From 3a4992633ee62d5edfbb484d9c6bcb3cf158489d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 31 Jul 2023 14:34:36 +0200 Subject: [PATCH] Migrate server to ESM Sorry for the very big commit that may lead to git log issues and merge conflicts, but it's a major step forward: * Server can be faster at startup because imports() are async and we can easily lazy import big modules * Angular doesn't seem to support ES import (with .js extension), so we had to correctly organize peertube into a monorepo: * Use yarn workspace feature * Use typescript reference projects for dependencies * Shared projects have been moved into "packages", each one is now a node module (with a dedicated package.json/tsconfig.json) * server/tools have been moved into apps/ and is now a dedicated app bundled and published on NPM so users don't have to build peertube cli tools manually * server/tests have been moved into packages/ so we don't compile them every time we want to run the server * Use isolatedModule option: * Had to move from const enum to const (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums) * Had to explictely specify "type" imports when used in decorators * Prefer tsx (that uses esbuild under the hood) instead of ts-node to load typescript files (tests with mocha or scripts): * To reduce test complexity as esbuild doesn't support decorator metadata, we only test server files that do not import server models * We still build tests files into js files for a faster CI * Remove unmaintained peertube CLI import script * Removed some barrels to speed up execution (less imports) --- .eslintrc.json | 23 +- .github/CONTRIBUTING.md | 31 +- .../action.yml | 10 +- .github/workflows/benchmark.yml | 2 +- .github/workflows/codeql/codeql-config.yml | 2 +- .github/workflows/stats.yml | 4 +- .gitignore | 16 +- .mocharc.cjs | 10 + apps/peertube-cli/.npmignore | 4 + apps/peertube-cli/README.md | 43 + apps/peertube-cli/package.json | 19 + apps/peertube-cli/scripts/build.js | 27 + apps/peertube-cli/scripts/watch.js | 7 + apps/peertube-cli/src/peertube-auth.ts | 171 ++ .../src/peertube-get-access-token.ts | 39 + apps/peertube-cli/src/peertube-plugins.ts | 167 ++ apps/peertube-cli/src/peertube-redundancy.ts | 186 ++ apps/peertube-cli/src/peertube-upload.ts | 167 ++ apps/peertube-cli/src/peertube.ts | 64 + apps/peertube-cli/src/shared/cli.ts | 195 ++ apps/peertube-cli/src/shared/index.ts | 1 + apps/peertube-cli/tsconfig.json | 15 + apps/peertube-cli/yarn.lock | 374 +++ {packages => apps}/peertube-runner/.gitignore | 0 apps/peertube-runner/.npmignore | 4 + apps/peertube-runner/README.md | 43 + apps/peertube-runner/package.json | 20 + apps/peertube-runner/scripts/build.js | 26 + apps/peertube-runner/src/peertube-runner.ts | 91 + apps/peertube-runner/src/register/index.ts | 1 + apps/peertube-runner/src/register/register.ts | 36 + apps/peertube-runner/src/server/index.ts | 1 + .../src/server/process/index.ts | 2 + .../src/server/process/process.ts | 34 + .../src/server/process/shared/common.ts | 106 + .../src/server/process/shared/index.ts | 3 + .../src/server/process/shared/process-live.ts | 338 +++ .../server/process/shared/process-studio.ts | 165 ++ .../src/server/process/shared/process-vod.ts | 201 ++ .../process/shared/transcoding-logger.ts | 10 + apps/peertube-runner/src/server/server.ts | 307 +++ .../src/server/shared/index.ts | 1 + .../src/server/shared/supported-job.ts | 43 + .../src/shared/config-manager.ts | 140 ++ apps/peertube-runner/src/shared/http.ts | 67 + apps/peertube-runner/src/shared/index.ts | 3 + apps/peertube-runner/src/shared/ipc/index.ts | 2 + .../src/shared/ipc/ipc-client.ts | 88 + .../src/shared/ipc/ipc-server.ts | 61 + .../src/shared/ipc/shared/index.ts | 2 + .../shared/ipc/shared/ipc-request.model.ts | 0 .../shared/ipc/shared/ipc-response.model.ts | 0 apps/peertube-runner/src/shared/logger.ts | 12 + apps/peertube-runner/tsconfig.json | 16 + {packages => apps}/peertube-runner/yarn.lock | 0 client/.eslintrc.json | 1 + client/e2e/wdio.main.conf.ts | 8 - client/package.json | 10 +- .../about-follows/about-follows.component.ts | 2 +- .../about-instance.component.ts | 2 +- .../about-instance/about-instance.resolver.ts | 2 +- .../contact-admin-modal.component.ts | 2 +- .../instance-statistics.component.ts | 2 +- .../account-video-channels.component.ts | 2 +- .../account-videos.component.ts | 2 +- .../src/app/+accounts/accounts.component.ts | 2 +- client/src/app/+admin/admin.component.ts | 2 +- client/src/app/+admin/config/config.routes.ts | 2 +- .../edit-basic-configuration.component.ts | 2 +- .../edit-custom-config.component.ts | 2 +- .../edit-live-configuration.component.ts | 2 +- .../edit-vod-transcoding.component.ts | 2 +- .../+admin/config/shared/config.service.ts | 2 +- .../followers-list.component.ts | 2 +- .../following-list.component.ts | 2 +- .../src/app/+admin/follows/follows.routes.ts | 2 +- .../video-redundancies-list.component.ts | 3 +- .../video-redundancy-information.component.ts | 2 +- .../+admin/moderation/moderation.routes.ts | 2 +- .../admin-registration.service.ts | 4 +- .../process-registration-modal.component.ts | 2 +- .../registration-list.component.ts | 2 +- .../video-block-list.component.ts | 6 +- .../comments/video-comment-list.component.ts | 2 +- .../overview/comments/video-comment.routes.ts | 2 +- .../users/user-edit/user-create.component.ts | 2 +- .../overview/users/user-edit/user-edit.ts | 5 +- .../user-edit/user-password.component.ts | 2 +- .../users/user-edit/user-update.component.ts | 2 +- .../users/user-list/user-list.component.ts | 4 +- .../app/+admin/overview/users/users.routes.ts | 2 +- .../overview/videos/video-admin.service.ts | 4 +- .../overview/videos/video-list.component.ts | 4 +- .../+admin/overview/videos/video.routes.ts | 2 +- .../plugin-list-installed.component.ts | 8 +- .../plugin-search/plugin-search.component.ts | 6 +- .../plugin-show-installed.component.ts | 2 +- .../src/app/+admin/plugins/plugins.routes.ts | 2 +- .../plugins/shared/plugin-api.service.ts | 19 +- .../plugins/shared/plugin-card.component.ts | 4 +- .../shared/plugin-navigation.component.ts | 4 +- .../shared/user-email-info.component.ts | 2 +- .../shared/user-real-quota-info.component.ts | 2 +- .../+admin/system/debug/debug.component.ts | 2 +- .../app/+admin/system/debug/debug.service.ts | 2 +- .../src/app/+admin/system/jobs/job.service.ts | 2 +- .../app/+admin/system/jobs/jobs.component.ts | 4 +- .../app/+admin/system/logs/log-row.model.ts | 2 +- .../app/+admin/system/logs/logs.component.ts | 2 +- .../app/+admin/system/logs/logs.service.ts | 2 +- .../runner-job-list.component.ts | 2 +- .../runner-list/runner-list.component.ts | 2 +- ...unner-registration-token-list.component.ts | 2 +- .../+admin/system/runners/runner.service.ts | 5 +- .../+admin/system/runners/runners.routes.ts | 2 +- client/src/app/+admin/system/system.routes.ts | 2 +- .../app/+error-page/error-page.component.ts | 4 +- client/src/app/+login/login.component.ts | 4 +- .../video-channel-create.component.ts | 2 +- .../video-channel-update.component.ts | 4 +- .../my-account-applications.component.ts | 3 +- .../my-account-change-email.component.ts | 2 +- .../my-account-change-password.component.ts | 2 +- .../my-account-email-preferences.component.ts | 2 +- ...ount-notification-preferences.component.ts | 6 +- .../my-account-settings.component.ts | 2 +- .../my-follows/my-followers.component.ts | 2 +- .../app/+my-library/my-library.component.ts | 2 +- .../my-accept-ownership.component.ts | 2 +- .../my-ownership/my-ownership.component.ts | 4 +- .../my-video-channel-syncs.component.ts | 5 +- .../video-channel-sync-edit.component.ts | 2 +- .../my-video-imports.component.ts | 4 +- .../my-video-playlist-create.component.ts | 3 +- .../my-video-playlist-edit.ts | 5 +- .../my-video-playlist-elements.component.ts | 2 +- .../my-video-playlist-update.component.ts | 2 +- .../my-video-playlists.component.ts | 2 +- .../my-videos/my-videos.component.ts | 2 +- .../app/+search/search-filters.component.ts | 2 +- client/src/app/+search/search.component.ts | 2 +- .../shared/abstract-lazy-load.resolver.ts | 2 +- .../+signup/+register/register.component.ts | 3 +- .../src/app/+signup/shared/signup.service.ts | 2 +- .../app/+stats/video/video-stats.component.ts | 6 +- .../app/+stats/video/video-stats.service.ts | 2 +- .../video-channel-videos.component.ts | 2 +- .../video-channels.component.ts | 2 +- .../edit/video-studio-edit.component.ts | 4 +- .../shared/video-studio.service.ts | 2 +- .../video-caption-add-modal.component.ts | 2 +- ...eo-caption-edit-modal-content.component.ts | 2 +- .../shared/video-edit.component.ts | 15 +- .../shared/video-upload.service.ts | 2 +- .../video-go-live.component.ts | 2 +- .../video-import-torrent.component.ts | 2 +- .../video-import-url.component.ts | 2 +- .../video-add-components/video-send.ts | 8 +- .../video-upload.component.ts | 2 +- .../+video-edit/video-add.component.ts | 2 +- .../+video-edit/video-update.component.ts | 5 +- .../+video-edit/video-update.resolver.ts | 4 +- .../action-buttons.component.ts | 2 +- .../action-buttons/video-rate.component.ts | 2 +- .../comment/video-comment-add.component.ts | 2 +- .../shared/comment/video-comment.component.ts | 2 +- .../comment/video-comments.component.ts | 2 +- .../information/privacy-concerns.component.ts | 2 +- .../information/video-alert.component.ts | 2 +- .../video-watch-playlist.component.ts | 2 +- .../recent-videos-recommendation.service.ts | 2 +- .../+video-watch/video-watch.component.ts | 13 +- .../video-list/overview/overview.service.ts | 5 +- .../overview/videos-overview.model.ts | 2 +- .../video-user-subscriptions.component.ts | 2 +- .../videos-list-common-page.component.ts | 2 +- client/src/app/app-routing.module.ts | 2 +- client/src/app/app.component.ts | 4 +- client/src/app/core/auth/auth-user.model.ts | 10 +- client/src/app/core/auth/auth.service.ts | 2 +- client/src/app/core/menu/menu.service.ts | 2 +- .../notification/peertube-socket.service.ts | 2 +- client/src/app/core/plugins/hooks.service.ts | 2 +- client/src/app/core/plugins/plugin.service.ts | 10 +- .../core/renderer/html-renderer.service.ts | 2 +- .../src/app/core/renderer/markdown.service.ts | 5 +- .../app/core/rest/rest-extractor.service.ts | 10 +- .../routing/homepage-redirect.component.ts | 2 +- client/src/app/core/routing/meta.service.ts | 2 +- .../scoped-tokens/scoped-tokens.service.ts | 2 +- client/src/app/core/server/server.service.ts | 22 +- client/src/app/core/theme/theme.service.ts | 2 +- .../core/users/user-local-storage.service.ts | 10 +- client/src/app/core/users/user.model.ts | 15 +- client/src/app/core/users/user.service.ts | 2 +- .../app/header/search-typeahead.component.ts | 2 +- client/src/app/helpers/utils/channel.ts | 2 +- client/src/app/helpers/utils/upload.ts | 2 +- .../app/menu/language-chooser.component.ts | 3 +- client/src/app/menu/menu.component.ts | 4 +- ...instance-config-warning-modal.component.ts | 2 +- .../video-playlist-validators.ts | 4 +- .../abuse-details.component.ts | 2 +- .../abuse-list-table.component.ts | 4 +- .../abuse-message-modal.component.ts | 2 +- .../moderation-comment-modal.component.ts | 2 +- .../processed-abuse.model.ts | 2 +- .../actor-avatar.component.ts | 2 +- .../custom-markup.service.ts | 2 +- .../dynamic-element.service.ts | 2 +- .../channel-miniature-markup.component.ts | 2 +- .../embed-markup.component.ts | 2 +- .../video-miniature-markup.component.ts | 2 +- .../videos-list-markup.component.ts | 4 +- .../dynamic-form-field.component.ts | 2 +- .../shared-forms/form-validator.service.ts | 2 +- .../markdown-textarea.component.ts | 2 +- .../shared-forms/preview-upload.component.ts | 2 +- .../shared-forms/timestamp-input.component.ts | 2 +- .../instance-about-accordion.component.ts | 3 +- .../instance-features-table.component.ts | 2 +- .../instance-follow.service.ts | 4 +- .../shared-instance/instance.service.ts | 5 +- .../shared-main/account/account.model.ts | 2 +- .../shared-main/account/account.service.ts | 2 +- .../shared/shared-main/account/actor.model.ts | 2 +- .../auth/auth-interceptor.service.ts | 3 +- .../custom-page/custom-page.service.ts | 2 +- .../shared-main/feeds/syndication.model.ts | 4 +- .../shared/shared-main/misc/help.component.ts | 2 +- .../plugins/plugin-placeholder.component.ts | 2 +- .../plugins/plugin-selector.directive.ts | 2 +- .../shared-main/users/user-history.service.ts | 2 +- .../users/user-notification.model.ts | 15 +- .../users/user-notification.service.ts | 2 +- .../users/user-notifications.component.ts | 2 +- .../video-caption/video-caption.service.ts | 4 +- .../video-channel-sync.service.ts | 3 +- .../video-channel/video-channel.model.ts | 2 +- .../video-channel/video-channel.service.ts | 2 +- .../shared-main/video/embed.component.ts | 4 +- .../shared-main/video/redundancy.service.ts | 2 +- .../shared-main/video/video-details.model.ts | 6 +- .../shared-main/video/video-edit.model.ts | 6 +- .../video/video-file-token.service.ts | 2 +- .../shared-main/video/video-import.service.ts | 4 +- .../video/video-ownership.service.ts | 2 +- .../video/video-password.service.ts | 2 +- .../shared/shared-main/video/video.model.ts | 11 +- .../shared/shared-main/video/video.service.ts | 17 +- .../shared/shared-moderation/abuse.service.ts | 2 +- .../shared-moderation/account-block.model.ts | 2 +- .../shared-moderation/blocklist.service.ts | 4 +- .../shared/shared-moderation/bulk.service.ts | 2 +- .../report-modals/account-report.component.ts | 4 +- .../report-modals/comment-report.component.ts | 4 +- .../report-modals/video-report.component.ts | 4 +- .../server-blocklist.component.ts | 2 +- .../user-ban-modal.component.ts | 2 +- .../user-moderation-dropdown.component.ts | 2 +- .../shared-moderation/video-block.service.ts | 6 +- .../shared-search/advanced-search.model.ts | 2 +- .../shared-search/find-in-bulk.service.ts | 4 +- .../shared/shared-search/search.service.ts | 2 +- .../video-share.component.ts | 4 +- .../support-modal.component.ts | 2 +- .../video-thumbnail.component.ts | 2 +- .../user-interface-settings.component.ts | 2 +- .../user-video-settings.component.ts | 3 +- .../subscribe-button.component.ts | 2 +- .../user-subscription.service.ts | 2 +- .../shared/shared-users/two-factor.service.ts | 2 +- .../shared/shared-users/user-admin.service.ts | 4 +- .../video-comment-thread-tree.model.ts | 2 +- .../video-comment.model.ts | 2 +- .../video-comment.service.ts | 2 +- .../live-stream-information.component.ts | 4 +- .../shared-video-live/live-video.service.ts | 2 +- .../video-actions-dropdown.component.ts | 2 +- .../video-download.component.ts | 4 +- .../video-filters-header.component.ts | 2 +- .../video-filters.model.ts | 18 +- .../video-miniature.component.ts | 2 +- .../videos-list.component.ts | 4 +- .../videos-selection.component.ts | 4 +- .../video-add-to-playlist.component.ts | 4 +- ...eo-playlist-element-miniature.component.ts | 4 +- .../video-playlist-element.model.ts | 4 +- .../video-playlist.model.ts | 14 +- .../video-playlist.service.ts | 2 +- client/src/assets/player/peertube-player.ts | 10 +- .../src/assets/player/shared/common/utils.ts | 2 +- .../control-bar/peertube-link-button.ts | 2 +- .../player/shared/metrics/metrics-plugin.ts | 4 +- .../p2p-media-loader-plugin.ts | 2 +- .../p2p-media-loader/segment-validator.ts | 2 +- .../player/shared/peertube/peertube-plugin.ts | 4 +- .../hls-options-builder.ts | 2 +- .../shared/playlist/playlist-menu-item.ts | 4 +- .../player/shared/playlist/playlist-menu.ts | 2 +- .../assets/player/shared/stats/stats-card.ts | 2 +- .../shared/web-video/web-video-plugin.ts | 4 +- .../src/assets/player/translations-manager.ts | 2 +- .../player/types/peertube-player-options.ts | 4 +- .../player/types/peertube-videojs-typings.ts | 2 +- client/src/assets/player/utils.ts | 2 +- client/src/root-helpers/logger.ts | 2 +- client/src/root-helpers/plugins-manager.ts | 7 +- client/src/root-helpers/video.ts | 8 +- client/src/standalone/videos/embed.ts | 2 +- .../src/standalone/videos/shared/auth-http.ts | 4 +- .../standalone/videos/shared/live-manager.ts | 4 +- .../videos/shared/peertube-plugin.ts | 4 +- .../standalone/videos/shared/player-html.ts | 2 +- .../videos/shared/player-options-builder.ts | 4 +- .../videos/shared/playlist-fetcher.ts | 2 +- .../videos/shared/playlist-tracker.ts | 2 +- .../standalone/videos/shared/video-fetcher.ts | 2 +- client/src/types/job-state-client.type.ts | 2 +- client/src/types/job-type-client.type.ts | 2 +- .../src/types/register-client-option.model.ts | 2 +- client/src/types/server-error.model.ts | 6 +- client/tsconfig.eslint.json | 5 + client/tsconfig.json | 28 +- client/tsconfig.types.json | 7 +- client/webpack/webpack.video-embed.js | 4 +- client/yarn.lock | 118 +- config/default.yaml | 1 + config/production.yaml.example | 1 + package.json | 89 +- packages/core-utils/package.json | 19 + .../src/abuse/abuse-predefined-reasons.ts | 14 + packages/core-utils/src/abuse/index.ts | 1 + .../core-utils/src}/common/array.ts | 0 .../core-utils/src}/common/date.ts | 0 packages/core-utils/src/common/index.ts | 10 + .../core-utils/src}/common/number.ts | 0 .../core-utils/src}/common/object.ts | 0 .../core-utils/src}/common/promises.ts | 0 .../core-utils/src}/common/random.ts | 0 .../core-utils/src}/common/regexp.ts | 0 .../core-utils/src}/common/time.ts | 0 packages/core-utils/src/common/url.ts | 150 ++ .../core-utils/src}/common/version.ts | 0 .../core-utils/src}/i18n/i18n.ts | 0 packages/core-utils/src/i18n/index.ts | 1 + packages/core-utils/src/index.ts | 7 + packages/core-utils/src/plugins/hooks.ts | 60 + packages/core-utils/src/plugins/index.ts | 1 + .../core-utils/src}/renderer/html.ts | 0 packages/core-utils/src/renderer/index.ts | 2 + .../core-utils/src}/renderer/markdown.ts | 0 packages/core-utils/src/users/index.ts | 1 + packages/core-utils/src/users/user-role.ts | 37 + packages/core-utils/src/videos/bitrate.ts | 113 + packages/core-utils/src/videos/common.ts | 24 + packages/core-utils/src/videos/index.ts | 2 + packages/core-utils/tsconfig.json | 11 + packages/ffmpeg/package.json | 19 + packages/ffmpeg/src/ffmpeg-command-wrapper.ts | 246 ++ .../src/ffmpeg-default-transcoding-profile.ts | 187 ++ packages/ffmpeg/src/ffmpeg-edition.ts | 239 ++ packages/ffmpeg/src/ffmpeg-images.ts | 92 + packages/ffmpeg/src/ffmpeg-live.ts | 184 ++ packages/ffmpeg/src/ffmpeg-utils.ts | 17 + .../ffmpeg/src}/ffmpeg-version.ts | 0 packages/ffmpeg/src/ffmpeg-vod.ts | 256 ++ packages/ffmpeg/src/ffprobe.ts | 184 ++ packages/ffmpeg/src/index.ts | 9 + packages/ffmpeg/src/shared/encoder-options.ts | 39 + packages/ffmpeg/src/shared/index.ts | 2 + packages/ffmpeg/src/shared/presets.ts | 93 + packages/ffmpeg/tsconfig.json | 12 + packages/models/package.json | 19 + packages/models/src/activitypub/activity.ts | 135 ++ .../src/activitypub/activitypub-actor.ts | 34 + .../src/activitypub/activitypub-collection.ts | 9 + .../activitypub-ordered-collection.ts | 0 .../src/activitypub/activitypub-root.ts | 5 + .../src}/activitypub/activitypub-signature.ts | 0 .../models/src}/activitypub/context.ts | 0 packages/models/src/activitypub/index.ts | 9 + .../src/activitypub/objects/abuse-object.ts | 15 + .../activitypub/objects/activitypub-object.ts | 17 + .../activitypub/objects/cache-file-object.ts | 9 + .../src/activitypub/objects/common-objects.ts | 130 ++ .../models/src/activitypub/objects/index.ts | 9 + .../objects/playlist-element-object.ts | 0 .../activitypub/objects/playlist-object.ts | 29 + .../objects/video-comment-object.ts | 16 + .../src/activitypub/objects/video-object.ts | 74 + .../objects/watch-action-object.ts | 0 .../models/src}/activitypub/webfinger.ts | 0 packages/models/src/actors/account.model.ts | 22 + .../models/src}/actors/actor-image.model.ts | 0 .../models/src/actors/actor-image.type.ts | 6 + packages/models/src/actors/actor.model.ts | 13 + .../models/src}/actors/custom-page.model.ts | 0 packages/models/src/actors/follow.model.ts | 13 + packages/models/src/actors/index.ts | 6 + .../bulk-remove-comments-of-body.model.ts | 0 packages/models/src/bulk/index.ts | 1 + packages/models/src/common/index.ts | 1 + .../models/src}/common/result-list.model.ts | 0 .../custom-markup/custom-markup-data.model.ts | 0 packages/models/src/custom-markup/index.ts | 1 + packages/models/src/feeds/feed-format.enum.ts | 7 + packages/models/src/feeds/index.ts | 1 + packages/models/src/http/http-methods.ts | 23 + packages/models/src/http/http-status-codes.ts | 366 +++ packages/models/src/http/index.ts | 2 + packages/models/src/index.ts | 20 + packages/models/src/joinpeertube/index.ts | 1 + .../src}/joinpeertube/versions.model.ts | 0 packages/models/src/metrics/index.ts | 1 + .../metrics/playback-metric-create.model.ts | 22 + .../moderation/abuse/abuse-create.model.ts | 21 + .../moderation/abuse/abuse-filter.type.ts | 0 .../moderation/abuse/abuse-message.model.ts | 10 + .../moderation/abuse/abuse-reason.model.ts | 22 + .../src/moderation/abuse/abuse-state.model.ts | 7 + .../moderation/abuse/abuse-update.model.ts | 7 + .../moderation/abuse/abuse-video-is.type.ts | 0 .../src/moderation/abuse/abuse.model.ts | 70 + packages/models/src/moderation/abuse/index.ts | 8 + .../src/moderation/account-block.model.ts | 7 + .../src}/moderation/block-status.model.ts | 0 packages/models/src/moderation/index.ts | 4 + .../src/moderation/server-block.model.ts | 9 + packages/models/src/nodeinfo/index.ts | 1 + .../models/src}/nodeinfo/nodeinfo.model.ts | 0 packages/models/src/overviews/index.ts | 1 + .../src/overviews/videos-overview.model.ts | 24 + .../src}/plugins/client/client-hook.model.ts | 0 packages/models/src/plugins/client/index.ts | 8 + .../client/plugin-client-scope.type.ts | 0 .../client/plugin-element-placeholder.type.ts | 0 .../plugins/client/plugin-selector-id.type.ts | 0 .../register-client-form-field.model.ts | 0 .../client/register-client-hook.model.ts | 7 + .../client/register-client-route.model.ts | 0 .../register-client-settings-script.model.ts | 8 + packages/models/src/plugins/hook-type.enum.ts | 7 + packages/models/src/plugins/index.ts | 6 + .../models/src/plugins/plugin-index/index.ts | 3 + .../peertube-plugin-index-list.model.ts | 10 + .../peertube-plugin-index.model.ts | 0 .../peertube-plugin-latest-version.model.ts | 0 .../src/plugins/plugin-package-json.model.ts | 29 + packages/models/src/plugins/plugin.type.ts | 6 + .../models/src/plugins/server/api/index.ts | 3 + .../server/api/install-plugin.model.ts | 0 .../plugins/server/api/manage-plugin.model.ts | 0 .../server/api/peertube-plugin.model.ts | 16 + packages/models/src/plugins/server/index.ts | 7 + .../src/plugins/server/managers/index.ts | 9 + .../plugin-playlist-privacy-manager.model.ts | 12 + .../managers/plugin-settings-manager.model.ts | 0 .../managers/plugin-storage-manager.model.ts | 0 .../plugin-transcoding-manager.model.ts | 13 + .../plugin-video-category-manager.model.ts | 13 + .../plugin-video-language-manager.model.ts | 13 + .../plugin-video-licence-manager.model.ts | 13 + .../plugin-video-privacy-manager.model.ts | 13 + .../server/plugin-constant-manager.model.ts | 0 .../server/plugin-translation.model.ts | 0 .../server/register-server-hook.model.ts | 7 + .../src}/plugins/server/server-hook.model.ts | 0 .../src/plugins/server/settings/index.ts | 2 + .../server/settings/public-server.setting.ts | 5 + .../settings/register-server-setting.model.ts | 12 + packages/models/src/redundancy/index.ts | 4 + .../video-redundancies-filters.model.ts | 0 .../video-redundancy-config-filter.type.ts | 0 .../src}/redundancy/video-redundancy.model.ts | 0 .../videos-redundancy-strategy.model.ts | 0 .../runners/abort-runner-job-body.model.ts | 0 .../runners/accept-runner-job-body.model.ts | 0 .../runners/accept-runner-job-result.model.ts | 6 + .../runners/error-runner-job-body.model.ts | 0 packages/models/src/runners/index.ts | 21 + .../runners/list-runner-jobs-query.model.ts | 9 + .../list-runner-registration-tokens.model.ts | 0 .../src}/runners/list-runners-query.model.ts | 0 .../runners/register-runner-body.model.ts | 0 .../runners/register-runner-result.model.ts | 0 .../runners/request-runner-job-body.model.ts | 0 .../request-runner-job-result.model.ts | 10 + .../src/runners/runner-job-payload.model.ts | 79 + .../runner-job-private-payload.model.ts | 45 + .../src/runners/runner-job-state.model.ts | 13 + .../runners/runner-job-success-body.model.ts | 0 .../src}/runners/runner-job-type.type.ts | 0 .../runners/runner-job-update-body.model.ts | 0 .../models/src/runners/runner-job.model.ts | 45 + .../src}/runners/runner-registration-token.ts | 0 .../models/src}/runners/runner.model.ts | 0 .../runners/unregister-runner-body.model.ts | 0 .../src}/search/boolean-both-query.model.ts | 0 packages/models/src/search/index.ts | 6 + .../src}/search/search-target-query.model.ts | 0 .../video-channels-search-query.model.ts | 18 + .../video-playlists-search-query.model.ts | 20 + .../src/search/videos-common-query.model.ts | 45 + .../src/search/videos-search-query.model.ts | 26 + .../models/src}/server/about.model.ts | 0 .../server/broadcast-message-level.type.ts | 0 .../src/server/client-log-create.model.ts | 11 + .../src}/server/client-log-level.type.ts | 0 .../models/src}/server/contact-form.model.ts | 0 .../models/src/server/custom-config.model.ts | 259 +++ .../models/src}/server/debug.model.ts | 0 .../models/src}/server/emailer.model.ts | 0 packages/models/src/server/index.ts | 16 + packages/models/src/server/job.model.ts | 303 +++ .../server/peertube-problem-document.model.ts | 32 + .../models/src/server/server-config.model.ts | 305 +++ .../models/src}/server/server-debug.model.ts | 0 .../src/server/server-error-code.enum.ts | 92 + .../src}/server/server-follow-create.model.ts | 0 .../src}/server/server-log-level.type.ts | 0 .../models/src/server/server-stats.model.ts | 47 + packages/models/src/tokens/index.ts | 1 + .../src}/tokens/oauth-client-local.model.ts | 0 packages/models/src/users/index.ts | 16 + .../models/src/users/registration/index.ts | 5 + .../users/registration/user-register.model.ts | 0 .../user-registration-request.model.ts | 5 + .../user-registration-state.model.ts | 7 + .../user-registration-update-state.model.ts | 0 .../registration/user-registration.model.ts | 29 + .../users/two-factor-enable-result.model.ts | 0 .../src}/users/user-create-result.model.ts | 0 .../models/src/users/user-create.model.ts | 13 + packages/models/src/users/user-flag.model.ts | 6 + .../models/src}/users/user-login.model.ts | 0 .../users/user-notification-setting.model.ts | 34 + .../src/users/user-notification.model.ts | 140 ++ .../src}/users/user-refresh-token.model.ts | 0 packages/models/src/users/user-right.enum.ts | 53 + packages/models/src/users/user-role.ts | 8 + .../models/src}/users/user-scoped-token.ts | 0 .../models/src/users/user-update-me.model.ts | 26 + .../models/src/users/user-update.model.ts | 13 + .../src}/users/user-video-quota.model.ts | 0 packages/models/src/users/user.model.ts | 78 + packages/models/src/videos/blacklist/index.ts | 3 + .../blacklist/video-blacklist-create.model.ts | 0 .../blacklist/video-blacklist-update.model.ts | 0 .../videos/blacklist/video-blacklist.model.ts | 20 + packages/models/src/videos/caption/index.ts | 2 + .../caption/video-caption-update.model.ts | 0 .../src/videos/caption/video-caption.model.ts | 7 + .../src/videos/change-ownership/index.ts | 3 + .../video-change-ownership-accept.model.ts | 0 .../video-change-ownership-create.model.ts | 0 .../video-change-ownership.model.ts | 19 + .../models/src/videos/channel-sync/index.ts | 3 + .../video-channel-sync-create.model.ts | 0 .../video-channel-sync-state.enum.ts | 8 + .../channel-sync/video-channel-sync.model.ts | 14 + packages/models/src/videos/channel/index.ts | 4 + .../video-channel-create-result.model.ts | 0 .../channel/video-channel-create.model.ts | 0 .../channel/video-channel-update.model.ts | 0 .../src/videos/channel/video-channel.model.ts | 34 + packages/models/src/videos/comment/index.ts | 2 + .../comment/video-comment-create.model.ts | 0 .../src/videos/comment/video-comment.model.ts | 45 + packages/models/src/videos/file/index.ts | 3 + .../videos/file/video-file-metadata.model.ts | 0 .../src/videos/file/video-file.model.ts | 22 + .../src/videos/file/video-resolution.enum.ts | 13 + packages/models/src/videos/import/index.ts | 4 + .../import/video-import-create.model.ts | 9 + .../videos/import/video-import-state.enum.ts | 10 + .../src/videos/import/video-import.model.ts | 24 + .../videos-import-in-channel-create.model.ts | 0 packages/models/src/videos/index.ts | 43 + packages/models/src/videos/live/index.ts | 8 + .../videos/live/live-video-create.model.ts | 11 + .../src/videos/live/live-video-error.enum.ts | 11 + .../live/live-video-event-payload.model.ts | 7 + .../src}/videos/live/live-video-event.type.ts | 0 .../live/live-video-latency-mode.enum.ts | 7 + .../videos/live/live-video-session.model.ts | 22 + .../videos/live/live-video-update.model.ts | 9 + .../src/videos/live/live-video.model.ts | 14 + .../models/src}/videos/nsfw-policy.type.ts | 0 packages/models/src/videos/playlist/index.ts | 12 + .../playlist/video-exist-in-playlist.model.ts | 0 .../video-playlist-create-result.model.ts | 0 .../playlist/video-playlist-create.model.ts | 11 + ...eo-playlist-element-create-result.model.ts | 0 .../video-playlist-element-create.model.ts | 0 .../video-playlist-element-update.model.ts | 0 .../playlist/video-playlist-element.model.ts | 21 + .../playlist/video-playlist-privacy.model.ts | 7 + .../playlist/video-playlist-reorder.model.ts | 0 .../playlist/video-playlist-type.model.ts | 6 + .../playlist/video-playlist-update.model.ts | 10 + .../videos/playlist/video-playlist.model.ts | 35 + .../videos/rate/account-video-rate.model.ts | 7 + packages/models/src/videos/rate/index.ts | 5 + .../rate/user-video-rate-update.model.ts | 5 + .../src/videos/rate/user-video-rate.model.ts | 6 + .../src}/videos/rate/user-video-rate.type.ts | 0 packages/models/src/videos/stats/index.ts | 6 + .../stats/video-stats-overall-query.model.ts | 0 .../videos/stats/video-stats-overall.model.ts | 0 .../stats/video-stats-retention.model.ts | 0 .../video-stats-timeserie-metric.type.ts | 0 .../video-stats-timeserie-query.model.ts | 0 .../stats/video-stats-timeserie.model.ts | 0 .../models/src}/videos/storyboard.model.ts | 0 packages/models/src/videos/studio/index.ts | 1 + .../studio/video-studio-create-edit.model.ts | 0 packages/models/src/videos/thumbnail.type.ts | 6 + .../models/src/videos/transcoding/index.ts | 3 + .../video-transcoding-create.model.ts | 0 .../video-transcoding-fps.model.ts | 0 .../transcoding/video-transcoding.model.ts | 65 + .../src}/videos/video-constant.model.ts | 0 .../src}/videos/video-create-result.model.ts | 0 .../models/src/videos/video-create.model.ts | 25 + .../models/src/videos/video-include.enum.ts | 10 + .../src}/videos/video-password.model.ts | 0 .../models/src/videos/video-privacy.enum.ts | 9 + .../models/src}/videos/video-rate.type.ts | 0 .../src/videos/video-schedule-update.model.ts | 7 + .../src}/videos/video-sort-field.type.ts | 0 .../models/src/videos/video-source.model.ts | 0 .../models/src/videos/video-state.enum.ts | 13 + .../models/src/videos/video-storage.enum.ts | 6 + .../videos/video-streaming-playlist.model.ts | 15 + .../videos/video-streaming-playlist.type.ts | 5 + .../models/src}/videos/video-token.model.ts | 0 .../models/src/videos/video-update.model.ts | 25 + .../models/src}/videos/video-view.model.ts | 0 packages/models/src/videos/video.model.ts | 99 + packages/models/tsconfig.json | 8 + packages/models/tsconfig.types.json | 10 + packages/node-utils/package.json | 19 + .../node-utils/src}/crypto.ts | 0 packages/node-utils/src/env.ts | 58 + packages/node-utils/src/file.ts | 11 + packages/node-utils/src/index.ts | 5 + packages/node-utils/src/path.ts | 50 + packages/node-utils/src/uuid.ts | 32 + packages/node-utils/tsconfig.json | 8 + packages/peertube-runner/.npmignore | 6 - packages/peertube-runner/README.md | 29 - packages/peertube-runner/package.json | 17 - packages/peertube-runner/peertube-runner.ts | 93 - packages/peertube-runner/register/index.ts | 1 - packages/peertube-runner/register/register.ts | 36 - packages/peertube-runner/server/index.ts | 1 - .../peertube-runner/server/process/index.ts | 2 - .../peertube-runner/server/process/process.ts | 34 - .../server/process/shared/common.ts | 106 - .../server/process/shared/index.ts | 3 - .../server/process/shared/process-live.ts | 338 --- .../server/process/shared/process-studio.ts | 165 -- .../server/process/shared/process-vod.ts | 201 -- .../process/shared/transcoding-logger.ts | 10 - packages/peertube-runner/server/server.ts | 306 --- .../peertube-runner/server/shared/index.ts | 1 - .../server/shared/supported-job.ts | 43 - .../peertube-runner/shared/config-manager.ts | 139 -- packages/peertube-runner/shared/http.ts | 66 - packages/peertube-runner/shared/index.ts | 3 - packages/peertube-runner/shared/ipc/index.ts | 2 - .../peertube-runner/shared/ipc/ipc-client.ts | 88 - .../peertube-runner/shared/ipc/ipc-server.ts | 61 - .../shared/ipc/shared/index.ts | 2 - packages/peertube-runner/shared/logger.ts | 12 - packages/peertube-runner/tsconfig.json | 9 - packages/server-commands/package.json | 19 + .../server-commands/src/bulk/bulk-command.ts | 20 + packages/server-commands/src/bulk/index.ts | 1 + .../server-commands/src/cli/cli-command.ts | 27 + packages/server-commands/src/cli/index.ts | 1 + .../src/custom-pages/custom-pages-command.ts | 33 + .../server-commands/src/custom-pages/index.ts | 1 + .../src/feeds/feeds-command.ts | 78 + packages/server-commands/src/feeds/index.ts | 1 + packages/server-commands/src/index.ts | 14 + packages/server-commands/src/logs/index.ts | 1 + .../server-commands/src/logs/logs-command.ts | 56 + .../src/moderation/abuses-command.ts | 228 ++ .../server-commands/src/moderation/index.ts | 1 + .../server-commands/src/overviews/index.ts | 1 + .../src/overviews/overviews-command.ts | 23 + .../server-commands/src/requests/index.ts | 1 + .../server-commands/src/requests/requests.ts | 260 +++ packages/server-commands/src/runners/index.ts | 3 + .../src/runners/runner-jobs-command.ts | 297 +++ .../runner-registration-tokens-command.ts | 55 + .../src/runners/runners-command.ts | 85 + packages/server-commands/src/search/index.ts | 1 + .../src/search/search-command.ts | 98 + .../src/server/config-command.ts | 576 +++++ .../src/server/contact-form-command.ts | 30 + .../src/server/debug-command.ts | 33 + .../src/server/follows-command.ts | 139 ++ .../server-commands/src/server/follows.ts | 20 + packages/server-commands/src/server/index.ts | 15 + .../src/server/jobs-command.ts | 84 + packages/server-commands/src/server/jobs.ts | 117 + .../src/server/metrics-command.ts | 18 + .../src/server/object-storage-command.ts | 165 ++ .../src/server/plugins-command.ts | 258 +++ .../src/server/redundancy-command.ts | 80 + packages/server-commands/src/server/server.ts | 451 ++++ .../src/server/servers-command.ts | 104 + .../server-commands/src/server/servers.ts | 68 + .../src/server/stats-command.ts | 25 + .../src/shared/abstract-command.ts | 225 ++ packages/server-commands/src/shared/index.ts | 1 + packages/server-commands/src/socket/index.ts | 1 + .../src/socket/socket-io-command.ts | 24 + .../src/users/accounts-command.ts | 76 + .../server-commands/src/users/accounts.ts | 15 + .../src/users/blocklist-command.ts | 165 ++ packages/server-commands/src/users/index.ts | 10 + .../src/users/login-command.ts | 159 ++ packages/server-commands/src/users/login.ts | 19 + .../src/users/notifications-command.ts | 85 + .../src/users/registrations-command.ts | 157 ++ .../src/users/subscriptions-command.ts | 83 + .../src/users/two-factor-command.ts | 92 + .../src/users/users-command.ts | 389 ++++ .../src/videos/blacklist-command.ts | 74 + .../src/videos/captions-command.ts | 67 + .../src/videos/change-ownership-command.ts | 67 + .../src/videos/channel-syncs-command.ts | 55 + .../src/videos/channels-command.ts | 202 ++ .../server-commands/src/videos/channels.ts | 29 + .../src/videos/comments-command.ts | 159 ++ .../src/videos/history-command.ts | 54 + .../src/videos/imports-command.ts | 76 + packages/server-commands/src/videos/index.ts | 22 + .../src/videos/live-command.ts | 339 +++ packages/server-commands/src/videos/live.ts | 129 ++ .../src/videos/playlists-command.ts | 281 +++ .../src/videos/services-command.ts | 29 + .../src/videos/storyboard-command.ts | 19 + .../src/videos/streaming-playlists-command.ts | 119 + .../src/videos/video-passwords-command.ts | 56 + .../src/videos/video-stats-command.ts | 62 + .../src/videos/video-studio-command.ts | 67 + .../src/videos/video-token-command.ts | 34 + .../src/videos/videos-command.ts | 831 +++++++ .../src/videos/views-command.ts | 51 + packages/server-commands/tsconfig.json | 14 + .../tests/fixtures/60fps_720p_small.mp4 | Bin .../mastodon/bad-body-http-signature.json | 0 .../ap-json/mastodon/bad-http-signature.json | 0 .../ap-json/mastodon/bad-public-key.json | 0 .../mastodon/create-bad-signature.json | 0 .../fixtures/ap-json/mastodon/create.json | 0 .../ap-json/mastodon/http-signature.json | 0 .../fixtures/ap-json/mastodon/public-key.json | 0 .../peertube/announce-without-context.json | 0 .../ap-json/peertube/invalid-keys.json | 0 .../tests/fixtures/ap-json/peertube/keys.json | 0 .../tests/fixtures/avatar-big.png | Bin .../tests/fixtures/avatar-resized-120x120.gif | Bin .../tests/fixtures/avatar-resized-120x120.png | Bin .../tests/fixtures/avatar-resized-48x48.gif | Bin .../tests/fixtures/avatar-resized-48x48.png | Bin .../tests/fixtures/avatar.gif | Bin .../tests/fixtures/avatar.png | Bin .../fixtures/avatar2-resized-120x120.png | Bin .../tests/fixtures/avatar2-resized-48x48.png | Bin .../tests/fixtures/avatar2.png | Bin .../tests/fixtures/banner-resized.jpg | Bin .../tests/fixtures/banner.jpg | Bin .../tests/fixtures/custom-preview-big.png | Bin .../tests/fixtures/custom-preview.jpg | Bin .../tests/fixtures/custom-thumbnail-big.jpg | Bin .../tests/fixtures/custom-thumbnail.jpg | Bin .../tests/fixtures/custom-thumbnail.png | Bin {server => packages}/tests/fixtures/exif.jpg | Bin {server => packages}/tests/fixtures/exif.png | Bin .../tests/fixtures/live/0-000067.ts | Bin .../tests/fixtures/live/0-000068.ts | Bin .../tests/fixtures/live/0-000069.ts | Bin .../tests/fixtures/live/0-000070.ts | Bin .../tests/fixtures/live/0.m3u8 | 0 .../tests/fixtures/live/1-000067.ts | Bin .../tests/fixtures/live/1-000068.ts | Bin .../tests/fixtures/live/1-000069.ts | Bin .../tests/fixtures/live/1-000070.ts | Bin .../tests/fixtures/live/1.m3u8 | 0 .../tests/fixtures/live/master.m3u8 | 0 .../tests/fixtures/low-bitrate.mp4 | Bin .../peertube-plugin-test-broken/main.js | 0 .../peertube-plugin-test-broken/package.json | 0 .../main.js | 0 .../package.json | 0 .../main.js | 0 .../package.json | 0 .../main.js | 0 .../package.json | 0 .../languages/fr.json | 0 .../languages/it.json | 0 .../main.js | 0 .../package.json | 0 .../peertube-plugin-test-five/main.js | 0 .../peertube-plugin-test-five/package.json | 0 .../peertube-plugin-test-four/main.js | 0 .../peertube-plugin-test-four/package.json | 0 .../main.js | 0 .../package.json | 0 .../main.js | 0 .../package.json | 0 .../main.js | 0 .../package.json | 0 .../peertube-plugin-test-native/main.js | 0 .../peertube-plugin-test-native/package.json | 0 .../main.js | 0 .../package.json | 0 .../fixtures/peertube-plugin-test-six/main.js | 0 .../peertube-plugin-test-six/package.json | 0 .../main.js | 0 .../package.json | 0 .../main.js | 0 .../package.json | 0 .../peertube-plugin-test-unloading/lib.js | 0 .../peertube-plugin-test-unloading/main.js | 0 .../package.json | 0 .../main.js | 0 .../package.json | 0 .../peertube-plugin-test-websocket/main.js | 0 .../package.json | 0 .../peertube-plugin-test/languages/fr.json | 0 .../fixtures/peertube-plugin-test/main.js | 0 .../peertube-plugin-test/package.json | 0 .../tests/fixtures/rtmps.cert | 0 {server => packages}/tests/fixtures/rtmps.key | 0 .../tests/fixtures/sample.ogg | Bin .../tests/fixtures/subtitle-bad.txt | 0 .../tests/fixtures/subtitle-good.srt | 0 .../tests/fixtures/subtitle-good1.vtt | 0 .../tests/fixtures/subtitle-good2.vtt | 0 .../tests/fixtures/thumbnail-playlist.jpg | Bin .../tests/fixtures/video-720p.torrent | Bin .../tests/fixtures/video_import_preview.jpg | Bin .../fixtures/video_import_preview_yt_dlp.jpg | Bin .../tests/fixtures/video_import_thumbnail.jpg | Bin .../video_import_thumbnail_yt_dlp.jpg | Bin .../tests/fixtures/video_short.avi | Bin .../tests/fixtures/video_short.mkv | Bin .../tests/fixtures/video_short.mp4 | Bin .../tests/fixtures/video_short.mp4.jpg | Bin .../tests/fixtures/video_short.ogv | Bin .../tests/fixtures/video_short.ogv.jpg | Bin .../tests/fixtures/video_short.webm | Bin .../tests/fixtures/video_short.webm.jpg | Bin .../fixtures/video_short1-preview.webm.jpg | Bin .../tests/fixtures/video_short1.webm | Bin .../tests/fixtures/video_short1.webm.jpg | Bin .../tests/fixtures/video_short2.webm | Bin .../tests/fixtures/video_short2.webm.jpg | Bin .../tests/fixtures/video_short3.webm | Bin .../tests/fixtures/video_short3.webm.jpg | Bin .../tests/fixtures/video_short_0p.mp4 | Bin .../tests/fixtures/video_short_144p.m3u8 | 0 .../tests/fixtures/video_short_144p.mp4 | Bin .../tests/fixtures/video_short_240p.m3u8 | 0 .../tests/fixtures/video_short_240p.mp4 | Bin .../tests/fixtures/video_short_360p.m3u8 | 0 .../tests/fixtures/video_short_360p.mp4 | Bin .../tests/fixtures/video_short_480.webm | Bin .../tests/fixtures/video_short_480p.m3u8 | 0 .../tests/fixtures/video_short_480p.mp4 | Bin .../tests/fixtures/video_short_4k.mp4 | Bin .../tests/fixtures/video_short_720p.m3u8 | 0 .../tests/fixtures/video_short_720p.mp4 | Bin .../tests/fixtures/video_short_fake.webm | 0 .../tests/fixtures/video_short_mp3_256k.mp4 | Bin .../tests/fixtures/video_short_no_audio.mp4 | Bin .../tests/fixtures/video_very_long_10p.mp4 | Bin .../tests/fixtures/video_very_short_240p.mp4 | Bin packages/tests/package.json | 12 + packages/tests/src/api/activitypub/cleaner.ts | 342 +++ packages/tests/src/api/activitypub/client.ts | 136 ++ packages/tests/src/api/activitypub/fetch.ts | 82 + packages/tests/src/api/activitypub/index.ts | 5 + .../tests/src/api/activitypub/refresher.ts | 157 ++ .../tests/src/api/activitypub/security.ts | 331 +++ packages/tests/src/api/check-params/abuses.ts | 438 ++++ .../tests/src/api/check-params/accounts.ts | 43 + .../tests/src/api/check-params/blocklist.ts | 556 +++++ packages/tests/src/api/check-params/bulk.ts | 86 + .../api/check-params/channel-import-videos.ts | 209 ++ packages/tests/src/api/check-params/config.ts | 428 ++++ .../src/api/check-params/contact-form.ts | 86 + .../src/api/check-params/custom-pages.ts | 79 + packages/tests/src/api/check-params/debug.ts | 67 + .../tests/src/api/check-params/follows.ts | 369 +++ packages/tests/src/api/check-params/index.ts | 45 + packages/tests/src/api/check-params/jobs.ts | 125 + packages/tests/src/api/check-params/live.ts | 590 +++++ packages/tests/src/api/check-params/logs.ts | 163 ++ .../tests/src/api/check-params/metrics.ts | 214 ++ .../tests/src/api/check-params/my-user.ts | 492 ++++ .../tests/src/api/check-params/plugins.ts | 490 ++++ .../tests/src/api/check-params/redundancy.ts | 240 ++ .../src/api/check-params/registrations.ts | 446 ++++ .../tests/src/api/check-params/runners.ts | 911 ++++++++ packages/tests/src/api/check-params/search.ts | 278 +++ .../tests/src/api/check-params/services.ts | 207 ++ .../tests/src/api/check-params/transcoding.ts | 112 + .../tests/src/api/check-params/two-factor.ts | 294 +++ .../src/api/check-params/upload-quota.ts | 134 ++ .../api/check-params/user-notifications.ts | 290 +++ .../api/check-params/user-subscriptions.ts | 298 +++ .../tests/src/api/check-params/users-admin.ts | 457 ++++ .../src/api/check-params/users-emails.ts | 122 + .../src/api/check-params/video-blacklist.ts | 292 +++ .../src/api/check-params/video-captions.ts | 307 +++ .../api/check-params/video-channel-syncs.ts | 319 +++ .../src/api/check-params/video-channels.ts | 379 +++ .../src/api/check-params/video-comments.ts | 484 ++++ .../tests/src/api/check-params/video-files.ts | 195 ++ .../src/api/check-params/video-imports.ts | 433 ++++ .../src/api/check-params/video-passwords.ts | 604 +++++ .../src/api/check-params/video-playlists.ts | 695 ++++++ .../src/api/check-params/video-source.ts | 154 ++ .../src/api/check-params/video-storyboards.ts | 45 + .../src/api/check-params/video-studio.ts | 392 ++++ .../tests/src/api/check-params/video-token.ts | 70 + .../api/check-params/videos-common-filters.ts | 171 ++ .../src/api/check-params/videos-history.ts | 145 ++ .../src/api/check-params/videos-overviews.ts | 31 + packages/tests/src/api/check-params/videos.ts | 883 +++++++ packages/tests/src/api/check-params/views.ts | 227 ++ packages/tests/src/api/live/index.ts | 7 + .../tests/src/api/live/live-constraints.ts | 237 ++ .../tests/src/api/live/live-fast-restream.ts | 153 ++ packages/tests/src/api/live/live-permanent.ts | 204 ++ packages/tests/src/api/live/live-rtmps.ts | 143 ++ .../tests/src/api/live/live-save-replay.ts | 583 +++++ .../src/api/live/live-socket-messages.ts | 186 ++ packages/tests/src/api/live/live.ts | 766 ++++++ packages/tests/src/api/moderation/abuses.ts | 887 +++++++ .../api/moderation/blocklist-notification.ts | 231 ++ .../tests/src/api/moderation/blocklist.ts | 902 ++++++++ packages/tests/src/api/moderation/index.ts | 4 + .../src/api/moderation/video-blacklist.ts | 414 ++++ .../api/notifications/admin-notifications.ts | 154 ++ .../notifications/comments-notifications.ts | 300 +++ packages/tests/src/api/notifications/index.ts | 6 + .../notifications/moderation-notifications.ts | 609 +++++ .../api/notifications/notifications-api.ts | 206 ++ .../registrations-notifications.ts | 83 + .../api/notifications/user-notifications.ts | 574 +++++ .../tests/src/api/object-storage/index.ts | 4 + packages/tests/src/api/object-storage/live.ts | 314 +++ .../src/api/object-storage/video-imports.ts | 112 + .../video-static-file-privacy.ts | 573 +++++ .../tests/src/api/object-storage/videos.ts | 434 ++++ packages/tests/src/api/redundancy/index.ts | 3 + .../src/api/redundancy/manage-redundancy.ts | 324 +++ .../api/redundancy/redundancy-constraints.ts | 191 ++ .../tests/src/api/redundancy/redundancy.ts | 743 ++++++ packages/tests/src/api/runners/index.ts | 5 + .../tests/src/api/runners/runner-common.ts | 744 ++++++ .../api/runners/runner-live-transcoding.ts | 332 +++ .../tests/src/api/runners/runner-socket.ts | 120 + .../api/runners/runner-studio-transcoding.ts | 169 ++ .../src/api/runners/runner-vod-transcoding.ts | 545 +++++ packages/tests/src/api/search/index.ts | 7 + .../search-activitypub-video-channels.ts | 255 ++ .../search-activitypub-video-playlists.ts | 214 ++ .../api/search/search-activitypub-videos.ts | 196 ++ .../tests/src/api/search/search-channels.ts | 159 ++ packages/tests/src/api/search/search-index.ts | 438 ++++ .../tests/src/api/search/search-playlists.ts | 180 ++ .../tests/src/api/search/search-videos.ts | 568 +++++ packages/tests/src/api/server/auto-follows.ts | 189 ++ packages/tests/src/api/server/bulk.ts | 185 ++ .../tests/src/api/server/config-defaults.ts | 294 +++ packages/tests/src/api/server/config.ts | 645 ++++++ packages/tests/src/api/server/contact-form.ts | 101 + packages/tests/src/api/server/email.ts | 371 +++ .../src/api/server/follow-constraints.ts | 321 +++ .../src/api/server/follows-moderation.ts | 364 +++ packages/tests/src/api/server/follows.ts | 644 ++++++ packages/tests/src/api/server/handle-down.ts | 339 +++ packages/tests/src/api/server/homepage.ts | 81 + packages/tests/src/api/server/index.ts | 22 + packages/tests/src/api/server/jobs.ts | 128 + packages/tests/src/api/server/logs.ts | 265 +++ packages/tests/src/api/server/no-client.ts | 24 + .../tests/src/api/server/open-telemetry.ts | 193 ++ packages/tests/src/api/server/plugins.ts | 410 ++++ packages/tests/src/api/server/proxy.ts | 173 ++ .../tests/src/api/server/reverse-proxy.ts | 156 ++ packages/tests/src/api/server/services.ts | 143 ++ packages/tests/src/api/server/slow-follows.ts | 85 + packages/tests/src/api/server/stats.ts | 279 +++ packages/tests/src/api/server/tracker.ts | 110 + .../tests/src/api/transcoding/audio-only.ts | 104 + .../src/api/transcoding/create-transcoding.ts | 267 +++ packages/tests/src/api/transcoding/hls.ts | 176 ++ packages/tests/src/api/transcoding/index.ts | 6 + .../tests/src/api/transcoding/transcoder.ts | 802 +++++++ .../transcoding/update-while-transcoding.ts | 161 ++ .../tests/src/api/transcoding/video-studio.ts | 379 +++ packages/tests/src/api/users/index.ts | 8 + packages/tests/src/api/users/oauth.ts | 203 ++ packages/tests/src/api/users/registrations.ts | 415 ++++ packages/tests/src/api/users/two-factor.ts | 206 ++ .../tests/src/api/users/user-subscriptions.ts | 614 +++++ packages/tests/src/api/users/user-videos.ts | 219 ++ .../src/api/users/users-email-verification.ts | 165 ++ .../src/api/users/users-multiple-servers.ts | 213 ++ packages/tests/src/api/users/users.ts | 529 +++++ .../src/api/videos/channel-import-videos.ts | 161 ++ packages/tests/src/api/videos/index.ts | 23 + .../tests/src/api/videos/multiple-servers.ts | 1095 +++++++++ .../tests/src/api/videos/resumable-upload.ts | 316 +++ .../tests/src/api/videos/single-server.ts | 461 ++++ .../tests/src/api/videos/video-captions.ts | 189 ++ .../src/api/videos/video-change-ownership.ts | 314 +++ .../src/api/videos/video-channel-syncs.ts | 321 +++ .../tests/src/api/videos/video-channels.ts | 556 +++++ .../tests/src/api/videos/video-comments.ts | 335 +++ .../tests/src/api/videos/video-description.ts | 103 + packages/tests/src/api/videos/video-files.ts | 202 ++ .../tests/src/api/videos/video-imports.ts | 634 +++++ packages/tests/src/api/videos/video-nsfw.ts | 227 ++ .../tests/src/api/videos/video-passwords.ts | 97 + .../api/videos/video-playlist-thumbnails.ts | 234 ++ .../tests/src/api/videos/video-playlists.ts | 1210 ++++++++++ .../tests/src/api/videos/video-privacy.ts | 294 +++ .../src/api/videos/video-schedule-update.ts | 155 ++ packages/tests/src/api/videos/video-source.ts | 448 ++++ .../api/videos/video-static-file-privacy.ts | 602 +++++ .../tests/src/api/videos/video-storyboard.ts | 213 ++ .../src/api/videos/videos-common-filters.ts | 499 ++++ .../tests/src/api/videos/videos-history.ts | 230 ++ .../tests/src/api/videos/videos-overview.ts | 129 ++ packages/tests/src/api/views/index.ts | 5 + .../src/api/views/video-views-counter.ts | 153 ++ .../api/views/video-views-overall-stats.ts | 368 +++ .../api/views/video-views-retention-stats.ts | 53 + .../api/views/video-views-timeserie-stats.ts | 253 ++ .../src/api/views/videos-views-cleaner.ts | 98 + .../src/cli/create-generate-storyboard-job.ts | 121 + .../src/cli/create-import-video-file-job.ts | 168 ++ .../src/cli/create-move-video-storage-job.ts | 125 + .../tests => packages/tests/src}/cli/index.ts | 0 packages/tests/src/cli/peertube.ts | 257 +++ packages/tests/src/cli/plugins.ts | 76 + packages/tests/src/cli/prune-storage.ts | 224 ++ .../tests/src/cli/regenerate-thumbnails.ts | 122 + packages/tests/src/cli/reset-password.ts | 26 + packages/tests/src/cli/update-host.ts | 134 ++ packages/tests/src/client.ts | 556 +++++ .../tests/src/external-plugins/akismet.ts | 160 ++ .../tests/src/external-plugins/auth-ldap.ts | 117 + .../src/external-plugins/auto-block-videos.ts | 167 ++ .../tests/src/external-plugins/auto-mute.ts | 216 ++ .../tests/src}/external-plugins/index.ts | 0 packages/tests/src/feeds/feeds.ts | 697 ++++++ .../tests/src}/feeds/index.ts | 0 packages/tests/src/misc-endpoints.ts | 243 ++ .../tests/src/peertube-runner/client-cli.ts | 76 + packages/tests/src/peertube-runner/index.ts | 4 + .../src/peertube-runner/live-transcoding.ts | 200 ++ .../src/peertube-runner/studio-transcoding.ts | 127 + .../src/peertube-runner/vod-transcoding.ts | 349 +++ packages/tests/src/plugins/action-hooks.ts | 298 +++ packages/tests/src/plugins/external-auth.ts | 436 ++++ packages/tests/src/plugins/filter-hooks.ts | 909 ++++++++ packages/tests/src/plugins/html-injection.ts | 73 + .../tests/src/plugins/id-and-pass-auth.ts | 248 ++ .../tests/src}/plugins/index.ts | 0 packages/tests/src/plugins/plugin-helpers.ts | 383 +++ packages/tests/src/plugins/plugin-router.ts | 105 + packages/tests/src/plugins/plugin-storage.ts | 95 + .../tests/src/plugins/plugin-transcoding.ts | 279 +++ .../tests/src/plugins/plugin-unloading.ts | 75 + .../tests/src/plugins/plugin-websocket.ts | 76 + packages/tests/src/plugins/translations.ts | 80 + packages/tests/src/plugins/video-constants.ts | 180 ++ .../tests/src/server-helpers/activitypub.ts | 176 ++ .../tests/src/server-helpers/core-utils.ts | 150 ++ packages/tests/src/server-helpers/crypto.ts | 33 + packages/tests/src/server-helpers/dns.ts | 16 + packages/tests/src/server-helpers/image.ts | 97 + packages/tests/src/server-helpers/index.ts | 10 + packages/tests/src/server-helpers/markdown.ts | 39 + packages/tests/src/server-helpers/mentions.ts | 17 + packages/tests/src/server-helpers/request.ts | 64 + .../tests/src/server-helpers/validator.ts | 35 + packages/tests/src/server-helpers/version.ts | 36 + packages/tests/src/server-lib/index.ts | 1 + .../video-constant-registry-factory.ts | 151 ++ packages/tests/src/shared/actors.ts | 70 + packages/tests/src/shared/captions.ts | 21 + packages/tests/src/shared/checks.ts | 177 ++ packages/tests/src/shared/directories.ts | 44 + packages/tests/src/shared/generate.ts | 79 + packages/tests/src/shared/live.ts | 186 ++ .../tests/src/shared/mock-servers/index.ts | 8 + .../tests/src/shared/mock-servers/mock-429.ts | 33 + .../src/shared/mock-servers/mock-email.ts | 62 + .../src/shared/mock-servers/mock-http.ts | 23 + .../mock-servers/mock-instances-index.ts | 46 + .../mock-joinpeertube-versions.ts | 34 + .../mock-servers/mock-object-storage.ts | 41 + .../mock-servers/mock-plugin-blocklist.ts | 36 + .../src/shared/mock-servers/mock-proxy.ts | 24 + .../tests/src}/shared/mock-servers/shared.ts | 0 packages/tests/src/shared/notifications.ts | 891 +++++++ .../src/shared/peertube-runner-process.ts | 104 + packages/tests/src/shared/plugins.ts | 18 + packages/tests/src/shared/requests.ts | 12 + packages/tests/src/shared/sql-command.ts | 150 ++ .../tests/src/shared/streaming-playlists.ts | 302 +++ .../tests/src}/shared/tests.ts | 0 packages/tests/src/shared/tracker.ts | 27 + packages/tests/src/shared/video-playlists.ts | 22 + packages/tests/src/shared/videos.ts | 323 +++ packages/tests/src/shared/views.ts | 93 + packages/tests/src/shared/webtorrent.ts | 58 + packages/tests/tsconfig.json | 27 + packages/{types => types-generator}/README.md | 0 packages/types-generator/generate-package.ts | 107 + packages/types-generator/package.json | 9 + packages/types-generator/src/client/index.ts | 1 + .../src/client/tsconfig.types.json | 19 + packages/types-generator/src/index.ts | 3 + packages/types-generator/tests/test.ts | 32 + packages/types-generator/tsconfig.dist.json | 17 + packages/types-generator/tsconfig.json | 11 + packages/types-generator/tsconfig.types.json | 23 + packages/types/generate-package.ts | 96 - packages/types/src/client/index.ts | 1 - packages/types/src/client/tsconfig.json | 15 - packages/types/src/index.ts | 3 - packages/types/tests/test.ts | 32 - packages/types/tsconfig.dist.json | 16 - packages/types/tsconfig.json | 23 - packages/typescript-utils/package.json | 19 + packages/typescript-utils/src/index.ts | 1 + .../typescript-utils/src}/types.ts | 0 packages/typescript-utils/tsconfig.json | 8 + packages/typescript-utils/tsconfig.types.json | 10 + scripts/benchmark.ts | 12 +- scripts/build/peertube-cli.sh | 12 + scripts/build/peertube-runner.sh | 5 +- scripts/build/server.sh | 9 +- scripts/build/tests.sh | 9 + scripts/ci.sh | 104 +- scripts/client-build-stats.ts | 4 +- scripts/create-generate-storyboard-job.ts | 85 - scripts/create-import-video-file-job.ts | 50 - scripts/create-move-video-storage-job.ts | 99 - scripts/dev/cli.sh | 16 - scripts/dev/peertube-cli.sh | 11 + scripts/dev/peertube-runner.sh | 6 +- scripts/dev/server.sh | 8 +- scripts/generate-code-contributors.ts | 2 +- scripts/i18n/create-custom-files.ts | 14 +- scripts/migrations/peertube-4.0.ts | 104 - scripts/migrations/peertube-4.2.ts | 124 - scripts/migrations/peertube-5.0.ts | 71 - scripts/nightly.sh | 11 + scripts/parse-log.ts | 160 -- scripts/plugin/install.ts | 41 - scripts/plugin/uninstall.ts | 29 - scripts/prune-storage.ts | 184 -- scripts/regenerate-thumbnails.ts | 64 - scripts/release.sh | 13 +- scripts/reset-password.ts | 58 - scripts/setup/cli.sh | 17 - scripts/simulate-many-viewers.ts | 4 +- scripts/tsconfig.json | 5 +- scripts/update-host.ts | 140 -- server.ts | 376 --- server/controllers/activitypub/client.ts | 482 ---- server/controllers/activitypub/inbox.ts | 85 - server/controllers/activitypub/index.ts | 17 - server/controllers/activitypub/outbox.ts | 86 - server/controllers/api/abuse.ts | 259 --- server/controllers/api/accounts.ts | 266 --- server/controllers/api/blocklist.ts | 110 - server/controllers/api/bulk.ts | 44 - server/controllers/api/config.ts | 377 --- server/controllers/api/custom-page.ts | 48 - server/controllers/api/index.ts | 73 - server/controllers/api/jobs.ts | 109 - server/controllers/api/metrics.ts | 34 - server/controllers/api/oauth-clients.ts | 54 - server/controllers/api/overviews.ts | 139 -- server/controllers/api/plugins.ts | 230 -- server/controllers/api/runners/index.ts | 20 - server/controllers/api/runners/jobs-files.ts | 112 - server/controllers/api/runners/jobs.ts | 416 ---- .../controllers/api/runners/manage-runners.ts | 112 - .../api/runners/registration-tokens.ts | 91 - server/controllers/api/search/index.ts | 19 - .../api/search/search-video-channels.ts | 152 -- .../api/search/search-video-playlists.ts | 131 -- .../controllers/api/search/search-videos.ts | 167 -- server/controllers/api/search/shared/index.ts | 1 - server/controllers/api/server/contact.ts | 34 - server/controllers/api/server/debug.ts | 56 - server/controllers/api/server/follows.ts | 214 -- server/controllers/api/server/index.ts | 27 - server/controllers/api/server/logs.ts | 203 -- server/controllers/api/server/redundancy.ts | 116 - .../api/server/server-blocklist.ts | 158 -- server/controllers/api/server/stats.ts | 26 - .../api/users/email-verification.ts | 72 - server/controllers/api/users/index.ts | 319 --- server/controllers/api/users/me.ts | 277 --- server/controllers/api/users/my-abuses.ts | 48 - server/controllers/api/users/my-blocklist.ts | 149 -- server/controllers/api/users/my-history.ts | 75 - .../controllers/api/users/my-notifications.ts | 116 - .../controllers/api/users/my-subscriptions.ts | 193 -- .../api/users/my-video-playlists.ts | 51 - server/controllers/api/users/registrations.ts | 249 -- server/controllers/api/users/token.ts | 131 -- server/controllers/api/users/two-factor.ts | 95 - server/controllers/api/video-channel-sync.ts | 79 - server/controllers/api/video-channel.ts | 431 ---- server/controllers/api/video-playlist.ts | 514 ----- server/controllers/api/videos/blacklist.ts | 112 - server/controllers/api/videos/captions.ts | 93 - server/controllers/api/videos/comment.ts | 234 -- server/controllers/api/videos/files.ts | 122 - server/controllers/api/videos/import.ts | 262 --- server/controllers/api/videos/index.ts | 228 -- server/controllers/api/videos/live.ts | 224 -- server/controllers/api/videos/ownership.ts | 138 -- server/controllers/api/videos/passwords.ts | 105 - server/controllers/api/videos/rate.ts | 87 - server/controllers/api/videos/source.ts | 206 -- server/controllers/api/videos/stats.ts | 75 - server/controllers/api/videos/storyboard.ts | 29 - server/controllers/api/videos/studio.ts | 143 -- server/controllers/api/videos/token.ts | 33 - server/controllers/api/videos/transcoding.ts | 60 - server/controllers/api/videos/update.ts | 210 -- server/controllers/api/videos/upload.ts | 287 --- server/controllers/api/videos/view.ts | 60 - server/controllers/client.ts | 236 -- server/controllers/download.ts | 213 -- server/controllers/feeds/comment-feeds.ts | 96 - server/controllers/feeds/index.ts | 25 - .../feeds/shared/common-feed-utils.ts | 149 -- server/controllers/feeds/shared/index.ts | 2 - .../feeds/shared/video-feed-utils.ts | 66 - server/controllers/feeds/video-feeds.ts | 189 -- .../controllers/feeds/video-podcast-feeds.ts | 313 --- server/controllers/index.ts | 14 - server/controllers/lazy-static.ts | 128 - server/controllers/misc.ts | 210 -- server/controllers/object-storage-proxy.ts | 60 - server/controllers/plugins.ts | 175 -- server/controllers/services.ts | 165 -- server/controllers/sitemap.ts | 115 - server/controllers/static.ts | 116 - server/controllers/tracker.ts | 148 -- server/controllers/well-known.ts | 125 - server/helpers/actors.ts | 17 - server/helpers/audit-logger.ts | 287 --- server/helpers/captions-utils.ts | 53 - server/helpers/core-utils.ts | 315 --- server/helpers/custom-jsonld-signature.ts | 91 - server/helpers/custom-validators/abuses.ts | 68 - server/helpers/custom-validators/accounts.ts | 22 - .../custom-validators/activitypub/activity.ts | 151 -- .../custom-validators/activitypub/actor.ts | 142 -- .../activitypub/cache-file.ts | 26 - .../custom-validators/activitypub/misc.ts | 76 - .../custom-validators/activitypub/playlist.ts | 29 - .../activitypub/signature.ts | 22 - .../activitypub/video-comments.ts | 59 - .../custom-validators/activitypub/videos.ts | 241 -- .../activitypub/watch-action.ts | 37 - .../helpers/custom-validators/actor-images.ts | 24 - server/helpers/custom-validators/feeds.ts | 23 - server/helpers/custom-validators/follows.ts | 30 - server/helpers/custom-validators/jobs.ts | 21 - server/helpers/custom-validators/logs.ts | 42 - server/helpers/custom-validators/misc.ts | 190 -- server/helpers/custom-validators/plugins.ts | 178 -- .../helpers/custom-validators/runners/jobs.ts | 197 -- .../custom-validators/runners/runners.ts | 30 - server/helpers/custom-validators/search.ts | 37 - server/helpers/custom-validators/servers.ts | 42 - .../custom-validators/user-notifications.ts | 23 - .../custom-validators/user-registration.ts | 25 - server/helpers/custom-validators/users.ts | 123 - .../custom-validators/video-blacklist.ts | 22 - .../custom-validators/video-captions.ts | 43 - .../custom-validators/video-channel-syncs.ts | 6 - .../custom-validators/video-channels.ts | 32 - .../custom-validators/video-comments.ts | 14 - .../custom-validators/video-imports.ts | 46 - .../helpers/custom-validators/video-lives.ts | 11 - .../custom-validators/video-ownership.ts | 20 - .../custom-validators/video-playlists.ts | 35 - .../custom-validators/video-redundancies.ts | 12 - .../helpers/custom-validators/video-stats.ts | 16 - .../helpers/custom-validators/video-studio.ts | 53 - .../custom-validators/video-transcoding.ts | 12 - .../helpers/custom-validators/video-view.ts | 12 - server/helpers/custom-validators/videos.ts | 218 -- server/helpers/custom-validators/webfinger.ts | 21 - server/helpers/database-utils.ts | 121 - server/helpers/decache.ts | 78 - server/helpers/dns.ts | 29 - server/helpers/express-utils.ts | 156 -- server/helpers/ffmpeg/codecs.ts | 64 - server/helpers/ffmpeg/ffmpeg-image.ts | 14 - server/helpers/ffmpeg/ffmpeg-options.ts | 45 - server/helpers/ffmpeg/framerate.ts | 44 - server/helpers/ffmpeg/index.ts | 4 - server/helpers/geo-ip.ts | 78 - server/helpers/image-utils.ts | 179 -- server/helpers/logger.ts | 208 -- server/helpers/markdown.ts | 90 - server/helpers/otp.ts | 58 - server/helpers/peertube-crypto.ts | 208 -- server/helpers/query.ts | 81 - server/helpers/requests.ts | 246 -- server/helpers/token-generator.ts | 19 - server/helpers/upload.ts | 14 - server/helpers/utils.ts | 70 - server/helpers/version.ts | 36 - server/helpers/video.ts | 51 - server/helpers/webtorrent.ts | 258 --- server/helpers/youtube-dl/index.ts | 3 - server/helpers/youtube-dl/youtube-dl-cli.ts | 259 --- .../youtube-dl/youtube-dl-info-builder.ts | 205 -- .../helpers/youtube-dl/youtube-dl-wrapper.ts | 154 -- server/initializers/checker-after-init.ts | 325 --- server/initializers/checker-before-init.ts | 162 -- server/initializers/config.ts | 688 ------ server/initializers/constants.ts | 1396 ----------- server/initializers/database.ts | 233 -- server/initializers/installer.ts | 199 -- .../0530-playlist-multiple-video.ts | 46 - .../migrations/0560-user-feed-token.ts | 51 - .../migrations/0605-actor-missing-keys.ts | 33 - .../migrations/0660-object-storage.ts | 56 - .../migrations/0690-live-latency-mode.ts | 35 - server/initializers/migrator.ts | 105 - server/lib/activitypub/activity.ts | 74 - server/lib/activitypub/actors/get.ts | 143 -- server/lib/activitypub/actors/image.ts | 112 - server/lib/activitypub/actors/index.ts | 6 - server/lib/activitypub/actors/keys.ts | 16 - server/lib/activitypub/actors/refresh.ts | 81 - .../lib/activitypub/actors/shared/creator.ts | 149 -- server/lib/activitypub/actors/shared/index.ts | 3 - .../shared/object-to-model-attributes.ts | 84 - .../actors/shared/url-to-object.ts | 56 - server/lib/activitypub/actors/updater.ts | 91 - server/lib/activitypub/actors/webfinger.ts | 67 - server/lib/activitypub/audience.ts | 34 - server/lib/activitypub/cache-file.ts | 82 - server/lib/activitypub/collection.ts | 63 - server/lib/activitypub/context.ts | 212 -- server/lib/activitypub/crawl.ts | 58 - server/lib/activitypub/follow.ts | 51 - server/lib/activitypub/inbox-manager.ts | 47 - server/lib/activitypub/local-video-viewer.ts | 44 - server/lib/activitypub/outbox.ts | 24 - .../activitypub/playlists/create-update.ts | 157 -- server/lib/activitypub/playlists/get.ts | 35 - server/lib/activitypub/playlists/index.ts | 3 - server/lib/activitypub/playlists/refresh.ts | 53 - .../lib/activitypub/playlists/shared/index.ts | 2 - .../shared/object-to-model-attributes.ts | 40 - .../playlists/shared/url-to-object.ts | 47 - server/lib/activitypub/process/index.ts | 1 - .../lib/activitypub/process/process-accept.ts | 32 - .../activitypub/process/process-announce.ts | 75 - .../lib/activitypub/process/process-create.ts | 170 -- .../lib/activitypub/process/process-delete.ts | 153 -- .../activitypub/process/process-dislike.ts | 58 - .../lib/activitypub/process/process-flag.ts | 103 - .../lib/activitypub/process/process-follow.ts | 156 -- .../lib/activitypub/process/process-like.ts | 60 - .../lib/activitypub/process/process-reject.ts | 33 - .../lib/activitypub/process/process-undo.ts | 183 -- .../lib/activitypub/process/process-update.ts | 119 - .../lib/activitypub/process/process-view.ts | 42 - server/lib/activitypub/process/process.ts | 92 - server/lib/activitypub/send/http.ts | 73 - server/lib/activitypub/send/index.ts | 10 - server/lib/activitypub/send/send-accept.ts | 47 - server/lib/activitypub/send/send-announce.ts | 58 - server/lib/activitypub/send/send-create.ts | 226 -- server/lib/activitypub/send/send-delete.ts | 158 -- server/lib/activitypub/send/send-dislike.ts | 40 - server/lib/activitypub/send/send-flag.ts | 42 - server/lib/activitypub/send/send-follow.ts | 37 - server/lib/activitypub/send/send-like.ts | 40 - server/lib/activitypub/send/send-reject.ts | 39 - server/lib/activitypub/send/send-undo.ts | 172 -- server/lib/activitypub/send/send-update.ts | 157 -- server/lib/activitypub/send/send-view.ts | 62 - .../activitypub/send/shared/audience-utils.ts | 74 - server/lib/activitypub/send/shared/index.ts | 2 - .../lib/activitypub/send/shared/send-utils.ts | 291 --- server/lib/activitypub/share.ts | 120 - server/lib/activitypub/url.ts | 177 -- server/lib/activitypub/video-comments.ts | 205 -- server/lib/activitypub/video-rates.ts | 59 - server/lib/activitypub/videos/federate.ts | 29 - server/lib/activitypub/videos/get.ts | 116 - server/lib/activitypub/videos/index.ts | 4 - server/lib/activitypub/videos/refresh.ts | 68 - .../videos/shared/abstract-builder.ts | 190 -- .../lib/activitypub/videos/shared/creator.ts | 65 - server/lib/activitypub/videos/shared/index.ts | 6 - .../shared/object-to-model-attributes.ts | 285 --- .../lib/activitypub/videos/shared/trackers.ts | 43 - .../videos/shared/url-to-object.ts | 25 - .../videos/shared/video-sync-attributes.ts | 107 - server/lib/activitypub/videos/updater.ts | 180 -- server/lib/actor-follow-health-cache.ts | 86 - server/lib/actor-image.ts | 14 - server/lib/auth/external-auth.ts | 231 -- server/lib/auth/oauth-model.ts | 294 --- server/lib/auth/oauth.ts | 223 -- server/lib/auth/tokens-cache.ts | 52 - server/lib/blocklist.ts | 62 - server/lib/client-html.ts | 623 ----- server/lib/emailer.ts | 284 --- .../avatar-permanent-file-cache.ts | 27 - server/lib/files-cache/index.ts | 6 - .../shared/abstract-permanent-file-cache.ts | 132 -- .../shared/abstract-simple-file-cache.ts | 30 - server/lib/files-cache/shared/index.ts | 2 - .../video-captions-simple-file-cache.ts | 61 - .../video-miniature-permanent-file-cache.ts | 28 - .../video-previews-simple-file-cache.ts | 58 - .../video-storyboards-simple-file-cache.ts | 53 - .../video-torrents-simple-file-cache.ts | 70 - server/lib/hls.ts | 285 --- server/lib/internal-event-emitter.ts | 35 - .../job-queue/handlers/activitypub-cleaner.ts | 202 -- .../job-queue/handlers/activitypub-follow.ts | 82 - .../handlers/activitypub-http-broadcast.ts | 49 - .../handlers/activitypub-http-fetcher.ts | 41 - .../handlers/activitypub-http-unicast.ts | 38 - .../handlers/activitypub-refresher.ts | 60 - server/lib/job-queue/handlers/actor-keys.ts | 20 - .../handlers/after-video-channel-import.ts | 37 - server/lib/job-queue/handlers/email.ts | 17 - .../lib/job-queue/handlers/federate-video.ts | 28 - .../job-queue/handlers/generate-storyboard.ts | 163 -- .../handlers/manage-video-torrent.ts | 110 - .../handlers/move-to-object-storage.ts | 159 -- server/lib/job-queue/handlers/notify.ts | 27 - .../handlers/transcoding-job-builder.ts | 48 - .../handlers/video-channel-import.ts | 43 - .../job-queue/handlers/video-file-import.ts | 83 - server/lib/job-queue/handlers/video-import.ts | 344 --- .../job-queue/handlers/video-live-ending.ts | 279 --- .../job-queue/handlers/video-redundancy.ts | 17 - .../handlers/video-studio-edition.ts | 180 -- .../job-queue/handlers/video-transcoding.ts | 150 -- .../job-queue/handlers/video-views-stats.ts | 57 - server/lib/job-queue/index.ts | 1 - server/lib/job-queue/job-queue.ts | 537 ----- server/lib/live/index.ts | 4 - server/lib/live/live-manager.ts | 552 ----- server/lib/live/live-segment-sha-store.ts | 95 - server/lib/live/live-utils.ts | 99 - server/lib/live/shared/index.ts | 1 - server/lib/live/shared/muxing-session.ts | 518 ----- .../abstract-transcoding-wrapper.ts | 110 - .../ffmpeg-transcoding-wrapper.ts | 107 - .../live/shared/transcoding-wrapper/index.ts | 3 - .../remote-transcoding-wrapper.ts | 21 - server/lib/local-actor.ts | 102 - server/lib/model-loaders/actor.ts | 17 - server/lib/model-loaders/index.ts | 2 - server/lib/model-loaders/video.ts | 66 - server/lib/moderation.ts | 258 --- server/lib/notifier/index.ts | 1 - server/lib/notifier/notifier.ts | 284 --- .../abuse/abstract-new-abuse-message.ts | 67 - .../abuse/abuse-state-change-for-reporter.ts | 74 - server/lib/notifier/shared/abuse/index.ts | 4 - .../shared/abuse/new-abuse-for-moderators.ts | 119 - .../abuse/new-abuse-message-for-moderators.ts | 32 - .../abuse/new-abuse-message-for-reporter.ts | 36 - server/lib/notifier/shared/blacklist/index.ts | 3 - .../new-auto-blacklist-for-moderators.ts | 60 - .../blacklist/new-blacklist-for-owner.ts | 58 - .../shared/blacklist/unblacklist-for-owner.ts | 55 - .../shared/comment/comment-mention.ts | 111 - server/lib/notifier/shared/comment/index.ts | 2 - .../comment/new-comment-for-video-owner.ts | 76 - .../shared/common/abstract-notification.ts | 23 - server/lib/notifier/shared/common/index.ts | 1 - .../shared/follow/auto-follow-for-instance.ts | 51 - .../shared/follow/follow-for-instance.ts | 68 - .../notifier/shared/follow/follow-for-user.ts | 82 - server/lib/notifier/shared/follow/index.ts | 3 - server/lib/notifier/shared/index.ts | 7 - .../direct-registration-for-moderators.ts | 49 - server/lib/notifier/shared/instance/index.ts | 4 - .../new-peertube-version-for-admins.ts | 54 - .../instance/new-plugin-version-for-admins.ts | 58 - .../registration-request-for-moderators.ts | 48 - .../abstract-owned-video-publication.ts | 57 - .../import-finished-for-owner.ts | 97 - .../shared/video-publication/index.ts | 6 - .../new-video-for-subscribers.ts | 61 - ...wned-publication-after-auto-unblacklist.ts | 11 - ...owned-publication-after-schedule-update.ts | 10 - .../owned-publication-after-transcoding.ts | 9 - .../studio-edition-finished-for-owner.ts | 57 - server/lib/object-storage/index.ts | 5 - server/lib/object-storage/keys.ts | 20 - server/lib/object-storage/pre-signed-urls.ts | 46 - server/lib/object-storage/proxy.ts | 97 - server/lib/object-storage/shared/client.ts | 71 - server/lib/object-storage/shared/index.ts | 3 - server/lib/object-storage/shared/logger.ts | 7 - .../shared/object-storage-helpers.ts | 328 --- server/lib/object-storage/urls.ts | 63 - server/lib/object-storage/videos.ts | 197 -- .../lib/opentelemetry/metric-helpers/index.ts | 7 - .../job-queue-observers-builder.ts | 24 - .../metric-helpers/lives-observers-builder.ts | 21 - .../nodejs-observers-builder.ts | 202 -- .../metric-helpers/playback-metrics.ts | 85 - .../metric-helpers/stats-observers-builder.ts | 186 -- .../viewers-observers-builder.ts | 24 - server/lib/opentelemetry/metrics.ts | 123 - server/lib/opentelemetry/tracing.ts | 94 - server/lib/paths.ts | 92 - server/lib/peertube-socket.ts | 129 -- server/lib/plugins/hooks.ts | 35 - server/lib/plugins/plugin-helpers-builder.ts | 262 --- server/lib/plugins/plugin-index.ts | 85 - server/lib/plugins/plugin-manager.ts | 665 ------ server/lib/plugins/register-helpers.ts | 340 --- server/lib/plugins/theme-utils.ts | 24 - .../plugins/video-constant-manager-factory.ts | 139 -- server/lib/plugins/yarn.ts | 73 - server/lib/redis.ts | 465 ---- server/lib/redundancy.ts | 59 - server/lib/runners/index.ts | 3 - .../job-handlers/abstract-job-handler.ts | 269 --- .../abstract-vod-transcoding-job-handler.ts | 66 - server/lib/runners/job-handlers/index.ts | 7 - .../live-rtmp-hls-transcoding-job-handler.ts | 173 -- .../job-handlers/runner-job-handlers.ts | 20 - .../lib/runners/job-handlers/shared/index.ts | 1 - .../job-handlers/shared/vod-helpers.ts | 44 - .../video-studio-transcoding-job-handler.ts | 157 -- ...vod-audio-merge-transcoding-job-handler.ts | 97 - .../vod-hls-transcoding-job-handler.ts | 114 - .../vod-web-video-transcoding-job-handler.ts | 84 - server/lib/runners/runner-urls.ts | 13 - server/lib/runners/runner.ts | 49 - server/lib/schedulers/abstract-scheduler.ts | 35 - .../lib/schedulers/actor-follow-scheduler.ts | 54 - .../schedulers/auto-follow-index-instances.ts | 75 - .../lib/schedulers/geo-ip-update-scheduler.ts | 22 - .../peertube-version-check-scheduler.ts | 55 - .../lib/schedulers/plugins-check-scheduler.ts | 74 - ...ve-dangling-resumable-uploads-scheduler.ts | 40 - .../remove-old-history-scheduler.ts | 31 - .../schedulers/remove-old-views-scheduler.ts | 31 - .../runner-job-watch-dog-scheduler.ts | 42 - .../lib/schedulers/update-videos-scheduler.ts | 89 - .../video-channel-sync-latest-scheduler.ts | 50 - .../video-views-buffer-scheduler.ts | 52 - .../schedulers/videos-redundancy-scheduler.ts | 375 --- .../schedulers/youtube-dl-update-scheduler.ts | 22 - server/lib/search.ts | 49 - server/lib/server-config-manager.ts | 384 --- server/lib/signup.ts | 75 - server/lib/stat-manager.ts | 182 -- server/lib/sync-channel.ts | 111 - server/lib/thumbnail.ts | 327 --- server/lib/timeserie.ts | 61 - .../lib/transcoding/create-transcoding-job.ts | 37 - .../default-transcoding-profiles.ts | 143 -- server/lib/transcoding/ended-transcoding.ts | 18 - server/lib/transcoding/hls-transcoding.ts | 180 -- .../lib/transcoding/shared/ffmpeg-builder.ts | 18 - server/lib/transcoding/shared/index.ts | 2 - .../job-builders/abstract-job-builder.ts | 21 - .../transcoding/shared/job-builders/index.ts | 2 - .../transcoding-job-queue-builder.ts | 322 --- .../transcoding-runner-job-builder.ts | 196 -- .../lib/transcoding/transcoding-priority.ts | 24 - .../transcoding-quick-transcode.ts | 12 - .../transcoding/transcoding-resolutions.ts | 73 - server/lib/transcoding/web-transcoding.ts | 263 --- server/lib/uploadx.ts | 37 - server/lib/user.ts | 301 --- server/lib/video-blacklist.ts | 145 -- server/lib/video-channel.ts | 50 - server/lib/video-comment.ts | 116 - server/lib/video-file.ts | 145 -- server/lib/video-path-manager.ts | 174 -- server/lib/video-playlist.ts | 30 - server/lib/video-pre-import.ts | 323 --- server/lib/video-privacy.ts | 133 -- server/lib/video-state.ts | 154 -- server/lib/video-studio.ts | 130 -- server/lib/video-tokens-manager.ts | 78 - server/lib/video-urls.ts | 31 - server/lib/video.ts | 189 -- server/lib/views/shared/index.ts | 3 - .../lib/views/shared/video-viewer-counters.ts | 198 -- server/lib/views/shared/video-viewer-stats.ts | 196 -- server/lib/views/shared/video-views.ts | 70 - server/lib/views/video-views-manager.ts | 100 - server/lib/worker/parent-process.ts | 77 - server/lib/worker/workers/http-broadcast.ts | 32 - server/lib/worker/workers/image-downloader.ts | 35 - server/lib/worker/workers/image-processor.ts | 7 - server/middlewares/activitypub.ts | 156 -- server/middlewares/async.ts | 44 - server/middlewares/auth.ts | 113 - server/middlewares/cache/cache.ts | 38 - server/middlewares/cache/index.ts | 1 - server/middlewares/cache/shared/api-cache.ts | 314 --- server/middlewares/cache/shared/index.ts | 1 - server/middlewares/csp.ts | 40 - server/middlewares/error.ts | 63 - server/middlewares/index.ts | 15 - server/middlewares/pagination.ts | 19 - server/middlewares/rate-limiter.ts | 59 - server/middlewares/servers.ts | 29 - server/middlewares/user-right.ts | 26 - server/middlewares/validators/abuse.ts | 255 -- server/middlewares/validators/account.ts | 35 - .../validators/activitypub/activity.ts | 29 - .../validators/activitypub/index.ts | 3 - .../validators/activitypub/pagination.ts | 25 - .../validators/activitypub/signature.ts | 39 - server/middlewares/validators/actor-image.ts | 27 - server/middlewares/validators/blocklist.ts | 179 -- server/middlewares/validators/bulk.ts | 38 - server/middlewares/validators/config.ts | 194 -- server/middlewares/validators/feeds.ts | 178 -- server/middlewares/validators/follows.ts | 158 -- server/middlewares/validators/index.ts | 31 - server/middlewares/validators/jobs.ts | 29 - server/middlewares/validators/logs.ts | 93 - server/middlewares/validators/metrics.ts | 60 - .../validators/object-storage-proxy.ts | 20 - server/middlewares/validators/oembed.ts | 158 -- server/middlewares/validators/pagination.ts | 30 - server/middlewares/validators/plugins.ts | 218 -- server/middlewares/validators/redundancy.ts | 198 -- .../middlewares/validators/runners/index.ts | 3 - .../validators/runners/job-files.ts | 60 - server/middlewares/validators/runners/jobs.ts | 216 -- .../validators/runners/registration-token.ts | 37 - .../middlewares/validators/runners/runners.ts | 104 - server/middlewares/validators/search.ts | 112 - server/middlewares/validators/server.ts | 75 - .../middlewares/validators/shared/abuses.ts | 26 - .../middlewares/validators/shared/accounts.ts | 66 - server/middlewares/validators/shared/index.ts | 14 - .../validators/shared/user-registrations.ts | 60 - server/middlewares/validators/shared/users.ts | 63 - server/middlewares/validators/shared/utils.ts | 69 - .../validators/shared/video-blacklists.ts | 24 - .../validators/shared/video-captions.ts | 25 - .../validators/shared/video-channel-syncs.ts | 24 - .../validators/shared/video-channels.ts | 36 - .../validators/shared/video-comments.ts | 80 - .../validators/shared/video-imports.ts | 22 - .../validators/shared/video-ownerships.ts | 25 - .../validators/shared/video-passwords.ts | 80 - .../validators/shared/video-playlists.ts | 39 - .../middlewares/validators/shared/videos.ts | 311 --- server/middlewares/validators/sort.ts | 66 - server/middlewares/validators/static.ts | 184 -- server/middlewares/validators/themes.ts | 46 - server/middlewares/validators/two-factor.ts | 81 - .../validators/user-email-verification.ts | 94 - server/middlewares/validators/user-history.ts | 47 - .../validators/user-notifications.ts | 71 - .../validators/user-registrations.ts | 208 -- .../validators/user-subscriptions.ts | 111 - server/middlewares/validators/users.ts | 489 ---- server/middlewares/validators/videos/index.ts | 19 - .../validators/videos/shared/index.ts | 2 - .../validators/videos/shared/upload.ts | 39 - .../videos/shared/video-validators.ts | 100 - .../validators/videos/video-blacklist.ts | 87 - .../validators/videos/video-captions.ts | 83 - .../validators/videos/video-channel-sync.ts | 56 - .../validators/videos/video-channels.ts | 194 -- .../validators/videos/video-comments.ts | 249 -- .../validators/videos/video-files.ts | 163 -- .../validators/videos/video-imports.ts | 209 -- .../validators/videos/video-live.ts | 342 --- .../videos/video-ownership-changes.ts | 107 - .../validators/videos/video-passwords.ts | 77 - .../validators/videos/video-playlists.ts | 419 ---- .../validators/videos/video-rates.ts | 72 - .../validators/videos/video-shares.ts | 35 - .../validators/videos/video-source.ts | 130 -- .../validators/videos/video-stats.ts | 108 - .../validators/videos/video-studio.ts | 105 - .../validators/videos/video-token.ts | 24 - .../validators/videos/video-transcoding.ts | 61 - .../validators/videos/video-view.ts | 61 - .../middlewares/validators/videos/videos.ts | 575 ----- server/middlewares/validators/webfinger.ts | 37 - server/models/abuse/abuse-message.ts | 114 - server/models/abuse/abuse.ts | 624 ----- .../models/abuse/sql/abuse-query-builder.ts | 167 -- server/models/abuse/video-abuse.ts | 64 - server/models/abuse/video-comment-abuse.ts | 48 - server/models/account/account-blocklist.ts | 236 -- server/models/account/account-video-rate.ts | 259 --- server/models/account/account.ts | 468 ---- server/models/account/actor-custom-page.ts | 69 - server/models/actor/actor-follow.ts | 662 ------ server/models/actor/actor-image.ts | 171 -- server/models/actor/actor.ts | 686 ------ .../instance-list-followers-query-builder.ts | 69 - .../instance-list-following-query-builder.ts | 69 - .../shared/actor-follow-table-attributes.ts | 28 - .../instance-list-follows-query-builder.ts | 97 - server/models/application/application.ts | 79 - server/models/migrations.ts | 27 - server/models/oauth/oauth-client.ts | 63 - server/models/oauth/oauth-token.ts | 220 -- server/models/redundancy/video-redundancy.ts | 793 ------- server/models/runner/runner-job.ts | 357 --- .../runner/runner-registration-token.ts | 103 - server/models/runner/runner.ts | 124 - server/models/server/plugin.ts | 305 --- server/models/server/server-blocklist.ts | 190 -- server/models/server/server.ts | 104 - server/models/server/tracker.ts | 74 - server/models/server/video-tracker.ts | 31 - server/models/shared/index.ts | 8 - server/models/shared/model-builder.ts | 118 - server/models/shared/model-cache.ts | 90 - server/models/shared/query.ts | 82 - server/models/shared/sql.ts | 68 - .../user-notitication-list-query-builder.ts | 273 --- .../models/user/user-notification-setting.ts | 232 -- server/models/user/user-notification.ts | 534 ----- server/models/user/user-registration.ts | 259 --- server/models/user/user-video-history.ts | 111 - server/models/user/user.ts | 983 -------- server/models/video/formatter/index.ts | 2 - server/models/video/formatter/shared/index.ts | 1 - .../formatter/shared/video-format-utils.ts | 7 - .../formatter/video-activity-pub-format.ts | 296 --- .../video/formatter/video-api-format.ts | 305 --- server/models/video/schedule-video-update.ts | 95 - .../video-comment-list-query-builder.ts | 400 ---- .../comment/video-comment-table-attributes.ts | 43 - server/models/video/sql/video/index.ts | 3 - .../shared/abstract-video-query-builder.ts | 340 --- .../video/shared/video-file-query-builder.ts | 75 - .../sql/video/shared/video-model-builder.ts | 408 ---- .../video/video-model-get-query-builder.ts | 189 -- .../sql/video/videos-id-list-query-builder.ts | 728 ------ .../video/videos-model-list-query-builder.ts | 103 - server/models/video/storyboard.ts | 169 -- server/models/video/tag.ts | 86 - server/models/video/thumbnail.ts | 208 -- server/models/video/video-blacklist.ts | 134 -- server/models/video/video-caption.ts | 247 -- server/models/video/video-change-ownership.ts | 137 -- server/models/video/video-channel-sync.ts | 176 -- server/models/video/video-channel.ts | 860 ------- server/models/video/video-comment.ts | 683 ------ server/models/video/video-file.ts | 635 ----- server/models/video/video-import.ts | 267 --- server/models/video/video-job-info.ts | 121 - .../models/video/video-live-replay-setting.ts | 42 - server/models/video/video-live-session.ts | 217 -- server/models/video/video-live.ts | 184 -- server/models/video/video-password.ts | 137 -- server/models/video/video-playlist-element.ts | 370 --- server/models/video/video-playlist.ts | 725 ------ server/models/video/video-share.ts | 216 -- server/models/video/video-source.ts | 56 - .../models/video/video-streaming-playlist.ts | 328 --- server/models/video/video-tag.ts | 31 - server/models/video/video.ts | 2047 ---------------- .../view/local-video-viewer-watch-section.ts | 63 - server/models/view/local-video-viewer.ts | 368 --- server/models/view/video-view.ts | 67 - server/package.json | 12 + .../scripts/create-generate-storyboard-job.ts | 85 + .../scripts/create-import-video-file-job.ts | 50 + .../scripts/create-move-video-storage-job.ts | 99 + server/scripts/migrations/peertube-4.0.ts | 110 + server/scripts/migrations/peertube-4.2.ts | 123 + server/scripts/migrations/peertube-5.0.ts | 71 + server/scripts/parse-log.ts | 161 ++ server/scripts/plugin/install.ts | 41 + server/scripts/plugin/uninstall.ts | 29 + server/scripts/prune-storage.ts | 187 ++ server/scripts/regenerate-thumbnails.ts | 64 + server/scripts/reset-password.ts | 58 + server/scripts/update-host.ts | 140 ++ {scripts => server/scripts}/upgrade.sh | 0 server/server.ts | 376 +++ .../assets/default-audio-background.jpg | Bin .../assets/default-live-background.jpg | Bin .../server/controllers/activitypub/client.ts | 485 ++++ .../server/controllers/activitypub/inbox.ts | 84 + .../server/controllers/activitypub/index.ts | 17 + .../server/controllers/activitypub/outbox.ts | 86 + .../controllers/activitypub/utils.ts | 0 server/server/controllers/api/abuse.ts | 259 +++ server/server/controllers/api/accounts.ts | 266 +++ server/server/controllers/api/blocklist.ts | 110 + server/server/controllers/api/bulk.ts | 43 + server/server/controllers/api/config.ts | 377 +++ server/server/controllers/api/custom-page.ts | 48 + server/server/controllers/api/index.ts | 73 + server/server/controllers/api/jobs.ts | 109 + server/server/controllers/api/metrics.ts | 34 + .../server/controllers/api/oauth-clients.ts | 54 + server/server/controllers/api/overviews.ts | 139 ++ server/server/controllers/api/plugins.ts | 230 ++ .../server/controllers/api/runners/index.ts | 20 + .../controllers/api/runners/jobs-files.ts | 112 + server/server/controllers/api/runners/jobs.ts | 416 ++++ .../controllers/api/runners/manage-runners.ts | 116 + .../api/runners/registration-tokens.ts | 91 + server/server/controllers/api/search/index.ts | 19 + .../api/search/search-video-channels.ts | 151 ++ .../api/search/search-video-playlists.ts | 131 ++ .../controllers/api/search/search-videos.ts | 166 ++ .../controllers/api/search/shared/index.ts | 1 + .../controllers/api/search/shared/utils.ts | 0 .../server/controllers/api/server/contact.ts | 33 + server/server/controllers/api/server/debug.ts | 54 + .../server/controllers/api/server/follows.ts | 212 ++ server/server/controllers/api/server/index.ts | 27 + server/server/controllers/api/server/logs.ts | 201 ++ .../controllers/api/server/redundancy.ts | 115 + .../api/server/server-blocklist.ts | 162 ++ server/server/controllers/api/server/stats.ts | 26 + .../api/users/email-verification.ts | 72 + server/server/controllers/api/users/index.ts | 319 +++ server/server/controllers/api/users/me.ts | 283 +++ .../server/controllers/api/users/my-abuses.ts | 48 + .../controllers/api/users/my-blocklist.ts | 154 ++ .../controllers/api/users/my-history.ts | 75 + .../controllers/api/users/my-notifications.ts | 115 + .../controllers/api/users/my-subscriptions.ts | 193 ++ .../api/users/my-video-playlists.ts | 51 + .../controllers/api/users/registrations.ts | 249 ++ server/server/controllers/api/users/token.ts | 131 ++ .../controllers/api/users/two-factor.ts | 95 + .../controllers/api/video-channel-sync.ts | 79 + .../server/controllers/api/video-channel.ts | 437 ++++ .../server/controllers/api/video-playlist.ts | 518 +++++ .../controllers/api/videos/blacklist.ts | 112 + .../server/controllers/api/videos/captions.ts | 93 + .../server/controllers/api/videos/comment.ts | 238 ++ server/server/controllers/api/videos/files.ts | 122 + .../server/controllers/api/videos/import.ts | 270 +++ server/server/controllers/api/videos/index.ts | 228 ++ server/server/controllers/api/videos/live.ts | 232 ++ .../controllers/api/videos/ownership.ts | 137 ++ .../controllers/api/videos/passwords.ts | 104 + server/server/controllers/api/videos/rate.ts | 87 + .../server/controllers/api/videos/source.ts | 206 ++ server/server/controllers/api/videos/stats.ts | 75 + .../controllers/api/videos/storyboard.ts | 29 + .../server/controllers/api/videos/studio.ts | 143 ++ server/server/controllers/api/videos/token.ts | 33 + .../controllers/api/videos/transcoding.ts | 60 + .../server/controllers/api/videos/update.ts | 210 ++ .../server/controllers/api/videos/upload.ts | 287 +++ server/server/controllers/api/videos/view.ts | 66 + server/server/controllers/client.ts | 236 ++ server/server/controllers/download.ts | 213 ++ .../server/controllers/feeds/comment-feeds.ts | 96 + server/server/controllers/feeds/index.ts | 25 + .../feeds/shared/common-feed-utils.ts | 149 ++ .../server/controllers/feeds/shared/index.ts | 2 + .../feeds/shared/video-feed-utils.ts | 66 + .../server/controllers/feeds/video-feeds.ts | 189 ++ .../controllers/feeds/video-podcast-feeds.ts | 313 +++ server/server/controllers/index.ts | 14 + server/server/controllers/lazy-static.ts | 128 + server/server/controllers/misc.ts | 209 ++ .../controllers/object-storage-proxy.ts | 60 + server/server/controllers/plugins.ts | 174 ++ server/server/controllers/services.ts | 164 ++ .../controllers/shared/m3u8-playlist.ts | 0 server/server/controllers/sitemap.ts | 115 + server/server/controllers/static.ts | 116 + server/server/controllers/tracker.ts | 148 ++ server/server/controllers/well-known.ts | 125 + server/server/helpers/activity-pub-utils.ts | 230 ++ server/server/helpers/actors.ts | 17 + server/server/helpers/audit-logger.ts | 298 +++ server/server/helpers/captions-utils.ts | 55 + server/server/helpers/core-utils.ts | 288 +++ .../server/helpers/custom-jsonld-signature.ts | 90 + .../helpers/custom-validators/abuses.ts | 68 + .../helpers/custom-validators/accounts.ts | 22 + .../custom-validators/activitypub/activity.ts | 151 ++ .../custom-validators/activitypub/actor.ts | 142 ++ .../activitypub/cache-file.ts | 26 + .../custom-validators/activitypub/misc.ts | 76 + .../custom-validators/activitypub/playlist.ts | 29 + .../activitypub/signature.ts | 22 + .../activitypub/video-comments.ts | 59 + .../custom-validators/activitypub/videos.ts | 247 ++ .../activitypub/watch-action.ts | 37 + .../helpers/custom-validators/actor-images.ts | 24 + .../helpers/custom-validators/bulk.ts | 0 .../server/helpers/custom-validators/feeds.ts | 23 + .../helpers/custom-validators/follows.ts | 30 + .../server/helpers/custom-validators/jobs.ts | 21 + .../server/helpers/custom-validators/logs.ts | 42 + .../helpers/custom-validators/metrics.ts | 0 .../server/helpers/custom-validators/misc.ts | 190 ++ .../helpers/custom-validators/plugins.ts | 177 ++ .../helpers/custom-validators/runners/jobs.ts | 197 ++ .../custom-validators/runners/runners.ts | 30 + .../helpers/custom-validators/search.ts | 37 + .../helpers/custom-validators/servers.ts | 42 + .../custom-validators/user-notifications.ts | 23 + .../custom-validators/user-registration.ts | 25 + .../server/helpers/custom-validators/users.ts | 125 + .../custom-validators/video-blacklist.ts | 22 + .../custom-validators/video-captions.ts | 43 + .../custom-validators/video-channel-syncs.ts | 6 + .../custom-validators/video-channels.ts | 32 + .../custom-validators/video-comments.ts | 14 + .../custom-validators/video-imports.ts | 46 + .../helpers/custom-validators/video-lives.ts | 11 + .../custom-validators/video-ownership.ts | 20 + .../custom-validators/video-playlists.ts | 35 + .../helpers/custom-validators/video-rates.ts | 0 .../custom-validators/video-redundancies.ts | 12 + .../helpers/custom-validators/video-stats.ts | 16 + .../helpers/custom-validators/video-studio.ts | 53 + .../custom-validators/video-transcoding.ts | 12 + .../helpers/custom-validators/video-view.ts | 12 + .../helpers/custom-validators/videos.ts | 218 ++ .../helpers/custom-validators/webfinger.ts | 21 + server/server/helpers/database-utils.ts | 121 + server/{ => server}/helpers/debounce.ts | 0 server/server/helpers/decache.ts | 79 + server/server/helpers/dns.ts | 29 + server/server/helpers/express-utils.ts | 156 ++ server/server/helpers/ffmpeg/codecs.ts | 64 + server/server/helpers/ffmpeg/ffmpeg-image.ts | 14 + .../server/helpers/ffmpeg/ffmpeg-options.ts | 45 + server/server/helpers/ffmpeg/framerate.ts | 43 + server/server/helpers/ffmpeg/index.ts | 4 + server/server/helpers/geo-ip.ts | 79 + server/server/helpers/image-utils.ts | 184 ++ server/server/helpers/logger.ts | 208 ++ server/server/helpers/markdown.ts | 89 + server/{ => server}/helpers/memoize.ts | 0 server/server/helpers/mentions.ts | 42 + server/server/helpers/otp.ts | 58 + server/server/helpers/peertube-crypto.ts | 208 ++ server/{ => server}/helpers/promise-cache.ts | 0 server/{ => server}/helpers/proxy.ts | 0 server/server/helpers/query.ts | 81 + server/{ => server}/helpers/regexp.ts | 0 server/server/helpers/requests.ts | 258 +++ .../{ => server}/helpers/stream-replacer.ts | 0 server/server/helpers/token-generator.ts | 19 + server/server/helpers/upload.ts | 14 + server/server/helpers/utils.ts | 70 + server/server/helpers/version.ts | 36 + server/server/helpers/video.ts | 51 + server/server/helpers/webtorrent.ts | 263 +++ server/server/helpers/youtube-dl/index.ts | 3 + .../helpers/youtube-dl/youtube-dl-cli.ts | 260 +++ .../youtube-dl/youtube-dl-info-builder.ts | 198 ++ .../helpers/youtube-dl/youtube-dl-wrapper.ts | 156 ++ .../server/initializers/checker-after-init.ts | 326 +++ .../initializers/checker-before-init.ts | 159 ++ server/server/initializers/config.ts | 699 ++++++ server/server/initializers/constants.ts | 1411 +++++++++++ server/server/initializers/database.ts | 234 ++ server/server/initializers/installer.ts | 200 ++ .../migrations/0505-user-last-login-date.ts | 0 .../migrations/0510-video-file-metadata.ts | 0 .../0515-video-abuse-reason-timestamps.ts | 0 .../migrations/0520-abuses-split.ts | 0 .../migrations/0525-abuse-messages.ts | 0 .../0530-playlist-multiple-video.ts | 46 + .../migrations/0535-video-live.ts | 0 .../migrations/0540-video-file-infohash.ts | 0 .../migrations/0545-video-live-save-replay.ts | 0 .../migrations/0550-actor-follow-cleanup.ts | 0 .../migrations/0555-actor-follow-url.ts | 0 .../migrations/0560-user-feed-token.ts | 51 + .../migrations/0565-actor-follow-local-url.ts | 0 .../migrations/0570-permanent-live.ts | 0 .../migrations/0575-duplicate-thumbnail.ts | 0 .../migrations/0580-caption-filename.ts | 0 .../migrations/0585-video-file-names.ts | 0 .../initializers/migrations/0590-trackers.ts | 0 .../migrations/0595-remote-url.ts | 0 .../migrations/0600-duplicate-video-files.ts | 0 .../migrations/0605-actor-missing-keys.ts | 33 + .../migrations/0610-views-index copy.ts | 0 .../migrations/0612-captions-unique.ts | 0 ...5-latest-versions-notification-settings.ts | 0 .../0620-latest-versions-application.ts | 0 .../0625-latest-versions-notification.ts | 0 .../initializers/migrations/0630-banner.ts | 0 .../migrations/0635-actor-image-size.ts | 0 .../migrations/0640-unique-keys.ts | 0 .../0645-actor-remote-creation-date.ts | 0 .../migrations/0650-actor-custom-pages.ts | 0 .../0655-streaming-playlist-filenames.ts | 0 .../migrations/0660-object-storage.ts | 56 + .../0665-no-account-warning-modal.ts | 0 .../migrations/0670-pending-job-default.ts | 0 .../migrations/0675-p2p-enabled.ts | 0 .../migrations/0680-files-storage-default.ts | 0 .../migrations/0685-multiple-actor-images.ts | 0 .../migrations/0690-live-latency-mode.ts | 35 + .../migrations/0695-remove-remote-rates.ts | 0 .../0700-edition-finished-notification.ts | 0 .../migrations/0705-local-video-viewers.ts | 0 .../migrations/0710-live-sessions.ts | 0 .../migrations/0715-video-source.ts | 0 .../0720-session-ending-processed.ts | 0 .../migrations/0725-node-version.ts | 0 .../migrations/0730-video-channel-sync.ts | 0 ...5-video-channel-sync-import-foreign-key.ts | 0 .../migrations/0740-fix-old-enums.ts | 0 .../initializers/migrations/0745-user-otp.ts | 0 .../migrations/0750-user-registration.ts | 0 .../migrations/0755-unique-viewer-url.ts | 0 .../0760-video-live-replay-setting.ts | 0 .../migrations/0765-remote-transcoding.ts | 0 .../0770-actor-preferred-username.ts | 0 .../0775-add-user-is-email-public.ts | 0 .../0780-notification-registration.ts | 0 .../0785-video-password-protection.ts | 0 .../migrations/0790-thumbnail-disk.ts | 0 .../migrations/0795-duplicate-runner-name.ts | 0 .../migrations/0800-video-replace-file.ts | 0 server/server/initializers/migrator.ts | 106 + server/server/lib/activitypub/activity.ts | 74 + server/server/lib/activitypub/actors/get.ts | 149 ++ server/server/lib/activitypub/actors/image.ts | 112 + server/server/lib/activitypub/actors/index.ts | 6 + server/server/lib/activitypub/actors/keys.ts | 16 + .../server/lib/activitypub/actors/refresh.ts | 81 + .../lib/activitypub/actors/shared/creator.ts | 158 ++ .../lib/activitypub/actors/shared/index.ts | 3 + .../shared/object-to-model-attributes.ts | 83 + .../actors/shared/url-to-object.ts | 56 + .../server/lib/activitypub/actors/updater.ts | 91 + .../lib/activitypub/actors/webfinger.ts | 67 + server/server/lib/activitypub/audience.ts | 34 + server/server/lib/activitypub/cache-file.ts | 82 + server/server/lib/activitypub/collection.ts | 63 + server/server/lib/activitypub/context.ts | 10 + server/server/lib/activitypub/crawl.ts | 58 + server/server/lib/activitypub/follow.ts | 51 + .../server/lib/activitypub/inbox-manager.ts | 47 + .../lib/activitypub/local-video-viewer.ts | 44 + server/server/lib/activitypub/outbox.ts | 24 + .../activitypub/playlists/create-update.ts | 157 ++ .../server/lib/activitypub/playlists/get.ts | 35 + .../server/lib/activitypub/playlists/index.ts | 3 + .../lib/activitypub/playlists/refresh.ts | 53 + .../lib/activitypub/playlists/shared/index.ts | 2 + .../shared/object-to-model-attributes.ts | 40 + .../playlists/shared/url-to-object.ts | 47 + .../server/lib/activitypub/process/index.ts | 1 + .../lib/activitypub/process/process-accept.ts | 32 + .../activitypub/process/process-announce.ts | 75 + .../lib/activitypub/process/process-create.ts | 170 ++ .../lib/activitypub/process/process-delete.ts | 153 ++ .../activitypub/process/process-dislike.ts | 58 + .../lib/activitypub/process/process-flag.ts | 103 + .../lib/activitypub/process/process-follow.ts | 156 ++ .../lib/activitypub/process/process-like.ts | 60 + .../lib/activitypub/process/process-reject.ts | 33 + .../lib/activitypub/process/process-undo.ts | 183 ++ .../lib/activitypub/process/process-update.ts | 124 + .../lib/activitypub/process/process-view.ts | 42 + .../server/lib/activitypub/process/process.ts | 92 + server/server/lib/activitypub/send/http.ts | 50 + server/server/lib/activitypub/send/index.ts | 10 + .../lib/activitypub/send/send-accept.ts | 47 + .../lib/activitypub/send/send-announce.ts | 58 + .../lib/activitypub/send/send-create.ts | 226 ++ .../lib/activitypub/send/send-delete.ts | 158 ++ .../lib/activitypub/send/send-dislike.ts | 40 + .../server/lib/activitypub/send/send-flag.ts | 42 + .../lib/activitypub/send/send-follow.ts | 37 + .../server/lib/activitypub/send/send-like.ts | 40 + .../lib/activitypub/send/send-reject.ts | 39 + .../server/lib/activitypub/send/send-undo.ts | 172 ++ .../lib/activitypub/send/send-update.ts | 157 ++ .../server/lib/activitypub/send/send-view.ts | 62 + .../activitypub/send/shared/audience-utils.ts | 74 + .../lib/activitypub/send/shared/index.ts | 2 + .../lib/activitypub/send/shared/send-utils.ts | 298 +++ server/server/lib/activitypub/share.ts | 120 + server/server/lib/activitypub/url.ts | 177 ++ .../server/lib/activitypub/video-comments.ts | 204 ++ server/server/lib/activitypub/video-rates.ts | 59 + .../server/lib/activitypub/videos/federate.ts | 29 + server/server/lib/activitypub/videos/get.ts | 116 + server/server/lib/activitypub/videos/index.ts | 4 + .../server/lib/activitypub/videos/refresh.ts | 68 + .../videos/shared/abstract-builder.ts | 190 ++ .../lib/activitypub/videos/shared/creator.ts | 65 + .../lib/activitypub/videos/shared/index.ts | 6 + .../shared/object-to-model-attributes.ts | 286 +++ .../lib/activitypub/videos/shared/trackers.ts | 43 + .../videos/shared/url-to-object.ts | 25 + .../videos/shared/video-sync-attributes.ts | 101 + .../server/lib/activitypub/videos/updater.ts | 186 ++ .../server/lib/actor-follow-health-cache.ts | 86 + server/server/lib/actor-image.ts | 14 + server/server/lib/auth/external-auth.ts | 230 ++ server/server/lib/auth/oauth-model.ts | 294 +++ server/server/lib/auth/oauth.ts | 230 ++ server/server/lib/auth/tokens-cache.ts | 52 + server/server/lib/blocklist.ts | 62 + server/server/lib/client-html.ts | 619 +++++ server/server/lib/emailer.ts | 283 +++ .../lib/emails/abuse-new-message/html.pug | 0 .../lib/emails/abuse-state-change/html.pug | 0 .../lib/emails/account-abuse-new/html.pug | 0 .../{ => server}/lib/emails/common/base.pug | 0 .../lib/emails/common/greetings.pug | 0 .../{ => server}/lib/emails/common/html.pug | 0 .../{ => server}/lib/emails/common/mixins.pug | 0 .../lib/emails/contact-form/html.pug | 0 .../lib/emails/follower-on-channel/html.pug | 0 .../lib/emails/password-create/html.pug | 0 .../lib/emails/password-reset/html.pug | 0 .../lib/emails/peertube-version-new/html.pug | 0 .../lib/emails/plugin-version-new/html.pug | 0 .../lib/emails/user-registered/html.pug | 0 .../html.pug | 0 .../html.pug | 0 .../emails/user-registration-request/html.pug | 0 .../lib/emails/verify-email/html.pug | 0 .../lib/emails/video-abuse-new/html.pug | 0 .../emails/video-auto-blacklist-new/html.pug | 0 .../emails/video-comment-abuse-new/html.pug | 0 .../lib/emails/video-comment-mention/html.pug | 0 .../lib/emails/video-comment-new/html.pug | 0 .../avatar-permanent-file-cache.ts | 27 + server/server/lib/files-cache/index.ts | 6 + .../shared/abstract-permanent-file-cache.ts | 132 ++ .../shared/abstract-simple-file-cache.ts | 30 + server/server/lib/files-cache/shared/index.ts | 2 + .../video-captions-simple-file-cache.ts | 61 + .../video-miniature-permanent-file-cache.ts | 28 + .../video-previews-simple-file-cache.ts | 58 + .../video-storyboards-simple-file-cache.ts | 53 + .../video-torrents-simple-file-cache.ts | 70 + server/server/lib/hls.ts | 286 +++ server/server/lib/internal-event-emitter.ts | 35 + .../job-queue/handlers/activitypub-cleaner.ts | 202 ++ .../job-queue/handlers/activitypub-follow.ts | 82 + .../handlers/activitypub-http-broadcast.ts | 50 + .../handlers/activitypub-http-fetcher.ts | 41 + .../handlers/activitypub-http-unicast.ts | 39 + .../handlers/activitypub-refresher.ts | 60 + .../lib/job-queue/handlers/actor-keys.ts | 20 + .../handlers/after-video-channel-import.ts | 37 + server/server/lib/job-queue/handlers/email.ts | 17 + .../lib/job-queue/handlers/federate-video.ts | 28 + .../job-queue/handlers/generate-storyboard.ts | 163 ++ .../handlers/manage-video-torrent.ts | 110 + .../handlers/move-to-object-storage.ts | 159 ++ .../server/lib/job-queue/handlers/notify.ts | 27 + .../handlers/transcoding-job-builder.ts | 48 + .../handlers/video-channel-import.ts | 43 + .../job-queue/handlers/video-file-import.ts | 84 + .../lib/job-queue/handlers/video-import.ts | 356 +++ .../job-queue/handlers/video-live-ending.ts | 280 +++ .../job-queue/handlers/video-redundancy.ts | 17 + .../handlers/video-studio-edition.ts | 180 ++ .../job-queue/handlers/video-transcoding.ts | 150 ++ .../job-queue/handlers/video-views-stats.ts | 57 + server/server/lib/job-queue/index.ts | 1 + server/server/lib/job-queue/job-queue.ts | 540 +++++ server/server/lib/live/index.ts | 4 + server/server/lib/live/live-manager.ts | 557 +++++ .../{ => server}/lib/live/live-quota-store.ts | 0 .../server/lib/live/live-segment-sha-store.ts | 96 + server/server/lib/live/live-utils.ts | 100 + server/server/lib/live/shared/index.ts | 1 + .../server/lib/live/shared/muxing-session.ts | 519 +++++ .../abstract-transcoding-wrapper.ts | 111 + .../ffmpeg-transcoding-wrapper.ts | 107 + .../live/shared/transcoding-wrapper/index.ts | 3 + .../remote-transcoding-wrapper.ts | 21 + server/server/lib/local-actor.ts | 101 + server/server/lib/model-loaders/actor.ts | 16 + server/server/lib/model-loaders/index.ts | 2 + server/server/lib/model-loaders/video.ts | 66 + server/server/lib/moderation.ts | 257 +++ server/server/lib/notifier/index.ts | 1 + server/server/lib/notifier/notifier.ts | 292 +++ .../abuse/abstract-new-abuse-message.ts | 73 + .../abuse/abuse-state-change-for-reporter.ts | 74 + .../server/lib/notifier/shared/abuse/index.ts | 4 + .../shared/abuse/new-abuse-for-moderators.ts | 119 + .../abuse/new-abuse-message-for-moderators.ts | 32 + .../abuse/new-abuse-message-for-reporter.ts | 36 + .../lib/notifier/shared/blacklist/index.ts | 3 + .../new-auto-blacklist-for-moderators.ts | 65 + .../blacklist/new-blacklist-for-owner.ts | 63 + .../shared/blacklist/unblacklist-for-owner.ts | 55 + .../shared/comment/comment-mention.ts | 111 + .../lib/notifier/shared/comment/index.ts | 2 + .../comment/new-comment-for-video-owner.ts | 76 + .../shared/common/abstract-notification.ts | 23 + .../lib/notifier/shared/common/index.ts | 1 + .../shared/follow/auto-follow-for-instance.ts | 51 + .../shared/follow/follow-for-instance.ts | 68 + .../notifier/shared/follow/follow-for-user.ts | 82 + .../lib/notifier/shared/follow/index.ts | 3 + server/server/lib/notifier/shared/index.ts | 7 + .../direct-registration-for-moderators.ts | 49 + .../lib/notifier/shared/instance/index.ts | 4 + .../new-peertube-version-for-admins.ts | 54 + .../instance/new-plugin-version-for-admins.ts | 58 + .../registration-request-for-moderators.ts | 48 + .../abstract-owned-video-publication.ts | 57 + .../import-finished-for-owner.ts | 97 + .../shared/video-publication/index.ts | 6 + .../new-video-for-subscribers.ts | 61 + ...wned-publication-after-auto-unblacklist.ts | 11 + ...owned-publication-after-schedule-update.ts | 10 + .../owned-publication-after-transcoding.ts | 9 + .../studio-edition-finished-for-owner.ts | 57 + server/server/lib/object-storage/index.ts | 5 + server/server/lib/object-storage/keys.ts | 20 + .../lib/object-storage/pre-signed-urls.ts | 50 + server/server/lib/object-storage/proxy.ts | 98 + .../lib/object-storage/shared/client.ts | 78 + .../server/lib/object-storage/shared/index.ts | 3 + .../lib/object-storage/shared/logger.ts | 7 + .../shared/object-storage-helpers.ts | 345 +++ server/server/lib/object-storage/urls.ts | 63 + server/server/lib/object-storage/videos.ts | 197 ++ .../bittorrent-tracker-observers-builder.ts | 0 .../lib/opentelemetry/metric-helpers/index.ts | 7 + .../job-queue-observers-builder.ts | 24 + .../metric-helpers/lives-observers-builder.ts | 21 + .../nodejs-observers-builder.ts | 202 ++ .../metric-helpers/playback-metrics.ts | 85 + .../metric-helpers/stats-observers-builder.ts | 186 ++ .../viewers-observers-builder.ts | 24 + server/server/lib/opentelemetry/metrics.ts | 123 + server/server/lib/opentelemetry/tracing.ts | 140 ++ server/server/lib/paths.ts | 92 + server/server/lib/peertube-socket.ts | 129 ++ server/server/lib/plugins/hooks.ts | 35 + .../lib/plugins/plugin-helpers-builder.ts | 262 +++ server/server/lib/plugins/plugin-index.ts | 85 + server/server/lib/plugins/plugin-manager.ts | 674 ++++++ server/server/lib/plugins/register-helpers.ts | 340 +++ server/server/lib/plugins/theme-utils.ts | 24 + .../plugins/video-constant-manager-factory.ts | 139 ++ server/server/lib/plugins/yarn.ts | 73 + server/server/lib/redis.ts | 465 ++++ server/server/lib/redundancy.ts | 59 + server/server/lib/runners/index.ts | 3 + .../job-handlers/abstract-job-handler.ts | 270 +++ .../abstract-vod-transcoding-job-handler.ts | 72 + .../server/lib/runners/job-handlers/index.ts | 7 + .../live-rtmp-hls-transcoding-job-handler.ts | 173 ++ .../job-handlers/runner-job-handlers.ts | 20 + .../lib/runners/job-handlers/shared/index.ts | 1 + .../job-handlers/shared/vod-helpers.ts | 44 + .../video-studio-transcoding-job-handler.ts | 158 ++ ...vod-audio-merge-transcoding-job-handler.ts | 97 + .../vod-hls-transcoding-job-handler.ts | 114 + .../vod-web-video-transcoding-job-handler.ts | 84 + server/server/lib/runners/runner-urls.ts | 13 + server/server/lib/runners/runner.ts | 49 + .../lib/schedulers/abstract-scheduler.ts | 35 + .../lib/schedulers/actor-follow-scheduler.ts | 54 + .../schedulers/auto-follow-index-instances.ts | 75 + .../lib/schedulers/geo-ip-update-scheduler.ts | 22 + .../peertube-version-check-scheduler.ts | 55 + .../lib/schedulers/plugins-check-scheduler.ts | 74 + ...ve-dangling-resumable-uploads-scheduler.ts | 40 + .../remove-old-history-scheduler.ts | 31 + .../schedulers/remove-old-views-scheduler.ts | 31 + .../runner-job-watch-dog-scheduler.ts | 42 + .../lib/schedulers/update-videos-scheduler.ts | 89 + .../video-channel-sync-latest-scheduler.ts | 50 + .../video-views-buffer-scheduler.ts | 52 + .../schedulers/videos-redundancy-scheduler.ts | 375 +++ .../schedulers/youtube-dl-update-scheduler.ts | 22 + server/server/lib/search.ts | 49 + server/server/lib/server-config-manager.ts | 390 ++++ server/server/lib/signup.ts | 74 + server/server/lib/stat-manager.ts | 182 ++ server/server/lib/sync-channel.ts | 111 + server/server/lib/thumbnail.ts | 327 +++ server/server/lib/timeserie.ts | 61 + .../lib/transcoding/create-transcoding-job.ts | 37 + .../default-transcoding-profiles.ts | 143 ++ .../lib/transcoding/ended-transcoding.ts | 18 + .../server/lib/transcoding/hls-transcoding.ts | 180 ++ .../lib/transcoding/shared/ffmpeg-builder.ts | 18 + server/server/lib/transcoding/shared/index.ts | 2 + .../job-builders/abstract-job-builder.ts | 21 + .../transcoding/shared/job-builders/index.ts | 2 + .../transcoding-job-queue-builder.ts | 322 +++ .../transcoding-runner-job-builder.ts | 200 ++ .../lib/transcoding/transcoding-priority.ts | 24 + .../transcoding-quick-transcode.ts | 12 + .../transcoding/transcoding-resolutions.ts | 73 + .../server/lib/transcoding/web-transcoding.ts | 264 +++ server/server/lib/uploadx.ts | 37 + server/server/lib/user.ts | 308 +++ server/server/lib/video-blacklist.ts | 144 ++ server/server/lib/video-channel.ts | 50 + server/server/lib/video-comment.ts | 115 + server/server/lib/video-file.ts | 144 ++ server/server/lib/video-path-manager.ts | 180 ++ server/server/lib/video-playlist.ts | 29 + server/server/lib/video-pre-import.ts | 331 +++ server/server/lib/video-privacy.ts | 133 ++ server/server/lib/video-state.ts | 154 ++ server/server/lib/video-studio.ts | 130 ++ server/server/lib/video-tokens-manager.ts | 78 + server/server/lib/video-urls.ts | 31 + server/server/lib/video.ts | 201 ++ server/server/lib/views/shared/index.ts | 3 + .../lib/views/shared/video-viewer-counters.ts | 197 ++ .../lib/views/shared/video-viewer-stats.ts | 196 ++ server/server/lib/views/shared/video-views.ts | 70 + .../server/lib/views/video-views-manager.ts | 100 + server/server/lib/worker/parent-process.ts | 77 + .../lib/worker/workers/http-broadcast.ts | 28 + .../lib/worker/workers/image-downloader.ts | 31 + .../lib/worker/workers/image-processor.ts | 3 + server/server/middlewares/activitypub.ts | 156 ++ server/server/middlewares/async.ts | 44 + server/server/middlewares/auth.ts | 112 + server/server/middlewares/cache/cache.ts | 38 + server/server/middlewares/cache/index.ts | 1 + .../middlewares/cache/shared/api-cache.ts | 315 +++ .../server/middlewares/cache/shared/index.ts | 1 + server/server/middlewares/csp.ts | 40 + server/{ => server}/middlewares/dnt.ts | 0 server/{ => server}/middlewares/doc.ts | 0 server/server/middlewares/error.ts | 63 + server/server/middlewares/index.ts | 15 + server/server/middlewares/pagination.ts | 19 + server/server/middlewares/rate-limiter.ts | 59 + server/{ => server}/middlewares/robots.ts | 0 server/server/middlewares/servers.ts | 29 + server/{ => server}/middlewares/sort.ts | 0 server/server/middlewares/user-right.ts | 26 + server/server/middlewares/validators/abuse.ts | 254 ++ .../server/middlewares/validators/account.ts | 35 + .../validators/activitypub/activity.ts | 29 + .../validators/activitypub/index.ts | 3 + .../validators/activitypub/pagination.ts | 25 + .../validators/activitypub/signature.ts | 39 + .../middlewares/validators/actor-image.ts | 27 + .../middlewares/validators/blocklist.ts | 179 ++ server/server/middlewares/validators/bulk.ts | 37 + .../server/middlewares/validators/config.ts | 193 ++ .../middlewares/validators/express.ts | 0 server/server/middlewares/validators/feeds.ts | 178 ++ .../server/middlewares/validators/follows.ts | 157 ++ server/server/middlewares/validators/index.ts | 32 + server/server/middlewares/validators/jobs.ts | 29 + server/server/middlewares/validators/logs.ts | 93 + .../server/middlewares/validators/metrics.ts | 60 + .../validators/object-storage-proxy.ts | 20 + .../server/middlewares/validators/oembed.ts | 157 ++ .../middlewares/validators/pagination.ts | 30 + .../server/middlewares/validators/plugins.ts | 216 ++ .../middlewares/validators/redundancy.ts | 198 ++ .../middlewares/validators/runners/index.ts | 3 + .../validators/runners/job-files.ts | 60 + .../middlewares/validators/runners/jobs.ts | 217 ++ .../validators/runners/registration-token.ts | 37 + .../middlewares/validators/runners/runners.ts | 104 + .../server/middlewares/validators/search.ts | 112 + .../server/middlewares/validators/server.ts | 75 + .../middlewares/validators/shared/abuses.ts | 26 + .../middlewares/validators/shared/accounts.ts | 66 + .../middlewares/validators/shared/index.ts | 14 + .../validators/shared/user-registrations.ts | 60 + .../middlewares/validators/shared/users.ts | 63 + .../middlewares/validators/shared/utils.ts | 69 + .../validators/shared/video-blacklists.ts | 24 + .../validators/shared/video-captions.ts | 25 + .../validators/shared/video-channel-syncs.ts | 24 + .../validators/shared/video-channels.ts | 36 + .../validators/shared/video-comments.ts | 80 + .../validators/shared/video-imports.ts | 22 + .../validators/shared/video-ownerships.ts | 25 + .../validators/shared/video-passwords.ts | 80 + .../validators/shared/video-playlists.ts | 39 + .../middlewares/validators/shared/videos.ts | 311 +++ server/server/middlewares/validators/sort.ts | 66 + .../server/middlewares/validators/static.ts | 184 ++ .../server/middlewares/validators/themes.ts | 46 + .../middlewares/validators/two-factor.ts | 81 + .../validators/user-email-verification.ts | 94 + .../middlewares/validators/user-history.ts | 47 + .../validators/user-notifications.ts | 71 + .../validators/user-registrations.ts | 208 ++ .../validators/user-subscriptions.ts | 110 + server/server/middlewares/validators/users.ts | 489 ++++ .../middlewares/validators/videos/index.ts | 19 + .../validators/videos/shared/index.ts | 2 + .../validators/videos/shared/upload.ts | 39 + .../videos/shared/video-validators.ts | 105 + .../validators/videos/video-blacklist.ts | 87 + .../validators/videos/video-captions.ts | 83 + .../validators/videos/video-channel-sync.ts | 56 + .../validators/videos/video-channels.ts | 193 ++ .../validators/videos/video-comments.ts | 249 ++ .../validators/videos/video-files.ts | 163 ++ .../validators/videos/video-imports.ts | 204 ++ .../validators/videos/video-live.ts | 342 +++ .../videos/video-ownership-changes.ts | 107 + .../validators/videos/video-passwords.ts | 77 + .../validators/videos/video-playlists.ts | 425 ++++ .../validators/videos/video-rates.ts | 71 + .../validators/videos/video-shares.ts | 35 + .../validators/videos/video-source.ts | 130 ++ .../validators/videos/video-stats.ts | 108 + .../validators/videos/video-studio.ts | 105 + .../validators/videos/video-token.ts | 23 + .../validators/videos/video-transcoding.ts | 61 + .../validators/videos/video-view.ts | 61 + .../middlewares/validators/videos/videos.ts | 575 +++++ .../middlewares/validators/webfinger.ts | 37 + server/server/models/abuse/abuse-message.ts | 114 + server/server/models/abuse/abuse.ts | 631 +++++ .../models/abuse/sql/abuse-query-builder.ts | 167 ++ server/server/models/abuse/video-abuse.ts | 64 + .../models/abuse/video-comment-abuse.ts | 48 + .../models/account/account-blocklist.ts | 236 ++ .../models/account/account-video-rate.ts | 259 +++ server/server/models/account/account.ts | 468 ++++ .../models/account/actor-custom-page.ts | 69 + server/server/models/actor/actor-follow.ts | 661 ++++++ server/server/models/actor/actor-image.ts | 170 ++ server/server/models/actor/actor.ts | 690 ++++++ .../instance-list-followers-query-builder.ts | 69 + .../instance-list-following-query-builder.ts | 69 + .../shared/actor-follow-table-attributes.ts | 28 + .../instance-list-follows-query-builder.ts | 97 + .../server/models/application/application.ts | 79 + server/server/models/oauth/oauth-client.ts | 63 + server/server/models/oauth/oauth-token.ts | 222 ++ .../models/redundancy/video-redundancy.ts | 793 +++++++ server/server/models/runner/runner-job.ts | 366 +++ .../runner/runner-registration-token.ts | 103 + server/server/models/runner/runner.ts | 124 + server/server/models/server/plugin.ts | 317 +++ .../server/models/server/server-blocklist.ts | 190 ++ server/server/models/server/server.ts | 104 + server/server/models/server/tracker.ts | 74 + server/server/models/server/video-tracker.ts | 31 + .../models/shared/abstract-run-query.ts | 0 server/server/models/shared/index.ts | 8 + server/server/models/shared/model-builder.ts | 118 + server/server/models/shared/model-cache.ts | 90 + server/server/models/shared/query.ts | 84 + .../models/shared/sequelize-helpers.ts | 0 server/{ => server}/models/shared/sort.ts | 0 server/server/models/shared/sql.ts | 71 + server/{ => server}/models/shared/update.ts | 0 .../user-notitication-list-query-builder.ts | 273 +++ .../models/user/user-notification-setting.ts | 232 ++ .../server/models/user/user-notification.ts | 534 +++++ .../server/models/user/user-registration.ts | 259 +++ .../server/models/user/user-video-history.ts | 113 + server/server/models/user/user.ts | 989 ++++++++ server/server/models/video/formatter/index.ts | 2 + .../models/video/formatter/shared/index.ts | 1 + .../formatter/shared/video-format-utils.ts | 7 + .../formatter/video-activity-pub-format.ts | 296 +++ .../video/formatter/video-api-format.ts | 305 +++ .../models/video/schedule-video-update.ts | 95 + .../video-comment-list-query-builder.ts | 400 ++++ .../comment/video-comment-table-attributes.ts | 43 + server/server/models/video/sql/video/index.ts | 3 + .../shared/abstract-video-query-builder.ts | 340 +++ .../video/shared/video-file-query-builder.ts | 75 + .../sql/video/shared/video-model-builder.ts | 407 ++++ .../video/shared/video-table-attributes.ts | 0 .../video/video-model-get-query-builder.ts | 189 ++ .../sql/video/videos-id-list-query-builder.ts | 728 ++++++ .../video/videos-model-list-query-builder.ts | 103 + server/server/models/video/storyboard.ts | 169 ++ server/server/models/video/tag.ts | 86 + server/server/models/video/thumbnail.ts | 208 ++ server/server/models/video/video-blacklist.ts | 134 ++ server/server/models/video/video-caption.ts | 253 ++ .../models/video/video-change-ownership.ts | 137 ++ .../server/models/video/video-channel-sync.ts | 176 ++ server/server/models/video/video-channel.ts | 859 +++++++ server/server/models/video/video-comment.ts | 646 ++++++ server/server/models/video/video-file.ts | 635 +++++ server/server/models/video/video-import.ts | 267 +++ server/server/models/video/video-job-info.ts | 121 + .../models/video/video-live-replay-setting.ts | 42 + .../server/models/video/video-live-session.ts | 217 ++ server/server/models/video/video-live.ts | 184 ++ server/server/models/video/video-password.ts | 137 ++ .../models/video/video-playlist-element.ts | 375 +++ server/server/models/video/video-playlist.ts | 733 ++++++ server/server/models/video/video-share.ts | 216 ++ server/server/models/video/video-source.ts | 56 + .../models/video/video-streaming-playlist.ts | 332 +++ server/server/models/video/video-tag.ts | 31 + server/server/models/video/video.ts | 2055 +++++++++++++++++ .../view/local-video-viewer-watch-section.ts | 63 + .../server/models/view/local-video-viewer.ts | 374 +++ server/server/models/view/video-view.ts | 67 + .../static/dnt-policy/dnt-policy-1.0.txt | 0 .../types/activitypub-processor.model.ts | 9 + server/{ => server}/types/express-handler.ts | 0 server/server/types/express.d.ts | 222 ++ server/server/types/index.ts | 3 + server/{ => server}/types/lib.d.ts | 0 .../types/models/abuse/abuse-message.ts | 20 + server/server/types/models/abuse/abuse.ts | 114 + server/server/types/models/abuse/index.ts | 2 + .../types/models/account/account-blocklist.ts | 27 + server/server/types/models/account/account.ts | 108 + .../types/models/account/actor-custom-page.ts | 3 + server/server/types/models/account/index.ts | 3 + .../server/types/models/actor/actor-follow.ts | 65 + .../server/types/models/actor/actor-image.ts | 12 + server/server/types/models/actor/actor.ts | 170 ++ server/server/types/models/actor/index.ts | 3 + .../types/models/application/application.ts | 5 + .../server/types/models/application/index.ts | 1 + server/server/types/models/index.ts | 8 + server/server/types/models/oauth/index.ts | 2 + .../server/types/models/oauth/oauth-client.ts | 3 + .../server/types/models/oauth/oauth-token.ts | 14 + server/server/types/models/runners/index.ts | 3 + .../server/types/models/runners/runner-job.ts | 20 + .../runners/runner-registration-token.ts | 5 + server/server/types/models/runners/runner.ts | 5 + server/server/types/models/server/index.ts | 3 + server/server/types/models/server/plugin.ts | 11 + .../types/models/server/server-blocklist.ts | 26 + server/server/types/models/server/server.ts | 26 + server/server/types/models/server/tracker.ts | 7 + server/server/types/models/user/index.ts | 5 + .../models/user/user-notification-setting.ts | 9 + .../types/models/user/user-notification.ts | 122 + .../types/models/user/user-registration.ts | 15 + .../types/models/user/user-video-history.ts | 5 + server/server/types/models/user/user.ts | 89 + server/server/types/models/video/index.ts | 26 + .../video/local-video-viewer-watch-section.ts | 5 + .../types/models/video/local-video-viewer.ts | 19 + .../models/video/schedule-video-update.ts | 11 + .../server/types/models/video/storyboard.ts | 15 + server/server/types/models/video/tag.ts | 3 + server/server/types/models/video/thumbnail.ts | 15 + .../types/models/video/video-blacklist.ts | 30 + .../types/models/video/video-caption.ts | 27 + .../models/video/video-change-ownership.ts | 26 + .../types/models/video/video-channel-sync.ts | 17 + .../types/models/video/video-channels.ts | 153 ++ .../types/models/video/video-comment.ts | 71 + .../server/types/models/video/video-file.ts | 43 + .../server/types/models/video/video-import.ts | 36 + .../models/video/video-live-replay-setting.ts | 3 + .../types/models/video/video-live-session.ts | 17 + .../server/types/models/video/video-live.ts | 22 + .../types/models/video/video-password.ts | 3 + .../models/video/video-playlist-element.ts | 39 + .../types/models/video/video-playlist.ts | 104 + .../server/types/models/video/video-rate.ts | 27 + .../types/models/video/video-redundancy.ts | 43 + .../server/types/models/video/video-share.ts | 19 + .../server/types/models/video/video-source.ts | 3 + .../models/video/video-streaming-playlist.ts | 43 + server/server/types/models/video/video.ts | 225 ++ server/server/types/plugins/index.ts | 4 + .../types/plugins/plugin-library.model.ts | 7 + .../plugins/register-server-auth.model.ts | 72 + .../plugins/register-server-option.model.ts | 171 ++ .../register-server-websocket-route.model.ts | 0 server/server/types/sequelize.ts | 19 + server/tests/api/activitypub/cleaner.ts | 342 --- server/tests/api/activitypub/client.ts | 136 -- server/tests/api/activitypub/fetch.ts | 82 - server/tests/api/activitypub/helpers.ts | 167 -- server/tests/api/activitypub/index.ts | 6 - server/tests/api/activitypub/refresher.ts | 157 -- server/tests/api/activitypub/security.ts | 321 --- server/tests/api/check-params/abuses.ts | 438 ---- server/tests/api/check-params/accounts.ts | 43 - server/tests/api/check-params/blocklist.ts | 556 ----- server/tests/api/check-params/bulk.ts | 80 - .../api/check-params/channel-import-videos.ts | 209 -- server/tests/api/check-params/config.ts | 428 ---- server/tests/api/check-params/contact-form.ts | 86 - server/tests/api/check-params/custom-pages.ts | 79 - server/tests/api/check-params/debug.ts | 61 - server/tests/api/check-params/follows.ts | 369 --- server/tests/api/check-params/index.ts | 45 - server/tests/api/check-params/jobs.ts | 125 - server/tests/api/check-params/live.ts | 589 ----- server/tests/api/check-params/logs.ts | 157 -- server/tests/api/check-params/metrics.ts | 208 -- server/tests/api/check-params/my-user.ts | 491 ---- server/tests/api/check-params/plugins.ts | 490 ---- server/tests/api/check-params/redundancy.ts | 240 -- .../tests/api/check-params/registrations.ts | 446 ---- server/tests/api/check-params/runners.ts | 910 -------- server/tests/api/check-params/search.ts | 272 --- server/tests/api/check-params/services.ts | 195 -- server/tests/api/check-params/transcoding.ts | 112 - server/tests/api/check-params/two-factor.ts | 288 --- server/tests/api/check-params/upload-quota.ts | 134 -- .../api/check-params/user-notifications.ts | 290 --- .../api/check-params/user-subscriptions.ts | 298 --- server/tests/api/check-params/users-admin.ts | 456 ---- server/tests/api/check-params/users-emails.ts | 116 - .../tests/api/check-params/video-blacklist.ts | 292 --- .../tests/api/check-params/video-captions.ts | 307 --- .../api/check-params/video-channel-syncs.ts | 318 --- .../tests/api/check-params/video-channels.ts | 378 --- .../tests/api/check-params/video-comments.ts | 484 ---- server/tests/api/check-params/video-files.ts | 195 -- .../tests/api/check-params/video-imports.ts | 431 ---- .../tests/api/check-params/video-passwords.ts | 609 ----- .../tests/api/check-params/video-playlists.ts | 695 ------ server/tests/api/check-params/video-source.ts | 154 -- .../api/check-params/video-storyboards.ts | 45 - server/tests/api/check-params/video-studio.ts | 388 ---- server/tests/api/check-params/video-token.ts | 70 - .../api/check-params/videos-common-filters.ts | 163 -- .../tests/api/check-params/videos-history.ts | 145 -- .../api/check-params/videos-overviews.ts | 31 - server/tests/api/check-params/videos.ts | 881 ------- server/tests/api/check-params/views.ts | 227 -- server/tests/api/index.ts | 13 - server/tests/api/live/index.ts | 7 - server/tests/api/live/live-constraints.ts | 237 -- server/tests/api/live/live-fast-restream.ts | 153 -- server/tests/api/live/live-permanent.ts | 204 -- server/tests/api/live/live-rtmps.ts | 143 -- server/tests/api/live/live-save-replay.ts | 570 ----- server/tests/api/live/live-socket-messages.ts | 186 -- server/tests/api/live/live.ts | 764 ------ server/tests/api/moderation/abuses.ts | 887 ------- .../api/moderation/blocklist-notification.ts | 231 -- server/tests/api/moderation/blocklist.ts | 902 -------- server/tests/api/moderation/index.ts | 4 - .../tests/api/moderation/video-blacklist.ts | 414 ---- .../api/notifications/admin-notifications.ts | 159 -- .../notifications/comments-notifications.ts | 305 --- server/tests/api/notifications/index.ts | 6 - .../notifications/moderation-notifications.ts | 609 ----- .../api/notifications/notifications-api.ts | 206 -- .../registrations-notifications.ts | 88 - .../api/notifications/user-notifications.ts | 574 ----- server/tests/api/object-storage/index.ts | 4 - server/tests/api/object-storage/live.ts | 311 --- .../tests/api/object-storage/video-imports.ts | 111 - .../video-static-file-privacy.ts | 570 ----- server/tests/api/object-storage/videos.ts | 438 ---- server/tests/api/redundancy/index.ts | 3 - .../tests/api/redundancy/manage-redundancy.ts | 324 --- .../api/redundancy/redundancy-constraints.ts | 191 -- server/tests/api/redundancy/redundancy.ts | 742 ------ server/tests/api/runners/index.ts | 5 - server/tests/api/runners/runner-common.ts | 743 ------ .../api/runners/runner-live-transcoding.ts | 330 --- server/tests/api/runners/runner-socket.ts | 120 - .../api/runners/runner-studio-transcoding.ts | 168 -- .../api/runners/runner-vod-transcoding.ts | 545 ----- server/tests/api/search/index.ts | 7 - .../search-activitypub-video-channels.ts | 255 -- .../search-activitypub-video-playlists.ts | 214 -- .../api/search/search-activitypub-videos.ts | 196 -- server/tests/api/search/search-channels.ts | 159 -- server/tests/api/search/search-index.ts | 432 ---- server/tests/api/search/search-playlists.ts | 180 -- server/tests/api/search/search-videos.ts | 568 ----- server/tests/api/server/auto-follows.ts | 189 -- server/tests/api/server/bulk.ts | 185 -- server/tests/api/server/config-defaults.ts | 288 --- server/tests/api/server/config.ts | 645 ------ server/tests/api/server/contact-form.ts | 101 - server/tests/api/server/email.ts | 371 --- server/tests/api/server/follow-constraints.ts | 321 --- server/tests/api/server/follows-moderation.ts | 364 --- server/tests/api/server/follows.ts | 641 ----- server/tests/api/server/handle-down.ts | 338 --- server/tests/api/server/homepage.ts | 81 - server/tests/api/server/index.ts | 22 - server/tests/api/server/jobs.ts | 128 - server/tests/api/server/logs.ts | 265 --- server/tests/api/server/no-client.ts | 24 - server/tests/api/server/open-telemetry.ts | 186 -- server/tests/api/server/plugins.ts | 409 ---- server/tests/api/server/proxy.ts | 171 -- server/tests/api/server/reverse-proxy.ts | 156 -- server/tests/api/server/services.ts | 137 -- server/tests/api/server/slow-follows.ts | 85 - server/tests/api/server/stats.ts | 279 --- server/tests/api/server/tracker.ts | 104 - server/tests/api/transcoding/audio-only.ts | 104 - .../api/transcoding/create-transcoding.ts | 266 --- server/tests/api/transcoding/hls.ts | 175 -- server/tests/api/transcoding/index.ts | 6 - server/tests/api/transcoding/transcoder.ts | 800 ------- .../transcoding/update-while-transcoding.ts | 160 -- server/tests/api/transcoding/video-studio.ts | 377 --- server/tests/api/users/index.ts | 8 - server/tests/api/users/oauth.ts | 197 -- server/tests/api/users/registrations.ts | 415 ---- server/tests/api/users/two-factor.ts | 200 -- server/tests/api/users/user-subscriptions.ts | 614 ----- server/tests/api/users/user-videos.ts | 219 -- .../api/users/users-email-verification.ts | 165 -- .../tests/api/users/users-multiple-servers.ts | 216 -- server/tests/api/users/users.ts | 529 ----- .../tests/api/videos/channel-import-videos.ts | 161 -- server/tests/api/videos/index.ts | 23 - server/tests/api/videos/multiple-servers.ts | 1099 --------- server/tests/api/videos/resumable-upload.ts | 310 --- server/tests/api/videos/single-server.ts | 460 ---- server/tests/api/videos/video-captions.ts | 188 -- .../api/videos/video-change-ownership.ts | 314 --- .../tests/api/videos/video-channel-syncs.ts | 320 --- server/tests/api/videos/video-channels.ts | 555 ----- server/tests/api/videos/video-comments.ts | 335 --- server/tests/api/videos/video-description.ts | 103 - server/tests/api/videos/video-files.ts | 202 -- server/tests/api/videos/video-imports.ts | 631 ----- server/tests/api/videos/video-nsfw.ts | 227 -- server/tests/api/videos/video-passwords.ts | 97 - .../api/videos/video-playlist-thumbnails.ts | 234 -- server/tests/api/videos/video-playlists.ts | 1208 ---------- server/tests/api/videos/video-privacy.ts | 287 --- .../tests/api/videos/video-schedule-update.ts | 155 -- server/tests/api/videos/video-source.ts | 447 ---- .../api/videos/video-static-file-privacy.ts | 600 ----- server/tests/api/videos/video-storyboard.ts | 213 -- .../tests/api/videos/videos-common-filters.ts | 489 ---- server/tests/api/videos/videos-history.ts | 224 -- server/tests/api/videos/videos-overview.ts | 129 -- server/tests/api/views/index.ts | 5 - server/tests/api/views/video-views-counter.ts | 153 -- .../api/views/video-views-overall-stats.ts | 368 --- .../api/views/video-views-retention-stats.ts | 53 - .../api/views/video-views-timeserie-stats.ts | 253 -- .../tests/api/views/videos-views-cleaner.ts | 98 - .../cli/create-generate-storyboard-job.ts | 120 - .../tests/cli/create-import-video-file-job.ts | 168 -- .../cli/create-move-video-storage-job.ts | 124 - server/tests/cli/peertube.ts | 331 --- server/tests/cli/plugins.ts | 76 - server/tests/cli/prune-storage.ts | 223 -- server/tests/cli/regenerate-thumbnails.ts | 122 - server/tests/cli/reset-password.ts | 26 - server/tests/cli/update-host.ts | 134 -- server/tests/client.ts | 556 ----- server/tests/external-plugins/akismet.ts | 160 -- server/tests/external-plugins/auth-ldap.ts | 117 - .../external-plugins/auto-block-videos.ts | 167 -- server/tests/external-plugins/auto-mute.ts | 216 -- server/tests/feeds/feeds.ts | 695 ------ server/tests/helpers/comment-model.ts | 24 - server/tests/helpers/core-utils.ts | 150 -- server/tests/helpers/crypto.ts | 33 - server/tests/helpers/dns.ts | 16 - server/tests/helpers/image.ts | 96 - server/tests/helpers/index.ts | 9 - server/tests/helpers/markdown.ts | 39 - server/tests/helpers/request.ts | 62 - server/tests/helpers/validator.ts | 32 - server/tests/helpers/version.ts | 36 - server/tests/index.ts | 10 - server/tests/lib/index.ts | 1 - .../lib/video-constant-registry-factory.ts | 154 -- server/tests/misc-endpoints.ts | 237 -- server/tests/peertube-runner/client-cli.ts | 70 - server/tests/peertube-runner/index.ts | 4 - .../tests/peertube-runner/live-transcoding.ts | 201 -- .../peertube-runner/studio-transcoding.ts | 124 - .../tests/peertube-runner/vod-transcoding.ts | 350 --- server/tests/plugins/action-hooks.ts | 298 --- server/tests/plugins/external-auth.ts | 436 ---- server/tests/plugins/filter-hooks.ts | 909 -------- server/tests/plugins/html-injection.ts | 73 - server/tests/plugins/id-and-pass-auth.ts | 242 -- server/tests/plugins/plugin-helpers.ts | 383 --- server/tests/plugins/plugin-router.ts | 105 - server/tests/plugins/plugin-storage.ts | 94 - server/tests/plugins/plugin-transcoding.ts | 279 --- server/tests/plugins/plugin-unloading.ts | 75 - server/tests/plugins/plugin-websocket.ts | 70 - server/tests/plugins/translations.ts | 74 - server/tests/plugins/video-constants.ts | 180 -- server/tests/shared/actors.ts | 69 - server/tests/shared/captions.ts | 21 - server/tests/shared/checks.ts | 174 -- server/tests/shared/directories.ts | 43 - server/tests/shared/generate.ts | 74 - server/tests/shared/index.ts | 19 - server/tests/shared/live.ts | 185 -- server/tests/shared/mock-servers/index.ts | 8 - server/tests/shared/mock-servers/mock-429.ts | 33 - .../tests/shared/mock-servers/mock-email.ts | 61 - server/tests/shared/mock-servers/mock-http.ts | 23 - .../mock-servers/mock-instances-index.ts | 46 - .../mock-joinpeertube-versions.ts | 34 - .../mock-servers/mock-object-storage.ts | 41 - .../mock-servers/mock-plugin-blocklist.ts | 36 - .../tests/shared/mock-servers/mock-proxy.ts | 24 - server/tests/shared/notifications.ts | 889 ------- .../tests/shared/peertube-runner-process.ts | 98 - server/tests/shared/plugins.ts | 18 - server/tests/shared/requests.ts | 12 - server/tests/shared/sql-command.ts | 150 -- server/tests/shared/streaming-playlists.ts | 296 --- server/tests/shared/tracker.ts | 27 - server/tests/shared/video-playlists.ts | 22 - server/tests/shared/videos.ts | 315 --- server/tests/shared/views.ts | 93 - server/tests/shared/webtorrent.ts | 58 - server/tools/README.md | 3 - server/tools/package.json | 11 - server/tools/peertube-auth.ts | 171 -- server/tools/peertube-get-access-token.ts | 34 - server/tools/peertube-import-videos.ts | 351 --- server/tools/peertube-plugins.ts | 165 -- server/tools/peertube-redundancy.ts | 172 -- server/tools/peertube-upload.ts | 77 - server/tools/peertube.ts | 72 - server/tools/shared/cli.ts | 262 --- server/tools/shared/index.ts | 1 - server/tools/tsconfig.json | 12 - server/tools/yarn.lock | 373 --- server/tsconfig.json | 19 +- server/tsconfig.lib.json | 12 + server/tsconfig.types.json | 11 +- server/types/activitypub-processor.model.ts | 9 - server/types/express.d.ts | 222 -- server/types/index.ts | 3 - server/types/models/abuse/abuse-message.ts | 20 - server/types/models/abuse/abuse.ts | 114 - server/types/models/abuse/index.ts | 2 - .../types/models/account/account-blocklist.ts | 27 - server/types/models/account/account.ts | 108 - .../types/models/account/actor-custom-page.ts | 3 - server/types/models/account/index.ts | 3 - server/types/models/actor/actor-follow.ts | 65 - server/types/models/actor/actor-image.ts | 12 - server/types/models/actor/actor.ts | 170 -- server/types/models/actor/index.ts | 3 - .../types/models/application/application.ts | 5 - server/types/models/application/index.ts | 1 - server/types/models/index.ts | 8 - server/types/models/oauth/index.ts | 2 - server/types/models/oauth/oauth-client.ts | 3 - server/types/models/oauth/oauth-token.ts | 14 - server/types/models/runners/index.ts | 3 - server/types/models/runners/runner-job.ts | 20 - .../runners/runner-registration-token.ts | 5 - server/types/models/runners/runner.ts | 5 - server/types/models/server/index.ts | 3 - server/types/models/server/plugin.ts | 11 - .../types/models/server/server-blocklist.ts | 26 - server/types/models/server/server.ts | 26 - server/types/models/server/tracker.ts | 7 - server/types/models/user/index.ts | 5 - .../models/user/user-notification-setting.ts | 9 - server/types/models/user/user-notification.ts | 122 - server/types/models/user/user-registration.ts | 15 - .../types/models/user/user-video-history.ts | 5 - server/types/models/user/user.ts | 89 - server/types/models/video/index.ts | 26 - .../video/local-video-viewer-watch-section.ts | 5 - .../types/models/video/local-video-viewer.ts | 19 - .../models/video/schedule-video-update.ts | 11 - server/types/models/video/storyboard.ts | 15 - server/types/models/video/tag.ts | 3 - server/types/models/video/thumbnail.ts | 15 - server/types/models/video/video-blacklist.ts | 30 - server/types/models/video/video-caption.ts | 27 - .../models/video/video-change-ownership.ts | 26 - .../types/models/video/video-channel-sync.ts | 17 - server/types/models/video/video-channels.ts | 153 -- server/types/models/video/video-comment.ts | 71 - server/types/models/video/video-file.ts | 43 - server/types/models/video/video-import.ts | 36 - .../models/video/video-live-replay-setting.ts | 3 - .../types/models/video/video-live-session.ts | 17 - server/types/models/video/video-live.ts | 22 - server/types/models/video/video-password.ts | 3 - .../models/video/video-playlist-element.ts | 39 - server/types/models/video/video-playlist.ts | 104 - server/types/models/video/video-rate.ts | 27 - server/types/models/video/video-redundancy.ts | 43 - server/types/models/video/video-share.ts | 19 - server/types/models/video/video-source.ts | 3 - .../models/video/video-streaming-playlist.ts | 43 - server/types/models/video/video.ts | 225 -- server/types/plugins/index.ts | 4 - server/types/plugins/plugin-library.model.ts | 7 - .../plugins/register-server-auth.model.ts | 72 - .../plugins/register-server-option.model.ts | 171 -- server/types/sequelize.ts | 19 - .../abuse/abuse-predefined-reasons.ts | 14 - shared/core-utils/abuse/index.ts | 1 - shared/core-utils/common/env.ts | 46 - shared/core-utils/common/index.ts | 12 - shared/core-utils/common/path.ts | 48 - shared/core-utils/common/url.ts | 150 -- shared/core-utils/i18n/index.ts | 1 - shared/core-utils/index.ts | 7 - shared/core-utils/plugins/hooks.ts | 61 - shared/core-utils/plugins/index.ts | 1 - shared/core-utils/renderer/index.ts | 2 - shared/core-utils/users/index.ts | 1 - shared/core-utils/users/user-role.ts | 37 - shared/core-utils/videos/bitrate.ts | 113 - shared/core-utils/videos/common.ts | 26 - shared/core-utils/videos/index.ts | 2 - shared/extra-utils/file.ts | 11 - shared/extra-utils/index.ts | 3 - shared/extra-utils/uuid.ts | 32 - shared/ffmpeg/ffmpeg-command-wrapper.ts | 246 -- .../ffmpeg-default-transcoding-profile.ts | 187 -- shared/ffmpeg/ffmpeg-edition.ts | 239 -- shared/ffmpeg/ffmpeg-images.ts | 92 - shared/ffmpeg/ffmpeg-live.ts | 184 -- shared/ffmpeg/ffmpeg-utils.ts | 17 - shared/ffmpeg/ffmpeg-vod.ts | 256 -- shared/ffmpeg/ffprobe.ts | 184 -- shared/ffmpeg/index.ts | 9 - shared/ffmpeg/shared/encoder-options.ts | 39 - shared/ffmpeg/shared/index.ts | 2 - shared/ffmpeg/shared/presets.ts | 93 - shared/models/activitypub/activity.ts | 135 -- .../models/activitypub/activitypub-actor.ts | 34 - .../activitypub/activitypub-collection.ts | 9 - shared/models/activitypub/activitypub-root.ts | 5 - shared/models/activitypub/index.ts | 9 - .../activitypub/objects/abuse-object.ts | 15 - .../activitypub/objects/activitypub-object.ts | 17 - .../activitypub/objects/cache-file-object.ts | 9 - .../activitypub/objects/common-objects.ts | 130 -- shared/models/activitypub/objects/index.ts | 9 - .../activitypub/objects/playlist-object.ts | 29 - .../objects/video-comment-object.ts | 16 - .../activitypub/objects/video-object.ts | 74 - shared/models/actors/account.model.ts | 22 - shared/models/actors/actor-image.type.ts | 4 - shared/models/actors/actor.model.ts | 13 - shared/models/actors/follow.model.ts | 13 - shared/models/actors/index.ts | 6 - shared/models/bulk/index.ts | 1 - shared/models/common/index.ts | 1 - shared/models/custom-markup/index.ts | 1 - shared/models/feeds/feed-format.enum.ts | 5 - shared/models/feeds/index.ts | 1 - shared/models/http/http-error-codes.ts | 364 --- shared/models/http/http-methods.ts | 21 - shared/models/http/index.ts | 2 - shared/models/index.ts | 19 - shared/models/joinpeertube/index.ts | 1 - shared/models/metrics/index.ts | 1 - .../metrics/playback-metric-create.model.ts | 22 - .../moderation/abuse/abuse-create.model.ts | 21 - .../moderation/abuse/abuse-message.model.ts | 10 - .../moderation/abuse/abuse-reason.model.ts | 20 - .../moderation/abuse/abuse-state.model.ts | 5 - .../moderation/abuse/abuse-update.model.ts | 7 - shared/models/moderation/abuse/abuse.model.ts | 70 - shared/models/moderation/abuse/index.ts | 8 - .../models/moderation/account-block.model.ts | 7 - shared/models/moderation/index.ts | 4 - .../models/moderation/server-block.model.ts | 9 - shared/models/nodeinfo/index.ts | 1 - shared/models/overviews/index.ts | 1 - .../models/overviews/videos-overview.model.ts | 24 - shared/models/plugins/client/index.ts | 8 - .../client/register-client-hook.model.ts | 7 - .../register-client-settings-script.model.ts | 8 - shared/models/plugins/hook-type.enum.ts | 5 - shared/models/plugins/index.ts | 6 - shared/models/plugins/plugin-index/index.ts | 3 - .../peertube-plugin-index-list.model.ts | 10 - .../plugins/plugin-package-json.model.ts | 29 - shared/models/plugins/plugin.type.ts | 4 - shared/models/plugins/server/api/index.ts | 3 - .../server/api/peertube-plugin.model.ts | 16 - shared/models/plugins/server/index.ts | 6 - .../models/plugins/server/managers/index.ts | 9 - .../plugin-playlist-privacy-manager.model.ts | 12 - .../plugin-transcoding-manager.model.ts | 13 - .../plugin-video-category-manager.model.ts | 13 - .../plugin-video-language-manager.model.ts | 13 - .../plugin-video-licence-manager.model.ts | 13 - .../plugin-video-privacy-manager.model.ts | 13 - .../server/register-server-hook.model.ts | 7 - .../models/plugins/server/settings/index.ts | 2 - .../server/settings/public-server.setting.ts | 5 - .../settings/register-server-setting.model.ts | 12 - shared/models/redundancy/index.ts | 4 - .../runners/accept-runner-job-result.model.ts | 6 - shared/models/runners/index.ts | 21 - .../runners/list-runner-jobs-query.model.ts | 9 - .../request-runner-job-result.model.ts | 10 - .../runners/runner-job-payload.model.ts | 79 - .../runner-job-private-payload.model.ts | 45 - .../models/runners/runner-job-state.model.ts | 11 - shared/models/runners/runner-job.model.ts | 45 - shared/models/search/index.ts | 6 - .../video-channels-search-query.model.ts | 18 - .../video-playlists-search-query.model.ts | 20 - .../search/videos-common-query.model.ts | 45 - .../search/videos-search-query.model.ts | 26 - .../models/server/client-log-create.model.ts | 11 - shared/models/server/custom-config.model.ts | 259 --- shared/models/server/index.ts | 16 - shared/models/server/job.model.ts | 304 --- .../server/peertube-problem-document.model.ts | 32 - shared/models/server/server-config.model.ts | 305 --- .../models/server/server-error-code.enum.ts | 89 - shared/models/server/server-stats.model.ts | 47 - shared/models/tokens/index.ts | 1 - shared/models/users/index.ts | 16 - shared/models/users/registration/index.ts | 5 - .../user-registration-request.model.ts | 5 - .../user-registration-state.model.ts | 5 - .../registration/user-registration.model.ts | 29 - shared/models/users/user-create.model.ts | 13 - shared/models/users/user-flag.model.ts | 4 - .../users/user-notification-setting.model.ts | 32 - .../models/users/user-notification.model.ts | 138 -- shared/models/users/user-right.enum.ts | 51 - shared/models/users/user-role.ts | 6 - shared/models/users/user-update-me.model.ts | 26 - shared/models/users/user-update.model.ts | 13 - shared/models/users/user.model.ts | 78 - shared/models/videos/blacklist/index.ts | 3 - .../videos/blacklist/video-blacklist.model.ts | 18 - shared/models/videos/caption/index.ts | 2 - .../videos/caption/video-caption.model.ts | 7 - .../models/videos/change-ownership/index.ts | 3 - .../video-change-ownership.model.ts | 17 - shared/models/videos/channel-sync/index.ts | 3 - .../video-channel-sync-state.enum.ts | 6 - .../channel-sync/video-channel-sync.model.ts | 14 - shared/models/videos/channel/index.ts | 4 - .../videos/channel/video-channel.model.ts | 34 - shared/models/videos/comment/index.ts | 2 - .../videos/comment/video-comment.model.ts | 45 - shared/models/videos/file/index.ts | 3 - shared/models/videos/file/video-file.model.ts | 23 - .../videos/file/video-resolution.enum.ts | 11 - shared/models/videos/import/index.ts | 4 - .../import/video-import-create.model.ts | 9 - .../videos/import/video-import-state.enum.ts | 8 - .../videos/import/video-import.model.ts | 24 - shared/models/videos/index.ts | 42 - shared/models/videos/live/index.ts | 8 - .../videos/live/live-video-create.model.ts | 11 - .../videos/live/live-video-error.enum.ts | 9 - .../live/live-video-event-payload.model.ts | 7 - .../live/live-video-latency-mode.enum.ts | 5 - .../videos/live/live-video-session.model.ts | 22 - .../videos/live/live-video-update.model.ts | 9 - shared/models/videos/live/live-video.model.ts | 14 - shared/models/videos/playlist/index.ts | 12 - .../playlist/video-playlist-create.model.ts | 11 - .../playlist/video-playlist-element.model.ts | 19 - .../playlist/video-playlist-privacy.model.ts | 5 - .../playlist/video-playlist-type.model.ts | 4 - .../playlist/video-playlist-update.model.ts | 10 - .../videos/playlist/video-playlist.model.ts | 34 - .../videos/rate/account-video-rate.model.ts | 7 - shared/models/videos/rate/index.ts | 5 - .../rate/user-video-rate-update.model.ts | 5 - .../videos/rate/user-video-rate.model.ts | 6 - shared/models/videos/stats/index.ts | 6 - shared/models/videos/studio/index.ts | 1 - shared/models/videos/thumbnail.type.ts | 4 - shared/models/videos/transcoding/index.ts | 3 - .../transcoding/video-transcoding.model.ts | 67 - shared/models/videos/video-create.model.ts | 25 - shared/models/videos/video-include.enum.ts | 8 - shared/models/videos/video-privacy.enum.ts | 7 - .../videos/video-schedule-update.model.ts | 6 - shared/models/videos/video-state.enum.ts | 11 - shared/models/videos/video-storage.enum.ts | 4 - .../videos/video-streaming-playlist.model.ts | 15 - .../videos/video-streaming-playlist.type.ts | 3 - shared/models/videos/video-update.model.ts | 25 - shared/models/videos/video.model.ts | 99 - shared/server-commands/bulk/bulk-command.ts | 20 - shared/server-commands/bulk/index.ts | 1 - shared/server-commands/cli/cli-command.ts | 27 - shared/server-commands/cli/index.ts | 1 - .../custom-pages/custom-pages-command.ts | 33 - shared/server-commands/custom-pages/index.ts | 1 - shared/server-commands/feeds/feeds-command.ts | 78 - shared/server-commands/feeds/index.ts | 1 - shared/server-commands/index.ts | 14 - shared/server-commands/logs/index.ts | 1 - shared/server-commands/logs/logs-command.ts | 56 - .../moderation/abuses-command.ts | 228 -- shared/server-commands/moderation/index.ts | 1 - shared/server-commands/overviews/index.ts | 1 - .../overviews/overviews-command.ts | 23 - shared/server-commands/requests/index.ts | 1 - shared/server-commands/requests/requests.ts | 259 --- shared/server-commands/runners/index.ts | 3 - .../runners/runner-jobs-command.ts | 294 --- .../runner-registration-tokens-command.ts | 55 - .../runners/runners-command.ts | 78 - shared/server-commands/search/index.ts | 1 - .../server-commands/search/search-command.ts | 98 - .../server-commands/server/config-command.ts | 576 ----- .../server/contact-form-command.ts | 31 - .../server-commands/server/debug-command.ts | 33 - .../server-commands/server/follows-command.ts | 139 -- shared/server-commands/server/follows.ts | 20 - shared/server-commands/server/index.ts | 15 - shared/server-commands/server/jobs-command.ts | 84 - shared/server-commands/server/jobs.ts | 118 - .../server-commands/server/metrics-command.ts | 18 - .../server/object-storage-command.ts | 165 -- .../server-commands/server/plugins-command.ts | 257 --- .../server/redundancy-command.ts | 80 - shared/server-commands/server/server.ts | 450 ---- .../server-commands/server/servers-command.ts | 103 - shared/server-commands/server/servers.ts | 68 - .../server-commands/server/stats-command.ts | 25 - .../shared/abstract-command.ts | 223 -- shared/server-commands/shared/index.ts | 1 - shared/server-commands/socket/index.ts | 1 - .../socket/socket-io-command.ts | 24 - .../server-commands/users/accounts-command.ts | 76 - shared/server-commands/users/accounts.ts | 15 - .../users/blocklist-command.ts | 165 -- shared/server-commands/users/index.ts | 10 - shared/server-commands/users/login-command.ts | 159 -- shared/server-commands/users/login.ts | 19 - .../users/notifications-command.ts | 85 - .../users/registrations-command.ts | 151 -- .../users/subscriptions-command.ts | 83 - .../users/two-factor-command.ts | 92 - shared/server-commands/users/users-command.ts | 388 ---- .../videos/blacklist-command.ts | 75 - .../videos/captions-command.ts | 67 - .../videos/change-ownership-command.ts | 68 - .../videos/channel-syncs-command.ts | 55 - .../videos/channels-command.ts | 202 -- shared/server-commands/videos/channels.ts | 29 - .../videos/comments-command.ts | 159 -- .../server-commands/videos/history-command.ts | 54 - .../server-commands/videos/imports-command.ts | 77 - shared/server-commands/videos/index.ts | 21 - shared/server-commands/videos/live-command.ts | 337 --- shared/server-commands/videos/live.ts | 128 - .../videos/playlists-command.ts | 281 --- .../videos/services-command.ts | 29 - .../videos/storyboard-command.ts | 19 - .../videos/streaming-playlists-command.ts | 119 - .../videos/video-passwords-command.ts | 55 - .../videos/video-stats-command.ts | 56 - .../videos/video-studio-command.ts | 67 - .../videos/video-token-command.ts | 34 - .../server-commands/videos/videos-command.ts | 829 ------- .../server-commands/videos/views-command.ts | 51 - shared/tsconfig.json | 6 - shared/tsconfig.types.json | 12 - shared/typescript-utils/index.ts | 1 - support/doc/api/embeds.md | 2 +- support/doc/api/openapi.yaml | 4 +- support/doc/development/lib.md | 2 +- support/doc/development/localization.md | 2 +- support/doc/development/monitoring.md | 4 +- support/doc/development/server.md | 66 +- support/doc/development/tests.md | 15 +- support/doc/plugins/guide.md | 43 +- support/doc/tools.md | 544 ++--- support/nginx/peertube | 2 +- tsconfig.base.json | 10 +- tsconfig.eslint.json | 32 + tsconfig.json | 20 - yarn.lock | 1072 ++++++--- 3230 files changed, 162039 insertions(+), 160923 deletions(-) create mode 100644 .mocharc.cjs create mode 100644 apps/peertube-cli/.npmignore create mode 100644 apps/peertube-cli/README.md create mode 100644 apps/peertube-cli/package.json create mode 100644 apps/peertube-cli/scripts/build.js create mode 100644 apps/peertube-cli/scripts/watch.js create mode 100644 apps/peertube-cli/src/peertube-auth.ts create mode 100644 apps/peertube-cli/src/peertube-get-access-token.ts create mode 100644 apps/peertube-cli/src/peertube-plugins.ts create mode 100644 apps/peertube-cli/src/peertube-redundancy.ts create mode 100644 apps/peertube-cli/src/peertube-upload.ts create mode 100644 apps/peertube-cli/src/peertube.ts create mode 100644 apps/peertube-cli/src/shared/cli.ts create mode 100644 apps/peertube-cli/src/shared/index.ts create mode 100644 apps/peertube-cli/tsconfig.json create mode 100644 apps/peertube-cli/yarn.lock rename {packages => apps}/peertube-runner/.gitignore (100%) create mode 100644 apps/peertube-runner/.npmignore create mode 100644 apps/peertube-runner/README.md create mode 100644 apps/peertube-runner/package.json create mode 100644 apps/peertube-runner/scripts/build.js create mode 100644 apps/peertube-runner/src/peertube-runner.ts create mode 100644 apps/peertube-runner/src/register/index.ts create mode 100644 apps/peertube-runner/src/register/register.ts create mode 100644 apps/peertube-runner/src/server/index.ts create mode 100644 apps/peertube-runner/src/server/process/index.ts create mode 100644 apps/peertube-runner/src/server/process/process.ts create mode 100644 apps/peertube-runner/src/server/process/shared/common.ts create mode 100644 apps/peertube-runner/src/server/process/shared/index.ts create mode 100644 apps/peertube-runner/src/server/process/shared/process-live.ts create mode 100644 apps/peertube-runner/src/server/process/shared/process-studio.ts create mode 100644 apps/peertube-runner/src/server/process/shared/process-vod.ts create mode 100644 apps/peertube-runner/src/server/process/shared/transcoding-logger.ts create mode 100644 apps/peertube-runner/src/server/server.ts create mode 100644 apps/peertube-runner/src/server/shared/index.ts create mode 100644 apps/peertube-runner/src/server/shared/supported-job.ts create mode 100644 apps/peertube-runner/src/shared/config-manager.ts create mode 100644 apps/peertube-runner/src/shared/http.ts create mode 100644 apps/peertube-runner/src/shared/index.ts create mode 100644 apps/peertube-runner/src/shared/ipc/index.ts create mode 100644 apps/peertube-runner/src/shared/ipc/ipc-client.ts create mode 100644 apps/peertube-runner/src/shared/ipc/ipc-server.ts create mode 100644 apps/peertube-runner/src/shared/ipc/shared/index.ts rename {packages/peertube-runner => apps/peertube-runner/src}/shared/ipc/shared/ipc-request.model.ts (100%) rename {packages/peertube-runner => apps/peertube-runner/src}/shared/ipc/shared/ipc-response.model.ts (100%) create mode 100644 apps/peertube-runner/src/shared/logger.ts create mode 100644 apps/peertube-runner/tsconfig.json rename {packages => apps}/peertube-runner/yarn.lock (100%) create mode 100644 packages/core-utils/package.json create mode 100644 packages/core-utils/src/abuse/abuse-predefined-reasons.ts create mode 100644 packages/core-utils/src/abuse/index.ts rename {shared/core-utils => packages/core-utils/src}/common/array.ts (100%) rename {shared/core-utils => packages/core-utils/src}/common/date.ts (100%) create mode 100644 packages/core-utils/src/common/index.ts rename {shared/core-utils => packages/core-utils/src}/common/number.ts (100%) rename {shared/core-utils => packages/core-utils/src}/common/object.ts (100%) rename {shared/core-utils => packages/core-utils/src}/common/promises.ts (100%) rename {shared/core-utils => packages/core-utils/src}/common/random.ts (100%) rename {shared/core-utils => packages/core-utils/src}/common/regexp.ts (100%) rename {shared/core-utils => packages/core-utils/src}/common/time.ts (100%) create mode 100644 packages/core-utils/src/common/url.ts rename {shared/core-utils => packages/core-utils/src}/common/version.ts (100%) rename {shared/core-utils => packages/core-utils/src}/i18n/i18n.ts (100%) create mode 100644 packages/core-utils/src/i18n/index.ts create mode 100644 packages/core-utils/src/index.ts create mode 100644 packages/core-utils/src/plugins/hooks.ts create mode 100644 packages/core-utils/src/plugins/index.ts rename {shared/core-utils => packages/core-utils/src}/renderer/html.ts (100%) create mode 100644 packages/core-utils/src/renderer/index.ts rename {shared/core-utils => packages/core-utils/src}/renderer/markdown.ts (100%) create mode 100644 packages/core-utils/src/users/index.ts create mode 100644 packages/core-utils/src/users/user-role.ts create mode 100644 packages/core-utils/src/videos/bitrate.ts create mode 100644 packages/core-utils/src/videos/common.ts create mode 100644 packages/core-utils/src/videos/index.ts create mode 100644 packages/core-utils/tsconfig.json create mode 100644 packages/ffmpeg/package.json create mode 100644 packages/ffmpeg/src/ffmpeg-command-wrapper.ts create mode 100644 packages/ffmpeg/src/ffmpeg-default-transcoding-profile.ts create mode 100644 packages/ffmpeg/src/ffmpeg-edition.ts create mode 100644 packages/ffmpeg/src/ffmpeg-images.ts create mode 100644 packages/ffmpeg/src/ffmpeg-live.ts create mode 100644 packages/ffmpeg/src/ffmpeg-utils.ts rename {shared/ffmpeg => packages/ffmpeg/src}/ffmpeg-version.ts (100%) create mode 100644 packages/ffmpeg/src/ffmpeg-vod.ts create mode 100644 packages/ffmpeg/src/ffprobe.ts create mode 100644 packages/ffmpeg/src/index.ts create mode 100644 packages/ffmpeg/src/shared/encoder-options.ts create mode 100644 packages/ffmpeg/src/shared/index.ts create mode 100644 packages/ffmpeg/src/shared/presets.ts create mode 100644 packages/ffmpeg/tsconfig.json create mode 100644 packages/models/package.json create mode 100644 packages/models/src/activitypub/activity.ts create mode 100644 packages/models/src/activitypub/activitypub-actor.ts create mode 100644 packages/models/src/activitypub/activitypub-collection.ts rename {shared/models => packages/models/src}/activitypub/activitypub-ordered-collection.ts (100%) create mode 100644 packages/models/src/activitypub/activitypub-root.ts rename {shared/models => packages/models/src}/activitypub/activitypub-signature.ts (100%) rename {shared/models => packages/models/src}/activitypub/context.ts (100%) create mode 100644 packages/models/src/activitypub/index.ts create mode 100644 packages/models/src/activitypub/objects/abuse-object.ts create mode 100644 packages/models/src/activitypub/objects/activitypub-object.ts create mode 100644 packages/models/src/activitypub/objects/cache-file-object.ts create mode 100644 packages/models/src/activitypub/objects/common-objects.ts create mode 100644 packages/models/src/activitypub/objects/index.ts rename {shared/models => packages/models/src}/activitypub/objects/playlist-element-object.ts (100%) create mode 100644 packages/models/src/activitypub/objects/playlist-object.ts create mode 100644 packages/models/src/activitypub/objects/video-comment-object.ts create mode 100644 packages/models/src/activitypub/objects/video-object.ts rename {shared/models => packages/models/src}/activitypub/objects/watch-action-object.ts (100%) rename {shared/models => packages/models/src}/activitypub/webfinger.ts (100%) create mode 100644 packages/models/src/actors/account.model.ts rename {shared/models => packages/models/src}/actors/actor-image.model.ts (100%) create mode 100644 packages/models/src/actors/actor-image.type.ts create mode 100644 packages/models/src/actors/actor.model.ts rename {shared/models => packages/models/src}/actors/custom-page.model.ts (100%) create mode 100644 packages/models/src/actors/follow.model.ts create mode 100644 packages/models/src/actors/index.ts rename {shared/models => packages/models/src}/bulk/bulk-remove-comments-of-body.model.ts (100%) create mode 100644 packages/models/src/bulk/index.ts create mode 100644 packages/models/src/common/index.ts rename {shared/models => packages/models/src}/common/result-list.model.ts (100%) rename {shared/models => packages/models/src}/custom-markup/custom-markup-data.model.ts (100%) create mode 100644 packages/models/src/custom-markup/index.ts create mode 100644 packages/models/src/feeds/feed-format.enum.ts create mode 100644 packages/models/src/feeds/index.ts create mode 100644 packages/models/src/http/http-methods.ts create mode 100644 packages/models/src/http/http-status-codes.ts create mode 100644 packages/models/src/http/index.ts create mode 100644 packages/models/src/index.ts create mode 100644 packages/models/src/joinpeertube/index.ts rename {shared/models => packages/models/src}/joinpeertube/versions.model.ts (100%) create mode 100644 packages/models/src/metrics/index.ts create mode 100644 packages/models/src/metrics/playback-metric-create.model.ts create mode 100644 packages/models/src/moderation/abuse/abuse-create.model.ts rename {shared/models => packages/models/src}/moderation/abuse/abuse-filter.type.ts (100%) create mode 100644 packages/models/src/moderation/abuse/abuse-message.model.ts create mode 100644 packages/models/src/moderation/abuse/abuse-reason.model.ts create mode 100644 packages/models/src/moderation/abuse/abuse-state.model.ts create mode 100644 packages/models/src/moderation/abuse/abuse-update.model.ts rename {shared/models => packages/models/src}/moderation/abuse/abuse-video-is.type.ts (100%) create mode 100644 packages/models/src/moderation/abuse/abuse.model.ts create mode 100644 packages/models/src/moderation/abuse/index.ts create mode 100644 packages/models/src/moderation/account-block.model.ts rename {shared/models => packages/models/src}/moderation/block-status.model.ts (100%) create mode 100644 packages/models/src/moderation/index.ts create mode 100644 packages/models/src/moderation/server-block.model.ts create mode 100644 packages/models/src/nodeinfo/index.ts rename {shared/models => packages/models/src}/nodeinfo/nodeinfo.model.ts (100%) create mode 100644 packages/models/src/overviews/index.ts create mode 100644 packages/models/src/overviews/videos-overview.model.ts rename {shared/models => packages/models/src}/plugins/client/client-hook.model.ts (100%) create mode 100644 packages/models/src/plugins/client/index.ts rename {shared/models => packages/models/src}/plugins/client/plugin-client-scope.type.ts (100%) rename {shared/models => packages/models/src}/plugins/client/plugin-element-placeholder.type.ts (100%) rename {shared/models => packages/models/src}/plugins/client/plugin-selector-id.type.ts (100%) rename {shared/models => packages/models/src}/plugins/client/register-client-form-field.model.ts (100%) create mode 100644 packages/models/src/plugins/client/register-client-hook.model.ts rename {shared/models => packages/models/src}/plugins/client/register-client-route.model.ts (100%) create mode 100644 packages/models/src/plugins/client/register-client-settings-script.model.ts create mode 100644 packages/models/src/plugins/hook-type.enum.ts create mode 100644 packages/models/src/plugins/index.ts create mode 100644 packages/models/src/plugins/plugin-index/index.ts create mode 100644 packages/models/src/plugins/plugin-index/peertube-plugin-index-list.model.ts rename {shared/models => packages/models/src}/plugins/plugin-index/peertube-plugin-index.model.ts (100%) rename {shared/models => packages/models/src}/plugins/plugin-index/peertube-plugin-latest-version.model.ts (100%) create mode 100644 packages/models/src/plugins/plugin-package-json.model.ts create mode 100644 packages/models/src/plugins/plugin.type.ts create mode 100644 packages/models/src/plugins/server/api/index.ts rename {shared/models => packages/models/src}/plugins/server/api/install-plugin.model.ts (100%) rename {shared/models => packages/models/src}/plugins/server/api/manage-plugin.model.ts (100%) create mode 100644 packages/models/src/plugins/server/api/peertube-plugin.model.ts create mode 100644 packages/models/src/plugins/server/index.ts create mode 100644 packages/models/src/plugins/server/managers/index.ts create mode 100644 packages/models/src/plugins/server/managers/plugin-playlist-privacy-manager.model.ts rename {shared/models => packages/models/src}/plugins/server/managers/plugin-settings-manager.model.ts (100%) rename {shared/models => packages/models/src}/plugins/server/managers/plugin-storage-manager.model.ts (100%) create mode 100644 packages/models/src/plugins/server/managers/plugin-transcoding-manager.model.ts create mode 100644 packages/models/src/plugins/server/managers/plugin-video-category-manager.model.ts create mode 100644 packages/models/src/plugins/server/managers/plugin-video-language-manager.model.ts create mode 100644 packages/models/src/plugins/server/managers/plugin-video-licence-manager.model.ts create mode 100644 packages/models/src/plugins/server/managers/plugin-video-privacy-manager.model.ts rename {shared/models => packages/models/src}/plugins/server/plugin-constant-manager.model.ts (100%) rename {shared/models => packages/models/src}/plugins/server/plugin-translation.model.ts (100%) create mode 100644 packages/models/src/plugins/server/register-server-hook.model.ts rename {shared/models => packages/models/src}/plugins/server/server-hook.model.ts (100%) create mode 100644 packages/models/src/plugins/server/settings/index.ts create mode 100644 packages/models/src/plugins/server/settings/public-server.setting.ts create mode 100644 packages/models/src/plugins/server/settings/register-server-setting.model.ts create mode 100644 packages/models/src/redundancy/index.ts rename {shared/models => packages/models/src}/redundancy/video-redundancies-filters.model.ts (100%) rename {shared/models => packages/models/src}/redundancy/video-redundancy-config-filter.type.ts (100%) rename {shared/models => packages/models/src}/redundancy/video-redundancy.model.ts (100%) rename {shared/models => packages/models/src}/redundancy/videos-redundancy-strategy.model.ts (100%) rename {shared/models => packages/models/src}/runners/abort-runner-job-body.model.ts (100%) rename {shared/models => packages/models/src}/runners/accept-runner-job-body.model.ts (100%) create mode 100644 packages/models/src/runners/accept-runner-job-result.model.ts rename {shared/models => packages/models/src}/runners/error-runner-job-body.model.ts (100%) create mode 100644 packages/models/src/runners/index.ts create mode 100644 packages/models/src/runners/list-runner-jobs-query.model.ts rename {shared/models => packages/models/src}/runners/list-runner-registration-tokens.model.ts (100%) rename {shared/models => packages/models/src}/runners/list-runners-query.model.ts (100%) rename {shared/models => packages/models/src}/runners/register-runner-body.model.ts (100%) rename {shared/models => packages/models/src}/runners/register-runner-result.model.ts (100%) rename {shared/models => packages/models/src}/runners/request-runner-job-body.model.ts (100%) create mode 100644 packages/models/src/runners/request-runner-job-result.model.ts create mode 100644 packages/models/src/runners/runner-job-payload.model.ts create mode 100644 packages/models/src/runners/runner-job-private-payload.model.ts create mode 100644 packages/models/src/runners/runner-job-state.model.ts rename {shared/models => packages/models/src}/runners/runner-job-success-body.model.ts (100%) rename {shared/models => packages/models/src}/runners/runner-job-type.type.ts (100%) rename {shared/models => packages/models/src}/runners/runner-job-update-body.model.ts (100%) create mode 100644 packages/models/src/runners/runner-job.model.ts rename {shared/models => packages/models/src}/runners/runner-registration-token.ts (100%) rename {shared/models => packages/models/src}/runners/runner.model.ts (100%) rename {shared/models => packages/models/src}/runners/unregister-runner-body.model.ts (100%) rename {shared/models => packages/models/src}/search/boolean-both-query.model.ts (100%) create mode 100644 packages/models/src/search/index.ts rename {shared/models => packages/models/src}/search/search-target-query.model.ts (100%) create mode 100644 packages/models/src/search/video-channels-search-query.model.ts create mode 100644 packages/models/src/search/video-playlists-search-query.model.ts create mode 100644 packages/models/src/search/videos-common-query.model.ts create mode 100644 packages/models/src/search/videos-search-query.model.ts rename {shared/models => packages/models/src}/server/about.model.ts (100%) rename {shared/models => packages/models/src}/server/broadcast-message-level.type.ts (100%) create mode 100644 packages/models/src/server/client-log-create.model.ts rename {shared/models => packages/models/src}/server/client-log-level.type.ts (100%) rename {shared/models => packages/models/src}/server/contact-form.model.ts (100%) create mode 100644 packages/models/src/server/custom-config.model.ts rename {shared/models => packages/models/src}/server/debug.model.ts (100%) rename {shared/models => packages/models/src}/server/emailer.model.ts (100%) create mode 100644 packages/models/src/server/index.ts create mode 100644 packages/models/src/server/job.model.ts create mode 100644 packages/models/src/server/peertube-problem-document.model.ts create mode 100644 packages/models/src/server/server-config.model.ts rename {shared/models => packages/models/src}/server/server-debug.model.ts (100%) create mode 100644 packages/models/src/server/server-error-code.enum.ts rename {shared/models => packages/models/src}/server/server-follow-create.model.ts (100%) rename {shared/models => packages/models/src}/server/server-log-level.type.ts (100%) create mode 100644 packages/models/src/server/server-stats.model.ts create mode 100644 packages/models/src/tokens/index.ts rename {shared/models => packages/models/src}/tokens/oauth-client-local.model.ts (100%) create mode 100644 packages/models/src/users/index.ts create mode 100644 packages/models/src/users/registration/index.ts rename {shared/models => packages/models/src}/users/registration/user-register.model.ts (100%) create mode 100644 packages/models/src/users/registration/user-registration-request.model.ts create mode 100644 packages/models/src/users/registration/user-registration-state.model.ts rename {shared/models => packages/models/src}/users/registration/user-registration-update-state.model.ts (100%) create mode 100644 packages/models/src/users/registration/user-registration.model.ts rename {shared/models => packages/models/src}/users/two-factor-enable-result.model.ts (100%) rename {shared/models => packages/models/src}/users/user-create-result.model.ts (100%) create mode 100644 packages/models/src/users/user-create.model.ts create mode 100644 packages/models/src/users/user-flag.model.ts rename {shared/models => packages/models/src}/users/user-login.model.ts (100%) create mode 100644 packages/models/src/users/user-notification-setting.model.ts create mode 100644 packages/models/src/users/user-notification.model.ts rename {shared/models => packages/models/src}/users/user-refresh-token.model.ts (100%) create mode 100644 packages/models/src/users/user-right.enum.ts create mode 100644 packages/models/src/users/user-role.ts rename {shared/models => packages/models/src}/users/user-scoped-token.ts (100%) create mode 100644 packages/models/src/users/user-update-me.model.ts create mode 100644 packages/models/src/users/user-update.model.ts rename {shared/models => packages/models/src}/users/user-video-quota.model.ts (100%) create mode 100644 packages/models/src/users/user.model.ts create mode 100644 packages/models/src/videos/blacklist/index.ts rename {shared/models => packages/models/src}/videos/blacklist/video-blacklist-create.model.ts (100%) rename {shared/models => packages/models/src}/videos/blacklist/video-blacklist-update.model.ts (100%) create mode 100644 packages/models/src/videos/blacklist/video-blacklist.model.ts create mode 100644 packages/models/src/videos/caption/index.ts rename {shared/models => packages/models/src}/videos/caption/video-caption-update.model.ts (100%) create mode 100644 packages/models/src/videos/caption/video-caption.model.ts create mode 100644 packages/models/src/videos/change-ownership/index.ts rename {shared/models => packages/models/src}/videos/change-ownership/video-change-ownership-accept.model.ts (100%) rename {shared/models => packages/models/src}/videos/change-ownership/video-change-ownership-create.model.ts (100%) create mode 100644 packages/models/src/videos/change-ownership/video-change-ownership.model.ts create mode 100644 packages/models/src/videos/channel-sync/index.ts rename {shared/models => packages/models/src}/videos/channel-sync/video-channel-sync-create.model.ts (100%) create mode 100644 packages/models/src/videos/channel-sync/video-channel-sync-state.enum.ts create mode 100644 packages/models/src/videos/channel-sync/video-channel-sync.model.ts create mode 100644 packages/models/src/videos/channel/index.ts rename {shared/models => packages/models/src}/videos/channel/video-channel-create-result.model.ts (100%) rename {shared/models => packages/models/src}/videos/channel/video-channel-create.model.ts (100%) rename {shared/models => packages/models/src}/videos/channel/video-channel-update.model.ts (100%) create mode 100644 packages/models/src/videos/channel/video-channel.model.ts create mode 100644 packages/models/src/videos/comment/index.ts rename {shared/models => packages/models/src}/videos/comment/video-comment-create.model.ts (100%) create mode 100644 packages/models/src/videos/comment/video-comment.model.ts create mode 100644 packages/models/src/videos/file/index.ts rename {shared/models => packages/models/src}/videos/file/video-file-metadata.model.ts (100%) create mode 100644 packages/models/src/videos/file/video-file.model.ts create mode 100644 packages/models/src/videos/file/video-resolution.enum.ts create mode 100644 packages/models/src/videos/import/index.ts create mode 100644 packages/models/src/videos/import/video-import-create.model.ts create mode 100644 packages/models/src/videos/import/video-import-state.enum.ts create mode 100644 packages/models/src/videos/import/video-import.model.ts rename {shared/models => packages/models/src}/videos/import/videos-import-in-channel-create.model.ts (100%) create mode 100644 packages/models/src/videos/index.ts create mode 100644 packages/models/src/videos/live/index.ts create mode 100644 packages/models/src/videos/live/live-video-create.model.ts create mode 100644 packages/models/src/videos/live/live-video-error.enum.ts create mode 100644 packages/models/src/videos/live/live-video-event-payload.model.ts rename {shared/models => packages/models/src}/videos/live/live-video-event.type.ts (100%) create mode 100644 packages/models/src/videos/live/live-video-latency-mode.enum.ts create mode 100644 packages/models/src/videos/live/live-video-session.model.ts create mode 100644 packages/models/src/videos/live/live-video-update.model.ts create mode 100644 packages/models/src/videos/live/live-video.model.ts rename {shared/models => packages/models/src}/videos/nsfw-policy.type.ts (100%) create mode 100644 packages/models/src/videos/playlist/index.ts rename {shared/models => packages/models/src}/videos/playlist/video-exist-in-playlist.model.ts (100%) rename {shared/models => packages/models/src}/videos/playlist/video-playlist-create-result.model.ts (100%) create mode 100644 packages/models/src/videos/playlist/video-playlist-create.model.ts rename {shared/models => packages/models/src}/videos/playlist/video-playlist-element-create-result.model.ts (100%) rename {shared/models => packages/models/src}/videos/playlist/video-playlist-element-create.model.ts (100%) rename {shared/models => packages/models/src}/videos/playlist/video-playlist-element-update.model.ts (100%) create mode 100644 packages/models/src/videos/playlist/video-playlist-element.model.ts create mode 100644 packages/models/src/videos/playlist/video-playlist-privacy.model.ts rename {shared/models => packages/models/src}/videos/playlist/video-playlist-reorder.model.ts (100%) create mode 100644 packages/models/src/videos/playlist/video-playlist-type.model.ts create mode 100644 packages/models/src/videos/playlist/video-playlist-update.model.ts create mode 100644 packages/models/src/videos/playlist/video-playlist.model.ts create mode 100644 packages/models/src/videos/rate/account-video-rate.model.ts create mode 100644 packages/models/src/videos/rate/index.ts create mode 100644 packages/models/src/videos/rate/user-video-rate-update.model.ts create mode 100644 packages/models/src/videos/rate/user-video-rate.model.ts rename {shared/models => packages/models/src}/videos/rate/user-video-rate.type.ts (100%) create mode 100644 packages/models/src/videos/stats/index.ts rename {shared/models => packages/models/src}/videos/stats/video-stats-overall-query.model.ts (100%) rename {shared/models => packages/models/src}/videos/stats/video-stats-overall.model.ts (100%) rename {shared/models => packages/models/src}/videos/stats/video-stats-retention.model.ts (100%) rename {shared/models => packages/models/src}/videos/stats/video-stats-timeserie-metric.type.ts (100%) rename {shared/models => packages/models/src}/videos/stats/video-stats-timeserie-query.model.ts (100%) rename {shared/models => packages/models/src}/videos/stats/video-stats-timeserie.model.ts (100%) rename {shared/models => packages/models/src}/videos/storyboard.model.ts (100%) create mode 100644 packages/models/src/videos/studio/index.ts rename {shared/models => packages/models/src}/videos/studio/video-studio-create-edit.model.ts (100%) create mode 100644 packages/models/src/videos/thumbnail.type.ts create mode 100644 packages/models/src/videos/transcoding/index.ts rename {shared/models => packages/models/src}/videos/transcoding/video-transcoding-create.model.ts (100%) rename {shared/models => packages/models/src}/videos/transcoding/video-transcoding-fps.model.ts (100%) create mode 100644 packages/models/src/videos/transcoding/video-transcoding.model.ts rename {shared/models => packages/models/src}/videos/video-constant.model.ts (100%) rename {shared/models => packages/models/src}/videos/video-create-result.model.ts (100%) create mode 100644 packages/models/src/videos/video-create.model.ts create mode 100644 packages/models/src/videos/video-include.enum.ts rename {shared/models => packages/models/src}/videos/video-password.model.ts (100%) create mode 100644 packages/models/src/videos/video-privacy.enum.ts rename {shared/models => packages/models/src}/videos/video-rate.type.ts (100%) create mode 100644 packages/models/src/videos/video-schedule-update.model.ts rename {shared/models => packages/models/src}/videos/video-sort-field.type.ts (100%) rename shared/models/videos/video-source.ts => packages/models/src/videos/video-source.model.ts (100%) create mode 100644 packages/models/src/videos/video-state.enum.ts create mode 100644 packages/models/src/videos/video-storage.enum.ts create mode 100644 packages/models/src/videos/video-streaming-playlist.model.ts create mode 100644 packages/models/src/videos/video-streaming-playlist.type.ts rename {shared/models => packages/models/src}/videos/video-token.model.ts (100%) create mode 100644 packages/models/src/videos/video-update.model.ts rename {shared/models => packages/models/src}/videos/video-view.model.ts (100%) create mode 100644 packages/models/src/videos/video.model.ts create mode 100644 packages/models/tsconfig.json create mode 100644 packages/models/tsconfig.types.json create mode 100644 packages/node-utils/package.json rename {shared/extra-utils => packages/node-utils/src}/crypto.ts (100%) create mode 100644 packages/node-utils/src/env.ts create mode 100644 packages/node-utils/src/file.ts create mode 100644 packages/node-utils/src/index.ts create mode 100644 packages/node-utils/src/path.ts create mode 100644 packages/node-utils/src/uuid.ts create mode 100644 packages/node-utils/tsconfig.json delete mode 100644 packages/peertube-runner/.npmignore delete mode 100644 packages/peertube-runner/README.md delete mode 100644 packages/peertube-runner/package.json delete mode 100644 packages/peertube-runner/peertube-runner.ts delete mode 100644 packages/peertube-runner/register/index.ts delete mode 100644 packages/peertube-runner/register/register.ts delete mode 100644 packages/peertube-runner/server/index.ts delete mode 100644 packages/peertube-runner/server/process/index.ts delete mode 100644 packages/peertube-runner/server/process/process.ts delete mode 100644 packages/peertube-runner/server/process/shared/common.ts delete mode 100644 packages/peertube-runner/server/process/shared/index.ts delete mode 100644 packages/peertube-runner/server/process/shared/process-live.ts delete mode 100644 packages/peertube-runner/server/process/shared/process-studio.ts delete mode 100644 packages/peertube-runner/server/process/shared/process-vod.ts delete mode 100644 packages/peertube-runner/server/process/shared/transcoding-logger.ts delete mode 100644 packages/peertube-runner/server/server.ts delete mode 100644 packages/peertube-runner/server/shared/index.ts delete mode 100644 packages/peertube-runner/server/shared/supported-job.ts delete mode 100644 packages/peertube-runner/shared/config-manager.ts delete mode 100644 packages/peertube-runner/shared/http.ts delete mode 100644 packages/peertube-runner/shared/index.ts delete mode 100644 packages/peertube-runner/shared/ipc/index.ts delete mode 100644 packages/peertube-runner/shared/ipc/ipc-client.ts delete mode 100644 packages/peertube-runner/shared/ipc/ipc-server.ts delete mode 100644 packages/peertube-runner/shared/ipc/shared/index.ts delete mode 100644 packages/peertube-runner/shared/logger.ts delete mode 100644 packages/peertube-runner/tsconfig.json create mode 100644 packages/server-commands/package.json create mode 100644 packages/server-commands/src/bulk/bulk-command.ts create mode 100644 packages/server-commands/src/bulk/index.ts create mode 100644 packages/server-commands/src/cli/cli-command.ts create mode 100644 packages/server-commands/src/cli/index.ts create mode 100644 packages/server-commands/src/custom-pages/custom-pages-command.ts create mode 100644 packages/server-commands/src/custom-pages/index.ts create mode 100644 packages/server-commands/src/feeds/feeds-command.ts create mode 100644 packages/server-commands/src/feeds/index.ts create mode 100644 packages/server-commands/src/index.ts create mode 100644 packages/server-commands/src/logs/index.ts create mode 100644 packages/server-commands/src/logs/logs-command.ts create mode 100644 packages/server-commands/src/moderation/abuses-command.ts create mode 100644 packages/server-commands/src/moderation/index.ts create mode 100644 packages/server-commands/src/overviews/index.ts create mode 100644 packages/server-commands/src/overviews/overviews-command.ts create mode 100644 packages/server-commands/src/requests/index.ts create mode 100644 packages/server-commands/src/requests/requests.ts create mode 100644 packages/server-commands/src/runners/index.ts create mode 100644 packages/server-commands/src/runners/runner-jobs-command.ts create mode 100644 packages/server-commands/src/runners/runner-registration-tokens-command.ts create mode 100644 packages/server-commands/src/runners/runners-command.ts create mode 100644 packages/server-commands/src/search/index.ts create mode 100644 packages/server-commands/src/search/search-command.ts create mode 100644 packages/server-commands/src/server/config-command.ts create mode 100644 packages/server-commands/src/server/contact-form-command.ts create mode 100644 packages/server-commands/src/server/debug-command.ts create mode 100644 packages/server-commands/src/server/follows-command.ts create mode 100644 packages/server-commands/src/server/follows.ts create mode 100644 packages/server-commands/src/server/index.ts create mode 100644 packages/server-commands/src/server/jobs-command.ts create mode 100644 packages/server-commands/src/server/jobs.ts create mode 100644 packages/server-commands/src/server/metrics-command.ts create mode 100644 packages/server-commands/src/server/object-storage-command.ts create mode 100644 packages/server-commands/src/server/plugins-command.ts create mode 100644 packages/server-commands/src/server/redundancy-command.ts create mode 100644 packages/server-commands/src/server/server.ts create mode 100644 packages/server-commands/src/server/servers-command.ts create mode 100644 packages/server-commands/src/server/servers.ts create mode 100644 packages/server-commands/src/server/stats-command.ts create mode 100644 packages/server-commands/src/shared/abstract-command.ts create mode 100644 packages/server-commands/src/shared/index.ts create mode 100644 packages/server-commands/src/socket/index.ts create mode 100644 packages/server-commands/src/socket/socket-io-command.ts create mode 100644 packages/server-commands/src/users/accounts-command.ts create mode 100644 packages/server-commands/src/users/accounts.ts create mode 100644 packages/server-commands/src/users/blocklist-command.ts create mode 100644 packages/server-commands/src/users/index.ts create mode 100644 packages/server-commands/src/users/login-command.ts create mode 100644 packages/server-commands/src/users/login.ts create mode 100644 packages/server-commands/src/users/notifications-command.ts create mode 100644 packages/server-commands/src/users/registrations-command.ts create mode 100644 packages/server-commands/src/users/subscriptions-command.ts create mode 100644 packages/server-commands/src/users/two-factor-command.ts create mode 100644 packages/server-commands/src/users/users-command.ts create mode 100644 packages/server-commands/src/videos/blacklist-command.ts create mode 100644 packages/server-commands/src/videos/captions-command.ts create mode 100644 packages/server-commands/src/videos/change-ownership-command.ts create mode 100644 packages/server-commands/src/videos/channel-syncs-command.ts create mode 100644 packages/server-commands/src/videos/channels-command.ts create mode 100644 packages/server-commands/src/videos/channels.ts create mode 100644 packages/server-commands/src/videos/comments-command.ts create mode 100644 packages/server-commands/src/videos/history-command.ts create mode 100644 packages/server-commands/src/videos/imports-command.ts create mode 100644 packages/server-commands/src/videos/index.ts create mode 100644 packages/server-commands/src/videos/live-command.ts create mode 100644 packages/server-commands/src/videos/live.ts create mode 100644 packages/server-commands/src/videos/playlists-command.ts create mode 100644 packages/server-commands/src/videos/services-command.ts create mode 100644 packages/server-commands/src/videos/storyboard-command.ts create mode 100644 packages/server-commands/src/videos/streaming-playlists-command.ts create mode 100644 packages/server-commands/src/videos/video-passwords-command.ts create mode 100644 packages/server-commands/src/videos/video-stats-command.ts create mode 100644 packages/server-commands/src/videos/video-studio-command.ts create mode 100644 packages/server-commands/src/videos/video-token-command.ts create mode 100644 packages/server-commands/src/videos/videos-command.ts create mode 100644 packages/server-commands/src/videos/views-command.ts create mode 100644 packages/server-commands/tsconfig.json rename {server => packages}/tests/fixtures/60fps_720p_small.mp4 (100%) rename {server => packages}/tests/fixtures/ap-json/mastodon/bad-body-http-signature.json (100%) rename {server => packages}/tests/fixtures/ap-json/mastodon/bad-http-signature.json (100%) rename {server => packages}/tests/fixtures/ap-json/mastodon/bad-public-key.json (100%) rename {server => packages}/tests/fixtures/ap-json/mastodon/create-bad-signature.json (100%) rename {server => packages}/tests/fixtures/ap-json/mastodon/create.json (100%) rename {server => packages}/tests/fixtures/ap-json/mastodon/http-signature.json (100%) rename {server => packages}/tests/fixtures/ap-json/mastodon/public-key.json (100%) rename {server => packages}/tests/fixtures/ap-json/peertube/announce-without-context.json (100%) rename {server => packages}/tests/fixtures/ap-json/peertube/invalid-keys.json (100%) rename {server => packages}/tests/fixtures/ap-json/peertube/keys.json (100%) rename {server => packages}/tests/fixtures/avatar-big.png (100%) rename {server => packages}/tests/fixtures/avatar-resized-120x120.gif (100%) rename {server => packages}/tests/fixtures/avatar-resized-120x120.png (100%) rename {server => packages}/tests/fixtures/avatar-resized-48x48.gif (100%) rename {server => packages}/tests/fixtures/avatar-resized-48x48.png (100%) rename {server => packages}/tests/fixtures/avatar.gif (100%) rename {server => packages}/tests/fixtures/avatar.png (100%) rename {server => packages}/tests/fixtures/avatar2-resized-120x120.png (100%) rename {server => packages}/tests/fixtures/avatar2-resized-48x48.png (100%) rename {server => packages}/tests/fixtures/avatar2.png (100%) rename {server => packages}/tests/fixtures/banner-resized.jpg (100%) rename {server => packages}/tests/fixtures/banner.jpg (100%) rename {server => packages}/tests/fixtures/custom-preview-big.png (100%) rename {server => packages}/tests/fixtures/custom-preview.jpg (100%) rename {server => packages}/tests/fixtures/custom-thumbnail-big.jpg (100%) rename {server => packages}/tests/fixtures/custom-thumbnail.jpg (100%) rename {server => packages}/tests/fixtures/custom-thumbnail.png (100%) rename {server => packages}/tests/fixtures/exif.jpg (100%) rename {server => packages}/tests/fixtures/exif.png (100%) rename {server => packages}/tests/fixtures/live/0-000067.ts (100%) rename {server => packages}/tests/fixtures/live/0-000068.ts (100%) rename {server => packages}/tests/fixtures/live/0-000069.ts (100%) rename {server => packages}/tests/fixtures/live/0-000070.ts (100%) rename {server => packages}/tests/fixtures/live/0.m3u8 (100%) rename {server => packages}/tests/fixtures/live/1-000067.ts (100%) rename {server => packages}/tests/fixtures/live/1-000068.ts (100%) rename {server => packages}/tests/fixtures/live/1-000069.ts (100%) rename {server => packages}/tests/fixtures/live/1-000070.ts (100%) rename {server => packages}/tests/fixtures/live/1.m3u8 (100%) rename {server => packages}/tests/fixtures/live/master.m3u8 (100%) rename {server => packages}/tests/fixtures/low-bitrate.mp4 (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-broken/main.js (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-broken/package.json (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-external-auth-one/main.js (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-external-auth-one/package.json (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-external-auth-three/main.js (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-external-auth-three/package.json (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-external-auth-two/main.js (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-external-auth-two/package.json (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-filter-translations/languages/fr.json (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-filter-translations/languages/it.json (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-filter-translations/main.js (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-filter-translations/package.json (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-five/main.js (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-five/package.json (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-four/main.js (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-four/package.json (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-id-pass-auth-one/main.js (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-id-pass-auth-one/package.json (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-id-pass-auth-three/main.js (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-id-pass-auth-three/package.json (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-id-pass-auth-two/main.js (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-id-pass-auth-two/package.json (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-native/main.js (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-native/package.json (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-podcast-custom-tags/main.js (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-podcast-custom-tags/package.json (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-six/main.js (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-six/package.json (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-transcoding-one/main.js (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-transcoding-one/package.json (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-transcoding-two/main.js (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-transcoding-two/package.json (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-unloading/lib.js (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-unloading/main.js (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-unloading/package.json (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-video-constants/main.js (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-video-constants/package.json (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-websocket/main.js (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test-websocket/package.json (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test/languages/fr.json (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test/main.js (100%) rename {server => packages}/tests/fixtures/peertube-plugin-test/package.json (100%) rename {server => packages}/tests/fixtures/rtmps.cert (100%) rename {server => packages}/tests/fixtures/rtmps.key (100%) rename {server => packages}/tests/fixtures/sample.ogg (100%) rename {server => packages}/tests/fixtures/subtitle-bad.txt (100%) rename {server => packages}/tests/fixtures/subtitle-good.srt (100%) rename {server => packages}/tests/fixtures/subtitle-good1.vtt (100%) rename {server => packages}/tests/fixtures/subtitle-good2.vtt (100%) rename {server => packages}/tests/fixtures/thumbnail-playlist.jpg (100%) rename {server => packages}/tests/fixtures/video-720p.torrent (100%) rename {server => packages}/tests/fixtures/video_import_preview.jpg (100%) rename {server => packages}/tests/fixtures/video_import_preview_yt_dlp.jpg (100%) rename {server => packages}/tests/fixtures/video_import_thumbnail.jpg (100%) rename {server => packages}/tests/fixtures/video_import_thumbnail_yt_dlp.jpg (100%) rename {server => packages}/tests/fixtures/video_short.avi (100%) rename {server => packages}/tests/fixtures/video_short.mkv (100%) rename {server => packages}/tests/fixtures/video_short.mp4 (100%) rename {server => packages}/tests/fixtures/video_short.mp4.jpg (100%) rename {server => packages}/tests/fixtures/video_short.ogv (100%) rename {server => packages}/tests/fixtures/video_short.ogv.jpg (100%) rename {server => packages}/tests/fixtures/video_short.webm (100%) rename {server => packages}/tests/fixtures/video_short.webm.jpg (100%) rename {server => packages}/tests/fixtures/video_short1-preview.webm.jpg (100%) rename {server => packages}/tests/fixtures/video_short1.webm (100%) rename {server => packages}/tests/fixtures/video_short1.webm.jpg (100%) rename {server => packages}/tests/fixtures/video_short2.webm (100%) rename {server => packages}/tests/fixtures/video_short2.webm.jpg (100%) rename {server => packages}/tests/fixtures/video_short3.webm (100%) rename {server => packages}/tests/fixtures/video_short3.webm.jpg (100%) rename {server => packages}/tests/fixtures/video_short_0p.mp4 (100%) rename {server => packages}/tests/fixtures/video_short_144p.m3u8 (100%) rename {server => packages}/tests/fixtures/video_short_144p.mp4 (100%) rename {server => packages}/tests/fixtures/video_short_240p.m3u8 (100%) rename {server => packages}/tests/fixtures/video_short_240p.mp4 (100%) rename {server => packages}/tests/fixtures/video_short_360p.m3u8 (100%) rename {server => packages}/tests/fixtures/video_short_360p.mp4 (100%) rename {server => packages}/tests/fixtures/video_short_480.webm (100%) rename {server => packages}/tests/fixtures/video_short_480p.m3u8 (100%) rename {server => packages}/tests/fixtures/video_short_480p.mp4 (100%) rename {server => packages}/tests/fixtures/video_short_4k.mp4 (100%) rename {server => packages}/tests/fixtures/video_short_720p.m3u8 (100%) rename {server => packages}/tests/fixtures/video_short_720p.mp4 (100%) rename {server => packages}/tests/fixtures/video_short_fake.webm (100%) rename {server => packages}/tests/fixtures/video_short_mp3_256k.mp4 (100%) rename {server => packages}/tests/fixtures/video_short_no_audio.mp4 (100%) rename {server => packages}/tests/fixtures/video_very_long_10p.mp4 (100%) rename {server => packages}/tests/fixtures/video_very_short_240p.mp4 (100%) create mode 100644 packages/tests/package.json create mode 100644 packages/tests/src/api/activitypub/cleaner.ts create mode 100644 packages/tests/src/api/activitypub/client.ts create mode 100644 packages/tests/src/api/activitypub/fetch.ts create mode 100644 packages/tests/src/api/activitypub/index.ts create mode 100644 packages/tests/src/api/activitypub/refresher.ts create mode 100644 packages/tests/src/api/activitypub/security.ts create mode 100644 packages/tests/src/api/check-params/abuses.ts create mode 100644 packages/tests/src/api/check-params/accounts.ts create mode 100644 packages/tests/src/api/check-params/blocklist.ts create mode 100644 packages/tests/src/api/check-params/bulk.ts create mode 100644 packages/tests/src/api/check-params/channel-import-videos.ts create mode 100644 packages/tests/src/api/check-params/config.ts create mode 100644 packages/tests/src/api/check-params/contact-form.ts create mode 100644 packages/tests/src/api/check-params/custom-pages.ts create mode 100644 packages/tests/src/api/check-params/debug.ts create mode 100644 packages/tests/src/api/check-params/follows.ts create mode 100644 packages/tests/src/api/check-params/index.ts create mode 100644 packages/tests/src/api/check-params/jobs.ts create mode 100644 packages/tests/src/api/check-params/live.ts create mode 100644 packages/tests/src/api/check-params/logs.ts create mode 100644 packages/tests/src/api/check-params/metrics.ts create mode 100644 packages/tests/src/api/check-params/my-user.ts create mode 100644 packages/tests/src/api/check-params/plugins.ts create mode 100644 packages/tests/src/api/check-params/redundancy.ts create mode 100644 packages/tests/src/api/check-params/registrations.ts create mode 100644 packages/tests/src/api/check-params/runners.ts create mode 100644 packages/tests/src/api/check-params/search.ts create mode 100644 packages/tests/src/api/check-params/services.ts create mode 100644 packages/tests/src/api/check-params/transcoding.ts create mode 100644 packages/tests/src/api/check-params/two-factor.ts create mode 100644 packages/tests/src/api/check-params/upload-quota.ts create mode 100644 packages/tests/src/api/check-params/user-notifications.ts create mode 100644 packages/tests/src/api/check-params/user-subscriptions.ts create mode 100644 packages/tests/src/api/check-params/users-admin.ts create mode 100644 packages/tests/src/api/check-params/users-emails.ts create mode 100644 packages/tests/src/api/check-params/video-blacklist.ts create mode 100644 packages/tests/src/api/check-params/video-captions.ts create mode 100644 packages/tests/src/api/check-params/video-channel-syncs.ts create mode 100644 packages/tests/src/api/check-params/video-channels.ts create mode 100644 packages/tests/src/api/check-params/video-comments.ts create mode 100644 packages/tests/src/api/check-params/video-files.ts create mode 100644 packages/tests/src/api/check-params/video-imports.ts create mode 100644 packages/tests/src/api/check-params/video-passwords.ts create mode 100644 packages/tests/src/api/check-params/video-playlists.ts create mode 100644 packages/tests/src/api/check-params/video-source.ts create mode 100644 packages/tests/src/api/check-params/video-storyboards.ts create mode 100644 packages/tests/src/api/check-params/video-studio.ts create mode 100644 packages/tests/src/api/check-params/video-token.ts create mode 100644 packages/tests/src/api/check-params/videos-common-filters.ts create mode 100644 packages/tests/src/api/check-params/videos-history.ts create mode 100644 packages/tests/src/api/check-params/videos-overviews.ts create mode 100644 packages/tests/src/api/check-params/videos.ts create mode 100644 packages/tests/src/api/check-params/views.ts create mode 100644 packages/tests/src/api/live/index.ts create mode 100644 packages/tests/src/api/live/live-constraints.ts create mode 100644 packages/tests/src/api/live/live-fast-restream.ts create mode 100644 packages/tests/src/api/live/live-permanent.ts create mode 100644 packages/tests/src/api/live/live-rtmps.ts create mode 100644 packages/tests/src/api/live/live-save-replay.ts create mode 100644 packages/tests/src/api/live/live-socket-messages.ts create mode 100644 packages/tests/src/api/live/live.ts create mode 100644 packages/tests/src/api/moderation/abuses.ts create mode 100644 packages/tests/src/api/moderation/blocklist-notification.ts create mode 100644 packages/tests/src/api/moderation/blocklist.ts create mode 100644 packages/tests/src/api/moderation/index.ts create mode 100644 packages/tests/src/api/moderation/video-blacklist.ts create mode 100644 packages/tests/src/api/notifications/admin-notifications.ts create mode 100644 packages/tests/src/api/notifications/comments-notifications.ts create mode 100644 packages/tests/src/api/notifications/index.ts create mode 100644 packages/tests/src/api/notifications/moderation-notifications.ts create mode 100644 packages/tests/src/api/notifications/notifications-api.ts create mode 100644 packages/tests/src/api/notifications/registrations-notifications.ts create mode 100644 packages/tests/src/api/notifications/user-notifications.ts create mode 100644 packages/tests/src/api/object-storage/index.ts create mode 100644 packages/tests/src/api/object-storage/live.ts create mode 100644 packages/tests/src/api/object-storage/video-imports.ts create mode 100644 packages/tests/src/api/object-storage/video-static-file-privacy.ts create mode 100644 packages/tests/src/api/object-storage/videos.ts create mode 100644 packages/tests/src/api/redundancy/index.ts create mode 100644 packages/tests/src/api/redundancy/manage-redundancy.ts create mode 100644 packages/tests/src/api/redundancy/redundancy-constraints.ts create mode 100644 packages/tests/src/api/redundancy/redundancy.ts create mode 100644 packages/tests/src/api/runners/index.ts create mode 100644 packages/tests/src/api/runners/runner-common.ts create mode 100644 packages/tests/src/api/runners/runner-live-transcoding.ts create mode 100644 packages/tests/src/api/runners/runner-socket.ts create mode 100644 packages/tests/src/api/runners/runner-studio-transcoding.ts create mode 100644 packages/tests/src/api/runners/runner-vod-transcoding.ts create mode 100644 packages/tests/src/api/search/index.ts create mode 100644 packages/tests/src/api/search/search-activitypub-video-channels.ts create mode 100644 packages/tests/src/api/search/search-activitypub-video-playlists.ts create mode 100644 packages/tests/src/api/search/search-activitypub-videos.ts create mode 100644 packages/tests/src/api/search/search-channels.ts create mode 100644 packages/tests/src/api/search/search-index.ts create mode 100644 packages/tests/src/api/search/search-playlists.ts create mode 100644 packages/tests/src/api/search/search-videos.ts create mode 100644 packages/tests/src/api/server/auto-follows.ts create mode 100644 packages/tests/src/api/server/bulk.ts create mode 100644 packages/tests/src/api/server/config-defaults.ts create mode 100644 packages/tests/src/api/server/config.ts create mode 100644 packages/tests/src/api/server/contact-form.ts create mode 100644 packages/tests/src/api/server/email.ts create mode 100644 packages/tests/src/api/server/follow-constraints.ts create mode 100644 packages/tests/src/api/server/follows-moderation.ts create mode 100644 packages/tests/src/api/server/follows.ts create mode 100644 packages/tests/src/api/server/handle-down.ts create mode 100644 packages/tests/src/api/server/homepage.ts create mode 100644 packages/tests/src/api/server/index.ts create mode 100644 packages/tests/src/api/server/jobs.ts create mode 100644 packages/tests/src/api/server/logs.ts create mode 100644 packages/tests/src/api/server/no-client.ts create mode 100644 packages/tests/src/api/server/open-telemetry.ts create mode 100644 packages/tests/src/api/server/plugins.ts create mode 100644 packages/tests/src/api/server/proxy.ts create mode 100644 packages/tests/src/api/server/reverse-proxy.ts create mode 100644 packages/tests/src/api/server/services.ts create mode 100644 packages/tests/src/api/server/slow-follows.ts create mode 100644 packages/tests/src/api/server/stats.ts create mode 100644 packages/tests/src/api/server/tracker.ts create mode 100644 packages/tests/src/api/transcoding/audio-only.ts create mode 100644 packages/tests/src/api/transcoding/create-transcoding.ts create mode 100644 packages/tests/src/api/transcoding/hls.ts create mode 100644 packages/tests/src/api/transcoding/index.ts create mode 100644 packages/tests/src/api/transcoding/transcoder.ts create mode 100644 packages/tests/src/api/transcoding/update-while-transcoding.ts create mode 100644 packages/tests/src/api/transcoding/video-studio.ts create mode 100644 packages/tests/src/api/users/index.ts create mode 100644 packages/tests/src/api/users/oauth.ts create mode 100644 packages/tests/src/api/users/registrations.ts create mode 100644 packages/tests/src/api/users/two-factor.ts create mode 100644 packages/tests/src/api/users/user-subscriptions.ts create mode 100644 packages/tests/src/api/users/user-videos.ts create mode 100644 packages/tests/src/api/users/users-email-verification.ts create mode 100644 packages/tests/src/api/users/users-multiple-servers.ts create mode 100644 packages/tests/src/api/users/users.ts create mode 100644 packages/tests/src/api/videos/channel-import-videos.ts create mode 100644 packages/tests/src/api/videos/index.ts create mode 100644 packages/tests/src/api/videos/multiple-servers.ts create mode 100644 packages/tests/src/api/videos/resumable-upload.ts create mode 100644 packages/tests/src/api/videos/single-server.ts create mode 100644 packages/tests/src/api/videos/video-captions.ts create mode 100644 packages/tests/src/api/videos/video-change-ownership.ts create mode 100644 packages/tests/src/api/videos/video-channel-syncs.ts create mode 100644 packages/tests/src/api/videos/video-channels.ts create mode 100644 packages/tests/src/api/videos/video-comments.ts create mode 100644 packages/tests/src/api/videos/video-description.ts create mode 100644 packages/tests/src/api/videos/video-files.ts create mode 100644 packages/tests/src/api/videos/video-imports.ts create mode 100644 packages/tests/src/api/videos/video-nsfw.ts create mode 100644 packages/tests/src/api/videos/video-passwords.ts create mode 100644 packages/tests/src/api/videos/video-playlist-thumbnails.ts create mode 100644 packages/tests/src/api/videos/video-playlists.ts create mode 100644 packages/tests/src/api/videos/video-privacy.ts create mode 100644 packages/tests/src/api/videos/video-schedule-update.ts create mode 100644 packages/tests/src/api/videos/video-source.ts create mode 100644 packages/tests/src/api/videos/video-static-file-privacy.ts create mode 100644 packages/tests/src/api/videos/video-storyboard.ts create mode 100644 packages/tests/src/api/videos/videos-common-filters.ts create mode 100644 packages/tests/src/api/videos/videos-history.ts create mode 100644 packages/tests/src/api/videos/videos-overview.ts create mode 100644 packages/tests/src/api/views/index.ts create mode 100644 packages/tests/src/api/views/video-views-counter.ts create mode 100644 packages/tests/src/api/views/video-views-overall-stats.ts create mode 100644 packages/tests/src/api/views/video-views-retention-stats.ts create mode 100644 packages/tests/src/api/views/video-views-timeserie-stats.ts create mode 100644 packages/tests/src/api/views/videos-views-cleaner.ts create mode 100644 packages/tests/src/cli/create-generate-storyboard-job.ts create mode 100644 packages/tests/src/cli/create-import-video-file-job.ts create mode 100644 packages/tests/src/cli/create-move-video-storage-job.ts rename {server/tests => packages/tests/src}/cli/index.ts (100%) create mode 100644 packages/tests/src/cli/peertube.ts create mode 100644 packages/tests/src/cli/plugins.ts create mode 100644 packages/tests/src/cli/prune-storage.ts create mode 100644 packages/tests/src/cli/regenerate-thumbnails.ts create mode 100644 packages/tests/src/cli/reset-password.ts create mode 100644 packages/tests/src/cli/update-host.ts create mode 100644 packages/tests/src/client.ts create mode 100644 packages/tests/src/external-plugins/akismet.ts create mode 100644 packages/tests/src/external-plugins/auth-ldap.ts create mode 100644 packages/tests/src/external-plugins/auto-block-videos.ts create mode 100644 packages/tests/src/external-plugins/auto-mute.ts rename {server/tests => packages/tests/src}/external-plugins/index.ts (100%) create mode 100644 packages/tests/src/feeds/feeds.ts rename {server/tests => packages/tests/src}/feeds/index.ts (100%) create mode 100644 packages/tests/src/misc-endpoints.ts create mode 100644 packages/tests/src/peertube-runner/client-cli.ts create mode 100644 packages/tests/src/peertube-runner/index.ts create mode 100644 packages/tests/src/peertube-runner/live-transcoding.ts create mode 100644 packages/tests/src/peertube-runner/studio-transcoding.ts create mode 100644 packages/tests/src/peertube-runner/vod-transcoding.ts create mode 100644 packages/tests/src/plugins/action-hooks.ts create mode 100644 packages/tests/src/plugins/external-auth.ts create mode 100644 packages/tests/src/plugins/filter-hooks.ts create mode 100644 packages/tests/src/plugins/html-injection.ts create mode 100644 packages/tests/src/plugins/id-and-pass-auth.ts rename {server/tests => packages/tests/src}/plugins/index.ts (100%) create mode 100644 packages/tests/src/plugins/plugin-helpers.ts create mode 100644 packages/tests/src/plugins/plugin-router.ts create mode 100644 packages/tests/src/plugins/plugin-storage.ts create mode 100644 packages/tests/src/plugins/plugin-transcoding.ts create mode 100644 packages/tests/src/plugins/plugin-unloading.ts create mode 100644 packages/tests/src/plugins/plugin-websocket.ts create mode 100644 packages/tests/src/plugins/translations.ts create mode 100644 packages/tests/src/plugins/video-constants.ts create mode 100644 packages/tests/src/server-helpers/activitypub.ts create mode 100644 packages/tests/src/server-helpers/core-utils.ts create mode 100644 packages/tests/src/server-helpers/crypto.ts create mode 100644 packages/tests/src/server-helpers/dns.ts create mode 100644 packages/tests/src/server-helpers/image.ts create mode 100644 packages/tests/src/server-helpers/index.ts create mode 100644 packages/tests/src/server-helpers/markdown.ts create mode 100644 packages/tests/src/server-helpers/mentions.ts create mode 100644 packages/tests/src/server-helpers/request.ts create mode 100644 packages/tests/src/server-helpers/validator.ts create mode 100644 packages/tests/src/server-helpers/version.ts create mode 100644 packages/tests/src/server-lib/index.ts create mode 100644 packages/tests/src/server-lib/video-constant-registry-factory.ts create mode 100644 packages/tests/src/shared/actors.ts create mode 100644 packages/tests/src/shared/captions.ts create mode 100644 packages/tests/src/shared/checks.ts create mode 100644 packages/tests/src/shared/directories.ts create mode 100644 packages/tests/src/shared/generate.ts create mode 100644 packages/tests/src/shared/live.ts create mode 100644 packages/tests/src/shared/mock-servers/index.ts create mode 100644 packages/tests/src/shared/mock-servers/mock-429.ts create mode 100644 packages/tests/src/shared/mock-servers/mock-email.ts create mode 100644 packages/tests/src/shared/mock-servers/mock-http.ts create mode 100644 packages/tests/src/shared/mock-servers/mock-instances-index.ts create mode 100644 packages/tests/src/shared/mock-servers/mock-joinpeertube-versions.ts create mode 100644 packages/tests/src/shared/mock-servers/mock-object-storage.ts create mode 100644 packages/tests/src/shared/mock-servers/mock-plugin-blocklist.ts create mode 100644 packages/tests/src/shared/mock-servers/mock-proxy.ts rename {server/tests => packages/tests/src}/shared/mock-servers/shared.ts (100%) create mode 100644 packages/tests/src/shared/notifications.ts create mode 100644 packages/tests/src/shared/peertube-runner-process.ts create mode 100644 packages/tests/src/shared/plugins.ts create mode 100644 packages/tests/src/shared/requests.ts create mode 100644 packages/tests/src/shared/sql-command.ts create mode 100644 packages/tests/src/shared/streaming-playlists.ts rename {server/tests => packages/tests/src}/shared/tests.ts (100%) create mode 100644 packages/tests/src/shared/tracker.ts create mode 100644 packages/tests/src/shared/video-playlists.ts create mode 100644 packages/tests/src/shared/videos.ts create mode 100644 packages/tests/src/shared/views.ts create mode 100644 packages/tests/src/shared/webtorrent.ts create mode 100644 packages/tests/tsconfig.json rename packages/{types => types-generator}/README.md (100%) create mode 100644 packages/types-generator/generate-package.ts create mode 100644 packages/types-generator/package.json create mode 100644 packages/types-generator/src/client/index.ts create mode 100644 packages/types-generator/src/client/tsconfig.types.json create mode 100644 packages/types-generator/src/index.ts create mode 100644 packages/types-generator/tests/test.ts create mode 100644 packages/types-generator/tsconfig.dist.json create mode 100644 packages/types-generator/tsconfig.json create mode 100644 packages/types-generator/tsconfig.types.json delete mode 100644 packages/types/generate-package.ts delete mode 100644 packages/types/src/client/index.ts delete mode 100644 packages/types/src/client/tsconfig.json delete mode 100644 packages/types/src/index.ts delete mode 100644 packages/types/tests/test.ts delete mode 100644 packages/types/tsconfig.dist.json delete mode 100644 packages/types/tsconfig.json create mode 100644 packages/typescript-utils/package.json create mode 100644 packages/typescript-utils/src/index.ts rename {shared/typescript-utils => packages/typescript-utils/src}/types.ts (100%) create mode 100644 packages/typescript-utils/tsconfig.json create mode 100644 packages/typescript-utils/tsconfig.types.json create mode 100644 scripts/build/peertube-cli.sh create mode 100755 scripts/build/tests.sh delete mode 100644 scripts/create-generate-storyboard-job.ts delete mode 100644 scripts/create-import-video-file-job.ts delete mode 100644 scripts/create-move-video-storage-job.ts delete mode 100755 scripts/dev/cli.sh create mode 100755 scripts/dev/peertube-cli.sh delete mode 100644 scripts/migrations/peertube-4.0.ts delete mode 100644 scripts/migrations/peertube-4.2.ts delete mode 100644 scripts/migrations/peertube-5.0.ts delete mode 100755 scripts/parse-log.ts delete mode 100755 scripts/plugin/install.ts delete mode 100755 scripts/plugin/uninstall.ts delete mode 100755 scripts/prune-storage.ts delete mode 100644 scripts/regenerate-thumbnails.ts delete mode 100755 scripts/reset-password.ts delete mode 100755 scripts/setup/cli.sh delete mode 100755 scripts/update-host.ts delete mode 100644 server.ts delete mode 100644 server/controllers/activitypub/client.ts delete mode 100644 server/controllers/activitypub/inbox.ts delete mode 100644 server/controllers/activitypub/index.ts delete mode 100644 server/controllers/activitypub/outbox.ts delete mode 100644 server/controllers/api/abuse.ts delete mode 100644 server/controllers/api/accounts.ts delete mode 100644 server/controllers/api/blocklist.ts delete mode 100644 server/controllers/api/bulk.ts delete mode 100644 server/controllers/api/config.ts delete mode 100644 server/controllers/api/custom-page.ts delete mode 100644 server/controllers/api/index.ts delete mode 100644 server/controllers/api/jobs.ts delete mode 100644 server/controllers/api/metrics.ts delete mode 100644 server/controllers/api/oauth-clients.ts delete mode 100644 server/controllers/api/overviews.ts delete mode 100644 server/controllers/api/plugins.ts delete mode 100644 server/controllers/api/runners/index.ts delete mode 100644 server/controllers/api/runners/jobs-files.ts delete mode 100644 server/controllers/api/runners/jobs.ts delete mode 100644 server/controllers/api/runners/manage-runners.ts delete mode 100644 server/controllers/api/runners/registration-tokens.ts delete mode 100644 server/controllers/api/search/index.ts delete mode 100644 server/controllers/api/search/search-video-channels.ts delete mode 100644 server/controllers/api/search/search-video-playlists.ts delete mode 100644 server/controllers/api/search/search-videos.ts delete mode 100644 server/controllers/api/search/shared/index.ts delete mode 100644 server/controllers/api/server/contact.ts delete mode 100644 server/controllers/api/server/debug.ts delete mode 100644 server/controllers/api/server/follows.ts delete mode 100644 server/controllers/api/server/index.ts delete mode 100644 server/controllers/api/server/logs.ts delete mode 100644 server/controllers/api/server/redundancy.ts delete mode 100644 server/controllers/api/server/server-blocklist.ts delete mode 100644 server/controllers/api/server/stats.ts delete mode 100644 server/controllers/api/users/email-verification.ts delete mode 100644 server/controllers/api/users/index.ts delete mode 100644 server/controllers/api/users/me.ts delete mode 100644 server/controllers/api/users/my-abuses.ts delete mode 100644 server/controllers/api/users/my-blocklist.ts delete mode 100644 server/controllers/api/users/my-history.ts delete mode 100644 server/controllers/api/users/my-notifications.ts delete mode 100644 server/controllers/api/users/my-subscriptions.ts delete mode 100644 server/controllers/api/users/my-video-playlists.ts delete mode 100644 server/controllers/api/users/registrations.ts delete mode 100644 server/controllers/api/users/token.ts delete mode 100644 server/controllers/api/users/two-factor.ts delete mode 100644 server/controllers/api/video-channel-sync.ts delete mode 100644 server/controllers/api/video-channel.ts delete mode 100644 server/controllers/api/video-playlist.ts delete mode 100644 server/controllers/api/videos/blacklist.ts delete mode 100644 server/controllers/api/videos/captions.ts delete mode 100644 server/controllers/api/videos/comment.ts delete mode 100644 server/controllers/api/videos/files.ts delete mode 100644 server/controllers/api/videos/import.ts delete mode 100644 server/controllers/api/videos/index.ts delete mode 100644 server/controllers/api/videos/live.ts delete mode 100644 server/controllers/api/videos/ownership.ts delete mode 100644 server/controllers/api/videos/passwords.ts delete mode 100644 server/controllers/api/videos/rate.ts delete mode 100644 server/controllers/api/videos/source.ts delete mode 100644 server/controllers/api/videos/stats.ts delete mode 100644 server/controllers/api/videos/storyboard.ts delete mode 100644 server/controllers/api/videos/studio.ts delete mode 100644 server/controllers/api/videos/token.ts delete mode 100644 server/controllers/api/videos/transcoding.ts delete mode 100644 server/controllers/api/videos/update.ts delete mode 100644 server/controllers/api/videos/upload.ts delete mode 100644 server/controllers/api/videos/view.ts delete mode 100644 server/controllers/client.ts delete mode 100644 server/controllers/download.ts delete mode 100644 server/controllers/feeds/comment-feeds.ts delete mode 100644 server/controllers/feeds/index.ts delete mode 100644 server/controllers/feeds/shared/common-feed-utils.ts delete mode 100644 server/controllers/feeds/shared/index.ts delete mode 100644 server/controllers/feeds/shared/video-feed-utils.ts delete mode 100644 server/controllers/feeds/video-feeds.ts delete mode 100644 server/controllers/feeds/video-podcast-feeds.ts delete mode 100644 server/controllers/index.ts delete mode 100644 server/controllers/lazy-static.ts delete mode 100644 server/controllers/misc.ts delete mode 100644 server/controllers/object-storage-proxy.ts delete mode 100644 server/controllers/plugins.ts delete mode 100644 server/controllers/services.ts delete mode 100644 server/controllers/sitemap.ts delete mode 100644 server/controllers/static.ts delete mode 100644 server/controllers/tracker.ts delete mode 100644 server/controllers/well-known.ts delete mode 100644 server/helpers/actors.ts delete mode 100644 server/helpers/audit-logger.ts delete mode 100644 server/helpers/captions-utils.ts delete mode 100644 server/helpers/core-utils.ts delete mode 100644 server/helpers/custom-jsonld-signature.ts delete mode 100644 server/helpers/custom-validators/abuses.ts delete mode 100644 server/helpers/custom-validators/accounts.ts delete mode 100644 server/helpers/custom-validators/activitypub/activity.ts delete mode 100644 server/helpers/custom-validators/activitypub/actor.ts delete mode 100644 server/helpers/custom-validators/activitypub/cache-file.ts delete mode 100644 server/helpers/custom-validators/activitypub/misc.ts delete mode 100644 server/helpers/custom-validators/activitypub/playlist.ts delete mode 100644 server/helpers/custom-validators/activitypub/signature.ts delete mode 100644 server/helpers/custom-validators/activitypub/video-comments.ts delete mode 100644 server/helpers/custom-validators/activitypub/videos.ts delete mode 100644 server/helpers/custom-validators/activitypub/watch-action.ts delete mode 100644 server/helpers/custom-validators/actor-images.ts delete mode 100644 server/helpers/custom-validators/feeds.ts delete mode 100644 server/helpers/custom-validators/follows.ts delete mode 100644 server/helpers/custom-validators/jobs.ts delete mode 100644 server/helpers/custom-validators/logs.ts delete mode 100644 server/helpers/custom-validators/misc.ts delete mode 100644 server/helpers/custom-validators/plugins.ts delete mode 100644 server/helpers/custom-validators/runners/jobs.ts delete mode 100644 server/helpers/custom-validators/runners/runners.ts delete mode 100644 server/helpers/custom-validators/search.ts delete mode 100644 server/helpers/custom-validators/servers.ts delete mode 100644 server/helpers/custom-validators/user-notifications.ts delete mode 100644 server/helpers/custom-validators/user-registration.ts delete mode 100644 server/helpers/custom-validators/users.ts delete mode 100644 server/helpers/custom-validators/video-blacklist.ts delete mode 100644 server/helpers/custom-validators/video-captions.ts delete mode 100644 server/helpers/custom-validators/video-channel-syncs.ts delete mode 100644 server/helpers/custom-validators/video-channels.ts delete mode 100644 server/helpers/custom-validators/video-comments.ts delete mode 100644 server/helpers/custom-validators/video-imports.ts delete mode 100644 server/helpers/custom-validators/video-lives.ts delete mode 100644 server/helpers/custom-validators/video-ownership.ts delete mode 100644 server/helpers/custom-validators/video-playlists.ts delete mode 100644 server/helpers/custom-validators/video-redundancies.ts delete mode 100644 server/helpers/custom-validators/video-stats.ts delete mode 100644 server/helpers/custom-validators/video-studio.ts delete mode 100644 server/helpers/custom-validators/video-transcoding.ts delete mode 100644 server/helpers/custom-validators/video-view.ts delete mode 100644 server/helpers/custom-validators/videos.ts delete mode 100644 server/helpers/custom-validators/webfinger.ts delete mode 100644 server/helpers/database-utils.ts delete mode 100644 server/helpers/decache.ts delete mode 100644 server/helpers/dns.ts delete mode 100644 server/helpers/express-utils.ts delete mode 100644 server/helpers/ffmpeg/codecs.ts delete mode 100644 server/helpers/ffmpeg/ffmpeg-image.ts delete mode 100644 server/helpers/ffmpeg/ffmpeg-options.ts delete mode 100644 server/helpers/ffmpeg/framerate.ts delete mode 100644 server/helpers/ffmpeg/index.ts delete mode 100644 server/helpers/geo-ip.ts delete mode 100644 server/helpers/image-utils.ts delete mode 100644 server/helpers/logger.ts delete mode 100644 server/helpers/markdown.ts delete mode 100644 server/helpers/otp.ts delete mode 100644 server/helpers/peertube-crypto.ts delete mode 100644 server/helpers/query.ts delete mode 100644 server/helpers/requests.ts delete mode 100644 server/helpers/token-generator.ts delete mode 100644 server/helpers/upload.ts delete mode 100644 server/helpers/utils.ts delete mode 100644 server/helpers/version.ts delete mode 100644 server/helpers/video.ts delete mode 100644 server/helpers/webtorrent.ts delete mode 100644 server/helpers/youtube-dl/index.ts delete mode 100644 server/helpers/youtube-dl/youtube-dl-cli.ts delete mode 100644 server/helpers/youtube-dl/youtube-dl-info-builder.ts delete mode 100644 server/helpers/youtube-dl/youtube-dl-wrapper.ts delete mode 100644 server/initializers/checker-after-init.ts delete mode 100644 server/initializers/checker-before-init.ts delete mode 100644 server/initializers/config.ts delete mode 100644 server/initializers/constants.ts delete mode 100644 server/initializers/database.ts delete mode 100644 server/initializers/installer.ts delete mode 100644 server/initializers/migrations/0530-playlist-multiple-video.ts delete mode 100644 server/initializers/migrations/0560-user-feed-token.ts delete mode 100644 server/initializers/migrations/0605-actor-missing-keys.ts delete mode 100644 server/initializers/migrations/0660-object-storage.ts delete mode 100644 server/initializers/migrations/0690-live-latency-mode.ts delete mode 100644 server/initializers/migrator.ts delete mode 100644 server/lib/activitypub/activity.ts delete mode 100644 server/lib/activitypub/actors/get.ts delete mode 100644 server/lib/activitypub/actors/image.ts delete mode 100644 server/lib/activitypub/actors/index.ts delete mode 100644 server/lib/activitypub/actors/keys.ts delete mode 100644 server/lib/activitypub/actors/refresh.ts delete mode 100644 server/lib/activitypub/actors/shared/creator.ts delete mode 100644 server/lib/activitypub/actors/shared/index.ts delete mode 100644 server/lib/activitypub/actors/shared/object-to-model-attributes.ts delete mode 100644 server/lib/activitypub/actors/shared/url-to-object.ts delete mode 100644 server/lib/activitypub/actors/updater.ts delete mode 100644 server/lib/activitypub/actors/webfinger.ts delete mode 100644 server/lib/activitypub/audience.ts delete mode 100644 server/lib/activitypub/cache-file.ts delete mode 100644 server/lib/activitypub/collection.ts delete mode 100644 server/lib/activitypub/context.ts delete mode 100644 server/lib/activitypub/crawl.ts delete mode 100644 server/lib/activitypub/follow.ts delete mode 100644 server/lib/activitypub/inbox-manager.ts delete mode 100644 server/lib/activitypub/local-video-viewer.ts delete mode 100644 server/lib/activitypub/outbox.ts delete mode 100644 server/lib/activitypub/playlists/create-update.ts delete mode 100644 server/lib/activitypub/playlists/get.ts delete mode 100644 server/lib/activitypub/playlists/index.ts delete mode 100644 server/lib/activitypub/playlists/refresh.ts delete mode 100644 server/lib/activitypub/playlists/shared/index.ts delete mode 100644 server/lib/activitypub/playlists/shared/object-to-model-attributes.ts delete mode 100644 server/lib/activitypub/playlists/shared/url-to-object.ts delete mode 100644 server/lib/activitypub/process/index.ts delete mode 100644 server/lib/activitypub/process/process-accept.ts delete mode 100644 server/lib/activitypub/process/process-announce.ts delete mode 100644 server/lib/activitypub/process/process-create.ts delete mode 100644 server/lib/activitypub/process/process-delete.ts delete mode 100644 server/lib/activitypub/process/process-dislike.ts delete mode 100644 server/lib/activitypub/process/process-flag.ts delete mode 100644 server/lib/activitypub/process/process-follow.ts delete mode 100644 server/lib/activitypub/process/process-like.ts delete mode 100644 server/lib/activitypub/process/process-reject.ts delete mode 100644 server/lib/activitypub/process/process-undo.ts delete mode 100644 server/lib/activitypub/process/process-update.ts delete mode 100644 server/lib/activitypub/process/process-view.ts delete mode 100644 server/lib/activitypub/process/process.ts delete mode 100644 server/lib/activitypub/send/http.ts delete mode 100644 server/lib/activitypub/send/index.ts delete mode 100644 server/lib/activitypub/send/send-accept.ts delete mode 100644 server/lib/activitypub/send/send-announce.ts delete mode 100644 server/lib/activitypub/send/send-create.ts delete mode 100644 server/lib/activitypub/send/send-delete.ts delete mode 100644 server/lib/activitypub/send/send-dislike.ts delete mode 100644 server/lib/activitypub/send/send-flag.ts delete mode 100644 server/lib/activitypub/send/send-follow.ts delete mode 100644 server/lib/activitypub/send/send-like.ts delete mode 100644 server/lib/activitypub/send/send-reject.ts delete mode 100644 server/lib/activitypub/send/send-undo.ts delete mode 100644 server/lib/activitypub/send/send-update.ts delete mode 100644 server/lib/activitypub/send/send-view.ts delete mode 100644 server/lib/activitypub/send/shared/audience-utils.ts delete mode 100644 server/lib/activitypub/send/shared/index.ts delete mode 100644 server/lib/activitypub/send/shared/send-utils.ts delete mode 100644 server/lib/activitypub/share.ts delete mode 100644 server/lib/activitypub/url.ts delete mode 100644 server/lib/activitypub/video-comments.ts delete mode 100644 server/lib/activitypub/video-rates.ts delete mode 100644 server/lib/activitypub/videos/federate.ts delete mode 100644 server/lib/activitypub/videos/get.ts delete mode 100644 server/lib/activitypub/videos/index.ts delete mode 100644 server/lib/activitypub/videos/refresh.ts delete mode 100644 server/lib/activitypub/videos/shared/abstract-builder.ts delete mode 100644 server/lib/activitypub/videos/shared/creator.ts delete mode 100644 server/lib/activitypub/videos/shared/index.ts delete mode 100644 server/lib/activitypub/videos/shared/object-to-model-attributes.ts delete mode 100644 server/lib/activitypub/videos/shared/trackers.ts delete mode 100644 server/lib/activitypub/videos/shared/url-to-object.ts delete mode 100644 server/lib/activitypub/videos/shared/video-sync-attributes.ts delete mode 100644 server/lib/activitypub/videos/updater.ts delete mode 100644 server/lib/actor-follow-health-cache.ts delete mode 100644 server/lib/actor-image.ts delete mode 100644 server/lib/auth/external-auth.ts delete mode 100644 server/lib/auth/oauth-model.ts delete mode 100644 server/lib/auth/oauth.ts delete mode 100644 server/lib/auth/tokens-cache.ts delete mode 100644 server/lib/blocklist.ts delete mode 100644 server/lib/client-html.ts delete mode 100644 server/lib/emailer.ts delete mode 100644 server/lib/files-cache/avatar-permanent-file-cache.ts delete mode 100644 server/lib/files-cache/index.ts delete mode 100644 server/lib/files-cache/shared/abstract-permanent-file-cache.ts delete mode 100644 server/lib/files-cache/shared/abstract-simple-file-cache.ts delete mode 100644 server/lib/files-cache/shared/index.ts delete mode 100644 server/lib/files-cache/video-captions-simple-file-cache.ts delete mode 100644 server/lib/files-cache/video-miniature-permanent-file-cache.ts delete mode 100644 server/lib/files-cache/video-previews-simple-file-cache.ts delete mode 100644 server/lib/files-cache/video-storyboards-simple-file-cache.ts delete mode 100644 server/lib/files-cache/video-torrents-simple-file-cache.ts delete mode 100644 server/lib/hls.ts delete mode 100644 server/lib/internal-event-emitter.ts delete mode 100644 server/lib/job-queue/handlers/activitypub-cleaner.ts delete mode 100644 server/lib/job-queue/handlers/activitypub-follow.ts delete mode 100644 server/lib/job-queue/handlers/activitypub-http-broadcast.ts delete mode 100644 server/lib/job-queue/handlers/activitypub-http-fetcher.ts delete mode 100644 server/lib/job-queue/handlers/activitypub-http-unicast.ts delete mode 100644 server/lib/job-queue/handlers/activitypub-refresher.ts delete mode 100644 server/lib/job-queue/handlers/actor-keys.ts delete mode 100644 server/lib/job-queue/handlers/after-video-channel-import.ts delete mode 100644 server/lib/job-queue/handlers/email.ts delete mode 100644 server/lib/job-queue/handlers/federate-video.ts delete mode 100644 server/lib/job-queue/handlers/generate-storyboard.ts delete mode 100644 server/lib/job-queue/handlers/manage-video-torrent.ts delete mode 100644 server/lib/job-queue/handlers/move-to-object-storage.ts delete mode 100644 server/lib/job-queue/handlers/notify.ts delete mode 100644 server/lib/job-queue/handlers/transcoding-job-builder.ts delete mode 100644 server/lib/job-queue/handlers/video-channel-import.ts delete mode 100644 server/lib/job-queue/handlers/video-file-import.ts delete mode 100644 server/lib/job-queue/handlers/video-import.ts delete mode 100644 server/lib/job-queue/handlers/video-live-ending.ts delete mode 100644 server/lib/job-queue/handlers/video-redundancy.ts delete mode 100644 server/lib/job-queue/handlers/video-studio-edition.ts delete mode 100644 server/lib/job-queue/handlers/video-transcoding.ts delete mode 100644 server/lib/job-queue/handlers/video-views-stats.ts delete mode 100644 server/lib/job-queue/index.ts delete mode 100644 server/lib/job-queue/job-queue.ts delete mode 100644 server/lib/live/index.ts delete mode 100644 server/lib/live/live-manager.ts delete mode 100644 server/lib/live/live-segment-sha-store.ts delete mode 100644 server/lib/live/live-utils.ts delete mode 100644 server/lib/live/shared/index.ts delete mode 100644 server/lib/live/shared/muxing-session.ts delete mode 100644 server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts delete mode 100644 server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts delete mode 100644 server/lib/live/shared/transcoding-wrapper/index.ts delete mode 100644 server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts delete mode 100644 server/lib/local-actor.ts delete mode 100644 server/lib/model-loaders/actor.ts delete mode 100644 server/lib/model-loaders/index.ts delete mode 100644 server/lib/model-loaders/video.ts delete mode 100644 server/lib/moderation.ts delete mode 100644 server/lib/notifier/index.ts delete mode 100644 server/lib/notifier/notifier.ts delete mode 100644 server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts delete mode 100644 server/lib/notifier/shared/abuse/abuse-state-change-for-reporter.ts delete mode 100644 server/lib/notifier/shared/abuse/index.ts delete mode 100644 server/lib/notifier/shared/abuse/new-abuse-for-moderators.ts delete mode 100644 server/lib/notifier/shared/abuse/new-abuse-message-for-moderators.ts delete mode 100644 server/lib/notifier/shared/abuse/new-abuse-message-for-reporter.ts delete mode 100644 server/lib/notifier/shared/blacklist/index.ts delete mode 100644 server/lib/notifier/shared/blacklist/new-auto-blacklist-for-moderators.ts delete mode 100644 server/lib/notifier/shared/blacklist/new-blacklist-for-owner.ts delete mode 100644 server/lib/notifier/shared/blacklist/unblacklist-for-owner.ts delete mode 100644 server/lib/notifier/shared/comment/comment-mention.ts delete mode 100644 server/lib/notifier/shared/comment/index.ts delete mode 100644 server/lib/notifier/shared/comment/new-comment-for-video-owner.ts delete mode 100644 server/lib/notifier/shared/common/abstract-notification.ts delete mode 100644 server/lib/notifier/shared/common/index.ts delete mode 100644 server/lib/notifier/shared/follow/auto-follow-for-instance.ts delete mode 100644 server/lib/notifier/shared/follow/follow-for-instance.ts delete mode 100644 server/lib/notifier/shared/follow/follow-for-user.ts delete mode 100644 server/lib/notifier/shared/follow/index.ts delete mode 100644 server/lib/notifier/shared/index.ts delete mode 100644 server/lib/notifier/shared/instance/direct-registration-for-moderators.ts delete mode 100644 server/lib/notifier/shared/instance/index.ts delete mode 100644 server/lib/notifier/shared/instance/new-peertube-version-for-admins.ts delete mode 100644 server/lib/notifier/shared/instance/new-plugin-version-for-admins.ts delete mode 100644 server/lib/notifier/shared/instance/registration-request-for-moderators.ts delete mode 100644 server/lib/notifier/shared/video-publication/abstract-owned-video-publication.ts delete mode 100644 server/lib/notifier/shared/video-publication/import-finished-for-owner.ts delete mode 100644 server/lib/notifier/shared/video-publication/index.ts delete mode 100644 server/lib/notifier/shared/video-publication/new-video-for-subscribers.ts delete mode 100644 server/lib/notifier/shared/video-publication/owned-publication-after-auto-unblacklist.ts delete mode 100644 server/lib/notifier/shared/video-publication/owned-publication-after-schedule-update.ts delete mode 100644 server/lib/notifier/shared/video-publication/owned-publication-after-transcoding.ts delete mode 100644 server/lib/notifier/shared/video-publication/studio-edition-finished-for-owner.ts delete mode 100644 server/lib/object-storage/index.ts delete mode 100644 server/lib/object-storage/keys.ts delete mode 100644 server/lib/object-storage/pre-signed-urls.ts delete mode 100644 server/lib/object-storage/proxy.ts delete mode 100644 server/lib/object-storage/shared/client.ts delete mode 100644 server/lib/object-storage/shared/index.ts delete mode 100644 server/lib/object-storage/shared/logger.ts delete mode 100644 server/lib/object-storage/shared/object-storage-helpers.ts delete mode 100644 server/lib/object-storage/urls.ts delete mode 100644 server/lib/object-storage/videos.ts delete mode 100644 server/lib/opentelemetry/metric-helpers/index.ts delete mode 100644 server/lib/opentelemetry/metric-helpers/job-queue-observers-builder.ts delete mode 100644 server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts delete mode 100644 server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts delete mode 100644 server/lib/opentelemetry/metric-helpers/playback-metrics.ts delete mode 100644 server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts delete mode 100644 server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts delete mode 100644 server/lib/opentelemetry/metrics.ts delete mode 100644 server/lib/opentelemetry/tracing.ts delete mode 100644 server/lib/paths.ts delete mode 100644 server/lib/peertube-socket.ts delete mode 100644 server/lib/plugins/hooks.ts delete mode 100644 server/lib/plugins/plugin-helpers-builder.ts delete mode 100644 server/lib/plugins/plugin-index.ts delete mode 100644 server/lib/plugins/plugin-manager.ts delete mode 100644 server/lib/plugins/register-helpers.ts delete mode 100644 server/lib/plugins/theme-utils.ts delete mode 100644 server/lib/plugins/video-constant-manager-factory.ts delete mode 100644 server/lib/plugins/yarn.ts delete mode 100644 server/lib/redis.ts delete mode 100644 server/lib/redundancy.ts delete mode 100644 server/lib/runners/index.ts delete mode 100644 server/lib/runners/job-handlers/abstract-job-handler.ts delete mode 100644 server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts delete mode 100644 server/lib/runners/job-handlers/index.ts delete mode 100644 server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts delete mode 100644 server/lib/runners/job-handlers/runner-job-handlers.ts delete mode 100644 server/lib/runners/job-handlers/shared/index.ts delete mode 100644 server/lib/runners/job-handlers/shared/vod-helpers.ts delete mode 100644 server/lib/runners/job-handlers/video-studio-transcoding-job-handler.ts delete mode 100644 server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts delete mode 100644 server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts delete mode 100644 server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts delete mode 100644 server/lib/runners/runner-urls.ts delete mode 100644 server/lib/runners/runner.ts delete mode 100644 server/lib/schedulers/abstract-scheduler.ts delete mode 100644 server/lib/schedulers/actor-follow-scheduler.ts delete mode 100644 server/lib/schedulers/auto-follow-index-instances.ts delete mode 100644 server/lib/schedulers/geo-ip-update-scheduler.ts delete mode 100644 server/lib/schedulers/peertube-version-check-scheduler.ts delete mode 100644 server/lib/schedulers/plugins-check-scheduler.ts delete mode 100644 server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts delete mode 100644 server/lib/schedulers/remove-old-history-scheduler.ts delete mode 100644 server/lib/schedulers/remove-old-views-scheduler.ts delete mode 100644 server/lib/schedulers/runner-job-watch-dog-scheduler.ts delete mode 100644 server/lib/schedulers/update-videos-scheduler.ts delete mode 100644 server/lib/schedulers/video-channel-sync-latest-scheduler.ts delete mode 100644 server/lib/schedulers/video-views-buffer-scheduler.ts delete mode 100644 server/lib/schedulers/videos-redundancy-scheduler.ts delete mode 100644 server/lib/schedulers/youtube-dl-update-scheduler.ts delete mode 100644 server/lib/search.ts delete mode 100644 server/lib/server-config-manager.ts delete mode 100644 server/lib/signup.ts delete mode 100644 server/lib/stat-manager.ts delete mode 100644 server/lib/sync-channel.ts delete mode 100644 server/lib/thumbnail.ts delete mode 100644 server/lib/timeserie.ts delete mode 100644 server/lib/transcoding/create-transcoding-job.ts delete mode 100644 server/lib/transcoding/default-transcoding-profiles.ts delete mode 100644 server/lib/transcoding/ended-transcoding.ts delete mode 100644 server/lib/transcoding/hls-transcoding.ts delete mode 100644 server/lib/transcoding/shared/ffmpeg-builder.ts delete mode 100644 server/lib/transcoding/shared/index.ts delete mode 100644 server/lib/transcoding/shared/job-builders/abstract-job-builder.ts delete mode 100644 server/lib/transcoding/shared/job-builders/index.ts delete mode 100644 server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts delete mode 100644 server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts delete mode 100644 server/lib/transcoding/transcoding-priority.ts delete mode 100644 server/lib/transcoding/transcoding-quick-transcode.ts delete mode 100644 server/lib/transcoding/transcoding-resolutions.ts delete mode 100644 server/lib/transcoding/web-transcoding.ts delete mode 100644 server/lib/uploadx.ts delete mode 100644 server/lib/user.ts delete mode 100644 server/lib/video-blacklist.ts delete mode 100644 server/lib/video-channel.ts delete mode 100644 server/lib/video-comment.ts delete mode 100644 server/lib/video-file.ts delete mode 100644 server/lib/video-path-manager.ts delete mode 100644 server/lib/video-playlist.ts delete mode 100644 server/lib/video-pre-import.ts delete mode 100644 server/lib/video-privacy.ts delete mode 100644 server/lib/video-state.ts delete mode 100644 server/lib/video-studio.ts delete mode 100644 server/lib/video-tokens-manager.ts delete mode 100644 server/lib/video-urls.ts delete mode 100644 server/lib/video.ts delete mode 100644 server/lib/views/shared/index.ts delete mode 100644 server/lib/views/shared/video-viewer-counters.ts delete mode 100644 server/lib/views/shared/video-viewer-stats.ts delete mode 100644 server/lib/views/shared/video-views.ts delete mode 100644 server/lib/views/video-views-manager.ts delete mode 100644 server/lib/worker/parent-process.ts delete mode 100644 server/lib/worker/workers/http-broadcast.ts delete mode 100644 server/lib/worker/workers/image-downloader.ts delete mode 100644 server/lib/worker/workers/image-processor.ts delete mode 100644 server/middlewares/activitypub.ts delete mode 100644 server/middlewares/async.ts delete mode 100644 server/middlewares/auth.ts delete mode 100644 server/middlewares/cache/cache.ts delete mode 100644 server/middlewares/cache/index.ts delete mode 100644 server/middlewares/cache/shared/api-cache.ts delete mode 100644 server/middlewares/cache/shared/index.ts delete mode 100644 server/middlewares/csp.ts delete mode 100644 server/middlewares/error.ts delete mode 100644 server/middlewares/index.ts delete mode 100644 server/middlewares/pagination.ts delete mode 100644 server/middlewares/rate-limiter.ts delete mode 100644 server/middlewares/servers.ts delete mode 100644 server/middlewares/user-right.ts delete mode 100644 server/middlewares/validators/abuse.ts delete mode 100644 server/middlewares/validators/account.ts delete mode 100644 server/middlewares/validators/activitypub/activity.ts delete mode 100644 server/middlewares/validators/activitypub/index.ts delete mode 100644 server/middlewares/validators/activitypub/pagination.ts delete mode 100644 server/middlewares/validators/activitypub/signature.ts delete mode 100644 server/middlewares/validators/actor-image.ts delete mode 100644 server/middlewares/validators/blocklist.ts delete mode 100644 server/middlewares/validators/bulk.ts delete mode 100644 server/middlewares/validators/config.ts delete mode 100644 server/middlewares/validators/feeds.ts delete mode 100644 server/middlewares/validators/follows.ts delete mode 100644 server/middlewares/validators/index.ts delete mode 100644 server/middlewares/validators/jobs.ts delete mode 100644 server/middlewares/validators/logs.ts delete mode 100644 server/middlewares/validators/metrics.ts delete mode 100644 server/middlewares/validators/object-storage-proxy.ts delete mode 100644 server/middlewares/validators/oembed.ts delete mode 100644 server/middlewares/validators/pagination.ts delete mode 100644 server/middlewares/validators/plugins.ts delete mode 100644 server/middlewares/validators/redundancy.ts delete mode 100644 server/middlewares/validators/runners/index.ts delete mode 100644 server/middlewares/validators/runners/job-files.ts delete mode 100644 server/middlewares/validators/runners/jobs.ts delete mode 100644 server/middlewares/validators/runners/registration-token.ts delete mode 100644 server/middlewares/validators/runners/runners.ts delete mode 100644 server/middlewares/validators/search.ts delete mode 100644 server/middlewares/validators/server.ts delete mode 100644 server/middlewares/validators/shared/abuses.ts delete mode 100644 server/middlewares/validators/shared/accounts.ts delete mode 100644 server/middlewares/validators/shared/index.ts delete mode 100644 server/middlewares/validators/shared/user-registrations.ts delete mode 100644 server/middlewares/validators/shared/users.ts delete mode 100644 server/middlewares/validators/shared/utils.ts delete mode 100644 server/middlewares/validators/shared/video-blacklists.ts delete mode 100644 server/middlewares/validators/shared/video-captions.ts delete mode 100644 server/middlewares/validators/shared/video-channel-syncs.ts delete mode 100644 server/middlewares/validators/shared/video-channels.ts delete mode 100644 server/middlewares/validators/shared/video-comments.ts delete mode 100644 server/middlewares/validators/shared/video-imports.ts delete mode 100644 server/middlewares/validators/shared/video-ownerships.ts delete mode 100644 server/middlewares/validators/shared/video-passwords.ts delete mode 100644 server/middlewares/validators/shared/video-playlists.ts delete mode 100644 server/middlewares/validators/shared/videos.ts delete mode 100644 server/middlewares/validators/sort.ts delete mode 100644 server/middlewares/validators/static.ts delete mode 100644 server/middlewares/validators/themes.ts delete mode 100644 server/middlewares/validators/two-factor.ts delete mode 100644 server/middlewares/validators/user-email-verification.ts delete mode 100644 server/middlewares/validators/user-history.ts delete mode 100644 server/middlewares/validators/user-notifications.ts delete mode 100644 server/middlewares/validators/user-registrations.ts delete mode 100644 server/middlewares/validators/user-subscriptions.ts delete mode 100644 server/middlewares/validators/users.ts delete mode 100644 server/middlewares/validators/videos/index.ts delete mode 100644 server/middlewares/validators/videos/shared/index.ts delete mode 100644 server/middlewares/validators/videos/shared/upload.ts delete mode 100644 server/middlewares/validators/videos/shared/video-validators.ts delete mode 100644 server/middlewares/validators/videos/video-blacklist.ts delete mode 100644 server/middlewares/validators/videos/video-captions.ts delete mode 100644 server/middlewares/validators/videos/video-channel-sync.ts delete mode 100644 server/middlewares/validators/videos/video-channels.ts delete mode 100644 server/middlewares/validators/videos/video-comments.ts delete mode 100644 server/middlewares/validators/videos/video-files.ts delete mode 100644 server/middlewares/validators/videos/video-imports.ts delete mode 100644 server/middlewares/validators/videos/video-live.ts delete mode 100644 server/middlewares/validators/videos/video-ownership-changes.ts delete mode 100644 server/middlewares/validators/videos/video-passwords.ts delete mode 100644 server/middlewares/validators/videos/video-playlists.ts delete mode 100644 server/middlewares/validators/videos/video-rates.ts delete mode 100644 server/middlewares/validators/videos/video-shares.ts delete mode 100644 server/middlewares/validators/videos/video-source.ts delete mode 100644 server/middlewares/validators/videos/video-stats.ts delete mode 100644 server/middlewares/validators/videos/video-studio.ts delete mode 100644 server/middlewares/validators/videos/video-token.ts delete mode 100644 server/middlewares/validators/videos/video-transcoding.ts delete mode 100644 server/middlewares/validators/videos/video-view.ts delete mode 100644 server/middlewares/validators/videos/videos.ts delete mode 100644 server/middlewares/validators/webfinger.ts delete mode 100644 server/models/abuse/abuse-message.ts delete mode 100644 server/models/abuse/abuse.ts delete mode 100644 server/models/abuse/sql/abuse-query-builder.ts delete mode 100644 server/models/abuse/video-abuse.ts delete mode 100644 server/models/abuse/video-comment-abuse.ts delete mode 100644 server/models/account/account-blocklist.ts delete mode 100644 server/models/account/account-video-rate.ts delete mode 100644 server/models/account/account.ts delete mode 100644 server/models/account/actor-custom-page.ts delete mode 100644 server/models/actor/actor-follow.ts delete mode 100644 server/models/actor/actor-image.ts delete mode 100644 server/models/actor/actor.ts delete mode 100644 server/models/actor/sql/instance-list-followers-query-builder.ts delete mode 100644 server/models/actor/sql/instance-list-following-query-builder.ts delete mode 100644 server/models/actor/sql/shared/actor-follow-table-attributes.ts delete mode 100644 server/models/actor/sql/shared/instance-list-follows-query-builder.ts delete mode 100644 server/models/application/application.ts delete mode 100644 server/models/migrations.ts delete mode 100644 server/models/oauth/oauth-client.ts delete mode 100644 server/models/oauth/oauth-token.ts delete mode 100644 server/models/redundancy/video-redundancy.ts delete mode 100644 server/models/runner/runner-job.ts delete mode 100644 server/models/runner/runner-registration-token.ts delete mode 100644 server/models/runner/runner.ts delete mode 100644 server/models/server/plugin.ts delete mode 100644 server/models/server/server-blocklist.ts delete mode 100644 server/models/server/server.ts delete mode 100644 server/models/server/tracker.ts delete mode 100644 server/models/server/video-tracker.ts delete mode 100644 server/models/shared/index.ts delete mode 100644 server/models/shared/model-builder.ts delete mode 100644 server/models/shared/model-cache.ts delete mode 100644 server/models/shared/query.ts delete mode 100644 server/models/shared/sql.ts delete mode 100644 server/models/user/sql/user-notitication-list-query-builder.ts delete mode 100644 server/models/user/user-notification-setting.ts delete mode 100644 server/models/user/user-notification.ts delete mode 100644 server/models/user/user-registration.ts delete mode 100644 server/models/user/user-video-history.ts delete mode 100644 server/models/user/user.ts delete mode 100644 server/models/video/formatter/index.ts delete mode 100644 server/models/video/formatter/shared/index.ts delete mode 100644 server/models/video/formatter/shared/video-format-utils.ts delete mode 100644 server/models/video/formatter/video-activity-pub-format.ts delete mode 100644 server/models/video/formatter/video-api-format.ts delete mode 100644 server/models/video/schedule-video-update.ts delete mode 100644 server/models/video/sql/comment/video-comment-list-query-builder.ts delete mode 100644 server/models/video/sql/comment/video-comment-table-attributes.ts delete mode 100644 server/models/video/sql/video/index.ts delete mode 100644 server/models/video/sql/video/shared/abstract-video-query-builder.ts delete mode 100644 server/models/video/sql/video/shared/video-file-query-builder.ts delete mode 100644 server/models/video/sql/video/shared/video-model-builder.ts delete mode 100644 server/models/video/sql/video/video-model-get-query-builder.ts delete mode 100644 server/models/video/sql/video/videos-id-list-query-builder.ts delete mode 100644 server/models/video/sql/video/videos-model-list-query-builder.ts delete mode 100644 server/models/video/storyboard.ts delete mode 100644 server/models/video/tag.ts delete mode 100644 server/models/video/thumbnail.ts delete mode 100644 server/models/video/video-blacklist.ts delete mode 100644 server/models/video/video-caption.ts delete mode 100644 server/models/video/video-change-ownership.ts delete mode 100644 server/models/video/video-channel-sync.ts delete mode 100644 server/models/video/video-channel.ts delete mode 100644 server/models/video/video-comment.ts delete mode 100644 server/models/video/video-file.ts delete mode 100644 server/models/video/video-import.ts delete mode 100644 server/models/video/video-job-info.ts delete mode 100644 server/models/video/video-live-replay-setting.ts delete mode 100644 server/models/video/video-live-session.ts delete mode 100644 server/models/video/video-live.ts delete mode 100644 server/models/video/video-password.ts delete mode 100644 server/models/video/video-playlist-element.ts delete mode 100644 server/models/video/video-playlist.ts delete mode 100644 server/models/video/video-share.ts delete mode 100644 server/models/video/video-source.ts delete mode 100644 server/models/video/video-streaming-playlist.ts delete mode 100644 server/models/video/video-tag.ts delete mode 100644 server/models/video/video.ts delete mode 100644 server/models/view/local-video-viewer-watch-section.ts delete mode 100644 server/models/view/local-video-viewer.ts delete mode 100644 server/models/view/video-view.ts create mode 100644 server/package.json create mode 100644 server/scripts/create-generate-storyboard-job.ts create mode 100644 server/scripts/create-import-video-file-job.ts create mode 100644 server/scripts/create-move-video-storage-job.ts create mode 100644 server/scripts/migrations/peertube-4.0.ts create mode 100644 server/scripts/migrations/peertube-4.2.ts create mode 100644 server/scripts/migrations/peertube-5.0.ts create mode 100755 server/scripts/parse-log.ts create mode 100755 server/scripts/plugin/install.ts create mode 100755 server/scripts/plugin/uninstall.ts create mode 100755 server/scripts/prune-storage.ts create mode 100644 server/scripts/regenerate-thumbnails.ts create mode 100755 server/scripts/reset-password.ts create mode 100755 server/scripts/update-host.ts rename {scripts => server/scripts}/upgrade.sh (100%) create mode 100644 server/server.ts rename server/{ => server}/assets/default-audio-background.jpg (100%) rename server/{ => server}/assets/default-live-background.jpg (100%) create mode 100644 server/server/controllers/activitypub/client.ts create mode 100644 server/server/controllers/activitypub/inbox.ts create mode 100644 server/server/controllers/activitypub/index.ts create mode 100644 server/server/controllers/activitypub/outbox.ts rename server/{ => server}/controllers/activitypub/utils.ts (100%) create mode 100644 server/server/controllers/api/abuse.ts create mode 100644 server/server/controllers/api/accounts.ts create mode 100644 server/server/controllers/api/blocklist.ts create mode 100644 server/server/controllers/api/bulk.ts create mode 100644 server/server/controllers/api/config.ts create mode 100644 server/server/controllers/api/custom-page.ts create mode 100644 server/server/controllers/api/index.ts create mode 100644 server/server/controllers/api/jobs.ts create mode 100644 server/server/controllers/api/metrics.ts create mode 100644 server/server/controllers/api/oauth-clients.ts create mode 100644 server/server/controllers/api/overviews.ts create mode 100644 server/server/controllers/api/plugins.ts create mode 100644 server/server/controllers/api/runners/index.ts create mode 100644 server/server/controllers/api/runners/jobs-files.ts create mode 100644 server/server/controllers/api/runners/jobs.ts create mode 100644 server/server/controllers/api/runners/manage-runners.ts create mode 100644 server/server/controllers/api/runners/registration-tokens.ts create mode 100644 server/server/controllers/api/search/index.ts create mode 100644 server/server/controllers/api/search/search-video-channels.ts create mode 100644 server/server/controllers/api/search/search-video-playlists.ts create mode 100644 server/server/controllers/api/search/search-videos.ts create mode 100644 server/server/controllers/api/search/shared/index.ts rename server/{ => server}/controllers/api/search/shared/utils.ts (100%) create mode 100644 server/server/controllers/api/server/contact.ts create mode 100644 server/server/controllers/api/server/debug.ts create mode 100644 server/server/controllers/api/server/follows.ts create mode 100644 server/server/controllers/api/server/index.ts create mode 100644 server/server/controllers/api/server/logs.ts create mode 100644 server/server/controllers/api/server/redundancy.ts create mode 100644 server/server/controllers/api/server/server-blocklist.ts create mode 100644 server/server/controllers/api/server/stats.ts create mode 100644 server/server/controllers/api/users/email-verification.ts create mode 100644 server/server/controllers/api/users/index.ts create mode 100644 server/server/controllers/api/users/me.ts create mode 100644 server/server/controllers/api/users/my-abuses.ts create mode 100644 server/server/controllers/api/users/my-blocklist.ts create mode 100644 server/server/controllers/api/users/my-history.ts create mode 100644 server/server/controllers/api/users/my-notifications.ts create mode 100644 server/server/controllers/api/users/my-subscriptions.ts create mode 100644 server/server/controllers/api/users/my-video-playlists.ts create mode 100644 server/server/controllers/api/users/registrations.ts create mode 100644 server/server/controllers/api/users/token.ts create mode 100644 server/server/controllers/api/users/two-factor.ts create mode 100644 server/server/controllers/api/video-channel-sync.ts create mode 100644 server/server/controllers/api/video-channel.ts create mode 100644 server/server/controllers/api/video-playlist.ts create mode 100644 server/server/controllers/api/videos/blacklist.ts create mode 100644 server/server/controllers/api/videos/captions.ts create mode 100644 server/server/controllers/api/videos/comment.ts create mode 100644 server/server/controllers/api/videos/files.ts create mode 100644 server/server/controllers/api/videos/import.ts create mode 100644 server/server/controllers/api/videos/index.ts create mode 100644 server/server/controllers/api/videos/live.ts create mode 100644 server/server/controllers/api/videos/ownership.ts create mode 100644 server/server/controllers/api/videos/passwords.ts create mode 100644 server/server/controllers/api/videos/rate.ts create mode 100644 server/server/controllers/api/videos/source.ts create mode 100644 server/server/controllers/api/videos/stats.ts create mode 100644 server/server/controllers/api/videos/storyboard.ts create mode 100644 server/server/controllers/api/videos/studio.ts create mode 100644 server/server/controllers/api/videos/token.ts create mode 100644 server/server/controllers/api/videos/transcoding.ts create mode 100644 server/server/controllers/api/videos/update.ts create mode 100644 server/server/controllers/api/videos/upload.ts create mode 100644 server/server/controllers/api/videos/view.ts create mode 100644 server/server/controllers/client.ts create mode 100644 server/server/controllers/download.ts create mode 100644 server/server/controllers/feeds/comment-feeds.ts create mode 100644 server/server/controllers/feeds/index.ts create mode 100644 server/server/controllers/feeds/shared/common-feed-utils.ts create mode 100644 server/server/controllers/feeds/shared/index.ts create mode 100644 server/server/controllers/feeds/shared/video-feed-utils.ts create mode 100644 server/server/controllers/feeds/video-feeds.ts create mode 100644 server/server/controllers/feeds/video-podcast-feeds.ts create mode 100644 server/server/controllers/index.ts create mode 100644 server/server/controllers/lazy-static.ts create mode 100644 server/server/controllers/misc.ts create mode 100644 server/server/controllers/object-storage-proxy.ts create mode 100644 server/server/controllers/plugins.ts create mode 100644 server/server/controllers/services.ts rename server/{ => server}/controllers/shared/m3u8-playlist.ts (100%) create mode 100644 server/server/controllers/sitemap.ts create mode 100644 server/server/controllers/static.ts create mode 100644 server/server/controllers/tracker.ts create mode 100644 server/server/controllers/well-known.ts create mode 100644 server/server/helpers/activity-pub-utils.ts create mode 100644 server/server/helpers/actors.ts create mode 100644 server/server/helpers/audit-logger.ts create mode 100644 server/server/helpers/captions-utils.ts create mode 100644 server/server/helpers/core-utils.ts create mode 100644 server/server/helpers/custom-jsonld-signature.ts create mode 100644 server/server/helpers/custom-validators/abuses.ts create mode 100644 server/server/helpers/custom-validators/accounts.ts create mode 100644 server/server/helpers/custom-validators/activitypub/activity.ts create mode 100644 server/server/helpers/custom-validators/activitypub/actor.ts create mode 100644 server/server/helpers/custom-validators/activitypub/cache-file.ts create mode 100644 server/server/helpers/custom-validators/activitypub/misc.ts create mode 100644 server/server/helpers/custom-validators/activitypub/playlist.ts create mode 100644 server/server/helpers/custom-validators/activitypub/signature.ts create mode 100644 server/server/helpers/custom-validators/activitypub/video-comments.ts create mode 100644 server/server/helpers/custom-validators/activitypub/videos.ts create mode 100644 server/server/helpers/custom-validators/activitypub/watch-action.ts create mode 100644 server/server/helpers/custom-validators/actor-images.ts rename server/{ => server}/helpers/custom-validators/bulk.ts (100%) create mode 100644 server/server/helpers/custom-validators/feeds.ts create mode 100644 server/server/helpers/custom-validators/follows.ts create mode 100644 server/server/helpers/custom-validators/jobs.ts create mode 100644 server/server/helpers/custom-validators/logs.ts rename server/{ => server}/helpers/custom-validators/metrics.ts (100%) create mode 100644 server/server/helpers/custom-validators/misc.ts create mode 100644 server/server/helpers/custom-validators/plugins.ts create mode 100644 server/server/helpers/custom-validators/runners/jobs.ts create mode 100644 server/server/helpers/custom-validators/runners/runners.ts create mode 100644 server/server/helpers/custom-validators/search.ts create mode 100644 server/server/helpers/custom-validators/servers.ts create mode 100644 server/server/helpers/custom-validators/user-notifications.ts create mode 100644 server/server/helpers/custom-validators/user-registration.ts create mode 100644 server/server/helpers/custom-validators/users.ts create mode 100644 server/server/helpers/custom-validators/video-blacklist.ts create mode 100644 server/server/helpers/custom-validators/video-captions.ts create mode 100644 server/server/helpers/custom-validators/video-channel-syncs.ts create mode 100644 server/server/helpers/custom-validators/video-channels.ts create mode 100644 server/server/helpers/custom-validators/video-comments.ts create mode 100644 server/server/helpers/custom-validators/video-imports.ts create mode 100644 server/server/helpers/custom-validators/video-lives.ts create mode 100644 server/server/helpers/custom-validators/video-ownership.ts create mode 100644 server/server/helpers/custom-validators/video-playlists.ts rename server/{ => server}/helpers/custom-validators/video-rates.ts (100%) create mode 100644 server/server/helpers/custom-validators/video-redundancies.ts create mode 100644 server/server/helpers/custom-validators/video-stats.ts create mode 100644 server/server/helpers/custom-validators/video-studio.ts create mode 100644 server/server/helpers/custom-validators/video-transcoding.ts create mode 100644 server/server/helpers/custom-validators/video-view.ts create mode 100644 server/server/helpers/custom-validators/videos.ts create mode 100644 server/server/helpers/custom-validators/webfinger.ts create mode 100644 server/server/helpers/database-utils.ts rename server/{ => server}/helpers/debounce.ts (100%) create mode 100644 server/server/helpers/decache.ts create mode 100644 server/server/helpers/dns.ts create mode 100644 server/server/helpers/express-utils.ts create mode 100644 server/server/helpers/ffmpeg/codecs.ts create mode 100644 server/server/helpers/ffmpeg/ffmpeg-image.ts create mode 100644 server/server/helpers/ffmpeg/ffmpeg-options.ts create mode 100644 server/server/helpers/ffmpeg/framerate.ts create mode 100644 server/server/helpers/ffmpeg/index.ts create mode 100644 server/server/helpers/geo-ip.ts create mode 100644 server/server/helpers/image-utils.ts create mode 100644 server/server/helpers/logger.ts create mode 100644 server/server/helpers/markdown.ts rename server/{ => server}/helpers/memoize.ts (100%) create mode 100644 server/server/helpers/mentions.ts create mode 100644 server/server/helpers/otp.ts create mode 100644 server/server/helpers/peertube-crypto.ts rename server/{ => server}/helpers/promise-cache.ts (100%) rename server/{ => server}/helpers/proxy.ts (100%) create mode 100644 server/server/helpers/query.ts rename server/{ => server}/helpers/regexp.ts (100%) create mode 100644 server/server/helpers/requests.ts rename server/{ => server}/helpers/stream-replacer.ts (100%) create mode 100644 server/server/helpers/token-generator.ts create mode 100644 server/server/helpers/upload.ts create mode 100644 server/server/helpers/utils.ts create mode 100644 server/server/helpers/version.ts create mode 100644 server/server/helpers/video.ts create mode 100644 server/server/helpers/webtorrent.ts create mode 100644 server/server/helpers/youtube-dl/index.ts create mode 100644 server/server/helpers/youtube-dl/youtube-dl-cli.ts create mode 100644 server/server/helpers/youtube-dl/youtube-dl-info-builder.ts create mode 100644 server/server/helpers/youtube-dl/youtube-dl-wrapper.ts create mode 100644 server/server/initializers/checker-after-init.ts create mode 100644 server/server/initializers/checker-before-init.ts create mode 100644 server/server/initializers/config.ts create mode 100644 server/server/initializers/constants.ts create mode 100644 server/server/initializers/database.ts create mode 100644 server/server/initializers/installer.ts rename server/{ => server}/initializers/migrations/0505-user-last-login-date.ts (100%) rename server/{ => server}/initializers/migrations/0510-video-file-metadata.ts (100%) rename server/{ => server}/initializers/migrations/0515-video-abuse-reason-timestamps.ts (100%) rename server/{ => server}/initializers/migrations/0520-abuses-split.ts (100%) rename server/{ => server}/initializers/migrations/0525-abuse-messages.ts (100%) create mode 100644 server/server/initializers/migrations/0530-playlist-multiple-video.ts rename server/{ => server}/initializers/migrations/0535-video-live.ts (100%) rename server/{ => server}/initializers/migrations/0540-video-file-infohash.ts (100%) rename server/{ => server}/initializers/migrations/0545-video-live-save-replay.ts (100%) rename server/{ => server}/initializers/migrations/0550-actor-follow-cleanup.ts (100%) rename server/{ => server}/initializers/migrations/0555-actor-follow-url.ts (100%) create mode 100644 server/server/initializers/migrations/0560-user-feed-token.ts rename server/{ => server}/initializers/migrations/0565-actor-follow-local-url.ts (100%) rename server/{ => server}/initializers/migrations/0570-permanent-live.ts (100%) rename server/{ => server}/initializers/migrations/0575-duplicate-thumbnail.ts (100%) rename server/{ => server}/initializers/migrations/0580-caption-filename.ts (100%) rename server/{ => server}/initializers/migrations/0585-video-file-names.ts (100%) rename server/{ => server}/initializers/migrations/0590-trackers.ts (100%) rename server/{ => server}/initializers/migrations/0595-remote-url.ts (100%) rename server/{ => server}/initializers/migrations/0600-duplicate-video-files.ts (100%) create mode 100644 server/server/initializers/migrations/0605-actor-missing-keys.ts rename server/{ => server}/initializers/migrations/0610-views-index copy.ts (100%) rename server/{ => server}/initializers/migrations/0612-captions-unique.ts (100%) rename server/{ => server}/initializers/migrations/0615-latest-versions-notification-settings.ts (100%) rename server/{ => server}/initializers/migrations/0620-latest-versions-application.ts (100%) rename server/{ => server}/initializers/migrations/0625-latest-versions-notification.ts (100%) rename server/{ => server}/initializers/migrations/0630-banner.ts (100%) rename server/{ => server}/initializers/migrations/0635-actor-image-size.ts (100%) rename server/{ => server}/initializers/migrations/0640-unique-keys.ts (100%) rename server/{ => server}/initializers/migrations/0645-actor-remote-creation-date.ts (100%) rename server/{ => server}/initializers/migrations/0650-actor-custom-pages.ts (100%) rename server/{ => server}/initializers/migrations/0655-streaming-playlist-filenames.ts (100%) create mode 100644 server/server/initializers/migrations/0660-object-storage.ts rename server/{ => server}/initializers/migrations/0665-no-account-warning-modal.ts (100%) rename server/{ => server}/initializers/migrations/0670-pending-job-default.ts (100%) rename server/{ => server}/initializers/migrations/0675-p2p-enabled.ts (100%) rename server/{ => server}/initializers/migrations/0680-files-storage-default.ts (100%) rename server/{ => server}/initializers/migrations/0685-multiple-actor-images.ts (100%) create mode 100644 server/server/initializers/migrations/0690-live-latency-mode.ts rename server/{ => server}/initializers/migrations/0695-remove-remote-rates.ts (100%) rename server/{ => server}/initializers/migrations/0700-edition-finished-notification.ts (100%) rename server/{ => server}/initializers/migrations/0705-local-video-viewers.ts (100%) rename server/{ => server}/initializers/migrations/0710-live-sessions.ts (100%) rename server/{ => server}/initializers/migrations/0715-video-source.ts (100%) rename server/{ => server}/initializers/migrations/0720-session-ending-processed.ts (100%) rename server/{ => server}/initializers/migrations/0725-node-version.ts (100%) rename server/{ => server}/initializers/migrations/0730-video-channel-sync.ts (100%) rename server/{ => server}/initializers/migrations/0735-video-channel-sync-import-foreign-key.ts (100%) rename server/{ => server}/initializers/migrations/0740-fix-old-enums.ts (100%) rename server/{ => server}/initializers/migrations/0745-user-otp.ts (100%) rename server/{ => server}/initializers/migrations/0750-user-registration.ts (100%) rename server/{ => server}/initializers/migrations/0755-unique-viewer-url.ts (100%) rename server/{ => server}/initializers/migrations/0760-video-live-replay-setting.ts (100%) rename server/{ => server}/initializers/migrations/0765-remote-transcoding.ts (100%) rename server/{ => server}/initializers/migrations/0770-actor-preferred-username.ts (100%) rename server/{ => server}/initializers/migrations/0775-add-user-is-email-public.ts (100%) rename server/{ => server}/initializers/migrations/0780-notification-registration.ts (100%) rename server/{ => server}/initializers/migrations/0785-video-password-protection.ts (100%) rename server/{ => server}/initializers/migrations/0790-thumbnail-disk.ts (100%) rename server/{ => server}/initializers/migrations/0795-duplicate-runner-name.ts (100%) rename server/{ => server}/initializers/migrations/0800-video-replace-file.ts (100%) create mode 100644 server/server/initializers/migrator.ts create mode 100644 server/server/lib/activitypub/activity.ts create mode 100644 server/server/lib/activitypub/actors/get.ts create mode 100644 server/server/lib/activitypub/actors/image.ts create mode 100644 server/server/lib/activitypub/actors/index.ts create mode 100644 server/server/lib/activitypub/actors/keys.ts create mode 100644 server/server/lib/activitypub/actors/refresh.ts create mode 100644 server/server/lib/activitypub/actors/shared/creator.ts create mode 100644 server/server/lib/activitypub/actors/shared/index.ts create mode 100644 server/server/lib/activitypub/actors/shared/object-to-model-attributes.ts create mode 100644 server/server/lib/activitypub/actors/shared/url-to-object.ts create mode 100644 server/server/lib/activitypub/actors/updater.ts create mode 100644 server/server/lib/activitypub/actors/webfinger.ts create mode 100644 server/server/lib/activitypub/audience.ts create mode 100644 server/server/lib/activitypub/cache-file.ts create mode 100644 server/server/lib/activitypub/collection.ts create mode 100644 server/server/lib/activitypub/context.ts create mode 100644 server/server/lib/activitypub/crawl.ts create mode 100644 server/server/lib/activitypub/follow.ts create mode 100644 server/server/lib/activitypub/inbox-manager.ts create mode 100644 server/server/lib/activitypub/local-video-viewer.ts create mode 100644 server/server/lib/activitypub/outbox.ts create mode 100644 server/server/lib/activitypub/playlists/create-update.ts create mode 100644 server/server/lib/activitypub/playlists/get.ts create mode 100644 server/server/lib/activitypub/playlists/index.ts create mode 100644 server/server/lib/activitypub/playlists/refresh.ts create mode 100644 server/server/lib/activitypub/playlists/shared/index.ts create mode 100644 server/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts create mode 100644 server/server/lib/activitypub/playlists/shared/url-to-object.ts create mode 100644 server/server/lib/activitypub/process/index.ts create mode 100644 server/server/lib/activitypub/process/process-accept.ts create mode 100644 server/server/lib/activitypub/process/process-announce.ts create mode 100644 server/server/lib/activitypub/process/process-create.ts create mode 100644 server/server/lib/activitypub/process/process-delete.ts create mode 100644 server/server/lib/activitypub/process/process-dislike.ts create mode 100644 server/server/lib/activitypub/process/process-flag.ts create mode 100644 server/server/lib/activitypub/process/process-follow.ts create mode 100644 server/server/lib/activitypub/process/process-like.ts create mode 100644 server/server/lib/activitypub/process/process-reject.ts create mode 100644 server/server/lib/activitypub/process/process-undo.ts create mode 100644 server/server/lib/activitypub/process/process-update.ts create mode 100644 server/server/lib/activitypub/process/process-view.ts create mode 100644 server/server/lib/activitypub/process/process.ts create mode 100644 server/server/lib/activitypub/send/http.ts create mode 100644 server/server/lib/activitypub/send/index.ts create mode 100644 server/server/lib/activitypub/send/send-accept.ts create mode 100644 server/server/lib/activitypub/send/send-announce.ts create mode 100644 server/server/lib/activitypub/send/send-create.ts create mode 100644 server/server/lib/activitypub/send/send-delete.ts create mode 100644 server/server/lib/activitypub/send/send-dislike.ts create mode 100644 server/server/lib/activitypub/send/send-flag.ts create mode 100644 server/server/lib/activitypub/send/send-follow.ts create mode 100644 server/server/lib/activitypub/send/send-like.ts create mode 100644 server/server/lib/activitypub/send/send-reject.ts create mode 100644 server/server/lib/activitypub/send/send-undo.ts create mode 100644 server/server/lib/activitypub/send/send-update.ts create mode 100644 server/server/lib/activitypub/send/send-view.ts create mode 100644 server/server/lib/activitypub/send/shared/audience-utils.ts create mode 100644 server/server/lib/activitypub/send/shared/index.ts create mode 100644 server/server/lib/activitypub/send/shared/send-utils.ts create mode 100644 server/server/lib/activitypub/share.ts create mode 100644 server/server/lib/activitypub/url.ts create mode 100644 server/server/lib/activitypub/video-comments.ts create mode 100644 server/server/lib/activitypub/video-rates.ts create mode 100644 server/server/lib/activitypub/videos/federate.ts create mode 100644 server/server/lib/activitypub/videos/get.ts create mode 100644 server/server/lib/activitypub/videos/index.ts create mode 100644 server/server/lib/activitypub/videos/refresh.ts create mode 100644 server/server/lib/activitypub/videos/shared/abstract-builder.ts create mode 100644 server/server/lib/activitypub/videos/shared/creator.ts create mode 100644 server/server/lib/activitypub/videos/shared/index.ts create mode 100644 server/server/lib/activitypub/videos/shared/object-to-model-attributes.ts create mode 100644 server/server/lib/activitypub/videos/shared/trackers.ts create mode 100644 server/server/lib/activitypub/videos/shared/url-to-object.ts create mode 100644 server/server/lib/activitypub/videos/shared/video-sync-attributes.ts create mode 100644 server/server/lib/activitypub/videos/updater.ts create mode 100644 server/server/lib/actor-follow-health-cache.ts create mode 100644 server/server/lib/actor-image.ts create mode 100644 server/server/lib/auth/external-auth.ts create mode 100644 server/server/lib/auth/oauth-model.ts create mode 100644 server/server/lib/auth/oauth.ts create mode 100644 server/server/lib/auth/tokens-cache.ts create mode 100644 server/server/lib/blocklist.ts create mode 100644 server/server/lib/client-html.ts create mode 100644 server/server/lib/emailer.ts rename server/{ => server}/lib/emails/abuse-new-message/html.pug (100%) rename server/{ => server}/lib/emails/abuse-state-change/html.pug (100%) rename server/{ => server}/lib/emails/account-abuse-new/html.pug (100%) rename server/{ => server}/lib/emails/common/base.pug (100%) rename server/{ => server}/lib/emails/common/greetings.pug (100%) rename server/{ => server}/lib/emails/common/html.pug (100%) rename server/{ => server}/lib/emails/common/mixins.pug (100%) rename server/{ => server}/lib/emails/contact-form/html.pug (100%) rename server/{ => server}/lib/emails/follower-on-channel/html.pug (100%) rename server/{ => server}/lib/emails/password-create/html.pug (100%) rename server/{ => server}/lib/emails/password-reset/html.pug (100%) rename server/{ => server}/lib/emails/peertube-version-new/html.pug (100%) rename server/{ => server}/lib/emails/plugin-version-new/html.pug (100%) rename server/{ => server}/lib/emails/user-registered/html.pug (100%) rename server/{ => server}/lib/emails/user-registration-request-accepted/html.pug (100%) rename server/{ => server}/lib/emails/user-registration-request-rejected/html.pug (100%) rename server/{ => server}/lib/emails/user-registration-request/html.pug (100%) rename server/{ => server}/lib/emails/verify-email/html.pug (100%) rename server/{ => server}/lib/emails/video-abuse-new/html.pug (100%) rename server/{ => server}/lib/emails/video-auto-blacklist-new/html.pug (100%) rename server/{ => server}/lib/emails/video-comment-abuse-new/html.pug (100%) rename server/{ => server}/lib/emails/video-comment-mention/html.pug (100%) rename server/{ => server}/lib/emails/video-comment-new/html.pug (100%) create mode 100644 server/server/lib/files-cache/avatar-permanent-file-cache.ts create mode 100644 server/server/lib/files-cache/index.ts create mode 100644 server/server/lib/files-cache/shared/abstract-permanent-file-cache.ts create mode 100644 server/server/lib/files-cache/shared/abstract-simple-file-cache.ts create mode 100644 server/server/lib/files-cache/shared/index.ts create mode 100644 server/server/lib/files-cache/video-captions-simple-file-cache.ts create mode 100644 server/server/lib/files-cache/video-miniature-permanent-file-cache.ts create mode 100644 server/server/lib/files-cache/video-previews-simple-file-cache.ts create mode 100644 server/server/lib/files-cache/video-storyboards-simple-file-cache.ts create mode 100644 server/server/lib/files-cache/video-torrents-simple-file-cache.ts create mode 100644 server/server/lib/hls.ts create mode 100644 server/server/lib/internal-event-emitter.ts create mode 100644 server/server/lib/job-queue/handlers/activitypub-cleaner.ts create mode 100644 server/server/lib/job-queue/handlers/activitypub-follow.ts create mode 100644 server/server/lib/job-queue/handlers/activitypub-http-broadcast.ts create mode 100644 server/server/lib/job-queue/handlers/activitypub-http-fetcher.ts create mode 100644 server/server/lib/job-queue/handlers/activitypub-http-unicast.ts create mode 100644 server/server/lib/job-queue/handlers/activitypub-refresher.ts create mode 100644 server/server/lib/job-queue/handlers/actor-keys.ts create mode 100644 server/server/lib/job-queue/handlers/after-video-channel-import.ts create mode 100644 server/server/lib/job-queue/handlers/email.ts create mode 100644 server/server/lib/job-queue/handlers/federate-video.ts create mode 100644 server/server/lib/job-queue/handlers/generate-storyboard.ts create mode 100644 server/server/lib/job-queue/handlers/manage-video-torrent.ts create mode 100644 server/server/lib/job-queue/handlers/move-to-object-storage.ts create mode 100644 server/server/lib/job-queue/handlers/notify.ts create mode 100644 server/server/lib/job-queue/handlers/transcoding-job-builder.ts create mode 100644 server/server/lib/job-queue/handlers/video-channel-import.ts create mode 100644 server/server/lib/job-queue/handlers/video-file-import.ts create mode 100644 server/server/lib/job-queue/handlers/video-import.ts create mode 100644 server/server/lib/job-queue/handlers/video-live-ending.ts create mode 100644 server/server/lib/job-queue/handlers/video-redundancy.ts create mode 100644 server/server/lib/job-queue/handlers/video-studio-edition.ts create mode 100644 server/server/lib/job-queue/handlers/video-transcoding.ts create mode 100644 server/server/lib/job-queue/handlers/video-views-stats.ts create mode 100644 server/server/lib/job-queue/index.ts create mode 100644 server/server/lib/job-queue/job-queue.ts create mode 100644 server/server/lib/live/index.ts create mode 100644 server/server/lib/live/live-manager.ts rename server/{ => server}/lib/live/live-quota-store.ts (100%) create mode 100644 server/server/lib/live/live-segment-sha-store.ts create mode 100644 server/server/lib/live/live-utils.ts create mode 100644 server/server/lib/live/shared/index.ts create mode 100644 server/server/lib/live/shared/muxing-session.ts create mode 100644 server/server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts create mode 100644 server/server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts create mode 100644 server/server/lib/live/shared/transcoding-wrapper/index.ts create mode 100644 server/server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts create mode 100644 server/server/lib/local-actor.ts create mode 100644 server/server/lib/model-loaders/actor.ts create mode 100644 server/server/lib/model-loaders/index.ts create mode 100644 server/server/lib/model-loaders/video.ts create mode 100644 server/server/lib/moderation.ts create mode 100644 server/server/lib/notifier/index.ts create mode 100644 server/server/lib/notifier/notifier.ts create mode 100644 server/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts create mode 100644 server/server/lib/notifier/shared/abuse/abuse-state-change-for-reporter.ts create mode 100644 server/server/lib/notifier/shared/abuse/index.ts create mode 100644 server/server/lib/notifier/shared/abuse/new-abuse-for-moderators.ts create mode 100644 server/server/lib/notifier/shared/abuse/new-abuse-message-for-moderators.ts create mode 100644 server/server/lib/notifier/shared/abuse/new-abuse-message-for-reporter.ts create mode 100644 server/server/lib/notifier/shared/blacklist/index.ts create mode 100644 server/server/lib/notifier/shared/blacklist/new-auto-blacklist-for-moderators.ts create mode 100644 server/server/lib/notifier/shared/blacklist/new-blacklist-for-owner.ts create mode 100644 server/server/lib/notifier/shared/blacklist/unblacklist-for-owner.ts create mode 100644 server/server/lib/notifier/shared/comment/comment-mention.ts create mode 100644 server/server/lib/notifier/shared/comment/index.ts create mode 100644 server/server/lib/notifier/shared/comment/new-comment-for-video-owner.ts create mode 100644 server/server/lib/notifier/shared/common/abstract-notification.ts create mode 100644 server/server/lib/notifier/shared/common/index.ts create mode 100644 server/server/lib/notifier/shared/follow/auto-follow-for-instance.ts create mode 100644 server/server/lib/notifier/shared/follow/follow-for-instance.ts create mode 100644 server/server/lib/notifier/shared/follow/follow-for-user.ts create mode 100644 server/server/lib/notifier/shared/follow/index.ts create mode 100644 server/server/lib/notifier/shared/index.ts create mode 100644 server/server/lib/notifier/shared/instance/direct-registration-for-moderators.ts create mode 100644 server/server/lib/notifier/shared/instance/index.ts create mode 100644 server/server/lib/notifier/shared/instance/new-peertube-version-for-admins.ts create mode 100644 server/server/lib/notifier/shared/instance/new-plugin-version-for-admins.ts create mode 100644 server/server/lib/notifier/shared/instance/registration-request-for-moderators.ts create mode 100644 server/server/lib/notifier/shared/video-publication/abstract-owned-video-publication.ts create mode 100644 server/server/lib/notifier/shared/video-publication/import-finished-for-owner.ts create mode 100644 server/server/lib/notifier/shared/video-publication/index.ts create mode 100644 server/server/lib/notifier/shared/video-publication/new-video-for-subscribers.ts create mode 100644 server/server/lib/notifier/shared/video-publication/owned-publication-after-auto-unblacklist.ts create mode 100644 server/server/lib/notifier/shared/video-publication/owned-publication-after-schedule-update.ts create mode 100644 server/server/lib/notifier/shared/video-publication/owned-publication-after-transcoding.ts create mode 100644 server/server/lib/notifier/shared/video-publication/studio-edition-finished-for-owner.ts create mode 100644 server/server/lib/object-storage/index.ts create mode 100644 server/server/lib/object-storage/keys.ts create mode 100644 server/server/lib/object-storage/pre-signed-urls.ts create mode 100644 server/server/lib/object-storage/proxy.ts create mode 100644 server/server/lib/object-storage/shared/client.ts create mode 100644 server/server/lib/object-storage/shared/index.ts create mode 100644 server/server/lib/object-storage/shared/logger.ts create mode 100644 server/server/lib/object-storage/shared/object-storage-helpers.ts create mode 100644 server/server/lib/object-storage/urls.ts create mode 100644 server/server/lib/object-storage/videos.ts rename server/{ => server}/lib/opentelemetry/metric-helpers/bittorrent-tracker-observers-builder.ts (100%) create mode 100644 server/server/lib/opentelemetry/metric-helpers/index.ts create mode 100644 server/server/lib/opentelemetry/metric-helpers/job-queue-observers-builder.ts create mode 100644 server/server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts create mode 100644 server/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts create mode 100644 server/server/lib/opentelemetry/metric-helpers/playback-metrics.ts create mode 100644 server/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts create mode 100644 server/server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts create mode 100644 server/server/lib/opentelemetry/metrics.ts create mode 100644 server/server/lib/opentelemetry/tracing.ts create mode 100644 server/server/lib/paths.ts create mode 100644 server/server/lib/peertube-socket.ts create mode 100644 server/server/lib/plugins/hooks.ts create mode 100644 server/server/lib/plugins/plugin-helpers-builder.ts create mode 100644 server/server/lib/plugins/plugin-index.ts create mode 100644 server/server/lib/plugins/plugin-manager.ts create mode 100644 server/server/lib/plugins/register-helpers.ts create mode 100644 server/server/lib/plugins/theme-utils.ts create mode 100644 server/server/lib/plugins/video-constant-manager-factory.ts create mode 100644 server/server/lib/plugins/yarn.ts create mode 100644 server/server/lib/redis.ts create mode 100644 server/server/lib/redundancy.ts create mode 100644 server/server/lib/runners/index.ts create mode 100644 server/server/lib/runners/job-handlers/abstract-job-handler.ts create mode 100644 server/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts create mode 100644 server/server/lib/runners/job-handlers/index.ts create mode 100644 server/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts create mode 100644 server/server/lib/runners/job-handlers/runner-job-handlers.ts create mode 100644 server/server/lib/runners/job-handlers/shared/index.ts create mode 100644 server/server/lib/runners/job-handlers/shared/vod-helpers.ts create mode 100644 server/server/lib/runners/job-handlers/video-studio-transcoding-job-handler.ts create mode 100644 server/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts create mode 100644 server/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts create mode 100644 server/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts create mode 100644 server/server/lib/runners/runner-urls.ts create mode 100644 server/server/lib/runners/runner.ts create mode 100644 server/server/lib/schedulers/abstract-scheduler.ts create mode 100644 server/server/lib/schedulers/actor-follow-scheduler.ts create mode 100644 server/server/lib/schedulers/auto-follow-index-instances.ts create mode 100644 server/server/lib/schedulers/geo-ip-update-scheduler.ts create mode 100644 server/server/lib/schedulers/peertube-version-check-scheduler.ts create mode 100644 server/server/lib/schedulers/plugins-check-scheduler.ts create mode 100644 server/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts create mode 100644 server/server/lib/schedulers/remove-old-history-scheduler.ts create mode 100644 server/server/lib/schedulers/remove-old-views-scheduler.ts create mode 100644 server/server/lib/schedulers/runner-job-watch-dog-scheduler.ts create mode 100644 server/server/lib/schedulers/update-videos-scheduler.ts create mode 100644 server/server/lib/schedulers/video-channel-sync-latest-scheduler.ts create mode 100644 server/server/lib/schedulers/video-views-buffer-scheduler.ts create mode 100644 server/server/lib/schedulers/videos-redundancy-scheduler.ts create mode 100644 server/server/lib/schedulers/youtube-dl-update-scheduler.ts create mode 100644 server/server/lib/search.ts create mode 100644 server/server/lib/server-config-manager.ts create mode 100644 server/server/lib/signup.ts create mode 100644 server/server/lib/stat-manager.ts create mode 100644 server/server/lib/sync-channel.ts create mode 100644 server/server/lib/thumbnail.ts create mode 100644 server/server/lib/timeserie.ts create mode 100644 server/server/lib/transcoding/create-transcoding-job.ts create mode 100644 server/server/lib/transcoding/default-transcoding-profiles.ts create mode 100644 server/server/lib/transcoding/ended-transcoding.ts create mode 100644 server/server/lib/transcoding/hls-transcoding.ts create mode 100644 server/server/lib/transcoding/shared/ffmpeg-builder.ts create mode 100644 server/server/lib/transcoding/shared/index.ts create mode 100644 server/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts create mode 100644 server/server/lib/transcoding/shared/job-builders/index.ts create mode 100644 server/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts create mode 100644 server/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts create mode 100644 server/server/lib/transcoding/transcoding-priority.ts create mode 100644 server/server/lib/transcoding/transcoding-quick-transcode.ts create mode 100644 server/server/lib/transcoding/transcoding-resolutions.ts create mode 100644 server/server/lib/transcoding/web-transcoding.ts create mode 100644 server/server/lib/uploadx.ts create mode 100644 server/server/lib/user.ts create mode 100644 server/server/lib/video-blacklist.ts create mode 100644 server/server/lib/video-channel.ts create mode 100644 server/server/lib/video-comment.ts create mode 100644 server/server/lib/video-file.ts create mode 100644 server/server/lib/video-path-manager.ts create mode 100644 server/server/lib/video-playlist.ts create mode 100644 server/server/lib/video-pre-import.ts create mode 100644 server/server/lib/video-privacy.ts create mode 100644 server/server/lib/video-state.ts create mode 100644 server/server/lib/video-studio.ts create mode 100644 server/server/lib/video-tokens-manager.ts create mode 100644 server/server/lib/video-urls.ts create mode 100644 server/server/lib/video.ts create mode 100644 server/server/lib/views/shared/index.ts create mode 100644 server/server/lib/views/shared/video-viewer-counters.ts create mode 100644 server/server/lib/views/shared/video-viewer-stats.ts create mode 100644 server/server/lib/views/shared/video-views.ts create mode 100644 server/server/lib/views/video-views-manager.ts create mode 100644 server/server/lib/worker/parent-process.ts create mode 100644 server/server/lib/worker/workers/http-broadcast.ts create mode 100644 server/server/lib/worker/workers/image-downloader.ts create mode 100644 server/server/lib/worker/workers/image-processor.ts create mode 100644 server/server/middlewares/activitypub.ts create mode 100644 server/server/middlewares/async.ts create mode 100644 server/server/middlewares/auth.ts create mode 100644 server/server/middlewares/cache/cache.ts create mode 100644 server/server/middlewares/cache/index.ts create mode 100644 server/server/middlewares/cache/shared/api-cache.ts create mode 100644 server/server/middlewares/cache/shared/index.ts create mode 100644 server/server/middlewares/csp.ts rename server/{ => server}/middlewares/dnt.ts (100%) rename server/{ => server}/middlewares/doc.ts (100%) create mode 100644 server/server/middlewares/error.ts create mode 100644 server/server/middlewares/index.ts create mode 100644 server/server/middlewares/pagination.ts create mode 100644 server/server/middlewares/rate-limiter.ts rename server/{ => server}/middlewares/robots.ts (100%) create mode 100644 server/server/middlewares/servers.ts rename server/{ => server}/middlewares/sort.ts (100%) create mode 100644 server/server/middlewares/user-right.ts create mode 100644 server/server/middlewares/validators/abuse.ts create mode 100644 server/server/middlewares/validators/account.ts create mode 100644 server/server/middlewares/validators/activitypub/activity.ts create mode 100644 server/server/middlewares/validators/activitypub/index.ts create mode 100644 server/server/middlewares/validators/activitypub/pagination.ts create mode 100644 server/server/middlewares/validators/activitypub/signature.ts create mode 100644 server/server/middlewares/validators/actor-image.ts create mode 100644 server/server/middlewares/validators/blocklist.ts create mode 100644 server/server/middlewares/validators/bulk.ts create mode 100644 server/server/middlewares/validators/config.ts rename server/{ => server}/middlewares/validators/express.ts (100%) create mode 100644 server/server/middlewares/validators/feeds.ts create mode 100644 server/server/middlewares/validators/follows.ts create mode 100644 server/server/middlewares/validators/index.ts create mode 100644 server/server/middlewares/validators/jobs.ts create mode 100644 server/server/middlewares/validators/logs.ts create mode 100644 server/server/middlewares/validators/metrics.ts create mode 100644 server/server/middlewares/validators/object-storage-proxy.ts create mode 100644 server/server/middlewares/validators/oembed.ts create mode 100644 server/server/middlewares/validators/pagination.ts create mode 100644 server/server/middlewares/validators/plugins.ts create mode 100644 server/server/middlewares/validators/redundancy.ts create mode 100644 server/server/middlewares/validators/runners/index.ts create mode 100644 server/server/middlewares/validators/runners/job-files.ts create mode 100644 server/server/middlewares/validators/runners/jobs.ts create mode 100644 server/server/middlewares/validators/runners/registration-token.ts create mode 100644 server/server/middlewares/validators/runners/runners.ts create mode 100644 server/server/middlewares/validators/search.ts create mode 100644 server/server/middlewares/validators/server.ts create mode 100644 server/server/middlewares/validators/shared/abuses.ts create mode 100644 server/server/middlewares/validators/shared/accounts.ts create mode 100644 server/server/middlewares/validators/shared/index.ts create mode 100644 server/server/middlewares/validators/shared/user-registrations.ts create mode 100644 server/server/middlewares/validators/shared/users.ts create mode 100644 server/server/middlewares/validators/shared/utils.ts create mode 100644 server/server/middlewares/validators/shared/video-blacklists.ts create mode 100644 server/server/middlewares/validators/shared/video-captions.ts create mode 100644 server/server/middlewares/validators/shared/video-channel-syncs.ts create mode 100644 server/server/middlewares/validators/shared/video-channels.ts create mode 100644 server/server/middlewares/validators/shared/video-comments.ts create mode 100644 server/server/middlewares/validators/shared/video-imports.ts create mode 100644 server/server/middlewares/validators/shared/video-ownerships.ts create mode 100644 server/server/middlewares/validators/shared/video-passwords.ts create mode 100644 server/server/middlewares/validators/shared/video-playlists.ts create mode 100644 server/server/middlewares/validators/shared/videos.ts create mode 100644 server/server/middlewares/validators/sort.ts create mode 100644 server/server/middlewares/validators/static.ts create mode 100644 server/server/middlewares/validators/themes.ts create mode 100644 server/server/middlewares/validators/two-factor.ts create mode 100644 server/server/middlewares/validators/user-email-verification.ts create mode 100644 server/server/middlewares/validators/user-history.ts create mode 100644 server/server/middlewares/validators/user-notifications.ts create mode 100644 server/server/middlewares/validators/user-registrations.ts create mode 100644 server/server/middlewares/validators/user-subscriptions.ts create mode 100644 server/server/middlewares/validators/users.ts create mode 100644 server/server/middlewares/validators/videos/index.ts create mode 100644 server/server/middlewares/validators/videos/shared/index.ts create mode 100644 server/server/middlewares/validators/videos/shared/upload.ts create mode 100644 server/server/middlewares/validators/videos/shared/video-validators.ts create mode 100644 server/server/middlewares/validators/videos/video-blacklist.ts create mode 100644 server/server/middlewares/validators/videos/video-captions.ts create mode 100644 server/server/middlewares/validators/videos/video-channel-sync.ts create mode 100644 server/server/middlewares/validators/videos/video-channels.ts create mode 100644 server/server/middlewares/validators/videos/video-comments.ts create mode 100644 server/server/middlewares/validators/videos/video-files.ts create mode 100644 server/server/middlewares/validators/videos/video-imports.ts create mode 100644 server/server/middlewares/validators/videos/video-live.ts create mode 100644 server/server/middlewares/validators/videos/video-ownership-changes.ts create mode 100644 server/server/middlewares/validators/videos/video-passwords.ts create mode 100644 server/server/middlewares/validators/videos/video-playlists.ts create mode 100644 server/server/middlewares/validators/videos/video-rates.ts create mode 100644 server/server/middlewares/validators/videos/video-shares.ts create mode 100644 server/server/middlewares/validators/videos/video-source.ts create mode 100644 server/server/middlewares/validators/videos/video-stats.ts create mode 100644 server/server/middlewares/validators/videos/video-studio.ts create mode 100644 server/server/middlewares/validators/videos/video-token.ts create mode 100644 server/server/middlewares/validators/videos/video-transcoding.ts create mode 100644 server/server/middlewares/validators/videos/video-view.ts create mode 100644 server/server/middlewares/validators/videos/videos.ts create mode 100644 server/server/middlewares/validators/webfinger.ts create mode 100644 server/server/models/abuse/abuse-message.ts create mode 100644 server/server/models/abuse/abuse.ts create mode 100644 server/server/models/abuse/sql/abuse-query-builder.ts create mode 100644 server/server/models/abuse/video-abuse.ts create mode 100644 server/server/models/abuse/video-comment-abuse.ts create mode 100644 server/server/models/account/account-blocklist.ts create mode 100644 server/server/models/account/account-video-rate.ts create mode 100644 server/server/models/account/account.ts create mode 100644 server/server/models/account/actor-custom-page.ts create mode 100644 server/server/models/actor/actor-follow.ts create mode 100644 server/server/models/actor/actor-image.ts create mode 100644 server/server/models/actor/actor.ts create mode 100644 server/server/models/actor/sql/instance-list-followers-query-builder.ts create mode 100644 server/server/models/actor/sql/instance-list-following-query-builder.ts create mode 100644 server/server/models/actor/sql/shared/actor-follow-table-attributes.ts create mode 100644 server/server/models/actor/sql/shared/instance-list-follows-query-builder.ts create mode 100644 server/server/models/application/application.ts create mode 100644 server/server/models/oauth/oauth-client.ts create mode 100644 server/server/models/oauth/oauth-token.ts create mode 100644 server/server/models/redundancy/video-redundancy.ts create mode 100644 server/server/models/runner/runner-job.ts create mode 100644 server/server/models/runner/runner-registration-token.ts create mode 100644 server/server/models/runner/runner.ts create mode 100644 server/server/models/server/plugin.ts create mode 100644 server/server/models/server/server-blocklist.ts create mode 100644 server/server/models/server/server.ts create mode 100644 server/server/models/server/tracker.ts create mode 100644 server/server/models/server/video-tracker.ts rename server/{ => server}/models/shared/abstract-run-query.ts (100%) create mode 100644 server/server/models/shared/index.ts create mode 100644 server/server/models/shared/model-builder.ts create mode 100644 server/server/models/shared/model-cache.ts create mode 100644 server/server/models/shared/query.ts rename server/{ => server}/models/shared/sequelize-helpers.ts (100%) rename server/{ => server}/models/shared/sort.ts (100%) create mode 100644 server/server/models/shared/sql.ts rename server/{ => server}/models/shared/update.ts (100%) create mode 100644 server/server/models/user/sql/user-notitication-list-query-builder.ts create mode 100644 server/server/models/user/user-notification-setting.ts create mode 100644 server/server/models/user/user-notification.ts create mode 100644 server/server/models/user/user-registration.ts create mode 100644 server/server/models/user/user-video-history.ts create mode 100644 server/server/models/user/user.ts create mode 100644 server/server/models/video/formatter/index.ts create mode 100644 server/server/models/video/formatter/shared/index.ts create mode 100644 server/server/models/video/formatter/shared/video-format-utils.ts create mode 100644 server/server/models/video/formatter/video-activity-pub-format.ts create mode 100644 server/server/models/video/formatter/video-api-format.ts create mode 100644 server/server/models/video/schedule-video-update.ts create mode 100644 server/server/models/video/sql/comment/video-comment-list-query-builder.ts create mode 100644 server/server/models/video/sql/comment/video-comment-table-attributes.ts create mode 100644 server/server/models/video/sql/video/index.ts create mode 100644 server/server/models/video/sql/video/shared/abstract-video-query-builder.ts create mode 100644 server/server/models/video/sql/video/shared/video-file-query-builder.ts create mode 100644 server/server/models/video/sql/video/shared/video-model-builder.ts rename server/{ => server}/models/video/sql/video/shared/video-table-attributes.ts (100%) create mode 100644 server/server/models/video/sql/video/video-model-get-query-builder.ts create mode 100644 server/server/models/video/sql/video/videos-id-list-query-builder.ts create mode 100644 server/server/models/video/sql/video/videos-model-list-query-builder.ts create mode 100644 server/server/models/video/storyboard.ts create mode 100644 server/server/models/video/tag.ts create mode 100644 server/server/models/video/thumbnail.ts create mode 100644 server/server/models/video/video-blacklist.ts create mode 100644 server/server/models/video/video-caption.ts create mode 100644 server/server/models/video/video-change-ownership.ts create mode 100644 server/server/models/video/video-channel-sync.ts create mode 100644 server/server/models/video/video-channel.ts create mode 100644 server/server/models/video/video-comment.ts create mode 100644 server/server/models/video/video-file.ts create mode 100644 server/server/models/video/video-import.ts create mode 100644 server/server/models/video/video-job-info.ts create mode 100644 server/server/models/video/video-live-replay-setting.ts create mode 100644 server/server/models/video/video-live-session.ts create mode 100644 server/server/models/video/video-live.ts create mode 100644 server/server/models/video/video-password.ts create mode 100644 server/server/models/video/video-playlist-element.ts create mode 100644 server/server/models/video/video-playlist.ts create mode 100644 server/server/models/video/video-share.ts create mode 100644 server/server/models/video/video-source.ts create mode 100644 server/server/models/video/video-streaming-playlist.ts create mode 100644 server/server/models/video/video-tag.ts create mode 100644 server/server/models/video/video.ts create mode 100644 server/server/models/view/local-video-viewer-watch-section.ts create mode 100644 server/server/models/view/local-video-viewer.ts create mode 100644 server/server/models/view/video-view.ts rename server/{ => server}/static/dnt-policy/dnt-policy-1.0.txt (100%) create mode 100644 server/server/types/activitypub-processor.model.ts rename server/{ => server}/types/express-handler.ts (100%) create mode 100644 server/server/types/express.d.ts create mode 100644 server/server/types/index.ts rename server/{ => server}/types/lib.d.ts (100%) create mode 100644 server/server/types/models/abuse/abuse-message.ts create mode 100644 server/server/types/models/abuse/abuse.ts create mode 100644 server/server/types/models/abuse/index.ts create mode 100644 server/server/types/models/account/account-blocklist.ts create mode 100644 server/server/types/models/account/account.ts create mode 100644 server/server/types/models/account/actor-custom-page.ts create mode 100644 server/server/types/models/account/index.ts create mode 100644 server/server/types/models/actor/actor-follow.ts create mode 100644 server/server/types/models/actor/actor-image.ts create mode 100644 server/server/types/models/actor/actor.ts create mode 100644 server/server/types/models/actor/index.ts create mode 100644 server/server/types/models/application/application.ts create mode 100644 server/server/types/models/application/index.ts create mode 100644 server/server/types/models/index.ts create mode 100644 server/server/types/models/oauth/index.ts create mode 100644 server/server/types/models/oauth/oauth-client.ts create mode 100644 server/server/types/models/oauth/oauth-token.ts create mode 100644 server/server/types/models/runners/index.ts create mode 100644 server/server/types/models/runners/runner-job.ts create mode 100644 server/server/types/models/runners/runner-registration-token.ts create mode 100644 server/server/types/models/runners/runner.ts create mode 100644 server/server/types/models/server/index.ts create mode 100644 server/server/types/models/server/plugin.ts create mode 100644 server/server/types/models/server/server-blocklist.ts create mode 100644 server/server/types/models/server/server.ts create mode 100644 server/server/types/models/server/tracker.ts create mode 100644 server/server/types/models/user/index.ts create mode 100644 server/server/types/models/user/user-notification-setting.ts create mode 100644 server/server/types/models/user/user-notification.ts create mode 100644 server/server/types/models/user/user-registration.ts create mode 100644 server/server/types/models/user/user-video-history.ts create mode 100644 server/server/types/models/user/user.ts create mode 100644 server/server/types/models/video/index.ts create mode 100644 server/server/types/models/video/local-video-viewer-watch-section.ts create mode 100644 server/server/types/models/video/local-video-viewer.ts create mode 100644 server/server/types/models/video/schedule-video-update.ts create mode 100644 server/server/types/models/video/storyboard.ts create mode 100644 server/server/types/models/video/tag.ts create mode 100644 server/server/types/models/video/thumbnail.ts create mode 100644 server/server/types/models/video/video-blacklist.ts create mode 100644 server/server/types/models/video/video-caption.ts create mode 100644 server/server/types/models/video/video-change-ownership.ts create mode 100644 server/server/types/models/video/video-channel-sync.ts create mode 100644 server/server/types/models/video/video-channels.ts create mode 100644 server/server/types/models/video/video-comment.ts create mode 100644 server/server/types/models/video/video-file.ts create mode 100644 server/server/types/models/video/video-import.ts create mode 100644 server/server/types/models/video/video-live-replay-setting.ts create mode 100644 server/server/types/models/video/video-live-session.ts create mode 100644 server/server/types/models/video/video-live.ts create mode 100644 server/server/types/models/video/video-password.ts create mode 100644 server/server/types/models/video/video-playlist-element.ts create mode 100644 server/server/types/models/video/video-playlist.ts create mode 100644 server/server/types/models/video/video-rate.ts create mode 100644 server/server/types/models/video/video-redundancy.ts create mode 100644 server/server/types/models/video/video-share.ts create mode 100644 server/server/types/models/video/video-source.ts create mode 100644 server/server/types/models/video/video-streaming-playlist.ts create mode 100644 server/server/types/models/video/video.ts create mode 100644 server/server/types/plugins/index.ts create mode 100644 server/server/types/plugins/plugin-library.model.ts create mode 100644 server/server/types/plugins/register-server-auth.model.ts create mode 100644 server/server/types/plugins/register-server-option.model.ts rename server/{ => server}/types/plugins/register-server-websocket-route.model.ts (100%) create mode 100644 server/server/types/sequelize.ts delete mode 100644 server/tests/api/activitypub/cleaner.ts delete mode 100644 server/tests/api/activitypub/client.ts delete mode 100644 server/tests/api/activitypub/fetch.ts delete mode 100644 server/tests/api/activitypub/helpers.ts delete mode 100644 server/tests/api/activitypub/index.ts delete mode 100644 server/tests/api/activitypub/refresher.ts delete mode 100644 server/tests/api/activitypub/security.ts delete mode 100644 server/tests/api/check-params/abuses.ts delete mode 100644 server/tests/api/check-params/accounts.ts delete mode 100644 server/tests/api/check-params/blocklist.ts delete mode 100644 server/tests/api/check-params/bulk.ts delete mode 100644 server/tests/api/check-params/channel-import-videos.ts delete mode 100644 server/tests/api/check-params/config.ts delete mode 100644 server/tests/api/check-params/contact-form.ts delete mode 100644 server/tests/api/check-params/custom-pages.ts delete mode 100644 server/tests/api/check-params/debug.ts delete mode 100644 server/tests/api/check-params/follows.ts delete mode 100644 server/tests/api/check-params/index.ts delete mode 100644 server/tests/api/check-params/jobs.ts delete mode 100644 server/tests/api/check-params/live.ts delete mode 100644 server/tests/api/check-params/logs.ts delete mode 100644 server/tests/api/check-params/metrics.ts delete mode 100644 server/tests/api/check-params/my-user.ts delete mode 100644 server/tests/api/check-params/plugins.ts delete mode 100644 server/tests/api/check-params/redundancy.ts delete mode 100644 server/tests/api/check-params/registrations.ts delete mode 100644 server/tests/api/check-params/runners.ts delete mode 100644 server/tests/api/check-params/search.ts delete mode 100644 server/tests/api/check-params/services.ts delete mode 100644 server/tests/api/check-params/transcoding.ts delete mode 100644 server/tests/api/check-params/two-factor.ts delete mode 100644 server/tests/api/check-params/upload-quota.ts delete mode 100644 server/tests/api/check-params/user-notifications.ts delete mode 100644 server/tests/api/check-params/user-subscriptions.ts delete mode 100644 server/tests/api/check-params/users-admin.ts delete mode 100644 server/tests/api/check-params/users-emails.ts delete mode 100644 server/tests/api/check-params/video-blacklist.ts delete mode 100644 server/tests/api/check-params/video-captions.ts delete mode 100644 server/tests/api/check-params/video-channel-syncs.ts delete mode 100644 server/tests/api/check-params/video-channels.ts delete mode 100644 server/tests/api/check-params/video-comments.ts delete mode 100644 server/tests/api/check-params/video-files.ts delete mode 100644 server/tests/api/check-params/video-imports.ts delete mode 100644 server/tests/api/check-params/video-passwords.ts delete mode 100644 server/tests/api/check-params/video-playlists.ts delete mode 100644 server/tests/api/check-params/video-source.ts delete mode 100644 server/tests/api/check-params/video-storyboards.ts delete mode 100644 server/tests/api/check-params/video-studio.ts delete mode 100644 server/tests/api/check-params/video-token.ts delete mode 100644 server/tests/api/check-params/videos-common-filters.ts delete mode 100644 server/tests/api/check-params/videos-history.ts delete mode 100644 server/tests/api/check-params/videos-overviews.ts delete mode 100644 server/tests/api/check-params/videos.ts delete mode 100644 server/tests/api/check-params/views.ts delete mode 100644 server/tests/api/index.ts delete mode 100644 server/tests/api/live/index.ts delete mode 100644 server/tests/api/live/live-constraints.ts delete mode 100644 server/tests/api/live/live-fast-restream.ts delete mode 100644 server/tests/api/live/live-permanent.ts delete mode 100644 server/tests/api/live/live-rtmps.ts delete mode 100644 server/tests/api/live/live-save-replay.ts delete mode 100644 server/tests/api/live/live-socket-messages.ts delete mode 100644 server/tests/api/live/live.ts delete mode 100644 server/tests/api/moderation/abuses.ts delete mode 100644 server/tests/api/moderation/blocklist-notification.ts delete mode 100644 server/tests/api/moderation/blocklist.ts delete mode 100644 server/tests/api/moderation/index.ts delete mode 100644 server/tests/api/moderation/video-blacklist.ts delete mode 100644 server/tests/api/notifications/admin-notifications.ts delete mode 100644 server/tests/api/notifications/comments-notifications.ts delete mode 100644 server/tests/api/notifications/index.ts delete mode 100644 server/tests/api/notifications/moderation-notifications.ts delete mode 100644 server/tests/api/notifications/notifications-api.ts delete mode 100644 server/tests/api/notifications/registrations-notifications.ts delete mode 100644 server/tests/api/notifications/user-notifications.ts delete mode 100644 server/tests/api/object-storage/index.ts delete mode 100644 server/tests/api/object-storage/live.ts delete mode 100644 server/tests/api/object-storage/video-imports.ts delete mode 100644 server/tests/api/object-storage/video-static-file-privacy.ts delete mode 100644 server/tests/api/object-storage/videos.ts delete mode 100644 server/tests/api/redundancy/index.ts delete mode 100644 server/tests/api/redundancy/manage-redundancy.ts delete mode 100644 server/tests/api/redundancy/redundancy-constraints.ts delete mode 100644 server/tests/api/redundancy/redundancy.ts delete mode 100644 server/tests/api/runners/index.ts delete mode 100644 server/tests/api/runners/runner-common.ts delete mode 100644 server/tests/api/runners/runner-live-transcoding.ts delete mode 100644 server/tests/api/runners/runner-socket.ts delete mode 100644 server/tests/api/runners/runner-studio-transcoding.ts delete mode 100644 server/tests/api/runners/runner-vod-transcoding.ts delete mode 100644 server/tests/api/search/index.ts delete mode 100644 server/tests/api/search/search-activitypub-video-channels.ts delete mode 100644 server/tests/api/search/search-activitypub-video-playlists.ts delete mode 100644 server/tests/api/search/search-activitypub-videos.ts delete mode 100644 server/tests/api/search/search-channels.ts delete mode 100644 server/tests/api/search/search-index.ts delete mode 100644 server/tests/api/search/search-playlists.ts delete mode 100644 server/tests/api/search/search-videos.ts delete mode 100644 server/tests/api/server/auto-follows.ts delete mode 100644 server/tests/api/server/bulk.ts delete mode 100644 server/tests/api/server/config-defaults.ts delete mode 100644 server/tests/api/server/config.ts delete mode 100644 server/tests/api/server/contact-form.ts delete mode 100644 server/tests/api/server/email.ts delete mode 100644 server/tests/api/server/follow-constraints.ts delete mode 100644 server/tests/api/server/follows-moderation.ts delete mode 100644 server/tests/api/server/follows.ts delete mode 100644 server/tests/api/server/handle-down.ts delete mode 100644 server/tests/api/server/homepage.ts delete mode 100644 server/tests/api/server/index.ts delete mode 100644 server/tests/api/server/jobs.ts delete mode 100644 server/tests/api/server/logs.ts delete mode 100644 server/tests/api/server/no-client.ts delete mode 100644 server/tests/api/server/open-telemetry.ts delete mode 100644 server/tests/api/server/plugins.ts delete mode 100644 server/tests/api/server/proxy.ts delete mode 100644 server/tests/api/server/reverse-proxy.ts delete mode 100644 server/tests/api/server/services.ts delete mode 100644 server/tests/api/server/slow-follows.ts delete mode 100644 server/tests/api/server/stats.ts delete mode 100644 server/tests/api/server/tracker.ts delete mode 100644 server/tests/api/transcoding/audio-only.ts delete mode 100644 server/tests/api/transcoding/create-transcoding.ts delete mode 100644 server/tests/api/transcoding/hls.ts delete mode 100644 server/tests/api/transcoding/index.ts delete mode 100644 server/tests/api/transcoding/transcoder.ts delete mode 100644 server/tests/api/transcoding/update-while-transcoding.ts delete mode 100644 server/tests/api/transcoding/video-studio.ts delete mode 100644 server/tests/api/users/index.ts delete mode 100644 server/tests/api/users/oauth.ts delete mode 100644 server/tests/api/users/registrations.ts delete mode 100644 server/tests/api/users/two-factor.ts delete mode 100644 server/tests/api/users/user-subscriptions.ts delete mode 100644 server/tests/api/users/user-videos.ts delete mode 100644 server/tests/api/users/users-email-verification.ts delete mode 100644 server/tests/api/users/users-multiple-servers.ts delete mode 100644 server/tests/api/users/users.ts delete mode 100644 server/tests/api/videos/channel-import-videos.ts delete mode 100644 server/tests/api/videos/index.ts delete mode 100644 server/tests/api/videos/multiple-servers.ts delete mode 100644 server/tests/api/videos/resumable-upload.ts delete mode 100644 server/tests/api/videos/single-server.ts delete mode 100644 server/tests/api/videos/video-captions.ts delete mode 100644 server/tests/api/videos/video-change-ownership.ts delete mode 100644 server/tests/api/videos/video-channel-syncs.ts delete mode 100644 server/tests/api/videos/video-channels.ts delete mode 100644 server/tests/api/videos/video-comments.ts delete mode 100644 server/tests/api/videos/video-description.ts delete mode 100644 server/tests/api/videos/video-files.ts delete mode 100644 server/tests/api/videos/video-imports.ts delete mode 100644 server/tests/api/videos/video-nsfw.ts delete mode 100644 server/tests/api/videos/video-passwords.ts delete mode 100644 server/tests/api/videos/video-playlist-thumbnails.ts delete mode 100644 server/tests/api/videos/video-playlists.ts delete mode 100644 server/tests/api/videos/video-privacy.ts delete mode 100644 server/tests/api/videos/video-schedule-update.ts delete mode 100644 server/tests/api/videos/video-source.ts delete mode 100644 server/tests/api/videos/video-static-file-privacy.ts delete mode 100644 server/tests/api/videos/video-storyboard.ts delete mode 100644 server/tests/api/videos/videos-common-filters.ts delete mode 100644 server/tests/api/videos/videos-history.ts delete mode 100644 server/tests/api/videos/videos-overview.ts delete mode 100644 server/tests/api/views/index.ts delete mode 100644 server/tests/api/views/video-views-counter.ts delete mode 100644 server/tests/api/views/video-views-overall-stats.ts delete mode 100644 server/tests/api/views/video-views-retention-stats.ts delete mode 100644 server/tests/api/views/video-views-timeserie-stats.ts delete mode 100644 server/tests/api/views/videos-views-cleaner.ts delete mode 100644 server/tests/cli/create-generate-storyboard-job.ts delete mode 100644 server/tests/cli/create-import-video-file-job.ts delete mode 100644 server/tests/cli/create-move-video-storage-job.ts delete mode 100644 server/tests/cli/peertube.ts delete mode 100644 server/tests/cli/plugins.ts delete mode 100644 server/tests/cli/prune-storage.ts delete mode 100644 server/tests/cli/regenerate-thumbnails.ts delete mode 100644 server/tests/cli/reset-password.ts delete mode 100644 server/tests/cli/update-host.ts delete mode 100644 server/tests/client.ts delete mode 100644 server/tests/external-plugins/akismet.ts delete mode 100644 server/tests/external-plugins/auth-ldap.ts delete mode 100644 server/tests/external-plugins/auto-block-videos.ts delete mode 100644 server/tests/external-plugins/auto-mute.ts delete mode 100644 server/tests/feeds/feeds.ts delete mode 100644 server/tests/helpers/comment-model.ts delete mode 100644 server/tests/helpers/core-utils.ts delete mode 100644 server/tests/helpers/crypto.ts delete mode 100644 server/tests/helpers/dns.ts delete mode 100644 server/tests/helpers/image.ts delete mode 100644 server/tests/helpers/index.ts delete mode 100644 server/tests/helpers/markdown.ts delete mode 100644 server/tests/helpers/request.ts delete mode 100644 server/tests/helpers/validator.ts delete mode 100644 server/tests/helpers/version.ts delete mode 100644 server/tests/index.ts delete mode 100644 server/tests/lib/index.ts delete mode 100644 server/tests/lib/video-constant-registry-factory.ts delete mode 100644 server/tests/misc-endpoints.ts delete mode 100644 server/tests/peertube-runner/client-cli.ts delete mode 100644 server/tests/peertube-runner/index.ts delete mode 100644 server/tests/peertube-runner/live-transcoding.ts delete mode 100644 server/tests/peertube-runner/studio-transcoding.ts delete mode 100644 server/tests/peertube-runner/vod-transcoding.ts delete mode 100644 server/tests/plugins/action-hooks.ts delete mode 100644 server/tests/plugins/external-auth.ts delete mode 100644 server/tests/plugins/filter-hooks.ts delete mode 100644 server/tests/plugins/html-injection.ts delete mode 100644 server/tests/plugins/id-and-pass-auth.ts delete mode 100644 server/tests/plugins/plugin-helpers.ts delete mode 100644 server/tests/plugins/plugin-router.ts delete mode 100644 server/tests/plugins/plugin-storage.ts delete mode 100644 server/tests/plugins/plugin-transcoding.ts delete mode 100644 server/tests/plugins/plugin-unloading.ts delete mode 100644 server/tests/plugins/plugin-websocket.ts delete mode 100644 server/tests/plugins/translations.ts delete mode 100644 server/tests/plugins/video-constants.ts delete mode 100644 server/tests/shared/actors.ts delete mode 100644 server/tests/shared/captions.ts delete mode 100644 server/tests/shared/checks.ts delete mode 100644 server/tests/shared/directories.ts delete mode 100644 server/tests/shared/generate.ts delete mode 100644 server/tests/shared/index.ts delete mode 100644 server/tests/shared/live.ts delete mode 100644 server/tests/shared/mock-servers/index.ts delete mode 100644 server/tests/shared/mock-servers/mock-429.ts delete mode 100644 server/tests/shared/mock-servers/mock-email.ts delete mode 100644 server/tests/shared/mock-servers/mock-http.ts delete mode 100644 server/tests/shared/mock-servers/mock-instances-index.ts delete mode 100644 server/tests/shared/mock-servers/mock-joinpeertube-versions.ts delete mode 100644 server/tests/shared/mock-servers/mock-object-storage.ts delete mode 100644 server/tests/shared/mock-servers/mock-plugin-blocklist.ts delete mode 100644 server/tests/shared/mock-servers/mock-proxy.ts delete mode 100644 server/tests/shared/notifications.ts delete mode 100644 server/tests/shared/peertube-runner-process.ts delete mode 100644 server/tests/shared/plugins.ts delete mode 100644 server/tests/shared/requests.ts delete mode 100644 server/tests/shared/sql-command.ts delete mode 100644 server/tests/shared/streaming-playlists.ts delete mode 100644 server/tests/shared/tracker.ts delete mode 100644 server/tests/shared/video-playlists.ts delete mode 100644 server/tests/shared/videos.ts delete mode 100644 server/tests/shared/views.ts delete mode 100644 server/tests/shared/webtorrent.ts delete mode 100644 server/tools/README.md delete mode 100644 server/tools/package.json delete mode 100644 server/tools/peertube-auth.ts delete mode 100644 server/tools/peertube-get-access-token.ts delete mode 100644 server/tools/peertube-import-videos.ts delete mode 100644 server/tools/peertube-plugins.ts delete mode 100644 server/tools/peertube-redundancy.ts delete mode 100644 server/tools/peertube-upload.ts delete mode 100644 server/tools/peertube.ts delete mode 100644 server/tools/shared/cli.ts delete mode 100644 server/tools/shared/index.ts delete mode 100644 server/tools/tsconfig.json delete mode 100644 server/tools/yarn.lock create mode 100644 server/tsconfig.lib.json delete mode 100644 server/types/activitypub-processor.model.ts delete mode 100644 server/types/express.d.ts delete mode 100644 server/types/index.ts delete mode 100644 server/types/models/abuse/abuse-message.ts delete mode 100644 server/types/models/abuse/abuse.ts delete mode 100644 server/types/models/abuse/index.ts delete mode 100644 server/types/models/account/account-blocklist.ts delete mode 100644 server/types/models/account/account.ts delete mode 100644 server/types/models/account/actor-custom-page.ts delete mode 100644 server/types/models/account/index.ts delete mode 100644 server/types/models/actor/actor-follow.ts delete mode 100644 server/types/models/actor/actor-image.ts delete mode 100644 server/types/models/actor/actor.ts delete mode 100644 server/types/models/actor/index.ts delete mode 100644 server/types/models/application/application.ts delete mode 100644 server/types/models/application/index.ts delete mode 100644 server/types/models/index.ts delete mode 100644 server/types/models/oauth/index.ts delete mode 100644 server/types/models/oauth/oauth-client.ts delete mode 100644 server/types/models/oauth/oauth-token.ts delete mode 100644 server/types/models/runners/index.ts delete mode 100644 server/types/models/runners/runner-job.ts delete mode 100644 server/types/models/runners/runner-registration-token.ts delete mode 100644 server/types/models/runners/runner.ts delete mode 100644 server/types/models/server/index.ts delete mode 100644 server/types/models/server/plugin.ts delete mode 100644 server/types/models/server/server-blocklist.ts delete mode 100644 server/types/models/server/server.ts delete mode 100644 server/types/models/server/tracker.ts delete mode 100644 server/types/models/user/index.ts delete mode 100644 server/types/models/user/user-notification-setting.ts delete mode 100644 server/types/models/user/user-notification.ts delete mode 100644 server/types/models/user/user-registration.ts delete mode 100644 server/types/models/user/user-video-history.ts delete mode 100644 server/types/models/user/user.ts delete mode 100644 server/types/models/video/index.ts delete mode 100644 server/types/models/video/local-video-viewer-watch-section.ts delete mode 100644 server/types/models/video/local-video-viewer.ts delete mode 100644 server/types/models/video/schedule-video-update.ts delete mode 100644 server/types/models/video/storyboard.ts delete mode 100644 server/types/models/video/tag.ts delete mode 100644 server/types/models/video/thumbnail.ts delete mode 100644 server/types/models/video/video-blacklist.ts delete mode 100644 server/types/models/video/video-caption.ts delete mode 100644 server/types/models/video/video-change-ownership.ts delete mode 100644 server/types/models/video/video-channel-sync.ts delete mode 100644 server/types/models/video/video-channels.ts delete mode 100644 server/types/models/video/video-comment.ts delete mode 100644 server/types/models/video/video-file.ts delete mode 100644 server/types/models/video/video-import.ts delete mode 100644 server/types/models/video/video-live-replay-setting.ts delete mode 100644 server/types/models/video/video-live-session.ts delete mode 100644 server/types/models/video/video-live.ts delete mode 100644 server/types/models/video/video-password.ts delete mode 100644 server/types/models/video/video-playlist-element.ts delete mode 100644 server/types/models/video/video-playlist.ts delete mode 100644 server/types/models/video/video-rate.ts delete mode 100644 server/types/models/video/video-redundancy.ts delete mode 100644 server/types/models/video/video-share.ts delete mode 100644 server/types/models/video/video-source.ts delete mode 100644 server/types/models/video/video-streaming-playlist.ts delete mode 100644 server/types/models/video/video.ts delete mode 100644 server/types/plugins/index.ts delete mode 100644 server/types/plugins/plugin-library.model.ts delete mode 100644 server/types/plugins/register-server-auth.model.ts delete mode 100644 server/types/plugins/register-server-option.model.ts delete mode 100644 server/types/sequelize.ts delete mode 100644 shared/core-utils/abuse/abuse-predefined-reasons.ts delete mode 100644 shared/core-utils/abuse/index.ts delete mode 100644 shared/core-utils/common/env.ts delete mode 100644 shared/core-utils/common/index.ts delete mode 100644 shared/core-utils/common/path.ts delete mode 100644 shared/core-utils/common/url.ts delete mode 100644 shared/core-utils/i18n/index.ts delete mode 100644 shared/core-utils/index.ts delete mode 100644 shared/core-utils/plugins/hooks.ts delete mode 100644 shared/core-utils/plugins/index.ts delete mode 100644 shared/core-utils/renderer/index.ts delete mode 100644 shared/core-utils/users/index.ts delete mode 100644 shared/core-utils/users/user-role.ts delete mode 100644 shared/core-utils/videos/bitrate.ts delete mode 100644 shared/core-utils/videos/common.ts delete mode 100644 shared/core-utils/videos/index.ts delete mode 100644 shared/extra-utils/file.ts delete mode 100644 shared/extra-utils/index.ts delete mode 100644 shared/extra-utils/uuid.ts delete mode 100644 shared/ffmpeg/ffmpeg-command-wrapper.ts delete mode 100644 shared/ffmpeg/ffmpeg-default-transcoding-profile.ts delete mode 100644 shared/ffmpeg/ffmpeg-edition.ts delete mode 100644 shared/ffmpeg/ffmpeg-images.ts delete mode 100644 shared/ffmpeg/ffmpeg-live.ts delete mode 100644 shared/ffmpeg/ffmpeg-utils.ts delete mode 100644 shared/ffmpeg/ffmpeg-vod.ts delete mode 100644 shared/ffmpeg/ffprobe.ts delete mode 100644 shared/ffmpeg/index.ts delete mode 100644 shared/ffmpeg/shared/encoder-options.ts delete mode 100644 shared/ffmpeg/shared/index.ts delete mode 100644 shared/ffmpeg/shared/presets.ts delete mode 100644 shared/models/activitypub/activity.ts delete mode 100644 shared/models/activitypub/activitypub-actor.ts delete mode 100644 shared/models/activitypub/activitypub-collection.ts delete mode 100644 shared/models/activitypub/activitypub-root.ts delete mode 100644 shared/models/activitypub/index.ts delete mode 100644 shared/models/activitypub/objects/abuse-object.ts delete mode 100644 shared/models/activitypub/objects/activitypub-object.ts delete mode 100644 shared/models/activitypub/objects/cache-file-object.ts delete mode 100644 shared/models/activitypub/objects/common-objects.ts delete mode 100644 shared/models/activitypub/objects/index.ts delete mode 100644 shared/models/activitypub/objects/playlist-object.ts delete mode 100644 shared/models/activitypub/objects/video-comment-object.ts delete mode 100644 shared/models/activitypub/objects/video-object.ts delete mode 100644 shared/models/actors/account.model.ts delete mode 100644 shared/models/actors/actor-image.type.ts delete mode 100644 shared/models/actors/actor.model.ts delete mode 100644 shared/models/actors/follow.model.ts delete mode 100644 shared/models/actors/index.ts delete mode 100644 shared/models/bulk/index.ts delete mode 100644 shared/models/common/index.ts delete mode 100644 shared/models/custom-markup/index.ts delete mode 100644 shared/models/feeds/feed-format.enum.ts delete mode 100644 shared/models/feeds/index.ts delete mode 100644 shared/models/http/http-error-codes.ts delete mode 100644 shared/models/http/http-methods.ts delete mode 100644 shared/models/http/index.ts delete mode 100644 shared/models/index.ts delete mode 100644 shared/models/joinpeertube/index.ts delete mode 100644 shared/models/metrics/index.ts delete mode 100644 shared/models/metrics/playback-metric-create.model.ts delete mode 100644 shared/models/moderation/abuse/abuse-create.model.ts delete mode 100644 shared/models/moderation/abuse/abuse-message.model.ts delete mode 100644 shared/models/moderation/abuse/abuse-reason.model.ts delete mode 100644 shared/models/moderation/abuse/abuse-state.model.ts delete mode 100644 shared/models/moderation/abuse/abuse-update.model.ts delete mode 100644 shared/models/moderation/abuse/abuse.model.ts delete mode 100644 shared/models/moderation/abuse/index.ts delete mode 100644 shared/models/moderation/account-block.model.ts delete mode 100644 shared/models/moderation/index.ts delete mode 100644 shared/models/moderation/server-block.model.ts delete mode 100644 shared/models/nodeinfo/index.ts delete mode 100644 shared/models/overviews/index.ts delete mode 100644 shared/models/overviews/videos-overview.model.ts delete mode 100644 shared/models/plugins/client/index.ts delete mode 100644 shared/models/plugins/client/register-client-hook.model.ts delete mode 100644 shared/models/plugins/client/register-client-settings-script.model.ts delete mode 100644 shared/models/plugins/hook-type.enum.ts delete mode 100644 shared/models/plugins/index.ts delete mode 100644 shared/models/plugins/plugin-index/index.ts delete mode 100644 shared/models/plugins/plugin-index/peertube-plugin-index-list.model.ts delete mode 100644 shared/models/plugins/plugin-package-json.model.ts delete mode 100644 shared/models/plugins/plugin.type.ts delete mode 100644 shared/models/plugins/server/api/index.ts delete mode 100644 shared/models/plugins/server/api/peertube-plugin.model.ts delete mode 100644 shared/models/plugins/server/index.ts delete mode 100644 shared/models/plugins/server/managers/index.ts delete mode 100644 shared/models/plugins/server/managers/plugin-playlist-privacy-manager.model.ts delete mode 100644 shared/models/plugins/server/managers/plugin-transcoding-manager.model.ts delete mode 100644 shared/models/plugins/server/managers/plugin-video-category-manager.model.ts delete mode 100644 shared/models/plugins/server/managers/plugin-video-language-manager.model.ts delete mode 100644 shared/models/plugins/server/managers/plugin-video-licence-manager.model.ts delete mode 100644 shared/models/plugins/server/managers/plugin-video-privacy-manager.model.ts delete mode 100644 shared/models/plugins/server/register-server-hook.model.ts delete mode 100644 shared/models/plugins/server/settings/index.ts delete mode 100644 shared/models/plugins/server/settings/public-server.setting.ts delete mode 100644 shared/models/plugins/server/settings/register-server-setting.model.ts delete mode 100644 shared/models/redundancy/index.ts delete mode 100644 shared/models/runners/accept-runner-job-result.model.ts delete mode 100644 shared/models/runners/index.ts delete mode 100644 shared/models/runners/list-runner-jobs-query.model.ts delete mode 100644 shared/models/runners/request-runner-job-result.model.ts delete mode 100644 shared/models/runners/runner-job-payload.model.ts delete mode 100644 shared/models/runners/runner-job-private-payload.model.ts delete mode 100644 shared/models/runners/runner-job-state.model.ts delete mode 100644 shared/models/runners/runner-job.model.ts delete mode 100644 shared/models/search/index.ts delete mode 100644 shared/models/search/video-channels-search-query.model.ts delete mode 100644 shared/models/search/video-playlists-search-query.model.ts delete mode 100644 shared/models/search/videos-common-query.model.ts delete mode 100644 shared/models/search/videos-search-query.model.ts delete mode 100644 shared/models/server/client-log-create.model.ts delete mode 100644 shared/models/server/custom-config.model.ts delete mode 100644 shared/models/server/index.ts delete mode 100644 shared/models/server/job.model.ts delete mode 100644 shared/models/server/peertube-problem-document.model.ts delete mode 100644 shared/models/server/server-config.model.ts delete mode 100644 shared/models/server/server-error-code.enum.ts delete mode 100644 shared/models/server/server-stats.model.ts delete mode 100644 shared/models/tokens/index.ts delete mode 100644 shared/models/users/index.ts delete mode 100644 shared/models/users/registration/index.ts delete mode 100644 shared/models/users/registration/user-registration-request.model.ts delete mode 100644 shared/models/users/registration/user-registration-state.model.ts delete mode 100644 shared/models/users/registration/user-registration.model.ts delete mode 100644 shared/models/users/user-create.model.ts delete mode 100644 shared/models/users/user-flag.model.ts delete mode 100644 shared/models/users/user-notification-setting.model.ts delete mode 100644 shared/models/users/user-notification.model.ts delete mode 100644 shared/models/users/user-right.enum.ts delete mode 100644 shared/models/users/user-role.ts delete mode 100644 shared/models/users/user-update-me.model.ts delete mode 100644 shared/models/users/user-update.model.ts delete mode 100644 shared/models/users/user.model.ts delete mode 100644 shared/models/videos/blacklist/index.ts delete mode 100644 shared/models/videos/blacklist/video-blacklist.model.ts delete mode 100644 shared/models/videos/caption/index.ts delete mode 100644 shared/models/videos/caption/video-caption.model.ts delete mode 100644 shared/models/videos/change-ownership/index.ts delete mode 100644 shared/models/videos/change-ownership/video-change-ownership.model.ts delete mode 100644 shared/models/videos/channel-sync/index.ts delete mode 100644 shared/models/videos/channel-sync/video-channel-sync-state.enum.ts delete mode 100644 shared/models/videos/channel-sync/video-channel-sync.model.ts delete mode 100644 shared/models/videos/channel/index.ts delete mode 100644 shared/models/videos/channel/video-channel.model.ts delete mode 100644 shared/models/videos/comment/index.ts delete mode 100644 shared/models/videos/comment/video-comment.model.ts delete mode 100644 shared/models/videos/file/index.ts delete mode 100644 shared/models/videos/file/video-file.model.ts delete mode 100644 shared/models/videos/file/video-resolution.enum.ts delete mode 100644 shared/models/videos/import/index.ts delete mode 100644 shared/models/videos/import/video-import-create.model.ts delete mode 100644 shared/models/videos/import/video-import-state.enum.ts delete mode 100644 shared/models/videos/import/video-import.model.ts delete mode 100644 shared/models/videos/index.ts delete mode 100644 shared/models/videos/live/index.ts delete mode 100644 shared/models/videos/live/live-video-create.model.ts delete mode 100644 shared/models/videos/live/live-video-error.enum.ts delete mode 100644 shared/models/videos/live/live-video-event-payload.model.ts delete mode 100644 shared/models/videos/live/live-video-latency-mode.enum.ts delete mode 100644 shared/models/videos/live/live-video-session.model.ts delete mode 100644 shared/models/videos/live/live-video-update.model.ts delete mode 100644 shared/models/videos/live/live-video.model.ts delete mode 100644 shared/models/videos/playlist/index.ts delete mode 100644 shared/models/videos/playlist/video-playlist-create.model.ts delete mode 100644 shared/models/videos/playlist/video-playlist-element.model.ts delete mode 100644 shared/models/videos/playlist/video-playlist-privacy.model.ts delete mode 100644 shared/models/videos/playlist/video-playlist-type.model.ts delete mode 100644 shared/models/videos/playlist/video-playlist-update.model.ts delete mode 100644 shared/models/videos/playlist/video-playlist.model.ts delete mode 100644 shared/models/videos/rate/account-video-rate.model.ts delete mode 100644 shared/models/videos/rate/index.ts delete mode 100644 shared/models/videos/rate/user-video-rate-update.model.ts delete mode 100644 shared/models/videos/rate/user-video-rate.model.ts delete mode 100644 shared/models/videos/stats/index.ts delete mode 100644 shared/models/videos/studio/index.ts delete mode 100644 shared/models/videos/thumbnail.type.ts delete mode 100644 shared/models/videos/transcoding/index.ts delete mode 100644 shared/models/videos/transcoding/video-transcoding.model.ts delete mode 100644 shared/models/videos/video-create.model.ts delete mode 100644 shared/models/videos/video-include.enum.ts delete mode 100644 shared/models/videos/video-privacy.enum.ts delete mode 100644 shared/models/videos/video-schedule-update.model.ts delete mode 100644 shared/models/videos/video-state.enum.ts delete mode 100644 shared/models/videos/video-storage.enum.ts delete mode 100644 shared/models/videos/video-streaming-playlist.model.ts delete mode 100644 shared/models/videos/video-streaming-playlist.type.ts delete mode 100644 shared/models/videos/video-update.model.ts delete mode 100644 shared/models/videos/video.model.ts delete mode 100644 shared/server-commands/bulk/bulk-command.ts delete mode 100644 shared/server-commands/bulk/index.ts delete mode 100644 shared/server-commands/cli/cli-command.ts delete mode 100644 shared/server-commands/cli/index.ts delete mode 100644 shared/server-commands/custom-pages/custom-pages-command.ts delete mode 100644 shared/server-commands/custom-pages/index.ts delete mode 100644 shared/server-commands/feeds/feeds-command.ts delete mode 100644 shared/server-commands/feeds/index.ts delete mode 100644 shared/server-commands/index.ts delete mode 100644 shared/server-commands/logs/index.ts delete mode 100644 shared/server-commands/logs/logs-command.ts delete mode 100644 shared/server-commands/moderation/abuses-command.ts delete mode 100644 shared/server-commands/moderation/index.ts delete mode 100644 shared/server-commands/overviews/index.ts delete mode 100644 shared/server-commands/overviews/overviews-command.ts delete mode 100644 shared/server-commands/requests/index.ts delete mode 100644 shared/server-commands/requests/requests.ts delete mode 100644 shared/server-commands/runners/index.ts delete mode 100644 shared/server-commands/runners/runner-jobs-command.ts delete mode 100644 shared/server-commands/runners/runner-registration-tokens-command.ts delete mode 100644 shared/server-commands/runners/runners-command.ts delete mode 100644 shared/server-commands/search/index.ts delete mode 100644 shared/server-commands/search/search-command.ts delete mode 100644 shared/server-commands/server/config-command.ts delete mode 100644 shared/server-commands/server/contact-form-command.ts delete mode 100644 shared/server-commands/server/debug-command.ts delete mode 100644 shared/server-commands/server/follows-command.ts delete mode 100644 shared/server-commands/server/follows.ts delete mode 100644 shared/server-commands/server/index.ts delete mode 100644 shared/server-commands/server/jobs-command.ts delete mode 100644 shared/server-commands/server/jobs.ts delete mode 100644 shared/server-commands/server/metrics-command.ts delete mode 100644 shared/server-commands/server/object-storage-command.ts delete mode 100644 shared/server-commands/server/plugins-command.ts delete mode 100644 shared/server-commands/server/redundancy-command.ts delete mode 100644 shared/server-commands/server/server.ts delete mode 100644 shared/server-commands/server/servers-command.ts delete mode 100644 shared/server-commands/server/servers.ts delete mode 100644 shared/server-commands/server/stats-command.ts delete mode 100644 shared/server-commands/shared/abstract-command.ts delete mode 100644 shared/server-commands/shared/index.ts delete mode 100644 shared/server-commands/socket/index.ts delete mode 100644 shared/server-commands/socket/socket-io-command.ts delete mode 100644 shared/server-commands/users/accounts-command.ts delete mode 100644 shared/server-commands/users/accounts.ts delete mode 100644 shared/server-commands/users/blocklist-command.ts delete mode 100644 shared/server-commands/users/index.ts delete mode 100644 shared/server-commands/users/login-command.ts delete mode 100644 shared/server-commands/users/login.ts delete mode 100644 shared/server-commands/users/notifications-command.ts delete mode 100644 shared/server-commands/users/registrations-command.ts delete mode 100644 shared/server-commands/users/subscriptions-command.ts delete mode 100644 shared/server-commands/users/two-factor-command.ts delete mode 100644 shared/server-commands/users/users-command.ts delete mode 100644 shared/server-commands/videos/blacklist-command.ts delete mode 100644 shared/server-commands/videos/captions-command.ts delete mode 100644 shared/server-commands/videos/change-ownership-command.ts delete mode 100644 shared/server-commands/videos/channel-syncs-command.ts delete mode 100644 shared/server-commands/videos/channels-command.ts delete mode 100644 shared/server-commands/videos/channels.ts delete mode 100644 shared/server-commands/videos/comments-command.ts delete mode 100644 shared/server-commands/videos/history-command.ts delete mode 100644 shared/server-commands/videos/imports-command.ts delete mode 100644 shared/server-commands/videos/index.ts delete mode 100644 shared/server-commands/videos/live-command.ts delete mode 100644 shared/server-commands/videos/live.ts delete mode 100644 shared/server-commands/videos/playlists-command.ts delete mode 100644 shared/server-commands/videos/services-command.ts delete mode 100644 shared/server-commands/videos/storyboard-command.ts delete mode 100644 shared/server-commands/videos/streaming-playlists-command.ts delete mode 100644 shared/server-commands/videos/video-passwords-command.ts delete mode 100644 shared/server-commands/videos/video-stats-command.ts delete mode 100644 shared/server-commands/videos/video-studio-command.ts delete mode 100644 shared/server-commands/videos/video-token-command.ts delete mode 100644 shared/server-commands/videos/videos-command.ts delete mode 100644 shared/server-commands/videos/views-command.ts delete mode 100644 shared/tsconfig.json delete mode 100644 shared/tsconfig.types.json delete mode 100644 shared/typescript-utils/index.ts create mode 100644 tsconfig.eslint.json delete mode 100644 tsconfig.json diff --git a/.eslintrc.json b/.eslintrc.json index 4e80ff224..3bfdee6eb 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -1,5 +1,6 @@ { "extends": "standard-with-typescript", + "root": true, "rules": { "eol-last": [ "error", @@ -126,18 +127,20 @@ ] }, "ignorePatterns": [ - "node_modules/", - "server/tests/fixtures" + "node_modules", + "packages/tests/fixtures", + "apps/**/dist", + "packages/**/dist", + "server/dist", + "packages/types-generator/tests", + "*.js", + "/client", + "/dist" ], "parserOptions": { - "EXPERIMENTAL_useSourceOfProjectReferenceRedirect": true, "project": [ - "./tsconfig.json", - "./shared/tsconfig.json", - "./scripts/tsconfig.json", - "./server/tsconfig.json", - "./server/tools/tsconfig.json", - "./packages/peertube-runner/tsconfig.json" - ] + "./tsconfig.eslint.json" + ], + "EXPERIMENTAL_useSourceOfProjectReferenceRedirect": true } } diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 80b7cba3c..16863eceb 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -53,13 +53,25 @@ interested in, user interface, design, decentralized architecture... You can help to write the documentation of the REST API, code, architecture, demonstrations. -For the REST API you can see the documentation in [/support/doc/api](https://github.com/Chocobozzz/PeerTube/tree/develop/support/doc/api) directory. -Then, you can just open the `openapi.yaml` file in a special editor like [http://editor.swagger.io/](http://editor.swagger.io/) to easily see and edit the documentation. You can also use [redoc-cli](https://github.com/Redocly/redoc/blob/master/cli/README.md) and run `redoc-cli serve --watch support/doc/api/openapi.yaml` to see the final result. +### User documentation + +The official user documentation is available on https://docs.joinpeertube.org/ + +You can update it by writing markdown files in the following repository: https://framagit.org/framasoft/peertube/documentation/ + +### REST API documentation + +The [REST API documentation](https://docs.joinpeertube.org/api-rest-reference.html) is generated from `support/doc/api/openapi.yaml` file. +To quickly get a preview of your changes, you can generate the documentation *on the fly* using the following command: + +``` +npx @redocly/cli preview-docs ./support/doc/api/openapi.yaml +``` Some hints: - * Routes are defined in [/server/controllers/](https://github.com/Chocobozzz/PeerTube/tree/develop/server/controllers) directory - * Parameters validators are defined in [/server/middlewares/validators](https://github.com/Chocobozzz/PeerTube/tree/develop/server/middlewares/validators) directory - * Models sent/received by the controllers are defined in [/shared/models](https://github.com/Chocobozzz/PeerTube/tree/develop/shared/models) directory + * Routes are defined in [/server/server/controllers/](https://github.com/Chocobozzz/PeerTube/tree/develop/server/server/controllers) directory + * Parameters validators are defined in [/server/server/middlewares/validators](https://github.com/Chocobozzz/PeerTube/tree/develop/server/server/middlewares/validators) directory + * Models sent/received by the controllers are defined in [/packages/models](https://github.com/Chocobozzz/PeerTube/tree/develop/packages/models) directory ## Improve the website @@ -242,15 +254,6 @@ To test emails with PeerTube: * Run [mailslurper](http://mailslurper.com/) * Run PeerTube using mailslurper SMTP port: `NODE_CONFIG='{ "smtp": { "hostname": "localhost", "port": 2500, "tls": false } }' NODE_ENV=dev node dist/server` -### OpenAPI documentation - -The [REST API documentation](https://docs.joinpeertube.org/api-rest-reference.html) is generated from `support/doc/api/openapi.yaml` file. -To quickly get a preview of your changes, you can generate the documentation *on the fly* using the following command: - -``` -npx @redocly/cli preview-docs ./support/doc/api/openapi.yaml -``` - ### Environment variables PeerTube can be configured using environment variables. diff --git a/.github/actions/reusable-prepare-peertube-build/action.yml b/.github/actions/reusable-prepare-peertube-build/action.yml index 03034f6cc..13e22ceb4 100644 --- a/.github/actions/reusable-prepare-peertube-build/action.yml +++ b/.github/actions/reusable-prepare-peertube-build/action.yml @@ -32,4 +32,12 @@ runs: - name: Install peertube runner dependencies shell: bash - run: cd packages/peertube-runner && yarn install --frozen-lockfile + run: cd apps/peertube-runner && yarn install --frozen-lockfile + + - name: Install peertube CLI dependencies + shell: bash + run: cd apps/peertube-cli && yarn install --frozen-lockfile + + - name: Display PeerTube dependencies + shell: bash + run: ls -l node_modules/@peertube diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index ab1780c74..5f56db3ad 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -71,7 +71,7 @@ jobs: - name: Run benchmark run: | - node dist/scripts/benchmark.js -o benchmark.json + npm run benchmark-server -- -o benchmark.json - name: Display result run: | diff --git a/.github/workflows/codeql/codeql-config.yml b/.github/workflows/codeql/codeql-config.yml index 8b771ae99..c96a4693e 100644 --- a/.github/workflows/codeql/codeql-config.yml +++ b/.github/workflows/codeql/codeql-config.yml @@ -1,4 +1,4 @@ name: "PeerTube CodeQL config" paths-ignore: - - server/tests + - packages/tests diff --git a/.github/workflows/stats.yml b/.github/workflows/stats.yml index e772fdb81..573b42975 100644 --- a/.github/workflows/stats.yml +++ b/.github/workflows/stats.yml @@ -36,12 +36,12 @@ jobs: run: | wget "https://github.com/boyter/scc/releases/download/v3.0.0/scc-3.0.0-x86_64-unknown-linux.zip" unzip "scc-3.0.0-x86_64-unknown-linux.zip" - ./scc --format=json --exclude-dir .git,node_modules,client/node_modules,client/dist,dist,yarn.lock,client/yarn.lock,client/src/locale,test1,test2,test3,client/src/assets/images,config,storage,server/tests/fixtures,support/openapi,.idea,.vscode,docker-volume,ffmpeg-3,ffmpeg-4 > ./scc.json + ./scc --format=json --exclude-dir .git,node_modules,client/node_modules,client/dist,dist,yarn.lock,client/yarn.lock,client/src/locale,test1,test2,test3,client/src/assets/images,config,storage,packages/tests/fixtures,support/openapi,.idea,.vscode,docker-volume,ffmpeg-3,ffmpeg-4 > ./scc.json - name: PeerTube client stats if: github.event_name != 'pull_request' run: | - node dist/scripts/client-build-stats.js > client-build-stats.json + npm run client:build-stats > client-build-stats.json - name: PeerTube client lighthouse report if: github.event_name != 'pull_request' diff --git a/.gitignore b/.gitignore index e0004004d..55707fb80 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,8 @@ # NPM instalation -/node_modules/ -/server/tools/node_modules +node_modules *npm-debug.log yarn-error.log +.yarn # Testing /test1/ @@ -11,8 +11,8 @@ yarn-error.log /test4/ /test5/ /test6/ -/server/tests/fixtures/video_high_bitrate_1080p.mp4 -/server/tests/fixtures/video_59fps.mp4 +/packages/tests/fixtures/video_high_bitrate_1080p.mp4 +/packages/tests/fixtures/video_59fps.mp4 # Production /storage @@ -49,12 +49,14 @@ yarn-error.log /*.tar.xz /*.asc *.DS_Store -/server/tools/import-mediacore.ts /docker-volume/ /init.mp4 # TypeScript *.tsbuildinfo -# Packages -/packages/types/dist/ +# EsLint +.eslintcache + +# Compiled output +dist diff --git a/.mocharc.cjs b/.mocharc.cjs new file mode 100644 index 000000000..88566cfcb --- /dev/null +++ b/.mocharc.cjs @@ -0,0 +1,10 @@ +process.env.ESBK_TSCONFIG_PATH = './packages/tests/tsconfig.json' + +module.exports = { + "node-option": [ + "loader=tsx", + "no-warnings", + "conditions=peertube:tsx" + ], + "timeout": 30000 +} diff --git a/apps/peertube-cli/.npmignore b/apps/peertube-cli/.npmignore new file mode 100644 index 000000000..af17b9f32 --- /dev/null +++ b/apps/peertube-cli/.npmignore @@ -0,0 +1,4 @@ +src +meta.json +tsconfig.json +scripts diff --git a/apps/peertube-cli/README.md b/apps/peertube-cli/README.md new file mode 100644 index 000000000..b5b379090 --- /dev/null +++ b/apps/peertube-cli/README.md @@ -0,0 +1,43 @@ +# PeerTube CLI + +## Usage + +See https://docs.joinpeertube.org/maintain/tools#remote-tools + +## Dev + +## Install dependencies + +```bash +cd peertube-root +yarn install --pure-lockfile +cd apps/peertube-cli && yarn install --pure-lockfile +``` + +## Develop + +```bash +cd peertube-root +npm run dev:peertube-cli +``` + +## Build + +```bash +cd peertube-root +npm run build:peertube-cli +``` + +## Run + +```bash +cd peertube-root +node apps/peertube-cli/dist/peertube-cli.js --help +``` + +## Publish on NPM + +```bash +cd peertube-root +(cd apps/peertube-cli && npm version patch) && npm run build:peertube-cli && (cd apps/peertube-cli && npm publish --access=public) +``` diff --git a/apps/peertube-cli/package.json b/apps/peertube-cli/package.json new file mode 100644 index 000000000..a78319be2 --- /dev/null +++ b/apps/peertube-cli/package.json @@ -0,0 +1,19 @@ +{ + "name": "@peertube/peertube-cli", + "version": "1.0.1", + "type": "module", + "main": "dist/peertube.js", + "bin": "dist/peertube.js", + "engines": { + "node": ">=16.x" + }, + "scripts": {}, + "license": "AGPL-3.0", + "private": false, + "devDependencies": { + "application-config": "^2.0.0", + "cli-table3": "^0.6.0", + "netrc-parser": "^3.1.6" + }, + "dependencies": {} +} diff --git a/apps/peertube-cli/scripts/build.js b/apps/peertube-cli/scripts/build.js new file mode 100644 index 000000000..a9139acfa --- /dev/null +++ b/apps/peertube-cli/scripts/build.js @@ -0,0 +1,27 @@ +import * as esbuild from 'esbuild' +import { readFileSync } from 'fs' + +const packageJSON = JSON.parse(readFileSync(new URL('../package.json', import.meta.url))) + +export const esbuildOptions = { + entryPoints: [ './src/peertube.ts' ], + bundle: true, + platform: 'node', + format: 'esm', + target: 'node16', + external: [ + './lib-cov/fluent-ffmpeg', + 'pg-hstore' + ], + outfile: './dist/peertube.js', + banner: { + js: `const require = (await import("node:module")).createRequire(import.meta.url);` + + `const __filename = (await import("node:url")).fileURLToPath(import.meta.url);` + + `const __dirname = (await import("node:path")).dirname(__filename);` + }, + define: { + 'process.env.PACKAGE_VERSION': `'${packageJSON.version}'` + } +} + +await esbuild.build(esbuildOptions) diff --git a/apps/peertube-cli/scripts/watch.js b/apps/peertube-cli/scripts/watch.js new file mode 100644 index 000000000..94e57199c --- /dev/null +++ b/apps/peertube-cli/scripts/watch.js @@ -0,0 +1,7 @@ +import * as esbuild from 'esbuild' +import { esbuildOptions } from './build.js' + +const context = await esbuild.context(esbuildOptions) + +// Enable watch mode +await context.watch() diff --git a/apps/peertube-cli/src/peertube-auth.ts b/apps/peertube-cli/src/peertube-auth.ts new file mode 100644 index 000000000..1d30207c7 --- /dev/null +++ b/apps/peertube-cli/src/peertube-auth.ts @@ -0,0 +1,171 @@ +import CliTable3 from 'cli-table3' +import prompt from 'prompt' +import { Command } from '@commander-js/extra-typings' +import { assignToken, buildServer, getNetrc, getSettings, writeSettings } from './shared/index.js' + +export function defineAuthProgram () { + const program = new Command() + .name('auth') + .description('Register your accounts on remote instances to use them with other commands') + + program + .command('add') + .description('remember your accounts on remote instances for easier use') + .option('-u, --url ', 'Server url') + .option('-U, --username ', 'Username') + .option('-p, --password ', 'Password') + .option('--default', 'add the entry as the new default') + .action(options => { + /* eslint-disable no-import-assign */ + prompt.override = options + prompt.start() + prompt.get({ + properties: { + url: { + description: 'instance url', + conform: value => isURLaPeerTubeInstance(value), + message: 'It should be an URL (https://peertube.example.com)', + required: true + }, + username: { + conform: value => typeof value === 'string' && value.length !== 0, + message: 'Name must be only letters, spaces, or dashes', + required: true + }, + password: { + hidden: true, + replace: '*', + required: true + } + } + }, async (_, result) => { + + // Check credentials + try { + // Strip out everything after the domain:port. + // See https://github.com/Chocobozzz/PeerTube/issues/3520 + result.url = stripExtraneousFromPeerTubeUrl(result.url) + + const server = buildServer(result.url) + await assignToken(server, result.username, result.password) + } catch (err) { + console.error(err.message) + process.exit(-1) + } + + await setInstance(result.url, result.username, result.password, options.default) + + process.exit(0) + }) + }) + + program + .command('del ') + .description('Unregisters a remote instance') + .action(async url => { + await delInstance(url) + + process.exit(0) + }) + + program + .command('list') + .description('List registered remote instances') + .action(async () => { + const [ settings, netrc ] = await Promise.all([ getSettings(), getNetrc() ]) + + const table = new CliTable3({ + head: [ 'instance', 'login' ], + colWidths: [ 30, 30 ] + }) as any + + settings.remotes.forEach(element => { + if (!netrc.machines[element]) return + + table.push([ + element, + netrc.machines[element].login + ]) + }) + + console.log(table.toString()) + + process.exit(0) + }) + + program + .command('set-default ') + .description('Set an existing entry as default') + .action(async url => { + const settings = await getSettings() + const instanceExists = settings.remotes.includes(url) + + if (instanceExists) { + settings.default = settings.remotes.indexOf(url) + await writeSettings(settings) + + process.exit(0) + } else { + console.log(' is not a registered instance.') + process.exit(-1) + } + }) + + program.addHelpText('after', '\n\n Examples:\n\n' + + ' $ peertube auth add -u https://peertube.cpy.re -U "PEERTUBE_USER" --password "PEERTUBE_PASSWORD"\n' + + ' $ peertube auth add -u https://peertube.cpy.re -U root\n' + + ' $ peertube auth list\n' + + ' $ peertube auth del https://peertube.cpy.re\n' + ) + + return program +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +async function delInstance (url: string) { + const [ settings, netrc ] = await Promise.all([ getSettings(), getNetrc() ]) + + const index = settings.remotes.indexOf(url) + settings.remotes.splice(index) + + if (settings.default === index) settings.default = -1 + + await writeSettings(settings) + + delete netrc.machines[url] + + await netrc.save() +} + +async function setInstance (url: string, username: string, password: string, isDefault: boolean) { + const [ settings, netrc ] = await Promise.all([ getSettings(), getNetrc() ]) + + if (settings.remotes.includes(url) === false) { + settings.remotes.push(url) + } + + if (isDefault || settings.remotes.length === 1) { + settings.default = settings.remotes.length - 1 + } + + await writeSettings(settings) + + netrc.machines[url] = { login: username, password } + await netrc.save() +} + +function isURLaPeerTubeInstance (url: string) { + return url.startsWith('http://') || url.startsWith('https://') +} + +function stripExtraneousFromPeerTubeUrl (url: string) { + // Get everything before the 3rd /. + const urlLength = url.includes('/', 8) + ? url.indexOf('/', 8) + : url.length + + return url.substring(0, urlLength) +} diff --git a/apps/peertube-cli/src/peertube-get-access-token.ts b/apps/peertube-cli/src/peertube-get-access-token.ts new file mode 100644 index 000000000..3e0013182 --- /dev/null +++ b/apps/peertube-cli/src/peertube-get-access-token.ts @@ -0,0 +1,39 @@ +import { Command } from '@commander-js/extra-typings' +import { assignToken, buildServer } from './shared/index.js' + +export function defineGetAccessProgram () { + const program = new Command() + .name('get-access-token') + .description('Get a peertube access token') + .alias('token') + + program + .option('-u, --url ', 'Server url') + .option('-n, --username ', 'Username') + .option('-p, --password ', 'Password') + .action(async options => { + try { + if ( + !options.url || + !options.username || + !options.password + ) { + if (!options.url) console.error('--url field is required.') + if (!options.username) console.error('--username field is required.') + if (!options.password) console.error('--password field is required.') + + process.exit(-1) + } + + const server = buildServer(options.url) + await assignToken(server, options.username, options.password) + + console.log(server.accessToken) + } catch (err) { + console.error('Cannot get access token: ' + err.message) + process.exit(-1) + } + }) + + return program +} diff --git a/apps/peertube-cli/src/peertube-plugins.ts b/apps/peertube-cli/src/peertube-plugins.ts new file mode 100644 index 000000000..c9da56266 --- /dev/null +++ b/apps/peertube-cli/src/peertube-plugins.ts @@ -0,0 +1,167 @@ +import CliTable3 from 'cli-table3' +import { isAbsolute } from 'path' +import { Command } from '@commander-js/extra-typings' +import { PluginType, PluginType_Type } from '@peertube/peertube-models' +import { assignToken, buildServer, CommonProgramOptions, getServerCredentials } from './shared/index.js' + +export function definePluginsProgram () { + const program = new Command() + + program + .name('plugins') + .description('Manage instance plugins/themes') + .alias('p') + + program + .command('list') + .description('List installed plugins') + .option('-u, --url ', 'Server url') + .option('-U, --username ', 'Username') + .option('-p, --password ', 'Password') + .option('-t, --only-themes', 'List themes only') + .option('-P, --only-plugins', 'List plugins only') + .action(async options => { + try { + await pluginsListCLI(options) + } catch (err) { + console.error('Cannot list plugins: ' + err.message) + process.exit(-1) + } + }) + + program + .command('install') + .description('Install a plugin or a theme') + .option('-u, --url ', 'Server url') + .option('-U, --username ', 'Username') + .option('-p, --password ', 'Password') + .option('-P --path ', 'Install from a path') + .option('-n, --npm-name ', 'Install from npm') + .option('--plugin-version ', 'Specify the plugin version to install (only available when installing from npm)') + .action(async options => { + try { + await installPluginCLI(options) + } catch (err) { + console.error('Cannot install plugin: ' + err.message) + process.exit(-1) + } + }) + + program + .command('update') + .description('Update a plugin or a theme') + .option('-u, --url ', 'Server url') + .option('-U, --username ', 'Username') + .option('-p, --password ', 'Password') + .option('-P --path ', 'Update from a path') + .option('-n, --npm-name ', 'Update from npm') + .action(async options => { + try { + await updatePluginCLI(options) + } catch (err) { + console.error('Cannot update plugin: ' + err.message) + process.exit(-1) + } + }) + + program + .command('uninstall') + .description('Uninstall a plugin or a theme') + .option('-u, --url ', 'Server url') + .option('-U, --username ', 'Username') + .option('-p, --password ', 'Password') + .option('-n, --npm-name ', 'NPM plugin/theme name') + .action(async options => { + try { + await uninstallPluginCLI(options) + } catch (err) { + console.error('Cannot uninstall plugin: ' + err.message) + process.exit(-1) + } + }) + + return program +} + +// ---------------------------------------------------------------------------- + +async function pluginsListCLI (options: CommonProgramOptions & { onlyThemes?: true, onlyPlugins?: true }) { + const { url, username, password } = await getServerCredentials(options) + const server = buildServer(url) + await assignToken(server, username, password) + + let pluginType: PluginType_Type + if (options.onlyThemes) pluginType = PluginType.THEME + if (options.onlyPlugins) pluginType = PluginType.PLUGIN + + const { data } = await server.plugins.list({ start: 0, count: 100, sort: 'name', pluginType }) + + const table = new CliTable3({ + head: [ 'name', 'version', 'homepage' ], + colWidths: [ 50, 20, 50 ] + }) as any + + for (const plugin of data) { + const npmName = plugin.type === PluginType.PLUGIN + ? 'peertube-plugin-' + plugin.name + : 'peertube-theme-' + plugin.name + + table.push([ + npmName, + plugin.version, + plugin.homepage + ]) + } + + console.log(table.toString()) +} + +async function installPluginCLI (options: CommonProgramOptions & { path?: string, npmName?: string, pluginVersion?: string }) { + if (!options.path && !options.npmName) { + throw new Error('You need to specify the npm name or the path of the plugin you want to install.') + } + + if (options.path && !isAbsolute(options.path)) { + throw new Error('Path should be absolute.') + } + + const { url, username, password } = await getServerCredentials(options) + const server = buildServer(url) + await assignToken(server, username, password) + + await server.plugins.install({ npmName: options.npmName, path: options.path, pluginVersion: options.pluginVersion }) + + console.log('Plugin installed.') +} + +async function updatePluginCLI (options: CommonProgramOptions & { path?: string, npmName?: string }) { + if (!options.path && !options.npmName) { + throw new Error('You need to specify the npm name or the path of the plugin you want to update.') + } + + if (options.path && !isAbsolute(options.path)) { + throw new Error('Path should be absolute.') + } + + const { url, username, password } = await getServerCredentials(options) + const server = buildServer(url) + await assignToken(server, username, password) + + await server.plugins.update({ npmName: options.npmName, path: options.path }) + + console.log('Plugin updated.') +} + +async function uninstallPluginCLI (options: CommonProgramOptions & { npmName?: string }) { + if (!options.npmName) { + throw new Error('You need to specify the npm name of the plugin/theme you want to uninstall.') + } + + const { url, username, password } = await getServerCredentials(options) + const server = buildServer(url) + await assignToken(server, username, password) + + await server.plugins.uninstall({ npmName: options.npmName }) + + console.log('Plugin uninstalled.') +} diff --git a/apps/peertube-cli/src/peertube-redundancy.ts b/apps/peertube-cli/src/peertube-redundancy.ts new file mode 100644 index 000000000..56fc6366b --- /dev/null +++ b/apps/peertube-cli/src/peertube-redundancy.ts @@ -0,0 +1,186 @@ +import bytes from 'bytes' +import CliTable3 from 'cli-table3' +import { URL } from 'url' +import { Command } from '@commander-js/extra-typings' +import { forceNumber, uniqify } from '@peertube/peertube-core-utils' +import { HttpStatusCode, VideoRedundanciesTarget } from '@peertube/peertube-models' +import { assignToken, buildServer, CommonProgramOptions, getServerCredentials } from './shared/index.js' + +export function defineRedundancyProgram () { + const program = new Command() + .name('redundancy') + .description('Manage instance redundancies') + .alias('r') + + program + .command('list-remote-redundancies') + .description('List remote redundancies on your videos') + .option('-u, --url ', 'Server url') + .option('-U, --username ', 'Username') + .option('-p, --password ', 'Password') + .action(async options => { + try { + await listRedundanciesCLI({ target: 'my-videos', ...options }) + } catch (err) { + console.error('Cannot list remote redundancies: ' + err.message) + process.exit(-1) + } + }) + + program + .command('list-my-redundancies') + .description('List your redundancies of remote videos') + .option('-u, --url ', 'Server url') + .option('-U, --username ', 'Username') + .option('-p, --password ', 'Password') + .action(async options => { + try { + await listRedundanciesCLI({ target: 'remote-videos', ...options }) + } catch (err) { + console.error('Cannot list redundancies: ' + err.message) + process.exit(-1) + } + }) + + program + .command('add') + .description('Duplicate a video in your redundancy system') + .option('-u, --url ', 'Server url') + .option('-U, --username ', 'Username') + .option('-p, --password ', 'Password') + .requiredOption('-v, --video ', 'Video id to duplicate', parseInt) + .action(async options => { + try { + await addRedundancyCLI(options) + } catch (err) { + console.error('Cannot duplicate video: ' + err.message) + process.exit(-1) + } + }) + + program + .command('remove') + .description('Remove a video from your redundancies') + .option('-u, --url ', 'Server url') + .option('-U, --username ', 'Username') + .option('-p, --password ', 'Password') + .requiredOption('-v, --video ', 'Video id to remove from redundancies', parseInt) + .action(async options => { + try { + await removeRedundancyCLI(options) + } catch (err) { + console.error('Cannot remove redundancy: ' + err) + process.exit(-1) + } + }) + + return program +} + +// ---------------------------------------------------------------------------- + +async function listRedundanciesCLI (options: CommonProgramOptions & { target: VideoRedundanciesTarget }) { + const { target } = options + + const { url, username, password } = await getServerCredentials(options) + const server = buildServer(url) + await assignToken(server, username, password) + + const { data } = await server.redundancy.listVideos({ start: 0, count: 100, sort: 'name', target }) + + const table = new CliTable3({ + head: [ 'video id', 'video name', 'video url', 'files', 'playlists', 'by instances', 'total size' ] + }) as any + + for (const redundancy of data) { + const webVideoFiles = redundancy.redundancies.files + const streamingPlaylists = redundancy.redundancies.streamingPlaylists + + let totalSize = '' + if (target === 'remote-videos') { + const tmp = webVideoFiles.concat(streamingPlaylists) + .reduce((a, b) => a + b.size, 0) + + // FIXME: don't use external dependency to stringify bytes: we already have the functions in the client + totalSize = bytes(tmp) + } + + const instances = uniqify( + webVideoFiles.concat(streamingPlaylists) + .map(r => r.fileUrl) + .map(u => new URL(u).host) + ) + + table.push([ + redundancy.id.toString(), + redundancy.name, + redundancy.url, + webVideoFiles.length, + streamingPlaylists.length, + instances.join('\n'), + totalSize + ]) + } + + console.log(table.toString()) +} + +async function addRedundancyCLI (options: { video: number } & CommonProgramOptions) { + const { url, username, password } = await getServerCredentials(options) + const server = buildServer(url) + await assignToken(server, username, password) + + if (!options.video || isNaN(options.video)) { + throw new Error('You need to specify the video id to duplicate and it should be a number.') + } + + try { + await server.redundancy.addVideo({ videoId: options.video }) + + console.log('Video will be duplicated by your instance!') + } catch (err) { + if (err.message.includes(HttpStatusCode.CONFLICT_409)) { + throw new Error('This video is already duplicated by your instance.') + } + + if (err.message.includes(HttpStatusCode.NOT_FOUND_404)) { + throw new Error('This video id does not exist.') + } + + throw err + } +} + +async function removeRedundancyCLI (options: CommonProgramOptions & { video: number }) { + const { url, username, password } = await getServerCredentials(options) + const server = buildServer(url) + await assignToken(server, username, password) + + if (!options.video || isNaN(options.video)) { + throw new Error('You need to specify the video id to remove from your redundancies') + } + + const videoId = forceNumber(options.video) + + const myVideoRedundancies = await server.redundancy.listVideos({ target: 'my-videos' }) + let videoRedundancy = myVideoRedundancies.data.find(r => videoId === r.id) + + if (!videoRedundancy) { + const remoteVideoRedundancies = await server.redundancy.listVideos({ target: 'remote-videos' }) + videoRedundancy = remoteVideoRedundancies.data.find(r => videoId === r.id) + } + + if (!videoRedundancy) { + throw new Error('Video redundancy not found.') + } + + const ids = videoRedundancy.redundancies.files + .concat(videoRedundancy.redundancies.streamingPlaylists) + .map(r => r.id) + + for (const id of ids) { + await server.redundancy.removeVideo({ redundancyId: id }) + } + + console.log('Video redundancy removed!') +} diff --git a/apps/peertube-cli/src/peertube-upload.ts b/apps/peertube-cli/src/peertube-upload.ts new file mode 100644 index 000000000..443f8ce1f --- /dev/null +++ b/apps/peertube-cli/src/peertube-upload.ts @@ -0,0 +1,167 @@ +import { access, constants } from 'fs/promises' +import { isAbsolute } from 'path' +import { inspect } from 'util' +import { Command } from '@commander-js/extra-typings' +import { VideoPrivacy } from '@peertube/peertube-models' +import { PeerTubeServer } from '@peertube/peertube-server-commands' +import { assignToken, buildServer, getServerCredentials, listOptions } from './shared/index.js' + +type UploadOptions = { + url?: string + username?: string + password?: string + thumbnail?: string + preview?: string + file?: string + videoName?: string + category?: string + licence?: string + language?: string + tags?: string + nsfw?: true + videoDescription?: string + privacy?: number + channelName?: string + noCommentsEnabled?: true + support?: string + noWaitTranscoding?: true + noDownloadEnabled?: true +} + +export function defineUploadProgram () { + const program = new Command('upload') + .description('Upload a video on a PeerTube instance') + .alias('up') + + program + .option('-u, --url ', 'Server url') + .option('-U, --username ', 'Username') + .option('-p, --password ', 'Password') + .option('-b, --thumbnail ', 'Thumbnail path') + .option('-v, --preview ', 'Preview path') + .option('-f, --file ', 'Video absolute file path') + .option('-n, --video-name ', 'Video name') + .option('-c, --category ', 'Category number') + .option('-l, --licence ', 'Licence number') + .option('-L, --language ', 'Language ISO 639 code (fr or en...)') + .option('-t, --tags ', 'Video tags', listOptions) + .option('-N, --nsfw', 'Video is Not Safe For Work') + .option('-d, --video-description ', 'Video description') + .option('-P, --privacy ', 'Privacy', parseInt) + .option('-C, --channel-name ', 'Channel name') + .option('--no-comments-enabled', 'Disable video comments') + .option('-s, --support ', 'Video support text') + .option('--no-wait-transcoding', 'Do not wait transcoding before publishing the video') + .option('--no-download-enabled', 'Disable video download') + .option('-v, --verbose ', 'Verbosity, from 0/\'error\' to 4/\'debug\'', 'info') + .action(async options => { + try { + const { url, username, password } = await getServerCredentials(options) + + if (!options.videoName || !options.file) { + if (!options.videoName) console.error('--video-name is required.') + if (!options.file) console.error('--file is required.') + + process.exit(-1) + } + + if (isAbsolute(options.file) === false) { + console.error('File path should be absolute.') + process.exit(-1) + } + + await run({ ...options, url, username, password }) + } catch (err) { + console.error('Cannot upload video: ' + err.message) + process.exit(-1) + } + }) + + return program +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +async function run (options: UploadOptions) { + const { url, username, password } = options + + const server = buildServer(url) + await assignToken(server, username, password) + + await access(options.file, constants.F_OK) + + console.log('Uploading %s video...', options.videoName) + + const baseAttributes = await buildVideoAttributesFromCommander(server, options) + + const attributes = { + ...baseAttributes, + + fixture: options.file, + thumbnailfile: options.thumbnail, + previewfile: options.preview + } + + try { + await server.videos.upload({ attributes }) + console.log(`Video ${options.videoName} uploaded.`) + process.exit(0) + } catch (err) { + const message = err.message || '' + if (message.includes('413')) { + console.error('Aborted: user quota is exceeded or video file is too big for this PeerTube instance.') + } else { + console.error(inspect(err)) + } + + process.exit(-1) + } +} + +async function buildVideoAttributesFromCommander (server: PeerTubeServer, options: UploadOptions, defaultAttributes: any = {}) { + const defaultBooleanAttributes = { + nsfw: false, + commentsEnabled: true, + downloadEnabled: true, + waitTranscoding: true + } + + const booleanAttributes: { [id in keyof typeof defaultBooleanAttributes]: boolean } | {} = {} + + for (const key of Object.keys(defaultBooleanAttributes)) { + if (options[key] !== undefined) { + booleanAttributes[key] = options[key] + } else if (defaultAttributes[key] !== undefined) { + booleanAttributes[key] = defaultAttributes[key] + } else { + booleanAttributes[key] = defaultBooleanAttributes[key] + } + } + + const videoAttributes = { + name: options.videoName || defaultAttributes.name, + category: options.category || defaultAttributes.category || undefined, + licence: options.licence || defaultAttributes.licence || undefined, + language: options.language || defaultAttributes.language || undefined, + privacy: options.privacy || defaultAttributes.privacy || VideoPrivacy.PUBLIC, + support: options.support || defaultAttributes.support || undefined, + description: options.videoDescription || defaultAttributes.description || undefined, + tags: options.tags || defaultAttributes.tags || undefined + } + + Object.assign(videoAttributes, booleanAttributes) + + if (options.channelName) { + const videoChannel = await server.channels.get({ channelName: options.channelName }) + + Object.assign(videoAttributes, { channelId: videoChannel.id }) + + if (!videoAttributes.support && videoChannel.support) { + Object.assign(videoAttributes, { support: videoChannel.support }) + } + } + + return videoAttributes +} diff --git a/apps/peertube-cli/src/peertube.ts b/apps/peertube-cli/src/peertube.ts new file mode 100644 index 000000000..e3565bb1a --- /dev/null +++ b/apps/peertube-cli/src/peertube.ts @@ -0,0 +1,64 @@ +#!/usr/bin/env node + +import { Command } from '@commander-js/extra-typings' +import { defineAuthProgram } from './peertube-auth.js' +import { defineGetAccessProgram } from './peertube-get-access-token.js' +import { definePluginsProgram } from './peertube-plugins.js' +import { defineRedundancyProgram } from './peertube-redundancy.js' +import { defineUploadProgram } from './peertube-upload.js' +import { getSettings, version } from './shared/index.js' + +const program = new Command() + +program + .version(version, '-v, --version') + .usage('[command] [options]') + +program.addCommand(defineAuthProgram()) +program.addCommand(defineUploadProgram()) +program.addCommand(defineRedundancyProgram()) +program.addCommand(definePluginsProgram()) +program.addCommand(defineGetAccessProgram()) + +// help on no command +if (!process.argv.slice(2).length) { + const logo = '░P░e░e░r░T░u░b░e░' + console.log(` + ___/),.._ ` + logo + ` +/' ,. ."'._ +( "' '-.__"-._ ,- +\\'='='), "\\ -._-"-. -"/ + / ""/"\\,_\\,__"" _" /,- + / / -" _/"/ + / | ._\\\\ |\\ |_.".-" / + / | __\\)|)|),/|_." _,." + / \\_." " ") | ).-""---''-- + ( "/.""7__-""'' + | " ."._--._ + \\ \\ (_ __ "" ".,_ + \\.,. \\ "" -"".-" + ".,_, (",_-,,,-".- + "'-,\\_ __,-" + ",)" ") + /"\\-" + ,"\\/ + _,.__/"\\/_ (the CLI for red chocobos) + / \\) "./, ". + --/---"---" "-) )---- by Chocobozzz et al.\n`) +} + +getSettings() + .then(settings => { + const state = (settings.default === undefined || settings.default === -1) + ? 'no instance selected, commands will require explicit arguments' + : 'instance ' + settings.remotes[settings.default] + ' selected' + + program + .addHelpText('after', '\n\n State: ' + state + '\n\n' + + ' Examples:\n\n' + + ' $ peertube auth add -u "PEERTUBE_URL" -U "PEERTUBE_USER" --password "PEERTUBE_PASSWORD"\n' + + ' $ peertube up \n' + ) + .parse(process.argv) + }) + .catch(err => console.error(err)) diff --git a/apps/peertube-cli/src/shared/cli.ts b/apps/peertube-cli/src/shared/cli.ts new file mode 100644 index 000000000..080eb8237 --- /dev/null +++ b/apps/peertube-cli/src/shared/cli.ts @@ -0,0 +1,195 @@ +import applicationConfig from 'application-config' +import { Netrc } from 'netrc-parser' +import { join } from 'path' +import { createLogger, format, transports } from 'winston' +import { UserRole } from '@peertube/peertube-models' +import { getAppNumber, isTestInstance, root } from '@peertube/peertube-node-utils' +import { PeerTubeServer } from '@peertube/peertube-server-commands' + +export type CommonProgramOptions = { + url?: string + username?: string + password?: string +} + +let configName = 'PeerTube/CLI' +if (isTestInstance()) configName += `-${getAppNumber()}` + +const config = applicationConfig(configName) + +const version: string = process.env.PACKAGE_VERSION + +async function getAdminTokenOrDie (server: PeerTubeServer, username: string, password: string) { + const token = await server.login.getAccessToken(username, password) + const me = await server.users.getMyInfo({ token }) + + if (me.role.id !== UserRole.ADMINISTRATOR) { + console.error('You must be an administrator.') + process.exit(-1) + } + + return token +} + +interface Settings { + remotes: any[] + default: number +} + +async function getSettings () { + const defaultSettings: Settings = { + remotes: [], + default: -1 + } + + const data = await config.read() as Promise + + return Object.keys(data).length === 0 + ? defaultSettings + : data +} + +async function getNetrc () { + const netrc = isTestInstance() + ? new Netrc(join(root(import.meta.url), 'test' + getAppNumber(), 'netrc')) + : new Netrc() + + await netrc.load() + + return netrc +} + +function writeSettings (settings: Settings) { + return config.write(settings) +} + +function deleteSettings () { + return config.trash() +} + +function getRemoteObjectOrDie ( + options: CommonProgramOptions, + settings: Settings, + netrc: Netrc +): { url: string, username: string, password: string } { + + function exitIfNoOptions (optionNames: string[], errorPrefix: string = '') { + let exit = false + + for (const key of optionNames) { + if (!options[key]) { + if (exit === false && errorPrefix) console.error(errorPrefix) + + console.error(`--${key} field is required`) + exit = true + } + } + + if (exit) process.exit(-1) + } + + // If username or password are specified, both are mandatory + if (options.username || options.password) { + exitIfNoOptions([ 'username', 'password' ]) + } + + // If no available machines, url, username and password args are mandatory + if (Object.keys(netrc.machines).length === 0) { + exitIfNoOptions([ 'url', 'username', 'password' ], 'No account found in netrc') + } + + if (settings.remotes.length === 0 || settings.default === -1) { + exitIfNoOptions([ 'url' ], 'No default instance found') + } + + let url: string = options.url + let username: string = options.username + let password: string = options.password + + if (!url && settings.default !== -1) url = settings.remotes[settings.default] + + const machine = netrc.machines[url] + if ((!username || !password) && !machine) { + console.error('Cannot find existing configuration for %s.', url) + process.exit(-1) + } + + if (!username && machine) username = machine.login + if (!password && machine) password = machine.password + + return { url, username, password } +} + +function listOptions (val: any) { + return val.split(',') +} + +function getServerCredentials (options: CommonProgramOptions) { + return Promise.all([ getSettings(), getNetrc() ]) + .then(([ settings, netrc ]) => { + return getRemoteObjectOrDie(options, settings, netrc) + }) +} + +function buildServer (url: string) { + return new PeerTubeServer({ url }) +} + +async function assignToken (server: PeerTubeServer, username: string, password: string) { + const bodyClient = await server.login.getClient() + const client = { id: bodyClient.client_id, secret: bodyClient.client_secret } + + const body = await server.login.login({ client, user: { username, password } }) + + server.accessToken = body.access_token +} + +function getLogger (logLevel = 'info') { + const logLevels = { + 0: 0, + error: 0, + 1: 1, + warn: 1, + 2: 2, + info: 2, + 3: 3, + verbose: 3, + 4: 4, + debug: 4 + } + + const logger = createLogger({ + levels: logLevels, + format: format.combine( + format.splat(), + format.simple() + ), + transports: [ + new (transports.Console)({ + level: logLevel + }) + ] + }) + + return logger +} + +// --------------------------------------------------------------------------- + +export { + version, + getLogger, + getSettings, + getNetrc, + getRemoteObjectOrDie, + writeSettings, + deleteSettings, + + getServerCredentials, + + listOptions, + + getAdminTokenOrDie, + buildServer, + assignToken +} diff --git a/apps/peertube-cli/src/shared/index.ts b/apps/peertube-cli/src/shared/index.ts new file mode 100644 index 000000000..a1fc9470b --- /dev/null +++ b/apps/peertube-cli/src/shared/index.ts @@ -0,0 +1 @@ +export * from './cli.js' diff --git a/apps/peertube-cli/tsconfig.json b/apps/peertube-cli/tsconfig.json new file mode 100644 index 000000000..636bdb95a --- /dev/null +++ b/apps/peertube-cli/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "baseUrl": "./", + "outDir": "./dist", + "rootDir": "src", + "tsBuildInfoFile": "./dist/.tsbuildinfo" + }, + "references": [ + { "path": "../../packages/core-utils" }, + { "path": "../../packages/models" }, + { "path": "../../packages/node-utils" }, + { "path": "../../packages/server-commands" } + ] +} diff --git a/apps/peertube-cli/yarn.lock b/apps/peertube-cli/yarn.lock new file mode 100644 index 000000000..76b36ee73 --- /dev/null +++ b/apps/peertube-cli/yarn.lock @@ -0,0 +1,374 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@babel/code-frame@^7.0.0": + version "7.22.10" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.22.10.tgz#1c20e612b768fefa75f6e90d6ecb86329247f0a3" + integrity sha512-/KKIMG4UEL35WmI9OlvMhurwtytjvXoFcGNrOvyG9zIzA8YmPjVtIZUf7b05+TPO7G7/GEmLHDaoCgACHl9hhA== + dependencies: + "@babel/highlight" "^7.22.10" + chalk "^2.4.2" + +"@babel/helper-validator-identifier@^7.22.5": + version "7.22.5" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.5.tgz#9544ef6a33999343c8740fa51350f30eeaaaf193" + integrity sha512-aJXu+6lErq8ltp+JhkJUfk1MTGyuA4v7f3pA+BJ5HLfNC6nAQ0Cpi9uOquUj8Hehg0aUiHzWQbOVJGao6ztBAQ== + +"@babel/highlight@^7.22.10": + version "7.22.10" + resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.22.10.tgz#02a3f6d8c1cb4521b2fd0ab0da8f4739936137d7" + integrity sha512-78aUtVcT7MUscr0K5mIEnkwxPE0MaxkR5RxRwuHaQ+JuU5AmTPhY+do2mdzVTnIJJpyBglql2pehuBIWHug+WQ== + dependencies: + "@babel/helper-validator-identifier" "^7.22.5" + chalk "^2.4.2" + js-tokens "^4.0.0" + +"@colors/colors@1.5.0": + version "1.5.0" + resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9" + integrity sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ== + +ansi-regex@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" + integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== + +ansi-styles@^3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" + integrity sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA== + dependencies: + color-convert "^1.9.0" + +application-config-path@^0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/application-config-path/-/application-config-path-0.1.1.tgz#8b5ac64ff6afdd9bd70ce69f6f64b6998f5f756e" + integrity sha512-zy9cHePtMP0YhwG+CfHm0bgwdnga2X3gZexpdCwEj//dpb+TKajtiC8REEUJUSq6Ab4f9cgNy2l8ObXzCXFkEw== + +application-config@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/application-config/-/application-config-2.0.0.tgz#15b4d54d61c0c082f9802227e3e85de876b47747" + integrity sha512-NC5/0guSZK3/UgUDfCk/riByXzqz0owL1L3r63JPSBzYk5QALrp3bLxbsR7qeSfvYfFmAhnp3dbqYsW3U9MpZQ== + dependencies: + application-config-path "^0.1.0" + load-json-file "^6.2.0" + write-json-file "^4.2.0" + +chalk@^2.4.2: + version "2.4.2" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" + integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== + dependencies: + ansi-styles "^3.2.1" + escape-string-regexp "^1.0.5" + supports-color "^5.3.0" + +cli-table3@^0.6.0: + version "0.6.3" + resolved "https://registry.yarnpkg.com/cli-table3/-/cli-table3-0.6.3.tgz#61ab765aac156b52f222954ffc607a6f01dbeeb2" + integrity sha512-w5Jac5SykAeZJKntOxJCrm63Eg5/4dhMWIcuTbo9rpE+brgaSZo0RuNJZeOyMgsUdhDeojvgyQLmjI+K50ZGyg== + dependencies: + string-width "^4.2.0" + optionalDependencies: + "@colors/colors" "1.5.0" + +color-convert@^1.9.0: + version "1.9.3" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" + integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== + dependencies: + color-name "1.1.3" + +color-name@1.1.3: + version "1.1.3" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" + integrity sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw== + +cross-spawn@^6.0.0: + version "6.0.5" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4" + integrity sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ== + dependencies: + nice-try "^1.0.4" + path-key "^2.0.1" + semver "^5.5.0" + shebang-command "^1.2.0" + which "^1.2.9" + +debug@^3.1.0: + version "3.2.7" + resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" + integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== + dependencies: + ms "^2.1.1" + +detect-indent@^6.0.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/detect-indent/-/detect-indent-6.1.0.tgz#592485ebbbf6b3b1ab2be175c8393d04ca0d57e6" + integrity sha512-reYkTUJAZb9gUuZ2RvVCNhVHdg62RHnJ7WJl8ftMi4diZ6NWlciOzQN88pUhSELEwflJht4oQDv0F0BMlwaYtA== + +emoji-regex@^8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" + integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== + +error-ex@^1.3.1: + version "1.3.2" + resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.2.tgz#b4ac40648107fdcdcfae242f428bea8a14d4f1bf" + integrity sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g== + dependencies: + is-arrayish "^0.2.1" + +escape-string-regexp@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" + integrity sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg== + +execa@^0.10.0: + version "0.10.0" + resolved "https://registry.yarnpkg.com/execa/-/execa-0.10.0.tgz#ff456a8f53f90f8eccc71a96d11bdfc7f082cb50" + integrity sha512-7XOMnz8Ynx1gGo/3hyV9loYNPWM94jG3+3T3Y8tsfSstFmETmENCMU/A/zj8Lyaj1lkgEepKepvd6240tBRvlw== + dependencies: + cross-spawn "^6.0.0" + get-stream "^3.0.0" + is-stream "^1.1.0" + npm-run-path "^2.0.0" + p-finally "^1.0.0" + signal-exit "^3.0.0" + strip-eof "^1.0.0" + +get-stream@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-3.0.0.tgz#8e943d1358dc37555054ecbe2edb05aa174ede14" + integrity sha512-GlhdIUuVakc8SJ6kK0zAFbiGzRFzNnY4jUuEbV9UROo4Y+0Ny4fjvcZFVTeDA4odpFyOQzaw6hXukJSq/f28sQ== + +graceful-fs@^4.1.15: + version "4.2.11" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" + integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== + +has-flag@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" + integrity sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw== + +imurmurhash@^0.1.4: + version "0.1.4" + resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" + integrity sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA== + +is-arrayish@^0.2.1: + version "0.2.1" + resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" + integrity sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg== + +is-fullwidth-code-point@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d" + integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg== + +is-plain-obj@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-2.1.0.tgz#45e42e37fccf1f40da8e5f76ee21515840c09287" + integrity sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA== + +is-stream@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44" + integrity sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ== + +is-typedarray@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" + integrity sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA== + +isexe@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" + integrity sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw== + +js-tokens@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" + integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== + +json-parse-even-better-errors@^2.3.0: + version "2.3.1" + resolved "https://registry.yarnpkg.com/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz#7c47805a94319928e05777405dc12e1f7a4ee02d" + integrity sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w== + +lines-and-columns@^1.1.6: + version "1.2.4" + resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632" + integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg== + +load-json-file@^6.2.0: + version "6.2.0" + resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-6.2.0.tgz#5c7770b42cafa97074ca2848707c61662f4251a1" + integrity sha512-gUD/epcRms75Cw8RT1pUdHugZYM5ce64ucs2GEISABwkRsOQr0q2wm/MV2TKThycIe5e0ytRweW2RZxclogCdQ== + dependencies: + graceful-fs "^4.1.15" + parse-json "^5.0.0" + strip-bom "^4.0.0" + type-fest "^0.6.0" + +make-dir@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f" + integrity sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw== + dependencies: + semver "^6.0.0" + +ms@^2.1.1: + version "2.1.3" + resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" + integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== + +netrc-parser@^3.1.6: + version "3.1.6" + resolved "https://registry.yarnpkg.com/netrc-parser/-/netrc-parser-3.1.6.tgz#7243c9ec850b8e805b9bdc7eae7b1450d4a96e72" + integrity sha512-lY+fmkqSwntAAjfP63jB4z5p5WbuZwyMCD3pInT7dpHU/Gc6Vv90SAC6A0aNiqaRGHiuZFBtiwu+pu8W/Eyotw== + dependencies: + debug "^3.1.0" + execa "^0.10.0" + +nice-try@^1.0.4: + version "1.0.5" + resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" + integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== + +npm-run-path@^2.0.0: + version "2.0.2" + resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-2.0.2.tgz#35a9232dfa35d7067b4cb2ddf2357b1871536c5f" + integrity sha512-lJxZYlT4DW/bRUtFh1MQIWqmLwQfAxnqWG4HhEdjMlkrJYnJn0Jrr2u3mgxqaWsdiBc76TYkTG/mhrnYTuzfHw== + dependencies: + path-key "^2.0.0" + +p-finally@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae" + integrity sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow== + +parse-json@^5.0.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-5.2.0.tgz#c76fc66dee54231c962b22bcc8a72cf2f99753cd" + integrity sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg== + dependencies: + "@babel/code-frame" "^7.0.0" + error-ex "^1.3.1" + json-parse-even-better-errors "^2.3.0" + lines-and-columns "^1.1.6" + +path-key@^2.0.0, path-key@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40" + integrity sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw== + +semver@^5.5.0: + version "5.7.2" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.2.tgz#48d55db737c3287cd4835e17fa13feace1c41ef8" + integrity sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g== + +semver@^6.0.0: + version "6.3.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" + integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== + +shebang-command@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-1.2.0.tgz#44aac65b695b03398968c39f363fee5deafdf1ea" + integrity sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg== + dependencies: + shebang-regex "^1.0.0" + +shebang-regex@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-1.0.0.tgz#da42f49740c0b42db2ca9728571cb190c98efea3" + integrity sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ== + +signal-exit@^3.0.0, signal-exit@^3.0.2: + version "3.0.7" + resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" + integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== + +sort-keys@^4.0.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/sort-keys/-/sort-keys-4.2.0.tgz#6b7638cee42c506fff8c1cecde7376d21315be18" + integrity sha512-aUYIEU/UviqPgc8mHR6IW1EGxkAXpeRETYcrzg8cLAvUPZcpAlleSXHV2mY7G12GphSH6Gzv+4MMVSSkbdteHg== + dependencies: + is-plain-obj "^2.0.0" + +string-width@^4.2.0: + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + +strip-bom@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-4.0.0.tgz#9c3505c1db45bcedca3d9cf7a16f5c5aa3901878" + integrity sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w== + +strip-eof@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/strip-eof/-/strip-eof-1.0.0.tgz#bb43ff5598a6eb05d89b59fcd129c983313606bf" + integrity sha512-7FCwGGmx8mD5xQd3RPUvnSpUXHM3BWuzjtpD4TXsfcZ9EL4azvVVUscFYwD9nx8Kh+uCBC00XBtAykoMHwTh8Q== + +supports-color@^5.3.0: + version "5.5.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" + integrity sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow== + dependencies: + has-flag "^3.0.0" + +type-fest@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.6.0.tgz#8d2a2370d3df886eb5c90ada1c5bf6188acf838b" + integrity sha512-q+MB8nYR1KDLrgr4G5yemftpMC7/QLqVndBmEEdqzmNj5dcFOO4Oo8qlwZE3ULT3+Zim1F8Kq4cBnikNhlCMlg== + +typedarray-to-buffer@^3.1.5: + version "3.1.5" + resolved "https://registry.yarnpkg.com/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz#a97ee7a9ff42691b9f783ff1bc5112fe3fca9080" + integrity sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q== + dependencies: + is-typedarray "^1.0.0" + +which@^1.2.9: + version "1.3.1" + resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a" + integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ== + dependencies: + isexe "^2.0.0" + +write-file-atomic@^3.0.0: + version "3.0.3" + resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-3.0.3.tgz#56bd5c5a5c70481cd19c571bd39ab965a5de56e8" + integrity sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q== + dependencies: + imurmurhash "^0.1.4" + is-typedarray "^1.0.0" + signal-exit "^3.0.2" + typedarray-to-buffer "^3.1.5" + +write-json-file@^4.2.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/write-json-file/-/write-json-file-4.3.0.tgz#908493d6fd23225344af324016e4ca8f702dd12d" + integrity sha512-PxiShnxf0IlnQuMYOPPhPkhExoCQuTUNPOa/2JWCYTmBquU9njyyDuwRKN26IZBlp4yn1nt+Agh2HOOBl+55HQ== + dependencies: + detect-indent "^6.0.0" + graceful-fs "^4.1.15" + is-plain-obj "^2.0.0" + make-dir "^3.0.0" + sort-keys "^4.0.0" + write-file-atomic "^3.0.0" diff --git a/packages/peertube-runner/.gitignore b/apps/peertube-runner/.gitignore similarity index 100% rename from packages/peertube-runner/.gitignore rename to apps/peertube-runner/.gitignore diff --git a/apps/peertube-runner/.npmignore b/apps/peertube-runner/.npmignore new file mode 100644 index 000000000..af17b9f32 --- /dev/null +++ b/apps/peertube-runner/.npmignore @@ -0,0 +1,4 @@ +src +meta.json +tsconfig.json +scripts diff --git a/apps/peertube-runner/README.md b/apps/peertube-runner/README.md new file mode 100644 index 000000000..37760f867 --- /dev/null +++ b/apps/peertube-runner/README.md @@ -0,0 +1,43 @@ +# PeerTube runner + +Runner program to execute jobs (transcoding...) of remote PeerTube instances. + +Commands below has to be run at the root of PeerTube git repository. + +## Dev + +### Install dependencies + +```bash +cd peertube-root +yarn install --pure-lockfile +cd apps/peertube-runner && yarn install --pure-lockfile +``` + +### Develop + +```bash +cd peertube-root +npm run dev:peertube-runner +``` + +### Build + +```bash +cd peertube-root +npm run build:peertube-runner +``` + +### Run + +```bash +cd peertube-root +node apps/peertube-runner/dist/peertube-runner.js --help +``` + +### Publish on NPM + +```bash +cd peertube-root +(cd apps/peertube-runner && npm version patch) && npm run build:peertube-runner && (cd apps/peertube-runner && npm publish --access=public) +``` diff --git a/apps/peertube-runner/package.json b/apps/peertube-runner/package.json new file mode 100644 index 000000000..1dca15451 --- /dev/null +++ b/apps/peertube-runner/package.json @@ -0,0 +1,20 @@ +{ + "name": "@peertube/peertube-runner", + "version": "0.0.5", + "type": "module", + "main": "dist/peertube-runner.js", + "bin": "dist/peertube-runner.js", + "engines": { + "node": ">=16.x" + }, + "license": "AGPL-3.0", + "dependencies": {}, + "devDependencies": { + "@commander-js/extra-typings": "^10.0.3", + "@iarna/toml": "^2.2.5", + "env-paths": "^3.0.0", + "net-ipc": "^2.0.1", + "pino": "^8.11.0", + "pino-pretty": "^10.0.0" + } +} diff --git a/apps/peertube-runner/scripts/build.js b/apps/peertube-runner/scripts/build.js new file mode 100644 index 000000000..f54ca35f3 --- /dev/null +++ b/apps/peertube-runner/scripts/build.js @@ -0,0 +1,26 @@ +import * as esbuild from 'esbuild' + +const packageJSON = JSON.parse(readFileSync(new URL('../package.json', import.meta.url))) + +export const esbuildOptions = { + entryPoints: [ './src/peertube-runner.ts' ], + bundle: true, + platform: 'node', + format: 'esm', + target: 'node16', + external: [ + './lib-cov/fluent-ffmpeg', + 'pg-hstore' + ], + outfile: './dist/peertube-runner.js', + banner: { + js: `const require = (await import("node:module")).createRequire(import.meta.url);` + + `const __filename = (await import("node:url")).fileURLToPath(import.meta.url);` + + `const __dirname = (await import("node:path")).dirname(__filename);` + }, + define: { + 'process.env.PACKAGE_VERSION': `'${packageJSON.version}'` + } +} + +await esbuild.build(esbuildOptions) diff --git a/apps/peertube-runner/src/peertube-runner.ts b/apps/peertube-runner/src/peertube-runner.ts new file mode 100644 index 000000000..67ca0e0ac --- /dev/null +++ b/apps/peertube-runner/src/peertube-runner.ts @@ -0,0 +1,91 @@ +#!/usr/bin/env node + +import { Command, InvalidArgumentError } from '@commander-js/extra-typings' +import { listRegistered, registerRunner, unregisterRunner } from './register/index.js' +import { RunnerServer } from './server/index.js' +import { ConfigManager, logger } from './shared/index.js' + +const program = new Command() + .version(process.env.PACKAGE_VERSION) + .option( + '--id ', + 'Runner server id, so you can run multiple PeerTube server runners with different configurations on the same machine', + 'default' + ) + .option('--verbose', 'Run in verbose mode') + .hook('preAction', thisCommand => { + const options = thisCommand.opts() + + ConfigManager.Instance.init(options.id) + + if (options.verbose === true) { + logger.level = 'debug' + } + }) + +program.command('server') + .description('Run in server mode, to execute remote jobs of registered PeerTube instances') + .action(async () => { + try { + await RunnerServer.Instance.run() + } catch (err) { + logger.error(err, 'Cannot run PeerTube runner as server mode') + process.exit(-1) + } + }) + +program.command('register') + .description('Register a new PeerTube instance to process runner jobs') + .requiredOption('--url ', 'PeerTube instance URL', parseUrl) + .requiredOption('--registration-token ', 'Runner registration token (can be found in PeerTube instance administration') + .requiredOption('--runner-name ', 'Runner name') + .option('--runner-description ', 'Runner description') + .action(async options => { + try { + await registerRunner(options) + } catch (err) { + console.error('Cannot register this PeerTube runner.') + console.error(err) + process.exit(-1) + } + }) + +program.command('unregister') + .description('Unregister the runner from PeerTube instance') + .requiredOption('--url ', 'PeerTube instance URL', parseUrl) + .requiredOption('--runner-name ', 'Runner name') + .action(async options => { + try { + await unregisterRunner(options) + } catch (err) { + console.error('Cannot unregister this PeerTube runner.') + console.error(err) + process.exit(-1) + } + }) + +program.command('list-registered') + .description('List registered PeerTube instances') + .action(async () => { + try { + await listRegistered() + } catch (err) { + console.error('Cannot list registered PeerTube instances.') + console.error(err) + process.exit(-1) + } + }) + +program.parse() + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +function parseUrl (url: string) { + if (url.startsWith('http://') !== true && url.startsWith('https://') !== true) { + throw new InvalidArgumentError('URL should start with a http:// or https://') + } + + return url +} diff --git a/apps/peertube-runner/src/register/index.ts b/apps/peertube-runner/src/register/index.ts new file mode 100644 index 000000000..a7d6cf457 --- /dev/null +++ b/apps/peertube-runner/src/register/index.ts @@ -0,0 +1 @@ +export * from './register.js' diff --git a/apps/peertube-runner/src/register/register.ts b/apps/peertube-runner/src/register/register.ts new file mode 100644 index 000000000..e8af21661 --- /dev/null +++ b/apps/peertube-runner/src/register/register.ts @@ -0,0 +1,36 @@ +import { IPCClient } from '../shared/ipc/index.js' + +export async function registerRunner (options: { + url: string + registrationToken: string + runnerName: string + runnerDescription?: string +}) { + const client = new IPCClient() + await client.run() + + await client.askRegister(options) + + client.stop() +} + +export async function unregisterRunner (options: { + url: string + runnerName: string +}) { + const client = new IPCClient() + await client.run() + + await client.askUnregister(options) + + client.stop() +} + +export async function listRegistered () { + const client = new IPCClient() + await client.run() + + await client.askListRegistered() + + client.stop() +} diff --git a/apps/peertube-runner/src/server/index.ts b/apps/peertube-runner/src/server/index.ts new file mode 100644 index 000000000..e56cda526 --- /dev/null +++ b/apps/peertube-runner/src/server/index.ts @@ -0,0 +1 @@ +export * from './server.js' diff --git a/apps/peertube-runner/src/server/process/index.ts b/apps/peertube-runner/src/server/process/index.ts new file mode 100644 index 000000000..64a7b00fc --- /dev/null +++ b/apps/peertube-runner/src/server/process/index.ts @@ -0,0 +1,2 @@ +export * from './shared/index.js' +export * from './process.js' diff --git a/apps/peertube-runner/src/server/process/process.ts b/apps/peertube-runner/src/server/process/process.ts new file mode 100644 index 000000000..e8a1d7c28 --- /dev/null +++ b/apps/peertube-runner/src/server/process/process.ts @@ -0,0 +1,34 @@ +import { + RunnerJobLiveRTMPHLSTranscodingPayload, + RunnerJobStudioTranscodingPayload, + RunnerJobVODAudioMergeTranscodingPayload, + RunnerJobVODHLSTranscodingPayload, + RunnerJobVODWebVideoTranscodingPayload +} from '@peertube/peertube-models' +import { logger } from '../../shared/index.js' +import { processAudioMergeTranscoding, processHLSTranscoding, ProcessOptions, processWebVideoTranscoding } from './shared/index.js' +import { ProcessLiveRTMPHLSTranscoding } from './shared/process-live.js' +import { processStudioTranscoding } from './shared/process-studio.js' + +export async function processJob (options: ProcessOptions) { + const { server, job } = options + + logger.info(`[${server.url}] Processing job of type ${job.type}: ${job.uuid}`, { payload: job.payload }) + + if (job.type === 'vod-audio-merge-transcoding') { + await processAudioMergeTranscoding(options as ProcessOptions) + } else if (job.type === 'vod-web-video-transcoding') { + await processWebVideoTranscoding(options as ProcessOptions) + } else if (job.type === 'vod-hls-transcoding') { + await processHLSTranscoding(options as ProcessOptions) + } else if (job.type === 'live-rtmp-hls-transcoding') { + await new ProcessLiveRTMPHLSTranscoding(options as ProcessOptions).process() + } else if (job.type === 'video-studio-transcoding') { + await processStudioTranscoding(options as ProcessOptions) + } else { + logger.error(`Unknown job ${job.type} to process`) + return + } + + logger.info(`[${server.url}] Finished processing job of type ${job.type}: ${job.uuid}`) +} diff --git a/apps/peertube-runner/src/server/process/shared/common.ts b/apps/peertube-runner/src/server/process/shared/common.ts new file mode 100644 index 000000000..09241d93b --- /dev/null +++ b/apps/peertube-runner/src/server/process/shared/common.ts @@ -0,0 +1,106 @@ +import { remove } from 'fs-extra/esm' +import { join } from 'path' +import { FFmpegEdition, FFmpegLive, FFmpegVOD, getDefaultAvailableEncoders, getDefaultEncodersToTry } from '@peertube/peertube-ffmpeg' +import { RunnerJob, RunnerJobPayload } from '@peertube/peertube-models' +import { buildUUID } from '@peertube/peertube-node-utils' +import { PeerTubeServer } from '@peertube/peertube-server-commands' +import { ConfigManager, downloadFile, logger } from '../../../shared/index.js' +import { getTranscodingLogger } from './transcoding-logger.js' + +export type JobWithToken = RunnerJob & { jobToken: string } + +export type ProcessOptions = { + server: PeerTubeServer + job: JobWithToken + runnerToken: string +} + +export async function downloadInputFile (options: { + url: string + job: JobWithToken + runnerToken: string +}) { + const { url, job, runnerToken } = options + const destination = join(ConfigManager.Instance.getTranscodingDirectory(), buildUUID()) + + try { + await downloadFile({ url, jobToken: job.jobToken, runnerToken, destination }) + } catch (err) { + remove(destination) + .catch(err => logger.error({ err }, `Cannot remove ${destination}`)) + + throw err + } + + return destination +} + +export function scheduleTranscodingProgress (options: { + server: PeerTubeServer + runnerToken: string + job: JobWithToken + progressGetter: () => number +}) { + const { job, server, progressGetter, runnerToken } = options + + const updateInterval = ConfigManager.Instance.isTestInstance() + ? 500 + : 60000 + + const update = () => { + server.runnerJobs.update({ jobToken: job.jobToken, jobUUID: job.uuid, runnerToken, progress: progressGetter() }) + .catch(err => logger.error({ err }, 'Cannot send job progress')) + } + + const interval = setInterval(() => { + update() + }, updateInterval) + + update() + + return interval +} + +// --------------------------------------------------------------------------- + +export function buildFFmpegVOD (options: { + onJobProgress: (progress: number) => void +}) { + const { onJobProgress } = options + + return new FFmpegVOD({ + ...getCommonFFmpegOptions(), + + updateJobProgress: arg => { + const progress = arg < 0 || arg > 100 + ? undefined + : arg + + onJobProgress(progress) + } + }) +} + +export function buildFFmpegLive () { + return new FFmpegLive(getCommonFFmpegOptions()) +} + +export function buildFFmpegEdition () { + return new FFmpegEdition(getCommonFFmpegOptions()) +} + +function getCommonFFmpegOptions () { + const config = ConfigManager.Instance.getConfig() + + return { + niceness: config.ffmpeg.nice, + threads: config.ffmpeg.threads, + tmpDirectory: ConfigManager.Instance.getTranscodingDirectory(), + profile: 'default', + availableEncoders: { + available: getDefaultAvailableEncoders(), + encodersToTry: getDefaultEncodersToTry() + }, + logger: getTranscodingLogger() + } +} diff --git a/apps/peertube-runner/src/server/process/shared/index.ts b/apps/peertube-runner/src/server/process/shared/index.ts new file mode 100644 index 000000000..638bf127f --- /dev/null +++ b/apps/peertube-runner/src/server/process/shared/index.ts @@ -0,0 +1,3 @@ +export * from './common.js' +export * from './process-vod.js' +export * from './transcoding-logger.js' diff --git a/apps/peertube-runner/src/server/process/shared/process-live.ts b/apps/peertube-runner/src/server/process/shared/process-live.ts new file mode 100644 index 000000000..0dc4e5b13 --- /dev/null +++ b/apps/peertube-runner/src/server/process/shared/process-live.ts @@ -0,0 +1,338 @@ +import { FSWatcher, watch } from 'chokidar' +import { FfmpegCommand } from 'fluent-ffmpeg' +import { ensureDir, remove } from 'fs-extra/esm' +import { basename, join } from 'path' +import { wait } from '@peertube/peertube-core-utils' +import { ffprobePromise, getVideoStreamBitrate, getVideoStreamDimensionsInfo, hasAudioStream } from '@peertube/peertube-ffmpeg' +import { + LiveRTMPHLSTranscodingSuccess, + LiveRTMPHLSTranscodingUpdatePayload, + PeerTubeProblemDocument, + RunnerJobLiveRTMPHLSTranscodingPayload, + ServerErrorCode +} from '@peertube/peertube-models' +import { buildUUID } from '@peertube/peertube-node-utils' +import { ConfigManager } from '../../../shared/config-manager.js' +import { logger } from '../../../shared/index.js' +import { buildFFmpegLive, ProcessOptions } from './common.js' + +export class ProcessLiveRTMPHLSTranscoding { + + private readonly outputPath: string + private readonly fsWatchers: FSWatcher[] = [] + + // Playlist name -> chunks + private readonly pendingChunksPerPlaylist = new Map() + + private readonly playlistsCreated = new Set() + private allPlaylistsCreated = false + + private ffmpegCommand: FfmpegCommand + + private ended = false + private errored = false + + constructor (private readonly options: ProcessOptions) { + this.outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), buildUUID()) + + logger.debug(`Using ${this.outputPath} to process live rtmp hls transcoding job ${options.job.uuid}`) + } + + process () { + const job = this.options.job + const payload = job.payload + + return new Promise(async (res, rej) => { + try { + await ensureDir(this.outputPath) + + logger.info(`Probing ${payload.input.rtmpUrl}`) + const probe = await ffprobePromise(payload.input.rtmpUrl) + logger.info({ probe }, `Probed ${payload.input.rtmpUrl}`) + + const hasAudio = await hasAudioStream(payload.input.rtmpUrl, probe) + const bitrate = await getVideoStreamBitrate(payload.input.rtmpUrl, probe) + const { ratio } = await getVideoStreamDimensionsInfo(payload.input.rtmpUrl, probe) + + const m3u8Watcher = watch(this.outputPath + '/*.m3u8') + this.fsWatchers.push(m3u8Watcher) + + const tsWatcher = watch(this.outputPath + '/*.ts') + this.fsWatchers.push(tsWatcher) + + m3u8Watcher.on('change', p => { + logger.debug(`${p} m3u8 playlist changed`) + }) + + m3u8Watcher.on('add', p => { + this.playlistsCreated.add(p) + + if (this.playlistsCreated.size === this.options.job.payload.output.toTranscode.length + 1) { + this.allPlaylistsCreated = true + logger.info('All m3u8 playlists are created.') + } + }) + + tsWatcher.on('add', async p => { + try { + await this.sendPendingChunks() + } catch (err) { + this.onUpdateError({ err, rej, res }) + } + + const playlistName = this.getPlaylistIdFromTS(p) + + const pendingChunks = this.pendingChunksPerPlaylist.get(playlistName) || [] + pendingChunks.push(p) + + this.pendingChunksPerPlaylist.set(playlistName, pendingChunks) + }) + + tsWatcher.on('unlink', p => { + this.sendDeletedChunkUpdate(p) + .catch(err => this.onUpdateError({ err, rej, res })) + }) + + this.ffmpegCommand = await buildFFmpegLive().getLiveTranscodingCommand({ + inputUrl: payload.input.rtmpUrl, + + outPath: this.outputPath, + masterPlaylistName: 'master.m3u8', + + segmentListSize: payload.output.segmentListSize, + segmentDuration: payload.output.segmentDuration, + + toTranscode: payload.output.toTranscode, + + bitrate, + ratio, + + hasAudio + }) + + logger.info(`Running live transcoding for ${payload.input.rtmpUrl}`) + + this.ffmpegCommand.on('error', (err, stdout, stderr) => { + this.onFFmpegError({ err, stdout, stderr }) + + res() + }) + + this.ffmpegCommand.on('end', () => { + this.onFFmpegEnded() + .catch(err => logger.error({ err }, 'Error in FFmpeg end handler')) + + res() + }) + + this.ffmpegCommand.run() + } catch (err) { + rej(err) + } + }) + } + + // --------------------------------------------------------------------------- + + private onUpdateError (options: { + err: Error + res: () => void + rej: (reason?: any) => void + }) { + const { err, res, rej } = options + + if (this.errored) return + if (this.ended) return + + this.errored = true + + this.ffmpegCommand.kill('SIGINT') + + const type = ((err as any).res?.body as PeerTubeProblemDocument)?.code + if (type === ServerErrorCode.RUNNER_JOB_NOT_IN_PROCESSING_STATE) { + logger.info({ err }, 'Stopping transcoding as the job is not in processing state anymore') + + res() + } else { + logger.error({ err }, 'Cannot send update after added/deleted chunk, stopping live transcoding') + + this.sendError(err) + .catch(subErr => logger.error({ err: subErr }, 'Cannot send error')) + + rej(err) + } + + this.cleanup() + } + + // --------------------------------------------------------------------------- + + private onFFmpegError (options: { + err: any + stdout: string + stderr: string + }) { + const { err, stdout, stderr } = options + + // Don't care that we killed the ffmpeg process + if (err?.message?.includes('Exiting normally')) return + if (this.errored) return + if (this.ended) return + + this.errored = true + + logger.error({ err, stdout, stderr }, 'FFmpeg transcoding error.') + + this.sendError(err) + .catch(subErr => logger.error({ err: subErr }, 'Cannot send error')) + + this.cleanup() + } + + private async sendError (err: Error) { + await this.options.server.runnerJobs.error({ + jobToken: this.options.job.jobToken, + jobUUID: this.options.job.uuid, + runnerToken: this.options.runnerToken, + message: err.message + }) + } + + // --------------------------------------------------------------------------- + + private async onFFmpegEnded () { + if (this.ended) return + + this.ended = true + logger.info('FFmpeg ended, sending success to server') + + // Wait last ffmpeg chunks generation + await wait(1500) + + this.sendSuccess() + .catch(err => logger.error({ err }, 'Cannot send success')) + + this.cleanup() + } + + private async sendSuccess () { + const successBody: LiveRTMPHLSTranscodingSuccess = {} + + await this.options.server.runnerJobs.success({ + jobToken: this.options.job.jobToken, + jobUUID: this.options.job.uuid, + runnerToken: this.options.runnerToken, + payload: successBody + }) + } + + // --------------------------------------------------------------------------- + + private sendDeletedChunkUpdate (deletedChunk: string): Promise { + if (this.ended) return Promise.resolve() + + logger.debug(`Sending removed live chunk ${deletedChunk} update`) + + const videoChunkFilename = basename(deletedChunk) + + let payload: LiveRTMPHLSTranscodingUpdatePayload = { + type: 'remove-chunk', + videoChunkFilename + } + + if (this.allPlaylistsCreated) { + const playlistName = this.getPlaylistName(videoChunkFilename) + + payload = { + ...payload, + masterPlaylistFile: join(this.outputPath, 'master.m3u8'), + resolutionPlaylistFilename: playlistName, + resolutionPlaylistFile: join(this.outputPath, playlistName) + } + } + + return this.updateWithRetry(payload) + } + + private async sendPendingChunks (): Promise { + if (this.ended) return Promise.resolve() + + const promises: Promise[] = [] + + for (const playlist of this.pendingChunksPerPlaylist.keys()) { + for (const chunk of this.pendingChunksPerPlaylist.get(playlist)) { + logger.debug(`Sending added live chunk ${chunk} update`) + + const videoChunkFilename = basename(chunk) + + let payload: LiveRTMPHLSTranscodingUpdatePayload = { + type: 'add-chunk', + videoChunkFilename, + videoChunkFile: chunk + } + + if (this.allPlaylistsCreated) { + const playlistName = this.getPlaylistName(videoChunkFilename) + + payload = { + ...payload, + masterPlaylistFile: join(this.outputPath, 'master.m3u8'), + resolutionPlaylistFilename: playlistName, + resolutionPlaylistFile: join(this.outputPath, playlistName) + } + } + + promises.push(this.updateWithRetry(payload)) + } + + this.pendingChunksPerPlaylist.set(playlist, []) + } + + await Promise.all(promises) + } + + private async updateWithRetry (payload: LiveRTMPHLSTranscodingUpdatePayload, currentTry = 1): Promise { + if (this.ended || this.errored) return + + try { + await this.options.server.runnerJobs.update({ + jobToken: this.options.job.jobToken, + jobUUID: this.options.job.uuid, + runnerToken: this.options.runnerToken, + payload + }) + } catch (err) { + if (currentTry >= 3) throw err + if ((err.res?.body as PeerTubeProblemDocument)?.code === ServerErrorCode.RUNNER_JOB_NOT_IN_PROCESSING_STATE) throw err + + logger.warn({ err }, 'Will retry update after error') + await wait(250) + + return this.updateWithRetry(payload, currentTry + 1) + } + } + + private getPlaylistName (videoChunkFilename: string) { + return `${videoChunkFilename.split('-')[0]}.m3u8` + } + + private getPlaylistIdFromTS (segmentPath: string) { + const playlistIdMatcher = /^([\d+])-/ + + return basename(segmentPath).match(playlistIdMatcher)[1] + } + + // --------------------------------------------------------------------------- + + private cleanup () { + logger.debug(`Cleaning up job ${this.options.job.uuid}`) + + for (const fsWatcher of this.fsWatchers) { + fsWatcher.close() + .catch(err => logger.error({ err }, 'Cannot close watcher')) + } + + remove(this.outputPath) + .catch(err => logger.error({ err }, `Cannot remove ${this.outputPath}`)) + } +} diff --git a/apps/peertube-runner/src/server/process/shared/process-studio.ts b/apps/peertube-runner/src/server/process/shared/process-studio.ts new file mode 100644 index 000000000..11b7b7d9a --- /dev/null +++ b/apps/peertube-runner/src/server/process/shared/process-studio.ts @@ -0,0 +1,165 @@ +import { remove } from 'fs-extra/esm' +import { join } from 'path' +import { pick } from '@peertube/peertube-core-utils' +import { + RunnerJobStudioTranscodingPayload, + VideoStudioTask, + VideoStudioTaskCutPayload, + VideoStudioTaskIntroPayload, + VideoStudioTaskOutroPayload, + VideoStudioTaskPayload, + VideoStudioTaskWatermarkPayload, + VideoStudioTranscodingSuccess +} from '@peertube/peertube-models' +import { buildUUID } from '@peertube/peertube-node-utils' +import { ConfigManager } from '../../../shared/config-manager.js' +import { logger } from '../../../shared/index.js' +import { buildFFmpegEdition, downloadInputFile, JobWithToken, ProcessOptions, scheduleTranscodingProgress } from './common.js' + +export async function processStudioTranscoding (options: ProcessOptions) { + const { server, job, runnerToken } = options + const payload = job.payload + + let inputPath: string + let outputPath: string + let tmpInputFilePath: string + + let tasksProgress = 0 + + const updateProgressInterval = scheduleTranscodingProgress({ + job, + server, + runnerToken, + progressGetter: () => tasksProgress + }) + + try { + logger.info(`Downloading input file ${payload.input.videoFileUrl} for job ${job.jobToken}`) + + inputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job }) + tmpInputFilePath = inputPath + + logger.info(`Input file ${payload.input.videoFileUrl} downloaded for job ${job.jobToken}. Running studio transcoding tasks.`) + + for (const task of payload.tasks) { + const outputFilename = 'output-edition-' + buildUUID() + '.mp4' + outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), outputFilename) + + await processTask({ + inputPath: tmpInputFilePath, + outputPath, + task, + job, + runnerToken + }) + + if (tmpInputFilePath) await remove(tmpInputFilePath) + + // For the next iteration + tmpInputFilePath = outputPath + + tasksProgress += Math.floor(100 / payload.tasks.length) + } + + const successBody: VideoStudioTranscodingSuccess = { + videoFile: outputPath + } + + await server.runnerJobs.success({ + jobToken: job.jobToken, + jobUUID: job.uuid, + runnerToken, + payload: successBody + }) + } finally { + if (tmpInputFilePath) await remove(tmpInputFilePath) + if (outputPath) await remove(outputPath) + if (updateProgressInterval) clearInterval(updateProgressInterval) + } +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +type TaskProcessorOptions = { + inputPath: string + outputPath: string + task: T + runnerToken: string + job: JobWithToken +} + +const taskProcessors: { [id in VideoStudioTask['name']]: (options: TaskProcessorOptions) => Promise } = { + 'add-intro': processAddIntroOutro, + 'add-outro': processAddIntroOutro, + 'cut': processCut, + 'add-watermark': processAddWatermark +} + +async function processTask (options: TaskProcessorOptions) { + const { task } = options + + const processor = taskProcessors[options.task.name] + if (!process) throw new Error('Unknown task ' + task.name) + + return processor(options) +} + +async function processAddIntroOutro (options: TaskProcessorOptions) { + const { inputPath, task, runnerToken, job } = options + + logger.debug('Adding intro/outro to ' + inputPath) + + const introOutroPath = await downloadInputFile({ url: task.options.file, runnerToken, job }) + + try { + await buildFFmpegEdition().addIntroOutro({ + ...pick(options, [ 'inputPath', 'outputPath' ]), + + introOutroPath, + type: task.name === 'add-intro' + ? 'intro' + : 'outro' + }) + } finally { + await remove(introOutroPath) + } +} + +function processCut (options: TaskProcessorOptions) { + const { inputPath, task } = options + + logger.debug(`Cutting ${inputPath}`) + + return buildFFmpegEdition().cutVideo({ + ...pick(options, [ 'inputPath', 'outputPath' ]), + + start: task.options.start, + end: task.options.end + }) +} + +async function processAddWatermark (options: TaskProcessorOptions) { + const { inputPath, task, runnerToken, job } = options + + logger.debug('Adding watermark to ' + inputPath) + + const watermarkPath = await downloadInputFile({ url: task.options.file, runnerToken, job }) + + try { + await buildFFmpegEdition().addWatermark({ + ...pick(options, [ 'inputPath', 'outputPath' ]), + + watermarkPath, + + videoFilters: { + watermarkSizeRatio: task.options.watermarkSizeRatio, + horitonzalMarginRatio: task.options.horitonzalMarginRatio, + verticalMarginRatio: task.options.verticalMarginRatio + } + }) + } finally { + await remove(watermarkPath) + } +} diff --git a/apps/peertube-runner/src/server/process/shared/process-vod.ts b/apps/peertube-runner/src/server/process/shared/process-vod.ts new file mode 100644 index 000000000..fe1715ca9 --- /dev/null +++ b/apps/peertube-runner/src/server/process/shared/process-vod.ts @@ -0,0 +1,201 @@ +import { remove } from 'fs-extra/esm' +import { join } from 'path' +import { + RunnerJobVODAudioMergeTranscodingPayload, + RunnerJobVODHLSTranscodingPayload, + RunnerJobVODWebVideoTranscodingPayload, + VODAudioMergeTranscodingSuccess, + VODHLSTranscodingSuccess, + VODWebVideoTranscodingSuccess +} from '@peertube/peertube-models' +import { buildUUID } from '@peertube/peertube-node-utils' +import { ConfigManager } from '../../../shared/config-manager.js' +import { logger } from '../../../shared/index.js' +import { buildFFmpegVOD, downloadInputFile, ProcessOptions, scheduleTranscodingProgress } from './common.js' + +export async function processWebVideoTranscoding (options: ProcessOptions) { + const { server, job, runnerToken } = options + + const payload = job.payload + + let ffmpegProgress: number + let inputPath: string + + const outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), `output-${buildUUID()}.mp4`) + + const updateProgressInterval = scheduleTranscodingProgress({ + job, + server, + runnerToken, + progressGetter: () => ffmpegProgress + }) + + try { + logger.info(`Downloading input file ${payload.input.videoFileUrl} for web video transcoding job ${job.jobToken}`) + + inputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job }) + + logger.info(`Downloaded input file ${payload.input.videoFileUrl} for job ${job.jobToken}. Running web video transcoding.`) + + const ffmpegVod = buildFFmpegVOD({ + onJobProgress: progress => { ffmpegProgress = progress } + }) + + await ffmpegVod.transcode({ + type: 'video', + + inputPath, + + outputPath, + + inputFileMutexReleaser: () => {}, + + resolution: payload.output.resolution, + fps: payload.output.fps + }) + + const successBody: VODWebVideoTranscodingSuccess = { + videoFile: outputPath + } + + await server.runnerJobs.success({ + jobToken: job.jobToken, + jobUUID: job.uuid, + runnerToken, + payload: successBody + }) + } finally { + if (inputPath) await remove(inputPath) + if (outputPath) await remove(outputPath) + if (updateProgressInterval) clearInterval(updateProgressInterval) + } +} + +export async function processHLSTranscoding (options: ProcessOptions) { + const { server, job, runnerToken } = options + const payload = job.payload + + let ffmpegProgress: number + let inputPath: string + + const uuid = buildUUID() + const outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), `${uuid}-${payload.output.resolution}.m3u8`) + const videoFilename = `${uuid}-${payload.output.resolution}-fragmented.mp4` + const videoPath = join(join(ConfigManager.Instance.getTranscodingDirectory(), videoFilename)) + + const updateProgressInterval = scheduleTranscodingProgress({ + job, + server, + runnerToken, + progressGetter: () => ffmpegProgress + }) + + try { + logger.info(`Downloading input file ${payload.input.videoFileUrl} for HLS transcoding job ${job.jobToken}`) + + inputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job }) + + logger.info(`Downloaded input file ${payload.input.videoFileUrl} for job ${job.jobToken}. Running HLS transcoding.`) + + const ffmpegVod = buildFFmpegVOD({ + onJobProgress: progress => { ffmpegProgress = progress } + }) + + await ffmpegVod.transcode({ + type: 'hls', + copyCodecs: false, + inputPath, + hlsPlaylist: { videoFilename }, + outputPath, + + inputFileMutexReleaser: () => {}, + + resolution: payload.output.resolution, + fps: payload.output.fps + }) + + const successBody: VODHLSTranscodingSuccess = { + resolutionPlaylistFile: outputPath, + videoFile: videoPath + } + + await server.runnerJobs.success({ + jobToken: job.jobToken, + jobUUID: job.uuid, + runnerToken, + payload: successBody + }) + } finally { + if (inputPath) await remove(inputPath) + if (outputPath) await remove(outputPath) + if (videoPath) await remove(videoPath) + if (updateProgressInterval) clearInterval(updateProgressInterval) + } +} + +export async function processAudioMergeTranscoding (options: ProcessOptions) { + const { server, job, runnerToken } = options + const payload = job.payload + + let ffmpegProgress: number + let audioPath: string + let inputPath: string + + const outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), `output-${buildUUID()}.mp4`) + + const updateProgressInterval = scheduleTranscodingProgress({ + job, + server, + runnerToken, + progressGetter: () => ffmpegProgress + }) + + try { + logger.info( + `Downloading input files ${payload.input.audioFileUrl} and ${payload.input.previewFileUrl} ` + + `for audio merge transcoding job ${job.jobToken}` + ) + + audioPath = await downloadInputFile({ url: payload.input.audioFileUrl, runnerToken, job }) + inputPath = await downloadInputFile({ url: payload.input.previewFileUrl, runnerToken, job }) + + logger.info( + `Downloaded input files ${payload.input.audioFileUrl} and ${payload.input.previewFileUrl} ` + + `for job ${job.jobToken}. Running audio merge transcoding.` + ) + + const ffmpegVod = buildFFmpegVOD({ + onJobProgress: progress => { ffmpegProgress = progress } + }) + + await ffmpegVod.transcode({ + type: 'merge-audio', + + audioPath, + inputPath, + + outputPath, + + inputFileMutexReleaser: () => {}, + + resolution: payload.output.resolution, + fps: payload.output.fps + }) + + const successBody: VODAudioMergeTranscodingSuccess = { + videoFile: outputPath + } + + await server.runnerJobs.success({ + jobToken: job.jobToken, + jobUUID: job.uuid, + runnerToken, + payload: successBody + }) + } finally { + if (audioPath) await remove(audioPath) + if (inputPath) await remove(inputPath) + if (outputPath) await remove(outputPath) + if (updateProgressInterval) clearInterval(updateProgressInterval) + } +} diff --git a/apps/peertube-runner/src/server/process/shared/transcoding-logger.ts b/apps/peertube-runner/src/server/process/shared/transcoding-logger.ts new file mode 100644 index 000000000..041dd62eb --- /dev/null +++ b/apps/peertube-runner/src/server/process/shared/transcoding-logger.ts @@ -0,0 +1,10 @@ +import { logger } from '../../../shared/index.js' + +export function getTranscodingLogger () { + return { + info: logger.info.bind(logger), + debug: logger.debug.bind(logger), + warn: logger.warn.bind(logger), + error: logger.error.bind(logger) + } +} diff --git a/apps/peertube-runner/src/server/server.ts b/apps/peertube-runner/src/server/server.ts new file mode 100644 index 000000000..825e3f297 --- /dev/null +++ b/apps/peertube-runner/src/server/server.ts @@ -0,0 +1,307 @@ +import { ensureDir, remove } from 'fs-extra/esm' +import { readdir } from 'fs/promises' +import { join } from 'path' +import { io, Socket } from 'socket.io-client' +import { pick, shuffle, wait } from '@peertube/peertube-core-utils' +import { PeerTubeProblemDocument, ServerErrorCode } from '@peertube/peertube-models' +import { PeerTubeServer as PeerTubeServerCommand } from '@peertube/peertube-server-commands' +import { ConfigManager } from '../shared/index.js' +import { IPCServer } from '../shared/ipc/index.js' +import { logger } from '../shared/logger.js' +import { JobWithToken, processJob } from './process/index.js' +import { isJobSupported } from './shared/index.js' + +type PeerTubeServer = PeerTubeServerCommand & { + runnerToken: string + runnerName: string + runnerDescription?: string +} + +export class RunnerServer { + private static instance: RunnerServer + + private servers: PeerTubeServer[] = [] + private processingJobs: { job: JobWithToken, server: PeerTubeServer }[] = [] + + private checkingAvailableJobs = false + + private cleaningUp = false + + private readonly sockets = new Map() + + private constructor () {} + + async run () { + logger.info('Running PeerTube runner in server mode') + + await ConfigManager.Instance.load() + + for (const registered of ConfigManager.Instance.getConfig().registeredInstances) { + const serverCommand = new PeerTubeServerCommand({ url: registered.url }) + + this.loadServer(Object.assign(serverCommand, registered)) + + logger.info(`Loading registered instance ${registered.url}`) + } + + // Run IPC + const ipcServer = new IPCServer() + try { + await ipcServer.run(this) + } catch (err) { + logger.error('Cannot start local socket for IPC communication', err) + process.exit(-1) + } + + // Cleanup on exit + for (const code of [ 'SIGINT', 'SIGUSR1', 'SIGUSR2', 'uncaughtException' ]) { + process.on(code, async (err, origin) => { + if (code === 'uncaughtException') { + logger.error({ err, origin }, 'uncaughtException') + } + + await this.onExit() + }) + } + + // Process jobs + await ensureDir(ConfigManager.Instance.getTranscodingDirectory()) + await this.cleanupTMP() + + logger.info(`Using ${ConfigManager.Instance.getTranscodingDirectory()} for transcoding directory`) + + await this.checkAvailableJobs() + } + + // --------------------------------------------------------------------------- + + async registerRunner (options: { + url: string + registrationToken: string + runnerName: string + runnerDescription?: string + }) { + const { url, registrationToken, runnerName, runnerDescription } = options + + logger.info(`Registering runner ${runnerName} on ${url}...`) + + const serverCommand = new PeerTubeServerCommand({ url }) + const { runnerToken } = await serverCommand.runners.register({ name: runnerName, description: runnerDescription, registrationToken }) + + const server: PeerTubeServer = Object.assign(serverCommand, { + runnerToken, + runnerName, + runnerDescription + }) + + this.loadServer(server) + await this.saveRegisteredInstancesInConf() + + logger.info(`Registered runner ${runnerName} on ${url}`) + + await this.checkAvailableJobs() + } + + private loadServer (server: PeerTubeServer) { + this.servers.push(server) + + const url = server.url + '/runners' + const socket = io(url, { + auth: { + runnerToken: server.runnerToken + }, + transports: [ 'websocket' ] + }) + + socket.on('connect_error', err => logger.warn({ err }, `Cannot connect to ${url} socket`)) + socket.on('connect', () => logger.info(`Connected to ${url} socket`)) + socket.on('available-jobs', () => this.checkAvailableJobs()) + + this.sockets.set(server, socket) + } + + async unregisterRunner (options: { + url: string + runnerName: string + }) { + const { url, runnerName } = options + + const server = this.servers.find(s => s.url === url && s.runnerName === runnerName) + if (!server) { + logger.error(`Unknown server ${url} - ${runnerName} to unregister`) + return + } + + logger.info(`Unregistering runner ${runnerName} on ${url}...`) + + try { + await server.runners.unregister({ runnerToken: server.runnerToken }) + } catch (err) { + logger.error({ err }, `Cannot unregister runner ${runnerName} on ${url}`) + } + + this.unloadServer(server) + await this.saveRegisteredInstancesInConf() + + logger.info(`Unregistered runner ${runnerName} on ${url}`) + } + + private unloadServer (server: PeerTubeServer) { + this.servers = this.servers.filter(s => s !== server) + + const socket = this.sockets.get(server) + socket.disconnect() + + this.sockets.delete(server) + } + + listRegistered () { + return { + servers: this.servers.map(s => { + return { + url: s.url, + runnerName: s.runnerName, + runnerDescription: s.runnerDescription + } + }) + } + } + + // --------------------------------------------------------------------------- + + private async checkAvailableJobs () { + if (this.checkingAvailableJobs) return + + this.checkingAvailableJobs = true + + let hadAvailableJob = false + + for (const server of shuffle([ ...this.servers ])) { + try { + logger.info('Checking available jobs on ' + server.url) + + const job = await this.requestJob(server) + if (!job) continue + + hadAvailableJob = true + + await this.tryToExecuteJobAsync(server, job) + } catch (err) { + const code = (err.res?.body as PeerTubeProblemDocument)?.code + + if (code === ServerErrorCode.RUNNER_JOB_NOT_IN_PROCESSING_STATE) { + logger.debug({ err }, 'Runner job is not in processing state anymore, retry later') + return + } + + if (code === ServerErrorCode.UNKNOWN_RUNNER_TOKEN) { + logger.error({ err }, `Unregistering ${server.url} as the runner token ${server.runnerToken} is invalid`) + + await this.unregisterRunner({ url: server.url, runnerName: server.runnerName }) + return + } + + logger.error({ err }, `Cannot request/accept job on ${server.url} for runner ${server.runnerName}`) + } + } + + this.checkingAvailableJobs = false + + if (hadAvailableJob && this.canProcessMoreJobs()) { + await wait(2500) + + this.checkAvailableJobs() + .catch(err => logger.error({ err }, 'Cannot check more available jobs')) + } + } + + private async requestJob (server: PeerTubeServer) { + logger.debug(`Requesting jobs on ${server.url} for runner ${server.runnerName}`) + + const { availableJobs } = await server.runnerJobs.request({ runnerToken: server.runnerToken }) + + const filtered = availableJobs.filter(j => isJobSupported(j)) + + if (filtered.length === 0) { + logger.debug(`No job available on ${server.url} for runner ${server.runnerName}`) + return undefined + } + + return filtered[0] + } + + private async tryToExecuteJobAsync (server: PeerTubeServer, jobToAccept: { uuid: string }) { + if (!this.canProcessMoreJobs()) return + + const { job } = await server.runnerJobs.accept({ runnerToken: server.runnerToken, jobUUID: jobToAccept.uuid }) + + const processingJob = { job, server } + this.processingJobs.push(processingJob) + + processJob({ server, job, runnerToken: server.runnerToken }) + .catch(err => { + logger.error({ err }, 'Cannot process job') + + server.runnerJobs.error({ jobToken: job.jobToken, jobUUID: job.uuid, runnerToken: server.runnerToken, message: err.message }) + .catch(err2 => logger.error({ err: err2 }, 'Cannot abort job after error')) + }) + .finally(() => { + this.processingJobs = this.processingJobs.filter(p => p !== processingJob) + + return this.checkAvailableJobs() + }) + } + + // --------------------------------------------------------------------------- + + private saveRegisteredInstancesInConf () { + const data = this.servers.map(s => { + return pick(s, [ 'url', 'runnerToken', 'runnerName', 'runnerDescription' ]) + }) + + return ConfigManager.Instance.setRegisteredInstances(data) + } + + private canProcessMoreJobs () { + return this.processingJobs.length < ConfigManager.Instance.getConfig().jobs.concurrency + } + + // --------------------------------------------------------------------------- + + private async cleanupTMP () { + const files = await readdir(ConfigManager.Instance.getTranscodingDirectory()) + + for (const file of files) { + await remove(join(ConfigManager.Instance.getTranscodingDirectory(), file)) + } + } + + private async onExit () { + if (this.cleaningUp) return + this.cleaningUp = true + + logger.info('Cleaning up after program exit') + + try { + for (const { server, job } of this.processingJobs) { + await server.runnerJobs.abort({ + jobToken: job.jobToken, + jobUUID: job.uuid, + reason: 'Runner stopped', + runnerToken: server.runnerToken + }) + } + + await this.cleanupTMP() + } catch (err) { + logger.error(err) + process.exit(-1) + } + + process.exit() + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/apps/peertube-runner/src/server/shared/index.ts b/apps/peertube-runner/src/server/shared/index.ts new file mode 100644 index 000000000..34d51196b --- /dev/null +++ b/apps/peertube-runner/src/server/shared/index.ts @@ -0,0 +1 @@ +export * from './supported-job.js' diff --git a/apps/peertube-runner/src/server/shared/supported-job.ts b/apps/peertube-runner/src/server/shared/supported-job.ts new file mode 100644 index 000000000..d905b5de2 --- /dev/null +++ b/apps/peertube-runner/src/server/shared/supported-job.ts @@ -0,0 +1,43 @@ +import { + RunnerJobLiveRTMPHLSTranscodingPayload, + RunnerJobPayload, + RunnerJobType, + RunnerJobStudioTranscodingPayload, + RunnerJobVODAudioMergeTranscodingPayload, + RunnerJobVODHLSTranscodingPayload, + RunnerJobVODWebVideoTranscodingPayload, + VideoStudioTaskPayload +} from '@peertube/peertube-models' + +const supportedMatrix = { + 'vod-web-video-transcoding': (_payload: RunnerJobVODWebVideoTranscodingPayload) => { + return true + }, + 'vod-hls-transcoding': (_payload: RunnerJobVODHLSTranscodingPayload) => { + return true + }, + 'vod-audio-merge-transcoding': (_payload: RunnerJobVODAudioMergeTranscodingPayload) => { + return true + }, + 'live-rtmp-hls-transcoding': (_payload: RunnerJobLiveRTMPHLSTranscodingPayload) => { + return true + }, + 'video-studio-transcoding': (payload: RunnerJobStudioTranscodingPayload) => { + const tasks = payload?.tasks + const supported = new Set([ 'add-intro', 'add-outro', 'add-watermark', 'cut' ]) + + if (!Array.isArray(tasks)) return false + + return tasks.every(t => t && supported.has(t.name)) + } +} + +export function isJobSupported (job: { + type: RunnerJobType + payload: RunnerJobPayload +}) { + const fn = supportedMatrix[job.type] + if (!fn) return false + + return fn(job.payload as any) +} diff --git a/apps/peertube-runner/src/shared/config-manager.ts b/apps/peertube-runner/src/shared/config-manager.ts new file mode 100644 index 000000000..84a326a16 --- /dev/null +++ b/apps/peertube-runner/src/shared/config-manager.ts @@ -0,0 +1,140 @@ +import { parse, stringify } from '@iarna/toml' +import envPaths from 'env-paths' +import { ensureDir, pathExists, remove } from 'fs-extra/esm' +import { readFile, writeFile } from 'fs/promises' +import merge from 'lodash-es/merge.js' +import { dirname, join } from 'path' +import { logger } from '../shared/index.js' + +const paths = envPaths('peertube-runner') + +type Config = { + jobs: { + concurrency: number + } + + ffmpeg: { + threads: number + nice: number + } + + registeredInstances: { + url: string + runnerToken: string + runnerName: string + runnerDescription?: string + }[] +} + +export class ConfigManager { + private static instance: ConfigManager + + private config: Config = { + jobs: { + concurrency: 2 + }, + ffmpeg: { + threads: 2, + nice: 20 + }, + registeredInstances: [] + } + + private id: string + private configFilePath: string + + private constructor () {} + + init (id: string) { + this.id = id + this.configFilePath = join(this.getConfigDir(), 'config.toml') + } + + async load () { + logger.info(`Using ${this.configFilePath} as configuration file`) + + if (this.isTestInstance()) { + logger.info('Removing configuration file as we are using the "test" id') + await remove(this.configFilePath) + } + + await ensureDir(dirname(this.configFilePath)) + + if (!await pathExists(this.configFilePath)) { + await this.save() + } + + const file = await readFile(this.configFilePath, 'utf-8') + + this.config = merge(this.config, parse(file)) + } + + save () { + return writeFile(this.configFilePath, stringify(this.config)) + } + + // --------------------------------------------------------------------------- + + async setRegisteredInstances (registeredInstances: { + url: string + runnerToken: string + runnerName: string + runnerDescription?: string + }[]) { + this.config.registeredInstances = registeredInstances + + await this.save() + } + + // --------------------------------------------------------------------------- + + getConfig () { + return this.deepFreeze(this.config) + } + + // --------------------------------------------------------------------------- + + getTranscodingDirectory () { + return join(paths.cache, this.id, 'transcoding') + } + + getSocketDirectory () { + return join(paths.data, this.id) + } + + getSocketPath () { + return join(this.getSocketDirectory(), 'peertube-runner.sock') + } + + getConfigDir () { + return join(paths.config, this.id) + } + + // --------------------------------------------------------------------------- + + isTestInstance () { + return typeof this.id === 'string' && this.id.match(/^test-\d$/) + } + + // --------------------------------------------------------------------------- + + // Thanks: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/freeze + private deepFreeze (object: T) { + const propNames = Reflect.ownKeys(object) + + // Freeze properties before freezing self + for (const name of propNames) { + const value = object[name] + + if ((value && typeof value === 'object') || typeof value === 'function') { + this.deepFreeze(value) + } + } + + return Object.freeze({ ...object }) + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/apps/peertube-runner/src/shared/http.ts b/apps/peertube-runner/src/shared/http.ts new file mode 100644 index 000000000..42886279c --- /dev/null +++ b/apps/peertube-runner/src/shared/http.ts @@ -0,0 +1,67 @@ +import { createWriteStream } from 'fs' +import { remove } from 'fs-extra/esm' +import { request as requestHTTP } from 'http' +import { request as requestHTTPS, RequestOptions } from 'https' +import { logger } from './logger.js' + +export function downloadFile (options: { + url: string + destination: string + runnerToken: string + jobToken: string +}) { + const { url, destination, runnerToken, jobToken } = options + + logger.debug(`Downloading file ${url}`) + + return new Promise((res, rej) => { + const parsed = new URL(url) + + const body = JSON.stringify({ + runnerToken, + jobToken + }) + + const getOptions: RequestOptions = { + method: 'POST', + hostname: parsed.hostname, + port: parsed.port, + path: parsed.pathname, + headers: { + 'Content-Type': 'application/json', + 'Content-Length': Buffer.byteLength(body, 'utf-8') + } + } + + const request = getRequest(url)(getOptions, response => { + const code = response.statusCode ?? 0 + + if (code >= 400) { + return rej(new Error(response.statusMessage)) + } + + const file = createWriteStream(destination) + file.on('finish', () => res()) + + response.pipe(file) + }) + + request.on('error', err => { + remove(destination) + .catch(err => logger.error(err)) + + return rej(err) + }) + + request.write(body) + request.end() + }) +} + +// --------------------------------------------------------------------------- + +function getRequest (url: string) { + if (url.startsWith('https://')) return requestHTTPS + + return requestHTTP +} diff --git a/apps/peertube-runner/src/shared/index.ts b/apps/peertube-runner/src/shared/index.ts new file mode 100644 index 000000000..951eef55b --- /dev/null +++ b/apps/peertube-runner/src/shared/index.ts @@ -0,0 +1,3 @@ +export * from './config-manager.js' +export * from './http.js' +export * from './logger.js' diff --git a/apps/peertube-runner/src/shared/ipc/index.ts b/apps/peertube-runner/src/shared/ipc/index.ts new file mode 100644 index 000000000..337d4de16 --- /dev/null +++ b/apps/peertube-runner/src/shared/ipc/index.ts @@ -0,0 +1,2 @@ +export * from './ipc-client.js' +export * from './ipc-server.js' diff --git a/apps/peertube-runner/src/shared/ipc/ipc-client.ts b/apps/peertube-runner/src/shared/ipc/ipc-client.ts new file mode 100644 index 000000000..aa5740dd1 --- /dev/null +++ b/apps/peertube-runner/src/shared/ipc/ipc-client.ts @@ -0,0 +1,88 @@ +import CliTable3 from 'cli-table3' +import { ensureDir } from 'fs-extra/esm' +import { Client as NetIPC } from 'net-ipc' +import { ConfigManager } from '../config-manager.js' +import { IPCReponse, IPCReponseData, IPCRequest } from './shared/index.js' + +export class IPCClient { + private netIPC: NetIPC + + async run () { + await ensureDir(ConfigManager.Instance.getSocketDirectory()) + + const socketPath = ConfigManager.Instance.getSocketPath() + + this.netIPC = new NetIPC({ path: socketPath }) + + try { + await this.netIPC.connect() + } catch (err) { + if (err.code === 'ECONNREFUSED') { + throw new Error( + 'This runner is not currently running in server mode on this system. ' + + 'Please run it using the `server` command first (in another terminal for example) and then retry your command.' + ) + } + + throw err + } + } + + async askRegister (options: { + url: string + registrationToken: string + runnerName: string + runnerDescription?: string + }) { + const req: IPCRequest = { + type: 'register', + ...options + } + + const { success, error } = await this.netIPC.request(req) as IPCReponse + + if (success) console.log('PeerTube instance registered') + else console.error('Could not register PeerTube instance on runner server side', error) + } + + async askUnregister (options: { + url: string + runnerName: string + }) { + const req: IPCRequest = { + type: 'unregister', + ...options + } + + const { success, error } = await this.netIPC.request(req) as IPCReponse + + if (success) console.log('PeerTube instance unregistered') + else console.error('Could not unregister PeerTube instance on runner server side', error) + } + + async askListRegistered () { + const req: IPCRequest = { + type: 'list-registered' + } + + const { success, error, data } = await this.netIPC.request(req) as IPCReponse + if (!success) { + console.error('Could not list registered PeerTube instances', error) + return + } + + const table = new CliTable3({ + head: [ 'instance', 'runner name', 'runner description' ] + }) + + for (const server of data.servers) { + table.push([ server.url, server.runnerName, server.runnerDescription ]) + } + + console.log(table.toString()) + } + + stop () { + this.netIPC.destroy() + } +} diff --git a/apps/peertube-runner/src/shared/ipc/ipc-server.ts b/apps/peertube-runner/src/shared/ipc/ipc-server.ts new file mode 100644 index 000000000..c68438504 --- /dev/null +++ b/apps/peertube-runner/src/shared/ipc/ipc-server.ts @@ -0,0 +1,61 @@ +import { ensureDir } from 'fs-extra/esm' +import { Server as NetIPC } from 'net-ipc' +import { pick } from '@peertube/peertube-core-utils' +import { RunnerServer } from '../../server/index.js' +import { ConfigManager } from '../config-manager.js' +import { logger } from '../logger.js' +import { IPCReponse, IPCReponseData, IPCRequest } from './shared/index.js' + +export class IPCServer { + private netIPC: NetIPC + private runnerServer: RunnerServer + + async run (runnerServer: RunnerServer) { + this.runnerServer = runnerServer + + await ensureDir(ConfigManager.Instance.getSocketDirectory()) + + const socketPath = ConfigManager.Instance.getSocketPath() + this.netIPC = new NetIPC({ path: socketPath }) + await this.netIPC.start() + + logger.info(`IPC socket created on ${socketPath}`) + + this.netIPC.on('request', async (req: IPCRequest, res) => { + try { + const data = await this.process(req) + + this.sendReponse(res, { success: true, data }) + } catch (err) { + logger.error('Cannot execute RPC call', err) + this.sendReponse(res, { success: false, error: err.message }) + } + }) + } + + private async process (req: IPCRequest) { + switch (req.type) { + case 'register': + await this.runnerServer.registerRunner(pick(req, [ 'url', 'registrationToken', 'runnerName', 'runnerDescription' ])) + return undefined + + case 'unregister': + await this.runnerServer.unregisterRunner(pick(req, [ 'url', 'runnerName' ])) + return undefined + + case 'list-registered': + return Promise.resolve(this.runnerServer.listRegistered()) + + default: + throw new Error('Unknown RPC call ' + (req as any).type) + } + } + + private sendReponse ( + response: (data: any) => Promise, + body: IPCReponse + ) { + response(body) + .catch(err => logger.error('Cannot send response after IPC request', err)) + } +} diff --git a/apps/peertube-runner/src/shared/ipc/shared/index.ts b/apps/peertube-runner/src/shared/ipc/shared/index.ts new file mode 100644 index 000000000..986acafb0 --- /dev/null +++ b/apps/peertube-runner/src/shared/ipc/shared/index.ts @@ -0,0 +1,2 @@ +export * from './ipc-request.model.js' +export * from './ipc-response.model.js' diff --git a/packages/peertube-runner/shared/ipc/shared/ipc-request.model.ts b/apps/peertube-runner/src/shared/ipc/shared/ipc-request.model.ts similarity index 100% rename from packages/peertube-runner/shared/ipc/shared/ipc-request.model.ts rename to apps/peertube-runner/src/shared/ipc/shared/ipc-request.model.ts diff --git a/packages/peertube-runner/shared/ipc/shared/ipc-response.model.ts b/apps/peertube-runner/src/shared/ipc/shared/ipc-response.model.ts similarity index 100% rename from packages/peertube-runner/shared/ipc/shared/ipc-response.model.ts rename to apps/peertube-runner/src/shared/ipc/shared/ipc-response.model.ts diff --git a/apps/peertube-runner/src/shared/logger.ts b/apps/peertube-runner/src/shared/logger.ts new file mode 100644 index 000000000..ef5283892 --- /dev/null +++ b/apps/peertube-runner/src/shared/logger.ts @@ -0,0 +1,12 @@ +import { pino } from 'pino' +import pretty from 'pino-pretty' + +const logger = pino(pretty.default({ + colorize: true +})) + +logger.level = 'info' + +export { + logger +} diff --git a/apps/peertube-runner/tsconfig.json b/apps/peertube-runner/tsconfig.json new file mode 100644 index 000000000..03660b0eb --- /dev/null +++ b/apps/peertube-runner/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "baseUrl": "./", + "outDir": "./dist", + "rootDir": "src", + "tsBuildInfoFile": "./dist/.tsbuildinfo" + }, + "references": [ + { "path": "../../packages/core-utils" }, + { "path": "../../packages/ffmpeg" }, + { "path": "../../packages/models" }, + { "path": "../../packages/node-utils" }, + { "path": "../../packages/server-commands" } + ] +} diff --git a/packages/peertube-runner/yarn.lock b/apps/peertube-runner/yarn.lock similarity index 100% rename from packages/peertube-runner/yarn.lock rename to apps/peertube-runner/yarn.lock diff --git a/client/.eslintrc.json b/client/.eslintrc.json index c5685b9dc..e4c8d901b 100644 --- a/client/.eslintrc.json +++ b/client/.eslintrc.json @@ -14,6 +14,7 @@ "project": [ "tsconfig.eslint.json" ], + "EXPERIMENTAL_useSourceOfProjectReferenceRedirect": true, "createDefaultProgram": false }, "extends": [ diff --git a/client/e2e/wdio.main.conf.ts b/client/e2e/wdio.main.conf.ts index 5ae7fa18b..8e6ef51cf 100644 --- a/client/e2e/wdio.main.conf.ts +++ b/client/e2e/wdio.main.conf.ts @@ -107,14 +107,6 @@ export const config = { tsNodeOpts: { project: require('path').join(__dirname, './tsconfig.json') - }, - - tsConfigPathsOpts: { - baseUrl: './', - paths: { - '@server/*': [ '../../server/*' ], - '@shared/*': [ '../../shared/*' ] - } } }, diff --git a/client/package.json b/client/package.json index 149322192..9c311622b 100644 --- a/client/package.json +++ b/client/package.json @@ -14,7 +14,7 @@ }, "scripts": { "lint": "npm run lint-ts && npm run lint-scss", - "lint-ts": "eslint --ext .ts src/standalone/**/*.ts && npm run ng lint", + "lint-ts": "eslint --cache --ext .ts src/standalone/**/*.ts && npm run ng lint", "lint-scss": "stylelint 'src/**/*.scss'", "webpack": "webpack", "eslint": "eslint", @@ -24,6 +24,9 @@ "ngx-extractor": "ngx-extractor", "stylelint": "stylelint" }, + "workspaces": [ + "../packages/*" + ], "typings": "*.d.ts", "devDependencies": { "@angular-devkit/build-angular": "^16.0.2", @@ -57,6 +60,8 @@ "@peertube/maildev": "^1.2.0", "@peertube/p2p-media-loader-core": "^1.0.14", "@peertube/p2p-media-loader-hlsjs": "^1.0.14", + "@peertube/peertube-core-utils": "*", + "@peertube/peertube-models": "*", "@peertube/videojs-contextmenu": "^5.5.0", "@peertube/xliffmerge": "^2.0.3", "@popperjs/core": "^2.11.5", @@ -86,7 +91,7 @@ "buffer": "^6.0.3", "chart.js": "^4.3.0", "chartjs-plugin-zoom": "~2.0.1", - "chromedriver": "^113.0.0", + "chromedriver": "^115.0.1", "core-js": "^3.22.8", "css-loader": "^6.2.0", "debug": "^4.3.1", @@ -122,6 +127,7 @@ "stylelint": "^15.1.0", "stylelint-config-sass-guidelines": "^10.0.0", "ts-loader": "^9.3.0", + "ts-node": "^10.9.1", "tslib": "^2.4.0", "typescript": "~4.9.5", "video.js": "^7.19.2", diff --git a/client/src/app/+about/about-follows/about-follows.component.ts b/client/src/app/+about/about-follows/about-follows.component.ts index e1df8b813..a542cdbf1 100644 --- a/client/src/app/+about/about-follows/about-follows.component.ts +++ b/client/src/app/+about/about-follows/about-follows.component.ts @@ -2,7 +2,7 @@ import { SortMeta } from 'primeng/api' import { Component, OnInit } from '@angular/core' import { ComponentPagination, hasMoreItems, Notifier, RestService, ServerService } from '@app/core' import { InstanceFollowService } from '@app/shared/shared-instance' -import { Actor } from '@shared/models/actors' +import { Actor } from '@peertube/peertube-models' @Component({ selector: 'my-about-follows', diff --git a/client/src/app/+about/about-instance/about-instance.component.ts b/client/src/app/+about/about-instance/about-instance.component.ts index fc5214215..85e973d7b 100644 --- a/client/src/app/+about/about-instance/about-instance.component.ts +++ b/client/src/app/+about/about-instance/about-instance.component.ts @@ -3,8 +3,8 @@ import { AfterViewChecked, Component, ElementRef, OnInit, ViewChild } from '@ang import { ActivatedRoute } from '@angular/router' import { Notifier, ServerService } from '@app/core' import { AboutHTML } from '@app/shared/shared-instance' +import { HTMLServerConfig, ServerStats } from '@peertube/peertube-models' import { copyToClipboard } from '@root-helpers/utils' -import { HTMLServerConfig, ServerStats } from '@shared/models/server' import { ResolverData } from './about-instance.resolver' import { ContactAdminModalComponent } from './contact-admin-modal.component' diff --git a/client/src/app/+about/about-instance/about-instance.resolver.ts b/client/src/app/+about/about-instance/about-instance.resolver.ts index f52a95b88..b5e8ccaa2 100644 --- a/client/src/app/+about/about-instance/about-instance.resolver.ts +++ b/client/src/app/+about/about-instance/about-instance.resolver.ts @@ -4,7 +4,7 @@ import { Injectable } from '@angular/core' import { ServerService } from '@app/core' import { CustomMarkupService } from '@app/shared/shared-custom-markup' import { AboutHTML, InstanceService } from '@app/shared/shared-instance' -import { About, ServerStats } from '@shared/models/server' +import { About, ServerStats } from '@peertube/peertube-models' export type ResolverData = { serverStats: ServerStats diff --git a/client/src/app/+about/about-instance/contact-admin-modal.component.ts b/client/src/app/+about/about-instance/contact-admin-modal.component.ts index 0e2bf51e8..38e577fcd 100644 --- a/client/src/app/+about/about-instance/contact-admin-modal.component.ts +++ b/client/src/app/+about/about-instance/contact-admin-modal.component.ts @@ -11,7 +11,7 @@ import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' import { InstanceService } from '@app/shared/shared-instance' import { NgbModal } from '@ng-bootstrap/ng-bootstrap' import { NgbModalRef } from '@ng-bootstrap/ng-bootstrap/modal/modal-ref' -import { HTMLServerConfig, HttpStatusCode } from '@shared/models' +import { HTMLServerConfig, HttpStatusCode } from '@peertube/peertube-models' type Prefill = { subject?: string diff --git a/client/src/app/+about/about-instance/instance-statistics.component.ts b/client/src/app/+about/about-instance/instance-statistics.component.ts index ac6984438..9eb56d0a4 100644 --- a/client/src/app/+about/about-instance/instance-statistics.component.ts +++ b/client/src/app/+about/about-instance/instance-statistics.component.ts @@ -1,5 +1,5 @@ import { Component, Input } from '@angular/core' -import { ServerStats } from '@shared/models/server' +import { ServerStats } from '@peertube/peertube-models' @Component({ selector: 'my-instance-statistics', diff --git a/client/src/app/+accounts/account-video-channels/account-video-channels.component.ts b/client/src/app/+accounts/account-video-channels/account-video-channels.component.ts index 2ee168492..b8afa66ff 100644 --- a/client/src/app/+accounts/account-video-channels/account-video-channels.component.ts +++ b/client/src/app/+accounts/account-video-channels/account-video-channels.component.ts @@ -5,7 +5,7 @@ import { ComponentPagination, hasMoreItems, MarkdownService, User, UserService } import { SimpleMemoize } from '@app/helpers' import { Account, AccountService, Video, VideoChannel, VideoChannelService, VideoService } from '@app/shared/shared-main' import { MiniatureDisplayOptions } from '@app/shared/shared-video-miniature' -import { NSFWPolicyType, VideoSortField } from '@shared/models' +import { NSFWPolicyType, VideoSortField } from '@peertube/peertube-models' @Component({ selector: 'my-account-video-channels', diff --git a/client/src/app/+accounts/account-videos/account-videos.component.ts b/client/src/app/+accounts/account-videos/account-videos.component.ts index 13d1f857d..d8e8377e1 100644 --- a/client/src/app/+accounts/account-videos/account-videos.component.ts +++ b/client/src/app/+accounts/account-videos/account-videos.component.ts @@ -4,7 +4,7 @@ import { Component, OnDestroy, OnInit } from '@angular/core' import { ComponentPaginationLight, DisableForReuseHook, ScreenService } from '@app/core' import { Account, AccountService, VideoService } from '@app/shared/shared-main' import { VideoFilters } from '@app/shared/shared-video-miniature' -import { VideoSortField } from '@shared/models' +import { VideoSortField } from '@peertube/peertube-models' @Component({ selector: 'my-account-videos', diff --git a/client/src/app/+accounts/accounts.component.ts b/client/src/app/+accounts/accounts.component.ts index 6d912e325..156f35804 100644 --- a/client/src/app/+accounts/accounts.component.ts +++ b/client/src/app/+accounts/accounts.component.ts @@ -13,7 +13,7 @@ import { VideoService } from '@app/shared/shared-main' import { AccountReportComponent, BlocklistService } from '@app/shared/shared-moderation' -import { HttpStatusCode, User, UserRight } from '@shared/models' +import { HttpStatusCode, User, UserRight } from '@peertube/peertube-models' @Component({ templateUrl: './accounts.component.html', diff --git a/client/src/app/+admin/admin.component.ts b/client/src/app/+admin/admin.component.ts index 49092ea2a..c0d7db99e 100644 --- a/client/src/app/+admin/admin.component.ts +++ b/client/src/app/+admin/admin.component.ts @@ -2,7 +2,7 @@ import { Component, OnInit } from '@angular/core' import { AuthService, ScreenService, ServerService } from '@app/core' import { ListOverflowItem } from '@app/shared/shared-main' import { TopMenuDropdownParam } from '@app/shared/shared-main/misc/top-menu-dropdown.component' -import { UserRight } from '@shared/models' +import { UserRight } from '@peertube/peertube-models' @Component({ templateUrl: './admin.component.html', diff --git a/client/src/app/+admin/config/config.routes.ts b/client/src/app/+admin/config/config.routes.ts index 6d255ac46..96a4f3771 100644 --- a/client/src/app/+admin/config/config.routes.ts +++ b/client/src/app/+admin/config/config.routes.ts @@ -1,7 +1,7 @@ import { Routes } from '@angular/router' import { EditCustomConfigComponent } from '@app/+admin/config/edit-custom-config' import { UserRightGuard } from '@app/core' -import { UserRight } from '@shared/models' +import { UserRight } from '@peertube/peertube-models' export const ConfigRoutes: Routes = [ { diff --git a/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.ts index 2122e67b2..953c7d540 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.ts +++ b/client/src/app/+admin/config/edit-custom-config/edit-basic-configuration.component.ts @@ -3,7 +3,7 @@ import { SelectOptionsItem } from 'src/types/select-options-item.model' import { Component, Input, OnChanges, OnInit, SimpleChanges } from '@angular/core' import { FormGroup } from '@angular/forms' import { MenuService, ThemeService } from '@app/core' -import { HTMLServerConfig } from '@shared/models' +import { HTMLServerConfig } from '@peertube/peertube-models' import { ConfigService } from '../shared/config.service' @Component({ diff --git a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts index c3b85b196..54c076b74 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts +++ b/client/src/app/+admin/config/edit-custom-config/edit-custom-config.component.ts @@ -27,7 +27,7 @@ import { import { USER_VIDEO_QUOTA_DAILY_VALIDATOR, USER_VIDEO_QUOTA_VALIDATOR } from '@app/shared/form-validators/user-validators' import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' import { CustomPageService } from '@app/shared/shared-main/custom-page' -import { CustomConfig, CustomPage, HTMLServerConfig } from '@shared/models' +import { CustomConfig, CustomPage, HTMLServerConfig } from '@peertube/peertube-models' import { EditConfigurationService } from './edit-configuration.service' type ComponentCustomConfig = CustomConfig & { diff --git a/client/src/app/+admin/config/edit-custom-config/edit-live-configuration.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-live-configuration.component.ts index 1d1fecf90..59629aa20 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-live-configuration.component.ts +++ b/client/src/app/+admin/config/edit-custom-config/edit-live-configuration.component.ts @@ -2,7 +2,7 @@ import { SelectOptionsItem } from 'src/types/select-options-item.model' import { Component, Input, OnChanges, OnInit, SimpleChanges } from '@angular/core' import { FormGroup } from '@angular/forms' -import { HTMLServerConfig } from '@shared/models' +import { HTMLServerConfig } from '@peertube/peertube-models' import { ConfigService } from '../shared/config.service' import { EditConfigurationService, ResolutionOption } from './edit-configuration.service' diff --git a/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.ts b/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.ts index 6496e8753..a2cd04396 100644 --- a/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.ts +++ b/client/src/app/+admin/config/edit-custom-config/edit-vod-transcoding.component.ts @@ -2,7 +2,7 @@ import { SelectOptionsItem } from 'src/types/select-options-item.model' import { Component, Input, OnChanges, OnInit, SimpleChanges } from '@angular/core' import { FormGroup } from '@angular/forms' -import { HTMLServerConfig } from '@shared/models' +import { HTMLServerConfig } from '@peertube/peertube-models' import { ConfigService } from '../shared/config.service' import { EditConfigurationService, ResolutionOption } from './edit-configuration.service' diff --git a/client/src/app/+admin/config/shared/config.service.ts b/client/src/app/+admin/config/shared/config.service.ts index 80f495b41..3c3894945 100644 --- a/client/src/app/+admin/config/shared/config.service.ts +++ b/client/src/app/+admin/config/shared/config.service.ts @@ -2,7 +2,7 @@ import { catchError } from 'rxjs/operators' import { HttpClient } from '@angular/common/http' import { Injectable } from '@angular/core' import { RestExtractor } from '@app/core' -import { CustomConfig } from '@shared/models' +import { CustomConfig } from '@peertube/peertube-models' import { SelectOptionsItem } from '../../../../types/select-options-item.model' import { environment } from '../../../../environments/environment' diff --git a/client/src/app/+admin/follows/followers-list/followers-list.component.ts b/client/src/app/+admin/follows/followers-list/followers-list.component.ts index 618892242..656a7bf87 100644 --- a/client/src/app/+admin/follows/followers-list/followers-list.component.ts +++ b/client/src/app/+admin/follows/followers-list/followers-list.component.ts @@ -5,7 +5,7 @@ import { formatICU } from '@app/helpers' import { AdvancedInputFilter } from '@app/shared/shared-forms' import { InstanceFollowService } from '@app/shared/shared-instance' import { DropdownAction } from '@app/shared/shared-main' -import { ActorFollow } from '@shared/models' +import { ActorFollow } from '@peertube/peertube-models' @Component({ selector: 'my-followers-list', diff --git a/client/src/app/+admin/follows/following-list/following-list.component.ts b/client/src/app/+admin/follows/following-list/following-list.component.ts index 6c8723c16..da6647f6b 100644 --- a/client/src/app/+admin/follows/following-list/following-list.component.ts +++ b/client/src/app/+admin/follows/following-list/following-list.component.ts @@ -3,7 +3,7 @@ import { Component, OnInit, ViewChild } from '@angular/core' import { ConfirmService, Notifier, RestPagination, RestTable } from '@app/core' import { AdvancedInputFilter } from '@app/shared/shared-forms' import { InstanceFollowService } from '@app/shared/shared-instance' -import { ActorFollow } from '@shared/models' +import { ActorFollow } from '@peertube/peertube-models' import { FollowModalComponent } from './follow-modal.component' import { DropdownAction } from '@app/shared/shared-main' import { formatICU } from '@app/helpers' diff --git a/client/src/app/+admin/follows/follows.routes.ts b/client/src/app/+admin/follows/follows.routes.ts index 718493dc7..e187f83ea 100644 --- a/client/src/app/+admin/follows/follows.routes.ts +++ b/client/src/app/+admin/follows/follows.routes.ts @@ -1,7 +1,7 @@ import { Routes } from '@angular/router' import { VideoRedundanciesListComponent } from '@app/+admin/follows/video-redundancies-list' import { UserRightGuard } from '@app/core' -import { UserRight } from '@shared/models' +import { UserRight } from '@peertube/peertube-models' import { FollowersListComponent } from './followers-list' import { FollowingListComponent } from './following-list/following-list.component' diff --git a/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.ts b/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.ts index efcefd509..09fc038ce 100644 --- a/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.ts +++ b/client/src/app/+admin/follows/video-redundancies-list/video-redundancies-list.component.ts @@ -3,9 +3,8 @@ import { SortMeta } from 'primeng/api' import { Component, OnInit } from '@angular/core' import { ConfirmService, Notifier, RestPagination, RestTable, ServerService } from '@app/core' import { BytesPipe, RedundancyService } from '@app/shared/shared-main' +import { VideoRedundanciesTarget, VideoRedundancy, VideosRedundancyStats } from '@peertube/peertube-models' import { peertubeLocalStorage } from '@root-helpers/peertube-web-storage' -import { VideoRedundanciesTarget, VideoRedundancy } from '@shared/models' -import { VideosRedundancyStats } from '@shared/models/server' @Component({ selector: 'my-video-redundancies-list', diff --git a/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.ts b/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.ts index 6f3090c08..779d19059 100644 --- a/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.ts +++ b/client/src/app/+admin/follows/video-redundancies-list/video-redundancy-information.component.ts @@ -1,5 +1,5 @@ import { Component, Input } from '@angular/core' -import { FileRedundancyInformation, StreamingPlaylistRedundancyInformation } from '@shared/models' +import { FileRedundancyInformation, StreamingPlaylistRedundancyInformation } from '@peertube/peertube-models' @Component({ selector: 'my-video-redundancy-information', diff --git a/client/src/app/+admin/moderation/moderation.routes.ts b/client/src/app/+admin/moderation/moderation.routes.ts index 378d2bed7..f0494de7b 100644 --- a/client/src/app/+admin/moderation/moderation.routes.ts +++ b/client/src/app/+admin/moderation/moderation.routes.ts @@ -3,7 +3,7 @@ import { AbuseListComponent } from '@app/+admin/moderation/abuse-list' import { InstanceAccountBlocklistComponent, InstanceServerBlocklistComponent } from '@app/+admin/moderation/instance-blocklist' import { VideoBlockListComponent } from '@app/+admin/moderation/video-block-list' import { UserRightGuard } from '@app/core' -import { UserRight } from '@shared/models' +import { UserRight } from '@peertube/peertube-models' import { RegistrationListComponent } from './registration-list' export const ModerationRoutes: Routes = [ diff --git a/client/src/app/+admin/moderation/registration-list/admin-registration.service.ts b/client/src/app/+admin/moderation/registration-list/admin-registration.service.ts index a9f13cf2f..f8ab04c71 100644 --- a/client/src/app/+admin/moderation/registration-list/admin-registration.service.ts +++ b/client/src/app/+admin/moderation/registration-list/admin-registration.service.ts @@ -4,8 +4,8 @@ import { catchError, concatMap, toArray } from 'rxjs/operators' import { HttpClient, HttpParams } from '@angular/common/http' import { Injectable } from '@angular/core' import { RestExtractor, RestPagination, RestService } from '@app/core' -import { arrayify } from '@shared/core-utils' -import { ResultList, UserRegistration, UserRegistrationUpdateState } from '@shared/models' +import { arrayify } from '@peertube/peertube-core-utils' +import { ResultList, UserRegistration, UserRegistrationUpdateState } from '@peertube/peertube-models' import { environment } from '../../../../environments/environment' @Injectable() diff --git a/client/src/app/+admin/moderation/registration-list/process-registration-modal.component.ts b/client/src/app/+admin/moderation/registration-list/process-registration-modal.component.ts index 8f013cbd5..f8e346f50 100644 --- a/client/src/app/+admin/moderation/registration-list/process-registration-modal.component.ts +++ b/client/src/app/+admin/moderation/registration-list/process-registration-modal.component.ts @@ -3,7 +3,7 @@ import { Notifier, ServerService } from '@app/core' import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' import { NgbModal } from '@ng-bootstrap/ng-bootstrap' import { NgbModalRef } from '@ng-bootstrap/ng-bootstrap/modal/modal-ref' -import { UserRegistration } from '@shared/models' +import { UserRegistration } from '@peertube/peertube-models' import { AdminRegistrationService } from './admin-registration.service' import { REGISTRATION_MODERATION_RESPONSE_VALIDATOR } from './process-registration-validators' diff --git a/client/src/app/+admin/moderation/registration-list/registration-list.component.ts b/client/src/app/+admin/moderation/registration-list/registration-list.component.ts index 35d9d13d7..1dc5e9077 100644 --- a/client/src/app/+admin/moderation/registration-list/registration-list.component.ts +++ b/client/src/app/+admin/moderation/registration-list/registration-list.component.ts @@ -5,7 +5,7 @@ import { ConfirmService, MarkdownService, Notifier, RestPagination, RestTable, S import { formatICU } from '@app/helpers' import { AdvancedInputFilter } from '@app/shared/shared-forms' import { DropdownAction } from '@app/shared/shared-main' -import { UserRegistration, UserRegistrationState } from '@shared/models' +import { UserRegistration, UserRegistrationState } from '@peertube/peertube-models' import { AdminRegistrationService } from './admin-registration.service' import { ProcessRegistrationModalComponent } from './process-registration-modal.component' diff --git a/client/src/app/+admin/moderation/video-block-list/video-block-list.component.ts b/client/src/app/+admin/moderation/video-block-list/video-block-list.component.ts index f365a2500..3c6bda16c 100644 --- a/client/src/app/+admin/moderation/video-block-list/video-block-list.component.ts +++ b/client/src/app/+admin/moderation/video-block-list/video-block-list.component.ts @@ -7,9 +7,9 @@ import { ConfirmService, MarkdownService, Notifier, RestPagination, RestTable, S import { AdvancedInputFilter } from '@app/shared/shared-forms' import { DropdownAction, VideoService } from '@app/shared/shared-main' import { VideoBlockService } from '@app/shared/shared-moderation' +import { buildVideoEmbedLink, decorateVideoLink } from '@peertube/peertube-core-utils' +import { VideoBlacklist, VideoBlacklistType, VideoBlacklistType_Type } from '@peertube/peertube-models' import { buildVideoOrPlaylistEmbed } from '@root-helpers/video' -import { buildVideoEmbedLink, decorateVideoLink } from '@shared/core-utils' -import { VideoBlacklist, VideoBlacklistType } from '@shared/models' @Component({ selector: 'my-video-block-list', @@ -21,7 +21,7 @@ export class VideoBlockListComponent extends RestTable implements OnInit { totalRecords = 0 sort: SortMeta = { field: 'createdAt', order: -1 } pagination: RestPagination = { count: this.rowsPerPage, start: 0 } - blocklistTypeFilter: VideoBlacklistType = undefined + blocklistTypeFilter: VideoBlacklistType_Type videoBlocklistActions: DropdownAction[][] = [] diff --git a/client/src/app/+admin/overview/comments/video-comment-list.component.ts b/client/src/app/+admin/overview/comments/video-comment-list.component.ts index b77072665..254e76a60 100644 --- a/client/src/app/+admin/overview/comments/video-comment-list.component.ts +++ b/client/src/app/+admin/overview/comments/video-comment-list.component.ts @@ -6,7 +6,7 @@ import { AdvancedInputFilter } from '@app/shared/shared-forms' import { DropdownAction } from '@app/shared/shared-main' import { BulkService } from '@app/shared/shared-moderation' import { VideoCommentAdmin, VideoCommentService } from '@app/shared/shared-video-comment' -import { FeedFormat, UserRight } from '@shared/models' +import { FeedFormat, UserRight } from '@peertube/peertube-models' import { formatICU } from '@app/helpers' @Component({ diff --git a/client/src/app/+admin/overview/comments/video-comment.routes.ts b/client/src/app/+admin/overview/comments/video-comment.routes.ts index f0bd440ad..f67027430 100644 --- a/client/src/app/+admin/overview/comments/video-comment.routes.ts +++ b/client/src/app/+admin/overview/comments/video-comment.routes.ts @@ -1,6 +1,6 @@ import { Routes } from '@angular/router' import { UserRightGuard } from '@app/core' -import { UserRight } from '@shared/models' +import { UserRight } from '@peertube/peertube-models' import { VideoCommentListComponent } from './video-comment-list.component' export const commentRoutes: Routes = [ diff --git a/client/src/app/+admin/overview/users/user-edit/user-create.component.ts b/client/src/app/+admin/overview/users/user-edit/user-create.component.ts index 0627aa887..77acb9988 100644 --- a/client/src/app/+admin/overview/users/user-edit/user-create.component.ts +++ b/client/src/app/+admin/overview/users/user-edit/user-create.component.ts @@ -14,7 +14,7 @@ import { } from '@app/shared/form-validators/user-validators' import { FormReactiveService } from '@app/shared/shared-forms' import { UserAdminService } from '@app/shared/shared-users' -import { UserCreate, UserRole } from '@shared/models' +import { UserCreate, UserRole } from '@peertube/peertube-models' import { UserEdit } from './user-edit' @Component({ diff --git a/client/src/app/+admin/overview/users/user-edit/user-edit.ts b/client/src/app/+admin/overview/users/user-edit/user-edit.ts index 9547da2d1..d61b7b068 100644 --- a/client/src/app/+admin/overview/users/user-edit/user-edit.ts +++ b/client/src/app/+admin/overview/users/user-edit/user-edit.ts @@ -2,9 +2,8 @@ import { Directive, OnInit } from '@angular/core' import { ConfigService } from '@app/+admin/config/shared/config.service' import { AuthService, ScreenService, ServerService, User } from '@app/core' import { FormReactive } from '@app/shared/shared-forms' -import { peertubeTranslate } from '@shared/core-utils' -import { USER_ROLE_LABELS } from '@shared/core-utils/users' -import { HTMLServerConfig, UserAdminFlag, UserRole } from '@shared/models' +import { peertubeTranslate, USER_ROLE_LABELS } from '@peertube/peertube-core-utils' +import { HTMLServerConfig, UserAdminFlag, UserRole } from '@peertube/peertube-models' import { SelectOptionsItem } from '../../../../../types/select-options-item.model' @Directive() diff --git a/client/src/app/+admin/overview/users/user-edit/user-password.component.ts b/client/src/app/+admin/overview/users/user-edit/user-password.component.ts index ec93619f5..af39c82af 100644 --- a/client/src/app/+admin/overview/users/user-edit/user-password.component.ts +++ b/client/src/app/+admin/overview/users/user-edit/user-password.component.ts @@ -3,7 +3,7 @@ import { Notifier } from '@app/core' import { USER_PASSWORD_VALIDATOR } from '@app/shared/form-validators/user-validators' import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' import { UserAdminService } from '@app/shared/shared-users' -import { UserUpdate } from '@shared/models' +import { UserUpdate } from '@peertube/peertube-models' @Component({ selector: 'my-user-password', diff --git a/client/src/app/+admin/overview/users/user-edit/user-update.component.ts b/client/src/app/+admin/overview/users/user-edit/user-update.component.ts index 25d02f000..b55a519f3 100644 --- a/client/src/app/+admin/overview/users/user-edit/user-update.component.ts +++ b/client/src/app/+admin/overview/users/user-edit/user-update.component.ts @@ -11,7 +11,7 @@ import { } from '@app/shared/form-validators/user-validators' import { FormReactiveService } from '@app/shared/shared-forms' import { TwoFactorService, UserAdminService } from '@app/shared/shared-users' -import { User as UserType, UserAdminFlag, UserRole, UserUpdate } from '@shared/models' +import { User as UserType, UserAdminFlag, UserRole, UserUpdate } from '@peertube/peertube-models' import { UserEdit } from './user-edit' @Component({ diff --git a/client/src/app/+admin/overview/users/user-list/user-list.component.ts b/client/src/app/+admin/overview/users/user-list/user-list.component.ts index 5d5abf6f4..a5a1552da 100644 --- a/client/src/app/+admin/overview/users/user-list/user-list.component.ts +++ b/client/src/app/+admin/overview/users/user-list/user-list.component.ts @@ -7,8 +7,8 @@ import { AdvancedInputFilter } from '@app/shared/shared-forms' import { Actor, DropdownAction } from '@app/shared/shared-main' import { AccountMutedStatus, BlocklistService, UserBanModalComponent, UserModerationDisplayType } from '@app/shared/shared-moderation' import { UserAdminService } from '@app/shared/shared-users' +import { User, UserRole, UserRoleType } from '@peertube/peertube-models' import { logger } from '@root-helpers/logger' -import { User, UserRole } from '@shared/models' type UserForList = User & { rawVideoQuota: number @@ -166,7 +166,7 @@ export class UserListComponent extends RestTable implements OnInit { return 'UserListComponent' } - getRoleClass (role: UserRole) { + getRoleClass (role: UserRoleType) { switch (role) { case UserRole.ADMINISTRATOR: return 'badge-purple' diff --git a/client/src/app/+admin/overview/users/users.routes.ts b/client/src/app/+admin/overview/users/users.routes.ts index c9724e5fb..d66b8c762 100644 --- a/client/src/app/+admin/overview/users/users.routes.ts +++ b/client/src/app/+admin/overview/users/users.routes.ts @@ -1,6 +1,6 @@ import { Routes } from '@angular/router' import { UserRightGuard } from '@app/core' -import { UserRight } from '@shared/models' +import { UserRight } from '@peertube/peertube-models' import { UserCreateComponent, UserUpdateComponent } from './user-edit' import { UserListComponent } from './user-list' diff --git a/client/src/app/+admin/overview/videos/video-admin.service.ts b/client/src/app/+admin/overview/videos/video-admin.service.ts index 722495706..9b33ed8e5 100644 --- a/client/src/app/+admin/overview/videos/video-admin.service.ts +++ b/client/src/app/+admin/overview/videos/video-admin.service.ts @@ -5,8 +5,8 @@ import { Injectable } from '@angular/core' import { RestExtractor, RestPagination, RestService } from '@app/core' import { AdvancedInputFilter } from '@app/shared/shared-forms' import { CommonVideoParams, Video, VideoService } from '@app/shared/shared-main' -import { ResultList, VideoInclude, VideoPrivacy } from '@shared/models' -import { getAllPrivacies } from '@shared/core-utils' +import { ResultList, VideoInclude, VideoPrivacy } from '@peertube/peertube-models' +import { getAllPrivacies } from '@peertube/peertube-core-utils' @Injectable() export class VideoAdminService { diff --git a/client/src/app/+admin/overview/videos/video-list.component.ts b/client/src/app/+admin/overview/videos/video-list.component.ts index 2792a2d8a..2e12a2b31 100644 --- a/client/src/app/+admin/overview/videos/video-list.component.ts +++ b/client/src/app/+admin/overview/videos/video-list.component.ts @@ -8,8 +8,8 @@ import { AdvancedInputFilter } from '@app/shared/shared-forms' import { DropdownAction, Video, VideoService } from '@app/shared/shared-main' import { VideoBlockComponent, VideoBlockService } from '@app/shared/shared-moderation' import { VideoActionsDisplayType } from '@app/shared/shared-video-miniature' -import { getAllFiles } from '@shared/core-utils' -import { UserRight, VideoFile, VideoPrivacy, VideoState, VideoStreamingPlaylistType } from '@shared/models' +import { getAllFiles } from '@peertube/peertube-core-utils' +import { UserRight, VideoFile, VideoPrivacy, VideoState, VideoStreamingPlaylistType } from '@peertube/peertube-models' import { VideoAdminService } from './video-admin.service' @Component({ diff --git a/client/src/app/+admin/overview/videos/video.routes.ts b/client/src/app/+admin/overview/videos/video.routes.ts index 01cb5b497..dfffd2696 100644 --- a/client/src/app/+admin/overview/videos/video.routes.ts +++ b/client/src/app/+admin/overview/videos/video.routes.ts @@ -1,6 +1,6 @@ import { Routes } from '@angular/router' import { UserRightGuard } from '@app/core' -import { UserRight } from '@shared/models' +import { UserRight } from '@peertube/peertube-models' import { VideoListComponent } from './video-list.component' export const videosRoutes: Routes = [ diff --git a/client/src/app/+admin/plugins/plugin-list-installed/plugin-list-installed.component.ts b/client/src/app/+admin/plugins/plugin-list-installed/plugin-list-installed.component.ts index 3fa1c56dc..1b78a00cd 100644 --- a/client/src/app/+admin/plugins/plugin-list-installed/plugin-list-installed.component.ts +++ b/client/src/app/+admin/plugins/plugin-list-installed/plugin-list-installed.component.ts @@ -4,8 +4,8 @@ import { ActivatedRoute, Router } from '@angular/router' import { PluginApiService } from '@app/+admin/plugins/shared/plugin-api.service' import { ComponentPagination, ConfirmService, hasMoreItems, Notifier } from '@app/core' import { PluginService } from '@app/core/plugins/plugin.service' -import { compareSemVer } from '@shared/core-utils' -import { PeerTubePlugin, PluginType } from '@shared/models' +import { compareSemVer } from '@peertube/peertube-core-utils' +import { PeerTubePlugin, PluginType, PluginType_Type } from '@peertube/peertube-models' @Component({ selector: 'my-plugin-list-installed', @@ -13,7 +13,7 @@ import { PeerTubePlugin, PluginType } from '@shared/models' styleUrls: [ './plugin-list-installed.component.scss' ] }) export class PluginListInstalledComponent implements OnInit { - pluginType: PluginType + pluginType: PluginType_Type pagination: ComponentPagination = { currentPage: 1, @@ -48,7 +48,7 @@ export class PluginListInstalledComponent implements OnInit { this.route.queryParams.subscribe(query => { if (!query['pluginType']) return - this.pluginType = parseInt(query['pluginType'], 10) + this.pluginType = parseInt(query['pluginType'], 10) as PluginType_Type this.reloadPlugins() }) diff --git a/client/src/app/+admin/plugins/plugin-search/plugin-search.component.ts b/client/src/app/+admin/plugins/plugin-search/plugin-search.component.ts index c03e37aa5..5539d1c13 100644 --- a/client/src/app/+admin/plugins/plugin-search/plugin-search.component.ts +++ b/client/src/app/+admin/plugins/plugin-search/plugin-search.component.ts @@ -4,8 +4,8 @@ import { Component, OnInit } from '@angular/core' import { ActivatedRoute, Router } from '@angular/router' import { PluginApiService } from '@app/+admin/plugins/shared/plugin-api.service' import { ComponentPagination, ConfirmService, hasMoreItems, Notifier, PluginService } from '@app/core' +import { PeerTubePluginIndex, PluginType, PluginType_Type } from '@peertube/peertube-models' import { logger } from '@root-helpers/logger' -import { PeerTubePluginIndex, PluginType } from '@shared/models' @Component({ selector: 'my-plugin-search', @@ -13,7 +13,7 @@ import { PeerTubePluginIndex, PluginType } from '@shared/models' styleUrls: [ './plugin-search.component.scss' ] }) export class PluginSearchComponent implements OnInit { - pluginType: PluginType + pluginType: PluginType_Type pagination: ComponentPagination = { currentPage: 1, @@ -53,7 +53,7 @@ export class PluginSearchComponent implements OnInit { this.route.queryParams.subscribe(query => { if (!query['pluginType']) return - this.pluginType = parseInt(query['pluginType'], 10) + this.pluginType = parseInt(query['pluginType'], 10) as PluginType_Type this.search = query['search'] || '' this.reloadPlugins() diff --git a/client/src/app/+admin/plugins/plugin-show-installed/plugin-show-installed.component.ts b/client/src/app/+admin/plugins/plugin-show-installed/plugin-show-installed.component.ts index b1a41567e..9eee1a901 100644 --- a/client/src/app/+admin/plugins/plugin-show-installed/plugin-show-installed.component.ts +++ b/client/src/app/+admin/plugins/plugin-show-installed/plugin-show-installed.component.ts @@ -5,7 +5,7 @@ import { ActivatedRoute } from '@angular/router' import { HooksService, Notifier, PluginService } from '@app/core' import { BuildFormArgument } from '@app/shared/form-validators' import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' -import { PeerTubePlugin, RegisterServerSettingOptions } from '@shared/models' +import { PeerTubePlugin, RegisterServerSettingOptions } from '@peertube/peertube-models' import { PluginApiService } from '../shared/plugin-api.service' @Component({ diff --git a/client/src/app/+admin/plugins/plugins.routes.ts b/client/src/app/+admin/plugins/plugins.routes.ts index f735a490b..40660f1f4 100644 --- a/client/src/app/+admin/plugins/plugins.routes.ts +++ b/client/src/app/+admin/plugins/plugins.routes.ts @@ -3,7 +3,7 @@ import { PluginListInstalledComponent } from '@app/+admin/plugins/plugin-list-in import { PluginSearchComponent } from '@app/+admin/plugins/plugin-search/plugin-search.component' import { PluginShowInstalledComponent } from '@app/+admin/plugins/plugin-show-installed/plugin-show-installed.component' import { UserRightGuard } from '@app/core' -import { UserRight } from '@shared/models' +import { UserRight } from '@peertube/peertube-models' export const PluginsRoutes: Routes = [ { diff --git a/client/src/app/+admin/plugins/shared/plugin-api.service.ts b/client/src/app/+admin/plugins/shared/plugin-api.service.ts index fbfdaea18..e1bd2f125 100644 --- a/client/src/app/+admin/plugins/shared/plugin-api.service.ts +++ b/client/src/app/+admin/plugins/shared/plugin-api.service.ts @@ -9,9 +9,10 @@ import { PeerTubePlugin, PeerTubePluginIndex, PluginType, + PluginType_Type, RegisteredServerSettings, ResultList -} from '@shared/models' +} from '@peertube/peertube-models' import { environment } from '../../../../environments/environment' @Injectable() @@ -25,7 +26,7 @@ export class PluginApiService { private pluginService: PluginService ) { } - getPluginTypeLabel (type: PluginType) { + getPluginTypeLabel (type: PluginType_Type) { if (type === PluginType.PLUGIN) { return $localize`plugin` } @@ -34,7 +35,7 @@ export class PluginApiService { } getPlugins ( - pluginType: PluginType, + pluginType: PluginType_Type, componentPagination: ComponentPagination, sort: string ) { @@ -49,7 +50,7 @@ export class PluginApiService { } searchAvailablePlugins ( - pluginType: PluginType, + pluginType: PluginType_Type, componentPagination: ComponentPagination, sort: string, search?: string @@ -73,7 +74,7 @@ export class PluginApiService { .pipe(catchError(res => this.restExtractor.handleError(res))) } - getPluginRegisteredSettings (pluginName: string, pluginType: PluginType) { + getPluginRegisteredSettings (pluginName: string, pluginType: PluginType_Type) { const npmName = this.pluginService.nameToNpmName(pluginName, pluginType) const path = PluginApiService.BASE_PLUGIN_URL + '/' + npmName + '/registered-settings' @@ -83,7 +84,7 @@ export class PluginApiService { ) } - updatePluginSettings (pluginName: string, pluginType: PluginType, settings: any) { + updatePluginSettings (pluginName: string, pluginType: PluginType_Type, settings: any) { const npmName = this.pluginService.nameToNpmName(pluginName, pluginType) const path = PluginApiService.BASE_PLUGIN_URL + '/' + npmName + '/settings' @@ -91,7 +92,7 @@ export class PluginApiService { .pipe(catchError(res => this.restExtractor.handleError(res))) } - uninstall (pluginName: string, pluginType: PluginType) { + uninstall (pluginName: string, pluginType: PluginType_Type) { const body: ManagePlugin = { npmName: this.pluginService.nameToNpmName(pluginName, pluginType) } @@ -100,7 +101,7 @@ export class PluginApiService { .pipe(catchError(res => this.restExtractor.handleError(res))) } - update (pluginName: string, pluginType: PluginType) { + update (pluginName: string, pluginType: PluginType_Type) { const body: ManagePlugin = { npmName: this.pluginService.nameToNpmName(pluginName, pluginType) } @@ -118,7 +119,7 @@ export class PluginApiService { .pipe(catchError(res => this.restExtractor.handleError(res))) } - getPluginOrThemeHref (type: PluginType, name: string) { + getPluginOrThemeHref (type: PluginType_Type, name: string) { const typeString = type === PluginType.PLUGIN ? 'plugin' : 'theme' diff --git a/client/src/app/+admin/plugins/shared/plugin-card.component.ts b/client/src/app/+admin/plugins/shared/plugin-card.component.ts index 462a6c213..ae91f6887 100644 --- a/client/src/app/+admin/plugins/shared/plugin-card.component.ts +++ b/client/src/app/+admin/plugins/shared/plugin-card.component.ts @@ -1,5 +1,5 @@ import { Component, Input } from '@angular/core' -import { PeerTubePlugin, PeerTubePluginIndex, PluginType } from '@shared/models' +import { PeerTubePlugin, PeerTubePluginIndex, PluginType_Type } from '@peertube/peertube-models' import { PluginApiService } from './plugin-api.service' @Component({ @@ -11,7 +11,7 @@ import { PluginApiService } from './plugin-api.service' export class PluginCardComponent { @Input() plugin: PeerTubePluginIndex | PeerTubePlugin @Input() version: string - @Input() pluginType: PluginType + @Input() pluginType: PluginType_Type constructor ( private pluginApiService: PluginApiService diff --git a/client/src/app/+admin/plugins/shared/plugin-navigation.component.ts b/client/src/app/+admin/plugins/shared/plugin-navigation.component.ts index 1c963f521..c829bc975 100644 --- a/client/src/app/+admin/plugins/shared/plugin-navigation.component.ts +++ b/client/src/app/+admin/plugins/shared/plugin-navigation.component.ts @@ -1,5 +1,5 @@ import { Component, Input } from '@angular/core' -import { PluginType } from '@shared/models/plugins' +import { PluginType_Type } from '@peertube/peertube-models' @Component({ selector: 'my-plugin-navigation', @@ -7,5 +7,5 @@ import { PluginType } from '@shared/models/plugins' styleUrls: [ './plugin-navigation.component.scss' ] }) export class PluginNavigationComponent { - @Input() pluginType: PluginType + @Input() pluginType: PluginType_Type } diff --git a/client/src/app/+admin/shared/user-email-info.component.ts b/client/src/app/+admin/shared/user-email-info.component.ts index e33948b60..0af905c84 100644 --- a/client/src/app/+admin/shared/user-email-info.component.ts +++ b/client/src/app/+admin/shared/user-email-info.component.ts @@ -1,5 +1,5 @@ import { Component, Input } from '@angular/core' -import { User, UserRegistration } from '@shared/models/users' +import { User, UserRegistration } from '@peertube/peertube-models' @Component({ selector: 'my-user-email-info', diff --git a/client/src/app/+admin/shared/user-real-quota-info.component.ts b/client/src/app/+admin/shared/user-real-quota-info.component.ts index 0a342c799..dd78fa9f0 100644 --- a/client/src/app/+admin/shared/user-real-quota-info.component.ts +++ b/client/src/app/+admin/shared/user-real-quota-info.component.ts @@ -1,6 +1,6 @@ import { Component, Input, OnInit } from '@angular/core' import { ServerService } from '@app/core' -import { HTMLServerConfig, VideoResolution } from '@shared/models/index' +import { HTMLServerConfig, VideoResolution } from '@peertube/peertube-models' @Component({ selector: 'my-user-real-quota-info', diff --git a/client/src/app/+admin/system/debug/debug.component.ts b/client/src/app/+admin/system/debug/debug.component.ts index 1f4e71e8a..5c86803ef 100644 --- a/client/src/app/+admin/system/debug/debug.component.ts +++ b/client/src/app/+admin/system/debug/debug.component.ts @@ -1,6 +1,6 @@ import { Component, OnInit } from '@angular/core' import { Notifier } from '@app/core' -import { Debug } from '@shared/models' +import { Debug } from '@peertube/peertube-models' import { DebugService } from './debug.service' @Component({ diff --git a/client/src/app/+admin/system/debug/debug.service.ts b/client/src/app/+admin/system/debug/debug.service.ts index ab1d0a7fa..24d3b2ab8 100644 --- a/client/src/app/+admin/system/debug/debug.service.ts +++ b/client/src/app/+admin/system/debug/debug.service.ts @@ -3,7 +3,7 @@ import { catchError } from 'rxjs/operators' import { HttpClient } from '@angular/common/http' import { Injectable } from '@angular/core' import { RestExtractor } from '@app/core' -import { Debug } from '@shared/models' +import { Debug } from '@peertube/peertube-models' import { environment } from '../../../../environments/environment' @Injectable() diff --git a/client/src/app/+admin/system/jobs/job.service.ts b/client/src/app/+admin/system/jobs/job.service.ts index 031e2bad8..eae1dea7d 100644 --- a/client/src/app/+admin/system/jobs/job.service.ts +++ b/client/src/app/+admin/system/jobs/job.service.ts @@ -4,7 +4,7 @@ import { catchError, map } from 'rxjs/operators' import { HttpClient, HttpParams } from '@angular/common/http' import { Injectable } from '@angular/core' import { RestExtractor, RestPagination, RestService } from '@app/core' -import { Job, ResultList } from '@shared/models' +import { Job, ResultList } from '@peertube/peertube-models' import { environment } from '../../../../environments/environment' import { JobStateClient } from '../../../../types/job-state-client.type' import { JobTypeClient } from '../../../../types/job-type-client.type' diff --git a/client/src/app/+admin/system/jobs/jobs.component.ts b/client/src/app/+admin/system/jobs/jobs.component.ts index 6e10c81ff..4e6b4bf7b 100644 --- a/client/src/app/+admin/system/jobs/jobs.component.ts +++ b/client/src/app/+admin/system/jobs/jobs.component.ts @@ -1,9 +1,9 @@ import { SortMeta } from 'primeng/api' import { Component, OnInit } from '@angular/core' import { Notifier, RestPagination, RestTable } from '@app/core' +import { escapeHTML } from '@peertube/peertube-core-utils' +import { Job, JobState, JobType } from '@peertube/peertube-models' import { peertubeLocalStorage } from '@root-helpers/peertube-web-storage' -import { escapeHTML } from '@shared/core-utils/renderer' -import { Job, JobState, JobType } from '@shared/models' import { JobStateClient } from '../../../../types/job-state-client.type' import { JobTypeClient } from '../../../../types/job-type-client.type' import { JobService } from './job.service' diff --git a/client/src/app/+admin/system/logs/log-row.model.ts b/client/src/app/+admin/system/logs/log-row.model.ts index e83c7b064..15799e8b0 100644 --- a/client/src/app/+admin/system/logs/log-row.model.ts +++ b/client/src/app/+admin/system/logs/log-row.model.ts @@ -1,6 +1,6 @@ import omit from 'lodash-es/omit' import { logger } from '@root-helpers/logger' -import { ServerLogLevel } from '@shared/models' +import { ServerLogLevel } from '@peertube/peertube-models' export class LogRow { date: Date diff --git a/client/src/app/+admin/system/logs/logs.component.ts b/client/src/app/+admin/system/logs/logs.component.ts index 939e710d7..22375fcd9 100644 --- a/client/src/app/+admin/system/logs/logs.component.ts +++ b/client/src/app/+admin/system/logs/logs.component.ts @@ -1,6 +1,6 @@ import { Component, ElementRef, OnInit, ViewChild } from '@angular/core' import { LocalStorageService, Notifier } from '@app/core' -import { ServerLogLevel } from '@shared/models' +import { ServerLogLevel } from '@peertube/peertube-models' import { LogRow } from './log-row.model' import { LogsService } from './logs.service' diff --git a/client/src/app/+admin/system/logs/logs.service.ts b/client/src/app/+admin/system/logs/logs.service.ts index 933a074a8..9e774d7fd 100644 --- a/client/src/app/+admin/system/logs/logs.service.ts +++ b/client/src/app/+admin/system/logs/logs.service.ts @@ -3,7 +3,7 @@ import { catchError, map } from 'rxjs/operators' import { HttpClient, HttpParams } from '@angular/common/http' import { Injectable } from '@angular/core' import { RestExtractor, RestService } from '@app/core' -import { ServerLogLevel } from '@shared/models' +import { ServerLogLevel } from '@peertube/peertube-models' import { environment } from '../../../../environments/environment' import { LogRow } from './log-row.model' diff --git a/client/src/app/+admin/system/runners/runner-job-list/runner-job-list.component.ts b/client/src/app/+admin/system/runners/runner-job-list/runner-job-list.component.ts index 2670eac86..e75446d8c 100644 --- a/client/src/app/+admin/system/runners/runner-job-list/runner-job-list.component.ts +++ b/client/src/app/+admin/system/runners/runner-job-list/runner-job-list.component.ts @@ -3,7 +3,7 @@ import { Component, OnInit } from '@angular/core' import { ConfirmService, Notifier, RestPagination, RestTable } from '@app/core' import { formatICU } from '@app/helpers' import { DropdownAction } from '@app/shared/shared-main' -import { RunnerJob, RunnerJobState } from '@shared/models' +import { RunnerJob, RunnerJobState } from '@peertube/peertube-models' import { RunnerJobFormatted, RunnerService } from '../runner.service' import { AdvancedInputFilter } from '@app/shared/shared-forms' diff --git a/client/src/app/+admin/system/runners/runner-list/runner-list.component.ts b/client/src/app/+admin/system/runners/runner-list/runner-list.component.ts index 7566f967e..0964471f9 100644 --- a/client/src/app/+admin/system/runners/runner-list/runner-list.component.ts +++ b/client/src/app/+admin/system/runners/runner-list/runner-list.component.ts @@ -2,7 +2,7 @@ import { SortMeta } from 'primeng/api' import { Component, OnInit } from '@angular/core' import { ConfirmService, Notifier, RestPagination, RestTable } from '@app/core' import { DropdownAction } from '@app/shared/shared-main' -import { Runner } from '@shared/models' +import { Runner } from '@peertube/peertube-models' import { RunnerService } from '../runner.service' @Component({ diff --git a/client/src/app/+admin/system/runners/runner-registration-token-list/runner-registration-token-list.component.ts b/client/src/app/+admin/system/runners/runner-registration-token-list/runner-registration-token-list.component.ts index 77908a2e1..c8a597b18 100644 --- a/client/src/app/+admin/system/runners/runner-registration-token-list/runner-registration-token-list.component.ts +++ b/client/src/app/+admin/system/runners/runner-registration-token-list/runner-registration-token-list.component.ts @@ -2,7 +2,7 @@ import { SortMeta } from 'primeng/api' import { Component, OnInit } from '@angular/core' import { ConfirmService, Notifier, RestPagination, RestTable } from '@app/core' import { DropdownAction } from '@app/shared/shared-main' -import { RunnerRegistrationToken } from '@shared/models' +import { RunnerRegistrationToken } from '@peertube/peertube-models' import { RunnerService } from '../runner.service' @Component({ diff --git a/client/src/app/+admin/system/runners/runner.service.ts b/client/src/app/+admin/system/runners/runner.service.ts index 3ab36c4ff..94bdaad78 100644 --- a/client/src/app/+admin/system/runners/runner.service.ts +++ b/client/src/app/+admin/system/runners/runner.service.ts @@ -4,9 +4,8 @@ import { catchError, concatMap, forkJoin, from, map, toArray } from 'rxjs' import { HttpClient, HttpParams } from '@angular/common/http' import { Injectable } from '@angular/core' import { RestExtractor, RestPagination, RestService, ServerService } from '@app/core' -import { arrayify, peertubeTranslate } from '@shared/core-utils' -import { ResultList } from '@shared/models/common' -import { Runner, RunnerJob, RunnerJobAdmin, RunnerJobState, RunnerRegistrationToken } from '@shared/models/runners' +import { arrayify, peertubeTranslate } from '@peertube/peertube-core-utils' +import { ResultList, Runner, RunnerJob, RunnerJobAdmin, RunnerJobState, RunnerRegistrationToken } from '@peertube/peertube-models' import { environment } from '../../../../environments/environment' export type RunnerJobFormatted = RunnerJob & { diff --git a/client/src/app/+admin/system/runners/runners.routes.ts b/client/src/app/+admin/system/runners/runners.routes.ts index fabe687d6..004c3bedd 100644 --- a/client/src/app/+admin/system/runners/runners.routes.ts +++ b/client/src/app/+admin/system/runners/runners.routes.ts @@ -1,6 +1,6 @@ import { Routes } from '@angular/router' import { UserRightGuard } from '@app/core' -import { UserRight } from '@shared/models' +import { UserRight } from '@peertube/peertube-models' import { RunnerJobListComponent } from './runner-job-list' import { RunnerListComponent } from './runner-list' import { RunnerRegistrationTokenListComponent } from './runner-registration-token-list' diff --git a/client/src/app/+admin/system/system.routes.ts b/client/src/app/+admin/system/system.routes.ts index 87e4b25b3..169d52952 100644 --- a/client/src/app/+admin/system/system.routes.ts +++ b/client/src/app/+admin/system/system.routes.ts @@ -1,6 +1,6 @@ import { Routes } from '@angular/router' import { UserRightGuard } from '@app/core' -import { UserRight } from '@shared/models' +import { UserRight } from '@peertube/peertube-models' import { DebugComponent } from './debug' import { JobsComponent } from './jobs/jobs.component' import { LogsComponent } from './logs' diff --git a/client/src/app/+error-page/error-page.component.ts b/client/src/app/+error-page/error-page.component.ts index 4fee01350..9cc4646df 100644 --- a/client/src/app/+error-page/error-page.component.ts +++ b/client/src/app/+error-page/error-page.component.ts @@ -1,7 +1,7 @@ import { Component, OnInit } from '@angular/core' import { Title } from '@angular/platform-browser' import { Router } from '@angular/router' -import { HttpStatusCode } from '@shared/models' +import { HttpStatusCode, HttpStatusCodeType } from '@peertube/peertube-models' @Component({ selector: 'my-error-page', @@ -9,7 +9,7 @@ import { HttpStatusCode } from '@shared/models' styleUrls: [ './error-page.component.scss' ] }) export class ErrorPageComponent implements OnInit { - status = HttpStatusCode.NOT_FOUND_404 + status: HttpStatusCodeType = HttpStatusCode.NOT_FOUND_404 type: 'video' | 'other' = 'other' public constructor ( diff --git a/client/src/app/+login/login.component.ts b/client/src/app/+login/login.component.ts index e486df61d..a6906efa2 100644 --- a/client/src/app/+login/login.component.ts +++ b/client/src/app/+login/login.component.ts @@ -8,8 +8,8 @@ import { USER_OTP_TOKEN_VALIDATOR } from '@app/shared/form-validators/user-valid import { FormReactive, FormReactiveService, InputTextComponent } from '@app/shared/shared-forms' import { InstanceAboutAccordionComponent } from '@app/shared/shared-instance' import { NgbAccordionDirective, NgbModal, NgbModalRef } from '@ng-bootstrap/ng-bootstrap' -import { getExternalAuthHref } from '@shared/core-utils' -import { RegisteredExternalAuthConfig, ServerConfig, ServerErrorCode } from '@shared/models' +import { getExternalAuthHref } from '@peertube/peertube-core-utils' +import { RegisteredExternalAuthConfig, ServerConfig, ServerErrorCode } from '@peertube/peertube-models' @Component({ selector: 'my-login', diff --git a/client/src/app/+manage/video-channel-edit/video-channel-create.component.ts b/client/src/app/+manage/video-channel-edit/video-channel-create.component.ts index 8ca94b0b3..3f876078f 100644 --- a/client/src/app/+manage/video-channel-edit/video-channel-create.component.ts +++ b/client/src/app/+manage/video-channel-edit/video-channel-create.component.ts @@ -11,7 +11,7 @@ import { } from '@app/shared/form-validators/video-channel-validators' import { FormReactiveService } from '@app/shared/shared-forms' import { VideoChannel, VideoChannelService } from '@app/shared/shared-main' -import { HttpStatusCode, VideoChannelCreate } from '@shared/models' +import { HttpStatusCode, VideoChannelCreate } from '@peertube/peertube-models' import { VideoChannelEdit } from './video-channel-edit' @Component({ diff --git a/client/src/app/+manage/video-channel-edit/video-channel-update.component.ts b/client/src/app/+manage/video-channel-edit/video-channel-update.component.ts index f9045db35..3992e298e 100644 --- a/client/src/app/+manage/video-channel-edit/video-channel-update.component.ts +++ b/client/src/app/+manage/video-channel-edit/video-channel-update.component.ts @@ -11,9 +11,9 @@ import { } from '@app/shared/form-validators/video-channel-validators' import { FormReactiveService } from '@app/shared/shared-forms' import { VideoChannel, VideoChannelService } from '@app/shared/shared-main' -import { HTMLServerConfig, VideoChannelUpdate } from '@shared/models' +import { HTMLServerConfig, VideoChannelUpdate } from '@peertube/peertube-models' import { VideoChannelEdit } from './video-channel-edit' -import { shallowCopy } from '@shared/core-utils' +import { shallowCopy } from '@peertube/peertube-core-utils' @Component({ selector: 'my-video-channel-update', diff --git a/client/src/app/+my-account/my-account-applications/my-account-applications.component.ts b/client/src/app/+my-account/my-account-applications/my-account-applications.component.ts index e88cdd228..281a12eda 100644 --- a/client/src/app/+my-account/my-account-applications/my-account-applications.component.ts +++ b/client/src/app/+my-account/my-account-applications/my-account-applications.component.ts @@ -1,8 +1,7 @@ import { Component, OnInit } from '@angular/core' import { AuthService, ConfirmService, Notifier, ScopedTokensService } from '@app/core' import { VideoService } from '@app/shared/shared-main' -import { FeedFormat } from '@shared/models' -import { ScopedToken } from '@shared/models/users/user-scoped-token' +import { FeedFormat, ScopedToken } from '@peertube/peertube-models' import { environment } from '../../../environments/environment' @Component({ diff --git a/client/src/app/+my-account/my-account-settings/my-account-change-email/my-account-change-email.component.ts b/client/src/app/+my-account/my-account-settings/my-account-change-email/my-account-change-email.component.ts index 1e8fa2a56..7bbd240d0 100644 --- a/client/src/app/+my-account/my-account-settings/my-account-change-email/my-account-change-email.component.ts +++ b/client/src/app/+my-account/my-account-settings/my-account-change-email/my-account-change-email.component.ts @@ -4,7 +4,7 @@ import { Component, OnInit } from '@angular/core' import { AuthService, Notifier, ServerService, UserService } from '@app/core' import { USER_EMAIL_VALIDATOR, USER_PASSWORD_VALIDATOR } from '@app/shared/form-validators/user-validators' import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' -import { HttpStatusCode, User } from '@shared/models' +import { HttpStatusCode, User } from '@peertube/peertube-models' @Component({ selector: 'my-account-change-email', diff --git a/client/src/app/+my-account/my-account-settings/my-account-change-password/my-account-change-password.component.ts b/client/src/app/+my-account/my-account-settings/my-account-change-password/my-account-change-password.component.ts index 805d50070..f916740be 100644 --- a/client/src/app/+my-account/my-account-settings/my-account-change-password/my-account-change-password.component.ts +++ b/client/src/app/+my-account/my-account-settings/my-account-change-password/my-account-change-password.component.ts @@ -7,7 +7,7 @@ import { USER_PASSWORD_VALIDATOR } from '@app/shared/form-validators/user-validators' import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' -import { HttpStatusCode, User } from '@shared/models' +import { HttpStatusCode, User } from '@peertube/peertube-models' @Component({ selector: 'my-account-change-password', diff --git a/client/src/app/+my-account/my-account-settings/my-account-email-preferences/my-account-email-preferences.component.ts b/client/src/app/+my-account/my-account-settings/my-account-email-preferences/my-account-email-preferences.component.ts index 381d18922..bf6accc3e 100644 --- a/client/src/app/+my-account/my-account-settings/my-account-email-preferences/my-account-email-preferences.component.ts +++ b/client/src/app/+my-account/my-account-settings/my-account-email-preferences/my-account-email-preferences.component.ts @@ -2,7 +2,7 @@ import { Subject } from 'rxjs' import { Component, Input, OnInit } from '@angular/core' import { Notifier, UserService } from '@app/core' import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' -import { User, UserUpdateMe } from '@shared/models' +import { User, UserUpdateMe } from '@peertube/peertube-models' @Component({ selector: 'my-account-email-preferences', diff --git a/client/src/app/+my-account/my-account-settings/my-account-notification-preferences/my-account-notification-preferences.component.ts b/client/src/app/+my-account/my-account-settings/my-account-notification-preferences/my-account-notification-preferences.component.ts index 2adc276a9..f4181340d 100644 --- a/client/src/app/+my-account/my-account-settings/my-account-notification-preferences/my-account-notification-preferences.component.ts +++ b/client/src/app/+my-account/my-account-settings/my-account-notification-preferences/my-account-notification-preferences.component.ts @@ -3,8 +3,8 @@ import { Subject } from 'rxjs' import { Component, Input, OnInit } from '@angular/core' import { Notifier, ServerService, User } from '@app/core' import { UserNotificationService } from '@app/shared/shared-main' -import { objectKeysTyped } from '@shared/core-utils' -import { UserNotificationSetting, UserNotificationSettingValue, UserRight } from '@shared/models' +import { objectKeysTyped } from '@peertube/peertube-core-utils' +import { UserNotificationSetting, UserNotificationSettingValue, UserRight, UserRightType } from '@peertube/peertube-models' @Component({ selector: 'my-account-notification-preferences', @@ -19,7 +19,7 @@ export class MyAccountNotificationPreferencesComponent implements OnInit { emailNotifications: { [ id in keyof UserNotificationSetting ]?: boolean } = {} webNotifications: { [ id in keyof UserNotificationSetting ]?: boolean } = {} labelNotifications: { [ id in keyof UserNotificationSetting ]?: string } = {} - rightNotifications: { [ id in keyof Partial ]?: UserRight } = {} + rightNotifications: { [ id in keyof Partial ]?: UserRightType } = {} emailEnabled = false private savePreferences = debounce(this.savePreferencesImpl.bind(this), 500) diff --git a/client/src/app/+my-account/my-account-settings/my-account-settings.component.ts b/client/src/app/+my-account/my-account-settings/my-account-settings.component.ts index a276bb126..4b3b33bcc 100644 --- a/client/src/app/+my-account/my-account-settings/my-account-settings.component.ts +++ b/client/src/app/+my-account/my-account-settings/my-account-settings.component.ts @@ -3,7 +3,7 @@ import { HttpErrorResponse } from '@angular/common/http' import { AfterViewChecked, Component, OnInit } from '@angular/core' import { AuthService, Notifier, User, UserService } from '@app/core' import { genericUploadErrorHandler } from '@app/helpers' -import { shallowCopy } from '@shared/core-utils' +import { shallowCopy } from '@peertube/peertube-core-utils' @Component({ selector: 'my-account-settings', diff --git a/client/src/app/+my-library/my-follows/my-followers.component.ts b/client/src/app/+my-library/my-follows/my-followers.component.ts index 0dd9bf6f5..4e3e5bcc4 100644 --- a/client/src/app/+my-library/my-follows/my-followers.component.ts +++ b/client/src/app/+my-library/my-follows/my-followers.component.ts @@ -4,7 +4,7 @@ import { ActivatedRoute } from '@angular/router' import { AuthService, ComponentPagination, Notifier } from '@app/core' import { AdvancedInputFilter } from '@app/shared/shared-forms' import { UserSubscriptionService } from '@app/shared/shared-user-subscription' -import { ActorFollow } from '@shared/models' +import { ActorFollow } from '@peertube/peertube-models' @Component({ templateUrl: './my-followers.component.html', diff --git a/client/src/app/+my-library/my-library.component.ts b/client/src/app/+my-library/my-library.component.ts index ff901952f..35eb617ab 100644 --- a/client/src/app/+my-library/my-library.component.ts +++ b/client/src/app/+my-library/my-library.component.ts @@ -1,6 +1,6 @@ import { Component, OnInit } from '@angular/core' import { AuthService, AuthUser, ScreenService, ServerService } from '@app/core' -import { HTMLServerConfig } from '@shared/models' +import { HTMLServerConfig } from '@peertube/peertube-models' import { TopMenuDropdownParam } from '../shared/shared-main/misc/top-menu-dropdown.component' @Component({ diff --git a/client/src/app/+my-library/my-ownership/my-accept-ownership/my-accept-ownership.component.ts b/client/src/app/+my-library/my-ownership/my-accept-ownership/my-accept-ownership.component.ts index ca7eb680b..87e389411 100644 --- a/client/src/app/+my-library/my-ownership/my-accept-ownership/my-accept-ownership.component.ts +++ b/client/src/app/+my-library/my-ownership/my-accept-ownership/my-accept-ownership.component.ts @@ -6,7 +6,7 @@ import { OWNERSHIP_CHANGE_CHANNEL_VALIDATOR } from '@app/shared/form-validators/ import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' import { VideoOwnershipService } from '@app/shared/shared-main' import { NgbModal } from '@ng-bootstrap/ng-bootstrap' -import { VideoChangeOwnership } from '@shared/models' +import { VideoChangeOwnership } from '@peertube/peertube-models' @Component({ selector: 'my-accept-ownership', diff --git a/client/src/app/+my-library/my-ownership/my-ownership.component.ts b/client/src/app/+my-library/my-ownership/my-ownership.component.ts index 8d6a42dfb..4838eca27 100644 --- a/client/src/app/+my-library/my-ownership/my-ownership.component.ts +++ b/client/src/app/+my-library/my-ownership/my-ownership.component.ts @@ -2,7 +2,7 @@ import { SortMeta } from 'primeng/api' import { Component, OnInit, ViewChild } from '@angular/core' import { Notifier, RestPagination, RestTable } from '@app/core' import { Account, VideoOwnershipService } from '@app/shared/shared-main' -import { VideoChangeOwnership, VideoChangeOwnershipStatus } from '@shared/models' +import { VideoChangeOwnership, VideoChangeOwnershipStatus, VideoChangeOwnershipStatusType } from '@peertube/peertube-models' import { MyAcceptOwnershipComponent } from './my-accept-ownership/my-accept-ownership.component' @Component({ @@ -32,7 +32,7 @@ export class MyOwnershipComponent extends RestTable implements OnInit { return 'MyOwnershipComponent' } - getStatusClass (status: VideoChangeOwnershipStatus) { + getStatusClass (status: VideoChangeOwnershipStatusType) { switch (status) { case VideoChangeOwnershipStatus.ACCEPTED: return 'badge-green' diff --git a/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.ts b/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.ts index 1f7287f44..44e5c45b9 100644 --- a/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.ts +++ b/client/src/app/+my-library/my-video-channel-syncs/my-video-channel-syncs.component.ts @@ -3,8 +3,7 @@ import { mergeMap } from 'rxjs' import { Component, OnInit } from '@angular/core' import { AuthService, Notifier, RestPagination, RestTable, ServerService } from '@app/core' import { DropdownAction, VideoChannelService, VideoChannelSyncService } from '@app/shared/shared-main' -import { HTMLServerConfig } from '@shared/models/server' -import { VideoChannelSync, VideoChannelSyncState } from '@shared/models/videos' +import { HTMLServerConfig, VideoChannelSync, VideoChannelSyncState, VideoChannelSyncStateType } from '@peertube/peertube-models' @Component({ templateUrl: './my-video-channel-syncs.component.html', @@ -124,7 +123,7 @@ export class MyVideoChannelSyncsComponent extends RestTable implements OnInit { return '/my-library/video-channel-syncs/create' } - getSyncStateClass (stateId: VideoChannelSyncState) { + getSyncStateClass (stateId: VideoChannelSyncStateType) { return [ 'pt-badge', MyVideoChannelSyncsComponent.STATE_CLASS_BY_ID[stateId] ] } diff --git a/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.ts b/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.ts index a14ab5b92..a40a68764 100644 --- a/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.ts +++ b/client/src/app/+my-library/my-video-channel-syncs/video-channel-sync-edit/video-channel-sync-edit.component.ts @@ -7,7 +7,7 @@ import { listUserChannelsForSelect } from '@app/helpers' import { VIDEO_CHANNEL_EXTERNAL_URL_VALIDATOR } from '@app/shared/form-validators/video-channel-validators' import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' import { VideoChannelService, VideoChannelSyncService } from '@app/shared/shared-main' -import { VideoChannelSyncCreate } from '@shared/models/videos' +import { VideoChannelSyncCreate } from '@peertube/peertube-models' @Component({ selector: 'my-video-channel-sync-edit', diff --git a/client/src/app/+my-library/my-video-imports/my-video-imports.component.ts b/client/src/app/+my-library/my-video-imports/my-video-imports.component.ts index 7d82f62b9..76cefa7bd 100644 --- a/client/src/app/+my-library/my-video-imports/my-video-imports.component.ts +++ b/client/src/app/+my-library/my-video-imports/my-video-imports.component.ts @@ -2,7 +2,7 @@ import { SortMeta } from 'primeng/api' import { Component, OnInit } from '@angular/core' import { Notifier, RestPagination, RestTable } from '@app/core' import { Video, VideoImportService } from '@app/shared/shared-main' -import { VideoImport, VideoImportState } from '@shared/models' +import { VideoImport, VideoImportState, VideoImportStateType } from '@peertube/peertube-models' @Component({ templateUrl: './my-video-imports.component.html', @@ -29,7 +29,7 @@ export class MyVideoImportsComponent extends RestTable implements OnInit { return 'MyVideoImportsComponent' } - getVideoImportStateClass (state: VideoImportState) { + getVideoImportStateClass (state: VideoImportStateType) { switch (state) { case VideoImportState.FAILED: return 'badge-red' diff --git a/client/src/app/+my-library/my-video-playlists/my-video-playlist-create.component.ts b/client/src/app/+my-library/my-video-playlists/my-video-playlist-create.component.ts index 63f72df3f..a54b3ca86 100644 --- a/client/src/app/+my-library/my-video-playlists/my-video-playlist-create.component.ts +++ b/client/src/app/+my-library/my-video-playlists/my-video-playlist-create.component.ts @@ -11,8 +11,7 @@ import { } from '@app/shared/form-validators/video-playlist-validators' import { FormReactiveService } from '@app/shared/shared-forms' import { VideoPlaylistService } from '@app/shared/shared-video-playlist' -import { VideoPlaylistCreate } from '@shared/models/videos/playlist/video-playlist-create.model' -import { VideoPlaylistPrivacy } from '@shared/models/videos/playlist/video-playlist-privacy.model' +import { VideoPlaylistCreate, VideoPlaylistPrivacy } from '@peertube/peertube-models' import { MyVideoPlaylistEdit } from './my-video-playlist-edit' @Component({ diff --git a/client/src/app/+my-library/my-video-playlists/my-video-playlist-edit.ts b/client/src/app/+my-library/my-video-playlists/my-video-playlist-edit.ts index 71db0592a..127960a58 100644 --- a/client/src/app/+my-library/my-video-playlists/my-video-playlist-edit.ts +++ b/client/src/app/+my-library/my-video-playlists/my-video-playlist-edit.ts @@ -1,13 +1,12 @@ import { FormReactive } from '@app/shared/shared-forms' -import { VideoConstant, VideoPlaylistPrivacy } from '@shared/models' -import { VideoPlaylist } from '@shared/models/videos/playlist/video-playlist.model' +import { VideoConstant, VideoPlaylist, VideoPlaylistPrivacyType } from '@peertube/peertube-models' import { SelectChannelItem } from '../../../types/select-options-item.model' export abstract class MyVideoPlaylistEdit extends FormReactive { // Declare it here to avoid errors in create template videoPlaylistToUpdate: VideoPlaylist userVideoChannels: SelectChannelItem[] = [] - videoPlaylistPrivacies: VideoConstant[] = [] + videoPlaylistPrivacies: VideoConstant[] = [] abstract isCreation (): boolean abstract getFormButtonTitle (): string diff --git a/client/src/app/+my-library/my-video-playlists/my-video-playlist-elements.component.ts b/client/src/app/+my-library/my-video-playlists/my-video-playlist-elements.component.ts index c9739b6cc..7a9588743 100644 --- a/client/src/app/+my-library/my-video-playlists/my-video-playlist-elements.component.ts +++ b/client/src/app/+my-library/my-video-playlists/my-video-playlist-elements.component.ts @@ -6,7 +6,7 @@ import { ComponentPagination, ConfirmService, HooksService, Notifier, ScreenServ import { DropdownAction } from '@app/shared/shared-main' import { VideoShareComponent } from '@app/shared/shared-share-modal' import { VideoPlaylist, VideoPlaylistElement, VideoPlaylistService } from '@app/shared/shared-video-playlist' -import { VideoPlaylistType } from '@shared/models' +import { VideoPlaylistType } from '@peertube/peertube-models' @Component({ templateUrl: './my-video-playlist-elements.component.html', diff --git a/client/src/app/+my-library/my-video-playlists/my-video-playlist-update.component.ts b/client/src/app/+my-library/my-video-playlists/my-video-playlist-update.component.ts index bbe8a5f80..f29a0cc45 100644 --- a/client/src/app/+my-library/my-video-playlists/my-video-playlist-update.component.ts +++ b/client/src/app/+my-library/my-video-playlists/my-video-playlist-update.component.ts @@ -13,7 +13,7 @@ import { } from '@app/shared/form-validators/video-playlist-validators' import { FormReactiveService } from '@app/shared/shared-forms' import { VideoPlaylist, VideoPlaylistService } from '@app/shared/shared-video-playlist' -import { VideoPlaylistUpdate } from '@shared/models' +import { VideoPlaylistUpdate } from '@peertube/peertube-models' import { MyVideoPlaylistEdit } from './my-video-playlist-edit' @Component({ diff --git a/client/src/app/+my-library/my-video-playlists/my-video-playlists.component.ts b/client/src/app/+my-library/my-video-playlists/my-video-playlists.component.ts index 634176744..08bd94fae 100644 --- a/client/src/app/+my-library/my-video-playlists/my-video-playlists.component.ts +++ b/client/src/app/+my-library/my-video-playlists/my-video-playlists.component.ts @@ -3,7 +3,7 @@ import { mergeMap } from 'rxjs/operators' import { Component } from '@angular/core' import { AuthService, ComponentPagination, ConfirmService, Notifier } from '@app/core' import { VideoPlaylist, VideoPlaylistService } from '@app/shared/shared-video-playlist' -import { VideoPlaylistType } from '@shared/models' +import { VideoPlaylistType } from '@peertube/peertube-models' @Component({ templateUrl: './my-video-playlists.component.html', diff --git a/client/src/app/+my-library/my-videos/my-videos.component.ts b/client/src/app/+my-library/my-videos/my-videos.component.ts index 1827d6a0b..4a7604878 100644 --- a/client/src/app/+my-library/my-videos/my-videos.component.ts +++ b/client/src/app/+my-library/my-videos/my-videos.component.ts @@ -16,7 +16,7 @@ import { VideosSelectionComponent } from '@app/shared/shared-video-miniature' import { VideoPlaylistService } from '@app/shared/shared-video-playlist' -import { VideoChannel, VideoExistInPlaylist, VideosExistInPlaylists, VideoSortField } from '@shared/models' +import { VideoChannel, VideoExistInPlaylist, VideosExistInPlaylists, VideoSortField } from '@peertube/peertube-models' import { VideoChangeOwnershipComponent } from './modals/video-change-ownership.component' @Component({ diff --git a/client/src/app/+search/search-filters.component.ts b/client/src/app/+search/search-filters.component.ts index a6fc51383..b98acee18 100644 --- a/client/src/app/+search/search-filters.component.ts +++ b/client/src/app/+search/search-filters.component.ts @@ -1,7 +1,7 @@ import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core' import { ServerService } from '@app/core' import { AdvancedSearch } from '@app/shared/shared-search' -import { HTMLServerConfig, VideoConstant } from '@shared/models' +import { HTMLServerConfig, VideoConstant } from '@peertube/peertube-models' type FormOption = { id: string, label: string } diff --git a/client/src/app/+search/search.component.ts b/client/src/app/+search/search.component.ts index 31394a1d1..62e349c40 100644 --- a/client/src/app/+search/search.component.ts +++ b/client/src/app/+search/search.component.ts @@ -9,7 +9,7 @@ import { Video, VideoChannel } from '@app/shared/shared-main' import { AdvancedSearch, SearchService } from '@app/shared/shared-search' import { MiniatureDisplayOptions } from '@app/shared/shared-video-miniature' import { VideoPlaylist } from '@app/shared/shared-video-playlist' -import { HTMLServerConfig, SearchTargetType } from '@shared/models' +import { HTMLServerConfig, SearchTargetType } from '@peertube/peertube-models' @Component({ selector: 'my-search', diff --git a/client/src/app/+search/shared/abstract-lazy-load.resolver.ts b/client/src/app/+search/shared/abstract-lazy-load.resolver.ts index 6765ba15e..4e8b71293 100644 --- a/client/src/app/+search/shared/abstract-lazy-load.resolver.ts +++ b/client/src/app/+search/shared/abstract-lazy-load.resolver.ts @@ -2,7 +2,7 @@ import { Observable } from 'rxjs' import { map } from 'rxjs/operators' import { ActivatedRouteSnapshot, Router } from '@angular/router' import { logger } from '@root-helpers/logger' -import { ResultList } from '@shared/models' +import { ResultList } from '@peertube/peertube-models' export abstract class AbstractLazyLoadResolver { protected router: Router diff --git a/client/src/app/+signup/+register/register.component.ts b/client/src/app/+signup/+register/register.component.ts index be51c36d1..396971d98 100644 --- a/client/src/app/+signup/+register/register.component.ts +++ b/client/src/app/+signup/+register/register.component.ts @@ -5,8 +5,7 @@ import { ActivatedRoute } from '@angular/router' import { AuthService } from '@app/core' import { HooksService } from '@app/core/plugins/hooks.service' import { InstanceAboutAccordionComponent } from '@app/shared/shared-instance' -import { UserRegister } from '@shared/models' -import { ServerConfig } from '@shared/models/server' +import { ServerConfig, UserRegister } from '@peertube/peertube-models' import { SignupService } from '../shared/signup.service' @Component({ diff --git a/client/src/app/+signup/shared/signup.service.ts b/client/src/app/+signup/shared/signup.service.ts index f647298be..7c331437e 100644 --- a/client/src/app/+signup/shared/signup.service.ts +++ b/client/src/app/+signup/shared/signup.service.ts @@ -2,7 +2,7 @@ import { catchError, tap } from 'rxjs/operators' import { HttpClient } from '@angular/common/http' import { Injectable } from '@angular/core' import { RestExtractor, UserService } from '@app/core' -import { UserRegister, UserRegistrationRequest } from '@shared/models' +import { UserRegister, UserRegistrationRequest } from '@peertube/peertube-models' @Injectable() export class SignupService { diff --git a/client/src/app/+stats/video/video-stats.component.ts b/client/src/app/+stats/video/video-stats.component.ts index fa5e33ab6..5bde5b01d 100644 --- a/client/src/app/+stats/video/video-stats.component.ts +++ b/client/src/app/+stats/video/video-stats.component.ts @@ -7,15 +7,15 @@ import { ActivatedRoute } from '@angular/router' import { Notifier, PeerTubeRouterService } from '@app/core' import { NumberFormatterPipe, VideoDetails } from '@app/shared/shared-main' import { LiveVideoService } from '@app/shared/shared-video-live' -import { secondsToTime } from '@shared/core-utils' -import { HttpStatusCode } from '@shared/models/http' +import { secondsToTime } from '@peertube/peertube-core-utils' import { + HttpStatusCode, LiveVideoSession, VideoStatsOverall, VideoStatsRetention, VideoStatsTimeserie, VideoStatsTimeserieMetric -} from '@shared/models/videos' +} from '@peertube/peertube-models' import { VideoStatsService } from './video-stats.service' type ActiveGraphId = VideoStatsTimeserieMetric | 'retention' | 'countries' diff --git a/client/src/app/+stats/video/video-stats.service.ts b/client/src/app/+stats/video/video-stats.service.ts index e019c87f7..5d4817892 100644 --- a/client/src/app/+stats/video/video-stats.service.ts +++ b/client/src/app/+stats/video/video-stats.service.ts @@ -4,7 +4,7 @@ import { HttpClient, HttpParams } from '@angular/common/http' import { Injectable } from '@angular/core' import { RestExtractor } from '@app/core' import { VideoService } from '@app/shared/shared-main' -import { VideoStatsOverall, VideoStatsRetention, VideoStatsTimeserie, VideoStatsTimeserieMetric } from '@shared/models/videos' +import { VideoStatsOverall, VideoStatsRetention, VideoStatsTimeserie, VideoStatsTimeserieMetric } from '@peertube/peertube-models' @Injectable({ providedIn: 'root' diff --git a/client/src/app/+video-channels/video-channel-videos/video-channel-videos.component.ts b/client/src/app/+video-channels/video-channel-videos/video-channel-videos.component.ts index 5e3946bf5..725990300 100644 --- a/client/src/app/+video-channels/video-channel-videos/video-channel-videos.component.ts +++ b/client/src/app/+video-channels/video-channel-videos/video-channel-videos.component.ts @@ -4,7 +4,7 @@ import { AfterViewInit, Component, OnDestroy, OnInit } from '@angular/core' import { ComponentPaginationLight, DisableForReuseHook, HooksService, ScreenService } from '@app/core' import { VideoChannel, VideoChannelService, VideoService } from '@app/shared/shared-main' import { MiniatureDisplayOptions, VideoFilters } from '@app/shared/shared-video-miniature' -import { Video, VideoSortField } from '@shared/models' +import { Video, VideoSortField } from '@peertube/peertube-models' @Component({ selector: 'my-video-channel-videos', diff --git a/client/src/app/+video-channels/video-channels.component.ts b/client/src/app/+video-channels/video-channels.component.ts index f5bea66ec..40b3b19b7 100644 --- a/client/src/app/+video-channels/video-channels.component.ts +++ b/client/src/app/+video-channels/video-channels.component.ts @@ -8,7 +8,7 @@ import { Account, ListOverflowItem, VideoChannel, VideoChannelService, VideoServ import { BlocklistService } from '@app/shared/shared-moderation' import { SupportModalComponent } from '@app/shared/shared-support-modal' import { SubscribeButtonComponent } from '@app/shared/shared-user-subscription' -import { HttpStatusCode, UserRight } from '@shared/models' +import { HttpStatusCode, UserRight } from '@peertube/peertube-models' @Component({ templateUrl: './video-channels.component.html', diff --git a/client/src/app/+video-studio/edit/video-studio-edit.component.ts b/client/src/app/+video-studio/edit/video-studio-edit.component.ts index 3d618fbe1..40e9cf40a 100644 --- a/client/src/app/+video-studio/edit/video-studio-edit.component.ts +++ b/client/src/app/+video-studio/edit/video-studio-edit.component.ts @@ -5,8 +5,8 @@ import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' import { VideoDetails } from '@app/shared/shared-main' import { LoadingBarService } from '@ngx-loading-bar/core' import { logger } from '@root-helpers/logger' -import { secondsToTime } from '@shared/core-utils' -import { VideoStudioTask, VideoStudioTaskCut } from '@shared/models' +import { secondsToTime } from '@peertube/peertube-core-utils' +import { VideoStudioTask, VideoStudioTaskCut } from '@peertube/peertube-models' import { VideoStudioService } from '../shared' @Component({ diff --git a/client/src/app/+video-studio/shared/video-studio.service.ts b/client/src/app/+video-studio/shared/video-studio.service.ts index 8d8b2f0e5..a3aabd347 100644 --- a/client/src/app/+video-studio/shared/video-studio.service.ts +++ b/client/src/app/+video-studio/shared/video-studio.service.ts @@ -4,7 +4,7 @@ import { Injectable } from '@angular/core' import { RestExtractor } from '@app/core' import { objectToFormData } from '@app/helpers' import { VideoService } from '@app/shared/shared-main' -import { VideoStudioCreateEdition, VideoStudioTask } from '@shared/models' +import { VideoStudioCreateEdition, VideoStudioTask } from '@peertube/peertube-models' @Injectable() export class VideoStudioService { diff --git a/client/src/app/+videos/+video-edit/shared/video-caption-add-modal.component.ts b/client/src/app/+videos/+video-edit/shared/video-caption-add-modal.component.ts index 4ab2d42db..e595cf2c9 100644 --- a/client/src/app/+videos/+video-edit/shared/video-caption-add-modal.component.ts +++ b/client/src/app/+videos/+video-edit/shared/video-caption-add-modal.component.ts @@ -4,7 +4,7 @@ import { VIDEO_CAPTION_FILE_VALIDATOR, VIDEO_CAPTION_LANGUAGE_VALIDATOR } from ' import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' import { VideoCaptionEdit } from '@app/shared/shared-main' import { NgbModal, NgbModalRef } from '@ng-bootstrap/ng-bootstrap' -import { HTMLServerConfig, VideoConstant } from '@shared/models' +import { HTMLServerConfig, VideoConstant } from '@peertube/peertube-models' @Component({ selector: 'my-video-caption-add-modal', diff --git a/client/src/app/+videos/+video-edit/shared/video-caption-edit-modal-content/video-caption-edit-modal-content.component.ts b/client/src/app/+videos/+video-edit/shared/video-caption-edit-modal-content/video-caption-edit-modal-content.component.ts index 2cb470a24..6c6a30d96 100644 --- a/client/src/app/+videos/+video-edit/shared/video-caption-edit-modal-content/video-caption-edit-modal-content.component.ts +++ b/client/src/app/+videos/+video-edit/shared/video-caption-edit-modal-content/video-caption-edit-modal-content.component.ts @@ -3,7 +3,7 @@ import { VIDEO_CAPTION_FILE_CONTENT_VALIDATOR } from '@app/shared/form-validator import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' import { VideoCaptionEdit, VideoCaptionService, VideoCaptionWithPathEdit } from '@app/shared/shared-main' import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap' -import { HTMLServerConfig, VideoConstant } from '@shared/models' +import { HTMLServerConfig, VideoConstant } from '@peertube/peertube-models' import { ServerService } from '../../../../core' /** diff --git a/client/src/app/+videos/+video-edit/shared/video-edit.component.ts b/client/src/app/+videos/+video-edit/shared/video-edit.component.ts index 460960a01..b0c1352f3 100644 --- a/client/src/app/+videos/+video-edit/shared/video-edit.component.ts +++ b/client/src/app/+videos/+video-edit/shared/video-edit.component.ts @@ -24,8 +24,6 @@ import { FormReactiveValidationMessages, FormValidatorService } from '@app/share import { InstanceService } from '@app/shared/shared-instance' import { VideoCaptionEdit, VideoCaptionWithPathEdit, VideoEdit, VideoService } from '@app/shared/shared-main' import { NgbModal } from '@ng-bootstrap/ng-bootstrap' -import { logger } from '@root-helpers/logger' -import { PluginInfo } from '@root-helpers/plugins-manager' import { HTMLServerConfig, LiveVideo, @@ -34,9 +32,12 @@ import { RegisterClientVideoFieldOptions, VideoConstant, VideoDetails, - VideoPrivacy -} from '@shared/models' -import { VideoSource } from '@shared/models/videos/video-source' + VideoPrivacy, + VideoPrivacyType, + VideoSource +} from '@peertube/peertube-models' +import { logger } from '@root-helpers/logger' +import { PluginInfo } from '@root-helpers/plugins-manager' import { I18nPrimengCalendarService } from './i18n-primeng-calendar.service' import { VideoCaptionAddModalComponent } from './video-caption-add-modal.component' import { VideoCaptionEditModalContentComponent } from './video-caption-edit-modal-content/video-caption-edit-modal-content.component' @@ -81,8 +82,8 @@ export class VideoEditComponent implements OnInit, OnDestroy { // So that it can be accessed in the template readonly SPECIAL_SCHEDULED_PRIVACY = VideoEdit.SPECIAL_SCHEDULED_PRIVACY - videoPrivacies: VideoConstant [] = [] - replayPrivacies: VideoConstant [] = [] + videoPrivacies: VideoConstant [] = [] + replayPrivacies: VideoConstant [] = [] videoCategories: VideoConstant[] = [] videoLicences: VideoConstant[] = [] videoLanguages: VideoLanguages[] = [] diff --git a/client/src/app/+videos/+video-edit/shared/video-upload.service.ts b/client/src/app/+videos/+video-edit/shared/video-upload.service.ts index 50ca1a60b..c3f8936a9 100644 --- a/client/src/app/+videos/+video-edit/shared/video-upload.service.ts +++ b/client/src/app/+videos/+video-edit/shared/video-upload.service.ts @@ -3,7 +3,7 @@ import { HttpErrorResponse, HttpEventType, HttpHeaders } from '@angular/common/h import { Injectable } from '@angular/core' import { AuthService, Notifier, ServerService } from '@app/core' import { BytesPipe, VideoService } from '@app/shared/shared-main' -import { HttpStatusCode } from '@shared/models' +import { HttpStatusCode } from '@peertube/peertube-models' import { UploaderXFormData } from './uploaderx-form-data' @Injectable() diff --git a/client/src/app/+videos/+video-edit/video-add-components/video-go-live.component.ts b/client/src/app/+videos/+video-edit/video-add-components/video-go-live.component.ts index ba612f553..f7a570ed3 100644 --- a/client/src/app/+videos/+video-edit/video-add-components/video-go-live.component.ts +++ b/client/src/app/+videos/+video-edit/video-add-components/video-go-live.component.ts @@ -16,7 +16,7 @@ import { PeerTubeProblemDocument, ServerErrorCode, VideoPrivacy -} from '@shared/models' +} from '@peertube/peertube-models' import { VideoSend } from './video-send' @Component({ diff --git a/client/src/app/+videos/+video-edit/video-add-components/video-import-torrent.component.ts b/client/src/app/+videos/+video-edit/video-add-components/video-import-torrent.component.ts index 4a1408a4a..97517e1c7 100644 --- a/client/src/app/+videos/+video-edit/video-add-components/video-import-torrent.component.ts +++ b/client/src/app/+videos/+video-edit/video-add-components/video-import-torrent.component.ts @@ -7,7 +7,7 @@ import { FormReactiveService } from '@app/shared/shared-forms' import { VideoCaptionService, VideoEdit, VideoImportService, VideoService } from '@app/shared/shared-main' import { LoadingBarService } from '@ngx-loading-bar/core' import { logger } from '@root-helpers/logger' -import { PeerTubeProblemDocument, ServerErrorCode, VideoUpdate } from '@shared/models' +import { PeerTubeProblemDocument, ServerErrorCode, VideoUpdate } from '@peertube/peertube-models' import { hydrateFormFromVideo } from '../shared/video-edit-utils' import { VideoSend } from './video-send' diff --git a/client/src/app/+videos/+video-edit/video-add-components/video-import-url.component.ts b/client/src/app/+videos/+video-edit/video-add-components/video-import-url.component.ts index 502f3818e..634bd9914 100644 --- a/client/src/app/+videos/+video-edit/video-add-components/video-import-url.component.ts +++ b/client/src/app/+videos/+video-edit/video-add-components/video-import-url.component.ts @@ -8,7 +8,7 @@ import { FormReactiveService } from '@app/shared/shared-forms' import { VideoCaptionService, VideoEdit, VideoImportService, VideoService } from '@app/shared/shared-main' import { LoadingBarService } from '@ngx-loading-bar/core' import { logger } from '@root-helpers/logger' -import { VideoUpdate } from '@shared/models' +import { VideoUpdate } from '@peertube/peertube-models' import { hydrateFormFromVideo } from '../shared/video-edit-utils' import { VideoSend } from './video-send' diff --git a/client/src/app/+videos/+video-edit/video-add-components/video-send.ts b/client/src/app/+videos/+video-edit/video-add-components/video-send.ts index 9de373cd3..56dcfa0e6 100644 --- a/client/src/app/+videos/+video-edit/video-add-components/video-send.ts +++ b/client/src/app/+videos/+video-edit/video-add-components/video-send.ts @@ -6,16 +6,16 @@ import { listUserChannelsForSelect } from '@app/helpers' import { FormReactive } from '@app/shared/shared-forms' import { VideoCaptionEdit, VideoCaptionService, VideoEdit, VideoService } from '@app/shared/shared-main' import { LoadingBarService } from '@ngx-loading-bar/core' -import { HTMLServerConfig, VideoConstant, VideoPrivacy } from '@shared/models' +import { HTMLServerConfig, VideoConstant, VideoPrivacyType } from '@peertube/peertube-models' @Directive() // eslint-disable-next-line @angular-eslint/directive-class-suffix export abstract class VideoSend extends FormReactive implements OnInit { userVideoChannels: SelectChannelItem[] = [] - videoPrivacies: VideoConstant[] = [] + videoPrivacies: VideoConstant[] = [] videoCaptions: VideoCaptionEdit[] = [] - firstStepPrivacyId: VideoPrivacy + firstStepPrivacyId: VideoPrivacyType firstStepChannelId: number abstract firstStepDone: EventEmitter @@ -31,7 +31,7 @@ export abstract class VideoSend extends FormReactive implements OnInit { protected serverConfig: HTMLServerConfig - protected highestPrivacy: VideoPrivacy + protected highestPrivacy: VideoPrivacyType abstract canDeactivate (): CanComponentDeactivateResult diff --git a/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.ts b/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.ts index f7e1872a5..cbf43ee5f 100644 --- a/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.ts +++ b/client/src/app/+videos/+video-edit/video-add-components/video-upload.component.ts @@ -10,7 +10,7 @@ import { FormReactiveService } from '@app/shared/shared-forms' import { Video, VideoCaptionService, VideoEdit, VideoService } from '@app/shared/shared-main' import { LoadingBarService } from '@ngx-loading-bar/core' import { logger } from '@root-helpers/logger' -import { HttpStatusCode, VideoCreateResult } from '@shared/models' +import { HttpStatusCode, VideoCreateResult } from '@peertube/peertube-models' import { VideoUploadService } from '../shared/video-upload.service' import { VideoSend } from './video-send' diff --git a/client/src/app/+videos/+video-edit/video-add.component.ts b/client/src/app/+videos/+video-edit/video-add.component.ts index 460c37a38..413fe4780 100644 --- a/client/src/app/+videos/+video-edit/video-add.component.ts +++ b/client/src/app/+videos/+video-edit/video-add.component.ts @@ -8,7 +8,7 @@ import { ServerService, UserService } from '@app/core' -import { HTMLServerConfig } from '@shared/models' +import { HTMLServerConfig } from '@peertube/peertube-models' import { VideoEditType } from './shared/video-edit.type' import { VideoGoLiveComponent } from './video-add-components/video-go-live.component' import { VideoImportTorrentComponent } from './video-add-components/video-import-torrent.component' diff --git a/client/src/app/+videos/+video-edit/video-update.component.ts b/client/src/app/+videos/+video-edit/video-update.component.ts index 6ad08cbad..24d91a69a 100644 --- a/client/src/app/+videos/+video-edit/video-update.component.ts +++ b/client/src/app/+videos/+video-edit/video-update.component.ts @@ -12,9 +12,8 @@ import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' import { Video, VideoCaptionEdit, VideoCaptionService, VideoDetails, VideoEdit, VideoService } from '@app/shared/shared-main' import { LiveVideoService } from '@app/shared/shared-video-live' import { LoadingBarService } from '@ngx-loading-bar/core' -import { pick, simpleObjectsDeepEqual } from '@shared/core-utils' -import { HttpStatusCode, LiveVideo, LiveVideoUpdate, VideoPrivacy, VideoState } from '@shared/models' -import { VideoSource } from '@shared/models/videos/video-source' +import { pick, simpleObjectsDeepEqual } from '@peertube/peertube-core-utils' +import { HttpStatusCode, LiveVideo, LiveVideoUpdate, VideoPrivacy, VideoSource, VideoState } from '@peertube/peertube-models' import { hydrateFormFromVideo } from './shared/video-edit-utils' import { VideoUploadService } from './shared/video-upload.service' diff --git a/client/src/app/+videos/+video-edit/video-update.resolver.ts b/client/src/app/+videos/+video-edit/video-update.resolver.ts index 2c99b36a8..d114bfb2d 100644 --- a/client/src/app/+videos/+video-edit/video-update.resolver.ts +++ b/client/src/app/+videos/+video-edit/video-update.resolver.ts @@ -4,9 +4,9 @@ import { Injectable } from '@angular/core' import { ActivatedRouteSnapshot } from '@angular/router' import { AuthService } from '@app/core' import { listUserChannelsForSelect } from '@app/helpers' -import { VideoCaptionService, VideoDetails, VideoService, VideoPasswordService } from '@app/shared/shared-main' +import { VideoCaptionService, VideoDetails, VideoPasswordService, VideoService } from '@app/shared/shared-main' import { LiveVideoService } from '@app/shared/shared-video-live' -import { VideoPrivacy } from '@shared/models/videos' +import { VideoPrivacy } from '@peertube/peertube-models' @Injectable() export class VideoUpdateResolver { diff --git a/client/src/app/+videos/+video-watch/shared/action-buttons/action-buttons.component.ts b/client/src/app/+videos/+video-watch/shared/action-buttons/action-buttons.component.ts index e6c0d4de1..7c89b7e62 100644 --- a/client/src/app/+videos/+video-watch/shared/action-buttons/action-buttons.component.ts +++ b/client/src/app/+videos/+video-watch/shared/action-buttons/action-buttons.component.ts @@ -5,7 +5,7 @@ import { VideoShareComponent } from '@app/shared/shared-share-modal' import { SupportModalComponent } from '@app/shared/shared-support-modal' import { VideoActionsDisplayType, VideoDownloadComponent } from '@app/shared/shared-video-miniature' import { VideoPlaylist } from '@app/shared/shared-video-playlist' -import { UserVideoRateType, VideoCaption, VideoPrivacy } from '@shared/models/videos' +import { UserVideoRateType, VideoCaption, VideoPrivacy } from '@peertube/peertube-models' @Component({ selector: 'my-action-buttons', diff --git a/client/src/app/+videos/+video-watch/shared/action-buttons/video-rate.component.ts b/client/src/app/+videos/+video-watch/shared/action-buttons/video-rate.component.ts index 11966ce34..13a709cb0 100644 --- a/client/src/app/+videos/+video-watch/shared/action-buttons/video-rate.component.ts +++ b/client/src/app/+videos/+video-watch/shared/action-buttons/video-rate.component.ts @@ -3,7 +3,7 @@ import { Observable } from 'rxjs' import { Component, EventEmitter, Input, OnChanges, OnDestroy, OnInit, Output } from '@angular/core' import { Notifier, ScreenService } from '@app/core' import { VideoDetails, VideoService } from '@app/shared/shared-main' -import { UserVideoRateType } from '@shared/models' +import { UserVideoRateType } from '@peertube/peertube-models' @Component({ selector: 'my-video-rate', diff --git a/client/src/app/+videos/+video-watch/shared/comment/video-comment-add.component.ts b/client/src/app/+videos/+video-watch/shared/comment/video-comment-add.component.ts index 1d9e10d0a..a01bd31fe 100644 --- a/client/src/app/+videos/+video-watch/shared/comment/video-comment-add.component.ts +++ b/client/src/app/+videos/+video-watch/shared/comment/video-comment-add.component.ts @@ -19,7 +19,7 @@ import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' import { Video } from '@app/shared/shared-main' import { VideoComment, VideoCommentService } from '@app/shared/shared-video-comment' import { NgbModal } from '@ng-bootstrap/ng-bootstrap' -import { VideoCommentCreate } from '@shared/models' +import { VideoCommentCreate } from '@peertube/peertube-models' @Component({ selector: 'my-video-comment-add', diff --git a/client/src/app/+videos/+video-watch/shared/comment/video-comment.component.ts b/client/src/app/+videos/+video-watch/shared/comment/video-comment.component.ts index 4c85df657..14422010f 100644 --- a/client/src/app/+videos/+video-watch/shared/comment/video-comment.component.ts +++ b/client/src/app/+videos/+video-watch/shared/comment/video-comment.component.ts @@ -5,7 +5,7 @@ import { AuthService } from '@app/core/auth' import { Account, DropdownAction, Video } from '@app/shared/shared-main' import { CommentReportComponent } from '@app/shared/shared-moderation/report-modals/comment-report.component' import { VideoComment, VideoCommentThreadTree } from '@app/shared/shared-video-comment' -import { User, UserRight } from '@shared/models' +import { User, UserRight } from '@peertube/peertube-models' @Component({ selector: 'my-video-comment', diff --git a/client/src/app/+videos/+video-watch/shared/comment/video-comments.component.ts b/client/src/app/+videos/+video-watch/shared/comment/video-comments.component.ts index 848936f91..1a8c89bdc 100644 --- a/client/src/app/+videos/+video-watch/shared/comment/video-comments.component.ts +++ b/client/src/app/+videos/+video-watch/shared/comment/video-comments.component.ts @@ -5,7 +5,7 @@ import { AuthService, ComponentPagination, ConfirmService, hasMoreItems, Notifie import { HooksService } from '@app/core/plugins/hooks.service' import { Syndication, VideoDetails } from '@app/shared/shared-main' import { VideoComment, VideoCommentService, VideoCommentThreadTree } from '@app/shared/shared-video-comment' -import { PeerTubeProblemDocument, ServerErrorCode } from '@shared/models' +import { PeerTubeProblemDocument, ServerErrorCode } from '@peertube/peertube-models' @Component({ selector: 'my-video-comments', diff --git a/client/src/app/+videos/+video-watch/shared/information/privacy-concerns.component.ts b/client/src/app/+videos/+video-watch/shared/information/privacy-concerns.component.ts index b51457e02..d2f8aa45d 100644 --- a/client/src/app/+videos/+video-watch/shared/information/privacy-concerns.component.ts +++ b/client/src/app/+videos/+video-watch/shared/information/privacy-concerns.component.ts @@ -2,7 +2,7 @@ import { Component, Input, OnInit } from '@angular/core' import { ServerService, User, UserService } from '@app/core' import { peertubeLocalStorage } from '@root-helpers/peertube-web-storage' import { isP2PEnabled } from '@root-helpers/video' -import { HTMLServerConfig, Video } from '@shared/models' +import { HTMLServerConfig, Video } from '@peertube/peertube-models' @Component({ selector: 'my-privacy-concerns', diff --git a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts index 8781ead7e..497c48813 100644 --- a/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts +++ b/client/src/app/+videos/+video-watch/shared/information/video-alert.component.ts @@ -1,7 +1,7 @@ import { Component, Input } from '@angular/core' import { AuthUser } from '@app/core' import { VideoDetails } from '@app/shared/shared-main' -import { VideoPrivacy, VideoState } from '@shared/models' +import { VideoPrivacy, VideoState } from '@peertube/peertube-models' @Component({ selector: 'my-video-alert', diff --git a/client/src/app/+videos/+video-watch/shared/playlist/video-watch-playlist.component.ts b/client/src/app/+videos/+video-watch/shared/playlist/video-watch-playlist.component.ts index 97d71a510..eca7cf87c 100644 --- a/client/src/app/+videos/+video-watch/shared/playlist/video-watch-playlist.component.ts +++ b/client/src/app/+videos/+video-watch/shared/playlist/video-watch-playlist.component.ts @@ -5,7 +5,7 @@ import { isInViewport } from '@app/helpers' import { VideoPlaylist, VideoPlaylistElement, VideoPlaylistService } from '@app/shared/shared-video-playlist' import { getBoolOrDefault } from '@root-helpers/local-storage-utils' import { peertubeSessionStorage } from '@root-helpers/peertube-web-storage' -import { VideoPlaylistPrivacy } from '@shared/models' +import { VideoPlaylistPrivacy } from '@peertube/peertube-models' @Component({ selector: 'my-video-watch-playlist', diff --git a/client/src/app/+videos/+video-watch/shared/recommendations/recent-videos-recommendation.service.ts b/client/src/app/+videos/+video-watch/shared/recommendations/recent-videos-recommendation.service.ts index ba0d30f3d..174fd6610 100644 --- a/client/src/app/+videos/+video-watch/shared/recommendations/recent-videos-recommendation.service.ts +++ b/client/src/app/+videos/+video-watch/shared/recommendations/recent-videos-recommendation.service.ts @@ -4,7 +4,7 @@ import { Injectable } from '@angular/core' import { ServerService, UserService } from '@app/core' import { Video, VideoService } from '@app/shared/shared-main' import { AdvancedSearch, SearchService } from '@app/shared/shared-search' -import { HTMLServerConfig } from '@shared/models' +import { HTMLServerConfig } from '@peertube/peertube-models' import { RecommendationInfo } from './recommendation-info.model' import { RecommendationService } from './recommendations.service' diff --git a/client/src/app/+videos/+video-watch/video-watch.component.ts b/client/src/app/+videos/+video-watch/video-watch.component.ts index 45de62519..f109427cc 100644 --- a/client/src/app/+videos/+video-watch/video-watch.component.ts +++ b/client/src/app/+videos/+video-watch/video-watch.component.ts @@ -22,9 +22,7 @@ import { Video, VideoCaptionService, VideoDetails, VideoFileTokenService, VideoS import { SubscribeButtonComponent } from '@app/shared/shared-user-subscription' import { LiveVideoService } from '@app/shared/shared-video-live' import { VideoPlaylist, VideoPlaylistService } from '@app/shared/shared-video-playlist' -import { logger } from '@root-helpers/logger' -import { isP2PEnabled, videoRequiresFileToken, videoRequiresUserAuth } from '@root-helpers/video' -import { timeToInt } from '@shared/core-utils' +import { timeToInt } from '@peertube/peertube-core-utils' import { HTMLServerConfig, HttpStatusCode, @@ -34,8 +32,11 @@ import { Storyboard, VideoCaption, VideoPrivacy, - VideoState -} from '@shared/models' + VideoState, + VideoStateType +} from '@peertube/peertube-models' +import { logger } from '@root-helpers/logger' +import { isP2PEnabled, videoRequiresFileToken, videoRequiresUserAuth } from '@root-helpers/video' import { HLSOptions, PeerTubePlayer, @@ -812,7 +813,7 @@ export class VideoWatchComponent implements OnInit, OnDestroy { }) } - private handleLiveStateChange (newState: VideoState) { + private handleLiveStateChange (newState: VideoStateType) { if (newState !== VideoState.PUBLISHED) return logger.info('Loading video after live update.') diff --git a/client/src/app/+videos/video-list/overview/overview.service.ts b/client/src/app/+videos/video-list/overview/overview.service.ts index 67a1adb28..581c6b846 100644 --- a/client/src/app/+videos/video-list/overview/overview.service.ts +++ b/client/src/app/+videos/video-list/overview/overview.service.ts @@ -5,9 +5,8 @@ import { Injectable } from '@angular/core' import { RestExtractor, ServerService } from '@app/core' import { immutableAssign } from '@app/helpers' import { VideoService } from '@app/shared/shared-main' -import { objectKeysTyped } from '@shared/core-utils' -import { peertubeTranslate } from '@shared/core-utils/i18n' -import { VideosOverview as VideosOverviewServer } from '@shared/models' +import { objectKeysTyped, peertubeTranslate } from '@peertube/peertube-core-utils' +import { VideosOverview as VideosOverviewServer } from '@peertube/peertube-models' import { environment } from '../../../../environments/environment' import { VideosOverview } from './videos-overview.model' diff --git a/client/src/app/+videos/video-list/overview/videos-overview.model.ts b/client/src/app/+videos/video-list/overview/videos-overview.model.ts index 6765ad9b7..1208d5a00 100644 --- a/client/src/app/+videos/video-list/overview/videos-overview.model.ts +++ b/client/src/app/+videos/video-list/overview/videos-overview.model.ts @@ -1,5 +1,5 @@ import { Video } from '@app/shared/shared-main' -import { VideoChannelSummary, VideoConstant, VideosOverview as VideosOverviewServer } from '@shared/models' +import { VideoChannelSummary, VideoConstant, VideosOverview as VideosOverviewServer } from '@peertube/peertube-models' export class VideosOverview implements VideosOverviewServer { channels: { diff --git a/client/src/app/+videos/video-list/video-user-subscriptions.component.ts b/client/src/app/+videos/video-list/video-user-subscriptions.component.ts index 04f02c138..c9926488d 100644 --- a/client/src/app/+videos/video-list/video-user-subscriptions.component.ts +++ b/client/src/app/+videos/video-list/video-user-subscriptions.component.ts @@ -7,7 +7,7 @@ import { HooksService } from '@app/core/plugins/hooks.service' import { VideoService } from '@app/shared/shared-main' import { UserSubscriptionService } from '@app/shared/shared-user-subscription' import { VideoFilters } from '@app/shared/shared-video-miniature' -import { VideoSortField } from '@shared/models' +import { VideoSortField } from '@peertube/peertube-models' @Component({ selector: 'my-videos-user-subscriptions', diff --git a/client/src/app/+videos/video-list/videos-list-common-page.component.ts b/client/src/app/+videos/video-list/videos-list-common-page.component.ts index c8fa8ef30..32954f373 100644 --- a/client/src/app/+videos/video-list/videos-list-common-page.component.ts +++ b/client/src/app/+videos/video-list/videos-list-common-page.component.ts @@ -4,7 +4,7 @@ import { ComponentPaginationLight, DisableForReuseHook, MetaService, RedirectSer import { HooksService } from '@app/core/plugins/hooks.service' import { VideoService } from '@app/shared/shared-main' import { VideoFilters, VideoFilterScope } from '@app/shared/shared-video-miniature/video-filters.model' -import { ClientFilterHookName, VideoSortField } from '@shared/models' +import { ClientFilterHookName, VideoSortField } from '@peertube/peertube-models' import { Subscription } from 'rxjs' export type VideosListCommonPageRouteData = { diff --git a/client/src/app/app-routing.module.ts b/client/src/app/app-routing.module.ts index 40e4ec35d..360524d69 100644 --- a/client/src/app/app-routing.module.ts +++ b/client/src/app/app-routing.module.ts @@ -2,7 +2,7 @@ import { NgModule } from '@angular/core' import { RouteReuseStrategy, RouterModule, Routes, UrlMatchResult, UrlSegment } from '@angular/router' import { CustomReuseStrategy } from '@app/core/routing/custom-reuse-strategy' import { MenuGuards } from '@app/core/routing/menu-guard.service' -import { POSSIBLE_LOCALES } from '@shared/core-utils/i18n' +import { POSSIBLE_LOCALES } from '@peertube/peertube-core-utils' import { HomepageRedirectComponent, MetaGuard, PreloadSelectedModulesList } from './core' import { EmptyComponent } from './empty.component' import { USER_USERNAME_REGEX_CHARACTERS } from './shared/form-validators/user-validators' diff --git a/client/src/app/app.component.ts b/client/src/app/app.component.ts index f6d90cb64..4ad20bfd6 100644 --- a/client/src/app/app.component.ts +++ b/client/src/app/app.component.ts @@ -25,10 +25,10 @@ import { CustomModalComponent } from '@app/modal/custom-modal.component' import { InstanceConfigWarningModalComponent } from '@app/modal/instance-config-warning-modal.component' import { NgbConfig, NgbModal } from '@ng-bootstrap/ng-bootstrap' import { LoadingBarService } from '@ngx-loading-bar/core' +import { getShortLocale } from '@peertube/peertube-core-utils' +import { BroadcastMessageLevel, HTMLServerConfig, UserRole } from '@peertube/peertube-models' import { logger } from '@root-helpers/logger' import { peertubeLocalStorage } from '@root-helpers/peertube-web-storage' -import { getShortLocale } from '@shared/core-utils/i18n' -import { BroadcastMessageLevel, HTMLServerConfig, UserRole } from '@shared/models' import { MenuService } from './core/menu/menu.service' import { POP_STATE_MODAL_DISMISS } from './helpers' import { GlobalIconName } from './shared/shared-icons' diff --git a/client/src/app/core/auth/auth-user.model.ts b/client/src/app/core/auth/auth-user.model.ts index 226075265..a1e23bfbb 100644 --- a/client/src/app/core/auth/auth-user.model.ts +++ b/client/src/app/core/auth/auth-user.model.ts @@ -1,16 +1,16 @@ import { Observable, of } from 'rxjs' import { map } from 'rxjs/operators' import { User } from '@app/core/users/user.model' -import { OAuthUserTokens } from '@root-helpers/users' -import { hasUserRight } from '@shared/core-utils/users' +import { hasUserRight } from '@peertube/peertube-core-utils' import { MyUser as ServerMyUserModel, MyUserSpecialPlaylist, User as ServerUserModel, - UserRight, + UserRightType, UserRole, UserVideoQuota -} from '@shared/models' +} from '@peertube/peertube-models' +import { OAuthUserTokens } from '@root-helpers/users' export class AuthUser extends User implements ServerMyUserModel { oauthTokens: OAuthUserTokens @@ -42,7 +42,7 @@ export class AuthUser extends User implements ServerMyUserModel { this.oauthTokens.refreshToken = refreshToken } - hasRight (right: UserRight) { + hasRight (right: UserRightType) { return hasUserRight(this.role.id, right) } diff --git a/client/src/app/core/auth/auth.service.ts b/client/src/app/core/auth/auth.service.ts index 6fe601d8d..bc67ab7a0 100644 --- a/client/src/app/core/auth/auth.service.ts +++ b/client/src/app/core/auth/auth.service.ts @@ -6,7 +6,7 @@ import { Injectable } from '@angular/core' import { Router } from '@angular/router' import { Notifier } from '@app/core/notification/notifier.service' import { logger, OAuthUserTokens, objectToUrlEncoded, peertubeLocalStorage } from '@root-helpers/index' -import { HttpStatusCode, MyUser as UserServerModel, OAuthClientLocal, User, UserLogin, UserRefreshToken } from '@shared/models' +import { HttpStatusCode, MyUser as UserServerModel, OAuthClientLocal, User, UserLogin, UserRefreshToken } from '@peertube/peertube-models' import { environment } from '../../../environments/environment' import { RestExtractor } from '../rest/rest-extractor.service' import { RedirectService } from '../routing' diff --git a/client/src/app/core/menu/menu.service.ts b/client/src/app/core/menu/menu.service.ts index d865c7da2..39e89f6e3 100644 --- a/client/src/app/core/menu/menu.service.ts +++ b/client/src/app/core/menu/menu.service.ts @@ -2,7 +2,7 @@ import { fromEvent } from 'rxjs' import { debounceTime } from 'rxjs/operators' import { Injectable } from '@angular/core' import { GlobalIconName } from '@app/shared/shared-icons' -import { HTMLServerConfig } from '@shared/models/server' +import { HTMLServerConfig } from '@peertube/peertube-models' import { ScreenService } from '../wrappers' export type MenuLink = { diff --git a/client/src/app/core/notification/peertube-socket.service.ts b/client/src/app/core/notification/peertube-socket.service.ts index 50a11e948..15af9a310 100644 --- a/client/src/app/core/notification/peertube-socket.service.ts +++ b/client/src/app/core/notification/peertube-socket.service.ts @@ -1,7 +1,7 @@ import { Subject } from 'rxjs' import { ManagerOptions, Socket, SocketOptions } from 'socket.io-client' import { Injectable } from '@angular/core' -import { LiveVideoEventPayload, LiveVideoEventType, UserNotification as UserNotificationServer } from '@shared/models' +import { LiveVideoEventPayload, LiveVideoEventType, UserNotification as UserNotificationServer } from '@peertube/peertube-models' import { environment } from '../../../environments/environment' import { AuthService } from '../auth' diff --git a/client/src/app/core/plugins/hooks.service.ts b/client/src/app/core/plugins/hooks.service.ts index d9fef8389..59c627c05 100644 --- a/client/src/app/core/plugins/hooks.service.ts +++ b/client/src/app/core/plugins/hooks.service.ts @@ -3,7 +3,7 @@ import { mergeMap, switchMap } from 'rxjs/operators' import { Injectable } from '@angular/core' import { PluginService } from '@app/core/plugins/plugin.service' import { logger } from '@root-helpers/logger' -import { ClientActionHookName, ClientFilterHookName, PluginClientScope } from '@shared/models' +import { ClientActionHookName, ClientFilterHookName, PluginClientScope } from '@peertube/peertube-models' import { AuthService, AuthStatus } from '../auth' type RawFunction = (params: U) => T diff --git a/client/src/app/core/plugins/plugin.service.ts b/client/src/app/core/plugins/plugin.service.ts index bd8c61d9a..d37b2e5f7 100644 --- a/client/src/app/core/plugins/plugin.service.ts +++ b/client/src/app/core/plugins/plugin.service.ts @@ -10,22 +10,22 @@ import { RestExtractor } from '@app/core/rest' import { ServerService } from '@app/core/server/server.service' import { getDevLocale, isOnDevLocale } from '@app/helpers' import { CustomModalComponent } from '@app/modal/custom-modal.component' -import { PluginInfo, PluginsManager } from '@root-helpers/plugins-manager' -import { getKeys } from '@shared/core-utils' -import { getCompleteLocale, isDefaultLocale, peertubeTranslate } from '@shared/core-utils/i18n' +import { getCompleteLocale, getKeys, isDefaultLocale, peertubeTranslate } from '@peertube/peertube-core-utils' import { ClientHook, ClientHookName, PluginClientScope, PluginTranslation, PluginType, + PluginType_Type, PublicServerSetting, RegisterClientFormFieldOptions, RegisterClientRouteOptions, RegisterClientSettingsScriptOptions, RegisterClientVideoFieldOptions, ServerConfigPlugin -} from '@shared/models' +} from '@peertube/peertube-models' +import { PluginInfo, PluginsManager } from '@root-helpers/plugins-manager' import { environment } from '../../../environments/environment' import { RegisterClientHelpers } from '../../../types/register-client-option.model' @@ -110,7 +110,7 @@ export class PluginService implements ClientHook { return this.pluginsManager.removePlugin(plugin) } - nameToNpmName (name: string, type: PluginType) { + nameToNpmName (name: string, type: PluginType_Type) { const prefix = type === PluginType.PLUGIN ? 'peertube-plugin-' : 'peertube-theme-' diff --git a/client/src/app/core/renderer/html-renderer.service.ts b/client/src/app/core/renderer/html-renderer.service.ts index 7776ccad5..37741c079 100644 --- a/client/src/app/core/renderer/html-renderer.service.ts +++ b/client/src/app/core/renderer/html-renderer.service.ts @@ -1,5 +1,5 @@ import { Injectable } from '@angular/core' -import { getCustomMarkupSanitizeOptions, getDefaultSanitizeOptions } from '@shared/core-utils/renderer/html' +import { getCustomMarkupSanitizeOptions, getDefaultSanitizeOptions } from '@peertube/peertube-core-utils' import { LinkifierService } from './linkifier.service' @Injectable() diff --git a/client/src/app/core/renderer/markdown.service.ts b/client/src/app/core/renderer/markdown.service.ts index ec3b683bb..907b92232 100644 --- a/client/src/app/core/renderer/markdown.service.ts +++ b/client/src/app/core/renderer/markdown.service.ts @@ -1,13 +1,14 @@ import * as MarkdownIt from 'markdown-it' import { Injectable } from '@angular/core' -import { buildVideoLink, decorateVideoLink } from '@shared/core-utils' import { + buildVideoLink, COMPLETE_RULES, + decorateVideoLink, ENHANCED_RULES, ENHANCED_WITH_HTML_RULES, TEXT_RULES, TEXT_WITH_HTML_RULES -} from '@shared/core-utils/renderer/markdown' +} from '@peertube/peertube-core-utils' import { HtmlRendererService } from './html-renderer.service' type MarkdownParsers = { diff --git a/client/src/app/core/rest/rest-extractor.service.ts b/client/src/app/core/rest/rest-extractor.service.ts index c6c1e183f..bcc50c0f4 100644 --- a/client/src/app/core/rest/rest-extractor.service.ts +++ b/client/src/app/core/rest/rest-extractor.service.ts @@ -1,10 +1,10 @@ import { throwError as observableThrowError } from 'rxjs' +import { HttpHeaderResponse } from '@angular/common/http' import { Inject, Injectable, LOCALE_ID } from '@angular/core' import { Router } from '@angular/router' import { DateFormat, dateToHuman } from '@app/helpers' +import { HttpStatusCode, HttpStatusCodeType, ResultList } from '@peertube/peertube-models' import { logger } from '@root-helpers/logger' -import { HttpStatusCode, ResultList } from '@shared/models' -import { HttpHeaderResponse } from '@angular/common/http' @Injectable() export class RestExtractor { @@ -45,7 +45,11 @@ export class RestExtractor { return target } - redirectTo404IfNotFound (obj: { status: number }, type: 'video' | 'other', status = [ HttpStatusCode.NOT_FOUND_404 ]) { + redirectTo404IfNotFound ( + obj: { status: HttpStatusCodeType }, + type: 'video' | 'other', + status: HttpStatusCodeType[] = [ HttpStatusCode.NOT_FOUND_404 ] + ) { if (obj?.status && status.includes(obj.status)) { // Do not use redirectService to avoid circular dependencies this.router.navigate([ '/404' ], { state: { type, obj }, skipLocationChange: true }) diff --git a/client/src/app/core/routing/homepage-redirect.component.ts b/client/src/app/core/routing/homepage-redirect.component.ts index 9e3848038..e9be832e0 100644 --- a/client/src/app/core/routing/homepage-redirect.component.ts +++ b/client/src/app/core/routing/homepage-redirect.component.ts @@ -1,6 +1,6 @@ import { Component, OnInit } from '@angular/core' import { ActivatedRoute } from '@angular/router' -import { is18nPath } from '@shared/core-utils/i18n/i18n' +import { is18nPath } from '@peertube/peertube-core-utils' import { RedirectService } from './redirect.service' /* diff --git a/client/src/app/core/routing/meta.service.ts b/client/src/app/core/routing/meta.service.ts index 97e440faf..d50f0d65a 100644 --- a/client/src/app/core/routing/meta.service.ts +++ b/client/src/app/core/routing/meta.service.ts @@ -1,6 +1,6 @@ import { Injectable } from '@angular/core' import { Meta, Title } from '@angular/platform-browser' -import { HTMLServerConfig } from '@shared/models/server' +import { HTMLServerConfig } from '@peertube/peertube-models' import { ServerService } from '../server' export interface MetaSettings { diff --git a/client/src/app/core/scoped-tokens/scoped-tokens.service.ts b/client/src/app/core/scoped-tokens/scoped-tokens.service.ts index 038e5031c..f7d192feb 100644 --- a/client/src/app/core/scoped-tokens/scoped-tokens.service.ts +++ b/client/src/app/core/scoped-tokens/scoped-tokens.service.ts @@ -1,7 +1,7 @@ import { catchError } from 'rxjs/operators' import { HttpClient } from '@angular/common/http' import { Injectable } from '@angular/core' -import { ScopedToken } from '@shared/models/users/user-scoped-token' +import { ScopedToken } from '@peertube/peertube-models' import { environment } from '../../../environments/environment' import { RestExtractor } from '../rest' diff --git a/client/src/app/core/server/server.service.ts b/client/src/app/core/server/server.service.ts index 41cb4791a..75ac8ddc1 100644 --- a/client/src/app/core/server/server.service.ts +++ b/client/src/app/core/server/server.service.ts @@ -3,9 +3,16 @@ import { first, map, share, shareReplay, switchMap, tap } from 'rxjs/operators' import { HttpClient } from '@angular/common/http' import { Inject, Injectable, LOCALE_ID } from '@angular/core' import { getDevLocale, isOnDevLocale } from '@app/helpers' +import { getCompleteLocale, isDefaultLocale, peertubeTranslate } from '@peertube/peertube-core-utils' +import { + HTMLServerConfig, + ServerConfig, + ServerStats, + VideoConstant, + VideoPlaylistPrivacyType, + VideoPrivacyType +} from '@peertube/peertube-models' import { logger } from '@root-helpers/logger' -import { getCompleteLocale, isDefaultLocale, peertubeTranslate } from '@shared/core-utils/i18n' -import { HTMLServerConfig, ServerConfig, ServerStats, VideoConstant } from '@shared/models' import { environment } from '../../../environments/environment' @Injectable() @@ -21,8 +28,8 @@ export class ServerService { private localeObservable: Observable private videoLicensesObservable: Observable[]> private videoCategoriesObservable: Observable[]> - private videoPrivaciesObservable: Observable[]> - private videoPlaylistPrivaciesObservable: Observable[]> + private videoPrivaciesObservable: Observable[]> + private videoPlaylistPrivaciesObservable: Observable[]> private videoLanguagesObservable: Observable[]> private configObservable: Observable @@ -123,7 +130,7 @@ export class ServerService { getVideoPrivacies () { if (!this.videoPrivaciesObservable) { - this.videoPrivaciesObservable = this.loadAttributeEnum(ServerService.BASE_VIDEO_URL, 'privacies') + this.videoPrivaciesObservable = this.loadAttributeEnum(ServerService.BASE_VIDEO_URL, 'privacies') } return this.videoPrivaciesObservable.pipe(first()) @@ -131,7 +138,10 @@ export class ServerService { getVideoPlaylistPrivacies () { if (!this.videoPlaylistPrivaciesObservable) { - this.videoPlaylistPrivaciesObservable = this.loadAttributeEnum(ServerService.BASE_VIDEO_PLAYLIST_URL, 'privacies') + this.videoPlaylistPrivaciesObservable = this.loadAttributeEnum( + ServerService.BASE_VIDEO_PLAYLIST_URL, + 'privacies' + ) } return this.videoPlaylistPrivaciesObservable.pipe(first()) diff --git a/client/src/app/core/theme/theme.service.ts b/client/src/app/core/theme/theme.service.ts index ead1770ba..22eb5ddd3 100644 --- a/client/src/app/core/theme/theme.service.ts +++ b/client/src/app/core/theme/theme.service.ts @@ -2,7 +2,7 @@ import { Injectable } from '@angular/core' import { logger } from '@root-helpers/logger' import { capitalizeFirstLetter } from '@root-helpers/string' import { UserLocalStorageKeys } from '@root-helpers/users' -import { HTMLServerConfig, ServerConfigTheme } from '@shared/models' +import { HTMLServerConfig, ServerConfigTheme } from '@peertube/peertube-models' import { environment } from '../../../environments/environment' import { AuthService } from '../auth' import { PluginService } from '../plugins/plugin.service' diff --git a/client/src/app/core/users/user-local-storage.service.ts b/client/src/app/core/users/user-local-storage.service.ts index a87f3b98a..431a57343 100644 --- a/client/src/app/core/users/user-local-storage.service.ts +++ b/client/src/app/core/users/user-local-storage.service.ts @@ -1,13 +1,11 @@ - import { filter, throttleTime } from 'rxjs' import { Injectable } from '@angular/core' import { AuthService, AuthStatus } from '@app/core/auth' +import { objectKeysTyped } from '@peertube/peertube-core-utils' +import { NSFWPolicyType, UserRoleType, UserUpdateMe } from '@peertube/peertube-models' import { getBoolOrDefault } from '@root-helpers/local-storage-utils' import { logger } from '@root-helpers/logger' import { OAuthUserTokens, UserLocalStorageKeys } from '@root-helpers/users' -import { objectKeysTyped } from '@shared/core-utils' -import { UserRole, UserUpdateMe } from '@shared/models' -import { NSFWPolicyType } from '@shared/models/videos' import { ServerService } from '../server' import { LocalStorageService } from '../wrappers/storage.service' @@ -61,7 +59,7 @@ export class UserLocalStorageService { username: this.localStorageService.getItem(UserLocalStorageKeys.USERNAME), email: this.localStorageService.getItem(UserLocalStorageKeys.EMAIL), role: { - id: parseInt(this.localStorageService.getItem(UserLocalStorageKeys.ROLE), 10) as UserRole, + id: parseInt(this.localStorageService.getItem(UserLocalStorageKeys.ROLE), 10) as UserRoleType, label: '' }, @@ -74,7 +72,7 @@ export class UserLocalStorageService { username: string email: string role: { - id: UserRole + id: UserRoleType } }) { this.localStorageService.setItem(UserLocalStorageKeys.ID, user.id.toString()) diff --git a/client/src/app/core/users/user.model.ts b/client/src/app/core/users/user.model.ts index 54b749a4c..ffc7c2b44 100644 --- a/client/src/app/core/users/user.model.ts +++ b/client/src/app/core/users/user.model.ts @@ -1,17 +1,18 @@ import { Account } from '@app/shared/shared-main/account/account.model' -import { objectKeysTyped } from '@shared/core-utils' -import { hasUserRight } from '@shared/core-utils/users' +import { hasUserRight, objectKeysTyped } from '@peertube/peertube-core-utils' import { ActorImage, HTMLServerConfig, NSFWPolicyType, User as UserServerModel, UserAdminFlag, + UserAdminFlagType, UserNotificationSetting, - UserRight, + UserRightType, UserRole, + UserRoleType, VideoChannel -} from '@shared/models' +} from '@peertube/peertube-models' export class User implements UserServerModel { id: number @@ -23,7 +24,7 @@ export class User implements UserServerModel { emailPublic: boolean nsfwPolicy: NSFWPolicyType - adminFlags?: UserAdminFlag + adminFlags?: UserAdminFlagType autoPlayVideo: boolean autoPlayNextVideo: boolean @@ -35,7 +36,7 @@ export class User implements UserServerModel { videoLanguages: string[] role: { - id: UserRole + id: UserRoleType label: string } @@ -124,7 +125,7 @@ export class User implements UserServerModel { } } - hasRight (right: UserRight) { + hasRight (right: UserRightType) { return hasUserRight(this.role.id, right) } diff --git a/client/src/app/core/users/user.service.ts b/client/src/app/core/users/user.service.ts index b4024c02d..7ad0ee9bf 100644 --- a/client/src/app/core/users/user.service.ts +++ b/client/src/app/core/users/user.service.ts @@ -3,7 +3,7 @@ import { catchError, first, map, shareReplay } from 'rxjs/operators' import { HttpClient, HttpParams } from '@angular/common/http' import { Injectable } from '@angular/core' import { AuthService } from '@app/core/auth' -import { ActorImage, User as UserServerModel, UserUpdateMe, UserVideoQuota } from '@shared/models' +import { ActorImage, User as UserServerModel, UserUpdateMe, UserVideoQuota } from '@peertube/peertube-models' import { environment } from '../../../environments/environment' import { RestExtractor } from '../rest' import { UserLocalStorageService } from './user-local-storage.service' diff --git a/client/src/app/header/search-typeahead.component.ts b/client/src/app/header/search-typeahead.component.ts index d2549315c..a3a04041f 100644 --- a/client/src/app/header/search-typeahead.component.ts +++ b/client/src/app/header/search-typeahead.component.ts @@ -5,7 +5,7 @@ import { AfterViewChecked, Component, OnDestroy, OnInit, QueryList, ViewChildren import { ActivatedRoute, Params, Router } from '@angular/router' import { AuthService, ServerService } from '@app/core' import { logger } from '@root-helpers/logger' -import { HTMLServerConfig, SearchTargetType } from '@shared/models' +import { HTMLServerConfig, SearchTargetType } from '@peertube/peertube-models' import { SuggestionComponent, SuggestionPayload, SuggestionPayloadType } from './suggestion.component' @Component({ diff --git a/client/src/app/helpers/utils/channel.ts b/client/src/app/helpers/utils/channel.ts index 83f36b70f..fe59ea567 100644 --- a/client/src/app/helpers/utils/channel.ts +++ b/client/src/app/helpers/utils/channel.ts @@ -1,7 +1,7 @@ import { minBy } from 'lodash-es' import { first, map } from 'rxjs/operators' import { SelectChannelItem } from 'src/types/select-options-item.model' -import { VideoChannel } from '@shared/models' +import { VideoChannel } from '@peertube/peertube-models' import { AuthService } from '../../core/auth' function listUserChannelsForSelect (authService: AuthService) { diff --git a/client/src/app/helpers/utils/upload.ts b/client/src/app/helpers/utils/upload.ts index b60951612..b55415064 100644 --- a/client/src/app/helpers/utils/upload.ts +++ b/client/src/app/helpers/utils/upload.ts @@ -1,6 +1,6 @@ import { HttpErrorResponse } from '@angular/common/http' import { Notifier } from '@app/core' -import { HttpStatusCode } from '@shared/models' +import { HttpStatusCode } from '@peertube/peertube-models' function genericUploadErrorHandler (options: { err: Pick diff --git a/client/src/app/menu/language-chooser.component.ts b/client/src/app/menu/language-chooser.component.ts index f7ae69717..1ec5987c2 100644 --- a/client/src/app/menu/language-chooser.component.ts +++ b/client/src/app/menu/language-chooser.component.ts @@ -1,8 +1,7 @@ import { Component, ElementRef, Inject, LOCALE_ID, ViewChild } from '@angular/core' import { getDevLocale, isOnDevLocale, sortBy } from '@app/helpers' import { NgbModal } from '@ng-bootstrap/ng-bootstrap' -import { objectKeysTyped } from '@shared/core-utils' -import { getCompleteLocale, getShortLocale, I18N_LOCALES } from '@shared/core-utils/i18n' +import { getCompleteLocale, getShortLocale, I18N_LOCALES, objectKeysTyped } from '@peertube/peertube-core-utils' @Component({ selector: 'my-language-chooser', diff --git a/client/src/app/menu/menu.component.ts b/client/src/app/menu/menu.component.ts index 410abe6fa..6d309f15a 100644 --- a/client/src/app/menu/menu.component.ts +++ b/client/src/app/menu/menu.component.ts @@ -22,7 +22,7 @@ import { LanguageChooserComponent } from '@app/menu/language-chooser.component' import { QuickSettingsModalComponent } from '@app/modal/quick-settings-modal.component' import { PeertubeModalService } from '@app/shared/shared-main/peertube-modal/peertube-modal.service' import { NgbDropdown } from '@ng-bootstrap/ng-bootstrap' -import { HTMLServerConfig, ServerConfig, UserRight, VideoConstant } from '@shared/models' +import { HTMLServerConfig, ServerConfig, UserRight, UserRightType, VideoConstant } from '@peertube/peertube-models' const debugLogger = debug('peertube:menu:MenuComponent') @@ -54,7 +54,7 @@ export class MenuComponent implements OnInit, OnDestroy { private htmlServerConfig: HTMLServerConfig private serverConfig: ServerConfig - private routesPerRight: { [role in UserRight]?: string } = { + private routesPerRight: { [role in UserRightType]?: string } = { [UserRight.MANAGE_USERS]: '/admin/users', [UserRight.MANAGE_SERVER_FOLLOW]: '/admin/friends', [UserRight.MANAGE_ABUSES]: '/admin/moderation/abuses', diff --git a/client/src/app/modal/instance-config-warning-modal.component.ts b/client/src/app/modal/instance-config-warning-modal.component.ts index 23c2c777e..f8ab155ae 100644 --- a/client/src/app/modal/instance-config-warning-modal.component.ts +++ b/client/src/app/modal/instance-config-warning-modal.component.ts @@ -2,9 +2,9 @@ import { Location } from '@angular/common' import { Component, ElementRef, ViewChild } from '@angular/core' import { Notifier, User, UserService } from '@app/core' import { NgbModal } from '@ng-bootstrap/ng-bootstrap' +import { About, ServerConfig } from '@peertube/peertube-models' import { logger } from '@root-helpers/logger' import { peertubeLocalStorage } from '@root-helpers/peertube-web-storage' -import { About, ServerConfig } from '@shared/models/server' @Component({ selector: 'my-instance-config-warning-modal', diff --git a/client/src/app/shared/form-validators/video-playlist-validators.ts b/client/src/app/shared/form-validators/video-playlist-validators.ts index 63af637a3..3cddcaad2 100644 --- a/client/src/app/shared/form-validators/video-playlist-validators.ts +++ b/client/src/app/shared/form-validators/video-playlist-validators.ts @@ -1,6 +1,6 @@ import { Validators, AbstractControl } from '@angular/forms' import { BuildFormValidator } from './form-validator.model' -import { VideoPlaylistPrivacy } from '@shared/models' +import { VideoPlaylistPrivacy, VideoPlaylistPrivacyType } from '@peertube/peertube-models' export const VIDEO_PLAYLIST_DISPLAY_NAME_VALIDATOR: BuildFormValidator = { VALIDATORS: [ @@ -42,7 +42,7 @@ export const VIDEO_PLAYLIST_CHANNEL_ID_VALIDATOR: BuildFormValidator = { } } -export function setPlaylistChannelValidator (channelControl: AbstractControl, privacy: VideoPlaylistPrivacy) { +export function setPlaylistChannelValidator (channelControl: AbstractControl, privacy: VideoPlaylistPrivacyType) { if (privacy.toString() === VideoPlaylistPrivacy.PUBLIC.toString()) { channelControl.setValidators([ Validators.required ]) } else { diff --git a/client/src/app/shared/shared-abuse-list/abuse-details.component.ts b/client/src/app/shared/shared-abuse-list/abuse-details.component.ts index e15edf8c2..357dc4522 100644 --- a/client/src/app/shared/shared-abuse-list/abuse-details.component.ts +++ b/client/src/app/shared/shared-abuse-list/abuse-details.component.ts @@ -1,6 +1,6 @@ import { Component, Input, OnInit } from '@angular/core' import { durationToString } from '@app/helpers' -import { AbusePredefinedReasonsString } from '@shared/models' +import { AbusePredefinedReasonsString } from '@peertube/peertube-models' import { ProcessedAbuse } from './processed-abuse.model' @Component({ diff --git a/client/src/app/shared/shared-abuse-list/abuse-list-table.component.ts b/client/src/app/shared/shared-abuse-list/abuse-list-table.component.ts index d8470e927..c38e1286f 100644 --- a/client/src/app/shared/shared-abuse-list/abuse-list-table.component.ts +++ b/client/src/app/shared/shared-abuse-list/abuse-list-table.component.ts @@ -7,8 +7,8 @@ import { ConfirmService, MarkdownService, Notifier, RestPagination, RestTable } import { Account, Actor, DropdownAction, Video, VideoService } from '@app/shared/shared-main' import { AbuseService, BlocklistService, VideoBlockService } from '@app/shared/shared-moderation' import { VideoCommentService } from '@app/shared/shared-video-comment' +import { AbuseState, AbuseStateType, AdminAbuse } from '@peertube/peertube-models' import { logger } from '@root-helpers/logger' -import { AbuseState, AdminAbuse } from '@shared/models' import { AdvancedInputFilter } from '../shared-forms' import { AbuseMessageModalComponent } from './abuse-message-modal.component' import { ModerationCommentModalComponent } from './moderation-comment-modal.component' @@ -144,7 +144,7 @@ export class AbuseListTableComponent extends RestTable implements OnInit { }) } - updateAbuseState (abuse: AdminAbuse, state: AbuseState) { + updateAbuseState (abuse: AdminAbuse, state: AbuseStateType) { this.abuseService.updateAbuse(abuse, { state }) .subscribe({ next: () => this.reloadData(), diff --git a/client/src/app/shared/shared-abuse-list/abuse-message-modal.component.ts b/client/src/app/shared/shared-abuse-list/abuse-message-modal.component.ts index 12d503f56..8d20166f6 100644 --- a/client/src/app/shared/shared-abuse-list/abuse-message-modal.component.ts +++ b/client/src/app/shared/shared-abuse-list/abuse-message-modal.component.ts @@ -4,7 +4,7 @@ import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' import { NgbModal } from '@ng-bootstrap/ng-bootstrap' import { NgbModalRef } from '@ng-bootstrap/ng-bootstrap/modal/modal-ref' import { logger } from '@root-helpers/logger' -import { AbuseMessage, UserAbuse } from '@shared/models' +import { AbuseMessage, UserAbuse } from '@peertube/peertube-models' import { ABUSE_MESSAGE_VALIDATOR } from '../form-validators/abuse-validators' import { AbuseService } from '../shared-moderation' diff --git a/client/src/app/shared/shared-abuse-list/moderation-comment-modal.component.ts b/client/src/app/shared/shared-abuse-list/moderation-comment-modal.component.ts index 4ad807d25..e42939f96 100644 --- a/client/src/app/shared/shared-abuse-list/moderation-comment-modal.component.ts +++ b/client/src/app/shared/shared-abuse-list/moderation-comment-modal.component.ts @@ -4,7 +4,7 @@ import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' import { AbuseService } from '@app/shared/shared-moderation' import { NgbModal } from '@ng-bootstrap/ng-bootstrap' import { NgbModalRef } from '@ng-bootstrap/ng-bootstrap/modal/modal-ref' -import { AdminAbuse } from '@shared/models' +import { AdminAbuse } from '@peertube/peertube-models' import { ABUSE_MODERATION_COMMENT_VALIDATOR } from '../form-validators/abuse-validators' @Component({ diff --git a/client/src/app/shared/shared-abuse-list/processed-abuse.model.ts b/client/src/app/shared/shared-abuse-list/processed-abuse.model.ts index b9a9bd889..076ccb40b 100644 --- a/client/src/app/shared/shared-abuse-list/processed-abuse.model.ts +++ b/client/src/app/shared/shared-abuse-list/processed-abuse.model.ts @@ -1,5 +1,5 @@ import { Account } from '@app/shared/shared-main' -import { AdminAbuse } from '@shared/models' +import { AdminAbuse } from '@peertube/peertube-models' // Don't use an abuse model because we need external services to compute some properties // And this model is only used in this component diff --git a/client/src/app/shared/shared-actor-image/actor-avatar.component.ts b/client/src/app/shared/shared-actor-image/actor-avatar.component.ts index ab2e02ad7..36babbe34 100644 --- a/client/src/app/shared/shared-actor-image/actor-avatar.component.ts +++ b/client/src/app/shared/shared-actor-image/actor-avatar.component.ts @@ -1,7 +1,7 @@ import { Component, Input, OnChanges, OnInit } from '@angular/core' import { VideoChannel } from '../shared-main' import { Account } from '../shared-main/account/account.model' -import { objectKeysTyped } from '@shared/core-utils' +import { objectKeysTyped } from '@peertube/peertube-core-utils' type ActorInput = { name: string diff --git a/client/src/app/shared/shared-custom-markup/custom-markup.service.ts b/client/src/app/shared/shared-custom-markup/custom-markup.service.ts index b49f64834..4ab78a250 100644 --- a/client/src/app/shared/shared-custom-markup/custom-markup.service.ts +++ b/client/src/app/shared/shared-custom-markup/custom-markup.service.ts @@ -10,7 +10,7 @@ import { PlaylistMiniatureMarkupData, VideoMiniatureMarkupData, VideosListMarkupData -} from '@shared/models' +} from '@peertube/peertube-models' import { DynamicElementService } from './dynamic-element.service' import { ButtonMarkupComponent, diff --git a/client/src/app/shared/shared-custom-markup/dynamic-element.service.ts b/client/src/app/shared/shared-custom-markup/dynamic-element.service.ts index a12907055..dc4f683ac 100644 --- a/client/src/app/shared/shared-custom-markup/dynamic-element.service.ts +++ b/client/src/app/shared/shared-custom-markup/dynamic-element.service.ts @@ -10,7 +10,7 @@ import { SimpleChanges, Type } from '@angular/core' -import { objectKeysTyped } from '@shared/core-utils' +import { objectKeysTyped } from '@peertube/peertube-core-utils' @Injectable() export class DynamicElementService { diff --git a/client/src/app/shared/shared-custom-markup/peertube-custom-tags/channel-miniature-markup.component.ts b/client/src/app/shared/shared-custom-markup/peertube-custom-tags/channel-miniature-markup.component.ts index 4f00eabd3..b731ccc64 100644 --- a/client/src/app/shared/shared-custom-markup/peertube-custom-tags/channel-miniature-markup.component.ts +++ b/client/src/app/shared/shared-custom-markup/peertube-custom-tags/channel-miniature-markup.component.ts @@ -3,7 +3,7 @@ import { finalize, map, switchMap, tap } from 'rxjs/operators' import { ChangeDetectionStrategy, ChangeDetectorRef, Component, EventEmitter, Input, OnInit, Output } from '@angular/core' import { MarkdownService, Notifier, UserService } from '@app/core' import { FindInBulkService } from '@app/shared/shared-search' -import { VideoSortField } from '@shared/models' +import { VideoSortField } from '@peertube/peertube-models' import { Video, VideoChannel, VideoService } from '../../shared-main' import { CustomMarkupComponent } from './shared' diff --git a/client/src/app/shared/shared-custom-markup/peertube-custom-tags/embed-markup.component.ts b/client/src/app/shared/shared-custom-markup/peertube-custom-tags/embed-markup.component.ts index 0baf2428b..bca7444ec 100644 --- a/client/src/app/shared/shared-custom-markup/peertube-custom-tags/embed-markup.component.ts +++ b/client/src/app/shared/shared-custom-markup/peertube-custom-tags/embed-markup.component.ts @@ -1,7 +1,7 @@ import { environment } from 'src/environments/environment' import { Component, ElementRef, Input, OnInit } from '@angular/core' import { buildVideoOrPlaylistEmbed } from '@root-helpers/video' -import { buildPlaylistEmbedLink, buildVideoEmbedLink } from '@shared/core-utils' +import { buildPlaylistEmbedLink, buildVideoEmbedLink } from '@peertube/peertube-core-utils' import { CustomMarkupComponent } from './shared' @Component({ diff --git a/client/src/app/shared/shared-custom-markup/peertube-custom-tags/video-miniature-markup.component.ts b/client/src/app/shared/shared-custom-markup/peertube-custom-tags/video-miniature-markup.component.ts index bd93929c9..d692abbe3 100644 --- a/client/src/app/shared/shared-custom-markup/peertube-custom-tags/video-miniature-markup.component.ts +++ b/client/src/app/shared/shared-custom-markup/peertube-custom-tags/video-miniature-markup.component.ts @@ -2,7 +2,7 @@ import { finalize } from 'rxjs/operators' import { ChangeDetectionStrategy, ChangeDetectorRef, Component, EventEmitter, Input, OnInit, Output } from '@angular/core' import { AuthService, Notifier } from '@app/core' import { FindInBulkService } from '@app/shared/shared-search' -import { objectKeysTyped } from '@shared/core-utils' +import { objectKeysTyped } from '@peertube/peertube-core-utils' import { Video } from '../../shared-main' import { MiniatureDisplayOptions } from '../../shared-video-miniature' import { CustomMarkupComponent } from './shared' diff --git a/client/src/app/shared/shared-custom-markup/peertube-custom-tags/videos-list-markup.component.ts b/client/src/app/shared/shared-custom-markup/peertube-custom-tags/videos-list-markup.component.ts index 81363be87..cbd5c7bf5 100644 --- a/client/src/app/shared/shared-custom-markup/peertube-custom-tags/videos-list-markup.component.ts +++ b/client/src/app/shared/shared-custom-markup/peertube-custom-tags/videos-list-markup.component.ts @@ -1,8 +1,8 @@ import { finalize } from 'rxjs/operators' import { ChangeDetectionStrategy, ChangeDetectorRef, Component, EventEmitter, Input, OnInit, Output } from '@angular/core' import { AuthService, Notifier } from '@app/core' -import { objectKeysTyped } from '@shared/core-utils' -import { VideoSortField } from '@shared/models' +import { objectKeysTyped } from '@peertube/peertube-core-utils' +import { VideoSortField } from '@peertube/peertube-models' import { Video, VideoService } from '../../shared-main' import { MiniatureDisplayOptions } from '../../shared-video-miniature' import { CustomMarkupComponent } from './shared' diff --git a/client/src/app/shared/shared-forms/dynamic-form-field.component.ts b/client/src/app/shared/shared-forms/dynamic-form-field.component.ts index e1a1f8034..a95463944 100644 --- a/client/src/app/shared/shared-forms/dynamic-form-field.component.ts +++ b/client/src/app/shared/shared-forms/dynamic-form-field.component.ts @@ -1,6 +1,6 @@ import { Component, Input } from '@angular/core' import { FormGroup } from '@angular/forms' -import { RegisterClientFormFieldOptions } from '@shared/models' +import { RegisterClientFormFieldOptions } from '@peertube/peertube-models' @Component({ selector: 'my-dynamic-form-field', diff --git a/client/src/app/shared/shared-forms/form-validator.service.ts b/client/src/app/shared/shared-forms/form-validator.service.ts index 14ee044b5..e7dedf52a 100644 --- a/client/src/app/shared/shared-forms/form-validator.service.ts +++ b/client/src/app/shared/shared-forms/form-validator.service.ts @@ -1,6 +1,6 @@ import { Injectable } from '@angular/core' import { AsyncValidatorFn, FormArray, FormBuilder, FormControl, FormGroup, ValidatorFn } from '@angular/forms' -import { objectKeysTyped } from '@shared/core-utils' +import { objectKeysTyped } from '@peertube/peertube-core-utils' import { BuildFormArgument, BuildFormDefaultValues } from '../form-validators/form-validator.model' import { FormReactiveErrors, FormReactiveValidationMessages } from './form-reactive.service' diff --git a/client/src/app/shared/shared-forms/markdown-textarea.component.ts b/client/src/app/shared/shared-forms/markdown-textarea.component.ts index 036fab3d9..7edcf868c 100644 --- a/client/src/app/shared/shared-forms/markdown-textarea.component.ts +++ b/client/src/app/shared/shared-forms/markdown-textarea.component.ts @@ -6,7 +6,7 @@ import { Component, ElementRef, forwardRef, Input, OnInit, ViewChild } from '@an import { ControlValueAccessor, NG_VALUE_ACCESSOR } from '@angular/forms' import { SafeHtml } from '@angular/platform-browser' import { MarkdownService, ScreenService } from '@app/core' -import { Video } from '@shared/models' +import { Video } from '@peertube/peertube-models' @Component({ selector: 'my-markdown-textarea', diff --git a/client/src/app/shared/shared-forms/preview-upload.component.ts b/client/src/app/shared/shared-forms/preview-upload.component.ts index cdfa26a23..3db7c34f7 100644 --- a/client/src/app/shared/shared-forms/preview-upload.component.ts +++ b/client/src/app/shared/shared-forms/preview-upload.component.ts @@ -2,7 +2,7 @@ import { Component, forwardRef, Input, OnInit } from '@angular/core' import { ControlValueAccessor, NG_VALUE_ACCESSOR } from '@angular/forms' import { ServerService } from '@app/core' import { imageToDataURL } from '@root-helpers/images' -import { HTMLServerConfig } from '@shared/models' +import { HTMLServerConfig } from '@peertube/peertube-models' import { BytesPipe } from '../shared-main' @Component({ diff --git a/client/src/app/shared/shared-forms/timestamp-input.component.ts b/client/src/app/shared/shared-forms/timestamp-input.component.ts index 79ca63673..280491852 100644 --- a/client/src/app/shared/shared-forms/timestamp-input.component.ts +++ b/client/src/app/shared/shared-forms/timestamp-input.component.ts @@ -1,6 +1,6 @@ import { ChangeDetectorRef, Component, EventEmitter, forwardRef, Input, OnInit, Output } from '@angular/core' import { ControlValueAccessor, NG_VALUE_ACCESSOR } from '@angular/forms' -import { secondsToTime, timeToInt } from '@shared/core-utils' +import { secondsToTime, timeToInt } from '@peertube/peertube-core-utils' @Component({ selector: 'my-timestamp-input', diff --git a/client/src/app/shared/shared-instance/instance-about-accordion.component.ts b/client/src/app/shared/shared-instance/instance-about-accordion.component.ts index a7c521876..78d960d93 100644 --- a/client/src/app/shared/shared-instance/instance-about-accordion.component.ts +++ b/client/src/app/shared/shared-instance/instance-about-accordion.component.ts @@ -1,8 +1,7 @@ import { Component, EventEmitter, Input, OnInit, Output, ViewChild } from '@angular/core' import { HooksService, Notifier } from '@app/core' import { NgbAccordionDirective } from '@ng-bootstrap/ng-bootstrap' -import { ClientFilterHookName, PluginClientScope } from '@shared/models/plugins' -import { About } from '@shared/models/server' +import { About, ClientFilterHookName, PluginClientScope } from '@peertube/peertube-models' import { InstanceService } from './instance.service' @Component({ diff --git a/client/src/app/shared/shared-instance/instance-features-table.component.ts b/client/src/app/shared/shared-instance/instance-features-table.component.ts index ab1b1458a..11c6cc0ac 100644 --- a/client/src/app/shared/shared-instance/instance-features-table.component.ts +++ b/client/src/app/shared/shared-instance/instance-features-table.component.ts @@ -1,7 +1,7 @@ import { Component, OnInit } from '@angular/core' import { ServerService } from '@app/core' import { formatICU } from '@app/helpers' -import { ServerConfig } from '@shared/models' +import { ServerConfig } from '@peertube/peertube-models' @Component({ selector: 'my-instance-features-table', diff --git a/client/src/app/shared/shared-instance/instance-follow.service.ts b/client/src/app/shared/shared-instance/instance-follow.service.ts index 7568fbbf4..f243273ba 100644 --- a/client/src/app/shared/shared-instance/instance-follow.service.ts +++ b/client/src/app/shared/shared-instance/instance-follow.service.ts @@ -4,8 +4,8 @@ import { catchError, concatMap, toArray } from 'rxjs/operators' import { HttpClient, HttpParams } from '@angular/common/http' import { Injectable } from '@angular/core' import { RestExtractor, RestPagination, RestService } from '@app/core' -import { arrayify } from '@shared/core-utils' -import { ActivityPubActorType, ActorFollow, FollowState, ResultList, ServerFollowCreate } from '@shared/models' +import { arrayify } from '@peertube/peertube-core-utils' +import { ActivityPubActorType, ActorFollow, FollowState, ResultList, ServerFollowCreate } from '@peertube/peertube-models' import { environment } from '../../../environments/environment' import { AdvancedInputFilter } from '../shared-forms' diff --git a/client/src/app/shared/shared-instance/instance.service.ts b/client/src/app/shared/shared-instance/instance.service.ts index 3088f0899..9a55cf972 100644 --- a/client/src/app/shared/shared-instance/instance.service.ts +++ b/client/src/app/shared/shared-instance/instance.service.ts @@ -3,9 +3,8 @@ import { catchError, map } from 'rxjs/operators' import { HttpClient } from '@angular/common/http' import { Injectable } from '@angular/core' import { MarkdownService, RestExtractor, ServerService } from '@app/core' -import { objectKeysTyped } from '@shared/core-utils' -import { peertubeTranslate } from '@shared/core-utils/i18n' -import { About } from '@shared/models' +import { objectKeysTyped, peertubeTranslate } from '@peertube/peertube-core-utils' +import { About } from '@peertube/peertube-models' import { environment } from '../../../environments/environment' export type AboutHTML = Pick + state: VideoConstant likesPercent: number dislikesPercent: number diff --git a/client/src/app/shared/shared-main/video/video-edit.model.ts b/client/src/app/shared/shared-main/video/video-edit.model.ts index 9129ab93f..a3e736c0f 100644 --- a/client/src/app/shared/shared-main/video/video-edit.model.ts +++ b/client/src/app/shared/shared-main/video/video-edit.model.ts @@ -1,7 +1,7 @@ import { getAbsoluteAPIUrl } from '@app/helpers' -import { VideoPassword, VideoPrivacy, VideoScheduleUpdate, VideoUpdate } from '@shared/models' +import { objectKeysTyped } from '@peertube/peertube-core-utils' +import { VideoPassword, VideoPrivacy, VideoPrivacyType, VideoScheduleUpdate, VideoUpdate } from '@peertube/peertube-models' import { VideoDetails } from './video-details.model' -import { objectKeysTyped } from '@shared/core-utils' export class VideoEdit implements VideoUpdate { static readonly SPECIAL_SCHEDULED_PRIVACY = -1 @@ -17,7 +17,7 @@ export class VideoEdit implements VideoUpdate { downloadEnabled: boolean waitTranscoding: boolean channelId: number - privacy: VideoPrivacy + privacy: VideoPrivacyType videoPassword?: string support: string thumbnailfile?: any diff --git a/client/src/app/shared/shared-main/video/video-file-token.service.ts b/client/src/app/shared/shared-main/video/video-file-token.service.ts index 9bca5b9ec..87a952895 100644 --- a/client/src/app/shared/shared-main/video/video-file-token.service.ts +++ b/client/src/app/shared/shared-main/video/video-file-token.service.ts @@ -2,7 +2,7 @@ import { catchError, map, of, tap } from 'rxjs' import { HttpClient } from '@angular/common/http' import { Injectable } from '@angular/core' import { RestExtractor } from '@app/core' -import { VideoToken } from '@shared/models' +import { VideoToken } from '@peertube/peertube-models' import { VideoService } from './video.service' import { VideoPasswordService } from './video-password.service' diff --git a/client/src/app/shared/shared-main/video/video-import.service.ts b/client/src/app/shared/shared-main/video/video-import.service.ts index 607c08d71..bb9052401 100644 --- a/client/src/app/shared/shared-main/video/video-import.service.ts +++ b/client/src/app/shared/shared-main/video/video-import.service.ts @@ -5,8 +5,8 @@ import { HttpClient, HttpParams } from '@angular/common/http' import { Injectable } from '@angular/core' import { RestExtractor, RestPagination, RestService, ServerService, UserService } from '@app/core' import { objectToFormData } from '@app/helpers' -import { peertubeTranslate } from '@shared/core-utils/i18n' -import { ResultList, VideoImport, VideoImportCreate, VideoUpdate } from '@shared/models' +import { peertubeTranslate } from '@peertube/peertube-core-utils' +import { ResultList, VideoImport, VideoImportCreate, VideoUpdate } from '@peertube/peertube-models' import { environment } from '../../../../environments/environment' @Injectable() diff --git a/client/src/app/shared/shared-main/video/video-ownership.service.ts b/client/src/app/shared/shared-main/video/video-ownership.service.ts index 1e8f7f68c..03e8fc946 100644 --- a/client/src/app/shared/shared-main/video/video-ownership.service.ts +++ b/client/src/app/shared/shared-main/video/video-ownership.service.ts @@ -4,7 +4,7 @@ import { catchError } from 'rxjs/operators' import { HttpClient, HttpParams } from '@angular/common/http' import { Injectable } from '@angular/core' import { RestExtractor, RestPagination, RestService } from '@app/core' -import { ResultList, VideoChangeOwnership, VideoChangeOwnershipAccept, VideoChangeOwnershipCreate } from '@shared/models' +import { ResultList, VideoChangeOwnership, VideoChangeOwnershipAccept, VideoChangeOwnershipCreate } from '@peertube/peertube-models' import { environment } from '../../../../environments/environment' @Injectable() diff --git a/client/src/app/shared/shared-main/video/video-password.service.ts b/client/src/app/shared/shared-main/video/video-password.service.ts index d5b0406f8..156efd60f 100644 --- a/client/src/app/shared/shared-main/video/video-password.service.ts +++ b/client/src/app/shared/shared-main/video/video-password.service.ts @@ -1,4 +1,4 @@ -import { ResultList, VideoPassword } from '@shared/models' +import { ResultList, VideoPassword } from '@peertube/peertube-models' import { Injectable } from '@angular/core' import { catchError, switchMap } from 'rxjs' import { HttpClient, HttpHeaders } from '@angular/common/http' diff --git a/client/src/app/shared/shared-main/video/video.model.ts b/client/src/app/shared/shared-main/video/video.model.ts index 1d077a613..ed28fb3f8 100644 --- a/client/src/app/shared/shared-main/video/video.model.ts +++ b/client/src/app/shared/shared-main/video/video.model.ts @@ -2,8 +2,7 @@ import { AuthUser } from '@app/core' import { User } from '@app/core/users/user.model' import { durationToString, formatICU, getAbsoluteAPIUrl, getAbsoluteEmbedUrl } from '@app/helpers' import { Actor } from '@app/shared/shared-main/account/actor.model' -import { buildVideoWatchPath, getAllFiles } from '@shared/core-utils' -import { peertubeTranslate } from '@shared/core-utils/i18n' +import { buildVideoWatchPath, getAllFiles, peertubeTranslate } from '@peertube/peertube-core-utils' import { ActorImage, HTMLServerConfig, @@ -12,11 +11,13 @@ import { VideoConstant, VideoFile, VideoPrivacy, + VideoPrivacyType, VideoScheduleUpdate, VideoState, + VideoStateType, VideoStreamingPlaylist, VideoStreamingPlaylistType -} from '@shared/models' +} from '@peertube/peertube-models' export class Video implements VideoServerModel { byVideoChannel: string @@ -30,7 +31,7 @@ export class Video implements VideoServerModel { category: VideoConstant licence: VideoConstant language: VideoConstant - privacy: VideoConstant + privacy: VideoConstant truncatedDescription: string description: string @@ -70,7 +71,7 @@ export class Video implements VideoServerModel { originInstanceHost: string waitTranscoding?: boolean - state?: VideoConstant + state?: VideoConstant scheduledUpdate?: VideoScheduleUpdate blacklisted?: boolean diff --git a/client/src/app/shared/shared-main/video/video.service.ts b/client/src/app/shared/shared-main/video/video.service.ts index a980c2dcf..9b2bc5dee 100644 --- a/client/src/app/shared/shared-main/video/video.service.ts +++ b/client/src/app/shared/shared-main/video/video.service.ts @@ -5,7 +5,7 @@ import { HttpClient, HttpParams, HttpRequest } from '@angular/common/http' import { Injectable } from '@angular/core' import { AuthService, ComponentPaginationLight, ConfirmService, RestExtractor, RestService, ServerService, UserService } from '@app/core' import { objectToFormData } from '@app/helpers' -import { arrayify } from '@shared/core-utils' +import { arrayify } from '@peertube/peertube-core-utils' import { BooleanBothQuery, FeedFormat, @@ -21,13 +21,14 @@ import { VideoConstant, VideoDetails as VideoDetailsServerModel, VideoFileMetadata, - VideoInclude, + VideoIncludeType, VideoPrivacy, + VideoPrivacyType, VideoSortField, + VideoSource, VideoTranscodingCreate, VideoUpdate -} from '@shared/models' -import { VideoSource } from '@shared/models/videos/video-source' +} from '@peertube/peertube-models' import { environment } from '../../../../environments/environment' import { Account } from '../account/account.model' import { AccountService } from '../account/account.service' @@ -40,11 +41,11 @@ import { Video } from './video.model' export type CommonVideoParams = { videoPagination?: ComponentPaginationLight sort: VideoSortField | SortMeta - include?: VideoInclude + include?: VideoIncludeType isLocal?: boolean categoryOneOf?: number[] languageOneOf?: string[] - privacyOneOf?: VideoPrivacy[] + privacyOneOf?: VideoPrivacyType[] isLive?: boolean skipCount?: boolean nsfw?: BooleanBothQuery @@ -455,7 +456,7 @@ export class VideoService { ) } - explainedPrivacyLabels (serverPrivacies: VideoConstant[], defaultPrivacyId = VideoPrivacy.PUBLIC) { + explainedPrivacyLabels (serverPrivacies: VideoConstant[], defaultPrivacyId: VideoPrivacyType = VideoPrivacy.PUBLIC) { const descriptions = { [VideoPrivacy.PRIVATE]: $localize`Only I can see this video`, [VideoPrivacy.UNLISTED]: $localize`Only shareable via a private link`, @@ -478,7 +479,7 @@ export class VideoService { } } - getHighestAvailablePrivacy (serverPrivacies: VideoConstant[]) { + getHighestAvailablePrivacy (serverPrivacies: VideoConstant[]) { // We do not add a password as this requires additional configuration. const order = [ VideoPrivacy.PRIVATE, diff --git a/client/src/app/shared/shared-moderation/abuse.service.ts b/client/src/app/shared/shared-moderation/abuse.service.ts index 5d1539f69..8055b6dd1 100644 --- a/client/src/app/shared/shared-moderation/abuse.service.ts +++ b/client/src/app/shared/shared-moderation/abuse.service.ts @@ -15,7 +15,7 @@ import { AdminAbuse, ResultList, UserAbuse -} from '@shared/models' +} from '@peertube/peertube-models' import { environment } from '../../../environments/environment' @Injectable() diff --git a/client/src/app/shared/shared-moderation/account-block.model.ts b/client/src/app/shared/shared-moderation/account-block.model.ts index 8f76c69dc..a5bde327a 100644 --- a/client/src/app/shared/shared-moderation/account-block.model.ts +++ b/client/src/app/shared/shared-moderation/account-block.model.ts @@ -1,4 +1,4 @@ -import { AccountBlock as AccountBlockServer } from '@shared/models' +import { AccountBlock as AccountBlockServer } from '@peertube/peertube-models' import { Account } from '@app/shared/shared-main' export class AccountBlock implements AccountBlockServer { diff --git a/client/src/app/shared/shared-moderation/blocklist.service.ts b/client/src/app/shared/shared-moderation/blocklist.service.ts index 0fb7536e5..f755b812a 100644 --- a/client/src/app/shared/shared-moderation/blocklist.service.ts +++ b/client/src/app/shared/shared-moderation/blocklist.service.ts @@ -4,8 +4,8 @@ import { catchError, concatMap, map, toArray } from 'rxjs/operators' import { HttpClient, HttpParams } from '@angular/common/http' import { Injectable } from '@angular/core' import { RestExtractor, RestPagination, RestService } from '@app/core' -import { arrayify } from '@shared/core-utils' -import { AccountBlock as AccountBlockServer, BlockStatus, ResultList, ServerBlock } from '@shared/models' +import { arrayify } from '@peertube/peertube-core-utils' +import { AccountBlock as AccountBlockServer, BlockStatus, ResultList, ServerBlock } from '@peertube/peertube-models' import { environment } from '../../../environments/environment' import { Account } from '../shared-main' import { AccountBlock } from './account-block.model' diff --git a/client/src/app/shared/shared-moderation/bulk.service.ts b/client/src/app/shared/shared-moderation/bulk.service.ts index f0b869421..36d1b0b1e 100644 --- a/client/src/app/shared/shared-moderation/bulk.service.ts +++ b/client/src/app/shared/shared-moderation/bulk.service.ts @@ -2,7 +2,7 @@ import { catchError } from 'rxjs/operators' import { HttpClient } from '@angular/common/http' import { Injectable } from '@angular/core' import { RestExtractor } from '@app/core' -import { BulkRemoveCommentsOfBody } from '@shared/models' +import { BulkRemoveCommentsOfBody } from '@peertube/peertube-models' import { environment } from '../../../environments/environment' @Injectable() diff --git a/client/src/app/shared/shared-moderation/report-modals/account-report.component.ts b/client/src/app/shared/shared-moderation/report-modals/account-report.component.ts index d587a9709..042b57aa7 100644 --- a/client/src/app/shared/shared-moderation/report-modals/account-report.component.ts +++ b/client/src/app/shared/shared-moderation/report-modals/account-report.component.ts @@ -6,8 +6,8 @@ import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' import { Account } from '@app/shared/shared-main' import { NgbModal } from '@ng-bootstrap/ng-bootstrap' import { NgbModalRef } from '@ng-bootstrap/ng-bootstrap/modal/modal-ref' -import { abusePredefinedReasonsMap } from '@shared/core-utils/abuse' -import { AbusePredefinedReasonsString } from '@shared/models' +import { abusePredefinedReasonsMap } from '@peertube/peertube-core-utils' +import { AbusePredefinedReasonsString } from '@peertube/peertube-models' import { AbuseService } from '../abuse.service' @Component({ diff --git a/client/src/app/shared/shared-moderation/report-modals/comment-report.component.ts b/client/src/app/shared/shared-moderation/report-modals/comment-report.component.ts index e35d70c8f..fd50b745a 100644 --- a/client/src/app/shared/shared-moderation/report-modals/comment-report.component.ts +++ b/client/src/app/shared/shared-moderation/report-modals/comment-report.component.ts @@ -6,8 +6,8 @@ import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' import { VideoComment } from '@app/shared/shared-video-comment' import { NgbModal } from '@ng-bootstrap/ng-bootstrap' import { NgbModalRef } from '@ng-bootstrap/ng-bootstrap/modal/modal-ref' -import { abusePredefinedReasonsMap } from '@shared/core-utils/abuse' -import { AbusePredefinedReasonsString } from '@shared/models' +import { abusePredefinedReasonsMap } from '@peertube/peertube-core-utils' +import { AbusePredefinedReasonsString } from '@peertube/peertube-models' import { AbuseService } from '../abuse.service' @Component({ diff --git a/client/src/app/shared/shared-moderation/report-modals/video-report.component.ts b/client/src/app/shared/shared-moderation/report-modals/video-report.component.ts index 16be8e0a1..479957d21 100644 --- a/client/src/app/shared/shared-moderation/report-modals/video-report.component.ts +++ b/client/src/app/shared/shared-moderation/report-modals/video-report.component.ts @@ -6,8 +6,8 @@ import { ABUSE_REASON_VALIDATOR } from '@app/shared/form-validators/abuse-valida import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' import { NgbModal } from '@ng-bootstrap/ng-bootstrap' import { NgbModalRef } from '@ng-bootstrap/ng-bootstrap/modal/modal-ref' -import { abusePredefinedReasonsMap } from '@shared/core-utils/abuse' -import { AbusePredefinedReasonsString } from '@shared/models' +import { abusePredefinedReasonsMap } from '@peertube/peertube-core-utils' +import { AbusePredefinedReasonsString } from '@peertube/peertube-models' import { Video } from '../../shared-main' import { AbuseService } from '../abuse.service' diff --git a/client/src/app/shared/shared-moderation/server-blocklist.component.ts b/client/src/app/shared/shared-moderation/server-blocklist.component.ts index f1bcbd561..4105645fa 100644 --- a/client/src/app/shared/shared-moderation/server-blocklist.component.ts +++ b/client/src/app/shared/shared-moderation/server-blocklist.component.ts @@ -2,7 +2,7 @@ import { SortMeta } from 'primeng/api' import { Directive, OnInit, ViewChild } from '@angular/core' import { Notifier, RestPagination, RestTable } from '@app/core' import { BatchDomainsModalComponent } from '@app/shared/shared-moderation/batch-domains-modal.component' -import { ServerBlock } from '@shared/models' +import { ServerBlock } from '@peertube/peertube-models' import { BlocklistComponentType, BlocklistService } from './blocklist.service' @Directive() diff --git a/client/src/app/shared/shared-moderation/user-ban-modal.component.ts b/client/src/app/shared/shared-moderation/user-ban-modal.component.ts index 34295c34a..fcada7acc 100644 --- a/client/src/app/shared/shared-moderation/user-ban-modal.component.ts +++ b/client/src/app/shared/shared-moderation/user-ban-modal.component.ts @@ -5,7 +5,7 @@ import { formatICU } from '@app/helpers' import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' import { NgbModal } from '@ng-bootstrap/ng-bootstrap' import { NgbModalRef } from '@ng-bootstrap/ng-bootstrap/modal/modal-ref' -import { User } from '@shared/models' +import { User } from '@peertube/peertube-models' import { USER_BAN_REASON_VALIDATOR } from '../form-validators/user-validators' import { Account } from '../shared-main' import { UserAdminService } from '../shared-users' diff --git a/client/src/app/shared/shared-moderation/user-moderation-dropdown.component.ts b/client/src/app/shared/shared-moderation/user-moderation-dropdown.component.ts index 50dccf862..7de152e60 100644 --- a/client/src/app/shared/shared-moderation/user-moderation-dropdown.component.ts +++ b/client/src/app/shared/shared-moderation/user-moderation-dropdown.component.ts @@ -1,7 +1,7 @@ import { Component, EventEmitter, Input, OnChanges, OnInit, Output, ViewChild } from '@angular/core' import { AuthService, ConfirmService, Notifier, ServerService } from '@app/core' import { Account, DropdownAction } from '@app/shared/shared-main' -import { BulkRemoveCommentsOfBody, User, UserRight } from '@shared/models' +import { BulkRemoveCommentsOfBody, User, UserRight } from '@peertube/peertube-models' import { UserAdminService } from '../shared-users' import { BlocklistService } from './blocklist.service' import { BulkService } from './bulk.service' diff --git a/client/src/app/shared/shared-moderation/video-block.service.ts b/client/src/app/shared/shared-moderation/video-block.service.ts index ab352a2d6..18950c92b 100644 --- a/client/src/app/shared/shared-moderation/video-block.service.ts +++ b/client/src/app/shared/shared-moderation/video-block.service.ts @@ -4,8 +4,8 @@ import { catchError, concatMap, toArray } from 'rxjs/operators' import { HttpClient, HttpParams } from '@angular/common/http' import { Injectable } from '@angular/core' import { RestExtractor, RestPagination, RestService } from '@app/core' -import { arrayify } from '@shared/core-utils' -import { ResultList, VideoBlacklist, VideoBlacklistType } from '@shared/models' +import { arrayify } from '@peertube/peertube-core-utils' +import { ResultList, VideoBlacklist, VideoBlacklistType, VideoBlacklistType_Type } from '@peertube/peertube-models' import { environment } from '../../../environments/environment' @Injectable() @@ -22,7 +22,7 @@ export class VideoBlockService { pagination: RestPagination sort: SortMeta search?: string - type?: VideoBlacklistType + type?: VideoBlacklistType_Type }): Observable> { const { pagination, sort, search, type } = options diff --git a/client/src/app/shared/shared-search/advanced-search.model.ts b/client/src/app/shared/shared-search/advanced-search.model.ts index 29fe3e8dc..b977a4801 100644 --- a/client/src/app/shared/shared-search/advanced-search.model.ts +++ b/client/src/app/shared/shared-search/advanced-search.model.ts @@ -6,7 +6,7 @@ import { VideoChannelsSearchQuery, VideoPlaylistsSearchQuery, VideosSearchQuery -} from '@shared/models' +} from '@peertube/peertube-models' export type AdvancedSearchResultType = 'videos' | 'playlists' | 'channels' diff --git a/client/src/app/shared/shared-search/find-in-bulk.service.ts b/client/src/app/shared/shared-search/find-in-bulk.service.ts index 125d5e2b8..de57c7f64 100644 --- a/client/src/app/shared/shared-search/find-in-bulk.service.ts +++ b/client/src/app/shared/shared-search/find-in-bulk.service.ts @@ -3,11 +3,11 @@ import { Observable, Subject } from 'rxjs' import { filter, first, map } from 'rxjs/operators' import { Injectable } from '@angular/core' import { buildBulkObservable } from '@app/helpers' -import { ResultList } from '@shared/models/common' +import { ResultList } from '@peertube/peertube-models' import { Video, VideoChannel } from '../shared-main' import { VideoPlaylist } from '../shared-video-playlist' -import { SearchService } from './search.service' import { AdvancedSearch } from './advanced-search.model' +import { SearchService } from './search.service' const debugLogger = debug('peertube:search:FindInBulkService') diff --git a/client/src/app/shared/shared-search/search.service.ts b/client/src/app/shared/shared-search/search.service.ts index ad2de0f37..281e0b4bd 100644 --- a/client/src/app/shared/shared-search/search.service.ts +++ b/client/src/app/shared/shared-search/search.service.ts @@ -9,7 +9,7 @@ import { Video as VideoServerModel, VideoChannel as VideoChannelServerModel, VideoPlaylist as VideoPlaylistServerModel -} from '@shared/models' +} from '@peertube/peertube-models' import { environment } from '../../../environments/environment' import { VideoPlaylist, VideoPlaylistService } from '../shared-video-playlist' import { AdvancedSearch } from './advanced-search.model' diff --git a/client/src/app/shared/shared-share-modal/video-share.component.ts b/client/src/app/shared/shared-share-modal/video-share.component.ts index b09222c3e..24c9cdeca 100644 --- a/client/src/app/shared/shared-share-modal/video-share.component.ts +++ b/client/src/app/shared/shared-share-modal/video-share.component.ts @@ -5,8 +5,8 @@ import { VideoDetails } from '@app/shared/shared-main' import { VideoPlaylist } from '@app/shared/shared-video-playlist' import { NgbModal } from '@ng-bootstrap/ng-bootstrap' import { buildVideoOrPlaylistEmbed } from '@root-helpers/video' -import { buildPlaylistLink, buildVideoLink, decoratePlaylistLink, decorateVideoLink } from '@shared/core-utils' -import { VideoCaption, VideoPlaylistPrivacy, VideoPrivacy } from '@shared/models' +import { buildPlaylistLink, buildVideoLink, decoratePlaylistLink, decorateVideoLink } from '@peertube/peertube-core-utils' +import { VideoCaption, VideoPlaylistPrivacy, VideoPrivacy } from '@peertube/peertube-models' type Customizations = { startAtCheckbox: boolean diff --git a/client/src/app/shared/shared-support-modal/support-modal.component.ts b/client/src/app/shared/shared-support-modal/support-modal.component.ts index f330228e1..d911b45d8 100644 --- a/client/src/app/shared/shared-support-modal/support-modal.component.ts +++ b/client/src/app/shared/shared-support-modal/support-modal.component.ts @@ -2,7 +2,7 @@ import { Component, Input, ViewChild } from '@angular/core' import { MarkdownService } from '@app/core' import { VideoDetails } from '@app/shared/shared-main' import { NgbModal } from '@ng-bootstrap/ng-bootstrap' -import { VideoChannel } from '@shared/models' +import { VideoChannel } from '@peertube/peertube-models' @Component({ selector: 'my-support-modal', diff --git a/client/src/app/shared/shared-thumbnail/video-thumbnail.component.ts b/client/src/app/shared/shared-thumbnail/video-thumbnail.component.ts index ad5d30db2..c9a5c97db 100644 --- a/client/src/app/shared/shared-thumbnail/video-thumbnail.component.ts +++ b/client/src/app/shared/shared-thumbnail/video-thumbnail.component.ts @@ -1,6 +1,6 @@ import { Component, EventEmitter, Input, Output } from '@angular/core' import { ScreenService } from '@app/core' -import { VideoState } from '@shared/models' +import { VideoState } from '@peertube/peertube-models' import { Video } from '../shared-main' @Component({ diff --git a/client/src/app/shared/shared-user-settings/user-interface-settings.component.ts b/client/src/app/shared/shared-user-settings/user-interface-settings.component.ts index c2c30d38b..08c6b6933 100644 --- a/client/src/app/shared/shared-user-settings/user-interface-settings.component.ts +++ b/client/src/app/shared/shared-user-settings/user-interface-settings.component.ts @@ -2,7 +2,7 @@ import { Subject, Subscription } from 'rxjs' import { Component, Input, OnDestroy, OnInit } from '@angular/core' import { AuthService, Notifier, ServerService, ThemeService, UserService } from '@app/core' import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' -import { HTMLServerConfig, User, UserUpdateMe } from '@shared/models' +import { HTMLServerConfig, User, UserUpdateMe } from '@peertube/peertube-models' import { SelectOptionsItem } from 'src/types' @Component({ diff --git a/client/src/app/shared/shared-user-settings/user-video-settings.component.ts b/client/src/app/shared/shared-user-settings/user-video-settings.component.ts index ed6e7fffd..234d5b217 100644 --- a/client/src/app/shared/shared-user-settings/user-video-settings.component.ts +++ b/client/src/app/shared/shared-user-settings/user-video-settings.component.ts @@ -4,8 +4,7 @@ import { first } from 'rxjs/operators' import { Component, Input, OnDestroy, OnInit } from '@angular/core' import { AuthService, Notifier, ServerService, User, UserService } from '@app/core' import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' -import { UserUpdateMe } from '@shared/models' -import { NSFWPolicyType } from '@shared/models/videos/nsfw-policy.type' +import { NSFWPolicyType, UserUpdateMe } from '@peertube/peertube-models' @Component({ selector: 'my-user-video-settings', diff --git a/client/src/app/shared/shared-user-subscription/subscribe-button.component.ts b/client/src/app/shared/shared-user-subscription/subscribe-button.component.ts index a002bf4e7..2a5751824 100644 --- a/client/src/app/shared/shared-user-subscription/subscribe-button.component.ts +++ b/client/src/app/shared/shared-user-subscription/subscribe-button.component.ts @@ -2,7 +2,7 @@ import { concat, forkJoin, merge } from 'rxjs' import { Component, Input, OnChanges, OnInit } from '@angular/core' import { AuthService, Notifier, RedirectService } from '@app/core' import { Account, VideoChannel, VideoService } from '@app/shared/shared-main' -import { FeedFormat } from '@shared/models' +import { FeedFormat } from '@peertube/peertube-models' import { UserSubscriptionService } from './user-subscription.service' @Component({ diff --git a/client/src/app/shared/shared-user-subscription/user-subscription.service.ts b/client/src/app/shared/shared-user-subscription/user-subscription.service.ts index 9cf6b4d16..b83f7ebc5 100644 --- a/client/src/app/shared/shared-user-subscription/user-subscription.service.ts +++ b/client/src/app/shared/shared-user-subscription/user-subscription.service.ts @@ -6,7 +6,7 @@ import { Injectable } from '@angular/core' import { ComponentPaginationLight, RestExtractor, RestService } from '@app/core' import { buildBulkObservable } from '@app/helpers' import { Video, VideoChannel, VideoChannelService, VideoService } from '@app/shared/shared-main' -import { ActorFollow, ResultList, VideoChannel as VideoChannelServer, VideoSortField } from '@shared/models' +import { ActorFollow, ResultList, VideoChannel as VideoChannelServer, VideoSortField } from '@peertube/peertube-models' import { environment } from '../../../environments/environment' const debugLogger = debug('peertube:subscriptions:UserSubscriptionService') diff --git a/client/src/app/shared/shared-users/two-factor.service.ts b/client/src/app/shared/shared-users/two-factor.service.ts index 9ff916f15..cb4450e8f 100644 --- a/client/src/app/shared/shared-users/two-factor.service.ts +++ b/client/src/app/shared/shared-users/two-factor.service.ts @@ -2,7 +2,7 @@ import { catchError } from 'rxjs/operators' import { HttpClient } from '@angular/common/http' import { Injectable } from '@angular/core' import { RestExtractor, UserService } from '@app/core' -import { TwoFactorEnableResult } from '@shared/models' +import { TwoFactorEnableResult } from '@peertube/peertube-models' @Injectable() export class TwoFactorService { diff --git a/client/src/app/shared/shared-users/user-admin.service.ts b/client/src/app/shared/shared-users/user-admin.service.ts index 5842bd271..cc706343f 100644 --- a/client/src/app/shared/shared-users/user-admin.service.ts +++ b/client/src/app/shared/shared-users/user-admin.service.ts @@ -5,8 +5,8 @@ import { HttpClient, HttpParams } from '@angular/common/http' import { Injectable } from '@angular/core' import { RestExtractor, RestPagination, RestService, ServerService, UserService } from '@app/core' import { getBytes } from '@root-helpers/bytes' -import { arrayify, peertubeTranslate } from '@shared/core-utils' -import { ResultList, User as UserServerModel, UserCreate, UserUpdate } from '@shared/models' +import { arrayify, peertubeTranslate } from '@peertube/peertube-core-utils' +import { ResultList, User as UserServerModel, UserCreate, UserUpdate } from '@peertube/peertube-models' @Injectable() export class UserAdminService { diff --git a/client/src/app/shared/shared-video-comment/video-comment-thread-tree.model.ts b/client/src/app/shared/shared-video-comment/video-comment-thread-tree.model.ts index 9956c88a6..62683f57f 100644 --- a/client/src/app/shared/shared-video-comment/video-comment-thread-tree.model.ts +++ b/client/src/app/shared/shared-video-comment/video-comment-thread-tree.model.ts @@ -1,4 +1,4 @@ -import { VideoCommentThreadTree as VideoCommentThreadTreeServerModel } from '@shared/models' +import { VideoCommentThreadTree as VideoCommentThreadTreeServerModel } from '@peertube/peertube-models' import { VideoComment } from './video-comment.model' export class VideoCommentThreadTree implements VideoCommentThreadTreeServerModel { diff --git a/client/src/app/shared/shared-video-comment/video-comment.model.ts b/client/src/app/shared/shared-video-comment/video-comment.model.ts index adab4cfbd..7048ed66f 100644 --- a/client/src/app/shared/shared-video-comment/video-comment.model.ts +++ b/client/src/app/shared/shared-video-comment/video-comment.model.ts @@ -4,7 +4,7 @@ import { Account as AccountInterface, VideoComment as VideoCommentServerModel, VideoCommentAdmin as VideoCommentAdminServerModel -} from '@shared/models' +} from '@peertube/peertube-models' export class VideoComment implements VideoCommentServerModel { id: number diff --git a/client/src/app/shared/shared-video-comment/video-comment.service.ts b/client/src/app/shared/shared-video-comment/video-comment.service.ts index 3906652be..d1db773c4 100644 --- a/client/src/app/shared/shared-video-comment/video-comment.service.ts +++ b/client/src/app/shared/shared-video-comment/video-comment.service.ts @@ -14,7 +14,7 @@ import { VideoCommentAdmin, VideoCommentCreate, VideoCommentThreadTree as VideoCommentThreadTreeServerModel -} from '@shared/models' +} from '@peertube/peertube-models' import { environment } from '../../../environments/environment' import { VideoCommentThreadTree } from './video-comment-thread-tree.model' import { VideoComment } from './video-comment.model' diff --git a/client/src/app/shared/shared-video-live/live-stream-information.component.ts b/client/src/app/shared/shared-video-live/live-stream-information.component.ts index 400a6fa01..4089c88fb 100644 --- a/client/src/app/shared/shared-video-live/live-stream-information.component.ts +++ b/client/src/app/shared/shared-video-live/live-stream-information.component.ts @@ -1,7 +1,7 @@ import { Component, ElementRef, ViewChild } from '@angular/core' import { Video } from '@app/shared/shared-main' import { NgbModal } from '@ng-bootstrap/ng-bootstrap' -import { LiveVideo, LiveVideoError, LiveVideoSession } from '@shared/models' +import { LiveVideo, LiveVideoError, LiveVideoErrorType, LiveVideoSession } from '@peertube/peertube-models' import { LiveVideoService } from './live-video.service' @Component({ @@ -38,7 +38,7 @@ export class LiveStreamInformationComponent { getErrorLabel (session: LiveVideoSession) { if (!session.error) return undefined - const errors: { [ id in LiveVideoError ]: string } = { + const errors: { [ id in LiveVideoErrorType ]: string } = { [LiveVideoError.BAD_SOCKET_HEALTH]: $localize`Server too slow`, [LiveVideoError.BLACKLISTED]: $localize`Live blacklisted`, [LiveVideoError.DURATION_EXCEEDED]: $localize`Max duration exceeded`, diff --git a/client/src/app/shared/shared-video-live/live-video.service.ts b/client/src/app/shared/shared-video-live/live-video.service.ts index 89bfd84a0..8ac0eb924 100644 --- a/client/src/app/shared/shared-video-live/live-video.service.ts +++ b/client/src/app/shared/shared-video-live/live-video.service.ts @@ -2,7 +2,7 @@ import { catchError } from 'rxjs/operators' import { HttpClient } from '@angular/common/http' import { Injectable } from '@angular/core' import { RestExtractor } from '@app/core' -import { LiveVideo, LiveVideoCreate, LiveVideoSession, LiveVideoUpdate, ResultList, VideoCreateResult } from '@shared/models' +import { LiveVideo, LiveVideoCreate, LiveVideoSession, LiveVideoUpdate, ResultList, VideoCreateResult } from '@peertube/peertube-models' import { environment } from '../../../environments/environment' import { VideoService } from '../shared-main' diff --git a/client/src/app/shared/shared-video-miniature/video-actions-dropdown.component.ts b/client/src/app/shared/shared-video-miniature/video-actions-dropdown.component.ts index 9891aae2e..4b3ed6e99 100644 --- a/client/src/app/shared/shared-video-miniature/video-actions-dropdown.component.ts +++ b/client/src/app/shared/shared-video-miniature/video-actions-dropdown.component.ts @@ -2,7 +2,7 @@ import { Component, EventEmitter, Input, OnChanges, Output, ViewChild } from '@a import { AuthService, ConfirmService, Notifier, ScreenService, ServerService } from '@app/core' import { BlocklistService, VideoBlockComponent, VideoBlockService, VideoReportComponent } from '@app/shared/shared-moderation' import { NgbDropdown } from '@ng-bootstrap/ng-bootstrap' -import { VideoCaption } from '@shared/models' +import { VideoCaption } from '@peertube/peertube-models' import { Actor, DropdownAction, diff --git a/client/src/app/shared/shared-video-miniature/video-download.component.ts b/client/src/app/shared/shared-video-miniature/video-download.component.ts index 146ea7dfe..123f40b2f 100644 --- a/client/src/app/shared/shared-video-miniature/video-download.component.ts +++ b/client/src/app/shared/shared-video-miniature/video-download.component.ts @@ -6,8 +6,8 @@ import { HooksService } from '@app/core' import { NgbModal, NgbModalRef } from '@ng-bootstrap/ng-bootstrap' import { logger } from '@root-helpers/logger' import { videoRequiresFileToken } from '@root-helpers/video' -import { objectKeysTyped, pick } from '@shared/core-utils' -import { VideoCaption, VideoFile } from '@shared/models' +import { objectKeysTyped, pick } from '@peertube/peertube-core-utils' +import { VideoCaption, VideoFile } from '@peertube/peertube-models' import { BytesPipe, NumberFormatterPipe, VideoDetails, VideoFileTokenService, VideoService } from '../shared-main' type DownloadType = 'video' | 'subtitles' diff --git a/client/src/app/shared/shared-video-miniature/video-filters-header.component.ts b/client/src/app/shared/shared-video-miniature/video-filters-header.component.ts index a5da9ebf3..2826408e7 100644 --- a/client/src/app/shared/shared-video-miniature/video-filters-header.component.ts +++ b/client/src/app/shared/shared-video-miniature/video-filters-header.component.ts @@ -4,7 +4,7 @@ import { Component, EventEmitter, Input, OnDestroy, OnInit, Output } from '@angu import { FormBuilder, FormGroup } from '@angular/forms' import { AuthService } from '@app/core' import { ServerService } from '@app/core/server/server.service' -import { UserRight } from '@shared/models' +import { UserRight } from '@peertube/peertube-models' import { PeertubeModalService } from '../shared-main' import { VideoFilters } from './video-filters.model' diff --git a/client/src/app/shared/shared-video-miniature/video-filters.model.ts b/client/src/app/shared/shared-video-miniature/video-filters.model.ts index f57a45eb1..8db17c015 100644 --- a/client/src/app/shared/shared-video-miniature/video-filters.model.ts +++ b/client/src/app/shared/shared-video-miniature/video-filters.model.ts @@ -1,8 +1,14 @@ import { splitIntoArray, toBoolean } from '@app/helpers' -import { getAllPrivacies } from '@shared/core-utils' -import { escapeHTML } from '@shared/core-utils/renderer' -import { BooleanBothQuery, NSFWPolicyType, VideoInclude, VideoPrivacy, VideoSortField } from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' +import { escapeHTML, getAllPrivacies } from '@peertube/peertube-core-utils' +import { + BooleanBothQuery, + NSFWPolicyType, + VideoInclude, + VideoIncludeType, + VideoPrivacyType, + VideoSortField +} from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' type VideoFiltersKeys = { [ id in keyof AttributesOnly ]: any @@ -207,8 +213,8 @@ export class VideoFilters { toVideosAPIObject () { let isLocal: boolean - let include: VideoInclude - let privacyOneOf: VideoPrivacy[] + let include: VideoIncludeType + let privacyOneOf: VideoPrivacyType[] if (this.scope === 'local') { isLocal = true diff --git a/client/src/app/shared/shared-video-miniature/video-miniature.component.ts b/client/src/app/shared/shared-video-miniature/video-miniature.component.ts index d453f37a1..11cd6726e 100644 --- a/client/src/app/shared/shared-video-miniature/video-miniature.component.ts +++ b/client/src/app/shared/shared-video-miniature/video-miniature.component.ts @@ -11,7 +11,7 @@ import { Output } from '@angular/core' import { AuthService, ScreenService, ServerService, User } from '@app/core' -import { HTMLServerConfig, VideoExistInPlaylist, VideoPlaylistType, VideoPrivacy, VideoState } from '@shared/models' +import { HTMLServerConfig, VideoExistInPlaylist, VideoPlaylistType, VideoPrivacy, VideoState } from '@peertube/peertube-models' import { LinkType } from '../../../types/link.type' import { ActorAvatarSize } from '../shared-actor-image/actor-avatar.component' import { Video, VideoService } from '../shared-main' diff --git a/client/src/app/shared/shared-video-miniature/videos-list.component.ts b/client/src/app/shared/shared-video-miniature/videos-list.component.ts index 14a5abd7a..afdef5ace 100644 --- a/client/src/app/shared/shared-video-miniature/videos-list.component.ts +++ b/client/src/app/shared/shared-video-miniature/videos-list.component.ts @@ -15,8 +15,8 @@ import { } from '@app/core' import { GlobalIconName } from '@app/shared/shared-icons' import { logger } from '@root-helpers/logger' -import { isLastMonth, isLastWeek, isThisMonth, isToday, isYesterday } from '@shared/core-utils' -import { ResultList, UserRight, VideoSortField } from '@shared/models' +import { isLastMonth, isLastWeek, isThisMonth, isToday, isYesterday } from '@peertube/peertube-core-utils' +import { ResultList, UserRight, VideoSortField } from '@peertube/peertube-models' import { Syndication, Video } from '../shared-main' import { VideoFilters, VideoFilterScope } from './video-filters.model' import { MiniatureDisplayOptions } from './video-miniature.component' diff --git a/client/src/app/shared/shared-video-miniature/videos-selection.component.ts b/client/src/app/shared/shared-video-miniature/videos-selection.component.ts index 86fe502e2..286b33dd4 100644 --- a/client/src/app/shared/shared-video-miniature/videos-selection.component.ts +++ b/client/src/app/shared/shared-video-miniature/videos-selection.component.ts @@ -2,8 +2,8 @@ import { Observable, Subject } from 'rxjs' import { AfterContentInit, Component, ContentChildren, EventEmitter, Input, Output, QueryList, TemplateRef } from '@angular/core' import { ComponentPagination, Notifier, User } from '@app/core' import { logger } from '@root-helpers/logger' -import { objectKeysTyped } from '@shared/core-utils' -import { ResultList, VideosExistInPlaylists, VideoSortField } from '@shared/models' +import { objectKeysTyped } from '@peertube/peertube-core-utils' +import { ResultList, VideosExistInPlaylists, VideoSortField } from '@peertube/peertube-models' import { PeerTubeTemplateDirective, Video } from '../shared-main' import { MiniatureDisplayOptions } from './video-miniature.component' diff --git a/client/src/app/shared/shared-video-playlist/video-add-to-playlist.component.ts b/client/src/app/shared/shared-video-playlist/video-add-to-playlist.component.ts index f802416a4..84173ba69 100644 --- a/client/src/app/shared/shared-video-playlist/video-add-to-playlist.component.ts +++ b/client/src/app/shared/shared-video-playlist/video-add-to-playlist.component.ts @@ -4,7 +4,7 @@ import { debounceTime, filter } from 'rxjs/operators' import { ChangeDetectionStrategy, ChangeDetectorRef, Component, Input, OnChanges, OnDestroy, OnInit, SimpleChanges } from '@angular/core' import { AuthService, DisableForReuseHook, Notifier } from '@app/core' import { FormReactive, FormReactiveService } from '@app/shared/shared-forms' -import { secondsToTime } from '@shared/core-utils' +import { secondsToTime } from '@peertube/peertube-core-utils' import { CachedVideoExistInPlaylist, Video, @@ -12,7 +12,7 @@ import { VideoPlaylistElementCreate, VideoPlaylistElementUpdate, VideoPlaylistPrivacy -} from '@shared/models' +} from '@peertube/peertube-models' import { VIDEO_PLAYLIST_DISPLAY_NAME_VALIDATOR } from '../form-validators/video-playlist-validators' import { CachedPlaylist, VideoPlaylistService } from './video-playlist.service' diff --git a/client/src/app/shared/shared-video-playlist/video-playlist-element-miniature.component.ts b/client/src/app/shared/shared-video-playlist/video-playlist-element-miniature.component.ts index b9a1d9623..0c0f11ecc 100644 --- a/client/src/app/shared/shared-video-playlist/video-playlist-element-miniature.component.ts +++ b/client/src/app/shared/shared-video-playlist/video-playlist-element-miniature.component.ts @@ -2,8 +2,8 @@ import { ChangeDetectionStrategy, ChangeDetectorRef, Component, EventEmitter, In import { AuthService, Notifier, ServerService } from '@app/core' import { Video, VideoService } from '@app/shared/shared-main' import { NgbDropdown } from '@ng-bootstrap/ng-bootstrap' -import { secondsToTime } from '@shared/core-utils' -import { HTMLServerConfig, VideoPlaylistElementType, VideoPlaylistElementUpdate, VideoPrivacy } from '@shared/models' +import { secondsToTime } from '@peertube/peertube-core-utils' +import { HTMLServerConfig, VideoPlaylistElementType, VideoPlaylistElementUpdate, VideoPrivacy } from '@peertube/peertube-models' import { VideoPlaylistElement } from './video-playlist-element.model' import { VideoPlaylist } from './video-playlist.model' import { VideoPlaylistService } from './video-playlist.service' diff --git a/client/src/app/shared/shared-video-playlist/video-playlist-element.model.ts b/client/src/app/shared/shared-video-playlist/video-playlist-element.model.ts index b661378bd..16b212281 100644 --- a/client/src/app/shared/shared-video-playlist/video-playlist-element.model.ts +++ b/client/src/app/shared/shared-video-playlist/video-playlist-element.model.ts @@ -1,5 +1,5 @@ import { Video } from '@app/shared/shared-main' -import { VideoPlaylistElement as ServerVideoPlaylistElement, VideoPlaylistElementType } from '@shared/models' +import { VideoPlaylistElement as ServerVideoPlaylistElement, VideoPlaylistElementType_Type } from '@peertube/peertube-models' export class VideoPlaylistElement implements ServerVideoPlaylistElement { id: number @@ -7,7 +7,7 @@ export class VideoPlaylistElement implements ServerVideoPlaylistElement { startTimestamp: number stopTimestamp: number - type: VideoPlaylistElementType + type: VideoPlaylistElementType_Type video?: Video diff --git a/client/src/app/shared/shared-video-playlist/video-playlist.model.ts b/client/src/app/shared/shared-video-playlist/video-playlist.model.ts index 6b38d9ca3..24f1041ce 100644 --- a/client/src/app/shared/shared-video-playlist/video-playlist.model.ts +++ b/client/src/app/shared/shared-video-playlist/video-playlist.model.ts @@ -1,15 +1,15 @@ import { getAbsoluteAPIUrl, getAbsoluteEmbedUrl } from '@app/helpers' import { Actor } from '@app/shared/shared-main' -import { buildPlaylistWatchPath } from '@shared/core-utils' -import { peertubeTranslate } from '@shared/core-utils/i18n' +import { buildPlaylistWatchPath, peertubeTranslate } from '@peertube/peertube-core-utils' import { AccountSummary, VideoChannelSummary, VideoConstant, VideoPlaylist as ServerVideoPlaylist, - VideoPlaylistPrivacy, - VideoPlaylistType -} from '@shared/models' + VideoPlaylistPrivacyType, + VideoPlaylistType, + VideoPlaylistType_Type +} from '@peertube/peertube-models' export class VideoPlaylist implements ServerVideoPlaylist { id: number @@ -22,11 +22,11 @@ export class VideoPlaylist implements ServerVideoPlaylist { displayName: string description: string - privacy: VideoConstant + privacy: VideoConstant videosLength: number - type: VideoConstant + type: VideoConstant createdAt: Date | string updatedAt: Date | string diff --git a/client/src/app/shared/shared-video-playlist/video-playlist.service.ts b/client/src/app/shared/shared-video-playlist/video-playlist.service.ts index bc9fb0d74..7f0da2be8 100644 --- a/client/src/app/shared/shared-video-playlist/video-playlist.service.ts +++ b/client/src/app/shared/shared-video-playlist/video-playlist.service.ts @@ -20,7 +20,7 @@ import { VideoPlaylistReorder, VideoPlaylistUpdate, VideosExistInPlaylists -} from '@shared/models' +} from '@peertube/peertube-models' import { environment } from '../../../environments/environment' import { VideoPlaylistElement } from './video-playlist-element.model' import { VideoPlaylist } from './video-playlist.model' diff --git a/client/src/assets/player/peertube-player.ts b/client/src/assets/player/peertube-player.ts index 69ca1a566..4da681a08 100644 --- a/client/src/assets/player/peertube-player.ts +++ b/client/src/assets/player/peertube-player.ts @@ -30,7 +30,7 @@ import { PluginsManager } from '@root-helpers/plugins-manager' import { copyToClipboard } from '@root-helpers/utils' import { buildVideoOrPlaylistEmbed } from '@root-helpers/video' import { isMobile } from '@root-helpers/web-browser' -import { buildVideoLink, decorateVideoLink, isDefaultLocale, pick } from '@shared/core-utils' +import { buildVideoLink, decorateVideoLink, isDefaultLocale, pick } from '@peertube/peertube-core-utils' import { saveAverageBandwidth } from './peertube-player-local-storage' import { ControlBarOptionsBuilder, HLSOptionsBuilder, WebVideoOptionsBuilder } from './shared/player-options-builder' import { TranslationsManager } from './translations-manager' @@ -51,6 +51,8 @@ if (PlayProgressBar.prototype.options_.children.includes('timeTooltip') !== true PlayProgressBar.prototype.options_.children.push('timeTooltip') } +export { videojs } + export class PeerTubePlayer { private pluginsManager: PluginsManager @@ -516,9 +518,3 @@ export class PeerTubePlayer { return { content } } } - -// ############################################################################ - -export { - videojs -} diff --git a/client/src/assets/player/shared/common/utils.ts b/client/src/assets/player/shared/common/utils.ts index 609240626..4a7182021 100644 --- a/client/src/assets/player/shared/common/utils.ts +++ b/client/src/assets/player/shared/common/utils.ts @@ -1,4 +1,4 @@ -import { VideoFile } from '@shared/models' +import { VideoFile } from '@peertube/peertube-models' function toTitleCase (str: string) { return str.charAt(0).toUpperCase() + str.slice(1) diff --git a/client/src/assets/player/shared/control-bar/peertube-link-button.ts b/client/src/assets/player/shared/control-bar/peertube-link-button.ts index 8242b9cea..f93c265d6 100644 --- a/client/src/assets/player/shared/control-bar/peertube-link-button.ts +++ b/client/src/assets/player/shared/control-bar/peertube-link-button.ts @@ -1,5 +1,5 @@ import videojs from 'video.js' -import { buildVideoLink, decorateVideoLink } from '@shared/core-utils' +import { buildVideoLink, decorateVideoLink } from '@peertube/peertube-core-utils' import { PeerTubeLinkButtonOptions } from '../../types' const Component = videojs.getComponent('Component') diff --git a/client/src/assets/player/shared/metrics/metrics-plugin.ts b/client/src/assets/player/shared/metrics/metrics-plugin.ts index 06ca0c2f2..0ad16338c 100644 --- a/client/src/assets/player/shared/metrics/metrics-plugin.ts +++ b/client/src/assets/player/shared/metrics/metrics-plugin.ts @@ -1,7 +1,7 @@ import debug from 'debug' import videojs from 'video.js' +import { PlaybackMetricCreate, VideoResolutionType } from '@peertube/peertube-models' import { logger } from '@root-helpers/logger' -import { PlaybackMetricCreate } from '../../../../../../shared/models' import { MetricsPluginOptions, PlayerNetworkInfo } from '../../types' const debugLogger = debug('peertube:player:metrics') @@ -102,7 +102,7 @@ class MetricsPlugin extends Plugin { } const body: PlaybackMetricCreate = { - resolution, + resolution: resolution as VideoResolutionType, fps, playerMode: this.options_.mode(), diff --git a/client/src/assets/player/shared/p2p-media-loader/p2p-media-loader-plugin.ts b/client/src/assets/player/shared/p2p-media-loader/p2p-media-loader-plugin.ts index 8c376cd21..1e47fe486 100644 --- a/client/src/assets/player/shared/p2p-media-loader/p2p-media-loader-plugin.ts +++ b/client/src/assets/player/shared/p2p-media-loader/p2p-media-loader-plugin.ts @@ -3,7 +3,7 @@ import videojs from 'video.js' import { Events, Segment } from '@peertube/p2p-media-loader-core' import { Engine, initHlsJsPlayer, initVideoJsContribHlsJsPlayer } from '@peertube/p2p-media-loader-hlsjs' import { logger } from '@root-helpers/logger' -import { addQueryParams } from '@shared/core-utils' +import { addQueryParams } from '@peertube/peertube-core-utils' import { P2PMediaLoaderPluginOptions, PlayerNetworkInfo } from '../../types' import { SettingsButton } from '../settings/settings-menu-button' diff --git a/client/src/assets/player/shared/p2p-media-loader/segment-validator.ts b/client/src/assets/player/shared/p2p-media-loader/segment-validator.ts index 9cb6344a9..75d483015 100644 --- a/client/src/assets/player/shared/p2p-media-loader/segment-validator.ts +++ b/client/src/assets/player/shared/p2p-media-loader/segment-validator.ts @@ -2,7 +2,7 @@ import { basename } from 'path' import { Segment } from '@peertube/p2p-media-loader-core' import { logger } from '@root-helpers/logger' import { wait } from '@root-helpers/utils' -import { removeQueryParams } from '@shared/core-utils' +import { removeQueryParams } from '@peertube/peertube-core-utils' import { isSameOrigin } from '../common' type SegmentsJSON = { [filename: string]: string | { [byterange: string]: string } } diff --git a/client/src/assets/player/shared/peertube/peertube-plugin.ts b/client/src/assets/player/shared/peertube/peertube-plugin.ts index f93593415..cf866723c 100644 --- a/client/src/assets/player/shared/peertube/peertube-plugin.ts +++ b/client/src/assets/player/shared/peertube/peertube-plugin.ts @@ -1,9 +1,9 @@ import debug from 'debug' import videojs from 'video.js' +import { timeToInt } from '@peertube/peertube-core-utils' +import { VideoView, VideoViewEvent } from '@peertube/peertube-models' import { logger } from '@root-helpers/logger' import { isIOS, isMobile, isSafari } from '@root-helpers/web-browser' -import { timeToInt } from '@shared/core-utils' -import { VideoView, VideoViewEvent } from '@shared/models/videos' import { getStoredLastSubtitle, getStoredMute, diff --git a/client/src/assets/player/shared/player-options-builder/hls-options-builder.ts b/client/src/assets/player/shared/player-options-builder/hls-options-builder.ts index fd632d90d..41198afbe 100644 --- a/client/src/assets/player/shared/player-options-builder/hls-options-builder.ts +++ b/client/src/assets/player/shared/player-options-builder/hls-options-builder.ts @@ -1,7 +1,7 @@ import { HybridLoaderSettings } from '@peertube/p2p-media-loader-core' import { HlsJsEngineSettings } from '@peertube/p2p-media-loader-hlsjs' import { logger } from '@root-helpers/logger' -import { LiveVideoLatencyMode } from '@shared/models' +import { LiveVideoLatencyMode } from '@peertube/peertube-models' import { getAverageBandwidthInStore } from '../../peertube-player-local-storage' import { P2PMediaLoader, P2PMediaLoaderPluginOptions, PeerTubePlayerContructorOptions, PeerTubePlayerLoadOptions } from '../../types' import { getRtcConfig, isSameOrigin } from '../common' diff --git a/client/src/assets/player/shared/playlist/playlist-menu-item.ts b/client/src/assets/player/shared/playlist/playlist-menu-item.ts index f9366332d..66c92d9e5 100644 --- a/client/src/assets/player/shared/playlist/playlist-menu-item.ts +++ b/client/src/assets/player/shared/playlist/playlist-menu-item.ts @@ -1,6 +1,6 @@ import videojs from 'video.js' -import { secondsToTime } from '@shared/core-utils' -import { VideoPlaylistElement } from '@shared/models' +import { secondsToTime } from '@peertube/peertube-core-utils' +import { VideoPlaylistElement } from '@peertube/peertube-models' import { PlaylistItemOptions } from '../../types' const Component = videojs.getComponent('Component') diff --git a/client/src/assets/player/shared/playlist/playlist-menu.ts b/client/src/assets/player/shared/playlist/playlist-menu.ts index 53a5a7274..f6795390c 100644 --- a/client/src/assets/player/shared/playlist/playlist-menu.ts +++ b/client/src/assets/player/shared/playlist/playlist-menu.ts @@ -1,5 +1,5 @@ import videojs from 'video.js' -import { VideoPlaylistElement } from '@shared/models' +import { VideoPlaylistElement } from '@peertube/peertube-models' import { PlaylistPluginOptions } from '../../types' import { PlaylistMenuItem } from './playlist-menu-item' diff --git a/client/src/assets/player/shared/stats/stats-card.ts b/client/src/assets/player/shared/stats/stats-card.ts index 13334d91a..fefe8483e 100644 --- a/client/src/assets/player/shared/stats/stats-card.ts +++ b/client/src/assets/player/shared/stats/stats-card.ts @@ -1,6 +1,6 @@ import videojs from 'video.js' import { logger } from '@root-helpers/logger' -import { secondsToTime } from '@shared/core-utils' +import { secondsToTime } from '@peertube/peertube-core-utils' import { PlayerNetworkInfo as EventPlayerNetworkInfo } from '../../types' import { bytes } from '../common' diff --git a/client/src/assets/player/shared/web-video/web-video-plugin.ts b/client/src/assets/player/shared/web-video/web-video-plugin.ts index b839062f2..18d911108 100644 --- a/client/src/assets/player/shared/web-video/web-video-plugin.ts +++ b/client/src/assets/player/shared/web-video/web-video-plugin.ts @@ -1,8 +1,8 @@ import debug from 'debug' import videojs from 'video.js' import { logger } from '@root-helpers/logger' -import { addQueryParams } from '@shared/core-utils' -import { VideoFile } from '@shared/models' +import { addQueryParams } from '@peertube/peertube-core-utils' +import { VideoFile } from '@peertube/peertube-models' import { PeerTubeResolution, PlayerNetworkInfo, WebVideoPluginOptions } from '../../types' const debugLogger = debug('peertube:player:web-video-plugin') diff --git a/client/src/assets/player/translations-manager.ts b/client/src/assets/player/translations-manager.ts index bf9c2d471..03d5dd91e 100644 --- a/client/src/assets/player/translations-manager.ts +++ b/client/src/assets/player/translations-manager.ts @@ -1,5 +1,5 @@ +import { getCompleteLocale, getShortLocale, is18nLocale, isDefaultLocale } from '@peertube/peertube-core-utils' import { logger } from '@root-helpers/logger' -import { getCompleteLocale, getShortLocale, is18nLocale, isDefaultLocale } from '@shared/core-utils/i18n' export class TranslationsManager { private static videojsLocaleCache: { [ path: string ]: any } = {} diff --git a/client/src/assets/player/types/peertube-player-options.ts b/client/src/assets/player/types/peertube-player-options.ts index e1b8c7fab..352f7d8dd 100644 --- a/client/src/assets/player/types/peertube-player-options.ts +++ b/client/src/assets/player/types/peertube-player-options.ts @@ -1,5 +1,5 @@ +import { LiveVideoLatencyModeType, VideoFile } from '@peertube/peertube-models' import { PluginsManager } from '@root-helpers/plugins-manager' -import { LiveVideoLatencyMode, VideoFile } from '@shared/models' import { PeerTubeDockPluginOptions } from '../shared/dock/peertube-dock-plugin' import { PlaylistPluginOptions, VideoJSCaption, VideoJSStoryboard } from './peertube-videojs-typings' @@ -59,7 +59,7 @@ export type PeerTubePlayerLoadOptions = { isLive: boolean liveOptions?: { - latencyMode: LiveVideoLatencyMode + latencyMode: LiveVideoLatencyModeType } videoCaptions: VideoJSCaption[] diff --git a/client/src/assets/player/types/peertube-videojs-typings.ts b/client/src/assets/player/types/peertube-videojs-typings.ts index 73b38d0d3..dae9e14c8 100644 --- a/client/src/assets/player/types/peertube-videojs-typings.ts +++ b/client/src/assets/player/types/peertube-videojs-typings.ts @@ -1,7 +1,7 @@ import { HlsConfig, Level } from 'hls.js' import videojs from 'video.js' import { Engine } from '@peertube/p2p-media-loader-hlsjs' -import { VideoFile, VideoPlaylist, VideoPlaylistElement } from '@shared/models' +import { VideoFile, VideoPlaylist, VideoPlaylistElement } from '@peertube/peertube-models' import { BezelsPlugin } from '../shared/bezels/bezels-plugin' import { StoryboardPlugin } from '../shared/control-bar/storyboard-plugin' import { PeerTubeDockPlugin, PeerTubeDockPluginOptions } from '../shared/dock/peertube-dock-plugin' diff --git a/client/src/assets/player/utils.ts b/client/src/assets/player/utils.ts index cc303b80b..b56e3df8f 100644 --- a/client/src/assets/player/utils.ts +++ b/client/src/assets/player/utils.ts @@ -1,4 +1,4 @@ -import { HTMLServerConfig, Video, VideoFile } from '@shared/models' +import { HTMLServerConfig, Video, VideoFile } from '@peertube/peertube-models' function toTitleCase (str: string) { return str.charAt(0).toUpperCase() + str.slice(1) diff --git a/client/src/root-helpers/logger.ts b/client/src/root-helpers/logger.ts index 8181c13f3..108228d84 100644 --- a/client/src/root-helpers/logger.ts +++ b/client/src/root-helpers/logger.ts @@ -1,4 +1,4 @@ -import { ClientLogCreate } from '@shared/models/server' +import { ClientLogCreate } from '@peertube/peertube-models' import { peertubeLocalStorage } from './peertube-web-storage' import { OAuthUserTokens } from './users' diff --git a/client/src/root-helpers/plugins-manager.ts b/client/src/root-helpers/plugins-manager.ts index fd7b5233b..e987f16d6 100644 --- a/client/src/root-helpers/plugins-manager.ts +++ b/client/src/root-helpers/plugins-manager.ts @@ -3,7 +3,7 @@ import * as debug from 'debug' import { firstValueFrom, ReplaySubject } from 'rxjs' import { first, shareReplay } from 'rxjs/operators' import { RegisterClientHelpers } from 'src/types/register-client-option.model' -import { getExternalAuthHref, getHookType, internalRunHook } from '@shared/core-utils/plugins/hooks' +import { getExternalAuthHref, getHookType, internalRunHook } from '@peertube/peertube-core-utils' import { ClientHookName, clientHookObject, @@ -11,13 +11,14 @@ import { HTMLServerConfig, PluginClientScope, PluginType, + PluginType_Type, RegisterClientFormFieldOptions, RegisterClientHookOptions, RegisterClientRouteOptions, RegisterClientSettingsScriptOptions, RegisterClientVideoFieldOptions, ServerConfigPlugin -} from '@shared/models' +} from '@peertube/peertube-models' import { environment } from '../environments/environment' import { ClientScript } from '../types' import { logger } from './logger' @@ -32,7 +33,7 @@ type Hooks = { [ name: string ]: HookStructValue[] } type PluginInfo = { plugin: ServerConfigPlugin clientScript: ClientScriptJSON - pluginType: PluginType + pluginType: PluginType_Type isTheme: boolean } diff --git a/client/src/root-helpers/video.ts b/client/src/root-helpers/video.ts index 4a44615fb..4ee29df13 100644 --- a/client/src/root-helpers/video.ts +++ b/client/src/root-helpers/video.ts @@ -1,4 +1,4 @@ -import { HTMLServerConfig, Video, VideoPrivacy } from '@shared/models' +import { HTMLServerConfig, Video, VideoPrivacy, VideoPrivacyType } from '@peertube/peertube-models' function buildVideoOrPlaylistEmbed (options: { embedUrl: string @@ -42,13 +42,13 @@ function isP2PEnabled (video: Video, config: HTMLServerConfig, userP2PEnabled: b } function videoRequiresUserAuth (video: Video, videoPassword?: string) { - return new Set([ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]).has(video.privacy.id) || + return new Set([ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]).has(video.privacy.id) || (video.privacy.id === VideoPrivacy.PASSWORD_PROTECTED && !videoPassword) } -function videoRequiresFileToken (video: Video, videoPassword?: string) { - return new Set([ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL, VideoPrivacy.PASSWORD_PROTECTED ]).has(video.privacy.id) +function videoRequiresFileToken (video: Video) { + return new Set([ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL, VideoPrivacy.PASSWORD_PROTECTED ]).has(video.privacy.id) } export { diff --git a/client/src/standalone/videos/embed.ts b/client/src/standalone/videos/embed.ts index 78b812ffd..e4f723079 100644 --- a/client/src/standalone/videos/embed.ts +++ b/client/src/standalone/videos/embed.ts @@ -11,7 +11,7 @@ import { VideoPlaylist, VideoPlaylistElement, VideoState -} from '../../../../shared/models' +} from '@peertube/peertube-models' import { PeerTubePlayer } from '../../assets/player/peertube-player' import { TranslationsManager } from '../../assets/player/translations-manager' import { getParamString, logger, videoRequiresFileToken } from '../../root-helpers' diff --git a/client/src/standalone/videos/shared/auth-http.ts b/client/src/standalone/videos/shared/auth-http.ts index c1e9f7750..3dfc47b88 100644 --- a/client/src/standalone/videos/shared/auth-http.ts +++ b/client/src/standalone/videos/shared/auth-http.ts @@ -1,4 +1,4 @@ -import { HttpStatusCode, OAuth2ErrorCode, UserRefreshToken } from '../../../../../shared/models' +import { HttpStatusCode, OAuth2ErrorCode, OAuth2ErrorCodeType, UserRefreshToken } from '@peertube/peertube-models' import { OAuthUserTokens, objectToUrlEncoded } from '../../../root-helpers' import { peertubeLocalStorage } from '../../../root-helpers/peertube-web-storage' @@ -66,7 +66,7 @@ export class AuthHTTP { if (res.status === HttpStatusCode.UNAUTHORIZED_401) return undefined return res.json() - }).then((obj: UserRefreshToken & { code?: OAuth2ErrorCode }) => { + }).then((obj: UserRefreshToken & { code?: OAuth2ErrorCodeType }) => { if (!obj || obj.code === OAuth2ErrorCode.INVALID_GRANT) { OAuthUserTokens.flushLocalStorage(peertubeLocalStorage) this.removeTokensFromHeaders() diff --git a/client/src/standalone/videos/shared/live-manager.ts b/client/src/standalone/videos/shared/live-manager.ts index 5fac229ba..274f70d9c 100644 --- a/client/src/standalone/videos/shared/live-manager.ts +++ b/client/src/standalone/videos/shared/live-manager.ts @@ -1,5 +1,5 @@ import { Socket } from 'socket.io-client' -import { LiveVideoEventPayload, VideoDetails, VideoState } from '../../../../../shared/models' +import { LiveVideoEventPayload, VideoDetails, VideoState, VideoStateType } from '@peertube/peertube-models' import { PlayerHTML } from './player-html' import { Translations } from './translations' @@ -49,7 +49,7 @@ export class LiveManager { } displayInfo (options: { - state: VideoState + state: VideoStateType translations: Translations }) { const { state, translations } = options diff --git a/client/src/standalone/videos/shared/peertube-plugin.ts b/client/src/standalone/videos/shared/peertube-plugin.ts index daf6f2b03..95433299e 100644 --- a/client/src/standalone/videos/shared/peertube-plugin.ts +++ b/client/src/standalone/videos/shared/peertube-plugin.ts @@ -1,5 +1,5 @@ -import { peertubeTranslate } from '../../../../../shared/core-utils/i18n' -import { HTMLServerConfig, PublicServerSetting } from '../../../../../shared/models' +import { peertubeTranslate } from '@peertube/peertube-core-utils' +import { HTMLServerConfig, PublicServerSetting } from '@peertube/peertube-models' import { PluginInfo, PluginsManager } from '../../../root-helpers' import { RegisterClientHelpers } from '../../../types' import { AuthHTTP } from './auth-http' diff --git a/client/src/standalone/videos/shared/player-html.ts b/client/src/standalone/videos/shared/player-html.ts index 0defa0d70..ada2aaaf7 100644 --- a/client/src/standalone/videos/shared/player-html.ts +++ b/client/src/standalone/videos/shared/player-html.ts @@ -1,4 +1,4 @@ -import { peertubeTranslate } from '../../../../../shared/core-utils/i18n' +import { peertubeTranslate } from '@peertube/peertube-core-utils' import { logger } from '../../../root-helpers' import { Translations } from './translations' diff --git a/client/src/standalone/videos/shared/player-options-builder.ts b/client/src/standalone/videos/shared/player-options-builder.ts index 8a4e32444..3437ef421 100644 --- a/client/src/standalone/videos/shared/player-options-builder.ts +++ b/client/src/standalone/videos/shared/player-options-builder.ts @@ -1,4 +1,4 @@ -import { peertubeTranslate } from '../../../../../shared/core-utils/i18n' +import { peertubeTranslate } from '@peertube/peertube-core-utils' import { HTMLServerConfig, LiveVideo, @@ -9,7 +9,7 @@ import { VideoPlaylistElement, VideoState, VideoStreamingPlaylistType -} from '../../../../../shared/models' +} from '@peertube/peertube-models' import { HLSOptions, PeerTubePlayerContructorOptions, PeerTubePlayerLoadOptions, PlayerMode, VideoJSCaption } from '../../../assets/player' import { getBoolOrDefault, diff --git a/client/src/standalone/videos/shared/playlist-fetcher.ts b/client/src/standalone/videos/shared/playlist-fetcher.ts index 713d82e3a..db38e3d6c 100644 --- a/client/src/standalone/videos/shared/playlist-fetcher.ts +++ b/client/src/standalone/videos/shared/playlist-fetcher.ts @@ -1,4 +1,4 @@ -import { HttpStatusCode, ResultList, VideoPlaylistElement } from '../../../../../shared/models' +import { HttpStatusCode, ResultList, VideoPlaylistElement } from '@peertube/peertube-models' import { logger } from '../../../root-helpers' import { AuthHTTP } from './auth-http' diff --git a/client/src/standalone/videos/shared/playlist-tracker.ts b/client/src/standalone/videos/shared/playlist-tracker.ts index d8708826d..dc9a084b7 100644 --- a/client/src/standalone/videos/shared/playlist-tracker.ts +++ b/client/src/standalone/videos/shared/playlist-tracker.ts @@ -1,4 +1,4 @@ -import { VideoPlaylist, VideoPlaylistElement } from '../../../../../shared/models' +import { VideoPlaylist, VideoPlaylistElement } from '@peertube/peertube-models' import { logger } from '../../../root-helpers' export class PlaylistTracker { diff --git a/client/src/standalone/videos/shared/video-fetcher.ts b/client/src/standalone/videos/shared/video-fetcher.ts index 7fb94fbf3..9149d946e 100644 --- a/client/src/standalone/videos/shared/video-fetcher.ts +++ b/client/src/standalone/videos/shared/video-fetcher.ts @@ -1,4 +1,4 @@ -import { HttpStatusCode, LiveVideo, VideoDetails, VideoToken } from '../../../../../shared/models' +import { HttpStatusCode, LiveVideo, VideoDetails, VideoToken } from '@peertube/peertube-models' import { logger } from '../../../root-helpers' import { PeerTubeServerError } from '../../../types' import { AuthHTTP } from './auth-http' diff --git a/client/src/types/job-state-client.type.ts b/client/src/types/job-state-client.type.ts index 6123678df..6697539a3 100644 --- a/client/src/types/job-state-client.type.ts +++ b/client/src/types/job-state-client.type.ts @@ -1,3 +1,3 @@ -import { JobState } from '@shared/models' +import { JobState } from '@peertube/peertube-models' export type JobStateClient = JobState diff --git a/client/src/types/job-type-client.type.ts b/client/src/types/job-type-client.type.ts index 7d51f1db2..930809081 100644 --- a/client/src/types/job-type-client.type.ts +++ b/client/src/types/job-type-client.type.ts @@ -1,3 +1,3 @@ -import { JobType } from '@shared/models' +import { JobType } from '@peertube/peertube-models' export type JobTypeClient = 'all' | JobType diff --git a/client/src/types/register-client-option.model.ts b/client/src/types/register-client-option.model.ts index 2c09f15a7..2336119bb 100644 --- a/client/src/types/register-client-option.model.ts +++ b/client/src/types/register-client-option.model.ts @@ -5,7 +5,7 @@ import { RegisterClientSettingsScriptOptions, RegisterClientVideoFieldOptions, ServerConfig, SettingEntries -} from '@shared/models' +} from '@peertube/peertube-models' export type RegisterClientOptions = { registerHook: (options: RegisterClientHookOptions) => void diff --git a/client/src/types/server-error.model.ts b/client/src/types/server-error.model.ts index 4a57287fe..096deb50b 100644 --- a/client/src/types/server-error.model.ts +++ b/client/src/types/server-error.model.ts @@ -1,9 +1,9 @@ -import { ServerErrorCode } from '@shared/models/index' +import { ServerErrorCodeType } from '@peertube/peertube-models' export class PeerTubeServerError extends Error { - serverCode: ServerErrorCode + serverCode: ServerErrorCodeType - constructor (message: string, serverCode: ServerErrorCode) { + constructor (message: string, serverCode: ServerErrorCodeType) { super(message) this.name = 'CustomError' this.serverCode = serverCode diff --git a/client/tsconfig.eslint.json b/client/tsconfig.eslint.json index a18c461fe..91bb53e1d 100644 --- a/client/tsconfig.eslint.json +++ b/client/tsconfig.eslint.json @@ -4,5 +4,10 @@ // adjust "includes" to what makes sense for you and your project "src/**/*.ts", "e2e/**/*.ts" + ], + "references": [ + { "path": "../packages/core-utils" }, + { "path": "../packages/models" }, + { "path": "../packages/typescript-utils" } ] } diff --git a/client/tsconfig.json b/client/tsconfig.json index 5dee39362..48c2ff7f7 100644 --- a/client/tsconfig.json +++ b/client/tsconfig.json @@ -11,6 +11,7 @@ "noImplicitAny": true, "noImplicitThis": true, "alwaysStrict": true, + "allowJs": true, "importHelpers": true, "allowSyntheticDefaultImports": true, "strictBindCallApply": true, @@ -37,24 +38,6 @@ "@app/*": [ "src/app/*" ], - "@shared/models/*": [ - "../shared/models/*" - ], - "@shared/models": [ - "../shared/models" - ], - "@shared/core-utils": [ - "../shared/core-utils" - ], - "@shared/core-utils/*": [ - "../shared/core-utils/*" - ], - "@shared/typescript-utils": [ - "../shared/typescript-utils" - ], - "@shared/typescript-utils/*": [ - "../shared/typescript-utils/*" - ], "@root-helpers/*": [ "src/root-helpers/*" ], @@ -70,6 +53,11 @@ }, "useDefineForClassFields": false }, + "references": [ + { "path": "../packages/core-utils" }, + { "path": "../packages/models" }, + { "path": "../packages/typescript-utils" } + ], "files": [ "src/polyfills.ts" ], @@ -78,10 +66,6 @@ "src/**/*.d.ts", "src/shims/*.ts" ], - "exclude": [ - "../node_modules", - "../server" - ], "angularCompilerOptions": { "strictInjectionParameters": true, "fullTemplateTypeCheck": true, diff --git a/client/tsconfig.types.json b/client/tsconfig.types.json index 99d96d413..1481bc9b9 100644 --- a/client/tsconfig.types.json +++ b/client/tsconfig.types.json @@ -4,15 +4,12 @@ "stripInternal": true, "removeComments": false, "declaration": true, - "outDir": "../packages/types/dist/client/", + "outDir": "../packages/types-generator/dist/client/", "emitDeclarationOnly": true, "composite": true, "rootDir": "src/", - "tsBuildInfoFile": "../packages/types/dist/tsconfig.client.tsbuildinfo" + "tsBuildInfoFile": "../packages/types-generator/dist/tsconfig.client.tsbuildinfo" }, - "references": [ - { "path": "../shared/tsconfig.types.json" } - ], "files": [ "src/types/index.ts" ], "include": [ "src/types/**/*" diff --git a/client/webpack/webpack.video-embed.js b/client/webpack/webpack.video-embed.js index 47d440c25..30a8ec27c 100644 --- a/client/webpack/webpack.video-embed.js +++ b/client/webpack/webpack.video-embed.js @@ -29,9 +29,7 @@ module.exports = function () { alias: { 'video.js$': path.resolve('node_modules/video.js/core.js'), 'hls.js$': path.resolve('node_modules/hls.js/dist/hls.light.js'), - '@root-helpers': path.resolve('src/root-helpers'), - '@shared/models': path.resolve('../shared/models'), - '@shared/core-utils': path.resolve('../shared/core-utils') + '@root-helpers': path.resolve('src/root-helpers') }, fallback: { diff --git a/client/yarn.lock b/client/yarn.lock index 5c9f4bf42..cc0bd23a6 100644 --- a/client/yarn.lock +++ b/client/yarn.lock @@ -1353,6 +1353,13 @@ "@babel/helper-validator-identifier" "^7.19.1" to-fast-properties "^2.0.0" +"@cspotcode/source-map-support@^0.8.0": + version "0.8.1" + resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz#00629c35a688e05a88b1cda684fb9d5e73f000a1" + integrity sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw== + dependencies: + "@jridgewell/trace-mapping" "0.3.9" + "@csstools/css-parser-algorithms@^2.1.1": version "2.1.1" resolved "https://registry.yarnpkg.com/@csstools/css-parser-algorithms/-/css-parser-algorithms-2.1.1.tgz#7b62e6412a468a2d1096ed267edd1e4a7fd4a119" @@ -1818,6 +1825,11 @@ resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz#2203b118c157721addfe69d47b70465463066d78" integrity sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w== +"@jridgewell/resolve-uri@^3.0.3": + version "3.1.1" + resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz#c08679063f279615a3326583ba3a90d1d82cc721" + integrity sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA== + "@jridgewell/set-array@^1.0.1": version "1.1.2" resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.1.2.tgz#7c6cf998d6d20b914c0a55a91ae928ff25965e72" @@ -1841,6 +1853,14 @@ resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz#d7c6e6755c78567a951e04ab52ef0fd26de59f32" integrity sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg== +"@jridgewell/trace-mapping@0.3.9": + version "0.3.9" + resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz#6534fd5933a53ba7cbf3a17615e273a0d1273ff9" + integrity sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ== + dependencies: + "@jridgewell/resolve-uri" "^3.0.3" + "@jridgewell/sourcemap-codec" "^1.4.10" + "@jridgewell/trace-mapping@^0.3.17", "@jridgewell/trace-mapping@^0.3.9": version "0.3.18" resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.18.tgz#25783b2086daf6ff1dcb53c9249ae480e4dd4cd6" @@ -2204,6 +2224,26 @@ resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-2.0.0.tgz#f544a148d3ab35801c1f633a7441fd87c2e484bf" integrity sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A== +"@tsconfig/node10@^1.0.7": + version "1.0.9" + resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.9.tgz#df4907fc07a886922637b15e02d4cebc4c0021b2" + integrity sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA== + +"@tsconfig/node12@^1.0.7": + version "1.0.11" + resolved "https://registry.yarnpkg.com/@tsconfig/node12/-/node12-1.0.11.tgz#ee3def1f27d9ed66dac6e46a295cffb0152e058d" + integrity sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag== + +"@tsconfig/node14@^1.0.0": + version "1.0.3" + resolved "https://registry.yarnpkg.com/@tsconfig/node14/-/node14-1.0.3.tgz#e4386316284f00b98435bf40f72f75a09dabf6c1" + integrity sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow== + +"@tsconfig/node16@^1.0.2": + version "1.0.4" + resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.4.tgz#0b92dcc0cc1c81f6f306a381f28e31b1a56536e9" + integrity sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA== + "@tufjs/canonical-json@1.0.0": version "1.0.0" resolved "https://registry.yarnpkg.com/@tufjs/canonical-json/-/canonical-json-1.0.0.tgz#eade9fd1f537993bc1f0949f3aea276ecc4fab31" @@ -3140,7 +3180,7 @@ acorn-jsx@^5.3.2: resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937" integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ== -acorn-walk@^8.0.0: +acorn-walk@^8.0.0, acorn-walk@^8.1.1: version "8.2.0" resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.2.0.tgz#741210f2e2426454508853a2f44d0ab83b7f69c1" integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA== @@ -3150,6 +3190,11 @@ acorn@^8.0.4, acorn@^8.5.0, acorn@^8.7.1, acorn@^8.8.0: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.2.tgz#1b2f25db02af965399b9776b0c2c391276d37c4a" integrity sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw== +acorn@^8.4.1: + version "8.10.0" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.10.0.tgz#8be5b3907a67221a81ab23c7889c4c5526b62ec5" + integrity sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw== + addr-to-ip-port@^1.0.1: version "1.5.4" resolved "https://registry.yarnpkg.com/addr-to-ip-port/-/addr-to-ip-port-1.5.4.tgz#9542b1c6219fdb8c9ce6cc72c14ee880ab7ddd88" @@ -3392,6 +3437,11 @@ are-we-there-yet@^3.0.0: delegates "^1.0.0" readable-stream "^3.6.0" +arg@^4.1.0: + version "4.1.3" + resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089" + integrity sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA== + argparse@^1.0.7: version "1.0.10" resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911" @@ -3512,7 +3562,7 @@ available-typed-arrays@^1.0.5: resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz#92f95616501069d07d10edb2fc37d3e1c65123b7" integrity sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw== -axios@^1.0.0, axios@^1.2.1: +axios@^1.0.0, axios@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/axios/-/axios-1.4.0.tgz#38a7bf1224cd308de271146038b551d725f0be1f" integrity sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA== @@ -4088,14 +4138,14 @@ chrome-trace-event@^1.0.2: resolved "https://registry.yarnpkg.com/chrome-trace-event/-/chrome-trace-event-1.0.3.tgz#1015eced4741e15d06664a957dbbf50d041e26ac" integrity sha512-p3KULyQg4S7NIHixdwbGX+nFHkoBiA4YQmyWtjb8XngSKV124nJmRysgAeujbUVb15vh+RvFUfCPqU7rXk+hZg== -chromedriver@^113.0.0: - version "113.0.0" - resolved "https://registry.yarnpkg.com/chromedriver/-/chromedriver-113.0.0.tgz#d4855f156ee51cea4282e04aadd29fa154e44dbb" - integrity sha512-UnQlt2kPicYXVNHPzy9HfcWvEbKJjjKAEaatdcnP/lCIRwuSoZFVLH0HVDAGdbraXp3dNVhfE2Qx7gw8TnHnPw== +chromedriver@^115.0.1: + version "115.0.1" + resolved "https://registry.yarnpkg.com/chromedriver/-/chromedriver-115.0.1.tgz#76cbf35f16e0c1f5e29ab821fb3b8b06d22c3e40" + integrity sha512-faE6WvIhXfhnoZ3nAxUXYzeDCKy612oPwpkUp0mVkA7fZPg2JHSUiYOQhUYgzHQgGvDWD5Fy2+M2xV55GKHBVQ== dependencies: "@testim/chrome-version" "^1.1.3" - axios "^1.2.1" - compare-versions "^5.0.1" + axios "^1.4.0" + compare-versions "^6.0.0" extract-zip "^2.0.1" https-proxy-agent "^5.0.1" proxy-from-env "^1.1.0" @@ -4307,10 +4357,10 @@ compact2string@^1.4.1: dependencies: ipaddr.js ">= 0.1.5" -compare-versions@^5.0.1: - version "5.0.3" - resolved "https://registry.yarnpkg.com/compare-versions/-/compare-versions-5.0.3.tgz#a9b34fea217472650ef4a2651d905f42c28ebfd7" - integrity sha512-4UZlZP8Z99MGEY+Ovg/uJxJuvoXuN4M6B3hKaiackiHrgzQFEe3diJi1mf1PNHbFujM7FvLrK2bpgIaImbtZ1A== +compare-versions@^6.0.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/compare-versions/-/compare-versions-6.1.0.tgz#3f2131e3ae93577df111dba133e6db876ffe127a" + integrity sha512-LNZQXhqUvqUTotpZ00qLSaify3b4VFD588aRr8MKFw4CMUr98ytzCW5wDH5qx/DEY5kCDXcbcRuCqL0szEf2tg== compress-commons@^4.1.0: version "4.1.1" @@ -4453,6 +4503,11 @@ crc32-stream@^4.0.2: crc-32 "^1.2.0" readable-stream "^3.4.0" +create-require@^1.1.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" + integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ== + critters@0.0.16: version "0.0.16" resolved "https://registry.yarnpkg.com/critters/-/critters-0.0.16.tgz#ffa2c5561a65b43c53b940036237ce72dcebfe93" @@ -4799,6 +4854,11 @@ diff@5.0.0: resolved "https://registry.yarnpkg.com/diff/-/diff-5.0.0.tgz#7ed6ad76d859d030787ec35855f5b1daf31d852b" integrity sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w== +diff@^4.0.1: + version "4.0.2" + resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d" + integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A== + diff@^5.0.0: version "5.1.0" resolved "https://registry.yarnpkg.com/diff/-/diff-5.1.0.tgz#bc52d298c5ea8df9194800224445ed43ffc87e40" @@ -7694,6 +7754,11 @@ make-dir@^3.0.2: dependencies: semver "^6.0.0" +make-error@^1.1.1: + version "1.3.6" + resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2" + integrity sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw== + make-fetch-happen@^10.0.3: version "10.2.1" resolved "https://registry.yarnpkg.com/make-fetch-happen/-/make-fetch-happen-10.2.1.tgz#f5e3835c5e9817b617f2770870d9492d28678164" @@ -10802,6 +10867,25 @@ ts-loader@^9.3.0: micromatch "^4.0.0" semver "^7.3.4" +ts-node@^10.9.1: + version "10.9.1" + resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.9.1.tgz#e73de9102958af9e1f0b168a6ff320e25adcff4b" + integrity sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw== + dependencies: + "@cspotcode/source-map-support" "^0.8.0" + "@tsconfig/node10" "^1.0.7" + "@tsconfig/node12" "^1.0.7" + "@tsconfig/node14" "^1.0.0" + "@tsconfig/node16" "^1.0.2" + acorn "^8.4.1" + acorn-walk "^8.1.1" + arg "^4.1.0" + create-require "^1.1.0" + diff "^4.0.1" + make-error "^1.1.1" + v8-compile-cache-lib "^3.0.1" + yn "3.1.1" + tsconfig-paths@^3.14.1: version "3.14.2" resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.14.2.tgz#6e32f1f79412decd261f92d633a9dc1cfa99f088" @@ -11096,6 +11180,11 @@ uuid@^9.0.0: resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.0.tgz#592f550650024a38ceb0c562f2f6aa435761efb5" integrity sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg== +v8-compile-cache-lib@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf" + integrity sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg== + v8-compile-cache@2.3.0, v8-compile-cache@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz#2de19618c66dc247dcfb6f99338035d8245a2cee" @@ -11792,6 +11881,11 @@ yauzl@^2.10.0: buffer-crc32 "~0.2.3" fd-slicer "~1.1.0" +yn@3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50" + integrity sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q== + yocto-queue@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" diff --git a/config/default.yaml b/config/default.yaml index 9d217b56b..fcd634d1d 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -261,6 +261,7 @@ open_telemetry: port: 9091 tracing: + # If tracing is enabled, you must provide --experimental-loader=@opentelemetry/instrumentation/hook.mjs flag to the node binary enabled: false # Send traces to a Jaeger compatible endpoint diff --git a/config/production.yaml.example b/config/production.yaml.example index 2afc5f982..2ec7c3fca 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -259,6 +259,7 @@ open_telemetry: port: 9091 tracing: + # If tracing is enabled, you must provide --experimental-loader=@opentelemetry/instrumentation/hook.mjs flag to the node binary enabled: false # Send traces to a Jaeger compatible endpoint diff --git a/package.json b/package.json index 7573232a1..89f8fd4ed 100644 --- a/package.json +++ b/package.json @@ -8,9 +8,7 @@ "node": ">=16.x", "yarn": ">=1.x" }, - "bin": { - "peertube": "dist/server/tools/peertube.js" - }, + "type": "module", "author": { "name": "Chocobozzz", "email": "chocobozzz@framasoft.org", @@ -21,49 +19,60 @@ "url": "git+https://github.com/Chocobozzz/PeerTube.git" }, "typings": "*.d.ts", + "workspaces": [ + "packages/*", + "server" + ], "scripts": { "e2e:browserstack": "bash ./scripts/e2e/browserstack.sh", "e2e:local": "bash ./scripts/e2e/local.sh", - "setup:cli": "bash ./scripts/setup/cli.sh", "build": "bash ./scripts/build/index.sh", "build:embed": "bash ./scripts/build/embed.sh", "build:server": "bash ./scripts/build/server.sh", "build:client": "bash ./scripts/build/client.sh", "build:peertube-runner": "bash ./scripts/build/peertube-runner.sh", + "build:peertube-cli": "bash ./scripts/build/peertube-cli.sh", + "build:tests": "bash ./scripts/build/tests.sh", "clean:client": "bash ./scripts/clean/client/index.sh", "clean:server:test": "bash ./scripts/clean/server/test.sh", "i18n:update": "bash ./scripts/i18n/update.sh", - "plugin:install": "node ./dist/scripts/plugin/install.js", - "plugin:uninstall": "node ./dist/scripts/plugin/uninstall.js", - "i18n:create-custom-files": "node ./dist/scripts/i18n/create-custom-files.js", - "reset-password": "node ./dist/scripts/reset-password.js", "dev": "bash ./scripts/dev/index.sh", "dev:server": "bash ./scripts/dev/server.sh", "dev:embed": "bash ./scripts/dev/embed.sh", "dev:client": "bash ./scripts/dev/client.sh", - "dev:cli": "bash ./scripts/dev/cli.sh", + "dev:peertube-cli": "bash ./scripts/dev/peertube-cli.sh", "dev:peertube-runner": "bash ./scripts/dev/peertube-runner.sh", "start": "node dist/server", "start:server": "node dist/server --no-client", + "plugin:install": "node ./dist/scripts/plugin/install.js", + "plugin:uninstall": "node ./dist/scripts/plugin/uninstall.js", + "reset-password": "node ./dist/scripts/reset-password.js", "update-host": "node ./dist/scripts/update-host.js", "regenerate-thumbnails": "node ./dist/scripts/regenerate-thumbnails.js", "create-import-video-file-job": "node ./dist/scripts/create-import-video-file-job.js", "create-move-video-storage-job": "node ./dist/scripts/create-move-video-storage-job.js", "create-generate-storyboard-job": "node ./dist/scripts/create-generate-storyboard-job.js", - "test": "bash ./scripts/test.sh", - "generate-cli-doc": "bash ./scripts/generate-cli-doc.sh", - "generate-types-package": "ts-node ./packages/types/generate-package.ts", "parse-log": "node ./dist/scripts/parse-log.js", "prune-storage": "node ./dist/scripts/prune-storage.js", + "test": "bash ./scripts/test.sh", + "generate-cli-doc": "bash ./scripts/generate-cli-doc.sh", + "generate-types-package": "tsx --conditions=peertube:tsx ./packages/types-generator/generate-package.ts", + "i18n:create-custom-files": "tsx --conditions=peertube:tsx ./scripts/i18n/create-custom-files.ts", + "benchmark-server": "tsx --conditions=peertube:tsx ./scripts/benchmark.ts", + "client:build-stats": "tsx --conditions=peertube:tsx ./scripts/client-build-stats.ts", + "generate-code-contributors": "tsx --conditions=peertube:tsx ./scripts/generate-code-contributors.ts", + "simulate-many-viewers": "tsx --conditions=peertube:tsx ./scripts/simulate-many-viewers.ts", "postinstall": "test -n \"$NOCLIENT\" || (cd client && yarn install --pure-lockfile)", "tsc": "tsc", "commander": "commander", "lint": "npm run ci -- lint", "ng": "ng", - "ts-node": "ts-node", + "tsx": "tsx", "eslint": "eslint", "resolve-tspaths": "resolve-tspaths", - "resolve-tspaths:server": "npm run resolve-tspaths -- --project tsconfig.json --src . --out dist", + "resolve-tspaths:server": "npm run resolve-tspaths -- --project server/tsconfig.json --src server --out dist", + "resolve-tspaths:server-lib": "npm run resolve-tspaths -- --project server/tsconfig.lib.json --src server --out server/dist", + "resolve-tspaths:tests": "npm run resolve-tspaths -- --project packages/tests/tsconfig.json --src packages/tests/src --out packages/tests/dist", "concurrently": "concurrently", "mocha": "mocha", "ci": "bash ./scripts/ci.sh", @@ -80,29 +89,30 @@ "@aws-sdk/node-http-handler": "^3.190.0", "@aws-sdk/s3-request-presigner": "^3.345.0", "@babel/parser": "^7.17.8", + "@commander-js/extra-typings": "^11.0.0", "@node-oauth/oauth2-server": "^4.2.0", "@opentelemetry/api": "^1.1.0", - "@opentelemetry/exporter-jaeger": "^1.3.1", - "@opentelemetry/exporter-prometheus": "~0.39.1", - "@opentelemetry/instrumentation": "^0.39.1", - "@opentelemetry/instrumentation-dns": "^0.31.2", - "@opentelemetry/instrumentation-express": "^0.32.1", - "@opentelemetry/instrumentation-fs": "^0.7.0", - "@opentelemetry/instrumentation-http": "^0.39.1", - "@opentelemetry/instrumentation-ioredis": "^0.34.2", - "@opentelemetry/instrumentation-pg": "^0.35.2", - "@opentelemetry/resources": "^1.3.1", - "@opentelemetry/sdk-metrics": "^1.8.0", - "@opentelemetry/sdk-trace-base": "^1.3.1", - "@opentelemetry/sdk-trace-node": "^1.3.1", - "@opentelemetry/semantic-conventions": "^1.3.1", + "@opentelemetry/exporter-jaeger": "^1.15.1", + "@opentelemetry/exporter-prometheus": "~0.41.1", + "@opentelemetry/instrumentation": "^0.41.1", + "@opentelemetry/instrumentation-dns": "^0.32.0", + "@opentelemetry/instrumentation-express": "^0.33.0", + "@opentelemetry/instrumentation-fs": "^0.8.0", + "@opentelemetry/instrumentation-http": "^0.41.1", + "@opentelemetry/instrumentation-ioredis": "^0.35.0", + "@opentelemetry/instrumentation-pg": "^0.36.0", + "@opentelemetry/resources": "^1.15.1", + "@opentelemetry/sdk-metrics": "^1.15.1", + "@opentelemetry/sdk-trace-base": "^1.15.1", + "@opentelemetry/sdk-trace-node": "^1.15.1", + "@opentelemetry/semantic-conventions": "^1.15.1", "@peertube/feed": "^5.1.0", "@peertube/http-signature": "^1.7.0", "@uploadx/core": "^6.0.0", "async-lru": "^1.1.1", "async-mutex": "^0.4.0", "bcrypt": "5.1.0", - "bencode": "^2.0.2", + "bencode": "^3.1.1", "bittorrent-tracker": "^9", "bluebird": "^3.5.0", "bullmq": "^3.6.6", @@ -122,7 +132,7 @@ "flat": "^5.0.0", "fluent-ffmpeg": "^2.1.0", "fs-extra": "^11.1.0", - "got": "^11.8.2", + "got": "^13.0.0", "helmet": "^7.0.0", "hpagent": "^1.0.0", "http-problem-details": "^0.1.5", @@ -130,11 +140,11 @@ "ip-anonymize": "^0.1.0", "ipaddr.js": "2.0.1", "is-cidr": "^4.0.0", - "iso-639-3": "2.2.0", + "iso-639-3": "3.0.1", "jimp": "^0.22.4", "js-yaml": "^4.0.0", "jsonld": "~8.2.0", - "lodash": "^4.17.21", + "lodash-es": "^4.17.21", "lru-cache": "^9.1.1", "magnet-uri": "^6", "markdown-it": "^13.0.1", @@ -145,9 +155,9 @@ "multer": "^1.4.5-lts.1", "node-media-server": "^2.1.4", "nodemailer": "^6.0.0", - "opentelemetry-instrumentation-sequelize": "^0.35.0", + "opentelemetry-instrumentation-sequelize": "^0.39.1", "otpauth": "^9.0.2", - "p-queue": "^6", + "p-queue": "^7.3.4", "parse-torrent": "^9", "password-generator": "^2.0.2", "pg": "^8.2.1", @@ -164,7 +174,6 @@ "socket.io": "^4.5.4", "sql-formatter": "^12.0.1", "srt-to-vtt": "^1.1.2", - "tsconfig-paths": "^4.0.0", "tslib": "^2.0.0", "useragent": "^2.3.0", "validator": "^13.0.0", @@ -175,6 +184,7 @@ }, "devDependencies": { "@peertube/maildev": "^1.2.0", + "@peertube/resolve-tspaths": "^0.8.14", "@types/bcrypt": "^5.0.0", "@types/bencode": "^2.0.0", "@types/bluebird": "^3.5.33", @@ -188,7 +198,8 @@ "@types/express": "4.17.9", "@types/fluent-ffmpeg": "^2.1.16", "@types/fs-extra": "^11.0.1", - "@types/lodash": "^4.14.64", + "@types/jsonld": "^1.5.9", + "@types/lodash-es": "^4.17.8", "@types/magnet-uri": "^5.1.1", "@types/maildev": "^0.0.4", "@types/memoizee": "^0.4.2", @@ -200,7 +211,7 @@ "@types/oauth2-server": "^3.0.8", "@types/request": "^2.0.3", "@types/supertest": "^2.0.3", - "@types/validator": "^13.0.0", + "@types/validator": "^13.9.0", "@types/webtorrent": "^0.109.0", "@types/ws": "^8.2.0", "@typescript-eslint/eslint-plugin": "^5.0.0", @@ -210,6 +221,7 @@ "chai-xml": "^0.4.0", "concurrently": "^8.0.1", "depcheck": "^1.4.2", + "esbuild": "^0.19.0", "eslint": "8.41.0", "eslint-config-standard-with-typescript": "34.0.1", "eslint-plugin-import": "^2.20.1", @@ -222,12 +234,11 @@ "pixelmatch": "^5.3.0", "pngjs": "^7.0.0", "proxy": "^2.1.1", - "resolve-tspaths": "^0.8.8", "socket.io-client": "^4.5.4", "supertest": "^6.0.1", "swagger-cli": "^4.0.2", - "ts-node": "^10.8.1", "tsc-watch": "^6.0.0", + "tsx": "^3.12.7", "typescript": "~5.0.4" }, "bundlewatch": { diff --git a/packages/core-utils/package.json b/packages/core-utils/package.json new file mode 100644 index 000000000..d3bf18335 --- /dev/null +++ b/packages/core-utils/package.json @@ -0,0 +1,19 @@ +{ + "name": "@peertube/peertube-core-utils", + "private": true, + "version": "0.0.0", + "main": "dist/index.js", + "files": [ "dist" ], + "exports": { + "types": "./dist/index.d.ts", + "peertube:tsx": "./src/index.ts", + "default": "./dist/index.js" + }, + "type": "module", + "devDependencies": {}, + "scripts": { + "build": "tsc", + "watch": "tsc -w" + }, + "dependencies": {} +} diff --git a/packages/core-utils/src/abuse/abuse-predefined-reasons.ts b/packages/core-utils/src/abuse/abuse-predefined-reasons.ts new file mode 100644 index 000000000..68534a1e0 --- /dev/null +++ b/packages/core-utils/src/abuse/abuse-predefined-reasons.ts @@ -0,0 +1,14 @@ +import { AbusePredefinedReasons, AbusePredefinedReasonsString, AbusePredefinedReasonsType } from '@peertube/peertube-models' + +export const abusePredefinedReasonsMap: { + [key in AbusePredefinedReasonsString]: AbusePredefinedReasonsType +} = { + violentOrRepulsive: AbusePredefinedReasons.VIOLENT_OR_REPULSIVE, + hatefulOrAbusive: AbusePredefinedReasons.HATEFUL_OR_ABUSIVE, + spamOrMisleading: AbusePredefinedReasons.SPAM_OR_MISLEADING, + privacy: AbusePredefinedReasons.PRIVACY, + rights: AbusePredefinedReasons.RIGHTS, + serverRules: AbusePredefinedReasons.SERVER_RULES, + thumbnails: AbusePredefinedReasons.THUMBNAILS, + captions: AbusePredefinedReasons.CAPTIONS +} as const diff --git a/packages/core-utils/src/abuse/index.ts b/packages/core-utils/src/abuse/index.ts new file mode 100644 index 000000000..b79b86155 --- /dev/null +++ b/packages/core-utils/src/abuse/index.ts @@ -0,0 +1 @@ +export * from './abuse-predefined-reasons.js' diff --git a/shared/core-utils/common/array.ts b/packages/core-utils/src/common/array.ts similarity index 100% rename from shared/core-utils/common/array.ts rename to packages/core-utils/src/common/array.ts diff --git a/shared/core-utils/common/date.ts b/packages/core-utils/src/common/date.ts similarity index 100% rename from shared/core-utils/common/date.ts rename to packages/core-utils/src/common/date.ts diff --git a/packages/core-utils/src/common/index.ts b/packages/core-utils/src/common/index.ts new file mode 100644 index 000000000..d7d8599aa --- /dev/null +++ b/packages/core-utils/src/common/index.ts @@ -0,0 +1,10 @@ +export * from './array.js' +export * from './random.js' +export * from './date.js' +export * from './number.js' +export * from './object.js' +export * from './regexp.js' +export * from './time.js' +export * from './promises.js' +export * from './url.js' +export * from './version.js' diff --git a/shared/core-utils/common/number.ts b/packages/core-utils/src/common/number.ts similarity index 100% rename from shared/core-utils/common/number.ts rename to packages/core-utils/src/common/number.ts diff --git a/shared/core-utils/common/object.ts b/packages/core-utils/src/common/object.ts similarity index 100% rename from shared/core-utils/common/object.ts rename to packages/core-utils/src/common/object.ts diff --git a/shared/core-utils/common/promises.ts b/packages/core-utils/src/common/promises.ts similarity index 100% rename from shared/core-utils/common/promises.ts rename to packages/core-utils/src/common/promises.ts diff --git a/shared/core-utils/common/random.ts b/packages/core-utils/src/common/random.ts similarity index 100% rename from shared/core-utils/common/random.ts rename to packages/core-utils/src/common/random.ts diff --git a/shared/core-utils/common/regexp.ts b/packages/core-utils/src/common/regexp.ts similarity index 100% rename from shared/core-utils/common/regexp.ts rename to packages/core-utils/src/common/regexp.ts diff --git a/shared/core-utils/common/time.ts b/packages/core-utils/src/common/time.ts similarity index 100% rename from shared/core-utils/common/time.ts rename to packages/core-utils/src/common/time.ts diff --git a/packages/core-utils/src/common/url.ts b/packages/core-utils/src/common/url.ts new file mode 100644 index 000000000..449b6c9dc --- /dev/null +++ b/packages/core-utils/src/common/url.ts @@ -0,0 +1,150 @@ +import { Video, VideoPlaylist } from '@peertube/peertube-models' +import { secondsToTime } from './date.js' + +function addQueryParams (url: string, params: { [ id: string ]: string }) { + const objUrl = new URL(url) + + for (const key of Object.keys(params)) { + objUrl.searchParams.append(key, params[key]) + } + + return objUrl.toString() +} + +function removeQueryParams (url: string) { + const objUrl = new URL(url) + + objUrl.searchParams.forEach((_v, k) => objUrl.searchParams.delete(k)) + + return objUrl.toString() +} + +function buildPlaylistLink (playlist: Pick, base?: string) { + return (base ?? window.location.origin) + buildPlaylistWatchPath(playlist) +} + +function buildPlaylistWatchPath (playlist: Pick) { + return '/w/p/' + playlist.shortUUID +} + +function buildVideoWatchPath (video: Pick) { + return '/w/' + video.shortUUID +} + +function buildVideoLink (video: Pick, base?: string) { + return (base ?? window.location.origin) + buildVideoWatchPath(video) +} + +function buildPlaylistEmbedPath (playlist: Pick) { + return '/video-playlists/embed/' + playlist.uuid +} + +function buildPlaylistEmbedLink (playlist: Pick, base?: string) { + return (base ?? window.location.origin) + buildPlaylistEmbedPath(playlist) +} + +function buildVideoEmbedPath (video: Pick) { + return '/videos/embed/' + video.uuid +} + +function buildVideoEmbedLink (video: Pick, base?: string) { + return (base ?? window.location.origin) + buildVideoEmbedPath(video) +} + +function decorateVideoLink (options: { + url: string + + startTime?: number + stopTime?: number + + subtitle?: string + + loop?: boolean + autoplay?: boolean + muted?: boolean + + // Embed options + title?: boolean + warningTitle?: boolean + + controls?: boolean + controlBar?: boolean + + peertubeLink?: boolean + p2p?: boolean +}) { + const { url } = options + + const params = new URLSearchParams() + + if (options.startTime !== undefined && options.startTime !== null) { + const startTimeInt = Math.floor(options.startTime) + params.set('start', secondsToTime(startTimeInt)) + } + + if (options.stopTime) { + const stopTimeInt = Math.floor(options.stopTime) + params.set('stop', secondsToTime(stopTimeInt)) + } + + if (options.subtitle) params.set('subtitle', options.subtitle) + + if (options.loop === true) params.set('loop', '1') + if (options.autoplay === true) params.set('autoplay', '1') + if (options.muted === true) params.set('muted', '1') + if (options.title === false) params.set('title', '0') + if (options.warningTitle === false) params.set('warningTitle', '0') + + if (options.controls === false) params.set('controls', '0') + if (options.controlBar === false) params.set('controlBar', '0') + + if (options.peertubeLink === false) params.set('peertubeLink', '0') + if (options.p2p !== undefined) params.set('p2p', options.p2p ? '1' : '0') + + return buildUrl(url, params) +} + +function decoratePlaylistLink (options: { + url: string + + playlistPosition?: number +}) { + const { url } = options + + const params = new URLSearchParams() + + if (options.playlistPosition) params.set('playlistPosition', '' + options.playlistPosition) + + return buildUrl(url, params) +} + +// --------------------------------------------------------------------------- + +export { + addQueryParams, + removeQueryParams, + + buildPlaylistLink, + buildVideoLink, + + buildVideoWatchPath, + buildPlaylistWatchPath, + + buildPlaylistEmbedPath, + buildVideoEmbedPath, + + buildPlaylistEmbedLink, + buildVideoEmbedLink, + + decorateVideoLink, + decoratePlaylistLink +} + +function buildUrl (url: string, params: URLSearchParams) { + let hasParams = false + params.forEach(() => { hasParams = true }) + + if (hasParams) return url + '?' + params.toString() + + return url +} diff --git a/shared/core-utils/common/version.ts b/packages/core-utils/src/common/version.ts similarity index 100% rename from shared/core-utils/common/version.ts rename to packages/core-utils/src/common/version.ts diff --git a/shared/core-utils/i18n/i18n.ts b/packages/core-utils/src/i18n/i18n.ts similarity index 100% rename from shared/core-utils/i18n/i18n.ts rename to packages/core-utils/src/i18n/i18n.ts diff --git a/packages/core-utils/src/i18n/index.ts b/packages/core-utils/src/i18n/index.ts new file mode 100644 index 000000000..758e54b73 --- /dev/null +++ b/packages/core-utils/src/i18n/index.ts @@ -0,0 +1 @@ +export * from './i18n.js' diff --git a/packages/core-utils/src/index.ts b/packages/core-utils/src/index.ts new file mode 100644 index 000000000..3ca5d9d47 --- /dev/null +++ b/packages/core-utils/src/index.ts @@ -0,0 +1,7 @@ +export * from './abuse/index.js' +export * from './common/index.js' +export * from './i18n/index.js' +export * from './plugins/index.js' +export * from './renderer/index.js' +export * from './users/index.js' +export * from './videos/index.js' diff --git a/packages/core-utils/src/plugins/hooks.ts b/packages/core-utils/src/plugins/hooks.ts new file mode 100644 index 000000000..fe7c4a74f --- /dev/null +++ b/packages/core-utils/src/plugins/hooks.ts @@ -0,0 +1,60 @@ +import { HookType, HookType_Type, RegisteredExternalAuthConfig } from '@peertube/peertube-models' +import { isCatchable, isPromise } from '../common/promises.js' + +function getHookType (hookName: string) { + if (hookName.startsWith('filter:')) return HookType.FILTER + if (hookName.startsWith('action:')) return HookType.ACTION + + return HookType.STATIC +} + +async function internalRunHook (options: { + handler: Function + hookType: HookType_Type + result: T + params: any + onError: (err: Error) => void +}) { + const { handler, hookType, result, params, onError } = options + + try { + if (hookType === HookType.FILTER) { + const p = handler(result, params) + + const newResult = isPromise(p) + ? await p + : p + + return newResult + } + + // Action/static hooks do not have result value + const p = handler(params) + + if (hookType === HookType.STATIC) { + if (isPromise(p)) await p + + return undefined + } + + if (hookType === HookType.ACTION) { + if (isCatchable(p)) p.catch((err: any) => onError(err)) + + return undefined + } + } catch (err) { + onError(err) + } + + return result +} + +function getExternalAuthHref (apiUrl: string, auth: RegisteredExternalAuthConfig) { + return apiUrl + `/plugins/${auth.name}/${auth.version}/auth/${auth.authName}` +} + +export { + getHookType, + internalRunHook, + getExternalAuthHref +} diff --git a/packages/core-utils/src/plugins/index.ts b/packages/core-utils/src/plugins/index.ts new file mode 100644 index 000000000..3462bf41e --- /dev/null +++ b/packages/core-utils/src/plugins/index.ts @@ -0,0 +1 @@ +export * from './hooks.js' diff --git a/shared/core-utils/renderer/html.ts b/packages/core-utils/src/renderer/html.ts similarity index 100% rename from shared/core-utils/renderer/html.ts rename to packages/core-utils/src/renderer/html.ts diff --git a/packages/core-utils/src/renderer/index.ts b/packages/core-utils/src/renderer/index.ts new file mode 100644 index 000000000..0dd0a8808 --- /dev/null +++ b/packages/core-utils/src/renderer/index.ts @@ -0,0 +1,2 @@ +export * from './markdown.js' +export * from './html.js' diff --git a/shared/core-utils/renderer/markdown.ts b/packages/core-utils/src/renderer/markdown.ts similarity index 100% rename from shared/core-utils/renderer/markdown.ts rename to packages/core-utils/src/renderer/markdown.ts diff --git a/packages/core-utils/src/users/index.ts b/packages/core-utils/src/users/index.ts new file mode 100644 index 000000000..3fd9dc448 --- /dev/null +++ b/packages/core-utils/src/users/index.ts @@ -0,0 +1 @@ +export * from './user-role.js' diff --git a/packages/core-utils/src/users/user-role.ts b/packages/core-utils/src/users/user-role.ts new file mode 100644 index 000000000..0add3a0a8 --- /dev/null +++ b/packages/core-utils/src/users/user-role.ts @@ -0,0 +1,37 @@ +import { UserRight, UserRightType, UserRole, UserRoleType } from '@peertube/peertube-models' + +export const USER_ROLE_LABELS: { [ id in UserRoleType ]: string } = { + [UserRole.USER]: 'User', + [UserRole.MODERATOR]: 'Moderator', + [UserRole.ADMINISTRATOR]: 'Administrator' +} + +const userRoleRights: { [ id in UserRoleType ]: UserRightType[] } = { + [UserRole.ADMINISTRATOR]: [ + UserRight.ALL + ], + + [UserRole.MODERATOR]: [ + UserRight.MANAGE_VIDEO_BLACKLIST, + UserRight.MANAGE_ABUSES, + UserRight.MANAGE_ANY_VIDEO_CHANNEL, + UserRight.REMOVE_ANY_VIDEO, + UserRight.REMOVE_ANY_VIDEO_PLAYLIST, + UserRight.REMOVE_ANY_VIDEO_COMMENT, + UserRight.UPDATE_ANY_VIDEO, + UserRight.SEE_ALL_VIDEOS, + UserRight.MANAGE_ACCOUNTS_BLOCKLIST, + UserRight.MANAGE_SERVERS_BLOCKLIST, + UserRight.MANAGE_USERS, + UserRight.SEE_ALL_COMMENTS, + UserRight.MANAGE_REGISTRATIONS + ], + + [UserRole.USER]: [] +} + +export function hasUserRight (userRole: UserRoleType, userRight: UserRightType) { + const userRights = userRoleRights[userRole] + + return userRights.includes(UserRight.ALL) || userRights.includes(userRight) +} diff --git a/packages/core-utils/src/videos/bitrate.ts b/packages/core-utils/src/videos/bitrate.ts new file mode 100644 index 000000000..b28eaf460 --- /dev/null +++ b/packages/core-utils/src/videos/bitrate.ts @@ -0,0 +1,113 @@ +import { VideoResolution, VideoResolutionType } from '@peertube/peertube-models' + +type BitPerPixel = { [ id in VideoResolutionType ]: number } + +// https://bitmovin.com/video-bitrate-streaming-hls-dash/ + +const minLimitBitPerPixel: BitPerPixel = { + [VideoResolution.H_NOVIDEO]: 0, + [VideoResolution.H_144P]: 0.02, + [VideoResolution.H_240P]: 0.02, + [VideoResolution.H_360P]: 0.02, + [VideoResolution.H_480P]: 0.02, + [VideoResolution.H_720P]: 0.02, + [VideoResolution.H_1080P]: 0.02, + [VideoResolution.H_1440P]: 0.02, + [VideoResolution.H_4K]: 0.02 +} + +const averageBitPerPixel: BitPerPixel = { + [VideoResolution.H_NOVIDEO]: 0, + [VideoResolution.H_144P]: 0.19, + [VideoResolution.H_240P]: 0.17, + [VideoResolution.H_360P]: 0.15, + [VideoResolution.H_480P]: 0.12, + [VideoResolution.H_720P]: 0.11, + [VideoResolution.H_1080P]: 0.10, + [VideoResolution.H_1440P]: 0.09, + [VideoResolution.H_4K]: 0.08 +} + +const maxBitPerPixel: BitPerPixel = { + [VideoResolution.H_NOVIDEO]: 0, + [VideoResolution.H_144P]: 0.32, + [VideoResolution.H_240P]: 0.29, + [VideoResolution.H_360P]: 0.26, + [VideoResolution.H_480P]: 0.22, + [VideoResolution.H_720P]: 0.19, + [VideoResolution.H_1080P]: 0.17, + [VideoResolution.H_1440P]: 0.16, + [VideoResolution.H_4K]: 0.14 +} + +function getAverageTheoreticalBitrate (options: { + resolution: number + ratio: number + fps: number +}) { + const targetBitrate = calculateBitrate({ ...options, bitPerPixel: averageBitPerPixel }) + if (!targetBitrate) return 192 * 1000 + + return targetBitrate +} + +function getMaxTheoreticalBitrate (options: { + resolution: number + ratio: number + fps: number +}) { + const targetBitrate = calculateBitrate({ ...options, bitPerPixel: maxBitPerPixel }) + if (!targetBitrate) return 256 * 1000 + + return targetBitrate +} + +function getMinTheoreticalBitrate (options: { + resolution: number + ratio: number + fps: number +}) { + const minLimitBitrate = calculateBitrate({ ...options, bitPerPixel: minLimitBitPerPixel }) + if (!minLimitBitrate) return 10 * 1000 + + return minLimitBitrate +} + +// --------------------------------------------------------------------------- + +export { + getAverageTheoreticalBitrate, + getMaxTheoreticalBitrate, + getMinTheoreticalBitrate +} + +// --------------------------------------------------------------------------- + +function calculateBitrate (options: { + bitPerPixel: BitPerPixel + resolution: number + ratio: number + fps: number +}) { + const { bitPerPixel, resolution, ratio, fps } = options + + const resolutionsOrder = [ + VideoResolution.H_4K, + VideoResolution.H_1440P, + VideoResolution.H_1080P, + VideoResolution.H_720P, + VideoResolution.H_480P, + VideoResolution.H_360P, + VideoResolution.H_240P, + VideoResolution.H_144P, + VideoResolution.H_NOVIDEO + ] + + for (const toTestResolution of resolutionsOrder) { + if (toTestResolution <= resolution) { + return Math.floor(resolution * resolution * ratio * fps * bitPerPixel[toTestResolution]) + } + } + + throw new Error('Unknown resolution ' + resolution) +} diff --git a/packages/core-utils/src/videos/common.ts b/packages/core-utils/src/videos/common.ts new file mode 100644 index 000000000..47564fb2a --- /dev/null +++ b/packages/core-utils/src/videos/common.ts @@ -0,0 +1,24 @@ +import { VideoDetails, VideoPrivacy, VideoStreamingPlaylistType } from '@peertube/peertube-models' + +function getAllPrivacies () { + return [ VideoPrivacy.PUBLIC, VideoPrivacy.INTERNAL, VideoPrivacy.PRIVATE, VideoPrivacy.UNLISTED, VideoPrivacy.PASSWORD_PROTECTED ] +} + +function getAllFiles (video: Partial>) { + const files = video.files + + const hls = getHLS(video) + if (hls) return files.concat(hls.files) + + return files +} + +function getHLS (video: Partial>) { + return video.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) +} + +export { + getAllPrivacies, + getAllFiles, + getHLS +} diff --git a/packages/core-utils/src/videos/index.ts b/packages/core-utils/src/videos/index.ts new file mode 100644 index 000000000..7d3dacdd4 --- /dev/null +++ b/packages/core-utils/src/videos/index.ts @@ -0,0 +1,2 @@ +export * from './bitrate.js' +export * from './common.js' diff --git a/packages/core-utils/tsconfig.json b/packages/core-utils/tsconfig.json new file mode 100644 index 000000000..56ebffbb3 --- /dev/null +++ b/packages/core-utils/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "src", + "tsBuildInfoFile": "./dist/.tsbuildinfo" + }, + "references": [ + { "path": "../models" } + ] +} diff --git a/packages/ffmpeg/package.json b/packages/ffmpeg/package.json new file mode 100644 index 000000000..fca86df25 --- /dev/null +++ b/packages/ffmpeg/package.json @@ -0,0 +1,19 @@ +{ + "name": "@peertube/peertube-ffmpeg", + "private": true, + "version": "0.0.0", + "main": "dist/index.js", + "files": [ "dist" ], + "exports": { + "types": "./dist/index.d.ts", + "peertube:tsx": "./src/index.ts", + "default": "./dist/index.js" + }, + "type": "module", + "devDependencies": {}, + "scripts": { + "build": "tsc", + "watch": "tsc -w" + }, + "dependencies": {} +} diff --git a/packages/ffmpeg/src/ffmpeg-command-wrapper.ts b/packages/ffmpeg/src/ffmpeg-command-wrapper.ts new file mode 100644 index 000000000..647ee3996 --- /dev/null +++ b/packages/ffmpeg/src/ffmpeg-command-wrapper.ts @@ -0,0 +1,246 @@ +import ffmpeg, { FfmpegCommand } from 'fluent-ffmpeg' +import { pick, promisify0 } from '@peertube/peertube-core-utils' +import { AvailableEncoders, EncoderOptionsBuilder, EncoderOptionsBuilderParams, EncoderProfile } from '@peertube/peertube-models' + +type FFmpegLogger = { + info: (msg: string, obj?: any) => void + debug: (msg: string, obj?: any) => void + warn: (msg: string, obj?: any) => void + error: (msg: string, obj?: any) => void +} + +export interface FFmpegCommandWrapperOptions { + availableEncoders?: AvailableEncoders + profile?: string + + niceness: number + tmpDirectory: string + threads: number + + logger: FFmpegLogger + lTags?: { tags: string[] } + + updateJobProgress?: (progress?: number) => void + onEnd?: () => void + onError?: (err: Error) => void +} + +export class FFmpegCommandWrapper { + private static supportedEncoders: Map + + private readonly availableEncoders: AvailableEncoders + private readonly profile: string + + private readonly niceness: number + private readonly tmpDirectory: string + private readonly threads: number + + private readonly logger: FFmpegLogger + private readonly lTags: { tags: string[] } + + private readonly updateJobProgress: (progress?: number) => void + private readonly onEnd?: () => void + private readonly onError?: (err: Error) => void + + private command: FfmpegCommand + + constructor (options: FFmpegCommandWrapperOptions) { + this.availableEncoders = options.availableEncoders + this.profile = options.profile + this.niceness = options.niceness + this.tmpDirectory = options.tmpDirectory + this.threads = options.threads + this.logger = options.logger + this.lTags = options.lTags || { tags: [] } + + this.updateJobProgress = options.updateJobProgress + + this.onEnd = options.onEnd + this.onError = options.onError + } + + getAvailableEncoders () { + return this.availableEncoders + } + + getProfile () { + return this.profile + } + + getCommand () { + return this.command + } + + // --------------------------------------------------------------------------- + + debugLog (msg: string, meta: any) { + this.logger.debug(msg, { ...meta, ...this.lTags }) + } + + // --------------------------------------------------------------------------- + + buildCommand (input: string) { + if (this.command) throw new Error('Command is already built') + + // We set cwd explicitly because ffmpeg appears to create temporary files when trancoding which fails in read-only file systems + this.command = ffmpeg(input, { + niceness: this.niceness, + cwd: this.tmpDirectory + }) + + if (this.threads > 0) { + // If we don't set any threads ffmpeg will chose automatically + this.command.outputOption('-threads ' + this.threads) + } + + return this.command + } + + async runCommand (options: { + silent?: boolean // false by default + } = {}) { + const { silent = false } = options + + return new Promise((res, rej) => { + let shellCommand: string + + this.command.on('start', cmdline => { shellCommand = cmdline }) + + this.command.on('error', (err, stdout, stderr) => { + if (silent !== true) this.logger.error('Error in ffmpeg.', { stdout, stderr, shellCommand, ...this.lTags }) + + if (this.onError) this.onError(err) + + rej(err) + }) + + this.command.on('end', (stdout, stderr) => { + this.logger.debug('FFmpeg command ended.', { stdout, stderr, shellCommand, ...this.lTags }) + + if (this.onEnd) this.onEnd() + + res() + }) + + if (this.updateJobProgress) { + this.command.on('progress', progress => { + if (!progress.percent) return + + // Sometimes ffmpeg returns an invalid progress + let percent = Math.round(progress.percent) + if (percent < 0) percent = 0 + if (percent > 100) percent = 100 + + this.updateJobProgress(percent) + }) + } + + this.command.run() + }) + } + + // --------------------------------------------------------------------------- + + static resetSupportedEncoders () { + FFmpegCommandWrapper.supportedEncoders = undefined + } + + // Run encoder builder depending on available encoders + // Try encoders by priority: if the encoder is available, run the chosen profile or fallback to the default one + // If the default one does not exist, check the next encoder + async getEncoderBuilderResult (options: EncoderOptionsBuilderParams & { + streamType: 'video' | 'audio' + input: string + + videoType: 'vod' | 'live' + }) { + if (!this.availableEncoders) { + throw new Error('There is no available encoders') + } + + const { streamType, videoType } = options + + const encodersToTry = this.availableEncoders.encodersToTry[videoType][streamType] + const encoders = this.availableEncoders.available[videoType] + + for (const encoder of encodersToTry) { + if (!(await this.checkFFmpegEncoders(this.availableEncoders)).get(encoder)) { + this.logger.debug(`Encoder ${encoder} not available in ffmpeg, skipping.`, this.lTags) + continue + } + + if (!encoders[encoder]) { + this.logger.debug(`Encoder ${encoder} not available in peertube encoders, skipping.`, this.lTags) + continue + } + + // An object containing available profiles for this encoder + const builderProfiles: EncoderProfile = encoders[encoder] + let builder = builderProfiles[this.profile] + + if (!builder) { + this.logger.debug(`Profile ${this.profile} for encoder ${encoder} not available. Fallback to default.`, this.lTags) + builder = builderProfiles.default + + if (!builder) { + this.logger.debug(`Default profile for encoder ${encoder} not available. Try next available encoder.`, this.lTags) + continue + } + } + + const result = await builder( + pick(options, [ + 'input', + 'canCopyAudio', + 'canCopyVideo', + 'resolution', + 'inputBitrate', + 'fps', + 'inputRatio', + 'streamNum' + ]) + ) + + return { + result, + + // If we don't have output options, then copy the input stream + encoder: result.copy === true + ? 'copy' + : encoder + } + } + + return null + } + + // Detect supported encoders by ffmpeg + private async checkFFmpegEncoders (peertubeAvailableEncoders: AvailableEncoders): Promise> { + if (FFmpegCommandWrapper.supportedEncoders !== undefined) { + return FFmpegCommandWrapper.supportedEncoders + } + + const getAvailableEncodersPromise = promisify0(ffmpeg.getAvailableEncoders) + const availableFFmpegEncoders = await getAvailableEncodersPromise() + + const searchEncoders = new Set() + for (const type of [ 'live', 'vod' ]) { + for (const streamType of [ 'audio', 'video' ]) { + for (const encoder of peertubeAvailableEncoders.encodersToTry[type][streamType]) { + searchEncoders.add(encoder) + } + } + } + + const supportedEncoders = new Map() + + for (const searchEncoder of searchEncoders) { + supportedEncoders.set(searchEncoder, availableFFmpegEncoders[searchEncoder] !== undefined) + } + + this.logger.info('Built supported ffmpeg encoders.', { supportedEncoders, searchEncoders, ...this.lTags }) + + FFmpegCommandWrapper.supportedEncoders = supportedEncoders + return supportedEncoders + } +} diff --git a/packages/ffmpeg/src/ffmpeg-default-transcoding-profile.ts b/packages/ffmpeg/src/ffmpeg-default-transcoding-profile.ts new file mode 100644 index 000000000..0d3538512 --- /dev/null +++ b/packages/ffmpeg/src/ffmpeg-default-transcoding-profile.ts @@ -0,0 +1,187 @@ +import { FfprobeData } from 'fluent-ffmpeg' +import { getAverageTheoreticalBitrate, getMaxTheoreticalBitrate, getMinTheoreticalBitrate } from '@peertube/peertube-core-utils' +import { + buildStreamSuffix, + ffprobePromise, + getAudioStream, + getMaxAudioBitrate, + getVideoStream, + getVideoStreamBitrate, + getVideoStreamDimensionsInfo, + getVideoStreamFPS +} from '@peertube/peertube-ffmpeg' +import { EncoderOptionsBuilder, EncoderOptionsBuilderParams } from '@peertube/peertube-models' + +const defaultX264VODOptionsBuilder: EncoderOptionsBuilder = (options: EncoderOptionsBuilderParams) => { + const { fps, inputRatio, inputBitrate, resolution } = options + + const targetBitrate = getTargetBitrate({ inputBitrate, ratio: inputRatio, fps, resolution }) + + return { + outputOptions: [ + ...getCommonOutputOptions(targetBitrate), + + `-r ${fps}` + ] + } +} + +const defaultX264LiveOptionsBuilder: EncoderOptionsBuilder = (options: EncoderOptionsBuilderParams) => { + const { streamNum, fps, inputBitrate, inputRatio, resolution } = options + + const targetBitrate = getTargetBitrate({ inputBitrate, ratio: inputRatio, fps, resolution }) + + return { + outputOptions: [ + ...getCommonOutputOptions(targetBitrate, streamNum), + + `${buildStreamSuffix('-r:v', streamNum)} ${fps}`, + `${buildStreamSuffix('-b:v', streamNum)} ${targetBitrate}` + ] + } +} + +const defaultAACOptionsBuilder: EncoderOptionsBuilder = async ({ input, streamNum, canCopyAudio }) => { + const probe = await ffprobePromise(input) + + if (canCopyAudio && await canDoQuickAudioTranscode(input, probe)) { + return { copy: true, outputOptions: [ ] } + } + + const parsedAudio = await getAudioStream(input, probe) + + // We try to reduce the ceiling bitrate by making rough matches of bitrates + // Of course this is far from perfect, but it might save some space in the end + + const audioCodecName = parsedAudio.audioStream['codec_name'] + + const bitrate = getMaxAudioBitrate(audioCodecName, parsedAudio.bitrate) + + // Force stereo as it causes some issues with HLS playback in Chrome + const base = [ '-channel_layout', 'stereo' ] + + if (bitrate !== -1) { + return { outputOptions: base.concat([ buildStreamSuffix('-b:a', streamNum), bitrate + 'k' ]) } + } + + return { outputOptions: base } +} + +const defaultLibFDKAACVODOptionsBuilder: EncoderOptionsBuilder = ({ streamNum }) => { + return { outputOptions: [ buildStreamSuffix('-q:a', streamNum), '5' ] } +} + +export function getDefaultAvailableEncoders () { + return { + vod: { + libx264: { + default: defaultX264VODOptionsBuilder + }, + aac: { + default: defaultAACOptionsBuilder + }, + libfdk_aac: { + default: defaultLibFDKAACVODOptionsBuilder + } + }, + live: { + libx264: { + default: defaultX264LiveOptionsBuilder + }, + aac: { + default: defaultAACOptionsBuilder + } + } + } +} + +export function getDefaultEncodersToTry () { + return { + vod: { + video: [ 'libx264' ], + audio: [ 'libfdk_aac', 'aac' ] + }, + + live: { + video: [ 'libx264' ], + audio: [ 'libfdk_aac', 'aac' ] + } + } +} + +export async function canDoQuickAudioTranscode (path: string, probe?: FfprobeData): Promise { + const parsedAudio = await getAudioStream(path, probe) + + if (!parsedAudio.audioStream) return true + + if (parsedAudio.audioStream['codec_name'] !== 'aac') return false + + const audioBitrate = parsedAudio.bitrate + if (!audioBitrate) return false + + const maxAudioBitrate = getMaxAudioBitrate('aac', audioBitrate) + if (maxAudioBitrate !== -1 && audioBitrate > maxAudioBitrate) return false + + const channelLayout = parsedAudio.audioStream['channel_layout'] + // Causes playback issues with Chrome + if (!channelLayout || channelLayout === 'unknown' || channelLayout === 'quad') return false + + return true +} + +export async function canDoQuickVideoTranscode (path: string, probe?: FfprobeData): Promise { + const videoStream = await getVideoStream(path, probe) + const fps = await getVideoStreamFPS(path, probe) + const bitRate = await getVideoStreamBitrate(path, probe) + const resolutionData = await getVideoStreamDimensionsInfo(path, probe) + + // If ffprobe did not manage to guess the bitrate + if (!bitRate) return false + + // check video params + if (!videoStream) return false + if (videoStream['codec_name'] !== 'h264') return false + if (videoStream['pix_fmt'] !== 'yuv420p') return false + if (fps < 2 || fps > 65) return false + if (bitRate > getMaxTheoreticalBitrate({ ...resolutionData, fps })) return false + + return true +} + +// --------------------------------------------------------------------------- + +function getTargetBitrate (options: { + inputBitrate: number + resolution: number + ratio: number + fps: number +}) { + const { inputBitrate, resolution, ratio, fps } = options + + const capped = capBitrate(inputBitrate, getAverageTheoreticalBitrate({ resolution, fps, ratio })) + const limit = getMinTheoreticalBitrate({ resolution, fps, ratio }) + + return Math.max(limit, capped) +} + +function capBitrate (inputBitrate: number, targetBitrate: number) { + if (!inputBitrate) return targetBitrate + + // Add 30% margin to input bitrate + const inputBitrateWithMargin = inputBitrate + (inputBitrate * 0.3) + + return Math.min(targetBitrate, inputBitrateWithMargin) +} + +function getCommonOutputOptions (targetBitrate: number, streamNum?: number) { + return [ + `-preset veryfast`, + `${buildStreamSuffix('-maxrate:v', streamNum)} ${targetBitrate}`, + `${buildStreamSuffix('-bufsize:v', streamNum)} ${targetBitrate * 2}`, + + // NOTE: b-strategy 1 - heuristic algorithm, 16 is optimal B-frames for it + `-b_strategy 1`, + // NOTE: Why 16: https://github.com/Chocobozzz/PeerTube/pull/774. b-strategy 2 -> B-frames<16 + `-bf 16` + ] +} diff --git a/packages/ffmpeg/src/ffmpeg-edition.ts b/packages/ffmpeg/src/ffmpeg-edition.ts new file mode 100644 index 000000000..021342930 --- /dev/null +++ b/packages/ffmpeg/src/ffmpeg-edition.ts @@ -0,0 +1,239 @@ +import { FilterSpecification } from 'fluent-ffmpeg' +import { FFmpegCommandWrapper, FFmpegCommandWrapperOptions } from './ffmpeg-command-wrapper.js' +import { presetVOD } from './shared/presets.js' +import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS, hasAudioStream } from './ffprobe.js' + +export class FFmpegEdition { + private readonly commandWrapper: FFmpegCommandWrapper + + constructor (options: FFmpegCommandWrapperOptions) { + this.commandWrapper = new FFmpegCommandWrapper(options) + } + + async cutVideo (options: { + inputPath: string + outputPath: string + start?: number + end?: number + }) { + const { inputPath, outputPath } = options + + const mainProbe = await ffprobePromise(inputPath) + const fps = await getVideoStreamFPS(inputPath, mainProbe) + const { resolution } = await getVideoStreamDimensionsInfo(inputPath, mainProbe) + + const command = this.commandWrapper.buildCommand(inputPath) + .output(outputPath) + + await presetVOD({ + commandWrapper: this.commandWrapper, + input: inputPath, + resolution, + fps, + canCopyAudio: false, + canCopyVideo: false + }) + + if (options.start) { + command.outputOption('-ss ' + options.start) + } + + if (options.end) { + command.outputOption('-to ' + options.end) + } + + await this.commandWrapper.runCommand() + } + + async addWatermark (options: { + inputPath: string + watermarkPath: string + outputPath: string + + videoFilters: { + watermarkSizeRatio: number + horitonzalMarginRatio: number + verticalMarginRatio: number + } + }) { + const { watermarkPath, inputPath, outputPath, videoFilters } = options + + const videoProbe = await ffprobePromise(inputPath) + const fps = await getVideoStreamFPS(inputPath, videoProbe) + const { resolution } = await getVideoStreamDimensionsInfo(inputPath, videoProbe) + + const command = this.commandWrapper.buildCommand(inputPath) + .output(outputPath) + + command.input(watermarkPath) + + await presetVOD({ + commandWrapper: this.commandWrapper, + input: inputPath, + resolution, + fps, + canCopyAudio: true, + canCopyVideo: false + }) + + const complexFilter: FilterSpecification[] = [ + // Scale watermark + { + inputs: [ '[1]', '[0]' ], + filter: 'scale2ref', + options: { + w: 'oh*mdar', + h: `ih*${videoFilters.watermarkSizeRatio}` + }, + outputs: [ '[watermark]', '[video]' ] + }, + + { + inputs: [ '[video]', '[watermark]' ], + filter: 'overlay', + options: { + x: `main_w - overlay_w - (main_h * ${videoFilters.horitonzalMarginRatio})`, + y: `main_h * ${videoFilters.verticalMarginRatio}` + } + } + ] + + command.complexFilter(complexFilter) + + await this.commandWrapper.runCommand() + } + + async addIntroOutro (options: { + inputPath: string + introOutroPath: string + outputPath: string + type: 'intro' | 'outro' + }) { + const { introOutroPath, inputPath, outputPath, type } = options + + const mainProbe = await ffprobePromise(inputPath) + const fps = await getVideoStreamFPS(inputPath, mainProbe) + const { resolution } = await getVideoStreamDimensionsInfo(inputPath, mainProbe) + const mainHasAudio = await hasAudioStream(inputPath, mainProbe) + + const introOutroProbe = await ffprobePromise(introOutroPath) + const introOutroHasAudio = await hasAudioStream(introOutroPath, introOutroProbe) + + const command = this.commandWrapper.buildCommand(inputPath) + .output(outputPath) + + command.input(introOutroPath) + + if (!introOutroHasAudio && mainHasAudio) { + const duration = await getVideoStreamDuration(introOutroPath, introOutroProbe) + + command.input('anullsrc') + command.withInputFormat('lavfi') + command.withInputOption('-t ' + duration) + } + + await presetVOD({ + commandWrapper: this.commandWrapper, + input: inputPath, + resolution, + fps, + canCopyAudio: false, + canCopyVideo: false + }) + + // Add black background to correctly scale intro/outro with padding + const complexFilter: FilterSpecification[] = [ + { + inputs: [ '1', '0' ], + filter: 'scale2ref', + options: { + w: 'iw', + h: `ih` + }, + outputs: [ 'intro-outro', 'main' ] + }, + { + inputs: [ 'intro-outro', 'main' ], + filter: 'scale2ref', + options: { + w: 'iw', + h: `ih` + }, + outputs: [ 'to-scale', 'main' ] + }, + { + inputs: 'to-scale', + filter: 'drawbox', + options: { + t: 'fill' + }, + outputs: [ 'to-scale-bg' ] + }, + { + inputs: [ '1', 'to-scale-bg' ], + filter: 'scale2ref', + options: { + w: 'iw', + h: 'ih', + force_original_aspect_ratio: 'decrease', + flags: 'spline' + }, + outputs: [ 'to-scale', 'to-scale-bg' ] + }, + { + inputs: [ 'to-scale-bg', 'to-scale' ], + filter: 'overlay', + options: { + x: '(main_w - overlay_w)/2', + y: '(main_h - overlay_h)/2' + }, + outputs: 'intro-outro-resized' + } + ] + + const concatFilter = { + inputs: [], + filter: 'concat', + options: { + n: 2, + v: 1, + unsafe: 1 + }, + outputs: [ 'v' ] + } + + const introOutroFilterInputs = [ 'intro-outro-resized' ] + const mainFilterInputs = [ 'main' ] + + if (mainHasAudio) { + mainFilterInputs.push('0:a') + + if (introOutroHasAudio) { + introOutroFilterInputs.push('1:a') + } else { + // Silent input + introOutroFilterInputs.push('2:a') + } + } + + if (type === 'intro') { + concatFilter.inputs = [ ...introOutroFilterInputs, ...mainFilterInputs ] + } else { + concatFilter.inputs = [ ...mainFilterInputs, ...introOutroFilterInputs ] + } + + if (mainHasAudio) { + concatFilter.options['a'] = 1 + concatFilter.outputs.push('a') + + command.outputOption('-map [a]') + } + + command.outputOption('-map [v]') + + complexFilter.push(concatFilter) + command.complexFilter(complexFilter) + + await this.commandWrapper.runCommand() + } +} diff --git a/packages/ffmpeg/src/ffmpeg-images.ts b/packages/ffmpeg/src/ffmpeg-images.ts new file mode 100644 index 000000000..4cd37aa80 --- /dev/null +++ b/packages/ffmpeg/src/ffmpeg-images.ts @@ -0,0 +1,92 @@ +import { FFmpegCommandWrapper, FFmpegCommandWrapperOptions } from './ffmpeg-command-wrapper.js' +import { getVideoStreamDuration } from './ffprobe.js' + +export class FFmpegImage { + private readonly commandWrapper: FFmpegCommandWrapper + + constructor (options: FFmpegCommandWrapperOptions) { + this.commandWrapper = new FFmpegCommandWrapper(options) + } + + convertWebPToJPG (options: { + path: string + destination: string + }): Promise { + const { path, destination } = options + + this.commandWrapper.buildCommand(path) + .output(destination) + + return this.commandWrapper.runCommand({ silent: true }) + } + + processGIF (options: { + path: string + destination: string + newSize: { width: number, height: number } + }): Promise { + const { path, destination, newSize } = options + + this.commandWrapper.buildCommand(path) + .fps(20) + .size(`${newSize.width}x${newSize.height}`) + .output(destination) + + return this.commandWrapper.runCommand() + } + + async generateThumbnailFromVideo (options: { + fromPath: string + output: string + }) { + const { fromPath, output } = options + + let duration = await getVideoStreamDuration(fromPath) + if (isNaN(duration)) duration = 0 + + this.commandWrapper.buildCommand(fromPath) + .seekInput(duration / 2) + .videoFilter('thumbnail=500') + .outputOption('-frames:v 1') + .output(output) + + return this.commandWrapper.runCommand() + } + + async generateStoryboardFromVideo (options: { + path: string + destination: string + + sprites: { + size: { + width: number + height: number + } + + count: { + width: number + height: number + } + + duration: number + } + }) { + const { path, destination, sprites } = options + + const command = this.commandWrapper.buildCommand(path) + + const filter = [ + `setpts=N/round(FRAME_RATE)/TB`, + `select='not(mod(t,${options.sprites.duration}))'`, + `scale=${sprites.size.width}:${sprites.size.height}`, + `tile=layout=${sprites.count.width}x${sprites.count.height}` + ].join(',') + + command.outputOption('-filter_complex', filter) + command.outputOption('-frames:v', '1') + command.outputOption('-q:v', '2') + command.output(destination) + + return this.commandWrapper.runCommand() + } +} diff --git a/packages/ffmpeg/src/ffmpeg-live.ts b/packages/ffmpeg/src/ffmpeg-live.ts new file mode 100644 index 000000000..20318f63c --- /dev/null +++ b/packages/ffmpeg/src/ffmpeg-live.ts @@ -0,0 +1,184 @@ +import { FilterSpecification } from 'fluent-ffmpeg' +import { join } from 'path' +import { pick } from '@peertube/peertube-core-utils' +import { FFmpegCommandWrapper, FFmpegCommandWrapperOptions } from './ffmpeg-command-wrapper.js' +import { buildStreamSuffix, getScaleFilter, StreamType } from './ffmpeg-utils.js' +import { addDefaultEncoderGlobalParams, addDefaultEncoderParams, applyEncoderOptions } from './shared/index.js' + +export class FFmpegLive { + private readonly commandWrapper: FFmpegCommandWrapper + + constructor (options: FFmpegCommandWrapperOptions) { + this.commandWrapper = new FFmpegCommandWrapper(options) + } + + async getLiveTranscodingCommand (options: { + inputUrl: string + + outPath: string + masterPlaylistName: string + + toTranscode: { + resolution: number + fps: number + }[] + + // Input information + bitrate: number + ratio: number + hasAudio: boolean + + segmentListSize: number + segmentDuration: number + }) { + const { + inputUrl, + outPath, + toTranscode, + bitrate, + masterPlaylistName, + ratio, + hasAudio + } = options + const command = this.commandWrapper.buildCommand(inputUrl) + + const varStreamMap: string[] = [] + + const complexFilter: FilterSpecification[] = [ + { + inputs: '[v:0]', + filter: 'split', + options: toTranscode.length, + outputs: toTranscode.map(t => `vtemp${t.resolution}`) + } + ] + + command.outputOption('-sc_threshold 0') + + addDefaultEncoderGlobalParams(command) + + for (let i = 0; i < toTranscode.length; i++) { + const streamMap: string[] = [] + const { resolution, fps } = toTranscode[i] + + const baseEncoderBuilderParams = { + input: inputUrl, + + canCopyAudio: true, + canCopyVideo: true, + + inputBitrate: bitrate, + inputRatio: ratio, + + resolution, + fps, + + streamNum: i, + videoType: 'live' as 'live' + } + + { + const streamType: StreamType = 'video' + const builderResult = await this.commandWrapper.getEncoderBuilderResult({ ...baseEncoderBuilderParams, streamType }) + if (!builderResult) { + throw new Error('No available live video encoder found') + } + + command.outputOption(`-map [vout${resolution}]`) + + addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps, streamNum: i }) + + this.commandWrapper.debugLog( + `Apply ffmpeg live video params from ${builderResult.encoder} using ${this.commandWrapper.getProfile()} profile.`, + { builderResult, fps, toTranscode } + ) + + command.outputOption(`${buildStreamSuffix('-c:v', i)} ${builderResult.encoder}`) + applyEncoderOptions(command, builderResult.result) + + complexFilter.push({ + inputs: `vtemp${resolution}`, + filter: getScaleFilter(builderResult.result), + options: `w=-2:h=${resolution}`, + outputs: `vout${resolution}` + }) + + streamMap.push(`v:${i}`) + } + + if (hasAudio) { + const streamType: StreamType = 'audio' + const builderResult = await this.commandWrapper.getEncoderBuilderResult({ ...baseEncoderBuilderParams, streamType }) + if (!builderResult) { + throw new Error('No available live audio encoder found') + } + + command.outputOption('-map a:0') + + addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps, streamNum: i }) + + this.commandWrapper.debugLog( + `Apply ffmpeg live audio params from ${builderResult.encoder} using ${this.commandWrapper.getProfile()} profile.`, + { builderResult, fps, resolution } + ) + + command.outputOption(`${buildStreamSuffix('-c:a', i)} ${builderResult.encoder}`) + applyEncoderOptions(command, builderResult.result) + + streamMap.push(`a:${i}`) + } + + varStreamMap.push(streamMap.join(',')) + } + + command.complexFilter(complexFilter) + + this.addDefaultLiveHLSParams({ ...pick(options, [ 'segmentDuration', 'segmentListSize' ]), outPath, masterPlaylistName }) + + command.outputOption('-var_stream_map', varStreamMap.join(' ')) + + return command + } + + getLiveMuxingCommand (options: { + inputUrl: string + outPath: string + masterPlaylistName: string + + segmentListSize: number + segmentDuration: number + }) { + const { inputUrl, outPath, masterPlaylistName } = options + + const command = this.commandWrapper.buildCommand(inputUrl) + + command.outputOption('-c:v copy') + command.outputOption('-c:a copy') + command.outputOption('-map 0:a?') + command.outputOption('-map 0:v?') + + this.addDefaultLiveHLSParams({ ...pick(options, [ 'segmentDuration', 'segmentListSize' ]), outPath, masterPlaylistName }) + + return command + } + + private addDefaultLiveHLSParams (options: { + outPath: string + masterPlaylistName: string + segmentListSize: number + segmentDuration: number + }) { + const { outPath, masterPlaylistName, segmentListSize, segmentDuration } = options + + const command = this.commandWrapper.getCommand() + + command.outputOption('-hls_time ' + segmentDuration) + command.outputOption('-hls_list_size ' + segmentListSize) + command.outputOption('-hls_flags delete_segments+independent_segments+program_date_time') + command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%06d.ts')}`) + command.outputOption('-master_pl_name ' + masterPlaylistName) + command.outputOption(`-f hls`) + + command.output(join(outPath, '%v.m3u8')) + } +} diff --git a/packages/ffmpeg/src/ffmpeg-utils.ts b/packages/ffmpeg/src/ffmpeg-utils.ts new file mode 100644 index 000000000..56fd8c0b3 --- /dev/null +++ b/packages/ffmpeg/src/ffmpeg-utils.ts @@ -0,0 +1,17 @@ +import { EncoderOptions } from '@peertube/peertube-models' + +export type StreamType = 'audio' | 'video' + +export function buildStreamSuffix (base: string, streamNum?: number) { + if (streamNum !== undefined) { + return `${base}:${streamNum}` + } + + return base +} + +export function getScaleFilter (options: EncoderOptions): string { + if (options.scaleFilter) return options.scaleFilter.name + + return 'scale' +} diff --git a/shared/ffmpeg/ffmpeg-version.ts b/packages/ffmpeg/src/ffmpeg-version.ts similarity index 100% rename from shared/ffmpeg/ffmpeg-version.ts rename to packages/ffmpeg/src/ffmpeg-version.ts diff --git a/packages/ffmpeg/src/ffmpeg-vod.ts b/packages/ffmpeg/src/ffmpeg-vod.ts new file mode 100644 index 000000000..6dd272b8d --- /dev/null +++ b/packages/ffmpeg/src/ffmpeg-vod.ts @@ -0,0 +1,256 @@ +import { MutexInterface } from 'async-mutex' +import { FfmpegCommand } from 'fluent-ffmpeg' +import { readFile, writeFile } from 'fs/promises' +import { dirname } from 'path' +import { pick } from '@peertube/peertube-core-utils' +import { VideoResolution } from '@peertube/peertube-models' +import { FFmpegCommandWrapper, FFmpegCommandWrapperOptions } from './ffmpeg-command-wrapper.js' +import { ffprobePromise, getVideoStreamDimensionsInfo } from './ffprobe.js' +import { presetCopy, presetOnlyAudio, presetVOD } from './shared/presets.js' + +export type TranscodeVODOptionsType = 'hls' | 'hls-from-ts' | 'quick-transcode' | 'video' | 'merge-audio' | 'only-audio' + +export interface BaseTranscodeVODOptions { + type: TranscodeVODOptionsType + + inputPath: string + outputPath: string + + // Will be released after the ffmpeg started + // To prevent a bug where the input file does not exist anymore when running ffmpeg + inputFileMutexReleaser: MutexInterface.Releaser + + resolution: number + fps: number +} + +export interface HLSTranscodeOptions extends BaseTranscodeVODOptions { + type: 'hls' + + copyCodecs: boolean + + hlsPlaylist: { + videoFilename: string + } +} + +export interface HLSFromTSTranscodeOptions extends BaseTranscodeVODOptions { + type: 'hls-from-ts' + + isAAC: boolean + + hlsPlaylist: { + videoFilename: string + } +} + +export interface QuickTranscodeOptions extends BaseTranscodeVODOptions { + type: 'quick-transcode' +} + +export interface VideoTranscodeOptions extends BaseTranscodeVODOptions { + type: 'video' +} + +export interface MergeAudioTranscodeOptions extends BaseTranscodeVODOptions { + type: 'merge-audio' + audioPath: string +} + +export interface OnlyAudioTranscodeOptions extends BaseTranscodeVODOptions { + type: 'only-audio' +} + +export type TranscodeVODOptions = + HLSTranscodeOptions + | HLSFromTSTranscodeOptions + | VideoTranscodeOptions + | MergeAudioTranscodeOptions + | OnlyAudioTranscodeOptions + | QuickTranscodeOptions + +// --------------------------------------------------------------------------- + +export class FFmpegVOD { + private readonly commandWrapper: FFmpegCommandWrapper + + private ended = false + + constructor (options: FFmpegCommandWrapperOptions) { + this.commandWrapper = new FFmpegCommandWrapper(options) + } + + async transcode (options: TranscodeVODOptions) { + const builders: { + [ type in TranscodeVODOptionsType ]: (options: TranscodeVODOptions) => Promise | void + } = { + 'quick-transcode': this.buildQuickTranscodeCommand.bind(this), + 'hls': this.buildHLSVODCommand.bind(this), + 'hls-from-ts': this.buildHLSVODFromTSCommand.bind(this), + 'merge-audio': this.buildAudioMergeCommand.bind(this), + // TODO: remove, we merge this in buildWebVideoCommand + 'only-audio': this.buildOnlyAudioCommand.bind(this), + 'video': this.buildWebVideoCommand.bind(this) + } + + this.commandWrapper.debugLog('Will run transcode.', { options }) + + const command = this.commandWrapper.buildCommand(options.inputPath) + .output(options.outputPath) + + await builders[options.type](options) + + command.on('start', () => { + setTimeout(() => { + options.inputFileMutexReleaser() + }, 1000) + }) + + await this.commandWrapper.runCommand() + + await this.fixHLSPlaylistIfNeeded(options) + + this.ended = true + } + + isEnded () { + return this.ended + } + + private async buildWebVideoCommand (options: TranscodeVODOptions) { + const { resolution, fps, inputPath } = options + + if (resolution === VideoResolution.H_NOVIDEO) { + presetOnlyAudio(this.commandWrapper) + return + } + + let scaleFilterValue: string + + if (resolution !== undefined) { + const probe = await ffprobePromise(inputPath) + const videoStreamInfo = await getVideoStreamDimensionsInfo(inputPath, probe) + + scaleFilterValue = videoStreamInfo?.isPortraitMode === true + ? `w=${resolution}:h=-2` + : `w=-2:h=${resolution}` + } + + await presetVOD({ + commandWrapper: this.commandWrapper, + + resolution, + input: inputPath, + canCopyAudio: true, + canCopyVideo: true, + fps, + scaleFilterValue + }) + } + + private buildQuickTranscodeCommand (_options: TranscodeVODOptions) { + const command = this.commandWrapper.getCommand() + + presetCopy(this.commandWrapper) + + command.outputOption('-map_metadata -1') // strip all metadata + .outputOption('-movflags faststart') + } + + // --------------------------------------------------------------------------- + // Audio transcoding + // --------------------------------------------------------------------------- + + private async buildAudioMergeCommand (options: MergeAudioTranscodeOptions) { + const command = this.commandWrapper.getCommand() + + command.loop(undefined) + + await presetVOD({ + ...pick(options, [ 'resolution' ]), + + commandWrapper: this.commandWrapper, + input: options.audioPath, + canCopyAudio: true, + canCopyVideo: true, + fps: options.fps, + scaleFilterValue: this.getMergeAudioScaleFilterValue() + }) + + command.outputOption('-preset:v veryfast') + + command.input(options.audioPath) + .outputOption('-tune stillimage') + .outputOption('-shortest') + } + + private buildOnlyAudioCommand (_options: OnlyAudioTranscodeOptions) { + presetOnlyAudio(this.commandWrapper) + } + + // Avoid "height not divisible by 2" error + private getMergeAudioScaleFilterValue () { + return 'trunc(iw/2)*2:trunc(ih/2)*2' + } + + // --------------------------------------------------------------------------- + // HLS transcoding + // --------------------------------------------------------------------------- + + private async buildHLSVODCommand (options: HLSTranscodeOptions) { + const command = this.commandWrapper.getCommand() + + const videoPath = this.getHLSVideoPath(options) + + if (options.copyCodecs) presetCopy(this.commandWrapper) + else if (options.resolution === VideoResolution.H_NOVIDEO) presetOnlyAudio(this.commandWrapper) + else await this.buildWebVideoCommand(options) + + this.addCommonHLSVODCommandOptions(command, videoPath) + } + + private buildHLSVODFromTSCommand (options: HLSFromTSTranscodeOptions) { + const command = this.commandWrapper.getCommand() + + const videoPath = this.getHLSVideoPath(options) + + command.outputOption('-c copy') + + if (options.isAAC) { + // Required for example when copying an AAC stream from an MPEG-TS + // Since it's a bitstream filter, we don't need to reencode the audio + command.outputOption('-bsf:a aac_adtstoasc') + } + + this.addCommonHLSVODCommandOptions(command, videoPath) + } + + private addCommonHLSVODCommandOptions (command: FfmpegCommand, outputPath: string) { + return command.outputOption('-hls_time 4') + .outputOption('-hls_list_size 0') + .outputOption('-hls_playlist_type vod') + .outputOption('-hls_segment_filename ' + outputPath) + .outputOption('-hls_segment_type fmp4') + .outputOption('-f hls') + .outputOption('-hls_flags single_file') + } + + private async fixHLSPlaylistIfNeeded (options: TranscodeVODOptions) { + if (options.type !== 'hls' && options.type !== 'hls-from-ts') return + + const fileContent = await readFile(options.outputPath) + + const videoFileName = options.hlsPlaylist.videoFilename + const videoFilePath = this.getHLSVideoPath(options) + + // Fix wrong mapping with some ffmpeg versions + const newContent = fileContent.toString() + .replace(`#EXT-X-MAP:URI="${videoFilePath}",`, `#EXT-X-MAP:URI="${videoFileName}",`) + + await writeFile(options.outputPath, newContent) + } + + private getHLSVideoPath (options: HLSTranscodeOptions | HLSFromTSTranscodeOptions) { + return `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}` + } +} diff --git a/packages/ffmpeg/src/ffprobe.ts b/packages/ffmpeg/src/ffprobe.ts new file mode 100644 index 000000000..ed1742ab1 --- /dev/null +++ b/packages/ffmpeg/src/ffprobe.ts @@ -0,0 +1,184 @@ +import ffmpeg, { FfprobeData } from 'fluent-ffmpeg' +import { forceNumber } from '@peertube/peertube-core-utils' +import { VideoResolution } from '@peertube/peertube-models' + +/** + * + * Helpers to run ffprobe and extract data from the JSON output + * + */ + +function ffprobePromise (path: string) { + return new Promise((res, rej) => { + ffmpeg.ffprobe(path, (err, data) => { + if (err) return rej(err) + + return res(data) + }) + }) +} + +// --------------------------------------------------------------------------- +// Audio +// --------------------------------------------------------------------------- + +const imageCodecs = new Set([ + 'ansi', 'apng', 'bintext', 'bmp', 'brender_pix', 'dpx', 'exr', 'fits', 'gem', 'gif', 'jpeg2000', 'jpgls', 'mjpeg', 'mjpegb', 'msp2', + 'pam', 'pbm', 'pcx', 'pfm', 'pgm', 'pgmyuv', 'pgx', 'photocd', 'pictor', 'png', 'ppm', 'psd', 'sgi', 'sunrast', 'svg', 'targa', 'tiff', + 'txd', 'webp', 'xbin', 'xbm', 'xface', 'xpm', 'xwd' +]) + +async function isAudioFile (path: string, existingProbe?: FfprobeData) { + const videoStream = await getVideoStream(path, existingProbe) + if (!videoStream) return true + + if (imageCodecs.has(videoStream.codec_name)) return true + + return false +} + +async function hasAudioStream (path: string, existingProbe?: FfprobeData) { + const { audioStream } = await getAudioStream(path, existingProbe) + + return !!audioStream +} + +async function getAudioStream (videoPath: string, existingProbe?: FfprobeData) { + // without position, ffprobe considers the last input only + // we make it consider the first input only + // if you pass a file path to pos, then ffprobe acts on that file directly + const data = existingProbe || await ffprobePromise(videoPath) + + if (Array.isArray(data.streams)) { + const audioStream = data.streams.find(stream => stream['codec_type'] === 'audio') + + if (audioStream) { + return { + absolutePath: data.format.filename, + audioStream, + bitrate: forceNumber(audioStream['bit_rate']) + } + } + } + + return { absolutePath: data.format.filename } +} + +function getMaxAudioBitrate (type: 'aac' | 'mp3' | string, bitrate: number) { + const maxKBitrate = 384 + const kToBits = (kbits: number) => kbits * 1000 + + // If we did not manage to get the bitrate, use an average value + if (!bitrate) return 256 + + if (type === 'aac') { + switch (true) { + case bitrate > kToBits(maxKBitrate): + return maxKBitrate + + default: + return -1 // we interpret it as a signal to copy the audio stream as is + } + } + + /* + a 192kbit/sec mp3 doesn't hold as much information as a 192kbit/sec aac. + That's why, when using aac, we can go to lower kbit/sec. The equivalences + made here are not made to be accurate, especially with good mp3 encoders. + */ + switch (true) { + case bitrate <= kToBits(192): + return 128 + + case bitrate <= kToBits(384): + return 256 + + default: + return maxKBitrate + } +} + +// --------------------------------------------------------------------------- +// Video +// --------------------------------------------------------------------------- + +async function getVideoStreamDimensionsInfo (path: string, existingProbe?: FfprobeData) { + const videoStream = await getVideoStream(path, existingProbe) + if (!videoStream) { + return { + width: 0, + height: 0, + ratio: 0, + resolution: VideoResolution.H_NOVIDEO, + isPortraitMode: false + } + } + + return { + width: videoStream.width, + height: videoStream.height, + ratio: Math.max(videoStream.height, videoStream.width) / Math.min(videoStream.height, videoStream.width), + resolution: Math.min(videoStream.height, videoStream.width), + isPortraitMode: videoStream.height > videoStream.width + } +} + +async function getVideoStreamFPS (path: string, existingProbe?: FfprobeData) { + const videoStream = await getVideoStream(path, existingProbe) + if (!videoStream) return 0 + + for (const key of [ 'avg_frame_rate', 'r_frame_rate' ]) { + const valuesText: string = videoStream[key] + if (!valuesText) continue + + const [ frames, seconds ] = valuesText.split('/') + if (!frames || !seconds) continue + + const result = parseInt(frames, 10) / parseInt(seconds, 10) + if (result > 0) return Math.round(result) + } + + return 0 +} + +async function getVideoStreamBitrate (path: string, existingProbe?: FfprobeData): Promise { + const metadata = existingProbe || await ffprobePromise(path) + + let bitrate = metadata.format.bit_rate + if (bitrate && !isNaN(bitrate)) return bitrate + + const videoStream = await getVideoStream(path, existingProbe) + if (!videoStream) return undefined + + bitrate = forceNumber(videoStream?.bit_rate) + if (bitrate && !isNaN(bitrate)) return bitrate + + return undefined +} + +async function getVideoStreamDuration (path: string, existingProbe?: FfprobeData) { + const metadata = existingProbe || await ffprobePromise(path) + + return Math.round(metadata.format.duration) +} + +async function getVideoStream (path: string, existingProbe?: FfprobeData) { + const metadata = existingProbe || await ffprobePromise(path) + + return metadata.streams.find(s => s.codec_type === 'video') +} + +// --------------------------------------------------------------------------- + +export { + getVideoStreamDimensionsInfo, + getMaxAudioBitrate, + getVideoStream, + getVideoStreamDuration, + getAudioStream, + getVideoStreamFPS, + isAudioFile, + ffprobePromise, + getVideoStreamBitrate, + hasAudioStream +} diff --git a/packages/ffmpeg/src/index.ts b/packages/ffmpeg/src/index.ts new file mode 100644 index 000000000..511409a50 --- /dev/null +++ b/packages/ffmpeg/src/index.ts @@ -0,0 +1,9 @@ +export * from './ffmpeg-command-wrapper.js' +export * from './ffmpeg-default-transcoding-profile.js' +export * from './ffmpeg-edition.js' +export * from './ffmpeg-images.js' +export * from './ffmpeg-live.js' +export * from './ffmpeg-utils.js' +export * from './ffmpeg-version.js' +export * from './ffmpeg-vod.js' +export * from './ffprobe.js' diff --git a/packages/ffmpeg/src/shared/encoder-options.ts b/packages/ffmpeg/src/shared/encoder-options.ts new file mode 100644 index 000000000..376a19186 --- /dev/null +++ b/packages/ffmpeg/src/shared/encoder-options.ts @@ -0,0 +1,39 @@ +import { FfmpegCommand } from 'fluent-ffmpeg' +import { EncoderOptions } from '@peertube/peertube-models' +import { buildStreamSuffix } from '../ffmpeg-utils.js' + +export function addDefaultEncoderGlobalParams (command: FfmpegCommand) { + // avoid issues when transcoding some files: https://trac.ffmpeg.org/ticket/6375 + command.outputOption('-max_muxing_queue_size 1024') + // strip all metadata + .outputOption('-map_metadata -1') + // allows import of source material with incompatible pixel formats (e.g. MJPEG video) + .outputOption('-pix_fmt yuv420p') +} + +export function addDefaultEncoderParams (options: { + command: FfmpegCommand + encoder: 'libx264' | string + fps: number + + streamNum?: number +}) { + const { command, encoder, fps, streamNum } = options + + if (encoder === 'libx264') { + // 3.1 is the minimal resource allocation for our highest supported resolution + command.outputOption(buildStreamSuffix('-level:v', streamNum) + ' 3.1') + + if (fps) { + // Keyframe interval of 2 seconds for faster seeking and resolution switching. + // https://streaminglearningcenter.com/blogs/whats-the-right-keyframe-interval.html + // https://superuser.com/a/908325 + command.outputOption(buildStreamSuffix('-g:v', streamNum) + ' ' + (fps * 2)) + } + } +} + +export function applyEncoderOptions (command: FfmpegCommand, options: EncoderOptions) { + command.inputOptions(options.inputOptions ?? []) + .outputOptions(options.outputOptions ?? []) +} diff --git a/packages/ffmpeg/src/shared/index.ts b/packages/ffmpeg/src/shared/index.ts new file mode 100644 index 000000000..81e8ff0b5 --- /dev/null +++ b/packages/ffmpeg/src/shared/index.ts @@ -0,0 +1,2 @@ +export * from './encoder-options.js' +export * from './presets.js' diff --git a/packages/ffmpeg/src/shared/presets.ts b/packages/ffmpeg/src/shared/presets.ts new file mode 100644 index 000000000..17bd7b031 --- /dev/null +++ b/packages/ffmpeg/src/shared/presets.ts @@ -0,0 +1,93 @@ +import { pick } from '@peertube/peertube-core-utils' +import { FFmpegCommandWrapper } from '../ffmpeg-command-wrapper.js' +import { getScaleFilter, StreamType } from '../ffmpeg-utils.js' +import { ffprobePromise, getVideoStreamBitrate, getVideoStreamDimensionsInfo, hasAudioStream } from '../ffprobe.js' +import { addDefaultEncoderGlobalParams, addDefaultEncoderParams, applyEncoderOptions } from './encoder-options.js' + +export async function presetVOD (options: { + commandWrapper: FFmpegCommandWrapper + + input: string + + canCopyAudio: boolean + canCopyVideo: boolean + + resolution: number + fps: number + + scaleFilterValue?: string +}) { + const { commandWrapper, input, resolution, fps, scaleFilterValue } = options + const command = commandWrapper.getCommand() + + command.format('mp4') + .outputOption('-movflags faststart') + + addDefaultEncoderGlobalParams(command) + + const probe = await ffprobePromise(input) + + // Audio encoder + const bitrate = await getVideoStreamBitrate(input, probe) + const videoStreamDimensions = await getVideoStreamDimensionsInfo(input, probe) + + let streamsToProcess: StreamType[] = [ 'audio', 'video' ] + + if (!await hasAudioStream(input, probe)) { + command.noAudio() + streamsToProcess = [ 'video' ] + } + + for (const streamType of streamsToProcess) { + const builderResult = await commandWrapper.getEncoderBuilderResult({ + ...pick(options, [ 'canCopyAudio', 'canCopyVideo' ]), + + input, + inputBitrate: bitrate, + inputRatio: videoStreamDimensions?.ratio || 0, + + resolution, + fps, + streamType, + + videoType: 'vod' as 'vod' + }) + + if (!builderResult) { + throw new Error('No available encoder found for stream ' + streamType) + } + + commandWrapper.debugLog( + `Apply ffmpeg params from ${builderResult.encoder} for ${streamType} ` + + `stream of input ${input} using ${commandWrapper.getProfile()} profile.`, + { builderResult, resolution, fps } + ) + + if (streamType === 'video') { + command.videoCodec(builderResult.encoder) + + if (scaleFilterValue) { + command.outputOption(`-vf ${getScaleFilter(builderResult.result)}=${scaleFilterValue}`) + } + } else if (streamType === 'audio') { + command.audioCodec(builderResult.encoder) + } + + applyEncoderOptions(command, builderResult.result) + addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps }) + } +} + +export function presetCopy (commandWrapper: FFmpegCommandWrapper) { + commandWrapper.getCommand() + .format('mp4') + .videoCodec('copy') + .audioCodec('copy') +} + +export function presetOnlyAudio (commandWrapper: FFmpegCommandWrapper) { + commandWrapper.getCommand() + .format('mp4') + .audioCodec('copy') + .noVideo() +} diff --git a/packages/ffmpeg/tsconfig.json b/packages/ffmpeg/tsconfig.json new file mode 100644 index 000000000..c8aeb3c14 --- /dev/null +++ b/packages/ffmpeg/tsconfig.json @@ -0,0 +1,12 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "src", + "tsBuildInfoFile": "./dist/.tsbuildinfo" + }, + "references": [ + { "path": "../models" }, + { "path": "../core-utils" } + ] +} diff --git a/packages/models/package.json b/packages/models/package.json new file mode 100644 index 000000000..58a993add --- /dev/null +++ b/packages/models/package.json @@ -0,0 +1,19 @@ +{ + "name": "@peertube/peertube-models", + "private": true, + "version": "0.0.0", + "main": "dist/index.js", + "type": "module", + "files": [ "dist" ], + "exports": { + "types": "./dist/index.d.ts", + "peertube:tsx": "./src/index.ts", + "default": "./dist/index.js" + }, + "devDependencies": {}, + "scripts": { + "build": "tsc", + "watch": "tsc -w" + }, + "dependencies": {} +} diff --git a/packages/models/src/activitypub/activity.ts b/packages/models/src/activitypub/activity.ts new file mode 100644 index 000000000..78a3ab33b --- /dev/null +++ b/packages/models/src/activitypub/activity.ts @@ -0,0 +1,135 @@ +import { ActivityPubActor } from './activitypub-actor.js' +import { ActivityPubSignature } from './activitypub-signature.js' +import { + ActivityFlagReasonObject, + ActivityObject, + APObjectId, + CacheFileObject, + PlaylistObject, + VideoCommentObject, + VideoObject, + WatchActionObject +} from './objects/index.js' + +export type ActivityUpdateObject = + Extract | ActivityPubActor + +// Cannot Extract from Activity because of circular reference +export type ActivityUndoObject = + ActivityFollow | ActivityLike | ActivityDislike | ActivityCreate | ActivityAnnounce + +export type ActivityCreateObject = + Extract + +export type Activity = + ActivityCreate | + ActivityUpdate | + ActivityDelete | + ActivityFollow | + ActivityAccept | + ActivityAnnounce | + ActivityUndo | + ActivityLike | + ActivityReject | + ActivityView | + ActivityDislike | + ActivityFlag + +export type ActivityType = + 'Create' | + 'Update' | + 'Delete' | + 'Follow' | + 'Accept' | + 'Announce' | + 'Undo' | + 'Like' | + 'Reject' | + 'View' | + 'Dislike' | + 'Flag' + +export interface ActivityAudience { + to: string[] + cc: string[] +} + +export interface BaseActivity { + '@context'?: any[] + id: string + to?: string[] + cc?: string[] + actor: string | ActivityPubActor + type: ActivityType + signature?: ActivityPubSignature +} + +export interface ActivityCreate extends BaseActivity { + type: 'Create' + object: T +} + +export interface ActivityUpdate extends BaseActivity { + type: 'Update' + object: T +} + +export interface ActivityDelete extends BaseActivity { + type: 'Delete' + object: APObjectId +} + +export interface ActivityFollow extends BaseActivity { + type: 'Follow' + object: string +} + +export interface ActivityAccept extends BaseActivity { + type: 'Accept' + object: ActivityFollow +} + +export interface ActivityReject extends BaseActivity { + type: 'Reject' + object: ActivityFollow +} + +export interface ActivityAnnounce extends BaseActivity { + type: 'Announce' + object: APObjectId +} + +export interface ActivityUndo extends BaseActivity { + type: 'Undo' + object: T +} + +export interface ActivityLike extends BaseActivity { + type: 'Like' + object: APObjectId +} + +export interface ActivityView extends BaseActivity { + type: 'View' + actor: string + object: APObjectId + + // If sending a "viewer" event + expires?: string +} + +export interface ActivityDislike extends BaseActivity { + id: string + type: 'Dislike' + actor: string + object: APObjectId +} + +export interface ActivityFlag extends BaseActivity { + type: 'Flag' + content: string + object: APObjectId | APObjectId[] + tag?: ActivityFlagReasonObject[] + startAt?: number + endAt?: number +} diff --git a/packages/models/src/activitypub/activitypub-actor.ts b/packages/models/src/activitypub/activitypub-actor.ts new file mode 100644 index 000000000..85b37c5ad --- /dev/null +++ b/packages/models/src/activitypub/activitypub-actor.ts @@ -0,0 +1,34 @@ +import { ActivityIconObject, ActivityPubAttributedTo } from './objects/common-objects.js' + +export type ActivityPubActorType = 'Person' | 'Application' | 'Group' | 'Service' | 'Organization' + +export interface ActivityPubActor { + '@context': any[] + type: ActivityPubActorType + id: string + following: string + followers: string + playlists?: string + inbox: string + outbox: string + preferredUsername: string + url: string + name: string + endpoints: { + sharedInbox: string + } + summary: string + attributedTo: ActivityPubAttributedTo[] + + support?: string + publicKey: { + id: string + owner: string + publicKeyPem: string + } + + image?: ActivityIconObject | ActivityIconObject[] + icon?: ActivityIconObject | ActivityIconObject[] + + published?: string +} diff --git a/packages/models/src/activitypub/activitypub-collection.ts b/packages/models/src/activitypub/activitypub-collection.ts new file mode 100644 index 000000000..b98ad37c2 --- /dev/null +++ b/packages/models/src/activitypub/activitypub-collection.ts @@ -0,0 +1,9 @@ +import { Activity } from './activity.js' + +export interface ActivityPubCollection { + '@context': string[] + type: 'Collection' | 'CollectionPage' + totalItems: number + partOf?: string + items: Activity[] +} diff --git a/shared/models/activitypub/activitypub-ordered-collection.ts b/packages/models/src/activitypub/activitypub-ordered-collection.ts similarity index 100% rename from shared/models/activitypub/activitypub-ordered-collection.ts rename to packages/models/src/activitypub/activitypub-ordered-collection.ts diff --git a/packages/models/src/activitypub/activitypub-root.ts b/packages/models/src/activitypub/activitypub-root.ts new file mode 100644 index 000000000..2fa1970c7 --- /dev/null +++ b/packages/models/src/activitypub/activitypub-root.ts @@ -0,0 +1,5 @@ +import { Activity } from './activity.js' +import { ActivityPubCollection } from './activitypub-collection.js' +import { ActivityPubOrderedCollection } from './activitypub-ordered-collection.js' + +export type RootActivity = Activity | ActivityPubCollection | ActivityPubOrderedCollection diff --git a/shared/models/activitypub/activitypub-signature.ts b/packages/models/src/activitypub/activitypub-signature.ts similarity index 100% rename from shared/models/activitypub/activitypub-signature.ts rename to packages/models/src/activitypub/activitypub-signature.ts diff --git a/shared/models/activitypub/context.ts b/packages/models/src/activitypub/context.ts similarity index 100% rename from shared/models/activitypub/context.ts rename to packages/models/src/activitypub/context.ts diff --git a/packages/models/src/activitypub/index.ts b/packages/models/src/activitypub/index.ts new file mode 100644 index 000000000..f36aa1bc5 --- /dev/null +++ b/packages/models/src/activitypub/index.ts @@ -0,0 +1,9 @@ +export * from './objects/index.js' +export * from './activity.js' +export * from './activitypub-actor.js' +export * from './activitypub-collection.js' +export * from './activitypub-ordered-collection.js' +export * from './activitypub-root.js' +export * from './activitypub-signature.js' +export * from './context.js' +export * from './webfinger.js' diff --git a/packages/models/src/activitypub/objects/abuse-object.ts b/packages/models/src/activitypub/objects/abuse-object.ts new file mode 100644 index 000000000..2c0f2832b --- /dev/null +++ b/packages/models/src/activitypub/objects/abuse-object.ts @@ -0,0 +1,15 @@ +import { ActivityFlagReasonObject } from './common-objects.js' + +export interface AbuseObject { + type: 'Flag' + + content: string + mediaType: 'text/markdown' + + object: string | string[] + + tag?: ActivityFlagReasonObject[] + + startAt?: number + endAt?: number +} diff --git a/packages/models/src/activitypub/objects/activitypub-object.ts b/packages/models/src/activitypub/objects/activitypub-object.ts new file mode 100644 index 000000000..93c925ae0 --- /dev/null +++ b/packages/models/src/activitypub/objects/activitypub-object.ts @@ -0,0 +1,17 @@ +import { AbuseObject } from './abuse-object.js' +import { CacheFileObject } from './cache-file-object.js' +import { PlaylistObject } from './playlist-object.js' +import { VideoCommentObject } from './video-comment-object.js' +import { VideoObject } from './video-object.js' +import { WatchActionObject } from './watch-action-object.js' + +export type ActivityObject = + VideoObject | + AbuseObject | + VideoCommentObject | + CacheFileObject | + PlaylistObject | + WatchActionObject | + string + +export type APObjectId = string | { id: string } diff --git a/packages/models/src/activitypub/objects/cache-file-object.ts b/packages/models/src/activitypub/objects/cache-file-object.ts new file mode 100644 index 000000000..a40ef339c --- /dev/null +++ b/packages/models/src/activitypub/objects/cache-file-object.ts @@ -0,0 +1,9 @@ +import { ActivityVideoUrlObject, ActivityPlaylistUrlObject } from './common-objects.js' + +export interface CacheFileObject { + id: string + type: 'CacheFile' + object: string + expires: string + url: ActivityVideoUrlObject | ActivityPlaylistUrlObject +} diff --git a/packages/models/src/activitypub/objects/common-objects.ts b/packages/models/src/activitypub/objects/common-objects.ts new file mode 100644 index 000000000..a332c26f3 --- /dev/null +++ b/packages/models/src/activitypub/objects/common-objects.ts @@ -0,0 +1,130 @@ +import { AbusePredefinedReasonsString } from '../../moderation/abuse/abuse-reason.model.js' + +export interface ActivityIdentifierObject { + identifier: string + name: string + url?: string +} + +export interface ActivityIconObject { + type: 'Image' + url: string + mediaType: string + width?: number + height?: number +} + +export type ActivityVideoUrlObject = { + type: 'Link' + mediaType: 'video/mp4' | 'video/webm' | 'video/ogg' + href: string + height: number + size: number + fps: number +} + +export type ActivityPlaylistSegmentHashesObject = { + type: 'Link' + name: 'sha256' + mediaType: 'application/json' + href: string +} + +export type ActivityVideoFileMetadataUrlObject = { + type: 'Link' + rel: [ 'metadata', any ] + mediaType: 'application/json' + height: number + href: string + fps: number +} + +export type ActivityTrackerUrlObject = { + type: 'Link' + rel: [ 'tracker', 'websocket' | 'http' ] + name: string + href: string +} + +export type ActivityStreamingPlaylistInfohashesObject = { + type: 'Infohash' + name: string +} + +export type ActivityPlaylistUrlObject = { + type: 'Link' + mediaType: 'application/x-mpegURL' + href: string + tag?: ActivityTagObject[] +} + +export type ActivityBitTorrentUrlObject = { + type: 'Link' + mediaType: 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet' + href: string + height: number +} + +export type ActivityMagnetUrlObject = { + type: 'Link' + mediaType: 'application/x-bittorrent;x-scheme-handler/magnet' + href: string + height: number +} + +export type ActivityHtmlUrlObject = { + type: 'Link' + mediaType: 'text/html' + href: string +} + +export interface ActivityHashTagObject { + type: 'Hashtag' + href?: string + name: string +} + +export interface ActivityMentionObject { + type: 'Mention' + href?: string + name: string +} + +export interface ActivityFlagReasonObject { + type: 'Hashtag' + name: AbusePredefinedReasonsString +} + +export type ActivityTagObject = + ActivityPlaylistSegmentHashesObject + | ActivityStreamingPlaylistInfohashesObject + | ActivityVideoUrlObject + | ActivityHashTagObject + | ActivityMentionObject + | ActivityBitTorrentUrlObject + | ActivityMagnetUrlObject + | ActivityVideoFileMetadataUrlObject + +export type ActivityUrlObject = + ActivityVideoUrlObject + | ActivityPlaylistUrlObject + | ActivityBitTorrentUrlObject + | ActivityMagnetUrlObject + | ActivityHtmlUrlObject + | ActivityVideoFileMetadataUrlObject + | ActivityTrackerUrlObject + +export type ActivityPubAttributedTo = { type: 'Group' | 'Person', id: string } | string + +export interface ActivityTombstoneObject { + '@context'?: any + id: string + url?: string + type: 'Tombstone' + name?: string + formerType?: string + inReplyTo?: string + published: string + updated: string + deleted: string +} diff --git a/packages/models/src/activitypub/objects/index.ts b/packages/models/src/activitypub/objects/index.ts new file mode 100644 index 000000000..510f621ea --- /dev/null +++ b/packages/models/src/activitypub/objects/index.ts @@ -0,0 +1,9 @@ +export * from './abuse-object.js' +export * from './activitypub-object.js' +export * from './cache-file-object.js' +export * from './common-objects.js' +export * from './playlist-element-object.js' +export * from './playlist-object.js' +export * from './video-comment-object.js' +export * from './video-object.js' +export * from './watch-action-object.js' diff --git a/shared/models/activitypub/objects/playlist-element-object.ts b/packages/models/src/activitypub/objects/playlist-element-object.ts similarity index 100% rename from shared/models/activitypub/objects/playlist-element-object.ts rename to packages/models/src/activitypub/objects/playlist-element-object.ts diff --git a/packages/models/src/activitypub/objects/playlist-object.ts b/packages/models/src/activitypub/objects/playlist-object.ts new file mode 100644 index 000000000..c68a28780 --- /dev/null +++ b/packages/models/src/activitypub/objects/playlist-object.ts @@ -0,0 +1,29 @@ +import { ActivityIconObject, ActivityPubAttributedTo } from './common-objects.js' + +export interface PlaylistObject { + id: string + type: 'Playlist' + + name: string + + content: string + mediaType: 'text/markdown' + + uuid: string + + totalItems: number + attributedTo: ActivityPubAttributedTo[] + + icon?: ActivityIconObject + + published: string + updated: string + + orderedItems?: string[] + + partOf?: string + next?: string + first?: string + + to?: string[] +} diff --git a/packages/models/src/activitypub/objects/video-comment-object.ts b/packages/models/src/activitypub/objects/video-comment-object.ts new file mode 100644 index 000000000..880dd2ee2 --- /dev/null +++ b/packages/models/src/activitypub/objects/video-comment-object.ts @@ -0,0 +1,16 @@ +import { ActivityPubAttributedTo, ActivityTagObject } from './common-objects.js' + +export interface VideoCommentObject { + type: 'Note' + id: string + + content: string + mediaType: 'text/markdown' + + inReplyTo: string + published: string + updated: string + url: string + attributedTo: ActivityPubAttributedTo + tag: ActivityTagObject[] +} diff --git a/packages/models/src/activitypub/objects/video-object.ts b/packages/models/src/activitypub/objects/video-object.ts new file mode 100644 index 000000000..14afd85a2 --- /dev/null +++ b/packages/models/src/activitypub/objects/video-object.ts @@ -0,0 +1,74 @@ +import { LiveVideoLatencyModeType, VideoStateType } from '../../videos/index.js' +import { + ActivityIconObject, + ActivityIdentifierObject, + ActivityPubAttributedTo, + ActivityTagObject, + ActivityUrlObject +} from './common-objects.js' + +export interface VideoObject { + type: 'Video' + id: string + name: string + duration: string + uuid: string + tag: ActivityTagObject[] + category: ActivityIdentifierObject + licence: ActivityIdentifierObject + language: ActivityIdentifierObject + subtitleLanguage: ActivityIdentifierObject[] + views: number + + sensitive: boolean + + isLiveBroadcast: boolean + liveSaveReplay: boolean + permanentLive: boolean + latencyMode: LiveVideoLatencyModeType + + commentsEnabled: boolean + downloadEnabled: boolean + waitTranscoding: boolean + state: VideoStateType + + published: string + originallyPublishedAt: string + updated: string + uploadDate: string + + mediaType: 'text/markdown' + content: string + + support: string + + icon: ActivityIconObject[] + + url: ActivityUrlObject[] + + likes: string + dislikes: string + shares: string + comments: string + + attributedTo: ActivityPubAttributedTo[] + + preview?: ActivityPubStoryboard[] + + to?: string[] + cc?: string[] +} + +export interface ActivityPubStoryboard { + type: 'Image' + rel: [ 'storyboard' ] + url: { + href: string + mediaType: string + width: number + height: number + tileWidth: number + tileHeight: number + tileDuration: string + }[] +} diff --git a/shared/models/activitypub/objects/watch-action-object.ts b/packages/models/src/activitypub/objects/watch-action-object.ts similarity index 100% rename from shared/models/activitypub/objects/watch-action-object.ts rename to packages/models/src/activitypub/objects/watch-action-object.ts diff --git a/shared/models/activitypub/webfinger.ts b/packages/models/src/activitypub/webfinger.ts similarity index 100% rename from shared/models/activitypub/webfinger.ts rename to packages/models/src/activitypub/webfinger.ts diff --git a/packages/models/src/actors/account.model.ts b/packages/models/src/actors/account.model.ts new file mode 100644 index 000000000..370a273cf --- /dev/null +++ b/packages/models/src/actors/account.model.ts @@ -0,0 +1,22 @@ +import { ActorImage } from './actor-image.model.js' +import { Actor } from './actor.model.js' + +export interface Account extends Actor { + displayName: string + description: string + avatars: ActorImage[] + + updatedAt: Date | string + + userId?: number +} + +export interface AccountSummary { + id: number + name: string + displayName: string + url: string + host: string + + avatars: ActorImage[] +} diff --git a/shared/models/actors/actor-image.model.ts b/packages/models/src/actors/actor-image.model.ts similarity index 100% rename from shared/models/actors/actor-image.model.ts rename to packages/models/src/actors/actor-image.model.ts diff --git a/packages/models/src/actors/actor-image.type.ts b/packages/models/src/actors/actor-image.type.ts new file mode 100644 index 000000000..3a808b110 --- /dev/null +++ b/packages/models/src/actors/actor-image.type.ts @@ -0,0 +1,6 @@ +export const ActorImageType = { + AVATAR: 1, + BANNER: 2 +} as const + +export type ActorImageType_Type = typeof ActorImageType[keyof typeof ActorImageType] diff --git a/packages/models/src/actors/actor.model.ts b/packages/models/src/actors/actor.model.ts new file mode 100644 index 000000000..d18053b4b --- /dev/null +++ b/packages/models/src/actors/actor.model.ts @@ -0,0 +1,13 @@ +import { ActorImage } from './actor-image.model.js' + +export interface Actor { + id: number + url: string + name: string + host: string + followingCount: number + followersCount: number + createdAt: Date | string + + avatars: ActorImage[] +} diff --git a/shared/models/actors/custom-page.model.ts b/packages/models/src/actors/custom-page.model.ts similarity index 100% rename from shared/models/actors/custom-page.model.ts rename to packages/models/src/actors/custom-page.model.ts diff --git a/packages/models/src/actors/follow.model.ts b/packages/models/src/actors/follow.model.ts new file mode 100644 index 000000000..7f3f52ac5 --- /dev/null +++ b/packages/models/src/actors/follow.model.ts @@ -0,0 +1,13 @@ +import { Actor } from './actor.model.js' + +export type FollowState = 'pending' | 'accepted' | 'rejected' + +export interface ActorFollow { + id: number + follower: Actor & { hostRedundancyAllowed: boolean } + following: Actor & { hostRedundancyAllowed: boolean } + score: number + state: FollowState + createdAt: Date + updatedAt: Date +} diff --git a/packages/models/src/actors/index.ts b/packages/models/src/actors/index.ts new file mode 100644 index 000000000..c44063c81 --- /dev/null +++ b/packages/models/src/actors/index.ts @@ -0,0 +1,6 @@ +export * from './account.model.js' +export * from './actor-image.model.js' +export * from './actor-image.type.js' +export * from './actor.model.js' +export * from './custom-page.model.js' +export * from './follow.model.js' diff --git a/shared/models/bulk/bulk-remove-comments-of-body.model.ts b/packages/models/src/bulk/bulk-remove-comments-of-body.model.ts similarity index 100% rename from shared/models/bulk/bulk-remove-comments-of-body.model.ts rename to packages/models/src/bulk/bulk-remove-comments-of-body.model.ts diff --git a/packages/models/src/bulk/index.ts b/packages/models/src/bulk/index.ts new file mode 100644 index 000000000..3597fda36 --- /dev/null +++ b/packages/models/src/bulk/index.ts @@ -0,0 +1 @@ +export * from './bulk-remove-comments-of-body.model.js' diff --git a/packages/models/src/common/index.ts b/packages/models/src/common/index.ts new file mode 100644 index 000000000..957851ae4 --- /dev/null +++ b/packages/models/src/common/index.ts @@ -0,0 +1 @@ +export * from './result-list.model.js' diff --git a/shared/models/common/result-list.model.ts b/packages/models/src/common/result-list.model.ts similarity index 100% rename from shared/models/common/result-list.model.ts rename to packages/models/src/common/result-list.model.ts diff --git a/shared/models/custom-markup/custom-markup-data.model.ts b/packages/models/src/custom-markup/custom-markup-data.model.ts similarity index 100% rename from shared/models/custom-markup/custom-markup-data.model.ts rename to packages/models/src/custom-markup/custom-markup-data.model.ts diff --git a/packages/models/src/custom-markup/index.ts b/packages/models/src/custom-markup/index.ts new file mode 100644 index 000000000..1ce10d8e2 --- /dev/null +++ b/packages/models/src/custom-markup/index.ts @@ -0,0 +1 @@ +export * from './custom-markup-data.model.js' diff --git a/packages/models/src/feeds/feed-format.enum.ts b/packages/models/src/feeds/feed-format.enum.ts new file mode 100644 index 000000000..71f2f6c15 --- /dev/null +++ b/packages/models/src/feeds/feed-format.enum.ts @@ -0,0 +1,7 @@ +export const FeedFormat = { + RSS: 'xml', + ATOM: 'atom', + JSON: 'json' +} as const + +export type FeedFormatType = typeof FeedFormat[keyof typeof FeedFormat] diff --git a/packages/models/src/feeds/index.ts b/packages/models/src/feeds/index.ts new file mode 100644 index 000000000..1eda1d6c3 --- /dev/null +++ b/packages/models/src/feeds/index.ts @@ -0,0 +1 @@ +export * from './feed-format.enum.js' diff --git a/packages/models/src/http/http-methods.ts b/packages/models/src/http/http-methods.ts new file mode 100644 index 000000000..ec3c855d8 --- /dev/null +++ b/packages/models/src/http/http-methods.ts @@ -0,0 +1,23 @@ +/** HTTP request method to indicate the desired action to be performed for a given resource. */ +export const HttpMethod = { + /** The CONNECT method establishes a tunnel to the server identified by the target resource. */ + CONNECT: 'CONNECT', + /** The DELETE method deletes the specified resource. */ + DELETE: 'DELETE', + /** The GET method requests a representation of the specified resource. Requests using GET should only retrieve data. */ + GET: 'GET', + /** The HEAD method asks for a response identical to that of a GET request, but without the response body. */ + HEAD: 'HEAD', + /** The OPTIONS method is used to describe the communication options for the target resource. */ + OPTIONS: 'OPTIONS', + /** The PATCH method is used to apply partial modifications to a resource. */ + PATCH: 'PATCH', + /** The POST method is used to submit an entity to the specified resource */ + POST: 'POST', + /** The PUT method replaces all current representations of the target resource with the request payload. */ + PUT: 'PUT', + /** The TRACE method performs a message loop-back test along the path to the target resource. */ + TRACE: 'TRACE' +} as const + +export type HttpMethodType = typeof HttpMethod[keyof typeof HttpMethod] diff --git a/packages/models/src/http/http-status-codes.ts b/packages/models/src/http/http-status-codes.ts new file mode 100644 index 000000000..920b9a2e9 --- /dev/null +++ b/packages/models/src/http/http-status-codes.ts @@ -0,0 +1,366 @@ +/** + * Hypertext Transfer Protocol (HTTP) response status codes. + * @see {@link https://en.wikipedia.org/wiki/List_of_HTTP_status_codes} + * + * WebDAV and other codes useless with regards to PeerTube are not listed. + */ +export const HttpStatusCode = { + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.2.1 + * + * The server has received the request headers and the client should proceed to send the request body + * (in the case of a request for which a body needs to be sent; for example, a POST request). + * Sending a large request body to a server after a request has been rejected for inappropriate headers would be inefficient. + * To have a server check the request's headers, a client must send Expect: 100-continue as a header in its initial request + * and receive a 100 Continue status code in response before sending the body. The response 417 Expectation Failed indicates + * the request should not be continued. + */ + CONTINUE_100: 100, + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.2.2 + * + * This code is sent in response to an Upgrade request header by the client, and indicates the protocol the server is switching too. + */ + SWITCHING_PROTOCOLS_101: 101, + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.3.1 + * + * Standard response for successful HTTP requests. The actual response will depend on the request method used: + * GET: The resource has been fetched and is transmitted in the message body. + * HEAD: The entity headers are in the message body. + * POST: The resource describing the result of the action is transmitted in the message body. + * TRACE: The message body contains the request message as received by the server + */ + OK_200: 200, + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.3.2 + * + * The request has been fulfilled, resulting in the creation of a new resource, typically after a PUT. + */ + CREATED_201: 201, + + /** + * The request has been accepted for processing, but the processing has not been completed. + * The request might or might not be eventually acted upon, and may be disallowed when processing occurs. + */ + ACCEPTED_202: 202, + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.3.5 + * + * There is no content to send for this request, but the headers may be useful. + * The user-agent may update its cached headers for this resource with the new ones. + */ + NO_CONTENT_204: 204, + + /** + * The server successfully processed the request, but is not returning any content. + * Unlike a 204 response, this response requires that the requester reset the document view. + */ + RESET_CONTENT_205: 205, + + /** + * The server is delivering only part of the resource (byte serving) due to a range header sent by the client. + * The range header is used by HTTP clients to enable resuming of interrupted downloads, + * or split a download into multiple simultaneous streams. + */ + PARTIAL_CONTENT_206: 206, + + /** + * Indicates multiple options for the resource from which the client may choose (via agent-driven content negotiation). + * For example, this code could be used to present multiple video format options, + * to list files with different filename extensions, or to suggest word-sense disambiguation. + */ + MULTIPLE_CHOICES_300: 300, + + /** + * This and all future requests should be directed to the given URI. + */ + MOVED_PERMANENTLY_301: 301, + + /** + * This is an example of industry practice contradicting the standard. + * The HTTP/1.0 specification (RFC 1945) required the client to perform a temporary redirect + * (the original describing phrase was "Moved Temporarily"), but popular browsers implemented 302 + * with the functionality of a 303 See Other. Therefore, HTTP/1.1 added status codes 303 and 307 + * to distinguish between the two behaviours. However, some Web applications and frameworks + * use the 302 status code as if it were the 303. + */ + FOUND_302: 302, + + /** + * SINCE HTTP/1.1 + * The response to the request can be found under another URI using a GET method. + * When received in response to a POST (or PUT/DELETE), the client should presume that + * the server has received the data and should issue a redirect with a separate GET message. + */ + SEE_OTHER_303: 303, + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc7232#section-4.1 + * + * Indicates that the resource has not been modified since the version specified by the request headers + * `If-Modified-Since` or `If-None-Match`. + * In such case, there is no need to retransmit the resource since the client still has a previously-downloaded copy. + */ + NOT_MODIFIED_304: 304, + + /** + * SINCE HTTP/1.1 + * In this case, the request should be repeated with another URI; however, future requests should still use the original URI. + * In contrast to how 302 was historically implemented, the request method is not allowed to be changed when reissuing the + * original request. + * For example, a POST request should be repeated using another POST request. + */ + TEMPORARY_REDIRECT_307: 307, + + /** + * The request and all future requests should be repeated using another URI. + * 307 and 308 parallel the behaviors of 302 and 301, but do not allow the HTTP method to change. + * So, for example, submitting a form to a permanently redirected resource may continue smoothly. + */ + PERMANENT_REDIRECT_308: 308, + + /** + * The server cannot or will not process the request due to an apparent client error + * (e.g., malformed request syntax, too large size, invalid request message framing, or deceptive request routing). + */ + BAD_REQUEST_400: 400, + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc7235#section-3.1 + * + * Similar to 403 Forbidden, but specifically for use when authentication is required and has failed or has not yet + * been provided. The response must include a `WWW-Authenticate` header field containing a challenge applicable to the + * requested resource. See Basic access authentication and Digest access authentication. 401 semantically means + * "unauthenticated",i.e. the user does not have the necessary credentials. + */ + UNAUTHORIZED_401: 401, + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.2 + * + * Reserved for future use. The original intention was that this code might be used as part of some form of digital + * cash or micro payment scheme, but that has not happened, and this code is not usually used. + * Google Developers API uses this status if a particular developer has exceeded the daily limit on requests. + */ + PAYMENT_REQUIRED_402: 402, + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.3 + * + * The client does not have access rights to the content, i.e. they are unauthorized, so server is rejecting to + * give proper response. Unlike 401, the client's identity is known to the server. + */ + FORBIDDEN_403: 403, + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.6.2 + * + * The requested resource could not be found but may be available in the future. + * Subsequent requests by the client are permissible. + */ + NOT_FOUND_404: 404, + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.5 + * + * A request method is not supported for the requested resource; + * for example, a GET request on a form that requires data to be presented via POST, or a PUT request on a read-only resource. + */ + METHOD_NOT_ALLOWED_405: 405, + + /** + * The requested resource is capable of generating only content not acceptable according to the Accept headers sent in the request. + */ + NOT_ACCEPTABLE_406: 406, + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.7 + * + * This response is sent on an idle connection by some servers, even without any previous request by the client. + * It means that the server would like to shut down this unused connection. This response is used much more since + * some browsers, like Chrome, Firefox 27+, or IE9, use HTTP pre-connection mechanisms to speed up surfing. Also + * note that some servers merely shut down the connection without sending this message. + * + * @ + */ + REQUEST_TIMEOUT_408: 408, + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.8 + * + * Indicates that the request could not be processed because of conflict in the request, + * such as an edit conflict between multiple simultaneous updates. + * + * @see HttpStatusCode.UNPROCESSABLE_ENTITY_422 to denote a disabled feature + */ + CONFLICT_409: 409, + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.9 + * + * Indicates that the resource requested is no longer available and will not be available again. + * This should be used when a resource has been intentionally removed and the resource should be purged. + * Upon receiving a 410 status code, the client should not request the resource in the future. + * Clients such as search engines should remove the resource from their indices. + * Most use cases do not require clients and search engines to purge the resource, and a "404 Not Found" may be used instead. + */ + GONE_410: 410, + + /** + * The request did not specify the length of its content, which is required by the requested resource. + */ + LENGTH_REQUIRED_411: 411, + + /** + * The server does not meet one of the preconditions that the requester put on the request. + */ + PRECONDITION_FAILED_412: 412, + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.11 + * + * The request is larger than the server is willing or able to process ; the server might close the connection + * or return an Retry-After header field. + * Previously called "Request Entity Too Large". + */ + PAYLOAD_TOO_LARGE_413: 413, + + /** + * The URI provided was too long for the server to process. Often the result of too much data being encoded as a + * query-string of a GET request, in which case it should be converted to a POST request. + * Called "Request-URI Too Long" previously. + */ + URI_TOO_LONG_414: 414, + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.13 + * + * The request entity has a media type which the server or resource does not support. + * For example, the client uploads an image as image/svg+xml, but the server requires that images use a different format. + */ + UNSUPPORTED_MEDIA_TYPE_415: 415, + + /** + * The client has asked for a portion of the file (byte serving), but the server cannot supply that portion. + * For example, if the client asked for a part of the file that lies beyond the end of the file. + * Called "Requested Range Not Satisfiable" previously. + */ + RANGE_NOT_SATISFIABLE_416: 416, + + /** + * The server cannot meet the requirements of the `Expect` request-header field. + */ + EXPECTATION_FAILED_417: 417, + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc2324 + * + * This code was defined in 1998 as one of the traditional IETF April Fools' jokes, in RFC 2324, Hyper Text Coffee Pot Control Protocol, + * and is not expected to be implemented by actual HTTP servers. The RFC specifies this code should be returned by + * teapots requested to brew coffee. This HTTP status is used as an Easter egg in some websites, including PeerTube instances ;-). + */ + I_AM_A_TEAPOT_418: 418, + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc2518#section-10.3 + * + * The request was well-formed but was unable to be followed due to semantic errors. + * The server understands the content type of the request entity (hence a 415 (Unsupported Media Type) status code is inappropriate), + * and the syntax of the request entity is correct (thus a 400 (Bad Request) status code is inappropriate) but was unable to process + * the contained instructions. For example, this error condition may occur if an JSON request body contains well-formed (i.e., + * syntactically correct), but semantically erroneous, JSON instructions. + * + * Can also be used to denote disabled features (akin to disabled syntax). + * + * @see HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415 if the `Content-Type` was not supported. + * @see HttpStatusCode.BAD_REQUEST_400 if the request was not parsable (broken JSON, XML) + */ + UNPROCESSABLE_ENTITY_422: 422, + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc4918#section-11.3 + * + * The resource that is being accessed is locked. WebDAV-specific but used by some HTTP services. + * + * @deprecated use `If-Match` / `If-None-Match` instead + * @see {@link https://evertpot.com/http/423-locked} + */ + LOCKED_423: 423, + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc6585#section-4 + * + * The user has sent too many requests in a given amount of time. Intended for use with rate-limiting schemes. + */ + TOO_MANY_REQUESTS_429: 429, + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc6585#section-5 + * + * The server is unwilling to process the request because either an individual header field, + * or all the header fields collectively, are too large. + */ + REQUEST_HEADER_FIELDS_TOO_LARGE_431: 431, + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc7725 + * + * A server operator has received a legal demand to deny access to a resource or to a set of resources + * that includes the requested resource. The code 451 was chosen as a reference to the novel Fahrenheit 451. + */ + UNAVAILABLE_FOR_LEGAL_REASONS_451: 451, + + /** + * A generic error message, given when an unexpected condition was encountered and no more specific message is suitable. + */ + INTERNAL_SERVER_ERROR_500: 500, + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.6.2 + * + * The server either does not recognize the request method, or it lacks the ability to fulfill the request. + * Usually this implies future availability (e.g., a new feature of a web-service API). + */ + NOT_IMPLEMENTED_501: 501, + + /** + * The server was acting as a gateway or proxy and received an invalid response from the upstream server. + */ + BAD_GATEWAY_502: 502, + + /** + * The server is currently unavailable (because it is overloaded or down for maintenance). + * Generally, this is a temporary state. + */ + SERVICE_UNAVAILABLE_503: 503, + + /** + * The server was acting as a gateway or proxy and did not receive a timely response from the upstream server. + */ + GATEWAY_TIMEOUT_504: 504, + + /** + * The server does not support the HTTP protocol version used in the request + */ + HTTP_VERSION_NOT_SUPPORTED_505: 505, + + /** + * Official Documentation @ https://tools.ietf.org/html/rfc2518#section-10.6 + * + * The 507 (Insufficient Storage) status code means the method could not be performed on the resource because the + * server is unable to store the representation needed to successfully complete the request. This condition is + * considered to be temporary. If the request which received this status code was the result of a user action, + * the request MUST NOT be repeated until it is requested by a separate user action. + * + * @see HttpStatusCode.PAYLOAD_TOO_LARGE_413 for quota errors + */ + INSUFFICIENT_STORAGE_507: 507 +} as const + +export type HttpStatusCodeType = typeof HttpStatusCode[keyof typeof HttpStatusCode] diff --git a/packages/models/src/http/index.ts b/packages/models/src/http/index.ts new file mode 100644 index 000000000..f0ad040ed --- /dev/null +++ b/packages/models/src/http/index.ts @@ -0,0 +1,2 @@ +export * from './http-status-codes.js' +export * from './http-methods.js' diff --git a/packages/models/src/index.ts b/packages/models/src/index.ts new file mode 100644 index 000000000..b76703dff --- /dev/null +++ b/packages/models/src/index.ts @@ -0,0 +1,20 @@ +export * from './activitypub/index.js' +export * from './actors/index.js' +export * from './bulk/index.js' +export * from './common/index.js' +export * from './custom-markup/index.js' +export * from './feeds/index.js' +export * from './http/index.js' +export * from './joinpeertube/index.js' +export * from './metrics/index.js' +export * from './moderation/index.js' +export * from './nodeinfo/index.js' +export * from './overviews/index.js' +export * from './plugins/index.js' +export * from './redundancy/index.js' +export * from './runners/index.js' +export * from './search/index.js' +export * from './server/index.js' +export * from './tokens/index.js' +export * from './users/index.js' +export * from './videos/index.js' diff --git a/packages/models/src/joinpeertube/index.ts b/packages/models/src/joinpeertube/index.ts new file mode 100644 index 000000000..a51d34190 --- /dev/null +++ b/packages/models/src/joinpeertube/index.ts @@ -0,0 +1 @@ +export * from './versions.model.js' diff --git a/shared/models/joinpeertube/versions.model.ts b/packages/models/src/joinpeertube/versions.model.ts similarity index 100% rename from shared/models/joinpeertube/versions.model.ts rename to packages/models/src/joinpeertube/versions.model.ts diff --git a/packages/models/src/metrics/index.ts b/packages/models/src/metrics/index.ts new file mode 100644 index 000000000..def6f8095 --- /dev/null +++ b/packages/models/src/metrics/index.ts @@ -0,0 +1 @@ +export * from './playback-metric-create.model.js' diff --git a/packages/models/src/metrics/playback-metric-create.model.ts b/packages/models/src/metrics/playback-metric-create.model.ts new file mode 100644 index 000000000..3ae91b295 --- /dev/null +++ b/packages/models/src/metrics/playback-metric-create.model.ts @@ -0,0 +1,22 @@ +import { VideoResolutionType } from '../videos/index.js' + +export interface PlaybackMetricCreate { + playerMode: 'p2p-media-loader' | 'webtorrent' | 'web-video' // FIXME: remove webtorrent player mode not used anymore in PeerTube v6 + + resolution?: VideoResolutionType + fps?: number + + p2pEnabled: boolean + p2pPeers?: number + + resolutionChanges: number + + errors: number + + downloadedBytesP2P: number + downloadedBytesHTTP: number + + uploadedBytesP2P: number + + videoId: number | string +} diff --git a/packages/models/src/moderation/abuse/abuse-create.model.ts b/packages/models/src/moderation/abuse/abuse-create.model.ts new file mode 100644 index 000000000..1c2723b1c --- /dev/null +++ b/packages/models/src/moderation/abuse/abuse-create.model.ts @@ -0,0 +1,21 @@ +import { AbusePredefinedReasonsString } from './abuse-reason.model.js' + +export interface AbuseCreate { + reason: string + + predefinedReasons?: AbusePredefinedReasonsString[] + + account?: { + id: number + } + + video?: { + id: number | string + startAt?: number + endAt?: number + } + + comment?: { + id: number + } +} diff --git a/shared/models/moderation/abuse/abuse-filter.type.ts b/packages/models/src/moderation/abuse/abuse-filter.type.ts similarity index 100% rename from shared/models/moderation/abuse/abuse-filter.type.ts rename to packages/models/src/moderation/abuse/abuse-filter.type.ts diff --git a/packages/models/src/moderation/abuse/abuse-message.model.ts b/packages/models/src/moderation/abuse/abuse-message.model.ts new file mode 100644 index 000000000..9ba95e724 --- /dev/null +++ b/packages/models/src/moderation/abuse/abuse-message.model.ts @@ -0,0 +1,10 @@ +import { AccountSummary } from '../../actors/account.model.js' + +export interface AbuseMessage { + id: number + message: string + byModerator: boolean + createdAt: Date | string + + account: AccountSummary +} diff --git a/packages/models/src/moderation/abuse/abuse-reason.model.ts b/packages/models/src/moderation/abuse/abuse-reason.model.ts new file mode 100644 index 000000000..770b9d47a --- /dev/null +++ b/packages/models/src/moderation/abuse/abuse-reason.model.ts @@ -0,0 +1,22 @@ +export const AbusePredefinedReasons = { + VIOLENT_OR_REPULSIVE: 1, + HATEFUL_OR_ABUSIVE: 2, + SPAM_OR_MISLEADING: 3, + PRIVACY: 4, + RIGHTS: 5, + SERVER_RULES: 6, + THUMBNAILS: 7, + CAPTIONS: 8 +} as const + +export type AbusePredefinedReasonsType = typeof AbusePredefinedReasons[keyof typeof AbusePredefinedReasons] + +export type AbusePredefinedReasonsString = + 'violentOrRepulsive' | + 'hatefulOrAbusive' | + 'spamOrMisleading' | + 'privacy' | + 'rights' | + 'serverRules' | + 'thumbnails' | + 'captions' diff --git a/packages/models/src/moderation/abuse/abuse-state.model.ts b/packages/models/src/moderation/abuse/abuse-state.model.ts new file mode 100644 index 000000000..5582d73c4 --- /dev/null +++ b/packages/models/src/moderation/abuse/abuse-state.model.ts @@ -0,0 +1,7 @@ +export const AbuseState = { + PENDING: 1, + REJECTED: 2, + ACCEPTED: 3 +} as const + +export type AbuseStateType = typeof AbuseState[keyof typeof AbuseState] diff --git a/packages/models/src/moderation/abuse/abuse-update.model.ts b/packages/models/src/moderation/abuse/abuse-update.model.ts new file mode 100644 index 000000000..22a01be89 --- /dev/null +++ b/packages/models/src/moderation/abuse/abuse-update.model.ts @@ -0,0 +1,7 @@ +import { AbuseStateType } from './abuse-state.model.js' + +export interface AbuseUpdate { + moderationComment?: string + + state?: AbuseStateType +} diff --git a/shared/models/moderation/abuse/abuse-video-is.type.ts b/packages/models/src/moderation/abuse/abuse-video-is.type.ts similarity index 100% rename from shared/models/moderation/abuse/abuse-video-is.type.ts rename to packages/models/src/moderation/abuse/abuse-video-is.type.ts diff --git a/packages/models/src/moderation/abuse/abuse.model.ts b/packages/models/src/moderation/abuse/abuse.model.ts new file mode 100644 index 000000000..253a3f44c --- /dev/null +++ b/packages/models/src/moderation/abuse/abuse.model.ts @@ -0,0 +1,70 @@ +import { Account } from '../../actors/account.model.js' +import { AbuseStateType } from './abuse-state.model.js' +import { AbusePredefinedReasonsString } from './abuse-reason.model.js' +import { VideoConstant } from '../../videos/video-constant.model.js' +import { VideoChannel } from '../../videos/channel/video-channel.model.js' + +export interface AdminVideoAbuse { + id: number + name: string + uuid: string + nsfw: boolean + + deleted: boolean + blacklisted: boolean + + startAt: number | null + endAt: number | null + + thumbnailPath?: string + channel?: VideoChannel + + countReports: number + nthReport: number +} + +export interface AdminVideoCommentAbuse { + id: number + threadId: number + + video: { + id: number + name: string + uuid: string + } + + text: string + + deleted: boolean +} + +export interface AdminAbuse { + id: number + + reason: string + predefinedReasons?: AbusePredefinedReasonsString[] + + reporterAccount: Account + flaggedAccount: Account + + state: VideoConstant + moderationComment?: string + + video?: AdminVideoAbuse + comment?: AdminVideoCommentAbuse + + createdAt: Date + updatedAt: Date + + countReportsForReporter?: number + countReportsForReportee?: number + + countMessages: number +} + +export type UserVideoAbuse = Omit + +export type UserVideoCommentAbuse = AdminVideoCommentAbuse + +export type UserAbuse = Omit diff --git a/packages/models/src/moderation/abuse/index.ts b/packages/models/src/moderation/abuse/index.ts new file mode 100644 index 000000000..27fca7076 --- /dev/null +++ b/packages/models/src/moderation/abuse/index.ts @@ -0,0 +1,8 @@ +export * from './abuse-create.model.js' +export * from './abuse-filter.type.js' +export * from './abuse-message.model.js' +export * from './abuse-reason.model.js' +export * from './abuse-state.model.js' +export * from './abuse-update.model.js' +export * from './abuse-video-is.type.js' +export * from './abuse.model.js' diff --git a/packages/models/src/moderation/account-block.model.ts b/packages/models/src/moderation/account-block.model.ts new file mode 100644 index 000000000..2d070da62 --- /dev/null +++ b/packages/models/src/moderation/account-block.model.ts @@ -0,0 +1,7 @@ +import { Account } from '../actors/index.js' + +export interface AccountBlock { + byAccount: Account + blockedAccount: Account + createdAt: Date | string +} diff --git a/shared/models/moderation/block-status.model.ts b/packages/models/src/moderation/block-status.model.ts similarity index 100% rename from shared/models/moderation/block-status.model.ts rename to packages/models/src/moderation/block-status.model.ts diff --git a/packages/models/src/moderation/index.ts b/packages/models/src/moderation/index.ts new file mode 100644 index 000000000..52e21e7b3 --- /dev/null +++ b/packages/models/src/moderation/index.ts @@ -0,0 +1,4 @@ +export * from './abuse/index.js' +export * from './block-status.model.js' +export * from './account-block.model.js' +export * from './server-block.model.js' diff --git a/packages/models/src/moderation/server-block.model.ts b/packages/models/src/moderation/server-block.model.ts new file mode 100644 index 000000000..b85646fc6 --- /dev/null +++ b/packages/models/src/moderation/server-block.model.ts @@ -0,0 +1,9 @@ +import { Account } from '../actors/index.js' + +export interface ServerBlock { + byAccount: Account + blockedServer: { + host: string + } + createdAt: Date | string +} diff --git a/packages/models/src/nodeinfo/index.ts b/packages/models/src/nodeinfo/index.ts new file mode 100644 index 000000000..932288795 --- /dev/null +++ b/packages/models/src/nodeinfo/index.ts @@ -0,0 +1 @@ +export * from './nodeinfo.model.js' diff --git a/shared/models/nodeinfo/nodeinfo.model.ts b/packages/models/src/nodeinfo/nodeinfo.model.ts similarity index 100% rename from shared/models/nodeinfo/nodeinfo.model.ts rename to packages/models/src/nodeinfo/nodeinfo.model.ts diff --git a/packages/models/src/overviews/index.ts b/packages/models/src/overviews/index.ts new file mode 100644 index 000000000..20dc105e2 --- /dev/null +++ b/packages/models/src/overviews/index.ts @@ -0,0 +1 @@ +export * from './videos-overview.model.js' diff --git a/packages/models/src/overviews/videos-overview.model.ts b/packages/models/src/overviews/videos-overview.model.ts new file mode 100644 index 000000000..3a1ba1760 --- /dev/null +++ b/packages/models/src/overviews/videos-overview.model.ts @@ -0,0 +1,24 @@ +import { Video, VideoChannelSummary, VideoConstant } from '../videos/index.js' + +export interface ChannelOverview { + channel: VideoChannelSummary + videos: Video[] +} + +export interface CategoryOverview { + category: VideoConstant + videos: Video[] +} + +export interface TagOverview { + tag: string + videos: Video[] +} + +export interface VideosOverview { + channels: ChannelOverview[] + + categories: CategoryOverview[] + + tags: TagOverview[] +} diff --git a/shared/models/plugins/client/client-hook.model.ts b/packages/models/src/plugins/client/client-hook.model.ts similarity index 100% rename from shared/models/plugins/client/client-hook.model.ts rename to packages/models/src/plugins/client/client-hook.model.ts diff --git a/packages/models/src/plugins/client/index.ts b/packages/models/src/plugins/client/index.ts new file mode 100644 index 000000000..04fa32d6d --- /dev/null +++ b/packages/models/src/plugins/client/index.ts @@ -0,0 +1,8 @@ +export * from './client-hook.model.js' +export * from './plugin-client-scope.type.js' +export * from './plugin-element-placeholder.type.js' +export * from './plugin-selector-id.type.js' +export * from './register-client-form-field.model.js' +export * from './register-client-hook.model.js' +export * from './register-client-route.model.js' +export * from './register-client-settings-script.model.js' diff --git a/shared/models/plugins/client/plugin-client-scope.type.ts b/packages/models/src/plugins/client/plugin-client-scope.type.ts similarity index 100% rename from shared/models/plugins/client/plugin-client-scope.type.ts rename to packages/models/src/plugins/client/plugin-client-scope.type.ts diff --git a/shared/models/plugins/client/plugin-element-placeholder.type.ts b/packages/models/src/plugins/client/plugin-element-placeholder.type.ts similarity index 100% rename from shared/models/plugins/client/plugin-element-placeholder.type.ts rename to packages/models/src/plugins/client/plugin-element-placeholder.type.ts diff --git a/shared/models/plugins/client/plugin-selector-id.type.ts b/packages/models/src/plugins/client/plugin-selector-id.type.ts similarity index 100% rename from shared/models/plugins/client/plugin-selector-id.type.ts rename to packages/models/src/plugins/client/plugin-selector-id.type.ts diff --git a/shared/models/plugins/client/register-client-form-field.model.ts b/packages/models/src/plugins/client/register-client-form-field.model.ts similarity index 100% rename from shared/models/plugins/client/register-client-form-field.model.ts rename to packages/models/src/plugins/client/register-client-form-field.model.ts diff --git a/packages/models/src/plugins/client/register-client-hook.model.ts b/packages/models/src/plugins/client/register-client-hook.model.ts new file mode 100644 index 000000000..19159ed1e --- /dev/null +++ b/packages/models/src/plugins/client/register-client-hook.model.ts @@ -0,0 +1,7 @@ +import { ClientHookName } from './client-hook.model.js' + +export interface RegisterClientHookOptions { + target: ClientHookName + handler: Function + priority?: number +} diff --git a/shared/models/plugins/client/register-client-route.model.ts b/packages/models/src/plugins/client/register-client-route.model.ts similarity index 100% rename from shared/models/plugins/client/register-client-route.model.ts rename to packages/models/src/plugins/client/register-client-route.model.ts diff --git a/packages/models/src/plugins/client/register-client-settings-script.model.ts b/packages/models/src/plugins/client/register-client-settings-script.model.ts new file mode 100644 index 000000000..7de3c1c28 --- /dev/null +++ b/packages/models/src/plugins/client/register-client-settings-script.model.ts @@ -0,0 +1,8 @@ +import { RegisterServerSettingOptions } from '../server/index.js' + +export interface RegisterClientSettingsScriptOptions { + isSettingHidden (options: { + setting: RegisterServerSettingOptions + formValues: { [name: string]: any } + }): boolean +} diff --git a/packages/models/src/plugins/hook-type.enum.ts b/packages/models/src/plugins/hook-type.enum.ts new file mode 100644 index 000000000..7acc5f48a --- /dev/null +++ b/packages/models/src/plugins/hook-type.enum.ts @@ -0,0 +1,7 @@ +export const HookType = { + STATIC: 1, + ACTION: 2, + FILTER: 3 +} as const + +export type HookType_Type = typeof HookType[keyof typeof HookType] diff --git a/packages/models/src/plugins/index.ts b/packages/models/src/plugins/index.ts new file mode 100644 index 000000000..1117a946e --- /dev/null +++ b/packages/models/src/plugins/index.ts @@ -0,0 +1,6 @@ +export * from './client/index.js' +export * from './plugin-index/index.js' +export * from './server/index.js' +export * from './hook-type.enum.js' +export * from './plugin-package-json.model.js' +export * from './plugin.type.js' diff --git a/packages/models/src/plugins/plugin-index/index.ts b/packages/models/src/plugins/plugin-index/index.ts new file mode 100644 index 000000000..f53b88084 --- /dev/null +++ b/packages/models/src/plugins/plugin-index/index.ts @@ -0,0 +1,3 @@ +export * from './peertube-plugin-index-list.model.js' +export * from './peertube-plugin-index.model.js' +export * from './peertube-plugin-latest-version.model.js' diff --git a/packages/models/src/plugins/plugin-index/peertube-plugin-index-list.model.ts b/packages/models/src/plugins/plugin-index/peertube-plugin-index-list.model.ts new file mode 100644 index 000000000..98301bbc1 --- /dev/null +++ b/packages/models/src/plugins/plugin-index/peertube-plugin-index-list.model.ts @@ -0,0 +1,10 @@ +import { PluginType_Type } from '../plugin.type.js' + +export interface PeertubePluginIndexList { + start: number + count: number + sort: string + pluginType?: PluginType_Type + currentPeerTubeEngine?: string + search?: string +} diff --git a/shared/models/plugins/plugin-index/peertube-plugin-index.model.ts b/packages/models/src/plugins/plugin-index/peertube-plugin-index.model.ts similarity index 100% rename from shared/models/plugins/plugin-index/peertube-plugin-index.model.ts rename to packages/models/src/plugins/plugin-index/peertube-plugin-index.model.ts diff --git a/shared/models/plugins/plugin-index/peertube-plugin-latest-version.model.ts b/packages/models/src/plugins/plugin-index/peertube-plugin-latest-version.model.ts similarity index 100% rename from shared/models/plugins/plugin-index/peertube-plugin-latest-version.model.ts rename to packages/models/src/plugins/plugin-index/peertube-plugin-latest-version.model.ts diff --git a/packages/models/src/plugins/plugin-package-json.model.ts b/packages/models/src/plugins/plugin-package-json.model.ts new file mode 100644 index 000000000..5b9ccec56 --- /dev/null +++ b/packages/models/src/plugins/plugin-package-json.model.ts @@ -0,0 +1,29 @@ +import { PluginClientScope } from './client/plugin-client-scope.type.js' + +export type PluginTranslationPathsJSON = { + [ locale: string ]: string +} + +export type ClientScriptJSON = { + script: string + scopes: PluginClientScope[] +} + +export type PluginPackageJSON = { + name: string + version: string + description: string + engine: { peertube: string } + + homepage: string + author: string + bugs: string + library: string + + staticDirs: { [ name: string ]: string } + css: string[] + + clientScripts: ClientScriptJSON[] + + translations: PluginTranslationPathsJSON +} diff --git a/packages/models/src/plugins/plugin.type.ts b/packages/models/src/plugins/plugin.type.ts new file mode 100644 index 000000000..7d03012e6 --- /dev/null +++ b/packages/models/src/plugins/plugin.type.ts @@ -0,0 +1,6 @@ +export const PluginType = { + PLUGIN: 1, + THEME: 2 +} as const + +export type PluginType_Type = typeof PluginType[keyof typeof PluginType] diff --git a/packages/models/src/plugins/server/api/index.ts b/packages/models/src/plugins/server/api/index.ts new file mode 100644 index 000000000..1e3842c46 --- /dev/null +++ b/packages/models/src/plugins/server/api/index.ts @@ -0,0 +1,3 @@ +export * from './install-plugin.model.js' +export * from './manage-plugin.model.js' +export * from './peertube-plugin.model.js' diff --git a/shared/models/plugins/server/api/install-plugin.model.ts b/packages/models/src/plugins/server/api/install-plugin.model.ts similarity index 100% rename from shared/models/plugins/server/api/install-plugin.model.ts rename to packages/models/src/plugins/server/api/install-plugin.model.ts diff --git a/shared/models/plugins/server/api/manage-plugin.model.ts b/packages/models/src/plugins/server/api/manage-plugin.model.ts similarity index 100% rename from shared/models/plugins/server/api/manage-plugin.model.ts rename to packages/models/src/plugins/server/api/manage-plugin.model.ts diff --git a/packages/models/src/plugins/server/api/peertube-plugin.model.ts b/packages/models/src/plugins/server/api/peertube-plugin.model.ts new file mode 100644 index 000000000..0bc1b095b --- /dev/null +++ b/packages/models/src/plugins/server/api/peertube-plugin.model.ts @@ -0,0 +1,16 @@ +import { PluginType_Type } from '../../plugin.type.js' + +export interface PeerTubePlugin { + name: string + type: PluginType_Type + latestVersion: string + version: string + enabled: boolean + uninstalled: boolean + peertubeEngine: string + description: string + homepage: string + settings: { [ name: string ]: string } + createdAt: Date + updatedAt: Date +} diff --git a/packages/models/src/plugins/server/index.ts b/packages/models/src/plugins/server/index.ts new file mode 100644 index 000000000..04412318b --- /dev/null +++ b/packages/models/src/plugins/server/index.ts @@ -0,0 +1,7 @@ +export * from './api/index.js' +export * from './managers/index.js' +export * from './settings/index.js' +export * from './plugin-constant-manager.model.js' +export * from './plugin-translation.model.js' +export * from './register-server-hook.model.js' +export * from './server-hook.model.js' diff --git a/packages/models/src/plugins/server/managers/index.ts b/packages/models/src/plugins/server/managers/index.ts new file mode 100644 index 000000000..2433dd9bf --- /dev/null +++ b/packages/models/src/plugins/server/managers/index.ts @@ -0,0 +1,9 @@ + +export * from './plugin-playlist-privacy-manager.model.js' +export * from './plugin-settings-manager.model.js' +export * from './plugin-storage-manager.model.js' +export * from './plugin-transcoding-manager.model.js' +export * from './plugin-video-category-manager.model.js' +export * from './plugin-video-language-manager.model.js' +export * from './plugin-video-licence-manager.model.js' +export * from './plugin-video-privacy-manager.model.js' diff --git a/packages/models/src/plugins/server/managers/plugin-playlist-privacy-manager.model.ts b/packages/models/src/plugins/server/managers/plugin-playlist-privacy-manager.model.ts new file mode 100644 index 000000000..212c910c5 --- /dev/null +++ b/packages/models/src/plugins/server/managers/plugin-playlist-privacy-manager.model.ts @@ -0,0 +1,12 @@ +import { VideoPlaylistPrivacyType } from '../../../videos/playlist/video-playlist-privacy.model.js' +import { ConstantManager } from '../plugin-constant-manager.model.js' + +export interface PluginPlaylistPrivacyManager extends ConstantManager { + /** + * PUBLIC = 1, + * UNLISTED = 2, + * PRIVATE = 3 + * @deprecated use `deleteConstant` instead + */ + deletePlaylistPrivacy: (privacyKey: VideoPlaylistPrivacyType) => boolean +} diff --git a/shared/models/plugins/server/managers/plugin-settings-manager.model.ts b/packages/models/src/plugins/server/managers/plugin-settings-manager.model.ts similarity index 100% rename from shared/models/plugins/server/managers/plugin-settings-manager.model.ts rename to packages/models/src/plugins/server/managers/plugin-settings-manager.model.ts diff --git a/shared/models/plugins/server/managers/plugin-storage-manager.model.ts b/packages/models/src/plugins/server/managers/plugin-storage-manager.model.ts similarity index 100% rename from shared/models/plugins/server/managers/plugin-storage-manager.model.ts rename to packages/models/src/plugins/server/managers/plugin-storage-manager.model.ts diff --git a/packages/models/src/plugins/server/managers/plugin-transcoding-manager.model.ts b/packages/models/src/plugins/server/managers/plugin-transcoding-manager.model.ts new file mode 100644 index 000000000..235f5c65d --- /dev/null +++ b/packages/models/src/plugins/server/managers/plugin-transcoding-manager.model.ts @@ -0,0 +1,13 @@ +import { EncoderOptionsBuilder } from '../../../videos/transcoding/index.js' + +export interface PluginTranscodingManager { + addLiveProfile (encoder: string, profile: string, builder: EncoderOptionsBuilder): boolean + + addVODProfile (encoder: string, profile: string, builder: EncoderOptionsBuilder): boolean + + addLiveEncoderPriority (streamType: 'audio' | 'video', encoder: string, priority: number): void + + addVODEncoderPriority (streamType: 'audio' | 'video', encoder: string, priority: number): void + + removeAllProfilesAndEncoderPriorities(): void +} diff --git a/packages/models/src/plugins/server/managers/plugin-video-category-manager.model.ts b/packages/models/src/plugins/server/managers/plugin-video-category-manager.model.ts new file mode 100644 index 000000000..9da691e11 --- /dev/null +++ b/packages/models/src/plugins/server/managers/plugin-video-category-manager.model.ts @@ -0,0 +1,13 @@ +import { ConstantManager } from '../plugin-constant-manager.model.js' + +export interface PluginVideoCategoryManager extends ConstantManager { + /** + * @deprecated use `addConstant` instead + */ + addCategory: (categoryKey: number, categoryLabel: string) => boolean + + /** + * @deprecated use `deleteConstant` instead + */ + deleteCategory: (categoryKey: number) => boolean +} diff --git a/packages/models/src/plugins/server/managers/plugin-video-language-manager.model.ts b/packages/models/src/plugins/server/managers/plugin-video-language-manager.model.ts new file mode 100644 index 000000000..712486075 --- /dev/null +++ b/packages/models/src/plugins/server/managers/plugin-video-language-manager.model.ts @@ -0,0 +1,13 @@ +import { ConstantManager } from '../plugin-constant-manager.model.js' + +export interface PluginVideoLanguageManager extends ConstantManager { + /** + * @deprecated use `addConstant` instead + */ + addLanguage: (languageKey: string, languageLabel: string) => boolean + + /** + * @deprecated use `deleteConstant` instead + */ + deleteLanguage: (languageKey: string) => boolean +} diff --git a/packages/models/src/plugins/server/managers/plugin-video-licence-manager.model.ts b/packages/models/src/plugins/server/managers/plugin-video-licence-manager.model.ts new file mode 100644 index 000000000..cebae8d95 --- /dev/null +++ b/packages/models/src/plugins/server/managers/plugin-video-licence-manager.model.ts @@ -0,0 +1,13 @@ +import { ConstantManager } from '../plugin-constant-manager.model.js' + +export interface PluginVideoLicenceManager extends ConstantManager { + /** + * @deprecated use `addConstant` instead + */ + addLicence: (licenceKey: number, licenceLabel: string) => boolean + + /** + * @deprecated use `deleteConstant` instead + */ + deleteLicence: (licenceKey: number) => boolean +} diff --git a/packages/models/src/plugins/server/managers/plugin-video-privacy-manager.model.ts b/packages/models/src/plugins/server/managers/plugin-video-privacy-manager.model.ts new file mode 100644 index 000000000..260cee683 --- /dev/null +++ b/packages/models/src/plugins/server/managers/plugin-video-privacy-manager.model.ts @@ -0,0 +1,13 @@ +import { VideoPrivacyType } from '../../../videos/video-privacy.enum.js' +import { ConstantManager } from '../plugin-constant-manager.model.js' + +export interface PluginVideoPrivacyManager extends ConstantManager { + /** + * PUBLIC = 1, + * UNLISTED = 2, + * PRIVATE = 3 + * INTERNAL = 4 + * @deprecated use `deleteConstant` instead + */ + deletePrivacy: (privacyKey: VideoPrivacyType) => boolean +} diff --git a/shared/models/plugins/server/plugin-constant-manager.model.ts b/packages/models/src/plugins/server/plugin-constant-manager.model.ts similarity index 100% rename from shared/models/plugins/server/plugin-constant-manager.model.ts rename to packages/models/src/plugins/server/plugin-constant-manager.model.ts diff --git a/shared/models/plugins/server/plugin-translation.model.ts b/packages/models/src/plugins/server/plugin-translation.model.ts similarity index 100% rename from shared/models/plugins/server/plugin-translation.model.ts rename to packages/models/src/plugins/server/plugin-translation.model.ts diff --git a/packages/models/src/plugins/server/register-server-hook.model.ts b/packages/models/src/plugins/server/register-server-hook.model.ts new file mode 100644 index 000000000..05c883f1f --- /dev/null +++ b/packages/models/src/plugins/server/register-server-hook.model.ts @@ -0,0 +1,7 @@ +import { ServerHookName } from './server-hook.model.js' + +export interface RegisterServerHookOptions { + target: ServerHookName + handler: Function + priority?: number +} diff --git a/shared/models/plugins/server/server-hook.model.ts b/packages/models/src/plugins/server/server-hook.model.ts similarity index 100% rename from shared/models/plugins/server/server-hook.model.ts rename to packages/models/src/plugins/server/server-hook.model.ts diff --git a/packages/models/src/plugins/server/settings/index.ts b/packages/models/src/plugins/server/settings/index.ts new file mode 100644 index 000000000..4bdccaa4a --- /dev/null +++ b/packages/models/src/plugins/server/settings/index.ts @@ -0,0 +1,2 @@ +export * from './public-server.setting.js' +export * from './register-server-setting.model.js' diff --git a/packages/models/src/plugins/server/settings/public-server.setting.ts b/packages/models/src/plugins/server/settings/public-server.setting.ts new file mode 100644 index 000000000..0b6251aa3 --- /dev/null +++ b/packages/models/src/plugins/server/settings/public-server.setting.ts @@ -0,0 +1,5 @@ +import { SettingEntries } from '../managers/plugin-settings-manager.model.js' + +export interface PublicServerSetting { + publicSettings: SettingEntries +} diff --git a/packages/models/src/plugins/server/settings/register-server-setting.model.ts b/packages/models/src/plugins/server/settings/register-server-setting.model.ts new file mode 100644 index 000000000..8cde8eaaa --- /dev/null +++ b/packages/models/src/plugins/server/settings/register-server-setting.model.ts @@ -0,0 +1,12 @@ +import { RegisterClientFormFieldOptions } from '../../client/index.js' + +export type RegisterServerSettingOptions = RegisterClientFormFieldOptions & { + // If the setting is not private, anyone can view its value (client code included) + // If the setting is private, only server-side hooks can access it + // Mainly used by the PeerTube client to get admin config + private: boolean +} + +export interface RegisteredServerSettings { + registeredSettings: RegisterServerSettingOptions[] +} diff --git a/packages/models/src/redundancy/index.ts b/packages/models/src/redundancy/index.ts new file mode 100644 index 000000000..89e8fe464 --- /dev/null +++ b/packages/models/src/redundancy/index.ts @@ -0,0 +1,4 @@ +export * from './video-redundancies-filters.model.js' +export * from './video-redundancy-config-filter.type.js' +export * from './video-redundancy.model.js' +export * from './videos-redundancy-strategy.model.js' diff --git a/shared/models/redundancy/video-redundancies-filters.model.ts b/packages/models/src/redundancy/video-redundancies-filters.model.ts similarity index 100% rename from shared/models/redundancy/video-redundancies-filters.model.ts rename to packages/models/src/redundancy/video-redundancies-filters.model.ts diff --git a/shared/models/redundancy/video-redundancy-config-filter.type.ts b/packages/models/src/redundancy/video-redundancy-config-filter.type.ts similarity index 100% rename from shared/models/redundancy/video-redundancy-config-filter.type.ts rename to packages/models/src/redundancy/video-redundancy-config-filter.type.ts diff --git a/shared/models/redundancy/video-redundancy.model.ts b/packages/models/src/redundancy/video-redundancy.model.ts similarity index 100% rename from shared/models/redundancy/video-redundancy.model.ts rename to packages/models/src/redundancy/video-redundancy.model.ts diff --git a/shared/models/redundancy/videos-redundancy-strategy.model.ts b/packages/models/src/redundancy/videos-redundancy-strategy.model.ts similarity index 100% rename from shared/models/redundancy/videos-redundancy-strategy.model.ts rename to packages/models/src/redundancy/videos-redundancy-strategy.model.ts diff --git a/shared/models/runners/abort-runner-job-body.model.ts b/packages/models/src/runners/abort-runner-job-body.model.ts similarity index 100% rename from shared/models/runners/abort-runner-job-body.model.ts rename to packages/models/src/runners/abort-runner-job-body.model.ts diff --git a/shared/models/runners/accept-runner-job-body.model.ts b/packages/models/src/runners/accept-runner-job-body.model.ts similarity index 100% rename from shared/models/runners/accept-runner-job-body.model.ts rename to packages/models/src/runners/accept-runner-job-body.model.ts diff --git a/packages/models/src/runners/accept-runner-job-result.model.ts b/packages/models/src/runners/accept-runner-job-result.model.ts new file mode 100644 index 000000000..ebb605930 --- /dev/null +++ b/packages/models/src/runners/accept-runner-job-result.model.ts @@ -0,0 +1,6 @@ +import { RunnerJobPayload } from './runner-job-payload.model.js' +import { RunnerJob } from './runner-job.model.js' + +export interface AcceptRunnerJobResult { + job: RunnerJob & { jobToken: string } +} diff --git a/shared/models/runners/error-runner-job-body.model.ts b/packages/models/src/runners/error-runner-job-body.model.ts similarity index 100% rename from shared/models/runners/error-runner-job-body.model.ts rename to packages/models/src/runners/error-runner-job-body.model.ts diff --git a/packages/models/src/runners/index.ts b/packages/models/src/runners/index.ts new file mode 100644 index 000000000..cfe997b64 --- /dev/null +++ b/packages/models/src/runners/index.ts @@ -0,0 +1,21 @@ +export * from './abort-runner-job-body.model.js' +export * from './accept-runner-job-body.model.js' +export * from './accept-runner-job-result.model.js' +export * from './error-runner-job-body.model.js' +export * from './list-runner-jobs-query.model.js' +export * from './list-runner-registration-tokens.model.js' +export * from './list-runners-query.model.js' +export * from './register-runner-body.model.js' +export * from './register-runner-result.model.js' +export * from './request-runner-job-body.model.js' +export * from './request-runner-job-result.model.js' +export * from './runner-job-payload.model.js' +export * from './runner-job-private-payload.model.js' +export * from './runner-job-state.model.js' +export * from './runner-job-success-body.model.js' +export * from './runner-job-type.type.js' +export * from './runner-job-update-body.model.js' +export * from './runner-job.model.js' +export * from './runner-registration-token.js' +export * from './runner.model.js' +export * from './unregister-runner-body.model.js' diff --git a/packages/models/src/runners/list-runner-jobs-query.model.ts b/packages/models/src/runners/list-runner-jobs-query.model.ts new file mode 100644 index 000000000..395fe4b92 --- /dev/null +++ b/packages/models/src/runners/list-runner-jobs-query.model.ts @@ -0,0 +1,9 @@ +import { RunnerJobStateType } from './runner-job-state.model.js' + +export interface ListRunnerJobsQuery { + start?: number + count?: number + sort?: string + search?: string + stateOneOf?: RunnerJobStateType[] +} diff --git a/shared/models/runners/list-runner-registration-tokens.model.ts b/packages/models/src/runners/list-runner-registration-tokens.model.ts similarity index 100% rename from shared/models/runners/list-runner-registration-tokens.model.ts rename to packages/models/src/runners/list-runner-registration-tokens.model.ts diff --git a/shared/models/runners/list-runners-query.model.ts b/packages/models/src/runners/list-runners-query.model.ts similarity index 100% rename from shared/models/runners/list-runners-query.model.ts rename to packages/models/src/runners/list-runners-query.model.ts diff --git a/shared/models/runners/register-runner-body.model.ts b/packages/models/src/runners/register-runner-body.model.ts similarity index 100% rename from shared/models/runners/register-runner-body.model.ts rename to packages/models/src/runners/register-runner-body.model.ts diff --git a/shared/models/runners/register-runner-result.model.ts b/packages/models/src/runners/register-runner-result.model.ts similarity index 100% rename from shared/models/runners/register-runner-result.model.ts rename to packages/models/src/runners/register-runner-result.model.ts diff --git a/shared/models/runners/request-runner-job-body.model.ts b/packages/models/src/runners/request-runner-job-body.model.ts similarity index 100% rename from shared/models/runners/request-runner-job-body.model.ts rename to packages/models/src/runners/request-runner-job-body.model.ts diff --git a/packages/models/src/runners/request-runner-job-result.model.ts b/packages/models/src/runners/request-runner-job-result.model.ts new file mode 100644 index 000000000..30c8c640c --- /dev/null +++ b/packages/models/src/runners/request-runner-job-result.model.ts @@ -0,0 +1,10 @@ +import { RunnerJobPayload } from './runner-job-payload.model.js' +import { RunnerJobType } from './runner-job-type.type.js' + +export interface RequestRunnerJobResult

{ + availableJobs: { + uuid: string + type: RunnerJobType + payload: P + }[] +} diff --git a/packages/models/src/runners/runner-job-payload.model.ts b/packages/models/src/runners/runner-job-payload.model.ts new file mode 100644 index 000000000..19b9d649b --- /dev/null +++ b/packages/models/src/runners/runner-job-payload.model.ts @@ -0,0 +1,79 @@ +import { VideoStudioTaskPayload } from '../server/index.js' + +export type RunnerJobVODPayload = + RunnerJobVODWebVideoTranscodingPayload | + RunnerJobVODHLSTranscodingPayload | + RunnerJobVODAudioMergeTranscodingPayload + +export type RunnerJobPayload = + RunnerJobVODPayload | + RunnerJobLiveRTMPHLSTranscodingPayload | + RunnerJobStudioTranscodingPayload + +// --------------------------------------------------------------------------- + +export interface RunnerJobVODWebVideoTranscodingPayload { + input: { + videoFileUrl: string + } + + output: { + resolution: number + fps: number + } +} + +export interface RunnerJobVODHLSTranscodingPayload { + input: { + videoFileUrl: string + } + + output: { + resolution: number + fps: number + } +} + +export interface RunnerJobVODAudioMergeTranscodingPayload { + input: { + audioFileUrl: string + previewFileUrl: string + } + + output: { + resolution: number + fps: number + } +} + +export interface RunnerJobStudioTranscodingPayload { + input: { + videoFileUrl: string + } + + tasks: VideoStudioTaskPayload[] +} + +// --------------------------------------------------------------------------- + +export function isAudioMergeTranscodingPayload (payload: RunnerJobPayload): payload is RunnerJobVODAudioMergeTranscodingPayload { + return !!(payload as RunnerJobVODAudioMergeTranscodingPayload).input.audioFileUrl +} + +// --------------------------------------------------------------------------- + +export interface RunnerJobLiveRTMPHLSTranscodingPayload { + input: { + rtmpUrl: string + } + + output: { + toTranscode: { + resolution: number + fps: number + }[] + + segmentDuration: number + segmentListSize: number + } +} diff --git a/packages/models/src/runners/runner-job-private-payload.model.ts b/packages/models/src/runners/runner-job-private-payload.model.ts new file mode 100644 index 000000000..c1205984e --- /dev/null +++ b/packages/models/src/runners/runner-job-private-payload.model.ts @@ -0,0 +1,45 @@ +import { VideoStudioTaskPayload } from '../server/index.js' + +export type RunnerJobVODPrivatePayload = + RunnerJobVODWebVideoTranscodingPrivatePayload | + RunnerJobVODAudioMergeTranscodingPrivatePayload | + RunnerJobVODHLSTranscodingPrivatePayload + +export type RunnerJobPrivatePayload = + RunnerJobVODPrivatePayload | + RunnerJobLiveRTMPHLSTranscodingPrivatePayload | + RunnerJobVideoStudioTranscodingPrivatePayload + +// --------------------------------------------------------------------------- + +export interface RunnerJobVODWebVideoTranscodingPrivatePayload { + videoUUID: string + isNewVideo: boolean +} + +export interface RunnerJobVODAudioMergeTranscodingPrivatePayload { + videoUUID: string + isNewVideo: boolean +} + +export interface RunnerJobVODHLSTranscodingPrivatePayload { + videoUUID: string + isNewVideo: boolean + deleteWebVideoFiles: boolean +} + +// --------------------------------------------------------------------------- + +export interface RunnerJobLiveRTMPHLSTranscodingPrivatePayload { + videoUUID: string + masterPlaylistName: string + outputDirectory: string + sessionId: string +} + +// --------------------------------------------------------------------------- + +export interface RunnerJobVideoStudioTranscodingPrivatePayload { + videoUUID: string + originalTasks: VideoStudioTaskPayload[] +} diff --git a/packages/models/src/runners/runner-job-state.model.ts b/packages/models/src/runners/runner-job-state.model.ts new file mode 100644 index 000000000..07e135121 --- /dev/null +++ b/packages/models/src/runners/runner-job-state.model.ts @@ -0,0 +1,13 @@ +export const RunnerJobState = { + PENDING: 1, + PROCESSING: 2, + COMPLETED: 3, + ERRORED: 4, + WAITING_FOR_PARENT_JOB: 5, + CANCELLED: 6, + PARENT_ERRORED: 7, + PARENT_CANCELLED: 8, + COMPLETING: 9 +} as const + +export type RunnerJobStateType = typeof RunnerJobState[keyof typeof RunnerJobState] diff --git a/shared/models/runners/runner-job-success-body.model.ts b/packages/models/src/runners/runner-job-success-body.model.ts similarity index 100% rename from shared/models/runners/runner-job-success-body.model.ts rename to packages/models/src/runners/runner-job-success-body.model.ts diff --git a/shared/models/runners/runner-job-type.type.ts b/packages/models/src/runners/runner-job-type.type.ts similarity index 100% rename from shared/models/runners/runner-job-type.type.ts rename to packages/models/src/runners/runner-job-type.type.ts diff --git a/shared/models/runners/runner-job-update-body.model.ts b/packages/models/src/runners/runner-job-update-body.model.ts similarity index 100% rename from shared/models/runners/runner-job-update-body.model.ts rename to packages/models/src/runners/runner-job-update-body.model.ts diff --git a/packages/models/src/runners/runner-job.model.ts b/packages/models/src/runners/runner-job.model.ts new file mode 100644 index 000000000..6d6427396 --- /dev/null +++ b/packages/models/src/runners/runner-job.model.ts @@ -0,0 +1,45 @@ +import { VideoConstant } from '../videos/index.js' +import { RunnerJobPayload } from './runner-job-payload.model.js' +import { RunnerJobPrivatePayload } from './runner-job-private-payload.model.js' +import { RunnerJobStateType } from './runner-job-state.model.js' +import { RunnerJobType } from './runner-job-type.type.js' + +export interface RunnerJob { + uuid: string + + type: RunnerJobType + + state: VideoConstant + + payload: T + + failures: number + error: string | null + + progress: number + priority: number + + startedAt: Date | string + createdAt: Date | string + updatedAt: Date | string + finishedAt: Date | string + + parent?: { + type: RunnerJobType + state: VideoConstant + uuid: string + } + + // If associated to a runner + runner?: { + id: number + name: string + + description: string + } +} + +// eslint-disable-next-line max-len +export interface RunnerJobAdmin extends RunnerJob { + privatePayload: U +} diff --git a/shared/models/runners/runner-registration-token.ts b/packages/models/src/runners/runner-registration-token.ts similarity index 100% rename from shared/models/runners/runner-registration-token.ts rename to packages/models/src/runners/runner-registration-token.ts diff --git a/shared/models/runners/runner.model.ts b/packages/models/src/runners/runner.model.ts similarity index 100% rename from shared/models/runners/runner.model.ts rename to packages/models/src/runners/runner.model.ts diff --git a/shared/models/runners/unregister-runner-body.model.ts b/packages/models/src/runners/unregister-runner-body.model.ts similarity index 100% rename from shared/models/runners/unregister-runner-body.model.ts rename to packages/models/src/runners/unregister-runner-body.model.ts diff --git a/shared/models/search/boolean-both-query.model.ts b/packages/models/src/search/boolean-both-query.model.ts similarity index 100% rename from shared/models/search/boolean-both-query.model.ts rename to packages/models/src/search/boolean-both-query.model.ts diff --git a/packages/models/src/search/index.ts b/packages/models/src/search/index.ts new file mode 100644 index 000000000..5c4de1eea --- /dev/null +++ b/packages/models/src/search/index.ts @@ -0,0 +1,6 @@ +export * from './boolean-both-query.model.js' +export * from './search-target-query.model.js' +export * from './videos-common-query.model.js' +export * from './video-channels-search-query.model.js' +export * from './video-playlists-search-query.model.js' +export * from './videos-search-query.model.js' diff --git a/shared/models/search/search-target-query.model.ts b/packages/models/src/search/search-target-query.model.ts similarity index 100% rename from shared/models/search/search-target-query.model.ts rename to packages/models/src/search/search-target-query.model.ts diff --git a/packages/models/src/search/video-channels-search-query.model.ts b/packages/models/src/search/video-channels-search-query.model.ts new file mode 100644 index 000000000..7e84359cf --- /dev/null +++ b/packages/models/src/search/video-channels-search-query.model.ts @@ -0,0 +1,18 @@ +import { SearchTargetQuery } from './search-target-query.model.js' + +export interface VideoChannelsSearchQuery extends SearchTargetQuery { + search?: string + + start?: number + count?: number + sort?: string + + host?: string + handles?: string[] +} + +export interface VideoChannelsSearchQueryAfterSanitize extends VideoChannelsSearchQuery { + start: number + count: number + sort: string +} diff --git a/packages/models/src/search/video-playlists-search-query.model.ts b/packages/models/src/search/video-playlists-search-query.model.ts new file mode 100644 index 000000000..65ac7b4d7 --- /dev/null +++ b/packages/models/src/search/video-playlists-search-query.model.ts @@ -0,0 +1,20 @@ +import { SearchTargetQuery } from './search-target-query.model.js' + +export interface VideoPlaylistsSearchQuery extends SearchTargetQuery { + search?: string + + start?: number + count?: number + sort?: string + + host?: string + + // UUIDs or short UUIDs + uuids?: string[] +} + +export interface VideoPlaylistsSearchQueryAfterSanitize extends VideoPlaylistsSearchQuery { + start: number + count: number + sort: string +} diff --git a/packages/models/src/search/videos-common-query.model.ts b/packages/models/src/search/videos-common-query.model.ts new file mode 100644 index 000000000..45181a739 --- /dev/null +++ b/packages/models/src/search/videos-common-query.model.ts @@ -0,0 +1,45 @@ +import { VideoIncludeType } from '../videos/video-include.enum.js' +import { VideoPrivacyType } from '../videos/video-privacy.enum.js' +import { BooleanBothQuery } from './boolean-both-query.model.js' + +// These query parameters can be used with any endpoint that list videos +export interface VideosCommonQuery { + start?: number + count?: number + sort?: string + + nsfw?: BooleanBothQuery + + isLive?: boolean + + isLocal?: boolean + include?: VideoIncludeType + + categoryOneOf?: number[] + + licenceOneOf?: number[] + + languageOneOf?: string[] + + privacyOneOf?: VideoPrivacyType[] + + tagsOneOf?: string[] + tagsAllOf?: string[] + + hasHLSFiles?: boolean + + hasWebtorrentFiles?: boolean // TODO: remove in v7 + hasWebVideoFiles?: boolean + + skipCount?: boolean + + search?: string + + excludeAlreadyWatched?: boolean +} + +export interface VideosCommonQueryAfterSanitize extends VideosCommonQuery { + start: number + count: number + sort: string +} diff --git a/packages/models/src/search/videos-search-query.model.ts b/packages/models/src/search/videos-search-query.model.ts new file mode 100644 index 000000000..bbaa8d23f --- /dev/null +++ b/packages/models/src/search/videos-search-query.model.ts @@ -0,0 +1,26 @@ +import { SearchTargetQuery } from './search-target-query.model.js' +import { VideosCommonQuery } from './videos-common-query.model.js' + +export interface VideosSearchQuery extends SearchTargetQuery, VideosCommonQuery { + search?: string + + host?: string + + startDate?: string // ISO 8601 + endDate?: string // ISO 8601 + + originallyPublishedStartDate?: string // ISO 8601 + originallyPublishedEndDate?: string // ISO 8601 + + durationMin?: number // seconds + durationMax?: number // seconds + + // UUIDs or short UUIDs + uuids?: string[] +} + +export interface VideosSearchQueryAfterSanitize extends VideosSearchQuery { + start: number + count: number + sort: string +} diff --git a/shared/models/server/about.model.ts b/packages/models/src/server/about.model.ts similarity index 100% rename from shared/models/server/about.model.ts rename to packages/models/src/server/about.model.ts diff --git a/shared/models/server/broadcast-message-level.type.ts b/packages/models/src/server/broadcast-message-level.type.ts similarity index 100% rename from shared/models/server/broadcast-message-level.type.ts rename to packages/models/src/server/broadcast-message-level.type.ts diff --git a/packages/models/src/server/client-log-create.model.ts b/packages/models/src/server/client-log-create.model.ts new file mode 100644 index 000000000..543af0d3d --- /dev/null +++ b/packages/models/src/server/client-log-create.model.ts @@ -0,0 +1,11 @@ +import { ClientLogLevel } from './client-log-level.type.js' + +export interface ClientLogCreate { + message: string + url: string + level: ClientLogLevel + + stackTrace?: string + userAgent?: string + meta?: string +} diff --git a/shared/models/server/client-log-level.type.ts b/packages/models/src/server/client-log-level.type.ts similarity index 100% rename from shared/models/server/client-log-level.type.ts rename to packages/models/src/server/client-log-level.type.ts diff --git a/shared/models/server/contact-form.model.ts b/packages/models/src/server/contact-form.model.ts similarity index 100% rename from shared/models/server/contact-form.model.ts rename to packages/models/src/server/contact-form.model.ts diff --git a/packages/models/src/server/custom-config.model.ts b/packages/models/src/server/custom-config.model.ts new file mode 100644 index 000000000..df4176ba7 --- /dev/null +++ b/packages/models/src/server/custom-config.model.ts @@ -0,0 +1,259 @@ +import { NSFWPolicyType } from '../videos/nsfw-policy.type.js' +import { BroadcastMessageLevel } from './broadcast-message-level.type.js' + +export type ConfigResolutions = { + '144p': boolean + '240p': boolean + '360p': boolean + '480p': boolean + '720p': boolean + '1080p': boolean + '1440p': boolean + '2160p': boolean +} + +export interface CustomConfig { + instance: { + name: string + shortDescription: string + description: string + terms: string + codeOfConduct: string + + creationReason: string + moderationInformation: string + administrator: string + maintenanceLifetime: string + businessModel: string + hardwareInformation: string + + languages: string[] + categories: number[] + + isNSFW: boolean + defaultNSFWPolicy: NSFWPolicyType + + defaultClientRoute: string + + customizations: { + javascript?: string + css?: string + } + } + + theme: { + default: string + } + + services: { + twitter: { + username: string + whitelisted: boolean + } + } + + client: { + videos: { + miniature: { + preferAuthorDisplayName: boolean + } + } + + menu: { + login: { + redirectOnSingleExternalAuth: boolean + } + } + } + + cache: { + previews: { + size: number + } + + captions: { + size: number + } + + torrents: { + size: number + } + + storyboards: { + size: number + } + } + + signup: { + enabled: boolean + limit: number + requiresApproval: boolean + requiresEmailVerification: boolean + minimumAge: number + } + + admin: { + email: string + } + + contactForm: { + enabled: boolean + } + + user: { + history: { + videos: { + enabled: boolean + } + } + videoQuota: number + videoQuotaDaily: number + } + + videoChannels: { + maxPerUser: number + } + + transcoding: { + enabled: boolean + + allowAdditionalExtensions: boolean + allowAudioFiles: boolean + + remoteRunners: { + enabled: boolean + } + + threads: number + concurrency: number + + profile: string + + resolutions: ConfigResolutions & { '0p': boolean } + + alwaysTranscodeOriginalResolution: boolean + + webVideos: { + enabled: boolean + } + + hls: { + enabled: boolean + } + } + + live: { + enabled: boolean + + allowReplay: boolean + + latencySetting: { + enabled: boolean + } + + maxDuration: number + maxInstanceLives: number + maxUserLives: number + + transcoding: { + enabled: boolean + remoteRunners: { + enabled: boolean + } + threads: number + profile: string + resolutions: ConfigResolutions + alwaysTranscodeOriginalResolution: boolean + } + } + + videoStudio: { + enabled: boolean + + remoteRunners: { + enabled: boolean + } + } + + videoFile: { + update: { + enabled: boolean + } + } + + import: { + videos: { + concurrency: number + + http: { + enabled: boolean + } + torrent: { + enabled: boolean + } + } + videoChannelSynchronization: { + enabled: boolean + maxPerUser: number + } + } + + trending: { + videos: { + algorithms: { + enabled: string[] + default: string + } + } + } + + autoBlacklist: { + videos: { + ofUsers: { + enabled: boolean + } + } + } + + followers: { + instance: { + enabled: boolean + manualApproval: boolean + } + } + + followings: { + instance: { + autoFollowBack: { + enabled: boolean + } + + autoFollowIndex: { + enabled: boolean + indexUrl: string + } + } + } + + broadcastMessage: { + enabled: boolean + message: string + level: BroadcastMessageLevel + dismissable: boolean + } + + search: { + remoteUri: { + users: boolean + anonymous: boolean + } + + searchIndex: { + enabled: boolean + url: string + disableLocalSearch: boolean + isDefaultSearch: boolean + } + } + +} diff --git a/shared/models/server/debug.model.ts b/packages/models/src/server/debug.model.ts similarity index 100% rename from shared/models/server/debug.model.ts rename to packages/models/src/server/debug.model.ts diff --git a/shared/models/server/emailer.model.ts b/packages/models/src/server/emailer.model.ts similarity index 100% rename from shared/models/server/emailer.model.ts rename to packages/models/src/server/emailer.model.ts diff --git a/packages/models/src/server/index.ts b/packages/models/src/server/index.ts new file mode 100644 index 000000000..ba6af8f6f --- /dev/null +++ b/packages/models/src/server/index.ts @@ -0,0 +1,16 @@ +export * from './about.model.js' +export * from './broadcast-message-level.type.js' +export * from './client-log-create.model.js' +export * from './client-log-level.type.js' +export * from './contact-form.model.js' +export * from './custom-config.model.js' +export * from './debug.model.js' +export * from './emailer.model.js' +export * from './job.model.js' +export * from './peertube-problem-document.model.js' +export * from './server-config.model.js' +export * from './server-debug.model.js' +export * from './server-error-code.enum.js' +export * from './server-follow-create.model.js' +export * from './server-log-level.type.js' +export * from './server-stats.model.js' diff --git a/packages/models/src/server/job.model.ts b/packages/models/src/server/job.model.ts new file mode 100644 index 000000000..f86a20e28 --- /dev/null +++ b/packages/models/src/server/job.model.ts @@ -0,0 +1,303 @@ +import { ContextType } from '../activitypub/context.js' +import { VideoStateType } from '../videos/index.js' +import { VideoStudioTaskCut } from '../videos/studio/index.js' +import { SendEmailOptions } from './emailer.model.js' + +export type JobState = 'active' | 'completed' | 'failed' | 'waiting' | 'delayed' | 'paused' | 'waiting-children' + +export type JobType = + | 'activitypub-cleaner' + | 'activitypub-follow' + | 'activitypub-http-broadcast-parallel' + | 'activitypub-http-broadcast' + | 'activitypub-http-fetcher' + | 'activitypub-http-unicast' + | 'activitypub-refresher' + | 'actor-keys' + | 'after-video-channel-import' + | 'email' + | 'federate-video' + | 'transcoding-job-builder' + | 'manage-video-torrent' + | 'move-to-object-storage' + | 'notify' + | 'video-channel-import' + | 'video-file-import' + | 'video-import' + | 'video-live-ending' + | 'video-redundancy' + | 'video-studio-edition' + | 'video-transcoding' + | 'videos-views-stats' + | 'generate-video-storyboard' + +export interface Job { + id: number | string + state: JobState | 'unknown' + type: JobType + data: any + priority: number + progress: number + error: any + createdAt: Date | string + finishedOn: Date | string + processedOn: Date | string + + parent?: { + id: string + } +} + +export type ActivitypubHttpBroadcastPayload = { + uris: string[] + contextType: ContextType + body: any + signatureActorId?: number +} + +export type ActivitypubFollowPayload = { + followerActorId: number + name: string + host: string + isAutoFollow?: boolean + assertIsChannel?: boolean +} + +export type FetchType = 'activity' | 'video-shares' | 'video-comments' | 'account-playlists' +export type ActivitypubHttpFetcherPayload = { + uri: string + type: FetchType + videoId?: number +} + +export type ActivitypubHttpUnicastPayload = { + uri: string + contextType: ContextType + signatureActorId?: number + body: object +} + +export type RefreshPayload = { + type: 'video' | 'video-playlist' | 'actor' + url: string +} + +export type EmailPayload = SendEmailOptions + +export type VideoFileImportPayload = { + videoUUID: string + filePath: string +} + +// --------------------------------------------------------------------------- + +export type VideoImportTorrentPayloadType = 'magnet-uri' | 'torrent-file' +export type VideoImportYoutubeDLPayloadType = 'youtube-dl' + +export interface VideoImportYoutubeDLPayload { + type: VideoImportYoutubeDLPayloadType + videoImportId: number + + fileExt?: string +} + +export interface VideoImportTorrentPayload { + type: VideoImportTorrentPayloadType + videoImportId: number +} + +export type VideoImportPayload = (VideoImportYoutubeDLPayload | VideoImportTorrentPayload) & { + preventException: boolean +} + +export interface VideoImportPreventExceptionResult { + resultType: 'success' | 'error' +} + +// --------------------------------------------------------------------------- + +export type VideoRedundancyPayload = { + videoId: number +} + +export type ManageVideoTorrentPayload = + { + action: 'create' + videoId: number + videoFileId: number + } | { + action: 'update-metadata' + + videoId?: number + streamingPlaylistId?: number + + videoFileId: number + } + +// Video transcoding payloads + +interface BaseTranscodingPayload { + videoUUID: string + isNewVideo?: boolean +} + +export interface HLSTranscodingPayload extends BaseTranscodingPayload { + type: 'new-resolution-to-hls' + resolution: number + fps: number + copyCodecs: boolean + + deleteWebVideoFiles: boolean +} + +export interface NewWebVideoResolutionTranscodingPayload extends BaseTranscodingPayload { + type: 'new-resolution-to-web-video' + resolution: number + fps: number +} + +export interface MergeAudioTranscodingPayload extends BaseTranscodingPayload { + type: 'merge-audio-to-web-video' + + resolution: number + fps: number + + hasChildren: boolean +} + +export interface OptimizeTranscodingPayload extends BaseTranscodingPayload { + type: 'optimize-to-web-video' + + quickTranscode: boolean + + hasChildren: boolean +} + +export type VideoTranscodingPayload = + HLSTranscodingPayload + | NewWebVideoResolutionTranscodingPayload + | OptimizeTranscodingPayload + | MergeAudioTranscodingPayload + +export interface VideoLiveEndingPayload { + videoId: number + publishedAt: string + liveSessionId: number + streamingPlaylistId: number + + replayDirectory?: string +} + +export interface ActorKeysPayload { + actorId: number +} + +export interface DeleteResumableUploadMetaFilePayload { + filepath: string +} + +export interface MoveObjectStoragePayload { + videoUUID: string + isNewVideo: boolean + previousVideoState: VideoStateType +} + +export type VideoStudioTaskCutPayload = VideoStudioTaskCut + +export type VideoStudioTaskIntroPayload = { + name: 'add-intro' + + options: { + file: string + } +} + +export type VideoStudioTaskOutroPayload = { + name: 'add-outro' + + options: { + file: string + } +} + +export type VideoStudioTaskWatermarkPayload = { + name: 'add-watermark' + + options: { + file: string + + watermarkSizeRatio: number + horitonzalMarginRatio: number + verticalMarginRatio: number + } +} + +export type VideoStudioTaskPayload = + VideoStudioTaskCutPayload | + VideoStudioTaskIntroPayload | + VideoStudioTaskOutroPayload | + VideoStudioTaskWatermarkPayload + +export interface VideoStudioEditionPayload { + videoUUID: string + tasks: VideoStudioTaskPayload[] +} + +// --------------------------------------------------------------------------- + +export interface VideoChannelImportPayload { + externalChannelUrl: string + videoChannelId: number + + partOfChannelSyncId?: number +} + +export interface AfterVideoChannelImportPayload { + channelSyncId: number +} + +// --------------------------------------------------------------------------- + +export type NotifyPayload = + { + action: 'new-video' + videoUUID: string + } + +// --------------------------------------------------------------------------- + +export interface FederateVideoPayload { + videoUUID: string + isNewVideo: boolean +} + +// --------------------------------------------------------------------------- + +export interface TranscodingJobBuilderPayload { + videoUUID: string + + optimizeJob?: { + isNewVideo: boolean + } + + // Array of jobs to create + jobs?: { + type: 'video-transcoding' + payload: VideoTranscodingPayload + priority?: number + }[] + + // Array of sequential jobs to create + sequentialJobs?: { + type: 'video-transcoding' + payload: VideoTranscodingPayload + priority?: number + }[][] +} + +// --------------------------------------------------------------------------- + +export interface GenerateStoryboardPayload { + videoUUID: string + federate: boolean +} diff --git a/packages/models/src/server/peertube-problem-document.model.ts b/packages/models/src/server/peertube-problem-document.model.ts new file mode 100644 index 000000000..c717fc152 --- /dev/null +++ b/packages/models/src/server/peertube-problem-document.model.ts @@ -0,0 +1,32 @@ +import { HttpStatusCodeType } from '../http/http-status-codes.js' +import { OAuth2ErrorCodeType, ServerErrorCodeType } from './server-error-code.enum.js' + +export interface PeerTubeProblemDocumentData { + 'invalid-params'?: Record + + originUrl?: string + + keyId?: string + + targetUrl?: string + + actorUrl?: string + + // Feeds + format?: string + url?: string +} + +export interface PeerTubeProblemDocument extends PeerTubeProblemDocumentData { + type: string + title: string + + detail: string + // FIXME: Compat PeerTube <= 3.2 + error: string + + status: HttpStatusCodeType + + docs?: string + code?: OAuth2ErrorCodeType | ServerErrorCodeType +} diff --git a/packages/models/src/server/server-config.model.ts b/packages/models/src/server/server-config.model.ts new file mode 100644 index 000000000..a2a2bd5aa --- /dev/null +++ b/packages/models/src/server/server-config.model.ts @@ -0,0 +1,305 @@ +import { ClientScriptJSON } from '../plugins/plugin-package-json.model.js' +import { NSFWPolicyType } from '../videos/nsfw-policy.type.js' +import { VideoPrivacyType } from '../videos/video-privacy.enum.js' +import { BroadcastMessageLevel } from './broadcast-message-level.type.js' + +export interface ServerConfigPlugin { + name: string + npmName: string + version: string + description: string + clientScripts: { [name: string]: ClientScriptJSON } +} + +export interface ServerConfigTheme extends ServerConfigPlugin { + css: string[] +} + +export interface RegisteredExternalAuthConfig { + npmName: string + name: string + version: string + authName: string + authDisplayName: string +} + +export interface RegisteredIdAndPassAuthConfig { + npmName: string + name: string + version: string + authName: string + weight: number +} + +export interface ServerConfig { + serverVersion: string + serverCommit?: string + + client: { + videos: { + miniature: { + displayAuthorAvatar: boolean + preferAuthorDisplayName: boolean + } + resumableUpload: { + maxChunkSize: number + } + } + + menu: { + login: { + redirectOnSingleExternalAuth: boolean + } + } + } + + defaults: { + publish: { + downloadEnabled: boolean + commentsEnabled: boolean + privacy: VideoPrivacyType + licence: number + } + + p2p: { + webapp: { + enabled: boolean + } + + embed: { + enabled: boolean + } + } + } + + webadmin: { + configuration: { + edition: { + allowed: boolean + } + } + } + + instance: { + name: string + shortDescription: string + isNSFW: boolean + defaultNSFWPolicy: NSFWPolicyType + defaultClientRoute: string + customizations: { + javascript: string + css: string + } + } + + search: { + remoteUri: { + users: boolean + anonymous: boolean + } + + searchIndex: { + enabled: boolean + url: string + disableLocalSearch: boolean + isDefaultSearch: boolean + } + } + + plugin: { + registered: ServerConfigPlugin[] + + registeredExternalAuths: RegisteredExternalAuthConfig[] + + registeredIdAndPassAuths: RegisteredIdAndPassAuthConfig[] + } + + theme: { + registered: ServerConfigTheme[] + default: string + } + + email: { + enabled: boolean + } + + contactForm: { + enabled: boolean + } + + signup: { + allowed: boolean + allowedForCurrentIP: boolean + requiresEmailVerification: boolean + requiresApproval: boolean + minimumAge: number + } + + transcoding: { + hls: { + enabled: boolean + } + + web_videos: { + enabled: boolean + } + + enabledResolutions: number[] + + profile: string + availableProfiles: string[] + + remoteRunners: { + enabled: boolean + } + } + + live: { + enabled: boolean + + allowReplay: boolean + latencySetting: { + enabled: boolean + } + + maxDuration: number + maxInstanceLives: number + maxUserLives: number + + transcoding: { + enabled: boolean + + remoteRunners: { + enabled: boolean + } + + enabledResolutions: number[] + + profile: string + availableProfiles: string[] + } + + rtmp: { + port: number + } + } + + videoStudio: { + enabled: boolean + + remoteRunners: { + enabled: boolean + } + } + + videoFile: { + update: { + enabled: boolean + } + } + + import: { + videos: { + http: { + enabled: boolean + } + torrent: { + enabled: boolean + } + } + videoChannelSynchronization: { + enabled: boolean + } + } + + autoBlacklist: { + videos: { + ofUsers: { + enabled: boolean + } + } + } + + avatar: { + file: { + size: { + max: number + } + extensions: string[] + } + } + + banner: { + file: { + size: { + max: number + } + extensions: string[] + } + } + + video: { + image: { + size: { + max: number + } + extensions: string[] + } + file: { + extensions: string[] + } + } + + videoCaption: { + file: { + size: { + max: number + } + extensions: string[] + } + } + + user: { + videoQuota: number + videoQuotaDaily: number + } + + videoChannels: { + maxPerUser: number + } + + trending: { + videos: { + intervalDays: number + algorithms: { + enabled: string[] + default: string + } + } + } + + tracker: { + enabled: boolean + } + + followings: { + instance: { + autoFollowIndex: { + indexUrl: string + } + } + } + + broadcastMessage: { + enabled: boolean + message: string + level: BroadcastMessageLevel + dismissable: boolean + } + + homepage: { + enabled: boolean + } +} + +export type HTMLServerConfig = Omit diff --git a/shared/models/server/server-debug.model.ts b/packages/models/src/server/server-debug.model.ts similarity index 100% rename from shared/models/server/server-debug.model.ts rename to packages/models/src/server/server-debug.model.ts diff --git a/packages/models/src/server/server-error-code.enum.ts b/packages/models/src/server/server-error-code.enum.ts new file mode 100644 index 000000000..dc200c1ea --- /dev/null +++ b/packages/models/src/server/server-error-code.enum.ts @@ -0,0 +1,92 @@ +export const ServerErrorCode = { + /** + * The simplest form of payload too large: when the file size is over the + * global file size limit + */ + MAX_FILE_SIZE_REACHED:'max_file_size_reached', + + /** + * The payload is too large for the user quota set + */ + QUOTA_REACHED:'quota_reached', + + /** + * Error yielded upon trying to access a video that is not federated, nor can + * be. This may be due to: remote videos on instances that are not followed by + * yours, and with your instance disallowing unknown instances being accessed. + */ + DOES_NOT_RESPECT_FOLLOW_CONSTRAINTS:'does_not_respect_follow_constraints', + + LIVE_NOT_ENABLED:'live_not_enabled', + LIVE_NOT_ALLOWING_REPLAY:'live_not_allowing_replay', + LIVE_CONFLICTING_PERMANENT_AND_SAVE_REPLAY:'live_conflicting_permanent_and_save_replay', + /** + * Pretty self-explanatory: the set maximum number of simultaneous lives was + * reached, and this error is typically there to inform the user trying to + * broadcast one. + */ + MAX_INSTANCE_LIVES_LIMIT_REACHED:'max_instance_lives_limit_reached', + /** + * Pretty self-explanatory: the set maximum number of simultaneous lives FOR + * THIS USER was reached, and this error is typically there to inform the user + * trying to broadcast one. + */ + MAX_USER_LIVES_LIMIT_REACHED:'max_user_lives_limit_reached', + + /** + * A torrent should have at most one correct video file. Any more and we will + * not be able to choose automatically. + */ + INCORRECT_FILES_IN_TORRENT:'incorrect_files_in_torrent', + + COMMENT_NOT_ASSOCIATED_TO_VIDEO:'comment_not_associated_to_video', + + MISSING_TWO_FACTOR:'missing_two_factor', + INVALID_TWO_FACTOR:'invalid_two_factor', + + ACCOUNT_WAITING_FOR_APPROVAL:'account_waiting_for_approval', + ACCOUNT_APPROVAL_REJECTED:'account_approval_rejected', + + RUNNER_JOB_NOT_IN_PROCESSING_STATE:'runner_job_not_in_processing_state', + RUNNER_JOB_NOT_IN_PENDING_STATE:'runner_job_not_in_pending_state', + UNKNOWN_RUNNER_TOKEN:'unknown_runner_token', + + VIDEO_REQUIRES_PASSWORD:'video_requires_password', + INCORRECT_VIDEO_PASSWORD:'incorrect_video_password', + + VIDEO_ALREADY_BEING_TRANSCODED:'video_already_being_transcoded' +} as const + +/** + * oauthjs/oauth2-server error codes + * @see https://datatracker.ietf.org/doc/html/rfc6749#section-5.2 + **/ +export const OAuth2ErrorCode = { + /** + * The provided authorization grant (e.g., authorization code, resource owner + * credentials) or refresh token is invalid, expired, revoked, does not match + * the redirection URI used in the authorization request, or was issued to + * another client. + * + * @see https://github.com/oauthjs/node-oauth2-server/blob/master/lib/errors/invalid-grant-error.js + */ + INVALID_GRANT: 'invalid_grant', + + /** + * Client authentication failed (e.g., unknown client, no client authentication + * included, or unsupported authentication method). + * + * @see https://github.com/oauthjs/node-oauth2-server/blob/master/lib/errors/invalid-client-error.js + */ + INVALID_CLIENT: 'invalid_client', + + /** + * The access token provided is expired, revoked, malformed, or invalid for other reasons + * + * @see https://github.com/oauthjs/node-oauth2-server/blob/master/lib/errors/invalid-token-error.js + */ + INVALID_TOKEN: 'invalid_token' +} as const + +export type OAuth2ErrorCodeType = typeof OAuth2ErrorCode[keyof typeof OAuth2ErrorCode] +export type ServerErrorCodeType = typeof ServerErrorCode[keyof typeof ServerErrorCode] diff --git a/shared/models/server/server-follow-create.model.ts b/packages/models/src/server/server-follow-create.model.ts similarity index 100% rename from shared/models/server/server-follow-create.model.ts rename to packages/models/src/server/server-follow-create.model.ts diff --git a/shared/models/server/server-log-level.type.ts b/packages/models/src/server/server-log-level.type.ts similarity index 100% rename from shared/models/server/server-log-level.type.ts rename to packages/models/src/server/server-log-level.type.ts diff --git a/packages/models/src/server/server-stats.model.ts b/packages/models/src/server/server-stats.model.ts new file mode 100644 index 000000000..5870ee73d --- /dev/null +++ b/packages/models/src/server/server-stats.model.ts @@ -0,0 +1,47 @@ +import { ActivityType } from '../activitypub/index.js' +import { VideoRedundancyStrategyWithManual } from '../redundancy/index.js' + +type ActivityPubMessagesSuccess = Record<`totalActivityPub${ActivityType}MessagesSuccesses`, number> +type ActivityPubMessagesErrors = Record<`totalActivityPub${ActivityType}MessagesErrors`, number> + +export interface ServerStats extends ActivityPubMessagesSuccess, ActivityPubMessagesErrors { + totalUsers: number + totalDailyActiveUsers: number + totalWeeklyActiveUsers: number + totalMonthlyActiveUsers: number + + totalLocalVideos: number + totalLocalVideoViews: number + totalLocalVideoComments: number + totalLocalVideoFilesSize: number + + totalVideos: number + totalVideoComments: number + + totalLocalVideoChannels: number + totalLocalDailyActiveVideoChannels: number + totalLocalWeeklyActiveVideoChannels: number + totalLocalMonthlyActiveVideoChannels: number + + totalLocalPlaylists: number + + totalInstanceFollowers: number + totalInstanceFollowing: number + + videosRedundancy: VideosRedundancyStats[] + + totalActivityPubMessagesProcessed: number + totalActivityPubMessagesSuccesses: number + totalActivityPubMessagesErrors: number + + activityPubMessagesProcessedPerSecond: number + totalActivityPubMessagesWaiting: number +} + +export interface VideosRedundancyStats { + strategy: VideoRedundancyStrategyWithManual + totalSize: number + totalUsed: number + totalVideoFiles: number + totalVideos: number +} diff --git a/packages/models/src/tokens/index.ts b/packages/models/src/tokens/index.ts new file mode 100644 index 000000000..db2d63d21 --- /dev/null +++ b/packages/models/src/tokens/index.ts @@ -0,0 +1 @@ +export * from './oauth-client-local.model.js' diff --git a/shared/models/tokens/oauth-client-local.model.ts b/packages/models/src/tokens/oauth-client-local.model.ts similarity index 100% rename from shared/models/tokens/oauth-client-local.model.ts rename to packages/models/src/tokens/oauth-client-local.model.ts diff --git a/packages/models/src/users/index.ts b/packages/models/src/users/index.ts new file mode 100644 index 000000000..6f5218234 --- /dev/null +++ b/packages/models/src/users/index.ts @@ -0,0 +1,16 @@ +export * from './registration/index.js' +export * from './two-factor-enable-result.model.js' +export * from './user-create-result.model.js' +export * from './user-create.model.js' +export * from './user-flag.model.js' +export * from './user-login.model.js' +export * from './user-notification-setting.model.js' +export * from './user-notification.model.js' +export * from './user-refresh-token.model.js' +export * from './user-right.enum.js' +export * from './user-role.js' +export * from './user-scoped-token.js' +export * from './user-update-me.model.js' +export * from './user-update.model.js' +export * from './user-video-quota.model.js' +export * from './user.model.js' diff --git a/packages/models/src/users/registration/index.ts b/packages/models/src/users/registration/index.ts new file mode 100644 index 000000000..dcf16ef9d --- /dev/null +++ b/packages/models/src/users/registration/index.ts @@ -0,0 +1,5 @@ +export * from './user-register.model.js' +export * from './user-registration-request.model.js' +export * from './user-registration-state.model.js' +export * from './user-registration-update-state.model.js' +export * from './user-registration.model.js' diff --git a/shared/models/users/registration/user-register.model.ts b/packages/models/src/users/registration/user-register.model.ts similarity index 100% rename from shared/models/users/registration/user-register.model.ts rename to packages/models/src/users/registration/user-register.model.ts diff --git a/packages/models/src/users/registration/user-registration-request.model.ts b/packages/models/src/users/registration/user-registration-request.model.ts new file mode 100644 index 000000000..ed369f96a --- /dev/null +++ b/packages/models/src/users/registration/user-registration-request.model.ts @@ -0,0 +1,5 @@ +import { UserRegister } from './user-register.model.js' + +export interface UserRegistrationRequest extends UserRegister { + registrationReason: string +} diff --git a/packages/models/src/users/registration/user-registration-state.model.ts b/packages/models/src/users/registration/user-registration-state.model.ts new file mode 100644 index 000000000..7c51f3f9d --- /dev/null +++ b/packages/models/src/users/registration/user-registration-state.model.ts @@ -0,0 +1,7 @@ +export const UserRegistrationState = { + PENDING: 1, + REJECTED: 2, + ACCEPTED: 3 +} + +export type UserRegistrationStateType = typeof UserRegistrationState[keyof typeof UserRegistrationState] diff --git a/shared/models/users/registration/user-registration-update-state.model.ts b/packages/models/src/users/registration/user-registration-update-state.model.ts similarity index 100% rename from shared/models/users/registration/user-registration-update-state.model.ts rename to packages/models/src/users/registration/user-registration-update-state.model.ts diff --git a/packages/models/src/users/registration/user-registration.model.ts b/packages/models/src/users/registration/user-registration.model.ts new file mode 100644 index 000000000..0d01add36 --- /dev/null +++ b/packages/models/src/users/registration/user-registration.model.ts @@ -0,0 +1,29 @@ +import { UserRegistrationStateType } from './user-registration-state.model.js' + +export interface UserRegistration { + id: number + + state: { + id: UserRegistrationStateType + label: string + } + + registrationReason: string + moderationResponse: string + + username: string + email: string + emailVerified: boolean + + accountDisplayName: string + + channelHandle: string + channelDisplayName: string + + createdAt: Date + updatedAt: Date + + user?: { + id: number + } +} diff --git a/shared/models/users/two-factor-enable-result.model.ts b/packages/models/src/users/two-factor-enable-result.model.ts similarity index 100% rename from shared/models/users/two-factor-enable-result.model.ts rename to packages/models/src/users/two-factor-enable-result.model.ts diff --git a/shared/models/users/user-create-result.model.ts b/packages/models/src/users/user-create-result.model.ts similarity index 100% rename from shared/models/users/user-create-result.model.ts rename to packages/models/src/users/user-create-result.model.ts diff --git a/packages/models/src/users/user-create.model.ts b/packages/models/src/users/user-create.model.ts new file mode 100644 index 000000000..b62cf692f --- /dev/null +++ b/packages/models/src/users/user-create.model.ts @@ -0,0 +1,13 @@ +import { UserAdminFlagType } from './user-flag.model.js' +import { UserRoleType } from './user-role.js' + +export interface UserCreate { + username: string + password: string + email: string + videoQuota: number + videoQuotaDaily: number + role: UserRoleType + adminFlags?: UserAdminFlagType + channelName?: string +} diff --git a/packages/models/src/users/user-flag.model.ts b/packages/models/src/users/user-flag.model.ts new file mode 100644 index 000000000..0ecbacecc --- /dev/null +++ b/packages/models/src/users/user-flag.model.ts @@ -0,0 +1,6 @@ +export const UserAdminFlag = { + NONE: 0, + BYPASS_VIDEO_AUTO_BLACKLIST: 1 << 0 +} as const + +export type UserAdminFlagType = typeof UserAdminFlag[keyof typeof UserAdminFlag] diff --git a/shared/models/users/user-login.model.ts b/packages/models/src/users/user-login.model.ts similarity index 100% rename from shared/models/users/user-login.model.ts rename to packages/models/src/users/user-login.model.ts diff --git a/packages/models/src/users/user-notification-setting.model.ts b/packages/models/src/users/user-notification-setting.model.ts new file mode 100644 index 000000000..fbd94994e --- /dev/null +++ b/packages/models/src/users/user-notification-setting.model.ts @@ -0,0 +1,34 @@ +export const UserNotificationSettingValue = { + NONE: 0, + WEB: 1 << 0, + EMAIL: 1 << 1 +} as const + +export type UserNotificationSettingValueType = typeof UserNotificationSettingValue[keyof typeof UserNotificationSettingValue] + +export interface UserNotificationSetting { + abuseAsModerator: UserNotificationSettingValueType + videoAutoBlacklistAsModerator: UserNotificationSettingValueType + newUserRegistration: UserNotificationSettingValueType + + newVideoFromSubscription: UserNotificationSettingValueType + + blacklistOnMyVideo: UserNotificationSettingValueType + myVideoPublished: UserNotificationSettingValueType + myVideoImportFinished: UserNotificationSettingValueType + + commentMention: UserNotificationSettingValueType + newCommentOnMyVideo: UserNotificationSettingValueType + + newFollow: UserNotificationSettingValueType + newInstanceFollower: UserNotificationSettingValueType + autoInstanceFollowing: UserNotificationSettingValueType + + abuseStateChange: UserNotificationSettingValueType + abuseNewMessage: UserNotificationSettingValueType + + newPeerTubeVersion: UserNotificationSettingValueType + newPluginVersion: UserNotificationSettingValueType + + myVideoStudioEditionFinished: UserNotificationSettingValueType +} diff --git a/packages/models/src/users/user-notification.model.ts b/packages/models/src/users/user-notification.model.ts new file mode 100644 index 000000000..991fe6728 --- /dev/null +++ b/packages/models/src/users/user-notification.model.ts @@ -0,0 +1,140 @@ +import { FollowState } from '../actors/index.js' +import { AbuseStateType } from '../moderation/index.js' +import { PluginType_Type } from '../plugins/index.js' + +export const UserNotificationType = { + NEW_VIDEO_FROM_SUBSCRIPTION: 1, + NEW_COMMENT_ON_MY_VIDEO: 2, + NEW_ABUSE_FOR_MODERATORS: 3, + + BLACKLIST_ON_MY_VIDEO: 4, + UNBLACKLIST_ON_MY_VIDEO: 5, + + MY_VIDEO_PUBLISHED: 6, + + MY_VIDEO_IMPORT_SUCCESS: 7, + MY_VIDEO_IMPORT_ERROR: 8, + + NEW_USER_REGISTRATION: 9, + NEW_FOLLOW: 10, + COMMENT_MENTION: 11, + + VIDEO_AUTO_BLACKLIST_FOR_MODERATORS: 12, + + NEW_INSTANCE_FOLLOWER: 13, + + AUTO_INSTANCE_FOLLOWING: 14, + + ABUSE_STATE_CHANGE: 15, + + ABUSE_NEW_MESSAGE: 16, + + NEW_PLUGIN_VERSION: 17, + NEW_PEERTUBE_VERSION: 18, + + MY_VIDEO_STUDIO_EDITION_FINISHED: 19, + + NEW_USER_REGISTRATION_REQUEST: 20 +} as const + +export type UserNotificationType_Type = typeof UserNotificationType[keyof typeof UserNotificationType] + +export interface VideoInfo { + id: number + uuid: string + shortUUID: string + name: string +} + +export interface AvatarInfo { + width: number + path: string +} + +export interface ActorInfo { + id: number + displayName: string + name: string + host: string + + avatars: AvatarInfo[] + avatar: AvatarInfo +} + +export interface UserNotification { + id: number + type: UserNotificationType_Type + read: boolean + + video?: VideoInfo & { + channel: ActorInfo + } + + videoImport?: { + id: number + video?: VideoInfo + torrentName?: string + magnetUri?: string + targetUrl?: string + } + + comment?: { + id: number + threadId: number + account: ActorInfo + video: VideoInfo + } + + abuse?: { + id: number + state: AbuseStateType + + video?: VideoInfo + + comment?: { + threadId: number + + video: VideoInfo + } + + account?: ActorInfo + } + + videoBlacklist?: { + id: number + video: VideoInfo + } + + account?: ActorInfo + + actorFollow?: { + id: number + follower: ActorInfo + state: FollowState + + following: { + type: 'account' | 'channel' | 'instance' + name: string + displayName: string + host: string + } + } + + plugin?: { + name: string + type: PluginType_Type + latestVersion: string + } + + peertube?: { + latestVersion: string + } + + registration?: { + id: number + username: string + } + + createdAt: string + updatedAt: string +} diff --git a/shared/models/users/user-refresh-token.model.ts b/packages/models/src/users/user-refresh-token.model.ts similarity index 100% rename from shared/models/users/user-refresh-token.model.ts rename to packages/models/src/users/user-refresh-token.model.ts diff --git a/packages/models/src/users/user-right.enum.ts b/packages/models/src/users/user-right.enum.ts new file mode 100644 index 000000000..534b9feb0 --- /dev/null +++ b/packages/models/src/users/user-right.enum.ts @@ -0,0 +1,53 @@ +export const UserRight = { + ALL: 0, + + MANAGE_USERS: 1, + + MANAGE_SERVER_FOLLOW: 2, + + MANAGE_LOGS: 3, + + MANAGE_DEBUG: 4, + + MANAGE_SERVER_REDUNDANCY: 5, + + MANAGE_ABUSES: 6, + + MANAGE_JOBS: 7, + + MANAGE_CONFIGURATION: 8, + MANAGE_INSTANCE_CUSTOM_PAGE: 9, + + MANAGE_ACCOUNTS_BLOCKLIST: 10, + MANAGE_SERVERS_BLOCKLIST: 11, + + MANAGE_VIDEO_BLACKLIST: 12, + MANAGE_ANY_VIDEO_CHANNEL: 13, + + REMOVE_ANY_VIDEO: 14, + REMOVE_ANY_VIDEO_PLAYLIST: 15, + REMOVE_ANY_VIDEO_COMMENT: 16, + + UPDATE_ANY_VIDEO: 17, + UPDATE_ANY_VIDEO_PLAYLIST: 18, + + GET_ANY_LIVE: 19, + SEE_ALL_VIDEOS: 20, + SEE_ALL_COMMENTS: 21, + CHANGE_VIDEO_OWNERSHIP: 22, + + MANAGE_PLUGINS: 23, + + MANAGE_VIDEOS_REDUNDANCIES: 24, + + MANAGE_VIDEO_FILES: 25, + RUN_VIDEO_TRANSCODING: 26, + + MANAGE_VIDEO_IMPORTS: 27, + + MANAGE_REGISTRATIONS: 28, + + MANAGE_RUNNERS: 29 +} as const + +export type UserRightType = typeof UserRight[keyof typeof UserRight] diff --git a/packages/models/src/users/user-role.ts b/packages/models/src/users/user-role.ts new file mode 100644 index 000000000..b496f8153 --- /dev/null +++ b/packages/models/src/users/user-role.ts @@ -0,0 +1,8 @@ +// Always keep this order to prevent security issue since we store these values in the database +export const UserRole = { + ADMINISTRATOR: 0, + MODERATOR: 1, + USER: 2 +} as const + +export type UserRoleType = typeof UserRole[keyof typeof UserRole] diff --git a/shared/models/users/user-scoped-token.ts b/packages/models/src/users/user-scoped-token.ts similarity index 100% rename from shared/models/users/user-scoped-token.ts rename to packages/models/src/users/user-scoped-token.ts diff --git a/packages/models/src/users/user-update-me.model.ts b/packages/models/src/users/user-update-me.model.ts new file mode 100644 index 000000000..ba9672136 --- /dev/null +++ b/packages/models/src/users/user-update-me.model.ts @@ -0,0 +1,26 @@ +import { NSFWPolicyType } from '../videos/nsfw-policy.type.js' + +export interface UserUpdateMe { + displayName?: string + description?: string + nsfwPolicy?: NSFWPolicyType + + p2pEnabled?: boolean + + autoPlayVideo?: boolean + autoPlayNextVideo?: boolean + autoPlayNextVideoPlaylist?: boolean + videosHistoryEnabled?: boolean + videoLanguages?: string[] + + email?: string + emailPublic?: boolean + currentPassword?: string + password?: string + + theme?: string + + noInstanceConfigWarningModal?: boolean + noWelcomeModal?: boolean + noAccountSetupWarningModal?: boolean +} diff --git a/packages/models/src/users/user-update.model.ts b/packages/models/src/users/user-update.model.ts new file mode 100644 index 000000000..283255629 --- /dev/null +++ b/packages/models/src/users/user-update.model.ts @@ -0,0 +1,13 @@ +import { UserAdminFlagType } from './user-flag.model.js' +import { UserRoleType } from './user-role.js' + +export interface UserUpdate { + password?: string + email?: string + emailVerified?: boolean + videoQuota?: number + videoQuotaDaily?: number + role?: UserRoleType + adminFlags?: UserAdminFlagType + pluginAuth?: string +} diff --git a/shared/models/users/user-video-quota.model.ts b/packages/models/src/users/user-video-quota.model.ts similarity index 100% rename from shared/models/users/user-video-quota.model.ts rename to packages/models/src/users/user-video-quota.model.ts diff --git a/packages/models/src/users/user.model.ts b/packages/models/src/users/user.model.ts new file mode 100644 index 000000000..57b4c1aab --- /dev/null +++ b/packages/models/src/users/user.model.ts @@ -0,0 +1,78 @@ +import { Account } from '../actors/index.js' +import { VideoChannel } from '../videos/channel/video-channel.model.js' +import { NSFWPolicyType } from '../videos/nsfw-policy.type.js' +import { VideoPlaylistType_Type } from '../videos/playlist/video-playlist-type.model.js' +import { UserAdminFlagType } from './user-flag.model.js' +import { UserNotificationSetting } from './user-notification-setting.model.js' +import { UserRoleType } from './user-role.js' + +export interface User { + id: number + username: string + email: string + pendingEmail: string | null + + emailVerified: boolean + emailPublic: boolean + nsfwPolicy: NSFWPolicyType + + adminFlags?: UserAdminFlagType + + autoPlayVideo: boolean + autoPlayNextVideo: boolean + autoPlayNextVideoPlaylist: boolean + + p2pEnabled: boolean + + videosHistoryEnabled: boolean + videoLanguages: string[] + + role: { + id: UserRoleType + label: string + } + + videoQuota: number + videoQuotaDaily: number + videoQuotaUsed?: number + videoQuotaUsedDaily?: number + + videosCount?: number + + abusesCount?: number + abusesAcceptedCount?: number + abusesCreatedCount?: number + + videoCommentsCount?: number + + theme: string + + account: Account + notificationSettings?: UserNotificationSetting + videoChannels?: VideoChannel[] + + blocked: boolean + blockedReason?: string + + noInstanceConfigWarningModal: boolean + noWelcomeModal: boolean + noAccountSetupWarningModal: boolean + + createdAt: Date + + pluginAuth: string | null + + lastLoginDate: Date | null + + twoFactorEnabled: boolean +} + +export interface MyUserSpecialPlaylist { + id: number + name: string + type: VideoPlaylistType_Type +} + +export interface MyUser extends User { + specialPlaylists: MyUserSpecialPlaylist[] +} diff --git a/packages/models/src/videos/blacklist/index.ts b/packages/models/src/videos/blacklist/index.ts new file mode 100644 index 000000000..5eb36ad48 --- /dev/null +++ b/packages/models/src/videos/blacklist/index.ts @@ -0,0 +1,3 @@ +export * from './video-blacklist.model.js' +export * from './video-blacklist-create.model.js' +export * from './video-blacklist-update.model.js' diff --git a/shared/models/videos/blacklist/video-blacklist-create.model.ts b/packages/models/src/videos/blacklist/video-blacklist-create.model.ts similarity index 100% rename from shared/models/videos/blacklist/video-blacklist-create.model.ts rename to packages/models/src/videos/blacklist/video-blacklist-create.model.ts diff --git a/shared/models/videos/blacklist/video-blacklist-update.model.ts b/packages/models/src/videos/blacklist/video-blacklist-update.model.ts similarity index 100% rename from shared/models/videos/blacklist/video-blacklist-update.model.ts rename to packages/models/src/videos/blacklist/video-blacklist-update.model.ts diff --git a/packages/models/src/videos/blacklist/video-blacklist.model.ts b/packages/models/src/videos/blacklist/video-blacklist.model.ts new file mode 100644 index 000000000..1ca5bbbb7 --- /dev/null +++ b/packages/models/src/videos/blacklist/video-blacklist.model.ts @@ -0,0 +1,20 @@ +import { Video } from '../video.model.js' + +export const VideoBlacklistType = { + MANUAL: 1, + AUTO_BEFORE_PUBLISHED: 2 +} as const + +export type VideoBlacklistType_Type = typeof VideoBlacklistType[keyof typeof VideoBlacklistType] + +export interface VideoBlacklist { + id: number + unfederated: boolean + reason?: string + type: VideoBlacklistType_Type + + video: Video + + createdAt: Date + updatedAt: Date +} diff --git a/packages/models/src/videos/caption/index.ts b/packages/models/src/videos/caption/index.ts new file mode 100644 index 000000000..a175768ce --- /dev/null +++ b/packages/models/src/videos/caption/index.ts @@ -0,0 +1,2 @@ +export * from './video-caption.model.js' +export * from './video-caption-update.model.js' diff --git a/shared/models/videos/caption/video-caption-update.model.ts b/packages/models/src/videos/caption/video-caption-update.model.ts similarity index 100% rename from shared/models/videos/caption/video-caption-update.model.ts rename to packages/models/src/videos/caption/video-caption-update.model.ts diff --git a/packages/models/src/videos/caption/video-caption.model.ts b/packages/models/src/videos/caption/video-caption.model.ts new file mode 100644 index 000000000..d6d625ff7 --- /dev/null +++ b/packages/models/src/videos/caption/video-caption.model.ts @@ -0,0 +1,7 @@ +import { VideoConstant } from '../video-constant.model.js' + +export interface VideoCaption { + language: VideoConstant + captionPath: string + updatedAt: string +} diff --git a/packages/models/src/videos/change-ownership/index.ts b/packages/models/src/videos/change-ownership/index.ts new file mode 100644 index 000000000..6cf568f4e --- /dev/null +++ b/packages/models/src/videos/change-ownership/index.ts @@ -0,0 +1,3 @@ +export * from './video-change-ownership-accept.model.js' +export * from './video-change-ownership-create.model.js' +export * from './video-change-ownership.model.js' diff --git a/shared/models/videos/change-ownership/video-change-ownership-accept.model.ts b/packages/models/src/videos/change-ownership/video-change-ownership-accept.model.ts similarity index 100% rename from shared/models/videos/change-ownership/video-change-ownership-accept.model.ts rename to packages/models/src/videos/change-ownership/video-change-ownership-accept.model.ts diff --git a/shared/models/videos/change-ownership/video-change-ownership-create.model.ts b/packages/models/src/videos/change-ownership/video-change-ownership-create.model.ts similarity index 100% rename from shared/models/videos/change-ownership/video-change-ownership-create.model.ts rename to packages/models/src/videos/change-ownership/video-change-ownership-create.model.ts diff --git a/packages/models/src/videos/change-ownership/video-change-ownership.model.ts b/packages/models/src/videos/change-ownership/video-change-ownership.model.ts new file mode 100644 index 000000000..353db37f0 --- /dev/null +++ b/packages/models/src/videos/change-ownership/video-change-ownership.model.ts @@ -0,0 +1,19 @@ +import { Account } from '../../actors/index.js' +import { Video } from '../video.model.js' + +export interface VideoChangeOwnership { + id: number + status: VideoChangeOwnershipStatusType + initiatorAccount: Account + nextOwnerAccount: Account + video: Video + createdAt: Date +} + +export const VideoChangeOwnershipStatus = { + WAITING: 'WAITING', + ACCEPTED: 'ACCEPTED', + REFUSED: 'REFUSED' +} as const + +export type VideoChangeOwnershipStatusType = typeof VideoChangeOwnershipStatus[keyof typeof VideoChangeOwnershipStatus] diff --git a/packages/models/src/videos/channel-sync/index.ts b/packages/models/src/videos/channel-sync/index.ts new file mode 100644 index 000000000..206cbe1b6 --- /dev/null +++ b/packages/models/src/videos/channel-sync/index.ts @@ -0,0 +1,3 @@ +export * from './video-channel-sync-state.enum.js' +export * from './video-channel-sync.model.js' +export * from './video-channel-sync-create.model.js' diff --git a/shared/models/videos/channel-sync/video-channel-sync-create.model.ts b/packages/models/src/videos/channel-sync/video-channel-sync-create.model.ts similarity index 100% rename from shared/models/videos/channel-sync/video-channel-sync-create.model.ts rename to packages/models/src/videos/channel-sync/video-channel-sync-create.model.ts diff --git a/packages/models/src/videos/channel-sync/video-channel-sync-state.enum.ts b/packages/models/src/videos/channel-sync/video-channel-sync-state.enum.ts new file mode 100644 index 000000000..047444bbc --- /dev/null +++ b/packages/models/src/videos/channel-sync/video-channel-sync-state.enum.ts @@ -0,0 +1,8 @@ +export const VideoChannelSyncState = { + WAITING_FIRST_RUN: 1, + PROCESSING: 2, + SYNCED: 3, + FAILED: 4 +} as const + +export type VideoChannelSyncStateType = typeof VideoChannelSyncState[keyof typeof VideoChannelSyncState] diff --git a/packages/models/src/videos/channel-sync/video-channel-sync.model.ts b/packages/models/src/videos/channel-sync/video-channel-sync.model.ts new file mode 100644 index 000000000..38ac99668 --- /dev/null +++ b/packages/models/src/videos/channel-sync/video-channel-sync.model.ts @@ -0,0 +1,14 @@ +import { VideoChannelSummary } from '../channel/video-channel.model.js' +import { VideoConstant } from '../video-constant.model.js' +import { VideoChannelSyncStateType } from './video-channel-sync-state.enum.js' + +export interface VideoChannelSync { + id: number + + externalChannelUrl: string + + createdAt: string + channel: VideoChannelSummary + state: VideoConstant + lastSyncAt: string +} diff --git a/packages/models/src/videos/channel/index.ts b/packages/models/src/videos/channel/index.ts new file mode 100644 index 000000000..3c96e80f0 --- /dev/null +++ b/packages/models/src/videos/channel/index.ts @@ -0,0 +1,4 @@ +export * from './video-channel-create-result.model.js' +export * from './video-channel-create.model.js' +export * from './video-channel-update.model.js' +export * from './video-channel.model.js' diff --git a/shared/models/videos/channel/video-channel-create-result.model.ts b/packages/models/src/videos/channel/video-channel-create-result.model.ts similarity index 100% rename from shared/models/videos/channel/video-channel-create-result.model.ts rename to packages/models/src/videos/channel/video-channel-create-result.model.ts diff --git a/shared/models/videos/channel/video-channel-create.model.ts b/packages/models/src/videos/channel/video-channel-create.model.ts similarity index 100% rename from shared/models/videos/channel/video-channel-create.model.ts rename to packages/models/src/videos/channel/video-channel-create.model.ts diff --git a/shared/models/videos/channel/video-channel-update.model.ts b/packages/models/src/videos/channel/video-channel-update.model.ts similarity index 100% rename from shared/models/videos/channel/video-channel-update.model.ts rename to packages/models/src/videos/channel/video-channel-update.model.ts diff --git a/packages/models/src/videos/channel/video-channel.model.ts b/packages/models/src/videos/channel/video-channel.model.ts new file mode 100644 index 000000000..bb10f6da5 --- /dev/null +++ b/packages/models/src/videos/channel/video-channel.model.ts @@ -0,0 +1,34 @@ +import { Account, ActorImage } from '../../actors/index.js' +import { Actor } from '../../actors/actor.model.js' + +export type ViewsPerDate = { + date: Date + views: number +} + +export interface VideoChannel extends Actor { + displayName: string + description: string + support: string + isLocal: boolean + + updatedAt: Date | string + + ownerAccount?: Account + + videosCount?: number + viewsPerDay?: ViewsPerDate[] // chronologically ordered + totalViews?: number + + banners: ActorImage[] +} + +export interface VideoChannelSummary { + id: number + name: string + displayName: string + url: string + host: string + + avatars: ActorImage[] +} diff --git a/packages/models/src/videos/comment/index.ts b/packages/models/src/videos/comment/index.ts new file mode 100644 index 000000000..bd26c652d --- /dev/null +++ b/packages/models/src/videos/comment/index.ts @@ -0,0 +1,2 @@ +export * from './video-comment-create.model.js' +export * from './video-comment.model.js' diff --git a/shared/models/videos/comment/video-comment-create.model.ts b/packages/models/src/videos/comment/video-comment-create.model.ts similarity index 100% rename from shared/models/videos/comment/video-comment-create.model.ts rename to packages/models/src/videos/comment/video-comment-create.model.ts diff --git a/packages/models/src/videos/comment/video-comment.model.ts b/packages/models/src/videos/comment/video-comment.model.ts new file mode 100644 index 000000000..e2266545a --- /dev/null +++ b/packages/models/src/videos/comment/video-comment.model.ts @@ -0,0 +1,45 @@ +import { ResultList } from '../../common/index.js' +import { Account } from '../../actors/index.js' + +export interface VideoComment { + id: number + url: string + text: string + threadId: number + inReplyToCommentId: number + videoId: number + createdAt: Date | string + updatedAt: Date | string + deletedAt: Date | string + isDeleted: boolean + totalRepliesFromVideoAuthor: number + totalReplies: number + account: Account +} + +export interface VideoCommentAdmin { + id: number + url: string + text: string + + threadId: number + inReplyToCommentId: number + + createdAt: Date | string + updatedAt: Date | string + + account: Account + + video: { + id: number + uuid: string + name: string + } +} + +export type VideoCommentThreads = ResultList & { totalNotDeletedComments: number } + +export interface VideoCommentThreadTree { + comment: VideoComment + children: VideoCommentThreadTree[] +} diff --git a/packages/models/src/videos/file/index.ts b/packages/models/src/videos/file/index.ts new file mode 100644 index 000000000..ee06f4e20 --- /dev/null +++ b/packages/models/src/videos/file/index.ts @@ -0,0 +1,3 @@ +export * from './video-file-metadata.model.js' +export * from './video-file.model.js' +export * from './video-resolution.enum.js' diff --git a/shared/models/videos/file/video-file-metadata.model.ts b/packages/models/src/videos/file/video-file-metadata.model.ts similarity index 100% rename from shared/models/videos/file/video-file-metadata.model.ts rename to packages/models/src/videos/file/video-file-metadata.model.ts diff --git a/packages/models/src/videos/file/video-file.model.ts b/packages/models/src/videos/file/video-file.model.ts new file mode 100644 index 000000000..2ed1ac4be --- /dev/null +++ b/packages/models/src/videos/file/video-file.model.ts @@ -0,0 +1,22 @@ +import { VideoConstant } from '../video-constant.model.js' +import { VideoFileMetadata } from './video-file-metadata.model.js' + +export interface VideoFile { + id: number + + resolution: VideoConstant + size: number // Bytes + + torrentUrl: string + torrentDownloadUrl: string + + fileUrl: string + fileDownloadUrl: string + + fps: number + + metadata?: VideoFileMetadata + metadataUrl?: string + + magnetUri: string | null +} diff --git a/packages/models/src/videos/file/video-resolution.enum.ts b/packages/models/src/videos/file/video-resolution.enum.ts new file mode 100644 index 000000000..434e8c36d --- /dev/null +++ b/packages/models/src/videos/file/video-resolution.enum.ts @@ -0,0 +1,13 @@ +export const VideoResolution = { + H_NOVIDEO: 0, + H_144P: 144, + H_240P: 240, + H_360P: 360, + H_480P: 480, + H_720P: 720, + H_1080P: 1080, + H_1440P: 1440, + H_4K: 2160 +} as const + +export type VideoResolutionType = typeof VideoResolution[keyof typeof VideoResolution] diff --git a/packages/models/src/videos/import/index.ts b/packages/models/src/videos/import/index.ts new file mode 100644 index 000000000..6701674c5 --- /dev/null +++ b/packages/models/src/videos/import/index.ts @@ -0,0 +1,4 @@ +export * from './video-import-create.model.js' +export * from './video-import-state.enum.js' +export * from './video-import.model.js' +export * from './videos-import-in-channel-create.model.js' diff --git a/packages/models/src/videos/import/video-import-create.model.ts b/packages/models/src/videos/import/video-import-create.model.ts new file mode 100644 index 000000000..3ec0d22f3 --- /dev/null +++ b/packages/models/src/videos/import/video-import-create.model.ts @@ -0,0 +1,9 @@ +import { VideoUpdate } from '../video-update.model.js' + +export interface VideoImportCreate extends VideoUpdate { + targetUrl?: string + magnetUri?: string + torrentfile?: Blob + + channelId: number // Required +} diff --git a/packages/models/src/videos/import/video-import-state.enum.ts b/packages/models/src/videos/import/video-import-state.enum.ts new file mode 100644 index 000000000..475fdbe66 --- /dev/null +++ b/packages/models/src/videos/import/video-import-state.enum.ts @@ -0,0 +1,10 @@ +export const VideoImportState = { + PENDING: 1, + SUCCESS: 2, + FAILED: 3, + REJECTED: 4, + CANCELLED: 5, + PROCESSING: 6 +} as const + +export type VideoImportStateType = typeof VideoImportState[keyof typeof VideoImportState] diff --git a/packages/models/src/videos/import/video-import.model.ts b/packages/models/src/videos/import/video-import.model.ts new file mode 100644 index 000000000..eef23f401 --- /dev/null +++ b/packages/models/src/videos/import/video-import.model.ts @@ -0,0 +1,24 @@ +import { VideoConstant } from '../video-constant.model.js' +import { Video } from '../video.model.js' +import { VideoImportStateType } from './video-import-state.enum.js' + +export interface VideoImport { + id: number + + targetUrl: string + magnetUri: string + torrentName: string + + createdAt: string + updatedAt: string + originallyPublishedAt?: string + state: VideoConstant + error?: string + + video?: Video & { tags: string[] } + + videoChannelSync?: { + id: number + externalChannelUrl: string + } +} diff --git a/shared/models/videos/import/videos-import-in-channel-create.model.ts b/packages/models/src/videos/import/videos-import-in-channel-create.model.ts similarity index 100% rename from shared/models/videos/import/videos-import-in-channel-create.model.ts rename to packages/models/src/videos/import/videos-import-in-channel-create.model.ts diff --git a/packages/models/src/videos/index.ts b/packages/models/src/videos/index.ts new file mode 100644 index 000000000..d131212c9 --- /dev/null +++ b/packages/models/src/videos/index.ts @@ -0,0 +1,43 @@ +export * from './blacklist/index.js' +export * from './caption/index.js' +export * from './change-ownership/index.js' +export * from './channel/index.js' +export * from './comment/index.js' +export * from './studio/index.js' +export * from './live/index.js' +export * from './file/index.js' +export * from './import/index.js' +export * from './playlist/index.js' +export * from './rate/index.js' +export * from './stats/index.js' +export * from './transcoding/index.js' +export * from './channel-sync/index.js' + +export * from './nsfw-policy.type.js' + +export * from './storyboard.model.js' +export * from './thumbnail.type.js' + +export * from './video-constant.model.js' +export * from './video-create.model.js' + +export * from './video-privacy.enum.js' +export * from './video-include.enum.js' +export * from './video-rate.type.js' + +export * from './video-schedule-update.model.js' +export * from './video-sort-field.type.js' +export * from './video-state.enum.js' +export * from './video-storage.enum.js' +export * from './video-source.model.js' + +export * from './video-streaming-playlist.model.js' +export * from './video-streaming-playlist.type.js' + +export * from './video-token.model.js' + +export * from './video-update.model.js' +export * from './video-view.model.js' +export * from './video.model.js' +export * from './video-create-result.model.js' +export * from './video-password.model.js' diff --git a/packages/models/src/videos/live/index.ts b/packages/models/src/videos/live/index.ts new file mode 100644 index 000000000..1763eb574 --- /dev/null +++ b/packages/models/src/videos/live/index.ts @@ -0,0 +1,8 @@ +export * from './live-video-create.model.js' +export * from './live-video-error.enum.js' +export * from './live-video-event-payload.model.js' +export * from './live-video-event.type.js' +export * from './live-video-latency-mode.enum.js' +export * from './live-video-session.model.js' +export * from './live-video-update.model.js' +export * from './live-video.model.js' diff --git a/packages/models/src/videos/live/live-video-create.model.ts b/packages/models/src/videos/live/live-video-create.model.ts new file mode 100644 index 000000000..e4e39518c --- /dev/null +++ b/packages/models/src/videos/live/live-video-create.model.ts @@ -0,0 +1,11 @@ +import { VideoCreate } from '../video-create.model.js' +import { VideoPrivacyType } from '../video-privacy.enum.js' +import { LiveVideoLatencyModeType } from './live-video-latency-mode.enum.js' + +export interface LiveVideoCreate extends VideoCreate { + permanentLive?: boolean + latencyMode?: LiveVideoLatencyModeType + + saveReplay?: boolean + replaySettings?: { privacy: VideoPrivacyType } +} diff --git a/packages/models/src/videos/live/live-video-error.enum.ts b/packages/models/src/videos/live/live-video-error.enum.ts new file mode 100644 index 000000000..cd92a1cff --- /dev/null +++ b/packages/models/src/videos/live/live-video-error.enum.ts @@ -0,0 +1,11 @@ +export const LiveVideoError = { + BAD_SOCKET_HEALTH: 1, + DURATION_EXCEEDED: 2, + QUOTA_EXCEEDED: 3, + FFMPEG_ERROR: 4, + BLACKLISTED: 5, + RUNNER_JOB_ERROR: 6, + RUNNER_JOB_CANCEL: 7 +} as const + +export type LiveVideoErrorType = typeof LiveVideoError[keyof typeof LiveVideoError] diff --git a/packages/models/src/videos/live/live-video-event-payload.model.ts b/packages/models/src/videos/live/live-video-event-payload.model.ts new file mode 100644 index 000000000..507f8d153 --- /dev/null +++ b/packages/models/src/videos/live/live-video-event-payload.model.ts @@ -0,0 +1,7 @@ +import { VideoStateType } from '../video-state.enum.js' + +export interface LiveVideoEventPayload { + state?: VideoStateType + + viewers?: number +} diff --git a/shared/models/videos/live/live-video-event.type.ts b/packages/models/src/videos/live/live-video-event.type.ts similarity index 100% rename from shared/models/videos/live/live-video-event.type.ts rename to packages/models/src/videos/live/live-video-event.type.ts diff --git a/packages/models/src/videos/live/live-video-latency-mode.enum.ts b/packages/models/src/videos/live/live-video-latency-mode.enum.ts new file mode 100644 index 000000000..6fd8fe8e9 --- /dev/null +++ b/packages/models/src/videos/live/live-video-latency-mode.enum.ts @@ -0,0 +1,7 @@ +export const LiveVideoLatencyMode = { + DEFAULT: 1, + HIGH_LATENCY: 2, + SMALL_LATENCY: 3 +} as const + +export type LiveVideoLatencyModeType = typeof LiveVideoLatencyMode[keyof typeof LiveVideoLatencyMode] diff --git a/packages/models/src/videos/live/live-video-session.model.ts b/packages/models/src/videos/live/live-video-session.model.ts new file mode 100644 index 000000000..8d45bc86a --- /dev/null +++ b/packages/models/src/videos/live/live-video-session.model.ts @@ -0,0 +1,22 @@ +import { VideoPrivacyType } from '../video-privacy.enum.js' +import { LiveVideoErrorType } from './live-video-error.enum.js' + +export interface LiveVideoSession { + id: number + + startDate: string + endDate: string + + error: LiveVideoErrorType + + saveReplay: boolean + endingProcessed: boolean + + replaySettings?: { privacy: VideoPrivacyType } + + replayVideo: { + id: number + uuid: string + shortUUID: string + } +} diff --git a/packages/models/src/videos/live/live-video-update.model.ts b/packages/models/src/videos/live/live-video-update.model.ts new file mode 100644 index 000000000..b4d91e447 --- /dev/null +++ b/packages/models/src/videos/live/live-video-update.model.ts @@ -0,0 +1,9 @@ +import { VideoPrivacyType } from '../video-privacy.enum.js' +import { LiveVideoLatencyModeType } from './live-video-latency-mode.enum.js' + +export interface LiveVideoUpdate { + permanentLive?: boolean + saveReplay?: boolean + replaySettings?: { privacy: VideoPrivacyType } + latencyMode?: LiveVideoLatencyModeType +} diff --git a/packages/models/src/videos/live/live-video.model.ts b/packages/models/src/videos/live/live-video.model.ts new file mode 100644 index 000000000..3e91f677c --- /dev/null +++ b/packages/models/src/videos/live/live-video.model.ts @@ -0,0 +1,14 @@ +import { VideoPrivacyType } from '../video-privacy.enum.js' +import { LiveVideoLatencyModeType } from './live-video-latency-mode.enum.js' + +export interface LiveVideo { + // If owner + rtmpUrl?: string + rtmpsUrl?: string + streamKey?: string + + saveReplay: boolean + replaySettings?: { privacy: VideoPrivacyType } + permanentLive: boolean + latencyMode: LiveVideoLatencyModeType +} diff --git a/shared/models/videos/nsfw-policy.type.ts b/packages/models/src/videos/nsfw-policy.type.ts similarity index 100% rename from shared/models/videos/nsfw-policy.type.ts rename to packages/models/src/videos/nsfw-policy.type.ts diff --git a/packages/models/src/videos/playlist/index.ts b/packages/models/src/videos/playlist/index.ts new file mode 100644 index 000000000..0e139657a --- /dev/null +++ b/packages/models/src/videos/playlist/index.ts @@ -0,0 +1,12 @@ +export * from './video-exist-in-playlist.model.js' +export * from './video-playlist-create-result.model.js' +export * from './video-playlist-create.model.js' +export * from './video-playlist-element-create-result.model.js' +export * from './video-playlist-element-create.model.js' +export * from './video-playlist-element-update.model.js' +export * from './video-playlist-element.model.js' +export * from './video-playlist-privacy.model.js' +export * from './video-playlist-reorder.model.js' +export * from './video-playlist-type.model.js' +export * from './video-playlist-update.model.js' +export * from './video-playlist.model.js' diff --git a/shared/models/videos/playlist/video-exist-in-playlist.model.ts b/packages/models/src/videos/playlist/video-exist-in-playlist.model.ts similarity index 100% rename from shared/models/videos/playlist/video-exist-in-playlist.model.ts rename to packages/models/src/videos/playlist/video-exist-in-playlist.model.ts diff --git a/shared/models/videos/playlist/video-playlist-create-result.model.ts b/packages/models/src/videos/playlist/video-playlist-create-result.model.ts similarity index 100% rename from shared/models/videos/playlist/video-playlist-create-result.model.ts rename to packages/models/src/videos/playlist/video-playlist-create-result.model.ts diff --git a/packages/models/src/videos/playlist/video-playlist-create.model.ts b/packages/models/src/videos/playlist/video-playlist-create.model.ts new file mode 100644 index 000000000..f9dd1e0d1 --- /dev/null +++ b/packages/models/src/videos/playlist/video-playlist-create.model.ts @@ -0,0 +1,11 @@ +import { VideoPlaylistPrivacyType } from './video-playlist-privacy.model.js' + +export interface VideoPlaylistCreate { + displayName: string + privacy: VideoPlaylistPrivacyType + + description?: string + videoChannelId?: number + + thumbnailfile?: any +} diff --git a/shared/models/videos/playlist/video-playlist-element-create-result.model.ts b/packages/models/src/videos/playlist/video-playlist-element-create-result.model.ts similarity index 100% rename from shared/models/videos/playlist/video-playlist-element-create-result.model.ts rename to packages/models/src/videos/playlist/video-playlist-element-create-result.model.ts diff --git a/shared/models/videos/playlist/video-playlist-element-create.model.ts b/packages/models/src/videos/playlist/video-playlist-element-create.model.ts similarity index 100% rename from shared/models/videos/playlist/video-playlist-element-create.model.ts rename to packages/models/src/videos/playlist/video-playlist-element-create.model.ts diff --git a/shared/models/videos/playlist/video-playlist-element-update.model.ts b/packages/models/src/videos/playlist/video-playlist-element-update.model.ts similarity index 100% rename from shared/models/videos/playlist/video-playlist-element-update.model.ts rename to packages/models/src/videos/playlist/video-playlist-element-update.model.ts diff --git a/packages/models/src/videos/playlist/video-playlist-element.model.ts b/packages/models/src/videos/playlist/video-playlist-element.model.ts new file mode 100644 index 000000000..a4711f919 --- /dev/null +++ b/packages/models/src/videos/playlist/video-playlist-element.model.ts @@ -0,0 +1,21 @@ +import { Video } from '../video.model.js' + +export const VideoPlaylistElementType = { + REGULAR: 0, + DELETED: 1, + PRIVATE: 2, + UNAVAILABLE: 3 // Blacklisted, blocked by the user/instance, NSFW... +} as const + +export type VideoPlaylistElementType_Type = typeof VideoPlaylistElementType[keyof typeof VideoPlaylistElementType] + +export interface VideoPlaylistElement { + id: number + position: number + startTimestamp: number + stopTimestamp: number + + type: VideoPlaylistElementType_Type + + video?: Video +} diff --git a/packages/models/src/videos/playlist/video-playlist-privacy.model.ts b/packages/models/src/videos/playlist/video-playlist-privacy.model.ts new file mode 100644 index 000000000..23f6a1a16 --- /dev/null +++ b/packages/models/src/videos/playlist/video-playlist-privacy.model.ts @@ -0,0 +1,7 @@ +export const VideoPlaylistPrivacy = { + PUBLIC: 1, + UNLISTED: 2, + PRIVATE: 3 +} as const + +export type VideoPlaylistPrivacyType = typeof VideoPlaylistPrivacy[keyof typeof VideoPlaylistPrivacy] diff --git a/shared/models/videos/playlist/video-playlist-reorder.model.ts b/packages/models/src/videos/playlist/video-playlist-reorder.model.ts similarity index 100% rename from shared/models/videos/playlist/video-playlist-reorder.model.ts rename to packages/models/src/videos/playlist/video-playlist-reorder.model.ts diff --git a/packages/models/src/videos/playlist/video-playlist-type.model.ts b/packages/models/src/videos/playlist/video-playlist-type.model.ts new file mode 100644 index 000000000..183439f98 --- /dev/null +++ b/packages/models/src/videos/playlist/video-playlist-type.model.ts @@ -0,0 +1,6 @@ +export const VideoPlaylistType = { + REGULAR: 1, + WATCH_LATER: 2 +} as const + +export type VideoPlaylistType_Type = typeof VideoPlaylistType[keyof typeof VideoPlaylistType] diff --git a/packages/models/src/videos/playlist/video-playlist-update.model.ts b/packages/models/src/videos/playlist/video-playlist-update.model.ts new file mode 100644 index 000000000..ed536367e --- /dev/null +++ b/packages/models/src/videos/playlist/video-playlist-update.model.ts @@ -0,0 +1,10 @@ +import { VideoPlaylistPrivacyType } from './video-playlist-privacy.model.js' + +export interface VideoPlaylistUpdate { + displayName?: string + privacy?: VideoPlaylistPrivacyType + + description?: string + videoChannelId?: number + thumbnailfile?: any +} diff --git a/packages/models/src/videos/playlist/video-playlist.model.ts b/packages/models/src/videos/playlist/video-playlist.model.ts new file mode 100644 index 000000000..4261aac25 --- /dev/null +++ b/packages/models/src/videos/playlist/video-playlist.model.ts @@ -0,0 +1,35 @@ +import { AccountSummary } from '../../actors/index.js' +import { VideoChannelSummary } from '../channel/index.js' +import { VideoConstant } from '../video-constant.model.js' +import { VideoPlaylistPrivacyType } from './video-playlist-privacy.model.js' +import { VideoPlaylistType_Type } from './video-playlist-type.model.js' + +export interface VideoPlaylist { + id: number + uuid: string + shortUUID: string + + isLocal: boolean + + url: string + + displayName: string + description: string + privacy: VideoConstant + + thumbnailPath: string + thumbnailUrl?: string + + videosLength: number + + type: VideoConstant + + embedPath: string + embedUrl?: string + + createdAt: Date | string + updatedAt: Date | string + + ownerAccount: AccountSummary + videoChannel?: VideoChannelSummary +} diff --git a/packages/models/src/videos/rate/account-video-rate.model.ts b/packages/models/src/videos/rate/account-video-rate.model.ts new file mode 100644 index 000000000..d19ccdbdd --- /dev/null +++ b/packages/models/src/videos/rate/account-video-rate.model.ts @@ -0,0 +1,7 @@ +import { UserVideoRateType } from './user-video-rate.type.js' +import { Video } from '../video.model.js' + +export interface AccountVideoRate { + video: Video + rating: UserVideoRateType +} diff --git a/packages/models/src/videos/rate/index.ts b/packages/models/src/videos/rate/index.ts new file mode 100644 index 000000000..ecbe3523d --- /dev/null +++ b/packages/models/src/videos/rate/index.ts @@ -0,0 +1,5 @@ + +export * from './user-video-rate-update.model.js' +export * from './user-video-rate.model.js' +export * from './account-video-rate.model.js' +export * from './user-video-rate.type.js' diff --git a/packages/models/src/videos/rate/user-video-rate-update.model.ts b/packages/models/src/videos/rate/user-video-rate-update.model.ts new file mode 100644 index 000000000..8ee1e78ca --- /dev/null +++ b/packages/models/src/videos/rate/user-video-rate-update.model.ts @@ -0,0 +1,5 @@ +import { UserVideoRateType } from './user-video-rate.type.js' + +export interface UserVideoRateUpdate { + rating: UserVideoRateType +} diff --git a/packages/models/src/videos/rate/user-video-rate.model.ts b/packages/models/src/videos/rate/user-video-rate.model.ts new file mode 100644 index 000000000..344cf9a68 --- /dev/null +++ b/packages/models/src/videos/rate/user-video-rate.model.ts @@ -0,0 +1,6 @@ +import { UserVideoRateType } from './user-video-rate.type.js' + +export interface UserVideoRate { + videoId: number + rating: UserVideoRateType +} diff --git a/shared/models/videos/rate/user-video-rate.type.ts b/packages/models/src/videos/rate/user-video-rate.type.ts similarity index 100% rename from shared/models/videos/rate/user-video-rate.type.ts rename to packages/models/src/videos/rate/user-video-rate.type.ts diff --git a/packages/models/src/videos/stats/index.ts b/packages/models/src/videos/stats/index.ts new file mode 100644 index 000000000..7187cac26 --- /dev/null +++ b/packages/models/src/videos/stats/index.ts @@ -0,0 +1,6 @@ +export * from './video-stats-overall-query.model.js' +export * from './video-stats-overall.model.js' +export * from './video-stats-retention.model.js' +export * from './video-stats-timeserie-query.model.js' +export * from './video-stats-timeserie-metric.type.js' +export * from './video-stats-timeserie.model.js' diff --git a/shared/models/videos/stats/video-stats-overall-query.model.ts b/packages/models/src/videos/stats/video-stats-overall-query.model.ts similarity index 100% rename from shared/models/videos/stats/video-stats-overall-query.model.ts rename to packages/models/src/videos/stats/video-stats-overall-query.model.ts diff --git a/shared/models/videos/stats/video-stats-overall.model.ts b/packages/models/src/videos/stats/video-stats-overall.model.ts similarity index 100% rename from shared/models/videos/stats/video-stats-overall.model.ts rename to packages/models/src/videos/stats/video-stats-overall.model.ts diff --git a/shared/models/videos/stats/video-stats-retention.model.ts b/packages/models/src/videos/stats/video-stats-retention.model.ts similarity index 100% rename from shared/models/videos/stats/video-stats-retention.model.ts rename to packages/models/src/videos/stats/video-stats-retention.model.ts diff --git a/shared/models/videos/stats/video-stats-timeserie-metric.type.ts b/packages/models/src/videos/stats/video-stats-timeserie-metric.type.ts similarity index 100% rename from shared/models/videos/stats/video-stats-timeserie-metric.type.ts rename to packages/models/src/videos/stats/video-stats-timeserie-metric.type.ts diff --git a/shared/models/videos/stats/video-stats-timeserie-query.model.ts b/packages/models/src/videos/stats/video-stats-timeserie-query.model.ts similarity index 100% rename from shared/models/videos/stats/video-stats-timeserie-query.model.ts rename to packages/models/src/videos/stats/video-stats-timeserie-query.model.ts diff --git a/shared/models/videos/stats/video-stats-timeserie.model.ts b/packages/models/src/videos/stats/video-stats-timeserie.model.ts similarity index 100% rename from shared/models/videos/stats/video-stats-timeserie.model.ts rename to packages/models/src/videos/stats/video-stats-timeserie.model.ts diff --git a/shared/models/videos/storyboard.model.ts b/packages/models/src/videos/storyboard.model.ts similarity index 100% rename from shared/models/videos/storyboard.model.ts rename to packages/models/src/videos/storyboard.model.ts diff --git a/packages/models/src/videos/studio/index.ts b/packages/models/src/videos/studio/index.ts new file mode 100644 index 000000000..0d8ad3227 --- /dev/null +++ b/packages/models/src/videos/studio/index.ts @@ -0,0 +1 @@ +export * from './video-studio-create-edit.model.js' diff --git a/shared/models/videos/studio/video-studio-create-edit.model.ts b/packages/models/src/videos/studio/video-studio-create-edit.model.ts similarity index 100% rename from shared/models/videos/studio/video-studio-create-edit.model.ts rename to packages/models/src/videos/studio/video-studio-create-edit.model.ts diff --git a/packages/models/src/videos/thumbnail.type.ts b/packages/models/src/videos/thumbnail.type.ts new file mode 100644 index 000000000..0cb7483ec --- /dev/null +++ b/packages/models/src/videos/thumbnail.type.ts @@ -0,0 +1,6 @@ +export const ThumbnailType = { + MINIATURE: 1, + PREVIEW: 2 +} as const + +export type ThumbnailType_Type = typeof ThumbnailType[keyof typeof ThumbnailType] diff --git a/packages/models/src/videos/transcoding/index.ts b/packages/models/src/videos/transcoding/index.ts new file mode 100644 index 000000000..e1d931bd5 --- /dev/null +++ b/packages/models/src/videos/transcoding/index.ts @@ -0,0 +1,3 @@ +export * from './video-transcoding-create.model.js' +export * from './video-transcoding-fps.model.js' +export * from './video-transcoding.model.js' diff --git a/shared/models/videos/transcoding/video-transcoding-create.model.ts b/packages/models/src/videos/transcoding/video-transcoding-create.model.ts similarity index 100% rename from shared/models/videos/transcoding/video-transcoding-create.model.ts rename to packages/models/src/videos/transcoding/video-transcoding-create.model.ts diff --git a/shared/models/videos/transcoding/video-transcoding-fps.model.ts b/packages/models/src/videos/transcoding/video-transcoding-fps.model.ts similarity index 100% rename from shared/models/videos/transcoding/video-transcoding-fps.model.ts rename to packages/models/src/videos/transcoding/video-transcoding-fps.model.ts diff --git a/packages/models/src/videos/transcoding/video-transcoding.model.ts b/packages/models/src/videos/transcoding/video-transcoding.model.ts new file mode 100644 index 000000000..e2c2a56e5 --- /dev/null +++ b/packages/models/src/videos/transcoding/video-transcoding.model.ts @@ -0,0 +1,65 @@ +// Types used by plugins and ffmpeg-utils + +export type EncoderOptionsBuilderParams = { + input: string + + resolution: number + + // If PeerTube applies a filter, transcoding profile must not copy input stream + canCopyAudio: boolean + canCopyVideo: boolean + + fps: number + + // Could be undefined if we could not get input bitrate (some RTMP streams for example) + inputBitrate: number + inputRatio: number + + // For lives + streamNum?: number +} + +export type EncoderOptionsBuilder = (params: EncoderOptionsBuilderParams) => Promise | EncoderOptions + +export interface EncoderOptions { + copy?: boolean // Copy stream? Default to false + + scaleFilter?: { + name: string + } + + inputOptions?: string[] + outputOptions?: string[] +} + +// All our encoders + +export interface EncoderProfile { + [ profile: string ]: T + + default: T +} + +export type AvailableEncoders = { + available: { + live: { + [ encoder: string ]: EncoderProfile + } + + vod: { + [ encoder: string ]: EncoderProfile + } + } + + encodersToTry: { + vod: { + video: string[] + audio: string[] + } + + live: { + video: string[] + audio: string[] + } + } +} diff --git a/shared/models/videos/video-constant.model.ts b/packages/models/src/videos/video-constant.model.ts similarity index 100% rename from shared/models/videos/video-constant.model.ts rename to packages/models/src/videos/video-constant.model.ts diff --git a/shared/models/videos/video-create-result.model.ts b/packages/models/src/videos/video-create-result.model.ts similarity index 100% rename from shared/models/videos/video-create-result.model.ts rename to packages/models/src/videos/video-create-result.model.ts diff --git a/packages/models/src/videos/video-create.model.ts b/packages/models/src/videos/video-create.model.ts new file mode 100644 index 000000000..472201211 --- /dev/null +++ b/packages/models/src/videos/video-create.model.ts @@ -0,0 +1,25 @@ +import { VideoPrivacyType } from './video-privacy.enum.js' +import { VideoScheduleUpdate } from './video-schedule-update.model.js' + +export interface VideoCreate { + name: string + channelId: number + + category?: number + licence?: number + language?: string + description?: string + support?: string + nsfw?: boolean + waitTranscoding?: boolean + tags?: string[] + commentsEnabled?: boolean + downloadEnabled?: boolean + privacy: VideoPrivacyType + scheduleUpdate?: VideoScheduleUpdate + originallyPublishedAt?: Date | string + videoPasswords?: string[] + + thumbnailfile?: Blob | string + previewfile?: Blob | string +} diff --git a/packages/models/src/videos/video-include.enum.ts b/packages/models/src/videos/video-include.enum.ts new file mode 100644 index 000000000..7d88a6890 --- /dev/null +++ b/packages/models/src/videos/video-include.enum.ts @@ -0,0 +1,10 @@ +export const VideoInclude = { + NONE: 0, + NOT_PUBLISHED_STATE: 1 << 0, + BLACKLISTED: 1 << 1, + BLOCKED_OWNER: 1 << 2, + FILES: 1 << 3, + CAPTIONS: 1 << 4 +} as const + +export type VideoIncludeType = typeof VideoInclude[keyof typeof VideoInclude] diff --git a/shared/models/videos/video-password.model.ts b/packages/models/src/videos/video-password.model.ts similarity index 100% rename from shared/models/videos/video-password.model.ts rename to packages/models/src/videos/video-password.model.ts diff --git a/packages/models/src/videos/video-privacy.enum.ts b/packages/models/src/videos/video-privacy.enum.ts new file mode 100644 index 000000000..cbcc91b3f --- /dev/null +++ b/packages/models/src/videos/video-privacy.enum.ts @@ -0,0 +1,9 @@ +export const VideoPrivacy = { + PUBLIC: 1, + UNLISTED: 2, + PRIVATE: 3, + INTERNAL: 4, + PASSWORD_PROTECTED: 5 +} as const + +export type VideoPrivacyType = typeof VideoPrivacy[keyof typeof VideoPrivacy] diff --git a/shared/models/videos/video-rate.type.ts b/packages/models/src/videos/video-rate.type.ts similarity index 100% rename from shared/models/videos/video-rate.type.ts rename to packages/models/src/videos/video-rate.type.ts diff --git a/packages/models/src/videos/video-schedule-update.model.ts b/packages/models/src/videos/video-schedule-update.model.ts new file mode 100644 index 000000000..2e6a5551d --- /dev/null +++ b/packages/models/src/videos/video-schedule-update.model.ts @@ -0,0 +1,7 @@ +import { VideoPrivacy } from './video-privacy.enum.js' + +export interface VideoScheduleUpdate { + updateAt: Date | string + // Cannot schedule an update to PRIVATE + privacy?: typeof VideoPrivacy.PUBLIC | typeof VideoPrivacy.UNLISTED | typeof VideoPrivacy.INTERNAL +} diff --git a/shared/models/videos/video-sort-field.type.ts b/packages/models/src/videos/video-sort-field.type.ts similarity index 100% rename from shared/models/videos/video-sort-field.type.ts rename to packages/models/src/videos/video-sort-field.type.ts diff --git a/shared/models/videos/video-source.ts b/packages/models/src/videos/video-source.model.ts similarity index 100% rename from shared/models/videos/video-source.ts rename to packages/models/src/videos/video-source.model.ts diff --git a/packages/models/src/videos/video-state.enum.ts b/packages/models/src/videos/video-state.enum.ts new file mode 100644 index 000000000..ae7c6a0c4 --- /dev/null +++ b/packages/models/src/videos/video-state.enum.ts @@ -0,0 +1,13 @@ +export const VideoState = { + PUBLISHED: 1, + TO_TRANSCODE: 2, + TO_IMPORT: 3, + WAITING_FOR_LIVE: 4, + LIVE_ENDED: 5, + TO_MOVE_TO_EXTERNAL_STORAGE: 6, + TRANSCODING_FAILED: 7, + TO_MOVE_TO_EXTERNAL_STORAGE_FAILED: 8, + TO_EDIT: 9 +} as const + +export type VideoStateType = typeof VideoState[keyof typeof VideoState] diff --git a/packages/models/src/videos/video-storage.enum.ts b/packages/models/src/videos/video-storage.enum.ts new file mode 100644 index 000000000..de5c92e0d --- /dev/null +++ b/packages/models/src/videos/video-storage.enum.ts @@ -0,0 +1,6 @@ +export const VideoStorage = { + FILE_SYSTEM: 0, + OBJECT_STORAGE: 1 +} as const + +export type VideoStorageType = typeof VideoStorage[keyof typeof VideoStorage] diff --git a/packages/models/src/videos/video-streaming-playlist.model.ts b/packages/models/src/videos/video-streaming-playlist.model.ts new file mode 100644 index 000000000..80aa70e3c --- /dev/null +++ b/packages/models/src/videos/video-streaming-playlist.model.ts @@ -0,0 +1,15 @@ +import { VideoFile } from './file/index.js' +import { VideoStreamingPlaylistType_Type } from './video-streaming-playlist.type.js' + +export interface VideoStreamingPlaylist { + id: number + type: VideoStreamingPlaylistType_Type + playlistUrl: string + segmentsSha256Url: string + + redundancies: { + baseUrl: string + }[] + + files: VideoFile[] +} diff --git a/packages/models/src/videos/video-streaming-playlist.type.ts b/packages/models/src/videos/video-streaming-playlist.type.ts new file mode 100644 index 000000000..07a2c207f --- /dev/null +++ b/packages/models/src/videos/video-streaming-playlist.type.ts @@ -0,0 +1,5 @@ +export const VideoStreamingPlaylistType = { + HLS: 1 +} as const + +export type VideoStreamingPlaylistType_Type = typeof VideoStreamingPlaylistType[keyof typeof VideoStreamingPlaylistType] diff --git a/shared/models/videos/video-token.model.ts b/packages/models/src/videos/video-token.model.ts similarity index 100% rename from shared/models/videos/video-token.model.ts rename to packages/models/src/videos/video-token.model.ts diff --git a/packages/models/src/videos/video-update.model.ts b/packages/models/src/videos/video-update.model.ts new file mode 100644 index 000000000..8af298160 --- /dev/null +++ b/packages/models/src/videos/video-update.model.ts @@ -0,0 +1,25 @@ +import { VideoPrivacyType } from './video-privacy.enum.js' +import { VideoScheduleUpdate } from './video-schedule-update.model.js' + +export interface VideoUpdate { + name?: string + category?: number + licence?: number + language?: string + description?: string + support?: string + privacy?: VideoPrivacyType + tags?: string[] + commentsEnabled?: boolean + downloadEnabled?: boolean + nsfw?: boolean + waitTranscoding?: boolean + channelId?: number + thumbnailfile?: Blob + previewfile?: Blob + scheduleUpdate?: VideoScheduleUpdate + originallyPublishedAt?: Date | string + videoPasswords?: string[] + + pluginData?: any +} diff --git a/shared/models/videos/video-view.model.ts b/packages/models/src/videos/video-view.model.ts similarity index 100% rename from shared/models/videos/video-view.model.ts rename to packages/models/src/videos/video-view.model.ts diff --git a/packages/models/src/videos/video.model.ts b/packages/models/src/videos/video.model.ts new file mode 100644 index 000000000..a750e220d --- /dev/null +++ b/packages/models/src/videos/video.model.ts @@ -0,0 +1,99 @@ +import { Account, AccountSummary } from '../actors/index.js' +import { VideoChannel, VideoChannelSummary } from './channel/video-channel.model.js' +import { VideoFile } from './file/index.js' +import { VideoConstant } from './video-constant.model.js' +import { VideoPrivacyType } from './video-privacy.enum.js' +import { VideoScheduleUpdate } from './video-schedule-update.model.js' +import { VideoStateType } from './video-state.enum.js' +import { VideoStreamingPlaylist } from './video-streaming-playlist.model.js' + +export interface Video extends Partial { + id: number + uuid: string + shortUUID: string + + createdAt: Date | string + updatedAt: Date | string + publishedAt: Date | string + originallyPublishedAt: Date | string + category: VideoConstant + licence: VideoConstant + language: VideoConstant + privacy: VideoConstant + + // Deprecated in 5.0 in favour of truncatedDescription + description: string + truncatedDescription: string + + duration: number + isLocal: boolean + name: string + + isLive: boolean + + thumbnailPath: string + thumbnailUrl?: string + + previewPath: string + previewUrl?: string + + embedPath: string + embedUrl?: string + + url: string + + views: number + viewers: number + + likes: number + dislikes: number + nsfw: boolean + + account: AccountSummary + channel: VideoChannelSummary + + userHistory?: { + currentTime: number + } + + pluginData?: any +} + +// Not included by default, needs query params +export interface VideoAdditionalAttributes { + waitTranscoding: boolean + state: VideoConstant + scheduledUpdate: VideoScheduleUpdate + + blacklisted: boolean + blacklistedReason: string + + blockedOwner: boolean + blockedServer: boolean + + files: VideoFile[] + streamingPlaylists: VideoStreamingPlaylist[] +} + +export interface VideoDetails extends Video { + // Deprecated in 5.0 + descriptionPath: string + + support: string + channel: VideoChannel + account: Account + tags: string[] + commentsEnabled: boolean + downloadEnabled: boolean + + // Not optional in details (unlike in parent Video) + waitTranscoding: boolean + state: VideoConstant + + trackerUrls: string[] + + files: VideoFile[] + streamingPlaylists: VideoStreamingPlaylist[] + + inputFileUpdatedAt: string | Date +} diff --git a/packages/models/tsconfig.json b/packages/models/tsconfig.json new file mode 100644 index 000000000..58fa2330b --- /dev/null +++ b/packages/models/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "src", + "tsBuildInfoFile": "./dist/.tsbuildinfo" + } +} diff --git a/packages/models/tsconfig.types.json b/packages/models/tsconfig.types.json new file mode 100644 index 000000000..997161c21 --- /dev/null +++ b/packages/models/tsconfig.types.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../types-generator/dist/peertube-models", + "tsBuildInfoFile": "../types-generator/dist/peertube-models/.tsbuildinfo", + "stripInternal": true, + "removeComments": false, + "emitDeclarationOnly": true + } +} diff --git a/packages/node-utils/package.json b/packages/node-utils/package.json new file mode 100644 index 000000000..cd7d8ac80 --- /dev/null +++ b/packages/node-utils/package.json @@ -0,0 +1,19 @@ +{ + "name": "@peertube/peertube-node-utils", + "private": true, + "version": "0.0.0", + "main": "dist/index.js", + "files": [ "dist" ], + "exports": { + "types": "./dist/index.d.ts", + "peertube:tsx": "./src/index.ts", + "default": "./dist/index.js" + }, + "type": "module", + "devDependencies": {}, + "scripts": { + "build": "tsc", + "watch": "tsc -w" + }, + "dependencies": {} +} diff --git a/shared/extra-utils/crypto.ts b/packages/node-utils/src/crypto.ts similarity index 100% rename from shared/extra-utils/crypto.ts rename to packages/node-utils/src/crypto.ts diff --git a/packages/node-utils/src/env.ts b/packages/node-utils/src/env.ts new file mode 100644 index 000000000..1a28f509e --- /dev/null +++ b/packages/node-utils/src/env.ts @@ -0,0 +1,58 @@ +export function parallelTests () { + return process.env.MOCHA_PARALLEL === 'true' +} + +export function isGithubCI () { + return !!process.env.GITHUB_WORKSPACE +} + +export function areHttpImportTestsDisabled () { + const disabled = process.env.DISABLE_HTTP_IMPORT_TESTS === 'true' + + if (disabled) console.log('DISABLE_HTTP_IMPORT_TESTS env set to "true" so import tests are disabled') + + return disabled +} + +export function areMockObjectStorageTestsDisabled () { + const disabled = process.env.ENABLE_OBJECT_STORAGE_TESTS !== 'true' + + if (disabled) console.log('ENABLE_OBJECT_STORAGE_TESTS env is not set to "true" so object storage tests are disabled') + + return disabled +} + +export function areScalewayObjectStorageTestsDisabled () { + if (areMockObjectStorageTestsDisabled()) return true + + const enabled = process.env.OBJECT_STORAGE_SCALEWAY_KEY_ID && process.env.OBJECT_STORAGE_SCALEWAY_ACCESS_KEY + if (!enabled) { + console.log( + 'OBJECT_STORAGE_SCALEWAY_KEY_ID and/or OBJECT_STORAGE_SCALEWAY_ACCESS_KEY are not set, so scaleway object storage tests are disabled' + ) + + return true + } + + return false +} + +export function isTestInstance () { + return process.env.NODE_ENV === 'test' +} + +export function isDevInstance () { + return process.env.NODE_ENV === 'dev' +} + +export function isTestOrDevInstance () { + return isTestInstance() || isDevInstance() +} + +export function isProdInstance () { + return process.env.NODE_ENV === 'production' +} + +export function getAppNumber () { + return process.env.NODE_APP_INSTANCE || '' +} diff --git a/packages/node-utils/src/file.ts b/packages/node-utils/src/file.ts new file mode 100644 index 000000000..89cf5fe0f --- /dev/null +++ b/packages/node-utils/src/file.ts @@ -0,0 +1,11 @@ +import { stat } from 'fs/promises' + +async function getFileSize (path: string) { + const stats = await stat(path) + + return stats.size +} + +export { + getFileSize +} diff --git a/packages/node-utils/src/index.ts b/packages/node-utils/src/index.ts new file mode 100644 index 000000000..89f22e7d3 --- /dev/null +++ b/packages/node-utils/src/index.ts @@ -0,0 +1,5 @@ +export * from './crypto.js' +export * from './env.js' +export * from './file.js' +export * from './path.js' +export * from './uuid.js' diff --git a/packages/node-utils/src/path.ts b/packages/node-utils/src/path.ts new file mode 100644 index 000000000..1d569833e --- /dev/null +++ b/packages/node-utils/src/path.ts @@ -0,0 +1,50 @@ +import { basename, extname, isAbsolute, join, resolve } from 'path' +import { fileURLToPath } from 'url' + +let rootPath: string + +export function currentDir (metaUrl: string) { + return resolve(fileURLToPath(metaUrl), '..') +} + +export function root (metaUrl?: string) { + if (rootPath) return rootPath + + if (!metaUrl) { + metaUrl = import.meta.url + + const filename = basename(metaUrl) === 'path.js' || basename(metaUrl) === 'path.ts' + if (!filename) throw new Error('meta url must be specified as this file has been bundled in another one') + } + + rootPath = currentDir(metaUrl) + + if (basename(rootPath) === 'src' || basename(rootPath) === 'dist') rootPath = resolve(rootPath, '..') + if ([ 'node-utils', 'peertube-cli', 'peertube-runner' ].includes(basename(rootPath))) rootPath = resolve(rootPath, '..') + if ([ 'packages', 'apps' ].includes(basename(rootPath))) rootPath = resolve(rootPath, '..') + if (basename(rootPath) === 'dist') rootPath = resolve(rootPath, '..') + + return rootPath +} + +export function buildPath (path: string) { + if (isAbsolute(path)) return path + + return join(root(), path) +} + +export function getLowercaseExtension (filename: string) { + const ext = extname(filename) || '' + + return ext.toLowerCase() +} + +export function buildAbsoluteFixturePath (path: string, customCIPath = false) { + if (isAbsolute(path)) return path + + if (customCIPath && process.env.GITHUB_WORKSPACE) { + return join(process.env.GITHUB_WORKSPACE, 'fixtures', path) + } + + return join(root(), 'packages', 'tests', 'fixtures', path) +} diff --git a/packages/node-utils/src/uuid.ts b/packages/node-utils/src/uuid.ts new file mode 100644 index 000000000..f158ec487 --- /dev/null +++ b/packages/node-utils/src/uuid.ts @@ -0,0 +1,32 @@ +import short from 'short-uuid' + +const translator = short() + +function buildUUID () { + return short.uuid() +} + +function uuidToShort (uuid: string) { + if (!uuid) return uuid + + return translator.fromUUID(uuid) +} + +function shortToUUID (shortUUID: string) { + if (!shortUUID) return shortUUID + + return translator.toUUID(shortUUID) +} + +function isShortUUID (value: string) { + if (!value) return false + + return value.length === translator.maxLength +} + +export { + buildUUID, + uuidToShort, + shortToUUID, + isShortUUID +} diff --git a/packages/node-utils/tsconfig.json b/packages/node-utils/tsconfig.json new file mode 100644 index 000000000..58fa2330b --- /dev/null +++ b/packages/node-utils/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "src", + "tsBuildInfoFile": "./dist/.tsbuildinfo" + } +} diff --git a/packages/peertube-runner/.npmignore b/packages/peertube-runner/.npmignore deleted file mode 100644 index f38d9947c..000000000 --- a/packages/peertube-runner/.npmignore +++ /dev/null @@ -1,6 +0,0 @@ -register -server -shared -meta.json -peertube-runner.ts -tsconfig.json diff --git a/packages/peertube-runner/README.md b/packages/peertube-runner/README.md deleted file mode 100644 index 84974bb80..000000000 --- a/packages/peertube-runner/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# PeerTube runner - -Runner program to execute jobs (transcoding...) of remote PeerTube instances. - -Commands below has to be run at the root of PeerTube git repository. - -## Develop - -```bash -npm run dev:peertube-runner -``` - -## Build - -```bash -npm run build:peertube-runner -``` - -## Run - -```bash -node packages/peertube-runner/dist/peertube-runner.js --help -``` - -## Publish on NPM - -```bash -(cd packages/peertube-runner && npm version patch) && npm run build:peertube-runner && (cd packages/peertube-runner && npm publish --access=public) -``` diff --git a/packages/peertube-runner/package.json b/packages/peertube-runner/package.json deleted file mode 100644 index 1c525691a..000000000 --- a/packages/peertube-runner/package.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "@peertube/peertube-runner", - "version": "0.0.5", - "main": "dist/peertube-runner.js", - "bin": "dist/peertube-runner.js", - "license": "AGPL-3.0", - "dependencies": {}, - "devDependencies": { - "@commander-js/extra-typings": "^10.0.3", - "@iarna/toml": "^2.2.5", - "env-paths": "^3.0.0", - "esbuild": "^0.17.15", - "net-ipc": "^2.0.1", - "pino": "^8.11.0", - "pino-pretty": "^10.0.0" - } -} diff --git a/packages/peertube-runner/peertube-runner.ts b/packages/peertube-runner/peertube-runner.ts deleted file mode 100644 index 32586c4d9..000000000 --- a/packages/peertube-runner/peertube-runner.ts +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env node - -import { Command, InvalidArgumentError } from '@commander-js/extra-typings' -import { listRegistered, registerRunner, unregisterRunner } from './register' -import { RunnerServer } from './server' -import { ConfigManager, logger } from './shared' - -const packageJSON = require('./package.json') - -const program = new Command() - .version(packageJSON.version) - .option( - '--id ', - 'Runner server id, so you can run multiple PeerTube server runners with different configurations on the same machine', - 'default' - ) - .option('--verbose', 'Run in verbose mode') - .hook('preAction', thisCommand => { - const options = thisCommand.opts() - - ConfigManager.Instance.init(options.id) - - if (options.verbose === true) { - logger.level = 'debug' - } - }) - -program.command('server') - .description('Run in server mode, to execute remote jobs of registered PeerTube instances') - .action(async () => { - try { - await RunnerServer.Instance.run() - } catch (err) { - logger.error(err, 'Cannot run PeerTube runner as server mode') - process.exit(-1) - } - }) - -program.command('register') - .description('Register a new PeerTube instance to process runner jobs') - .requiredOption('--url ', 'PeerTube instance URL', parseUrl) - .requiredOption('--registration-token ', 'Runner registration token (can be found in PeerTube instance administration') - .requiredOption('--runner-name ', 'Runner name') - .option('--runner-description ', 'Runner description') - .action(async options => { - try { - await registerRunner(options) - } catch (err) { - console.error('Cannot register this PeerTube runner.') - console.error(err) - process.exit(-1) - } - }) - -program.command('unregister') - .description('Unregister the runner from PeerTube instance') - .requiredOption('--url ', 'PeerTube instance URL', parseUrl) - .requiredOption('--runner-name ', 'Runner name') - .action(async options => { - try { - await unregisterRunner(options) - } catch (err) { - console.error('Cannot unregister this PeerTube runner.') - console.error(err) - process.exit(-1) - } - }) - -program.command('list-registered') - .description('List registered PeerTube instances') - .action(async () => { - try { - await listRegistered() - } catch (err) { - console.error('Cannot list registered PeerTube instances.') - console.error(err) - process.exit(-1) - } - }) - -program.parse() - -// --------------------------------------------------------------------------- -// Private -// --------------------------------------------------------------------------- - -function parseUrl (url: string) { - if (url.startsWith('http://') !== true && url.startsWith('https://') !== true) { - throw new InvalidArgumentError('URL should start with a http:// or https://') - } - - return url -} diff --git a/packages/peertube-runner/register/index.ts b/packages/peertube-runner/register/index.ts deleted file mode 100644 index 3d4273ef8..000000000 --- a/packages/peertube-runner/register/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './register' diff --git a/packages/peertube-runner/register/register.ts b/packages/peertube-runner/register/register.ts deleted file mode 100644 index ca1bf0f5a..000000000 --- a/packages/peertube-runner/register/register.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { IPCClient } from '../shared/ipc' - -export async function registerRunner (options: { - url: string - registrationToken: string - runnerName: string - runnerDescription?: string -}) { - const client = new IPCClient() - await client.run() - - await client.askRegister(options) - - client.stop() -} - -export async function unregisterRunner (options: { - url: string - runnerName: string -}) { - const client = new IPCClient() - await client.run() - - await client.askUnregister(options) - - client.stop() -} - -export async function listRegistered () { - const client = new IPCClient() - await client.run() - - await client.askListRegistered() - - client.stop() -} diff --git a/packages/peertube-runner/server/index.ts b/packages/peertube-runner/server/index.ts deleted file mode 100644 index 371836515..000000000 --- a/packages/peertube-runner/server/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './server' diff --git a/packages/peertube-runner/server/process/index.ts b/packages/peertube-runner/server/process/index.ts deleted file mode 100644 index 6caedbdaf..000000000 --- a/packages/peertube-runner/server/process/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './shared' -export * from './process' diff --git a/packages/peertube-runner/server/process/process.ts b/packages/peertube-runner/server/process/process.ts deleted file mode 100644 index 1caafda8c..000000000 --- a/packages/peertube-runner/server/process/process.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { logger } from 'packages/peertube-runner/shared/logger' -import { - RunnerJobLiveRTMPHLSTranscodingPayload, - RunnerJobStudioTranscodingPayload, - RunnerJobVODAudioMergeTranscodingPayload, - RunnerJobVODHLSTranscodingPayload, - RunnerJobVODWebVideoTranscodingPayload -} from '@shared/models' -import { processAudioMergeTranscoding, processHLSTranscoding, ProcessOptions, processWebVideoTranscoding } from './shared' -import { ProcessLiveRTMPHLSTranscoding } from './shared/process-live' -import { processStudioTranscoding } from './shared/process-studio' - -export async function processJob (options: ProcessOptions) { - const { server, job } = options - - logger.info(`[${server.url}] Processing job of type ${job.type}: ${job.uuid}`, { payload: job.payload }) - - if (job.type === 'vod-audio-merge-transcoding') { - await processAudioMergeTranscoding(options as ProcessOptions) - } else if (job.type === 'vod-web-video-transcoding') { - await processWebVideoTranscoding(options as ProcessOptions) - } else if (job.type === 'vod-hls-transcoding') { - await processHLSTranscoding(options as ProcessOptions) - } else if (job.type === 'live-rtmp-hls-transcoding') { - await new ProcessLiveRTMPHLSTranscoding(options as ProcessOptions).process() - } else if (job.type === 'video-studio-transcoding') { - await processStudioTranscoding(options as ProcessOptions) - } else { - logger.error(`Unknown job ${job.type} to process`) - return - } - - logger.info(`[${server.url}] Finished processing job of type ${job.type}: ${job.uuid}`) -} diff --git a/packages/peertube-runner/server/process/shared/common.ts b/packages/peertube-runner/server/process/shared/common.ts deleted file mode 100644 index a9b37bbc4..000000000 --- a/packages/peertube-runner/server/process/shared/common.ts +++ /dev/null @@ -1,106 +0,0 @@ -import { remove } from 'fs-extra' -import { ConfigManager, downloadFile, logger } from 'packages/peertube-runner/shared' -import { join } from 'path' -import { buildUUID } from '@shared/extra-utils' -import { FFmpegEdition, FFmpegLive, FFmpegVOD, getDefaultAvailableEncoders, getDefaultEncodersToTry } from '@shared/ffmpeg' -import { RunnerJob, RunnerJobPayload } from '@shared/models' -import { PeerTubeServer } from '@shared/server-commands' -import { getTranscodingLogger } from './transcoding-logger' - -export type JobWithToken = RunnerJob & { jobToken: string } - -export type ProcessOptions = { - server: PeerTubeServer - job: JobWithToken - runnerToken: string -} - -export async function downloadInputFile (options: { - url: string - job: JobWithToken - runnerToken: string -}) { - const { url, job, runnerToken } = options - const destination = join(ConfigManager.Instance.getTranscodingDirectory(), buildUUID()) - - try { - await downloadFile({ url, jobToken: job.jobToken, runnerToken, destination }) - } catch (err) { - remove(destination) - .catch(err => logger.error({ err }, `Cannot remove ${destination}`)) - - throw err - } - - return destination -} - -export function scheduleTranscodingProgress (options: { - server: PeerTubeServer - runnerToken: string - job: JobWithToken - progressGetter: () => number -}) { - const { job, server, progressGetter, runnerToken } = options - - const updateInterval = ConfigManager.Instance.isTestInstance() - ? 500 - : 60000 - - const update = () => { - server.runnerJobs.update({ jobToken: job.jobToken, jobUUID: job.uuid, runnerToken, progress: progressGetter() }) - .catch(err => logger.error({ err }, 'Cannot send job progress')) - } - - const interval = setInterval(() => { - update() - }, updateInterval) - - update() - - return interval -} - -// --------------------------------------------------------------------------- - -export function buildFFmpegVOD (options: { - onJobProgress: (progress: number) => void -}) { - const { onJobProgress } = options - - return new FFmpegVOD({ - ...getCommonFFmpegOptions(), - - updateJobProgress: arg => { - const progress = arg < 0 || arg > 100 - ? undefined - : arg - - onJobProgress(progress) - } - }) -} - -export function buildFFmpegLive () { - return new FFmpegLive(getCommonFFmpegOptions()) -} - -export function buildFFmpegEdition () { - return new FFmpegEdition(getCommonFFmpegOptions()) -} - -function getCommonFFmpegOptions () { - const config = ConfigManager.Instance.getConfig() - - return { - niceness: config.ffmpeg.nice, - threads: config.ffmpeg.threads, - tmpDirectory: ConfigManager.Instance.getTranscodingDirectory(), - profile: 'default', - availableEncoders: { - available: getDefaultAvailableEncoders(), - encodersToTry: getDefaultEncodersToTry() - }, - logger: getTranscodingLogger() - } -} diff --git a/packages/peertube-runner/server/process/shared/index.ts b/packages/peertube-runner/server/process/shared/index.ts deleted file mode 100644 index 556c51365..000000000 --- a/packages/peertube-runner/server/process/shared/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './common' -export * from './process-vod' -export * from './transcoding-logger' diff --git a/packages/peertube-runner/server/process/shared/process-live.ts b/packages/peertube-runner/server/process/shared/process-live.ts deleted file mode 100644 index e1fc0e34e..000000000 --- a/packages/peertube-runner/server/process/shared/process-live.ts +++ /dev/null @@ -1,338 +0,0 @@ -import { FSWatcher, watch } from 'chokidar' -import { FfmpegCommand } from 'fluent-ffmpeg' -import { ensureDir, remove } from 'fs-extra' -import { logger } from 'packages/peertube-runner/shared' -import { basename, join } from 'path' -import { wait } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { ffprobePromise, getVideoStreamBitrate, getVideoStreamDimensionsInfo, hasAudioStream } from '@shared/ffmpeg' -import { - LiveRTMPHLSTranscodingSuccess, - LiveRTMPHLSTranscodingUpdatePayload, - PeerTubeProblemDocument, - RunnerJobLiveRTMPHLSTranscodingPayload, - ServerErrorCode -} from '@shared/models' -import { ConfigManager } from '../../../shared/config-manager' -import { buildFFmpegLive, ProcessOptions } from './common' - -export class ProcessLiveRTMPHLSTranscoding { - - private readonly outputPath: string - private readonly fsWatchers: FSWatcher[] = [] - - // Playlist name -> chunks - private readonly pendingChunksPerPlaylist = new Map() - - private readonly playlistsCreated = new Set() - private allPlaylistsCreated = false - - private ffmpegCommand: FfmpegCommand - - private ended = false - private errored = false - - constructor (private readonly options: ProcessOptions) { - this.outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), buildUUID()) - - logger.debug(`Using ${this.outputPath} to process live rtmp hls transcoding job ${options.job.uuid}`) - } - - process () { - const job = this.options.job - const payload = job.payload - - return new Promise(async (res, rej) => { - try { - await ensureDir(this.outputPath) - - logger.info(`Probing ${payload.input.rtmpUrl}`) - const probe = await ffprobePromise(payload.input.rtmpUrl) - logger.info({ probe }, `Probed ${payload.input.rtmpUrl}`) - - const hasAudio = await hasAudioStream(payload.input.rtmpUrl, probe) - const bitrate = await getVideoStreamBitrate(payload.input.rtmpUrl, probe) - const { ratio } = await getVideoStreamDimensionsInfo(payload.input.rtmpUrl, probe) - - const m3u8Watcher = watch(this.outputPath + '/*.m3u8') - this.fsWatchers.push(m3u8Watcher) - - const tsWatcher = watch(this.outputPath + '/*.ts') - this.fsWatchers.push(tsWatcher) - - m3u8Watcher.on('change', p => { - logger.debug(`${p} m3u8 playlist changed`) - }) - - m3u8Watcher.on('add', p => { - this.playlistsCreated.add(p) - - if (this.playlistsCreated.size === this.options.job.payload.output.toTranscode.length + 1) { - this.allPlaylistsCreated = true - logger.info('All m3u8 playlists are created.') - } - }) - - tsWatcher.on('add', async p => { - try { - await this.sendPendingChunks() - } catch (err) { - this.onUpdateError({ err, rej, res }) - } - - const playlistName = this.getPlaylistIdFromTS(p) - - const pendingChunks = this.pendingChunksPerPlaylist.get(playlistName) || [] - pendingChunks.push(p) - - this.pendingChunksPerPlaylist.set(playlistName, pendingChunks) - }) - - tsWatcher.on('unlink', p => { - this.sendDeletedChunkUpdate(p) - .catch(err => this.onUpdateError({ err, rej, res })) - }) - - this.ffmpegCommand = await buildFFmpegLive().getLiveTranscodingCommand({ - inputUrl: payload.input.rtmpUrl, - - outPath: this.outputPath, - masterPlaylistName: 'master.m3u8', - - segmentListSize: payload.output.segmentListSize, - segmentDuration: payload.output.segmentDuration, - - toTranscode: payload.output.toTranscode, - - bitrate, - ratio, - - hasAudio - }) - - logger.info(`Running live transcoding for ${payload.input.rtmpUrl}`) - - this.ffmpegCommand.on('error', (err, stdout, stderr) => { - this.onFFmpegError({ err, stdout, stderr }) - - res() - }) - - this.ffmpegCommand.on('end', () => { - this.onFFmpegEnded() - .catch(err => logger.error({ err }, 'Error in FFmpeg end handler')) - - res() - }) - - this.ffmpegCommand.run() - } catch (err) { - rej(err) - } - }) - } - - // --------------------------------------------------------------------------- - - private onUpdateError (options: { - err: Error - res: () => void - rej: (reason?: any) => void - }) { - const { err, res, rej } = options - - if (this.errored) return - if (this.ended) return - - this.errored = true - - this.ffmpegCommand.kill('SIGINT') - - const type = ((err as any).res?.body as PeerTubeProblemDocument)?.code - if (type === ServerErrorCode.RUNNER_JOB_NOT_IN_PROCESSING_STATE) { - logger.info({ err }, 'Stopping transcoding as the job is not in processing state anymore') - - res() - } else { - logger.error({ err }, 'Cannot send update after added/deleted chunk, stopping live transcoding') - - this.sendError(err) - .catch(subErr => logger.error({ err: subErr }, 'Cannot send error')) - - rej(err) - } - - this.cleanup() - } - - // --------------------------------------------------------------------------- - - private onFFmpegError (options: { - err: any - stdout: string - stderr: string - }) { - const { err, stdout, stderr } = options - - // Don't care that we killed the ffmpeg process - if (err?.message?.includes('Exiting normally')) return - if (this.errored) return - if (this.ended) return - - this.errored = true - - logger.error({ err, stdout, stderr }, 'FFmpeg transcoding error.') - - this.sendError(err) - .catch(subErr => logger.error({ err: subErr }, 'Cannot send error')) - - this.cleanup() - } - - private async sendError (err: Error) { - await this.options.server.runnerJobs.error({ - jobToken: this.options.job.jobToken, - jobUUID: this.options.job.uuid, - runnerToken: this.options.runnerToken, - message: err.message - }) - } - - // --------------------------------------------------------------------------- - - private async onFFmpegEnded () { - if (this.ended) return - - this.ended = true - logger.info('FFmpeg ended, sending success to server') - - // Wait last ffmpeg chunks generation - await wait(1500) - - this.sendSuccess() - .catch(err => logger.error({ err }, 'Cannot send success')) - - this.cleanup() - } - - private async sendSuccess () { - const successBody: LiveRTMPHLSTranscodingSuccess = {} - - await this.options.server.runnerJobs.success({ - jobToken: this.options.job.jobToken, - jobUUID: this.options.job.uuid, - runnerToken: this.options.runnerToken, - payload: successBody - }) - } - - // --------------------------------------------------------------------------- - - private sendDeletedChunkUpdate (deletedChunk: string): Promise { - if (this.ended) return Promise.resolve() - - logger.debug(`Sending removed live chunk ${deletedChunk} update`) - - const videoChunkFilename = basename(deletedChunk) - - let payload: LiveRTMPHLSTranscodingUpdatePayload = { - type: 'remove-chunk', - videoChunkFilename - } - - if (this.allPlaylistsCreated) { - const playlistName = this.getPlaylistName(videoChunkFilename) - - payload = { - ...payload, - masterPlaylistFile: join(this.outputPath, 'master.m3u8'), - resolutionPlaylistFilename: playlistName, - resolutionPlaylistFile: join(this.outputPath, playlistName) - } - } - - return this.updateWithRetry(payload) - } - - private async sendPendingChunks (): Promise { - if (this.ended) return Promise.resolve() - - const promises: Promise[] = [] - - for (const playlist of this.pendingChunksPerPlaylist.keys()) { - for (const chunk of this.pendingChunksPerPlaylist.get(playlist)) { - logger.debug(`Sending added live chunk ${chunk} update`) - - const videoChunkFilename = basename(chunk) - - let payload: LiveRTMPHLSTranscodingUpdatePayload = { - type: 'add-chunk', - videoChunkFilename, - videoChunkFile: chunk - } - - if (this.allPlaylistsCreated) { - const playlistName = this.getPlaylistName(videoChunkFilename) - - payload = { - ...payload, - masterPlaylistFile: join(this.outputPath, 'master.m3u8'), - resolutionPlaylistFilename: playlistName, - resolutionPlaylistFile: join(this.outputPath, playlistName) - } - } - - promises.push(this.updateWithRetry(payload)) - } - - this.pendingChunksPerPlaylist.set(playlist, []) - } - - await Promise.all(promises) - } - - private async updateWithRetry (payload: LiveRTMPHLSTranscodingUpdatePayload, currentTry = 1): Promise { - if (this.ended || this.errored) return - - try { - await this.options.server.runnerJobs.update({ - jobToken: this.options.job.jobToken, - jobUUID: this.options.job.uuid, - runnerToken: this.options.runnerToken, - payload - }) - } catch (err) { - if (currentTry >= 3) throw err - if ((err.res?.body as PeerTubeProblemDocument)?.code === ServerErrorCode.RUNNER_JOB_NOT_IN_PROCESSING_STATE) throw err - - logger.warn({ err }, 'Will retry update after error') - await wait(250) - - return this.updateWithRetry(payload, currentTry + 1) - } - } - - private getPlaylistName (videoChunkFilename: string) { - return `${videoChunkFilename.split('-')[0]}.m3u8` - } - - private getPlaylistIdFromTS (segmentPath: string) { - const playlistIdMatcher = /^([\d+])-/ - - return basename(segmentPath).match(playlistIdMatcher)[1] - } - - // --------------------------------------------------------------------------- - - private cleanup () { - logger.debug(`Cleaning up job ${this.options.job.uuid}`) - - for (const fsWatcher of this.fsWatchers) { - fsWatcher.close() - .catch(err => logger.error({ err }, 'Cannot close watcher')) - } - - remove(this.outputPath) - .catch(err => logger.error({ err }, `Cannot remove ${this.outputPath}`)) - } -} diff --git a/packages/peertube-runner/server/process/shared/process-studio.ts b/packages/peertube-runner/server/process/shared/process-studio.ts deleted file mode 100644 index 7bb209e80..000000000 --- a/packages/peertube-runner/server/process/shared/process-studio.ts +++ /dev/null @@ -1,165 +0,0 @@ -import { remove } from 'fs-extra' -import { logger } from 'packages/peertube-runner/shared' -import { join } from 'path' -import { pick } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { - RunnerJobStudioTranscodingPayload, - VideoStudioTask, - VideoStudioTaskCutPayload, - VideoStudioTaskIntroPayload, - VideoStudioTaskOutroPayload, - VideoStudioTaskPayload, - VideoStudioTaskWatermarkPayload, - VideoStudioTranscodingSuccess -} from '@shared/models' -import { ConfigManager } from '../../../shared/config-manager' -import { buildFFmpegEdition, downloadInputFile, JobWithToken, ProcessOptions, scheduleTranscodingProgress } from './common' - -export async function processStudioTranscoding (options: ProcessOptions) { - const { server, job, runnerToken } = options - const payload = job.payload - - let inputPath: string - let outputPath: string - let tmpInputFilePath: string - - let tasksProgress = 0 - - const updateProgressInterval = scheduleTranscodingProgress({ - job, - server, - runnerToken, - progressGetter: () => tasksProgress - }) - - try { - logger.info(`Downloading input file ${payload.input.videoFileUrl} for job ${job.jobToken}`) - - inputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job }) - tmpInputFilePath = inputPath - - logger.info(`Input file ${payload.input.videoFileUrl} downloaded for job ${job.jobToken}. Running studio transcoding tasks.`) - - for (const task of payload.tasks) { - const outputFilename = 'output-edition-' + buildUUID() + '.mp4' - outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), outputFilename) - - await processTask({ - inputPath: tmpInputFilePath, - outputPath, - task, - job, - runnerToken - }) - - if (tmpInputFilePath) await remove(tmpInputFilePath) - - // For the next iteration - tmpInputFilePath = outputPath - - tasksProgress += Math.floor(100 / payload.tasks.length) - } - - const successBody: VideoStudioTranscodingSuccess = { - videoFile: outputPath - } - - await server.runnerJobs.success({ - jobToken: job.jobToken, - jobUUID: job.uuid, - runnerToken, - payload: successBody - }) - } finally { - if (tmpInputFilePath) await remove(tmpInputFilePath) - if (outputPath) await remove(outputPath) - if (updateProgressInterval) clearInterval(updateProgressInterval) - } -} - -// --------------------------------------------------------------------------- -// Private -// --------------------------------------------------------------------------- - -type TaskProcessorOptions = { - inputPath: string - outputPath: string - task: T - runnerToken: string - job: JobWithToken -} - -const taskProcessors: { [id in VideoStudioTask['name']]: (options: TaskProcessorOptions) => Promise } = { - 'add-intro': processAddIntroOutro, - 'add-outro': processAddIntroOutro, - 'cut': processCut, - 'add-watermark': processAddWatermark -} - -async function processTask (options: TaskProcessorOptions) { - const { task } = options - - const processor = taskProcessors[options.task.name] - if (!process) throw new Error('Unknown task ' + task.name) - - return processor(options) -} - -async function processAddIntroOutro (options: TaskProcessorOptions) { - const { inputPath, task, runnerToken, job } = options - - logger.debug('Adding intro/outro to ' + inputPath) - - const introOutroPath = await downloadInputFile({ url: task.options.file, runnerToken, job }) - - try { - await buildFFmpegEdition().addIntroOutro({ - ...pick(options, [ 'inputPath', 'outputPath' ]), - - introOutroPath, - type: task.name === 'add-intro' - ? 'intro' - : 'outro' - }) - } finally { - await remove(introOutroPath) - } -} - -function processCut (options: TaskProcessorOptions) { - const { inputPath, task } = options - - logger.debug(`Cutting ${inputPath}`) - - return buildFFmpegEdition().cutVideo({ - ...pick(options, [ 'inputPath', 'outputPath' ]), - - start: task.options.start, - end: task.options.end - }) -} - -async function processAddWatermark (options: TaskProcessorOptions) { - const { inputPath, task, runnerToken, job } = options - - logger.debug('Adding watermark to ' + inputPath) - - const watermarkPath = await downloadInputFile({ url: task.options.file, runnerToken, job }) - - try { - await buildFFmpegEdition().addWatermark({ - ...pick(options, [ 'inputPath', 'outputPath' ]), - - watermarkPath, - - videoFilters: { - watermarkSizeRatio: task.options.watermarkSizeRatio, - horitonzalMarginRatio: task.options.horitonzalMarginRatio, - verticalMarginRatio: task.options.verticalMarginRatio - } - }) - } finally { - await remove(watermarkPath) - } -} diff --git a/packages/peertube-runner/server/process/shared/process-vod.ts b/packages/peertube-runner/server/process/shared/process-vod.ts deleted file mode 100644 index f7c076b27..000000000 --- a/packages/peertube-runner/server/process/shared/process-vod.ts +++ /dev/null @@ -1,201 +0,0 @@ -import { remove } from 'fs-extra' -import { logger } from 'packages/peertube-runner/shared' -import { join } from 'path' -import { buildUUID } from '@shared/extra-utils' -import { - RunnerJobVODAudioMergeTranscodingPayload, - RunnerJobVODHLSTranscodingPayload, - RunnerJobVODWebVideoTranscodingPayload, - VODAudioMergeTranscodingSuccess, - VODHLSTranscodingSuccess, - VODWebVideoTranscodingSuccess -} from '@shared/models' -import { ConfigManager } from '../../../shared/config-manager' -import { buildFFmpegVOD, downloadInputFile, ProcessOptions, scheduleTranscodingProgress } from './common' - -export async function processWebVideoTranscoding (options: ProcessOptions) { - const { server, job, runnerToken } = options - - const payload = job.payload - - let ffmpegProgress: number - let inputPath: string - - const outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), `output-${buildUUID()}.mp4`) - - const updateProgressInterval = scheduleTranscodingProgress({ - job, - server, - runnerToken, - progressGetter: () => ffmpegProgress - }) - - try { - logger.info(`Downloading input file ${payload.input.videoFileUrl} for web video transcoding job ${job.jobToken}`) - - inputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job }) - - logger.info(`Downloaded input file ${payload.input.videoFileUrl} for job ${job.jobToken}. Running web video transcoding.`) - - const ffmpegVod = buildFFmpegVOD({ - onJobProgress: progress => { ffmpegProgress = progress } - }) - - await ffmpegVod.transcode({ - type: 'video', - - inputPath, - - outputPath, - - inputFileMutexReleaser: () => {}, - - resolution: payload.output.resolution, - fps: payload.output.fps - }) - - const successBody: VODWebVideoTranscodingSuccess = { - videoFile: outputPath - } - - await server.runnerJobs.success({ - jobToken: job.jobToken, - jobUUID: job.uuid, - runnerToken, - payload: successBody - }) - } finally { - if (inputPath) await remove(inputPath) - if (outputPath) await remove(outputPath) - if (updateProgressInterval) clearInterval(updateProgressInterval) - } -} - -export async function processHLSTranscoding (options: ProcessOptions) { - const { server, job, runnerToken } = options - const payload = job.payload - - let ffmpegProgress: number - let inputPath: string - - const uuid = buildUUID() - const outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), `${uuid}-${payload.output.resolution}.m3u8`) - const videoFilename = `${uuid}-${payload.output.resolution}-fragmented.mp4` - const videoPath = join(join(ConfigManager.Instance.getTranscodingDirectory(), videoFilename)) - - const updateProgressInterval = scheduleTranscodingProgress({ - job, - server, - runnerToken, - progressGetter: () => ffmpegProgress - }) - - try { - logger.info(`Downloading input file ${payload.input.videoFileUrl} for HLS transcoding job ${job.jobToken}`) - - inputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job }) - - logger.info(`Downloaded input file ${payload.input.videoFileUrl} for job ${job.jobToken}. Running HLS transcoding.`) - - const ffmpegVod = buildFFmpegVOD({ - onJobProgress: progress => { ffmpegProgress = progress } - }) - - await ffmpegVod.transcode({ - type: 'hls', - copyCodecs: false, - inputPath, - hlsPlaylist: { videoFilename }, - outputPath, - - inputFileMutexReleaser: () => {}, - - resolution: payload.output.resolution, - fps: payload.output.fps - }) - - const successBody: VODHLSTranscodingSuccess = { - resolutionPlaylistFile: outputPath, - videoFile: videoPath - } - - await server.runnerJobs.success({ - jobToken: job.jobToken, - jobUUID: job.uuid, - runnerToken, - payload: successBody - }) - } finally { - if (inputPath) await remove(inputPath) - if (outputPath) await remove(outputPath) - if (videoPath) await remove(videoPath) - if (updateProgressInterval) clearInterval(updateProgressInterval) - } -} - -export async function processAudioMergeTranscoding (options: ProcessOptions) { - const { server, job, runnerToken } = options - const payload = job.payload - - let ffmpegProgress: number - let audioPath: string - let inputPath: string - - const outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), `output-${buildUUID()}.mp4`) - - const updateProgressInterval = scheduleTranscodingProgress({ - job, - server, - runnerToken, - progressGetter: () => ffmpegProgress - }) - - try { - logger.info( - `Downloading input files ${payload.input.audioFileUrl} and ${payload.input.previewFileUrl} ` + - `for audio merge transcoding job ${job.jobToken}` - ) - - audioPath = await downloadInputFile({ url: payload.input.audioFileUrl, runnerToken, job }) - inputPath = await downloadInputFile({ url: payload.input.previewFileUrl, runnerToken, job }) - - logger.info( - `Downloaded input files ${payload.input.audioFileUrl} and ${payload.input.previewFileUrl} ` + - `for job ${job.jobToken}. Running audio merge transcoding.` - ) - - const ffmpegVod = buildFFmpegVOD({ - onJobProgress: progress => { ffmpegProgress = progress } - }) - - await ffmpegVod.transcode({ - type: 'merge-audio', - - audioPath, - inputPath, - - outputPath, - - inputFileMutexReleaser: () => {}, - - resolution: payload.output.resolution, - fps: payload.output.fps - }) - - const successBody: VODAudioMergeTranscodingSuccess = { - videoFile: outputPath - } - - await server.runnerJobs.success({ - jobToken: job.jobToken, - jobUUID: job.uuid, - runnerToken, - payload: successBody - }) - } finally { - if (audioPath) await remove(audioPath) - if (inputPath) await remove(inputPath) - if (outputPath) await remove(outputPath) - if (updateProgressInterval) clearInterval(updateProgressInterval) - } -} diff --git a/packages/peertube-runner/server/process/shared/transcoding-logger.ts b/packages/peertube-runner/server/process/shared/transcoding-logger.ts deleted file mode 100644 index d0f928914..000000000 --- a/packages/peertube-runner/server/process/shared/transcoding-logger.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { logger } from 'packages/peertube-runner/shared/logger' - -export function getTranscodingLogger () { - return { - info: logger.info.bind(logger), - debug: logger.debug.bind(logger), - warn: logger.warn.bind(logger), - error: logger.error.bind(logger) - } -} diff --git a/packages/peertube-runner/server/server.ts b/packages/peertube-runner/server/server.ts deleted file mode 100644 index 5fa86fa1a..000000000 --- a/packages/peertube-runner/server/server.ts +++ /dev/null @@ -1,306 +0,0 @@ -import { ensureDir, readdir, remove } from 'fs-extra' -import { join } from 'path' -import { io, Socket } from 'socket.io-client' -import { pick, shuffle, wait } from '@shared/core-utils' -import { PeerTubeProblemDocument, ServerErrorCode } from '@shared/models' -import { PeerTubeServer as PeerTubeServerCommand } from '@shared/server-commands' -import { ConfigManager } from '../shared' -import { IPCServer } from '../shared/ipc' -import { logger } from '../shared/logger' -import { JobWithToken, processJob } from './process' -import { isJobSupported } from './shared' - -type PeerTubeServer = PeerTubeServerCommand & { - runnerToken: string - runnerName: string - runnerDescription?: string -} - -export class RunnerServer { - private static instance: RunnerServer - - private servers: PeerTubeServer[] = [] - private processingJobs: { job: JobWithToken, server: PeerTubeServer }[] = [] - - private checkingAvailableJobs = false - - private cleaningUp = false - - private readonly sockets = new Map() - - private constructor () {} - - async run () { - logger.info('Running PeerTube runner in server mode') - - await ConfigManager.Instance.load() - - for (const registered of ConfigManager.Instance.getConfig().registeredInstances) { - const serverCommand = new PeerTubeServerCommand({ url: registered.url }) - - this.loadServer(Object.assign(serverCommand, registered)) - - logger.info(`Loading registered instance ${registered.url}`) - } - - // Run IPC - const ipcServer = new IPCServer() - try { - await ipcServer.run(this) - } catch (err) { - logger.error('Cannot start local socket for IPC communication', err) - process.exit(-1) - } - - // Cleanup on exit - for (const code of [ 'SIGINT', 'SIGUSR1', 'SIGUSR2', 'uncaughtException' ]) { - process.on(code, async (err, origin) => { - if (code === 'uncaughtException') { - logger.error({ err, origin }, 'uncaughtException') - } - - await this.onExit() - }) - } - - // Process jobs - await ensureDir(ConfigManager.Instance.getTranscodingDirectory()) - await this.cleanupTMP() - - logger.info(`Using ${ConfigManager.Instance.getTranscodingDirectory()} for transcoding directory`) - - await this.checkAvailableJobs() - } - - // --------------------------------------------------------------------------- - - async registerRunner (options: { - url: string - registrationToken: string - runnerName: string - runnerDescription?: string - }) { - const { url, registrationToken, runnerName, runnerDescription } = options - - logger.info(`Registering runner ${runnerName} on ${url}...`) - - const serverCommand = new PeerTubeServerCommand({ url }) - const { runnerToken } = await serverCommand.runners.register({ name: runnerName, description: runnerDescription, registrationToken }) - - const server: PeerTubeServer = Object.assign(serverCommand, { - runnerToken, - runnerName, - runnerDescription - }) - - this.loadServer(server) - await this.saveRegisteredInstancesInConf() - - logger.info(`Registered runner ${runnerName} on ${url}`) - - await this.checkAvailableJobs() - } - - private loadServer (server: PeerTubeServer) { - this.servers.push(server) - - const url = server.url + '/runners' - const socket = io(url, { - auth: { - runnerToken: server.runnerToken - }, - transports: [ 'websocket' ] - }) - - socket.on('connect_error', err => logger.warn({ err }, `Cannot connect to ${url} socket`)) - socket.on('connect', () => logger.info(`Connected to ${url} socket`)) - socket.on('available-jobs', () => this.checkAvailableJobs()) - - this.sockets.set(server, socket) - } - - async unregisterRunner (options: { - url: string - runnerName: string - }) { - const { url, runnerName } = options - - const server = this.servers.find(s => s.url === url && s.runnerName === runnerName) - if (!server) { - logger.error(`Unknown server ${url} - ${runnerName} to unregister`) - return - } - - logger.info(`Unregistering runner ${runnerName} on ${url}...`) - - try { - await server.runners.unregister({ runnerToken: server.runnerToken }) - } catch (err) { - logger.error({ err }, `Cannot unregister runner ${runnerName} on ${url}`) - } - - this.unloadServer(server) - await this.saveRegisteredInstancesInConf() - - logger.info(`Unregistered runner ${runnerName} on ${url}`) - } - - private unloadServer (server: PeerTubeServer) { - this.servers = this.servers.filter(s => s !== server) - - const socket = this.sockets.get(server) - socket.disconnect() - - this.sockets.delete(server) - } - - listRegistered () { - return { - servers: this.servers.map(s => { - return { - url: s.url, - runnerName: s.runnerName, - runnerDescription: s.runnerDescription - } - }) - } - } - - // --------------------------------------------------------------------------- - - private async checkAvailableJobs () { - if (this.checkingAvailableJobs) return - - this.checkingAvailableJobs = true - - let hadAvailableJob = false - - for (const server of shuffle([ ...this.servers ])) { - try { - logger.info('Checking available jobs on ' + server.url) - - const job = await this.requestJob(server) - if (!job) continue - - hadAvailableJob = true - - await this.tryToExecuteJobAsync(server, job) - } catch (err) { - const code = (err.res?.body as PeerTubeProblemDocument)?.code - - if (code === ServerErrorCode.RUNNER_JOB_NOT_IN_PROCESSING_STATE) { - logger.debug({ err }, 'Runner job is not in processing state anymore, retry later') - return - } - - if (code === ServerErrorCode.UNKNOWN_RUNNER_TOKEN) { - logger.error({ err }, `Unregistering ${server.url} as the runner token ${server.runnerToken} is invalid`) - - await this.unregisterRunner({ url: server.url, runnerName: server.runnerName }) - return - } - - logger.error({ err }, `Cannot request/accept job on ${server.url} for runner ${server.runnerName}`) - } - } - - this.checkingAvailableJobs = false - - if (hadAvailableJob && this.canProcessMoreJobs()) { - await wait(2500) - - this.checkAvailableJobs() - .catch(err => logger.error({ err }, 'Cannot check more available jobs')) - } - } - - private async requestJob (server: PeerTubeServer) { - logger.debug(`Requesting jobs on ${server.url} for runner ${server.runnerName}`) - - const { availableJobs } = await server.runnerJobs.request({ runnerToken: server.runnerToken }) - - const filtered = availableJobs.filter(j => isJobSupported(j)) - - if (filtered.length === 0) { - logger.debug(`No job available on ${server.url} for runner ${server.runnerName}`) - return undefined - } - - return filtered[0] - } - - private async tryToExecuteJobAsync (server: PeerTubeServer, jobToAccept: { uuid: string }) { - if (!this.canProcessMoreJobs()) return - - const { job } = await server.runnerJobs.accept({ runnerToken: server.runnerToken, jobUUID: jobToAccept.uuid }) - - const processingJob = { job, server } - this.processingJobs.push(processingJob) - - processJob({ server, job, runnerToken: server.runnerToken }) - .catch(err => { - logger.error({ err }, 'Cannot process job') - - server.runnerJobs.error({ jobToken: job.jobToken, jobUUID: job.uuid, runnerToken: server.runnerToken, message: err.message }) - .catch(err2 => logger.error({ err: err2 }, 'Cannot abort job after error')) - }) - .finally(() => { - this.processingJobs = this.processingJobs.filter(p => p !== processingJob) - - return this.checkAvailableJobs() - }) - } - - // --------------------------------------------------------------------------- - - private saveRegisteredInstancesInConf () { - const data = this.servers.map(s => { - return pick(s, [ 'url', 'runnerToken', 'runnerName', 'runnerDescription' ]) - }) - - return ConfigManager.Instance.setRegisteredInstances(data) - } - - private canProcessMoreJobs () { - return this.processingJobs.length < ConfigManager.Instance.getConfig().jobs.concurrency - } - - // --------------------------------------------------------------------------- - - private async cleanupTMP () { - const files = await readdir(ConfigManager.Instance.getTranscodingDirectory()) - - for (const file of files) { - await remove(join(ConfigManager.Instance.getTranscodingDirectory(), file)) - } - } - - private async onExit () { - if (this.cleaningUp) return - this.cleaningUp = true - - logger.info('Cleaning up after program exit') - - try { - for (const { server, job } of this.processingJobs) { - await server.runnerJobs.abort({ - jobToken: job.jobToken, - jobUUID: job.uuid, - reason: 'Runner stopped', - runnerToken: server.runnerToken - }) - } - - await this.cleanupTMP() - } catch (err) { - logger.error(err) - process.exit(-1) - } - - process.exit() - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/packages/peertube-runner/server/shared/index.ts b/packages/peertube-runner/server/shared/index.ts deleted file mode 100644 index 5c86bafc0..000000000 --- a/packages/peertube-runner/server/shared/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './supported-job' diff --git a/packages/peertube-runner/server/shared/supported-job.ts b/packages/peertube-runner/server/shared/supported-job.ts deleted file mode 100644 index 1137d8206..000000000 --- a/packages/peertube-runner/server/shared/supported-job.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { - RunnerJobLiveRTMPHLSTranscodingPayload, - RunnerJobPayload, - RunnerJobType, - RunnerJobStudioTranscodingPayload, - RunnerJobVODAudioMergeTranscodingPayload, - RunnerJobVODHLSTranscodingPayload, - RunnerJobVODWebVideoTranscodingPayload, - VideoStudioTaskPayload -} from '@shared/models' - -const supportedMatrix = { - 'vod-web-video-transcoding': (_payload: RunnerJobVODWebVideoTranscodingPayload) => { - return true - }, - 'vod-hls-transcoding': (_payload: RunnerJobVODHLSTranscodingPayload) => { - return true - }, - 'vod-audio-merge-transcoding': (_payload: RunnerJobVODAudioMergeTranscodingPayload) => { - return true - }, - 'live-rtmp-hls-transcoding': (_payload: RunnerJobLiveRTMPHLSTranscodingPayload) => { - return true - }, - 'video-studio-transcoding': (payload: RunnerJobStudioTranscodingPayload) => { - const tasks = payload?.tasks - const supported = new Set([ 'add-intro', 'add-outro', 'add-watermark', 'cut' ]) - - if (!Array.isArray(tasks)) return false - - return tasks.every(t => t && supported.has(t.name)) - } -} - -export function isJobSupported (job: { - type: RunnerJobType - payload: RunnerJobPayload -}) { - const fn = supportedMatrix[job.type] - if (!fn) return false - - return fn(job.payload as any) -} diff --git a/packages/peertube-runner/shared/config-manager.ts b/packages/peertube-runner/shared/config-manager.ts deleted file mode 100644 index 548eeab85..000000000 --- a/packages/peertube-runner/shared/config-manager.ts +++ /dev/null @@ -1,139 +0,0 @@ -import envPaths from 'env-paths' -import { ensureDir, pathExists, readFile, remove, writeFile } from 'fs-extra' -import { merge } from 'lodash' -import { logger } from 'packages/peertube-runner/shared/logger' -import { dirname, join } from 'path' -import { parse, stringify } from '@iarna/toml' - -const paths = envPaths('peertube-runner') - -type Config = { - jobs: { - concurrency: number - } - - ffmpeg: { - threads: number - nice: number - } - - registeredInstances: { - url: string - runnerToken: string - runnerName: string - runnerDescription?: string - }[] -} - -export class ConfigManager { - private static instance: ConfigManager - - private config: Config = { - jobs: { - concurrency: 2 - }, - ffmpeg: { - threads: 2, - nice: 20 - }, - registeredInstances: [] - } - - private id: string - private configFilePath: string - - private constructor () {} - - init (id: string) { - this.id = id - this.configFilePath = join(this.getConfigDir(), 'config.toml') - } - - async load () { - logger.info(`Using ${this.configFilePath} as configuration file`) - - if (this.isTestInstance()) { - logger.info('Removing configuration file as we are using the "test" id') - await remove(this.configFilePath) - } - - await ensureDir(dirname(this.configFilePath)) - - if (!await pathExists(this.configFilePath)) { - await this.save() - } - - const file = await readFile(this.configFilePath, 'utf-8') - - this.config = merge(this.config, parse(file)) - } - - save () { - return writeFile(this.configFilePath, stringify(this.config)) - } - - // --------------------------------------------------------------------------- - - async setRegisteredInstances (registeredInstances: { - url: string - runnerToken: string - runnerName: string - runnerDescription?: string - }[]) { - this.config.registeredInstances = registeredInstances - - await this.save() - } - - // --------------------------------------------------------------------------- - - getConfig () { - return this.deepFreeze(this.config) - } - - // --------------------------------------------------------------------------- - - getTranscodingDirectory () { - return join(paths.cache, this.id, 'transcoding') - } - - getSocketDirectory () { - return join(paths.data, this.id) - } - - getSocketPath () { - return join(this.getSocketDirectory(), 'peertube-runner.sock') - } - - getConfigDir () { - return join(paths.config, this.id) - } - - // --------------------------------------------------------------------------- - - isTestInstance () { - return typeof this.id === 'string' && this.id.match(/^test-\d$/) - } - - // --------------------------------------------------------------------------- - - // Thanks: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/freeze - private deepFreeze (object: T) { - const propNames = Reflect.ownKeys(object) - - // Freeze properties before freezing self - for (const name of propNames) { - const value = object[name] - - if ((value && typeof value === 'object') || typeof value === 'function') { - this.deepFreeze(value) - } - } - - return Object.freeze({ ...object }) - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/packages/peertube-runner/shared/http.ts b/packages/peertube-runner/shared/http.ts deleted file mode 100644 index df64dc168..000000000 --- a/packages/peertube-runner/shared/http.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { createWriteStream, remove } from 'fs-extra' -import { request as requestHTTP } from 'http' -import { request as requestHTTPS, RequestOptions } from 'https' -import { logger } from './logger' - -export function downloadFile (options: { - url: string - destination: string - runnerToken: string - jobToken: string -}) { - const { url, destination, runnerToken, jobToken } = options - - logger.debug(`Downloading file ${url}`) - - return new Promise((res, rej) => { - const parsed = new URL(url) - - const body = JSON.stringify({ - runnerToken, - jobToken - }) - - const getOptions: RequestOptions = { - method: 'POST', - hostname: parsed.hostname, - port: parsed.port, - path: parsed.pathname, - headers: { - 'Content-Type': 'application/json', - 'Content-Length': Buffer.byteLength(body, 'utf-8') - } - } - - const request = getRequest(url)(getOptions, response => { - const code = response.statusCode ?? 0 - - if (code >= 400) { - return rej(new Error(response.statusMessage)) - } - - const file = createWriteStream(destination) - file.on('finish', () => res()) - - response.pipe(file) - }) - - request.on('error', err => { - remove(destination) - .catch(err => logger.error(err)) - - return rej(err) - }) - - request.write(body) - request.end() - }) -} - -// --------------------------------------------------------------------------- - -function getRequest (url: string) { - if (url.startsWith('https://')) return requestHTTPS - - return requestHTTP -} diff --git a/packages/peertube-runner/shared/index.ts b/packages/peertube-runner/shared/index.ts deleted file mode 100644 index d0b5a2e3e..000000000 --- a/packages/peertube-runner/shared/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './config-manager' -export * from './http' -export * from './logger' diff --git a/packages/peertube-runner/shared/ipc/index.ts b/packages/peertube-runner/shared/ipc/index.ts deleted file mode 100644 index ad4590281..000000000 --- a/packages/peertube-runner/shared/ipc/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './ipc-client' -export * from './ipc-server' diff --git a/packages/peertube-runner/shared/ipc/ipc-client.ts b/packages/peertube-runner/shared/ipc/ipc-client.ts deleted file mode 100644 index f8e72f97f..000000000 --- a/packages/peertube-runner/shared/ipc/ipc-client.ts +++ /dev/null @@ -1,88 +0,0 @@ -import CliTable3 from 'cli-table3' -import { ensureDir } from 'fs-extra' -import { Client as NetIPC } from 'net-ipc' -import { ConfigManager } from '../config-manager' -import { IPCReponse, IPCReponseData, IPCRequest } from './shared' - -export class IPCClient { - private netIPC: NetIPC - - async run () { - await ensureDir(ConfigManager.Instance.getSocketDirectory()) - - const socketPath = ConfigManager.Instance.getSocketPath() - - this.netIPC = new NetIPC({ path: socketPath }) - - try { - await this.netIPC.connect() - } catch (err) { - if (err.code === 'ECONNREFUSED') { - throw new Error( - 'This runner is not currently running in server mode on this system. ' + - 'Please run it using the `server` command first (in another terminal for example) and then retry your command.' - ) - } - - throw err - } - } - - async askRegister (options: { - url: string - registrationToken: string - runnerName: string - runnerDescription?: string - }) { - const req: IPCRequest = { - type: 'register', - ...options - } - - const { success, error } = await this.netIPC.request(req) as IPCReponse - - if (success) console.log('PeerTube instance registered') - else console.error('Could not register PeerTube instance on runner server side', error) - } - - async askUnregister (options: { - url: string - runnerName: string - }) { - const req: IPCRequest = { - type: 'unregister', - ...options - } - - const { success, error } = await this.netIPC.request(req) as IPCReponse - - if (success) console.log('PeerTube instance unregistered') - else console.error('Could not unregister PeerTube instance on runner server side', error) - } - - async askListRegistered () { - const req: IPCRequest = { - type: 'list-registered' - } - - const { success, error, data } = await this.netIPC.request(req) as IPCReponse - if (!success) { - console.error('Could not list registered PeerTube instances', error) - return - } - - const table = new CliTable3({ - head: [ 'instance', 'runner name', 'runner description' ] - }) - - for (const server of data.servers) { - table.push([ server.url, server.runnerName, server.runnerDescription ]) - } - - console.log(table.toString()) - } - - stop () { - this.netIPC.destroy() - } -} diff --git a/packages/peertube-runner/shared/ipc/ipc-server.ts b/packages/peertube-runner/shared/ipc/ipc-server.ts deleted file mode 100644 index 4b67d01ae..000000000 --- a/packages/peertube-runner/shared/ipc/ipc-server.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { ensureDir } from 'fs-extra' -import { Server as NetIPC } from 'net-ipc' -import { pick } from '@shared/core-utils' -import { RunnerServer } from '../../server' -import { ConfigManager } from '../config-manager' -import { logger } from '../logger' -import { IPCReponse, IPCReponseData, IPCRequest } from './shared' - -export class IPCServer { - private netIPC: NetIPC - private runnerServer: RunnerServer - - async run (runnerServer: RunnerServer) { - this.runnerServer = runnerServer - - await ensureDir(ConfigManager.Instance.getSocketDirectory()) - - const socketPath = ConfigManager.Instance.getSocketPath() - this.netIPC = new NetIPC({ path: socketPath }) - await this.netIPC.start() - - logger.info(`IPC socket created on ${socketPath}`) - - this.netIPC.on('request', async (req: IPCRequest, res) => { - try { - const data = await this.process(req) - - this.sendReponse(res, { success: true, data }) - } catch (err) { - logger.error('Cannot execute RPC call', err) - this.sendReponse(res, { success: false, error: err.message }) - } - }) - } - - private async process (req: IPCRequest) { - switch (req.type) { - case 'register': - await this.runnerServer.registerRunner(pick(req, [ 'url', 'registrationToken', 'runnerName', 'runnerDescription' ])) - return undefined - - case 'unregister': - await this.runnerServer.unregisterRunner(pick(req, [ 'url', 'runnerName' ])) - return undefined - - case 'list-registered': - return Promise.resolve(this.runnerServer.listRegistered()) - - default: - throw new Error('Unknown RPC call ' + (req as any).type) - } - } - - private sendReponse ( - response: (data: any) => Promise, - body: IPCReponse - ) { - response(body) - .catch(err => logger.error('Cannot send response after IPC request', err)) - } -} diff --git a/packages/peertube-runner/shared/ipc/shared/index.ts b/packages/peertube-runner/shared/ipc/shared/index.ts deleted file mode 100644 index deaaa152e..000000000 --- a/packages/peertube-runner/shared/ipc/shared/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './ipc-request.model' -export * from './ipc-response.model' diff --git a/packages/peertube-runner/shared/logger.ts b/packages/peertube-runner/shared/logger.ts deleted file mode 100644 index bf0f41828..000000000 --- a/packages/peertube-runner/shared/logger.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { pino } from 'pino' -import pretty from 'pino-pretty' - -const logger = pino(pretty({ - colorize: true -})) - -logger.level = 'info' - -export { - logger -} diff --git a/packages/peertube-runner/tsconfig.json b/packages/peertube-runner/tsconfig.json deleted file mode 100644 index b6c62bc34..000000000 --- a/packages/peertube-runner/tsconfig.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist" - }, - "references": [ - { "path": "../../shared" } - ] -} diff --git a/packages/server-commands/package.json b/packages/server-commands/package.json new file mode 100644 index 000000000..df9778198 --- /dev/null +++ b/packages/server-commands/package.json @@ -0,0 +1,19 @@ +{ + "name": "@peertube/peertube-server-commands", + "private": true, + "version": "0.0.0", + "main": "dist/index.js", + "files": [ "dist" ], + "exports": { + "types": "./dist/index.d.ts", + "peertube:tsx": "./src/index.ts", + "default": "./dist/index.js" + }, + "type": "module", + "devDependencies": {}, + "scripts": { + "build": "tsc", + "watch": "tsc -w" + }, + "dependencies": {} +} diff --git a/packages/server-commands/src/bulk/bulk-command.ts b/packages/server-commands/src/bulk/bulk-command.ts new file mode 100644 index 000000000..784836e19 --- /dev/null +++ b/packages/server-commands/src/bulk/bulk-command.ts @@ -0,0 +1,20 @@ +import { BulkRemoveCommentsOfBody, HttpStatusCode } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class BulkCommand extends AbstractCommand { + + removeCommentsOf (options: OverrideCommandOptions & { + attributes: BulkRemoveCommentsOfBody + }) { + const { attributes } = options + + return this.postBodyRequest({ + ...options, + + path: '/api/v1/bulk/remove-comments-of', + fields: attributes, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/packages/server-commands/src/bulk/index.ts b/packages/server-commands/src/bulk/index.ts new file mode 100644 index 000000000..903f7a282 --- /dev/null +++ b/packages/server-commands/src/bulk/index.ts @@ -0,0 +1 @@ +export * from './bulk-command.js' diff --git a/packages/server-commands/src/cli/cli-command.ts b/packages/server-commands/src/cli/cli-command.ts new file mode 100644 index 000000000..8b9400c85 --- /dev/null +++ b/packages/server-commands/src/cli/cli-command.ts @@ -0,0 +1,27 @@ +import { exec } from 'child_process' +import { AbstractCommand } from '../shared/index.js' + +export class CLICommand extends AbstractCommand { + + static exec (command: string) { + return new Promise((res, rej) => { + exec(command, (err, stdout, _stderr) => { + if (err) return rej(err) + + return res(stdout) + }) + }) + } + + getEnv () { + return `NODE_ENV=test NODE_APP_INSTANCE=${this.server.internalServerNumber}` + } + + async execWithEnv (command: string, configOverride?: any) { + const prefix = configOverride + ? `NODE_CONFIG='${JSON.stringify(configOverride)}'` + : '' + + return CLICommand.exec(`${prefix} ${this.getEnv()} ${command}`) + } +} diff --git a/packages/server-commands/src/cli/index.ts b/packages/server-commands/src/cli/index.ts new file mode 100644 index 000000000..d79b13a76 --- /dev/null +++ b/packages/server-commands/src/cli/index.ts @@ -0,0 +1 @@ +export * from './cli-command.js' diff --git a/packages/server-commands/src/custom-pages/custom-pages-command.ts b/packages/server-commands/src/custom-pages/custom-pages-command.ts new file mode 100644 index 000000000..412f3f763 --- /dev/null +++ b/packages/server-commands/src/custom-pages/custom-pages-command.ts @@ -0,0 +1,33 @@ +import { CustomPage, HttpStatusCode } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class CustomPagesCommand extends AbstractCommand { + + getInstanceHomepage (options: OverrideCommandOptions = {}) { + const path = '/api/v1/custom-pages/homepage/instance' + + return this.getRequestBody({ + ...options, + + path, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + updateInstanceHomepage (options: OverrideCommandOptions & { + content: string + }) { + const { content } = options + const path = '/api/v1/custom-pages/homepage/instance' + + return this.putBodyRequest({ + ...options, + + path, + fields: { content }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/packages/server-commands/src/custom-pages/index.ts b/packages/server-commands/src/custom-pages/index.ts new file mode 100644 index 000000000..67f537f07 --- /dev/null +++ b/packages/server-commands/src/custom-pages/index.ts @@ -0,0 +1 @@ +export * from './custom-pages-command.js' diff --git a/packages/server-commands/src/feeds/feeds-command.ts b/packages/server-commands/src/feeds/feeds-command.ts new file mode 100644 index 000000000..51bc45b7f --- /dev/null +++ b/packages/server-commands/src/feeds/feeds-command.ts @@ -0,0 +1,78 @@ +import { buildUUID } from '@peertube/peertube-node-utils' +import { HttpStatusCode } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +type FeedType = 'videos' | 'video-comments' | 'subscriptions' + +export class FeedCommand extends AbstractCommand { + + getXML (options: OverrideCommandOptions & { + feed: FeedType + ignoreCache: boolean + format?: string + }) { + const { feed, format, ignoreCache } = options + const path = '/feeds/' + feed + '.xml' + + const query: { [id: string]: string } = {} + + if (ignoreCache) query.v = buildUUID() + if (format) query.format = format + + return this.getRequestText({ + ...options, + + path, + query, + accept: 'application/xml', + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getPodcastXML (options: OverrideCommandOptions & { + ignoreCache: boolean + channelId: number + }) { + const { ignoreCache, channelId } = options + const path = `/feeds/podcast/videos.xml` + + const query: { [id: string]: string } = {} + + if (ignoreCache) query.v = buildUUID() + if (channelId) query.videoChannelId = channelId + '' + + return this.getRequestText({ + ...options, + + path, + query, + accept: 'application/xml', + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getJSON (options: OverrideCommandOptions & { + feed: FeedType + ignoreCache: boolean + query?: { [ id: string ]: any } + }) { + const { feed, query = {}, ignoreCache } = options + const path = '/feeds/' + feed + '.json' + + const cacheQuery = ignoreCache + ? { v: buildUUID() } + : {} + + return this.getRequestText({ + ...options, + + path, + query: { ...query, ...cacheQuery }, + accept: 'application/json', + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } +} diff --git a/packages/server-commands/src/feeds/index.ts b/packages/server-commands/src/feeds/index.ts new file mode 100644 index 000000000..316ebb974 --- /dev/null +++ b/packages/server-commands/src/feeds/index.ts @@ -0,0 +1 @@ +export * from './feeds-command.js' diff --git a/packages/server-commands/src/index.ts b/packages/server-commands/src/index.ts new file mode 100644 index 000000000..382fe966e --- /dev/null +++ b/packages/server-commands/src/index.ts @@ -0,0 +1,14 @@ +export * from './bulk/index.js' +export * from './cli/index.js' +export * from './custom-pages/index.js' +export * from './feeds/index.js' +export * from './logs/index.js' +export * from './moderation/index.js' +export * from './overviews/index.js' +export * from './requests/index.js' +export * from './runners/index.js' +export * from './search/index.js' +export * from './server/index.js' +export * from './socket/index.js' +export * from './users/index.js' +export * from './videos/index.js' diff --git a/packages/server-commands/src/logs/index.ts b/packages/server-commands/src/logs/index.ts new file mode 100644 index 000000000..37e77901c --- /dev/null +++ b/packages/server-commands/src/logs/index.ts @@ -0,0 +1 @@ +export * from './logs-command.js' diff --git a/packages/server-commands/src/logs/logs-command.ts b/packages/server-commands/src/logs/logs-command.ts new file mode 100644 index 000000000..d5d11b997 --- /dev/null +++ b/packages/server-commands/src/logs/logs-command.ts @@ -0,0 +1,56 @@ +import { ClientLogCreate, HttpStatusCode, ServerLogLevel } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class LogsCommand extends AbstractCommand { + + createLogClient (options: OverrideCommandOptions & { payload: ClientLogCreate }) { + const path = '/api/v1/server/logs/client' + + return this.postBodyRequest({ + ...options, + + path, + fields: options.payload, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + getLogs (options: OverrideCommandOptions & { + startDate: Date + endDate?: Date + level?: ServerLogLevel + tagsOneOf?: string[] + }) { + const { startDate, endDate, tagsOneOf, level } = options + const path = '/api/v1/server/logs' + + return this.getRequestBody({ + ...options, + + path, + query: { startDate, endDate, level, tagsOneOf }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getAuditLogs (options: OverrideCommandOptions & { + startDate: Date + endDate?: Date + }) { + const { startDate, endDate } = options + + const path = '/api/v1/server/audit-logs' + + return this.getRequestBody({ + ...options, + + path, + query: { startDate, endDate }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + +} diff --git a/packages/server-commands/src/moderation/abuses-command.ts b/packages/server-commands/src/moderation/abuses-command.ts new file mode 100644 index 000000000..e267709e2 --- /dev/null +++ b/packages/server-commands/src/moderation/abuses-command.ts @@ -0,0 +1,228 @@ +import { pick } from '@peertube/peertube-core-utils' +import { + AbuseFilter, + AbuseMessage, + AbusePredefinedReasonsString, + AbuseStateType, + AbuseUpdate, + AbuseVideoIs, + AdminAbuse, + HttpStatusCode, + ResultList, + UserAbuse +} from '@peertube/peertube-models' +import { unwrapBody } from '../requests/requests.js' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class AbusesCommand extends AbstractCommand { + + report (options: OverrideCommandOptions & { + reason: string + + accountId?: number + videoId?: number + commentId?: number + + predefinedReasons?: AbusePredefinedReasonsString[] + + startAt?: number + endAt?: number + }) { + const path = '/api/v1/abuses' + + const video = options.videoId + ? { + id: options.videoId, + startAt: options.startAt, + endAt: options.endAt + } + : undefined + + const comment = options.commentId + ? { id: options.commentId } + : undefined + + const account = options.accountId + ? { id: options.accountId } + : undefined + + const body = { + account, + video, + comment, + + reason: options.reason, + predefinedReasons: options.predefinedReasons + } + + return unwrapBody<{ abuse: { id: number } }>(this.postBodyRequest({ + ...options, + + path, + fields: body, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + } + + getAdminList (options: OverrideCommandOptions & { + start?: number + count?: number + sort?: string + + id?: number + predefinedReason?: AbusePredefinedReasonsString + search?: string + filter?: AbuseFilter + state?: AbuseStateType + videoIs?: AbuseVideoIs + searchReporter?: string + searchReportee?: string + searchVideo?: string + searchVideoChannel?: string + } = {}) { + const toPick: (keyof typeof options)[] = [ + 'count', + 'filter', + 'id', + 'predefinedReason', + 'search', + 'searchReportee', + 'searchReporter', + 'searchVideo', + 'searchVideoChannel', + 'sort', + 'start', + 'state', + 'videoIs' + ] + + const path = '/api/v1/abuses' + + const defaultQuery = { sort: 'createdAt' } + const query = { ...defaultQuery, ...pick(options, toPick) } + + return this.getRequestBody>({ + ...options, + + path, + query, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getUserList (options: OverrideCommandOptions & { + start?: number + count?: number + sort?: string + + id?: number + search?: string + state?: AbuseStateType + }) { + const toPick: (keyof typeof options)[] = [ + 'id', + 'search', + 'state', + 'start', + 'count', + 'sort' + ] + + const path = '/api/v1/users/me/abuses' + + const defaultQuery = { sort: 'createdAt' } + const query = { ...defaultQuery, ...pick(options, toPick) } + + return this.getRequestBody>({ + ...options, + + path, + query, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + update (options: OverrideCommandOptions & { + abuseId: number + body: AbuseUpdate + }) { + const { abuseId, body } = options + const path = '/api/v1/abuses/' + abuseId + + return this.putBodyRequest({ + ...options, + + path, + fields: body, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + delete (options: OverrideCommandOptions & { + abuseId: number + }) { + const { abuseId } = options + const path = '/api/v1/abuses/' + abuseId + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + listMessages (options: OverrideCommandOptions & { + abuseId: number + }) { + const { abuseId } = options + const path = '/api/v1/abuses/' + abuseId + '/messages' + + return this.getRequestBody>({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + deleteMessage (options: OverrideCommandOptions & { + abuseId: number + messageId: number + }) { + const { abuseId, messageId } = options + const path = '/api/v1/abuses/' + abuseId + '/messages/' + messageId + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + addMessage (options: OverrideCommandOptions & { + abuseId: number + message: string + }) { + const { abuseId, message } = options + const path = '/api/v1/abuses/' + abuseId + '/messages' + + return this.postBodyRequest({ + ...options, + + path, + fields: { message }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + +} diff --git a/packages/server-commands/src/moderation/index.ts b/packages/server-commands/src/moderation/index.ts new file mode 100644 index 000000000..8164afd7c --- /dev/null +++ b/packages/server-commands/src/moderation/index.ts @@ -0,0 +1 @@ +export * from './abuses-command.js' diff --git a/packages/server-commands/src/overviews/index.ts b/packages/server-commands/src/overviews/index.ts new file mode 100644 index 000000000..54c90705a --- /dev/null +++ b/packages/server-commands/src/overviews/index.ts @@ -0,0 +1 @@ +export * from './overviews-command.js' diff --git a/packages/server-commands/src/overviews/overviews-command.ts b/packages/server-commands/src/overviews/overviews-command.ts new file mode 100644 index 000000000..decd2fd8e --- /dev/null +++ b/packages/server-commands/src/overviews/overviews-command.ts @@ -0,0 +1,23 @@ +import { HttpStatusCode, VideosOverview } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class OverviewsCommand extends AbstractCommand { + + getVideos (options: OverrideCommandOptions & { + page: number + }) { + const { page } = options + const path = '/api/v1/overviews/videos' + + const query = { page } + + return this.getRequestBody({ + ...options, + + path, + query, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } +} diff --git a/packages/server-commands/src/requests/index.ts b/packages/server-commands/src/requests/index.ts new file mode 100644 index 000000000..4c818659e --- /dev/null +++ b/packages/server-commands/src/requests/index.ts @@ -0,0 +1 @@ +export * from './requests.js' diff --git a/packages/server-commands/src/requests/requests.ts b/packages/server-commands/src/requests/requests.ts new file mode 100644 index 000000000..ac143ea5d --- /dev/null +++ b/packages/server-commands/src/requests/requests.ts @@ -0,0 +1,260 @@ +/* eslint-disable @typescript-eslint/no-floating-promises */ + +import { decode } from 'querystring' +import request from 'supertest' +import { URL } from 'url' +import { pick } from '@peertube/peertube-core-utils' +import { HttpStatusCode, HttpStatusCodeType } from '@peertube/peertube-models' +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' + +export type CommonRequestParams = { + url: string + path?: string + contentType?: string + responseType?: string + range?: string + redirects?: number + accept?: string + host?: string + token?: string + headers?: { [ name: string ]: string } + type?: string + xForwardedFor?: string + expectedStatus?: HttpStatusCodeType +} + +function makeRawRequest (options: { + url: string + token?: string + expectedStatus?: HttpStatusCodeType + range?: string + query?: { [ id: string ]: string } + method?: 'GET' | 'POST' + headers?: { [ name: string ]: string } +}) { + const { host, protocol, pathname } = new URL(options.url) + + const reqOptions = { + url: `${protocol}//${host}`, + path: pathname, + contentType: undefined, + + ...pick(options, [ 'expectedStatus', 'range', 'token', 'query', 'headers' ]) + } + + if (options.method === 'POST') { + return makePostBodyRequest(reqOptions) + } + + return makeGetRequest(reqOptions) +} + +function makeGetRequest (options: CommonRequestParams & { + query?: any + rawQuery?: string +}) { + const req = request(options.url).get(options.path) + + if (options.query) req.query(options.query) + if (options.rawQuery) req.query(options.rawQuery) + + return buildRequest(req, { contentType: 'application/json', expectedStatus: HttpStatusCode.BAD_REQUEST_400, ...options }) +} + +function makeHTMLRequest (url: string, path: string) { + return makeGetRequest({ + url, + path, + accept: 'text/html', + expectedStatus: HttpStatusCode.OK_200 + }) +} + +function makeActivityPubGetRequest (url: string, path: string, expectedStatus: HttpStatusCodeType = HttpStatusCode.OK_200) { + return makeGetRequest({ + url, + path, + expectedStatus, + accept: 'application/activity+json,text/html;q=0.9,\\*/\\*;q=0.8' + }) +} + +function makeDeleteRequest (options: CommonRequestParams & { + query?: any + rawQuery?: string +}) { + const req = request(options.url).delete(options.path) + + if (options.query) req.query(options.query) + if (options.rawQuery) req.query(options.rawQuery) + + return buildRequest(req, { accept: 'application/json', expectedStatus: HttpStatusCode.BAD_REQUEST_400, ...options }) +} + +function makeUploadRequest (options: CommonRequestParams & { + method?: 'POST' | 'PUT' + + fields: { [ fieldName: string ]: any } + attaches?: { [ attachName: string ]: any | any[] } +}) { + let req = options.method === 'PUT' + ? request(options.url).put(options.path) + : request(options.url).post(options.path) + + req = buildRequest(req, { accept: 'application/json', expectedStatus: HttpStatusCode.BAD_REQUEST_400, ...options }) + + buildFields(req, options.fields) + + Object.keys(options.attaches || {}).forEach(attach => { + const value = options.attaches[attach] + if (!value) return + + if (Array.isArray(value)) { + req.attach(attach, buildAbsoluteFixturePath(value[0]), value[1]) + } else { + req.attach(attach, buildAbsoluteFixturePath(value)) + } + }) + + return req +} + +function makePostBodyRequest (options: CommonRequestParams & { + fields?: { [ fieldName: string ]: any } +}) { + const req = request(options.url).post(options.path) + .send(options.fields) + + return buildRequest(req, { accept: 'application/json', expectedStatus: HttpStatusCode.BAD_REQUEST_400, ...options }) +} + +function makePutBodyRequest (options: { + url: string + path: string + token?: string + fields: { [ fieldName: string ]: any } + expectedStatus?: HttpStatusCodeType + headers?: { [name: string]: string } +}) { + const req = request(options.url).put(options.path) + .send(options.fields) + + return buildRequest(req, { accept: 'application/json', expectedStatus: HttpStatusCode.BAD_REQUEST_400, ...options }) +} + +function decodeQueryString (path: string) { + return decode(path.split('?')[1]) +} + +// --------------------------------------------------------------------------- + +function unwrapBody (test: request.Test): Promise { + return test.then(res => res.body) +} + +function unwrapText (test: request.Test): Promise { + return test.then(res => res.text) +} + +function unwrapBodyOrDecodeToJSON (test: request.Test): Promise { + return test.then(res => { + if (res.body instanceof Buffer) { + try { + return JSON.parse(new TextDecoder().decode(res.body)) + } catch (err) { + console.error('Cannot decode JSON.', { res, body: res.body instanceof Buffer ? res.body.toString() : res.body }) + throw err + } + } + + if (res.text) { + try { + return JSON.parse(res.text) + } catch (err) { + console.error('Cannot decode json', { res, text: res.text }) + throw err + } + } + + return res.body + }) +} + +function unwrapTextOrDecode (test: request.Test): Promise { + return test.then(res => res.text || new TextDecoder().decode(res.body)) +} + +// --------------------------------------------------------------------------- + +export { + makeHTMLRequest, + makeGetRequest, + decodeQueryString, + makeUploadRequest, + makePostBodyRequest, + makePutBodyRequest, + makeDeleteRequest, + makeRawRequest, + makeActivityPubGetRequest, + unwrapBody, + unwrapTextOrDecode, + unwrapBodyOrDecodeToJSON, + unwrapText +} + +// --------------------------------------------------------------------------- + +function buildRequest (req: request.Test, options: CommonRequestParams) { + if (options.contentType) req.set('Accept', options.contentType) + if (options.responseType) req.responseType(options.responseType) + if (options.token) req.set('Authorization', 'Bearer ' + options.token) + if (options.range) req.set('Range', options.range) + if (options.accept) req.set('Accept', options.accept) + if (options.host) req.set('Host', options.host) + if (options.redirects) req.redirects(options.redirects) + if (options.xForwardedFor) req.set('X-Forwarded-For', options.xForwardedFor) + if (options.type) req.type(options.type) + + Object.keys(options.headers || {}).forEach(name => { + req.set(name, options.headers[name]) + }) + + return req.expect(res => { + if (options.expectedStatus && res.status !== options.expectedStatus) { + const err = new Error(`Expected status ${options.expectedStatus}, got ${res.status}. ` + + `\nThe server responded: "${res.body?.error ?? res.text}".\n` + + 'You may take a closer look at the logs. To see how to do so, check out this page: ' + + 'https://github.com/Chocobozzz/PeerTube/blob/develop/support/doc/development/tests.md#debug-server-logs'); + + (err as any).res = res + + throw err + } + + return res + }) +} + +function buildFields (req: request.Test, fields: { [ fieldName: string ]: any }, namespace?: string) { + if (!fields) return + + let formKey: string + + for (const key of Object.keys(fields)) { + if (namespace) formKey = `${namespace}[${key}]` + else formKey = key + + if (fields[key] === undefined) continue + + if (Array.isArray(fields[key]) && fields[key].length === 0) { + req.field(key, []) + continue + } + + if (fields[key] !== null && typeof fields[key] === 'object') { + buildFields(req, fields[key], formKey) + } else { + req.field(formKey, fields[key]) + } + } +} diff --git a/packages/server-commands/src/runners/index.ts b/packages/server-commands/src/runners/index.ts new file mode 100644 index 000000000..c868fa78e --- /dev/null +++ b/packages/server-commands/src/runners/index.ts @@ -0,0 +1,3 @@ +export * from './runner-jobs-command.js' +export * from './runner-registration-tokens-command.js' +export * from './runners-command.js' diff --git a/packages/server-commands/src/runners/runner-jobs-command.ts b/packages/server-commands/src/runners/runner-jobs-command.ts new file mode 100644 index 000000000..4e702199f --- /dev/null +++ b/packages/server-commands/src/runners/runner-jobs-command.ts @@ -0,0 +1,297 @@ +import { omit, pick, wait } from '@peertube/peertube-core-utils' +import { + AbortRunnerJobBody, + AcceptRunnerJobBody, + AcceptRunnerJobResult, + ErrorRunnerJobBody, + HttpStatusCode, + isHLSTranscodingPayloadSuccess, + isLiveRTMPHLSTranscodingUpdatePayload, + isWebVideoOrAudioMergeTranscodingPayloadSuccess, + ListRunnerJobsQuery, + RequestRunnerJobBody, + RequestRunnerJobResult, + ResultList, + RunnerJobAdmin, + RunnerJobLiveRTMPHLSTranscodingPayload, + RunnerJobPayload, + RunnerJobState, + RunnerJobStateType, + RunnerJobSuccessBody, + RunnerJobSuccessPayload, + RunnerJobType, + RunnerJobUpdateBody, + RunnerJobVODPayload, + VODHLSTranscodingSuccess, + VODWebVideoTranscodingSuccess +} from '@peertube/peertube-models' +import { unwrapBody } from '../requests/index.js' +import { waitJobs } from '../server/jobs.js' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class RunnerJobsCommand extends AbstractCommand { + + list (options: OverrideCommandOptions & ListRunnerJobsQuery = {}) { + const path = '/api/v1/runners/jobs' + + return this.getRequestBody>({ + ...options, + + path, + query: pick(options, [ 'start', 'count', 'sort', 'search', 'stateOneOf' ]), + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + cancelByAdmin (options: OverrideCommandOptions & { jobUUID: string }) { + const path = '/api/v1/runners/jobs/' + options.jobUUID + '/cancel' + + return this.postBodyRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + deleteByAdmin (options: OverrideCommandOptions & { jobUUID: string }) { + const path = '/api/v1/runners/jobs/' + options.jobUUID + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + // --------------------------------------------------------------------------- + + request (options: OverrideCommandOptions & RequestRunnerJobBody) { + const path = '/api/v1/runners/jobs/request' + + return unwrapBody(this.postBodyRequest({ + ...options, + + path, + fields: pick(options, [ 'runnerToken' ]), + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + } + + async requestVOD (options: OverrideCommandOptions & RequestRunnerJobBody) { + const vodTypes = new Set([ 'vod-audio-merge-transcoding', 'vod-hls-transcoding', 'vod-web-video-transcoding' ]) + + const { availableJobs } = await this.request(options) + + return { + availableJobs: availableJobs.filter(j => vodTypes.has(j.type)) + } as RequestRunnerJobResult + } + + async requestLive (options: OverrideCommandOptions & RequestRunnerJobBody) { + const vodTypes = new Set([ 'live-rtmp-hls-transcoding' ]) + + const { availableJobs } = await this.request(options) + + return { + availableJobs: availableJobs.filter(j => vodTypes.has(j.type)) + } as RequestRunnerJobResult + } + + // --------------------------------------------------------------------------- + + accept (options: OverrideCommandOptions & AcceptRunnerJobBody & { jobUUID: string }) { + const path = '/api/v1/runners/jobs/' + options.jobUUID + '/accept' + + return unwrapBody>(this.postBodyRequest({ + ...options, + + path, + fields: pick(options, [ 'runnerToken' ]), + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + } + + abort (options: OverrideCommandOptions & AbortRunnerJobBody & { jobUUID: string }) { + const path = '/api/v1/runners/jobs/' + options.jobUUID + '/abort' + + return this.postBodyRequest({ + ...options, + + path, + fields: pick(options, [ 'reason', 'jobToken', 'runnerToken' ]), + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + update (options: OverrideCommandOptions & RunnerJobUpdateBody & { jobUUID: string }) { + const path = '/api/v1/runners/jobs/' + options.jobUUID + '/update' + + const { payload } = options + const attaches: { [id: string]: any } = {} + let payloadWithoutFiles = payload + + if (isLiveRTMPHLSTranscodingUpdatePayload(payload)) { + if (payload.masterPlaylistFile) { + attaches[`payload[masterPlaylistFile]`] = payload.masterPlaylistFile + } + + attaches[`payload[resolutionPlaylistFile]`] = payload.resolutionPlaylistFile + attaches[`payload[videoChunkFile]`] = payload.videoChunkFile + + payloadWithoutFiles = omit(payloadWithoutFiles, [ 'masterPlaylistFile', 'resolutionPlaylistFile', 'videoChunkFile' ]) + } + + return this.postUploadRequest({ + ...options, + + path, + fields: { + ...pick(options, [ 'progress', 'jobToken', 'runnerToken' ]), + + payload: payloadWithoutFiles + }, + attaches, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + error (options: OverrideCommandOptions & ErrorRunnerJobBody & { jobUUID: string }) { + const path = '/api/v1/runners/jobs/' + options.jobUUID + '/error' + + return this.postBodyRequest({ + ...options, + + path, + fields: pick(options, [ 'message', 'jobToken', 'runnerToken' ]), + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + success (options: OverrideCommandOptions & RunnerJobSuccessBody & { jobUUID: string }) { + const { payload } = options + + const path = '/api/v1/runners/jobs/' + options.jobUUID + '/success' + const attaches: { [id: string]: any } = {} + let payloadWithoutFiles = payload + + if ((isWebVideoOrAudioMergeTranscodingPayloadSuccess(payload) || isHLSTranscodingPayloadSuccess(payload)) && payload.videoFile) { + attaches[`payload[videoFile]`] = payload.videoFile + + payloadWithoutFiles = omit(payloadWithoutFiles as VODWebVideoTranscodingSuccess, [ 'videoFile' ]) + } + + if (isHLSTranscodingPayloadSuccess(payload) && payload.resolutionPlaylistFile) { + attaches[`payload[resolutionPlaylistFile]`] = payload.resolutionPlaylistFile + + payloadWithoutFiles = omit(payloadWithoutFiles as VODHLSTranscodingSuccess, [ 'resolutionPlaylistFile' ]) + } + + return this.postUploadRequest({ + ...options, + + path, + attaches, + fields: { + ...pick(options, [ 'jobToken', 'runnerToken' ]), + + payload: payloadWithoutFiles + }, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + getJobFile (options: OverrideCommandOptions & { url: string, jobToken: string, runnerToken: string }) { + const { host, protocol, pathname } = new URL(options.url) + + return this.postBodyRequest({ + url: `${protocol}//${host}`, + path: pathname, + + fields: pick(options, [ 'jobToken', 'runnerToken' ]), + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + // --------------------------------------------------------------------------- + + async autoAccept (options: OverrideCommandOptions & RequestRunnerJobBody & { type?: RunnerJobType }) { + const { availableJobs } = await this.request(options) + + const job = options.type + ? availableJobs.find(j => j.type === options.type) + : availableJobs[0] + + return this.accept({ ...options, jobUUID: job.uuid }) + } + + async autoProcessWebVideoJob (runnerToken: string, jobUUIDToProcess?: string) { + let jobUUID = jobUUIDToProcess + + if (!jobUUID) { + const { availableJobs } = await this.request({ runnerToken }) + jobUUID = availableJobs[0].uuid + } + + const { job } = await this.accept({ runnerToken, jobUUID }) + const jobToken = job.jobToken + + const payload: RunnerJobSuccessPayload = { videoFile: 'video_short.mp4' } + await this.success({ runnerToken, jobUUID, jobToken, payload }) + + await waitJobs([ this.server ]) + + return job + } + + async cancelAllJobs (options: { state?: RunnerJobStateType } = {}) { + const { state } = options + + const { data } = await this.list({ count: 100 }) + + const allowedStates = new Set([ + RunnerJobState.PENDING, + RunnerJobState.PROCESSING, + RunnerJobState.WAITING_FOR_PARENT_JOB + ]) + + for (const job of data) { + if (state && job.state.id !== state) continue + else if (allowedStates.has(job.state.id) !== true) continue + + await this.cancelByAdmin({ jobUUID: job.uuid }) + } + } + + async getJob (options: OverrideCommandOptions & { uuid: string }) { + const { data } = await this.list({ ...options, count: 100, sort: '-updatedAt' }) + + return data.find(j => j.uuid === options.uuid) + } + + async requestLiveJob (runnerToken: string) { + let availableJobs: RequestRunnerJobResult['availableJobs'] = [] + + while (availableJobs.length === 0) { + const result = await this.requestLive({ runnerToken }) + availableJobs = result.availableJobs + + if (availableJobs.length === 1) break + + await wait(150) + } + + return availableJobs[0] + } +} diff --git a/packages/server-commands/src/runners/runner-registration-tokens-command.ts b/packages/server-commands/src/runners/runner-registration-tokens-command.ts new file mode 100644 index 000000000..86b6e5f93 --- /dev/null +++ b/packages/server-commands/src/runners/runner-registration-tokens-command.ts @@ -0,0 +1,55 @@ +import { pick } from '@peertube/peertube-core-utils' +import { HttpStatusCode, ResultList, RunnerRegistrationToken } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class RunnerRegistrationTokensCommand extends AbstractCommand { + + list (options: OverrideCommandOptions & { + start?: number + count?: number + sort?: string + } = {}) { + const path = '/api/v1/runners/registration-tokens' + + return this.getRequestBody>({ + ...options, + + path, + query: pick(options, [ 'start', 'count', 'sort' ]), + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + generate (options: OverrideCommandOptions = {}) { + const path = '/api/v1/runners/registration-tokens/generate' + + return this.postBodyRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + delete (options: OverrideCommandOptions & { + id: number + }) { + const path = '/api/v1/runners/registration-tokens/' + options.id + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + async getFirstRegistrationToken (options: OverrideCommandOptions = {}) { + const { data } = await this.list(options) + + return data[0].registrationToken + } +} diff --git a/packages/server-commands/src/runners/runners-command.ts b/packages/server-commands/src/runners/runners-command.ts new file mode 100644 index 000000000..376a1dff9 --- /dev/null +++ b/packages/server-commands/src/runners/runners-command.ts @@ -0,0 +1,85 @@ +import { pick } from '@peertube/peertube-core-utils' +import { + HttpStatusCode, + RegisterRunnerBody, + RegisterRunnerResult, + ResultList, + Runner, + UnregisterRunnerBody +} from '@peertube/peertube-models' +import { buildUUID } from '@peertube/peertube-node-utils' +import { unwrapBody } from '../requests/index.js' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class RunnersCommand extends AbstractCommand { + + list (options: OverrideCommandOptions & { + start?: number + count?: number + sort?: string + } = {}) { + const path = '/api/v1/runners' + + return this.getRequestBody>({ + ...options, + + path, + query: pick(options, [ 'start', 'count', 'sort' ]), + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + register (options: OverrideCommandOptions & RegisterRunnerBody) { + const path = '/api/v1/runners/register' + + return unwrapBody(this.postBodyRequest({ + ...options, + + path, + fields: pick(options, [ 'name', 'registrationToken', 'description' ]), + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + } + + unregister (options: OverrideCommandOptions & UnregisterRunnerBody) { + const path = '/api/v1/runners/unregister' + + return this.postBodyRequest({ + ...options, + + path, + fields: pick(options, [ 'runnerToken' ]), + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + delete (options: OverrideCommandOptions & { + id: number + }) { + const path = '/api/v1/runners/' + options.id + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + // --------------------------------------------------------------------------- + + async autoRegisterRunner () { + const { data } = await this.server.runnerRegistrationTokens.list({ sort: 'createdAt' }) + + const { runnerToken } = await this.register({ + name: 'runner ' + buildUUID(), + registrationToken: data[0].registrationToken + }) + + return runnerToken + } +} diff --git a/packages/server-commands/src/search/index.ts b/packages/server-commands/src/search/index.ts new file mode 100644 index 000000000..ca56fc669 --- /dev/null +++ b/packages/server-commands/src/search/index.ts @@ -0,0 +1 @@ +export * from './search-command.js' diff --git a/packages/server-commands/src/search/search-command.ts b/packages/server-commands/src/search/search-command.ts new file mode 100644 index 000000000..e766a2861 --- /dev/null +++ b/packages/server-commands/src/search/search-command.ts @@ -0,0 +1,98 @@ +import { + HttpStatusCode, + ResultList, + Video, + VideoChannel, + VideoChannelsSearchQuery, + VideoPlaylist, + VideoPlaylistsSearchQuery, + VideosSearchQuery +} from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class SearchCommand extends AbstractCommand { + + searchChannels (options: OverrideCommandOptions & { + search: string + }) { + return this.advancedChannelSearch({ + ...options, + + search: { search: options.search } + }) + } + + advancedChannelSearch (options: OverrideCommandOptions & { + search: VideoChannelsSearchQuery + }) { + const { search } = options + const path = '/api/v1/search/video-channels' + + return this.getRequestBody>({ + ...options, + + path, + query: search, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + searchPlaylists (options: OverrideCommandOptions & { + search: string + }) { + return this.advancedPlaylistSearch({ + ...options, + + search: { search: options.search } + }) + } + + advancedPlaylistSearch (options: OverrideCommandOptions & { + search: VideoPlaylistsSearchQuery + }) { + const { search } = options + const path = '/api/v1/search/video-playlists' + + return this.getRequestBody>({ + ...options, + + path, + query: search, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + searchVideos (options: OverrideCommandOptions & { + search: string + sort?: string + }) { + const { search, sort } = options + + return this.advancedVideoSearch({ + ...options, + + search: { + search, + sort: sort ?? '-publishedAt' + } + }) + } + + advancedVideoSearch (options: OverrideCommandOptions & { + search: VideosSearchQuery + }) { + const { search } = options + const path = '/api/v1/search/videos' + + return this.getRequestBody>({ + ...options, + + path, + query: search, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } +} diff --git a/packages/server-commands/src/server/config-command.ts b/packages/server-commands/src/server/config-command.ts new file mode 100644 index 000000000..8fcf0bd51 --- /dev/null +++ b/packages/server-commands/src/server/config-command.ts @@ -0,0 +1,576 @@ +import merge from 'lodash-es/merge.js' +import { About, CustomConfig, HttpStatusCode, ServerConfig } from '@peertube/peertube-models' +import { DeepPartial } from '@peertube/peertube-typescript-utils' +import { AbstractCommand, OverrideCommandOptions } from '../shared/abstract-command.js' + +export class ConfigCommand extends AbstractCommand { + + static getCustomConfigResolutions (enabled: boolean, with0p = false) { + return { + '0p': enabled && with0p, + '144p': enabled, + '240p': enabled, + '360p': enabled, + '480p': enabled, + '720p': enabled, + '1080p': enabled, + '1440p': enabled, + '2160p': enabled + } + } + + // --------------------------------------------------------------------------- + + static getEmailOverrideConfig (emailPort: number) { + return { + smtp: { + hostname: '127.0.0.1', + port: emailPort + } + } + } + + // --------------------------------------------------------------------------- + + enableSignup (requiresApproval: boolean, limit = -1) { + return this.updateExistingSubConfig({ + newConfig: { + signup: { + enabled: true, + requiresApproval, + limit + } + } + }) + } + + // --------------------------------------------------------------------------- + + disableImports () { + return this.setImportsEnabled(false) + } + + enableImports () { + return this.setImportsEnabled(true) + } + + private setImportsEnabled (enabled: boolean) { + return this.updateExistingSubConfig({ + newConfig: { + import: { + videos: { + http: { + enabled + }, + + torrent: { + enabled + } + } + } + } + }) + } + + // --------------------------------------------------------------------------- + + disableFileUpdate () { + return this.setFileUpdateEnabled(false) + } + + enableFileUpdate () { + return this.setFileUpdateEnabled(true) + } + + private setFileUpdateEnabled (enabled: boolean) { + return this.updateExistingSubConfig({ + newConfig: { + videoFile: { + update: { + enabled + } + } + } + }) + } + + // --------------------------------------------------------------------------- + + enableChannelSync () { + return this.setChannelSyncEnabled(true) + } + + disableChannelSync () { + return this.setChannelSyncEnabled(false) + } + + private setChannelSyncEnabled (enabled: boolean) { + return this.updateExistingSubConfig({ + newConfig: { + import: { + videoChannelSynchronization: { + enabled + } + } + } + }) + } + + // --------------------------------------------------------------------------- + + enableLive (options: { + allowReplay?: boolean + transcoding?: boolean + resolutions?: 'min' | 'max' // Default max + } = {}) { + const { allowReplay, transcoding, resolutions = 'max' } = options + + return this.updateExistingSubConfig({ + newConfig: { + live: { + enabled: true, + allowReplay: allowReplay ?? true, + transcoding: { + enabled: transcoding ?? true, + resolutions: ConfigCommand.getCustomConfigResolutions(resolutions === 'max') + } + } + } + }) + } + + disableTranscoding () { + return this.updateExistingSubConfig({ + newConfig: { + transcoding: { + enabled: false + }, + videoStudio: { + enabled: false + } + } + }) + } + + enableTranscoding (options: { + webVideo?: boolean // default true + hls?: boolean // default true + with0p?: boolean // default false + } = {}) { + const { webVideo = true, hls = true, with0p = false } = options + + return this.updateExistingSubConfig({ + newConfig: { + transcoding: { + enabled: true, + + allowAudioFiles: true, + allowAdditionalExtensions: true, + + resolutions: ConfigCommand.getCustomConfigResolutions(true, with0p), + + webVideos: { + enabled: webVideo + }, + hls: { + enabled: hls + } + } + } + }) + } + + enableMinimumTranscoding (options: { + webVideo?: boolean // default true + hls?: boolean // default true + } = {}) { + const { webVideo = true, hls = true } = options + + return this.updateExistingSubConfig({ + newConfig: { + transcoding: { + enabled: true, + + allowAudioFiles: true, + allowAdditionalExtensions: true, + + resolutions: { + ...ConfigCommand.getCustomConfigResolutions(false), + + '240p': true + }, + + webVideos: { + enabled: webVideo + }, + hls: { + enabled: hls + } + } + } + }) + } + + enableRemoteTranscoding () { + return this.updateExistingSubConfig({ + newConfig: { + transcoding: { + remoteRunners: { + enabled: true + } + }, + live: { + transcoding: { + remoteRunners: { + enabled: true + } + } + } + } + }) + } + + enableRemoteStudio () { + return this.updateExistingSubConfig({ + newConfig: { + videoStudio: { + remoteRunners: { + enabled: true + } + } + } + }) + } + + // --------------------------------------------------------------------------- + + enableStudio () { + return this.updateExistingSubConfig({ + newConfig: { + videoStudio: { + enabled: true + } + } + }) + } + + // --------------------------------------------------------------------------- + + getConfig (options: OverrideCommandOptions = {}) { + const path = '/api/v1/config' + + return this.getRequestBody({ + ...options, + + path, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + async getIndexHTMLConfig (options: OverrideCommandOptions = {}) { + const text = await this.getRequestText({ + ...options, + + path: '/', + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + + const match = text.match('') + + // We parse the string twice, first to extract the string and then to extract the JSON + return JSON.parse(JSON.parse(match[1])) as ServerConfig + } + + getAbout (options: OverrideCommandOptions = {}) { + const path = '/api/v1/config/about' + + return this.getRequestBody({ + ...options, + + path, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getCustomConfig (options: OverrideCommandOptions = {}) { + const path = '/api/v1/config/custom' + + return this.getRequestBody({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + updateCustomConfig (options: OverrideCommandOptions & { + newCustomConfig: CustomConfig + }) { + const path = '/api/v1/config/custom' + + return this.putBodyRequest({ + ...options, + + path, + fields: options.newCustomConfig, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + deleteCustomConfig (options: OverrideCommandOptions = {}) { + const path = '/api/v1/config/custom' + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + async updateExistingSubConfig (options: OverrideCommandOptions & { + newConfig: DeepPartial + }) { + const existing = await this.getCustomConfig({ ...options, expectedStatus: HttpStatusCode.OK_200 }) + + return this.updateCustomConfig({ ...options, newCustomConfig: merge({}, existing, options.newConfig) }) + } + + updateCustomSubConfig (options: OverrideCommandOptions & { + newConfig: DeepPartial + }) { + const newCustomConfig: CustomConfig = { + instance: { + name: 'PeerTube updated', + shortDescription: 'my short description', + description: 'my super description', + terms: 'my super terms', + codeOfConduct: 'my super coc', + + creationReason: 'my super creation reason', + moderationInformation: 'my super moderation information', + administrator: 'Kuja', + maintenanceLifetime: 'forever', + businessModel: 'my super business model', + hardwareInformation: '2vCore 3GB RAM', + + languages: [ 'en', 'es' ], + categories: [ 1, 2 ], + + isNSFW: true, + defaultNSFWPolicy: 'blur', + + defaultClientRoute: '/videos/recently-added', + + customizations: { + javascript: 'alert("coucou")', + css: 'body { background-color: red; }' + } + }, + theme: { + default: 'default' + }, + services: { + twitter: { + username: '@MySuperUsername', + whitelisted: true + } + }, + client: { + videos: { + miniature: { + preferAuthorDisplayName: false + } + }, + menu: { + login: { + redirectOnSingleExternalAuth: false + } + } + }, + cache: { + previews: { + size: 2 + }, + captions: { + size: 3 + }, + torrents: { + size: 4 + }, + storyboards: { + size: 5 + } + }, + signup: { + enabled: false, + limit: 5, + requiresApproval: true, + requiresEmailVerification: false, + minimumAge: 16 + }, + admin: { + email: 'superadmin1@example.com' + }, + contactForm: { + enabled: true + }, + user: { + history: { + videos: { + enabled: true + } + }, + videoQuota: 5242881, + videoQuotaDaily: 318742 + }, + videoChannels: { + maxPerUser: 20 + }, + transcoding: { + enabled: true, + remoteRunners: { + enabled: false + }, + allowAdditionalExtensions: true, + allowAudioFiles: true, + threads: 1, + concurrency: 3, + profile: 'default', + resolutions: { + '0p': false, + '144p': false, + '240p': false, + '360p': true, + '480p': true, + '720p': false, + '1080p': false, + '1440p': false, + '2160p': false + }, + alwaysTranscodeOriginalResolution: true, + webVideos: { + enabled: true + }, + hls: { + enabled: false + } + }, + live: { + enabled: true, + allowReplay: false, + latencySetting: { + enabled: false + }, + maxDuration: -1, + maxInstanceLives: -1, + maxUserLives: 50, + transcoding: { + enabled: true, + remoteRunners: { + enabled: false + }, + threads: 4, + profile: 'default', + resolutions: { + '144p': true, + '240p': true, + '360p': true, + '480p': true, + '720p': true, + '1080p': true, + '1440p': true, + '2160p': true + }, + alwaysTranscodeOriginalResolution: true + } + }, + videoStudio: { + enabled: false, + remoteRunners: { + enabled: false + } + }, + videoFile: { + update: { + enabled: false + } + }, + import: { + videos: { + concurrency: 3, + http: { + enabled: false + }, + torrent: { + enabled: false + } + }, + videoChannelSynchronization: { + enabled: false, + maxPerUser: 10 + } + }, + trending: { + videos: { + algorithms: { + enabled: [ 'hot', 'most-viewed', 'most-liked' ], + default: 'hot' + } + } + }, + autoBlacklist: { + videos: { + ofUsers: { + enabled: false + } + } + }, + followers: { + instance: { + enabled: true, + manualApproval: false + } + }, + followings: { + instance: { + autoFollowBack: { + enabled: false + }, + autoFollowIndex: { + indexUrl: 'https://instances.joinpeertube.org/api/v1/instances/hosts', + enabled: false + } + } + }, + broadcastMessage: { + enabled: true, + level: 'warning', + message: 'hello', + dismissable: true + }, + search: { + remoteUri: { + users: true, + anonymous: true + }, + searchIndex: { + enabled: true, + url: 'https://search.joinpeertube.org', + disableLocalSearch: true, + isDefaultSearch: true + } + } + } + + merge(newCustomConfig, options.newConfig) + + return this.updateCustomConfig({ ...options, newCustomConfig }) + } +} diff --git a/packages/server-commands/src/server/contact-form-command.ts b/packages/server-commands/src/server/contact-form-command.ts new file mode 100644 index 000000000..399e06d2f --- /dev/null +++ b/packages/server-commands/src/server/contact-form-command.ts @@ -0,0 +1,30 @@ +import { ContactForm, HttpStatusCode } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class ContactFormCommand extends AbstractCommand { + + send (options: OverrideCommandOptions & { + fromEmail: string + fromName: string + subject: string + body: string + }) { + const path = '/api/v1/server/contact' + + const body: ContactForm = { + fromEmail: options.fromEmail, + fromName: options.fromName, + subject: options.subject, + body: options.body + } + + return this.postBodyRequest({ + ...options, + + path, + fields: body, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/packages/server-commands/src/server/debug-command.ts b/packages/server-commands/src/server/debug-command.ts new file mode 100644 index 000000000..9bb7fda10 --- /dev/null +++ b/packages/server-commands/src/server/debug-command.ts @@ -0,0 +1,33 @@ +import { Debug, HttpStatusCode, SendDebugCommand } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class DebugCommand extends AbstractCommand { + + getDebug (options: OverrideCommandOptions = {}) { + const path = '/api/v1/server/debug' + + return this.getRequestBody({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + sendCommand (options: OverrideCommandOptions & { + body: SendDebugCommand + }) { + const { body } = options + const path = '/api/v1/server/debug/run-command' + + return this.postBodyRequest({ + ...options, + + path, + fields: body, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/packages/server-commands/src/server/follows-command.ts b/packages/server-commands/src/server/follows-command.ts new file mode 100644 index 000000000..cdc263982 --- /dev/null +++ b/packages/server-commands/src/server/follows-command.ts @@ -0,0 +1,139 @@ +import { pick } from '@peertube/peertube-core-utils' +import { ActivityPubActorType, ActorFollow, FollowState, HttpStatusCode, ResultList, ServerFollowCreate } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' +import { PeerTubeServer } from './server.js' + +export class FollowsCommand extends AbstractCommand { + + getFollowers (options: OverrideCommandOptions & { + start?: number + count?: number + sort?: string + search?: string + actorType?: ActivityPubActorType + state?: FollowState + } = {}) { + const path = '/api/v1/server/followers' + + const query = pick(options, [ 'start', 'count', 'sort', 'search', 'state', 'actorType' ]) + + return this.getRequestBody>({ + ...options, + + path, + query, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getFollowings (options: OverrideCommandOptions & { + start?: number + count?: number + sort?: string + search?: string + actorType?: ActivityPubActorType + state?: FollowState + } = {}) { + const path = '/api/v1/server/following' + + const query = pick(options, [ 'start', 'count', 'sort', 'search', 'state', 'actorType' ]) + + return this.getRequestBody>({ + ...options, + + path, + query, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + follow (options: OverrideCommandOptions & { + hosts?: string[] + handles?: string[] + }) { + const path = '/api/v1/server/following' + + const fields: ServerFollowCreate = {} + + if (options.hosts) { + fields.hosts = options.hosts.map(f => f.replace(/^http:\/\//, '')) + } + + if (options.handles) { + fields.handles = options.handles + } + + return this.postBodyRequest({ + ...options, + + path, + fields, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + async unfollow (options: OverrideCommandOptions & { + target: PeerTubeServer | string + }) { + const { target } = options + + const handle = typeof target === 'string' + ? target + : target.host + + const path = '/api/v1/server/following/' + handle + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + acceptFollower (options: OverrideCommandOptions & { + follower: string + }) { + const path = '/api/v1/server/followers/' + options.follower + '/accept' + + return this.postBodyRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + rejectFollower (options: OverrideCommandOptions & { + follower: string + }) { + const path = '/api/v1/server/followers/' + options.follower + '/reject' + + return this.postBodyRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + removeFollower (options: OverrideCommandOptions & { + follower: PeerTubeServer + }) { + const path = '/api/v1/server/followers/peertube@' + options.follower.host + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/packages/server-commands/src/server/follows.ts b/packages/server-commands/src/server/follows.ts new file mode 100644 index 000000000..32304495a --- /dev/null +++ b/packages/server-commands/src/server/follows.ts @@ -0,0 +1,20 @@ +import { waitJobs } from './jobs.js' +import { PeerTubeServer } from './server.js' + +async function doubleFollow (server1: PeerTubeServer, server2: PeerTubeServer) { + await Promise.all([ + server1.follows.follow({ hosts: [ server2.url ] }), + server2.follows.follow({ hosts: [ server1.url ] }) + ]) + + // Wait request propagation + await waitJobs([ server1, server2 ]) + + return true +} + +// --------------------------------------------------------------------------- + +export { + doubleFollow +} diff --git a/packages/server-commands/src/server/index.ts b/packages/server-commands/src/server/index.ts new file mode 100644 index 000000000..c13972eca --- /dev/null +++ b/packages/server-commands/src/server/index.ts @@ -0,0 +1,15 @@ +export * from './config-command.js' +export * from './contact-form-command.js' +export * from './debug-command.js' +export * from './follows-command.js' +export * from './follows.js' +export * from './jobs.js' +export * from './jobs-command.js' +export * from './metrics-command.js' +export * from './object-storage-command.js' +export * from './plugins-command.js' +export * from './redundancy-command.js' +export * from './server.js' +export * from './servers-command.js' +export * from './servers.js' +export * from './stats-command.js' diff --git a/packages/server-commands/src/server/jobs-command.ts b/packages/server-commands/src/server/jobs-command.ts new file mode 100644 index 000000000..18aa0cd95 --- /dev/null +++ b/packages/server-commands/src/server/jobs-command.ts @@ -0,0 +1,84 @@ +import { pick } from '@peertube/peertube-core-utils' +import { HttpStatusCode, Job, JobState, JobType, ResultList } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class JobsCommand extends AbstractCommand { + + async getLatest (options: OverrideCommandOptions & { + jobType: JobType + }) { + const { data } = await this.list({ ...options, start: 0, count: 1, sort: '-createdAt' }) + + if (data.length === 0) return undefined + + return data[0] + } + + pauseJobQueue (options: OverrideCommandOptions = {}) { + const path = '/api/v1/jobs/pause' + + return this.postBodyRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + resumeJobQueue (options: OverrideCommandOptions = {}) { + const path = '/api/v1/jobs/resume' + + return this.postBodyRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + list (options: OverrideCommandOptions & { + state?: JobState + jobType?: JobType + start?: number + count?: number + sort?: string + } = {}) { + const path = this.buildJobsUrl(options.state) + + const query = pick(options, [ 'start', 'count', 'sort', 'jobType' ]) + + return this.getRequestBody>({ + ...options, + + path, + query, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + listFailed (options: OverrideCommandOptions & { + jobType?: JobType + }) { + const path = this.buildJobsUrl('failed') + + return this.getRequestBody>({ + ...options, + + path, + query: { start: 0, count: 50 }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + private buildJobsUrl (state?: JobState) { + let path = '/api/v1/jobs' + + if (state) path += '/' + state + + return path + } +} diff --git a/packages/server-commands/src/server/jobs.ts b/packages/server-commands/src/server/jobs.ts new file mode 100644 index 000000000..1f3b1f745 --- /dev/null +++ b/packages/server-commands/src/server/jobs.ts @@ -0,0 +1,117 @@ +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { JobState, JobType, RunnerJobState } from '@peertube/peertube-models' +import { PeerTubeServer } from './server.js' + +async function waitJobs ( + serversArg: PeerTubeServer[] | PeerTubeServer, + options: { + skipDelayed?: boolean // default false + runnerJobs?: boolean // default false + } = {} +) { + const { skipDelayed = false, runnerJobs = false } = options + + const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT + ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10) + : 250 + + let servers: PeerTubeServer[] + + if (Array.isArray(serversArg) === false) servers = [ serversArg as PeerTubeServer ] + else servers = serversArg as PeerTubeServer[] + + const states: JobState[] = [ 'waiting', 'active' ] + if (!skipDelayed) states.push('delayed') + + const repeatableJobs: JobType[] = [ 'videos-views-stats', 'activitypub-cleaner' ] + let pendingRequests: boolean + + function tasksBuilder () { + const tasks: Promise[] = [] + + // Check if each server has pending request + for (const server of servers) { + if (process.env.DEBUG) console.log('Checking ' + server.url) + + for (const state of states) { + + const jobPromise = server.jobs.list({ + state, + start: 0, + count: 10, + sort: '-createdAt' + }).then(body => body.data) + .then(jobs => jobs.filter(j => !repeatableJobs.includes(j.type))) + .then(jobs => { + if (jobs.length !== 0) { + pendingRequests = true + + if (process.env.DEBUG) { + console.log(jobs) + } + } + }) + + tasks.push(jobPromise) + } + + const debugPromise = server.debug.getDebug() + .then(obj => { + if (obj.activityPubMessagesWaiting !== 0) { + pendingRequests = true + + if (process.env.DEBUG) { + console.log('AP messages waiting: ' + obj.activityPubMessagesWaiting) + } + } + }) + tasks.push(debugPromise) + + if (runnerJobs) { + const runnerJobsPromise = server.runnerJobs.list({ count: 100 }) + .then(({ data }) => { + for (const job of data) { + if (job.state.id !== RunnerJobState.COMPLETED) { + pendingRequests = true + + if (process.env.DEBUG) { + console.log(job) + } + } + } + }) + tasks.push(runnerJobsPromise) + } + } + + return tasks + } + + do { + pendingRequests = false + await Promise.all(tasksBuilder()) + + // Retry, in case of new jobs were created + if (pendingRequests === false) { + await wait(pendingJobWait) + await Promise.all(tasksBuilder()) + } + + if (pendingRequests) { + await wait(pendingJobWait) + } + } while (pendingRequests) +} + +async function expectNoFailedTranscodingJob (server: PeerTubeServer) { + const { data } = await server.jobs.listFailed({ jobType: 'video-transcoding' }) + expect(data).to.have.lengthOf(0) +} + +// --------------------------------------------------------------------------- + +export { + waitJobs, + expectNoFailedTranscodingJob +} diff --git a/packages/server-commands/src/server/metrics-command.ts b/packages/server-commands/src/server/metrics-command.ts new file mode 100644 index 000000000..1f969a024 --- /dev/null +++ b/packages/server-commands/src/server/metrics-command.ts @@ -0,0 +1,18 @@ +import { HttpStatusCode, PlaybackMetricCreate } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class MetricsCommand extends AbstractCommand { + + addPlaybackMetric (options: OverrideCommandOptions & { metrics: PlaybackMetricCreate }) { + const path = '/api/v1/metrics/playback' + + return this.postBodyRequest({ + ...options, + + path, + fields: options.metrics, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/packages/server-commands/src/server/object-storage-command.ts b/packages/server-commands/src/server/object-storage-command.ts new file mode 100644 index 000000000..ff8d5d75c --- /dev/null +++ b/packages/server-commands/src/server/object-storage-command.ts @@ -0,0 +1,165 @@ +import { randomInt } from 'crypto' +import { HttpStatusCode } from '@peertube/peertube-models' +import { makePostBodyRequest } from '../requests/index.js' + +export class ObjectStorageCommand { + static readonly DEFAULT_SCALEWAY_BUCKET = 'peertube-ci-test' + + private readonly bucketsCreated: string[] = [] + private readonly seed: number + + // --------------------------------------------------------------------------- + + constructor () { + this.seed = randomInt(0, 10000) + } + + static getMockCredentialsConfig () { + return { + access_key_id: 'AKIAIOSFODNN7EXAMPLE', + secret_access_key: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' + } + } + + static getMockEndpointHost () { + return 'localhost:9444' + } + + static getMockRegion () { + return 'us-east-1' + } + + getDefaultMockConfig () { + return { + object_storage: { + enabled: true, + endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(), + region: ObjectStorageCommand.getMockRegion(), + + credentials: ObjectStorageCommand.getMockCredentialsConfig(), + + streaming_playlists: { + bucket_name: this.getMockStreamingPlaylistsBucketName() + }, + + web_videos: { + bucket_name: this.getMockWebVideosBucketName() + } + } + } + } + + getMockWebVideosBaseUrl () { + return `http://${this.getMockWebVideosBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/` + } + + getMockPlaylistBaseUrl () { + return `http://${this.getMockStreamingPlaylistsBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/` + } + + async prepareDefaultMockBuckets () { + await this.createMockBucket(this.getMockStreamingPlaylistsBucketName()) + await this.createMockBucket(this.getMockWebVideosBucketName()) + } + + async createMockBucket (name: string) { + this.bucketsCreated.push(name) + + await this.deleteMockBucket(name) + + await makePostBodyRequest({ + url: ObjectStorageCommand.getMockEndpointHost(), + path: '/ui/' + name + '?create', + expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 + }) + + await makePostBodyRequest({ + url: ObjectStorageCommand.getMockEndpointHost(), + path: '/ui/' + name + '?make-public', + expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 + }) + } + + async cleanupMock () { + for (const name of this.bucketsCreated) { + await this.deleteMockBucket(name) + } + } + + getMockStreamingPlaylistsBucketName (name = 'streaming-playlists') { + return this.getMockBucketName(name) + } + + getMockWebVideosBucketName (name = 'web-videos') { + return this.getMockBucketName(name) + } + + getMockBucketName (name: string) { + return `${this.seed}-${name}` + } + + private async deleteMockBucket (name: string) { + await makePostBodyRequest({ + url: ObjectStorageCommand.getMockEndpointHost(), + path: '/ui/' + name + '?delete', + expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 + }) + } + + // --------------------------------------------------------------------------- + + static getDefaultScalewayConfig (options: { + serverNumber: number + enablePrivateProxy?: boolean // default true + privateACL?: 'private' | 'public-read' // default 'private' + }) { + const { serverNumber, enablePrivateProxy = true, privateACL = 'private' } = options + + return { + object_storage: { + enabled: true, + endpoint: this.getScalewayEndpointHost(), + region: this.getScalewayRegion(), + + credentials: this.getScalewayCredentialsConfig(), + + upload_acl: { + private: privateACL + }, + + proxy: { + proxify_private_files: enablePrivateProxy + }, + + streaming_playlists: { + bucket_name: this.DEFAULT_SCALEWAY_BUCKET, + prefix: `test:server-${serverNumber}-streaming-playlists:` + }, + + web_videos: { + bucket_name: this.DEFAULT_SCALEWAY_BUCKET, + prefix: `test:server-${serverNumber}-web-videos:` + } + } + } + } + + static getScalewayCredentialsConfig () { + return { + access_key_id: process.env.OBJECT_STORAGE_SCALEWAY_KEY_ID, + secret_access_key: process.env.OBJECT_STORAGE_SCALEWAY_ACCESS_KEY + } + } + + static getScalewayEndpointHost () { + return 's3.fr-par.scw.cloud' + } + + static getScalewayRegion () { + return 'fr-par' + } + + static getScalewayBaseUrl () { + return `https://${this.DEFAULT_SCALEWAY_BUCKET}.${this.getScalewayEndpointHost()}/` + } +} diff --git a/packages/server-commands/src/server/plugins-command.ts b/packages/server-commands/src/server/plugins-command.ts new file mode 100644 index 000000000..f85ef0330 --- /dev/null +++ b/packages/server-commands/src/server/plugins-command.ts @@ -0,0 +1,258 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { readJSON, writeJSON } from 'fs-extra/esm' +import { join } from 'path' +import { + HttpStatusCode, + HttpStatusCodeType, + PeerTubePlugin, + PeerTubePluginIndex, + PeertubePluginIndexList, + PluginPackageJSON, + PluginTranslation, + PluginType_Type, + PublicServerSetting, + RegisteredServerSettings, + ResultList +} from '@peertube/peertube-models' +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class PluginsCommand extends AbstractCommand { + + static getPluginTestPath (suffix = '') { + return buildAbsoluteFixturePath('peertube-plugin-test' + suffix) + } + + list (options: OverrideCommandOptions & { + start?: number + count?: number + sort?: string + pluginType?: PluginType_Type + uninstalled?: boolean + }) { + const { start, count, sort, pluginType, uninstalled } = options + const path = '/api/v1/plugins' + + return this.getRequestBody>({ + ...options, + + path, + query: { + start, + count, + sort, + pluginType, + uninstalled + }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + listAvailable (options: OverrideCommandOptions & { + start?: number + count?: number + sort?: string + pluginType?: PluginType_Type + currentPeerTubeEngine?: string + search?: string + expectedStatus?: HttpStatusCodeType + }) { + const { start, count, sort, pluginType, search, currentPeerTubeEngine } = options + const path = '/api/v1/plugins/available' + + const query: PeertubePluginIndexList = { + start, + count, + sort, + pluginType, + currentPeerTubeEngine, + search + } + + return this.getRequestBody>({ + ...options, + + path, + query, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + get (options: OverrideCommandOptions & { + npmName: string + }) { + const path = '/api/v1/plugins/' + options.npmName + + return this.getRequestBody({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + updateSettings (options: OverrideCommandOptions & { + npmName: string + settings: any + }) { + const { npmName, settings } = options + const path = '/api/v1/plugins/' + npmName + '/settings' + + return this.putBodyRequest({ + ...options, + + path, + fields: { settings }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + getRegisteredSettings (options: OverrideCommandOptions & { + npmName: string + }) { + const path = '/api/v1/plugins/' + options.npmName + '/registered-settings' + + return this.getRequestBody({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getPublicSettings (options: OverrideCommandOptions & { + npmName: string + }) { + const { npmName } = options + const path = '/api/v1/plugins/' + npmName + '/public-settings' + + return this.getRequestBody({ + ...options, + + path, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getTranslations (options: OverrideCommandOptions & { + locale: string + }) { + const { locale } = options + const path = '/plugins/translations/' + locale + '.json' + + return this.getRequestBody({ + ...options, + + path, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + install (options: OverrideCommandOptions & { + path?: string + npmName?: string + pluginVersion?: string + }) { + const { npmName, path, pluginVersion } = options + const apiPath = '/api/v1/plugins/install' + + return this.postBodyRequest({ + ...options, + + path: apiPath, + fields: { npmName, path, pluginVersion }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + update (options: OverrideCommandOptions & { + path?: string + npmName?: string + }) { + const { npmName, path } = options + const apiPath = '/api/v1/plugins/update' + + return this.postBodyRequest({ + ...options, + + path: apiPath, + fields: { npmName, path }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + uninstall (options: OverrideCommandOptions & { + npmName: string + }) { + const { npmName } = options + const apiPath = '/api/v1/plugins/uninstall' + + return this.postBodyRequest({ + ...options, + + path: apiPath, + fields: { npmName }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + getCSS (options: OverrideCommandOptions = {}) { + const path = '/plugins/global.css' + + return this.getRequestText({ + ...options, + + path, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getExternalAuth (options: OverrideCommandOptions & { + npmName: string + npmVersion: string + authName: string + query?: any + }) { + const { npmName, npmVersion, authName, query } = options + + const path = '/plugins/' + npmName + '/' + npmVersion + '/auth/' + authName + + return this.getRequest({ + ...options, + + path, + query, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200, + redirects: 0 + }) + } + + updatePackageJSON (npmName: string, json: any) { + const path = this.getPackageJSONPath(npmName) + + return writeJSON(path, json) + } + + getPackageJSON (npmName: string): Promise { + const path = this.getPackageJSONPath(npmName) + + return readJSON(path) + } + + private getPackageJSONPath (npmName: string) { + return this.server.servers.buildDirectory(join('plugins', 'node_modules', npmName, 'package.json')) + } +} diff --git a/packages/server-commands/src/server/redundancy-command.ts b/packages/server-commands/src/server/redundancy-command.ts new file mode 100644 index 000000000..a0ec3e80e --- /dev/null +++ b/packages/server-commands/src/server/redundancy-command.ts @@ -0,0 +1,80 @@ +import { HttpStatusCode, ResultList, VideoRedundanciesTarget, VideoRedundancy } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class RedundancyCommand extends AbstractCommand { + + updateRedundancy (options: OverrideCommandOptions & { + host: string + redundancyAllowed: boolean + }) { + const { host, redundancyAllowed } = options + const path = '/api/v1/server/redundancy/' + host + + return this.putBodyRequest({ + ...options, + + path, + fields: { redundancyAllowed }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + listVideos (options: OverrideCommandOptions & { + target: VideoRedundanciesTarget + start?: number + count?: number + sort?: string + }) { + const path = '/api/v1/server/redundancy/videos' + + const { target, start, count, sort } = options + + return this.getRequestBody>({ + ...options, + + path, + + query: { + start: start ?? 0, + count: count ?? 5, + sort: sort ?? 'name', + target + }, + + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + addVideo (options: OverrideCommandOptions & { + videoId: number + }) { + const path = '/api/v1/server/redundancy/videos' + const { videoId } = options + + return this.postBodyRequest({ + ...options, + + path, + fields: { videoId }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + removeVideo (options: OverrideCommandOptions & { + redundancyId: number + }) { + const { redundancyId } = options + const path = '/api/v1/server/redundancy/videos/' + redundancyId + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/packages/server-commands/src/server/server.ts b/packages/server-commands/src/server/server.ts new file mode 100644 index 000000000..57a897c17 --- /dev/null +++ b/packages/server-commands/src/server/server.ts @@ -0,0 +1,451 @@ +import { ChildProcess, fork } from 'child_process' +import { copy } from 'fs-extra/esm' +import { join } from 'path' +import { randomInt } from '@peertube/peertube-core-utils' +import { Video, VideoChannel, VideoChannelSync, VideoCreateResult, VideoDetails } from '@peertube/peertube-models' +import { parallelTests, root } from '@peertube/peertube-node-utils' +import { BulkCommand } from '../bulk/index.js' +import { CLICommand } from '../cli/index.js' +import { CustomPagesCommand } from '../custom-pages/index.js' +import { FeedCommand } from '../feeds/index.js' +import { LogsCommand } from '../logs/index.js' +import { AbusesCommand } from '../moderation/index.js' +import { OverviewsCommand } from '../overviews/index.js' +import { RunnerJobsCommand, RunnerRegistrationTokensCommand, RunnersCommand } from '../runners/index.js' +import { SearchCommand } from '../search/index.js' +import { SocketIOCommand } from '../socket/index.js' +import { + AccountsCommand, + BlocklistCommand, + LoginCommand, + NotificationsCommand, + RegistrationsCommand, + SubscriptionsCommand, + TwoFactorCommand, + UsersCommand +} from '../users/index.js' +import { + BlacklistCommand, + CaptionsCommand, + ChangeOwnershipCommand, + ChannelsCommand, + ChannelSyncsCommand, + CommentsCommand, + HistoryCommand, + ImportsCommand, + LiveCommand, + PlaylistsCommand, + ServicesCommand, + StoryboardCommand, + StreamingPlaylistsCommand, + VideoPasswordsCommand, + VideosCommand, + VideoStatsCommand, + VideoStudioCommand, + VideoTokenCommand, + ViewsCommand +} from '../videos/index.js' +import { ConfigCommand } from './config-command.js' +import { ContactFormCommand } from './contact-form-command.js' +import { DebugCommand } from './debug-command.js' +import { FollowsCommand } from './follows-command.js' +import { JobsCommand } from './jobs-command.js' +import { MetricsCommand } from './metrics-command.js' +import { PluginsCommand } from './plugins-command.js' +import { RedundancyCommand } from './redundancy-command.js' +import { ServersCommand } from './servers-command.js' +import { StatsCommand } from './stats-command.js' + +export type RunServerOptions = { + hideLogs?: boolean + nodeArgs?: string[] + peertubeArgs?: string[] + env?: { [ id: string ]: string } +} + +export class PeerTubeServer { + app?: ChildProcess + + url: string + host?: string + hostname?: string + port?: number + + rtmpPort?: number + rtmpsPort?: number + + parallel?: boolean + internalServerNumber: number + + serverNumber?: number + customConfigFile?: string + + store?: { + client?: { + id?: string + secret?: string + } + + user?: { + username: string + password: string + email?: string + } + + channel?: VideoChannel + videoChannelSync?: Partial + + video?: Video + videoCreated?: VideoCreateResult + videoDetails?: VideoDetails + + videos?: { id: number, uuid: string }[] + } + + accessToken?: string + refreshToken?: string + + bulk?: BulkCommand + cli?: CLICommand + customPage?: CustomPagesCommand + feed?: FeedCommand + logs?: LogsCommand + abuses?: AbusesCommand + overviews?: OverviewsCommand + search?: SearchCommand + contactForm?: ContactFormCommand + debug?: DebugCommand + follows?: FollowsCommand + jobs?: JobsCommand + metrics?: MetricsCommand + plugins?: PluginsCommand + redundancy?: RedundancyCommand + stats?: StatsCommand + config?: ConfigCommand + socketIO?: SocketIOCommand + accounts?: AccountsCommand + blocklist?: BlocklistCommand + subscriptions?: SubscriptionsCommand + live?: LiveCommand + services?: ServicesCommand + blacklist?: BlacklistCommand + captions?: CaptionsCommand + changeOwnership?: ChangeOwnershipCommand + playlists?: PlaylistsCommand + history?: HistoryCommand + imports?: ImportsCommand + channelSyncs?: ChannelSyncsCommand + streamingPlaylists?: StreamingPlaylistsCommand + channels?: ChannelsCommand + comments?: CommentsCommand + notifications?: NotificationsCommand + servers?: ServersCommand + login?: LoginCommand + users?: UsersCommand + videoStudio?: VideoStudioCommand + videos?: VideosCommand + videoStats?: VideoStatsCommand + views?: ViewsCommand + twoFactor?: TwoFactorCommand + videoToken?: VideoTokenCommand + registrations?: RegistrationsCommand + videoPasswords?: VideoPasswordsCommand + + storyboard?: StoryboardCommand + + runners?: RunnersCommand + runnerRegistrationTokens?: RunnerRegistrationTokensCommand + runnerJobs?: RunnerJobsCommand + + constructor (options: { serverNumber: number } | { url: string }) { + if ((options as any).url) { + this.setUrl((options as any).url) + } else { + this.setServerNumber((options as any).serverNumber) + } + + this.store = { + client: { + id: null, + secret: null + }, + user: { + username: null, + password: null + } + } + + this.assignCommands() + } + + setServerNumber (serverNumber: number) { + this.serverNumber = serverNumber + + this.parallel = parallelTests() + + this.internalServerNumber = this.parallel ? this.randomServer() : this.serverNumber + this.rtmpPort = this.parallel ? this.randomRTMP() : 1936 + this.rtmpsPort = this.parallel ? this.randomRTMP() : 1937 + this.port = 9000 + this.internalServerNumber + + this.url = `http://127.0.0.1:${this.port}` + this.host = `127.0.0.1:${this.port}` + this.hostname = '127.0.0.1' + } + + setUrl (url: string) { + const parsed = new URL(url) + + this.url = url + this.host = parsed.host + this.hostname = parsed.hostname + this.port = parseInt(parsed.port) + } + + getDirectoryPath (directoryName: string) { + const testDirectory = 'test' + this.internalServerNumber + + return join(root(), testDirectory, directoryName) + } + + async flushAndRun (configOverride?: object, options: RunServerOptions = {}) { + await ServersCommand.flushTests(this.internalServerNumber) + + return this.run(configOverride, options) + } + + async run (configOverrideArg?: any, options: RunServerOptions = {}) { + // These actions are async so we need to be sure that they have both been done + const serverRunString = { + 'HTTP server listening': false + } + const key = 'Database peertube_test' + this.internalServerNumber + ' is ready' + serverRunString[key] = false + + const regexps = { + client_id: 'Client id: (.+)', + client_secret: 'Client secret: (.+)', + user_username: 'Username: (.+)', + user_password: 'User password: (.+)' + } + + await this.assignCustomConfigFile() + + const configOverride = this.buildConfigOverride() + + if (configOverrideArg !== undefined) { + Object.assign(configOverride, configOverrideArg) + } + + // Share the environment + const env = { ...process.env } + env['NODE_ENV'] = 'test' + env['NODE_APP_INSTANCE'] = this.internalServerNumber.toString() + env['NODE_CONFIG'] = JSON.stringify(configOverride) + + if (options.env) { + Object.assign(env, options.env) + } + + const execArgv = options.nodeArgs || [] + // FIXME: too slow :/ + // execArgv.push('--enable-source-maps') + + const forkOptions = { + silent: true, + env, + detached: false, + execArgv + } + + const peertubeArgs = options.peertubeArgs || [] + + return new Promise((res, rej) => { + const self = this + let aggregatedLogs = '' + + this.app = fork(join(root(), 'dist', 'server.js'), peertubeArgs, forkOptions) + + const onPeerTubeExit = () => rej(new Error('Process exited:\n' + aggregatedLogs)) + const onParentExit = () => { + if (!this.app?.pid) return + + try { + process.kill(self.app.pid) + } catch { /* empty */ } + } + + this.app.on('exit', onPeerTubeExit) + process.on('exit', onParentExit) + + this.app.stdout.on('data', function onStdout (data) { + let dontContinue = false + + const log: string = data.toString() + aggregatedLogs += log + + // Capture things if we want to + for (const key of Object.keys(regexps)) { + const regexp = regexps[key] + const matches = log.match(regexp) + if (matches !== null) { + if (key === 'client_id') self.store.client.id = matches[1] + else if (key === 'client_secret') self.store.client.secret = matches[1] + else if (key === 'user_username') self.store.user.username = matches[1] + else if (key === 'user_password') self.store.user.password = matches[1] + } + } + + // Check if all required sentences are here + for (const key of Object.keys(serverRunString)) { + if (log.includes(key)) serverRunString[key] = true + if (serverRunString[key] === false) dontContinue = true + } + + // If no, there is maybe one thing not already initialized (client/user credentials generation...) + if (dontContinue === true) return + + if (options.hideLogs === false) { + console.log(log) + } else { + process.removeListener('exit', onParentExit) + self.app.stdout.removeListener('data', onStdout) + self.app.removeListener('exit', onPeerTubeExit) + } + + res() + }) + }) + } + + kill () { + if (!this.app) return Promise.resolve() + + process.kill(this.app.pid) + + this.app = null + + return Promise.resolve() + } + + private randomServer () { + const low = 2500 + const high = 10000 + + return randomInt(low, high) + } + + private randomRTMP () { + const low = 1900 + const high = 2100 + + return randomInt(low, high) + } + + private async assignCustomConfigFile () { + if (this.internalServerNumber === this.serverNumber) return + + const basePath = join(root(), 'config') + + const tmpConfigFile = join(basePath, `test-${this.internalServerNumber}.yaml`) + await copy(join(basePath, `test-${this.serverNumber}.yaml`), tmpConfigFile) + + this.customConfigFile = tmpConfigFile + } + + private buildConfigOverride () { + if (!this.parallel) return {} + + return { + listen: { + port: this.port + }, + webserver: { + port: this.port + }, + database: { + suffix: '_test' + this.internalServerNumber + }, + storage: { + tmp: this.getDirectoryPath('tmp') + '/', + tmp_persistent: this.getDirectoryPath('tmp-persistent') + '/', + bin: this.getDirectoryPath('bin') + '/', + avatars: this.getDirectoryPath('avatars') + '/', + web_videos: this.getDirectoryPath('web-videos') + '/', + streaming_playlists: this.getDirectoryPath('streaming-playlists') + '/', + redundancy: this.getDirectoryPath('redundancy') + '/', + logs: this.getDirectoryPath('logs') + '/', + previews: this.getDirectoryPath('previews') + '/', + thumbnails: this.getDirectoryPath('thumbnails') + '/', + storyboards: this.getDirectoryPath('storyboards') + '/', + torrents: this.getDirectoryPath('torrents') + '/', + captions: this.getDirectoryPath('captions') + '/', + cache: this.getDirectoryPath('cache') + '/', + plugins: this.getDirectoryPath('plugins') + '/', + well_known: this.getDirectoryPath('well-known') + '/' + }, + admin: { + email: `admin${this.internalServerNumber}@example.com` + }, + live: { + rtmp: { + port: this.rtmpPort + } + } + } + } + + private assignCommands () { + this.bulk = new BulkCommand(this) + this.cli = new CLICommand(this) + this.customPage = new CustomPagesCommand(this) + this.feed = new FeedCommand(this) + this.logs = new LogsCommand(this) + this.abuses = new AbusesCommand(this) + this.overviews = new OverviewsCommand(this) + this.search = new SearchCommand(this) + this.contactForm = new ContactFormCommand(this) + this.debug = new DebugCommand(this) + this.follows = new FollowsCommand(this) + this.jobs = new JobsCommand(this) + this.metrics = new MetricsCommand(this) + this.plugins = new PluginsCommand(this) + this.redundancy = new RedundancyCommand(this) + this.stats = new StatsCommand(this) + this.config = new ConfigCommand(this) + this.socketIO = new SocketIOCommand(this) + this.accounts = new AccountsCommand(this) + this.blocklist = new BlocklistCommand(this) + this.subscriptions = new SubscriptionsCommand(this) + this.live = new LiveCommand(this) + this.services = new ServicesCommand(this) + this.blacklist = new BlacklistCommand(this) + this.captions = new CaptionsCommand(this) + this.changeOwnership = new ChangeOwnershipCommand(this) + this.playlists = new PlaylistsCommand(this) + this.history = new HistoryCommand(this) + this.imports = new ImportsCommand(this) + this.channelSyncs = new ChannelSyncsCommand(this) + this.streamingPlaylists = new StreamingPlaylistsCommand(this) + this.channels = new ChannelsCommand(this) + this.comments = new CommentsCommand(this) + this.notifications = new NotificationsCommand(this) + this.servers = new ServersCommand(this) + this.login = new LoginCommand(this) + this.users = new UsersCommand(this) + this.videos = new VideosCommand(this) + this.videoStudio = new VideoStudioCommand(this) + this.videoStats = new VideoStatsCommand(this) + this.views = new ViewsCommand(this) + this.twoFactor = new TwoFactorCommand(this) + this.videoToken = new VideoTokenCommand(this) + this.registrations = new RegistrationsCommand(this) + + this.storyboard = new StoryboardCommand(this) + + this.runners = new RunnersCommand(this) + this.runnerRegistrationTokens = new RunnerRegistrationTokensCommand(this) + this.runnerJobs = new RunnerJobsCommand(this) + this.videoPasswords = new VideoPasswordsCommand(this) + } +} diff --git a/packages/server-commands/src/server/servers-command.ts b/packages/server-commands/src/server/servers-command.ts new file mode 100644 index 000000000..0b722b62f --- /dev/null +++ b/packages/server-commands/src/server/servers-command.ts @@ -0,0 +1,104 @@ +import { exec } from 'child_process' +import { copy, ensureDir, remove } from 'fs-extra/esm' +import { readdir, readFile } from 'fs/promises' +import { basename, join } from 'path' +import { wait } from '@peertube/peertube-core-utils' +import { HttpStatusCode } from '@peertube/peertube-models' +import { getFileSize, isGithubCI, root } from '@peertube/peertube-node-utils' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class ServersCommand extends AbstractCommand { + + static flushTests (internalServerNumber: number) { + return new Promise((res, rej) => { + const suffix = ` -- ${internalServerNumber}` + + return exec('npm run clean:server:test' + suffix, (err, _stdout, stderr) => { + if (err || stderr) return rej(err || new Error(stderr)) + + return res() + }) + }) + } + + ping (options: OverrideCommandOptions = {}) { + return this.getRequestBody({ + ...options, + + path: '/api/v1/ping', + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + cleanupTests () { + const promises: Promise[] = [] + + const saveGithubLogsIfNeeded = async () => { + if (!isGithubCI()) return + + await ensureDir('artifacts') + + const origin = this.buildDirectory('logs/peertube.log') + const destname = `peertube-${this.server.internalServerNumber}.log` + console.log('Saving logs %s.', destname) + + await copy(origin, join('artifacts', destname)) + } + + if (this.server.parallel) { + const promise = saveGithubLogsIfNeeded() + .then(() => ServersCommand.flushTests(this.server.internalServerNumber)) + + promises.push(promise) + } + + if (this.server.customConfigFile) { + promises.push(remove(this.server.customConfigFile)) + } + + return promises + } + + async waitUntilLog (str: string, count = 1, strictCount = true) { + const logfile = this.buildDirectory('logs/peertube.log') + + while (true) { + const buf = await readFile(logfile) + + const matches = buf.toString().match(new RegExp(str, 'g')) + if (matches && matches.length === count) return + if (matches && strictCount === false && matches.length >= count) return + + await wait(1000) + } + } + + buildDirectory (directory: string) { + return join(root(), 'test' + this.server.internalServerNumber, directory) + } + + async countFiles (directory: string) { + const files = await readdir(this.buildDirectory(directory)) + + return files.length + } + + buildWebVideoFilePath (fileUrl: string) { + return this.buildDirectory(join('web-videos', basename(fileUrl))) + } + + buildFragmentedFilePath (videoUUID: string, fileUrl: string) { + return this.buildDirectory(join('streaming-playlists', 'hls', videoUUID, basename(fileUrl))) + } + + getLogContent () { + return readFile(this.buildDirectory('logs/peertube.log')) + } + + async getServerFileSize (subPath: string) { + const path = this.server.servers.buildDirectory(subPath) + + return getFileSize(path) + } +} diff --git a/packages/server-commands/src/server/servers.ts b/packages/server-commands/src/server/servers.ts new file mode 100644 index 000000000..caf9866e1 --- /dev/null +++ b/packages/server-commands/src/server/servers.ts @@ -0,0 +1,68 @@ +import { ensureDir } from 'fs-extra/esm' +import { isGithubCI } from '@peertube/peertube-node-utils' +import { PeerTubeServer, RunServerOptions } from './server.js' + +async function createSingleServer (serverNumber: number, configOverride?: object, options: RunServerOptions = {}) { + const server = new PeerTubeServer({ serverNumber }) + + await server.flushAndRun(configOverride, options) + + return server +} + +function createMultipleServers (totalServers: number, configOverride?: object, options: RunServerOptions = {}) { + const serverPromises: Promise[] = [] + + for (let i = 1; i <= totalServers; i++) { + serverPromises.push(createSingleServer(i, configOverride, options)) + } + + return Promise.all(serverPromises) +} + +function killallServers (servers: PeerTubeServer[]) { + return Promise.all(servers.map(s => s.kill())) +} + +async function cleanupTests (servers: PeerTubeServer[]) { + await killallServers(servers) + + if (isGithubCI()) { + await ensureDir('artifacts') + } + + let p: Promise[] = [] + for (const server of servers) { + p = p.concat(server.servers.cleanupTests()) + } + + return Promise.all(p) +} + +function getServerImportConfig (mode: 'youtube-dl' | 'yt-dlp') { + return { + import: { + videos: { + http: { + youtube_dl_release: { + url: mode === 'youtube-dl' + ? 'https://yt-dl.org/downloads/latest/youtube-dl' + : 'https://api.github.com/repos/yt-dlp/yt-dlp/releases', + + name: mode + } + } + } + } + } +} + +// --------------------------------------------------------------------------- + +export { + createSingleServer, + createMultipleServers, + cleanupTests, + killallServers, + getServerImportConfig +} diff --git a/packages/server-commands/src/server/stats-command.ts b/packages/server-commands/src/server/stats-command.ts new file mode 100644 index 000000000..80acd7bdc --- /dev/null +++ b/packages/server-commands/src/server/stats-command.ts @@ -0,0 +1,25 @@ +import { HttpStatusCode, ServerStats } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class StatsCommand extends AbstractCommand { + + get (options: OverrideCommandOptions & { + useCache?: boolean // default false + } = {}) { + const { useCache = false } = options + const path = '/api/v1/server/stats' + + const query = { + t: useCache ? undefined : new Date().getTime() + } + + return this.getRequestBody({ + ...options, + + path, + query, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } +} diff --git a/packages/server-commands/src/shared/abstract-command.ts b/packages/server-commands/src/shared/abstract-command.ts new file mode 100644 index 000000000..bb6522e07 --- /dev/null +++ b/packages/server-commands/src/shared/abstract-command.ts @@ -0,0 +1,225 @@ +import { isAbsolute } from 'path' +import { HttpStatusCodeType } from '@peertube/peertube-models' +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import { + makeDeleteRequest, + makeGetRequest, + makePostBodyRequest, + makePutBodyRequest, + makeUploadRequest, + unwrapBody, + unwrapText +} from '../requests/requests.js' + +import type { PeerTubeServer } from '../server/server.js' + +export interface OverrideCommandOptions { + token?: string + expectedStatus?: HttpStatusCodeType +} + +interface InternalCommonCommandOptions extends OverrideCommandOptions { + // Default to server.url + url?: string + + path: string + // If we automatically send the server token if the token is not provided + implicitToken: boolean + defaultExpectedStatus: HttpStatusCodeType + + // Common optional request parameters + contentType?: string + accept?: string + redirects?: number + range?: string + host?: string + headers?: { [ name: string ]: string } + requestType?: string + responseType?: string + xForwardedFor?: string +} + +interface InternalGetCommandOptions extends InternalCommonCommandOptions { + query?: { [ id: string ]: any } +} + +interface InternalDeleteCommandOptions extends InternalCommonCommandOptions { + query?: { [ id: string ]: any } + rawQuery?: string +} + +abstract class AbstractCommand { + + constructor ( + protected server: PeerTubeServer + ) { + + } + + protected getRequestBody (options: InternalGetCommandOptions) { + return unwrapBody(this.getRequest(options)) + } + + protected getRequestText (options: InternalGetCommandOptions) { + return unwrapText(this.getRequest(options)) + } + + protected getRawRequest (options: Omit) { + const { url, range } = options + const { host, protocol, pathname } = new URL(url) + + return this.getRequest({ + ...options, + + token: this.buildCommonRequestToken(options), + defaultExpectedStatus: this.buildExpectedStatus(options), + + url: `${protocol}//${host}`, + path: pathname, + range + }) + } + + protected getRequest (options: InternalGetCommandOptions) { + const { query } = options + + return makeGetRequest({ + ...this.buildCommonRequestOptions(options), + + query + }) + } + + protected deleteRequest (options: InternalDeleteCommandOptions) { + const { query, rawQuery } = options + + return makeDeleteRequest({ + ...this.buildCommonRequestOptions(options), + + query, + rawQuery + }) + } + + protected putBodyRequest (options: InternalCommonCommandOptions & { + fields?: { [ fieldName: string ]: any } + headers?: { [name: string]: string } + }) { + const { fields, headers } = options + + return makePutBodyRequest({ + ...this.buildCommonRequestOptions(options), + + fields, + headers + }) + } + + protected postBodyRequest (options: InternalCommonCommandOptions & { + fields?: { [ fieldName: string ]: any } + headers?: { [name: string]: string } + }) { + const { fields, headers } = options + + return makePostBodyRequest({ + ...this.buildCommonRequestOptions(options), + + fields, + headers + }) + } + + protected postUploadRequest (options: InternalCommonCommandOptions & { + fields?: { [ fieldName: string ]: any } + attaches?: { [ fieldName: string ]: any } + }) { + const { fields, attaches } = options + + return makeUploadRequest({ + ...this.buildCommonRequestOptions(options), + + method: 'POST', + fields, + attaches + }) + } + + protected putUploadRequest (options: InternalCommonCommandOptions & { + fields?: { [ fieldName: string ]: any } + attaches?: { [ fieldName: string ]: any } + }) { + const { fields, attaches } = options + + return makeUploadRequest({ + ...this.buildCommonRequestOptions(options), + + method: 'PUT', + fields, + attaches + }) + } + + protected updateImageRequest (options: InternalCommonCommandOptions & { + fixture: string + fieldname: string + }) { + const filePath = isAbsolute(options.fixture) + ? options.fixture + : buildAbsoluteFixturePath(options.fixture) + + return this.postUploadRequest({ + ...options, + + fields: {}, + attaches: { [options.fieldname]: filePath } + }) + } + + protected buildCommonRequestOptions (options: InternalCommonCommandOptions) { + const { url, path, redirects, contentType, accept, range, host, headers, requestType, xForwardedFor, responseType } = options + + return { + url: url ?? this.server.url, + path, + + token: this.buildCommonRequestToken(options), + expectedStatus: this.buildExpectedStatus(options), + + redirects, + contentType, + range, + host, + accept, + headers, + type: requestType, + responseType, + xForwardedFor + } + } + + protected buildCommonRequestToken (options: Pick) { + const { token } = options + + const fallbackToken = options.implicitToken + ? this.server.accessToken + : undefined + + return token !== undefined ? token : fallbackToken + } + + protected buildExpectedStatus (options: Pick) { + const { expectedStatus, defaultExpectedStatus } = options + + return expectedStatus !== undefined ? expectedStatus : defaultExpectedStatus + } + + protected buildVideoPasswordHeader (videoPassword: string) { + return videoPassword !== undefined && videoPassword !== null + ? { 'x-peertube-video-password': videoPassword } + : undefined + } +} + +export { + AbstractCommand +} diff --git a/packages/server-commands/src/shared/index.ts b/packages/server-commands/src/shared/index.ts new file mode 100644 index 000000000..795db3d55 --- /dev/null +++ b/packages/server-commands/src/shared/index.ts @@ -0,0 +1 @@ +export * from './abstract-command.js' diff --git a/packages/server-commands/src/socket/index.ts b/packages/server-commands/src/socket/index.ts new file mode 100644 index 000000000..24b8f4b46 --- /dev/null +++ b/packages/server-commands/src/socket/index.ts @@ -0,0 +1 @@ +export * from './socket-io-command.js' diff --git a/packages/server-commands/src/socket/socket-io-command.ts b/packages/server-commands/src/socket/socket-io-command.ts new file mode 100644 index 000000000..9c18c2a1f --- /dev/null +++ b/packages/server-commands/src/socket/socket-io-command.ts @@ -0,0 +1,24 @@ +import { io } from 'socket.io-client' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class SocketIOCommand extends AbstractCommand { + + getUserNotificationSocket (options: OverrideCommandOptions = {}) { + return io(this.server.url + '/user-notifications', { + query: { accessToken: options.token ?? this.server.accessToken } + }) + } + + getLiveNotificationSocket () { + return io(this.server.url + '/live-videos') + } + + getRunnersSocket (options: { + runnerToken: string + }) { + return io(this.server.url + '/runners', { + reconnection: false, + auth: { runnerToken: options.runnerToken } + }) + } +} diff --git a/packages/server-commands/src/users/accounts-command.ts b/packages/server-commands/src/users/accounts-command.ts new file mode 100644 index 000000000..fd98b7eea --- /dev/null +++ b/packages/server-commands/src/users/accounts-command.ts @@ -0,0 +1,76 @@ +import { Account, AccountVideoRate, ActorFollow, HttpStatusCode, ResultList, VideoRateType } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class AccountsCommand extends AbstractCommand { + + list (options: OverrideCommandOptions & { + sort?: string // default -createdAt + } = {}) { + const { sort = '-createdAt' } = options + const path = '/api/v1/accounts' + + return this.getRequestBody>({ + ...options, + + path, + query: { sort }, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + get (options: OverrideCommandOptions & { + accountName: string + }) { + const path = '/api/v1/accounts/' + options.accountName + + return this.getRequestBody({ + ...options, + + path, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + listRatings (options: OverrideCommandOptions & { + accountName: string + rating?: VideoRateType + }) { + const { rating, accountName } = options + const path = '/api/v1/accounts/' + accountName + '/ratings' + + const query = { rating } + + return this.getRequestBody>({ + ...options, + + path, + query, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + listFollowers (options: OverrideCommandOptions & { + accountName: string + start?: number + count?: number + sort?: string + search?: string + }) { + const { accountName, start, count, sort, search } = options + const path = '/api/v1/accounts/' + accountName + '/followers' + + const query = { start, count, sort, search } + + return this.getRequestBody>({ + ...options, + + path, + query, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } +} diff --git a/packages/server-commands/src/users/accounts.ts b/packages/server-commands/src/users/accounts.ts new file mode 100644 index 000000000..3b8b9d36a --- /dev/null +++ b/packages/server-commands/src/users/accounts.ts @@ -0,0 +1,15 @@ +import { PeerTubeServer } from '../server/server.js' + +async function setDefaultAccountAvatar (serversArg: PeerTubeServer | PeerTubeServer[], token?: string) { + const servers = Array.isArray(serversArg) + ? serversArg + : [ serversArg ] + + for (const server of servers) { + await server.users.updateMyAvatar({ fixture: 'avatar.png', token }) + } +} + +export { + setDefaultAccountAvatar +} diff --git a/packages/server-commands/src/users/blocklist-command.ts b/packages/server-commands/src/users/blocklist-command.ts new file mode 100644 index 000000000..c77c56131 --- /dev/null +++ b/packages/server-commands/src/users/blocklist-command.ts @@ -0,0 +1,165 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { AccountBlock, BlockStatus, HttpStatusCode, ResultList, ServerBlock } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +type ListBlocklistOptions = OverrideCommandOptions & { + start: number + count: number + + sort?: string // default -createdAt + + search?: string +} + +export class BlocklistCommand extends AbstractCommand { + + listMyAccountBlocklist (options: ListBlocklistOptions) { + const path = '/api/v1/users/me/blocklist/accounts' + + return this.listBlocklist(options, path) + } + + listMyServerBlocklist (options: ListBlocklistOptions) { + const path = '/api/v1/users/me/blocklist/servers' + + return this.listBlocklist(options, path) + } + + listServerAccountBlocklist (options: ListBlocklistOptions) { + const path = '/api/v1/server/blocklist/accounts' + + return this.listBlocklist(options, path) + } + + listServerServerBlocklist (options: ListBlocklistOptions) { + const path = '/api/v1/server/blocklist/servers' + + return this.listBlocklist(options, path) + } + + // --------------------------------------------------------------------------- + + getStatus (options: OverrideCommandOptions & { + accounts?: string[] + hosts?: string[] + }) { + const { accounts, hosts } = options + + const path = '/api/v1/blocklist/status' + + return this.getRequestBody({ + ...options, + + path, + query: { + accounts, + hosts + }, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + // --------------------------------------------------------------------------- + + addToMyBlocklist (options: OverrideCommandOptions & { + account?: string + server?: string + }) { + const { account, server } = options + + const path = account + ? '/api/v1/users/me/blocklist/accounts' + : '/api/v1/users/me/blocklist/servers' + + return this.postBodyRequest({ + ...options, + + path, + fields: { + accountName: account, + host: server + }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + addToServerBlocklist (options: OverrideCommandOptions & { + account?: string + server?: string + }) { + const { account, server } = options + + const path = account + ? '/api/v1/server/blocklist/accounts' + : '/api/v1/server/blocklist/servers' + + return this.postBodyRequest({ + ...options, + + path, + fields: { + accountName: account, + host: server + }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + // --------------------------------------------------------------------------- + + removeFromMyBlocklist (options: OverrideCommandOptions & { + account?: string + server?: string + }) { + const { account, server } = options + + const path = account + ? '/api/v1/users/me/blocklist/accounts/' + account + : '/api/v1/users/me/blocklist/servers/' + server + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + removeFromServerBlocklist (options: OverrideCommandOptions & { + account?: string + server?: string + }) { + const { account, server } = options + + const path = account + ? '/api/v1/server/blocklist/accounts/' + account + : '/api/v1/server/blocklist/servers/' + server + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + private listBlocklist (options: ListBlocklistOptions, path: string) { + const { start, count, search, sort = '-createdAt' } = options + + return this.getRequestBody>({ + ...options, + + path, + query: { start, count, sort, search }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + +} diff --git a/packages/server-commands/src/users/index.ts b/packages/server-commands/src/users/index.ts new file mode 100644 index 000000000..baa048a43 --- /dev/null +++ b/packages/server-commands/src/users/index.ts @@ -0,0 +1,10 @@ +export * from './accounts-command.js' +export * from './accounts.js' +export * from './blocklist-command.js' +export * from './login.js' +export * from './login-command.js' +export * from './notifications-command.js' +export * from './registrations-command.js' +export * from './subscriptions-command.js' +export * from './two-factor-command.js' +export * from './users-command.js' diff --git a/packages/server-commands/src/users/login-command.ts b/packages/server-commands/src/users/login-command.ts new file mode 100644 index 000000000..92d123dfc --- /dev/null +++ b/packages/server-commands/src/users/login-command.ts @@ -0,0 +1,159 @@ +import { HttpStatusCode, PeerTubeProblemDocument } from '@peertube/peertube-models' +import { unwrapBody } from '../requests/index.js' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +type LoginOptions = OverrideCommandOptions & { + client?: { id?: string, secret?: string } + user?: { username: string, password?: string } + otpToken?: string +} + +export class LoginCommand extends AbstractCommand { + + async login (options: LoginOptions = {}) { + const res = await this._login(options) + + return this.unwrapLoginBody(res.body) + } + + async loginAndGetResponse (options: LoginOptions = {}) { + const res = await this._login(options) + + return { + res, + body: this.unwrapLoginBody(res.body) + } + } + + getAccessToken (arg1?: { username: string, password?: string }): Promise + getAccessToken (arg1: string, password?: string): Promise + async getAccessToken (arg1?: { username: string, password?: string } | string, password?: string) { + let user: { username: string, password?: string } + + if (!arg1) user = this.server.store.user + else if (typeof arg1 === 'object') user = arg1 + else user = { username: arg1, password } + + try { + const body = await this.login({ user }) + + return body.access_token + } catch (err) { + throw new Error(`Cannot authenticate. Please check your username/password. (${err})`) + } + } + + loginUsingExternalToken (options: OverrideCommandOptions & { + username: string + externalAuthToken: string + }) { + const { username, externalAuthToken } = options + const path = '/api/v1/users/token' + + const body = { + client_id: this.server.store.client.id, + client_secret: this.server.store.client.secret, + username, + response_type: 'code', + grant_type: 'password', + scope: 'upload', + externalAuthToken + } + + return this.postBodyRequest({ + ...options, + + path, + requestType: 'form', + fields: body, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + logout (options: OverrideCommandOptions & { + token: string + }) { + const path = '/api/v1/users/revoke-token' + + return unwrapBody<{ redirectUrl: string }>(this.postBodyRequest({ + ...options, + + path, + requestType: 'form', + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + } + + refreshToken (options: OverrideCommandOptions & { + refreshToken: string + }) { + const path = '/api/v1/users/token' + + const body = { + client_id: this.server.store.client.id, + client_secret: this.server.store.client.secret, + refresh_token: options.refreshToken, + response_type: 'code', + grant_type: 'refresh_token' + } + + return this.postBodyRequest({ + ...options, + + path, + requestType: 'form', + fields: body, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getClient (options: OverrideCommandOptions = {}) { + const path = '/api/v1/oauth-clients/local' + + return this.getRequestBody<{ client_id: string, client_secret: string }>({ + ...options, + + path, + host: this.server.host, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + private _login (options: LoginOptions) { + const { client = this.server.store.client, user = this.server.store.user, otpToken } = options + const path = '/api/v1/users/token' + + const body = { + client_id: client.id, + client_secret: client.secret, + username: user.username, + password: user.password ?? 'password', + response_type: 'code', + grant_type: 'password', + scope: 'upload' + } + + const headers = otpToken + ? { 'x-peertube-otp': otpToken } + : {} + + return this.postBodyRequest({ + ...options, + + path, + headers, + requestType: 'form', + fields: body, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + private unwrapLoginBody (body: any) { + return body as { access_token: string, refresh_token: string } & PeerTubeProblemDocument + } +} diff --git a/packages/server-commands/src/users/login.ts b/packages/server-commands/src/users/login.ts new file mode 100644 index 000000000..c48c42c72 --- /dev/null +++ b/packages/server-commands/src/users/login.ts @@ -0,0 +1,19 @@ +import { PeerTubeServer } from '../server/server.js' + +function setAccessTokensToServers (servers: PeerTubeServer[]) { + const tasks: Promise[] = [] + + for (const server of servers) { + const p = server.login.getAccessToken() + .then(t => { server.accessToken = t }) + tasks.push(p) + } + + return Promise.all(tasks) +} + +// --------------------------------------------------------------------------- + +export { + setAccessTokensToServers +} diff --git a/packages/server-commands/src/users/notifications-command.ts b/packages/server-commands/src/users/notifications-command.ts new file mode 100644 index 000000000..d90d56900 --- /dev/null +++ b/packages/server-commands/src/users/notifications-command.ts @@ -0,0 +1,85 @@ +import { HttpStatusCode, ResultList, UserNotification, UserNotificationSetting } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class NotificationsCommand extends AbstractCommand { + + updateMySettings (options: OverrideCommandOptions & { + settings: UserNotificationSetting + }) { + const path = '/api/v1/users/me/notification-settings' + + return this.putBodyRequest({ + ...options, + + path, + fields: options.settings, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + list (options: OverrideCommandOptions & { + start?: number + count?: number + unread?: boolean + sort?: string + }) { + const { start, count, unread, sort = '-createdAt' } = options + const path = '/api/v1/users/me/notifications' + + return this.getRequestBody>({ + ...options, + + path, + query: { + start, + count, + sort, + unread + }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + markAsRead (options: OverrideCommandOptions & { + ids: number[] + }) { + const { ids } = options + const path = '/api/v1/users/me/notifications/read' + + return this.postBodyRequest({ + ...options, + + path, + fields: { ids }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + markAsReadAll (options: OverrideCommandOptions) { + const path = '/api/v1/users/me/notifications/read-all' + + return this.postBodyRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + async getLatest (options: OverrideCommandOptions = {}) { + const { total, data } = await this.list({ + ...options, + start: 0, + count: 1, + sort: '-createdAt' + }) + + if (total === 0) return undefined + + return data[0] + } +} diff --git a/packages/server-commands/src/users/registrations-command.ts b/packages/server-commands/src/users/registrations-command.ts new file mode 100644 index 000000000..2111fbd39 --- /dev/null +++ b/packages/server-commands/src/users/registrations-command.ts @@ -0,0 +1,157 @@ +import { pick } from '@peertube/peertube-core-utils' +import { + HttpStatusCode, + ResultList, + UserRegistration, + UserRegistrationRequest, + UserRegistrationUpdateState +} from '@peertube/peertube-models' +import { unwrapBody } from '../requests/index.js' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class RegistrationsCommand extends AbstractCommand { + + register (options: OverrideCommandOptions & Partial & Pick) { + const { password = 'password', email = options.username + '@example.com' } = options + const path = '/api/v1/users/register' + + return this.postBodyRequest({ + ...options, + + path, + fields: { + ...pick(options, [ 'username', 'displayName', 'channel' ]), + + password, + email + }, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + requestRegistration ( + options: OverrideCommandOptions & Partial & Pick + ) { + const { password = 'password', email = options.username + '@example.com' } = options + const path = '/api/v1/users/registrations/request' + + return unwrapBody(this.postBodyRequest({ + ...options, + + path, + fields: { + ...pick(options, [ 'username', 'displayName', 'channel', 'registrationReason' ]), + + password, + email + }, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + } + + // --------------------------------------------------------------------------- + + accept (options: OverrideCommandOptions & { id: number } & UserRegistrationUpdateState) { + const { id } = options + const path = '/api/v1/users/registrations/' + id + '/accept' + + return this.postBodyRequest({ + ...options, + + path, + fields: pick(options, [ 'moderationResponse', 'preventEmailDelivery' ]), + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + reject (options: OverrideCommandOptions & { id: number } & UserRegistrationUpdateState) { + const { id } = options + const path = '/api/v1/users/registrations/' + id + '/reject' + + return this.postBodyRequest({ + ...options, + + path, + fields: pick(options, [ 'moderationResponse', 'preventEmailDelivery' ]), + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + // --------------------------------------------------------------------------- + + delete (options: OverrideCommandOptions & { + id: number + }) { + const { id } = options + const path = '/api/v1/users/registrations/' + id + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + // --------------------------------------------------------------------------- + + list (options: OverrideCommandOptions & { + start?: number + count?: number + sort?: string + search?: string + } = {}) { + const path = '/api/v1/users/registrations' + + return this.getRequestBody>({ + ...options, + + path, + query: pick(options, [ 'start', 'count', 'sort', 'search' ]), + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + // --------------------------------------------------------------------------- + + askSendVerifyEmail (options: OverrideCommandOptions & { + email: string + }) { + const { email } = options + const path = '/api/v1/users/registrations/ask-send-verify-email' + + return this.postBodyRequest({ + ...options, + + path, + fields: { email }, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + verifyEmail (options: OverrideCommandOptions & { + registrationId: number + verificationString: string + }) { + const { registrationId, verificationString } = options + const path = '/api/v1/users/registrations/' + registrationId + '/verify-email' + + return this.postBodyRequest({ + ...options, + + path, + fields: { + verificationString + }, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/packages/server-commands/src/users/subscriptions-command.ts b/packages/server-commands/src/users/subscriptions-command.ts new file mode 100644 index 000000000..52a1f0e51 --- /dev/null +++ b/packages/server-commands/src/users/subscriptions-command.ts @@ -0,0 +1,83 @@ +import { HttpStatusCode, ResultList, VideoChannel } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class SubscriptionsCommand extends AbstractCommand { + + add (options: OverrideCommandOptions & { + targetUri: string + }) { + const path = '/api/v1/users/me/subscriptions' + + return this.postBodyRequest({ + ...options, + + path, + fields: { uri: options.targetUri }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + list (options: OverrideCommandOptions & { + sort?: string // default -createdAt + search?: string + } = {}) { + const { sort = '-createdAt', search } = options + const path = '/api/v1/users/me/subscriptions' + + return this.getRequestBody>({ + ...options, + + path, + query: { + sort, + search + }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + get (options: OverrideCommandOptions & { + uri: string + }) { + const path = '/api/v1/users/me/subscriptions/' + options.uri + + return this.getRequestBody({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + remove (options: OverrideCommandOptions & { + uri: string + }) { + const path = '/api/v1/users/me/subscriptions/' + options.uri + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + exist (options: OverrideCommandOptions & { + uris: string[] + }) { + const path = '/api/v1/users/me/subscriptions/exist' + + return this.getRequestBody<{ [id: string ]: boolean }>({ + ...options, + + path, + query: { 'uris[]': options.uris }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } +} diff --git a/packages/server-commands/src/users/two-factor-command.ts b/packages/server-commands/src/users/two-factor-command.ts new file mode 100644 index 000000000..cf3d6cb68 --- /dev/null +++ b/packages/server-commands/src/users/two-factor-command.ts @@ -0,0 +1,92 @@ +import { TOTP } from 'otpauth' +import { HttpStatusCode, TwoFactorEnableResult } from '@peertube/peertube-models' +import { unwrapBody } from '../requests/index.js' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class TwoFactorCommand extends AbstractCommand { + + static buildOTP (options: { + secret: string + }) { + const { secret } = options + + return new TOTP({ + issuer: 'PeerTube', + algorithm: 'SHA1', + digits: 6, + period: 30, + secret + }) + } + + request (options: OverrideCommandOptions & { + userId: number + currentPassword?: string + }) { + const { currentPassword, userId } = options + + const path = '/api/v1/users/' + userId + '/two-factor/request' + + return unwrapBody(this.postBodyRequest({ + ...options, + + path, + fields: { currentPassword }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + } + + confirmRequest (options: OverrideCommandOptions & { + userId: number + requestToken: string + otpToken: string + }) { + const { userId, requestToken, otpToken } = options + + const path = '/api/v1/users/' + userId + '/two-factor/confirm-request' + + return this.postBodyRequest({ + ...options, + + path, + fields: { requestToken, otpToken }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + disable (options: OverrideCommandOptions & { + userId: number + currentPassword?: string + }) { + const { userId, currentPassword } = options + const path = '/api/v1/users/' + userId + '/two-factor/disable' + + return this.postBodyRequest({ + ...options, + + path, + fields: { currentPassword }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + async requestAndConfirm (options: OverrideCommandOptions & { + userId: number + currentPassword?: string + }) { + const { userId, currentPassword } = options + + const { otpRequest } = await this.request({ userId, currentPassword }) + + await this.confirmRequest({ + userId, + requestToken: otpRequest.requestToken, + otpToken: TwoFactorCommand.buildOTP({ secret: otpRequest.secret }).generate() + }) + + return otpRequest + } +} diff --git a/packages/server-commands/src/users/users-command.ts b/packages/server-commands/src/users/users-command.ts new file mode 100644 index 000000000..d3b11939e --- /dev/null +++ b/packages/server-commands/src/users/users-command.ts @@ -0,0 +1,389 @@ +import { omit, pick } from '@peertube/peertube-core-utils' +import { + HttpStatusCode, + MyUser, + ResultList, + ScopedToken, + User, + UserAdminFlagType, + UserCreateResult, + UserRole, + UserRoleType, + UserUpdate, + UserUpdateMe, + UserVideoQuota, + UserVideoRate +} from '@peertube/peertube-models' +import { unwrapBody } from '../requests/index.js' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class UsersCommand extends AbstractCommand { + + askResetPassword (options: OverrideCommandOptions & { + email: string + }) { + const { email } = options + const path = '/api/v1/users/ask-reset-password' + + return this.postBodyRequest({ + ...options, + + path, + fields: { email }, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + resetPassword (options: OverrideCommandOptions & { + userId: number + verificationString: string + password: string + }) { + const { userId, verificationString, password } = options + const path = '/api/v1/users/' + userId + '/reset-password' + + return this.postBodyRequest({ + ...options, + + path, + fields: { password, verificationString }, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + // --------------------------------------------------------------------------- + + askSendVerifyEmail (options: OverrideCommandOptions & { + email: string + }) { + const { email } = options + const path = '/api/v1/users/ask-send-verify-email' + + return this.postBodyRequest({ + ...options, + + path, + fields: { email }, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + verifyEmail (options: OverrideCommandOptions & { + userId: number + verificationString: string + isPendingEmail?: boolean // default false + }) { + const { userId, verificationString, isPendingEmail = false } = options + const path = '/api/v1/users/' + userId + '/verify-email' + + return this.postBodyRequest({ + ...options, + + path, + fields: { + verificationString, + isPendingEmail + }, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + // --------------------------------------------------------------------------- + + banUser (options: OverrideCommandOptions & { + userId: number + reason?: string + }) { + const { userId, reason } = options + const path = '/api/v1/users' + '/' + userId + '/block' + + return this.postBodyRequest({ + ...options, + + path, + fields: { reason }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + unbanUser (options: OverrideCommandOptions & { + userId: number + }) { + const { userId } = options + const path = '/api/v1/users' + '/' + userId + '/unblock' + + return this.postBodyRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + // --------------------------------------------------------------------------- + + getMyScopedTokens (options: OverrideCommandOptions = {}) { + const path = '/api/v1/users/scoped-tokens' + + return this.getRequestBody({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + renewMyScopedTokens (options: OverrideCommandOptions = {}) { + const path = '/api/v1/users/scoped-tokens' + + return this.postBodyRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + // --------------------------------------------------------------------------- + + create (options: OverrideCommandOptions & { + username: string + password?: string + videoQuota?: number + videoQuotaDaily?: number + role?: UserRoleType + adminFlags?: UserAdminFlagType + }) { + const { + username, + adminFlags, + password = 'password', + videoQuota, + videoQuotaDaily, + role = UserRole.USER + } = options + + const path = '/api/v1/users' + + return unwrapBody<{ user: UserCreateResult }>(this.postBodyRequest({ + ...options, + + path, + fields: { + username, + password, + role, + adminFlags, + email: username + '@example.com', + videoQuota, + videoQuotaDaily + }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + })).then(res => res.user) + } + + async generate (username: string, role?: UserRoleType) { + const password = 'password' + const user = await this.create({ username, password, role }) + + const token = await this.server.login.getAccessToken({ username, password }) + + const me = await this.getMyInfo({ token }) + + return { + token, + userId: user.id, + userChannelId: me.videoChannels[0].id, + userChannelName: me.videoChannels[0].name, + password + } + } + + async generateUserAndToken (username: string, role?: UserRoleType) { + const password = 'password' + await this.create({ username, password, role }) + + return this.server.login.getAccessToken({ username, password }) + } + + // --------------------------------------------------------------------------- + + getMyInfo (options: OverrideCommandOptions = {}) { + const path = '/api/v1/users/me' + + return this.getRequestBody({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getMyQuotaUsed (options: OverrideCommandOptions = {}) { + const path = '/api/v1/users/me/video-quota-used' + + return this.getRequestBody({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getMyRating (options: OverrideCommandOptions & { + videoId: number | string + }) { + const { videoId } = options + const path = '/api/v1/users/me/videos/' + videoId + '/rating' + + return this.getRequestBody({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + deleteMe (options: OverrideCommandOptions = {}) { + const path = '/api/v1/users/me' + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + updateMe (options: OverrideCommandOptions & UserUpdateMe) { + const path = '/api/v1/users/me' + + const toSend: UserUpdateMe = omit(options, [ 'expectedStatus', 'token' ]) + + return this.putBodyRequest({ + ...options, + + path, + fields: toSend, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + updateMyAvatar (options: OverrideCommandOptions & { + fixture: string + }) { + const { fixture } = options + const path = '/api/v1/users/me/avatar/pick' + + return this.updateImageRequest({ + ...options, + + path, + fixture, + fieldname: 'avatarfile', + + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + // --------------------------------------------------------------------------- + + get (options: OverrideCommandOptions & { + userId: number + withStats?: boolean // default false + }) { + const { userId, withStats } = options + const path = '/api/v1/users/' + userId + + return this.getRequestBody({ + ...options, + + path, + query: { withStats }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + list (options: OverrideCommandOptions & { + start?: number + count?: number + sort?: string + search?: string + blocked?: boolean + } = {}) { + const path = '/api/v1/users' + + return this.getRequestBody>({ + ...options, + + path, + query: pick(options, [ 'start', 'count', 'sort', 'search', 'blocked' ]), + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + remove (options: OverrideCommandOptions & { + userId: number + }) { + const { userId } = options + const path = '/api/v1/users/' + userId + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + update (options: OverrideCommandOptions & { + userId: number + email?: string + emailVerified?: boolean + videoQuota?: number + videoQuotaDaily?: number + password?: string + adminFlags?: UserAdminFlagType + pluginAuth?: string + role?: UserRoleType + }) { + const path = '/api/v1/users/' + options.userId + + const toSend: UserUpdate = {} + if (options.password !== undefined && options.password !== null) toSend.password = options.password + if (options.email !== undefined && options.email !== null) toSend.email = options.email + if (options.emailVerified !== undefined && options.emailVerified !== null) toSend.emailVerified = options.emailVerified + if (options.videoQuota !== undefined && options.videoQuota !== null) toSend.videoQuota = options.videoQuota + if (options.videoQuotaDaily !== undefined && options.videoQuotaDaily !== null) toSend.videoQuotaDaily = options.videoQuotaDaily + if (options.role !== undefined && options.role !== null) toSend.role = options.role + if (options.adminFlags !== undefined && options.adminFlags !== null) toSend.adminFlags = options.adminFlags + if (options.pluginAuth !== undefined) toSend.pluginAuth = options.pluginAuth + + return this.putBodyRequest({ + ...options, + + path, + fields: toSend, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/packages/server-commands/src/videos/blacklist-command.ts b/packages/server-commands/src/videos/blacklist-command.ts new file mode 100644 index 000000000..d41001e26 --- /dev/null +++ b/packages/server-commands/src/videos/blacklist-command.ts @@ -0,0 +1,74 @@ +import { HttpStatusCode, ResultList, VideoBlacklist, VideoBlacklistType_Type } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class BlacklistCommand extends AbstractCommand { + + add (options: OverrideCommandOptions & { + videoId: number | string + reason?: string + unfederate?: boolean + }) { + const { videoId, reason, unfederate } = options + const path = '/api/v1/videos/' + videoId + '/blacklist' + + return this.postBodyRequest({ + ...options, + + path, + fields: { reason, unfederate }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + update (options: OverrideCommandOptions & { + videoId: number | string + reason?: string + }) { + const { videoId, reason } = options + const path = '/api/v1/videos/' + videoId + '/blacklist' + + return this.putBodyRequest({ + ...options, + + path, + fields: { reason }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + remove (options: OverrideCommandOptions & { + videoId: number | string + }) { + const { videoId } = options + const path = '/api/v1/videos/' + videoId + '/blacklist' + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + list (options: OverrideCommandOptions & { + sort?: string + type?: VideoBlacklistType_Type + } = {}) { + const { sort, type } = options + const path = '/api/v1/videos/blacklist/' + + const query = { sort, type } + + return this.getRequestBody>({ + ...options, + + path, + query, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } +} diff --git a/packages/server-commands/src/videos/captions-command.ts b/packages/server-commands/src/videos/captions-command.ts new file mode 100644 index 000000000..a8336aa27 --- /dev/null +++ b/packages/server-commands/src/videos/captions-command.ts @@ -0,0 +1,67 @@ +import { HttpStatusCode, ResultList, VideoCaption } from '@peertube/peertube-models' +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class CaptionsCommand extends AbstractCommand { + + add (options: OverrideCommandOptions & { + videoId: string | number + language: string + fixture: string + mimeType?: string + }) { + const { videoId, language, fixture, mimeType } = options + + const path = '/api/v1/videos/' + videoId + '/captions/' + language + + const captionfile = buildAbsoluteFixturePath(fixture) + const captionfileAttach = mimeType + ? [ captionfile, { contentType: mimeType } ] + : captionfile + + return this.putUploadRequest({ + ...options, + + path, + fields: {}, + attaches: { + captionfile: captionfileAttach + }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + list (options: OverrideCommandOptions & { + videoId: string | number + videoPassword?: string + }) { + const { videoId, videoPassword } = options + const path = '/api/v1/videos/' + videoId + '/captions' + + return this.getRequestBody>({ + ...options, + + path, + headers: this.buildVideoPasswordHeader(videoPassword), + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + delete (options: OverrideCommandOptions & { + videoId: string | number + language: string + }) { + const { videoId, language } = options + const path = '/api/v1/videos/' + videoId + '/captions/' + language + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/packages/server-commands/src/videos/change-ownership-command.ts b/packages/server-commands/src/videos/change-ownership-command.ts new file mode 100644 index 000000000..1dc7c2c0f --- /dev/null +++ b/packages/server-commands/src/videos/change-ownership-command.ts @@ -0,0 +1,67 @@ +import { HttpStatusCode, ResultList, VideoChangeOwnership } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class ChangeOwnershipCommand extends AbstractCommand { + + create (options: OverrideCommandOptions & { + videoId: number | string + username: string + }) { + const { videoId, username } = options + const path = '/api/v1/videos/' + videoId + '/give-ownership' + + return this.postBodyRequest({ + ...options, + + path, + fields: { username }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + list (options: OverrideCommandOptions = {}) { + const path = '/api/v1/videos/ownership' + + return this.getRequestBody>({ + ...options, + + path, + query: { sort: '-createdAt' }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + accept (options: OverrideCommandOptions & { + ownershipId: number + channelId: number + }) { + const { ownershipId, channelId } = options + const path = '/api/v1/videos/ownership/' + ownershipId + '/accept' + + return this.postBodyRequest({ + ...options, + + path, + fields: { channelId }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + refuse (options: OverrideCommandOptions & { + ownershipId: number + }) { + const { ownershipId } = options + const path = '/api/v1/videos/ownership/' + ownershipId + '/refuse' + + return this.postBodyRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/packages/server-commands/src/videos/channel-syncs-command.ts b/packages/server-commands/src/videos/channel-syncs-command.ts new file mode 100644 index 000000000..718000c8a --- /dev/null +++ b/packages/server-commands/src/videos/channel-syncs-command.ts @@ -0,0 +1,55 @@ +import { HttpStatusCode, ResultList, VideoChannelSync, VideoChannelSyncCreate } from '@peertube/peertube-models' +import { pick } from '@peertube/peertube-core-utils' +import { unwrapBody } from '../requests/index.js' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class ChannelSyncsCommand extends AbstractCommand { + private static readonly API_PATH = '/api/v1/video-channel-syncs' + + listByAccount (options: OverrideCommandOptions & { + accountName: string + start?: number + count?: number + sort?: string + }) { + const { accountName, sort = 'createdAt' } = options + + const path = `/api/v1/accounts/${accountName}/video-channel-syncs` + + return this.getRequestBody>({ + ...options, + + path, + query: { sort, ...pick(options, [ 'start', 'count' ]) }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + async create (options: OverrideCommandOptions & { + attributes: VideoChannelSyncCreate + }) { + return unwrapBody<{ videoChannelSync: VideoChannelSync }>(this.postBodyRequest({ + ...options, + + path: ChannelSyncsCommand.API_PATH, + fields: options.attributes, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + } + + delete (options: OverrideCommandOptions & { + channelSyncId: number + }) { + const path = `${ChannelSyncsCommand.API_PATH}/${options.channelSyncId}` + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/packages/server-commands/src/videos/channels-command.ts b/packages/server-commands/src/videos/channels-command.ts new file mode 100644 index 000000000..772677d39 --- /dev/null +++ b/packages/server-commands/src/videos/channels-command.ts @@ -0,0 +1,202 @@ +import { pick } from '@peertube/peertube-core-utils' +import { + ActorFollow, + HttpStatusCode, + ResultList, + VideoChannel, + VideoChannelCreate, + VideoChannelCreateResult, + VideoChannelUpdate, + VideosImportInChannelCreate +} from '@peertube/peertube-models' +import { unwrapBody } from '../requests/index.js' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class ChannelsCommand extends AbstractCommand { + + list (options: OverrideCommandOptions & { + start?: number + count?: number + sort?: string + withStats?: boolean + } = {}) { + const path = '/api/v1/video-channels' + + return this.getRequestBody>({ + ...options, + + path, + query: pick(options, [ 'start', 'count', 'sort', 'withStats' ]), + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + listByAccount (options: OverrideCommandOptions & { + accountName: string + start?: number + count?: number + sort?: string + withStats?: boolean + search?: string + }) { + const { accountName, sort = 'createdAt' } = options + const path = '/api/v1/accounts/' + accountName + '/video-channels' + + return this.getRequestBody>({ + ...options, + + path, + query: { sort, ...pick(options, [ 'start', 'count', 'withStats', 'search' ]) }, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + async create (options: OverrideCommandOptions & { + attributes: Partial + }) { + const path = '/api/v1/video-channels/' + + // Default attributes + const defaultAttributes = { + displayName: 'my super video channel', + description: 'my super channel description', + support: 'my super channel support' + } + const attributes = { ...defaultAttributes, ...options.attributes } + + const body = await unwrapBody<{ videoChannel: VideoChannelCreateResult }>(this.postBodyRequest({ + ...options, + + path, + fields: attributes, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + + return body.videoChannel + } + + update (options: OverrideCommandOptions & { + channelName: string + attributes: VideoChannelUpdate + }) { + const { channelName, attributes } = options + const path = '/api/v1/video-channels/' + channelName + + return this.putBodyRequest({ + ...options, + + path, + fields: attributes, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + delete (options: OverrideCommandOptions & { + channelName: string + }) { + const path = '/api/v1/video-channels/' + options.channelName + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + get (options: OverrideCommandOptions & { + channelName: string + }) { + const path = '/api/v1/video-channels/' + options.channelName + + return this.getRequestBody({ + ...options, + + path, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + updateImage (options: OverrideCommandOptions & { + fixture: string + channelName: string | number + type: 'avatar' | 'banner' + }) { + const { channelName, fixture, type } = options + + const path = `/api/v1/video-channels/${channelName}/${type}/pick` + + return this.updateImageRequest({ + ...options, + + path, + fixture, + fieldname: type + 'file', + + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + deleteImage (options: OverrideCommandOptions & { + channelName: string | number + type: 'avatar' | 'banner' + }) { + const { channelName, type } = options + + const path = `/api/v1/video-channels/${channelName}/${type}` + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + listFollowers (options: OverrideCommandOptions & { + channelName: string + start?: number + count?: number + sort?: string + search?: string + }) { + const { channelName, start, count, sort, search } = options + const path = '/api/v1/video-channels/' + channelName + '/followers' + + const query = { start, count, sort, search } + + return this.getRequestBody>({ + ...options, + + path, + query, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + importVideos (options: OverrideCommandOptions & VideosImportInChannelCreate & { + channelName: string + }) { + const { channelName, externalChannelUrl, videoChannelSyncId } = options + + const path = `/api/v1/video-channels/${channelName}/import-videos` + + return this.postBodyRequest({ + ...options, + + path, + fields: { externalChannelUrl, videoChannelSyncId }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/packages/server-commands/src/videos/channels.ts b/packages/server-commands/src/videos/channels.ts new file mode 100644 index 000000000..e3487d024 --- /dev/null +++ b/packages/server-commands/src/videos/channels.ts @@ -0,0 +1,29 @@ +import { PeerTubeServer } from '../server/server.js' + +function setDefaultVideoChannel (servers: PeerTubeServer[]) { + const tasks: Promise[] = [] + + for (const server of servers) { + const p = server.users.getMyInfo() + .then(user => { server.store.channel = user.videoChannels[0] }) + + tasks.push(p) + } + + return Promise.all(tasks) +} + +async function setDefaultChannelAvatar (serversArg: PeerTubeServer | PeerTubeServer[], channelName: string = 'root_channel') { + const servers = Array.isArray(serversArg) + ? serversArg + : [ serversArg ] + + for (const server of servers) { + await server.channels.updateImage({ channelName, fixture: 'avatar.png', type: 'avatar' }) + } +} + +export { + setDefaultVideoChannel, + setDefaultChannelAvatar +} diff --git a/packages/server-commands/src/videos/comments-command.ts b/packages/server-commands/src/videos/comments-command.ts new file mode 100644 index 000000000..4835ae1fb --- /dev/null +++ b/packages/server-commands/src/videos/comments-command.ts @@ -0,0 +1,159 @@ +import { pick } from '@peertube/peertube-core-utils' +import { HttpStatusCode, ResultList, VideoComment, VideoCommentThreads, VideoCommentThreadTree } from '@peertube/peertube-models' +import { unwrapBody } from '../requests/index.js' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class CommentsCommand extends AbstractCommand { + + private lastVideoId: number | string + private lastThreadId: number + private lastReplyId: number + + listForAdmin (options: OverrideCommandOptions & { + start?: number + count?: number + sort?: string + isLocal?: boolean + onLocalVideo?: boolean + search?: string + searchAccount?: string + searchVideo?: string + } = {}) { + const { sort = '-createdAt' } = options + const path = '/api/v1/videos/comments' + + const query = { sort, ...pick(options, [ 'start', 'count', 'isLocal', 'onLocalVideo', 'search', 'searchAccount', 'searchVideo' ]) } + + return this.getRequestBody>({ + ...options, + + path, + query, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + listThreads (options: OverrideCommandOptions & { + videoId: number | string + videoPassword?: string + start?: number + count?: number + sort?: string + }) { + const { start, count, sort, videoId, videoPassword } = options + const path = '/api/v1/videos/' + videoId + '/comment-threads' + + return this.getRequestBody({ + ...options, + + path, + query: { start, count, sort }, + headers: this.buildVideoPasswordHeader(videoPassword), + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getThread (options: OverrideCommandOptions & { + videoId: number | string + threadId: number + }) { + const { videoId, threadId } = options + const path = '/api/v1/videos/' + videoId + '/comment-threads/' + threadId + + return this.getRequestBody({ + ...options, + + path, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + async createThread (options: OverrideCommandOptions & { + videoId: number | string + text: string + videoPassword?: string + }) { + const { videoId, text, videoPassword } = options + const path = '/api/v1/videos/' + videoId + '/comment-threads' + + const body = await unwrapBody<{ comment: VideoComment }>(this.postBodyRequest({ + ...options, + + path, + fields: { text }, + headers: this.buildVideoPasswordHeader(videoPassword), + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + + this.lastThreadId = body.comment?.id + this.lastVideoId = videoId + + return body.comment + } + + async addReply (options: OverrideCommandOptions & { + videoId: number | string + toCommentId: number + text: string + videoPassword?: string + }) { + const { videoId, toCommentId, text, videoPassword } = options + const path = '/api/v1/videos/' + videoId + '/comments/' + toCommentId + + const body = await unwrapBody<{ comment: VideoComment }>(this.postBodyRequest({ + ...options, + + path, + fields: { text }, + headers: this.buildVideoPasswordHeader(videoPassword), + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + + this.lastReplyId = body.comment?.id + + return body.comment + } + + async addReplyToLastReply (options: OverrideCommandOptions & { + text: string + }) { + return this.addReply({ ...options, videoId: this.lastVideoId, toCommentId: this.lastReplyId }) + } + + async addReplyToLastThread (options: OverrideCommandOptions & { + text: string + }) { + return this.addReply({ ...options, videoId: this.lastVideoId, toCommentId: this.lastThreadId }) + } + + async findCommentId (options: OverrideCommandOptions & { + videoId: number | string + text: string + }) { + const { videoId, text } = options + const { data } = await this.listThreads({ videoId, count: 25, sort: '-createdAt' }) + + return data.find(c => c.text === text).id + } + + delete (options: OverrideCommandOptions & { + videoId: number | string + commentId: number + }) { + const { videoId, commentId } = options + const path = '/api/v1/videos/' + videoId + '/comments/' + commentId + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/packages/server-commands/src/videos/history-command.ts b/packages/server-commands/src/videos/history-command.ts new file mode 100644 index 000000000..fd032504a --- /dev/null +++ b/packages/server-commands/src/videos/history-command.ts @@ -0,0 +1,54 @@ +import { HttpStatusCode, ResultList, Video } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class HistoryCommand extends AbstractCommand { + + list (options: OverrideCommandOptions & { + search?: string + } = {}) { + const { search } = options + const path = '/api/v1/users/me/history/videos' + + return this.getRequestBody>({ + ...options, + + path, + query: { + search + }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + removeElement (options: OverrideCommandOptions & { + videoId: number + }) { + const { videoId } = options + const path = '/api/v1/users/me/history/videos/' + videoId + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + removeAll (options: OverrideCommandOptions & { + beforeDate?: string + } = {}) { + const { beforeDate } = options + const path = '/api/v1/users/me/history/videos/remove' + + return this.postBodyRequest({ + ...options, + + path, + fields: { beforeDate }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/packages/server-commands/src/videos/imports-command.ts b/packages/server-commands/src/videos/imports-command.ts new file mode 100644 index 000000000..1a1931d64 --- /dev/null +++ b/packages/server-commands/src/videos/imports-command.ts @@ -0,0 +1,76 @@ + +import { HttpStatusCode, ResultList, VideoImport, VideoImportCreate } from '@peertube/peertube-models' +import { unwrapBody } from '../requests/index.js' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class ImportsCommand extends AbstractCommand { + + importVideo (options: OverrideCommandOptions & { + attributes: (VideoImportCreate | { torrentfile?: string, previewfile?: string, thumbnailfile?: string }) + }) { + const { attributes } = options + const path = '/api/v1/videos/imports' + + let attaches: any = {} + if (attributes.torrentfile) attaches = { torrentfile: attributes.torrentfile } + if (attributes.thumbnailfile) attaches = { thumbnailfile: attributes.thumbnailfile } + if (attributes.previewfile) attaches = { previewfile: attributes.previewfile } + + return unwrapBody(this.postUploadRequest({ + ...options, + + path, + attaches, + fields: options.attributes, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + } + + delete (options: OverrideCommandOptions & { + importId: number + }) { + const path = '/api/v1/videos/imports/' + options.importId + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + cancel (options: OverrideCommandOptions & { + importId: number + }) { + const path = '/api/v1/videos/imports/' + options.importId + '/cancel' + + return this.postBodyRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + getMyVideoImports (options: OverrideCommandOptions & { + sort?: string + targetUrl?: string + videoChannelSyncId?: number + search?: string + } = {}) { + const { sort, targetUrl, videoChannelSyncId, search } = options + const path = '/api/v1/users/me/videos/imports' + + return this.getRequestBody>({ + ...options, + + path, + query: { sort, targetUrl, videoChannelSyncId, search }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } +} diff --git a/packages/server-commands/src/videos/index.ts b/packages/server-commands/src/videos/index.ts new file mode 100644 index 000000000..970026d51 --- /dev/null +++ b/packages/server-commands/src/videos/index.ts @@ -0,0 +1,22 @@ +export * from './blacklist-command.js' +export * from './captions-command.js' +export * from './change-ownership-command.js' +export * from './channels.js' +export * from './channels-command.js' +export * from './channel-syncs-command.js' +export * from './comments-command.js' +export * from './history-command.js' +export * from './imports-command.js' +export * from './live-command.js' +export * from './live.js' +export * from './playlists-command.js' +export * from './services-command.js' +export * from './storyboard-command.js' +export * from './streaming-playlists-command.js' +export * from './comments-command.js' +export * from './video-studio-command.js' +export * from './video-token-command.js' +export * from './views-command.js' +export * from './videos-command.js' +export * from './video-passwords-command.js' +export * from './video-stats-command.js' diff --git a/packages/server-commands/src/videos/live-command.ts b/packages/server-commands/src/videos/live-command.ts new file mode 100644 index 000000000..793b64f40 --- /dev/null +++ b/packages/server-commands/src/videos/live-command.ts @@ -0,0 +1,339 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { readdir } from 'fs/promises' +import { join } from 'path' +import { omit, wait } from '@peertube/peertube-core-utils' +import { + HttpStatusCode, + LiveVideo, + LiveVideoCreate, + LiveVideoSession, + LiveVideoUpdate, + ResultList, + VideoCreateResult, + VideoDetails, + VideoPrivacy, + VideoPrivacyType, + VideoState, + VideoStateType +} from '@peertube/peertube-models' +import { unwrapBody } from '../requests/index.js' +import { ObjectStorageCommand, PeerTubeServer } from '../server/index.js' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' +import { sendRTMPStream, testFfmpegStreamError } from './live.js' + +export class LiveCommand extends AbstractCommand { + + get (options: OverrideCommandOptions & { + videoId: number | string + }) { + const path = '/api/v1/videos/live' + + return this.getRequestBody({ + ...options, + + path: path + '/' + options.videoId, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + // --------------------------------------------------------------------------- + + listSessions (options: OverrideCommandOptions & { + videoId: number | string + }) { + const path = `/api/v1/videos/live/${options.videoId}/sessions` + + return this.getRequestBody>({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + async findLatestSession (options: OverrideCommandOptions & { + videoId: number | string + }) { + const { data: sessions } = await this.listSessions(options) + + return sessions[sessions.length - 1] + } + + getReplaySession (options: OverrideCommandOptions & { + videoId: number | string + }) { + const path = `/api/v1/videos/${options.videoId}/live-session` + + return this.getRequestBody({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + // --------------------------------------------------------------------------- + + update (options: OverrideCommandOptions & { + videoId: number | string + fields: LiveVideoUpdate + }) { + const { videoId, fields } = options + const path = '/api/v1/videos/live' + + return this.putBodyRequest({ + ...options, + + path: path + '/' + videoId, + fields, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + async create (options: OverrideCommandOptions & { + fields: LiveVideoCreate + }) { + const { fields } = options + const path = '/api/v1/videos/live' + + const attaches: any = {} + if (fields.thumbnailfile) attaches.thumbnailfile = fields.thumbnailfile + if (fields.previewfile) attaches.previewfile = fields.previewfile + + const body = await unwrapBody<{ video: VideoCreateResult }>(this.postUploadRequest({ + ...options, + + path, + attaches, + fields: omit(fields, [ 'thumbnailfile', 'previewfile' ]), + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + + return body.video + } + + async quickCreate (options: OverrideCommandOptions & { + saveReplay: boolean + permanentLive: boolean + privacy?: VideoPrivacyType + videoPasswords?: string[] + }) { + const { saveReplay, permanentLive, privacy = VideoPrivacy.PUBLIC, videoPasswords } = options + + const replaySettings = privacy === VideoPrivacy.PASSWORD_PROTECTED + ? { privacy: VideoPrivacy.PRIVATE } + : { privacy } + + const { uuid } = await this.create({ + ...options, + + fields: { + name: 'live', + permanentLive, + saveReplay, + replaySettings, + channelId: this.server.store.channel.id, + privacy, + videoPasswords + } + }) + + const video = await this.server.videos.getWithToken({ id: uuid }) + const live = await this.get({ videoId: uuid }) + + return { video, live } + } + + // --------------------------------------------------------------------------- + + async sendRTMPStreamInVideo (options: OverrideCommandOptions & { + videoId: number | string + fixtureName?: string + copyCodecs?: boolean + }) { + const { videoId, fixtureName, copyCodecs } = options + const videoLive = await this.get({ videoId }) + + return sendRTMPStream({ rtmpBaseUrl: videoLive.rtmpUrl, streamKey: videoLive.streamKey, fixtureName, copyCodecs }) + } + + async runAndTestStreamError (options: OverrideCommandOptions & { + videoId: number | string + shouldHaveError: boolean + }) { + const command = await this.sendRTMPStreamInVideo(options) + + return testFfmpegStreamError(command, options.shouldHaveError) + } + + // --------------------------------------------------------------------------- + + waitUntilPublished (options: OverrideCommandOptions & { + videoId: number | string + }) { + const { videoId } = options + return this.waitUntilState({ videoId, state: VideoState.PUBLISHED }) + } + + waitUntilWaiting (options: OverrideCommandOptions & { + videoId: number | string + }) { + const { videoId } = options + return this.waitUntilState({ videoId, state: VideoState.WAITING_FOR_LIVE }) + } + + waitUntilEnded (options: OverrideCommandOptions & { + videoId: number | string + }) { + const { videoId } = options + return this.waitUntilState({ videoId, state: VideoState.LIVE_ENDED }) + } + + async waitUntilSegmentGeneration (options: OverrideCommandOptions & { + server: PeerTubeServer + videoUUID: string + playlistNumber: number + segment: number + objectStorage?: ObjectStorageCommand + objectStorageBaseUrl?: string + }) { + const { + server, + objectStorage, + playlistNumber, + segment, + videoUUID, + objectStorageBaseUrl + } = options + + const segmentName = `${playlistNumber}-00000${segment}.ts` + const baseUrl = objectStorage + ? join(objectStorageBaseUrl || objectStorage.getMockPlaylistBaseUrl(), 'hls') + : server.url + '/static/streaming-playlists/hls' + + let error = true + + while (error) { + try { + // Check fragment exists + await this.getRawRequest({ + ...options, + + url: `${baseUrl}/${videoUUID}/${segmentName}`, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + + const video = await server.videos.get({ id: videoUUID }) + const hlsPlaylist = video.streamingPlaylists[0] + + // Check SHA generation + const shaBody = await server.streamingPlaylists.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url, withRetry: !!objectStorage }) + if (!shaBody[segmentName]) { + throw new Error('Segment SHA does not exist') + } + + // Check fragment is in m3u8 playlist + const subPlaylist = await server.streamingPlaylists.get({ url: `${baseUrl}/${video.uuid}/${playlistNumber}.m3u8` }) + if (!subPlaylist.includes(segmentName)) throw new Error('Fragment does not exist in playlist') + + error = false + } catch { + error = true + await wait(100) + } + } + } + + async waitUntilReplacedByReplay (options: OverrideCommandOptions & { + videoId: number | string + }) { + let video: VideoDetails + + do { + video = await this.server.videos.getWithToken({ token: options.token, id: options.videoId }) + + await wait(500) + } while (video.isLive === true || video.state.id !== VideoState.PUBLISHED) + } + + // --------------------------------------------------------------------------- + + getSegmentFile (options: OverrideCommandOptions & { + videoUUID: string + playlistNumber: number + segment: number + objectStorage?: ObjectStorageCommand + }) { + const { playlistNumber, segment, videoUUID, objectStorage } = options + + const segmentName = `${playlistNumber}-00000${segment}.ts` + const baseUrl = objectStorage + ? objectStorage.getMockPlaylistBaseUrl() + : `${this.server.url}/static/streaming-playlists/hls` + + const url = `${baseUrl}/${videoUUID}/${segmentName}` + + return this.getRawRequest({ + ...options, + + url, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getPlaylistFile (options: OverrideCommandOptions & { + videoUUID: string + playlistName: string + objectStorage?: ObjectStorageCommand + }) { + const { playlistName, videoUUID, objectStorage } = options + + const baseUrl = objectStorage + ? objectStorage.getMockPlaylistBaseUrl() + : `${this.server.url}/static/streaming-playlists/hls` + + const url = `${baseUrl}/${videoUUID}/${playlistName}` + + return this.getRawRequest({ + ...options, + + url, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + // --------------------------------------------------------------------------- + + async countPlaylists (options: OverrideCommandOptions & { + videoUUID: string + }) { + const basePath = this.server.servers.buildDirectory('streaming-playlists') + const hlsPath = join(basePath, 'hls', options.videoUUID) + + const files = await readdir(hlsPath) + + return files.filter(f => f.endsWith('.m3u8')).length + } + + private async waitUntilState (options: OverrideCommandOptions & { + videoId: number | string + state: VideoStateType + }) { + let video: VideoDetails + + do { + video = await this.server.videos.getWithToken({ token: options.token, id: options.videoId }) + + await wait(500) + } while (video.state.id !== options.state) + } +} diff --git a/packages/server-commands/src/videos/live.ts b/packages/server-commands/src/videos/live.ts new file mode 100644 index 000000000..05bfa1113 --- /dev/null +++ b/packages/server-commands/src/videos/live.ts @@ -0,0 +1,129 @@ +import { wait } from '@peertube/peertube-core-utils' +import { VideoDetails, VideoInclude, VideoPrivacy } from '@peertube/peertube-models' +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import ffmpeg, { FfmpegCommand } from 'fluent-ffmpeg' +import truncate from 'lodash-es/truncate.js' +import { PeerTubeServer } from '../server/server.js' + +function sendRTMPStream (options: { + rtmpBaseUrl: string + streamKey: string + fixtureName?: string // default video_short.mp4 + copyCodecs?: boolean // default false +}) { + const { rtmpBaseUrl, streamKey, fixtureName = 'video_short.mp4', copyCodecs = false } = options + + const fixture = buildAbsoluteFixturePath(fixtureName) + + const command = ffmpeg(fixture) + command.inputOption('-stream_loop -1') + command.inputOption('-re') + + if (copyCodecs) { + command.outputOption('-c copy') + } else { + command.outputOption('-c:v libx264') + command.outputOption('-g 120') + command.outputOption('-x264-params "no-scenecut=1"') + command.outputOption('-r 60') + } + + command.outputOption('-f flv') + + const rtmpUrl = rtmpBaseUrl + '/' + streamKey + command.output(rtmpUrl) + + command.on('error', err => { + if (err?.message?.includes('Exiting normally')) return + + if (process.env.DEBUG) console.error(err) + }) + + if (process.env.DEBUG) { + command.on('stderr', data => console.log(data)) + command.on('stdout', data => console.log(data)) + } + + command.run() + + return command +} + +function waitFfmpegUntilError (command: FfmpegCommand, successAfterMS = 10000) { + return new Promise((res, rej) => { + command.on('error', err => { + return rej(err) + }) + + setTimeout(() => { + res() + }, successAfterMS) + }) +} + +async function testFfmpegStreamError (command: FfmpegCommand, shouldHaveError: boolean) { + let error: Error + + try { + await waitFfmpegUntilError(command, 45000) + } catch (err) { + error = err + } + + await stopFfmpeg(command) + + if (shouldHaveError && !error) throw new Error('Ffmpeg did not have an error') + if (!shouldHaveError && error) throw error +} + +async function stopFfmpeg (command: FfmpegCommand) { + command.kill('SIGINT') + + await wait(500) +} + +async function waitUntilLivePublishedOnAllServers (servers: PeerTubeServer[], videoId: string) { + for (const server of servers) { + await server.live.waitUntilPublished({ videoId }) + } +} + +async function waitUntilLiveWaitingOnAllServers (servers: PeerTubeServer[], videoId: string) { + for (const server of servers) { + await server.live.waitUntilWaiting({ videoId }) + } +} + +async function waitUntilLiveReplacedByReplayOnAllServers (servers: PeerTubeServer[], videoId: string) { + for (const server of servers) { + await server.live.waitUntilReplacedByReplay({ videoId }) + } +} + +async function findExternalSavedVideo (server: PeerTubeServer, liveDetails: VideoDetails) { + const include = VideoInclude.BLACKLISTED + const privacyOneOf = [ VideoPrivacy.INTERNAL, VideoPrivacy.PRIVATE, VideoPrivacy.PUBLIC, VideoPrivacy.UNLISTED ] + + const { data } = await server.videos.list({ token: server.accessToken, sort: '-publishedAt', include, privacyOneOf }) + + const videoNameSuffix = ` - ${new Date(liveDetails.publishedAt).toLocaleString()}` + const truncatedVideoName = truncate(liveDetails.name, { + length: 120 - videoNameSuffix.length + }) + const toFind = truncatedVideoName + videoNameSuffix + + return data.find(v => v.name === toFind) +} + +export { + sendRTMPStream, + waitFfmpegUntilError, + testFfmpegStreamError, + stopFfmpeg, + + waitUntilLivePublishedOnAllServers, + waitUntilLiveReplacedByReplayOnAllServers, + waitUntilLiveWaitingOnAllServers, + + findExternalSavedVideo +} diff --git a/packages/server-commands/src/videos/playlists-command.ts b/packages/server-commands/src/videos/playlists-command.ts new file mode 100644 index 000000000..2e483f318 --- /dev/null +++ b/packages/server-commands/src/videos/playlists-command.ts @@ -0,0 +1,281 @@ +import { omit, pick } from '@peertube/peertube-core-utils' +import { + BooleanBothQuery, + HttpStatusCode, + ResultList, + VideoExistInPlaylist, + VideoPlaylist, + VideoPlaylistCreate, + VideoPlaylistCreateResult, + VideoPlaylistElement, + VideoPlaylistElementCreate, + VideoPlaylistElementCreateResult, + VideoPlaylistElementUpdate, + VideoPlaylistReorder, + VideoPlaylistType_Type, + VideoPlaylistUpdate +} from '@peertube/peertube-models' +import { unwrapBody } from '../requests/index.js' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class PlaylistsCommand extends AbstractCommand { + + list (options: OverrideCommandOptions & { + start?: number + count?: number + sort?: string + playlistType?: VideoPlaylistType_Type + }) { + const path = '/api/v1/video-playlists' + const query = pick(options, [ 'start', 'count', 'sort', 'playlistType' ]) + + return this.getRequestBody>({ + ...options, + + path, + query, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + listByChannel (options: OverrideCommandOptions & { + handle: string + start?: number + count?: number + sort?: string + playlistType?: VideoPlaylistType_Type + }) { + const path = '/api/v1/video-channels/' + options.handle + '/video-playlists' + const query = pick(options, [ 'start', 'count', 'sort', 'playlistType' ]) + + return this.getRequestBody>({ + ...options, + + path, + query, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + listByAccount (options: OverrideCommandOptions & { + handle: string + start?: number + count?: number + sort?: string + search?: string + playlistType?: VideoPlaylistType_Type + }) { + const path = '/api/v1/accounts/' + options.handle + '/video-playlists' + const query = pick(options, [ 'start', 'count', 'sort', 'search', 'playlistType' ]) + + return this.getRequestBody>({ + ...options, + + path, + query, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + get (options: OverrideCommandOptions & { + playlistId: number | string + }) { + const { playlistId } = options + const path = '/api/v1/video-playlists/' + playlistId + + return this.getRequestBody({ + ...options, + + path, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + listVideos (options: OverrideCommandOptions & { + playlistId: number | string + start?: number + count?: number + query?: { nsfw?: BooleanBothQuery } + }) { + const path = '/api/v1/video-playlists/' + options.playlistId + '/videos' + const query = options.query ?? {} + + return this.getRequestBody>({ + ...options, + + path, + query: { + ...query, + start: options.start, + count: options.count + }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + delete (options: OverrideCommandOptions & { + playlistId: number | string + }) { + const path = '/api/v1/video-playlists/' + options.playlistId + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + async create (options: OverrideCommandOptions & { + attributes: VideoPlaylistCreate + }) { + const path = '/api/v1/video-playlists' + + const fields = omit(options.attributes, [ 'thumbnailfile' ]) + + const attaches = options.attributes.thumbnailfile + ? { thumbnailfile: options.attributes.thumbnailfile } + : {} + + const body = await unwrapBody<{ videoPlaylist: VideoPlaylistCreateResult }>(this.postUploadRequest({ + ...options, + + path, + fields, + attaches, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + + return body.videoPlaylist + } + + update (options: OverrideCommandOptions & { + attributes: VideoPlaylistUpdate + playlistId: number | string + }) { + const path = '/api/v1/video-playlists/' + options.playlistId + + const fields = omit(options.attributes, [ 'thumbnailfile' ]) + + const attaches = options.attributes.thumbnailfile + ? { thumbnailfile: options.attributes.thumbnailfile } + : {} + + return this.putUploadRequest({ + ...options, + + path, + fields, + attaches, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + async addElement (options: OverrideCommandOptions & { + playlistId: number | string + attributes: VideoPlaylistElementCreate | { videoId: string } + }) { + const attributes = { + ...options.attributes, + + videoId: await this.server.videos.getId({ ...options, uuid: options.attributes.videoId }) + } + + const path = '/api/v1/video-playlists/' + options.playlistId + '/videos' + + const body = await unwrapBody<{ videoPlaylistElement: VideoPlaylistElementCreateResult }>(this.postBodyRequest({ + ...options, + + path, + fields: attributes, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + + return body.videoPlaylistElement + } + + updateElement (options: OverrideCommandOptions & { + playlistId: number | string + elementId: number | string + attributes: VideoPlaylistElementUpdate + }) { + const path = '/api/v1/video-playlists/' + options.playlistId + '/videos/' + options.elementId + + return this.putBodyRequest({ + ...options, + + path, + fields: options.attributes, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + removeElement (options: OverrideCommandOptions & { + playlistId: number | string + elementId: number + }) { + const path = '/api/v1/video-playlists/' + options.playlistId + '/videos/' + options.elementId + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + reorderElements (options: OverrideCommandOptions & { + playlistId: number | string + attributes: VideoPlaylistReorder + }) { + const path = '/api/v1/video-playlists/' + options.playlistId + '/videos/reorder' + + return this.postBodyRequest({ + ...options, + + path, + fields: options.attributes, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + getPrivacies (options: OverrideCommandOptions = {}) { + const path = '/api/v1/video-playlists/privacies' + + return this.getRequestBody<{ [ id: number ]: string }>({ + ...options, + + path, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + videosExist (options: OverrideCommandOptions & { + videoIds: number[] + }) { + const { videoIds } = options + const path = '/api/v1/users/me/video-playlists/videos-exist' + + return this.getRequestBody({ + ...options, + + path, + query: { videoIds }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } +} diff --git a/packages/server-commands/src/videos/services-command.ts b/packages/server-commands/src/videos/services-command.ts new file mode 100644 index 000000000..ade10cd3a --- /dev/null +++ b/packages/server-commands/src/videos/services-command.ts @@ -0,0 +1,29 @@ +import { HttpStatusCode } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class ServicesCommand extends AbstractCommand { + + getOEmbed (options: OverrideCommandOptions & { + oembedUrl: string + format?: string + maxHeight?: number + maxWidth?: number + }) { + const path = '/services/oembed' + const query = { + url: options.oembedUrl, + format: options.format, + maxheight: options.maxHeight, + maxwidth: options.maxWidth + } + + return this.getRequest({ + ...options, + + path, + query, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } +} diff --git a/packages/server-commands/src/videos/storyboard-command.ts b/packages/server-commands/src/videos/storyboard-command.ts new file mode 100644 index 000000000..a692ad612 --- /dev/null +++ b/packages/server-commands/src/videos/storyboard-command.ts @@ -0,0 +1,19 @@ +import { HttpStatusCode, Storyboard } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class StoryboardCommand extends AbstractCommand { + + list (options: OverrideCommandOptions & { + id: number | string + }) { + const path = '/api/v1/videos/' + options.id + '/storyboards' + + return this.getRequestBody<{ storyboards: Storyboard[] }>({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } +} diff --git a/packages/server-commands/src/videos/streaming-playlists-command.ts b/packages/server-commands/src/videos/streaming-playlists-command.ts new file mode 100644 index 000000000..2406dd023 --- /dev/null +++ b/packages/server-commands/src/videos/streaming-playlists-command.ts @@ -0,0 +1,119 @@ +import { wait } from '@peertube/peertube-core-utils' +import { HttpStatusCode } from '@peertube/peertube-models' +import { unwrapBody, unwrapBodyOrDecodeToJSON, unwrapTextOrDecode } from '../requests/index.js' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class StreamingPlaylistsCommand extends AbstractCommand { + + async get (options: OverrideCommandOptions & { + url: string + + videoFileToken?: string + reinjectVideoFileToken?: boolean + + withRetry?: boolean // default false + currentRetry?: number + }): Promise { + const { videoFileToken, reinjectVideoFileToken, expectedStatus, withRetry = false, currentRetry = 1 } = options + + try { + const result = await unwrapTextOrDecode(this.getRawRequest({ + ...options, + + url: options.url, + query: { + videoFileToken, + reinjectVideoFileToken + }, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + + // master.m3u8 could be empty + if (!result && (!expectedStatus || expectedStatus === HttpStatusCode.OK_200)) { + throw new Error('Empty result') + } + + return result + } catch (err) { + if (!withRetry || currentRetry > 10) throw err + + await wait(250) + + return this.get({ + ...options, + + withRetry, + currentRetry: currentRetry + 1 + }) + } + } + + async getFragmentedSegment (options: OverrideCommandOptions & { + url: string + range?: string + + withRetry?: boolean // default false + currentRetry?: number + }) { + const { withRetry = false, currentRetry = 1 } = options + + try { + const result = await unwrapBody(this.getRawRequest({ + ...options, + + url: options.url, + range: options.range, + implicitToken: false, + responseType: 'application/octet-stream', + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + + return result + } catch (err) { + if (!withRetry || currentRetry > 10) throw err + + await wait(250) + + return this.getFragmentedSegment({ + ...options, + + withRetry, + currentRetry: currentRetry + 1 + }) + } + } + + async getSegmentSha256 (options: OverrideCommandOptions & { + url: string + + withRetry?: boolean // default false + currentRetry?: number + }) { + const { withRetry = false, currentRetry = 1 } = options + + try { + const result = await unwrapBodyOrDecodeToJSON<{ [ id: string ]: string }>(this.getRawRequest({ + ...options, + + url: options.url, + contentType: 'application/json', + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + + return result + } catch (err) { + if (!withRetry || currentRetry > 10) throw err + + await wait(250) + + return this.getSegmentSha256({ + ...options, + + withRetry, + currentRetry: currentRetry + 1 + }) + } + } +} diff --git a/packages/server-commands/src/videos/video-passwords-command.ts b/packages/server-commands/src/videos/video-passwords-command.ts new file mode 100644 index 000000000..7a56311ca --- /dev/null +++ b/packages/server-commands/src/videos/video-passwords-command.ts @@ -0,0 +1,56 @@ +import { HttpStatusCode, ResultList, VideoPassword } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class VideoPasswordsCommand extends AbstractCommand { + + list (options: OverrideCommandOptions & { + videoId: number | string + start?: number + count?: number + sort?: string + }) { + const { start, count, sort, videoId } = options + const path = '/api/v1/videos/' + videoId + '/passwords' + + return this.getRequestBody>({ + ...options, + + path, + query: { start, count, sort }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + updateAll (options: OverrideCommandOptions & { + videoId: number | string + passwords: string[] + }) { + const { videoId, passwords } = options + const path = `/api/v1/videos/${videoId}/passwords` + + return this.putBodyRequest({ + ...options, + path, + fields: { passwords }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + remove (options: OverrideCommandOptions & { + id: number + videoId: number | string + }) { + const { id, videoId } = options + const path = `/api/v1/videos/${videoId}/passwords/${id}` + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/packages/server-commands/src/videos/video-stats-command.ts b/packages/server-commands/src/videos/video-stats-command.ts new file mode 100644 index 000000000..1b7a9b592 --- /dev/null +++ b/packages/server-commands/src/videos/video-stats-command.ts @@ -0,0 +1,62 @@ +import { pick } from '@peertube/peertube-core-utils' +import { + HttpStatusCode, + VideoStatsOverall, + VideoStatsRetention, + VideoStatsTimeserie, + VideoStatsTimeserieMetric +} from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class VideoStatsCommand extends AbstractCommand { + + getOverallStats (options: OverrideCommandOptions & { + videoId: number | string + startDate?: string + endDate?: string + }) { + const path = '/api/v1/videos/' + options.videoId + '/stats/overall' + + return this.getRequestBody({ + ...options, + path, + + query: pick(options, [ 'startDate', 'endDate' ]), + + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getTimeserieStats (options: OverrideCommandOptions & { + videoId: number | string + metric: VideoStatsTimeserieMetric + startDate?: Date + endDate?: Date + }) { + const path = '/api/v1/videos/' + options.videoId + '/stats/timeseries/' + options.metric + + return this.getRequestBody({ + ...options, + path, + + query: pick(options, [ 'startDate', 'endDate' ]), + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getRetentionStats (options: OverrideCommandOptions & { + videoId: number | string + }) { + const path = '/api/v1/videos/' + options.videoId + '/stats/retention' + + return this.getRequestBody({ + ...options, + path, + + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } +} diff --git a/packages/server-commands/src/videos/video-studio-command.ts b/packages/server-commands/src/videos/video-studio-command.ts new file mode 100644 index 000000000..8c5ff169a --- /dev/null +++ b/packages/server-commands/src/videos/video-studio-command.ts @@ -0,0 +1,67 @@ +import { HttpStatusCode, VideoStudioTask } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class VideoStudioCommand extends AbstractCommand { + + static getComplexTask (): VideoStudioTask[] { + return [ + // Total duration: 2 + { + name: 'cut', + options: { + start: 1, + end: 3 + } + }, + + // Total duration: 7 + { + name: 'add-outro', + options: { + file: 'video_short.webm' + } + }, + + { + name: 'add-watermark', + options: { + file: 'custom-thumbnail.png' + } + }, + + // Total duration: 9 + { + name: 'add-intro', + options: { + file: 'video_very_short_240p.mp4' + } + } + ] + } + + createEditionTasks (options: OverrideCommandOptions & { + videoId: number | string + tasks: VideoStudioTask[] + }) { + const path = '/api/v1/videos/' + options.videoId + '/studio/edit' + const attaches: { [id: string]: any } = {} + + for (let i = 0; i < options.tasks.length; i++) { + const task = options.tasks[i] + + if (task.name === 'add-intro' || task.name === 'add-outro' || task.name === 'add-watermark') { + attaches[`tasks[${i}][options][file]`] = task.options.file + } + } + + return this.postUploadRequest({ + ...options, + + path, + attaches, + fields: { tasks: options.tasks }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/packages/server-commands/src/videos/video-token-command.ts b/packages/server-commands/src/videos/video-token-command.ts new file mode 100644 index 000000000..5812e484a --- /dev/null +++ b/packages/server-commands/src/videos/video-token-command.ts @@ -0,0 +1,34 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */ + +import { HttpStatusCode, VideoToken } from '@peertube/peertube-models' +import { unwrapBody } from '../requests/index.js' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class VideoTokenCommand extends AbstractCommand { + + create (options: OverrideCommandOptions & { + videoId: number | string + videoPassword?: string + }) { + const { videoId, videoPassword } = options + const path = '/api/v1/videos/' + videoId + '/token' + + return unwrapBody(this.postBodyRequest({ + ...options, + headers: this.buildVideoPasswordHeader(videoPassword), + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + } + + async getVideoFileToken (options: OverrideCommandOptions & { + videoId: number | string + videoPassword?: string + }) { + const { files } = await this.create(options) + + return files.token + } +} diff --git a/packages/server-commands/src/videos/videos-command.ts b/packages/server-commands/src/videos/videos-command.ts new file mode 100644 index 000000000..72dc58a4b --- /dev/null +++ b/packages/server-commands/src/videos/videos-command.ts @@ -0,0 +1,831 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */ + +import { expect } from 'chai' +import { createReadStream } from 'fs' +import { stat } from 'fs/promises' +import got, { Response as GotResponse } from 'got' +import validator from 'validator' +import { getAllPrivacies, omit, pick, wait } from '@peertube/peertube-core-utils' +import { + HttpStatusCode, + HttpStatusCodeType, + ResultList, + UserVideoRateType, + Video, + VideoCreate, + VideoCreateResult, + VideoDetails, + VideoFileMetadata, + VideoInclude, + VideoPrivacy, + VideoPrivacyType, + VideosCommonQuery, + VideoSource, + VideoTranscodingCreate +} from '@peertube/peertube-models' +import { buildAbsoluteFixturePath, buildUUID } from '@peertube/peertube-node-utils' +import { unwrapBody } from '../requests/index.js' +import { waitJobs } from '../server/jobs.js' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export type VideoEdit = Partial> & { + fixture?: string + thumbnailfile?: string + previewfile?: string +} + +export class VideosCommand extends AbstractCommand { + + getCategories (options: OverrideCommandOptions = {}) { + const path = '/api/v1/videos/categories' + + return this.getRequestBody<{ [id: number]: string }>({ + ...options, + path, + + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getLicences (options: OverrideCommandOptions = {}) { + const path = '/api/v1/videos/licences' + + return this.getRequestBody<{ [id: number]: string }>({ + ...options, + path, + + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getLanguages (options: OverrideCommandOptions = {}) { + const path = '/api/v1/videos/languages' + + return this.getRequestBody<{ [id: string]: string }>({ + ...options, + path, + + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getPrivacies (options: OverrideCommandOptions = {}) { + const path = '/api/v1/videos/privacies' + + return this.getRequestBody<{ [id in VideoPrivacyType]: string }>({ + ...options, + path, + + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + // --------------------------------------------------------------------------- + + getDescription (options: OverrideCommandOptions & { + descriptionPath: string + }) { + return this.getRequestBody<{ description: string }>({ + ...options, + path: options.descriptionPath, + + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getFileMetadata (options: OverrideCommandOptions & { + url: string + }) { + return unwrapBody(this.getRawRequest({ + ...options, + + url: options.url, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + } + + // --------------------------------------------------------------------------- + + rate (options: OverrideCommandOptions & { + id: number | string + rating: UserVideoRateType + videoPassword?: string + }) { + const { id, rating, videoPassword } = options + const path = '/api/v1/videos/' + id + '/rate' + + return this.putBodyRequest({ + ...options, + + path, + fields: { rating }, + headers: this.buildVideoPasswordHeader(videoPassword), + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + // --------------------------------------------------------------------------- + + get (options: OverrideCommandOptions & { + id: number | string + }) { + const path = '/api/v1/videos/' + options.id + + return this.getRequestBody({ + ...options, + + path, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getWithToken (options: OverrideCommandOptions & { + id: number | string + }) { + return this.get({ + ...options, + + token: this.buildCommonRequestToken({ ...options, implicitToken: true }) + }) + } + + getWithPassword (options: OverrideCommandOptions & { + id: number | string + password?: string + }) { + const path = '/api/v1/videos/' + options.id + + return this.getRequestBody({ + ...options, + headers:{ + 'x-peertube-video-password': options.password + }, + path, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getSource (options: OverrideCommandOptions & { + id: number | string + }) { + const path = '/api/v1/videos/' + options.id + '/source' + + return this.getRequestBody({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + async getId (options: OverrideCommandOptions & { + uuid: number | string + }) { + const { uuid } = options + + if (validator.default.isUUID('' + uuid) === false) return uuid as number + + const { id } = await this.get({ ...options, id: uuid }) + + return id + } + + async listFiles (options: OverrideCommandOptions & { + id: number | string + }) { + const video = await this.get(options) + + const files = video.files || [] + const hlsFiles = video.streamingPlaylists[0]?.files || [] + + return files.concat(hlsFiles) + } + + // --------------------------------------------------------------------------- + + listMyVideos (options: OverrideCommandOptions & { + start?: number + count?: number + sort?: string + search?: string + isLive?: boolean + channelId?: number + } = {}) { + const path = '/api/v1/users/me/videos' + + return this.getRequestBody>({ + ...options, + + path, + query: pick(options, [ 'start', 'count', 'sort', 'search', 'isLive', 'channelId' ]), + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + listMySubscriptionVideos (options: OverrideCommandOptions & VideosCommonQuery = {}) { + const { sort = '-createdAt' } = options + const path = '/api/v1/users/me/subscriptions/videos' + + return this.getRequestBody>({ + ...options, + + path, + query: { sort, ...this.buildListQuery(options) }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + // --------------------------------------------------------------------------- + + list (options: OverrideCommandOptions & VideosCommonQuery = {}) { + const path = '/api/v1/videos' + + const query = this.buildListQuery(options) + + return this.getRequestBody>({ + ...options, + + path, + query: { sort: 'name', ...query }, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + listWithToken (options: OverrideCommandOptions & VideosCommonQuery = {}) { + return this.list({ + ...options, + + token: this.buildCommonRequestToken({ ...options, implicitToken: true }) + }) + } + + listAllForAdmin (options: OverrideCommandOptions & VideosCommonQuery = {}) { + const include = VideoInclude.NOT_PUBLISHED_STATE | VideoInclude.BLACKLISTED | VideoInclude.BLOCKED_OWNER + const nsfw = 'both' + const privacyOneOf = getAllPrivacies() + + return this.list({ + ...options, + + include, + nsfw, + privacyOneOf, + + token: this.buildCommonRequestToken({ ...options, implicitToken: true }) + }) + } + + listByAccount (options: OverrideCommandOptions & VideosCommonQuery & { + handle: string + }) { + const { handle, search } = options + const path = '/api/v1/accounts/' + handle + '/videos' + + return this.getRequestBody>({ + ...options, + + path, + query: { search, ...this.buildListQuery(options) }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + listByChannel (options: OverrideCommandOptions & VideosCommonQuery & { + handle: string + }) { + const { handle } = options + const path = '/api/v1/video-channels/' + handle + '/videos' + + return this.getRequestBody>({ + ...options, + + path, + query: this.buildListQuery(options), + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + // --------------------------------------------------------------------------- + + async find (options: OverrideCommandOptions & { + name: string + }) { + const { data } = await this.list(options) + + return data.find(v => v.name === options.name) + } + + // --------------------------------------------------------------------------- + + update (options: OverrideCommandOptions & { + id: number | string + attributes?: VideoEdit + }) { + const { id, attributes = {} } = options + const path = '/api/v1/videos/' + id + + // Upload request + if (attributes.thumbnailfile || attributes.previewfile) { + const attaches: any = {} + if (attributes.thumbnailfile) attaches.thumbnailfile = attributes.thumbnailfile + if (attributes.previewfile) attaches.previewfile = attributes.previewfile + + return this.putUploadRequest({ + ...options, + + path, + fields: options.attributes, + attaches: { + thumbnailfile: attributes.thumbnailfile, + previewfile: attributes.previewfile + }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + return this.putBodyRequest({ + ...options, + + path, + fields: options.attributes, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + remove (options: OverrideCommandOptions & { + id: number | string + }) { + const path = '/api/v1/videos/' + options.id + + return unwrapBody(this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + })) + } + + async removeAll () { + const { data } = await this.list() + + for (const v of data) { + await this.remove({ id: v.id }) + } + } + + // --------------------------------------------------------------------------- + + async upload (options: OverrideCommandOptions & { + attributes?: VideoEdit + mode?: 'legacy' | 'resumable' // default legacy + waitTorrentGeneration?: boolean // default true + completedExpectedStatus?: HttpStatusCodeType + } = {}) { + const { mode = 'legacy', waitTorrentGeneration = true } = options + let defaultChannelId = 1 + + try { + const { videoChannels } = await this.server.users.getMyInfo({ token: options.token }) + defaultChannelId = videoChannels[0].id + } catch (e) { /* empty */ } + + // Override default attributes + const attributes = { + name: 'my super video', + category: 5, + licence: 4, + language: 'zh', + channelId: defaultChannelId, + nsfw: true, + waitTranscoding: false, + description: 'my super description', + support: 'my super support text', + tags: [ 'tag' ], + privacy: VideoPrivacy.PUBLIC, + commentsEnabled: true, + downloadEnabled: true, + fixture: 'video_short.webm', + + ...options.attributes + } + + const created = mode === 'legacy' + ? await this.buildLegacyUpload({ ...options, attributes }) + : await this.buildResumeUpload({ ...options, path: '/api/v1/videos/upload-resumable', attributes }) + + // Wait torrent generation + const expectedStatus = this.buildExpectedStatus({ ...options, defaultExpectedStatus: HttpStatusCode.OK_200 }) + if (expectedStatus === HttpStatusCode.OK_200 && waitTorrentGeneration) { + let video: VideoDetails + + do { + video = await this.getWithToken({ ...options, id: created.uuid }) + + await wait(50) + } while (!video.files[0].torrentUrl) + } + + return created + } + + async buildLegacyUpload (options: OverrideCommandOptions & { + attributes: VideoEdit + }): Promise { + const path = '/api/v1/videos/upload' + + return unwrapBody<{ video: VideoCreateResult }>(this.postUploadRequest({ + ...options, + + path, + fields: this.buildUploadFields(options.attributes), + attaches: this.buildUploadAttaches(options.attributes), + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + })).then(body => body.video || body as any) + } + + async buildResumeUpload (options: OverrideCommandOptions & { + path: string + attributes: { fixture?: string } & { [id: string]: any } + completedExpectedStatus?: HttpStatusCodeType // When the upload is finished + }): Promise { + const { path, attributes, expectedStatus = HttpStatusCode.OK_200, completedExpectedStatus } = options + + let size = 0 + let videoFilePath: string + let mimetype = 'video/mp4' + + if (attributes.fixture) { + videoFilePath = buildAbsoluteFixturePath(attributes.fixture) + size = (await stat(videoFilePath)).size + + if (videoFilePath.endsWith('.mkv')) { + mimetype = 'video/x-matroska' + } else if (videoFilePath.endsWith('.webm')) { + mimetype = 'video/webm' + } + } + + // Do not check status automatically, we'll check it manually + const initializeSessionRes = await this.prepareResumableUpload({ + ...options, + + path, + expectedStatus: null, + attributes, + size, + mimetype + }) + const initStatus = initializeSessionRes.status + + if (videoFilePath && initStatus === HttpStatusCode.CREATED_201) { + const locationHeader = initializeSessionRes.header['location'] + expect(locationHeader).to.not.be.undefined + + const pathUploadId = locationHeader.split('?')[1] + + const result = await this.sendResumableChunks({ + ...options, + + path, + pathUploadId, + videoFilePath, + size, + expectedStatus: completedExpectedStatus + }) + + if (result.statusCode === HttpStatusCode.OK_200) { + await this.endResumableUpload({ + ...options, + + expectedStatus: HttpStatusCode.NO_CONTENT_204, + path, + pathUploadId + }) + } + + return result.body?.video || result.body as any + } + + const expectedInitStatus = expectedStatus === HttpStatusCode.OK_200 + ? HttpStatusCode.CREATED_201 + : expectedStatus + + expect(initStatus).to.equal(expectedInitStatus) + + return initializeSessionRes.body.video || initializeSessionRes.body + } + + async prepareResumableUpload (options: OverrideCommandOptions & { + path: string + attributes: { fixture?: string } & { [id: string]: any } + size: number + mimetype: string + + originalName?: string + lastModified?: number + }) { + const { path, attributes, originalName, lastModified, size, mimetype } = options + + const attaches = this.buildUploadAttaches(omit(options.attributes, [ 'fixture' ])) + + const uploadOptions = { + ...options, + + path, + headers: { + 'X-Upload-Content-Type': mimetype, + 'X-Upload-Content-Length': size.toString() + }, + fields: { + filename: attributes.fixture, + originalName, + lastModified, + + ...this.buildUploadFields(options.attributes) + }, + + // Fixture will be sent later + attaches: this.buildUploadAttaches(omit(options.attributes, [ 'fixture' ])), + implicitToken: true, + + defaultExpectedStatus: null + } + + if (Object.keys(attaches).length === 0) return this.postBodyRequest(uploadOptions) + + return this.postUploadRequest(uploadOptions) + } + + sendResumableChunks (options: OverrideCommandOptions & { + pathUploadId: string + path: string + videoFilePath: string + size: number + contentLength?: number + contentRangeBuilder?: (start: number, chunk: any) => string + digestBuilder?: (chunk: any) => string + }) { + const { + path, + pathUploadId, + videoFilePath, + size, + contentLength, + contentRangeBuilder, + digestBuilder, + expectedStatus = HttpStatusCode.OK_200 + } = options + + let start = 0 + + const token = this.buildCommonRequestToken({ ...options, implicitToken: true }) + + const readable = createReadStream(videoFilePath, { highWaterMark: 8 * 1024 }) + const server = this.server + return new Promise>((resolve, reject) => { + readable.on('data', async function onData (chunk) { + try { + readable.pause() + + const byterangeStart = start + chunk.length - 1 + + const headers = { + 'Authorization': 'Bearer ' + token, + 'Content-Type': 'application/octet-stream', + 'Content-Range': contentRangeBuilder + ? contentRangeBuilder(start, chunk) + : `bytes ${start}-${byterangeStart}/${size}`, + 'Content-Length': contentLength ? contentLength + '' : chunk.length + '' + } + + if (digestBuilder) { + Object.assign(headers, { digest: digestBuilder(chunk) }) + } + + const res = await got<{ video: VideoCreateResult }>({ + url: new URL(path + '?' + pathUploadId, server.url).toString(), + method: 'put', + headers, + body: chunk, + responseType: 'json', + throwHttpErrors: false + }) + + start += chunk.length + + // Last request, check final status + if (byterangeStart + 1 === size) { + if (res.statusCode === expectedStatus) { + return resolve(res) + } + + if (res.statusCode !== HttpStatusCode.PERMANENT_REDIRECT_308) { + readable.off('data', onData) + + // eslint-disable-next-line max-len + const message = `Incorrect transient behaviour sending intermediary chunks. Status code is ${res.statusCode} instead of ${expectedStatus}` + return reject(new Error(message)) + } + } + + readable.resume() + } catch (err) { + reject(err) + } + }) + }) + } + + endResumableUpload (options: OverrideCommandOptions & { + path: string + pathUploadId: string + }) { + return this.deleteRequest({ + ...options, + + path: options.path, + rawQuery: options.pathUploadId, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + quickUpload (options: OverrideCommandOptions & { + name: string + nsfw?: boolean + privacy?: VideoPrivacyType + fixture?: string + videoPasswords?: string[] + }) { + const attributes: VideoEdit = { name: options.name } + if (options.nsfw) attributes.nsfw = options.nsfw + if (options.privacy) attributes.privacy = options.privacy + if (options.fixture) attributes.fixture = options.fixture + if (options.videoPasswords) attributes.videoPasswords = options.videoPasswords + + return this.upload({ ...options, attributes }) + } + + async randomUpload (options: OverrideCommandOptions & { + wait?: boolean // default true + additionalParams?: VideoEdit & { prefixName?: string } + } = {}) { + const { wait = true, additionalParams } = options + const prefixName = additionalParams?.prefixName || '' + const name = prefixName + buildUUID() + + const attributes = { name, ...additionalParams } + + const result = await this.upload({ ...options, attributes }) + + if (wait) await waitJobs([ this.server ]) + + return { ...result, name } + } + + // --------------------------------------------------------------------------- + + replaceSourceFile (options: OverrideCommandOptions & { + videoId: number | string + fixture: string + completedExpectedStatus?: HttpStatusCodeType + }) { + return this.buildResumeUpload({ + ...options, + + path: '/api/v1/videos/' + options.videoId + '/source/replace-resumable', + attributes: { fixture: options.fixture } + }) + } + + // --------------------------------------------------------------------------- + + removeHLSPlaylist (options: OverrideCommandOptions & { + videoId: number | string + }) { + const path = '/api/v1/videos/' + options.videoId + '/hls' + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + removeHLSFile (options: OverrideCommandOptions & { + videoId: number | string + fileId: number + }) { + const path = '/api/v1/videos/' + options.videoId + '/hls/' + options.fileId + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + removeAllWebVideoFiles (options: OverrideCommandOptions & { + videoId: number | string + }) { + const path = '/api/v1/videos/' + options.videoId + '/web-videos' + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + removeWebVideoFile (options: OverrideCommandOptions & { + videoId: number | string + fileId: number + }) { + const path = '/api/v1/videos/' + options.videoId + '/web-videos/' + options.fileId + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + runTranscoding (options: OverrideCommandOptions & VideoTranscodingCreate & { + videoId: number | string + }) { + const path = '/api/v1/videos/' + options.videoId + '/transcoding' + + return this.postBodyRequest({ + ...options, + + path, + fields: pick(options, [ 'transcodingType', 'forceTranscoding' ]), + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + // --------------------------------------------------------------------------- + + private buildListQuery (options: VideosCommonQuery) { + return pick(options, [ + 'start', + 'count', + 'sort', + 'nsfw', + 'isLive', + 'categoryOneOf', + 'licenceOneOf', + 'languageOneOf', + 'privacyOneOf', + 'tagsOneOf', + 'tagsAllOf', + 'isLocal', + 'include', + 'skipCount' + ]) + } + + private buildUploadFields (attributes: VideoEdit) { + return omit(attributes, [ 'fixture', 'thumbnailfile', 'previewfile' ]) + } + + private buildUploadAttaches (attributes: VideoEdit) { + const attaches: { [ name: string ]: string } = {} + + for (const key of [ 'thumbnailfile', 'previewfile' ]) { + if (attributes[key]) attaches[key] = buildAbsoluteFixturePath(attributes[key]) + } + + if (attributes.fixture) attaches.videofile = buildAbsoluteFixturePath(attributes.fixture) + + return attaches + } +} diff --git a/packages/server-commands/src/videos/views-command.ts b/packages/server-commands/src/videos/views-command.ts new file mode 100644 index 000000000..048bd3fda --- /dev/null +++ b/packages/server-commands/src/videos/views-command.ts @@ -0,0 +1,51 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */ +import { HttpStatusCode, VideoViewEvent } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class ViewsCommand extends AbstractCommand { + + view (options: OverrideCommandOptions & { + id: number | string + currentTime: number + viewEvent?: VideoViewEvent + xForwardedFor?: string + }) { + const { id, xForwardedFor, viewEvent, currentTime } = options + const path = '/api/v1/videos/' + id + '/views' + + return this.postBodyRequest({ + ...options, + + path, + xForwardedFor, + fields: { + currentTime, + viewEvent + }, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + async simulateView (options: OverrideCommandOptions & { + id: number | string + xForwardedFor?: string + }) { + await this.view({ ...options, currentTime: 0 }) + await this.view({ ...options, currentTime: 5 }) + } + + async simulateViewer (options: OverrideCommandOptions & { + id: number | string + currentTimes: number[] + xForwardedFor?: string + }) { + let viewEvent: VideoViewEvent = 'seek' + + for (const currentTime of options.currentTimes) { + await this.view({ ...options, currentTime, viewEvent }) + + viewEvent = undefined + } + } +} diff --git a/packages/server-commands/tsconfig.json b/packages/server-commands/tsconfig.json new file mode 100644 index 000000000..eb942f295 --- /dev/null +++ b/packages/server-commands/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "baseUrl": "./", + "outDir": "./dist", + "rootDir": "src", + "tsBuildInfoFile": "./dist/.tsbuildinfo" + }, + "references": [ + { "path": "../models" }, + { "path": "../core-utils" }, + { "path": "../typescript-utils" } + ] +} diff --git a/server/tests/fixtures/60fps_720p_small.mp4 b/packages/tests/fixtures/60fps_720p_small.mp4 similarity index 100% rename from server/tests/fixtures/60fps_720p_small.mp4 rename to packages/tests/fixtures/60fps_720p_small.mp4 diff --git a/server/tests/fixtures/ap-json/mastodon/bad-body-http-signature.json b/packages/tests/fixtures/ap-json/mastodon/bad-body-http-signature.json similarity index 100% rename from server/tests/fixtures/ap-json/mastodon/bad-body-http-signature.json rename to packages/tests/fixtures/ap-json/mastodon/bad-body-http-signature.json diff --git a/server/tests/fixtures/ap-json/mastodon/bad-http-signature.json b/packages/tests/fixtures/ap-json/mastodon/bad-http-signature.json similarity index 100% rename from server/tests/fixtures/ap-json/mastodon/bad-http-signature.json rename to packages/tests/fixtures/ap-json/mastodon/bad-http-signature.json diff --git a/server/tests/fixtures/ap-json/mastodon/bad-public-key.json b/packages/tests/fixtures/ap-json/mastodon/bad-public-key.json similarity index 100% rename from server/tests/fixtures/ap-json/mastodon/bad-public-key.json rename to packages/tests/fixtures/ap-json/mastodon/bad-public-key.json diff --git a/server/tests/fixtures/ap-json/mastodon/create-bad-signature.json b/packages/tests/fixtures/ap-json/mastodon/create-bad-signature.json similarity index 100% rename from server/tests/fixtures/ap-json/mastodon/create-bad-signature.json rename to packages/tests/fixtures/ap-json/mastodon/create-bad-signature.json diff --git a/server/tests/fixtures/ap-json/mastodon/create.json b/packages/tests/fixtures/ap-json/mastodon/create.json similarity index 100% rename from server/tests/fixtures/ap-json/mastodon/create.json rename to packages/tests/fixtures/ap-json/mastodon/create.json diff --git a/server/tests/fixtures/ap-json/mastodon/http-signature.json b/packages/tests/fixtures/ap-json/mastodon/http-signature.json similarity index 100% rename from server/tests/fixtures/ap-json/mastodon/http-signature.json rename to packages/tests/fixtures/ap-json/mastodon/http-signature.json diff --git a/server/tests/fixtures/ap-json/mastodon/public-key.json b/packages/tests/fixtures/ap-json/mastodon/public-key.json similarity index 100% rename from server/tests/fixtures/ap-json/mastodon/public-key.json rename to packages/tests/fixtures/ap-json/mastodon/public-key.json diff --git a/server/tests/fixtures/ap-json/peertube/announce-without-context.json b/packages/tests/fixtures/ap-json/peertube/announce-without-context.json similarity index 100% rename from server/tests/fixtures/ap-json/peertube/announce-without-context.json rename to packages/tests/fixtures/ap-json/peertube/announce-without-context.json diff --git a/server/tests/fixtures/ap-json/peertube/invalid-keys.json b/packages/tests/fixtures/ap-json/peertube/invalid-keys.json similarity index 100% rename from server/tests/fixtures/ap-json/peertube/invalid-keys.json rename to packages/tests/fixtures/ap-json/peertube/invalid-keys.json diff --git a/server/tests/fixtures/ap-json/peertube/keys.json b/packages/tests/fixtures/ap-json/peertube/keys.json similarity index 100% rename from server/tests/fixtures/ap-json/peertube/keys.json rename to packages/tests/fixtures/ap-json/peertube/keys.json diff --git a/server/tests/fixtures/avatar-big.png b/packages/tests/fixtures/avatar-big.png similarity index 100% rename from server/tests/fixtures/avatar-big.png rename to packages/tests/fixtures/avatar-big.png diff --git a/server/tests/fixtures/avatar-resized-120x120.gif b/packages/tests/fixtures/avatar-resized-120x120.gif similarity index 100% rename from server/tests/fixtures/avatar-resized-120x120.gif rename to packages/tests/fixtures/avatar-resized-120x120.gif diff --git a/server/tests/fixtures/avatar-resized-120x120.png b/packages/tests/fixtures/avatar-resized-120x120.png similarity index 100% rename from server/tests/fixtures/avatar-resized-120x120.png rename to packages/tests/fixtures/avatar-resized-120x120.png diff --git a/server/tests/fixtures/avatar-resized-48x48.gif b/packages/tests/fixtures/avatar-resized-48x48.gif similarity index 100% rename from server/tests/fixtures/avatar-resized-48x48.gif rename to packages/tests/fixtures/avatar-resized-48x48.gif diff --git a/server/tests/fixtures/avatar-resized-48x48.png b/packages/tests/fixtures/avatar-resized-48x48.png similarity index 100% rename from server/tests/fixtures/avatar-resized-48x48.png rename to packages/tests/fixtures/avatar-resized-48x48.png diff --git a/server/tests/fixtures/avatar.gif b/packages/tests/fixtures/avatar.gif similarity index 100% rename from server/tests/fixtures/avatar.gif rename to packages/tests/fixtures/avatar.gif diff --git a/server/tests/fixtures/avatar.png b/packages/tests/fixtures/avatar.png similarity index 100% rename from server/tests/fixtures/avatar.png rename to packages/tests/fixtures/avatar.png diff --git a/server/tests/fixtures/avatar2-resized-120x120.png b/packages/tests/fixtures/avatar2-resized-120x120.png similarity index 100% rename from server/tests/fixtures/avatar2-resized-120x120.png rename to packages/tests/fixtures/avatar2-resized-120x120.png diff --git a/server/tests/fixtures/avatar2-resized-48x48.png b/packages/tests/fixtures/avatar2-resized-48x48.png similarity index 100% rename from server/tests/fixtures/avatar2-resized-48x48.png rename to packages/tests/fixtures/avatar2-resized-48x48.png diff --git a/server/tests/fixtures/avatar2.png b/packages/tests/fixtures/avatar2.png similarity index 100% rename from server/tests/fixtures/avatar2.png rename to packages/tests/fixtures/avatar2.png diff --git a/server/tests/fixtures/banner-resized.jpg b/packages/tests/fixtures/banner-resized.jpg similarity index 100% rename from server/tests/fixtures/banner-resized.jpg rename to packages/tests/fixtures/banner-resized.jpg diff --git a/server/tests/fixtures/banner.jpg b/packages/tests/fixtures/banner.jpg similarity index 100% rename from server/tests/fixtures/banner.jpg rename to packages/tests/fixtures/banner.jpg diff --git a/server/tests/fixtures/custom-preview-big.png b/packages/tests/fixtures/custom-preview-big.png similarity index 100% rename from server/tests/fixtures/custom-preview-big.png rename to packages/tests/fixtures/custom-preview-big.png diff --git a/server/tests/fixtures/custom-preview.jpg b/packages/tests/fixtures/custom-preview.jpg similarity index 100% rename from server/tests/fixtures/custom-preview.jpg rename to packages/tests/fixtures/custom-preview.jpg diff --git a/server/tests/fixtures/custom-thumbnail-big.jpg b/packages/tests/fixtures/custom-thumbnail-big.jpg similarity index 100% rename from server/tests/fixtures/custom-thumbnail-big.jpg rename to packages/tests/fixtures/custom-thumbnail-big.jpg diff --git a/server/tests/fixtures/custom-thumbnail.jpg b/packages/tests/fixtures/custom-thumbnail.jpg similarity index 100% rename from server/tests/fixtures/custom-thumbnail.jpg rename to packages/tests/fixtures/custom-thumbnail.jpg diff --git a/server/tests/fixtures/custom-thumbnail.png b/packages/tests/fixtures/custom-thumbnail.png similarity index 100% rename from server/tests/fixtures/custom-thumbnail.png rename to packages/tests/fixtures/custom-thumbnail.png diff --git a/server/tests/fixtures/exif.jpg b/packages/tests/fixtures/exif.jpg similarity index 100% rename from server/tests/fixtures/exif.jpg rename to packages/tests/fixtures/exif.jpg diff --git a/server/tests/fixtures/exif.png b/packages/tests/fixtures/exif.png similarity index 100% rename from server/tests/fixtures/exif.png rename to packages/tests/fixtures/exif.png diff --git a/server/tests/fixtures/live/0-000067.ts b/packages/tests/fixtures/live/0-000067.ts similarity index 100% rename from server/tests/fixtures/live/0-000067.ts rename to packages/tests/fixtures/live/0-000067.ts diff --git a/server/tests/fixtures/live/0-000068.ts b/packages/tests/fixtures/live/0-000068.ts similarity index 100% rename from server/tests/fixtures/live/0-000068.ts rename to packages/tests/fixtures/live/0-000068.ts diff --git a/server/tests/fixtures/live/0-000069.ts b/packages/tests/fixtures/live/0-000069.ts similarity index 100% rename from server/tests/fixtures/live/0-000069.ts rename to packages/tests/fixtures/live/0-000069.ts diff --git a/server/tests/fixtures/live/0-000070.ts b/packages/tests/fixtures/live/0-000070.ts similarity index 100% rename from server/tests/fixtures/live/0-000070.ts rename to packages/tests/fixtures/live/0-000070.ts diff --git a/server/tests/fixtures/live/0.m3u8 b/packages/tests/fixtures/live/0.m3u8 similarity index 100% rename from server/tests/fixtures/live/0.m3u8 rename to packages/tests/fixtures/live/0.m3u8 diff --git a/server/tests/fixtures/live/1-000067.ts b/packages/tests/fixtures/live/1-000067.ts similarity index 100% rename from server/tests/fixtures/live/1-000067.ts rename to packages/tests/fixtures/live/1-000067.ts diff --git a/server/tests/fixtures/live/1-000068.ts b/packages/tests/fixtures/live/1-000068.ts similarity index 100% rename from server/tests/fixtures/live/1-000068.ts rename to packages/tests/fixtures/live/1-000068.ts diff --git a/server/tests/fixtures/live/1-000069.ts b/packages/tests/fixtures/live/1-000069.ts similarity index 100% rename from server/tests/fixtures/live/1-000069.ts rename to packages/tests/fixtures/live/1-000069.ts diff --git a/server/tests/fixtures/live/1-000070.ts b/packages/tests/fixtures/live/1-000070.ts similarity index 100% rename from server/tests/fixtures/live/1-000070.ts rename to packages/tests/fixtures/live/1-000070.ts diff --git a/server/tests/fixtures/live/1.m3u8 b/packages/tests/fixtures/live/1.m3u8 similarity index 100% rename from server/tests/fixtures/live/1.m3u8 rename to packages/tests/fixtures/live/1.m3u8 diff --git a/server/tests/fixtures/live/master.m3u8 b/packages/tests/fixtures/live/master.m3u8 similarity index 100% rename from server/tests/fixtures/live/master.m3u8 rename to packages/tests/fixtures/live/master.m3u8 diff --git a/server/tests/fixtures/low-bitrate.mp4 b/packages/tests/fixtures/low-bitrate.mp4 similarity index 100% rename from server/tests/fixtures/low-bitrate.mp4 rename to packages/tests/fixtures/low-bitrate.mp4 diff --git a/server/tests/fixtures/peertube-plugin-test-broken/main.js b/packages/tests/fixtures/peertube-plugin-test-broken/main.js similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-broken/main.js rename to packages/tests/fixtures/peertube-plugin-test-broken/main.js diff --git a/server/tests/fixtures/peertube-plugin-test-broken/package.json b/packages/tests/fixtures/peertube-plugin-test-broken/package.json similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-broken/package.json rename to packages/tests/fixtures/peertube-plugin-test-broken/package.json diff --git a/server/tests/fixtures/peertube-plugin-test-external-auth-one/main.js b/packages/tests/fixtures/peertube-plugin-test-external-auth-one/main.js similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-external-auth-one/main.js rename to packages/tests/fixtures/peertube-plugin-test-external-auth-one/main.js diff --git a/server/tests/fixtures/peertube-plugin-test-external-auth-one/package.json b/packages/tests/fixtures/peertube-plugin-test-external-auth-one/package.json similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-external-auth-one/package.json rename to packages/tests/fixtures/peertube-plugin-test-external-auth-one/package.json diff --git a/server/tests/fixtures/peertube-plugin-test-external-auth-three/main.js b/packages/tests/fixtures/peertube-plugin-test-external-auth-three/main.js similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-external-auth-three/main.js rename to packages/tests/fixtures/peertube-plugin-test-external-auth-three/main.js diff --git a/server/tests/fixtures/peertube-plugin-test-external-auth-three/package.json b/packages/tests/fixtures/peertube-plugin-test-external-auth-three/package.json similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-external-auth-three/package.json rename to packages/tests/fixtures/peertube-plugin-test-external-auth-three/package.json diff --git a/server/tests/fixtures/peertube-plugin-test-external-auth-two/main.js b/packages/tests/fixtures/peertube-plugin-test-external-auth-two/main.js similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-external-auth-two/main.js rename to packages/tests/fixtures/peertube-plugin-test-external-auth-two/main.js diff --git a/server/tests/fixtures/peertube-plugin-test-external-auth-two/package.json b/packages/tests/fixtures/peertube-plugin-test-external-auth-two/package.json similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-external-auth-two/package.json rename to packages/tests/fixtures/peertube-plugin-test-external-auth-two/package.json diff --git a/server/tests/fixtures/peertube-plugin-test-filter-translations/languages/fr.json b/packages/tests/fixtures/peertube-plugin-test-filter-translations/languages/fr.json similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-filter-translations/languages/fr.json rename to packages/tests/fixtures/peertube-plugin-test-filter-translations/languages/fr.json diff --git a/server/tests/fixtures/peertube-plugin-test-filter-translations/languages/it.json b/packages/tests/fixtures/peertube-plugin-test-filter-translations/languages/it.json similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-filter-translations/languages/it.json rename to packages/tests/fixtures/peertube-plugin-test-filter-translations/languages/it.json diff --git a/server/tests/fixtures/peertube-plugin-test-filter-translations/main.js b/packages/tests/fixtures/peertube-plugin-test-filter-translations/main.js similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-filter-translations/main.js rename to packages/tests/fixtures/peertube-plugin-test-filter-translations/main.js diff --git a/server/tests/fixtures/peertube-plugin-test-filter-translations/package.json b/packages/tests/fixtures/peertube-plugin-test-filter-translations/package.json similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-filter-translations/package.json rename to packages/tests/fixtures/peertube-plugin-test-filter-translations/package.json diff --git a/server/tests/fixtures/peertube-plugin-test-five/main.js b/packages/tests/fixtures/peertube-plugin-test-five/main.js similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-five/main.js rename to packages/tests/fixtures/peertube-plugin-test-five/main.js diff --git a/server/tests/fixtures/peertube-plugin-test-five/package.json b/packages/tests/fixtures/peertube-plugin-test-five/package.json similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-five/package.json rename to packages/tests/fixtures/peertube-plugin-test-five/package.json diff --git a/server/tests/fixtures/peertube-plugin-test-four/main.js b/packages/tests/fixtures/peertube-plugin-test-four/main.js similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-four/main.js rename to packages/tests/fixtures/peertube-plugin-test-four/main.js diff --git a/server/tests/fixtures/peertube-plugin-test-four/package.json b/packages/tests/fixtures/peertube-plugin-test-four/package.json similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-four/package.json rename to packages/tests/fixtures/peertube-plugin-test-four/package.json diff --git a/server/tests/fixtures/peertube-plugin-test-id-pass-auth-one/main.js b/packages/tests/fixtures/peertube-plugin-test-id-pass-auth-one/main.js similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-id-pass-auth-one/main.js rename to packages/tests/fixtures/peertube-plugin-test-id-pass-auth-one/main.js diff --git a/server/tests/fixtures/peertube-plugin-test-id-pass-auth-one/package.json b/packages/tests/fixtures/peertube-plugin-test-id-pass-auth-one/package.json similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-id-pass-auth-one/package.json rename to packages/tests/fixtures/peertube-plugin-test-id-pass-auth-one/package.json diff --git a/server/tests/fixtures/peertube-plugin-test-id-pass-auth-three/main.js b/packages/tests/fixtures/peertube-plugin-test-id-pass-auth-three/main.js similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-id-pass-auth-three/main.js rename to packages/tests/fixtures/peertube-plugin-test-id-pass-auth-three/main.js diff --git a/server/tests/fixtures/peertube-plugin-test-id-pass-auth-three/package.json b/packages/tests/fixtures/peertube-plugin-test-id-pass-auth-three/package.json similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-id-pass-auth-three/package.json rename to packages/tests/fixtures/peertube-plugin-test-id-pass-auth-three/package.json diff --git a/server/tests/fixtures/peertube-plugin-test-id-pass-auth-two/main.js b/packages/tests/fixtures/peertube-plugin-test-id-pass-auth-two/main.js similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-id-pass-auth-two/main.js rename to packages/tests/fixtures/peertube-plugin-test-id-pass-auth-two/main.js diff --git a/server/tests/fixtures/peertube-plugin-test-id-pass-auth-two/package.json b/packages/tests/fixtures/peertube-plugin-test-id-pass-auth-two/package.json similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-id-pass-auth-two/package.json rename to packages/tests/fixtures/peertube-plugin-test-id-pass-auth-two/package.json diff --git a/server/tests/fixtures/peertube-plugin-test-native/main.js b/packages/tests/fixtures/peertube-plugin-test-native/main.js similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-native/main.js rename to packages/tests/fixtures/peertube-plugin-test-native/main.js diff --git a/server/tests/fixtures/peertube-plugin-test-native/package.json b/packages/tests/fixtures/peertube-plugin-test-native/package.json similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-native/package.json rename to packages/tests/fixtures/peertube-plugin-test-native/package.json diff --git a/server/tests/fixtures/peertube-plugin-test-podcast-custom-tags/main.js b/packages/tests/fixtures/peertube-plugin-test-podcast-custom-tags/main.js similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-podcast-custom-tags/main.js rename to packages/tests/fixtures/peertube-plugin-test-podcast-custom-tags/main.js diff --git a/server/tests/fixtures/peertube-plugin-test-podcast-custom-tags/package.json b/packages/tests/fixtures/peertube-plugin-test-podcast-custom-tags/package.json similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-podcast-custom-tags/package.json rename to packages/tests/fixtures/peertube-plugin-test-podcast-custom-tags/package.json diff --git a/server/tests/fixtures/peertube-plugin-test-six/main.js b/packages/tests/fixtures/peertube-plugin-test-six/main.js similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-six/main.js rename to packages/tests/fixtures/peertube-plugin-test-six/main.js diff --git a/server/tests/fixtures/peertube-plugin-test-six/package.json b/packages/tests/fixtures/peertube-plugin-test-six/package.json similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-six/package.json rename to packages/tests/fixtures/peertube-plugin-test-six/package.json diff --git a/server/tests/fixtures/peertube-plugin-test-transcoding-one/main.js b/packages/tests/fixtures/peertube-plugin-test-transcoding-one/main.js similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-transcoding-one/main.js rename to packages/tests/fixtures/peertube-plugin-test-transcoding-one/main.js diff --git a/server/tests/fixtures/peertube-plugin-test-transcoding-one/package.json b/packages/tests/fixtures/peertube-plugin-test-transcoding-one/package.json similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-transcoding-one/package.json rename to packages/tests/fixtures/peertube-plugin-test-transcoding-one/package.json diff --git a/server/tests/fixtures/peertube-plugin-test-transcoding-two/main.js b/packages/tests/fixtures/peertube-plugin-test-transcoding-two/main.js similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-transcoding-two/main.js rename to packages/tests/fixtures/peertube-plugin-test-transcoding-two/main.js diff --git a/server/tests/fixtures/peertube-plugin-test-transcoding-two/package.json b/packages/tests/fixtures/peertube-plugin-test-transcoding-two/package.json similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-transcoding-two/package.json rename to packages/tests/fixtures/peertube-plugin-test-transcoding-two/package.json diff --git a/server/tests/fixtures/peertube-plugin-test-unloading/lib.js b/packages/tests/fixtures/peertube-plugin-test-unloading/lib.js similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-unloading/lib.js rename to packages/tests/fixtures/peertube-plugin-test-unloading/lib.js diff --git a/server/tests/fixtures/peertube-plugin-test-unloading/main.js b/packages/tests/fixtures/peertube-plugin-test-unloading/main.js similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-unloading/main.js rename to packages/tests/fixtures/peertube-plugin-test-unloading/main.js diff --git a/server/tests/fixtures/peertube-plugin-test-unloading/package.json b/packages/tests/fixtures/peertube-plugin-test-unloading/package.json similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-unloading/package.json rename to packages/tests/fixtures/peertube-plugin-test-unloading/package.json diff --git a/server/tests/fixtures/peertube-plugin-test-video-constants/main.js b/packages/tests/fixtures/peertube-plugin-test-video-constants/main.js similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-video-constants/main.js rename to packages/tests/fixtures/peertube-plugin-test-video-constants/main.js diff --git a/server/tests/fixtures/peertube-plugin-test-video-constants/package.json b/packages/tests/fixtures/peertube-plugin-test-video-constants/package.json similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-video-constants/package.json rename to packages/tests/fixtures/peertube-plugin-test-video-constants/package.json diff --git a/server/tests/fixtures/peertube-plugin-test-websocket/main.js b/packages/tests/fixtures/peertube-plugin-test-websocket/main.js similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-websocket/main.js rename to packages/tests/fixtures/peertube-plugin-test-websocket/main.js diff --git a/server/tests/fixtures/peertube-plugin-test-websocket/package.json b/packages/tests/fixtures/peertube-plugin-test-websocket/package.json similarity index 100% rename from server/tests/fixtures/peertube-plugin-test-websocket/package.json rename to packages/tests/fixtures/peertube-plugin-test-websocket/package.json diff --git a/server/tests/fixtures/peertube-plugin-test/languages/fr.json b/packages/tests/fixtures/peertube-plugin-test/languages/fr.json similarity index 100% rename from server/tests/fixtures/peertube-plugin-test/languages/fr.json rename to packages/tests/fixtures/peertube-plugin-test/languages/fr.json diff --git a/server/tests/fixtures/peertube-plugin-test/main.js b/packages/tests/fixtures/peertube-plugin-test/main.js similarity index 100% rename from server/tests/fixtures/peertube-plugin-test/main.js rename to packages/tests/fixtures/peertube-plugin-test/main.js diff --git a/server/tests/fixtures/peertube-plugin-test/package.json b/packages/tests/fixtures/peertube-plugin-test/package.json similarity index 100% rename from server/tests/fixtures/peertube-plugin-test/package.json rename to packages/tests/fixtures/peertube-plugin-test/package.json diff --git a/server/tests/fixtures/rtmps.cert b/packages/tests/fixtures/rtmps.cert similarity index 100% rename from server/tests/fixtures/rtmps.cert rename to packages/tests/fixtures/rtmps.cert diff --git a/server/tests/fixtures/rtmps.key b/packages/tests/fixtures/rtmps.key similarity index 100% rename from server/tests/fixtures/rtmps.key rename to packages/tests/fixtures/rtmps.key diff --git a/server/tests/fixtures/sample.ogg b/packages/tests/fixtures/sample.ogg similarity index 100% rename from server/tests/fixtures/sample.ogg rename to packages/tests/fixtures/sample.ogg diff --git a/server/tests/fixtures/subtitle-bad.txt b/packages/tests/fixtures/subtitle-bad.txt similarity index 100% rename from server/tests/fixtures/subtitle-bad.txt rename to packages/tests/fixtures/subtitle-bad.txt diff --git a/server/tests/fixtures/subtitle-good.srt b/packages/tests/fixtures/subtitle-good.srt similarity index 100% rename from server/tests/fixtures/subtitle-good.srt rename to packages/tests/fixtures/subtitle-good.srt diff --git a/server/tests/fixtures/subtitle-good1.vtt b/packages/tests/fixtures/subtitle-good1.vtt similarity index 100% rename from server/tests/fixtures/subtitle-good1.vtt rename to packages/tests/fixtures/subtitle-good1.vtt diff --git a/server/tests/fixtures/subtitle-good2.vtt b/packages/tests/fixtures/subtitle-good2.vtt similarity index 100% rename from server/tests/fixtures/subtitle-good2.vtt rename to packages/tests/fixtures/subtitle-good2.vtt diff --git a/server/tests/fixtures/thumbnail-playlist.jpg b/packages/tests/fixtures/thumbnail-playlist.jpg similarity index 100% rename from server/tests/fixtures/thumbnail-playlist.jpg rename to packages/tests/fixtures/thumbnail-playlist.jpg diff --git a/server/tests/fixtures/video-720p.torrent b/packages/tests/fixtures/video-720p.torrent similarity index 100% rename from server/tests/fixtures/video-720p.torrent rename to packages/tests/fixtures/video-720p.torrent diff --git a/server/tests/fixtures/video_import_preview.jpg b/packages/tests/fixtures/video_import_preview.jpg similarity index 100% rename from server/tests/fixtures/video_import_preview.jpg rename to packages/tests/fixtures/video_import_preview.jpg diff --git a/server/tests/fixtures/video_import_preview_yt_dlp.jpg b/packages/tests/fixtures/video_import_preview_yt_dlp.jpg similarity index 100% rename from server/tests/fixtures/video_import_preview_yt_dlp.jpg rename to packages/tests/fixtures/video_import_preview_yt_dlp.jpg diff --git a/server/tests/fixtures/video_import_thumbnail.jpg b/packages/tests/fixtures/video_import_thumbnail.jpg similarity index 100% rename from server/tests/fixtures/video_import_thumbnail.jpg rename to packages/tests/fixtures/video_import_thumbnail.jpg diff --git a/server/tests/fixtures/video_import_thumbnail_yt_dlp.jpg b/packages/tests/fixtures/video_import_thumbnail_yt_dlp.jpg similarity index 100% rename from server/tests/fixtures/video_import_thumbnail_yt_dlp.jpg rename to packages/tests/fixtures/video_import_thumbnail_yt_dlp.jpg diff --git a/server/tests/fixtures/video_short.avi b/packages/tests/fixtures/video_short.avi similarity index 100% rename from server/tests/fixtures/video_short.avi rename to packages/tests/fixtures/video_short.avi diff --git a/server/tests/fixtures/video_short.mkv b/packages/tests/fixtures/video_short.mkv similarity index 100% rename from server/tests/fixtures/video_short.mkv rename to packages/tests/fixtures/video_short.mkv diff --git a/server/tests/fixtures/video_short.mp4 b/packages/tests/fixtures/video_short.mp4 similarity index 100% rename from server/tests/fixtures/video_short.mp4 rename to packages/tests/fixtures/video_short.mp4 diff --git a/server/tests/fixtures/video_short.mp4.jpg b/packages/tests/fixtures/video_short.mp4.jpg similarity index 100% rename from server/tests/fixtures/video_short.mp4.jpg rename to packages/tests/fixtures/video_short.mp4.jpg diff --git a/server/tests/fixtures/video_short.ogv b/packages/tests/fixtures/video_short.ogv similarity index 100% rename from server/tests/fixtures/video_short.ogv rename to packages/tests/fixtures/video_short.ogv diff --git a/server/tests/fixtures/video_short.ogv.jpg b/packages/tests/fixtures/video_short.ogv.jpg similarity index 100% rename from server/tests/fixtures/video_short.ogv.jpg rename to packages/tests/fixtures/video_short.ogv.jpg diff --git a/server/tests/fixtures/video_short.webm b/packages/tests/fixtures/video_short.webm similarity index 100% rename from server/tests/fixtures/video_short.webm rename to packages/tests/fixtures/video_short.webm diff --git a/server/tests/fixtures/video_short.webm.jpg b/packages/tests/fixtures/video_short.webm.jpg similarity index 100% rename from server/tests/fixtures/video_short.webm.jpg rename to packages/tests/fixtures/video_short.webm.jpg diff --git a/server/tests/fixtures/video_short1-preview.webm.jpg b/packages/tests/fixtures/video_short1-preview.webm.jpg similarity index 100% rename from server/tests/fixtures/video_short1-preview.webm.jpg rename to packages/tests/fixtures/video_short1-preview.webm.jpg diff --git a/server/tests/fixtures/video_short1.webm b/packages/tests/fixtures/video_short1.webm similarity index 100% rename from server/tests/fixtures/video_short1.webm rename to packages/tests/fixtures/video_short1.webm diff --git a/server/tests/fixtures/video_short1.webm.jpg b/packages/tests/fixtures/video_short1.webm.jpg similarity index 100% rename from server/tests/fixtures/video_short1.webm.jpg rename to packages/tests/fixtures/video_short1.webm.jpg diff --git a/server/tests/fixtures/video_short2.webm b/packages/tests/fixtures/video_short2.webm similarity index 100% rename from server/tests/fixtures/video_short2.webm rename to packages/tests/fixtures/video_short2.webm diff --git a/server/tests/fixtures/video_short2.webm.jpg b/packages/tests/fixtures/video_short2.webm.jpg similarity index 100% rename from server/tests/fixtures/video_short2.webm.jpg rename to packages/tests/fixtures/video_short2.webm.jpg diff --git a/server/tests/fixtures/video_short3.webm b/packages/tests/fixtures/video_short3.webm similarity index 100% rename from server/tests/fixtures/video_short3.webm rename to packages/tests/fixtures/video_short3.webm diff --git a/server/tests/fixtures/video_short3.webm.jpg b/packages/tests/fixtures/video_short3.webm.jpg similarity index 100% rename from server/tests/fixtures/video_short3.webm.jpg rename to packages/tests/fixtures/video_short3.webm.jpg diff --git a/server/tests/fixtures/video_short_0p.mp4 b/packages/tests/fixtures/video_short_0p.mp4 similarity index 100% rename from server/tests/fixtures/video_short_0p.mp4 rename to packages/tests/fixtures/video_short_0p.mp4 diff --git a/server/tests/fixtures/video_short_144p.m3u8 b/packages/tests/fixtures/video_short_144p.m3u8 similarity index 100% rename from server/tests/fixtures/video_short_144p.m3u8 rename to packages/tests/fixtures/video_short_144p.m3u8 diff --git a/server/tests/fixtures/video_short_144p.mp4 b/packages/tests/fixtures/video_short_144p.mp4 similarity index 100% rename from server/tests/fixtures/video_short_144p.mp4 rename to packages/tests/fixtures/video_short_144p.mp4 diff --git a/server/tests/fixtures/video_short_240p.m3u8 b/packages/tests/fixtures/video_short_240p.m3u8 similarity index 100% rename from server/tests/fixtures/video_short_240p.m3u8 rename to packages/tests/fixtures/video_short_240p.m3u8 diff --git a/server/tests/fixtures/video_short_240p.mp4 b/packages/tests/fixtures/video_short_240p.mp4 similarity index 100% rename from server/tests/fixtures/video_short_240p.mp4 rename to packages/tests/fixtures/video_short_240p.mp4 diff --git a/server/tests/fixtures/video_short_360p.m3u8 b/packages/tests/fixtures/video_short_360p.m3u8 similarity index 100% rename from server/tests/fixtures/video_short_360p.m3u8 rename to packages/tests/fixtures/video_short_360p.m3u8 diff --git a/server/tests/fixtures/video_short_360p.mp4 b/packages/tests/fixtures/video_short_360p.mp4 similarity index 100% rename from server/tests/fixtures/video_short_360p.mp4 rename to packages/tests/fixtures/video_short_360p.mp4 diff --git a/server/tests/fixtures/video_short_480.webm b/packages/tests/fixtures/video_short_480.webm similarity index 100% rename from server/tests/fixtures/video_short_480.webm rename to packages/tests/fixtures/video_short_480.webm diff --git a/server/tests/fixtures/video_short_480p.m3u8 b/packages/tests/fixtures/video_short_480p.m3u8 similarity index 100% rename from server/tests/fixtures/video_short_480p.m3u8 rename to packages/tests/fixtures/video_short_480p.m3u8 diff --git a/server/tests/fixtures/video_short_480p.mp4 b/packages/tests/fixtures/video_short_480p.mp4 similarity index 100% rename from server/tests/fixtures/video_short_480p.mp4 rename to packages/tests/fixtures/video_short_480p.mp4 diff --git a/server/tests/fixtures/video_short_4k.mp4 b/packages/tests/fixtures/video_short_4k.mp4 similarity index 100% rename from server/tests/fixtures/video_short_4k.mp4 rename to packages/tests/fixtures/video_short_4k.mp4 diff --git a/server/tests/fixtures/video_short_720p.m3u8 b/packages/tests/fixtures/video_short_720p.m3u8 similarity index 100% rename from server/tests/fixtures/video_short_720p.m3u8 rename to packages/tests/fixtures/video_short_720p.m3u8 diff --git a/server/tests/fixtures/video_short_720p.mp4 b/packages/tests/fixtures/video_short_720p.mp4 similarity index 100% rename from server/tests/fixtures/video_short_720p.mp4 rename to packages/tests/fixtures/video_short_720p.mp4 diff --git a/server/tests/fixtures/video_short_fake.webm b/packages/tests/fixtures/video_short_fake.webm similarity index 100% rename from server/tests/fixtures/video_short_fake.webm rename to packages/tests/fixtures/video_short_fake.webm diff --git a/server/tests/fixtures/video_short_mp3_256k.mp4 b/packages/tests/fixtures/video_short_mp3_256k.mp4 similarity index 100% rename from server/tests/fixtures/video_short_mp3_256k.mp4 rename to packages/tests/fixtures/video_short_mp3_256k.mp4 diff --git a/server/tests/fixtures/video_short_no_audio.mp4 b/packages/tests/fixtures/video_short_no_audio.mp4 similarity index 100% rename from server/tests/fixtures/video_short_no_audio.mp4 rename to packages/tests/fixtures/video_short_no_audio.mp4 diff --git a/server/tests/fixtures/video_very_long_10p.mp4 b/packages/tests/fixtures/video_very_long_10p.mp4 similarity index 100% rename from server/tests/fixtures/video_very_long_10p.mp4 rename to packages/tests/fixtures/video_very_long_10p.mp4 diff --git a/server/tests/fixtures/video_very_short_240p.mp4 b/packages/tests/fixtures/video_very_short_240p.mp4 similarity index 100% rename from server/tests/fixtures/video_very_short_240p.mp4 rename to packages/tests/fixtures/video_very_short_240p.mp4 diff --git a/packages/tests/package.json b/packages/tests/package.json new file mode 100644 index 000000000..02882ebc7 --- /dev/null +++ b/packages/tests/package.json @@ -0,0 +1,12 @@ +{ + "name": "@peertube/tests", + "private": true, + "version": "0.0.0", + "type": "module", + "devDependencies": {}, + "scripts": { + "build": "tsc", + "watch": "tsc -w" + }, + "dependencies": {} +} diff --git a/packages/tests/src/api/activitypub/cleaner.ts b/packages/tests/src/api/activitypub/cleaner.ts new file mode 100644 index 000000000..4476aea85 --- /dev/null +++ b/packages/tests/src/api/activitypub/cleaner.ts @@ -0,0 +1,342 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { SQLCommand } from '@tests/shared/sql-command.js' +import { wait } from '@peertube/peertube-core-utils' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test AP cleaner', function () { + let servers: PeerTubeServer[] = [] + const sqlCommands: SQLCommand[] = [] + + let videoUUID1: string + let videoUUID2: string + let videoUUID3: string + + let videoUUIDs: string[] + + before(async function () { + this.timeout(120000) + + const config = { + federation: { + videos: { cleanup_remote_interactions: true } + } + } + servers = await createMultipleServers(3, config) + + // Get the access tokens + await setAccessTokensToServers(servers) + + await Promise.all([ + doubleFollow(servers[0], servers[1]), + doubleFollow(servers[1], servers[2]), + doubleFollow(servers[0], servers[2]) + ]) + + // Update 1 local share, check 6 shares + + // Create 1 comment per video + // Update 1 remote URL and 1 local URL on + + videoUUID1 = (await servers[0].videos.quickUpload({ name: 'server 1' })).uuid + videoUUID2 = (await servers[1].videos.quickUpload({ name: 'server 2' })).uuid + videoUUID3 = (await servers[2].videos.quickUpload({ name: 'server 3' })).uuid + + videoUUIDs = [ videoUUID1, videoUUID2, videoUUID3 ] + + await waitJobs(servers) + + for (const server of servers) { + for (const uuid of videoUUIDs) { + await server.videos.rate({ id: uuid, rating: 'like' }) + await server.comments.createThread({ videoId: uuid, text: 'comment' }) + } + + sqlCommands.push(new SQLCommand(server)) + } + + await waitJobs(servers) + }) + + it('Should have the correct likes', async function () { + for (const server of servers) { + for (const uuid of videoUUIDs) { + const video = await server.videos.get({ id: uuid }) + + expect(video.likes).to.equal(3) + expect(video.dislikes).to.equal(0) + } + } + }) + + it('Should destroy server 3 internal likes and correctly clean them', async function () { + this.timeout(20000) + + await sqlCommands[2].deleteAll('accountVideoRate') + for (const uuid of videoUUIDs) { + await sqlCommands[2].setVideoField(uuid, 'likes', '0') + } + + await wait(5000) + await waitJobs(servers) + + // Updated rates of my video + { + const video = await servers[0].videos.get({ id: videoUUID1 }) + expect(video.likes).to.equal(2) + expect(video.dislikes).to.equal(0) + } + + // Did not update rates of a remote video + { + const video = await servers[0].videos.get({ id: videoUUID2 }) + expect(video.likes).to.equal(3) + expect(video.dislikes).to.equal(0) + } + }) + + it('Should update rates to dislikes', async function () { + this.timeout(20000) + + for (const server of servers) { + for (const uuid of videoUUIDs) { + await server.videos.rate({ id: uuid, rating: 'dislike' }) + } + } + + await waitJobs(servers) + + for (const server of servers) { + for (const uuid of videoUUIDs) { + const video = await server.videos.get({ id: uuid }) + expect(video.likes).to.equal(0) + expect(video.dislikes).to.equal(3) + } + } + }) + + it('Should destroy server 3 internal dislikes and correctly clean them', async function () { + this.timeout(20000) + + await sqlCommands[2].deleteAll('accountVideoRate') + + for (const uuid of videoUUIDs) { + await sqlCommands[2].setVideoField(uuid, 'dislikes', '0') + } + + await wait(5000) + await waitJobs(servers) + + // Updated rates of my video + { + const video = await servers[0].videos.get({ id: videoUUID1 }) + expect(video.likes).to.equal(0) + expect(video.dislikes).to.equal(2) + } + + // Did not update rates of a remote video + { + const video = await servers[0].videos.get({ id: videoUUID2 }) + expect(video.likes).to.equal(0) + expect(video.dislikes).to.equal(3) + } + }) + + it('Should destroy server 3 internal shares and correctly clean them', async function () { + this.timeout(20000) + + const preCount = await sqlCommands[0].getVideoShareCount() + expect(preCount).to.equal(6) + + await sqlCommands[2].deleteAll('videoShare') + await wait(5000) + await waitJobs(servers) + + // Still 6 because we don't have remote shares on local videos + const postCount = await sqlCommands[0].getVideoShareCount() + expect(postCount).to.equal(6) + }) + + it('Should destroy server 3 internal comments and correctly clean them', async function () { + this.timeout(20000) + + { + const { total } = await servers[0].comments.listThreads({ videoId: videoUUID1 }) + expect(total).to.equal(3) + } + + await sqlCommands[2].deleteAll('videoComment') + + await wait(5000) + await waitJobs(servers) + + { + const { total } = await servers[0].comments.listThreads({ videoId: videoUUID1 }) + expect(total).to.equal(2) + } + }) + + it('Should correctly update rate URLs', async function () { + this.timeout(30000) + + async function check (like: string, ofServerUrl: string, urlSuffix: string, remote: 'true' | 'false') { + const query = `SELECT "videoId", "accountVideoRate".url FROM "accountVideoRate" ` + + `INNER JOIN video ON "accountVideoRate"."videoId" = video.id AND remote IS ${remote} WHERE "accountVideoRate"."url" LIKE '${like}'` + const res = await sqlCommands[0].selectQuery<{ url: string }>(query) + + for (const rate of res) { + const matcher = new RegExp(`^${ofServerUrl}/accounts/root/dislikes/\\d+${urlSuffix}$`) + expect(rate.url).to.match(matcher) + } + } + + async function checkLocal () { + const startsWith = 'http://' + servers[0].host + '%' + // On local videos + await check(startsWith, servers[0].url, '', 'false') + // On remote videos + await check(startsWith, servers[0].url, '', 'true') + } + + async function checkRemote (suffix: string) { + const startsWith = 'http://' + servers[1].host + '%' + // On local videos + await check(startsWith, servers[1].url, suffix, 'false') + // On remote videos, we should not update URLs so no suffix + await check(startsWith, servers[1].url, '', 'true') + } + + await checkLocal() + await checkRemote('') + + { + const query = `UPDATE "accountVideoRate" SET url = url || 'stan'` + await sqlCommands[1].updateQuery(query) + + await wait(5000) + await waitJobs(servers) + } + + await checkLocal() + await checkRemote('stan') + }) + + it('Should correctly update comment URLs', async function () { + this.timeout(30000) + + async function check (like: string, ofServerUrl: string, urlSuffix: string, remote: 'true' | 'false') { + const query = `SELECT "videoId", "videoComment".url, uuid as "videoUUID" FROM "videoComment" ` + + `INNER JOIN video ON "videoComment"."videoId" = video.id AND remote IS ${remote} WHERE "videoComment"."url" LIKE '${like}'` + + const res = await sqlCommands[0].selectQuery<{ url: string, videoUUID: string }>(query) + + for (const comment of res) { + const matcher = new RegExp(`${ofServerUrl}/videos/watch/${comment.videoUUID}/comments/\\d+${urlSuffix}`) + expect(comment.url).to.match(matcher) + } + } + + async function checkLocal () { + const startsWith = 'http://' + servers[0].host + '%' + // On local videos + await check(startsWith, servers[0].url, '', 'false') + // On remote videos + await check(startsWith, servers[0].url, '', 'true') + } + + async function checkRemote (suffix: string) { + const startsWith = 'http://' + servers[1].host + '%' + // On local videos + await check(startsWith, servers[1].url, suffix, 'false') + // On remote videos, we should not update URLs so no suffix + await check(startsWith, servers[1].url, '', 'true') + } + + { + const query = `UPDATE "videoComment" SET url = url || 'kyle'` + await sqlCommands[1].updateQuery(query) + + await wait(5000) + await waitJobs(servers) + } + + await checkLocal() + await checkRemote('kyle') + }) + + it('Should remove unavailable remote resources', async function () { + this.timeout(240000) + + async function expectNotDeleted () { + { + const video = await servers[0].videos.get({ id: uuid }) + + expect(video.likes).to.equal(3) + expect(video.dislikes).to.equal(0) + } + + { + const { total } = await servers[0].comments.listThreads({ videoId: uuid }) + expect(total).to.equal(3) + } + } + + async function expectDeleted () { + { + const video = await servers[0].videos.get({ id: uuid }) + + expect(video.likes).to.equal(2) + expect(video.dislikes).to.equal(0) + } + + { + const { total } = await servers[0].comments.listThreads({ videoId: uuid }) + expect(total).to.equal(2) + } + } + + const uuid = (await servers[0].videos.quickUpload({ name: 'server 1 video 2' })).uuid + + await waitJobs(servers) + + for (const server of servers) { + await server.videos.rate({ id: uuid, rating: 'like' }) + await server.comments.createThread({ videoId: uuid, text: 'comment' }) + } + + await waitJobs(servers) + + await expectNotDeleted() + + await servers[1].kill() + + await wait(5000) + await expectNotDeleted() + + let continueWhile = true + + do { + try { + await expectDeleted() + continueWhile = false + } catch { + } + } while (continueWhile) + }) + + after(async function () { + for (const sql of sqlCommands) { + await sql.cleanup() + } + + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/activitypub/client.ts b/packages/tests/src/api/activitypub/client.ts new file mode 100644 index 000000000..fb9575d31 --- /dev/null +++ b/packages/tests/src/api/activitypub/client.ts @@ -0,0 +1,136 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { processViewersStats } from '@tests/shared/views.js' +import { HttpStatusCode, VideoPlaylistPrivacy, WatchActionObject } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + makeActivityPubGetRequest, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel +} from '@peertube/peertube-server-commands' + +describe('Test activitypub', function () { + let servers: PeerTubeServer[] = [] + let video: { id: number, uuid: string, shortUUID: string } + let playlist: { id: number, uuid: string, shortUUID: string } + + async function testAccount (path: string) { + const res = await makeActivityPubGetRequest(servers[0].url, path) + const object = res.body + + expect(object.type).to.equal('Person') + expect(object.id).to.equal(servers[0].url + '/accounts/root') + expect(object.name).to.equal('root') + expect(object.preferredUsername).to.equal('root') + } + + async function testChannel (path: string) { + const res = await makeActivityPubGetRequest(servers[0].url, path) + const object = res.body + + expect(object.type).to.equal('Group') + expect(object.id).to.equal(servers[0].url + '/video-channels/root_channel') + expect(object.name).to.equal('Main root channel') + expect(object.preferredUsername).to.equal('root_channel') + } + + async function testVideo (path: string) { + const res = await makeActivityPubGetRequest(servers[0].url, path) + const object = res.body + + expect(object.type).to.equal('Video') + expect(object.id).to.equal(servers[0].url + '/videos/watch/' + video.uuid) + expect(object.name).to.equal('video') + } + + async function testPlaylist (path: string) { + const res = await makeActivityPubGetRequest(servers[0].url, path) + const object = res.body + + expect(object.type).to.equal('Playlist') + expect(object.id).to.equal(servers[0].url + '/video-playlists/' + playlist.uuid) + expect(object.name).to.equal('playlist') + } + + before(async function () { + this.timeout(30000) + + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + { + video = await servers[0].videos.quickUpload({ name: 'video' }) + } + + { + const attributes = { displayName: 'playlist', privacy: VideoPlaylistPrivacy.PUBLIC, videoChannelId: servers[0].store.channel.id } + playlist = await servers[0].playlists.create({ attributes }) + } + + await doubleFollow(servers[0], servers[1]) + }) + + it('Should return the account object', async function () { + await testAccount('/accounts/root') + await testAccount('/a/root') + }) + + it('Should return the channel object', async function () { + await testChannel('/video-channels/root_channel') + await testChannel('/c/root_channel') + }) + + it('Should return the video object', async function () { + await testVideo('/videos/watch/' + video.id) + await testVideo('/videos/watch/' + video.uuid) + await testVideo('/videos/watch/' + video.shortUUID) + await testVideo('/w/' + video.id) + await testVideo('/w/' + video.uuid) + await testVideo('/w/' + video.shortUUID) + }) + + it('Should return the playlist object', async function () { + await testPlaylist('/video-playlists/' + playlist.id) + await testPlaylist('/video-playlists/' + playlist.uuid) + await testPlaylist('/video-playlists/' + playlist.shortUUID) + await testPlaylist('/w/p/' + playlist.id) + await testPlaylist('/w/p/' + playlist.uuid) + await testPlaylist('/w/p/' + playlist.shortUUID) + await testPlaylist('/videos/watch/playlist/' + playlist.id) + await testPlaylist('/videos/watch/playlist/' + playlist.uuid) + await testPlaylist('/videos/watch/playlist/' + playlist.shortUUID) + }) + + it('Should redirect to the origin video object', async function () { + const res = await makeActivityPubGetRequest(servers[1].url, '/videos/watch/' + video.uuid, HttpStatusCode.FOUND_302) + + expect(res.header.location).to.equal(servers[0].url + '/videos/watch/' + video.uuid) + }) + + it('Should return the watch action', async function () { + this.timeout(50000) + + await servers[0].views.simulateViewer({ id: video.uuid, currentTimes: [ 0, 2 ] }) + await processViewersStats(servers) + + const res = await makeActivityPubGetRequest(servers[0].url, '/videos/local-viewer/1', HttpStatusCode.OK_200) + + const object: WatchActionObject = res.body + expect(object.type).to.equal('WatchAction') + expect(object.duration).to.equal('PT2S') + expect(object.actionStatus).to.equal('CompletedActionStatus') + expect(object.watchSections).to.have.lengthOf(1) + expect(object.watchSections[0].startTimestamp).to.equal(0) + expect(object.watchSections[0].endTimestamp).to.equal(2) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/activitypub/fetch.ts b/packages/tests/src/api/activitypub/fetch.ts new file mode 100644 index 000000000..c7f5288cc --- /dev/null +++ b/packages/tests/src/api/activitypub/fetch.ts @@ -0,0 +1,82 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { SQLCommand } from '@tests/shared/sql-command.js' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test ActivityPub fetcher', function () { + let servers: PeerTubeServer[] + let sqlCommandServer1: SQLCommand + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(60000) + + servers = await createMultipleServers(3) + + // Get the access tokens + await setAccessTokensToServers(servers) + + const user = { username: 'user1', password: 'password' } + for (const server of servers) { + await server.users.create({ username: user.username, password: user.password }) + } + + const userAccessToken = await servers[0].login.getAccessToken(user) + + await servers[0].videos.upload({ attributes: { name: 'video root' } }) + const { uuid } = await servers[0].videos.upload({ attributes: { name: 'bad video root' } }) + await servers[0].videos.upload({ token: userAccessToken, attributes: { name: 'video user' } }) + + sqlCommandServer1 = new SQLCommand(servers[0]) + + { + const to = servers[0].url + '/accounts/user1' + const value = servers[1].url + '/accounts/user1' + await sqlCommandServer1.setActorField(to, 'url', value) + } + + { + const value = servers[2].url + '/videos/watch/' + uuid + await sqlCommandServer1.setVideoField(uuid, 'url', value) + } + }) + + it('Should add only the video with a valid actor URL', async function () { + this.timeout(60000) + + await doubleFollow(servers[0], servers[1]) + await waitJobs(servers) + + { + const { total, data } = await servers[0].videos.list({ sort: 'createdAt' }) + + expect(total).to.equal(3) + expect(data[0].name).to.equal('video root') + expect(data[1].name).to.equal('bad video root') + expect(data[2].name).to.equal('video user') + } + + { + const { total, data } = await servers[1].videos.list({ sort: 'createdAt' }) + + expect(total).to.equal(1) + expect(data[0].name).to.equal('video root') + } + }) + + after(async function () { + this.timeout(20000) + + await sqlCommandServer1.cleanup() + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/activitypub/index.ts b/packages/tests/src/api/activitypub/index.ts new file mode 100644 index 000000000..ef4f1aafb --- /dev/null +++ b/packages/tests/src/api/activitypub/index.ts @@ -0,0 +1,5 @@ +import './cleaner.js' +import './client.js' +import './fetch.js' +import './refresher.js' +import './security.js' diff --git a/packages/tests/src/api/activitypub/refresher.ts b/packages/tests/src/api/activitypub/refresher.ts new file mode 100644 index 000000000..90aa1a5ad --- /dev/null +++ b/packages/tests/src/api/activitypub/refresher.ts @@ -0,0 +1,157 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { SQLCommand } from '@tests/shared/sql-command.js' +import { wait } from '@peertube/peertube-core-utils' +import { HttpStatusCode, VideoPlaylistPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + killallServers, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test AP refresher', function () { + let servers: PeerTubeServer[] = [] + let sqlCommandServer2: SQLCommand + let videoUUID1: string + let videoUUID2: string + let videoUUID3: string + let playlistUUID1: string + let playlistUUID2: string + + before(async function () { + this.timeout(240000) + + servers = await createMultipleServers(2) + + // Get the access tokens + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + for (const server of servers) { + await server.config.disableTranscoding() + } + + { + videoUUID1 = (await servers[1].videos.quickUpload({ name: 'video1' })).uuid + videoUUID2 = (await servers[1].videos.quickUpload({ name: 'video2' })).uuid + videoUUID3 = (await servers[1].videos.quickUpload({ name: 'video3' })).uuid + } + + { + const token1 = await servers[1].users.generateUserAndToken('user1') + await servers[1].videos.upload({ token: token1, attributes: { name: 'video4' } }) + + const token2 = await servers[1].users.generateUserAndToken('user2') + await servers[1].videos.upload({ token: token2, attributes: { name: 'video5' } }) + } + + { + const attributes = { displayName: 'playlist1', privacy: VideoPlaylistPrivacy.PUBLIC, videoChannelId: servers[1].store.channel.id } + const created = await servers[1].playlists.create({ attributes }) + playlistUUID1 = created.uuid + } + + { + const attributes = { displayName: 'playlist2', privacy: VideoPlaylistPrivacy.PUBLIC, videoChannelId: servers[1].store.channel.id } + const created = await servers[1].playlists.create({ attributes }) + playlistUUID2 = created.uuid + } + + await doubleFollow(servers[0], servers[1]) + + sqlCommandServer2 = new SQLCommand(servers[1]) + }) + + describe('Videos refresher', function () { + + it('Should remove a deleted remote video', async function () { + this.timeout(60000) + + await wait(10000) + + // Change UUID so the remote server returns a 404 + await sqlCommandServer2.setVideoField(videoUUID1, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174f') + + await servers[0].videos.get({ id: videoUUID1 }) + await servers[0].videos.get({ id: videoUUID2 }) + + await waitJobs(servers) + + await servers[0].videos.get({ id: videoUUID1, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + await servers[0].videos.get({ id: videoUUID2 }) + }) + + it('Should not update a remote video if the remote instance is down', async function () { + this.timeout(70000) + + await killallServers([ servers[1] ]) + + await sqlCommandServer2.setVideoField(videoUUID3, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174e') + + // Video will need a refresh + await wait(10000) + + await servers[0].videos.get({ id: videoUUID3 }) + // The refresh should fail + await waitJobs([ servers[0] ]) + + await servers[1].run() + + await servers[0].videos.get({ id: videoUUID3 }) + }) + }) + + describe('Actors refresher', function () { + + it('Should remove a deleted actor', async function () { + this.timeout(60000) + + const command = servers[0].accounts + + await wait(10000) + + // Change actor name so the remote server returns a 404 + const to = servers[1].url + '/accounts/user2' + await sqlCommandServer2.setActorField(to, 'preferredUsername', 'toto') + + await command.get({ accountName: 'user1@' + servers[1].host }) + await command.get({ accountName: 'user2@' + servers[1].host }) + + await waitJobs(servers) + + await command.get({ accountName: 'user1@' + servers[1].host, expectedStatus: HttpStatusCode.OK_200 }) + await command.get({ accountName: 'user2@' + servers[1].host, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + }) + + describe('Playlist refresher', function () { + + it('Should remove a deleted playlist', async function () { + this.timeout(60000) + + await wait(10000) + + // Change UUID so the remote server returns a 404 + await sqlCommandServer2.setPlaylistField(playlistUUID2, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b178e') + + await servers[0].playlists.get({ playlistId: playlistUUID1 }) + await servers[0].playlists.get({ playlistId: playlistUUID2 }) + + await waitJobs(servers) + + await servers[0].playlists.get({ playlistId: playlistUUID1, expectedStatus: HttpStatusCode.OK_200 }) + await servers[0].playlists.get({ playlistId: playlistUUID2, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + }) + + after(async function () { + await sqlCommandServer2.cleanup() + + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/activitypub/security.ts b/packages/tests/src/api/activitypub/security.ts new file mode 100644 index 000000000..d9649de50 --- /dev/null +++ b/packages/tests/src/api/activitypub/security.ts @@ -0,0 +1,331 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { wait } from '@peertube/peertube-core-utils' +import { HttpStatusCode } from '@peertube/peertube-models' +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import { PeerTubeServer, cleanupTests, createMultipleServers, killallServers } from '@peertube/peertube-server-commands' +import { + activityPubContextify, + buildGlobalHTTPHeaders, + signAndContextify +} from '@peertube/peertube-server/server/helpers/activity-pub-utils.js' +import { buildDigest } from '@peertube/peertube-server/server/helpers/peertube-crypto.js' +import { ACTIVITY_PUB, HTTP_SIGNATURE } from '@peertube/peertube-server/server/initializers/constants.js' +import { makePOSTAPRequest } from '@tests/shared/requests.js' +import { SQLCommand } from '@tests/shared/sql-command.js' +import { expect } from 'chai' +import { readJsonSync } from 'fs-extra/esm' + +function fakeFilter () { + return (data: any) => Promise.resolve(data) +} + +function setKeysOfServer (onServer: SQLCommand, ofServerUrl: string, publicKey: string, privateKey: string) { + const url = ofServerUrl + '/accounts/peertube' + + return Promise.all([ + onServer.setActorField(url, 'publicKey', publicKey), + onServer.setActorField(url, 'privateKey', privateKey) + ]) +} + +function setUpdatedAtOfServer (onServer: SQLCommand, ofServerUrl: string, updatedAt: string) { + const url = ofServerUrl + '/accounts/peertube' + + return Promise.all([ + onServer.setActorField(url, 'createdAt', updatedAt), + onServer.setActorField(url, 'updatedAt', updatedAt) + ]) +} + +function getAnnounceWithoutContext (server: PeerTubeServer) { + const json = readJsonSync(buildAbsoluteFixturePath('./ap-json/peertube/announce-without-context.json')) + const result: typeof json = {} + + for (const key of Object.keys(json)) { + if (Array.isArray(json[key])) { + result[key] = json[key].map(v => v.replace(':9002', `:${server.port}`)) + } else { + result[key] = json[key].replace(':9002', `:${server.port}`) + } + } + + return result +} + +async function makeFollowRequest (to: { url: string }, by: { url: string, privateKey }) { + const follow = { + type: 'Follow', + id: by.url + '/' + new Date().getTime(), + actor: by.url, + object: to.url + } + + const body = await activityPubContextify(follow, 'Follow', fakeFilter()) + + const httpSignature = { + algorithm: HTTP_SIGNATURE.ALGORITHM, + authorizationHeaderName: HTTP_SIGNATURE.HEADER_NAME, + keyId: by.url, + key: by.privateKey, + headers: HTTP_SIGNATURE.HEADERS_TO_SIGN_WITH_PAYLOAD + } + const headers = { + 'digest': buildDigest(body), + 'content-type': 'application/activity+json', + 'accept': ACTIVITY_PUB.ACCEPT_HEADER + } + + return makePOSTAPRequest(to.url + '/inbox', body, httpSignature, headers) +} + +describe('Test ActivityPub security', function () { + let servers: PeerTubeServer[] + let sqlCommands: SQLCommand[] = [] + + let url: string + + const keys = readJsonSync(buildAbsoluteFixturePath('./ap-json/peertube/keys.json')) + const invalidKeys = readJsonSync(buildAbsoluteFixturePath('./ap-json/peertube/invalid-keys.json')) + const baseHttpSignature = () => ({ + algorithm: HTTP_SIGNATURE.ALGORITHM, + authorizationHeaderName: HTTP_SIGNATURE.HEADER_NAME, + keyId: 'acct:peertube@' + servers[1].host, + key: keys.privateKey, + headers: HTTP_SIGNATURE.HEADERS_TO_SIGN_WITH_PAYLOAD + }) + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(60000) + + servers = await createMultipleServers(3) + + sqlCommands = servers.map(s => new SQLCommand(s)) + + url = servers[0].url + '/inbox' + + await setKeysOfServer(sqlCommands[0], servers[1].url, keys.publicKey, null) + await setKeysOfServer(sqlCommands[1], servers[1].url, keys.publicKey, keys.privateKey) + + const to = { url: servers[0].url + '/accounts/peertube' } + const by = { url: servers[1].url + '/accounts/peertube', privateKey: keys.privateKey } + await makeFollowRequest(to, by) + }) + + describe('When checking HTTP signature', function () { + + it('Should fail with an invalid digest', async function () { + const body = await activityPubContextify(getAnnounceWithoutContext(servers[1]), 'Announce', fakeFilter()) + const headers = { + Digest: buildDigest({ hello: 'coucou' }) + } + + try { + await makePOSTAPRequest(url, body, baseHttpSignature(), headers) + expect(true, 'Did not throw').to.be.false + } catch (err) { + expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) + } + }) + + it('Should fail with an invalid date', async function () { + const body = await activityPubContextify(getAnnounceWithoutContext(servers[1]), 'Announce', fakeFilter()) + const headers = buildGlobalHTTPHeaders(body) + headers['date'] = 'Wed, 21 Oct 2015 07:28:00 GMT' + + try { + await makePOSTAPRequest(url, body, baseHttpSignature(), headers) + expect(true, 'Did not throw').to.be.false + } catch (err) { + expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) + } + }) + + it('Should fail with bad keys', async function () { + await setKeysOfServer(sqlCommands[0], servers[1].url, invalidKeys.publicKey, invalidKeys.privateKey) + await setKeysOfServer(sqlCommands[1], servers[1].url, invalidKeys.publicKey, invalidKeys.privateKey) + + const body = await activityPubContextify(getAnnounceWithoutContext(servers[1]), 'Announce', fakeFilter()) + const headers = buildGlobalHTTPHeaders(body) + + try { + await makePOSTAPRequest(url, body, baseHttpSignature(), headers) + expect(true, 'Did not throw').to.be.false + } catch (err) { + expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) + } + }) + + it('Should reject requests without appropriate signed headers', async function () { + await setKeysOfServer(sqlCommands[0], servers[1].url, keys.publicKey, keys.privateKey) + await setKeysOfServer(sqlCommands[1], servers[1].url, keys.publicKey, keys.privateKey) + + const body = await activityPubContextify(getAnnounceWithoutContext(servers[1]), 'Announce', fakeFilter()) + const headers = buildGlobalHTTPHeaders(body) + + const signatureOptions = baseHttpSignature() + const badHeadersMatrix = [ + [ '(request-target)', 'date', 'digest' ], + [ 'host', 'date', 'digest' ], + [ '(request-target)', 'host', 'digest' ] + ] + + for (const badHeaders of badHeadersMatrix) { + signatureOptions.headers = badHeaders + + try { + await makePOSTAPRequest(url, body, signatureOptions, headers) + expect(true, 'Did not throw').to.be.false + } catch (err) { + expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) + } + } + }) + + it('Should succeed with a valid HTTP signature draft 11 (without date but with (created))', async function () { + const body = await activityPubContextify(getAnnounceWithoutContext(servers[1]), 'Announce', fakeFilter()) + const headers = buildGlobalHTTPHeaders(body) + + const signatureOptions = baseHttpSignature() + signatureOptions.headers = [ '(request-target)', '(created)', 'host', 'digest' ] + + const { statusCode } = await makePOSTAPRequest(url, body, signatureOptions, headers) + expect(statusCode).to.equal(HttpStatusCode.NO_CONTENT_204) + }) + + it('Should succeed with a valid HTTP signature', async function () { + const body = await activityPubContextify(getAnnounceWithoutContext(servers[1]), 'Announce', fakeFilter()) + const headers = buildGlobalHTTPHeaders(body) + + const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers) + expect(statusCode).to.equal(HttpStatusCode.NO_CONTENT_204) + }) + + it('Should refresh the actor keys', async function () { + this.timeout(20000) + + // Update keys of server 2 to invalid keys + // Server 1 should refresh the actor and fail + await setKeysOfServer(sqlCommands[1], servers[1].url, invalidKeys.publicKey, invalidKeys.privateKey) + await setUpdatedAtOfServer(sqlCommands[0], servers[1].url, '2015-07-17 22:00:00+00') + + // Invalid peertube actor cache + await killallServers([ servers[1] ]) + await servers[1].run() + + const body = await activityPubContextify(getAnnounceWithoutContext(servers[1]), 'Announce', fakeFilter()) + const headers = buildGlobalHTTPHeaders(body) + + try { + await makePOSTAPRequest(url, body, baseHttpSignature(), headers) + expect(true, 'Did not throw').to.be.false + } catch (err) { + console.error(err) + expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) + } + }) + }) + + describe('When checking Linked Data Signature', function () { + before(async function () { + await setKeysOfServer(sqlCommands[0], servers[1].url, keys.publicKey, keys.privateKey) + await setKeysOfServer(sqlCommands[1], servers[1].url, keys.publicKey, keys.privateKey) + await setKeysOfServer(sqlCommands[2], servers[2].url, keys.publicKey, keys.privateKey) + + const to = { url: servers[0].url + '/accounts/peertube' } + const by = { url: servers[2].url + '/accounts/peertube', privateKey: keys.privateKey } + await makeFollowRequest(to, by) + }) + + it('Should fail with bad keys', async function () { + await setKeysOfServer(sqlCommands[0], servers[2].url, invalidKeys.publicKey, invalidKeys.privateKey) + await setKeysOfServer(sqlCommands[2], servers[2].url, invalidKeys.publicKey, invalidKeys.privateKey) + + const body = getAnnounceWithoutContext(servers[1]) + body.actor = servers[2].url + '/accounts/peertube' + + const signer: any = { privateKey: invalidKeys.privateKey, url: servers[2].url + '/accounts/peertube' } + const signedBody = await signAndContextify(signer, body, 'Announce', fakeFilter()) + + const headers = buildGlobalHTTPHeaders(signedBody) + + try { + await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) + expect(true, 'Did not throw').to.be.false + } catch (err) { + expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) + } + }) + + it('Should fail with an altered body', async function () { + await setKeysOfServer(sqlCommands[0], servers[2].url, keys.publicKey, keys.privateKey) + await setKeysOfServer(sqlCommands[0], servers[2].url, keys.publicKey, keys.privateKey) + + const body = getAnnounceWithoutContext(servers[1]) + body.actor = servers[2].url + '/accounts/peertube' + + const signer: any = { privateKey: keys.privateKey, url: servers[2].url + '/accounts/peertube' } + const signedBody = await signAndContextify(signer, body, 'Announce', fakeFilter()) + + signedBody.actor = servers[2].url + '/account/peertube' + + const headers = buildGlobalHTTPHeaders(signedBody) + + try { + await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) + expect(true, 'Did not throw').to.be.false + } catch (err) { + expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) + } + }) + + it('Should succeed with a valid signature', async function () { + const body = getAnnounceWithoutContext(servers[1]) + body.actor = servers[2].url + '/accounts/peertube' + + const signer: any = { privateKey: keys.privateKey, url: servers[2].url + '/accounts/peertube' } + const signedBody = await signAndContextify(signer, body, 'Announce', fakeFilter()) + + const headers = buildGlobalHTTPHeaders(signedBody) + + const { statusCode } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) + expect(statusCode).to.equal(HttpStatusCode.NO_CONTENT_204) + }) + + it('Should refresh the actor keys', async function () { + this.timeout(20000) + + // Wait refresh invalidation + await wait(10000) + + // Update keys of server 3 to invalid keys + // Server 1 should refresh the actor and fail + await setKeysOfServer(sqlCommands[2], servers[2].url, invalidKeys.publicKey, invalidKeys.privateKey) + + const body = getAnnounceWithoutContext(servers[1]) + body.actor = servers[2].url + '/accounts/peertube' + + const signer: any = { privateKey: keys.privateKey, url: servers[2].url + '/accounts/peertube' } + const signedBody = await signAndContextify(signer, body, 'Announce', fakeFilter()) + + const headers = buildGlobalHTTPHeaders(signedBody) + + try { + await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) + expect(true, 'Did not throw').to.be.false + } catch (err) { + expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) + } + }) + }) + + after(async function () { + for (const sql of sqlCommands) { + await sql.cleanup() + } + + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/check-params/abuses.ts b/packages/tests/src/api/check-params/abuses.ts new file mode 100644 index 000000000..1effc82b1 --- /dev/null +++ b/packages/tests/src/api/check-params/abuses.ts @@ -0,0 +1,438 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@tests/shared/checks.js' +import { AbuseCreate, AbuseState, HttpStatusCode } from '@peertube/peertube-models' +import { + AbusesCommand, + cleanupTests, + createSingleServer, + doubleFollow, + makeGetRequest, + makePostBodyRequest, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test abuses API validators', function () { + const basePath = '/api/v1/abuses/' + + let server: PeerTubeServer + + let userToken = '' + let userToken2 = '' + let abuseId: number + let messageId: number + + let command: AbusesCommand + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + + userToken = await server.users.generateUserAndToken('user_1') + userToken2 = await server.users.generateUserAndToken('user_2') + + server.store.videoCreated = await server.videos.upload() + + command = server.abuses + }) + + describe('When listing abuses for admins', function () { + const path = basePath + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a non authenticated user', async function () { + await makeGetRequest({ + url: server.url, + path, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a non admin user', async function () { + await makeGetRequest({ + url: server.url, + path, + token: userToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with a bad id filter', async function () { + await makeGetRequest({ url: server.url, path, token: server.accessToken, query: { id: 'toto' } }) + }) + + it('Should fail with a bad filter', async function () { + await makeGetRequest({ url: server.url, path, token: server.accessToken, query: { filter: 'toto' } }) + await makeGetRequest({ url: server.url, path, token: server.accessToken, query: { filter: 'videos' } }) + }) + + it('Should fail with bad predefined reason', async function () { + await makeGetRequest({ url: server.url, path, token: server.accessToken, query: { predefinedReason: 'violentOrRepulsives' } }) + }) + + it('Should fail with a bad state filter', async function () { + await makeGetRequest({ url: server.url, path, token: server.accessToken, query: { state: 'toto' } }) + await makeGetRequest({ url: server.url, path, token: server.accessToken, query: { state: 0 } }) + }) + + it('Should fail with a bad videoIs filter', async function () { + await makeGetRequest({ url: server.url, path, token: server.accessToken, query: { videoIs: 'toto' } }) + }) + + it('Should succeed with the correct params', async function () { + const query = { + id: 13, + predefinedReason: 'violentOrRepulsive', + filter: 'comment', + state: 2, + videoIs: 'deleted' + } + + await makeGetRequest({ url: server.url, path, token: server.accessToken, query, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('When listing abuses for users', function () { + const path = '/api/v1/users/me/abuses' + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, userToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, userToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, userToken) + }) + + it('Should fail with a non authenticated user', async function () { + await makeGetRequest({ + url: server.url, + path, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a bad id filter', async function () { + await makeGetRequest({ url: server.url, path, token: userToken, query: { id: 'toto' } }) + }) + + it('Should fail with a bad state filter', async function () { + await makeGetRequest({ url: server.url, path, token: userToken, query: { state: 'toto' } }) + await makeGetRequest({ url: server.url, path, token: userToken, query: { state: 0 } }) + }) + + it('Should succeed with the correct params', async function () { + const query = { + id: 13, + state: 2 + } + + await makeGetRequest({ url: server.url, path, token: userToken, query, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('When reporting an abuse', function () { + const path = basePath + + it('Should fail with nothing', async function () { + const fields = {} + await makePostBodyRequest({ url: server.url, path, token: userToken, fields }) + }) + + it('Should fail with a wrong video', async function () { + const fields = { video: { id: 'blabla' }, reason: 'my super reason' } + await makePostBodyRequest({ url: server.url, path, token: userToken, fields }) + }) + + it('Should fail with an unknown video', async function () { + const fields = { video: { id: 42 }, reason: 'my super reason' } + await makePostBodyRequest({ + url: server.url, + path, + token: userToken, + fields, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail with a wrong comment', async function () { + const fields = { comment: { id: 'blabla' }, reason: 'my super reason' } + await makePostBodyRequest({ url: server.url, path, token: userToken, fields }) + }) + + it('Should fail with an unknown comment', async function () { + const fields = { comment: { id: 42 }, reason: 'my super reason' } + await makePostBodyRequest({ + url: server.url, + path, + token: userToken, + fields, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail with a wrong account', async function () { + const fields = { account: { id: 'blabla' }, reason: 'my super reason' } + await makePostBodyRequest({ url: server.url, path, token: userToken, fields }) + }) + + it('Should fail with an unknown account', async function () { + const fields = { account: { id: 42 }, reason: 'my super reason' } + await makePostBodyRequest({ + url: server.url, + path, + token: userToken, + fields, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail with not account, comment or video', async function () { + const fields = { reason: 'my super reason' } + await makePostBodyRequest({ + url: server.url, + path, + token: userToken, + fields, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with a non authenticated user', async function () { + const fields = { video: { id: server.store.videoCreated.id }, reason: 'my super reason' } + + await makePostBodyRequest({ url: server.url, path, token: 'hello', fields, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with a reason too short', async function () { + const fields = { video: { id: server.store.videoCreated.id }, reason: 'h' } + + await makePostBodyRequest({ url: server.url, path, token: userToken, fields }) + }) + + it('Should fail with a too big reason', async function () { + const fields = { video: { id: server.store.videoCreated.id }, reason: 'super'.repeat(605) } + + await makePostBodyRequest({ url: server.url, path, token: userToken, fields }) + }) + + it('Should succeed with the correct parameters (basic)', async function () { + const fields: AbuseCreate = { video: { id: server.store.videoCreated.shortUUID }, reason: 'my super reason' } + + const res = await makePostBodyRequest({ + url: server.url, + path, + token: userToken, + fields, + expectedStatus: HttpStatusCode.OK_200 + }) + abuseId = res.body.abuse.id + }) + + it('Should fail with a wrong predefined reason', async function () { + const fields = { video: server.store.videoCreated, reason: 'my super reason', predefinedReasons: [ 'wrongPredefinedReason' ] } + + await makePostBodyRequest({ url: server.url, path, token: userToken, fields }) + }) + + it('Should fail with negative timestamps', async function () { + const fields = { video: { id: server.store.videoCreated.id, startAt: -1 }, reason: 'my super reason' } + + await makePostBodyRequest({ url: server.url, path, token: userToken, fields }) + }) + + it('Should fail mith misordered startAt/endAt', async function () { + const fields = { video: { id: server.store.videoCreated.id, startAt: 5, endAt: 1 }, reason: 'my super reason' } + + await makePostBodyRequest({ url: server.url, path, token: userToken, fields }) + }) + + it('Should succeed with the correct parameters (advanced)', async function () { + const fields: AbuseCreate = { + video: { + id: server.store.videoCreated.id, + startAt: 1, + endAt: 5 + }, + reason: 'my super reason', + predefinedReasons: [ 'serverRules' ] + } + + await makePostBodyRequest({ url: server.url, path, token: userToken, fields, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('When updating an abuse', function () { + + it('Should fail with a non authenticated user', async function () { + await command.update({ token: 'blabla', abuseId, body: {}, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with a non admin user', async function () { + await command.update({ token: userToken, abuseId, body: {}, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail with a bad abuse id', async function () { + await command.update({ abuseId: 45, body: {}, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail with a bad state', async function () { + const body = { state: 5 as any } + await command.update({ abuseId, body, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with a bad moderation comment', async function () { + const body = { moderationComment: 'b'.repeat(3001) } + await command.update({ abuseId, body, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should succeed with the correct params', async function () { + const body = { state: AbuseState.ACCEPTED } + await command.update({ abuseId, body }) + }) + }) + + describe('When creating an abuse message', function () { + const message = 'my super message' + + it('Should fail with an invalid abuse id', async function () { + await command.addMessage({ token: userToken2, abuseId: 888, message, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail with a non authenticated user', async function () { + await command.addMessage({ token: 'fake_token', abuseId, message, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with an invalid logged in user', async function () { + await command.addMessage({ token: userToken2, abuseId, message, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail with an invalid message', async function () { + await command.addMessage({ token: userToken, abuseId, message: 'a'.repeat(5000), expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should succeed with the correct params', async function () { + const res = await command.addMessage({ token: userToken, abuseId, message }) + messageId = res.body.abuseMessage.id + }) + }) + + describe('When listing abuse messages', function () { + + it('Should fail with an invalid abuse id', async function () { + await command.listMessages({ token: userToken, abuseId: 888, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail with a non authenticated user', async function () { + await command.listMessages({ token: 'fake_token', abuseId, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with an invalid logged in user', async function () { + await command.listMessages({ token: userToken2, abuseId, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should succeed with the correct params', async function () { + await command.listMessages({ token: userToken, abuseId }) + }) + }) + + describe('When deleting an abuse message', function () { + it('Should fail with an invalid abuse id', async function () { + await command.deleteMessage({ token: userToken, abuseId: 888, messageId, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail with an invalid message id', async function () { + await command.deleteMessage({ token: userToken, abuseId, messageId: 888, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail with a non authenticated user', async function () { + await command.deleteMessage({ token: 'fake_token', abuseId, messageId, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with an invalid logged in user', async function () { + await command.deleteMessage({ token: userToken2, abuseId, messageId, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should succeed with the correct params', async function () { + await command.deleteMessage({ token: userToken, abuseId, messageId }) + }) + }) + + describe('When deleting a video abuse', function () { + + it('Should fail with a non authenticated user', async function () { + await command.delete({ token: 'blabla', abuseId, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with a non admin user', async function () { + await command.delete({ token: userToken, abuseId, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail with a bad abuse id', async function () { + await command.delete({ abuseId: 45, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should succeed with the correct params', async function () { + await command.delete({ abuseId }) + }) + }) + + describe('When trying to manage messages of a remote abuse', function () { + let remoteAbuseId: number + let anotherServer: PeerTubeServer + + before(async function () { + this.timeout(50000) + + anotherServer = await createSingleServer(2) + await setAccessTokensToServers([ anotherServer ]) + + await doubleFollow(anotherServer, server) + + const server2VideoId = await anotherServer.videos.getId({ uuid: server.store.videoCreated.uuid }) + await anotherServer.abuses.report({ reason: 'remote server', videoId: server2VideoId }) + + await waitJobs([ server, anotherServer ]) + + const body = await command.getAdminList({ sort: '-createdAt' }) + remoteAbuseId = body.data[0].id + }) + + it('Should fail when listing abuse messages of a remote abuse', async function () { + await command.listMessages({ abuseId: remoteAbuseId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail when creating abuse message of a remote abuse', async function () { + await command.addMessage({ abuseId: remoteAbuseId, message: 'message', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + after(async function () { + await cleanupTests([ anotherServer ]) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/accounts.ts b/packages/tests/src/api/check-params/accounts.ts new file mode 100644 index 000000000..87810bbd3 --- /dev/null +++ b/packages/tests/src/api/check-params/accounts.ts @@ -0,0 +1,43 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@tests/shared/checks.js' +import { HttpStatusCode } from '@peertube/peertube-models' +import { cleanupTests, createSingleServer, PeerTubeServer } from '@peertube/peertube-server-commands' + +describe('Test accounts API validators', function () { + const path = '/api/v1/accounts/' + let server: PeerTubeServer + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + }) + + describe('When listing accounts', function () { + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, server.accessToken) + }) + }) + + describe('When getting an account', function () { + + it('Should return 404 with a non existing name', async function () { + await server.accounts.get({ accountName: 'arfaze', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/blocklist.ts b/packages/tests/src/api/check-params/blocklist.ts new file mode 100644 index 000000000..fcd6d08f8 --- /dev/null +++ b/packages/tests/src/api/check-params/blocklist.ts @@ -0,0 +1,556 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@tests/shared/checks.js' +import { HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + makeDeleteRequest, + makeGetRequest, + makePostBodyRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test blocklist API validators', function () { + let servers: PeerTubeServer[] + let server: PeerTubeServer + let userAccessToken: string + + before(async function () { + this.timeout(60000) + + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + + server = servers[0] + + const user = { username: 'user1', password: 'password' } + await server.users.create({ username: user.username, password: user.password }) + + userAccessToken = await server.login.getAccessToken(user) + + await doubleFollow(servers[0], servers[1]) + }) + + // --------------------------------------------------------------- + + describe('When managing user blocklist', function () { + + describe('When managing user accounts blocklist', function () { + const path = '/api/v1/users/me/blocklist/accounts' + + describe('When listing blocked accounts', function () { + it('Should fail with an unauthenticated user', async function () { + await makeGetRequest({ + url: server.url, + path, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, server.accessToken) + }) + }) + + describe('When blocking an account', function () { + it('Should fail with an unauthenticated user', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { accountName: 'user1' }, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with an unknown account', async function () { + await makePostBodyRequest({ + url: server.url, + token: server.accessToken, + path, + fields: { accountName: 'user2' }, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail to block ourselves', async function () { + await makePostBodyRequest({ + url: server.url, + token: server.accessToken, + path, + fields: { accountName: 'root' }, + expectedStatus: HttpStatusCode.CONFLICT_409 + }) + }) + + it('Should succeed with the correct params', async function () { + await makePostBodyRequest({ + url: server.url, + token: server.accessToken, + path, + fields: { accountName: 'user1' }, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + describe('When unblocking an account', function () { + it('Should fail with an unauthenticated user', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/user1', + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with an unknown account block', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/user2', + token: server.accessToken, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should succeed with the correct params', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/user1', + token: server.accessToken, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + }) + + describe('When managing user servers blocklist', function () { + const path = '/api/v1/users/me/blocklist/servers' + + describe('When listing blocked servers', function () { + it('Should fail with an unauthenticated user', async function () { + await makeGetRequest({ + url: server.url, + path, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, server.accessToken) + }) + }) + + describe('When blocking a server', function () { + it('Should fail with an unauthenticated user', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { host: '127.0.0.1:9002' }, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should succeed with an unknown server', async function () { + await makePostBodyRequest({ + url: server.url, + token: server.accessToken, + path, + fields: { host: '127.0.0.1:9003' }, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + + it('Should fail with our own server', async function () { + await makePostBodyRequest({ + url: server.url, + token: server.accessToken, + path, + fields: { host: server.host }, + expectedStatus: HttpStatusCode.CONFLICT_409 + }) + }) + + it('Should succeed with the correct params', async function () { + await makePostBodyRequest({ + url: server.url, + token: server.accessToken, + path, + fields: { host: servers[1].host }, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + describe('When unblocking a server', function () { + it('Should fail with an unauthenticated user', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/' + servers[1].host, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with an unknown server block', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/127.0.0.1:9004', + token: server.accessToken, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should succeed with the correct params', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/' + servers[1].host, + token: server.accessToken, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + }) + }) + + describe('When managing server blocklist', function () { + + describe('When managing server accounts blocklist', function () { + const path = '/api/v1/server/blocklist/accounts' + + describe('When listing blocked accounts', function () { + it('Should fail with an unauthenticated user', async function () { + await makeGetRequest({ + url: server.url, + path, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a user without the appropriate rights', async function () { + await makeGetRequest({ + url: server.url, + token: userAccessToken, + path, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, server.accessToken) + }) + }) + + describe('When blocking an account', function () { + it('Should fail with an unauthenticated user', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { accountName: 'user1' }, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a user without the appropriate rights', async function () { + await makePostBodyRequest({ + url: server.url, + token: userAccessToken, + path, + fields: { accountName: 'user1' }, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with an unknown account', async function () { + await makePostBodyRequest({ + url: server.url, + token: server.accessToken, + path, + fields: { accountName: 'user2' }, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail to block ourselves', async function () { + await makePostBodyRequest({ + url: server.url, + token: server.accessToken, + path, + fields: { accountName: 'root' }, + expectedStatus: HttpStatusCode.CONFLICT_409 + }) + }) + + it('Should succeed with the correct params', async function () { + await makePostBodyRequest({ + url: server.url, + token: server.accessToken, + path, + fields: { accountName: 'user1' }, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + describe('When unblocking an account', function () { + it('Should fail with an unauthenticated user', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/user1', + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a user without the appropriate rights', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/user1', + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with an unknown account block', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/user2', + token: server.accessToken, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should succeed with the correct params', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/user1', + token: server.accessToken, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + }) + + describe('When managing server servers blocklist', function () { + const path = '/api/v1/server/blocklist/servers' + + describe('When listing blocked servers', function () { + it('Should fail with an unauthenticated user', async function () { + await makeGetRequest({ + url: server.url, + path, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a user without the appropriate rights', async function () { + await makeGetRequest({ + url: server.url, + token: userAccessToken, + path, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, server.accessToken) + }) + }) + + describe('When blocking a server', function () { + it('Should fail with an unauthenticated user', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { host: servers[1].host }, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a user without the appropriate rights', async function () { + await makePostBodyRequest({ + url: server.url, + token: userAccessToken, + path, + fields: { host: servers[1].host }, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should succeed with an unknown server', async function () { + await makePostBodyRequest({ + url: server.url, + token: server.accessToken, + path, + fields: { host: '127.0.0.1:9003' }, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + + it('Should fail with our own server', async function () { + await makePostBodyRequest({ + url: server.url, + token: server.accessToken, + path, + fields: { host: server.host }, + expectedStatus: HttpStatusCode.CONFLICT_409 + }) + }) + + it('Should succeed with the correct params', async function () { + await makePostBodyRequest({ + url: server.url, + token: server.accessToken, + path, + fields: { host: servers[1].host }, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + describe('When unblocking a server', function () { + it('Should fail with an unauthenticated user', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/' + servers[1].host, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a user without the appropriate rights', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/' + servers[1].host, + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with an unknown server block', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/127.0.0.1:9004', + token: server.accessToken, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should succeed with the correct params', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/' + servers[1].host, + token: server.accessToken, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + }) + }) + + describe('When getting blocklist status', function () { + const path = '/api/v1/blocklist/status' + + it('Should fail with a bad token', async function () { + await makeGetRequest({ + url: server.url, + path, + token: 'false', + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a bad accounts field', async function () { + await makeGetRequest({ + url: server.url, + path, + query: { + accounts: 1 + }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + + await makeGetRequest({ + url: server.url, + path, + query: { + accounts: [ 1 ] + }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with a bad hosts field', async function () { + await makeGetRequest({ + url: server.url, + path, + query: { + hosts: 1 + }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + + await makeGetRequest({ + url: server.url, + path, + query: { + hosts: [ 1 ] + }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should succeed with the correct parameters', async function () { + await makeGetRequest({ + url: server.url, + path, + query: {}, + expectedStatus: HttpStatusCode.OK_200 + }) + + await makeGetRequest({ + url: server.url, + path, + query: { + hosts: [ 'example.com' ], + accounts: [ 'john@example.com' ] + }, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/check-params/bulk.ts b/packages/tests/src/api/check-params/bulk.ts new file mode 100644 index 000000000..def0c38eb --- /dev/null +++ b/packages/tests/src/api/check-params/bulk.ts @@ -0,0 +1,86 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + makePostBodyRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test bulk API validators', function () { + let server: PeerTubeServer + let userAccessToken: string + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(120000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + + const user = { username: 'user1', password: 'password' } + await server.users.create({ username: user.username, password: user.password }) + + userAccessToken = await server.login.getAccessToken(user) + }) + + describe('When removing comments of', function () { + const path = '/api/v1/bulk/remove-comments-of' + + it('Should fail with an unauthenticated user', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { accountName: 'user1', scope: 'my-videos' }, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with an unknown account', async function () { + await makePostBodyRequest({ + url: server.url, + token: server.accessToken, + path, + fields: { accountName: 'user2', scope: 'my-videos' }, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail with an invalid scope', async function () { + await makePostBodyRequest({ + url: server.url, + token: server.accessToken, + path, + fields: { accountName: 'user1', scope: 'my-videoss' }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail to delete comments of the instance without the appropriate rights', async function () { + await makePostBodyRequest({ + url: server.url, + token: userAccessToken, + path, + fields: { accountName: 'user1', scope: 'instance' }, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should succeed with the correct params', async function () { + await makePostBodyRequest({ + url: server.url, + token: server.accessToken, + path, + fields: { accountName: 'user1', scope: 'instance' }, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/channel-import-videos.ts b/packages/tests/src/api/check-params/channel-import-videos.ts new file mode 100644 index 000000000..0e897dad7 --- /dev/null +++ b/packages/tests/src/api/check-params/channel-import-videos.ts @@ -0,0 +1,209 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { FIXTURE_URLS } from '@tests/shared/tests.js' +import { areHttpImportTestsDisabled } from '@peertube/peertube-node-utils' +import { HttpStatusCode } from '@peertube/peertube-models' +import { + ChannelsCommand, + cleanupTests, + createSingleServer, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel +} from '@peertube/peertube-server-commands' + +describe('Test videos import in a channel API validator', function () { + let server: PeerTubeServer + const userInfo = { + accessToken: '', + channelName: 'fake_channel', + channelId: -1, + id: -1, + videoQuota: -1, + videoQuotaDaily: -1, + channelSyncId: -1 + } + let command: ChannelsCommand + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(120000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + await server.config.enableImports() + await server.config.enableChannelSync() + + const userCreds = { + username: 'fake', + password: 'fake_password' + } + + { + const user = await server.users.create({ username: userCreds.username, password: userCreds.password }) + userInfo.id = user.id + userInfo.accessToken = await server.login.getAccessToken(userCreds) + + const info = await server.users.getMyInfo({ token: userInfo.accessToken }) + userInfo.channelId = info.videoChannels[0].id + } + + { + const { videoChannelSync } = await server.channelSyncs.create({ + token: userInfo.accessToken, + attributes: { + externalChannelUrl: FIXTURE_URLS.youtubeChannel, + videoChannelId: userInfo.channelId + } + }) + userInfo.channelSyncId = videoChannelSync.id + } + + command = server.channels + }) + + it('Should fail when HTTP upload is disabled', async function () { + await server.config.disableChannelSync() + await server.config.disableImports() + + await command.importVideos({ + channelName: server.store.channel.name, + externalChannelUrl: FIXTURE_URLS.youtubeChannel, + token: server.accessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + + await server.config.enableImports() + }) + + it('Should fail when externalChannelUrl is not provided', async function () { + await command.importVideos({ + channelName: server.store.channel.name, + externalChannelUrl: null, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail when externalChannelUrl is malformed', async function () { + await command.importVideos({ + channelName: server.store.channel.name, + externalChannelUrl: 'not-a-url', + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with a bad sync id', async function () { + await command.importVideos({ + channelName: server.store.channel.name, + externalChannelUrl: FIXTURE_URLS.youtubeChannel, + videoChannelSyncId: 'toto' as any, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with a unknown sync id', async function () { + await command.importVideos({ + channelName: server.store.channel.name, + externalChannelUrl: FIXTURE_URLS.youtubeChannel, + videoChannelSyncId: 42, + token: server.accessToken, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail with a sync id of another channel', async function () { + await command.importVideos({ + channelName: server.store.channel.name, + externalChannelUrl: FIXTURE_URLS.youtubeChannel, + videoChannelSyncId: userInfo.channelSyncId, + token: server.accessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with no authentication', async function () { + await command.importVideos({ + channelName: server.store.channel.name, + externalChannelUrl: FIXTURE_URLS.youtubeChannel, + token: null, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail when sync is not owned by the user', async function () { + await command.importVideos({ + channelName: server.store.channel.name, + externalChannelUrl: FIXTURE_URLS.youtubeChannel, + token: userInfo.accessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail when the user has no quota', async function () { + await server.users.update({ + userId: userInfo.id, + videoQuota: 0 + }) + + await command.importVideos({ + channelName: 'fake_channel', + externalChannelUrl: FIXTURE_URLS.youtubeChannel, + token: userInfo.accessToken, + expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413 + }) + + await server.users.update({ + userId: userInfo.id, + videoQuota: userInfo.videoQuota + }) + }) + + it('Should fail when the user has no daily quota', async function () { + await server.users.update({ + userId: userInfo.id, + videoQuotaDaily: 0 + }) + + await command.importVideos({ + channelName: 'fake_channel', + externalChannelUrl: FIXTURE_URLS.youtubeChannel, + token: userInfo.accessToken, + expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413 + }) + + await server.users.update({ + userId: userInfo.id, + videoQuotaDaily: userInfo.videoQuotaDaily + }) + }) + + it('Should succeed when sync is run by its owner', async function () { + if (!areHttpImportTestsDisabled()) return + + await command.importVideos({ + channelName: 'fake_channel', + externalChannelUrl: FIXTURE_URLS.youtubeChannel, + token: userInfo.accessToken + }) + }) + + it('Should succeed when sync is run with root and for another user\'s channel', async function () { + if (!areHttpImportTestsDisabled()) return + + await command.importVideos({ + channelName: 'fake_channel', + externalChannelUrl: FIXTURE_URLS.youtubeChannel + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/config.ts b/packages/tests/src/api/check-params/config.ts new file mode 100644 index 000000000..8179a8815 --- /dev/null +++ b/packages/tests/src/api/check-params/config.ts @@ -0,0 +1,428 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ +import merge from 'lodash-es/merge.js' +import { omit } from '@peertube/peertube-core-utils' +import { CustomConfig, HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + makeDeleteRequest, + makeGetRequest, + makePutBodyRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test config API validators', function () { + const path = '/api/v1/config/custom' + let server: PeerTubeServer + let userAccessToken: string + const updateParams: CustomConfig = { + instance: { + name: 'PeerTube updated', + shortDescription: 'my short description', + description: 'my super description', + terms: 'my super terms', + codeOfConduct: 'my super coc', + + creationReason: 'my super reason', + moderationInformation: 'my super moderation information', + administrator: 'Kuja', + maintenanceLifetime: 'forever', + businessModel: 'my super business model', + hardwareInformation: '2vCore 3GB RAM', + + languages: [ 'en', 'es' ], + categories: [ 1, 2 ], + + isNSFW: true, + defaultNSFWPolicy: 'blur', + + defaultClientRoute: '/videos/recently-added', + + customizations: { + javascript: 'alert("coucou")', + css: 'body { background-color: red; }' + } + }, + theme: { + default: 'default' + }, + services: { + twitter: { + username: '@MySuperUsername', + whitelisted: true + } + }, + client: { + videos: { + miniature: { + preferAuthorDisplayName: false + } + }, + menu: { + login: { + redirectOnSingleExternalAuth: false + } + } + }, + cache: { + previews: { + size: 2 + }, + captions: { + size: 3 + }, + torrents: { + size: 4 + }, + storyboards: { + size: 5 + } + }, + signup: { + enabled: false, + limit: 5, + requiresApproval: false, + requiresEmailVerification: false, + minimumAge: 16 + }, + admin: { + email: 'superadmin1@example.com' + }, + contactForm: { + enabled: false + }, + user: { + history: { + videos: { + enabled: true + } + }, + videoQuota: 5242881, + videoQuotaDaily: 318742 + }, + videoChannels: { + maxPerUser: 20 + }, + transcoding: { + enabled: true, + remoteRunners: { + enabled: true + }, + allowAdditionalExtensions: true, + allowAudioFiles: true, + concurrency: 1, + threads: 1, + profile: 'vod_profile', + resolutions: { + '0p': false, + '144p': false, + '240p': false, + '360p': true, + '480p': true, + '720p': false, + '1080p': false, + '1440p': false, + '2160p': false + }, + alwaysTranscodeOriginalResolution: false, + webVideos: { + enabled: true + }, + hls: { + enabled: false + } + }, + live: { + enabled: true, + + allowReplay: false, + latencySetting: { + enabled: false + }, + maxDuration: 30, + maxInstanceLives: -1, + maxUserLives: 50, + + transcoding: { + enabled: true, + remoteRunners: { + enabled: true + }, + threads: 4, + profile: 'live_profile', + resolutions: { + '144p': true, + '240p': true, + '360p': true, + '480p': true, + '720p': true, + '1080p': true, + '1440p': true, + '2160p': true + }, + alwaysTranscodeOriginalResolution: false + } + }, + videoStudio: { + enabled: true, + remoteRunners: { + enabled: true + } + }, + videoFile: { + update: { + enabled: true + } + }, + import: { + videos: { + concurrency: 1, + http: { + enabled: false + }, + torrent: { + enabled: false + } + }, + videoChannelSynchronization: { + enabled: false, + maxPerUser: 10 + } + }, + trending: { + videos: { + algorithms: { + enabled: [ 'hot', 'most-viewed', 'most-liked' ], + default: 'most-viewed' + } + } + }, + autoBlacklist: { + videos: { + ofUsers: { + enabled: false + } + } + }, + followers: { + instance: { + enabled: false, + manualApproval: true + } + }, + followings: { + instance: { + autoFollowBack: { + enabled: true + }, + autoFollowIndex: { + enabled: true, + indexUrl: 'https://index.example.com' + } + } + }, + broadcastMessage: { + enabled: true, + dismissable: true, + message: 'super message', + level: 'warning' + }, + search: { + remoteUri: { + users: true, + anonymous: true + }, + searchIndex: { + enabled: true, + url: 'https://search.joinpeertube.org', + disableLocalSearch: true, + isDefaultSearch: true + } + } + } + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + + const user = { + username: 'user1', + password: 'password' + } + await server.users.create({ username: user.username, password: user.password }) + userAccessToken = await server.login.getAccessToken(user) + }) + + describe('When getting the configuration', function () { + it('Should fail without token', async function () { + await makeGetRequest({ + url: server.url, + path, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail if the user is not an administrator', async function () { + await makeGetRequest({ + url: server.url, + path, + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + }) + + describe('When updating the configuration', function () { + it('Should fail without token', async function () { + await makePutBodyRequest({ + url: server.url, + path, + fields: updateParams, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail if the user is not an administrator', async function () { + await makePutBodyRequest({ + url: server.url, + path, + fields: updateParams, + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail if it misses a key', async function () { + const newUpdateParams = { ...updateParams, admin: omit(updateParams.admin, [ 'email' ]) } + + await makePutBodyRequest({ + url: server.url, + path, + fields: newUpdateParams, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with a bad default NSFW policy', async function () { + const newUpdateParams = { + ...updateParams, + + instance: { + defaultNSFWPolicy: 'hello' + } + } + + await makePutBodyRequest({ + url: server.url, + path, + fields: newUpdateParams, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail if email disabled and signup requires email verification', async function () { + // opposite scenario - success when enable enabled - covered via tests/api/users/user-verification.ts + const newUpdateParams = { + ...updateParams, + + signup: { + enabled: true, + limit: 5, + requiresApproval: true, + requiresEmailVerification: true + } + } + + await makePutBodyRequest({ + url: server.url, + path, + fields: newUpdateParams, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with a disabled web videos & hls transcoding', async function () { + const newUpdateParams = { + ...updateParams, + + transcoding: { + hls: { + enabled: false + }, + web_videos: { + enabled: false + } + } + } + + await makePutBodyRequest({ + url: server.url, + path, + fields: newUpdateParams, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with a disabled http upload & enabled sync', async function () { + const newUpdateParams: CustomConfig = merge({}, updateParams, { + import: { + videos: { + http: { enabled: false } + }, + videoChannelSynchronization: { enabled: true } + } + }) + + await makePutBodyRequest({ + url: server.url, + path, + fields: newUpdateParams, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should succeed with the correct parameters', async function () { + await makePutBodyRequest({ + url: server.url, + path, + fields: updateParams, + token: server.accessToken, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + + describe('When deleting the configuration', function () { + it('Should fail without token', async function () { + await makeDeleteRequest({ + url: server.url, + path, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail if the user is not an administrator', async function () { + await makeDeleteRequest({ + url: server.url, + path, + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/contact-form.ts b/packages/tests/src/api/check-params/contact-form.ts new file mode 100644 index 000000000..009cb2ad9 --- /dev/null +++ b/packages/tests/src/api/check-params/contact-form.ts @@ -0,0 +1,86 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { MockSmtpServer } from '@tests/shared/mock-servers/index.js' +import { HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + ConfigCommand, + ContactFormCommand, + createSingleServer, + killallServers, + PeerTubeServer +} from '@peertube/peertube-server-commands' + +describe('Test contact form API validators', function () { + let server: PeerTubeServer + const emails: object[] = [] + const defaultBody = { + fromName: 'super name', + fromEmail: 'toto@example.com', + subject: 'my subject', + body: 'Hello, how are you?' + } + let emailPort: number + let command: ContactFormCommand + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(60000) + + emailPort = await MockSmtpServer.Instance.collectEmails(emails) + + // Email is disabled + server = await createSingleServer(1) + command = server.contactForm + }) + + it('Should not accept a contact form if emails are disabled', async function () { + await command.send({ ...defaultBody, expectedStatus: HttpStatusCode.CONFLICT_409 }) + }) + + it('Should not accept a contact form if it is disabled in the configuration', async function () { + this.timeout(25000) + + await killallServers([ server ]) + + // Contact form is disabled + await server.run({ ...ConfigCommand.getEmailOverrideConfig(emailPort), contact_form: { enabled: false } }) + await command.send({ ...defaultBody, expectedStatus: HttpStatusCode.CONFLICT_409 }) + }) + + it('Should not accept a contact form if from email is invalid', async function () { + this.timeout(25000) + + await killallServers([ server ]) + + // Email & contact form enabled + await server.run(ConfigCommand.getEmailOverrideConfig(emailPort)) + + await command.send({ ...defaultBody, fromEmail: 'badEmail', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await command.send({ ...defaultBody, fromEmail: 'badEmail@', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await command.send({ ...defaultBody, fromEmail: undefined, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should not accept a contact form if from name is invalid', async function () { + await command.send({ ...defaultBody, fromName: 'name'.repeat(100), expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await command.send({ ...defaultBody, fromName: '', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await command.send({ ...defaultBody, fromName: undefined, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should not accept a contact form if body is invalid', async function () { + await command.send({ ...defaultBody, body: 'body'.repeat(5000), expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await command.send({ ...defaultBody, body: 'a', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await command.send({ ...defaultBody, body: undefined, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should accept a contact form with the correct parameters', async function () { + await command.send(defaultBody) + }) + + after(async function () { + MockSmtpServer.Instance.kill() + + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/custom-pages.ts b/packages/tests/src/api/check-params/custom-pages.ts new file mode 100644 index 000000000..180a5e406 --- /dev/null +++ b/packages/tests/src/api/check-params/custom-pages.ts @@ -0,0 +1,79 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + makeGetRequest, + makePutBodyRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test custom pages validators', function () { + const path = '/api/v1/custom-pages/homepage/instance' + + let server: PeerTubeServer + let userAccessToken: string + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(120000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + + const user = { username: 'user1', password: 'password' } + await server.users.create({ username: user.username, password: user.password }) + + userAccessToken = await server.login.getAccessToken(user) + }) + + describe('When updating instance homepage', function () { + + it('Should fail with an unauthenticated user', async function () { + await makePutBodyRequest({ + url: server.url, + path, + fields: { content: 'super content' }, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a non admin user', async function () { + await makePutBodyRequest({ + url: server.url, + path, + token: userAccessToken, + fields: { content: 'super content' }, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should succeed with the correct params', async function () { + await makePutBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields: { content: 'super content' }, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + describe('When getting instance homapage', function () { + + it('Should succeed with the correct params', async function () { + await makeGetRequest({ + url: server.url, + path, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/debug.ts b/packages/tests/src/api/check-params/debug.ts new file mode 100644 index 000000000..4a7c18a62 --- /dev/null +++ b/packages/tests/src/api/check-params/debug.ts @@ -0,0 +1,67 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + makeGetRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test debug API validators', function () { + const path = '/api/v1/server/debug' + let server: PeerTubeServer + let userAccessToken = '' + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(120000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + + const user = { + username: 'user1', + password: 'my super password' + } + await server.users.create({ username: user.username, password: user.password }) + userAccessToken = await server.login.getAccessToken(user) + }) + + describe('When getting debug endpoint', function () { + + it('Should fail with a non authenticated user', async function () { + await makeGetRequest({ + url: server.url, + path, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a non admin user', async function () { + await makeGetRequest({ + url: server.url, + path, + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should succeed with the correct params', async function () { + await makeGetRequest({ + url: server.url, + path, + token: server.accessToken, + query: { startDate: new Date().toISOString() }, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/follows.ts b/packages/tests/src/api/check-params/follows.ts new file mode 100644 index 000000000..e92a3acd6 --- /dev/null +++ b/packages/tests/src/api/check-params/follows.ts @@ -0,0 +1,369 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@tests/shared/checks.js' +import { HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + makeDeleteRequest, + makeGetRequest, + makePostBodyRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test server follows API validators', function () { + let server: PeerTubeServer + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + }) + + describe('When managing following', function () { + let userAccessToken = null + + before(async function () { + userAccessToken = await server.users.generateUserAndToken('user1') + }) + + describe('When adding follows', function () { + const path = '/api/v1/server/following' + + it('Should fail with nothing', async function () { + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail if hosts is not composed by hosts', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { hosts: [ '127.0.0.1:9002', '127.0.0.1:coucou' ] }, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail if hosts is composed with http schemes', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { hosts: [ '127.0.0.1:9002', 'http://127.0.0.1:9003' ] }, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail if hosts are not unique', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { urls: [ '127.0.0.1:9002', '127.0.0.1:9002' ] }, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail if handles is not composed by handles', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { handles: [ 'hello@example.com', '127.0.0.1:9001' ] }, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail if handles are not unique', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { urls: [ 'hello@example.com', 'hello@example.com' ] }, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with an invalid token', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { hosts: [ '127.0.0.1:9002' ] }, + token: 'fake_token', + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail if the user is not an administrator', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { hosts: [ '127.0.0.1:9002' ] }, + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + }) + + describe('When listing followings', function () { + const path = '/api/v1/server/following' + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path) + }) + + it('Should fail with an incorrect state', async function () { + await makeGetRequest({ + url: server.url, + path, + query: { + state: 'blabla' + } + }) + }) + + it('Should fail with an incorrect actor type', async function () { + await makeGetRequest({ + url: server.url, + path, + query: { + actorType: 'blabla' + } + }) + }) + + it('Should fail succeed with the correct params', async function () { + await makeGetRequest({ + url: server.url, + path, + expectedStatus: HttpStatusCode.OK_200, + query: { + state: 'accepted', + actorType: 'Application' + } + }) + }) + }) + + describe('When listing followers', function () { + const path = '/api/v1/server/followers' + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path) + }) + + it('Should fail with an incorrect actor type', async function () { + await makeGetRequest({ + url: server.url, + path, + query: { + actorType: 'blabla' + } + }) + }) + + it('Should fail with an incorrect state', async function () { + await makeGetRequest({ + url: server.url, + path, + query: { + state: 'blabla', + actorType: 'Application' + } + }) + }) + + it('Should fail succeed with the correct params', async function () { + await makeGetRequest({ + url: server.url, + path, + expectedStatus: HttpStatusCode.OK_200, + query: { + state: 'accepted' + } + }) + }) + }) + + describe('When removing a follower', function () { + const path = '/api/v1/server/followers' + + it('Should fail with an invalid token', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/toto@127.0.0.1:9002', + token: 'fake_token', + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail if the user is not an administrator', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/toto@127.0.0.1:9002', + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with an invalid follower', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/toto', + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with an unknown follower', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/toto@127.0.0.1:9003', + token: server.accessToken, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + }) + + describe('When accepting a follower', function () { + const path = '/api/v1/server/followers' + + it('Should fail with an invalid token', async function () { + await makePostBodyRequest({ + url: server.url, + path: path + '/toto@127.0.0.1:9002/accept', + token: 'fake_token', + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail if the user is not an administrator', async function () { + await makePostBodyRequest({ + url: server.url, + path: path + '/toto@127.0.0.1:9002/accept', + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with an invalid follower', async function () { + await makePostBodyRequest({ + url: server.url, + path: path + '/toto/accept', + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with an unknown follower', async function () { + await makePostBodyRequest({ + url: server.url, + path: path + '/toto@127.0.0.1:9003/accept', + token: server.accessToken, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + }) + + describe('When rejecting a follower', function () { + const path = '/api/v1/server/followers' + + it('Should fail with an invalid token', async function () { + await makePostBodyRequest({ + url: server.url, + path: path + '/toto@127.0.0.1:9002/reject', + token: 'fake_token', + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail if the user is not an administrator', async function () { + await makePostBodyRequest({ + url: server.url, + path: path + '/toto@127.0.0.1:9002/reject', + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with an invalid follower', async function () { + await makePostBodyRequest({ + url: server.url, + path: path + '/toto/reject', + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with an unknown follower', async function () { + await makePostBodyRequest({ + url: server.url, + path: path + '/toto@127.0.0.1:9003/reject', + token: server.accessToken, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + }) + + describe('When removing following', function () { + const path = '/api/v1/server/following' + + it('Should fail with an invalid token', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/127.0.0.1:9002', + token: 'fake_token', + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail if the user is not an administrator', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/127.0.0.1:9002', + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail if we do not follow this server', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/example.com', + token: server.accessToken, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/index.ts b/packages/tests/src/api/check-params/index.ts new file mode 100644 index 000000000..ed5fe6b06 --- /dev/null +++ b/packages/tests/src/api/check-params/index.ts @@ -0,0 +1,45 @@ +import './abuses.js' +import './accounts.js' +import './blocklist.js' +import './bulk.js' +import './channel-import-videos.js' +import './config.js' +import './contact-form.js' +import './custom-pages.js' +import './debug.js' +import './follows.js' +import './jobs.js' +import './live.js' +import './logs.js' +import './metrics.js' +import './my-user.js' +import './plugins.js' +import './redundancy.js' +import './registrations.js' +import './runners.js' +import './search.js' +import './services.js' +import './transcoding.js' +import './two-factor.js' +import './upload-quota.js' +import './user-notifications.js' +import './user-subscriptions.js' +import './users-admin.js' +import './users-emails.js' +import './video-blacklist.js' +import './video-captions.js' +import './video-channel-syncs.js' +import './video-channels.js' +import './video-comments.js' +import './video-files.js' +import './video-imports.js' +import './video-playlists.js' +import './video-storyboards.js' +import './video-source.js' +import './video-studio.js' +import './video-token.js' +import './videos-common-filters.js' +import './videos-history.js' +import './videos-overviews.js' +import './videos.js' +import './views.js' diff --git a/packages/tests/src/api/check-params/jobs.ts b/packages/tests/src/api/check-params/jobs.ts new file mode 100644 index 000000000..331d58c6a --- /dev/null +++ b/packages/tests/src/api/check-params/jobs.ts @@ -0,0 +1,125 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@tests/shared/checks.js' +import { HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + makeGetRequest, + makePostBodyRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test jobs API validators', function () { + const path = '/api/v1/jobs/failed' + let server: PeerTubeServer + let userAccessToken = '' + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(120000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + + const user = { + username: 'user1', + password: 'my super password' + } + await server.users.create({ username: user.username, password: user.password }) + userAccessToken = await server.login.getAccessToken(user) + }) + + describe('When listing jobs', function () { + + it('Should fail with a bad state', async function () { + await makeGetRequest({ + url: server.url, + token: server.accessToken, + path: path + 'ade' + }) + }) + + it('Should fail with an incorrect job type', async function () { + await makeGetRequest({ + url: server.url, + token: server.accessToken, + path, + query: { + jobType: 'toto' + } + }) + }) + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a non authenticated user', async function () { + await makeGetRequest({ + url: server.url, + path, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a non admin user', async function () { + await makeGetRequest({ + url: server.url, + path, + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + }) + + describe('When pausing/resuming the job queue', async function () { + const commands = [ 'pause', 'resume' ] + + it('Should fail with a non authenticated user', async function () { + for (const command of commands) { + await makePostBodyRequest({ + url: server.url, + path: '/api/v1/jobs/' + command, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + } + }) + + it('Should fail with a non admin user', async function () { + for (const command of commands) { + await makePostBodyRequest({ + url: server.url, + path: '/api/v1/jobs/' + command, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + } + }) + + it('Should succeed with the correct params', async function () { + for (const command of commands) { + await makePostBodyRequest({ + url: server.url, + path: '/api/v1/jobs/' + command, + token: server.accessToken, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/live.ts b/packages/tests/src/api/check-params/live.ts new file mode 100644 index 000000000..5900823ea --- /dev/null +++ b/packages/tests/src/api/check-params/live.ts @@ -0,0 +1,590 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { omit } from '@peertube/peertube-core-utils' +import { HttpStatusCode, LiveVideoLatencyMode, VideoCreateResult, VideoPrivacy } from '@peertube/peertube-models' +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import { + cleanupTests, + createSingleServer, + LiveCommand, + makePostBodyRequest, + makeUploadRequest, + PeerTubeServer, + sendRTMPStream, + setAccessTokensToServers, + stopFfmpeg +} from '@peertube/peertube-server-commands' + +describe('Test video lives API validator', function () { + const path = '/api/v1/videos/live' + let server: PeerTubeServer + let userAccessToken = '' + let channelId: number + let video: VideoCreateResult + let videoIdNotLive: number + let command: LiveCommand + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + + await server.config.updateCustomSubConfig({ + newConfig: { + live: { + enabled: true, + latencySetting: { + enabled: false + }, + maxInstanceLives: 20, + maxUserLives: 20, + allowReplay: true + } + } + }) + + const username = 'user1' + const password = 'my super password' + await server.users.create({ username, password }) + userAccessToken = await server.login.getAccessToken({ username, password }) + + { + const { videoChannels } = await server.users.getMyInfo() + channelId = videoChannels[0].id + } + + { + videoIdNotLive = (await server.videos.quickUpload({ name: 'not live' })).id + } + + command = server.live + }) + + describe('When creating a live', function () { + let baseCorrectParams + + before(function () { + baseCorrectParams = { + name: 'my super name', + category: 5, + licence: 1, + language: 'pt', + nsfw: false, + commentsEnabled: true, + downloadEnabled: true, + waitTranscoding: true, + description: 'my super description', + support: 'my super support text', + tags: [ 'tag1', 'tag2' ], + privacy: VideoPrivacy.PUBLIC, + channelId, + saveReplay: false, + replaySettings: undefined, + permanentLive: false, + latencyMode: LiveVideoLatencyMode.DEFAULT + } + }) + + it('Should fail with nothing', async function () { + const fields = {} + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a long name', async function () { + const fields = { ...baseCorrectParams, name: 'super'.repeat(65) } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a bad category', async function () { + const fields = { ...baseCorrectParams, category: 125 } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a bad licence', async function () { + const fields = { ...baseCorrectParams, licence: 125 } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a bad language', async function () { + const fields = { ...baseCorrectParams, language: 'a'.repeat(15) } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a long description', async function () { + const fields = { ...baseCorrectParams, description: 'super'.repeat(2500) } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a long support text', async function () { + const fields = { ...baseCorrectParams, support: 'super'.repeat(201) } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail without a channel', async function () { + const fields = omit(baseCorrectParams, [ 'channelId' ]) + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a bad channel', async function () { + const fields = { ...baseCorrectParams, channelId: 545454 } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a bad privacy for replay settings', async function () { + const fields = { ...baseCorrectParams, saveReplay: true, replaySettings: { privacy: 999 } } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with another user channel', async function () { + const user = { + username: 'fake', + password: 'fake_password' + } + await server.users.create({ username: user.username, password: user.password }) + + const accessTokenUser = await server.login.getAccessToken(user) + const { videoChannels } = await server.users.getMyInfo({ token: accessTokenUser }) + const customChannelId = videoChannels[0].id + + const fields = { ...baseCorrectParams, channelId: customChannelId } + + await makePostBodyRequest({ url: server.url, path, token: userAccessToken, fields }) + }) + + it('Should fail with too many tags', async function () { + const fields = { ...baseCorrectParams, tags: [ 'tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6' ] } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a tag length too low', async function () { + const fields = { ...baseCorrectParams, tags: [ 'tag1', 't' ] } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a tag length too big', async function () { + const fields = { ...baseCorrectParams, tags: [ 'tag1', 'my_super_tag_too_long_long_long_long_long_long' ] } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with an incorrect thumbnail file', async function () { + const fields = baseCorrectParams + const attaches = { + thumbnailfile: buildAbsoluteFixturePath('video_short.mp4') + } + + await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches }) + }) + + it('Should fail with a big thumbnail file', async function () { + const fields = baseCorrectParams + const attaches = { + thumbnailfile: buildAbsoluteFixturePath('custom-preview-big.png') + } + + await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches }) + }) + + it('Should fail with an incorrect preview file', async function () { + const fields = baseCorrectParams + const attaches = { + previewfile: buildAbsoluteFixturePath('video_short.mp4') + } + + await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches }) + }) + + it('Should fail with a big preview file', async function () { + const fields = baseCorrectParams + const attaches = { + previewfile: buildAbsoluteFixturePath('custom-preview-big.png') + } + + await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches }) + }) + + it('Should fail with bad latency setting', async function () { + const fields = { ...baseCorrectParams, latencyMode: 42 } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail to set latency if the server does not allow it', async function () { + const fields = { ...baseCorrectParams, latencyMode: LiveVideoLatencyMode.HIGH_LATENCY } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should succeed with the correct parameters', async function () { + this.timeout(30000) + + const res = await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields: baseCorrectParams, + expectedStatus: HttpStatusCode.OK_200 + }) + + video = res.body.video + }) + + it('Should forbid if live is disabled', async function () { + await server.config.updateCustomSubConfig({ + newConfig: { + live: { + enabled: false + } + } + }) + + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields: baseCorrectParams, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should forbid to save replay if not enabled by the admin', async function () { + const fields = { ...baseCorrectParams, saveReplay: true, replaySettings: { privacy: VideoPrivacy.PUBLIC } } + + await server.config.updateCustomSubConfig({ + newConfig: { + live: { + enabled: true, + allowReplay: false + } + } + }) + + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should allow to save replay if enabled by the admin', async function () { + const fields = { ...baseCorrectParams, saveReplay: true, replaySettings: { privacy: VideoPrivacy.PUBLIC } } + + await server.config.updateCustomSubConfig({ + newConfig: { + live: { + enabled: true, + allowReplay: true + } + } + }) + + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + + it('Should not allow live if max instance lives is reached', async function () { + await server.config.updateCustomSubConfig({ + newConfig: { + live: { + enabled: true, + maxInstanceLives: 1 + } + } + }) + + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields: baseCorrectParams, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should not allow live if max user lives is reached', async function () { + await server.config.updateCustomSubConfig({ + newConfig: { + live: { + enabled: true, + maxInstanceLives: 20, + maxUserLives: 1 + } + } + }) + + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields: baseCorrectParams, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + }) + + describe('When getting live information', function () { + + it('Should fail with a bad access token', async function () { + await command.get({ token: 'toto', videoId: video.id, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should not display private information without access token', async function () { + const live = await command.get({ token: '', videoId: video.id }) + + expect(live.rtmpUrl).to.not.exist + expect(live.streamKey).to.not.exist + expect(live.latencyMode).to.exist + }) + + it('Should not display private information with token of another user', async function () { + const live = await command.get({ token: userAccessToken, videoId: video.id }) + + expect(live.rtmpUrl).to.not.exist + expect(live.streamKey).to.not.exist + expect(live.latencyMode).to.exist + }) + + it('Should display private information with appropriate token', async function () { + const live = await command.get({ videoId: video.id }) + + expect(live.rtmpUrl).to.exist + expect(live.streamKey).to.exist + expect(live.latencyMode).to.exist + }) + + it('Should fail with a bad video id', async function () { + await command.get({ videoId: 'toto', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with an unknown video id', async function () { + await command.get({ videoId: 454555, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail with a non live video', async function () { + await command.get({ videoId: videoIdNotLive, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should succeed with the correct params', async function () { + await command.get({ videoId: video.id }) + await command.get({ videoId: video.uuid }) + await command.get({ videoId: video.shortUUID }) + }) + }) + + describe('When getting live sessions', function () { + + it('Should fail with a bad access token', async function () { + await command.listSessions({ token: 'toto', videoId: video.id, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail without token', async function () { + await command.listSessions({ token: null, videoId: video.id, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with the token of another user', async function () { + await command.listSessions({ token: userAccessToken, videoId: video.id, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail with a bad video id', async function () { + await command.listSessions({ videoId: 'toto', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with an unknown video id', async function () { + await command.listSessions({ videoId: 454555, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail with a non live video', async function () { + await command.listSessions({ videoId: videoIdNotLive, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should succeed with the correct params', async function () { + await command.listSessions({ videoId: video.id }) + }) + }) + + describe('When getting live session of a replay', function () { + + it('Should fail with a bad video id', async function () { + await command.getReplaySession({ videoId: 'toto', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with an unknown video id', async function () { + await command.getReplaySession({ videoId: 454555, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail with a non replay video', async function () { + await command.getReplaySession({ videoId: videoIdNotLive, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + }) + + describe('When updating live information', async function () { + + it('Should fail without access token', async function () { + await command.update({ token: '', videoId: video.id, fields: {}, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with a bad access token', async function () { + await command.update({ token: 'toto', videoId: video.id, fields: {}, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with access token of another user', async function () { + await command.update({ token: userAccessToken, videoId: video.id, fields: {}, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail with a bad video id', async function () { + await command.update({ videoId: 'toto', fields: {}, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with an unknown video id', async function () { + await command.update({ videoId: 454555, fields: {}, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail with a non live video', async function () { + await command.update({ videoId: videoIdNotLive, fields: {}, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail with bad latency setting', async function () { + const fields = { latencyMode: 42 as any } + + await command.update({ videoId: video.id, fields, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with a bad privacy for replay settings', async function () { + const fields = { saveReplay: true, replaySettings: { privacy: 999 as any } } + + await command.update({ videoId: video.id, fields, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with save replay enabled but without replay settings', async function () { + await server.config.updateCustomSubConfig({ + newConfig: { + live: { + enabled: true, + allowReplay: true + } + } + }) + + const fields = { saveReplay: true } + + await command.update({ videoId: video.id, fields, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with save replay disabled and replay settings', async function () { + const fields = { saveReplay: false, replaySettings: { privacy: VideoPrivacy.INTERNAL } } + + await command.update({ videoId: video.id, fields, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with only replay settings when save replay is disabled', async function () { + const fields = { replaySettings: { privacy: VideoPrivacy.INTERNAL } } + + await command.update({ videoId: video.id, fields, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail to set latency if the server does not allow it', async function () { + const fields = { latencyMode: LiveVideoLatencyMode.HIGH_LATENCY } + + await command.update({ videoId: video.id, fields, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should succeed with the correct params', async function () { + await command.update({ videoId: video.id, fields: { saveReplay: false } }) + await command.update({ videoId: video.uuid, fields: { saveReplay: false } }) + await command.update({ videoId: video.shortUUID, fields: { saveReplay: false } }) + + await command.update({ videoId: video.id, fields: { saveReplay: true, replaySettings: { privacy: VideoPrivacy.PUBLIC } } }) + + }) + + it('Should fail to update replay status if replay is not allowed on the instance', async function () { + await server.config.updateCustomSubConfig({ + newConfig: { + live: { + enabled: true, + allowReplay: false + } + } + }) + + await command.update({ videoId: video.id, fields: { saveReplay: true }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail to update a live if it has already started', async function () { + this.timeout(40000) + + const live = await command.get({ videoId: video.id }) + + const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) + + await command.waitUntilPublished({ videoId: video.id }) + await command.update({ videoId: video.id, fields: {}, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + + await stopFfmpeg(ffmpegCommand) + }) + + it('Should fail to change live privacy if it has already started', async function () { + this.timeout(40000) + + const live = await command.get({ videoId: video.id }) + + const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) + + await command.waitUntilPublished({ videoId: video.id }) + + await server.videos.update({ + id: video.id, + attributes: { privacy: VideoPrivacy.PUBLIC } // Same privacy, it's fine + }) + + await server.videos.update({ + id: video.id, + attributes: { privacy: VideoPrivacy.UNLISTED }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + + await stopFfmpeg(ffmpegCommand) + }) + + it('Should fail to stream twice in the save live', async function () { + this.timeout(40000) + + const live = await command.get({ videoId: video.id }) + + const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) + + await command.waitUntilPublished({ videoId: video.id }) + + await command.runAndTestStreamError({ videoId: video.id, shouldHaveError: true }) + + await stopFfmpeg(ffmpegCommand) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/logs.ts b/packages/tests/src/api/check-params/logs.ts new file mode 100644 index 000000000..629530e30 --- /dev/null +++ b/packages/tests/src/api/check-params/logs.ts @@ -0,0 +1,163 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + makeGetRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test logs API validators', function () { + const path = '/api/v1/server/logs' + let server: PeerTubeServer + let userAccessToken = '' + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(120000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + + const user = { + username: 'user1', + password: 'my super password' + } + await server.users.create({ username: user.username, password: user.password }) + userAccessToken = await server.login.getAccessToken(user) + }) + + describe('When getting logs', function () { + + it('Should fail with a non authenticated user', async function () { + await makeGetRequest({ + url: server.url, + path, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a non admin user', async function () { + await makeGetRequest({ + url: server.url, + path, + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with a missing startDate query', async function () { + await makeGetRequest({ + url: server.url, + path, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with a bad startDate query', async function () { + await makeGetRequest({ + url: server.url, + path, + token: server.accessToken, + query: { startDate: 'toto' }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with a bad endDate query', async function () { + await makeGetRequest({ + url: server.url, + path, + token: server.accessToken, + query: { startDate: new Date().toISOString(), endDate: 'toto' }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with a bad level parameter', async function () { + await makeGetRequest({ + url: server.url, + path, + token: server.accessToken, + query: { startDate: new Date().toISOString(), level: 'toto' }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should succeed with the correct params', async function () { + await makeGetRequest({ + url: server.url, + path, + token: server.accessToken, + query: { startDate: new Date().toISOString() }, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + + describe('When creating client logs', function () { + const base = { + level: 'warn' as 'warn', + message: 'my super message', + url: 'https://example.com/toto' + } + const expectedStatus = HttpStatusCode.BAD_REQUEST_400 + + it('Should fail with an invalid level', async function () { + await server.logs.createLogClient({ payload: { ...base, level: '' as any }, expectedStatus }) + await server.logs.createLogClient({ payload: { ...base, level: undefined }, expectedStatus }) + await server.logs.createLogClient({ payload: { ...base, level: 'toto' as any }, expectedStatus }) + }) + + it('Should fail with an invalid message', async function () { + await server.logs.createLogClient({ payload: { ...base, message: undefined }, expectedStatus }) + await server.logs.createLogClient({ payload: { ...base, message: '' }, expectedStatus }) + await server.logs.createLogClient({ payload: { ...base, message: 'm'.repeat(2500) }, expectedStatus }) + }) + + it('Should fail with an invalid url', async function () { + await server.logs.createLogClient({ payload: { ...base, url: undefined }, expectedStatus }) + await server.logs.createLogClient({ payload: { ...base, url: 'toto' }, expectedStatus }) + }) + + it('Should fail with an invalid stackTrace', async function () { + await server.logs.createLogClient({ payload: { ...base, stackTrace: 's'.repeat(20000) }, expectedStatus }) + }) + + it('Should fail with an invalid userAgent', async function () { + await server.logs.createLogClient({ payload: { ...base, userAgent: 's'.repeat(500) }, expectedStatus }) + }) + + it('Should fail with an invalid meta', async function () { + await server.logs.createLogClient({ payload: { ...base, meta: 's'.repeat(10000) }, expectedStatus }) + }) + + it('Should succeed with the correct params', async function () { + await server.logs.createLogClient({ payload: { ...base, stackTrace: 'stackTrace', meta: '{toto}', userAgent: 'userAgent' } }) + }) + + it('Should rate limit log creation', async function () { + let fail = false + + for (let i = 0; i < 10; i++) { + try { + await server.logs.createLogClient({ token: null, payload: base }) + } catch { + fail = true + } + } + + expect(fail).to.be.true + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/metrics.ts b/packages/tests/src/api/check-params/metrics.ts new file mode 100644 index 000000000..cda854554 --- /dev/null +++ b/packages/tests/src/api/check-params/metrics.ts @@ -0,0 +1,214 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { omit } from '@peertube/peertube-core-utils' +import { HttpStatusCode, PlaybackMetricCreate, VideoResolution } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + makePostBodyRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test metrics API validators', function () { + let server: PeerTubeServer + let videoUUID: string + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(120000) + + server = await createSingleServer(1, { + open_telemetry: { + metrics: { + enabled: true + } + } + }) + + await setAccessTokensToServers([ server ]) + + const { uuid } = await server.videos.quickUpload({ name: 'video' }) + videoUUID = uuid + }) + + describe('When adding playback metrics', function () { + const path = '/api/v1/metrics/playback' + let baseParams: PlaybackMetricCreate + + before(function () { + baseParams = { + playerMode: 'p2p-media-loader', + resolution: VideoResolution.H_1080P, + fps: 30, + resolutionChanges: 1, + errors: 2, + p2pEnabled: true, + downloadedBytesP2P: 0, + downloadedBytesHTTP: 0, + uploadedBytesP2P: 0, + videoId: videoUUID + } + }) + + it('Should fail with an invalid resolution', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { ...baseParams, resolution: 'toto' } + }) + }) + + it('Should fail with an invalid fps', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { ...baseParams, fps: 'toto' } + }) + }) + + it('Should fail with a missing/invalid player mode', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: omit(baseParams, [ 'playerMode' ]) + }) + + await makePostBodyRequest({ + url: server.url, + path, + fields: { ...baseParams, playerMode: 'toto' } + }) + }) + + it('Should fail with an missing/invalid resolution changes', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: omit(baseParams, [ 'resolutionChanges' ]) + }) + + await makePostBodyRequest({ + url: server.url, + path, + fields: { ...baseParams, resolutionChanges: 'toto' } + }) + }) + + it('Should fail with an missing/invalid errors', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: omit(baseParams, [ 'errors' ]) + }) + + await makePostBodyRequest({ + url: server.url, + path, + fields: { ...baseParams, errors: 'toto' } + }) + }) + + it('Should fail with an missing/invalid downloadedBytesP2P', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: omit(baseParams, [ 'downloadedBytesP2P' ]) + }) + + await makePostBodyRequest({ + url: server.url, + path, + fields: { ...baseParams, downloadedBytesP2P: 'toto' } + }) + }) + + it('Should fail with an missing/invalid downloadedBytesHTTP', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: omit(baseParams, [ 'downloadedBytesHTTP' ]) + }) + + await makePostBodyRequest({ + url: server.url, + path, + fields: { ...baseParams, downloadedBytesHTTP: 'toto' } + }) + }) + + it('Should fail with an missing/invalid uploadedBytesP2P', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: omit(baseParams, [ 'uploadedBytesP2P' ]) + }) + + await makePostBodyRequest({ + url: server.url, + path, + fields: { ...baseParams, uploadedBytesP2P: 'toto' } + }) + }) + + it('Should fail with a missing/invalid p2pEnabled', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: omit(baseParams, [ 'p2pEnabled' ]) + }) + + await makePostBodyRequest({ + url: server.url, + path, + fields: { ...baseParams, p2pEnabled: 'toto' } + }) + }) + + it('Should fail with an invalid totalPeers', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { ...baseParams, p2pPeers: 'toto' } + }) + }) + + it('Should fail with a bad video id', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { ...baseParams, videoId: 'toto' } + }) + }) + + it('Should fail with an unknown video', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { ...baseParams, videoId: 42 }, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should succeed with the correct params', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: baseParams, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + + await makePostBodyRequest({ + url: server.url, + path, + fields: { ...baseParams, p2pEnabled: false, totalPeers: 32 }, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/my-user.ts b/packages/tests/src/api/check-params/my-user.ts new file mode 100644 index 000000000..2ef2e242a --- /dev/null +++ b/packages/tests/src/api/check-params/my-user.ts @@ -0,0 +1,492 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@tests/shared/checks.js' +import { MockSmtpServer } from '@tests/shared/mock-servers/index.js' +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import { HttpStatusCode, UserRole, VideoCreateResult } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + makeGetRequest, + makePutBodyRequest, + makeUploadRequest, + PeerTubeServer, + setAccessTokensToServers, + UsersCommand +} from '@peertube/peertube-server-commands' + +describe('Test my user API validators', function () { + const path = '/api/v1/users/' + let userId: number + let rootId: number + let moderatorId: number + let video: VideoCreateResult + let server: PeerTubeServer + let userToken = '' + let moderatorToken = '' + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + { + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + } + + { + const result = await server.users.generate('user1') + userToken = result.token + userId = result.userId + } + + { + const result = await server.users.generate('moderator1', UserRole.MODERATOR) + moderatorToken = result.token + } + + { + const result = await server.users.generate('moderator2', UserRole.MODERATOR) + moderatorId = result.userId + } + + { + video = await server.videos.upload() + } + }) + + describe('When updating my account', function () { + + it('Should fail with an invalid email attribute', async function () { + const fields = { + email: 'blabla' + } + + await makePutBodyRequest({ url: server.url, path: path + 'me', token: server.accessToken, fields }) + }) + + it('Should fail with a too small password', async function () { + const fields = { + currentPassword: 'password', + password: 'bla' + } + + await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) + }) + + it('Should fail with a too long password', async function () { + const fields = { + currentPassword: 'password', + password: 'super'.repeat(61) + } + + await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) + }) + + it('Should fail without the current password', async function () { + const fields = { + currentPassword: 'password', + password: 'super'.repeat(61) + } + + await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) + }) + + it('Should fail with an invalid current password', async function () { + const fields = { + currentPassword: 'my super password fail', + password: 'super'.repeat(61) + } + + await makePutBodyRequest({ + url: server.url, + path: path + 'me', + token: userToken, + fields, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with an invalid NSFW policy attribute', async function () { + const fields = { + nsfwPolicy: 'hello' + } + + await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) + }) + + it('Should fail with an invalid autoPlayVideo attribute', async function () { + const fields = { + autoPlayVideo: -1 + } + + await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) + }) + + it('Should fail with an invalid autoPlayNextVideo attribute', async function () { + const fields = { + autoPlayNextVideo: -1 + } + + await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) + }) + + it('Should fail with an invalid videosHistoryEnabled attribute', async function () { + const fields = { + videosHistoryEnabled: -1 + } + + await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) + }) + + it('Should fail with an non authenticated user', async function () { + const fields = { + currentPassword: 'password', + password: 'my super password' + } + + await makePutBodyRequest({ + url: server.url, + path: path + 'me', + token: 'super token', + fields, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a too long description', async function () { + const fields = { + description: 'super'.repeat(201) + } + + await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) + }) + + it('Should fail with an invalid videoLanguages attribute', async function () { + { + const fields = { + videoLanguages: 'toto' + } + + await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) + } + + { + const languages = [] + for (let i = 0; i < 1000; i++) { + languages.push('fr') + } + + const fields = { + videoLanguages: languages + } + + await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) + } + }) + + it('Should fail with an invalid theme', async function () { + const fields = { theme: 'invalid' } + await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) + }) + + it('Should fail with an unknown theme', async function () { + const fields = { theme: 'peertube-theme-unknown' } + await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) + }) + + it('Should fail with invalid no modal attributes', async function () { + const keys = [ + 'noInstanceConfigWarningModal', + 'noAccountSetupWarningModal', + 'noWelcomeModal' + ] + + for (const key of keys) { + const fields = { + [key]: -1 + } + + await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) + } + }) + + it('Should succeed to change password with the correct params', async function () { + const fields = { + currentPassword: 'password', + password: 'my super password', + nsfwPolicy: 'blur', + autoPlayVideo: false, + email: 'super_email@example.com', + theme: 'default', + noInstanceConfigWarningModal: true, + noWelcomeModal: true, + noAccountSetupWarningModal: true + } + + await makePutBodyRequest({ + url: server.url, + path: path + 'me', + token: userToken, + fields, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + + it('Should succeed without password change with the correct params', async function () { + const fields = { + nsfwPolicy: 'blur', + autoPlayVideo: false + } + + await makePutBodyRequest({ + url: server.url, + path: path + 'me', + token: userToken, + fields, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + describe('When updating my avatar', function () { + it('Should fail without an incorrect input file', async function () { + const fields = {} + const attaches = { + avatarfile: buildAbsoluteFixturePath('video_short.mp4') + } + await makeUploadRequest({ url: server.url, path: path + '/me/avatar/pick', token: server.accessToken, fields, attaches }) + }) + + it('Should fail with a big file', async function () { + const fields = {} + const attaches = { + avatarfile: buildAbsoluteFixturePath('avatar-big.png') + } + await makeUploadRequest({ url: server.url, path: path + '/me/avatar/pick', token: server.accessToken, fields, attaches }) + }) + + it('Should fail with an unauthenticated user', async function () { + const fields = {} + const attaches = { + avatarfile: buildAbsoluteFixturePath('avatar.png') + } + await makeUploadRequest({ + url: server.url, + path: path + '/me/avatar/pick', + fields, + attaches, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should succeed with the correct params', async function () { + const fields = {} + const attaches = { + avatarfile: buildAbsoluteFixturePath('avatar.png') + } + await makeUploadRequest({ + url: server.url, + path: path + '/me/avatar/pick', + token: server.accessToken, + fields, + attaches, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + + describe('When managing my scoped tokens', function () { + + it('Should fail to get my scoped tokens with an non authenticated user', async function () { + await server.users.getMyScopedTokens({ token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail to get my scoped tokens with a bad token', async function () { + await server.users.getMyScopedTokens({ token: 'bad', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + + }) + + it('Should succeed to get my scoped tokens', async function () { + await server.users.getMyScopedTokens() + }) + + it('Should fail to renew my scoped tokens with an non authenticated user', async function () { + await server.users.renewMyScopedTokens({ token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail to renew my scoped tokens with a bad token', async function () { + await server.users.renewMyScopedTokens({ token: 'bad', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should succeed to renew my scoped tokens', async function () { + await server.users.renewMyScopedTokens() + }) + }) + + describe('When getting my information', function () { + it('Should fail with a non authenticated user', async function () { + await server.users.getMyInfo({ token: 'fake_token', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should success with the correct parameters', async function () { + await server.users.getMyInfo({ token: userToken }) + }) + }) + + describe('When getting my video rating', function () { + let command: UsersCommand + + before(function () { + command = server.users + }) + + it('Should fail with a non authenticated user', async function () { + await command.getMyRating({ token: 'fake_token', videoId: video.id, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with an incorrect video uuid', async function () { + await command.getMyRating({ videoId: 'blabla', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with an unknown video', async function () { + await command.getMyRating({ videoId: '4da6fde3-88f7-4d16-b119-108df5630b06', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should succeed with the correct parameters', async function () { + await command.getMyRating({ videoId: video.id }) + await command.getMyRating({ videoId: video.uuid }) + await command.getMyRating({ videoId: video.shortUUID }) + }) + }) + + describe('When retrieving my global ratings', function () { + const path = '/api/v1/accounts/user1/ratings' + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, userToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, userToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, userToken) + }) + + it('Should fail with a unauthenticated user', async function () { + await makeGetRequest({ url: server.url, path, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with a another user', async function () { + await makeGetRequest({ url: server.url, path, token: server.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail with a bad type', async function () { + await makeGetRequest({ + url: server.url, + path, + token: userToken, + query: { rating: 'toto ' }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should succeed with the correct params', async function () { + await makeGetRequest({ url: server.url, path, token: userToken, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('When getting my global followers', function () { + const path = '/api/v1/accounts/user1/followers' + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, userToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, userToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, userToken) + }) + + it('Should fail with a unauthenticated user', async function () { + await makeGetRequest({ url: server.url, path, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with a another user', async function () { + await makeGetRequest({ url: server.url, path, token: server.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should succeed with the correct params', async function () { + await makeGetRequest({ url: server.url, path, token: userToken, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('When blocking/unblocking/removing user', function () { + + it('Should fail with an incorrect id', async function () { + const options = { userId: 'blabla' as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 } + + await server.users.remove(options) + await server.users.banUser({ userId: 'blabla' as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await server.users.unbanUser({ userId: 'blabla' as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with the root user', async function () { + const options = { userId: rootId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 } + + await server.users.remove(options) + await server.users.banUser(options) + await server.users.unbanUser(options) + }) + + it('Should return 404 with a non existing id', async function () { + const options = { userId: 4545454, expectedStatus: HttpStatusCode.NOT_FOUND_404 } + + await server.users.remove(options) + await server.users.banUser(options) + await server.users.unbanUser(options) + }) + + it('Should fail with a non admin user', async function () { + const options = { userId, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 } + + await server.users.remove(options) + await server.users.banUser(options) + await server.users.unbanUser(options) + }) + + it('Should fail on a moderator with a moderator', async function () { + const options = { userId: moderatorId, token: moderatorToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 } + + await server.users.remove(options) + await server.users.banUser(options) + await server.users.unbanUser(options) + }) + + it('Should succeed on a user with a moderator', async function () { + const options = { userId, token: moderatorToken } + + await server.users.banUser(options) + await server.users.unbanUser(options) + }) + }) + + describe('When deleting our account', function () { + + it('Should fail with with the root account', async function () { + await server.users.deleteMe({ expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + }) + + after(async function () { + MockSmtpServer.Instance.kill() + + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/plugins.ts b/packages/tests/src/api/check-params/plugins.ts new file mode 100644 index 000000000..ab2a426fe --- /dev/null +++ b/packages/tests/src/api/check-params/plugins.ts @@ -0,0 +1,490 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@tests/shared/checks.js' +import { HttpStatusCode, PeerTubePlugin, PluginType } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + makeGetRequest, + makePostBodyRequest, + makePutBodyRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test server plugins API validators', function () { + let server: PeerTubeServer + let userAccessToken = null + + const npmPlugin = 'peertube-plugin-hello-world' + const pluginName = 'hello-world' + let npmVersion: string + + const themePlugin = 'peertube-theme-background-red' + const themeName = 'background-red' + let themeVersion: string + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(60000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + + const user = { + username: 'user1', + password: 'password' + } + + await server.users.create({ username: user.username, password: user.password }) + userAccessToken = await server.login.getAccessToken(user) + + { + const res = await server.plugins.install({ npmName: npmPlugin }) + const plugin = res.body as PeerTubePlugin + npmVersion = plugin.version + } + + { + const res = await server.plugins.install({ npmName: themePlugin }) + const plugin = res.body as PeerTubePlugin + themeVersion = plugin.version + } + }) + + describe('With static plugin routes', function () { + it('Should fail with an unknown plugin name/plugin version', async function () { + const paths = [ + '/plugins/' + pluginName + '/0.0.1/auth/fake-auth', + '/plugins/' + pluginName + '/0.0.1/static/images/chocobo.png', + '/plugins/' + pluginName + '/0.0.1/client-scripts/client/common-client-plugin.js', + '/themes/' + themeName + '/0.0.1/static/images/chocobo.png', + '/themes/' + themeName + '/0.0.1/client-scripts/client/video-watch-client-plugin.js', + '/themes/' + themeName + '/0.0.1/css/assets/style1.css' + ] + + for (const p of paths) { + await makeGetRequest({ url: server.url, path: p, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + } + }) + + it('Should fail when requesting a plugin in the theme path', async function () { + await makeGetRequest({ + url: server.url, + path: '/themes/' + pluginName + '/' + npmVersion + '/static/images/chocobo.png', + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail with invalid versions', async function () { + const paths = [ + '/plugins/' + pluginName + '/0.0.1.1/auth/fake-auth', + '/plugins/' + pluginName + '/0.0.1.1/static/images/chocobo.png', + '/plugins/' + pluginName + '/0.1/client-scripts/client/common-client-plugin.js', + '/themes/' + themeName + '/1/static/images/chocobo.png', + '/themes/' + themeName + '/0.0.1000a/client-scripts/client/video-watch-client-plugin.js', + '/themes/' + themeName + '/0.a.1/css/assets/style1.css' + ] + + for (const p of paths) { + await makeGetRequest({ url: server.url, path: p, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + } + }) + + it('Should fail with invalid paths', async function () { + const paths = [ + '/plugins/' + pluginName + '/' + npmVersion + '/static/images/../chocobo.png', + '/plugins/' + pluginName + '/' + npmVersion + '/client-scripts/../client/common-client-plugin.js', + '/themes/' + themeName + '/' + themeVersion + '/static/../images/chocobo.png', + '/themes/' + themeName + '/' + themeVersion + '/client-scripts/client/video-watch-client-plugin.js/..', + '/themes/' + themeName + '/' + themeVersion + '/css/../assets/style1.css' + ] + + for (const p of paths) { + await makeGetRequest({ url: server.url, path: p, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + } + }) + + it('Should fail with an unknown auth name', async function () { + const path = '/plugins/' + pluginName + '/' + npmVersion + '/auth/bad-auth' + + await makeGetRequest({ url: server.url, path, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail with an unknown static file', async function () { + const paths = [ + '/plugins/' + pluginName + '/' + npmVersion + '/static/fake/chocobo.png', + '/plugins/' + pluginName + '/' + npmVersion + '/client-scripts/client/fake.js', + '/themes/' + themeName + '/' + themeVersion + '/static/fake/chocobo.png', + '/themes/' + themeName + '/' + themeVersion + '/client-scripts/client/fake.js' + ] + + for (const p of paths) { + await makeGetRequest({ url: server.url, path: p, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + } + }) + + it('Should fail with an unknown CSS file', async function () { + await makeGetRequest({ + url: server.url, + path: '/themes/' + themeName + '/' + themeVersion + '/css/assets/fake.css', + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should succeed with the correct parameters', async function () { + const paths = [ + '/plugins/' + pluginName + '/' + npmVersion + '/static/images/chocobo.png', + '/plugins/' + pluginName + '/' + npmVersion + '/client-scripts/client/common-client-plugin.js', + '/themes/' + themeName + '/' + themeVersion + '/static/images/chocobo.png', + '/themes/' + themeName + '/' + themeVersion + '/client-scripts/client/video-watch-client-plugin.js', + '/themes/' + themeName + '/' + themeVersion + '/css/assets/style1.css' + ] + + for (const p of paths) { + await makeGetRequest({ url: server.url, path: p, expectedStatus: HttpStatusCode.OK_200 }) + } + + const authPath = '/plugins/' + pluginName + '/' + npmVersion + '/auth/fake-auth' + await makeGetRequest({ url: server.url, path: authPath, expectedStatus: HttpStatusCode.FOUND_302 }) + }) + }) + + describe('When listing available plugins/themes', function () { + const path = '/api/v1/plugins/available' + const baseQuery = { + search: 'super search', + pluginType: PluginType.PLUGIN, + currentPeerTubeEngine: '1.2.3' + } + + it('Should fail with an invalid token', async function () { + await makeGetRequest({ + url: server.url, + path, + token: 'fake_token', + query: baseQuery, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail if the user is not an administrator', async function () { + await makeGetRequest({ + url: server.url, + path, + token: userAccessToken, + query: baseQuery, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an invalid plugin type', async function () { + const query = { ...baseQuery, pluginType: 5 } + + await makeGetRequest({ + url: server.url, + path, + token: server.accessToken, + query + }) + }) + + it('Should fail with an invalid current peertube engine', async function () { + const query = { ...baseQuery, currentPeerTubeEngine: '1.0' } + + await makeGetRequest({ + url: server.url, + path, + token: server.accessToken, + query + }) + }) + + it('Should success with the correct parameters', async function () { + await makeGetRequest({ + url: server.url, + path, + token: server.accessToken, + query: baseQuery, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + + describe('When listing local plugins/themes', function () { + const path = '/api/v1/plugins' + const baseQuery = { + pluginType: PluginType.THEME + } + + it('Should fail with an invalid token', async function () { + await makeGetRequest({ + url: server.url, + path, + token: 'fake_token', + query: baseQuery, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail if the user is not an administrator', async function () { + await makeGetRequest({ + url: server.url, + path, + token: userAccessToken, + query: baseQuery, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an invalid plugin type', async function () { + const query = { ...baseQuery, pluginType: 5 } + + await makeGetRequest({ + url: server.url, + path, + token: server.accessToken, + query + }) + }) + + it('Should success with the correct parameters', async function () { + await makeGetRequest({ + url: server.url, + path, + token: server.accessToken, + query: baseQuery, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + + describe('When getting a plugin or the registered settings or public settings', function () { + const path = '/api/v1/plugins/' + + it('Should fail with an invalid token', async function () { + for (const suffix of [ npmPlugin, `${npmPlugin}/registered-settings` ]) { + await makeGetRequest({ + url: server.url, + path: path + suffix, + token: 'fake_token', + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + } + }) + + it('Should fail if the user is not an administrator', async function () { + for (const suffix of [ npmPlugin, `${npmPlugin}/registered-settings` ]) { + await makeGetRequest({ + url: server.url, + path: path + suffix, + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + } + }) + + it('Should fail with an invalid npm name', async function () { + for (const suffix of [ 'toto', 'toto/registered-settings', 'toto/public-settings' ]) { + await makeGetRequest({ + url: server.url, + path: path + suffix, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + } + + for (const suffix of [ 'peertube-plugin-TOTO', 'peertube-plugin-TOTO/registered-settings', 'peertube-plugin-TOTO/public-settings' ]) { + await makeGetRequest({ + url: server.url, + path: path + suffix, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + } + }) + + it('Should fail with an unknown plugin', async function () { + for (const suffix of [ 'peertube-plugin-toto', 'peertube-plugin-toto/registered-settings', 'peertube-plugin-toto/public-settings' ]) { + await makeGetRequest({ + url: server.url, + path: path + suffix, + token: server.accessToken, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + } + }) + + it('Should succeed with the correct parameters', async function () { + for (const suffix of [ npmPlugin, `${npmPlugin}/registered-settings`, `${npmPlugin}/public-settings` ]) { + await makeGetRequest({ + url: server.url, + path: path + suffix, + token: server.accessToken, + expectedStatus: HttpStatusCode.OK_200 + }) + } + }) + }) + + describe('When updating plugin settings', function () { + const path = '/api/v1/plugins/' + const settings = { setting1: 'value1' } + + it('Should fail with an invalid token', async function () { + await makePutBodyRequest({ + url: server.url, + path: path + npmPlugin + '/settings', + fields: { settings }, + token: 'fake_token', + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail if the user is not an administrator', async function () { + await makePutBodyRequest({ + url: server.url, + path: path + npmPlugin + '/settings', + fields: { settings }, + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with an invalid npm name', async function () { + await makePutBodyRequest({ + url: server.url, + path: path + 'toto/settings', + fields: { settings }, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + + await makePutBodyRequest({ + url: server.url, + path: path + 'peertube-plugin-TOTO/settings', + fields: { settings }, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with an unknown plugin', async function () { + await makePutBodyRequest({ + url: server.url, + path: path + 'peertube-plugin-toto/settings', + fields: { settings }, + token: server.accessToken, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should succeed with the correct parameters', async function () { + await makePutBodyRequest({ + url: server.url, + path: path + npmPlugin + '/settings', + fields: { settings }, + token: server.accessToken, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + describe('When installing/updating/uninstalling a plugin', function () { + const path = '/api/v1/plugins/' + + it('Should fail with an invalid token', async function () { + for (const suffix of [ 'install', 'update', 'uninstall' ]) { + await makePostBodyRequest({ + url: server.url, + path: path + suffix, + fields: { npmName: npmPlugin }, + token: 'fake_token', + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + } + }) + + it('Should fail if the user is not an administrator', async function () { + for (const suffix of [ 'install', 'update', 'uninstall' ]) { + await makePostBodyRequest({ + url: server.url, + path: path + suffix, + fields: { npmName: npmPlugin }, + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + } + }) + + it('Should fail with an invalid npm name', async function () { + for (const suffix of [ 'install', 'update', 'uninstall' ]) { + await makePostBodyRequest({ + url: server.url, + path: path + suffix, + fields: { npmName: 'toto' }, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + } + + for (const suffix of [ 'install', 'update', 'uninstall' ]) { + await makePostBodyRequest({ + url: server.url, + path: path + suffix, + fields: { npmName: 'peertube-plugin-TOTO' }, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + } + }) + + it('Should succeed with the correct parameters', async function () { + const it = [ + { suffix: 'install', status: HttpStatusCode.OK_200 }, + { suffix: 'update', status: HttpStatusCode.OK_200 }, + { suffix: 'uninstall', status: HttpStatusCode.NO_CONTENT_204 } + ] + + for (const obj of it) { + await makePostBodyRequest({ + url: server.url, + path: path + obj.suffix, + fields: { npmName: npmPlugin }, + token: server.accessToken, + expectedStatus: obj.status + }) + } + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/redundancy.ts b/packages/tests/src/api/check-params/redundancy.ts new file mode 100644 index 000000000..16a5d0a3d --- /dev/null +++ b/packages/tests/src/api/check-params/redundancy.ts @@ -0,0 +1,240 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@tests/shared/checks.js' +import { HttpStatusCode, VideoCreateResult } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + makeDeleteRequest, + makeGetRequest, + makePostBodyRequest, + makePutBodyRequest, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test server redundancy API validators', function () { + let servers: PeerTubeServer[] + let userAccessToken = null + let videoIdLocal: number + let videoRemote: VideoCreateResult + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(160000) + + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + await doubleFollow(servers[0], servers[1]) + + const user = { + username: 'user1', + password: 'password' + } + + await servers[0].users.create({ username: user.username, password: user.password }) + userAccessToken = await servers[0].login.getAccessToken(user) + + videoIdLocal = (await servers[0].videos.quickUpload({ name: 'video' })).id + + const remoteUUID = (await servers[1].videos.quickUpload({ name: 'video' })).uuid + + await waitJobs(servers) + + videoRemote = await servers[0].videos.get({ id: remoteUUID }) + }) + + describe('When listing redundancies', function () { + const path = '/api/v1/server/redundancy/videos' + + let url: string + let token: string + + before(function () { + url = servers[0].url + token = servers[0].accessToken + }) + + it('Should fail with an invalid token', async function () { + await makeGetRequest({ url, path, token: 'fake_token', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail if the user is not an administrator', async function () { + await makeGetRequest({ url, path, token: userAccessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(url, path, servers[0].accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(url, path, servers[0].accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(url, path, servers[0].accessToken) + }) + + it('Should fail with a bad target', async function () { + await makeGetRequest({ url, path, token, query: { target: 'bad target' } }) + }) + + it('Should fail without target', async function () { + await makeGetRequest({ url, path, token }) + }) + + it('Should succeed with the correct params', async function () { + await makeGetRequest({ url, path, token, query: { target: 'my-videos' }, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('When manually adding a redundancy', function () { + const path = '/api/v1/server/redundancy/videos' + + let url: string + let token: string + + before(function () { + url = servers[0].url + token = servers[0].accessToken + }) + + it('Should fail with an invalid token', async function () { + await makePostBodyRequest({ url, path, token: 'fake_token', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail if the user is not an administrator', async function () { + await makePostBodyRequest({ url, path, token: userAccessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail without a video id', async function () { + await makePostBodyRequest({ url, path, token }) + }) + + it('Should fail with an incorrect video id', async function () { + await makePostBodyRequest({ url, path, token, fields: { videoId: 'peertube' } }) + }) + + it('Should fail with a not found video id', async function () { + await makePostBodyRequest({ url, path, token, fields: { videoId: 6565 }, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail with a local a video id', async function () { + await makePostBodyRequest({ url, path, token, fields: { videoId: videoIdLocal } }) + }) + + it('Should succeed with the correct params', async function () { + await makePostBodyRequest({ + url, + path, + token, + fields: { videoId: videoRemote.shortUUID }, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + + it('Should fail if the video is already duplicated', async function () { + this.timeout(30000) + + await waitJobs(servers) + + await makePostBodyRequest({ + url, + path, + token, + fields: { videoId: videoRemote.uuid }, + expectedStatus: HttpStatusCode.CONFLICT_409 + }) + }) + }) + + describe('When manually removing a redundancy', function () { + const path = '/api/v1/server/redundancy/videos/' + + let url: string + let token: string + + before(function () { + url = servers[0].url + token = servers[0].accessToken + }) + + it('Should fail with an invalid token', async function () { + await makeDeleteRequest({ url, path: path + '1', token: 'fake_token', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail if the user is not an administrator', async function () { + await makeDeleteRequest({ url, path: path + '1', token: userAccessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail with an incorrect video id', async function () { + await makeDeleteRequest({ url, path: path + 'toto', token }) + }) + + it('Should fail with a not found video redundancy', async function () { + await makeDeleteRequest({ url, path: path + '454545', token, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + }) + + describe('When updating server redundancy', function () { + const path = '/api/v1/server/redundancy' + + it('Should fail with an invalid token', async function () { + await makePutBodyRequest({ + url: servers[0].url, + path: path + '/' + servers[1].host, + fields: { redundancyAllowed: true }, + token: 'fake_token', + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail if the user is not an administrator', async function () { + await makePutBodyRequest({ + url: servers[0].url, + path: path + '/' + servers[1].host, + fields: { redundancyAllowed: true }, + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail if we do not follow this server', async function () { + await makePutBodyRequest({ + url: servers[0].url, + path: path + '/example.com', + fields: { redundancyAllowed: true }, + token: servers[0].accessToken, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail without de redundancyAllowed param', async function () { + await makePutBodyRequest({ + url: servers[0].url, + path: path + '/' + servers[1].host, + fields: { blabla: true }, + token: servers[0].accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should succeed with the correct parameters', async function () { + await makePutBodyRequest({ + url: servers[0].url, + path: path + '/' + servers[1].host, + fields: { redundancyAllowed: true }, + token: servers[0].accessToken, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/check-params/registrations.ts b/packages/tests/src/api/check-params/registrations.ts new file mode 100644 index 000000000..e4e46da2a --- /dev/null +++ b/packages/tests/src/api/check-params/registrations.ts @@ -0,0 +1,446 @@ +import { omit } from '@peertube/peertube-core-utils' +import { HttpStatusCode, HttpStatusCodeType, UserRole } from '@peertube/peertube-models' +import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@tests/shared/checks.js' +import { + cleanupTests, + createSingleServer, + makePostBodyRequest, + PeerTubeServer, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultChannelAvatar +} from '@peertube/peertube-server-commands' + +describe('Test registrations API validators', function () { + let server: PeerTubeServer + let userToken: string + let moderatorToken: string + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + await setDefaultAccountAvatar([ server ]) + await setDefaultChannelAvatar([ server ]) + + await server.config.enableSignup(false); + + ({ token: moderatorToken } = await server.users.generate('moderator', UserRole.MODERATOR)); + ({ token: userToken } = await server.users.generate('user', UserRole.USER)) + }) + + describe('Register', function () { + const registrationPath = '/api/v1/users/register' + const registrationRequestPath = '/api/v1/users/registrations/request' + + const baseCorrectParams = { + username: 'user3', + displayName: 'super user', + email: 'test3@example.com', + password: 'my super password', + registrationReason: 'my super registration reason' + } + + describe('When registering a new user or requesting user registration', function () { + + async function check (fields: any, expectedStatus: HttpStatusCodeType = HttpStatusCode.BAD_REQUEST_400) { + await server.config.enableSignup(false) + await makePostBodyRequest({ url: server.url, path: registrationPath, fields, expectedStatus }) + + await server.config.enableSignup(true) + await makePostBodyRequest({ url: server.url, path: registrationRequestPath, fields, expectedStatus }) + } + + it('Should fail with a too small username', async function () { + const fields = { ...baseCorrectParams, username: '' } + + await check(fields) + }) + + it('Should fail with a too long username', async function () { + const fields = { ...baseCorrectParams, username: 'super'.repeat(50) } + + await check(fields) + }) + + it('Should fail with an incorrect username', async function () { + const fields = { ...baseCorrectParams, username: 'my username' } + + await check(fields) + }) + + it('Should fail with a missing email', async function () { + const fields = omit(baseCorrectParams, [ 'email' ]) + + await check(fields) + }) + + it('Should fail with an invalid email', async function () { + const fields = { ...baseCorrectParams, email: 'test_example.com' } + + await check(fields) + }) + + it('Should fail with a too small password', async function () { + const fields = { ...baseCorrectParams, password: 'bla' } + + await check(fields) + }) + + it('Should fail with a too long password', async function () { + const fields = { ...baseCorrectParams, password: 'super'.repeat(61) } + + await check(fields) + }) + + it('Should fail if we register a user with the same username', async function () { + const fields = { ...baseCorrectParams, username: 'root' } + + await check(fields, HttpStatusCode.CONFLICT_409) + }) + + it('Should fail with a "peertube" username', async function () { + const fields = { ...baseCorrectParams, username: 'peertube' } + + await check(fields, HttpStatusCode.CONFLICT_409) + }) + + it('Should fail if we register a user with the same email', async function () { + const fields = { ...baseCorrectParams, email: 'admin' + server.internalServerNumber + '@example.com' } + + await check(fields, HttpStatusCode.CONFLICT_409) + }) + + it('Should fail with a bad display name', async function () { + const fields = { ...baseCorrectParams, displayName: 'a'.repeat(150) } + + await check(fields) + }) + + it('Should fail with a bad channel name', async function () { + const fields = { ...baseCorrectParams, channel: { name: '[]azf', displayName: 'toto' } } + + await check(fields) + }) + + it('Should fail with a bad channel display name', async function () { + const fields = { ...baseCorrectParams, channel: { name: 'toto', displayName: '' } } + + await check(fields) + }) + + it('Should fail with a channel name that is the same as username', async function () { + const source = { username: 'super_user', channel: { name: 'super_user', displayName: 'display name' } } + const fields = { ...baseCorrectParams, ...source } + + await check(fields) + }) + + it('Should fail with an existing channel', async function () { + const attributes = { name: 'existing_channel', displayName: 'hello', description: 'super description' } + await server.channels.create({ attributes }) + + const fields = { ...baseCorrectParams, channel: { name: 'existing_channel', displayName: 'toto' } } + + await check(fields, HttpStatusCode.CONFLICT_409) + }) + + it('Should fail on a server with registration disabled', async function () { + this.timeout(60000) + + await server.config.updateExistingSubConfig({ + newConfig: { + signup: { + enabled: false + } + } + }) + + await server.registrations.register({ username: 'user4', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await server.registrations.requestRegistration({ + username: 'user4', + registrationReason: 'reason', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail if the user limit is reached', async function () { + this.timeout(60000) + + const { total } = await server.users.list() + + await server.config.enableSignup(false, total) + await server.registrations.register({ username: 'user42', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + + await server.config.enableSignup(true, total) + await server.registrations.requestRegistration({ + username: 'user42', + registrationReason: 'reason', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should succeed if the user limit is not reached', async function () { + this.timeout(60000) + + const { total } = await server.users.list() + + await server.config.enableSignup(false, total + 1) + await server.registrations.register({ username: 'user43', expectedStatus: HttpStatusCode.NO_CONTENT_204 }) + + await server.config.enableSignup(true, total + 2) + await server.registrations.requestRegistration({ + username: 'user44', + registrationReason: 'reason', + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + + describe('On direct registration', function () { + + it('Should succeed with the correct params', async function () { + await server.config.enableSignup(false) + + const fields = { + username: 'user_direct_1', + displayName: 'super user direct 1', + email: 'user_direct_1@example.com', + password: 'my super password', + channel: { name: 'super_user_direct_1_channel', displayName: 'super user direct 1 channel' } + } + + await makePostBodyRequest({ url: server.url, path: registrationPath, fields, expectedStatus: HttpStatusCode.NO_CONTENT_204 }) + }) + + it('Should fail if the instance requires approval', async function () { + this.timeout(60000) + + await server.config.enableSignup(true) + await server.registrations.register({ username: 'user42', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + }) + + describe('On registration request', function () { + + before(async function () { + this.timeout(60000) + + await server.config.enableSignup(true) + }) + + it('Should fail with an invalid registration reason', async function () { + for (const registrationReason of [ '', 't', 't'.repeat(5000) ]) { + await server.registrations.requestRegistration({ + username: 'user_request_1', + registrationReason, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + } + }) + + it('Should succeed with the correct params', async function () { + await server.registrations.requestRegistration({ + username: 'user_request_2', + registrationReason: 'tt', + channel: { + displayName: 'my user request 2 channel', + name: 'user_request_2_channel' + } + }) + }) + + it('Should fail if the username is already awaiting registration approval', async function () { + await server.registrations.requestRegistration({ + username: 'user_request_2', + registrationReason: 'tt', + channel: { + displayName: 'my user request 42 channel', + name: 'user_request_42_channel' + }, + expectedStatus: HttpStatusCode.CONFLICT_409 + }) + }) + + it('Should fail if the email is already awaiting registration approval', async function () { + await server.registrations.requestRegistration({ + username: 'user42', + email: 'user_request_2@example.com', + registrationReason: 'tt', + channel: { + displayName: 'my user request 42 channel', + name: 'user_request_42_channel' + }, + expectedStatus: HttpStatusCode.CONFLICT_409 + }) + }) + + it('Should fail if the channel is already awaiting registration approval', async function () { + await server.registrations.requestRegistration({ + username: 'user42', + registrationReason: 'tt', + channel: { + displayName: 'my user request 2 channel', + name: 'user_request_2_channel' + }, + expectedStatus: HttpStatusCode.CONFLICT_409 + }) + }) + + it('Should fail if the instance does not require approval', async function () { + this.timeout(60000) + + await server.config.enableSignup(false) + + await server.registrations.requestRegistration({ + username: 'user42', + registrationReason: 'toto', + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + }) + }) + + describe('Registrations accept/reject', function () { + let id1: number + let id2: number + + before(async function () { + this.timeout(60000) + + await server.config.enableSignup(true); + + ({ id: id1 } = await server.registrations.requestRegistration({ username: 'request_2', registrationReason: 'toto' })); + ({ id: id2 } = await server.registrations.requestRegistration({ username: 'request_3', registrationReason: 'toto' })) + }) + + it('Should fail to accept/reject registration without token', async function () { + const options = { id: id1, moderationResponse: 'tt', token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 } + await server.registrations.accept(options) + await server.registrations.reject(options) + }) + + it('Should fail to accept/reject registration with a non moderator user', async function () { + const options = { id: id1, moderationResponse: 'tt', token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 } + await server.registrations.accept(options) + await server.registrations.reject(options) + }) + + it('Should fail to accept/reject registration with a bad registration id', async function () { + { + const options = { id: 't' as any, moderationResponse: 'tt', token: moderatorToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 } + await server.registrations.accept(options) + await server.registrations.reject(options) + } + + { + const options = { id: 42, moderationResponse: 'tt', token: moderatorToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 } + await server.registrations.accept(options) + await server.registrations.reject(options) + } + }) + + it('Should fail to accept/reject registration with a bad moderation resposne', async function () { + for (const moderationResponse of [ '', 't', 't'.repeat(5000) ]) { + const options = { id: id1, moderationResponse, token: moderatorToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 } + await server.registrations.accept(options) + await server.registrations.reject(options) + } + }) + + it('Should succeed to accept a registration', async function () { + await server.registrations.accept({ id: id1, moderationResponse: 'tt', token: moderatorToken }) + }) + + it('Should succeed to reject a registration', async function () { + await server.registrations.reject({ id: id2, moderationResponse: 'tt', token: moderatorToken }) + }) + + it('Should fail to accept/reject a registration that was already accepted/rejected', async function () { + for (const id of [ id1, id2 ]) { + const options = { id, moderationResponse: 'tt', token: moderatorToken, expectedStatus: HttpStatusCode.CONFLICT_409 } + await server.registrations.accept(options) + await server.registrations.reject(options) + } + }) + }) + + describe('Registrations deletion', function () { + let id1: number + let id2: number + let id3: number + + before(async function () { + ({ id: id1 } = await server.registrations.requestRegistration({ username: 'request_4', registrationReason: 'toto' })); + ({ id: id2 } = await server.registrations.requestRegistration({ username: 'request_5', registrationReason: 'toto' })); + ({ id: id3 } = await server.registrations.requestRegistration({ username: 'request_6', registrationReason: 'toto' })) + + await server.registrations.accept({ id: id2, moderationResponse: 'tt' }) + await server.registrations.reject({ id: id3, moderationResponse: 'tt' }) + }) + + it('Should fail to delete registration without token', async function () { + await server.registrations.delete({ id: id1, token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail to delete registration with a non moderator user', async function () { + await server.registrations.delete({ id: id1, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail to delete registration with a bad registration id', async function () { + await server.registrations.delete({ id: 't' as any, token: moderatorToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await server.registrations.delete({ id: 42, token: moderatorToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should succeed with the correct params', async function () { + await server.registrations.delete({ id: id1, token: moderatorToken }) + await server.registrations.delete({ id: id2, token: moderatorToken }) + await server.registrations.delete({ id: id3, token: moderatorToken }) + }) + }) + + describe('Listing registrations', function () { + const path = '/api/v1/users/registrations' + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a non authenticated user', async function () { + await server.registrations.list({ + token: null, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a non admin user', async function () { + await server.registrations.list({ + token: userToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should succeed with the correct params', async function () { + await server.registrations.list({ + token: moderatorToken, + search: 'toto' + }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/runners.ts b/packages/tests/src/api/check-params/runners.ts new file mode 100644 index 000000000..dd2d2f0a1 --- /dev/null +++ b/packages/tests/src/api/check-params/runners.ts @@ -0,0 +1,911 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ +import { basename } from 'path' +import { + HttpStatusCode, + HttpStatusCodeType, + isVideoStudioTaskIntro, + RunnerJob, + RunnerJobState, + RunnerJobStudioTranscodingPayload, + RunnerJobSuccessPayload, + RunnerJobUpdatePayload, + VideoPrivacy, + VideoStudioTaskIntro +} from '@peertube/peertube-models' +import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@tests/shared/checks.js' +import { + cleanupTests, + createSingleServer, + makePostBodyRequest, + PeerTubeServer, + sendRTMPStream, + setAccessTokensToServers, + setDefaultVideoChannel, + stopFfmpeg, + VideoStudioCommand, + waitJobs +} from '@peertube/peertube-server-commands' + +const badUUID = '910ec12a-d9e6-458b-a274-0abb655f9464' + +describe('Test managing runners', function () { + let server: PeerTubeServer + + let userToken: string + + let registrationTokenId: number + let registrationToken: string + + let runnerToken: string + let runnerToken2: string + + let completedJobToken: string + let completedJobUUID: string + + let cancelledJobToken: string + let cancelledJobUUID: string + + before(async function () { + this.timeout(120000) + + const config = { + rates_limit: { + api: { + max: 5000 + } + } + } + + server = await createSingleServer(1, config) + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + userToken = await server.users.generateUserAndToken('user1') + + const { data } = await server.runnerRegistrationTokens.list() + registrationToken = data[0].registrationToken + registrationTokenId = data[0].id + + await server.config.enableTranscoding({ hls: true, webVideo: true }) + await server.config.enableStudio() + await server.config.enableRemoteTranscoding() + await server.config.enableRemoteStudio() + + runnerToken = await server.runners.autoRegisterRunner() + runnerToken2 = await server.runners.autoRegisterRunner() + + { + await server.videos.quickUpload({ name: 'video 1' }) + await server.videos.quickUpload({ name: 'video 2' }) + + await waitJobs([ server ]) + + { + const job = await server.runnerJobs.autoProcessWebVideoJob(runnerToken) + completedJobToken = job.jobToken + completedJobUUID = job.uuid + } + + { + const { job } = await server.runnerJobs.autoAccept({ runnerToken }) + cancelledJobToken = job.jobToken + cancelledJobUUID = job.uuid + await server.runnerJobs.cancelByAdmin({ jobUUID: cancelledJobUUID }) + } + } + }) + + describe('Managing runner registration tokens', function () { + + describe('Common', function () { + + it('Should fail to generate, list or delete runner registration token without oauth token', async function () { + const expectedStatus = HttpStatusCode.UNAUTHORIZED_401 + + await server.runnerRegistrationTokens.generate({ token: null, expectedStatus }) + await server.runnerRegistrationTokens.list({ token: null, expectedStatus }) + await server.runnerRegistrationTokens.delete({ token: null, id: registrationTokenId, expectedStatus }) + }) + + it('Should fail to generate, list or delete runner registration token without admin rights', async function () { + const expectedStatus = HttpStatusCode.FORBIDDEN_403 + + await server.runnerRegistrationTokens.generate({ token: userToken, expectedStatus }) + await server.runnerRegistrationTokens.list({ token: userToken, expectedStatus }) + await server.runnerRegistrationTokens.delete({ token: userToken, id: registrationTokenId, expectedStatus }) + }) + }) + + describe('Delete', function () { + + it('Should fail to delete with a bad id', async function () { + await server.runnerRegistrationTokens.delete({ id: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + }) + + describe('List', function () { + const path = '/api/v1/runners/registration-tokens' + + it('Should fail to list with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, server.accessToken) + }) + + it('Should fail to list with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, server.accessToken) + }) + + it('Should fail to list with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, server.accessToken) + }) + + it('Should succeed to list with the correct params', async function () { + await server.runnerRegistrationTokens.list({ start: 0, count: 5, sort: '-createdAt' }) + }) + }) + }) + + describe('Managing runners', function () { + let toDeleteId: number + + describe('Register', function () { + const name = 'runner name' + + it('Should fail with a bad registration token', async function () { + const expectedStatus = HttpStatusCode.BAD_REQUEST_400 + + await server.runners.register({ name, registrationToken: 'a'.repeat(4000), expectedStatus }) + await server.runners.register({ name, registrationToken: null, expectedStatus }) + }) + + it('Should fail with an unknown registration token', async function () { + await server.runners.register({ name, registrationToken: 'aaa', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail with a bad name', async function () { + const expectedStatus = HttpStatusCode.BAD_REQUEST_400 + + await server.runners.register({ name: '', registrationToken, expectedStatus }) + await server.runners.register({ name: 'a'.repeat(200), registrationToken, expectedStatus }) + }) + + it('Should fail with an invalid description', async function () { + const expectedStatus = HttpStatusCode.BAD_REQUEST_400 + + await server.runners.register({ name, description: '', registrationToken, expectedStatus }) + await server.runners.register({ name, description: 'a'.repeat(5000), registrationToken, expectedStatus }) + }) + + it('Should succeed with the correct params', async function () { + const { id } = await server.runners.register({ name, description: 'super description', registrationToken }) + + toDeleteId = id + }) + + it('Should fail with the same runner name', async function () { + await server.runners.register({ + name, + description: 'super description', + registrationToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + }) + + describe('Delete', function () { + + it('Should fail without oauth token', async function () { + await server.runners.delete({ token: null, id: toDeleteId, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail without admin rights', async function () { + await server.runners.delete({ token: userToken, id: toDeleteId, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail with a bad id', async function () { + await server.runners.delete({ id: 'hi' as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with an unknown id', async function () { + await server.runners.delete({ id: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should succeed with the correct params', async function () { + await server.runners.delete({ id: toDeleteId }) + }) + }) + + describe('List', function () { + const path = '/api/v1/runners' + + it('Should fail without oauth token', async function () { + await server.runners.list({ token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail without admin rights', async function () { + await server.runners.list({ token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail to list with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, server.accessToken) + }) + + it('Should fail to list with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, server.accessToken) + }) + + it('Should fail to list with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an invalid state', async function () { + await server.runners.list({ start: 0, count: 5, sort: '-createdAt' }) + }) + + it('Should succeed to list with the correct params', async function () { + await server.runners.list({ start: 0, count: 5, sort: '-createdAt' }) + }) + }) + + }) + + describe('Runner jobs by admin', function () { + + describe('Cancel', function () { + let jobUUID: string + + before(async function () { + this.timeout(60000) + + await server.videos.quickUpload({ name: 'video' }) + await waitJobs([ server ]) + + const { availableJobs } = await server.runnerJobs.request({ runnerToken }) + jobUUID = availableJobs[0].uuid + }) + + it('Should fail without oauth token', async function () { + await server.runnerJobs.cancelByAdmin({ token: null, jobUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail without admin rights', async function () { + await server.runnerJobs.cancelByAdmin({ token: userToken, jobUUID, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail with a bad job uuid', async function () { + await server.runnerJobs.cancelByAdmin({ jobUUID: 'hello', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with an unknown job uuid', async function () { + const jobUUID = badUUID + await server.runnerJobs.cancelByAdmin({ jobUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail with an already cancelled job', async function () { + await server.runnerJobs.cancelByAdmin({ jobUUID: cancelledJobUUID, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should succeed with the correct params', async function () { + await server.runnerJobs.cancelByAdmin({ jobUUID }) + }) + }) + + describe('List', function () { + const path = '/api/v1/runners/jobs' + + it('Should fail without oauth token', async function () { + await server.runnerJobs.list({ token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail without admin rights', async function () { + await server.runnerJobs.list({ token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail to list with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, server.accessToken) + }) + + it('Should fail to list with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, server.accessToken) + }) + + it('Should fail to list with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an invalid state', async function () { + await server.runnerJobs.list({ start: 0, count: 5, sort: '-createdAt', stateOneOf: 42 as any }) + await server.runnerJobs.list({ start: 0, count: 5, sort: '-createdAt', stateOneOf: [ 42 ] as any }) + }) + + it('Should succeed with the correct params', async function () { + await server.runnerJobs.list({ start: 0, count: 5, sort: '-createdAt', stateOneOf: [ RunnerJobState.COMPLETED ] }) + }) + }) + + describe('Delete', function () { + let jobUUID: string + + before(async function () { + this.timeout(60000) + + await server.videos.quickUpload({ name: 'video' }) + await waitJobs([ server ]) + + const { availableJobs } = await server.runnerJobs.request({ runnerToken }) + jobUUID = availableJobs[0].uuid + }) + + it('Should fail without oauth token', async function () { + await server.runnerJobs.deleteByAdmin({ token: null, jobUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail without admin rights', async function () { + await server.runnerJobs.deleteByAdmin({ token: userToken, jobUUID, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail with a bad job uuid', async function () { + await server.runnerJobs.deleteByAdmin({ jobUUID: 'hello', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with an unknown job uuid', async function () { + const jobUUID = badUUID + await server.runnerJobs.deleteByAdmin({ jobUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should succeed with the correct params', async function () { + await server.runnerJobs.deleteByAdmin({ jobUUID }) + }) + }) + + }) + + describe('Runner jobs by runners', function () { + let jobUUID: string + let jobToken: string + let videoUUID: string + + let jobUUID2: string + let jobToken2: string + + let videoUUID2: string + + let pendingUUID: string + + let videoStudioUUID: string + let studioFile: string + + let liveAcceptedJob: RunnerJob & { jobToken: string } + let studioAcceptedJob: RunnerJob & { jobToken: string } + + async function fetchVideoInputFiles (options: { + jobUUID: string + videoUUID: string + runnerToken: string + jobToken: string + expectedStatus: HttpStatusCodeType + }) { + const { jobUUID, expectedStatus, videoUUID, runnerToken, jobToken } = options + + const basePath = '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + const paths = [ `${basePath}/max-quality`, `${basePath}/previews/max-quality` ] + + for (const path of paths) { + await makePostBodyRequest({ url: server.url, path, fields: { runnerToken, jobToken }, expectedStatus }) + } + } + + async function fetchStudioFiles (options: { + jobUUID: string + videoUUID: string + runnerToken: string + jobToken: string + studioFile?: string + expectedStatus: HttpStatusCodeType + }) { + const { jobUUID, expectedStatus, videoUUID, runnerToken, jobToken, studioFile } = options + + const path = `/api/v1/runners/jobs/${jobUUID}/files/videos/${videoUUID}/studio/task-files/${studioFile}` + + await makePostBodyRequest({ url: server.url, path, fields: { runnerToken, jobToken }, expectedStatus }) + } + + before(async function () { + this.timeout(120000) + + { + await server.runnerJobs.cancelAllJobs({ state: RunnerJobState.PENDING }) + } + + { + const { uuid } = await server.videos.quickUpload({ name: 'video' }) + videoUUID = uuid + + await waitJobs([ server ]) + + const { job } = await server.runnerJobs.autoAccept({ runnerToken }) + jobUUID = job.uuid + jobToken = job.jobToken + } + + { + const { uuid } = await server.videos.quickUpload({ name: 'video' }) + videoUUID2 = uuid + + await waitJobs([ server ]) + + const { job } = await server.runnerJobs.autoAccept({ runnerToken: runnerToken2 }) + jobUUID2 = job.uuid + jobToken2 = job.jobToken + } + + { + await server.videos.quickUpload({ name: 'video' }) + await waitJobs([ server ]) + + const { availableJobs } = await server.runnerJobs.request({ runnerToken }) + pendingUUID = availableJobs[0].uuid + } + + { + await server.config.disableTranscoding() + + const { uuid } = await server.videos.quickUpload({ name: 'video studio' }) + videoStudioUUID = uuid + + await server.config.enableTranscoding({ hls: true, webVideo: true }) + await server.config.enableStudio() + + await server.videoStudio.createEditionTasks({ + videoId: videoStudioUUID, + tasks: VideoStudioCommand.getComplexTask() + }) + + const { job } = await server.runnerJobs.autoAccept({ runnerToken, type: 'video-studio-transcoding' }) + studioAcceptedJob = job + + const tasks = (job.payload as RunnerJobStudioTranscodingPayload).tasks + const fileUrl = (tasks.find(t => isVideoStudioTaskIntro(t)) as VideoStudioTaskIntro).options.file as string + studioFile = basename(fileUrl) + } + + { + await server.config.enableLive({ + allowReplay: false, + resolutions: 'max', + transcoding: true + }) + + const { live } = await server.live.quickCreate({ permanentLive: true, saveReplay: false, privacy: VideoPrivacy.PUBLIC }) + + const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) + await waitJobs([ server ]) + + await server.runnerJobs.requestLiveJob(runnerToken) + + const { job } = await server.runnerJobs.autoAccept({ runnerToken, type: 'live-rtmp-hls-transcoding' }) + liveAcceptedJob = job + + await stopFfmpeg(ffmpegCommand) + } + }) + + describe('Common runner tokens validations', function () { + + async function testEndpoints (options: { + jobUUID: string + runnerToken: string + jobToken: string + expectedStatus: HttpStatusCodeType + }) { + await server.runnerJobs.abort({ ...options, reason: 'reason' }) + await server.runnerJobs.update({ ...options }) + await server.runnerJobs.error({ ...options, message: 'message' }) + await server.runnerJobs.success({ ...options, payload: { videoFile: 'video_short.mp4' } }) + } + + it('Should fail with an invalid job uuid', async function () { + const options = { jobUUID: 'a', runnerToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 } + + await testEndpoints({ ...options, jobToken }) + await fetchVideoInputFiles({ ...options, videoUUID, jobToken }) + await fetchStudioFiles({ ...options, videoUUID, jobToken: studioAcceptedJob.jobToken, studioFile }) + }) + + it('Should fail with an unknown job uuid', async function () { + const options = { jobUUID: badUUID, runnerToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 } + + await testEndpoints({ ...options, jobToken }) + await fetchVideoInputFiles({ ...options, videoUUID, jobToken }) + await fetchStudioFiles({ ...options, jobToken: studioAcceptedJob.jobToken, videoUUID, studioFile }) + }) + + it('Should fail with an invalid runner token', async function () { + const options = { runnerToken: '', expectedStatus: HttpStatusCode.BAD_REQUEST_400 } + + await testEndpoints({ ...options, jobUUID, jobToken }) + await fetchVideoInputFiles({ ...options, jobUUID, videoUUID, jobToken }) + await fetchStudioFiles({ + ...options, + jobToken: studioAcceptedJob.jobToken, + jobUUID: studioAcceptedJob.uuid, + videoUUID: videoStudioUUID, + studioFile + }) + }) + + it('Should fail with an unknown runner token', async function () { + const options = { runnerToken: badUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 } + + await testEndpoints({ ...options, jobUUID, jobToken }) + await fetchVideoInputFiles({ ...options, jobUUID, videoUUID, jobToken }) + await fetchStudioFiles({ + ...options, + jobToken: studioAcceptedJob.jobToken, + jobUUID: studioAcceptedJob.uuid, + videoUUID: videoStudioUUID, + studioFile + }) + }) + + it('Should fail with an invalid job token job uuid', async function () { + const options = { runnerToken, jobToken: '', expectedStatus: HttpStatusCode.BAD_REQUEST_400 } + + await testEndpoints({ ...options, jobUUID }) + await fetchVideoInputFiles({ ...options, jobUUID, videoUUID }) + await fetchStudioFiles({ ...options, jobUUID: studioAcceptedJob.uuid, videoUUID: videoStudioUUID, studioFile }) + }) + + it('Should fail with an unknown job token job uuid', async function () { + const options = { runnerToken, jobToken: badUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 } + + await testEndpoints({ ...options, jobUUID }) + await fetchVideoInputFiles({ ...options, jobUUID, videoUUID }) + await fetchStudioFiles({ ...options, jobUUID: studioAcceptedJob.uuid, videoUUID: videoStudioUUID, studioFile }) + }) + + it('Should fail with a runner token not associated to this job', async function () { + const options = { runnerToken: runnerToken2, expectedStatus: HttpStatusCode.NOT_FOUND_404 } + + await testEndpoints({ ...options, jobUUID, jobToken }) + await fetchVideoInputFiles({ ...options, jobUUID, videoUUID, jobToken }) + await fetchStudioFiles({ + ...options, + jobToken: studioAcceptedJob.jobToken, + jobUUID: studioAcceptedJob.uuid, + videoUUID: videoStudioUUID, + studioFile + }) + }) + + it('Should fail with a job uuid not associated to the job token', async function () { + { + const options = { jobUUID: jobUUID2, runnerToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 } + + await testEndpoints({ ...options, jobToken }) + await fetchVideoInputFiles({ ...options, jobToken, videoUUID }) + await fetchStudioFiles({ ...options, jobToken: studioAcceptedJob.jobToken, videoUUID: videoStudioUUID, studioFile }) + } + + { + const options = { runnerToken, jobToken: jobToken2, expectedStatus: HttpStatusCode.NOT_FOUND_404 } + + await testEndpoints({ ...options, jobUUID }) + await fetchVideoInputFiles({ ...options, jobUUID, videoUUID }) + await fetchStudioFiles({ ...options, jobUUID: studioAcceptedJob.uuid, videoUUID: videoStudioUUID, studioFile }) + } + }) + }) + + describe('Unregister', function () { + + it('Should fail without a runner token', async function () { + await server.runners.unregister({ runnerToken: null, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with a bad a runner token', async function () { + await server.runners.unregister({ runnerToken: '', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with an unknown runner token', async function () { + await server.runners.unregister({ runnerToken: badUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + }) + + describe('Request', function () { + + it('Should fail without a runner token', async function () { + await server.runnerJobs.request({ runnerToken: null, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with a bad a runner token', async function () { + await server.runnerJobs.request({ runnerToken: '', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with an unknown runner token', async function () { + await server.runnerJobs.request({ runnerToken: badUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + }) + + describe('Accept', function () { + + it('Should fail with a bad a job uuid', async function () { + await server.runnerJobs.accept({ jobUUID: '', runnerToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with an unknown job uuid', async function () { + await server.runnerJobs.accept({ jobUUID: badUUID, runnerToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail with a job not in pending state', async function () { + await server.runnerJobs.accept({ jobUUID: completedJobUUID, runnerToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await server.runnerJobs.accept({ jobUUID: cancelledJobUUID, runnerToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail without a runner token', async function () { + await server.runnerJobs.accept({ jobUUID: pendingUUID, runnerToken: null, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with a bad a runner token', async function () { + await server.runnerJobs.accept({ jobUUID: pendingUUID, runnerToken: '', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with an unknown runner token', async function () { + await server.runnerJobs.accept({ jobUUID: pendingUUID, runnerToken: badUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + }) + + describe('Abort', function () { + + it('Should fail without a reason', async function () { + await server.runnerJobs.abort({ jobUUID, jobToken, runnerToken, reason: null, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with a bad reason', async function () { + const reason = 'reason'.repeat(5000) + await server.runnerJobs.abort({ jobUUID, jobToken, runnerToken, reason, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with a job not in processing state', async function () { + await server.runnerJobs.abort({ + jobUUID: completedJobUUID, + jobToken: completedJobToken, + runnerToken, + reason: 'reason', + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + }) + + describe('Update', function () { + + describe('Common', function () { + + it('Should fail with an invalid progress', async function () { + await server.runnerJobs.update({ jobUUID, jobToken, runnerToken, progress: 101, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with a job not in processing state', async function () { + await server.runnerJobs.update({ + jobUUID: cancelledJobUUID, + jobToken: cancelledJobToken, + runnerToken, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + }) + + describe('Live RTMP to HLS', function () { + const base: RunnerJobUpdatePayload = { + masterPlaylistFile: 'live/master.m3u8', + resolutionPlaylistFilename: '0.m3u8', + resolutionPlaylistFile: 'live/1.m3u8', + type: 'add-chunk', + videoChunkFile: 'live/1-000069.ts', + videoChunkFilename: '1-000068.ts' + } + + function testUpdate (payload: RunnerJobUpdatePayload) { + return server.runnerJobs.update({ + jobUUID: liveAcceptedJob.uuid, + jobToken: liveAcceptedJob.jobToken, + payload, + runnerToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + } + + it('Should fail with an invalid resolutionPlaylistFilename', async function () { + await testUpdate({ ...base, resolutionPlaylistFilename: undefined }) + await testUpdate({ ...base, resolutionPlaylistFilename: 'coucou/hello' }) + await testUpdate({ ...base, resolutionPlaylistFilename: 'hello' }) + }) + + it('Should fail with an invalid videoChunkFilename', async function () { + await testUpdate({ ...base, resolutionPlaylistFilename: undefined }) + await testUpdate({ ...base, resolutionPlaylistFilename: 'coucou/hello' }) + await testUpdate({ ...base, resolutionPlaylistFilename: 'hello' }) + }) + + it('Should fail with an invalid type', async function () { + await testUpdate({ ...base, type: undefined }) + await testUpdate({ ...base, type: 'toto' as any }) + }) + }) + }) + + describe('Error', function () { + + it('Should fail with a missing error message', async function () { + await server.runnerJobs.error({ jobUUID, jobToken, runnerToken, message: null, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with an invalid error messgae', async function () { + const message = 'a'.repeat(6000) + await server.runnerJobs.error({ jobUUID, jobToken, runnerToken, message, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with a job not in processing state', async function () { + await server.runnerJobs.error({ + jobUUID: completedJobUUID, + jobToken: completedJobToken, + message: 'my message', + runnerToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + }) + + describe('Success', function () { + let vodJobUUID: string + let vodJobToken: string + + describe('Common', function () { + + it('Should fail with a job not in processing state', async function () { + await server.runnerJobs.success({ + jobUUID: completedJobUUID, + jobToken: completedJobToken, + payload: { videoFile: 'video_short.mp4' }, + runnerToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + }) + + describe('VOD', function () { + + it('Should fail with an invalid vod web video payload', async function () { + const { job } = await server.runnerJobs.autoAccept({ runnerToken, type: 'vod-web-video-transcoding' }) + + await server.runnerJobs.success({ + jobUUID: job.uuid, + jobToken: job.jobToken, + payload: { hello: 'video_short.mp4' } as any, + runnerToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + + vodJobUUID = job.uuid + vodJobToken = job.jobToken + }) + + it('Should fail with an invalid vod hls payload', async function () { + // To create HLS jobs + const payload: RunnerJobSuccessPayload = { videoFile: 'video_short.mp4' } + await server.runnerJobs.success({ runnerToken, jobUUID: vodJobUUID, jobToken: vodJobToken, payload }) + + await waitJobs([ server ]) + + const { job } = await server.runnerJobs.autoAccept({ runnerToken, type: 'vod-hls-transcoding' }) + + await server.runnerJobs.success({ + jobUUID: job.uuid, + jobToken: job.jobToken, + payload: { videoFile: 'video_short.mp4' } as any, + runnerToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with an invalid vod audio merge payload', async function () { + const attributes = { name: 'audio_with_preview', previewfile: 'custom-preview.jpg', fixture: 'sample.ogg' } + await server.videos.upload({ attributes, mode: 'legacy' }) + + await waitJobs([ server ]) + + const { job } = await server.runnerJobs.autoAccept({ runnerToken, type: 'vod-audio-merge-transcoding' }) + + await server.runnerJobs.success({ + jobUUID: job.uuid, + jobToken: job.jobToken, + payload: { hello: 'video_short.mp4' } as any, + runnerToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + }) + + describe('Video studio', function () { + + it('Should fail with an invalid video studio transcoding payload', async function () { + await server.runnerJobs.success({ + jobUUID: studioAcceptedJob.uuid, + jobToken: studioAcceptedJob.jobToken, + payload: { hello: 'video_short.mp4' } as any, + runnerToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + }) + }) + + describe('Job files', function () { + + describe('Check video param for common job file routes', function () { + + async function fetchFiles (options: { + videoUUID?: string + expectedStatus: HttpStatusCodeType + }) { + await fetchVideoInputFiles({ videoUUID, ...options, jobToken, jobUUID, runnerToken }) + + await fetchStudioFiles({ + videoUUID: videoStudioUUID, + + ...options, + + jobToken: studioAcceptedJob.jobToken, + jobUUID: studioAcceptedJob.uuid, + runnerToken, + studioFile + }) + } + + it('Should fail with an invalid video id', async function () { + await fetchFiles({ + videoUUID: 'a', + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with an unknown video id', async function () { + const videoUUID = '910ec12a-d9e6-458b-a274-0abb655f9464' + + await fetchFiles({ + videoUUID, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail with a video id not associated to this job', async function () { + await fetchFiles({ + videoUUID: videoUUID2, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should succeed with the correct params', async function () { + await fetchFiles({ expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('Video studio tasks file routes', function () { + + it('Should fail with an invalid studio filename', async function () { + await fetchStudioFiles({ + videoUUID: videoStudioUUID, + jobUUID: studioAcceptedJob.uuid, + runnerToken, + jobToken: studioAcceptedJob.jobToken, + studioFile: 'toto', + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/search.ts b/packages/tests/src/api/check-params/search.ts new file mode 100644 index 000000000..b886cbc82 --- /dev/null +++ b/packages/tests/src/api/check-params/search.ts @@ -0,0 +1,278 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { HttpStatusCode } from '@peertube/peertube-models' +import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@tests/shared/checks.js' +import { + cleanupTests, + createSingleServer, + makeGetRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +function updateSearchIndex (server: PeerTubeServer, enabled: boolean, disableLocalSearch = false) { + return server.config.updateCustomSubConfig({ + newConfig: { + search: { + searchIndex: { + enabled, + disableLocalSearch + } + } + } + }) +} + +describe('Test videos API validator', function () { + let server: PeerTubeServer + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + }) + + describe('When searching videos', function () { + const path = '/api/v1/search/videos/' + + const query = { + search: 'coucou' + } + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, null, query) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, null, query) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, null, query) + }) + + it('Should succeed with the correct parameters', async function () { + await makeGetRequest({ url: server.url, path, query, expectedStatus: HttpStatusCode.OK_200 }) + }) + + it('Should fail with an invalid category', async function () { + const customQuery1 = { ...query, categoryOneOf: [ 'aa', 'b' ] } + await makeGetRequest({ url: server.url, path, query: customQuery1, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + + const customQuery2 = { ...query, categoryOneOf: 'a' } + await makeGetRequest({ url: server.url, path, query: customQuery2, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should succeed with a valid category', async function () { + const customQuery1 = { ...query, categoryOneOf: [ 1, 7 ] } + await makeGetRequest({ url: server.url, path, query: customQuery1, expectedStatus: HttpStatusCode.OK_200 }) + + const customQuery2 = { ...query, categoryOneOf: 1 } + await makeGetRequest({ url: server.url, path, query: customQuery2, expectedStatus: HttpStatusCode.OK_200 }) + }) + + it('Should fail with an invalid licence', async function () { + const customQuery1 = { ...query, licenceOneOf: [ 'aa', 'b' ] } + await makeGetRequest({ url: server.url, path, query: customQuery1, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + + const customQuery2 = { ...query, licenceOneOf: 'a' } + await makeGetRequest({ url: server.url, path, query: customQuery2, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should succeed with a valid licence', async function () { + const customQuery1 = { ...query, licenceOneOf: [ 1, 2 ] } + await makeGetRequest({ url: server.url, path, query: customQuery1, expectedStatus: HttpStatusCode.OK_200 }) + + const customQuery2 = { ...query, licenceOneOf: 1 } + await makeGetRequest({ url: server.url, path, query: customQuery2, expectedStatus: HttpStatusCode.OK_200 }) + }) + + it('Should succeed with a valid language', async function () { + const customQuery1 = { ...query, languageOneOf: [ 'fr', 'en' ] } + await makeGetRequest({ url: server.url, path, query: customQuery1, expectedStatus: HttpStatusCode.OK_200 }) + + const customQuery2 = { ...query, languageOneOf: 'fr' } + await makeGetRequest({ url: server.url, path, query: customQuery2, expectedStatus: HttpStatusCode.OK_200 }) + }) + + it('Should succeed with valid tags', async function () { + const customQuery1 = { ...query, tagsOneOf: [ 'tag1', 'tag2' ] } + await makeGetRequest({ url: server.url, path, query: customQuery1, expectedStatus: HttpStatusCode.OK_200 }) + + const customQuery2 = { ...query, tagsOneOf: 'tag1' } + await makeGetRequest({ url: server.url, path, query: customQuery2, expectedStatus: HttpStatusCode.OK_200 }) + + const customQuery3 = { ...query, tagsAllOf: [ 'tag1', 'tag2' ] } + await makeGetRequest({ url: server.url, path, query: customQuery3, expectedStatus: HttpStatusCode.OK_200 }) + + const customQuery4 = { ...query, tagsAllOf: 'tag1' } + await makeGetRequest({ url: server.url, path, query: customQuery4, expectedStatus: HttpStatusCode.OK_200 }) + }) + + it('Should fail with invalid durations', async function () { + const customQuery1 = { ...query, durationMin: 'hello' } + await makeGetRequest({ url: server.url, path, query: customQuery1, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + + const customQuery2 = { ...query, durationMax: 'hello' } + await makeGetRequest({ url: server.url, path, query: customQuery2, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with invalid dates', async function () { + const customQuery1 = { ...query, startDate: 'hello' } + await makeGetRequest({ url: server.url, path, query: customQuery1, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + + const customQuery2 = { ...query, endDate: 'hello' } + await makeGetRequest({ url: server.url, path, query: customQuery2, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + + const customQuery3 = { ...query, originallyPublishedStartDate: 'hello' } + await makeGetRequest({ url: server.url, path, query: customQuery3, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + + const customQuery4 = { ...query, originallyPublishedEndDate: 'hello' } + await makeGetRequest({ url: server.url, path, query: customQuery4, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with an invalid host', async function () { + const customQuery = { ...query, host: '6565' } + await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should succeed with a host', async function () { + const customQuery = { ...query, host: 'example.com' } + await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.OK_200 }) + }) + + it('Should fail with invalid uuids', async function () { + const customQuery = { ...query, uuids: [ '6565', 'dfd70b83-639f-4980-94af-304a56ab4b35' ] } + await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should succeed with valid uuids', async function () { + const customQuery = { ...query, uuids: [ 'dfd70b83-639f-4980-94af-304a56ab4b35' ] } + await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('When searching video playlists', function () { + const path = '/api/v1/search/video-playlists/' + + const query = { + search: 'coucou', + host: 'example.com' + } + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, null, query) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, null, query) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, null, query) + }) + + it('Should fail with an invalid host', async function () { + await makeGetRequest({ url: server.url, path, query: { ...query, host: '6565' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with invalid uuids', async function () { + const customQuery = { ...query, uuids: [ '6565', 'dfd70b83-639f-4980-94af-304a56ab4b35' ] } + await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should succeed with the correct parameters', async function () { + await makeGetRequest({ url: server.url, path, query, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('When searching video channels', function () { + const path = '/api/v1/search/video-channels/' + + const query = { + search: 'coucou', + host: 'example.com' + } + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, null, query) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, null, query) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, null, query) + }) + + it('Should fail with an invalid host', async function () { + await makeGetRequest({ url: server.url, path, query: { ...query, host: '6565' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with invalid handles', async function () { + await makeGetRequest({ url: server.url, path, query: { ...query, handles: [ '' ] }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should succeed with the correct parameters', async function () { + await makeGetRequest({ url: server.url, path, query, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('Search target', function () { + + it('Should fail/succeed depending on the search target', async function () { + const query = { search: 'coucou' } + const paths = [ + '/api/v1/search/video-playlists/', + '/api/v1/search/video-channels/', + '/api/v1/search/videos/' + ] + + for (const path of paths) { + { + const customQuery = { ...query, searchTarget: 'hello' } + await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + } + + { + const customQuery = { ...query, searchTarget: undefined } + await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.OK_200 }) + } + + { + const customQuery = { ...query, searchTarget: 'local' } + await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.OK_200 }) + } + + { + const customQuery = { ...query, searchTarget: 'search-index' } + await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + } + + await updateSearchIndex(server, true, true) + + { + const customQuery = { ...query, searchTarget: 'search-index' } + await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.OK_200 }) + } + + await updateSearchIndex(server, true, false) + + { + const customQuery = { ...query, searchTarget: 'local' } + await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.OK_200 }) + } + + await updateSearchIndex(server, false, false) + } + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/services.ts b/packages/tests/src/api/check-params/services.ts new file mode 100644 index 000000000..0b0466d84 --- /dev/null +++ b/packages/tests/src/api/check-params/services.ts @@ -0,0 +1,207 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { + HttpStatusCode, + HttpStatusCodeType, + VideoCreateResult, + VideoPlaylistCreateResult, + VideoPlaylistPrivacy, + VideoPrivacy +} from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + makeGetRequest, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel +} from '@peertube/peertube-server-commands' + +describe('Test services API validators', function () { + let server: PeerTubeServer + let playlistUUID: string + + let privateVideo: VideoCreateResult + let unlistedVideo: VideoCreateResult + + let privatePlaylist: VideoPlaylistCreateResult + let unlistedPlaylist: VideoPlaylistCreateResult + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(60000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + server.store.videoCreated = await server.videos.upload({ attributes: { name: 'my super name' } }) + + privateVideo = await server.videos.quickUpload({ name: 'private', privacy: VideoPrivacy.PRIVATE }) + unlistedVideo = await server.videos.quickUpload({ name: 'unlisted', privacy: VideoPrivacy.UNLISTED }) + + { + const created = await server.playlists.create({ + attributes: { + displayName: 'super playlist', + privacy: VideoPlaylistPrivacy.PUBLIC, + videoChannelId: server.store.channel.id + } + }) + + playlistUUID = created.uuid + + privatePlaylist = await server.playlists.create({ + attributes: { + displayName: 'private', + privacy: VideoPlaylistPrivacy.PRIVATE, + videoChannelId: server.store.channel.id + } + }) + + unlistedPlaylist = await server.playlists.create({ + attributes: { + displayName: 'unlisted', + privacy: VideoPlaylistPrivacy.UNLISTED, + videoChannelId: server.store.channel.id + } + }) + } + }) + + describe('Test oEmbed API validators', function () { + + it('Should fail with an invalid url', async function () { + const embedUrl = 'hello.com' + await checkParamEmbed(server, embedUrl) + }) + + it('Should fail with an invalid host', async function () { + const embedUrl = 'http://hello.com/videos/watch/' + server.store.videoCreated.uuid + await checkParamEmbed(server, embedUrl) + }) + + it('Should fail with an invalid element id', async function () { + const embedUrl = `${server.url}/videos/watch/blabla` + await checkParamEmbed(server, embedUrl) + }) + + it('Should fail with an unknown element', async function () { + const embedUrl = `${server.url}/videos/watch/88fc0165-d1f0-4a35-a51a-3b47f668689c` + await checkParamEmbed(server, embedUrl, HttpStatusCode.NOT_FOUND_404) + }) + + it('Should fail with an invalid path', async function () { + const embedUrl = `${server.url}/videos/watchs/${server.store.videoCreated.uuid}` + + await checkParamEmbed(server, embedUrl) + }) + + it('Should fail with an invalid max height', async function () { + const embedUrl = `${server.url}/videos/watch/${server.store.videoCreated.uuid}` + + await checkParamEmbed(server, embedUrl, HttpStatusCode.BAD_REQUEST_400, { maxheight: 'hello' }) + }) + + it('Should fail with an invalid max width', async function () { + const embedUrl = `${server.url}/videos/watch/${server.store.videoCreated.uuid}` + + await checkParamEmbed(server, embedUrl, HttpStatusCode.BAD_REQUEST_400, { maxwidth: 'hello' }) + }) + + it('Should fail with an invalid format', async function () { + const embedUrl = `${server.url}/videos/watch/${server.store.videoCreated.uuid}` + + await checkParamEmbed(server, embedUrl, HttpStatusCode.BAD_REQUEST_400, { format: 'blabla' }) + }) + + it('Should fail with a non supported format', async function () { + const embedUrl = `${server.url}/videos/watch/${server.store.videoCreated.uuid}` + + await checkParamEmbed(server, embedUrl, HttpStatusCode.NOT_IMPLEMENTED_501, { format: 'xml' }) + }) + + it('Should fail with a private video', async function () { + const embedUrl = `${server.url}/videos/watch/${privateVideo.uuid}` + + await checkParamEmbed(server, embedUrl, HttpStatusCode.FORBIDDEN_403) + }) + + it('Should fail with an unlisted video with the int id', async function () { + const embedUrl = `${server.url}/videos/watch/${unlistedVideo.id}` + + await checkParamEmbed(server, embedUrl, HttpStatusCode.FORBIDDEN_403) + }) + + it('Should succeed with an unlisted video using the uuid id', async function () { + for (const uuid of [ unlistedVideo.uuid, unlistedVideo.shortUUID ]) { + const embedUrl = `${server.url}/videos/watch/${uuid}` + + await checkParamEmbed(server, embedUrl, HttpStatusCode.OK_200) + } + }) + + it('Should fail with a private playlist', async function () { + const embedUrl = `${server.url}/videos/watch/playlist/${privatePlaylist.uuid}` + + await checkParamEmbed(server, embedUrl, HttpStatusCode.FORBIDDEN_403) + }) + + it('Should fail with an unlisted playlist using the int id', async function () { + const embedUrl = `${server.url}/videos/watch/playlist/${unlistedPlaylist.id}` + + await checkParamEmbed(server, embedUrl, HttpStatusCode.FORBIDDEN_403) + }) + + it('Should succeed with an unlisted playlist using the uuid id', async function () { + for (const uuid of [ unlistedPlaylist.uuid, unlistedPlaylist.shortUUID ]) { + const embedUrl = `${server.url}/videos/watch/playlist/${uuid}` + + await checkParamEmbed(server, embedUrl, HttpStatusCode.OK_200) + } + }) + + it('Should succeed with the correct params with a video', async function () { + const embedUrl = `${server.url}/videos/watch/${server.store.videoCreated.uuid}` + const query = { + format: 'json', + maxheight: 400, + maxwidth: 400 + } + + await checkParamEmbed(server, embedUrl, HttpStatusCode.OK_200, query) + }) + + it('Should succeed with the correct params with a playlist', async function () { + const embedUrl = `${server.url}/videos/watch/playlist/${playlistUUID}` + const query = { + format: 'json', + maxheight: 400, + maxwidth: 400 + } + + await checkParamEmbed(server, embedUrl, HttpStatusCode.OK_200, query) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) + +function checkParamEmbed ( + server: PeerTubeServer, + embedUrl: string, + expectedStatus: HttpStatusCodeType = HttpStatusCode.BAD_REQUEST_400, + query = {} +) { + const path = '/services/oembed' + + return makeGetRequest({ + url: server.url, + path, + query: Object.assign(query, { url: embedUrl }), + expectedStatus + }) +} diff --git a/packages/tests/src/api/check-params/transcoding.ts b/packages/tests/src/api/check-params/transcoding.ts new file mode 100644 index 000000000..50935c59e --- /dev/null +++ b/packages/tests/src/api/check-params/transcoding.ts @@ -0,0 +1,112 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { HttpStatusCode, UserRole } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test transcoding API validators', function () { + let servers: PeerTubeServer[] + + let userToken: string + let moderatorToken: string + + let remoteId: string + let validId: string + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + + await doubleFollow(servers[0], servers[1]) + + userToken = await servers[0].users.generateUserAndToken('user', UserRole.USER) + moderatorToken = await servers[0].users.generateUserAndToken('moderator', UserRole.MODERATOR) + + { + const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' }) + remoteId = uuid + } + + { + const { uuid } = await servers[0].videos.quickUpload({ name: 'both 1' }) + validId = uuid + } + + await waitJobs(servers) + + await servers[0].config.enableTranscoding() + }) + + it('Should not run transcoding of a unknown video', async function () { + await servers[0].videos.runTranscoding({ videoId: 404, transcodingType: 'hls', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + await servers[0].videos.runTranscoding({ videoId: 404, transcodingType: 'web-video', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should not run transcoding of a remote video', async function () { + const expectedStatus = HttpStatusCode.BAD_REQUEST_400 + + await servers[0].videos.runTranscoding({ videoId: remoteId, transcodingType: 'hls', expectedStatus }) + await servers[0].videos.runTranscoding({ videoId: remoteId, transcodingType: 'web-video', expectedStatus }) + }) + + it('Should not run transcoding by a non admin user', async function () { + const expectedStatus = HttpStatusCode.FORBIDDEN_403 + + await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'hls', token: userToken, expectedStatus }) + await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'web-video', token: moderatorToken, expectedStatus }) + }) + + it('Should not run transcoding without transcoding type', async function () { + await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: undefined, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should not run transcoding with an incorrect transcoding type', async function () { + const expectedStatus = HttpStatusCode.BAD_REQUEST_400 + + await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'toto' as any, expectedStatus }) + }) + + it('Should not run transcoding if the instance disabled it', async function () { + const expectedStatus = HttpStatusCode.BAD_REQUEST_400 + + await servers[0].config.disableTranscoding() + + await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'hls', expectedStatus }) + await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'web-video', expectedStatus }) + }) + + it('Should run transcoding', async function () { + this.timeout(120_000) + + await servers[0].config.enableTranscoding() + + await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'hls' }) + await waitJobs(servers) + + await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'web-video', forceTranscoding: true }) + await waitJobs(servers) + }) + + it('Should not run transcoding on a video that is already being transcoded if forceTranscoding is not set', async function () { + await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'web-video' }) + + const expectedStatus = HttpStatusCode.CONFLICT_409 + await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'web-video', expectedStatus }) + + await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'web-video', forceTranscoding: true }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/check-params/two-factor.ts b/packages/tests/src/api/check-params/two-factor.ts new file mode 100644 index 000000000..0b1766eca --- /dev/null +++ b/packages/tests/src/api/check-params/two-factor.ts @@ -0,0 +1,294 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + PeerTubeServer, + setAccessTokensToServers, + TwoFactorCommand +} from '@peertube/peertube-server-commands' + +describe('Test two factor API validators', function () { + let server: PeerTubeServer + + let rootId: number + let rootPassword: string + let rootRequestToken: string + let rootOTPToken: string + + let userId: number + let userToken = '' + let userPassword: string + let userRequestToken: string + let userOTPToken: string + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + { + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + } + + { + const result = await server.users.generate('user1') + userToken = result.token + userId = result.userId + userPassword = result.password + } + + { + const { id } = await server.users.getMyInfo() + rootId = id + rootPassword = server.store.user.password + } + }) + + describe('When requesting two factor', function () { + + it('Should fail with an unknown user id', async function () { + await server.twoFactor.request({ userId: 42, currentPassword: rootPassword, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail with an invalid user id', async function () { + await server.twoFactor.request({ + userId: 'invalid' as any, + currentPassword: rootPassword, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail to request another user two factor without the appropriate rights', async function () { + await server.twoFactor.request({ + userId: rootId, + token: userToken, + currentPassword: userPassword, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should succeed to request another user two factor with the appropriate rights', async function () { + await server.twoFactor.request({ userId, currentPassword: rootPassword }) + }) + + it('Should fail to request two factor without a password', async function () { + await server.twoFactor.request({ + userId, + token: userToken, + currentPassword: undefined, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail to request two factor with an incorrect password', async function () { + await server.twoFactor.request({ + userId, + token: userToken, + currentPassword: rootPassword, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should succeed to request two factor without a password when targeting a remote user with an admin account', async function () { + await server.twoFactor.request({ userId }) + }) + + it('Should fail to request two factor without a password when targeting myself with an admin account', async function () { + await server.twoFactor.request({ userId: rootId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await server.twoFactor.request({ userId: rootId, currentPassword: 'bad', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should succeed to request my two factor auth', async function () { + { + const { otpRequest } = await server.twoFactor.request({ userId, token: userToken, currentPassword: userPassword }) + userRequestToken = otpRequest.requestToken + userOTPToken = TwoFactorCommand.buildOTP({ secret: otpRequest.secret }).generate() + } + + { + const { otpRequest } = await server.twoFactor.request({ userId: rootId, currentPassword: rootPassword }) + rootRequestToken = otpRequest.requestToken + rootOTPToken = TwoFactorCommand.buildOTP({ secret: otpRequest.secret }).generate() + } + }) + }) + + describe('When confirming two factor request', function () { + + it('Should fail with an unknown user id', async function () { + await server.twoFactor.confirmRequest({ + userId: 42, + requestToken: rootRequestToken, + otpToken: rootOTPToken, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail with an invalid user id', async function () { + await server.twoFactor.confirmRequest({ + userId: 'invalid' as any, + requestToken: rootRequestToken, + otpToken: rootOTPToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail to confirm another user two factor request without the appropriate rights', async function () { + await server.twoFactor.confirmRequest({ + userId: rootId, + token: userToken, + requestToken: rootRequestToken, + otpToken: rootOTPToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail without request token', async function () { + await server.twoFactor.confirmRequest({ + userId, + requestToken: undefined, + otpToken: userOTPToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with an invalid request token', async function () { + await server.twoFactor.confirmRequest({ + userId, + requestToken: 'toto', + otpToken: userOTPToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with request token of another user', async function () { + await server.twoFactor.confirmRequest({ + userId, + requestToken: rootRequestToken, + otpToken: userOTPToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail without an otp token', async function () { + await server.twoFactor.confirmRequest({ + userId, + requestToken: userRequestToken, + otpToken: undefined, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with a bad otp token', async function () { + await server.twoFactor.confirmRequest({ + userId, + requestToken: userRequestToken, + otpToken: '123456', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should succeed to confirm another user two factor request with the appropriate rights', async function () { + await server.twoFactor.confirmRequest({ + userId, + requestToken: userRequestToken, + otpToken: userOTPToken + }) + + // Reinit + await server.twoFactor.disable({ userId, currentPassword: rootPassword }) + }) + + it('Should succeed to confirm my two factor request', async function () { + await server.twoFactor.confirmRequest({ + userId, + token: userToken, + requestToken: userRequestToken, + otpToken: userOTPToken + }) + }) + + it('Should fail to confirm again two factor request', async function () { + await server.twoFactor.confirmRequest({ + userId, + token: userToken, + requestToken: userRequestToken, + otpToken: userOTPToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + }) + + describe('When disabling two factor', function () { + + it('Should fail with an unknown user id', async function () { + await server.twoFactor.disable({ + userId: 42, + currentPassword: rootPassword, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail with an invalid user id', async function () { + await server.twoFactor.disable({ + userId: 'invalid' as any, + currentPassword: rootPassword, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail to disable another user two factor without the appropriate rights', async function () { + await server.twoFactor.disable({ + userId: rootId, + token: userToken, + currentPassword: userPassword, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail to disable two factor with an incorrect password', async function () { + await server.twoFactor.disable({ + userId, + token: userToken, + currentPassword: rootPassword, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should succeed to disable two factor without a password when targeting a remote user with an admin account', async function () { + await server.twoFactor.disable({ userId }) + await server.twoFactor.requestAndConfirm({ userId }) + }) + + it('Should fail to disable two factor without a password when targeting myself with an admin account', async function () { + await server.twoFactor.disable({ userId: rootId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await server.twoFactor.disable({ userId: rootId, currentPassword: 'bad', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should succeed to disable another user two factor with the appropriate rights', async function () { + await server.twoFactor.disable({ userId, currentPassword: rootPassword }) + + await server.twoFactor.requestAndConfirm({ userId }) + }) + + it('Should succeed to update my two factor auth', async function () { + await server.twoFactor.disable({ userId, token: userToken, currentPassword: userPassword }) + }) + + it('Should fail to disable again two factor', async function () { + await server.twoFactor.disable({ + userId, + token: userToken, + currentPassword: userPassword, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/upload-quota.ts b/packages/tests/src/api/check-params/upload-quota.ts new file mode 100644 index 000000000..a77792822 --- /dev/null +++ b/packages/tests/src/api/check-params/upload-quota.ts @@ -0,0 +1,134 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { FIXTURE_URLS } from '@tests/shared/tests.js' +import { randomInt } from '@peertube/peertube-core-utils' +import { HttpStatusCode, VideoImportState, VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + VideosCommand, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test upload quota', function () { + let server: PeerTubeServer + let rootId: number + let command: VideosCommand + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + const user = await server.users.getMyInfo() + rootId = user.id + + await server.users.update({ userId: rootId, videoQuota: 42 }) + + command = server.videos + }) + + describe('When having a video quota', function () { + + it('Should fail with a registered user having too many videos with legacy upload', async function () { + this.timeout(120000) + + const user = { username: 'registered' + randomInt(1, 1500), password: 'password' } + await server.registrations.register(user) + const userToken = await server.login.getAccessToken(user) + + const attributes = { fixture: 'video_short2.webm' } + for (let i = 0; i < 5; i++) { + await command.upload({ token: userToken, attributes }) + } + + await command.upload({ token: userToken, attributes, expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413, mode: 'legacy' }) + }) + + it('Should fail with a registered user having too many videos with resumable upload', async function () { + this.timeout(120000) + + const user = { username: 'registered' + randomInt(1, 1500), password: 'password' } + await server.registrations.register(user) + const userToken = await server.login.getAccessToken(user) + + const attributes = { fixture: 'video_short2.webm' } + for (let i = 0; i < 5; i++) { + await command.upload({ token: userToken, attributes }) + } + + await command.upload({ token: userToken, attributes, expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413, mode: 'resumable' }) + }) + + it('Should fail to import with HTTP/Torrent/magnet', async function () { + this.timeout(120_000) + + const baseAttributes = { + channelId: server.store.channel.id, + privacy: VideoPrivacy.PUBLIC + } + await server.imports.importVideo({ attributes: { ...baseAttributes, targetUrl: FIXTURE_URLS.goodVideo } }) + await server.imports.importVideo({ attributes: { ...baseAttributes, magnetUri: FIXTURE_URLS.magnet } }) + await server.imports.importVideo({ attributes: { ...baseAttributes, torrentfile: 'video-720p.torrent' as any } }) + + await waitJobs([ server ]) + + const { total, data: videoImports } = await server.imports.getMyVideoImports() + expect(total).to.equal(3) + + expect(videoImports).to.have.lengthOf(3) + + for (const videoImport of videoImports) { + expect(videoImport.state.id).to.equal(VideoImportState.FAILED) + expect(videoImport.error).not.to.be.undefined + expect(videoImport.error).to.contain('user video quota is exceeded') + } + }) + }) + + describe('When having a daily video quota', function () { + + it('Should fail with a user having too many videos daily', async function () { + await server.users.update({ userId: rootId, videoQuotaDaily: 42 }) + + await command.upload({ expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413, mode: 'legacy' }) + await command.upload({ expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413, mode: 'resumable' }) + }) + }) + + describe('When having an absolute and daily video quota', function () { + it('Should fail if exceeding total quota', async function () { + await server.users.update({ + userId: rootId, + videoQuota: 42, + videoQuotaDaily: 1024 * 1024 * 1024 + }) + + await command.upload({ expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413, mode: 'legacy' }) + await command.upload({ expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413, mode: 'resumable' }) + }) + + it('Should fail if exceeding daily quota', async function () { + await server.users.update({ + userId: rootId, + videoQuota: 1024 * 1024 * 1024, + videoQuotaDaily: 42 + }) + + await command.upload({ expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413, mode: 'legacy' }) + await command.upload({ expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413, mode: 'resumable' }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/user-notifications.ts b/packages/tests/src/api/check-params/user-notifications.ts new file mode 100644 index 000000000..cf20324a1 --- /dev/null +++ b/packages/tests/src/api/check-params/user-notifications.ts @@ -0,0 +1,290 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { io } from 'socket.io-client' +import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@tests/shared/checks.js' +import { wait } from '@peertube/peertube-core-utils' +import { HttpStatusCode, UserNotificationSetting, UserNotificationSettingValue } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + makeGetRequest, + makePostBodyRequest, + makePutBodyRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test user notifications API validators', function () { + let server: PeerTubeServer + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + }) + + describe('When listing my notifications', function () { + const path = '/api/v1/users/me/notifications' + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an incorrect unread parameter', async function () { + await makeGetRequest({ + url: server.url, + path, + query: { + unread: 'toto' + }, + token: server.accessToken, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + + it('Should fail with a non authenticated user', async function () { + await makeGetRequest({ + url: server.url, + path, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should succeed with the correct parameters', async function () { + await makeGetRequest({ + url: server.url, + path, + token: server.accessToken, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + + describe('When marking as read my notifications', function () { + const path = '/api/v1/users/me/notifications/read' + + it('Should fail with wrong ids parameters', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { + ids: [ 'hello' ] + }, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + + await makePostBodyRequest({ + url: server.url, + path, + fields: { + ids: [ ] + }, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + + await makePostBodyRequest({ + url: server.url, + path, + fields: { + ids: 5 + }, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with a non authenticated user', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { + ids: [ 5 ] + }, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should succeed with the correct parameters', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { + ids: [ 5 ] + }, + token: server.accessToken, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + describe('When marking as read my notifications', function () { + const path = '/api/v1/users/me/notifications/read-all' + + it('Should fail with a non authenticated user', async function () { + await makePostBodyRequest({ + url: server.url, + path, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should succeed with the correct parameters', async function () { + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + describe('When updating my notification settings', function () { + const path = '/api/v1/users/me/notification-settings' + const correctFields: UserNotificationSetting = { + newVideoFromSubscription: UserNotificationSettingValue.WEB, + newCommentOnMyVideo: UserNotificationSettingValue.WEB, + abuseAsModerator: UserNotificationSettingValue.WEB, + videoAutoBlacklistAsModerator: UserNotificationSettingValue.WEB, + blacklistOnMyVideo: UserNotificationSettingValue.WEB, + myVideoImportFinished: UserNotificationSettingValue.WEB, + myVideoPublished: UserNotificationSettingValue.WEB, + commentMention: UserNotificationSettingValue.WEB, + newFollow: UserNotificationSettingValue.WEB, + newUserRegistration: UserNotificationSettingValue.WEB, + newInstanceFollower: UserNotificationSettingValue.WEB, + autoInstanceFollowing: UserNotificationSettingValue.WEB, + abuseNewMessage: UserNotificationSettingValue.WEB, + abuseStateChange: UserNotificationSettingValue.WEB, + newPeerTubeVersion: UserNotificationSettingValue.WEB, + myVideoStudioEditionFinished: UserNotificationSettingValue.WEB, + newPluginVersion: UserNotificationSettingValue.WEB + } + + it('Should fail with missing fields', async function () { + await makePutBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields: { newVideoFromSubscription: UserNotificationSettingValue.WEB }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with incorrect field values', async function () { + { + const fields = { ...correctFields, newCommentOnMyVideo: 15 } + + await makePutBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + } + + { + const fields = { ...correctFields, newCommentOnMyVideo: 'toto' } + + await makePutBodyRequest({ + url: server.url, + path, + fields, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + } + }) + + it('Should fail with a non authenticated user', async function () { + await makePutBodyRequest({ + url: server.url, + path, + fields: correctFields, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should succeed with the correct parameters', async function () { + await makePutBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields: correctFields, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + describe('When connecting to my notification socket', function () { + + it('Should fail with no token', function (next) { + const socket = io(`${server.url}/user-notifications`, { reconnection: false }) + + socket.once('connect_error', function () { + socket.disconnect() + next() + }) + + socket.on('connect', () => { + socket.disconnect() + next(new Error('Connected with a missing token.')) + }) + }) + + it('Should fail with an invalid token', function (next) { + const socket = io(`${server.url}/user-notifications`, { + query: { accessToken: 'bad_access_token' }, + reconnection: false + }) + + socket.once('connect_error', function () { + socket.disconnect() + next() + }) + + socket.on('connect', () => { + socket.disconnect() + next(new Error('Connected with an invalid token.')) + }) + }) + + it('Should success with the correct token', function (next) { + const socket = io(`${server.url}/user-notifications`, { + query: { accessToken: server.accessToken }, + reconnection: false + }) + + function errorListener (err) { + next(new Error('Error in connection: ' + err)) + } + + socket.on('connect_error', errorListener) + + socket.once('connect', async () => { + socket.disconnect() + + await wait(500) + next() + }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/user-subscriptions.ts b/packages/tests/src/api/check-params/user-subscriptions.ts new file mode 100644 index 000000000..e97f513a0 --- /dev/null +++ b/packages/tests/src/api/check-params/user-subscriptions.ts @@ -0,0 +1,298 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { + cleanupTests, + createSingleServer, + makeDeleteRequest, + makeGetRequest, + makePostBodyRequest, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' +import { HttpStatusCode } from '@peertube/peertube-models' +import { checkBadStartPagination, checkBadCountPagination, checkBadSortPagination } from '@tests/shared/checks.js' + +describe('Test user subscriptions API validators', function () { + const path = '/api/v1/users/me/subscriptions' + let server: PeerTubeServer + let userAccessToken = '' + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + + const user = { + username: 'user1', + password: 'my super password' + } + await server.users.create({ username: user.username, password: user.password }) + userAccessToken = await server.login.getAccessToken(user) + }) + + describe('When listing my subscriptions', function () { + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a non authenticated user', async function () { + await makeGetRequest({ + url: server.url, + path, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should succeed with the correct parameters', async function () { + await makeGetRequest({ + url: server.url, + path, + token: userAccessToken, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + + describe('When listing my subscriptions videos', function () { + const path = '/api/v1/users/me/subscriptions/videos' + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a non authenticated user', async function () { + await makeGetRequest({ + url: server.url, + path, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should succeed with the correct parameters', async function () { + await makeGetRequest({ + url: server.url, + path, + token: userAccessToken, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + + describe('When adding a subscription', function () { + it('Should fail with a non authenticated user', async function () { + await makePostBodyRequest({ + url: server.url, + path, + fields: { uri: 'user1_channel@' + server.host }, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with bad URIs', async function () { + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields: { uri: 'root' }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields: { uri: 'root@' }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields: { uri: 'root@hello@' }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should succeed with the correct parameters', async function () { + this.timeout(20000) + + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields: { uri: 'user1_channel@' + server.host }, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + + await waitJobs([ server ]) + }) + }) + + describe('When getting a subscription', function () { + it('Should fail with a non authenticated user', async function () { + await makeGetRequest({ + url: server.url, + path: path + '/user1_channel@' + server.host, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with bad URIs', async function () { + await makeGetRequest({ + url: server.url, + path: path + '/root', + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + + await makeGetRequest({ + url: server.url, + path: path + '/root@', + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + + await makeGetRequest({ + url: server.url, + path: path + '/root@hello@', + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with an unknown subscription', async function () { + await makeGetRequest({ + url: server.url, + path: path + '/root1@' + server.host, + token: server.accessToken, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should succeed with the correct parameters', async function () { + await makeGetRequest({ + url: server.url, + path: path + '/user1_channel@' + server.host, + token: server.accessToken, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + + describe('When checking if subscriptions exist', function () { + const existPath = path + '/exist' + + it('Should fail with a non authenticated user', async function () { + await makeGetRequest({ + url: server.url, + path: existPath, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with bad URIs', async function () { + await makeGetRequest({ + url: server.url, + path: existPath, + query: { uris: 'toto' }, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + + await makeGetRequest({ + url: server.url, + path: existPath, + query: { 'uris[]': 1 }, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should succeed with the correct parameters', async function () { + await makeGetRequest({ + url: server.url, + path: existPath, + query: { 'uris[]': 'coucou@' + server.host }, + token: server.accessToken, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + + describe('When removing a subscription', function () { + it('Should fail with a non authenticated user', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/user1_channel@' + server.host, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with bad URIs', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/root', + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + + await makeDeleteRequest({ + url: server.url, + path: path + '/root@', + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + + await makeDeleteRequest({ + url: server.url, + path: path + '/root@hello@', + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with an unknown subscription', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/root1@' + server.host, + token: server.accessToken, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should succeed with the correct parameters', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '/user1_channel@' + server.host, + token: server.accessToken, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/users-admin.ts b/packages/tests/src/api/check-params/users-admin.ts new file mode 100644 index 000000000..1ad222ddc --- /dev/null +++ b/packages/tests/src/api/check-params/users-admin.ts @@ -0,0 +1,457 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@tests/shared/checks.js' +import { MockSmtpServer } from '@tests/shared/mock-servers/index.js' +import { omit } from '@peertube/peertube-core-utils' +import { HttpStatusCode, UserAdminFlag, UserRole } from '@peertube/peertube-models' +import { + cleanupTests, + ConfigCommand, + createSingleServer, + killallServers, + makeGetRequest, + makePostBodyRequest, + makePutBodyRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test users admin API validators', function () { + const path = '/api/v1/users/' + let userId: number + let rootId: number + let moderatorId: number + let server: PeerTubeServer + let userToken = '' + let moderatorToken = '' + let emailPort: number + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + const emails: object[] = [] + emailPort = await MockSmtpServer.Instance.collectEmails(emails) + + { + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + } + + { + const result = await server.users.generate('user1') + userToken = result.token + userId = result.userId + } + + { + const result = await server.users.generate('moderator1', UserRole.MODERATOR) + moderatorToken = result.token + } + + { + const result = await server.users.generate('moderator2', UserRole.MODERATOR) + moderatorId = result.userId + } + }) + + describe('When listing users', function () { + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a non authenticated user', async function () { + await makeGetRequest({ + url: server.url, + path, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a non admin user', async function () { + await makeGetRequest({ + url: server.url, + path, + token: userToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + }) + + describe('When adding a new user', function () { + const baseCorrectParams = { + username: 'user2', + email: 'test@example.com', + password: 'my super password', + videoQuota: -1, + videoQuotaDaily: -1, + role: UserRole.USER, + adminFlags: UserAdminFlag.BYPASS_VIDEO_AUTO_BLACKLIST + } + + it('Should fail with a too small username', async function () { + const fields = { ...baseCorrectParams, username: '' } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a too long username', async function () { + const fields = { ...baseCorrectParams, username: 'super'.repeat(50) } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a not lowercase username', async function () { + const fields = { ...baseCorrectParams, username: 'Toto' } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with an incorrect username', async function () { + const fields = { ...baseCorrectParams, username: 'my username' } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a missing email', async function () { + const fields = omit(baseCorrectParams, [ 'email' ]) + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with an invalid email', async function () { + const fields = { ...baseCorrectParams, email: 'test_example.com' } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a too small password', async function () { + const fields = { ...baseCorrectParams, password: 'bla' } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a too long password', async function () { + const fields = { ...baseCorrectParams, password: 'super'.repeat(61) } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with empty password and no smtp configured', async function () { + const fields = { ...baseCorrectParams, password: '' } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should succeed with no password on a server with smtp enabled', async function () { + this.timeout(20000) + + await killallServers([ server ]) + + await server.run(ConfigCommand.getEmailOverrideConfig(emailPort)) + + const fields = { + ...baseCorrectParams, + + password: '', + username: 'create_password', + email: 'create_password@example.com' + } + + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + + it('Should fail with invalid admin flags', async function () { + const fields = { ...baseCorrectParams, adminFlags: 'toto' } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with an non authenticated user', async function () { + await makePostBodyRequest({ + url: server.url, + path, + token: 'super token', + fields: baseCorrectParams, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail if we add a user with the same username', async function () { + const fields = { ...baseCorrectParams, username: 'user1' } + + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields, + expectedStatus: HttpStatusCode.CONFLICT_409 + }) + }) + + it('Should fail if we add a user with the same email', async function () { + const fields = { ...baseCorrectParams, email: 'user1@example.com' } + + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields, + expectedStatus: HttpStatusCode.CONFLICT_409 + }) + }) + + it('Should fail with an invalid videoQuota', async function () { + const fields = { ...baseCorrectParams, videoQuota: -5 } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with an invalid videoQuotaDaily', async function () { + const fields = { ...baseCorrectParams, videoQuotaDaily: -7 } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail without a user role', async function () { + const fields = omit(baseCorrectParams, [ 'role' ]) + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with an invalid user role', async function () { + const fields = { ...baseCorrectParams, role: 88989 } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a "peertube" username', async function () { + const fields = { ...baseCorrectParams, username: 'peertube' } + + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields, + expectedStatus: HttpStatusCode.CONFLICT_409 + }) + }) + + it('Should fail to create a moderator or an admin with a moderator', async function () { + for (const role of [ UserRole.MODERATOR, UserRole.ADMINISTRATOR ]) { + const fields = { ...baseCorrectParams, role } + + await makePostBodyRequest({ + url: server.url, + path, + token: moderatorToken, + fields, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + } + }) + + it('Should succeed to create a user with a moderator', async function () { + const fields = { ...baseCorrectParams, username: 'a4656', email: 'a4656@example.com', role: UserRole.USER } + + await makePostBodyRequest({ + url: server.url, + path, + token: moderatorToken, + fields, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + + it('Should succeed with the correct params', async function () { + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields: baseCorrectParams, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + + it('Should fail with a non admin user', async function () { + const user = { username: 'user1' } + userToken = await server.login.getAccessToken(user) + + const fields = { + username: 'user3', + email: 'test@example.com', + password: 'my super password', + videoQuota: 42000000 + } + await makePostBodyRequest({ url: server.url, path, token: userToken, fields, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + }) + + describe('When getting a user', function () { + + it('Should fail with an non authenticated user', async function () { + await makeGetRequest({ + url: server.url, + path: path + userId, + token: 'super token', + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a non admin user', async function () { + await makeGetRequest({ url: server.url, path, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should succeed with the correct params', async function () { + await makeGetRequest({ url: server.url, path: path + userId, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('When updating a user', function () { + + it('Should fail with an invalid email attribute', async function () { + const fields = { + email: 'blabla' + } + + await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields }) + }) + + it('Should fail with an invalid emailVerified attribute', async function () { + const fields = { + emailVerified: 'yes' + } + + await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields }) + }) + + it('Should fail with an invalid videoQuota attribute', async function () { + const fields = { + videoQuota: -90 + } + + await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields }) + }) + + it('Should fail with an invalid user role attribute', async function () { + const fields = { + role: 54878 + } + + await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields }) + }) + + it('Should fail with a too small password', async function () { + const fields = { + currentPassword: 'password', + password: 'bla' + } + + await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields }) + }) + + it('Should fail with a too long password', async function () { + const fields = { + currentPassword: 'password', + password: 'super'.repeat(61) + } + + await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields }) + }) + + it('Should fail with an non authenticated user', async function () { + const fields = { + videoQuota: 42 + } + + await makePutBodyRequest({ + url: server.url, + path: path + userId, + token: 'super token', + fields, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail when updating root role', async function () { + const fields = { + role: UserRole.MODERATOR + } + + await makePutBodyRequest({ url: server.url, path: path + rootId, token: server.accessToken, fields }) + }) + + it('Should fail with invalid admin flags', async function () { + const fields = { adminFlags: 'toto' } + + await makePutBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail to update an admin with a moderator', async function () { + const fields = { + videoQuota: 42 + } + + await makePutBodyRequest({ + url: server.url, + path: path + moderatorId, + token: moderatorToken, + fields, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should succeed to update a user with a moderator', async function () { + const fields = { + videoQuota: 42 + } + + await makePutBodyRequest({ + url: server.url, + path: path + userId, + token: moderatorToken, + fields, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + + it('Should succeed with the correct params', async function () { + const fields = { + email: 'email@example.com', + emailVerified: true, + videoQuota: 42, + role: UserRole.USER + } + + await makePutBodyRequest({ + url: server.url, + path: path + userId, + token: server.accessToken, + fields, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + after(async function () { + MockSmtpServer.Instance.kill() + + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/users-emails.ts b/packages/tests/src/api/check-params/users-emails.ts new file mode 100644 index 000000000..e382190ec --- /dev/null +++ b/packages/tests/src/api/check-params/users-emails.ts @@ -0,0 +1,122 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ +import { HttpStatusCode, UserRole } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + makePostBodyRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test users API validators', function () { + let server: PeerTubeServer + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1, { + rates_limit: { + ask_send_email: { + max: 10 + } + } + }) + + await setAccessTokensToServers([ server ]) + await server.config.enableSignup(true) + + await server.users.generate('moderator2', UserRole.MODERATOR) + + await server.registrations.requestRegistration({ + username: 'request1', + registrationReason: 'tt' + }) + }) + + describe('When asking a password reset', function () { + const path = '/api/v1/users/ask-reset-password' + + it('Should fail with a missing email', async function () { + const fields = {} + + await makePostBodyRequest({ url: server.url, path, fields }) + }) + + it('Should fail with an invalid email', async function () { + const fields = { email: 'hello' } + + await makePostBodyRequest({ url: server.url, path, fields }) + }) + + it('Should success with the correct params', async function () { + const fields = { email: 'admin@example.com' } + + await makePostBodyRequest({ + url: server.url, + path, + fields, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + describe('When asking for an account verification email', function () { + const path = '/api/v1/users/ask-send-verify-email' + + it('Should fail with a missing email', async function () { + const fields = {} + + await makePostBodyRequest({ url: server.url, path, fields }) + }) + + it('Should fail with an invalid email', async function () { + const fields = { email: 'hello' } + + await makePostBodyRequest({ url: server.url, path, fields }) + }) + + it('Should succeed with the correct params', async function () { + const fields = { email: 'admin@example.com' } + + await makePostBodyRequest({ + url: server.url, + path, + fields, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + describe('When asking for a registration verification email', function () { + const path = '/api/v1/users/registrations/ask-send-verify-email' + + it('Should fail with a missing email', async function () { + const fields = {} + + await makePostBodyRequest({ url: server.url, path, fields }) + }) + + it('Should fail with an invalid email', async function () { + const fields = { email: 'hello' } + + await makePostBodyRequest({ url: server.url, path, fields }) + }) + + it('Should succeed with the correct params', async function () { + const fields = { email: 'request1@example.com' } + + await makePostBodyRequest({ + url: server.url, + path, + fields, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/video-blacklist.ts b/packages/tests/src/api/check-params/video-blacklist.ts new file mode 100644 index 000000000..6ec070b9b --- /dev/null +++ b/packages/tests/src/api/check-params/video-blacklist.ts @@ -0,0 +1,292 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@tests/shared/checks.js' +import { HttpStatusCode, VideoBlacklistType } from '@peertube/peertube-models' +import { + BlacklistCommand, + cleanupTests, + createMultipleServers, + doubleFollow, + makePostBodyRequest, + makePutBodyRequest, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test video blacklist API validators', function () { + let servers: PeerTubeServer[] + let notBlacklistedVideoId: string + let remoteVideoUUID: string + let userAccessToken1 = '' + let userAccessToken2 = '' + let command: BlacklistCommand + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + await doubleFollow(servers[0], servers[1]) + + { + const username = 'user1' + const password = 'my super password' + await servers[0].users.create({ username, password }) + userAccessToken1 = await servers[0].login.getAccessToken({ username, password }) + } + + { + const username = 'user2' + const password = 'my super password' + await servers[0].users.create({ username, password }) + userAccessToken2 = await servers[0].login.getAccessToken({ username, password }) + } + + { + servers[0].store.videoCreated = await servers[0].videos.upload({ token: userAccessToken1 }) + } + + { + const { uuid } = await servers[0].videos.upload() + notBlacklistedVideoId = uuid + } + + { + const { uuid } = await servers[1].videos.upload() + remoteVideoUUID = uuid + } + + await waitJobs(servers) + + command = servers[0].blacklist + }) + + describe('When adding a video in blacklist', function () { + const basePath = '/api/v1/videos/' + + it('Should fail with nothing', async function () { + const path = basePath + servers[0].store.videoCreated + '/blacklist' + const fields = {} + await makePostBodyRequest({ url: servers[0].url, path, token: servers[0].accessToken, fields }) + }) + + it('Should fail with a wrong video', async function () { + const wrongPath = '/api/v1/videos/blabla/blacklist' + const fields = {} + await makePostBodyRequest({ url: servers[0].url, path: wrongPath, token: servers[0].accessToken, fields }) + }) + + it('Should fail with a non authenticated user', async function () { + const path = basePath + servers[0].store.videoCreated + '/blacklist' + const fields = {} + await makePostBodyRequest({ url: servers[0].url, path, token: 'hello', fields, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with a non admin user', async function () { + const path = basePath + servers[0].store.videoCreated + '/blacklist' + const fields = {} + await makePostBodyRequest({ + url: servers[0].url, + path, + token: userAccessToken2, + fields, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with an invalid reason', async function () { + const path = basePath + servers[0].store.videoCreated.uuid + '/blacklist' + const fields = { reason: 'a'.repeat(305) } + + await makePostBodyRequest({ url: servers[0].url, path, token: servers[0].accessToken, fields }) + }) + + it('Should fail to unfederate a remote video', async function () { + const path = basePath + remoteVideoUUID + '/blacklist' + const fields = { unfederate: true } + + await makePostBodyRequest({ + url: servers[0].url, + path, + token: servers[0].accessToken, + fields, + expectedStatus: HttpStatusCode.CONFLICT_409 + }) + }) + + it('Should succeed with the correct params', async function () { + const path = basePath + servers[0].store.videoCreated.uuid + '/blacklist' + const fields = {} + + await makePostBodyRequest({ + url: servers[0].url, + path, + token: servers[0].accessToken, + fields, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + describe('When updating a video in blacklist', function () { + const basePath = '/api/v1/videos/' + + it('Should fail with a wrong video', async function () { + const wrongPath = '/api/v1/videos/blabla/blacklist' + const fields = {} + await makePutBodyRequest({ url: servers[0].url, path: wrongPath, token: servers[0].accessToken, fields }) + }) + + it('Should fail with a video not blacklisted', async function () { + const path = '/api/v1/videos/' + notBlacklistedVideoId + '/blacklist' + const fields = {} + await makePutBodyRequest({ + url: servers[0].url, + path, + token: servers[0].accessToken, + fields, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail with a non authenticated user', async function () { + const path = basePath + servers[0].store.videoCreated + '/blacklist' + const fields = {} + await makePutBodyRequest({ url: servers[0].url, path, token: 'hello', fields, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with a non admin user', async function () { + const path = basePath + servers[0].store.videoCreated + '/blacklist' + const fields = {} + await makePutBodyRequest({ + url: servers[0].url, + path, + token: userAccessToken2, + fields, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with an invalid reason', async function () { + const path = basePath + servers[0].store.videoCreated.uuid + '/blacklist' + const fields = { reason: 'a'.repeat(305) } + + await makePutBodyRequest({ url: servers[0].url, path, token: servers[0].accessToken, fields }) + }) + + it('Should succeed with the correct params', async function () { + const path = basePath + servers[0].store.videoCreated.shortUUID + '/blacklist' + const fields = { reason: 'hello' } + + await makePutBodyRequest({ + url: servers[0].url, + path, + token: servers[0].accessToken, + fields, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + describe('When getting blacklisted video', function () { + + it('Should fail with a non authenticated user', async function () { + await servers[0].videos.get({ id: servers[0].store.videoCreated.uuid, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with another user', async function () { + await servers[0].videos.getWithToken({ + token: userAccessToken2, + id: servers[0].store.videoCreated.uuid, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should succeed with the owner authenticated user', async function () { + const video = await servers[0].videos.getWithToken({ token: userAccessToken1, id: servers[0].store.videoCreated.uuid }) + expect(video.blacklisted).to.be.true + }) + + it('Should succeed with an admin', async function () { + const video = servers[0].store.videoCreated + + for (const id of [ video.id, video.uuid, video.shortUUID ]) { + const video = await servers[0].videos.getWithToken({ id, expectedStatus: HttpStatusCode.OK_200 }) + expect(video.blacklisted).to.be.true + } + }) + }) + + describe('When removing a video in blacklist', function () { + + it('Should fail with a non authenticated user', async function () { + await command.remove({ + token: 'fake token', + videoId: servers[0].store.videoCreated.uuid, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a non admin user', async function () { + await command.remove({ + token: userAccessToken2, + videoId: servers[0].store.videoCreated.uuid, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with an incorrect id', async function () { + await command.remove({ videoId: 'hello', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with a not blacklisted video', async function () { + // The video was not added to the blacklist so it should fail + await command.remove({ videoId: notBlacklistedVideoId, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should succeed with the correct params', async function () { + await command.remove({ videoId: servers[0].store.videoCreated.uuid, expectedStatus: HttpStatusCode.NO_CONTENT_204 }) + }) + }) + + describe('When listing videos in blacklist', function () { + const basePath = '/api/v1/videos/blacklist/' + + it('Should fail with a non authenticated user', async function () { + await servers[0].blacklist.list({ token: 'fake token', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with a non admin user', async function () { + await servers[0].blacklist.list({ token: userAccessToken2, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(servers[0].url, basePath, servers[0].accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(servers[0].url, basePath, servers[0].accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(servers[0].url, basePath, servers[0].accessToken) + }) + + it('Should fail with an invalid type', async function () { + await servers[0].blacklist.list({ type: 0 as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should succeed with the correct parameters', async function () { + await servers[0].blacklist.list({ type: VideoBlacklistType.MANUAL }) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/check-params/video-captions.ts b/packages/tests/src/api/check-params/video-captions.ts new file mode 100644 index 000000000..4150b095f --- /dev/null +++ b/packages/tests/src/api/check-params/video-captions.ts @@ -0,0 +1,307 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import { HttpStatusCode, VideoCreateResult, VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + makeDeleteRequest, + makeGetRequest, + makeUploadRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test video captions API validator', function () { + const path = '/api/v1/videos/' + + let server: PeerTubeServer + let userAccessToken: string + let video: VideoCreateResult + let privateVideo: VideoCreateResult + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(120000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + + video = await server.videos.upload() + privateVideo = await server.videos.upload({ attributes: { privacy: VideoPrivacy.PRIVATE } }) + + { + const user = { + username: 'user1', + password: 'my super password' + } + await server.users.create({ username: user.username, password: user.password }) + userAccessToken = await server.login.getAccessToken(user) + } + }) + + describe('When adding video caption', function () { + const fields = { } + const attaches = { + captionfile: buildAbsoluteFixturePath('subtitle-good1.vtt') + } + + it('Should fail without a valid uuid', async function () { + await makeUploadRequest({ + method: 'PUT', + url: server.url, + path: path + '4da6fde3-88f7-4d16-b119-108df563d0b06/captions/fr', + token: server.accessToken, + fields, + attaches + }) + }) + + it('Should fail with an unknown id', async function () { + await makeUploadRequest({ + method: 'PUT', + url: server.url, + path: path + '4da6fde3-88f7-4d16-b119-108df5630b06/captions/fr', + token: server.accessToken, + fields, + attaches, + expectedStatus: 404 + }) + }) + + it('Should fail with a missing language in path', async function () { + const captionPath = path + video.uuid + '/captions' + await makeUploadRequest({ + method: 'PUT', + url: server.url, + path: captionPath, + token: server.accessToken, + fields, + attaches + }) + }) + + it('Should fail with an unknown language', async function () { + const captionPath = path + video.uuid + '/captions/15' + await makeUploadRequest({ + method: 'PUT', + url: server.url, + path: captionPath, + token: server.accessToken, + fields, + attaches + }) + }) + + it('Should fail without access token', async function () { + const captionPath = path + video.uuid + '/captions/fr' + await makeUploadRequest({ + method: 'PUT', + url: server.url, + path: captionPath, + fields, + attaches, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a bad access token', async function () { + const captionPath = path + video.uuid + '/captions/fr' + await makeUploadRequest({ + method: 'PUT', + url: server.url, + path: captionPath, + token: 'blabla', + fields, + attaches, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + // We accept any file now + // it('Should fail with an invalid captionfile extension', async function () { + // const attaches = { + // 'captionfile': buildAbsoluteFixturePath('subtitle-bad.txt') + // } + // + // const captionPath = path + video.uuid + '/captions/fr' + // await makeUploadRequest({ + // method: 'PUT', + // url: server.url, + // path: captionPath, + // token: server.accessToken, + // fields, + // attaches, + // expectedStatus: HttpStatusCode.BAD_REQUEST_400 + // }) + // }) + + // We don't check the extension yet + // it('Should fail with an invalid captionfile extension and octet-stream mime type', async function () { + // await createVideoCaption({ + // url: server.url, + // accessToken: server.accessToken, + // language: 'zh', + // videoId: video.uuid, + // fixture: 'subtitle-bad.txt', + // mimeType: 'application/octet-stream', + // expectedStatus: HttpStatusCode.BAD_REQUEST_400 + // }) + // }) + + it('Should succeed with a valid captionfile extension and octet-stream mime type', async function () { + await server.captions.add({ + language: 'zh', + videoId: video.uuid, + fixture: 'subtitle-good.srt', + mimeType: 'application/octet-stream' + }) + }) + + // We don't check the file validity yet + // it('Should fail with an invalid captionfile srt', async function () { + // const attaches = { + // 'captionfile': buildAbsoluteFixturePath('subtitle-bad.srt') + // } + // + // const captionPath = path + video.uuid + '/captions/fr' + // await makeUploadRequest({ + // method: 'PUT', + // url: server.url, + // path: captionPath, + // token: server.accessToken, + // fields, + // attaches, + // expectedStatus: HttpStatusCode.INTERNAL_SERVER_ERROR_500 + // }) + // }) + + it('Should success with the correct parameters', async function () { + const captionPath = path + video.uuid + '/captions/fr' + await makeUploadRequest({ + method: 'PUT', + url: server.url, + path: captionPath, + token: server.accessToken, + fields, + attaches, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + describe('When listing video captions', function () { + it('Should fail without a valid uuid', async function () { + await makeGetRequest({ url: server.url, path: path + '4da6fde3-88f7-4d16-b119-108df563d0b06/captions' }) + }) + + it('Should fail with an unknown id', async function () { + await makeGetRequest({ + url: server.url, + path: path + '4da6fde3-88f7-4d16-b119-108df5630b06/captions', + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail with a private video without token', async function () { + await makeGetRequest({ + url: server.url, + path: path + privateVideo.shortUUID + '/captions', + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with another user token', async function () { + await makeGetRequest({ + url: server.url, + token: userAccessToken, + path: path + privateVideo.shortUUID + '/captions', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should success with the correct parameters', async function () { + await makeGetRequest({ url: server.url, path: path + video.shortUUID + '/captions', expectedStatus: HttpStatusCode.OK_200 }) + + await makeGetRequest({ + url: server.url, + path: path + privateVideo.shortUUID + '/captions', + token: server.accessToken, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + + describe('When deleting video caption', function () { + it('Should fail without a valid uuid', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '4da6fde3-88f7-4d16-b119-108df563d0b06/captions/fr', + token: server.accessToken + }) + }) + + it('Should fail with an unknown id', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '4da6fde3-88f7-4d16-b119-108df5630b06/captions/fr', + token: server.accessToken, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail with an invalid language', async function () { + await makeDeleteRequest({ + url: server.url, + path: path + '4da6fde3-88f7-4d16-b119-108df5630b06/captions/16', + token: server.accessToken + }) + }) + + it('Should fail with a missing language', async function () { + const captionPath = path + video.shortUUID + '/captions' + await makeDeleteRequest({ url: server.url, path: captionPath, token: server.accessToken }) + }) + + it('Should fail with an unknown language', async function () { + const captionPath = path + video.shortUUID + '/captions/15' + await makeDeleteRequest({ url: server.url, path: captionPath, token: server.accessToken }) + }) + + it('Should fail without access token', async function () { + const captionPath = path + video.shortUUID + '/captions/fr' + await makeDeleteRequest({ url: server.url, path: captionPath, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with a bad access token', async function () { + const captionPath = path + video.shortUUID + '/captions/fr' + await makeDeleteRequest({ url: server.url, path: captionPath, token: 'coucou', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with another user', async function () { + const captionPath = path + video.shortUUID + '/captions/fr' + await makeDeleteRequest({ + url: server.url, + path: captionPath, + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should success with the correct parameters', async function () { + const captionPath = path + video.shortUUID + '/captions/fr' + await makeDeleteRequest({ + url: server.url, + path: captionPath, + token: server.accessToken, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/video-channel-syncs.ts b/packages/tests/src/api/check-params/video-channel-syncs.ts new file mode 100644 index 000000000..d95f3319a --- /dev/null +++ b/packages/tests/src/api/check-params/video-channel-syncs.ts @@ -0,0 +1,319 @@ +import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@tests/shared/checks.js' +import { FIXTURE_URLS } from '@tests/shared/tests.js' +import { HttpStatusCode, VideoChannelSyncCreate } from '@peertube/peertube-models' +import { + ChannelSyncsCommand, + createSingleServer, + makePostBodyRequest, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel +} from '@peertube/peertube-server-commands' + +describe('Test video channel sync API validator', () => { + const path = '/api/v1/video-channel-syncs' + let server: PeerTubeServer + let command: ChannelSyncsCommand + let rootChannelId: number + let rootChannelSyncId: number + const userInfo = { + accessToken: '', + username: 'user1', + id: -1, + channelId: -1, + syncId: -1 + } + + async function withChannelSyncDisabled (callback: () => Promise): Promise { + try { + await server.config.disableChannelSync() + await callback() + } finally { + await server.config.enableChannelSync() + } + } + + async function withMaxSyncsPerUser (maxSync: number, callback: () => Promise): Promise { + const origConfig = await server.config.getCustomConfig() + + await server.config.updateExistingSubConfig({ + newConfig: { + import: { + videoChannelSynchronization: { + maxPerUser: maxSync + } + } + } + }) + + try { + await callback() + } finally { + await server.config.updateCustomConfig({ newCustomConfig: origConfig }) + } + } + + before(async function () { + this.timeout(30_000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + command = server.channelSyncs + + rootChannelId = server.store.channel.id + + { + userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username) + + const { videoChannels, id: userId } = await server.users.getMyInfo({ token: userInfo.accessToken }) + userInfo.id = userId + userInfo.channelId = videoChannels[0].id + } + + await server.config.enableChannelSync() + }) + + describe('When creating a sync', function () { + let baseCorrectParams: VideoChannelSyncCreate + + before(function () { + baseCorrectParams = { + externalChannelUrl: FIXTURE_URLS.youtubeChannel, + videoChannelId: rootChannelId + } + }) + + it('Should fail when sync is disabled', async function () { + await withChannelSyncDisabled(async () => { + await command.create({ + token: server.accessToken, + attributes: baseCorrectParams, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + }) + + it('Should fail with nothing', async function () { + const fields = {} + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with no authentication', async function () { + await command.create({ + token: null, + attributes: baseCorrectParams, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail without a target url', async function () { + const attributes: VideoChannelSyncCreate = { + ...baseCorrectParams, + externalChannelUrl: null + } + await command.create({ + token: server.accessToken, + attributes, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail without a channelId', async function () { + const attributes: VideoChannelSyncCreate = { + ...baseCorrectParams, + videoChannelId: null + } + await command.create({ + token: server.accessToken, + attributes, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with a channelId refering nothing', async function () { + const attributes: VideoChannelSyncCreate = { + ...baseCorrectParams, + videoChannelId: 42 + } + await command.create({ + token: server.accessToken, + attributes, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail to create a sync when the user does not own the channel', async function () { + await command.create({ + token: userInfo.accessToken, + attributes: baseCorrectParams, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should succeed to create a sync with root and for another user\'s channel', async function () { + const { videoChannelSync } = await command.create({ + token: server.accessToken, + attributes: { + ...baseCorrectParams, + videoChannelId: userInfo.channelId + }, + expectedStatus: HttpStatusCode.OK_200 + }) + userInfo.syncId = videoChannelSync.id + }) + + it('Should succeed with the correct parameters', async function () { + const { videoChannelSync } = await command.create({ + token: server.accessToken, + attributes: baseCorrectParams, + expectedStatus: HttpStatusCode.OK_200 + }) + rootChannelSyncId = videoChannelSync.id + }) + + it('Should fail when the user exceeds allowed number of synchronizations', async function () { + await withMaxSyncsPerUser(1, async () => { + await command.create({ + token: server.accessToken, + attributes: { + ...baseCorrectParams, + videoChannelId: userInfo.channelId + }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + }) + }) + + describe('When listing my channel syncs', function () { + const myPath = '/api/v1/accounts/root/video-channel-syncs' + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, myPath, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, myPath, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, myPath, server.accessToken) + }) + + it('Should succeed with the correct parameters', async function () { + await command.listByAccount({ + accountName: 'root', + token: server.accessToken, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + + it('Should fail with no authentication', async function () { + await command.listByAccount({ + accountName: 'root', + token: null, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail when a simple user lists another user\'s synchronizations', async function () { + await command.listByAccount({ + accountName: 'root', + token: userInfo.accessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should succeed when root lists another user\'s synchronizations', async function () { + await command.listByAccount({ + accountName: userInfo.username, + token: server.accessToken, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + + it('Should succeed even with synchronization disabled', async function () { + await withChannelSyncDisabled(async function () { + await command.listByAccount({ + accountName: 'root', + token: server.accessToken, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + }) + + describe('When triggering deletion', function () { + it('should fail with no authentication', async function () { + await command.delete({ + channelSyncId: userInfo.syncId, + token: null, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail when channelSyncId does not refer to any sync', async function () { + await command.delete({ + channelSyncId: 42, + token: server.accessToken, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail when sync is not owned by the user', async function () { + await command.delete({ + channelSyncId: rootChannelSyncId, + token: userInfo.accessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should succeed when root delete a sync they do not own', async function () { + await command.delete({ + channelSyncId: userInfo.syncId, + token: server.accessToken, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + + it('should succeed when user delete a sync they own', async function () { + const { videoChannelSync } = await command.create({ + attributes: { + externalChannelUrl: FIXTURE_URLS.youtubeChannel, + videoChannelId: userInfo.channelId + }, + token: server.accessToken, + expectedStatus: HttpStatusCode.OK_200 + }) + + await command.delete({ + channelSyncId: videoChannelSync.id, + token: server.accessToken, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + + it('Should succeed even when synchronization is disabled', async function () { + await withChannelSyncDisabled(async function () { + await command.delete({ + channelSyncId: rootChannelSyncId, + token: server.accessToken, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + }) + + after(async function () { + await server?.kill() + }) +}) diff --git a/packages/tests/src/api/check-params/video-channels.ts b/packages/tests/src/api/check-params/video-channels.ts new file mode 100644 index 000000000..84b962b19 --- /dev/null +++ b/packages/tests/src/api/check-params/video-channels.ts @@ -0,0 +1,379 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { omit } from '@peertube/peertube-core-utils' +import { HttpStatusCode, VideoChannelUpdate } from '@peertube/peertube-models' +import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@tests/shared/checks.js' +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import { + ChannelsCommand, + cleanupTests, + createSingleServer, + makeGetRequest, + makePostBodyRequest, + makePutBodyRequest, + makeUploadRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test video channels API validator', function () { + const videoChannelPath = '/api/v1/video-channels' + let server: PeerTubeServer + const userInfo = { + accessToken: '', + channelName: 'fake_channel', + id: -1, + videoQuota: -1, + videoQuotaDaily: -1 + } + let command: ChannelsCommand + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + + const userCreds = { + username: 'fake', + password: 'fake_password' + } + + { + const user = await server.users.create({ username: userCreds.username, password: userCreds.password }) + userInfo.id = user.id + userInfo.accessToken = await server.login.getAccessToken(userCreds) + } + + command = server.channels + }) + + describe('When listing a video channels', function () { + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, videoChannelPath, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, videoChannelPath, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, videoChannelPath, server.accessToken) + }) + }) + + describe('When listing account video channels', function () { + const accountChannelPath = '/api/v1/accounts/fake/video-channels' + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, accountChannelPath, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, accountChannelPath, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, accountChannelPath, server.accessToken) + }) + + it('Should fail with a unknown account', async function () { + await server.channels.listByAccount({ accountName: 'unknown', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should succeed with the correct parameters', async function () { + await makeGetRequest({ + url: server.url, + path: accountChannelPath, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + + describe('When adding a video channel', function () { + const baseCorrectParams = { + name: 'super_channel', + displayName: 'hello', + description: 'super description', + support: 'super support text' + } + + it('Should fail with a non authenticated user', async function () { + await makePostBodyRequest({ + url: server.url, + path: videoChannelPath, + token: 'none', + fields: baseCorrectParams, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with nothing', async function () { + const fields = {} + await makePostBodyRequest({ url: server.url, path: videoChannelPath, token: server.accessToken, fields }) + }) + + it('Should fail without a name', async function () { + const fields = omit(baseCorrectParams, [ 'name' ]) + await makePostBodyRequest({ url: server.url, path: videoChannelPath, token: server.accessToken, fields }) + }) + + it('Should fail with a bad name', async function () { + const fields = { ...baseCorrectParams, name: 'super name' } + await makePostBodyRequest({ url: server.url, path: videoChannelPath, token: server.accessToken, fields }) + }) + + it('Should fail without a name', async function () { + const fields = omit(baseCorrectParams, [ 'displayName' ]) + await makePostBodyRequest({ url: server.url, path: videoChannelPath, token: server.accessToken, fields }) + }) + + it('Should fail with a long name', async function () { + const fields = { ...baseCorrectParams, displayName: 'super'.repeat(25) } + await makePostBodyRequest({ url: server.url, path: videoChannelPath, token: server.accessToken, fields }) + }) + + it('Should fail with a long description', async function () { + const fields = { ...baseCorrectParams, description: 'super'.repeat(201) } + await makePostBodyRequest({ url: server.url, path: videoChannelPath, token: server.accessToken, fields }) + }) + + it('Should fail with a long support text', async function () { + const fields = { ...baseCorrectParams, support: 'super'.repeat(201) } + await makePostBodyRequest({ url: server.url, path: videoChannelPath, token: server.accessToken, fields }) + }) + + it('Should succeed with the correct parameters', async function () { + await makePostBodyRequest({ + url: server.url, + path: videoChannelPath, + token: server.accessToken, + fields: baseCorrectParams, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + + it('Should fail when adding a channel with the same username', async function () { + await makePostBodyRequest({ + url: server.url, + path: videoChannelPath, + token: server.accessToken, + fields: baseCorrectParams, + expectedStatus: HttpStatusCode.CONFLICT_409 + }) + }) + }) + + describe('When updating a video channel', function () { + const baseCorrectParams: VideoChannelUpdate = { + displayName: 'hello', + description: 'super description', + support: 'toto', + bulkVideosSupportUpdate: false + } + let path: string + + before(async function () { + path = videoChannelPath + '/super_channel' + }) + + it('Should fail with a non authenticated user', async function () { + await makePutBodyRequest({ + url: server.url, + path, + token: 'hi', + fields: baseCorrectParams, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with another authenticated user', async function () { + await makePutBodyRequest({ + url: server.url, + path, + token: userInfo.accessToken, + fields: baseCorrectParams, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with a long name', async function () { + const fields = { ...baseCorrectParams, displayName: 'super'.repeat(25) } + await makePutBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a long description', async function () { + const fields = { ...baseCorrectParams, description: 'super'.repeat(201) } + await makePutBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a long support text', async function () { + const fields = { ...baseCorrectParams, support: 'super'.repeat(201) } + await makePutBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a bad bulkVideosSupportUpdate field', async function () { + const fields = { ...baseCorrectParams, bulkVideosSupportUpdate: 'super' } + await makePutBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should succeed with the correct parameters', async function () { + await makePutBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields: baseCorrectParams, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + describe('When updating video channel avatars/banners', function () { + const types = [ 'avatar', 'banner' ] + let path: string + + before(async function () { + path = videoChannelPath + '/super_channel' + }) + + it('Should fail with an incorrect input file', async function () { + for (const type of types) { + const fields = {} + const attaches = { + [type + 'file']: buildAbsoluteFixturePath('video_short.mp4') + } + + await makeUploadRequest({ url: server.url, path: `${path}/${type}/pick`, token: server.accessToken, fields, attaches }) + } + }) + + it('Should fail with a big file', async function () { + for (const type of types) { + const fields = {} + const attaches = { + [type + 'file']: buildAbsoluteFixturePath('avatar-big.png') + } + await makeUploadRequest({ url: server.url, path: `${path}/${type}/pick`, token: server.accessToken, fields, attaches }) + } + }) + + it('Should fail with an unauthenticated user', async function () { + for (const type of types) { + const fields = {} + const attaches = { + [type + 'file']: buildAbsoluteFixturePath('avatar.png') + } + await makeUploadRequest({ + url: server.url, + path: `${path}/${type}/pick`, + fields, + attaches, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + } + }) + + it('Should succeed with the correct params', async function () { + for (const type of types) { + const fields = {} + const attaches = { + [type + 'file']: buildAbsoluteFixturePath('avatar.png') + } + await makeUploadRequest({ + url: server.url, + path: `${path}/${type}/pick`, + token: server.accessToken, + fields, + attaches, + expectedStatus: HttpStatusCode.OK_200 + }) + } + }) + }) + + describe('When getting a video channel', function () { + it('Should return the list of the video channels with nothing', async function () { + const res = await makeGetRequest({ + url: server.url, + path: videoChannelPath, + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(res.body.data).to.be.an('array') + }) + + it('Should return 404 with an incorrect video channel', async function () { + await makeGetRequest({ + url: server.url, + path: videoChannelPath + '/super_channel2', + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should succeed with the correct parameters', async function () { + await makeGetRequest({ + url: server.url, + path: videoChannelPath + '/super_channel', + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + + describe('When getting channel followers', function () { + const path = '/api/v1/video-channels/super_channel/followers' + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a unauthenticated user', async function () { + await makeGetRequest({ url: server.url, path, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with a another user', async function () { + await makeGetRequest({ url: server.url, path, token: userInfo.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should succeed with the correct params', async function () { + await makeGetRequest({ url: server.url, path, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('When deleting a video channel', function () { + it('Should fail with a non authenticated user', async function () { + await command.delete({ token: 'coucou', channelName: 'super_channel', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with another authenticated user', async function () { + await command.delete({ token: userInfo.accessToken, channelName: 'super_channel', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail with an unknown video channel id', async function () { + await command.delete({ channelName: 'super_channel2', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should succeed with the correct parameters', async function () { + await command.delete({ channelName: 'super_channel' }) + }) + + it('Should fail to delete the last user video channel', async function () { + await command.delete({ channelName: 'root_channel', expectedStatus: HttpStatusCode.CONFLICT_409 }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/video-comments.ts b/packages/tests/src/api/check-params/video-comments.ts new file mode 100644 index 000000000..177361606 --- /dev/null +++ b/packages/tests/src/api/check-params/video-comments.ts @@ -0,0 +1,484 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@tests/shared/checks.js' +import { HttpStatusCode, VideoCreateResult, VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + makeDeleteRequest, + makeGetRequest, + makePostBodyRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test video comments API validator', function () { + let pathThread: string + let pathComment: string + + let server: PeerTubeServer + + let video: VideoCreateResult + + let userAccessToken: string + let userAccessToken2: string + + let commentId: number + let privateCommentId: number + let privateVideo: VideoCreateResult + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(120000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + + { + video = await server.videos.upload({ attributes: {} }) + pathThread = '/api/v1/videos/' + video.uuid + '/comment-threads' + } + + { + privateVideo = await server.videos.upload({ attributes: { privacy: VideoPrivacy.PRIVATE } }) + } + + { + const created = await server.comments.createThread({ videoId: video.uuid, text: 'coucou' }) + commentId = created.id + pathComment = '/api/v1/videos/' + video.uuid + '/comments/' + commentId + } + + { + const created = await server.comments.createThread({ videoId: privateVideo.uuid, text: 'coucou' }) + privateCommentId = created.id + } + + { + const user = { username: 'user1', password: 'my super password' } + await server.users.create({ username: user.username, password: user.password }) + userAccessToken = await server.login.getAccessToken(user) + } + + { + const user = { username: 'user2', password: 'my super password' } + await server.users.create({ username: user.username, password: user.password }) + userAccessToken2 = await server.login.getAccessToken(user) + } + }) + + describe('When listing video comment threads', function () { + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, pathThread, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, pathThread, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, pathThread, server.accessToken) + }) + + it('Should fail with an incorrect video', async function () { + await makeGetRequest({ + url: server.url, + path: '/api/v1/videos/ba708d62-e3d7-45d9-9d73-41b9097cc02d/comment-threads', + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail with a private video without token', async function () { + await makeGetRequest({ + url: server.url, + path: '/api/v1/videos/' + privateVideo.shortUUID + '/comment-threads', + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with another user token', async function () { + await makeGetRequest({ + url: server.url, + token: userAccessToken, + path: '/api/v1/videos/' + privateVideo.shortUUID + '/comment-threads', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should succeed with the correct params', async function () { + await makeGetRequest({ + url: server.url, + token: server.accessToken, + path: '/api/v1/videos/' + privateVideo.shortUUID + '/comment-threads', + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + + describe('When listing comments of a thread', function () { + it('Should fail with an incorrect video', async function () { + await makeGetRequest({ + url: server.url, + path: '/api/v1/videos/ba708d62-e3d7-45d9-9d73-41b9097cc02d/comment-threads/' + commentId, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail with an incorrect thread id', async function () { + await makeGetRequest({ + url: server.url, + path: '/api/v1/videos/' + video.shortUUID + '/comment-threads/156', + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail with a private video without token', async function () { + await makeGetRequest({ + url: server.url, + path: '/api/v1/videos/' + privateVideo.shortUUID + '/comment-threads/' + privateCommentId, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with another user token', async function () { + await makeGetRequest({ + url: server.url, + token: userAccessToken, + path: '/api/v1/videos/' + privateVideo.shortUUID + '/comment-threads/' + privateCommentId, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should success with the correct params', async function () { + await makeGetRequest({ + url: server.url, + token: server.accessToken, + path: '/api/v1/videos/' + privateVideo.shortUUID + '/comment-threads/' + privateCommentId, + expectedStatus: HttpStatusCode.OK_200 + }) + + await makeGetRequest({ + url: server.url, + path: '/api/v1/videos/' + video.shortUUID + '/comment-threads/' + commentId, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + + describe('When adding a video thread', function () { + + it('Should fail with a non authenticated user', async function () { + const fields = { + text: 'text' + } + await makePostBodyRequest({ + url: server.url, + path: pathThread, + token: 'none', + fields, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with nothing', async function () { + const fields = {} + await makePostBodyRequest({ url: server.url, path: pathThread, token: server.accessToken, fields }) + }) + + it('Should fail with a short comment', async function () { + const fields = { + text: '' + } + await makePostBodyRequest({ url: server.url, path: pathThread, token: server.accessToken, fields }) + }) + + it('Should fail with a long comment', async function () { + const fields = { + text: 'h'.repeat(10001) + } + await makePostBodyRequest({ url: server.url, path: pathThread, token: server.accessToken, fields }) + }) + + it('Should fail with an incorrect video', async function () { + const path = '/api/v1/videos/ba708d62-e3d7-45d9-9d73-41b9097cc02d/comment-threads' + const fields = { text: 'super comment' } + + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail with a private video of another user', async function () { + const fields = { text: 'super comment' } + + await makePostBodyRequest({ + url: server.url, + path: '/api/v1/videos/' + privateVideo.shortUUID + '/comment-threads', + token: userAccessToken, + fields, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should succeed with the correct parameters', async function () { + const fields = { text: 'super comment' } + + await makePostBodyRequest({ + url: server.url, + path: pathThread, + token: server.accessToken, + fields, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + + describe('When adding a comment to a thread', function () { + + it('Should fail with a non authenticated user', async function () { + const fields = { + text: 'text' + } + await makePostBodyRequest({ + url: server.url, + path: pathComment, + token: 'none', + fields, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with nothing', async function () { + const fields = {} + await makePostBodyRequest({ url: server.url, path: pathComment, token: server.accessToken, fields }) + }) + + it('Should fail with a short comment', async function () { + const fields = { + text: '' + } + await makePostBodyRequest({ url: server.url, path: pathComment, token: server.accessToken, fields }) + }) + + it('Should fail with a long comment', async function () { + const fields = { + text: 'h'.repeat(10001) + } + await makePostBodyRequest({ url: server.url, path: pathComment, token: server.accessToken, fields }) + }) + + it('Should fail with an incorrect video', async function () { + const path = '/api/v1/videos/ba708d62-e3d7-45d9-9d73-41b9097cc02d/comments/' + commentId + const fields = { + text: 'super comment' + } + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail with a private video of another user', async function () { + const fields = { text: 'super comment' } + + await makePostBodyRequest({ + url: server.url, + path: '/api/v1/videos/' + privateVideo.uuid + '/comments/' + privateCommentId, + token: userAccessToken, + fields, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with an incorrect comment', async function () { + const path = '/api/v1/videos/' + video.uuid + '/comments/124' + const fields = { + text: 'super comment' + } + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should succeed with the correct parameters', async function () { + const fields = { + text: 'super comment' + } + await makePostBodyRequest({ + url: server.url, + path: pathComment, + token: server.accessToken, + fields, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + + describe('When removing video comments', function () { + it('Should fail with a non authenticated user', async function () { + await makeDeleteRequest({ url: server.url, path: pathComment, token: 'none', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with another user', async function () { + await makeDeleteRequest({ + url: server.url, + path: pathComment, + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with an incorrect video', async function () { + const path = '/api/v1/videos/ba708d62-e3d7-45d9-9d73-41b9097cc02d/comments/' + commentId + await makeDeleteRequest({ url: server.url, path, token: server.accessToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail with an incorrect comment', async function () { + const path = '/api/v1/videos/' + video.uuid + '/comments/124' + await makeDeleteRequest({ url: server.url, path, token: server.accessToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should succeed with the same user', async function () { + let commentToDelete: number + + { + const created = await server.comments.createThread({ videoId: video.uuid, token: userAccessToken, text: 'hello' }) + commentToDelete = created.id + } + + const path = '/api/v1/videos/' + video.uuid + '/comments/' + commentToDelete + + await makeDeleteRequest({ url: server.url, path, token: userAccessToken2, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeDeleteRequest({ url: server.url, path, token: userAccessToken, expectedStatus: HttpStatusCode.NO_CONTENT_204 }) + }) + + it('Should succeed with the owner of the video', async function () { + let commentToDelete: number + let anotherVideoUUID: string + + { + const { uuid } = await server.videos.upload({ token: userAccessToken, attributes: { name: 'video' } }) + anotherVideoUUID = uuid + } + + { + const created = await server.comments.createThread({ videoId: anotherVideoUUID, text: 'hello' }) + commentToDelete = created.id + } + + const path = '/api/v1/videos/' + anotherVideoUUID + '/comments/' + commentToDelete + + await makeDeleteRequest({ url: server.url, path, token: userAccessToken2, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeDeleteRequest({ url: server.url, path, token: userAccessToken, expectedStatus: HttpStatusCode.NO_CONTENT_204 }) + }) + + it('Should succeed with the correct parameters', async function () { + await makeDeleteRequest({ + url: server.url, + path: pathComment, + token: server.accessToken, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + describe('When a video has comments disabled', function () { + before(async function () { + video = await server.videos.upload({ attributes: { commentsEnabled: false } }) + pathThread = '/api/v1/videos/' + video.uuid + '/comment-threads' + }) + + it('Should return an empty thread list', async function () { + const res = await makeGetRequest({ + url: server.url, + path: pathThread, + expectedStatus: HttpStatusCode.OK_200 + }) + expect(res.body.total).to.equal(0) + expect(res.body.data).to.have.lengthOf(0) + }) + + it('Should return an thread comments list') + + it('Should return conflict on thread add', async function () { + const fields = { + text: 'super comment' + } + await makePostBodyRequest({ + url: server.url, + path: pathThread, + token: server.accessToken, + fields, + expectedStatus: HttpStatusCode.CONFLICT_409 + }) + }) + + it('Should return conflict on comment thread add') + }) + + describe('When listing admin comments threads', function () { + const path = '/api/v1/videos/comments' + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a non authenticated user', async function () { + await makeGetRequest({ + url: server.url, + path, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a non admin user', async function () { + await makeGetRequest({ + url: server.url, + path, + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should succeed with the correct params', async function () { + await makeGetRequest({ + url: server.url, + path, + token: server.accessToken, + query: { + isLocal: false, + search: 'toto', + searchAccount: 'toto', + searchVideo: 'toto' + }, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/video-files.ts b/packages/tests/src/api/check-params/video-files.ts new file mode 100644 index 000000000..b5819ff19 --- /dev/null +++ b/packages/tests/src/api/check-params/video-files.ts @@ -0,0 +1,195 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { getAllFiles } from '@peertube/peertube-core-utils' +import { HttpStatusCode, UserRole, VideoDetails, VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + makeRawRequest, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test videos files', function () { + let servers: PeerTubeServer[] + + let userToken: string + let moderatorToken: string + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(300_000) + + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + + await doubleFollow(servers[0], servers[1]) + + userToken = await servers[0].users.generateUserAndToken('user', UserRole.USER) + moderatorToken = await servers[0].users.generateUserAndToken('moderator', UserRole.MODERATOR) + }) + + describe('Getting metadata', function () { + let video: VideoDetails + + before(async function () { + const { uuid } = await servers[0].videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) + video = await servers[0].videos.getWithToken({ id: uuid }) + }) + + it('Should not get metadata of private video without token', async function () { + for (const file of getAllFiles(video)) { + await makeRawRequest({ url: file.metadataUrl, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + } + }) + + it('Should not get metadata of private video without the appropriate token', async function () { + for (const file of getAllFiles(video)) { + await makeRawRequest({ url: file.metadataUrl, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + } + }) + + it('Should get metadata of private video with the appropriate token', async function () { + for (const file of getAllFiles(video)) { + await makeRawRequest({ url: file.metadataUrl, token: servers[0].accessToken, expectedStatus: HttpStatusCode.OK_200 }) + } + }) + }) + + describe('Deleting files', function () { + let webVideoId: string + let hlsId: string + let remoteId: string + + let validId1: string + let validId2: string + + let hlsFileId: number + let webVideoFileId: number + + let remoteHLSFileId: number + let remoteWebVideoFileId: number + + before(async function () { + this.timeout(300_000) + + { + const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' }) + await waitJobs(servers) + + const video = await servers[1].videos.get({ id: uuid }) + remoteId = video.uuid + remoteHLSFileId = video.streamingPlaylists[0].files[0].id + remoteWebVideoFileId = video.files[0].id + } + + { + await servers[0].config.enableTranscoding({ hls: true, webVideo: true }) + + { + const { uuid } = await servers[0].videos.quickUpload({ name: 'both 1' }) + await waitJobs(servers) + + const video = await servers[0].videos.get({ id: uuid }) + validId1 = video.uuid + hlsFileId = video.streamingPlaylists[0].files[0].id + webVideoFileId = video.files[0].id + } + + { + const { uuid } = await servers[0].videos.quickUpload({ name: 'both 2' }) + validId2 = uuid + } + } + + await waitJobs(servers) + + { + await servers[0].config.enableTranscoding({ hls: true, webVideo: false }) + const { uuid } = await servers[0].videos.quickUpload({ name: 'hls' }) + hlsId = uuid + } + + await waitJobs(servers) + + { + await servers[0].config.enableTranscoding({ webVideo: true, hls: false }) + const { uuid } = await servers[0].videos.quickUpload({ name: 'web-video' }) + webVideoId = uuid + } + + await waitJobs(servers) + }) + + it('Should not delete files of a unknown video', async function () { + const expectedStatus = HttpStatusCode.NOT_FOUND_404 + + await servers[0].videos.removeHLSPlaylist({ videoId: 404, expectedStatus }) + await servers[0].videos.removeAllWebVideoFiles({ videoId: 404, expectedStatus }) + + await servers[0].videos.removeHLSFile({ videoId: 404, fileId: hlsFileId, expectedStatus }) + await servers[0].videos.removeWebVideoFile({ videoId: 404, fileId: webVideoFileId, expectedStatus }) + }) + + it('Should not delete unknown files', async function () { + const expectedStatus = HttpStatusCode.NOT_FOUND_404 + + await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: webVideoFileId, expectedStatus }) + await servers[0].videos.removeWebVideoFile({ videoId: validId1, fileId: hlsFileId, expectedStatus }) + }) + + it('Should not delete files of a remote video', async function () { + const expectedStatus = HttpStatusCode.BAD_REQUEST_400 + + await servers[0].videos.removeHLSPlaylist({ videoId: remoteId, expectedStatus }) + await servers[0].videos.removeAllWebVideoFiles({ videoId: remoteId, expectedStatus }) + + await servers[0].videos.removeHLSFile({ videoId: remoteId, fileId: remoteHLSFileId, expectedStatus }) + await servers[0].videos.removeWebVideoFile({ videoId: remoteId, fileId: remoteWebVideoFileId, expectedStatus }) + }) + + it('Should not delete files by a non admin user', async function () { + const expectedStatus = HttpStatusCode.FORBIDDEN_403 + + await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: userToken, expectedStatus }) + await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: moderatorToken, expectedStatus }) + + await servers[0].videos.removeAllWebVideoFiles({ videoId: validId1, token: userToken, expectedStatus }) + await servers[0].videos.removeAllWebVideoFiles({ videoId: validId1, token: moderatorToken, expectedStatus }) + + await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: userToken, expectedStatus }) + await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: moderatorToken, expectedStatus }) + + await servers[0].videos.removeWebVideoFile({ videoId: validId1, fileId: webVideoFileId, token: userToken, expectedStatus }) + await servers[0].videos.removeWebVideoFile({ videoId: validId1, fileId: webVideoFileId, token: moderatorToken, expectedStatus }) + }) + + it('Should not delete files if the files are not available', async function () { + await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await servers[0].videos.removeAllWebVideoFiles({ videoId: webVideoId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + + await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + await servers[0].videos.removeWebVideoFile({ videoId: webVideoId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should not delete files if no both versions are available', async function () { + await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await servers[0].videos.removeAllWebVideoFiles({ videoId: webVideoId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should delete files if both versions are available', async function () { + await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId }) + await servers[0].videos.removeWebVideoFile({ videoId: validId1, fileId: webVideoFileId }) + + await servers[0].videos.removeHLSPlaylist({ videoId: validId1 }) + await servers[0].videos.removeAllWebVideoFiles({ videoId: validId2 }) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/check-params/video-imports.ts b/packages/tests/src/api/check-params/video-imports.ts new file mode 100644 index 000000000..e078cedd6 --- /dev/null +++ b/packages/tests/src/api/check-params/video-imports.ts @@ -0,0 +1,433 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { omit } from '@peertube/peertube-core-utils' +import { HttpStatusCode, VideoPrivacy } from '@peertube/peertube-models' +import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@tests/shared/checks.js' +import { FIXTURE_URLS } from '@tests/shared/tests.js' +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import { + cleanupTests, + createSingleServer, + makeGetRequest, + makePostBodyRequest, + makeUploadRequest, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test video imports API validator', function () { + const path = '/api/v1/videos/imports' + let server: PeerTubeServer + let userAccessToken = '' + let channelId: number + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + const username = 'user1' + const password = 'my super password' + await server.users.create({ username, password }) + userAccessToken = await server.login.getAccessToken({ username, password }) + + { + const { videoChannels } = await server.users.getMyInfo() + channelId = videoChannels[0].id + } + }) + + describe('When listing my video imports', function () { + const myPath = '/api/v1/users/me/videos/imports' + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, myPath, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, myPath, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, myPath, server.accessToken) + }) + + it('Should fail with a bad videoChannelSyncId param', async function () { + await makeGetRequest({ + url: server.url, + path: myPath, + query: { videoChannelSyncId: 'toto' }, + token: server.accessToken + }) + }) + + it('Should success with the correct parameters', async function () { + await makeGetRequest({ url: server.url, path: myPath, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken }) + }) + }) + + describe('When adding a video import', function () { + let baseCorrectParams + + before(function () { + baseCorrectParams = { + targetUrl: FIXTURE_URLS.goodVideo, + name: 'my super name', + category: 5, + licence: 1, + language: 'pt', + nsfw: false, + commentsEnabled: true, + downloadEnabled: true, + waitTranscoding: true, + description: 'my super description', + support: 'my super support text', + tags: [ 'tag1', 'tag2' ], + privacy: VideoPrivacy.PUBLIC, + channelId + } + }) + + it('Should fail with nothing', async function () { + const fields = {} + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail without a target url', async function () { + const fields = omit(baseCorrectParams, [ 'targetUrl' ]) + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with a bad target url', async function () { + const fields = { ...baseCorrectParams, targetUrl: 'htt://hello' } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with localhost', async function () { + const fields = { ...baseCorrectParams, targetUrl: 'http://localhost:8000' } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a private IP target urls', async function () { + const targetUrls = [ + 'http://127.0.0.1:8000', + 'http://127.0.0.1', + 'http://127.0.0.1/hello', + 'https://192.168.1.42', + 'http://192.168.1.42', + 'http://127.0.0.1.cpy.re' + ] + + for (const targetUrl of targetUrls) { + const fields = { ...baseCorrectParams, targetUrl } + + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + } + }) + + it('Should fail with a long name', async function () { + const fields = { ...baseCorrectParams, name: 'super'.repeat(65) } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a bad category', async function () { + const fields = { ...baseCorrectParams, category: 125 } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a bad licence', async function () { + const fields = { ...baseCorrectParams, licence: 125 } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a bad language', async function () { + const fields = { ...baseCorrectParams, language: 'a'.repeat(15) } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a long description', async function () { + const fields = { ...baseCorrectParams, description: 'super'.repeat(2500) } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a long support text', async function () { + const fields = { ...baseCorrectParams, support: 'super'.repeat(201) } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail without a channel', async function () { + const fields = omit(baseCorrectParams, [ 'channelId' ]) + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a bad channel', async function () { + const fields = { ...baseCorrectParams, channelId: 545454 } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with another user channel', async function () { + const user = { + username: 'fake', + password: 'fake_password' + } + await server.users.create({ username: user.username, password: user.password }) + + const accessTokenUser = await server.login.getAccessToken(user) + const { videoChannels } = await server.users.getMyInfo({ token: accessTokenUser }) + const customChannelId = videoChannels[0].id + + const fields = { ...baseCorrectParams, channelId: customChannelId } + + await makePostBodyRequest({ url: server.url, path, token: userAccessToken, fields }) + }) + + it('Should fail with too many tags', async function () { + const fields = { ...baseCorrectParams, tags: [ 'tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6' ] } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a tag length too low', async function () { + const fields = { ...baseCorrectParams, tags: [ 'tag1', 't' ] } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with a tag length too big', async function () { + const fields = { ...baseCorrectParams, tags: [ 'tag1', 'my_super_tag_too_long_long_long_long_long_long' ] } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail with an incorrect thumbnail file', async function () { + const fields = baseCorrectParams + const attaches = { + thumbnailfile: buildAbsoluteFixturePath('video_short.mp4') + } + + await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches }) + }) + + it('Should fail with a big thumbnail file', async function () { + const fields = baseCorrectParams + const attaches = { + thumbnailfile: buildAbsoluteFixturePath('custom-preview-big.png') + } + + await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches }) + }) + + it('Should fail with an incorrect preview file', async function () { + const fields = baseCorrectParams + const attaches = { + previewfile: buildAbsoluteFixturePath('video_short.mp4') + } + + await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches }) + }) + + it('Should fail with a big preview file', async function () { + const fields = baseCorrectParams + const attaches = { + previewfile: buildAbsoluteFixturePath('custom-preview-big.png') + } + + await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches }) + }) + + it('Should fail with an invalid torrent file', async function () { + const fields = omit(baseCorrectParams, [ 'targetUrl' ]) + const attaches = { + torrentfile: buildAbsoluteFixturePath('avatar-big.png') + } + + await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches }) + }) + + it('Should fail with an invalid magnet URI', async function () { + let fields = omit(baseCorrectParams, [ 'targetUrl' ]) + fields = { ...fields, magnetUri: 'blabla' } + + await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should succeed with the correct parameters', async function () { + this.timeout(120000) + + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields: baseCorrectParams, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + + it('Should forbid to import http videos', async function () { + await server.config.updateCustomSubConfig({ + newConfig: { + import: { + videos: { + http: { + enabled: false + }, + torrent: { + enabled: true + } + } + } + } + }) + + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields: baseCorrectParams, + expectedStatus: HttpStatusCode.CONFLICT_409 + }) + }) + + it('Should forbid to import torrent videos', async function () { + await server.config.updateCustomSubConfig({ + newConfig: { + import: { + videos: { + http: { + enabled: true + }, + torrent: { + enabled: false + } + } + } + } + }) + + let fields = omit(baseCorrectParams, [ 'targetUrl' ]) + fields = { ...fields, magnetUri: FIXTURE_URLS.magnet } + + await makePostBodyRequest({ + url: server.url, + path, + token: server.accessToken, + fields, + expectedStatus: HttpStatusCode.CONFLICT_409 + }) + + fields = omit(fields, [ 'magnetUri' ]) + const attaches = { + torrentfile: buildAbsoluteFixturePath('video-720p.torrent') + } + + await makeUploadRequest({ + url: server.url, + path, + token: server.accessToken, + fields, + attaches, + expectedStatus: HttpStatusCode.CONFLICT_409 + }) + }) + }) + + describe('Deleting/cancelling a video import', function () { + let importId: number + + async function importVideo () { + const attributes = { channelId: server.store.channel.id, targetUrl: FIXTURE_URLS.goodVideo } + const res = await server.imports.importVideo({ attributes }) + + return res.id + } + + before(async function () { + importId = await importVideo() + }) + + it('Should fail with an invalid import id', async function () { + await server.imports.cancel({ importId: 'artyom' as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await server.imports.delete({ importId: 'artyom' as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with an unknown import id', async function () { + await server.imports.cancel({ importId: 42, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + await server.imports.delete({ importId: 42, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail without token', async function () { + await server.imports.cancel({ importId, token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + await server.imports.delete({ importId, token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with another user token', async function () { + await server.imports.cancel({ importId, token: userAccessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await server.imports.delete({ importId, token: userAccessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail to cancel non pending import', async function () { + this.timeout(60000) + + await waitJobs([ server ]) + + await server.imports.cancel({ importId, expectedStatus: HttpStatusCode.CONFLICT_409 }) + }) + + it('Should succeed to delete an import', async function () { + await server.imports.delete({ importId }) + }) + + it('Should fail to delete a pending import', async function () { + await server.jobs.pauseJobQueue() + + importId = await importVideo() + + await server.imports.delete({ importId, expectedStatus: HttpStatusCode.CONFLICT_409 }) + }) + + it('Should succeed to cancel an import', async function () { + importId = await importVideo() + + await server.imports.cancel({ importId }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/video-passwords.ts b/packages/tests/src/api/check-params/video-passwords.ts new file mode 100644 index 000000000..3f57ebe74 --- /dev/null +++ b/packages/tests/src/api/check-params/video-passwords.ts @@ -0,0 +1,604 @@ +import { expect } from 'chai' +import { + HttpStatusCode, + HttpStatusCodeType, + PeerTubeProblemDocument, + ServerErrorCode, + VideoCreateResult, + VideoPrivacy +} from '@peertube/peertube-models' +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import { + cleanupTests, + createSingleServer, + makePostBodyRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' +import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@tests/shared/checks.js' +import { FIXTURE_URLS } from '@tests/shared/tests.js' +import { checkUploadVideoParam } from '@tests/shared/videos.js' + +describe('Test video passwords validator', function () { + let path: string + let server: PeerTubeServer + let userAccessToken = '' + let video: VideoCreateResult + let channelId: number + let publicVideo: VideoCreateResult + let commentId: number + // --------------------------------------------------------------- + + before(async function () { + this.timeout(50000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + + await server.config.updateCustomSubConfig({ + newConfig: { + live: { + enabled: true, + latencySetting: { + enabled: false + }, + allowReplay: false + }, + import: { + videos: { + http:{ + enabled: true + } + } + } + } + }) + + userAccessToken = await server.users.generateUserAndToken('user1') + + { + const body = await server.users.getMyInfo() + channelId = body.videoChannels[0].id + } + + { + video = await server.videos.quickUpload({ + name: 'password protected video', + privacy: VideoPrivacy.PASSWORD_PROTECTED, + videoPasswords: [ 'password1', 'password2' ] + }) + } + path = '/api/v1/videos/' + }) + + async function checkVideoPasswordOptions (options: { + server: PeerTubeServer + token: string + videoPasswords: string[] + expectedStatus: HttpStatusCodeType + mode: 'uploadLegacy' | 'uploadResumable' | 'import' | 'updateVideo' | 'updatePasswords' | 'live' + }) { + const { server, token, videoPasswords, expectedStatus = HttpStatusCode.OK_200, mode } = options + const attaches = { + fixture: buildAbsoluteFixturePath('video_short.webm') + } + const baseCorrectParams = { + name: 'my super name', + category: 5, + licence: 1, + language: 'pt', + nsfw: false, + commentsEnabled: true, + downloadEnabled: true, + waitTranscoding: true, + description: 'my super description', + support: 'my super support text', + tags: [ 'tag1', 'tag2' ], + privacy: VideoPrivacy.PASSWORD_PROTECTED, + channelId, + originallyPublishedAt: new Date().toISOString() + } + if (mode === 'uploadLegacy') { + const fields = { ...baseCorrectParams, videoPasswords } + return checkUploadVideoParam({ server, token, attributes: { ...fields, ...attaches }, expectedStatus, mode: 'legacy' }) + } + + if (mode === 'uploadResumable') { + const fields = { ...baseCorrectParams, videoPasswords } + return checkUploadVideoParam({ server, token, attributes: { ...fields, ...attaches }, expectedStatus, mode: 'resumable' }) + } + + if (mode === 'import') { + const attributes = { ...baseCorrectParams, targetUrl: FIXTURE_URLS.goodVideo, videoPasswords } + return server.imports.importVideo({ attributes, expectedStatus }) + } + + if (mode === 'updateVideo') { + const attributes = { ...baseCorrectParams, videoPasswords } + return server.videos.update({ token, expectedStatus, id: video.id, attributes }) + } + + if (mode === 'updatePasswords') { + return server.videoPasswords.updateAll({ token, expectedStatus, videoId: video.id, passwords: videoPasswords }) + } + + if (mode === 'live') { + const fields = { ...baseCorrectParams, videoPasswords } + + return server.live.create({ fields, expectedStatus }) + } + } + + function validateVideoPasswordList (mode: 'uploadLegacy' | 'uploadResumable' | 'import' | 'updateVideo' | 'updatePasswords' | 'live') { + + it('Should fail with a password protected privacy without providing a password', async function () { + await checkVideoPasswordOptions({ + server, + token: server.accessToken, + videoPasswords: undefined, + expectedStatus: HttpStatusCode.BAD_REQUEST_400, + mode + }) + }) + + it('Should fail with a password protected privacy and an empty password list', async function () { + const videoPasswords = [] + + await checkVideoPasswordOptions({ + server, + token: server.accessToken, + videoPasswords, + expectedStatus: HttpStatusCode.BAD_REQUEST_400, + mode + }) + }) + + it('Should fail with a password protected privacy and a too short password', async function () { + const videoPasswords = [ 'p' ] + + await checkVideoPasswordOptions({ + server, + token: server.accessToken, + videoPasswords, + expectedStatus: HttpStatusCode.BAD_REQUEST_400, + mode + }) + }) + + it('Should fail with a password protected privacy and a too long password', async function () { + const videoPasswords = [ 'Very very very very very very very very very very very very very very very very very very long password' ] + + await checkVideoPasswordOptions({ + server, + token: server.accessToken, + videoPasswords, + expectedStatus: HttpStatusCode.BAD_REQUEST_400, + mode + }) + }) + + it('Should fail with a password protected privacy and an empty password', async function () { + const videoPasswords = [ '' ] + + await checkVideoPasswordOptions({ + server, + token: server.accessToken, + videoPasswords, + expectedStatus: HttpStatusCode.BAD_REQUEST_400, + mode + }) + }) + + it('Should fail with a password protected privacy and duplicated passwords', async function () { + const videoPasswords = [ 'password', 'password' ] + + await checkVideoPasswordOptions({ + server, + token: server.accessToken, + videoPasswords, + expectedStatus: HttpStatusCode.BAD_REQUEST_400, + mode + }) + }) + + if (mode === 'updatePasswords') { + it('Should fail for an unauthenticated user', async function () { + const videoPasswords = [ 'password' ] + await checkVideoPasswordOptions({ + server, + token: null, + videoPasswords, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401, + mode + }) + }) + + it('Should fail for an unauthorized user', async function () { + const videoPasswords = [ 'password' ] + await checkVideoPasswordOptions({ + server, + token: userAccessToken, + videoPasswords, + expectedStatus: HttpStatusCode.FORBIDDEN_403, + mode + }) + }) + } + + it('Should succeed with a password protected privacy and correct passwords', async function () { + const videoPasswords = [ 'password1', 'password2' ] + const expectedStatus = mode === 'updatePasswords' || mode === 'updateVideo' + ? HttpStatusCode.NO_CONTENT_204 + : HttpStatusCode.OK_200 + + await checkVideoPasswordOptions({ server, token: server.accessToken, videoPasswords, expectedStatus, mode }) + }) + } + + describe('When adding or updating a video', function () { + describe('Resumable upload', function () { + validateVideoPasswordList('uploadResumable') + }) + + describe('Legacy upload', function () { + validateVideoPasswordList('uploadLegacy') + }) + + describe('When importing a video', function () { + validateVideoPasswordList('import') + }) + + describe('When updating a video', function () { + validateVideoPasswordList('updateVideo') + }) + + describe('When updating the password list of a video', function () { + validateVideoPasswordList('updatePasswords') + }) + + describe('When creating a live', function () { + validateVideoPasswordList('live') + }) + }) + + async function checkVideoAccessOptions (options: { + server: PeerTubeServer + token?: string + videoPassword?: string + expectedStatus: HttpStatusCodeType + mode: 'get' | 'getWithPassword' | 'getWithToken' | 'listCaptions' | 'createThread' | 'listThreads' | 'replyThread' | 'rate' | 'token' + }) { + const { server, token = null, videoPassword, expectedStatus, mode } = options + + if (mode === 'get') { + return server.videos.get({ id: video.id, expectedStatus }) + } + + if (mode === 'getWithToken') { + return server.videos.getWithToken({ + id: video.id, + token, + expectedStatus + }) + } + + if (mode === 'getWithPassword') { + return server.videos.getWithPassword({ + id: video.id, + token, + expectedStatus, + password: videoPassword + }) + } + + if (mode === 'rate') { + return server.videos.rate({ + id: video.id, + token, + expectedStatus, + rating: 'like', + videoPassword + }) + } + + if (mode === 'createThread') { + const fields = { text: 'super comment' } + const headers = videoPassword !== undefined && videoPassword !== null + ? { 'x-peertube-video-password': videoPassword } + : undefined + const body = await makePostBodyRequest({ + url: server.url, + path: path + video.uuid + '/comment-threads', + token, + fields, + headers, + expectedStatus + }) + return JSON.parse(body.text) + } + + if (mode === 'replyThread') { + const fields = { text: 'super reply' } + const headers = videoPassword !== undefined && videoPassword !== null + ? { 'x-peertube-video-password': videoPassword } + : undefined + return makePostBodyRequest({ + url: server.url, + path: path + video.uuid + '/comments/' + commentId, + token, + fields, + headers, + expectedStatus + }) + } + if (mode === 'listThreads') { + return server.comments.listThreads({ + videoId: video.id, + token, + expectedStatus, + videoPassword + }) + } + + if (mode === 'listCaptions') { + return server.captions.list({ + videoId: video.id, + token, + expectedStatus, + videoPassword + }) + } + + if (mode === 'token') { + return server.videoToken.create({ + videoId: video.id, + token, + expectedStatus, + videoPassword + }) + } + } + + function checkVideoError (error: any, mode: 'providePassword' | 'incorrectPassword') { + const serverCode = mode === 'providePassword' + ? ServerErrorCode.VIDEO_REQUIRES_PASSWORD + : ServerErrorCode.INCORRECT_VIDEO_PASSWORD + + const message = mode === 'providePassword' + ? 'Please provide a password to access this password protected video' + : 'Incorrect video password. Access to the video is denied.' + + if (!error.code) { + error = JSON.parse(error.text) + } + + expect(error.code).to.equal(serverCode) + expect(error.detail).to.equal(message) + expect(error.error).to.equal(message) + + expect(error.status).to.equal(HttpStatusCode.FORBIDDEN_403) + } + + function validateVideoAccess (mode: 'get' | 'listCaptions' | 'createThread' | 'listThreads' | 'replyThread' | 'rate' | 'token') { + const requiresUserAuth = [ 'createThread', 'replyThread', 'rate' ].includes(mode) + let tokens: string[] + if (!requiresUserAuth) { + it('Should fail without providing a password for an unlogged user', async function () { + const body = await checkVideoAccessOptions({ server, expectedStatus: HttpStatusCode.FORBIDDEN_403, mode }) + const error = body as unknown as PeerTubeProblemDocument + + checkVideoError(error, 'providePassword') + }) + } + + it('Should fail without providing a password for an unauthorised user', async function () { + const tmp = mode === 'get' ? 'getWithToken' : mode + + const body = await checkVideoAccessOptions({ + server, + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403, + mode: tmp + }) + + const error = body as unknown as PeerTubeProblemDocument + + checkVideoError(error, 'providePassword') + }) + + it('Should fail if a wrong password is entered', async function () { + const tmp = mode === 'get' ? 'getWithPassword' : mode + tokens = [ userAccessToken, server.accessToken ] + + if (!requiresUserAuth) tokens.push(null) + + for (const token of tokens) { + const body = await checkVideoAccessOptions({ + server, + token, + videoPassword: 'toto', + expectedStatus: HttpStatusCode.FORBIDDEN_403, + mode: tmp + }) + const error = body as unknown as PeerTubeProblemDocument + + checkVideoError(error, 'incorrectPassword') + } + }) + + it('Should fail if an empty password is entered', async function () { + const tmp = mode === 'get' ? 'getWithPassword' : mode + + for (const token of tokens) { + const body = await checkVideoAccessOptions({ + server, + token, + videoPassword: '', + expectedStatus: HttpStatusCode.FORBIDDEN_403, + mode: tmp + }) + const error = body as unknown as PeerTubeProblemDocument + + checkVideoError(error, 'incorrectPassword') + } + }) + + it('Should fail if an inccorect password containing the correct password is entered', async function () { + const tmp = mode === 'get' ? 'getWithPassword' : mode + + for (const token of tokens) { + const body = await checkVideoAccessOptions({ + server, + token, + videoPassword: 'password11', + expectedStatus: HttpStatusCode.FORBIDDEN_403, + mode: tmp + }) + const error = body as unknown as PeerTubeProblemDocument + + checkVideoError(error, 'incorrectPassword') + } + }) + + it('Should succeed without providing a password for an authorised user', async function () { + const tmp = mode === 'get' ? 'getWithToken' : mode + const expectedStatus = mode === 'rate' ? HttpStatusCode.NO_CONTENT_204 : HttpStatusCode.OK_200 + + const body = await checkVideoAccessOptions({ server, token: server.accessToken, expectedStatus, mode: tmp }) + + if (mode === 'createThread') commentId = body.comment.id + }) + + it('Should succeed using correct passwords', async function () { + const tmp = mode === 'get' ? 'getWithPassword' : mode + const expectedStatus = mode === 'rate' ? HttpStatusCode.NO_CONTENT_204 : HttpStatusCode.OK_200 + + for (const token of tokens) { + await checkVideoAccessOptions({ server, videoPassword: 'password1', token, expectedStatus, mode: tmp }) + await checkVideoAccessOptions({ server, videoPassword: 'password2', token, expectedStatus, mode: tmp }) + } + }) + } + + describe('When accessing password protected video', function () { + + describe('For getting a password protected video', function () { + validateVideoAccess('get') + }) + + describe('For rating a video', function () { + validateVideoAccess('rate') + }) + + describe('For creating a thread', function () { + validateVideoAccess('createThread') + }) + + describe('For replying to a thread', function () { + validateVideoAccess('replyThread') + }) + + describe('For listing threads', function () { + validateVideoAccess('listThreads') + }) + + describe('For getting captions', function () { + validateVideoAccess('listCaptions') + }) + + describe('For creating video file token', function () { + validateVideoAccess('token') + }) + }) + + describe('When listing passwords', function () { + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path + video.uuid + '/passwords', server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path + video.uuid + '/passwords', server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path + video.uuid + '/passwords', server.accessToken) + }) + + it('Should fail for unauthenticated user', async function () { + await server.videoPasswords.list({ + token: null, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401, + videoId: video.id + }) + }) + + it('Should fail for unauthorized user', async function () { + await server.videoPasswords.list({ + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403, + videoId: video.id + }) + }) + + it('Should succeed with the correct parameters', async function () { + await server.videoPasswords.list({ + token: server.accessToken, + expectedStatus: HttpStatusCode.OK_200, + videoId: video.id + }) + }) + }) + + describe('When deleting a password', async function () { + const passwords = (await server.videoPasswords.list({ videoId: video.id })).data + + it('Should fail with wrong password id', async function () { + await server.videoPasswords.remove({ id: -1, videoId: video.id, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail for unauthenticated user', async function () { + await server.videoPasswords.remove({ + id: passwords[0].id, + token: null, + videoId: video.id, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail for unauthorized user', async function () { + await server.videoPasswords.remove({ + id: passwords[0].id, + token: userAccessToken, + videoId: video.id, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail for non password protected video', async function () { + publicVideo = await server.videos.quickUpload({ name: 'public video' }) + await server.videoPasswords.remove({ id: passwords[0].id, videoId: publicVideo.id, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail for password not linked to correct video', async function () { + const video2 = await server.videos.quickUpload({ + name: 'password protected video', + privacy: VideoPrivacy.PASSWORD_PROTECTED, + videoPasswords: [ 'password1', 'password2' ] + }) + await server.videoPasswords.remove({ id: passwords[0].id, videoId: video2.id, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should succeed with correct parameter', async function () { + await server.videoPasswords.remove({ id: passwords[0].id, videoId: video.id, expectedStatus: HttpStatusCode.NO_CONTENT_204 }) + }) + + it('Should fail for last password of a video', async function () { + await server.videoPasswords.remove({ id: passwords[1].id, videoId: video.id, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/video-playlists.ts b/packages/tests/src/api/check-params/video-playlists.ts new file mode 100644 index 000000000..7f5be18d4 --- /dev/null +++ b/packages/tests/src/api/check-params/video-playlists.ts @@ -0,0 +1,695 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@tests/shared/checks.js' +import { + HttpStatusCode, + VideoPlaylistCreate, + VideoPlaylistCreateResult, + VideoPlaylistElementCreate, + VideoPlaylistElementUpdate, + VideoPlaylistPrivacy, + VideoPlaylistReorder, + VideoPlaylistType +} from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + makeGetRequest, + PeerTubeServer, + PlaylistsCommand, + setAccessTokensToServers, + setDefaultVideoChannel +} from '@peertube/peertube-server-commands' + +describe('Test video playlists API validator', function () { + let server: PeerTubeServer + let userAccessToken: string + + let playlist: VideoPlaylistCreateResult + let privatePlaylistUUID: string + + let watchLaterPlaylistId: number + let videoId: number + let elementId: number + + let command: PlaylistsCommand + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + userAccessToken = await server.users.generateUserAndToken('user1') + videoId = (await server.videos.quickUpload({ name: 'video 1' })).id + + command = server.playlists + + { + const { data } = await command.listByAccount({ + token: server.accessToken, + handle: 'root', + start: 0, + count: 5, + playlistType: VideoPlaylistType.WATCH_LATER + }) + watchLaterPlaylistId = data[0].id + } + + { + playlist = await command.create({ + attributes: { + displayName: 'super playlist', + privacy: VideoPlaylistPrivacy.PUBLIC, + videoChannelId: server.store.channel.id + } + }) + } + + { + const created = await command.create({ + attributes: { + displayName: 'private', + privacy: VideoPlaylistPrivacy.PRIVATE + } + }) + privatePlaylistUUID = created.uuid + } + }) + + describe('When listing playlists', function () { + const globalPath = '/api/v1/video-playlists' + const accountPath = '/api/v1/accounts/root/video-playlists' + const videoChannelPath = '/api/v1/video-channels/root_channel/video-playlists' + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, globalPath, server.accessToken) + await checkBadStartPagination(server.url, accountPath, server.accessToken) + await checkBadStartPagination(server.url, videoChannelPath, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, globalPath, server.accessToken) + await checkBadCountPagination(server.url, accountPath, server.accessToken) + await checkBadCountPagination(server.url, videoChannelPath, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, globalPath, server.accessToken) + await checkBadSortPagination(server.url, accountPath, server.accessToken) + await checkBadSortPagination(server.url, videoChannelPath, server.accessToken) + }) + + it('Should fail with a bad playlist type', async function () { + await makeGetRequest({ url: server.url, path: globalPath, query: { playlistType: 3 } }) + await makeGetRequest({ url: server.url, path: accountPath, query: { playlistType: 3 } }) + await makeGetRequest({ url: server.url, path: videoChannelPath, query: { playlistType: 3 } }) + }) + + it('Should fail with a bad account parameter', async function () { + const accountPath = '/api/v1/accounts/root2/video-playlists' + + await makeGetRequest({ + url: server.url, + path: accountPath, + expectedStatus: HttpStatusCode.NOT_FOUND_404, + token: server.accessToken + }) + }) + + it('Should fail with a bad video channel parameter', async function () { + const accountPath = '/api/v1/video-channels/bad_channel/video-playlists' + + await makeGetRequest({ + url: server.url, + path: accountPath, + expectedStatus: HttpStatusCode.NOT_FOUND_404, + token: server.accessToken + }) + }) + + it('Should success with the correct parameters', async function () { + await makeGetRequest({ url: server.url, path: globalPath, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken }) + await makeGetRequest({ url: server.url, path: accountPath, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken }) + await makeGetRequest({ + url: server.url, + path: videoChannelPath, + expectedStatus: HttpStatusCode.OK_200, + token: server.accessToken + }) + }) + }) + + describe('When listing videos of a playlist', function () { + const path = '/api/v1/video-playlists/' + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path + playlist.shortUUID + '/videos', server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path + playlist.shortUUID + '/videos', server.accessToken) + }) + + it('Should success with the correct parameters', async function () { + await makeGetRequest({ url: server.url, path: path + playlist.shortUUID + '/videos', expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('When getting a video playlist', function () { + it('Should fail with a bad id or uuid', async function () { + await command.get({ playlistId: 'toto', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with an unknown playlist', async function () { + await command.get({ playlistId: 42, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail to get an unlisted playlist with the number id', async function () { + const playlist = await command.create({ + attributes: { + displayName: 'super playlist', + videoChannelId: server.store.channel.id, + privacy: VideoPlaylistPrivacy.UNLISTED + } + }) + + await command.get({ playlistId: playlist.id, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + await command.get({ playlistId: playlist.uuid, expectedStatus: HttpStatusCode.OK_200 }) + }) + + it('Should succeed with the correct params', async function () { + await command.get({ playlistId: playlist.uuid, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('When creating/updating a video playlist', function () { + const getBase = ( + attributes?: Partial, + wrapper?: Partial[0]> + ) => { + return { + attributes: { + displayName: 'display name', + privacy: VideoPlaylistPrivacy.UNLISTED, + thumbnailfile: 'custom-thumbnail.jpg', + videoChannelId: server.store.channel.id, + + ...attributes + }, + + expectedStatus: HttpStatusCode.BAD_REQUEST_400, + + ...wrapper + } + } + const getUpdate = (params: any, playlistId: number | string) => { + return { ...params, playlistId } + } + + it('Should fail with an unauthenticated user', async function () { + const params = getBase({}, { token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + + await command.create(params) + await command.update(getUpdate(params, playlist.shortUUID)) + }) + + it('Should fail without displayName', async function () { + const params = getBase({ displayName: undefined }) + + await command.create(params) + }) + + it('Should fail with an incorrect display name', async function () { + const params = getBase({ displayName: 's'.repeat(300) }) + + await command.create(params) + await command.update(getUpdate(params, playlist.shortUUID)) + }) + + it('Should fail with an incorrect description', async function () { + const params = getBase({ description: 't' }) + + await command.create(params) + await command.update(getUpdate(params, playlist.shortUUID)) + }) + + it('Should fail with an incorrect privacy', async function () { + const params = getBase({ privacy: 45 as any }) + + await command.create(params) + await command.update(getUpdate(params, playlist.shortUUID)) + }) + + it('Should fail with an unknown video channel id', async function () { + const params = getBase({ videoChannelId: 42 }, { expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + + await command.create(params) + await command.update(getUpdate(params, playlist.shortUUID)) + }) + + it('Should fail with an incorrect thumbnail file', async function () { + const params = getBase({ thumbnailfile: 'video_short.mp4' }) + + await command.create(params) + await command.update(getUpdate(params, playlist.shortUUID)) + }) + + it('Should fail with a thumbnail file too big', async function () { + const params = getBase({ thumbnailfile: 'custom-preview-big.png' }) + + await command.create(params) + await command.update(getUpdate(params, playlist.shortUUID)) + }) + + it('Should fail to set "public" a playlist not assigned to a channel', async function () { + const params = getBase({ privacy: VideoPlaylistPrivacy.PUBLIC, videoChannelId: undefined }) + const params2 = getBase({ privacy: VideoPlaylistPrivacy.PUBLIC, videoChannelId: 'null' as any }) + const params3 = getBase({ privacy: undefined, videoChannelId: 'null' as any }) + + await command.create(params) + await command.create(params2) + await command.update(getUpdate(params, privatePlaylistUUID)) + await command.update(getUpdate(params2, playlist.shortUUID)) + await command.update(getUpdate(params3, playlist.shortUUID)) + }) + + it('Should fail with an unknown playlist to update', async function () { + await command.update(getUpdate( + getBase({}, { expectedStatus: HttpStatusCode.NOT_FOUND_404 }), + 42 + )) + }) + + it('Should fail to update a playlist of another user', async function () { + await command.update(getUpdate( + getBase({}, { token: userAccessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }), + playlist.shortUUID + )) + }) + + it('Should fail to update the watch later playlist', async function () { + await command.update(getUpdate( + getBase({}, { expectedStatus: HttpStatusCode.BAD_REQUEST_400 }), + watchLaterPlaylistId + )) + }) + + it('Should succeed with the correct params', async function () { + { + const params = getBase({}, { expectedStatus: HttpStatusCode.OK_200 }) + await command.create(params) + } + + { + const params = getBase({}, { expectedStatus: HttpStatusCode.NO_CONTENT_204 }) + await command.update(getUpdate(params, playlist.shortUUID)) + } + }) + }) + + describe('When adding an element in a playlist', function () { + const getBase = ( + attributes?: Partial, + wrapper?: Partial[0]> + ) => { + return { + attributes: { + videoId, + startTimestamp: 2, + stopTimestamp: 3, + + ...attributes + }, + + expectedStatus: HttpStatusCode.BAD_REQUEST_400, + playlistId: playlist.id, + + ...wrapper + } + } + + it('Should fail with an unauthenticated user', async function () { + const params = getBase({}, { token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + await command.addElement(params) + }) + + it('Should fail with the playlist of another user', async function () { + const params = getBase({}, { token: userAccessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await command.addElement(params) + }) + + it('Should fail with an unknown or incorrect playlist id', async function () { + { + const params = getBase({}, { playlistId: 'toto' }) + await command.addElement(params) + } + + { + const params = getBase({}, { playlistId: 42, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + await command.addElement(params) + } + }) + + it('Should fail with an unknown or incorrect video id', async function () { + const params = getBase({ videoId: 42 }, { expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + await command.addElement(params) + }) + + it('Should fail with a bad start/stop timestamp', async function () { + { + const params = getBase({ startTimestamp: -42 }) + await command.addElement(params) + } + + { + const params = getBase({ stopTimestamp: 'toto' as any }) + await command.addElement(params) + } + }) + + it('Succeed with the correct params', async function () { + const params = getBase({}, { expectedStatus: HttpStatusCode.OK_200 }) + const created = await command.addElement(params) + elementId = created.id + }) + }) + + describe('When updating an element in a playlist', function () { + const getBase = ( + attributes?: Partial, + wrapper?: Partial[0]> + ) => { + return { + attributes: { + startTimestamp: 1, + stopTimestamp: 2, + + ...attributes + }, + + elementId, + playlistId: playlist.id, + expectedStatus: HttpStatusCode.BAD_REQUEST_400, + + ...wrapper + } + } + + it('Should fail with an unauthenticated user', async function () { + const params = getBase({}, { token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + await command.updateElement(params) + }) + + it('Should fail with the playlist of another user', async function () { + const params = getBase({}, { token: userAccessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await command.updateElement(params) + }) + + it('Should fail with an unknown or incorrect playlist id', async function () { + { + const params = getBase({}, { playlistId: 'toto' }) + await command.updateElement(params) + } + + { + const params = getBase({}, { playlistId: 42, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + await command.updateElement(params) + } + }) + + it('Should fail with an unknown or incorrect playlistElement id', async function () { + { + const params = getBase({}, { elementId: 'toto' }) + await command.updateElement(params) + } + + { + const params = getBase({}, { elementId: 42, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + await command.updateElement(params) + } + }) + + it('Should fail with a bad start/stop timestamp', async function () { + { + const params = getBase({ startTimestamp: 'toto' as any }) + await command.updateElement(params) + } + + { + const params = getBase({ stopTimestamp: -42 }) + await command.updateElement(params) + } + }) + + it('Should fail with an unknown element', async function () { + const params = getBase({}, { elementId: 888, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + await command.updateElement(params) + }) + + it('Succeed with the correct params', async function () { + const params = getBase({}, { expectedStatus: HttpStatusCode.NO_CONTENT_204 }) + await command.updateElement(params) + }) + }) + + describe('When reordering elements of a playlist', function () { + let videoId3: number + let videoId4: number + + const getBase = ( + attributes?: Partial, + wrapper?: Partial[0]> + ) => { + return { + attributes: { + startPosition: 1, + insertAfterPosition: 2, + reorderLength: 3, + + ...attributes + }, + + playlistId: playlist.shortUUID, + expectedStatus: HttpStatusCode.BAD_REQUEST_400, + + ...wrapper + } + } + + before(async function () { + videoId3 = (await server.videos.quickUpload({ name: 'video 3' })).id + videoId4 = (await server.videos.quickUpload({ name: 'video 4' })).id + + for (const id of [ videoId3, videoId4 ]) { + await command.addElement({ playlistId: playlist.shortUUID, attributes: { videoId: id } }) + } + }) + + it('Should fail with an unauthenticated user', async function () { + const params = getBase({}, { token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + await command.reorderElements(params) + }) + + it('Should fail with the playlist of another user', async function () { + const params = getBase({}, { token: userAccessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await command.reorderElements(params) + }) + + it('Should fail with an invalid playlist', async function () { + { + const params = getBase({}, { playlistId: 'toto' }) + await command.reorderElements(params) + } + + { + const params = getBase({}, { playlistId: 42, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + await command.reorderElements(params) + } + }) + + it('Should fail with an invalid start position', async function () { + { + const params = getBase({ startPosition: -1 }) + await command.reorderElements(params) + } + + { + const params = getBase({ startPosition: 'toto' as any }) + await command.reorderElements(params) + } + + { + const params = getBase({ startPosition: 42 }) + await command.reorderElements(params) + } + }) + + it('Should fail with an invalid insert after position', async function () { + { + const params = getBase({ insertAfterPosition: 'toto' as any }) + await command.reorderElements(params) + } + + { + const params = getBase({ insertAfterPosition: -2 }) + await command.reorderElements(params) + } + + { + const params = getBase({ insertAfterPosition: 42 }) + await command.reorderElements(params) + } + }) + + it('Should fail with an invalid reorder length', async function () { + { + const params = getBase({ reorderLength: 'toto' as any }) + await command.reorderElements(params) + } + + { + const params = getBase({ reorderLength: -2 }) + await command.reorderElements(params) + } + + { + const params = getBase({ reorderLength: 42 }) + await command.reorderElements(params) + } + }) + + it('Succeed with the correct params', async function () { + const params = getBase({}, { expectedStatus: HttpStatusCode.NO_CONTENT_204 }) + await command.reorderElements(params) + }) + }) + + describe('When checking exists in playlist endpoint', function () { + const path = '/api/v1/users/me/video-playlists/videos-exist' + + it('Should fail with an unauthenticated user', async function () { + await makeGetRequest({ + url: server.url, + path, + query: { videoIds: [ 1, 2 ] }, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with invalid video ids', async function () { + await makeGetRequest({ + url: server.url, + token: server.accessToken, + path, + query: { videoIds: 'toto' } + }) + + await makeGetRequest({ + url: server.url, + token: server.accessToken, + path, + query: { videoIds: [ 'toto' ] } + }) + + await makeGetRequest({ + url: server.url, + token: server.accessToken, + path, + query: { videoIds: [ 1, 'toto' ] } + }) + }) + + it('Should succeed with the correct params', async function () { + await makeGetRequest({ + url: server.url, + token: server.accessToken, + path, + query: { videoIds: [ 1, 2 ] }, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + }) + + describe('When deleting an element in a playlist', function () { + const getBase = (wrapper: Partial[0]>) => { + return { + elementId, + playlistId: playlist.uuid, + expectedStatus: HttpStatusCode.BAD_REQUEST_400, + + ...wrapper + } + } + + it('Should fail with an unauthenticated user', async function () { + const params = getBase({ token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + await command.removeElement(params) + }) + + it('Should fail with the playlist of another user', async function () { + const params = getBase({ token: userAccessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await command.removeElement(params) + }) + + it('Should fail with an unknown or incorrect playlist id', async function () { + { + const params = getBase({ playlistId: 'toto' }) + await command.removeElement(params) + } + + { + const params = getBase({ playlistId: 42, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + await command.removeElement(params) + } + }) + + it('Should fail with an unknown or incorrect video id', async function () { + { + const params = getBase({ elementId: 'toto' as any }) + await command.removeElement(params) + } + + { + const params = getBase({ elementId: 42, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + await command.removeElement(params) + } + }) + + it('Should fail with an unknown element', async function () { + const params = getBase({ elementId: 888, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + await command.removeElement(params) + }) + + it('Succeed with the correct params', async function () { + const params = getBase({ expectedStatus: HttpStatusCode.NO_CONTENT_204 }) + await command.removeElement(params) + }) + }) + + describe('When deleting a playlist', function () { + it('Should fail with an unknown playlist', async function () { + await command.delete({ playlistId: 42, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail with a playlist of another user', async function () { + await command.delete({ token: userAccessToken, playlistId: playlist.uuid, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail with the watch later playlist', async function () { + await command.delete({ playlistId: watchLaterPlaylistId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should succeed with the correct params', async function () { + await command.delete({ playlistId: playlist.uuid }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/video-source.ts b/packages/tests/src/api/check-params/video-source.ts new file mode 100644 index 000000000..918182b8d --- /dev/null +++ b/packages/tests/src/api/check-params/video-source.ts @@ -0,0 +1,154 @@ +import { HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test video sources API validator', function () { + let server: PeerTubeServer = null + let uuid: string + let userToken: string + + before(async function () { + this.timeout(120000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + userToken = await server.users.generateUserAndToken('user1') + }) + + describe('When getting latest source', function () { + + before(async function () { + const created = await server.videos.quickUpload({ name: 'video' }) + uuid = created.uuid + }) + + it('Should fail without a valid uuid', async function () { + await server.videos.getSource({ id: '4da6fde3-88f7-4d16-b119-108df563d0b0', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should receive 404 when passing a non existing video id', async function () { + await server.videos.getSource({ id: '4da6fde3-88f7-4d16-b119-108df5630b06', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should not get the source as unauthenticated', async function () { + await server.videos.getSource({ id: uuid, expectedStatus: HttpStatusCode.UNAUTHORIZED_401, token: null }) + }) + + it('Should not get the source with another user', async function () { + await server.videos.getSource({ id: uuid, expectedStatus: HttpStatusCode.FORBIDDEN_403, token: userToken }) + }) + + it('Should succeed with the correct parameters get the source as another user', async function () { + await server.videos.getSource({ id: uuid }) + }) + }) + + describe('When updating source video file', function () { + let userAccessToken: string + let userId: number + + let videoId: string + let userVideoId: string + + before(async function () { + const res = await server.users.generate('user2') + userAccessToken = res.token + userId = res.userId + + const { uuid } = await server.videos.quickUpload({ name: 'video' }) + videoId = uuid + + await waitJobs([ server ]) + }) + + it('Should fail if not enabled on the instance', async function () { + await server.config.disableFileUpdate() + + await server.videos.replaceSourceFile({ videoId, fixture: 'video_short.mp4', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail on an unknown video', async function () { + await server.config.enableFileUpdate() + + await server.videos.replaceSourceFile({ videoId: 404, fixture: 'video_short.mp4', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail with an invalid video', async function () { + await server.config.enableLive({ allowReplay: false }) + + const { video } = await server.live.quickCreate({ saveReplay: false, permanentLive: true }) + await server.videos.replaceSourceFile({ + videoId: video.uuid, + fixture: 'video_short.mp4', + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail without token', async function () { + await server.videos.replaceSourceFile({ + token: null, + videoId, + fixture: 'video_short.mp4', + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with another user', async function () { + await server.videos.replaceSourceFile({ + token: userAccessToken, + videoId, + fixture: 'video_short.mp4', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with an incorrect input file', async function () { + await server.videos.replaceSourceFile({ + fixture: 'video_short_fake.webm', + videoId, + completedExpectedStatus: HttpStatusCode.UNPROCESSABLE_ENTITY_422 + }) + + await server.videos.replaceSourceFile({ + fixture: 'video_short.mkv', + videoId, + expectedStatus: HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415 + }) + }) + + it('Should fail if quota is exceeded', async function () { + this.timeout(60000) + + const { uuid } = await server.videos.quickUpload({ name: 'user video' }) + userVideoId = uuid + await waitJobs([ server ]) + + await server.users.update({ userId, videoQuota: 1 }) + await server.videos.replaceSourceFile({ + token: userAccessToken, + videoId: uuid, + fixture: 'video_short.mp4', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should succeed with the correct params', async function () { + this.timeout(60000) + + await server.users.update({ userId, videoQuota: 1000 * 1000 * 1000 }) + await server.videos.replaceSourceFile({ videoId: userVideoId, fixture: 'video_short.mp4' }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/video-storyboards.ts b/packages/tests/src/api/check-params/video-storyboards.ts new file mode 100644 index 000000000..f83b541d8 --- /dev/null +++ b/packages/tests/src/api/check-params/video-storyboards.ts @@ -0,0 +1,45 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { HttpStatusCode, VideoPrivacy } from '@peertube/peertube-models' +import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@peertube/peertube-server-commands' + +describe('Test video storyboards API validator', function () { + let server: PeerTubeServer + + let publicVideo: { uuid: string } + let privateVideo: { uuid: string } + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(120000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + + publicVideo = await server.videos.quickUpload({ name: 'public' }) + privateVideo = await server.videos.quickUpload({ name: 'private', privacy: VideoPrivacy.PRIVATE }) + }) + + it('Should fail without a valid uuid', async function () { + await server.storyboard.list({ id: '4da6fde3-88f7-4d16-b119-108df563d0b0', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should receive 404 when passing a non existing video id', async function () { + await server.storyboard.list({ id: '4da6fde3-88f7-4d16-b119-108df5630b06', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should not get the private storyboard without the appropriate token', async function () { + await server.storyboard.list({ id: privateVideo.uuid, expectedStatus: HttpStatusCode.UNAUTHORIZED_401, token: null }) + await server.storyboard.list({ id: publicVideo.uuid, expectedStatus: HttpStatusCode.OK_200, token: null }) + }) + + it('Should succeed with the correct parameters', async function () { + await server.storyboard.list({ id: privateVideo.uuid }) + await server.storyboard.list({ id: publicVideo.uuid }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/video-studio.ts b/packages/tests/src/api/check-params/video-studio.ts new file mode 100644 index 000000000..ae83f3590 --- /dev/null +++ b/packages/tests/src/api/check-params/video-studio.ts @@ -0,0 +1,392 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { HttpStatusCode, HttpStatusCodeType, VideoStudioTask } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + PeerTubeServer, + setAccessTokensToServers, + VideoStudioCommand, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test video studio API validator', function () { + let server: PeerTubeServer + let command: VideoStudioCommand + let userAccessToken: string + let videoUUID: string + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(120_000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + userAccessToken = await server.users.generateUserAndToken('user1') + + await server.config.enableMinimumTranscoding() + + const { uuid } = await server.videos.quickUpload({ name: 'video' }) + videoUUID = uuid + + command = server.videoStudio + + await waitJobs([ server ]) + }) + + describe('Task creation', function () { + + describe('Config settings', function () { + + it('Should fail if studio is disabled', async function () { + await server.config.updateExistingSubConfig({ + newConfig: { + videoStudio: { + enabled: false + } + } + }) + + await command.createEditionTasks({ + videoId: videoUUID, + tasks: VideoStudioCommand.getComplexTask(), + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail to enable studio if transcoding is disabled', async function () { + await server.config.updateExistingSubConfig({ + newConfig: { + videoStudio: { + enabled: true + }, + transcoding: { + enabled: false + } + }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should succeed to enable video studio', async function () { + await server.config.updateExistingSubConfig({ + newConfig: { + videoStudio: { + enabled: true + }, + transcoding: { + enabled: true + } + } + }) + }) + }) + + describe('Common tasks', function () { + + it('Should fail without token', async function () { + await command.createEditionTasks({ + token: null, + videoId: videoUUID, + tasks: VideoStudioCommand.getComplexTask(), + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with another user token', async function () { + await command.createEditionTasks({ + token: userAccessToken, + videoId: videoUUID, + tasks: VideoStudioCommand.getComplexTask(), + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with an invalid video', async function () { + await command.createEditionTasks({ + videoId: 'tintin', + tasks: VideoStudioCommand.getComplexTask(), + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with an unknown video', async function () { + await command.createEditionTasks({ + videoId: 42, + tasks: VideoStudioCommand.getComplexTask(), + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail with an already in transcoding state video', async function () { + this.timeout(60000) + + const { uuid } = await server.videos.quickUpload({ name: 'transcoded video' }) + await waitJobs([ server ]) + + await server.jobs.pauseJobQueue() + await server.videos.runTranscoding({ videoId: uuid, transcodingType: 'hls' }) + + await command.createEditionTasks({ + videoId: uuid, + tasks: VideoStudioCommand.getComplexTask(), + expectedStatus: HttpStatusCode.CONFLICT_409 + }) + + await server.jobs.resumeJobQueue() + }) + + it('Should fail with a bad complex task', async function () { + await command.createEditionTasks({ + videoId: videoUUID, + tasks: [ + { + name: 'cut', + options: { + start: 1, + end: 2 + } + }, + { + name: 'hadock', + options: { + start: 1, + end: 2 + } + } + ] as any, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail without task', async function () { + await command.createEditionTasks({ + videoId: videoUUID, + tasks: [], + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with too many tasks', async function () { + const tasks: VideoStudioTask[] = [] + + for (let i = 0; i < 110; i++) { + tasks.push({ + name: 'cut', + options: { + start: 1 + } + }) + } + + await command.createEditionTasks({ + videoId: videoUUID, + tasks, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should succeed with correct parameters', async function () { + await server.jobs.pauseJobQueue() + + await command.createEditionTasks({ + videoId: videoUUID, + tasks: VideoStudioCommand.getComplexTask(), + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + + it('Should fail with a video that is already waiting for edition', async function () { + this.timeout(120000) + + await command.createEditionTasks({ + videoId: videoUUID, + tasks: VideoStudioCommand.getComplexTask(), + expectedStatus: HttpStatusCode.CONFLICT_409 + }) + + await server.jobs.resumeJobQueue() + + await waitJobs([ server ]) + }) + }) + + describe('Cut task', function () { + + async function cut (start: number, end: number, expectedStatus: HttpStatusCodeType = HttpStatusCode.BAD_REQUEST_400) { + await command.createEditionTasks({ + videoId: videoUUID, + tasks: [ + { + name: 'cut', + options: { + start, + end + } + } + ], + expectedStatus + }) + } + + it('Should fail with bad start/end', async function () { + const invalid = [ + 'tintin', + -1, + undefined + ] + + for (const value of invalid) { + await cut(value as any, undefined) + await cut(undefined, value as any) + } + }) + + it('Should fail with the same start/end', async function () { + await cut(2, 2) + }) + + it('Should fail with inconsistents start/end', async function () { + await cut(2, 1) + }) + + it('Should fail without start and end', async function () { + await cut(undefined, undefined) + }) + + it('Should succeed with the correct params', async function () { + this.timeout(120000) + + await cut(0, 2, HttpStatusCode.NO_CONTENT_204) + + await waitJobs([ server ]) + }) + }) + + describe('Watermark task', function () { + + async function addWatermark (file: string, expectedStatus: HttpStatusCodeType = HttpStatusCode.BAD_REQUEST_400) { + await command.createEditionTasks({ + videoId: videoUUID, + tasks: [ + { + name: 'add-watermark', + options: { + file + } + } + ], + expectedStatus + }) + } + + it('Should fail without waterkmark', async function () { + await addWatermark(undefined) + }) + + it('Should fail with an invalid watermark', async function () { + await addWatermark('video_short.mp4') + }) + + it('Should succeed with the correct params', async function () { + this.timeout(120000) + + await addWatermark('custom-thumbnail.jpg', HttpStatusCode.NO_CONTENT_204) + + await waitJobs([ server ]) + }) + }) + + describe('Intro/Outro task', function () { + + async function addIntroOutro ( + type: 'add-intro' | 'add-outro', + file: string, + expectedStatus: HttpStatusCodeType = HttpStatusCode.BAD_REQUEST_400 + ) { + await command.createEditionTasks({ + videoId: videoUUID, + tasks: [ + { + name: type, + options: { + file + } + } + ], + expectedStatus + }) + } + + it('Should fail without file', async function () { + await addIntroOutro('add-intro', undefined) + await addIntroOutro('add-outro', undefined) + }) + + it('Should fail with an invalid file', async function () { + await addIntroOutro('add-intro', 'custom-thumbnail.jpg') + await addIntroOutro('add-outro', 'custom-thumbnail.jpg') + }) + + it('Should fail with a file that does not contain video stream', async function () { + await addIntroOutro('add-intro', 'sample.ogg') + await addIntroOutro('add-outro', 'sample.ogg') + + }) + + it('Should succeed with the correct params', async function () { + this.timeout(120000) + + await addIntroOutro('add-intro', 'video_very_short_240p.mp4', HttpStatusCode.NO_CONTENT_204) + await waitJobs([ server ]) + + await addIntroOutro('add-outro', 'video_very_short_240p.mp4', HttpStatusCode.NO_CONTENT_204) + await waitJobs([ server ]) + }) + + it('Should check total quota when creating the task', async function () { + this.timeout(120000) + + const user = await server.users.create({ username: 'user_quota_1' }) + const token = await server.login.getAccessToken('user_quota_1') + const { uuid } = await server.videos.quickUpload({ token, name: 'video_quota_1', fixture: 'video_short.mp4' }) + + const addIntroOutroByUser = (type: 'add-intro' | 'add-outro', expectedStatus: HttpStatusCodeType) => { + return command.createEditionTasks({ + token, + videoId: uuid, + tasks: [ + { + name: type, + options: { + file: 'video_short.mp4' + } + } + ], + expectedStatus + }) + } + + await waitJobs([ server ]) + + const { videoQuotaUsed } = await server.users.getMyQuotaUsed({ token }) + await server.users.update({ userId: user.id, videoQuota: Math.round(videoQuotaUsed * 2.5) }) + + // Still valid + await addIntroOutroByUser('add-intro', HttpStatusCode.NO_CONTENT_204) + + await waitJobs([ server ]) + + // Too much quota + await addIntroOutroByUser('add-intro', HttpStatusCode.PAYLOAD_TOO_LARGE_413) + await addIntroOutroByUser('add-outro', HttpStatusCode.PAYLOAD_TOO_LARGE_413) + }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/video-token.ts b/packages/tests/src/api/check-params/video-token.ts new file mode 100644 index 000000000..5f838102d --- /dev/null +++ b/packages/tests/src/api/check-params/video-token.ts @@ -0,0 +1,70 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { HttpStatusCode, VideoPrivacy } from '@peertube/peertube-models' +import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@peertube/peertube-server-commands' + +describe('Test video tokens', function () { + let server: PeerTubeServer + let privateVideoId: string + let passwordProtectedVideoId: string + let userToken: string + + const videoPassword = 'password' + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(300_000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + { + const { uuid } = await server.videos.quickUpload({ name: 'private video', privacy: VideoPrivacy.PRIVATE }) + privateVideoId = uuid + } + { + const { uuid } = await server.videos.quickUpload({ + name: 'password protected video', + privacy: VideoPrivacy.PASSWORD_PROTECTED, + videoPasswords: [ videoPassword ] + }) + passwordProtectedVideoId = uuid + } + userToken = await server.users.generateUserAndToken('user1') + }) + + it('Should not generate tokens on private video for unauthenticated user', async function () { + await server.videoToken.create({ videoId: privateVideoId, token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should not generate tokens of unknown video', async function () { + await server.videoToken.create({ videoId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should not generate tokens with incorrect password', async function () { + await server.videoToken.create({ + videoId: passwordProtectedVideoId, + token: null, + expectedStatus: HttpStatusCode.FORBIDDEN_403, + videoPassword: 'incorrectPassword' + }) + }) + + it('Should not generate tokens of a non owned video', async function () { + await server.videoToken.create({ videoId: privateVideoId, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should generate token', async function () { + await server.videoToken.create({ videoId: privateVideoId }) + }) + + it('Should generate token on password protected video', async function () { + await server.videoToken.create({ videoId: passwordProtectedVideoId, videoPassword, token: null }) + await server.videoToken.create({ videoId: passwordProtectedVideoId, videoPassword, token: userToken }) + await server.videoToken.create({ videoId: passwordProtectedVideoId, videoPassword }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/videos-common-filters.ts b/packages/tests/src/api/check-params/videos-common-filters.ts new file mode 100644 index 000000000..dbae3010c --- /dev/null +++ b/packages/tests/src/api/check-params/videos-common-filters.ts @@ -0,0 +1,171 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { + HttpStatusCode, + HttpStatusCodeType, + UserRole, + VideoInclude, + VideoIncludeType, + VideoPrivacy, + VideoPrivacyType +} from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + makeGetRequest, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel +} from '@peertube/peertube-server-commands' + +describe('Test video filters validators', function () { + let server: PeerTubeServer + let userAccessToken: string + let moderatorAccessToken: string + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + const user = { username: 'user1', password: 'my super password' } + await server.users.create({ username: user.username, password: user.password }) + userAccessToken = await server.login.getAccessToken(user) + + const moderator = { username: 'moderator', password: 'my super password' } + await server.users.create({ username: moderator.username, password: moderator.password, role: UserRole.MODERATOR }) + + moderatorAccessToken = await server.login.getAccessToken(moderator) + }) + + describe('When setting video filters', function () { + + const validIncludes = [ + VideoInclude.NONE, + VideoInclude.BLOCKED_OWNER, + VideoInclude.NOT_PUBLISHED_STATE | VideoInclude.BLACKLISTED + ] + + async function testEndpoints (options: { + token?: string + isLocal?: boolean + include?: VideoIncludeType + privacyOneOf?: VideoPrivacyType[] + expectedStatus: HttpStatusCodeType + excludeAlreadyWatched?: boolean + unauthenticatedUser?: boolean + }) { + const paths = [ + '/api/v1/video-channels/root_channel/videos', + '/api/v1/accounts/root/videos', + '/api/v1/videos', + '/api/v1/search/videos' + ] + + for (const path of paths) { + const token = options.unauthenticatedUser + ? undefined + : options.token || server.accessToken + + await makeGetRequest({ + url: server.url, + path, + token, + query: { + isLocal: options.isLocal, + privacyOneOf: options.privacyOneOf, + include: options.include, + excludeAlreadyWatched: options.excludeAlreadyWatched + }, + expectedStatus: options.expectedStatus + }) + } + } + + it('Should fail with a bad privacyOneOf', async function () { + await testEndpoints({ privacyOneOf: [ 'toto' ] as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should succeed with a good privacyOneOf', async function () { + await testEndpoints({ privacyOneOf: [ VideoPrivacy.INTERNAL ], expectedStatus: HttpStatusCode.OK_200 }) + }) + + it('Should fail to use privacyOneOf with a simple user', async function () { + await testEndpoints({ + privacyOneOf: [ VideoPrivacy.INTERNAL ], + token: userAccessToken, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with a bad include', async function () { + await testEndpoints({ include: 'toto' as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should succeed with a good include', async function () { + for (const include of validIncludes) { + await testEndpoints({ include, expectedStatus: HttpStatusCode.OK_200 }) + } + }) + + it('Should fail to include more videos with a simple user', async function () { + for (const include of validIncludes) { + await testEndpoints({ token: userAccessToken, include, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + } + }) + + it('Should succeed to list all local/all with a moderator', async function () { + for (const include of validIncludes) { + await testEndpoints({ token: moderatorAccessToken, include, expectedStatus: HttpStatusCode.OK_200 }) + } + }) + + it('Should succeed to list all local/all with an admin', async function () { + for (const include of validIncludes) { + await testEndpoints({ token: server.accessToken, include, expectedStatus: HttpStatusCode.OK_200 }) + } + }) + + // Because we cannot authenticate the user on the RSS endpoint + it('Should fail on the feeds endpoint with the all filter', async function () { + for (const include of [ VideoInclude.NOT_PUBLISHED_STATE ]) { + await makeGetRequest({ + url: server.url, + path: '/feeds/videos.json', + expectedStatus: HttpStatusCode.UNAUTHORIZED_401, + query: { + include + } + }) + } + }) + + it('Should succeed on the feeds endpoint with the local filter', async function () { + await makeGetRequest({ + url: server.url, + path: '/feeds/videos.json', + expectedStatus: HttpStatusCode.OK_200, + query: { + isLocal: true + } + }) + }) + + it('Should fail when trying to exclude already watched videos for an unlogged user', async function () { + await testEndpoints({ excludeAlreadyWatched: true, unauthenticatedUser: true, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should succeed when trying to exclude already watched videos for a logged user', async function () { + await testEndpoints({ token: userAccessToken, excludeAlreadyWatched: true, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/videos-history.ts b/packages/tests/src/api/check-params/videos-history.ts new file mode 100644 index 000000000..65d1e9fac --- /dev/null +++ b/packages/tests/src/api/check-params/videos-history.ts @@ -0,0 +1,145 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { checkBadCountPagination, checkBadStartPagination } from '@tests/shared/checks.js' +import { HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + makeDeleteRequest, + makeGetRequest, + makePostBodyRequest, + makePutBodyRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test videos history API validator', function () { + const myHistoryPath = '/api/v1/users/me/history/videos' + const myHistoryRemove = myHistoryPath + '/remove' + let viewPath: string + let server: PeerTubeServer + let videoId: number + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + + const { id, uuid } = await server.videos.upload() + viewPath = '/api/v1/videos/' + uuid + '/views' + videoId = id + }) + + describe('When notifying a user is watching a video', function () { + + it('Should fail with a bad token', async function () { + const fields = { currentTime: 5 } + await makePutBodyRequest({ url: server.url, path: viewPath, fields, token: 'bad', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should succeed with the correct parameters', async function () { + const fields = { currentTime: 5 } + + await makePutBodyRequest({ + url: server.url, + path: viewPath, + fields, + token: server.accessToken, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + describe('When listing user videos history', function () { + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, myHistoryPath, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, myHistoryPath, server.accessToken) + }) + + it('Should fail with an unauthenticated user', async function () { + await makeGetRequest({ url: server.url, path: myHistoryPath, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should succeed with the correct params', async function () { + await makeGetRequest({ url: server.url, token: server.accessToken, path: myHistoryPath, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('When removing a specific user video history element', function () { + let path: string + + before(function () { + path = myHistoryPath + '/' + videoId + }) + + it('Should fail with an unauthenticated user', async function () { + await makeDeleteRequest({ url: server.url, path, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with a bad videoId parameter', async function () { + await makeDeleteRequest({ + url: server.url, + token: server.accessToken, + path: myHistoryRemove + '/hi', + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should succeed with the correct parameters', async function () { + await makeDeleteRequest({ + url: server.url, + token: server.accessToken, + path, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + describe('When removing all user videos history', function () { + it('Should fail with an unauthenticated user', async function () { + await makePostBodyRequest({ url: server.url, path: myHistoryPath + '/remove', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with a bad beforeDate parameter', async function () { + const body = { beforeDate: '15' } + await makePostBodyRequest({ + url: server.url, + token: server.accessToken, + path: myHistoryRemove, + fields: body, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should succeed with a valid beforeDate param', async function () { + const body = { beforeDate: new Date().toISOString() } + await makePostBodyRequest({ + url: server.url, + token: server.accessToken, + path: myHistoryRemove, + fields: body, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + + it('Should succeed without body', async function () { + await makePostBodyRequest({ + url: server.url, + token: server.accessToken, + path: myHistoryRemove, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/videos-overviews.ts b/packages/tests/src/api/check-params/videos-overviews.ts new file mode 100644 index 000000000..ba6f6ac69 --- /dev/null +++ b/packages/tests/src/api/check-params/videos-overviews.ts @@ -0,0 +1,31 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { cleanupTests, createSingleServer, PeerTubeServer } from '@peertube/peertube-server-commands' + +describe('Test videos overview API validator', function () { + let server: PeerTubeServer + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + }) + + describe('When getting videos overview', function () { + + it('Should fail with a bad pagination', async function () { + await server.overviews.getVideos({ page: 0, expectedStatus: 400 }) + await server.overviews.getVideos({ page: 100, expectedStatus: 400 }) + }) + + it('Should succeed with a good pagination', async function () { + await server.overviews.getVideos({ page: 1 }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/videos.ts b/packages/tests/src/api/check-params/videos.ts new file mode 100644 index 000000000..c349ed9fe --- /dev/null +++ b/packages/tests/src/api/check-params/videos.ts @@ -0,0 +1,883 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { join } from 'path' +import { omit, randomInt } from '@peertube/peertube-core-utils' +import { HttpStatusCode, PeerTubeProblemDocument, VideoCreateResult, VideoPrivacy } from '@peertube/peertube-models' +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import { + cleanupTests, + createSingleServer, + makeDeleteRequest, + makeGetRequest, + makePutBodyRequest, + makeUploadRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' +import { checkBadStartPagination, checkBadCountPagination, checkBadSortPagination } from '@tests/shared/checks.js' +import { checkUploadVideoParam } from '@tests/shared/videos.js' + +describe('Test videos API validator', function () { + const path = '/api/v1/videos/' + let server: PeerTubeServer + let userAccessToken = '' + let accountName: string + let channelId: number + let channelName: string + let video: VideoCreateResult + let privateVideo: VideoCreateResult + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + + userAccessToken = await server.users.generateUserAndToken('user1') + + { + const body = await server.users.getMyInfo() + channelId = body.videoChannels[0].id + channelName = body.videoChannels[0].name + accountName = body.account.name + '@' + body.account.host + } + + { + privateVideo = await server.videos.quickUpload({ name: 'private video', privacy: VideoPrivacy.PRIVATE }) + } + }) + + describe('When listing videos', function () { + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path) + }) + + it('Should fail with a bad skipVideos query', async function () { + await makeGetRequest({ url: server.url, path, expectedStatus: HttpStatusCode.OK_200, query: { skipCount: 'toto' } }) + }) + + it('Should success with the correct parameters', async function () { + await makeGetRequest({ url: server.url, path, expectedStatus: HttpStatusCode.OK_200, query: { skipCount: false } }) + }) + }) + + describe('When searching a video', function () { + + it('Should fail with nothing', async function () { + await makeGetRequest({ + url: server.url, + path: join(path, 'search'), + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, join(path, 'search', 'test')) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, join(path, 'search', 'test')) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, join(path, 'search', 'test')) + }) + + it('Should success with the correct parameters', async function () { + await makeGetRequest({ url: server.url, path, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('When listing my videos', function () { + const path = '/api/v1/users/me/videos' + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an invalid channel', async function () { + await makeGetRequest({ url: server.url, token: server.accessToken, path, query: { channelId: 'toto' } }) + }) + + it('Should fail with an unknown channel', async function () { + await makeGetRequest({ + url: server.url, + token: server.accessToken, + path, + query: { channelId: 89898 }, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should success with the correct parameters', async function () { + await makeGetRequest({ url: server.url, token: server.accessToken, path, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('When listing account videos', function () { + let path: string + + before(async function () { + path = '/api/v1/accounts/' + accountName + '/videos' + }) + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, server.accessToken) + }) + + it('Should success with the correct parameters', async function () { + await makeGetRequest({ url: server.url, path, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('When listing video channel videos', function () { + let path: string + + before(async function () { + path = '/api/v1/video-channels/' + channelName + '/videos' + }) + + it('Should fail with a bad start pagination', async function () { + await checkBadStartPagination(server.url, path, server.accessToken) + }) + + it('Should fail with a bad count pagination', async function () { + await checkBadCountPagination(server.url, path, server.accessToken) + }) + + it('Should fail with an incorrect sort', async function () { + await checkBadSortPagination(server.url, path, server.accessToken) + }) + + it('Should success with the correct parameters', async function () { + await makeGetRequest({ url: server.url, path, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('When adding a video', function () { + let baseCorrectParams + const baseCorrectAttaches = { + fixture: buildAbsoluteFixturePath('video_short.webm') + } + + before(function () { + // Put in before to have channelId + baseCorrectParams = { + name: 'my super name', + category: 5, + licence: 1, + language: 'pt', + nsfw: false, + commentsEnabled: true, + downloadEnabled: true, + waitTranscoding: true, + description: 'my super description', + support: 'my super support text', + tags: [ 'tag1', 'tag2' ], + privacy: VideoPrivacy.PUBLIC, + channelId, + originallyPublishedAt: new Date().toISOString() + } + }) + + function runSuite (mode: 'legacy' | 'resumable') { + + const baseOptions = () => { + return { + server, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400, + mode + } + } + + it('Should fail with nothing', async function () { + const fields = {} + const attaches = {} + await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) + }) + + it('Should fail without name', async function () { + const fields = omit(baseCorrectParams, [ 'name' ]) + const attaches = baseCorrectAttaches + + await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) + }) + + it('Should fail with a long name', async function () { + const fields = { ...baseCorrectParams, name: 'super'.repeat(65) } + const attaches = baseCorrectAttaches + + await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) + }) + + it('Should fail with a bad category', async function () { + const fields = { ...baseCorrectParams, category: 125 } + const attaches = baseCorrectAttaches + + await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) + }) + + it('Should fail with a bad licence', async function () { + const fields = { ...baseCorrectParams, licence: 125 } + const attaches = baseCorrectAttaches + + await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) + }) + + it('Should fail with a bad language', async function () { + const fields = { ...baseCorrectParams, language: 'a'.repeat(15) } + const attaches = baseCorrectAttaches + + await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) + }) + + it('Should fail with a long description', async function () { + const fields = { ...baseCorrectParams, description: 'super'.repeat(2500) } + const attaches = baseCorrectAttaches + + await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) + }) + + it('Should fail with a long support text', async function () { + const fields = { ...baseCorrectParams, support: 'super'.repeat(201) } + const attaches = baseCorrectAttaches + + await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) + }) + + it('Should fail without a channel', async function () { + const fields = omit(baseCorrectParams, [ 'channelId' ]) + const attaches = baseCorrectAttaches + + await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) + }) + + it('Should fail with a bad channel', async function () { + const fields = { ...baseCorrectParams, channelId: 545454 } + const attaches = baseCorrectAttaches + + await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) + }) + + it('Should fail with another user channel', async function () { + const user = { + username: 'fake' + randomInt(0, 1500), + password: 'fake_password' + } + await server.users.create({ username: user.username, password: user.password }) + + const accessTokenUser = await server.login.getAccessToken(user) + const { videoChannels } = await server.users.getMyInfo({ token: accessTokenUser }) + const customChannelId = videoChannels[0].id + + const fields = { ...baseCorrectParams, channelId: customChannelId } + const attaches = baseCorrectAttaches + + await checkUploadVideoParam({ + ...baseOptions(), + token: userAccessToken, + attributes: { ...fields, ...attaches } + }) + }) + + it('Should fail with too many tags', async function () { + const fields = { ...baseCorrectParams, tags: [ 'tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6' ] } + const attaches = baseCorrectAttaches + + await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) + }) + + it('Should fail with a tag length too low', async function () { + const fields = { ...baseCorrectParams, tags: [ 'tag1', 't' ] } + const attaches = baseCorrectAttaches + + await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) + }) + + it('Should fail with a tag length too big', async function () { + const fields = { ...baseCorrectParams, tags: [ 'tag1', 'my_super_tag_too_long_long_long_long_long_long' ] } + const attaches = baseCorrectAttaches + + await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) + }) + + it('Should fail with a bad schedule update (miss updateAt)', async function () { + const fields = { ...baseCorrectParams, scheduleUpdate: { privacy: VideoPrivacy.PUBLIC } } + const attaches = baseCorrectAttaches + + await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) + }) + + it('Should fail with a bad schedule update (wrong updateAt)', async function () { + const fields = { + ...baseCorrectParams, + + scheduleUpdate: { + privacy: VideoPrivacy.PUBLIC, + updateAt: 'toto' + } + } + const attaches = baseCorrectAttaches + + await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) + }) + + it('Should fail with a bad originally published at attribute', async function () { + const fields = { ...baseCorrectParams, originallyPublishedAt: 'toto' } + const attaches = baseCorrectAttaches + + await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) + }) + + it('Should fail without an input file', async function () { + const fields = baseCorrectParams + const attaches = {} + await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) + }) + + it('Should fail with an incorrect input file', async function () { + const fields = baseCorrectParams + let attaches = { fixture: buildAbsoluteFixturePath('video_short_fake.webm') } + + await checkUploadVideoParam({ + ...baseOptions(), + attributes: { ...fields, ...attaches }, + // 200 for the init request, 422 when the file has finished being uploaded + expectedStatus: undefined, + completedExpectedStatus: HttpStatusCode.UNPROCESSABLE_ENTITY_422 + }) + + attaches = { fixture: buildAbsoluteFixturePath('video_short.mkv') } + await checkUploadVideoParam({ + ...baseOptions(), + attributes: { ...fields, ...attaches }, + expectedStatus: HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415 + }) + }) + + it('Should fail with an incorrect thumbnail file', async function () { + const fields = baseCorrectParams + const attaches = { + thumbnailfile: buildAbsoluteFixturePath('video_short.mp4'), + fixture: buildAbsoluteFixturePath('video_short.mp4') + } + + await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) + }) + + it('Should fail with a big thumbnail file', async function () { + const fields = baseCorrectParams + const attaches = { + thumbnailfile: buildAbsoluteFixturePath('custom-preview-big.png'), + fixture: buildAbsoluteFixturePath('video_short.mp4') + } + + await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) + }) + + it('Should fail with an incorrect preview file', async function () { + const fields = baseCorrectParams + const attaches = { + previewfile: buildAbsoluteFixturePath('video_short.mp4'), + fixture: buildAbsoluteFixturePath('video_short.mp4') + } + + await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) + }) + + it('Should fail with a big preview file', async function () { + const fields = baseCorrectParams + const attaches = { + previewfile: buildAbsoluteFixturePath('custom-preview-big.png'), + fixture: buildAbsoluteFixturePath('video_short.mp4') + } + + await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) + }) + + it('Should report the appropriate error', async function () { + const fields = { ...baseCorrectParams, language: 'a'.repeat(15) } + const attaches = baseCorrectAttaches + + const attributes = { ...fields, ...attaches } + const body = await checkUploadVideoParam({ ...baseOptions(), attributes }) + + const error = body as unknown as PeerTubeProblemDocument + + if (mode === 'legacy') { + expect(error.docs).to.equal('https://docs.joinpeertube.org/api-rest-reference.html#operation/uploadLegacy') + } else { + expect(error.docs).to.equal('https://docs.joinpeertube.org/api-rest-reference.html#operation/uploadResumableInit') + } + + expect(error.type).to.equal('about:blank') + expect(error.title).to.equal('Bad Request') + + expect(error.detail).to.equal('Incorrect request parameters: language') + expect(error.error).to.equal('Incorrect request parameters: language') + + expect(error.status).to.equal(HttpStatusCode.BAD_REQUEST_400) + expect(error['invalid-params'].language).to.exist + }) + + it('Should succeed with the correct parameters', async function () { + this.timeout(30000) + + const fields = baseCorrectParams + + { + const attaches = baseCorrectAttaches + await checkUploadVideoParam({ + ...baseOptions(), + attributes: { ...fields, ...attaches }, + expectedStatus: HttpStatusCode.OK_200 + }) + } + + { + const attaches = { + ...baseCorrectAttaches, + + videofile: buildAbsoluteFixturePath('video_short.mp4') + } + + await checkUploadVideoParam({ + ...baseOptions(), + attributes: { ...fields, ...attaches }, + expectedStatus: HttpStatusCode.OK_200 + }) + } + + { + const attaches = { + ...baseCorrectAttaches, + + videofile: buildAbsoluteFixturePath('video_short.ogv') + } + + await checkUploadVideoParam({ + ...baseOptions(), + attributes: { ...fields, ...attaches }, + expectedStatus: HttpStatusCode.OK_200 + }) + } + }) + } + + describe('Resumable upload', function () { + runSuite('resumable') + }) + + describe('Legacy upload', function () { + runSuite('legacy') + }) + }) + + describe('When updating a video', function () { + const baseCorrectParams = { + name: 'my super name', + category: 5, + licence: 2, + language: 'pt', + nsfw: false, + commentsEnabled: false, + downloadEnabled: false, + description: 'my super description', + privacy: VideoPrivacy.PUBLIC, + tags: [ 'tag1', 'tag2' ] + } + + before(async function () { + const { data } = await server.videos.list() + video = data[0] + }) + + it('Should fail with nothing', async function () { + const fields = {} + await makePutBodyRequest({ url: server.url, path, token: server.accessToken, fields }) + }) + + it('Should fail without a valid uuid', async function () { + const fields = baseCorrectParams + await makePutBodyRequest({ url: server.url, path: path + 'blabla', token: server.accessToken, fields }) + }) + + it('Should fail with an unknown id', async function () { + const fields = baseCorrectParams + + await makePutBodyRequest({ + url: server.url, + path: path + '4da6fde3-88f7-4d16-b119-108df5630b06', + token: server.accessToken, + fields, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail with a long name', async function () { + const fields = { ...baseCorrectParams, name: 'super'.repeat(65) } + + await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) + }) + + it('Should fail with a bad category', async function () { + const fields = { ...baseCorrectParams, category: 125 } + + await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) + }) + + it('Should fail with a bad licence', async function () { + const fields = { ...baseCorrectParams, licence: 125 } + + await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) + }) + + it('Should fail with a bad language', async function () { + const fields = { ...baseCorrectParams, language: 'a'.repeat(15) } + + await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) + }) + + it('Should fail with a long description', async function () { + const fields = { ...baseCorrectParams, description: 'super'.repeat(2500) } + + await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) + }) + + it('Should fail with a long support text', async function () { + const fields = { ...baseCorrectParams, support: 'super'.repeat(201) } + + await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) + }) + + it('Should fail with a bad channel', async function () { + const fields = { ...baseCorrectParams, channelId: 545454 } + + await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) + }) + + it('Should fail with too many tags', async function () { + const fields = { ...baseCorrectParams, tags: [ 'tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6' ] } + + await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) + }) + + it('Should fail with a tag length too low', async function () { + const fields = { ...baseCorrectParams, tags: [ 'tag1', 't' ] } + + await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) + }) + + it('Should fail with a tag length too big', async function () { + const fields = { ...baseCorrectParams, tags: [ 'tag1', 'my_super_tag_too_long_long_long_long_long_long' ] } + + await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) + }) + + it('Should fail with a bad schedule update (miss updateAt)', async function () { + const fields = { ...baseCorrectParams, scheduleUpdate: { privacy: VideoPrivacy.PUBLIC } } + + await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) + }) + + it('Should fail with a bad schedule update (wrong updateAt)', async function () { + const fields = { ...baseCorrectParams, scheduleUpdate: { updateAt: 'toto', privacy: VideoPrivacy.PUBLIC } } + + await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) + }) + + it('Should fail with a bad originally published at param', async function () { + const fields = { ...baseCorrectParams, originallyPublishedAt: 'toto' } + + await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) + }) + + it('Should fail with an incorrect thumbnail file', async function () { + const fields = baseCorrectParams + const attaches = { + thumbnailfile: buildAbsoluteFixturePath('video_short.mp4') + } + + await makeUploadRequest({ + url: server.url, + method: 'PUT', + path: path + video.shortUUID, + token: server.accessToken, + fields, + attaches + }) + }) + + it('Should fail with a big thumbnail file', async function () { + const fields = baseCorrectParams + const attaches = { + thumbnailfile: buildAbsoluteFixturePath('custom-preview-big.png') + } + + await makeUploadRequest({ + url: server.url, + method: 'PUT', + path: path + video.shortUUID, + token: server.accessToken, + fields, + attaches + }) + }) + + it('Should fail with an incorrect preview file', async function () { + const fields = baseCorrectParams + const attaches = { + previewfile: buildAbsoluteFixturePath('video_short.mp4') + } + + await makeUploadRequest({ + url: server.url, + method: 'PUT', + path: path + video.shortUUID, + token: server.accessToken, + fields, + attaches + }) + }) + + it('Should fail with a big preview file', async function () { + const fields = baseCorrectParams + const attaches = { + previewfile: buildAbsoluteFixturePath('custom-preview-big.png') + } + + await makeUploadRequest({ + url: server.url, + method: 'PUT', + path: path + video.shortUUID, + token: server.accessToken, + fields, + attaches + }) + }) + + it('Should fail with a video of another user without the appropriate right', async function () { + const fields = baseCorrectParams + + await makePutBodyRequest({ + url: server.url, + path: path + video.shortUUID, + token: userAccessToken, + fields, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with a video of another server') + + it('Shoud report the appropriate error', async function () { + const fields = { ...baseCorrectParams, licence: 125 } + + const res = await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) + const error = res.body as PeerTubeProblemDocument + + expect(error.docs).to.equal('https://docs.joinpeertube.org/api-rest-reference.html#operation/putVideo') + + expect(error.type).to.equal('about:blank') + expect(error.title).to.equal('Bad Request') + + expect(error.detail).to.equal('Incorrect request parameters: licence') + expect(error.error).to.equal('Incorrect request parameters: licence') + + expect(error.status).to.equal(HttpStatusCode.BAD_REQUEST_400) + expect(error['invalid-params'].licence).to.exist + }) + + it('Should succeed with the correct parameters', async function () { + const fields = baseCorrectParams + + await makePutBodyRequest({ + url: server.url, + path: path + video.shortUUID, + token: server.accessToken, + fields, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + describe('When getting a video', function () { + it('Should return the list of the videos with nothing', async function () { + const res = await makeGetRequest({ + url: server.url, + path, + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(6) + }) + + it('Should fail without a correct uuid', async function () { + await server.videos.get({ id: 'coucou', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should return 404 with an incorrect video', async function () { + await server.videos.get({ id: '4da6fde3-88f7-4d16-b119-108df5630b06', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Shoud report the appropriate error', async function () { + const body = await server.videos.get({ id: 'hi', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + const error = body as unknown as PeerTubeProblemDocument + + expect(error.docs).to.equal('https://docs.joinpeertube.org/api-rest-reference.html#operation/getVideo') + + expect(error.type).to.equal('about:blank') + expect(error.title).to.equal('Bad Request') + + expect(error.detail).to.equal('Incorrect request parameters: id') + expect(error.error).to.equal('Incorrect request parameters: id') + + expect(error.status).to.equal(HttpStatusCode.BAD_REQUEST_400) + expect(error['invalid-params'].id).to.exist + }) + + it('Should succeed with the correct parameters', async function () { + await server.videos.get({ id: video.shortUUID }) + }) + }) + + describe('When rating a video', function () { + let videoId: number + + before(async function () { + const { data } = await server.videos.list() + videoId = data[0].id + }) + + it('Should fail without a valid uuid', async function () { + const fields = { + rating: 'like' + } + await makePutBodyRequest({ url: server.url, path: path + 'blabla/rate', token: server.accessToken, fields }) + }) + + it('Should fail with an unknown id', async function () { + const fields = { + rating: 'like' + } + await makePutBodyRequest({ + url: server.url, + path: path + '4da6fde3-88f7-4d16-b119-108df5630b06/rate', + token: server.accessToken, + fields, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should fail with a wrong rating', async function () { + const fields = { + rating: 'likes' + } + await makePutBodyRequest({ url: server.url, path: path + videoId + '/rate', token: server.accessToken, fields }) + }) + + it('Should fail with a private video of another user', async function () { + const fields = { + rating: 'like' + } + await makePutBodyRequest({ + url: server.url, + path: path + privateVideo.uuid + '/rate', + token: userAccessToken, + fields, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should succeed with the correct parameters', async function () { + const fields = { + rating: 'like' + } + await makePutBodyRequest({ + url: server.url, + path: path + videoId + '/rate', + token: server.accessToken, + fields, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + }) + }) + + describe('When removing a video', function () { + it('Should have 404 with nothing', async function () { + await makeDeleteRequest({ + url: server.url, + path, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail without a correct uuid', async function () { + await server.videos.remove({ id: 'hello', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with a video which does not exist', async function () { + await server.videos.remove({ id: '4da6fde3-88f7-4d16-b119-108df5630b06', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should fail with a video of another user without the appropriate right', async function () { + await server.videos.remove({ token: userAccessToken, id: video.uuid, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail with a video of another server') + + it('Shoud report the appropriate error', async function () { + const body = await server.videos.remove({ id: 'hello', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + const error = body as PeerTubeProblemDocument + + expect(error.docs).to.equal('https://docs.joinpeertube.org/api-rest-reference.html#operation/delVideo') + + expect(error.type).to.equal('about:blank') + expect(error.title).to.equal('Bad Request') + + expect(error.detail).to.equal('Incorrect request parameters: id') + expect(error.error).to.equal('Incorrect request parameters: id') + + expect(error.status).to.equal(HttpStatusCode.BAD_REQUEST_400) + expect(error['invalid-params'].id).to.exist + }) + + it('Should succeed with the correct parameters', async function () { + await server.videos.remove({ id: video.uuid }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/check-params/views.ts b/packages/tests/src/api/check-params/views.ts new file mode 100644 index 000000000..c454d4b80 --- /dev/null +++ b/packages/tests/src/api/check-params/views.ts @@ -0,0 +1,227 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { HttpStatusCode, VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel +} from '@peertube/peertube-server-commands' + +describe('Test videos views', function () { + let servers: PeerTubeServer[] + let liveVideoId: string + let videoId: string + let remoteVideoId: string + let userAccessToken: string + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + await servers[0].config.enableLive({ allowReplay: false, transcoding: false }); + + ({ uuid: videoId } = await servers[0].videos.quickUpload({ name: 'video' })); + ({ uuid: remoteVideoId } = await servers[1].videos.quickUpload({ name: 'video' })); + ({ uuid: liveVideoId } = await servers[0].live.create({ + fields: { + name: 'live', + privacy: VideoPrivacy.PUBLIC, + channelId: servers[0].store.channel.id + } + })) + + userAccessToken = await servers[0].users.generateUserAndToken('user') + + await doubleFollow(servers[0], servers[1]) + }) + + describe('When viewing a video', async function () { + + it('Should fail without current time', async function () { + await servers[0].views.view({ id: videoId, currentTime: undefined, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with an invalid current time', async function () { + await servers[0].views.view({ id: videoId, currentTime: -1, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await servers[0].views.view({ id: videoId, currentTime: 10, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should succeed with correct parameters', async function () { + await servers[0].views.view({ id: videoId, currentTime: 1 }) + }) + }) + + describe('When getting overall stats', function () { + + it('Should fail with a remote video', async function () { + await servers[0].videoStats.getOverallStats({ videoId: remoteVideoId, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should fail without token', async function () { + await servers[0].videoStats.getOverallStats({ videoId, token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should fail with another token', async function () { + await servers[0].videoStats.getOverallStats({ + videoId, + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with an invalid start date', async function () { + await servers[0].videoStats.getOverallStats({ + videoId, + startDate: 'fake' as any, + endDate: new Date().toISOString(), + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with an invalid end date', async function () { + await servers[0].videoStats.getOverallStats({ + videoId, + startDate: new Date().toISOString(), + endDate: 'fake' as any, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should succeed with the correct parameters', async function () { + await servers[0].videoStats.getOverallStats({ + videoId, + startDate: new Date().toISOString(), + endDate: new Date().toISOString() + }) + }) + }) + + describe('When getting timeserie stats', function () { + + it('Should fail with a remote video', async function () { + await servers[0].videoStats.getTimeserieStats({ + videoId: remoteVideoId, + metric: 'viewers', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail without token', async function () { + await servers[0].videoStats.getTimeserieStats({ + videoId, + token: null, + metric: 'viewers', + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with another token', async function () { + await servers[0].videoStats.getTimeserieStats({ + videoId, + token: userAccessToken, + metric: 'viewers', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail with an invalid metric', async function () { + await servers[0].videoStats.getTimeserieStats({ videoId, metric: 'hello' as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should fail with an invalid start date', async function () { + await servers[0].videoStats.getTimeserieStats({ + videoId, + metric: 'viewers', + startDate: 'fake' as any, + endDate: new Date(), + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with an invalid end date', async function () { + await servers[0].videoStats.getTimeserieStats({ + videoId, + metric: 'viewers', + startDate: new Date(), + endDate: 'fake' as any, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail if start date is specified but not end date', async function () { + await servers[0].videoStats.getTimeserieStats({ + videoId, + metric: 'viewers', + startDate: new Date(), + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail if end date is specified but not start date', async function () { + await servers[0].videoStats.getTimeserieStats({ + videoId, + metric: 'viewers', + endDate: new Date(), + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should fail with a too big interval', async function () { + await servers[0].videoStats.getTimeserieStats({ + videoId, + metric: 'viewers', + startDate: new Date('2000-04-07T08:31:57.126Z'), + endDate: new Date(), + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should succeed with the correct parameters', async function () { + await servers[0].videoStats.getTimeserieStats({ videoId, metric: 'viewers' }) + }) + }) + + describe('When getting retention stats', function () { + + it('Should fail with a remote video', async function () { + await servers[0].videoStats.getRetentionStats({ + videoId: remoteVideoId, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail without token', async function () { + await servers[0].videoStats.getRetentionStats({ + videoId, + token: null, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + }) + + it('Should fail with another token', async function () { + await servers[0].videoStats.getRetentionStats({ + videoId, + token: userAccessToken, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should fail on live video', async function () { + await servers[0].videoStats.getRetentionStats({ videoId: liveVideoId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should succeed with the correct parameters', async function () { + await servers[0].videoStats.getRetentionStats({ videoId }) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/live/index.ts b/packages/tests/src/api/live/index.ts new file mode 100644 index 000000000..e61e6c611 --- /dev/null +++ b/packages/tests/src/api/live/index.ts @@ -0,0 +1,7 @@ +import './live-constraints.js' +import './live-fast-restream.js' +import './live-socket-messages.js' +import './live-permanent.js' +import './live-rtmps.js' +import './live-save-replay.js' +import './live.js' diff --git a/packages/tests/src/api/live/live-constraints.ts b/packages/tests/src/api/live/live-constraints.ts new file mode 100644 index 000000000..f62994cbd --- /dev/null +++ b/packages/tests/src/api/live/live-constraints.ts @@ -0,0 +1,237 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { LiveVideoError, UserVideoQuota, VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + ConfigCommand, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + stopFfmpeg, + waitJobs, + waitUntilLiveReplacedByReplayOnAllServers, + waitUntilLiveWaitingOnAllServers +} from '@peertube/peertube-server-commands' +import { checkLiveCleanup } from '../../shared/live.js' + +describe('Test live constraints', function () { + let servers: PeerTubeServer[] = [] + let userId: number + let userAccessToken: string + let userChannelId: number + + async function createLiveWrapper (options: { replay: boolean, permanent: boolean }) { + const { replay, permanent } = options + + const liveAttributes = { + name: 'user live', + channelId: userChannelId, + privacy: VideoPrivacy.PUBLIC, + saveReplay: replay, + replaySettings: options.replay ? { privacy: VideoPrivacy.PUBLIC } : undefined, + permanentLive: permanent + } + + const { uuid } = await servers[0].live.create({ token: userAccessToken, fields: liveAttributes }) + return uuid + } + + async function checkSaveReplay (videoId: string, resolutions = [ 720 ]) { + for (const server of servers) { + const video = await server.videos.get({ id: videoId }) + expect(video.isLive).to.be.false + expect(video.duration).to.be.greaterThan(0) + } + + await checkLiveCleanup({ server: servers[0], permanent: false, videoUUID: videoId, savedResolutions: resolutions }) + } + + function updateQuota (options: { total: number, daily: number }) { + return servers[0].users.update({ + userId, + videoQuota: options.total, + videoQuotaDaily: options.daily + }) + } + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(2) + + // Get the access tokens + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + await servers[0].config.updateCustomSubConfig({ + newConfig: { + live: { + enabled: true, + allowReplay: true, + transcoding: { + enabled: false + } + } + } + }) + + { + const res = await servers[0].users.generate('user1') + userId = res.userId + userChannelId = res.userChannelId + userAccessToken = res.token + + await updateQuota({ total: 1, daily: -1 }) + } + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + }) + + it('Should not have size limit if save replay is disabled', async function () { + this.timeout(60000) + + const userVideoLiveoId = await createLiveWrapper({ replay: false, permanent: false }) + await servers[0].live.runAndTestStreamError({ token: userAccessToken, videoId: userVideoLiveoId, shouldHaveError: false }) + }) + + it('Should have size limit depending on user global quota if save replay is enabled on non permanent live', async function () { + this.timeout(60000) + + // Wait for user quota memoize cache invalidation + await wait(5000) + + const userVideoLiveoId = await createLiveWrapper({ replay: true, permanent: false }) + await servers[0].live.runAndTestStreamError({ token: userAccessToken, videoId: userVideoLiveoId, shouldHaveError: true }) + + await waitUntilLiveReplacedByReplayOnAllServers(servers, userVideoLiveoId) + await waitJobs(servers) + + await checkSaveReplay(userVideoLiveoId) + + const session = await servers[0].live.getReplaySession({ videoId: userVideoLiveoId }) + expect(session.error).to.equal(LiveVideoError.QUOTA_EXCEEDED) + }) + + it('Should have size limit depending on user global quota if save replay is enabled on a permanent live', async function () { + this.timeout(60000) + + // Wait for user quota memoize cache invalidation + await wait(5000) + + const userVideoLiveoId = await createLiveWrapper({ replay: true, permanent: true }) + await servers[0].live.runAndTestStreamError({ token: userAccessToken, videoId: userVideoLiveoId, shouldHaveError: true }) + + await waitJobs(servers) + await waitUntilLiveWaitingOnAllServers(servers, userVideoLiveoId) + + const session = await servers[0].live.findLatestSession({ videoId: userVideoLiveoId }) + expect(session.error).to.equal(LiveVideoError.QUOTA_EXCEEDED) + }) + + it('Should have size limit depending on user daily quota if save replay is enabled', async function () { + this.timeout(60000) + + // Wait for user quota memoize cache invalidation + await wait(5000) + + await updateQuota({ total: -1, daily: 1 }) + + const userVideoLiveoId = await createLiveWrapper({ replay: true, permanent: false }) + await servers[0].live.runAndTestStreamError({ token: userAccessToken, videoId: userVideoLiveoId, shouldHaveError: true }) + + await waitUntilLiveReplacedByReplayOnAllServers(servers, userVideoLiveoId) + await waitJobs(servers) + + await checkSaveReplay(userVideoLiveoId) + + const session = await servers[0].live.getReplaySession({ videoId: userVideoLiveoId }) + expect(session.error).to.equal(LiveVideoError.QUOTA_EXCEEDED) + }) + + it('Should succeed without quota limit', async function () { + this.timeout(60000) + + // Wait for user quota memoize cache invalidation + await wait(5000) + + await updateQuota({ total: 10 * 1000 * 1000, daily: -1 }) + + const userVideoLiveoId = await createLiveWrapper({ replay: true, permanent: false }) + await servers[0].live.runAndTestStreamError({ token: userAccessToken, videoId: userVideoLiveoId, shouldHaveError: false }) + }) + + it('Should have the same quota in admin and as a user', async function () { + this.timeout(120000) + + const userVideoLiveoId = await createLiveWrapper({ replay: true, permanent: false }) + const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ token: userAccessToken, videoId: userVideoLiveoId }) + + await servers[0].live.waitUntilPublished({ videoId: userVideoLiveoId }) + // Wait previous live cleanups + await wait(3000) + + const baseQuota = await servers[0].users.getMyQuotaUsed({ token: userAccessToken }) + + let quotaUser: UserVideoQuota + + do { + await wait(500) + + quotaUser = await servers[0].users.getMyQuotaUsed({ token: userAccessToken }) + } while (quotaUser.videoQuotaUsed <= baseQuota.videoQuotaUsed) + + const { data } = await servers[0].users.list() + const quotaAdmin = data.find(u => u.username === 'user1') + + expect(quotaUser.videoQuotaUsed).to.be.above(baseQuota.videoQuotaUsed) + expect(quotaUser.videoQuotaUsedDaily).to.be.above(baseQuota.videoQuotaUsedDaily) + + expect(quotaAdmin.videoQuotaUsed).to.be.above(baseQuota.videoQuotaUsed) + expect(quotaAdmin.videoQuotaUsedDaily).to.be.above(baseQuota.videoQuotaUsedDaily) + + expect(quotaUser.videoQuotaUsed).to.be.above(10) + expect(quotaUser.videoQuotaUsedDaily).to.be.above(10) + expect(quotaAdmin.videoQuotaUsed).to.be.above(10) + expect(quotaAdmin.videoQuotaUsedDaily).to.be.above(10) + + await stopFfmpeg(ffmpegCommand) + }) + + it('Should have max duration limit', async function () { + this.timeout(240000) + + await servers[0].config.updateCustomSubConfig({ + newConfig: { + live: { + enabled: true, + allowReplay: true, + maxDuration: 15, + transcoding: { + enabled: true, + resolutions: ConfigCommand.getCustomConfigResolutions(true) + } + } + } + }) + + const userVideoLiveoId = await createLiveWrapper({ replay: true, permanent: false }) + await servers[0].live.runAndTestStreamError({ token: userAccessToken, videoId: userVideoLiveoId, shouldHaveError: true }) + + await waitUntilLiveReplacedByReplayOnAllServers(servers, userVideoLiveoId) + await waitJobs(servers) + + await checkSaveReplay(userVideoLiveoId, [ 720, 480, 360, 240, 144 ]) + + const session = await servers[0].live.getReplaySession({ videoId: userVideoLiveoId }) + expect(session.error).to.equal(LiveVideoError.DURATION_EXCEEDED) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/live/live-fast-restream.ts b/packages/tests/src/api/live/live-fast-restream.ts new file mode 100644 index 000000000..d34b00cbe --- /dev/null +++ b/packages/tests/src/api/live/live-fast-restream.ts @@ -0,0 +1,153 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { LiveVideoCreate, VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + stopFfmpeg, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Fast restream in live', function () { + let server: PeerTubeServer + + async function createLiveWrapper (options: { permanent: boolean, replay: boolean }) { + const attributes: LiveVideoCreate = { + channelId: server.store.channel.id, + privacy: VideoPrivacy.PUBLIC, + name: 'my super live', + saveReplay: options.replay, + replaySettings: options.replay ? { privacy: VideoPrivacy.PUBLIC } : undefined, + permanentLive: options.permanent + } + + const { uuid } = await server.live.create({ fields: attributes }) + return uuid + } + + async function fastRestreamWrapper ({ replay }: { replay: boolean }) { + const liveVideoUUID = await createLiveWrapper({ permanent: true, replay }) + await waitJobs([ server ]) + + const rtmpOptions = { + videoId: liveVideoUUID, + copyCodecs: true, + fixtureName: 'video_short.mp4' + } + + // Streaming session #1 + let ffmpegCommand = await server.live.sendRTMPStreamInVideo(rtmpOptions) + await server.live.waitUntilPublished({ videoId: liveVideoUUID }) + + const video = await server.videos.get({ id: liveVideoUUID }) + const session1PlaylistId = video.streamingPlaylists[0].id + + await stopFfmpeg(ffmpegCommand) + await server.live.waitUntilWaiting({ videoId: liveVideoUUID }) + + // Streaming session #2 + ffmpegCommand = await server.live.sendRTMPStreamInVideo(rtmpOptions) + + let hasNewPlaylist = false + do { + const video = await server.videos.get({ id: liveVideoUUID }) + hasNewPlaylist = video.streamingPlaylists.length === 1 && video.streamingPlaylists[0].id !== session1PlaylistId + + await wait(100) + } while (!hasNewPlaylist) + + await server.live.waitUntilSegmentGeneration({ + server, + videoUUID: liveVideoUUID, + segment: 1, + playlistNumber: 0 + }) + + return { ffmpegCommand, liveVideoUUID } + } + + async function ensureLastLiveWorks (liveId: string) { + // Equivalent to PEERTUBE_TEST_CONSTANTS_VIDEO_LIVE_CLEANUP_DELAY + for (let i = 0; i < 100; i++) { + const video = await server.videos.get({ id: liveId }) + expect(video.streamingPlaylists).to.have.lengthOf(1) + + try { + await server.live.getSegmentFile({ videoUUID: liveId, segment: 0, playlistNumber: 0 }) + await server.streamingPlaylists.get({ url: video.streamingPlaylists[0].playlistUrl }) + await server.streamingPlaylists.getSegmentSha256({ url: video.streamingPlaylists[0].segmentsSha256Url }) + } catch (err) { + // FIXME: try to debug error in CI "Unexpected end of JSON input" + console.error(err) + throw err + } + + await wait(100) + } + } + + async function runTest (replay: boolean) { + const { ffmpegCommand, liveVideoUUID } = await fastRestreamWrapper({ replay }) + + // TODO: remove, we try to debug a test timeout failure here + console.log('Ensuring last live works') + + await ensureLastLiveWorks(liveVideoUUID) + + await stopFfmpeg(ffmpegCommand) + await server.live.waitUntilWaiting({ videoId: liveVideoUUID }) + + // Wait for replays + await waitJobs([ server ]) + + const { total, data: sessions } = await server.live.listSessions({ videoId: liveVideoUUID }) + + expect(total).to.equal(2) + expect(sessions).to.have.lengthOf(2) + + for (const session of sessions) { + expect(session.error).to.be.null + + if (replay) { + expect(session.replayVideo).to.exist + + await server.videos.get({ id: session.replayVideo.uuid }) + } else { + expect(session.replayVideo).to.not.exist + } + } + } + + before(async function () { + this.timeout(120000) + + const env = { PEERTUBE_TEST_CONSTANTS_VIDEO_LIVE_CLEANUP_DELAY: '10000' } + server = await createSingleServer(1, {}, { env }) + + // Get the access tokens + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + await server.config.enableMinimumTranscoding({ webVideo: false, hls: true }) + await server.config.enableLive({ allowReplay: true, transcoding: true, resolutions: 'min' }) + }) + + it('Should correctly fast restream in a permanent live with and without save replay', async function () { + this.timeout(480000) + + // A test can take a long time, so prefer to run them in parallel + await Promise.all([ + runTest(true), + runTest(false) + ]) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/live/live-permanent.ts b/packages/tests/src/api/live/live-permanent.ts new file mode 100644 index 000000000..4ffcc7ed4 --- /dev/null +++ b/packages/tests/src/api/live/live-permanent.ts @@ -0,0 +1,204 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { LiveVideoCreate, VideoPrivacy, VideoState, VideoStateType } from '@peertube/peertube-models' +import { checkLiveCleanup } from '@tests/shared/live.js' +import { + cleanupTests, + ConfigCommand, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + stopFfmpeg, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Permanent live', function () { + let servers: PeerTubeServer[] = [] + let videoUUID: string + + async function createLiveWrapper (permanentLive: boolean) { + const attributes: LiveVideoCreate = { + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC, + name: 'my super live', + saveReplay: false, + permanentLive + } + + const { uuid } = await servers[0].live.create({ fields: attributes }) + return uuid + } + + async function checkVideoState (videoId: string, state: VideoStateType) { + for (const server of servers) { + const video = await server.videos.get({ id: videoId }) + expect(video.state.id).to.equal(state) + } + } + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(2) + + // Get the access tokens + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + + await servers[0].config.updateCustomSubConfig({ + newConfig: { + live: { + enabled: true, + allowReplay: true, + maxDuration: -1, + transcoding: { + enabled: true, + resolutions: ConfigCommand.getCustomConfigResolutions(true) + } + } + } + }) + }) + + it('Should create a non permanent live and update it to be a permanent live', async function () { + this.timeout(20000) + + const videoUUID = await createLiveWrapper(false) + + { + const live = await servers[0].live.get({ videoId: videoUUID }) + expect(live.permanentLive).to.be.false + } + + await servers[0].live.update({ videoId: videoUUID, fields: { permanentLive: true } }) + + { + const live = await servers[0].live.get({ videoId: videoUUID }) + expect(live.permanentLive).to.be.true + } + }) + + it('Should create a permanent live', async function () { + this.timeout(20000) + + videoUUID = await createLiveWrapper(true) + + const live = await servers[0].live.get({ videoId: videoUUID }) + expect(live.permanentLive).to.be.true + + await waitJobs(servers) + }) + + it('Should stream into this permanent live', async function () { + this.timeout(240_000) + + const beforePublication = new Date() + const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID }) + + for (const server of servers) { + await server.live.waitUntilPublished({ videoId: videoUUID }) + } + + await checkVideoState(videoUUID, VideoState.PUBLISHED) + + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + expect(new Date(video.publishedAt)).greaterThan(beforePublication) + } + + await stopFfmpeg(ffmpegCommand) + await servers[0].live.waitUntilWaiting({ videoId: videoUUID }) + + await waitJobs(servers) + }) + + it('Should have cleaned up this live', async function () { + this.timeout(40000) + + await wait(5000) + await waitJobs(servers) + + for (const server of servers) { + const videoDetails = await server.videos.get({ id: videoUUID }) + + expect(videoDetails.streamingPlaylists).to.have.lengthOf(0) + } + + await checkLiveCleanup({ server: servers[0], permanent: true, videoUUID }) + }) + + it('Should have set this live to waiting for live state', async function () { + this.timeout(20000) + + await checkVideoState(videoUUID, VideoState.WAITING_FOR_LIVE) + }) + + it('Should be able to stream again in the permanent live', async function () { + this.timeout(60000) + + await servers[0].config.updateCustomSubConfig({ + newConfig: { + live: { + enabled: true, + allowReplay: true, + maxDuration: -1, + transcoding: { + enabled: true, + resolutions: ConfigCommand.getCustomConfigResolutions(false) + } + } + } + }) + + const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID }) + + for (const server of servers) { + await server.live.waitUntilPublished({ videoId: videoUUID }) + } + + await checkVideoState(videoUUID, VideoState.PUBLISHED) + + const count = await servers[0].live.countPlaylists({ videoUUID }) + // master playlist and 720p playlist + expect(count).to.equal(2) + + await stopFfmpeg(ffmpegCommand) + }) + + it('Should have appropriate sessions', async function () { + this.timeout(60000) + + await servers[0].live.waitUntilWaiting({ videoId: videoUUID }) + + const { data, total } = await servers[0].live.listSessions({ videoId: videoUUID }) + expect(total).to.equal(2) + expect(data).to.have.lengthOf(2) + + for (const session of data) { + expect(session.startDate).to.exist + expect(session.endDate).to.exist + + expect(session.error).to.not.exist + } + }) + + it('Should remove the live and have cleaned up the directory', async function () { + this.timeout(60000) + + await servers[0].videos.remove({ id: videoUUID }) + await waitJobs(servers) + + await checkLiveCleanup({ server: servers[0], permanent: true, videoUUID }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/live/live-rtmps.ts b/packages/tests/src/api/live/live-rtmps.ts new file mode 100644 index 000000000..4ab59ed4c --- /dev/null +++ b/packages/tests/src/api/live/live-rtmps.ts @@ -0,0 +1,143 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import { VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + PeerTubeServer, + sendRTMPStream, + setAccessTokensToServers, + setDefaultVideoChannel, + stopFfmpeg, + testFfmpegStreamError, + waitUntilLivePublishedOnAllServers +} from '@peertube/peertube-server-commands' + +describe('Test live RTMPS', function () { + let server: PeerTubeServer + let rtmpUrl: string + let rtmpsUrl: string + + async function createLiveWrapper () { + const liveAttributes = { + name: 'live', + channelId: server.store.channel.id, + privacy: VideoPrivacy.PUBLIC, + saveReplay: false + } + + const { uuid } = await server.live.create({ fields: liveAttributes }) + + const live = await server.live.get({ videoId: uuid }) + const video = await server.videos.get({ id: uuid }) + + return Object.assign(video, live) + } + + before(async function () { + this.timeout(120000) + + server = await createSingleServer(1) + + // Get the access tokens + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + await server.config.updateCustomSubConfig({ + newConfig: { + live: { + enabled: true, + allowReplay: true, + transcoding: { + enabled: false + } + } + } + }) + + rtmpUrl = 'rtmp://' + server.hostname + ':' + server.rtmpPort + '/live' + rtmpsUrl = 'rtmps://' + server.hostname + ':' + server.rtmpsPort + '/live' + }) + + it('Should enable RTMPS endpoint only', async function () { + this.timeout(240000) + + await server.kill() + await server.run({ + live: { + rtmp: { + enabled: false + }, + rtmps: { + enabled: true, + port: server.rtmpsPort, + key_file: buildAbsoluteFixturePath('rtmps.key'), + cert_file: buildAbsoluteFixturePath('rtmps.cert') + } + } + }) + + { + const liveVideo = await createLiveWrapper() + + expect(liveVideo.rtmpUrl).to.not.exist + expect(liveVideo.rtmpsUrl).to.equal(rtmpsUrl) + + const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl, streamKey: liveVideo.streamKey }) + await testFfmpegStreamError(command, true) + } + + { + const liveVideo = await createLiveWrapper() + + const command = sendRTMPStream({ rtmpBaseUrl: rtmpsUrl, streamKey: liveVideo.streamKey }) + await waitUntilLivePublishedOnAllServers([ server ], liveVideo.uuid) + await stopFfmpeg(command) + } + }) + + it('Should enable both RTMP and RTMPS', async function () { + this.timeout(240000) + + await server.kill() + await server.run({ + live: { + rtmp: { + enabled: true, + port: server.rtmpPort + }, + rtmps: { + enabled: true, + port: server.rtmpsPort, + key_file: buildAbsoluteFixturePath('rtmps.key'), + cert_file: buildAbsoluteFixturePath('rtmps.cert') + } + } + }) + + { + const liveVideo = await createLiveWrapper() + + expect(liveVideo.rtmpUrl).to.equal(rtmpUrl) + expect(liveVideo.rtmpsUrl).to.equal(rtmpsUrl) + + const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl, streamKey: liveVideo.streamKey }) + await waitUntilLivePublishedOnAllServers([ server ], liveVideo.uuid) + await stopFfmpeg(command) + } + + { + const liveVideo = await createLiveWrapper() + + const command = sendRTMPStream({ rtmpBaseUrl: rtmpsUrl, streamKey: liveVideo.streamKey }) + await waitUntilLivePublishedOnAllServers([ server ], liveVideo.uuid) + await stopFfmpeg(command) + } + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/live/live-save-replay.ts b/packages/tests/src/api/live/live-save-replay.ts new file mode 100644 index 000000000..84135365b --- /dev/null +++ b/packages/tests/src/api/live/live-save-replay.ts @@ -0,0 +1,583 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { FfmpegCommand } from 'fluent-ffmpeg' +import { wait } from '@peertube/peertube-core-utils' +import { + HttpStatusCode, + HttpStatusCodeType, + LiveVideoCreate, + LiveVideoError, + VideoPrivacy, + VideoPrivacyType, + VideoState, + VideoStateType +} from '@peertube/peertube-models' +import { checkLiveCleanup } from '@tests/shared/live.js' +import { + cleanupTests, + ConfigCommand, + createMultipleServers, + doubleFollow, + findExternalSavedVideo, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + stopFfmpeg, + testFfmpegStreamError, + waitJobs, + waitUntilLivePublishedOnAllServers, + waitUntilLiveReplacedByReplayOnAllServers, + waitUntilLiveWaitingOnAllServers +} from '@peertube/peertube-server-commands' + +describe('Save replay setting', function () { + let servers: PeerTubeServer[] = [] + let liveVideoUUID: string + let ffmpegCommand: FfmpegCommand + + async function createLiveWrapper (options: { permanent: boolean, replay: boolean, replaySettings?: { privacy: VideoPrivacyType } }) { + if (liveVideoUUID) { + try { + await servers[0].videos.remove({ id: liveVideoUUID }) + await waitJobs(servers) + } catch {} + } + + const attributes: LiveVideoCreate = { + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC, + name: 'live'.repeat(30), + saveReplay: options.replay, + replaySettings: options.replaySettings, + permanentLive: options.permanent + } + + const { uuid } = await servers[0].live.create({ fields: attributes }) + return uuid + } + + async function publishLive (options: { permanent: boolean, replay: boolean, replaySettings?: { privacy: VideoPrivacyType } }) { + liveVideoUUID = await createLiveWrapper(options) + + const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) + await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) + + const liveDetails = await servers[0].videos.get({ id: liveVideoUUID }) + + await waitJobs(servers) + await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) + + return { ffmpegCommand, liveDetails } + } + + async function publishLiveAndDelete (options: { permanent: boolean, replay: boolean, replaySettings?: { privacy: VideoPrivacyType } }) { + const { ffmpegCommand, liveDetails } = await publishLive(options) + + await Promise.all([ + servers[0].videos.remove({ id: liveVideoUUID }), + testFfmpegStreamError(ffmpegCommand, true) + ]) + + await waitJobs(servers) + await wait(5000) + await waitJobs(servers) + + return { liveDetails } + } + + async function publishLiveAndBlacklist (options: { + permanent: boolean + replay: boolean + replaySettings?: { privacy: VideoPrivacyType } + }) { + const { ffmpegCommand, liveDetails } = await publishLive(options) + + await Promise.all([ + servers[0].blacklist.add({ videoId: liveVideoUUID, reason: 'bad live', unfederate: true }), + testFfmpegStreamError(ffmpegCommand, true) + ]) + + await waitJobs(servers) + await wait(5000) + await waitJobs(servers) + + return { liveDetails } + } + + async function checkVideosExist (videoId: string, existsInList: boolean, expectedStatus?: HttpStatusCodeType) { + for (const server of servers) { + const length = existsInList ? 1 : 0 + + const { data, total } = await server.videos.list() + expect(data).to.have.lengthOf(length) + expect(total).to.equal(length) + + if (expectedStatus) { + await server.videos.get({ id: videoId, expectedStatus }) + } + } + } + + async function checkVideoState (videoId: string, state: VideoStateType) { + for (const server of servers) { + const video = await server.videos.get({ id: videoId }) + expect(video.state.id).to.equal(state) + } + } + + async function checkVideoPrivacy (videoId: string, privacy: VideoPrivacyType) { + for (const server of servers) { + const video = await server.videos.get({ id: videoId }) + expect(video.privacy.id).to.equal(privacy) + } + } + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(2) + + // Get the access tokens + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + + await servers[0].config.updateCustomSubConfig({ + newConfig: { + live: { + enabled: true, + allowReplay: true, + maxDuration: -1, + transcoding: { + enabled: false, + resolutions: ConfigCommand.getCustomConfigResolutions(true) + } + } + } + }) + }) + + describe('With save replay disabled', function () { + let sessionStartDateMin: Date + let sessionStartDateMax: Date + let sessionEndDateMin: Date + + it('Should correctly create and federate the "waiting for stream" live', async function () { + this.timeout(40000) + + liveVideoUUID = await createLiveWrapper({ permanent: false, replay: false }) + + await waitJobs(servers) + + await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200) + await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE) + }) + + it('Should correctly have updated the live and federated it when streaming in the live', async function () { + this.timeout(120000) + + ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) + + sessionStartDateMin = new Date() + await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) + sessionStartDateMax = new Date() + + await waitJobs(servers) + + await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) + await checkVideoState(liveVideoUUID, VideoState.PUBLISHED) + }) + + it('Should correctly delete the video files after the stream ended', async function () { + this.timeout(120000) + + sessionEndDateMin = new Date() + await stopFfmpeg(ffmpegCommand) + + for (const server of servers) { + await server.live.waitUntilEnded({ videoId: liveVideoUUID }) + } + await waitJobs(servers) + + // Live still exist, but cannot be played anymore + await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200) + await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED) + + // No resolutions saved since we did not save replay + await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false }) + }) + + it('Should have appropriate ended session', async function () { + const { data, total } = await servers[0].live.listSessions({ videoId: liveVideoUUID }) + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + + const session = data[0] + + const startDate = new Date(session.startDate) + expect(startDate).to.be.above(sessionStartDateMin) + expect(startDate).to.be.below(sessionStartDateMax) + + expect(session.endDate).to.exist + expect(new Date(session.endDate)).to.be.above(sessionEndDateMin) + + expect(session.saveReplay).to.be.false + expect(session.error).to.not.exist + expect(session.replayVideo).to.not.exist + }) + + it('Should correctly terminate the stream on blacklist and delete the live', async function () { + this.timeout(120000) + + await publishLiveAndBlacklist({ permanent: false, replay: false }) + + await checkVideosExist(liveVideoUUID, false) + + await servers[0].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + await servers[1].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + + await wait(5000) + await waitJobs(servers) + await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false }) + }) + + it('Should have blacklisted session error', async function () { + const session = await servers[0].live.findLatestSession({ videoId: liveVideoUUID }) + expect(session.startDate).to.exist + expect(session.endDate).to.exist + + expect(session.error).to.equal(LiveVideoError.BLACKLISTED) + expect(session.replayVideo).to.not.exist + }) + + it('Should correctly terminate the stream on delete and delete the video', async function () { + this.timeout(120000) + + await publishLiveAndDelete({ permanent: false, replay: false }) + + await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) + await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false }) + }) + }) + + describe('With save replay enabled on non permanent live', function () { + + it('Should correctly create and federate the "waiting for stream" live', async function () { + this.timeout(120000) + + liveVideoUUID = await createLiveWrapper({ permanent: false, replay: true, replaySettings: { privacy: VideoPrivacy.UNLISTED } }) + + await waitJobs(servers) + + await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200) + await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE) + await checkVideoPrivacy(liveVideoUUID, VideoPrivacy.PUBLIC) + }) + + it('Should correctly have updated the live and federated it when streaming in the live', async function () { + this.timeout(120000) + + ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) + await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) + + await waitJobs(servers) + + await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) + await checkVideoState(liveVideoUUID, VideoState.PUBLISHED) + await checkVideoPrivacy(liveVideoUUID, VideoPrivacy.PUBLIC) + }) + + it('Should correctly have saved the live and federated it after the streaming', async function () { + this.timeout(120000) + + const session = await servers[0].live.findLatestSession({ videoId: liveVideoUUID }) + expect(session.endDate).to.not.exist + expect(session.endingProcessed).to.be.false + expect(session.saveReplay).to.be.true + expect(session.replaySettings).to.exist + expect(session.replaySettings.privacy).to.equal(VideoPrivacy.UNLISTED) + + await stopFfmpeg(ffmpegCommand) + + await waitUntilLiveReplacedByReplayOnAllServers(servers, liveVideoUUID) + await waitJobs(servers) + + // Live has been transcoded + await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200) + await checkVideoState(liveVideoUUID, VideoState.PUBLISHED) + await checkVideoPrivacy(liveVideoUUID, VideoPrivacy.UNLISTED) + }) + + it('Should find the replay live session', async function () { + const session = await servers[0].live.getReplaySession({ videoId: liveVideoUUID }) + + expect(session).to.exist + + expect(session.startDate).to.exist + expect(session.endDate).to.exist + + expect(session.error).to.not.exist + expect(session.saveReplay).to.be.true + expect(session.endingProcessed).to.be.true + expect(session.replaySettings).to.exist + expect(session.replaySettings.privacy).to.equal(VideoPrivacy.UNLISTED) + + expect(session.replayVideo).to.exist + expect(session.replayVideo.id).to.exist + expect(session.replayVideo.shortUUID).to.exist + expect(session.replayVideo.uuid).to.equal(liveVideoUUID) + }) + + it('Should update the saved live and correctly federate the updated attributes', async function () { + this.timeout(120000) + + await servers[0].videos.update({ id: liveVideoUUID, attributes: { name: 'video updated', privacy: VideoPrivacy.PUBLIC } }) + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: liveVideoUUID }) + expect(video.name).to.equal('video updated') + expect(video.isLive).to.be.false + expect(video.privacy.id).to.equal(VideoPrivacy.PUBLIC) + } + }) + + it('Should have cleaned up the live files', async function () { + await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false, savedResolutions: [ 720 ] }) + }) + + it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () { + this.timeout(120000) + + await publishLiveAndBlacklist({ permanent: false, replay: true, replaySettings: { privacy: VideoPrivacy.PUBLIC } }) + + await checkVideosExist(liveVideoUUID, false) + + await servers[0].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + await servers[1].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + + await wait(5000) + await waitJobs(servers) + await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false, savedResolutions: [ 720 ] }) + }) + + it('Should correctly terminate the stream on delete and delete the video', async function () { + this.timeout(120000) + + await publishLiveAndDelete({ permanent: false, replay: true, replaySettings: { privacy: VideoPrivacy.PUBLIC } }) + + await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) + await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false }) + }) + }) + + describe('With save replay enabled on permanent live', function () { + let lastReplayUUID: string + + describe('With a first live and its replay', function () { + + it('Should correctly create and federate the "waiting for stream" live', async function () { + this.timeout(120000) + + liveVideoUUID = await createLiveWrapper({ permanent: true, replay: true, replaySettings: { privacy: VideoPrivacy.UNLISTED } }) + + await waitJobs(servers) + + await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200) + await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE) + await checkVideoPrivacy(liveVideoUUID, VideoPrivacy.PUBLIC) + }) + + it('Should correctly have updated the live and federated it when streaming in the live', async function () { + this.timeout(120000) + + ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) + await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) + + await waitJobs(servers) + + await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) + await checkVideoState(liveVideoUUID, VideoState.PUBLISHED) + await checkVideoPrivacy(liveVideoUUID, VideoPrivacy.PUBLIC) + }) + + it('Should correctly have saved the live and federated it after the streaming', async function () { + this.timeout(120000) + + const liveDetails = await servers[0].videos.get({ id: liveVideoUUID }) + + await stopFfmpeg(ffmpegCommand) + + await waitUntilLiveWaitingOnAllServers(servers, liveVideoUUID) + await waitJobs(servers) + + const video = await findExternalSavedVideo(servers[0], liveDetails) + expect(video).to.exist + + for (const server of servers) { + await server.videos.get({ id: video.uuid }) + } + + lastReplayUUID = video.uuid + }) + + it('Should have appropriate ended session and replay live session', async function () { + const { data, total } = await servers[0].live.listSessions({ videoId: liveVideoUUID }) + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + + const sessionFromLive = data[0] + const sessionFromReplay = await servers[0].live.getReplaySession({ videoId: lastReplayUUID }) + + for (const session of [ sessionFromLive, sessionFromReplay ]) { + expect(session.startDate).to.exist + expect(session.endDate).to.exist + + expect(session.replaySettings).to.exist + expect(session.replaySettings.privacy).to.equal(VideoPrivacy.UNLISTED) + + expect(session.error).to.not.exist + + expect(session.replayVideo).to.exist + expect(session.replayVideo.id).to.exist + expect(session.replayVideo.shortUUID).to.exist + expect(session.replayVideo.uuid).to.equal(lastReplayUUID) + } + }) + + it('Should have the first live replay with correct settings', async function () { + await checkVideosExist(lastReplayUUID, false, HttpStatusCode.OK_200) + await checkVideoState(lastReplayUUID, VideoState.PUBLISHED) + await checkVideoPrivacy(lastReplayUUID, VideoPrivacy.UNLISTED) + }) + }) + + describe('With a second live and its replay', function () { + + it('Should update the replay settings', async function () { + await servers[0].live.update({ videoId: liveVideoUUID, fields: { replaySettings: { privacy: VideoPrivacy.PUBLIC } } }) + await waitJobs(servers) + + const live = await servers[0].live.get({ videoId: liveVideoUUID }) + + expect(live.saveReplay).to.be.true + expect(live.replaySettings).to.exist + expect(live.replaySettings.privacy).to.equal(VideoPrivacy.PUBLIC) + + }) + + it('Should correctly have updated the live and federated it when streaming in the live', async function () { + this.timeout(120000) + + ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) + await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) + + await waitJobs(servers) + + await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) + await checkVideoState(liveVideoUUID, VideoState.PUBLISHED) + await checkVideoPrivacy(liveVideoUUID, VideoPrivacy.PUBLIC) + }) + + it('Should correctly have saved the live and federated it after the streaming', async function () { + this.timeout(120000) + + const liveDetails = await servers[0].videos.get({ id: liveVideoUUID }) + + await stopFfmpeg(ffmpegCommand) + + await waitUntilLiveWaitingOnAllServers(servers, liveVideoUUID) + await waitJobs(servers) + + const video = await findExternalSavedVideo(servers[0], liveDetails) + expect(video).to.exist + + for (const server of servers) { + await server.videos.get({ id: video.uuid }) + } + + lastReplayUUID = video.uuid + }) + + it('Should have appropriate ended session and replay live session', async function () { + const { data, total } = await servers[0].live.listSessions({ videoId: liveVideoUUID }) + expect(total).to.equal(2) + expect(data).to.have.lengthOf(2) + + const sessionFromLive = data[1] + const sessionFromReplay = await servers[0].live.getReplaySession({ videoId: lastReplayUUID }) + + for (const session of [ sessionFromLive, sessionFromReplay ]) { + expect(session.startDate).to.exist + expect(session.endDate).to.exist + + expect(session.replaySettings).to.exist + expect(session.replaySettings.privacy).to.equal(VideoPrivacy.PUBLIC) + + expect(session.error).to.not.exist + + expect(session.replayVideo).to.exist + expect(session.replayVideo.id).to.exist + expect(session.replayVideo.shortUUID).to.exist + expect(session.replayVideo.uuid).to.equal(lastReplayUUID) + } + }) + + it('Should have the first live replay with correct settings', async function () { + await checkVideosExist(lastReplayUUID, true, HttpStatusCode.OK_200) + await checkVideoState(lastReplayUUID, VideoState.PUBLISHED) + await checkVideoPrivacy(lastReplayUUID, VideoPrivacy.PUBLIC) + }) + + it('Should have cleaned up the live files', async function () { + await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false }) + }) + + it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () { + this.timeout(120000) + + await servers[0].videos.remove({ id: lastReplayUUID }) + const { liveDetails } = await publishLiveAndBlacklist({ + permanent: true, + replay: true, + replaySettings: { privacy: VideoPrivacy.PUBLIC } + }) + + const replay = await findExternalSavedVideo(servers[0], liveDetails) + expect(replay).to.exist + + for (const videoId of [ liveVideoUUID, replay.uuid ]) { + await checkVideosExist(videoId, false) + + await servers[0].videos.get({ id: videoId, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + await servers[1].videos.get({ id: videoId, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + } + + await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false }) + }) + + it('Should correctly terminate the stream on delete and not save the video', async function () { + this.timeout(120000) + + const { liveDetails } = await publishLiveAndDelete({ + permanent: true, + replay: true, + replaySettings: { privacy: VideoPrivacy.PUBLIC } + }) + + const replay = await findExternalSavedVideo(servers[0], liveDetails) + expect(replay).to.not.exist + + await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) + await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false }) + }) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/live/live-socket-messages.ts b/packages/tests/src/api/live/live-socket-messages.ts new file mode 100644 index 000000000..80bae154c --- /dev/null +++ b/packages/tests/src/api/live/live-socket-messages.ts @@ -0,0 +1,186 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { LiveVideoEventPayload, VideoPrivacy, VideoState, VideoStateType } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + stopFfmpeg, + waitJobs, + waitUntilLivePublishedOnAllServers +} from '@peertube/peertube-server-commands' + +describe('Test live socket messages', function () { + let servers: PeerTubeServer[] = [] + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(2) + + // Get the access tokens + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + await servers[0].config.updateCustomSubConfig({ + newConfig: { + live: { + enabled: true, + allowReplay: true, + transcoding: { + enabled: false + } + } + } + }) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + }) + + describe('Live socket messages', function () { + + async function createLiveWrapper () { + const liveAttributes = { + name: 'live video', + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC + } + + const { uuid } = await servers[0].live.create({ fields: liveAttributes }) + return uuid + } + + it('Should correctly send a message when the live starts and ends', async function () { + this.timeout(60000) + + const localStateChanges: VideoStateType[] = [] + const remoteStateChanges: VideoStateType[] = [] + + const liveVideoUUID = await createLiveWrapper() + await waitJobs(servers) + + { + const videoId = await servers[0].videos.getId({ uuid: liveVideoUUID }) + + const localSocket = servers[0].socketIO.getLiveNotificationSocket() + localSocket.on('state-change', data => localStateChanges.push(data.state)) + localSocket.emit('subscribe', { videoId }) + } + + { + const videoId = await servers[1].videos.getId({ uuid: liveVideoUUID }) + + const remoteSocket = servers[1].socketIO.getLiveNotificationSocket() + remoteSocket.on('state-change', data => remoteStateChanges.push(data.state)) + remoteSocket.emit('subscribe', { videoId }) + } + + const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) + + await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) + await waitJobs(servers) + + for (const stateChanges of [ localStateChanges, remoteStateChanges ]) { + expect(stateChanges).to.have.length.at.least(1) + expect(stateChanges[stateChanges.length - 1]).to.equal(VideoState.PUBLISHED) + } + + await stopFfmpeg(ffmpegCommand) + + for (const server of servers) { + await server.live.waitUntilEnded({ videoId: liveVideoUUID }) + } + await waitJobs(servers) + + for (const stateChanges of [ localStateChanges, remoteStateChanges ]) { + expect(stateChanges).to.have.length.at.least(2) + expect(stateChanges[stateChanges.length - 1]).to.equal(VideoState.LIVE_ENDED) + } + }) + + it('Should correctly send views change notification', async function () { + this.timeout(60000) + + let localLastVideoViews = 0 + let remoteLastVideoViews = 0 + + const liveVideoUUID = await createLiveWrapper() + await waitJobs(servers) + + { + const videoId = await servers[0].videos.getId({ uuid: liveVideoUUID }) + + const localSocket = servers[0].socketIO.getLiveNotificationSocket() + localSocket.on('views-change', (data: LiveVideoEventPayload) => { localLastVideoViews = data.viewers }) + localSocket.emit('subscribe', { videoId }) + } + + { + const videoId = await servers[1].videos.getId({ uuid: liveVideoUUID }) + + const remoteSocket = servers[1].socketIO.getLiveNotificationSocket() + remoteSocket.on('views-change', (data: LiveVideoEventPayload) => { remoteLastVideoViews = data.viewers }) + remoteSocket.emit('subscribe', { videoId }) + } + + const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) + + await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) + await waitJobs(servers) + + expect(localLastVideoViews).to.equal(0) + expect(remoteLastVideoViews).to.equal(0) + + await servers[0].views.simulateView({ id: liveVideoUUID }) + await servers[1].views.simulateView({ id: liveVideoUUID }) + + await waitJobs(servers) + + expect(localLastVideoViews).to.equal(2) + expect(remoteLastVideoViews).to.equal(2) + + await stopFfmpeg(ffmpegCommand) + }) + + it('Should not receive a notification after unsubscribe', async function () { + this.timeout(120000) + + const stateChanges: VideoStateType[] = [] + + const liveVideoUUID = await createLiveWrapper() + await waitJobs(servers) + + const videoId = await servers[0].videos.getId({ uuid: liveVideoUUID }) + + const socket = servers[0].socketIO.getLiveNotificationSocket() + socket.on('state-change', data => stateChanges.push(data.state)) + socket.emit('subscribe', { videoId }) + + const command = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) + + await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) + await waitJobs(servers) + + // Notifier waits before sending a notification + await wait(10000) + + expect(stateChanges).to.have.lengthOf(1) + socket.emit('unsubscribe', { videoId }) + + await stopFfmpeg(command) + await waitJobs(servers) + + expect(stateChanges).to.have.lengthOf(1) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/live/live.ts b/packages/tests/src/api/live/live.ts new file mode 100644 index 000000000..20804f889 --- /dev/null +++ b/packages/tests/src/api/live/live.ts @@ -0,0 +1,766 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { basename, join } from 'path' +import { getAllFiles, wait } from '@peertube/peertube-core-utils' +import { ffprobePromise, getVideoStream } from '@peertube/peertube-ffmpeg' +import { + HttpStatusCode, + LiveVideo, + LiveVideoCreate, + LiveVideoLatencyMode, + VideoDetails, + VideoPrivacy, + VideoState, + VideoStreamingPlaylistType +} from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + killallServers, + LiveCommand, + makeGetRequest, + makeRawRequest, + PeerTubeServer, + sendRTMPStream, + setAccessTokensToServers, + setDefaultVideoChannel, + stopFfmpeg, + testFfmpegStreamError, + waitJobs, + waitUntilLivePublishedOnAllServers +} from '@peertube/peertube-server-commands' +import { testImageGeneratedByFFmpeg } from '@tests/shared/checks.js' +import { testLiveVideoResolutions } from '@tests/shared/live.js' +import { SQLCommand } from '@tests/shared/sql-command.js' + +describe('Test live', function () { + let servers: PeerTubeServer[] = [] + let commands: LiveCommand[] + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(2) + + // Get the access tokens + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + await servers[0].config.updateCustomSubConfig({ + newConfig: { + live: { + enabled: true, + allowReplay: true, + latencySetting: { + enabled: true + }, + transcoding: { + enabled: false + } + } + } + }) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + + commands = servers.map(s => s.live) + }) + + describe('Live creation, update and delete', function () { + let liveVideoUUID: string + + it('Should create a live with the appropriate parameters', async function () { + this.timeout(20000) + + const attributes: LiveVideoCreate = { + category: 1, + licence: 2, + language: 'fr', + description: 'super live description', + support: 'support field', + channelId: servers[0].store.channel.id, + nsfw: false, + waitTranscoding: false, + name: 'my super live', + tags: [ 'tag1', 'tag2' ], + commentsEnabled: false, + downloadEnabled: false, + saveReplay: true, + replaySettings: { privacy: VideoPrivacy.PUBLIC }, + latencyMode: LiveVideoLatencyMode.SMALL_LATENCY, + privacy: VideoPrivacy.PUBLIC, + previewfile: 'video_short1-preview.webm.jpg', + thumbnailfile: 'video_short1.webm.jpg' + } + + const live = await commands[0].create({ fields: attributes }) + liveVideoUUID = live.uuid + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: liveVideoUUID }) + + expect(video.category.id).to.equal(1) + expect(video.licence.id).to.equal(2) + expect(video.language.id).to.equal('fr') + expect(video.description).to.equal('super live description') + expect(video.support).to.equal('support field') + + expect(video.channel.name).to.equal(servers[0].store.channel.name) + expect(video.channel.host).to.equal(servers[0].store.channel.host) + + expect(video.isLive).to.be.true + + expect(video.nsfw).to.be.false + expect(video.waitTranscoding).to.be.false + expect(video.name).to.equal('my super live') + expect(video.tags).to.deep.equal([ 'tag1', 'tag2' ]) + expect(video.commentsEnabled).to.be.false + expect(video.downloadEnabled).to.be.false + expect(video.privacy.id).to.equal(VideoPrivacy.PUBLIC) + + await testImageGeneratedByFFmpeg(server.url, 'video_short1-preview.webm', video.previewPath) + await testImageGeneratedByFFmpeg(server.url, 'video_short1.webm', video.thumbnailPath) + + const live = await server.live.get({ videoId: liveVideoUUID }) + + if (server.url === servers[0].url) { + expect(live.rtmpUrl).to.equal('rtmp://' + server.hostname + ':' + servers[0].rtmpPort + '/live') + expect(live.streamKey).to.not.be.empty + + expect(live.replaySettings).to.exist + expect(live.replaySettings.privacy).to.equal(VideoPrivacy.PUBLIC) + } else { + expect(live.rtmpUrl).to.not.exist + expect(live.streamKey).to.not.exist + } + + expect(live.saveReplay).to.be.true + expect(live.latencyMode).to.equal(LiveVideoLatencyMode.SMALL_LATENCY) + } + }) + + it('Should have a default preview and thumbnail', async function () { + this.timeout(20000) + + const attributes: LiveVideoCreate = { + name: 'default live thumbnail', + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.UNLISTED, + nsfw: true + } + + const live = await commands[0].create({ fields: attributes }) + const videoId = live.uuid + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: videoId }) + expect(video.privacy.id).to.equal(VideoPrivacy.UNLISTED) + expect(video.nsfw).to.be.true + + await makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) + await makeGetRequest({ url: server.url, path: video.previewPath, expectedStatus: HttpStatusCode.OK_200 }) + } + }) + + it('Should not have the live listed since nobody streams into', async function () { + for (const server of servers) { + const { total, data } = await server.videos.list() + + expect(total).to.equal(0) + expect(data).to.have.lengthOf(0) + } + }) + + it('Should not be able to update a live of another server', async function () { + await commands[1].update({ videoId: liveVideoUUID, fields: { saveReplay: false }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should update the live', async function () { + await commands[0].update({ videoId: liveVideoUUID, fields: { saveReplay: false, latencyMode: LiveVideoLatencyMode.DEFAULT } }) + await waitJobs(servers) + }) + + it('Have the live updated', async function () { + for (const server of servers) { + const live = await server.live.get({ videoId: liveVideoUUID }) + + if (server.url === servers[0].url) { + expect(live.rtmpUrl).to.equal('rtmp://' + server.hostname + ':' + servers[0].rtmpPort + '/live') + expect(live.streamKey).to.not.be.empty + } else { + expect(live.rtmpUrl).to.not.exist + expect(live.streamKey).to.not.exist + } + + expect(live.saveReplay).to.be.false + expect(live.replaySettings).to.not.exist + expect(live.latencyMode).to.equal(LiveVideoLatencyMode.DEFAULT) + } + }) + + it('Delete the live', async function () { + await servers[0].videos.remove({ id: liveVideoUUID }) + await waitJobs(servers) + }) + + it('Should have the live deleted', async function () { + for (const server of servers) { + await server.videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + await server.live.get({ videoId: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + } + }) + }) + + describe('Live filters', function () { + let ffmpegCommand: any + let liveVideoId: string + let vodVideoId: string + + before(async function () { + this.timeout(240000) + + vodVideoId = (await servers[0].videos.quickUpload({ name: 'vod video' })).uuid + + const liveOptions = { name: 'live', privacy: VideoPrivacy.PUBLIC, channelId: servers[0].store.channel.id } + const live = await commands[0].create({ fields: liveOptions }) + liveVideoId = live.uuid + + ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoId }) + await waitUntilLivePublishedOnAllServers(servers, liveVideoId) + await waitJobs(servers) + }) + + it('Should only display lives', async function () { + const { data, total } = await servers[0].videos.list({ isLive: true }) + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + expect(data[0].name).to.equal('live') + }) + + it('Should not display lives', async function () { + const { data, total } = await servers[0].videos.list({ isLive: false }) + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + expect(data[0].name).to.equal('vod video') + }) + + it('Should display my lives', async function () { + this.timeout(60000) + + await stopFfmpeg(ffmpegCommand) + await waitJobs(servers) + + const { data } = await servers[0].videos.listMyVideos({ isLive: true }) + + const result = data.every(v => v.isLive) + expect(result).to.be.true + }) + + it('Should not display my lives', async function () { + const { data } = await servers[0].videos.listMyVideos({ isLive: false }) + + const result = data.every(v => !v.isLive) + expect(result).to.be.true + }) + + after(async function () { + await servers[0].videos.remove({ id: vodVideoId }) + await servers[0].videos.remove({ id: liveVideoId }) + }) + }) + + describe('Stream checks', function () { + let liveVideo: LiveVideo & VideoDetails + let rtmpUrl: string + + before(function () { + rtmpUrl = 'rtmp://' + servers[0].hostname + ':' + servers[0].rtmpPort + '' + }) + + async function createLiveWrapper () { + const liveAttributes = { + name: 'user live', + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC, + saveReplay: false + } + + const { uuid } = await commands[0].create({ fields: liveAttributes }) + + const live = await commands[0].get({ videoId: uuid }) + const video = await servers[0].videos.get({ id: uuid }) + + return Object.assign(video, live) + } + + it('Should not allow a stream without the appropriate path', async function () { + this.timeout(60000) + + liveVideo = await createLiveWrapper() + + const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/bad-live', streamKey: liveVideo.streamKey }) + await testFfmpegStreamError(command, true) + }) + + it('Should not allow a stream without the appropriate stream key', async function () { + this.timeout(60000) + + const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: 'bad-stream-key' }) + await testFfmpegStreamError(command, true) + }) + + it('Should succeed with the correct params', async function () { + this.timeout(60000) + + const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: liveVideo.streamKey }) + await testFfmpegStreamError(command, false) + }) + + it('Should list this live now someone stream into it', async function () { + for (const server of servers) { + const { total, data } = await server.videos.list() + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + + const video = data[0] + expect(video.name).to.equal('user live') + expect(video.isLive).to.be.true + } + }) + + it('Should not allow a stream on a live that was blacklisted', async function () { + this.timeout(60000) + + liveVideo = await createLiveWrapper() + + await servers[0].blacklist.add({ videoId: liveVideo.uuid }) + + const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: liveVideo.streamKey }) + await testFfmpegStreamError(command, true) + }) + + it('Should not allow a stream on a live that was deleted', async function () { + this.timeout(60000) + + liveVideo = await createLiveWrapper() + + await servers[0].videos.remove({ id: liveVideo.uuid }) + + const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: liveVideo.streamKey }) + await testFfmpegStreamError(command, true) + }) + }) + + describe('Live transcoding', function () { + let liveVideoId: string + let sqlCommandServer1: SQLCommand + + async function createLiveWrapper (saveReplay: boolean) { + const liveAttributes = { + name: 'live video', + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC, + saveReplay, + replaySettings: saveReplay + ? { privacy: VideoPrivacy.PUBLIC } + : undefined + } + + const { uuid } = await commands[0].create({ fields: liveAttributes }) + return uuid + } + + function updateConf (resolutions: number[]) { + return servers[0].config.updateCustomSubConfig({ + newConfig: { + live: { + enabled: true, + allowReplay: true, + maxDuration: -1, + transcoding: { + enabled: true, + resolutions: { + '144p': resolutions.includes(144), + '240p': resolutions.includes(240), + '360p': resolutions.includes(360), + '480p': resolutions.includes(480), + '720p': resolutions.includes(720), + '1080p': resolutions.includes(1080), + '2160p': resolutions.includes(2160) + } + } + } + } + }) + } + + before(async function () { + await updateConf([]) + + sqlCommandServer1 = new SQLCommand(servers[0]) + }) + + it('Should enable transcoding without additional resolutions', async function () { + this.timeout(120000) + + liveVideoId = await createLiveWrapper(false) + + const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId }) + await waitUntilLivePublishedOnAllServers(servers, liveVideoId) + await waitJobs(servers) + + await testLiveVideoResolutions({ + originServer: servers[0], + sqlCommand: sqlCommandServer1, + servers, + liveVideoId, + resolutions: [ 720 ], + transcoded: true + }) + + await stopFfmpeg(ffmpegCommand) + }) + + it('Should transcode audio only RTMP stream', async function () { + this.timeout(120000) + + liveVideoId = await createLiveWrapper(false) + + const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_short_no_audio.mp4' }) + await waitUntilLivePublishedOnAllServers(servers, liveVideoId) + await waitJobs(servers) + + await stopFfmpeg(ffmpegCommand) + }) + + it('Should enable transcoding with some resolutions', async function () { + this.timeout(240000) + + const resolutions = [ 240, 480 ] + await updateConf(resolutions) + liveVideoId = await createLiveWrapper(false) + + const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId }) + await waitUntilLivePublishedOnAllServers(servers, liveVideoId) + await waitJobs(servers) + + await testLiveVideoResolutions({ + originServer: servers[0], + sqlCommand: sqlCommandServer1, + servers, + liveVideoId, + resolutions: resolutions.concat([ 720 ]), + transcoded: true + }) + + await stopFfmpeg(ffmpegCommand) + }) + + it('Should correctly set the appropriate bitrate depending on the input', async function () { + this.timeout(120000) + + liveVideoId = await createLiveWrapper(false) + + const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ + videoId: liveVideoId, + fixtureName: 'video_short.mp4', + copyCodecs: true + }) + await waitUntilLivePublishedOnAllServers(servers, liveVideoId) + await waitJobs(servers) + + const video = await servers[0].videos.get({ id: liveVideoId }) + + const masterPlaylist = video.streamingPlaylists[0].playlistUrl + const probe = await ffprobePromise(masterPlaylist) + + const bitrates = probe.streams.map(s => parseInt(s.tags.variant_bitrate)) + for (const bitrate of bitrates) { + expect(bitrate).to.exist + expect(isNaN(bitrate)).to.be.false + expect(bitrate).to.be.below(61_000_000) // video_short.mp4 bitrate + } + + await stopFfmpeg(ffmpegCommand) + }) + + it('Should enable transcoding with some resolutions and correctly save them', async function () { + this.timeout(500_000) + + const resolutions = [ 240, 360, 720 ] + + await updateConf(resolutions) + liveVideoId = await createLiveWrapper(true) + + const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_short2.webm' }) + await waitUntilLivePublishedOnAllServers(servers, liveVideoId) + await waitJobs(servers) + + await testLiveVideoResolutions({ + originServer: servers[0], + sqlCommand: sqlCommandServer1, + servers, + liveVideoId, + resolutions, + transcoded: true + }) + + await stopFfmpeg(ffmpegCommand) + await commands[0].waitUntilEnded({ videoId: liveVideoId }) + + await waitJobs(servers) + + await waitUntilLivePublishedOnAllServers(servers, liveVideoId) + + const maxBitrateLimits = { + 720: 6500 * 1000, // 60FPS + 360: 1250 * 1000, + 240: 700 * 1000 + } + + const minBitrateLimits = { + 720: 4800 * 1000, + 360: 1000 * 1000, + 240: 550 * 1000 + } + + for (const server of servers) { + const video = await server.videos.get({ id: liveVideoId }) + + expect(video.state.id).to.equal(VideoState.PUBLISHED) + expect(video.duration).to.be.greaterThan(1) + expect(video.files).to.have.lengthOf(0) + + const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS) + await makeRawRequest({ url: hlsPlaylist.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url: hlsPlaylist.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) + + // We should have generated random filenames + expect(basename(hlsPlaylist.playlistUrl)).to.not.equal('master.m3u8') + expect(basename(hlsPlaylist.segmentsSha256Url)).to.not.equal('segments-sha256.json') + + expect(hlsPlaylist.files).to.have.lengthOf(resolutions.length) + + for (const resolution of resolutions) { + const file = hlsPlaylist.files.find(f => f.resolution.id === resolution) + + expect(file).to.exist + expect(file.size).to.be.greaterThan(1) + + if (resolution >= 720) { + expect(file.fps).to.be.approximately(60, 10) + } else { + expect(file.fps).to.be.approximately(30, 3) + } + + const filename = basename(file.fileUrl) + expect(filename).to.not.contain(video.uuid) + + const segmentPath = servers[0].servers.buildDirectory(join('streaming-playlists', 'hls', video.uuid, filename)) + + const probe = await ffprobePromise(segmentPath) + const videoStream = await getVideoStream(segmentPath, probe) + + expect(probe.format.bit_rate).to.be.below(maxBitrateLimits[videoStream.height]) + expect(probe.format.bit_rate).to.be.at.least(minBitrateLimits[videoStream.height]) + + await makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + } + } + }) + + it('Should not generate an upper resolution than original file', async function () { + this.timeout(500_000) + + const resolutions = [ 240, 480 ] + await updateConf(resolutions) + + await servers[0].config.updateExistingSubConfig({ + newConfig: { + live: { + transcoding: { + alwaysTranscodeOriginalResolution: false + } + } + } + }) + + liveVideoId = await createLiveWrapper(true) + + const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_short2.webm' }) + await waitUntilLivePublishedOnAllServers(servers, liveVideoId) + await waitJobs(servers) + + await testLiveVideoResolutions({ + originServer: servers[0], + sqlCommand: sqlCommandServer1, + servers, + liveVideoId, + resolutions, + transcoded: true + }) + + await stopFfmpeg(ffmpegCommand) + await commands[0].waitUntilEnded({ videoId: liveVideoId }) + + await waitJobs(servers) + + await waitUntilLivePublishedOnAllServers(servers, liveVideoId) + + const video = await servers[0].videos.get({ id: liveVideoId }) + const hlsFiles = video.streamingPlaylists[0].files + + expect(video.files).to.have.lengthOf(0) + expect(hlsFiles).to.have.lengthOf(resolutions.length) + + // eslint-disable-next-line @typescript-eslint/require-array-sort-compare + expect(getAllFiles(video).map(f => f.resolution.id).sort()).to.deep.equal(resolutions) + }) + + it('Should only keep the original resolution if all resolutions are disabled', async function () { + this.timeout(600_000) + + await updateConf([]) + liveVideoId = await createLiveWrapper(true) + + const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_short2.webm' }) + await waitUntilLivePublishedOnAllServers(servers, liveVideoId) + await waitJobs(servers) + + await testLiveVideoResolutions({ + originServer: servers[0], + sqlCommand: sqlCommandServer1, + servers, + liveVideoId, + resolutions: [ 720 ], + transcoded: true + }) + + await stopFfmpeg(ffmpegCommand) + await commands[0].waitUntilEnded({ videoId: liveVideoId }) + + await waitJobs(servers) + + await waitUntilLivePublishedOnAllServers(servers, liveVideoId) + + const video = await servers[0].videos.get({ id: liveVideoId }) + const hlsFiles = video.streamingPlaylists[0].files + + expect(video.files).to.have.lengthOf(0) + expect(hlsFiles).to.have.lengthOf(1) + + expect(hlsFiles[0].resolution.id).to.equal(720) + }) + + after(async function () { + await sqlCommandServer1.cleanup() + }) + }) + + describe('After a server restart', function () { + let liveVideoId: string + let liveVideoReplayId: string + let permanentLiveVideoReplayId: string + + let permanentLiveReplayName: string + + let beforeServerRestart: Date + + async function createLiveWrapper (options: { saveReplay: boolean, permanent: boolean }) { + const liveAttributes: LiveVideoCreate = { + name: 'live video', + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC, + saveReplay: options.saveReplay, + replaySettings: options.saveReplay + ? { privacy: VideoPrivacy.PUBLIC } + : undefined, + permanentLive: options.permanent + } + + const { uuid } = await commands[0].create({ fields: liveAttributes }) + return uuid + } + + before(async function () { + this.timeout(600_000) + + liveVideoId = await createLiveWrapper({ saveReplay: false, permanent: false }) + liveVideoReplayId = await createLiveWrapper({ saveReplay: true, permanent: false }) + permanentLiveVideoReplayId = await createLiveWrapper({ saveReplay: true, permanent: true }) + + await Promise.all([ + commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId }), + commands[0].sendRTMPStreamInVideo({ videoId: permanentLiveVideoReplayId }), + commands[0].sendRTMPStreamInVideo({ videoId: liveVideoReplayId }) + ]) + + await Promise.all([ + commands[0].waitUntilPublished({ videoId: liveVideoId }), + commands[0].waitUntilPublished({ videoId: permanentLiveVideoReplayId }), + commands[0].waitUntilPublished({ videoId: liveVideoReplayId }) + ]) + + for (const videoUUID of [ liveVideoId, liveVideoReplayId, permanentLiveVideoReplayId ]) { + await commands[0].waitUntilSegmentGeneration({ + server: servers[0], + videoUUID, + playlistNumber: 0, + segment: 2 + }) + } + + { + const video = await servers[0].videos.get({ id: permanentLiveVideoReplayId }) + permanentLiveReplayName = video.name + ' - ' + new Date(video.publishedAt).toLocaleString() + } + + await killallServers([ servers[0] ]) + + beforeServerRestart = new Date() + await servers[0].run() + + await wait(5000) + await waitJobs(servers) + }) + + it('Should cleanup lives', async function () { + this.timeout(60000) + + await commands[0].waitUntilEnded({ videoId: liveVideoId }) + await commands[0].waitUntilWaiting({ videoId: permanentLiveVideoReplayId }) + }) + + it('Should save a non permanent live replay', async function () { + this.timeout(240000) + + await commands[0].waitUntilPublished({ videoId: liveVideoReplayId }) + + const session = await commands[0].getReplaySession({ videoId: liveVideoReplayId }) + expect(session.endDate).to.exist + expect(new Date(session.endDate)).to.be.above(beforeServerRestart) + }) + + it('Should have saved a permanent live replay', async function () { + this.timeout(120000) + + const { data } = await servers[0].videos.listMyVideos({ sort: '-publishedAt' }) + expect(data.find(v => v.name === permanentLiveReplayName)).to.exist + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/moderation/abuses.ts b/packages/tests/src/api/moderation/abuses.ts new file mode 100644 index 000000000..649de224e --- /dev/null +++ b/packages/tests/src/api/moderation/abuses.ts @@ -0,0 +1,887 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { AbuseMessage, AbusePredefinedReasonsString, AbuseState, AdminAbuse, UserAbuse } from '@peertube/peertube-models' +import { + AbusesCommand, + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultChannelAvatar, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test abuses', function () { + let servers: PeerTubeServer[] = [] + let abuseServer1: AdminAbuse + let abuseServer2: AdminAbuse + let commands: AbusesCommand[] + + before(async function () { + this.timeout(50000) + + // Run servers + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + await setDefaultChannelAvatar(servers) + await setDefaultAccountAvatar(servers) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + + commands = servers.map(s => s.abuses) + }) + + describe('Video abuses', function () { + + before(async function () { + this.timeout(50000) + + // Upload some videos on each servers + { + const attributes = { + name: 'my super name for server 1', + description: 'my super description for server 1' + } + await servers[0].videos.upload({ attributes }) + } + + { + const attributes = { + name: 'my super name for server 2', + description: 'my super description for server 2' + } + await servers[1].videos.upload({ attributes }) + } + + // Wait videos propagation, server 2 has transcoding enabled + await waitJobs(servers) + + const { data } = await servers[0].videos.list() + expect(data.length).to.equal(2) + + servers[0].store.videoCreated = data.find(video => video.name === 'my super name for server 1') + servers[1].store.videoCreated = data.find(video => video.name === 'my super name for server 2') + }) + + it('Should not have abuses', async function () { + const body = await commands[0].getAdminList() + + expect(body.total).to.equal(0) + expect(body.data).to.be.an('array') + expect(body.data.length).to.equal(0) + }) + + it('Should report abuse on a local video', async function () { + this.timeout(15000) + + const reason = 'my super bad reason' + await commands[0].report({ videoId: servers[0].store.videoCreated.id, reason }) + + // We wait requests propagation, even if the server 1 is not supposed to make a request to server 2 + await waitJobs(servers) + }) + + it('Should have 1 video abuses on server 1 and 0 on server 2', async function () { + { + const body = await commands[0].getAdminList() + + expect(body.total).to.equal(1) + expect(body.data).to.be.an('array') + expect(body.data.length).to.equal(1) + + const abuse = body.data[0] + expect(abuse.reason).to.equal('my super bad reason') + + expect(abuse.reporterAccount.name).to.equal('root') + expect(abuse.reporterAccount.host).to.equal(servers[0].host) + + expect(abuse.video.id).to.equal(servers[0].store.videoCreated.id) + expect(abuse.video.channel).to.exist + + expect(abuse.comment).to.be.null + + expect(abuse.flaggedAccount.name).to.equal('root') + expect(abuse.flaggedAccount.host).to.equal(servers[0].host) + + expect(abuse.video.countReports).to.equal(1) + expect(abuse.video.nthReport).to.equal(1) + + expect(abuse.countReportsForReporter).to.equal(1) + expect(abuse.countReportsForReportee).to.equal(1) + } + + { + const body = await commands[1].getAdminList() + expect(body.total).to.equal(0) + expect(body.data).to.be.an('array') + expect(body.data.length).to.equal(0) + } + }) + + it('Should report abuse on a remote video', async function () { + const reason = 'my super bad reason 2' + const videoId = await servers[0].videos.getId({ uuid: servers[1].store.videoCreated.uuid }) + await commands[0].report({ videoId, reason }) + + // We wait requests propagation + await waitJobs(servers) + }) + + it('Should have 2 video abuses on server 1 and 1 on server 2', async function () { + { + const body = await commands[0].getAdminList() + + expect(body.total).to.equal(2) + expect(body.data.length).to.equal(2) + + const abuse1 = body.data[0] + expect(abuse1.reason).to.equal('my super bad reason') + expect(abuse1.reporterAccount.name).to.equal('root') + expect(abuse1.reporterAccount.host).to.equal(servers[0].host) + + expect(abuse1.video.id).to.equal(servers[0].store.videoCreated.id) + expect(abuse1.video.countReports).to.equal(1) + expect(abuse1.video.nthReport).to.equal(1) + + expect(abuse1.comment).to.be.null + + expect(abuse1.flaggedAccount.name).to.equal('root') + expect(abuse1.flaggedAccount.host).to.equal(servers[0].host) + + expect(abuse1.state.id).to.equal(AbuseState.PENDING) + expect(abuse1.state.label).to.equal('Pending') + expect(abuse1.moderationComment).to.be.null + + const abuse2 = body.data[1] + expect(abuse2.reason).to.equal('my super bad reason 2') + + expect(abuse2.reporterAccount.name).to.equal('root') + expect(abuse2.reporterAccount.host).to.equal(servers[0].host) + + expect(abuse2.video.uuid).to.equal(servers[1].store.videoCreated.uuid) + + expect(abuse2.comment).to.be.null + + expect(abuse2.flaggedAccount.name).to.equal('root') + expect(abuse2.flaggedAccount.host).to.equal(servers[1].host) + + expect(abuse2.state.id).to.equal(AbuseState.PENDING) + expect(abuse2.state.label).to.equal('Pending') + expect(abuse2.moderationComment).to.be.null + } + + { + const body = await commands[1].getAdminList() + expect(body.total).to.equal(1) + expect(body.data.length).to.equal(1) + + abuseServer2 = body.data[0] + expect(abuseServer2.reason).to.equal('my super bad reason 2') + expect(abuseServer2.reporterAccount.name).to.equal('root') + expect(abuseServer2.reporterAccount.host).to.equal(servers[0].host) + + expect(abuseServer2.flaggedAccount.name).to.equal('root') + expect(abuseServer2.flaggedAccount.host).to.equal(servers[1].host) + + expect(abuseServer2.state.id).to.equal(AbuseState.PENDING) + expect(abuseServer2.state.label).to.equal('Pending') + expect(abuseServer2.moderationComment).to.be.null + } + }) + + it('Should hide video abuses from blocked accounts', async function () { + { + const videoId = await servers[1].videos.getId({ uuid: servers[0].store.videoCreated.uuid }) + await commands[1].report({ videoId, reason: 'will mute this' }) + await waitJobs(servers) + + const body = await commands[0].getAdminList() + expect(body.total).to.equal(3) + } + + const accountToBlock = 'root@' + servers[1].host + + { + await servers[0].blocklist.addToServerBlocklist({ account: accountToBlock }) + + const body = await commands[0].getAdminList() + expect(body.total).to.equal(2) + + const abuse = body.data.find(a => a.reason === 'will mute this') + expect(abuse).to.be.undefined + } + + { + await servers[0].blocklist.removeFromServerBlocklist({ account: accountToBlock }) + + const body = await commands[0].getAdminList() + expect(body.total).to.equal(3) + } + }) + + it('Should hide video abuses from blocked servers', async function () { + const serverToBlock = servers[1].host + + { + await servers[0].blocklist.addToServerBlocklist({ server: serverToBlock }) + + const body = await commands[0].getAdminList() + expect(body.total).to.equal(2) + + const abuse = body.data.find(a => a.reason === 'will mute this') + expect(abuse).to.be.undefined + } + + { + await servers[0].blocklist.removeFromServerBlocklist({ server: serverToBlock }) + + const body = await commands[0].getAdminList() + expect(body.total).to.equal(3) + } + }) + + it('Should keep the video abuse when deleting the video', async function () { + await servers[1].videos.remove({ id: abuseServer2.video.uuid }) + + await waitJobs(servers) + + const body = await commands[1].getAdminList() + expect(body.total).to.equal(2, 'wrong number of videos returned') + expect(body.data).to.have.lengthOf(2, 'wrong number of videos returned') + + const abuse = body.data[0] + expect(abuse.id).to.equal(abuseServer2.id, 'wrong origin server id for first video') + expect(abuse.video.id).to.equal(abuseServer2.video.id, 'wrong video id') + expect(abuse.video.channel).to.exist + expect(abuse.video.deleted).to.be.true + }) + + it('Should include counts of reports from reporter and reportee', async function () { + // register a second user to have two reporters/reportees + const user = { username: 'user2', password: 'password' } + await servers[0].users.create({ ...user }) + const userAccessToken = await servers[0].login.getAccessToken(user) + + // upload a third video via this user + const attributes = { + name: 'my second super name for server 1', + description: 'my second super description for server 1' + } + const { id } = await servers[0].videos.upload({ token: userAccessToken, attributes }) + const video3Id = id + + // resume with the test + const reason3 = 'my super bad reason 3' + await commands[0].report({ videoId: video3Id, reason: reason3 }) + + const reason4 = 'my super bad reason 4' + await commands[0].report({ token: userAccessToken, videoId: servers[0].store.videoCreated.id, reason: reason4 }) + + { + const body = await commands[0].getAdminList() + const abuses = body.data + + const abuseVideo3 = body.data.find(a => a.video.id === video3Id) + expect(abuseVideo3).to.not.be.undefined + expect(abuseVideo3.video.countReports).to.equal(1, 'wrong reports count for video 3') + expect(abuseVideo3.video.nthReport).to.equal(1, 'wrong report position in report list for video 3') + expect(abuseVideo3.countReportsForReportee).to.equal(1, 'wrong reports count for reporter on video 3 abuse') + expect(abuseVideo3.countReportsForReporter).to.equal(3, 'wrong reports count for reportee on video 3 abuse') + + const abuseServer1 = abuses.find(a => a.video.id === servers[0].store.videoCreated.id) + expect(abuseServer1.countReportsForReportee).to.equal(3, 'wrong reports count for reporter on video 1 abuse') + } + }) + + it('Should list predefined reasons as well as timestamps for the reported video', async function () { + const reason5 = 'my super bad reason 5' + const predefinedReasons5: AbusePredefinedReasonsString[] = [ 'violentOrRepulsive', 'captions' ] + const createRes = await commands[0].report({ + videoId: servers[0].store.videoCreated.id, + reason: reason5, + predefinedReasons: predefinedReasons5, + startAt: 1, + endAt: 5 + }) + + const body = await commands[0].getAdminList() + + { + const abuse = body.data.find(a => a.id === createRes.abuse.id) + expect(abuse.reason).to.equals(reason5) + expect(abuse.predefinedReasons).to.deep.equals(predefinedReasons5, 'predefined reasons do not match the one reported') + expect(abuse.video.startAt).to.equal(1, "starting timestamp doesn't match the one reported") + expect(abuse.video.endAt).to.equal(5, "ending timestamp doesn't match the one reported") + } + }) + + it('Should delete the video abuse', async function () { + await commands[1].delete({ abuseId: abuseServer2.id }) + + await waitJobs(servers) + + { + const body = await commands[1].getAdminList() + expect(body.total).to.equal(1) + expect(body.data.length).to.equal(1) + expect(body.data[0].id).to.not.equal(abuseServer2.id) + } + + { + const body = await commands[0].getAdminList() + expect(body.total).to.equal(6) + } + }) + + it('Should list and filter video abuses', async function () { + async function list (query: Parameters[0]) { + const body = await commands[0].getAdminList(query) + + return body.data + } + + expect(await list({ id: 56 })).to.have.lengthOf(0) + expect(await list({ id: 1 })).to.have.lengthOf(1) + + expect(await list({ search: 'my super name for server 1' })).to.have.lengthOf(4) + expect(await list({ search: 'aaaaaaaaaaaaaaaaaaaaaaaaaa' })).to.have.lengthOf(0) + + expect(await list({ searchVideo: 'my second super name for server 1' })).to.have.lengthOf(1) + + expect(await list({ searchVideoChannel: 'root' })).to.have.lengthOf(4) + expect(await list({ searchVideoChannel: 'aaaa' })).to.have.lengthOf(0) + + expect(await list({ searchReporter: 'user2' })).to.have.lengthOf(1) + expect(await list({ searchReporter: 'root' })).to.have.lengthOf(5) + + expect(await list({ searchReportee: 'root' })).to.have.lengthOf(5) + expect(await list({ searchReportee: 'aaaa' })).to.have.lengthOf(0) + + expect(await list({ videoIs: 'deleted' })).to.have.lengthOf(1) + expect(await list({ videoIs: 'blacklisted' })).to.have.lengthOf(0) + + expect(await list({ state: AbuseState.ACCEPTED })).to.have.lengthOf(0) + expect(await list({ state: AbuseState.PENDING })).to.have.lengthOf(6) + + expect(await list({ predefinedReason: 'violentOrRepulsive' })).to.have.lengthOf(1) + expect(await list({ predefinedReason: 'serverRules' })).to.have.lengthOf(0) + }) + }) + + describe('Comment abuses', function () { + + async function getComment (server: PeerTubeServer, videoIdArg: number | string) { + const videoId = typeof videoIdArg === 'string' + ? await server.videos.getId({ uuid: videoIdArg }) + : videoIdArg + + const { data } = await server.comments.listThreads({ videoId }) + + return data[0] + } + + before(async function () { + this.timeout(50000) + + servers[0].store.videoCreated = await servers[0].videos.quickUpload({ name: 'server 1' }) + servers[1].store.videoCreated = await servers[1].videos.quickUpload({ name: 'server 2' }) + + await servers[0].comments.createThread({ videoId: servers[0].store.videoCreated.id, text: 'comment server 1' }) + await servers[1].comments.createThread({ videoId: servers[1].store.videoCreated.id, text: 'comment server 2' }) + + await waitJobs(servers) + }) + + it('Should report abuse on a comment', async function () { + this.timeout(15000) + + const comment = await getComment(servers[0], servers[0].store.videoCreated.id) + + const reason = 'it is a bad comment' + await commands[0].report({ commentId: comment.id, reason }) + + await waitJobs(servers) + }) + + it('Should have 1 comment abuse on server 1 and 0 on server 2', async function () { + { + const comment = await getComment(servers[0], servers[0].store.videoCreated.id) + const body = await commands[0].getAdminList({ filter: 'comment' }) + + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + + const abuse = body.data[0] + expect(abuse.reason).to.equal('it is a bad comment') + + expect(abuse.reporterAccount.name).to.equal('root') + expect(abuse.reporterAccount.host).to.equal(servers[0].host) + + expect(abuse.video).to.be.null + + expect(abuse.comment.deleted).to.be.false + expect(abuse.comment.id).to.equal(comment.id) + expect(abuse.comment.text).to.equal(comment.text) + expect(abuse.comment.video.name).to.equal('server 1') + expect(abuse.comment.video.id).to.equal(servers[0].store.videoCreated.id) + expect(abuse.comment.video.uuid).to.equal(servers[0].store.videoCreated.uuid) + + expect(abuse.countReportsForReporter).to.equal(5) + expect(abuse.countReportsForReportee).to.equal(5) + } + + { + const body = await commands[1].getAdminList({ filter: 'comment' }) + expect(body.total).to.equal(0) + expect(body.data.length).to.equal(0) + } + }) + + it('Should report abuse on a remote comment', async function () { + const comment = await getComment(servers[0], servers[1].store.videoCreated.uuid) + + const reason = 'it is a really bad comment' + await commands[0].report({ commentId: comment.id, reason }) + + await waitJobs(servers) + }) + + it('Should have 2 comment abuses on server 1 and 1 on server 2', async function () { + const commentServer2 = await getComment(servers[0], servers[1].store.videoCreated.shortUUID) + + { + const body = await commands[0].getAdminList({ filter: 'comment' }) + expect(body.total).to.equal(2) + expect(body.data.length).to.equal(2) + + const abuse = body.data[0] + expect(abuse.reason).to.equal('it is a bad comment') + expect(abuse.countReportsForReporter).to.equal(6) + expect(abuse.countReportsForReportee).to.equal(5) + + const abuse2 = body.data[1] + + expect(abuse2.reason).to.equal('it is a really bad comment') + + expect(abuse2.reporterAccount.name).to.equal('root') + expect(abuse2.reporterAccount.host).to.equal(servers[0].host) + + expect(abuse2.video).to.be.null + + expect(abuse2.comment.deleted).to.be.false + expect(abuse2.comment.id).to.equal(commentServer2.id) + expect(abuse2.comment.text).to.equal(commentServer2.text) + expect(abuse2.comment.video.name).to.equal('server 2') + expect(abuse2.comment.video.uuid).to.equal(servers[1].store.videoCreated.uuid) + + expect(abuse2.state.id).to.equal(AbuseState.PENDING) + expect(abuse2.state.label).to.equal('Pending') + + expect(abuse2.moderationComment).to.be.null + + expect(abuse2.countReportsForReporter).to.equal(6) + expect(abuse2.countReportsForReportee).to.equal(2) + } + + { + const body = await commands[1].getAdminList({ filter: 'comment' }) + expect(body.total).to.equal(1) + expect(body.data.length).to.equal(1) + + abuseServer2 = body.data[0] + expect(abuseServer2.reason).to.equal('it is a really bad comment') + expect(abuseServer2.reporterAccount.name).to.equal('root') + expect(abuseServer2.reporterAccount.host).to.equal(servers[0].host) + + expect(abuseServer2.state.id).to.equal(AbuseState.PENDING) + expect(abuseServer2.state.label).to.equal('Pending') + + expect(abuseServer2.moderationComment).to.be.null + + expect(abuseServer2.countReportsForReporter).to.equal(1) + expect(abuseServer2.countReportsForReportee).to.equal(1) + } + }) + + it('Should keep the comment abuse when deleting the comment', async function () { + const commentServer2 = await getComment(servers[0], servers[1].store.videoCreated.uuid) + + await servers[0].comments.delete({ videoId: servers[1].store.videoCreated.uuid, commentId: commentServer2.id }) + + await waitJobs(servers) + + const body = await commands[0].getAdminList({ filter: 'comment' }) + expect(body.total).to.equal(2) + expect(body.data).to.have.lengthOf(2) + + const abuse = body.data.find(a => a.comment?.id === commentServer2.id) + expect(abuse).to.not.be.undefined + + expect(abuse.comment.text).to.be.empty + expect(abuse.comment.video.name).to.equal('server 2') + expect(abuse.comment.deleted).to.be.true + }) + + it('Should delete the comment abuse', async function () { + await commands[1].delete({ abuseId: abuseServer2.id }) + + await waitJobs(servers) + + { + const body = await commands[1].getAdminList({ filter: 'comment' }) + expect(body.total).to.equal(0) + expect(body.data.length).to.equal(0) + } + + { + const body = await commands[0].getAdminList({ filter: 'comment' }) + expect(body.total).to.equal(2) + } + }) + + it('Should list and filter video abuses', async function () { + { + const body = await commands[0].getAdminList({ filter: 'comment', searchReportee: 'foo' }) + expect(body.total).to.equal(0) + } + + { + const body = await commands[0].getAdminList({ filter: 'comment', searchReportee: 'ot' }) + expect(body.total).to.equal(2) + } + + { + const body = await commands[0].getAdminList({ filter: 'comment', start: 1, count: 1, sort: 'createdAt' }) + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].comment.text).to.be.empty + } + + { + const body = await commands[0].getAdminList({ filter: 'comment', start: 1, count: 1, sort: '-createdAt' }) + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].comment.text).to.equal('comment server 1') + } + }) + }) + + describe('Account abuses', function () { + + function getAccountFromServer (server: PeerTubeServer, targetName: string, targetServer: PeerTubeServer) { + return server.accounts.get({ accountName: targetName + '@' + targetServer.host }) + } + + before(async function () { + this.timeout(50000) + + await servers[0].users.create({ username: 'user_1', password: 'donald' }) + + const token = await servers[1].users.generateUserAndToken('user_2') + await servers[1].videos.upload({ token, attributes: { name: 'super video' } }) + + await waitJobs(servers) + }) + + it('Should report abuse on an account', async function () { + this.timeout(15000) + + const account = await getAccountFromServer(servers[0], 'user_1', servers[0]) + + const reason = 'it is a bad account' + await commands[0].report({ accountId: account.id, reason }) + + await waitJobs(servers) + }) + + it('Should have 1 account abuse on server 1 and 0 on server 2', async function () { + { + const body = await commands[0].getAdminList({ filter: 'account' }) + + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + + const abuse = body.data[0] + expect(abuse.reason).to.equal('it is a bad account') + + expect(abuse.reporterAccount.name).to.equal('root') + expect(abuse.reporterAccount.host).to.equal(servers[0].host) + + expect(abuse.video).to.be.null + expect(abuse.comment).to.be.null + + expect(abuse.flaggedAccount.name).to.equal('user_1') + expect(abuse.flaggedAccount.host).to.equal(servers[0].host) + } + + { + const body = await commands[1].getAdminList({ filter: 'comment' }) + expect(body.total).to.equal(0) + expect(body.data.length).to.equal(0) + } + }) + + it('Should report abuse on a remote account', async function () { + const account = await getAccountFromServer(servers[0], 'user_2', servers[1]) + + const reason = 'it is a really bad account' + await commands[0].report({ accountId: account.id, reason }) + + await waitJobs(servers) + }) + + it('Should have 2 comment abuses on server 1 and 1 on server 2', async function () { + { + const body = await commands[0].getAdminList({ filter: 'account' }) + expect(body.total).to.equal(2) + expect(body.data.length).to.equal(2) + + const abuse: AdminAbuse = body.data[0] + expect(abuse.reason).to.equal('it is a bad account') + + const abuse2: AdminAbuse = body.data[1] + expect(abuse2.reason).to.equal('it is a really bad account') + + expect(abuse2.reporterAccount.name).to.equal('root') + expect(abuse2.reporterAccount.host).to.equal(servers[0].host) + + expect(abuse2.video).to.be.null + expect(abuse2.comment).to.be.null + + expect(abuse2.state.id).to.equal(AbuseState.PENDING) + expect(abuse2.state.label).to.equal('Pending') + + expect(abuse2.moderationComment).to.be.null + } + + { + const body = await commands[1].getAdminList({ filter: 'account' }) + expect(body.total).to.equal(1) + expect(body.data.length).to.equal(1) + + abuseServer2 = body.data[0] + + expect(abuseServer2.reason).to.equal('it is a really bad account') + + expect(abuseServer2.reporterAccount.name).to.equal('root') + expect(abuseServer2.reporterAccount.host).to.equal(servers[0].host) + + expect(abuseServer2.state.id).to.equal(AbuseState.PENDING) + expect(abuseServer2.state.label).to.equal('Pending') + + expect(abuseServer2.moderationComment).to.be.null + } + }) + + it('Should keep the account abuse when deleting the account', async function () { + const account = await getAccountFromServer(servers[1], 'user_2', servers[1]) + await servers[1].users.remove({ userId: account.userId }) + + await waitJobs(servers) + + const body = await commands[0].getAdminList({ filter: 'account' }) + expect(body.total).to.equal(2) + expect(body.data).to.have.lengthOf(2) + + const abuse = body.data.find(a => a.reason === 'it is a really bad account') + expect(abuse).to.not.be.undefined + }) + + it('Should delete the account abuse', async function () { + await commands[1].delete({ abuseId: abuseServer2.id }) + + await waitJobs(servers) + + { + const body = await commands[1].getAdminList({ filter: 'account' }) + expect(body.total).to.equal(0) + expect(body.data.length).to.equal(0) + } + + { + const body = await commands[0].getAdminList({ filter: 'account' }) + expect(body.total).to.equal(2) + + abuseServer1 = body.data[0] + } + }) + }) + + describe('Common actions on abuses', function () { + + it('Should update the state of an abuse', async function () { + await commands[0].update({ abuseId: abuseServer1.id, body: { state: AbuseState.REJECTED } }) + + const body = await commands[0].getAdminList({ id: abuseServer1.id }) + expect(body.data[0].state.id).to.equal(AbuseState.REJECTED) + }) + + it('Should add a moderation comment', async function () { + await commands[0].update({ abuseId: abuseServer1.id, body: { state: AbuseState.ACCEPTED, moderationComment: 'Valid' } }) + + const body = await commands[0].getAdminList({ id: abuseServer1.id }) + expect(body.data[0].state.id).to.equal(AbuseState.ACCEPTED) + expect(body.data[0].moderationComment).to.equal('Valid') + }) + }) + + describe('My abuses', async function () { + let abuseId1: number + let userAccessToken: string + + before(async function () { + userAccessToken = await servers[0].users.generateUserAndToken('user_42') + + await commands[0].report({ token: userAccessToken, videoId: servers[0].store.videoCreated.id, reason: 'user reason 1' }) + + const videoId = await servers[0].videos.getId({ uuid: servers[1].store.videoCreated.uuid }) + await commands[0].report({ token: userAccessToken, videoId, reason: 'user reason 2' }) + }) + + it('Should correctly list my abuses', async function () { + { + const body = await commands[0].getUserList({ token: userAccessToken, start: 0, count: 5, sort: 'createdAt' }) + expect(body.total).to.equal(2) + + const abuses = body.data + expect(abuses[0].reason).to.equal('user reason 1') + expect(abuses[1].reason).to.equal('user reason 2') + + abuseId1 = abuses[0].id + } + + { + const body = await commands[0].getUserList({ token: userAccessToken, start: 1, count: 1, sort: 'createdAt' }) + expect(body.total).to.equal(2) + + const abuses: UserAbuse[] = body.data + expect(abuses[0].reason).to.equal('user reason 2') + } + + { + const body = await commands[0].getUserList({ token: userAccessToken, start: 1, count: 1, sort: '-createdAt' }) + expect(body.total).to.equal(2) + + const abuses: UserAbuse[] = body.data + expect(abuses[0].reason).to.equal('user reason 1') + } + }) + + it('Should correctly filter my abuses by id', async function () { + const body = await commands[0].getUserList({ token: userAccessToken, id: abuseId1 }) + expect(body.total).to.equal(1) + + const abuses: UserAbuse[] = body.data + expect(abuses[0].reason).to.equal('user reason 1') + }) + + it('Should correctly filter my abuses by search', async function () { + const body = await commands[0].getUserList({ token: userAccessToken, search: 'server 2' }) + expect(body.total).to.equal(1) + + const abuses: UserAbuse[] = body.data + expect(abuses[0].reason).to.equal('user reason 2') + }) + + it('Should correctly filter my abuses by state', async function () { + await commands[0].update({ abuseId: abuseId1, body: { state: AbuseState.REJECTED } }) + + const body = await commands[0].getUserList({ token: userAccessToken, state: AbuseState.REJECTED }) + expect(body.total).to.equal(1) + + const abuses: UserAbuse[] = body.data + expect(abuses[0].reason).to.equal('user reason 1') + }) + }) + + describe('Abuse messages', async function () { + let abuseId: number + let userToken: string + let abuseMessageUserId: number + let abuseMessageModerationId: number + + before(async function () { + userToken = await servers[0].users.generateUserAndToken('user_43') + + const body = await commands[0].report({ token: userToken, videoId: servers[0].store.videoCreated.id, reason: 'user 43 reason 1' }) + abuseId = body.abuse.id + }) + + it('Should create some messages on the abuse', async function () { + await commands[0].addMessage({ token: userToken, abuseId, message: 'message 1' }) + await commands[0].addMessage({ abuseId, message: 'message 2' }) + await commands[0].addMessage({ abuseId, message: 'message 3' }) + await commands[0].addMessage({ token: userToken, abuseId, message: 'message 4' }) + }) + + it('Should have the correct messages count when listing abuses', async function () { + const results = await Promise.all([ + commands[0].getAdminList({ start: 0, count: 50 }), + commands[0].getUserList({ token: userToken, start: 0, count: 50 }) + ]) + + for (const body of results) { + const abuses = body.data + const abuse = abuses.find(a => a.id === abuseId) + expect(abuse.countMessages).to.equal(4) + } + }) + + it('Should correctly list messages of this abuse', async function () { + const results = await Promise.all([ + commands[0].listMessages({ abuseId }), + commands[0].listMessages({ token: userToken, abuseId }) + ]) + + for (const body of results) { + expect(body.total).to.equal(4) + + const abuseMessages: AbuseMessage[] = body.data + + expect(abuseMessages[0].message).to.equal('message 1') + expect(abuseMessages[0].byModerator).to.be.false + expect(abuseMessages[0].account.name).to.equal('user_43') + + abuseMessageUserId = abuseMessages[0].id + + expect(abuseMessages[1].message).to.equal('message 2') + expect(abuseMessages[1].byModerator).to.be.true + expect(abuseMessages[1].account.name).to.equal('root') + + expect(abuseMessages[2].message).to.equal('message 3') + expect(abuseMessages[2].byModerator).to.be.true + expect(abuseMessages[2].account.name).to.equal('root') + abuseMessageModerationId = abuseMessages[2].id + + expect(abuseMessages[3].message).to.equal('message 4') + expect(abuseMessages[3].byModerator).to.be.false + expect(abuseMessages[3].account.name).to.equal('user_43') + } + }) + + it('Should delete messages', async function () { + await commands[0].deleteMessage({ abuseId, messageId: abuseMessageModerationId }) + await commands[0].deleteMessage({ token: userToken, abuseId, messageId: abuseMessageUserId }) + + const results = await Promise.all([ + commands[0].listMessages({ abuseId }), + commands[0].listMessages({ token: userToken, abuseId }) + ]) + + for (const body of results) { + expect(body.total).to.equal(2) + + const abuseMessages: AbuseMessage[] = body.data + expect(abuseMessages[0].message).to.equal('message 2') + expect(abuseMessages[1].message).to.equal('message 4') + } + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/moderation/blocklist-notification.ts b/packages/tests/src/api/moderation/blocklist-notification.ts new file mode 100644 index 000000000..abf36313b --- /dev/null +++ b/packages/tests/src/api/moderation/blocklist-notification.ts @@ -0,0 +1,231 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { UserNotificationType, UserNotificationType_Type } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +async function checkNotifications (server: PeerTubeServer, token: string, expected: UserNotificationType_Type[]) { + const { data } = await server.notifications.list({ token, start: 0, count: 10, unread: true }) + expect(data).to.have.lengthOf(expected.length) + + for (const type of expected) { + expect(data.find(n => n.type === type)).to.exist + } +} + +describe('Test blocklist notifications', function () { + let servers: PeerTubeServer[] + let videoUUID: string + + let userToken1: string + let userToken2: string + let remoteUserToken: string + + async function resetState () { + try { + await servers[1].subscriptions.remove({ token: remoteUserToken, uri: 'user1_channel@' + servers[0].host }) + await servers[1].subscriptions.remove({ token: remoteUserToken, uri: 'user2_channel@' + servers[0].host }) + } catch {} + + await waitJobs(servers) + + await servers[0].notifications.markAsReadAll({ token: userToken1 }) + await servers[0].notifications.markAsReadAll({ token: userToken2 }) + + { + const { uuid } = await servers[0].videos.upload({ token: userToken1, attributes: { name: 'video' } }) + videoUUID = uuid + + await waitJobs(servers) + } + + { + await servers[1].comments.createThread({ + token: remoteUserToken, + videoId: videoUUID, + text: '@user2@' + servers[0].host + ' hello' + }) + } + + { + + await servers[1].subscriptions.add({ token: remoteUserToken, targetUri: 'user1_channel@' + servers[0].host }) + await servers[1].subscriptions.add({ token: remoteUserToken, targetUri: 'user2_channel@' + servers[0].host }) + } + + await waitJobs(servers) + } + + before(async function () { + this.timeout(60000) + + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + + { + const user = { username: 'user1', password: 'password' } + await servers[0].users.create({ + username: user.username, + password: user.password, + videoQuota: -1, + videoQuotaDaily: -1 + }) + + userToken1 = await servers[0].login.getAccessToken(user) + await servers[0].videos.upload({ token: userToken1, attributes: { name: 'video user 1' } }) + } + + { + const user = { username: 'user2', password: 'password' } + await servers[0].users.create({ username: user.username, password: user.password }) + + userToken2 = await servers[0].login.getAccessToken(user) + } + + { + const user = { username: 'user3', password: 'password' } + await servers[1].users.create({ username: user.username, password: user.password }) + + remoteUserToken = await servers[1].login.getAccessToken(user) + } + + await doubleFollow(servers[0], servers[1]) + }) + + describe('User blocks another user', function () { + + before(async function () { + this.timeout(30000) + + await resetState() + }) + + it('Should have appropriate notifications', async function () { + const notifs = [ UserNotificationType.NEW_COMMENT_ON_MY_VIDEO, UserNotificationType.NEW_FOLLOW ] + await checkNotifications(servers[0], userToken1, notifs) + }) + + it('Should block an account', async function () { + await servers[0].blocklist.addToMyBlocklist({ token: userToken1, account: 'user3@' + servers[1].host }) + await waitJobs(servers) + }) + + it('Should not have notifications from this account', async function () { + await checkNotifications(servers[0], userToken1, []) + }) + + it('Should have notifications of this account on user 2', async function () { + const notifs = [ UserNotificationType.COMMENT_MENTION, UserNotificationType.NEW_FOLLOW ] + + await checkNotifications(servers[0], userToken2, notifs) + + await servers[0].blocklist.removeFromMyBlocklist({ token: userToken1, account: 'user3@' + servers[1].host }) + }) + }) + + describe('User blocks another server', function () { + + before(async function () { + this.timeout(30000) + + await resetState() + }) + + it('Should have appropriate notifications', async function () { + const notifs = [ UserNotificationType.NEW_COMMENT_ON_MY_VIDEO, UserNotificationType.NEW_FOLLOW ] + await checkNotifications(servers[0], userToken1, notifs) + }) + + it('Should block an account', async function () { + await servers[0].blocklist.addToMyBlocklist({ token: userToken1, server: servers[1].host }) + await waitJobs(servers) + }) + + it('Should not have notifications from this account', async function () { + await checkNotifications(servers[0], userToken1, []) + }) + + it('Should have notifications of this account on user 2', async function () { + const notifs = [ UserNotificationType.COMMENT_MENTION, UserNotificationType.NEW_FOLLOW ] + + await checkNotifications(servers[0], userToken2, notifs) + + await servers[0].blocklist.removeFromMyBlocklist({ token: userToken1, server: servers[1].host }) + }) + }) + + describe('Server blocks a user', function () { + + before(async function () { + this.timeout(30000) + + await resetState() + }) + + it('Should have appropriate notifications', async function () { + { + const notifs = [ UserNotificationType.NEW_COMMENT_ON_MY_VIDEO, UserNotificationType.NEW_FOLLOW ] + await checkNotifications(servers[0], userToken1, notifs) + } + + { + const notifs = [ UserNotificationType.COMMENT_MENTION, UserNotificationType.NEW_FOLLOW ] + await checkNotifications(servers[0], userToken2, notifs) + } + }) + + it('Should block an account', async function () { + await servers[0].blocklist.addToServerBlocklist({ account: 'user3@' + servers[1].host }) + await waitJobs(servers) + }) + + it('Should not have notifications from this account', async function () { + await checkNotifications(servers[0], userToken1, []) + await checkNotifications(servers[0], userToken2, []) + + await servers[0].blocklist.removeFromServerBlocklist({ account: 'user3@' + servers[1].host }) + }) + }) + + describe('Server blocks a server', function () { + + before(async function () { + this.timeout(30000) + + await resetState() + }) + + it('Should have appropriate notifications', async function () { + { + const notifs = [ UserNotificationType.NEW_COMMENT_ON_MY_VIDEO, UserNotificationType.NEW_FOLLOW ] + await checkNotifications(servers[0], userToken1, notifs) + } + + { + const notifs = [ UserNotificationType.COMMENT_MENTION, UserNotificationType.NEW_FOLLOW ] + await checkNotifications(servers[0], userToken2, notifs) + } + }) + + it('Should block an account', async function () { + await servers[0].blocklist.addToServerBlocklist({ server: servers[1].host }) + await waitJobs(servers) + }) + + it('Should not have notifications from this account', async function () { + await checkNotifications(servers[0], userToken1, []) + await checkNotifications(servers[0], userToken2, []) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/moderation/blocklist.ts b/packages/tests/src/api/moderation/blocklist.ts new file mode 100644 index 000000000..a84515241 --- /dev/null +++ b/packages/tests/src/api/moderation/blocklist.ts @@ -0,0 +1,902 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { UserNotificationType } from '@peertube/peertube-models' +import { + BlocklistCommand, + cleanupTests, + CommentsCommand, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + setDefaultAccountAvatar, + waitJobs +} from '@peertube/peertube-server-commands' + +async function checkAllVideos (server: PeerTubeServer, token: string) { + { + const { data } = await server.videos.listWithToken({ token }) + expect(data).to.have.lengthOf(5) + } + + { + const { data } = await server.videos.list() + expect(data).to.have.lengthOf(5) + } +} + +async function checkAllComments (server: PeerTubeServer, token: string, videoUUID: string) { + const { data } = await server.comments.listThreads({ videoId: videoUUID, start: 0, count: 25, sort: '-createdAt', token }) + + const threads = data.filter(t => t.isDeleted === false) + expect(threads).to.have.lengthOf(2) + + for (const thread of threads) { + const tree = await server.comments.getThread({ videoId: videoUUID, threadId: thread.id, token }) + expect(tree.children).to.have.lengthOf(1) + } +} + +async function checkCommentNotification ( + mainServer: PeerTubeServer, + comment: { server: PeerTubeServer, token: string, videoUUID: string, text: string }, + check: 'presence' | 'absence' +) { + const command = comment.server.comments + + const { threadId, createdAt } = await command.createThread({ token: comment.token, videoId: comment.videoUUID, text: comment.text }) + + await waitJobs([ mainServer, comment.server ]) + + const { data } = await mainServer.notifications.list({ start: 0, count: 30 }) + const commentNotifications = data.filter(n => n.comment && n.comment.video.uuid === comment.videoUUID && n.createdAt >= createdAt) + + if (check === 'presence') expect(commentNotifications).to.have.lengthOf(1) + else expect(commentNotifications).to.have.lengthOf(0) + + await command.delete({ token: comment.token, videoId: comment.videoUUID, commentId: threadId }) + + await waitJobs([ mainServer, comment.server ]) +} + +describe('Test blocklist', function () { + let servers: PeerTubeServer[] + let videoUUID1: string + let videoUUID2: string + let videoUUID3: string + let userToken1: string + let userModeratorToken: string + let userToken2: string + + let command: BlocklistCommand + let commentsCommand: CommentsCommand[] + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(3) + await setAccessTokensToServers(servers) + await setDefaultAccountAvatar(servers) + + command = servers[0].blocklist + commentsCommand = servers.map(s => s.comments) + + { + const user = { username: 'user1', password: 'password' } + await servers[0].users.create({ username: user.username, password: user.password }) + + userToken1 = await servers[0].login.getAccessToken(user) + await servers[0].videos.upload({ token: userToken1, attributes: { name: 'video user 1' } }) + } + + { + const user = { username: 'moderator', password: 'password' } + await servers[0].users.create({ username: user.username, password: user.password }) + + userModeratorToken = await servers[0].login.getAccessToken(user) + } + + { + const user = { username: 'user2', password: 'password' } + await servers[1].users.create({ username: user.username, password: user.password }) + + userToken2 = await servers[1].login.getAccessToken(user) + await servers[1].videos.upload({ token: userToken2, attributes: { name: 'video user 2' } }) + } + + { + const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video server 1' } }) + videoUUID1 = uuid + } + + { + const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video server 2' } }) + videoUUID2 = uuid + } + + { + const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video 2 server 1' } }) + videoUUID3 = uuid + } + + await doubleFollow(servers[0], servers[1]) + await doubleFollow(servers[0], servers[2]) + + { + const created = await commentsCommand[0].createThread({ videoId: videoUUID1, text: 'comment root 1' }) + const reply = await commentsCommand[0].addReply({ + token: userToken1, + videoId: videoUUID1, + toCommentId: created.id, + text: 'comment user 1' + }) + await commentsCommand[0].addReply({ videoId: videoUUID1, toCommentId: reply.id, text: 'comment root 1' }) + } + + { + const created = await commentsCommand[0].createThread({ token: userToken1, videoId: videoUUID1, text: 'comment user 1' }) + await commentsCommand[0].addReply({ videoId: videoUUID1, toCommentId: created.id, text: 'comment root 1' }) + } + + await waitJobs(servers) + }) + + describe('User blocklist', function () { + + describe('When managing account blocklist', function () { + it('Should list all videos', function () { + return checkAllVideos(servers[0], servers[0].accessToken) + }) + + it('Should list the comments', function () { + return checkAllComments(servers[0], servers[0].accessToken, videoUUID1) + }) + + it('Should block a remote account', async function () { + await command.addToMyBlocklist({ account: 'user2@' + servers[1].host }) + }) + + it('Should hide its videos', async function () { + const { data } = await servers[0].videos.listWithToken() + + expect(data).to.have.lengthOf(4) + + const v = data.find(v => v.name === 'video user 2') + expect(v).to.be.undefined + }) + + it('Should block a local account', async function () { + await command.addToMyBlocklist({ account: 'user1' }) + }) + + it('Should hide its videos', async function () { + const { data } = await servers[0].videos.listWithToken() + + expect(data).to.have.lengthOf(3) + + const v = data.find(v => v.name === 'video user 1') + expect(v).to.be.undefined + }) + + it('Should hide its comments', async function () { + const { data } = await commentsCommand[0].listThreads({ + token: servers[0].accessToken, + videoId: videoUUID1, + start: 0, + count: 25, + sort: '-createdAt' + }) + + expect(data).to.have.lengthOf(1) + expect(data[0].totalReplies).to.equal(1) + + const t = data.find(t => t.text === 'comment user 1') + expect(t).to.be.undefined + + for (const thread of data) { + const tree = await commentsCommand[0].getThread({ + videoId: videoUUID1, + threadId: thread.id, + token: servers[0].accessToken + }) + expect(tree.children).to.have.lengthOf(0) + } + }) + + it('Should not have notifications from blocked accounts', async function () { + this.timeout(20000) + + { + const comment = { server: servers[0], token: userToken1, videoUUID: videoUUID1, text: 'hidden comment' } + await checkCommentNotification(servers[0], comment, 'absence') + } + + { + const comment = { + server: servers[0], + token: userToken1, + videoUUID: videoUUID2, + text: 'hello @root@' + servers[0].host + } + await checkCommentNotification(servers[0], comment, 'absence') + } + }) + + it('Should list all the videos with another user', async function () { + return checkAllVideos(servers[0], userToken1) + }) + + it('Should list blocked accounts', async function () { + { + const body = await command.listMyAccountBlocklist({ start: 0, count: 1, sort: 'createdAt' }) + expect(body.total).to.equal(2) + + const block = body.data[0] + expect(block.byAccount.displayName).to.equal('root') + expect(block.byAccount.name).to.equal('root') + expect(block.blockedAccount.displayName).to.equal('user2') + expect(block.blockedAccount.name).to.equal('user2') + expect(block.blockedAccount.host).to.equal('' + servers[1].host) + } + + { + const body = await command.listMyAccountBlocklist({ start: 1, count: 2, sort: 'createdAt' }) + expect(body.total).to.equal(2) + + const block = body.data[0] + expect(block.byAccount.displayName).to.equal('root') + expect(block.byAccount.name).to.equal('root') + expect(block.blockedAccount.displayName).to.equal('user1') + expect(block.blockedAccount.name).to.equal('user1') + expect(block.blockedAccount.host).to.equal('' + servers[0].host) + } + }) + + it('Should search blocked accounts', async function () { + const body = await command.listMyAccountBlocklist({ start: 0, count: 10, search: 'user2' }) + expect(body.total).to.equal(1) + + expect(body.data[0].blockedAccount.name).to.equal('user2') + }) + + it('Should get blocked status', async function () { + const remoteHandle = 'user2@' + servers[1].host + const localHandle = 'user1@' + servers[0].host + const unknownHandle = 'user5@' + servers[0].host + + { + const status = await command.getStatus({ accounts: [ remoteHandle ] }) + expect(Object.keys(status.accounts)).to.have.lengthOf(1) + expect(status.accounts[remoteHandle].blockedByUser).to.be.false + expect(status.accounts[remoteHandle].blockedByServer).to.be.false + + expect(Object.keys(status.hosts)).to.have.lengthOf(0) + } + + { + const status = await command.getStatus({ token: servers[0].accessToken, accounts: [ remoteHandle ] }) + expect(Object.keys(status.accounts)).to.have.lengthOf(1) + expect(status.accounts[remoteHandle].blockedByUser).to.be.true + expect(status.accounts[remoteHandle].blockedByServer).to.be.false + + expect(Object.keys(status.hosts)).to.have.lengthOf(0) + } + + { + const status = await command.getStatus({ token: servers[0].accessToken, accounts: [ localHandle, remoteHandle, unknownHandle ] }) + expect(Object.keys(status.accounts)).to.have.lengthOf(3) + + for (const handle of [ localHandle, remoteHandle ]) { + expect(status.accounts[handle].blockedByUser).to.be.true + expect(status.accounts[handle].blockedByServer).to.be.false + } + + expect(status.accounts[unknownHandle].blockedByUser).to.be.false + expect(status.accounts[unknownHandle].blockedByServer).to.be.false + + expect(Object.keys(status.hosts)).to.have.lengthOf(0) + } + }) + + it('Should not allow a remote blocked user to comment my videos', async function () { + this.timeout(60000) + + { + await commentsCommand[1].createThread({ token: userToken2, videoId: videoUUID3, text: 'comment user 2' }) + await waitJobs(servers) + + await commentsCommand[0].createThread({ token: servers[0].accessToken, videoId: videoUUID3, text: 'uploader' }) + await waitJobs(servers) + + const commentId = await commentsCommand[1].findCommentId({ videoId: videoUUID3, text: 'uploader' }) + const message = 'reply by user 2' + const reply = await commentsCommand[1].addReply({ token: userToken2, videoId: videoUUID3, toCommentId: commentId, text: message }) + await commentsCommand[1].addReply({ videoId: videoUUID3, toCommentId: reply.id, text: 'another reply' }) + + await waitJobs(servers) + } + + // Server 2 has all the comments + { + const { data } = await commentsCommand[1].listThreads({ videoId: videoUUID3, count: 25, sort: '-createdAt' }) + + expect(data).to.have.lengthOf(2) + expect(data[0].text).to.equal('uploader') + expect(data[1].text).to.equal('comment user 2') + + const tree = await commentsCommand[1].getThread({ videoId: videoUUID3, threadId: data[0].id }) + expect(tree.children).to.have.lengthOf(1) + expect(tree.children[0].comment.text).to.equal('reply by user 2') + expect(tree.children[0].children).to.have.lengthOf(1) + expect(tree.children[0].children[0].comment.text).to.equal('another reply') + } + + // Server 1 and 3 should only have uploader comments + for (const server of [ servers[0], servers[2] ]) { + const { data } = await server.comments.listThreads({ videoId: videoUUID3, count: 25, sort: '-createdAt' }) + + expect(data).to.have.lengthOf(1) + expect(data[0].text).to.equal('uploader') + + const tree = await server.comments.getThread({ videoId: videoUUID3, threadId: data[0].id }) + + if (server.serverNumber === 1) expect(tree.children).to.have.lengthOf(0) + else expect(tree.children).to.have.lengthOf(1) + } + }) + + it('Should unblock the remote account', async function () { + await command.removeFromMyBlocklist({ account: 'user2@' + servers[1].host }) + }) + + it('Should display its videos', async function () { + const { data } = await servers[0].videos.listWithToken() + expect(data).to.have.lengthOf(4) + + const v = data.find(v => v.name === 'video user 2') + expect(v).not.to.be.undefined + }) + + it('Should display its comments on my video', async function () { + for (const server of servers) { + const { data } = await server.comments.listThreads({ videoId: videoUUID3, count: 25, sort: '-createdAt' }) + + // Server 3 should not have 2 comment threads, because server 1 did not forward the server 2 comment + if (server.serverNumber === 3) { + expect(data).to.have.lengthOf(1) + continue + } + + expect(data).to.have.lengthOf(2) + expect(data[0].text).to.equal('uploader') + expect(data[1].text).to.equal('comment user 2') + + const tree = await server.comments.getThread({ videoId: videoUUID3, threadId: data[0].id }) + expect(tree.children).to.have.lengthOf(1) + expect(tree.children[0].comment.text).to.equal('reply by user 2') + expect(tree.children[0].children).to.have.lengthOf(1) + expect(tree.children[0].children[0].comment.text).to.equal('another reply') + } + }) + + it('Should unblock the local account', async function () { + await command.removeFromMyBlocklist({ account: 'user1' }) + }) + + it('Should display its comments', function () { + return checkAllComments(servers[0], servers[0].accessToken, videoUUID1) + }) + + it('Should have a notification from a non blocked account', async function () { + this.timeout(20000) + + { + const comment = { server: servers[1], token: userToken2, videoUUID: videoUUID1, text: 'displayed comment' } + await checkCommentNotification(servers[0], comment, 'presence') + } + + { + const comment = { + server: servers[0], + token: userToken1, + videoUUID: videoUUID2, + text: 'hello @root@' + servers[0].host + } + await checkCommentNotification(servers[0], comment, 'presence') + } + }) + }) + + describe('When managing server blocklist', function () { + + it('Should list all videos', function () { + return checkAllVideos(servers[0], servers[0].accessToken) + }) + + it('Should list the comments', function () { + return checkAllComments(servers[0], servers[0].accessToken, videoUUID1) + }) + + it('Should block a remote server', async function () { + await command.addToMyBlocklist({ server: '' + servers[1].host }) + }) + + it('Should hide its videos', async function () { + const { data } = await servers[0].videos.listWithToken() + + expect(data).to.have.lengthOf(3) + + const v1 = data.find(v => v.name === 'video user 2') + const v2 = data.find(v => v.name === 'video server 2') + + expect(v1).to.be.undefined + expect(v2).to.be.undefined + }) + + it('Should list all the videos with another user', async function () { + return checkAllVideos(servers[0], userToken1) + }) + + it('Should hide its comments', async function () { + const { id } = await commentsCommand[1].createThread({ token: userToken2, videoId: videoUUID1, text: 'hidden comment 2' }) + + await waitJobs(servers) + + await checkAllComments(servers[0], servers[0].accessToken, videoUUID1) + + await commentsCommand[1].delete({ token: userToken2, videoId: videoUUID1, commentId: id }) + }) + + it('Should not have notifications from blocked server', async function () { + this.timeout(20000) + + { + const comment = { server: servers[1], token: userToken2, videoUUID: videoUUID1, text: 'hidden comment' } + await checkCommentNotification(servers[0], comment, 'absence') + } + + { + const comment = { + server: servers[1], + token: userToken2, + videoUUID: videoUUID1, + text: 'hello @root@' + servers[0].host + } + await checkCommentNotification(servers[0], comment, 'absence') + } + }) + + it('Should list blocked servers', async function () { + const body = await command.listMyServerBlocklist({ start: 0, count: 1, sort: 'createdAt' }) + expect(body.total).to.equal(1) + + const block = body.data[0] + expect(block.byAccount.displayName).to.equal('root') + expect(block.byAccount.name).to.equal('root') + expect(block.blockedServer.host).to.equal('' + servers[1].host) + }) + + it('Should search blocked servers', async function () { + const body = await command.listMyServerBlocklist({ start: 0, count: 10, search: servers[1].host }) + expect(body.total).to.equal(1) + + expect(body.data[0].blockedServer.host).to.equal(servers[1].host) + }) + + it('Should get blocklist status', async function () { + const blockedServer = servers[1].host + const notBlockedServer = 'example.com' + + { + const status = await command.getStatus({ hosts: [ blockedServer, notBlockedServer ] }) + expect(Object.keys(status.accounts)).to.have.lengthOf(0) + + expect(Object.keys(status.hosts)).to.have.lengthOf(2) + expect(status.hosts[blockedServer].blockedByUser).to.be.false + expect(status.hosts[blockedServer].blockedByServer).to.be.false + + expect(status.hosts[notBlockedServer].blockedByUser).to.be.false + expect(status.hosts[notBlockedServer].blockedByServer).to.be.false + } + + { + const status = await command.getStatus({ token: servers[0].accessToken, hosts: [ blockedServer, notBlockedServer ] }) + expect(Object.keys(status.accounts)).to.have.lengthOf(0) + + expect(Object.keys(status.hosts)).to.have.lengthOf(2) + expect(status.hosts[blockedServer].blockedByUser).to.be.true + expect(status.hosts[blockedServer].blockedByServer).to.be.false + + expect(status.hosts[notBlockedServer].blockedByUser).to.be.false + expect(status.hosts[notBlockedServer].blockedByServer).to.be.false + } + }) + + it('Should unblock the remote server', async function () { + await command.removeFromMyBlocklist({ server: '' + servers[1].host }) + }) + + it('Should display its videos', function () { + return checkAllVideos(servers[0], servers[0].accessToken) + }) + + it('Should display its comments', function () { + return checkAllComments(servers[0], servers[0].accessToken, videoUUID1) + }) + + it('Should have notification from unblocked server', async function () { + this.timeout(20000) + + { + const comment = { server: servers[1], token: userToken2, videoUUID: videoUUID1, text: 'displayed comment' } + await checkCommentNotification(servers[0], comment, 'presence') + } + + { + const comment = { + server: servers[1], + token: userToken2, + videoUUID: videoUUID1, + text: 'hello @root@' + servers[0].host + } + await checkCommentNotification(servers[0], comment, 'presence') + } + }) + }) + }) + + describe('Server blocklist', function () { + + describe('When managing account blocklist', function () { + it('Should list all videos', async function () { + for (const token of [ userModeratorToken, servers[0].accessToken ]) { + await checkAllVideos(servers[0], token) + } + }) + + it('Should list the comments', async function () { + for (const token of [ userModeratorToken, servers[0].accessToken ]) { + await checkAllComments(servers[0], token, videoUUID1) + } + }) + + it('Should block a remote account', async function () { + await command.addToServerBlocklist({ account: 'user2@' + servers[1].host }) + }) + + it('Should hide its videos', async function () { + for (const token of [ userModeratorToken, servers[0].accessToken ]) { + const { data } = await servers[0].videos.listWithToken({ token }) + + expect(data).to.have.lengthOf(4) + + const v = data.find(v => v.name === 'video user 2') + expect(v).to.be.undefined + } + }) + + it('Should block a local account', async function () { + await command.addToServerBlocklist({ account: 'user1' }) + }) + + it('Should hide its videos', async function () { + for (const token of [ userModeratorToken, servers[0].accessToken ]) { + const { data } = await servers[0].videos.listWithToken({ token }) + + expect(data).to.have.lengthOf(3) + + const v = data.find(v => v.name === 'video user 1') + expect(v).to.be.undefined + } + }) + + it('Should hide its comments', async function () { + for (const token of [ userModeratorToken, servers[0].accessToken ]) { + const { data } = await commentsCommand[0].listThreads({ videoId: videoUUID1, count: 20, sort: '-createdAt', token }) + const threads = data.filter(t => t.isDeleted === false) + + expect(threads).to.have.lengthOf(1) + expect(threads[0].totalReplies).to.equal(1) + + const t = threads.find(t => t.text === 'comment user 1') + expect(t).to.be.undefined + + for (const thread of threads) { + const tree = await commentsCommand[0].getThread({ videoId: videoUUID1, threadId: thread.id, token }) + expect(tree.children).to.have.lengthOf(0) + } + } + }) + + it('Should not have notification from blocked accounts by instance', async function () { + this.timeout(20000) + + { + const comment = { server: servers[0], token: userToken1, videoUUID: videoUUID1, text: 'hidden comment' } + await checkCommentNotification(servers[0], comment, 'absence') + } + + { + const comment = { + server: servers[1], + token: userToken2, + videoUUID: videoUUID1, + text: 'hello @root@' + servers[0].host + } + await checkCommentNotification(servers[0], comment, 'absence') + } + }) + + it('Should list blocked accounts', async function () { + { + const body = await command.listServerAccountBlocklist({ start: 0, count: 1, sort: 'createdAt' }) + expect(body.total).to.equal(2) + + const block = body.data[0] + expect(block.byAccount.displayName).to.equal('peertube') + expect(block.byAccount.name).to.equal('peertube') + expect(block.blockedAccount.displayName).to.equal('user2') + expect(block.blockedAccount.name).to.equal('user2') + expect(block.blockedAccount.host).to.equal('' + servers[1].host) + } + + { + const body = await command.listServerAccountBlocklist({ start: 1, count: 2, sort: 'createdAt' }) + expect(body.total).to.equal(2) + + const block = body.data[0] + expect(block.byAccount.displayName).to.equal('peertube') + expect(block.byAccount.name).to.equal('peertube') + expect(block.blockedAccount.displayName).to.equal('user1') + expect(block.blockedAccount.name).to.equal('user1') + expect(block.blockedAccount.host).to.equal('' + servers[0].host) + } + }) + + it('Should search blocked accounts', async function () { + const body = await command.listServerAccountBlocklist({ start: 0, count: 10, search: 'user2' }) + expect(body.total).to.equal(1) + + expect(body.data[0].blockedAccount.name).to.equal('user2') + }) + + it('Should get blocked status', async function () { + const remoteHandle = 'user2@' + servers[1].host + const localHandle = 'user1@' + servers[0].host + const unknownHandle = 'user5@' + servers[0].host + + for (const token of [ undefined, servers[0].accessToken ]) { + const status = await command.getStatus({ token, accounts: [ localHandle, remoteHandle, unknownHandle ] }) + expect(Object.keys(status.accounts)).to.have.lengthOf(3) + + for (const handle of [ localHandle, remoteHandle ]) { + expect(status.accounts[handle].blockedByUser).to.be.false + expect(status.accounts[handle].blockedByServer).to.be.true + } + + expect(status.accounts[unknownHandle].blockedByUser).to.be.false + expect(status.accounts[unknownHandle].blockedByServer).to.be.false + + expect(Object.keys(status.hosts)).to.have.lengthOf(0) + } + }) + + it('Should unblock the remote account', async function () { + await command.removeFromServerBlocklist({ account: 'user2@' + servers[1].host }) + }) + + it('Should display its videos', async function () { + for (const token of [ userModeratorToken, servers[0].accessToken ]) { + const { data } = await servers[0].videos.listWithToken({ token }) + expect(data).to.have.lengthOf(4) + + const v = data.find(v => v.name === 'video user 2') + expect(v).not.to.be.undefined + } + }) + + it('Should unblock the local account', async function () { + await command.removeFromServerBlocklist({ account: 'user1' }) + }) + + it('Should display its comments', async function () { + for (const token of [ userModeratorToken, servers[0].accessToken ]) { + await checkAllComments(servers[0], token, videoUUID1) + } + }) + + it('Should have notifications from unblocked accounts', async function () { + this.timeout(20000) + + { + const comment = { server: servers[0], token: userToken1, videoUUID: videoUUID1, text: 'displayed comment' } + await checkCommentNotification(servers[0], comment, 'presence') + } + + { + const comment = { + server: servers[1], + token: userToken2, + videoUUID: videoUUID1, + text: 'hello @root@' + servers[0].host + } + await checkCommentNotification(servers[0], comment, 'presence') + } + }) + }) + + describe('When managing server blocklist', function () { + + it('Should list all videos', async function () { + for (const token of [ userModeratorToken, servers[0].accessToken ]) { + await checkAllVideos(servers[0], token) + } + }) + + it('Should list the comments', async function () { + for (const token of [ userModeratorToken, servers[0].accessToken ]) { + await checkAllComments(servers[0], token, videoUUID1) + } + }) + + it('Should block a remote server', async function () { + await command.addToServerBlocklist({ server: '' + servers[1].host }) + }) + + it('Should hide its videos', async function () { + for (const token of [ userModeratorToken, servers[0].accessToken ]) { + const requests = [ + servers[0].videos.list(), + servers[0].videos.listWithToken({ token }) + ] + + for (const req of requests) { + const { data } = await req + expect(data).to.have.lengthOf(3) + + const v1 = data.find(v => v.name === 'video user 2') + const v2 = data.find(v => v.name === 'video server 2') + + expect(v1).to.be.undefined + expect(v2).to.be.undefined + } + } + }) + + it('Should hide its comments', async function () { + const { id } = await commentsCommand[1].createThread({ token: userToken2, videoId: videoUUID1, text: 'hidden comment 2' }) + + await waitJobs(servers) + + await checkAllComments(servers[0], servers[0].accessToken, videoUUID1) + + await commentsCommand[1].delete({ token: userToken2, videoId: videoUUID1, commentId: id }) + }) + + it('Should not have notification from blocked instances by instance', async function () { + this.timeout(50000) + + { + const comment = { server: servers[1], token: userToken2, videoUUID: videoUUID1, text: 'hidden comment' } + await checkCommentNotification(servers[0], comment, 'absence') + } + + { + const comment = { + server: servers[1], + token: userToken2, + videoUUID: videoUUID1, + text: 'hello @root@' + servers[0].host + } + await checkCommentNotification(servers[0], comment, 'absence') + } + + { + const now = new Date() + await servers[1].follows.unfollow({ target: servers[0] }) + await waitJobs(servers) + await servers[1].follows.follow({ hosts: [ servers[0].host ] }) + + await waitJobs(servers) + + const { data } = await servers[0].notifications.list({ start: 0, count: 30 }) + const commentNotifications = data.filter(n => { + return n.type === UserNotificationType.NEW_INSTANCE_FOLLOWER && n.createdAt >= now.toISOString() + }) + + expect(commentNotifications).to.have.lengthOf(0) + } + }) + + it('Should list blocked servers', async function () { + const body = await command.listServerServerBlocklist({ start: 0, count: 1, sort: 'createdAt' }) + expect(body.total).to.equal(1) + + const block = body.data[0] + expect(block.byAccount.displayName).to.equal('peertube') + expect(block.byAccount.name).to.equal('peertube') + expect(block.blockedServer.host).to.equal('' + servers[1].host) + }) + + it('Should search blocked servers', async function () { + const body = await command.listServerServerBlocklist({ start: 0, count: 10, search: servers[1].host }) + expect(body.total).to.equal(1) + + expect(body.data[0].blockedServer.host).to.equal(servers[1].host) + }) + + it('Should get blocklist status', async function () { + const blockedServer = servers[1].host + const notBlockedServer = 'example.com' + + for (const token of [ undefined, servers[0].accessToken ]) { + const status = await command.getStatus({ token, hosts: [ blockedServer, notBlockedServer ] }) + expect(Object.keys(status.accounts)).to.have.lengthOf(0) + + expect(Object.keys(status.hosts)).to.have.lengthOf(2) + expect(status.hosts[blockedServer].blockedByUser).to.be.false + expect(status.hosts[blockedServer].blockedByServer).to.be.true + + expect(status.hosts[notBlockedServer].blockedByUser).to.be.false + expect(status.hosts[notBlockedServer].blockedByServer).to.be.false + } + }) + + it('Should unblock the remote server', async function () { + await command.removeFromServerBlocklist({ server: '' + servers[1].host }) + }) + + it('Should list all videos', async function () { + for (const token of [ userModeratorToken, servers[0].accessToken ]) { + await checkAllVideos(servers[0], token) + } + }) + + it('Should list the comments', async function () { + for (const token of [ userModeratorToken, servers[0].accessToken ]) { + await checkAllComments(servers[0], token, videoUUID1) + } + }) + + it('Should have notification from unblocked instances', async function () { + this.timeout(50000) + + { + const comment = { server: servers[1], token: userToken2, videoUUID: videoUUID1, text: 'displayed comment' } + await checkCommentNotification(servers[0], comment, 'presence') + } + + { + const comment = { + server: servers[1], + token: userToken2, + videoUUID: videoUUID1, + text: 'hello @root@' + servers[0].host + } + await checkCommentNotification(servers[0], comment, 'presence') + } + + { + const now = new Date() + await servers[1].follows.unfollow({ target: servers[0] }) + await waitJobs(servers) + await servers[1].follows.follow({ hosts: [ servers[0].host ] }) + + await waitJobs(servers) + + const { data } = await servers[0].notifications.list({ start: 0, count: 30 }) + const commentNotifications = data.filter(n => { + return n.type === UserNotificationType.NEW_INSTANCE_FOLLOWER && n.createdAt >= now.toISOString() + }) + + expect(commentNotifications).to.have.lengthOf(1) + } + }) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/moderation/index.ts b/packages/tests/src/api/moderation/index.ts new file mode 100644 index 000000000..e3794d01e --- /dev/null +++ b/packages/tests/src/api/moderation/index.ts @@ -0,0 +1,4 @@ +export * from './abuses.js' +export * from './blocklist-notification.js' +export * from './blocklist.js' +export * from './video-blacklist.js' diff --git a/packages/tests/src/api/moderation/video-blacklist.ts b/packages/tests/src/api/moderation/video-blacklist.ts new file mode 100644 index 000000000..341dadad0 --- /dev/null +++ b/packages/tests/src/api/moderation/video-blacklist.ts @@ -0,0 +1,414 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { FIXTURE_URLS } from '@tests/shared/tests.js' +import { sortObjectComparator } from '@peertube/peertube-core-utils' +import { UserAdminFlag, UserRole, VideoBlacklist, VideoBlacklistType } from '@peertube/peertube-models' +import { + BlacklistCommand, + cleanupTests, + createMultipleServers, + doubleFollow, + killallServers, + PeerTubeServer, + setAccessTokensToServers, + setDefaultChannelAvatar, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test video blacklist', function () { + let servers: PeerTubeServer[] = [] + let videoId: number + let command: BlacklistCommand + + async function blacklistVideosOnServer (server: PeerTubeServer) { + const { data } = await server.videos.list() + + for (const video of data) { + await server.blacklist.add({ videoId: video.id, reason: 'super reason' }) + } + } + + before(async function () { + this.timeout(120000) + + // Run servers + servers = await createMultipleServers(2) + + // Get the access tokens + await setAccessTokensToServers(servers) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + await setDefaultChannelAvatar(servers[0]) + + // Upload 2 videos on server 2 + await servers[1].videos.upload({ attributes: { name: 'My 1st video', description: 'A video on server 2' } }) + await servers[1].videos.upload({ attributes: { name: 'My 2nd video', description: 'A video on server 2' } }) + + // Wait videos propagation, server 2 has transcoding enabled + await waitJobs(servers) + + command = servers[0].blacklist + + // Blacklist the two videos on server 1 + await blacklistVideosOnServer(servers[0]) + }) + + describe('When listing/searching videos', function () { + + it('Should not have the video blacklisted in videos list/search on server 1', async function () { + { + const { total, data } = await servers[0].videos.list() + + expect(total).to.equal(0) + expect(data).to.be.an('array') + expect(data.length).to.equal(0) + } + + { + const body = await servers[0].search.searchVideos({ search: 'video' }) + + expect(body.total).to.equal(0) + expect(body.data).to.be.an('array') + expect(body.data.length).to.equal(0) + } + }) + + it('Should have the blacklisted video in videos list/search on server 2', async function () { + { + const { total, data } = await servers[1].videos.list() + + expect(total).to.equal(2) + expect(data).to.be.an('array') + expect(data.length).to.equal(2) + } + + { + const body = await servers[1].search.searchVideos({ search: 'video' }) + + expect(body.total).to.equal(2) + expect(body.data).to.be.an('array') + expect(body.data.length).to.equal(2) + } + }) + }) + + describe('When listing manually blacklisted videos', function () { + it('Should display all the blacklisted videos', async function () { + const body = await command.list() + expect(body.total).to.equal(2) + + const blacklistedVideos = body.data + expect(blacklistedVideos).to.be.an('array') + expect(blacklistedVideos.length).to.equal(2) + + for (const blacklistedVideo of blacklistedVideos) { + expect(blacklistedVideo.reason).to.equal('super reason') + videoId = blacklistedVideo.video.id + } + }) + + it('Should display all the blacklisted videos when applying manual type filter', async function () { + const body = await command.list({ type: VideoBlacklistType.MANUAL }) + expect(body.total).to.equal(2) + + const blacklistedVideos = body.data + expect(blacklistedVideos).to.be.an('array') + expect(blacklistedVideos.length).to.equal(2) + }) + + it('Should display nothing when applying automatic type filter', async function () { + const body = await command.list({ type: VideoBlacklistType.AUTO_BEFORE_PUBLISHED }) + expect(body.total).to.equal(0) + + const blacklistedVideos = body.data + expect(blacklistedVideos).to.be.an('array') + expect(blacklistedVideos.length).to.equal(0) + }) + + it('Should get the correct sort when sorting by descending id', async function () { + const body = await command.list({ sort: '-id' }) + expect(body.total).to.equal(2) + + const blacklistedVideos = body.data + expect(blacklistedVideos).to.be.an('array') + expect(blacklistedVideos.length).to.equal(2) + + const result = [ ...body.data ].sort(sortObjectComparator('id', 'desc')) + expect(blacklistedVideos).to.deep.equal(result) + }) + + it('Should get the correct sort when sorting by descending video name', async function () { + const body = await command.list({ sort: '-name' }) + expect(body.total).to.equal(2) + + const blacklistedVideos = body.data + expect(blacklistedVideos).to.be.an('array') + expect(blacklistedVideos.length).to.equal(2) + + const result = [ ...body.data ].sort(sortObjectComparator('name', 'desc')) + expect(blacklistedVideos).to.deep.equal(result) + }) + + it('Should get the correct sort when sorting by ascending creation date', async function () { + const body = await command.list({ sort: 'createdAt' }) + expect(body.total).to.equal(2) + + const blacklistedVideos = body.data + expect(blacklistedVideos).to.be.an('array') + expect(blacklistedVideos.length).to.equal(2) + + const result = [ ...body.data ].sort(sortObjectComparator('createdAt', 'asc')) + expect(blacklistedVideos).to.deep.equal(result) + }) + }) + + describe('When updating blacklisted videos', function () { + it('Should change the reason', async function () { + await command.update({ videoId, reason: 'my super reason updated' }) + + const body = await command.list({ sort: '-name' }) + const video = body.data.find(b => b.video.id === videoId) + + expect(video.reason).to.equal('my super reason updated') + }) + }) + + describe('When listing my videos', function () { + it('Should display blacklisted videos', async function () { + await blacklistVideosOnServer(servers[1]) + + const { total, data } = await servers[1].videos.listMyVideos() + + expect(total).to.equal(2) + expect(data).to.have.lengthOf(2) + + for (const video of data) { + expect(video.blacklisted).to.be.true + expect(video.blacklistedReason).to.equal('super reason') + } + }) + }) + + describe('When removing a blacklisted video', function () { + let videoToRemove: VideoBlacklist + let blacklist = [] + + it('Should not have any video in videos list on server 1', async function () { + const { total, data } = await servers[0].videos.list() + expect(total).to.equal(0) + expect(data).to.be.an('array') + expect(data.length).to.equal(0) + }) + + it('Should remove a video from the blacklist on server 1', async function () { + // Get one video in the blacklist + const body = await command.list({ sort: '-name' }) + videoToRemove = body.data[0] + blacklist = body.data.slice(1) + + // Remove it + await command.remove({ videoId: videoToRemove.video.id }) + }) + + it('Should have the ex-blacklisted video in videos list on server 1', async function () { + const { total, data } = await servers[0].videos.list() + expect(total).to.equal(1) + + expect(data).to.be.an('array') + expect(data.length).to.equal(1) + + expect(data[0].name).to.equal(videoToRemove.video.name) + expect(data[0].id).to.equal(videoToRemove.video.id) + }) + + it('Should not have the ex-blacklisted video in videos blacklist list on server 1', async function () { + const body = await command.list({ sort: '-name' }) + expect(body.total).to.equal(1) + + const videos = body.data + expect(videos).to.be.an('array') + expect(videos.length).to.equal(1) + expect(videos).to.deep.equal(blacklist) + }) + }) + + describe('When blacklisting local videos', function () { + let video3UUID: string + let video4UUID: string + + before(async function () { + { + const { uuid } = await servers[0].videos.upload({ attributes: { name: 'Video 3' } }) + video3UUID = uuid + } + { + const { uuid } = await servers[0].videos.upload({ attributes: { name: 'Video 4' } }) + video4UUID = uuid + } + + await waitJobs(servers) + }) + + it('Should blacklist video 3 and keep it federated', async function () { + await command.add({ videoId: video3UUID, reason: 'super reason', unfederate: false }) + + await waitJobs(servers) + + { + const { data } = await servers[0].videos.list() + expect(data.find(v => v.uuid === video3UUID)).to.be.undefined + } + + { + const { data } = await servers[1].videos.list() + expect(data.find(v => v.uuid === video3UUID)).to.not.be.undefined + } + }) + + it('Should unfederate the video', async function () { + await command.add({ videoId: video4UUID, reason: 'super reason', unfederate: true }) + + await waitJobs(servers) + + for (const server of servers) { + const { data } = await server.videos.list() + expect(data.find(v => v.uuid === video4UUID)).to.be.undefined + } + }) + + it('Should have the video unfederated even after an Update AP message', async function () { + await servers[0].videos.update({ id: video4UUID, attributes: { description: 'super description' } }) + + await waitJobs(servers) + + for (const server of servers) { + const { data } = await server.videos.list() + expect(data.find(v => v.uuid === video4UUID)).to.be.undefined + } + }) + + it('Should have the correct video blacklist unfederate attribute', async function () { + const body = await command.list({ sort: 'createdAt' }) + + const blacklistedVideos = body.data + const video3Blacklisted = blacklistedVideos.find(b => b.video.uuid === video3UUID) + const video4Blacklisted = blacklistedVideos.find(b => b.video.uuid === video4UUID) + + expect(video3Blacklisted.unfederated).to.be.false + expect(video4Blacklisted.unfederated).to.be.true + }) + + it('Should remove the video from blacklist and refederate the video', async function () { + await command.remove({ videoId: video4UUID }) + + await waitJobs(servers) + + for (const server of servers) { + const { data } = await server.videos.list() + expect(data.find(v => v.uuid === video4UUID)).to.not.be.undefined + } + }) + + }) + + describe('When auto blacklist videos', function () { + let userWithoutFlag: string + let userWithFlag: string + let channelOfUserWithoutFlag: number + + before(async function () { + this.timeout(20000) + + await killallServers([ servers[0] ]) + + const config = { + auto_blacklist: { + videos: { + of_users: { + enabled: true + } + } + } + } + await servers[0].run(config) + + { + const user = { username: 'user_without_flag', password: 'password' } + await servers[0].users.create({ + username: user.username, + adminFlags: UserAdminFlag.NONE, + password: user.password, + role: UserRole.USER + }) + + userWithoutFlag = await servers[0].login.getAccessToken(user) + + const { videoChannels } = await servers[0].users.getMyInfo({ token: userWithoutFlag }) + channelOfUserWithoutFlag = videoChannels[0].id + } + + { + const user = { username: 'user_with_flag', password: 'password' } + await servers[0].users.create({ + username: user.username, + adminFlags: UserAdminFlag.BYPASS_VIDEO_AUTO_BLACKLIST, + password: user.password, + role: UserRole.USER + }) + + userWithFlag = await servers[0].login.getAccessToken(user) + } + + await waitJobs(servers) + }) + + it('Should auto blacklist a video on upload', async function () { + await servers[0].videos.upload({ token: userWithoutFlag, attributes: { name: 'blacklisted' } }) + + const body = await command.list({ type: VideoBlacklistType.AUTO_BEFORE_PUBLISHED }) + expect(body.total).to.equal(1) + expect(body.data[0].video.name).to.equal('blacklisted') + }) + + it('Should auto blacklist a video on URL import', async function () { + this.timeout(15000) + + const attributes = { + targetUrl: FIXTURE_URLS.goodVideo, + name: 'URL import', + channelId: channelOfUserWithoutFlag + } + await servers[0].imports.importVideo({ token: userWithoutFlag, attributes }) + + const body = await command.list({ sort: 'createdAt', type: VideoBlacklistType.AUTO_BEFORE_PUBLISHED }) + expect(body.total).to.equal(2) + expect(body.data[1].video.name).to.equal('URL import') + }) + + it('Should auto blacklist a video on torrent import', async function () { + const attributes = { + magnetUri: FIXTURE_URLS.magnet, + name: 'Torrent import', + channelId: channelOfUserWithoutFlag + } + await servers[0].imports.importVideo({ token: userWithoutFlag, attributes }) + + const body = await command.list({ sort: 'createdAt', type: VideoBlacklistType.AUTO_BEFORE_PUBLISHED }) + expect(body.total).to.equal(3) + expect(body.data[2].video.name).to.equal('Torrent import') + }) + + it('Should not auto blacklist a video on upload if the user has the bypass blacklist flag', async function () { + await servers[0].videos.upload({ token: userWithFlag, attributes: { name: 'not blacklisted' } }) + + const body = await command.list({ type: VideoBlacklistType.AUTO_BEFORE_PUBLISHED }) + expect(body.total).to.equal(3) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/notifications/admin-notifications.ts b/packages/tests/src/api/notifications/admin-notifications.ts new file mode 100644 index 000000000..2186dc55a --- /dev/null +++ b/packages/tests/src/api/notifications/admin-notifications.ts @@ -0,0 +1,154 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { PluginType, UserNotification, UserNotificationType } from '@peertube/peertube-models' +import { cleanupTests, PeerTubeServer } from '@peertube/peertube-server-commands' +import { MockSmtpServer } from '@tests/shared/mock-servers/mock-email.js' +import { MockJoinPeerTubeVersions } from '@tests/shared/mock-servers/mock-joinpeertube-versions.js' +import { CheckerBaseParams, prepareNotificationsTest, checkNewPeerTubeVersion, checkNewPluginVersion } from '@tests/shared/notifications.js' +import { SQLCommand } from '@tests/shared/sql-command.js' + +describe('Test admin notifications', function () { + let server: PeerTubeServer + let sqlCommand: SQLCommand + let userNotifications: UserNotification[] = [] + let adminNotifications: UserNotification[] = [] + let emails: object[] = [] + let baseParams: CheckerBaseParams + let joinPeerTubeServer: MockJoinPeerTubeVersions + + before(async function () { + this.timeout(120000) + + joinPeerTubeServer = new MockJoinPeerTubeVersions() + const port = await joinPeerTubeServer.initialize() + + const config = { + peertube: { + check_latest_version: { + enabled: true, + url: `http://127.0.0.1:${port}/versions.json` + } + }, + plugins: { + index: { + enabled: true, + check_latest_versions_interval: '3 seconds' + } + } + } + + const res = await prepareNotificationsTest(1, config) + emails = res.emails + server = res.servers[0] + + userNotifications = res.userNotifications + adminNotifications = res.adminNotifications + + baseParams = { + server, + emails, + socketNotifications: adminNotifications, + token: server.accessToken + } + + await server.plugins.install({ npmName: 'peertube-plugin-hello-world' }) + await server.plugins.install({ npmName: 'peertube-theme-background-red' }) + + sqlCommand = new SQLCommand(server) + }) + + describe('Latest PeerTube version notification', function () { + + it('Should not send a notification to admins if there is no new version', async function () { + this.timeout(30000) + + joinPeerTubeServer.setLatestVersion('1.4.2') + + await wait(3000) + await checkNewPeerTubeVersion({ ...baseParams, latestVersion: '1.4.2', checkType: 'absence' }) + }) + + it('Should send a notification to admins on new version', async function () { + this.timeout(30000) + + joinPeerTubeServer.setLatestVersion('15.4.2') + + await wait(3000) + await checkNewPeerTubeVersion({ ...baseParams, latestVersion: '15.4.2', checkType: 'presence' }) + }) + + it('Should not send the same notification to admins', async function () { + this.timeout(30000) + + await wait(3000) + expect(adminNotifications.filter(n => n.type === UserNotificationType.NEW_PEERTUBE_VERSION)).to.have.lengthOf(1) + }) + + it('Should not have sent a notification to users', async function () { + this.timeout(30000) + + expect(userNotifications.filter(n => n.type === UserNotificationType.NEW_PEERTUBE_VERSION)).to.have.lengthOf(0) + }) + + it('Should send a new notification after a new release', async function () { + this.timeout(30000) + + joinPeerTubeServer.setLatestVersion('15.4.3') + + await wait(3000) + await checkNewPeerTubeVersion({ ...baseParams, latestVersion: '15.4.3', checkType: 'presence' }) + expect(adminNotifications.filter(n => n.type === UserNotificationType.NEW_PEERTUBE_VERSION)).to.have.lengthOf(2) + }) + }) + + describe('Latest plugin version notification', function () { + + it('Should not send a notification to admins if there is no new plugin version', async function () { + this.timeout(30000) + + await wait(6000) + await checkNewPluginVersion({ ...baseParams, pluginType: PluginType.PLUGIN, pluginName: 'hello-world', checkType: 'absence' }) + }) + + it('Should send a notification to admins on new plugin version', async function () { + this.timeout(30000) + + await sqlCommand.setPluginVersion('hello-world', '0.0.1') + await sqlCommand.setPluginLatestVersion('hello-world', '0.0.1') + await wait(6000) + + await checkNewPluginVersion({ ...baseParams, pluginType: PluginType.PLUGIN, pluginName: 'hello-world', checkType: 'presence' }) + }) + + it('Should not send the same notification to admins', async function () { + this.timeout(30000) + + await wait(6000) + + expect(adminNotifications.filter(n => n.type === UserNotificationType.NEW_PLUGIN_VERSION)).to.have.lengthOf(1) + }) + + it('Should not have sent a notification to users', async function () { + expect(userNotifications.filter(n => n.type === UserNotificationType.NEW_PLUGIN_VERSION)).to.have.lengthOf(0) + }) + + it('Should send a new notification after a new plugin release', async function () { + this.timeout(30000) + + await sqlCommand.setPluginVersion('hello-world', '0.0.1') + await sqlCommand.setPluginLatestVersion('hello-world', '0.0.1') + await wait(6000) + + expect(adminNotifications.filter(n => n.type === UserNotificationType.NEW_PEERTUBE_VERSION)).to.have.lengthOf(2) + }) + }) + + after(async function () { + MockSmtpServer.Instance.kill() + + await sqlCommand.cleanup() + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/notifications/comments-notifications.ts b/packages/tests/src/api/notifications/comments-notifications.ts new file mode 100644 index 000000000..5647d1286 --- /dev/null +++ b/packages/tests/src/api/notifications/comments-notifications.ts @@ -0,0 +1,300 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { UserNotification } from '@peertube/peertube-models' +import { cleanupTests, PeerTubeServer, waitJobs } from '@peertube/peertube-server-commands' +import { MockSmtpServer } from '@tests/shared/mock-servers/mock-email.js' +import { prepareNotificationsTest, CheckerBaseParams, checkNewCommentOnMyVideo, checkCommentMention } from '@tests/shared/notifications.js' + +describe('Test comments notifications', function () { + let servers: PeerTubeServer[] = [] + let userToken: string + let userNotifications: UserNotification[] = [] + let emails: object[] = [] + + const commentText = '**hello** world,

what do you think about peertube?

' + const expectedHtml = 'hello world' + + ',

what do you think about peertube?' + + before(async function () { + this.timeout(120000) + + const res = await prepareNotificationsTest(2) + emails = res.emails + userToken = res.userAccessToken + servers = res.servers + userNotifications = res.userNotifications + }) + + describe('Comment on my video notifications', function () { + let baseParams: CheckerBaseParams + + before(() => { + baseParams = { + server: servers[0], + emails, + socketNotifications: userNotifications, + token: userToken + } + }) + + it('Should not send a new comment notification after a comment on another video', async function () { + this.timeout(30000) + + const { uuid, shortUUID } = await servers[0].videos.upload({ attributes: { name: 'super video' } }) + + const created = await servers[0].comments.createThread({ videoId: uuid, text: 'comment' }) + const commentId = created.id + + await waitJobs(servers) + await checkNewCommentOnMyVideo({ ...baseParams, shortUUID, threadId: commentId, commentId, checkType: 'absence' }) + }) + + it('Should not send a new comment notification if I comment my own video', async function () { + this.timeout(30000) + + const { uuid, shortUUID } = await servers[0].videos.upload({ token: userToken, attributes: { name: 'super video' } }) + + const created = await servers[0].comments.createThread({ token: userToken, videoId: uuid, text: 'comment' }) + const commentId = created.id + + await waitJobs(servers) + await checkNewCommentOnMyVideo({ ...baseParams, shortUUID, threadId: commentId, commentId, checkType: 'absence' }) + }) + + it('Should not send a new comment notification if the account is muted', async function () { + this.timeout(30000) + + await servers[0].blocklist.addToMyBlocklist({ token: userToken, account: 'root' }) + + const { uuid, shortUUID } = await servers[0].videos.upload({ token: userToken, attributes: { name: 'super video' } }) + + const created = await servers[0].comments.createThread({ videoId: uuid, text: 'comment' }) + const commentId = created.id + + await waitJobs(servers) + await checkNewCommentOnMyVideo({ ...baseParams, shortUUID, threadId: commentId, commentId, checkType: 'absence' }) + + await servers[0].blocklist.removeFromMyBlocklist({ token: userToken, account: 'root' }) + }) + + it('Should send a new comment notification after a local comment on my video', async function () { + this.timeout(30000) + + const { uuid, shortUUID } = await servers[0].videos.upload({ token: userToken, attributes: { name: 'super video' } }) + + const created = await servers[0].comments.createThread({ videoId: uuid, text: 'comment' }) + const commentId = created.id + + await waitJobs(servers) + await checkNewCommentOnMyVideo({ ...baseParams, shortUUID, threadId: commentId, commentId, checkType: 'presence' }) + }) + + it('Should send a new comment notification after a remote comment on my video', async function () { + this.timeout(30000) + + const { uuid, shortUUID } = await servers[0].videos.upload({ token: userToken, attributes: { name: 'super video' } }) + + await waitJobs(servers) + + await servers[1].comments.createThread({ videoId: uuid, text: 'comment' }) + + await waitJobs(servers) + + const { data } = await servers[0].comments.listThreads({ videoId: uuid }) + expect(data).to.have.lengthOf(1) + + const commentId = data[0].id + await checkNewCommentOnMyVideo({ ...baseParams, shortUUID, threadId: commentId, commentId, checkType: 'presence' }) + }) + + it('Should send a new comment notification after a local reply on my video', async function () { + this.timeout(30000) + + const { uuid, shortUUID } = await servers[0].videos.upload({ token: userToken, attributes: { name: 'super video' } }) + + const { id: threadId } = await servers[0].comments.createThread({ videoId: uuid, text: 'comment' }) + + const { id: commentId } = await servers[0].comments.addReply({ videoId: uuid, toCommentId: threadId, text: 'reply' }) + + await waitJobs(servers) + await checkNewCommentOnMyVideo({ ...baseParams, shortUUID, threadId, commentId, checkType: 'presence' }) + }) + + it('Should send a new comment notification after a remote reply on my video', async function () { + this.timeout(30000) + + const { uuid, shortUUID } = await servers[0].videos.upload({ token: userToken, attributes: { name: 'super video' } }) + await waitJobs(servers) + + { + const created = await servers[1].comments.createThread({ videoId: uuid, text: 'comment' }) + const threadId = created.id + await servers[1].comments.addReply({ videoId: uuid, toCommentId: threadId, text: 'reply' }) + } + + await waitJobs(servers) + + const { data } = await servers[0].comments.listThreads({ videoId: uuid }) + expect(data).to.have.lengthOf(1) + + const threadId = data[0].id + const tree = await servers[0].comments.getThread({ videoId: uuid, threadId }) + + expect(tree.children).to.have.lengthOf(1) + const commentId = tree.children[0].comment.id + + await checkNewCommentOnMyVideo({ ...baseParams, shortUUID, threadId, commentId, checkType: 'presence' }) + }) + + it('Should convert markdown in comment to html', async function () { + this.timeout(30000) + + const { uuid } = await servers[0].videos.upload({ token: userToken, attributes: { name: 'cool video' } }) + + await servers[0].comments.createThread({ videoId: uuid, text: commentText }) + + await waitJobs(servers) + + const latestEmail = emails[emails.length - 1] + expect(latestEmail['html']).to.contain(expectedHtml) + }) + }) + + describe('Mention notifications', function () { + let baseParams: CheckerBaseParams + const byAccountDisplayName = 'super root name' + + before(async function () { + baseParams = { + server: servers[0], + emails, + socketNotifications: userNotifications, + token: userToken + } + + await servers[0].users.updateMe({ displayName: 'super root name' }) + await servers[1].users.updateMe({ displayName: 'super root 2 name' }) + }) + + it('Should not send a new mention comment notification if I mention the video owner', async function () { + this.timeout(30000) + + const { uuid, shortUUID } = await servers[0].videos.upload({ token: userToken, attributes: { name: 'super video' } }) + + const { id: commentId } = await servers[0].comments.createThread({ videoId: uuid, text: '@user_1 hello' }) + + await waitJobs(servers) + await checkCommentMention({ ...baseParams, shortUUID, threadId: commentId, commentId, byAccountDisplayName, checkType: 'absence' }) + }) + + it('Should not send a new mention comment notification if I mention myself', async function () { + this.timeout(30000) + + const { uuid, shortUUID } = await servers[0].videos.upload({ attributes: { name: 'super video' } }) + + const { id: commentId } = await servers[0].comments.createThread({ token: userToken, videoId: uuid, text: '@user_1 hello' }) + + await waitJobs(servers) + await checkCommentMention({ ...baseParams, shortUUID, threadId: commentId, commentId, byAccountDisplayName, checkType: 'absence' }) + }) + + it('Should not send a new mention notification if the account is muted', async function () { + this.timeout(30000) + + await servers[0].blocklist.addToMyBlocklist({ token: userToken, account: 'root' }) + + const { uuid, shortUUID } = await servers[0].videos.upload({ attributes: { name: 'super video' } }) + + const { id: commentId } = await servers[0].comments.createThread({ videoId: uuid, text: '@user_1 hello' }) + + await waitJobs(servers) + await checkCommentMention({ ...baseParams, shortUUID, threadId: commentId, commentId, byAccountDisplayName, checkType: 'absence' }) + + await servers[0].blocklist.removeFromMyBlocklist({ token: userToken, account: 'root' }) + }) + + it('Should not send a new mention notification if the remote account mention a local account', async function () { + this.timeout(30000) + + const { uuid, shortUUID } = await servers[0].videos.upload({ attributes: { name: 'super video' } }) + + await waitJobs(servers) + const { id: threadId } = await servers[1].comments.createThread({ videoId: uuid, text: '@user_1 hello' }) + + await waitJobs(servers) + + const byAccountDisplayName = 'super root 2 name' + await checkCommentMention({ ...baseParams, shortUUID, threadId, commentId: threadId, byAccountDisplayName, checkType: 'absence' }) + }) + + it('Should send a new mention notification after local comments', async function () { + this.timeout(30000) + + const { uuid, shortUUID } = await servers[0].videos.upload({ attributes: { name: 'super video' } }) + + const { id: threadId } = await servers[0].comments.createThread({ videoId: uuid, text: '@user_1 hellotext: 1' }) + + await waitJobs(servers) + await checkCommentMention({ ...baseParams, shortUUID, threadId, commentId: threadId, byAccountDisplayName, checkType: 'presence' }) + + const { id: commentId } = await servers[0].comments.addReply({ videoId: uuid, toCommentId: threadId, text: 'hello 2 @user_1' }) + + await waitJobs(servers) + await checkCommentMention({ ...baseParams, shortUUID, commentId, threadId, byAccountDisplayName, checkType: 'presence' }) + }) + + it('Should send a new mention notification after remote comments', async function () { + this.timeout(30000) + + const { uuid, shortUUID } = await servers[0].videos.upload({ attributes: { name: 'super video' } }) + + await waitJobs(servers) + + const text1 = `hello @user_1@${servers[0].host} 1` + const { id: server2ThreadId } = await servers[1].comments.createThread({ videoId: uuid, text: text1 }) + + await waitJobs(servers) + + const { data } = await servers[0].comments.listThreads({ videoId: uuid }) + expect(data).to.have.lengthOf(1) + + const byAccountDisplayName = 'super root 2 name' + const threadId = data[0].id + await checkCommentMention({ ...baseParams, shortUUID, commentId: threadId, threadId, byAccountDisplayName, checkType: 'presence' }) + + const text2 = `@user_1@${servers[0].host} hello 2 @root@${servers[0].host}` + await servers[1].comments.addReply({ videoId: uuid, toCommentId: server2ThreadId, text: text2 }) + + await waitJobs(servers) + + const tree = await servers[0].comments.getThread({ videoId: uuid, threadId }) + + expect(tree.children).to.have.lengthOf(1) + const commentId = tree.children[0].comment.id + + await checkCommentMention({ ...baseParams, shortUUID, commentId, threadId, byAccountDisplayName, checkType: 'presence' }) + }) + + it('Should convert markdown in comment to html', async function () { + this.timeout(30000) + + const { uuid } = await servers[0].videos.upload({ attributes: { name: 'super video' } }) + + const { id: threadId } = await servers[0].comments.createThread({ videoId: uuid, text: '@user_1 hello 1' }) + + await servers[0].comments.addReply({ videoId: uuid, toCommentId: threadId, text: '@user_1 ' + commentText }) + + await waitJobs(servers) + + const latestEmail = emails[emails.length - 1] + expect(latestEmail['html']).to.contain(expectedHtml) + }) + }) + + after(async function () { + MockSmtpServer.Instance.kill() + + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/notifications/index.ts b/packages/tests/src/api/notifications/index.ts new file mode 100644 index 000000000..d63d94182 --- /dev/null +++ b/packages/tests/src/api/notifications/index.ts @@ -0,0 +1,6 @@ +import './admin-notifications.js' +import './comments-notifications.js' +import './moderation-notifications.js' +import './notifications-api.js' +import './registrations-notifications.js' +import './user-notifications.js' diff --git a/packages/tests/src/api/notifications/moderation-notifications.ts b/packages/tests/src/api/notifications/moderation-notifications.ts new file mode 100644 index 000000000..493764882 --- /dev/null +++ b/packages/tests/src/api/notifications/moderation-notifications.ts @@ -0,0 +1,609 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { wait } from '@peertube/peertube-core-utils' +import { AbuseState, CustomConfig, UserNotification, UserRole, VideoPrivacy } from '@peertube/peertube-models' +import { buildUUID } from '@peertube/peertube-node-utils' +import { cleanupTests, PeerTubeServer, waitJobs } from '@peertube/peertube-server-commands' +import { MockSmtpServer } from '@tests/shared/mock-servers/mock-email.js' +import { MockInstancesIndex } from '@tests/shared/mock-servers/mock-instances-index.js' +import { + prepareNotificationsTest, + CheckerBaseParams, + checkNewVideoAbuseForModerators, + checkNewCommentAbuseForModerators, + checkNewAccountAbuseForModerators, + checkAbuseStateChange, + checkNewAbuseMessage, + checkNewBlacklistOnMyVideo, + checkNewInstanceFollower, + checkAutoInstanceFollowing, + checkVideoAutoBlacklistForModerators, + checkVideoIsPublished, + checkNewVideoFromSubscription +} from '@tests/shared/notifications.js' + +describe('Test moderation notifications', function () { + let servers: PeerTubeServer[] = [] + let userToken1: string + let userToken2: string + + let userNotifications: UserNotification[] = [] + let adminNotifications: UserNotification[] = [] + let adminNotificationsServer2: UserNotification[] = [] + let emails: object[] = [] + + before(async function () { + this.timeout(120000) + + const res = await prepareNotificationsTest(3) + emails = res.emails + userToken1 = res.userAccessToken + servers = res.servers + userNotifications = res.userNotifications + adminNotifications = res.adminNotifications + adminNotificationsServer2 = res.adminNotificationsServer2 + + userToken2 = await servers[1].users.generateUserAndToken('user2', UserRole.USER) + }) + + describe('Abuse for moderators notification', function () { + let baseParams: CheckerBaseParams + + before(() => { + baseParams = { + server: servers[0], + emails, + socketNotifications: adminNotifications, + token: servers[0].accessToken + } + }) + + it('Should not send a notification to moderators on local abuse reported by an admin', async function () { + this.timeout(50000) + + const name = 'video for abuse ' + buildUUID() + const video = await servers[0].videos.upload({ token: userToken1, attributes: { name } }) + + await servers[0].abuses.report({ videoId: video.id, reason: 'super reason' }) + + await waitJobs(servers) + await checkNewVideoAbuseForModerators({ ...baseParams, shortUUID: video.shortUUID, videoName: name, checkType: 'absence' }) + }) + + it('Should send a notification to moderators on local video abuse', async function () { + this.timeout(50000) + + const name = 'video for abuse ' + buildUUID() + const video = await servers[0].videos.upload({ token: userToken1, attributes: { name } }) + + await servers[0].abuses.report({ token: userToken1, videoId: video.id, reason: 'super reason' }) + + await waitJobs(servers) + await checkNewVideoAbuseForModerators({ ...baseParams, shortUUID: video.shortUUID, videoName: name, checkType: 'presence' }) + }) + + it('Should send a notification to moderators on remote video abuse', async function () { + this.timeout(50000) + + const name = 'video for abuse ' + buildUUID() + const video = await servers[0].videos.upload({ token: userToken1, attributes: { name } }) + + await waitJobs(servers) + + const videoId = await servers[1].videos.getId({ uuid: video.uuid }) + await servers[1].abuses.report({ token: userToken2, videoId, reason: 'super reason' }) + + await waitJobs(servers) + await checkNewVideoAbuseForModerators({ ...baseParams, shortUUID: video.shortUUID, videoName: name, checkType: 'presence' }) + }) + + it('Should send a notification to moderators on local comment abuse', async function () { + this.timeout(50000) + + const name = 'video for abuse ' + buildUUID() + const video = await servers[0].videos.upload({ token: userToken1, attributes: { name } }) + const comment = await servers[0].comments.createThread({ + token: userToken1, + videoId: video.id, + text: 'comment abuse ' + buildUUID() + }) + + await waitJobs(servers) + + await servers[0].abuses.report({ token: userToken1, commentId: comment.id, reason: 'super reason' }) + + await waitJobs(servers) + await checkNewCommentAbuseForModerators({ ...baseParams, shortUUID: video.shortUUID, videoName: name, checkType: 'presence' }) + }) + + it('Should send a notification to moderators on remote comment abuse', async function () { + this.timeout(50000) + + const name = 'video for abuse ' + buildUUID() + const video = await servers[0].videos.upload({ token: userToken1, attributes: { name } }) + + await servers[0].comments.createThread({ + token: userToken1, + videoId: video.id, + text: 'comment abuse ' + buildUUID() + }) + + await waitJobs(servers) + + const { data } = await servers[1].comments.listThreads({ videoId: video.uuid }) + const commentId = data[0].id + await servers[1].abuses.report({ token: userToken2, commentId, reason: 'super reason' }) + + await waitJobs(servers) + await checkNewCommentAbuseForModerators({ ...baseParams, shortUUID: video.shortUUID, videoName: name, checkType: 'presence' }) + }) + + it('Should send a notification to moderators on local account abuse', async function () { + this.timeout(50000) + + const username = 'user' + new Date().getTime() + const { account } = await servers[0].users.create({ username, password: 'donald' }) + const accountId = account.id + + await servers[0].abuses.report({ token: userToken1, accountId, reason: 'super reason' }) + + await waitJobs(servers) + await checkNewAccountAbuseForModerators({ ...baseParams, displayName: username, checkType: 'presence' }) + }) + + it('Should send a notification to moderators on remote account abuse', async function () { + this.timeout(50000) + + const username = 'user' + new Date().getTime() + const tmpToken = await servers[0].users.generateUserAndToken(username) + await servers[0].videos.upload({ token: tmpToken, attributes: { name: 'super video' } }) + + await waitJobs(servers) + + const account = await servers[1].accounts.get({ accountName: username + '@' + servers[0].host }) + await servers[1].abuses.report({ token: userToken2, accountId: account.id, reason: 'super reason' }) + + await waitJobs(servers) + await checkNewAccountAbuseForModerators({ ...baseParams, displayName: username, checkType: 'presence' }) + }) + }) + + describe('Abuse state change notification', function () { + let baseParams: CheckerBaseParams + let abuseId: number + + before(async function () { + baseParams = { + server: servers[0], + emails, + socketNotifications: userNotifications, + token: userToken1 + } + + const name = 'abuse ' + buildUUID() + const video = await servers[0].videos.upload({ token: userToken1, attributes: { name } }) + + const body = await servers[0].abuses.report({ token: userToken1, videoId: video.id, reason: 'super reason' }) + abuseId = body.abuse.id + }) + + it('Should send a notification to reporter if the abuse has been accepted', async function () { + this.timeout(30000) + + await servers[0].abuses.update({ abuseId, body: { state: AbuseState.ACCEPTED } }) + await waitJobs(servers) + + await checkAbuseStateChange({ ...baseParams, abuseId, state: AbuseState.ACCEPTED, checkType: 'presence' }) + }) + + it('Should send a notification to reporter if the abuse has been rejected', async function () { + this.timeout(30000) + + await servers[0].abuses.update({ abuseId, body: { state: AbuseState.REJECTED } }) + await waitJobs(servers) + + await checkAbuseStateChange({ ...baseParams, abuseId, state: AbuseState.REJECTED, checkType: 'presence' }) + }) + }) + + describe('New abuse message notification', function () { + let baseParamsUser: CheckerBaseParams + let baseParamsAdmin: CheckerBaseParams + let abuseId: number + let abuseId2: number + + before(async function () { + baseParamsUser = { + server: servers[0], + emails, + socketNotifications: userNotifications, + token: userToken1 + } + + baseParamsAdmin = { + server: servers[0], + emails, + socketNotifications: adminNotifications, + token: servers[0].accessToken + } + + const name = 'abuse ' + buildUUID() + const video = await servers[0].videos.upload({ token: userToken1, attributes: { name } }) + + { + const body = await servers[0].abuses.report({ token: userToken1, videoId: video.id, reason: 'super reason' }) + abuseId = body.abuse.id + } + + { + const body = await servers[0].abuses.report({ token: userToken1, videoId: video.id, reason: 'super reason 2' }) + abuseId2 = body.abuse.id + } + }) + + it('Should send a notification to reporter on new message', async function () { + this.timeout(30000) + + const message = 'my super message to users' + await servers[0].abuses.addMessage({ abuseId, message }) + await waitJobs(servers) + + await checkNewAbuseMessage({ ...baseParamsUser, abuseId, message, toEmail: 'user_1@example.com', checkType: 'presence' }) + }) + + it('Should not send a notification to the admin if sent by the admin', async function () { + this.timeout(30000) + + const message = 'my super message that should not be sent to the admin' + await servers[0].abuses.addMessage({ abuseId, message }) + await waitJobs(servers) + + const toEmail = 'admin' + servers[0].internalServerNumber + '@example.com' + await checkNewAbuseMessage({ ...baseParamsAdmin, abuseId, message, toEmail, checkType: 'absence' }) + }) + + it('Should send a notification to moderators', async function () { + this.timeout(30000) + + const message = 'my super message to moderators' + await servers[0].abuses.addMessage({ token: userToken1, abuseId: abuseId2, message }) + await waitJobs(servers) + + const toEmail = 'admin' + servers[0].internalServerNumber + '@example.com' + await checkNewAbuseMessage({ ...baseParamsAdmin, abuseId: abuseId2, message, toEmail, checkType: 'presence' }) + }) + + it('Should not send a notification to reporter if sent by the reporter', async function () { + this.timeout(30000) + + const message = 'my super message that should not be sent to reporter' + await servers[0].abuses.addMessage({ token: userToken1, abuseId: abuseId2, message }) + await waitJobs(servers) + + const toEmail = 'user_1@example.com' + await checkNewAbuseMessage({ ...baseParamsUser, abuseId: abuseId2, message, toEmail, checkType: 'absence' }) + }) + }) + + describe('Video blacklist on my video', function () { + let baseParams: CheckerBaseParams + + before(() => { + baseParams = { + server: servers[0], + emails, + socketNotifications: userNotifications, + token: userToken1 + } + }) + + it('Should send a notification to video owner on blacklist', async function () { + this.timeout(30000) + + const name = 'video for abuse ' + buildUUID() + const { uuid, shortUUID } = await servers[0].videos.upload({ token: userToken1, attributes: { name } }) + + await servers[0].blacklist.add({ videoId: uuid }) + + await waitJobs(servers) + await checkNewBlacklistOnMyVideo({ ...baseParams, shortUUID, videoName: name, blacklistType: 'blacklist' }) + }) + + it('Should send a notification to video owner on unblacklist', async function () { + this.timeout(30000) + + const name = 'video for abuse ' + buildUUID() + const { uuid, shortUUID } = await servers[0].videos.upload({ token: userToken1, attributes: { name } }) + + await servers[0].blacklist.add({ videoId: uuid }) + + await waitJobs(servers) + await servers[0].blacklist.remove({ videoId: uuid }) + await waitJobs(servers) + + await wait(500) + await checkNewBlacklistOnMyVideo({ ...baseParams, shortUUID, videoName: name, blacklistType: 'unblacklist' }) + }) + }) + + describe('New instance follows', function () { + const instanceIndexServer = new MockInstancesIndex() + let config: any + let baseParams: CheckerBaseParams + + before(async function () { + baseParams = { + server: servers[0], + emails, + socketNotifications: adminNotifications, + token: servers[0].accessToken + } + + const port = await instanceIndexServer.initialize() + instanceIndexServer.addInstance(servers[1].host) + + config = { + followings: { + instance: { + autoFollowIndex: { + indexUrl: `http://127.0.0.1:${port}/api/v1/instances/hosts`, + enabled: true + } + } + } + } + }) + + it('Should send a notification only to admin when there is a new instance follower', async function () { + this.timeout(60000) + + await servers[2].follows.follow({ hosts: [ servers[0].url ] }) + + await waitJobs(servers) + + await checkNewInstanceFollower({ ...baseParams, followerHost: servers[2].host, checkType: 'presence' }) + + const userOverride = { socketNotifications: userNotifications, token: userToken1, check: { web: true, mail: false } } + await checkNewInstanceFollower({ ...baseParams, ...userOverride, followerHost: servers[2].host, checkType: 'absence' }) + }) + + it('Should send a notification on auto follow back', async function () { + this.timeout(40000) + + await servers[2].follows.unfollow({ target: servers[0] }) + await waitJobs(servers) + + const config = { + followings: { + instance: { + autoFollowBack: { enabled: true } + } + } + } + await servers[0].config.updateCustomSubConfig({ newConfig: config }) + + await servers[2].follows.follow({ hosts: [ servers[0].url ] }) + + await waitJobs(servers) + + const followerHost = servers[0].host + const followingHost = servers[2].host + await checkAutoInstanceFollowing({ ...baseParams, followerHost, followingHost, checkType: 'presence' }) + + const userOverride = { socketNotifications: userNotifications, token: userToken1, check: { web: true, mail: false } } + await checkAutoInstanceFollowing({ ...baseParams, ...userOverride, followerHost, followingHost, checkType: 'absence' }) + + config.followings.instance.autoFollowBack.enabled = false + await servers[0].config.updateCustomSubConfig({ newConfig: config }) + await servers[0].follows.unfollow({ target: servers[2] }) + await servers[2].follows.unfollow({ target: servers[0] }) + }) + + it('Should send a notification on auto instances index follow', async function () { + this.timeout(30000) + await servers[0].follows.unfollow({ target: servers[1] }) + + await servers[0].config.updateCustomSubConfig({ newConfig: config }) + + await wait(5000) + await waitJobs(servers) + + const followerHost = servers[0].host + const followingHost = servers[1].host + await checkAutoInstanceFollowing({ ...baseParams, followerHost, followingHost, checkType: 'presence' }) + + config.followings.instance.autoFollowIndex.enabled = false + await servers[0].config.updateCustomSubConfig({ newConfig: config }) + await servers[0].follows.unfollow({ target: servers[1] }) + }) + }) + + describe('Video-related notifications when video auto-blacklist is enabled', function () { + let userBaseParams: CheckerBaseParams + let adminBaseParamsServer1: CheckerBaseParams + let adminBaseParamsServer2: CheckerBaseParams + let uuid: string + let shortUUID: string + let videoName: string + let currentCustomConfig: CustomConfig + + before(async function () { + + adminBaseParamsServer1 = { + server: servers[0], + emails, + socketNotifications: adminNotifications, + token: servers[0].accessToken + } + + adminBaseParamsServer2 = { + server: servers[1], + emails, + socketNotifications: adminNotificationsServer2, + token: servers[1].accessToken + } + + userBaseParams = { + server: servers[0], + emails, + socketNotifications: userNotifications, + token: userToken1 + } + + currentCustomConfig = await servers[0].config.getCustomConfig() + + const autoBlacklistTestsCustomConfig = { + ...currentCustomConfig, + + autoBlacklist: { + videos: { + ofUsers: { + enabled: true + } + } + } + } + + // enable transcoding otherwise own publish notification after transcoding not expected + autoBlacklistTestsCustomConfig.transcoding.enabled = true + await servers[0].config.updateCustomConfig({ newCustomConfig: autoBlacklistTestsCustomConfig }) + + await servers[0].subscriptions.add({ targetUri: 'user_1_channel@' + servers[0].host }) + await servers[1].subscriptions.add({ targetUri: 'user_1_channel@' + servers[0].host }) + }) + + it('Should send notification to moderators on new video with auto-blacklist', async function () { + this.timeout(120000) + + videoName = 'video with auto-blacklist ' + buildUUID() + const video = await servers[0].videos.upload({ token: userToken1, attributes: { name: videoName } }) + shortUUID = video.shortUUID + uuid = video.uuid + + await waitJobs(servers) + await checkVideoAutoBlacklistForModerators({ ...adminBaseParamsServer1, shortUUID, videoName, checkType: 'presence' }) + }) + + it('Should not send video publish notification if auto-blacklisted', async function () { + this.timeout(120000) + + await checkVideoIsPublished({ ...userBaseParams, videoName, shortUUID, checkType: 'absence' }) + }) + + it('Should not send a local user subscription notification if auto-blacklisted', async function () { + this.timeout(120000) + + await checkNewVideoFromSubscription({ ...adminBaseParamsServer1, videoName, shortUUID, checkType: 'absence' }) + }) + + it('Should not send a remote user subscription notification if auto-blacklisted', async function () { + await checkNewVideoFromSubscription({ ...adminBaseParamsServer2, videoName, shortUUID, checkType: 'absence' }) + }) + + it('Should send video published and unblacklist after video unblacklisted', async function () { + this.timeout(120000) + + await servers[0].blacklist.remove({ videoId: uuid }) + + await waitJobs(servers) + + // FIXME: Can't test as two notifications sent to same user and util only checks last one + // One notification might be better anyways + // await checkNewBlacklistOnMyVideo(userBaseParams, videoUUID, videoName, 'unblacklist') + // await checkVideoIsPublished(userBaseParams, videoName, videoUUID, 'presence') + }) + + it('Should send a local user subscription notification after removed from blacklist', async function () { + this.timeout(120000) + + await checkNewVideoFromSubscription({ ...adminBaseParamsServer1, videoName, shortUUID, checkType: 'presence' }) + }) + + it('Should send a remote user subscription notification after removed from blacklist', async function () { + this.timeout(120000) + + await checkNewVideoFromSubscription({ ...adminBaseParamsServer2, videoName, shortUUID, checkType: 'presence' }) + }) + + it('Should send unblacklist but not published/subscription notes after unblacklisted if scheduled update pending', async function () { + this.timeout(120000) + + const updateAt = new Date(new Date().getTime() + 1000000) + + const name = 'video with auto-blacklist and future schedule ' + buildUUID() + + const attributes = { + name, + privacy: VideoPrivacy.PRIVATE, + scheduleUpdate: { + updateAt: updateAt.toISOString(), + privacy: VideoPrivacy.PUBLIC + } + } + + const { shortUUID, uuid } = await servers[0].videos.upload({ token: userToken1, attributes }) + + await servers[0].blacklist.remove({ videoId: uuid }) + + await waitJobs(servers) + await checkNewBlacklistOnMyVideo({ ...userBaseParams, shortUUID, videoName: name, blacklistType: 'unblacklist' }) + + // FIXME: Can't test absence as two notifications sent to same user and util only checks last one + // One notification might be better anyways + // await checkVideoIsPublished(userBaseParams, name, uuid, 'absence') + + await checkNewVideoFromSubscription({ ...adminBaseParamsServer1, videoName: name, shortUUID, checkType: 'absence' }) + await checkNewVideoFromSubscription({ ...adminBaseParamsServer2, videoName: name, shortUUID, checkType: 'absence' }) + }) + + it('Should not send publish/subscription notifications after scheduled update if video still auto-blacklisted', async function () { + this.timeout(120000) + + // In 2 seconds + const updateAt = new Date(new Date().getTime() + 2000) + + const name = 'video with schedule done and still auto-blacklisted ' + buildUUID() + + const attributes = { + name, + privacy: VideoPrivacy.PRIVATE, + scheduleUpdate: { + updateAt: updateAt.toISOString(), + privacy: VideoPrivacy.PUBLIC + } + } + + const { shortUUID } = await servers[0].videos.upload({ token: userToken1, attributes }) + + await wait(6000) + await checkVideoIsPublished({ ...userBaseParams, videoName: name, shortUUID, checkType: 'absence' }) + await checkNewVideoFromSubscription({ ...adminBaseParamsServer1, videoName: name, shortUUID, checkType: 'absence' }) + await checkNewVideoFromSubscription({ ...adminBaseParamsServer2, videoName: name, shortUUID, checkType: 'absence' }) + }) + + it('Should not send a notification to moderators on new video without auto-blacklist', async function () { + this.timeout(120000) + + const name = 'video without auto-blacklist ' + buildUUID() + + // admin with blacklist right will not be auto-blacklisted + const { shortUUID } = await servers[0].videos.upload({ attributes: { name } }) + + await waitJobs(servers) + await checkVideoAutoBlacklistForModerators({ ...adminBaseParamsServer1, shortUUID, videoName: name, checkType: 'absence' }) + }) + + after(async () => { + await servers[0].config.updateCustomConfig({ newCustomConfig: currentCustomConfig }) + + await servers[0].subscriptions.remove({ uri: 'user_1_channel@' + servers[0].host }) + await servers[1].subscriptions.remove({ uri: 'user_1_channel@' + servers[0].host }) + }) + }) + + after(async function () { + MockSmtpServer.Instance.kill() + + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/notifications/notifications-api.ts b/packages/tests/src/api/notifications/notifications-api.ts new file mode 100644 index 000000000..1c7461553 --- /dev/null +++ b/packages/tests/src/api/notifications/notifications-api.ts @@ -0,0 +1,206 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { UserNotification, UserNotificationSettingValue } from '@peertube/peertube-models' +import { cleanupTests, PeerTubeServer, waitJobs } from '@peertube/peertube-server-commands' +import { MockSmtpServer } from '@tests/shared/mock-servers/mock-email.js' +import { + prepareNotificationsTest, + CheckerBaseParams, + getAllNotificationsSettings, + checkNewVideoFromSubscription +} from '@tests/shared/notifications.js' + +describe('Test notifications API', function () { + let server: PeerTubeServer + let userNotifications: UserNotification[] = [] + let userToken: string + let emails: object[] = [] + + before(async function () { + this.timeout(120000) + + const res = await prepareNotificationsTest(1) + emails = res.emails + userToken = res.userAccessToken + userNotifications = res.userNotifications + server = res.servers[0] + + await server.subscriptions.add({ token: userToken, targetUri: 'root_channel@' + server.host }) + + for (let i = 0; i < 10; i++) { + await server.videos.randomUpload({ wait: false }) + } + + await waitJobs([ server ]) + }) + + describe('Notification list & count', function () { + + it('Should correctly list notifications', async function () { + const { data, total } = await server.notifications.list({ token: userToken, start: 0, count: 2 }) + + expect(data).to.have.lengthOf(2) + expect(total).to.equal(10) + }) + }) + + describe('Mark as read', function () { + + it('Should mark as read some notifications', async function () { + const { data } = await server.notifications.list({ token: userToken, start: 2, count: 3 }) + const ids = data.map(n => n.id) + + await server.notifications.markAsRead({ token: userToken, ids }) + }) + + it('Should have the notifications marked as read', async function () { + const { data } = await server.notifications.list({ token: userToken, start: 0, count: 10 }) + + expect(data[0].read).to.be.false + expect(data[1].read).to.be.false + expect(data[2].read).to.be.true + expect(data[3].read).to.be.true + expect(data[4].read).to.be.true + expect(data[5].read).to.be.false + }) + + it('Should only list read notifications', async function () { + const { data } = await server.notifications.list({ token: userToken, start: 0, count: 10, unread: false }) + + for (const notification of data) { + expect(notification.read).to.be.true + } + }) + + it('Should only list unread notifications', async function () { + const { data } = await server.notifications.list({ token: userToken, start: 0, count: 10, unread: true }) + + for (const notification of data) { + expect(notification.read).to.be.false + } + }) + + it('Should mark as read all notifications', async function () { + await server.notifications.markAsReadAll({ token: userToken }) + + const body = await server.notifications.list({ token: userToken, start: 0, count: 10, unread: true }) + + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + }) + }) + + describe('Notification settings', function () { + let baseParams: CheckerBaseParams + + before(() => { + baseParams = { + server, + emails, + socketNotifications: userNotifications, + token: userToken + } + }) + + it('Should not have notifications', async function () { + this.timeout(40000) + + await server.notifications.updateMySettings({ + token: userToken, + settings: { ...getAllNotificationsSettings(), newVideoFromSubscription: UserNotificationSettingValue.NONE } + }) + + { + const info = await server.users.getMyInfo({ token: userToken }) + expect(info.notificationSettings.newVideoFromSubscription).to.equal(UserNotificationSettingValue.NONE) + } + + const { name, shortUUID } = await server.videos.randomUpload() + + const check = { web: true, mail: true } + await checkNewVideoFromSubscription({ ...baseParams, check, videoName: name, shortUUID, checkType: 'absence' }) + }) + + it('Should only have web notifications', async function () { + this.timeout(20000) + + await server.notifications.updateMySettings({ + token: userToken, + settings: { ...getAllNotificationsSettings(), newVideoFromSubscription: UserNotificationSettingValue.WEB } + }) + + { + const info = await server.users.getMyInfo({ token: userToken }) + expect(info.notificationSettings.newVideoFromSubscription).to.equal(UserNotificationSettingValue.WEB) + } + + const { name, shortUUID } = await server.videos.randomUpload() + + { + const check = { mail: true, web: false } + await checkNewVideoFromSubscription({ ...baseParams, check, videoName: name, shortUUID, checkType: 'absence' }) + } + + { + const check = { mail: false, web: true } + await checkNewVideoFromSubscription({ ...baseParams, check, videoName: name, shortUUID, checkType: 'presence' }) + } + }) + + it('Should only have mail notifications', async function () { + this.timeout(20000) + + await server.notifications.updateMySettings({ + token: userToken, + settings: { ...getAllNotificationsSettings(), newVideoFromSubscription: UserNotificationSettingValue.EMAIL } + }) + + { + const info = await server.users.getMyInfo({ token: userToken }) + expect(info.notificationSettings.newVideoFromSubscription).to.equal(UserNotificationSettingValue.EMAIL) + } + + const { name, shortUUID } = await server.videos.randomUpload() + + { + const check = { mail: false, web: true } + await checkNewVideoFromSubscription({ ...baseParams, check, videoName: name, shortUUID, checkType: 'absence' }) + } + + { + const check = { mail: true, web: false } + await checkNewVideoFromSubscription({ ...baseParams, check, videoName: name, shortUUID, checkType: 'presence' }) + } + }) + + it('Should have email and web notifications', async function () { + this.timeout(20000) + + await server.notifications.updateMySettings({ + token: userToken, + settings: { + ...getAllNotificationsSettings(), + newVideoFromSubscription: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL + } + }) + + { + const info = await server.users.getMyInfo({ token: userToken }) + expect(info.notificationSettings.newVideoFromSubscription).to.equal( + UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL + ) + } + + const { name, shortUUID } = await server.videos.randomUpload() + + await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) + }) + }) + + after(async function () { + MockSmtpServer.Instance.kill() + + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/notifications/registrations-notifications.ts b/packages/tests/src/api/notifications/registrations-notifications.ts new file mode 100644 index 000000000..1f166cb36 --- /dev/null +++ b/packages/tests/src/api/notifications/registrations-notifications.ts @@ -0,0 +1,83 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { UserNotification } from '@peertube/peertube-models' +import { cleanupTests, PeerTubeServer, waitJobs } from '@peertube/peertube-server-commands' +import { MockSmtpServer } from '@tests/shared/mock-servers/mock-email.js' +import { CheckerBaseParams, prepareNotificationsTest, checkUserRegistered, checkRegistrationRequest } from '@tests/shared/notifications.js' + +describe('Test registrations notifications', function () { + let server: PeerTubeServer + let userToken1: string + + let userNotifications: UserNotification[] = [] + let adminNotifications: UserNotification[] = [] + let emails: object[] = [] + + let baseParams: CheckerBaseParams + + before(async function () { + this.timeout(120000) + + const res = await prepareNotificationsTest(1) + + server = res.servers[0] + emails = res.emails + userToken1 = res.userAccessToken + adminNotifications = res.adminNotifications + userNotifications = res.userNotifications + + baseParams = { + server, + emails, + socketNotifications: adminNotifications, + token: server.accessToken + } + }) + + describe('New direct registration for moderators', function () { + + before(async function () { + await server.config.enableSignup(false) + }) + + it('Should send a notification only to moderators when a user registers on the instance', async function () { + this.timeout(50000) + + await server.registrations.register({ username: 'user_10' }) + + await waitJobs([ server ]) + + await checkUserRegistered({ ...baseParams, username: 'user_10', checkType: 'presence' }) + + const userOverride = { socketNotifications: userNotifications, token: userToken1, check: { web: true, mail: false } } + await checkUserRegistered({ ...baseParams, ...userOverride, username: 'user_10', checkType: 'absence' }) + }) + }) + + describe('New registration request for moderators', function () { + + before(async function () { + await server.config.enableSignup(true) + }) + + it('Should send a notification on new registration request', async function () { + this.timeout(50000) + + const registrationReason = 'my reason' + await server.registrations.requestRegistration({ username: 'user_11', registrationReason }) + + await waitJobs([ server ]) + + await checkRegistrationRequest({ ...baseParams, username: 'user_11', registrationReason, checkType: 'presence' }) + + const userOverride = { socketNotifications: userNotifications, token: userToken1, check: { web: true, mail: false } } + await checkRegistrationRequest({ ...baseParams, ...userOverride, username: 'user_11', registrationReason, checkType: 'absence' }) + }) + }) + + after(async function () { + MockSmtpServer.Instance.kill() + + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/notifications/user-notifications.ts b/packages/tests/src/api/notifications/user-notifications.ts new file mode 100644 index 000000000..4c03cdb47 --- /dev/null +++ b/packages/tests/src/api/notifications/user-notifications.ts @@ -0,0 +1,574 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { UserNotification, UserNotificationType, VideoPrivacy, VideoStudioTask } from '@peertube/peertube-models' +import { buildUUID } from '@peertube/peertube-node-utils' +import { cleanupTests, findExternalSavedVideo, PeerTubeServer, stopFfmpeg, waitJobs } from '@peertube/peertube-server-commands' +import { MockSmtpServer } from '@tests/shared/mock-servers/mock-email.js' +import { + prepareNotificationsTest, + CheckerBaseParams, + checkNewVideoFromSubscription, + checkVideoIsPublished, + checkVideoStudioEditionIsFinished, + checkMyVideoImportIsFinished, + checkNewActorFollow +} from '@tests/shared/notifications.js' +import { FIXTURE_URLS } from '@tests/shared/tests.js' +import { uploadRandomVideoOnServers } from '@tests/shared/videos.js' + +describe('Test user notifications', function () { + let servers: PeerTubeServer[] = [] + let userAccessToken: string + + let userNotifications: UserNotification[] = [] + let adminNotifications: UserNotification[] = [] + let adminNotificationsServer2: UserNotification[] = [] + let emails: object[] = [] + + let channelId: number + + before(async function () { + this.timeout(120000) + + const res = await prepareNotificationsTest(3) + emails = res.emails + userAccessToken = res.userAccessToken + servers = res.servers + userNotifications = res.userNotifications + adminNotifications = res.adminNotifications + adminNotificationsServer2 = res.adminNotificationsServer2 + channelId = res.channelId + }) + + describe('New video from my subscription notification', function () { + let baseParams: CheckerBaseParams + + before(() => { + baseParams = { + server: servers[0], + emails, + socketNotifications: userNotifications, + token: userAccessToken + } + }) + + it('Should not send notifications if the user does not follow the video publisher', async function () { + this.timeout(50000) + + await uploadRandomVideoOnServers(servers, 1) + + const notification = await servers[0].notifications.getLatest({ token: userAccessToken }) + expect(notification).to.be.undefined + + expect(emails).to.have.lengthOf(0) + expect(userNotifications).to.have.lengthOf(0) + }) + + it('Should send a new video notification if the user follows the local video publisher', async function () { + this.timeout(15000) + + await servers[0].subscriptions.add({ token: userAccessToken, targetUri: 'root_channel@' + servers[0].host }) + await waitJobs(servers) + + const { name, shortUUID } = await uploadRandomVideoOnServers(servers, 1) + await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) + }) + + it('Should send a new video notification from a remote account', async function () { + this.timeout(150000) // Server 2 has transcoding enabled + + await servers[0].subscriptions.add({ token: userAccessToken, targetUri: 'root_channel@' + servers[1].host }) + await waitJobs(servers) + + const { name, shortUUID } = await uploadRandomVideoOnServers(servers, 2) + await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) + }) + + it('Should send a new video notification on a scheduled publication', async function () { + this.timeout(50000) + + // In 2 seconds + const updateAt = new Date(new Date().getTime() + 2000) + + const data = { + privacy: VideoPrivacy.PRIVATE, + scheduleUpdate: { + updateAt: updateAt.toISOString(), + privacy: VideoPrivacy.PUBLIC + } + } + const { name, shortUUID } = await uploadRandomVideoOnServers(servers, 1, data) + + await wait(6000) + await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) + }) + + it('Should send a new video notification on a remote scheduled publication', async function () { + this.timeout(100000) + + // In 2 seconds + const updateAt = new Date(new Date().getTime() + 2000) + + const data = { + privacy: VideoPrivacy.PRIVATE, + scheduleUpdate: { + updateAt: updateAt.toISOString(), + privacy: VideoPrivacy.PUBLIC + } + } + const { name, shortUUID } = await uploadRandomVideoOnServers(servers, 2, data) + await waitJobs(servers) + + await wait(6000) + await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) + }) + + it('Should not send a notification before the video is published', async function () { + this.timeout(150000) + + const updateAt = new Date(new Date().getTime() + 1000000) + + const data = { + privacy: VideoPrivacy.PRIVATE, + scheduleUpdate: { + updateAt: updateAt.toISOString(), + privacy: VideoPrivacy.PUBLIC + } + } + const { name, shortUUID } = await uploadRandomVideoOnServers(servers, 1, data) + + await wait(6000) + await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'absence' }) + }) + + it('Should send a new video notification when a video becomes public', async function () { + this.timeout(50000) + + const data = { privacy: VideoPrivacy.PRIVATE } + const { name, uuid, shortUUID } = await uploadRandomVideoOnServers(servers, 1, data) + + await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'absence' }) + + await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } }) + + await waitJobs(servers) + await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) + }) + + it('Should send a new video notification when a remote video becomes public', async function () { + this.timeout(120000) + + const data = { privacy: VideoPrivacy.PRIVATE } + const { name, uuid, shortUUID } = await uploadRandomVideoOnServers(servers, 2, data) + + await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'absence' }) + + await servers[1].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } }) + + await waitJobs(servers) + await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) + }) + + it('Should not send a new video notification when a video becomes unlisted', async function () { + this.timeout(50000) + + const data = { privacy: VideoPrivacy.PRIVATE } + const { name, uuid, shortUUID } = await uploadRandomVideoOnServers(servers, 1, data) + + await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.UNLISTED } }) + + await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'absence' }) + }) + + it('Should not send a new video notification when a remote video becomes unlisted', async function () { + this.timeout(100000) + + const data = { privacy: VideoPrivacy.PRIVATE } + const { name, uuid, shortUUID } = await uploadRandomVideoOnServers(servers, 2, data) + + await servers[1].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.UNLISTED } }) + + await waitJobs(servers) + await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'absence' }) + }) + + it('Should send a new video notification after a video import', async function () { + this.timeout(100000) + + const name = 'video import ' + buildUUID() + + const attributes = { + name, + channelId, + privacy: VideoPrivacy.PUBLIC, + targetUrl: FIXTURE_URLS.goodVideo + } + const { video } = await servers[0].imports.importVideo({ attributes }) + + await waitJobs(servers) + + await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID: video.shortUUID, checkType: 'presence' }) + }) + }) + + describe('My video is published', function () { + let baseParams: CheckerBaseParams + + before(() => { + baseParams = { + server: servers[1], + emails, + socketNotifications: adminNotificationsServer2, + token: servers[1].accessToken + } + }) + + it('Should not send a notification if transcoding is not enabled', async function () { + this.timeout(50000) + + const { name, shortUUID } = await uploadRandomVideoOnServers(servers, 1) + await waitJobs(servers) + + await checkVideoIsPublished({ ...baseParams, videoName: name, shortUUID, checkType: 'absence' }) + }) + + it('Should not send a notification if the wait transcoding is false', async function () { + this.timeout(100_000) + + await uploadRandomVideoOnServers(servers, 2, { waitTranscoding: false }) + await waitJobs(servers) + + const notification = await servers[0].notifications.getLatest({ token: userAccessToken }) + if (notification) { + expect(notification.type).to.not.equal(UserNotificationType.MY_VIDEO_PUBLISHED) + } + }) + + it('Should send a notification even if the video is not transcoded in other resolutions', async function () { + this.timeout(100_000) + + const { name, shortUUID } = await uploadRandomVideoOnServers(servers, 2, { waitTranscoding: true, fixture: 'video_short_240p.mp4' }) + await waitJobs(servers) + + await checkVideoIsPublished({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) + }) + + it('Should send a notification with a transcoded video', async function () { + this.timeout(100_000) + + const { name, shortUUID } = await uploadRandomVideoOnServers(servers, 2, { waitTranscoding: true }) + await waitJobs(servers) + + await checkVideoIsPublished({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) + }) + + it('Should send a notification when an imported video is transcoded', async function () { + this.timeout(120000) + + const name = 'video import ' + buildUUID() + + const attributes = { + name, + channelId, + privacy: VideoPrivacy.PUBLIC, + targetUrl: FIXTURE_URLS.goodVideo, + waitTranscoding: true + } + const { video } = await servers[1].imports.importVideo({ attributes }) + + await waitJobs(servers) + await checkVideoIsPublished({ ...baseParams, videoName: name, shortUUID: video.shortUUID, checkType: 'presence' }) + }) + + it('Should send a notification when the scheduled update has been proceeded', async function () { + this.timeout(70000) + + // In 2 seconds + const updateAt = new Date(new Date().getTime() + 2000) + + const data = { + privacy: VideoPrivacy.PRIVATE, + scheduleUpdate: { + updateAt: updateAt.toISOString(), + privacy: VideoPrivacy.PUBLIC + } + } + const { name, shortUUID } = await uploadRandomVideoOnServers(servers, 2, data) + + await wait(6000) + await checkVideoIsPublished({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) + }) + + it('Should not send a notification before the video is published', async function () { + this.timeout(150000) + + const updateAt = new Date(new Date().getTime() + 1000000) + + const data = { + privacy: VideoPrivacy.PRIVATE, + scheduleUpdate: { + updateAt: updateAt.toISOString(), + privacy: VideoPrivacy.PUBLIC + } + } + const { name, shortUUID } = await uploadRandomVideoOnServers(servers, 2, data) + + await wait(6000) + await checkVideoIsPublished({ ...baseParams, videoName: name, shortUUID, checkType: 'absence' }) + }) + }) + + describe('My live replay is published', function () { + + let baseParams: CheckerBaseParams + + before(() => { + baseParams = { + server: servers[1], + emails, + socketNotifications: adminNotificationsServer2, + token: servers[1].accessToken + } + }) + + it('Should send a notification is a live replay of a non permanent live is published', async function () { + this.timeout(120000) + + const { shortUUID } = await servers[1].live.create({ + fields: { + name: 'non permanent live', + privacy: VideoPrivacy.PUBLIC, + channelId: servers[1].store.channel.id, + saveReplay: true, + replaySettings: { privacy: VideoPrivacy.PUBLIC }, + permanentLive: false + } + }) + + const ffmpegCommand = await servers[1].live.sendRTMPStreamInVideo({ videoId: shortUUID }) + + await waitJobs(servers) + await servers[1].live.waitUntilPublished({ videoId: shortUUID }) + + await stopFfmpeg(ffmpegCommand) + await servers[1].live.waitUntilReplacedByReplay({ videoId: shortUUID }) + + await waitJobs(servers) + await checkVideoIsPublished({ ...baseParams, videoName: 'non permanent live', shortUUID, checkType: 'presence' }) + }) + + it('Should send a notification is a live replay of a permanent live is published', async function () { + this.timeout(120000) + + const { shortUUID } = await servers[1].live.create({ + fields: { + name: 'permanent live', + privacy: VideoPrivacy.PUBLIC, + channelId: servers[1].store.channel.id, + saveReplay: true, + replaySettings: { privacy: VideoPrivacy.PUBLIC }, + permanentLive: true + } + }) + + const ffmpegCommand = await servers[1].live.sendRTMPStreamInVideo({ videoId: shortUUID }) + + await waitJobs(servers) + await servers[1].live.waitUntilPublished({ videoId: shortUUID }) + + const liveDetails = await servers[1].videos.get({ id: shortUUID }) + + await stopFfmpeg(ffmpegCommand) + + await servers[1].live.waitUntilWaiting({ videoId: shortUUID }) + await waitJobs(servers) + + const video = await findExternalSavedVideo(servers[1], liveDetails) + expect(video).to.exist + + await checkVideoIsPublished({ ...baseParams, videoName: video.name, shortUUID: video.shortUUID, checkType: 'presence' }) + }) + }) + + describe('Video studio', function () { + let baseParams: CheckerBaseParams + + before(() => { + baseParams = { + server: servers[1], + emails, + socketNotifications: adminNotificationsServer2, + token: servers[1].accessToken + } + }) + + it('Should send a notification after studio edition', async function () { + this.timeout(240000) + + const { name, shortUUID, id } = await uploadRandomVideoOnServers(servers, 2, { waitTranscoding: true }) + + await waitJobs(servers) + await checkVideoIsPublished({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) + + const tasks: VideoStudioTask[] = [ + { + name: 'cut', + options: { + start: 0, + end: 1 + } + } + ] + await servers[1].videoStudio.createEditionTasks({ videoId: id, tasks }) + await waitJobs(servers) + + await checkVideoStudioEditionIsFinished({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) + }) + }) + + describe('My video is imported', function () { + let baseParams: CheckerBaseParams + + before(() => { + baseParams = { + server: servers[0], + emails, + socketNotifications: adminNotifications, + token: servers[0].accessToken + } + }) + + it('Should send a notification when the video import failed', async function () { + this.timeout(70000) + + const name = 'video import ' + buildUUID() + + const attributes = { + name, + channelId, + privacy: VideoPrivacy.PRIVATE, + targetUrl: FIXTURE_URLS.badVideo + } + const { video: { shortUUID } } = await servers[0].imports.importVideo({ attributes }) + + await waitJobs(servers) + + const url = FIXTURE_URLS.badVideo + await checkMyVideoImportIsFinished({ ...baseParams, videoName: name, shortUUID, url, success: false, checkType: 'presence' }) + }) + + it('Should send a notification when the video import succeeded', async function () { + this.timeout(70000) + + const name = 'video import ' + buildUUID() + + const attributes = { + name, + channelId, + privacy: VideoPrivacy.PRIVATE, + targetUrl: FIXTURE_URLS.goodVideo + } + const { video: { shortUUID } } = await servers[0].imports.importVideo({ attributes }) + + await waitJobs(servers) + + const url = FIXTURE_URLS.goodVideo + await checkMyVideoImportIsFinished({ ...baseParams, videoName: name, shortUUID, url, success: true, checkType: 'presence' }) + }) + }) + + describe('New actor follow', function () { + let baseParams: CheckerBaseParams + const myChannelName = 'super channel name' + const myUserName = 'super user name' + + before(async function () { + baseParams = { + server: servers[0], + emails, + socketNotifications: userNotifications, + token: userAccessToken + } + + await servers[0].users.updateMe({ displayName: 'super root name' }) + + await servers[0].users.updateMe({ + token: userAccessToken, + displayName: myUserName + }) + + await servers[1].users.updateMe({ displayName: 'super root 2 name' }) + + await servers[0].channels.update({ + token: userAccessToken, + channelName: 'user_1_channel', + attributes: { displayName: myChannelName } + }) + }) + + it('Should notify when a local channel is following one of our channel', async function () { + this.timeout(50000) + + await servers[0].subscriptions.add({ targetUri: 'user_1_channel@' + servers[0].host }) + await waitJobs(servers) + + await checkNewActorFollow({ + ...baseParams, + followType: 'channel', + followerName: 'root', + followerDisplayName: 'super root name', + followingDisplayName: myChannelName, + checkType: 'presence' + }) + + await servers[0].subscriptions.remove({ uri: 'user_1_channel@' + servers[0].host }) + }) + + it('Should notify when a remote channel is following one of our channel', async function () { + this.timeout(50000) + + await servers[1].subscriptions.add({ targetUri: 'user_1_channel@' + servers[0].host }) + await waitJobs(servers) + + await checkNewActorFollow({ + ...baseParams, + followType: 'channel', + followerName: 'root', + followerDisplayName: 'super root 2 name', + followingDisplayName: myChannelName, + checkType: 'presence' + }) + + await servers[1].subscriptions.remove({ uri: 'user_1_channel@' + servers[0].host }) + }) + + // PeerTube does not support account -> account follows + // it('Should notify when a local account is following one of our channel', async function () { + // this.timeout(50000) + // + // await addUserSubscription(servers[0].url, servers[0].accessToken, 'user_1@' + servers[0].host) + // + // await waitJobs(servers) + // + // await checkNewActorFollow(baseParams, 'account', 'root', 'super root name', myUserName, 'presence') + // }) + + // it('Should notify when a remote account is following one of our channel', async function () { + // this.timeout(50000) + // + // await addUserSubscription(servers[1].url, servers[1].accessToken, 'user_1@' + servers[0].host) + // + // await waitJobs(servers) + // + // await checkNewActorFollow(baseParams, 'account', 'root', 'super root 2 name', myUserName, 'presence') + // }) + }) + + after(async function () { + MockSmtpServer.Instance.kill() + + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/object-storage/index.ts b/packages/tests/src/api/object-storage/index.ts new file mode 100644 index 000000000..51d2a29a0 --- /dev/null +++ b/packages/tests/src/api/object-storage/index.ts @@ -0,0 +1,4 @@ +export * from './live.js' +export * from './video-imports.js' +export * from './video-static-file-privacy.js' +export * from './videos.js' diff --git a/packages/tests/src/api/object-storage/live.ts b/packages/tests/src/api/object-storage/live.ts new file mode 100644 index 000000000..c8c214af5 --- /dev/null +++ b/packages/tests/src/api/object-storage/live.ts @@ -0,0 +1,314 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' +import { HttpStatusCode, LiveVideoCreate, VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + findExternalSavedVideo, + makeRawRequest, + ObjectStorageCommand, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + stopFfmpeg, + waitJobs, + waitUntilLivePublishedOnAllServers, + waitUntilLiveReplacedByReplayOnAllServers, + waitUntilLiveWaitingOnAllServers +} from '@peertube/peertube-server-commands' +import { expectStartWith } from '@tests/shared/checks.js' +import { testLiveVideoResolutions } from '@tests/shared/live.js' +import { MockObjectStorageProxy } from '@tests/shared/mock-servers/mock-object-storage.js' +import { SQLCommand } from '@tests/shared/sql-command.js' + +async function createLive (server: PeerTubeServer, permanent: boolean) { + const attributes: LiveVideoCreate = { + channelId: server.store.channel.id, + privacy: VideoPrivacy.PUBLIC, + name: 'my super live', + saveReplay: true, + replaySettings: { privacy: VideoPrivacy.PUBLIC }, + permanentLive: permanent + } + + const { uuid } = await server.live.create({ fields: attributes }) + + return uuid +} + +async function checkFilesExist (options: { + servers: PeerTubeServer[] + videoUUID: string + numberOfFiles: number + objectStorage: ObjectStorageCommand +}) { + const { servers, videoUUID, numberOfFiles, objectStorage } = options + + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + + expect(video.files).to.have.lengthOf(0) + expect(video.streamingPlaylists).to.have.lengthOf(1) + + const files = video.streamingPlaylists[0].files + expect(files).to.have.lengthOf(numberOfFiles) + + for (const file of files) { + expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl()) + + await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + } + } +} + +async function checkFilesCleanup (options: { + server: PeerTubeServer + videoUUID: string + resolutions: number[] + objectStorage: ObjectStorageCommand +}) { + const { server, videoUUID, resolutions, objectStorage } = options + + const resolutionFiles = resolutions.map((_value, i) => `${i}.m3u8`) + + for (const playlistName of [ 'master.m3u8' ].concat(resolutionFiles)) { + await server.live.getPlaylistFile({ + videoUUID, + playlistName, + expectedStatus: HttpStatusCode.NOT_FOUND_404, + objectStorage + }) + } + + await server.live.getSegmentFile({ + videoUUID, + playlistNumber: 0, + segment: 0, + objectStorage, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) +} + +describe('Object storage for lives', function () { + if (areMockObjectStorageTestsDisabled()) return + + let servers: PeerTubeServer[] + let sqlCommandServer1: SQLCommand + const objectStorage = new ObjectStorageCommand() + + before(async function () { + this.timeout(120000) + + await objectStorage.prepareDefaultMockBuckets() + servers = await createMultipleServers(2, objectStorage.getDefaultMockConfig()) + + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + await doubleFollow(servers[0], servers[1]) + + await servers[0].config.enableTranscoding() + + sqlCommandServer1 = new SQLCommand(servers[0]) + }) + + describe('Without live transcoding', function () { + let videoUUID: string + + before(async function () { + await servers[0].config.enableLive({ transcoding: false }) + + videoUUID = await createLive(servers[0], false) + }) + + it('Should create a live and publish it on object storage', async function () { + this.timeout(220000) + + const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID }) + await waitUntilLivePublishedOnAllServers(servers, videoUUID) + + await testLiveVideoResolutions({ + originServer: servers[0], + sqlCommand: sqlCommandServer1, + servers, + liveVideoId: videoUUID, + resolutions: [ 720 ], + transcoded: false, + objectStorage + }) + + await stopFfmpeg(ffmpegCommand) + }) + + it('Should have saved the replay on object storage', async function () { + this.timeout(220000) + + await waitUntilLiveReplacedByReplayOnAllServers(servers, videoUUID) + await waitJobs(servers) + + await checkFilesExist({ servers, videoUUID, numberOfFiles: 1, objectStorage }) + }) + + it('Should have cleaned up live files from object storage', async function () { + await checkFilesCleanup({ server: servers[0], videoUUID, resolutions: [ 720 ], objectStorage }) + }) + }) + + describe('With live transcoding', function () { + const resolutions = [ 720, 480, 360, 240, 144 ] + + before(async function () { + await servers[0].config.enableLive({ transcoding: true }) + }) + + describe('Normal replay', function () { + let videoUUIDNonPermanent: string + + before(async function () { + videoUUIDNonPermanent = await createLive(servers[0], false) + }) + + it('Should create a live and publish it on object storage', async function () { + this.timeout(240000) + + const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUIDNonPermanent }) + await waitUntilLivePublishedOnAllServers(servers, videoUUIDNonPermanent) + + await testLiveVideoResolutions({ + originServer: servers[0], + sqlCommand: sqlCommandServer1, + servers, + liveVideoId: videoUUIDNonPermanent, + resolutions, + transcoded: true, + objectStorage + }) + + await stopFfmpeg(ffmpegCommand) + }) + + it('Should have saved the replay on object storage', async function () { + this.timeout(220000) + + await waitUntilLiveReplacedByReplayOnAllServers(servers, videoUUIDNonPermanent) + await waitJobs(servers) + + await checkFilesExist({ servers, videoUUID: videoUUIDNonPermanent, numberOfFiles: 5, objectStorage }) + }) + + it('Should have cleaned up live files from object storage', async function () { + await checkFilesCleanup({ server: servers[0], videoUUID: videoUUIDNonPermanent, resolutions, objectStorage }) + }) + }) + + describe('Permanent replay', function () { + let videoUUIDPermanent: string + + before(async function () { + videoUUIDPermanent = await createLive(servers[0], true) + }) + + it('Should create a live and publish it on object storage', async function () { + this.timeout(240000) + + const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUIDPermanent }) + await waitUntilLivePublishedOnAllServers(servers, videoUUIDPermanent) + + await testLiveVideoResolutions({ + originServer: servers[0], + sqlCommand: sqlCommandServer1, + servers, + liveVideoId: videoUUIDPermanent, + resolutions, + transcoded: true, + objectStorage + }) + + await stopFfmpeg(ffmpegCommand) + }) + + it('Should have saved the replay on object storage', async function () { + this.timeout(220000) + + await waitUntilLiveWaitingOnAllServers(servers, videoUUIDPermanent) + await waitJobs(servers) + + const videoLiveDetails = await servers[0].videos.get({ id: videoUUIDPermanent }) + const replay = await findExternalSavedVideo(servers[0], videoLiveDetails) + + await checkFilesExist({ servers, videoUUID: replay.uuid, numberOfFiles: 5, objectStorage }) + }) + + it('Should have cleaned up live files from object storage', async function () { + await checkFilesCleanup({ server: servers[0], videoUUID: videoUUIDPermanent, resolutions, objectStorage }) + }) + }) + }) + + describe('With object storage base url', function () { + const mockObjectStorageProxy = new MockObjectStorageProxy() + let baseMockUrl: string + + before(async function () { + this.timeout(120000) + + const port = await mockObjectStorageProxy.initialize() + const bucketName = objectStorage.getMockStreamingPlaylistsBucketName() + baseMockUrl = `http://127.0.0.1:${port}/${bucketName}` + + await objectStorage.prepareDefaultMockBuckets() + + const config = { + object_storage: { + enabled: true, + endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(), + region: ObjectStorageCommand.getMockRegion(), + + credentials: ObjectStorageCommand.getMockCredentialsConfig(), + + streaming_playlists: { + bucket_name: bucketName, + prefix: '', + base_url: baseMockUrl + } + } + } + + await servers[0].kill() + await servers[0].run(config) + + await servers[0].config.enableLive({ transcoding: true, resolutions: 'min' }) + }) + + it('Should publish a live and replace the base url', async function () { + this.timeout(240000) + + const videoUUIDPermanent = await createLive(servers[0], true) + + const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUIDPermanent }) + await waitUntilLivePublishedOnAllServers(servers, videoUUIDPermanent) + + await testLiveVideoResolutions({ + originServer: servers[0], + sqlCommand: sqlCommandServer1, + servers, + liveVideoId: videoUUIDPermanent, + resolutions: [ 720 ], + transcoded: true, + objectStorage, + objectStorageBaseUrl: baseMockUrl + }) + + await stopFfmpeg(ffmpegCommand) + }) + }) + + after(async function () { + await sqlCommandServer1.cleanup() + await objectStorage.cleanupMock() + + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/object-storage/video-imports.ts b/packages/tests/src/api/object-storage/video-imports.ts new file mode 100644 index 000000000..43f769842 --- /dev/null +++ b/packages/tests/src/api/object-storage/video-imports.ts @@ -0,0 +1,112 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { expectStartWith } from '@tests/shared/checks.js' +import { FIXTURE_URLS } from '@tests/shared/tests.js' +import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' +import { HttpStatusCode, VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + makeRawRequest, + ObjectStorageCommand, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' + +async function importVideo (server: PeerTubeServer) { + const attributes = { + name: 'import 2', + privacy: VideoPrivacy.PUBLIC, + channelId: server.store.channel.id, + targetUrl: FIXTURE_URLS.goodVideo720 + } + + const { video: { uuid } } = await server.imports.importVideo({ attributes }) + + return uuid +} + +describe('Object storage for video import', function () { + if (areMockObjectStorageTestsDisabled()) return + + let server: PeerTubeServer + const objectStorage = new ObjectStorageCommand() + + before(async function () { + this.timeout(120000) + + await objectStorage.prepareDefaultMockBuckets() + + server = await createSingleServer(1, objectStorage.getDefaultMockConfig()) + + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + await server.config.enableImports() + }) + + describe('Without transcoding', async function () { + + before(async function () { + await server.config.disableTranscoding() + }) + + it('Should import a video and have sent it to object storage', async function () { + this.timeout(120000) + + const uuid = await importVideo(server) + await waitJobs(server) + + const video = await server.videos.get({ id: uuid }) + + expect(video.files).to.have.lengthOf(1) + expect(video.streamingPlaylists).to.have.lengthOf(0) + + const fileUrl = video.files[0].fileUrl + expectStartWith(fileUrl, objectStorage.getMockWebVideosBaseUrl()) + + await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('With transcoding', async function () { + + before(async function () { + await server.config.enableTranscoding() + }) + + it('Should import a video and have sent it to object storage', async function () { + this.timeout(120000) + + const uuid = await importVideo(server) + await waitJobs(server) + + const video = await server.videos.get({ id: uuid }) + + expect(video.files).to.have.lengthOf(5) + expect(video.streamingPlaylists).to.have.lengthOf(1) + expect(video.streamingPlaylists[0].files).to.have.lengthOf(5) + + for (const file of video.files) { + expectStartWith(file.fileUrl, objectStorage.getMockWebVideosBaseUrl()) + + await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + } + + for (const file of video.streamingPlaylists[0].files) { + expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl()) + + await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + } + }) + }) + + after(async function () { + await objectStorage.cleanupMock() + + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/object-storage/video-static-file-privacy.ts b/packages/tests/src/api/object-storage/video-static-file-privacy.ts new file mode 100644 index 000000000..cf6e9b4b9 --- /dev/null +++ b/packages/tests/src/api/object-storage/video-static-file-privacy.ts @@ -0,0 +1,573 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { basename } from 'path' +import { getAllFiles, getHLS } from '@peertube/peertube-core-utils' +import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@peertube/peertube-models' +import { areScalewayObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' +import { + cleanupTests, + createSingleServer, + findExternalSavedVideo, + makeRawRequest, + ObjectStorageCommand, + PeerTubeServer, + sendRTMPStream, + setAccessTokensToServers, + setDefaultVideoChannel, + stopFfmpeg, + waitJobs +} from '@peertube/peertube-server-commands' +import { expectStartWith } from '@tests/shared/checks.js' +import { SQLCommand } from '@tests/shared/sql-command.js' +import { checkVideoFileTokenReinjection } from '@tests/shared/streaming-playlists.js' + +function extractFilenameFromUrl (url: string) { + const parts = basename(url).split(':') + + return parts[parts.length - 1] +} + +describe('Object storage for video static file privacy', function () { + // We need real world object storage to check ACL + if (areScalewayObjectStorageTestsDisabled()) return + + let server: PeerTubeServer + let sqlCommand: SQLCommand + let userToken: string + + // --------------------------------------------------------------------------- + + async function checkPrivateVODFiles (uuid: string) { + const video = await server.videos.getWithToken({ id: uuid }) + + for (const file of video.files) { + expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/web-videos/private/') + + await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + } + + for (const file of getAllFiles(video)) { + const internalFileUrl = await sqlCommand.getInternalFileUrl(file.id) + expectStartWith(internalFileUrl, ObjectStorageCommand.getScalewayBaseUrl()) + await makeRawRequest({ url: internalFileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + } + + const hls = getHLS(video) + + if (hls) { + for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) { + expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') + } + + await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + + for (const file of hls.files) { + expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') + + await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + } + } + } + + async function checkPublicVODFiles (uuid: string) { + const video = await server.videos.getWithToken({ id: uuid }) + + for (const file of getAllFiles(video)) { + expectStartWith(file.fileUrl, ObjectStorageCommand.getScalewayBaseUrl()) + + await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + } + + const hls = getHLS(video) + + if (hls) { + expectStartWith(hls.playlistUrl, ObjectStorageCommand.getScalewayBaseUrl()) + expectStartWith(hls.segmentsSha256Url, ObjectStorageCommand.getScalewayBaseUrl()) + + await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) + } + } + + // --------------------------------------------------------------------------- + + before(async function () { + this.timeout(120000) + + server = await createSingleServer(1, ObjectStorageCommand.getDefaultScalewayConfig({ serverNumber: 1 })) + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + await server.config.enableMinimumTranscoding() + + userToken = await server.users.generateUserAndToken('user1') + + sqlCommand = new SQLCommand(server) + }) + + describe('VOD', function () { + let privateVideoUUID: string + let publicVideoUUID: string + let passwordProtectedVideoUUID: string + let userPrivateVideoUUID: string + + const correctPassword = 'my super password' + const correctPasswordHeader = { 'x-peertube-video-password': correctPassword } + const incorrectPasswordHeader = { 'x-peertube-video-password': correctPassword + 'toto' } + + // --------------------------------------------------------------------------- + + async function getSampleFileUrls (videoId: string) { + const video = await server.videos.getWithToken({ id: videoId }) + + return { + webVideoFile: video.files[0].fileUrl, + hlsFile: getHLS(video).files[0].fileUrl + } + } + + // --------------------------------------------------------------------------- + + it('Should upload a private video and have appropriate object storage ACL', async function () { + this.timeout(120000) + + { + const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) + privateVideoUUID = uuid + } + + { + const { uuid } = await server.videos.quickUpload({ name: 'user video', token: userToken, privacy: VideoPrivacy.PRIVATE }) + userPrivateVideoUUID = uuid + } + + await waitJobs([ server ]) + + await checkPrivateVODFiles(privateVideoUUID) + }) + + it('Should upload a password protected video and have appropriate object storage ACL', async function () { + this.timeout(120000) + + { + const { uuid } = await server.videos.quickUpload({ + name: 'video', + privacy: VideoPrivacy.PASSWORD_PROTECTED, + videoPasswords: [ correctPassword ] + }) + passwordProtectedVideoUUID = uuid + } + await waitJobs([ server ]) + + await checkPrivateVODFiles(passwordProtectedVideoUUID) + }) + + it('Should upload a public video and have appropriate object storage ACL', async function () { + this.timeout(120000) + + const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.UNLISTED }) + await waitJobs([ server ]) + + publicVideoUUID = uuid + + await checkPublicVODFiles(publicVideoUUID) + }) + + it('Should not get files without appropriate OAuth token', async function () { + this.timeout(60000) + + const { webVideoFile, hlsFile } = await getSampleFileUrls(privateVideoUUID) + + await makeRawRequest({ url: webVideoFile, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url: webVideoFile, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + + await makeRawRequest({ url: hlsFile, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url: hlsFile, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + }) + + it('Should not get files without appropriate password or appropriate OAuth token', async function () { + this.timeout(60000) + + const { webVideoFile, hlsFile } = await getSampleFileUrls(passwordProtectedVideoUUID) + + await makeRawRequest({ url: webVideoFile, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ + url: webVideoFile, + token: null, + headers: incorrectPasswordHeader, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + await makeRawRequest({ url: webVideoFile, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ + url: webVideoFile, + token: null, + headers: correctPasswordHeader, + expectedStatus: HttpStatusCode.OK_200 + }) + + await makeRawRequest({ url: hlsFile, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ + url: hlsFile, + token: null, + headers: incorrectPasswordHeader, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + await makeRawRequest({ url: hlsFile, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ + url: hlsFile, + token: null, + headers: correctPasswordHeader, + expectedStatus: HttpStatusCode.OK_200 + }) + }) + + it('Should not get HLS file of another video', async function () { + this.timeout(60000) + + const privateVideo = await server.videos.getWithToken({ id: privateVideoUUID }) + const hlsFilename = basename(getHLS(privateVideo).files[0].fileUrl) + + const badUrl = server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + userPrivateVideoUUID + '/' + hlsFilename + const goodUrl = server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + privateVideoUUID + '/' + hlsFilename + + await makeRawRequest({ url: badUrl, token: server.accessToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + await makeRawRequest({ url: goodUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + }) + + it('Should correctly check OAuth, video file token of private video', async function () { + this.timeout(60000) + + const badVideoFileToken = await server.videoToken.getVideoFileToken({ token: userToken, videoId: userPrivateVideoUUID }) + const goodVideoFileToken = await server.videoToken.getVideoFileToken({ videoId: privateVideoUUID }) + + const { webVideoFile, hlsFile } = await getSampleFileUrls(privateVideoUUID) + + for (const url of [ webVideoFile, hlsFile ]) { + await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + + await makeRawRequest({ url, query: { videoFileToken: badVideoFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url, query: { videoFileToken: goodVideoFileToken }, expectedStatus: HttpStatusCode.OK_200 }) + + } + }) + + it('Should correctly check OAuth, video file token or video password of password protected video', async function () { + this.timeout(60000) + + const badVideoFileToken = await server.videoToken.getVideoFileToken({ token: userToken, videoId: userPrivateVideoUUID }) + const goodVideoFileToken = await server.videoToken.getVideoFileToken({ + videoId: passwordProtectedVideoUUID, + videoPassword: correctPassword + }) + + const { webVideoFile, hlsFile } = await getSampleFileUrls(passwordProtectedVideoUUID) + + for (const url of [ hlsFile, webVideoFile ]) { + await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + + await makeRawRequest({ url, query: { videoFileToken: badVideoFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url, query: { videoFileToken: goodVideoFileToken }, expectedStatus: HttpStatusCode.OK_200 }) + + await makeRawRequest({ + url, + headers: incorrectPasswordHeader, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + await makeRawRequest({ url, headers: correctPasswordHeader, expectedStatus: HttpStatusCode.OK_200 }) + } + }) + + it('Should reinject video file token', async function () { + this.timeout(120000) + + const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: privateVideoUUID }) + + await checkVideoFileTokenReinjection({ + server, + videoUUID: privateVideoUUID, + videoFileToken, + resolutions: [ 240, 720 ], + isLive: false + }) + }) + + it('Should update public video to private', async function () { + this.timeout(60000) + + await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.INTERNAL } }) + + await checkPrivateVODFiles(publicVideoUUID) + }) + + it('Should update private video to public', async function () { + this.timeout(60000) + + await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.PUBLIC } }) + + await checkPublicVODFiles(publicVideoUUID) + }) + }) + + describe('Live', function () { + let normalLiveId: string + let normalLive: LiveVideo + + let permanentLiveId: string + let permanentLive: LiveVideo + + let passwordProtectedLiveId: string + let passwordProtectedLive: LiveVideo + + const correctPassword = 'my super password' + + let unrelatedFileToken: string + + // --------------------------------------------------------------------------- + + async function checkLiveFiles (live: LiveVideo, liveId: string, videoPassword?: string) { + const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) + await server.live.waitUntilPublished({ videoId: liveId }) + + const video = videoPassword + ? await server.videos.getWithPassword({ id: liveId, password: videoPassword }) + : await server.videos.getWithToken({ id: liveId }) + + const fileToken = videoPassword + ? await server.videoToken.getVideoFileToken({ token: null, videoId: video.uuid, videoPassword }) + : await server.videoToken.getVideoFileToken({ videoId: video.uuid }) + + const hls = video.streamingPlaylists[0] + + for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) { + expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') + + await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + + await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) + if (videoPassword) { + await makeRawRequest({ url, headers: { 'x-peertube-video-password': videoPassword }, expectedStatus: HttpStatusCode.OK_200 }) + } + await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + if (videoPassword) { + await makeRawRequest({ + url, + headers: { 'x-peertube-video-password': 'incorrectPassword' }, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + } + } + + await stopFfmpeg(ffmpegCommand) + } + + async function checkReplay (replay: VideoDetails) { + const fileToken = await server.videoToken.getVideoFileToken({ videoId: replay.uuid }) + + const hls = replay.streamingPlaylists[0] + expect(hls.files).to.not.have.lengthOf(0) + + for (const file of hls.files) { + await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url: file.fileUrl, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) + + await makeRawRequest({ url: file.fileUrl, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ + url: file.fileUrl, + query: { videoFileToken: unrelatedFileToken }, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + } + + for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) { + expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') + + await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) + + await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + } + } + + // --------------------------------------------------------------------------- + + before(async function () { + await server.config.enableMinimumTranscoding() + + const { uuid } = await server.videos.quickUpload({ name: 'another video' }) + unrelatedFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid }) + + await server.config.enableLive({ + allowReplay: true, + transcoding: true, + resolutions: 'min' + }) + + { + const { video, live } = await server.live.quickCreate({ + saveReplay: true, + permanentLive: false, + privacy: VideoPrivacy.PRIVATE + }) + normalLiveId = video.uuid + normalLive = live + } + + { + const { video, live } = await server.live.quickCreate({ + saveReplay: true, + permanentLive: true, + privacy: VideoPrivacy.PRIVATE + }) + permanentLiveId = video.uuid + permanentLive = live + } + + { + const { video, live } = await server.live.quickCreate({ + saveReplay: false, + permanentLive: false, + privacy: VideoPrivacy.PASSWORD_PROTECTED, + videoPasswords: [ correctPassword ] + }) + passwordProtectedLiveId = video.uuid + passwordProtectedLive = live + } + }) + + it('Should create a private normal live and have a private static path', async function () { + this.timeout(240000) + + await checkLiveFiles(normalLive, normalLiveId) + }) + + it('Should create a private permanent live and have a private static path', async function () { + this.timeout(240000) + + await checkLiveFiles(permanentLive, permanentLiveId) + }) + + it('Should create a password protected live and have a private static path', async function () { + this.timeout(240000) + + await checkLiveFiles(passwordProtectedLive, passwordProtectedLiveId, correctPassword) + }) + + it('Should reinject video file token in permanent live', async function () { + this.timeout(240000) + + const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: permanentLive.rtmpUrl, streamKey: permanentLive.streamKey }) + await server.live.waitUntilPublished({ videoId: permanentLiveId }) + + const video = await server.videos.getWithToken({ id: permanentLiveId }) + const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid }) + + await checkVideoFileTokenReinjection({ + server, + videoUUID: permanentLiveId, + videoFileToken, + resolutions: [ 720 ], + isLive: true + }) + + await stopFfmpeg(ffmpegCommand) + }) + + it('Should have created a replay of the normal live with a private static path', async function () { + this.timeout(240000) + + await server.live.waitUntilReplacedByReplay({ videoId: normalLiveId }) + + const replay = await server.videos.getWithToken({ id: normalLiveId }) + await checkReplay(replay) + }) + + it('Should have created a replay of the permanent live with a private static path', async function () { + this.timeout(240000) + + await server.live.waitUntilWaiting({ videoId: permanentLiveId }) + await waitJobs([ server ]) + + const live = await server.videos.getWithToken({ id: permanentLiveId }) + const replayFromList = await findExternalSavedVideo(server, live) + const replay = await server.videos.getWithToken({ id: replayFromList.id }) + + await checkReplay(replay) + }) + }) + + describe('With private files proxy disabled and public ACL for private files', function () { + let videoUUID: string + + before(async function () { + this.timeout(240000) + + await server.kill() + + const config = ObjectStorageCommand.getDefaultScalewayConfig({ + serverNumber: 1, + enablePrivateProxy: false, + privateACL: 'public-read' + }) + await server.run(config) + + const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) + videoUUID = uuid + + await waitJobs([ server ]) + }) + + it('Should display object storage path for a private video and be able to access them', async function () { + this.timeout(60000) + + await checkPublicVODFiles(videoUUID) + }) + + it('Should not be able to access object storage proxy', async function () { + const privateVideo = await server.videos.getWithToken({ id: videoUUID }) + const webVideoFilename = extractFilenameFromUrl(privateVideo.files[0].fileUrl) + const hlsFilename = extractFilenameFromUrl(getHLS(privateVideo).files[0].fileUrl) + + await makeRawRequest({ + url: server.url + '/object-storage-proxy/web-videos/private/' + webVideoFilename, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + + await makeRawRequest({ + url: server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + videoUUID + '/' + hlsFilename, + token: server.accessToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + }) + + after(async function () { + this.timeout(240000) + + const { data } = await server.videos.listAllForAdmin() + + for (const v of data) { + await server.videos.remove({ id: v.uuid }) + } + + for (const v of data) { + await server.servers.waitUntilLog('Removed files of video ' + v.url) + } + + await sqlCommand.cleanup() + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/object-storage/videos.ts b/packages/tests/src/api/object-storage/videos.ts new file mode 100644 index 000000000..66bca5cc8 --- /dev/null +++ b/packages/tests/src/api/object-storage/videos.ts @@ -0,0 +1,434 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import bytes from 'bytes' +import { expect } from 'chai' +import { stat } from 'fs/promises' +import merge from 'lodash-es/merge.js' +import { HttpStatusCode, VideoDetails } from '@peertube/peertube-models' +import { areMockObjectStorageTestsDisabled, sha1 } from '@peertube/peertube-node-utils' +import { + cleanupTests, + createMultipleServers, + createSingleServer, + doubleFollow, + killallServers, + makeRawRequest, + ObjectStorageCommand, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' +import { expectStartWith, expectLogDoesNotContain } from '@tests/shared/checks.js' +import { checkTmpIsEmpty } from '@tests/shared/directories.js' +import { generateHighBitrateVideo } from '@tests/shared/generate.js' +import { MockObjectStorageProxy } from '@tests/shared/mock-servers/mock-object-storage.js' +import { SQLCommand } from '@tests/shared/sql-command.js' +import { checkWebTorrentWorks } from '@tests/shared/webtorrent.js' + +async function checkFiles (options: { + server: PeerTubeServer + originServer: PeerTubeServer + originSQLCommand: SQLCommand + + video: VideoDetails + + baseMockUrl?: string + + playlistBucket: string + playlistPrefix?: string + + webVideoBucket: string + webVideoPrefix?: string +}) { + const { + server, + originServer, + originSQLCommand, + video, + playlistBucket, + webVideoBucket, + baseMockUrl, + playlistPrefix, + webVideoPrefix + } = options + + let allFiles = video.files + + for (const file of video.files) { + const baseUrl = baseMockUrl + ? `${baseMockUrl}/${webVideoBucket}/` + : `http://${webVideoBucket}.${ObjectStorageCommand.getMockEndpointHost()}/` + + const prefix = webVideoPrefix || '' + const start = baseUrl + prefix + + expectStartWith(file.fileUrl, start) + + const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 }) + const location = res.headers['location'] + expectStartWith(location, start) + + await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 }) + } + + const hls = video.streamingPlaylists[0] + + if (hls) { + allFiles = allFiles.concat(hls.files) + + const baseUrl = baseMockUrl + ? `${baseMockUrl}/${playlistBucket}/` + : `http://${playlistBucket}.${ObjectStorageCommand.getMockEndpointHost()}/` + + const prefix = playlistPrefix || '' + const start = baseUrl + prefix + + expectStartWith(hls.playlistUrl, start) + expectStartWith(hls.segmentsSha256Url, start) + + await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) + + const resSha = await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) + expect(JSON.stringify(resSha.body)).to.not.throw + + let i = 0 + for (const file of hls.files) { + expectStartWith(file.fileUrl, start) + + const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 }) + const location = res.headers['location'] + expectStartWith(location, start) + + await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 }) + + if (originServer.internalServerNumber === server.internalServerNumber) { + const infohash = sha1(`${2 + hls.playlistUrl}+V${i}`) + const dbInfohashes = await originSQLCommand.getPlaylistInfohash(hls.id) + + expect(dbInfohashes).to.include(infohash) + } + + i++ + } + } + + for (const file of allFiles) { + await checkWebTorrentWorks(file.magnetUri) + + const res = await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + expect(res.body).to.have.length.above(100) + } + + return allFiles.map(f => f.fileUrl) +} + +function runTestSuite (options: { + fixture?: string + + maxUploadPart?: string + + playlistBucket: string + playlistPrefix?: string + + webVideoBucket: string + webVideoPrefix?: string + + useMockBaseUrl?: boolean +}) { + const mockObjectStorageProxy = new MockObjectStorageProxy() + const { fixture } = options + let baseMockUrl: string + + let servers: PeerTubeServer[] + let sqlCommands: SQLCommand[] = [] + const objectStorage = new ObjectStorageCommand() + + let keptUrls: string[] = [] + + const uuidsToDelete: string[] = [] + let deletedUrls: string[] = [] + + before(async function () { + this.timeout(240000) + + const port = await mockObjectStorageProxy.initialize() + baseMockUrl = options.useMockBaseUrl + ? `http://127.0.0.1:${port}` + : undefined + + await objectStorage.createMockBucket(options.playlistBucket) + await objectStorage.createMockBucket(options.webVideoBucket) + + const config = { + object_storage: { + enabled: true, + endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(), + region: ObjectStorageCommand.getMockRegion(), + + credentials: ObjectStorageCommand.getMockCredentialsConfig(), + + max_upload_part: options.maxUploadPart || '5MB', + + streaming_playlists: { + bucket_name: options.playlistBucket, + prefix: options.playlistPrefix, + base_url: baseMockUrl + ? `${baseMockUrl}/${options.playlistBucket}` + : undefined + }, + + web_videos: { + bucket_name: options.webVideoBucket, + prefix: options.webVideoPrefix, + base_url: baseMockUrl + ? `${baseMockUrl}/${options.webVideoBucket}` + : undefined + } + } + } + + servers = await createMultipleServers(2, config) + + await setAccessTokensToServers(servers) + await doubleFollow(servers[0], servers[1]) + + for (const server of servers) { + const { uuid } = await server.videos.quickUpload({ name: 'video to keep' }) + await waitJobs(servers) + + const files = await server.videos.listFiles({ id: uuid }) + keptUrls = keptUrls.concat(files.map(f => f.fileUrl)) + } + + sqlCommands = servers.map(s => new SQLCommand(s)) + }) + + it('Should upload a video and move it to the object storage without transcoding', async function () { + this.timeout(40000) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1', fixture }) + uuidsToDelete.push(uuid) + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + const files = await checkFiles({ ...options, server, originServer: servers[0], originSQLCommand: sqlCommands[0], video, baseMockUrl }) + + deletedUrls = deletedUrls.concat(files) + } + }) + + it('Should upload a video and move it to the object storage with transcoding', async function () { + this.timeout(120000) + + const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2', fixture }) + uuidsToDelete.push(uuid) + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + const files = await checkFiles({ ...options, server, originServer: servers[0], originSQLCommand: sqlCommands[0], video, baseMockUrl }) + + deletedUrls = deletedUrls.concat(files) + } + }) + + it('Should fetch correctly all the files', async function () { + for (const url of deletedUrls.concat(keptUrls)) { + await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) + } + }) + + it('Should correctly delete the files', async function () { + await servers[0].videos.remove({ id: uuidsToDelete[0] }) + await servers[1].videos.remove({ id: uuidsToDelete[1] }) + + await waitJobs(servers) + + for (const url of deletedUrls) { + await makeRawRequest({ url, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + } + }) + + it('Should have kept other files', async function () { + for (const url of keptUrls) { + await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) + } + }) + + it('Should have an empty tmp directory', async function () { + for (const server of servers) { + await checkTmpIsEmpty(server) + } + }) + + it('Should not have downloaded files from object storage', async function () { + for (const server of servers) { + await expectLogDoesNotContain(server, 'from object storage') + } + }) + + after(async function () { + await mockObjectStorageProxy.terminate() + await objectStorage.cleanupMock() + + for (const sqlCommand of sqlCommands) { + await sqlCommand.cleanup() + } + + await cleanupTests(servers) + }) +} + +describe('Object storage for videos', function () { + if (areMockObjectStorageTestsDisabled()) return + + const objectStorage = new ObjectStorageCommand() + + describe('Test config', function () { + let server: PeerTubeServer + + const baseConfig = objectStorage.getDefaultMockConfig() + + const badCredentials = { + access_key_id: 'AKIAIOSFODNN7EXAMPLE', + secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' + } + + it('Should fail with same bucket names without prefix', function (done) { + const config = merge({}, baseConfig, { + object_storage: { + streaming_playlists: { + bucket_name: 'aaa' + }, + + web_videos: { + bucket_name: 'aaa' + } + } + }) + + createSingleServer(1, config) + .then(() => done(new Error('Did not throw'))) + .catch(() => done()) + }) + + it('Should fail with bad credentials', async function () { + this.timeout(60000) + + await objectStorage.prepareDefaultMockBuckets() + + const config = merge({}, baseConfig, { + object_storage: { + credentials: badCredentials + } + }) + + server = await createSingleServer(1, config) + await setAccessTokensToServers([ server ]) + + const { uuid } = await server.videos.quickUpload({ name: 'video' }) + + await waitJobs([ server ], { skipDelayed: true }) + const video = await server.videos.get({ id: uuid }) + + expectStartWith(video.files[0].fileUrl, server.url) + + await killallServers([ server ]) + }) + + it('Should succeed with credentials from env', async function () { + this.timeout(60000) + + await objectStorage.prepareDefaultMockBuckets() + + const config = merge({}, baseConfig, { + object_storage: { + credentials: { + access_key_id: '', + secret_access_key: '' + } + } + }) + + const goodCredentials = ObjectStorageCommand.getMockCredentialsConfig() + + server = await createSingleServer(1, config, { + env: { + AWS_ACCESS_KEY_ID: goodCredentials.access_key_id, + AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key + } + }) + + await setAccessTokensToServers([ server ]) + + const { uuid } = await server.videos.quickUpload({ name: 'video' }) + + await waitJobs([ server ], { skipDelayed: true }) + const video = await server.videos.get({ id: uuid }) + + expectStartWith(video.files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl()) + }) + + after(async function () { + await objectStorage.cleanupMock() + + await cleanupTests([ server ]) + }) + }) + + describe('Test simple object storage', function () { + runTestSuite({ + playlistBucket: objectStorage.getMockBucketName('streaming-playlists'), + webVideoBucket: objectStorage.getMockBucketName('web-videos') + }) + }) + + describe('Test object storage with prefix', function () { + runTestSuite({ + playlistBucket: objectStorage.getMockBucketName('mybucket'), + webVideoBucket: objectStorage.getMockBucketName('mybucket'), + + playlistPrefix: 'streaming-playlists_', + webVideoPrefix: 'webvideo_' + }) + }) + + describe('Test object storage with prefix and base URL', function () { + runTestSuite({ + playlistBucket: objectStorage.getMockBucketName('mybucket'), + webVideoBucket: objectStorage.getMockBucketName('mybucket'), + + playlistPrefix: 'streaming-playlists/', + webVideoPrefix: 'webvideo/', + + useMockBaseUrl: true + }) + }) + + describe('Test object storage with file bigger than upload part', function () { + let fixture: string + const maxUploadPart = '5MB' + + before(async function () { + this.timeout(120000) + + fixture = await generateHighBitrateVideo() + + const { size } = await stat(fixture) + + if (bytes.parse(maxUploadPart) > size) { + throw Error(`Fixture file is too small (${size}) to make sense for this test.`) + } + }) + + runTestSuite({ + maxUploadPart, + playlistBucket: objectStorage.getMockBucketName('streaming-playlists'), + webVideoBucket: objectStorage.getMockBucketName('web-videos'), + fixture + }) + }) +}) diff --git a/packages/tests/src/api/redundancy/index.ts b/packages/tests/src/api/redundancy/index.ts new file mode 100644 index 000000000..f6b70c8af --- /dev/null +++ b/packages/tests/src/api/redundancy/index.ts @@ -0,0 +1,3 @@ +import './redundancy-constraints.js' +import './redundancy.js' +import './manage-redundancy.js' diff --git a/packages/tests/src/api/redundancy/manage-redundancy.ts b/packages/tests/src/api/redundancy/manage-redundancy.ts new file mode 100644 index 000000000..14556e26c --- /dev/null +++ b/packages/tests/src/api/redundancy/manage-redundancy.ts @@ -0,0 +1,324 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + RedundancyCommand, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' +import { VideoPrivacy, VideoRedundanciesTarget } from '@peertube/peertube-models' + +describe('Test manage videos redundancy', function () { + const targets: VideoRedundanciesTarget[] = [ 'my-videos', 'remote-videos' ] + + let servers: PeerTubeServer[] + let video1Server2UUID: string + let video2Server2UUID: string + let redundanciesToRemove: number[] = [] + + let commands: RedundancyCommand[] + + before(async function () { + this.timeout(120000) + + const config = { + transcoding: { + hls: { + enabled: true + } + }, + redundancy: { + videos: { + check_interval: '1 second', + strategies: [ + { + strategy: 'recently-added', + min_lifetime: '1 hour', + size: '10MB', + min_views: 0 + } + ] + } + } + } + servers = await createMultipleServers(3, config) + + // Get the access tokens + await setAccessTokensToServers(servers) + + commands = servers.map(s => s.redundancy) + + { + const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video 1 server 2' } }) + video1Server2UUID = uuid + } + + { + const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video 2 server 2' } }) + video2Server2UUID = uuid + } + + await waitJobs(servers) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + await doubleFollow(servers[0], servers[2]) + await commands[0].updateRedundancy({ host: servers[1].host, redundancyAllowed: true }) + + await waitJobs(servers) + }) + + it('Should not have redundancies on server 3', async function () { + for (const target of targets) { + const body = await commands[2].listVideos({ target }) + + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + } + }) + + it('Should correctly list followings by redundancy', async function () { + const body = await servers[0].follows.getFollowings({ sort: '-redundancyAllowed' }) + + expect(body.total).to.equal(2) + expect(body.data).to.have.lengthOf(2) + + expect(body.data[0].following.host).to.equal(servers[1].host) + expect(body.data[1].following.host).to.equal(servers[2].host) + }) + + it('Should not have "remote-videos" redundancies on server 2', async function () { + this.timeout(120000) + + await waitJobs(servers) + await servers[0].servers.waitUntilLog('Duplicated ', 10) + await waitJobs(servers) + + const body = await commands[1].listVideos({ target: 'remote-videos' }) + + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + }) + + it('Should have "my-videos" redundancies on server 2', async function () { + this.timeout(120000) + + const body = await commands[1].listVideos({ target: 'my-videos' }) + expect(body.total).to.equal(2) + + const videos = body.data + expect(videos).to.have.lengthOf(2) + + const videos1 = videos.find(v => v.uuid === video1Server2UUID) + const videos2 = videos.find(v => v.uuid === video2Server2UUID) + + expect(videos1.name).to.equal('video 1 server 2') + expect(videos2.name).to.equal('video 2 server 2') + + expect(videos1.redundancies.files).to.have.lengthOf(4) + expect(videos1.redundancies.streamingPlaylists).to.have.lengthOf(1) + + const redundancies = videos1.redundancies.files.concat(videos1.redundancies.streamingPlaylists) + + for (const r of redundancies) { + expect(r.strategy).to.be.null + expect(r.fileUrl).to.exist + expect(r.createdAt).to.exist + expect(r.updatedAt).to.exist + expect(r.expiresOn).to.exist + } + }) + + it('Should not have "my-videos" redundancies on server 1', async function () { + const body = await commands[0].listVideos({ target: 'my-videos' }) + + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + }) + + it('Should have "remote-videos" redundancies on server 1', async function () { + this.timeout(120000) + + const body = await commands[0].listVideos({ target: 'remote-videos' }) + expect(body.total).to.equal(2) + + const videos = body.data + expect(videos).to.have.lengthOf(2) + + const videos1 = videos.find(v => v.uuid === video1Server2UUID) + const videos2 = videos.find(v => v.uuid === video2Server2UUID) + + expect(videos1.name).to.equal('video 1 server 2') + expect(videos2.name).to.equal('video 2 server 2') + + expect(videos1.redundancies.files).to.have.lengthOf(4) + expect(videos1.redundancies.streamingPlaylists).to.have.lengthOf(1) + + const redundancies = videos1.redundancies.files.concat(videos1.redundancies.streamingPlaylists) + + for (const r of redundancies) { + expect(r.strategy).to.equal('recently-added') + expect(r.fileUrl).to.exist + expect(r.createdAt).to.exist + expect(r.updatedAt).to.exist + expect(r.expiresOn).to.exist + } + }) + + it('Should correctly paginate and sort results', async function () { + { + const body = await commands[0].listVideos({ + target: 'remote-videos', + sort: 'name', + start: 0, + count: 2 + }) + + const videos = body.data + expect(videos[0].name).to.equal('video 1 server 2') + expect(videos[1].name).to.equal('video 2 server 2') + } + + { + const body = await commands[0].listVideos({ + target: 'remote-videos', + sort: '-name', + start: 0, + count: 2 + }) + + const videos = body.data + expect(videos[0].name).to.equal('video 2 server 2') + expect(videos[1].name).to.equal('video 1 server 2') + } + + { + const body = await commands[0].listVideos({ + target: 'remote-videos', + sort: '-name', + start: 1, + count: 1 + }) + + expect(body.data[0].name).to.equal('video 1 server 2') + } + }) + + it('Should manually add a redundancy and list it', async function () { + this.timeout(120000) + + const uuid = (await servers[1].videos.quickUpload({ name: 'video 3 server 2', privacy: VideoPrivacy.UNLISTED })).uuid + await waitJobs(servers) + const videoId = await servers[0].videos.getId({ uuid }) + + await commands[0].addVideo({ videoId }) + + await waitJobs(servers) + await servers[0].servers.waitUntilLog('Duplicated ', 15) + await waitJobs(servers) + + { + const body = await commands[0].listVideos({ + target: 'remote-videos', + sort: '-name', + start: 0, + count: 5 + }) + + const video = body.data[0] + + expect(video.name).to.equal('video 3 server 2') + expect(video.redundancies.files).to.have.lengthOf(4) + expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1) + + const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists) + + for (const r of redundancies) { + redundanciesToRemove.push(r.id) + + expect(r.strategy).to.equal('manual') + expect(r.fileUrl).to.exist + expect(r.createdAt).to.exist + expect(r.updatedAt).to.exist + expect(r.expiresOn).to.be.null + } + } + + const body = await commands[1].listVideos({ + target: 'my-videos', + sort: '-name', + start: 0, + count: 5 + }) + + const video = body.data[0] + expect(video.name).to.equal('video 3 server 2') + expect(video.redundancies.files).to.have.lengthOf(4) + expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1) + + const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists) + + for (const r of redundancies) { + expect(r.strategy).to.be.null + expect(r.fileUrl).to.exist + expect(r.createdAt).to.exist + expect(r.updatedAt).to.exist + expect(r.expiresOn).to.be.null + } + }) + + it('Should manually remove a redundancy and remove it from the list', async function () { + this.timeout(120000) + + for (const redundancyId of redundanciesToRemove) { + await commands[0].removeVideo({ redundancyId }) + } + + { + const body = await commands[0].listVideos({ + target: 'remote-videos', + sort: '-name', + start: 0, + count: 5 + }) + + const videos = body.data + + expect(videos).to.have.lengthOf(2) + + const video = videos[0] + expect(video.name).to.equal('video 2 server 2') + expect(video.redundancies.files).to.have.lengthOf(4) + expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1) + + const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists) + + redundanciesToRemove = redundancies.map(r => r.id) + } + }) + + it('Should remove another (auto) redundancy', async function () { + for (const redundancyId of redundanciesToRemove) { + await commands[0].removeVideo({ redundancyId }) + } + + const body = await commands[0].listVideos({ + target: 'remote-videos', + sort: '-name', + start: 0, + count: 5 + }) + + const videos = body.data + expect(videos).to.have.lengthOf(1) + expect(videos[0].name).to.equal('video 1 server 2') + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/redundancy/redundancy-constraints.ts b/packages/tests/src/api/redundancy/redundancy-constraints.ts new file mode 100644 index 000000000..24966b270 --- /dev/null +++ b/packages/tests/src/api/redundancy/redundancy-constraints.ts @@ -0,0 +1,191 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + killallServers, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test redundancy constraints', function () { + let remoteServer: PeerTubeServer + let localServer: PeerTubeServer + let servers: PeerTubeServer[] + + const remoteServerConfig = { + redundancy: { + videos: { + check_interval: '1 second', + strategies: [ + { + strategy: 'recently-added', + min_lifetime: '1 hour', + size: '100MB', + min_views: 0 + } + ] + } + } + } + + async function uploadWrapper (videoName: string) { + // Wait for transcoding + const { id } = await localServer.videos.upload({ attributes: { name: 'to transcode', privacy: VideoPrivacy.PRIVATE } }) + await waitJobs([ localServer ]) + + // Update video to schedule a federation + await localServer.videos.update({ id, attributes: { name: videoName, privacy: VideoPrivacy.PUBLIC } }) + } + + async function getTotalRedundanciesLocalServer () { + const body = await localServer.redundancy.listVideos({ target: 'my-videos' }) + + return body.total + } + + async function getTotalRedundanciesRemoteServer () { + const body = await remoteServer.redundancy.listVideos({ target: 'remote-videos' }) + + return body.total + } + + before(async function () { + this.timeout(120000) + + { + remoteServer = await createSingleServer(1, remoteServerConfig) + } + + { + const config = { + remote_redundancy: { + videos: { + accept_from: 'nobody' + } + } + } + localServer = await createSingleServer(2, config) + } + + servers = [ remoteServer, localServer ] + + // Get the access tokens + await setAccessTokensToServers(servers) + + await localServer.videos.upload({ attributes: { name: 'video 1 server 2' } }) + + await waitJobs(servers) + + // Server 1 and server 2 follow each other + await remoteServer.follows.follow({ hosts: [ localServer.url ] }) + await waitJobs(servers) + await remoteServer.redundancy.updateRedundancy({ host: localServer.host, redundancyAllowed: true }) + + await waitJobs(servers) + }) + + it('Should have redundancy on server 1 but not on server 2 with a nobody filter', async function () { + this.timeout(120000) + + await waitJobs(servers) + await remoteServer.servers.waitUntilLog('Duplicated ', 5) + await waitJobs(servers) + + { + const total = await getTotalRedundanciesRemoteServer() + expect(total).to.equal(1) + } + + { + const total = await getTotalRedundanciesLocalServer() + expect(total).to.equal(0) + } + }) + + it('Should have redundancy on server 1 and on server 2 with an anybody filter', async function () { + this.timeout(120000) + + const config = { + remote_redundancy: { + videos: { + accept_from: 'anybody' + } + } + } + await killallServers([ localServer ]) + await localServer.run(config) + + await uploadWrapper('video 2 server 2') + + await remoteServer.servers.waitUntilLog('Duplicated ', 10) + await waitJobs(servers) + + { + const total = await getTotalRedundanciesRemoteServer() + expect(total).to.equal(2) + } + + { + const total = await getTotalRedundanciesLocalServer() + expect(total).to.equal(1) + } + }) + + it('Should have redundancy on server 1 but not on server 2 with a followings filter', async function () { + this.timeout(120000) + + const config = { + remote_redundancy: { + videos: { + accept_from: 'followings' + } + } + } + await killallServers([ localServer ]) + await localServer.run(config) + + await uploadWrapper('video 3 server 2') + + await remoteServer.servers.waitUntilLog('Duplicated ', 15) + await waitJobs(servers) + + { + const total = await getTotalRedundanciesRemoteServer() + expect(total).to.equal(3) + } + + { + const total = await getTotalRedundanciesLocalServer() + expect(total).to.equal(1) + } + }) + + it('Should have redundancy on server 1 and on server 2 with followings filter now server 2 follows server 1', async function () { + this.timeout(120000) + + await localServer.follows.follow({ hosts: [ remoteServer.url ] }) + await waitJobs(servers) + + await uploadWrapper('video 4 server 2') + await remoteServer.servers.waitUntilLog('Duplicated ', 20) + await waitJobs(servers) + + { + const total = await getTotalRedundanciesRemoteServer() + expect(total).to.equal(4) + } + + { + const total = await getTotalRedundanciesLocalServer() + expect(total).to.equal(2) + } + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/redundancy/redundancy.ts b/packages/tests/src/api/redundancy/redundancy.ts new file mode 100644 index 000000000..69afae037 --- /dev/null +++ b/packages/tests/src/api/redundancy/redundancy.ts @@ -0,0 +1,743 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { readdir } from 'fs/promises' +import { decode as magnetUriDecode } from 'magnet-uri' +import { basename, join } from 'path' +import { wait } from '@peertube/peertube-core-utils' +import { + HttpStatusCode, + VideoDetails, + VideoFile, + VideoPrivacy, + VideoRedundancyStrategy, + VideoRedundancyStrategyWithManual +} from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + killallServers, + makeRawRequest, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' +import { checkSegmentHash } from '@tests/shared/streaming-playlists.js' +import { checkVideoFilesWereRemoved, saveVideoInServers } from '@tests/shared/videos.js' + +let servers: PeerTubeServer[] = [] +let video1Server2: VideoDetails + +async function checkMagnetWebseeds (file: VideoFile, baseWebseeds: string[], server: PeerTubeServer) { + const parsed = magnetUriDecode(file.magnetUri) + + for (const ws of baseWebseeds) { + const found = parsed.urlList.find(url => url === `${ws}${basename(file.fileUrl)}`) + expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined + } + + expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) + + for (const url of parsed.urlList) { + await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) + } +} + +async function createServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebVideo = true) { + const strategies: any[] = [] + + if (strategy !== null) { + strategies.push( + { + min_lifetime: '1 hour', + strategy, + size: '400KB', + + ...additionalParams + } + ) + } + + const config = { + transcoding: { + web_videos: { + enabled: withWebVideo + }, + hls: { + enabled: true + } + }, + redundancy: { + videos: { + check_interval: '5 seconds', + strategies + } + } + } + + servers = await createMultipleServers(3, config) + + // Get the access tokens + await setAccessTokensToServers(servers) + + { + const { id } = await servers[1].videos.upload({ attributes: { name: 'video 1 server 2' } }) + video1Server2 = await servers[1].videos.get({ id }) + + await servers[1].views.simulateView({ id }) + } + + await waitJobs(servers) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + // Server 1 and server 3 follow each other + await doubleFollow(servers[0], servers[2]) + // Server 2 and server 3 follow each other + await doubleFollow(servers[1], servers[2]) + + await waitJobs(servers) +} + +async function ensureSameFilenames (videoUUID: string) { + let webVideoFilenames: string[] + let hlsFilenames: string[] + + for (const server of servers) { + const video = await server.videos.getWithToken({ id: videoUUID }) + + // Ensure we use the same filenames that the origin + + const localWebVideoFilenames = video.files.map(f => basename(f.fileUrl)).sort() + const localHLSFilenames = video.streamingPlaylists[0].files.map(f => basename(f.fileUrl)).sort() + + if (webVideoFilenames) expect(webVideoFilenames).to.deep.equal(localWebVideoFilenames) + else webVideoFilenames = localWebVideoFilenames + + if (hlsFilenames) expect(hlsFilenames).to.deep.equal(localHLSFilenames) + else hlsFilenames = localHLSFilenames + } + + return { webVideoFilenames, hlsFilenames } +} + +async function check1WebSeed (videoUUID?: string) { + if (!videoUUID) videoUUID = video1Server2.uuid + + const webseeds = [ + `${servers[1].url}/static/web-videos/` + ] + + for (const server of servers) { + // With token to avoid issues with video follow constraints + const video = await server.videos.getWithToken({ id: videoUUID }) + + for (const f of video.files) { + await checkMagnetWebseeds(f, webseeds, server) + } + } + + await ensureSameFilenames(videoUUID) +} + +async function check2Webseeds (videoUUID?: string) { + if (!videoUUID) videoUUID = video1Server2.uuid + + const webseeds = [ + `${servers[0].url}/static/redundancy/`, + `${servers[1].url}/static/web-videos/` + ] + + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + + for (const file of video.files) { + await checkMagnetWebseeds(file, webseeds, server) + } + } + + const { webVideoFilenames } = await ensureSameFilenames(videoUUID) + + const directories = [ + servers[0].getDirectoryPath('redundancy'), + servers[1].getDirectoryPath('web-videos') + ] + + for (const directory of directories) { + const files = await readdir(directory) + expect(files).to.have.length.at.least(4) + + // Ensure we files exist on disk + expect(files.find(f => webVideoFilenames.includes(f))).to.exist + } +} + +async function check0PlaylistRedundancies (videoUUID?: string) { + if (!videoUUID) videoUUID = video1Server2.uuid + + for (const server of servers) { + // With token to avoid issues with video follow constraints + const video = await server.videos.getWithToken({ id: videoUUID }) + + expect(video.streamingPlaylists).to.be.an('array') + expect(video.streamingPlaylists).to.have.lengthOf(1) + expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0) + } + + await ensureSameFilenames(videoUUID) +} + +async function check1PlaylistRedundancies (videoUUID?: string) { + if (!videoUUID) videoUUID = video1Server2.uuid + + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + + expect(video.streamingPlaylists).to.have.lengthOf(1) + expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1) + + const redundancy = video.streamingPlaylists[0].redundancies[0] + + expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID) + } + + const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls/' + videoUUID + const baseUrlSegment = servers[0].url + '/static/redundancy/hls/' + videoUUID + + const video = await servers[0].videos.get({ id: videoUUID }) + const hlsPlaylist = video.streamingPlaylists[0] + + for (const resolution of [ 240, 360, 480, 720 ]) { + await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, resolution, hlsPlaylist }) + } + + const { hlsFilenames } = await ensureSameFilenames(videoUUID) + + const directories = [ + servers[0].getDirectoryPath('redundancy/hls'), + servers[1].getDirectoryPath('streaming-playlists/hls') + ] + + for (const directory of directories) { + const files = await readdir(join(directory, videoUUID)) + expect(files).to.have.length.at.least(4) + + // Ensure we files exist on disk + expect(files.find(f => hlsFilenames.includes(f))).to.exist + } +} + +async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) { + let totalSize: number = null + let statsLength = 1 + + if (strategy !== 'manual') { + totalSize = 409600 + statsLength = 2 + } + + const data = await servers[0].stats.get() + expect(data.videosRedundancy).to.have.lengthOf(statsLength) + + const stat = data.videosRedundancy[0] + expect(stat.strategy).to.equal(strategy) + expect(stat.totalSize).to.equal(totalSize) + + return stat +} + +async function checkStatsWith1Redundancy (strategy: VideoRedundancyStrategyWithManual, onlyHls = false) { + const stat = await checkStatsGlobal(strategy) + + expect(stat.totalUsed).to.be.at.least(1).and.below(409601) + expect(stat.totalVideoFiles).to.equal(onlyHls ? 4 : 8) + expect(stat.totalVideos).to.equal(1) +} + +async function checkStatsWithoutRedundancy (strategy: VideoRedundancyStrategyWithManual) { + const stat = await checkStatsGlobal(strategy) + + expect(stat.totalUsed).to.equal(0) + expect(stat.totalVideoFiles).to.equal(0) + expect(stat.totalVideos).to.equal(0) +} + +async function findServerFollows () { + const body = await servers[0].follows.getFollowings({ start: 0, count: 5, sort: '-createdAt' }) + const follows = body.data + const server2 = follows.find(f => f.following.host === `${servers[1].host}`) + const server3 = follows.find(f => f.following.host === `${servers[2].host}`) + + return { server2, server3 } +} + +async function enableRedundancyOnServer1 () { + await servers[0].redundancy.updateRedundancy({ host: servers[1].host, redundancyAllowed: true }) + + const { server2, server3 } = await findServerFollows() + + expect(server3).to.not.be.undefined + expect(server3.following.hostRedundancyAllowed).to.be.false + + expect(server2).to.not.be.undefined + expect(server2.following.hostRedundancyAllowed).to.be.true +} + +async function disableRedundancyOnServer1 () { + await servers[0].redundancy.updateRedundancy({ host: servers[1].host, redundancyAllowed: false }) + + const { server2, server3 } = await findServerFollows() + + expect(server3).to.not.be.undefined + expect(server3.following.hostRedundancyAllowed).to.be.false + + expect(server2).to.not.be.undefined + expect(server2.following.hostRedundancyAllowed).to.be.false +} + +describe('Test videos redundancy', function () { + + describe('With most-views strategy', function () { + const strategy = 'most-views' + + before(function () { + this.timeout(240000) + + return createServers(strategy) + }) + + it('Should have 1 webseed on the first video', async function () { + await check1WebSeed() + await check0PlaylistRedundancies() + await checkStatsWithoutRedundancy(strategy) + }) + + it('Should enable redundancy on server 1', function () { + return enableRedundancyOnServer1() + }) + + it('Should have 2 webseeds on the first video', async function () { + this.timeout(80000) + + await waitJobs(servers) + await servers[0].servers.waitUntilLog('Duplicated ', 5) + await waitJobs(servers) + + await check2Webseeds() + await check1PlaylistRedundancies() + await checkStatsWith1Redundancy(strategy) + }) + + it('Should undo redundancy on server 1 and remove duplicated videos', async function () { + this.timeout(80000) + + await disableRedundancyOnServer1() + + await waitJobs(servers) + await wait(5000) + + await check1WebSeed() + await check0PlaylistRedundancies() + + await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true }) + }) + + after(async function () { + return cleanupTests(servers) + }) + }) + + describe('With trending strategy', function () { + const strategy = 'trending' + + before(function () { + this.timeout(240000) + + return createServers(strategy) + }) + + it('Should have 1 webseed on the first video', async function () { + await check1WebSeed() + await check0PlaylistRedundancies() + await checkStatsWithoutRedundancy(strategy) + }) + + it('Should enable redundancy on server 1', function () { + return enableRedundancyOnServer1() + }) + + it('Should have 2 webseeds on the first video', async function () { + this.timeout(80000) + + await waitJobs(servers) + await servers[0].servers.waitUntilLog('Duplicated ', 5) + await waitJobs(servers) + + await check2Webseeds() + await check1PlaylistRedundancies() + await checkStatsWith1Redundancy(strategy) + }) + + it('Should unfollow server 3 and keep duplicated videos', async function () { + this.timeout(80000) + + await servers[0].follows.unfollow({ target: servers[2] }) + + await waitJobs(servers) + await wait(5000) + + await check2Webseeds() + await check1PlaylistRedundancies() + await checkStatsWith1Redundancy(strategy) + }) + + it('Should unfollow server 2 and remove duplicated videos', async function () { + this.timeout(80000) + + await servers[0].follows.unfollow({ target: servers[1] }) + + await waitJobs(servers) + await wait(5000) + + await check1WebSeed() + await check0PlaylistRedundancies() + + await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true }) + }) + + after(async function () { + await cleanupTests(servers) + }) + }) + + describe('With recently added strategy', function () { + const strategy = 'recently-added' + + before(function () { + this.timeout(240000) + + return createServers(strategy, { min_views: 3 }) + }) + + it('Should have 1 webseed on the first video', async function () { + await check1WebSeed() + await check0PlaylistRedundancies() + await checkStatsWithoutRedundancy(strategy) + }) + + it('Should enable redundancy on server 1', function () { + return enableRedundancyOnServer1() + }) + + it('Should still have 1 webseed on the first video', async function () { + this.timeout(80000) + + await waitJobs(servers) + await wait(15000) + await waitJobs(servers) + + await check1WebSeed() + await check0PlaylistRedundancies() + await checkStatsWithoutRedundancy(strategy) + }) + + it('Should view 2 times the first video to have > min_views config', async function () { + this.timeout(80000) + + await servers[0].views.simulateView({ id: video1Server2.uuid }) + await servers[2].views.simulateView({ id: video1Server2.uuid }) + + await wait(10000) + await waitJobs(servers) + }) + + it('Should have 2 webseeds on the first video', async function () { + this.timeout(80000) + + await waitJobs(servers) + await servers[0].servers.waitUntilLog('Duplicated ', 5) + await waitJobs(servers) + + await check2Webseeds() + await check1PlaylistRedundancies() + await checkStatsWith1Redundancy(strategy) + }) + + it('Should remove the video and the redundancy files', async function () { + this.timeout(20000) + + await saveVideoInServers(servers, video1Server2.uuid) + await servers[1].videos.remove({ id: video1Server2.uuid }) + + await waitJobs(servers) + + for (const server of servers) { + await checkVideoFilesWereRemoved({ server, video: server.store.videoDetails }) + } + }) + + after(async function () { + await cleanupTests(servers) + }) + }) + + describe('With only HLS files', function () { + const strategy = 'recently-added' + + before(async function () { + this.timeout(240000) + + await createServers(strategy, { min_views: 3 }, false) + }) + + it('Should have 0 playlist redundancy on the first video', async function () { + await check1WebSeed() + await check0PlaylistRedundancies() + }) + + it('Should enable redundancy on server 1', function () { + return enableRedundancyOnServer1() + }) + + it('Should still have 0 redundancy on the first video', async function () { + this.timeout(80000) + + await waitJobs(servers) + await wait(15000) + await waitJobs(servers) + + await check0PlaylistRedundancies() + await checkStatsWithoutRedundancy(strategy) + }) + + it('Should have 1 redundancy on the first video', async function () { + this.timeout(160000) + + await servers[0].views.simulateView({ id: video1Server2.uuid }) + await servers[2].views.simulateView({ id: video1Server2.uuid }) + + await wait(10000) + await waitJobs(servers) + + await waitJobs(servers) + await servers[0].servers.waitUntilLog('Duplicated ', 1) + await waitJobs(servers) + + await check1PlaylistRedundancies() + await checkStatsWith1Redundancy(strategy, true) + }) + + it('Should remove the video and the redundancy files', async function () { + this.timeout(20000) + + await saveVideoInServers(servers, video1Server2.uuid) + await servers[1].videos.remove({ id: video1Server2.uuid }) + + await waitJobs(servers) + + for (const server of servers) { + await checkVideoFilesWereRemoved({ server, video: server.store.videoDetails }) + } + }) + + after(async function () { + await cleanupTests(servers) + }) + }) + + describe('With manual strategy', function () { + before(function () { + this.timeout(240000) + + return createServers(null) + }) + + it('Should have 1 webseed on the first video', async function () { + await check1WebSeed() + await check0PlaylistRedundancies() + await checkStatsWithoutRedundancy('manual') + }) + + it('Should create a redundancy on first video', async function () { + await servers[0].redundancy.addVideo({ videoId: video1Server2.id }) + }) + + it('Should have 2 webseeds on the first video', async function () { + this.timeout(80000) + + await waitJobs(servers) + await servers[0].servers.waitUntilLog('Duplicated ', 5) + await waitJobs(servers) + + await check2Webseeds() + await check1PlaylistRedundancies() + await checkStatsWith1Redundancy('manual') + }) + + it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () { + this.timeout(80000) + + const body = await servers[0].redundancy.listVideos({ target: 'remote-videos' }) + + const videos = body.data + expect(videos).to.have.lengthOf(1) + + const video = videos[0] + + for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) { + await servers[0].redundancy.removeVideo({ redundancyId: r.id }) + } + + await waitJobs(servers) + await wait(5000) + + await check1WebSeed() + await check0PlaylistRedundancies() + + await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true }) + }) + + after(async function () { + await cleanupTests(servers) + }) + }) + + describe('Test expiration', function () { + const strategy = 'recently-added' + + async function checkContains (servers: PeerTubeServer[], str: string) { + for (const server of servers) { + const video = await server.videos.get({ id: video1Server2.uuid }) + + for (const f of video.files) { + expect(f.magnetUri).to.contain(str) + } + } + } + + async function checkNotContains (servers: PeerTubeServer[], str: string) { + for (const server of servers) { + const video = await server.videos.get({ id: video1Server2.uuid }) + + for (const f of video.files) { + expect(f.magnetUri).to.not.contain(str) + } + } + } + + before(async function () { + this.timeout(240000) + + await createServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) + + await enableRedundancyOnServer1() + }) + + it('Should still have 2 webseeds after 10 seconds', async function () { + this.timeout(80000) + + await wait(10000) + + try { + await checkContains(servers, 'http%3A%2F%2F' + servers[0].hostname + '%3A' + servers[0].port) + } catch { + // Maybe a server deleted a redundancy in the scheduler + await wait(2000) + + await checkContains(servers, 'http%3A%2F%2F' + servers[0].hostname + '%3A' + servers[0].port) + } + }) + + it('Should stop server 1 and expire video redundancy', async function () { + this.timeout(80000) + + await killallServers([ servers[0] ]) + + await wait(15000) + + await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2F' + servers[0].port + '%3A' + servers[0].port) + }) + + after(async function () { + await cleanupTests(servers) + }) + }) + + describe('Test file replacement', function () { + let video2Server2UUID: string + const strategy = 'recently-added' + + before(async function () { + this.timeout(240000) + + await createServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) + + await enableRedundancyOnServer1() + + await waitJobs(servers) + await servers[0].servers.waitUntilLog('Duplicated ', 5) + await waitJobs(servers) + + await check2Webseeds() + await check1PlaylistRedundancies() + await checkStatsWith1Redundancy(strategy) + + const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video 2 server 2', privacy: VideoPrivacy.PRIVATE } }) + video2Server2UUID = uuid + + // Wait transcoding before federation + await waitJobs(servers) + + await servers[1].videos.update({ id: video2Server2UUID, attributes: { privacy: VideoPrivacy.PUBLIC } }) + }) + + it('Should cache video 2 webseeds on the first video', async function () { + this.timeout(240000) + + await waitJobs(servers) + + let checked = false + + while (checked === false) { + await wait(1000) + + try { + await check1WebSeed() + await check0PlaylistRedundancies() + + await check2Webseeds(video2Server2UUID) + await check1PlaylistRedundancies(video2Server2UUID) + + checked = true + } catch { + checked = false + } + } + }) + + it('Should disable strategy and remove redundancies', async function () { + this.timeout(80000) + + await waitJobs(servers) + + await killallServers([ servers[0] ]) + await servers[0].run({ + redundancy: { + videos: { + check_interval: '1 second', + strategies: [] + } + } + }) + + await waitJobs(servers) + + await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true }) + }) + + after(async function () { + await cleanupTests(servers) + }) + }) +}) diff --git a/packages/tests/src/api/runners/index.ts b/packages/tests/src/api/runners/index.ts new file mode 100644 index 000000000..441ddc874 --- /dev/null +++ b/packages/tests/src/api/runners/index.ts @@ -0,0 +1,5 @@ +export * from './runner-common.js' +export * from './runner-live-transcoding.js' +export * from './runner-socket.js' +export * from './runner-studio-transcoding.js' +export * from './runner-vod-transcoding.js' diff --git a/packages/tests/src/api/runners/runner-common.ts b/packages/tests/src/api/runners/runner-common.ts new file mode 100644 index 000000000..53ea321d0 --- /dev/null +++ b/packages/tests/src/api/runners/runner-common.ts @@ -0,0 +1,744 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { wait } from '@peertube/peertube-core-utils' +import { + HttpStatusCode, + Runner, + RunnerJob, + RunnerJobAdmin, + RunnerJobState, + RunnerJobStateType, + RunnerJobVODWebVideoTranscodingPayload, + RunnerRegistrationToken +} from '@peertube/peertube-models' +import { + PeerTubeServer, + cleanupTests, + createSingleServer, + setAccessTokensToServers, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' +import { expect } from 'chai' + +describe('Test runner common actions', function () { + let server: PeerTubeServer + let registrationToken: string + let runnerToken: string + let jobMaxPriority: string + + before(async function () { + this.timeout(120_000) + + server = await createSingleServer(1, { + remote_runners: { + stalled_jobs: { + vod: '5 seconds' + } + } + }) + + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + await server.config.enableTranscoding({ hls: true, webVideo: true }) + await server.config.enableRemoteTranscoding() + }) + + describe('Managing runner registration tokens', function () { + let base: RunnerRegistrationToken[] + let registrationTokenToDelete: RunnerRegistrationToken + + it('Should have a default registration token', async function () { + const { total, data } = await server.runnerRegistrationTokens.list() + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + + const token = data[0] + expect(token.id).to.exist + expect(token.createdAt).to.exist + expect(token.updatedAt).to.exist + expect(token.registeredRunnersCount).to.equal(0) + expect(token.registrationToken).to.exist + }) + + it('Should create other registration tokens', async function () { + await server.runnerRegistrationTokens.generate() + await server.runnerRegistrationTokens.generate() + + const { total, data } = await server.runnerRegistrationTokens.list() + expect(total).to.equal(3) + expect(data).to.have.lengthOf(3) + }) + + it('Should list registration tokens', async function () { + { + const { total, data } = await server.runnerRegistrationTokens.list({ sort: 'createdAt' }) + expect(total).to.equal(3) + expect(data).to.have.lengthOf(3) + expect(new Date(data[0].createdAt)).to.be.below(new Date(data[1].createdAt)) + expect(new Date(data[1].createdAt)).to.be.below(new Date(data[2].createdAt)) + + base = data + + registrationTokenToDelete = data[0] + registrationToken = data[1].registrationToken + } + + { + const { total, data } = await server.runnerRegistrationTokens.list({ sort: '-createdAt', start: 2, count: 1 }) + expect(total).to.equal(3) + expect(data).to.have.lengthOf(1) + expect(data[0].registrationToken).to.equal(base[0].registrationToken) + } + }) + + it('Should have appropriate registeredRunnersCount for registration tokens', async function () { + await server.runners.register({ name: 'to delete 1', registrationToken: registrationTokenToDelete.registrationToken }) + await server.runners.register({ name: 'to delete 2', registrationToken: registrationTokenToDelete.registrationToken }) + + const { data } = await server.runnerRegistrationTokens.list() + + for (const d of data) { + if (d.registrationToken === registrationTokenToDelete.registrationToken) { + expect(d.registeredRunnersCount).to.equal(2) + } else { + expect(d.registeredRunnersCount).to.equal(0) + } + } + + const { data: runners } = await server.runners.list() + expect(runners).to.have.lengthOf(2) + }) + + it('Should delete a registration token', async function () { + await server.runnerRegistrationTokens.delete({ id: registrationTokenToDelete.id }) + + const { total, data } = await server.runnerRegistrationTokens.list({ sort: 'createdAt' }) + expect(total).to.equal(2) + expect(data).to.have.lengthOf(2) + + for (const d of data) { + expect(d.registeredRunnersCount).to.equal(0) + expect(d.registrationToken).to.not.equal(registrationTokenToDelete.registrationToken) + } + }) + + it('Should have removed runners of this registration token', async function () { + const { data: runners } = await server.runners.list() + expect(runners).to.have.lengthOf(0) + }) + }) + + describe('Managing runners', function () { + let toDelete: Runner + + it('Should not have runners available', async function () { + const { total, data } = await server.runners.list() + + expect(data).to.have.lengthOf(0) + expect(total).to.equal(0) + }) + + it('Should register runners', async function () { + const now = new Date() + + const result = await server.runners.register({ + name: 'runner 1', + description: 'my super runner 1', + registrationToken + }) + expect(result.runnerToken).to.exist + runnerToken = result.runnerToken + + await server.runners.register({ + name: 'runner 2', + registrationToken + }) + + const { total, data } = await server.runners.list({ sort: 'createdAt' }) + expect(total).to.equal(2) + expect(data).to.have.lengthOf(2) + + for (const d of data) { + expect(d.id).to.exist + expect(d.createdAt).to.exist + expect(d.updatedAt).to.exist + expect(new Date(d.createdAt)).to.be.above(now) + expect(new Date(d.updatedAt)).to.be.above(now) + expect(new Date(d.lastContact)).to.be.above(now) + expect(d.ip).to.exist + } + + expect(data[0].name).to.equal('runner 1') + expect(data[0].description).to.equal('my super runner 1') + + expect(data[1].name).to.equal('runner 2') + expect(data[1].description).to.be.null + + toDelete = data[1] + }) + + it('Should list runners', async function () { + const { total, data } = await server.runners.list({ sort: '-createdAt', start: 1, count: 1 }) + + expect(total).to.equal(2) + expect(data).to.have.lengthOf(1) + expect(data[0].name).to.equal('runner 1') + }) + + it('Should delete a runner', async function () { + await server.runners.delete({ id: toDelete.id }) + + const { total, data } = await server.runners.list() + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + expect(data[0].name).to.equal('runner 1') + }) + + it('Should unregister a runner', async function () { + const registered = await server.runners.autoRegisterRunner() + + { + const { total, data } = await server.runners.list() + expect(total).to.equal(2) + expect(data).to.have.lengthOf(2) + } + + await server.runners.unregister({ runnerToken: registered }) + + { + const { total, data } = await server.runners.list() + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + expect(data[0].name).to.equal('runner 1') + } + }) + }) + + describe('Managing runner jobs', function () { + let jobUUID: string + let jobToken: string + let lastRunnerContact: Date + let failedJob: RunnerJob + + async function checkMainJobState ( + mainJobState: RunnerJobStateType, + otherJobStates: RunnerJobStateType[] = [ RunnerJobState.PENDING, RunnerJobState.WAITING_FOR_PARENT_JOB ] + ) { + const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' }) + + for (const job of data) { + if (job.uuid === jobUUID) { + expect(job.state.id).to.equal(mainJobState) + } else { + expect(otherJobStates).to.include(job.state.id) + } + } + } + + function getMainJob () { + return server.runnerJobs.getJob({ uuid: jobUUID }) + } + + describe('List jobs', function () { + + it('Should not have jobs', async function () { + const { total, data } = await server.runnerJobs.list() + + expect(data).to.have.lengthOf(0) + expect(total).to.equal(0) + }) + + it('Should upload a video and have available jobs', async function () { + await server.videos.quickUpload({ name: 'to transcode' }) + await waitJobs([ server ]) + + const { total, data } = await server.runnerJobs.list() + + expect(data).to.have.lengthOf(10) + expect(total).to.equal(10) + + for (const job of data) { + expect(job.startedAt).to.not.exist + expect(job.finishedAt).to.not.exist + expect(job.payload).to.exist + expect(job.privatePayload).to.exist + } + + const hlsJobs = data.filter(d => d.type === 'vod-hls-transcoding') + const webVideoJobs = data.filter(d => d.type === 'vod-web-video-transcoding') + + expect(hlsJobs).to.have.lengthOf(5) + expect(webVideoJobs).to.have.lengthOf(5) + + const pendingJobs = data.filter(d => d.state.id === RunnerJobState.PENDING) + const waitingJobs = data.filter(d => d.state.id === RunnerJobState.WAITING_FOR_PARENT_JOB) + + expect(pendingJobs).to.have.lengthOf(1) + expect(waitingJobs).to.have.lengthOf(9) + }) + + it('Should upload another video and list/sort jobs', async function () { + await server.videos.quickUpload({ name: 'to transcode 2' }) + await waitJobs([ server ]) + + { + const { total, data } = await server.runnerJobs.list({ start: 0, count: 30 }) + + expect(data).to.have.lengthOf(20) + expect(total).to.equal(20) + + jobUUID = data[16].uuid + } + + { + const { total, data } = await server.runnerJobs.list({ start: 3, count: 1, sort: 'createdAt' }) + expect(total).to.equal(20) + + expect(data).to.have.lengthOf(1) + expect(data[0].uuid).to.equal(jobUUID) + } + + { + let previousPriority = Infinity + const { total, data } = await server.runnerJobs.list({ start: 0, count: 100, sort: '-priority' }) + expect(total).to.equal(20) + + for (const job of data) { + expect(job.priority).to.be.at.most(previousPriority) + previousPriority = job.priority + + if (job.state.id === RunnerJobState.PENDING) { + jobMaxPriority = job.uuid + } + } + } + }) + + it('Should search jobs', async function () { + { + const { total, data } = await server.runnerJobs.list({ search: jobUUID }) + + expect(data).to.have.lengthOf(1) + expect(total).to.equal(1) + + expect(data[0].uuid).to.equal(jobUUID) + } + + { + const { total, data } = await server.runnerJobs.list({ search: 'toto' }) + + expect(data).to.have.lengthOf(0) + expect(total).to.equal(0) + } + + { + const { total, data } = await server.runnerJobs.list({ search: 'hls' }) + + expect(data).to.not.have.lengthOf(0) + expect(total).to.not.equal(0) + + for (const job of data) { + expect(job.type).to.include('hls') + } + } + }) + + it('Should filter jobs', async function () { + { + const { total, data } = await server.runnerJobs.list({ stateOneOf: [ RunnerJobState.WAITING_FOR_PARENT_JOB ] }) + + expect(data).to.not.have.lengthOf(0) + expect(total).to.not.equal(0) + + for (const job of data) { + expect(job.state.label).to.equal('Waiting for parent job to finish') + } + } + + { + const { total, data } = await server.runnerJobs.list({ stateOneOf: [ RunnerJobState.COMPLETED ] }) + + expect(data).to.have.lengthOf(0) + expect(total).to.equal(0) + } + }) + }) + + describe('Accept/update/abort/process a job', function () { + + it('Should request available jobs', async function () { + lastRunnerContact = new Date() + + const { availableJobs } = await server.runnerJobs.request({ runnerToken }) + + // Only optimize jobs are available + expect(availableJobs).to.have.lengthOf(2) + + for (const job of availableJobs) { + expect(job.uuid).to.exist + expect(job.payload.input).to.exist + expect((job.payload as RunnerJobVODWebVideoTranscodingPayload).output).to.exist + + expect((job as RunnerJobAdmin).privatePayload).to.not.exist + } + + const hlsJobs = availableJobs.filter(d => d.type === 'vod-hls-transcoding') + const webVideoJobs = availableJobs.filter(d => d.type === 'vod-web-video-transcoding') + + expect(hlsJobs).to.have.lengthOf(0) + expect(webVideoJobs).to.have.lengthOf(2) + + jobUUID = webVideoJobs[0].uuid + }) + + it('Should have sorted available jobs by priority', async function () { + const { availableJobs } = await server.runnerJobs.request({ runnerToken }) + + expect(availableJobs[0].uuid).to.equal(jobMaxPriority) + }) + + it('Should have last runner contact updated', async function () { + await wait(1000) + + const { data } = await server.runners.list({ sort: 'createdAt' }) + expect(new Date(data[0].lastContact)).to.be.above(lastRunnerContact) + }) + + it('Should accept a job', async function () { + const startedAt = new Date() + + const { job } = await server.runnerJobs.accept({ runnerToken, jobUUID }) + jobToken = job.jobToken + + const checkProcessingJob = (job: RunnerJob & { jobToken?: string }, fromAccept: boolean) => { + expect(job.uuid).to.equal(jobUUID) + + expect(job.type).to.equal('vod-web-video-transcoding') + expect(job.state.label).to.equal('Processing') + expect(job.state.id).to.equal(RunnerJobState.PROCESSING) + + expect(job.runner).to.exist + expect(job.runner.name).to.equal('runner 1') + expect(job.runner.description).to.equal('my super runner 1') + + expect(job.progress).to.be.null + + expect(job.startedAt).to.exist + expect(new Date(job.startedAt)).to.be.above(startedAt) + + expect(job.finishedAt).to.not.exist + + expect(job.failures).to.equal(0) + + expect(job.payload).to.exist + + if (fromAccept) { + expect(job.jobToken).to.exist + expect((job as RunnerJobAdmin).privatePayload).to.not.exist + } else { + expect(job.jobToken).to.not.exist + expect((job as RunnerJobAdmin).privatePayload).to.exist + } + } + + checkProcessingJob(job, true) + + const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' }) + + const processingJob = data.find(j => j.uuid === jobUUID) + checkProcessingJob(processingJob, false) + + await checkMainJobState(RunnerJobState.PROCESSING) + }) + + it('Should update a job', async function () { + await server.runnerJobs.update({ runnerToken, jobUUID, jobToken, progress: 53 }) + + const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' }) + + for (const job of data) { + if (job.state.id === RunnerJobState.PROCESSING) { + expect(job.progress).to.equal(53) + } else { + expect(job.progress).to.be.null + } + } + }) + + it('Should abort a job', async function () { + await server.runnerJobs.abort({ runnerToken, jobUUID, jobToken, reason: 'for tests' }) + + await checkMainJobState(RunnerJobState.PENDING) + + const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' }) + for (const job of data) { + expect(job.progress).to.be.null + } + }) + + it('Should accept the same job again and post a success', async function () { + const { availableJobs } = await server.runnerJobs.request({ runnerToken }) + expect(availableJobs.find(j => j.uuid === jobUUID)).to.exist + + const { job } = await server.runnerJobs.accept({ runnerToken, jobUUID }) + jobToken = job.jobToken + + await checkMainJobState(RunnerJobState.PROCESSING) + + const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' }) + + for (const job of data) { + expect(job.progress).to.be.null + } + + const payload = { + videoFile: 'video_short.mp4' + } + + await server.runnerJobs.success({ runnerToken, jobUUID, jobToken, payload }) + }) + + it('Should not have available jobs anymore', async function () { + await checkMainJobState(RunnerJobState.COMPLETED) + + const job = await getMainJob() + expect(job.finishedAt).to.exist + + const { availableJobs } = await server.runnerJobs.request({ runnerToken }) + expect(availableJobs.find(j => j.uuid === jobUUID)).to.not.exist + }) + }) + + describe('Error job', function () { + + it('Should accept another job and post an error', async function () { + await server.runnerJobs.cancelAllJobs() + await server.videos.quickUpload({ name: 'video' }) + await waitJobs([ server ]) + + const { availableJobs } = await server.runnerJobs.request({ runnerToken }) + jobUUID = availableJobs[0].uuid + + const { job } = await server.runnerJobs.accept({ runnerToken, jobUUID }) + jobToken = job.jobToken + + await server.runnerJobs.error({ runnerToken, jobUUID, jobToken, message: 'Error' }) + }) + + it('Should have job failures increased', async function () { + const job = await getMainJob() + expect(job.state.id).to.equal(RunnerJobState.PENDING) + expect(job.failures).to.equal(1) + expect(job.error).to.be.null + expect(job.progress).to.be.null + expect(job.finishedAt).to.not.exist + }) + + it('Should error a job when job attempts is too big', async function () { + for (let i = 0; i < 4; i++) { + const { job } = await server.runnerJobs.accept({ runnerToken, jobUUID }) + jobToken = job.jobToken + + await server.runnerJobs.error({ runnerToken, jobUUID, jobToken, message: 'Error ' + i }) + } + + const job = await getMainJob() + expect(job.failures).to.equal(5) + expect(job.state.id).to.equal(RunnerJobState.ERRORED) + expect(job.state.label).to.equal('Errored') + expect(job.error).to.equal('Error 3') + expect(job.progress).to.be.null + expect(job.finishedAt).to.exist + + failedJob = job + }) + + it('Should have failed children jobs too', async function () { + const { data } = await server.runnerJobs.list({ count: 50, sort: '-updatedAt' }) + + const children = data.filter(j => j.parent?.uuid === failedJob.uuid) + expect(children).to.have.lengthOf(9) + + for (const child of children) { + expect(child.parent.uuid).to.equal(failedJob.uuid) + expect(child.parent.type).to.equal(failedJob.type) + expect(child.parent.state.id).to.equal(failedJob.state.id) + expect(child.parent.state.label).to.equal(failedJob.state.label) + + expect(child.state.id).to.equal(RunnerJobState.PARENT_ERRORED) + expect(child.state.label).to.equal('Parent job failed') + } + }) + }) + + describe('Cancel', function () { + + it('Should cancel a pending job', async function () { + await server.videos.quickUpload({ name: 'video' }) + await waitJobs([ server ]) + + { + const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' }) + + const pendingJob = data.find(j => j.state.id === RunnerJobState.PENDING) + jobUUID = pendingJob.uuid + + await server.runnerJobs.cancelByAdmin({ jobUUID }) + } + + { + const job = await getMainJob() + expect(job.state.id).to.equal(RunnerJobState.CANCELLED) + expect(job.state.label).to.equal('Cancelled') + } + + { + const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' }) + const children = data.filter(j => j.parent?.uuid === jobUUID) + expect(children).to.have.lengthOf(9) + + for (const child of children) { + expect(child.state.id).to.equal(RunnerJobState.PARENT_CANCELLED) + } + } + }) + + it('Should cancel an already accepted job and skip success/error', async function () { + await server.videos.quickUpload({ name: 'video' }) + await waitJobs([ server ]) + + const { availableJobs } = await server.runnerJobs.request({ runnerToken }) + jobUUID = availableJobs[0].uuid + + const { job } = await server.runnerJobs.accept({ runnerToken, jobUUID }) + jobToken = job.jobToken + + await server.runnerJobs.cancelByAdmin({ jobUUID }) + + await server.runnerJobs.abort({ runnerToken, jobUUID, jobToken, reason: 'aborted', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + }) + + describe('Remove', function () { + + it('Should remove a pending job', async function () { + await server.videos.quickUpload({ name: 'video' }) + await waitJobs([ server ]) + + { + const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' }) + + const pendingJob = data.find(j => j.state.id === RunnerJobState.PENDING) + jobUUID = pendingJob.uuid + + await server.runnerJobs.deleteByAdmin({ jobUUID }) + } + + { + const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' }) + + const parent = data.find(j => j.uuid === jobUUID) + expect(parent).to.not.exist + + const children = data.filter(j => j.parent?.uuid === jobUUID) + expect(children).to.have.lengthOf(0) + } + }) + }) + + describe('Stalled jobs', function () { + + it('Should abort stalled jobs', async function () { + this.timeout(60000) + + await server.videos.quickUpload({ name: 'video' }) + await server.videos.quickUpload({ name: 'video' }) + await waitJobs([ server ]) + + const { job: job1 } = await server.runnerJobs.autoAccept({ runnerToken }) + const { job: stalledJob } = await server.runnerJobs.autoAccept({ runnerToken }) + + for (let i = 0; i < 6; i++) { + await wait(2000) + + await server.runnerJobs.update({ runnerToken, jobToken: job1.jobToken, jobUUID: job1.uuid }) + } + + const refreshedJob1 = await server.runnerJobs.getJob({ uuid: job1.uuid }) + const refreshedStalledJob = await server.runnerJobs.getJob({ uuid: stalledJob.uuid }) + + expect(refreshedJob1.state.id).to.equal(RunnerJobState.PROCESSING) + expect(refreshedStalledJob.state.id).to.equal(RunnerJobState.PENDING) + }) + }) + + describe('Rate limit', function () { + + before(async function () { + this.timeout(60000) + + await server.kill() + + await server.run({ + rates_limit: { + api: { + max: 10 + } + } + }) + }) + + it('Should rate limit an unknown runner, but not a registered one', async function () { + this.timeout(60000) + + await server.videos.quickUpload({ name: 'video' }) + await waitJobs([ server ]) + + const { job } = await server.runnerJobs.autoAccept({ runnerToken }) + + for (let i = 0; i < 20; i++) { + try { + await server.runnerJobs.request({ runnerToken }) + await server.runnerJobs.update({ runnerToken, jobToken: job.jobToken, jobUUID: job.uuid }) + } catch {} + } + + // Invalid + { + await server.runnerJobs.request({ runnerToken: 'toto', expectedStatus: HttpStatusCode.TOO_MANY_REQUESTS_429 }) + await server.runnerJobs.update({ + runnerToken: 'toto', + jobToken: job.jobToken, + jobUUID: job.uuid, + expectedStatus: HttpStatusCode.TOO_MANY_REQUESTS_429 + }) + } + + // Not provided + { + await server.runnerJobs.request({ runnerToken: undefined, expectedStatus: HttpStatusCode.TOO_MANY_REQUESTS_429 }) + await server.runnerJobs.update({ + runnerToken: undefined, + jobToken: job.jobToken, + jobUUID: job.uuid, + expectedStatus: HttpStatusCode.TOO_MANY_REQUESTS_429 + }) + } + + // Registered + { + await server.runnerJobs.request({ runnerToken }) + await server.runnerJobs.update({ runnerToken, jobToken: job.jobToken, jobUUID: job.uuid }) + } + }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/runners/runner-live-transcoding.ts b/packages/tests/src/api/runners/runner-live-transcoding.ts new file mode 100644 index 000000000..20c1e5c2a --- /dev/null +++ b/packages/tests/src/api/runners/runner-live-transcoding.ts @@ -0,0 +1,332 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { FfmpegCommand } from 'fluent-ffmpeg' +import { readFile } from 'fs/promises' +import { wait } from '@peertube/peertube-core-utils' +import { + HttpStatusCode, + LiveRTMPHLSTranscodingUpdatePayload, + LiveVideo, + LiveVideoError, + LiveVideoErrorType, + RunnerJob, + RunnerJobLiveRTMPHLSTranscodingPayload, + Video, + VideoPrivacy, + VideoState +} from '@peertube/peertube-models' +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import { + cleanupTests, + createSingleServer, + makeRawRequest, + PeerTubeServer, + sendRTMPStream, + setAccessTokensToServers, + setDefaultVideoChannel, + stopFfmpeg, + testFfmpegStreamError, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test runner live transcoding', function () { + let server: PeerTubeServer + let runnerToken: string + let baseUrl: string + + before(async function () { + this.timeout(120_000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + await server.config.enableRemoteTranscoding() + await server.config.enableTranscoding() + runnerToken = await server.runners.autoRegisterRunner() + + baseUrl = server.url + '/static/streaming-playlists/hls' + }) + + describe('Without transcoding enabled', function () { + + before(async function () { + await server.config.enableLive({ + allowReplay: false, + resolutions: 'min', + transcoding: false + }) + }) + + it('Should not have available jobs', async function () { + this.timeout(120000) + + const { live, video } = await server.live.quickCreate({ permanentLive: true, saveReplay: false, privacy: VideoPrivacy.PUBLIC }) + + const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) + await server.live.waitUntilPublished({ videoId: video.id }) + + await waitJobs([ server ]) + + const { availableJobs } = await server.runnerJobs.requestLive({ runnerToken }) + expect(availableJobs).to.have.lengthOf(0) + + await stopFfmpeg(ffmpegCommand) + }) + }) + + describe('With transcoding enabled on classic live', function () { + let live: LiveVideo + let video: Video + let ffmpegCommand: FfmpegCommand + let jobUUID: string + let acceptedJob: RunnerJob & { jobToken: string } + + async function testPlaylistFile (fixture: string, expected: string) { + const text = await server.streamingPlaylists.get({ url: `${baseUrl}/${video.uuid}/${fixture}` }) + expect(await readFile(buildAbsoluteFixturePath(expected), 'utf-8')).to.equal(text) + + } + + async function testTSFile (fixture: string, expected: string) { + const { body } = await makeRawRequest({ url: `${baseUrl}/${video.uuid}/${fixture}`, expectedStatus: HttpStatusCode.OK_200 }) + expect(await readFile(buildAbsoluteFixturePath(expected))).to.deep.equal(body) + } + + before(async function () { + await server.config.enableLive({ + allowReplay: true, + resolutions: 'max', + transcoding: true + }) + }) + + it('Should publish a a live and have available jobs', async function () { + this.timeout(120000) + + const data = await server.live.quickCreate({ permanentLive: false, saveReplay: false, privacy: VideoPrivacy.PUBLIC }) + live = data.live + video = data.video + + ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) + await waitJobs([ server ]) + + const job = await server.runnerJobs.requestLiveJob(runnerToken) + jobUUID = job.uuid + + expect(job.type).to.equal('live-rtmp-hls-transcoding') + expect(job.payload.input.rtmpUrl).to.exist + + expect(job.payload.output.toTranscode).to.have.lengthOf(5) + + for (const { resolution, fps } of job.payload.output.toTranscode) { + expect([ 720, 480, 360, 240, 144 ]).to.contain(resolution) + + expect(fps).to.be.above(25) + expect(fps).to.be.below(70) + } + }) + + it('Should update the live with a new chunk', async function () { + this.timeout(120000) + + const { job } = await server.runnerJobs.accept({ jobUUID, runnerToken }) + acceptedJob = job + + { + const payload: LiveRTMPHLSTranscodingUpdatePayload = { + masterPlaylistFile: 'live/master.m3u8', + resolutionPlaylistFile: 'live/0.m3u8', + resolutionPlaylistFilename: '0.m3u8', + type: 'add-chunk', + videoChunkFile: 'live/0-000067.ts', + videoChunkFilename: '0-000067.ts' + } + await server.runnerJobs.update({ jobUUID, runnerToken, jobToken: job.jobToken, payload, progress: 50 }) + + const updatedJob = await server.runnerJobs.getJob({ uuid: job.uuid }) + expect(updatedJob.progress).to.equal(50) + } + + { + const payload: LiveRTMPHLSTranscodingUpdatePayload = { + resolutionPlaylistFile: 'live/1.m3u8', + resolutionPlaylistFilename: '1.m3u8', + type: 'add-chunk', + videoChunkFile: 'live/1-000068.ts', + videoChunkFilename: '1-000068.ts' + } + await server.runnerJobs.update({ jobUUID, runnerToken, jobToken: job.jobToken, payload }) + } + + await wait(1000) + + await testPlaylistFile('master.m3u8', 'live/master.m3u8') + await testPlaylistFile('0.m3u8', 'live/0.m3u8') + await testPlaylistFile('1.m3u8', 'live/1.m3u8') + + await testTSFile('0-000067.ts', 'live/0-000067.ts') + await testTSFile('1-000068.ts', 'live/1-000068.ts') + }) + + it('Should replace existing m3u8 on update', async function () { + this.timeout(120000) + + const payload: LiveRTMPHLSTranscodingUpdatePayload = { + masterPlaylistFile: 'live/1.m3u8', + resolutionPlaylistFilename: '0.m3u8', + resolutionPlaylistFile: 'live/1.m3u8', + type: 'add-chunk', + videoChunkFile: 'live/1-000069.ts', + videoChunkFilename: '1-000068.ts' + } + await server.runnerJobs.update({ jobUUID, runnerToken, jobToken: acceptedJob.jobToken, payload }) + await wait(1000) + + await testPlaylistFile('master.m3u8', 'live/1.m3u8') + await testPlaylistFile('0.m3u8', 'live/1.m3u8') + await testTSFile('1-000068.ts', 'live/1-000069.ts') + }) + + it('Should update the live with removed chunks', async function () { + this.timeout(120000) + + const payload: LiveRTMPHLSTranscodingUpdatePayload = { + resolutionPlaylistFile: 'live/0.m3u8', + resolutionPlaylistFilename: '0.m3u8', + type: 'remove-chunk', + videoChunkFilename: '1-000068.ts' + } + await server.runnerJobs.update({ jobUUID, runnerToken, jobToken: acceptedJob.jobToken, payload }) + + await wait(1000) + + await server.streamingPlaylists.get({ url: `${baseUrl}/${video.uuid}/master.m3u8` }) + await server.streamingPlaylists.get({ url: `${baseUrl}/${video.uuid}/0.m3u8` }) + await server.streamingPlaylists.get({ url: `${baseUrl}/${video.uuid}/1.m3u8` }) + await makeRawRequest({ url: `${baseUrl}/${video.uuid}/0-000067.ts`, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url: `${baseUrl}/${video.uuid}/1-000068.ts`, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should complete the live and save the replay', async function () { + this.timeout(120000) + + for (const segment of [ '0-000069.ts', '0-000070.ts' ]) { + const payload: LiveRTMPHLSTranscodingUpdatePayload = { + masterPlaylistFile: 'live/master.m3u8', + resolutionPlaylistFilename: '0.m3u8', + resolutionPlaylistFile: 'live/0.m3u8', + type: 'add-chunk', + videoChunkFile: 'live/' + segment, + videoChunkFilename: segment + } + await server.runnerJobs.update({ jobUUID, runnerToken, jobToken: acceptedJob.jobToken, payload }) + + await wait(1000) + } + + await waitJobs([ server ]) + + { + const { state } = await server.videos.get({ id: video.uuid }) + expect(state.id).to.equal(VideoState.PUBLISHED) + } + + await stopFfmpeg(ffmpegCommand) + + await server.runnerJobs.success({ jobUUID, runnerToken, jobToken: acceptedJob.jobToken, payload: {} }) + + await wait(1500) + await waitJobs([ server ]) + + { + const { state } = await server.videos.get({ id: video.uuid }) + expect(state.id).to.equal(VideoState.LIVE_ENDED) + + const session = await server.live.findLatestSession({ videoId: video.uuid }) + expect(session.error).to.be.null + } + }) + }) + + describe('With transcoding enabled on cancelled/aborted/errored live', function () { + let live: LiveVideo + let video: Video + let ffmpegCommand: FfmpegCommand + + async function prepare () { + ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) + await server.runnerJobs.requestLiveJob(runnerToken) + + const { job } = await server.runnerJobs.autoAccept({ runnerToken, type: 'live-rtmp-hls-transcoding' }) + + return job + } + + async function checkSessionError (error: LiveVideoErrorType) { + await wait(1500) + await waitJobs([ server ]) + + const session = await server.live.findLatestSession({ videoId: video.uuid }) + expect(session.error).to.equal(error) + } + + before(async function () { + await server.config.enableLive({ + allowReplay: true, + resolutions: 'max', + transcoding: true + }) + + const data = await server.live.quickCreate({ permanentLive: true, saveReplay: false, privacy: VideoPrivacy.PUBLIC }) + live = data.live + video = data.video + }) + + it('Should abort a running live', async function () { + this.timeout(120000) + + const job = await prepare() + + await Promise.all([ + server.runnerJobs.abort({ jobUUID: job.uuid, runnerToken, jobToken: job.jobToken, reason: 'abort' }), + testFfmpegStreamError(ffmpegCommand, true) + ]) + + // Abort is not supported + await checkSessionError(LiveVideoError.RUNNER_JOB_ERROR) + }) + + it('Should cancel a running live', async function () { + this.timeout(120000) + + const job = await prepare() + + await Promise.all([ + server.runnerJobs.cancelByAdmin({ jobUUID: job.uuid }), + testFfmpegStreamError(ffmpegCommand, true) + ]) + + await checkSessionError(LiveVideoError.RUNNER_JOB_CANCEL) + }) + + it('Should error a running live', async function () { + this.timeout(120000) + + const job = await prepare() + + await Promise.all([ + server.runnerJobs.error({ jobUUID: job.uuid, runnerToken, jobToken: job.jobToken, message: 'error' }), + testFfmpegStreamError(ffmpegCommand, true) + ]) + + await checkSessionError(LiveVideoError.RUNNER_JOB_ERROR) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/runners/runner-socket.ts b/packages/tests/src/api/runners/runner-socket.ts new file mode 100644 index 000000000..726ef084f --- /dev/null +++ b/packages/tests/src/api/runners/runner-socket.ts @@ -0,0 +1,120 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { + cleanupTests, + createSingleServer, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test runner socket', function () { + let server: PeerTubeServer + let runnerToken: string + + before(async function () { + this.timeout(120_000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + await server.config.enableTranscoding({ hls: true, webVideo: true }) + await server.config.enableRemoteTranscoding() + runnerToken = await server.runners.autoRegisterRunner() + }) + + it('Should throw an error without runner token', function (done) { + const localSocket = server.socketIO.getRunnersSocket({ runnerToken: null }) + localSocket.on('connect_error', err => { + expect(err.message).to.contain('No runner token provided') + done() + }) + }) + + it('Should throw an error with a bad runner token', function (done) { + const localSocket = server.socketIO.getRunnersSocket({ runnerToken: 'ergag' }) + localSocket.on('connect_error', err => { + expect(err.message).to.contain('Invalid runner token') + done() + }) + }) + + it('Should not send ping if there is no available jobs', async function () { + let pings = 0 + const localSocket = server.socketIO.getRunnersSocket({ runnerToken }) + localSocket.on('available-jobs', () => pings++) + + expect(pings).to.equal(0) + }) + + it('Should send a ping on available job', async function () { + let pings = 0 + const localSocket = server.socketIO.getRunnersSocket({ runnerToken }) + localSocket.on('available-jobs', () => pings++) + + await server.videos.quickUpload({ name: 'video1' }) + await waitJobs([ server ]) + + // eslint-disable-next-line no-unmodified-loop-condition + while (pings !== 1) { + await wait(500) + } + + await server.videos.quickUpload({ name: 'video2' }) + await waitJobs([ server ]) + + // eslint-disable-next-line no-unmodified-loop-condition + while ((pings as number) !== 2) { + await wait(500) + } + + await server.runnerJobs.cancelAllJobs() + }) + + it('Should send a ping when a child is ready', async function () { + let pings = 0 + const localSocket = server.socketIO.getRunnersSocket({ runnerToken }) + localSocket.on('available-jobs', () => pings++) + + await server.videos.quickUpload({ name: 'video3' }) + await waitJobs([ server ]) + + // eslint-disable-next-line no-unmodified-loop-condition + while (pings !== 1) { + await wait(500) + } + + await server.runnerJobs.autoProcessWebVideoJob(runnerToken) + await waitJobs([ server ]) + + // eslint-disable-next-line no-unmodified-loop-condition + while ((pings as number) !== 2) { + await wait(500) + } + }) + + it('Should not send a ping if the ended job does not have a child', async function () { + let pings = 0 + const localSocket = server.socketIO.getRunnersSocket({ runnerToken }) + localSocket.on('available-jobs', () => pings++) + + const { availableJobs } = await server.runnerJobs.request({ runnerToken }) + const job = availableJobs.find(j => j.type === 'vod-web-video-transcoding') + await server.runnerJobs.autoProcessWebVideoJob(runnerToken, job.uuid) + + // Wait for debounce + await wait(1000) + await waitJobs([ server ]) + + expect(pings).to.equal(0) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/runners/runner-studio-transcoding.ts b/packages/tests/src/api/runners/runner-studio-transcoding.ts new file mode 100644 index 000000000..adf6941c3 --- /dev/null +++ b/packages/tests/src/api/runners/runner-studio-transcoding.ts @@ -0,0 +1,169 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { readFile } from 'fs/promises' +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import { + RunnerJobStudioTranscodingPayload, + VideoStudioTranscodingSuccess, + VideoState, + VideoStudioTask, + VideoStudioTaskIntro +} from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + VideoStudioCommand, + waitJobs +} from '@peertube/peertube-server-commands' +import { checkVideoDuration } from '@tests/shared/checks.js' +import { checkPersistentTmpIsEmpty } from '@tests/shared/directories.js' + +describe('Test runner video studio transcoding', function () { + let servers: PeerTubeServer[] = [] + let runnerToken: string + let videoUUID: string + let jobUUID: string + + async function renewStudio (tasks: VideoStudioTask[] = VideoStudioCommand.getComplexTask()) { + const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) + videoUUID = uuid + + await waitJobs(servers) + + await servers[0].videoStudio.createEditionTasks({ videoId: uuid, tasks }) + await waitJobs(servers) + + const { availableJobs } = await servers[0].runnerJobs.request({ runnerToken }) + expect(availableJobs).to.have.lengthOf(1) + + jobUUID = availableJobs[0].uuid + } + + before(async function () { + this.timeout(120_000) + + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + await doubleFollow(servers[0], servers[1]) + + await servers[0].config.enableTranscoding({ hls: true, webVideo: true }) + await servers[0].config.enableStudio() + await servers[0].config.enableRemoteStudio() + + runnerToken = await servers[0].runners.autoRegisterRunner() + }) + + it('Should error a studio transcoding job', async function () { + this.timeout(60000) + + await renewStudio() + + for (let i = 0; i < 5; i++) { + const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) + const jobToken = job.jobToken + + await servers[0].runnerJobs.error({ runnerToken, jobUUID, jobToken, message: 'Error' }) + } + + const video = await servers[0].videos.get({ id: videoUUID }) + expect(video.state.id).to.equal(VideoState.PUBLISHED) + + await checkPersistentTmpIsEmpty(servers[0]) + }) + + it('Should cancel a transcoding job', async function () { + this.timeout(60000) + + await renewStudio() + + await servers[0].runnerJobs.cancelByAdmin({ jobUUID }) + + const video = await servers[0].videos.get({ id: videoUUID }) + expect(video.state.id).to.equal(VideoState.PUBLISHED) + + await checkPersistentTmpIsEmpty(servers[0]) + }) + + it('Should execute a remote studio job', async function () { + this.timeout(240_000) + + const tasks = [ + { + name: 'add-outro' as 'add-outro', + options: { + file: 'video_short.webm' + } + }, + { + name: 'add-watermark' as 'add-watermark', + options: { + file: 'custom-thumbnail.png' + } + }, + { + name: 'add-intro' as 'add-intro', + options: { + file: 'video_very_short_240p.mp4' + } + } + ] + + await renewStudio(tasks) + + for (const server of servers) { + await checkVideoDuration(server, videoUUID, 5) + } + + const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) + const jobToken = job.jobToken + + expect(job.type === 'video-studio-transcoding') + expect(job.payload.input.videoFileUrl).to.exist + + // Check video input file + { + await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) + } + + // Check task files + for (let i = 0; i < tasks.length; i++) { + const task = tasks[i] + const payloadTask = job.payload.tasks[i] + + expect(payloadTask.name).to.equal(task.name) + + const inputFile = await readFile(buildAbsoluteFixturePath(task.options.file)) + + const { body } = await servers[0].runnerJobs.getJobFile({ + url: (payloadTask as VideoStudioTaskIntro).options.file as string, + jobToken, + runnerToken + }) + + expect(body).to.deep.equal(inputFile) + } + + const payload: VideoStudioTranscodingSuccess = { videoFile: 'video_very_short_240p.mp4' } + await servers[0].runnerJobs.success({ runnerToken, jobUUID, jobToken, payload }) + + await waitJobs(servers) + + for (const server of servers) { + await checkVideoDuration(server, videoUUID, 2) + } + + await checkPersistentTmpIsEmpty(servers[0]) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/runners/runner-vod-transcoding.ts b/packages/tests/src/api/runners/runner-vod-transcoding.ts new file mode 100644 index 000000000..fe1c8f0b2 --- /dev/null +++ b/packages/tests/src/api/runners/runner-vod-transcoding.ts @@ -0,0 +1,545 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { readFile } from 'fs/promises' +import { completeCheckHlsPlaylist } from '@tests/shared/streaming-playlists.js' +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import { + HttpStatusCode, + RunnerJobSuccessPayload, + RunnerJobVODAudioMergeTranscodingPayload, + RunnerJobVODHLSTranscodingPayload, + RunnerJobVODPayload, + RunnerJobVODWebVideoTranscodingPayload, + VideoState, + VODAudioMergeTranscodingSuccess, + VODHLSTranscodingSuccess, + VODWebVideoTranscodingSuccess +} from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + makeGetRequest, + makeRawRequest, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' + +async function processAllJobs (server: PeerTubeServer, runnerToken: string) { + do { + const { availableJobs } = await server.runnerJobs.requestVOD({ runnerToken }) + if (availableJobs.length === 0) break + + const { job } = await server.runnerJobs.accept({ runnerToken, jobUUID: availableJobs[0].uuid }) + + const payload: RunnerJobSuccessPayload = { + videoFile: `video_short_${job.payload.output.resolution}p.mp4`, + resolutionPlaylistFile: `video_short_${job.payload.output.resolution}p.m3u8` + } + await server.runnerJobs.success({ runnerToken, jobUUID: job.uuid, jobToken: job.jobToken, payload }) + } while (true) + + await waitJobs([ server ]) +} + +describe('Test runner VOD transcoding', function () { + let servers: PeerTubeServer[] = [] + let runnerToken: string + + before(async function () { + this.timeout(120_000) + + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + await doubleFollow(servers[0], servers[1]) + + await servers[0].config.enableRemoteTranscoding() + runnerToken = await servers[0].runners.autoRegisterRunner() + }) + + describe('Without transcoding', function () { + + before(async function () { + this.timeout(60000) + + await servers[0].config.disableTranscoding() + await servers[0].videos.quickUpload({ name: 'video' }) + + await waitJobs(servers) + }) + + it('Should not have available jobs', async function () { + const { availableJobs } = await servers[0].runnerJobs.requestVOD({ runnerToken }) + expect(availableJobs).to.have.lengthOf(0) + }) + }) + + describe('With classic transcoding enabled', function () { + + before(async function () { + this.timeout(60000) + + await servers[0].config.enableTranscoding({ hls: true, webVideo: true }) + }) + + it('Should error a transcoding job', async function () { + this.timeout(60000) + + await servers[0].runnerJobs.cancelAllJobs() + const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) + await waitJobs(servers) + + const { availableJobs } = await servers[0].runnerJobs.request({ runnerToken }) + const jobUUID = availableJobs[0].uuid + + for (let i = 0; i < 5; i++) { + const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) + const jobToken = job.jobToken + + await servers[0].runnerJobs.error({ runnerToken, jobUUID, jobToken, message: 'Error' }) + } + + const video = await servers[0].videos.get({ id: uuid }) + expect(video.state.id).to.equal(VideoState.TRANSCODING_FAILED) + }) + + it('Should cancel a transcoding job', async function () { + await servers[0].runnerJobs.cancelAllJobs() + const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) + await waitJobs(servers) + + const { availableJobs } = await servers[0].runnerJobs.request({ runnerToken }) + const jobUUID = availableJobs[0].uuid + + await servers[0].runnerJobs.cancelByAdmin({ jobUUID }) + + const video = await servers[0].videos.get({ id: uuid }) + expect(video.state.id).to.equal(VideoState.PUBLISHED) + }) + }) + + describe('Web video transcoding only', function () { + let videoUUID: string + let jobToken: string + let jobUUID: string + + before(async function () { + this.timeout(60000) + + await servers[0].runnerJobs.cancelAllJobs() + await servers[0].config.enableTranscoding({ hls: false, webVideo: true }) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'web video', fixture: 'video_short.webm' }) + videoUUID = uuid + + await waitJobs(servers) + }) + + it('Should have jobs available for remote runners', async function () { + const { availableJobs } = await servers[0].runnerJobs.requestVOD({ runnerToken }) + expect(availableJobs).to.have.lengthOf(1) + + jobUUID = availableJobs[0].uuid + }) + + it('Should have a valid first transcoding job', async function () { + const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) + jobToken = job.jobToken + + expect(job.type === 'vod-web-video-transcoding') + expect(job.payload.input.videoFileUrl).to.exist + expect(job.payload.output.resolution).to.equal(720) + expect(job.payload.output.fps).to.equal(25) + + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) + const inputFile = await readFile(buildAbsoluteFixturePath('video_short.webm')) + + expect(body).to.deep.equal(inputFile) + }) + + it('Should transcode the max video resolution and send it back to the server', async function () { + this.timeout(60000) + + const payload: VODWebVideoTranscodingSuccess = { + videoFile: 'video_short.mp4' + } + await servers[0].runnerJobs.success({ runnerToken, jobUUID, jobToken, payload }) + + await waitJobs(servers) + }) + + it('Should have the video updated', async function () { + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + expect(video.files).to.have.lengthOf(1) + expect(video.streamingPlaylists).to.have.lengthOf(0) + + const { body } = await makeRawRequest({ url: video.files[0].fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + expect(body).to.deep.equal(await readFile(buildAbsoluteFixturePath('video_short.mp4'))) + } + }) + + it('Should have 4 lower resolution to transcode', async function () { + const { availableJobs } = await servers[0].runnerJobs.requestVOD({ runnerToken }) + expect(availableJobs).to.have.lengthOf(4) + + for (const resolution of [ 480, 360, 240, 144 ]) { + const job = availableJobs.find(j => j.payload.output.resolution === resolution) + expect(job).to.exist + expect(job.type).to.equal('vod-web-video-transcoding') + + if (resolution === 240) jobUUID = job.uuid + } + }) + + it('Should process one of these transcoding jobs', async function () { + const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) + jobToken = job.jobToken + + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) + const inputFile = await readFile(buildAbsoluteFixturePath('video_short.mp4')) + + expect(body).to.deep.equal(inputFile) + + const payload: VODWebVideoTranscodingSuccess = { videoFile: `video_short_${job.payload.output.resolution}p.mp4` } + await servers[0].runnerJobs.success({ runnerToken, jobUUID, jobToken, payload }) + }) + + it('Should process all other jobs', async function () { + const { availableJobs } = await servers[0].runnerJobs.requestVOD({ runnerToken }) + expect(availableJobs).to.have.lengthOf(3) + + for (const resolution of [ 480, 360, 144 ]) { + const availableJob = availableJobs.find(j => j.payload.output.resolution === resolution) + expect(availableJob).to.exist + jobUUID = availableJob.uuid + + const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) + jobToken = job.jobToken + + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) + const inputFile = await readFile(buildAbsoluteFixturePath('video_short.mp4')) + expect(body).to.deep.equal(inputFile) + + const payload: VODWebVideoTranscodingSuccess = { videoFile: `video_short_${resolution}p.mp4` } + await servers[0].runnerJobs.success({ runnerToken, jobUUID, jobToken, payload }) + } + + await waitJobs(servers) + }) + + it('Should have the video updated', async function () { + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + expect(video.files).to.have.lengthOf(5) + expect(video.streamingPlaylists).to.have.lengthOf(0) + + const { body } = await makeRawRequest({ url: video.files[0].fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + expect(body).to.deep.equal(await readFile(buildAbsoluteFixturePath('video_short.mp4'))) + + for (const file of video.files) { + await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 }) + } + } + }) + + it('Should not have available jobs anymore', async function () { + const { availableJobs } = await servers[0].runnerJobs.requestVOD({ runnerToken }) + expect(availableJobs).to.have.lengthOf(0) + }) + }) + + describe('HLS transcoding only', function () { + let videoUUID: string + let jobToken: string + let jobUUID: string + + before(async function () { + this.timeout(60000) + + await servers[0].config.enableTranscoding({ hls: true, webVideo: false }) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'hls video', fixture: 'video_short.webm' }) + videoUUID = uuid + + await waitJobs(servers) + }) + + it('Should run the optimize job', async function () { + this.timeout(60000) + + await servers[0].runnerJobs.autoProcessWebVideoJob(runnerToken) + }) + + it('Should have 5 HLS resolution to transcode', async function () { + const { availableJobs } = await servers[0].runnerJobs.requestVOD({ runnerToken }) + expect(availableJobs).to.have.lengthOf(5) + + for (const resolution of [ 720, 480, 360, 240, 144 ]) { + const job = availableJobs.find(j => j.payload.output.resolution === resolution) + expect(job).to.exist + expect(job.type).to.equal('vod-hls-transcoding') + + if (resolution === 480) jobUUID = job.uuid + } + }) + + it('Should process one of these transcoding jobs', async function () { + this.timeout(60000) + + const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) + jobToken = job.jobToken + + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) + const inputFile = await readFile(buildAbsoluteFixturePath('video_short.mp4')) + + expect(body).to.deep.equal(inputFile) + + const payload: VODHLSTranscodingSuccess = { + videoFile: 'video_short_480p.mp4', + resolutionPlaylistFile: 'video_short_480p.m3u8' + } + await servers[0].runnerJobs.success({ runnerToken, jobUUID, jobToken, payload }) + + await waitJobs(servers) + }) + + it('Should have the video updated', async function () { + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + + expect(video.files).to.have.lengthOf(1) + expect(video.streamingPlaylists).to.have.lengthOf(1) + + const hls = video.streamingPlaylists[0] + expect(hls.files).to.have.lengthOf(1) + + await completeCheckHlsPlaylist({ videoUUID, hlsOnly: false, servers, resolutions: [ 480 ] }) + } + }) + + it('Should process all other jobs', async function () { + this.timeout(60000) + + const { availableJobs } = await servers[0].runnerJobs.requestVOD({ runnerToken }) + expect(availableJobs).to.have.lengthOf(4) + + let maxQualityFile = 'video_short.mp4' + + for (const resolution of [ 720, 360, 240, 144 ]) { + const availableJob = availableJobs.find(j => j.payload.output.resolution === resolution) + expect(availableJob).to.exist + jobUUID = availableJob.uuid + + const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) + jobToken = job.jobToken + + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) + const inputFile = await readFile(buildAbsoluteFixturePath(maxQualityFile)) + expect(body).to.deep.equal(inputFile) + + const payload: VODHLSTranscodingSuccess = { + videoFile: `video_short_${resolution}p.mp4`, + resolutionPlaylistFile: `video_short_${resolution}p.m3u8` + } + await servers[0].runnerJobs.success({ runnerToken, jobUUID, jobToken, payload }) + + if (resolution === 720) { + maxQualityFile = 'video_short_720p.mp4' + } + } + + await waitJobs(servers) + }) + + it('Should have the video updated', async function () { + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + + expect(video.files).to.have.lengthOf(0) + expect(video.streamingPlaylists).to.have.lengthOf(1) + + const hls = video.streamingPlaylists[0] + expect(hls.files).to.have.lengthOf(5) + + await completeCheckHlsPlaylist({ videoUUID, hlsOnly: true, servers, resolutions: [ 720, 480, 360, 240, 144 ] }) + } + }) + + it('Should not have available jobs anymore', async function () { + const { availableJobs } = await servers[0].runnerJobs.requestVOD({ runnerToken }) + expect(availableJobs).to.have.lengthOf(0) + }) + }) + + describe('Web video and HLS transcoding', function () { + + before(async function () { + this.timeout(60000) + + await servers[0].config.enableTranscoding({ hls: true, webVideo: true }) + + await servers[0].videos.quickUpload({ name: 'web video and hls video', fixture: 'video_short.webm' }) + + await waitJobs(servers) + }) + + it('Should process the first optimize job', async function () { + this.timeout(60000) + + await servers[0].runnerJobs.autoProcessWebVideoJob(runnerToken) + }) + + it('Should have 9 jobs to process', async function () { + const { availableJobs } = await servers[0].runnerJobs.requestVOD({ runnerToken }) + + expect(availableJobs).to.have.lengthOf(9) + + const webVideoJobs = availableJobs.filter(j => j.type === 'vod-web-video-transcoding') + const hlsJobs = availableJobs.filter(j => j.type === 'vod-hls-transcoding') + + expect(webVideoJobs).to.have.lengthOf(4) + expect(hlsJobs).to.have.lengthOf(5) + }) + + it('Should process all available jobs', async function () { + await processAllJobs(servers[0], runnerToken) + }) + }) + + describe('Audio merge transcoding', function () { + let videoUUID: string + let jobToken: string + let jobUUID: string + + before(async function () { + this.timeout(60000) + + await servers[0].config.enableTranscoding({ hls: true, webVideo: true }) + + const attributes = { name: 'audio_with_preview', previewfile: 'custom-preview.jpg', fixture: 'sample.ogg' } + const { uuid } = await servers[0].videos.upload({ attributes, mode: 'legacy' }) + videoUUID = uuid + + await waitJobs(servers) + }) + + it('Should have an audio merge transcoding job', async function () { + const { availableJobs } = await servers[0].runnerJobs.requestVOD({ runnerToken }) + expect(availableJobs).to.have.lengthOf(1) + + expect(availableJobs[0].type).to.equal('vod-audio-merge-transcoding') + + jobUUID = availableJobs[0].uuid + }) + + it('Should have a valid remote audio merge transcoding job', async function () { + const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) + jobToken = job.jobToken + + expect(job.type === 'vod-audio-merge-transcoding') + expect(job.payload.input.audioFileUrl).to.exist + expect(job.payload.input.previewFileUrl).to.exist + expect(job.payload.output.resolution).to.equal(480) + + { + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.audioFileUrl, jobToken, runnerToken }) + const inputFile = await readFile(buildAbsoluteFixturePath('sample.ogg')) + expect(body).to.deep.equal(inputFile) + } + + { + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.previewFileUrl, jobToken, runnerToken }) + + const video = await servers[0].videos.get({ id: videoUUID }) + const { body: inputFile } = await makeGetRequest({ + url: servers[0].url, + path: video.previewPath, + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(body).to.deep.equal(inputFile) + } + }) + + it('Should merge the audio', async function () { + this.timeout(60000) + + const payload: VODAudioMergeTranscodingSuccess = { videoFile: 'video_short_480p.mp4' } + await servers[0].runnerJobs.success({ runnerToken, jobUUID, jobToken, payload }) + + await waitJobs(servers) + }) + + it('Should have the video updated', async function () { + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + expect(video.files).to.have.lengthOf(1) + expect(video.streamingPlaylists).to.have.lengthOf(0) + + const { body } = await makeRawRequest({ url: video.files[0].fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + expect(body).to.deep.equal(await readFile(buildAbsoluteFixturePath('video_short_480p.mp4'))) + } + }) + + it('Should have 7 lower resolutions to transcode', async function () { + const { availableJobs } = await servers[0].runnerJobs.requestVOD({ runnerToken }) + expect(availableJobs).to.have.lengthOf(7) + + for (const resolution of [ 360, 240, 144 ]) { + const jobs = availableJobs.filter(j => j.payload.output.resolution === resolution) + expect(jobs).to.have.lengthOf(2) + } + + jobUUID = availableJobs.find(j => j.payload.output.resolution === 480).uuid + }) + + it('Should process one other job', async function () { + this.timeout(60000) + + const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) + jobToken = job.jobToken + + const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) + const inputFile = await readFile(buildAbsoluteFixturePath('video_short_480p.mp4')) + expect(body).to.deep.equal(inputFile) + + const payload: VODHLSTranscodingSuccess = { + videoFile: `video_short_480p.mp4`, + resolutionPlaylistFile: `video_short_480p.m3u8` + } + await servers[0].runnerJobs.success({ runnerToken, jobUUID, jobToken, payload }) + + await waitJobs(servers) + }) + + it('Should have the video updated', async function () { + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + + expect(video.files).to.have.lengthOf(1) + expect(video.streamingPlaylists).to.have.lengthOf(1) + + const hls = video.streamingPlaylists[0] + expect(hls.files).to.have.lengthOf(1) + + await completeCheckHlsPlaylist({ videoUUID, hlsOnly: false, servers, resolutions: [ 480 ] }) + } + }) + + it('Should process all available jobs', async function () { + await processAllJobs(servers[0], runnerToken) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/search/index.ts b/packages/tests/src/api/search/index.ts new file mode 100644 index 000000000..f4420261d --- /dev/null +++ b/packages/tests/src/api/search/index.ts @@ -0,0 +1,7 @@ +import './search-activitypub-video-playlists.js' +import './search-activitypub-video-channels.js' +import './search-activitypub-videos.js' +import './search-channels.js' +import './search-index.js' +import './search-playlists.js' +import './search-videos.js' diff --git a/packages/tests/src/api/search/search-activitypub-video-channels.ts b/packages/tests/src/api/search/search-activitypub-video-channels.ts new file mode 100644 index 000000000..b63f45b18 --- /dev/null +++ b/packages/tests/src/api/search/search-activitypub-video-channels.ts @@ -0,0 +1,255 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { VideoChannel } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + PeerTubeServer, + SearchCommand, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test ActivityPub video channels search', function () { + let servers: PeerTubeServer[] + let userServer2Token: string + let videoServer2UUID: string + let channelIdServer2: number + let command: SearchCommand + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + await setDefaultAccountAvatar(servers) + + { + await servers[0].users.create({ username: 'user1_server1', password: 'password' }) + const channel = { + name: 'channel1_server1', + displayName: 'Channel 1 server 1' + } + await servers[0].channels.create({ attributes: channel }) + } + + { + const user = { username: 'user1_server2', password: 'password' } + await servers[1].users.create({ username: user.username, password: user.password }) + userServer2Token = await servers[1].login.getAccessToken(user) + + const channel = { + name: 'channel1_server2', + displayName: 'Channel 1 server 2' + } + const created = await servers[1].channels.create({ token: userServer2Token, attributes: channel }) + channelIdServer2 = created.id + + const attributes = { name: 'video 1 server 2', channelId: channelIdServer2 } + const { uuid } = await servers[1].videos.upload({ token: userServer2Token, attributes }) + videoServer2UUID = uuid + } + + await waitJobs(servers) + + command = servers[0].search + }) + + it('Should not find a remote video channel', async function () { + this.timeout(15000) + + { + const search = servers[1].url + '/video-channels/channel1_server3' + const body = await command.searchChannels({ search, token: servers[0].accessToken }) + + expect(body.total).to.equal(0) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(0) + } + + { + // Without token + const search = servers[1].url + '/video-channels/channel1_server2' + const body = await command.searchChannels({ search }) + + expect(body.total).to.equal(0) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(0) + } + }) + + it('Should search a local video channel', async function () { + const searches = [ + servers[0].url + '/video-channels/channel1_server1', + 'channel1_server1@' + servers[0].host + ] + + for (const search of searches) { + const body = await command.searchChannels({ search }) + + expect(body.total).to.equal(1) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].name).to.equal('channel1_server1') + expect(body.data[0].displayName).to.equal('Channel 1 server 1') + } + }) + + it('Should search a local video channel with an alternative URL', async function () { + const search = servers[0].url + '/c/channel1_server1' + + for (const token of [ undefined, servers[0].accessToken ]) { + const body = await command.searchChannels({ search, token }) + + expect(body.total).to.equal(1) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].name).to.equal('channel1_server1') + expect(body.data[0].displayName).to.equal('Channel 1 server 1') + } + }) + + it('Should search a local video channel with a query in URL', async function () { + const searches = [ + servers[0].url + '/video-channels/channel1_server1', + servers[0].url + '/c/channel1_server1' + ] + + for (const search of searches) { + for (const token of [ undefined, servers[0].accessToken ]) { + const body = await command.searchChannels({ search: search + '?param=2', token }) + + expect(body.total).to.equal(1) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].name).to.equal('channel1_server1') + expect(body.data[0].displayName).to.equal('Channel 1 server 1') + } + } + }) + + it('Should search a remote video channel with URL or handle', async function () { + const searches = [ + servers[1].url + '/video-channels/channel1_server2', + servers[1].url + '/c/channel1_server2', + servers[1].url + '/c/channel1_server2/videos', + 'channel1_server2@' + servers[1].host + ] + + for (const search of searches) { + const body = await command.searchChannels({ search, token: servers[0].accessToken }) + + expect(body.total).to.equal(1) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].name).to.equal('channel1_server2') + expect(body.data[0].displayName).to.equal('Channel 1 server 2') + } + }) + + it('Should not list this remote video channel', async function () { + const body = await servers[0].channels.list() + expect(body.total).to.equal(3) + expect(body.data).to.have.lengthOf(3) + expect(body.data[0].name).to.equal('channel1_server1') + expect(body.data[1].name).to.equal('user1_server1_channel') + expect(body.data[2].name).to.equal('root_channel') + }) + + it('Should list video channel videos of server 2 without token', async function () { + this.timeout(30000) + + await waitJobs(servers) + + const { total, data } = await servers[0].videos.listByChannel({ + token: null, + handle: 'channel1_server2@' + servers[1].host + }) + expect(total).to.equal(0) + expect(data).to.have.lengthOf(0) + }) + + it('Should list video channel videos of server 2 with token', async function () { + const { total, data } = await servers[0].videos.listByChannel({ + handle: 'channel1_server2@' + servers[1].host + }) + + expect(total).to.equal(1) + expect(data[0].name).to.equal('video 1 server 2') + }) + + it('Should update video channel of server 2, and refresh it on server 1', async function () { + this.timeout(120000) + + await servers[1].channels.update({ + token: userServer2Token, + channelName: 'channel1_server2', + attributes: { displayName: 'channel updated' } + }) + await servers[1].users.updateMe({ token: userServer2Token, displayName: 'user updated' }) + + await waitJobs(servers) + // Expire video channel + await wait(10000) + + const search = servers[1].url + '/video-channels/channel1_server2' + const body = await command.searchChannels({ search, token: servers[0].accessToken }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + + const videoChannel: VideoChannel = body.data[0] + expect(videoChannel.displayName).to.equal('channel updated') + + // We don't return the owner account for now + // expect(videoChannel.ownerAccount.displayName).to.equal('user updated') + }) + + it('Should update and add a video on server 2, and update it on server 1 after a search', async function () { + this.timeout(120000) + + await servers[1].videos.update({ token: userServer2Token, id: videoServer2UUID, attributes: { name: 'video 1 updated' } }) + await servers[1].videos.upload({ token: userServer2Token, attributes: { name: 'video 2 server 2', channelId: channelIdServer2 } }) + + await waitJobs(servers) + + // Expire video channel + await wait(10000) + + const search = servers[1].url + '/video-channels/channel1_server2' + await command.searchChannels({ search, token: servers[0].accessToken }) + + await waitJobs(servers) + + const handle = 'channel1_server2@' + servers[1].host + const { total, data } = await servers[0].videos.listByChannel({ handle, sort: '-createdAt' }) + + expect(total).to.equal(2) + expect(data[0].name).to.equal('video 2 server 2') + expect(data[1].name).to.equal('video 1 updated') + }) + + it('Should delete video channel of server 2, and delete it on server 1', async function () { + this.timeout(120000) + + await servers[1].channels.delete({ token: userServer2Token, channelName: 'channel1_server2' }) + + await waitJobs(servers) + // Expire video + await wait(10000) + + const search = servers[1].url + '/video-channels/channel1_server2' + const body = await command.searchChannels({ search, token: servers[0].accessToken }) + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/search/search-activitypub-video-playlists.ts b/packages/tests/src/api/search/search-activitypub-video-playlists.ts new file mode 100644 index 000000000..33ecfd8e7 --- /dev/null +++ b/packages/tests/src/api/search/search-activitypub-video-playlists.ts @@ -0,0 +1,214 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { VideoPlaylistPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + PeerTubeServer, + SearchCommand, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test ActivityPub playlists search', function () { + let servers: PeerTubeServer[] + let playlistServer1UUID: string + let playlistServer2UUID: string + let video2Server2: string + + let command: SearchCommand + + before(async function () { + this.timeout(240000) + + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + await setDefaultAccountAvatar(servers) + + { + const video1 = (await servers[0].videos.quickUpload({ name: 'video 1' })).uuid + const video2 = (await servers[0].videos.quickUpload({ name: 'video 2' })).uuid + + const attributes = { + displayName: 'playlist 1 on server 1', + privacy: VideoPlaylistPrivacy.PUBLIC, + videoChannelId: servers[0].store.channel.id + } + const created = await servers[0].playlists.create({ attributes }) + playlistServer1UUID = created.uuid + + for (const videoId of [ video1, video2 ]) { + await servers[0].playlists.addElement({ playlistId: playlistServer1UUID, attributes: { videoId } }) + } + } + + { + const videoId = (await servers[1].videos.quickUpload({ name: 'video 1' })).uuid + video2Server2 = (await servers[1].videos.quickUpload({ name: 'video 2' })).uuid + + const attributes = { + displayName: 'playlist 1 on server 2', + privacy: VideoPlaylistPrivacy.PUBLIC, + videoChannelId: servers[1].store.channel.id + } + const created = await servers[1].playlists.create({ attributes }) + playlistServer2UUID = created.uuid + + await servers[1].playlists.addElement({ playlistId: playlistServer2UUID, attributes: { videoId } }) + } + + await waitJobs(servers) + + command = servers[0].search + }) + + it('Should not find a remote playlist', async function () { + { + const search = servers[1].url + '/video-playlists/43' + const body = await command.searchPlaylists({ search, token: servers[0].accessToken }) + + expect(body.total).to.equal(0) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(0) + } + + { + // Without token + const search = servers[1].url + '/video-playlists/' + playlistServer2UUID + const body = await command.searchPlaylists({ search }) + + expect(body.total).to.equal(0) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(0) + } + }) + + it('Should search a local playlist', async function () { + const search = servers[0].url + '/video-playlists/' + playlistServer1UUID + const body = await command.searchPlaylists({ search }) + + expect(body.total).to.equal(1) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].displayName).to.equal('playlist 1 on server 1') + expect(body.data[0].videosLength).to.equal(2) + }) + + it('Should search a local playlist with an alternative URL', async function () { + const searches = [ + servers[0].url + '/videos/watch/playlist/' + playlistServer1UUID, + servers[0].url + '/w/p/' + playlistServer1UUID + ] + + for (const search of searches) { + for (const token of [ undefined, servers[0].accessToken ]) { + const body = await command.searchPlaylists({ search, token }) + + expect(body.total).to.equal(1) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].displayName).to.equal('playlist 1 on server 1') + expect(body.data[0].videosLength).to.equal(2) + } + } + }) + + it('Should search a local playlist with a query in URL', async function () { + const searches = [ + servers[0].url + '/videos/watch/playlist/' + playlistServer1UUID, + servers[0].url + '/w/p/' + playlistServer1UUID + ] + + for (const search of searches) { + for (const token of [ undefined, servers[0].accessToken ]) { + const body = await command.searchPlaylists({ search: search + '?param=1', token }) + + expect(body.total).to.equal(1) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].displayName).to.equal('playlist 1 on server 1') + expect(body.data[0].videosLength).to.equal(2) + } + } + }) + + it('Should search a remote playlist', async function () { + const searches = [ + servers[1].url + '/video-playlists/' + playlistServer2UUID, + servers[1].url + '/videos/watch/playlist/' + playlistServer2UUID, + servers[1].url + '/w/p/' + playlistServer2UUID + ] + + for (const search of searches) { + const body = await command.searchPlaylists({ search, token: servers[0].accessToken }) + + expect(body.total).to.equal(1) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].displayName).to.equal('playlist 1 on server 2') + expect(body.data[0].videosLength).to.equal(1) + } + }) + + it('Should not list this remote playlist', async function () { + const body = await servers[0].playlists.list({ start: 0, count: 10 }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].displayName).to.equal('playlist 1 on server 1') + }) + + it('Should update the playlist of server 2, and refresh it on server 1', async function () { + this.timeout(60000) + + await servers[1].playlists.addElement({ playlistId: playlistServer2UUID, attributes: { videoId: video2Server2 } }) + + await waitJobs(servers) + // Expire playlist + await wait(10000) + + // Will run refresh async + const search = servers[1].url + '/video-playlists/' + playlistServer2UUID + await command.searchPlaylists({ search, token: servers[0].accessToken }) + + // Wait refresh + await wait(5000) + + const body = await command.searchPlaylists({ search, token: servers[0].accessToken }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + + const playlist = body.data[0] + expect(playlist.videosLength).to.equal(2) + }) + + it('Should delete playlist of server 2, and delete it on server 1', async function () { + this.timeout(60000) + + await servers[1].playlists.delete({ playlistId: playlistServer2UUID }) + + await waitJobs(servers) + // Expiration + await wait(10000) + + // Will run refresh async + const search = servers[1].url + '/video-playlists/' + playlistServer2UUID + await command.searchPlaylists({ search, token: servers[0].accessToken }) + + // Wait refresh + await wait(5000) + + const body = await command.searchPlaylists({ search, token: servers[0].accessToken }) + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/search/search-activitypub-videos.ts b/packages/tests/src/api/search/search-activitypub-videos.ts new file mode 100644 index 000000000..72759f21e --- /dev/null +++ b/packages/tests/src/api/search/search-activitypub-videos.ts @@ -0,0 +1,196 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + PeerTubeServer, + SearchCommand, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test ActivityPub videos search', function () { + let servers: PeerTubeServer[] + let videoServer1UUID: string + let videoServer2UUID: string + + let command: SearchCommand + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + await setDefaultAccountAvatar(servers) + + { + const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video 1 on server 1' } }) + videoServer1UUID = uuid + } + + { + const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video 1 on server 2' } }) + videoServer2UUID = uuid + } + + await waitJobs(servers) + + command = servers[0].search + }) + + it('Should not find a remote video', async function () { + { + const search = servers[1].url + '/videos/watch/43' + const body = await command.searchVideos({ search, token: servers[0].accessToken }) + + expect(body.total).to.equal(0) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(0) + } + + { + // Without token + const search = servers[1].url + '/videos/watch/' + videoServer2UUID + const body = await command.searchVideos({ search }) + + expect(body.total).to.equal(0) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(0) + } + }) + + it('Should search a local video', async function () { + const search = servers[0].url + '/videos/watch/' + videoServer1UUID + const body = await command.searchVideos({ search }) + + expect(body.total).to.equal(1) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].name).to.equal('video 1 on server 1') + }) + + it('Should search a local video with an alternative URL', async function () { + const search = servers[0].url + '/w/' + videoServer1UUID + const body1 = await command.searchVideos({ search }) + const body2 = await command.searchVideos({ search, token: servers[0].accessToken }) + + for (const body of [ body1, body2 ]) { + expect(body.total).to.equal(1) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].name).to.equal('video 1 on server 1') + } + }) + + it('Should search a local video with a query in URL', async function () { + const searches = [ + servers[0].url + '/w/' + videoServer1UUID, + servers[0].url + '/videos/watch/' + videoServer1UUID + ] + + for (const search of searches) { + for (const token of [ undefined, servers[0].accessToken ]) { + const body = await command.searchVideos({ search: search + '?startTime=4', token }) + + expect(body.total).to.equal(1) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].name).to.equal('video 1 on server 1') + } + } + }) + + it('Should search a remote video', async function () { + const searches = [ + servers[1].url + '/w/' + videoServer2UUID, + servers[1].url + '/videos/watch/' + videoServer2UUID + ] + + for (const search of searches) { + const body = await command.searchVideos({ search, token: servers[0].accessToken }) + + expect(body.total).to.equal(1) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].name).to.equal('video 1 on server 2') + } + }) + + it('Should not list this remote video', async function () { + const { total, data } = await servers[0].videos.list() + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + expect(data[0].name).to.equal('video 1 on server 1') + }) + + it('Should update video of server 2, and refresh it on server 1', async function () { + this.timeout(120000) + + const channelAttributes = { + name: 'super_channel', + displayName: 'super channel' + } + const created = await servers[1].channels.create({ attributes: channelAttributes }) + const videoChannelId = created.id + + const attributes = { + name: 'updated', + tag: [ 'tag1', 'tag2' ], + privacy: VideoPrivacy.UNLISTED, + channelId: videoChannelId + } + await servers[1].videos.update({ id: videoServer2UUID, attributes }) + + await waitJobs(servers) + // Expire video + await wait(10000) + + // Will run refresh async + const search = servers[1].url + '/videos/watch/' + videoServer2UUID + await command.searchVideos({ search, token: servers[0].accessToken }) + + // Wait refresh + await wait(5000) + + const body = await command.searchVideos({ search, token: servers[0].accessToken }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + + const video = body.data[0] + expect(video.name).to.equal('updated') + expect(video.channel.name).to.equal('super_channel') + expect(video.privacy.id).to.equal(VideoPrivacy.UNLISTED) + }) + + it('Should delete video of server 2, and delete it on server 1', async function () { + this.timeout(120000) + + await servers[1].videos.remove({ id: videoServer2UUID }) + + await waitJobs(servers) + // Expire video + await wait(10000) + + // Will run refresh async + const search = servers[1].url + '/videos/watch/' + videoServer2UUID + await command.searchVideos({ search, token: servers[0].accessToken }) + + // Wait refresh + await wait(5000) + + const body = await command.searchVideos({ search, token: servers[0].accessToken }) + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/search/search-channels.ts b/packages/tests/src/api/search/search-channels.ts new file mode 100644 index 000000000..36596e036 --- /dev/null +++ b/packages/tests/src/api/search/search-channels.ts @@ -0,0 +1,159 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { VideoChannel } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + doubleFollow, + PeerTubeServer, + SearchCommand, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultChannelAvatar +} from '@peertube/peertube-server-commands' + +describe('Test channels search', function () { + let server: PeerTubeServer + let remoteServer: PeerTubeServer + let command: SearchCommand + + before(async function () { + this.timeout(120000) + + const servers = await Promise.all([ + createSingleServer(1), + createSingleServer(2) + ]) + server = servers[0] + remoteServer = servers[1] + + await setAccessTokensToServers([ server, remoteServer ]) + await setDefaultChannelAvatar(server) + await setDefaultAccountAvatar(server) + + await servers[1].config.disableTranscoding() + + { + await server.users.create({ username: 'user1' }) + const channel = { + name: 'squall_channel', + displayName: 'Squall channel' + } + await server.channels.create({ attributes: channel }) + } + + { + await remoteServer.users.create({ username: 'user1' }) + const channel = { + name: 'zell_channel', + displayName: 'Zell channel' + } + const { id } = await remoteServer.channels.create({ attributes: channel }) + + await remoteServer.videos.upload({ attributes: { channelId: id } }) + } + + await doubleFollow(server, remoteServer) + + command = server.search + }) + + it('Should make a simple search and not have results', async function () { + const body = await command.searchChannels({ search: 'abc' }) + + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + }) + + it('Should make a search and have results', async function () { + { + const search = { + search: 'Squall', + start: 0, + count: 1 + } + const body = await command.advancedChannelSearch({ search }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + + const channel: VideoChannel = body.data[0] + expect(channel.name).to.equal('squall_channel') + expect(channel.displayName).to.equal('Squall channel') + } + + { + const search = { + search: 'Squall', + start: 1, + count: 1 + } + + const body = await command.advancedChannelSearch({ search }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(0) + } + }) + + it('Should filter by host', async function () { + { + const search = { search: 'channel', host: remoteServer.host } + + const body = await command.advancedChannelSearch({ search }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].displayName).to.equal('Zell channel') + } + + { + const search = { search: 'Sq', host: server.host } + + const body = await command.advancedChannelSearch({ search }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].displayName).to.equal('Squall channel') + } + + { + const search = { search: 'Squall', host: 'example.com' } + + const body = await command.advancedChannelSearch({ search }) + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + } + }) + + it('Should filter by names', async function () { + { + const body = await command.advancedChannelSearch({ search: { handles: [ 'squall_channel', 'zell_channel' ] } }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].displayName).to.equal('Squall channel') + } + + { + const body = await command.advancedChannelSearch({ search: { handles: [ 'squall_channel@' + server.host ] } }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].displayName).to.equal('Squall channel') + } + + { + const body = await command.advancedChannelSearch({ search: { handles: [ 'chocobozzz_channel' ] } }) + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + } + + { + const body = await command.advancedChannelSearch({ search: { handles: [ 'squall_channel', 'zell_channel@' + remoteServer.host ] } }) + expect(body.total).to.equal(2) + expect(body.data).to.have.lengthOf(2) + expect(body.data[0].displayName).to.equal('Squall channel') + expect(body.data[1].displayName).to.equal('Zell channel') + } + }) + + after(async function () { + await cleanupTests([ server, remoteServer ]) + }) +}) diff --git a/packages/tests/src/api/search/search-index.ts b/packages/tests/src/api/search/search-index.ts new file mode 100644 index 000000000..4bac7ea94 --- /dev/null +++ b/packages/tests/src/api/search/search-index.ts @@ -0,0 +1,438 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { + BooleanBothQuery, + VideoChannelsSearchQuery, + VideoPlaylistPrivacy, + VideoPlaylistsSearchQuery, + VideoPlaylistType, + VideosSearchQuery +} from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + PeerTubeServer, + SearchCommand, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test index search', function () { + const localVideoName = 'local video' + new Date().toISOString() + + let server: PeerTubeServer = null + let command: SearchCommand + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + + await server.videos.upload({ attributes: { name: localVideoName } }) + + command = server.search + }) + + describe('Default search', async function () { + + it('Should make a local videos search by default', async function () { + await server.config.updateCustomSubConfig({ + newConfig: { + search: { + searchIndex: { + enabled: true, + isDefaultSearch: false, + disableLocalSearch: false + } + } + } + }) + + const body = await command.searchVideos({ search: 'local video' }) + + expect(body.total).to.equal(1) + expect(body.data[0].name).to.equal(localVideoName) + }) + + it('Should make a local channels search by default', async function () { + const body = await command.searchChannels({ search: 'root' }) + + expect(body.total).to.equal(1) + expect(body.data[0].name).to.equal('root_channel') + expect(body.data[0].host).to.equal(server.host) + }) + + it('Should make an index videos search by default', async function () { + await server.config.updateCustomSubConfig({ + newConfig: { + search: { + searchIndex: { + enabled: true, + isDefaultSearch: true, + disableLocalSearch: false + } + } + } + }) + + const body = await command.searchVideos({ search: 'local video' }) + expect(body.total).to.be.greaterThan(2) + }) + + it('Should make an index channels search by default', async function () { + const body = await command.searchChannels({ search: 'root' }) + expect(body.total).to.be.greaterThan(2) + }) + }) + + describe('Videos search', async function () { + + async function check (search: VideosSearchQuery, exists = true) { + const body = await command.advancedVideoSearch({ search }) + + if (exists === false) { + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + return + } + + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + + const video = body.data[0] + + expect(video.name).to.equal('What is PeerTube?') + expect(video.category.label).to.equal('Science & Technology') + expect(video.licence.label).to.equal('Attribution - Share Alike') + expect(video.privacy.label).to.equal('Public') + expect(video.duration).to.equal(113) + expect(video.thumbnailUrl.startsWith('https://framatube.org/static/thumbnails')).to.be.true + + expect(video.account.host).to.equal('framatube.org') + expect(video.account.name).to.equal('framasoft') + expect(video.account.url).to.equal('https://framatube.org/accounts/framasoft') + expect(video.account.avatars.length).to.equal(2, 'Account should have one avatar image') + + expect(video.channel.host).to.equal('framatube.org') + expect(video.channel.name).to.equal('joinpeertube') + expect(video.channel.url).to.equal('https://framatube.org/video-channels/joinpeertube') + expect(video.channel.avatars.length).to.equal(2, 'Channel should have one avatar image') + } + + const baseSearch: VideosSearchQuery = { + search: 'what is peertube', + start: 0, + count: 2, + categoryOneOf: [ 15 ], + licenceOneOf: [ 2 ], + tagsAllOf: [ 'framasoft', 'peertube' ], + startDate: '2018-10-01T10:50:46.396Z', + endDate: '2018-10-01T10:55:46.396Z' + } + + it('Should make a simple search and not have results', async function () { + const body = await command.searchVideos({ search: 'djidane'.repeat(50) }) + + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + }) + + it('Should make a simple search and have results', async function () { + const body = await command.searchVideos({ search: 'What is PeerTube' }) + + expect(body.total).to.be.greaterThan(1) + }) + + it('Should make a simple search', async function () { + await check(baseSearch) + }) + + it('Should search by start date', async function () { + const search = { ...baseSearch, startDate: '2018-10-01T10:54:46.396Z' } + await check(search, false) + }) + + it('Should search by tags', async function () { + const search = { ...baseSearch, tagsAllOf: [ 'toto', 'framasoft' ] } + await check(search, false) + }) + + it('Should search by duration', async function () { + const search = { ...baseSearch, durationMin: 2000 } + await check(search, false) + }) + + it('Should search by nsfw attribute', async function () { + { + const search = { ...baseSearch, nsfw: 'true' as BooleanBothQuery } + await check(search, false) + } + + { + const search = { ...baseSearch, nsfw: 'false' as BooleanBothQuery } + await check(search, true) + } + + { + const search = { ...baseSearch, nsfw: 'both' as BooleanBothQuery } + await check(search, true) + } + }) + + it('Should search by host', async function () { + { + const search = { ...baseSearch, host: 'example.com' } + await check(search, false) + } + + { + const search = { ...baseSearch, host: 'framatube.org' } + await check(search, true) + } + }) + + it('Should search by uuids', async function () { + const goodUUID = '9c9de5e8-0a1e-484a-b099-e80766180a6d' + const goodShortUUID = 'kkGMgK9ZtnKfYAgnEtQxbv' + const badUUID = 'c29c5b77-4a04-493d-96a9-2e9267e308f0' + const badShortUUID = 'rP5RgUeX9XwTSrspCdkDej' + + { + const uuidsMatrix = [ + [ goodUUID ], + [ goodUUID, badShortUUID ], + [ badShortUUID, goodShortUUID ], + [ goodUUID, goodShortUUID ] + ] + + for (const uuids of uuidsMatrix) { + const search = { ...baseSearch, uuids } + await check(search, true) + } + } + + { + const uuidsMatrix = [ + [ badUUID ], + [ badShortUUID ] + ] + + for (const uuids of uuidsMatrix) { + const search = { ...baseSearch, uuids } + await check(search, false) + } + } + }) + + it('Should have a correct pagination', async function () { + const search = { + search: 'video', + start: 0, + count: 5 + } + + const body = await command.advancedVideoSearch({ search }) + + expect(body.total).to.be.greaterThan(5) + expect(body.data).to.have.lengthOf(5) + }) + + it('Should use the nsfw instance policy as default', async function () { + let nsfwUUID: string + + { + await server.config.updateCustomSubConfig({ + newConfig: { + instance: { defaultNSFWPolicy: 'display' } + } + }) + + const body = await command.searchVideos({ search: 'NSFW search index', sort: '-match' }) + expect(body.data).to.have.length.greaterThan(0) + + const video = body.data[0] + expect(video.nsfw).to.be.true + + nsfwUUID = video.uuid + } + + { + await server.config.updateCustomSubConfig({ + newConfig: { + instance: { defaultNSFWPolicy: 'do_not_list' } + } + }) + + const body = await command.searchVideos({ search: 'NSFW search index', sort: '-match' }) + + try { + expect(body.data).to.have.lengthOf(0) + } catch { + const video = body.data[0] + + expect(video.uuid).not.equal(nsfwUUID) + } + } + }) + }) + + describe('Channels search', async function () { + + async function check (search: VideoChannelsSearchQuery, exists = true) { + const body = await command.advancedChannelSearch({ search }) + + if (exists === false) { + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + return + } + + expect(body.total).to.be.greaterThan(0) + expect(body.data).to.have.length.greaterThan(0) + + const videoChannel = body.data[0] + expect(videoChannel.url).to.equal('https://framatube.org/video-channels/bf54d359-cfad-4935-9d45-9d6be93f63e8') + expect(videoChannel.host).to.equal('framatube.org') + expect(videoChannel.avatars.length).to.equal(2, 'Channel should have two avatar images') + expect(videoChannel.displayName).to.exist + + expect(videoChannel.ownerAccount.url).to.equal('https://framatube.org/accounts/framasoft') + expect(videoChannel.ownerAccount.name).to.equal('framasoft') + expect(videoChannel.ownerAccount.host).to.equal('framatube.org') + expect(videoChannel.ownerAccount.avatars.length).to.equal(2, 'Account should have two avatar images') + } + + it('Should make a simple search and not have results', async function () { + const body = await command.searchChannels({ search: 'a'.repeat(500) }) + + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + }) + + it('Should make a search and have results', async function () { + await check({ search: 'Framasoft', sort: 'createdAt' }, true) + }) + + it('Should make host search and have appropriate results', async function () { + await check({ search: 'Framasoft videos', host: 'example.com' }, false) + await check({ search: 'Framasoft videos', host: 'framatube.org' }, true) + }) + + it('Should make handles search and have appropriate results', async function () { + await check({ handles: [ 'bf54d359-cfad-4935-9d45-9d6be93f63e8@framatube.org' ] }, true) + await check({ handles: [ 'jeanine', 'bf54d359-cfad-4935-9d45-9d6be93f63e8@framatube.org' ] }, true) + await check({ handles: [ 'jeanine', 'chocobozzz_channel2@peertube2.cpy.re' ] }, false) + }) + + it('Should have a correct pagination', async function () { + const body = await command.advancedChannelSearch({ search: { search: 'root', start: 0, count: 2 } }) + + expect(body.total).to.be.greaterThan(2) + expect(body.data).to.have.lengthOf(2) + }) + }) + + describe('Playlists search', async function () { + + async function check (search: VideoPlaylistsSearchQuery, exists = true) { + const body = await command.advancedPlaylistSearch({ search }) + + if (exists === false) { + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + return + } + + expect(body.total).to.be.greaterThan(0) + expect(body.data).to.have.length.greaterThan(0) + + const videoPlaylist = body.data[0] + + expect(videoPlaylist.url).to.equal('https://peertube2.cpy.re/videos/watch/playlist/73804a40-da9a-40c2-b1eb-2c6d9eec8f0a') + expect(videoPlaylist.thumbnailUrl).to.exist + expect(videoPlaylist.embedUrl).to.equal('https://peertube2.cpy.re/video-playlists/embed/73804a40-da9a-40c2-b1eb-2c6d9eec8f0a') + + expect(videoPlaylist.type.id).to.equal(VideoPlaylistType.REGULAR) + expect(videoPlaylist.privacy.id).to.equal(VideoPlaylistPrivacy.PUBLIC) + expect(videoPlaylist.videosLength).to.exist + + expect(videoPlaylist.createdAt).to.exist + expect(videoPlaylist.updatedAt).to.exist + + expect(videoPlaylist.uuid).to.equal('73804a40-da9a-40c2-b1eb-2c6d9eec8f0a') + expect(videoPlaylist.displayName).to.exist + + expect(videoPlaylist.ownerAccount.url).to.equal('https://peertube2.cpy.re/accounts/chocobozzz') + expect(videoPlaylist.ownerAccount.name).to.equal('chocobozzz') + expect(videoPlaylist.ownerAccount.host).to.equal('peertube2.cpy.re') + expect(videoPlaylist.ownerAccount.avatars.length).to.equal(2, 'Account should have two avatar images') + + expect(videoPlaylist.videoChannel.url).to.equal('https://peertube2.cpy.re/video-channels/chocobozzz_channel') + expect(videoPlaylist.videoChannel.name).to.equal('chocobozzz_channel') + expect(videoPlaylist.videoChannel.host).to.equal('peertube2.cpy.re') + expect(videoPlaylist.videoChannel.avatars.length).to.equal(2, 'Channel should have two avatar images') + } + + it('Should make a simple search and not have results', async function () { + const body = await command.searchPlaylists({ search: 'a'.repeat(500) }) + + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + }) + + it('Should make a search and have results', async function () { + await check({ search: 'E2E playlist', sort: '-match' }, true) + }) + + it('Should make host search and have appropriate results', async function () { + await check({ search: 'E2E playlist', host: 'example.com' }, false) + await check({ search: 'E2E playlist', host: 'peertube2.cpy.re', sort: '-match' }, true) + }) + + it('Should make a search by uuids and have appropriate results', async function () { + const goodUUID = '73804a40-da9a-40c2-b1eb-2c6d9eec8f0a' + const goodShortUUID = 'fgei1ws1oa6FCaJ2qZPG29' + const badUUID = 'c29c5b77-4a04-493d-96a9-2e9267e308f0' + const badShortUUID = 'rP5RgUeX9XwTSrspCdkDej' + + { + const uuidsMatrix = [ + [ goodUUID ], + [ goodUUID, badShortUUID ], + [ badShortUUID, goodShortUUID ], + [ goodUUID, goodShortUUID ] + ] + + for (const uuids of uuidsMatrix) { + const search = { search: 'E2E playlist', sort: '-match', uuids } + await check(search, true) + } + } + + { + const uuidsMatrix = [ + [ badUUID ], + [ badShortUUID ] + ] + + for (const uuids of uuidsMatrix) { + const search = { search: 'E2E playlist', sort: '-match', uuids } + await check(search, false) + } + } + }) + + it('Should have a correct pagination', async function () { + const body = await command.advancedChannelSearch({ search: { search: 'root', start: 0, count: 2 } }) + + expect(body.total).to.be.greaterThan(2) + expect(body.data).to.have.lengthOf(2) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/search/search-playlists.ts b/packages/tests/src/api/search/search-playlists.ts new file mode 100644 index 000000000..cd16e202e --- /dev/null +++ b/packages/tests/src/api/search/search-playlists.ts @@ -0,0 +1,180 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { VideoPlaylistPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + doubleFollow, + PeerTubeServer, + SearchCommand, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultChannelAvatar, + setDefaultVideoChannel +} from '@peertube/peertube-server-commands' + +describe('Test playlists search', function () { + let server: PeerTubeServer + let remoteServer: PeerTubeServer + let command: SearchCommand + let playlistUUID: string + let playlistShortUUID: string + + before(async function () { + this.timeout(120000) + + const servers = await Promise.all([ + createSingleServer(1), + createSingleServer(2) + ]) + server = servers[0] + remoteServer = servers[1] + + await setAccessTokensToServers([ remoteServer, server ]) + await setDefaultVideoChannel([ remoteServer, server ]) + await setDefaultChannelAvatar([ remoteServer, server ]) + await setDefaultAccountAvatar([ remoteServer, server ]) + + await servers[1].config.disableTranscoding() + + { + const videoId = (await server.videos.upload()).uuid + + const attributes = { + displayName: 'Dr. Kenzo Tenma hospital videos', + privacy: VideoPlaylistPrivacy.PUBLIC, + videoChannelId: server.store.channel.id + } + const created = await server.playlists.create({ attributes }) + playlistUUID = created.uuid + playlistShortUUID = created.shortUUID + + await server.playlists.addElement({ playlistId: created.id, attributes: { videoId } }) + } + + { + const videoId = (await remoteServer.videos.upload()).uuid + + const attributes = { + displayName: 'Johan & Anna Libert music videos', + privacy: VideoPlaylistPrivacy.PUBLIC, + videoChannelId: remoteServer.store.channel.id + } + const created = await remoteServer.playlists.create({ attributes }) + + await remoteServer.playlists.addElement({ playlistId: created.id, attributes: { videoId } }) + } + + { + const attributes = { + displayName: 'Inspector Lunge playlist', + privacy: VideoPlaylistPrivacy.PUBLIC, + videoChannelId: server.store.channel.id + } + await server.playlists.create({ attributes }) + } + + await doubleFollow(server, remoteServer) + + command = server.search + }) + + it('Should make a simple search and not have results', async function () { + const body = await command.searchPlaylists({ search: 'abc' }) + + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + }) + + it('Should make a search and have results', async function () { + { + const search = { + search: 'tenma', + start: 0, + count: 1 + } + const body = await command.advancedPlaylistSearch({ search }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + + const playlist = body.data[0] + expect(playlist.displayName).to.equal('Dr. Kenzo Tenma hospital videos') + expect(playlist.url).to.equal(server.url + '/video-playlists/' + playlist.uuid) + } + + { + const search = { + search: 'Anna Livert music', + start: 0, + count: 1 + } + const body = await command.advancedPlaylistSearch({ search }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + + const playlist = body.data[0] + expect(playlist.displayName).to.equal('Johan & Anna Libert music videos') + } + }) + + it('Should filter by host', async function () { + { + const search = { search: 'tenma', host: server.host } + const body = await command.advancedPlaylistSearch({ search }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + + const playlist = body.data[0] + expect(playlist.displayName).to.equal('Dr. Kenzo Tenma hospital videos') + } + + { + const search = { search: 'Anna', host: 'example.com' } + const body = await command.advancedPlaylistSearch({ search }) + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + } + + { + const search = { search: 'video', host: remoteServer.host } + const body = await command.advancedPlaylistSearch({ search }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + + const playlist = body.data[0] + expect(playlist.displayName).to.equal('Johan & Anna Libert music videos') + } + }) + + it('Should filter by UUIDs', async function () { + for (const uuid of [ playlistUUID, playlistShortUUID ]) { + const body = await command.advancedPlaylistSearch({ search: { uuids: [ uuid ] } }) + + expect(body.total).to.equal(1) + expect(body.data[0].displayName).to.equal('Dr. Kenzo Tenma hospital videos') + } + + { + const body = await command.advancedPlaylistSearch({ search: { uuids: [ 'dfd70b83-639f-4980-94af-304a56ab4b35' ] } }) + + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + } + }) + + it('Should not display playlists without videos', async function () { + const search = { + search: 'Lunge', + start: 0, + count: 1 + } + const body = await command.advancedPlaylistSearch({ search }) + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + }) + + after(async function () { + await cleanupTests([ server, remoteServer ]) + }) +}) diff --git a/packages/tests/src/api/search/search-videos.ts b/packages/tests/src/api/search/search-videos.ts new file mode 100644 index 000000000..0739f0886 --- /dev/null +++ b/packages/tests/src/api/search/search-videos.ts @@ -0,0 +1,568 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + doubleFollow, + PeerTubeServer, + SearchCommand, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultChannelAvatar, + setDefaultVideoChannel, + stopFfmpeg +} from '@peertube/peertube-server-commands' + +describe('Test videos search', function () { + let server: PeerTubeServer + let remoteServer: PeerTubeServer + let startDate: string + let videoUUID: string + let videoShortUUID: string + + let command: SearchCommand + + before(async function () { + this.timeout(360000) + + const servers = await Promise.all([ + createSingleServer(1), + createSingleServer(2) + ]) + server = servers[0] + remoteServer = servers[1] + + await setAccessTokensToServers([ server, remoteServer ]) + await setDefaultVideoChannel([ server, remoteServer ]) + await setDefaultChannelAvatar(server) + await setDefaultAccountAvatar(servers) + + { + const attributes1 = { + name: '1111 2222 3333', + fixture: '60fps_720p_small.mp4', // 2 seconds + category: 1, + licence: 1, + nsfw: false, + language: 'fr' + } + await server.videos.upload({ attributes: attributes1 }) + + const attributes2 = { ...attributes1, name: attributes1.name + ' - 2', fixture: 'video_short.mp4' } + await server.videos.upload({ attributes: attributes2 }) + + { + const attributes3 = { ...attributes1, name: attributes1.name + ' - 3', language: undefined } + const { id, uuid, shortUUID } = await server.videos.upload({ attributes: attributes3 }) + videoUUID = uuid + videoShortUUID = shortUUID + + await server.captions.add({ + language: 'en', + videoId: id, + fixture: 'subtitle-good2.vtt', + mimeType: 'application/octet-stream' + }) + + await server.captions.add({ + language: 'aa', + videoId: id, + fixture: 'subtitle-good2.vtt', + mimeType: 'application/octet-stream' + }) + } + + const attributes4 = { ...attributes1, name: attributes1.name + ' - 4', language: 'pl', nsfw: true } + await server.videos.upload({ attributes: attributes4 }) + + await wait(1000) + + startDate = new Date().toISOString() + + const attributes5 = { ...attributes1, name: attributes1.name + ' - 5', licence: 2, language: undefined } + await server.videos.upload({ attributes: attributes5 }) + + const attributes6 = { ...attributes1, name: attributes1.name + ' - 6', tags: [ 't1', 't2' ] } + await server.videos.upload({ attributes: attributes6 }) + + const attributes7 = { ...attributes1, name: attributes1.name + ' - 7', originallyPublishedAt: '2019-02-12T09:58:08.286Z' } + await server.videos.upload({ attributes: attributes7 }) + + const attributes8 = { ...attributes1, name: attributes1.name + ' - 8', licence: 4 } + await server.videos.upload({ attributes: attributes8 }) + } + + { + const attributes = { + name: '3333 4444 5555', + fixture: 'video_short.mp4', + category: 2, + licence: 2, + language: 'en' + } + await server.videos.upload({ attributes }) + + await server.videos.upload({ attributes: { ...attributes, name: attributes.name + ' duplicate' } }) + } + + { + const attributes = { + name: '6666 7777 8888', + fixture: 'video_short.mp4', + category: 3, + licence: 3, + language: 'pl' + } + await server.videos.upload({ attributes }) + } + + { + const attributes1 = { + name: '9999', + tags: [ 'aaaa', 'bbbb', 'cccc' ], + category: 1 + } + await server.videos.upload({ attributes: attributes1 }) + await server.videos.upload({ attributes: { ...attributes1, category: 2 } }) + + await server.videos.upload({ attributes: { ...attributes1, tags: [ 'cccc', 'dddd' ] } }) + await server.videos.upload({ attributes: { ...attributes1, tags: [ 'eeee', 'ffff' ] } }) + } + + { + const attributes1 = { + name: 'aaaa 2', + category: 1 + } + await server.videos.upload({ attributes: attributes1 }) + await server.videos.upload({ attributes: { ...attributes1, category: 2 } }) + } + + { + await remoteServer.videos.upload({ attributes: { name: 'remote video 1' } }) + await remoteServer.videos.upload({ attributes: { name: 'remote video 2' } }) + } + + await doubleFollow(server, remoteServer) + + command = server.search + }) + + it('Should make a simple search and not have results', async function () { + const body = await command.searchVideos({ search: 'abc' }) + + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + }) + + it('Should make a simple search and have results', async function () { + const body = await command.searchVideos({ search: '4444 5555 duplicate' }) + + expect(body.total).to.equal(2) + + const videos = body.data + expect(videos).to.have.lengthOf(2) + + // bestmatch + expect(videos[0].name).to.equal('3333 4444 5555 duplicate') + expect(videos[1].name).to.equal('3333 4444 5555') + }) + + it('Should make a search on tags too, and have results', async function () { + const search = { + search: 'aaaa', + categoryOneOf: [ 1 ] + } + const body = await command.advancedVideoSearch({ search }) + + expect(body.total).to.equal(2) + + const videos = body.data + expect(videos).to.have.lengthOf(2) + + // bestmatch + expect(videos[0].name).to.equal('aaaa 2') + expect(videos[1].name).to.equal('9999') + }) + + it('Should filter on tags without a search', async function () { + const search = { + tagsAllOf: [ 'bbbb' ] + } + const body = await command.advancedVideoSearch({ search }) + + expect(body.total).to.equal(2) + + const videos = body.data + expect(videos).to.have.lengthOf(2) + + expect(videos[0].name).to.equal('9999') + expect(videos[1].name).to.equal('9999') + }) + + it('Should filter on category without a search', async function () { + const search = { + categoryOneOf: [ 3 ] + } + const body = await command.advancedVideoSearch({ search }) + + expect(body.total).to.equal(1) + + const videos = body.data + expect(videos).to.have.lengthOf(1) + + expect(videos[0].name).to.equal('6666 7777 8888') + }) + + it('Should search by tags (one of)', async function () { + const query = { + search: '9999', + categoryOneOf: [ 1 ], + tagsOneOf: [ 'aAaa', 'ffff' ] + } + + { + const body = await command.advancedVideoSearch({ search: query }) + expect(body.total).to.equal(2) + } + + { + const body = await command.advancedVideoSearch({ search: { ...query, tagsOneOf: [ 'blabla' ] } }) + expect(body.total).to.equal(0) + } + }) + + it('Should search by tags (all of)', async function () { + const query = { + search: '9999', + categoryOneOf: [ 1 ], + tagsAllOf: [ 'CCcc' ] + } + + { + const body = await command.advancedVideoSearch({ search: query }) + expect(body.total).to.equal(2) + } + + { + const body = await command.advancedVideoSearch({ search: { ...query, tagsAllOf: [ 'blAbla' ] } }) + expect(body.total).to.equal(0) + } + + { + const body = await command.advancedVideoSearch({ search: { ...query, tagsAllOf: [ 'bbbb', 'CCCC' ] } }) + expect(body.total).to.equal(1) + } + }) + + it('Should search by category', async function () { + const query = { + search: '6666', + categoryOneOf: [ 3 ] + } + + { + const body = await command.advancedVideoSearch({ search: query }) + expect(body.total).to.equal(1) + expect(body.data[0].name).to.equal('6666 7777 8888') + } + + { + const body = await command.advancedVideoSearch({ search: { ...query, categoryOneOf: [ 2 ] } }) + expect(body.total).to.equal(0) + } + }) + + it('Should search by licence', async function () { + const query = { + search: '4444 5555', + licenceOneOf: [ 2 ] + } + + { + const body = await command.advancedVideoSearch({ search: query }) + expect(body.total).to.equal(2) + expect(body.data[0].name).to.equal('3333 4444 5555') + expect(body.data[1].name).to.equal('3333 4444 5555 duplicate') + } + + { + const body = await command.advancedVideoSearch({ search: { ...query, licenceOneOf: [ 3 ] } }) + expect(body.total).to.equal(0) + } + }) + + it('Should search by languages', async function () { + const query = { + search: '1111 2222 3333', + languageOneOf: [ 'pl', 'en' ] + } + + { + const body = await command.advancedVideoSearch({ search: query }) + expect(body.total).to.equal(2) + expect(body.data[0].name).to.equal('1111 2222 3333 - 3') + expect(body.data[1].name).to.equal('1111 2222 3333 - 4') + } + + { + const body = await command.advancedVideoSearch({ search: { ...query, languageOneOf: [ 'pl', 'en', '_unknown' ] } }) + expect(body.total).to.equal(3) + expect(body.data[0].name).to.equal('1111 2222 3333 - 3') + expect(body.data[1].name).to.equal('1111 2222 3333 - 4') + expect(body.data[2].name).to.equal('1111 2222 3333 - 5') + } + + { + const body = await command.advancedVideoSearch({ search: { ...query, languageOneOf: [ 'eo' ] } }) + expect(body.total).to.equal(0) + } + }) + + it('Should search by start date', async function () { + const query = { + search: '1111 2222 3333', + startDate + } + + const body = await command.advancedVideoSearch({ search: query }) + expect(body.total).to.equal(4) + + const videos = body.data + expect(videos[0].name).to.equal('1111 2222 3333 - 5') + expect(videos[1].name).to.equal('1111 2222 3333 - 6') + expect(videos[2].name).to.equal('1111 2222 3333 - 7') + expect(videos[3].name).to.equal('1111 2222 3333 - 8') + }) + + it('Should make an advanced search', async function () { + const query = { + search: '1111 2222 3333', + languageOneOf: [ 'pl', 'fr' ], + durationMax: 4, + nsfw: 'false' as 'false', + licenceOneOf: [ 1, 4 ] + } + + const body = await command.advancedVideoSearch({ search: query }) + expect(body.total).to.equal(4) + + const videos = body.data + expect(videos[0].name).to.equal('1111 2222 3333') + expect(videos[1].name).to.equal('1111 2222 3333 - 6') + expect(videos[2].name).to.equal('1111 2222 3333 - 7') + expect(videos[3].name).to.equal('1111 2222 3333 - 8') + }) + + it('Should make an advanced search and sort results', async function () { + const query = { + search: '1111 2222 3333', + languageOneOf: [ 'pl', 'fr' ], + durationMax: 4, + nsfw: 'false' as 'false', + licenceOneOf: [ 1, 4 ], + sort: '-name' + } + + const body = await command.advancedVideoSearch({ search: query }) + expect(body.total).to.equal(4) + + const videos = body.data + expect(videos[0].name).to.equal('1111 2222 3333 - 8') + expect(videos[1].name).to.equal('1111 2222 3333 - 7') + expect(videos[2].name).to.equal('1111 2222 3333 - 6') + expect(videos[3].name).to.equal('1111 2222 3333') + }) + + it('Should make an advanced search and only show the first result', async function () { + const query = { + search: '1111 2222 3333', + languageOneOf: [ 'pl', 'fr' ], + durationMax: 4, + nsfw: 'false' as 'false', + licenceOneOf: [ 1, 4 ], + sort: '-name', + start: 0, + count: 1 + } + + const body = await command.advancedVideoSearch({ search: query }) + expect(body.total).to.equal(4) + + const videos = body.data + expect(videos[0].name).to.equal('1111 2222 3333 - 8') + }) + + it('Should make an advanced search and only show the last result', async function () { + const query = { + search: '1111 2222 3333', + languageOneOf: [ 'pl', 'fr' ], + durationMax: 4, + nsfw: 'false' as 'false', + licenceOneOf: [ 1, 4 ], + sort: '-name', + start: 3, + count: 1 + } + + const body = await command.advancedVideoSearch({ search: query }) + expect(body.total).to.equal(4) + + const videos = body.data + expect(videos[0].name).to.equal('1111 2222 3333') + }) + + it('Should search on originally published date', async function () { + const baseQuery = { + search: '1111 2222 3333', + languageOneOf: [ 'pl', 'fr' ], + durationMax: 4, + nsfw: 'false' as 'false', + licenceOneOf: [ 1, 4 ] + } + + { + const query = { ...baseQuery, originallyPublishedStartDate: '2019-02-11T09:58:08.286Z' } + const body = await command.advancedVideoSearch({ search: query }) + + expect(body.total).to.equal(1) + expect(body.data[0].name).to.equal('1111 2222 3333 - 7') + } + + { + const query = { ...baseQuery, originallyPublishedEndDate: '2019-03-11T09:58:08.286Z' } + const body = await command.advancedVideoSearch({ search: query }) + + expect(body.total).to.equal(1) + expect(body.data[0].name).to.equal('1111 2222 3333 - 7') + } + + { + const query = { ...baseQuery, originallyPublishedEndDate: '2019-01-11T09:58:08.286Z' } + const body = await command.advancedVideoSearch({ search: query }) + + expect(body.total).to.equal(0) + } + + { + const query = { ...baseQuery, originallyPublishedStartDate: '2019-03-11T09:58:08.286Z' } + const body = await command.advancedVideoSearch({ search: query }) + + expect(body.total).to.equal(0) + } + + { + const query = { + ...baseQuery, + originallyPublishedStartDate: '2019-01-11T09:58:08.286Z', + originallyPublishedEndDate: '2019-01-10T09:58:08.286Z' + } + const body = await command.advancedVideoSearch({ search: query }) + + expect(body.total).to.equal(0) + } + + { + const query = { + ...baseQuery, + originallyPublishedStartDate: '2019-01-11T09:58:08.286Z', + originallyPublishedEndDate: '2019-04-11T09:58:08.286Z' + } + const body = await command.advancedVideoSearch({ search: query }) + + expect(body.total).to.equal(1) + expect(body.data[0].name).to.equal('1111 2222 3333 - 7') + } + }) + + it('Should search by UUID', async function () { + const search = videoUUID + const body = await command.advancedVideoSearch({ search: { search } }) + + expect(body.total).to.equal(1) + expect(body.data[0].name).to.equal('1111 2222 3333 - 3') + }) + + it('Should filter by UUIDs', async function () { + for (const uuid of [ videoUUID, videoShortUUID ]) { + const body = await command.advancedVideoSearch({ search: { uuids: [ uuid ] } }) + + expect(body.total).to.equal(1) + expect(body.data[0].name).to.equal('1111 2222 3333 - 3') + } + + { + const body = await command.advancedVideoSearch({ search: { uuids: [ 'dfd70b83-639f-4980-94af-304a56ab4b35' ] } }) + + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + } + }) + + it('Should search by host', async function () { + { + const body = await command.advancedVideoSearch({ search: { search: '6666 7777 8888', host: server.host } }) + expect(body.total).to.equal(1) + expect(body.data[0].name).to.equal('6666 7777 8888') + } + + { + const body = await command.advancedVideoSearch({ search: { search: '1111', host: 'example.com' } }) + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + } + + { + const body = await command.advancedVideoSearch({ search: { search: 'remote', host: remoteServer.host } }) + expect(body.total).to.equal(2) + expect(body.data).to.have.lengthOf(2) + expect(body.data[0].name).to.equal('remote video 1') + expect(body.data[1].name).to.equal('remote video 2') + } + }) + + it('Should search by live', async function () { + this.timeout(120000) + + { + const newConfig = { + search: { + searchIndex: { enabled: false } + }, + live: { enabled: true } + } + await server.config.updateCustomSubConfig({ newConfig }) + } + + { + const body = await command.advancedVideoSearch({ search: { isLive: true } }) + + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + } + + { + const liveCommand = server.live + + const liveAttributes = { name: 'live', privacy: VideoPrivacy.PUBLIC, channelId: server.store.channel.id } + const live = await liveCommand.create({ fields: liveAttributes }) + + const ffmpegCommand = await liveCommand.sendRTMPStreamInVideo({ videoId: live.id }) + await liveCommand.waitUntilPublished({ videoId: live.id }) + + const body = await command.advancedVideoSearch({ search: { isLive: true } }) + + expect(body.total).to.equal(1) + expect(body.data[0].name).to.equal('live') + + await stopFfmpeg(ffmpegCommand) + } + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/server/auto-follows.ts b/packages/tests/src/api/server/auto-follows.ts new file mode 100644 index 000000000..aa272ebcc --- /dev/null +++ b/packages/tests/src/api/server/auto-follows.ts @@ -0,0 +1,189 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { MockInstancesIndex } from '@tests/shared/mock-servers/index.js' +import { wait } from '@peertube/peertube-core-utils' +import { cleanupTests, createMultipleServers, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@peertube/peertube-server-commands' + +async function checkFollow (follower: PeerTubeServer, following: PeerTubeServer, exists: boolean) { + { + const body = await following.follows.getFollowers({ start: 0, count: 5, sort: '-createdAt' }) + const follow = body.data.find(f => f.follower.host === follower.host && f.state === 'accepted') + + if (exists === true) expect(follow, `Follower ${follower.url} should exist on ${following.url}`).to.exist + else expect(follow, `Follower ${follower.url} should not exist on ${following.url}`).to.be.undefined + } + + { + const body = await follower.follows.getFollowings({ start: 0, count: 5, sort: '-createdAt' }) + const follow = body.data.find(f => f.following.host === following.host && f.state === 'accepted') + + if (exists === true) expect(follow, `Following ${following.url} should exist on ${follower.url}`).to.exist + else expect(follow, `Following ${following.url} should not exist on ${follower.url}`).to.be.undefined + } +} + +async function server1Follows2 (servers: PeerTubeServer[]) { + await servers[0].follows.follow({ hosts: [ servers[1].host ] }) + + await waitJobs(servers) +} + +async function resetFollows (servers: PeerTubeServer[]) { + try { + await servers[0].follows.unfollow({ target: servers[1] }) + await servers[1].follows.unfollow({ target: servers[0] }) + } catch { /* empty */ + } + + await waitJobs(servers) + + await checkFollow(servers[0], servers[1], false) + await checkFollow(servers[1], servers[0], false) +} + +describe('Test auto follows', function () { + let servers: PeerTubeServer[] = [] + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(3) + + // Get the access tokens + await setAccessTokensToServers(servers) + }) + + describe('Auto follow back', function () { + + it('Should not auto follow back if the option is not enabled', async function () { + this.timeout(15000) + + await server1Follows2(servers) + + await checkFollow(servers[0], servers[1], true) + await checkFollow(servers[1], servers[0], false) + + await resetFollows(servers) + }) + + it('Should auto follow back on auto accept if the option is enabled', async function () { + this.timeout(15000) + + const config = { + followings: { + instance: { + autoFollowBack: { enabled: true } + } + } + } + await servers[1].config.updateCustomSubConfig({ newConfig: config }) + + await server1Follows2(servers) + + await checkFollow(servers[0], servers[1], true) + await checkFollow(servers[1], servers[0], true) + + await resetFollows(servers) + }) + + it('Should wait the acceptation before auto follow back', async function () { + this.timeout(30000) + + const config = { + followings: { + instance: { + autoFollowBack: { enabled: true } + } + }, + followers: { + instance: { + manualApproval: true + } + } + } + await servers[1].config.updateCustomSubConfig({ newConfig: config }) + + await server1Follows2(servers) + + await checkFollow(servers[0], servers[1], false) + await checkFollow(servers[1], servers[0], false) + + await servers[1].follows.acceptFollower({ follower: 'peertube@' + servers[0].host }) + await waitJobs(servers) + + await checkFollow(servers[0], servers[1], true) + await checkFollow(servers[1], servers[0], true) + + await resetFollows(servers) + + config.followings.instance.autoFollowBack.enabled = false + config.followers.instance.manualApproval = false + await servers[1].config.updateCustomSubConfig({ newConfig: config }) + }) + }) + + describe('Auto follow index', function () { + const instanceIndexServer = new MockInstancesIndex() + let port: number + + before(async function () { + port = await instanceIndexServer.initialize() + }) + + it('Should not auto follow index if the option is not enabled', async function () { + this.timeout(30000) + + await wait(5000) + await waitJobs(servers) + + await checkFollow(servers[0], servers[1], false) + await checkFollow(servers[1], servers[0], false) + }) + + it('Should auto follow the index', async function () { + this.timeout(30000) + + instanceIndexServer.addInstance(servers[1].host) + + const config = { + followings: { + instance: { + autoFollowIndex: { + indexUrl: `http://127.0.0.1:${port}/api/v1/instances/hosts`, + enabled: true + } + } + } + } + await servers[0].config.updateCustomSubConfig({ newConfig: config }) + + await wait(5000) + await waitJobs(servers) + + await checkFollow(servers[0], servers[1], true) + + await resetFollows(servers) + }) + + it('Should follow new added instances in the index but not old ones', async function () { + this.timeout(30000) + + instanceIndexServer.addInstance(servers[2].host) + + await wait(5000) + await waitJobs(servers) + + await checkFollow(servers[0], servers[1], false) + await checkFollow(servers[0], servers[2], true) + }) + + after(async function () { + await instanceIndexServer.terminate() + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/server/bulk.ts b/packages/tests/src/api/server/bulk.ts new file mode 100644 index 000000000..725bcfef2 --- /dev/null +++ b/packages/tests/src/api/server/bulk.ts @@ -0,0 +1,185 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { + BulkCommand, + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test bulk actions', function () { + const commentsUser3: { videoId: number, commentId: number }[] = [] + + let servers: PeerTubeServer[] = [] + let user1Token: string + let user2Token: string + let user3Token: string + + let bulkCommand: BulkCommand + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(2) + + // Get the access tokens + await setAccessTokensToServers(servers) + + { + const user = { username: 'user1', password: 'password' } + await servers[0].users.create({ username: user.username, password: user.password }) + + user1Token = await servers[0].login.getAccessToken(user) + } + + { + const user = { username: 'user2', password: 'password' } + await servers[0].users.create({ username: user.username, password: user.password }) + + user2Token = await servers[0].login.getAccessToken(user) + } + + { + const user = { username: 'user3', password: 'password' } + await servers[1].users.create({ username: user.username, password: user.password }) + + user3Token = await servers[1].login.getAccessToken(user) + } + + await doubleFollow(servers[0], servers[1]) + + bulkCommand = new BulkCommand(servers[0]) + }) + + describe('Bulk remove comments', function () { + async function checkInstanceCommentsRemoved () { + { + const { data } = await servers[0].videos.list() + + // Server 1 should not have these comments anymore + for (const video of data) { + const { data } = await servers[0].comments.listThreads({ videoId: video.id }) + const comment = data.find(c => c.text === 'comment by user 3') + + expect(comment).to.not.exist + } + } + + { + const { data } = await servers[1].videos.list() + + // Server 1 should not have these comments on videos of server 1 + for (const video of data) { + const { data } = await servers[1].comments.listThreads({ videoId: video.id }) + const comment = data.find(c => c.text === 'comment by user 3') + + if (video.account.host === servers[0].host) { + expect(comment).to.not.exist + } else { + expect(comment).to.exist + } + } + } + } + + before(async function () { + this.timeout(240000) + + await servers[0].videos.upload({ attributes: { name: 'video 1 server 1' } }) + await servers[0].videos.upload({ attributes: { name: 'video 2 server 1' } }) + await servers[0].videos.upload({ token: user1Token, attributes: { name: 'video 3 server 1' } }) + + await servers[1].videos.upload({ attributes: { name: 'video 1 server 2' } }) + + await waitJobs(servers) + + { + const { data } = await servers[0].videos.list() + for (const video of data) { + await servers[0].comments.createThread({ videoId: video.id, text: 'comment by root server 1' }) + await servers[0].comments.createThread({ token: user1Token, videoId: video.id, text: 'comment by user 1' }) + await servers[0].comments.createThread({ token: user2Token, videoId: video.id, text: 'comment by user 2' }) + } + } + + { + const { data } = await servers[1].videos.list() + + for (const video of data) { + await servers[1].comments.createThread({ videoId: video.id, text: 'comment by root server 2' }) + + const comment = await servers[1].comments.createThread({ token: user3Token, videoId: video.id, text: 'comment by user 3' }) + commentsUser3.push({ videoId: video.id, commentId: comment.id }) + } + } + + await waitJobs(servers) + }) + + it('Should delete comments of an account on my videos', async function () { + this.timeout(60000) + + await bulkCommand.removeCommentsOf({ + token: user1Token, + attributes: { + accountName: 'user2', + scope: 'my-videos' + } + }) + + await waitJobs(servers) + + for (const server of servers) { + const { data } = await server.videos.list() + + for (const video of data) { + const { data } = await server.comments.listThreads({ videoId: video.id }) + const comment = data.find(c => c.text === 'comment by user 2') + + if (video.name === 'video 3 server 1') expect(comment).to.not.exist + else expect(comment).to.exist + } + } + }) + + it('Should delete comments of an account on the instance', async function () { + this.timeout(60000) + + await bulkCommand.removeCommentsOf({ + attributes: { + accountName: 'user3@' + servers[1].host, + scope: 'instance' + } + }) + + await waitJobs(servers) + + await checkInstanceCommentsRemoved() + }) + + it('Should not re create the comment on video update', async function () { + this.timeout(60000) + + for (const obj of commentsUser3) { + await servers[1].comments.addReply({ + token: user3Token, + videoId: obj.videoId, + toCommentId: obj.commentId, + text: 'comment by user 3 bis' + }) + } + + await waitJobs(servers) + + await checkInstanceCommentsRemoved() + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/server/config-defaults.ts b/packages/tests/src/api/server/config-defaults.ts new file mode 100644 index 000000000..e874af012 --- /dev/null +++ b/packages/tests/src/api/server/config-defaults.ts @@ -0,0 +1,294 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { VideoDetails, VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel +} from '@peertube/peertube-server-commands' +import { FIXTURE_URLS } from '@tests/shared/tests.js' + +describe('Test config defaults', function () { + let server: PeerTubeServer + let channelId: number + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + channelId = server.store.channel.id + }) + + describe('Default publish values', function () { + + before(async function () { + const overrideConfig = { + defaults: { + publish: { + comments_enabled: false, + download_enabled: false, + privacy: VideoPrivacy.INTERNAL, + licence: 4 + } + } + } + + await server.kill() + await server.run(overrideConfig) + }) + + const attributes = { + name: 'video', + downloadEnabled: undefined, + commentsEnabled: undefined, + licence: undefined, + privacy: VideoPrivacy.PUBLIC // Privacy is mandatory for server + } + + function checkVideo (video: VideoDetails) { + expect(video.downloadEnabled).to.be.false + expect(video.commentsEnabled).to.be.false + expect(video.licence.id).to.equal(4) + } + + before(async function () { + await server.config.disableTranscoding() + await server.config.enableImports() + await server.config.enableLive({ allowReplay: false, transcoding: false }) + }) + + it('Should have the correct server configuration', async function () { + const config = await server.config.getConfig() + + expect(config.defaults.publish.commentsEnabled).to.be.false + expect(config.defaults.publish.downloadEnabled).to.be.false + expect(config.defaults.publish.licence).to.equal(4) + expect(config.defaults.publish.privacy).to.equal(VideoPrivacy.INTERNAL) + }) + + it('Should respect default values when uploading a video', async function () { + for (const mode of [ 'legacy' as 'legacy', 'resumable' as 'resumable' ]) { + const { id } = await server.videos.upload({ attributes, mode }) + + const video = await server.videos.get({ id }) + checkVideo(video) + } + }) + + it('Should respect default values when importing a video using URL', async function () { + const { video: { id } } = await server.imports.importVideo({ + attributes: { + ...attributes, + channelId, + targetUrl: FIXTURE_URLS.goodVideo + } + }) + + const video = await server.videos.get({ id }) + checkVideo(video) + }) + + it('Should respect default values when importing a video using magnet URI', async function () { + const { video: { id } } = await server.imports.importVideo({ + attributes: { + ...attributes, + channelId, + magnetUri: FIXTURE_URLS.magnet + } + }) + + const video = await server.videos.get({ id }) + checkVideo(video) + }) + + it('Should respect default values when creating a live', async function () { + const { id } = await server.live.create({ + fields: { + ...attributes, + channelId + } + }) + + const video = await server.videos.get({ id }) + checkVideo(video) + }) + }) + + describe('Default P2P values', function () { + + describe('Webapp default value', function () { + + before(async function () { + const overrideConfig = { + defaults: { + p2p: { + webapp: { + enabled: false + } + } + } + } + + await server.kill() + await server.run(overrideConfig) + }) + + it('Should have appropriate P2P config', async function () { + const config = await server.config.getConfig() + + expect(config.defaults.p2p.webapp.enabled).to.be.false + expect(config.defaults.p2p.embed.enabled).to.be.true + }) + + it('Should create a user with this default setting', async function () { + await server.users.create({ username: 'user_p2p_1' }) + const userToken = await server.login.getAccessToken('user_p2p_1') + + const { p2pEnabled } = await server.users.getMyInfo({ token: userToken }) + expect(p2pEnabled).to.be.false + }) + + it('Should register a user with this default setting', async function () { + await server.registrations.register({ username: 'user_p2p_2' }) + + const userToken = await server.login.getAccessToken('user_p2p_2') + + const { p2pEnabled } = await server.users.getMyInfo({ token: userToken }) + expect(p2pEnabled).to.be.false + }) + }) + + describe('Embed default value', function () { + + before(async function () { + const overrideConfig = { + defaults: { + p2p: { + embed: { + enabled: false + } + } + }, + signup: { + limit: 15 + } + } + + await server.kill() + await server.run(overrideConfig) + }) + + it('Should have appropriate P2P config', async function () { + const config = await server.config.getConfig() + + expect(config.defaults.p2p.webapp.enabled).to.be.true + expect(config.defaults.p2p.embed.enabled).to.be.false + }) + + it('Should create a user with this default setting', async function () { + await server.users.create({ username: 'user_p2p_3' }) + const userToken = await server.login.getAccessToken('user_p2p_3') + + const { p2pEnabled } = await server.users.getMyInfo({ token: userToken }) + expect(p2pEnabled).to.be.true + }) + + it('Should register a user with this default setting', async function () { + await server.registrations.register({ username: 'user_p2p_4' }) + + const userToken = await server.login.getAccessToken('user_p2p_4') + + const { p2pEnabled } = await server.users.getMyInfo({ token: userToken }) + expect(p2pEnabled).to.be.true + }) + }) + }) + + describe('Default user attributes', function () { + it('Should create a user and register a user with the default config', async function () { + await server.config.updateCustomSubConfig({ + newConfig: { + user: { + history: { + videos: { + enabled: true + } + }, + videoQuota : -1, + videoQuotaDaily: -1 + }, + signup: { + enabled: true, + requiresApproval: false + } + } + }) + + const config = await server.config.getConfig() + + expect(config.user.videoQuota).to.equal(-1) + expect(config.user.videoQuotaDaily).to.equal(-1) + + const user1Token = await server.users.generateUserAndToken('user1') + const user1 = await server.users.getMyInfo({ token: user1Token }) + + const user = { displayName: 'super user 2', username: 'user2', password: 'super password' } + const channel = { name: 'my_user_2_channel', displayName: 'my channel' } + await server.registrations.register({ ...user, channel }) + const user2Token = await server.login.getAccessToken(user) + const user2 = await server.users.getMyInfo({ token: user2Token }) + + for (const user of [ user1, user2 ]) { + expect(user.videosHistoryEnabled).to.be.true + expect(user.videoQuota).to.equal(-1) + expect(user.videoQuotaDaily).to.equal(-1) + } + }) + + it('Should update config and create a user and register a user with the new default config', async function () { + await server.config.updateCustomSubConfig({ + newConfig: { + user: { + history: { + videos: { + enabled: false + } + }, + videoQuota : 5242881, + videoQuotaDaily: 318742 + }, + signup: { + enabled: true, + requiresApproval: false + } + } + }) + + const user3Token = await server.users.generateUserAndToken('user3') + const user3 = await server.users.getMyInfo({ token: user3Token }) + + const user = { displayName: 'super user 4', username: 'user4', password: 'super password' } + const channel = { name: 'my_user_4_channel', displayName: 'my channel' } + await server.registrations.register({ ...user, channel }) + const user4Token = await server.login.getAccessToken(user) + const user4 = await server.users.getMyInfo({ token: user4Token }) + + for (const user of [ user3, user4 ]) { + expect(user.videosHistoryEnabled).to.be.false + expect(user.videoQuota).to.equal(5242881) + expect(user.videoQuotaDaily).to.equal(318742) + } + }) + + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/server/config.ts b/packages/tests/src/api/server/config.ts new file mode 100644 index 000000000..ce64668f8 --- /dev/null +++ b/packages/tests/src/api/server/config.ts @@ -0,0 +1,645 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { parallelTests } from '@peertube/peertube-node-utils' +import { CustomConfig, HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + killallServers, + makeGetRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +function checkInitialConfig (server: PeerTubeServer, data: CustomConfig) { + expect(data.instance.name).to.equal('PeerTube') + expect(data.instance.shortDescription).to.equal( + 'PeerTube, an ActivityPub-federated video streaming platform using P2P directly in your web browser.' + ) + expect(data.instance.description).to.equal('Welcome to this PeerTube instance!') + + expect(data.instance.terms).to.equal('No terms for now.') + expect(data.instance.creationReason).to.be.empty + expect(data.instance.codeOfConduct).to.be.empty + expect(data.instance.moderationInformation).to.be.empty + expect(data.instance.administrator).to.be.empty + expect(data.instance.maintenanceLifetime).to.be.empty + expect(data.instance.businessModel).to.be.empty + expect(data.instance.hardwareInformation).to.be.empty + + expect(data.instance.languages).to.have.lengthOf(0) + expect(data.instance.categories).to.have.lengthOf(0) + + expect(data.instance.defaultClientRoute).to.equal('/videos/trending') + expect(data.instance.isNSFW).to.be.false + expect(data.instance.defaultNSFWPolicy).to.equal('display') + expect(data.instance.customizations.css).to.be.empty + expect(data.instance.customizations.javascript).to.be.empty + + expect(data.services.twitter.username).to.equal('@Chocobozzz') + expect(data.services.twitter.whitelisted).to.be.false + + expect(data.client.videos.miniature.preferAuthorDisplayName).to.be.false + expect(data.client.menu.login.redirectOnSingleExternalAuth).to.be.false + + expect(data.cache.previews.size).to.equal(1) + expect(data.cache.captions.size).to.equal(1) + expect(data.cache.torrents.size).to.equal(1) + expect(data.cache.storyboards.size).to.equal(1) + + expect(data.signup.enabled).to.be.true + expect(data.signup.limit).to.equal(4) + expect(data.signup.minimumAge).to.equal(16) + expect(data.signup.requiresApproval).to.be.false + expect(data.signup.requiresEmailVerification).to.be.false + + expect(data.admin.email).to.equal('admin' + server.internalServerNumber + '@example.com') + expect(data.contactForm.enabled).to.be.true + + expect(data.user.history.videos.enabled).to.be.true + expect(data.user.videoQuota).to.equal(5242880) + expect(data.user.videoQuotaDaily).to.equal(-1) + + expect(data.videoChannels.maxPerUser).to.equal(20) + + expect(data.transcoding.enabled).to.be.false + expect(data.transcoding.remoteRunners.enabled).to.be.false + expect(data.transcoding.allowAdditionalExtensions).to.be.false + expect(data.transcoding.allowAudioFiles).to.be.false + expect(data.transcoding.threads).to.equal(2) + expect(data.transcoding.concurrency).to.equal(2) + expect(data.transcoding.profile).to.equal('default') + expect(data.transcoding.resolutions['144p']).to.be.false + expect(data.transcoding.resolutions['240p']).to.be.true + expect(data.transcoding.resolutions['360p']).to.be.true + expect(data.transcoding.resolutions['480p']).to.be.true + expect(data.transcoding.resolutions['720p']).to.be.true + expect(data.transcoding.resolutions['1080p']).to.be.true + expect(data.transcoding.resolutions['1440p']).to.be.true + expect(data.transcoding.resolutions['2160p']).to.be.true + expect(data.transcoding.alwaysTranscodeOriginalResolution).to.be.true + expect(data.transcoding.webVideos.enabled).to.be.true + expect(data.transcoding.hls.enabled).to.be.true + + expect(data.live.enabled).to.be.false + expect(data.live.allowReplay).to.be.false + expect(data.live.latencySetting.enabled).to.be.true + expect(data.live.maxDuration).to.equal(-1) + expect(data.live.maxInstanceLives).to.equal(20) + expect(data.live.maxUserLives).to.equal(3) + expect(data.live.transcoding.enabled).to.be.false + expect(data.live.transcoding.remoteRunners.enabled).to.be.false + expect(data.live.transcoding.threads).to.equal(2) + expect(data.live.transcoding.profile).to.equal('default') + expect(data.live.transcoding.resolutions['144p']).to.be.false + expect(data.live.transcoding.resolutions['240p']).to.be.false + expect(data.live.transcoding.resolutions['360p']).to.be.false + expect(data.live.transcoding.resolutions['480p']).to.be.false + expect(data.live.transcoding.resolutions['720p']).to.be.false + expect(data.live.transcoding.resolutions['1080p']).to.be.false + expect(data.live.transcoding.resolutions['1440p']).to.be.false + expect(data.live.transcoding.resolutions['2160p']).to.be.false + expect(data.live.transcoding.alwaysTranscodeOriginalResolution).to.be.true + + expect(data.videoStudio.enabled).to.be.false + expect(data.videoStudio.remoteRunners.enabled).to.be.false + + expect(data.videoFile.update.enabled).to.be.false + + expect(data.import.videos.concurrency).to.equal(2) + expect(data.import.videos.http.enabled).to.be.true + expect(data.import.videos.torrent.enabled).to.be.true + expect(data.autoBlacklist.videos.ofUsers.enabled).to.be.false + + expect(data.followers.instance.enabled).to.be.true + expect(data.followers.instance.manualApproval).to.be.false + + expect(data.followings.instance.autoFollowBack.enabled).to.be.false + expect(data.followings.instance.autoFollowIndex.enabled).to.be.false + expect(data.followings.instance.autoFollowIndex.indexUrl).to.equal('') + + expect(data.broadcastMessage.enabled).to.be.false + expect(data.broadcastMessage.level).to.equal('info') + expect(data.broadcastMessage.message).to.equal('') + expect(data.broadcastMessage.dismissable).to.be.false +} + +function checkUpdatedConfig (data: CustomConfig) { + expect(data.instance.name).to.equal('PeerTube updated') + expect(data.instance.shortDescription).to.equal('my short description') + expect(data.instance.description).to.equal('my super description') + + expect(data.instance.terms).to.equal('my super terms') + expect(data.instance.creationReason).to.equal('my super creation reason') + expect(data.instance.codeOfConduct).to.equal('my super coc') + expect(data.instance.moderationInformation).to.equal('my super moderation information') + expect(data.instance.administrator).to.equal('Kuja') + expect(data.instance.maintenanceLifetime).to.equal('forever') + expect(data.instance.businessModel).to.equal('my super business model') + expect(data.instance.hardwareInformation).to.equal('2vCore 3GB RAM') + + expect(data.instance.languages).to.deep.equal([ 'en', 'es' ]) + expect(data.instance.categories).to.deep.equal([ 1, 2 ]) + + expect(data.instance.defaultClientRoute).to.equal('/videos/recently-added') + expect(data.instance.isNSFW).to.be.true + expect(data.instance.defaultNSFWPolicy).to.equal('blur') + expect(data.instance.customizations.javascript).to.equal('alert("coucou")') + expect(data.instance.customizations.css).to.equal('body { background-color: red; }') + + expect(data.services.twitter.username).to.equal('@Kuja') + expect(data.services.twitter.whitelisted).to.be.true + + expect(data.client.videos.miniature.preferAuthorDisplayName).to.be.true + expect(data.client.menu.login.redirectOnSingleExternalAuth).to.be.true + + expect(data.cache.previews.size).to.equal(2) + expect(data.cache.captions.size).to.equal(3) + expect(data.cache.torrents.size).to.equal(4) + expect(data.cache.storyboards.size).to.equal(5) + + expect(data.signup.enabled).to.be.false + expect(data.signup.limit).to.equal(5) + expect(data.signup.requiresApproval).to.be.false + expect(data.signup.requiresEmailVerification).to.be.false + expect(data.signup.minimumAge).to.equal(10) + + // We override admin email in parallel tests, so skip this exception + if (parallelTests() === false) { + expect(data.admin.email).to.equal('superadmin1@example.com') + } + + expect(data.contactForm.enabled).to.be.false + + expect(data.user.history.videos.enabled).to.be.false + expect(data.user.videoQuota).to.equal(5242881) + expect(data.user.videoQuotaDaily).to.equal(318742) + + expect(data.videoChannels.maxPerUser).to.equal(24) + + expect(data.transcoding.enabled).to.be.true + expect(data.transcoding.remoteRunners.enabled).to.be.true + expect(data.transcoding.threads).to.equal(1) + expect(data.transcoding.concurrency).to.equal(3) + expect(data.transcoding.allowAdditionalExtensions).to.be.true + expect(data.transcoding.allowAudioFiles).to.be.true + expect(data.transcoding.profile).to.equal('vod_profile') + expect(data.transcoding.resolutions['144p']).to.be.false + expect(data.transcoding.resolutions['240p']).to.be.false + expect(data.transcoding.resolutions['360p']).to.be.true + expect(data.transcoding.resolutions['480p']).to.be.true + expect(data.transcoding.resolutions['720p']).to.be.false + expect(data.transcoding.resolutions['1080p']).to.be.false + expect(data.transcoding.resolutions['2160p']).to.be.false + expect(data.transcoding.alwaysTranscodeOriginalResolution).to.be.false + expect(data.transcoding.hls.enabled).to.be.false + expect(data.transcoding.webVideos.enabled).to.be.true + + expect(data.live.enabled).to.be.true + expect(data.live.allowReplay).to.be.true + expect(data.live.latencySetting.enabled).to.be.false + expect(data.live.maxDuration).to.equal(5000) + expect(data.live.maxInstanceLives).to.equal(-1) + expect(data.live.maxUserLives).to.equal(10) + expect(data.live.transcoding.enabled).to.be.true + expect(data.live.transcoding.remoteRunners.enabled).to.be.true + expect(data.live.transcoding.threads).to.equal(4) + expect(data.live.transcoding.profile).to.equal('live_profile') + expect(data.live.transcoding.resolutions['144p']).to.be.true + expect(data.live.transcoding.resolutions['240p']).to.be.true + expect(data.live.transcoding.resolutions['360p']).to.be.true + expect(data.live.transcoding.resolutions['480p']).to.be.true + expect(data.live.transcoding.resolutions['720p']).to.be.true + expect(data.live.transcoding.resolutions['1080p']).to.be.true + expect(data.live.transcoding.resolutions['2160p']).to.be.true + expect(data.live.transcoding.alwaysTranscodeOriginalResolution).to.be.false + + expect(data.videoStudio.enabled).to.be.true + expect(data.videoStudio.remoteRunners.enabled).to.be.true + + expect(data.videoFile.update.enabled).to.be.true + + expect(data.import.videos.concurrency).to.equal(4) + expect(data.import.videos.http.enabled).to.be.false + expect(data.import.videos.torrent.enabled).to.be.false + expect(data.autoBlacklist.videos.ofUsers.enabled).to.be.true + + expect(data.followers.instance.enabled).to.be.false + expect(data.followers.instance.manualApproval).to.be.true + + expect(data.followings.instance.autoFollowBack.enabled).to.be.true + expect(data.followings.instance.autoFollowIndex.enabled).to.be.true + expect(data.followings.instance.autoFollowIndex.indexUrl).to.equal('https://updated.example.com') + + expect(data.broadcastMessage.enabled).to.be.true + expect(data.broadcastMessage.level).to.equal('error') + expect(data.broadcastMessage.message).to.equal('super bad message') + expect(data.broadcastMessage.dismissable).to.be.true +} + +const newCustomConfig: CustomConfig = { + instance: { + name: 'PeerTube updated', + shortDescription: 'my short description', + description: 'my super description', + terms: 'my super terms', + codeOfConduct: 'my super coc', + + creationReason: 'my super creation reason', + moderationInformation: 'my super moderation information', + administrator: 'Kuja', + maintenanceLifetime: 'forever', + businessModel: 'my super business model', + hardwareInformation: '2vCore 3GB RAM', + + languages: [ 'en', 'es' ], + categories: [ 1, 2 ], + + isNSFW: true, + defaultNSFWPolicy: 'blur' as 'blur', + + defaultClientRoute: '/videos/recently-added', + + customizations: { + javascript: 'alert("coucou")', + css: 'body { background-color: red; }' + } + }, + theme: { + default: 'default' + }, + services: { + twitter: { + username: '@Kuja', + whitelisted: true + } + }, + client: { + videos: { + miniature: { + preferAuthorDisplayName: true + } + }, + menu: { + login: { + redirectOnSingleExternalAuth: true + } + } + }, + cache: { + previews: { + size: 2 + }, + captions: { + size: 3 + }, + torrents: { + size: 4 + }, + storyboards: { + size: 5 + } + }, + signup: { + enabled: false, + limit: 5, + requiresApproval: false, + requiresEmailVerification: false, + minimumAge: 10 + }, + admin: { + email: 'superadmin1@example.com' + }, + contactForm: { + enabled: false + }, + user: { + history: { + videos: { + enabled: false + } + }, + videoQuota: 5242881, + videoQuotaDaily: 318742 + }, + videoChannels: { + maxPerUser: 24 + }, + transcoding: { + enabled: true, + remoteRunners: { + enabled: true + }, + allowAdditionalExtensions: true, + allowAudioFiles: true, + threads: 1, + concurrency: 3, + profile: 'vod_profile', + resolutions: { + '0p': false, + '144p': false, + '240p': false, + '360p': true, + '480p': true, + '720p': false, + '1080p': false, + '1440p': false, + '2160p': false + }, + alwaysTranscodeOriginalResolution: false, + webVideos: { + enabled: true + }, + hls: { + enabled: false + } + }, + live: { + enabled: true, + allowReplay: true, + latencySetting: { + enabled: false + }, + maxDuration: 5000, + maxInstanceLives: -1, + maxUserLives: 10, + transcoding: { + enabled: true, + remoteRunners: { + enabled: true + }, + threads: 4, + profile: 'live_profile', + resolutions: { + '144p': true, + '240p': true, + '360p': true, + '480p': true, + '720p': true, + '1080p': true, + '1440p': true, + '2160p': true + }, + alwaysTranscodeOriginalResolution: false + } + }, + videoStudio: { + enabled: true, + remoteRunners: { + enabled: true + } + }, + videoFile: { + update: { + enabled: true + } + }, + import: { + videos: { + concurrency: 4, + http: { + enabled: false + }, + torrent: { + enabled: false + } + }, + videoChannelSynchronization: { + enabled: false, + maxPerUser: 10 + } + }, + trending: { + videos: { + algorithms: { + enabled: [ 'hot', 'most-viewed', 'most-liked' ], + default: 'hot' + } + } + }, + autoBlacklist: { + videos: { + ofUsers: { + enabled: true + } + } + }, + followers: { + instance: { + enabled: false, + manualApproval: true + } + }, + followings: { + instance: { + autoFollowBack: { + enabled: true + }, + autoFollowIndex: { + enabled: true, + indexUrl: 'https://updated.example.com' + } + } + }, + broadcastMessage: { + enabled: true, + level: 'error', + message: 'super bad message', + dismissable: true + }, + search: { + remoteUri: { + anonymous: true, + users: true + }, + searchIndex: { + enabled: true, + url: 'https://search.joinpeertube.org', + disableLocalSearch: true, + isDefaultSearch: true + } + } +} + +describe('Test static config', function () { + let server: PeerTubeServer = null + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1, { webadmin: { configuration: { edition: { allowed: false } } } }) + await setAccessTokensToServers([ server ]) + }) + + it('Should tell the client that edits are not allowed', async function () { + const data = await server.config.getConfig() + + expect(data.webadmin.configuration.edition.allowed).to.be.false + }) + + it('Should error when client tries to update', async function () { + await server.config.updateCustomConfig({ newCustomConfig, expectedStatus: 405 }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) + +describe('Test config', function () { + let server: PeerTubeServer = null + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + }) + + it('Should have a correct config on a server with registration enabled', async function () { + const data = await server.config.getConfig() + + expect(data.signup.allowed).to.be.true + }) + + it('Should have a correct config on a server with registration enabled and a users limit', async function () { + this.timeout(5000) + + await Promise.all([ + server.registrations.register({ username: 'user1' }), + server.registrations.register({ username: 'user2' }), + server.registrations.register({ username: 'user3' }) + ]) + + const data = await server.config.getConfig() + + expect(data.signup.allowed).to.be.false + }) + + it('Should have the correct video allowed extensions', async function () { + const data = await server.config.getConfig() + + expect(data.video.file.extensions).to.have.lengthOf(3) + expect(data.video.file.extensions).to.contain('.mp4') + expect(data.video.file.extensions).to.contain('.webm') + expect(data.video.file.extensions).to.contain('.ogv') + + await server.videos.upload({ attributes: { fixture: 'video_short.mkv' }, expectedStatus: HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415 }) + await server.videos.upload({ attributes: { fixture: 'sample.ogg' }, expectedStatus: HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415 }) + + expect(data.contactForm.enabled).to.be.true + }) + + it('Should get the customized configuration', async function () { + const data = await server.config.getCustomConfig() + + checkInitialConfig(server, data) + }) + + it('Should update the customized configuration', async function () { + await server.config.updateCustomConfig({ newCustomConfig }) + + const data = await server.config.getCustomConfig() + checkUpdatedConfig(data) + }) + + it('Should have the correct updated video allowed extensions', async function () { + this.timeout(30000) + + const data = await server.config.getConfig() + + expect(data.video.file.extensions).to.have.length.above(4) + expect(data.video.file.extensions).to.contain('.mp4') + expect(data.video.file.extensions).to.contain('.webm') + expect(data.video.file.extensions).to.contain('.ogv') + expect(data.video.file.extensions).to.contain('.flv') + expect(data.video.file.extensions).to.contain('.wmv') + expect(data.video.file.extensions).to.contain('.mkv') + expect(data.video.file.extensions).to.contain('.mp3') + expect(data.video.file.extensions).to.contain('.ogg') + expect(data.video.file.extensions).to.contain('.flac') + + await server.videos.upload({ attributes: { fixture: 'video_short.mkv' }, expectedStatus: HttpStatusCode.OK_200 }) + await server.videos.upload({ attributes: { fixture: 'sample.ogg' }, expectedStatus: HttpStatusCode.OK_200 }) + }) + + it('Should have the configuration updated after a restart', async function () { + this.timeout(30000) + + await killallServers([ server ]) + + await server.run() + + const data = await server.config.getCustomConfig() + + checkUpdatedConfig(data) + }) + + it('Should fetch the about information', async function () { + const data = await server.config.getAbout() + + expect(data.instance.name).to.equal('PeerTube updated') + expect(data.instance.shortDescription).to.equal('my short description') + expect(data.instance.description).to.equal('my super description') + expect(data.instance.terms).to.equal('my super terms') + expect(data.instance.codeOfConduct).to.equal('my super coc') + + expect(data.instance.creationReason).to.equal('my super creation reason') + expect(data.instance.moderationInformation).to.equal('my super moderation information') + expect(data.instance.administrator).to.equal('Kuja') + expect(data.instance.maintenanceLifetime).to.equal('forever') + expect(data.instance.businessModel).to.equal('my super business model') + expect(data.instance.hardwareInformation).to.equal('2vCore 3GB RAM') + + expect(data.instance.languages).to.deep.equal([ 'en', 'es' ]) + expect(data.instance.categories).to.deep.equal([ 1, 2 ]) + }) + + it('Should remove the custom configuration', async function () { + await server.config.deleteCustomConfig() + + const data = await server.config.getCustomConfig() + checkInitialConfig(server, data) + }) + + it('Should enable/disable security headers', async function () { + this.timeout(25000) + + { + const res = await makeGetRequest({ + url: server.url, + path: '/api/v1/config', + expectedStatus: 200 + }) + + expect(res.headers['x-frame-options']).to.exist + expect(res.headers['x-powered-by']).to.equal('PeerTube') + } + + await killallServers([ server ]) + + const config = { + security: { + frameguard: { enabled: false }, + powered_by_header: { enabled: false } + } + } + await server.run(config) + + { + const res = await makeGetRequest({ + url: server.url, + path: '/api/v1/config', + expectedStatus: 200 + }) + + expect(res.headers['x-frame-options']).to.not.exist + expect(res.headers['x-powered-by']).to.not.exist + } + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/server/contact-form.ts b/packages/tests/src/api/server/contact-form.ts new file mode 100644 index 000000000..03389aa64 --- /dev/null +++ b/packages/tests/src/api/server/contact-form.ts @@ -0,0 +1,101 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { MockSmtpServer } from '@tests/shared/mock-servers/index.js' +import { wait } from '@peertube/peertube-core-utils' +import { HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + ConfigCommand, + ContactFormCommand, + createSingleServer, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test contact form', function () { + let server: PeerTubeServer + const emails: object[] = [] + let command: ContactFormCommand + + before(async function () { + this.timeout(30000) + + const port = await MockSmtpServer.Instance.collectEmails(emails) + + server = await createSingleServer(1, ConfigCommand.getEmailOverrideConfig(port)) + await setAccessTokensToServers([ server ]) + + command = server.contactForm + }) + + it('Should send a contact form', async function () { + await command.send({ + fromEmail: 'toto@example.com', + body: 'my super message', + subject: 'my subject', + fromName: 'Super toto' + }) + + await waitJobs(server) + + expect(emails).to.have.lengthOf(1) + + const email = emails[0] + + expect(email['from'][0]['address']).equal('test-admin@127.0.0.1') + expect(email['replyTo'][0]['address']).equal('toto@example.com') + expect(email['to'][0]['address']).equal('admin' + server.internalServerNumber + '@example.com') + expect(email['subject']).contains('my subject') + expect(email['text']).contains('my super message') + }) + + it('Should not have duplicated email address in text message', async function () { + const text = emails[0]['text'] as string + + const matches = text.match(/toto@example.com/g) + expect(matches).to.have.lengthOf(1) + }) + + it('Should not be able to send another contact form because of the anti spam checker', async function () { + await wait(1000) + + await command.send({ + fromEmail: 'toto@example.com', + body: 'my super message', + subject: 'my subject', + fromName: 'Super toto' + }) + + await command.send({ + fromEmail: 'toto@example.com', + body: 'my super message', + fromName: 'Super toto', + subject: 'my subject', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should be able to send another contact form after a while', async function () { + await wait(1000) + + await command.send({ + fromEmail: 'toto@example.com', + fromName: 'Super toto', + subject: 'my subject', + body: 'my super message' + }) + }) + + it('Should not have the manage preferences link in the email', async function () { + const email = emails[0] + expect(email['text']).to.not.contain('Manage your notification preferences') + }) + + after(async function () { + MockSmtpServer.Instance.kill() + + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/server/email.ts b/packages/tests/src/api/server/email.ts new file mode 100644 index 000000000..6d3f3f3bb --- /dev/null +++ b/packages/tests/src/api/server/email.ts @@ -0,0 +1,371 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { MockSmtpServer } from '@tests/shared/mock-servers/index.js' +import { HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + ConfigCommand, + createSingleServer, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test emails', function () { + let server: PeerTubeServer + let userId: number + let userId2: number + let userAccessToken: string + + let videoShortUUID: string + let videoId: number + + let videoUserUUID: string + + let verificationString: string + let verificationString2: string + + const emails: object[] = [] + const user = { + username: 'user_1', + password: 'super_password' + } + + before(async function () { + this.timeout(120000) + + const emailPort = await MockSmtpServer.Instance.collectEmails(emails) + server = await createSingleServer(1, ConfigCommand.getEmailOverrideConfig(emailPort)) + + await setAccessTokensToServers([ server ]) + await server.config.enableSignup(true) + + { + const created = await server.users.create({ username: user.username, password: user.password }) + userId = created.id + + userAccessToken = await server.login.getAccessToken(user) + } + + { + const attributes = { name: 'my super user video' } + const { uuid } = await server.videos.upload({ token: userAccessToken, attributes }) + videoUserUUID = uuid + } + + { + const attributes = { + name: 'my super name' + } + const { shortUUID, id } = await server.videos.upload({ attributes }) + videoShortUUID = shortUUID + videoId = id + } + }) + + describe('When resetting user password', function () { + + it('Should ask to reset the password', async function () { + await server.users.askResetPassword({ email: 'user_1@example.com' }) + + await waitJobs(server) + expect(emails).to.have.lengthOf(1) + + const email = emails[0] + + expect(email['from'][0]['name']).equal('PeerTube') + expect(email['from'][0]['address']).equal('test-admin@127.0.0.1') + expect(email['to'][0]['address']).equal('user_1@example.com') + expect(email['subject']).contains('password') + + const verificationStringMatches = /verificationString=([a-z0-9]+)/.exec(email['text']) + expect(verificationStringMatches).not.to.be.null + + verificationString = verificationStringMatches[1] + expect(verificationString).to.have.length.above(2) + + const userIdMatches = /userId=([0-9]+)/.exec(email['text']) + expect(userIdMatches).not.to.be.null + + userId = parseInt(userIdMatches[1], 10) + expect(verificationString).to.not.be.undefined + }) + + it('Should not reset the password with an invalid verification string', async function () { + await server.users.resetPassword({ + userId, + verificationString: verificationString + 'b', + password: 'super_password2', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should reset the password', async function () { + await server.users.resetPassword({ userId, verificationString, password: 'super_password2' }) + }) + + it('Should not reset the password with the same verification string', async function () { + await server.users.resetPassword({ + userId, + verificationString, + password: 'super_password3', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should login with this new password', async function () { + user.password = 'super_password2' + + await server.login.getAccessToken(user) + }) + }) + + describe('When creating a user without password', function () { + + it('Should send a create password email', async function () { + await server.users.create({ username: 'create_password', password: '' }) + + await waitJobs(server) + expect(emails).to.have.lengthOf(2) + + const email = emails[1] + + expect(email['from'][0]['name']).equal('PeerTube') + expect(email['from'][0]['address']).equal('test-admin@127.0.0.1') + expect(email['to'][0]['address']).equal('create_password@example.com') + expect(email['subject']).contains('account') + expect(email['subject']).contains('password') + + const verificationStringMatches = /verificationString=([a-z0-9]+)/.exec(email['text']) + expect(verificationStringMatches).not.to.be.null + + verificationString2 = verificationStringMatches[1] + expect(verificationString2).to.have.length.above(2) + + const userIdMatches = /userId=([0-9]+)/.exec(email['text']) + expect(userIdMatches).not.to.be.null + + userId2 = parseInt(userIdMatches[1], 10) + }) + + it('Should not reset the password with an invalid verification string', async function () { + await server.users.resetPassword({ + userId: userId2, + verificationString: verificationString2 + 'c', + password: 'newly_created_password', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should reset the password', async function () { + await server.users.resetPassword({ + userId: userId2, + verificationString: verificationString2, + password: 'newly_created_password' + }) + }) + + it('Should login with this new password', async function () { + await server.login.getAccessToken({ + username: 'create_password', + password: 'newly_created_password' + }) + }) + }) + + describe('When creating an abuse', function () { + + it('Should send the notification email', async function () { + const reason = 'my super bad reason' + await server.abuses.report({ token: userAccessToken, videoId, reason }) + + await waitJobs(server) + expect(emails).to.have.lengthOf(3) + + const email = emails[2] + + expect(email['from'][0]['name']).equal('PeerTube') + expect(email['from'][0]['address']).equal('test-admin@127.0.0.1') + expect(email['to'][0]['address']).equal('admin' + server.internalServerNumber + '@example.com') + expect(email['subject']).contains('abuse') + expect(email['text']).contains(videoShortUUID) + }) + }) + + describe('When blocking/unblocking user', function () { + + it('Should send the notification email when blocking a user', async function () { + const reason = 'my super bad reason' + await server.users.banUser({ userId, reason }) + + await waitJobs(server) + expect(emails).to.have.lengthOf(4) + + const email = emails[3] + + expect(email['from'][0]['name']).equal('PeerTube') + expect(email['from'][0]['address']).equal('test-admin@127.0.0.1') + expect(email['to'][0]['address']).equal('user_1@example.com') + expect(email['subject']).contains(' blocked') + expect(email['text']).contains(' blocked') + expect(email['text']).contains('bad reason') + }) + + it('Should send the notification email when unblocking a user', async function () { + await server.users.unbanUser({ userId }) + + await waitJobs(server) + expect(emails).to.have.lengthOf(5) + + const email = emails[4] + + expect(email['from'][0]['name']).equal('PeerTube') + expect(email['from'][0]['address']).equal('test-admin@127.0.0.1') + expect(email['to'][0]['address']).equal('user_1@example.com') + expect(email['subject']).contains(' unblocked') + expect(email['text']).contains(' unblocked') + }) + }) + + describe('When blacklisting a video', function () { + it('Should send the notification email', async function () { + const reason = 'my super reason' + await server.blacklist.add({ videoId: videoUserUUID, reason }) + + await waitJobs(server) + expect(emails).to.have.lengthOf(6) + + const email = emails[5] + + expect(email['from'][0]['name']).equal('PeerTube') + expect(email['from'][0]['address']).equal('test-admin@127.0.0.1') + expect(email['to'][0]['address']).equal('user_1@example.com') + expect(email['subject']).contains(' blacklisted') + expect(email['text']).contains('my super user video') + expect(email['text']).contains('my super reason') + }) + + it('Should send the notification email', async function () { + await server.blacklist.remove({ videoId: videoUserUUID }) + + await waitJobs(server) + expect(emails).to.have.lengthOf(7) + + const email = emails[6] + + expect(email['from'][0]['name']).equal('PeerTube') + expect(email['from'][0]['address']).equal('test-admin@127.0.0.1') + expect(email['to'][0]['address']).equal('user_1@example.com') + expect(email['subject']).contains(' unblacklisted') + expect(email['text']).contains('my super user video') + }) + + it('Should have the manage preferences link in the email', async function () { + const email = emails[6] + expect(email['text']).to.contain('Manage your notification preferences') + }) + }) + + describe('When verifying a user email', function () { + + it('Should ask to send the verification email', async function () { + await server.users.askSendVerifyEmail({ email: 'user_1@example.com' }) + + await waitJobs(server) + expect(emails).to.have.lengthOf(8) + + const email = emails[7] + + expect(email['from'][0]['name']).equal('PeerTube') + expect(email['from'][0]['address']).equal('test-admin@127.0.0.1') + expect(email['to'][0]['address']).equal('user_1@example.com') + expect(email['subject']).contains('Verify') + + const verificationStringMatches = /verificationString=([a-z0-9]+)/.exec(email['text']) + expect(verificationStringMatches).not.to.be.null + + verificationString = verificationStringMatches[1] + expect(verificationString).to.not.be.undefined + expect(verificationString).to.have.length.above(2) + + const userIdMatches = /userId=([0-9]+)/.exec(email['text']) + expect(userIdMatches).not.to.be.null + + userId = parseInt(userIdMatches[1], 10) + }) + + it('Should not verify the email with an invalid verification string', async function () { + await server.users.verifyEmail({ + userId, + verificationString: verificationString + 'b', + isPendingEmail: false, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should verify the email', async function () { + await server.users.verifyEmail({ userId, verificationString }) + }) + }) + + describe('When verifying a registration email', function () { + let registrationId: number + let registrationIdEmail: number + + before(async function () { + const { id } = await server.registrations.requestRegistration({ + username: 'request_1', + email: 'request_1@example.com', + registrationReason: 'tt' + }) + registrationId = id + }) + + it('Should ask to send the verification email', async function () { + await server.registrations.askSendVerifyEmail({ email: 'request_1@example.com' }) + + await waitJobs(server) + expect(emails).to.have.lengthOf(9) + + const email = emails[8] + + expect(email['from'][0]['name']).equal('PeerTube') + expect(email['from'][0]['address']).equal('test-admin@127.0.0.1') + expect(email['to'][0]['address']).equal('request_1@example.com') + expect(email['subject']).contains('Verify') + + const verificationStringMatches = /verificationString=([a-z0-9]+)/.exec(email['text']) + expect(verificationStringMatches).not.to.be.null + + verificationString = verificationStringMatches[1] + expect(verificationString).to.not.be.undefined + expect(verificationString).to.have.length.above(2) + + const registrationIdMatches = /registrationId=([0-9]+)/.exec(email['text']) + expect(registrationIdMatches).not.to.be.null + + registrationIdEmail = parseInt(registrationIdMatches[1], 10) + + expect(registrationId).to.equal(registrationIdEmail) + }) + + it('Should not verify the email with an invalid verification string', async function () { + await server.registrations.verifyEmail({ + registrationId: registrationIdEmail, + verificationString: verificationString + 'b', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should verify the email', async function () { + await server.registrations.verifyEmail({ registrationId: registrationIdEmail, verificationString }) + }) + }) + + after(async function () { + MockSmtpServer.Instance.kill() + + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/server/follow-constraints.ts b/packages/tests/src/api/server/follow-constraints.ts new file mode 100644 index 000000000..8d277c906 --- /dev/null +++ b/packages/tests/src/api/server/follow-constraints.ts @@ -0,0 +1,321 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { HttpStatusCode, PeerTubeProblemDocument, ServerErrorCode } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test follow constraints', function () { + let servers: PeerTubeServer[] = [] + let video1UUID: string + let video2UUID: string + let userToken: string + + before(async function () { + this.timeout(240000) + + servers = await createMultipleServers(2) + + // Get the access tokens + await setAccessTokensToServers(servers) + + { + const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video server 1' } }) + video1UUID = uuid + } + { + const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video server 2' } }) + video2UUID = uuid + } + + const user = { + username: 'user1', + password: 'super_password' + } + await servers[0].users.create({ username: user.username, password: user.password }) + userToken = await servers[0].login.getAccessToken(user) + + await doubleFollow(servers[0], servers[1]) + }) + + describe('With a followed instance', function () { + + describe('With an unlogged user', function () { + + it('Should get the local video', async function () { + await servers[0].videos.get({ id: video1UUID }) + }) + + it('Should get the remote video', async function () { + await servers[0].videos.get({ id: video2UUID }) + }) + + it('Should list local account videos', async function () { + const { total, data } = await servers[0].videos.listByAccount({ handle: 'root@' + servers[0].host }) + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + }) + + it('Should list remote account videos', async function () { + const { total, data } = await servers[0].videos.listByAccount({ handle: 'root@' + servers[1].host }) + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + }) + + it('Should list local channel videos', async function () { + const handle = 'root_channel@' + servers[0].host + const { total, data } = await servers[0].videos.listByChannel({ handle }) + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + }) + + it('Should list remote channel videos', async function () { + const handle = 'root_channel@' + servers[1].host + const { total, data } = await servers[0].videos.listByChannel({ handle }) + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + }) + }) + + describe('With a logged user', function () { + it('Should get the local video', async function () { + await servers[0].videos.getWithToken({ token: userToken, id: video1UUID }) + }) + + it('Should get the remote video', async function () { + await servers[0].videos.getWithToken({ token: userToken, id: video2UUID }) + }) + + it('Should list local account videos', async function () { + const { total, data } = await servers[0].videos.listByAccount({ token: userToken, handle: 'root@' + servers[0].host }) + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + }) + + it('Should list remote account videos', async function () { + const { total, data } = await servers[0].videos.listByAccount({ token: userToken, handle: 'root@' + servers[1].host }) + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + }) + + it('Should list local channel videos', async function () { + const handle = 'root_channel@' + servers[0].host + const { total, data } = await servers[0].videos.listByChannel({ token: userToken, handle }) + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + }) + + it('Should list remote channel videos', async function () { + const handle = 'root_channel@' + servers[1].host + const { total, data } = await servers[0].videos.listByChannel({ token: userToken, handle }) + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + }) + }) + }) + + describe('With a non followed instance', function () { + + before(async function () { + this.timeout(30000) + + await servers[0].follows.unfollow({ target: servers[1] }) + }) + + describe('With an unlogged user', function () { + + it('Should get the local video', async function () { + await servers[0].videos.get({ id: video1UUID }) + }) + + it('Should not get the remote video', async function () { + const body = await servers[0].videos.get({ id: video2UUID, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + const error = body as unknown as PeerTubeProblemDocument + + const doc = 'https://docs.joinpeertube.org/api-rest-reference.html#section/Errors/does_not_respect_follow_constraints' + expect(error.type).to.equal(doc) + expect(error.code).to.equal(ServerErrorCode.DOES_NOT_RESPECT_FOLLOW_CONSTRAINTS) + + expect(error.detail).to.equal('Cannot get this video regarding follow constraints') + expect(error.error).to.equal(error.detail) + + expect(error.status).to.equal(HttpStatusCode.FORBIDDEN_403) + + expect(error.originUrl).to.contains(servers[1].url) + }) + + it('Should list local account videos', async function () { + const { total, data } = await servers[0].videos.listByAccount({ + token: null, + handle: 'root@' + servers[0].host + }) + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + }) + + it('Should not list remote account videos', async function () { + const { total, data } = await servers[0].videos.listByAccount({ + token: null, + handle: 'root@' + servers[1].host + }) + + expect(total).to.equal(0) + expect(data).to.have.lengthOf(0) + }) + + it('Should list local channel videos', async function () { + const handle = 'root_channel@' + servers[0].host + const { total, data } = await servers[0].videos.listByChannel({ token: null, handle }) + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + }) + + it('Should not list remote channel videos', async function () { + const handle = 'root_channel@' + servers[1].host + const { total, data } = await servers[0].videos.listByChannel({ token: null, handle }) + + expect(total).to.equal(0) + expect(data).to.have.lengthOf(0) + }) + }) + + describe('With a logged user', function () { + + it('Should get the local video', async function () { + await servers[0].videos.getWithToken({ token: userToken, id: video1UUID }) + }) + + it('Should get the remote video', async function () { + await servers[0].videos.getWithToken({ token: userToken, id: video2UUID }) + }) + + it('Should list local account videos', async function () { + const { total, data } = await servers[0].videos.listByAccount({ token: userToken, handle: 'root@' + servers[0].host }) + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + }) + + it('Should list remote account videos', async function () { + const { total, data } = await servers[0].videos.listByAccount({ token: userToken, handle: 'root@' + servers[1].host }) + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + }) + + it('Should list local channel videos', async function () { + const handle = 'root_channel@' + servers[0].host + const { total, data } = await servers[0].videos.listByChannel({ token: userToken, handle }) + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + }) + + it('Should list remote channel videos', async function () { + const handle = 'root_channel@' + servers[1].host + const { total, data } = await servers[0].videos.listByChannel({ token: userToken, handle }) + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + }) + }) + }) + + describe('When following a remote account', function () { + + before(async function () { + this.timeout(60000) + + await servers[0].follows.follow({ handles: [ 'root@' + servers[1].host ] }) + await waitJobs(servers) + }) + + it('Should get the remote video with an unlogged user', async function () { + await servers[0].videos.get({ id: video2UUID }) + }) + + it('Should get the remote video with a logged in user', async function () { + await servers[0].videos.getWithToken({ token: userToken, id: video2UUID }) + }) + }) + + describe('When unfollowing a remote account', function () { + + before(async function () { + this.timeout(60000) + + await servers[0].follows.unfollow({ target: 'root@' + servers[1].host }) + await waitJobs(servers) + }) + + it('Should not get the remote video with an unlogged user', async function () { + const body = await servers[0].videos.get({ id: video2UUID, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + + const error = body as unknown as PeerTubeProblemDocument + expect(error.code).to.equal(ServerErrorCode.DOES_NOT_RESPECT_FOLLOW_CONSTRAINTS) + }) + + it('Should get the remote video with a logged in user', async function () { + await servers[0].videos.getWithToken({ token: userToken, id: video2UUID }) + }) + }) + + describe('When following a remote channel', function () { + + before(async function () { + this.timeout(60000) + + await servers[0].follows.follow({ handles: [ 'root_channel@' + servers[1].host ] }) + await waitJobs(servers) + }) + + it('Should get the remote video with an unlogged user', async function () { + await servers[0].videos.get({ id: video2UUID }) + }) + + it('Should get the remote video with a logged in user', async function () { + await servers[0].videos.getWithToken({ token: userToken, id: video2UUID }) + }) + }) + + describe('When unfollowing a remote channel', function () { + + before(async function () { + this.timeout(60000) + + await servers[0].follows.unfollow({ target: 'root_channel@' + servers[1].host }) + await waitJobs(servers) + }) + + it('Should not get the remote video with an unlogged user', async function () { + const body = await servers[0].videos.get({ id: video2UUID, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + + const error = body as unknown as PeerTubeProblemDocument + expect(error.code).to.equal(ServerErrorCode.DOES_NOT_RESPECT_FOLLOW_CONSTRAINTS) + }) + + it('Should get the remote video with a logged in user', async function () { + await servers[0].videos.getWithToken({ token: userToken, id: video2UUID }) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/server/follows-moderation.ts b/packages/tests/src/api/server/follows-moderation.ts new file mode 100644 index 000000000..811dd5c22 --- /dev/null +++ b/packages/tests/src/api/server/follows-moderation.ts @@ -0,0 +1,364 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { expectStartWith } from '@tests/shared/checks.js' +import { ActorFollow, FollowState } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + FollowsCommand, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +async function checkServer1And2HasFollowers (servers: PeerTubeServer[], state = 'accepted') { + const fns = [ + servers[0].follows.getFollowings.bind(servers[0].follows), + servers[1].follows.getFollowers.bind(servers[1].follows) + ] + + for (const fn of fns) { + const body = await fn({ start: 0, count: 5, sort: 'createdAt' }) + expect(body.total).to.equal(1) + + const follow = body.data[0] + expect(follow.state).to.equal(state) + expect(follow.follower.url).to.equal(servers[0].url + '/accounts/peertube') + expect(follow.following.url).to.equal(servers[1].url + '/accounts/peertube') + } +} + +async function checkFollows (options: { + follower: PeerTubeServer + followerState: FollowState | 'deleted' + + following: PeerTubeServer + followingState: FollowState | 'deleted' +}) { + const { follower, followerState, followingState, following } = options + + const followerUrl = follower.url + '/accounts/peertube' + const followingUrl = following.url + '/accounts/peertube' + const finder = (d: ActorFollow) => d.follower.url === followerUrl && d.following.url === followingUrl + + { + const { data } = await follower.follows.getFollowings() + const follow = data.find(finder) + + if (followerState === 'deleted') { + expect(follow).to.not.exist + } else { + expect(follow.state).to.equal(followerState) + expect(follow.follower.url).to.equal(followerUrl) + expect(follow.following.url).to.equal(followingUrl) + } + } + + { + const { data } = await following.follows.getFollowers() + const follow = data.find(finder) + + if (followingState === 'deleted') { + expect(follow).to.not.exist + } else { + expect(follow.state).to.equal(followingState) + expect(follow.follower.url).to.equal(followerUrl) + expect(follow.following.url).to.equal(followingUrl) + } + } +} + +async function checkNoFollowers (servers: PeerTubeServer[]) { + const fns = [ + servers[0].follows.getFollowings.bind(servers[0].follows), + servers[1].follows.getFollowers.bind(servers[1].follows) + ] + + for (const fn of fns) { + const body = await fn({ start: 0, count: 5, sort: 'createdAt', state: 'accepted' }) + expect(body.total).to.equal(0) + } +} + +describe('Test follows moderation', function () { + let servers: PeerTubeServer[] = [] + let commands: FollowsCommand[] + + before(async function () { + this.timeout(240000) + + servers = await createMultipleServers(3) + + // Get the access tokens + await setAccessTokensToServers(servers) + + commands = servers.map(s => s.follows) + }) + + describe('Default behaviour', function () { + + it('Should have server 1 following server 2', async function () { + this.timeout(30000) + + await commands[0].follow({ hosts: [ servers[1].url ] }) + + await waitJobs(servers) + }) + + it('Should have correct follows', async function () { + await checkServer1And2HasFollowers(servers) + }) + + it('Should remove follower on server 2', async function () { + await commands[1].removeFollower({ follower: servers[0] }) + + await waitJobs(servers) + }) + + it('Should not not have follows anymore', async function () { + await checkNoFollowers(servers) + }) + }) + + describe('Disabled/Enabled followers', function () { + + it('Should disable followers on server 2', async function () { + const subConfig = { + followers: { + instance: { + enabled: false, + manualApproval: false + } + } + } + + await servers[1].config.updateCustomSubConfig({ newConfig: subConfig }) + + await commands[0].follow({ hosts: [ servers[1].url ] }) + await waitJobs(servers) + + await checkNoFollowers(servers) + }) + + it('Should re enable followers on server 2', async function () { + const subConfig = { + followers: { + instance: { + enabled: true, + manualApproval: false + } + } + } + + await servers[1].config.updateCustomSubConfig({ newConfig: subConfig }) + + await commands[0].follow({ hosts: [ servers[1].url ] }) + await waitJobs(servers) + + await checkServer1And2HasFollowers(servers) + }) + }) + + describe('Manual approbation', function () { + + it('Should manually approve followers', async function () { + this.timeout(20000) + + await commands[0].unfollow({ target: servers[1] }) + await waitJobs(servers) + + const subConfig = { + followers: { + instance: { + enabled: true, + manualApproval: true + } + } + } + + await servers[1].config.updateCustomSubConfig({ newConfig: subConfig }) + await servers[2].config.updateCustomSubConfig({ newConfig: subConfig }) + + await commands[0].follow({ hosts: [ servers[1].url ] }) + await waitJobs(servers) + + await checkServer1And2HasFollowers(servers, 'pending') + }) + + it('Should accept a follower', async function () { + await commands[1].acceptFollower({ follower: 'peertube@' + servers[0].host }) + await waitJobs(servers) + + await checkServer1And2HasFollowers(servers) + }) + + it('Should reject another follower', async function () { + this.timeout(20000) + + await commands[0].follow({ hosts: [ servers[2].url ] }) + await waitJobs(servers) + + { + const body = await commands[0].getFollowings() + expect(body.total).to.equal(2) + } + + { + const body = await commands[1].getFollowers() + expect(body.total).to.equal(1) + } + + { + const body = await commands[2].getFollowers() + expect(body.total).to.equal(1) + } + + await commands[2].rejectFollower({ follower: 'peertube@' + servers[0].host }) + await waitJobs(servers) + + { // server 1 + { + const { data } = await commands[0].getFollowings({ state: 'accepted' }) + expect(data).to.have.lengthOf(1) + } + + { + const { data } = await commands[0].getFollowings({ state: 'rejected' }) + expect(data).to.have.lengthOf(1) + expectStartWith(data[0].following.url, servers[2].url) + } + } + + { // server 3 + { + const { data } = await commands[2].getFollowers({ state: 'accepted' }) + expect(data).to.have.lengthOf(0) + } + + { + const { data } = await commands[2].getFollowers({ state: 'rejected' }) + expect(data).to.have.lengthOf(1) + expectStartWith(data[0].follower.url, servers[0].url) + } + } + }) + + it('Should still auto accept channel followers', async function () { + await commands[0].follow({ handles: [ 'root_channel@' + servers[1].host ] }) + + await waitJobs(servers) + + const body = await commands[0].getFollowings() + const follow = body.data[0] + expect(follow.following.name).to.equal('root_channel') + expect(follow.state).to.equal('accepted') + }) + }) + + describe('Accept/reject state', function () { + + it('Should not change the follow on refollow with and without auto accept', async function () { + const run = async () => { + await commands[0].follow({ hosts: [ servers[2].url ] }) + await waitJobs(servers) + + await checkFollows({ + follower: servers[0], + followerState: 'rejected', + following: servers[2], + followingState: 'rejected' + }) + } + + await servers[2].config.updateExistingSubConfig({ newConfig: { followers: { instance: { manualApproval: false } } } }) + await run() + + await servers[2].config.updateExistingSubConfig({ newConfig: { followers: { instance: { manualApproval: true } } } }) + await run() + }) + + it('Should not change the rejected status on unfollow', async function () { + await commands[0].unfollow({ target: servers[2] }) + await waitJobs(servers) + + await checkFollows({ + follower: servers[0], + followerState: 'deleted', + following: servers[2], + followingState: 'rejected' + }) + }) + + it('Should delete the follower and add again the follower', async function () { + await commands[2].removeFollower({ follower: servers[0] }) + await waitJobs(servers) + + await commands[0].follow({ hosts: [ servers[2].url ] }) + await waitJobs(servers) + + await checkFollows({ + follower: servers[0], + followerState: 'pending', + following: servers[2], + followingState: 'pending' + }) + }) + + it('Should be able to reject a previously accepted follower', async function () { + await commands[1].rejectFollower({ follower: 'peertube@' + servers[0].host }) + await waitJobs(servers) + + await checkFollows({ + follower: servers[0], + followerState: 'rejected', + following: servers[1], + followingState: 'rejected' + }) + }) + + it('Should be able to re accept a previously rejected follower', async function () { + await commands[1].acceptFollower({ follower: 'peertube@' + servers[0].host }) + await waitJobs(servers) + + await checkFollows({ + follower: servers[0], + followerState: 'accepted', + following: servers[1], + followingState: 'accepted' + }) + }) + }) + + describe('Muted servers', function () { + + it('Should ignore follow requests of muted servers', async function () { + await servers[1].blocklist.addToServerBlocklist({ server: servers[0].host }) + + await commands[0].unfollow({ target: servers[1] }) + + await waitJobs(servers) + + await checkFollows({ + follower: servers[0], + followerState: 'deleted', + following: servers[1], + followingState: 'deleted' + }) + + await commands[0].follow({ hosts: [ servers[1].host ] }) + await waitJobs(servers) + + await checkFollows({ + follower: servers[0], + followerState: 'rejected', + following: servers[1], + followingState: 'deleted' + }) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/server/follows.ts b/packages/tests/src/api/server/follows.ts new file mode 100644 index 000000000..fbe2e87da --- /dev/null +++ b/packages/tests/src/api/server/follows.ts @@ -0,0 +1,644 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { Video, VideoPrivacy } from '@peertube/peertube-models' +import { cleanupTests, createMultipleServers, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@peertube/peertube-server-commands' +import { expectAccountFollows, expectChannelsFollows } from '@tests/shared/actors.js' +import { testCaptionFile } from '@tests/shared/captions.js' +import { dateIsValid } from '@tests/shared/checks.js' +import { completeVideoCheck } from '@tests/shared/videos.js' + +describe('Test follows', function () { + + describe('Complex follow', function () { + let servers: PeerTubeServer[] = [] + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(3) + + // Get the access tokens + await setAccessTokensToServers(servers) + }) + + describe('Data propagation after follow', function () { + + it('Should not have followers/followings', async function () { + for (const server of servers) { + const bodies = await Promise.all([ + server.follows.getFollowers({ start: 0, count: 5, sort: 'createdAt' }), + server.follows.getFollowings({ start: 0, count: 5, sort: 'createdAt' }) + ]) + + for (const body of bodies) { + expect(body.total).to.equal(0) + + const follows = body.data + expect(follows).to.be.an('array') + expect(follows).to.have.lengthOf(0) + } + } + }) + + it('Should have server 1 following root account of server 2 and server 3', async function () { + this.timeout(30000) + + await servers[0].follows.follow({ + hosts: [ servers[2].url ], + handles: [ 'root@' + servers[1].host ] + }) + + await waitJobs(servers) + }) + + it('Should have 2 followings on server 1', async function () { + const body = await servers[0].follows.getFollowings({ start: 0, count: 1, sort: 'createdAt' }) + expect(body.total).to.equal(2) + + let follows = body.data + expect(follows).to.be.an('array') + expect(follows).to.have.lengthOf(1) + + const body2 = await servers[0].follows.getFollowings({ start: 1, count: 1, sort: 'createdAt' }) + follows = follows.concat(body2.data) + + const server2Follow = follows.find(f => f.following.host === servers[1].host) + const server3Follow = follows.find(f => f.following.host === servers[2].host) + + expect(server2Follow).to.not.be.undefined + expect(server2Follow.following.name).to.equal('root') + expect(server2Follow.state).to.equal('accepted') + + expect(server3Follow).to.not.be.undefined + expect(server3Follow.following.name).to.equal('peertube') + expect(server3Follow.state).to.equal('accepted') + }) + + it('Should have 0 followings on server 2 and 3', async function () { + for (const server of [ servers[1], servers[2] ]) { + const body = await server.follows.getFollowings({ start: 0, count: 5, sort: 'createdAt' }) + expect(body.total).to.equal(0) + + const follows = body.data + expect(follows).to.be.an('array') + expect(follows).to.have.lengthOf(0) + } + }) + + it('Should have 1 followers on server 3', async function () { + const body = await servers[2].follows.getFollowers({ start: 0, count: 1, sort: 'createdAt' }) + expect(body.total).to.equal(1) + + const follows = body.data + expect(follows).to.be.an('array') + expect(follows).to.have.lengthOf(1) + expect(follows[0].follower.host).to.equal(servers[0].host) + }) + + it('Should have 0 followers on server 1 and 2', async function () { + for (const server of [ servers[0], servers[1] ]) { + const body = await server.follows.getFollowers({ start: 0, count: 5, sort: 'createdAt' }) + expect(body.total).to.equal(0) + + const follows = body.data + expect(follows).to.be.an('array') + expect(follows).to.have.lengthOf(0) + } + }) + + it('Should search/filter followings on server 1', async function () { + const sort = 'createdAt' + const start = 0 + const count = 1 + + { + const search = ':' + servers[1].port + + { + const body = await servers[0].follows.getFollowings({ start, count, sort, search }) + expect(body.total).to.equal(1) + + const follows = body.data + expect(follows).to.have.lengthOf(1) + expect(follows[0].following.host).to.equal(servers[1].host) + } + + { + const body = await servers[0].follows.getFollowings({ start, count, sort, search, state: 'accepted' }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + } + + { + const body = await servers[0].follows.getFollowings({ start, count, sort, search, state: 'accepted', actorType: 'Person' }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + } + + { + const body = await servers[0].follows.getFollowings({ + start, + count, + sort, + search, + state: 'accepted', + actorType: 'Application' + }) + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + } + + { + const body = await servers[0].follows.getFollowings({ start, count, sort, search, state: 'pending' }) + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + } + } + + { + const body = await servers[0].follows.getFollowings({ start, count, sort, search: 'root' }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + } + + { + const body = await servers[0].follows.getFollowings({ start, count, sort, search: 'bla' }) + expect(body.total).to.equal(0) + + expect(body.data).to.have.lengthOf(0) + } + }) + + it('Should search/filter followers on server 2', async function () { + const start = 0 + const count = 5 + const sort = 'createdAt' + + { + const search = servers[0].port + '' + + { + const body = await servers[2].follows.getFollowers({ start, count, sort, search }) + expect(body.total).to.equal(1) + + const follows = body.data + expect(follows).to.have.lengthOf(1) + expect(follows[0].following.host).to.equal(servers[2].host) + } + + { + const body = await servers[2].follows.getFollowers({ start, count, sort, search, state: 'accepted' }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + } + + { + const body = await servers[2].follows.getFollowers({ start, count, sort, search, state: 'accepted', actorType: 'Person' }) + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + } + + { + const body = await servers[2].follows.getFollowers({ + start, + count, + sort, + search, + state: 'accepted', + actorType: 'Application' + }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + } + + { + const body = await servers[2].follows.getFollowers({ start, count, sort, search, state: 'pending' }) + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + } + } + + { + const body = await servers[2].follows.getFollowers({ start, count, sort, search: 'bla' }) + expect(body.total).to.equal(0) + + const follows = body.data + expect(follows).to.have.lengthOf(0) + } + }) + + it('Should have the correct follows counts', async function () { + await expectAccountFollows({ server: servers[0], handle: 'peertube@' + servers[0].host, followers: 0, following: 2 }) + await expectAccountFollows({ server: servers[0], handle: 'root@' + servers[1].host, followers: 1, following: 0 }) + await expectAccountFollows({ server: servers[0], handle: 'peertube@' + servers[2].host, followers: 1, following: 0 }) + + // Server 2 and 3 does not know server 1 follow another server (there was not a refresh) + await expectAccountFollows({ server: servers[1], handle: 'peertube@' + servers[0].host, followers: 0, following: 1 }) + await expectAccountFollows({ server: servers[1], handle: 'root@' + servers[1].host, followers: 1, following: 0 }) + await expectAccountFollows({ server: servers[1], handle: 'peertube@' + servers[1].host, followers: 0, following: 0 }) + + await expectAccountFollows({ server: servers[2], handle: 'peertube@' + servers[0].host, followers: 0, following: 1 }) + await expectAccountFollows({ server: servers[2], handle: 'peertube@' + servers[2].host, followers: 1, following: 0 }) + }) + + it('Should unfollow server 3 on server 1', async function () { + this.timeout(15000) + + await servers[0].follows.unfollow({ target: servers[2] }) + + await waitJobs(servers) + }) + + it('Should not follow server 3 on server 1 anymore', async function () { + const body = await servers[0].follows.getFollowings({ start: 0, count: 2, sort: 'createdAt' }) + expect(body.total).to.equal(1) + + const follows = body.data + expect(follows).to.be.an('array') + expect(follows).to.have.lengthOf(1) + + expect(follows[0].following.host).to.equal(servers[1].host) + }) + + it('Should not have server 1 as follower on server 3 anymore', async function () { + const body = await servers[2].follows.getFollowers({ start: 0, count: 1, sort: 'createdAt' }) + expect(body.total).to.equal(0) + + const follows = body.data + expect(follows).to.be.an('array') + expect(follows).to.have.lengthOf(0) + }) + + it('Should have the correct follows counts after the unfollow', async function () { + await expectAccountFollows({ server: servers[0], handle: 'peertube@' + servers[0].host, followers: 0, following: 1 }) + await expectAccountFollows({ server: servers[0], handle: 'root@' + servers[1].host, followers: 1, following: 0 }) + await expectAccountFollows({ server: servers[0], handle: 'peertube@' + servers[2].host, followers: 0, following: 0 }) + + await expectAccountFollows({ server: servers[1], handle: 'peertube@' + servers[0].host, followers: 0, following: 1 }) + await expectAccountFollows({ server: servers[1], handle: 'root@' + servers[1].host, followers: 1, following: 0 }) + await expectAccountFollows({ server: servers[1], handle: 'peertube@' + servers[1].host, followers: 0, following: 0 }) + + await expectAccountFollows({ server: servers[2], handle: 'peertube@' + servers[0].host, followers: 0, following: 0 }) + await expectAccountFollows({ server: servers[2], handle: 'peertube@' + servers[2].host, followers: 0, following: 0 }) + }) + + it('Should upload a video on server 2 and 3 and propagate only the video of server 2', async function () { + this.timeout(160000) + + await servers[1].videos.upload({ attributes: { name: 'server2' } }) + await servers[2].videos.upload({ attributes: { name: 'server3' } }) + + await waitJobs(servers) + + { + const { total, data } = await servers[0].videos.list() + expect(total).to.equal(1) + expect(data[0].name).to.equal('server2') + } + + { + const { total, data } = await servers[1].videos.list() + expect(total).to.equal(1) + expect(data[0].name).to.equal('server2') + } + + { + const { total, data } = await servers[2].videos.list() + expect(total).to.equal(1) + expect(data[0].name).to.equal('server3') + } + }) + + it('Should remove account follow', async function () { + this.timeout(15000) + + await servers[0].follows.unfollow({ target: 'root@' + servers[1].host }) + + await waitJobs(servers) + }) + + it('Should have removed the account follow', async function () { + await expectAccountFollows({ server: servers[0], handle: 'root@' + servers[1].host, followers: 0, following: 0 }) + await expectAccountFollows({ server: servers[1], handle: 'root@' + servers[1].host, followers: 0, following: 0 }) + + { + const { total, data } = await servers[0].follows.getFollowings() + expect(total).to.equal(0) + expect(data).to.have.lengthOf(0) + } + + { + const { total, data } = await servers[0].videos.list() + expect(total).to.equal(0) + expect(data).to.have.lengthOf(0) + } + }) + + it('Should follow a channel', async function () { + this.timeout(15000) + + await servers[0].follows.follow({ + handles: [ 'root_channel@' + servers[1].host ] + }) + + await waitJobs(servers) + + await expectChannelsFollows({ server: servers[0], handle: 'root_channel@' + servers[1].host, followers: 1, following: 0 }) + await expectChannelsFollows({ server: servers[1], handle: 'root_channel@' + servers[1].host, followers: 1, following: 0 }) + + { + const { total, data } = await servers[0].follows.getFollowings() + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + } + + { + const { total, data } = await servers[0].videos.list() + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + } + }) + }) + + describe('Should propagate data on a new server follow', function () { + let video4: Video + + before(async function () { + this.timeout(240000) + + const video4Attributes = { + name: 'server3-4', + category: 2, + nsfw: true, + licence: 6, + tags: [ 'tag1', 'tag2', 'tag3' ] + } + + await servers[2].videos.upload({ attributes: { name: 'server3-2' } }) + await servers[2].videos.upload({ attributes: { name: 'server3-3' } }) + + const video4CreateResult = await servers[2].videos.upload({ attributes: video4Attributes }) + + await servers[2].videos.upload({ attributes: { name: 'server3-5' } }) + await servers[2].videos.upload({ attributes: { name: 'server3-6' } }) + + { + const userAccessToken = await servers[2].users.generateUserAndToken('captain') + + await servers[2].videos.rate({ id: video4CreateResult.id, rating: 'like' }) + await servers[2].videos.rate({ token: userAccessToken, id: video4CreateResult.id, rating: 'dislike' }) + } + + { + await servers[2].comments.createThread({ videoId: video4CreateResult.id, text: 'my super first comment' }) + + await servers[2].comments.addReplyToLastThread({ text: 'my super answer to thread 1' }) + await servers[2].comments.addReplyToLastReply({ text: 'my super answer to answer of thread 1' }) + await servers[2].comments.addReplyToLastThread({ text: 'my second answer to thread 1' }) + } + + { + const { id: threadId } = await servers[2].comments.createThread({ videoId: video4CreateResult.id, text: 'will be deleted' }) + await servers[2].comments.addReplyToLastThread({ text: 'answer to deleted' }) + + const { id: replyId } = await servers[2].comments.addReplyToLastThread({ text: 'will also be deleted' }) + + await servers[2].comments.addReplyToLastReply({ text: 'my second answer to deleted' }) + + await servers[2].comments.delete({ videoId: video4CreateResult.id, commentId: threadId }) + await servers[2].comments.delete({ videoId: video4CreateResult.id, commentId: replyId }) + } + + await servers[2].captions.add({ + language: 'ar', + videoId: video4CreateResult.id, + fixture: 'subtitle-good2.vtt' + }) + + await waitJobs(servers) + + // Server 1 follows server 3 + await servers[0].follows.follow({ hosts: [ servers[2].url ] }) + + await waitJobs(servers) + }) + + it('Should have the correct follows counts', async function () { + await expectAccountFollows({ server: servers[0], handle: 'peertube@' + servers[0].host, followers: 0, following: 2 }) + await expectAccountFollows({ server: servers[0], handle: 'root@' + servers[1].host, followers: 0, following: 0 }) + await expectChannelsFollows({ server: servers[0], handle: 'root_channel@' + servers[1].host, followers: 1, following: 0 }) + await expectAccountFollows({ server: servers[0], handle: 'peertube@' + servers[2].host, followers: 1, following: 0 }) + + await expectAccountFollows({ server: servers[1], handle: 'peertube@' + servers[0].host, followers: 0, following: 1 }) + await expectAccountFollows({ server: servers[1], handle: 'peertube@' + servers[1].host, followers: 0, following: 0 }) + await expectAccountFollows({ server: servers[1], handle: 'root@' + servers[1].host, followers: 0, following: 0 }) + await expectChannelsFollows({ server: servers[1], handle: 'root_channel@' + servers[1].host, followers: 1, following: 0 }) + + await expectAccountFollows({ server: servers[2], handle: 'peertube@' + servers[0].host, followers: 0, following: 1 }) + await expectAccountFollows({ server: servers[2], handle: 'peertube@' + servers[2].host, followers: 1, following: 0 }) + }) + + it('Should have propagated videos', async function () { + const { total, data } = await servers[0].videos.list() + expect(total).to.equal(7) + + const video2 = data.find(v => v.name === 'server3-2') + video4 = data.find(v => v.name === 'server3-4') + const video6 = data.find(v => v.name === 'server3-6') + + expect(video2).to.not.be.undefined + expect(video4).to.not.be.undefined + expect(video6).to.not.be.undefined + + const isLocal = false + const checkAttributes = { + name: 'server3-4', + category: 2, + licence: 6, + language: 'zh', + nsfw: true, + description: 'my super description', + support: 'my super support text', + account: { + name: 'root', + host: servers[2].host + }, + isLocal, + commentsEnabled: true, + downloadEnabled: true, + duration: 5, + tags: [ 'tag1', 'tag2', 'tag3' ], + privacy: VideoPrivacy.PUBLIC, + likes: 1, + dislikes: 1, + channel: { + displayName: 'Main root channel', + name: 'root_channel', + description: '', + isLocal + }, + fixture: 'video_short.webm', + files: [ + { + resolution: 720, + size: 218910 + } + ] + } + await completeVideoCheck({ + server: servers[0], + originServer: servers[2], + videoUUID: video4.uuid, + attributes: checkAttributes + }) + }) + + it('Should have propagated comments', async function () { + const { total, data } = await servers[0].comments.listThreads({ videoId: video4.id, sort: 'createdAt' }) + + expect(total).to.equal(2) + expect(data).to.be.an('array') + expect(data).to.have.lengthOf(2) + + { + const comment = data[0] + expect(comment.inReplyToCommentId).to.be.null + expect(comment.text).equal('my super first comment') + expect(comment.videoId).to.equal(video4.id) + expect(comment.id).to.equal(comment.threadId) + expect(comment.account.name).to.equal('root') + expect(comment.account.host).to.equal(servers[2].host) + expect(comment.totalReplies).to.equal(3) + expect(dateIsValid(comment.createdAt as string)).to.be.true + expect(dateIsValid(comment.updatedAt as string)).to.be.true + + const threadId = comment.threadId + + const tree = await servers[0].comments.getThread({ videoId: video4.id, threadId }) + expect(tree.comment.text).equal('my super first comment') + expect(tree.children).to.have.lengthOf(2) + + const firstChild = tree.children[0] + expect(firstChild.comment.text).to.equal('my super answer to thread 1') + expect(firstChild.children).to.have.lengthOf(1) + + const childOfFirstChild = firstChild.children[0] + expect(childOfFirstChild.comment.text).to.equal('my super answer to answer of thread 1') + expect(childOfFirstChild.children).to.have.lengthOf(0) + + const secondChild = tree.children[1] + expect(secondChild.comment.text).to.equal('my second answer to thread 1') + expect(secondChild.children).to.have.lengthOf(0) + } + + { + const deletedComment = data[1] + expect(deletedComment).to.not.be.undefined + expect(deletedComment.isDeleted).to.be.true + expect(deletedComment.deletedAt).to.not.be.null + expect(deletedComment.text).to.equal('') + expect(deletedComment.inReplyToCommentId).to.be.null + expect(deletedComment.account).to.be.null + expect(deletedComment.totalReplies).to.equal(2) + expect(dateIsValid(deletedComment.deletedAt as string)).to.be.true + + const tree = await servers[0].comments.getThread({ videoId: video4.id, threadId: deletedComment.threadId }) + const [ commentRoot, deletedChildRoot ] = tree.children + + expect(deletedChildRoot).to.not.be.undefined + expect(deletedChildRoot.comment.isDeleted).to.be.true + expect(deletedChildRoot.comment.deletedAt).to.not.be.null + expect(deletedChildRoot.comment.text).to.equal('') + expect(deletedChildRoot.comment.inReplyToCommentId).to.equal(deletedComment.id) + expect(deletedChildRoot.comment.account).to.be.null + expect(deletedChildRoot.children).to.have.lengthOf(1) + + const answerToDeletedChild = deletedChildRoot.children[0] + expect(answerToDeletedChild.comment).to.not.be.undefined + expect(answerToDeletedChild.comment.inReplyToCommentId).to.equal(deletedChildRoot.comment.id) + expect(answerToDeletedChild.comment.text).to.equal('my second answer to deleted') + expect(answerToDeletedChild.comment.account.name).to.equal('root') + + expect(commentRoot.comment).to.not.be.undefined + expect(commentRoot.comment.inReplyToCommentId).to.equal(deletedComment.id) + expect(commentRoot.comment.text).to.equal('answer to deleted') + expect(commentRoot.comment.account.name).to.equal('root') + } + }) + + it('Should have propagated captions', async function () { + const body = await servers[0].captions.list({ videoId: video4.id }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + + const caption1 = body.data[0] + expect(caption1.language.id).to.equal('ar') + expect(caption1.language.label).to.equal('Arabic') + expect(caption1.captionPath).to.match(new RegExp('^/lazy-static/video-captions/.+-ar.vtt$')) + await testCaptionFile(servers[0].url, caption1.captionPath, 'Subtitle good 2.') + }) + + it('Should unfollow server 3 on server 1 and does not list server 3 videos', async function () { + this.timeout(5000) + + await servers[0].follows.unfollow({ target: servers[2] }) + + await waitJobs(servers) + + const { total } = await servers[0].videos.list() + expect(total).to.equal(1) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) + }) + + describe('Simple data propagation propagate data on a new channel follow', function () { + let servers: PeerTubeServer[] = [] + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(3) + await setAccessTokensToServers(servers) + + await servers[0].videos.upload({ attributes: { name: 'video to add' } }) + + await waitJobs(servers) + + for (const server of [ servers[1], servers[2] ]) { + const video = await server.videos.find({ name: 'video to add' }) + expect(video).to.not.exist + } + }) + + it('Should have propagated video after new channel follow', async function () { + this.timeout(60000) + + await servers[1].follows.follow({ handles: [ 'root_channel@' + servers[0].host ] }) + + await waitJobs(servers) + + const video = await servers[1].videos.find({ name: 'video to add' }) + expect(video).to.exist + }) + + it('Should have propagated video after new account follow', async function () { + this.timeout(60000) + + await servers[2].follows.follow({ handles: [ 'root@' + servers[0].host ] }) + + await waitJobs(servers) + + const video = await servers[2].videos.find({ name: 'video to add' }) + expect(video).to.exist + }) + + after(async function () { + await cleanupTests(servers) + }) + }) +}) diff --git a/packages/tests/src/api/server/handle-down.ts b/packages/tests/src/api/server/handle-down.ts new file mode 100644 index 000000000..604df129f --- /dev/null +++ b/packages/tests/src/api/server/handle-down.ts @@ -0,0 +1,339 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { HttpStatusCode, JobState, VideoCreateResult, VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + CommentsCommand, + createMultipleServers, + killallServers, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' +import { SQLCommand } from '@tests/shared/sql-command.js' +import { completeVideoCheck } from '@tests/shared/videos.js' + +describe('Test handle downs', function () { + let servers: PeerTubeServer[] = [] + let sqlCommands: SQLCommand[] = [] + + let threadIdServer1: number + let threadIdServer2: number + let commentIdServer1: number + let commentIdServer2: number + let missedVideo1: VideoCreateResult + let missedVideo2: VideoCreateResult + let unlistedVideo: VideoCreateResult + + const videoIdsServer1: string[] = [] + + const videoAttributes = { + name: 'my super name for server 1', + category: 5, + licence: 4, + language: 'ja', + nsfw: true, + privacy: VideoPrivacy.PUBLIC, + description: 'my super description for server 1', + support: 'my super support text for server 1', + tags: [ 'tag1p1', 'tag2p1' ], + fixture: 'video_short1.webm' + } + + const unlistedVideoAttributes = { ...videoAttributes, privacy: VideoPrivacy.UNLISTED } + + let checkAttributes: any + let unlistedCheckAttributes: any + + let commentCommands: CommentsCommand[] + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(3) + commentCommands = servers.map(s => s.comments) + + checkAttributes = { + name: 'my super name for server 1', + category: 5, + licence: 4, + language: 'ja', + nsfw: true, + description: 'my super description for server 1', + support: 'my super support text for server 1', + account: { + name: 'root', + host: servers[0].host + }, + isLocal: false, + duration: 10, + tags: [ 'tag1p1', 'tag2p1' ], + privacy: VideoPrivacy.PUBLIC, + commentsEnabled: true, + downloadEnabled: true, + channel: { + name: 'root_channel', + displayName: 'Main root channel', + description: '', + isLocal: false + }, + fixture: 'video_short1.webm', + files: [ + { + resolution: 720, + size: 572456 + } + ] + } + unlistedCheckAttributes = { ...checkAttributes, privacy: VideoPrivacy.UNLISTED } + + // Get the access tokens + await setAccessTokensToServers(servers) + + sqlCommands = servers.map(s => new SQLCommand(s)) + }) + + it('Should remove followers that are often down', async function () { + this.timeout(240000) + + // Server 2 and 3 follow server 1 + await servers[1].follows.follow({ hosts: [ servers[0].url ] }) + await servers[2].follows.follow({ hosts: [ servers[0].url ] }) + + await waitJobs(servers) + + // Upload a video to server 1 + await servers[0].videos.upload({ attributes: videoAttributes }) + + await waitJobs(servers) + + // And check all servers have this video + for (const server of servers) { + const { data } = await server.videos.list() + expect(data).to.be.an('array') + expect(data).to.have.lengthOf(1) + } + + // Kill server 2 + await killallServers([ servers[1] ]) + + // Remove server 2 follower + for (let i = 0; i < 10; i++) { + await servers[0].videos.upload({ attributes: videoAttributes }) + } + + await waitJobs([ servers[0], servers[2] ]) + + // Kill server 3 + await killallServers([ servers[2] ]) + + missedVideo1 = await servers[0].videos.upload({ attributes: videoAttributes }) + + missedVideo2 = await servers[0].videos.upload({ attributes: videoAttributes }) + + // Unlisted video + unlistedVideo = await servers[0].videos.upload({ attributes: unlistedVideoAttributes }) + + // Add comments to video 2 + { + const text = 'thread 1' + let comment = await commentCommands[0].createThread({ videoId: missedVideo2.uuid, text }) + threadIdServer1 = comment.id + + comment = await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: comment.id, text: 'comment 1-1' }) + + const created = await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: comment.id, text: 'comment 1-2' }) + commentIdServer1 = created.id + } + + await waitJobs(servers[0]) + // Wait scheduler + await wait(11000) + + // Only server 3 is still a follower of server 1 + const body = await servers[0].follows.getFollowers({ start: 0, count: 2, sort: 'createdAt' }) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].follower.host).to.equal(servers[2].host) + }) + + it('Should not have pending/processing jobs anymore', async function () { + const states: JobState[] = [ 'waiting', 'active' ] + + for (const state of states) { + const body = await servers[0].jobs.list({ + state, + start: 0, + count: 50, + sort: '-createdAt' + }) + expect(body.data).to.have.length(0) + } + }) + + it('Should re-follow server 1', async function () { + this.timeout(70000) + + await servers[1].run() + await servers[2].run() + + await servers[1].follows.unfollow({ target: servers[0] }) + await waitJobs(servers) + + await servers[1].follows.follow({ hosts: [ servers[0].url ] }) + + await waitJobs(servers) + + const body = await servers[0].follows.getFollowers({ start: 0, count: 2, sort: 'createdAt' }) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(2) + }) + + it('Should send an update to server 3, and automatically fetch the video', async function () { + this.timeout(15000) + + { + const { data } = await servers[2].videos.list() + expect(data).to.be.an('array') + expect(data).to.have.lengthOf(11) + } + + await servers[0].videos.update({ id: missedVideo1.uuid }) + await servers[0].videos.update({ id: unlistedVideo.uuid }) + + await waitJobs(servers) + + { + const { data } = await servers[2].videos.list() + expect(data).to.be.an('array') + // 1 video is unlisted + expect(data).to.have.lengthOf(12) + } + + // Check unlisted video + const video = await servers[2].videos.get({ id: unlistedVideo.uuid }) + await completeVideoCheck({ server: servers[2], originServer: servers[0], videoUUID: video.uuid, attributes: unlistedCheckAttributes }) + }) + + it('Should send comments on a video to server 3, and automatically fetch the video', async function () { + this.timeout(25000) + + await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: commentIdServer1, text: 'comment 1-3' }) + + await waitJobs(servers) + + await servers[2].videos.get({ id: missedVideo2.uuid }) + + { + const { data } = await servers[2].comments.listThreads({ videoId: missedVideo2.uuid }) + expect(data).to.be.an('array') + expect(data).to.have.lengthOf(1) + + threadIdServer2 = data[0].id + + const tree = await servers[2].comments.getThread({ videoId: missedVideo2.uuid, threadId: threadIdServer2 }) + expect(tree.comment.text).equal('thread 1') + expect(tree.children).to.have.lengthOf(1) + + const firstChild = tree.children[0] + expect(firstChild.comment.text).to.equal('comment 1-1') + expect(firstChild.children).to.have.lengthOf(1) + + const childOfFirstChild = firstChild.children[0] + expect(childOfFirstChild.comment.text).to.equal('comment 1-2') + expect(childOfFirstChild.children).to.have.lengthOf(1) + + const childOfChildFirstChild = childOfFirstChild.children[0] + expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3') + expect(childOfChildFirstChild.children).to.have.lengthOf(0) + + commentIdServer2 = childOfChildFirstChild.comment.id + } + }) + + it('Should correctly reply to the comment', async function () { + this.timeout(15000) + + await servers[2].comments.addReply({ videoId: missedVideo2.uuid, toCommentId: commentIdServer2, text: 'comment 1-4' }) + + await waitJobs(servers) + + const tree = await commentCommands[0].getThread({ videoId: missedVideo2.uuid, threadId: threadIdServer1 }) + + expect(tree.comment.text).equal('thread 1') + expect(tree.children).to.have.lengthOf(1) + + const firstChild = tree.children[0] + expect(firstChild.comment.text).to.equal('comment 1-1') + expect(firstChild.children).to.have.lengthOf(1) + + const childOfFirstChild = firstChild.children[0] + expect(childOfFirstChild.comment.text).to.equal('comment 1-2') + expect(childOfFirstChild.children).to.have.lengthOf(1) + + const childOfChildFirstChild = childOfFirstChild.children[0] + expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3') + expect(childOfChildFirstChild.children).to.have.lengthOf(1) + + const childOfChildOfChildOfFirstChild = childOfChildFirstChild.children[0] + expect(childOfChildOfChildOfFirstChild.comment.text).to.equal('comment 1-4') + expect(childOfChildOfChildOfFirstChild.children).to.have.lengthOf(0) + }) + + it('Should upload many videos on server 1', async function () { + this.timeout(240000) + + for (let i = 0; i < 10; i++) { + const uuid = (await servers[0].videos.quickUpload({ name: 'video ' + i })).uuid + videoIdsServer1.push(uuid) + } + + await waitJobs(servers) + + for (const id of videoIdsServer1) { + await servers[1].videos.get({ id }) + } + + await waitJobs(servers) + await sqlCommands[1].setActorFollowScores(20) + + // Wait video expiration + await wait(11000) + + // Refresh video -> score + 10 = 30 + await servers[1].videos.get({ id: videoIdsServer1[0] }) + + await waitJobs(servers) + }) + + it('Should remove followings that are down', async function () { + this.timeout(120000) + + await killallServers([ servers[0] ]) + + // Wait video expiration + await wait(11000) + + for (let i = 0; i < 5; i++) { + try { + await servers[1].videos.get({ id: videoIdsServer1[i] }) + await waitJobs([ servers[1] ]) + await wait(1500) + } catch {} + } + + for (const id of videoIdsServer1) { + await servers[1].videos.get({ id, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + } + }) + + after(async function () { + for (const sqlCommand of sqlCommands) { + await sqlCommand.cleanup() + } + + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/server/homepage.ts b/packages/tests/src/api/server/homepage.ts new file mode 100644 index 000000000..082a2fb91 --- /dev/null +++ b/packages/tests/src/api/server/homepage.ts @@ -0,0 +1,81 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + CustomPagesCommand, + killallServers, + PeerTubeServer, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultChannelAvatar +} from '@peertube/peertube-server-commands' + +async function getHomepageState (server: PeerTubeServer) { + const config = await server.config.getConfig() + + return config.homepage.enabled +} + +describe('Test instance homepage actions', function () { + let server: PeerTubeServer + let command: CustomPagesCommand + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + await setDefaultChannelAvatar(server) + await setDefaultAccountAvatar(server) + + command = server.customPage + }) + + it('Should not have a homepage', async function () { + const state = await getHomepageState(server) + expect(state).to.be.false + + await command.getInstanceHomepage({ expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should set a homepage', async function () { + await command.updateInstanceHomepage({ content: '' }) + + const page = await command.getInstanceHomepage() + expect(page.content).to.equal('') + + const state = await getHomepageState(server) + expect(state).to.be.true + }) + + it('Should have the same homepage after a restart', async function () { + this.timeout(30000) + + await killallServers([ server ]) + + await server.run() + + const page = await command.getInstanceHomepage() + expect(page.content).to.equal('') + + const state = await getHomepageState(server) + expect(state).to.be.true + }) + + it('Should empty the homepage', async function () { + await command.updateInstanceHomepage({ content: '' }) + + const page = await command.getInstanceHomepage() + expect(page.content).to.be.empty + + const state = await getHomepageState(server) + expect(state).to.be.false + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/server/index.ts b/packages/tests/src/api/server/index.ts new file mode 100644 index 000000000..5c80a5a37 --- /dev/null +++ b/packages/tests/src/api/server/index.ts @@ -0,0 +1,22 @@ +import './auto-follows.js' +import './bulk.js' +import './config-defaults.js' +import './config.js' +import './contact-form.js' +import './email.js' +import './follow-constraints.js' +import './follows.js' +import './follows-moderation.js' +import './homepage.js' +import './handle-down.js' +import './jobs.js' +import './logs.js' +import './reverse-proxy.js' +import './services.js' +import './slow-follows.js' +import './stats.js' +import './tracker.js' +import './no-client.js' +import './open-telemetry.js' +import './plugins.js' +import './proxy.js' diff --git a/packages/tests/src/api/server/jobs.ts b/packages/tests/src/api/server/jobs.ts new file mode 100644 index 000000000..3d60b1431 --- /dev/null +++ b/packages/tests/src/api/server/jobs.ts @@ -0,0 +1,128 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { dateIsValid } from '@tests/shared/checks.js' +import { wait } from '@peertube/peertube-core-utils' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test jobs', function () { + let servers: PeerTubeServer[] + + before(async function () { + this.timeout(240000) + + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + }) + + it('Should create some jobs', async function () { + this.timeout(240000) + + await servers[1].videos.upload({ attributes: { name: 'video1' } }) + await servers[1].videos.upload({ attributes: { name: 'video2' } }) + + await waitJobs(servers) + }) + + it('Should list jobs', async function () { + const body = await servers[1].jobs.list({ state: 'completed' }) + expect(body.total).to.be.above(2) + expect(body.data).to.have.length.above(2) + }) + + it('Should list jobs with sort, pagination and job type', async function () { + { + const body = await servers[1].jobs.list({ + state: 'completed', + start: 1, + count: 2, + sort: 'createdAt' + }) + expect(body.total).to.be.above(2) + expect(body.data).to.have.lengthOf(2) + + let job = body.data[0] + // Skip repeat jobs + if (job.type === 'videos-views-stats') job = body.data[1] + + expect(job.state).to.equal('completed') + expect(dateIsValid(job.createdAt as string)).to.be.true + expect(dateIsValid(job.processedOn as string)).to.be.true + expect(dateIsValid(job.finishedOn as string)).to.be.true + } + + { + const body = await servers[1].jobs.list({ + state: 'completed', + start: 0, + count: 100, + sort: 'createdAt', + jobType: 'activitypub-http-broadcast' + }) + expect(body.total).to.be.above(2) + + for (const j of body.data) { + expect(j.type).to.equal('activitypub-http-broadcast') + } + } + }) + + it('Should list all jobs', async function () { + const body = await servers[1].jobs.list() + expect(body.total).to.be.above(2) + + const jobs = body.data + expect(jobs).to.have.length.above(2) + + expect(jobs.find(j => j.state === 'completed')).to.not.be.undefined + }) + + it('Should pause the job queue', async function () { + this.timeout(120000) + + const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video2' } }) + await waitJobs(servers) + + await servers[1].jobs.pauseJobQueue() + await servers[1].videos.runTranscoding({ videoId: uuid, transcodingType: 'hls' }) + + await wait(5000) + + { + const body = await servers[1].jobs.list({ state: 'waiting', jobType: 'video-transcoding' }) + // waiting includes waiting-children + expect(body.data).to.have.lengthOf(4) + } + + { + const body = await servers[1].jobs.list({ state: 'waiting-children', jobType: 'video-transcoding' }) + expect(body.data).to.have.lengthOf(1) + } + }) + + it('Should resume the job queue', async function () { + this.timeout(120000) + + await servers[1].jobs.resumeJobQueue() + + await waitJobs(servers) + + const body = await servers[1].jobs.list({ state: 'waiting', jobType: 'video-transcoding' }) + expect(body.data).to.have.lengthOf(0) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/server/logs.ts b/packages/tests/src/api/server/logs.ts new file mode 100644 index 000000000..11c86d694 --- /dev/null +++ b/packages/tests/src/api/server/logs.ts @@ -0,0 +1,265 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + killallServers, + LogsCommand, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test logs', function () { + let server: PeerTubeServer + let logsCommand: LogsCommand + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + + logsCommand = server.logs + }) + + describe('With the standard log file', function () { + + it('Should get logs with a start date', async function () { + this.timeout(60000) + + await server.videos.upload({ attributes: { name: 'video 1' } }) + await waitJobs([ server ]) + + const now = new Date() + + await server.videos.upload({ attributes: { name: 'video 2' } }) + await waitJobs([ server ]) + + const body = await logsCommand.getLogs({ startDate: now }) + const logsString = JSON.stringify(body) + + expect(logsString.includes('Video with name video 1')).to.be.false + expect(logsString.includes('Video with name video 2')).to.be.true + }) + + it('Should get logs with an end date', async function () { + this.timeout(60000) + + await server.videos.upload({ attributes: { name: 'video 3' } }) + await waitJobs([ server ]) + + const now1 = new Date() + + await server.videos.upload({ attributes: { name: 'video 4' } }) + await waitJobs([ server ]) + + const now2 = new Date() + + await server.videos.upload({ attributes: { name: 'video 5' } }) + await waitJobs([ server ]) + + const body = await logsCommand.getLogs({ startDate: now1, endDate: now2 }) + const logsString = JSON.stringify(body) + + expect(logsString.includes('Video with name video 3')).to.be.false + expect(logsString.includes('Video with name video 4')).to.be.true + expect(logsString.includes('Video with name video 5')).to.be.false + }) + + it('Should filter by level', async function () { + this.timeout(60000) + + const now = new Date() + + await server.videos.upload({ attributes: { name: 'video 6' } }) + await waitJobs([ server ]) + + { + const body = await logsCommand.getLogs({ startDate: now, level: 'info' }) + const logsString = JSON.stringify(body) + + expect(logsString.includes('Video with name video 6')).to.be.true + } + + { + const body = await logsCommand.getLogs({ startDate: now, level: 'warn' }) + const logsString = JSON.stringify(body) + + expect(logsString.includes('Video with name video 6')).to.be.false + } + }) + + it('Should filter by tag', async function () { + const now = new Date() + + const { uuid } = await server.videos.upload({ attributes: { name: 'video 6' } }) + await waitJobs([ server ]) + + { + const body = await logsCommand.getLogs({ startDate: now, level: 'debug', tagsOneOf: [ 'toto' ] }) + expect(body).to.have.lengthOf(0) + } + + { + const body = await logsCommand.getLogs({ startDate: now, level: 'debug', tagsOneOf: [ uuid ] }) + expect(body).to.not.have.lengthOf(0) + + for (const line of body) { + expect(line.tags).to.contain(uuid) + } + } + }) + + it('Should log ping requests', async function () { + const now = new Date() + + await server.servers.ping() + + const body = await logsCommand.getLogs({ startDate: now, level: 'info' }) + const logsString = JSON.stringify(body) + + expect(logsString.includes('/api/v1/ping')).to.be.true + }) + + it('Should not log ping requests', async function () { + this.timeout(60000) + + await killallServers([ server ]) + + await server.run({ log: { log_ping_requests: false } }) + + const now = new Date() + + await server.servers.ping() + + const body = await logsCommand.getLogs({ startDate: now, level: 'info' }) + const logsString = JSON.stringify(body) + + expect(logsString.includes('/api/v1/ping')).to.be.false + }) + }) + + describe('With the audit log', function () { + + it('Should get logs with a start date', async function () { + this.timeout(60000) + + await server.videos.upload({ attributes: { name: 'video 7' } }) + await waitJobs([ server ]) + + const now = new Date() + + await server.videos.upload({ attributes: { name: 'video 8' } }) + await waitJobs([ server ]) + + const body = await logsCommand.getAuditLogs({ startDate: now }) + const logsString = JSON.stringify(body) + + expect(logsString.includes('video 7')).to.be.false + expect(logsString.includes('video 8')).to.be.true + + expect(body).to.have.lengthOf(1) + + const item = body[0] + + const message = JSON.parse(item.message) + expect(message.domain).to.equal('videos') + expect(message.action).to.equal('create') + }) + + it('Should get logs with an end date', async function () { + this.timeout(60000) + + await server.videos.upload({ attributes: { name: 'video 9' } }) + await waitJobs([ server ]) + + const now1 = new Date() + + await server.videos.upload({ attributes: { name: 'video 10' } }) + await waitJobs([ server ]) + + const now2 = new Date() + + await server.videos.upload({ attributes: { name: 'video 11' } }) + await waitJobs([ server ]) + + const body = await logsCommand.getAuditLogs({ startDate: now1, endDate: now2 }) + const logsString = JSON.stringify(body) + + expect(logsString.includes('video 9')).to.be.false + expect(logsString.includes('video 10')).to.be.true + expect(logsString.includes('video 11')).to.be.false + }) + }) + + describe('When creating log from the client', function () { + + it('Should create a warn client log', async function () { + const now = new Date() + + await server.logs.createLogClient({ + payload: { + level: 'warn', + url: 'http://example.com', + message: 'my super client message' + }, + token: null + }) + + const body = await logsCommand.getLogs({ startDate: now }) + const logsString = JSON.stringify(body) + + expect(logsString.includes('my super client message')).to.be.true + }) + + it('Should create an error authenticated client log', async function () { + const now = new Date() + + await server.logs.createLogClient({ + payload: { + url: 'https://example.com/page1', + level: 'error', + message: 'my super client message 2', + userAgent: 'super user agent', + meta: '{hello}', + stackTrace: 'super stack trace' + } + }) + + const body = await logsCommand.getLogs({ startDate: now }) + const logsString = JSON.stringify(body) + + expect(logsString.includes('my super client message 2')).to.be.true + expect(logsString.includes('super user agent')).to.be.true + expect(logsString.includes('super stack trace')).to.be.true + expect(logsString.includes('{hello}')).to.be.true + expect(logsString.includes('https://example.com/page1')).to.be.true + }) + + it('Should refuse to create client logs', async function () { + await server.kill() + + await server.run({ + log: { + accept_client_log: false + } + }) + + await server.logs.createLogClient({ + payload: { + level: 'warn', + url: 'http://example.com', + message: 'my super client message' + }, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/server/no-client.ts b/packages/tests/src/api/server/no-client.ts new file mode 100644 index 000000000..0f097d50b --- /dev/null +++ b/packages/tests/src/api/server/no-client.ts @@ -0,0 +1,24 @@ +import request from 'supertest' +import { HttpStatusCode } from '@peertube/peertube-models' +import { cleanupTests, createSingleServer, PeerTubeServer } from '@peertube/peertube-server-commands' + +describe('Start and stop server without web client routes', function () { + let server: PeerTubeServer + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1, {}, { peertubeArgs: [ '--no-client' ] }) + }) + + it('Should fail getting the client', function () { + const req = request(server.url) + .get('/') + + return req.expect(HttpStatusCode.NOT_FOUND_404) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/server/open-telemetry.ts b/packages/tests/src/api/server/open-telemetry.ts new file mode 100644 index 000000000..8ed3801db --- /dev/null +++ b/packages/tests/src/api/server/open-telemetry.ts @@ -0,0 +1,193 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { HttpStatusCode, PlaybackMetricCreate, VideoPrivacy, VideoResolution } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + makeRawRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' +import { expectLogDoesNotContain, expectLogContain } from '@tests/shared/checks.js' +import { MockHTTP } from '@tests/shared/mock-servers/mock-http.js' + +describe('Open Telemetry', function () { + let server: PeerTubeServer + + describe('Metrics', function () { + const metricsUrl = 'http://127.0.0.1:9092/metrics' + + it('Should not enable open telemetry metrics', async function () { + this.timeout(60000) + + server = await createSingleServer(1) + + let hasError = false + try { + await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + } catch (err) { + hasError = err.message.includes('ECONNREFUSED') + } + + expect(hasError).to.be.true + + await server.kill() + }) + + it('Should enable open telemetry metrics', async function () { + this.timeout(120000) + + await server.run({ + open_telemetry: { + metrics: { + enabled: true + } + } + }) + + // Simulate a HTTP request + await server.videos.list() + + const res = await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.OK_200 }) + expect(res.text).to.contain('peertube_job_queue_total{') + expect(res.text).to.contain('http_request_duration_ms_bucket{') + }) + + it('Should have playback metrics', async function () { + await setAccessTokensToServers([ server ]) + + const video = await server.videos.quickUpload({ name: 'video' }) + + await server.metrics.addPlaybackMetric({ + metrics: { + playerMode: 'p2p-media-loader', + resolution: VideoResolution.H_1080P, + fps: 30, + resolutionChanges: 1, + errors: 2, + downloadedBytesP2P: 0, + downloadedBytesHTTP: 0, + uploadedBytesP2P: 5, + p2pPeers: 1, + p2pEnabled: false, + videoId: video.uuid + } + }) + + const res = await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.OK_200 }) + + expect(res.text).to.contain('peertube_playback_http_downloaded_bytes_total{') + expect(res.text).to.contain('peertube_playback_p2p_peers{') + expect(res.text).to.contain('p2pEnabled="false"') + }) + + it('Should take the last playback metric', async function () { + await setAccessTokensToServers([ server ]) + + const video = await server.videos.quickUpload({ name: 'video' }) + + const metrics = { + playerMode: 'p2p-media-loader', + resolution: VideoResolution.H_1080P, + fps: 30, + resolutionChanges: 1, + errors: 2, + downloadedBytesP2P: 0, + downloadedBytesHTTP: 0, + uploadedBytesP2P: 5, + p2pPeers: 7, + p2pEnabled: false, + videoId: video.uuid + } as PlaybackMetricCreate + + await server.metrics.addPlaybackMetric({ metrics }) + + metrics.p2pPeers = 42 + await server.metrics.addPlaybackMetric({ metrics }) + + const res = await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.OK_200 }) + + // eslint-disable-next-line max-len + const label = `{videoOrigin="local",playerMode="p2p-media-loader",resolution="1080",fps="30",p2pEnabled="false",videoUUID="${video.uuid}"}` + expect(res.text).to.contain(`peertube_playback_p2p_peers${label} 42`) + expect(res.text).to.not.contain(`peertube_playback_p2p_peers${label} 7`) + }) + + it('Should disable http request duration metrics', async function () { + await server.kill() + + await server.run({ + open_telemetry: { + metrics: { + enabled: true, + http_request_duration: { + enabled: false + } + } + } + }) + + // Simulate a HTTP request + await server.videos.list() + + const res = await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.OK_200 }) + expect(res.text).to.not.contain('http_request_duration_ms_bucket{') + }) + + after(async function () { + await server.kill() + }) + }) + + describe('Tracing', function () { + let mockHTTP: MockHTTP + let mockPort: number + + before(async function () { + mockHTTP = new MockHTTP() + mockPort = await mockHTTP.initialize() + }) + + it('Should enable open telemetry tracing', async function () { + server = await createSingleServer(1) + + await expectLogDoesNotContain(server, 'Registering Open Telemetry tracing') + + await server.kill() + }) + + it('Should enable open telemetry metrics', async function () { + await server.run({ + open_telemetry: { + tracing: { + enabled: true, + jaeger_exporter: { + endpoint: 'http://127.0.0.1:' + mockPort + } + } + } + }) + + await expectLogContain(server, 'Registering Open Telemetry tracing') + }) + + it('Should upload a video and correctly works', async function () { + await setAccessTokensToServers([ server ]) + + const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PUBLIC }) + + const video = await server.videos.get({ id: uuid }) + + expect(video.name).to.equal('video') + }) + + after(async function () { + await mockHTTP.terminate() + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/server/plugins.ts b/packages/tests/src/api/server/plugins.ts new file mode 100644 index 000000000..a78cea025 --- /dev/null +++ b/packages/tests/src/api/server/plugins.ts @@ -0,0 +1,410 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { pathExists, remove } from 'fs-extra/esm' +import { join } from 'path' +import { wait } from '@peertube/peertube-core-utils' +import { HttpStatusCode, PluginType } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + killallServers, + makeGetRequest, + PeerTubeServer, + PluginsCommand, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' +import { SQLCommand } from '@tests/shared/sql-command.js' +import { testHelloWorldRegisteredSettings } from '@tests/shared/plugins.js' + +describe('Test plugins', function () { + let server: PeerTubeServer + let sqlCommand: SQLCommand + let command: PluginsCommand + + before(async function () { + this.timeout(30000) + + const configOverride = { + plugins: { + index: { check_latest_versions_interval: '5 seconds' } + } + } + server = await createSingleServer(1, configOverride) + await setAccessTokensToServers([ server ]) + + command = server.plugins + + sqlCommand = new SQLCommand(server) + }) + + it('Should list and search available plugins and themes', async function () { + this.timeout(30000) + + { + const body = await command.listAvailable({ + count: 1, + start: 0, + pluginType: PluginType.THEME, + search: 'background-red' + }) + + expect(body.total).to.be.at.least(1) + expect(body.data).to.have.lengthOf(1) + } + + { + const body1 = await command.listAvailable({ + count: 2, + start: 0, + sort: 'npmName' + }) + expect(body1.total).to.be.at.least(2) + + const data1 = body1.data + expect(data1).to.have.lengthOf(2) + + const body2 = await command.listAvailable({ + count: 2, + start: 0, + sort: '-npmName' + }) + expect(body2.total).to.be.at.least(2) + + const data2 = body2.data + expect(data2).to.have.lengthOf(2) + + expect(data1[0].npmName).to.not.equal(data2[0].npmName) + } + + { + const body = await command.listAvailable({ + count: 10, + start: 0, + pluginType: PluginType.THEME, + search: 'background-red', + currentPeerTubeEngine: '1.0.0' + }) + + const p = body.data.find(p => p.npmName === 'peertube-theme-background-red') + expect(p).to.be.undefined + } + }) + + it('Should install a plugin and a theme', async function () { + this.timeout(30000) + + await command.install({ npmName: 'peertube-plugin-hello-world' }) + await command.install({ npmName: 'peertube-theme-background-red' }) + }) + + it('Should have the plugin loaded in the configuration', async function () { + for (const config of [ await server.config.getConfig(), await server.config.getIndexHTMLConfig() ]) { + const theme = config.theme.registered.find(r => r.name === 'background-red') + expect(theme).to.not.be.undefined + expect(theme.npmName).to.equal('peertube-theme-background-red') + + const plugin = config.plugin.registered.find(r => r.name === 'hello-world') + expect(plugin).to.not.be.undefined + expect(plugin.npmName).to.equal('peertube-plugin-hello-world') + } + }) + + it('Should update the default theme in the configuration', async function () { + await server.config.updateCustomSubConfig({ + newConfig: { + theme: { default: 'background-red' } + } + }) + + for (const config of [ await server.config.getConfig(), await server.config.getIndexHTMLConfig() ]) { + expect(config.theme.default).to.equal('background-red') + } + }) + + it('Should update my default theme', async function () { + await server.users.updateMe({ theme: 'background-red' }) + + const user = await server.users.getMyInfo() + expect(user.theme).to.equal('background-red') + }) + + it('Should list plugins and themes', async function () { + { + const body = await command.list({ + count: 1, + start: 0, + pluginType: PluginType.THEME + }) + expect(body.total).to.be.at.least(1) + + const data = body.data + expect(data).to.have.lengthOf(1) + expect(data[0].name).to.equal('background-red') + } + + { + const { data } = await command.list({ + count: 2, + start: 0, + sort: 'name' + }) + + expect(data[0].name).to.equal('background-red') + expect(data[1].name).to.equal('hello-world') + } + + { + const body = await command.list({ + count: 2, + start: 1, + sort: 'name' + }) + + expect(body.data[0].name).to.equal('hello-world') + } + }) + + it('Should get registered settings', async function () { + await testHelloWorldRegisteredSettings(server) + }) + + it('Should get public settings', async function () { + const body = await command.getPublicSettings({ npmName: 'peertube-plugin-hello-world' }) + const publicSettings = body.publicSettings + + expect(Object.keys(publicSettings)).to.have.lengthOf(1) + expect(Object.keys(publicSettings)).to.deep.equal([ 'user-name' ]) + expect(publicSettings['user-name']).to.be.null + }) + + it('Should update the settings', async function () { + const settings = { + 'admin-name': 'Cid' + } + + await command.updateSettings({ + npmName: 'peertube-plugin-hello-world', + settings + }) + }) + + it('Should have watched settings changes', async function () { + await server.servers.waitUntilLog('Settings changed!') + }) + + it('Should get a plugin and a theme', async function () { + { + const plugin = await command.get({ npmName: 'peertube-plugin-hello-world' }) + + expect(plugin.type).to.equal(PluginType.PLUGIN) + expect(plugin.name).to.equal('hello-world') + expect(plugin.description).to.exist + expect(plugin.homepage).to.exist + expect(plugin.uninstalled).to.be.false + expect(plugin.enabled).to.be.true + expect(plugin.description).to.exist + expect(plugin.version).to.exist + expect(plugin.peertubeEngine).to.exist + expect(plugin.createdAt).to.exist + + expect(plugin.settings).to.not.be.undefined + expect(plugin.settings['admin-name']).to.equal('Cid') + } + + { + const plugin = await command.get({ npmName: 'peertube-theme-background-red' }) + + expect(plugin.type).to.equal(PluginType.THEME) + expect(plugin.name).to.equal('background-red') + expect(plugin.description).to.exist + expect(plugin.homepage).to.exist + expect(plugin.uninstalled).to.be.false + expect(plugin.enabled).to.be.true + expect(plugin.description).to.exist + expect(plugin.version).to.exist + expect(plugin.peertubeEngine).to.exist + expect(plugin.createdAt).to.exist + + expect(plugin.settings).to.be.null + } + }) + + it('Should update the plugin and the theme', async function () { + this.timeout(180000) + + // Wait the scheduler that get the latest plugins versions + await wait(6000) + + async function testUpdate (type: 'plugin' | 'theme', name: string) { + // Fake update our plugin version + await sqlCommand.setPluginVersion(name, '0.0.1') + + // Fake update package.json + const packageJSON = await command.getPackageJSON(`peertube-${type}-${name}`) + const oldVersion = packageJSON.version + + packageJSON.version = '0.0.1' + await command.updatePackageJSON(`peertube-${type}-${name}`, packageJSON) + + // Restart the server to take into account this change + await killallServers([ server ]) + await server.run() + + const checkConfig = async (version: string) => { + for (const config of [ await server.config.getConfig(), await server.config.getIndexHTMLConfig() ]) { + expect(config[type].registered.find(r => r.name === name).version).to.equal(version) + } + } + + const getPluginFromAPI = async () => { + const body = await command.list({ pluginType: type === 'plugin' ? PluginType.PLUGIN : PluginType.THEME }) + + return body.data.find(p => p.name === name) + } + + { + const plugin = await getPluginFromAPI() + expect(plugin.version).to.equal('0.0.1') + expect(plugin.latestVersion).to.exist + expect(plugin.latestVersion).to.not.equal('0.0.1') + + await checkConfig('0.0.1') + } + + { + await command.update({ npmName: `peertube-${type}-${name}` }) + + const plugin = await getPluginFromAPI() + expect(plugin.version).to.equal(oldVersion) + + const updatedPackageJSON = await command.getPackageJSON(`peertube-${type}-${name}`) + expect(updatedPackageJSON.version).to.equal(oldVersion) + + await checkConfig(oldVersion) + } + } + + await testUpdate('theme', 'background-red') + await testUpdate('plugin', 'hello-world') + }) + + it('Should uninstall the plugin', async function () { + await command.uninstall({ npmName: 'peertube-plugin-hello-world' }) + + const body = await command.list({ pluginType: PluginType.PLUGIN }) + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + }) + + it('Should list uninstalled plugins', async function () { + const body = await command.list({ pluginType: PluginType.PLUGIN, uninstalled: true }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + + const plugin = body.data[0] + expect(plugin.name).to.equal('hello-world') + expect(plugin.enabled).to.be.false + expect(plugin.uninstalled).to.be.true + }) + + it('Should uninstall the theme', async function () { + await command.uninstall({ npmName: 'peertube-theme-background-red' }) + }) + + it('Should have updated the configuration', async function () { + for (const config of [ await server.config.getConfig(), await server.config.getIndexHTMLConfig() ]) { + expect(config.theme.default).to.equal('default') + + const theme = config.theme.registered.find(r => r.name === 'background-red') + expect(theme).to.be.undefined + + const plugin = config.plugin.registered.find(r => r.name === 'hello-world') + expect(plugin).to.be.undefined + } + }) + + it('Should have updated the user theme', async function () { + const user = await server.users.getMyInfo() + expect(user.theme).to.equal('instance-default') + }) + + it('Should not install a broken plugin', async function () { + this.timeout(60000) + + async function check () { + const body = await command.list({ pluginType: PluginType.PLUGIN }) + const plugins = body.data + expect(plugins.find(p => p.name === 'test-broken')).to.not.exist + } + + await command.install({ + path: PluginsCommand.getPluginTestPath('-broken'), + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + + await check() + + await killallServers([ server ]) + await server.run() + + await check() + }) + + it('Should rebuild native modules on Node ABI change', async function () { + this.timeout(60000) + + const removeNativeModule = async () => { + await remove(join(baseNativeModule, 'build')) + await remove(join(baseNativeModule, 'prebuilds')) + } + + await command.install({ path: PluginsCommand.getPluginTestPath('-native') }) + + await makeGetRequest({ + url: server.url, + path: '/plugins/test-native/router', + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + + const query = `UPDATE "application" SET "nodeABIVersion" = 1` + await sqlCommand.updateQuery(query) + + const baseNativeModule = server.servers.buildDirectory(join('plugins', 'node_modules', 'a-native-example')) + + await removeNativeModule() + await server.kill() + await server.run() + + await wait(3000) + + expect(await pathExists(join(baseNativeModule, 'build'))).to.be.true + expect(await pathExists(join(baseNativeModule, 'prebuilds'))).to.be.true + + await makeGetRequest({ + url: server.url, + path: '/plugins/test-native/router', + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + + await removeNativeModule() + + await server.kill() + await server.run() + + expect(await pathExists(join(baseNativeModule, 'build'))).to.be.false + expect(await pathExists(join(baseNativeModule, 'prebuilds'))).to.be.false + + await makeGetRequest({ + url: server.url, + path: '/plugins/test-native/router', + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + after(async function () { + await sqlCommand.cleanup() + + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/server/proxy.ts b/packages/tests/src/api/server/proxy.ts new file mode 100644 index 000000000..c7d13f4ab --- /dev/null +++ b/packages/tests/src/api/server/proxy.ts @@ -0,0 +1,173 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { HttpStatusCode, HttpStatusCodeType, VideoPrivacy } from '@peertube/peertube-models' +import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + ObjectStorageCommand, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' +import { FIXTURE_URLS } from '@tests/shared/tests.js' +import { expectStartWith, expectNotStartWith } from '@tests/shared/checks.js' +import { MockProxy } from '@tests/shared/mock-servers/mock-proxy.js' + +describe('Test proxy', function () { + let servers: PeerTubeServer[] = [] + let proxy: MockProxy + + const goodEnv = { HTTP_PROXY: '' } + const badEnv = { HTTP_PROXY: 'http://127.0.0.1:9000' } + + before(async function () { + this.timeout(120000) + + proxy = new MockProxy() + + const proxyPort = await proxy.initialize() + servers = await createMultipleServers(2) + + goodEnv.HTTP_PROXY = 'http://127.0.0.1:' + proxyPort + + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + await doubleFollow(servers[0], servers[1]) + }) + + describe('Federation', function () { + + it('Should succeed federation with the appropriate proxy config', async function () { + this.timeout(40000) + + await servers[0].kill() + await servers[0].run({}, { env: goodEnv }) + + await servers[0].videos.quickUpload({ name: 'video 1' }) + + await waitJobs(servers) + + for (const server of servers) { + const { total, data } = await server.videos.list() + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + } + }) + + it('Should fail federation with a wrong proxy config', async function () { + this.timeout(40000) + + await servers[0].kill() + await servers[0].run({}, { env: badEnv }) + + await servers[0].videos.quickUpload({ name: 'video 2' }) + + await waitJobs(servers) + + { + const { total, data } = await servers[0].videos.list() + expect(total).to.equal(2) + expect(data).to.have.lengthOf(2) + } + + { + const { total, data } = await servers[1].videos.list() + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + } + }) + }) + + describe('Videos import', async function () { + + function quickImport (expectedStatus: HttpStatusCodeType = HttpStatusCode.OK_200) { + return servers[0].imports.importVideo({ + attributes: { + name: 'video import', + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC, + targetUrl: FIXTURE_URLS.peertube_long + }, + expectedStatus + }) + } + + it('Should succeed import with the appropriate proxy config', async function () { + this.timeout(240000) + + await servers[0].kill() + await servers[0].run({}, { env: goodEnv }) + + await quickImport() + + await waitJobs(servers) + + const { total, data } = await servers[0].videos.list() + expect(total).to.equal(3) + expect(data).to.have.lengthOf(3) + }) + + it('Should fail import with a wrong proxy config', async function () { + this.timeout(120000) + + await servers[0].kill() + await servers[0].run({}, { env: badEnv }) + + await quickImport(HttpStatusCode.BAD_REQUEST_400) + }) + }) + + describe('Object storage', function () { + if (areMockObjectStorageTestsDisabled()) return + + const objectStorage = new ObjectStorageCommand() + + before(async function () { + this.timeout(30000) + + await objectStorage.prepareDefaultMockBuckets() + }) + + it('Should succeed to upload to object storage with the appropriate proxy config', async function () { + this.timeout(120000) + + await servers[0].kill() + await servers[0].run(objectStorage.getDefaultMockConfig(), { env: goodEnv }) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) + await waitJobs(servers) + + const video = await servers[0].videos.get({ id: uuid }) + + expectStartWith(video.files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl()) + }) + + it('Should fail to upload to object storage with a wrong proxy config', async function () { + this.timeout(120000) + + await servers[0].kill() + await servers[0].run(objectStorage.getDefaultMockConfig(), { env: badEnv }) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) + await waitJobs(servers, { skipDelayed: true }) + + const video = await servers[0].videos.get({ id: uuid }) + + expectNotStartWith(video.files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl()) + }) + + after(async function () { + await objectStorage.cleanupMock() + }) + }) + + after(async function () { + await proxy.terminate() + + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/server/reverse-proxy.ts b/packages/tests/src/api/server/reverse-proxy.ts new file mode 100644 index 000000000..7e334cc3e --- /dev/null +++ b/packages/tests/src/api/server/reverse-proxy.ts @@ -0,0 +1,156 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { HttpStatusCode } from '@peertube/peertube-models' +import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@peertube/peertube-server-commands' + +describe('Test application behind a reverse proxy', function () { + let server: PeerTubeServer + let userAccessToken: string + let videoId: string + + before(async function () { + this.timeout(60000) + + const config = { + rates_limit: { + api: { + max: 50, + window: 5000 + }, + signup: { + max: 3, + window: 5000 + }, + login: { + max: 20 + } + }, + signup: { + limit: 20 + } + } + + server = await createSingleServer(1, config) + await setAccessTokensToServers([ server ]) + + userAccessToken = await server.users.generateUserAndToken('user') + + const { uuid } = await server.videos.upload() + videoId = uuid + }) + + it('Should view a video only once with the same IP by default', async function () { + this.timeout(40000) + + await server.views.simulateView({ id: videoId }) + await server.views.simulateView({ id: videoId }) + + // Wait the repeatable job + await wait(8000) + + const video = await server.videos.get({ id: videoId }) + expect(video.views).to.equal(1) + }) + + it('Should view a video 2 times with the X-Forwarded-For header set', async function () { + this.timeout(20000) + + await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.1,127.0.0.1' }) + await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.2,127.0.0.1' }) + + // Wait the repeatable job + await wait(8000) + + const video = await server.videos.get({ id: videoId }) + expect(video.views).to.equal(3) + }) + + it('Should view a video only once with the same client IP in the X-Forwarded-For header', async function () { + this.timeout(20000) + + await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.4,0.0.0.3,::ffff:127.0.0.1' }) + await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.5,0.0.0.3,127.0.0.1' }) + + // Wait the repeatable job + await wait(8000) + + const video = await server.videos.get({ id: videoId }) + expect(video.views).to.equal(4) + }) + + it('Should view a video two times with a different client IP in the X-Forwarded-For header', async function () { + this.timeout(20000) + + await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.8,0.0.0.6,127.0.0.1' }) + await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.8,0.0.0.7,127.0.0.1' }) + + // Wait the repeatable job + await wait(8000) + + const video = await server.videos.get({ id: videoId }) + expect(video.views).to.equal(6) + }) + + it('Should rate limit logins', async function () { + const user = { username: 'root', password: 'fail' } + + for (let i = 0; i < 18; i++) { + await server.login.login({ user, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + } + + await server.login.login({ user, expectedStatus: HttpStatusCode.TOO_MANY_REQUESTS_429 }) + }) + + it('Should rate limit signup', async function () { + for (let i = 0; i < 10; i++) { + try { + await server.registrations.register({ username: 'test' + i }) + } catch { + // empty + } + } + + await server.registrations.register({ username: 'test42', expectedStatus: HttpStatusCode.TOO_MANY_REQUESTS_429 }) + }) + + it('Should not rate limit failed signup', async function () { + this.timeout(30000) + + await wait(7000) + + for (let i = 0; i < 3; i++) { + await server.registrations.register({ username: 'test' + i, expectedStatus: HttpStatusCode.CONFLICT_409 }) + } + + await server.registrations.register({ username: 'test43', expectedStatus: HttpStatusCode.NO_CONTENT_204 }) + + }) + + it('Should rate limit API calls', async function () { + this.timeout(30000) + + await wait(7000) + + for (let i = 0; i < 100; i++) { + try { + await server.videos.get({ id: videoId }) + } catch { + // don't care if it fails + } + } + + await server.videos.get({ id: videoId, expectedStatus: HttpStatusCode.TOO_MANY_REQUESTS_429 }) + }) + + it('Should rate limit API calls with a user but not with an admin', async function () { + await server.videos.get({ id: videoId, token: userAccessToken, expectedStatus: HttpStatusCode.TOO_MANY_REQUESTS_429 }) + + await server.videos.get({ id: videoId, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/server/services.ts b/packages/tests/src/api/server/services.ts new file mode 100644 index 000000000..349d29a58 --- /dev/null +++ b/packages/tests/src/api/server/services.ts @@ -0,0 +1,143 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { Video, VideoPlaylistPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel +} from '@peertube/peertube-server-commands' + +describe('Test services', function () { + let server: PeerTubeServer = null + let playlistUUID: string + let playlistDisplayName: string + let video: Video + + const urlSuffixes = [ + { + input: '', + output: '' + }, + { + input: '?param=1', + output: '' + }, + { + input: '?muted=1&warningTitle=0&toto=1', + output: '?muted=1&warningTitle=0' + } + ] + + before(async function () { + this.timeout(120000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + { + const attributes = { name: 'my super name' } + await server.videos.upload({ attributes }) + + const { data } = await server.videos.list() + video = data[0] + } + + { + const created = await server.playlists.create({ + attributes: { + displayName: 'The Life and Times of Scrooge McDuck', + privacy: VideoPlaylistPrivacy.PUBLIC, + videoChannelId: server.store.channel.id + } + }) + + playlistUUID = created.uuid + playlistDisplayName = 'The Life and Times of Scrooge McDuck' + + await server.playlists.addElement({ + playlistId: created.id, + attributes: { + videoId: video.id + } + }) + } + }) + + it('Should have a valid oEmbed video response', async function () { + for (const basePath of [ '/videos/watch/', '/w/' ]) { + for (const suffix of urlSuffixes) { + const oembedUrl = server.url + basePath + video.uuid + suffix.input + + const res = await server.services.getOEmbed({ oembedUrl }) + const expectedHtml = '' + + const expectedThumbnailUrl = 'http://' + server.host + video.previewPath + + expect(res.body.html).to.equal(expectedHtml) + expect(res.body.title).to.equal(video.name) + expect(res.body.author_name).to.equal(server.store.channel.displayName) + expect(res.body.width).to.equal(560) + expect(res.body.height).to.equal(315) + expect(res.body.thumbnail_url).to.equal(expectedThumbnailUrl) + expect(res.body.thumbnail_width).to.equal(850) + expect(res.body.thumbnail_height).to.equal(480) + } + } + }) + + it('Should have a valid playlist oEmbed response', async function () { + for (const basePath of [ '/videos/watch/playlist/', '/w/p/' ]) { + for (const suffix of urlSuffixes) { + const oembedUrl = server.url + basePath + playlistUUID + suffix.input + + const res = await server.services.getOEmbed({ oembedUrl }) + const expectedHtml = '' + + expect(res.body.html).to.equal(expectedHtml) + expect(res.body.title).to.equal('The Life and Times of Scrooge McDuck') + expect(res.body.author_name).to.equal(server.store.channel.displayName) + expect(res.body.width).to.equal(560) + expect(res.body.height).to.equal(315) + expect(res.body.thumbnail_url).exist + expect(res.body.thumbnail_width).to.equal(280) + expect(res.body.thumbnail_height).to.equal(157) + } + } + }) + + it('Should have a valid oEmbed response with small max height query', async function () { + for (const basePath of [ '/videos/watch/', '/w/' ]) { + const oembedUrl = 'http://' + server.host + basePath + video.uuid + const format = 'json' + const maxHeight = 50 + const maxWidth = 50 + + const res = await server.services.getOEmbed({ oembedUrl, format, maxHeight, maxWidth }) + const expectedHtml = '' + + expect(res.body.html).to.equal(expectedHtml) + expect(res.body.title).to.equal(video.name) + expect(res.body.author_name).to.equal(server.store.channel.displayName) + expect(res.body.height).to.equal(50) + expect(res.body.width).to.equal(50) + expect(res.body).to.not.have.property('thumbnail_url') + expect(res.body).to.not.have.property('thumbnail_width') + expect(res.body).to.not.have.property('thumbnail_height') + } + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/server/slow-follows.ts b/packages/tests/src/api/server/slow-follows.ts new file mode 100644 index 000000000..d03109001 --- /dev/null +++ b/packages/tests/src/api/server/slow-follows.ts @@ -0,0 +1,85 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { Job } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test slow follows', function () { + let servers: PeerTubeServer[] = [] + + let afterFollows: Date + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(3) + + // Get the access tokens + await setAccessTokensToServers(servers) + + await doubleFollow(servers[0], servers[1]) + await doubleFollow(servers[0], servers[2]) + + afterFollows = new Date() + + for (let i = 0; i < 5; i++) { + await servers[0].videos.quickUpload({ name: 'video ' + i }) + } + + await waitJobs(servers) + }) + + it('Should only have broadcast jobs', async function () { + const { data } = await servers[0].jobs.list({ jobType: 'activitypub-http-unicast', sort: '-createdAt' }) + + for (const job of data) { + expect(new Date(job.createdAt)).below(afterFollows) + } + }) + + it('Should process bad follower', async function () { + this.timeout(30000) + + await servers[1].kill() + + // Set server 2 as bad follower + await servers[0].videos.quickUpload({ name: 'video 6' }) + await waitJobs(servers[0]) + + afterFollows = new Date() + const filter = (job: Job) => new Date(job.createdAt) > afterFollows + + // Resend another broadcast job + await servers[0].videos.quickUpload({ name: 'video 7' }) + await waitJobs(servers[0]) + + const resBroadcast = await servers[0].jobs.list({ jobType: 'activitypub-http-broadcast', sort: '-createdAt' }) + const resUnicast = await servers[0].jobs.list({ jobType: 'activitypub-http-unicast', sort: '-createdAt' }) + + const broadcast = resBroadcast.data.filter(filter) + const unicast = resUnicast.data.filter(filter) + + expect(unicast).to.have.lengthOf(2) + expect(broadcast).to.have.lengthOf(2) + + for (const u of unicast) { + expect(u.data.uri).to.equal(servers[1].url + '/inbox') + } + + for (const b of broadcast) { + expect(b.data.uris).to.have.lengthOf(1) + expect(b.data.uris[0]).to.equal(servers[2].url + '/inbox') + } + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/server/stats.ts b/packages/tests/src/api/server/stats.ts new file mode 100644 index 000000000..32ab323ce --- /dev/null +++ b/packages/tests/src/api/server/stats.ts @@ -0,0 +1,279 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { ActivityType, VideoPlaylistPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultChannelAvatar, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test stats (excluding redundancy)', function () { + let servers: PeerTubeServer[] = [] + let channelId + const user = { + username: 'user1', + password: 'super_password' + } + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(3) + + await setAccessTokensToServers(servers) + await setDefaultChannelAvatar(servers) + await setDefaultAccountAvatar(servers) + + await doubleFollow(servers[0], servers[1]) + + await servers[0].users.create({ username: user.username, password: user.password }) + + const { uuid } = await servers[0].videos.upload({ attributes: { fixture: 'video_short.webm' } }) + + await servers[0].comments.createThread({ videoId: uuid, text: 'comment' }) + + await servers[0].views.simulateView({ id: uuid }) + + // Wait the video views repeatable job + await wait(8000) + + await servers[2].follows.follow({ hosts: [ servers[0].url ] }) + await waitJobs(servers) + }) + + it('Should have the correct stats on instance 1', async function () { + const data = await servers[0].stats.get() + + expect(data.totalLocalVideoComments).to.equal(1) + expect(data.totalLocalVideos).to.equal(1) + expect(data.totalLocalVideoViews).to.equal(1) + expect(data.totalLocalVideoFilesSize).to.equal(218910) + expect(data.totalUsers).to.equal(2) + expect(data.totalVideoComments).to.equal(1) + expect(data.totalVideos).to.equal(1) + expect(data.totalInstanceFollowers).to.equal(2) + expect(data.totalInstanceFollowing).to.equal(1) + expect(data.totalLocalPlaylists).to.equal(0) + }) + + it('Should have the correct stats on instance 2', async function () { + const data = await servers[1].stats.get() + + expect(data.totalLocalVideoComments).to.equal(0) + expect(data.totalLocalVideos).to.equal(0) + expect(data.totalLocalVideoViews).to.equal(0) + expect(data.totalLocalVideoFilesSize).to.equal(0) + expect(data.totalUsers).to.equal(1) + expect(data.totalVideoComments).to.equal(1) + expect(data.totalVideos).to.equal(1) + expect(data.totalInstanceFollowers).to.equal(1) + expect(data.totalInstanceFollowing).to.equal(1) + expect(data.totalLocalPlaylists).to.equal(0) + }) + + it('Should have the correct stats on instance 3', async function () { + const data = await servers[2].stats.get() + + expect(data.totalLocalVideoComments).to.equal(0) + expect(data.totalLocalVideos).to.equal(0) + expect(data.totalLocalVideoViews).to.equal(0) + expect(data.totalUsers).to.equal(1) + expect(data.totalVideoComments).to.equal(1) + expect(data.totalVideos).to.equal(1) + expect(data.totalInstanceFollowing).to.equal(1) + expect(data.totalInstanceFollowers).to.equal(0) + expect(data.totalLocalPlaylists).to.equal(0) + }) + + it('Should have the correct total videos stats after an unfollow', async function () { + this.timeout(15000) + + await servers[2].follows.unfollow({ target: servers[0] }) + await waitJobs(servers) + + const data = await servers[2].stats.get() + + expect(data.totalVideos).to.equal(0) + }) + + it('Should have the correct active user stats', async function () { + const server = servers[0] + + { + const data = await server.stats.get() + + expect(data.totalDailyActiveUsers).to.equal(1) + expect(data.totalWeeklyActiveUsers).to.equal(1) + expect(data.totalMonthlyActiveUsers).to.equal(1) + } + + { + await server.login.getAccessToken(user) + + const data = await server.stats.get() + + expect(data.totalDailyActiveUsers).to.equal(2) + expect(data.totalWeeklyActiveUsers).to.equal(2) + expect(data.totalMonthlyActiveUsers).to.equal(2) + } + }) + + it('Should have the correct active channel stats', async function () { + const server = servers[0] + + { + const data = await server.stats.get() + + expect(data.totalLocalVideoChannels).to.equal(2) + expect(data.totalLocalDailyActiveVideoChannels).to.equal(1) + expect(data.totalLocalWeeklyActiveVideoChannels).to.equal(1) + expect(data.totalLocalMonthlyActiveVideoChannels).to.equal(1) + } + + { + const attributes = { + name: 'stats_channel', + displayName: 'My stats channel' + } + const created = await server.channels.create({ attributes }) + channelId = created.id + + const data = await server.stats.get() + + expect(data.totalLocalVideoChannels).to.equal(3) + expect(data.totalLocalDailyActiveVideoChannels).to.equal(1) + expect(data.totalLocalWeeklyActiveVideoChannels).to.equal(1) + expect(data.totalLocalMonthlyActiveVideoChannels).to.equal(1) + } + + { + await server.videos.upload({ attributes: { fixture: 'video_short.webm', channelId } }) + + const data = await server.stats.get() + + expect(data.totalLocalVideoChannels).to.equal(3) + expect(data.totalLocalDailyActiveVideoChannels).to.equal(2) + expect(data.totalLocalWeeklyActiveVideoChannels).to.equal(2) + expect(data.totalLocalMonthlyActiveVideoChannels).to.equal(2) + } + }) + + it('Should have the correct playlist stats', async function () { + const server = servers[0] + + { + const data = await server.stats.get() + expect(data.totalLocalPlaylists).to.equal(0) + } + + { + await server.playlists.create({ + attributes: { + displayName: 'playlist for count', + privacy: VideoPlaylistPrivacy.PUBLIC, + videoChannelId: channelId + } + }) + + const data = await server.stats.get() + expect(data.totalLocalPlaylists).to.equal(1) + } + }) + + it('Should correctly count video file sizes if transcoding is enabled', async function () { + this.timeout(120000) + + await servers[0].config.updateCustomSubConfig({ + newConfig: { + transcoding: { + enabled: true, + webVideos: { + enabled: true + }, + hls: { + enabled: true + }, + resolutions: { + '0p': false, + '144p': false, + '240p': false, + '360p': false, + '480p': false, + '720p': false, + '1080p': false, + '1440p': false, + '2160p': false + } + } + } + }) + + await servers[0].videos.upload({ attributes: { name: 'video', fixture: 'video_short.webm' } }) + + await waitJobs(servers) + + { + const data = await servers[1].stats.get() + expect(data.totalLocalVideoFilesSize).to.equal(0) + } + + { + const data = await servers[0].stats.get() + expect(data.totalLocalVideoFilesSize).to.be.greaterThan(500000) + expect(data.totalLocalVideoFilesSize).to.be.lessThan(600000) + } + }) + + it('Should have the correct AP stats', async function () { + this.timeout(120000) + + await servers[0].config.disableTranscoding() + + const first = await servers[1].stats.get() + + for (let i = 0; i < 10; i++) { + await servers[0].videos.upload({ attributes: { name: 'video' } }) + } + + await waitJobs(servers) + + await wait(6000) + + const second = await servers[1].stats.get() + expect(second.totalActivityPubMessagesProcessed).to.be.greaterThan(first.totalActivityPubMessagesProcessed) + + const apTypes: ActivityType[] = [ + 'Create', 'Update', 'Delete', 'Follow', 'Accept', 'Announce', 'Undo', 'Like', 'Reject', 'View', 'Dislike', 'Flag' + ] + + const processed = apTypes.reduce( + (previous, type) => previous + second['totalActivityPub' + type + 'MessagesSuccesses'], + 0 + ) + expect(second.totalActivityPubMessagesProcessed).to.equal(processed) + expect(second.totalActivityPubMessagesSuccesses).to.equal(processed) + + expect(second.totalActivityPubMessagesErrors).to.equal(0) + + for (const apType of apTypes) { + expect(second['totalActivityPub' + apType + 'MessagesErrors']).to.equal(0) + } + + await wait(6000) + + const third = await servers[1].stats.get() + expect(third.totalActivityPubMessagesWaiting).to.equal(0) + expect(third.activityPubMessagesProcessedPerSecond).to.be.lessThan(second.activityPubMessagesProcessedPerSecond) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/server/tracker.ts b/packages/tests/src/api/server/tracker.ts new file mode 100644 index 000000000..4df4e4613 --- /dev/null +++ b/packages/tests/src/api/server/tracker.ts @@ -0,0 +1,110 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await,@typescript-eslint/no-floating-promises */ + +import { decode as magnetUriDecode, encode as magnetUriEncode } from 'magnet-uri' +import WebTorrent from 'webtorrent' +import { + cleanupTests, + createSingleServer, + killallServers, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test tracker', function () { + let server: PeerTubeServer + let badMagnet: string + let goodMagnet: string + + before(async function () { + this.timeout(60000) + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + + { + const { uuid } = await server.videos.upload() + const video = await server.videos.get({ id: uuid }) + goodMagnet = video.files[0].magnetUri + + const parsed = magnetUriDecode(goodMagnet) + parsed.infoHash = '010597bb88b1968a5693a4fa8267c592ca65f2e9' + + badMagnet = magnetUriEncode(parsed) + } + }) + + it('Should succeed with the correct infohash', function (done) { + const webtorrent = new WebTorrent() + + const torrent = webtorrent.add(goodMagnet) + + torrent.on('error', done) + torrent.on('warning', warn => { + const message = typeof warn === 'string' ? warn : warn.message + if (message.includes('Unknown infoHash ')) return done(new Error('Error on infohash')) + }) + + torrent.on('done', done) + }) + + it('Should disable the tracker', function (done) { + this.timeout(20000) + + const errCb = () => done(new Error('Tracker is enabled')) + + killallServers([ server ]) + .then(() => server.run({ tracker: { enabled: false } })) + .then(() => { + const webtorrent = new WebTorrent() + + const torrent = webtorrent.add(goodMagnet) + + torrent.on('error', done) + torrent.on('warning', warn => { + const message = typeof warn === 'string' ? warn : warn.message + if (message.includes('disabled ')) { + torrent.off('done', errCb) + + return done() + } + }) + + torrent.on('done', errCb) + }) + }) + + it('Should return an error when adding an incorrect infohash', function (done) { + this.timeout(20000) + + killallServers([ server ]) + .then(() => server.run()) + .then(() => { + const webtorrent = new WebTorrent() + + const torrent = webtorrent.add(badMagnet) + + torrent.on('error', done) + torrent.on('warning', warn => { + const message = typeof warn === 'string' ? warn : warn.message + if (message.includes('Unknown infoHash ')) return done() + }) + + torrent.on('done', () => done(new Error('No error on infohash'))) + }) + }) + + it('Should block the IP after the failed infohash', function (done) { + const webtorrent = new WebTorrent() + + const torrent = webtorrent.add(goodMagnet) + + torrent.on('error', done) + torrent.on('warning', warn => { + const message = typeof warn === 'string' ? warn : warn.message + if (message.includes('Unsupported tracker protocol')) return done() + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/transcoding/audio-only.ts b/packages/tests/src/api/transcoding/audio-only.ts new file mode 100644 index 000000000..6d0410348 --- /dev/null +++ b/packages/tests/src/api/transcoding/audio-only.ts @@ -0,0 +1,104 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { getAudioStream, getVideoStreamDimensionsInfo } from '@peertube/peertube-ffmpeg' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test audio only video transcoding', function () { + let servers: PeerTubeServer[] = [] + let videoUUID: string + let webVideoAudioFileUrl: string + let fragmentedAudioFileUrl: string + + before(async function () { + this.timeout(120000) + + const configOverride = { + transcoding: { + enabled: true, + resolutions: { + '0p': true, + '144p': false, + '240p': true, + '360p': false, + '480p': false, + '720p': false, + '1080p': false, + '1440p': false, + '2160p': false + }, + hls: { + enabled: true + }, + web_videos: { + enabled: true + } + } + } + servers = await createMultipleServers(2, configOverride) + + // Get the access tokens + await setAccessTokensToServers(servers) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + }) + + it('Should upload a video and transcode it', async function () { + this.timeout(120000) + + const { uuid } = await servers[0].videos.upload({ attributes: { name: 'audio only' } }) + videoUUID = uuid + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + expect(video.streamingPlaylists).to.have.lengthOf(1) + + for (const files of [ video.files, video.streamingPlaylists[0].files ]) { + expect(files).to.have.lengthOf(3) + expect(files[0].resolution.id).to.equal(720) + expect(files[1].resolution.id).to.equal(240) + expect(files[2].resolution.id).to.equal(0) + } + + if (server.serverNumber === 1) { + webVideoAudioFileUrl = video.files[2].fileUrl + fragmentedAudioFileUrl = video.streamingPlaylists[0].files[2].fileUrl + } + } + }) + + it('0p transcoded video should not have video', async function () { + const paths = [ + servers[0].servers.buildWebVideoFilePath(webVideoAudioFileUrl), + servers[0].servers.buildFragmentedFilePath(videoUUID, fragmentedAudioFileUrl) + ] + + for (const path of paths) { + const { audioStream } = await getAudioStream(path) + expect(audioStream['codec_name']).to.be.equal('aac') + expect(audioStream['bit_rate']).to.be.at.most(384 * 8000) + + const size = await getVideoStreamDimensionsInfo(path) + + expect(size.height).to.equal(0) + expect(size.width).to.equal(0) + expect(size.isPortraitMode).to.be.false + expect(size.ratio).to.equal(0) + expect(size.resolution).to.equal(0) + } + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/transcoding/create-transcoding.ts b/packages/tests/src/api/transcoding/create-transcoding.ts new file mode 100644 index 000000000..b0a9c7556 --- /dev/null +++ b/packages/tests/src/api/transcoding/create-transcoding.ts @@ -0,0 +1,267 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' +import { HttpStatusCode, VideoDetails } from '@peertube/peertube-models' +import { + cleanupTests, + ConfigCommand, + createMultipleServers, + doubleFollow, + expectNoFailedTranscodingJob, + makeRawRequest, + ObjectStorageCommand, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' +import { expectStartWith } from '@tests/shared/checks.js' +import { checkResolutionsInMasterPlaylist } from '@tests/shared/streaming-playlists.js' + +async function checkFilesInObjectStorage (objectStorage: ObjectStorageCommand, video: VideoDetails) { + for (const file of video.files) { + expectStartWith(file.fileUrl, objectStorage.getMockWebVideosBaseUrl()) + await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + } + + if (video.streamingPlaylists.length === 0) return + + const hlsPlaylist = video.streamingPlaylists[0] + for (const file of hlsPlaylist.files) { + expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl()) + await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + } + + expectStartWith(hlsPlaylist.playlistUrl, objectStorage.getMockPlaylistBaseUrl()) + await makeRawRequest({ url: hlsPlaylist.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) + + expectStartWith(hlsPlaylist.segmentsSha256Url, objectStorage.getMockPlaylistBaseUrl()) + await makeRawRequest({ url: hlsPlaylist.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) +} + +function runTests (enableObjectStorage: boolean) { + let servers: PeerTubeServer[] = [] + let videoUUID: string + let publishedAt: string + + let shouldBeDeleted: string[] + const objectStorage = new ObjectStorageCommand() + + before(async function () { + this.timeout(120000) + + const config = enableObjectStorage + ? objectStorage.getDefaultMockConfig() + : {} + + // Run server 2 to have transcoding enabled + servers = await createMultipleServers(2, config) + await setAccessTokensToServers(servers) + + await servers[0].config.disableTranscoding() + + await doubleFollow(servers[0], servers[1]) + + if (enableObjectStorage) await objectStorage.prepareDefaultMockBuckets() + + const { shortUUID } = await servers[0].videos.quickUpload({ name: 'video' }) + videoUUID = shortUUID + + await waitJobs(servers) + + const video = await servers[0].videos.get({ id: videoUUID }) + publishedAt = video.publishedAt as string + + await servers[0].config.enableTranscoding() + }) + + it('Should generate HLS', async function () { + this.timeout(60000) + + await servers[0].videos.runTranscoding({ + videoId: videoUUID, + transcodingType: 'hls' + }) + + await waitJobs(servers) + await expectNoFailedTranscodingJob(servers[0]) + + for (const server of servers) { + const videoDetails = await server.videos.get({ id: videoUUID }) + + expect(videoDetails.files).to.have.lengthOf(1) + expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) + expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5) + + if (enableObjectStorage) await checkFilesInObjectStorage(objectStorage, videoDetails) + } + }) + + it('Should generate Web Video', async function () { + this.timeout(60000) + + await servers[0].videos.runTranscoding({ + videoId: videoUUID, + transcodingType: 'web-video' + }) + + await waitJobs(servers) + + for (const server of servers) { + const videoDetails = await server.videos.get({ id: videoUUID }) + + expect(videoDetails.files).to.have.lengthOf(5) + expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) + expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5) + + if (enableObjectStorage) await checkFilesInObjectStorage(objectStorage, videoDetails) + } + }) + + it('Should generate Web Video from HLS only video', async function () { + this.timeout(60000) + + await servers[0].videos.removeAllWebVideoFiles({ videoId: videoUUID }) + await waitJobs(servers) + + await servers[0].videos.runTranscoding({ videoId: videoUUID, transcodingType: 'web-video' }) + await waitJobs(servers) + + for (const server of servers) { + const videoDetails = await server.videos.get({ id: videoUUID }) + + expect(videoDetails.files).to.have.lengthOf(5) + expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) + expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5) + + if (enableObjectStorage) await checkFilesInObjectStorage(objectStorage, videoDetails) + } + }) + + it('Should only generate Web Video', async function () { + this.timeout(60000) + + await servers[0].videos.removeHLSPlaylist({ videoId: videoUUID }) + await waitJobs(servers) + + await servers[0].videos.runTranscoding({ videoId: videoUUID, transcodingType: 'web-video' }) + await waitJobs(servers) + + for (const server of servers) { + const videoDetails = await server.videos.get({ id: videoUUID }) + + expect(videoDetails.files).to.have.lengthOf(5) + expect(videoDetails.streamingPlaylists).to.have.lengthOf(0) + + if (enableObjectStorage) await checkFilesInObjectStorage(objectStorage, videoDetails) + } + }) + + it('Should correctly update HLS playlist on resolution change', async function () { + this.timeout(120000) + + await servers[0].config.updateExistingSubConfig({ + newConfig: { + transcoding: { + enabled: true, + resolutions: ConfigCommand.getCustomConfigResolutions(false), + + webVideos: { + enabled: true + }, + hls: { + enabled: true + } + } + } + }) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'quick' }) + + await waitJobs(servers) + + for (const server of servers) { + const videoDetails = await server.videos.get({ id: uuid }) + + expect(videoDetails.files).to.have.lengthOf(1) + expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) + expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(1) + + if (enableObjectStorage) await checkFilesInObjectStorage(objectStorage, videoDetails) + + shouldBeDeleted = [ + videoDetails.streamingPlaylists[0].files[0].fileUrl, + videoDetails.streamingPlaylists[0].playlistUrl, + videoDetails.streamingPlaylists[0].segmentsSha256Url + ] + } + + await servers[0].config.updateExistingSubConfig({ + newConfig: { + transcoding: { + enabled: true, + resolutions: ConfigCommand.getCustomConfigResolutions(true), + + webVideos: { + enabled: true + }, + hls: { + enabled: true + } + } + } + }) + + await servers[0].videos.runTranscoding({ videoId: uuid, transcodingType: 'hls' }) + await waitJobs(servers) + + for (const server of servers) { + const videoDetails = await server.videos.get({ id: uuid }) + + expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) + expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5) + + if (enableObjectStorage) { + await checkFilesInObjectStorage(objectStorage, videoDetails) + + const hlsPlaylist = videoDetails.streamingPlaylists[0] + const resolutions = hlsPlaylist.files.map(f => f.resolution.id) + await checkResolutionsInMasterPlaylist({ server: servers[0], playlistUrl: hlsPlaylist.playlistUrl, resolutions }) + + const shaBody = await servers[0].streamingPlaylists.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url, withRetry: true }) + expect(Object.keys(shaBody)).to.have.lengthOf(5) + } + } + }) + + it('Should have correctly deleted previous files', async function () { + for (const fileUrl of shouldBeDeleted) { + await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + } + }) + + it('Should not have updated published at attributes', async function () { + const video = await servers[0].videos.get({ id: videoUUID }) + + expect(video.publishedAt).to.equal(publishedAt) + }) + + after(async function () { + if (objectStorage) await objectStorage.cleanupMock() + + await cleanupTests(servers) + }) +} + +describe('Test create transcoding jobs from API', function () { + + describe('On filesystem', function () { + runTests(false) + }) + + describe('On object storage', function () { + if (areMockObjectStorageTestsDisabled()) return + + runTests(true) + }) +}) diff --git a/packages/tests/src/api/transcoding/hls.ts b/packages/tests/src/api/transcoding/hls.ts new file mode 100644 index 000000000..884f98e87 --- /dev/null +++ b/packages/tests/src/api/transcoding/hls.ts @@ -0,0 +1,176 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { join } from 'path' +import { HttpStatusCode } from '@peertube/peertube-models' +import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + ObjectStorageCommand, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' +import { DEFAULT_AUDIO_RESOLUTION } from '@peertube/peertube-server/server/initializers/constants.js' +import { checkDirectoryIsEmpty, checkTmpIsEmpty } from '@tests/shared/directories.js' +import { completeCheckHlsPlaylist } from '@tests/shared/streaming-playlists.js' + +describe('Test HLS videos', function () { + let servers: PeerTubeServer[] = [] + + function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) { + const videoUUIDs: string[] = [] + + it('Should upload a video and transcode it to HLS', async function () { + this.timeout(120000) + + const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video 1', fixture: 'video_short.webm' } }) + videoUUIDs.push(uuid) + + await waitJobs(servers) + + await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl }) + }) + + it('Should upload an audio file and transcode it to HLS', async function () { + this.timeout(120000) + + const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video audio', fixture: 'sample.ogg' } }) + videoUUIDs.push(uuid) + + await waitJobs(servers) + + await completeCheckHlsPlaylist({ + servers, + videoUUID: uuid, + hlsOnly, + resolutions: [ DEFAULT_AUDIO_RESOLUTION, 360, 240 ], + objectStorageBaseUrl + }) + }) + + it('Should update the video', async function () { + this.timeout(30000) + + await servers[0].videos.update({ id: videoUUIDs[0], attributes: { name: 'video 1 updated' } }) + + await waitJobs(servers) + + await completeCheckHlsPlaylist({ servers, videoUUID: videoUUIDs[0], hlsOnly, objectStorageBaseUrl }) + }) + + it('Should delete videos', async function () { + for (const uuid of videoUUIDs) { + await servers[0].videos.remove({ id: uuid }) + } + + await waitJobs(servers) + + for (const server of servers) { + for (const uuid of videoUUIDs) { + await server.videos.get({ id: uuid, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + } + } + }) + + it('Should have the playlists/segment deleted from the disk', async function () { + for (const server of servers) { + await checkDirectoryIsEmpty(server, 'web-videos', [ 'private' ]) + await checkDirectoryIsEmpty(server, join('web-videos', 'private')) + + await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls'), [ 'private' ]) + await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls', 'private')) + } + }) + + it('Should have an empty tmp directory', async function () { + for (const server of servers) { + await checkTmpIsEmpty(server) + } + }) + } + + before(async function () { + this.timeout(120000) + + const configOverride = { + transcoding: { + enabled: true, + allow_audio_files: true, + hls: { + enabled: true + } + } + } + servers = await createMultipleServers(2, configOverride) + + // Get the access tokens + await setAccessTokensToServers(servers) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + }) + + describe('With Web Video & HLS enabled', function () { + runTestSuite(false) + }) + + describe('With only HLS enabled', function () { + + before(async function () { + await servers[0].config.updateCustomSubConfig({ + newConfig: { + transcoding: { + enabled: true, + allowAudioFiles: true, + resolutions: { + '144p': false, + '240p': true, + '360p': true, + '480p': true, + '720p': true, + '1080p': true, + '1440p': true, + '2160p': true + }, + hls: { + enabled: true + }, + webVideos: { + enabled: false + } + } + } + }) + }) + + runTestSuite(true) + }) + + describe('With object storage enabled', function () { + if (areMockObjectStorageTestsDisabled()) return + + const objectStorage = new ObjectStorageCommand() + + before(async function () { + this.timeout(120000) + + const configOverride = objectStorage.getDefaultMockConfig() + await objectStorage.prepareDefaultMockBuckets() + + await servers[0].kill() + await servers[0].run(configOverride) + }) + + runTestSuite(true, objectStorage.getMockPlaylistBaseUrl()) + + after(async function () { + await objectStorage.cleanupMock() + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/transcoding/index.ts b/packages/tests/src/api/transcoding/index.ts new file mode 100644 index 000000000..c25cd51c3 --- /dev/null +++ b/packages/tests/src/api/transcoding/index.ts @@ -0,0 +1,6 @@ +export * from './audio-only.js' +export * from './create-transcoding.js' +export * from './hls.js' +export * from './transcoder.js' +export * from './update-while-transcoding.js' +export * from './video-studio.js' diff --git a/packages/tests/src/api/transcoding/transcoder.ts b/packages/tests/src/api/transcoding/transcoder.ts new file mode 100644 index 000000000..8900491f5 --- /dev/null +++ b/packages/tests/src/api/transcoding/transcoder.ts @@ -0,0 +1,802 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { getAllFiles, getMaxTheoreticalBitrate, getMinTheoreticalBitrate, omit } from '@peertube/peertube-core-utils' +import { HttpStatusCode, VideoFileMetadata, VideoState } from '@peertube/peertube-models' +import { canDoQuickTranscode } from '@peertube/peertube-server/server/lib/transcoding/transcoding-quick-transcode.js' +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import { + ffprobePromise, + getAudioStream, + getVideoStreamBitrate, + getVideoStreamDimensionsInfo, + getVideoStreamFPS, + hasAudioStream +} from '@peertube/peertube-ffmpeg' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + makeGetRequest, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' +import { generateVideoWithFramerate, generateHighBitrateVideo } from '@tests/shared/generate.js' +import { checkWebTorrentWorks } from '@tests/shared/webtorrent.js' + +function updateConfigForTranscoding (server: PeerTubeServer) { + return server.config.updateCustomSubConfig({ + newConfig: { + transcoding: { + enabled: true, + allowAdditionalExtensions: true, + allowAudioFiles: true, + hls: { enabled: true }, + webVideos: { enabled: true }, + resolutions: { + '0p': false, + '144p': true, + '240p': true, + '360p': true, + '480p': true, + '720p': true, + '1080p': true, + '1440p': true, + '2160p': true + } + } + } + }) +} + +describe('Test video transcoding', function () { + let servers: PeerTubeServer[] = [] + let video4k: string + + before(async function () { + this.timeout(30_000) + + // Run servers + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + + await doubleFollow(servers[0], servers[1]) + + await updateConfigForTranscoding(servers[1]) + }) + + describe('Basic transcoding (or not)', function () { + + it('Should not transcode video on server 1', async function () { + this.timeout(60_000) + + const attributes = { + name: 'my super name for server 1', + description: 'my super description for server 1', + fixture: 'video_short.webm' + } + await servers[0].videos.upload({ attributes }) + + await waitJobs(servers) + + for (const server of servers) { + const { data } = await server.videos.list() + const video = data[0] + + const videoDetails = await server.videos.get({ id: video.id }) + expect(videoDetails.files).to.have.lengthOf(1) + + const magnetUri = videoDetails.files[0].magnetUri + expect(magnetUri).to.match(/\.webm/) + + await checkWebTorrentWorks(magnetUri, /\.webm$/) + } + }) + + it('Should transcode video on server 2', async function () { + this.timeout(120_000) + + const attributes = { + name: 'my super name for server 2', + description: 'my super description for server 2', + fixture: 'video_short.webm' + } + await servers[1].videos.upload({ attributes }) + + await waitJobs(servers) + + for (const server of servers) { + const { data } = await server.videos.list() + + const video = data.find(v => v.name === attributes.name) + const videoDetails = await server.videos.get({ id: video.id }) + + expect(videoDetails.files).to.have.lengthOf(5) + + const magnetUri = videoDetails.files[0].magnetUri + expect(magnetUri).to.match(/\.mp4/) + + await checkWebTorrentWorks(magnetUri, /\.mp4$/) + } + }) + + it('Should wait for transcoding before publishing the video', async function () { + this.timeout(160_000) + + { + // Upload the video, but wait transcoding + const attributes = { + name: 'waiting video', + fixture: 'video_short1.webm', + waitTranscoding: true + } + const { uuid } = await servers[1].videos.upload({ attributes }) + const videoId = uuid + + // Should be in transcode state + const body = await servers[1].videos.get({ id: videoId }) + expect(body.name).to.equal('waiting video') + expect(body.state.id).to.equal(VideoState.TO_TRANSCODE) + expect(body.state.label).to.equal('To transcode') + expect(body.waitTranscoding).to.be.true + + { + // Should have my video + const { data } = await servers[1].videos.listMyVideos() + const videoToFindInMine = data.find(v => v.name === attributes.name) + expect(videoToFindInMine).not.to.be.undefined + expect(videoToFindInMine.state.id).to.equal(VideoState.TO_TRANSCODE) + expect(videoToFindInMine.state.label).to.equal('To transcode') + expect(videoToFindInMine.waitTranscoding).to.be.true + } + + { + // Should not list this video + const { data } = await servers[1].videos.list() + const videoToFindInList = data.find(v => v.name === attributes.name) + expect(videoToFindInList).to.be.undefined + } + + // Server 1 should not have the video yet + await servers[0].videos.get({ id: videoId, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + } + + await waitJobs(servers) + + for (const server of servers) { + const { data } = await server.videos.list() + const videoToFind = data.find(v => v.name === 'waiting video') + expect(videoToFind).not.to.be.undefined + + const videoDetails = await server.videos.get({ id: videoToFind.id }) + + expect(videoDetails.state.id).to.equal(VideoState.PUBLISHED) + expect(videoDetails.state.label).to.equal('Published') + expect(videoDetails.waitTranscoding).to.be.true + } + }) + + it('Should accept and transcode additional extensions', async function () { + this.timeout(300_000) + + for (const fixture of [ 'video_short.mkv', 'video_short.avi' ]) { + const attributes = { + name: fixture, + fixture + } + + await servers[1].videos.upload({ attributes }) + + await waitJobs(servers) + + for (const server of servers) { + const { data } = await server.videos.list() + + const video = data.find(v => v.name === attributes.name) + const videoDetails = await server.videos.get({ id: video.id }) + expect(videoDetails.files).to.have.lengthOf(5) + + const magnetUri = videoDetails.files[0].magnetUri + expect(magnetUri).to.contain('.mp4') + } + } + }) + + it('Should transcode a 4k video', async function () { + this.timeout(200_000) + + const attributes = { + name: '4k video', + fixture: 'video_short_4k.mp4' + } + + const { uuid } = await servers[1].videos.upload({ attributes }) + video4k = uuid + + await waitJobs(servers) + + const resolutions = [ 144, 240, 360, 480, 720, 1080, 1440, 2160 ] + + for (const server of servers) { + const videoDetails = await server.videos.get({ id: video4k }) + expect(videoDetails.files).to.have.lengthOf(resolutions.length) + + for (const r of resolutions) { + expect(videoDetails.files.find(f => f.resolution.id === r)).to.not.be.undefined + expect(videoDetails.streamingPlaylists[0].files.find(f => f.resolution.id === r)).to.not.be.undefined + } + } + }) + }) + + describe('Audio transcoding', function () { + + it('Should transcode high bit rate mp3 to proper bit rate', async function () { + this.timeout(60_000) + + const attributes = { + name: 'mp3_256k', + fixture: 'video_short_mp3_256k.mp4' + } + await servers[1].videos.upload({ attributes }) + + await waitJobs(servers) + + for (const server of servers) { + const { data } = await server.videos.list() + + const video = data.find(v => v.name === attributes.name) + const videoDetails = await server.videos.get({ id: video.id }) + + expect(videoDetails.files).to.have.lengthOf(5) + + const file = videoDetails.files.find(f => f.resolution.id === 240) + const path = servers[1].servers.buildWebVideoFilePath(file.fileUrl) + const probe = await getAudioStream(path) + + if (probe.audioStream) { + expect(probe.audioStream['codec_name']).to.be.equal('aac') + expect(probe.audioStream['bit_rate']).to.be.at.most(384 * 8000) + } else { + this.fail('Could not retrieve the audio stream on ' + probe.absolutePath) + } + } + }) + + it('Should transcode video with no audio and have no audio itself', async function () { + this.timeout(60_000) + + const attributes = { + name: 'no_audio', + fixture: 'video_short_no_audio.mp4' + } + await servers[1].videos.upload({ attributes }) + + await waitJobs(servers) + + for (const server of servers) { + const { data } = await server.videos.list() + + const video = data.find(v => v.name === attributes.name) + const videoDetails = await server.videos.get({ id: video.id }) + + const file = videoDetails.files.find(f => f.resolution.id === 240) + const path = servers[1].servers.buildWebVideoFilePath(file.fileUrl) + + expect(await hasAudioStream(path)).to.be.false + } + }) + + it('Should leave the audio untouched, but properly transcode the video', async function () { + this.timeout(60_000) + + const attributes = { + name: 'untouched_audio', + fixture: 'video_short.mp4' + } + await servers[1].videos.upload({ attributes }) + + await waitJobs(servers) + + for (const server of servers) { + const { data } = await server.videos.list() + + const video = data.find(v => v.name === attributes.name) + const videoDetails = await server.videos.get({ id: video.id }) + + expect(videoDetails.files).to.have.lengthOf(5) + + const fixturePath = buildAbsoluteFixturePath(attributes.fixture) + const fixtureVideoProbe = await getAudioStream(fixturePath) + + const file = videoDetails.files.find(f => f.resolution.id === 240) + const path = servers[1].servers.buildWebVideoFilePath(file.fileUrl) + + const videoProbe = await getAudioStream(path) + + if (videoProbe.audioStream && fixtureVideoProbe.audioStream) { + const toOmit = [ 'max_bit_rate', 'duration', 'duration_ts', 'nb_frames', 'start_time', 'start_pts' ] + expect(omit(videoProbe.audioStream, toOmit)).to.be.deep.equal(omit(fixtureVideoProbe.audioStream, toOmit)) + } else { + this.fail('Could not retrieve the audio stream on ' + videoProbe.absolutePath) + } + } + }) + }) + + describe('Audio upload', function () { + + function runSuite (mode: 'legacy' | 'resumable') { + + before(async function () { + await servers[1].config.updateCustomSubConfig({ + newConfig: { + transcoding: { + hls: { enabled: true }, + webVideos: { enabled: true }, + resolutions: { + '0p': false, + '144p': false, + '240p': false, + '360p': false, + '480p': false, + '720p': false, + '1080p': false, + '1440p': false, + '2160p': false + } + } + } + }) + }) + + it('Should merge an audio file with the preview file', async function () { + this.timeout(60_000) + + const attributes = { name: 'audio_with_preview', previewfile: 'custom-preview.jpg', fixture: 'sample.ogg' } + await servers[1].videos.upload({ attributes, mode }) + + await waitJobs(servers) + + for (const server of servers) { + const { data } = await server.videos.list() + + const video = data.find(v => v.name === 'audio_with_preview') + const videoDetails = await server.videos.get({ id: video.id }) + + expect(videoDetails.files).to.have.lengthOf(1) + + await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) + await makeGetRequest({ url: server.url, path: videoDetails.previewPath, expectedStatus: HttpStatusCode.OK_200 }) + + const magnetUri = videoDetails.files[0].magnetUri + expect(magnetUri).to.contain('.mp4') + } + }) + + it('Should upload an audio file and choose a default background image', async function () { + this.timeout(60_000) + + const attributes = { name: 'audio_without_preview', fixture: 'sample.ogg' } + await servers[1].videos.upload({ attributes, mode }) + + await waitJobs(servers) + + for (const server of servers) { + const { data } = await server.videos.list() + + const video = data.find(v => v.name === 'audio_without_preview') + const videoDetails = await server.videos.get({ id: video.id }) + + expect(videoDetails.files).to.have.lengthOf(1) + + await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) + await makeGetRequest({ url: server.url, path: videoDetails.previewPath, expectedStatus: HttpStatusCode.OK_200 }) + + const magnetUri = videoDetails.files[0].magnetUri + expect(magnetUri).to.contain('.mp4') + } + }) + + it('Should upload an audio file and create an audio version only', async function () { + this.timeout(60_000) + + await servers[1].config.updateCustomSubConfig({ + newConfig: { + transcoding: { + hls: { enabled: true }, + webVideos: { enabled: true }, + resolutions: { + '0p': true, + '144p': false, + '240p': false, + '360p': false + } + } + } + }) + + const attributes = { name: 'audio_with_preview', previewfile: 'custom-preview.jpg', fixture: 'sample.ogg' } + const { id } = await servers[1].videos.upload({ attributes, mode }) + + await waitJobs(servers) + + for (const server of servers) { + const videoDetails = await server.videos.get({ id }) + + for (const files of [ videoDetails.files, videoDetails.streamingPlaylists[0].files ]) { + expect(files).to.have.lengthOf(2) + expect(files.find(f => f.resolution.id === 0)).to.not.be.undefined + } + } + + await updateConfigForTranscoding(servers[1]) + }) + } + + describe('Legacy upload', function () { + runSuite('legacy') + }) + + describe('Resumable upload', function () { + runSuite('resumable') + }) + }) + + describe('Framerate', function () { + + it('Should transcode a 60 FPS video', async function () { + this.timeout(60_000) + + const attributes = { + name: 'my super 30fps name for server 2', + description: 'my super 30fps description for server 2', + fixture: '60fps_720p_small.mp4' + } + await servers[1].videos.upload({ attributes }) + + await waitJobs(servers) + + for (const server of servers) { + const { data } = await server.videos.list() + + const video = data.find(v => v.name === attributes.name) + const videoDetails = await server.videos.get({ id: video.id }) + + expect(videoDetails.files).to.have.lengthOf(5) + expect(videoDetails.files[0].fps).to.be.above(58).and.below(62) + expect(videoDetails.files[1].fps).to.be.below(31) + expect(videoDetails.files[2].fps).to.be.below(31) + expect(videoDetails.files[3].fps).to.be.below(31) + expect(videoDetails.files[4].fps).to.be.below(31) + + for (const resolution of [ 144, 240, 360, 480 ]) { + const file = videoDetails.files.find(f => f.resolution.id === resolution) + const path = servers[1].servers.buildWebVideoFilePath(file.fileUrl) + const fps = await getVideoStreamFPS(path) + + expect(fps).to.be.below(31) + } + + const file = videoDetails.files.find(f => f.resolution.id === 720) + const path = servers[1].servers.buildWebVideoFilePath(file.fileUrl) + const fps = await getVideoStreamFPS(path) + + expect(fps).to.be.above(58).and.below(62) + } + }) + + it('Should downscale to the closest divisor standard framerate', async function () { + this.timeout(200_000) + + let tempFixturePath: string + + { + tempFixturePath = await generateVideoWithFramerate(59) + + const fps = await getVideoStreamFPS(tempFixturePath) + expect(fps).to.be.equal(59) + } + + const attributes = { + name: '59fps video', + description: '59fps video', + fixture: tempFixturePath + } + + await servers[1].videos.upload({ attributes }) + + await waitJobs(servers) + + for (const server of servers) { + const { data } = await server.videos.list() + + const { id } = data.find(v => v.name === attributes.name) + const video = await server.videos.get({ id }) + + { + const file = video.files.find(f => f.resolution.id === 240) + const path = servers[1].servers.buildWebVideoFilePath(file.fileUrl) + const fps = await getVideoStreamFPS(path) + expect(fps).to.be.equal(25) + } + + { + const file = video.files.find(f => f.resolution.id === 720) + const path = servers[1].servers.buildWebVideoFilePath(file.fileUrl) + const fps = await getVideoStreamFPS(path) + expect(fps).to.be.equal(59) + } + } + }) + }) + + describe('Bitrate control', function () { + + it('Should respect maximum bitrate values', async function () { + this.timeout(160_000) + + const tempFixturePath = await generateHighBitrateVideo() + + const attributes = { + name: 'high bitrate video', + description: 'high bitrate video', + fixture: tempFixturePath + } + + await servers[1].videos.upload({ attributes }) + + await waitJobs(servers) + + for (const server of servers) { + const { data } = await server.videos.list() + + const { id } = data.find(v => v.name === attributes.name) + const video = await server.videos.get({ id }) + + for (const resolution of [ 240, 360, 480, 720, 1080 ]) { + const file = video.files.find(f => f.resolution.id === resolution) + const path = servers[1].servers.buildWebVideoFilePath(file.fileUrl) + + const bitrate = await getVideoStreamBitrate(path) + const fps = await getVideoStreamFPS(path) + const dataResolution = await getVideoStreamDimensionsInfo(path) + + expect(resolution).to.equal(resolution) + + const maxBitrate = getMaxTheoreticalBitrate({ ...dataResolution, fps }) + expect(bitrate).to.be.below(maxBitrate) + } + } + }) + + it('Should not transcode to an higher bitrate than the original file but above our low limit', async function () { + this.timeout(160_000) + + const newConfig = { + transcoding: { + enabled: true, + resolutions: { + '144p': true, + '240p': true, + '360p': true, + '480p': true, + '720p': true, + '1080p': true, + '1440p': true, + '2160p': true + }, + webVideos: { enabled: true }, + hls: { enabled: true } + } + } + await servers[1].config.updateCustomSubConfig({ newConfig }) + + const attributes = { + name: 'low bitrate', + fixture: 'low-bitrate.mp4' + } + + const { id } = await servers[1].videos.upload({ attributes }) + + await waitJobs(servers) + + const video = await servers[1].videos.get({ id }) + + const resolutions = [ 240, 360, 480, 720, 1080 ] + for (const r of resolutions) { + const file = video.files.find(f => f.resolution.id === r) + + const path = servers[1].servers.buildWebVideoFilePath(file.fileUrl) + const bitrate = await getVideoStreamBitrate(path) + + const inputBitrate = 60_000 + const limit = getMinTheoreticalBitrate({ fps: 10, ratio: 1, resolution: r }) + let belowValue = Math.max(inputBitrate, limit) + belowValue += belowValue * 0.20 // Apply 20% margin because bitrate control is not very precise + + expect(bitrate, `${path} not below ${limit}`).to.be.below(belowValue) + } + }) + }) + + describe('FFprobe', function () { + + it('Should provide valid ffprobe data', async function () { + this.timeout(160_000) + + const videoUUID = (await servers[1].videos.quickUpload({ name: 'ffprobe data' })).uuid + await waitJobs(servers) + + { + const video = await servers[1].videos.get({ id: videoUUID }) + const file = video.files.find(f => f.resolution.id === 240) + const path = servers[1].servers.buildWebVideoFilePath(file.fileUrl) + + const probe = await ffprobePromise(path) + const metadata = new VideoFileMetadata(probe) + + // expected format properties + for (const p of [ + 'tags.encoder', + 'format_long_name', + 'size', + 'bit_rate' + ]) { + expect(metadata.format).to.have.nested.property(p) + } + + // expected stream properties + for (const p of [ + 'codec_long_name', + 'profile', + 'width', + 'height', + 'display_aspect_ratio', + 'avg_frame_rate', + 'pix_fmt' + ]) { + expect(metadata.streams[0]).to.have.nested.property(p) + } + + expect(metadata).to.not.have.nested.property('format.filename') + } + + for (const server of servers) { + const videoDetails = await server.videos.get({ id: videoUUID }) + + const videoFiles = getAllFiles(videoDetails) + expect(videoFiles).to.have.lengthOf(10) + + for (const file of videoFiles) { + expect(file.metadata).to.be.undefined + expect(file.metadataUrl).to.exist + expect(file.metadataUrl).to.contain(servers[1].url) + expect(file.metadataUrl).to.contain(videoUUID) + + const metadata = await server.videos.getFileMetadata({ url: file.metadataUrl }) + expect(metadata).to.have.nested.property('format.size') + } + } + }) + + it('Should correctly detect if quick transcode is possible', async function () { + this.timeout(10_000) + + expect(await canDoQuickTranscode(buildAbsoluteFixturePath('video_short.mp4'))).to.be.true + expect(await canDoQuickTranscode(buildAbsoluteFixturePath('video_short.webm'))).to.be.false + }) + }) + + describe('Transcoding job queue', function () { + + it('Should have the appropriate priorities for transcoding jobs', async function () { + const body = await servers[1].jobs.list({ + start: 0, + count: 100, + sort: 'createdAt', + jobType: 'video-transcoding' + }) + + const jobs = body.data + const transcodingJobs = jobs.filter(j => j.data.videoUUID === video4k) + + expect(transcodingJobs).to.have.lengthOf(16) + + const hlsJobs = transcodingJobs.filter(j => j.data.type === 'new-resolution-to-hls') + const webVideoJobs = transcodingJobs.filter(j => j.data.type === 'new-resolution-to-web-video') + const optimizeJobs = transcodingJobs.filter(j => j.data.type === 'optimize-to-web-video') + + expect(hlsJobs).to.have.lengthOf(8) + expect(webVideoJobs).to.have.lengthOf(7) + expect(optimizeJobs).to.have.lengthOf(1) + + for (const j of optimizeJobs.concat(hlsJobs.concat(webVideoJobs))) { + expect(j.priority).to.be.greaterThan(100) + expect(j.priority).to.be.lessThan(150) + } + }) + }) + + describe('Bounded transcoding', function () { + + it('Should not generate an upper resolution than original file', async function () { + this.timeout(120_000) + + await servers[0].config.updateExistingSubConfig({ + newConfig: { + transcoding: { + enabled: true, + hls: { enabled: true }, + webVideos: { enabled: true }, + resolutions: { + '0p': false, + '144p': false, + '240p': true, + '360p': false, + '480p': true, + '720p': false, + '1080p': false, + '1440p': false, + '2160p': false + }, + alwaysTranscodeOriginalResolution: false + } + } + }) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'video', fixture: 'video_short.webm' }) + await waitJobs(servers) + + const video = await servers[0].videos.get({ id: uuid }) + const hlsFiles = video.streamingPlaylists[0].files + + expect(video.files).to.have.lengthOf(2) + expect(hlsFiles).to.have.lengthOf(2) + + // eslint-disable-next-line @typescript-eslint/require-array-sort-compare + const resolutions = getAllFiles(video).map(f => f.resolution.id).sort() + expect(resolutions).to.deep.equal([ 240, 240, 480, 480 ]) + }) + + it('Should only keep the original resolution if all resolutions are disabled', async function () { + this.timeout(120_000) + + await servers[0].config.updateExistingSubConfig({ + newConfig: { + transcoding: { + resolutions: { + '0p': false, + '144p': false, + '240p': false, + '360p': false, + '480p': false, + '720p': false, + '1080p': false, + '1440p': false, + '2160p': false + } + } + } + }) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'video', fixture: 'video_short.webm' }) + await waitJobs(servers) + + const video = await servers[0].videos.get({ id: uuid }) + const hlsFiles = video.streamingPlaylists[0].files + + expect(video.files).to.have.lengthOf(1) + expect(hlsFiles).to.have.lengthOf(1) + + expect(video.files[0].resolution.id).to.equal(720) + expect(hlsFiles[0].resolution.id).to.equal(720) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/transcoding/update-while-transcoding.ts b/packages/tests/src/api/transcoding/update-while-transcoding.ts new file mode 100644 index 000000000..9990bc745 --- /dev/null +++ b/packages/tests/src/api/transcoding/update-while-transcoding.ts @@ -0,0 +1,161 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { wait } from '@peertube/peertube-core-utils' +import { VideoPrivacy } from '@peertube/peertube-models' +import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + ObjectStorageCommand, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' +import { completeCheckHlsPlaylist } from '@tests/shared/streaming-playlists.js' + +describe('Test update video privacy while transcoding', function () { + let servers: PeerTubeServer[] = [] + + const videoUUIDs: string[] = [] + + function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) { + + it('Should not have an error while quickly updating a private video to public after upload #1', async function () { + this.timeout(360_000) + + const attributes = { + name: 'quick update', + privacy: VideoPrivacy.PRIVATE + } + + const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: false }) + await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } }) + videoUUIDs.push(uuid) + + await waitJobs(servers) + + await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl }) + }) + + it('Should not have an error while quickly updating a private video to public after upload #2', async function () { + this.timeout(60000) + + { + const attributes = { + name: 'quick update 2', + privacy: VideoPrivacy.PRIVATE + } + + const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: true }) + await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } }) + videoUUIDs.push(uuid) + + await waitJobs(servers) + + await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl }) + } + }) + + it('Should not have an error while quickly updating a private video to public after upload #3', async function () { + this.timeout(60000) + + const attributes = { + name: 'quick update 3', + privacy: VideoPrivacy.PRIVATE + } + + const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: true }) + await wait(1000) + await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } }) + videoUUIDs.push(uuid) + + await waitJobs(servers) + + await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl }) + }) + } + + before(async function () { + this.timeout(120000) + + const configOverride = { + transcoding: { + enabled: true, + allow_audio_files: true, + hls: { + enabled: true + } + } + } + servers = await createMultipleServers(2, configOverride) + + // Get the access tokens + await setAccessTokensToServers(servers) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + }) + + describe('With Web Video & HLS enabled', function () { + runTestSuite(false) + }) + + describe('With only HLS enabled', function () { + + before(async function () { + await servers[0].config.updateCustomSubConfig({ + newConfig: { + transcoding: { + enabled: true, + allowAudioFiles: true, + resolutions: { + '144p': false, + '240p': true, + '360p': true, + '480p': true, + '720p': true, + '1080p': true, + '1440p': true, + '2160p': true + }, + hls: { + enabled: true + }, + webVideos: { + enabled: false + } + } + } + }) + }) + + runTestSuite(true) + }) + + describe('With object storage enabled', function () { + if (areMockObjectStorageTestsDisabled()) return + + const objectStorage = new ObjectStorageCommand() + + before(async function () { + this.timeout(120000) + + const configOverride = objectStorage.getDefaultMockConfig() + await objectStorage.prepareDefaultMockBuckets() + + await servers[0].kill() + await servers[0].run(configOverride) + }) + + runTestSuite(true, objectStorage.getMockPlaylistBaseUrl()) + + after(async function () { + await objectStorage.cleanupMock() + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/transcoding/video-studio.ts b/packages/tests/src/api/transcoding/video-studio.ts new file mode 100644 index 000000000..8a3788aa6 --- /dev/null +++ b/packages/tests/src/api/transcoding/video-studio.ts @@ -0,0 +1,379 @@ +import { expect } from 'chai' +import { getAllFiles } from '@peertube/peertube-core-utils' +import { VideoStudioTask } from '@peertube/peertube-models' +import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + ObjectStorageCommand, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + VideoStudioCommand, + waitJobs +} from '@peertube/peertube-server-commands' +import { checkVideoDuration, expectStartWith } from '@tests/shared/checks.js' +import { checkPersistentTmpIsEmpty } from '@tests/shared/directories.js' + +describe('Test video studio', function () { + let servers: PeerTubeServer[] = [] + let videoUUID: string + + async function renewVideo (fixture = 'video_short.webm') { + const video = await servers[0].videos.quickUpload({ name: 'video', fixture }) + videoUUID = video.uuid + + await waitJobs(servers) + } + + async function createTasks (tasks: VideoStudioTask[]) { + await servers[0].videoStudio.createEditionTasks({ videoId: videoUUID, tasks }) + await waitJobs(servers) + } + + before(async function () { + this.timeout(120_000) + + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + await doubleFollow(servers[0], servers[1]) + + await servers[0].config.enableMinimumTranscoding() + + await servers[0].config.enableStudio() + }) + + describe('Cutting', function () { + + it('Should cut the beginning of the video', async function () { + this.timeout(120_000) + + await renewVideo() + await waitJobs(servers) + + const beforeTasks = new Date() + + await createTasks([ + { + name: 'cut', + options: { + start: 2 + } + } + ]) + + for (const server of servers) { + await checkVideoDuration(server, videoUUID, 3) + + const video = await server.videos.get({ id: videoUUID }) + expect(new Date(video.publishedAt)).to.be.below(beforeTasks) + } + }) + + it('Should cut the end of the video', async function () { + this.timeout(120_000) + await renewVideo() + + await createTasks([ + { + name: 'cut', + options: { + end: 2 + } + } + ]) + + for (const server of servers) { + await checkVideoDuration(server, videoUUID, 2) + } + }) + + it('Should cut start/end of the video', async function () { + this.timeout(120_000) + await renewVideo('video_short1.webm') // 10 seconds video duration + + await createTasks([ + { + name: 'cut', + options: { + start: 2, + end: 6 + } + } + ]) + + for (const server of servers) { + await checkVideoDuration(server, videoUUID, 4) + } + }) + }) + + describe('Intro/Outro', function () { + + it('Should add an intro', async function () { + this.timeout(120_000) + await renewVideo() + + await createTasks([ + { + name: 'add-intro', + options: { + file: 'video_short.webm' + } + } + ]) + + for (const server of servers) { + await checkVideoDuration(server, videoUUID, 10) + } + }) + + it('Should add an outro', async function () { + this.timeout(120_000) + await renewVideo() + + await createTasks([ + { + name: 'add-outro', + options: { + file: 'video_very_short_240p.mp4' + } + } + ]) + + for (const server of servers) { + await checkVideoDuration(server, videoUUID, 7) + } + }) + + it('Should add an intro/outro', async function () { + this.timeout(120_000) + await renewVideo() + + await createTasks([ + { + name: 'add-intro', + options: { + file: 'video_very_short_240p.mp4' + } + }, + { + name: 'add-outro', + options: { + // Different frame rate + file: 'video_short2.webm' + } + } + ]) + + for (const server of servers) { + await checkVideoDuration(server, videoUUID, 12) + } + }) + + it('Should add an intro to a video without audio', async function () { + this.timeout(120_000) + await renewVideo('video_short_no_audio.mp4') + + await createTasks([ + { + name: 'add-intro', + options: { + file: 'video_very_short_240p.mp4' + } + } + ]) + + for (const server of servers) { + await checkVideoDuration(server, videoUUID, 7) + } + }) + + it('Should add an outro without audio to a video with audio', async function () { + this.timeout(120_000) + await renewVideo() + + await createTasks([ + { + name: 'add-outro', + options: { + file: 'video_short_no_audio.mp4' + } + } + ]) + + for (const server of servers) { + await checkVideoDuration(server, videoUUID, 10) + } + }) + + it('Should add an outro without audio to a video with audio', async function () { + this.timeout(120_000) + await renewVideo('video_short_no_audio.mp4') + + await createTasks([ + { + name: 'add-outro', + options: { + file: 'video_short_no_audio.mp4' + } + } + ]) + + for (const server of servers) { + await checkVideoDuration(server, videoUUID, 10) + } + }) + }) + + describe('Watermark', function () { + + it('Should add a watermark to the video', async function () { + this.timeout(120_000) + await renewVideo() + + const video = await servers[0].videos.get({ id: videoUUID }) + const oldFileUrls = getAllFiles(video).map(f => f.fileUrl) + + await createTasks([ + { + name: 'add-watermark', + options: { + file: 'custom-thumbnail.png' + } + } + ]) + + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + const fileUrls = getAllFiles(video).map(f => f.fileUrl) + + for (const oldUrl of oldFileUrls) { + expect(fileUrls).to.not.include(oldUrl) + } + } + }) + }) + + describe('Complex tasks', function () { + it('Should run a complex task', async function () { + this.timeout(240_000) + await renewVideo() + + await createTasks(VideoStudioCommand.getComplexTask()) + + for (const server of servers) { + await checkVideoDuration(server, videoUUID, 9) + } + }) + }) + + describe('HLS only studio edition', function () { + + before(async function () { + // Disable Web Videos + await servers[0].config.updateExistingSubConfig({ + newConfig: { + transcoding: { + webVideos: { + enabled: false + } + } + } + }) + }) + + it('Should run a complex task on HLS only video', async function () { + this.timeout(240_000) + await renewVideo() + + await createTasks(VideoStudioCommand.getComplexTask()) + + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + expect(video.files).to.have.lengthOf(0) + + await checkVideoDuration(server, videoUUID, 9) + } + }) + }) + + describe('Server restart', function () { + + it('Should still be able to run video edition after a server restart', async function () { + this.timeout(240_000) + + await renewVideo() + await servers[0].videoStudio.createEditionTasks({ videoId: videoUUID, tasks: VideoStudioCommand.getComplexTask() }) + + await servers[0].kill() + await servers[0].run() + + await waitJobs(servers) + + for (const server of servers) { + await checkVideoDuration(server, videoUUID, 9) + } + }) + + it('Should have an empty persistent tmp directory', async function () { + await checkPersistentTmpIsEmpty(servers[0]) + }) + }) + + describe('Object storage studio edition', function () { + if (areMockObjectStorageTestsDisabled()) return + + const objectStorage = new ObjectStorageCommand() + + before(async function () { + await objectStorage.prepareDefaultMockBuckets() + + await servers[0].kill() + await servers[0].run(objectStorage.getDefaultMockConfig()) + + await servers[0].config.enableMinimumTranscoding() + }) + + it('Should run a complex task on a video in object storage', async function () { + this.timeout(240_000) + await renewVideo() + + const video = await servers[0].videos.get({ id: videoUUID }) + const oldFileUrls = getAllFiles(video).map(f => f.fileUrl) + + await createTasks(VideoStudioCommand.getComplexTask()) + + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + const files = getAllFiles(video) + + for (const f of files) { + expect(oldFileUrls).to.not.include(f.fileUrl) + } + + for (const webVideoFile of video.files) { + expectStartWith(webVideoFile.fileUrl, objectStorage.getMockWebVideosBaseUrl()) + } + + for (const hlsFile of video.streamingPlaylists[0].files) { + expectStartWith(hlsFile.fileUrl, objectStorage.getMockPlaylistBaseUrl()) + } + + await checkVideoDuration(server, videoUUID, 9) + } + }) + + after(async function () { + await objectStorage.cleanupMock() + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/users/index.ts b/packages/tests/src/api/users/index.ts new file mode 100644 index 000000000..830d4da62 --- /dev/null +++ b/packages/tests/src/api/users/index.ts @@ -0,0 +1,8 @@ +import './oauth.js' +import './registrations`.js' +import './two-factor.js' +import './user-subscriptions.js' +import './user-videos.js' +import './users.js' +import './users-multiple-servers.js' +import './users-email-verification.js' diff --git a/packages/tests/src/api/users/oauth.ts b/packages/tests/src/api/users/oauth.ts new file mode 100644 index 000000000..fe50872cb --- /dev/null +++ b/packages/tests/src/api/users/oauth.ts @@ -0,0 +1,203 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { HttpStatusCode, OAuth2ErrorCode, PeerTubeProblemDocument } from '@peertube/peertube-models' +import { SQLCommand } from '@tests/shared/sql-command.js' +import { + cleanupTests, + createSingleServer, + killallServers, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test oauth', function () { + let server: PeerTubeServer + let sqlCommand: SQLCommand + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1, { + rates_limit: { + login: { + max: 30 + } + } + }) + + await setAccessTokensToServers([ server ]) + + sqlCommand = new SQLCommand(server) + }) + + describe('OAuth client', function () { + + function expectInvalidClient (body: PeerTubeProblemDocument) { + expect(body.code).to.equal(OAuth2ErrorCode.INVALID_CLIENT) + expect(body.error).to.contain('client is invalid') + expect(body.type.startsWith('https://')).to.be.true + expect(body.type).to.contain(OAuth2ErrorCode.INVALID_CLIENT) + } + + it('Should create a new client') + + it('Should return the first client') + + it('Should remove the last client') + + it('Should not login with an invalid client id', async function () { + const client = { id: 'client', secret: server.store.client.secret } + const body = await server.login.login({ client, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + + expectInvalidClient(body) + }) + + it('Should not login with an invalid client secret', async function () { + const client = { id: server.store.client.id, secret: 'coucou' } + const body = await server.login.login({ client, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + + expectInvalidClient(body) + }) + }) + + describe('Login', function () { + + function expectInvalidCredentials (body: PeerTubeProblemDocument) { + expect(body.code).to.equal(OAuth2ErrorCode.INVALID_GRANT) + expect(body.error).to.contain('credentials are invalid') + expect(body.type.startsWith('https://')).to.be.true + expect(body.type).to.contain(OAuth2ErrorCode.INVALID_GRANT) + } + + it('Should not login with an invalid username', async function () { + const user = { username: 'captain crochet', password: server.store.user.password } + const body = await server.login.login({ user, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + + expectInvalidCredentials(body) + }) + + it('Should not login with an invalid password', async function () { + const user = { username: server.store.user.username, password: 'mew_three' } + const body = await server.login.login({ user, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + + expectInvalidCredentials(body) + }) + + it('Should be able to login', async function () { + await server.login.login({ expectedStatus: HttpStatusCode.OK_200 }) + }) + + it('Should be able to login with an insensitive username', async function () { + const user = { username: 'RoOt', password: server.store.user.password } + await server.login.login({ user, expectedStatus: HttpStatusCode.OK_200 }) + + const user2 = { username: 'rOoT', password: server.store.user.password } + await server.login.login({ user: user2, expectedStatus: HttpStatusCode.OK_200 }) + + const user3 = { username: 'ROOt', password: server.store.user.password } + await server.login.login({ user: user3, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('Logout', function () { + + it('Should logout (revoke token)', async function () { + await server.login.logout({ token: server.accessToken }) + }) + + it('Should not be able to get the user information', async function () { + await server.users.getMyInfo({ expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should not be able to upload a video', async function () { + await server.videos.upload({ attributes: { name: 'video' }, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should be able to login again', async function () { + const body = await server.login.login() + server.accessToken = body.access_token + server.refreshToken = body.refresh_token + }) + + it('Should be able to get my user information again', async function () { + await server.users.getMyInfo() + }) + + it('Should have an expired access token', async function () { + this.timeout(60000) + + await sqlCommand.setTokenField(server.accessToken, 'accessTokenExpiresAt', new Date().toISOString()) + await sqlCommand.setTokenField(server.accessToken, 'refreshTokenExpiresAt', new Date().toISOString()) + + await killallServers([ server ]) + await server.run() + + await server.users.getMyInfo({ expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should not be able to refresh an access token with an expired refresh token', async function () { + await server.login.refreshToken({ refreshToken: server.refreshToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should refresh the token', async function () { + this.timeout(50000) + + const futureDate = new Date(new Date().getTime() + 1000 * 60).toISOString() + await sqlCommand.setTokenField(server.accessToken, 'refreshTokenExpiresAt', futureDate) + + await killallServers([ server ]) + await server.run() + + const res = await server.login.refreshToken({ refreshToken: server.refreshToken }) + server.accessToken = res.body.access_token + server.refreshToken = res.body.refresh_token + }) + + it('Should be able to get my user information again', async function () { + await server.users.getMyInfo() + }) + }) + + describe('Custom token lifetime', function () { + before(async function () { + this.timeout(120_000) + + await server.kill() + await server.run({ + oauth2: { + token_lifetime: { + access_token: '2 seconds', + refresh_token: '2 seconds' + } + } + }) + }) + + it('Should have a very short access token lifetime', async function () { + this.timeout(50000) + + const { access_token: accessToken } = await server.login.login() + await server.users.getMyInfo({ token: accessToken }) + + await wait(3000) + await server.users.getMyInfo({ token: accessToken, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should have a very short refresh token lifetime', async function () { + this.timeout(50000) + + const { refresh_token: refreshToken } = await server.login.login() + await server.login.refreshToken({ refreshToken }) + + await wait(3000) + await server.login.refreshToken({ refreshToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + }) + + after(async function () { + await sqlCommand.cleanup() + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/users/registrations.ts b/packages/tests/src/api/users/registrations.ts new file mode 100644 index 000000000..dbe1bc4f5 --- /dev/null +++ b/packages/tests/src/api/users/registrations.ts @@ -0,0 +1,415 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { MockSmtpServer } from '@tests/shared/mock-servers/index.js' +import { UserRegistrationState, UserRole } from '@peertube/peertube-models' +import { + cleanupTests, + ConfigCommand, + createSingleServer, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test registrations', function () { + let server: PeerTubeServer + + const emails: object[] = [] + let emailPort: number + + before(async function () { + this.timeout(30000) + + emailPort = await MockSmtpServer.Instance.collectEmails(emails) + + server = await createSingleServer(1, ConfigCommand.getEmailOverrideConfig(emailPort)) + + await setAccessTokensToServers([ server ]) + await server.config.enableSignup(false) + }) + + describe('Direct registrations of a new user', function () { + let user1Token: string + + it('Should register a new user', async function () { + const user = { displayName: 'super user 1', username: 'user_1', password: 'my super password' } + const channel = { name: 'my_user_1_channel', displayName: 'my channel rocks' } + + await server.registrations.register({ ...user, channel }) + }) + + it('Should be able to login with this registered user', async function () { + const user1 = { username: 'user_1', password: 'my super password' } + + user1Token = await server.login.getAccessToken(user1) + }) + + it('Should have the correct display name', async function () { + const user = await server.users.getMyInfo({ token: user1Token }) + expect(user.account.displayName).to.equal('super user 1') + }) + + it('Should have the correct video quota', async function () { + const user = await server.users.getMyInfo({ token: user1Token }) + expect(user.videoQuota).to.equal(5 * 1024 * 1024) + }) + + it('Should have created the channel', async function () { + const { displayName } = await server.channels.get({ channelName: 'my_user_1_channel' }) + + expect(displayName).to.equal('my channel rocks') + }) + + it('Should remove me', async function () { + { + const { data } = await server.users.list() + expect(data.find(u => u.username === 'user_1')).to.not.be.undefined + } + + await server.users.deleteMe({ token: user1Token }) + + { + const { data } = await server.users.list() + expect(data.find(u => u.username === 'user_1')).to.be.undefined + } + }) + }) + + describe('Registration requests', function () { + let id2: number + let id3: number + let id4: number + + let user2Token: string + let user3Token: string + + before(async function () { + this.timeout(60000) + + await server.config.enableSignup(true) + + { + const { id } = await server.registrations.requestRegistration({ + username: 'user4', + registrationReason: 'registration reason 4' + }) + + id4 = id + } + }) + + it('Should request a registration without a channel', async function () { + { + const { id } = await server.registrations.requestRegistration({ + username: 'user2', + displayName: 'my super user 2', + email: 'user2@example.com', + password: 'user2password', + registrationReason: 'registration reason 2' + }) + + id2 = id + } + }) + + it('Should request a registration with a channel', async function () { + const { id } = await server.registrations.requestRegistration({ + username: 'user3', + displayName: 'my super user 3', + channel: { + displayName: 'my user 3 channel', + name: 'super_user3_channel' + }, + email: 'user3@example.com', + password: 'user3password', + registrationReason: 'registration reason 3' + }) + + id3 = id + }) + + it('Should list these registration requests', async function () { + { + const { total, data } = await server.registrations.list({ sort: '-createdAt' }) + expect(total).to.equal(3) + expect(data).to.have.lengthOf(3) + + { + expect(data[0].id).to.equal(id3) + expect(data[0].username).to.equal('user3') + expect(data[0].accountDisplayName).to.equal('my super user 3') + + expect(data[0].channelDisplayName).to.equal('my user 3 channel') + expect(data[0].channelHandle).to.equal('super_user3_channel') + + expect(data[0].createdAt).to.exist + expect(data[0].updatedAt).to.exist + + expect(data[0].email).to.equal('user3@example.com') + expect(data[0].emailVerified).to.be.null + + expect(data[0].moderationResponse).to.be.null + expect(data[0].registrationReason).to.equal('registration reason 3') + expect(data[0].state.id).to.equal(UserRegistrationState.PENDING) + expect(data[0].state.label).to.equal('Pending') + expect(data[0].user).to.be.null + } + + { + expect(data[1].id).to.equal(id2) + expect(data[1].username).to.equal('user2') + expect(data[1].accountDisplayName).to.equal('my super user 2') + + expect(data[1].channelDisplayName).to.be.null + expect(data[1].channelHandle).to.be.null + + expect(data[1].createdAt).to.exist + expect(data[1].updatedAt).to.exist + + expect(data[1].email).to.equal('user2@example.com') + expect(data[1].emailVerified).to.be.null + + expect(data[1].moderationResponse).to.be.null + expect(data[1].registrationReason).to.equal('registration reason 2') + expect(data[1].state.id).to.equal(UserRegistrationState.PENDING) + expect(data[1].state.label).to.equal('Pending') + expect(data[1].user).to.be.null + } + + { + expect(data[2].username).to.equal('user4') + } + } + + { + const { total, data } = await server.registrations.list({ count: 1, start: 1, sort: 'createdAt' }) + + expect(total).to.equal(3) + expect(data).to.have.lengthOf(1) + expect(data[0].id).to.equal(id2) + } + + { + const { total, data } = await server.registrations.list({ search: 'user3' }) + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + expect(data[0].id).to.equal(id3) + } + }) + + it('Should reject a registration request', async function () { + await server.registrations.reject({ id: id4, moderationResponse: 'I do not want id 4 on this instance' }) + }) + + it('Should have sent an email to the user explanining the registration has been rejected', async function () { + this.timeout(50000) + + await waitJobs([ server ]) + + const email = emails.find(e => e['to'][0]['address'] === 'user4@example.com') + expect(email).to.exist + + expect(email['subject']).to.contain('been rejected') + expect(email['text']).to.contain('been rejected') + expect(email['text']).to.contain('I do not want id 4 on this instance') + }) + + it('Should accept registration requests', async function () { + await server.registrations.accept({ id: id2, moderationResponse: 'Welcome id 2' }) + await server.registrations.accept({ id: id3, moderationResponse: 'Welcome id 3' }) + }) + + it('Should have sent an email to the user explanining the registration has been accepted', async function () { + this.timeout(50000) + + await waitJobs([ server ]) + + { + const email = emails.find(e => e['to'][0]['address'] === 'user2@example.com') + expect(email).to.exist + + expect(email['subject']).to.contain('been accepted') + expect(email['text']).to.contain('been accepted') + expect(email['text']).to.contain('Welcome id 2') + } + + { + const email = emails.find(e => e['to'][0]['address'] === 'user3@example.com') + expect(email).to.exist + + expect(email['subject']).to.contain('been accepted') + expect(email['text']).to.contain('been accepted') + expect(email['text']).to.contain('Welcome id 3') + } + }) + + it('Should login with these users', async function () { + user2Token = await server.login.getAccessToken({ username: 'user2', password: 'user2password' }) + user3Token = await server.login.getAccessToken({ username: 'user3', password: 'user3password' }) + }) + + it('Should have created the appropriate attributes for user 2', async function () { + const me = await server.users.getMyInfo({ token: user2Token }) + + expect(me.username).to.equal('user2') + expect(me.account.displayName).to.equal('my super user 2') + expect(me.videoQuota).to.equal(5 * 1024 * 1024) + expect(me.videoChannels[0].name).to.equal('user2_channel') + expect(me.videoChannels[0].displayName).to.equal('Main user2 channel') + expect(me.role.id).to.equal(UserRole.USER) + expect(me.email).to.equal('user2@example.com') + }) + + it('Should have created the appropriate attributes for user 3', async function () { + const me = await server.users.getMyInfo({ token: user3Token }) + + expect(me.username).to.equal('user3') + expect(me.account.displayName).to.equal('my super user 3') + expect(me.videoQuota).to.equal(5 * 1024 * 1024) + expect(me.videoChannels[0].name).to.equal('super_user3_channel') + expect(me.videoChannels[0].displayName).to.equal('my user 3 channel') + expect(me.role.id).to.equal(UserRole.USER) + expect(me.email).to.equal('user3@example.com') + }) + + it('Should list these accepted/rejected registration requests', async function () { + const { data } = await server.registrations.list({ sort: 'createdAt' }) + const { data: users } = await server.users.list() + + { + expect(data[0].id).to.equal(id4) + expect(data[0].state.id).to.equal(UserRegistrationState.REJECTED) + expect(data[0].state.label).to.equal('Rejected') + + expect(data[0].moderationResponse).to.equal('I do not want id 4 on this instance') + expect(data[0].user).to.be.null + + expect(users.find(u => u.username === 'user4')).to.not.exist + } + + { + expect(data[1].id).to.equal(id2) + expect(data[1].state.id).to.equal(UserRegistrationState.ACCEPTED) + expect(data[1].state.label).to.equal('Accepted') + + expect(data[1].moderationResponse).to.equal('Welcome id 2') + expect(data[1].user).to.exist + + const user2 = users.find(u => u.username === 'user2') + expect(data[1].user.id).to.equal(user2.id) + } + + { + expect(data[2].id).to.equal(id3) + expect(data[2].state.id).to.equal(UserRegistrationState.ACCEPTED) + expect(data[2].state.label).to.equal('Accepted') + + expect(data[2].moderationResponse).to.equal('Welcome id 3') + expect(data[2].user).to.exist + + const user3 = users.find(u => u.username === 'user3') + expect(data[2].user.id).to.equal(user3.id) + } + }) + + it('Shoulde delete a registration', async function () { + await server.registrations.delete({ id: id2 }) + await server.registrations.delete({ id: id3 }) + + const { total, data } = await server.registrations.list() + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + expect(data[0].id).to.equal(id4) + + const { data: users } = await server.users.list() + + for (const username of [ 'user2', 'user3' ]) { + expect(users.find(u => u.username === username)).to.exist + } + }) + + it('Should be able to prevent email delivery on accept/reject', async function () { + this.timeout(50000) + + let id1: number + let id2: number + + { + const { id } = await server.registrations.requestRegistration({ + username: 'user7', + email: 'user7@example.com', + registrationReason: 'tt' + }) + id1 = id + } + { + const { id } = await server.registrations.requestRegistration({ + username: 'user8', + email: 'user8@example.com', + registrationReason: 'tt' + }) + id2 = id + } + + await server.registrations.accept({ id: id1, moderationResponse: 'tt', preventEmailDelivery: true }) + await server.registrations.reject({ id: id2, moderationResponse: 'tt', preventEmailDelivery: true }) + + await waitJobs([ server ]) + + const filtered = emails.filter(e => { + const address = e['to'][0]['address'] + return address === 'user7@example.com' || address === 'user8@example.com' + }) + + expect(filtered).to.have.lengthOf(0) + }) + + it('Should request a registration without a channel, that will conflict with an already existing channel', async function () { + let id1: number + let id2: number + + { + const { id } = await server.registrations.requestRegistration({ + registrationReason: 'tt', + username: 'user5', + password: 'user5password', + channel: { + displayName: 'channel 6', + name: 'user6_channel' + } + }) + + id1 = id + } + + { + const { id } = await server.registrations.requestRegistration({ + registrationReason: 'tt', + username: 'user6', + password: 'user6password' + }) + + id2 = id + } + + await server.registrations.accept({ id: id1, moderationResponse: 'tt' }) + await server.registrations.accept({ id: id2, moderationResponse: 'tt' }) + + const user5Token = await server.login.getAccessToken('user5', 'user5password') + const user6Token = await server.login.getAccessToken('user6', 'user6password') + + const user5 = await server.users.getMyInfo({ token: user5Token }) + const user6 = await server.users.getMyInfo({ token: user6Token }) + + expect(user5.videoChannels[0].name).to.equal('user6_channel') + expect(user6.videoChannels[0].name).to.equal('user6_channel-1') + }) + }) + + after(async function () { + MockSmtpServer.Instance.kill() + + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/users/two-factor.ts b/packages/tests/src/api/users/two-factor.ts new file mode 100644 index 000000000..fda125d20 --- /dev/null +++ b/packages/tests/src/api/users/two-factor.ts @@ -0,0 +1,206 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { HttpStatusCode, HttpStatusCodeType } from '@peertube/peertube-models' +import { expectStartWith } from '@tests/shared/checks.js' +import { + cleanupTests, + createSingleServer, + PeerTubeServer, + setAccessTokensToServers, + TwoFactorCommand +} from '@peertube/peertube-server-commands' + +async function login (options: { + server: PeerTubeServer + username: string + password: string + otpToken?: string + expectedStatus?: HttpStatusCodeType +}) { + const { server, username, password, otpToken, expectedStatus } = options + + const user = { username, password } + const { res, body: { access_token: token } } = await server.login.loginAndGetResponse({ user, otpToken, expectedStatus }) + + return { res, token } +} + +describe('Test users', function () { + let server: PeerTubeServer + let otpSecret: string + let requestToken: string + + const userUsername = 'user1' + let userId: number + let userPassword: string + let userToken: string + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + const res = await server.users.generate(userUsername) + userId = res.userId + userPassword = res.password + userToken = res.token + }) + + it('Should not add the header on login if two factor is not enabled', async function () { + const { res, token } = await login({ server, username: userUsername, password: userPassword }) + + expect(res.header['x-peertube-otp']).to.not.exist + + await server.users.getMyInfo({ token }) + }) + + it('Should request two factor and get the secret and uri', async function () { + const { otpRequest } = await server.twoFactor.request({ userId, token: userToken, currentPassword: userPassword }) + + expect(otpRequest.requestToken).to.exist + + expect(otpRequest.secret).to.exist + expect(otpRequest.secret).to.have.lengthOf(32) + + expect(otpRequest.uri).to.exist + expectStartWith(otpRequest.uri, 'otpauth://') + expect(otpRequest.uri).to.include(otpRequest.secret) + + requestToken = otpRequest.requestToken + otpSecret = otpRequest.secret + }) + + it('Should not have two factor confirmed yet', async function () { + const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken }) + expect(twoFactorEnabled).to.be.false + }) + + it('Should confirm two factor', async function () { + await server.twoFactor.confirmRequest({ + userId, + token: userToken, + otpToken: TwoFactorCommand.buildOTP({ secret: otpSecret }).generate(), + requestToken + }) + }) + + it('Should not add the header on login if two factor is enabled and password is incorrect', async function () { + const { res, token } = await login({ server, username: userUsername, password: 'fake', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + + expect(res.header['x-peertube-otp']).to.not.exist + expect(token).to.not.exist + }) + + it('Should add the header on login if two factor is enabled and password is correct', async function () { + const { res, token } = await login({ + server, + username: userUsername, + password: userPassword, + expectedStatus: HttpStatusCode.UNAUTHORIZED_401 + }) + + expect(res.header['x-peertube-otp']).to.exist + expect(token).to.not.exist + + await server.users.getMyInfo({ token }) + }) + + it('Should not login with correct password and incorrect otp secret', async function () { + const otp = TwoFactorCommand.buildOTP({ secret: 'a'.repeat(32) }) + + const { res, token } = await login({ + server, + username: userUsername, + password: userPassword, + otpToken: otp.generate(), + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + + expect(res.header['x-peertube-otp']).to.not.exist + expect(token).to.not.exist + }) + + it('Should not login with correct password and incorrect otp code', async function () { + const { res, token } = await login({ + server, + username: userUsername, + password: userPassword, + otpToken: '123456', + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + + expect(res.header['x-peertube-otp']).to.not.exist + expect(token).to.not.exist + }) + + it('Should not login with incorrect password and correct otp code', async function () { + const otpToken = TwoFactorCommand.buildOTP({ secret: otpSecret }).generate() + + const { res, token } = await login({ + server, + username: userUsername, + password: 'fake', + otpToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + + expect(res.header['x-peertube-otp']).to.not.exist + expect(token).to.not.exist + }) + + it('Should correctly login with correct password and otp code', async function () { + const otpToken = TwoFactorCommand.buildOTP({ secret: otpSecret }).generate() + + const { res, token } = await login({ server, username: userUsername, password: userPassword, otpToken }) + + expect(res.header['x-peertube-otp']).to.not.exist + expect(token).to.exist + + await server.users.getMyInfo({ token }) + }) + + it('Should have two factor enabled when getting my info', async function () { + const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken }) + expect(twoFactorEnabled).to.be.true + }) + + it('Should disable two factor and be able to login without otp token', async function () { + await server.twoFactor.disable({ userId, token: userToken, currentPassword: userPassword }) + + const { res, token } = await login({ server, username: userUsername, password: userPassword }) + expect(res.header['x-peertube-otp']).to.not.exist + + await server.users.getMyInfo({ token }) + }) + + it('Should have two factor disabled when getting my info', async function () { + const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken }) + expect(twoFactorEnabled).to.be.false + }) + + it('Should enable two factor auth without password from an admin', async function () { + const { otpRequest } = await server.twoFactor.request({ userId }) + + await server.twoFactor.confirmRequest({ + userId, + otpToken: TwoFactorCommand.buildOTP({ secret: otpRequest.secret }).generate(), + requestToken: otpRequest.requestToken + }) + + const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken }) + expect(twoFactorEnabled).to.be.true + }) + + it('Should disable two factor auth without password from an admin', async function () { + await server.twoFactor.disable({ userId }) + + const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken }) + expect(twoFactorEnabled).to.be.false + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/users/user-subscriptions.ts b/packages/tests/src/api/users/user-subscriptions.ts new file mode 100644 index 000000000..eb4ea9539 --- /dev/null +++ b/packages/tests/src/api/users/user-subscriptions.ts @@ -0,0 +1,614 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultChannelAvatar, + SubscriptionsCommand, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test users subscriptions', function () { + let servers: PeerTubeServer[] = [] + const users: { accessToken: string }[] = [] + let video3UUID: string + + let command: SubscriptionsCommand + + before(async function () { + this.timeout(240000) + + servers = await createMultipleServers(3) + + // Get the access tokens + await setAccessTokensToServers(servers) + await setDefaultChannelAvatar(servers) + await setDefaultAccountAvatar(servers) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + + for (const server of servers) { + const user = { username: 'user' + server.serverNumber, password: 'password' } + await server.users.create({ username: user.username, password: user.password }) + + const accessToken = await server.login.getAccessToken(user) + users.push({ accessToken }) + + const videoName1 = 'video 1-' + server.serverNumber + await server.videos.upload({ token: accessToken, attributes: { name: videoName1 } }) + + const videoName2 = 'video 2-' + server.serverNumber + await server.videos.upload({ token: accessToken, attributes: { name: videoName2 } }) + } + + await waitJobs(servers) + + command = servers[0].subscriptions + }) + + describe('Destinction between server videos and user videos', function () { + it('Should display videos of server 2 on server 1', async function () { + const { total } = await servers[0].videos.list() + + expect(total).to.equal(4) + }) + + it('User of server 1 should follow user of server 3 and root of server 1', async function () { + this.timeout(60000) + + await command.add({ token: users[0].accessToken, targetUri: 'user3_channel@' + servers[2].host }) + await command.add({ token: users[0].accessToken, targetUri: 'root_channel@' + servers[0].host }) + + await waitJobs(servers) + + const attributes = { name: 'video server 3 added after follow' } + const { uuid } = await servers[2].videos.upload({ token: users[2].accessToken, attributes }) + video3UUID = uuid + + await waitJobs(servers) + }) + + it('Should not display videos of server 3 on server 1', async function () { + const { total, data } = await servers[0].videos.list() + expect(total).to.equal(4) + + for (const video of data) { + expect(video.name).to.not.contain('1-3') + expect(video.name).to.not.contain('2-3') + expect(video.name).to.not.contain('video server 3 added after follow') + } + }) + }) + + describe('Subscription endpoints', function () { + + it('Should list subscriptions', async function () { + { + const body = await command.list() + expect(body.total).to.equal(0) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(0) + } + + { + const body = await command.list({ token: users[0].accessToken, sort: 'createdAt' }) + expect(body.total).to.equal(2) + + const subscriptions = body.data + expect(subscriptions).to.be.an('array') + expect(subscriptions).to.have.lengthOf(2) + + expect(subscriptions[0].name).to.equal('user3_channel') + expect(subscriptions[1].name).to.equal('root_channel') + } + }) + + it('Should get subscription', async function () { + { + const videoChannel = await command.get({ token: users[0].accessToken, uri: 'user3_channel@' + servers[2].host }) + + expect(videoChannel.name).to.equal('user3_channel') + expect(videoChannel.host).to.equal(servers[2].host) + expect(videoChannel.displayName).to.equal('Main user3 channel') + expect(videoChannel.followingCount).to.equal(0) + expect(videoChannel.followersCount).to.equal(1) + } + + { + const videoChannel = await command.get({ token: users[0].accessToken, uri: 'root_channel@' + servers[0].host }) + + expect(videoChannel.name).to.equal('root_channel') + expect(videoChannel.host).to.equal(servers[0].host) + expect(videoChannel.displayName).to.equal('Main root channel') + expect(videoChannel.followingCount).to.equal(0) + expect(videoChannel.followersCount).to.equal(1) + } + }) + + it('Should return the existing subscriptions', async function () { + const uris = [ + 'user3_channel@' + servers[2].host, + 'root2_channel@' + servers[0].host, + 'root_channel@' + servers[0].host, + 'user3_channel@' + servers[0].host + ] + + const body = await command.exist({ token: users[0].accessToken, uris }) + + expect(body['user3_channel@' + servers[2].host]).to.be.true + expect(body['root2_channel@' + servers[0].host]).to.be.false + expect(body['root_channel@' + servers[0].host]).to.be.true + expect(body['user3_channel@' + servers[0].host]).to.be.false + }) + + it('Should search among subscriptions', async function () { + { + const body = await command.list({ token: users[0].accessToken, sort: '-createdAt', search: 'user3_channel' }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + } + + { + const body = await command.list({ token: users[0].accessToken, sort: '-createdAt', search: 'toto' }) + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + } + }) + }) + + describe('Subscription videos', function () { + + it('Should list subscription videos', async function () { + { + const body = await servers[0].videos.listMySubscriptionVideos() + expect(body.total).to.equal(0) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(0) + } + + { + const body = await servers[0].videos.listMySubscriptionVideos({ token: users[0].accessToken, sort: 'createdAt' }) + expect(body.total).to.equal(3) + + const videos = body.data + expect(videos).to.be.an('array') + expect(videos).to.have.lengthOf(3) + + expect(videos[0].name).to.equal('video 1-3') + expect(videos[1].name).to.equal('video 2-3') + expect(videos[2].name).to.equal('video server 3 added after follow') + } + + { + const body = await servers[0].videos.listMySubscriptionVideos({ token: users[0].accessToken, count: 1, start: 1 }) + expect(body.total).to.equal(3) + + const videos = body.data + expect(videos).to.be.an('array') + expect(videos).to.have.lengthOf(1) + + expect(videos[0].name).to.equal('video 2-3') + } + }) + + it('Should upload a video by root on server 1 and see it in the subscription videos', async function () { + this.timeout(60000) + + const videoName = 'video server 1 added after follow' + await servers[0].videos.upload({ attributes: { name: videoName } }) + + await waitJobs(servers) + + { + const body = await servers[0].videos.listMySubscriptionVideos() + expect(body.total).to.equal(0) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(0) + } + + { + const body = await servers[0].videos.listMySubscriptionVideos({ token: users[0].accessToken, sort: 'createdAt' }) + expect(body.total).to.equal(4) + + const videos = body.data + expect(videos).to.be.an('array') + expect(videos).to.have.lengthOf(4) + + expect(videos[0].name).to.equal('video 1-3') + expect(videos[1].name).to.equal('video 2-3') + expect(videos[2].name).to.equal('video server 3 added after follow') + expect(videos[3].name).to.equal('video server 1 added after follow') + } + + { + const { data, total } = await servers[0].videos.list() + expect(total).to.equal(5) + + for (const video of data) { + expect(video.name).to.not.contain('1-3') + expect(video.name).to.not.contain('2-3') + expect(video.name).to.not.contain('video server 3 added after follow') + } + } + }) + + it('Should have server 1 following server 3 and display server 3 videos', async function () { + this.timeout(60000) + + await servers[0].follows.follow({ hosts: [ servers[2].url ] }) + + await waitJobs(servers) + + const { data, total } = await servers[0].videos.list() + expect(total).to.equal(8) + + const names = [ '1-3', '2-3', 'video server 3 added after follow' ] + for (const name of names) { + const video = data.find(v => v.name.includes(name)) + expect(video).to.not.be.undefined + } + }) + + it('Should remove follow server 1 -> server 3 and hide server 3 videos', async function () { + this.timeout(60000) + + await servers[0].follows.unfollow({ target: servers[2] }) + + await waitJobs(servers) + + const { total, data } = await servers[0].videos.list() + expect(total).to.equal(5) + + for (const video of data) { + expect(video.name).to.not.contain('1-3') + expect(video.name).to.not.contain('2-3') + expect(video.name).to.not.contain('video server 3 added after follow') + } + }) + + it('Should still list subscription videos', async function () { + { + const body = await servers[0].videos.listMySubscriptionVideos() + expect(body.total).to.equal(0) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(0) + } + + { + const body = await servers[0].videos.listMySubscriptionVideos({ token: users[0].accessToken, sort: 'createdAt' }) + expect(body.total).to.equal(4) + + const videos = body.data + expect(videos).to.be.an('array') + expect(videos).to.have.lengthOf(4) + + expect(videos[0].name).to.equal('video 1-3') + expect(videos[1].name).to.equal('video 2-3') + expect(videos[2].name).to.equal('video server 3 added after follow') + expect(videos[3].name).to.equal('video server 1 added after follow') + } + }) + }) + + describe('Existing subscription video update', function () { + + it('Should update a video of server 3 and see the updated video on server 1', async function () { + this.timeout(30000) + + await servers[2].videos.update({ id: video3UUID, attributes: { name: 'video server 3 added after follow updated' } }) + + await waitJobs(servers) + + const body = await servers[0].videos.listMySubscriptionVideos({ token: users[0].accessToken, sort: 'createdAt' }) + expect(body.data[2].name).to.equal('video server 3 added after follow updated') + }) + }) + + describe('Subscription removal', function () { + + it('Should remove user of server 3 subscription', async function () { + this.timeout(30000) + + await command.remove({ token: users[0].accessToken, uri: 'user3_channel@' + servers[2].host }) + + await waitJobs(servers) + }) + + it('Should not display its videos anymore', async function () { + const body = await servers[0].videos.listMySubscriptionVideos({ token: users[0].accessToken, sort: 'createdAt' }) + expect(body.total).to.equal(1) + + const videos = body.data + expect(videos).to.be.an('array') + expect(videos).to.have.lengthOf(1) + + expect(videos[0].name).to.equal('video server 1 added after follow') + }) + + it('Should remove the root subscription and not display the videos anymore', async function () { + this.timeout(30000) + + await command.remove({ token: users[0].accessToken, uri: 'root_channel@' + servers[0].host }) + + await waitJobs(servers) + + { + const body = await command.list({ token: users[0].accessToken, sort: 'createdAt' }) + expect(body.total).to.equal(0) + + const videos = body.data + expect(videos).to.be.an('array') + expect(videos).to.have.lengthOf(0) + } + }) + + it('Should correctly display public videos on server 1', async function () { + const { total, data } = await servers[0].videos.list() + expect(total).to.equal(5) + + for (const video of data) { + expect(video.name).to.not.contain('1-3') + expect(video.name).to.not.contain('2-3') + expect(video.name).to.not.contain('video server 3 added after follow updated') + } + }) + }) + + describe('Re-follow', function () { + + it('Should follow user of server 3 again', async function () { + this.timeout(60000) + + await command.add({ token: users[0].accessToken, targetUri: 'user3_channel@' + servers[2].host }) + + await waitJobs(servers) + + { + const body = await servers[0].videos.listMySubscriptionVideos({ token: users[0].accessToken, sort: 'createdAt' }) + expect(body.total).to.equal(3) + + const videos = body.data + expect(videos).to.be.an('array') + expect(videos).to.have.lengthOf(3) + + expect(videos[0].name).to.equal('video 1-3') + expect(videos[1].name).to.equal('video 2-3') + expect(videos[2].name).to.equal('video server 3 added after follow updated') + } + + { + const { total, data } = await servers[0].videos.list() + expect(total).to.equal(5) + + for (const video of data) { + expect(video.name).to.not.contain('1-3') + expect(video.name).to.not.contain('2-3') + expect(video.name).to.not.contain('video server 3 added after follow updated') + } + } + }) + + it('Should follow user channels of server 3 by root of server 3', async function () { + this.timeout(60000) + + await servers[2].channels.create({ token: users[2].accessToken, attributes: { name: 'user3_channel2' } }) + + await servers[2].subscriptions.add({ token: servers[2].accessToken, targetUri: 'user3_channel@' + servers[2].host }) + await servers[2].subscriptions.add({ token: servers[2].accessToken, targetUri: 'user3_channel2@' + servers[2].host }) + + await waitJobs(servers) + }) + }) + + describe('Followers listing', function () { + + it('Should list user 3 followers', async function () { + { + const { total, data } = await servers[2].accounts.listFollowers({ + token: users[2].accessToken, + accountName: 'user3', + start: 0, + count: 5, + sort: 'createdAt' + }) + + expect(total).to.equal(3) + expect(data).to.have.lengthOf(3) + + expect(data[0].following.host).to.equal(servers[2].host) + expect(data[0].following.name).to.equal('user3_channel') + expect(data[0].follower.host).to.equal(servers[0].host) + expect(data[0].follower.name).to.equal('user1') + + expect(data[1].following.host).to.equal(servers[2].host) + expect(data[1].following.name).to.equal('user3_channel') + expect(data[1].follower.host).to.equal(servers[2].host) + expect(data[1].follower.name).to.equal('root') + + expect(data[2].following.host).to.equal(servers[2].host) + expect(data[2].following.name).to.equal('user3_channel2') + expect(data[2].follower.host).to.equal(servers[2].host) + expect(data[2].follower.name).to.equal('root') + } + + { + const { total, data } = await servers[2].accounts.listFollowers({ + token: users[2].accessToken, + accountName: 'user3', + start: 0, + count: 1, + sort: '-createdAt' + }) + + expect(total).to.equal(3) + expect(data).to.have.lengthOf(1) + + expect(data[0].following.host).to.equal(servers[2].host) + expect(data[0].following.name).to.equal('user3_channel2') + expect(data[0].follower.host).to.equal(servers[2].host) + expect(data[0].follower.name).to.equal('root') + } + + { + const { total, data } = await servers[2].accounts.listFollowers({ + token: users[2].accessToken, + accountName: 'user3', + start: 1, + count: 1, + sort: '-createdAt' + }) + + expect(total).to.equal(3) + expect(data).to.have.lengthOf(1) + + expect(data[0].following.host).to.equal(servers[2].host) + expect(data[0].following.name).to.equal('user3_channel') + expect(data[0].follower.host).to.equal(servers[2].host) + expect(data[0].follower.name).to.equal('root') + } + + { + const { total, data } = await servers[2].accounts.listFollowers({ + token: users[2].accessToken, + accountName: 'user3', + search: 'user1', + sort: '-createdAt' + }) + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + + expect(data[0].following.host).to.equal(servers[2].host) + expect(data[0].following.name).to.equal('user3_channel') + expect(data[0].follower.host).to.equal(servers[0].host) + expect(data[0].follower.name).to.equal('user1') + } + }) + + it('Should list user3_channel followers', async function () { + { + const { total, data } = await servers[2].channels.listFollowers({ + token: users[2].accessToken, + channelName: 'user3_channel', + start: 0, + count: 5, + sort: 'createdAt' + }) + + expect(total).to.equal(2) + expect(data).to.have.lengthOf(2) + + expect(data[0].following.host).to.equal(servers[2].host) + expect(data[0].following.name).to.equal('user3_channel') + expect(data[0].follower.host).to.equal(servers[0].host) + expect(data[0].follower.name).to.equal('user1') + + expect(data[1].following.host).to.equal(servers[2].host) + expect(data[1].following.name).to.equal('user3_channel') + expect(data[1].follower.host).to.equal(servers[2].host) + expect(data[1].follower.name).to.equal('root') + } + + { + const { total, data } = await servers[2].channels.listFollowers({ + token: users[2].accessToken, + channelName: 'user3_channel', + start: 0, + count: 1, + sort: '-createdAt' + }) + + expect(total).to.equal(2) + expect(data).to.have.lengthOf(1) + + expect(data[0].following.host).to.equal(servers[2].host) + expect(data[0].following.name).to.equal('user3_channel') + expect(data[0].follower.host).to.equal(servers[2].host) + expect(data[0].follower.name).to.equal('root') + } + + { + const { total, data } = await servers[2].channels.listFollowers({ + token: users[2].accessToken, + channelName: 'user3_channel', + start: 1, + count: 1, + sort: '-createdAt' + }) + + expect(total).to.equal(2) + expect(data).to.have.lengthOf(1) + + expect(data[0].following.host).to.equal(servers[2].host) + expect(data[0].following.name).to.equal('user3_channel') + expect(data[0].follower.host).to.equal(servers[0].host) + expect(data[0].follower.name).to.equal('user1') + } + + { + const { total, data } = await servers[2].channels.listFollowers({ + token: users[2].accessToken, + channelName: 'user3_channel', + search: 'user1', + sort: '-createdAt' + }) + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + + expect(data[0].following.host).to.equal(servers[2].host) + expect(data[0].following.name).to.equal('user3_channel') + expect(data[0].follower.host).to.equal(servers[0].host) + expect(data[0].follower.name).to.equal('user1') + } + }) + }) + + describe('Subscription videos privacy', function () { + + it('Should update video as internal and not see from remote server', async function () { + this.timeout(30000) + + await servers[2].videos.update({ id: video3UUID, attributes: { name: 'internal', privacy: VideoPrivacy.INTERNAL } }) + await waitJobs(servers) + + { + const { data } = await servers[0].videos.listMySubscriptionVideos({ token: users[0].accessToken }) + expect(data.find(v => v.name === 'internal')).to.not.exist + } + }) + + it('Should see internal from local user', async function () { + const { data } = await servers[2].videos.listMySubscriptionVideos({ token: servers[2].accessToken }) + expect(data.find(v => v.name === 'internal')).to.exist + }) + + it('Should update video as private and not see from anyone server', async function () { + this.timeout(30000) + + await servers[2].videos.update({ id: video3UUID, attributes: { name: 'private', privacy: VideoPrivacy.PRIVATE } }) + await waitJobs(servers) + + { + const { data } = await servers[0].videos.listMySubscriptionVideos({ token: users[0].accessToken }) + expect(data.find(v => v.name === 'private')).to.not.exist + } + + { + const { data } = await servers[2].videos.listMySubscriptionVideos({ token: servers[2].accessToken }) + expect(data.find(v => v.name === 'private')).to.not.exist + } + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/users/user-videos.ts b/packages/tests/src/api/users/user-videos.ts new file mode 100644 index 000000000..7b075d040 --- /dev/null +++ b/packages/tests/src/api/users/user-videos.ts @@ -0,0 +1,219 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + PeerTubeServer, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultChannelAvatar, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test user videos', function () { + let server: PeerTubeServer + let videoId: number + let videoId2: number + let token: string + let anotherUserToken: string + + before(async function () { + this.timeout(120000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + await setDefaultChannelAvatar([ server ]) + await setDefaultAccountAvatar([ server ]) + + await server.videos.quickUpload({ name: 'root video' }) + await server.videos.quickUpload({ name: 'root video 2' }) + + token = await server.users.generateUserAndToken('user') + anotherUserToken = await server.users.generateUserAndToken('user2') + }) + + describe('List my videos', function () { + + it('Should list my videos', async function () { + const { data, total } = await server.videos.listMyVideos() + + expect(total).to.equal(2) + expect(data).to.have.lengthOf(2) + }) + }) + + describe('Upload', function () { + + it('Should upload the video with the correct token', async function () { + await server.videos.upload({ token }) + const { data } = await server.videos.list() + const video = data[0] + + expect(video.account.name).to.equal('user') + videoId = video.id + }) + + it('Should upload the video again with the correct token', async function () { + const { id } = await server.videos.upload({ token }) + videoId2 = id + }) + }) + + describe('Ratings', function () { + + it('Should retrieve a video rating', async function () { + await server.videos.rate({ id: videoId, token, rating: 'like' }) + const rating = await server.users.getMyRating({ token, videoId }) + + expect(rating.videoId).to.equal(videoId) + expect(rating.rating).to.equal('like') + }) + + it('Should retrieve ratings list', async function () { + await server.videos.rate({ id: videoId, token, rating: 'like' }) + + const body = await server.accounts.listRatings({ accountName: 'user', token }) + + expect(body.total).to.equal(1) + expect(body.data[0].video.id).to.equal(videoId) + expect(body.data[0].rating).to.equal('like') + }) + + it('Should retrieve ratings list by rating type', async function () { + { + const body = await server.accounts.listRatings({ accountName: 'user', token, rating: 'like' }) + expect(body.data.length).to.equal(1) + } + + { + const body = await server.accounts.listRatings({ accountName: 'user', token, rating: 'dislike' }) + expect(body.data.length).to.equal(0) + } + }) + }) + + describe('Remove video', function () { + + it('Should not be able to remove the video with an incorrect token', async function () { + await server.videos.remove({ token: 'bad_token', id: videoId, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should not be able to remove the video with the token of another account', async function () { + await server.videos.remove({ token: anotherUserToken, id: videoId, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should be able to remove the video with the correct token', async function () { + await server.videos.remove({ token, id: videoId }) + await server.videos.remove({ token, id: videoId2 }) + }) + }) + + describe('My videos & quotas', function () { + + it('Should be able to upload a video with a user', async function () { + this.timeout(30000) + + const attributes = { + name: 'super user video', + fixture: 'video_short.webm' + } + await server.videos.upload({ token, attributes }) + + await server.channels.create({ token, attributes: { name: 'other_channel' } }) + }) + + it('Should have video quota updated', async function () { + const quota = await server.users.getMyQuotaUsed({ token }) + expect(quota.videoQuotaUsed).to.equal(218910) + expect(quota.videoQuotaUsedDaily).to.equal(218910) + + const { data } = await server.users.list() + const tmpUser = data.find(u => u.username === 'user') + expect(tmpUser.videoQuotaUsed).to.equal(218910) + expect(tmpUser.videoQuotaUsedDaily).to.equal(218910) + }) + + it('Should be able to list my videos', async function () { + const { total, data } = await server.videos.listMyVideos({ token }) + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + + const video = data[0] + expect(video.name).to.equal('super user video') + expect(video.thumbnailPath).to.not.be.null + expect(video.previewPath).to.not.be.null + }) + + it('Should be able to filter by channel in my videos', async function () { + const myInfo = await server.users.getMyInfo({ token }) + const mainChannel = myInfo.videoChannels.find(c => c.name !== 'other_channel') + const otherChannel = myInfo.videoChannels.find(c => c.name === 'other_channel') + + { + const { total, data } = await server.videos.listMyVideos({ token, channelId: mainChannel.id }) + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + + const video = data[0] + expect(video.name).to.equal('super user video') + expect(video.thumbnailPath).to.not.be.null + expect(video.previewPath).to.not.be.null + } + + { + const { total, data } = await server.videos.listMyVideos({ token, channelId: otherChannel.id }) + expect(total).to.equal(0) + expect(data).to.have.lengthOf(0) + } + }) + + it('Should be able to search in my videos', async function () { + { + const { total, data } = await server.videos.listMyVideos({ token, sort: '-createdAt', search: 'user video' }) + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + } + + { + const { total, data } = await server.videos.listMyVideos({ token, sort: '-createdAt', search: 'toto' }) + expect(total).to.equal(0) + expect(data).to.have.lengthOf(0) + } + }) + + it('Should disable web videos, enable HLS, and update my quota', async function () { + this.timeout(160000) + + { + const config = await server.config.getCustomConfig() + config.transcoding.webVideos.enabled = false + config.transcoding.hls.enabled = true + config.transcoding.enabled = true + await server.config.updateCustomSubConfig({ newConfig: config }) + } + + { + const attributes = { + name: 'super user video 2', + fixture: 'video_short.webm' + } + await server.videos.upload({ token, attributes }) + + await waitJobs([ server ]) + } + + { + const data = await server.users.getMyQuotaUsed({ token }) + expect(data.videoQuotaUsed).to.be.greaterThan(220000) + expect(data.videoQuotaUsedDaily).to.be.greaterThan(220000) + } + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/users/users-email-verification.ts b/packages/tests/src/api/users/users-email-verification.ts new file mode 100644 index 000000000..689e3c4bb --- /dev/null +++ b/packages/tests/src/api/users/users-email-verification.ts @@ -0,0 +1,165 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { MockSmtpServer } from '@tests/shared/mock-servers/index.js' +import { HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + ConfigCommand, + createSingleServer, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test users email verification', function () { + let server: PeerTubeServer + let userId: number + let userAccessToken: string + let verificationString: string + let expectedEmailsLength = 0 + const user1 = { + username: 'user_1', + password: 'super password' + } + const user2 = { + username: 'user_2', + password: 'super password' + } + const emails: object[] = [] + + before(async function () { + this.timeout(30000) + + const port = await MockSmtpServer.Instance.collectEmails(emails) + server = await createSingleServer(1, ConfigCommand.getEmailOverrideConfig(port)) + + await setAccessTokensToServers([ server ]) + }) + + it('Should register user and send verification email if verification required', async function () { + this.timeout(30000) + + await server.config.updateExistingSubConfig({ + newConfig: { + signup: { + enabled: true, + requiresApproval: false, + requiresEmailVerification: true, + limit: 10 + } + } + }) + + await server.registrations.register(user1) + + await waitJobs(server) + expectedEmailsLength++ + expect(emails).to.have.lengthOf(expectedEmailsLength) + + const email = emails[expectedEmailsLength - 1] + + const verificationStringMatches = /verificationString=([a-z0-9]+)/.exec(email['text']) + expect(verificationStringMatches).not.to.be.null + + verificationString = verificationStringMatches[1] + expect(verificationString).to.have.length.above(2) + + const userIdMatches = /userId=([0-9]+)/.exec(email['text']) + expect(userIdMatches).not.to.be.null + + userId = parseInt(userIdMatches[1], 10) + + const body = await server.users.get({ userId }) + expect(body.emailVerified).to.be.false + }) + + it('Should not allow login for user with unverified email', async function () { + const { detail } = await server.login.login({ user: user1, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + expect(detail).to.contain('User email is not verified.') + }) + + it('Should verify the user via email and allow login', async function () { + await server.users.verifyEmail({ userId, verificationString }) + + const body = await server.login.login({ user: user1 }) + userAccessToken = body.access_token + + const user = await server.users.get({ userId }) + expect(user.emailVerified).to.be.true + }) + + it('Should be able to change the user email', async function () { + let updateVerificationString: string + + { + await server.users.updateMe({ + token: userAccessToken, + email: 'updated@example.com', + currentPassword: user1.password + }) + + await waitJobs(server) + expectedEmailsLength++ + expect(emails).to.have.lengthOf(expectedEmailsLength) + + const email = emails[expectedEmailsLength - 1] + + const verificationStringMatches = /verificationString=([a-z0-9]+)/.exec(email['text']) + updateVerificationString = verificationStringMatches[1] + } + + { + const me = await server.users.getMyInfo({ token: userAccessToken }) + expect(me.email).to.equal('user_1@example.com') + expect(me.pendingEmail).to.equal('updated@example.com') + } + + { + await server.users.verifyEmail({ userId, verificationString: updateVerificationString, isPendingEmail: true }) + + const me = await server.users.getMyInfo({ token: userAccessToken }) + expect(me.email).to.equal('updated@example.com') + expect(me.pendingEmail).to.be.null + } + }) + + it('Should register user not requiring email verification if setting not enabled', async function () { + this.timeout(5000) + await server.config.updateExistingSubConfig({ + newConfig: { + signup: { + requiresEmailVerification: false + } + } + }) + + await server.registrations.register(user2) + + await waitJobs(server) + expect(emails).to.have.lengthOf(expectedEmailsLength) + + const accessToken = await server.login.getAccessToken(user2) + + const user = await server.users.getMyInfo({ token: accessToken }) + expect(user.emailVerified).to.be.null + }) + + it('Should allow login for user with unverified email when setting later enabled', async function () { + await server.config.updateCustomSubConfig({ + newConfig: { + signup: { + requiresEmailVerification: true + } + } + }) + + await server.login.getAccessToken(user2) + }) + + after(async function () { + MockSmtpServer.Instance.kill() + + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/users/users-multiple-servers.ts b/packages/tests/src/api/users/users-multiple-servers.ts new file mode 100644 index 000000000..61e3aa001 --- /dev/null +++ b/packages/tests/src/api/users/users-multiple-servers.ts @@ -0,0 +1,213 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { MyUser } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + setDefaultChannelAvatar, + waitJobs +} from '@peertube/peertube-server-commands' +import { checkActorFilesWereRemoved } from '@tests/shared/actors.js' +import { testImage } from '@tests/shared/checks.js' +import { checkTmpIsEmpty } from '@tests/shared/directories.js' +import { saveVideoInServers, checkVideoFilesWereRemoved } from '@tests/shared/videos.js' + +describe('Test users with multiple servers', function () { + let servers: PeerTubeServer[] = [] + + let user: MyUser + let userId: number + + let videoUUID: string + let userAccessToken: string + let userAvatarFilenames: string[] + + before(async function () { + this.timeout(120_000) + + servers = await createMultipleServers(3) + + // Get the access tokens + await setAccessTokensToServers(servers) + await setDefaultChannelAvatar(servers) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + // Server 1 and server 3 follow each other + await doubleFollow(servers[0], servers[2]) + // Server 2 and server 3 follow each other + await doubleFollow(servers[1], servers[2]) + + // The root user of server 1 is propagated to servers 2 and 3 + await servers[0].videos.upload() + + { + const username = 'user1' + const created = await servers[0].users.create({ username }) + userId = created.id + userAccessToken = await servers[0].login.getAccessToken(username) + } + + { + const { uuid } = await servers[0].videos.upload({ token: userAccessToken }) + videoUUID = uuid + + await waitJobs(servers) + + await saveVideoInServers(servers, videoUUID) + } + }) + + it('Should be able to update my display name', async function () { + await servers[0].users.updateMe({ displayName: 'my super display name' }) + + user = await servers[0].users.getMyInfo() + expect(user.account.displayName).to.equal('my super display name') + + await waitJobs(servers) + }) + + it('Should be able to update my description', async function () { + this.timeout(10_000) + + await servers[0].users.updateMe({ description: 'my super description updated' }) + + user = await servers[0].users.getMyInfo() + expect(user.account.displayName).to.equal('my super display name') + expect(user.account.description).to.equal('my super description updated') + + await waitJobs(servers) + }) + + it('Should be able to update my avatar', async function () { + this.timeout(10_000) + + const fixture = 'avatar2.png' + + await servers[0].users.updateMyAvatar({ fixture }) + + user = await servers[0].users.getMyInfo() + userAvatarFilenames = user.account.avatars.map(({ path }) => path) + + for (const avatar of user.account.avatars) { + await testImage(servers[0].url, `avatar2-resized-${avatar.width}x${avatar.width}`, avatar.path, '.png') + } + + await waitJobs(servers) + }) + + it('Should have updated my profile on other servers too', async function () { + let createdAt: string | Date + + for (const server of servers) { + const body = await server.accounts.list({ sort: '-createdAt' }) + + const resList = body.data.find(a => a.name === 'root' && a.host === servers[0].host) + expect(resList).not.to.be.undefined + + const account = await server.accounts.get({ accountName: resList.name + '@' + resList.host }) + + if (!createdAt) createdAt = account.createdAt + + expect(account.name).to.equal('root') + expect(account.host).to.equal(servers[0].host) + expect(account.displayName).to.equal('my super display name') + expect(account.description).to.equal('my super description updated') + expect(createdAt).to.equal(account.createdAt) + + if (server.serverNumber === 1) { + expect(account.userId).to.be.a('number') + } else { + expect(account.userId).to.be.undefined + } + + for (const avatar of account.avatars) { + await testImage(server.url, `avatar2-resized-${avatar.width}x${avatar.width}`, avatar.path, '.png') + } + } + }) + + it('Should list account videos', async function () { + for (const server of servers) { + const { total, data } = await server.videos.listByAccount({ handle: 'user1@' + servers[0].host }) + + expect(total).to.equal(1) + expect(data).to.be.an('array') + expect(data).to.have.lengthOf(1) + expect(data[0].uuid).to.equal(videoUUID) + } + }) + + it('Should search through account videos', async function () { + const created = await servers[0].videos.upload({ token: userAccessToken, attributes: { name: 'Kami no chikara' } }) + + await waitJobs(servers) + + for (const server of servers) { + const { total, data } = await server.videos.listByAccount({ handle: 'user1@' + servers[0].host, search: 'Kami' }) + + expect(total).to.equal(1) + expect(data).to.be.an('array') + expect(data).to.have.lengthOf(1) + expect(data[0].uuid).to.equal(created.uuid) + } + }) + + it('Should remove the user', async function () { + this.timeout(10_000) + + for (const server of servers) { + const body = await server.accounts.list({ sort: '-createdAt' }) + + const accountDeleted = body.data.find(a => a.name === 'user1' && a.host === servers[0].host) + expect(accountDeleted).not.to.be.undefined + + const { data } = await server.channels.list() + const videoChannelDeleted = data.find(a => a.displayName === 'Main user1 channel' && a.host === servers[0].host) + expect(videoChannelDeleted).not.to.be.undefined + } + + await servers[0].users.remove({ userId }) + + await waitJobs(servers) + + for (const server of servers) { + const body = await server.accounts.list({ sort: '-createdAt' }) + + const accountDeleted = body.data.find(a => a.name === 'user1' && a.host === servers[0].host) + expect(accountDeleted).to.be.undefined + + const { data } = await server.channels.list() + const videoChannelDeleted = data.find(a => a.name === 'Main user1 channel' && a.host === servers[0].host) + expect(videoChannelDeleted).to.be.undefined + } + }) + + it('Should not have actor files', async () => { + for (const server of servers) { + for (const userAvatarFilename of userAvatarFilenames) { + await checkActorFilesWereRemoved(userAvatarFilename, server) + } + } + }) + + it('Should not have video files', async () => { + for (const server of servers) { + await checkVideoFilesWereRemoved({ server, video: server.store.videoDetails }) + } + }) + + it('Should have an empty tmp directory', async function () { + for (const server of servers) { + await checkTmpIsEmpty(server) + } + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/users/users.ts b/packages/tests/src/api/users/users.ts new file mode 100644 index 000000000..a0090a463 --- /dev/null +++ b/packages/tests/src/api/users/users.ts @@ -0,0 +1,529 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { testImageSize } from '@tests/shared/checks.js' +import { AbuseState, HttpStatusCode, UserAdminFlag, UserRole, VideoPlaylistType } from '@peertube/peertube-models' +import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@peertube/peertube-server-commands' + +describe('Test users', function () { + let server: PeerTubeServer + let token: string + let userToken: string + let videoId: number + let userId: number + const user = { + username: 'user_1', + password: 'super password' + } + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1, { + rates_limit: { + login: { + max: 30 + } + } + }) + + await setAccessTokensToServers([ server ]) + + await server.plugins.install({ npmName: 'peertube-theme-background-red' }) + }) + + describe('Creating a user', function () { + + it('Should be able to create a new user', async function () { + await server.users.create({ ...user, videoQuota: 2 * 1024 * 1024, adminFlags: UserAdminFlag.BYPASS_VIDEO_AUTO_BLACKLIST }) + }) + + it('Should be able to login with this user', async function () { + userToken = await server.login.getAccessToken(user) + }) + + it('Should be able to get user information', async function () { + const userMe = await server.users.getMyInfo({ token: userToken }) + + const userGet = await server.users.get({ userId: userMe.id, withStats: true }) + + for (const user of [ userMe, userGet ]) { + expect(user.username).to.equal('user_1') + expect(user.email).to.equal('user_1@example.com') + expect(user.nsfwPolicy).to.equal('display') + expect(user.videoQuota).to.equal(2 * 1024 * 1024) + expect(user.role.label).to.equal('User') + expect(user.id).to.be.a('number') + expect(user.account.displayName).to.equal('user_1') + expect(user.account.description).to.be.null + } + + expect(userMe.adminFlags).to.equal(UserAdminFlag.BYPASS_VIDEO_AUTO_BLACKLIST) + expect(userGet.adminFlags).to.equal(UserAdminFlag.BYPASS_VIDEO_AUTO_BLACKLIST) + + expect(userMe.specialPlaylists).to.have.lengthOf(1) + expect(userMe.specialPlaylists[0].type).to.equal(VideoPlaylistType.WATCH_LATER) + + // Check stats are included with withStats + expect(userGet.videosCount).to.be.a('number') + expect(userGet.videosCount).to.equal(0) + expect(userGet.videoCommentsCount).to.be.a('number') + expect(userGet.videoCommentsCount).to.equal(0) + expect(userGet.abusesCount).to.be.a('number') + expect(userGet.abusesCount).to.equal(0) + expect(userGet.abusesAcceptedCount).to.be.a('number') + expect(userGet.abusesAcceptedCount).to.equal(0) + }) + }) + + describe('Users listing', function () { + + it('Should list all the users', async function () { + const { data, total } = await server.users.list() + + expect(total).to.equal(2) + expect(data).to.be.an('array') + expect(data.length).to.equal(2) + + const user = data[0] + expect(user.username).to.equal('user_1') + expect(user.email).to.equal('user_1@example.com') + expect(user.nsfwPolicy).to.equal('display') + + const rootUser = data[1] + expect(rootUser.username).to.equal('root') + expect(rootUser.email).to.equal('admin' + server.internalServerNumber + '@example.com') + expect(user.nsfwPolicy).to.equal('display') + + expect(rootUser.lastLoginDate).to.exist + expect(user.lastLoginDate).to.exist + + userId = user.id + }) + + it('Should list only the first user by username asc', async function () { + const { total, data } = await server.users.list({ start: 0, count: 1, sort: 'username' }) + + expect(total).to.equal(2) + expect(data.length).to.equal(1) + + const user = data[0] + expect(user.username).to.equal('root') + expect(user.email).to.equal('admin' + server.internalServerNumber + '@example.com') + expect(user.role.label).to.equal('Administrator') + expect(user.nsfwPolicy).to.equal('display') + }) + + it('Should list only the first user by username desc', async function () { + const { total, data } = await server.users.list({ start: 0, count: 1, sort: '-username' }) + + expect(total).to.equal(2) + expect(data.length).to.equal(1) + + const user = data[0] + expect(user.username).to.equal('user_1') + expect(user.email).to.equal('user_1@example.com') + expect(user.nsfwPolicy).to.equal('display') + }) + + it('Should list only the second user by createdAt desc', async function () { + const { data, total } = await server.users.list({ start: 0, count: 1, sort: '-createdAt' }) + expect(total).to.equal(2) + + expect(data.length).to.equal(1) + + const user = data[0] + expect(user.username).to.equal('user_1') + expect(user.email).to.equal('user_1@example.com') + expect(user.nsfwPolicy).to.equal('display') + }) + + it('Should list all the users by createdAt asc', async function () { + const { data, total } = await server.users.list({ start: 0, count: 2, sort: 'createdAt' }) + + expect(total).to.equal(2) + expect(data.length).to.equal(2) + + expect(data[0].username).to.equal('root') + expect(data[0].email).to.equal('admin' + server.internalServerNumber + '@example.com') + expect(data[0].nsfwPolicy).to.equal('display') + + expect(data[1].username).to.equal('user_1') + expect(data[1].email).to.equal('user_1@example.com') + expect(data[1].nsfwPolicy).to.equal('display') + }) + + it('Should search user by username', async function () { + const { data, total } = await server.users.list({ start: 0, count: 2, sort: 'createdAt', search: 'oot' }) + expect(total).to.equal(1) + expect(data.length).to.equal(1) + expect(data[0].username).to.equal('root') + }) + + it('Should search user by email', async function () { + { + const { total, data } = await server.users.list({ start: 0, count: 2, sort: 'createdAt', search: 'r_1@exam' }) + expect(total).to.equal(1) + expect(data.length).to.equal(1) + expect(data[0].username).to.equal('user_1') + expect(data[0].email).to.equal('user_1@example.com') + } + + { + const { total, data } = await server.users.list({ start: 0, count: 2, sort: 'createdAt', search: 'example' }) + expect(total).to.equal(2) + expect(data.length).to.equal(2) + expect(data[0].username).to.equal('root') + expect(data[1].username).to.equal('user_1') + } + }) + }) + + describe('Update my account', function () { + + it('Should update my password', async function () { + await server.users.updateMe({ + token: userToken, + currentPassword: 'super password', + password: 'new password' + }) + user.password = 'new password' + + await server.login.login({ user }) + }) + + it('Should be able to change the NSFW display attribute', async function () { + await server.users.updateMe({ + token: userToken, + nsfwPolicy: 'do_not_list' + }) + + const user = await server.users.getMyInfo({ token: userToken }) + expect(user.username).to.equal('user_1') + expect(user.email).to.equal('user_1@example.com') + expect(user.nsfwPolicy).to.equal('do_not_list') + expect(user.videoQuota).to.equal(2 * 1024 * 1024) + expect(user.id).to.be.a('number') + expect(user.account.displayName).to.equal('user_1') + expect(user.account.description).to.be.null + }) + + it('Should be able to change the autoPlayVideo attribute', async function () { + await server.users.updateMe({ + token: userToken, + autoPlayVideo: false + }) + + const user = await server.users.getMyInfo({ token: userToken }) + expect(user.autoPlayVideo).to.be.false + }) + + it('Should be able to change the autoPlayNextVideo attribute', async function () { + await server.users.updateMe({ + token: userToken, + autoPlayNextVideo: true + }) + + const user = await server.users.getMyInfo({ token: userToken }) + expect(user.autoPlayNextVideo).to.be.true + }) + + it('Should be able to change the p2p attribute', async function () { + await server.users.updateMe({ + token: userToken, + p2pEnabled: true + }) + + const user = await server.users.getMyInfo({ token: userToken }) + expect(user.p2pEnabled).to.be.true + }) + + it('Should be able to change the email attribute', async function () { + await server.users.updateMe({ + token: userToken, + currentPassword: 'new password', + email: 'updated@example.com' + }) + + const user = await server.users.getMyInfo({ token: userToken }) + expect(user.username).to.equal('user_1') + expect(user.email).to.equal('updated@example.com') + expect(user.nsfwPolicy).to.equal('do_not_list') + expect(user.videoQuota).to.equal(2 * 1024 * 1024) + expect(user.id).to.be.a('number') + expect(user.account.displayName).to.equal('user_1') + expect(user.account.description).to.be.null + }) + + it('Should be able to update my avatar with a gif', async function () { + const fixture = 'avatar.gif' + + await server.users.updateMyAvatar({ token: userToken, fixture }) + + const user = await server.users.getMyInfo({ token: userToken }) + for (const avatar of user.account.avatars) { + await testImageSize(server.url, `avatar-resized-${avatar.width}x${avatar.width}`, avatar.path, '.gif') + } + }) + + it('Should be able to update my avatar with a gif, and then a png', async function () { + for (const extension of [ '.png', '.gif' ]) { + const fixture = 'avatar' + extension + + await server.users.updateMyAvatar({ token: userToken, fixture }) + + const user = await server.users.getMyInfo({ token: userToken }) + for (const avatar of user.account.avatars) { + await testImageSize(server.url, `avatar-resized-${avatar.width}x${avatar.width}`, avatar.path, extension) + } + } + }) + + it('Should be able to update my display name', async function () { + await server.users.updateMe({ token: userToken, displayName: 'new display name' }) + + const user = await server.users.getMyInfo({ token: userToken }) + expect(user.username).to.equal('user_1') + expect(user.email).to.equal('updated@example.com') + expect(user.nsfwPolicy).to.equal('do_not_list') + expect(user.videoQuota).to.equal(2 * 1024 * 1024) + expect(user.id).to.be.a('number') + expect(user.account.displayName).to.equal('new display name') + expect(user.account.description).to.be.null + }) + + it('Should be able to update my description', async function () { + await server.users.updateMe({ token: userToken, description: 'my super description updated' }) + + const user = await server.users.getMyInfo({ token: userToken }) + expect(user.username).to.equal('user_1') + expect(user.email).to.equal('updated@example.com') + expect(user.nsfwPolicy).to.equal('do_not_list') + expect(user.videoQuota).to.equal(2 * 1024 * 1024) + expect(user.id).to.be.a('number') + expect(user.account.displayName).to.equal('new display name') + expect(user.account.description).to.equal('my super description updated') + expect(user.noWelcomeModal).to.be.false + expect(user.noInstanceConfigWarningModal).to.be.false + expect(user.noAccountSetupWarningModal).to.be.false + }) + + it('Should be able to update my theme', async function () { + for (const theme of [ 'background-red', 'default', 'instance-default' ]) { + await server.users.updateMe({ token: userToken, theme }) + + const user = await server.users.getMyInfo({ token: userToken }) + expect(user.theme).to.equal(theme) + } + }) + + it('Should be able to update my modal preferences', async function () { + await server.users.updateMe({ + token: userToken, + noInstanceConfigWarningModal: true, + noWelcomeModal: true, + noAccountSetupWarningModal: true + }) + + const user = await server.users.getMyInfo({ token: userToken }) + expect(user.noWelcomeModal).to.be.true + expect(user.noInstanceConfigWarningModal).to.be.true + expect(user.noAccountSetupWarningModal).to.be.true + }) + }) + + describe('Updating another user', function () { + + it('Should be able to update another user', async function () { + await server.users.update({ + userId, + token, + email: 'updated2@example.com', + emailVerified: true, + videoQuota: 42, + role: UserRole.MODERATOR, + adminFlags: UserAdminFlag.NONE, + pluginAuth: 'toto' + }) + + const user = await server.users.get({ token, userId }) + + expect(user.username).to.equal('user_1') + expect(user.email).to.equal('updated2@example.com') + expect(user.emailVerified).to.be.true + expect(user.nsfwPolicy).to.equal('do_not_list') + expect(user.videoQuota).to.equal(42) + expect(user.role.label).to.equal('Moderator') + expect(user.id).to.be.a('number') + expect(user.adminFlags).to.equal(UserAdminFlag.NONE) + expect(user.pluginAuth).to.equal('toto') + }) + + it('Should reset the auth plugin', async function () { + await server.users.update({ userId, token, pluginAuth: null }) + + const user = await server.users.get({ token, userId }) + expect(user.pluginAuth).to.be.null + }) + + it('Should have removed the user token', async function () { + await server.users.getMyQuotaUsed({ token: userToken, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + + userToken = await server.login.getAccessToken(user) + }) + + it('Should be able to update another user password', async function () { + await server.users.update({ userId, token, password: 'password updated' }) + + await server.users.getMyQuotaUsed({ token: userToken, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + + await server.login.login({ user, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + + user.password = 'password updated' + userToken = await server.login.getAccessToken(user) + }) + }) + + describe('Remove a user', function () { + + before(async function () { + await server.users.update({ + userId, + token, + videoQuota: 2 * 1024 * 1024 + }) + + await server.videos.quickUpload({ name: 'user video', token: userToken, fixture: 'video_short.webm' }) + await server.videos.quickUpload({ name: 'root video' }) + + const { total } = await server.videos.list() + expect(total).to.equal(2) + }) + + it('Should be able to remove this user', async function () { + await server.users.remove({ userId, token }) + }) + + it('Should not be able to login with this user', async function () { + await server.login.login({ user, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should not have videos of this user', async function () { + const { data, total } = await server.videos.list() + expect(total).to.equal(1) + + const video = data[0] + expect(video.account.name).to.equal('root') + }) + }) + + describe('User blocking', function () { + let user16Id: number + let user16AccessToken: string + + const user16 = { + username: 'user_16', + password: 'my super password' + } + + it('Should block a user', async function () { + const user = await server.users.create({ ...user16 }) + user16Id = user.id + + user16AccessToken = await server.login.getAccessToken(user16) + + await server.users.getMyInfo({ token: user16AccessToken, expectedStatus: HttpStatusCode.OK_200 }) + await server.users.banUser({ userId: user16Id }) + + await server.users.getMyInfo({ token: user16AccessToken, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + await server.login.login({ user: user16, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should search user by banned status', async function () { + { + const { data, total } = await server.users.list({ start: 0, count: 2, sort: 'createdAt', blocked: true }) + expect(total).to.equal(1) + expect(data.length).to.equal(1) + + expect(data[0].username).to.equal(user16.username) + } + + { + const { data, total } = await server.users.list({ start: 0, count: 2, sort: 'createdAt', blocked: false }) + expect(total).to.equal(1) + expect(data.length).to.equal(1) + + expect(data[0].username).to.not.equal(user16.username) + } + }) + + it('Should unblock a user', async function () { + await server.users.unbanUser({ userId: user16Id }) + user16AccessToken = await server.login.getAccessToken(user16) + await server.users.getMyInfo({ token: user16AccessToken, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('User stats', function () { + let user17Id: number + let user17AccessToken: string + + it('Should report correct initial statistics about a user', async function () { + const user17 = { + username: 'user_17', + password: 'my super password' + } + const created = await server.users.create({ ...user17 }) + + user17Id = created.id + user17AccessToken = await server.login.getAccessToken(user17) + + const user = await server.users.get({ userId: user17Id, withStats: true }) + expect(user.videosCount).to.equal(0) + expect(user.videoCommentsCount).to.equal(0) + expect(user.abusesCount).to.equal(0) + expect(user.abusesCreatedCount).to.equal(0) + expect(user.abusesAcceptedCount).to.equal(0) + }) + + it('Should report correct videos count', async function () { + const attributes = { name: 'video to test user stats' } + await server.videos.upload({ token: user17AccessToken, attributes }) + + const { data } = await server.videos.list() + videoId = data.find(video => video.name === attributes.name).id + + const user = await server.users.get({ userId: user17Id, withStats: true }) + expect(user.videosCount).to.equal(1) + }) + + it('Should report correct video comments for user', async function () { + const text = 'super comment' + await server.comments.createThread({ token: user17AccessToken, videoId, text }) + + const user = await server.users.get({ userId: user17Id, withStats: true }) + expect(user.videoCommentsCount).to.equal(1) + }) + + it('Should report correct abuses counts', async function () { + const reason = 'my super bad reason' + await server.abuses.report({ token: user17AccessToken, videoId, reason }) + + const body1 = await server.abuses.getAdminList() + const abuseId = body1.data[0].id + + const user2 = await server.users.get({ userId: user17Id, withStats: true }) + expect(user2.abusesCount).to.equal(1) // number of incriminations + expect(user2.abusesCreatedCount).to.equal(1) // number of reports created + + await server.abuses.update({ abuseId, body: { state: AbuseState.ACCEPTED } }) + + const user3 = await server.users.get({ userId: user17Id, withStats: true }) + expect(user3.abusesAcceptedCount).to.equal(1) // number of reports created accepted + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/videos/channel-import-videos.ts b/packages/tests/src/api/videos/channel-import-videos.ts new file mode 100644 index 000000000..d0e47fe95 --- /dev/null +++ b/packages/tests/src/api/videos/channel-import-videos.ts @@ -0,0 +1,161 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { FIXTURE_URLS } from '@tests/shared/tests.js' +import { areHttpImportTestsDisabled } from '@peertube/peertube-node-utils' +import { + createSingleServer, + getServerImportConfig, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test videos import in a channel', function () { + if (areHttpImportTestsDisabled()) return + + function runSuite (mode: 'youtube-dl' | 'yt-dlp') { + + describe('Import using ' + mode, function () { + let server: PeerTubeServer + + before(async function () { + this.timeout(120_000) + + server = await createSingleServer(1, getServerImportConfig(mode)) + + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + await server.config.enableChannelSync() + }) + + it('Should import a whole channel without specifying the sync id', async function () { + this.timeout(240_000) + + await server.channels.importVideos({ channelName: server.store.channel.name, externalChannelUrl: FIXTURE_URLS.youtubeChannel }) + await waitJobs(server) + + const videos = await server.videos.listByChannel({ handle: server.store.channel.name }) + expect(videos.total).to.equal(2) + }) + + it('These imports should not have a sync id', async function () { + const { total, data } = await server.imports.getMyVideoImports() + + expect(total).to.equal(2) + expect(data).to.have.lengthOf(2) + + for (const videoImport of data) { + expect(videoImport.videoChannelSync).to.not.exist + } + }) + + it('Should import a whole channel and specifying the sync id', async function () { + this.timeout(240_000) + + { + server.store.channel.name = 'channel2' + const { id } = await server.channels.create({ attributes: { name: server.store.channel.name } }) + server.store.channel.id = id + } + + { + const attributes = { + externalChannelUrl: FIXTURE_URLS.youtubeChannel, + videoChannelId: server.store.channel.id + } + + const { videoChannelSync } = await server.channelSyncs.create({ attributes }) + server.store.videoChannelSync = videoChannelSync + + await waitJobs(server) + } + + await server.channels.importVideos({ + channelName: server.store.channel.name, + externalChannelUrl: FIXTURE_URLS.youtubeChannel, + videoChannelSyncId: server.store.videoChannelSync.id + }) + + await waitJobs(server) + }) + + it('These imports should have a sync id', async function () { + const { total, data } = await server.imports.getMyVideoImports() + + expect(total).to.equal(4) + expect(data).to.have.lengthOf(4) + + const importsWithSyncId = data.filter(i => !!i.videoChannelSync) + expect(importsWithSyncId).to.have.lengthOf(2) + + for (const videoImport of importsWithSyncId) { + expect(videoImport.videoChannelSync).to.exist + expect(videoImport.videoChannelSync.id).to.equal(server.store.videoChannelSync.id) + } + }) + + it('Should be able to filter imports by this sync id', async function () { + const { total, data } = await server.imports.getMyVideoImports({ videoChannelSyncId: server.store.videoChannelSync.id }) + + expect(total).to.equal(2) + expect(data).to.have.lengthOf(2) + + for (const videoImport of data) { + expect(videoImport.videoChannelSync).to.exist + expect(videoImport.videoChannelSync.id).to.equal(server.store.videoChannelSync.id) + } + }) + + it('Should limit max amount of videos synced on full sync', async function () { + this.timeout(240_000) + + await server.kill() + await server.run({ + import: { + video_channel_synchronization: { + full_sync_videos_limit: 1 + } + } + }) + + const { id } = await server.channels.create({ attributes: { name: 'channel3' } }) + const channel3Id = id + + const { videoChannelSync } = await server.channelSyncs.create({ + attributes: { + externalChannelUrl: FIXTURE_URLS.youtubeChannel, + videoChannelId: channel3Id + } + }) + const syncId = videoChannelSync.id + + await waitJobs(server) + + await server.channels.importVideos({ + channelName: 'channel3', + externalChannelUrl: FIXTURE_URLS.youtubeChannel, + videoChannelSyncId: syncId + }) + + await waitJobs(server) + + const { total, data } = await server.videos.listByChannel({ handle: 'channel3' }) + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + }) + + after(async function () { + await server?.kill() + }) + }) + } + + runSuite('yt-dlp') + + // FIXME: With recent changes on youtube, youtube-dl doesn't fetch live replays which means the test suite fails + // runSuite('youtube-dl') +}) diff --git a/packages/tests/src/api/videos/index.ts b/packages/tests/src/api/videos/index.ts new file mode 100644 index 000000000..fcb1d5a81 --- /dev/null +++ b/packages/tests/src/api/videos/index.ts @@ -0,0 +1,23 @@ +import './multiple-servers.js' +import './resumable-upload.js' +import './single-server.js' +import './video-captions.js' +import './video-change-ownership.js' +import './video-channels.js' +import './channel-import-videos.js' +import './video-channel-syncs.js' +import './video-comments.js' +import './video-description.js' +import './video-files.js' +import './video-imports.js' +import './video-nsfw.js' +import './video-playlists.js' +import './video-playlist-thumbnails.js' +import './video-source.js' +import './video-privacy.js' +import './video-schedule-update.js' +import './videos-common-filters.js' +import './videos-history.js' +import './videos-overview.js' +import './video-static-file-privacy.js' +import './video-storyboard.js' diff --git a/packages/tests/src/api/videos/multiple-servers.ts b/packages/tests/src/api/videos/multiple-servers.ts new file mode 100644 index 000000000..03afd7cbb --- /dev/null +++ b/packages/tests/src/api/videos/multiple-servers.ts @@ -0,0 +1,1095 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import request from 'supertest' +import { wait } from '@peertube/peertube-core-utils' +import { HttpStatusCode, VideoCommentThreadTree, VideoPrivacy } from '@peertube/peertube-models' +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + makeGetRequest, + PeerTubeServer, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultChannelAvatar, + waitJobs +} from '@peertube/peertube-server-commands' +import { testImageGeneratedByFFmpeg, dateIsValid } from '@tests/shared/checks.js' +import { checkTmpIsEmpty } from '@tests/shared/directories.js' +import { completeVideoCheck, saveVideoInServers, checkVideoFilesWereRemoved } from '@tests/shared/videos.js' +import { checkWebTorrentWorks } from '@tests/shared/webtorrent.js' + +describe('Test multiple servers', function () { + let servers: PeerTubeServer[] = [] + const toRemove = [] + let videoUUID = '' + let videoChannelId: number + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(3) + + // Get the access tokens + await setAccessTokensToServers(servers) + + { + const videoChannel = { + name: 'super_channel_name', + displayName: 'my channel', + description: 'super channel' + } + await servers[0].channels.create({ attributes: videoChannel }) + await setDefaultChannelAvatar(servers[0], videoChannel.name) + await setDefaultAccountAvatar(servers) + + const { data } = await servers[0].channels.list({ start: 0, count: 1 }) + videoChannelId = data[0].id + } + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + // Server 1 and server 3 follow each other + await doubleFollow(servers[0], servers[2]) + // Server 2 and server 3 follow each other + await doubleFollow(servers[1], servers[2]) + }) + + it('Should not have videos for all servers', async function () { + for (const server of servers) { + const { data } = await server.videos.list() + expect(data).to.be.an('array') + expect(data.length).to.equal(0) + } + }) + + describe('Should upload the video and propagate on each server', function () { + + it('Should upload the video on server 1 and propagate on each server', async function () { + this.timeout(60000) + + const attributes = { + name: 'my super name for server 1', + category: 5, + licence: 4, + language: 'ja', + nsfw: true, + description: 'my super description for server 1', + support: 'my super support text for server 1', + originallyPublishedAt: '2019-02-10T13:38:14.449Z', + tags: [ 'tag1p1', 'tag2p1' ], + channelId: videoChannelId, + fixture: 'video_short1.webm' + } + await servers[0].videos.upload({ attributes }) + + await waitJobs(servers) + + // All servers should have this video + let publishedAt: string = null + for (const server of servers) { + const isLocal = server.port === servers[0].port + const checkAttributes = { + name: 'my super name for server 1', + category: 5, + licence: 4, + language: 'ja', + nsfw: true, + description: 'my super description for server 1', + support: 'my super support text for server 1', + originallyPublishedAt: '2019-02-10T13:38:14.449Z', + account: { + name: 'root', + host: servers[0].host + }, + isLocal, + publishedAt, + duration: 10, + tags: [ 'tag1p1', 'tag2p1' ], + privacy: VideoPrivacy.PUBLIC, + commentsEnabled: true, + downloadEnabled: true, + channel: { + displayName: 'my channel', + name: 'super_channel_name', + description: 'super channel', + isLocal + }, + fixture: 'video_short1.webm', + files: [ + { + resolution: 720, + size: 572456 + } + ] + } + + const { data } = await server.videos.list() + expect(data).to.be.an('array') + expect(data.length).to.equal(1) + const video = data[0] + + await completeVideoCheck({ server, originServer: servers[0], videoUUID: video.uuid, attributes: checkAttributes }) + publishedAt = video.publishedAt as string + + expect(video.channel.avatars).to.have.lengthOf(2) + expect(video.account.avatars).to.have.lengthOf(2) + + for (const image of [ ...video.channel.avatars, ...video.account.avatars ]) { + expect(image.createdAt).to.exist + expect(image.updatedAt).to.exist + expect(image.width).to.be.above(20).and.below(1000) + expect(image.path).to.exist + + await makeGetRequest({ + url: server.url, + path: image.path, + expectedStatus: HttpStatusCode.OK_200 + }) + } + } + }) + + it('Should upload the video on server 2 and propagate on each server', async function () { + this.timeout(240000) + + const user = { + username: 'user1', + password: 'super_password' + } + await servers[1].users.create({ username: user.username, password: user.password }) + const userAccessToken = await servers[1].login.getAccessToken(user) + + const attributes = { + name: 'my super name for server 2', + category: 4, + licence: 3, + language: 'de', + nsfw: true, + description: 'my super description for server 2', + support: 'my super support text for server 2', + tags: [ 'tag1p2', 'tag2p2', 'tag3p2' ], + fixture: 'video_short2.webm', + thumbnailfile: 'custom-thumbnail.jpg', + previewfile: 'custom-preview.jpg' + } + await servers[1].videos.upload({ token: userAccessToken, attributes, mode: 'resumable' }) + + // Transcoding + await waitJobs(servers) + + // All servers should have this video + for (const server of servers) { + const isLocal = server.url === servers[1].url + const checkAttributes = { + name: 'my super name for server 2', + category: 4, + licence: 3, + language: 'de', + nsfw: true, + description: 'my super description for server 2', + support: 'my super support text for server 2', + account: { + name: 'user1', + host: servers[1].host + }, + isLocal, + commentsEnabled: true, + downloadEnabled: true, + duration: 5, + tags: [ 'tag1p2', 'tag2p2', 'tag3p2' ], + privacy: VideoPrivacy.PUBLIC, + channel: { + displayName: 'Main user1 channel', + name: 'user1_channel', + description: 'super channel', + isLocal + }, + fixture: 'video_short2.webm', + files: [ + { + resolution: 240, + size: 270000 + }, + { + resolution: 360, + size: 359000 + }, + { + resolution: 480, + size: 465000 + }, + { + resolution: 720, + size: 750000 + } + ], + thumbnailfile: 'custom-thumbnail', + previewfile: 'custom-preview' + } + + const { data } = await server.videos.list() + expect(data).to.be.an('array') + expect(data.length).to.equal(2) + const video = data[1] + + await completeVideoCheck({ server, originServer: servers[1], videoUUID: video.uuid, attributes: checkAttributes }) + } + }) + + it('Should upload two videos on server 3 and propagate on each server', async function () { + this.timeout(45000) + + { + const attributes = { + name: 'my super name for server 3', + category: 6, + licence: 5, + language: 'de', + nsfw: true, + description: 'my super description for server 3', + support: 'my super support text for server 3', + tags: [ 'tag1p3' ], + fixture: 'video_short3.webm' + } + await servers[2].videos.upload({ attributes }) + } + + { + const attributes = { + name: 'my super name for server 3-2', + category: 7, + licence: 6, + language: 'ko', + nsfw: false, + description: 'my super description for server 3-2', + support: 'my super support text for server 3-2', + tags: [ 'tag2p3', 'tag3p3', 'tag4p3' ], + fixture: 'video_short.webm' + } + await servers[2].videos.upload({ attributes }) + } + + await waitJobs(servers) + + // All servers should have this video + for (const server of servers) { + const isLocal = server.url === servers[2].url + const { data } = await server.videos.list() + + expect(data).to.be.an('array') + expect(data.length).to.equal(4) + + // We not sure about the order of the two last uploads + let video1 = null + let video2 = null + if (data[2].name === 'my super name for server 3') { + video1 = data[2] + video2 = data[3] + } else { + video1 = data[3] + video2 = data[2] + } + + const checkAttributesVideo1 = { + name: 'my super name for server 3', + category: 6, + licence: 5, + language: 'de', + nsfw: true, + description: 'my super description for server 3', + support: 'my super support text for server 3', + account: { + name: 'root', + host: servers[2].host + }, + isLocal, + duration: 5, + commentsEnabled: true, + downloadEnabled: true, + tags: [ 'tag1p3' ], + privacy: VideoPrivacy.PUBLIC, + channel: { + displayName: 'Main root channel', + name: 'root_channel', + description: '', + isLocal + }, + fixture: 'video_short3.webm', + files: [ + { + resolution: 720, + size: 292677 + } + ] + } + await completeVideoCheck({ server, originServer: servers[2], videoUUID: video1.uuid, attributes: checkAttributesVideo1 }) + + const checkAttributesVideo2 = { + name: 'my super name for server 3-2', + category: 7, + licence: 6, + language: 'ko', + nsfw: false, + description: 'my super description for server 3-2', + support: 'my super support text for server 3-2', + account: { + name: 'root', + host: servers[2].host + }, + commentsEnabled: true, + downloadEnabled: true, + isLocal, + duration: 5, + tags: [ 'tag2p3', 'tag3p3', 'tag4p3' ], + privacy: VideoPrivacy.PUBLIC, + channel: { + displayName: 'Main root channel', + name: 'root_channel', + description: '', + isLocal + }, + fixture: 'video_short.webm', + files: [ + { + resolution: 720, + size: 218910 + } + ] + } + await completeVideoCheck({ server, originServer: servers[2], videoUUID: video2.uuid, attributes: checkAttributesVideo2 }) + } + }) + }) + + describe('It should list local videos', function () { + it('Should list only local videos on server 1', async function () { + const { data, total } = await servers[0].videos.list({ isLocal: true }) + + expect(total).to.equal(1) + expect(data).to.be.an('array') + expect(data.length).to.equal(1) + expect(data[0].name).to.equal('my super name for server 1') + }) + + it('Should list only local videos on server 2', async function () { + const { data, total } = await servers[1].videos.list({ isLocal: true }) + + expect(total).to.equal(1) + expect(data).to.be.an('array') + expect(data.length).to.equal(1) + expect(data[0].name).to.equal('my super name for server 2') + }) + + it('Should list only local videos on server 3', async function () { + const { data, total } = await servers[2].videos.list({ isLocal: true }) + + expect(total).to.equal(2) + expect(data).to.be.an('array') + expect(data.length).to.equal(2) + expect(data[0].name).to.equal('my super name for server 3') + expect(data[1].name).to.equal('my super name for server 3-2') + }) + }) + + describe('Should seed the uploaded video', function () { + + it('Should add the file 1 by asking server 3', async function () { + this.retries(2) + this.timeout(30000) + + const { data } = await servers[2].videos.list() + + const video = data[0] + toRemove.push(data[2]) + toRemove.push(data[3]) + + const videoDetails = await servers[2].videos.get({ id: video.id }) + + await checkWebTorrentWorks(videoDetails.files[0].magnetUri) + }) + + it('Should add the file 2 by asking server 1', async function () { + this.retries(2) + this.timeout(30000) + + const { data } = await servers[0].videos.list() + + const video = data[1] + const videoDetails = await servers[0].videos.get({ id: video.id }) + + await checkWebTorrentWorks(videoDetails.files[0].magnetUri) + }) + + it('Should add the file 3 by asking server 2', async function () { + this.retries(2) + this.timeout(30000) + + const { data } = await servers[1].videos.list() + + const video = data[2] + const videoDetails = await servers[1].videos.get({ id: video.id }) + + await checkWebTorrentWorks(videoDetails.files[0].magnetUri) + }) + + it('Should add the file 3-2 by asking server 1', async function () { + this.retries(2) + this.timeout(30000) + + const { data } = await servers[0].videos.list() + + const video = data[3] + const videoDetails = await servers[0].videos.get({ id: video.id }) + + await checkWebTorrentWorks(videoDetails.files[0].magnetUri) + }) + + it('Should add the file 2 in 360p by asking server 1', async function () { + this.retries(2) + this.timeout(30000) + + const { data } = await servers[0].videos.list() + + const video = data.find(v => v.name === 'my super name for server 2') + const videoDetails = await servers[0].videos.get({ id: video.id }) + + const file = videoDetails.files.find(f => f.resolution.id === 360) + expect(file).not.to.be.undefined + + await checkWebTorrentWorks(file.magnetUri) + }) + }) + + describe('Should update video views, likes and dislikes', function () { + let localVideosServer3 = [] + let remoteVideosServer1 = [] + let remoteVideosServer2 = [] + let remoteVideosServer3 = [] + + before(async function () { + { + const { data } = await servers[0].videos.list() + remoteVideosServer1 = data.filter(video => video.isLocal === false).map(video => video.uuid) + } + + { + const { data } = await servers[1].videos.list() + remoteVideosServer2 = data.filter(video => video.isLocal === false).map(video => video.uuid) + } + + { + const { data } = await servers[2].videos.list() + localVideosServer3 = data.filter(video => video.isLocal === true).map(video => video.uuid) + remoteVideosServer3 = data.filter(video => video.isLocal === false).map(video => video.uuid) + } + }) + + it('Should view multiple videos on owned servers', async function () { + this.timeout(30000) + + await servers[2].views.simulateView({ id: localVideosServer3[0] }) + await wait(1000) + + await servers[2].views.simulateView({ id: localVideosServer3[0] }) + await servers[2].views.simulateView({ id: localVideosServer3[1] }) + + await wait(1000) + + await servers[2].views.simulateView({ id: localVideosServer3[0] }) + await servers[2].views.simulateView({ id: localVideosServer3[0] }) + + await waitJobs(servers) + + for (const server of servers) { + await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } }) + } + + await waitJobs(servers) + + for (const server of servers) { + const { data } = await server.videos.list() + + const video0 = data.find(v => v.uuid === localVideosServer3[0]) + const video1 = data.find(v => v.uuid === localVideosServer3[1]) + + expect(video0.views).to.equal(3) + expect(video1.views).to.equal(1) + } + }) + + it('Should view multiple videos on each servers', async function () { + this.timeout(45000) + + const tasks: Promise[] = [] + tasks.push(servers[0].views.simulateView({ id: remoteVideosServer1[0] })) + tasks.push(servers[1].views.simulateView({ id: remoteVideosServer2[0] })) + tasks.push(servers[1].views.simulateView({ id: remoteVideosServer2[0] })) + tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[0] })) + tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[1] })) + tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[1] })) + tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[1] })) + tasks.push(servers[2].views.simulateView({ id: localVideosServer3[1] })) + tasks.push(servers[2].views.simulateView({ id: localVideosServer3[1] })) + tasks.push(servers[2].views.simulateView({ id: localVideosServer3[1] })) + + await Promise.all(tasks) + + await waitJobs(servers) + + for (const server of servers) { + await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } }) + } + + await waitJobs(servers) + + let baseVideos = null + + for (const server of servers) { + const { data } = await server.videos.list() + + // Initialize base videos for future comparisons + if (baseVideos === null) { + baseVideos = data + continue + } + + for (const baseVideo of baseVideos) { + const sameVideo = data.find(video => video.name === baseVideo.name) + expect(baseVideo.views).to.equal(sameVideo.views) + } + } + }) + + it('Should like and dislikes videos on different services', async function () { + this.timeout(50000) + + await servers[0].videos.rate({ id: remoteVideosServer1[0], rating: 'like' }) + await wait(500) + await servers[0].videos.rate({ id: remoteVideosServer1[0], rating: 'dislike' }) + await wait(500) + await servers[0].videos.rate({ id: remoteVideosServer1[0], rating: 'like' }) + await servers[2].videos.rate({ id: localVideosServer3[1], rating: 'like' }) + await wait(500) + await servers[2].videos.rate({ id: localVideosServer3[1], rating: 'dislike' }) + await servers[2].videos.rate({ id: remoteVideosServer3[1], rating: 'dislike' }) + await wait(500) + await servers[2].videos.rate({ id: remoteVideosServer3[0], rating: 'like' }) + + await waitJobs(servers) + await wait(5000) + await waitJobs(servers) + + let baseVideos = null + for (const server of servers) { + const { data } = await server.videos.list() + + // Initialize base videos for future comparisons + if (baseVideos === null) { + baseVideos = data + continue + } + + for (const baseVideo of baseVideos) { + const sameVideo = data.find(video => video.name === baseVideo.name) + expect(baseVideo.likes).to.equal(sameVideo.likes, `Likes of ${sameVideo.uuid} do not correspond`) + expect(baseVideo.dislikes).to.equal(sameVideo.dislikes, `Dislikes of ${sameVideo.uuid} do not correspond`) + } + } + }) + }) + + describe('Should manipulate these videos', function () { + let updatedAtMin: Date + + it('Should update video 3', async function () { + this.timeout(30000) + + const attributes = { + name: 'my super video updated', + category: 10, + licence: 7, + language: 'fr', + nsfw: true, + description: 'my super description updated', + support: 'my super support text updated', + tags: [ 'tag_up_1', 'tag_up_2' ], + thumbnailfile: 'custom-thumbnail.jpg', + originallyPublishedAt: '2019-02-11T13:38:14.449Z', + previewfile: 'custom-preview.jpg' + } + + updatedAtMin = new Date() + await servers[2].videos.update({ id: toRemove[0].id, attributes }) + + await waitJobs(servers) + }) + + it('Should have the video 3 updated on each server', async function () { + this.timeout(30000) + + for (const server of servers) { + const { data } = await server.videos.list() + + const videoUpdated = data.find(video => video.name === 'my super video updated') + expect(!!videoUpdated).to.be.true + + expect(new Date(videoUpdated.updatedAt)).to.be.greaterThan(updatedAtMin) + + const isLocal = server.url === servers[2].url + const checkAttributes = { + name: 'my super video updated', + category: 10, + licence: 7, + language: 'fr', + nsfw: true, + description: 'my super description updated', + support: 'my super support text updated', + originallyPublishedAt: '2019-02-11T13:38:14.449Z', + account: { + name: 'root', + host: servers[2].host + }, + isLocal, + duration: 5, + commentsEnabled: true, + downloadEnabled: true, + tags: [ 'tag_up_1', 'tag_up_2' ], + privacy: VideoPrivacy.PUBLIC, + channel: { + displayName: 'Main root channel', + name: 'root_channel', + description: '', + isLocal + }, + fixture: 'video_short3.webm', + files: [ + { + resolution: 720, + size: 292677 + } + ], + thumbnailfile: 'custom-thumbnail', + previewfile: 'custom-preview' + } + await completeVideoCheck({ server, originServer: servers[2], videoUUID: videoUpdated.uuid, attributes: checkAttributes }) + } + }) + + it('Should only update thumbnail and update updatedAt attribute', async function () { + this.timeout(30000) + + const attributes = { + thumbnailfile: 'custom-thumbnail.jpg' + } + + updatedAtMin = new Date() + await servers[2].videos.update({ id: toRemove[0].id, attributes }) + + await waitJobs(servers) + + for (const server of servers) { + const { data } = await server.videos.list() + + const videoUpdated = data.find(video => video.name === 'my super video updated') + expect(new Date(videoUpdated.updatedAt)).to.be.greaterThan(updatedAtMin) + } + }) + + it('Should remove the videos 3 and 3-2 by asking server 3 and correctly delete files', async function () { + this.timeout(30000) + + for (const id of [ toRemove[0].id, toRemove[1].id ]) { + await saveVideoInServers(servers, id) + + await servers[2].videos.remove({ id }) + + await waitJobs(servers) + + for (const server of servers) { + await checkVideoFilesWereRemoved({ server, video: server.store.videoDetails }) + } + } + }) + + it('Should have videos 1 and 3 on each server', async function () { + for (const server of servers) { + const { data } = await server.videos.list() + + expect(data).to.be.an('array') + expect(data.length).to.equal(2) + expect(data[0].name).not.to.equal(data[1].name) + expect(data[0].name).not.to.equal(toRemove[0].name) + expect(data[1].name).not.to.equal(toRemove[0].name) + expect(data[0].name).not.to.equal(toRemove[1].name) + expect(data[1].name).not.to.equal(toRemove[1].name) + + videoUUID = data.find(video => video.name === 'my super name for server 1').uuid + } + }) + + it('Should get the same video by UUID on each server', async function () { + let baseVideo = null + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + + if (baseVideo === null) { + baseVideo = video + continue + } + + expect(baseVideo.name).to.equal(video.name) + expect(baseVideo.uuid).to.equal(video.uuid) + expect(baseVideo.category.id).to.equal(video.category.id) + expect(baseVideo.language.id).to.equal(video.language.id) + expect(baseVideo.licence.id).to.equal(video.licence.id) + expect(baseVideo.nsfw).to.equal(video.nsfw) + expect(baseVideo.account.name).to.equal(video.account.name) + expect(baseVideo.account.displayName).to.equal(video.account.displayName) + expect(baseVideo.account.url).to.equal(video.account.url) + expect(baseVideo.account.host).to.equal(video.account.host) + expect(baseVideo.tags).to.deep.equal(video.tags) + } + }) + + it('Should get the preview from each server', async function () { + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + + await testImageGeneratedByFFmpeg(server.url, 'video_short1-preview.webm', video.previewPath) + } + }) + }) + + describe('Should comment these videos', function () { + let childOfFirstChild: VideoCommentThreadTree + + it('Should add comment (threads and replies)', async function () { + this.timeout(25000) + + { + const text = 'my super first comment' + await servers[0].comments.createThread({ videoId: videoUUID, text }) + } + + { + const text = 'my super second comment' + await servers[2].comments.createThread({ videoId: videoUUID, text }) + } + + await waitJobs(servers) + + { + const threadId = await servers[1].comments.findCommentId({ videoId: videoUUID, text: 'my super first comment' }) + + const text = 'my super answer to thread 1' + await servers[1].comments.addReply({ videoId: videoUUID, toCommentId: threadId, text }) + } + + await waitJobs(servers) + + { + const threadId = await servers[2].comments.findCommentId({ videoId: videoUUID, text: 'my super first comment' }) + + const body = await servers[2].comments.getThread({ videoId: videoUUID, threadId }) + const childCommentId = body.children[0].comment.id + + const text3 = 'my second answer to thread 1' + await servers[2].comments.addReply({ videoId: videoUUID, toCommentId: threadId, text: text3 }) + + const text2 = 'my super answer to answer of thread 1' + await servers[2].comments.addReply({ videoId: videoUUID, toCommentId: childCommentId, text: text2 }) + } + + await waitJobs(servers) + }) + + it('Should have these threads', async function () { + for (const server of servers) { + const body = await server.comments.listThreads({ videoId: videoUUID }) + + expect(body.total).to.equal(2) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(2) + + { + const comment = body.data.find(c => c.text === 'my super first comment') + expect(comment).to.not.be.undefined + expect(comment.inReplyToCommentId).to.be.null + expect(comment.account.name).to.equal('root') + expect(comment.account.host).to.equal(servers[0].host) + expect(comment.totalReplies).to.equal(3) + expect(dateIsValid(comment.createdAt as string)).to.be.true + expect(dateIsValid(comment.updatedAt as string)).to.be.true + } + + { + const comment = body.data.find(c => c.text === 'my super second comment') + expect(comment).to.not.be.undefined + expect(comment.inReplyToCommentId).to.be.null + expect(comment.account.name).to.equal('root') + expect(comment.account.host).to.equal(servers[2].host) + expect(comment.totalReplies).to.equal(0) + expect(dateIsValid(comment.createdAt as string)).to.be.true + expect(dateIsValid(comment.updatedAt as string)).to.be.true + } + } + }) + + it('Should have these comments', async function () { + for (const server of servers) { + const body = await server.comments.listThreads({ videoId: videoUUID }) + const threadId = body.data.find(c => c.text === 'my super first comment').id + + const tree = await server.comments.getThread({ videoId: videoUUID, threadId }) + + expect(tree.comment.text).equal('my super first comment') + expect(tree.comment.account.name).equal('root') + expect(tree.comment.account.host).equal(servers[0].host) + expect(tree.children).to.have.lengthOf(2) + + const firstChild = tree.children[0] + expect(firstChild.comment.text).to.equal('my super answer to thread 1') + expect(firstChild.comment.account.name).equal('root') + expect(firstChild.comment.account.host).equal(servers[1].host) + expect(firstChild.children).to.have.lengthOf(1) + + childOfFirstChild = firstChild.children[0] + expect(childOfFirstChild.comment.text).to.equal('my super answer to answer of thread 1') + expect(childOfFirstChild.comment.account.name).equal('root') + expect(childOfFirstChild.comment.account.host).equal(servers[2].host) + expect(childOfFirstChild.children).to.have.lengthOf(0) + + const secondChild = tree.children[1] + expect(secondChild.comment.text).to.equal('my second answer to thread 1') + expect(secondChild.comment.account.name).equal('root') + expect(secondChild.comment.account.host).equal(servers[2].host) + expect(secondChild.children).to.have.lengthOf(0) + } + }) + + it('Should delete a reply', async function () { + this.timeout(30000) + + await servers[2].comments.delete({ videoId: videoUUID, commentId: childOfFirstChild.comment.id }) + + await waitJobs(servers) + }) + + it('Should have this comment marked as deleted', async function () { + for (const server of servers) { + const { data } = await server.comments.listThreads({ videoId: videoUUID }) + const threadId = data.find(c => c.text === 'my super first comment').id + + const tree = await server.comments.getThread({ videoId: videoUUID, threadId }) + expect(tree.comment.text).equal('my super first comment') + + const firstChild = tree.children[0] + expect(firstChild.comment.text).to.equal('my super answer to thread 1') + expect(firstChild.children).to.have.lengthOf(1) + + const deletedComment = firstChild.children[0].comment + expect(deletedComment.isDeleted).to.be.true + expect(deletedComment.deletedAt).to.not.be.null + expect(deletedComment.account).to.be.null + expect(deletedComment.text).to.equal('') + + const secondChild = tree.children[1] + expect(secondChild.comment.text).to.equal('my second answer to thread 1') + } + }) + + it('Should delete the thread comments', async function () { + this.timeout(30000) + + const { data } = await servers[0].comments.listThreads({ videoId: videoUUID }) + const commentId = data.find(c => c.text === 'my super first comment').id + await servers[0].comments.delete({ videoId: videoUUID, commentId }) + + await waitJobs(servers) + }) + + it('Should have the threads marked as deleted on other servers too', async function () { + for (const server of servers) { + const body = await server.comments.listThreads({ videoId: videoUUID }) + + expect(body.total).to.equal(2) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(2) + + { + const comment = body.data[0] + expect(comment).to.not.be.undefined + expect(comment.inReplyToCommentId).to.be.null + expect(comment.account.name).to.equal('root') + expect(comment.account.host).to.equal(servers[2].host) + expect(comment.totalReplies).to.equal(0) + expect(dateIsValid(comment.createdAt as string)).to.be.true + expect(dateIsValid(comment.updatedAt as string)).to.be.true + } + + { + const deletedComment = body.data[1] + expect(deletedComment).to.not.be.undefined + expect(deletedComment.isDeleted).to.be.true + expect(deletedComment.deletedAt).to.not.be.null + expect(deletedComment.text).to.equal('') + expect(deletedComment.inReplyToCommentId).to.be.null + expect(deletedComment.account).to.be.null + expect(deletedComment.totalReplies).to.equal(2) + expect(dateIsValid(deletedComment.createdAt as string)).to.be.true + expect(dateIsValid(deletedComment.updatedAt as string)).to.be.true + expect(dateIsValid(deletedComment.deletedAt as string)).to.be.true + } + } + }) + + it('Should delete a remote thread by the origin server', async function () { + this.timeout(5000) + + const { data } = await servers[0].comments.listThreads({ videoId: videoUUID }) + const commentId = data.find(c => c.text === 'my super second comment').id + await servers[0].comments.delete({ videoId: videoUUID, commentId }) + + await waitJobs(servers) + }) + + it('Should have the threads marked as deleted on other servers too', async function () { + for (const server of servers) { + const body = await server.comments.listThreads({ videoId: videoUUID }) + + expect(body.total).to.equal(2) + expect(body.data).to.have.lengthOf(2) + + { + const comment = body.data[0] + expect(comment.text).to.equal('') + expect(comment.isDeleted).to.be.true + expect(comment.createdAt).to.not.be.null + expect(comment.deletedAt).to.not.be.null + expect(comment.account).to.be.null + expect(comment.totalReplies).to.equal(0) + } + + { + const comment = body.data[1] + expect(comment.text).to.equal('') + expect(comment.isDeleted).to.be.true + expect(comment.createdAt).to.not.be.null + expect(comment.deletedAt).to.not.be.null + expect(comment.account).to.be.null + expect(comment.totalReplies).to.equal(2) + } + } + }) + + it('Should disable comments and download', async function () { + this.timeout(20000) + + const attributes = { + commentsEnabled: false, + downloadEnabled: false + } + + await servers[0].videos.update({ id: videoUUID, attributes }) + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + expect(video.commentsEnabled).to.be.false + expect(video.downloadEnabled).to.be.false + + const text = 'my super forbidden comment' + await server.comments.createThread({ videoId: videoUUID, text, expectedStatus: HttpStatusCode.CONFLICT_409 }) + } + }) + }) + + describe('With minimum parameters', function () { + it('Should upload and propagate the video', async function () { + this.timeout(120000) + + const path = '/api/v1/videos/upload' + + const req = request(servers[1].url) + .post(path) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + servers[1].accessToken) + .field('name', 'minimum parameters') + .field('privacy', '1') + .field('channelId', '1') + + await req.attach('videofile', buildAbsoluteFixturePath('video_short.webm')) + .expect(HttpStatusCode.OK_200) + + await waitJobs(servers) + + for (const server of servers) { + const { data } = await server.videos.list() + const video = data.find(v => v.name === 'minimum parameters') + + const isLocal = server.url === servers[1].url + const checkAttributes = { + name: 'minimum parameters', + category: null, + licence: null, + language: null, + nsfw: false, + description: null, + support: null, + account: { + name: 'root', + host: servers[1].host + }, + isLocal, + duration: 5, + commentsEnabled: true, + downloadEnabled: true, + tags: [], + privacy: VideoPrivacy.PUBLIC, + channel: { + displayName: 'Main root channel', + name: 'root_channel', + description: '', + isLocal + }, + fixture: 'video_short.webm', + files: [ + { + resolution: 720, + size: 61000 + }, + { + resolution: 480, + size: 40000 + }, + { + resolution: 360, + size: 32000 + }, + { + resolution: 240, + size: 23000 + } + ] + } + await completeVideoCheck({ server, originServer: servers[1], videoUUID: video.uuid, attributes: checkAttributes }) + } + }) + }) + + describe('TMP directory', function () { + it('Should have an empty tmp directory', async function () { + for (const server of servers) { + await checkTmpIsEmpty(server) + } + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/videos/resumable-upload.ts b/packages/tests/src/api/videos/resumable-upload.ts new file mode 100644 index 000000000..628e0298c --- /dev/null +++ b/packages/tests/src/api/videos/resumable-upload.ts @@ -0,0 +1,316 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { pathExists } from 'fs-extra/esm' +import { readdir, stat } from 'fs/promises' +import { join } from 'path' +import { HttpStatusCode, HttpStatusCodeType, VideoPrivacy } from '@peertube/peertube-models' +import { buildAbsoluteFixturePath, sha1 } from '@peertube/peertube-node-utils' +import { + cleanupTests, + createSingleServer, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel +} from '@peertube/peertube-server-commands' + +// Most classic resumable upload tests are done in other test suites + +describe('Test resumable upload', function () { + const path = '/api/v1/videos/upload-resumable' + const defaultFixture = 'video_short.mp4' + let server: PeerTubeServer + let rootId: number + let userAccessToken: string + let userChannelId: number + + async function buildSize (fixture: string, size?: number) { + if (size !== undefined) return size + + const baseFixture = buildAbsoluteFixturePath(fixture) + return (await stat(baseFixture)).size + } + + async function prepareUpload (options: { + channelId?: number + token?: string + size?: number + originalName?: string + lastModified?: number + } = {}) { + const { token, originalName, lastModified } = options + + const size = await buildSize(defaultFixture, options.size) + + const attributes = { + name: 'video', + channelId: options.channelId ?? server.store.channel.id, + privacy: VideoPrivacy.PUBLIC, + fixture: defaultFixture + } + + const mimetype = 'video/mp4' + + const res = await server.videos.prepareResumableUpload({ path, token, attributes, size, mimetype, originalName, lastModified }) + + return res.header['location'].split('?')[1] + } + + async function sendChunks (options: { + token?: string + pathUploadId: string + size?: number + expectedStatus?: HttpStatusCodeType + contentLength?: number + contentRange?: string + contentRangeBuilder?: (start: number, chunk: any) => string + digestBuilder?: (chunk: any) => string + }) { + const { token, pathUploadId, expectedStatus, contentLength, contentRangeBuilder, digestBuilder } = options + + const size = await buildSize(defaultFixture, options.size) + const absoluteFilePath = buildAbsoluteFixturePath(defaultFixture) + + return server.videos.sendResumableChunks({ + token, + path, + pathUploadId, + videoFilePath: absoluteFilePath, + size, + contentLength, + contentRangeBuilder, + digestBuilder, + expectedStatus + }) + } + + async function checkFileSize (uploadIdArg: string, expectedSize: number | null) { + const uploadId = uploadIdArg.replace(/^upload_id=/, '') + + const subPath = join('tmp', 'resumable-uploads', `${rootId}-${uploadId}.mp4`) + const filePath = server.servers.buildDirectory(subPath) + const exists = await pathExists(filePath) + + if (expectedSize === null) { + expect(exists).to.be.false + return + } + + expect(exists).to.be.true + + expect((await stat(filePath)).size).to.equal(expectedSize) + } + + async function countResumableUploads (wait?: number) { + const subPath = join('tmp', 'resumable-uploads') + const filePath = server.servers.buildDirectory(subPath) + await new Promise(resolve => setTimeout(resolve, wait)) + const files = await readdir(filePath) + return files.length + } + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + const body = await server.users.getMyInfo() + rootId = body.id + + { + userAccessToken = await server.users.generateUserAndToken('user1') + const { videoChannels } = await server.users.getMyInfo({ token: userAccessToken }) + userChannelId = videoChannels[0].id + } + + await server.users.update({ userId: rootId, videoQuota: 10_000_000 }) + }) + + describe('Directory cleaning', function () { + + it('Should correctly delete files after an upload', async function () { + const uploadId = await prepareUpload() + await sendChunks({ pathUploadId: uploadId }) + await server.videos.endResumableUpload({ path, pathUploadId: uploadId }) + + expect(await countResumableUploads()).to.equal(0) + }) + + it('Should correctly delete corrupt files', async function () { + const uploadId = await prepareUpload({ size: 8 * 1024 }) + await sendChunks({ pathUploadId: uploadId, size: 8 * 1024, expectedStatus: HttpStatusCode.UNPROCESSABLE_ENTITY_422 }) + + expect(await countResumableUploads(2000)).to.equal(0) + }) + + it('Should not delete files after an unfinished upload', async function () { + await prepareUpload() + + expect(await countResumableUploads()).to.equal(2) + }) + + it('Should not delete recent uploads', async function () { + await server.debug.sendCommand({ body: { command: 'remove-dandling-resumable-uploads' } }) + + expect(await countResumableUploads()).to.equal(2) + }) + + it('Should delete old uploads', async function () { + await server.debug.sendCommand({ body: { command: 'remove-dandling-resumable-uploads' } }) + + expect(await countResumableUploads()).to.equal(0) + }) + }) + + describe('Resumable upload and chunks', function () { + + it('Should accept the same amount of chunks', async function () { + const uploadId = await prepareUpload() + await sendChunks({ pathUploadId: uploadId }) + + await checkFileSize(uploadId, null) + }) + + it('Should not accept more chunks than expected', async function () { + const uploadId = await prepareUpload({ size: 100 }) + + await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.CONFLICT_409 }) + await checkFileSize(uploadId, 0) + }) + + it('Should not accept more chunks than expected with an invalid content length/content range', async function () { + const uploadId = await prepareUpload({ size: 1500 }) + + // Content length check can be different depending on the node version + try { + await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.CONFLICT_409, contentLength: 1000 }) + await checkFileSize(uploadId, 0) + } catch { + await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.BAD_REQUEST_400, contentLength: 1000 }) + await checkFileSize(uploadId, 0) + } + }) + + it('Should not accept more chunks than expected with an invalid content length', async function () { + const uploadId = await prepareUpload({ size: 500 }) + + const size = 1000 + + // Content length check seems to have changed in v16 + const expectedStatus = process.version.startsWith('v16') + ? HttpStatusCode.CONFLICT_409 + : HttpStatusCode.BAD_REQUEST_400 + + const contentRangeBuilder = (start: number) => `bytes ${start}-${start + size - 1}/${size}` + await sendChunks({ pathUploadId: uploadId, expectedStatus, contentRangeBuilder, contentLength: size }) + await checkFileSize(uploadId, 0) + }) + + it('Should be able to accept 2 PUT requests', async function () { + const uploadId = await prepareUpload() + + const result1 = await sendChunks({ pathUploadId: uploadId }) + const result2 = await sendChunks({ pathUploadId: uploadId }) + + expect(result1.body.video.uuid).to.exist + expect(result1.body.video.uuid).to.equal(result2.body.video.uuid) + + expect(result1.headers['x-resumable-upload-cached']).to.not.exist + expect(result2.headers['x-resumable-upload-cached']).to.equal('true') + + await checkFileSize(uploadId, null) + }) + + it('Should not have the same upload id with 2 different users', async function () { + const originalName = 'toto.mp4' + const lastModified = new Date().getTime() + + const uploadId1 = await prepareUpload({ originalName, lastModified, token: server.accessToken }) + const uploadId2 = await prepareUpload({ originalName, lastModified, channelId: userChannelId, token: userAccessToken }) + + expect(uploadId1).to.not.equal(uploadId2) + }) + + it('Should have the same upload id with the same user', async function () { + const originalName = 'toto.mp4' + const lastModified = new Date().getTime() + + const uploadId1 = await prepareUpload({ originalName, lastModified }) + const uploadId2 = await prepareUpload({ originalName, lastModified }) + + expect(uploadId1).to.equal(uploadId2) + }) + + it('Should not cache a request with 2 different users', async function () { + const originalName = 'toto.mp4' + const lastModified = new Date().getTime() + + const uploadId = await prepareUpload({ originalName, lastModified, token: server.accessToken }) + + await sendChunks({ pathUploadId: uploadId, token: server.accessToken }) + await sendChunks({ pathUploadId: uploadId, token: userAccessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should not cache a request after a delete', async function () { + const originalName = 'toto.mp4' + const lastModified = new Date().getTime() + const uploadId1 = await prepareUpload({ originalName, lastModified, token: server.accessToken }) + + await sendChunks({ pathUploadId: uploadId1 }) + await server.videos.endResumableUpload({ path, pathUploadId: uploadId1 }) + + const uploadId2 = await prepareUpload({ originalName, lastModified, token: server.accessToken }) + expect(uploadId1).to.equal(uploadId2) + + const result2 = await sendChunks({ pathUploadId: uploadId1 }) + expect(result2.headers['x-resumable-upload-cached']).to.not.exist + }) + + it('Should not cache after video deletion', async function () { + const originalName = 'toto.mp4' + const lastModified = new Date().getTime() + + const uploadId1 = await prepareUpload({ originalName, lastModified }) + const result1 = await sendChunks({ pathUploadId: uploadId1 }) + await server.videos.remove({ id: result1.body.video.uuid }) + + const uploadId2 = await prepareUpload({ originalName, lastModified }) + const result2 = await sendChunks({ pathUploadId: uploadId2 }) + expect(result1.body.video.uuid).to.not.equal(result2.body.video.uuid) + + expect(result2.headers['x-resumable-upload-cached']).to.not.exist + + await checkFileSize(uploadId1, null) + await checkFileSize(uploadId2, null) + }) + + it('Should refuse an invalid digest', async function () { + const uploadId = await prepareUpload({ token: server.accessToken }) + + await sendChunks({ + pathUploadId: uploadId, + token: server.accessToken, + digestBuilder: () => 'sha=' + 'a'.repeat(40), + expectedStatus: 460 as any + }) + }) + + it('Should accept an appropriate digest', async function () { + const uploadId = await prepareUpload({ token: server.accessToken }) + + await sendChunks({ + pathUploadId: uploadId, + token: server.accessToken, + digestBuilder: (chunk: Buffer) => { + return 'sha1=' + sha1(chunk, 'base64') + } + }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/videos/single-server.ts b/packages/tests/src/api/videos/single-server.ts new file mode 100644 index 000000000..b87192a57 --- /dev/null +++ b/packages/tests/src/api/videos/single-server.ts @@ -0,0 +1,461 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { Video, VideoPrivacy } from '@peertube/peertube-models' +import { checkVideoFilesWereRemoved, completeVideoCheck } from '@tests/shared/videos.js' +import { testImageGeneratedByFFmpeg } from '@tests/shared/checks.js' +import { + cleanupTests, + createSingleServer, + PeerTubeServer, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultChannelAvatar, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test a single server', function () { + + function runSuite (mode: 'legacy' | 'resumable') { + let server: PeerTubeServer = null + let videoId: number | string + let videoId2: string + let videoUUID = '' + let videosListBase: any[] = null + + const getCheckAttributes = () => ({ + name: 'my super name', + category: 2, + licence: 6, + language: 'zh', + nsfw: true, + description: 'my super description', + support: 'my super support text', + account: { + name: 'root', + host: server.host + }, + isLocal: true, + duration: 5, + tags: [ 'tag1', 'tag2', 'tag3' ], + privacy: VideoPrivacy.PUBLIC, + commentsEnabled: true, + downloadEnabled: true, + channel: { + displayName: 'Main root channel', + name: 'root_channel', + description: '', + isLocal: true + }, + fixture: 'video_short.webm', + files: [ + { + resolution: 720, + size: 218910 + } + ] + }) + + const updateCheckAttributes = () => ({ + name: 'my super video updated', + category: 4, + licence: 2, + language: 'ar', + nsfw: false, + description: 'my super description updated', + support: 'my super support text updated', + account: { + name: 'root', + host: server.host + }, + isLocal: true, + tags: [ 'tagup1', 'tagup2' ], + privacy: VideoPrivacy.PUBLIC, + duration: 5, + commentsEnabled: false, + downloadEnabled: false, + channel: { + name: 'root_channel', + displayName: 'Main root channel', + description: '', + isLocal: true + }, + fixture: 'video_short3.webm', + files: [ + { + resolution: 720, + size: 292677 + } + ] + }) + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1, {}) + + await setAccessTokensToServers([ server ]) + await setDefaultChannelAvatar(server) + await setDefaultAccountAvatar(server) + }) + + it('Should list video categories', async function () { + const categories = await server.videos.getCategories() + expect(Object.keys(categories)).to.have.length.above(10) + + expect(categories[11]).to.equal('News & Politics') + }) + + it('Should list video licences', async function () { + const licences = await server.videos.getLicences() + expect(Object.keys(licences)).to.have.length.above(5) + + expect(licences[3]).to.equal('Attribution - No Derivatives') + }) + + it('Should list video languages', async function () { + const languages = await server.videos.getLanguages() + expect(Object.keys(languages)).to.have.length.above(5) + + expect(languages['ru']).to.equal('Russian') + }) + + it('Should list video privacies', async function () { + const privacies = await server.videos.getPrivacies() + expect(Object.keys(privacies)).to.have.length.at.least(3) + + expect(privacies[3]).to.equal('Private') + }) + + it('Should not have videos', async function () { + const { data, total } = await server.videos.list() + + expect(total).to.equal(0) + expect(data).to.be.an('array') + expect(data.length).to.equal(0) + }) + + it('Should upload the video', async function () { + const attributes = { + name: 'my super name', + category: 2, + nsfw: true, + licence: 6, + tags: [ 'tag1', 'tag2', 'tag3' ] + } + const video = await server.videos.upload({ attributes, mode }) + expect(video).to.not.be.undefined + expect(video.id).to.equal(1) + expect(video.uuid).to.have.length.above(5) + + videoId = video.id + videoUUID = video.uuid + }) + + it('Should get and seed the uploaded video', async function () { + this.timeout(5000) + + const { data, total } = await server.videos.list() + + expect(total).to.equal(1) + expect(data).to.be.an('array') + expect(data.length).to.equal(1) + + const video = data[0] + await completeVideoCheck({ server, originServer: server, videoUUID: video.uuid, attributes: getCheckAttributes() }) + }) + + it('Should get the video by UUID', async function () { + this.timeout(5000) + + const video = await server.videos.get({ id: videoUUID }) + await completeVideoCheck({ server, originServer: server, videoUUID: video.uuid, attributes: getCheckAttributes() }) + }) + + it('Should have the views updated', async function () { + this.timeout(20000) + + await server.views.simulateView({ id: videoId }) + await server.views.simulateView({ id: videoId }) + await server.views.simulateView({ id: videoId }) + + await wait(1500) + + await server.views.simulateView({ id: videoId }) + await server.views.simulateView({ id: videoId }) + + await wait(1500) + + await server.views.simulateView({ id: videoId }) + await server.views.simulateView({ id: videoId }) + + await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } }) + + const video = await server.videos.get({ id: videoId }) + expect(video.views).to.equal(3) + }) + + it('Should remove the video', async function () { + const video = await server.videos.get({ id: videoId }) + await server.videos.remove({ id: videoId }) + + await checkVideoFilesWereRemoved({ video, server }) + }) + + it('Should not have videos', async function () { + const { total, data } = await server.videos.list() + + expect(total).to.equal(0) + expect(data).to.be.an('array') + expect(data).to.have.lengthOf(0) + }) + + it('Should upload 6 videos', async function () { + this.timeout(120000) + + const videos = new Set([ + 'video_short.mp4', 'video_short.ogv', 'video_short.webm', + 'video_short1.webm', 'video_short2.webm', 'video_short3.webm' + ]) + + for (const video of videos) { + const attributes = { + name: video + ' name', + description: video + ' description', + category: 2, + licence: 1, + language: 'en', + nsfw: true, + tags: [ 'tag1', 'tag2', 'tag3' ], + fixture: video + } + + await server.videos.upload({ attributes, mode }) + } + }) + + it('Should have the correct durations', async function () { + const { total, data } = await server.videos.list() + + expect(total).to.equal(6) + expect(data).to.be.an('array') + expect(data).to.have.lengthOf(6) + + const videosByName: { [ name: string ]: Video } = {} + data.forEach(v => { videosByName[v.name] = v }) + + expect(videosByName['video_short.mp4 name'].duration).to.equal(5) + expect(videosByName['video_short.ogv name'].duration).to.equal(5) + expect(videosByName['video_short.webm name'].duration).to.equal(5) + expect(videosByName['video_short1.webm name'].duration).to.equal(10) + expect(videosByName['video_short2.webm name'].duration).to.equal(5) + expect(videosByName['video_short3.webm name'].duration).to.equal(5) + }) + + it('Should have the correct thumbnails', async function () { + const { data } = await server.videos.list() + + // For the next test + videosListBase = data + + for (const video of data) { + const videoName = video.name.replace(' name', '') + await testImageGeneratedByFFmpeg(server.url, videoName, video.thumbnailPath) + } + }) + + it('Should list only the two first videos', async function () { + const { total, data } = await server.videos.list({ start: 0, count: 2, sort: 'name' }) + + expect(total).to.equal(6) + expect(data.length).to.equal(2) + expect(data[0].name).to.equal(videosListBase[0].name) + expect(data[1].name).to.equal(videosListBase[1].name) + }) + + it('Should list only the next three videos', async function () { + const { total, data } = await server.videos.list({ start: 2, count: 3, sort: 'name' }) + + expect(total).to.equal(6) + expect(data.length).to.equal(3) + expect(data[0].name).to.equal(videosListBase[2].name) + expect(data[1].name).to.equal(videosListBase[3].name) + expect(data[2].name).to.equal(videosListBase[4].name) + }) + + it('Should list the last video', async function () { + const { total, data } = await server.videos.list({ start: 5, count: 6, sort: 'name' }) + + expect(total).to.equal(6) + expect(data.length).to.equal(1) + expect(data[0].name).to.equal(videosListBase[5].name) + }) + + it('Should not have the total field', async function () { + const { total, data } = await server.videos.list({ start: 5, count: 6, sort: 'name', skipCount: true }) + + expect(total).to.not.exist + expect(data.length).to.equal(1) + expect(data[0].name).to.equal(videosListBase[5].name) + }) + + it('Should list and sort by name in descending order', async function () { + const { total, data } = await server.videos.list({ sort: '-name' }) + + expect(total).to.equal(6) + expect(data.length).to.equal(6) + expect(data[0].name).to.equal('video_short.webm name') + expect(data[1].name).to.equal('video_short.ogv name') + expect(data[2].name).to.equal('video_short.mp4 name') + expect(data[3].name).to.equal('video_short3.webm name') + expect(data[4].name).to.equal('video_short2.webm name') + expect(data[5].name).to.equal('video_short1.webm name') + + videoId = data[3].uuid + videoId2 = data[5].uuid + }) + + it('Should list and sort by trending in descending order', async function () { + const { total, data } = await server.videos.list({ start: 0, count: 2, sort: '-trending' }) + + expect(total).to.equal(6) + expect(data.length).to.equal(2) + }) + + it('Should list and sort by hotness in descending order', async function () { + const { total, data } = await server.videos.list({ start: 0, count: 2, sort: '-hot' }) + + expect(total).to.equal(6) + expect(data.length).to.equal(2) + }) + + it('Should list and sort by best in descending order', async function () { + const { total, data } = await server.videos.list({ start: 0, count: 2, sort: '-best' }) + + expect(total).to.equal(6) + expect(data.length).to.equal(2) + }) + + it('Should update a video', async function () { + const attributes = { + name: 'my super video updated', + category: 4, + licence: 2, + language: 'ar', + nsfw: false, + description: 'my super description updated', + commentsEnabled: false, + downloadEnabled: false, + tags: [ 'tagup1', 'tagup2' ] + } + await server.videos.update({ id: videoId, attributes }) + }) + + it('Should have the video updated', async function () { + this.timeout(60000) + + await waitJobs([ server ]) + + const video = await server.videos.get({ id: videoId }) + + await completeVideoCheck({ server, originServer: server, videoUUID: video.uuid, attributes: updateCheckAttributes() }) + }) + + it('Should update only the tags of a video', async function () { + const attributes = { + tags: [ 'supertag', 'tag1', 'tag2' ] + } + await server.videos.update({ id: videoId, attributes }) + + const video = await server.videos.get({ id: videoId }) + + await completeVideoCheck({ + server, + originServer: server, + videoUUID: video.uuid, + attributes: Object.assign(updateCheckAttributes(), attributes) + }) + }) + + it('Should update only the description of a video', async function () { + const attributes = { + description: 'hello everybody' + } + await server.videos.update({ id: videoId, attributes }) + + const video = await server.videos.get({ id: videoId }) + + await completeVideoCheck({ + server, + originServer: server, + videoUUID: video.uuid, + attributes: Object.assign(updateCheckAttributes(), { tags: [ 'supertag', 'tag1', 'tag2' ] }, attributes) + }) + }) + + it('Should like a video', async function () { + await server.videos.rate({ id: videoId, rating: 'like' }) + + const video = await server.videos.get({ id: videoId }) + + expect(video.likes).to.equal(1) + expect(video.dislikes).to.equal(0) + }) + + it('Should dislike the same video', async function () { + await server.videos.rate({ id: videoId, rating: 'dislike' }) + + const video = await server.videos.get({ id: videoId }) + + expect(video.likes).to.equal(0) + expect(video.dislikes).to.equal(1) + }) + + it('Should sort by originallyPublishedAt', async function () { + { + const now = new Date() + const attributes = { originallyPublishedAt: now.toISOString() } + await server.videos.update({ id: videoId, attributes }) + + const { data } = await server.videos.list({ sort: '-originallyPublishedAt' }) + const names = data.map(v => v.name) + + expect(names[0]).to.equal('my super video updated') + expect(names[1]).to.equal('video_short2.webm name') + expect(names[2]).to.equal('video_short1.webm name') + expect(names[3]).to.equal('video_short.webm name') + expect(names[4]).to.equal('video_short.ogv name') + expect(names[5]).to.equal('video_short.mp4 name') + } + + { + const now = new Date() + const attributes = { originallyPublishedAt: now.toISOString() } + await server.videos.update({ id: videoId2, attributes }) + + const { data } = await server.videos.list({ sort: '-originallyPublishedAt' }) + const names = data.map(v => v.name) + + expect(names[0]).to.equal('video_short1.webm name') + expect(names[1]).to.equal('my super video updated') + expect(names[2]).to.equal('video_short2.webm name') + expect(names[3]).to.equal('video_short.webm name') + expect(names[4]).to.equal('video_short.ogv name') + expect(names[5]).to.equal('video_short.mp4 name') + } + }) + + after(async function () { + await cleanupTests([ server ]) + }) + } + + describe('Legacy upload', function () { + runSuite('legacy') + }) + + describe('Resumable upload', function () { + runSuite('resumable') + }) +}) diff --git a/packages/tests/src/api/videos/video-captions.ts b/packages/tests/src/api/videos/video-captions.ts new file mode 100644 index 000000000..027022549 --- /dev/null +++ b/packages/tests/src/api/videos/video-captions.ts @@ -0,0 +1,189 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' +import { testCaptionFile } from '@tests/shared/captions.js' +import { checkVideoFilesWereRemoved } from '@tests/shared/videos.js' + +describe('Test video captions', function () { + const uuidRegex = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' + + let servers: PeerTubeServer[] + let videoUUID: string + + before(async function () { + this.timeout(60000) + + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + await doubleFollow(servers[0], servers[1]) + + await waitJobs(servers) + + const { uuid } = await servers[0].videos.upload({ attributes: { name: 'my video name' } }) + videoUUID = uuid + + await waitJobs(servers) + }) + + it('Should list the captions and return an empty list', async function () { + for (const server of servers) { + const body = await server.captions.list({ videoId: videoUUID }) + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + } + }) + + it('Should create two new captions', async function () { + this.timeout(30000) + + await servers[0].captions.add({ + language: 'ar', + videoId: videoUUID, + fixture: 'subtitle-good1.vtt' + }) + + await servers[0].captions.add({ + language: 'zh', + videoId: videoUUID, + fixture: 'subtitle-good2.vtt', + mimeType: 'application/octet-stream' + }) + + await waitJobs(servers) + }) + + it('Should list these uploaded captions', async function () { + for (const server of servers) { + const body = await server.captions.list({ videoId: videoUUID }) + expect(body.total).to.equal(2) + expect(body.data).to.have.lengthOf(2) + + const caption1 = body.data[0] + expect(caption1.language.id).to.equal('ar') + expect(caption1.language.label).to.equal('Arabic') + expect(caption1.captionPath).to.match(new RegExp('^/lazy-static/video-captions/' + uuidRegex + '-ar.vtt$')) + await testCaptionFile(server.url, caption1.captionPath, 'Subtitle good 1.') + + const caption2 = body.data[1] + expect(caption2.language.id).to.equal('zh') + expect(caption2.language.label).to.equal('Chinese') + expect(caption2.captionPath).to.match(new RegExp('^/lazy-static/video-captions/' + uuidRegex + '-zh.vtt$')) + await testCaptionFile(server.url, caption2.captionPath, 'Subtitle good 2.') + } + }) + + it('Should replace an existing caption', async function () { + this.timeout(30000) + + await servers[0].captions.add({ + language: 'ar', + videoId: videoUUID, + fixture: 'subtitle-good2.vtt' + }) + + await waitJobs(servers) + }) + + it('Should have this caption updated', async function () { + for (const server of servers) { + const body = await server.captions.list({ videoId: videoUUID }) + expect(body.total).to.equal(2) + expect(body.data).to.have.lengthOf(2) + + const caption1 = body.data[0] + expect(caption1.language.id).to.equal('ar') + expect(caption1.language.label).to.equal('Arabic') + expect(caption1.captionPath).to.match(new RegExp('^/lazy-static/video-captions/' + uuidRegex + '-ar.vtt$')) + await testCaptionFile(server.url, caption1.captionPath, 'Subtitle good 2.') + } + }) + + it('Should replace an existing caption with a srt file and convert it', async function () { + this.timeout(30000) + + await servers[0].captions.add({ + language: 'ar', + videoId: videoUUID, + fixture: 'subtitle-good.srt' + }) + + await waitJobs(servers) + + // Cache invalidation + await wait(3000) + }) + + it('Should have this caption updated and converted', async function () { + for (const server of servers) { + const body = await server.captions.list({ videoId: videoUUID }) + expect(body.total).to.equal(2) + expect(body.data).to.have.lengthOf(2) + + const caption1 = body.data[0] + expect(caption1.language.id).to.equal('ar') + expect(caption1.language.label).to.equal('Arabic') + expect(caption1.captionPath).to.match(new RegExp('^/lazy-static/video-captions/' + uuidRegex + '-ar.vtt$')) + + const expected = 'WEBVTT FILE\r\n' + + '\r\n' + + '1\r\n' + + '00:00:01.600 --> 00:00:04.200\r\n' + + 'English (US)\r\n' + + '\r\n' + + '2\r\n' + + '00:00:05.900 --> 00:00:07.999\r\n' + + 'This is a subtitle in American English\r\n' + + '\r\n' + + '3\r\n' + + '00:00:10.000 --> 00:00:14.000\r\n' + + 'Adding subtitles is very easy to do\r\n' + await testCaptionFile(server.url, caption1.captionPath, expected) + } + }) + + it('Should remove one caption', async function () { + this.timeout(30000) + + await servers[0].captions.delete({ videoId: videoUUID, language: 'ar' }) + + await waitJobs(servers) + }) + + it('Should only list the caption that was not deleted', async function () { + for (const server of servers) { + const body = await server.captions.list({ videoId: videoUUID }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + + const caption = body.data[0] + + expect(caption.language.id).to.equal('zh') + expect(caption.language.label).to.equal('Chinese') + expect(caption.captionPath).to.match(new RegExp('^/lazy-static/video-captions/' + uuidRegex + '-zh.vtt$')) + await testCaptionFile(server.url, caption.captionPath, 'Subtitle good 2.') + } + }) + + it('Should remove the video, and thus all video captions', async function () { + const video = await servers[0].videos.get({ id: videoUUID }) + const { data: captions } = await servers[0].captions.list({ videoId: videoUUID }) + + await servers[0].videos.remove({ id: videoUUID }) + + await checkVideoFilesWereRemoved({ server: servers[0], video, captions }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/videos/video-change-ownership.ts b/packages/tests/src/api/videos/video-change-ownership.ts new file mode 100644 index 000000000..717c37469 --- /dev/null +++ b/packages/tests/src/api/videos/video-change-ownership.ts @@ -0,0 +1,314 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { + ChangeOwnershipCommand, + cleanupTests, + createMultipleServers, + createSingleServer, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' +import { HttpStatusCode, VideoPrivacy } from '@peertube/peertube-models' + +describe('Test video change ownership - nominal', function () { + let servers: PeerTubeServer[] = [] + + const firstUser = 'first' + const secondUser = 'second' + + let firstUserToken = '' + let firstUserChannelId: number + + let secondUserToken = '' + let secondUserChannelId: number + + let lastRequestId: number + + let liveId: number + + let command: ChangeOwnershipCommand + + before(async function () { + this.timeout(240000) + + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + await servers[0].config.updateCustomSubConfig({ + newConfig: { + transcoding: { + enabled: false + }, + live: { + enabled: true + } + } + }) + + firstUserToken = await servers[0].users.generateUserAndToken(firstUser) + secondUserToken = await servers[0].users.generateUserAndToken(secondUser) + + { + const { videoChannels } = await servers[0].users.getMyInfo({ token: firstUserToken }) + firstUserChannelId = videoChannels[0].id + } + + { + const { videoChannels } = await servers[0].users.getMyInfo({ token: secondUserToken }) + secondUserChannelId = videoChannels[0].id + } + + { + const attributes = { + name: 'my super name', + description: 'my super description' + } + const { id } = await servers[0].videos.upload({ token: firstUserToken, attributes }) + + servers[0].store.videoCreated = await servers[0].videos.get({ id }) + } + + { + const attributes = { name: 'live', channelId: firstUserChannelId, privacy: VideoPrivacy.PUBLIC } + const video = await servers[0].live.create({ token: firstUserToken, fields: attributes }) + + liveId = video.id + } + + command = servers[0].changeOwnership + + await doubleFollow(servers[0], servers[1]) + }) + + it('Should not have video change ownership', async function () { + { + const body = await command.list({ token: firstUserToken }) + + expect(body.total).to.equal(0) + expect(body.data).to.be.an('array') + expect(body.data.length).to.equal(0) + } + + { + const body = await command.list({ token: secondUserToken }) + + expect(body.total).to.equal(0) + expect(body.data).to.be.an('array') + expect(body.data.length).to.equal(0) + } + }) + + it('Should send a request to change ownership of a video', async function () { + this.timeout(15000) + + await command.create({ token: firstUserToken, videoId: servers[0].store.videoCreated.id, username: secondUser }) + }) + + it('Should only return a request to change ownership for the second user', async function () { + { + const body = await command.list({ token: firstUserToken }) + + expect(body.total).to.equal(0) + expect(body.data).to.be.an('array') + expect(body.data.length).to.equal(0) + } + + { + const body = await command.list({ token: secondUserToken }) + + expect(body.total).to.equal(1) + expect(body.data).to.be.an('array') + expect(body.data.length).to.equal(1) + + lastRequestId = body.data[0].id + } + }) + + it('Should accept the same change ownership request without crashing', async function () { + await command.create({ token: firstUserToken, videoId: servers[0].store.videoCreated.id, username: secondUser }) + }) + + it('Should not create multiple change ownership requests while one is waiting', async function () { + const body = await command.list({ token: secondUserToken }) + + expect(body.total).to.equal(1) + expect(body.data).to.be.an('array') + expect(body.data.length).to.equal(1) + }) + + it('Should not be possible to refuse the change of ownership from first user', async function () { + await command.refuse({ token: firstUserToken, ownershipId: lastRequestId, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should be possible to refuse the change of ownership from second user', async function () { + await command.refuse({ token: secondUserToken, ownershipId: lastRequestId }) + }) + + it('Should send a new request to change ownership of a video', async function () { + this.timeout(15000) + + await command.create({ token: firstUserToken, videoId: servers[0].store.videoCreated.id, username: secondUser }) + }) + + it('Should return two requests to change ownership for the second user', async function () { + { + const body = await command.list({ token: firstUserToken }) + + expect(body.total).to.equal(0) + expect(body.data).to.be.an('array') + expect(body.data.length).to.equal(0) + } + + { + const body = await command.list({ token: secondUserToken }) + + expect(body.total).to.equal(2) + expect(body.data).to.be.an('array') + expect(body.data.length).to.equal(2) + + lastRequestId = body.data[0].id + } + }) + + it('Should not be possible to accept the change of ownership from first user', async function () { + await command.accept({ + token: firstUserToken, + ownershipId: lastRequestId, + channelId: secondUserChannelId, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should be possible to accept the change of ownership from second user', async function () { + await command.accept({ token: secondUserToken, ownershipId: lastRequestId, channelId: secondUserChannelId }) + + await waitJobs(servers) + }) + + it('Should have the channel of the video updated', async function () { + for (const server of servers) { + const video = await server.videos.get({ id: servers[0].store.videoCreated.uuid }) + + expect(video.name).to.equal('my super name') + expect(video.channel.displayName).to.equal('Main second channel') + expect(video.channel.name).to.equal('second_channel') + } + }) + + it('Should send a request to change ownership of a live', async function () { + this.timeout(15000) + + await command.create({ token: firstUserToken, videoId: liveId, username: secondUser }) + + const body = await command.list({ token: secondUserToken }) + + expect(body.total).to.equal(3) + expect(body.data.length).to.equal(3) + + lastRequestId = body.data[0].id + }) + + it('Should accept a live ownership change', async function () { + this.timeout(20000) + + await command.accept({ token: secondUserToken, ownershipId: lastRequestId, channelId: secondUserChannelId }) + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: servers[0].store.videoCreated.uuid }) + + expect(video.name).to.equal('my super name') + expect(video.channel.displayName).to.equal('Main second channel') + expect(video.channel.name).to.equal('second_channel') + } + }) + + after(async function () { + await cleanupTests(servers) + }) +}) + +describe('Test video change ownership - quota too small', function () { + let server: PeerTubeServer + const firstUser = 'first' + const secondUser = 'second' + + let firstUserToken = '' + let secondUserToken = '' + let lastRequestId: number + + before(async function () { + this.timeout(50000) + + // Run one server + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + + await server.users.create({ username: secondUser, videoQuota: 10 }) + + firstUserToken = await server.users.generateUserAndToken(firstUser) + secondUserToken = await server.login.getAccessToken(secondUser) + + // Upload some videos on the server + const attributes = { + name: 'my super name', + description: 'my super description' + } + await server.videos.upload({ token: firstUserToken, attributes }) + + await waitJobs(server) + + const { data } = await server.videos.list() + expect(data.length).to.equal(1) + + server.store.videoCreated = data.find(video => video.name === 'my super name') + }) + + it('Should send a request to change ownership of a video', async function () { + this.timeout(15000) + + await server.changeOwnership.create({ token: firstUserToken, videoId: server.store.videoCreated.id, username: secondUser }) + }) + + it('Should only return a request to change ownership for the second user', async function () { + { + const body = await server.changeOwnership.list({ token: firstUserToken }) + + expect(body.total).to.equal(0) + expect(body.data).to.be.an('array') + expect(body.data.length).to.equal(0) + } + + { + const body = await server.changeOwnership.list({ token: secondUserToken }) + + expect(body.total).to.equal(1) + expect(body.data).to.be.an('array') + expect(body.data.length).to.equal(1) + + lastRequestId = body.data[0].id + } + }) + + it('Should not be possible to accept the change of ownership from second user because of exceeded quota', async function () { + const { videoChannels } = await server.users.getMyInfo({ token: secondUserToken }) + const channelId = videoChannels[0].id + + await server.changeOwnership.accept({ + token: secondUserToken, + ownershipId: lastRequestId, + channelId, + expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413 + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/videos/video-channel-syncs.ts b/packages/tests/src/api/videos/video-channel-syncs.ts new file mode 100644 index 000000000..54212bcb5 --- /dev/null +++ b/packages/tests/src/api/videos/video-channel-syncs.ts @@ -0,0 +1,321 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { areHttpImportTestsDisabled } from '@peertube/peertube-node-utils' +import { VideoChannelSyncState, VideoInclude, VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + getServerImportConfig, + PeerTubeServer, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultChannelAvatar, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' +import { SQLCommand } from '@tests/shared/sql-command.js' +import { FIXTURE_URLS } from '@tests/shared/tests.js' + +describe('Test channel synchronizations', function () { + if (areHttpImportTestsDisabled()) return + + function runSuite (mode: 'youtube-dl' | 'yt-dlp') { + + describe('Sync using ' + mode, function () { + let servers: PeerTubeServer[] + let sqlCommands: SQLCommand[] = [] + + let startTestDate: Date + + let rootChannelSyncId: number + const userInfo = { + accessToken: '', + username: 'user1', + channelName: 'user1_channel', + channelId: -1, + syncId: -1 + } + + async function changeDateForSync (channelSyncId: number, newDate: string) { + await sqlCommands[0].updateQuery( + `UPDATE "videoChannelSync" ` + + `SET "createdAt"='${newDate}', "lastSyncAt"='${newDate}' ` + + `WHERE id=${channelSyncId}` + ) + } + + async function listAllVideosOfChannel (channelName: string) { + return servers[0].videos.listByChannel({ + handle: channelName, + include: VideoInclude.NOT_PUBLISHED_STATE + }) + } + + async function forceSyncAll (videoChannelSyncId: number, fromDate = '1970-01-01') { + await changeDateForSync(videoChannelSyncId, fromDate) + + await servers[0].debug.sendCommand({ + body: { + command: 'process-video-channel-sync-latest' + } + }) + + await waitJobs(servers) + } + + before(async function () { + this.timeout(240_000) + + startTestDate = new Date() + + servers = await createMultipleServers(2, getServerImportConfig(mode)) + + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + await setDefaultChannelAvatar(servers) + await setDefaultAccountAvatar(servers) + + await servers[0].config.enableChannelSync() + + { + userInfo.accessToken = await servers[0].users.generateUserAndToken(userInfo.username) + + const { videoChannels } = await servers[0].users.getMyInfo({ token: userInfo.accessToken }) + userInfo.channelId = videoChannels[0].id + } + + sqlCommands = servers.map(s => new SQLCommand(s)) + }) + + it('Should fetch the latest channel videos of a remote channel', async function () { + this.timeout(120_000) + + { + const { video } = await servers[0].imports.importVideo({ + attributes: { + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC, + targetUrl: FIXTURE_URLS.youtube + } + }) + + expect(video.name).to.equal('small video - youtube') + expect(video.waitTranscoding).to.be.true + + const { total } = await listAllVideosOfChannel('root_channel') + expect(total).to.equal(1) + } + + const { videoChannelSync } = await servers[0].channelSyncs.create({ + attributes: { + externalChannelUrl: FIXTURE_URLS.youtubeChannel, + videoChannelId: servers[0].store.channel.id + } + }) + rootChannelSyncId = videoChannelSync.id + + await forceSyncAll(rootChannelSyncId) + + { + const { total, data } = await listAllVideosOfChannel('root_channel') + expect(total).to.equal(2) + expect(data[0].name).to.equal('test') + expect(data[0].waitTranscoding).to.be.true + } + }) + + it('Should add another synchronization', async function () { + const externalChannelUrl = FIXTURE_URLS.youtubeChannel + '?foo=bar' + + const { videoChannelSync } = await servers[0].channelSyncs.create({ + attributes: { + externalChannelUrl, + videoChannelId: servers[0].store.channel.id + } + }) + + expect(videoChannelSync.externalChannelUrl).to.equal(externalChannelUrl) + expect(videoChannelSync.channel.id).to.equal(servers[0].store.channel.id) + expect(videoChannelSync.channel.name).to.equal('root_channel') + expect(videoChannelSync.state.id).to.equal(VideoChannelSyncState.WAITING_FIRST_RUN) + expect(new Date(videoChannelSync.createdAt)).to.be.above(startTestDate).and.to.be.at.most(new Date()) + }) + + it('Should add a synchronization for another user', async function () { + const { videoChannelSync } = await servers[0].channelSyncs.create({ + attributes: { + externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux', + videoChannelId: userInfo.channelId + }, + token: userInfo.accessToken + }) + userInfo.syncId = videoChannelSync.id + }) + + it('Should not import a channel if not asked', async function () { + await waitJobs(servers) + + const { data } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username }) + + expect(data[0].state).to.contain({ + id: VideoChannelSyncState.WAITING_FIRST_RUN, + label: 'Waiting first run' + }) + }) + + it('Should only fetch the videos newer than the creation date', async function () { + this.timeout(120_000) + + await forceSyncAll(userInfo.syncId, '2019-03-01') + + const { data, total } = await listAllVideosOfChannel(userInfo.channelName) + + expect(total).to.equal(1) + expect(data[0].name).to.equal('test') + }) + + it('Should list channel synchronizations', async function () { + // Root + { + const { total, data } = await servers[0].channelSyncs.listByAccount({ accountName: 'root' }) + expect(total).to.equal(2) + + expect(data[0]).to.deep.contain({ + externalChannelUrl: FIXTURE_URLS.youtubeChannel, + state: { + id: VideoChannelSyncState.SYNCED, + label: 'Synchronized' + } + }) + + expect(new Date(data[0].lastSyncAt)).to.be.greaterThan(startTestDate) + + expect(data[0].channel).to.contain({ id: servers[0].store.channel.id }) + expect(data[1]).to.contain({ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?foo=bar' }) + } + + // User + { + const { total, data } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username }) + expect(total).to.equal(1) + expect(data[0]).to.deep.contain({ + externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux', + state: { + id: VideoChannelSyncState.SYNCED, + label: 'Synchronized' + } + }) + } + }) + + it('Should list imports of a channel synchronization', async function () { + const { total, data } = await servers[0].imports.getMyVideoImports({ videoChannelSyncId: rootChannelSyncId }) + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + expect(data[0].video.name).to.equal('test') + }) + + it('Should remove user\'s channel synchronizations', async function () { + await servers[0].channelSyncs.delete({ channelSyncId: userInfo.syncId }) + + const { total } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username }) + expect(total).to.equal(0) + }) + + // FIXME: youtube-dl/yt-dlp doesn't work when speicifying a port after the hostname + // it('Should import a remote PeerTube channel', async function () { + // this.timeout(240_000) + + // await servers[1].videos.quickUpload({ name: 'remote 1' }) + // await waitJobs(servers) + + // const { videoChannelSync } = await servers[0].channelSyncs.create({ + // attributes: { + // externalChannelUrl: servers[1].url + '/c/root_channel', + // videoChannelId: userInfo.channelId + // }, + // token: userInfo.accessToken + // }) + // await servers[0].channels.importVideos({ + // channelName: userInfo.channelName, + // externalChannelUrl: servers[1].url + '/c/root_channel', + // videoChannelSyncId: videoChannelSync.id, + // token: userInfo.accessToken + // }) + + // await waitJobs(servers) + + // const { data, total } = await servers[0].videos.listByChannel({ + // handle: userInfo.channelName, + // include: VideoInclude.NOT_PUBLISHED_STATE + // }) + + // expect(total).to.equal(2) + // expect(data[0].name).to.equal('remote 1') + // }) + + // it('Should keep synced a remote PeerTube channel', async function () { + // this.timeout(240_000) + + // await servers[1].videos.quickUpload({ name: 'remote 2' }) + // await waitJobs(servers) + + // await servers[0].debug.sendCommand({ + // body: { + // command: 'process-video-channel-sync-latest' + // } + // }) + + // await waitJobs(servers) + + // const { data, total } = await servers[0].videos.listByChannel({ + // handle: userInfo.channelName, + // include: VideoInclude.NOT_PUBLISHED_STATE + // }) + // expect(total).to.equal(2) + // expect(data[0].name).to.equal('remote 2') + // }) + + it('Should fetch the latest videos of a youtube playlist', async function () { + this.timeout(120_000) + + const { id: channelId } = await servers[0].channels.create({ + attributes: { + name: 'channel2' + } + }) + + const { videoChannelSync: { id: videoChannelSyncId } } = await servers[0].channelSyncs.create({ + attributes: { + externalChannelUrl: FIXTURE_URLS.youtubePlaylist, + videoChannelId: channelId + } + }) + + await forceSyncAll(videoChannelSyncId) + + { + + const { total, data } = await listAllVideosOfChannel('channel2') + expect(total).to.equal(2) + expect(data[0].name).to.equal('test') + expect(data[1].name).to.equal('small video - youtube') + } + }) + + after(async function () { + for (const sqlCommand of sqlCommands) { + await sqlCommand.cleanup() + } + + await cleanupTests(servers) + }) + }) + } + + // FIXME: suite is broken with youtube-dl + // runSuite('youtube-dl') + runSuite('yt-dlp') +}) diff --git a/packages/tests/src/api/videos/video-channels.ts b/packages/tests/src/api/videos/video-channels.ts new file mode 100644 index 000000000..64b1b9315 --- /dev/null +++ b/packages/tests/src/api/videos/video-channels.ts @@ -0,0 +1,556 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { basename } from 'path' +import { ACTOR_IMAGES_SIZE } from '@peertube/peertube-server/server/initializers/constants.js' +import { testFileExistsOrNot, testImage } from '@tests/shared/checks.js' +import { SQLCommand } from '@tests/shared/sql-command.js' +import { wait } from '@peertube/peertube-core-utils' +import { ActorImageType, User, VideoChannel } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' + +async function findChannel (server: PeerTubeServer, channelId: number) { + const body = await server.channels.list({ sort: '-name' }) + + return body.data.find(c => c.id === channelId) +} + +describe('Test video channels', function () { + let servers: PeerTubeServer[] + let sqlCommands: SQLCommand[] = [] + + let userInfo: User + let secondVideoChannelId: number + let totoChannel: number + let videoUUID: string + let accountName: string + let secondUserChannelName: string + + const avatarPaths: { [ port: number ]: string } = {} + const bannerPaths: { [ port: number ]: string } = {} + + before(async function () { + this.timeout(60000) + + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + await setDefaultAccountAvatar(servers) + + await doubleFollow(servers[0], servers[1]) + + sqlCommands = servers.map(s => new SQLCommand(s)) + }) + + it('Should have one video channel (created with root)', async () => { + const body = await servers[0].channels.list({ start: 0, count: 2 }) + + expect(body.total).to.equal(1) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(1) + }) + + it('Should create another video channel', async function () { + this.timeout(30000) + + { + const videoChannel = { + name: 'second_video_channel', + displayName: 'second video channel', + description: 'super video channel description', + support: 'super video channel support text' + } + const created = await servers[0].channels.create({ attributes: videoChannel }) + secondVideoChannelId = created.id + } + + // The channel is 1 is propagated to servers 2 + { + const attributes = { name: 'my video name', channelId: secondVideoChannelId, support: 'video support field' } + const { uuid } = await servers[0].videos.upload({ attributes }) + videoUUID = uuid + } + + await waitJobs(servers) + }) + + it('Should have two video channels when getting my information', async () => { + userInfo = await servers[0].users.getMyInfo() + + expect(userInfo.videoChannels).to.be.an('array') + expect(userInfo.videoChannels).to.have.lengthOf(2) + + const videoChannels = userInfo.videoChannels + expect(videoChannels[0].name).to.equal('root_channel') + expect(videoChannels[0].displayName).to.equal('Main root channel') + + expect(videoChannels[1].name).to.equal('second_video_channel') + expect(videoChannels[1].displayName).to.equal('second video channel') + expect(videoChannels[1].description).to.equal('super video channel description') + expect(videoChannels[1].support).to.equal('super video channel support text') + + accountName = userInfo.account.name + '@' + userInfo.account.host + }) + + it('Should have two video channels when getting account channels on server 1', async function () { + const body = await servers[0].channels.listByAccount({ accountName }) + expect(body.total).to.equal(2) + + const videoChannels = body.data + + expect(videoChannels).to.be.an('array') + expect(videoChannels).to.have.lengthOf(2) + + expect(videoChannels[0].name).to.equal('root_channel') + expect(videoChannels[0].displayName).to.equal('Main root channel') + + expect(videoChannels[1].name).to.equal('second_video_channel') + expect(videoChannels[1].displayName).to.equal('second video channel') + expect(videoChannels[1].description).to.equal('super video channel description') + expect(videoChannels[1].support).to.equal('super video channel support text') + }) + + it('Should paginate and sort account channels', async function () { + { + const body = await servers[0].channels.listByAccount({ + accountName, + start: 0, + count: 1, + sort: 'createdAt' + }) + + expect(body.total).to.equal(2) + expect(body.data).to.have.lengthOf(1) + + const videoChannel: VideoChannel = body.data[0] + expect(videoChannel.name).to.equal('root_channel') + } + + { + const body = await servers[0].channels.listByAccount({ + accountName, + start: 0, + count: 1, + sort: '-createdAt' + }) + + expect(body.total).to.equal(2) + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].name).to.equal('second_video_channel') + } + + { + const body = await servers[0].channels.listByAccount({ + accountName, + start: 1, + count: 1, + sort: '-createdAt' + }) + + expect(body.total).to.equal(2) + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].name).to.equal('root_channel') + } + }) + + it('Should have one video channel when getting account channels on server 2', async function () { + const body = await servers[1].channels.listByAccount({ accountName }) + + expect(body.total).to.equal(1) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(1) + + const videoChannel = body.data[0] + expect(videoChannel.name).to.equal('second_video_channel') + expect(videoChannel.displayName).to.equal('second video channel') + expect(videoChannel.description).to.equal('super video channel description') + expect(videoChannel.support).to.equal('super video channel support text') + }) + + it('Should list video channels', async function () { + const body = await servers[0].channels.list({ start: 1, count: 1, sort: '-name' }) + + expect(body.total).to.equal(2) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].name).to.equal('root_channel') + expect(body.data[0].displayName).to.equal('Main root channel') + }) + + it('Should update video channel', async function () { + this.timeout(15000) + + const videoChannelAttributes = { + displayName: 'video channel updated', + description: 'video channel description updated', + support: 'support updated' + } + + await servers[0].channels.update({ channelName: 'second_video_channel', attributes: videoChannelAttributes }) + + await waitJobs(servers) + }) + + it('Should have video channel updated', async function () { + for (const server of servers) { + const body = await server.channels.list({ start: 0, count: 1, sort: '-name' }) + + expect(body.total).to.equal(2) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(1) + + expect(body.data[0].name).to.equal('second_video_channel') + expect(body.data[0].displayName).to.equal('video channel updated') + expect(body.data[0].description).to.equal('video channel description updated') + expect(body.data[0].support).to.equal('support updated') + } + }) + + it('Should not have updated the video support field', async function () { + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + expect(video.support).to.equal('video support field') + } + }) + + it('Should update another accounts video channel', async function () { + this.timeout(15000) + + const result = await servers[0].users.generate('second_user') + secondUserChannelName = result.userChannelName + + await servers[0].videos.quickUpload({ name: 'video', token: result.token }) + + const videoChannelAttributes = { + displayName: 'video channel updated', + description: 'video channel description updated', + support: 'support updated' + } + + await servers[0].channels.update({ channelName: secondUserChannelName, attributes: videoChannelAttributes }) + + await waitJobs(servers) + }) + + it('Should have another accounts video channel updated', async function () { + for (const server of servers) { + const body = await server.channels.get({ channelName: `${secondUserChannelName}@${servers[0].host}` }) + + expect(body.displayName).to.equal('video channel updated') + expect(body.description).to.equal('video channel description updated') + expect(body.support).to.equal('support updated') + } + }) + + it('Should update the channel support field and update videos too', async function () { + this.timeout(35000) + + const videoChannelAttributes = { + support: 'video channel support text updated', + bulkVideosSupportUpdate: true + } + + await servers[0].channels.update({ channelName: 'second_video_channel', attributes: videoChannelAttributes }) + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + expect(video.support).to.equal(videoChannelAttributes.support) + } + }) + + it('Should update video channel avatar', async function () { + this.timeout(15000) + + const fixture = 'avatar.png' + + await servers[0].channels.updateImage({ + channelName: 'second_video_channel', + fixture, + type: 'avatar' + }) + + await waitJobs(servers) + + for (let i = 0; i < servers.length; i++) { + const server = servers[i] + + const videoChannel = await findChannel(server, secondVideoChannelId) + const expectedSizes = ACTOR_IMAGES_SIZE[ActorImageType.AVATAR] + + expect(videoChannel.avatars.length).to.equal(expectedSizes.length, 'Expected avatars to be generated in all sizes') + + for (const avatar of videoChannel.avatars) { + avatarPaths[server.port] = avatar.path + await testImage(server.url, `avatar-resized-${avatar.width}x${avatar.width}`, avatarPaths[server.port], '.png') + await testFileExistsOrNot(server, 'avatars', basename(avatarPaths[server.port]), true) + + const row = await sqlCommands[i].getActorImage(basename(avatarPaths[server.port])) + + expect(expectedSizes.some(({ height, width }) => row.height === height && row.width === width)).to.equal(true) + } + } + }) + + it('Should update video channel banner', async function () { + this.timeout(15000) + + const fixture = 'banner.jpg' + + await servers[0].channels.updateImage({ + channelName: 'second_video_channel', + fixture, + type: 'banner' + }) + + await waitJobs(servers) + + for (let i = 0; i < servers.length; i++) { + const server = servers[i] + + const videoChannel = await server.channels.get({ channelName: 'second_video_channel@' + servers[0].host }) + + bannerPaths[server.port] = videoChannel.banners[0].path + await testImage(server.url, 'banner-resized', bannerPaths[server.port]) + await testFileExistsOrNot(server, 'avatars', basename(bannerPaths[server.port]), true) + + const row = await sqlCommands[i].getActorImage(basename(bannerPaths[server.port])) + expect(row.height).to.equal(ACTOR_IMAGES_SIZE[ActorImageType.BANNER][0].height) + expect(row.width).to.equal(ACTOR_IMAGES_SIZE[ActorImageType.BANNER][0].width) + } + }) + + it('Should still correctly list channels', async function () { + { + const body = await servers[0].channels.list({ start: 1, count: 1, sort: 'createdAt' }) + + expect(body.total).to.equal(3) + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].name).to.equal('second_video_channel') + } + + { + const body = await servers[0].channels.listByAccount({ accountName, start: 1, count: 1, sort: 'createdAt' }) + + expect(body.total).to.equal(2) + expect(body.data).to.have.lengthOf(1) + expect(body.data[0].name).to.equal('second_video_channel') + } + }) + + it('Should delete the video channel avatar', async function () { + this.timeout(15000) + await servers[0].channels.deleteImage({ channelName: 'second_video_channel', type: 'avatar' }) + + await waitJobs(servers) + + for (const server of servers) { + const videoChannel = await findChannel(server, secondVideoChannelId) + await testFileExistsOrNot(server, 'avatars', basename(avatarPaths[server.port]), false) + + expect(videoChannel.avatars).to.be.empty + } + }) + + it('Should delete the video channel banner', async function () { + this.timeout(15000) + + await servers[0].channels.deleteImage({ channelName: 'second_video_channel', type: 'banner' }) + + await waitJobs(servers) + + for (const server of servers) { + const videoChannel = await findChannel(server, secondVideoChannelId) + await testFileExistsOrNot(server, 'avatars', basename(bannerPaths[server.port]), false) + + expect(videoChannel.banners).to.be.empty + } + }) + + it('Should list the second video channel videos', async function () { + for (const server of servers) { + const channelURI = 'second_video_channel@' + servers[0].host + const { total, data } = await server.videos.listByChannel({ handle: channelURI }) + + expect(total).to.equal(1) + expect(data).to.be.an('array') + expect(data).to.have.lengthOf(1) + expect(data[0].name).to.equal('my video name') + } + }) + + it('Should change the video channel of a video', async function () { + await servers[0].videos.update({ id: videoUUID, attributes: { channelId: servers[0].store.channel.id } }) + + await waitJobs(servers) + }) + + it('Should list the first video channel videos', async function () { + for (const server of servers) { + { + const secondChannelURI = 'second_video_channel@' + servers[0].host + const { total } = await server.videos.listByChannel({ handle: secondChannelURI }) + expect(total).to.equal(0) + } + + { + const channelURI = 'root_channel@' + servers[0].host + const { total, data } = await server.videos.listByChannel({ handle: channelURI }) + expect(total).to.equal(1) + + expect(data).to.be.an('array') + expect(data).to.have.lengthOf(1) + expect(data[0].name).to.equal('my video name') + } + } + }) + + it('Should delete video channel', async function () { + await servers[0].channels.delete({ channelName: 'second_video_channel' }) + }) + + it('Should have video channel deleted', async function () { + const body = await servers[0].channels.list({ start: 0, count: 10, sort: 'createdAt' }) + + expect(body.total).to.equal(2) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(2) + expect(body.data[0].displayName).to.equal('Main root channel') + expect(body.data[1].displayName).to.equal('video channel updated') + }) + + it('Should create the main channel with a suffix if there is a conflict', async function () { + { + const videoChannel = { name: 'toto_channel', displayName: 'My toto channel' } + const created = await servers[0].channels.create({ attributes: videoChannel }) + totoChannel = created.id + } + + { + await servers[0].users.create({ username: 'toto', password: 'password' }) + const accessToken = await servers[0].login.getAccessToken({ username: 'toto', password: 'password' }) + + const { videoChannels } = await servers[0].users.getMyInfo({ token: accessToken }) + expect(videoChannels[0].name).to.equal('toto_channel-1') + } + }) + + it('Should report correct channel views per days', async function () { + { + const { data } = await servers[0].channels.listByAccount({ accountName, withStats: true }) + + for (const channel of data) { + expect(channel).to.haveOwnProperty('viewsPerDay') + expect(channel.viewsPerDay).to.have.length(30 + 1) // daysPrior + today + + for (const v of channel.viewsPerDay) { + expect(v.date).to.be.an('string') + expect(v.views).to.equal(0) + } + } + } + + { + // video has been posted on channel servers[0].store.videoChannel.id since last update + await servers[0].views.simulateView({ id: videoUUID, xForwardedFor: '0.0.0.1,127.0.0.1' }) + await servers[0].views.simulateView({ id: videoUUID, xForwardedFor: '0.0.0.2,127.0.0.1' }) + + // Wait the repeatable job + await wait(8000) + + const { data } = await servers[0].channels.listByAccount({ accountName, withStats: true }) + const channelWithView = data.find(channel => channel.id === servers[0].store.channel.id) + expect(channelWithView.viewsPerDay.slice(-1)[0].views).to.equal(2) + } + }) + + it('Should report correct total views count', async function () { + // check if there's the property + { + const { data } = await servers[0].channels.listByAccount({ accountName, withStats: true }) + + for (const channel of data) { + expect(channel).to.haveOwnProperty('totalViews') + expect(channel.totalViews).to.be.a('number') + } + } + + // Check if the totalViews count can be updated + { + const { data } = await servers[0].channels.listByAccount({ accountName, withStats: true }) + const channelWithView = data.find(channel => channel.id === servers[0].store.channel.id) + expect(channelWithView.totalViews).to.equal(2) + } + }) + + it('Should report correct videos count', async function () { + const { data } = await servers[0].channels.listByAccount({ accountName, withStats: true }) + + const totoChannel = data.find(c => c.name === 'toto_channel') + const rootChannel = data.find(c => c.name === 'root_channel') + + expect(rootChannel.videosCount).to.equal(1) + expect(totoChannel.videosCount).to.equal(0) + }) + + it('Should search among account video channels', async function () { + { + const body = await servers[0].channels.listByAccount({ accountName, search: 'root' }) + expect(body.total).to.equal(1) + + const channels = body.data + expect(channels).to.have.lengthOf(1) + } + + { + const body = await servers[0].channels.listByAccount({ accountName, search: 'does not exist' }) + expect(body.total).to.equal(0) + + const channels = body.data + expect(channels).to.have.lengthOf(0) + } + }) + + it('Should list channels by updatedAt desc if a video has been uploaded', async function () { + this.timeout(30000) + + await servers[0].videos.upload({ attributes: { channelId: totoChannel } }) + await waitJobs(servers) + + for (const server of servers) { + const { data } = await server.channels.listByAccount({ accountName, sort: '-updatedAt' }) + + expect(data[0].name).to.equal('toto_channel') + expect(data[1].name).to.equal('root_channel') + } + + await servers[0].videos.upload({ attributes: { channelId: servers[0].store.channel.id } }) + await waitJobs(servers) + + for (const server of servers) { + const { data } = await server.channels.listByAccount({ accountName, sort: '-updatedAt' }) + + expect(data[0].name).to.equal('root_channel') + expect(data[1].name).to.equal('toto_channel') + } + }) + + after(async function () { + for (const sqlCommand of sqlCommands) { + await sqlCommand.cleanup() + } + + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/videos/video-comments.ts b/packages/tests/src/api/videos/video-comments.ts new file mode 100644 index 000000000..f17db9979 --- /dev/null +++ b/packages/tests/src/api/videos/video-comments.ts @@ -0,0 +1,335 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { dateIsValid, testImage } from '@tests/shared/checks.js' +import { + cleanupTests, + CommentsCommand, + createSingleServer, + PeerTubeServer, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultChannelAvatar +} from '@peertube/peertube-server-commands' + +describe('Test video comments', function () { + let server: PeerTubeServer + let videoId: number + let videoUUID: string + let threadId: number + let replyToDeleteId: number + + let userAccessTokenServer1: string + + let command: CommentsCommand + + before(async function () { + this.timeout(120000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + + const { id, uuid } = await server.videos.upload() + videoUUID = uuid + videoId = id + + await setDefaultChannelAvatar(server) + await setDefaultAccountAvatar(server) + + userAccessTokenServer1 = await server.users.generateUserAndToken('user1') + await setDefaultChannelAvatar(server, 'user1_channel') + await setDefaultAccountAvatar(server, userAccessTokenServer1) + + command = server.comments + }) + + describe('User comments', function () { + + it('Should not have threads on this video', async function () { + const body = await command.listThreads({ videoId: videoUUID }) + + expect(body.total).to.equal(0) + expect(body.totalNotDeletedComments).to.equal(0) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(0) + }) + + it('Should create a thread in this video', async function () { + const text = 'my super first comment' + + const comment = await command.createThread({ videoId: videoUUID, text }) + + expect(comment.inReplyToCommentId).to.be.null + expect(comment.text).equal('my super first comment') + expect(comment.videoId).to.equal(videoId) + expect(comment.id).to.equal(comment.threadId) + expect(comment.account.name).to.equal('root') + expect(comment.account.host).to.equal(server.host) + expect(comment.account.url).to.equal(server.url + '/accounts/root') + expect(comment.totalReplies).to.equal(0) + expect(comment.totalRepliesFromVideoAuthor).to.equal(0) + expect(dateIsValid(comment.createdAt as string)).to.be.true + expect(dateIsValid(comment.updatedAt as string)).to.be.true + }) + + it('Should list threads of this video', async function () { + const body = await command.listThreads({ videoId: videoUUID }) + + expect(body.total).to.equal(1) + expect(body.totalNotDeletedComments).to.equal(1) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(1) + + const comment = body.data[0] + expect(comment.inReplyToCommentId).to.be.null + expect(comment.text).equal('my super first comment') + expect(comment.videoId).to.equal(videoId) + expect(comment.id).to.equal(comment.threadId) + expect(comment.account.name).to.equal('root') + expect(comment.account.host).to.equal(server.host) + + for (const avatar of comment.account.avatars) { + await testImage(server.url, `avatar-resized-${avatar.width}x${avatar.width}`, avatar.path, '.png') + } + + expect(comment.totalReplies).to.equal(0) + expect(comment.totalRepliesFromVideoAuthor).to.equal(0) + expect(dateIsValid(comment.createdAt as string)).to.be.true + expect(dateIsValid(comment.updatedAt as string)).to.be.true + + threadId = comment.threadId + }) + + it('Should get all the thread created', async function () { + const body = await command.getThread({ videoId: videoUUID, threadId }) + + const rootComment = body.comment + expect(rootComment.inReplyToCommentId).to.be.null + expect(rootComment.text).equal('my super first comment') + expect(rootComment.videoId).to.equal(videoId) + expect(dateIsValid(rootComment.createdAt as string)).to.be.true + expect(dateIsValid(rootComment.updatedAt as string)).to.be.true + }) + + it('Should create multiple replies in this thread', async function () { + const text1 = 'my super answer to thread 1' + const created = await command.addReply({ videoId, toCommentId: threadId, text: text1 }) + const childCommentId = created.id + + const text2 = 'my super answer to answer of thread 1' + await command.addReply({ videoId, toCommentId: childCommentId, text: text2 }) + + const text3 = 'my second answer to thread 1' + await command.addReply({ videoId, toCommentId: threadId, text: text3 }) + }) + + it('Should get correctly the replies', async function () { + const tree = await command.getThread({ videoId: videoUUID, threadId }) + + expect(tree.comment.text).equal('my super first comment') + expect(tree.children).to.have.lengthOf(2) + + const firstChild = tree.children[0] + expect(firstChild.comment.text).to.equal('my super answer to thread 1') + expect(firstChild.children).to.have.lengthOf(1) + + const childOfFirstChild = firstChild.children[0] + expect(childOfFirstChild.comment.text).to.equal('my super answer to answer of thread 1') + expect(childOfFirstChild.children).to.have.lengthOf(0) + + const secondChild = tree.children[1] + expect(secondChild.comment.text).to.equal('my second answer to thread 1') + expect(secondChild.children).to.have.lengthOf(0) + + replyToDeleteId = secondChild.comment.id + }) + + it('Should create other threads', async function () { + const text1 = 'super thread 2' + await command.createThread({ videoId: videoUUID, text: text1 }) + + const text2 = 'super thread 3' + await command.createThread({ videoId: videoUUID, text: text2 }) + }) + + it('Should list the threads', async function () { + const body = await command.listThreads({ videoId: videoUUID, sort: 'createdAt' }) + + expect(body.total).to.equal(3) + expect(body.totalNotDeletedComments).to.equal(6) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(3) + + expect(body.data[0].text).to.equal('my super first comment') + expect(body.data[0].totalReplies).to.equal(3) + expect(body.data[1].text).to.equal('super thread 2') + expect(body.data[1].totalReplies).to.equal(0) + expect(body.data[2].text).to.equal('super thread 3') + expect(body.data[2].totalReplies).to.equal(0) + }) + + it('Should list the and sort them by total replies', async function () { + const body = await command.listThreads({ videoId: videoUUID, sort: 'totalReplies' }) + + expect(body.data[2].text).to.equal('my super first comment') + expect(body.data[2].totalReplies).to.equal(3) + }) + + it('Should delete a reply', async function () { + await command.delete({ videoId, commentId: replyToDeleteId }) + + { + const body = await command.listThreads({ videoId: videoUUID, sort: 'createdAt' }) + + expect(body.total).to.equal(3) + expect(body.totalNotDeletedComments).to.equal(5) + } + + { + const tree = await command.getThread({ videoId: videoUUID, threadId }) + + expect(tree.comment.text).equal('my super first comment') + expect(tree.children).to.have.lengthOf(2) + + const firstChild = tree.children[0] + expect(firstChild.comment.text).to.equal('my super answer to thread 1') + expect(firstChild.children).to.have.lengthOf(1) + + const childOfFirstChild = firstChild.children[0] + expect(childOfFirstChild.comment.text).to.equal('my super answer to answer of thread 1') + expect(childOfFirstChild.children).to.have.lengthOf(0) + + const deletedChildOfFirstChild = tree.children[1] + expect(deletedChildOfFirstChild.comment.text).to.equal('') + expect(deletedChildOfFirstChild.comment.isDeleted).to.be.true + expect(deletedChildOfFirstChild.comment.deletedAt).to.not.be.null + expect(deletedChildOfFirstChild.comment.account).to.be.null + expect(deletedChildOfFirstChild.children).to.have.lengthOf(0) + } + }) + + it('Should delete a complete thread', async function () { + await command.delete({ videoId, commentId: threadId }) + + const body = await command.listThreads({ videoId: videoUUID, sort: 'createdAt' }) + expect(body.total).to.equal(3) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(3) + + expect(body.data[0].text).to.equal('') + expect(body.data[0].isDeleted).to.be.true + expect(body.data[0].deletedAt).to.not.be.null + expect(body.data[0].account).to.be.null + expect(body.data[0].totalReplies).to.equal(2) + expect(body.data[1].text).to.equal('super thread 2') + expect(body.data[1].totalReplies).to.equal(0) + expect(body.data[2].text).to.equal('super thread 3') + expect(body.data[2].totalReplies).to.equal(0) + }) + + it('Should count replies from the video author correctly', async function () { + await command.createThread({ videoId: videoUUID, text: 'my super first comment' }) + + const { data } = await command.listThreads({ videoId: videoUUID }) + const threadId2 = data[0].threadId + + const text2 = 'a first answer to thread 4 by a third party' + await command.addReply({ token: userAccessTokenServer1, videoId, toCommentId: threadId2, text: text2 }) + + const text3 = 'my second answer to thread 4' + await command.addReply({ videoId, toCommentId: threadId2, text: text3 }) + + const tree = await command.getThread({ videoId: videoUUID, threadId: threadId2 }) + expect(tree.comment.totalRepliesFromVideoAuthor).to.equal(1) + expect(tree.comment.totalReplies).to.equal(2) + }) + }) + + describe('All instance comments', function () { + + it('Should list instance comments as admin', async function () { + { + const { data, total } = await command.listForAdmin({ start: 0, count: 1 }) + + expect(total).to.equal(7) + expect(data).to.have.lengthOf(1) + expect(data[0].text).to.equal('my second answer to thread 4') + expect(data[0].account.name).to.equal('root') + expect(data[0].account.displayName).to.equal('root') + expect(data[0].account.avatars).to.have.lengthOf(2) + } + + { + const { data, total } = await command.listForAdmin({ start: 1, count: 2 }) + + expect(total).to.equal(7) + expect(data).to.have.lengthOf(2) + + expect(data[0].account.avatars).to.have.lengthOf(2) + expect(data[1].account.avatars).to.have.lengthOf(2) + } + }) + + it('Should filter instance comments by isLocal', async function () { + const { total, data } = await command.listForAdmin({ isLocal: false }) + + expect(data).to.have.lengthOf(0) + expect(total).to.equal(0) + }) + + it('Should filter instance comments by onLocalVideo', async function () { + { + const { total, data } = await command.listForAdmin({ onLocalVideo: false }) + + expect(data).to.have.lengthOf(0) + expect(total).to.equal(0) + } + + { + const { total, data } = await command.listForAdmin({ onLocalVideo: true }) + + expect(data).to.not.have.lengthOf(0) + expect(total).to.not.equal(0) + } + }) + + it('Should search instance comments by account', async function () { + const { total, data } = await command.listForAdmin({ searchAccount: 'user' }) + + expect(data).to.have.lengthOf(1) + expect(total).to.equal(1) + + expect(data[0].text).to.equal('a first answer to thread 4 by a third party') + }) + + it('Should search instance comments by video', async function () { + { + const { total, data } = await command.listForAdmin({ searchVideo: 'video' }) + + expect(data).to.have.lengthOf(7) + expect(total).to.equal(7) + } + + { + const { total, data } = await command.listForAdmin({ searchVideo: 'hello' }) + + expect(data).to.have.lengthOf(0) + expect(total).to.equal(0) + } + }) + + it('Should search instance comments', async function () { + const { total, data } = await command.listForAdmin({ search: 'super thread 3' }) + + expect(total).to.equal(1) + + expect(data).to.have.lengthOf(1) + expect(data[0].text).to.equal('super thread 3') + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/videos/video-description.ts b/packages/tests/src/api/videos/video-description.ts new file mode 100644 index 000000000..eb41cd71c --- /dev/null +++ b/packages/tests/src/api/videos/video-description.ts @@ -0,0 +1,103 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test video description', function () { + let servers: PeerTubeServer[] = [] + let videoUUID = '' + let videoId: number + + const longDescription = 'my super description for server 1'.repeat(50) + + // 30 characters * 6 -> 240 characters + const truncatedDescription = 'my super description for server 1'.repeat(7) + 'my super descrip...' + + before(async function () { + this.timeout(40000) + + // Run servers + servers = await createMultipleServers(2) + + // Get the access tokens + await setAccessTokensToServers(servers) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + }) + + it('Should upload video with long description', async function () { + this.timeout(30000) + + const attributes = { + description: longDescription + } + await servers[0].videos.upload({ attributes }) + + await waitJobs(servers) + + const { data } = await servers[0].videos.list() + + videoId = data[0].id + videoUUID = data[0].uuid + }) + + it('Should have a truncated description on each server when listing videos', async function () { + for (const server of servers) { + const { data } = await server.videos.list() + const video = data.find(v => v.uuid === videoUUID) + + expect(video.description).to.equal(truncatedDescription) + expect(video.truncatedDescription).to.equal(truncatedDescription) + } + }) + + it('Should not have a truncated description on each server when getting videos', async function () { + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + + expect(video.description).to.equal(longDescription) + expect(video.truncatedDescription).to.equal(truncatedDescription) + } + }) + + it('Should fetch long description on each server', async function () { + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + + const { description } = await server.videos.getDescription({ descriptionPath: video.descriptionPath }) + expect(description).to.equal(longDescription) + } + }) + + it('Should update with a short description', async function () { + const attributes = { + description: 'short description' + } + await servers[0].videos.update({ id: videoId, attributes }) + + await waitJobs(servers) + }) + + it('Should have a small description on each server', async function () { + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + + expect(video.description).to.equal('short description') + + const { description } = await server.videos.getDescription({ descriptionPath: video.descriptionPath }) + expect(description).to.equal('short description') + } + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/videos/video-files.ts b/packages/tests/src/api/videos/video-files.ts new file mode 100644 index 000000000..1d7c218a4 --- /dev/null +++ b/packages/tests/src/api/videos/video-files.ts @@ -0,0 +1,202 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + makeRawRequest, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test videos files', function () { + let servers: PeerTubeServer[] + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(150_000) + + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + + await doubleFollow(servers[0], servers[1]) + + await servers[0].config.enableTranscoding({ hls: true, webVideo: true }) + }) + + describe('When deleting all files', function () { + let validId1: string + let validId2: string + + before(async function () { + this.timeout(360_000) + + { + const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' }) + validId1 = uuid + } + + { + const { uuid } = await servers[0].videos.quickUpload({ name: 'video 2' }) + validId2 = uuid + } + + await waitJobs(servers) + }) + + it('Should delete web video files', async function () { + this.timeout(30_000) + + await servers[0].videos.removeAllWebVideoFiles({ videoId: validId1 }) + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: validId1 }) + + expect(video.files).to.have.lengthOf(0) + expect(video.streamingPlaylists).to.have.lengthOf(1) + } + }) + + it('Should delete HLS files', async function () { + this.timeout(30_000) + + await servers[0].videos.removeHLSPlaylist({ videoId: validId2 }) + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: validId2 }) + + expect(video.files).to.have.length.above(0) + expect(video.streamingPlaylists).to.have.lengthOf(0) + } + }) + }) + + describe('When deleting a specific file', function () { + let webVideoId: string + let hlsId: string + + before(async function () { + this.timeout(120_000) + + { + const { uuid } = await servers[0].videos.quickUpload({ name: 'web-video' }) + webVideoId = uuid + } + + { + const { uuid } = await servers[0].videos.quickUpload({ name: 'hls' }) + hlsId = uuid + } + + await waitJobs(servers) + }) + + it('Shoulde delete a web video file', async function () { + this.timeout(30_000) + + const video = await servers[0].videos.get({ id: webVideoId }) + const files = video.files + + await servers[0].videos.removeWebVideoFile({ videoId: webVideoId, fileId: files[0].id }) + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: webVideoId }) + + expect(video.files).to.have.lengthOf(files.length - 1) + expect(video.files.find(f => f.id === files[0].id)).to.not.exist + } + }) + + it('Should delete all web video files', async function () { + this.timeout(30_000) + + const video = await servers[0].videos.get({ id: webVideoId }) + const files = video.files + + for (const file of files) { + await servers[0].videos.removeWebVideoFile({ videoId: webVideoId, fileId: file.id }) + } + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: webVideoId }) + + expect(video.files).to.have.lengthOf(0) + } + }) + + it('Should delete a hls file', async function () { + this.timeout(30_000) + + const video = await servers[0].videos.get({ id: hlsId }) + const files = video.streamingPlaylists[0].files + const toDelete = files[0] + + await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: toDelete.id }) + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: hlsId }) + + expect(video.streamingPlaylists[0].files).to.have.lengthOf(files.length - 1) + expect(video.streamingPlaylists[0].files.find(f => f.id === toDelete.id)).to.not.exist + + const { text } = await makeRawRequest({ url: video.streamingPlaylists[0].playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) + + expect(text.includes(`-${toDelete.resolution.id}.m3u8`)).to.be.false + expect(text.includes(`-${video.streamingPlaylists[0].files[0].resolution.id}.m3u8`)).to.be.true + } + }) + + it('Should delete all hls files', async function () { + this.timeout(30_000) + + const video = await servers[0].videos.get({ id: hlsId }) + const files = video.streamingPlaylists[0].files + + for (const file of files) { + await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: file.id }) + } + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: hlsId }) + + expect(video.streamingPlaylists).to.have.lengthOf(0) + } + }) + + it('Should not delete last file of a video', async function () { + this.timeout(60_000) + + const webVideoOnly = await servers[0].videos.get({ id: hlsId }) + const hlsOnly = await servers[0].videos.get({ id: webVideoId }) + + for (let i = 0; i < 4; i++) { + await servers[0].videos.removeWebVideoFile({ videoId: webVideoOnly.id, fileId: webVideoOnly.files[i].id }) + await servers[0].videos.removeHLSFile({ videoId: hlsOnly.id, fileId: hlsOnly.streamingPlaylists[0].files[i].id }) + } + + const expectedStatus = HttpStatusCode.BAD_REQUEST_400 + await servers[0].videos.removeWebVideoFile({ videoId: webVideoOnly.id, fileId: webVideoOnly.files[4].id, expectedStatus }) + await servers[0].videos.removeHLSFile({ videoId: hlsOnly.id, fileId: hlsOnly.streamingPlaylists[0].files[4].id, expectedStatus }) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/videos/video-imports.ts b/packages/tests/src/api/videos/video-imports.ts new file mode 100644 index 000000000..09efe9931 --- /dev/null +++ b/packages/tests/src/api/videos/video-imports.ts @@ -0,0 +1,634 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { pathExists, remove } from 'fs-extra/esm' +import { readdir } from 'fs/promises' +import { join } from 'path' +import { areHttpImportTestsDisabled } from '@peertube/peertube-node-utils' +import { CustomConfig, HttpStatusCode, Video, VideoImportState, VideoPrivacy, VideoResolution, VideoState } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + createSingleServer, + doubleFollow, + getServerImportConfig, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' +import { DeepPartial } from '@peertube/peertube-typescript-utils' +import { testCaptionFile } from '@tests/shared/captions.js' +import { testImageGeneratedByFFmpeg } from '@tests/shared/checks.js' +import { FIXTURE_URLS } from '@tests/shared/tests.js' + +async function checkVideosServer1 (server: PeerTubeServer, idHttp: string, idMagnet: string, idTorrent: string) { + const videoHttp = await server.videos.get({ id: idHttp }) + + expect(videoHttp.name).to.equal('small video - youtube') + expect(videoHttp.category.label).to.equal('News & Politics') + expect(videoHttp.licence.label).to.equal('Attribution') + expect(videoHttp.language.label).to.equal('Unknown') + expect(videoHttp.nsfw).to.be.false + expect(videoHttp.description).to.equal('this is a super description') + expect(videoHttp.tags).to.deep.equal([ 'tag1', 'tag2' ]) + expect(videoHttp.files).to.have.lengthOf(1) + + const originallyPublishedAt = new Date(videoHttp.originallyPublishedAt) + expect(originallyPublishedAt.getDate()).to.equal(14) + expect(originallyPublishedAt.getMonth()).to.equal(0) + expect(originallyPublishedAt.getFullYear()).to.equal(2019) + + const videoMagnet = await server.videos.get({ id: idMagnet }) + const videoTorrent = await server.videos.get({ id: idTorrent }) + + for (const video of [ videoMagnet, videoTorrent ]) { + expect(video.category.label).to.equal('Unknown') + expect(video.licence.label).to.equal('Unknown') + expect(video.language.label).to.equal('Unknown') + expect(video.nsfw).to.be.false + expect(video.description).to.equal('this is a super torrent description') + expect(video.tags).to.deep.equal([ 'tag_torrent1', 'tag_torrent2' ]) + expect(video.files).to.have.lengthOf(1) + } + + expect(videoTorrent.name).to.contain('你好 世界 720p.mp4') + expect(videoMagnet.name).to.contain('super peertube2 video') + + const bodyCaptions = await server.captions.list({ videoId: idHttp }) + expect(bodyCaptions.total).to.equal(2) +} + +async function checkVideoServer2 (server: PeerTubeServer, id: number | string) { + const video = await server.videos.get({ id }) + + expect(video.name).to.equal('my super name') + expect(video.category.label).to.equal('Entertainment') + expect(video.licence.label).to.equal('Public Domain Dedication') + expect(video.language.label).to.equal('English') + expect(video.nsfw).to.be.false + expect(video.description).to.equal('my super description') + expect(video.tags).to.deep.equal([ 'supertag1', 'supertag2' ]) + + await testImageGeneratedByFFmpeg(server.url, 'custom-thumbnail', video.thumbnailPath) + + expect(video.files).to.have.lengthOf(1) + + const bodyCaptions = await server.captions.list({ videoId: id }) + expect(bodyCaptions.total).to.equal(2) +} + +describe('Test video imports', function () { + + if (areHttpImportTestsDisabled()) return + + function runSuite (mode: 'youtube-dl' | 'yt-dlp') { + + describe('Import ' + mode, function () { + let servers: PeerTubeServer[] = [] + + before(async function () { + this.timeout(60_000) + + servers = await createMultipleServers(2, getServerImportConfig(mode)) + + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + for (const server of servers) { + await server.config.updateExistingSubConfig({ + newConfig: { + transcoding: { + alwaysTranscodeOriginalResolution: false + } + } + }) + } + + await doubleFollow(servers[0], servers[1]) + }) + + it('Should import videos on server 1', async function () { + this.timeout(60_000) + + const baseAttributes = { + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC + } + + { + const attributes = { ...baseAttributes, targetUrl: FIXTURE_URLS.youtube } + const { video } = await servers[0].imports.importVideo({ attributes }) + expect(video.name).to.equal('small video - youtube') + + { + expect(video.thumbnailPath).to.match(new RegExp(`^/lazy-static/thumbnails/.+.jpg$`)) + expect(video.previewPath).to.match(new RegExp(`^/lazy-static/previews/.+.jpg$`)) + + const suffix = mode === 'yt-dlp' + ? '_yt_dlp' + : '' + + await testImageGeneratedByFFmpeg(servers[0].url, 'video_import_thumbnail' + suffix, video.thumbnailPath) + await testImageGeneratedByFFmpeg(servers[0].url, 'video_import_preview' + suffix, video.previewPath) + } + + const bodyCaptions = await servers[0].captions.list({ videoId: video.id }) + const videoCaptions = bodyCaptions.data + expect(videoCaptions).to.have.lengthOf(2) + + { + const enCaption = videoCaptions.find(caption => caption.language.id === 'en') + expect(enCaption).to.exist + expect(enCaption.language.label).to.equal('English') + expect(enCaption.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/.+-en.vtt$`)) + + const regex = `WEBVTT[ \n]+Kind: captions[ \n]+` + + `(Language: en[ \n]+)?` + + `00:00:01.600 --> 00:00:04.200( position:\\d+% line:\\d+%)?[ \n]+English \\(US\\)[ \n]+` + + `00:00:05.900 --> 00:00:07.999( position:\\d+% line:\\d+%)?[ \n]+This is a subtitle in American English[ \n]+` + + `00:00:10.000 --> 00:00:14.000( position:\\d+% line:\\d+%)?[ \n]+Adding subtitles is very easy to do` + await testCaptionFile(servers[0].url, enCaption.captionPath, new RegExp(regex)) + } + + { + const frCaption = videoCaptions.find(caption => caption.language.id === 'fr') + expect(frCaption).to.exist + expect(frCaption.language.label).to.equal('French') + expect(frCaption.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/.+-fr.vtt`)) + + const regex = `WEBVTT[ \n]+Kind: captions[ \n]+` + + `(Language: fr[ \n]+)?` + + `00:00:01.600 --> 00:00:04.200( position:\\d+% line:\\d+%)?[ \n]+Français \\(FR\\)[ \n]+` + + `00:00:05.900 --> 00:00:07.999( position:\\d+% line:\\d+%)?[ \n]+C'est un sous-titre français[ \n]+` + + `00:00:10.000 --> 00:00:14.000( position:\\d+% line:\\d+%)?[ \n]+Ajouter un sous-titre est vraiment facile` + + await testCaptionFile(servers[0].url, frCaption.captionPath, new RegExp(regex)) + } + } + + { + const attributes = { + ...baseAttributes, + magnetUri: FIXTURE_URLS.magnet, + description: 'this is a super torrent description', + tags: [ 'tag_torrent1', 'tag_torrent2' ] + } + const { video } = await servers[0].imports.importVideo({ attributes }) + expect(video.name).to.equal('super peertube2 video') + } + + { + const attributes = { + ...baseAttributes, + torrentfile: 'video-720p.torrent' as any, + description: 'this is a super torrent description', + tags: [ 'tag_torrent1', 'tag_torrent2' ] + } + const { video } = await servers[0].imports.importVideo({ attributes }) + expect(video.name).to.equal('你好 世界 720p.mp4') + } + }) + + it('Should list the videos to import in my videos on server 1', async function () { + const { total, data } = await servers[0].videos.listMyVideos({ sort: 'createdAt' }) + + expect(total).to.equal(3) + + expect(data).to.have.lengthOf(3) + expect(data[0].name).to.equal('small video - youtube') + expect(data[1].name).to.equal('super peertube2 video') + expect(data[2].name).to.equal('你好 世界 720p.mp4') + }) + + it('Should list the videos to import in my imports on server 1', async function () { + const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ sort: '-createdAt' }) + expect(total).to.equal(3) + + expect(videoImports).to.have.lengthOf(3) + + expect(videoImports[2].targetUrl).to.equal(FIXTURE_URLS.youtube) + expect(videoImports[2].magnetUri).to.be.null + expect(videoImports[2].torrentName).to.be.null + expect(videoImports[2].video.name).to.equal('small video - youtube') + + expect(videoImports[1].targetUrl).to.be.null + expect(videoImports[1].magnetUri).to.equal(FIXTURE_URLS.magnet) + expect(videoImports[1].torrentName).to.be.null + expect(videoImports[1].video.name).to.equal('super peertube2 video') + + expect(videoImports[0].targetUrl).to.be.null + expect(videoImports[0].magnetUri).to.be.null + expect(videoImports[0].torrentName).to.equal('video-720p.torrent') + expect(videoImports[0].video.name).to.equal('你好 世界 720p.mp4') + }) + + it('Should filter my imports on target URL', async function () { + const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ targetUrl: FIXTURE_URLS.youtube }) + expect(total).to.equal(1) + expect(videoImports).to.have.lengthOf(1) + + expect(videoImports[0].targetUrl).to.equal(FIXTURE_URLS.youtube) + }) + + it('Should search in my imports', async function () { + const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ search: 'peertube2' }) + expect(total).to.equal(1) + expect(videoImports).to.have.lengthOf(1) + + expect(videoImports[0].magnetUri).to.equal(FIXTURE_URLS.magnet) + expect(videoImports[0].video.name).to.equal('super peertube2 video') + }) + + it('Should have the video listed on the two instances', async function () { + this.timeout(120_000) + + await waitJobs(servers) + + for (const server of servers) { + const { total, data } = await server.videos.list() + expect(total).to.equal(3) + expect(data).to.have.lengthOf(3) + + const [ videoHttp, videoMagnet, videoTorrent ] = data + await checkVideosServer1(server, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid) + } + }) + + it('Should import a video on server 2 with some fields', async function () { + this.timeout(60_000) + + const { video } = await servers[1].imports.importVideo({ + attributes: { + targetUrl: FIXTURE_URLS.youtube, + channelId: servers[1].store.channel.id, + privacy: VideoPrivacy.PUBLIC, + category: 10, + licence: 7, + language: 'en', + name: 'my super name', + description: 'my super description', + tags: [ 'supertag1', 'supertag2' ], + thumbnailfile: 'custom-thumbnail.jpg' + } + }) + expect(video.name).to.equal('my super name') + }) + + it('Should have the videos listed on the two instances', async function () { + this.timeout(120_000) + + await waitJobs(servers) + + for (const server of servers) { + const { total, data } = await server.videos.list() + expect(total).to.equal(4) + expect(data).to.have.lengthOf(4) + + await checkVideoServer2(server, data[0].uuid) + + const [ , videoHttp, videoMagnet, videoTorrent ] = data + await checkVideosServer1(server, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid) + } + }) + + it('Should import a video that will be transcoded', async function () { + this.timeout(240_000) + + const attributes = { + name: 'transcoded video', + magnetUri: FIXTURE_URLS.magnet, + channelId: servers[1].store.channel.id, + privacy: VideoPrivacy.PUBLIC + } + const { video } = await servers[1].imports.importVideo({ attributes }) + const videoUUID = video.uuid + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + + expect(video.name).to.equal('transcoded video') + expect(video.files).to.have.lengthOf(4) + } + }) + + it('Should import no HDR version on a HDR video', async function () { + this.timeout(300_000) + + const config: DeepPartial = { + transcoding: { + enabled: true, + resolutions: { + '0p': false, + '144p': true, + '240p': true, + '360p': false, + '480p': false, + '720p': false, + '1080p': false, // the resulting resolution shouldn't be higher than this, and not vp9.2/av01 + '1440p': false, + '2160p': false + }, + webVideos: { enabled: true }, + hls: { enabled: false } + } + } + await servers[0].config.updateExistingSubConfig({ newConfig: config }) + + const attributes = { + name: 'hdr video', + targetUrl: FIXTURE_URLS.youtubeHDR, + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC + } + const { video: videoImported } = await servers[0].imports.importVideo({ attributes }) + const videoUUID = videoImported.uuid + + await waitJobs(servers) + + // test resolution + const video = await servers[0].videos.get({ id: videoUUID }) + expect(video.name).to.equal('hdr video') + const maxResolution = Math.max.apply(Math, video.files.map(function (o) { return o.resolution.id })) + expect(maxResolution, 'expected max resolution not met').to.equals(VideoResolution.H_240P) + }) + + it('Should not import resolution higher than enabled transcoding resolution', async function () { + this.timeout(300_000) + + const config: DeepPartial = { + transcoding: { + enabled: true, + resolutions: { + '0p': false, + '144p': true, + '240p': false, + '360p': false, + '480p': false, + '720p': false, + '1080p': false, + '1440p': false, + '2160p': false + }, + alwaysTranscodeOriginalResolution: false + } + } + await servers[0].config.updateExistingSubConfig({ newConfig: config }) + + const attributes = { + name: 'small resolution video', + targetUrl: FIXTURE_URLS.youtube, + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC + } + const { video: videoImported } = await servers[0].imports.importVideo({ attributes }) + const videoUUID = videoImported.uuid + + await waitJobs(servers) + + // test resolution + const video = await servers[0].videos.get({ id: videoUUID }) + expect(video.name).to.equal('small resolution video') + expect(video.files).to.have.lengthOf(1) + expect(video.files[0].resolution.id).to.equal(144) + }) + + it('Should import resolution higher than enabled transcoding resolution', async function () { + this.timeout(300_000) + + const config: DeepPartial = { + transcoding: { + alwaysTranscodeOriginalResolution: true + } + } + await servers[0].config.updateExistingSubConfig({ newConfig: config }) + + const attributes = { + name: 'bigger resolution video', + targetUrl: FIXTURE_URLS.youtube, + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC + } + const { video: videoImported } = await servers[0].imports.importVideo({ attributes }) + const videoUUID = videoImported.uuid + + await waitJobs(servers) + + // test resolution + const video = await servers[0].videos.get({ id: videoUUID }) + expect(video.name).to.equal('bigger resolution video') + + expect(video.files).to.have.lengthOf(2) + expect(video.files.find(f => f.resolution.id === 240)).to.exist + expect(video.files.find(f => f.resolution.id === 144)).to.exist + }) + + it('Should import a peertube video', async function () { + this.timeout(120_000) + + const toTest = [ FIXTURE_URLS.peertube_long ] + + // TODO: include peertube_short when https://github.com/ytdl-org/youtube-dl/pull/29475 is merged + if (mode === 'yt-dlp') { + toTest.push(FIXTURE_URLS.peertube_short) + } + + for (const targetUrl of toTest) { + await servers[0].config.disableTranscoding() + + const attributes = { + targetUrl, + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC + } + const { video } = await servers[0].imports.importVideo({ attributes }) + const videoUUID = video.uuid + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: videoUUID }) + + expect(video.name).to.equal('E2E tests') + + const { data: captions } = await server.captions.list({ videoId: videoUUID }) + expect(captions).to.have.lengthOf(1) + expect(captions[0].language.id).to.equal('fr') + + const str = `WEBVTT FILE\r?\n\r?\n` + + `1\r?\n` + + `00:00:04.000 --> 00:00:09.000\r?\n` + + `January 1, 1994. The North American` + await testCaptionFile(server.url, captions[0].captionPath, new RegExp(str)) + } + } + }) + + after(async function () { + await cleanupTests(servers) + }) + }) + } + + // FIXME: youtube-dl seems broken + // runSuite('youtube-dl') + + runSuite('yt-dlp') + + describe('Delete/cancel an import', function () { + let server: PeerTubeServer + + let finishedImportId: number + let finishedVideo: Video + let pendingImportId: number + + async function importVideo (name: string) { + const attributes = { name, channelId: server.store.channel.id, targetUrl: FIXTURE_URLS.goodVideo } + const res = await server.imports.importVideo({ attributes }) + + return res.id + } + + before(async function () { + this.timeout(120_000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + finishedImportId = await importVideo('finished') + await waitJobs([ server ]) + + await server.jobs.pauseJobQueue() + pendingImportId = await importVideo('pending') + + const { data } = await server.imports.getMyVideoImports() + expect(data).to.have.lengthOf(2) + + finishedVideo = data.find(i => i.id === finishedImportId).video + }) + + it('Should delete a video import', async function () { + await server.imports.delete({ importId: finishedImportId }) + + const { data } = await server.imports.getMyVideoImports() + expect(data).to.have.lengthOf(1) + expect(data[0].id).to.equal(pendingImportId) + expect(data[0].state.id).to.equal(VideoImportState.PENDING) + }) + + it('Should not have deleted the associated video', async function () { + const video = await server.videos.get({ id: finishedVideo.id, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + expect(video.name).to.equal('finished') + expect(video.state.id).to.equal(VideoState.PUBLISHED) + }) + + it('Should cancel a video import', async function () { + await server.imports.cancel({ importId: pendingImportId }) + + const { data } = await server.imports.getMyVideoImports() + expect(data).to.have.lengthOf(1) + expect(data[0].id).to.equal(pendingImportId) + expect(data[0].state.id).to.equal(VideoImportState.CANCELLED) + }) + + it('Should not have processed the cancelled video import', async function () { + this.timeout(60_000) + + await server.jobs.resumeJobQueue() + + await waitJobs([ server ]) + + const { data } = await server.imports.getMyVideoImports() + expect(data).to.have.lengthOf(1) + expect(data[0].id).to.equal(pendingImportId) + expect(data[0].state.id).to.equal(VideoImportState.CANCELLED) + expect(data[0].video.state.id).to.equal(VideoState.TO_IMPORT) + }) + + it('Should delete the cancelled video import', async function () { + await server.imports.delete({ importId: pendingImportId }) + const { data } = await server.imports.getMyVideoImports() + expect(data).to.have.lengthOf(0) + }) + + after(async function () { + await cleanupTests([ server ]) + }) + }) + + describe('Auto update', function () { + let server: PeerTubeServer + + function quickPeerTubeImport () { + const attributes = { + targetUrl: FIXTURE_URLS.peertube_long, + channelId: server.store.channel.id, + privacy: VideoPrivacy.PUBLIC + } + + return server.imports.importVideo({ attributes }) + } + + async function testBinaryUpdate (releaseUrl: string, releaseName: string) { + await remove(join(server.servers.buildDirectory('bin'), releaseName)) + + await server.kill() + await server.run({ + import: { + videos: { + http: { + youtube_dl_release: { + url: releaseUrl, + name: releaseName + } + } + } + } + }) + + await quickPeerTubeImport() + + const base = server.servers.buildDirectory('bin') + const content = await readdir(base) + const binaryPath = join(base, releaseName) + + expect(await pathExists(binaryPath), `${binaryPath} does not exist in ${base} (${content.join(', ')})`).to.be.true + } + + before(async function () { + this.timeout(30_000) + + // Run servers + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + }) + + it('Should update youtube-dl from github URL', async function () { + this.timeout(120_000) + + await testBinaryUpdate('https://api.github.com/repos/ytdl-org/youtube-dl/releases', 'youtube-dl') + }) + + it('Should update youtube-dl from raw URL', async function () { + this.timeout(120_000) + + await testBinaryUpdate('https://yt-dl.org/downloads/latest/youtube-dl', 'youtube-dl') + }) + + it('Should update youtube-dl from youtube-dl fork', async function () { + this.timeout(120_000) + + await testBinaryUpdate('https://api.github.com/repos/yt-dlp/yt-dlp/releases', 'yt-dlp') + }) + + after(async function () { + await cleanupTests([ server ]) + }) + }) +}) diff --git a/packages/tests/src/api/videos/video-nsfw.ts b/packages/tests/src/api/videos/video-nsfw.ts new file mode 100644 index 000000000..fc5225dd2 --- /dev/null +++ b/packages/tests/src/api/videos/video-nsfw.ts @@ -0,0 +1,227 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@peertube/peertube-server-commands' +import { BooleanBothQuery, CustomConfig, ResultList, Video, VideosOverview } from '@peertube/peertube-models' + +function createOverviewRes (overview: VideosOverview) { + const videos = overview.categories[0].videos + return { data: videos, total: videos.length } +} + +describe('Test video NSFW policy', function () { + let server: PeerTubeServer + let userAccessToken: string + let customConfig: CustomConfig + + async function getVideosFunctions (token?: string, query: { nsfw?: BooleanBothQuery } = {}) { + const user = await server.users.getMyInfo() + + const channelName = user.videoChannels[0].name + const accountName = user.account.name + '@' + user.account.host + + const hasQuery = Object.keys(query).length !== 0 + let promises: Promise>[] + + if (token) { + promises = [ + server.search.advancedVideoSearch({ token, search: { search: 'n', sort: '-publishedAt', ...query } }), + server.videos.listWithToken({ token, ...query }), + server.videos.listByAccount({ token, handle: accountName, ...query }), + server.videos.listByChannel({ token, handle: channelName, ...query }) + ] + + // Overviews do not support video filters + if (!hasQuery) { + const p = server.overviews.getVideos({ page: 1, token }) + .then(res => createOverviewRes(res)) + promises.push(p) + } + + return Promise.all(promises) + } + + promises = [ + server.search.searchVideos({ search: 'n', sort: '-publishedAt' }), + server.videos.list(), + server.videos.listByAccount({ token: null, handle: accountName }), + server.videos.listByChannel({ token: null, handle: channelName }) + ] + + // Overviews do not support video filters + if (!hasQuery) { + const p = server.overviews.getVideos({ page: 1 }) + .then(res => createOverviewRes(res)) + promises.push(p) + } + + return Promise.all(promises) + } + + before(async function () { + this.timeout(50000) + server = await createSingleServer(1) + + // Get the access tokens + await setAccessTokensToServers([ server ]) + + { + const attributes = { name: 'nsfw', nsfw: true, category: 1 } + await server.videos.upload({ attributes }) + } + + { + const attributes = { name: 'normal', nsfw: false, category: 1 } + await server.videos.upload({ attributes }) + } + + customConfig = await server.config.getCustomConfig() + }) + + describe('Instance default NSFW policy', function () { + + it('Should display NSFW videos with display default NSFW policy', async function () { + const serverConfig = await server.config.getConfig() + expect(serverConfig.instance.defaultNSFWPolicy).to.equal('display') + + for (const body of await getVideosFunctions()) { + expect(body.total).to.equal(2) + + const videos = body.data + expect(videos).to.have.lengthOf(2) + expect(videos[0].name).to.equal('normal') + expect(videos[1].name).to.equal('nsfw') + } + }) + + it('Should not display NSFW videos with do_not_list default NSFW policy', async function () { + customConfig.instance.defaultNSFWPolicy = 'do_not_list' + await server.config.updateCustomConfig({ newCustomConfig: customConfig }) + + const serverConfig = await server.config.getConfig() + expect(serverConfig.instance.defaultNSFWPolicy).to.equal('do_not_list') + + for (const body of await getVideosFunctions()) { + expect(body.total).to.equal(1) + + const videos = body.data + expect(videos).to.have.lengthOf(1) + expect(videos[0].name).to.equal('normal') + } + }) + + it('Should display NSFW videos with blur default NSFW policy', async function () { + customConfig.instance.defaultNSFWPolicy = 'blur' + await server.config.updateCustomConfig({ newCustomConfig: customConfig }) + + const serverConfig = await server.config.getConfig() + expect(serverConfig.instance.defaultNSFWPolicy).to.equal('blur') + + for (const body of await getVideosFunctions()) { + expect(body.total).to.equal(2) + + const videos = body.data + expect(videos).to.have.lengthOf(2) + expect(videos[0].name).to.equal('normal') + expect(videos[1].name).to.equal('nsfw') + } + }) + }) + + describe('User NSFW policy', function () { + + it('Should create a user having the default nsfw policy', async function () { + const username = 'user1' + const password = 'my super password' + await server.users.create({ username, password }) + + userAccessToken = await server.login.getAccessToken({ username, password }) + + const user = await server.users.getMyInfo({ token: userAccessToken }) + expect(user.nsfwPolicy).to.equal('blur') + }) + + it('Should display NSFW videos with blur user NSFW policy', async function () { + customConfig.instance.defaultNSFWPolicy = 'do_not_list' + await server.config.updateCustomConfig({ newCustomConfig: customConfig }) + + for (const body of await getVideosFunctions(userAccessToken)) { + expect(body.total).to.equal(2) + + const videos = body.data + expect(videos).to.have.lengthOf(2) + expect(videos[0].name).to.equal('normal') + expect(videos[1].name).to.equal('nsfw') + } + }) + + it('Should display NSFW videos with display user NSFW policy', async function () { + await server.users.updateMe({ nsfwPolicy: 'display' }) + + for (const body of await getVideosFunctions(server.accessToken)) { + expect(body.total).to.equal(2) + + const videos = body.data + expect(videos).to.have.lengthOf(2) + expect(videos[0].name).to.equal('normal') + expect(videos[1].name).to.equal('nsfw') + } + }) + + it('Should not display NSFW videos with do_not_list user NSFW policy', async function () { + await server.users.updateMe({ nsfwPolicy: 'do_not_list' }) + + for (const body of await getVideosFunctions(server.accessToken)) { + expect(body.total).to.equal(1) + + const videos = body.data + expect(videos).to.have.lengthOf(1) + expect(videos[0].name).to.equal('normal') + } + }) + + it('Should be able to see my NSFW videos even with do_not_list user NSFW policy', async function () { + const { total, data } = await server.videos.listMyVideos() + expect(total).to.equal(2) + + expect(data).to.have.lengthOf(2) + expect(data[0].name).to.equal('normal') + expect(data[1].name).to.equal('nsfw') + }) + + it('Should display NSFW videos when the nsfw param === true', async function () { + for (const body of await getVideosFunctions(server.accessToken, { nsfw: 'true' })) { + expect(body.total).to.equal(1) + + const videos = body.data + expect(videos).to.have.lengthOf(1) + expect(videos[0].name).to.equal('nsfw') + } + }) + + it('Should hide NSFW videos when the nsfw param === true', async function () { + for (const body of await getVideosFunctions(server.accessToken, { nsfw: 'false' })) { + expect(body.total).to.equal(1) + + const videos = body.data + expect(videos).to.have.lengthOf(1) + expect(videos[0].name).to.equal('normal') + } + }) + + it('Should display both videos when the nsfw param === both', async function () { + for (const body of await getVideosFunctions(server.accessToken, { nsfw: 'both' })) { + expect(body.total).to.equal(2) + + const videos = body.data + expect(videos).to.have.lengthOf(2) + expect(videos[0].name).to.equal('normal') + expect(videos[1].name).to.equal('nsfw') + } + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/videos/video-passwords.ts b/packages/tests/src/api/videos/video-passwords.ts new file mode 100644 index 000000000..60e0e28bd --- /dev/null +++ b/packages/tests/src/api/videos/video-passwords.ts @@ -0,0 +1,97 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { + cleanupTests, + createSingleServer, + VideoPasswordsCommand, + PeerTubeServer, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultChannelAvatar +} from '@peertube/peertube-server-commands' +import { VideoPrivacy } from '@peertube/peertube-models' + +describe('Test video passwords', function () { + let server: PeerTubeServer + let videoUUID: string + + let userAccessTokenServer1: string + + let videoPasswords: string[] = [] + let command: VideoPasswordsCommand + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + + for (let i = 0; i < 10; i++) { + videoPasswords.push(`password ${i + 1}`) + } + const { uuid } = await server.videos.upload({ attributes: { privacy: VideoPrivacy.PASSWORD_PROTECTED, videoPasswords } }) + videoUUID = uuid + + await setDefaultChannelAvatar(server) + await setDefaultAccountAvatar(server) + + userAccessTokenServer1 = await server.users.generateUserAndToken('user1') + await setDefaultChannelAvatar(server, 'user1_channel') + await setDefaultAccountAvatar(server, userAccessTokenServer1) + + command = server.videoPasswords + }) + + it('Should list video passwords', async function () { + const body = await command.list({ videoId: videoUUID }) + + expect(body.total).to.equal(10) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(10) + }) + + it('Should filter passwords on this video', async function () { + const body = await command.list({ videoId: videoUUID, count: 2, start: 3, sort: 'createdAt' }) + + expect(body.total).to.equal(10) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(2) + expect(body.data[0].password).to.equal('password 4') + expect(body.data[1].password).to.equal('password 5') + }) + + it('Should update password for this video', async function () { + videoPasswords = [ 'my super new password 1', 'my super new password 2' ] + + await command.updateAll({ videoId: videoUUID, passwords: videoPasswords }) + const body = await command.list({ videoId: videoUUID }) + expect(body.total).to.equal(2) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(2) + expect(body.data[0].password).to.equal('my super new password 2') + expect(body.data[1].password).to.equal('my super new password 1') + }) + + it('Should delete one password', async function () { + { + const body = await command.list({ videoId: videoUUID }) + expect(body.total).to.equal(2) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(2) + await command.remove({ id: body.data[0].id, videoId: videoUUID }) + } + { + const body = await command.list({ videoId: videoUUID }) + + expect(body.total).to.equal(1) + expect(body.data).to.be.an('array') + expect(body.data).to.have.lengthOf(1) + } + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/videos/video-playlist-thumbnails.ts b/packages/tests/src/api/videos/video-playlist-thumbnails.ts new file mode 100644 index 000000000..d79c92f72 --- /dev/null +++ b/packages/tests/src/api/videos/video-playlist-thumbnails.ts @@ -0,0 +1,234 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { testImageGeneratedByFFmpeg } from '@tests/shared/checks.js' +import { VideoPlaylistPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Playlist thumbnail', function () { + let servers: PeerTubeServer[] = [] + + let playlistWithoutThumbnailId: number + let playlistWithThumbnailId: number + + let withThumbnailE1: number + let withThumbnailE2: number + let withoutThumbnailE1: number + let withoutThumbnailE2: number + + let video1: number + let video2: number + + async function getPlaylistWithoutThumbnail (server: PeerTubeServer) { + const body = await server.playlists.list({ start: 0, count: 10 }) + + return body.data.find(p => p.displayName === 'playlist without thumbnail') + } + + async function getPlaylistWithThumbnail (server: PeerTubeServer) { + const body = await server.playlists.list({ start: 0, count: 10 }) + + return body.data.find(p => p.displayName === 'playlist with thumbnail') + } + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(2) + + // Get the access tokens + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + for (const server of servers) { + await server.config.disableTranscoding() + } + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + + video1 = (await servers[0].videos.quickUpload({ name: 'video 1' })).id + video2 = (await servers[0].videos.quickUpload({ name: 'video 2' })).id + + await waitJobs(servers) + }) + + it('Should automatically update the thumbnail when adding an element', async function () { + this.timeout(30000) + + const created = await servers[1].playlists.create({ + attributes: { + displayName: 'playlist without thumbnail', + privacy: VideoPlaylistPrivacy.PUBLIC, + videoChannelId: servers[1].store.channel.id + } + }) + playlistWithoutThumbnailId = created.id + + const added = await servers[1].playlists.addElement({ + playlistId: playlistWithoutThumbnailId, + attributes: { videoId: video1 } + }) + withoutThumbnailE1 = added.id + + await waitJobs(servers) + + for (const server of servers) { + const p = await getPlaylistWithoutThumbnail(server) + await testImageGeneratedByFFmpeg(server.url, 'thumbnail-playlist', p.thumbnailPath) + } + }) + + it('Should not update the thumbnail if we explicitly uploaded a thumbnail', async function () { + this.timeout(30000) + + const created = await servers[1].playlists.create({ + attributes: { + displayName: 'playlist with thumbnail', + privacy: VideoPlaylistPrivacy.PUBLIC, + videoChannelId: servers[1].store.channel.id, + thumbnailfile: 'custom-thumbnail.jpg' + } + }) + playlistWithThumbnailId = created.id + + const added = await servers[1].playlists.addElement({ + playlistId: playlistWithThumbnailId, + attributes: { videoId: video1 } + }) + withThumbnailE1 = added.id + + await waitJobs(servers) + + for (const server of servers) { + const p = await getPlaylistWithThumbnail(server) + await testImageGeneratedByFFmpeg(server.url, 'custom-thumbnail', p.thumbnailPath) + } + }) + + it('Should automatically update the thumbnail when moving the first element', async function () { + this.timeout(30000) + + const added = await servers[1].playlists.addElement({ + playlistId: playlistWithoutThumbnailId, + attributes: { videoId: video2 } + }) + withoutThumbnailE2 = added.id + + await servers[1].playlists.reorderElements({ + playlistId: playlistWithoutThumbnailId, + attributes: { + startPosition: 1, + insertAfterPosition: 2 + } + }) + + await waitJobs(servers) + + for (const server of servers) { + const p = await getPlaylistWithoutThumbnail(server) + await testImageGeneratedByFFmpeg(server.url, 'thumbnail-playlist', p.thumbnailPath) + } + }) + + it('Should not update the thumbnail when moving the first element if we explicitly uploaded a thumbnail', async function () { + this.timeout(30000) + + const added = await servers[1].playlists.addElement({ + playlistId: playlistWithThumbnailId, + attributes: { videoId: video2 } + }) + withThumbnailE2 = added.id + + await servers[1].playlists.reorderElements({ + playlistId: playlistWithThumbnailId, + attributes: { + startPosition: 1, + insertAfterPosition: 2 + } + }) + + await waitJobs(servers) + + for (const server of servers) { + const p = await getPlaylistWithThumbnail(server) + await testImageGeneratedByFFmpeg(server.url, 'custom-thumbnail', p.thumbnailPath) + } + }) + + it('Should automatically update the thumbnail when deleting the first element', async function () { + this.timeout(30000) + + await servers[1].playlists.removeElement({ + playlistId: playlistWithoutThumbnailId, + elementId: withoutThumbnailE1 + }) + + await waitJobs(servers) + + for (const server of servers) { + const p = await getPlaylistWithoutThumbnail(server) + await testImageGeneratedByFFmpeg(server.url, 'thumbnail-playlist', p.thumbnailPath) + } + }) + + it('Should not update the thumbnail when deleting the first element if we explicitly uploaded a thumbnail', async function () { + this.timeout(30000) + + await servers[1].playlists.removeElement({ + playlistId: playlistWithThumbnailId, + elementId: withThumbnailE1 + }) + + await waitJobs(servers) + + for (const server of servers) { + const p = await getPlaylistWithThumbnail(server) + await testImageGeneratedByFFmpeg(server.url, 'custom-thumbnail', p.thumbnailPath) + } + }) + + it('Should the thumbnail when we delete the last element', async function () { + this.timeout(30000) + + await servers[1].playlists.removeElement({ + playlistId: playlistWithoutThumbnailId, + elementId: withoutThumbnailE2 + }) + + await waitJobs(servers) + + for (const server of servers) { + const p = await getPlaylistWithoutThumbnail(server) + expect(p.thumbnailPath).to.be.null + } + }) + + it('Should not update the thumbnail when we delete the last element if we explicitly uploaded a thumbnail', async function () { + this.timeout(30000) + + await servers[1].playlists.removeElement({ + playlistId: playlistWithThumbnailId, + elementId: withThumbnailE2 + }) + + await waitJobs(servers) + + for (const server of servers) { + const p = await getPlaylistWithThumbnail(server) + await testImageGeneratedByFFmpeg(server.url, 'custom-thumbnail', p.thumbnailPath) + } + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/videos/video-playlists.ts b/packages/tests/src/api/videos/video-playlists.ts new file mode 100644 index 000000000..578d01093 --- /dev/null +++ b/packages/tests/src/api/videos/video-playlists.ts @@ -0,0 +1,1210 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { + HttpStatusCode, + VideoPlaylist, + VideoPlaylistCreateResult, + VideoPlaylistElementType, + VideoPlaylistElementType_Type, + VideoPlaylistPrivacy, + VideoPlaylistType, + VideoPrivacy +} from '@peertube/peertube-models' +import { uuidToShort } from '@peertube/peertube-node-utils' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + PlaylistsCommand, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' +import { testImageGeneratedByFFmpeg } from '@tests/shared/checks.js' +import { checkPlaylistFilesWereRemoved } from '@tests/shared/video-playlists.js' + +async function checkPlaylistElementType ( + servers: PeerTubeServer[], + playlistId: string, + type: VideoPlaylistElementType_Type, + position: number, + name: string, + total: number +) { + for (const server of servers) { + const body = await server.playlists.listVideos({ token: server.accessToken, playlistId, start: 0, count: 10 }) + expect(body.total).to.equal(total) + + const videoElement = body.data.find(e => e.position === position) + expect(videoElement.type).to.equal(type, 'On server ' + server.url) + + if (type === VideoPlaylistElementType.REGULAR) { + expect(videoElement.video).to.not.be.null + expect(videoElement.video.name).to.equal(name) + } else { + expect(videoElement.video).to.be.null + } + } +} + +describe('Test video playlists', function () { + let servers: PeerTubeServer[] = [] + + let playlistServer2Id1: number + let playlistServer2Id2: number + let playlistServer2UUID2: string + + let playlistServer1Id: number + let playlistServer1DisplayName: string + let playlistServer1UUID: string + let playlistServer1UUID2: string + + let playlistElementServer1Video4: number + let playlistElementServer1Video5: number + let playlistElementNSFW: number + + let nsfwVideoServer1: number + + let userTokenServer1: string + + let commands: PlaylistsCommand[] + + before(async function () { + this.timeout(240000) + + servers = await createMultipleServers(3) + + // Get the access tokens + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + await setDefaultAccountAvatar(servers) + + for (const server of servers) { + await server.config.disableTranscoding() + } + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + // Server 1 and server 3 follow each other + await doubleFollow(servers[0], servers[2]) + + commands = servers.map(s => s.playlists) + + { + servers[0].store.videos = [] + servers[1].store.videos = [] + servers[2].store.videos = [] + + for (const server of servers) { + for (let i = 0; i < 7; i++) { + const name = `video ${i} server ${server.serverNumber}` + const video = await server.videos.upload({ attributes: { name, nsfw: false } }) + + server.store.videos.push(video) + } + } + } + + nsfwVideoServer1 = (await servers[0].videos.quickUpload({ name: 'NSFW video', nsfw: true })).id + + userTokenServer1 = await servers[0].users.generateUserAndToken('user1') + + await waitJobs(servers) + }) + + describe('Check playlists filters and privacies', function () { + + it('Should list video playlist privacies', async function () { + const privacies = await commands[0].getPrivacies() + + expect(Object.keys(privacies)).to.have.length.at.least(3) + expect(privacies[3]).to.equal('Private') + }) + + it('Should filter on playlist type', async function () { + this.timeout(30000) + + const token = servers[0].accessToken + + await commands[0].create({ + attributes: { + displayName: 'my super playlist', + privacy: VideoPlaylistPrivacy.PUBLIC, + description: 'my super description', + thumbnailfile: 'custom-thumbnail.jpg', + videoChannelId: servers[0].store.channel.id + } + }) + + { + const body = await commands[0].listByAccount({ token, handle: 'root', playlistType: VideoPlaylistType.WATCH_LATER }) + + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + + const playlist = body.data[0] + expect(playlist.displayName).to.equal('Watch later') + expect(playlist.type.id).to.equal(VideoPlaylistType.WATCH_LATER) + expect(playlist.type.label).to.equal('Watch later') + expect(playlist.privacy.id).to.equal(VideoPlaylistPrivacy.PRIVATE) + } + + { + const bodyList = await commands[0].list({ playlistType: VideoPlaylistType.WATCH_LATER }) + const bodyChannel = await commands[0].listByChannel({ handle: 'root_channel', playlistType: VideoPlaylistType.WATCH_LATER }) + + for (const body of [ bodyList, bodyChannel ]) { + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + } + } + + { + const bodyList = await commands[0].list({ playlistType: VideoPlaylistType.REGULAR }) + const bodyChannel = await commands[0].listByChannel({ handle: 'root_channel', playlistType: VideoPlaylistType.REGULAR }) + + let playlist: VideoPlaylist = null + for (const body of [ bodyList, bodyChannel ]) { + + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + + playlist = body.data[0] + expect(playlist.displayName).to.equal('my super playlist') + expect(playlist.privacy.id).to.equal(VideoPlaylistPrivacy.PUBLIC) + expect(playlist.type.id).to.equal(VideoPlaylistType.REGULAR) + } + + await commands[0].update({ + playlistId: playlist.id, + attributes: { + privacy: VideoPlaylistPrivacy.PRIVATE + } + }) + } + + { + const bodyList = await commands[0].list({ playlistType: VideoPlaylistType.REGULAR }) + const bodyChannel = await commands[0].listByChannel({ handle: 'root_channel', playlistType: VideoPlaylistType.REGULAR }) + + for (const body of [ bodyList, bodyChannel ]) { + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + } + } + + { + const body = await commands[0].listByAccount({ handle: 'root' }) + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + } + }) + + it('Should get private playlist for a classic user', async function () { + const token = await servers[0].users.generateUserAndToken('toto') + + const body = await commands[0].listByAccount({ token, handle: 'toto' }) + + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + + const playlistId = body.data[0].id + await commands[0].listVideos({ token, playlistId }) + }) + }) + + describe('Create and federate playlists', function () { + + it('Should create a playlist on server 1 and have the playlist on server 2 and 3', async function () { + this.timeout(30000) + + await commands[0].create({ + attributes: { + displayName: 'my super playlist', + privacy: VideoPlaylistPrivacy.PUBLIC, + description: 'my super description', + thumbnailfile: 'custom-thumbnail.jpg', + videoChannelId: servers[0].store.channel.id + } + }) + + await waitJobs(servers) + // Processing a playlist by the receiver could be long + await wait(3000) + + for (const server of servers) { + const body = await server.playlists.list({ start: 0, count: 5 }) + expect(body.total).to.equal(1) + expect(body.data).to.have.lengthOf(1) + + const playlistFromList = body.data[0] + + const playlistFromGet = await server.playlists.get({ playlistId: playlistFromList.uuid }) + + for (const playlist of [ playlistFromGet, playlistFromList ]) { + expect(playlist.id).to.be.a('number') + expect(playlist.uuid).to.be.a('string') + + expect(playlist.isLocal).to.equal(server.serverNumber === 1) + + expect(playlist.displayName).to.equal('my super playlist') + expect(playlist.description).to.equal('my super description') + expect(playlist.privacy.id).to.equal(VideoPlaylistPrivacy.PUBLIC) + expect(playlist.privacy.label).to.equal('Public') + expect(playlist.type.id).to.equal(VideoPlaylistType.REGULAR) + expect(playlist.type.label).to.equal('Regular') + expect(playlist.embedPath).to.equal('/video-playlists/embed/' + playlist.uuid) + + expect(playlist.videosLength).to.equal(0) + + expect(playlist.ownerAccount.name).to.equal('root') + expect(playlist.ownerAccount.displayName).to.equal('root') + expect(playlist.videoChannel.name).to.equal('root_channel') + expect(playlist.videoChannel.displayName).to.equal('Main root channel') + } + } + }) + + it('Should create a playlist on server 2 and have the playlist on server 1 but not on server 3', async function () { + this.timeout(30000) + + { + const playlist = await servers[1].playlists.create({ + attributes: { + displayName: 'playlist 2', + privacy: VideoPlaylistPrivacy.PUBLIC, + videoChannelId: servers[1].store.channel.id + } + }) + playlistServer2Id1 = playlist.id + } + + { + const playlist = await servers[1].playlists.create({ + attributes: { + displayName: 'playlist 3', + privacy: VideoPlaylistPrivacy.PUBLIC, + thumbnailfile: 'custom-thumbnail.jpg', + videoChannelId: servers[1].store.channel.id + } + }) + + playlistServer2Id2 = playlist.id + playlistServer2UUID2 = playlist.uuid + } + + for (const id of [ playlistServer2Id1, playlistServer2Id2 ]) { + await servers[1].playlists.addElement({ + playlistId: id, + attributes: { videoId: servers[1].store.videos[0].id, startTimestamp: 1, stopTimestamp: 2 } + }) + await servers[1].playlists.addElement({ + playlistId: id, + attributes: { videoId: servers[1].store.videos[1].id } + }) + } + + await waitJobs(servers) + await wait(3000) + + for (const server of [ servers[0], servers[1] ]) { + const body = await server.playlists.list({ start: 0, count: 5 }) + + const playlist2 = body.data.find(p => p.displayName === 'playlist 2') + expect(playlist2).to.not.be.undefined + await testImageGeneratedByFFmpeg(server.url, 'thumbnail-playlist', playlist2.thumbnailPath) + + const playlist3 = body.data.find(p => p.displayName === 'playlist 3') + expect(playlist3).to.not.be.undefined + await testImageGeneratedByFFmpeg(server.url, 'custom-thumbnail', playlist3.thumbnailPath) + } + + const body = await servers[2].playlists.list({ start: 0, count: 5 }) + expect(body.data.find(p => p.displayName === 'playlist 2')).to.be.undefined + expect(body.data.find(p => p.displayName === 'playlist 3')).to.be.undefined + }) + + it('Should have the playlist on server 3 after a new follow', async function () { + this.timeout(30000) + + // Server 2 and server 3 follow each other + await doubleFollow(servers[1], servers[2]) + + const body = await servers[2].playlists.list({ start: 0, count: 5 }) + + const playlist2 = body.data.find(p => p.displayName === 'playlist 2') + expect(playlist2).to.not.be.undefined + await testImageGeneratedByFFmpeg(servers[2].url, 'thumbnail-playlist', playlist2.thumbnailPath) + + expect(body.data.find(p => p.displayName === 'playlist 3')).to.not.be.undefined + }) + }) + + describe('List playlists', function () { + + it('Should correctly list the playlists', async function () { + this.timeout(30000) + + { + const body = await servers[2].playlists.list({ start: 1, count: 2, sort: 'createdAt' }) + expect(body.total).to.equal(3) + + const data = body.data + expect(data).to.have.lengthOf(2) + expect(data[0].displayName).to.equal('playlist 2') + expect(data[1].displayName).to.equal('playlist 3') + } + + { + const body = await servers[2].playlists.list({ start: 1, count: 2, sort: '-createdAt' }) + expect(body.total).to.equal(3) + + const data = body.data + expect(data).to.have.lengthOf(2) + expect(data[0].displayName).to.equal('playlist 2') + expect(data[1].displayName).to.equal('my super playlist') + } + }) + + it('Should list video channel playlists', async function () { + this.timeout(30000) + + { + const body = await commands[0].listByChannel({ handle: 'root_channel', start: 0, count: 2, sort: '-createdAt' }) + expect(body.total).to.equal(1) + + const data = body.data + expect(data).to.have.lengthOf(1) + expect(data[0].displayName).to.equal('my super playlist') + } + }) + + it('Should list account playlists', async function () { + this.timeout(30000) + + { + const body = await servers[1].playlists.listByAccount({ handle: 'root', start: 1, count: 2, sort: '-createdAt' }) + expect(body.total).to.equal(2) + + const data = body.data + expect(data).to.have.lengthOf(1) + expect(data[0].displayName).to.equal('playlist 2') + } + + { + const body = await servers[1].playlists.listByAccount({ handle: 'root', start: 1, count: 2, sort: 'createdAt' }) + expect(body.total).to.equal(2) + + const data = body.data + expect(data).to.have.lengthOf(1) + expect(data[0].displayName).to.equal('playlist 3') + } + + { + const body = await servers[1].playlists.listByAccount({ handle: 'root', sort: 'createdAt', search: '3' }) + expect(body.total).to.equal(1) + + const data = body.data + expect(data).to.have.lengthOf(1) + expect(data[0].displayName).to.equal('playlist 3') + } + + { + const body = await servers[1].playlists.listByAccount({ handle: 'root', sort: 'createdAt', search: '4' }) + expect(body.total).to.equal(0) + + const data = body.data + expect(data).to.have.lengthOf(0) + } + }) + }) + + describe('Playlist rights', function () { + let unlistedPlaylist: VideoPlaylistCreateResult + let privatePlaylist: VideoPlaylistCreateResult + + before(async function () { + this.timeout(30000) + + { + unlistedPlaylist = await servers[1].playlists.create({ + attributes: { + displayName: 'playlist unlisted', + privacy: VideoPlaylistPrivacy.UNLISTED, + videoChannelId: servers[1].store.channel.id + } + }) + } + + { + privatePlaylist = await servers[1].playlists.create({ + attributes: { + displayName: 'playlist private', + privacy: VideoPlaylistPrivacy.PRIVATE + } + }) + } + + await waitJobs(servers) + await wait(3000) + }) + + it('Should not list unlisted or private playlists', async function () { + for (const server of servers) { + const results = [ + await server.playlists.listByAccount({ handle: 'root@' + servers[1].host, sort: '-createdAt' }), + await server.playlists.list({ start: 0, count: 2, sort: '-createdAt' }) + ] + + expect(results[0].total).to.equal(2) + expect(results[1].total).to.equal(3) + + for (const body of results) { + const data = body.data + expect(data).to.have.lengthOf(2) + expect(data[0].displayName).to.equal('playlist 3') + expect(data[1].displayName).to.equal('playlist 2') + } + } + }) + + it('Should not get unlisted playlist using only the id', async function () { + await servers[1].playlists.get({ playlistId: unlistedPlaylist.id, expectedStatus: 404 }) + }) + + it('Should get unlisted playlist using uuid or shortUUID', async function () { + await servers[1].playlists.get({ playlistId: unlistedPlaylist.uuid }) + await servers[1].playlists.get({ playlistId: unlistedPlaylist.shortUUID }) + }) + + it('Should not get private playlist without token', async function () { + for (const id of [ privatePlaylist.id, privatePlaylist.uuid, privatePlaylist.shortUUID ]) { + await servers[1].playlists.get({ playlistId: id, expectedStatus: 401 }) + } + }) + + it('Should get private playlist with a token', async function () { + for (const id of [ privatePlaylist.id, privatePlaylist.uuid, privatePlaylist.shortUUID ]) { + await servers[1].playlists.get({ token: servers[1].accessToken, playlistId: id }) + } + }) + }) + + describe('Update playlists', function () { + + it('Should update a playlist', async function () { + this.timeout(30000) + + await servers[1].playlists.update({ + attributes: { + displayName: 'playlist 3 updated', + description: 'description updated', + privacy: VideoPlaylistPrivacy.UNLISTED, + thumbnailfile: 'custom-thumbnail.jpg', + videoChannelId: servers[1].store.channel.id + }, + playlistId: playlistServer2Id2 + }) + + await waitJobs(servers) + + for (const server of servers) { + const playlist = await server.playlists.get({ playlistId: playlistServer2UUID2 }) + + expect(playlist.displayName).to.equal('playlist 3 updated') + expect(playlist.description).to.equal('description updated') + + expect(playlist.privacy.id).to.equal(VideoPlaylistPrivacy.UNLISTED) + expect(playlist.privacy.label).to.equal('Unlisted') + + expect(playlist.type.id).to.equal(VideoPlaylistType.REGULAR) + expect(playlist.type.label).to.equal('Regular') + + expect(playlist.videosLength).to.equal(2) + + expect(playlist.ownerAccount.name).to.equal('root') + expect(playlist.ownerAccount.displayName).to.equal('root') + expect(playlist.videoChannel.name).to.equal('root_channel') + expect(playlist.videoChannel.displayName).to.equal('Main root channel') + } + }) + }) + + describe('Element timestamps', function () { + + it('Should create a playlist containing different startTimestamp/endTimestamp videos', async function () { + this.timeout(30000) + + const addVideo = (attributes: any) => { + return commands[0].addElement({ playlistId: playlistServer1Id, attributes }) + } + + const playlistDisplayName = 'playlist 4' + const playlist = await commands[0].create({ + attributes: { + displayName: playlistDisplayName, + privacy: VideoPlaylistPrivacy.PUBLIC, + videoChannelId: servers[0].store.channel.id + } + }) + + playlistServer1Id = playlist.id + playlistServer1DisplayName = playlistDisplayName + playlistServer1UUID = playlist.uuid + + await addVideo({ videoId: servers[0].store.videos[0].uuid, startTimestamp: 15, stopTimestamp: 28 }) + await addVideo({ videoId: servers[2].store.videos[1].uuid, startTimestamp: 35 }) + await addVideo({ videoId: servers[2].store.videos[2].uuid }) + { + const element = await addVideo({ videoId: servers[0].store.videos[3].uuid, stopTimestamp: 35 }) + playlistElementServer1Video4 = element.id + } + + { + const element = await addVideo({ videoId: servers[0].store.videos[4].uuid, startTimestamp: 45, stopTimestamp: 60 }) + playlistElementServer1Video5 = element.id + } + + { + const element = await addVideo({ videoId: nsfwVideoServer1, startTimestamp: 5 }) + playlistElementNSFW = element.id + + await addVideo({ videoId: nsfwVideoServer1, startTimestamp: 4 }) + await addVideo({ videoId: nsfwVideoServer1 }) + } + + await waitJobs(servers) + }) + + it('Should correctly list playlist videos', async function () { + this.timeout(30000) + + for (const server of servers) { + { + const body = await server.playlists.listVideos({ playlistId: playlistServer1UUID, start: 0, count: 10 }) + + expect(body.total).to.equal(8) + + const videoElements = body.data + expect(videoElements).to.have.lengthOf(8) + + expect(videoElements[0].video.name).to.equal('video 0 server 1') + expect(videoElements[0].position).to.equal(1) + expect(videoElements[0].startTimestamp).to.equal(15) + expect(videoElements[0].stopTimestamp).to.equal(28) + + expect(videoElements[1].video.name).to.equal('video 1 server 3') + expect(videoElements[1].position).to.equal(2) + expect(videoElements[1].startTimestamp).to.equal(35) + expect(videoElements[1].stopTimestamp).to.be.null + + expect(videoElements[2].video.name).to.equal('video 2 server 3') + expect(videoElements[2].position).to.equal(3) + expect(videoElements[2].startTimestamp).to.be.null + expect(videoElements[2].stopTimestamp).to.be.null + + expect(videoElements[3].video.name).to.equal('video 3 server 1') + expect(videoElements[3].position).to.equal(4) + expect(videoElements[3].startTimestamp).to.be.null + expect(videoElements[3].stopTimestamp).to.equal(35) + + expect(videoElements[4].video.name).to.equal('video 4 server 1') + expect(videoElements[4].position).to.equal(5) + expect(videoElements[4].startTimestamp).to.equal(45) + expect(videoElements[4].stopTimestamp).to.equal(60) + + expect(videoElements[5].video.name).to.equal('NSFW video') + expect(videoElements[5].position).to.equal(6) + expect(videoElements[5].startTimestamp).to.equal(5) + expect(videoElements[5].stopTimestamp).to.be.null + + expect(videoElements[6].video.name).to.equal('NSFW video') + expect(videoElements[6].position).to.equal(7) + expect(videoElements[6].startTimestamp).to.equal(4) + expect(videoElements[6].stopTimestamp).to.be.null + + expect(videoElements[7].video.name).to.equal('NSFW video') + expect(videoElements[7].position).to.equal(8) + expect(videoElements[7].startTimestamp).to.be.null + expect(videoElements[7].stopTimestamp).to.be.null + } + + { + const body = await server.playlists.listVideos({ playlistId: playlistServer1UUID, start: 0, count: 2 }) + expect(body.data).to.have.lengthOf(2) + } + } + }) + }) + + describe('Element type', function () { + let groupUser1: PeerTubeServer[] + let groupWithoutToken1: PeerTubeServer[] + let group1: PeerTubeServer[] + let group2: PeerTubeServer[] + + let video1: string + let video2: string + let video3: string + + before(async function () { + this.timeout(60000) + + groupUser1 = [ Object.assign({}, servers[0], { accessToken: userTokenServer1 }) ] + groupWithoutToken1 = [ Object.assign({}, servers[0], { accessToken: undefined }) ] + group1 = [ servers[0] ] + group2 = [ servers[1], servers[2] ] + + const playlist = await commands[0].create({ + token: userTokenServer1, + attributes: { + displayName: 'playlist 56', + privacy: VideoPlaylistPrivacy.PUBLIC, + videoChannelId: servers[0].store.channel.id + } + }) + + const playlistServer1Id2 = playlist.id + playlistServer1UUID2 = playlist.uuid + + const addVideo = (attributes: any) => { + return commands[0].addElement({ token: userTokenServer1, playlistId: playlistServer1Id2, attributes }) + } + + video1 = (await servers[0].videos.quickUpload({ name: 'video 89', token: userTokenServer1 })).uuid + video2 = (await servers[1].videos.quickUpload({ name: 'video 90' })).uuid + video3 = (await servers[0].videos.quickUpload({ name: 'video 91', nsfw: true })).uuid + + await waitJobs(servers) + + await addVideo({ videoId: video1, startTimestamp: 15, stopTimestamp: 28 }) + await addVideo({ videoId: video2, startTimestamp: 35 }) + await addVideo({ videoId: video3 }) + + await waitJobs(servers) + }) + + it('Should update the element type if the video is private/password protected', async function () { + this.timeout(20000) + + const name = 'video 89' + const position = 1 + + { + await servers[0].videos.update({ id: video1, attributes: { privacy: VideoPrivacy.PRIVATE } }) + await waitJobs(servers) + + await checkPlaylistElementType(groupUser1, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) + await checkPlaylistElementType(groupWithoutToken1, playlistServer1UUID2, VideoPlaylistElementType.PRIVATE, position, name, 3) + await checkPlaylistElementType(group1, playlistServer1UUID2, VideoPlaylistElementType.PRIVATE, position, name, 3) + await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.DELETED, position, name, 3) + } + + { + await servers[0].videos.update({ + id: video1, + attributes: { privacy: VideoPrivacy.PASSWORD_PROTECTED, videoPasswords: [ 'password' ] } + }) + await waitJobs(servers) + + await checkPlaylistElementType(groupUser1, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) + await checkPlaylistElementType(groupWithoutToken1, playlistServer1UUID2, VideoPlaylistElementType.PRIVATE, position, name, 3) + await checkPlaylistElementType(group1, playlistServer1UUID2, VideoPlaylistElementType.PRIVATE, position, name, 3) + await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.DELETED, position, name, 3) + } + + { + await servers[0].videos.update({ id: video1, attributes: { privacy: VideoPrivacy.PUBLIC } }) + await waitJobs(servers) + + await checkPlaylistElementType(groupUser1, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) + await checkPlaylistElementType(groupWithoutToken1, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) + await checkPlaylistElementType(group1, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) + // We deleted the video, so even if we recreated it, the old entry is still deleted + await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.DELETED, position, name, 3) + } + }) + + it('Should update the element type if the video is blacklisted', async function () { + this.timeout(20000) + + const name = 'video 89' + const position = 1 + + { + await servers[0].blacklist.add({ videoId: video1, reason: 'reason', unfederate: true }) + await waitJobs(servers) + + await checkPlaylistElementType(groupUser1, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) + await checkPlaylistElementType(groupWithoutToken1, playlistServer1UUID2, VideoPlaylistElementType.UNAVAILABLE, position, name, 3) + await checkPlaylistElementType(group1, playlistServer1UUID2, VideoPlaylistElementType.UNAVAILABLE, position, name, 3) + await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.DELETED, position, name, 3) + } + + { + await servers[0].blacklist.remove({ videoId: video1 }) + await waitJobs(servers) + + await checkPlaylistElementType(groupUser1, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) + await checkPlaylistElementType(groupWithoutToken1, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) + await checkPlaylistElementType(group1, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) + // We deleted the video (because unfederated), so even if we recreated it, the old entry is still deleted + await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.DELETED, position, name, 3) + } + }) + + it('Should update the element type if the account or server of the video is blocked', async function () { + this.timeout(90000) + + const command = servers[0].blocklist + + const name = 'video 90' + const position = 2 + + { + await command.addToMyBlocklist({ token: userTokenServer1, account: 'root@' + servers[1].host }) + await waitJobs(servers) + + await checkPlaylistElementType(groupUser1, playlistServer1UUID2, VideoPlaylistElementType.UNAVAILABLE, position, name, 3) + await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) + + await command.removeFromMyBlocklist({ token: userTokenServer1, account: 'root@' + servers[1].host }) + await waitJobs(servers) + + await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) + } + + { + await command.addToMyBlocklist({ token: userTokenServer1, server: servers[1].host }) + await waitJobs(servers) + + await checkPlaylistElementType(groupUser1, playlistServer1UUID2, VideoPlaylistElementType.UNAVAILABLE, position, name, 3) + await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) + + await command.removeFromMyBlocklist({ token: userTokenServer1, server: servers[1].host }) + await waitJobs(servers) + + await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) + } + + { + await command.addToServerBlocklist({ account: 'root@' + servers[1].host }) + await waitJobs(servers) + + await checkPlaylistElementType(groupUser1, playlistServer1UUID2, VideoPlaylistElementType.UNAVAILABLE, position, name, 3) + await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) + + await command.removeFromServerBlocklist({ account: 'root@' + servers[1].host }) + await waitJobs(servers) + + await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) + } + + { + await command.addToServerBlocklist({ server: servers[1].host }) + await waitJobs(servers) + + await checkPlaylistElementType(groupUser1, playlistServer1UUID2, VideoPlaylistElementType.UNAVAILABLE, position, name, 3) + await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) + + await command.removeFromServerBlocklist({ server: servers[1].host }) + await waitJobs(servers) + + await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) + } + }) + }) + + describe('Managing playlist elements', function () { + + it('Should reorder the playlist', async function () { + this.timeout(30000) + + { + await commands[0].reorderElements({ + playlistId: playlistServer1Id, + attributes: { + startPosition: 2, + insertAfterPosition: 3 + } + }) + + await waitJobs(servers) + + for (const server of servers) { + const body = await server.playlists.listVideos({ playlistId: playlistServer1UUID, start: 0, count: 10 }) + const names = body.data.map(v => v.video.name) + + expect(names).to.deep.equal([ + 'video 0 server 1', + 'video 2 server 3', + 'video 1 server 3', + 'video 3 server 1', + 'video 4 server 1', + 'NSFW video', + 'NSFW video', + 'NSFW video' + ]) + } + } + + { + await commands[0].reorderElements({ + playlistId: playlistServer1Id, + attributes: { + startPosition: 1, + reorderLength: 3, + insertAfterPosition: 4 + } + }) + + await waitJobs(servers) + + for (const server of servers) { + const body = await server.playlists.listVideos({ playlistId: playlistServer1UUID, start: 0, count: 10 }) + const names = body.data.map(v => v.video.name) + + expect(names).to.deep.equal([ + 'video 3 server 1', + 'video 0 server 1', + 'video 2 server 3', + 'video 1 server 3', + 'video 4 server 1', + 'NSFW video', + 'NSFW video', + 'NSFW video' + ]) + } + } + + { + await commands[0].reorderElements({ + playlistId: playlistServer1Id, + attributes: { + startPosition: 6, + insertAfterPosition: 3 + } + }) + + await waitJobs(servers) + + for (const server of servers) { + const { data: elements } = await server.playlists.listVideos({ playlistId: playlistServer1UUID, start: 0, count: 10 }) + const names = elements.map(v => v.video.name) + + expect(names).to.deep.equal([ + 'video 3 server 1', + 'video 0 server 1', + 'video 2 server 3', + 'NSFW video', + 'video 1 server 3', + 'video 4 server 1', + 'NSFW video', + 'NSFW video' + ]) + + for (let i = 1; i <= elements.length; i++) { + expect(elements[i - 1].position).to.equal(i) + } + } + } + }) + + it('Should update startTimestamp/endTimestamp of some elements', async function () { + this.timeout(30000) + + await commands[0].updateElement({ + playlistId: playlistServer1Id, + elementId: playlistElementServer1Video4, + attributes: { + startTimestamp: 1 + } + }) + + await commands[0].updateElement({ + playlistId: playlistServer1Id, + elementId: playlistElementServer1Video5, + attributes: { + stopTimestamp: null + } + }) + + await waitJobs(servers) + + for (const server of servers) { + const { data: elements } = await server.playlists.listVideos({ playlistId: playlistServer1UUID, start: 0, count: 10 }) + + expect(elements[0].video.name).to.equal('video 3 server 1') + expect(elements[0].position).to.equal(1) + expect(elements[0].startTimestamp).to.equal(1) + expect(elements[0].stopTimestamp).to.equal(35) + + expect(elements[5].video.name).to.equal('video 4 server 1') + expect(elements[5].position).to.equal(6) + expect(elements[5].startTimestamp).to.equal(45) + expect(elements[5].stopTimestamp).to.be.null + } + }) + + it('Should check videos existence in my playlist', async function () { + const videoIds = [ + servers[0].store.videos[0].id, + 42000, + servers[0].store.videos[3].id, + 43000, + servers[0].store.videos[4].id + ] + const obj = await commands[0].videosExist({ videoIds }) + + { + const elem = obj[servers[0].store.videos[0].id] + expect(elem).to.have.lengthOf(1) + expect(elem[0].playlistElementId).to.exist + expect(elem[0].playlistDisplayName).to.equal(playlistServer1DisplayName) + expect(elem[0].playlistShortUUID).to.equal(uuidToShort(playlistServer1UUID)) + expect(elem[0].playlistId).to.equal(playlistServer1Id) + expect(elem[0].startTimestamp).to.equal(15) + expect(elem[0].stopTimestamp).to.equal(28) + } + + { + const elem = obj[servers[0].store.videos[3].id] + expect(elem).to.have.lengthOf(1) + expect(elem[0].playlistElementId).to.equal(playlistElementServer1Video4) + expect(elem[0].playlistDisplayName).to.equal(playlistServer1DisplayName) + expect(elem[0].playlistShortUUID).to.equal(uuidToShort(playlistServer1UUID)) + expect(elem[0].playlistId).to.equal(playlistServer1Id) + expect(elem[0].startTimestamp).to.equal(1) + expect(elem[0].stopTimestamp).to.equal(35) + } + + { + const elem = obj[servers[0].store.videos[4].id] + expect(elem).to.have.lengthOf(1) + expect(elem[0].playlistId).to.equal(playlistServer1Id) + expect(elem[0].playlistDisplayName).to.equal(playlistServer1DisplayName) + expect(elem[0].playlistShortUUID).to.equal(uuidToShort(playlistServer1UUID)) + expect(elem[0].startTimestamp).to.equal(45) + expect(elem[0].stopTimestamp).to.equal(null) + } + + expect(obj[42000]).to.have.lengthOf(0) + expect(obj[43000]).to.have.lengthOf(0) + }) + + it('Should automatically update updatedAt field of playlists', async function () { + const server = servers[1] + const videoId = servers[1].store.videos[5].id + + async function getPlaylistNames () { + const { data } = await server.playlists.listByAccount({ token: server.accessToken, handle: 'root', sort: '-updatedAt' }) + + return data.map(p => p.displayName) + } + + const attributes = { videoId } + const element1 = await server.playlists.addElement({ playlistId: playlistServer2Id1, attributes }) + const element2 = await server.playlists.addElement({ playlistId: playlistServer2Id2, attributes }) + + const names1 = await getPlaylistNames() + expect(names1[0]).to.equal('playlist 3 updated') + expect(names1[1]).to.equal('playlist 2') + + await server.playlists.removeElement({ playlistId: playlistServer2Id1, elementId: element1.id }) + + const names2 = await getPlaylistNames() + expect(names2[0]).to.equal('playlist 2') + expect(names2[1]).to.equal('playlist 3 updated') + + await server.playlists.removeElement({ playlistId: playlistServer2Id2, elementId: element2.id }) + + const names3 = await getPlaylistNames() + expect(names3[0]).to.equal('playlist 3 updated') + expect(names3[1]).to.equal('playlist 2') + }) + + it('Should delete some elements', async function () { + this.timeout(30000) + + await commands[0].removeElement({ playlistId: playlistServer1Id, elementId: playlistElementServer1Video4 }) + await commands[0].removeElement({ playlistId: playlistServer1Id, elementId: playlistElementNSFW }) + + await waitJobs(servers) + + for (const server of servers) { + const body = await server.playlists.listVideos({ playlistId: playlistServer1UUID, start: 0, count: 10 }) + expect(body.total).to.equal(6) + + const elements = body.data + expect(elements).to.have.lengthOf(6) + + expect(elements[0].video.name).to.equal('video 0 server 1') + expect(elements[0].position).to.equal(1) + + expect(elements[1].video.name).to.equal('video 2 server 3') + expect(elements[1].position).to.equal(2) + + expect(elements[2].video.name).to.equal('video 1 server 3') + expect(elements[2].position).to.equal(3) + + expect(elements[3].video.name).to.equal('video 4 server 1') + expect(elements[3].position).to.equal(4) + + expect(elements[4].video.name).to.equal('NSFW video') + expect(elements[4].position).to.equal(5) + + expect(elements[5].video.name).to.equal('NSFW video') + expect(elements[5].position).to.equal(6) + } + }) + + it('Should be able to create a public playlist, and set it to private', async function () { + this.timeout(30000) + + const videoPlaylistIds = await commands[0].create({ + attributes: { + displayName: 'my super public playlist', + privacy: VideoPlaylistPrivacy.PUBLIC, + videoChannelId: servers[0].store.channel.id + } + }) + + await waitJobs(servers) + + for (const server of servers) { + await server.playlists.get({ playlistId: videoPlaylistIds.uuid, expectedStatus: HttpStatusCode.OK_200 }) + } + + const attributes = { privacy: VideoPlaylistPrivacy.PRIVATE } + await commands[0].update({ playlistId: videoPlaylistIds.id, attributes }) + + await waitJobs(servers) + + for (const server of [ servers[1], servers[2] ]) { + await server.playlists.get({ playlistId: videoPlaylistIds.uuid, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + } + + await commands[0].get({ playlistId: videoPlaylistIds.uuid, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + await commands[0].get({ token: servers[0].accessToken, playlistId: videoPlaylistIds.uuid, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + describe('Playlist deletion', function () { + + it('Should delete the playlist on server 1 and delete on server 2 and 3', async function () { + this.timeout(30000) + + await commands[0].delete({ playlistId: playlistServer1Id }) + + await waitJobs(servers) + + for (const server of servers) { + await server.playlists.get({ playlistId: playlistServer1UUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + } + }) + + it('Should have deleted the thumbnail on server 1, 2 and 3', async function () { + this.timeout(30000) + + for (const server of servers) { + await checkPlaylistFilesWereRemoved(playlistServer1UUID, server) + } + }) + + it('Should unfollow servers 1 and 2 and hide their playlists', async function () { + this.timeout(30000) + + const finder = (data: VideoPlaylist[]) => data.find(p => p.displayName === 'my super playlist') + + { + const body = await servers[2].playlists.list({ start: 0, count: 5 }) + expect(body.total).to.equal(3) + + expect(finder(body.data)).to.not.be.undefined + } + + await servers[2].follows.unfollow({ target: servers[0] }) + + { + const body = await servers[2].playlists.list({ start: 0, count: 5 }) + expect(body.total).to.equal(1) + + expect(finder(body.data)).to.be.undefined + } + }) + + it('Should delete a channel and put the associated playlist in private mode', async function () { + this.timeout(30000) + + const channel = await servers[0].channels.create({ attributes: { name: 'super_channel', displayName: 'super channel' } }) + + const playlistCreated = await commands[0].create({ + attributes: { + displayName: 'channel playlist', + privacy: VideoPlaylistPrivacy.PUBLIC, + videoChannelId: channel.id + } + }) + + await waitJobs(servers) + + await servers[0].channels.delete({ channelName: 'super_channel' }) + + await waitJobs(servers) + + const body = await commands[0].get({ token: servers[0].accessToken, playlistId: playlistCreated.uuid }) + expect(body.displayName).to.equal('channel playlist') + expect(body.privacy.id).to.equal(VideoPlaylistPrivacy.PRIVATE) + + await servers[1].playlists.get({ playlistId: playlistCreated.uuid, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + + it('Should delete an account and delete its playlists', async function () { + this.timeout(30000) + + const { userId, token } = await servers[0].users.generate('user_1') + + const { videoChannels } = await servers[0].users.getMyInfo({ token }) + const userChannel = videoChannels[0] + + await commands[0].create({ + attributes: { + displayName: 'playlist to be deleted', + privacy: VideoPlaylistPrivacy.PUBLIC, + videoChannelId: userChannel.id + } + }) + + await waitJobs(servers) + + const finder = (data: VideoPlaylist[]) => data.find(p => p.displayName === 'playlist to be deleted') + + { + for (const server of [ servers[0], servers[1] ]) { + const body = await server.playlists.list({ start: 0, count: 15 }) + + expect(finder(body.data)).to.not.be.undefined + } + } + + await servers[0].users.remove({ userId }) + await waitJobs(servers) + + { + for (const server of [ servers[0], servers[1] ]) { + const body = await server.playlists.list({ start: 0, count: 15 }) + + expect(finder(body.data)).to.be.undefined + } + } + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/videos/video-privacy.ts b/packages/tests/src/api/videos/video-privacy.ts new file mode 100644 index 000000000..9171463a4 --- /dev/null +++ b/packages/tests/src/api/videos/video-privacy.ts @@ -0,0 +1,294 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { HttpStatusCode, VideoCreateResult, VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test video privacy', function () { + const servers: PeerTubeServer[] = [] + let anotherUserToken: string + + let privateVideoId: number + let privateVideoUUID: string + + let internalVideoId: number + let internalVideoUUID: string + + let unlistedVideo: VideoCreateResult + let nonFederatedUnlistedVideoUUID: string + + let now: number + + const dontFederateUnlistedConfig = { + federation: { + videos: { + federate_unlisted: false + } + } + } + + before(async function () { + this.timeout(50000) + + // Run servers + servers.push(await createSingleServer(1, dontFederateUnlistedConfig)) + servers.push(await createSingleServer(2)) + + // Get the access tokens + await setAccessTokensToServers(servers) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + }) + + describe('Private and internal videos', function () { + + it('Should upload a private and internal videos on server 1', async function () { + this.timeout(50000) + + for (const privacy of [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]) { + const attributes = { privacy } + await servers[0].videos.upload({ attributes }) + } + + await waitJobs(servers) + }) + + it('Should not have these private and internal videos on server 2', async function () { + const { total, data } = await servers[1].videos.list() + + expect(total).to.equal(0) + expect(data).to.have.lengthOf(0) + }) + + it('Should not list the private and internal videos for an unauthenticated user on server 1', async function () { + const { total, data } = await servers[0].videos.list() + + expect(total).to.equal(0) + expect(data).to.have.lengthOf(0) + }) + + it('Should not list the private video and list the internal video for an authenticated user on server 1', async function () { + const { total, data } = await servers[0].videos.listWithToken() + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + + expect(data[0].privacy.id).to.equal(VideoPrivacy.INTERNAL) + }) + + it('Should list my (private and internal) videos', async function () { + const { total, data } = await servers[0].videos.listMyVideos() + + expect(total).to.equal(2) + expect(data).to.have.lengthOf(2) + + const privateVideo = data.find(v => v.privacy.id === VideoPrivacy.PRIVATE) + privateVideoId = privateVideo.id + privateVideoUUID = privateVideo.uuid + + const internalVideo = data.find(v => v.privacy.id === VideoPrivacy.INTERNAL) + internalVideoId = internalVideo.id + internalVideoUUID = internalVideo.uuid + }) + + it('Should not be able to watch the private/internal video with non authenticated user', async function () { + await servers[0].videos.get({ id: privateVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + await servers[0].videos.get({ id: internalVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should not be able to watch the private video with another user', async function () { + const user = { + username: 'hello', + password: 'super password' + } + await servers[0].users.create({ username: user.username, password: user.password }) + + anotherUserToken = await servers[0].login.getAccessToken(user) + + await servers[0].videos.getWithToken({ + token: anotherUserToken, + id: privateVideoUUID, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should be able to watch the internal video with another user', async function () { + await servers[0].videos.getWithToken({ token: anotherUserToken, id: internalVideoUUID }) + }) + + it('Should be able to watch the private video with the correct user', async function () { + await servers[0].videos.getWithToken({ id: privateVideoUUID }) + }) + }) + + describe('Unlisted videos', function () { + + it('Should upload an unlisted video on server 2', async function () { + this.timeout(120000) + + const attributes = { + name: 'unlisted video', + privacy: VideoPrivacy.UNLISTED + } + await servers[1].videos.upload({ attributes }) + + // Server 2 has transcoding enabled + await waitJobs(servers) + }) + + it('Should not have this unlisted video listed on server 1 and 2', async function () { + for (const server of servers) { + const { total, data } = await server.videos.list() + + expect(total).to.equal(0) + expect(data).to.have.lengthOf(0) + } + }) + + it('Should list my (unlisted) videos', async function () { + const { total, data } = await servers[1].videos.listMyVideos() + + expect(total).to.equal(1) + expect(data).to.have.lengthOf(1) + + unlistedVideo = data[0] + }) + + it('Should not be able to get this unlisted video using its id', async function () { + await servers[1].videos.get({ id: unlistedVideo.id, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should be able to get this unlisted video using its uuid/shortUUID', async function () { + for (const server of servers) { + for (const id of [ unlistedVideo.uuid, unlistedVideo.shortUUID ]) { + const video = await server.videos.get({ id }) + + expect(video.name).to.equal('unlisted video') + } + } + }) + + it('Should upload a non-federating unlisted video to server 1', async function () { + this.timeout(30000) + + const attributes = { + name: 'unlisted video', + privacy: VideoPrivacy.UNLISTED + } + await servers[0].videos.upload({ attributes }) + + await waitJobs(servers) + }) + + it('Should list my new unlisted video', async function () { + const { total, data } = await servers[0].videos.listMyVideos() + + expect(total).to.equal(3) + expect(data).to.have.lengthOf(3) + + nonFederatedUnlistedVideoUUID = data[0].uuid + }) + + it('Should be able to get non-federated unlisted video from origin', async function () { + const video = await servers[0].videos.get({ id: nonFederatedUnlistedVideoUUID }) + + expect(video.name).to.equal('unlisted video') + }) + + it('Should not be able to get non-federated unlisted video from federated server', async function () { + await servers[1].videos.get({ id: nonFederatedUnlistedVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + }) + }) + + describe('Privacy update', function () { + + it('Should update the private and internal videos to public on server 1', async function () { + this.timeout(100000) + + now = Date.now() + + { + const attributes = { + name: 'private video becomes public', + privacy: VideoPrivacy.PUBLIC + } + + await servers[0].videos.update({ id: privateVideoId, attributes }) + } + + { + const attributes = { + name: 'internal video becomes public', + privacy: VideoPrivacy.PUBLIC + } + await servers[0].videos.update({ id: internalVideoId, attributes }) + } + + await wait(10000) + await waitJobs(servers) + }) + + it('Should have this new public video listed on server 1 and 2', async function () { + for (const server of servers) { + const { total, data } = await server.videos.list() + expect(total).to.equal(2) + expect(data).to.have.lengthOf(2) + + const privateVideo = data.find(v => v.name === 'private video becomes public') + const internalVideo = data.find(v => v.name === 'internal video becomes public') + + expect(privateVideo).to.not.be.undefined + expect(internalVideo).to.not.be.undefined + + expect(new Date(privateVideo.publishedAt).getTime()).to.be.at.least(now) + // We don't change the publish date of internal videos + expect(new Date(internalVideo.publishedAt).getTime()).to.be.below(now) + + expect(privateVideo.privacy.id).to.equal(VideoPrivacy.PUBLIC) + expect(internalVideo.privacy.id).to.equal(VideoPrivacy.PUBLIC) + } + }) + + it('Should set these videos as private and internal', async function () { + await servers[0].videos.update({ id: internalVideoId, attributes: { privacy: VideoPrivacy.PRIVATE } }) + await servers[0].videos.update({ id: privateVideoId, attributes: { privacy: VideoPrivacy.INTERNAL } }) + + await waitJobs(servers) + + for (const server of servers) { + const { total, data } = await server.videos.list() + + expect(total).to.equal(0) + expect(data).to.have.lengthOf(0) + } + + { + const { total, data } = await servers[0].videos.listMyVideos() + expect(total).to.equal(3) + expect(data).to.have.lengthOf(3) + + const privateVideo = data.find(v => v.name === 'private video becomes public') + const internalVideo = data.find(v => v.name === 'internal video becomes public') + + expect(privateVideo).to.not.be.undefined + expect(internalVideo).to.not.be.undefined + + expect(privateVideo.privacy.id).to.equal(VideoPrivacy.INTERNAL) + expect(internalVideo.privacy.id).to.equal(VideoPrivacy.PRIVATE) + } + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/videos/video-schedule-update.ts b/packages/tests/src/api/videos/video-schedule-update.ts new file mode 100644 index 000000000..96d71933e --- /dev/null +++ b/packages/tests/src/api/videos/video-schedule-update.ts @@ -0,0 +1,155 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +function in10Seconds () { + const now = new Date() + now.setSeconds(now.getSeconds() + 10) + + return now +} + +describe('Test video update scheduler', function () { + let servers: PeerTubeServer[] = [] + let video2UUID: string + + before(async function () { + this.timeout(30000) + + // Run servers + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + + await doubleFollow(servers[0], servers[1]) + }) + + it('Should upload a video and schedule an update in 10 seconds', async function () { + const attributes = { + name: 'video 1', + privacy: VideoPrivacy.PRIVATE, + scheduleUpdate: { + updateAt: in10Seconds().toISOString(), + privacy: VideoPrivacy.PUBLIC + } + } + + await servers[0].videos.upload({ attributes }) + + await waitJobs(servers) + }) + + it('Should not list the video (in privacy mode)', async function () { + for (const server of servers) { + const { total } = await server.videos.list() + + expect(total).to.equal(0) + } + }) + + it('Should have my scheduled video in my account videos', async function () { + const { total, data } = await servers[0].videos.listMyVideos() + expect(total).to.equal(1) + + const videoFromList = data[0] + const videoFromGet = await servers[0].videos.getWithToken({ id: videoFromList.uuid }) + + for (const video of [ videoFromList, videoFromGet ]) { + expect(video.name).to.equal('video 1') + expect(video.privacy.id).to.equal(VideoPrivacy.PRIVATE) + expect(new Date(video.scheduledUpdate.updateAt)).to.be.above(new Date()) + expect(video.scheduledUpdate.privacy).to.equal(VideoPrivacy.PUBLIC) + } + }) + + it('Should wait some seconds and have the video in public privacy', async function () { + this.timeout(50000) + + await wait(15000) + await waitJobs(servers) + + for (const server of servers) { + const { total, data } = await server.videos.list() + + expect(total).to.equal(1) + expect(data[0].name).to.equal('video 1') + } + }) + + it('Should upload a video without scheduling an update', async function () { + const attributes = { + name: 'video 2', + privacy: VideoPrivacy.PRIVATE + } + + const { uuid } = await servers[0].videos.upload({ attributes }) + video2UUID = uuid + + await waitJobs(servers) + }) + + it('Should update a video by scheduling an update', async function () { + const attributes = { + name: 'video 2 updated', + scheduleUpdate: { + updateAt: in10Seconds().toISOString(), + privacy: VideoPrivacy.PUBLIC + } + } + + await servers[0].videos.update({ id: video2UUID, attributes }) + await waitJobs(servers) + }) + + it('Should not display the updated video', async function () { + for (const server of servers) { + const { total } = await server.videos.list() + + expect(total).to.equal(1) + } + }) + + it('Should have my scheduled updated video in my account videos', async function () { + const { total, data } = await servers[0].videos.listMyVideos() + expect(total).to.equal(2) + + const video = data.find(v => v.uuid === video2UUID) + expect(video).not.to.be.undefined + + expect(video.name).to.equal('video 2 updated') + expect(video.privacy.id).to.equal(VideoPrivacy.PRIVATE) + + expect(new Date(video.scheduledUpdate.updateAt)).to.be.above(new Date()) + expect(video.scheduledUpdate.privacy).to.equal(VideoPrivacy.PUBLIC) + }) + + it('Should wait some seconds and have the updated video in public privacy', async function () { + this.timeout(20000) + + await wait(15000) + await waitJobs(servers) + + for (const server of servers) { + const { total, data } = await server.videos.list() + expect(total).to.equal(2) + + const video = data.find(v => v.uuid === video2UUID) + expect(video).not.to.be.undefined + expect(video.name).to.equal('video 2 updated') + } + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/videos/video-source.ts b/packages/tests/src/api/videos/video-source.ts new file mode 100644 index 000000000..efe8c3802 --- /dev/null +++ b/packages/tests/src/api/videos/video-source.ts @@ -0,0 +1,448 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ +import { expect } from 'chai' +import { getAllFiles } from '@peertube/peertube-core-utils' +import { HttpStatusCode } from '@peertube/peertube-models' +import { expectStartWith } from '@tests/shared/checks.js' +import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + makeGetRequest, + makeRawRequest, + ObjectStorageCommand, + PeerTubeServer, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test a video file replacement', function () { + let servers: PeerTubeServer[] = [] + + let replaceDate: Date + let userToken: string + let uuid: string + + before(async function () { + this.timeout(50000) + + servers = await createMultipleServers(2) + + // Get the access tokens + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + await setDefaultAccountAvatar(servers) + + await servers[0].config.enableFileUpdate() + + userToken = await servers[0].users.generateUserAndToken('user1') + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + }) + + describe('Getting latest video source', () => { + const fixture = 'video_short.webm' + const uuids: string[] = [] + + it('Should get the source filename with legacy upload', async function () { + this.timeout(30000) + + const { uuid } = await servers[0].videos.upload({ attributes: { name: 'my video', fixture }, mode: 'legacy' }) + uuids.push(uuid) + + const source = await servers[0].videos.getSource({ id: uuid }) + expect(source.filename).to.equal(fixture) + }) + + it('Should get the source filename with resumable upload', async function () { + this.timeout(30000) + + const { uuid } = await servers[0].videos.upload({ attributes: { name: 'my video', fixture }, mode: 'resumable' }) + uuids.push(uuid) + + const source = await servers[0].videos.getSource({ id: uuid }) + expect(source.filename).to.equal(fixture) + }) + + after(async function () { + this.timeout(60000) + + for (const uuid of uuids) { + await servers[0].videos.remove({ id: uuid }) + } + + await waitJobs(servers) + }) + }) + + describe('Updating video source', function () { + + describe('Filesystem', function () { + + it('Should replace a video file with transcoding disabled', async function () { + this.timeout(120000) + + await servers[0].config.disableTranscoding() + + const { uuid } = await servers[0].videos.quickUpload({ name: 'fs without transcoding', fixture: 'video_short_720p.mp4' }) + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + + const files = getAllFiles(video) + expect(files).to.have.lengthOf(1) + expect(files[0].resolution.id).to.equal(720) + } + + await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_360p.mp4' }) + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + + const files = getAllFiles(video) + expect(files).to.have.lengthOf(1) + expect(files[0].resolution.id).to.equal(360) + } + }) + + it('Should replace a video file with transcoding enabled', async function () { + this.timeout(120000) + + const previousPaths: string[] = [] + + await servers[0].config.enableTranscoding({ hls: true, webVideo: true, with0p: true }) + + const { uuid: videoUUID } = await servers[0].videos.quickUpload({ name: 'fs with transcoding', fixture: 'video_short_720p.mp4' }) + uuid = videoUUID + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + expect(video.inputFileUpdatedAt).to.be.null + + const files = getAllFiles(video) + expect(files).to.have.lengthOf(6 * 2) + + // Grab old paths to ensure we'll regenerate + + previousPaths.push(video.previewPath) + previousPaths.push(video.thumbnailPath) + + for (const file of files) { + previousPaths.push(file.fileUrl) + previousPaths.push(file.torrentUrl) + previousPaths.push(file.metadataUrl) + + const metadata = await server.videos.getFileMetadata({ url: file.metadataUrl }) + previousPaths.push(JSON.stringify(metadata)) + } + + const { storyboards } = await server.storyboard.list({ id: uuid }) + for (const s of storyboards) { + previousPaths.push(s.storyboardPath) + } + } + + replaceDate = new Date() + + await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_360p.mp4' }) + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + + expect(video.inputFileUpdatedAt).to.not.be.null + expect(new Date(video.inputFileUpdatedAt)).to.be.above(replaceDate) + + const files = getAllFiles(video) + expect(files).to.have.lengthOf(4 * 2) + + expect(previousPaths).to.not.include(video.previewPath) + expect(previousPaths).to.not.include(video.thumbnailPath) + + await makeGetRequest({ url: server.url, path: video.previewPath, expectedStatus: HttpStatusCode.OK_200 }) + await makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) + + for (const file of files) { + expect(previousPaths).to.not.include(file.fileUrl) + expect(previousPaths).to.not.include(file.torrentUrl) + expect(previousPaths).to.not.include(file.metadataUrl) + + await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 }) + + const metadata = await server.videos.getFileMetadata({ url: file.metadataUrl }) + expect(previousPaths).to.not.include(JSON.stringify(metadata)) + } + + const { storyboards } = await server.storyboard.list({ id: uuid }) + for (const s of storyboards) { + expect(previousPaths).to.not.include(s.storyboardPath) + + await makeGetRequest({ url: server.url, path: s.storyboardPath, expectedStatus: HttpStatusCode.OK_200 }) + } + } + + await servers[0].config.enableMinimumTranscoding() + }) + + it('Should have cleaned up old files', async function () { + { + const count = await servers[0].servers.countFiles('storyboards') + expect(count).to.equal(2) + } + + { + const count = await servers[0].servers.countFiles('web-videos') + expect(count).to.equal(5 + 1) // +1 for private directory + } + + { + const count = await servers[0].servers.countFiles('streaming-playlists/hls') + expect(count).to.equal(1 + 1) // +1 for private directory + } + + { + const count = await servers[0].servers.countFiles('torrents') + expect(count).to.equal(9) + } + }) + + it('Should have the correct source input', async function () { + const source = await servers[0].videos.getSource({ id: uuid }) + + expect(source.filename).to.equal('video_short_360p.mp4') + expect(new Date(source.createdAt)).to.be.above(replaceDate) + }) + + it('Should not have regenerated miniatures that were previously uploaded', async function () { + this.timeout(120000) + + const { uuid } = await servers[0].videos.upload({ + attributes: { + name: 'custom miniatures', + thumbnailfile: 'custom-thumbnail.jpg', + previewfile: 'custom-preview.jpg' + } + }) + + await waitJobs(servers) + + const previousPaths: string[] = [] + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + + previousPaths.push(video.previewPath) + previousPaths.push(video.thumbnailPath) + + await makeGetRequest({ url: server.url, path: video.previewPath, expectedStatus: HttpStatusCode.OK_200 }) + await makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) + } + + await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_360p.mp4' }) + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + + expect(previousPaths).to.include(video.previewPath) + expect(previousPaths).to.include(video.thumbnailPath) + + await makeGetRequest({ url: server.url, path: video.previewPath, expectedStatus: HttpStatusCode.OK_200 }) + await makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) + } + }) + }) + + describe('Autoblacklist', function () { + + function updateAutoBlacklist (enabled: boolean) { + return servers[0].config.updateExistingSubConfig({ + newConfig: { + autoBlacklist: { + videos: { + ofUsers: { + enabled + } + } + } + } + }) + } + + async function expectBlacklist (uuid: string, value: boolean) { + const video = await servers[0].videos.getWithToken({ id: uuid }) + + expect(video.blacklisted).to.equal(value) + } + + before(async function () { + await updateAutoBlacklist(true) + }) + + it('Should auto blacklist an unblacklisted video after file replacement', async function () { + this.timeout(120000) + + const { uuid } = await servers[0].videos.quickUpload({ token: userToken, name: 'user video' }) + await waitJobs(servers) + await expectBlacklist(uuid, true) + + await servers[0].blacklist.remove({ videoId: uuid }) + await expectBlacklist(uuid, false) + + await servers[0].videos.replaceSourceFile({ videoId: uuid, token: userToken, fixture: 'video_short_360p.mp4' }) + await waitJobs(servers) + + await expectBlacklist(uuid, true) + }) + + it('Should auto blacklist an already blacklisted video after file replacement', async function () { + this.timeout(120000) + + const { uuid } = await servers[0].videos.quickUpload({ token: userToken, name: 'user video' }) + await waitJobs(servers) + await expectBlacklist(uuid, true) + + await servers[0].videos.replaceSourceFile({ videoId: uuid, token: userToken, fixture: 'video_short_360p.mp4' }) + await waitJobs(servers) + + await expectBlacklist(uuid, true) + }) + + it('Should not auto blacklist if auto blacklist has been disabled between the upload and the replacement', async function () { + this.timeout(120000) + + const { uuid } = await servers[0].videos.quickUpload({ token: userToken, name: 'user video' }) + await waitJobs(servers) + await expectBlacklist(uuid, true) + + await servers[0].blacklist.remove({ videoId: uuid }) + await expectBlacklist(uuid, false) + + await updateAutoBlacklist(false) + + await servers[0].videos.replaceSourceFile({ videoId: uuid, token: userToken, fixture: 'video_short1.webm' }) + await waitJobs(servers) + + await expectBlacklist(uuid, false) + }) + }) + + describe('With object storage enabled', function () { + if (areMockObjectStorageTestsDisabled()) return + + const objectStorage = new ObjectStorageCommand() + + before(async function () { + this.timeout(120000) + + const configOverride = objectStorage.getDefaultMockConfig() + await objectStorage.prepareDefaultMockBuckets() + + await servers[0].kill() + await servers[0].run(configOverride) + }) + + it('Should replace a video file with transcoding disabled', async function () { + this.timeout(120000) + + await servers[0].config.disableTranscoding() + + const { uuid } = await servers[0].videos.quickUpload({ + name: 'object storage without transcoding', + fixture: 'video_short_720p.mp4' + }) + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + + const files = getAllFiles(video) + expect(files).to.have.lengthOf(1) + expect(files[0].resolution.id).to.equal(720) + expectStartWith(files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl()) + } + + await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_360p.mp4' }) + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + + const files = getAllFiles(video) + expect(files).to.have.lengthOf(1) + expect(files[0].resolution.id).to.equal(360) + expectStartWith(files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl()) + } + }) + + it('Should replace a video file with transcoding enabled', async function () { + this.timeout(120000) + + const previousPaths: string[] = [] + + await servers[0].config.enableTranscoding({ hls: true, webVideo: true, with0p: true }) + + const { uuid: videoUUID } = await servers[0].videos.quickUpload({ + name: 'object storage with transcoding', + fixture: 'video_short_360p.mp4' + }) + uuid = videoUUID + + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + + const files = getAllFiles(video) + expect(files).to.have.lengthOf(4 * 2) + + for (const file of files) { + previousPaths.push(file.fileUrl) + } + + for (const file of video.files) { + expectStartWith(file.fileUrl, objectStorage.getMockWebVideosBaseUrl()) + } + + for (const file of video.streamingPlaylists[0].files) { + expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl()) + } + } + + await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_240p.mp4' }) + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + + const files = getAllFiles(video) + expect(files).to.have.lengthOf(3 * 2) + + for (const file of files) { + expect(previousPaths).to.not.include(file.fileUrl) + } + + for (const file of video.files) { + expectStartWith(file.fileUrl, objectStorage.getMockWebVideosBaseUrl()) + } + + for (const file of video.streamingPlaylists[0].files) { + expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl()) + } + } + }) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/videos/video-static-file-privacy.ts b/packages/tests/src/api/videos/video-static-file-privacy.ts new file mode 100644 index 000000000..7c8d14815 --- /dev/null +++ b/packages/tests/src/api/videos/video-static-file-privacy.ts @@ -0,0 +1,602 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { decode } from 'magnet-uri' +import { getAllFiles, wait } from '@peertube/peertube-core-utils' +import { HttpStatusCode, HttpStatusCodeType, LiveVideo, VideoDetails, VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + findExternalSavedVideo, + makeRawRequest, + PeerTubeServer, + sendRTMPStream, + setAccessTokensToServers, + setDefaultVideoChannel, + stopFfmpeg, + waitJobs +} from '@peertube/peertube-server-commands' +import { expectStartWith } from '@tests/shared/checks.js' +import { checkVideoFileTokenReinjection } from '@tests/shared/streaming-playlists.js' +import { parseTorrentVideo } from '@tests/shared/webtorrent.js' + +describe('Test video static file privacy', function () { + let server: PeerTubeServer + let userToken: string + + before(async function () { + this.timeout(50000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + userToken = await server.users.generateUserAndToken('user1') + }) + + describe('VOD static file path', function () { + + function runSuite () { + + async function checkPrivateFiles (uuid: string) { + const video = await server.videos.getWithToken({ id: uuid }) + + for (const file of video.files) { + expect(file.fileDownloadUrl).to.not.include('/private/') + expectStartWith(file.fileUrl, server.url + '/static/web-videos/private/') + + const torrent = await parseTorrentVideo(server, file) + expect(torrent.urlList).to.have.lengthOf(0) + + const magnet = decode(file.magnetUri) + expect(magnet.urlList).to.have.lengthOf(0) + + await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + } + + const hls = video.streamingPlaylists[0] + if (hls) { + expectStartWith(hls.playlistUrl, server.url + '/static/streaming-playlists/hls/private/') + expectStartWith(hls.segmentsSha256Url, server.url + '/static/streaming-playlists/hls/private/') + + await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + } + } + + async function checkPublicFiles (uuid: string) { + const video = await server.videos.get({ id: uuid }) + + for (const file of getAllFiles(video)) { + expect(file.fileDownloadUrl).to.not.include('/private/') + expect(file.fileUrl).to.not.include('/private/') + + const torrent = await parseTorrentVideo(server, file) + expect(torrent.urlList[0]).to.not.include('private') + + const magnet = decode(file.magnetUri) + expect(magnet.urlList[0]).to.not.include('private') + + await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url: torrent.urlList[0], expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url: magnet.urlList[0], expectedStatus: HttpStatusCode.OK_200 }) + } + + const hls = video.streamingPlaylists[0] + if (hls) { + expect(hls.playlistUrl).to.not.include('private') + expect(hls.segmentsSha256Url).to.not.include('private') + + await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) + } + } + + it('Should upload a private/internal/password protected video and have a private static path', async function () { + this.timeout(120000) + + for (const privacy of [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]) { + const { uuid } = await server.videos.quickUpload({ name: 'video', privacy }) + await waitJobs([ server ]) + + await checkPrivateFiles(uuid) + } + + const { uuid } = await server.videos.quickUpload({ + name: 'video', + privacy: VideoPrivacy.PASSWORD_PROTECTED, + videoPasswords: [ 'my super password' ] + }) + await waitJobs([ server ]) + + await checkPrivateFiles(uuid) + }) + + it('Should upload a public video and update it as private/internal to have a private static path', async function () { + this.timeout(120000) + + for (const privacy of [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]) { + const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PUBLIC }) + await waitJobs([ server ]) + + await server.videos.update({ id: uuid, attributes: { privacy } }) + await waitJobs([ server ]) + + await checkPrivateFiles(uuid) + } + }) + + it('Should upload a private video and update it to unlisted to have a public static path', async function () { + this.timeout(120000) + + const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) + await waitJobs([ server ]) + + await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.UNLISTED } }) + await waitJobs([ server ]) + + await checkPublicFiles(uuid) + }) + + it('Should upload an internal video and update it to public to have a public static path', async function () { + this.timeout(120000) + + const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.INTERNAL }) + await waitJobs([ server ]) + + await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } }) + await waitJobs([ server ]) + + await checkPublicFiles(uuid) + }) + + it('Should upload an internal video and schedule a public publish', async function () { + this.timeout(120000) + + const attributes = { + name: 'video', + privacy: VideoPrivacy.PRIVATE, + scheduleUpdate: { + updateAt: new Date(Date.now() + 1000).toISOString(), + privacy: VideoPrivacy.PUBLIC + } + } + + const { uuid } = await server.videos.upload({ attributes }) + + await waitJobs([ server ]) + await wait(1000) + await server.debug.sendCommand({ body: { command: 'process-update-videos-scheduler' } }) + + await waitJobs([ server ]) + + await checkPublicFiles(uuid) + }) + } + + describe('Without transcoding', function () { + runSuite() + }) + + describe('With transcoding', function () { + + before(async function () { + await server.config.enableMinimumTranscoding() + }) + + runSuite() + }) + }) + + describe('VOD static file right check', function () { + let unrelatedFileToken: string + + async function checkVideoFiles (options: { + id: string + expectedStatus: HttpStatusCodeType + token: string + videoFileToken: string + videoPassword?: string + }) { + const { id, expectedStatus, token, videoFileToken, videoPassword } = options + + const video = await server.videos.getWithToken({ id }) + + for (const file of getAllFiles(video)) { + await makeRawRequest({ url: file.fileUrl, token, expectedStatus }) + await makeRawRequest({ url: file.fileDownloadUrl, token, expectedStatus }) + + await makeRawRequest({ url: file.fileUrl, query: { videoFileToken }, expectedStatus }) + await makeRawRequest({ url: file.fileDownloadUrl, query: { videoFileToken }, expectedStatus }) + + if (videoPassword) { + const headers = { 'x-peertube-video-password': videoPassword } + await makeRawRequest({ url: file.fileUrl, headers, expectedStatus }) + await makeRawRequest({ url: file.fileDownloadUrl, headers, expectedStatus }) + } + } + + const hls = video.streamingPlaylists[0] + await makeRawRequest({ url: hls.playlistUrl, token, expectedStatus }) + await makeRawRequest({ url: hls.segmentsSha256Url, token, expectedStatus }) + + await makeRawRequest({ url: hls.playlistUrl, query: { videoFileToken }, expectedStatus }) + await makeRawRequest({ url: hls.segmentsSha256Url, query: { videoFileToken }, expectedStatus }) + + if (videoPassword) { + const headers = { 'x-peertube-video-password': videoPassword } + await makeRawRequest({ url: hls.playlistUrl, token: null, headers, expectedStatus }) + await makeRawRequest({ url: hls.segmentsSha256Url, token: null, headers, expectedStatus }) + } + } + + before(async function () { + await server.config.enableMinimumTranscoding() + + const { uuid } = await server.videos.quickUpload({ name: 'another video' }) + unrelatedFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid }) + }) + + it('Should not be able to access a private video files without OAuth token and file token', async function () { + this.timeout(120000) + + const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) + await waitJobs([ server ]) + + await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.FORBIDDEN_403, token: null, videoFileToken: null }) + }) + + it('Should not be able to access password protected video files without OAuth token, file token and password', async function () { + this.timeout(120000) + const videoPassword = 'my super password' + + const { uuid } = await server.videos.quickUpload({ + name: 'password protected video', + privacy: VideoPrivacy.PASSWORD_PROTECTED, + videoPasswords: [ videoPassword ] + }) + await waitJobs([ server ]) + + await checkVideoFiles({ + id: uuid, + expectedStatus: HttpStatusCode.FORBIDDEN_403, + token: null, + videoFileToken: null, + videoPassword: null + }) + }) + + it('Should not be able to access an password video files with incorrect OAuth token, file token and password', async function () { + this.timeout(120000) + const videoPassword = 'my super password' + + const { uuid } = await server.videos.quickUpload({ + name: 'password protected video', + privacy: VideoPrivacy.PASSWORD_PROTECTED, + videoPasswords: [ videoPassword ] + }) + await waitJobs([ server ]) + + await checkVideoFiles({ + id: uuid, + expectedStatus: HttpStatusCode.FORBIDDEN_403, + token: userToken, + videoFileToken: unrelatedFileToken, + videoPassword: 'incorrectPassword' + }) + }) + + it('Should not be able to access an private video files without appropriate OAuth token and file token', async function () { + this.timeout(120000) + + const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) + await waitJobs([ server ]) + + await checkVideoFiles({ + id: uuid, + expectedStatus: HttpStatusCode.FORBIDDEN_403, + token: userToken, + videoFileToken: unrelatedFileToken + }) + }) + + it('Should be able to access a private video files with appropriate OAuth token or file token', async function () { + this.timeout(120000) + + const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) + const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid }) + + await waitJobs([ server ]) + + await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken, videoFileToken }) + }) + + it('Should be able to access a password protected video files with appropriate OAuth token or file token', async function () { + this.timeout(120000) + const videoPassword = 'my super password' + + const { uuid } = await server.videos.quickUpload({ + name: 'video', + privacy: VideoPrivacy.PASSWORD_PROTECTED, + videoPasswords: [ videoPassword ] + }) + + const videoFileToken = await server.videoToken.getVideoFileToken({ token: null, videoId: uuid, videoPassword }) + + await waitJobs([ server ]) + + await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken, videoFileToken, videoPassword }) + }) + + it('Should reinject video file token', async function () { + this.timeout(120000) + + const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) + + const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid }) + await waitJobs([ server ]) + + { + const video = await server.videos.getWithToken({ id: uuid }) + const hls = video.streamingPlaylists[0] + const query = { videoFileToken } + const { text } = await makeRawRequest({ url: hls.playlistUrl, query, expectedStatus: HttpStatusCode.OK_200 }) + + expect(text).to.not.include(videoFileToken) + } + + { + await checkVideoFileTokenReinjection({ + server, + videoUUID: uuid, + videoFileToken, + resolutions: [ 240, 720 ], + isLive: false + }) + } + }) + + it('Should be able to access a private video of another user with an admin OAuth token or file token', async function () { + this.timeout(120000) + + const { uuid } = await server.videos.quickUpload({ name: 'video', token: userToken, privacy: VideoPrivacy.PRIVATE }) + const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid }) + + await waitJobs([ server ]) + + await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken, videoFileToken }) + }) + }) + + describe('Live static file path and check', function () { + let normalLiveId: string + let normalLive: LiveVideo + + let permanentLiveId: string + let permanentLive: LiveVideo + + let passwordProtectedLiveId: string + let passwordProtectedLive: LiveVideo + + const correctPassword = 'my super password' + + let unrelatedFileToken: string + + async function checkLiveFiles (options: { live: LiveVideo, liveId: string, videoPassword?: string }) { + const { live, liveId, videoPassword } = options + const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) + await server.live.waitUntilPublished({ videoId: liveId }) + + const video = await server.videos.getWithToken({ id: liveId }) + + const fileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid }) + + const hls = video.streamingPlaylists[0] + + for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) { + expectStartWith(url, server.url + '/static/streaming-playlists/hls/private/') + + await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) + + await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + + if (videoPassword) { + await makeRawRequest({ url, headers: { 'x-peertube-video-password': videoPassword }, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ + url, + headers: { 'x-peertube-video-password': 'incorrectPassword' }, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + } + + } + + await stopFfmpeg(ffmpegCommand) + } + + async function checkReplay (replay: VideoDetails) { + const fileToken = await server.videoToken.getVideoFileToken({ videoId: replay.uuid }) + + const hls = replay.streamingPlaylists[0] + expect(hls.files).to.not.have.lengthOf(0) + + for (const file of hls.files) { + await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url: file.fileUrl, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) + + await makeRawRequest({ url: file.fileUrl, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ + url: file.fileUrl, + query: { videoFileToken: unrelatedFileToken }, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + } + + for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) { + expectStartWith(url, server.url + '/static/streaming-playlists/hls/private/') + + await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) + + await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + } + } + + before(async function () { + await server.config.enableMinimumTranscoding() + + const { uuid } = await server.videos.quickUpload({ name: 'another video' }) + unrelatedFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid }) + + await server.config.enableLive({ + allowReplay: true, + transcoding: true, + resolutions: 'min' + }) + + { + const { video, live } = await server.live.quickCreate({ + saveReplay: true, + permanentLive: false, + privacy: VideoPrivacy.PRIVATE + }) + normalLiveId = video.uuid + normalLive = live + } + + { + const { video, live } = await server.live.quickCreate({ + saveReplay: true, + permanentLive: true, + privacy: VideoPrivacy.PRIVATE + }) + permanentLiveId = video.uuid + permanentLive = live + } + + { + const { video, live } = await server.live.quickCreate({ + saveReplay: false, + permanentLive: false, + privacy: VideoPrivacy.PASSWORD_PROTECTED, + videoPasswords: [ correctPassword ] + }) + passwordProtectedLiveId = video.uuid + passwordProtectedLive = live + } + }) + + it('Should create a private normal live and have a private static path', async function () { + this.timeout(240000) + + await checkLiveFiles({ live: normalLive, liveId: normalLiveId }) + }) + + it('Should create a private permanent live and have a private static path', async function () { + this.timeout(240000) + + await checkLiveFiles({ live: permanentLive, liveId: permanentLiveId }) + }) + + it('Should create a password protected live and have a private static path', async function () { + this.timeout(240000) + + await checkLiveFiles({ live: passwordProtectedLive, liveId: passwordProtectedLiveId, videoPassword: correctPassword }) + }) + + it('Should reinject video file token on permanent live', async function () { + this.timeout(240000) + + const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: permanentLive.rtmpUrl, streamKey: permanentLive.streamKey }) + await server.live.waitUntilPublished({ videoId: permanentLiveId }) + + const video = await server.videos.getWithToken({ id: permanentLiveId }) + const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid }) + const hls = video.streamingPlaylists[0] + + { + const query = { videoFileToken } + const { text } = await makeRawRequest({ url: hls.playlistUrl, query, expectedStatus: HttpStatusCode.OK_200 }) + + expect(text).to.not.include(videoFileToken) + } + + { + await checkVideoFileTokenReinjection({ + server, + videoUUID: permanentLiveId, + videoFileToken, + resolutions: [ 720 ], + isLive: true + }) + } + + await stopFfmpeg(ffmpegCommand) + }) + + it('Should have created a replay of the normal live with a private static path', async function () { + this.timeout(240000) + + await server.live.waitUntilReplacedByReplay({ videoId: normalLiveId }) + + const replay = await server.videos.getWithToken({ id: normalLiveId }) + await checkReplay(replay) + }) + + it('Should have created a replay of the permanent live with a private static path', async function () { + this.timeout(240000) + + await server.live.waitUntilWaiting({ videoId: permanentLiveId }) + await waitJobs([ server ]) + + const live = await server.videos.getWithToken({ id: permanentLiveId }) + const replayFromList = await findExternalSavedVideo(server, live) + const replay = await server.videos.getWithToken({ id: replayFromList.id }) + + await checkReplay(replay) + }) + }) + + describe('With static file right check disabled', function () { + let videoUUID: string + + before(async function () { + this.timeout(240000) + + await server.kill() + + await server.run({ + static_files: { + private_files_require_auth: false + } + }) + + const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.INTERNAL }) + videoUUID = uuid + + await waitJobs([ server ]) + }) + + it('Should not check auth for private static files', async function () { + const video = await server.videos.getWithToken({ id: videoUUID }) + + for (const file of getAllFiles(video)) { + await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + } + + const hls = video.streamingPlaylists[0] + await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/videos/video-storyboard.ts b/packages/tests/src/api/videos/video-storyboard.ts new file mode 100644 index 000000000..7d156aa7f --- /dev/null +++ b/packages/tests/src/api/videos/video-storyboard.ts @@ -0,0 +1,213 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { readdir } from 'fs/promises' +import { basename } from 'path' +import { FIXTURE_URLS } from '@tests/shared/tests.js' +import { areHttpImportTestsDisabled } from '@peertube/peertube-node-utils' +import { HttpStatusCode, VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + makeGetRequest, + PeerTubeServer, + sendRTMPStream, + setAccessTokensToServers, + setDefaultVideoChannel, + stopFfmpeg, + waitJobs +} from '@peertube/peertube-server-commands' + +async function checkStoryboard (options: { + server: PeerTubeServer + uuid: string + tilesCount?: number + minSize?: number +}) { + const { server, uuid, tilesCount, minSize = 1000 } = options + + const { storyboards } = await server.storyboard.list({ id: uuid }) + + expect(storyboards).to.have.lengthOf(1) + + const storyboard = storyboards[0] + + expect(storyboard.spriteDuration).to.equal(1) + expect(storyboard.spriteHeight).to.equal(108) + expect(storyboard.spriteWidth).to.equal(192) + expect(storyboard.storyboardPath).to.exist + + if (tilesCount) { + expect(storyboard.totalWidth).to.equal(192 * Math.min(tilesCount, 10)) + expect(storyboard.totalHeight).to.equal(108 * Math.max((tilesCount / 10), 1)) + } + + const { body } = await makeGetRequest({ url: server.url, path: storyboard.storyboardPath, expectedStatus: HttpStatusCode.OK_200 }) + expect(body.length).to.be.above(minSize) +} + +describe('Test video storyboard', function () { + let servers: PeerTubeServer[] + + let baseUUID: string + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + await doubleFollow(servers[0], servers[1]) + }) + + it('Should generate a storyboard after upload without transcoding', async function () { + this.timeout(120000) + + // 5s video + const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_short.webm' }) + baseUUID = uuid + await waitJobs(servers) + + for (const server of servers) { + await checkStoryboard({ server, uuid, tilesCount: 5 }) + } + }) + + it('Should generate a storyboard after upload without transcoding with a long video', async function () { + this.timeout(120000) + + // 124s video + const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_very_long_10p.mp4' }) + await waitJobs(servers) + + for (const server of servers) { + await checkStoryboard({ server, uuid, tilesCount: 100 }) + } + }) + + it('Should generate a storyboard after upload with transcoding', async function () { + this.timeout(120000) + + await servers[0].config.enableMinimumTranscoding() + + // 5s video + const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_short.webm' }) + await waitJobs(servers) + + for (const server of servers) { + await checkStoryboard({ server, uuid, tilesCount: 5 }) + } + }) + + it('Should generate a storyboard after an audio upload', async function () { + this.timeout(120000) + + // 6s audio + const attributes = { name: 'audio', fixture: 'sample.ogg' } + const { uuid } = await servers[0].videos.upload({ attributes, mode: 'legacy' }) + await waitJobs(servers) + + for (const server of servers) { + try { + await checkStoryboard({ server, uuid, tilesCount: 6, minSize: 250 }) + } catch { // FIXME: to remove after ffmpeg CI upgrade, ffmpeg CI version (4.3) generates a 7.6s length video + await checkStoryboard({ server, uuid, tilesCount: 8, minSize: 250 }) + } + } + }) + + it('Should generate a storyboard after HTTP import', async function () { + this.timeout(120000) + + if (areHttpImportTestsDisabled()) return + + // 3s video + const { video } = await servers[0].imports.importVideo({ + attributes: { + targetUrl: FIXTURE_URLS.goodVideo, + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC + } + }) + await waitJobs(servers) + + for (const server of servers) { + await checkStoryboard({ server, uuid: video.uuid, tilesCount: 3 }) + } + }) + + it('Should generate a storyboard after torrent import', async function () { + this.timeout(120000) + + if (areHttpImportTestsDisabled()) return + + // 10s video + const { video } = await servers[0].imports.importVideo({ + attributes: { + magnetUri: FIXTURE_URLS.magnet, + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC + } + }) + await waitJobs(servers) + + for (const server of servers) { + await checkStoryboard({ server, uuid: video.uuid, tilesCount: 10 }) + } + }) + + it('Should generate a storyboard after a live', async function () { + this.timeout(240000) + + await servers[0].config.enableLive({ allowReplay: true, transcoding: true, resolutions: 'min' }) + + const { live, video } = await servers[0].live.quickCreate({ + saveReplay: true, + permanentLive: false, + privacy: VideoPrivacy.PUBLIC + }) + + const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) + await servers[0].live.waitUntilPublished({ videoId: video.id }) + + await stopFfmpeg(ffmpegCommand) + + await servers[0].live.waitUntilReplacedByReplay({ videoId: video.id }) + await waitJobs(servers) + + for (const server of servers) { + await checkStoryboard({ server, uuid: video.uuid }) + } + }) + + it('Should cleanup storyboards on video deletion', async function () { + this.timeout(60000) + + const { storyboards } = await servers[0].storyboard.list({ id: baseUUID }) + const storyboardName = basename(storyboards[0].storyboardPath) + + const listFiles = () => { + const storyboardPath = servers[0].getDirectoryPath('storyboards') + return readdir(storyboardPath) + } + + { + const storyboads = await listFiles() + expect(storyboads).to.include(storyboardName) + } + + await servers[0].videos.remove({ id: baseUUID }) + await waitJobs(servers) + + { + const storyboads = await listFiles() + expect(storyboads).to.not.include(storyboardName) + } + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/videos/videos-common-filters.ts b/packages/tests/src/api/videos/videos-common-filters.ts new file mode 100644 index 000000000..9e75bd6ca --- /dev/null +++ b/packages/tests/src/api/videos/videos-common-filters.ts @@ -0,0 +1,499 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { pick } from '@peertube/peertube-core-utils' +import { + HttpStatusCode, + HttpStatusCodeType, + UserRole, + Video, + VideoDetails, + VideoInclude, + VideoIncludeType, + VideoPrivacy, + VideoPrivacyType +} from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + makeGetRequest, + PeerTubeServer, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test videos filter', function () { + let servers: PeerTubeServer[] + let paths: string[] + let remotePaths: string[] + + const subscriptionVideosPath = '/api/v1/users/me/subscriptions/videos' + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(240000) + + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + await setDefaultAccountAvatar(servers) + + await servers[1].config.enableMinimumTranscoding() + + for (const server of servers) { + const moderator = { username: 'moderator', password: 'my super password' } + await server.users.create({ username: moderator.username, password: moderator.password, role: UserRole.MODERATOR }) + server['moderatorAccessToken'] = await server.login.getAccessToken(moderator) + + await server.videos.upload({ attributes: { name: 'public ' + server.serverNumber } }) + + { + const attributes = { name: 'unlisted ' + server.serverNumber, privacy: VideoPrivacy.UNLISTED } + await server.videos.upload({ attributes }) + } + + { + const attributes = { name: 'private ' + server.serverNumber, privacy: VideoPrivacy.PRIVATE } + await server.videos.upload({ attributes }) + } + + // Subscribing to itself + await server.subscriptions.add({ targetUri: 'root_channel@' + server.host }) + } + + await doubleFollow(servers[0], servers[1]) + + paths = [ + `/api/v1/video-channels/root_channel/videos`, + `/api/v1/accounts/root/videos`, + '/api/v1/videos', + '/api/v1/search/videos', + subscriptionVideosPath + ] + + remotePaths = [ + `/api/v1/video-channels/root_channel@${servers[1].host}/videos`, + `/api/v1/accounts/root@${servers[1].host}/videos`, + '/api/v1/videos', + '/api/v1/search/videos' + ] + }) + + describe('Check videos filters', function () { + + async function listVideos (options: { + server: PeerTubeServer + path: string + isLocal?: boolean + hasWebVideoFiles?: boolean + hasHLSFiles?: boolean + include?: VideoIncludeType + privacyOneOf?: VideoPrivacyType[] + category?: number + tagsAllOf?: string[] + token?: string + expectedStatus?: HttpStatusCodeType + excludeAlreadyWatched?: boolean + }) { + const res = await makeGetRequest({ + url: options.server.url, + path: options.path, + token: options.token ?? options.server.accessToken, + query: { + ...pick(options, [ + 'isLocal', + 'include', + 'category', + 'tagsAllOf', + 'hasWebVideoFiles', + 'hasHLSFiles', + 'privacyOneOf', + 'excludeAlreadyWatched' + ]), + + sort: 'createdAt' + }, + expectedStatus: options.expectedStatus ?? HttpStatusCode.OK_200 + }) + + return res.body.data as Video[] + } + + async function getVideosNames ( + options: { + server: PeerTubeServer + isLocal?: boolean + include?: VideoIncludeType + privacyOneOf?: VideoPrivacyType[] + token?: string + expectedStatus?: HttpStatusCodeType + skipSubscription?: boolean + excludeAlreadyWatched?: boolean + } + ) { + const { skipSubscription = false } = options + const videosResults: string[][] = [] + + for (const path of paths) { + if (skipSubscription && path === subscriptionVideosPath) continue + + const videos = await listVideos({ ...options, path }) + + videosResults.push(videos.map(v => v.name)) + } + + return videosResults + } + + it('Should display local videos', async function () { + for (const server of servers) { + const namesResults = await getVideosNames({ server, isLocal: true }) + + for (const names of namesResults) { + expect(names).to.have.lengthOf(1) + expect(names[0]).to.equal('public ' + server.serverNumber) + } + } + }) + + it('Should display local videos with hidden privacy by the admin or the moderator', async function () { + for (const server of servers) { + for (const token of [ server.accessToken, server['moderatorAccessToken'] ]) { + + const namesResults = await getVideosNames( + { + server, + token, + isLocal: true, + privacyOneOf: [ VideoPrivacy.UNLISTED, VideoPrivacy.PUBLIC, VideoPrivacy.PRIVATE ], + skipSubscription: true + } + ) + + for (const names of namesResults) { + expect(names).to.have.lengthOf(3) + + expect(names[0]).to.equal('public ' + server.serverNumber) + expect(names[1]).to.equal('unlisted ' + server.serverNumber) + expect(names[2]).to.equal('private ' + server.serverNumber) + } + } + } + }) + + it('Should display all videos by the admin or the moderator', async function () { + for (const server of servers) { + for (const token of [ server.accessToken, server['moderatorAccessToken'] ]) { + + const [ channelVideos, accountVideos, videos, searchVideos ] = await getVideosNames({ + server, + token, + privacyOneOf: [ VideoPrivacy.UNLISTED, VideoPrivacy.PUBLIC, VideoPrivacy.PRIVATE ] + }) + + expect(channelVideos).to.have.lengthOf(3) + expect(accountVideos).to.have.lengthOf(3) + + expect(videos).to.have.lengthOf(5) + expect(searchVideos).to.have.lengthOf(5) + } + } + }) + + it('Should display only remote videos', async function () { + this.timeout(120000) + + await servers[1].videos.upload({ attributes: { name: 'remote video' } }) + + await waitJobs(servers) + + const finder = (videos: Video[]) => videos.find(v => v.name === 'remote video') + + for (const path of remotePaths) { + { + const videos = await listVideos({ server: servers[0], path }) + const video = finder(videos) + expect(video).to.exist + } + + { + const videos = await listVideos({ server: servers[0], path, isLocal: false }) + const video = finder(videos) + expect(video).to.exist + } + + { + const videos = await listVideos({ server: servers[0], path, isLocal: true }) + const video = finder(videos) + expect(video).to.not.exist + } + } + }) + + it('Should include not published videos', async function () { + await servers[0].config.enableLive({ allowReplay: false, transcoding: false }) + await servers[0].live.create({ fields: { name: 'live video', channelId: servers[0].store.channel.id, privacy: VideoPrivacy.PUBLIC } }) + + const finder = (videos: Video[]) => videos.find(v => v.name === 'live video') + + for (const path of paths) { + { + const videos = await listVideos({ server: servers[0], path }) + const video = finder(videos) + expect(video).to.not.exist + expect(videos[0].state).to.not.exist + expect(videos[0].waitTranscoding).to.not.exist + } + + { + const videos = await listVideos({ server: servers[0], path, include: VideoInclude.NOT_PUBLISHED_STATE }) + const video = finder(videos) + expect(video).to.exist + expect(video.state).to.exist + } + } + }) + + it('Should include blacklisted videos', async function () { + const { id } = await servers[0].videos.upload({ attributes: { name: 'blacklisted' } }) + + await servers[0].blacklist.add({ videoId: id }) + + const finder = (videos: Video[]) => videos.find(v => v.name === 'blacklisted') + + for (const path of paths) { + { + const videos = await listVideos({ server: servers[0], path }) + const video = finder(videos) + expect(video).to.not.exist + expect(videos[0].blacklisted).to.not.exist + } + + { + const videos = await listVideos({ server: servers[0], path, include: VideoInclude.BLACKLISTED }) + const video = finder(videos) + expect(video).to.exist + expect(video.blacklisted).to.be.true + } + } + }) + + it('Should include videos from muted account', async function () { + const finder = (videos: Video[]) => videos.find(v => v.name === 'remote video') + + await servers[0].blocklist.addToServerBlocklist({ account: 'root@' + servers[1].host }) + + for (const path of remotePaths) { + { + const videos = await listVideos({ server: servers[0], path }) + const video = finder(videos) + expect(video).to.not.exist + + // Some paths won't have videos + if (videos[0]) { + expect(videos[0].blockedOwner).to.not.exist + expect(videos[0].blockedServer).to.not.exist + } + } + + { + const videos = await listVideos({ server: servers[0], path, include: VideoInclude.BLOCKED_OWNER }) + + const video = finder(videos) + expect(video).to.exist + expect(video.blockedServer).to.be.false + expect(video.blockedOwner).to.be.true + } + } + + await servers[0].blocklist.removeFromServerBlocklist({ account: 'root@' + servers[1].host }) + }) + + it('Should include videos from muted server', async function () { + const finder = (videos: Video[]) => videos.find(v => v.name === 'remote video') + + await servers[0].blocklist.addToServerBlocklist({ server: servers[1].host }) + + for (const path of remotePaths) { + { + const videos = await listVideos({ server: servers[0], path }) + const video = finder(videos) + expect(video).to.not.exist + + // Some paths won't have videos + if (videos[0]) { + expect(videos[0].blockedOwner).to.not.exist + expect(videos[0].blockedServer).to.not.exist + } + } + + { + const videos = await listVideos({ server: servers[0], path, include: VideoInclude.BLOCKED_OWNER }) + const video = finder(videos) + expect(video).to.exist + expect(video.blockedServer).to.be.true + expect(video.blockedOwner).to.be.false + } + } + + await servers[0].blocklist.removeFromServerBlocklist({ server: servers[1].host }) + }) + + it('Should include video files', async function () { + for (const path of paths) { + { + const videos = await listVideos({ server: servers[0], path }) + + for (const video of videos) { + const videoWithFiles = video as VideoDetails + + expect(videoWithFiles.files).to.not.exist + expect(videoWithFiles.streamingPlaylists).to.not.exist + } + } + + { + const videos = await listVideos({ server: servers[0], path, include: VideoInclude.FILES }) + + for (const video of videos) { + const videoWithFiles = video as VideoDetails + + expect(videoWithFiles.files).to.exist + expect(videoWithFiles.files).to.have.length.at.least(1) + } + } + } + }) + + it('Should filter by tags and category', async function () { + await servers[0].videos.upload({ attributes: { name: 'tag filter', tags: [ 'tag1', 'tag2' ] } }) + await servers[0].videos.upload({ attributes: { name: 'tag filter with category', tags: [ 'tag3' ], category: 4 } }) + + for (const path of paths) { + { + const videos = await listVideos({ server: servers[0], path, tagsAllOf: [ 'tag1', 'tag2' ] }) + expect(videos).to.have.lengthOf(1) + expect(videos[0].name).to.equal('tag filter') + } + + { + const videos = await listVideos({ server: servers[0], path, tagsAllOf: [ 'tag1', 'tag3' ] }) + expect(videos).to.have.lengthOf(0) + } + + { + const { data, total } = await servers[0].videos.list({ tagsAllOf: [ 'tag3' ], categoryOneOf: [ 4 ] }) + expect(total).to.equal(1) + expect(data[0].name).to.equal('tag filter with category') + } + + { + const { total } = await servers[0].videos.list({ tagsAllOf: [ 'tag4' ], categoryOneOf: [ 4 ] }) + expect(total).to.equal(0) + } + } + }) + + it('Should filter by HLS or Web Video files', async function () { + this.timeout(360000) + + const finderFactory = (name: string) => (videos: Video[]) => videos.some(v => v.name === name) + + await servers[0].config.enableTranscoding({ hls: false, webVideo: true }) + await servers[0].videos.upload({ attributes: { name: 'web video' } }) + const hasWebVideo = finderFactory('web video') + + await waitJobs(servers) + + await servers[0].config.enableTranscoding({ hls: true, webVideo: false }) + await servers[0].videos.upload({ attributes: { name: 'hls video' } }) + const hasHLS = finderFactory('hls video') + + await waitJobs(servers) + + await servers[0].config.enableTranscoding({ hls: true, webVideo: true }) + await servers[0].videos.upload({ attributes: { name: 'hls and web video' } }) + const hasBoth = finderFactory('hls and web video') + + await waitJobs(servers) + + for (const path of paths) { + { + const videos = await listVideos({ server: servers[0], path, hasWebVideoFiles: true }) + + expect(hasWebVideo(videos)).to.be.true + expect(hasHLS(videos)).to.be.false + expect(hasBoth(videos)).to.be.true + } + + { + const videos = await listVideos({ server: servers[0], path, hasWebVideoFiles: false }) + + expect(hasWebVideo(videos)).to.be.false + expect(hasHLS(videos)).to.be.true + expect(hasBoth(videos)).to.be.false + } + + { + const videos = await listVideos({ server: servers[0], path, hasHLSFiles: true }) + + expect(hasWebVideo(videos)).to.be.false + expect(hasHLS(videos)).to.be.true + expect(hasBoth(videos)).to.be.true + } + + { + const videos = await listVideos({ server: servers[0], path, hasHLSFiles: false }) + + expect(hasWebVideo(videos)).to.be.true + expect(hasHLS(videos)).to.be.false + expect(hasBoth(videos)).to.be.false + } + + { + const videos = await listVideos({ server: servers[0], path, hasHLSFiles: false, hasWebVideoFiles: false }) + + expect(hasWebVideo(videos)).to.be.false + expect(hasHLS(videos)).to.be.false + expect(hasBoth(videos)).to.be.false + } + + { + const videos = await listVideos({ server: servers[0], path, hasHLSFiles: true, hasWebVideoFiles: true }) + + expect(hasWebVideo(videos)).to.be.false + expect(hasHLS(videos)).to.be.false + expect(hasBoth(videos)).to.be.true + } + } + }) + + it('Should filter already watched videos by the user', async function () { + const { id } = await servers[0].videos.upload({ attributes: { name: 'video for history' } }) + + for (const path of paths) { + const videos = await listVideos({ server: servers[0], path, isLocal: true, excludeAlreadyWatched: true }) + const foundVideo = videos.find(video => video.id === id) + + expect(foundVideo).to.not.be.undefined + } + await servers[0].views.view({ id, currentTime: 1, token: servers[0].accessToken }) + + for (const path of paths) { + const videos = await listVideos({ server: servers[0], path, excludeAlreadyWatched: true }) + const foundVideo = videos.find(video => video.id === id) + + expect(foundVideo).to.be.undefined + } + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/videos/videos-history.ts b/packages/tests/src/api/videos/videos-history.ts new file mode 100644 index 000000000..75c0fcebd --- /dev/null +++ b/packages/tests/src/api/videos/videos-history.ts @@ -0,0 +1,230 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { Video } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + killallServers, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test videos history', function () { + let server: PeerTubeServer = null + let video1Id: number + let video1UUID: string + let video2UUID: string + let video3UUID: string + let video3WatchedDate: Date + let userAccessToken: string + + before(async function () { + this.timeout(120000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + + // 10 seconds long + const fixture = 'video_short1.webm' + + { + const { id, uuid } = await server.videos.upload({ attributes: { name: 'video 1', fixture } }) + video1UUID = uuid + video1Id = id + } + + { + const { uuid } = await server.videos.upload({ attributes: { name: 'video 2', fixture } }) + video2UUID = uuid + } + + { + const { uuid } = await server.videos.upload({ attributes: { name: 'video 3', fixture } }) + video3UUID = uuid + } + + userAccessToken = await server.users.generateUserAndToken('user_1') + }) + + it('Should get videos, without watching history', async function () { + const { data } = await server.videos.listWithToken() + + for (const video of data) { + const videoDetails = await server.videos.getWithToken({ id: video.id }) + + expect(video.userHistory).to.be.undefined + expect(videoDetails.userHistory).to.be.undefined + } + }) + + it('Should watch the first and second video', async function () { + await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 }) + await server.views.view({ id: video1UUID, token: server.accessToken, currentTime: 3 }) + }) + + it('Should return the correct history when listing, searching and getting videos', async function () { + const videosOfVideos: Video[][] = [] + + { + const { data } = await server.videos.listWithToken() + videosOfVideos.push(data) + } + + { + const body = await server.search.searchVideos({ token: server.accessToken, search: 'video' }) + videosOfVideos.push(body.data) + } + + for (const videos of videosOfVideos) { + const video1 = videos.find(v => v.uuid === video1UUID) + const video2 = videos.find(v => v.uuid === video2UUID) + const video3 = videos.find(v => v.uuid === video3UUID) + + expect(video1.userHistory).to.not.be.undefined + expect(video1.userHistory.currentTime).to.equal(3) + + expect(video2.userHistory).to.not.be.undefined + expect(video2.userHistory.currentTime).to.equal(8) + + expect(video3.userHistory).to.be.undefined + } + + { + const videoDetails = await server.videos.getWithToken({ id: video1UUID }) + + expect(videoDetails.userHistory).to.not.be.undefined + expect(videoDetails.userHistory.currentTime).to.equal(3) + } + + { + const videoDetails = await server.videos.getWithToken({ id: video2UUID }) + + expect(videoDetails.userHistory).to.not.be.undefined + expect(videoDetails.userHistory.currentTime).to.equal(8) + } + + { + const videoDetails = await server.videos.getWithToken({ id: video3UUID }) + + expect(videoDetails.userHistory).to.be.undefined + } + }) + + it('Should have these videos when listing my history', async function () { + video3WatchedDate = new Date() + await server.views.view({ id: video3UUID, token: server.accessToken, currentTime: 2 }) + + const body = await server.history.list() + + expect(body.total).to.equal(3) + + const videos = body.data + expect(videos[0].name).to.equal('video 3') + expect(videos[1].name).to.equal('video 1') + expect(videos[2].name).to.equal('video 2') + }) + + it('Should not have videos history on another user', async function () { + const body = await server.history.list({ token: userAccessToken }) + + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) + }) + + it('Should be able to search through videos in my history', async function () { + const body = await server.history.list({ search: '2' }) + expect(body.total).to.equal(1) + + const videos = body.data + expect(videos[0].name).to.equal('video 2') + }) + + it('Should clear my history', async function () { + await server.history.removeAll({ beforeDate: video3WatchedDate.toISOString() }) + }) + + it('Should have my history cleared', async function () { + const body = await server.history.list() + expect(body.total).to.equal(1) + + const videos = body.data + expect(videos[0].name).to.equal('video 3') + }) + + it('Should disable videos history', async function () { + await server.users.updateMe({ + videosHistoryEnabled: false + }) + + await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 }) + + const { data } = await server.history.list() + expect(data[0].name).to.not.equal('video 2') + }) + + it('Should re-enable videos history', async function () { + await server.users.updateMe({ + videosHistoryEnabled: true + }) + + await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 }) + + const { data } = await server.history.list() + expect(data[0].name).to.equal('video 2') + }) + + it('Should not clean old history', async function () { + this.timeout(50000) + + await killallServers([ server ]) + + await server.run({ history: { videos: { max_age: '10 days' } } }) + + await wait(6000) + + // Should still have history + + const body = await server.history.list() + expect(body.total).to.equal(2) + }) + + it('Should clean old history', async function () { + this.timeout(50000) + + await killallServers([ server ]) + + await server.run({ history: { videos: { max_age: '5 seconds' } } }) + + await wait(6000) + + const body = await server.history.list() + expect(body.total).to.equal(0) + }) + + it('Should delete a specific history element', async function () { + { + await server.views.view({ id: video1UUID, token: server.accessToken, currentTime: 4 }) + await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 }) + } + + { + const body = await server.history.list() + expect(body.total).to.equal(2) + } + + { + await server.history.removeElement({ videoId: video1Id }) + + const body = await server.history.list() + expect(body.total).to.equal(1) + expect(body.data[0].uuid).to.equal(video2UUID) + } + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/videos/videos-overview.ts b/packages/tests/src/api/videos/videos-overview.ts new file mode 100644 index 000000000..7d74d6db2 --- /dev/null +++ b/packages/tests/src/api/videos/videos-overview.ts @@ -0,0 +1,129 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { VideosOverview } from '@peertube/peertube-models' +import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@peertube/peertube-server-commands' + +describe('Test a videos overview', function () { + let server: PeerTubeServer = null + + function testOverviewCount (overview: VideosOverview, expected: number) { + expect(overview.tags).to.have.lengthOf(expected) + expect(overview.categories).to.have.lengthOf(expected) + expect(overview.channels).to.have.lengthOf(expected) + } + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + }) + + it('Should send empty overview', async function () { + const body = await server.overviews.getVideos({ page: 1 }) + + testOverviewCount(body, 0) + }) + + it('Should upload 5 videos in a specific category, tag and channel but not include them in overview', async function () { + this.timeout(60000) + + await wait(3000) + + await server.videos.upload({ + attributes: { + name: 'video 0', + category: 3, + tags: [ 'coucou1', 'coucou2' ] + } + }) + + const body = await server.overviews.getVideos({ page: 1 }) + + testOverviewCount(body, 0) + }) + + it('Should upload another video and include all videos in the overview', async function () { + this.timeout(120000) + + { + for (let i = 1; i < 6; i++) { + await server.videos.upload({ + attributes: { + name: 'video ' + i, + category: 3, + tags: [ 'coucou1', 'coucou2' ] + } + }) + } + + await wait(3000) + } + + { + const body = await server.overviews.getVideos({ page: 1 }) + + testOverviewCount(body, 1) + } + + { + const overview = await server.overviews.getVideos({ page: 2 }) + + expect(overview.tags).to.have.lengthOf(1) + expect(overview.categories).to.have.lengthOf(0) + expect(overview.channels).to.have.lengthOf(0) + } + }) + + it('Should have the correct overview', async function () { + const overview1 = await server.overviews.getVideos({ page: 1 }) + const overview2 = await server.overviews.getVideos({ page: 2 }) + + for (const arr of [ overview1.tags, overview1.categories, overview1.channels, overview2.tags ]) { + expect(arr).to.have.lengthOf(1) + + const obj = arr[0] + + expect(obj.videos).to.have.lengthOf(6) + expect(obj.videos[0].name).to.equal('video 5') + expect(obj.videos[1].name).to.equal('video 4') + expect(obj.videos[2].name).to.equal('video 3') + expect(obj.videos[3].name).to.equal('video 2') + expect(obj.videos[4].name).to.equal('video 1') + expect(obj.videos[5].name).to.equal('video 0') + } + + const tags = [ overview1.tags[0].tag, overview2.tags[0].tag ] + expect(tags.find(t => t === 'coucou1')).to.not.be.undefined + expect(tags.find(t => t === 'coucou2')).to.not.be.undefined + + expect(overview1.categories[0].category.id).to.equal(3) + + expect(overview1.channels[0].channel.name).to.equal('root_channel') + }) + + it('Should hide muted accounts', async function () { + const token = await server.users.generateUserAndToken('choco') + + await server.blocklist.addToMyBlocklist({ token, account: 'root@' + server.host }) + + { + const body = await server.overviews.getVideos({ page: 1 }) + + testOverviewCount(body, 1) + } + + { + const body = await server.overviews.getVideos({ page: 1, token }) + + testOverviewCount(body, 0) + } + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/api/views/index.ts b/packages/tests/src/api/views/index.ts new file mode 100644 index 000000000..2b7334d1a --- /dev/null +++ b/packages/tests/src/api/views/index.ts @@ -0,0 +1,5 @@ +export * from './video-views-counter.js' +export * from './video-views-overall-stats.js' +export * from './video-views-retention-stats.js' +export * from './video-views-timeserie-stats.js' +export * from './videos-views-cleaner.js' diff --git a/packages/tests/src/api/views/video-views-counter.ts b/packages/tests/src/api/views/video-views-counter.ts new file mode 100644 index 000000000..d9afb0f18 --- /dev/null +++ b/packages/tests/src/api/views/video-views-counter.ts @@ -0,0 +1,153 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { FfmpegCommand } from 'fluent-ffmpeg' +import { prepareViewsServers, prepareViewsVideos, processViewsBuffer } from '@tests/shared/views.js' +import { wait } from '@peertube/peertube-core-utils' +import { cleanupTests, PeerTubeServer, stopFfmpeg, waitJobs } from '@peertube/peertube-server-commands' + +describe('Test video views/viewers counters', function () { + let servers: PeerTubeServer[] + + async function checkCounter (field: 'views' | 'viewers', id: string, expected: number) { + for (const server of servers) { + const video = await server.videos.get({ id }) + + const messageSuffix = video.isLive + ? 'live video' + : 'vod video' + + expect(video[field]).to.equal(expected, `${field} not valid on server ${server.serverNumber} for ${messageSuffix} ${video.uuid}`) + } + } + + before(async function () { + this.timeout(120000) + + servers = await prepareViewsServers() + }) + + describe('Test views counter on VOD', function () { + let videoUUID: string + + before(async function () { + this.timeout(120000) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) + videoUUID = uuid + + await waitJobs(servers) + }) + + it('Should not view a video if watch time is below the threshold', async function () { + await servers[0].views.simulateViewer({ id: videoUUID, currentTimes: [ 1, 2 ] }) + await processViewsBuffer(servers) + + await checkCounter('views', videoUUID, 0) + }) + + it('Should view a video if watch time is above the threshold', async function () { + await servers[0].views.simulateViewer({ id: videoUUID, currentTimes: [ 1, 4 ] }) + await processViewsBuffer(servers) + + await checkCounter('views', videoUUID, 1) + }) + + it('Should not view again this video with the same IP', async function () { + await servers[0].views.simulateViewer({ id: videoUUID, xForwardedFor: '0.0.0.1,127.0.0.1', currentTimes: [ 1, 4 ] }) + await servers[0].views.simulateViewer({ id: videoUUID, xForwardedFor: '0.0.0.1,127.0.0.1', currentTimes: [ 1, 4 ] }) + await processViewsBuffer(servers) + + await checkCounter('views', videoUUID, 2) + }) + + it('Should view the video from server 2 and send the event', async function () { + await servers[1].views.simulateViewer({ id: videoUUID, currentTimes: [ 1, 4 ] }) + await waitJobs(servers) + await processViewsBuffer(servers) + + await checkCounter('views', videoUUID, 3) + }) + }) + + describe('Test views and viewers counters on live and VOD', function () { + let liveVideoId: string + let vodVideoId: string + let command: FfmpegCommand + + before(async function () { + this.timeout(240000); + + ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) + }) + + it('Should display no views and viewers', async function () { + await checkCounter('views', liveVideoId, 0) + await checkCounter('viewers', liveVideoId, 0) + + await checkCounter('views', vodVideoId, 0) + await checkCounter('viewers', vodVideoId, 0) + }) + + it('Should view twice and display 1 view/viewer', async function () { + this.timeout(30000) + + await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) + await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) + await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) + await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) + + await waitJobs(servers) + await checkCounter('viewers', liveVideoId, 1) + await checkCounter('viewers', vodVideoId, 1) + + await processViewsBuffer(servers) + + await checkCounter('views', liveVideoId, 1) + await checkCounter('views', vodVideoId, 1) + }) + + it('Should wait and display 0 viewers but still have 1 view', async function () { + this.timeout(30000) + + await wait(12000) + await waitJobs(servers) + + await checkCounter('views', liveVideoId, 1) + await checkCounter('viewers', liveVideoId, 0) + + await checkCounter('views', vodVideoId, 1) + await checkCounter('viewers', vodVideoId, 0) + }) + + it('Should view on a remote and on local and display 2 viewers and 3 views', async function () { + this.timeout(30000) + + await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) + await servers[1].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) + await servers[1].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) + + await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) + await servers[1].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) + await servers[1].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) + + await waitJobs(servers) + + await checkCounter('viewers', liveVideoId, 2) + await checkCounter('viewers', vodVideoId, 2) + + await processViewsBuffer(servers) + + await checkCounter('views', liveVideoId, 3) + await checkCounter('views', vodVideoId, 3) + }) + + after(async function () { + await stopFfmpeg(command) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/views/video-views-overall-stats.ts b/packages/tests/src/api/views/video-views-overall-stats.ts new file mode 100644 index 000000000..6ea0da2d9 --- /dev/null +++ b/packages/tests/src/api/views/video-views-overall-stats.ts @@ -0,0 +1,368 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { FfmpegCommand } from 'fluent-ffmpeg' +import { prepareViewsServers, prepareViewsVideos, processViewersStats } from '@tests/shared/views.js' +import { cleanupTests, PeerTubeServer, stopFfmpeg, waitJobs } from '@peertube/peertube-server-commands' +import { wait } from '@peertube/peertube-core-utils' +import { VideoStatsOverall } from '@peertube/peertube-models' + +/** + * + * Simulate 5 sections of viewers + * * user0 started and ended before start date + * * user1 started before start date and ended in the interval + * * user2 started started in the interval and ended after end date + * * user3 started and ended in the interval + * * user4 started and ended after end date + */ +async function simulateComplexViewers (servers: PeerTubeServer[], videoUUID: string) { + const user0 = '8.8.8.8,127.0.0.1' + const user1 = '8.8.8.8,127.0.0.1' + const user2 = '8.8.8.9,127.0.0.1' + const user3 = '8.8.8.10,127.0.0.1' + const user4 = '8.8.8.11,127.0.0.1' + + await servers[0].views.view({ id: videoUUID, currentTime: 0, xForwardedFor: user0 }) // User 0 starts + await wait(500) + + await servers[0].views.view({ id: videoUUID, currentTime: 0, xForwardedFor: user1 }) // User 1 starts + await servers[0].views.view({ id: videoUUID, currentTime: 2, xForwardedFor: user0 }) // User 0 ends + await wait(500) + + const startDate = new Date().toISOString() + await servers[0].views.view({ id: videoUUID, currentTime: 0, xForwardedFor: user2 }) // User 2 starts + await wait(500) + + await servers[0].views.view({ id: videoUUID, currentTime: 0, xForwardedFor: user3 }) // User 3 starts + await wait(500) + + await servers[0].views.view({ id: videoUUID, currentTime: 4, xForwardedFor: user1 }) // User 1 ends + await wait(500) + + await servers[0].views.view({ id: videoUUID, currentTime: 3, xForwardedFor: user3 }) // User 3 ends + await wait(500) + + const endDate = new Date().toISOString() + await servers[0].views.view({ id: videoUUID, currentTime: 0, xForwardedFor: user4 }) // User 4 starts + await servers[0].views.view({ id: videoUUID, currentTime: 5, xForwardedFor: user2 }) // User 2 ends + await wait(500) + + await servers[0].views.view({ id: videoUUID, currentTime: 1, xForwardedFor: user4 }) // User 4 ends + + await processViewersStats(servers) + + return { startDate, endDate } +} + +describe('Test views overall stats', function () { + let servers: PeerTubeServer[] + + before(async function () { + this.timeout(120000) + + servers = await prepareViewsServers() + }) + + describe('Test watch time stats of local videos on live and VOD', function () { + let vodVideoId: string + let liveVideoId: string + let command: FfmpegCommand + + before(async function () { + this.timeout(240000); + + ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) + }) + + it('Should display overall stats of a video with no viewers', async function () { + for (const videoId of [ liveVideoId, vodVideoId ]) { + const stats = await servers[0].videoStats.getOverallStats({ videoId }) + const video = await servers[0].videos.get({ id: videoId }) + + expect(video.views).to.equal(0) + expect(stats.averageWatchTime).to.equal(0) + expect(stats.totalWatchTime).to.equal(0) + expect(stats.totalViewers).to.equal(0) + } + }) + + it('Should display overall stats with 1 viewer below the watch time limit', async function () { + this.timeout(60000) + + for (const videoId of [ liveVideoId, vodVideoId ]) { + await servers[0].views.simulateViewer({ id: videoId, currentTimes: [ 0, 1 ] }) + } + + await processViewersStats(servers) + + for (const videoId of [ liveVideoId, vodVideoId ]) { + const stats = await servers[0].videoStats.getOverallStats({ videoId }) + const video = await servers[0].videos.get({ id: videoId }) + + expect(video.views).to.equal(0) + expect(stats.averageWatchTime).to.equal(1) + expect(stats.totalWatchTime).to.equal(1) + expect(stats.totalViewers).to.equal(1) + } + }) + + it('Should display overall stats with 2 viewers', async function () { + this.timeout(60000) + + { + await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 3 ] }) + await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35, 40 ] }) + + await processViewersStats(servers) + + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId }) + const video = await servers[0].videos.get({ id: vodVideoId }) + + expect(video.views).to.equal(1) + expect(stats.averageWatchTime).to.equal(2) + expect(stats.totalWatchTime).to.equal(4) + expect(stats.totalViewers).to.equal(2) + } + + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: liveVideoId }) + const video = await servers[0].videos.get({ id: liveVideoId }) + + expect(video.views).to.equal(1) + expect(stats.averageWatchTime).to.equal(21) + expect(stats.totalWatchTime).to.equal(41) + expect(stats.totalViewers).to.equal(2) + } + } + }) + + it('Should display overall stats with a remote viewer below the watch time limit', async function () { + this.timeout(60000) + + for (const videoId of [ liveVideoId, vodVideoId ]) { + await servers[1].views.simulateViewer({ id: videoId, currentTimes: [ 0, 2 ] }) + } + + await processViewersStats(servers) + + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId }) + const video = await servers[0].videos.get({ id: vodVideoId }) + + expect(video.views).to.equal(1) + expect(stats.averageWatchTime).to.equal(2) + expect(stats.totalWatchTime).to.equal(6) + expect(stats.totalViewers).to.equal(3) + } + + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: liveVideoId }) + const video = await servers[0].videos.get({ id: liveVideoId }) + + expect(video.views).to.equal(1) + expect(stats.averageWatchTime).to.equal(14) + expect(stats.totalWatchTime).to.equal(43) + expect(stats.totalViewers).to.equal(3) + } + }) + + it('Should display overall stats with a remote viewer above the watch time limit', async function () { + this.timeout(60000) + + await servers[1].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) + await servers[1].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 45 ] }) + await processViewersStats(servers) + + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId }) + const video = await servers[0].videos.get({ id: vodVideoId }) + + expect(video.views).to.equal(2) + expect(stats.averageWatchTime).to.equal(3) + expect(stats.totalWatchTime).to.equal(11) + expect(stats.totalViewers).to.equal(4) + } + + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: liveVideoId }) + const video = await servers[0].videos.get({ id: liveVideoId }) + + expect(video.views).to.equal(2) + expect(stats.averageWatchTime).to.equal(22) + expect(stats.totalWatchTime).to.equal(88) + expect(stats.totalViewers).to.equal(4) + } + }) + + it('Should filter overall stats by date', async function () { + this.timeout(60000) + + const beforeView = new Date() + + await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 3 ] }) + await processViewersStats(servers) + + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId, startDate: beforeView.toISOString() }) + expect(stats.averageWatchTime).to.equal(3) + expect(stats.totalWatchTime).to.equal(3) + expect(stats.totalViewers).to.equal(1) + } + + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: liveVideoId, endDate: beforeView.toISOString() }) + expect(stats.averageWatchTime).to.equal(22) + expect(stats.totalWatchTime).to.equal(88) + expect(stats.totalViewers).to.equal(4) + } + }) + + after(async function () { + await stopFfmpeg(command) + }) + }) + + describe('Test watchers peak stats of local videos on VOD', function () { + let videoUUID: string + let before2Watchers: Date + + before(async function () { + this.timeout(240000); + + ({ vodVideoId: videoUUID } = await prepareViewsVideos({ servers, live: true, vod: true })) + }) + + it('Should not have watchers peak', async function () { + const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID }) + + expect(stats.viewersPeak).to.equal(0) + expect(stats.viewersPeakDate).to.be.null + }) + + it('Should have watcher peak with 1 watcher', async function () { + this.timeout(60000) + + const before = new Date() + await servers[0].views.simulateViewer({ id: videoUUID, currentTimes: [ 0, 2 ] }) + const after = new Date() + + await processViewersStats(servers) + + const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID }) + + expect(stats.viewersPeak).to.equal(1) + expect(new Date(stats.viewersPeakDate)).to.be.above(before).and.below(after) + }) + + it('Should have watcher peak with 2 watchers', async function () { + this.timeout(60000) + + before2Watchers = new Date() + await servers[0].views.view({ id: videoUUID, currentTime: 0 }) + await servers[1].views.view({ id: videoUUID, currentTime: 0 }) + await servers[0].views.view({ id: videoUUID, currentTime: 2 }) + await servers[1].views.view({ id: videoUUID, currentTime: 2 }) + const after = new Date() + + await processViewersStats(servers) + + const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID }) + + expect(stats.viewersPeak).to.equal(2) + expect(new Date(stats.viewersPeakDate)).to.be.above(before2Watchers).and.below(after) + }) + + it('Should filter peak viewers stats by date', async function () { + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID, startDate: new Date().toISOString() }) + expect(stats.viewersPeak).to.equal(0) + expect(stats.viewersPeakDate).to.not.exist + } + + { + const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID, endDate: before2Watchers.toISOString() }) + expect(stats.viewersPeak).to.equal(1) + expect(new Date(stats.viewersPeakDate)).to.be.below(before2Watchers) + } + }) + + it('Should complex filter peak viewers by date', async function () { + this.timeout(60000) + + const { startDate, endDate } = await simulateComplexViewers(servers, videoUUID) + + const expectCorrect = (stats: VideoStatsOverall) => { + expect(stats.viewersPeak).to.equal(3) + expect(new Date(stats.viewersPeakDate)).to.be.above(new Date(startDate)).and.below(new Date(endDate)) + } + + expectCorrect(await servers[0].videoStats.getOverallStats({ videoId: videoUUID, startDate, endDate })) + expectCorrect(await servers[0].videoStats.getOverallStats({ videoId: videoUUID, startDate })) + expectCorrect(await servers[0].videoStats.getOverallStats({ videoId: videoUUID, endDate })) + expectCorrect(await servers[0].videoStats.getOverallStats({ videoId: videoUUID })) + }) + }) + + describe('Test countries', function () { + let videoUUID: string + + it('Should not report countries if geoip is disabled', async function () { + this.timeout(120000) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) + await waitJobs(servers) + + await servers[1].views.view({ id: uuid, xForwardedFor: '8.8.8.8,127.0.0.1', currentTime: 1 }) + + await processViewersStats(servers) + + const stats = await servers[0].videoStats.getOverallStats({ videoId: uuid }) + expect(stats.countries).to.have.lengthOf(0) + }) + + it('Should report countries if geoip is enabled', async function () { + this.timeout(240000) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) + videoUUID = uuid + await waitJobs(servers) + + await Promise.all([ + servers[0].kill(), + servers[1].kill() + ]) + + const config = { geo_ip: { enabled: true } } + await Promise.all([ + servers[0].run(config), + servers[1].run(config) + ]) + + await servers[0].views.view({ id: uuid, xForwardedFor: '8.8.8.8,127.0.0.1', currentTime: 1 }) + await servers[1].views.view({ id: uuid, xForwardedFor: '8.8.8.4,127.0.0.1', currentTime: 3 }) + await servers[1].views.view({ id: uuid, xForwardedFor: '80.67.169.12,127.0.0.1', currentTime: 2 }) + + await processViewersStats(servers) + + const stats = await servers[0].videoStats.getOverallStats({ videoId: uuid }) + expect(stats.countries).to.have.lengthOf(2) + + expect(stats.countries[0].isoCode).to.equal('US') + expect(stats.countries[0].viewers).to.equal(2) + + expect(stats.countries[1].isoCode).to.equal('FR') + expect(stats.countries[1].viewers).to.equal(1) + }) + + it('Should filter countries stats by date', async function () { + const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID, startDate: new Date().toISOString() }) + expect(stats.countries).to.have.lengthOf(0) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/views/video-views-retention-stats.ts b/packages/tests/src/api/views/video-views-retention-stats.ts new file mode 100644 index 000000000..4cd0c7da9 --- /dev/null +++ b/packages/tests/src/api/views/video-views-retention-stats.ts @@ -0,0 +1,53 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { prepareViewsServers, prepareViewsVideos, processViewersStats } from '@tests/shared/views.js' +import { cleanupTests, PeerTubeServer } from '@peertube/peertube-server-commands' + +describe('Test views retention stats', function () { + let servers: PeerTubeServer[] + + before(async function () { + this.timeout(120000) + + servers = await prepareViewsServers() + }) + + describe('Test retention stats on VOD', function () { + let vodVideoId: string + + before(async function () { + this.timeout(240000); + + ({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true })) + }) + + it('Should display empty retention', async function () { + const { data } = await servers[0].videoStats.getRetentionStats({ videoId: vodVideoId }) + expect(data).to.have.lengthOf(6) + + for (let i = 0; i < 6; i++) { + expect(data[i].second).to.equal(i) + expect(data[i].retentionPercent).to.equal(0) + } + }) + + it('Should display appropriate retention metrics', async function () { + await servers[0].views.simulateViewer({ xForwardedFor: '127.0.0.2,127.0.0.1', id: vodVideoId, currentTimes: [ 0, 1 ] }) + await servers[0].views.simulateViewer({ xForwardedFor: '127.0.0.3,127.0.0.1', id: vodVideoId, currentTimes: [ 1, 3 ] }) + await servers[1].views.simulateViewer({ xForwardedFor: '127.0.0.2,127.0.0.1', id: vodVideoId, currentTimes: [ 4 ] }) + await servers[1].views.simulateViewer({ xForwardedFor: '127.0.0.3,127.0.0.1', id: vodVideoId, currentTimes: [ 0, 1 ] }) + + await processViewersStats(servers) + + const { data } = await servers[0].videoStats.getRetentionStats({ videoId: vodVideoId }) + expect(data).to.have.lengthOf(6) + + expect(data.map(d => d.retentionPercent)).to.deep.equal([ 50, 75, 25, 25, 25, 0 ]) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/views/video-views-timeserie-stats.ts b/packages/tests/src/api/views/video-views-timeserie-stats.ts new file mode 100644 index 000000000..44fccb644 --- /dev/null +++ b/packages/tests/src/api/views/video-views-timeserie-stats.ts @@ -0,0 +1,253 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { FfmpegCommand } from 'fluent-ffmpeg' +import { prepareViewsServers, prepareViewsVideos, processViewersStats } from '@tests/shared/views.js' +import { VideoStatsTimeserie, VideoStatsTimeserieMetric } from '@peertube/peertube-models' +import { cleanupTests, PeerTubeServer, stopFfmpeg } from '@peertube/peertube-server-commands' + +function buildOneMonthAgo () { + const monthAgo = new Date() + monthAgo.setHours(0, 0, 0, 0) + + monthAgo.setDate(monthAgo.getDate() - 29) + + return monthAgo +} + +describe('Test views timeserie stats', function () { + const availableMetrics: VideoStatsTimeserieMetric[] = [ 'viewers' ] + + let servers: PeerTubeServer[] + + before(async function () { + this.timeout(120000) + + servers = await prepareViewsServers() + }) + + describe('Common metric tests', function () { + let vodVideoId: string + + before(async function () { + this.timeout(240000); + + ({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true })) + }) + + it('Should display empty metric stats', async function () { + for (const metric of availableMetrics) { + const { data } = await servers[0].videoStats.getTimeserieStats({ videoId: vodVideoId, metric }) + + expect(data).to.have.length.at.least(1) + + for (const d of data) { + expect(d.value).to.equal(0) + } + } + }) + }) + + describe('Test viewer and watch time metrics on live and VOD', function () { + let vodVideoId: string + let liveVideoId: string + let command: FfmpegCommand + + function expectTodayLastValue (result: VideoStatsTimeserie, lastValue?: number) { + const { data } = result + + const last = data[data.length - 1] + const today = new Date().getDate() + expect(new Date(last.date).getDate()).to.equal(today) + + if (lastValue) expect(last.value).to.equal(lastValue) + } + + function expectTimeserieData (result: VideoStatsTimeserie, lastValue: number) { + const { data } = result + expect(data).to.have.length.at.least(25) + + expectTodayLastValue(result, lastValue) + + for (let i = 0; i < data.length - 2; i++) { + expect(data[i].value).to.equal(0) + } + } + + function expectInterval (result: VideoStatsTimeserie, intervalMs: number) { + const first = result.data[0] + const second = result.data[1] + expect(new Date(second.date).getTime() - new Date(first.date).getTime()).to.equal(intervalMs) + } + + before(async function () { + this.timeout(240000); + + ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) + }) + + it('Should display appropriate viewers metrics', async function () { + for (const videoId of [ vodVideoId, liveVideoId ]) { + await servers[0].views.simulateViewer({ id: videoId, currentTimes: [ 0, 3 ] }) + await servers[1].views.simulateViewer({ id: videoId, currentTimes: [ 0, 5 ] }) + } + + await processViewersStats(servers) + + for (const videoId of [ vodVideoId, liveVideoId ]) { + const result = await servers[0].videoStats.getTimeserieStats({ + videoId, + startDate: buildOneMonthAgo(), + endDate: new Date(), + metric: 'viewers' + }) + expectTimeserieData(result, 2) + } + }) + + it('Should display appropriate watch time metrics', async function () { + for (const videoId of [ vodVideoId, liveVideoId ]) { + const result = await servers[0].videoStats.getTimeserieStats({ + videoId, + startDate: buildOneMonthAgo(), + endDate: new Date(), + metric: 'aggregateWatchTime' + }) + expectTimeserieData(result, 8) + + await servers[1].views.simulateViewer({ id: videoId, currentTimes: [ 0, 1 ] }) + } + + await processViewersStats(servers) + + for (const videoId of [ vodVideoId, liveVideoId ]) { + const result = await servers[0].videoStats.getTimeserieStats({ + videoId, + startDate: buildOneMonthAgo(), + endDate: new Date(), + metric: 'aggregateWatchTime' + }) + expectTimeserieData(result, 9) + } + }) + + it('Should use a custom start/end date', async function () { + const now = new Date() + const twentyDaysAgo = new Date() + twentyDaysAgo.setDate(twentyDaysAgo.getDate() - 19) + + const result = await servers[0].videoStats.getTimeserieStats({ + videoId: vodVideoId, + metric: 'aggregateWatchTime', + startDate: twentyDaysAgo, + endDate: now + }) + + expect(result.groupInterval).to.equal('1 day') + expect(result.data).to.have.lengthOf(20) + + const first = result.data[0] + expect(new Date(first.date).toLocaleDateString()).to.equal(twentyDaysAgo.toLocaleDateString()) + + expectInterval(result, 24 * 3600 * 1000) + expectTodayLastValue(result, 9) + }) + + it('Should automatically group by months', async function () { + const now = new Date() + const heightYearsAgo = new Date() + heightYearsAgo.setFullYear(heightYearsAgo.getFullYear() - 7) + + const result = await servers[0].videoStats.getTimeserieStats({ + videoId: vodVideoId, + metric: 'aggregateWatchTime', + startDate: heightYearsAgo, + endDate: now + }) + + expect(result.groupInterval).to.equal('6 months') + expect(result.data).to.have.length.above(10).and.below(200) + }) + + it('Should automatically group by days', async function () { + const now = new Date() + const threeMonthsAgo = new Date() + threeMonthsAgo.setMonth(threeMonthsAgo.getMonth() - 3) + + const result = await servers[0].videoStats.getTimeserieStats({ + videoId: vodVideoId, + metric: 'aggregateWatchTime', + startDate: threeMonthsAgo, + endDate: now + }) + + expect(result.groupInterval).to.equal('2 days') + expect(result.data).to.have.length.above(10).and.below(200) + }) + + it('Should automatically group by hours', async function () { + const now = new Date() + const twoDaysAgo = new Date() + twoDaysAgo.setDate(twoDaysAgo.getDate() - 1) + + const result = await servers[0].videoStats.getTimeserieStats({ + videoId: vodVideoId, + metric: 'aggregateWatchTime', + startDate: twoDaysAgo, + endDate: now + }) + + expect(result.groupInterval).to.equal('1 hour') + expect(result.data).to.have.length.above(24).and.below(50) + + expectInterval(result, 3600 * 1000) + expectTodayLastValue(result, 9) + }) + + it('Should automatically group by ten minutes', async function () { + const now = new Date() + const twoHoursAgo = new Date() + twoHoursAgo.setHours(twoHoursAgo.getHours() - 4) + + const result = await servers[0].videoStats.getTimeserieStats({ + videoId: vodVideoId, + metric: 'aggregateWatchTime', + startDate: twoHoursAgo, + endDate: now + }) + + expect(result.groupInterval).to.equal('10 minutes') + expect(result.data).to.have.length.above(20).and.below(30) + + expectInterval(result, 60 * 10 * 1000) + expectTodayLastValue(result) + }) + + it('Should automatically group by one minute', async function () { + const now = new Date() + const thirtyAgo = new Date() + thirtyAgo.setMinutes(thirtyAgo.getMinutes() - 30) + + const result = await servers[0].videoStats.getTimeserieStats({ + videoId: vodVideoId, + metric: 'aggregateWatchTime', + startDate: thirtyAgo, + endDate: now + }) + + expect(result.groupInterval).to.equal('1 minute') + expect(result.data).to.have.length.above(20).and.below(40) + + expectInterval(result, 60 * 1000) + expectTodayLastValue(result) + }) + + after(async function () { + await stopFfmpeg(command) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/api/views/videos-views-cleaner.ts b/packages/tests/src/api/views/videos-views-cleaner.ts new file mode 100644 index 000000000..521dd9b5e --- /dev/null +++ b/packages/tests/src/api/views/videos-views-cleaner.ts @@ -0,0 +1,98 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { SQLCommand } from '@tests/shared/sql-command.js' +import { wait } from '@peertube/peertube-core-utils' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + killallServers, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test video views cleaner', function () { + let servers: PeerTubeServer[] + let sqlCommands: SQLCommand[] = [] + + let videoIdServer1: string + let videoIdServer2: string + + before(async function () { + this.timeout(240000) + + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + + await doubleFollow(servers[0], servers[1]) + + videoIdServer1 = (await servers[0].videos.quickUpload({ name: 'video server 1' })).uuid + videoIdServer2 = (await servers[1].videos.quickUpload({ name: 'video server 2' })).uuid + + await waitJobs(servers) + + await servers[0].views.simulateView({ id: videoIdServer1 }) + await servers[1].views.simulateView({ id: videoIdServer1 }) + await servers[0].views.simulateView({ id: videoIdServer2 }) + await servers[1].views.simulateView({ id: videoIdServer2 }) + + await waitJobs(servers) + + sqlCommands = servers.map(s => new SQLCommand(s)) + }) + + it('Should not clean old video views', async function () { + this.timeout(50000) + + await killallServers([ servers[0] ]) + + await servers[0].run({ views: { videos: { remote: { max_age: '10 days' } } } }) + + await wait(6000) + + // Should still have views + + for (let i = 0; i < servers.length; i++) { + const total = await sqlCommands[i].countVideoViewsOf(videoIdServer1) + expect(total).to.equal(2, 'Server ' + servers[i].serverNumber + ' does not have the correct amount of views') + } + + for (let i = 0; i < servers.length; i++) { + const total = await sqlCommands[i].countVideoViewsOf(videoIdServer2) + expect(total).to.equal(2, 'Server ' + servers[i].serverNumber + ' does not have the correct amount of views') + } + }) + + it('Should clean old video views', async function () { + this.timeout(50000) + + await killallServers([ servers[0] ]) + + await servers[0].run({ views: { videos: { remote: { max_age: '5 seconds' } } } }) + + await wait(6000) + + // Should still have views + + for (let i = 0; i < servers.length; i++) { + const total = await sqlCommands[i].countVideoViewsOf(videoIdServer1) + expect(total).to.equal(2) + } + + const totalServer1 = await sqlCommands[0].countVideoViewsOf(videoIdServer2) + expect(totalServer1).to.equal(0) + + const totalServer2 = await sqlCommands[1].countVideoViewsOf(videoIdServer2) + expect(totalServer2).to.equal(2) + }) + + after(async function () { + for (const sqlCommand of sqlCommands) { + await sqlCommand.cleanup() + } + + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/cli/create-generate-storyboard-job.ts b/packages/tests/src/cli/create-generate-storyboard-job.ts new file mode 100644 index 000000000..5a1c61ef1 --- /dev/null +++ b/packages/tests/src/cli/create-generate-storyboard-job.ts @@ -0,0 +1,121 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { remove } from 'fs-extra/esm' +import { readdir } from 'fs/promises' +import { join } from 'path' +import { HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + makeGetRequest, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' +import { SQLCommand } from '../shared/sql-command.js' + +function listStoryboardFiles (server: PeerTubeServer) { + const storage = server.getDirectoryPath('storyboards') + + return readdir(storage) +} + +describe('Test create generate storyboard job', function () { + let servers: PeerTubeServer[] = [] + const uuids: string[] = [] + let sql: SQLCommand + let existingStoryboardName: string + + before(async function () { + this.timeout(120000) + + // Run server 2 to have transcoding enabled + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + + await doubleFollow(servers[0], servers[1]) + + for (let i = 0; i < 3; i++) { + const { uuid } = await servers[0].videos.quickUpload({ name: 'video ' + i }) + uuids.push(uuid) + } + + await waitJobs(servers) + + const storage = servers[0].getDirectoryPath('storyboards') + for (const storyboard of await listStoryboardFiles(servers[0])) { + await remove(join(storage, storyboard)) + } + + sql = new SQLCommand(servers[0]) + await sql.deleteAll('storyboard') + + const { uuid } = await servers[0].videos.quickUpload({ name: 'video 4' }) + uuids.push(uuid) + + await waitJobs(servers) + + const storyboards = await listStoryboardFiles(servers[0]) + existingStoryboardName = storyboards[0] + }) + + it('Should create a storyboard of a video', async function () { + this.timeout(120000) + + for (const uuid of [ uuids[0], uuids[3] ]) { + const command = `npm run create-generate-storyboard-job -- -v ${uuid}` + await servers[0].cli.execWithEnv(command) + } + + await waitJobs(servers) + + { + const storyboards = await listStoryboardFiles(servers[0]) + expect(storyboards).to.have.lengthOf(2) + expect(storyboards).to.not.include(existingStoryboardName) + + existingStoryboardName = storyboards[0] + } + + for (const server of servers) { + for (const uuid of [ uuids[0], uuids[3] ]) { + const { storyboards } = await server.storyboard.list({ id: uuid }) + expect(storyboards).to.have.lengthOf(1) + + await makeGetRequest({ url: server.url, path: storyboards[0].storyboardPath, expectedStatus: HttpStatusCode.OK_200 }) + } + } + }) + + it('Should create missing storyboards', async function () { + this.timeout(120000) + + const command = `npm run create-generate-storyboard-job -- -a` + await servers[0].cli.execWithEnv(command) + + await waitJobs(servers) + + { + const storyboards = await listStoryboardFiles(servers[0]) + expect(storyboards).to.have.lengthOf(4) + expect(storyboards).to.include(existingStoryboardName) + } + + for (const server of servers) { + for (const uuid of uuids) { + const { storyboards } = await server.storyboard.list({ id: uuid }) + expect(storyboards).to.have.lengthOf(1) + + await makeGetRequest({ url: server.url, path: storyboards[0].storyboardPath, expectedStatus: HttpStatusCode.OK_200 }) + } + } + }) + + after(async function () { + await sql.cleanup() + + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/cli/create-import-video-file-job.ts b/packages/tests/src/cli/create-import-video-file-job.ts new file mode 100644 index 000000000..fa934510c --- /dev/null +++ b/packages/tests/src/cli/create-import-video-file-job.ts @@ -0,0 +1,168 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { HttpStatusCode, VideoDetails, VideoFile, VideoInclude } from '@peertube/peertube-models' +import { areMockObjectStorageTestsDisabled, buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import { + ObjectStorageCommand, + PeerTubeServer, + cleanupTests, + createMultipleServers, + doubleFollow, + makeRawRequest, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' +import { expect } from 'chai' +import { expectStartWith } from '../shared/checks.js' + +function assertVideoProperties (video: VideoFile, resolution: number, extname: string, size?: number) { + expect(video).to.have.nested.property('resolution.id', resolution) + expect(video).to.have.property('torrentUrl').that.includes(`-${resolution}.torrent`) + expect(video).to.have.property('fileUrl').that.includes(`.${extname}`) + expect(video).to.have.property('magnetUri').that.includes(`.${extname}`) + expect(video).to.have.property('size').that.is.above(0) + + if (size) expect(video.size).to.equal(size) +} + +async function checkFiles (video: VideoDetails, objectStorage: ObjectStorageCommand) { + for (const file of video.files) { + if (objectStorage) expectStartWith(file.fileUrl, objectStorage.getMockWebVideosBaseUrl()) + + await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + } +} + +function runTests (enableObjectStorage: boolean) { + let video1ShortId: string + let video2UUID: string + + let servers: PeerTubeServer[] = [] + + const objectStorage = new ObjectStorageCommand() + + before(async function () { + this.timeout(90000) + + const config = enableObjectStorage + ? objectStorage.getDefaultMockConfig() + : {} + + // Run server 2 to have transcoding enabled + servers = await createMultipleServers(2, config) + await setAccessTokensToServers(servers) + + await doubleFollow(servers[0], servers[1]) + + if (enableObjectStorage) await objectStorage.prepareDefaultMockBuckets() + + // Upload two videos for our needs + { + const { shortUUID } = await servers[0].videos.upload({ attributes: { name: 'video1' } }) + video1ShortId = shortUUID + } + + { + const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video2' } }) + video2UUID = uuid + } + + await waitJobs(servers) + + for (const server of servers) { + await server.config.enableTranscoding() + } + }) + + it('Should run a import job on video 1 with a lower resolution', async function () { + const command = `npm run create-import-video-file-job -- -v ${video1ShortId} -i ${buildAbsoluteFixturePath('video_short_480.webm')}` + await servers[0].cli.execWithEnv(command) + + await waitJobs(servers) + + for (const server of servers) { + const { data: videos } = await server.videos.list() + expect(videos).to.have.lengthOf(2) + + const video = videos.find(({ shortUUID }) => shortUUID === video1ShortId) + const videoDetails = await server.videos.get({ id: video.shortUUID }) + + expect(videoDetails.files).to.have.lengthOf(2) + const [ originalVideo, transcodedVideo ] = videoDetails.files + assertVideoProperties(originalVideo, 720, 'webm', 218910) + assertVideoProperties(transcodedVideo, 480, 'webm', 69217) + + await checkFiles(videoDetails, enableObjectStorage && objectStorage) + } + }) + + it('Should run a import job on video 2 with the same resolution and a different extension', async function () { + const command = `npm run create-import-video-file-job -- -v ${video2UUID} -i ${buildAbsoluteFixturePath('video_short.ogv')}` + await servers[1].cli.execWithEnv(command) + + await waitJobs(servers) + + for (const server of servers) { + const { data: videos } = await server.videos.listWithToken({ include: VideoInclude.NOT_PUBLISHED_STATE }) + expect(videos).to.have.lengthOf(2) + + const video = videos.find(({ uuid }) => uuid === video2UUID) + const videoDetails = await server.videos.get({ id: video.uuid }) + + expect(videoDetails.files).to.have.lengthOf(4) + const [ originalVideo, transcodedVideo420, transcodedVideo320, transcodedVideo240 ] = videoDetails.files + assertVideoProperties(originalVideo, 720, 'ogv', 140849) + assertVideoProperties(transcodedVideo420, 480, 'mp4') + assertVideoProperties(transcodedVideo320, 360, 'mp4') + assertVideoProperties(transcodedVideo240, 240, 'mp4') + + await checkFiles(videoDetails, enableObjectStorage && objectStorage) + } + }) + + it('Should run a import job on video 2 with the same resolution and the same extension', async function () { + const command = `npm run create-import-video-file-job -- -v ${video1ShortId} -i ${buildAbsoluteFixturePath('video_short2.webm')}` + await servers[0].cli.execWithEnv(command) + + await waitJobs(servers) + + for (const server of servers) { + const { data: videos } = await server.videos.listWithToken({ include: VideoInclude.NOT_PUBLISHED_STATE }) + expect(videos).to.have.lengthOf(2) + + const video = videos.find(({ shortUUID }) => shortUUID === video1ShortId) + const videoDetails = await server.videos.get({ id: video.uuid }) + + expect(videoDetails.files).to.have.lengthOf(2) + const [ video720, video480 ] = videoDetails.files + assertVideoProperties(video720, 720, 'webm', 942961) + assertVideoProperties(video480, 480, 'webm', 69217) + + await checkFiles(videoDetails, enableObjectStorage && objectStorage) + } + }) + + it('Should not have run transcoding after an import job', async function () { + const { data } = await servers[0].jobs.list({ jobType: 'video-transcoding' }) + expect(data).to.have.lengthOf(0) + }) + + after(async function () { + await objectStorage.cleanupMock() + + await cleanupTests(servers) + }) +} + +describe('Test create import video jobs', function () { + + describe('On filesystem', function () { + runTests(false) + }) + + describe('On object storage', function () { + if (areMockObjectStorageTestsDisabled()) return + + runTests(true) + }) +}) diff --git a/packages/tests/src/cli/create-move-video-storage-job.ts b/packages/tests/src/cli/create-move-video-storage-job.ts new file mode 100644 index 000000000..1bee7414f --- /dev/null +++ b/packages/tests/src/cli/create-move-video-storage-job.ts @@ -0,0 +1,125 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { join } from 'path' +import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' +import { HttpStatusCode, VideoDetails } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + makeRawRequest, + ObjectStorageCommand, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' +import { expectStartWith } from '../shared/checks.js' +import { checkDirectoryIsEmpty } from '@tests/shared/directories.js' + +async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectStorage?: ObjectStorageCommand) { + for (const file of video.files) { + const start = objectStorage + ? objectStorage.getMockWebVideosBaseUrl() + : origin.url + + expectStartWith(file.fileUrl, start) + + await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + } + + const start = objectStorage + ? objectStorage.getMockPlaylistBaseUrl() + : origin.url + + const hls = video.streamingPlaylists[0] + expectStartWith(hls.playlistUrl, start) + expectStartWith(hls.segmentsSha256Url, start) + + for (const file of hls.files) { + expectStartWith(file.fileUrl, start) + + await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + } +} + +describe('Test create move video storage job', function () { + if (areMockObjectStorageTestsDisabled()) return + + let servers: PeerTubeServer[] = [] + const uuids: string[] = [] + const objectStorage = new ObjectStorageCommand() + + before(async function () { + this.timeout(360000) + + // Run server 2 to have transcoding enabled + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + + await doubleFollow(servers[0], servers[1]) + + await objectStorage.prepareDefaultMockBuckets() + + await servers[0].config.enableTranscoding() + + for (let i = 0; i < 3; i++) { + const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' + i } }) + uuids.push(uuid) + } + + await waitJobs(servers) + + await servers[0].kill() + await servers[0].run(objectStorage.getDefaultMockConfig()) + }) + + it('Should move only one file', async function () { + this.timeout(120000) + + const command = `npm run create-move-video-storage-job -- --to-object-storage -v ${uuids[1]}` + await servers[0].cli.execWithEnv(command, objectStorage.getDefaultMockConfig()) + await waitJobs(servers) + + for (const server of servers) { + const video = await server.videos.get({ id: uuids[1] }) + + await checkFiles(servers[0], video, objectStorage) + + for (const id of [ uuids[0], uuids[2] ]) { + const video = await server.videos.get({ id }) + + await checkFiles(servers[0], video) + } + } + }) + + it('Should move all files', async function () { + this.timeout(120000) + + const command = `npm run create-move-video-storage-job -- --to-object-storage --all-videos` + await servers[0].cli.execWithEnv(command, objectStorage.getDefaultMockConfig()) + await waitJobs(servers) + + for (const server of servers) { + for (const id of [ uuids[0], uuids[2] ]) { + const video = await server.videos.get({ id }) + + await checkFiles(servers[0], video, objectStorage) + } + } + }) + + it('Should not have files on disk anymore', async function () { + await checkDirectoryIsEmpty(servers[0], 'web-videos', [ 'private' ]) + await checkDirectoryIsEmpty(servers[0], join('web-videos', 'private')) + + await checkDirectoryIsEmpty(servers[0], join('streaming-playlists', 'hls'), [ 'private' ]) + await checkDirectoryIsEmpty(servers[0], join('streaming-playlists', 'hls', 'private')) + }) + + after(async function () { + await objectStorage.cleanupMock() + + await cleanupTests(servers) + }) +}) diff --git a/server/tests/cli/index.ts b/packages/tests/src/cli/index.ts similarity index 100% rename from server/tests/cli/index.ts rename to packages/tests/src/cli/index.ts diff --git a/packages/tests/src/cli/peertube.ts b/packages/tests/src/cli/peertube.ts new file mode 100644 index 000000000..2c66b7a18 --- /dev/null +++ b/packages/tests/src/cli/peertube.ts @@ -0,0 +1,257 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import { + cleanupTests, + CLICommand, + createSingleServer, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' +import { testHelloWorldRegisteredSettings } from '../shared/plugins.js' + +describe('Test CLI wrapper', function () { + let server: PeerTubeServer + let userAccessToken: string + + let cliCommand: CLICommand + + const cmd = 'node ./apps/peertube-cli/dist/peertube.js' + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1, { + rates_limit: { + login: { + max: 30 + } + } + }) + await setAccessTokensToServers([ server ]) + + await server.users.create({ username: 'user_1', password: 'super_password' }) + + userAccessToken = await server.login.getAccessToken({ username: 'user_1', password: 'super_password' }) + + { + const attributes = { name: 'user_channel', displayName: 'User channel', support: 'super support text' } + await server.channels.create({ token: userAccessToken, attributes }) + } + + cliCommand = server.cli + }) + + describe('Authentication and instance selection', function () { + + it('Should get an access token', async function () { + const stdout = await cliCommand.execWithEnv(`${cmd} token --url ${server.url} --username user_1 --password super_password`) + const token = stdout.trim() + + const body = await server.users.getMyInfo({ token }) + expect(body.username).to.equal('user_1') + }) + + it('Should display no selected instance', async function () { + this.timeout(60000) + + const stdout = await cliCommand.execWithEnv(`${cmd} --help`) + expect(stdout).to.contain('no instance selected') + }) + + it('Should add a user', async function () { + this.timeout(60000) + + await cliCommand.execWithEnv(`${cmd} auth add -u ${server.url} -U user_1 -p super_password`) + }) + + it('Should not fail to add a user if there is a slash at the end of the instance URL', async function () { + this.timeout(60000) + + let fullServerURL = server.url + '/' + + await cliCommand.execWithEnv(`${cmd} auth add -u ${fullServerURL} -U user_1 -p super_password`) + + fullServerURL = server.url + '/asdfasdf' + await cliCommand.execWithEnv(`${cmd} auth add -u ${fullServerURL} -U user_1 -p super_password`) + }) + + it('Should default to this user', async function () { + this.timeout(60000) + + const stdout = await cliCommand.execWithEnv(`${cmd} --help`) + expect(stdout).to.contain(`instance ${server.url} selected`) + }) + + it('Should remember the user', async function () { + this.timeout(60000) + + const stdout = await cliCommand.execWithEnv(`${cmd} auth list`) + expect(stdout).to.contain(server.url) + }) + }) + + describe('Video upload', function () { + + it('Should upload a video', async function () { + this.timeout(60000) + + const fixture = buildAbsoluteFixturePath('60fps_720p_small.mp4') + const params = `-f ${fixture} --video-name 'test upload' --channel-name user_channel --support 'support_text'` + + await cliCommand.execWithEnv(`${cmd} upload ${params}`) + }) + + it('Should have the video uploaded', async function () { + const { total, data } = await server.videos.list() + expect(total).to.equal(1) + + const video = await server.videos.get({ id: data[0].uuid }) + expect(video.name).to.equal('test upload') + expect(video.support).to.equal('support_text') + expect(video.channel.name).to.equal('user_channel') + }) + }) + + describe('Admin auth', function () { + + it('Should remove the auth user', async function () { + await cliCommand.execWithEnv(`${cmd} auth del ${server.url}`) + + const stdout = await cliCommand.execWithEnv(`${cmd} --help`) + expect(stdout).to.contain('no instance selected') + }) + + it('Should add the admin user', async function () { + await cliCommand.execWithEnv(`${cmd} auth add -u ${server.url} -U root -p test${server.internalServerNumber}`) + }) + }) + + describe('Manage plugins', function () { + + it('Should install a plugin', async function () { + this.timeout(60000) + + await cliCommand.execWithEnv(`${cmd} plugins install --npm-name peertube-plugin-hello-world`) + }) + + it('Should have registered settings', async function () { + await testHelloWorldRegisteredSettings(server) + }) + + it('Should list installed plugins', async function () { + const res = await cliCommand.execWithEnv(`${cmd} plugins list`) + + expect(res).to.contain('peertube-plugin-hello-world') + }) + + it('Should uninstall the plugin', async function () { + const res = await cliCommand.execWithEnv(`${cmd} plugins uninstall --npm-name peertube-plugin-hello-world`) + + expect(res).to.not.contain('peertube-plugin-hello-world') + }) + + it('Should install a plugin in requested version', async function () { + this.timeout(60000) + + await cliCommand.execWithEnv(`${cmd} plugins install --npm-name peertube-plugin-hello-world --plugin-version 0.0.17`) + }) + + it('Should list installed plugins, in correct version', async function () { + const res = await cliCommand.execWithEnv(`${cmd} plugins list`) + + expect(res).to.contain('peertube-plugin-hello-world') + expect(res).to.contain('0.0.17') + }) + + it('Should uninstall the plugin again', async function () { + const res = await cliCommand.execWithEnv(`${cmd} plugins uninstall --npm-name peertube-plugin-hello-world`) + + expect(res).to.not.contain('peertube-plugin-hello-world') + }) + + it('Should install a plugin in requested beta version', async function () { + this.timeout(60000) + + await cliCommand.execWithEnv(`${cmd} plugins install --npm-name peertube-plugin-hello-world --plugin-version 0.0.21-beta.1`) + + const res = await cliCommand.execWithEnv(`${cmd} plugins list`) + + expect(res).to.contain('peertube-plugin-hello-world') + expect(res).to.contain('0.0.21-beta.1') + + await cliCommand.execWithEnv(`${cmd} plugins uninstall --npm-name peertube-plugin-hello-world`) + }) + }) + + describe('Manage video redundancies', function () { + let anotherServer: PeerTubeServer + let video1Server2: number + let servers: PeerTubeServer[] + + before(async function () { + this.timeout(120000) + + anotherServer = await createSingleServer(2) + await setAccessTokensToServers([ anotherServer ]) + + await doubleFollow(server, anotherServer) + + servers = [ server, anotherServer ] + await waitJobs(servers) + + const { uuid } = await anotherServer.videos.quickUpload({ name: 'super video' }) + await waitJobs(servers) + + video1Server2 = await server.videos.getId({ uuid }) + }) + + it('Should add a redundancy', async function () { + this.timeout(60000) + + const params = `add --video ${video1Server2}` + await cliCommand.execWithEnv(`${cmd} redundancy ${params}`) + + await waitJobs(servers) + }) + + it('Should list redundancies', async function () { + this.timeout(60000) + + { + const params = 'list-my-redundancies' + const stdout = await cliCommand.execWithEnv(`${cmd} redundancy ${params}`) + + expect(stdout).to.contain('super video') + expect(stdout).to.contain(server.host) + } + }) + + it('Should remove a redundancy', async function () { + this.timeout(60000) + + const params = `remove --video ${video1Server2}` + await cliCommand.execWithEnv(`${cmd} redundancy ${params}`) + + await waitJobs(servers) + + { + const params = 'list-my-redundancies' + const stdout = await cliCommand.execWithEnv(`${cmd} redundancy ${params}`) + + expect(stdout).to.not.contain('super video') + } + }) + + after(async function () { + await cleanupTests([ anotherServer ]) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/cli/plugins.ts b/packages/tests/src/cli/plugins.ts new file mode 100644 index 000000000..ab7f7dd85 --- /dev/null +++ b/packages/tests/src/cli/plugins.ts @@ -0,0 +1,76 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { + cleanupTests, + createSingleServer, + killallServers, + PeerTubeServer, + PluginsCommand, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test plugin scripts', function () { + let server: PeerTubeServer + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + }) + + it('Should install a plugin from stateless CLI', async function () { + this.timeout(60000) + + const packagePath = PluginsCommand.getPluginTestPath() + + await server.cli.execWithEnv(`npm run plugin:install -- --plugin-path ${packagePath}`) + }) + + it('Should install a theme from stateless CLI', async function () { + this.timeout(60000) + + await server.cli.execWithEnv(`npm run plugin:install -- --npm-name peertube-theme-background-red`) + }) + + it('Should have the theme and the plugin registered when we restart peertube', async function () { + this.timeout(30000) + + await killallServers([ server ]) + await server.run() + + const config = await server.config.getConfig() + + const plugin = config.plugin.registered + .find(p => p.name === 'test') + expect(plugin).to.not.be.undefined + + const theme = config.theme.registered + .find(t => t.name === 'background-red') + expect(theme).to.not.be.undefined + }) + + it('Should uninstall a plugin from stateless CLI', async function () { + this.timeout(60000) + + await server.cli.execWithEnv(`npm run plugin:uninstall -- --npm-name peertube-plugin-test`) + }) + + it('Should have removed the plugin on another peertube restart', async function () { + this.timeout(30000) + + await killallServers([ server ]) + await server.run() + + const config = await server.config.getConfig() + + const plugin = config.plugin.registered + .find(p => p.name === 'test') + expect(plugin).to.be.undefined + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/cli/prune-storage.ts b/packages/tests/src/cli/prune-storage.ts new file mode 100644 index 000000000..c07a2a975 --- /dev/null +++ b/packages/tests/src/cli/prune-storage.ts @@ -0,0 +1,224 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { createFile } from 'fs-extra/esm' +import { readdir } from 'fs/promises' +import { join } from 'path' +import { wait } from '@peertube/peertube-core-utils' +import { buildUUID } from '@peertube/peertube-node-utils' +import { HttpStatusCode, VideoPlaylistPrivacy, VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + CLICommand, + createMultipleServers, + doubleFollow, + killallServers, + makeGetRequest, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' + +async function assertNotExists (server: PeerTubeServer, directory: string, substring: string) { + const files = await readdir(server.servers.buildDirectory(directory)) + + for (const f of files) { + expect(f).to.not.contain(substring) + } +} + +async function assertCountAreOkay (servers: PeerTubeServer[]) { + for (const server of servers) { + const videosCount = await server.servers.countFiles('web-videos') + expect(videosCount).to.equal(9) // 2 videos with 4 resolutions + private directory + + const privateVideosCount = await server.servers.countFiles('web-videos/private') + expect(privateVideosCount).to.equal(4) + + const torrentsCount = await server.servers.countFiles('torrents') + expect(torrentsCount).to.equal(24) + + const previewsCount = await server.servers.countFiles('previews') + expect(previewsCount).to.equal(3) + + const thumbnailsCount = await server.servers.countFiles('thumbnails') + expect(thumbnailsCount).to.equal(5) // 3 local videos, 1 local playlist, 2 remotes videos (lazy downloaded) and 1 remote playlist + + const avatarsCount = await server.servers.countFiles('avatars') + expect(avatarsCount).to.equal(4) + + const hlsRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls')) + expect(hlsRootCount).to.equal(3) // 2 videos + private directory + + const hlsPrivateRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls', 'private')) + expect(hlsPrivateRootCount).to.equal(1) + } +} + +describe('Test prune storage scripts', function () { + let servers: PeerTubeServer[] + const badNames: { [directory: string]: string[] } = {} + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(2, { transcoding: { enabled: true } }) + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + for (const server of servers) { + await server.videos.upload({ attributes: { name: 'video 1', privacy: VideoPrivacy.PUBLIC } }) + await server.videos.upload({ attributes: { name: 'video 2', privacy: VideoPrivacy.PUBLIC } }) + + await server.videos.upload({ attributes: { name: 'video 3', privacy: VideoPrivacy.PRIVATE } }) + + await server.users.updateMyAvatar({ fixture: 'avatar.png' }) + + await server.playlists.create({ + attributes: { + displayName: 'playlist', + privacy: VideoPlaylistPrivacy.PUBLIC, + videoChannelId: server.store.channel.id, + thumbnailfile: 'custom-thumbnail.jpg' + } + }) + } + + await doubleFollow(servers[0], servers[1]) + + // Lazy load the remote avatars + { + const account = await servers[0].accounts.get({ accountName: 'root@' + servers[1].host }) + + for (const avatar of account.avatars) { + await makeGetRequest({ + url: servers[0].url, + path: avatar.path, + expectedStatus: HttpStatusCode.OK_200 + }) + } + } + + { + const account = await servers[1].accounts.get({ accountName: 'root@' + servers[0].host }) + for (const avatar of account.avatars) { + await makeGetRequest({ + url: servers[1].url, + path: avatar.path, + expectedStatus: HttpStatusCode.OK_200 + }) + } + } + + await wait(1000) + + await waitJobs(servers) + await killallServers(servers) + + await wait(1000) + }) + + it('Should have the files on the disk', async function () { + await assertCountAreOkay(servers) + }) + + it('Should create some dirty files', async function () { + for (let i = 0; i < 2; i++) { + { + const basePublic = servers[0].servers.buildDirectory('web-videos') + const basePrivate = servers[0].servers.buildDirectory(join('web-videos', 'private')) + + const n1 = buildUUID() + '.mp4' + const n2 = buildUUID() + '.webm' + + await createFile(join(basePublic, n1)) + await createFile(join(basePublic, n2)) + await createFile(join(basePrivate, n1)) + await createFile(join(basePrivate, n2)) + + badNames['web-videos'] = [ n1, n2 ] + } + + { + const base = servers[0].servers.buildDirectory('torrents') + + const n1 = buildUUID() + '-240.torrent' + const n2 = buildUUID() + '-480.torrent' + + await createFile(join(base, n1)) + await createFile(join(base, n2)) + + badNames['torrents'] = [ n1, n2 ] + } + + { + const base = servers[0].servers.buildDirectory('thumbnails') + + const n1 = buildUUID() + '.jpg' + const n2 = buildUUID() + '.jpg' + + await createFile(join(base, n1)) + await createFile(join(base, n2)) + + badNames['thumbnails'] = [ n1, n2 ] + } + + { + const base = servers[0].servers.buildDirectory('previews') + + const n1 = buildUUID() + '.jpg' + const n2 = buildUUID() + '.jpg' + + await createFile(join(base, n1)) + await createFile(join(base, n2)) + + badNames['previews'] = [ n1, n2 ] + } + + { + const base = servers[0].servers.buildDirectory('avatars') + + const n1 = buildUUID() + '.png' + const n2 = buildUUID() + '.jpg' + + await createFile(join(base, n1)) + await createFile(join(base, n2)) + + badNames['avatars'] = [ n1, n2 ] + } + + { + const directory = join('streaming-playlists', 'hls') + const basePublic = servers[0].servers.buildDirectory(directory) + const basePrivate = servers[0].servers.buildDirectory(join(directory, 'private')) + + const n1 = buildUUID() + await createFile(join(basePublic, n1)) + await createFile(join(basePrivate, n1)) + badNames[directory] = [ n1 ] + } + } + }) + + it('Should run prune storage', async function () { + this.timeout(30000) + + const env = servers[0].cli.getEnv() + await CLICommand.exec(`echo y | ${env} npm run prune-storage`) + }) + + it('Should have removed files', async function () { + await assertCountAreOkay(servers) + + for (const directory of Object.keys(badNames)) { + for (const name of badNames[directory]) { + await assertNotExists(servers[0], directory, name) + } + } + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/cli/regenerate-thumbnails.ts b/packages/tests/src/cli/regenerate-thumbnails.ts new file mode 100644 index 000000000..1448e5cfc --- /dev/null +++ b/packages/tests/src/cli/regenerate-thumbnails.ts @@ -0,0 +1,122 @@ +import { expect } from 'chai' +import { writeFile } from 'fs/promises' +import { basename, join } from 'path' +import { HttpStatusCode, Video } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + makeGetRequest, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +async function testThumbnail (server: PeerTubeServer, videoId: number | string) { + const video = await server.videos.get({ id: videoId }) + + const requests = [ + makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }), + makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) + ] + + for (const req of requests) { + const res = await req + expect(res.body).to.not.have.lengthOf(0) + } +} + +describe('Test regenerate thumbnails script', function () { + let servers: PeerTubeServer[] + + let video1: Video + let video2: Video + let remoteVideo: Video + + let thumbnail1Path: string + let thumbnailRemotePath: string + + before(async function () { + this.timeout(60000) + + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + + await doubleFollow(servers[0], servers[1]) + + { + const videoUUID1 = (await servers[0].videos.quickUpload({ name: 'video 1' })).uuid + video1 = await servers[0].videos.get({ id: videoUUID1 }) + + thumbnail1Path = join(servers[0].servers.buildDirectory('thumbnails'), basename(video1.thumbnailPath)) + + const videoUUID2 = (await servers[0].videos.quickUpload({ name: 'video 2' })).uuid + video2 = await servers[0].videos.get({ id: videoUUID2 }) + } + + { + const videoUUID = (await servers[1].videos.quickUpload({ name: 'video 3' })).uuid + await waitJobs(servers) + + remoteVideo = await servers[0].videos.get({ id: videoUUID }) + + // Load remote thumbnail on disk + await makeGetRequest({ url: servers[0].url, path: remoteVideo.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) + + thumbnailRemotePath = join(servers[0].servers.buildDirectory('thumbnails'), basename(remoteVideo.thumbnailPath)) + } + + await writeFile(thumbnail1Path, '') + await writeFile(thumbnailRemotePath, '') + }) + + it('Should have empty thumbnails', async function () { + { + const res = await makeGetRequest({ url: servers[0].url, path: video1.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) + expect(res.body).to.have.lengthOf(0) + } + + { + const res = await makeGetRequest({ url: servers[0].url, path: video2.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) + expect(res.body).to.not.have.lengthOf(0) + } + + { + const res = await makeGetRequest({ url: servers[0].url, path: remoteVideo.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) + expect(res.body).to.have.lengthOf(0) + } + }) + + it('Should regenerate local thumbnails from the CLI', async function () { + this.timeout(15000) + + await servers[0].cli.execWithEnv(`npm run regenerate-thumbnails`) + }) + + it('Should have generated new thumbnail files', async function () { + await testThumbnail(servers[0], video1.uuid) + await testThumbnail(servers[0], video2.uuid) + + const res = await makeGetRequest({ url: servers[0].url, path: remoteVideo.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) + expect(res.body).to.have.lengthOf(0) + }) + + it('Should have deleted old thumbnail files', async function () { + { + await makeGetRequest({ url: servers[0].url, path: video1.thumbnailPath, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + } + + { + await makeGetRequest({ url: servers[0].url, path: video2.thumbnailPath, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + } + + { + const res = await makeGetRequest({ url: servers[0].url, path: remoteVideo.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) + expect(res.body).to.have.lengthOf(0) + } + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/cli/reset-password.ts b/packages/tests/src/cli/reset-password.ts new file mode 100644 index 000000000..62e1a37a0 --- /dev/null +++ b/packages/tests/src/cli/reset-password.ts @@ -0,0 +1,26 @@ +import { cleanupTests, CLICommand, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@peertube/peertube-server-commands' + +describe('Test reset password scripts', function () { + let server: PeerTubeServer + + before(async function () { + this.timeout(30000) + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + + await server.users.create({ username: 'user_1', password: 'super password' }) + }) + + it('Should change the user password from CLI', async function () { + this.timeout(60000) + + const env = server.cli.getEnv() + await CLICommand.exec(`echo coucou | ${env} npm run reset-password -- -u user_1`) + + await server.login.login({ user: { username: 'user_1', password: 'coucou' } }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/cli/update-host.ts b/packages/tests/src/cli/update-host.ts new file mode 100644 index 000000000..e5f165e5e --- /dev/null +++ b/packages/tests/src/cli/update-host.ts @@ -0,0 +1,134 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { getAllFiles } from '@peertube/peertube-core-utils' +import { + cleanupTests, + createSingleServer, + killallServers, + makeActivityPubGetRequest, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' +import { parseTorrentVideo } from '@tests/shared/webtorrent.js' + +describe('Test update host scripts', function () { + let server: PeerTubeServer + + before(async function () { + this.timeout(60000) + + const overrideConfig = { + webserver: { + port: 9256 + } + } + // Run server 2 to have transcoding enabled + server = await createSingleServer(2, overrideConfig) + await setAccessTokensToServers([ server ]) + + // Upload two videos for our needs + const { uuid: video1UUID } = await server.videos.upload() + await server.videos.upload() + + // Create a user + await server.users.create({ username: 'toto', password: 'coucou' }) + + // Create channel + const videoChannel = { + name: 'second_channel', + displayName: 'second video channel', + description: 'super video channel description' + } + await server.channels.create({ attributes: videoChannel }) + + // Create comments + const text = 'my super first comment' + await server.comments.createThread({ videoId: video1UUID, text }) + + await waitJobs(server) + }) + + it('Should run update host', async function () { + this.timeout(30000) + + await killallServers([ server ]) + // Run server with standard configuration + await server.run() + + await server.cli.execWithEnv(`npm run update-host`) + }) + + it('Should have updated videos url', async function () { + const { total, data } = await server.videos.list() + expect(total).to.equal(2) + + for (const video of data) { + const { body } = await makeActivityPubGetRequest(server.url, '/videos/watch/' + video.uuid) + + expect(body.id).to.equal('http://127.0.0.1:9002/videos/watch/' + video.uuid) + + const videoDetails = await server.videos.get({ id: video.uuid }) + + expect(videoDetails.trackerUrls[0]).to.include(server.host) + expect(videoDetails.streamingPlaylists[0].playlistUrl).to.include(server.host) + expect(videoDetails.streamingPlaylists[0].segmentsSha256Url).to.include(server.host) + } + }) + + it('Should have updated video channels url', async function () { + const { data, total } = await server.channels.list({ sort: '-name' }) + expect(total).to.equal(3) + + for (const channel of data) { + const { body } = await makeActivityPubGetRequest(server.url, '/video-channels/' + channel.name) + + expect(body.id).to.equal('http://127.0.0.1:9002/video-channels/' + channel.name) + } + }) + + it('Should have updated accounts url', async function () { + const body = await server.accounts.list() + expect(body.total).to.equal(3) + + for (const account of body.data) { + const usernameWithDomain = account.name + const { body } = await makeActivityPubGetRequest(server.url, '/accounts/' + usernameWithDomain) + + expect(body.id).to.equal('http://127.0.0.1:9002/accounts/' + usernameWithDomain) + } + }) + + it('Should have updated torrent hosts', async function () { + this.timeout(30000) + + const { data } = await server.videos.list() + expect(data).to.have.lengthOf(2) + + for (const video of data) { + const videoDetails = await server.videos.get({ id: video.id }) + const files = getAllFiles(videoDetails) + + expect(files).to.have.lengthOf(8) + + for (const file of files) { + expect(file.magnetUri).to.contain('127.0.0.1%3A9002%2Ftracker%2Fsocket') + expect(file.magnetUri).to.contain('127.0.0.1%3A9002%2Fstatic%2F') + + const torrent = await parseTorrentVideo(server, file) + const announceWS = torrent.announce.find(a => a === 'ws://127.0.0.1:9002/tracker/socket') + expect(announceWS).to.not.be.undefined + + const announceHttp = torrent.announce.find(a => a === 'http://127.0.0.1:9002/tracker/announce') + expect(announceHttp).to.not.be.undefined + + expect(torrent.urlList[0]).to.contain('http://127.0.0.1:9002/static/') + } + } + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/client.ts b/packages/tests/src/client.ts new file mode 100644 index 000000000..a16205494 --- /dev/null +++ b/packages/tests/src/client.ts @@ -0,0 +1,556 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { omit } from '@peertube/peertube-core-utils' +import { + Account, + HTMLServerConfig, + HttpStatusCode, + ServerConfig, + VideoPlaylistCreateResult, + VideoPlaylistPrivacy, + VideoPrivacy +} from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + makeGetRequest, + makeHTMLRequest, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' + +function checkIndexTags (html: string, title: string, description: string, css: string, config: ServerConfig) { + expect(html).to.contain('' + title + '') + expect(html).to.contain('') + expect(html).to.contain('') + + const htmlConfig: HTMLServerConfig = omit(config, [ 'signup' ]) + const configObjectString = JSON.stringify(htmlConfig) + const configEscapedString = JSON.stringify(configObjectString) + + expect(html).to.contain(``) +} + +describe('Test a client controllers', function () { + let servers: PeerTubeServer[] = [] + let account: Account + + const videoName = 'my super name for server 1' + const videoDescription = 'my
super __description__ for *server* 1

' + const videoDescriptionPlainText = 'my super description for server 1' + + const playlistName = 'super playlist name' + const playlistDescription = 'super playlist description' + let playlist: VideoPlaylistCreateResult + + const channelDescription = 'my super channel description' + + const watchVideoBasePaths = [ '/videos/watch/', '/w/' ] + const watchPlaylistBasePaths = [ '/videos/watch/playlist/', '/w/p/' ] + + let videoIds: (string | number)[] = [] + let privateVideoId: string + let internalVideoId: string + let unlistedVideoId: string + let passwordProtectedVideoId: string + + let playlistIds: (string | number)[] = [] + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + + await doubleFollow(servers[0], servers[1]) + + await setDefaultVideoChannel(servers) + + await servers[0].channels.update({ + channelName: servers[0].store.channel.name, + attributes: { description: channelDescription } + }) + + // Public video + + { + const attributes = { name: videoName, description: videoDescription } + await servers[0].videos.upload({ attributes }) + + const { data } = await servers[0].videos.list() + expect(data.length).to.equal(1) + + const video = data[0] + servers[0].store.video = video + videoIds = [ video.id, video.uuid, video.shortUUID ] + } + + { + ({ uuid: privateVideoId } = await servers[0].videos.quickUpload({ name: 'private', privacy: VideoPrivacy.PRIVATE })); + ({ uuid: unlistedVideoId } = await servers[0].videos.quickUpload({ name: 'unlisted', privacy: VideoPrivacy.UNLISTED })); + ({ uuid: internalVideoId } = await servers[0].videos.quickUpload({ name: 'internal', privacy: VideoPrivacy.INTERNAL })); + ({ uuid: passwordProtectedVideoId } = await servers[0].videos.quickUpload({ + name: 'password protected', + privacy: VideoPrivacy.PASSWORD_PROTECTED, + videoPasswords: [ 'password' ] + })) + } + + // Playlist + + { + const attributes = { + displayName: playlistName, + description: playlistDescription, + privacy: VideoPlaylistPrivacy.PUBLIC, + videoChannelId: servers[0].store.channel.id + } + + playlist = await servers[0].playlists.create({ attributes }) + playlistIds = [ playlist.id, playlist.shortUUID, playlist.uuid ] + + await servers[0].playlists.addElement({ playlistId: playlist.shortUUID, attributes: { videoId: servers[0].store.video.id } }) + } + + // Account + + { + await servers[0].users.updateMe({ description: 'my account description' }) + + account = await servers[0].accounts.get({ accountName: `${servers[0].store.user.username}@${servers[0].host}` }) + } + + await waitJobs(servers) + }) + + describe('oEmbed', function () { + + it('Should have valid oEmbed discovery tags for videos', async function () { + for (const basePath of watchVideoBasePaths) { + for (const id of videoIds) { + const res = await makeGetRequest({ + url: servers[0].url, + path: basePath + id, + accept: 'text/html', + expectedStatus: HttpStatusCode.OK_200 + }) + + const expectedLink = `` + + expect(res.text).to.contain(expectedLink) + } + } + }) + + it('Should have valid oEmbed discovery tags for a playlist', async function () { + for (const basePath of watchPlaylistBasePaths) { + for (const id of playlistIds) { + const res = await makeGetRequest({ + url: servers[0].url, + path: basePath + id, + accept: 'text/html', + expectedStatus: HttpStatusCode.OK_200 + }) + + const expectedLink = `` + + expect(res.text).to.contain(expectedLink) + } + } + }) + }) + + describe('Open Graph', function () { + + async function accountPageTest (path: string) { + const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) + const text = res.text + + expect(text).to.contain(``) + expect(text).to.contain(``) + expect(text).to.contain('') + expect(text).to.contain(``) + } + + async function channelPageTest (path: string) { + const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) + const text = res.text + + expect(text).to.contain(``) + expect(text).to.contain(``) + expect(text).to.contain('') + expect(text).to.contain(``) + } + + async function watchVideoPageTest (path: string) { + const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) + const text = res.text + + expect(text).to.contain(``) + expect(text).to.contain(``) + expect(text).to.contain('') + expect(text).to.contain(``) + } + + async function watchPlaylistPageTest (path: string) { + const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) + const text = res.text + + expect(text).to.contain(``) + expect(text).to.contain(``) + expect(text).to.contain('') + expect(text).to.contain(``) + } + + it('Should have valid Open Graph tags on the account page', async function () { + await accountPageTest('/accounts/' + servers[0].store.user.username) + await accountPageTest('/a/' + servers[0].store.user.username) + await accountPageTest('/@' + servers[0].store.user.username) + }) + + it('Should have valid Open Graph tags on the channel page', async function () { + await channelPageTest('/video-channels/' + servers[0].store.channel.name) + await channelPageTest('/c/' + servers[0].store.channel.name) + await channelPageTest('/@' + servers[0].store.channel.name) + }) + + it('Should have valid Open Graph tags on the watch page', async function () { + for (const path of watchVideoBasePaths) { + for (const id of videoIds) { + await watchVideoPageTest(path + id) + } + } + }) + + it('Should have valid Open Graph tags on the watch page with thread id Angular param', async function () { + for (const path of watchVideoBasePaths) { + for (const id of videoIds) { + await watchVideoPageTest(path + id + ';threadId=1') + } + } + }) + + it('Should have valid Open Graph tags on the watch playlist page', async function () { + for (const path of watchPlaylistBasePaths) { + for (const id of playlistIds) { + await watchPlaylistPageTest(path + id) + } + } + }) + }) + + describe('Twitter card', async function () { + + describe('Not whitelisted', function () { + + async function accountPageTest (path: string) { + const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) + const text = res.text + + expect(text).to.contain('') + expect(text).to.contain('') + expect(text).to.contain(``) + expect(text).to.contain(``) + } + + async function channelPageTest (path: string) { + const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) + const text = res.text + + expect(text).to.contain('') + expect(text).to.contain('') + expect(text).to.contain(``) + expect(text).to.contain(``) + } + + async function watchVideoPageTest (path: string) { + const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) + const text = res.text + + expect(text).to.contain('') + expect(text).to.contain('') + expect(text).to.contain(``) + expect(text).to.contain(``) + } + + async function watchPlaylistPageTest (path: string) { + const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) + const text = res.text + + expect(text).to.contain('') + expect(text).to.contain('') + expect(text).to.contain(``) + expect(text).to.contain(``) + } + + it('Should have valid twitter card on the watch video page', async function () { + for (const path of watchVideoBasePaths) { + for (const id of videoIds) { + await watchVideoPageTest(path + id) + } + } + }) + + it('Should have valid twitter card on the watch playlist page', async function () { + for (const path of watchPlaylistBasePaths) { + for (const id of playlistIds) { + await watchPlaylistPageTest(path + id) + } + } + }) + + it('Should have valid twitter card on the account page', async function () { + await accountPageTest('/accounts/' + account.name) + await accountPageTest('/a/' + account.name) + await accountPageTest('/@' + account.name) + }) + + it('Should have valid twitter card on the channel page', async function () { + await channelPageTest('/video-channels/' + servers[0].store.channel.name) + await channelPageTest('/c/' + servers[0].store.channel.name) + await channelPageTest('/@' + servers[0].store.channel.name) + }) + }) + + describe('Whitelisted', function () { + + before(async function () { + const config = await servers[0].config.getCustomConfig() + config.services.twitter = { + username: '@Kuja', + whitelisted: true + } + + await servers[0].config.updateCustomConfig({ newCustomConfig: config }) + }) + + async function accountPageTest (path: string) { + const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) + const text = res.text + + expect(text).to.contain('') + expect(text).to.contain('') + } + + async function channelPageTest (path: string) { + const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) + const text = res.text + + expect(text).to.contain('') + expect(text).to.contain('') + } + + async function watchVideoPageTest (path: string) { + const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) + const text = res.text + + expect(text).to.contain('') + expect(text).to.contain('') + } + + async function watchPlaylistPageTest (path: string) { + const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) + const text = res.text + + expect(text).to.contain('') + expect(text).to.contain('') + } + + it('Should have valid twitter card on the watch video page', async function () { + for (const path of watchVideoBasePaths) { + for (const id of videoIds) { + await watchVideoPageTest(path + id) + } + } + }) + + it('Should have valid twitter card on the watch playlist page', async function () { + for (const path of watchPlaylistBasePaths) { + for (const id of playlistIds) { + await watchPlaylistPageTest(path + id) + } + } + }) + + it('Should have valid twitter card on the account page', async function () { + await accountPageTest('/accounts/' + account.name) + await accountPageTest('/a/' + account.name) + await accountPageTest('/@' + account.name) + }) + + it('Should have valid twitter card on the channel page', async function () { + await channelPageTest('/video-channels/' + servers[0].store.channel.name) + await channelPageTest('/c/' + servers[0].store.channel.name) + await channelPageTest('/@' + servers[0].store.channel.name) + }) + }) + }) + + describe('Index HTML', function () { + + it('Should have valid index html tags (title, description...)', async function () { + const config = await servers[0].config.getConfig() + const res = await makeHTMLRequest(servers[0].url, '/videos/trending') + + const description = 'PeerTube, an ActivityPub-federated video streaming platform using P2P directly in your web browser.' + checkIndexTags(res.text, 'PeerTube', description, '', config) + }) + + it('Should update the customized configuration and have the correct index html tags', async function () { + await servers[0].config.updateCustomSubConfig({ + newConfig: { + instance: { + name: 'PeerTube updated', + shortDescription: 'my short description', + description: 'my super description', + terms: 'my super terms', + defaultNSFWPolicy: 'blur', + defaultClientRoute: '/videos/recently-added', + customizations: { + javascript: 'alert("coucou")', + css: 'body { background-color: red; }' + } + } + } + }) + + const config = await servers[0].config.getConfig() + const res = await makeHTMLRequest(servers[0].url, '/videos/trending') + + checkIndexTags(res.text, 'PeerTube updated', 'my short description', 'body { background-color: red; }', config) + }) + + it('Should have valid index html updated tags (title, description...)', async function () { + const config = await servers[0].config.getConfig() + const res = await makeHTMLRequest(servers[0].url, '/videos/trending') + + checkIndexTags(res.text, 'PeerTube updated', 'my short description', 'body { background-color: red; }', config) + }) + + it('Should use the original video URL for the canonical tag', async function () { + for (const basePath of watchVideoBasePaths) { + for (const id of videoIds) { + const res = await makeHTMLRequest(servers[1].url, basePath + id) + expect(res.text).to.contain(``) + } + } + }) + + it('Should use the original account URL for the canonical tag', async function () { + const accountURLtest = res => { + expect(res.text).to.contain(``) + } + + accountURLtest(await makeHTMLRequest(servers[1].url, '/accounts/root@' + servers[0].host)) + accountURLtest(await makeHTMLRequest(servers[1].url, '/a/root@' + servers[0].host)) + accountURLtest(await makeHTMLRequest(servers[1].url, '/@root@' + servers[0].host)) + }) + + it('Should use the original channel URL for the canonical tag', async function () { + const channelURLtests = res => { + expect(res.text).to.contain(``) + } + + channelURLtests(await makeHTMLRequest(servers[1].url, '/video-channels/root_channel@' + servers[0].host)) + channelURLtests(await makeHTMLRequest(servers[1].url, '/c/root_channel@' + servers[0].host)) + channelURLtests(await makeHTMLRequest(servers[1].url, '/@root_channel@' + servers[0].host)) + }) + + it('Should use the original playlist URL for the canonical tag', async function () { + for (const basePath of watchPlaylistBasePaths) { + for (const id of playlistIds) { + const res = await makeHTMLRequest(servers[1].url, basePath + id) + expect(res.text).to.contain(``) + } + } + }) + + it('Should add noindex meta tag for remote accounts', async function () { + const handle = 'root@' + servers[0].host + const paths = [ '/accounts/', '/a/', '/@' ] + + for (const path of paths) { + { + const { text } = await makeHTMLRequest(servers[1].url, path + handle) + expect(text).to.contain('') + } + + { + const { text } = await makeHTMLRequest(servers[0].url, path + handle) + expect(text).to.not.contain('') + } + } + }) + + it('Should add noindex meta tag for remote channels', async function () { + const handle = 'root_channel@' + servers[0].host + const paths = [ '/video-channels/', '/c/', '/@' ] + + for (const path of paths) { + { + const { text } = await makeHTMLRequest(servers[1].url, path + handle) + expect(text).to.contain('') + } + + { + const { text } = await makeHTMLRequest(servers[0].url, path + handle) + expect(text).to.not.contain('') + } + } + }) + + it('Should not display internal/private/password protected video', async function () { + for (const basePath of watchVideoBasePaths) { + for (const id of [ privateVideoId, internalVideoId, passwordProtectedVideoId ]) { + const res = await makeGetRequest({ + url: servers[0].url, + path: basePath + id, + accept: 'text/html', + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + + expect(res.text).to.not.contain('internal') + expect(res.text).to.not.contain('private') + expect(res.text).to.not.contain('password protected') + } + } + }) + + it('Should add noindex meta tag for unlisted video', async function () { + for (const basePath of watchVideoBasePaths) { + const res = await makeGetRequest({ + url: servers[0].url, + path: basePath + unlistedVideoId, + accept: 'text/html', + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(res.text).to.contain('unlisted') + expect(res.text).to.contain('') + } + }) + }) + + describe('Embed HTML', function () { + + it('Should have the correct embed html tags', async function () { + const config = await servers[0].config.getConfig() + const res = await makeHTMLRequest(servers[0].url, servers[0].store.video.embedPath) + + checkIndexTags(res.text, 'PeerTube updated', 'my short description', 'body { background-color: red; }', config) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/external-plugins/akismet.ts b/packages/tests/src/external-plugins/akismet.ts new file mode 100644 index 000000000..c6d3b7752 --- /dev/null +++ b/packages/tests/src/external-plugins/akismet.ts @@ -0,0 +1,160 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Official plugin Akismet', function () { + let servers: PeerTubeServer[] + let videoUUID: string + + before(async function () { + this.timeout(30000) + + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + + await servers[0].plugins.install({ + npmName: 'peertube-plugin-akismet' + }) + + if (!process.env.AKISMET_KEY) throw new Error('Missing AKISMET_KEY from env') + + await servers[0].plugins.updateSettings({ + npmName: 'peertube-plugin-akismet', + settings: { + 'akismet-api-key': process.env.AKISMET_KEY + } + }) + + await doubleFollow(servers[0], servers[1]) + }) + + describe('Local threads/replies', function () { + + before(async function () { + const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' }) + videoUUID = uuid + }) + + it('Should not detect a thread as spam', async function () { + await servers[0].comments.createThread({ videoId: videoUUID, text: 'comment' }) + }) + + it('Should not detect a reply as spam', async function () { + await servers[0].comments.addReplyToLastThread({ text: 'reply' }) + }) + + it('Should detect a thread as spam', async function () { + await servers[0].comments.createThread({ + videoId: videoUUID, + text: 'akismet-guaranteed-spam', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should detect a thread as spam', async function () { + await servers[0].comments.createThread({ videoId: videoUUID, text: 'comment' }) + await servers[0].comments.addReplyToLastThread({ text: 'akismet-guaranteed-spam', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + }) + + describe('Remote threads/replies', function () { + + before(async function () { + this.timeout(60000) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' }) + videoUUID = uuid + + await waitJobs(servers) + }) + + it('Should not detect a thread as spam', async function () { + this.timeout(30000) + + await servers[1].comments.createThread({ videoId: videoUUID, text: 'remote comment 1' }) + await waitJobs(servers) + + const { data } = await servers[0].comments.listThreads({ videoId: videoUUID }) + expect(data).to.have.lengthOf(1) + }) + + it('Should not detect a reply as spam', async function () { + this.timeout(30000) + + await servers[1].comments.addReplyToLastThread({ text: 'I agree with you' }) + await waitJobs(servers) + + const { data } = await servers[0].comments.listThreads({ videoId: videoUUID }) + expect(data).to.have.lengthOf(1) + + const tree = await servers[0].comments.getThread({ videoId: videoUUID, threadId: data[0].id }) + expect(tree.children).to.have.lengthOf(1) + }) + + it('Should detect a thread as spam', async function () { + this.timeout(30000) + + await servers[1].comments.createThread({ videoId: videoUUID, text: 'akismet-guaranteed-spam' }) + await waitJobs(servers) + + const { data } = await servers[0].comments.listThreads({ videoId: videoUUID }) + expect(data).to.have.lengthOf(1) + }) + + it('Should detect a thread as spam', async function () { + this.timeout(30000) + + await servers[1].comments.addReplyToLastThread({ text: 'akismet-guaranteed-spam' }) + await waitJobs(servers) + + const { data } = await servers[0].comments.listThreads({ videoId: videoUUID }) + expect(data).to.have.lengthOf(1) + + const thread = data[0] + const tree = await servers[0].comments.getThread({ videoId: videoUUID, threadId: thread.id }) + expect(tree.children).to.have.lengthOf(1) + }) + }) + + describe('Signup', function () { + + before(async function () { + await servers[0].config.updateExistingSubConfig({ + newConfig: { + signup: { + enabled: true + } + } + }) + }) + + it('Should allow signup', async function () { + await servers[0].registrations.register({ + username: 'user1', + displayName: 'user 1' + }) + }) + + it('Should detect a signup as SPAM', async function () { + await servers[0].registrations.register({ + username: 'user2', + displayName: 'user 2', + email: 'akismet-guaranteed-spam@example.com', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/external-plugins/auth-ldap.ts b/packages/tests/src/external-plugins/auth-ldap.ts new file mode 100644 index 000000000..ad058110c --- /dev/null +++ b/packages/tests/src/external-plugins/auth-ldap.ts @@ -0,0 +1,117 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@peertube/peertube-server-commands' +import { HttpStatusCode } from '@peertube/peertube-models' + +describe('Official plugin auth-ldap', function () { + let server: PeerTubeServer + let accessToken: string + let userId: number + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + + await server.plugins.install({ npmName: 'peertube-plugin-auth-ldap' }) + }) + + it('Should not login with without LDAP settings', async function () { + await server.login.login({ user: { username: 'fry', password: 'fry' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should not login with bad LDAP settings', async function () { + await server.plugins.updateSettings({ + npmName: 'peertube-plugin-auth-ldap', + settings: { + 'bind-credentials': 'GoodNewsEveryone', + 'bind-dn': 'cn=admin,dc=planetexpress,dc=com', + 'insecure-tls': false, + 'mail-property': 'mail', + 'search-base': 'ou=people,dc=planetexpress,dc=com', + 'search-filter': '(|(mail={{username}})(uid={{username}}))', + 'url': 'ldap://127.0.0.1:390', + 'username-property': 'uid' + } + }) + + await server.login.login({ user: { username: 'fry', password: 'fry' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should not login with good LDAP settings but wrong username/password', async function () { + await server.plugins.updateSettings({ + npmName: 'peertube-plugin-auth-ldap', + settings: { + 'bind-credentials': 'GoodNewsEveryone', + 'bind-dn': 'cn=admin,dc=planetexpress,dc=com', + 'insecure-tls': false, + 'mail-property': 'mail', + 'search-base': 'ou=people,dc=planetexpress,dc=com', + 'search-filter': '(|(mail={{username}})(uid={{username}}))', + 'url': 'ldap://127.0.0.1:10389', + 'username-property': 'uid' + } + }) + + await server.login.login({ user: { username: 'fry', password: 'bad password' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await server.login.login({ user: { username: 'fryr', password: 'fry' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should login with the appropriate username/password', async function () { + accessToken = await server.login.getAccessToken({ username: 'fry', password: 'fry' }) + }) + + it('Should login with the appropriate email/password', async function () { + accessToken = await server.login.getAccessToken({ username: 'fry@planetexpress.com', password: 'fry' }) + }) + + it('Should login get my profile', async function () { + const body = await server.users.getMyInfo({ token: accessToken }) + expect(body.username).to.equal('fry') + expect(body.email).to.equal('fry@planetexpress.com') + + userId = body.id + }) + + it('Should upload a video', async function () { + await server.videos.upload({ token: accessToken, attributes: { name: 'my super video' } }) + }) + + it('Should not be able to login if the user is banned', async function () { + await server.users.banUser({ userId }) + + await server.login.login({ + user: { username: 'fry@planetexpress.com', password: 'fry' }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should be able to login if the user is unbanned', async function () { + await server.users.unbanUser({ userId }) + + await server.login.login({ user: { username: 'fry@planetexpress.com', password: 'fry' } }) + }) + + it('Should not be able to ask password reset', async function () { + await server.users.askResetPassword({ email: 'fry@planetexpress.com', expectedStatus: HttpStatusCode.CONFLICT_409 }) + }) + + it('Should not be able to ask email verification', async function () { + await server.users.askSendVerifyEmail({ email: 'fry@planetexpress.com', expectedStatus: HttpStatusCode.CONFLICT_409 }) + }) + + it('Should not login if the plugin is uninstalled', async function () { + await server.plugins.uninstall({ npmName: 'peertube-plugin-auth-ldap' }) + + await server.login.login({ + user: { username: 'fry@planetexpress.com', password: 'fry' }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/external-plugins/auto-block-videos.ts b/packages/tests/src/external-plugins/auto-block-videos.ts new file mode 100644 index 000000000..6146c827c --- /dev/null +++ b/packages/tests/src/external-plugins/auto-block-videos.ts @@ -0,0 +1,167 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { Video } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + killallServers, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' +import { MockBlocklist } from '../shared/mock-servers/index.js' + +async function check (server: PeerTubeServer, videoUUID: string, exists = true) { + const { data } = await server.videos.list() + + const video = data.find(v => v.uuid === videoUUID) + + if (exists) expect(video).to.not.be.undefined + else expect(video).to.be.undefined +} + +describe('Official plugin auto-block videos', function () { + let servers: PeerTubeServer[] + let blocklistServer: MockBlocklist + let server1Videos: Video[] = [] + let server2Videos: Video[] = [] + let port: number + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + + for (const server of servers) { + await server.plugins.install({ npmName: 'peertube-plugin-auto-block-videos' }) + } + + blocklistServer = new MockBlocklist() + port = await blocklistServer.initialize() + + await servers[0].videos.quickUpload({ name: 'video server 1' }) + await servers[1].videos.quickUpload({ name: 'video server 2' }) + await servers[1].videos.quickUpload({ name: 'video 2 server 2' }) + await servers[1].videos.quickUpload({ name: 'video 3 server 2' }) + + { + const { data } = await servers[0].videos.list() + server1Videos = data.map(v => Object.assign(v, { url: servers[0].url + '/videos/watch/' + v.uuid })) + } + + { + const { data } = await servers[1].videos.list() + server2Videos = data.map(v => Object.assign(v, { url: servers[1].url + '/videos/watch/' + v.uuid })) + } + + await doubleFollow(servers[0], servers[1]) + }) + + it('Should update plugin settings', async function () { + await servers[0].plugins.updateSettings({ + npmName: 'peertube-plugin-auto-block-videos', + settings: { + 'blocklist-urls': `http://127.0.0.1:${port}/blocklist`, + 'check-seconds-interval': 1 + } + }) + }) + + it('Should auto block a video', async function () { + await check(servers[0], server2Videos[0].uuid, true) + + blocklistServer.replace({ + data: [ + { + value: server2Videos[0].url + } + ] + }) + + await wait(2000) + + await check(servers[0], server2Videos[0].uuid, false) + }) + + it('Should have video in blacklists', async function () { + const body = await servers[0].blacklist.list() + + const videoBlacklists = body.data + expect(videoBlacklists).to.have.lengthOf(1) + expect(videoBlacklists[0].reason).to.contains('Automatically blocked from auto block plugin') + expect(videoBlacklists[0].video.name).to.equal(server2Videos[0].name) + }) + + it('Should not block a local video', async function () { + await check(servers[0], server1Videos[0].uuid, true) + + blocklistServer.replace({ + data: [ + { + value: server1Videos[0].url + } + ] + }) + + await wait(2000) + + await check(servers[0], server1Videos[0].uuid, true) + }) + + it('Should remove a video block', async function () { + await check(servers[0], server2Videos[0].uuid, false) + + blocklistServer.replace({ + data: [ + { + value: server2Videos[0].url, + action: 'remove' + } + ] + }) + + await wait(2000) + + await check(servers[0], server2Videos[0].uuid, true) + }) + + it('Should auto block a video, manually unblock it and do not reblock it automatically', async function () { + this.timeout(20000) + + const video = server2Videos[1] + + await check(servers[0], video.uuid, true) + + blocklistServer.replace({ + data: [ + { + value: video.url, + updatedAt: new Date().toISOString() + } + ] + }) + + await wait(2000) + + await check(servers[0], video.uuid, false) + + await servers[0].blacklist.remove({ videoId: video.uuid }) + + await check(servers[0], video.uuid, true) + + await killallServers([ servers[0] ]) + await servers[0].run() + await wait(2000) + + await check(servers[0], video.uuid, true) + }) + + after(async function () { + await blocklistServer.terminate() + + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/external-plugins/auto-mute.ts b/packages/tests/src/external-plugins/auto-mute.ts new file mode 100644 index 000000000..b4050e236 --- /dev/null +++ b/packages/tests/src/external-plugins/auto-mute.ts @@ -0,0 +1,216 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + killallServers, + makeGetRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' +import { MockBlocklist } from '../shared/mock-servers/index.js' + +describe('Official plugin auto-mute', function () { + const autoMuteListPath = '/plugins/auto-mute/router/api/v1/mute-list' + let servers: PeerTubeServer[] + let blocklistServer: MockBlocklist + let port: number + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + + for (const server of servers) { + await server.plugins.install({ npmName: 'peertube-plugin-auto-mute' }) + } + + blocklistServer = new MockBlocklist() + port = await blocklistServer.initialize() + + await servers[0].videos.quickUpload({ name: 'video server 1' }) + await servers[1].videos.quickUpload({ name: 'video server 2' }) + + await doubleFollow(servers[0], servers[1]) + }) + + it('Should update plugin settings', async function () { + await servers[0].plugins.updateSettings({ + npmName: 'peertube-plugin-auto-mute', + settings: { + 'blocklist-urls': `http://127.0.0.1:${port}/blocklist`, + 'check-seconds-interval': 1 + } + }) + }) + + it('Should add a server blocklist', async function () { + blocklistServer.replace({ + data: [ + { + value: servers[1].host + } + ] + }) + + await wait(2000) + + const { total } = await servers[0].videos.list() + expect(total).to.equal(1) + }) + + it('Should remove a server blocklist', async function () { + blocklistServer.replace({ + data: [ + { + value: servers[1].host, + action: 'remove' + } + ] + }) + + await wait(2000) + + const { total } = await servers[0].videos.list() + expect(total).to.equal(2) + }) + + it('Should add an account blocklist', async function () { + blocklistServer.replace({ + data: [ + { + value: 'root@' + servers[1].host + } + ] + }) + + await wait(2000) + + const { total } = await servers[0].videos.list() + expect(total).to.equal(1) + }) + + it('Should remove an account blocklist', async function () { + blocklistServer.replace({ + data: [ + { + value: 'root@' + servers[1].host, + action: 'remove' + } + ] + }) + + await wait(2000) + + const { total } = await servers[0].videos.list() + expect(total).to.equal(2) + }) + + it('Should auto mute an account, manually unmute it and do not remute it automatically', async function () { + this.timeout(20000) + + const account = 'root@' + servers[1].host + + blocklistServer.replace({ + data: [ + { + value: account, + updatedAt: new Date().toISOString() + } + ] + }) + + await wait(2000) + + { + const { total } = await servers[0].videos.list() + expect(total).to.equal(1) + } + + await servers[0].blocklist.removeFromServerBlocklist({ account }) + + { + const { total } = await servers[0].videos.list() + expect(total).to.equal(2) + } + + await killallServers([ servers[0] ]) + await servers[0].run() + await wait(2000) + + { + const { total } = await servers[0].videos.list() + expect(total).to.equal(2) + } + }) + + it('Should not expose the auto mute list', async function () { + await makeGetRequest({ + url: servers[0].url, + path: '/plugins/auto-mute/router/api/v1/mute-list', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should enable auto mute list', async function () { + await servers[0].plugins.updateSettings({ + npmName: 'peertube-plugin-auto-mute', + settings: { + 'blocklist-urls': '', + 'check-seconds-interval': 1, + 'expose-mute-list': true + } + }) + + await makeGetRequest({ + url: servers[0].url, + path: '/plugins/auto-mute/router/api/v1/mute-list', + expectedStatus: HttpStatusCode.OK_200 + }) + }) + + it('Should mute an account on server 1, and server 2 auto mutes it', async function () { + this.timeout(20000) + + await servers[1].plugins.updateSettings({ + npmName: 'peertube-plugin-auto-mute', + settings: { + 'blocklist-urls': 'http://' + servers[0].host + autoMuteListPath, + 'check-seconds-interval': 1, + 'expose-mute-list': false + } + }) + + await servers[0].blocklist.addToServerBlocklist({ account: 'root@' + servers[1].host }) + await servers[0].blocklist.addToMyBlocklist({ server: servers[1].host }) + + const res = await makeGetRequest({ + url: servers[0].url, + path: '/plugins/auto-mute/router/api/v1/mute-list', + expectedStatus: HttpStatusCode.OK_200 + }) + + const data = res.body.data + expect(data).to.have.lengthOf(1) + expect(data[0].updatedAt).to.exist + expect(data[0].value).to.equal('root@' + servers[1].host) + + await wait(2000) + + for (const server of servers) { + const { total } = await server.videos.list() + expect(total).to.equal(1) + } + }) + + after(async function () { + await blocklistServer.terminate() + + await cleanupTests(servers) + }) +}) diff --git a/server/tests/external-plugins/index.ts b/packages/tests/src/external-plugins/index.ts similarity index 100% rename from server/tests/external-plugins/index.ts rename to packages/tests/src/external-plugins/index.ts diff --git a/packages/tests/src/feeds/feeds.ts b/packages/tests/src/feeds/feeds.ts new file mode 100644 index 000000000..7587bb34e --- /dev/null +++ b/packages/tests/src/feeds/feeds.ts @@ -0,0 +1,697 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import * as chai from 'chai' +import chaiJSONSChema from 'chai-json-schema' +import chaiXML from 'chai-xml' +import { XMLParser, XMLValidator } from 'fast-xml-parser' +import { HttpStatusCode, VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + createSingleServer, + doubleFollow, + makeGetRequest, + makeRawRequest, + PeerTubeServer, + PluginsCommand, + setAccessTokensToServers, + setDefaultChannelAvatar, + setDefaultVideoChannel, + stopFfmpeg, + waitJobs +} from '@peertube/peertube-server-commands' + +chai.use(chaiXML) +chai.use(chaiJSONSChema) +chai.config.includeStack = true + +const expect = chai.expect + +describe('Test syndication feeds', () => { + let servers: PeerTubeServer[] = [] + let serverHLSOnly: PeerTubeServer + + let userAccessToken: string + let rootAccountId: number + let rootChannelId: number + + let userAccountId: number + let userChannelId: number + let userFeedToken: string + + let liveId: string + + before(async function () { + this.timeout(120000) + + // Run servers + servers = await createMultipleServers(2) + serverHLSOnly = await createSingleServer(3, { + transcoding: { + enabled: true, + web_videos: { enabled: false }, + hls: { enabled: true } + } + }) + + await setAccessTokensToServers([ ...servers, serverHLSOnly ]) + await setDefaultChannelAvatar(servers[0]) + await setDefaultVideoChannel(servers) + await doubleFollow(servers[0], servers[1]) + + await servers[0].config.enableLive({ allowReplay: false, transcoding: false }) + + { + const user = await servers[0].users.getMyInfo() + rootAccountId = user.account.id + rootChannelId = user.videoChannels[0].id + } + + { + userAccessToken = await servers[0].users.generateUserAndToken('john') + + const user = await servers[0].users.getMyInfo({ token: userAccessToken }) + userAccountId = user.account.id + userChannelId = user.videoChannels[0].id + + const token = await servers[0].users.getMyScopedTokens({ token: userAccessToken }) + userFeedToken = token.feedToken + } + + { + await servers[0].videos.upload({ token: userAccessToken, attributes: { name: 'user video' } }) + } + + { + const attributes = { + name: 'my super name for server 1', + description: 'my super description for server 1', + fixture: 'video_short.webm' + } + const { id } = await servers[0].videos.upload({ attributes }) + + await servers[0].comments.createThread({ videoId: id, text: 'super comment 1' }) + await servers[0].comments.createThread({ videoId: id, text: 'super comment 2' }) + } + + { + const attributes = { name: 'unlisted video', privacy: VideoPrivacy.UNLISTED } + const { id } = await servers[0].videos.upload({ attributes }) + + await servers[0].comments.createThread({ videoId: id, text: 'comment on unlisted video' }) + } + + { + const attributes = { name: 'password protected video', privacy: VideoPrivacy.PASSWORD_PROTECTED, videoPasswords: [ 'password' ] } + const { id } = await servers[0].videos.upload({ attributes }) + + await servers[0].comments.createThread({ videoId: id, text: 'comment on password protected video' }) + } + + await serverHLSOnly.videos.upload({ attributes: { name: 'hls only video' } }) + + await waitJobs([ ...servers, serverHLSOnly ]) + + await servers[0].plugins.install({ path: PluginsCommand.getPluginTestPath('-podcast-custom-tags') }) + }) + + describe('All feed', function () { + + it('Should be well formed XML (covers RSS 2.0 and ATOM 1.0 endpoints)', async function () { + for (const feed of [ 'video-comments' as 'video-comments', 'videos' as 'videos' ]) { + const rss = await servers[0].feed.getXML({ feed, ignoreCache: true }) + expect(rss).xml.to.be.valid() + + const atom = await servers[0].feed.getXML({ feed, format: 'atom', ignoreCache: true }) + expect(atom).xml.to.be.valid() + } + }) + + it('Should be well formed XML (covers Podcast endpoint)', async function () { + const podcast = await servers[0].feed.getPodcastXML({ ignoreCache: true, channelId: rootChannelId }) + expect(podcast).xml.to.be.valid() + }) + + it('Should be well formed JSON (covers JSON feed 1.0 endpoint)', async function () { + for (const feed of [ 'video-comments' as 'video-comments', 'videos' as 'videos' ]) { + const jsonText = await servers[0].feed.getJSON({ feed, ignoreCache: true }) + expect(JSON.parse(jsonText)).to.be.jsonSchema({ type: 'object' }) + } + }) + + it('Should serve the endpoint with a classic request', async function () { + await makeGetRequest({ + url: servers[0].url, + path: '/feeds/videos.xml', + accept: 'application/xml', + expectedStatus: HttpStatusCode.OK_200 + }) + }) + + it('Should refuse to serve the endpoint without accept header', async function () { + await makeGetRequest({ url: servers[0].url, path: '/feeds/videos.xml', expectedStatus: HttpStatusCode.NOT_ACCEPTABLE_406 }) + }) + }) + + describe('Videos feed', function () { + + describe('Podcast feed', function () { + + it('Should contain a valid podcast:alternateEnclosure', async function () { + // Since podcast feeds should only work on the server they originate on, + // only test the first server where the videos reside + const rss = await servers[0].feed.getPodcastXML({ ignoreCache: false, channelId: rootChannelId }) + expect(XMLValidator.validate(rss)).to.be.true + + const parser = new XMLParser({ parseAttributeValue: true, ignoreAttributes: false }) + const xmlDoc = parser.parse(rss) + + const itemGuid = xmlDoc.rss.channel.item.guid + expect(itemGuid).to.exist + expect(itemGuid['@_isPermaLink']).to.equal(true) + + const enclosure = xmlDoc.rss.channel.item.enclosure + expect(enclosure).to.exist + const alternateEnclosure = xmlDoc.rss.channel.item['podcast:alternateEnclosure'] + expect(alternateEnclosure).to.exist + + expect(alternateEnclosure['@_type']).to.equal('video/webm') + expect(alternateEnclosure['@_length']).to.equal(218910) + expect(alternateEnclosure['@_lang']).to.equal('zh') + expect(alternateEnclosure['@_title']).to.equal('720p') + expect(alternateEnclosure['@_default']).to.equal(true) + + expect(alternateEnclosure['podcast:source'][0]['@_uri']).to.contain('-720.webm') + expect(alternateEnclosure['podcast:source'][0]['@_uri']).to.equal(enclosure['@_url']) + expect(alternateEnclosure['podcast:source'][1]['@_uri']).to.contain('-720.torrent') + expect(alternateEnclosure['podcast:source'][1]['@_contentType']).to.equal('application/x-bittorrent') + expect(alternateEnclosure['podcast:source'][2]['@_uri']).to.contain('magnet:?') + }) + + it('Should contain a valid podcast:alternateEnclosure with HLS only', async function () { + const rss = await serverHLSOnly.feed.getPodcastXML({ ignoreCache: false, channelId: rootChannelId }) + expect(XMLValidator.validate(rss)).to.be.true + + const parser = new XMLParser({ parseAttributeValue: true, ignoreAttributes: false }) + const xmlDoc = parser.parse(rss) + + const itemGuid = xmlDoc.rss.channel.item.guid + expect(itemGuid).to.exist + expect(itemGuid['@_isPermaLink']).to.equal(true) + + const enclosure = xmlDoc.rss.channel.item.enclosure + const alternateEnclosure = xmlDoc.rss.channel.item['podcast:alternateEnclosure'] + expect(alternateEnclosure).to.exist + + expect(alternateEnclosure['@_type']).to.equal('application/x-mpegURL') + expect(alternateEnclosure['@_lang']).to.equal('zh') + expect(alternateEnclosure['@_title']).to.equal('HLS') + expect(alternateEnclosure['@_default']).to.equal(true) + + expect(alternateEnclosure['podcast:source']['@_uri']).to.contain('-master.m3u8') + expect(alternateEnclosure['podcast:source']['@_uri']).to.equal(enclosure['@_url']) + }) + + it('Should contain a valid podcast:socialInteract', async function () { + const rss = await servers[0].feed.getPodcastXML({ ignoreCache: false, channelId: rootChannelId }) + expect(XMLValidator.validate(rss)).to.be.true + + const parser = new XMLParser({ parseAttributeValue: true, ignoreAttributes: false }) + const xmlDoc = parser.parse(rss) + + const item = xmlDoc.rss.channel.item + const socialInteract = item['podcast:socialInteract'] + expect(socialInteract).to.exist + expect(socialInteract['@_protocol']).to.equal('activitypub') + expect(socialInteract['@_uri']).to.exist + expect(socialInteract['@_accountUrl']).to.exist + }) + + it('Should contain a valid support custom tags for plugins', async function () { + const rss = await servers[0].feed.getPodcastXML({ ignoreCache: false, channelId: userChannelId }) + expect(XMLValidator.validate(rss)).to.be.true + + const parser = new XMLParser({ parseAttributeValue: true, ignoreAttributes: false }) + const xmlDoc = parser.parse(rss) + + const fooTag = xmlDoc.rss.channel.fooTag + expect(fooTag).to.exist + expect(fooTag['@_bar']).to.equal('baz') + expect(fooTag['#text']).to.equal(42) + + const bizzBuzzItem = xmlDoc.rss.channel['biz:buzzItem'] + expect(bizzBuzzItem).to.exist + + let nestedTag = bizzBuzzItem.nestedTag + expect(nestedTag).to.exist + expect(nestedTag).to.equal('example nested tag') + + const item = xmlDoc.rss.channel.item + const fizzTag = item.fizzTag + expect(fizzTag).to.exist + expect(fizzTag['@_bar']).to.equal('baz') + expect(fizzTag['#text']).to.equal(21) + + const bizzBuzz = item['biz:buzz'] + expect(bizzBuzz).to.exist + + nestedTag = bizzBuzz.nestedTag + expect(nestedTag).to.exist + expect(nestedTag).to.equal('example nested tag') + }) + + it('Should contain a valid podcast:liveItem for live streams', async function () { + this.timeout(120000) + + const { uuid } = await servers[0].live.create({ + fields: { + name: 'live-0', + privacy: VideoPrivacy.PUBLIC, + channelId: rootChannelId, + permanentLive: false + } + }) + liveId = uuid + + const ffmpeg = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveId, copyCodecs: true, fixtureName: 'video_short.mp4' }) + await servers[0].live.waitUntilPublished({ videoId: liveId }) + + const rss = await servers[0].feed.getPodcastXML({ ignoreCache: false, channelId: rootChannelId }) + expect(XMLValidator.validate(rss)).to.be.true + + const parser = new XMLParser({ parseAttributeValue: true, ignoreAttributes: false }) + const xmlDoc = parser.parse(rss) + const liveItem = xmlDoc.rss.channel['podcast:liveItem'] + expect(liveItem.title).to.equal('live-0') + expect(liveItem.guid['@_isPermaLink']).to.equal(false) + expect(liveItem.guid['#text']).to.contain(`${uuid}_`) + expect(liveItem['@_status']).to.equal('live') + + const enclosure = liveItem.enclosure + const alternateEnclosure = liveItem['podcast:alternateEnclosure'] + expect(alternateEnclosure).to.exist + expect(alternateEnclosure['@_type']).to.equal('application/x-mpegURL') + expect(alternateEnclosure['@_title']).to.equal('HLS live stream') + expect(alternateEnclosure['@_default']).to.equal(true) + + expect(alternateEnclosure['podcast:source']['@_uri']).to.contain('/master.m3u8') + expect(alternateEnclosure['podcast:source']['@_uri']).to.equal(enclosure['@_url']) + + await stopFfmpeg(ffmpeg) + + await servers[0].live.waitUntilEnded({ videoId: liveId }) + + await waitJobs(servers) + }) + }) + + describe('JSON feed', function () { + + it('Should contain a valid \'attachments\' object', async function () { + for (const server of servers) { + const json = await server.feed.getJSON({ feed: 'videos', ignoreCache: true }) + const jsonObj = JSON.parse(json) + expect(jsonObj.items.length).to.be.equal(2) + expect(jsonObj.items[0].attachments).to.exist + expect(jsonObj.items[0].attachments.length).to.be.eq(1) + expect(jsonObj.items[0].attachments[0].mime_type).to.be.eq('application/x-bittorrent') + expect(jsonObj.items[0].attachments[0].size_in_bytes).to.be.eq(218910) + expect(jsonObj.items[0].attachments[0].url).to.contain('720.torrent') + } + }) + + it('Should filter by account', async function () { + { + const json = await servers[0].feed.getJSON({ feed: 'videos', query: { accountId: rootAccountId }, ignoreCache: true }) + const jsonObj = JSON.parse(json) + expect(jsonObj.items.length).to.be.equal(1) + expect(jsonObj.items[0].title).to.equal('my super name for server 1') + expect(jsonObj.items[0].author.name).to.equal('Main root channel') + } + + { + const json = await servers[0].feed.getJSON({ feed: 'videos', query: { accountId: userAccountId }, ignoreCache: true }) + const jsonObj = JSON.parse(json) + expect(jsonObj.items.length).to.be.equal(1) + expect(jsonObj.items[0].title).to.equal('user video') + expect(jsonObj.items[0].author.name).to.equal('Main john channel') + } + + for (const server of servers) { + { + const json = await server.feed.getJSON({ feed: 'videos', query: { accountName: 'root@' + servers[0].host }, ignoreCache: true }) + const jsonObj = JSON.parse(json) + expect(jsonObj.items.length).to.be.equal(1) + expect(jsonObj.items[0].title).to.equal('my super name for server 1') + } + + { + const json = await server.feed.getJSON({ feed: 'videos', query: { accountName: 'john@' + servers[0].host }, ignoreCache: true }) + const jsonObj = JSON.parse(json) + expect(jsonObj.items.length).to.be.equal(1) + expect(jsonObj.items[0].title).to.equal('user video') + } + } + }) + + it('Should filter by video channel', async function () { + { + const json = await servers[0].feed.getJSON({ feed: 'videos', query: { videoChannelId: rootChannelId }, ignoreCache: true }) + const jsonObj = JSON.parse(json) + expect(jsonObj.items.length).to.be.equal(1) + expect(jsonObj.items[0].title).to.equal('my super name for server 1') + expect(jsonObj.items[0].author.name).to.equal('Main root channel') + } + + { + const json = await servers[0].feed.getJSON({ feed: 'videos', query: { videoChannelId: userChannelId }, ignoreCache: true }) + const jsonObj = JSON.parse(json) + expect(jsonObj.items.length).to.be.equal(1) + expect(jsonObj.items[0].title).to.equal('user video') + expect(jsonObj.items[0].author.name).to.equal('Main john channel') + } + + for (const server of servers) { + { + const query = { videoChannelName: 'root_channel@' + servers[0].host } + const json = await server.feed.getJSON({ feed: 'videos', query, ignoreCache: true }) + const jsonObj = JSON.parse(json) + expect(jsonObj.items.length).to.be.equal(1) + expect(jsonObj.items[0].title).to.equal('my super name for server 1') + } + + { + const query = { videoChannelName: 'john_channel@' + servers[0].host } + const json = await server.feed.getJSON({ feed: 'videos', query, ignoreCache: true }) + const jsonObj = JSON.parse(json) + expect(jsonObj.items.length).to.be.equal(1) + expect(jsonObj.items[0].title).to.equal('user video') + } + } + }) + + it('Should correctly have videos feed with HLS only', async function () { + this.timeout(120000) + + const json = await serverHLSOnly.feed.getJSON({ feed: 'videos', ignoreCache: true }) + const jsonObj = JSON.parse(json) + expect(jsonObj.items.length).to.be.equal(1) + expect(jsonObj.items[0].attachments).to.exist + expect(jsonObj.items[0].attachments.length).to.be.eq(4) + + for (let i = 0; i < 4; i++) { + expect(jsonObj.items[0].attachments[i].mime_type).to.be.eq('application/x-bittorrent') + expect(jsonObj.items[0].attachments[i].size_in_bytes).to.be.greaterThan(0) + expect(jsonObj.items[0].attachments[i].url).to.exist + } + }) + + it('Should not display waiting live videos', async function () { + const { uuid } = await servers[0].live.create({ + fields: { + name: 'live', + privacy: VideoPrivacy.PUBLIC, + channelId: rootChannelId + } + }) + liveId = uuid + + const json = await servers[0].feed.getJSON({ feed: 'videos', ignoreCache: true }) + + const jsonObj = JSON.parse(json) + expect(jsonObj.items.length).to.be.equal(2) + expect(jsonObj.items[0].title).to.equal('my super name for server 1') + expect(jsonObj.items[1].title).to.equal('user video') + }) + + it('Should display published live videos', async function () { + this.timeout(120000) + + const ffmpeg = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveId, copyCodecs: true, fixtureName: 'video_short.mp4' }) + await servers[0].live.waitUntilPublished({ videoId: liveId }) + + const json = await servers[0].feed.getJSON({ feed: 'videos', ignoreCache: true }) + + const jsonObj = JSON.parse(json) + expect(jsonObj.items.length).to.be.equal(3) + expect(jsonObj.items[0].title).to.equal('live') + expect(jsonObj.items[1].title).to.equal('my super name for server 1') + expect(jsonObj.items[2].title).to.equal('user video') + + await stopFfmpeg(ffmpeg) + }) + + it('Should have the channel avatar as feed icon', async function () { + const json = await servers[0].feed.getJSON({ feed: 'videos', query: { videoChannelId: rootChannelId }, ignoreCache: true }) + + const jsonObj = JSON.parse(json) + const imageUrl = jsonObj.icon + expect(imageUrl).to.include('/lazy-static/avatars/') + await makeRawRequest({ url: imageUrl, expectedStatus: HttpStatusCode.OK_200 }) + }) + }) + }) + + describe('Video comments feed', function () { + + it('Should contain valid comments (covers JSON feed 1.0 endpoint) and not from unlisted/password protected videos', async function () { + for (const server of servers) { + const json = await server.feed.getJSON({ feed: 'video-comments', ignoreCache: true }) + + const jsonObj = JSON.parse(json) + expect(jsonObj.items.length).to.be.equal(2) + expect(jsonObj.items[0].content_html).to.contain('

super comment 2

') + expect(jsonObj.items[1].content_html).to.contain('

super comment 1

') + } + }) + + it('Should not list comments from muted accounts or instances', async function () { + this.timeout(30000) + + const remoteHandle = 'root@' + servers[0].host + + await servers[1].blocklist.addToServerBlocklist({ account: remoteHandle }) + + { + const json = await servers[1].feed.getJSON({ feed: 'video-comments', ignoreCache: true }) + const jsonObj = JSON.parse(json) + expect(jsonObj.items.length).to.be.equal(0) + } + + await servers[1].blocklist.removeFromServerBlocklist({ account: remoteHandle }) + + { + const videoUUID = (await servers[1].videos.quickUpload({ name: 'server 2' })).uuid + await waitJobs(servers) + await servers[0].comments.createThread({ videoId: videoUUID, text: 'super comment' }) + await waitJobs(servers) + + const json = await servers[1].feed.getJSON({ feed: 'video-comments', ignoreCache: true }) + const jsonObj = JSON.parse(json) + expect(jsonObj.items.length).to.be.equal(3) + } + + await servers[1].blocklist.addToMyBlocklist({ account: remoteHandle }) + + { + const json = await servers[1].feed.getJSON({ feed: 'video-comments', ignoreCache: true }) + const jsonObj = JSON.parse(json) + expect(jsonObj.items.length).to.be.equal(2) + } + }) + }) + + describe('Video feed from my subscriptions', function () { + let feeduserAccountId: number + let feeduserFeedToken: string + + it('Should list no videos for a user with no videos and no subscriptions', async function () { + const attr = { username: 'feeduser', password: 'password' } + await servers[0].users.create({ username: attr.username, password: attr.password }) + const feeduserAccessToken = await servers[0].login.getAccessToken(attr) + + { + const user = await servers[0].users.getMyInfo({ token: feeduserAccessToken }) + feeduserAccountId = user.account.id + } + + { + const token = await servers[0].users.getMyScopedTokens({ token: feeduserAccessToken }) + feeduserFeedToken = token.feedToken + } + + { + const body = await servers[0].videos.listMySubscriptionVideos({ token: feeduserAccessToken }) + expect(body.total).to.equal(0) + + const query = { accountId: feeduserAccountId, token: feeduserFeedToken } + const json = await servers[0].feed.getJSON({ feed: 'subscriptions', query, ignoreCache: true }) + const jsonObj = JSON.parse(json) + expect(jsonObj.items.length).to.be.equal(0) // no subscription, it should not list the instance's videos but list 0 videos + } + }) + + it('Should fail with an invalid token', async function () { + const query = { accountId: feeduserAccountId, token: 'toto' } + await servers[0].feed.getJSON({ feed: 'subscriptions', query, expectedStatus: HttpStatusCode.FORBIDDEN_403, ignoreCache: true }) + }) + + it('Should fail with a token of another user', async function () { + const query = { accountId: feeduserAccountId, token: userFeedToken } + await servers[0].feed.getJSON({ feed: 'subscriptions', query, expectedStatus: HttpStatusCode.FORBIDDEN_403, ignoreCache: true }) + }) + + it('Should list no videos for a user with videos but no subscriptions', async function () { + const body = await servers[0].videos.listMySubscriptionVideos({ token: userAccessToken }) + expect(body.total).to.equal(0) + + const query = { accountId: userAccountId, token: userFeedToken } + const json = await servers[0].feed.getJSON({ feed: 'subscriptions', query, ignoreCache: true }) + const jsonObj = JSON.parse(json) + expect(jsonObj.items.length).to.be.equal(0) // no subscription, it should not list the instance's videos but list 0 videos + }) + + it('Should list self videos for a user with a subscription to themselves', async function () { + this.timeout(30000) + + await servers[0].subscriptions.add({ token: userAccessToken, targetUri: 'john_channel@' + servers[0].host }) + await waitJobs(servers) + + { + const body = await servers[0].videos.listMySubscriptionVideos({ token: userAccessToken }) + expect(body.total).to.equal(1) + expect(body.data[0].name).to.equal('user video') + + const query = { accountId: userAccountId, token: userFeedToken } + const json = await servers[0].feed.getJSON({ feed: 'subscriptions', query, ignoreCache: true }) + const jsonObj = JSON.parse(json) + expect(jsonObj.items.length).to.be.equal(1) // subscribed to self, it should not list the instance's videos but list john's + } + }) + + it('Should list videos of a user\'s subscription', async function () { + this.timeout(30000) + + await servers[0].subscriptions.add({ token: userAccessToken, targetUri: 'root_channel@' + servers[0].host }) + await waitJobs(servers) + + { + const body = await servers[0].videos.listMySubscriptionVideos({ token: userAccessToken }) + expect(body.total).to.equal(2, 'there should be 2 videos part of the subscription') + + const query = { accountId: userAccountId, token: userFeedToken } + const json = await servers[0].feed.getJSON({ feed: 'subscriptions', query, ignoreCache: true }) + const jsonObj = JSON.parse(json) + expect(jsonObj.items.length).to.be.equal(2) // subscribed to root, it should not list the instance's videos but list root/john's + } + }) + + it('Should renew the token, and so have an invalid old token', async function () { + await servers[0].users.renewMyScopedTokens({ token: userAccessToken }) + + const query = { accountId: userAccountId, token: userFeedToken } + await servers[0].feed.getJSON({ feed: 'subscriptions', query, expectedStatus: HttpStatusCode.FORBIDDEN_403, ignoreCache: true }) + }) + + it('Should succeed with the new token', async function () { + const token = await servers[0].users.getMyScopedTokens({ token: userAccessToken }) + userFeedToken = token.feedToken + + const query = { accountId: userAccountId, token: userFeedToken } + await servers[0].feed.getJSON({ feed: 'subscriptions', query, ignoreCache: true }) + }) + + }) + + describe('Cache', function () { + const uuids: string[] = [] + + function doPodcastRequest () { + return makeGetRequest({ + url: servers[0].url, + path: '/feeds/podcast/videos.xml', + query: { videoChannelId: servers[0].store.channel.id }, + accept: 'application/xml', + expectedStatus: HttpStatusCode.OK_200 + }) + } + + function doVideosRequest (query: { [id: string]: string } = {}) { + return makeGetRequest({ + url: servers[0].url, + path: '/feeds/videos.xml', + query, + accept: 'application/xml', + expectedStatus: HttpStatusCode.OK_200 + }) + } + + before(async function () { + { + const { uuid } = await servers[0].videos.quickUpload({ name: 'cache 1' }) + uuids.push(uuid) + } + + { + const { uuid } = await servers[0].videos.quickUpload({ name: 'cache 2' }) + uuids.push(uuid) + } + }) + + it('Should serve the videos endpoint as a cached request', async function () { + await doVideosRequest() + + const res = await doVideosRequest() + + expect(res.headers['x-api-cache-cached']).to.equal('true') + }) + + it('Should not serve the videos endpoint as a cached request', async function () { + const res = await doVideosRequest({ v: '186' }) + + expect(res.headers['x-api-cache-cached']).to.not.exist + }) + + it('Should invalidate the podcast feed cache after video deletion', async function () { + await doPodcastRequest() + + { + const res = await doPodcastRequest() + expect(res.headers['x-api-cache-cached']).to.exist + } + + await servers[0].videos.remove({ id: uuids[0] }) + + { + const res = await doPodcastRequest() + expect(res.headers['x-api-cache-cached']).to.not.exist + } + }) + + it('Should invalidate the podcast feed cache after video deletion, even after server restart', async function () { + this.timeout(120000) + + await doPodcastRequest() + + { + const res = await doPodcastRequest() + expect(res.headers['x-api-cache-cached']).to.exist + } + + await servers[0].kill() + await servers[0].run() + + await servers[0].videos.remove({ id: uuids[1] }) + + const res = await doPodcastRequest() + expect(res.headers['x-api-cache-cached']).to.not.exist + }) + + }) + + after(async function () { + await servers[0].plugins.uninstall({ npmName: 'peertube-plugin-test-podcast-custom-tags' }) + + await cleanupTests([ ...servers, serverHLSOnly ]) + }) +}) diff --git a/server/tests/feeds/index.ts b/packages/tests/src/feeds/index.ts similarity index 100% rename from server/tests/feeds/index.ts rename to packages/tests/src/feeds/index.ts diff --git a/packages/tests/src/misc-endpoints.ts b/packages/tests/src/misc-endpoints.ts new file mode 100644 index 000000000..0067578ed --- /dev/null +++ b/packages/tests/src/misc-endpoints.ts @@ -0,0 +1,243 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { writeJson } from 'fs-extra/esm' +import { join } from 'path' +import { HttpStatusCode, VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + makeGetRequest, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' +import { expectLogDoesNotContain } from './shared/checks.js' + +describe('Test misc endpoints', function () { + let server: PeerTubeServer + let wellKnownPath: string + + before(async function () { + this.timeout(120000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + + wellKnownPath = server.getDirectoryPath('well-known') + }) + + describe('Test a well known endpoints', function () { + + it('Should get security.txt', async function () { + const res = await makeGetRequest({ + url: server.url, + path: '/.well-known/security.txt', + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(res.text).to.contain('security issue') + }) + + it('Should get nodeinfo', async function () { + const res = await makeGetRequest({ + url: server.url, + path: '/.well-known/nodeinfo', + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(res.body.links).to.be.an('array') + expect(res.body.links).to.have.lengthOf(1) + expect(res.body.links[0].rel).to.equal('http://nodeinfo.diaspora.software/ns/schema/2.0') + }) + + it('Should get dnt policy text', async function () { + const res = await makeGetRequest({ + url: server.url, + path: '/.well-known/dnt-policy.txt', + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(res.text).to.contain('http://www.w3.org/TR/tracking-dnt') + }) + + it('Should get dnt policy', async function () { + const res = await makeGetRequest({ + url: server.url, + path: '/.well-known/dnt', + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(res.body.tracking).to.equal('N') + }) + + it('Should get change-password location', async function () { + const res = await makeGetRequest({ + url: server.url, + path: '/.well-known/change-password', + expectedStatus: HttpStatusCode.FOUND_302 + }) + + expect(res.header.location).to.equal('/my-account/settings') + }) + + it('Should test webfinger', async function () { + const resource = 'acct:peertube@' + server.host + const accountUrl = server.url + '/accounts/peertube' + + const res = await makeGetRequest({ + url: server.url, + path: '/.well-known/webfinger?resource=' + resource, + expectedStatus: HttpStatusCode.OK_200 + }) + + const data = res.body + + expect(data.subject).to.equal(resource) + expect(data.aliases).to.contain(accountUrl) + + const self = data.links.find(l => l.rel === 'self') + expect(self).to.exist + expect(self.type).to.equal('application/activity+json') + expect(self.href).to.equal(accountUrl) + + const remoteInteract = data.links.find(l => l.rel === 'http://ostatus.org/schema/1.0/subscribe') + expect(remoteInteract).to.exist + expect(remoteInteract.template).to.equal(server.url + '/remote-interaction?uri={uri}') + }) + + it('Should return 404 for non-existing files in /.well-known', async function () { + await makeGetRequest({ + url: server.url, + path: '/.well-known/non-existing-file', + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should return custom file from /.well-known', async function () { + const filename = 'existing-file.json' + + await writeJson(join(wellKnownPath, filename), { iThink: 'therefore I am' }) + + const { body } = await makeGetRequest({ + url: server.url, + path: '/.well-known/' + filename, + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(body.iThink).to.equal('therefore I am') + }) + }) + + describe('Test classic static endpoints', function () { + + it('Should get robots.txt', async function () { + const res = await makeGetRequest({ + url: server.url, + path: '/robots.txt', + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(res.text).to.contain('User-agent') + }) + + it('Should get security.txt', async function () { + await makeGetRequest({ + url: server.url, + path: '/security.txt', + expectedStatus: HttpStatusCode.MOVED_PERMANENTLY_301 + }) + }) + + it('Should get nodeinfo', async function () { + const res = await makeGetRequest({ + url: server.url, + path: '/nodeinfo/2.0.json', + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(res.body.software.name).to.equal('peertube') + expect(res.body.usage.users.activeMonth).to.equal(1) + expect(res.body.usage.users.activeHalfyear).to.equal(1) + }) + }) + + describe('Test bots endpoints', function () { + + it('Should get the empty sitemap', async function () { + const res = await makeGetRequest({ + url: server.url, + path: '/sitemap.xml', + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"') + expect(res.text).to.contain('' + server.url + '/about/instance') + }) + + it('Should get the empty cached sitemap', async function () { + const res = await makeGetRequest({ + url: server.url, + path: '/sitemap.xml', + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"') + expect(res.text).to.contain('' + server.url + '/about/instance') + }) + + it('Should add videos, channel and accounts and get sitemap', async function () { + this.timeout(35000) + + await server.videos.upload({ attributes: { name: 'video 1', nsfw: false } }) + await server.videos.upload({ attributes: { name: 'video 2', nsfw: false } }) + await server.videos.upload({ attributes: { name: 'video 3', privacy: VideoPrivacy.PRIVATE } }) + + await server.channels.create({ attributes: { name: 'channel1', displayName: 'channel 1' } }) + await server.channels.create({ attributes: { name: 'channel2', displayName: 'channel 2' } }) + + await server.users.create({ username: 'user1', password: 'password' }) + await server.users.create({ username: 'user2', password: 'password' }) + + const res = await makeGetRequest({ + url: server.url, + path: '/sitemap.xml?t=1', // avoid using cache + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"') + expect(res.text).to.contain('' + server.url + '/about/instance') + + expect(res.text).to.contain('video 1') + expect(res.text).to.contain('video 2') + expect(res.text).to.not.contain('video 3') + + expect(res.text).to.contain('' + server.url + '/video-channels/channel1') + expect(res.text).to.contain('' + server.url + '/video-channels/channel2') + + expect(res.text).to.contain('' + server.url + '/accounts/user1') + expect(res.text).to.contain('' + server.url + '/accounts/user2') + }) + + it('Should not fail with big title/description videos', async function () { + const name = 'v'.repeat(115) + + await server.videos.upload({ attributes: { name, description: 'd'.repeat(2500), nsfw: false } }) + + const res = await makeGetRequest({ + url: server.url, + path: '/sitemap.xml?t=2', // avoid using cache + expectedStatus: HttpStatusCode.OK_200 + }) + + await expectLogDoesNotContain(server, 'Warning in sitemap generation') + await expectLogDoesNotContain(server, 'Error in sitemap generation') + + expect(res.text).to.contain(`${'v'.repeat(97)}...`) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/peertube-runner/client-cli.ts b/packages/tests/src/peertube-runner/client-cli.ts new file mode 100644 index 000000000..814b7f13a --- /dev/null +++ b/packages/tests/src/peertube-runner/client-cli.ts @@ -0,0 +1,76 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { PeerTubeRunnerProcess } from '@tests/shared/peertube-runner-process.js' +import { + cleanupTests, + createSingleServer, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel +} from '@peertube/peertube-server-commands' + +describe('Test peertube-runner program client CLI', function () { + let server: PeerTubeServer + let peertubeRunner: PeerTubeRunnerProcess + + before(async function () { + this.timeout(120_000) + + server = await createSingleServer(1) + + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + await server.config.enableRemoteTranscoding() + + peertubeRunner = new PeerTubeRunnerProcess(server) + await peertubeRunner.runServer() + }) + + it('Should not have PeerTube instance listed', async function () { + const data = await peertubeRunner.listRegisteredPeerTubeInstances() + + expect(data).to.not.contain(server.url) + }) + + it('Should register a new PeerTube instance', async function () { + const registrationToken = await server.runnerRegistrationTokens.getFirstRegistrationToken() + + await peertubeRunner.registerPeerTubeInstance({ + registrationToken, + runnerName: 'my super runner', + runnerDescription: 'super description' + }) + }) + + it('Should list this new PeerTube instance', async function () { + const data = await peertubeRunner.listRegisteredPeerTubeInstances() + + expect(data).to.contain(server.url) + expect(data).to.contain('my super runner') + expect(data).to.contain('super description') + }) + + it('Should still have the configuration after a restart', async function () { + peertubeRunner.kill() + + await peertubeRunner.runServer() + }) + + it('Should unregister the PeerTube instance', async function () { + await peertubeRunner.unregisterPeerTubeInstance({ runnerName: 'my super runner' }) + }) + + it('Should not have PeerTube instance listed', async function () { + const data = await peertubeRunner.listRegisteredPeerTubeInstances() + + expect(data).to.not.contain(server.url) + }) + + after(async function () { + peertubeRunner.kill() + + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/peertube-runner/index.ts b/packages/tests/src/peertube-runner/index.ts new file mode 100644 index 000000000..29f21694f --- /dev/null +++ b/packages/tests/src/peertube-runner/index.ts @@ -0,0 +1,4 @@ +export * from './client-cli.js' +export * from './live-transcoding.js' +export * from './studio-transcoding.js' +export * from './vod-transcoding.js' diff --git a/packages/tests/src/peertube-runner/live-transcoding.ts b/packages/tests/src/peertube-runner/live-transcoding.ts new file mode 100644 index 000000000..9351bc5e2 --- /dev/null +++ b/packages/tests/src/peertube-runner/live-transcoding.ts @@ -0,0 +1,200 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { HttpStatusCode, VideoPrivacy } from '@peertube/peertube-models' +import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + findExternalSavedVideo, + makeRawRequest, + ObjectStorageCommand, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + stopFfmpeg, + waitJobs, + waitUntilLivePublishedOnAllServers, + waitUntilLiveWaitingOnAllServers +} from '@peertube/peertube-server-commands' +import { expectStartWith } from '@tests/shared/checks.js' +import { checkPeerTubeRunnerCacheIsEmpty } from '@tests/shared/directories.js' +import { testLiveVideoResolutions } from '@tests/shared/live.js' +import { PeerTubeRunnerProcess } from '@tests/shared/peertube-runner-process.js' +import { SQLCommand } from '@tests/shared/sql-command.js' + +describe('Test Live transcoding in peertube-runner program', function () { + let servers: PeerTubeServer[] = [] + let peertubeRunner: PeerTubeRunnerProcess + let sqlCommandServer1: SQLCommand + + function runSuite (options: { + objectStorage?: ObjectStorageCommand + } = {}) { + const { objectStorage } = options + + it('Should enable transcoding without additional resolutions', async function () { + this.timeout(120000) + + const { video } = await servers[0].live.quickCreate({ permanentLive: true, saveReplay: false, privacy: VideoPrivacy.PUBLIC }) + + const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: video.uuid }) + await waitUntilLivePublishedOnAllServers(servers, video.uuid) + await waitJobs(servers) + + await testLiveVideoResolutions({ + originServer: servers[0], + sqlCommand: sqlCommandServer1, + servers, + liveVideoId: video.uuid, + resolutions: [ 720, 480, 360, 240, 144 ], + objectStorage, + transcoded: true + }) + + await stopFfmpeg(ffmpegCommand) + + await waitUntilLiveWaitingOnAllServers(servers, video.uuid) + await servers[0].videos.remove({ id: video.id }) + }) + + it('Should transcode audio only RTMP stream', async function () { + this.timeout(120000) + + const { video } = await servers[0].live.quickCreate({ permanentLive: true, saveReplay: false, privacy: VideoPrivacy.UNLISTED }) + + const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: video.uuid, fixtureName: 'video_short_no_audio.mp4' }) + await waitUntilLivePublishedOnAllServers(servers, video.uuid) + await waitJobs(servers) + + await stopFfmpeg(ffmpegCommand) + + await waitUntilLiveWaitingOnAllServers(servers, video.uuid) + await servers[0].videos.remove({ id: video.id }) + }) + + it('Should save a replay', async function () { + this.timeout(240000) + + const { video } = await servers[0].live.quickCreate({ permanentLive: true, saveReplay: true }) + + const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: video.uuid }) + await waitUntilLivePublishedOnAllServers(servers, video.uuid) + + await testLiveVideoResolutions({ + originServer: servers[0], + sqlCommand: sqlCommandServer1, + servers, + liveVideoId: video.uuid, + resolutions: [ 720, 480, 360, 240, 144 ], + objectStorage, + transcoded: true + }) + + await stopFfmpeg(ffmpegCommand) + + await waitUntilLiveWaitingOnAllServers(servers, video.uuid) + await waitJobs(servers) + + const session = await servers[0].live.findLatestSession({ videoId: video.uuid }) + expect(session.endingProcessed).to.be.true + expect(session.endDate).to.exist + expect(session.saveReplay).to.be.true + + const videoLiveDetails = await servers[0].videos.get({ id: video.uuid }) + const replay = await findExternalSavedVideo(servers[0], videoLiveDetails) + + for (const server of servers) { + const video = await server.videos.get({ id: replay.uuid }) + + expect(video.files).to.have.lengthOf(0) + expect(video.streamingPlaylists).to.have.lengthOf(1) + + const files = video.streamingPlaylists[0].files + expect(files).to.have.lengthOf(5) + + for (const file of files) { + if (objectStorage) { + expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl()) + } + + await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + } + } + }) + } + + before(async function () { + this.timeout(120_000) + + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + await doubleFollow(servers[0], servers[1]) + + sqlCommandServer1 = new SQLCommand(servers[0]) + + await servers[0].config.enableRemoteTranscoding() + await servers[0].config.enableTranscoding({ hls: true, webVideo: true, with0p: true }) + await servers[0].config.enableLive({ allowReplay: true, resolutions: 'max', transcoding: true }) + + const registrationToken = await servers[0].runnerRegistrationTokens.getFirstRegistrationToken() + + peertubeRunner = new PeerTubeRunnerProcess(servers[0]) + await peertubeRunner.runServer() + await peertubeRunner.registerPeerTubeInstance({ registrationToken, runnerName: 'runner' }) + }) + + describe('With lives on local filesystem storage', function () { + + before(async function () { + await servers[0].config.enableTranscoding({ webVideo: true, hls: false, with0p: true }) + }) + + runSuite() + }) + + describe('With lives on object storage', function () { + if (areMockObjectStorageTestsDisabled()) return + + const objectStorage = new ObjectStorageCommand() + + before(async function () { + await objectStorage.prepareDefaultMockBuckets() + + await servers[0].kill() + + await servers[0].run(objectStorage.getDefaultMockConfig()) + + // Wait for peertube runner socket reconnection + await wait(1500) + }) + + runSuite({ objectStorage }) + + after(async function () { + await objectStorage.cleanupMock() + }) + }) + + describe('Check cleanup', function () { + + it('Should have an empty cache directory', async function () { + await checkPeerTubeRunnerCacheIsEmpty(peertubeRunner) + }) + }) + + after(async function () { + if (peertubeRunner) { + await peertubeRunner.unregisterPeerTubeInstance({ runnerName: 'runner' }) + peertubeRunner.kill() + } + + if (sqlCommandServer1) await sqlCommandServer1.cleanup() + + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/peertube-runner/studio-transcoding.ts b/packages/tests/src/peertube-runner/studio-transcoding.ts new file mode 100644 index 000000000..50e61091a --- /dev/null +++ b/packages/tests/src/peertube-runner/studio-transcoding.ts @@ -0,0 +1,127 @@ + +import { expect } from 'chai' +import { getAllFiles, wait } from '@peertube/peertube-core-utils' +import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + ObjectStorageCommand, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + VideoStudioCommand, + waitJobs +} from '@peertube/peertube-server-commands' +import { expectStartWith, checkVideoDuration } from '@tests/shared/checks.js' +import { checkPeerTubeRunnerCacheIsEmpty } from '@tests/shared/directories.js' +import { PeerTubeRunnerProcess } from '@tests/shared/peertube-runner-process.js' + +describe('Test studio transcoding in peertube-runner program', function () { + let servers: PeerTubeServer[] = [] + let peertubeRunner: PeerTubeRunnerProcess + + function runSuite (options: { + objectStorage?: ObjectStorageCommand + } = {}) { + const { objectStorage } = options + + it('Should run a complex studio transcoding', async function () { + this.timeout(120000) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'mp4', fixture: 'video_short.mp4' }) + await waitJobs(servers) + + const video = await servers[0].videos.get({ id: uuid }) + const oldFileUrls = getAllFiles(video).map(f => f.fileUrl) + + await servers[0].videoStudio.createEditionTasks({ videoId: uuid, tasks: VideoStudioCommand.getComplexTask() }) + await waitJobs(servers, { runnerJobs: true }) + + for (const server of servers) { + const video = await server.videos.get({ id: uuid }) + const files = getAllFiles(video) + + for (const f of files) { + expect(oldFileUrls).to.not.include(f.fileUrl) + } + + if (objectStorage) { + for (const webVideoFile of video.files) { + expectStartWith(webVideoFile.fileUrl, objectStorage.getMockWebVideosBaseUrl()) + } + + for (const hlsFile of video.streamingPlaylists[0].files) { + expectStartWith(hlsFile.fileUrl, objectStorage.getMockPlaylistBaseUrl()) + } + } + + await checkVideoDuration(server, uuid, 9) + } + }) + } + + before(async function () { + this.timeout(120_000) + + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + await doubleFollow(servers[0], servers[1]) + + await servers[0].config.enableTranscoding({ hls: true, webVideo: true }) + await servers[0].config.enableStudio() + await servers[0].config.enableRemoteStudio() + + const registrationToken = await servers[0].runnerRegistrationTokens.getFirstRegistrationToken() + + peertubeRunner = new PeerTubeRunnerProcess(servers[0]) + await peertubeRunner.runServer() + await peertubeRunner.registerPeerTubeInstance({ registrationToken, runnerName: 'runner' }) + }) + + describe('With videos on local filesystem storage', function () { + runSuite() + }) + + describe('With videos on object storage', function () { + if (areMockObjectStorageTestsDisabled()) return + + const objectStorage = new ObjectStorageCommand() + + before(async function () { + await objectStorage.prepareDefaultMockBuckets() + + await servers[0].kill() + + await servers[0].run(objectStorage.getDefaultMockConfig()) + + // Wait for peertube runner socket reconnection + await wait(1500) + }) + + runSuite({ objectStorage }) + + after(async function () { + await objectStorage.cleanupMock() + }) + }) + + describe('Check cleanup', function () { + + it('Should have an empty cache directory', async function () { + await checkPeerTubeRunnerCacheIsEmpty(peertubeRunner) + }) + }) + + after(async function () { + if (peertubeRunner) { + await peertubeRunner.unregisterPeerTubeInstance({ runnerName: 'runner' }) + peertubeRunner.kill() + } + + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/peertube-runner/vod-transcoding.ts b/packages/tests/src/peertube-runner/vod-transcoding.ts new file mode 100644 index 000000000..ff5cefe36 --- /dev/null +++ b/packages/tests/src/peertube-runner/vod-transcoding.ts @@ -0,0 +1,349 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ +import { expect } from 'chai' +import { getAllFiles, wait } from '@peertube/peertube-core-utils' +import { VideoPrivacy } from '@peertube/peertube-models' +import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + ObjectStorageCommand, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' +import { checkPeerTubeRunnerCacheIsEmpty } from '@tests/shared/directories.js' +import { PeerTubeRunnerProcess } from '@tests/shared/peertube-runner-process.js' +import { completeCheckHlsPlaylist } from '@tests/shared/streaming-playlists.js' +import { completeWebVideoFilesCheck } from '@tests/shared/videos.js' + +describe('Test VOD transcoding in peertube-runner program', function () { + let servers: PeerTubeServer[] = [] + let peertubeRunner: PeerTubeRunnerProcess + + function runSuite (options: { + webVideoEnabled: boolean + hlsEnabled: boolean + objectStorage?: ObjectStorageCommand + }) { + const { webVideoEnabled, hlsEnabled, objectStorage } = options + + const objectStorageBaseUrlWebVideo = objectStorage + ? objectStorage.getMockWebVideosBaseUrl() + : undefined + + const objectStorageBaseUrlHLS = objectStorage + ? objectStorage.getMockPlaylistBaseUrl() + : undefined + + it('Should upload a classic video mp4 and transcode it', async function () { + this.timeout(120000) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'mp4', fixture: 'video_short.mp4' }) + + await waitJobs(servers, { runnerJobs: true }) + + for (const server of servers) { + if (webVideoEnabled) { + await completeWebVideoFilesCheck({ + server, + originServer: servers[0], + fixture: 'video_short.mp4', + videoUUID: uuid, + objectStorageBaseUrl: objectStorageBaseUrlWebVideo, + files: [ + { resolution: 0 }, + { resolution: 144 }, + { resolution: 240 }, + { resolution: 360 }, + { resolution: 480 }, + { resolution: 720 } + ] + }) + } + + if (hlsEnabled) { + await completeCheckHlsPlaylist({ + hlsOnly: !webVideoEnabled, + servers, + videoUUID: uuid, + objectStorageBaseUrl: objectStorageBaseUrlHLS, + resolutions: [ 720, 480, 360, 240, 144, 0 ] + }) + } + } + }) + + it('Should upload a webm video and transcode it', async function () { + this.timeout(120000) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'mp4', fixture: 'video_short.webm' }) + + await waitJobs(servers, { runnerJobs: true }) + + for (const server of servers) { + if (webVideoEnabled) { + await completeWebVideoFilesCheck({ + server, + originServer: servers[0], + fixture: 'video_short.webm', + videoUUID: uuid, + objectStorageBaseUrl: objectStorageBaseUrlWebVideo, + files: [ + { resolution: 0 }, + { resolution: 144 }, + { resolution: 240 }, + { resolution: 360 }, + { resolution: 480 }, + { resolution: 720 } + ] + }) + } + + if (hlsEnabled) { + await completeCheckHlsPlaylist({ + hlsOnly: !webVideoEnabled, + servers, + videoUUID: uuid, + objectStorageBaseUrl: objectStorageBaseUrlHLS, + resolutions: [ 720, 480, 360, 240, 144, 0 ] + }) + } + } + }) + + it('Should upload an audio only video and transcode it', async function () { + this.timeout(120000) + + const attributes = { name: 'audio_without_preview', fixture: 'sample.ogg' } + const { uuid } = await servers[0].videos.upload({ attributes, mode: 'resumable' }) + + await waitJobs(servers, { runnerJobs: true }) + + for (const server of servers) { + if (webVideoEnabled) { + await completeWebVideoFilesCheck({ + server, + originServer: servers[0], + fixture: 'sample.ogg', + videoUUID: uuid, + objectStorageBaseUrl: objectStorageBaseUrlWebVideo, + files: [ + { resolution: 0 }, + { resolution: 144 }, + { resolution: 240 }, + { resolution: 360 }, + { resolution: 480 } + ] + }) + } + + if (hlsEnabled) { + await completeCheckHlsPlaylist({ + hlsOnly: !webVideoEnabled, + servers, + videoUUID: uuid, + objectStorageBaseUrl: objectStorageBaseUrlHLS, + resolutions: [ 480, 360, 240, 144, 0 ] + }) + } + } + }) + + it('Should upload a private video and transcode it', async function () { + this.timeout(120000) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'mp4', fixture: 'video_short.mp4', privacy: VideoPrivacy.PRIVATE }) + + await waitJobs(servers, { runnerJobs: true }) + + if (webVideoEnabled) { + await completeWebVideoFilesCheck({ + server: servers[0], + originServer: servers[0], + fixture: 'video_short.mp4', + videoUUID: uuid, + objectStorageBaseUrl: objectStorageBaseUrlWebVideo, + files: [ + { resolution: 0 }, + { resolution: 144 }, + { resolution: 240 }, + { resolution: 360 }, + { resolution: 480 }, + { resolution: 720 } + ] + }) + } + + if (hlsEnabled) { + await completeCheckHlsPlaylist({ + hlsOnly: !webVideoEnabled, + servers: [ servers[0] ], + videoUUID: uuid, + objectStorageBaseUrl: objectStorageBaseUrlHLS, + resolutions: [ 720, 480, 360, 240, 144, 0 ] + }) + } + }) + + it('Should transcode videos on manual run', async function () { + this.timeout(120000) + + await servers[0].config.disableTranscoding() + + const { uuid } = await servers[0].videos.quickUpload({ name: 'manual transcoding', fixture: 'video_short.mp4' }) + await waitJobs(servers, { runnerJobs: true }) + + { + const video = await servers[0].videos.get({ id: uuid }) + expect(getAllFiles(video)).to.have.lengthOf(1) + } + + await servers[0].config.enableTranscoding({ hls: true, webVideo: true, with0p: true }) + + await servers[0].videos.runTranscoding({ transcodingType: 'web-video', videoId: uuid }) + await waitJobs(servers, { runnerJobs: true }) + + await completeWebVideoFilesCheck({ + server: servers[0], + originServer: servers[0], + fixture: 'video_short.mp4', + videoUUID: uuid, + objectStorageBaseUrl: objectStorageBaseUrlWebVideo, + files: [ + { resolution: 0 }, + { resolution: 144 }, + { resolution: 240 }, + { resolution: 360 }, + { resolution: 480 }, + { resolution: 720 } + ] + }) + + await servers[0].videos.runTranscoding({ transcodingType: 'hls', videoId: uuid }) + await waitJobs(servers, { runnerJobs: true }) + + await completeCheckHlsPlaylist({ + hlsOnly: false, + servers: [ servers[0] ], + videoUUID: uuid, + objectStorageBaseUrl: objectStorageBaseUrlHLS, + resolutions: [ 720, 480, 360, 240, 144, 0 ] + }) + }) + } + + before(async function () { + this.timeout(120_000) + + servers = await createMultipleServers(2) + + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + await doubleFollow(servers[0], servers[1]) + + await servers[0].config.enableRemoteTranscoding() + + const registrationToken = await servers[0].runnerRegistrationTokens.getFirstRegistrationToken() + + peertubeRunner = new PeerTubeRunnerProcess(servers[0]) + await peertubeRunner.runServer() + await peertubeRunner.registerPeerTubeInstance({ registrationToken, runnerName: 'runner' }) + }) + + describe('With videos on local filesystem storage', function () { + + describe('Web video only enabled', function () { + + before(async function () { + await servers[0].config.enableTranscoding({ webVideo: true, hls: false, with0p: true }) + }) + + runSuite({ webVideoEnabled: true, hlsEnabled: false }) + }) + + describe('HLS videos only enabled', function () { + + before(async function () { + await servers[0].config.enableTranscoding({ webVideo: false, hls: true, with0p: true }) + }) + + runSuite({ webVideoEnabled: false, hlsEnabled: true }) + }) + + describe('Web video & HLS enabled', function () { + + before(async function () { + await servers[0].config.enableTranscoding({ hls: true, webVideo: true, with0p: true }) + }) + + runSuite({ webVideoEnabled: true, hlsEnabled: true }) + }) + }) + + describe('With videos on object storage', function () { + if (areMockObjectStorageTestsDisabled()) return + + const objectStorage = new ObjectStorageCommand() + + before(async function () { + await objectStorage.prepareDefaultMockBuckets() + + await servers[0].kill() + + await servers[0].run(objectStorage.getDefaultMockConfig()) + + // Wait for peertube runner socket reconnection + await wait(1500) + }) + + describe('Web video only enabled', function () { + + before(async function () { + await servers[0].config.enableTranscoding({ webVideo: true, hls: false, with0p: true }) + }) + + runSuite({ webVideoEnabled: true, hlsEnabled: false, objectStorage }) + }) + + describe('HLS videos only enabled', function () { + + before(async function () { + await servers[0].config.enableTranscoding({ webVideo: false, hls: true, with0p: true }) + }) + + runSuite({ webVideoEnabled: false, hlsEnabled: true, objectStorage }) + }) + + describe('Web video & HLS enabled', function () { + + before(async function () { + await servers[0].config.enableTranscoding({ hls: true, webVideo: true, with0p: true }) + }) + + runSuite({ webVideoEnabled: true, hlsEnabled: true, objectStorage }) + }) + + after(async function () { + await objectStorage.cleanupMock() + }) + }) + + describe('Check cleanup', function () { + + it('Should have an empty cache directory', async function () { + await checkPeerTubeRunnerCacheIsEmpty(peertubeRunner) + }) + }) + + after(async function () { + if (peertubeRunner) { + await peertubeRunner.unregisterPeerTubeInstance({ runnerName: 'runner' }) + peertubeRunner.kill() + } + + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/plugins/action-hooks.ts b/packages/tests/src/plugins/action-hooks.ts new file mode 100644 index 000000000..136c7671b --- /dev/null +++ b/packages/tests/src/plugins/action-hooks.ts @@ -0,0 +1,298 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { ServerHookName, VideoPlaylistPrivacy, VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + killallServers, + PeerTubeServer, + PluginsCommand, + setAccessTokensToServers, + setDefaultVideoChannel, + stopFfmpeg, + waitJobs +} from '@peertube/peertube-server-commands' + +describe('Test plugin action hooks', function () { + let servers: PeerTubeServer[] + let videoUUID: string + let threadId: number + + function checkHook (hook: ServerHookName, strictCount = true, count = 1) { + return servers[0].servers.waitUntilLog('Run hook ' + hook, count, strictCount) + } + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + await servers[0].plugins.install({ path: PluginsCommand.getPluginTestPath() }) + + await killallServers([ servers[0] ]) + + await servers[0].run({ + live: { + enabled: true + } + }) + + await servers[0].config.enableFileUpdate() + + await doubleFollow(servers[0], servers[1]) + }) + + describe('Application hooks', function () { + it('Should run action:application.listening', async function () { + await checkHook('action:application.listening') + }) + }) + + describe('Videos hooks', function () { + + it('Should run action:api.video.uploaded', async function () { + const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' } }) + videoUUID = uuid + + await checkHook('action:api.video.uploaded') + }) + + it('Should run action:api.video.updated', async function () { + await servers[0].videos.update({ id: videoUUID, attributes: { name: 'video updated' } }) + + await checkHook('action:api.video.updated') + }) + + it('Should run action:api.video.viewed', async function () { + await servers[0].views.simulateView({ id: videoUUID }) + + await checkHook('action:api.video.viewed') + }) + + it('Should run action:api.video.file-updated', async function () { + await servers[0].videos.replaceSourceFile({ videoId: videoUUID, fixture: 'video_short.mp4' }) + + await checkHook('action:api.video.file-updated') + }) + + it('Should run action:api.video.deleted', async function () { + await servers[0].videos.remove({ id: videoUUID }) + + await checkHook('action:api.video.deleted') + }) + + after(async function () { + const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) + videoUUID = uuid + }) + }) + + describe('Video channel hooks', function () { + const channelName = 'my_super_channel' + + it('Should run action:api.video-channel.created', async function () { + await servers[0].channels.create({ attributes: { name: channelName } }) + + await checkHook('action:api.video-channel.created') + }) + + it('Should run action:api.video-channel.updated', async function () { + await servers[0].channels.update({ channelName, attributes: { displayName: 'my display name' } }) + + await checkHook('action:api.video-channel.updated') + }) + + it('Should run action:api.video-channel.deleted', async function () { + await servers[0].channels.delete({ channelName }) + + await checkHook('action:api.video-channel.deleted') + }) + }) + + describe('Live hooks', function () { + + it('Should run action:api.live-video.created', async function () { + const attributes = { + name: 'live', + privacy: VideoPrivacy.PUBLIC, + channelId: servers[0].store.channel.id + } + + await servers[0].live.create({ fields: attributes }) + + await checkHook('action:api.live-video.created') + }) + + it('Should run action:live.video.state.updated', async function () { + this.timeout(60000) + + const attributes = { + name: 'live', + privacy: VideoPrivacy.PUBLIC, + channelId: servers[0].store.channel.id + } + + const { uuid: liveVideoId } = await servers[0].live.create({ fields: attributes }) + const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoId }) + await servers[0].live.waitUntilPublished({ videoId: liveVideoId }) + await waitJobs(servers) + + await checkHook('action:live.video.state.updated', true, 1) + + await stopFfmpeg(ffmpegCommand) + await servers[0].live.waitUntilEnded({ videoId: liveVideoId }) + await waitJobs(servers) + + await checkHook('action:live.video.state.updated', true, 2) + }) + }) + + describe('Comments hooks', function () { + it('Should run action:api.video-thread.created', async function () { + const created = await servers[0].comments.createThread({ videoId: videoUUID, text: 'thread' }) + threadId = created.id + + await checkHook('action:api.video-thread.created') + }) + + it('Should run action:api.video-comment-reply.created', async function () { + await servers[0].comments.addReply({ videoId: videoUUID, toCommentId: threadId, text: 'reply' }) + + await checkHook('action:api.video-comment-reply.created') + }) + + it('Should run action:api.video-comment.deleted', async function () { + await servers[0].comments.delete({ videoId: videoUUID, commentId: threadId }) + + await checkHook('action:api.video-comment.deleted') + }) + }) + + describe('Captions hooks', function () { + it('Should run action:api.video-caption.created', async function () { + await servers[0].captions.add({ videoId: videoUUID, language: 'en', fixture: 'subtitle-good.srt' }) + + await checkHook('action:api.video-caption.created') + }) + + it('Should run action:api.video-caption.deleted', async function () { + await servers[0].captions.delete({ videoId: videoUUID, language: 'en' }) + + await checkHook('action:api.video-caption.deleted') + }) + }) + + describe('Users hooks', function () { + let userId: number + + it('Should run action:api.user.registered', async function () { + await servers[0].registrations.register({ username: 'registered_user' }) + + await checkHook('action:api.user.registered') + }) + + it('Should run action:api.user.created', async function () { + const user = await servers[0].users.create({ username: 'created_user' }) + userId = user.id + + await checkHook('action:api.user.created') + }) + + it('Should run action:api.user.oauth2-got-token', async function () { + await servers[0].login.login({ user: { username: 'created_user' } }) + + await checkHook('action:api.user.oauth2-got-token') + }) + + it('Should run action:api.user.blocked', async function () { + await servers[0].users.banUser({ userId }) + + await checkHook('action:api.user.blocked') + }) + + it('Should run action:api.user.unblocked', async function () { + await servers[0].users.unbanUser({ userId }) + + await checkHook('action:api.user.unblocked') + }) + + it('Should run action:api.user.updated', async function () { + await servers[0].users.update({ userId, videoQuota: 50 }) + + await checkHook('action:api.user.updated') + }) + + it('Should run action:api.user.deleted', async function () { + await servers[0].users.remove({ userId }) + + await checkHook('action:api.user.deleted') + }) + }) + + describe('Playlist hooks', function () { + let playlistId: number + let videoId: number + + before(async function () { + { + const { id } = await servers[0].playlists.create({ + attributes: { + displayName: 'My playlist', + privacy: VideoPlaylistPrivacy.PRIVATE + } + }) + playlistId = id + } + + { + const { id } = await servers[0].videos.upload({ attributes: { name: 'my super name' } }) + videoId = id + } + }) + + it('Should run action:api.video-playlist-element.created', async function () { + await servers[0].playlists.addElement({ playlistId, attributes: { videoId } }) + + await checkHook('action:api.video-playlist-element.created') + }) + }) + + describe('Notification hook', function () { + + it('Should run action:notifier.notification.created', async function () { + await checkHook('action:notifier.notification.created', false) + }) + }) + + describe('Activity Pub hooks', function () { + let videoUUID: string + + it('Should run action:activity-pub.remote-video.created', async function () { + this.timeout(30000) + + const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' }) + videoUUID = uuid + + await servers[0].servers.waitUntilLog('action:activity-pub.remote-video.created - AP remote video - video remote video') + }) + + it('Should run action:activity-pub.remote-video.updated', async function () { + this.timeout(30000) + + await servers[1].videos.update({ id: videoUUID, attributes: { name: 'remote video updated' } }) + + await servers[0].servers.waitUntilLog( + 'action:activity-pub.remote-video.updated - AP remote video updated - video remote video updated', + 1, + false + ) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/plugins/external-auth.ts b/packages/tests/src/plugins/external-auth.ts new file mode 100644 index 000000000..c7fe22185 --- /dev/null +++ b/packages/tests/src/plugins/external-auth.ts @@ -0,0 +1,436 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { HttpStatusCode, HttpStatusCodeType, UserAdminFlag, UserRole } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + decodeQueryString, + PeerTubeServer, + PluginsCommand, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +async function loginExternal (options: { + server: PeerTubeServer + npmName: string + authName: string + username: string + query?: any + expectedStatus?: HttpStatusCodeType + expectedStatusStep2?: HttpStatusCodeType +}) { + const res = await options.server.plugins.getExternalAuth({ + npmName: options.npmName, + npmVersion: '0.0.1', + authName: options.authName, + query: options.query, + expectedStatus: options.expectedStatus || HttpStatusCode.FOUND_302 + }) + + if (res.status !== HttpStatusCode.FOUND_302) return + + const location = res.header.location + const { externalAuthToken } = decodeQueryString(location) + + const resLogin = await options.server.login.loginUsingExternalToken({ + username: options.username, + externalAuthToken: externalAuthToken as string, + expectedStatus: options.expectedStatusStep2 + }) + + return resLogin.body +} + +describe('Test external auth plugins', function () { + let server: PeerTubeServer + + let cyanAccessToken: string + let cyanRefreshToken: string + + let kefkaAccessToken: string + let kefkaRefreshToken: string + let kefkaId: number + + let externalAuthToken: string + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1, { + rates_limit: { + login: { + max: 30 + } + } + }) + + await setAccessTokensToServers([ server ]) + + for (const suffix of [ 'one', 'two', 'three' ]) { + await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-external-auth-' + suffix) }) + } + }) + + it('Should display the correct configuration', async function () { + const config = await server.config.getConfig() + + const auths = config.plugin.registeredExternalAuths + expect(auths).to.have.lengthOf(9) + + const auth2 = auths.find((a) => a.authName === 'external-auth-2') + expect(auth2).to.exist + expect(auth2.authDisplayName).to.equal('External Auth 2') + expect(auth2.npmName).to.equal('peertube-plugin-test-external-auth-one') + }) + + it('Should redirect for a Cyan login', async function () { + const res = await server.plugins.getExternalAuth({ + npmName: 'test-external-auth-one', + npmVersion: '0.0.1', + authName: 'external-auth-1', + query: { + username: 'cyan' + }, + expectedStatus: HttpStatusCode.FOUND_302 + }) + + const location = res.header.location + expect(location.startsWith('/login?')).to.be.true + + const searchParams = decodeQueryString(location) + + expect(searchParams.externalAuthToken).to.exist + expect(searchParams.username).to.equal('cyan') + + externalAuthToken = searchParams.externalAuthToken as string + }) + + it('Should reject auto external login with a missing or invalid token', async function () { + const command = server.login + + await command.loginUsingExternalToken({ username: 'cyan', externalAuthToken: '', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await command.loginUsingExternalToken({ username: 'cyan', externalAuthToken: 'blabla', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should reject auto external login with a missing or invalid username', async function () { + const command = server.login + + await command.loginUsingExternalToken({ username: '', externalAuthToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await command.loginUsingExternalToken({ username: '', externalAuthToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should reject auto external login with an expired token', async function () { + this.timeout(15000) + + await wait(5000) + + await server.login.loginUsingExternalToken({ + username: 'cyan', + externalAuthToken, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + + await server.servers.waitUntilLog('expired external auth token', 4) + }) + + it('Should auto login Cyan, create the user and use the token', async function () { + { + const res = await loginExternal({ + server, + npmName: 'test-external-auth-one', + authName: 'external-auth-1', + query: { + username: 'cyan' + }, + username: 'cyan' + }) + + cyanAccessToken = res.access_token + cyanRefreshToken = res.refresh_token + } + + { + const body = await server.users.getMyInfo({ token: cyanAccessToken }) + expect(body.username).to.equal('cyan') + expect(body.account.displayName).to.equal('cyan') + expect(body.email).to.equal('cyan@example.com') + expect(body.role.id).to.equal(UserRole.USER) + expect(body.adminFlags).to.equal(UserAdminFlag.NONE) + expect(body.videoQuota).to.equal(5242880) + expect(body.videoQuotaDaily).to.equal(-1) + } + }) + + it('Should auto login Kefka, create the user and use the token', async function () { + { + const res = await loginExternal({ + server, + npmName: 'test-external-auth-one', + authName: 'external-auth-2', + username: 'kefka' + }) + + kefkaAccessToken = res.access_token + kefkaRefreshToken = res.refresh_token + } + + { + const body = await server.users.getMyInfo({ token: kefkaAccessToken }) + expect(body.username).to.equal('kefka') + expect(body.account.displayName).to.equal('Kefka Palazzo') + expect(body.email).to.equal('kefka@example.com') + expect(body.role.id).to.equal(UserRole.ADMINISTRATOR) + expect(body.adminFlags).to.equal(UserAdminFlag.BYPASS_VIDEO_AUTO_BLACKLIST) + expect(body.videoQuota).to.equal(42000) + expect(body.videoQuotaDaily).to.equal(42100) + + kefkaId = body.id + } + }) + + it('Should refresh Cyan token, but not Kefka token', async function () { + { + const resRefresh = await server.login.refreshToken({ refreshToken: cyanRefreshToken }) + cyanAccessToken = resRefresh.body.access_token + cyanRefreshToken = resRefresh.body.refresh_token + + const body = await server.users.getMyInfo({ token: cyanAccessToken }) + expect(body.username).to.equal('cyan') + } + + { + await server.login.refreshToken({ refreshToken: kefkaRefreshToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + } + }) + + it('Should update Cyan profile', async function () { + await server.users.updateMe({ + token: cyanAccessToken, + displayName: 'Cyan Garamonde', + description: 'Retainer to the king of Doma' + }) + + const body = await server.users.getMyInfo({ token: cyanAccessToken }) + expect(body.account.displayName).to.equal('Cyan Garamonde') + expect(body.account.description).to.equal('Retainer to the king of Doma') + }) + + it('Should logout Cyan', async function () { + await server.login.logout({ token: cyanAccessToken }) + }) + + it('Should have logged out Cyan', async function () { + await server.servers.waitUntilLog('On logout cyan') + + await server.users.getMyInfo({ token: cyanAccessToken, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should login Cyan and keep the old existing profile', async function () { + { + const res = await loginExternal({ + server, + npmName: 'test-external-auth-one', + authName: 'external-auth-1', + query: { + username: 'cyan' + }, + username: 'cyan' + }) + + cyanAccessToken = res.access_token + } + + const body = await server.users.getMyInfo({ token: cyanAccessToken }) + expect(body.username).to.equal('cyan') + expect(body.account.displayName).to.equal('Cyan Garamonde') + expect(body.account.description).to.equal('Retainer to the king of Doma') + expect(body.role.id).to.equal(UserRole.USER) + }) + + it('Should login Kefka and update the profile', async function () { + { + await server.users.update({ userId: kefkaId, videoQuota: 43000, videoQuotaDaily: 43100 }) + await server.users.updateMe({ token: kefkaAccessToken, displayName: 'kefka updated' }) + + const body = await server.users.getMyInfo({ token: kefkaAccessToken }) + expect(body.username).to.equal('kefka') + expect(body.account.displayName).to.equal('kefka updated') + expect(body.videoQuota).to.equal(43000) + expect(body.videoQuotaDaily).to.equal(43100) + } + + { + const res = await loginExternal({ + server, + npmName: 'test-external-auth-one', + authName: 'external-auth-2', + username: 'kefka' + }) + + kefkaAccessToken = res.access_token + kefkaRefreshToken = res.refresh_token + + const body = await server.users.getMyInfo({ token: kefkaAccessToken }) + expect(body.username).to.equal('kefka') + expect(body.account.displayName).to.equal('Kefka Palazzo') + expect(body.videoQuota).to.equal(42000) + expect(body.videoQuotaDaily).to.equal(43100) + } + }) + + it('Should not update an external auth email', async function () { + await server.users.updateMe({ + token: cyanAccessToken, + email: 'toto@example.com', + currentPassword: 'toto', + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should reject token of Kefka by the plugin hook', async function () { + await wait(5000) + + await server.users.getMyInfo({ token: kefkaAccessToken, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should unregister external-auth-2 and do not login existing Kefka', async function () { + await server.plugins.updateSettings({ + npmName: 'peertube-plugin-test-external-auth-one', + settings: { disableKefka: true } + }) + + await server.login.login({ user: { username: 'kefka', password: 'fake' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + + await loginExternal({ + server, + npmName: 'test-external-auth-one', + authName: 'external-auth-2', + query: { + username: 'kefka' + }, + username: 'kefka', + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should have disabled this auth', async function () { + const config = await server.config.getConfig() + + const auths = config.plugin.registeredExternalAuths + expect(auths).to.have.lengthOf(8) + + const auth1 = auths.find(a => a.authName === 'external-auth-2') + expect(auth1).to.not.exist + }) + + it('Should uninstall the plugin one and do not login Cyan', async function () { + await server.plugins.uninstall({ npmName: 'peertube-plugin-test-external-auth-one' }) + + await loginExternal({ + server, + npmName: 'test-external-auth-one', + authName: 'external-auth-1', + query: { + username: 'cyan' + }, + username: 'cyan', + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + + await server.login.login({ user: { username: 'cyan', password: null }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await server.login.login({ user: { username: 'cyan', password: '' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await server.login.login({ user: { username: 'cyan', password: 'fake' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should not login kefka with another plugin', async function () { + await loginExternal({ + server, + npmName: 'test-external-auth-two', + authName: 'external-auth-4', + username: 'kefka2', + expectedStatusStep2: HttpStatusCode.BAD_REQUEST_400 + }) + + await loginExternal({ + server, + npmName: 'test-external-auth-two', + authName: 'external-auth-4', + username: 'kefka', + expectedStatusStep2: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should not login an existing user email', async function () { + await server.users.create({ username: 'existing_user', password: 'super_password' }) + + await loginExternal({ + server, + npmName: 'test-external-auth-two', + authName: 'external-auth-6', + username: 'existing_user', + expectedStatusStep2: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should be able to login an existing user username and channel', async function () { + await server.users.create({ username: 'existing_user2' }) + await server.users.create({ username: 'existing_user2-1_channel' }) + + // Test twice to ensure we don't generate a username on every login + for (let i = 0; i < 2; i++) { + const res = await loginExternal({ + server, + npmName: 'test-external-auth-two', + authName: 'external-auth-7', + username: 'existing_user2' + }) + + const token = res.access_token + + const myInfo = await server.users.getMyInfo({ token }) + expect(myInfo.username).to.equal('existing_user2-1') + + expect(myInfo.videoChannels[0].name).to.equal('existing_user2-1_channel-1') + } + }) + + it('Should display the correct configuration', async function () { + const config = await server.config.getConfig() + + const auths = config.plugin.registeredExternalAuths + expect(auths).to.have.lengthOf(7) + + const auth2 = auths.find((a) => a.authName === 'external-auth-2') + expect(auth2).to.not.exist + }) + + after(async function () { + await cleanupTests([ server ]) + }) + + it('Should forward the redirectUrl if the plugin returns one', async function () { + const resLogin = await loginExternal({ + server, + npmName: 'test-external-auth-three', + authName: 'external-auth-7', + username: 'cid' + }) + + const { redirectUrl } = await server.login.logout({ token: resLogin.access_token }) + expect(redirectUrl).to.equal('https://example.com/redirectUrl') + }) + + it('Should call the plugin\'s onLogout method with the request', async function () { + const resLogin = await loginExternal({ + server, + npmName: 'test-external-auth-three', + authName: 'external-auth-8', + username: 'cid' + }) + + const { redirectUrl } = await server.login.logout({ token: resLogin.access_token }) + expect(redirectUrl).to.equal('https://example.com/redirectUrl?access_token=' + resLogin.access_token) + }) +}) diff --git a/packages/tests/src/plugins/filter-hooks.ts b/packages/tests/src/plugins/filter-hooks.ts new file mode 100644 index 000000000..88cfee631 --- /dev/null +++ b/packages/tests/src/plugins/filter-hooks.ts @@ -0,0 +1,909 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { + HttpStatusCode, + PeerTubeProblemDocument, + VideoDetails, + VideoImportState, + VideoPlaylist, + VideoPlaylistPrivacy, + VideoPrivacy +} from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + makeActivityPubGetRequest, + makeGetRequest, + makeRawRequest, + PeerTubeServer, + PluginsCommand, + setAccessTokensToServers, + setDefaultVideoChannel, + waitJobs +} from '@peertube/peertube-server-commands' +import { FIXTURE_URLS } from '../shared/tests.js' + +describe('Test plugin filter hooks', function () { + let servers: PeerTubeServer[] + let videoUUID: string + let threadId: number + let videoPlaylistUUID: string + + before(async function () { + this.timeout(120000) + + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + await doubleFollow(servers[0], servers[1]) + + await servers[0].plugins.install({ path: PluginsCommand.getPluginTestPath() }) + await servers[0].plugins.install({ path: PluginsCommand.getPluginTestPath('-filter-translations') }) + { + ({ uuid: videoPlaylistUUID } = await servers[0].playlists.create({ + attributes: { + displayName: 'my super playlist', + privacy: VideoPlaylistPrivacy.PUBLIC, + description: 'my super description', + videoChannelId: servers[0].store.channel.id + } + })) + } + + for (let i = 0; i < 10; i++) { + const video = await servers[0].videos.upload({ attributes: { name: 'default video ' + i } }) + await servers[0].playlists.addElement({ playlistId: videoPlaylistUUID, attributes: { videoId: video.id } }) + } + + const { data } = await servers[0].videos.list() + videoUUID = data[0].uuid + + await servers[0].config.updateCustomSubConfig({ + newConfig: { + live: { enabled: true }, + signup: { enabled: true }, + videoFile: { + update: { + enabled: true + } + }, + import: { + videos: { + http: { enabled: true }, + torrent: { enabled: true } + } + } + } + }) + + // Root subscribes to itself + await servers[0].subscriptions.add({ targetUri: 'root_channel@' + servers[0].host }) + }) + + describe('Videos', function () { + + it('Should run filter:api.videos.list.params', async function () { + const { data } = await servers[0].videos.list({ start: 0, count: 2 }) + + // 2 plugins do +1 to the count parameter + expect(data).to.have.lengthOf(4) + }) + + it('Should run filter:api.videos.list.result', async function () { + const { total } = await servers[0].videos.list({ start: 0, count: 0 }) + + // Plugin do +1 to the total result + expect(total).to.equal(11) + }) + + it('Should run filter:api.video-playlist.videos.list.params', async function () { + const { data } = await servers[0].playlists.listVideos({ + count: 2, + playlistId: videoPlaylistUUID + }) + + // 1 plugin do +1 to the count parameter + expect(data).to.have.lengthOf(3) + }) + + it('Should run filter:api.video-playlist.videos.list.result', async function () { + const { total } = await servers[0].playlists.listVideos({ + count: 0, + playlistId: videoPlaylistUUID + }) + + // Plugin do +1 to the total result + expect(total).to.equal(11) + }) + + it('Should run filter:api.accounts.videos.list.params', async function () { + const { data } = await servers[0].videos.listByAccount({ handle: 'root', start: 0, count: 2 }) + + // 1 plugin do +1 to the count parameter + expect(data).to.have.lengthOf(3) + }) + + it('Should run filter:api.accounts.videos.list.result', async function () { + const { total } = await servers[0].videos.listByAccount({ handle: 'root', start: 0, count: 2 }) + + // Plugin do +2 to the total result + expect(total).to.equal(12) + }) + + it('Should run filter:api.video-channels.videos.list.params', async function () { + const { data } = await servers[0].videos.listByChannel({ handle: 'root_channel', start: 0, count: 2 }) + + // 1 plugin do +3 to the count parameter + expect(data).to.have.lengthOf(5) + }) + + it('Should run filter:api.video-channels.videos.list.result', async function () { + const { total } = await servers[0].videos.listByChannel({ handle: 'root_channel', start: 0, count: 2 }) + + // Plugin do +3 to the total result + expect(total).to.equal(13) + }) + + it('Should run filter:api.user.me.videos.list.params', async function () { + const { data } = await servers[0].videos.listMyVideos({ start: 0, count: 2 }) + + // 1 plugin do +4 to the count parameter + expect(data).to.have.lengthOf(6) + }) + + it('Should run filter:api.user.me.videos.list.result', async function () { + const { total } = await servers[0].videos.listMyVideos({ start: 0, count: 2 }) + + // Plugin do +4 to the total result + expect(total).to.equal(14) + }) + + it('Should run filter:api.user.me.subscription-videos.list.params', async function () { + const { data } = await servers[0].videos.listMySubscriptionVideos({ start: 0, count: 2 }) + + // 1 plugin do +1 to the count parameter + expect(data).to.have.lengthOf(3) + }) + + it('Should run filter:api.user.me.subscription-videos.list.result', async function () { + const { total } = await servers[0].videos.listMySubscriptionVideos({ start: 0, count: 2 }) + + // Plugin do +4 to the total result + expect(total).to.equal(14) + }) + + it('Should run filter:api.video.get.result', async function () { + const video = await servers[0].videos.get({ id: videoUUID }) + expect(video.name).to.contain('<3') + }) + }) + + describe('Video/live/import accept', function () { + + it('Should run filter:api.video.upload.accept.result', async function () { + const options = { attributes: { name: 'video with bad word' }, expectedStatus: HttpStatusCode.FORBIDDEN_403 } + await servers[0].videos.upload({ mode: 'legacy', ...options }) + await servers[0].videos.upload({ mode: 'resumable', ...options }) + }) + + it('Should run filter:api.video.update-file.accept.result', async function () { + const res = await servers[0].videos.replaceSourceFile({ + videoId: videoUUID, + fixture: 'video_short1.webm', + completedExpectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + + expect((res as any)?.error).to.equal('no webm') + }) + + it('Should run filter:api.live-video.create.accept.result', async function () { + const attributes = { + name: 'video with bad word', + privacy: VideoPrivacy.PUBLIC, + channelId: servers[0].store.channel.id + } + + await servers[0].live.create({ fields: attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should run filter:api.video.pre-import-url.accept.result', async function () { + const attributes = { + name: 'normal title', + privacy: VideoPrivacy.PUBLIC, + channelId: servers[0].store.channel.id, + targetUrl: FIXTURE_URLS.goodVideo + 'bad' + } + await servers[0].imports.importVideo({ attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should run filter:api.video.pre-import-torrent.accept.result', async function () { + const attributes = { + name: 'bad torrent', + privacy: VideoPrivacy.PUBLIC, + channelId: servers[0].store.channel.id, + torrentfile: 'video-720p.torrent' as any + } + await servers[0].imports.importVideo({ attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + }) + + it('Should run filter:api.video.post-import-url.accept.result', async function () { + this.timeout(60000) + + let videoImportId: number + + { + const attributes = { + name: 'title with bad word', + privacy: VideoPrivacy.PUBLIC, + channelId: servers[0].store.channel.id, + targetUrl: FIXTURE_URLS.goodVideo + } + const body = await servers[0].imports.importVideo({ attributes }) + videoImportId = body.id + } + + await waitJobs(servers) + + { + const body = await servers[0].imports.getMyVideoImports() + const videoImports = body.data + + const videoImport = videoImports.find(i => i.id === videoImportId) + + expect(videoImport.state.id).to.equal(VideoImportState.REJECTED) + expect(videoImport.state.label).to.equal('Rejected') + } + }) + + it('Should run filter:api.video.post-import-torrent.accept.result', async function () { + this.timeout(60000) + + let videoImportId: number + + { + const attributes = { + name: 'title with bad word', + privacy: VideoPrivacy.PUBLIC, + channelId: servers[0].store.channel.id, + torrentfile: 'video-720p.torrent' as any + } + const body = await servers[0].imports.importVideo({ attributes }) + videoImportId = body.id + } + + await waitJobs(servers) + + { + const { data: videoImports } = await servers[0].imports.getMyVideoImports() + + const videoImport = videoImports.find(i => i.id === videoImportId) + + expect(videoImport.state.id).to.equal(VideoImportState.REJECTED) + expect(videoImport.state.label).to.equal('Rejected') + } + }) + }) + + describe('Video comments accept', function () { + + it('Should run filter:api.video-thread.create.accept.result', async function () { + await servers[0].comments.createThread({ + videoId: videoUUID, + text: 'comment with bad word', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + }) + + it('Should run filter:api.video-comment-reply.create.accept.result', async function () { + const created = await servers[0].comments.createThread({ videoId: videoUUID, text: 'thread' }) + threadId = created.id + + await servers[0].comments.addReply({ + videoId: videoUUID, + toCommentId: threadId, + text: 'comment with bad word', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + await servers[0].comments.addReply({ + videoId: videoUUID, + toCommentId: threadId, + text: 'comment with good word', + expectedStatus: HttpStatusCode.OK_200 + }) + }) + + it('Should run filter:activity-pub.remote-video-comment.create.accept.result on a thread creation', async function () { + this.timeout(30000) + + await servers[1].comments.createThread({ videoId: videoUUID, text: 'comment with bad word' }) + + await waitJobs(servers) + + { + const thread = await servers[0].comments.listThreads({ videoId: videoUUID }) + expect(thread.data).to.have.lengthOf(1) + expect(thread.data[0].text).to.not.include(' bad ') + } + + { + const thread = await servers[1].comments.listThreads({ videoId: videoUUID }) + expect(thread.data).to.have.lengthOf(2) + } + }) + + it('Should run filter:activity-pub.remote-video-comment.create.accept.result on a reply creation', async function () { + this.timeout(30000) + + const { data } = await servers[1].comments.listThreads({ videoId: videoUUID }) + const threadIdServer2 = data.find(t => t.text === 'thread').id + + await servers[1].comments.addReply({ + videoId: videoUUID, + toCommentId: threadIdServer2, + text: 'comment with bad word' + }) + + await waitJobs(servers) + + { + const tree = await servers[0].comments.getThread({ videoId: videoUUID, threadId }) + expect(tree.children).to.have.lengthOf(1) + expect(tree.children[0].comment.text).to.not.include(' bad ') + } + + { + const tree = await servers[1].comments.getThread({ videoId: videoUUID, threadId: threadIdServer2 }) + expect(tree.children).to.have.lengthOf(2) + } + }) + }) + + describe('Video comments', function () { + + it('Should run filter:api.video-threads.list.params', async function () { + const { data } = await servers[0].comments.listThreads({ videoId: videoUUID, start: 0, count: 0 }) + + // our plugin do +1 to the count parameter + expect(data).to.have.lengthOf(1) + }) + + it('Should run filter:api.video-threads.list.result', async function () { + const { total } = await servers[0].comments.listThreads({ videoId: videoUUID, start: 0, count: 0 }) + + // Plugin do +1 to the total result + expect(total).to.equal(2) + }) + + it('Should run filter:api.video-thread-comments.list.params') + + it('Should run filter:api.video-thread-comments.list.result', async function () { + const thread = await servers[0].comments.getThread({ videoId: videoUUID, threadId }) + + expect(thread.comment.text.endsWith(' <3')).to.be.true + }) + + it('Should run filter:api.overviews.videos.list.{params,result}', async function () { + await servers[0].overviews.getVideos({ page: 1 }) + + // 3 because we get 3 samples per page + await servers[0].servers.waitUntilLog('Run hook filter:api.overviews.videos.list.params', 3) + await servers[0].servers.waitUntilLog('Run hook filter:api.overviews.videos.list.result', 3) + }) + }) + + describe('filter:video.auto-blacklist.result', function () { + + async function checkIsBlacklisted (id: number | string, value: boolean) { + const video = await servers[0].videos.getWithToken({ id }) + expect(video.blacklisted).to.equal(value) + } + + it('Should blacklist on upload', async function () { + const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video please blacklist me' } }) + await checkIsBlacklisted(uuid, true) + }) + + it('Should blacklist on import', async function () { + this.timeout(15000) + + const attributes = { + name: 'video please blacklist me', + targetUrl: FIXTURE_URLS.goodVideo, + channelId: servers[0].store.channel.id + } + const body = await servers[0].imports.importVideo({ attributes }) + await checkIsBlacklisted(body.video.uuid, true) + }) + + it('Should blacklist on update', async function () { + const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' } }) + await checkIsBlacklisted(uuid, false) + + await servers[0].videos.update({ id: uuid, attributes: { name: 'please blacklist me' } }) + await checkIsBlacklisted(uuid, true) + }) + + it('Should blacklist on remote upload', async function () { + this.timeout(120000) + + const { uuid } = await servers[1].videos.upload({ attributes: { name: 'remote please blacklist me' } }) + await waitJobs(servers) + + await checkIsBlacklisted(uuid, true) + }) + + it('Should blacklist on remote update', async function () { + this.timeout(120000) + + const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video' } }) + await waitJobs(servers) + + await checkIsBlacklisted(uuid, false) + + await servers[1].videos.update({ id: uuid, attributes: { name: 'please blacklist me' } }) + await waitJobs(servers) + + await checkIsBlacklisted(uuid, true) + }) + }) + + describe('Should run filter:api.user.signup.allowed.result', function () { + + before(async function () { + await servers[0].config.updateExistingSubConfig({ newConfig: { signup: { requiresApproval: false } } }) + }) + + it('Should run on config endpoint', async function () { + const body = await servers[0].config.getConfig() + expect(body.signup.allowed).to.be.true + }) + + it('Should allow a signup', async function () { + await servers[0].registrations.register({ username: 'john1' }) + }) + + it('Should not allow a signup', async function () { + const res = await servers[0].registrations.register({ + username: 'jma 1', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + + expect(res.body.error).to.equal('No jma 1') + }) + }) + + describe('Should run filter:api.user.request-signup.allowed.result', function () { + + before(async function () { + await servers[0].config.updateExistingSubConfig({ newConfig: { signup: { requiresApproval: true } } }) + }) + + it('Should run on config endpoint', async function () { + const body = await servers[0].config.getConfig() + expect(body.signup.allowed).to.be.true + }) + + it('Should allow a signup request', async function () { + await servers[0].registrations.requestRegistration({ username: 'john2', registrationReason: 'tt' }) + }) + + it('Should not allow a signup request', async function () { + const body = await servers[0].registrations.requestRegistration({ + username: 'jma 2', + registrationReason: 'tt', + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + + expect((body as unknown as PeerTubeProblemDocument).error).to.equal('No jma 2') + }) + }) + + describe('Download hooks', function () { + const downloadVideos: VideoDetails[] = [] + let downloadVideo2Token: string + + before(async function () { + this.timeout(120000) + + await servers[0].config.updateCustomSubConfig({ + newConfig: { + transcoding: { + webVideos: { + enabled: true + }, + hls: { + enabled: true + } + } + } + }) + + const uuids: string[] = [] + + for (const name of [ 'bad torrent', 'bad file', 'bad playlist file' ]) { + const uuid = (await servers[0].videos.quickUpload({ name })).uuid + uuids.push(uuid) + } + + await waitJobs(servers) + + for (const uuid of uuids) { + downloadVideos.push(await servers[0].videos.get({ id: uuid })) + } + + downloadVideo2Token = await servers[0].videoToken.getVideoFileToken({ videoId: downloadVideos[2].uuid }) + }) + + it('Should run filter:api.download.torrent.allowed.result', async function () { + const res = await makeRawRequest({ url: downloadVideos[0].files[0].torrentDownloadUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + expect(res.body.error).to.equal('Liu Bei') + + await makeRawRequest({ url: downloadVideos[1].files[0].torrentDownloadUrl, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url: downloadVideos[2].files[0].torrentDownloadUrl, expectedStatus: HttpStatusCode.OK_200 }) + }) + + it('Should run filter:api.download.video.allowed.result', async function () { + { + const refused = downloadVideos[1].files[0].fileDownloadUrl + const allowed = [ + downloadVideos[0].files[0].fileDownloadUrl, + downloadVideos[2].files[0].fileDownloadUrl + ] + + const res = await makeRawRequest({ url: refused, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + expect(res.body.error).to.equal('Cao Cao') + + for (const url of allowed) { + await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) + } + } + + { + const refused = downloadVideos[2].streamingPlaylists[0].files[0].fileDownloadUrl + + const allowed = [ + downloadVideos[2].files[0].fileDownloadUrl, + downloadVideos[0].streamingPlaylists[0].files[0].fileDownloadUrl, + downloadVideos[1].streamingPlaylists[0].files[0].fileDownloadUrl + ] + + // Only streaming playlist is refuse + const res = await makeRawRequest({ url: refused, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + expect(res.body.error).to.equal('Sun Jian') + + // But not we there is a user in res + await makeRawRequest({ url: refused, token: servers[0].accessToken, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url: refused, query: { videoFileToken: downloadVideo2Token }, expectedStatus: HttpStatusCode.OK_200 }) + + // Other files work + for (const url of allowed) { + await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) + } + } + }) + }) + + describe('Embed filters', function () { + const embedVideos: VideoDetails[] = [] + const embedPlaylists: VideoPlaylist[] = [] + + before(async function () { + this.timeout(60000) + + await servers[0].config.disableTranscoding() + + for (const name of [ 'bad embed', 'good embed' ]) { + { + const uuid = (await servers[0].videos.quickUpload({ name })).uuid + embedVideos.push(await servers[0].videos.get({ id: uuid })) + } + + { + const attributes = { displayName: name, videoChannelId: servers[0].store.channel.id, privacy: VideoPlaylistPrivacy.PUBLIC } + const { id } = await servers[0].playlists.create({ attributes }) + + const playlist = await servers[0].playlists.get({ playlistId: id }) + embedPlaylists.push(playlist) + } + } + }) + + it('Should run filter:html.embed.video.allowed.result', async function () { + const res = await makeGetRequest({ url: servers[0].url, path: embedVideos[0].embedPath, expectedStatus: HttpStatusCode.OK_200 }) + expect(res.text).to.equal('Lu Bu') + }) + + it('Should run filter:html.embed.video-playlist.allowed.result', async function () { + const res = await makeGetRequest({ url: servers[0].url, path: embedPlaylists[0].embedPath, expectedStatus: HttpStatusCode.OK_200 }) + expect(res.text).to.equal('Diao Chan') + }) + }) + + describe('Client HTML filters', function () { + let videoUUID: string + + before(async function () { + this.timeout(60000) + + const { uuid } = await servers[0].videos.quickUpload({ name: 'html video' }) + videoUUID = uuid + }) + + it('Should run filter:html.client.json-ld.result', async function () { + const res = await makeGetRequest({ url: servers[0].url, path: '/w/' + videoUUID, expectedStatus: HttpStatusCode.OK_200 }) + expect(res.text).to.contain('"recordedAt":"http://example.com/recordedAt"') + }) + + it('Should not run filter:html.client.json-ld.result with an account', async function () { + const res = await makeGetRequest({ url: servers[0].url, path: '/a/root', expectedStatus: HttpStatusCode.OK_200 }) + expect(res.text).not.to.contain('"recordedAt":"http://example.com/recordedAt"') + }) + }) + + describe('Search filters', function () { + + before(async function () { + await servers[0].config.updateCustomSubConfig({ + newConfig: { + search: { + searchIndex: { + enabled: true, + isDefaultSearch: false, + disableLocalSearch: false + } + } + } + }) + }) + + it('Should run filter:api.search.videos.local.list.{params,result}', async function () { + await servers[0].search.advancedVideoSearch({ + search: { + search: 'Sun Quan' + } + }) + + await servers[0].servers.waitUntilLog('Run hook filter:api.search.videos.local.list.params', 1) + await servers[0].servers.waitUntilLog('Run hook filter:api.search.videos.local.list.result', 1) + }) + + it('Should run filter:api.search.videos.index.list.{params,result}', async function () { + await servers[0].search.advancedVideoSearch({ + search: { + search: 'Sun Quan', + searchTarget: 'search-index' + } + }) + + await servers[0].servers.waitUntilLog('Run hook filter:api.search.videos.local.list.params', 1) + await servers[0].servers.waitUntilLog('Run hook filter:api.search.videos.local.list.result', 1) + await servers[0].servers.waitUntilLog('Run hook filter:api.search.videos.index.list.params', 1) + await servers[0].servers.waitUntilLog('Run hook filter:api.search.videos.index.list.result', 1) + }) + + it('Should run filter:api.search.video-channels.local.list.{params,result}', async function () { + await servers[0].search.advancedChannelSearch({ + search: { + search: 'Sun Ce' + } + }) + + await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-channels.local.list.params', 1) + await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-channels.local.list.result', 1) + }) + + it('Should run filter:api.search.video-channels.index.list.{params,result}', async function () { + await servers[0].search.advancedChannelSearch({ + search: { + search: 'Sun Ce', + searchTarget: 'search-index' + } + }) + + await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-channels.local.list.params', 1) + await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-channels.local.list.result', 1) + await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-channels.index.list.params', 1) + await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-channels.index.list.result', 1) + }) + + it('Should run filter:api.search.video-playlists.local.list.{params,result}', async function () { + await servers[0].search.advancedPlaylistSearch({ + search: { + search: 'Sun Jian' + } + }) + + await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-playlists.local.list.params', 1) + await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-playlists.local.list.result', 1) + }) + + it('Should run filter:api.search.video-playlists.index.list.{params,result}', async function () { + await servers[0].search.advancedPlaylistSearch({ + search: { + search: 'Sun Jian', + searchTarget: 'search-index' + } + }) + + await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-playlists.local.list.params', 1) + await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-playlists.local.list.result', 1) + await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-playlists.index.list.params', 1) + await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-playlists.index.list.result', 1) + }) + }) + + describe('Upload/import/live attributes filters', function () { + + before(async function () { + await servers[0].config.enableLive({ transcoding: false, allowReplay: false }) + await servers[0].config.enableImports() + await servers[0].config.disableTranscoding() + }) + + it('Should run filter:api.video.upload.video-attribute.result', async function () { + for (const mode of [ 'legacy' as 'legacy', 'resumable' as 'resumable' ]) { + const { id } = await servers[0].videos.upload({ attributes: { name: 'video', description: 'upload' }, mode }) + + const video = await servers[0].videos.get({ id }) + expect(video.description).to.equal('upload - filter:api.video.upload.video-attribute.result') + } + }) + + it('Should run filter:api.video.import-url.video-attribute.result', async function () { + const attributes = { + name: 'video', + description: 'import url', + channelId: servers[0].store.channel.id, + targetUrl: FIXTURE_URLS.goodVideo, + privacy: VideoPrivacy.PUBLIC + } + const { video: { id } } = await servers[0].imports.importVideo({ attributes }) + + const video = await servers[0].videos.get({ id }) + expect(video.description).to.equal('import url - filter:api.video.import-url.video-attribute.result') + }) + + it('Should run filter:api.video.import-torrent.video-attribute.result', async function () { + const attributes = { + name: 'video', + description: 'import torrent', + channelId: servers[0].store.channel.id, + magnetUri: FIXTURE_URLS.magnet, + privacy: VideoPrivacy.PUBLIC + } + const { video: { id } } = await servers[0].imports.importVideo({ attributes }) + + const video = await servers[0].videos.get({ id }) + expect(video.description).to.equal('import torrent - filter:api.video.import-torrent.video-attribute.result') + }) + + it('Should run filter:api.video.live.video-attribute.result', async function () { + const fields = { + name: 'live', + description: 'live', + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC + } + const { id } = await servers[0].live.create({ fields }) + + const video = await servers[0].videos.get({ id }) + expect(video.description).to.equal('live - filter:api.video.live.video-attribute.result') + }) + }) + + describe('Stats filters', function () { + + it('Should run filter:api.server.stats.get.result', async function () { + const data = await servers[0].stats.get() + + expect((data as any).customStats).to.equal(14) + }) + + }) + + describe('Job queue filters', function () { + let videoUUID: string + + before(async function () { + this.timeout(120_000) + + await servers[0].config.enableMinimumTranscoding() + const { uuid } = await servers[0].videos.quickUpload({ name: 'studio' }) + + const video = await servers[0].videos.get({ id: uuid }) + expect(video.duration).at.least(2) + videoUUID = video.uuid + + await waitJobs(servers) + + await servers[0].config.enableStudio() + }) + + it('Should run filter:job-queue.process.params', async function () { + this.timeout(120_000) + + await servers[0].videoStudio.createEditionTasks({ + videoId: videoUUID, + tasks: [ + { + name: 'add-intro', + options: { + file: 'video_very_short_240p.mp4' + } + } + ] + }) + + await waitJobs(servers) + + await servers[0].servers.waitUntilLog('Run hook filter:job-queue.process.params', 1, false) + + const video = await servers[0].videos.get({ id: videoUUID }) + expect(video.duration).at.most(2) + }) + + it('Should run filter:job-queue.process.result', async function () { + await servers[0].servers.waitUntilLog('Run hook filter:job-queue.process.result', 1, false) + }) + }) + + describe('Transcoding filters', async function () { + + it('Should run filter:transcoding.auto.resolutions-to-transcode.result', async function () { + const { uuid } = await servers[0].videos.quickUpload({ name: 'transcode-filter' }) + + await waitJobs(servers) + + const video = await servers[0].videos.get({ id: uuid }) + expect(video.files).to.have.lengthOf(2) + expect(video.files.find(f => f.resolution.id === 100 as any)).to.exist + }) + }) + + describe('Video channel filters', async function () { + + it('Should run filter:api.video-channels.list.params', async function () { + const { data } = await servers[0].channels.list({ start: 0, count: 0 }) + + // plugin do +1 to the count parameter + expect(data).to.have.lengthOf(1) + }) + + it('Should run filter:api.video-channels.list.result', async function () { + const { total } = await servers[0].channels.list({ start: 0, count: 1 }) + + // plugin do +1 to the total parameter + expect(total).to.equal(4) + }) + + it('Should run filter:api.video-channel.get.result', async function () { + const channel = await servers[0].channels.get({ channelName: 'root_channel' }) + expect(channel.displayName).to.equal('Main root channel <3') + }) + }) + + describe('Activity Pub', function () { + + it('Should run filter:activity-pub.activity.context.build.result', async function () { + const { body } = await makeActivityPubGetRequest(servers[0].url, '/w/' + videoUUID) + expect(body.type).to.equal('Video') + + expect(body['@context'].some(c => { + return typeof c === 'object' && c.recordedAt === 'https://schema.org/recordedAt' + })).to.be.true + }) + + it('Should run filter:activity-pub.video.json-ld.build.result', async function () { + const { body } = await makeActivityPubGetRequest(servers[0].url, '/w/' + videoUUID) + expect(body.name).to.equal('default video 0') + expect(body.videoName).to.equal('default video 0') + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/plugins/html-injection.ts b/packages/tests/src/plugins/html-injection.ts new file mode 100644 index 000000000..269a45b98 --- /dev/null +++ b/packages/tests/src/plugins/html-injection.ts @@ -0,0 +1,73 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { + cleanupTests, + createSingleServer, + makeHTMLRequest, + PeerTubeServer, + PluginsCommand, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test plugins HTML injection', function () { + let server: PeerTubeServer = null + let command: PluginsCommand + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + + command = server.plugins + }) + + it('Should not inject global css file in HTML', async function () { + { + const text = await command.getCSS() + expect(text).to.be.empty + } + + for (const path of [ '/', '/videos/embed/1', '/video-playlists/embed/1' ]) { + const res = await makeHTMLRequest(server.url, path) + expect(res.text).to.not.include('link rel="stylesheet" href="/plugins/global.css') + } + }) + + it('Should install a plugin and a theme', async function () { + this.timeout(30000) + + await command.install({ npmName: 'peertube-plugin-hello-world' }) + }) + + it('Should have the correct global css', async function () { + { + const text = await command.getCSS() + expect(text).to.contain('background-color: red') + } + + for (const path of [ '/', '/videos/embed/1', '/video-playlists/embed/1' ]) { + const res = await makeHTMLRequest(server.url, path) + expect(res.text).to.include('link rel="stylesheet" href="/plugins/global.css') + } + }) + + it('Should have an empty global css on uninstall', async function () { + await command.uninstall({ npmName: 'peertube-plugin-hello-world' }) + + { + const text = await command.getCSS() + expect(text).to.be.empty + } + + for (const path of [ '/', '/videos/embed/1', '/video-playlists/embed/1' ]) { + const res = await makeHTMLRequest(server.url, path) + expect(res.text).to.not.include('link rel="stylesheet" href="/plugins/global.css') + } + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/plugins/id-and-pass-auth.ts b/packages/tests/src/plugins/id-and-pass-auth.ts new file mode 100644 index 000000000..a332f0eec --- /dev/null +++ b/packages/tests/src/plugins/id-and-pass-auth.ts @@ -0,0 +1,248 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { HttpStatusCode, UserRole } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + PeerTubeServer, + PluginsCommand, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test id and pass auth plugins', function () { + let server: PeerTubeServer + + let crashAccessToken: string + let crashRefreshToken: string + + let lagunaAccessToken: string + let lagunaRefreshToken: string + let lagunaId: number + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + + for (const suffix of [ 'one', 'two', 'three' ]) { + await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-id-pass-auth-' + suffix) }) + } + }) + + it('Should display the correct configuration', async function () { + const config = await server.config.getConfig() + + const auths = config.plugin.registeredIdAndPassAuths + expect(auths).to.have.lengthOf(8) + + const crashAuth = auths.find(a => a.authName === 'crash-auth') + expect(crashAuth).to.exist + expect(crashAuth.npmName).to.equal('peertube-plugin-test-id-pass-auth-one') + expect(crashAuth.weight).to.equal(50) + }) + + it('Should not login', async function () { + await server.login.login({ user: { username: 'toto', password: 'password' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should login Spyro, create the user and use the token', async function () { + const accessToken = await server.login.getAccessToken({ username: 'spyro', password: 'spyro password' }) + + const body = await server.users.getMyInfo({ token: accessToken }) + + expect(body.username).to.equal('spyro') + expect(body.account.displayName).to.equal('Spyro the Dragon') + expect(body.role.id).to.equal(UserRole.USER) + }) + + it('Should login Crash, create the user and use the token', async function () { + { + const body = await server.login.login({ user: { username: 'crash', password: 'crash password' } }) + crashAccessToken = body.access_token + crashRefreshToken = body.refresh_token + } + + { + const body = await server.users.getMyInfo({ token: crashAccessToken }) + + expect(body.username).to.equal('crash') + expect(body.account.displayName).to.equal('Crash Bandicoot') + expect(body.role.id).to.equal(UserRole.MODERATOR) + } + }) + + it('Should login the first Laguna, create the user and use the token', async function () { + { + const body = await server.login.login({ user: { username: 'laguna', password: 'laguna password' } }) + lagunaAccessToken = body.access_token + lagunaRefreshToken = body.refresh_token + } + + { + const body = await server.users.getMyInfo({ token: lagunaAccessToken }) + + expect(body.username).to.equal('laguna') + expect(body.account.displayName).to.equal('Laguna Loire') + expect(body.role.id).to.equal(UserRole.USER) + + lagunaId = body.id + } + }) + + it('Should refresh crash token, but not laguna token', async function () { + { + const resRefresh = await server.login.refreshToken({ refreshToken: crashRefreshToken }) + crashAccessToken = resRefresh.body.access_token + crashRefreshToken = resRefresh.body.refresh_token + + const body = await server.users.getMyInfo({ token: crashAccessToken }) + expect(body.username).to.equal('crash') + } + + { + await server.login.refreshToken({ refreshToken: lagunaRefreshToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + } + }) + + it('Should update Crash profile', async function () { + await server.users.updateMe({ + token: crashAccessToken, + displayName: 'Beautiful Crash', + description: 'Mutant eastern barred bandicoot' + }) + + const body = await server.users.getMyInfo({ token: crashAccessToken }) + + expect(body.account.displayName).to.equal('Beautiful Crash') + expect(body.account.description).to.equal('Mutant eastern barred bandicoot') + }) + + it('Should logout Crash', async function () { + await server.login.logout({ token: crashAccessToken }) + }) + + it('Should have logged out Crash', async function () { + await server.servers.waitUntilLog('On logout for auth 1 - 2') + + await server.users.getMyInfo({ token: crashAccessToken, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should login Crash and keep the old existing profile', async function () { + crashAccessToken = await server.login.getAccessToken({ username: 'crash', password: 'crash password' }) + + const body = await server.users.getMyInfo({ token: crashAccessToken }) + + expect(body.username).to.equal('crash') + expect(body.account.displayName).to.equal('Beautiful Crash') + expect(body.account.description).to.equal('Mutant eastern barred bandicoot') + expect(body.role.id).to.equal(UserRole.MODERATOR) + }) + + it('Should login Laguna and update the profile', async function () { + { + await server.users.update({ userId: lagunaId, videoQuota: 43000, videoQuotaDaily: 43100 }) + await server.users.updateMe({ token: lagunaAccessToken, displayName: 'laguna updated' }) + + const body = await server.users.getMyInfo({ token: lagunaAccessToken }) + expect(body.username).to.equal('laguna') + expect(body.account.displayName).to.equal('laguna updated') + expect(body.videoQuota).to.equal(43000) + expect(body.videoQuotaDaily).to.equal(43100) + } + + { + const body = await server.login.login({ user: { username: 'laguna', password: 'laguna password' } }) + lagunaAccessToken = body.access_token + lagunaRefreshToken = body.refresh_token + } + + { + const body = await server.users.getMyInfo({ token: lagunaAccessToken }) + expect(body.username).to.equal('laguna') + expect(body.account.displayName).to.equal('Laguna Loire') + expect(body.videoQuota).to.equal(42000) + expect(body.videoQuotaDaily).to.equal(43100) + } + }) + + it('Should reject token of laguna by the plugin hook', async function () { + await wait(5000) + + await server.users.getMyInfo({ token: lagunaAccessToken, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) + }) + + it('Should reject an invalid username, email, role or display name', async function () { + const command = server.login + + await command.login({ user: { username: 'ward', password: 'ward password' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await server.servers.waitUntilLog('valid username') + + await command.login({ user: { username: 'kiros', password: 'kiros password' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await server.servers.waitUntilLog('valid displayName') + + await command.login({ user: { username: 'raine', password: 'raine password' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await server.servers.waitUntilLog('valid role') + + await command.login({ user: { username: 'ellone', password: 'elonne password' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await server.servers.waitUntilLog('valid email') + }) + + it('Should unregister spyro-auth and do not login existing Spyro', async function () { + await server.plugins.updateSettings({ + npmName: 'peertube-plugin-test-id-pass-auth-one', + settings: { disableSpyro: true } + }) + + const command = server.login + await command.login({ user: { username: 'spyro', password: 'spyro password' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + await command.login({ user: { username: 'spyro', password: 'fake' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should have disabled this auth', async function () { + const config = await server.config.getConfig() + + const auths = config.plugin.registeredIdAndPassAuths + expect(auths).to.have.lengthOf(7) + + const spyroAuth = auths.find(a => a.authName === 'spyro-auth') + expect(spyroAuth).to.not.exist + }) + + it('Should uninstall the plugin one and do not login existing Crash', async function () { + await server.plugins.uninstall({ npmName: 'peertube-plugin-test-id-pass-auth-one' }) + + await server.login.login({ + user: { username: 'crash', password: 'crash password' }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + }) + + it('Should display the correct configuration', async function () { + const config = await server.config.getConfig() + + const auths = config.plugin.registeredIdAndPassAuths + expect(auths).to.have.lengthOf(6) + + const crashAuth = auths.find(a => a.authName === 'crash-auth') + expect(crashAuth).to.not.exist + }) + + it('Should display plugin auth information in users list', async function () { + const { data } = await server.users.list() + + const root = data.find(u => u.username === 'root') + const crash = data.find(u => u.username === 'crash') + const laguna = data.find(u => u.username === 'laguna') + + expect(root.pluginAuth).to.be.null + expect(crash.pluginAuth).to.equal('peertube-plugin-test-id-pass-auth-one') + expect(laguna.pluginAuth).to.equal('peertube-plugin-test-id-pass-auth-two') + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/server/tests/plugins/index.ts b/packages/tests/src/plugins/index.ts similarity index 100% rename from server/tests/plugins/index.ts rename to packages/tests/src/plugins/index.ts diff --git a/packages/tests/src/plugins/plugin-helpers.ts b/packages/tests/src/plugins/plugin-helpers.ts new file mode 100644 index 000000000..d2bd8596e --- /dev/null +++ b/packages/tests/src/plugins/plugin-helpers.ts @@ -0,0 +1,383 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { pathExists } from 'fs-extra/esm' +import { HttpStatusCode, ThumbnailType } from '@peertube/peertube-models' +import { + cleanupTests, + createMultipleServers, + doubleFollow, + makeGetRequest, + makePostBodyRequest, + makeRawRequest, + PeerTubeServer, + PluginsCommand, + setAccessTokensToServers, + waitJobs +} from '@peertube/peertube-server-commands' +import { checkVideoFilesWereRemoved } from '@tests/shared/videos.js' + +function postCommand (server: PeerTubeServer, command: string, bodyArg?: object) { + const body = { command } + if (bodyArg) Object.assign(body, bodyArg) + + return makePostBodyRequest({ + url: server.url, + path: '/plugins/test-four/router/commander', + fields: body, + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) +} + +describe('Test plugin helpers', function () { + let servers: PeerTubeServer[] + + before(async function () { + this.timeout(60000) + + servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + + await doubleFollow(servers[0], servers[1]) + + await servers[0].plugins.install({ path: PluginsCommand.getPluginTestPath('-four') }) + }) + + describe('Logger', function () { + + it('Should have logged things', async function () { + await servers[0].servers.waitUntilLog(servers[0].host + ' peertube-plugin-test-four', 1, false) + await servers[0].servers.waitUntilLog('Hello world from plugin four', 1) + }) + }) + + describe('Database', function () { + + it('Should have made a query', async function () { + await servers[0].servers.waitUntilLog(`root email is admin${servers[0].internalServerNumber}@example.com`) + }) + }) + + describe('Config', function () { + + it('Should have the correct webserver url', async function () { + await servers[0].servers.waitUntilLog(`server url is ${servers[0].url}`) + }) + + it('Should have the correct listening config', async function () { + const res = await makeGetRequest({ + url: servers[0].url, + path: '/plugins/test-four/router/server-listening-config', + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(res.body.config).to.exist + expect(res.body.config.hostname).to.equal('::') + expect(res.body.config.port).to.equal(servers[0].port) + }) + + it('Should have the correct config', async function () { + const res = await makeGetRequest({ + url: servers[0].url, + path: '/plugins/test-four/router/server-config', + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(res.body.serverConfig).to.exist + expect(res.body.serverConfig.instance.name).to.equal('PeerTube') + }) + }) + + describe('Server', function () { + + it('Should get the server actor', async function () { + await servers[0].servers.waitUntilLog('server actor name is peertube') + }) + }) + + describe('Socket', function () { + + it('Should sendNotification without any exceptions', async () => { + const user = await servers[0].users.create({ username: 'notis_redding', password: 'secret1234?' }) + await makePostBodyRequest({ + url: servers[0].url, + path: '/plugins/test-four/router/send-notification', + fields: { + userId: user.id + }, + expectedStatus: HttpStatusCode.CREATED_201 + }) + }) + + it('Should sendVideoLiveNewState without any exceptions', async () => { + const res = await servers[0].videos.quickUpload({ name: 'video server 1' }) + + await makePostBodyRequest({ + url: servers[0].url, + path: '/plugins/test-four/router/send-video-live-new-state/' + res.uuid, + expectedStatus: HttpStatusCode.CREATED_201 + }) + + await servers[0].videos.remove({ id: res.uuid }) + }) + }) + + describe('Plugin', function () { + + it('Should get the base static route', async function () { + const res = await makeGetRequest({ + url: servers[0].url, + path: '/plugins/test-four/router/static-route', + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(res.body.staticRoute).to.equal('/plugins/test-four/0.0.1/static/') + }) + + it('Should get the base static route', async function () { + const baseRouter = '/plugins/test-four/0.0.1/router/' + + const res = await makeGetRequest({ + url: servers[0].url, + path: baseRouter + 'router-route', + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(res.body.routerRoute).to.equal(baseRouter) + }) + }) + + describe('User', function () { + let rootId: number + + it('Should not get a user if not authenticated', async function () { + await makeGetRequest({ + url: servers[0].url, + path: '/plugins/test-four/router/user', + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should get a user if authenticated', async function () { + const res = await makeGetRequest({ + url: servers[0].url, + token: servers[0].accessToken, + path: '/plugins/test-four/router/user', + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(res.body.username).to.equal('root') + expect(res.body.displayName).to.equal('root') + expect(res.body.isAdmin).to.be.true + expect(res.body.isModerator).to.be.false + expect(res.body.isUser).to.be.false + + rootId = res.body.id + }) + + it('Should load a user by id', async function () { + { + const res = await makeGetRequest({ + url: servers[0].url, + path: '/plugins/test-four/router/user/' + rootId, + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(res.body.username).to.equal('root') + } + + { + await makeGetRequest({ + url: servers[0].url, + path: '/plugins/test-four/router/user/42', + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + } + }) + }) + + describe('Moderation', function () { + let videoUUIDServer1: string + + before(async function () { + this.timeout(60000) + + { + const res = await servers[0].videos.quickUpload({ name: 'video server 1' }) + videoUUIDServer1 = res.uuid + } + + { + await servers[1].videos.quickUpload({ name: 'video server 2' }) + } + + await waitJobs(servers) + + const { data } = await servers[0].videos.list() + + expect(data).to.have.lengthOf(2) + }) + + it('Should mute server 2', async function () { + await postCommand(servers[0], 'blockServer', { hostToBlock: servers[1].host }) + + const { data } = await servers[0].videos.list() + + expect(data).to.have.lengthOf(1) + expect(data[0].name).to.equal('video server 1') + }) + + it('Should unmute server 2', async function () { + await postCommand(servers[0], 'unblockServer', { hostToUnblock: servers[1].host }) + + const { data } = await servers[0].videos.list() + + expect(data).to.have.lengthOf(2) + }) + + it('Should mute account of server 2', async function () { + await postCommand(servers[0], 'blockAccount', { handleToBlock: `root@${servers[1].host}` }) + + const { data } = await servers[0].videos.list() + + expect(data).to.have.lengthOf(1) + expect(data[0].name).to.equal('video server 1') + }) + + it('Should unmute account of server 2', async function () { + await postCommand(servers[0], 'unblockAccount', { handleToUnblock: `root@${servers[1].host}` }) + + const { data } = await servers[0].videos.list() + + expect(data).to.have.lengthOf(2) + }) + + it('Should blacklist video', async function () { + await postCommand(servers[0], 'blacklist', { videoUUID: videoUUIDServer1, unfederate: true }) + + await waitJobs(servers) + + for (const server of servers) { + const { data } = await server.videos.list() + + expect(data).to.have.lengthOf(1) + expect(data[0].name).to.equal('video server 2') + } + }) + + it('Should unblacklist video', async function () { + await postCommand(servers[0], 'unblacklist', { videoUUID: videoUUIDServer1 }) + + await waitJobs(servers) + + for (const server of servers) { + const { data } = await server.videos.list() + + expect(data).to.have.lengthOf(2) + } + }) + }) + + describe('Videos', function () { + let videoUUID: string + let videoPath: string + + before(async function () { + this.timeout(240000) + + await servers[0].config.enableTranscoding() + + const res = await servers[0].videos.quickUpload({ name: 'video1' }) + videoUUID = res.uuid + + await waitJobs(servers) + }) + + it('Should get video files', async function () { + const { body } = await makeGetRequest({ + url: servers[0].url, + path: '/plugins/test-four/router/video-files/' + videoUUID, + expectedStatus: HttpStatusCode.OK_200 + }) + + // Video files check + { + expect(body.webVideo.videoFiles).to.be.an('array') + expect(body.hls.videoFiles).to.be.an('array') + + for (const resolution of [ 144, 240, 360, 480, 720 ]) { + for (const files of [ body.webVideo.videoFiles, body.hls.videoFiles ]) { + const file = files.find(f => f.resolution === resolution) + expect(file).to.exist + + expect(file.size).to.be.a('number') + expect(file.fps).to.equal(25) + + expect(await pathExists(file.path)).to.be.true + await makeRawRequest({ url: file.url, expectedStatus: HttpStatusCode.OK_200 }) + } + } + + videoPath = body.webVideo.videoFiles[0].path + } + + // Thumbnails check + { + expect(body.thumbnails).to.be.an('array') + + const miniature = body.thumbnails.find(t => t.type === ThumbnailType.MINIATURE) + expect(miniature).to.exist + expect(await pathExists(miniature.path)).to.be.true + await makeRawRequest({ url: miniature.url, expectedStatus: HttpStatusCode.OK_200 }) + + const preview = body.thumbnails.find(t => t.type === ThumbnailType.PREVIEW) + expect(preview).to.exist + expect(await pathExists(preview.path)).to.be.true + await makeRawRequest({ url: preview.url, expectedStatus: HttpStatusCode.OK_200 }) + } + }) + + it('Should probe a file', async function () { + const { body } = await makeGetRequest({ + url: servers[0].url, + path: '/plugins/test-four/router/ffprobe', + query: { + path: videoPath + }, + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(body.streams).to.be.an('array') + expect(body.streams).to.have.lengthOf(2) + }) + + it('Should remove a video after a view', async function () { + this.timeout(40000) + + // Should not throw -> video exists + const video = await servers[0].videos.get({ id: videoUUID }) + // Should delete the video + await servers[0].views.simulateView({ id: videoUUID }) + + await servers[0].servers.waitUntilLog('Video deleted by plugin four.') + + try { + // Should throw because the video should have been deleted + await servers[0].videos.get({ id: videoUUID }) + throw new Error('Video exists') + } catch (err) { + if (err.message.includes('exists')) throw err + } + + await checkVideoFilesWereRemoved({ server: servers[0], video }) + }) + + it('Should have fetched the video by URL', async function () { + await servers[0].servers.waitUntilLog(`video from DB uuid is ${videoUUID}`) + }) + }) + + after(async function () { + await cleanupTests(servers) + }) +}) diff --git a/packages/tests/src/plugins/plugin-router.ts b/packages/tests/src/plugins/plugin-router.ts new file mode 100644 index 000000000..6f3571c05 --- /dev/null +++ b/packages/tests/src/plugins/plugin-router.ts @@ -0,0 +1,105 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { + cleanupTests, + createSingleServer, + makeGetRequest, + makePostBodyRequest, + PeerTubeServer, + PluginsCommand, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' +import { HttpStatusCode } from '@peertube/peertube-models' + +describe('Test plugin helpers', function () { + let server: PeerTubeServer + const basePaths = [ + '/plugins/test-five/router/', + '/plugins/test-five/0.0.1/router/' + ] + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + + await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-five') }) + }) + + it('Should answer "pong"', async function () { + for (const path of basePaths) { + const res = await makeGetRequest({ + url: server.url, + path: path + 'ping', + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(res.body.message).to.equal('pong') + } + }) + + it('Should check if authenticated', async function () { + for (const path of basePaths) { + const res = await makeGetRequest({ + url: server.url, + path: path + 'is-authenticated', + token: server.accessToken, + expectedStatus: 200 + }) + + expect(res.body.isAuthenticated).to.equal(true) + + const secRes = await makeGetRequest({ + url: server.url, + path: path + 'is-authenticated', + expectedStatus: 200 + }) + + expect(secRes.body.isAuthenticated).to.equal(false) + } + }) + + it('Should mirror post body', async function () { + const body = { + hello: 'world', + riri: 'fifi', + loulou: 'picsou' + } + + for (const path of basePaths) { + const res = await makePostBodyRequest({ + url: server.url, + path: path + 'form/post/mirror', + fields: body, + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(res.body).to.deep.equal(body) + } + }) + + it('Should remove the plugin and remove the routes', async function () { + await server.plugins.uninstall({ npmName: 'peertube-plugin-test-five' }) + + for (const path of basePaths) { + await makeGetRequest({ + url: server.url, + path: path + 'ping', + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + + await makePostBodyRequest({ + url: server.url, + path: path + 'ping', + fields: {}, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + } + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/plugins/plugin-storage.ts b/packages/tests/src/plugins/plugin-storage.ts new file mode 100644 index 000000000..f9b0ead0c --- /dev/null +++ b/packages/tests/src/plugins/plugin-storage.ts @@ -0,0 +1,95 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { pathExists } from 'fs-extra/esm' +import { readdir, readFile } from 'fs/promises' +import { join } from 'path' +import { HttpStatusCode } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + makeGetRequest, + PeerTubeServer, + PluginsCommand, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test plugin storage', function () { + let server: PeerTubeServer + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + + await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-six') }) + }) + + describe('DB storage', function () { + it('Should correctly store a subkey', async function () { + await server.servers.waitUntilLog('superkey stored value is toto') + }) + + it('Should correctly retrieve an array as array from the storage.', async function () { + await server.servers.waitUntilLog('storedArrayKey isArray is true') + await server.servers.waitUntilLog('storedArrayKey stored value is toto, toto2') + }) + }) + + describe('Disk storage', function () { + let dataPath: string + let pluginDataPath: string + + async function getFileContent () { + const files = await readdir(pluginDataPath) + expect(files).to.have.lengthOf(1) + + return readFile(join(pluginDataPath, files[0]), 'utf8') + } + + before(function () { + dataPath = server.servers.buildDirectory('plugins/data') + pluginDataPath = join(dataPath, 'peertube-plugin-test-six') + }) + + it('Should have created the directory on install', async function () { + const dataPath = server.servers.buildDirectory('plugins/data') + const pluginDataPath = join(dataPath, 'peertube-plugin-test-six') + + expect(await pathExists(dataPath)).to.be.true + expect(await pathExists(pluginDataPath)).to.be.true + expect(await readdir(pluginDataPath)).to.have.lengthOf(0) + }) + + it('Should have created a file', async function () { + await makeGetRequest({ + url: server.url, + token: server.accessToken, + path: '/plugins/test-six/router/create-file', + expectedStatus: HttpStatusCode.OK_200 + }) + + const content = await getFileContent() + expect(content).to.equal('Prince Ali') + }) + + it('Should still have the file after an uninstallation', async function () { + await server.plugins.uninstall({ npmName: 'peertube-plugin-test-six' }) + + const content = await getFileContent() + expect(content).to.equal('Prince Ali') + }) + + it('Should still have the file after the reinstallation', async function () { + await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-six') }) + + const content = await getFileContent() + expect(content).to.equal('Prince Ali') + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/plugins/plugin-transcoding.ts b/packages/tests/src/plugins/plugin-transcoding.ts new file mode 100644 index 000000000..2f50f65ff --- /dev/null +++ b/packages/tests/src/plugins/plugin-transcoding.ts @@ -0,0 +1,279 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { getAudioStream, getVideoStream, getVideoStreamFPS } from '@peertube/peertube-ffmpeg' +import { VideoPrivacy } from '@peertube/peertube-models' +import { + cleanupTests, + createSingleServer, + PeerTubeServer, + PluginsCommand, + setAccessTokensToServers, + setDefaultVideoChannel, + testFfmpegStreamError, + waitJobs +} from '@peertube/peertube-server-commands' + +async function createLiveWrapper (server: PeerTubeServer) { + const liveAttributes = { + name: 'live video', + channelId: server.store.channel.id, + privacy: VideoPrivacy.PUBLIC + } + + const { uuid } = await server.live.create({ fields: liveAttributes }) + + return uuid +} + +function updateConf (server: PeerTubeServer, vodProfile: string, liveProfile: string) { + return server.config.updateCustomSubConfig({ + newConfig: { + transcoding: { + enabled: true, + profile: vodProfile, + hls: { + enabled: true + }, + webVideos: { + enabled: true + }, + resolutions: { + '240p': true, + '360p': false, + '480p': false, + '720p': true + } + }, + live: { + transcoding: { + profile: liveProfile, + enabled: true, + resolutions: { + '240p': true, + '360p': false, + '480p': false, + '720p': true + } + } + } + } + }) +} + +describe('Test transcoding plugins', function () { + let server: PeerTubeServer + + before(async function () { + this.timeout(60000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + await updateConf(server, 'default', 'default') + }) + + describe('When using a plugin adding profiles to existing encoders', function () { + + async function checkVideoFPS (uuid: string, type: 'above' | 'below', fps: number) { + const video = await server.videos.get({ id: uuid }) + const files = video.files.concat(...video.streamingPlaylists.map(p => p.files)) + + for (const file of files) { + if (type === 'above') { + expect(file.fps).to.be.above(fps) + } else { + expect(file.fps).to.be.below(fps) + } + } + } + + async function checkLiveFPS (uuid: string, type: 'above' | 'below', fps: number) { + const playlistUrl = `${server.url}/static/streaming-playlists/hls/${uuid}/0.m3u8` + const videoFPS = await getVideoStreamFPS(playlistUrl) + + if (type === 'above') { + expect(videoFPS).to.be.above(fps) + } else { + expect(videoFPS).to.be.below(fps) + } + } + + before(async function () { + await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-transcoding-one') }) + }) + + it('Should have the appropriate available profiles', async function () { + const config = await server.config.getConfig() + + expect(config.transcoding.availableProfiles).to.have.members([ 'default', 'low-vod', 'input-options-vod', 'bad-scale-vod' ]) + expect(config.live.transcoding.availableProfiles).to.have.members([ 'default', 'high-live', 'input-options-live', 'bad-scale-live' ]) + }) + + describe('VOD', function () { + + it('Should not use the plugin profile if not chosen by the admin', async function () { + this.timeout(240000) + + const videoUUID = (await server.videos.quickUpload({ name: 'video' })).uuid + await waitJobs([ server ]) + + await checkVideoFPS(videoUUID, 'above', 20) + }) + + it('Should use the vod profile', async function () { + this.timeout(240000) + + await updateConf(server, 'low-vod', 'default') + + const videoUUID = (await server.videos.quickUpload({ name: 'video' })).uuid + await waitJobs([ server ]) + + await checkVideoFPS(videoUUID, 'below', 12) + }) + + it('Should apply input options in vod profile', async function () { + this.timeout(240000) + + await updateConf(server, 'input-options-vod', 'default') + + const videoUUID = (await server.videos.quickUpload({ name: 'video' })).uuid + await waitJobs([ server ]) + + await checkVideoFPS(videoUUID, 'below', 6) + }) + + it('Should apply the scale filter in vod profile', async function () { + this.timeout(240000) + + await updateConf(server, 'bad-scale-vod', 'default') + + const videoUUID = (await server.videos.quickUpload({ name: 'video' })).uuid + await waitJobs([ server ]) + + // Transcoding failed + const video = await server.videos.get({ id: videoUUID }) + expect(video.files).to.have.lengthOf(1) + expect(video.streamingPlaylists).to.have.lengthOf(0) + }) + }) + + describe('Live', function () { + + it('Should not use the plugin profile if not chosen by the admin', async function () { + this.timeout(240000) + + const liveVideoId = await createLiveWrapper(server) + + await server.live.sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_very_short_240p.mp4' }) + await server.live.waitUntilPublished({ videoId: liveVideoId }) + await waitJobs([ server ]) + + await checkLiveFPS(liveVideoId, 'above', 20) + }) + + it('Should use the live profile', async function () { + this.timeout(240000) + + await updateConf(server, 'low-vod', 'high-live') + + const liveVideoId = await createLiveWrapper(server) + + await server.live.sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_very_short_240p.mp4' }) + await server.live.waitUntilPublished({ videoId: liveVideoId }) + await waitJobs([ server ]) + + await checkLiveFPS(liveVideoId, 'above', 45) + }) + + it('Should apply the input options on live profile', async function () { + this.timeout(240000) + + await updateConf(server, 'low-vod', 'input-options-live') + + const liveVideoId = await createLiveWrapper(server) + + await server.live.sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_very_short_240p.mp4' }) + await server.live.waitUntilPublished({ videoId: liveVideoId }) + await waitJobs([ server ]) + + await checkLiveFPS(liveVideoId, 'above', 45) + }) + + it('Should apply the scale filter name on live profile', async function () { + this.timeout(240000) + + await updateConf(server, 'low-vod', 'bad-scale-live') + + const liveVideoId = await createLiveWrapper(server) + + const command = await server.live.sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_very_short_240p.mp4' }) + await testFfmpegStreamError(command, true) + }) + + it('Should default to the default profile if the specified profile does not exist', async function () { + this.timeout(240000) + + await server.plugins.uninstall({ npmName: 'peertube-plugin-test-transcoding-one' }) + + const config = await server.config.getConfig() + + expect(config.transcoding.availableProfiles).to.deep.equal([ 'default' ]) + expect(config.live.transcoding.availableProfiles).to.deep.equal([ 'default' ]) + + const videoUUID = (await server.videos.quickUpload({ name: 'video', fixture: 'video_very_short_240p.mp4' })).uuid + await waitJobs([ server ]) + + await checkVideoFPS(videoUUID, 'above', 20) + }) + }) + + }) + + describe('When using a plugin adding new encoders', function () { + + before(async function () { + await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-transcoding-two') }) + + await updateConf(server, 'test-vod-profile', 'test-live-profile') + }) + + it('Should use the new vod encoders', async function () { + this.timeout(240000) + + const videoUUID = (await server.videos.quickUpload({ name: 'video', fixture: 'video_very_short_240p.mp4' })).uuid + await waitJobs([ server ]) + + const video = await server.videos.get({ id: videoUUID }) + + const path = server.servers.buildWebVideoFilePath(video.files[0].fileUrl) + const audioProbe = await getAudioStream(path) + expect(audioProbe.audioStream.codec_name).to.equal('opus') + + const videoProbe = await getVideoStream(path) + expect(videoProbe.codec_name).to.equal('vp9') + }) + + it('Should use the new live encoders', async function () { + this.timeout(240000) + + const liveVideoId = await createLiveWrapper(server) + + await server.live.sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_short2.webm' }) + await server.live.waitUntilPublished({ videoId: liveVideoId }) + await waitJobs([ server ]) + + const playlistUrl = `${server.url}/static/streaming-playlists/hls/${liveVideoId}/0.m3u8` + const audioProbe = await getAudioStream(playlistUrl) + expect(audioProbe.audioStream.codec_name).to.equal('opus') + + const videoProbe = await getVideoStream(playlistUrl) + expect(videoProbe.codec_name).to.equal('h264') + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/plugins/plugin-unloading.ts b/packages/tests/src/plugins/plugin-unloading.ts new file mode 100644 index 000000000..70310bc8c --- /dev/null +++ b/packages/tests/src/plugins/plugin-unloading.ts @@ -0,0 +1,75 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { + cleanupTests, + createSingleServer, + makeGetRequest, + PeerTubeServer, + PluginsCommand, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' +import { HttpStatusCode } from '@peertube/peertube-models' + +describe('Test plugins module unloading', function () { + let server: PeerTubeServer = null + const requestPath = '/plugins/test-unloading/router/get' + let value: string = null + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + + await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-unloading') }) + }) + + it('Should return a numeric value', async function () { + const res = await makeGetRequest({ + url: server.url, + path: requestPath, + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(res.body.message).to.match(/^\d+$/) + value = res.body.message + }) + + it('Should return the same value the second time', async function () { + const res = await makeGetRequest({ + url: server.url, + path: requestPath, + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(res.body.message).to.be.equal(value) + }) + + it('Should uninstall the plugin and free the route', async function () { + await server.plugins.uninstall({ npmName: 'peertube-plugin-test-unloading' }) + + await makeGetRequest({ + url: server.url, + path: requestPath, + expectedStatus: HttpStatusCode.NOT_FOUND_404 + }) + }) + + it('Should return a different numeric value', async function () { + await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-unloading') }) + + const res = await makeGetRequest({ + url: server.url, + path: requestPath, + expectedStatus: HttpStatusCode.OK_200 + }) + + expect(res.body.message).to.match(/^\d+$/) + expect(res.body.message).to.be.not.equal(value) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/plugins/plugin-websocket.ts b/packages/tests/src/plugins/plugin-websocket.ts new file mode 100644 index 000000000..832dcebd0 --- /dev/null +++ b/packages/tests/src/plugins/plugin-websocket.ts @@ -0,0 +1,76 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import WebSocket from 'ws' +import { + cleanupTests, + createSingleServer, + PeerTubeServer, + PluginsCommand, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +function buildWebSocket (server: PeerTubeServer, path: string) { + return new WebSocket('ws://' + server.host + path) +} + +function expectErrorOrTimeout (server: PeerTubeServer, path: string, expectedTimeout: number) { + return new Promise((res, rej) => { + const ws = buildWebSocket(server, path) + ws.on('error', () => res()) + + const timeout = setTimeout(() => res(), expectedTimeout) + + ws.on('open', () => { + clearTimeout(timeout) + + return rej(new Error('Connect did not timeout')) + }) + }) +} + +describe('Test plugin websocket', function () { + let server: PeerTubeServer + const basePaths = [ + '/plugins/test-websocket/ws/', + '/plugins/test-websocket/0.0.1/ws/' + ] + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + + await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-websocket') }) + }) + + it('Should not connect to the websocket without the appropriate path', async function () { + const paths = [ + '/plugins/unknown/ws/', + '/plugins/unknown/0.0.1/ws/' + ] + + for (const path of paths) { + await expectErrorOrTimeout(server, path, 1000) + } + }) + + it('Should not connect to the websocket without the appropriate sub path', async function () { + for (const path of basePaths) { + await expectErrorOrTimeout(server, path + '/unknown', 1000) + } + }) + + it('Should connect to the websocket and receive pong', function (done) { + const ws = buildWebSocket(server, basePaths[0]) + + ws.on('open', () => ws.send('ping')) + ws.on('message', data => { + if (data.toString() === 'pong') return done() + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/plugins/translations.ts b/packages/tests/src/plugins/translations.ts new file mode 100644 index 000000000..a69e14134 --- /dev/null +++ b/packages/tests/src/plugins/translations.ts @@ -0,0 +1,80 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { + cleanupTests, + createSingleServer, + PeerTubeServer, + PluginsCommand, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' + +describe('Test plugin translations', function () { + let server: PeerTubeServer + let command: PluginsCommand + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + + command = server.plugins + + await command.install({ path: PluginsCommand.getPluginTestPath() }) + await command.install({ path: PluginsCommand.getPluginTestPath('-filter-translations') }) + }) + + it('Should not have translations for locale pt', async function () { + const body = await command.getTranslations({ locale: 'pt' }) + + expect(body).to.deep.equal({}) + }) + + it('Should have translations for locale fr', async function () { + const body = await command.getTranslations({ locale: 'fr-FR' }) + + expect(body).to.deep.equal({ + 'peertube-plugin-test': { + Hi: 'Coucou' + }, + 'peertube-plugin-test-filter-translations': { + 'Hello world': 'Bonjour le monde' + } + }) + }) + + it('Should have translations of locale it', async function () { + const body = await command.getTranslations({ locale: 'it-IT' }) + + expect(body).to.deep.equal({ + 'peertube-plugin-test-filter-translations': { + 'Hello world': 'Ciao, mondo!' + } + }) + }) + + it('Should remove the plugin and remove the locales', async function () { + await command.uninstall({ npmName: 'peertube-plugin-test-filter-translations' }) + + { + const body = await command.getTranslations({ locale: 'fr-FR' }) + + expect(body).to.deep.equal({ + 'peertube-plugin-test': { + Hi: 'Coucou' + } + }) + } + + { + const body = await command.getTranslations({ locale: 'it-IT' }) + + expect(body).to.deep.equal({}) + } + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/plugins/video-constants.ts b/packages/tests/src/plugins/video-constants.ts new file mode 100644 index 000000000..b81240a64 --- /dev/null +++ b/packages/tests/src/plugins/video-constants.ts @@ -0,0 +1,180 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { + cleanupTests, + createSingleServer, + makeGetRequest, + PeerTubeServer, + PluginsCommand, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' +import { HttpStatusCode, VideoPlaylistPrivacy, VideoPrivacy } from '@peertube/peertube-models' + +describe('Test plugin altering video constants', function () { + let server: PeerTubeServer + + before(async function () { + this.timeout(30000) + + server = await createSingleServer(1) + await setAccessTokensToServers([ server ]) + + await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-video-constants') }) + }) + + it('Should have updated languages', async function () { + const languages = await server.videos.getLanguages() + + expect(languages['en']).to.not.exist + expect(languages['fr']).to.not.exist + + expect(languages['al_bhed']).to.equal('Al Bhed') + expect(languages['al_bhed2']).to.equal('Al Bhed 2') + expect(languages['al_bhed3']).to.not.exist + }) + + it('Should have updated categories', async function () { + const categories = await server.videos.getCategories() + + expect(categories[1]).to.not.exist + expect(categories[2]).to.not.exist + + expect(categories[42]).to.equal('Best category') + expect(categories[43]).to.equal('High best category') + }) + + it('Should have updated licences', async function () { + const licences = await server.videos.getLicences() + + expect(licences[1]).to.not.exist + expect(licences[7]).to.not.exist + + expect(licences[42]).to.equal('Best licence') + expect(licences[43]).to.equal('High best licence') + }) + + it('Should have updated video privacies', async function () { + const privacies = await server.videos.getPrivacies() + + expect(privacies[1]).to.exist + expect(privacies[2]).to.not.exist + expect(privacies[3]).to.exist + expect(privacies[4]).to.exist + }) + + it('Should have updated playlist privacies', async function () { + const playlistPrivacies = await server.playlists.getPrivacies() + + expect(playlistPrivacies[1]).to.exist + expect(playlistPrivacies[2]).to.exist + expect(playlistPrivacies[3]).to.not.exist + }) + + it('Should not be able to create a video with this privacy', async function () { + const attributes = { name: 'video', privacy: VideoPrivacy.UNLISTED } + await server.videos.upload({ attributes, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should not be able to create a video with this privacy', async function () { + const attributes = { displayName: 'video playlist', privacy: VideoPlaylistPrivacy.PRIVATE } + await server.playlists.create({ attributes, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + + it('Should be able to upload a video with these values', async function () { + const attributes = { name: 'video', category: 42, licence: 42, language: 'al_bhed2' } + const { uuid } = await server.videos.upload({ attributes }) + + const video = await server.videos.get({ id: uuid }) + expect(video.language.label).to.equal('Al Bhed 2') + expect(video.licence.label).to.equal('Best licence') + expect(video.category.label).to.equal('Best category') + }) + + it('Should uninstall the plugin and reset languages, categories, licences and privacies', async function () { + await server.plugins.uninstall({ npmName: 'peertube-plugin-test-video-constants' }) + + { + const languages = await server.videos.getLanguages() + + expect(languages['en']).to.equal('English') + expect(languages['fr']).to.equal('French') + + expect(languages['al_bhed']).to.not.exist + expect(languages['al_bhed2']).to.not.exist + expect(languages['al_bhed3']).to.not.exist + } + + { + const categories = await server.videos.getCategories() + + expect(categories[1]).to.equal('Music') + expect(categories[2]).to.equal('Films') + + expect(categories[42]).to.not.exist + expect(categories[43]).to.not.exist + } + + { + const licences = await server.videos.getLicences() + + expect(licences[1]).to.equal('Attribution') + expect(licences[7]).to.equal('Public Domain Dedication') + + expect(licences[42]).to.not.exist + expect(licences[43]).to.not.exist + } + + { + const privacies = await server.videos.getPrivacies() + + expect(privacies[1]).to.exist + expect(privacies[2]).to.exist + expect(privacies[3]).to.exist + expect(privacies[4]).to.exist + } + + { + const playlistPrivacies = await server.playlists.getPrivacies() + + expect(playlistPrivacies[1]).to.exist + expect(playlistPrivacies[2]).to.exist + expect(playlistPrivacies[3]).to.exist + } + }) + + it('Should be able to reset categories', async function () { + await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-video-constants') }) + + { + const categories = await server.videos.getCategories() + + expect(categories[1]).to.not.exist + expect(categories[2]).to.not.exist + + expect(categories[42]).to.exist + expect(categories[43]).to.exist + } + + await makeGetRequest({ + url: server.url, + token: server.accessToken, + path: '/plugins/test-video-constants/router/reset-categories', + expectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + + { + const categories = await server.videos.getCategories() + + expect(categories[1]).to.exist + expect(categories[2]).to.exist + + expect(categories[42]).to.not.exist + expect(categories[43]).to.not.exist + } + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/packages/tests/src/server-helpers/activitypub.ts b/packages/tests/src/server-helpers/activitypub.ts new file mode 100644 index 000000000..dfcd0389f --- /dev/null +++ b/packages/tests/src/server-helpers/activitypub.ts @@ -0,0 +1,176 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import { signAndContextify } from '@peertube/peertube-server/server/helpers/activity-pub-utils.js' +import { + isHTTPSignatureVerified, + isJsonLDSignatureVerified, + parseHTTPSignature +} from '@peertube/peertube-server/server/helpers/peertube-crypto.js' +import { buildRequestStub } from '@tests/shared/tests.js' +import { expect } from 'chai' +import { readJsonSync } from 'fs-extra/esm' +import cloneDeep from 'lodash-es/cloneDeep.js' + +function fakeFilter () { + return (data: any) => Promise.resolve(data) +} + +describe('Test activity pub helpers', function () { + + describe('When checking the Linked Signature', function () { + + it('Should fail with an invalid Mastodon signature', async function () { + const body = readJsonSync(buildAbsoluteFixturePath('./ap-json/mastodon/create-bad-signature.json')) + const publicKey = readJsonSync(buildAbsoluteFixturePath('./ap-json/mastodon/public-key.json')).publicKey + const fromActor = { publicKey, url: 'http://localhost:9002/accounts/peertube' } + + const result = await isJsonLDSignatureVerified(fromActor as any, body) + + expect(result).to.be.false + }) + + it('Should fail with an invalid public key', async function () { + const body = readJsonSync(buildAbsoluteFixturePath('./ap-json/mastodon/create.json')) + const publicKey = readJsonSync(buildAbsoluteFixturePath('./ap-json/mastodon/bad-public-key.json')).publicKey + const fromActor = { publicKey, url: 'http://localhost:9002/accounts/peertube' } + + const result = await isJsonLDSignatureVerified(fromActor as any, body) + + expect(result).to.be.false + }) + + it('Should succeed with a valid Mastodon signature', async function () { + const body = readJsonSync(buildAbsoluteFixturePath('./ap-json/mastodon/create.json')) + const publicKey = readJsonSync(buildAbsoluteFixturePath('./ap-json/mastodon/public-key.json')).publicKey + const fromActor = { publicKey, url: 'http://localhost:9002/accounts/peertube' } + + const result = await isJsonLDSignatureVerified(fromActor as any, body) + + expect(result).to.be.true + }) + + it('Should fail with an invalid PeerTube signature', async function () { + const keys = readJsonSync(buildAbsoluteFixturePath('./ap-json/peertube/invalid-keys.json')) + const body = readJsonSync(buildAbsoluteFixturePath('./ap-json/peertube/announce-without-context.json')) + + const actorSignature = { url: 'http://localhost:9002/accounts/peertube', privateKey: keys.privateKey } + const signedBody = await signAndContextify(actorSignature as any, body, 'Announce', fakeFilter()) + + const fromActor = { publicKey: keys.publicKey, url: 'http://localhost:9002/accounts/peertube' } + const result = await isJsonLDSignatureVerified(fromActor as any, signedBody) + + expect(result).to.be.false + }) + + it('Should succeed with a valid PeerTube signature', async function () { + const keys = readJsonSync(buildAbsoluteFixturePath('./ap-json/peertube/keys.json')) + const body = readJsonSync(buildAbsoluteFixturePath('./ap-json/peertube/announce-without-context.json')) + + const actorSignature = { url: 'http://localhost:9002/accounts/peertube', privateKey: keys.privateKey } + const signedBody = await signAndContextify(actorSignature as any, body, 'Announce', fakeFilter()) + + const fromActor = { publicKey: keys.publicKey, url: 'http://localhost:9002/accounts/peertube' } + const result = await isJsonLDSignatureVerified(fromActor as any, signedBody) + + expect(result).to.be.true + }) + }) + + describe('When checking HTTP signature', function () { + it('Should fail with an invalid http signature', async function () { + const req = buildRequestStub() + req.method = 'POST' + req.url = '/accounts/ronan/inbox' + + const mastodonObject = cloneDeep(readJsonSync(buildAbsoluteFixturePath('./ap-json/mastodon/bad-http-signature.json'))) + req.body = mastodonObject.body + req.headers = mastodonObject.headers + + const parsed = parseHTTPSignature(req, 3600 * 1000 * 365 * 10) + const publicKey = readJsonSync(buildAbsoluteFixturePath('./ap-json/mastodon/public-key.json')).publicKey + + const actor = { publicKey } + const verified = isHTTPSignatureVerified(parsed, actor as any) + + expect(verified).to.be.false + }) + + it('Should fail with an invalid public key', async function () { + const req = buildRequestStub() + req.method = 'POST' + req.url = '/accounts/ronan/inbox' + + const mastodonObject = cloneDeep(readJsonSync(buildAbsoluteFixturePath('./ap-json/mastodon/http-signature.json'))) + req.body = mastodonObject.body + req.headers = mastodonObject.headers + + const parsed = parseHTTPSignature(req, 3600 * 1000 * 365 * 10) + const publicKey = readJsonSync(buildAbsoluteFixturePath('./ap-json/mastodon/bad-public-key.json')).publicKey + + const actor = { publicKey } + const verified = isHTTPSignatureVerified(parsed, actor as any) + + expect(verified).to.be.false + }) + + it('Should fail because of clock skew', async function () { + const req = buildRequestStub() + req.method = 'POST' + req.url = '/accounts/ronan/inbox' + + const mastodonObject = cloneDeep(readJsonSync(buildAbsoluteFixturePath('./ap-json/mastodon/http-signature.json'))) + req.body = mastodonObject.body + req.headers = mastodonObject.headers + + let errored = false + try { + parseHTTPSignature(req) + } catch { + errored = true + } + + expect(errored).to.be.true + }) + + it('Should with a scheme', async function () { + const req = buildRequestStub() + req.method = 'POST' + req.url = '/accounts/ronan/inbox' + + const mastodonObject = cloneDeep(readJsonSync(buildAbsoluteFixturePath('./ap-json/mastodon/http-signature.json'))) + req.body = mastodonObject.body + req.headers = mastodonObject.headers + req.headers = 'Signature ' + mastodonObject.headers + + let errored = false + try { + parseHTTPSignature(req, 3600 * 1000 * 365 * 10) + } catch { + errored = true + } + + expect(errored).to.be.true + }) + + it('Should succeed with a valid signature', async function () { + const req = buildRequestStub() + req.method = 'POST' + req.url = '/accounts/ronan/inbox' + + const mastodonObject = cloneDeep(readJsonSync(buildAbsoluteFixturePath('./ap-json/mastodon/http-signature.json'))) + req.body = mastodonObject.body + req.headers = mastodonObject.headers + + const parsed = parseHTTPSignature(req, 3600 * 1000 * 365 * 10) + const publicKey = readJsonSync(buildAbsoluteFixturePath('./ap-json/mastodon/public-key.json')).publicKey + + const actor = { publicKey } + const verified = isHTTPSignatureVerified(parsed, actor as any) + + expect(verified).to.be.true + }) + + }) + +}) diff --git a/packages/tests/src/server-helpers/core-utils.ts b/packages/tests/src/server-helpers/core-utils.ts new file mode 100644 index 000000000..06c78591e --- /dev/null +++ b/packages/tests/src/server-helpers/core-utils.ts @@ -0,0 +1,150 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import snakeCase from 'lodash-es/snakeCase.js' +import validator from 'validator' +import { getAverageTheoreticalBitrate, getMaxTheoreticalBitrate } from '@peertube/peertube-core-utils' +import { VideoResolution } from '@peertube/peertube-models' +import { objectConverter, parseBytes, parseDurationToMs } from '@peertube/peertube-server/server/helpers/core-utils.js' + +describe('Parse Bytes', function () { + + it('Should pass on valid value', async function () { + // just return it + expect(parseBytes(-1024)).to.equal(-1024) + expect(parseBytes(1024)).to.equal(1024) + expect(parseBytes(1048576)).to.equal(1048576) + expect(parseBytes('1024')).to.equal(1024) + expect(parseBytes('1048576')).to.equal(1048576) + + // sizes + expect(parseBytes('1B')).to.equal(1024) + expect(parseBytes('1MB')).to.equal(1048576) + expect(parseBytes('1GB')).to.equal(1073741824) + expect(parseBytes('1TB')).to.equal(1099511627776) + + expect(parseBytes('5GB')).to.equal(5368709120) + expect(parseBytes('5TB')).to.equal(5497558138880) + + expect(parseBytes('1024B')).to.equal(1048576) + expect(parseBytes('1024MB')).to.equal(1073741824) + expect(parseBytes('1024GB')).to.equal(1099511627776) + expect(parseBytes('1024TB')).to.equal(1125899906842624) + + // with whitespace + expect(parseBytes('1 GB')).to.equal(1073741824) + expect(parseBytes('1\tGB')).to.equal(1073741824) + + // sum value + expect(parseBytes('1TB 1024MB')).to.equal(1100585369600) + expect(parseBytes('4GB 1024MB')).to.equal(5368709120) + expect(parseBytes('4TB 1024GB')).to.equal(5497558138880) + expect(parseBytes('4TB 1024GB 0MB')).to.equal(5497558138880) + expect(parseBytes('1024TB 1024GB 1024MB')).to.equal(1127000492212224) + }) + + it('Should be invalid when given invalid value', async function () { + expect(parseBytes('6GB 1GB')).to.equal(6) + }) +}) + +describe('Parse duration', function () { + + it('Should pass when given valid value', async function () { + expect(parseDurationToMs(35)).to.equal(35) + expect(parseDurationToMs(-35)).to.equal(-35) + expect(parseDurationToMs('35 seconds')).to.equal(35 * 1000) + expect(parseDurationToMs('1 minute')).to.equal(60 * 1000) + expect(parseDurationToMs('1 hour')).to.equal(3600 * 1000) + expect(parseDurationToMs('35 hours')).to.equal(3600 * 35 * 1000) + }) + + it('Should be invalid when given invalid value', async function () { + expect(parseBytes('35m 5s')).to.equal(35) + }) +}) + +describe('Object', function () { + + it('Should convert an object', async function () { + function keyConverter (k: string) { + return snakeCase(k) + } + + function valueConverter (v: any) { + if (validator.default.isNumeric(v + '')) return parseInt('' + v, 10) + + return v + } + + const obj = { + mySuperKey: 'hello', + mySuper2Key: '45', + mySuper3Key: { + mySuperSubKey: '15', + mySuperSub2Key: 'hello', + mySuperSub3Key: [ '1', 'hello', 2 ], + mySuperSub4Key: 4 + }, + mySuper4Key: 45, + toto: { + super_key: '15', + superKey2: 'hello' + }, + super_key: { + superKey4: 15 + } + } + + const res = objectConverter(obj, keyConverter, valueConverter) + + expect(res.my_super_key).to.equal('hello') + expect(res.my_super_2_key).to.equal(45) + expect(res.my_super_3_key.my_super_sub_key).to.equal(15) + expect(res.my_super_3_key.my_super_sub_2_key).to.equal('hello') + expect(res.my_super_3_key.my_super_sub_3_key).to.deep.equal([ 1, 'hello', 2 ]) + expect(res.my_super_3_key.my_super_sub_4_key).to.equal(4) + expect(res.toto.super_key).to.equal(15) + expect(res.toto.super_key_2).to.equal('hello') + expect(res.super_key.super_key_4).to.equal(15) + + // Immutable + expect(res.mySuperKey).to.be.undefined + expect(obj['my_super_key']).to.be.undefined + }) +}) + +describe('Bitrate', function () { + + it('Should get appropriate max bitrate', function () { + const tests = [ + { resolution: VideoResolution.H_144P, ratio: 16 / 9, fps: 24, min: 200, max: 400 }, + { resolution: VideoResolution.H_240P, ratio: 16 / 9, fps: 24, min: 600, max: 800 }, + { resolution: VideoResolution.H_360P, ratio: 16 / 9, fps: 24, min: 1200, max: 1600 }, + { resolution: VideoResolution.H_480P, ratio: 16 / 9, fps: 24, min: 2000, max: 2300 }, + { resolution: VideoResolution.H_720P, ratio: 16 / 9, fps: 24, min: 4000, max: 4400 }, + { resolution: VideoResolution.H_1080P, ratio: 16 / 9, fps: 24, min: 8000, max: 10000 }, + { resolution: VideoResolution.H_4K, ratio: 16 / 9, fps: 24, min: 25000, max: 30000 } + ] + + for (const test of tests) { + expect(getMaxTheoreticalBitrate(test)).to.be.above(test.min * 1000).and.below(test.max * 1000) + } + }) + + it('Should get appropriate average bitrate', function () { + const tests = [ + { resolution: VideoResolution.H_144P, ratio: 16 / 9, fps: 24, min: 50, max: 300 }, + { resolution: VideoResolution.H_240P, ratio: 16 / 9, fps: 24, min: 350, max: 450 }, + { resolution: VideoResolution.H_360P, ratio: 16 / 9, fps: 24, min: 700, max: 900 }, + { resolution: VideoResolution.H_480P, ratio: 16 / 9, fps: 24, min: 1100, max: 1300 }, + { resolution: VideoResolution.H_720P, ratio: 16 / 9, fps: 24, min: 2300, max: 2500 }, + { resolution: VideoResolution.H_1080P, ratio: 16 / 9, fps: 24, min: 4700, max: 5000 }, + { resolution: VideoResolution.H_4K, ratio: 16 / 9, fps: 24, min: 15000, max: 17000 } + ] + + for (const test of tests) { + expect(getAverageTheoreticalBitrate(test)).to.be.above(test.min * 1000).and.below(test.max * 1000) + } + }) +}) diff --git a/packages/tests/src/server-helpers/crypto.ts b/packages/tests/src/server-helpers/crypto.ts new file mode 100644 index 000000000..4bf5b8a45 --- /dev/null +++ b/packages/tests/src/server-helpers/crypto.ts @@ -0,0 +1,33 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { decrypt, encrypt } from '@peertube/peertube-server/server/helpers/peertube-crypto.js' + +describe('Encrypt/Descrypt', function () { + + it('Should encrypt and decrypt the string', async function () { + const secret = 'my_secret' + const str = 'my super string' + + const encrypted = await encrypt(str, secret) + const decrypted = await decrypt(encrypted, secret) + + expect(str).to.equal(decrypted) + }) + + it('Should not decrypt without the same secret', async function () { + const str = 'my super string' + + const encrypted = await encrypt(str, 'my_secret') + + let error = false + + try { + await decrypt(encrypted, 'my_sicret') + } catch (err) { + error = true + } + + expect(error).to.be.true + }) +}) diff --git a/packages/tests/src/server-helpers/dns.ts b/packages/tests/src/server-helpers/dns.ts new file mode 100644 index 000000000..64e3112a2 --- /dev/null +++ b/packages/tests/src/server-helpers/dns.ts @@ -0,0 +1,16 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { isResolvingToUnicastOnly } from '@peertube/peertube-server/server/helpers/dns.js' + +describe('DNS helpers', function () { + + it('Should correctly check unicast IPs', async function () { + expect(await isResolvingToUnicastOnly('cpy.re')).to.be.true + expect(await isResolvingToUnicastOnly('framasoft.org')).to.be.true + expect(await isResolvingToUnicastOnly('8.8.8.8')).to.be.true + + expect(await isResolvingToUnicastOnly('127.0.0.1')).to.be.false + expect(await isResolvingToUnicastOnly('127.0.0.1.cpy.re')).to.be.false + }) +}) diff --git a/packages/tests/src/server-helpers/image.ts b/packages/tests/src/server-helpers/image.ts new file mode 100644 index 000000000..34675d385 --- /dev/null +++ b/packages/tests/src/server-helpers/image.ts @@ -0,0 +1,97 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { remove } from 'fs-extra/esm' +import { readFile } from 'fs/promises' +import { join } from 'path' +import { buildAbsoluteFixturePath, root } from '@peertube/peertube-node-utils' +import { execPromise } from '@peertube/peertube-server/server/helpers/core-utils.js' +import { processImage } from '@peertube/peertube-server/server/helpers/image-utils.js' + +async function checkBuffers (path1: string, path2: string, equals: boolean) { + const [ buf1, buf2 ] = await Promise.all([ + readFile(path1), + readFile(path2) + ]) + + if (equals) { + expect(buf1.equals(buf2)).to.be.true + } else { + expect(buf1.equals(buf2)).to.be.false + } +} + +async function hasTitleExif (path: string) { + const result = JSON.parse(await execPromise(`exiftool -json ${path}`)) + + return result[0]?.Title === 'should be removed' +} + +describe('Image helpers', function () { + const imageDestDir = join(root(), 'test-images') + + const imageDestJPG = join(imageDestDir, 'test.jpg') + const imageDestPNG = join(imageDestDir, 'test.png') + + const thumbnailSize = { width: 280, height: 157 } + + it('Should skip processing if the source image is okay', async function () { + const input = buildAbsoluteFixturePath('custom-thumbnail.jpg') + await processImage({ path: input, destination: imageDestJPG, newSize: thumbnailSize, keepOriginal: true }) + + await checkBuffers(input, imageDestJPG, true) + }) + + it('Should not skip processing if the source image does not have the appropriate extension', async function () { + const input = buildAbsoluteFixturePath('custom-thumbnail.png') + await processImage({ path: input, destination: imageDestJPG, newSize: thumbnailSize, keepOriginal: true }) + + await checkBuffers(input, imageDestJPG, false) + }) + + it('Should not skip processing if the source image does not have the appropriate size', async function () { + const input = buildAbsoluteFixturePath('custom-preview.jpg') + await processImage({ path: input, destination: imageDestJPG, newSize: thumbnailSize, keepOriginal: true }) + + await checkBuffers(input, imageDestJPG, false) + }) + + it('Should not skip processing if the source image does not have the appropriate size', async function () { + const input = buildAbsoluteFixturePath('custom-thumbnail-big.jpg') + await processImage({ path: input, destination: imageDestJPG, newSize: thumbnailSize, keepOriginal: true }) + + await checkBuffers(input, imageDestJPG, false) + }) + + it('Should strip exif for a jpg file that can not be copied', async function () { + const input = buildAbsoluteFixturePath('exif.jpg') + expect(await hasTitleExif(input)).to.be.true + + await processImage({ path: input, destination: imageDestJPG, newSize: { width: 100, height: 100 }, keepOriginal: true }) + await checkBuffers(input, imageDestJPG, false) + + expect(await hasTitleExif(imageDestJPG)).to.be.false + }) + + it('Should strip exif for a jpg file that could be copied', async function () { + const input = buildAbsoluteFixturePath('exif.jpg') + expect(await hasTitleExif(input)).to.be.true + + await processImage({ path: input, destination: imageDestJPG, newSize: thumbnailSize, keepOriginal: true }) + await checkBuffers(input, imageDestJPG, false) + + expect(await hasTitleExif(imageDestJPG)).to.be.false + }) + + it('Should strip exif for png', async function () { + const input = buildAbsoluteFixturePath('exif.png') + expect(await hasTitleExif(input)).to.be.true + + await processImage({ path: input, destination: imageDestPNG, newSize: thumbnailSize, keepOriginal: true }) + expect(await hasTitleExif(imageDestPNG)).to.be.false + }) + + after(async function () { + await remove(imageDestDir) + }) +}) diff --git a/packages/tests/src/server-helpers/index.ts b/packages/tests/src/server-helpers/index.ts new file mode 100644 index 000000000..04a26560c --- /dev/null +++ b/packages/tests/src/server-helpers/index.ts @@ -0,0 +1,10 @@ +import './activitypub.js' +import './core-utils.js' +import './crypto.js' +import './dns.js' +import './image.js' +import './markdown.js' +import './mentions.js' +import './request.js' +import './validator.js' +import './version.js' diff --git a/packages/tests/src/server-helpers/markdown.ts b/packages/tests/src/server-helpers/markdown.ts new file mode 100644 index 000000000..96e3c34dc --- /dev/null +++ b/packages/tests/src/server-helpers/markdown.ts @@ -0,0 +1,39 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { mdToOneLinePlainText } from '@peertube/peertube-server/server/helpers/markdown.js' +import { expect } from 'chai' + +describe('Markdown helpers', function () { + + describe('Plain text', function () { + + it('Should convert a list to plain text', function () { + const result = mdToOneLinePlainText(`* list 1 +* list 2 +* list 3`) + + expect(result).to.equal('list 1, list 2, list 3') + }) + + it('Should convert a list with indentation to plain text', function () { + const result = mdToOneLinePlainText(`Hello: + * list 1 + * list 2 + * list 3`) + + expect(result).to.equal('Hello: list 1, list 2, list 3') + }) + + it('Should convert HTML to plain text', function () { + const result = mdToOneLinePlainText(`**Hello** coucou`) + + expect(result).to.equal('Hello coucou') + }) + + it('Should convert tags to plain text', function () { + const result = mdToOneLinePlainText(`#déconversion\n#newage\n#histoire`) + + expect(result).to.equal('#déconversion #newage #histoire') + }) + }) +}) diff --git a/packages/tests/src/server-helpers/mentions.ts b/packages/tests/src/server-helpers/mentions.ts new file mode 100644 index 000000000..153931d60 --- /dev/null +++ b/packages/tests/src/server-helpers/mentions.ts @@ -0,0 +1,17 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { extractMentions } from '@peertube/peertube-server/server/helpers/mentions.js' + +describe('Comment model', function () { + it('Should correctly extract mentions', async function () { + const text = '@florian @jean@localhost:9000 @flo @another@localhost:9000 @flo2@jean.com hello ' + + 'email@localhost:9000 coucou.com no? @chocobozzz @chocobozzz @end' + + const isOwned = true + + const result = extractMentions(text, isOwned).sort((a, b) => a.localeCompare(b)) + + expect(result).to.deep.equal([ 'another', 'chocobozzz', 'end', 'flo', 'florian', 'jean' ]) + }) +}) diff --git a/packages/tests/src/server-helpers/request.ts b/packages/tests/src/server-helpers/request.ts new file mode 100644 index 000000000..f4b9af52e --- /dev/null +++ b/packages/tests/src/server-helpers/request.ts @@ -0,0 +1,64 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { pathExists, remove } from 'fs-extra/esm' +import { join } from 'path' +import { wait } from '@peertube/peertube-core-utils' +import { root } from '@peertube/peertube-node-utils' +import { doRequest, doRequestAndSaveToFile } from '@peertube/peertube-server/server/helpers/requests.js' +import { Mock429 } from '@tests/shared/mock-servers/mock-429.js' +import { FIXTURE_URLS } from '@tests/shared/tests.js' + +describe('Request helpers', function () { + const destPath1 = join(root(), 'test-output-1.txt') + const destPath2 = join(root(), 'test-output-2.txt') + + it('Should throw an error when the bytes limit is exceeded for request', async function () { + try { + await doRequest(FIXTURE_URLS.file4K, { bodyKBLimit: 3 }) + } catch { + return + } + + throw new Error('No error thrown by do request') + }) + + it('Should throw an error when the bytes limit is exceeded for request and save file', async function () { + try { + await doRequestAndSaveToFile(FIXTURE_URLS.file4K, destPath1, { bodyKBLimit: 3 }) + } catch { + + await wait(500) + expect(await pathExists(destPath1)).to.be.false + return + } + + throw new Error('No error thrown by do request and save to file') + }) + + it('Should correctly retry on 429 error', async function () { + this.timeout(25000) + + const mock = new Mock429() + const port = await mock.initialize() + + const before = new Date().getTime() + await doRequest('http://127.0.0.1:' + port) + + expect(new Date().getTime() - before).to.be.greaterThan(2000) + + await mock.terminate() + }) + + it('Should succeed if the file is below the limit', async function () { + await doRequest(FIXTURE_URLS.file4K, { bodyKBLimit: 5 }) + await doRequestAndSaveToFile(FIXTURE_URLS.file4K, destPath2, { bodyKBLimit: 5 }) + + expect(await pathExists(destPath2)).to.be.true + }) + + after(async function () { + await remove(destPath1) + await remove(destPath2) + }) +}) diff --git a/packages/tests/src/server-helpers/validator.ts b/packages/tests/src/server-helpers/validator.ts new file mode 100644 index 000000000..792bd501c --- /dev/null +++ b/packages/tests/src/server-helpers/validator.ts @@ -0,0 +1,35 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { + isPluginStableOrUnstableVersionValid, + isPluginStableVersionValid +} from '@peertube/peertube-server/server/helpers/custom-validators/plugins.js' + +describe('Validators', function () { + + it('Should correctly check stable plugin versions', async function () { + expect(isPluginStableVersionValid('3.4.0')).to.be.true + expect(isPluginStableVersionValid('0.4.0')).to.be.true + expect(isPluginStableVersionValid('0.1.0')).to.be.true + + expect(isPluginStableVersionValid('0.1.0-beta-1')).to.be.false + expect(isPluginStableVersionValid('hello')).to.be.false + expect(isPluginStableVersionValid('0.x.a')).to.be.false + }) + + it('Should correctly check unstable plugin versions', async function () { + expect(isPluginStableOrUnstableVersionValid('3.4.0')).to.be.true + expect(isPluginStableOrUnstableVersionValid('0.4.0')).to.be.true + expect(isPluginStableOrUnstableVersionValid('0.1.0')).to.be.true + + expect(isPluginStableOrUnstableVersionValid('0.1.0-beta.1')).to.be.true + expect(isPluginStableOrUnstableVersionValid('0.1.0-alpha.45')).to.be.true + expect(isPluginStableOrUnstableVersionValid('0.1.0-rc.45')).to.be.true + + expect(isPluginStableOrUnstableVersionValid('hello')).to.be.false + expect(isPluginStableOrUnstableVersionValid('0.x.a')).to.be.false + expect(isPluginStableOrUnstableVersionValid('0.1.0-rc-45')).to.be.false + expect(isPluginStableOrUnstableVersionValid('0.1.0-rc.45d')).to.be.false + }) +}) diff --git a/packages/tests/src/server-helpers/version.ts b/packages/tests/src/server-helpers/version.ts new file mode 100644 index 000000000..76892d1e7 --- /dev/null +++ b/packages/tests/src/server-helpers/version.ts @@ -0,0 +1,36 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { compareSemVer } from '@peertube/peertube-core-utils' + +describe('Version', function () { + + it('Should correctly compare two stable versions', async function () { + expect(compareSemVer('3.4.0', '3.5.0')).to.be.below(0) + expect(compareSemVer('3.5.0', '3.4.0')).to.be.above(0) + + expect(compareSemVer('3.4.0', '4.1.0')).to.be.below(0) + expect(compareSemVer('4.1.0', '3.4.0')).to.be.above(0) + + expect(compareSemVer('3.4.0', '3.4.1')).to.be.below(0) + expect(compareSemVer('3.4.1', '3.4.0')).to.be.above(0) + }) + + it('Should correctly compare two unstable version', async function () { + expect(compareSemVer('3.4.0-alpha', '3.4.0-beta.1')).to.be.below(0) + expect(compareSemVer('3.4.0-alpha.1', '3.4.0-beta.1')).to.be.below(0) + expect(compareSemVer('3.4.0-beta.1', '3.4.0-beta.2')).to.be.below(0) + expect(compareSemVer('3.4.0-beta.1', '3.5.0-alpha.1')).to.be.below(0) + + expect(compareSemVer('3.4.0-alpha.1', '3.4.0-nightly.4')).to.be.below(0) + expect(compareSemVer('3.4.0-nightly.3', '3.4.0-nightly.4')).to.be.below(0) + expect(compareSemVer('3.3.0-nightly.5', '3.4.0-nightly.4')).to.be.below(0) + }) + + it('Should correctly compare a stable and unstable versions', async function () { + expect(compareSemVer('3.4.0', '3.4.1-beta.1')).to.be.below(0) + expect(compareSemVer('3.4.0-beta.1', '3.4.0-beta.2')).to.be.below(0) + expect(compareSemVer('3.4.0-beta.1', '3.4.0')).to.be.below(0) + expect(compareSemVer('3.4.0-nightly.4', '3.4.0')).to.be.below(0) + }) +}) diff --git a/packages/tests/src/server-lib/index.ts b/packages/tests/src/server-lib/index.ts new file mode 100644 index 000000000..873f53e15 --- /dev/null +++ b/packages/tests/src/server-lib/index.ts @@ -0,0 +1 @@ +export * from './video-constant-registry-factory.js' diff --git a/packages/tests/src/server-lib/video-constant-registry-factory.ts b/packages/tests/src/server-lib/video-constant-registry-factory.ts new file mode 100644 index 000000000..6bf2d1db6 --- /dev/null +++ b/packages/tests/src/server-lib/video-constant-registry-factory.ts @@ -0,0 +1,151 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions */ +import { expect } from 'chai' +import { VideoPlaylistPrivacyType, VideoPrivacyType } from '@peertube/peertube-models' +import { + VIDEO_CATEGORIES, + VIDEO_LANGUAGES, + VIDEO_LICENCES, + VIDEO_PLAYLIST_PRIVACIES, + VIDEO_PRIVACIES +} from '@peertube/peertube-server/server/initializers/constants.js' +import { VideoConstantManagerFactory } from '@peertube/peertube-server/server/lib/plugins/video-constant-manager-factory.js' + +describe('VideoConstantManagerFactory', function () { + const factory = new VideoConstantManagerFactory('peertube-plugin-constants') + + afterEach(() => { + factory.resetVideoConstants('peertube-plugin-constants') + }) + + describe('VideoCategoryManager', () => { + const videoCategoryManager = factory.createVideoConstantManager('category') + + it('Should be able to list all video category constants', () => { + const constants = videoCategoryManager.getConstants() + expect(constants).to.deep.equal(VIDEO_CATEGORIES) + }) + + it('Should be able to delete a video category constant', () => { + const successfullyDeleted = videoCategoryManager.deleteConstant(1) + expect(successfullyDeleted).to.be.true + expect(videoCategoryManager.getConstantValue(1)).to.be.undefined + }) + + it('Should be able to add a video category constant', () => { + const successfullyAdded = videoCategoryManager.addConstant(42, 'The meaning of life') + expect(successfullyAdded).to.be.true + expect(videoCategoryManager.getConstantValue(42)).to.equal('The meaning of life') + }) + + it('Should be able to reset video category constants', () => { + videoCategoryManager.deleteConstant(1) + videoCategoryManager.resetConstants() + expect(videoCategoryManager.getConstantValue(1)).not.be.undefined + }) + }) + + describe('VideoLicenceManager', () => { + const videoLicenceManager = factory.createVideoConstantManager('licence') + it('Should be able to list all video licence constants', () => { + const constants = videoLicenceManager.getConstants() + expect(constants).to.deep.equal(VIDEO_LICENCES) + }) + + it('Should be able to delete a video licence constant', () => { + const successfullyDeleted = videoLicenceManager.deleteConstant(1) + expect(successfullyDeleted).to.be.true + expect(videoLicenceManager.getConstantValue(1)).to.be.undefined + }) + + it('Should be able to add a video licence constant', () => { + const successfullyAdded = videoLicenceManager.addConstant(42, 'European Union Public Licence') + expect(successfullyAdded).to.be.true + expect(videoLicenceManager.getConstantValue(42 as any)).to.equal('European Union Public Licence') + }) + + it('Should be able to reset video licence constants', () => { + videoLicenceManager.deleteConstant(1) + videoLicenceManager.resetConstants() + expect(videoLicenceManager.getConstantValue(1)).not.be.undefined + }) + }) + + describe('PlaylistPrivacyManager', () => { + const playlistPrivacyManager = factory.createVideoConstantManager('playlistPrivacy') + it('Should be able to list all video playlist privacy constants', () => { + const constants = playlistPrivacyManager.getConstants() + expect(constants).to.deep.equal(VIDEO_PLAYLIST_PRIVACIES) + }) + + it('Should be able to delete a video playlist privacy constant', () => { + const successfullyDeleted = playlistPrivacyManager.deleteConstant(1) + expect(successfullyDeleted).to.be.true + expect(playlistPrivacyManager.getConstantValue(1)).to.be.undefined + }) + + it('Should be able to add a video playlist privacy constant', () => { + const successfullyAdded = playlistPrivacyManager.addConstant(42 as any, 'Friends only') + expect(successfullyAdded).to.be.true + expect(playlistPrivacyManager.getConstantValue(42 as any)).to.equal('Friends only') + }) + + it('Should be able to reset video playlist privacy constants', () => { + playlistPrivacyManager.deleteConstant(1) + playlistPrivacyManager.resetConstants() + expect(playlistPrivacyManager.getConstantValue(1)).not.be.undefined + }) + }) + + describe('VideoPrivacyManager', () => { + const videoPrivacyManager = factory.createVideoConstantManager('privacy') + it('Should be able to list all video privacy constants', () => { + const constants = videoPrivacyManager.getConstants() + expect(constants).to.deep.equal(VIDEO_PRIVACIES) + }) + + it('Should be able to delete a video privacy constant', () => { + const successfullyDeleted = videoPrivacyManager.deleteConstant(1) + expect(successfullyDeleted).to.be.true + expect(videoPrivacyManager.getConstantValue(1)).to.be.undefined + }) + + it('Should be able to add a video privacy constant', () => { + const successfullyAdded = videoPrivacyManager.addConstant(42 as any, 'Friends only') + expect(successfullyAdded).to.be.true + expect(videoPrivacyManager.getConstantValue(42 as any)).to.equal('Friends only') + }) + + it('Should be able to reset video privacy constants', () => { + videoPrivacyManager.deleteConstant(1) + videoPrivacyManager.resetConstants() + expect(videoPrivacyManager.getConstantValue(1)).not.be.undefined + }) + }) + + describe('VideoLanguageManager', () => { + const videoLanguageManager = factory.createVideoConstantManager('language') + it('Should be able to list all video language constants', () => { + const constants = videoLanguageManager.getConstants() + expect(constants).to.deep.equal(VIDEO_LANGUAGES) + }) + + it('Should be able to add a video language constant', () => { + const successfullyAdded = videoLanguageManager.addConstant('fr', 'Fr occitan') + expect(successfullyAdded).to.be.true + expect(videoLanguageManager.getConstantValue('fr')).to.equal('Fr occitan') + }) + + it('Should be able to delete a video language constant', () => { + videoLanguageManager.addConstant('fr', 'Fr occitan') + const successfullyDeleted = videoLanguageManager.deleteConstant('fr') + expect(successfullyDeleted).to.be.true + expect(videoLanguageManager.getConstantValue('fr')).to.be.undefined + }) + + it('Should be able to reset video language constants', () => { + videoLanguageManager.addConstant('fr', 'Fr occitan') + videoLanguageManager.resetConstants() + expect(videoLanguageManager.getConstantValue('fr')).to.be.undefined + }) + }) +}) diff --git a/packages/tests/src/shared/actors.ts b/packages/tests/src/shared/actors.ts new file mode 100644 index 000000000..02d507a49 --- /dev/null +++ b/packages/tests/src/shared/actors.ts @@ -0,0 +1,70 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { pathExists } from 'fs-extra/esm' +import { readdir } from 'fs/promises' +import { Account, VideoChannel } from '@peertube/peertube-models' +import { PeerTubeServer } from '@peertube/peertube-server-commands' + +async function expectChannelsFollows (options: { + server: PeerTubeServer + handle: string + followers: number + following: number +}) { + const { server } = options + const { data } = await server.channels.list() + + return expectActorFollow({ ...options, data }) +} + +async function expectAccountFollows (options: { + server: PeerTubeServer + handle: string + followers: number + following: number +}) { + const { server } = options + const { data } = await server.accounts.list() + + return expectActorFollow({ ...options, data }) +} + +async function checkActorFilesWereRemoved (filename: string, server: PeerTubeServer) { + for (const directory of [ 'avatars' ]) { + const directoryPath = server.getDirectoryPath(directory) + + const directoryExists = await pathExists(directoryPath) + expect(directoryExists).to.be.true + + const files = await readdir(directoryPath) + for (const file of files) { + expect(file).to.not.contain(filename) + } + } +} + +export { + expectAccountFollows, + expectChannelsFollows, + checkActorFilesWereRemoved +} + +// --------------------------------------------------------------------------- + +function expectActorFollow (options: { + server: PeerTubeServer + data: (Account | VideoChannel)[] + handle: string + followers: number + following: number +}) { + const { server, data, handle, followers, following } = options + + const actor = data.find(a => a.name + '@' + a.host === handle) + const message = `${handle} on ${server.url}` + + expect(actor, message).to.exist + expect(actor.followersCount).to.equal(followers, message) + expect(actor.followingCount).to.equal(following, message) +} diff --git a/packages/tests/src/shared/captions.ts b/packages/tests/src/shared/captions.ts new file mode 100644 index 000000000..436cf8dcc --- /dev/null +++ b/packages/tests/src/shared/captions.ts @@ -0,0 +1,21 @@ +import { expect } from 'chai' +import request from 'supertest' +import { HttpStatusCode } from '@peertube/peertube-models' + +async function testCaptionFile (url: string, captionPath: string, toTest: RegExp | string) { + const res = await request(url) + .get(captionPath) + .expect(HttpStatusCode.OK_200) + + if (toTest instanceof RegExp) { + expect(res.text).to.match(toTest) + } else { + expect(res.text).to.contain(toTest) + } +} + +// --------------------------------------------------------------------------- + +export { + testCaptionFile +} diff --git a/packages/tests/src/shared/checks.ts b/packages/tests/src/shared/checks.ts new file mode 100644 index 000000000..fea618a30 --- /dev/null +++ b/packages/tests/src/shared/checks.ts @@ -0,0 +1,177 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */ + +import { expect } from 'chai' +import { pathExists } from 'fs-extra/esm' +import { readFile } from 'fs/promises' +import { join } from 'path' +import { HttpStatusCode } from '@peertube/peertube-models' +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import { makeGetRequest, PeerTubeServer } from '@peertube/peertube-server-commands' + +// Default interval -> 5 minutes +function dateIsValid (dateString: string | Date, interval = 300000) { + const dateToCheck = new Date(dateString) + const now = new Date() + + return Math.abs(now.getTime() - dateToCheck.getTime()) <= interval +} + +function expectStartWith (str: string, start: string) { + expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.true +} + +function expectNotStartWith (str: string, start: string) { + expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.false +} + +function expectEndWith (str: string, end: string) { + expect(str.endsWith(end), `${str} does not end with ${end}`).to.be.true +} + +// --------------------------------------------------------------------------- + +async function expectLogDoesNotContain (server: PeerTubeServer, str: string) { + const content = await server.servers.getLogContent() + + expect(content.toString()).to.not.contain(str) +} + +async function expectLogContain (server: PeerTubeServer, str: string) { + const content = await server.servers.getLogContent() + + expect(content.toString()).to.contain(str) +} + +async function testImageSize (url: string, imageName: string, imageHTTPPath: string, extension = '.jpg') { + const res = await makeGetRequest({ + url, + path: imageHTTPPath, + expectedStatus: HttpStatusCode.OK_200 + }) + + const body = res.body + + const data = await readFile(buildAbsoluteFixturePath(imageName + extension)) + const minLength = data.length - ((40 * data.length) / 100) + const maxLength = data.length + ((40 * data.length) / 100) + + expect(body.length).to.be.above(minLength, 'the generated image is way smaller than the recorded fixture') + expect(body.length).to.be.below(maxLength, 'the generated image is way larger than the recorded fixture') +} + +async function testImageGeneratedByFFmpeg (url: string, imageName: string, imageHTTPPath: string, extension = '.jpg') { + if (process.env.ENABLE_FFMPEG_THUMBNAIL_PIXEL_COMPARISON_TESTS !== 'true') { + console.log( + 'Pixel comparison of image generated by ffmpeg is disabled. ' + + 'You can enable it using `ENABLE_FFMPEG_THUMBNAIL_PIXEL_COMPARISON_TESTS=true env variable') + return + } + + return testImage(url, imageName, imageHTTPPath, extension) +} + +async function testImage (url: string, imageName: string, imageHTTPPath: string, extension = '.jpg') { + const res = await makeGetRequest({ + url, + path: imageHTTPPath, + expectedStatus: HttpStatusCode.OK_200 + }) + + const body = res.body + const data = await readFile(buildAbsoluteFixturePath(imageName + extension)) + + const { PNG } = await import('pngjs') + const JPEG = await import('jpeg-js') + const pixelmatch = (await import('pixelmatch')).default + + const img1 = imageHTTPPath.endsWith('.png') + ? PNG.sync.read(body) + : JPEG.decode(body) + + const img2 = extension === '.png' + ? PNG.sync.read(data) + : JPEG.decode(data) + + const result = pixelmatch(img1.data, img2.data, null, img1.width, img1.height, { threshold: 0.1 }) + + expect(result).to.equal(0, `${imageHTTPPath} image is not the same as ${imageName}${extension}`) +} + +async function testFileExistsOrNot (server: PeerTubeServer, directory: string, filePath: string, exist: boolean) { + const base = server.servers.buildDirectory(directory) + + expect(await pathExists(join(base, filePath))).to.equal(exist) +} + +// --------------------------------------------------------------------------- + +function checkBadStartPagination (url: string, path: string, token?: string, query = {}) { + return makeGetRequest({ + url, + path, + token, + query: { ...query, start: 'hello' }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) +} + +async function checkBadCountPagination (url: string, path: string, token?: string, query = {}) { + await makeGetRequest({ + url, + path, + token, + query: { ...query, count: 'hello' }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) + + await makeGetRequest({ + url, + path, + token, + query: { ...query, count: 2000 }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) +} + +function checkBadSortPagination (url: string, path: string, token?: string, query = {}) { + return makeGetRequest({ + url, + path, + token, + query: { ...query, sort: 'hello' }, + expectedStatus: HttpStatusCode.BAD_REQUEST_400 + }) +} + +// --------------------------------------------------------------------------- + +async function checkVideoDuration (server: PeerTubeServer, videoUUID: string, duration: number) { + const video = await server.videos.get({ id: videoUUID }) + + expect(video.duration).to.be.approximately(duration, 1) + + for (const file of video.files) { + const metadata = await server.videos.getFileMetadata({ url: file.metadataUrl }) + + for (const stream of metadata.streams) { + expect(Math.round(stream.duration)).to.be.approximately(duration, 1) + } + } +} + +export { + dateIsValid, + testImageGeneratedByFFmpeg, + testImageSize, + testImage, + expectLogDoesNotContain, + testFileExistsOrNot, + expectStartWith, + expectNotStartWith, + expectEndWith, + checkBadStartPagination, + checkBadCountPagination, + checkBadSortPagination, + checkVideoDuration, + expectLogContain +} diff --git a/packages/tests/src/shared/directories.ts b/packages/tests/src/shared/directories.ts new file mode 100644 index 000000000..f21e7b7c6 --- /dev/null +++ b/packages/tests/src/shared/directories.ts @@ -0,0 +1,44 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { pathExists } from 'fs-extra/esm' +import { readdir } from 'fs/promises' +import { homedir } from 'os' +import { join } from 'path' +import { PeerTubeServer } from '@peertube/peertube-server-commands' +import { PeerTubeRunnerProcess } from './peertube-runner-process.js' + +export async function checkTmpIsEmpty (server: PeerTubeServer) { + await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls', 'resumable-uploads' ]) + + if (await pathExists(server.getDirectoryPath('tmp/hls'))) { + await checkDirectoryIsEmpty(server, 'tmp/hls') + } +} + +export async function checkPersistentTmpIsEmpty (server: PeerTubeServer) { + await checkDirectoryIsEmpty(server, 'tmp-persistent') +} + +export async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) { + const directoryPath = server.getDirectoryPath(directory) + + const directoryExists = await pathExists(directoryPath) + expect(directoryExists).to.be.true + + const files = await readdir(directoryPath) + const filtered = files.filter(f => exceptions.includes(f) === false) + + expect(filtered).to.have.lengthOf(0) +} + +export async function checkPeerTubeRunnerCacheIsEmpty (runner: PeerTubeRunnerProcess) { + const directoryPath = join(homedir(), '.cache', 'peertube-runner-nodejs', runner.getId(), 'transcoding') + + const directoryExists = await pathExists(directoryPath) + expect(directoryExists).to.be.true + + const files = await readdir(directoryPath) + + expect(files, 'Directory content: ' + files.join(', ')).to.have.lengthOf(0) +} diff --git a/packages/tests/src/shared/generate.ts b/packages/tests/src/shared/generate.ts new file mode 100644 index 000000000..ab2ecaf40 --- /dev/null +++ b/packages/tests/src/shared/generate.ts @@ -0,0 +1,79 @@ +import { expect } from 'chai' +import { ensureDir, pathExists } from 'fs-extra/esm' +import { dirname } from 'path' +import { getMaxTheoreticalBitrate } from '@peertube/peertube-core-utils' +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import { getVideoStreamBitrate, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@peertube/peertube-ffmpeg' + +async function ensureHasTooBigBitrate (fixturePath: string) { + const bitrate = await getVideoStreamBitrate(fixturePath) + const dataResolution = await getVideoStreamDimensionsInfo(fixturePath) + const fps = await getVideoStreamFPS(fixturePath) + + const maxBitrate = getMaxTheoreticalBitrate({ ...dataResolution, fps }) + expect(bitrate).to.be.above(maxBitrate) +} + +async function generateHighBitrateVideo () { + const tempFixturePath = buildAbsoluteFixturePath('video_high_bitrate_1080p.mp4', true) + + await ensureDir(dirname(tempFixturePath)) + + const exists = await pathExists(tempFixturePath) + + if (!exists) { + const ffmpeg = (await import('fluent-ffmpeg')).default + + console.log('Generating high bitrate video.') + + // Generate a random, high bitrate video on the fly, so we don't have to include + // a large file in the repo. The video needs to have a certain minimum length so + // that FFmpeg properly applies bitrate limits. + // https://stackoverflow.com/a/15795112 + return new Promise((res, rej) => { + ffmpeg() + .outputOptions([ '-f rawvideo', '-video_size 1920x1080', '-i /dev/urandom' ]) + .outputOptions([ '-ac 2', '-f s16le', '-i /dev/urandom', '-t 10' ]) + .outputOptions([ '-maxrate 10M', '-bufsize 10M' ]) + .output(tempFixturePath) + .on('error', rej) + .on('end', () => res(tempFixturePath)) + .run() + }) + } + + await ensureHasTooBigBitrate(tempFixturePath) + + return tempFixturePath +} + +async function generateVideoWithFramerate (fps = 60) { + const tempFixturePath = buildAbsoluteFixturePath(`video_${fps}fps.mp4`, true) + + await ensureDir(dirname(tempFixturePath)) + + const exists = await pathExists(tempFixturePath) + if (!exists) { + const ffmpeg = (await import('fluent-ffmpeg')).default + + console.log('Generating video with framerate %d.', fps) + + return new Promise((res, rej) => { + ffmpeg() + .outputOptions([ '-f rawvideo', '-video_size 1280x720', '-i /dev/urandom' ]) + .outputOptions([ '-ac 2', '-f s16le', '-i /dev/urandom', '-t 10' ]) + .outputOptions([ `-r ${fps}` ]) + .output(tempFixturePath) + .on('error', rej) + .on('end', () => res(tempFixturePath)) + .run() + }) + } + + return tempFixturePath +} + +export { + generateHighBitrateVideo, + generateVideoWithFramerate +} diff --git a/packages/tests/src/shared/live.ts b/packages/tests/src/shared/live.ts new file mode 100644 index 000000000..9c7991b0d --- /dev/null +++ b/packages/tests/src/shared/live.ts @@ -0,0 +1,186 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { pathExists } from 'fs-extra/esm' +import { readdir } from 'fs/promises' +import { join } from 'path' +import { sha1 } from '@peertube/peertube-node-utils' +import { LiveVideo, VideoStreamingPlaylistType } from '@peertube/peertube-models' +import { ObjectStorageCommand, PeerTubeServer } from '@peertube/peertube-server-commands' +import { SQLCommand } from './sql-command.js' +import { checkLiveSegmentHash, checkResolutionsInMasterPlaylist } from './streaming-playlists.js' + +async function checkLiveCleanup (options: { + server: PeerTubeServer + videoUUID: string + permanent: boolean + savedResolutions?: number[] +}) { + const { server, videoUUID, permanent, savedResolutions = [] } = options + + const basePath = server.servers.buildDirectory('streaming-playlists') + const hlsPath = join(basePath, 'hls', videoUUID) + + if (permanent) { + if (!await pathExists(hlsPath)) return + + const files = await readdir(hlsPath) + expect(files).to.have.lengthOf(0) + return + } + + if (savedResolutions.length === 0) { + return checkUnsavedLiveCleanup(server, videoUUID, hlsPath) + } + + return checkSavedLiveCleanup(hlsPath, savedResolutions) +} + +// --------------------------------------------------------------------------- + +async function testLiveVideoResolutions (options: { + sqlCommand: SQLCommand + originServer: PeerTubeServer + + servers: PeerTubeServer[] + liveVideoId: string + resolutions: number[] + transcoded: boolean + + objectStorage?: ObjectStorageCommand + objectStorageBaseUrl?: string +}) { + const { + originServer, + sqlCommand, + servers, + liveVideoId, + resolutions, + transcoded, + objectStorage, + objectStorageBaseUrl = objectStorage?.getMockPlaylistBaseUrl() + } = options + + for (const server of servers) { + const { data } = await server.videos.list() + expect(data.find(v => v.uuid === liveVideoId)).to.exist + + const video = await server.videos.get({ id: liveVideoId }) + expect(video.streamingPlaylists).to.have.lengthOf(1) + + const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS) + expect(hlsPlaylist).to.exist + expect(hlsPlaylist.files).to.have.lengthOf(0) // Only fragmented mp4 files are displayed + + await checkResolutionsInMasterPlaylist({ + server, + playlistUrl: hlsPlaylist.playlistUrl, + resolutions, + transcoded, + withRetry: !!objectStorage + }) + + if (objectStorage) { + expect(hlsPlaylist.playlistUrl).to.contain(objectStorageBaseUrl) + } + + for (let i = 0; i < resolutions.length; i++) { + const segmentNum = 3 + const segmentName = `${i}-00000${segmentNum}.ts` + await originServer.live.waitUntilSegmentGeneration({ + server: originServer, + videoUUID: video.uuid, + playlistNumber: i, + segment: segmentNum, + objectStorage, + objectStorageBaseUrl + }) + + const baseUrl = objectStorage + ? join(objectStorageBaseUrl, 'hls') + : originServer.url + '/static/streaming-playlists/hls' + + if (objectStorage) { + expect(hlsPlaylist.segmentsSha256Url).to.contain(objectStorageBaseUrl) + } + + const subPlaylist = await originServer.streamingPlaylists.get({ + url: `${baseUrl}/${video.uuid}/${i}.m3u8`, + withRetry: !!objectStorage // With object storage, the request may fail because of inconsistent data in S3 + }) + + expect(subPlaylist).to.contain(segmentName) + + await checkLiveSegmentHash({ + server, + baseUrlSegment: baseUrl, + videoUUID: video.uuid, + segmentName, + hlsPlaylist, + withRetry: !!objectStorage // With object storage, the request may fail because of inconsistent data in S3 + }) + + if (originServer.internalServerNumber === server.internalServerNumber) { + const infohash = sha1(`${2 + hlsPlaylist.playlistUrl}+V${i}`) + const dbInfohashes = await sqlCommand.getPlaylistInfohash(hlsPlaylist.id) + + expect(dbInfohashes).to.include(infohash) + } + } + } +} + +// --------------------------------------------------------------------------- + +export { + checkLiveCleanup, + testLiveVideoResolutions +} + +// --------------------------------------------------------------------------- + +async function checkSavedLiveCleanup (hlsPath: string, savedResolutions: number[] = []) { + const files = await readdir(hlsPath) + + // fragmented file and playlist per resolution + master playlist + segments sha256 json file + expect(files, `Directory content: ${files.join(', ')}`).to.have.lengthOf(savedResolutions.length * 2 + 2) + + for (const resolution of savedResolutions) { + const fragmentedFile = files.find(f => f.endsWith(`-${resolution}-fragmented.mp4`)) + expect(fragmentedFile).to.exist + + const playlistFile = files.find(f => f.endsWith(`${resolution}.m3u8`)) + expect(playlistFile).to.exist + } + + const masterPlaylistFile = files.find(f => f.endsWith('-master.m3u8')) + expect(masterPlaylistFile).to.exist + + const shaFile = files.find(f => f.endsWith('-segments-sha256.json')) + expect(shaFile).to.exist +} + +async function checkUnsavedLiveCleanup (server: PeerTubeServer, videoUUID: string, hlsPath: string) { + let live: LiveVideo + + try { + live = await server.live.get({ videoId: videoUUID }) + } catch {} + + if (live?.permanentLive) { + expect(await pathExists(hlsPath)).to.be.true + + const hlsFiles = await readdir(hlsPath) + expect(hlsFiles).to.have.lengthOf(1) // Only replays directory + + const replayDir = join(hlsPath, 'replay') + expect(await pathExists(replayDir)).to.be.true + + const replayFiles = await readdir(join(hlsPath, 'replay')) + expect(replayFiles).to.have.lengthOf(0) + + return + } + + expect(await pathExists(hlsPath)).to.be.false +} diff --git a/packages/tests/src/shared/mock-servers/index.ts b/packages/tests/src/shared/mock-servers/index.ts new file mode 100644 index 000000000..9d1c63c67 --- /dev/null +++ b/packages/tests/src/shared/mock-servers/index.ts @@ -0,0 +1,8 @@ +export * from './mock-429.js' +export * from './mock-email.js' +export * from './mock-http.js' +export * from './mock-instances-index.js' +export * from './mock-joinpeertube-versions.js' +export * from './mock-object-storage.js' +export * from './mock-plugin-blocklist.js' +export * from './mock-proxy.js' diff --git a/packages/tests/src/shared/mock-servers/mock-429.ts b/packages/tests/src/shared/mock-servers/mock-429.ts new file mode 100644 index 000000000..5fcb1447d --- /dev/null +++ b/packages/tests/src/shared/mock-servers/mock-429.ts @@ -0,0 +1,33 @@ +import express from 'express' +import { Server } from 'http' +import { getPort, randomListen, terminateServer } from './shared.js' + +export class Mock429 { + private server: Server + private responseSent = false + + async initialize () { + const app = express() + + app.get('/', (req: express.Request, res: express.Response, next: express.NextFunction) => { + + if (!this.responseSent) { + this.responseSent = true + + // Retry after 5 seconds + res.header('retry-after', '2') + return res.sendStatus(429) + } + + return res.sendStatus(200) + }) + + this.server = await randomListen(app) + + return getPort(this.server) + } + + terminate () { + return terminateServer(this.server) + } +} diff --git a/packages/tests/src/shared/mock-servers/mock-email.ts b/packages/tests/src/shared/mock-servers/mock-email.ts new file mode 100644 index 000000000..7c618e57f --- /dev/null +++ b/packages/tests/src/shared/mock-servers/mock-email.ts @@ -0,0 +1,62 @@ +import MailDev from '@peertube/maildev' +import { randomInt } from '@peertube/peertube-core-utils' +import { parallelTests } from '@peertube/peertube-node-utils' + +class MockSmtpServer { + + private static instance: MockSmtpServer + private started = false + private maildev: any + private emails: object[] + + private constructor () { } + + collectEmails (emailsCollection: object[]) { + return new Promise((res, rej) => { + const port = parallelTests() ? randomInt(1025, 2000) : 1025 + this.emails = emailsCollection + + if (this.started) { + return res(undefined) + } + + this.maildev = new MailDev({ + ip: '127.0.0.1', + smtp: port, + disableWeb: true, + silent: true + }) + + this.maildev.on('new', email => { + this.emails.push(email) + }) + + this.maildev.listen(err => { + if (err) return rej(err) + + this.started = true + + return res(port) + }) + }) + } + + kill () { + if (!this.maildev) return + + this.maildev.close() + + this.maildev = null + MockSmtpServer.instance = null + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} + +// --------------------------------------------------------------------------- + +export { + MockSmtpServer +} diff --git a/packages/tests/src/shared/mock-servers/mock-http.ts b/packages/tests/src/shared/mock-servers/mock-http.ts new file mode 100644 index 000000000..bc1a9ce91 --- /dev/null +++ b/packages/tests/src/shared/mock-servers/mock-http.ts @@ -0,0 +1,23 @@ +import express from 'express' +import { Server } from 'http' +import { getPort, randomListen, terminateServer } from './shared.js' + +export class MockHTTP { + private server: Server + + async initialize () { + const app = express() + + app.get('/*', (req: express.Request, res: express.Response, next: express.NextFunction) => { + return res.sendStatus(200) + }) + + this.server = await randomListen(app) + + return getPort(this.server) + } + + terminate () { + return terminateServer(this.server) + } +} diff --git a/packages/tests/src/shared/mock-servers/mock-instances-index.ts b/packages/tests/src/shared/mock-servers/mock-instances-index.ts new file mode 100644 index 000000000..a21367358 --- /dev/null +++ b/packages/tests/src/shared/mock-servers/mock-instances-index.ts @@ -0,0 +1,46 @@ +import express from 'express' +import { Server } from 'http' +import { getPort, randomListen, terminateServer } from './shared.js' + +export class MockInstancesIndex { + private server: Server + + private readonly indexInstances: { host: string, createdAt: string }[] = [] + + async initialize () { + const app = express() + + app.use('/', (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (process.env.DEBUG) console.log('Receiving request on mocked server %s.', req.url) + + return next() + }) + + app.get('/api/v1/instances/hosts', (req: express.Request, res: express.Response) => { + const since = req.query.since + + const filtered = this.indexInstances.filter(i => { + if (!since) return true + + return i.createdAt > since + }) + + return res.json({ + total: filtered.length, + data: filtered + }) + }) + + this.server = await randomListen(app) + + return getPort(this.server) + } + + addInstance (host: string) { + this.indexInstances.push({ host, createdAt: new Date().toISOString() }) + } + + terminate () { + return terminateServer(this.server) + } +} diff --git a/packages/tests/src/shared/mock-servers/mock-joinpeertube-versions.ts b/packages/tests/src/shared/mock-servers/mock-joinpeertube-versions.ts new file mode 100644 index 000000000..0783165e4 --- /dev/null +++ b/packages/tests/src/shared/mock-servers/mock-joinpeertube-versions.ts @@ -0,0 +1,34 @@ +import express from 'express' +import { Server } from 'http' +import { getPort, randomListen } from './shared.js' + +export class MockJoinPeerTubeVersions { + private server: Server + private latestVersion: string + + async initialize () { + const app = express() + + app.use('/', (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (process.env.DEBUG) console.log('Receiving request on mocked server %s.', req.url) + + return next() + }) + + app.get('/versions.json', (req: express.Request, res: express.Response) => { + return res.json({ + peertube: { + latestVersion: this.latestVersion + } + }) + }) + + this.server = await randomListen(app) + + return getPort(this.server) + } + + setLatestVersion (latestVersion: string) { + this.latestVersion = latestVersion + } +} diff --git a/packages/tests/src/shared/mock-servers/mock-object-storage.ts b/packages/tests/src/shared/mock-servers/mock-object-storage.ts new file mode 100644 index 000000000..f97c57fd7 --- /dev/null +++ b/packages/tests/src/shared/mock-servers/mock-object-storage.ts @@ -0,0 +1,41 @@ +import express from 'express' +import got, { RequestError } from 'got' +import { Server } from 'http' +import { pipeline } from 'stream' +import { ObjectStorageCommand } from '@peertube/peertube-server-commands' +import { getPort, randomListen, terminateServer } from './shared.js' + +export class MockObjectStorageProxy { + private server: Server + + async initialize () { + const app = express() + + app.get('/:bucketName/:path(*)', (req: express.Request, res: express.Response, next: express.NextFunction) => { + const url = `http://${req.params.bucketName}.${ObjectStorageCommand.getMockEndpointHost()}/${req.params.path}` + + if (process.env.DEBUG) { + console.log('Receiving request on mocked server %s.', req.url) + console.log('Proxifying request to %s', url) + } + + return pipeline( + got.stream(url, { throwHttpErrors: false }), + res, + (err: RequestError) => { + if (!err) return + + console.error('Pipeline failed.', err) + } + ) + }) + + this.server = await randomListen(app) + + return getPort(this.server) + } + + terminate () { + return terminateServer(this.server) + } +} diff --git a/packages/tests/src/shared/mock-servers/mock-plugin-blocklist.ts b/packages/tests/src/shared/mock-servers/mock-plugin-blocklist.ts new file mode 100644 index 000000000..c0b6518ba --- /dev/null +++ b/packages/tests/src/shared/mock-servers/mock-plugin-blocklist.ts @@ -0,0 +1,36 @@ +import express, { Request, Response } from 'express' +import { Server } from 'http' +import { getPort, randomListen, terminateServer } from './shared.js' + +type BlocklistResponse = { + data: { + value: string + action?: 'add' | 'remove' + updatedAt?: string + }[] +} + +export class MockBlocklist { + private body: BlocklistResponse + private server: Server + + async initialize () { + const app = express() + + app.get('/blocklist', (req: Request, res: Response) => { + return res.json(this.body) + }) + + this.server = await randomListen(app) + + return getPort(this.server) + } + + replace (body: BlocklistResponse) { + this.body = body + } + + terminate () { + return terminateServer(this.server) + } +} diff --git a/packages/tests/src/shared/mock-servers/mock-proxy.ts b/packages/tests/src/shared/mock-servers/mock-proxy.ts new file mode 100644 index 000000000..e731670d8 --- /dev/null +++ b/packages/tests/src/shared/mock-servers/mock-proxy.ts @@ -0,0 +1,24 @@ +import { createServer, Server } from 'http' +import { createProxy } from 'proxy' +import { getPort, terminateServer } from './shared.js' + +class MockProxy { + private server: Server + + initialize () { + return new Promise(res => { + this.server = createProxy(createServer()) + this.server.listen(0, () => res(getPort(this.server))) + }) + } + + terminate () { + return terminateServer(this.server) + } +} + +// --------------------------------------------------------------------------- + +export { + MockProxy +} diff --git a/server/tests/shared/mock-servers/shared.ts b/packages/tests/src/shared/mock-servers/shared.ts similarity index 100% rename from server/tests/shared/mock-servers/shared.ts rename to packages/tests/src/shared/mock-servers/shared.ts diff --git a/packages/tests/src/shared/notifications.ts b/packages/tests/src/shared/notifications.ts new file mode 100644 index 000000000..3accd7322 --- /dev/null +++ b/packages/tests/src/shared/notifications.ts @@ -0,0 +1,891 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { + AbuseState, + AbuseStateType, + PluginType_Type, + UserNotification, + UserNotificationSetting, + UserNotificationSettingValue, + UserNotificationType +} from '@peertube/peertube-models' +import { + ConfigCommand, + PeerTubeServer, + createMultipleServers, + doubleFollow, + setAccessTokensToServers, + setDefaultAccountAvatar, + setDefaultChannelAvatar, + setDefaultVideoChannel +} from '@peertube/peertube-server-commands' +import { expect } from 'chai' +import { inspect } from 'util' +import { MockSmtpServer } from './mock-servers/index.js' + +type CheckerBaseParams = { + server: PeerTubeServer + emails: any[] + socketNotifications: UserNotification[] + token: string + check?: { web: boolean, mail: boolean } +} + +type CheckerType = 'presence' | 'absence' + +function getAllNotificationsSettings (): UserNotificationSetting { + return { + newVideoFromSubscription: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + newCommentOnMyVideo: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + abuseAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + videoAutoBlacklistAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + blacklistOnMyVideo: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + myVideoImportFinished: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + myVideoPublished: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + commentMention: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + newFollow: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + newUserRegistration: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + newInstanceFollower: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + abuseNewMessage: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + abuseStateChange: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + autoInstanceFollowing: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + newPeerTubeVersion: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + myVideoStudioEditionFinished: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + newPluginVersion: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL + } +} + +async function checkNewVideoFromSubscription (options: CheckerBaseParams & { + videoName: string + shortUUID: string + checkType: CheckerType +}) { + const { videoName, shortUUID } = options + const notificationType = UserNotificationType.NEW_VIDEO_FROM_SUBSCRIPTION + + function notificationChecker (notification: UserNotification, checkType: CheckerType) { + if (checkType === 'presence') { + expect(notification).to.not.be.undefined + expect(notification.type).to.equal(notificationType) + + checkVideo(notification.video, videoName, shortUUID) + checkActor(notification.video.channel) + } else { + expect(notification).to.satisfy((n: UserNotification) => { + return n === undefined || n.type !== UserNotificationType.NEW_VIDEO_FROM_SUBSCRIPTION || n.video.name !== videoName + }) + } + } + + function emailNotificationFinder (email: object) { + const text = email['text'] + return text.indexOf(shortUUID) !== -1 && text.indexOf('Your subscription') !== -1 + } + + await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) +} + +async function checkVideoIsPublished (options: CheckerBaseParams & { + videoName: string + shortUUID: string + checkType: CheckerType +}) { + const { videoName, shortUUID } = options + const notificationType = UserNotificationType.MY_VIDEO_PUBLISHED + + function notificationChecker (notification: UserNotification, checkType: CheckerType) { + if (checkType === 'presence') { + expect(notification).to.not.be.undefined + expect(notification.type).to.equal(notificationType) + + checkVideo(notification.video, videoName, shortUUID) + checkActor(notification.video.channel) + } else { + expect(notification.video).to.satisfy(v => v === undefined || v.name !== videoName) + } + } + + function emailNotificationFinder (email: object) { + const text: string = email['text'] + return text.includes(shortUUID) && text.includes('Your video') + } + + await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) +} + +async function checkVideoStudioEditionIsFinished (options: CheckerBaseParams & { + videoName: string + shortUUID: string + checkType: CheckerType +}) { + const { videoName, shortUUID } = options + const notificationType = UserNotificationType.MY_VIDEO_STUDIO_EDITION_FINISHED + + function notificationChecker (notification: UserNotification, checkType: CheckerType) { + if (checkType === 'presence') { + expect(notification).to.not.be.undefined + expect(notification.type).to.equal(notificationType) + + checkVideo(notification.video, videoName, shortUUID) + checkActor(notification.video.channel) + } else { + expect(notification.video).to.satisfy(v => v === undefined || v.name !== videoName) + } + } + + function emailNotificationFinder (email: object) { + const text: string = email['text'] + return text.includes(shortUUID) && text.includes('Edition of your video') + } + + await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) +} + +async function checkMyVideoImportIsFinished (options: CheckerBaseParams & { + videoName: string + shortUUID: string + url: string + success: boolean + checkType: CheckerType +}) { + const { videoName, shortUUID, url, success } = options + + const notificationType = success ? UserNotificationType.MY_VIDEO_IMPORT_SUCCESS : UserNotificationType.MY_VIDEO_IMPORT_ERROR + + function notificationChecker (notification: UserNotification, checkType: CheckerType) { + if (checkType === 'presence') { + expect(notification).to.not.be.undefined + expect(notification.type).to.equal(notificationType) + + expect(notification.videoImport.targetUrl).to.equal(url) + + if (success) checkVideo(notification.videoImport.video, videoName, shortUUID) + } else { + expect(notification.videoImport).to.satisfy(i => i === undefined || i.targetUrl !== url) + } + } + + function emailNotificationFinder (email: object) { + const text: string = email['text'] + const toFind = success ? ' finished' : ' error' + + return text.includes(url) && text.includes(toFind) + } + + await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) +} + +// --------------------------------------------------------------------------- + +async function checkUserRegistered (options: CheckerBaseParams & { + username: string + checkType: CheckerType +}) { + const { username } = options + const notificationType = UserNotificationType.NEW_USER_REGISTRATION + + function notificationChecker (notification: UserNotification, checkType: CheckerType) { + if (checkType === 'presence') { + expect(notification).to.not.be.undefined + expect(notification.type).to.equal(notificationType) + + checkActor(notification.account, { withAvatar: false }) + expect(notification.account.name).to.equal(username) + } else { + expect(notification).to.satisfy(n => n.type !== notificationType || n.account.name !== username) + } + } + + function emailNotificationFinder (email: object) { + const text: string = email['text'] + + return text.includes(' registered.') && text.includes(username) + } + + await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) +} + +async function checkRegistrationRequest (options: CheckerBaseParams & { + username: string + registrationReason: string + checkType: CheckerType +}) { + const { username, registrationReason } = options + const notificationType = UserNotificationType.NEW_USER_REGISTRATION_REQUEST + + function notificationChecker (notification: UserNotification, checkType: CheckerType) { + if (checkType === 'presence') { + expect(notification).to.not.be.undefined + expect(notification.type).to.equal(notificationType) + + expect(notification.registration.username).to.equal(username) + } else { + expect(notification).to.satisfy(n => n.type !== notificationType || n.registration.username !== username) + } + } + + function emailNotificationFinder (email: object) { + const text: string = email['text'] + + return text.includes(' wants to register ') && text.includes(username) && text.includes(registrationReason) + } + + await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) +} + +// --------------------------------------------------------------------------- + +async function checkNewActorFollow (options: CheckerBaseParams & { + followType: 'channel' | 'account' + followerName: string + followerDisplayName: string + followingDisplayName: string + checkType: CheckerType +}) { + const { followType, followerName, followerDisplayName, followingDisplayName } = options + const notificationType = UserNotificationType.NEW_FOLLOW + + function notificationChecker (notification: UserNotification, checkType: CheckerType) { + if (checkType === 'presence') { + expect(notification).to.not.be.undefined + expect(notification.type).to.equal(notificationType) + + checkActor(notification.actorFollow.follower) + expect(notification.actorFollow.follower.displayName).to.equal(followerDisplayName) + expect(notification.actorFollow.follower.name).to.equal(followerName) + expect(notification.actorFollow.follower.host).to.not.be.undefined + + const following = notification.actorFollow.following + expect(following.displayName).to.equal(followingDisplayName) + expect(following.type).to.equal(followType) + } else { + expect(notification).to.satisfy(n => { + return n.type !== notificationType || + (n.actorFollow.follower.name !== followerName && n.actorFollow.following !== followingDisplayName) + }) + } + } + + function emailNotificationFinder (email: object) { + const text: string = email['text'] + + return text.includes(followType) && text.includes(followingDisplayName) && text.includes(followerDisplayName) + } + + await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) +} + +async function checkNewInstanceFollower (options: CheckerBaseParams & { + followerHost: string + checkType: CheckerType +}) { + const { followerHost } = options + const notificationType = UserNotificationType.NEW_INSTANCE_FOLLOWER + + function notificationChecker (notification: UserNotification, checkType: CheckerType) { + if (checkType === 'presence') { + expect(notification).to.not.be.undefined + expect(notification.type).to.equal(notificationType) + + checkActor(notification.actorFollow.follower, { withAvatar: false }) + expect(notification.actorFollow.follower.name).to.equal('peertube') + expect(notification.actorFollow.follower.host).to.equal(followerHost) + + expect(notification.actorFollow.following.name).to.equal('peertube') + } else { + expect(notification).to.satisfy(n => { + return n.type !== notificationType || n.actorFollow.follower.host !== followerHost + }) + } + } + + function emailNotificationFinder (email: object) { + const text: string = email['text'] + + return text.includes('instance has a new follower') && text.includes(followerHost) + } + + await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) +} + +async function checkAutoInstanceFollowing (options: CheckerBaseParams & { + followerHost: string + followingHost: string + checkType: CheckerType +}) { + const { followerHost, followingHost } = options + const notificationType = UserNotificationType.AUTO_INSTANCE_FOLLOWING + + function notificationChecker (notification: UserNotification, checkType: CheckerType) { + if (checkType === 'presence') { + expect(notification).to.not.be.undefined + expect(notification.type).to.equal(notificationType) + + const following = notification.actorFollow.following + + checkActor(following, { withAvatar: false }) + expect(following.name).to.equal('peertube') + expect(following.host).to.equal(followingHost) + + expect(notification.actorFollow.follower.name).to.equal('peertube') + expect(notification.actorFollow.follower.host).to.equal(followerHost) + } else { + expect(notification).to.satisfy(n => { + return n.type !== notificationType || n.actorFollow.following.host !== followingHost + }) + } + } + + function emailNotificationFinder (email: object) { + const text: string = email['text'] + + return text.includes(' automatically followed a new instance') && text.includes(followingHost) + } + + await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) +} + +async function checkCommentMention (options: CheckerBaseParams & { + shortUUID: string + commentId: number + threadId: number + byAccountDisplayName: string + checkType: CheckerType +}) { + const { shortUUID, commentId, threadId, byAccountDisplayName } = options + const notificationType = UserNotificationType.COMMENT_MENTION + + function notificationChecker (notification: UserNotification, checkType: CheckerType) { + if (checkType === 'presence') { + expect(notification).to.not.be.undefined + expect(notification.type).to.equal(notificationType) + + checkComment(notification.comment, commentId, threadId) + checkActor(notification.comment.account) + expect(notification.comment.account.displayName).to.equal(byAccountDisplayName) + + checkVideo(notification.comment.video, undefined, shortUUID) + } else { + expect(notification).to.satisfy(n => n.type !== notificationType || n.comment.id !== commentId) + } + } + + function emailNotificationFinder (email: object) { + const text: string = email['text'] + + return text.includes(' mentioned ') && text.includes(shortUUID) && text.includes(byAccountDisplayName) + } + + await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) +} + +let lastEmailCount = 0 + +async function checkNewCommentOnMyVideo (options: CheckerBaseParams & { + shortUUID: string + commentId: number + threadId: number + checkType: CheckerType +}) { + const { server, shortUUID, commentId, threadId, checkType, emails } = options + const notificationType = UserNotificationType.NEW_COMMENT_ON_MY_VIDEO + + function notificationChecker (notification: UserNotification, checkType: CheckerType) { + if (checkType === 'presence') { + expect(notification).to.not.be.undefined + expect(notification.type).to.equal(notificationType) + + checkComment(notification.comment, commentId, threadId) + checkActor(notification.comment.account) + checkVideo(notification.comment.video, undefined, shortUUID) + } else { + expect(notification).to.satisfy((n: UserNotification) => { + return n === undefined || n.comment === undefined || n.comment.id !== commentId + }) + } + } + + const commentUrl = `${server.url}/w/${shortUUID};threadId=${threadId}` + + function emailNotificationFinder (email: object) { + return email['text'].indexOf(commentUrl) !== -1 + } + + await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) + + if (checkType === 'presence') { + // We cannot detect email duplicates, so check we received another email + expect(emails).to.have.length.above(lastEmailCount) + lastEmailCount = emails.length + } +} + +async function checkNewVideoAbuseForModerators (options: CheckerBaseParams & { + shortUUID: string + videoName: string + checkType: CheckerType +}) { + const { shortUUID, videoName } = options + const notificationType = UserNotificationType.NEW_ABUSE_FOR_MODERATORS + + function notificationChecker (notification: UserNotification, checkType: CheckerType) { + if (checkType === 'presence') { + expect(notification).to.not.be.undefined + expect(notification.type).to.equal(notificationType) + + expect(notification.abuse.id).to.be.a('number') + checkVideo(notification.abuse.video, videoName, shortUUID) + } else { + expect(notification).to.satisfy((n: UserNotification) => { + return n === undefined || n.abuse === undefined || n.abuse.video.shortUUID !== shortUUID + }) + } + } + + function emailNotificationFinder (email: object) { + const text = email['text'] + return text.indexOf(shortUUID) !== -1 && text.indexOf('abuse') !== -1 + } + + await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) +} + +async function checkNewAbuseMessage (options: CheckerBaseParams & { + abuseId: number + message: string + toEmail: string + checkType: CheckerType +}) { + const { abuseId, message, toEmail } = options + const notificationType = UserNotificationType.ABUSE_NEW_MESSAGE + + function notificationChecker (notification: UserNotification, checkType: CheckerType) { + if (checkType === 'presence') { + expect(notification).to.not.be.undefined + expect(notification.type).to.equal(notificationType) + + expect(notification.abuse.id).to.equal(abuseId) + } else { + expect(notification).to.satisfy((n: UserNotification) => { + return n === undefined || n.type !== notificationType || n.abuse === undefined || n.abuse.id !== abuseId + }) + } + } + + function emailNotificationFinder (email: object) { + const text = email['text'] + const to = email['to'].filter(t => t.address === toEmail) + + return text.indexOf(message) !== -1 && to.length !== 0 + } + + await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) +} + +async function checkAbuseStateChange (options: CheckerBaseParams & { + abuseId: number + state: AbuseStateType + checkType: CheckerType +}) { + const { abuseId, state } = options + const notificationType = UserNotificationType.ABUSE_STATE_CHANGE + + function notificationChecker (notification: UserNotification, checkType: CheckerType) { + if (checkType === 'presence') { + expect(notification).to.not.be.undefined + expect(notification.type).to.equal(notificationType) + + expect(notification.abuse.id).to.equal(abuseId) + expect(notification.abuse.state).to.equal(state) + } else { + expect(notification).to.satisfy((n: UserNotification) => { + return n === undefined || n.abuse === undefined || n.abuse.id !== abuseId + }) + } + } + + function emailNotificationFinder (email: object) { + const text = email['text'] + + const contains = state === AbuseState.ACCEPTED + ? ' accepted' + : ' rejected' + + return text.indexOf(contains) !== -1 + } + + await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) +} + +async function checkNewCommentAbuseForModerators (options: CheckerBaseParams & { + shortUUID: string + videoName: string + checkType: CheckerType +}) { + const { shortUUID, videoName } = options + const notificationType = UserNotificationType.NEW_ABUSE_FOR_MODERATORS + + function notificationChecker (notification: UserNotification, checkType: CheckerType) { + if (checkType === 'presence') { + expect(notification).to.not.be.undefined + expect(notification.type).to.equal(notificationType) + + expect(notification.abuse.id).to.be.a('number') + checkVideo(notification.abuse.comment.video, videoName, shortUUID) + } else { + expect(notification).to.satisfy((n: UserNotification) => { + return n === undefined || n.abuse === undefined || n.abuse.comment.video.shortUUID !== shortUUID + }) + } + } + + function emailNotificationFinder (email: object) { + const text = email['text'] + return text.indexOf(shortUUID) !== -1 && text.indexOf('abuse') !== -1 + } + + await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) +} + +async function checkNewAccountAbuseForModerators (options: CheckerBaseParams & { + displayName: string + checkType: CheckerType +}) { + const { displayName } = options + const notificationType = UserNotificationType.NEW_ABUSE_FOR_MODERATORS + + function notificationChecker (notification: UserNotification, checkType: CheckerType) { + if (checkType === 'presence') { + expect(notification).to.not.be.undefined + expect(notification.type).to.equal(notificationType) + + expect(notification.abuse.id).to.be.a('number') + expect(notification.abuse.account.displayName).to.equal(displayName) + } else { + expect(notification).to.satisfy((n: UserNotification) => { + return n === undefined || n.abuse === undefined || n.abuse.account.displayName !== displayName + }) + } + } + + function emailNotificationFinder (email: object) { + const text = email['text'] + return text.indexOf(displayName) !== -1 && text.indexOf('abuse') !== -1 + } + + await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) +} + +async function checkVideoAutoBlacklistForModerators (options: CheckerBaseParams & { + shortUUID: string + videoName: string + checkType: CheckerType +}) { + const { shortUUID, videoName } = options + const notificationType = UserNotificationType.VIDEO_AUTO_BLACKLIST_FOR_MODERATORS + + function notificationChecker (notification: UserNotification, checkType: CheckerType) { + if (checkType === 'presence') { + expect(notification).to.not.be.undefined + expect(notification.type).to.equal(notificationType) + + expect(notification.videoBlacklist.video.id).to.be.a('number') + checkVideo(notification.videoBlacklist.video, videoName, shortUUID) + } else { + expect(notification).to.satisfy((n: UserNotification) => { + return n === undefined || n.video === undefined || n.video.shortUUID !== shortUUID + }) + } + } + + function emailNotificationFinder (email: object) { + const text = email['text'] + return text.indexOf(shortUUID) !== -1 && email['text'].indexOf('video-auto-blacklist/list') !== -1 + } + + await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) +} + +async function checkNewBlacklistOnMyVideo (options: CheckerBaseParams & { + shortUUID: string + videoName: string + blacklistType: 'blacklist' | 'unblacklist' +}) { + const { videoName, shortUUID, blacklistType } = options + const notificationType = blacklistType === 'blacklist' + ? UserNotificationType.BLACKLIST_ON_MY_VIDEO + : UserNotificationType.UNBLACKLIST_ON_MY_VIDEO + + function notificationChecker (notification: UserNotification) { + expect(notification).to.not.be.undefined + expect(notification.type).to.equal(notificationType) + + const video = blacklistType === 'blacklist' ? notification.videoBlacklist.video : notification.video + + checkVideo(video, videoName, shortUUID) + } + + function emailNotificationFinder (email: object) { + const text = email['text'] + const blacklistText = blacklistType === 'blacklist' + ? 'blacklisted' + : 'unblacklisted' + + return text.includes(shortUUID) && text.includes(blacklistText) + } + + await checkNotification({ ...options, notificationChecker, emailNotificationFinder, checkType: 'presence' }) +} + +async function checkNewPeerTubeVersion (options: CheckerBaseParams & { + latestVersion: string + checkType: CheckerType +}) { + const { latestVersion } = options + const notificationType = UserNotificationType.NEW_PEERTUBE_VERSION + + function notificationChecker (notification: UserNotification, checkType: CheckerType) { + if (checkType === 'presence') { + expect(notification).to.not.be.undefined + expect(notification.type).to.equal(notificationType) + + expect(notification.peertube).to.exist + expect(notification.peertube.latestVersion).to.equal(latestVersion) + } else { + expect(notification).to.satisfy((n: UserNotification) => { + return n === undefined || n.peertube === undefined || n.peertube.latestVersion !== latestVersion + }) + } + } + + function emailNotificationFinder (email: object) { + const text = email['text'] + + return text.includes(latestVersion) + } + + await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) +} + +async function checkNewPluginVersion (options: CheckerBaseParams & { + pluginType: PluginType_Type + pluginName: string + checkType: CheckerType +}) { + const { pluginName, pluginType } = options + const notificationType = UserNotificationType.NEW_PLUGIN_VERSION + + function notificationChecker (notification: UserNotification, checkType: CheckerType) { + if (checkType === 'presence') { + expect(notification).to.not.be.undefined + expect(notification.type).to.equal(notificationType) + + expect(notification.plugin.name).to.equal(pluginName) + expect(notification.plugin.type).to.equal(pluginType) + } else { + expect(notification).to.satisfy((n: UserNotification) => { + return n === undefined || n.plugin === undefined || n.plugin.name !== pluginName + }) + } + } + + function emailNotificationFinder (email: object) { + const text = email['text'] + + return text.includes(pluginName) + } + + await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) +} + +async function prepareNotificationsTest (serversCount = 3, overrideConfigArg: any = {}) { + const userNotifications: UserNotification[] = [] + const adminNotifications: UserNotification[] = [] + const adminNotificationsServer2: UserNotification[] = [] + const emails: object[] = [] + + const port = await MockSmtpServer.Instance.collectEmails(emails) + + const overrideConfig = { + ...ConfigCommand.getEmailOverrideConfig(port), + + signup: { + limit: 20 + } + } + const servers = await createMultipleServers(serversCount, Object.assign(overrideConfig, overrideConfigArg)) + + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + await setDefaultChannelAvatar(servers) + await setDefaultAccountAvatar(servers) + + if (servers[1]) { + await servers[1].config.enableStudio() + await servers[1].config.enableLive({ allowReplay: true, transcoding: false }) + } + + if (serversCount > 1) { + await doubleFollow(servers[0], servers[1]) + } + + const user = { username: 'user_1', password: 'super password' } + await servers[0].users.create({ ...user, videoQuota: 10 * 1000 * 1000 }) + const userAccessToken = await servers[0].login.getAccessToken(user) + + await servers[0].notifications.updateMySettings({ token: userAccessToken, settings: getAllNotificationsSettings() }) + await servers[0].users.updateMyAvatar({ token: userAccessToken, fixture: 'avatar.png' }) + await servers[0].channels.updateImage({ channelName: 'user_1_channel', token: userAccessToken, fixture: 'avatar.png', type: 'avatar' }) + + await servers[0].notifications.updateMySettings({ settings: getAllNotificationsSettings() }) + + if (serversCount > 1) { + await servers[1].notifications.updateMySettings({ settings: getAllNotificationsSettings() }) + } + + { + const socket = servers[0].socketIO.getUserNotificationSocket({ token: userAccessToken }) + socket.on('new-notification', n => userNotifications.push(n)) + } + { + const socket = servers[0].socketIO.getUserNotificationSocket() + socket.on('new-notification', n => adminNotifications.push(n)) + } + + if (serversCount > 1) { + const socket = servers[1].socketIO.getUserNotificationSocket() + socket.on('new-notification', n => adminNotificationsServer2.push(n)) + } + + const { videoChannels } = await servers[0].users.getMyInfo() + const channelId = videoChannels[0].id + + return { + userNotifications, + adminNotifications, + adminNotificationsServer2, + userAccessToken, + emails, + servers, + channelId, + baseOverrideConfig: overrideConfig + } +} + +// --------------------------------------------------------------------------- + +export { + type CheckerType, + type CheckerBaseParams, + + getAllNotificationsSettings, + + checkMyVideoImportIsFinished, + checkUserRegistered, + checkAutoInstanceFollowing, + checkVideoIsPublished, + checkNewVideoFromSubscription, + checkNewActorFollow, + checkNewCommentOnMyVideo, + checkNewBlacklistOnMyVideo, + checkCommentMention, + checkNewVideoAbuseForModerators, + checkVideoAutoBlacklistForModerators, + checkNewAbuseMessage, + checkAbuseStateChange, + checkNewInstanceFollower, + prepareNotificationsTest, + checkNewCommentAbuseForModerators, + checkNewAccountAbuseForModerators, + checkNewPeerTubeVersion, + checkNewPluginVersion, + checkVideoStudioEditionIsFinished, + checkRegistrationRequest +} + +// --------------------------------------------------------------------------- + +async function checkNotification (options: CheckerBaseParams & { + notificationChecker: (notification: UserNotification, checkType: CheckerType) => void + emailNotificationFinder: (email: object) => boolean + checkType: CheckerType +}) { + const { server, token, checkType, notificationChecker, emailNotificationFinder, socketNotifications, emails } = options + + const check = options.check || { web: true, mail: true } + + if (check.web) { + const notification = await server.notifications.getLatest({ token }) + + if (notification || checkType !== 'absence') { + notificationChecker(notification, checkType) + } + + const socketNotification = socketNotifications.find(n => { + try { + notificationChecker(n, 'presence') + return true + } catch { + return false + } + }) + + if (checkType === 'presence') { + const obj = inspect(socketNotifications, { depth: 5 }) + expect(socketNotification, 'The socket notification is absent when it should be present. ' + obj).to.not.be.undefined + } else { + const obj = inspect(socketNotification, { depth: 5 }) + expect(socketNotification, 'The socket notification is present when it should not be present. ' + obj).to.be.undefined + } + } + + if (check.mail) { + // Last email + const email = emails + .slice() + .reverse() + .find(e => emailNotificationFinder(e)) + + if (checkType === 'presence') { + const texts = emails.map(e => e.text) + expect(email, 'The email is absent when is should be present. ' + inspect(texts)).to.not.be.undefined + } else { + expect(email, 'The email is present when is should not be present. ' + inspect(email)).to.be.undefined + } + } +} + +function checkVideo (video: any, videoName?: string, shortUUID?: string) { + if (videoName) { + expect(video.name).to.be.a('string') + expect(video.name).to.not.be.empty + expect(video.name).to.equal(videoName) + } + + if (shortUUID) { + expect(video.shortUUID).to.be.a('string') + expect(video.shortUUID).to.not.be.empty + expect(video.shortUUID).to.equal(shortUUID) + } + + expect(video.id).to.be.a('number') +} + +function checkActor (actor: any, options: { withAvatar?: boolean } = {}) { + const { withAvatar = true } = options + + expect(actor.displayName).to.be.a('string') + expect(actor.displayName).to.not.be.empty + expect(actor.host).to.not.be.undefined + + if (withAvatar) { + expect(actor.avatars).to.be.an('array') + expect(actor.avatars).to.have.lengthOf(2) + expect(actor.avatars[0].path).to.exist.and.not.empty + } +} + +function checkComment (comment: any, commentId: number, threadId: number) { + expect(comment.id).to.equal(commentId) + expect(comment.threadId).to.equal(threadId) +} diff --git a/packages/tests/src/shared/peertube-runner-process.ts b/packages/tests/src/shared/peertube-runner-process.ts new file mode 100644 index 000000000..3d1f299f2 --- /dev/null +++ b/packages/tests/src/shared/peertube-runner-process.ts @@ -0,0 +1,104 @@ +import { ChildProcess, fork, ForkOptions } from 'child_process' +import execa from 'execa' +import { join } from 'path' +import { root } from '@peertube/peertube-node-utils' +import { PeerTubeServer } from '@peertube/peertube-server-commands' + +export class PeerTubeRunnerProcess { + private app?: ChildProcess + + constructor (private readonly server: PeerTubeServer) { + + } + + runServer (options: { + hideLogs?: boolean // default true + } = {}) { + const { hideLogs = true } = options + + return new Promise((res, rej) => { + const args = [ 'server', '--verbose', ...this.buildIdArg() ] + + const forkOptions: ForkOptions = { + detached: false, + silent: true, + execArgv: [] // Don't inject parent node options + } + + this.app = fork(this.getRunnerPath(), args, forkOptions) + + this.app.stdout.on('data', data => { + const str = data.toString() as string + + if (!hideLogs) { + console.log(str) + } + }) + + res() + }) + } + + registerPeerTubeInstance (options: { + registrationToken: string + runnerName: string + runnerDescription?: string + }) { + const { registrationToken, runnerName, runnerDescription } = options + + const args = [ + 'register', + '--url', this.server.url, + '--registration-token', registrationToken, + '--runner-name', runnerName, + ...this.buildIdArg() + ] + + if (runnerDescription) { + args.push('--runner-description') + args.push(runnerDescription) + } + + return this.runCommand(this.getRunnerPath(), args) + } + + unregisterPeerTubeInstance (options: { + runnerName: string + }) { + const { runnerName } = options + + const args = [ 'unregister', '--url', this.server.url, '--runner-name', runnerName, ...this.buildIdArg() ] + return this.runCommand(this.getRunnerPath(), args) + } + + async listRegisteredPeerTubeInstances () { + const args = [ 'list-registered', ...this.buildIdArg() ] + const { stdout } = await this.runCommand(this.getRunnerPath(), args) + + return stdout + } + + kill () { + if (!this.app) return + + process.kill(this.app.pid) + + this.app = null + } + + getId () { + return 'test-' + this.server.internalServerNumber + } + + private getRunnerPath () { + return join(root(), 'apps', 'peertube-runner', 'dist', 'peertube-runner.js') + } + + private buildIdArg () { + return [ '--id', this.getId() ] + } + + private runCommand (path: string, args: string[]) { + return execa.node(path, args, { env: { ...process.env, NODE_OPTIONS: '' } }) + } +} diff --git a/packages/tests/src/shared/plugins.ts b/packages/tests/src/shared/plugins.ts new file mode 100644 index 000000000..c2afcbcbf --- /dev/null +++ b/packages/tests/src/shared/plugins.ts @@ -0,0 +1,18 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { PeerTubeServer } from '@peertube/peertube-server-commands' + +async function testHelloWorldRegisteredSettings (server: PeerTubeServer) { + const body = await server.plugins.getRegisteredSettings({ npmName: 'peertube-plugin-hello-world' }) + + const registeredSettings = body.registeredSettings + expect(registeredSettings).to.have.length.at.least(1) + + const adminNameSettings = registeredSettings.find(s => s.name === 'admin-name') + expect(adminNameSettings).to.not.be.undefined +} + +export { + testHelloWorldRegisteredSettings +} diff --git a/packages/tests/src/shared/requests.ts b/packages/tests/src/shared/requests.ts new file mode 100644 index 000000000..fc70ad6ed --- /dev/null +++ b/packages/tests/src/shared/requests.ts @@ -0,0 +1,12 @@ +import { doRequest } from '@peertube/peertube-server/server/helpers/requests.js' + +export function makePOSTAPRequest (url: string, body: any, httpSignature: any, headers: any) { + const options = { + method: 'POST' as 'POST', + json: body, + httpSignature, + headers + } + + return doRequest(url, options) +} diff --git a/packages/tests/src/shared/sql-command.ts b/packages/tests/src/shared/sql-command.ts new file mode 100644 index 000000000..1c4f89351 --- /dev/null +++ b/packages/tests/src/shared/sql-command.ts @@ -0,0 +1,150 @@ +import { QueryTypes, Sequelize } from 'sequelize' +import { forceNumber } from '@peertube/peertube-core-utils' +import { PeerTubeServer } from '@peertube/peertube-server-commands' + +export class SQLCommand { + private sequelize: Sequelize + + constructor (private readonly server: PeerTubeServer) { + + } + + deleteAll (table: string) { + const seq = this.getSequelize() + + const options = { type: QueryTypes.DELETE } + + return seq.query(`DELETE FROM "${table}"`, options) + } + + async getVideoShareCount () { + const [ { total } ] = await this.selectQuery<{ total: string }>(`SELECT COUNT(*) as total FROM "videoShare"`) + if (total === null) return 0 + + return parseInt(total, 10) + } + + async getInternalFileUrl (fileId: number) { + return this.selectQuery<{ fileUrl: string }>(`SELECT "fileUrl" FROM "videoFile" WHERE id = :fileId`, { fileId }) + .then(rows => rows[0].fileUrl) + } + + setActorField (to: string, field: string, value: string) { + return this.updateQuery(`UPDATE actor SET ${this.escapeColumnName(field)} = :value WHERE url = :to`, { value, to }) + } + + setVideoField (uuid: string, field: string, value: string) { + return this.updateQuery(`UPDATE video SET ${this.escapeColumnName(field)} = :value WHERE uuid = :uuid`, { value, uuid }) + } + + setPlaylistField (uuid: string, field: string, value: string) { + return this.updateQuery(`UPDATE "videoPlaylist" SET ${this.escapeColumnName(field)} = :value WHERE uuid = :uuid`, { value, uuid }) + } + + async countVideoViewsOf (uuid: string) { + const query = 'SELECT SUM("videoView"."views") AS "total" FROM "videoView" ' + + `INNER JOIN "video" ON "video"."id" = "videoView"."videoId" WHERE "video"."uuid" = :uuid` + + const [ { total } ] = await this.selectQuery<{ total: number }>(query, { uuid }) + if (!total) return 0 + + return forceNumber(total) + } + + getActorImage (filename: string) { + return this.selectQuery<{ width: number, height: number }>(`SELECT * FROM "actorImage" WHERE filename = :filename`, { filename }) + .then(rows => rows[0]) + } + + // --------------------------------------------------------------------------- + + setPluginVersion (pluginName: string, newVersion: string) { + return this.setPluginField(pluginName, 'version', newVersion) + } + + setPluginLatestVersion (pluginName: string, newVersion: string) { + return this.setPluginField(pluginName, 'latestVersion', newVersion) + } + + setPluginField (pluginName: string, field: string, value: string) { + return this.updateQuery( + `UPDATE "plugin" SET ${this.escapeColumnName(field)} = :value WHERE "name" = :pluginName`, + { pluginName, value } + ) + } + + // --------------------------------------------------------------------------- + + selectQuery (query: string, replacements: { [id: string]: string | number } = {}) { + const seq = this.getSequelize() + const options = { + type: QueryTypes.SELECT as QueryTypes.SELECT, + replacements + } + + return seq.query(query, options) + } + + updateQuery (query: string, replacements: { [id: string]: string | number } = {}) { + const seq = this.getSequelize() + const options = { type: QueryTypes.UPDATE as QueryTypes.UPDATE, replacements } + + return seq.query(query, options) + } + + // --------------------------------------------------------------------------- + + async getPlaylistInfohash (playlistId: number) { + const query = 'SELECT "p2pMediaLoaderInfohashes" FROM "videoStreamingPlaylist" WHERE id = :playlistId' + + const result = await this.selectQuery<{ p2pMediaLoaderInfohashes: string }>(query, { playlistId }) + if (!result || result.length === 0) return [] + + return result[0].p2pMediaLoaderInfohashes + } + + // --------------------------------------------------------------------------- + + setActorFollowScores (newScore: number) { + return this.updateQuery(`UPDATE "actorFollow" SET "score" = :newScore`, { newScore }) + } + + setTokenField (accessToken: string, field: string, value: string) { + return this.updateQuery( + `UPDATE "oAuthToken" SET ${this.escapeColumnName(field)} = :value WHERE "accessToken" = :accessToken`, + { value, accessToken } + ) + } + + async cleanup () { + if (!this.sequelize) return + + await this.sequelize.close() + this.sequelize = undefined + } + + private getSequelize () { + if (this.sequelize) return this.sequelize + + const dbname = 'peertube_test' + this.server.internalServerNumber + const username = 'peertube' + const password = 'peertube' + const host = '127.0.0.1' + const port = 5432 + + this.sequelize = new Sequelize(dbname, username, password, { + dialect: 'postgres', + host, + port, + logging: false + }) + + return this.sequelize + } + + private escapeColumnName (columnName: string) { + return this.getSequelize().escape(columnName) + .replace(/^'/, '"') + .replace(/'$/, '"') + } +} diff --git a/packages/tests/src/shared/streaming-playlists.ts b/packages/tests/src/shared/streaming-playlists.ts new file mode 100644 index 000000000..f2f0fbe85 --- /dev/null +++ b/packages/tests/src/shared/streaming-playlists.ts @@ -0,0 +1,302 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { basename, dirname, join } from 'path' +import { removeFragmentedMP4Ext, uuidRegex } from '@peertube/peertube-core-utils' +import { + HttpStatusCode, + VideoPrivacy, + VideoResolution, + VideoStreamingPlaylist, + VideoStreamingPlaylistType +} from '@peertube/peertube-models' +import { sha256 } from '@peertube/peertube-node-utils' +import { makeRawRequest, PeerTubeServer } from '@peertube/peertube-server-commands' +import { expectStartWith } from './checks.js' +import { hlsInfohashExist } from './tracker.js' +import { checkWebTorrentWorks } from './webtorrent.js' + +async function checkSegmentHash (options: { + server: PeerTubeServer + baseUrlPlaylist: string + baseUrlSegment: string + resolution: number + hlsPlaylist: VideoStreamingPlaylist + token?: string +}) { + const { server, baseUrlPlaylist, baseUrlSegment, resolution, hlsPlaylist, token } = options + const command = server.streamingPlaylists + + const file = hlsPlaylist.files.find(f => f.resolution.id === resolution) + const videoName = basename(file.fileUrl) + + const playlist = await command.get({ url: `${baseUrlPlaylist}/${removeFragmentedMP4Ext(videoName)}.m3u8`, token }) + + const matches = /#EXT-X-BYTERANGE:(\d+)@(\d+)/.exec(playlist) + + const length = parseInt(matches[1], 10) + const offset = parseInt(matches[2], 10) + const range = `${offset}-${offset + length - 1}` + + const segmentBody = await command.getFragmentedSegment({ + url: `${baseUrlSegment}/${videoName}`, + expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206, + range: `bytes=${range}`, + token + }) + + const shaBody = await command.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url, token }) + expect(sha256(segmentBody)).to.equal(shaBody[videoName][range], `Invalid sha256 result for ${videoName} range ${range}`) +} + +// --------------------------------------------------------------------------- + +async function checkLiveSegmentHash (options: { + server: PeerTubeServer + baseUrlSegment: string + videoUUID: string + segmentName: string + hlsPlaylist: VideoStreamingPlaylist + withRetry?: boolean +}) { + const { server, baseUrlSegment, videoUUID, segmentName, hlsPlaylist, withRetry = false } = options + const command = server.streamingPlaylists + + const segmentBody = await command.getFragmentedSegment({ url: `${baseUrlSegment}/${videoUUID}/${segmentName}`, withRetry }) + const shaBody = await command.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url, withRetry }) + + expect(sha256(segmentBody)).to.equal(shaBody[segmentName]) +} + +// --------------------------------------------------------------------------- + +async function checkResolutionsInMasterPlaylist (options: { + server: PeerTubeServer + playlistUrl: string + resolutions: number[] + token?: string + transcoded?: boolean // default true + withRetry?: boolean // default false +}) { + const { server, playlistUrl, resolutions, token, withRetry = false, transcoded = true } = options + + const masterPlaylist = await server.streamingPlaylists.get({ url: playlistUrl, token, withRetry }) + + for (const resolution of resolutions) { + const base = '#EXT-X-STREAM-INF:BANDWIDTH=\\d+,RESOLUTION=\\d+x' + resolution + + if (resolution === VideoResolution.H_NOVIDEO) { + expect(masterPlaylist).to.match(new RegExp(`${base},CODECS="mp4a.40.2"`)) + } else if (transcoded) { + expect(masterPlaylist).to.match(new RegExp(`${base},(FRAME-RATE=\\d+,)?CODECS="avc1.64001f,mp4a.40.2"`)) + } else { + expect(masterPlaylist).to.match(new RegExp(`${base}`)) + } + } + + const playlistsLength = masterPlaylist.split('\n').filter(line => line.startsWith('#EXT-X-STREAM-INF:BANDWIDTH=')) + expect(playlistsLength).to.have.lengthOf(resolutions.length) +} + +async function completeCheckHlsPlaylist (options: { + servers: PeerTubeServer[] + videoUUID: string + hlsOnly: boolean + + resolutions?: number[] + objectStorageBaseUrl?: string +}) { + const { videoUUID, hlsOnly, objectStorageBaseUrl } = options + + const resolutions = options.resolutions ?? [ 240, 360, 480, 720 ] + + for (const server of options.servers) { + const videoDetails = await server.videos.getWithToken({ id: videoUUID }) + const requiresAuth = videoDetails.privacy.id === VideoPrivacy.PRIVATE || videoDetails.privacy.id === VideoPrivacy.INTERNAL + + const privatePath = requiresAuth + ? 'private/' + : '' + const token = requiresAuth + ? server.accessToken + : undefined + + const baseUrl = `http://${videoDetails.account.host}` + + expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) + + const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) + expect(hlsPlaylist).to.not.be.undefined + + const hlsFiles = hlsPlaylist.files + expect(hlsFiles).to.have.lengthOf(resolutions.length) + + if (hlsOnly) expect(videoDetails.files).to.have.lengthOf(0) + else expect(videoDetails.files).to.have.lengthOf(resolutions.length) + + // Check JSON files + for (const resolution of resolutions) { + const file = hlsFiles.find(f => f.resolution.id === resolution) + expect(file).to.not.be.undefined + + if (file.resolution.id === VideoResolution.H_NOVIDEO) { + expect(file.resolution.label).to.equal('Audio') + } else { + expect(file.resolution.label).to.equal(resolution + 'p') + } + + expect(file.magnetUri).to.have.lengthOf.above(2) + await checkWebTorrentWorks(file.magnetUri) + + { + const nameReg = `${uuidRegex}-${file.resolution.id}` + + expect(file.torrentUrl).to.match(new RegExp(`${server.url}/lazy-static/torrents/${nameReg}-hls.torrent`)) + + if (objectStorageBaseUrl && requiresAuth) { + // eslint-disable-next-line max-len + expect(file.fileUrl).to.match(new RegExp(`${server.url}/object-storage-proxy/streaming-playlists/hls/${privatePath}${videoDetails.uuid}/${nameReg}-fragmented.mp4`)) + } else if (objectStorageBaseUrl) { + expectStartWith(file.fileUrl, objectStorageBaseUrl) + } else { + expect(file.fileUrl).to.match( + new RegExp(`${baseUrl}/static/streaming-playlists/hls/${privatePath}${videoDetails.uuid}/${nameReg}-fragmented.mp4`) + ) + } + } + + { + await Promise.all([ + makeRawRequest({ url: file.torrentUrl, token, expectedStatus: HttpStatusCode.OK_200 }), + makeRawRequest({ url: file.torrentDownloadUrl, token, expectedStatus: HttpStatusCode.OK_200 }), + makeRawRequest({ url: file.metadataUrl, token, expectedStatus: HttpStatusCode.OK_200 }), + makeRawRequest({ url: file.fileUrl, token, expectedStatus: HttpStatusCode.OK_200 }), + + makeRawRequest({ + url: file.fileDownloadUrl, + token, + expectedStatus: objectStorageBaseUrl + ? HttpStatusCode.FOUND_302 + : HttpStatusCode.OK_200 + }) + ]) + } + } + + // Check master playlist + { + await checkResolutionsInMasterPlaylist({ server, token, playlistUrl: hlsPlaylist.playlistUrl, resolutions }) + + const masterPlaylist = await server.streamingPlaylists.get({ url: hlsPlaylist.playlistUrl, token }) + + let i = 0 + for (const resolution of resolutions) { + expect(masterPlaylist).to.contain(`${resolution}.m3u8`) + expect(masterPlaylist).to.contain(`${resolution}.m3u8`) + + const url = 'http://' + videoDetails.account.host + await hlsInfohashExist(url, hlsPlaylist.playlistUrl, i) + + i++ + } + } + + // Check resolution playlists + { + for (const resolution of resolutions) { + const file = hlsFiles.find(f => f.resolution.id === resolution) + const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8' + + let url: string + if (objectStorageBaseUrl && requiresAuth) { + url = `${baseUrl}/object-storage-proxy/streaming-playlists/hls/${privatePath}${videoUUID}/${playlistName}` + } else if (objectStorageBaseUrl) { + url = `${objectStorageBaseUrl}hls/${videoUUID}/${playlistName}` + } else { + url = `${baseUrl}/static/streaming-playlists/hls/${privatePath}${videoUUID}/${playlistName}` + } + + const subPlaylist = await server.streamingPlaylists.get({ url, token }) + + expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`)) + expect(subPlaylist).to.contain(basename(file.fileUrl)) + } + } + + { + let baseUrlAndPath: string + if (objectStorageBaseUrl && requiresAuth) { + baseUrlAndPath = `${baseUrl}/object-storage-proxy/streaming-playlists/hls/${privatePath}${videoUUID}` + } else if (objectStorageBaseUrl) { + baseUrlAndPath = `${objectStorageBaseUrl}hls/${videoUUID}` + } else { + baseUrlAndPath = `${baseUrl}/static/streaming-playlists/hls/${privatePath}${videoUUID}` + } + + for (const resolution of resolutions) { + await checkSegmentHash({ + server, + token, + baseUrlPlaylist: baseUrlAndPath, + baseUrlSegment: baseUrlAndPath, + resolution, + hlsPlaylist + }) + } + } + } +} + +async function checkVideoFileTokenReinjection (options: { + server: PeerTubeServer + videoUUID: string + videoFileToken: string + resolutions: number[] + isLive: boolean +}) { + const { server, resolutions, videoFileToken, videoUUID, isLive } = options + + const video = await server.videos.getWithToken({ id: videoUUID }) + const hls = video.streamingPlaylists[0] + + const query = { videoFileToken, reinjectVideoFileToken: 'true' } + const { text } = await makeRawRequest({ url: hls.playlistUrl, query, expectedStatus: HttpStatusCode.OK_200 }) + + for (let i = 0; i < resolutions.length; i++) { + const resolution = resolutions[i] + + const suffix = isLive + ? i + : `-${resolution}` + + expect(text).to.contain(`${suffix}.m3u8?videoFileToken=${videoFileToken}&reinjectVideoFileToken=true`) + } + + const resolutionPlaylists = extractResolutionPlaylistUrls(hls.playlistUrl, text) + expect(resolutionPlaylists).to.have.lengthOf(resolutions.length) + + for (const url of resolutionPlaylists) { + const { text } = await makeRawRequest({ url, query, expectedStatus: HttpStatusCode.OK_200 }) + + const extension = isLive + ? '.ts' + : '.mp4' + + expect(text).to.contain(`${extension}?videoFileToken=${videoFileToken}`) + expect(text).not.to.contain(`reinjectVideoFileToken=true`) + } +} + +function extractResolutionPlaylistUrls (masterPath: string, masterContent: string) { + return masterContent.match(/^([^.]+\.m3u8.*)/mg) + .map(filename => join(dirname(masterPath), filename)) +} + +export { + checkSegmentHash, + checkLiveSegmentHash, + checkResolutionsInMasterPlaylist, + completeCheckHlsPlaylist, + extractResolutionPlaylistUrls, + checkVideoFileTokenReinjection +} diff --git a/server/tests/shared/tests.ts b/packages/tests/src/shared/tests.ts similarity index 100% rename from server/tests/shared/tests.ts rename to packages/tests/src/shared/tests.ts diff --git a/packages/tests/src/shared/tracker.ts b/packages/tests/src/shared/tracker.ts new file mode 100644 index 000000000..6ab430456 --- /dev/null +++ b/packages/tests/src/shared/tracker.ts @@ -0,0 +1,27 @@ +import { expect } from 'chai' +import { sha1 } from '@peertube/peertube-node-utils' +import { makeGetRequest } from '@peertube/peertube-server-commands' + +async function hlsInfohashExist (serverUrl: string, masterPlaylistUrl: string, fileNumber: number) { + const path = '/tracker/announce' + + const infohash = sha1(`2${masterPlaylistUrl}+V${fileNumber}`) + + // From bittorrent-tracker + const infohashBinary = escape(Buffer.from(infohash, 'hex').toString('binary')).replace(/[@*/+]/g, function (char) { + return '%' + char.charCodeAt(0).toString(16).toUpperCase() + }) + + const res = await makeGetRequest({ + url: serverUrl, + path, + rawQuery: `peer_id=-WW0105-NkvYO/egUAr4&info_hash=${infohashBinary}&port=42100`, + expectedStatus: 200 + }) + + expect(res.text).to.not.contain('failure') +} + +export { + hlsInfohashExist +} diff --git a/packages/tests/src/shared/video-playlists.ts b/packages/tests/src/shared/video-playlists.ts new file mode 100644 index 000000000..81dc43ed6 --- /dev/null +++ b/packages/tests/src/shared/video-playlists.ts @@ -0,0 +1,22 @@ +import { expect } from 'chai' +import { readdir } from 'fs/promises' +import { PeerTubeServer } from '@peertube/peertube-server-commands' + +async function checkPlaylistFilesWereRemoved ( + playlistUUID: string, + server: PeerTubeServer, + directories = [ 'thumbnails' ] +) { + for (const directory of directories) { + const directoryPath = server.getDirectoryPath(directory) + + const files = await readdir(directoryPath) + for (const file of files) { + expect(file).to.not.contain(playlistUUID) + } + } +} + +export { + checkPlaylistFilesWereRemoved +} diff --git a/packages/tests/src/shared/videos.ts b/packages/tests/src/shared/videos.ts new file mode 100644 index 000000000..9bdcbf058 --- /dev/null +++ b/packages/tests/src/shared/videos.ts @@ -0,0 +1,323 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */ + +import { expect } from 'chai' +import { pathExists } from 'fs-extra/esm' +import { readdir } from 'fs/promises' +import { basename, join } from 'path' +import { pick, uuidRegex } from '@peertube/peertube-core-utils' +import { HttpStatusCode, HttpStatusCodeType, VideoCaption, VideoDetails, VideoPrivacy, VideoResolution } from '@peertube/peertube-models' +import { + loadLanguages, + VIDEO_CATEGORIES, + VIDEO_LANGUAGES, + VIDEO_LICENCES, + VIDEO_PRIVACIES +} from '@peertube/peertube-server/server/initializers/constants.js' +import { getLowercaseExtension } from '@peertube/peertube-node-utils' +import { makeRawRequest, PeerTubeServer, VideoEdit, waitJobs } from '@peertube/peertube-server-commands' +import { dateIsValid, expectStartWith, testImageGeneratedByFFmpeg } from './checks.js' +import { checkWebTorrentWorks } from './webtorrent.js' + +async function completeWebVideoFilesCheck (options: { + server: PeerTubeServer + originServer: PeerTubeServer + videoUUID: string + fixture: string + files: { + resolution: number + size?: number + }[] + objectStorageBaseUrl?: string +}) { + const { originServer, server, videoUUID, files, fixture, objectStorageBaseUrl } = options + const video = await server.videos.getWithToken({ id: videoUUID }) + const serverConfig = await originServer.config.getConfig() + const requiresAuth = video.privacy.id === VideoPrivacy.PRIVATE || video.privacy.id === VideoPrivacy.INTERNAL + + const transcodingEnabled = serverConfig.transcoding.web_videos.enabled + + for (const attributeFile of files) { + const file = video.files.find(f => f.resolution.id === attributeFile.resolution) + expect(file, `resolution ${attributeFile.resolution} does not exist`).not.to.be.undefined + + let extension = getLowercaseExtension(fixture) + // Transcoding enabled: extension will always be .mp4 + if (transcodingEnabled) extension = '.mp4' + + expect(file.id).to.exist + expect(file.magnetUri).to.have.lengthOf.above(2) + + { + const privatePath = requiresAuth + ? 'private/' + : '' + const nameReg = `${uuidRegex}-${file.resolution.id}` + + expect(file.torrentDownloadUrl).to.match(new RegExp(`${server.url}/download/torrents/${nameReg}.torrent`)) + expect(file.torrentUrl).to.match(new RegExp(`${server.url}/lazy-static/torrents/${nameReg}.torrent`)) + + if (objectStorageBaseUrl && requiresAuth) { + const regexp = new RegExp(`${originServer.url}/object-storage-proxy/web-videos/${privatePath}${nameReg}${extension}`) + expect(file.fileUrl).to.match(regexp) + } else if (objectStorageBaseUrl) { + expectStartWith(file.fileUrl, objectStorageBaseUrl) + } else { + expect(file.fileUrl).to.match(new RegExp(`${originServer.url}/static/web-videos/${privatePath}${nameReg}${extension}`)) + } + + expect(file.fileDownloadUrl).to.match(new RegExp(`${originServer.url}/download/videos/${nameReg}${extension}`)) + } + + { + const token = requiresAuth + ? server.accessToken + : undefined + + await Promise.all([ + makeRawRequest({ url: file.torrentUrl, token, expectedStatus: HttpStatusCode.OK_200 }), + makeRawRequest({ url: file.torrentDownloadUrl, token, expectedStatus: HttpStatusCode.OK_200 }), + makeRawRequest({ url: file.metadataUrl, token, expectedStatus: HttpStatusCode.OK_200 }), + makeRawRequest({ url: file.fileUrl, token, expectedStatus: HttpStatusCode.OK_200 }), + makeRawRequest({ + url: file.fileDownloadUrl, + token, + expectedStatus: objectStorageBaseUrl ? HttpStatusCode.FOUND_302 : HttpStatusCode.OK_200 + }) + ]) + } + + expect(file.resolution.id).to.equal(attributeFile.resolution) + + if (file.resolution.id === VideoResolution.H_NOVIDEO) { + expect(file.resolution.label).to.equal('Audio') + } else { + expect(file.resolution.label).to.equal(attributeFile.resolution + 'p') + } + + if (attributeFile.size) { + const minSize = attributeFile.size - ((10 * attributeFile.size) / 100) + const maxSize = attributeFile.size + ((10 * attributeFile.size) / 100) + expect( + file.size, + 'File size for resolution ' + file.resolution.label + ' outside confidence interval (' + minSize + '> size <' + maxSize + ')' + ).to.be.above(minSize).and.below(maxSize) + } + + await checkWebTorrentWorks(file.magnetUri) + } +} + +async function completeVideoCheck (options: { + server: PeerTubeServer + originServer: PeerTubeServer + videoUUID: string + attributes: { + name: string + category: number + licence: number + language: string + nsfw: boolean + commentsEnabled: boolean + downloadEnabled: boolean + description: string + publishedAt?: string + support: string + originallyPublishedAt?: string + account: { + name: string + host: string + } + isLocal: boolean + tags: string[] + privacy: number + likes?: number + dislikes?: number + duration: number + channel: { + displayName: string + name: string + description: string + isLocal: boolean + } + fixture: string + files: { + resolution: number + size: number + }[] + thumbnailfile?: string + previewfile?: string + } +}) { + const { attributes, originServer, server, videoUUID } = options + + await loadLanguages() + + const video = await server.videos.get({ id: videoUUID }) + + if (!attributes.likes) attributes.likes = 0 + if (!attributes.dislikes) attributes.dislikes = 0 + + expect(video.name).to.equal(attributes.name) + expect(video.category.id).to.equal(attributes.category) + expect(video.category.label).to.equal(attributes.category !== null ? VIDEO_CATEGORIES[attributes.category] : 'Unknown') + expect(video.licence.id).to.equal(attributes.licence) + expect(video.licence.label).to.equal(attributes.licence !== null ? VIDEO_LICENCES[attributes.licence] : 'Unknown') + expect(video.language.id).to.equal(attributes.language) + expect(video.language.label).to.equal(attributes.language !== null ? VIDEO_LANGUAGES[attributes.language] : 'Unknown') + expect(video.privacy.id).to.deep.equal(attributes.privacy) + expect(video.privacy.label).to.deep.equal(VIDEO_PRIVACIES[attributes.privacy]) + expect(video.nsfw).to.equal(attributes.nsfw) + expect(video.description).to.equal(attributes.description) + expect(video.account.id).to.be.a('number') + expect(video.account.host).to.equal(attributes.account.host) + expect(video.account.name).to.equal(attributes.account.name) + expect(video.channel.displayName).to.equal(attributes.channel.displayName) + expect(video.channel.name).to.equal(attributes.channel.name) + expect(video.likes).to.equal(attributes.likes) + expect(video.dislikes).to.equal(attributes.dislikes) + expect(video.isLocal).to.equal(attributes.isLocal) + expect(video.duration).to.equal(attributes.duration) + expect(video.url).to.contain(originServer.host) + expect(dateIsValid(video.createdAt)).to.be.true + expect(dateIsValid(video.publishedAt)).to.be.true + expect(dateIsValid(video.updatedAt)).to.be.true + + if (attributes.publishedAt) { + expect(video.publishedAt).to.equal(attributes.publishedAt) + } + + if (attributes.originallyPublishedAt) { + expect(video.originallyPublishedAt).to.equal(attributes.originallyPublishedAt) + } else { + expect(video.originallyPublishedAt).to.be.null + } + + expect(video.files).to.have.lengthOf(attributes.files.length) + expect(video.tags).to.deep.equal(attributes.tags) + expect(video.account.name).to.equal(attributes.account.name) + expect(video.account.host).to.equal(attributes.account.host) + expect(video.channel.displayName).to.equal(attributes.channel.displayName) + expect(video.channel.name).to.equal(attributes.channel.name) + expect(video.channel.host).to.equal(attributes.account.host) + expect(video.channel.isLocal).to.equal(attributes.channel.isLocal) + expect(video.channel.createdAt).to.exist + expect(dateIsValid(video.channel.updatedAt.toString())).to.be.true + expect(video.commentsEnabled).to.equal(attributes.commentsEnabled) + expect(video.downloadEnabled).to.equal(attributes.downloadEnabled) + + expect(video.thumbnailPath).to.exist + await testImageGeneratedByFFmpeg(server.url, attributes.thumbnailfile || attributes.fixture, video.thumbnailPath) + + if (attributes.previewfile) { + expect(video.previewPath).to.exist + await testImageGeneratedByFFmpeg(server.url, attributes.previewfile, video.previewPath) + } + + await completeWebVideoFilesCheck({ server, originServer, videoUUID: video.uuid, ...pick(attributes, [ 'fixture', 'files' ]) }) +} + +async function checkVideoFilesWereRemoved (options: { + server: PeerTubeServer + video: VideoDetails + captions?: VideoCaption[] + onlyVideoFiles?: boolean // default false +}) { + const { video, server, captions = [], onlyVideoFiles = false } = options + + const webVideoFiles = video.files || [] + const hlsFiles = video.streamingPlaylists[0]?.files || [] + + const thumbnailName = basename(video.thumbnailPath) + const previewName = basename(video.previewPath) + + const torrentNames = webVideoFiles.concat(hlsFiles).map(f => basename(f.torrentUrl)) + + const captionNames = captions.map(c => basename(c.captionPath)) + + const webVideoFilenames = webVideoFiles.map(f => basename(f.fileUrl)) + const hlsFilenames = hlsFiles.map(f => basename(f.fileUrl)) + + let directories: { [ directory: string ]: string[] } = { + videos: webVideoFilenames, + redundancy: webVideoFilenames, + [join('playlists', 'hls')]: hlsFilenames, + [join('redundancy', 'hls')]: hlsFilenames + } + + if (onlyVideoFiles !== true) { + directories = { + ...directories, + + thumbnails: [ thumbnailName ], + previews: [ previewName ], + torrents: torrentNames, + captions: captionNames + } + } + + for (const directory of Object.keys(directories)) { + const directoryPath = server.servers.buildDirectory(directory) + + const directoryExists = await pathExists(directoryPath) + if (directoryExists === false) continue + + const existingFiles = await readdir(directoryPath) + for (const existingFile of existingFiles) { + for (const shouldNotExist of directories[directory]) { + expect(existingFile, `File ${existingFile} should not exist in ${directoryPath}`).to.not.contain(shouldNotExist) + } + } + } +} + +async function saveVideoInServers (servers: PeerTubeServer[], uuid: string) { + for (const server of servers) { + server.store.videoDetails = await server.videos.get({ id: uuid }) + } +} + +function checkUploadVideoParam (options: { + server: PeerTubeServer + token: string + attributes: Partial + expectedStatus?: HttpStatusCodeType + completedExpectedStatus?: HttpStatusCodeType + mode?: 'legacy' | 'resumable' +}) { + const { server, token, attributes, completedExpectedStatus, expectedStatus, mode = 'legacy' } = options + + return mode === 'legacy' + ? server.videos.buildLegacyUpload({ token, attributes, expectedStatus: expectedStatus || completedExpectedStatus }) + : server.videos.buildResumeUpload({ + token, + attributes, + expectedStatus, + completedExpectedStatus, + path: '/api/v1/videos/upload-resumable' + }) +} + +// serverNumber starts from 1 +async function uploadRandomVideoOnServers ( + servers: PeerTubeServer[], + serverNumber: number, + additionalParams?: VideoEdit & { prefixName?: string } +) { + const server = servers.find(s => s.serverNumber === serverNumber) + const res = await server.videos.randomUpload({ wait: false, additionalParams }) + + await waitJobs(servers) + + return res +} + +// --------------------------------------------------------------------------- + +export { + completeVideoCheck, + completeWebVideoFilesCheck, + checkUploadVideoParam, + uploadRandomVideoOnServers, + checkVideoFilesWereRemoved, + saveVideoInServers +} diff --git a/packages/tests/src/shared/views.ts b/packages/tests/src/shared/views.ts new file mode 100644 index 000000000..b791eff25 --- /dev/null +++ b/packages/tests/src/shared/views.ts @@ -0,0 +1,93 @@ +import type { FfmpegCommand } from 'fluent-ffmpeg' +import { wait } from '@peertube/peertube-core-utils' +import { VideoCreateResult, VideoPrivacy } from '@peertube/peertube-models' +import { + createMultipleServers, + doubleFollow, + PeerTubeServer, + setAccessTokensToServers, + setDefaultVideoChannel, + waitJobs, + waitUntilLivePublishedOnAllServers +} from '@peertube/peertube-server-commands' + +async function processViewersStats (servers: PeerTubeServer[]) { + await wait(6000) + + for (const server of servers) { + await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } }) + await server.debug.sendCommand({ body: { command: 'process-video-viewers' } }) + } + + await waitJobs(servers) +} + +async function processViewsBuffer (servers: PeerTubeServer[]) { + for (const server of servers) { + await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } }) + } + + await waitJobs(servers) +} + +async function prepareViewsServers () { + const servers = await createMultipleServers(2) + await setAccessTokensToServers(servers) + await setDefaultVideoChannel(servers) + + await servers[0].config.updateCustomSubConfig({ + newConfig: { + live: { + enabled: true, + allowReplay: true, + transcoding: { + enabled: false + } + } + } + }) + + await doubleFollow(servers[0], servers[1]) + + return servers +} + +async function prepareViewsVideos (options: { + servers: PeerTubeServer[] + live: boolean + vod: boolean +}) { + const { servers } = options + + const liveAttributes = { + name: 'live video', + channelId: servers[0].store.channel.id, + privacy: VideoPrivacy.PUBLIC + } + + let ffmpegCommand: FfmpegCommand + let live: VideoCreateResult + let vod: VideoCreateResult + + if (options.live) { + live = await servers[0].live.create({ fields: liveAttributes }) + + ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: live.uuid }) + await waitUntilLivePublishedOnAllServers(servers, live.uuid) + } + + if (options.vod) { + vod = await servers[0].videos.quickUpload({ name: 'video' }) + } + + await waitJobs(servers) + + return { liveVideoId: live?.uuid, vodVideoId: vod?.uuid, ffmpegCommand } +} + +export { + processViewersStats, + prepareViewsServers, + processViewsBuffer, + prepareViewsVideos +} diff --git a/packages/tests/src/shared/webtorrent.ts b/packages/tests/src/shared/webtorrent.ts new file mode 100644 index 000000000..1be54426a --- /dev/null +++ b/packages/tests/src/shared/webtorrent.ts @@ -0,0 +1,58 @@ +import { expect } from 'chai' +import { readFile } from 'fs/promises' +import parseTorrent from 'parse-torrent' +import { basename, join } from 'path' +import type { Instance, Torrent } from 'webtorrent' +import { VideoFile } from '@peertube/peertube-models' +import { PeerTubeServer } from '@peertube/peertube-server-commands' + +let webtorrent: Instance + +export async function checkWebTorrentWorks (magnetUri: string, pathMatch?: RegExp) { + const torrent = await webtorrentAdd(magnetUri, true) + + expect(torrent.files).to.be.an('array') + expect(torrent.files.length).to.equal(1) + expect(torrent.files[0].path).to.exist.and.to.not.equal('') + + if (pathMatch) { + expect(torrent.files[0].path).match(pathMatch) + } +} + +export async function parseTorrentVideo (server: PeerTubeServer, file: VideoFile) { + const torrentName = basename(file.torrentUrl) + const torrentPath = server.servers.buildDirectory(join('torrents', torrentName)) + + const data = await readFile(torrentPath) + + return parseTorrent(data) +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +async function webtorrentAdd (torrentId: string, refreshWebTorrent = false) { + const WebTorrent = (await import('webtorrent')).default + + if (webtorrent && refreshWebTorrent) webtorrent.destroy() + if (!webtorrent || refreshWebTorrent) webtorrent = new WebTorrent() + + webtorrent.on('error', err => console.error('Error in webtorrent', err)) + + return new Promise(res => { + const torrent = webtorrent.add(torrentId, res) + + torrent.on('error', err => console.error('Error in webtorrent torrent', err)) + torrent.on('warning', warn => { + const msg = typeof warn === 'string' + ? warn + : warn.message + + if (msg.includes('Unsupported')) return + + console.error('Warning in webtorrent torrent', warn) + }) + }) +} diff --git a/packages/tests/tsconfig.json b/packages/tests/tsconfig.json new file mode 100644 index 000000000..91a74b4be --- /dev/null +++ b/packages/tests/tsconfig.json @@ -0,0 +1,27 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "baseUrl": "./", + "rootDir": "src", + "tsBuildInfoFile": "./dist/.tsbuildinfo", + "paths": { + "@tests/*": [ "src/*" ] + } + }, + "references": [ + { "path": "../core-utils" }, + { "path": "../ffmpeg" }, + { "path": "../models" }, + { "path": "../node-utils" }, + { "path": "../typescript-utils" }, + { "path": "../server-commands" }, + { "path": "../../server/tsconfig.lib.json" } + ], + "include": [ + "./src/**/*.ts" + ], + "exclude": [ + "./fixtures" + ] +} diff --git a/packages/types/README.md b/packages/types-generator/README.md similarity index 100% rename from packages/types/README.md rename to packages/types-generator/README.md diff --git a/packages/types-generator/generate-package.ts b/packages/types-generator/generate-package.ts new file mode 100644 index 000000000..2b2f51623 --- /dev/null +++ b/packages/types-generator/generate-package.ts @@ -0,0 +1,107 @@ +import { execSync } from 'child_process' +import depcheck, { PackageDependencies } from 'depcheck' +import { readJson, remove, writeJSON } from 'fs-extra/esm' +import { copyFile, writeFile } from 'fs/promises' +import { join, resolve } from 'path' +import { currentDir, root } from '@peertube/peertube-node-utils' + +if (!process.argv[2]) { + console.error('Need version as argument') + process.exit(-1) +} + +const version = process.argv[2] +console.log('Will generate package version %s.', version) + +run() + .then(() => process.exit(0)) + .catch(err => { + console.error(err) + process.exit(-1) + }) + +async function run () { + const typesPath = currentDir(import.meta.url) + const typesDistPath = join(typesPath, 'dist') + + await remove(typesDistPath) + + const typesDistPackageJsonPath = join(typesDistPath, 'package.json') + const typesDistGitIgnorePath = join(typesDistPath, '.gitignore') + + const mainPackageJson = await readJson(join(root(), 'package.json')) + + const typesTsConfigPath = join(typesPath, 'tsconfig.types.json') + + const distTsConfigPath = join(typesPath, 'tsconfig.dist.json') + const distTsConfig = await readJson(distTsConfigPath) + + const clientPackageJson = await readJson(join(root(), 'client', 'package.json')) + + await remove(typesDistPath) + execSync(`npm run tsc -- -b ${typesTsConfigPath} --verbose`, { stdio: 'inherit' }) + execSync(`npm run resolve-tspaths -- --project ${distTsConfigPath} --src ${typesDistPath} --out ${typesDistPath}`, { stdio: 'inherit' }) + + const allDependencies = Object.assign( + mainPackageJson.dependencies, + mainPackageJson.devDependencies, + clientPackageJson.dependencies, + clientPackageJson.devDependencies + ) as PackageDependencies + + const toIgnore = Object.keys(distTsConfig?.compilerOptions?.paths || []) + + // https://github.com/depcheck/depcheck#api + const depcheckOptions = { + parsers: { '**/*.ts': depcheck.parser.typescript }, + detectors: [ + depcheck.detector.requireCallExpression, + depcheck.detector.importDeclaration + ], + ignoreMatches: toIgnore, + package: { dependencies: allDependencies } + } + + const result = await depcheck(typesDistPath, depcheckOptions) + + if (Object.keys(result.invalidDirs).length !== 0) { + console.error('Invalid directories detected.', { invalidDirs: result.invalidDirs }) + process.exit(-1) + } + + if (Object.keys(result.invalidFiles).length !== 0) { + console.error('Invalid files detected.', { invalidFiles: result.invalidFiles }) + process.exit(-1) + } + + const unusedDependencies = result.dependencies + + console.log(`Removing ${Object.keys(unusedDependencies).length} unused dependencies.`) + const dependencies = Object + .keys(allDependencies) + .filter(dependencyName => !unusedDependencies.includes(dependencyName) && !toIgnore.includes(dependencyName)) + .reduce((dependencies, dependencyName) => { + dependencies[dependencyName] = allDependencies[dependencyName] + return dependencies + }, {}) + + const { description, licence, engines, author, repository } = mainPackageJson + const typesPackageJson = { + name: '@peertube/peertube-types', + description, + version, + private: false, + license: licence, + engines, + author, + repository, + dependencies + } + console.log(`Writing package.json to ${typesDistPackageJsonPath}`) + await writeJSON(typesDistPackageJsonPath, typesPackageJson, { spaces: 2 }) + + console.log(`Writing git ignore to ${typesDistGitIgnorePath}`) + await writeFile(typesDistGitIgnorePath, '*.tsbuildinfo') + + await copyFile(resolve(typesPath, './README.md'), resolve(typesDistPath, './README.md')) +} diff --git a/packages/types-generator/package.json b/packages/types-generator/package.json new file mode 100644 index 000000000..a72235851 --- /dev/null +++ b/packages/types-generator/package.json @@ -0,0 +1,9 @@ +{ + "name": "@peertube/peertube-types-generator", + "private": true, + "version": "0.0.0", + "type": "module", + "dependencies": { + "@peertube/peertube-core-utils": "*" + } +} diff --git a/packages/types-generator/src/client/index.ts b/packages/types-generator/src/client/index.ts new file mode 100644 index 000000000..8868dd5b0 --- /dev/null +++ b/packages/types-generator/src/client/index.ts @@ -0,0 +1 @@ +export * from '@client/types/index.js' diff --git a/packages/types-generator/src/client/tsconfig.types.json b/packages/types-generator/src/client/tsconfig.types.json new file mode 100644 index 000000000..f60b43f07 --- /dev/null +++ b/packages/types-generator/src/client/tsconfig.types.json @@ -0,0 +1,19 @@ +{ + "extends": "../../../../tsconfig.base.json", + "compilerOptions": { + "stripInternal": true, + "removeComments": false, + "emitDeclarationOnly": true, + "outDir": "../../dist/client/", + "rootDir": "./", + "baseUrl": "./", + "tsBuildInfoFile": "../../dist/tsconfig.client.types.tsbuildinfo", + "paths": { + "@client/*": [ "../../../../client/src/*" ] + } + }, + "references": [ + { "path": "../../../../client/tsconfig.types.json" } + ], + "files": [ "./index.ts" ] +} diff --git a/packages/types-generator/src/index.ts b/packages/types-generator/src/index.ts new file mode 100644 index 000000000..b5669ea24 --- /dev/null +++ b/packages/types-generator/src/index.ts @@ -0,0 +1,3 @@ +export * from '@server/types/index.js' +export * from '@server/types/models/index.js' +export * from '@peertube/peertube-models' diff --git a/packages/types-generator/tests/test.ts b/packages/types-generator/tests/test.ts new file mode 100644 index 000000000..bfdcdeed5 --- /dev/null +++ b/packages/types-generator/tests/test.ts @@ -0,0 +1,32 @@ +import { RegisterServerOptions, Video } from '../dist/index.js' +import { RegisterClientOptions } from '../dist/client/index.js' + +function register1 ({ registerHook }: RegisterServerOptions) { + registerHook({ + target: 'action:application.listening', + handler: () => console.log('hello') + }) +} + +function register2 ({ registerHook, peertubeHelpers }: RegisterClientOptions) { + registerHook({ + target: 'action:admin-plugin-settings.init', + handler: ({ npmName }: { npmName: string }) => { + if ('peertube-plugin-transcription' !== npmName) { + return + } + }, + }) + + registerHook({ + target: 'action:video-watch.video.loaded', + handler: ({ video }: { video: Video }) => { + fetch(`${peertubeHelpers.getBaseRouterRoute()}/videos/${video.uuid}/captions`, { + method: 'PUT', + headers: peertubeHelpers.getAuthHeader(), + }) + .then((res) => res.json()) + .then((data) => console.log('Hi %s.', data)) + }, + }) +} diff --git a/packages/types-generator/tsconfig.dist.json b/packages/types-generator/tsconfig.dist.json new file mode 100644 index 000000000..6c24a67ba --- /dev/null +++ b/packages/types-generator/tsconfig.dist.json @@ -0,0 +1,17 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "typeRoots": [ + "node_modules/@types", + "client/node_modules/@types" + ], + "baseUrl": "./dist", + "paths": { + "@server/*": [ "server/server/*" ], + "@client/*": [ "client/*" ], + "@peertube/peertube-models": [ "peertube-models" ], + "@peertube/peertube-typescript-utils": [ "peertube-typescript-utils" ] + } + } +} + diff --git a/packages/types-generator/tsconfig.json b/packages/types-generator/tsconfig.json new file mode 100644 index 000000000..fe09d9395 --- /dev/null +++ b/packages/types-generator/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": ".", + "noEmit": true + }, + "files": [ "./generate-package.ts" ], + "references": [ + { "path": "../node-utils" } + ] +} diff --git a/packages/types-generator/tsconfig.types.json b/packages/types-generator/tsconfig.types.json new file mode 100644 index 000000000..a3a1b7c0d --- /dev/null +++ b/packages/types-generator/tsconfig.types.json @@ -0,0 +1,23 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "stripInternal": true, + "removeComments": false, + "emitDeclarationOnly": true, + "sourceMap": false, + "outDir": "./dist/", + "baseUrl": "./", + "rootDir": "./src", + "tsBuildInfoFile": "./dist/tsconfig.server.types.tsbuildinfo", + "paths": { + "@server/*": [ "../../server/server/*" ] + } + }, + "references": [ + { "path": "../models/tsconfig.types.json" }, + { "path": "../typescript-utils/tsconfig.types.json" }, + { "path": "../../server/tsconfig.types.json" }, + { "path": "./src/client/tsconfig.types.json" } + ], + "files": ["./src/index.ts"] +} diff --git a/packages/types/generate-package.ts b/packages/types/generate-package.ts deleted file mode 100644 index 125259bb4..000000000 --- a/packages/types/generate-package.ts +++ /dev/null @@ -1,96 +0,0 @@ -import { execSync } from 'child_process' -import depcheck, { PackageDependencies } from 'depcheck' -import { copyFile, readJson, remove, writeFile, writeJSON } from 'fs-extra' -import { join, resolve } from 'path' -import { root } from '../../shared/core-utils' - -if (!process.argv[2]) { - console.error('Need version as argument') - process.exit(-1) -} - -const version = process.argv[2] -console.log('Will generate package version %s.', version) - -run() - .then(() => process.exit(0)) - .catch(err => { - console.error(err) - process.exit(-1) - }) - -async function run () { - const typesPath = __dirname - const typesDistPath = join(typesPath, 'dist') - const typesDistPackageJsonPath = join(typesDistPath, 'package.json') - const typesDistGitIgnorePath = join(typesDistPath, '.gitignore') - const mainPackageJson = await readJson(join(root(), 'package.json')) - const distTsConfigPath = join(typesPath, 'tsconfig.dist.json') - const distTsConfig = await readJson(distTsConfigPath) - const clientPackageJson = await readJson(join(root(), 'client', 'package.json')) - - await remove(typesDistPath) - execSync('npm run tsc -- -b --verbose packages/types', { stdio: 'inherit' }) - execSync(`npm run resolve-tspaths -- --project ${distTsConfigPath} --src ${typesDistPath} --out ${typesDistPath}`, { stdio: 'inherit' }) - - const allDependencies = Object.assign( - mainPackageJson.dependencies, - mainPackageJson.devDependencies, - clientPackageJson.dependencies, - clientPackageJson.devDependencies - ) as PackageDependencies - - // https://github.com/depcheck/depcheck#api - const depcheckOptions = { - parsers: { '**/*.ts': depcheck.parser.typescript }, - detectors: [ - depcheck.detector.requireCallExpression, - depcheck.detector.importDeclaration - ], - ignoreMatches: Object.keys(distTsConfig?.compilerOptions?.paths || []), - package: { dependencies: allDependencies } - } - - const result = await depcheck(typesDistPath, depcheckOptions) - - if (Object.keys(result.invalidDirs).length !== 0) { - console.error('Invalid directories detected.', { invalidDirs: result.invalidDirs }) - process.exit(-1) - } - - if (Object.keys(result.invalidFiles).length !== 0) { - console.error('Invalid files detected.', { invalidFiles: result.invalidFiles }) - process.exit(-1) - } - - const unusedDependencies = result.dependencies - - console.log(`Removing ${Object.keys(unusedDependencies).length} unused dependencies.`) - const dependencies = Object - .keys(allDependencies) - .filter(dependencyName => !unusedDependencies.includes(dependencyName)) - .reduce((dependencies, dependencyName) => { - dependencies[dependencyName] = allDependencies[dependencyName] - return dependencies - }, {}) - - const { description, licence, engines, author, repository } = mainPackageJson - const typesPackageJson = { - name: '@peertube/peertube-types', - description, - version, - private: false, - license: licence, - engines, - author, - repository, - dependencies - } - console.log(`Writing package.json to ${typesDistPackageJsonPath}`) - await writeJSON(typesDistPackageJsonPath, typesPackageJson, { spaces: 2 }) - - console.log(`Writing git ignore to ${typesDistGitIgnorePath}`) - await writeFile(typesDistGitIgnorePath, '*.tsbuildinfo') - - await copyFile(resolve(typesPath, './README.md'), resolve(typesDistPath, './README.md')) -} diff --git a/packages/types/src/client/index.ts b/packages/types/src/client/index.ts deleted file mode 100644 index 5ee10ecb8..000000000 --- a/packages/types/src/client/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from '@client/types' diff --git a/packages/types/src/client/tsconfig.json b/packages/types/src/client/tsconfig.json deleted file mode 100644 index bb76fbe21..000000000 --- a/packages/types/src/client/tsconfig.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "extends": "../../../../tsconfig.base.json", - "compilerOptions": { - "stripInternal": true, - "removeComments": false, - "emitDeclarationOnly": true, - "outDir": "../../dist/client/", - "rootDir": "./", - "tsBuildInfoFile": "../../dist/tsconfig.client.types.tsbuildinfo" - }, - "references": [ - { "path": "../../../../client/tsconfig.types.json" } - ], - "files": ["index.ts"] -} diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts deleted file mode 100644 index a8adca287..000000000 --- a/packages/types/src/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from '@server/types' -export * from '@server/types/models' -export * from '@shared/models' diff --git a/packages/types/tests/test.ts b/packages/types/tests/test.ts deleted file mode 100644 index 8c53320a1..000000000 --- a/packages/types/tests/test.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { RegisterServerOptions, Video } from '../dist' -import { RegisterClientOptions } from '../dist/client' - -function register1 ({ registerHook }: RegisterServerOptions) { - registerHook({ - target: 'action:application.listening', - handler: () => console.log('hello') - }) -} - -function register2 ({ registerHook, peertubeHelpers }: RegisterClientOptions) { - registerHook({ - target: 'action:admin-plugin-settings.init', - handler: ({ npmName }: { npmName: string }) => { - if ('peertube-plugin-transcription' !== npmName) { - return - } - }, - }) - - registerHook({ - target: 'action:video-watch.video.loaded', - handler: ({ video }: { video: Video }) => { - fetch(`${peertubeHelpers.getBaseRouterRoute()}/videos/${video.uuid}/captions`, { - method: 'PUT', - headers: peertubeHelpers.getAuthHeader(), - }) - .then((res) => res.json()) - .then((data) => console.log('Hi %s.', data)) - }, - }) -} diff --git a/packages/types/tsconfig.dist.json b/packages/types/tsconfig.dist.json deleted file mode 100644 index fbc92712b..000000000 --- a/packages/types/tsconfig.dist.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "extends": "./tsconfig.json", - "compilerOptions": { - "typeRoots": [ - "node_modules/@types", - "client/node_modules/@types" - ], - "baseUrl": "./dist", - "paths": { - "@server/*": [ "server/*" ], - "@shared/*": [ "shared/*" ], - "@client/*": [ "client/*" ] - } - } -} - diff --git a/packages/types/tsconfig.json b/packages/types/tsconfig.json deleted file mode 100644 index f8e16f6b4..000000000 --- a/packages/types/tsconfig.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "stripInternal": true, - "removeComments": false, - "emitDeclarationOnly": true, - "outDir": "./dist/", - "baseUrl": "./src/", - "rootDir": "./src/", - "tsBuildInfoFile": "./dist/tsconfig.server.types.tsbuildinfo", - "paths": { - "@server/*": [ "../../../server/*" ], - "@shared/*": [ "../../../shared/*" ], - "@client/*": [ "../../../client/src/*" ] - } - }, - "references": [ - { "path": "../../shared/tsconfig.types.json" }, - { "path": "../../server/tsconfig.types.json" }, - { "path": "./src/client/tsconfig.json" } - ], - "files": ["./src/index.ts"] -} diff --git a/packages/typescript-utils/package.json b/packages/typescript-utils/package.json new file mode 100644 index 000000000..9608bb018 --- /dev/null +++ b/packages/typescript-utils/package.json @@ -0,0 +1,19 @@ +{ + "name": "@peertube/peertube-typescript-utils", + "private": true, + "version": "0.0.0", + "main": "dist/index.js", + "files": [ "dist" ], + "exports": { + "types": "./dist/index.d.ts", + "peertube:tsx": "./src/index.ts", + "default": "./dist/index.js" + }, + "type": "module", + "devDependencies": {}, + "scripts": { + "build": "tsc", + "watch": "tsc -w" + }, + "dependencies": {} +} diff --git a/packages/typescript-utils/src/index.ts b/packages/typescript-utils/src/index.ts new file mode 100644 index 000000000..fdc633235 --- /dev/null +++ b/packages/typescript-utils/src/index.ts @@ -0,0 +1 @@ +export * from './types.js' diff --git a/shared/typescript-utils/types.ts b/packages/typescript-utils/src/types.ts similarity index 100% rename from shared/typescript-utils/types.ts rename to packages/typescript-utils/src/types.ts diff --git a/packages/typescript-utils/tsconfig.json b/packages/typescript-utils/tsconfig.json new file mode 100644 index 000000000..58fa2330b --- /dev/null +++ b/packages/typescript-utils/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "src", + "tsBuildInfoFile": "./dist/.tsbuildinfo" + } +} diff --git a/packages/typescript-utils/tsconfig.types.json b/packages/typescript-utils/tsconfig.types.json new file mode 100644 index 000000000..e666b4ca2 --- /dev/null +++ b/packages/typescript-utils/tsconfig.types.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../types-generator/dist/peertube-typescript-utils", + "tsBuildInfoFile": "../types-generator/dist/peertube-typescript-utils/.tsbuildinfo", + "stripInternal": true, + "removeComments": false, + "emitDeclarationOnly": true + } +} diff --git a/scripts/benchmark.ts b/scripts/benchmark.ts index 92fbd5490..c4e4c7275 100644 --- a/scripts/benchmark.ts +++ b/scripts/benchmark.ts @@ -1,8 +1,14 @@ import autocannon, { printResult } from 'autocannon' import { program } from 'commander' -import { writeJson } from 'fs-extra' -import { Video, VideoPrivacy } from '@shared/models' -import { createMultipleServers, doubleFollow, killallServers, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' +import { writeJson } from 'fs-extra/esm' +import { Video, VideoPrivacy } from '@peertube/peertube-models' +import { + createMultipleServers, + doubleFollow, + killallServers, + PeerTubeServer, + setAccessTokensToServers +} from '@peertube/peertube-server-commands' let servers: PeerTubeServer[] // First server diff --git a/scripts/build/peertube-cli.sh b/scripts/build/peertube-cli.sh new file mode 100644 index 000000000..51886a0db --- /dev/null +++ b/scripts/build/peertube-cli.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -eu + +cd ./apps/peertube-cli +rm -rf ./dist + +../../node_modules/.bin/tsc -b --verbose +rm -rf ./dist +mkdir ./dist + +node ./scripts/build.js diff --git a/scripts/build/peertube-runner.sh b/scripts/build/peertube-runner.sh index 7f6ad5ede..321d10b4b 100755 --- a/scripts/build/peertube-runner.sh +++ b/scripts/build/peertube-runner.sh @@ -2,12 +2,11 @@ set -eu - -cd ./packages/peertube-runner +cd ./apps/peertube-runner rm -rf ./dist ../../node_modules/.bin/tsc -b --verbose rm -rf ./dist mkdir ./dist -./node_modules/.bin/esbuild ./peertube-runner.ts --bundle --platform=node --target=node16 --external:"./lib-cov/fluent-ffmpeg" --external:pg-hstore --outfile=dist/peertube-runner.js +node ./scripts/build.js diff --git a/scripts/build/server.sh b/scripts/build/server.sh index a2dfc3dd9..bbab633a2 100755 --- a/scripts/build/server.sh +++ b/scripts/build/server.sh @@ -2,10 +2,11 @@ set -eu -rm -rf ./dist +rm -rf ./dist ./packages/*/dist -npm run tsc -- -b --verbose +npm run tsc -- -b --verbose server/tsconfig.json npm run resolve-tspaths:server -cp -r "./server/static" "./server/assets" "./dist/server" -cp -r "./server/lib/emails" "./dist/server/lib" +cp -r "./server/server/static" "./server/server/assets" ./dist/server +cp -r "./server/server/lib/emails" "./dist/server/lib" +cp "./server/scripts/upgrade.sh" "./dist/scripts" diff --git a/scripts/build/tests.sh b/scripts/build/tests.sh new file mode 100755 index 000000000..f94dde2a4 --- /dev/null +++ b/scripts/build/tests.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -eu + +rm -rf ./packages/tests/dist + +npm run tsc -- -b --verbose ./packages/tests/tsconfig.json +npm run resolve-tspaths:server-lib +npm run resolve-tspaths:tests diff --git a/scripts/ci.sh b/scripts/ci.sh index 9fb67f634..64968cf94 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -10,7 +10,7 @@ fi retries=3 speedFactor="${2:-1}" -runTest () { +runJSTest () { jobname=$1 shift @@ -24,7 +24,7 @@ runTest () { joblog="$jobname-ci.log" parallel -j $jobs --retries $retries \ - "echo Trying {} >> $joblog; npm run mocha -- -c --timeout 30000 --exit --bail {}" \ + "echo Trying {} >> $joblog; npm run mocha -- --timeout 30000 --no-config -c --exit --bail {}" \ ::: $files cat "$joblog" | sort | uniq -c @@ -32,92 +32,116 @@ runTest () { } findTestFiles () { - exception="-not -name index.js" + exception="-not -name index.js -not -name index.ts -not -name *.d.ts" if [ ! -z ${2+x} ]; then exception="$exception -not -name $2" fi - find $1 -type f -name "*.js" $exception | xargs echo + find $1 -type f \( -name "*.js" -o -name "*.ts" \) $exception | xargs echo } if [ "$1" = "types-package" ]; then npm run generate-types-package 0.0.0 - npm run tsc -- --noEmit --esModuleInterop packages/types/tests/test.ts + + # Test on in independent directory + rm -fr /tmp/types-generator + mkdir -p /tmp/types-generator + cp -r packages/types-generator/tests /tmp/types-generator/tests + cp -r packages/types-generator/dist /tmp/types-generator/dist + (cd /tmp/types-generator/dist && npm install) + + npm run tsc -- --noEmit --esModuleInterop --moduleResolution node16 /tmp/types-generator/tests/test.ts + rm -r /tmp/types-generator elif [ "$1" = "client" ]; then npm run build + npm run build:tests - feedsFiles=$(findTestFiles ./dist/server/tests/feeds) - helperFiles=$(findTestFiles ./dist/server/tests/helpers) - libFiles=$(findTestFiles ./dist/server/tests/lib) - miscFiles="./dist/server/tests/client.js ./dist/server/tests/misc-endpoints.js" + feedsFiles=$(findTestFiles ./packages/tests/dist/feeds) + miscFiles="./packages/tests/dist/client.js ./packages/tests/dist/misc-endpoints.js" # Not in their own task, they need an index.html - pluginFiles="./dist/server/tests/plugins/html-injection.js ./dist/server/tests/api/server/plugins.js" + pluginFiles="./packages/tests/dist/plugins/html-injection.js ./packages/tests/dist/api/server/plugins.js" - MOCHA_PARALLEL=true runTest "$1" $((2*$speedFactor)) $feedsFiles $helperFiles $miscFiles $pluginFiles $libFiles + MOCHA_PARALLEL=true runJSTest "$1" $((2*$speedFactor)) $feedsFiles $miscFiles $pluginFiles + + # Use TS tests directly because we import server files + helperFiles=$(findTestFiles ./packages/tests/src/server-helpers) + libFiles=$(findTestFiles ./packages/tests/src/server-lib) + + npm run mocha -- --timeout 30000 -c --exit --bail $libFiles $helperFiles elif [ "$1" = "cli-plugin" ]; then # Simulate HTML mkdir -p "./client/dist/en-US/" cp "./client/src/index.html" "./client/dist/en-US/index.html" npm run build:server - npm run setup:cli + npm run build:tests + npm run build:peertube-cli - pluginsFiles=$(findTestFiles ./dist/server/tests/plugins html-injection.js) - cliFiles=$(findTestFiles ./dist/server/tests/cli) + # html-injection test needs an HTML file + pluginsFiles=$(findTestFiles ./packages/tests/dist/plugins html-injection.js) + cliFiles=$(findTestFiles ./packages/tests/dist/cli) - MOCHA_PARALLEL=true runTest "$1" $((2*$speedFactor)) $pluginsFiles - runTest "$1" 1 $cliFiles + MOCHA_PARALLEL=true runJSTest "$1" $((2*$speedFactor)) $pluginsFiles + runJSTest "$1" 1 $cliFiles elif [ "$1" = "api-1" ]; then npm run build:server + npm run build:tests - checkParamFiles=$(findTestFiles ./dist/server/tests/api/check-params) - notificationsFiles=$(findTestFiles ./dist/server/tests/api/notifications) - searchFiles=$(findTestFiles ./dist/server/tests/api/search) + checkParamFiles=$(findTestFiles ./packages/tests/dist/api/check-params) + notificationsFiles=$(findTestFiles ./packages/tests/dist/api/notifications) + searchFiles=$(findTestFiles ./packages/tests/dist/api/search) - MOCHA_PARALLEL=true runTest "$1" $((3*$speedFactor)) $notificationsFiles $searchFiles $checkParamFiles + MOCHA_PARALLEL=true runJSTest "$1" $((3*$speedFactor)) $notificationsFiles $searchFiles $checkParamFiles elif [ "$1" = "api-2" ]; then npm run build:server + npm run build:tests - liveFiles=$(findTestFiles ./dist/server/tests/api/live) - serverFiles=$(findTestFiles ./dist/server/tests/api/server plugins.js) - usersFiles=$(findTestFiles ./dist/server/tests/api/users) + liveFiles=$(findTestFiles ./packages/tests/dist/api/live) + # plugins test needs an HTML file + serverFiles=$(findTestFiles ./packages/tests/dist/api/server plugins.js) + usersFiles=$(findTestFiles ./packages/tests/dist/api/users) - MOCHA_PARALLEL=true runTest "$1" $((3*$speedFactor)) $liveFiles $serverFiles $usersFiles + MOCHA_PARALLEL=true runJSTest "$1" $((3*$speedFactor)) $liveFiles $serverFiles $usersFiles elif [ "$1" = "api-3" ]; then npm run build:server + npm run build:tests - videosFiles=$(findTestFiles ./dist/server/tests/api/videos) - viewsFiles=$(findTestFiles ./dist/server/tests/api/views) + videosFiles=$(findTestFiles ./packages/tests/dist/api/videos) + viewsFiles=$(findTestFiles ./packages/tests/dist/api/views) - MOCHA_PARALLEL=true runTest "$1" $((3*$speedFactor)) $viewsFiles $videosFiles + MOCHA_PARALLEL=true runJSTest "$1" $((3*$speedFactor)) $viewsFiles $videosFiles elif [ "$1" = "api-4" ]; then npm run build:server + npm run build:tests - moderationFiles=$(findTestFiles ./dist/server/tests/api/moderation) - redundancyFiles=$(findTestFiles ./dist/server/tests/api/redundancy) - objectStorageFiles=$(findTestFiles ./dist/server/tests/api/object-storage) - activitypubFiles=$(findTestFiles ./dist/server/tests/api/activitypub) + moderationFiles=$(findTestFiles ./packages/tests/dist/api/moderation) + redundancyFiles=$(findTestFiles ./packages/tests/dist/api/redundancy) + objectStorageFiles=$(findTestFiles ./packages/tests/dist/api/object-storage) + activitypubFiles=$(findTestFiles ./packages/tests/dist/api/activitypub) - MOCHA_PARALLEL=true runTest "$1" $((2*$speedFactor)) $moderationFiles $redundancyFiles $activitypubFiles $objectStorageFiles + MOCHA_PARALLEL=true runJSTest "$1" $((2*$speedFactor)) $moderationFiles $redundancyFiles $activitypubFiles $objectStorageFiles elif [ "$1" = "api-5" ]; then npm run build:server + npm run build:tests - transcodingFiles=$(findTestFiles ./dist/server/tests/api/transcoding) - runnersFiles=$(findTestFiles ./dist/server/tests/api/runners) + transcodingFiles=$(findTestFiles ./packages/tests/dist/api/transcoding) + runnersFiles=$(findTestFiles ./packages/tests/dist/api/runners) - MOCHA_PARALLEL=true runTest "$1" $((2*$speedFactor)) $transcodingFiles $runnersFiles + MOCHA_PARALLEL=true runJSTest "$1" $((2*$speedFactor)) $transcodingFiles $runnersFiles elif [ "$1" = "external-plugins" ]; then npm run build:server + npm run build:tests npm run build:peertube-runner - externalPluginsFiles=$(findTestFiles ./dist/server/tests/external-plugins) - peertubeRunnerFiles=$(findTestFiles ./dist/server/tests/peertube-runner) + externalPluginsFiles=$(findTestFiles ./packages/tests/dist/external-plugins) + peertubeRunnerFiles=$(findTestFiles ./packages/tests/dist/peertube-runner) - runTest "$1" 1 $externalPluginsFiles - MOCHA_PARALLEL=true runTest "$1" $((2*$speedFactor)) $peertubeRunnerFiles + runJSTest "$1" 1 $externalPluginsFiles + MOCHA_PARALLEL=true runJSTest "$1" $((2*$speedFactor)) $peertubeRunnerFiles elif [ "$1" = "lint" ]; then - npm run eslint -- --ext .ts "./server/**/*.ts" "shared/**/*.ts" "scripts/**/*.ts" + npm run eslint -- --ext .ts "server/**/*.ts" "scripts/**/*.ts" "packages/**/*.ts" "apps/**/*.ts" + npm run swagger-cli -- validate support/doc/api/openapi.yaml ( cd client diff --git a/scripts/client-build-stats.ts b/scripts/client-build-stats.ts index d5ecd5fea..3b26aa647 100644 --- a/scripts/client-build-stats.ts +++ b/scripts/client-build-stats.ts @@ -1,6 +1,6 @@ -import { readdir, stat } from 'fs-extra' +import { readdir, stat } from 'fs/promises' import { join } from 'path' -import { root } from '@shared/core-utils' +import { root } from '@peertube/peertube-node-utils' async function run () { const result = { diff --git a/scripts/create-generate-storyboard-job.ts b/scripts/create-generate-storyboard-job.ts deleted file mode 100644 index 47c08edac..000000000 --- a/scripts/create-generate-storyboard-job.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { program } from 'commander' -import { toCompleteUUID } from '@server/helpers/custom-validators/misc' -import { initDatabaseModels } from '@server/initializers/database' -import { JobQueue } from '@server/lib/job-queue' -import { VideoModel } from '@server/models/video/video' -import { StoryboardModel } from '@server/models/video/storyboard' - -program - .description('Generate videos storyboard') - .option('-v, --video [videoUUID]', 'Generate the storyboard of a specific video') - .option('-a, --all-videos', 'Generate missing storyboards of local videos') - .parse(process.argv) - -const options = program.opts() - -if (!options['video'] && !options['allVideos']) { - console.error('You need to choose videos for storyboard generation.') - process.exit(-1) -} - -run() - .then(() => process.exit(0)) - .catch(err => { - console.error(err) - process.exit(-1) - }) - -async function run () { - await initDatabaseModels(true) - - JobQueue.Instance.init() - - let ids: number[] = [] - - if (options['video']) { - const video = await VideoModel.load(toCompleteUUID(options['video'])) - - if (!video) { - console.error('Unknown video ' + options['video']) - process.exit(-1) - } - - if (video.remote === true) { - console.error('Cannot process a remote video') - process.exit(-1) - } - - if (video.isLive) { - console.error('Cannot process live video') - process.exit(-1) - } - - ids.push(video.id) - } else { - ids = await listLocalMissingStoryboards() - } - - for (const id of ids) { - const videoFull = await VideoModel.load(id) - - if (videoFull.isLive) continue - - await JobQueue.Instance.createJob({ - type: 'generate-video-storyboard', - payload: { - videoUUID: videoFull.uuid, - federate: true - } - }) - - console.log(`Created generate-storyboard job for ${videoFull.name}.`) - } -} - -async function listLocalMissingStoryboards () { - const ids = await VideoModel.listLocalIds() - const results: number[] = [] - - for (const id of ids) { - const storyboard = await StoryboardModel.loadByVideo(id) - if (!storyboard) results.push(id) - } - - return results -} diff --git a/scripts/create-import-video-file-job.ts b/scripts/create-import-video-file-job.ts deleted file mode 100644 index 9cb387d2e..000000000 --- a/scripts/create-import-video-file-job.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { program } from 'commander' -import { resolve } from 'path' -import { isUUIDValid, toCompleteUUID } from '@server/helpers/custom-validators/misc' -import { initDatabaseModels } from '../server/initializers/database' -import { JobQueue } from '../server/lib/job-queue' -import { VideoModel } from '../server/models/video/video' - -program - .option('-v, --video [videoUUID]', 'Video UUID') - .option('-i, --import [videoFile]', 'Video file') - .description('Import a video file to replace an already uploaded file or to add a new resolution') - .parse(process.argv) - -const options = program.opts() - -if (options.video === undefined || options.import === undefined) { - console.error('All parameters are mandatory.') - process.exit(-1) -} - -run() - .then(() => process.exit(0)) - .catch(err => { - console.error(err) - process.exit(-1) - }) - -async function run () { - await initDatabaseModels(true) - - const uuid = toCompleteUUID(options.video) - - if (isUUIDValid(uuid) === false) { - console.error('%s is not a valid video UUID.', options.video) - return - } - - const video = await VideoModel.load(uuid) - if (!video) throw new Error('Video not found.') - if (video.isOwned() === false) throw new Error('Cannot import files of a non owned video.') - - const dataInput = { - videoUUID: video.uuid, - filePath: resolve(options.import) - } - - JobQueue.Instance.init() - await JobQueue.Instance.createJob({ type: 'video-file-import', payload: dataInput }) - console.log('Import job for video %s created.', video.uuid) -} diff --git a/scripts/create-move-video-storage-job.ts b/scripts/create-move-video-storage-job.ts deleted file mode 100644 index 8537114eb..000000000 --- a/scripts/create-move-video-storage-job.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { program } from 'commander' -import { toCompleteUUID } from '@server/helpers/custom-validators/misc' -import { CONFIG } from '@server/initializers/config' -import { initDatabaseModels } from '@server/initializers/database' -import { JobQueue } from '@server/lib/job-queue' -import { moveToExternalStorageState } from '@server/lib/video-state' -import { VideoModel } from '@server/models/video/video' -import { VideoState, VideoStorage } from '@shared/models' - -program - .description('Move videos to another storage.') - .option('-o, --to-object-storage', 'Move videos in object storage') - .option('-v, --video [videoUUID]', 'Move a specific video') - .option('-a, --all-videos', 'Migrate all videos') - .parse(process.argv) - -const options = program.opts() - -if (!options['toObjectStorage']) { - console.error('You need to choose where to send video files.') - process.exit(-1) -} - -if (!options['video'] && !options['allVideos']) { - console.error('You need to choose which videos to move.') - process.exit(-1) -} - -if (options['toObjectStorage'] && !CONFIG.OBJECT_STORAGE.ENABLED) { - console.error('Object storage is not enabled on this instance.') - process.exit(-1) -} - -run() - .then(() => process.exit(0)) - .catch(err => { - console.error(err) - process.exit(-1) - }) - -async function run () { - await initDatabaseModels(true) - - JobQueue.Instance.init() - - let ids: number[] = [] - - if (options['video']) { - const video = await VideoModel.load(toCompleteUUID(options['video'])) - - if (!video) { - console.error('Unknown video ' + options['video']) - process.exit(-1) - } - - if (video.remote === true) { - console.error('Cannot process a remote video') - process.exit(-1) - } - - if (video.isLive) { - console.error('Cannot process live video') - process.exit(-1) - } - - if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { - console.error('This video is already being moved to external storage') - process.exit(-1) - } - - ids.push(video.id) - } else { - ids = await VideoModel.listLocalIds() - } - - for (const id of ids) { - const videoFull = await VideoModel.loadFull(id) - - if (videoFull.isLive) continue - - const files = videoFull.VideoFiles || [] - const hls = videoFull.getHLSPlaylist() - - if (files.some(f => f.storage === VideoStorage.FILE_SYSTEM) || hls?.storage === VideoStorage.FILE_SYSTEM) { - console.log('Processing video %s.', videoFull.name) - - const success = await moveToExternalStorageState({ video: videoFull, isNewVideo: false, transaction: undefined }) - - if (!success) { - console.error( - 'Cannot create move job for %s: job creation may have failed or there may be pending transcoding jobs for this video', - videoFull.name - ) - } - } - - console.log(`Created move-to-object-storage job for ${videoFull.name}.`) - } -} diff --git a/scripts/dev/cli.sh b/scripts/dev/cli.sh deleted file mode 100755 index 39ecaad94..000000000 --- a/scripts/dev/cli.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -eu - -rm -rf ./dist/server/tools/ - -( - cd ./server/tools - yarn install --pure-lockfile -) - -mkdir -p "./dist/server/tools" -cp -r "./server/tools/node_modules" "./dist/server/tools" - -cd ./server/tools -../../node_modules/.bin/tsc-watch --build --verbose --onSuccess 'sh -c "cd ../../ && npm run resolve-tspaths:server"' diff --git a/scripts/dev/peertube-cli.sh b/scripts/dev/peertube-cli.sh new file mode 100755 index 000000000..172bf038e --- /dev/null +++ b/scripts/dev/peertube-cli.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -eu + +rm -rf ./apps/peertube-cli/dist + +cd ./apps/peertube-cli + +../../node_modules/.bin/concurrently -k \ + "../../node_modules/.bin/tsc -w --noEmit" \ + "node ./scripts/watch.js" diff --git a/scripts/dev/peertube-runner.sh b/scripts/dev/peertube-runner.sh index e39259372..7bd756123 100755 --- a/scripts/dev/peertube-runner.sh +++ b/scripts/dev/peertube-runner.sh @@ -2,10 +2,10 @@ set -eu -rm -rf ./packages/peertube-runner/dist +rm -rf ./apps/peertube-runner/dist -cd ./packages/peertube-runner +cd ./apps/peertube-runner ../../node_modules/.bin/concurrently -k \ "../../node_modules/.bin/tsc -w --noEmit" \ - "./node_modules/.bin/esbuild ./peertube-runner.ts --bundle --sourcemap --platform=node --external:"./lib-cov/fluent-ffmpeg" --external:pg-hstore --watch --outfile=dist/peertube-runner.js" + "node ./scripts/watch.js" diff --git a/scripts/dev/server.sh b/scripts/dev/server.sh index c52c5124c..4112cb2f8 100755 --- a/scripts/dev/server.sh +++ b/scripts/dev/server.sh @@ -16,10 +16,10 @@ cp -r "./client/src/locale" "./client/dist/locale" mkdir -p "./dist/server/lib" -npm run tsc -- -b -v --incremental +npm run tsc -- -b -v --incremental server/tsconfig.json npm run resolve-tspaths:server -cp -r ./server/static ./server/assets ./dist/server -cp -r "./server/lib/emails" "./dist/server/lib" +cp -r ./server/server/static ./server/server/assets ./dist/server +cp -r "./server/server/lib/emails" "./dist/server/lib" -./node_modules/.bin/tsc-watch --build --preserveWatchOutput --verbose --onSuccess 'sh -c "npm run resolve-tspaths:server && NODE_ENV=dev node dist/server"' +./node_modules/.bin/tsc-watch --build --preserveWatchOutput --verbose --onSuccess 'sh -c "npm run resolve-tspaths:server && NODE_ENV=dev node dist/server"' server/tsconfig.json diff --git a/scripts/generate-code-contributors.ts b/scripts/generate-code-contributors.ts index 2fd0ecdf3..408bbec5d 100755 --- a/scripts/generate-code-contributors.ts +++ b/scripts/generate-code-contributors.ts @@ -1,4 +1,4 @@ -import { CLICommand } from '@shared/server-commands' +import { CLICommand } from '@peertube/peertube-server-commands' run() .then(() => process.exit(0)) diff --git a/scripts/i18n/create-custom-files.ts b/scripts/i18n/create-custom-files.ts index 696a097b1..887ab86af 100755 --- a/scripts/i18n/create-custom-files.ts +++ b/scripts/i18n/create-custom-files.ts @@ -1,6 +1,7 @@ -import { writeJSON } from 'fs-extra' +import { readJsonSync, writeJSON } from 'fs-extra/esm' import { join } from 'path' -import { root, USER_ROLE_LABELS } from '@shared/core-utils' +import { I18N_LOCALES, USER_ROLE_LABELS } from '@peertube/peertube-core-utils' +import { root } from '@peertube/peertube-node-utils' import { ABUSE_STATES, buildLanguages, @@ -14,10 +15,9 @@ import { VIDEO_PLAYLIST_TYPES, VIDEO_PRIVACIES, VIDEO_STATES -} from '../../server/initializers/constants' -import { I18N_LOCALES } from '../../shared/core-utils/i18n' +} from '../../server/initializers/constants.js' -const videojs = require(join(root(), 'client', 'src', 'locale', 'videojs.en-US.json')) +const videojs = readJsonSync(join(root(), 'client', 'src', 'locale', 'videojs.en-US.json')) const playerKeys = { 'Quality': 'Quality', 'Auto': 'Auto', @@ -131,13 +131,13 @@ async function writeAll () { for (const key of Object.keys(I18N_LOCALES)) { const playerJsonPath = join(localePath, `player.${key}.json`) - const translatedPlayer = require(playerJsonPath) + const translatedPlayer = readJsonSync(playerJsonPath) const newTranslatedPlayer = Object.assign({}, playerKeys, translatedPlayer) await writeJSON(playerJsonPath, newTranslatedPlayer, { spaces: 4 }) const serverJsonPath = join(localePath, `server.${key}.json`) - const translatedServer = require(serverJsonPath) + const translatedServer = readJsonSync(serverJsonPath) const newTranslatedServer = Object.assign({}, serverKeys, translatedServer) await writeJSON(serverJsonPath, newTranslatedServer, { spaces: 4 }) diff --git a/scripts/migrations/peertube-4.0.ts b/scripts/migrations/peertube-4.0.ts deleted file mode 100644 index b0891c2e6..000000000 --- a/scripts/migrations/peertube-4.0.ts +++ /dev/null @@ -1,104 +0,0 @@ -import Bluebird from 'bluebird' -import { move, readFile, writeFile } from 'fs-extra' -import { join } from 'path' -import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' -import { JobQueue } from '@server/lib/job-queue' -import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from '@server/lib/paths' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { VideoModel } from '@server/models/video/video' -import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { initDatabaseModels } from '../../server/initializers/database' - -run() - .then(() => process.exit(0)) - .catch(err => { - console.error(err) - process.exit(-1) - }) - -async function run () { - console.log('Migrate old HLS paths to new format.') - - await initDatabaseModels(true) - - JobQueue.Instance.init() - - const ids = await VideoModel.listLocalIds() - - await Bluebird.map(ids, async id => { - try { - await processVideo(id) - } catch (err) { - console.error('Cannot process video %s.', { err }) - } - }, { concurrency: 5 }) - - console.log('Migration finished!') -} - -async function processVideo (videoId: number) { - const video = await VideoModel.loadWithFiles(videoId) - - const hls = video.getHLSPlaylist() - if (video.isLive || !hls || hls.playlistFilename !== 'master.m3u8' || hls.VideoFiles.length === 0) { - return - } - - console.log(`Renaming HLS playlist files of video ${video.name}.`) - - const playlist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id) - const hlsDirPath = VideoPathManager.Instance.getFSHLSOutputPath(video) - - const masterPlaylistPath = join(hlsDirPath, playlist.playlistFilename) - let masterPlaylistContent = await readFile(masterPlaylistPath, 'utf8') - - for (const videoFile of hls.VideoFiles) { - const srcName = `${videoFile.resolution}.m3u8` - const dstName = getHlsResolutionPlaylistFilename(videoFile.filename) - - const src = join(hlsDirPath, srcName) - const dst = join(hlsDirPath, dstName) - - try { - await move(src, dst) - - masterPlaylistContent = masterPlaylistContent.replace(new RegExp('^' + srcName + '$', 'm'), dstName) - } catch (err) { - console.error('Cannot move video file %s to %s.', src, dst, err) - } - } - - await writeFile(masterPlaylistPath, masterPlaylistContent) - - if (playlist.segmentsSha256Filename === 'segments-sha256.json') { - try { - const newName = generateHlsSha256SegmentsFilename(video.isLive) - - const dst = join(hlsDirPath, newName) - await move(join(hlsDirPath, playlist.segmentsSha256Filename), dst) - playlist.segmentsSha256Filename = newName - } catch (err) { - console.error(`Cannot rename ${video.name} segments-sha256.json file to a new name`, err) - } - } - - if (playlist.playlistFilename === 'master.m3u8') { - try { - const newName = generateHLSMasterPlaylistFilename(video.isLive) - - const dst = join(hlsDirPath, newName) - await move(join(hlsDirPath, playlist.playlistFilename), dst) - playlist.playlistFilename = newName - } catch (err) { - console.error(`Cannot rename ${video.name} master.m3u8 file to a new name`, err) - } - } - - // Everything worked, we can save the playlist now - await playlist.save() - - const allVideo = await VideoModel.loadFull(video.id) - await federateVideoIfNeeded(allVideo, false) - - console.log(`Successfully moved HLS files of ${video.name}.`) -} diff --git a/scripts/migrations/peertube-4.2.ts b/scripts/migrations/peertube-4.2.ts deleted file mode 100644 index d8929692b..000000000 --- a/scripts/migrations/peertube-4.2.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { minBy } from 'lodash' -import { join } from 'path' -import { getImageSize, processImage } from '@server/helpers/image-utils' -import { CONFIG } from '@server/initializers/config' -import { ACTOR_IMAGES_SIZE } from '@server/initializers/constants' -import { updateActorImages } from '@server/lib/activitypub/actors' -import { sendUpdateActor } from '@server/lib/activitypub/send' -import { getBiggestActorImage } from '@server/lib/actor-image' -import { JobQueue } from '@server/lib/job-queue' -import { AccountModel } from '@server/models/account/account' -import { ActorModel } from '@server/models/actor/actor' -import { VideoChannelModel } from '@server/models/video/video-channel' -import { MAccountDefault, MActorDefault, MChannelDefault } from '@server/types/models' -import { getLowercaseExtension } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { ActorImageType } from '@shared/models' -import { initDatabaseModels } from '../../server/initializers/database' - -run() - .then(() => process.exit(0)) - .catch(err => { - console.error(err) - process.exit(-1) - }) - -async function run () { - console.log('Generate avatar miniatures from existing avatars.') - - await initDatabaseModels(true) - JobQueue.Instance.init() - - const accounts: AccountModel[] = await AccountModel.findAll({ - include: [ - { - model: ActorModel, - required: true, - where: { - serverId: null - } - }, - { - model: VideoChannelModel, - include: [ - { - model: AccountModel - } - ] - } - ] - }) - - for (const account of accounts) { - try { - await fillAvatarSizeIfNeeded(account) - await generateSmallerAvatarIfNeeded(account) - } catch (err) { - console.error(`Cannot process account avatar ${account.name}`, err) - } - - for (const videoChannel of account.VideoChannels) { - try { - await fillAvatarSizeIfNeeded(videoChannel) - await generateSmallerAvatarIfNeeded(videoChannel) - } catch (err) { - console.error(`Cannot process channel avatar ${videoChannel.name}`, err) - } - } - } - - console.log('Generation finished!') -} - -async function fillAvatarSizeIfNeeded (accountOrChannel: MAccountDefault | MChannelDefault) { - const avatars = accountOrChannel.Actor.Avatars - - for (const avatar of avatars) { - if (avatar.width && avatar.height) continue - - console.log('Filling size of avatars of %s.', accountOrChannel.name) - - const { width, height } = await getImageSize(join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, avatar.filename)) - avatar.width = width - avatar.height = height - - await avatar.save() - } -} - -async function generateSmallerAvatarIfNeeded (accountOrChannel: MAccountDefault | MChannelDefault) { - const avatars = accountOrChannel.Actor.Avatars - if (avatars.length !== 1) { - return - } - - console.log(`Processing ${accountOrChannel.name}.`) - - await generateSmallerAvatar(accountOrChannel.Actor) - accountOrChannel.Actor = Object.assign(accountOrChannel.Actor, { Server: null }) - - return sendUpdateActor(accountOrChannel, undefined) -} - -async function generateSmallerAvatar (actor: MActorDefault) { - const bigAvatar = getBiggestActorImage(actor.Avatars) - - const imageSize = minBy(ACTOR_IMAGES_SIZE[ActorImageType.AVATAR], 'width') - const sourceFilename = bigAvatar.filename - - const newImageName = buildUUID() + getLowercaseExtension(sourceFilename) - const source = join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, sourceFilename) - const destination = join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, newImageName) - - await processImage({ path: source, destination, newSize: imageSize, keepOriginal: true }) - - const actorImageInfo = { - name: newImageName, - fileUrl: null, - height: imageSize.height, - width: imageSize.width, - onDisk: true - } - - await updateActorImages(actor, ActorImageType.AVATAR, [ actorImageInfo ], undefined) -} diff --git a/scripts/migrations/peertube-5.0.ts b/scripts/migrations/peertube-5.0.ts deleted file mode 100644 index a0f51a64c..000000000 --- a/scripts/migrations/peertube-5.0.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { ensureDir } from 'fs-extra' -import { Op } from 'sequelize' -import { updateTorrentMetadata } from '@server/helpers/webtorrent' -import { DIRECTORIES } from '@server/initializers/constants' -import { moveFilesIfPrivacyChanged } from '@server/lib/video-privacy' -import { VideoModel } from '@server/models/video/video' -import { MVideoFullLight } from '@server/types/models' -import { VideoPrivacy } from '@shared/models' -import { initDatabaseModels } from '../../server/initializers/database' - -run() - .then(() => process.exit(0)) - .catch(err => { - console.error(err) - process.exit(-1) - }) - -async function run () { - console.log('Moving private video files in dedicated folders.') - - await ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE) - await ensureDir(DIRECTORIES.VIDEOS.PRIVATE) - - await initDatabaseModels(true) - - const videos = await VideoModel.unscoped().findAll({ - attributes: [ 'uuid' ], - where: { - privacy: { - [Op.in]: [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ] - } - } - }) - - for (const { uuid } of videos) { - try { - console.log('Moving files of video %s.', uuid) - - const video = await VideoModel.loadFull(uuid) - - try { - await moveFilesIfPrivacyChanged(video, VideoPrivacy.PUBLIC) - } catch (err) { - console.error('Cannot move files of video %s.', uuid, err) - } - - try { - await updateTorrents(video) - } catch (err) { - console.error('Cannot regenerate torrents of video %s.', uuid, err) - } - } catch (err) { - console.error('Cannot process video %s.', uuid, err) - } - } -} - -async function updateTorrents (video: MVideoFullLight) { - for (const file of video.VideoFiles) { - await updateTorrentMetadata(video, file) - - await file.save() - } - - const playlist = video.getHLSPlaylist() - for (const file of (playlist?.VideoFiles || [])) { - await updateTorrentMetadata(playlist, file) - - await file.save() - } -} diff --git a/scripts/nightly.sh b/scripts/nightly.sh index 572277f9d..e911c549d 100755 --- a/scripts/nightly.sh +++ b/scripts/nightly.sh @@ -20,6 +20,13 @@ tar_name="peertube-nightly-$today.tar.xz" npm run build -- --source-map +# Clean up declaration files +find dist/ packages/core-utils/dist/ \ + packages/ffmpeg/dist/ \ + packages/node-utils/dist/ \ + packages/models/dist/ \ + \( -name '*.d.ts' -o -name '*.d.ts.map' \) -type f -delete + nightly_version="nightly-$today" sed -i 's/"version": "\([^"]\+\)"/"version": "\1-'"$nightly_version"'"/' ./package.json @@ -28,6 +35,10 @@ sed -i 's/"version": "\([^"]\+\)"/"version": "\1-'"$nightly_version"'"/' ./packa # local variables directories_to_archive=("$directory_name/CREDITS.md" "$directory_name/FAQ.md" \ "$directory_name/LICENSE" "$directory_name/README.md" \ + "$directory_name/packages/core-utils/dist/" "$directory_name/packages/core-utils/package.json" \ + "$directory_name/packages/ffmpeg/dist/" "$directory_name/packages/ffmpeg/package.json" \ + "$directory_name/packages/node-utils/dist/" "$directory_name/packages/node-utils/package.json" \ + "$directory_name/packages/models/dist/" "$directory_name/packages/models/package.json" \ "$directory_name/client/dist/" "$directory_name/client/yarn.lock" \ "$directory_name/client/package.json" "$directory_name/config" \ "$directory_name/dist" "$directory_name/package.json" \ diff --git a/scripts/parse-log.ts b/scripts/parse-log.ts deleted file mode 100755 index 6770f090b..000000000 --- a/scripts/parse-log.ts +++ /dev/null @@ -1,160 +0,0 @@ -import { program } from 'commander' -import { createReadStream, readdir } from 'fs-extra' -import { join } from 'path' -import { stdin } from 'process' -import { createInterface } from 'readline' -import { format as sqlFormat } from 'sql-formatter' -import { inspect } from 'util' -import * as winston from 'winston' -import { labelFormatter, mtimeSortFilesDesc } from '../server/helpers/logger' -import { CONFIG } from '../server/initializers/config' - -program - .option('-l, --level [level]', 'Level log (debug/info/warn/error)') - .option('-f, --files [file...]', 'Files to parse. If not provided, the script will parse the latest log file from config)') - .option('-t, --tags [tags...]', 'Display only lines with these tags') - .option('-nt, --not-tags [tags...]', 'Donrt display lines containing these tags') - .parse(process.argv) - -const options = program.opts() - -const excludedKeys = { - level: true, - message: true, - splat: true, - timestamp: true, - tags: true, - label: true, - sql: true -} -function keysExcluder (key, value) { - return excludedKeys[key] === true ? undefined : value -} - -const loggerFormat = winston.format.printf((info) => { - let additionalInfos = JSON.stringify(info, keysExcluder, 2) - if (additionalInfos === '{}') additionalInfos = '' - else additionalInfos = ' ' + additionalInfos - - if (info.sql) { - if (CONFIG.LOG.PRETTIFY_SQL) { - additionalInfos += '\n' + sqlFormat(info.sql, { - language: 'sql', - tabWidth: 2 - }) - } else { - additionalInfos += ' - ' + info.sql - } - } - - return `[${info.label}] ${toTimeFormat(info.timestamp)} ${info.level}: ${info.message}${additionalInfos}` -}) - -const logger = winston.createLogger({ - transports: [ - new winston.transports.Console({ - level: options.level || 'debug', - stderrLevels: [], - format: winston.format.combine( - winston.format.splat(), - labelFormatter(), - winston.format.colorize(), - loggerFormat - ) - }) - ], - exitOnError: true -}) - -const logLevels = { - error: logger.error.bind(logger), - warn: logger.warn.bind(logger), - info: logger.info.bind(logger), - debug: logger.debug.bind(logger) -} - -run() - .then(() => process.exit(0)) - .catch(err => console.error(err)) - -async function run () { - const files = await getFiles() - - for (const file of files) { - if (file === 'peertube-audit.log') continue - - await readFile(file) - } -} - -function readFile (file: string) { - console.log('Opening %s.', file) - - const stream = file === '-' ? stdin : createReadStream(file) - - const rl = createInterface({ - input: stream - }) - - return new Promise(res => { - rl.on('line', line => { - try { - const log = JSON.parse(line) - if (options.tags && !containsTags(log.tags, options.tags)) { - return - } - - if (options.notTags && containsTags(log.tags, options.notTags)) { - return - } - - // Don't know why but loggerFormat does not remove splat key - Object.assign(log, { splat: undefined }) - - logLevels[log.level](log) - } catch (err) { - console.error('Cannot parse line.', inspect(line)) - throw err - } - }) - - stream.once('end', () => res()) - }) -} - -// Thanks: https://stackoverflow.com/a/37014317 -async function getNewestFile (files: string[], basePath: string) { - const sorted = await mtimeSortFilesDesc(files, basePath) - - return (sorted.length > 0) ? sorted[0].file : '' -} - -async function getFiles () { - if (options.files) return options.files - - const logFiles = await readdir(CONFIG.STORAGE.LOG_DIR) - - const filename = await getNewestFile(logFiles, CONFIG.STORAGE.LOG_DIR) - return [ join(CONFIG.STORAGE.LOG_DIR, filename) ] -} - -function toTimeFormat (time: string) { - const timestamp = Date.parse(time) - - if (isNaN(timestamp) === true) return 'Unknown date' - - const d = new Date(timestamp) - return d.toLocaleString() + `.${d.getMilliseconds()}` -} - -function containsTags (loggerTags: string[], optionsTags: string[]) { - if (!loggerTags) return false - - for (const lt of loggerTags) { - for (const ot of optionsTags) { - if (lt === ot) return true - } - } - - return false -} diff --git a/scripts/plugin/install.ts b/scripts/plugin/install.ts deleted file mode 100755 index 138f34446..000000000 --- a/scripts/plugin/install.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { program } from 'commander' -import { isAbsolute } from 'path' -import { initDatabaseModels } from '../../server/initializers/database' -import { PluginManager } from '../../server/lib/plugins/plugin-manager' - -program - .option('-n, --npm-name [npmName]', 'Plugin to install') - .option('-v, --plugin-version [pluginVersion]', 'Plugin version to install') - .option('-p, --plugin-path [pluginPath]', 'Path of the plugin you want to install') - .parse(process.argv) - -const options = program.opts() - -if (!options.npmName && !options.pluginPath) { - console.error('You need to specify a plugin name with the desired version, or a plugin path.') - process.exit(-1) -} - -if (options.pluginPath && !isAbsolute(options.pluginPath)) { - console.error('Plugin path should be absolute.') - process.exit(-1) -} - -run() - .then(() => process.exit(0)) - .catch(err => { - console.error(err) - process.exit(-1) - }) - -async function run () { - await initDatabaseModels(true) - - const toInstall = options.npmName || options.pluginPath - await PluginManager.Instance.install({ - toInstall, - version: options.pluginVersion, - fromDisk: !!options.pluginPath, - register: false - }) -} diff --git a/scripts/plugin/uninstall.ts b/scripts/plugin/uninstall.ts deleted file mode 100755 index 770594685..000000000 --- a/scripts/plugin/uninstall.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { program } from 'commander' -import { initDatabaseModels } from '../../server/initializers/database' -import { PluginManager } from '../../server/lib/plugins/plugin-manager' - -program - .option('-n, --npm-name [npmName]', 'Package name to install') - .parse(process.argv) - -const options = program.opts() - -if (!options.npmName) { - console.error('You need to specify the plugin name.') - process.exit(-1) -} - -run() - .then(() => process.exit(0)) - .catch(err => { - console.error(err) - process.exit(-1) - }) - -async function run () { - - await initDatabaseModels(true) - - const toUninstall = options.npmName - await PluginManager.Instance.uninstall({ npmName: toUninstall, unregister: false }) -} diff --git a/scripts/prune-storage.ts b/scripts/prune-storage.ts deleted file mode 100755 index 9a73a8600..000000000 --- a/scripts/prune-storage.ts +++ /dev/null @@ -1,184 +0,0 @@ -import { map } from 'bluebird' -import { readdir, remove, stat } from 'fs-extra' -import { basename, join } from 'path' -import { get, start } from 'prompt' -import { DIRECTORIES } from '@server/initializers/constants' -import { VideoFileModel } from '@server/models/video/video-file' -import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { uniqify } from '@shared/core-utils' -import { ThumbnailType } from '@shared/models' -import { getUUIDFromFilename } from '../server/helpers/utils' -import { CONFIG } from '../server/initializers/config' -import { initDatabaseModels } from '../server/initializers/database' -import { ActorImageModel } from '../server/models/actor/actor-image' -import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy' -import { ThumbnailModel } from '../server/models/video/thumbnail' -import { VideoModel } from '../server/models/video/video' - -run() - .then(() => process.exit(0)) - .catch(err => { - console.error(err) - process.exit(-1) - }) - -async function run () { - const dirs = Object.values(CONFIG.STORAGE) - - if (uniqify(dirs).length !== dirs.length) { - console.error('Cannot prune storage because you put multiple storage keys in the same directory.') - process.exit(0) - } - - await initDatabaseModels(true) - - let toDelete: string[] = [] - - console.log('Detecting files to remove, it could take a while...') - - toDelete = toDelete.concat( - await pruneDirectory(DIRECTORIES.VIDEOS.PUBLIC, doesWebVideoFileExist()), - await pruneDirectory(DIRECTORIES.VIDEOS.PRIVATE, doesWebVideoFileExist()), - - await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, doesHLSPlaylistExist()), - await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, doesHLSPlaylistExist()), - - await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesTorrentFileExist()), - - await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist), - - await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true, ThumbnailType.PREVIEW)), - await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false, ThumbnailType.MINIATURE)), - - await pruneDirectory(CONFIG.STORAGE.ACTOR_IMAGES_DIR, doesActorImageExist) - ) - - const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR) - toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t))) - - if (toDelete.length === 0) { - console.log('No files to delete.') - return - } - - console.log('Will delete %d files:\n\n%s\n\n', toDelete.length, toDelete.join('\n')) - - const res = await askConfirmation() - if (res === true) { - console.log('Processing delete...\n') - - for (const path of toDelete) { - await remove(path) - } - - console.log('Done!') - } else { - console.log('Exiting without deleting files.') - } -} - -type ExistFun = (file: string) => Promise | boolean -async function pruneDirectory (directory: string, existFun: ExistFun) { - const files = await readdir(directory) - - const toDelete: string[] = [] - await map(files, async file => { - const filePath = join(directory, file) - - if (await existFun(filePath) !== true) { - toDelete.push(filePath) - } - }, { concurrency: 20 }) - - return toDelete -} - -function doesWebVideoFileExist () { - return (filePath: string) => { - // Don't delete private directory - if (filePath === DIRECTORIES.VIDEOS.PRIVATE) return true - - return VideoFileModel.doesOwnedWebVideoFileExist(basename(filePath)) - } -} - -function doesHLSPlaylistExist () { - return (hlsPath: string) => { - // Don't delete private directory - if (hlsPath === DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE) return true - - return VideoStreamingPlaylistModel.doesOwnedHLSPlaylistExist(basename(hlsPath)) - } -} - -function doesTorrentFileExist () { - return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath)) -} - -function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType) { - return async (filePath: string) => { - const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type) - if (!thumbnail) return false - - if (keepOnlyOwned) { - const video = await VideoModel.load(thumbnail.videoId) - if (video.isOwned() === false) return false - } - - return true - } -} - -async function doesActorImageExist (filePath: string) { - const image = await ActorImageModel.loadByName(basename(filePath)) - - return !!image -} - -async function doesRedundancyExist (filePath: string) { - const isPlaylist = (await stat(filePath)).isDirectory() - - if (isPlaylist) { - // Don't delete HLS redundancy directory - if (filePath === DIRECTORIES.HLS_REDUNDANCY) return true - - const uuid = getUUIDFromFilename(filePath) - const video = await VideoModel.loadWithFiles(uuid) - if (!video) return false - - const p = video.getHLSPlaylist() - if (!p) return false - - const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id) - return !!redundancy - } - - const file = await VideoFileModel.loadByFilename(basename(filePath)) - if (!file) return false - - const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) - return !!redundancy -} - -async function askConfirmation () { - return new Promise((res, rej) => { - start() - const schema = { - properties: { - confirm: { - type: 'string', - description: 'These following unused files can be deleted, but please check your backups first (bugs happen).' + - ' Notice PeerTube must have been stopped when your ran this script.' + - ' Can we delete these files?', - default: 'n', - required: true - } - } - } - get(schema, function (err, result) { - if (err) return rej(err) - - return res(result.confirm?.match(/y/) !== null) - }) - }) -} diff --git a/scripts/regenerate-thumbnails.ts b/scripts/regenerate-thumbnails.ts deleted file mode 100644 index 061819387..000000000 --- a/scripts/regenerate-thumbnails.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { map } from 'bluebird' -import { program } from 'commander' -import { pathExists, remove } from 'fs-extra' -import { generateImageFilename, processImage } from '@server/helpers/image-utils' -import { THUMBNAILS_SIZE } from '@server/initializers/constants' -import { initDatabaseModels } from '@server/initializers/database' -import { VideoModel } from '@server/models/video/video' - -program - .description('Regenerate local thumbnails using preview files') - .parse(process.argv) - -run() - .then(() => process.exit(0)) - .catch(err => console.error(err)) - -async function run () { - await initDatabaseModels(true) - - const ids = await VideoModel.listLocalIds() - - await map(ids, id => { - return processVideo(id) - .catch(err => console.error('Cannot process video %d.', id, err)) - }, { concurrency: 20 }) -} - -async function processVideo (id: number) { - const video = await VideoModel.loadWithFiles(id) - - console.log('Processing video %s.', video.name) - - const thumbnail = video.getMiniature() - const preview = video.getPreview() - - const previewPath = preview.getPath() - - if (!await pathExists(previewPath)) { - throw new Error(`Preview ${previewPath} does not exist on disk`) - } - - const size = { - width: THUMBNAILS_SIZE.width, - height: THUMBNAILS_SIZE.height - } - - const oldPath = thumbnail.getPath() - - // Update thumbnail - thumbnail.filename = generateImageFilename() - thumbnail.width = size.width - thumbnail.height = size.height - - const thumbnailPath = thumbnail.getPath() - await processImage({ path: previewPath, destination: thumbnailPath, newSize: size, keepOriginal: true }) - - // Save new attributes - await thumbnail.save() - - // Remove old thumbnail - await remove(oldPath) - - // Don't federate, remote instances will refresh the thumbnails after a while -} diff --git a/scripts/release.sh b/scripts/release.sh index 2b922a749..0df9efa1d 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -69,11 +69,22 @@ npm run build -- --source-map rm -f "./client/dist/en-US/stats.json" rm -f "./client/dist/embed-stats.json" +# Clean up declaration files +find dist/ packages/core-utils/dist/ \ + packages/ffmpeg/dist/ \ + packages/node-utils/dist/ \ + packages/models/dist/ \ + \( -name '*.d.ts' -o -name '*.d.ts.map' \) -type f -delete + # Creating the archives ( # local variables directories_to_archive=("$directory_name/CREDITS.md" "$directory_name/FAQ.md" \ "$directory_name/LICENSE" "$directory_name/README.md" \ + "$directory_name/packages/core-utils/dist/" "$directory_name/packages/core-utils/package.json" \ + "$directory_name/packages/ffmpeg/dist/" "$directory_name/packages/ffmpeg/package.json" \ + "$directory_name/packages/node-utils/dist/" "$directory_name/packages/node-utils/package.json" \ + "$directory_name/packages/models/dist/" "$directory_name/packages/models/package.json" \ "$directory_name/client/dist/" "$directory_name/client/yarn.lock" \ "$directory_name/client/package.json" "$directory_name/config" \ "$directory_name/dist" "$directory_name/package.json" \ @@ -124,7 +135,7 @@ rm -f "./client/dist/embed-stats.json" # Release types package npm run generate-types-package "$version" - cd packages/types/dist + cd packages/types-generator/dist npm publish --access public fi ) diff --git a/scripts/reset-password.ts b/scripts/reset-password.ts deleted file mode 100755 index b2e5639fb..000000000 --- a/scripts/reset-password.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { program } from 'commander' -import { isUserPasswordValid } from '../server/helpers/custom-validators/users' -import { initDatabaseModels } from '../server/initializers/database' -import { UserModel } from '../server/models/user/user' - -program - .option('-u, --user [user]', 'User') - .parse(process.argv) - -const options = program.opts() - -if (options.user === undefined) { - console.error('All parameters are mandatory.') - process.exit(-1) -} - -initDatabaseModels(true) - .then(() => { - return UserModel.loadByUsername(options.user) - }) - .then(user => { - if (!user) { - console.error('Unknown user.') - process.exit(-1) - } - - const readline = require('readline') - const Writable = require('stream').Writable - const mutableStdout = new Writable({ - write: function (_chunk, _encoding, callback) { - callback() - } - }) - const rl = readline.createInterface({ - input: process.stdin, - output: mutableStdout, - terminal: true - }) - - console.log('New password?') - rl.on('line', function (password) { - if (!isUserPasswordValid(password)) { - console.error('New password is invalid.') - process.exit(-1) - } - - user.password = password - - user.save() - .then(() => console.log('User password updated.')) - .catch(err => console.error(err)) - .finally(() => process.exit(0)) - }) - }) - .catch(err => { - console.error(err) - process.exit(-1) - }) diff --git a/scripts/setup/cli.sh b/scripts/setup/cli.sh deleted file mode 100755 index 2e9b8a505..000000000 --- a/scripts/setup/cli.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh - -set -eu - -NOCLIENT=1 yarn install --pure-lockfile - -rm -rf ./dist/server/tools/ - -( - cd ./server/tools - yarn install --pure-lockfile - ../../node_modules/.bin/tsc --build --verbose -) - -cp -r "./server/tools/node_modules" "./dist/server/tools" - -npm run resolve-tspaths:server diff --git a/scripts/simulate-many-viewers.ts b/scripts/simulate-many-viewers.ts index a993e175a..d6d9fd69e 100644 --- a/scripts/simulate-many-viewers.ts +++ b/scripts/simulate-many-viewers.ts @@ -1,5 +1,5 @@ import Bluebird from 'bluebird' -import { wait } from '@shared/core-utils' +import { wait } from '@peertube/peertube-core-utils' import { createSingleServer, doubleFollow, @@ -7,7 +7,7 @@ import { PeerTubeServer, setAccessTokensToServers, waitJobs -} from '@shared/server-commands' +} from '@peertube/peertube-server-commands' let servers: PeerTubeServer[] const viewers: { xForwardedFor: string }[] = [] diff --git a/scripts/tsconfig.json b/scripts/tsconfig.json index 0cfd927a6..0a7e07599 100644 --- a/scripts/tsconfig.json +++ b/scripts/tsconfig.json @@ -4,7 +4,10 @@ "outDir": "../dist/scripts" }, "references": [ - { "path": "../shared" }, + { "path": "../packages/core-utils" }, + { "path": "../packages/models" }, + { "path": "../packages/node-utils" }, + { "path": "../packages/server-commands" }, { "path": "../server" } ] } diff --git a/scripts/update-host.ts b/scripts/update-host.ts deleted file mode 100755 index 1d17ce152..000000000 --- a/scripts/update-host.ts +++ /dev/null @@ -1,140 +0,0 @@ -import { updateTorrentMetadata } from '@server/helpers/webtorrent' -import { getServerActor } from '@server/models/application/application' -import { WEBSERVER } from '../server/initializers/constants' -import { initDatabaseModels } from '../server/initializers/database' -import { - getLocalAccountActivityPubUrl, - getLocalVideoActivityPubUrl, - getLocalVideoAnnounceActivityPubUrl, - getLocalVideoChannelActivityPubUrl, - getLocalVideoCommentActivityPubUrl -} from '../server/lib/activitypub/url' -import { AccountModel } from '../server/models/account/account' -import { ActorModel } from '../server/models/actor/actor' -import { ActorFollowModel } from '../server/models/actor/actor-follow' -import { VideoModel } from '../server/models/video/video' -import { VideoChannelModel } from '../server/models/video/video-channel' -import { VideoCommentModel } from '../server/models/video/video-comment' -import { VideoShareModel } from '../server/models/video/video-share' - -run() - .then(() => process.exit(0)) - .catch(err => { - console.error(err) - process.exit(-1) - }) - -async function run () { - await initDatabaseModels(true) - - const serverAccount = await getServerActor() - - { - const res = await ActorFollowModel.listAcceptedFollowingUrlsForApi([ serverAccount.id ], undefined) - const hasFollowing = res.total > 0 - - if (hasFollowing === true) { - throw new Error('Cannot update host because you follow other servers!') - } - } - - console.log('Updating actors.') - - const actors: ActorModel[] = await ActorModel.unscoped().findAll({ - include: [ - { - model: VideoChannelModel.unscoped(), - required: false - }, - { - model: AccountModel.unscoped(), - required: false - } - ] - }) - for (const actor of actors) { - if (actor.isOwned() === false) continue - - console.log('Updating actor ' + actor.url) - - const newUrl = actor.Account - ? getLocalAccountActivityPubUrl(actor.preferredUsername) - : getLocalVideoChannelActivityPubUrl(actor.preferredUsername) - - actor.url = newUrl - actor.inboxUrl = newUrl + '/inbox' - actor.outboxUrl = newUrl + '/outbox' - actor.sharedInboxUrl = WEBSERVER.URL + '/inbox' - actor.followersUrl = newUrl + '/followers' - actor.followingUrl = newUrl + '/following' - - await actor.save() - } - - console.log('Updating video shares.') - - const videoShares: VideoShareModel[] = await VideoShareModel.findAll({ - include: [ VideoModel.unscoped(), ActorModel.unscoped() ] - }) - for (const videoShare of videoShares) { - if (videoShare.Video.isOwned() === false) continue - - console.log('Updating video share ' + videoShare.url) - - videoShare.url = getLocalVideoAnnounceActivityPubUrl(videoShare.Actor, videoShare.Video) - await videoShare.save() - } - - console.log('Updating video comments.') - const videoComments: VideoCommentModel[] = await VideoCommentModel.findAll({ - include: [ - { - model: VideoModel.unscoped() - }, - { - model: AccountModel.unscoped(), - include: [ - { - model: ActorModel.unscoped() - } - ] - } - ] - }) - for (const comment of videoComments) { - if (comment.isOwned() === false) continue - - console.log('Updating comment ' + comment.url) - - comment.url = getLocalVideoCommentActivityPubUrl(comment.Video, comment) - await comment.save() - } - - console.log('Updating video and torrent files.') - - const ids = await VideoModel.listLocalIds() - for (const id of ids) { - const video = await VideoModel.loadFull(id) - - console.log('Updating video ' + video.uuid) - - video.url = getLocalVideoActivityPubUrl(video) - await video.save() - - for (const file of video.VideoFiles) { - console.log('Updating torrent file %s of video %s.', file.resolution, video.uuid) - await updateTorrentMetadata(video, file) - - await file.save() - } - - const playlist = video.getHLSPlaylist() - for (const file of (playlist?.VideoFiles || [])) { - console.log('Updating fragmented torrent file %s of video %s.', file.resolution, video.uuid) - - await updateTorrentMetadata(playlist, file) - - await file.save() - } - } -} diff --git a/server.ts b/server.ts deleted file mode 100644 index 605b94be0..000000000 --- a/server.ts +++ /dev/null @@ -1,376 +0,0 @@ -// ----------- Node modules ----------- -import { registerOpentelemetryTracing } from './server/lib/opentelemetry/tracing' -registerOpentelemetryTracing() - -import express from 'express' -import morgan, { token } from 'morgan' -import cors from 'cors' -import cookieParser from 'cookie-parser' -import { frameguard } from 'helmet' -import { parse } from 'useragent' -import anonymize from 'ip-anonymize' -import { program as cli } from 'commander' - -process.title = 'peertube' - -// Create our main app -const app = express().disable('x-powered-by') - -// ----------- Core checker ----------- -import { checkMissedConfig, checkFFmpeg, checkNodeVersion } from './server/initializers/checker-before-init' - -// Do not use barrels because we don't want to load all modules here (we need to initialize database first) -import { CONFIG } from './server/initializers/config' -import { API_VERSION, WEBSERVER, loadLanguages } from './server/initializers/constants' -import { logger } from './server/helpers/logger' - -const missed = checkMissedConfig() -if (missed.length !== 0) { - logger.error('Your configuration files miss keys: ' + missed) - process.exit(-1) -} - -checkFFmpeg(CONFIG) - .catch(err => { - logger.error('Error in ffmpeg check.', { err }) - process.exit(-1) - }) - -try { - checkNodeVersion() -} catch (err) { - logger.error('Error in NodeJS check.', { err }) - process.exit(-1) -} - -import { checkConfig, checkActivityPubUrls, checkFFmpegVersion } from './server/initializers/checker-after-init' - -try { - checkConfig() -} catch (err) { - logger.error('Config error.', { err }) - process.exit(-1) -} - -// Trust our proxy (IP forwarding...) -app.set('trust proxy', CONFIG.TRUST_PROXY) - -app.use((_req, res, next) => { - // OpenTelemetry - res.locals.requestStart = Date.now() - - if (CONFIG.SECURITY.POWERED_BY_HEADER.ENABLED === true) { - res.setHeader('x-powered-by', 'PeerTube') - } - - return next() -}) - -// Security middleware -import { baseCSP } from './server/middlewares/csp' - -if (CONFIG.CSP.ENABLED) { - app.use(baseCSP) -} - -if (CONFIG.SECURITY.FRAMEGUARD.ENABLED) { - app.use(frameguard({ - action: 'deny' // we only allow it for /videos/embed, see server/controllers/client.ts - })) -} - -// ----------- Database ----------- - -// Initialize database and models -import { initDatabaseModels, checkDatabaseConnectionOrDie } from './server/initializers/database' -checkDatabaseConnectionOrDie() - -import { migrate } from './server/initializers/migrator' -migrate() - .then(() => initDatabaseModels(false)) - .then(() => startApplication()) - .catch(err => { - logger.error('Cannot start application.', { err }) - process.exit(-1) - }) - -// ----------- Initialize ----------- -loadLanguages() - -// ----------- PeerTube modules ----------- -import { installApplication } from './server/initializers/installer' -import { Emailer } from './server/lib/emailer' -import { JobQueue } from './server/lib/job-queue' -import { - activityPubRouter, - apiRouter, - miscRouter, - clientsRouter, - feedsRouter, - staticRouter, - wellKnownRouter, - lazyStaticRouter, - servicesRouter, - objectStorageProxyRouter, - pluginsRouter, - trackerRouter, - createWebsocketTrackerServer, - sitemapRouter, - downloadRouter -} from './server/controllers' -import { advertiseDoNotTrack } from './server/middlewares/dnt' -import { apiFailMiddleware } from './server/middlewares/error' -import { Redis } from './server/lib/redis' -import { ActorFollowScheduler } from './server/lib/schedulers/actor-follow-scheduler' -import { RemoveOldViewsScheduler } from './server/lib/schedulers/remove-old-views-scheduler' -import { UpdateVideosScheduler } from './server/lib/schedulers/update-videos-scheduler' -import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-update-scheduler' -import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler' -import { RemoveOldHistoryScheduler } from './server/lib/schedulers/remove-old-history-scheduler' -import { AutoFollowIndexInstances } from './server/lib/schedulers/auto-follow-index-instances' -import { RemoveDanglingResumableUploadsScheduler } from './server/lib/schedulers/remove-dangling-resumable-uploads-scheduler' -import { VideoViewsBufferScheduler } from './server/lib/schedulers/video-views-buffer-scheduler' -import { GeoIPUpdateScheduler } from './server/lib/schedulers/geo-ip-update-scheduler' -import { RunnerJobWatchDogScheduler } from './server/lib/schedulers/runner-job-watch-dog-scheduler' -import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto' -import { PeerTubeSocket } from './server/lib/peertube-socket' -import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls' -import { PluginsCheckScheduler } from './server/lib/schedulers/plugins-check-scheduler' -import { PeerTubeVersionCheckScheduler } from './server/lib/schedulers/peertube-version-check-scheduler' -import { Hooks } from './server/lib/plugins/hooks' -import { PluginManager } from './server/lib/plugins/plugin-manager' -import { LiveManager } from './server/lib/live' -import { HttpStatusCode } from './shared/models/http/http-error-codes' -import { ServerConfigManager } from '@server/lib/server-config-manager' -import { VideoViewsManager } from '@server/lib/views/video-views-manager' -import { isTestOrDevInstance } from './server/helpers/core-utils' -import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics' -import { ApplicationModel } from '@server/models/application/application' -import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler' - -// ----------- Command line ----------- - -cli - .option('--no-client', 'Start PeerTube without client interface') - .option('--no-plugins', 'Start PeerTube without plugins/themes enabled') - .option('--benchmark-startup', 'Automatically stop server when initialized') - .parse(process.argv) - -// ----------- App ----------- - -// Enable CORS for develop -if (isTestOrDevInstance()) { - app.use(cors({ - origin: '*', - exposedHeaders: 'Retry-After', - credentials: true - })) -} - -// For the logger -token('remote-addr', (req: express.Request) => { - if (CONFIG.LOG.ANONYMIZE_IP === true || req.get('DNT') === '1') { - return anonymize(req.ip, 16, 16) - } - - return req.ip -}) -token('user-agent', (req: express.Request) => { - if (req.get('DNT') === '1') { - return parse(req.get('user-agent')).family - } - - return req.get('user-agent') -}) -app.use(morgan('combined', { - stream: { - write: (str: string) => logger.info(str.trim(), { tags: [ 'http' ] }) - }, - skip: req => CONFIG.LOG.LOG_PING_REQUESTS === false && req.originalUrl === '/api/v1/ping' -})) - -// Add .fail() helper to response -app.use(apiFailMiddleware) - -// For body requests -app.use(express.urlencoded({ extended: false })) -app.use(express.json({ - type: [ 'application/json', 'application/*+json' ], - limit: '500kb', - verify: (req: express.Request, res: express.Response, buf: Buffer) => { - const valid = isHTTPSignatureDigestValid(buf, req) - - if (valid !== true) { - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Invalid digest' - }) - } - } -})) - -// Cookies -app.use(cookieParser()) - -// W3C DNT Tracking Status -app.use(advertiseDoNotTrack) - -// ----------- Open Telemetry ----------- - -OpenTelemetryMetrics.Instance.init(app) - -// ----------- Views, routes and static files ----------- - -app.use('/api/' + API_VERSION, apiRouter) - -// Services (oembed...) -app.use('/services', servicesRouter) - -// Plugins & themes -app.use('/', pluginsRouter) - -app.use('/', activityPubRouter) -app.use('/', feedsRouter) -app.use('/', trackerRouter) -app.use('/', sitemapRouter) - -// Static files -app.use('/', staticRouter) -app.use('/', wellKnownRouter) -app.use('/', miscRouter) -app.use('/', downloadRouter) -app.use('/', lazyStaticRouter) -app.use('/', objectStorageProxyRouter) - -// Client files, last valid routes! -const cliOptions = cli.opts<{ client: boolean, plugins: boolean }>() -if (cliOptions.client) app.use('/', clientsRouter) - -// ----------- Errors ----------- - -// Catch unmatched routes -app.use((_req, res: express.Response) => { - res.status(HttpStatusCode.NOT_FOUND_404).end() -}) - -// Catch thrown errors -app.use((err, _req, res: express.Response, _next) => { - // Format error to be logged - let error = 'Unknown error.' - if (err) { - error = err.stack || err.message || err - } - - // Handling Sequelize error traces - const sql = err?.parent ? err.parent.sql : undefined - - // Help us to debug SequelizeConnectionAcquireTimeoutError errors - const activeRequests = err?.name === 'SequelizeConnectionAcquireTimeoutError' && typeof (process as any)._getActiveRequests !== 'function' - ? (process as any)._getActiveRequests() - : undefined - - logger.error('Error in controller.', { err: error, sql, activeRequests }) - - return res.fail({ - status: err.status || HttpStatusCode.INTERNAL_SERVER_ERROR_500, - message: err.message, - type: err.name - }) -}) - -const { server, trackerServer } = createWebsocketTrackerServer(app) - -// ----------- Run ----------- - -async function startApplication () { - const port = CONFIG.LISTEN.PORT - const hostname = CONFIG.LISTEN.HOSTNAME - - await installApplication() - - // Check activity pub urls are valid - checkActivityPubUrls() - .catch(err => { - logger.error('Error in ActivityPub URLs checker.', { err }) - process.exit(-1) - }) - - checkFFmpegVersion() - .catch(err => logger.error('Cannot check ffmpeg version', { err })) - - Redis.Instance.init() - Emailer.Instance.init() - - await Promise.all([ - Emailer.Instance.checkConnection(), - JobQueue.Instance.init(), - ServerConfigManager.Instance.init() - ]) - - // Enable Schedulers - ActorFollowScheduler.Instance.enable() - UpdateVideosScheduler.Instance.enable() - YoutubeDlUpdateScheduler.Instance.enable() - VideosRedundancyScheduler.Instance.enable() - RemoveOldHistoryScheduler.Instance.enable() - RemoveOldViewsScheduler.Instance.enable() - PluginsCheckScheduler.Instance.enable() - PeerTubeVersionCheckScheduler.Instance.enable() - AutoFollowIndexInstances.Instance.enable() - RemoveDanglingResumableUploadsScheduler.Instance.enable() - VideoChannelSyncLatestScheduler.Instance.enable() - VideoViewsBufferScheduler.Instance.enable() - GeoIPUpdateScheduler.Instance.enable() - RunnerJobWatchDogScheduler.Instance.enable() - - OpenTelemetryMetrics.Instance.registerMetrics({ trackerServer }) - - PluginManager.Instance.init(server) - // Before PeerTubeSocket init - PluginManager.Instance.registerWebSocketRouter() - - PeerTubeSocket.Instance.init(server) - VideoViewsManager.Instance.init() - - updateStreamingPlaylistsInfohashesIfNeeded() - .catch(err => logger.error('Cannot update streaming playlist infohashes.', { err })) - - LiveManager.Instance.init() - if (CONFIG.LIVE.ENABLED) await LiveManager.Instance.run() - - // Make server listening - server.listen(port, hostname, async () => { - if (cliOptions.plugins) { - try { - await PluginManager.Instance.rebuildNativePluginsIfNeeded() - - await PluginManager.Instance.registerPluginsAndThemes() - } catch (err) { - logger.error('Cannot register plugins and themes.', { err }) - } - } - - ApplicationModel.updateNodeVersions() - .catch(err => logger.error('Cannot update node versions.', { err })) - - JobQueue.Instance.start() - .catch(err => { - logger.error('Cannot start job queue.', { err }) - process.exit(-1) - }) - - logger.info('HTTP server listening on %s:%d', hostname, port) - logger.info('Web server: %s', WEBSERVER.URL) - - Hooks.runAction('action:application.listening') - - if (cliOptions['benchmarkStartup']) process.exit(0) - }) - - process.on('exit', () => { - JobQueue.Instance.terminate() - .catch(err => logger.error('Cannot terminate job queue.', { err })) - }) - - process.on('SIGINT', () => process.exit(0)) -} diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts deleted file mode 100644 index be52f1662..000000000 --- a/server/controllers/activitypub/client.ts +++ /dev/null @@ -1,482 +0,0 @@ -import cors from 'cors' -import express from 'express' -import { activityPubCollectionPagination } from '@server/lib/activitypub/collection' -import { activityPubContextify } from '@server/lib/activitypub/context' -import { getServerActor } from '@server/models/application/application' -import { MAccountId, MActorId, MChannelId, MVideoId } from '@server/types/models' -import { VideoCommentObject } from '@shared/models' -import { VideoPrivacy, VideoRateType } from '../../../shared/models/videos' -import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model' -import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../../initializers/constants' -import { audiencify, getAudience } from '../../lib/activitypub/audience' -import { buildAnnounceWithVideoAudience, buildLikeActivity } from '../../lib/activitypub/send' -import { buildCreateActivity } from '../../lib/activitypub/send/send-create' -import { buildDislikeActivity } from '../../lib/activitypub/send/send-dislike' -import { - getLocalVideoCommentsActivityPubUrl, - getLocalVideoDislikesActivityPubUrl, - getLocalVideoLikesActivityPubUrl, - getLocalVideoSharesActivityPubUrl -} from '../../lib/activitypub/url' -import { - activityPubRateLimiter, - asyncMiddleware, - ensureIsLocalChannel, - executeIfActivityPub, - localAccountValidator, - videoChannelsNameWithHostValidator, - videosCustomGetValidator, - videosShareValidator -} from '../../middlewares' -import { cacheRoute } from '../../middlewares/cache/cache' -import { getAccountVideoRateValidatorFactory, getVideoLocalViewerValidator, videoCommentGetValidator } from '../../middlewares/validators' -import { videoFileRedundancyGetValidator, videoPlaylistRedundancyGetValidator } from '../../middlewares/validators/redundancy' -import { videoPlaylistElementAPGetValidator, videoPlaylistsGetValidator } from '../../middlewares/validators/videos/video-playlists' -import { AccountModel } from '../../models/account/account' -import { AccountVideoRateModel } from '../../models/account/account-video-rate' -import { ActorFollowModel } from '../../models/actor/actor-follow' -import { VideoCommentModel } from '../../models/video/video-comment' -import { VideoPlaylistModel } from '../../models/video/video-playlist' -import { VideoShareModel } from '../../models/video/video-share' -import { activityPubResponse } from './utils' - -const activityPubClientRouter = express.Router() -activityPubClientRouter.use(cors()) - -// Intercept ActivityPub client requests - -activityPubClientRouter.get( - [ '/accounts?/:name', '/accounts?/:name/video-channels', '/a/:name', '/a/:name/video-channels' ], - executeIfActivityPub, - activityPubRateLimiter, - asyncMiddleware(localAccountValidator), - asyncMiddleware(accountController) -) -activityPubClientRouter.get('/accounts?/:name/followers', - executeIfActivityPub, - activityPubRateLimiter, - asyncMiddleware(localAccountValidator), - asyncMiddleware(accountFollowersController) -) -activityPubClientRouter.get('/accounts?/:name/following', - executeIfActivityPub, - activityPubRateLimiter, - asyncMiddleware(localAccountValidator), - asyncMiddleware(accountFollowingController) -) -activityPubClientRouter.get('/accounts?/:name/playlists', - executeIfActivityPub, - activityPubRateLimiter, - asyncMiddleware(localAccountValidator), - asyncMiddleware(accountPlaylistsController) -) -activityPubClientRouter.get('/accounts?/:name/likes/:videoId', - executeIfActivityPub, - activityPubRateLimiter, - cacheRoute(ROUTE_CACHE_LIFETIME.ACTIVITY_PUB.VIDEOS), - asyncMiddleware(getAccountVideoRateValidatorFactory('like')), - asyncMiddleware(getAccountVideoRateFactory('like')) -) -activityPubClientRouter.get('/accounts?/:name/dislikes/:videoId', - executeIfActivityPub, - activityPubRateLimiter, - cacheRoute(ROUTE_CACHE_LIFETIME.ACTIVITY_PUB.VIDEOS), - asyncMiddleware(getAccountVideoRateValidatorFactory('dislike')), - asyncMiddleware(getAccountVideoRateFactory('dislike')) -) - -activityPubClientRouter.get( - [ '/videos/watch/:id', '/w/:id' ], - executeIfActivityPub, - activityPubRateLimiter, - cacheRoute(ROUTE_CACHE_LIFETIME.ACTIVITY_PUB.VIDEOS), - asyncMiddleware(videosCustomGetValidator('all')), - asyncMiddleware(videoController) -) -activityPubClientRouter.get('/videos/watch/:id/activity', - executeIfActivityPub, - activityPubRateLimiter, - asyncMiddleware(videosCustomGetValidator('all')), - asyncMiddleware(videoController) -) -activityPubClientRouter.get('/videos/watch/:id/announces', - executeIfActivityPub, - activityPubRateLimiter, - asyncMiddleware(videosCustomGetValidator('only-immutable-attributes')), - asyncMiddleware(videoAnnouncesController) -) -activityPubClientRouter.get('/videos/watch/:id/announces/:actorId', - executeIfActivityPub, - activityPubRateLimiter, - asyncMiddleware(videosShareValidator), - asyncMiddleware(videoAnnounceController) -) -activityPubClientRouter.get('/videos/watch/:id/likes', - executeIfActivityPub, - activityPubRateLimiter, - asyncMiddleware(videosCustomGetValidator('only-immutable-attributes')), - asyncMiddleware(videoLikesController) -) -activityPubClientRouter.get('/videos/watch/:id/dislikes', - executeIfActivityPub, - activityPubRateLimiter, - asyncMiddleware(videosCustomGetValidator('only-immutable-attributes')), - asyncMiddleware(videoDislikesController) -) -activityPubClientRouter.get('/videos/watch/:id/comments', - executeIfActivityPub, - activityPubRateLimiter, - asyncMiddleware(videosCustomGetValidator('only-immutable-attributes')), - asyncMiddleware(videoCommentsController) -) -activityPubClientRouter.get('/videos/watch/:videoId/comments/:commentId', - executeIfActivityPub, - activityPubRateLimiter, - asyncMiddleware(videoCommentGetValidator), - asyncMiddleware(videoCommentController) -) -activityPubClientRouter.get('/videos/watch/:videoId/comments/:commentId/activity', - executeIfActivityPub, - activityPubRateLimiter, - asyncMiddleware(videoCommentGetValidator), - asyncMiddleware(videoCommentController) -) - -activityPubClientRouter.get( - [ '/video-channels/:nameWithHost', '/video-channels/:nameWithHost/videos', '/c/:nameWithHost', '/c/:nameWithHost/videos' ], - executeIfActivityPub, - activityPubRateLimiter, - asyncMiddleware(videoChannelsNameWithHostValidator), - ensureIsLocalChannel, - asyncMiddleware(videoChannelController) -) -activityPubClientRouter.get('/video-channels/:nameWithHost/followers', - executeIfActivityPub, - activityPubRateLimiter, - asyncMiddleware(videoChannelsNameWithHostValidator), - ensureIsLocalChannel, - asyncMiddleware(videoChannelFollowersController) -) -activityPubClientRouter.get('/video-channels/:nameWithHost/following', - executeIfActivityPub, - activityPubRateLimiter, - asyncMiddleware(videoChannelsNameWithHostValidator), - ensureIsLocalChannel, - asyncMiddleware(videoChannelFollowingController) -) -activityPubClientRouter.get('/video-channels/:nameWithHost/playlists', - executeIfActivityPub, - activityPubRateLimiter, - asyncMiddleware(videoChannelsNameWithHostValidator), - ensureIsLocalChannel, - asyncMiddleware(videoChannelPlaylistsController) -) - -activityPubClientRouter.get('/redundancy/videos/:videoId/:resolution([0-9]+)(-:fps([0-9]+))?', - executeIfActivityPub, - activityPubRateLimiter, - asyncMiddleware(videoFileRedundancyGetValidator), - asyncMiddleware(videoRedundancyController) -) -activityPubClientRouter.get('/redundancy/streaming-playlists/:streamingPlaylistType/:videoId', - executeIfActivityPub, - activityPubRateLimiter, - asyncMiddleware(videoPlaylistRedundancyGetValidator), - asyncMiddleware(videoRedundancyController) -) - -activityPubClientRouter.get( - [ '/video-playlists/:playlistId', '/videos/watch/playlist/:playlistId', '/w/p/:playlistId' ], - executeIfActivityPub, - activityPubRateLimiter, - asyncMiddleware(videoPlaylistsGetValidator('all')), - asyncMiddleware(videoPlaylistController) -) -activityPubClientRouter.get('/video-playlists/:playlistId/videos/:playlistElementId', - executeIfActivityPub, - activityPubRateLimiter, - asyncMiddleware(videoPlaylistElementAPGetValidator), - asyncMiddleware(videoPlaylistElementController) -) - -activityPubClientRouter.get('/videos/local-viewer/:localViewerId', - executeIfActivityPub, - activityPubRateLimiter, - asyncMiddleware(getVideoLocalViewerValidator), - asyncMiddleware(getVideoLocalViewerController) -) - -// --------------------------------------------------------------------------- - -export { - activityPubClientRouter -} - -// --------------------------------------------------------------------------- - -async function accountController (req: express.Request, res: express.Response) { - const account = res.locals.account - - return activityPubResponse(activityPubContextify(await account.toActivityPubObject(), 'Actor'), res) -} - -async function accountFollowersController (req: express.Request, res: express.Response) { - const account = res.locals.account - const activityPubResult = await actorFollowers(req, account.Actor) - - return activityPubResponse(activityPubContextify(activityPubResult, 'Collection'), res) -} - -async function accountFollowingController (req: express.Request, res: express.Response) { - const account = res.locals.account - const activityPubResult = await actorFollowing(req, account.Actor) - - return activityPubResponse(activityPubContextify(activityPubResult, 'Collection'), res) -} - -async function accountPlaylistsController (req: express.Request, res: express.Response) { - const account = res.locals.account - const activityPubResult = await actorPlaylists(req, { account }) - - return activityPubResponse(activityPubContextify(activityPubResult, 'Collection'), res) -} - -async function videoChannelPlaylistsController (req: express.Request, res: express.Response) { - const channel = res.locals.videoChannel - const activityPubResult = await actorPlaylists(req, { channel }) - - return activityPubResponse(activityPubContextify(activityPubResult, 'Collection'), res) -} - -function getAccountVideoRateFactory (rateType: VideoRateType) { - return (req: express.Request, res: express.Response) => { - const accountVideoRate = res.locals.accountVideoRate - - const byActor = accountVideoRate.Account.Actor - const APObject = rateType === 'like' - ? buildLikeActivity(accountVideoRate.url, byActor, accountVideoRate.Video) - : buildDislikeActivity(accountVideoRate.url, byActor, accountVideoRate.Video) - - return activityPubResponse(activityPubContextify(APObject, 'Rate'), res) - } -} - -async function videoController (req: express.Request, res: express.Response) { - const video = res.locals.videoAll - - if (redirectIfNotOwned(video.url, res)) return - - // We need captions to render AP object - const videoAP = await video.lightAPToFullAP(undefined) - - const audience = getAudience(videoAP.VideoChannel.Account.Actor, videoAP.privacy === VideoPrivacy.PUBLIC) - const videoObject = audiencify(await videoAP.toActivityPubObject(), audience) - - if (req.path.endsWith('/activity')) { - const data = buildCreateActivity(videoAP.url, video.VideoChannel.Account.Actor, videoObject, audience) - return activityPubResponse(activityPubContextify(data, 'Video'), res) - } - - return activityPubResponse(activityPubContextify(videoObject, 'Video'), res) -} - -async function videoAnnounceController (req: express.Request, res: express.Response) { - const share = res.locals.videoShare - - if (redirectIfNotOwned(share.url, res)) return - - const { activity } = await buildAnnounceWithVideoAudience(share.Actor, share, res.locals.videoAll, undefined) - - return activityPubResponse(activityPubContextify(activity, 'Announce'), res) -} - -async function videoAnnouncesController (req: express.Request, res: express.Response) { - const video = res.locals.onlyImmutableVideo - - if (redirectIfNotOwned(video.url, res)) return - - const handler = async (start: number, count: number) => { - const result = await VideoShareModel.listAndCountByVideoId(video.id, start, count) - return { - total: result.total, - data: result.data.map(r => r.url) - } - } - const json = await activityPubCollectionPagination(getLocalVideoSharesActivityPubUrl(video), handler, req.query.page) - - return activityPubResponse(activityPubContextify(json, 'Collection'), res) -} - -async function videoLikesController (req: express.Request, res: express.Response) { - const video = res.locals.onlyImmutableVideo - - if (redirectIfNotOwned(video.url, res)) return - - const json = await videoRates(req, 'like', video, getLocalVideoLikesActivityPubUrl(video)) - - return activityPubResponse(activityPubContextify(json, 'Collection'), res) -} - -async function videoDislikesController (req: express.Request, res: express.Response) { - const video = res.locals.onlyImmutableVideo - - if (redirectIfNotOwned(video.url, res)) return - - const json = await videoRates(req, 'dislike', video, getLocalVideoDislikesActivityPubUrl(video)) - - return activityPubResponse(activityPubContextify(json, 'Collection'), res) -} - -async function videoCommentsController (req: express.Request, res: express.Response) { - const video = res.locals.onlyImmutableVideo - - if (redirectIfNotOwned(video.url, res)) return - - const handler = async (start: number, count: number) => { - const result = await VideoCommentModel.listAndCountByVideoForAP({ video, start, count }) - - return { - total: result.total, - data: result.data.map(r => r.url) - } - } - const json = await activityPubCollectionPagination(getLocalVideoCommentsActivityPubUrl(video), handler, req.query.page) - - return activityPubResponse(activityPubContextify(json, 'Collection'), res) -} - -async function videoChannelController (req: express.Request, res: express.Response) { - const videoChannel = res.locals.videoChannel - - return activityPubResponse(activityPubContextify(await videoChannel.toActivityPubObject(), 'Actor'), res) -} - -async function videoChannelFollowersController (req: express.Request, res: express.Response) { - const videoChannel = res.locals.videoChannel - const activityPubResult = await actorFollowers(req, videoChannel.Actor) - - return activityPubResponse(activityPubContextify(activityPubResult, 'Collection'), res) -} - -async function videoChannelFollowingController (req: express.Request, res: express.Response) { - const videoChannel = res.locals.videoChannel - const activityPubResult = await actorFollowing(req, videoChannel.Actor) - - return activityPubResponse(activityPubContextify(activityPubResult, 'Collection'), res) -} - -async function videoCommentController (req: express.Request, res: express.Response) { - const videoComment = res.locals.videoCommentFull - - if (redirectIfNotOwned(videoComment.url, res)) return - - const threadParentComments = await VideoCommentModel.listThreadParentComments(videoComment, undefined) - const isPublic = true // Comments are always public - let videoCommentObject = videoComment.toActivityPubObject(threadParentComments) - - if (videoComment.Account) { - const audience = getAudience(videoComment.Account.Actor, isPublic) - videoCommentObject = audiencify(videoCommentObject, audience) - - if (req.path.endsWith('/activity')) { - const data = buildCreateActivity(videoComment.url, videoComment.Account.Actor, videoCommentObject as VideoCommentObject, audience) - return activityPubResponse(activityPubContextify(data, 'Comment'), res) - } - } - - return activityPubResponse(activityPubContextify(videoCommentObject, 'Comment'), res) -} - -async function videoRedundancyController (req: express.Request, res: express.Response) { - const videoRedundancy = res.locals.videoRedundancy - - if (redirectIfNotOwned(videoRedundancy.url, res)) return - - const serverActor = await getServerActor() - - const audience = getAudience(serverActor) - const object = audiencify(videoRedundancy.toActivityPubObject(), audience) - - if (req.path.endsWith('/activity')) { - const data = buildCreateActivity(videoRedundancy.url, serverActor, object, audience) - return activityPubResponse(activityPubContextify(data, 'CacheFile'), res) - } - - return activityPubResponse(activityPubContextify(object, 'CacheFile'), res) -} - -async function videoPlaylistController (req: express.Request, res: express.Response) { - const playlist = res.locals.videoPlaylistFull - - if (redirectIfNotOwned(playlist.url, res)) return - - // We need more attributes - playlist.OwnerAccount = await AccountModel.load(playlist.ownerAccountId) - - const json = await playlist.toActivityPubObject(req.query.page, null) - const audience = getAudience(playlist.OwnerAccount.Actor, playlist.privacy === VideoPlaylistPrivacy.PUBLIC) - const object = audiencify(json, audience) - - return activityPubResponse(activityPubContextify(object, 'Playlist'), res) -} - -function videoPlaylistElementController (req: express.Request, res: express.Response) { - const videoPlaylistElement = res.locals.videoPlaylistElementAP - - if (redirectIfNotOwned(videoPlaylistElement.url, res)) return - - const json = videoPlaylistElement.toActivityPubObject() - return activityPubResponse(activityPubContextify(json, 'Playlist'), res) -} - -function getVideoLocalViewerController (req: express.Request, res: express.Response) { - const localViewer = res.locals.localViewerFull - - return activityPubResponse(activityPubContextify(localViewer.toActivityPubObject(), 'WatchAction'), res) -} - -// --------------------------------------------------------------------------- - -function actorFollowing (req: express.Request, actor: MActorId) { - const handler = (start: number, count: number) => { - return ActorFollowModel.listAcceptedFollowingUrlsForApi([ actor.id ], undefined, start, count) - } - - return activityPubCollectionPagination(WEBSERVER.URL + req.path, handler, req.query.page) -} - -function actorFollowers (req: express.Request, actor: MActorId) { - const handler = (start: number, count: number) => { - return ActorFollowModel.listAcceptedFollowerUrlsForAP([ actor.id ], undefined, start, count) - } - - return activityPubCollectionPagination(WEBSERVER.URL + req.path, handler, req.query.page) -} - -function actorPlaylists (req: express.Request, options: { account: MAccountId } | { channel: MChannelId }) { - const handler = (start: number, count: number) => { - return VideoPlaylistModel.listPublicUrlsOfForAP(options, start, count) - } - - return activityPubCollectionPagination(WEBSERVER.URL + req.path, handler, req.query.page) -} - -function videoRates (req: express.Request, rateType: VideoRateType, video: MVideoId, url: string) { - const handler = async (start: number, count: number) => { - const result = await AccountVideoRateModel.listAndCountAccountUrlsByVideoId(rateType, video.id, start, count) - return { - total: result.total, - data: result.data.map(r => r.url) - } - } - return activityPubCollectionPagination(url, handler, req.query.page) -} - -function redirectIfNotOwned (url: string, res: express.Response) { - if (url.startsWith(WEBSERVER.URL) === false) { - res.redirect(url) - return true - } - - return false -} diff --git a/server/controllers/activitypub/inbox.ts b/server/controllers/activitypub/inbox.ts deleted file mode 100644 index 862c7baf1..000000000 --- a/server/controllers/activitypub/inbox.ts +++ /dev/null @@ -1,85 +0,0 @@ -import express from 'express' -import { InboxManager } from '@server/lib/activitypub/inbox-manager' -import { Activity, ActivityPubCollection, ActivityPubOrderedCollection, RootActivity } from '@shared/models' -import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' -import { isActivityValid } from '../../helpers/custom-validators/activitypub/activity' -import { logger } from '../../helpers/logger' -import { - activityPubRateLimiter, - asyncMiddleware, - checkSignature, - ensureIsLocalChannel, - localAccountValidator, - signatureValidator, - videoChannelsNameWithHostValidator -} from '../../middlewares' -import { activityPubValidator } from '../../middlewares/validators/activitypub/activity' - -const inboxRouter = express.Router() - -inboxRouter.post('/inbox', - activityPubRateLimiter, - signatureValidator, - asyncMiddleware(checkSignature), - asyncMiddleware(activityPubValidator), - inboxController -) - -inboxRouter.post('/accounts/:name/inbox', - activityPubRateLimiter, - signatureValidator, - asyncMiddleware(checkSignature), - asyncMiddleware(localAccountValidator), - asyncMiddleware(activityPubValidator), - inboxController -) - -inboxRouter.post('/video-channels/:nameWithHost/inbox', - activityPubRateLimiter, - signatureValidator, - asyncMiddleware(checkSignature), - asyncMiddleware(videoChannelsNameWithHostValidator), - ensureIsLocalChannel, - asyncMiddleware(activityPubValidator), - inboxController -) - -// --------------------------------------------------------------------------- - -export { - inboxRouter -} - -// --------------------------------------------------------------------------- - -function inboxController (req: express.Request, res: express.Response) { - const rootActivity: RootActivity = req.body - let activities: Activity[] - - if ([ 'Collection', 'CollectionPage' ].includes(rootActivity.type)) { - activities = (rootActivity as ActivityPubCollection).items - } else if ([ 'OrderedCollection', 'OrderedCollectionPage' ].includes(rootActivity.type)) { - activities = (rootActivity as ActivityPubOrderedCollection).orderedItems - } else { - activities = [ rootActivity as Activity ] - } - - // Only keep activities we are able to process - logger.debug('Filtering %d activities...', activities.length) - activities = activities.filter(a => isActivityValid(a)) - logger.debug('We keep %d activities.', activities.length, { activities }) - - const accountOrChannel = res.locals.account || res.locals.videoChannel - - logger.info('Receiving inbox requests for %d activities by %s.', activities.length, res.locals.signature.actor.url) - - InboxManager.Instance.addInboxMessage({ - activities, - signatureActor: res.locals.signature.actor, - inboxActor: accountOrChannel - ? accountOrChannel.Actor - : undefined - }) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} diff --git a/server/controllers/activitypub/index.ts b/server/controllers/activitypub/index.ts deleted file mode 100644 index c14d95108..000000000 --- a/server/controllers/activitypub/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -import express from 'express' - -import { activityPubClientRouter } from './client' -import { inboxRouter } from './inbox' -import { outboxRouter } from './outbox' - -const activityPubRouter = express.Router() - -activityPubRouter.use('/', inboxRouter) -activityPubRouter.use('/', outboxRouter) -activityPubRouter.use('/', activityPubClientRouter) - -// --------------------------------------------------------------------------- - -export { - activityPubRouter -} diff --git a/server/controllers/activitypub/outbox.ts b/server/controllers/activitypub/outbox.ts deleted file mode 100644 index 8c88b6971..000000000 --- a/server/controllers/activitypub/outbox.ts +++ /dev/null @@ -1,86 +0,0 @@ -import express from 'express' -import { activityPubCollectionPagination } from '@server/lib/activitypub/collection' -import { activityPubContextify } from '@server/lib/activitypub/context' -import { MActorLight } from '@server/types/models' -import { Activity } from '../../../shared/models/activitypub/activity' -import { VideoPrivacy } from '../../../shared/models/videos' -import { logger } from '../../helpers/logger' -import { buildAudience } from '../../lib/activitypub/audience' -import { buildAnnounceActivity, buildCreateActivity } from '../../lib/activitypub/send' -import { - activityPubRateLimiter, - asyncMiddleware, - ensureIsLocalChannel, - localAccountValidator, - videoChannelsNameWithHostValidator -} from '../../middlewares' -import { apPaginationValidator } from '../../middlewares/validators/activitypub' -import { VideoModel } from '../../models/video/video' -import { activityPubResponse } from './utils' - -const outboxRouter = express.Router() - -outboxRouter.get('/accounts/:name/outbox', - activityPubRateLimiter, - apPaginationValidator, - localAccountValidator, - asyncMiddleware(outboxController) -) - -outboxRouter.get('/video-channels/:nameWithHost/outbox', - activityPubRateLimiter, - apPaginationValidator, - asyncMiddleware(videoChannelsNameWithHostValidator), - ensureIsLocalChannel, - asyncMiddleware(outboxController) -) - -// --------------------------------------------------------------------------- - -export { - outboxRouter -} - -// --------------------------------------------------------------------------- - -async function outboxController (req: express.Request, res: express.Response) { - const accountOrVideoChannel = res.locals.account || res.locals.videoChannel - const actor = accountOrVideoChannel.Actor - const actorOutboxUrl = actor.url + '/outbox' - - logger.info('Receiving outbox request for %s.', actorOutboxUrl) - - const handler = (start: number, count: number) => buildActivities(actor, start, count) - const json = await activityPubCollectionPagination(actorOutboxUrl, handler, req.query.page, req.query.size) - - return activityPubResponse(activityPubContextify(json, 'Collection'), res) -} - -async function buildActivities (actor: MActorLight, start: number, count: number) { - const data = await VideoModel.listAllAndSharedByActorForOutbox(actor.id, start, count) - const activities: Activity[] = [] - - for (const video of data.data) { - const byActor = video.VideoChannel.Account.Actor - const createActivityAudience = buildAudience([ byActor.followersUrl ], video.privacy === VideoPrivacy.PUBLIC) - - // This is a shared video - if (video.VideoShares !== undefined && video.VideoShares.length !== 0) { - const videoShare = video.VideoShares[0] - const announceActivity = buildAnnounceActivity(videoShare.url, actor, video.url, createActivityAudience) - - activities.push(announceActivity) - } else { - // FIXME: only use the video URL to reduce load. Breaks compat with PeerTube < 6.0.0 - const videoObject = await video.toActivityPubObject() - const createActivity = buildCreateActivity(video.url, byActor, videoObject, createActivityAudience) - - activities.push(createActivity) - } - } - - return { - data: activities, - total: data.total - } -} diff --git a/server/controllers/api/abuse.ts b/server/controllers/api/abuse.ts deleted file mode 100644 index d582f198d..000000000 --- a/server/controllers/api/abuse.ts +++ /dev/null @@ -1,259 +0,0 @@ -import express from 'express' -import { logger } from '@server/helpers/logger' -import { createAccountAbuse, createVideoAbuse, createVideoCommentAbuse } from '@server/lib/moderation' -import { Notifier } from '@server/lib/notifier' -import { AbuseModel } from '@server/models/abuse/abuse' -import { AbuseMessageModel } from '@server/models/abuse/abuse-message' -import { getServerActor } from '@server/models/application/application' -import { abusePredefinedReasonsMap } from '@shared/core-utils/abuse' -import { AbuseCreate, AbuseState, HttpStatusCode, UserRight } from '@shared/models' -import { getFormattedObjects } from '../../helpers/utils' -import { sequelizeTypescript } from '../../initializers/database' -import { - abuseGetValidator, - abuseListForAdminsValidator, - abuseReportValidator, - abusesSortValidator, - abuseUpdateValidator, - addAbuseMessageValidator, - apiRateLimiter, - asyncMiddleware, - asyncRetryTransactionMiddleware, - authenticate, - checkAbuseValidForMessagesValidator, - deleteAbuseMessageValidator, - ensureUserHasRight, - getAbuseValidator, - openapiOperationDoc, - paginationValidator, - setDefaultPagination, - setDefaultSort -} from '../../middlewares' -import { AccountModel } from '../../models/account/account' - -const abuseRouter = express.Router() - -abuseRouter.use(apiRateLimiter) - -abuseRouter.get('/', - openapiOperationDoc({ operationId: 'getAbuses' }), - authenticate, - ensureUserHasRight(UserRight.MANAGE_ABUSES), - paginationValidator, - abusesSortValidator, - setDefaultSort, - setDefaultPagination, - abuseListForAdminsValidator, - asyncMiddleware(listAbusesForAdmins) -) -abuseRouter.put('/:id', - authenticate, - ensureUserHasRight(UserRight.MANAGE_ABUSES), - asyncMiddleware(abuseUpdateValidator), - asyncRetryTransactionMiddleware(updateAbuse) -) -abuseRouter.post('/', - authenticate, - asyncMiddleware(abuseReportValidator), - asyncRetryTransactionMiddleware(reportAbuse) -) -abuseRouter.delete('/:id', - authenticate, - ensureUserHasRight(UserRight.MANAGE_ABUSES), - asyncMiddleware(abuseGetValidator), - asyncRetryTransactionMiddleware(deleteAbuse) -) - -abuseRouter.get('/:id/messages', - authenticate, - asyncMiddleware(getAbuseValidator), - checkAbuseValidForMessagesValidator, - asyncRetryTransactionMiddleware(listAbuseMessages) -) - -abuseRouter.post('/:id/messages', - authenticate, - asyncMiddleware(getAbuseValidator), - checkAbuseValidForMessagesValidator, - addAbuseMessageValidator, - asyncRetryTransactionMiddleware(addAbuseMessage) -) - -abuseRouter.delete('/:id/messages/:messageId', - authenticate, - asyncMiddleware(getAbuseValidator), - checkAbuseValidForMessagesValidator, - asyncMiddleware(deleteAbuseMessageValidator), - asyncRetryTransactionMiddleware(deleteAbuseMessage) -) - -// --------------------------------------------------------------------------- - -export { - abuseRouter -} - -// --------------------------------------------------------------------------- - -async function listAbusesForAdmins (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.user - const serverActor = await getServerActor() - - const resultList = await AbuseModel.listForAdminApi({ - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - id: req.query.id, - filter: req.query.filter, - predefinedReason: req.query.predefinedReason, - search: req.query.search, - state: req.query.state, - videoIs: req.query.videoIs, - searchReporter: req.query.searchReporter, - searchReportee: req.query.searchReportee, - searchVideo: req.query.searchVideo, - searchVideoChannel: req.query.searchVideoChannel, - serverAccountId: serverActor.Account.id, - user - }) - - return res.json({ - total: resultList.total, - data: resultList.data.map(d => d.toFormattedAdminJSON()) - }) -} - -async function updateAbuse (req: express.Request, res: express.Response) { - const abuse = res.locals.abuse - let stateUpdated = false - - if (req.body.moderationComment !== undefined) abuse.moderationComment = req.body.moderationComment - - if (req.body.state !== undefined) { - abuse.state = req.body.state - stateUpdated = true - } - - await sequelizeTypescript.transaction(t => { - return abuse.save({ transaction: t }) - }) - - if (stateUpdated === true) { - AbuseModel.loadFull(abuse.id) - .then(abuseFull => Notifier.Instance.notifyOnAbuseStateChange(abuseFull)) - .catch(err => logger.error('Cannot notify on abuse state change', { err })) - } - - // Do not send the delete to other instances, we updated OUR copy of this abuse - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function deleteAbuse (req: express.Request, res: express.Response) { - const abuse = res.locals.abuse - - await sequelizeTypescript.transaction(t => { - return abuse.destroy({ transaction: t }) - }) - - // Do not send the delete to other instances, we delete OUR copy of this abuse - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function reportAbuse (req: express.Request, res: express.Response) { - const videoInstance = res.locals.videoAll - const commentInstance = res.locals.videoCommentFull - const accountInstance = res.locals.account - - const body: AbuseCreate = req.body - - const { id } = await sequelizeTypescript.transaction(async t => { - const user = res.locals.oauth.token.User - // Don't send abuse notification if reporter is an admin/moderator - const skipNotification = user.hasRight(UserRight.MANAGE_ABUSES) - - const reporterAccount = await AccountModel.load(user.Account.id, t) - const predefinedReasons = body.predefinedReasons?.map(r => abusePredefinedReasonsMap[r]) - - const baseAbuse = { - reporterAccountId: reporterAccount.id, - reason: body.reason, - state: AbuseState.PENDING, - predefinedReasons - } - - if (body.video) { - return createVideoAbuse({ - baseAbuse, - videoInstance, - reporterAccount, - transaction: t, - startAt: body.video.startAt, - endAt: body.video.endAt, - skipNotification - }) - } - - if (body.comment) { - return createVideoCommentAbuse({ - baseAbuse, - commentInstance, - reporterAccount, - transaction: t, - skipNotification - }) - } - - // Account report - return createAccountAbuse({ - baseAbuse, - accountInstance, - reporterAccount, - transaction: t, - skipNotification - }) - }) - - return res.json({ abuse: { id } }) -} - -async function listAbuseMessages (req: express.Request, res: express.Response) { - const abuse = res.locals.abuse - - const resultList = await AbuseMessageModel.listForApi(abuse.id) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function addAbuseMessage (req: express.Request, res: express.Response) { - const abuse = res.locals.abuse - const user = res.locals.oauth.token.user - - const abuseMessage = await AbuseMessageModel.create({ - message: req.body.message, - byModerator: abuse.reporterAccountId !== user.Account.id, - accountId: user.Account.id, - abuseId: abuse.id - }) - - AbuseModel.loadFull(abuse.id) - .then(abuseFull => Notifier.Instance.notifyOnAbuseMessage(abuseFull, abuseMessage)) - .catch(err => logger.error('Cannot notify on new abuse message', { err })) - - return res.json({ - abuseMessage: { - id: abuseMessage.id - } - }) -} - -async function deleteAbuseMessage (req: express.Request, res: express.Response) { - const abuseMessage = res.locals.abuseMessage - - await sequelizeTypescript.transaction(t => { - return abuseMessage.destroy({ transaction: t }) - }) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} diff --git a/server/controllers/api/accounts.ts b/server/controllers/api/accounts.ts deleted file mode 100644 index 49cd7559a..000000000 --- a/server/controllers/api/accounts.ts +++ /dev/null @@ -1,266 +0,0 @@ -import express from 'express' -import { pickCommonVideoQuery } from '@server/helpers/query' -import { ActorFollowModel } from '@server/models/actor/actor-follow' -import { getServerActor } from '@server/models/application/application' -import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' -import { buildNSFWFilter, getCountVideos, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils' -import { getFormattedObjects } from '../../helpers/utils' -import { JobQueue } from '../../lib/job-queue' -import { Hooks } from '../../lib/plugins/hooks' -import { - apiRateLimiter, - asyncMiddleware, - authenticate, - commonVideosFiltersValidator, - optionalAuthenticate, - paginationValidator, - setDefaultPagination, - setDefaultSort, - setDefaultVideosSort, - videoPlaylistsSortValidator, - videoRatesSortValidator, - videoRatingValidator -} from '../../middlewares' -import { - accountNameWithHostGetValidator, - accountsFollowersSortValidator, - accountsSortValidator, - ensureAuthUserOwnsAccountValidator, - ensureCanManageChannelOrAccount, - videoChannelsSortValidator, - videoChannelStatsValidator, - videoChannelSyncsSortValidator, - videosSortValidator -} from '../../middlewares/validators' -import { commonVideoPlaylistFiltersValidator, videoPlaylistsSearchValidator } from '../../middlewares/validators/videos/video-playlists' -import { AccountModel } from '../../models/account/account' -import { AccountVideoRateModel } from '../../models/account/account-video-rate' -import { guessAdditionalAttributesFromQuery } from '../../models/video/formatter' -import { VideoModel } from '../../models/video/video' -import { VideoChannelModel } from '../../models/video/video-channel' -import { VideoPlaylistModel } from '../../models/video/video-playlist' - -const accountsRouter = express.Router() - -accountsRouter.use(apiRateLimiter) - -accountsRouter.get('/', - paginationValidator, - accountsSortValidator, - setDefaultSort, - setDefaultPagination, - asyncMiddleware(listAccounts) -) - -accountsRouter.get('/:accountName', - asyncMiddleware(accountNameWithHostGetValidator), - getAccount -) - -accountsRouter.get('/:accountName/videos', - asyncMiddleware(accountNameWithHostGetValidator), - paginationValidator, - videosSortValidator, - setDefaultVideosSort, - setDefaultPagination, - optionalAuthenticate, - commonVideosFiltersValidator, - asyncMiddleware(listAccountVideos) -) - -accountsRouter.get('/:accountName/video-channels', - asyncMiddleware(accountNameWithHostGetValidator), - videoChannelStatsValidator, - paginationValidator, - videoChannelsSortValidator, - setDefaultSort, - setDefaultPagination, - asyncMiddleware(listAccountChannels) -) - -accountsRouter.get('/:accountName/video-channel-syncs', - authenticate, - asyncMiddleware(accountNameWithHostGetValidator), - ensureCanManageChannelOrAccount, - paginationValidator, - videoChannelSyncsSortValidator, - setDefaultSort, - setDefaultPagination, - asyncMiddleware(listAccountChannelsSync) -) - -accountsRouter.get('/:accountName/video-playlists', - optionalAuthenticate, - asyncMiddleware(accountNameWithHostGetValidator), - paginationValidator, - videoPlaylistsSortValidator, - setDefaultSort, - setDefaultPagination, - commonVideoPlaylistFiltersValidator, - videoPlaylistsSearchValidator, - asyncMiddleware(listAccountPlaylists) -) - -accountsRouter.get('/:accountName/ratings', - authenticate, - asyncMiddleware(accountNameWithHostGetValidator), - ensureAuthUserOwnsAccountValidator, - paginationValidator, - videoRatesSortValidator, - setDefaultSort, - setDefaultPagination, - videoRatingValidator, - asyncMiddleware(listAccountRatings) -) - -accountsRouter.get('/:accountName/followers', - authenticate, - asyncMiddleware(accountNameWithHostGetValidator), - ensureAuthUserOwnsAccountValidator, - paginationValidator, - accountsFollowersSortValidator, - setDefaultSort, - setDefaultPagination, - asyncMiddleware(listAccountFollowers) -) - -// --------------------------------------------------------------------------- - -export { - accountsRouter -} - -// --------------------------------------------------------------------------- - -function getAccount (req: express.Request, res: express.Response) { - const account = res.locals.account - - if (account.isOutdated()) { - JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'actor', url: account.Actor.url } }) - } - - return res.json(account.toFormattedJSON()) -} - -async function listAccounts (req: express.Request, res: express.Response) { - const resultList = await AccountModel.listForApi(req.query.start, req.query.count, req.query.sort) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function listAccountChannels (req: express.Request, res: express.Response) { - const options = { - accountId: res.locals.account.id, - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - withStats: req.query.withStats, - search: req.query.search - } - - const resultList = await VideoChannelModel.listByAccountForAPI(options) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function listAccountChannelsSync (req: express.Request, res: express.Response) { - const options = { - accountId: res.locals.account.id, - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - search: req.query.search - } - - const resultList = await VideoChannelSyncModel.listByAccountForAPI(options) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function listAccountPlaylists (req: express.Request, res: express.Response) { - const serverActor = await getServerActor() - - // Allow users to see their private/unlisted video playlists - let listMyPlaylists = false - if (res.locals.oauth && res.locals.oauth.token.User.Account.id === res.locals.account.id) { - listMyPlaylists = true - } - - const resultList = await VideoPlaylistModel.listForApi({ - search: req.query.search, - followerActorId: serverActor.id, - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - accountId: res.locals.account.id, - listMyPlaylists, - type: req.query.playlistType - }) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function listAccountVideos (req: express.Request, res: express.Response) { - const serverActor = await getServerActor() - - const account = res.locals.account - - const displayOnlyForFollower = isUserAbleToSearchRemoteURI(res) - ? null - : { - actorId: serverActor.id, - orLocalVideos: true - } - - const countVideos = getCountVideos(req) - const query = pickCommonVideoQuery(req.query) - - const apiOptions = await Hooks.wrapObject({ - ...query, - - displayOnlyForFollower, - nsfw: buildNSFWFilter(res, query.nsfw), - accountId: account.id, - user: res.locals.oauth ? res.locals.oauth.token.User : undefined, - countVideos - }, 'filter:api.accounts.videos.list.params') - - const resultList = await Hooks.wrapPromiseFun( - VideoModel.listForApi, - apiOptions, - 'filter:api.accounts.videos.list.result' - ) - - return res.json(getFormattedObjects(resultList.data, resultList.total, guessAdditionalAttributesFromQuery(query))) -} - -async function listAccountRatings (req: express.Request, res: express.Response) { - const account = res.locals.account - - const resultList = await AccountVideoRateModel.listByAccountForApi({ - accountId: account.id, - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - type: req.query.rating - }) - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function listAccountFollowers (req: express.Request, res: express.Response) { - const account = res.locals.account - - const channels = await VideoChannelModel.listAllByAccount(account.id) - const actorIds = [ account.actorId ].concat(channels.map(c => c.actorId)) - - const resultList = await ActorFollowModel.listFollowersForApi({ - actorIds, - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - search: req.query.search, - state: 'accepted' - }) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} diff --git a/server/controllers/api/blocklist.ts b/server/controllers/api/blocklist.ts deleted file mode 100644 index dee12b108..000000000 --- a/server/controllers/api/blocklist.ts +++ /dev/null @@ -1,110 +0,0 @@ -import express from 'express' -import { handleToNameAndHost } from '@server/helpers/actors' -import { logger } from '@server/helpers/logger' -import { AccountBlocklistModel } from '@server/models/account/account-blocklist' -import { getServerActor } from '@server/models/application/application' -import { ServerBlocklistModel } from '@server/models/server/server-blocklist' -import { MActorAccountId, MUserAccountId } from '@server/types/models' -import { BlockStatus } from '@shared/models' -import { apiRateLimiter, asyncMiddleware, blocklistStatusValidator, optionalAuthenticate } from '../../middlewares' - -const blocklistRouter = express.Router() - -blocklistRouter.use(apiRateLimiter) - -blocklistRouter.get('/status', - optionalAuthenticate, - blocklistStatusValidator, - asyncMiddleware(getBlocklistStatus) -) - -// --------------------------------------------------------------------------- - -export { - blocklistRouter -} - -// --------------------------------------------------------------------------- - -async function getBlocklistStatus (req: express.Request, res: express.Response) { - const hosts = req.query.hosts as string[] - const accounts = req.query.accounts as string[] - const user = res.locals.oauth?.token.User - - const serverActor = await getServerActor() - - const byAccountIds = [ serverActor.Account.id ] - if (user) byAccountIds.push(user.Account.id) - - const status: BlockStatus = { - accounts: {}, - hosts: {} - } - - const baseOptions = { - byAccountIds, - user, - serverActor, - status - } - - await Promise.all([ - populateServerBlocklistStatus({ ...baseOptions, hosts }), - populateAccountBlocklistStatus({ ...baseOptions, accounts }) - ]) - - return res.json(status) -} - -async function populateServerBlocklistStatus (options: { - byAccountIds: number[] - user?: MUserAccountId - serverActor: MActorAccountId - hosts: string[] - status: BlockStatus -}) { - const { byAccountIds, user, serverActor, hosts, status } = options - - if (!hosts || hosts.length === 0) return - - const serverBlocklistStatus = await ServerBlocklistModel.getBlockStatus(byAccountIds, hosts) - - logger.debug('Got server blocklist status.', { serverBlocklistStatus, byAccountIds, hosts }) - - for (const host of hosts) { - const block = serverBlocklistStatus.find(b => b.host === host) - - status.hosts[host] = getStatus(block, serverActor, user) - } -} - -async function populateAccountBlocklistStatus (options: { - byAccountIds: number[] - user?: MUserAccountId - serverActor: MActorAccountId - accounts: string[] - status: BlockStatus -}) { - const { byAccountIds, user, serverActor, accounts, status } = options - - if (!accounts || accounts.length === 0) return - - const accountBlocklistStatus = await AccountBlocklistModel.getBlockStatus(byAccountIds, accounts) - - logger.debug('Got account blocklist status.', { accountBlocklistStatus, byAccountIds, accounts }) - - for (const account of accounts) { - const sanitizedHandle = handleToNameAndHost(account) - - const block = accountBlocklistStatus.find(b => b.name === sanitizedHandle.name && b.host === sanitizedHandle.host) - - status.accounts[sanitizedHandle.handle] = getStatus(block, serverActor, user) - } -} - -function getStatus (block: { accountId: number }, serverActor: MActorAccountId, user?: MUserAccountId) { - return { - blockedByServer: !!(block && block.accountId === serverActor.Account.id), - blockedByUser: !!(block && user && block.accountId === user.Account.id) - } -} diff --git a/server/controllers/api/bulk.ts b/server/controllers/api/bulk.ts deleted file mode 100644 index c41c7d378..000000000 --- a/server/controllers/api/bulk.ts +++ /dev/null @@ -1,44 +0,0 @@ -import express from 'express' -import { removeComment } from '@server/lib/video-comment' -import { bulkRemoveCommentsOfValidator } from '@server/middlewares/validators/bulk' -import { VideoCommentModel } from '@server/models/video/video-comment' -import { HttpStatusCode } from '@shared/models' -import { BulkRemoveCommentsOfBody } from '@shared/models/bulk/bulk-remove-comments-of-body.model' -import { apiRateLimiter, asyncMiddleware, authenticate } from '../../middlewares' - -const bulkRouter = express.Router() - -bulkRouter.use(apiRateLimiter) - -bulkRouter.post('/remove-comments-of', - authenticate, - asyncMiddleware(bulkRemoveCommentsOfValidator), - asyncMiddleware(bulkRemoveCommentsOf) -) - -// --------------------------------------------------------------------------- - -export { - bulkRouter -} - -// --------------------------------------------------------------------------- - -async function bulkRemoveCommentsOf (req: express.Request, res: express.Response) { - const account = res.locals.account - const body = req.body as BulkRemoveCommentsOfBody - const user = res.locals.oauth.token.User - - const filter = body.scope === 'my-videos' - ? { onVideosOfAccount: user.Account } - : {} - - const comments = await VideoCommentModel.listForBulkDelete(account, filter) - - // Don't wait result - res.status(HttpStatusCode.NO_CONTENT_204).end() - - for (const comment of comments) { - await removeComment(comment, req, res) - } -} diff --git a/server/controllers/api/config.ts b/server/controllers/api/config.ts deleted file mode 100644 index c5c4c8a74..000000000 --- a/server/controllers/api/config.ts +++ /dev/null @@ -1,377 +0,0 @@ -import express from 'express' -import { remove, writeJSON } from 'fs-extra' -import { snakeCase } from 'lodash' -import validator from 'validator' -import { ServerConfigManager } from '@server/lib/server-config-manager' -import { About, CustomConfig, UserRight } from '@shared/models' -import { auditLoggerFactory, CustomConfigAuditView, getAuditIdFromRes } from '../../helpers/audit-logger' -import { objectConverter } from '../../helpers/core-utils' -import { CONFIG, reloadConfig } from '../../initializers/config' -import { ClientHtml } from '../../lib/client-html' -import { apiRateLimiter, asyncMiddleware, authenticate, ensureUserHasRight, openapiOperationDoc } from '../../middlewares' -import { customConfigUpdateValidator, ensureConfigIsEditable } from '../../middlewares/validators/config' - -const configRouter = express.Router() - -configRouter.use(apiRateLimiter) - -const auditLogger = auditLoggerFactory('config') - -configRouter.get('/', - openapiOperationDoc({ operationId: 'getConfig' }), - asyncMiddleware(getConfig) -) - -configRouter.get('/about', - openapiOperationDoc({ operationId: 'getAbout' }), - getAbout -) - -configRouter.get('/custom', - openapiOperationDoc({ operationId: 'getCustomConfig' }), - authenticate, - ensureUserHasRight(UserRight.MANAGE_CONFIGURATION), - getCustomConfig -) - -configRouter.put('/custom', - openapiOperationDoc({ operationId: 'putCustomConfig' }), - authenticate, - ensureUserHasRight(UserRight.MANAGE_CONFIGURATION), - ensureConfigIsEditable, - customConfigUpdateValidator, - asyncMiddleware(updateCustomConfig) -) - -configRouter.delete('/custom', - openapiOperationDoc({ operationId: 'delCustomConfig' }), - authenticate, - ensureUserHasRight(UserRight.MANAGE_CONFIGURATION), - ensureConfigIsEditable, - asyncMiddleware(deleteCustomConfig) -) - -async function getConfig (req: express.Request, res: express.Response) { - const json = await ServerConfigManager.Instance.getServerConfig(req.ip) - - return res.json(json) -} - -function getAbout (req: express.Request, res: express.Response) { - const about: About = { - instance: { - name: CONFIG.INSTANCE.NAME, - shortDescription: CONFIG.INSTANCE.SHORT_DESCRIPTION, - description: CONFIG.INSTANCE.DESCRIPTION, - terms: CONFIG.INSTANCE.TERMS, - codeOfConduct: CONFIG.INSTANCE.CODE_OF_CONDUCT, - - hardwareInformation: CONFIG.INSTANCE.HARDWARE_INFORMATION, - - creationReason: CONFIG.INSTANCE.CREATION_REASON, - moderationInformation: CONFIG.INSTANCE.MODERATION_INFORMATION, - administrator: CONFIG.INSTANCE.ADMINISTRATOR, - maintenanceLifetime: CONFIG.INSTANCE.MAINTENANCE_LIFETIME, - businessModel: CONFIG.INSTANCE.BUSINESS_MODEL, - - languages: CONFIG.INSTANCE.LANGUAGES, - categories: CONFIG.INSTANCE.CATEGORIES - } - } - - return res.json(about) -} - -function getCustomConfig (req: express.Request, res: express.Response) { - const data = customConfig() - - return res.json(data) -} - -async function deleteCustomConfig (req: express.Request, res: express.Response) { - await remove(CONFIG.CUSTOM_FILE) - - auditLogger.delete(getAuditIdFromRes(res), new CustomConfigAuditView(customConfig())) - - reloadConfig() - ClientHtml.invalidCache() - - const data = customConfig() - - return res.json(data) -} - -async function updateCustomConfig (req: express.Request, res: express.Response) { - const oldCustomConfigAuditKeys = new CustomConfigAuditView(customConfig()) - - // camelCase to snake_case key + Force number conversion - const toUpdateJSON = convertCustomConfigBody(req.body) - - await writeJSON(CONFIG.CUSTOM_FILE, toUpdateJSON, { spaces: 2 }) - - reloadConfig() - ClientHtml.invalidCache() - - const data = customConfig() - - auditLogger.update( - getAuditIdFromRes(res), - new CustomConfigAuditView(data), - oldCustomConfigAuditKeys - ) - - return res.json(data) -} - -// --------------------------------------------------------------------------- - -export { - configRouter -} - -// --------------------------------------------------------------------------- - -function customConfig (): CustomConfig { - return { - instance: { - name: CONFIG.INSTANCE.NAME, - shortDescription: CONFIG.INSTANCE.SHORT_DESCRIPTION, - description: CONFIG.INSTANCE.DESCRIPTION, - terms: CONFIG.INSTANCE.TERMS, - codeOfConduct: CONFIG.INSTANCE.CODE_OF_CONDUCT, - - creationReason: CONFIG.INSTANCE.CREATION_REASON, - moderationInformation: CONFIG.INSTANCE.MODERATION_INFORMATION, - administrator: CONFIG.INSTANCE.ADMINISTRATOR, - maintenanceLifetime: CONFIG.INSTANCE.MAINTENANCE_LIFETIME, - businessModel: CONFIG.INSTANCE.BUSINESS_MODEL, - hardwareInformation: CONFIG.INSTANCE.HARDWARE_INFORMATION, - - languages: CONFIG.INSTANCE.LANGUAGES, - categories: CONFIG.INSTANCE.CATEGORIES, - - isNSFW: CONFIG.INSTANCE.IS_NSFW, - defaultNSFWPolicy: CONFIG.INSTANCE.DEFAULT_NSFW_POLICY, - - defaultClientRoute: CONFIG.INSTANCE.DEFAULT_CLIENT_ROUTE, - - customizations: { - css: CONFIG.INSTANCE.CUSTOMIZATIONS.CSS, - javascript: CONFIG.INSTANCE.CUSTOMIZATIONS.JAVASCRIPT - } - }, - theme: { - default: CONFIG.THEME.DEFAULT - }, - services: { - twitter: { - username: CONFIG.SERVICES.TWITTER.USERNAME, - whitelisted: CONFIG.SERVICES.TWITTER.WHITELISTED - } - }, - client: { - videos: { - miniature: { - preferAuthorDisplayName: CONFIG.CLIENT.VIDEOS.MINIATURE.PREFER_AUTHOR_DISPLAY_NAME - } - }, - menu: { - login: { - redirectOnSingleExternalAuth: CONFIG.CLIENT.MENU.LOGIN.REDIRECT_ON_SINGLE_EXTERNAL_AUTH - } - } - }, - cache: { - previews: { - size: CONFIG.CACHE.PREVIEWS.SIZE - }, - captions: { - size: CONFIG.CACHE.VIDEO_CAPTIONS.SIZE - }, - torrents: { - size: CONFIG.CACHE.TORRENTS.SIZE - }, - storyboards: { - size: CONFIG.CACHE.STORYBOARDS.SIZE - } - }, - signup: { - enabled: CONFIG.SIGNUP.ENABLED, - limit: CONFIG.SIGNUP.LIMIT, - requiresApproval: CONFIG.SIGNUP.REQUIRES_APPROVAL, - requiresEmailVerification: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION, - minimumAge: CONFIG.SIGNUP.MINIMUM_AGE - }, - admin: { - email: CONFIG.ADMIN.EMAIL - }, - contactForm: { - enabled: CONFIG.CONTACT_FORM.ENABLED - }, - user: { - history: { - videos: { - enabled: CONFIG.USER.HISTORY.VIDEOS.ENABLED - } - }, - videoQuota: CONFIG.USER.VIDEO_QUOTA, - videoQuotaDaily: CONFIG.USER.VIDEO_QUOTA_DAILY - }, - videoChannels: { - maxPerUser: CONFIG.VIDEO_CHANNELS.MAX_PER_USER - }, - transcoding: { - enabled: CONFIG.TRANSCODING.ENABLED, - remoteRunners: { - enabled: CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED - }, - allowAdditionalExtensions: CONFIG.TRANSCODING.ALLOW_ADDITIONAL_EXTENSIONS, - allowAudioFiles: CONFIG.TRANSCODING.ALLOW_AUDIO_FILES, - threads: CONFIG.TRANSCODING.THREADS, - concurrency: CONFIG.TRANSCODING.CONCURRENCY, - profile: CONFIG.TRANSCODING.PROFILE, - resolutions: { - '0p': CONFIG.TRANSCODING.RESOLUTIONS['0p'], - '144p': CONFIG.TRANSCODING.RESOLUTIONS['144p'], - '240p': CONFIG.TRANSCODING.RESOLUTIONS['240p'], - '360p': CONFIG.TRANSCODING.RESOLUTIONS['360p'], - '480p': CONFIG.TRANSCODING.RESOLUTIONS['480p'], - '720p': CONFIG.TRANSCODING.RESOLUTIONS['720p'], - '1080p': CONFIG.TRANSCODING.RESOLUTIONS['1080p'], - '1440p': CONFIG.TRANSCODING.RESOLUTIONS['1440p'], - '2160p': CONFIG.TRANSCODING.RESOLUTIONS['2160p'] - }, - alwaysTranscodeOriginalResolution: CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION, - webVideos: { - enabled: CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED - }, - hls: { - enabled: CONFIG.TRANSCODING.HLS.ENABLED - } - }, - live: { - enabled: CONFIG.LIVE.ENABLED, - allowReplay: CONFIG.LIVE.ALLOW_REPLAY, - latencySetting: { - enabled: CONFIG.LIVE.LATENCY_SETTING.ENABLED - }, - maxDuration: CONFIG.LIVE.MAX_DURATION, - maxInstanceLives: CONFIG.LIVE.MAX_INSTANCE_LIVES, - maxUserLives: CONFIG.LIVE.MAX_USER_LIVES, - transcoding: { - enabled: CONFIG.LIVE.TRANSCODING.ENABLED, - remoteRunners: { - enabled: CONFIG.LIVE.TRANSCODING.REMOTE_RUNNERS.ENABLED - }, - threads: CONFIG.LIVE.TRANSCODING.THREADS, - profile: CONFIG.LIVE.TRANSCODING.PROFILE, - resolutions: { - '144p': CONFIG.LIVE.TRANSCODING.RESOLUTIONS['144p'], - '240p': CONFIG.LIVE.TRANSCODING.RESOLUTIONS['240p'], - '360p': CONFIG.LIVE.TRANSCODING.RESOLUTIONS['360p'], - '480p': CONFIG.LIVE.TRANSCODING.RESOLUTIONS['480p'], - '720p': CONFIG.LIVE.TRANSCODING.RESOLUTIONS['720p'], - '1080p': CONFIG.LIVE.TRANSCODING.RESOLUTIONS['1080p'], - '1440p': CONFIG.LIVE.TRANSCODING.RESOLUTIONS['1440p'], - '2160p': CONFIG.LIVE.TRANSCODING.RESOLUTIONS['2160p'] - }, - alwaysTranscodeOriginalResolution: CONFIG.LIVE.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION - } - }, - videoStudio: { - enabled: CONFIG.VIDEO_STUDIO.ENABLED, - remoteRunners: { - enabled: CONFIG.VIDEO_STUDIO.REMOTE_RUNNERS.ENABLED - } - }, - videoFile: { - update: { - enabled: CONFIG.VIDEO_FILE.UPDATE.ENABLED - } - }, - import: { - videos: { - concurrency: CONFIG.IMPORT.VIDEOS.CONCURRENCY, - http: { - enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED - }, - torrent: { - enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED - } - }, - videoChannelSynchronization: { - enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED, - maxPerUser: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER - } - }, - trending: { - videos: { - algorithms: { - enabled: CONFIG.TRENDING.VIDEOS.ALGORITHMS.ENABLED, - default: CONFIG.TRENDING.VIDEOS.ALGORITHMS.DEFAULT - } - } - }, - autoBlacklist: { - videos: { - ofUsers: { - enabled: CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED - } - } - }, - followers: { - instance: { - enabled: CONFIG.FOLLOWERS.INSTANCE.ENABLED, - manualApproval: CONFIG.FOLLOWERS.INSTANCE.MANUAL_APPROVAL - } - }, - followings: { - instance: { - autoFollowBack: { - enabled: CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_BACK.ENABLED - }, - - autoFollowIndex: { - enabled: CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.ENABLED, - indexUrl: CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.INDEX_URL - } - } - }, - broadcastMessage: { - enabled: CONFIG.BROADCAST_MESSAGE.ENABLED, - message: CONFIG.BROADCAST_MESSAGE.MESSAGE, - level: CONFIG.BROADCAST_MESSAGE.LEVEL, - dismissable: CONFIG.BROADCAST_MESSAGE.DISMISSABLE - }, - search: { - remoteUri: { - users: CONFIG.SEARCH.REMOTE_URI.USERS, - anonymous: CONFIG.SEARCH.REMOTE_URI.ANONYMOUS - }, - searchIndex: { - enabled: CONFIG.SEARCH.SEARCH_INDEX.ENABLED, - url: CONFIG.SEARCH.SEARCH_INDEX.URL, - disableLocalSearch: CONFIG.SEARCH.SEARCH_INDEX.DISABLE_LOCAL_SEARCH, - isDefaultSearch: CONFIG.SEARCH.SEARCH_INDEX.IS_DEFAULT_SEARCH - } - } - } -} - -function convertCustomConfigBody (body: CustomConfig) { - function keyConverter (k: string) { - // Transcoding resolutions exception - if (/^\d{3,4}p$/.exec(k)) return k - if (k === '0p') return k - - return snakeCase(k) - } - - function valueConverter (v: any) { - if (validator.isNumeric(v + '')) return parseInt('' + v, 10) - - return v - } - - return objectConverter(body, keyConverter, valueConverter) -} diff --git a/server/controllers/api/custom-page.ts b/server/controllers/api/custom-page.ts deleted file mode 100644 index f4e1a0e79..000000000 --- a/server/controllers/api/custom-page.ts +++ /dev/null @@ -1,48 +0,0 @@ -import express from 'express' -import { ServerConfigManager } from '@server/lib/server-config-manager' -import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' -import { HttpStatusCode, UserRight } from '@shared/models' -import { apiRateLimiter, asyncMiddleware, authenticate, ensureUserHasRight } from '../../middlewares' - -const customPageRouter = express.Router() - -customPageRouter.use(apiRateLimiter) - -customPageRouter.get('/homepage/instance', - asyncMiddleware(getInstanceHomepage) -) - -customPageRouter.put('/homepage/instance', - authenticate, - ensureUserHasRight(UserRight.MANAGE_INSTANCE_CUSTOM_PAGE), - asyncMiddleware(updateInstanceHomepage) -) - -// --------------------------------------------------------------------------- - -export { - customPageRouter -} - -// --------------------------------------------------------------------------- - -async function getInstanceHomepage (req: express.Request, res: express.Response) { - const page = await ActorCustomPageModel.loadInstanceHomepage() - if (!page) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Instance homepage could not be found' - }) - } - - return res.json(page.toFormattedJSON()) -} - -async function updateInstanceHomepage (req: express.Request, res: express.Response) { - const content = req.body.content - - await ActorCustomPageModel.updateInstanceHomepage(content) - ServerConfigManager.Instance.updateHomepageState(content) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} diff --git a/server/controllers/api/index.ts b/server/controllers/api/index.ts deleted file mode 100644 index 38bd135d0..000000000 --- a/server/controllers/api/index.ts +++ /dev/null @@ -1,73 +0,0 @@ -import cors from 'cors' -import express from 'express' -import { logger } from '@server/helpers/logger' -import { HttpStatusCode } from '../../../shared/models' -import { abuseRouter } from './abuse' -import { accountsRouter } from './accounts' -import { blocklistRouter } from './blocklist' -import { bulkRouter } from './bulk' -import { configRouter } from './config' -import { customPageRouter } from './custom-page' -import { jobsRouter } from './jobs' -import { metricsRouter } from './metrics' -import { oauthClientsRouter } from './oauth-clients' -import { overviewsRouter } from './overviews' -import { pluginRouter } from './plugins' -import { runnersRouter } from './runners' -import { searchRouter } from './search' -import { serverRouter } from './server' -import { usersRouter } from './users' -import { videoChannelRouter } from './video-channel' -import { videoChannelSyncRouter } from './video-channel-sync' -import { videoPlaylistRouter } from './video-playlist' -import { videosRouter } from './videos' - -const apiRouter = express.Router() - -apiRouter.use(cors({ - origin: '*', - exposedHeaders: 'Retry-After', - credentials: true -})) - -apiRouter.use('/server', serverRouter) -apiRouter.use('/abuses', abuseRouter) -apiRouter.use('/bulk', bulkRouter) -apiRouter.use('/oauth-clients', oauthClientsRouter) -apiRouter.use('/config', configRouter) -apiRouter.use('/users', usersRouter) -apiRouter.use('/accounts', accountsRouter) -apiRouter.use('/video-channels', videoChannelRouter) -apiRouter.use('/video-channel-syncs', videoChannelSyncRouter) -apiRouter.use('/video-playlists', videoPlaylistRouter) -apiRouter.use('/videos', videosRouter) -apiRouter.use('/jobs', jobsRouter) -apiRouter.use('/metrics', metricsRouter) -apiRouter.use('/search', searchRouter) -apiRouter.use('/overviews', overviewsRouter) -apiRouter.use('/plugins', pluginRouter) -apiRouter.use('/custom-pages', customPageRouter) -apiRouter.use('/blocklist', blocklistRouter) -apiRouter.use('/runners', runnersRouter) - -// apiRouter.use(apiRateLimiter) -apiRouter.use('/ping', pong) -apiRouter.use('/*', badRequest) - -// --------------------------------------------------------------------------- - -export { apiRouter } - -// --------------------------------------------------------------------------- - -function pong (req: express.Request, res: express.Response) { - return res.send('pong').status(HttpStatusCode.OK_200).end() -} - -function badRequest (req: express.Request, res: express.Response) { - logger.debug(`API express handler not found: bad PeerTube request for ${req.method} - ${req.originalUrl}`) - - return res.type('json') - .status(HttpStatusCode.BAD_REQUEST_400) - .end() -} diff --git a/server/controllers/api/jobs.ts b/server/controllers/api/jobs.ts deleted file mode 100644 index c701bc970..000000000 --- a/server/controllers/api/jobs.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { Job as BullJob } from 'bullmq' -import express from 'express' -import { HttpStatusCode, Job, JobState, JobType, ResultList, UserRight } from '@shared/models' -import { isArray } from '../../helpers/custom-validators/misc' -import { JobQueue } from '../../lib/job-queue' -import { - apiRateLimiter, - asyncMiddleware, - authenticate, - ensureUserHasRight, - jobsSortValidator, - openapiOperationDoc, - paginationValidatorBuilder, - setDefaultPagination, - setDefaultSort -} from '../../middlewares' -import { listJobsValidator } from '../../middlewares/validators/jobs' - -const jobsRouter = express.Router() - -jobsRouter.use(apiRateLimiter) - -jobsRouter.post('/pause', - authenticate, - ensureUserHasRight(UserRight.MANAGE_JOBS), - asyncMiddleware(pauseJobQueue) -) - -jobsRouter.post('/resume', - authenticate, - ensureUserHasRight(UserRight.MANAGE_JOBS), - resumeJobQueue -) - -jobsRouter.get('/:state?', - openapiOperationDoc({ operationId: 'getJobs' }), - authenticate, - ensureUserHasRight(UserRight.MANAGE_JOBS), - paginationValidatorBuilder([ 'jobs' ]), - jobsSortValidator, - setDefaultSort, - setDefaultPagination, - listJobsValidator, - asyncMiddleware(listJobs) -) - -// --------------------------------------------------------------------------- - -export { - jobsRouter -} - -// --------------------------------------------------------------------------- - -async function pauseJobQueue (req: express.Request, res: express.Response) { - await JobQueue.Instance.pause() - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -function resumeJobQueue (req: express.Request, res: express.Response) { - JobQueue.Instance.resume() - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -async function listJobs (req: express.Request, res: express.Response) { - const state = req.params.state as JobState - const asc = req.query.sort === 'createdAt' - const jobType = req.query.jobType - - const jobs = await JobQueue.Instance.listForApi({ - state, - start: req.query.start, - count: req.query.count, - asc, - jobType - }) - const total = await JobQueue.Instance.count(state, jobType) - - const result: ResultList = { - total, - data: await Promise.all(jobs.map(j => formatJob(j, state))) - } - - return res.json(result) -} - -async function formatJob (job: BullJob, state?: JobState): Promise { - const error = isArray(job.stacktrace) && job.stacktrace.length !== 0 - ? job.stacktrace[0] - : null - - return { - id: job.id, - state: state || await job.getState(), - type: job.queueName as JobType, - data: job.data, - parent: job.parent - ? { id: job.parent.id } - : undefined, - progress: job.progress as number, - priority: job.opts.priority, - error, - createdAt: new Date(job.timestamp), - finishedOn: new Date(job.finishedOn), - processedOn: new Date(job.processedOn) - } -} diff --git a/server/controllers/api/metrics.ts b/server/controllers/api/metrics.ts deleted file mode 100644 index 909963fa7..000000000 --- a/server/controllers/api/metrics.ts +++ /dev/null @@ -1,34 +0,0 @@ -import express from 'express' -import { CONFIG } from '@server/initializers/config' -import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics' -import { HttpStatusCode, PlaybackMetricCreate } from '@shared/models' -import { addPlaybackMetricValidator, apiRateLimiter, asyncMiddleware } from '../../middlewares' - -const metricsRouter = express.Router() - -metricsRouter.use(apiRateLimiter) - -metricsRouter.post('/playback', - asyncMiddleware(addPlaybackMetricValidator), - addPlaybackMetric -) - -// --------------------------------------------------------------------------- - -export { - metricsRouter -} - -// --------------------------------------------------------------------------- - -function addPlaybackMetric (req: express.Request, res: express.Response) { - if (!CONFIG.OPEN_TELEMETRY.METRICS.ENABLED) { - return res.sendStatus(HttpStatusCode.FORBIDDEN_403) - } - - const body: PlaybackMetricCreate = req.body - - OpenTelemetryMetrics.Instance.observePlaybackMetric(res.locals.onlyImmutableVideo, body) - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} diff --git a/server/controllers/api/oauth-clients.ts b/server/controllers/api/oauth-clients.ts deleted file mode 100644 index 1899dbb02..000000000 --- a/server/controllers/api/oauth-clients.ts +++ /dev/null @@ -1,54 +0,0 @@ -import express from 'express' -import { isTestOrDevInstance } from '@server/helpers/core-utils' -import { OAuthClientModel } from '@server/models/oauth/oauth-client' -import { HttpStatusCode, OAuthClientLocal } from '@shared/models' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { apiRateLimiter, asyncMiddleware, openapiOperationDoc } from '../../middlewares' - -const oauthClientsRouter = express.Router() - -oauthClientsRouter.use(apiRateLimiter) - -oauthClientsRouter.get('/local', - openapiOperationDoc({ operationId: 'getOAuthClient' }), - asyncMiddleware(getLocalClient) -) - -// Get the client credentials for the PeerTube front end -async function getLocalClient (req: express.Request, res: express.Response, next: express.NextFunction) { - const serverHostname = CONFIG.WEBSERVER.HOSTNAME - const serverPort = CONFIG.WEBSERVER.PORT - let headerHostShouldBe = serverHostname - if (serverPort !== 80 && serverPort !== 443) { - headerHostShouldBe += ':' + serverPort - } - - // Don't make this check if this is a test instance - if (!isTestOrDevInstance() && req.get('host') !== headerHostShouldBe) { - logger.info( - 'Getting client tokens for host %s is forbidden (expected %s).', req.get('host'), headerHostShouldBe, - { webserverConfig: CONFIG.WEBSERVER } - ) - - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: `Getting client tokens for host ${req.get('host')} is forbidden` - }) - } - - const client = await OAuthClientModel.loadFirstClient() - if (!client) throw new Error('No client available.') - - const json: OAuthClientLocal = { - client_id: client.clientId, - client_secret: client.clientSecret - } - return res.json(json) -} - -// --------------------------------------------------------------------------- - -export { - oauthClientsRouter -} diff --git a/server/controllers/api/overviews.ts b/server/controllers/api/overviews.ts deleted file mode 100644 index fc616281e..000000000 --- a/server/controllers/api/overviews.ts +++ /dev/null @@ -1,139 +0,0 @@ -import express from 'express' -import memoizee from 'memoizee' -import { logger } from '@server/helpers/logger' -import { Hooks } from '@server/lib/plugins/hooks' -import { getServerActor } from '@server/models/application/application' -import { VideoModel } from '@server/models/video/video' -import { CategoryOverview, ChannelOverview, TagOverview, VideosOverview } from '../../../shared/models/overviews' -import { buildNSFWFilter } from '../../helpers/express-utils' -import { MEMOIZE_TTL, OVERVIEWS } from '../../initializers/constants' -import { apiRateLimiter, asyncMiddleware, optionalAuthenticate, videosOverviewValidator } from '../../middlewares' -import { TagModel } from '../../models/video/tag' - -const overviewsRouter = express.Router() - -overviewsRouter.use(apiRateLimiter) - -overviewsRouter.get('/videos', - videosOverviewValidator, - optionalAuthenticate, - asyncMiddleware(getVideosOverview) -) - -// --------------------------------------------------------------------------- - -export { overviewsRouter } - -// --------------------------------------------------------------------------- - -const buildSamples = memoizee(async function () { - const [ categories, channels, tags ] = await Promise.all([ - VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT), - VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT), - TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT) - ]) - - const result = { categories, channels, tags } - - logger.debug('Building samples for overview endpoint.', { result }) - - return result -}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE }) - -// This endpoint could be quite long, but we cache it -async function getVideosOverview (req: express.Request, res: express.Response) { - const attributes = await buildSamples() - - const page = req.query.page || 1 - const index = page - 1 - - const categories: CategoryOverview[] = [] - const channels: ChannelOverview[] = [] - const tags: TagOverview[] = [] - - await Promise.all([ - getVideosByCategory(attributes.categories, index, res, categories), - getVideosByChannel(attributes.channels, index, res, channels), - getVideosByTag(attributes.tags, index, res, tags) - ]) - - const result: VideosOverview = { - categories, - channels, - tags - } - - return res.json(result) -} - -async function getVideosByTag (tagsSample: string[], index: number, res: express.Response, acc: TagOverview[]) { - if (tagsSample.length <= index) return - - const tag = tagsSample[index] - const videos = await getVideos(res, { tagsOneOf: [ tag ] }) - - if (videos.length === 0) return - - acc.push({ - tag, - videos - }) -} - -async function getVideosByCategory (categoriesSample: number[], index: number, res: express.Response, acc: CategoryOverview[]) { - if (categoriesSample.length <= index) return - - const category = categoriesSample[index] - const videos = await getVideos(res, { categoryOneOf: [ category ] }) - - if (videos.length === 0) return - - acc.push({ - category: videos[0].category, - videos - }) -} - -async function getVideosByChannel (channelsSample: number[], index: number, res: express.Response, acc: ChannelOverview[]) { - if (channelsSample.length <= index) return - - const channelId = channelsSample[index] - const videos = await getVideos(res, { videoChannelId: channelId }) - - if (videos.length === 0) return - - acc.push({ - channel: videos[0].channel, - videos - }) -} - -async function getVideos ( - res: express.Response, - where: { videoChannelId?: number, tagsOneOf?: string[], categoryOneOf?: number[] } -) { - const serverActor = await getServerActor() - - const query = await Hooks.wrapObject({ - start: 0, - count: 12, - sort: '-createdAt', - displayOnlyForFollower: { - actorId: serverActor.id, - orLocalVideos: true - }, - nsfw: buildNSFWFilter(res), - user: res.locals.oauth ? res.locals.oauth.token.User : undefined, - countVideos: false, - - ...where - }, 'filter:api.overviews.videos.list.params') - - const { data } = await Hooks.wrapPromiseFun( - VideoModel.listForApi, - query, - 'filter:api.overviews.videos.list.result' - ) - - return data.map(d => d.toFormattedJSON()) -} diff --git a/server/controllers/api/plugins.ts b/server/controllers/api/plugins.ts deleted file mode 100644 index 337b72b2f..000000000 --- a/server/controllers/api/plugins.ts +++ /dev/null @@ -1,230 +0,0 @@ -import express from 'express' -import { logger } from '@server/helpers/logger' -import { getFormattedObjects } from '@server/helpers/utils' -import { listAvailablePluginsFromIndex } from '@server/lib/plugins/plugin-index' -import { PluginManager } from '@server/lib/plugins/plugin-manager' -import { - apiRateLimiter, - asyncMiddleware, - authenticate, - availablePluginsSortValidator, - ensureUserHasRight, - openapiOperationDoc, - paginationValidator, - pluginsSortValidator, - setDefaultPagination, - setDefaultSort -} from '@server/middlewares' -import { - existingPluginValidator, - installOrUpdatePluginValidator, - listAvailablePluginsValidator, - listPluginsValidator, - uninstallPluginValidator, - updatePluginSettingsValidator -} from '@server/middlewares/validators/plugins' -import { PluginModel } from '@server/models/server/plugin' -import { - HttpStatusCode, - InstallOrUpdatePlugin, - ManagePlugin, - PeertubePluginIndexList, - PublicServerSetting, - RegisteredServerSettings, - UserRight -} from '@shared/models' - -const pluginRouter = express.Router() - -pluginRouter.use(apiRateLimiter) - -pluginRouter.get('/available', - openapiOperationDoc({ operationId: 'getAvailablePlugins' }), - authenticate, - ensureUserHasRight(UserRight.MANAGE_PLUGINS), - listAvailablePluginsValidator, - paginationValidator, - availablePluginsSortValidator, - setDefaultSort, - setDefaultPagination, - asyncMiddleware(listAvailablePlugins) -) - -pluginRouter.get('/', - openapiOperationDoc({ operationId: 'getPlugins' }), - authenticate, - ensureUserHasRight(UserRight.MANAGE_PLUGINS), - listPluginsValidator, - paginationValidator, - pluginsSortValidator, - setDefaultSort, - setDefaultPagination, - asyncMiddleware(listPlugins) -) - -pluginRouter.get('/:npmName/registered-settings', - authenticate, - ensureUserHasRight(UserRight.MANAGE_PLUGINS), - asyncMiddleware(existingPluginValidator), - getPluginRegisteredSettings -) - -pluginRouter.get('/:npmName/public-settings', - asyncMiddleware(existingPluginValidator), - getPublicPluginSettings -) - -pluginRouter.put('/:npmName/settings', - authenticate, - ensureUserHasRight(UserRight.MANAGE_PLUGINS), - updatePluginSettingsValidator, - asyncMiddleware(existingPluginValidator), - asyncMiddleware(updatePluginSettings) -) - -pluginRouter.get('/:npmName', - authenticate, - ensureUserHasRight(UserRight.MANAGE_PLUGINS), - asyncMiddleware(existingPluginValidator), - getPlugin -) - -pluginRouter.post('/install', - openapiOperationDoc({ operationId: 'addPlugin' }), - authenticate, - ensureUserHasRight(UserRight.MANAGE_PLUGINS), - installOrUpdatePluginValidator, - asyncMiddleware(installPlugin) -) - -pluginRouter.post('/update', - openapiOperationDoc({ operationId: 'updatePlugin' }), - authenticate, - ensureUserHasRight(UserRight.MANAGE_PLUGINS), - installOrUpdatePluginValidator, - asyncMiddleware(updatePlugin) -) - -pluginRouter.post('/uninstall', - openapiOperationDoc({ operationId: 'uninstallPlugin' }), - authenticate, - ensureUserHasRight(UserRight.MANAGE_PLUGINS), - uninstallPluginValidator, - asyncMiddleware(uninstallPlugin) -) - -// --------------------------------------------------------------------------- - -export { - pluginRouter -} - -// --------------------------------------------------------------------------- - -async function listPlugins (req: express.Request, res: express.Response) { - const pluginType = req.query.pluginType - const uninstalled = req.query.uninstalled - - const resultList = await PluginModel.listForApi({ - pluginType, - uninstalled, - start: req.query.start, - count: req.query.count, - sort: req.query.sort - }) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -function getPlugin (req: express.Request, res: express.Response) { - const plugin = res.locals.plugin - - return res.json(plugin.toFormattedJSON()) -} - -async function installPlugin (req: express.Request, res: express.Response) { - const body: InstallOrUpdatePlugin = req.body - - const fromDisk = !!body.path - const toInstall = body.npmName || body.path - - const pluginVersion = body.pluginVersion && body.npmName - ? body.pluginVersion - : undefined - - try { - const plugin = await PluginManager.Instance.install({ toInstall, version: pluginVersion, fromDisk }) - - return res.json(plugin.toFormattedJSON()) - } catch (err) { - logger.warn('Cannot install plugin %s.', toInstall, { err }) - return res.fail({ message: 'Cannot install plugin ' + toInstall }) - } -} - -async function updatePlugin (req: express.Request, res: express.Response) { - const body: InstallOrUpdatePlugin = req.body - - const fromDisk = !!body.path - const toUpdate = body.npmName || body.path - try { - const plugin = await PluginManager.Instance.update(toUpdate, fromDisk) - - return res.json(plugin.toFormattedJSON()) - } catch (err) { - logger.warn('Cannot update plugin %s.', toUpdate, { err }) - return res.fail({ message: 'Cannot update plugin ' + toUpdate }) - } -} - -async function uninstallPlugin (req: express.Request, res: express.Response) { - const body: ManagePlugin = req.body - - await PluginManager.Instance.uninstall({ npmName: body.npmName }) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -function getPublicPluginSettings (req: express.Request, res: express.Response) { - const plugin = res.locals.plugin - const registeredSettings = PluginManager.Instance.getRegisteredSettings(req.params.npmName) - const publicSettings = plugin.getPublicSettings(registeredSettings) - - const json: PublicServerSetting = { publicSettings } - - return res.json(json) -} - -function getPluginRegisteredSettings (req: express.Request, res: express.Response) { - const registeredSettings = PluginManager.Instance.getRegisteredSettings(req.params.npmName) - - const json: RegisteredServerSettings = { registeredSettings } - - return res.json(json) -} - -async function updatePluginSettings (req: express.Request, res: express.Response) { - const plugin = res.locals.plugin - - plugin.settings = req.body.settings - await plugin.save() - - await PluginManager.Instance.onSettingsChanged(plugin.name, plugin.settings) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function listAvailablePlugins (req: express.Request, res: express.Response) { - const query: PeertubePluginIndexList = req.query - - const resultList = await listAvailablePluginsFromIndex(query) - - if (!resultList) { - return res.fail({ - status: HttpStatusCode.SERVICE_UNAVAILABLE_503, - message: 'Plugin index unavailable. Please retry later' - }) - } - - return res.json(resultList) -} diff --git a/server/controllers/api/runners/index.ts b/server/controllers/api/runners/index.ts deleted file mode 100644 index 9998fe4cc..000000000 --- a/server/controllers/api/runners/index.ts +++ /dev/null @@ -1,20 +0,0 @@ -import express from 'express' -import { runnerJobsRouter } from './jobs' -import { runnerJobFilesRouter } from './jobs-files' -import { manageRunnersRouter } from './manage-runners' -import { runnerRegistrationTokensRouter } from './registration-tokens' - -const runnersRouter = express.Router() - -// No api route limiter here, they are defined in child routers - -runnersRouter.use('/', manageRunnersRouter) -runnersRouter.use('/', runnerJobsRouter) -runnersRouter.use('/', runnerJobFilesRouter) -runnersRouter.use('/', runnerRegistrationTokensRouter) - -// --------------------------------------------------------------------------- - -export { - runnersRouter -} diff --git a/server/controllers/api/runners/jobs-files.ts b/server/controllers/api/runners/jobs-files.ts deleted file mode 100644 index d28f43701..000000000 --- a/server/controllers/api/runners/jobs-files.ts +++ /dev/null @@ -1,112 +0,0 @@ -import express from 'express' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { proxifyHLS, proxifyWebVideoFile } from '@server/lib/object-storage' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { getStudioTaskFilePath } from '@server/lib/video-studio' -import { apiRateLimiter, asyncMiddleware } from '@server/middlewares' -import { jobOfRunnerGetValidatorFactory } from '@server/middlewares/validators/runners' -import { - runnerJobGetVideoStudioTaskFileValidator, - runnerJobGetVideoTranscodingFileValidator -} from '@server/middlewares/validators/runners/job-files' -import { RunnerJobState, VideoStorage } from '@shared/models' - -const lTags = loggerTagsFactory('api', 'runner') - -const runnerJobFilesRouter = express.Router() - -runnerJobFilesRouter.post('/jobs/:jobUUID/files/videos/:videoId/max-quality', - apiRateLimiter, - asyncMiddleware(jobOfRunnerGetValidatorFactory([ RunnerJobState.PROCESSING ])), - asyncMiddleware(runnerJobGetVideoTranscodingFileValidator), - asyncMiddleware(getMaxQualityVideoFile) -) - -runnerJobFilesRouter.post('/jobs/:jobUUID/files/videos/:videoId/previews/max-quality', - apiRateLimiter, - asyncMiddleware(jobOfRunnerGetValidatorFactory([ RunnerJobState.PROCESSING ])), - asyncMiddleware(runnerJobGetVideoTranscodingFileValidator), - getMaxQualityVideoPreview -) - -runnerJobFilesRouter.post('/jobs/:jobUUID/files/videos/:videoId/studio/task-files/:filename', - apiRateLimiter, - asyncMiddleware(jobOfRunnerGetValidatorFactory([ RunnerJobState.PROCESSING ])), - asyncMiddleware(runnerJobGetVideoTranscodingFileValidator), - runnerJobGetVideoStudioTaskFileValidator, - getVideoStudioTaskFile -) - -// --------------------------------------------------------------------------- - -export { - runnerJobFilesRouter -} - -// --------------------------------------------------------------------------- - -async function getMaxQualityVideoFile (req: express.Request, res: express.Response) { - const runnerJob = res.locals.runnerJob - const runner = runnerJob.Runner - const video = res.locals.videoAll - - logger.info( - 'Get max quality file of video %s of job %s for runner %s', video.uuid, runnerJob.uuid, runner.name, - lTags(runner.name, runnerJob.id, runnerJob.type) - ) - - const file = video.getMaxQualityFile() - - if (file.storage === VideoStorage.OBJECT_STORAGE) { - if (file.isHLS()) { - return proxifyHLS({ - req, - res, - filename: file.filename, - playlist: video.getHLSPlaylist(), - reinjectVideoFileToken: false, - video - }) - } - - // Web video - return proxifyWebVideoFile({ - req, - res, - filename: file.filename - }) - } - - return VideoPathManager.Instance.makeAvailableVideoFile(file, videoPath => { - return res.sendFile(videoPath) - }) -} - -function getMaxQualityVideoPreview (req: express.Request, res: express.Response) { - const runnerJob = res.locals.runnerJob - const runner = runnerJob.Runner - const video = res.locals.videoAll - - logger.info( - 'Get max quality preview file of video %s of job %s for runner %s', video.uuid, runnerJob.uuid, runner.name, - lTags(runner.name, runnerJob.id, runnerJob.type) - ) - - const file = video.getPreview() - - return res.sendFile(file.getPath()) -} - -function getVideoStudioTaskFile (req: express.Request, res: express.Response) { - const runnerJob = res.locals.runnerJob - const runner = runnerJob.Runner - const video = res.locals.videoAll - const filename = req.params.filename - - logger.info( - 'Get video studio task file %s of video %s of job %s for runner %s', filename, video.uuid, runnerJob.uuid, runner.name, - lTags(runner.name, runnerJob.id, runnerJob.type) - ) - - return res.sendFile(getStudioTaskFilePath(filename)) -} diff --git a/server/controllers/api/runners/jobs.ts b/server/controllers/api/runners/jobs.ts deleted file mode 100644 index e9e2ddf49..000000000 --- a/server/controllers/api/runners/jobs.ts +++ /dev/null @@ -1,416 +0,0 @@ -import express, { UploadFiles } from 'express' -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { createReqFiles } from '@server/helpers/express-utils' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { generateRunnerJobToken } from '@server/helpers/token-generator' -import { MIMETYPES } from '@server/initializers/constants' -import { sequelizeTypescript } from '@server/initializers/database' -import { getRunnerJobHandlerClass, runnerJobCanBeCancelled, updateLastRunnerContact } from '@server/lib/runners' -import { - apiRateLimiter, - asyncMiddleware, - authenticate, - ensureUserHasRight, - paginationValidator, - runnerJobsSortValidator, - setDefaultPagination, - setDefaultSort -} from '@server/middlewares' -import { - abortRunnerJobValidator, - acceptRunnerJobValidator, - cancelRunnerJobValidator, - errorRunnerJobValidator, - getRunnerFromTokenValidator, - jobOfRunnerGetValidatorFactory, - listRunnerJobsValidator, - runnerJobGetValidator, - successRunnerJobValidator, - updateRunnerJobValidator -} from '@server/middlewares/validators/runners' -import { RunnerModel } from '@server/models/runner/runner' -import { RunnerJobModel } from '@server/models/runner/runner-job' -import { - AbortRunnerJobBody, - AcceptRunnerJobResult, - ErrorRunnerJobBody, - HttpStatusCode, - ListRunnerJobsQuery, - LiveRTMPHLSTranscodingUpdatePayload, - RequestRunnerJobResult, - RunnerJobState, - RunnerJobSuccessBody, - RunnerJobSuccessPayload, - RunnerJobType, - RunnerJobUpdateBody, - RunnerJobUpdatePayload, - ServerErrorCode, - UserRight, - VideoStudioTranscodingSuccess, - VODAudioMergeTranscodingSuccess, - VODHLSTranscodingSuccess, - VODWebVideoTranscodingSuccess -} from '@shared/models' - -const postRunnerJobSuccessVideoFiles = createReqFiles( - [ 'payload[videoFile]', 'payload[resolutionPlaylistFile]' ], - { ...MIMETYPES.VIDEO.MIMETYPE_EXT, ...MIMETYPES.M3U8.MIMETYPE_EXT } -) - -const runnerJobUpdateVideoFiles = createReqFiles( - [ 'payload[videoChunkFile]', 'payload[resolutionPlaylistFile]', 'payload[masterPlaylistFile]' ], - { ...MIMETYPES.VIDEO.MIMETYPE_EXT, ...MIMETYPES.M3U8.MIMETYPE_EXT } -) - -const lTags = loggerTagsFactory('api', 'runner') - -const runnerJobsRouter = express.Router() - -// --------------------------------------------------------------------------- -// Controllers for runners -// --------------------------------------------------------------------------- - -runnerJobsRouter.post('/jobs/request', - apiRateLimiter, - asyncMiddleware(getRunnerFromTokenValidator), - asyncMiddleware(requestRunnerJob) -) - -runnerJobsRouter.post('/jobs/:jobUUID/accept', - apiRateLimiter, - asyncMiddleware(runnerJobGetValidator), - acceptRunnerJobValidator, - asyncMiddleware(getRunnerFromTokenValidator), - asyncMiddleware(acceptRunnerJob) -) - -runnerJobsRouter.post('/jobs/:jobUUID/abort', - apiRateLimiter, - asyncMiddleware(jobOfRunnerGetValidatorFactory([ RunnerJobState.PROCESSING ])), - abortRunnerJobValidator, - asyncMiddleware(abortRunnerJob) -) - -runnerJobsRouter.post('/jobs/:jobUUID/update', - runnerJobUpdateVideoFiles, - apiRateLimiter, // Has to be after multer middleware to parse runner token - asyncMiddleware(jobOfRunnerGetValidatorFactory([ RunnerJobState.PROCESSING, RunnerJobState.COMPLETING, RunnerJobState.COMPLETED ])), - updateRunnerJobValidator, - asyncMiddleware(updateRunnerJobController) -) - -runnerJobsRouter.post('/jobs/:jobUUID/error', - asyncMiddleware(jobOfRunnerGetValidatorFactory([ RunnerJobState.PROCESSING ])), - errorRunnerJobValidator, - asyncMiddleware(errorRunnerJob) -) - -runnerJobsRouter.post('/jobs/:jobUUID/success', - postRunnerJobSuccessVideoFiles, - apiRateLimiter, // Has to be after multer middleware to parse runner token - asyncMiddleware(jobOfRunnerGetValidatorFactory([ RunnerJobState.PROCESSING ])), - successRunnerJobValidator, - asyncMiddleware(postRunnerJobSuccess) -) - -// --------------------------------------------------------------------------- -// Controllers for admins -// --------------------------------------------------------------------------- - -runnerJobsRouter.post('/jobs/:jobUUID/cancel', - authenticate, - ensureUserHasRight(UserRight.MANAGE_RUNNERS), - asyncMiddleware(runnerJobGetValidator), - cancelRunnerJobValidator, - asyncMiddleware(cancelRunnerJob) -) - -runnerJobsRouter.get('/jobs', - authenticate, - ensureUserHasRight(UserRight.MANAGE_RUNNERS), - paginationValidator, - runnerJobsSortValidator, - setDefaultSort, - setDefaultPagination, - listRunnerJobsValidator, - asyncMiddleware(listRunnerJobs) -) - -runnerJobsRouter.delete('/jobs/:jobUUID', - authenticate, - ensureUserHasRight(UserRight.MANAGE_RUNNERS), - asyncMiddleware(runnerJobGetValidator), - asyncMiddleware(deleteRunnerJob) -) - -// --------------------------------------------------------------------------- - -export { - runnerJobsRouter -} - -// --------------------------------------------------------------------------- - -// --------------------------------------------------------------------------- -// Controllers for runners -// --------------------------------------------------------------------------- - -async function requestRunnerJob (req: express.Request, res: express.Response) { - const runner = res.locals.runner - const availableJobs = await RunnerJobModel.listAvailableJobs() - - logger.debug('Runner %s requests for a job.', runner.name, { availableJobs, ...lTags(runner.name) }) - - const result: RequestRunnerJobResult = { - availableJobs: availableJobs.map(j => ({ - uuid: j.uuid, - type: j.type, - payload: j.payload - })) - } - - updateLastRunnerContact(req, runner) - - return res.json(result) -} - -async function acceptRunnerJob (req: express.Request, res: express.Response) { - const runner = res.locals.runner - const runnerJob = res.locals.runnerJob - - const newRunnerJob = await retryTransactionWrapper(() => { - return sequelizeTypescript.transaction(async transaction => { - await runnerJob.reload({ transaction }) - - if (runnerJob.state !== RunnerJobState.PENDING) { - res.fail({ - type: ServerErrorCode.RUNNER_JOB_NOT_IN_PENDING_STATE, - message: 'This job is not in pending state anymore', - status: HttpStatusCode.CONFLICT_409 - }) - - return undefined - } - - runnerJob.state = RunnerJobState.PROCESSING - runnerJob.processingJobToken = generateRunnerJobToken() - runnerJob.startedAt = new Date() - runnerJob.runnerId = runner.id - - return runnerJob.save({ transaction }) - }) - }) - if (!newRunnerJob) return - - newRunnerJob.Runner = runner as RunnerModel - - const result: AcceptRunnerJobResult = { - job: { - ...newRunnerJob.toFormattedJSON(), - - jobToken: newRunnerJob.processingJobToken - } - } - - updateLastRunnerContact(req, runner) - - logger.info( - 'Remote runner %s has accepted job %s (%s)', runner.name, runnerJob.uuid, runnerJob.type, - lTags(runner.name, runnerJob.uuid, runnerJob.type) - ) - - return res.json(result) -} - -async function abortRunnerJob (req: express.Request, res: express.Response) { - const runnerJob = res.locals.runnerJob - const runner = runnerJob.Runner - const body: AbortRunnerJobBody = req.body - - logger.info( - 'Remote runner %s is aborting job %s (%s)', runner.name, runnerJob.uuid, runnerJob.type, - { reason: body.reason, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) } - ) - - const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) - await new RunnerJobHandler().abort({ runnerJob }) - - updateLastRunnerContact(req, runnerJob.Runner) - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -async function errorRunnerJob (req: express.Request, res: express.Response) { - const runnerJob = res.locals.runnerJob - const runner = runnerJob.Runner - const body: ErrorRunnerJobBody = req.body - - runnerJob.failures += 1 - - logger.error( - 'Remote runner %s had an error with job %s (%s)', runner.name, runnerJob.uuid, runnerJob.type, - { errorMessage: body.message, totalFailures: runnerJob.failures, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) } - ) - - const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) - await new RunnerJobHandler().error({ runnerJob, message: body.message }) - - updateLastRunnerContact(req, runnerJob.Runner) - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -// --------------------------------------------------------------------------- - -const jobUpdateBuilders: { - [id in RunnerJobType]?: (payload: RunnerJobUpdatePayload, files?: UploadFiles) => RunnerJobUpdatePayload -} = { - 'live-rtmp-hls-transcoding': (payload: LiveRTMPHLSTranscodingUpdatePayload, files) => { - return { - ...payload, - - masterPlaylistFile: files['payload[masterPlaylistFile]']?.[0].path, - resolutionPlaylistFile: files['payload[resolutionPlaylistFile]']?.[0].path, - videoChunkFile: files['payload[videoChunkFile]']?.[0].path - } - } -} - -async function updateRunnerJobController (req: express.Request, res: express.Response) { - const runnerJob = res.locals.runnerJob - const runner = runnerJob.Runner - const body: RunnerJobUpdateBody = req.body - - if (runnerJob.state === RunnerJobState.COMPLETING || runnerJob.state === RunnerJobState.COMPLETED) { - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) - } - - const payloadBuilder = jobUpdateBuilders[runnerJob.type] - const updatePayload = payloadBuilder - ? payloadBuilder(body.payload, req.files as UploadFiles) - : undefined - - logger.debug( - 'Remote runner %s is updating job %s (%s)', runnerJob.Runner.name, runnerJob.uuid, runnerJob.type, - { body, updatePayload, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) } - ) - - const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) - await new RunnerJobHandler().update({ - runnerJob, - progress: req.body.progress, - updatePayload - }) - - updateLastRunnerContact(req, runnerJob.Runner) - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -// --------------------------------------------------------------------------- - -const jobSuccessPayloadBuilders: { - [id in RunnerJobType]: (payload: RunnerJobSuccessPayload, files?: UploadFiles) => RunnerJobSuccessPayload -} = { - 'vod-web-video-transcoding': (payload: VODWebVideoTranscodingSuccess, files) => { - return { - ...payload, - - videoFile: files['payload[videoFile]'][0].path - } - }, - - 'vod-hls-transcoding': (payload: VODHLSTranscodingSuccess, files) => { - return { - ...payload, - - videoFile: files['payload[videoFile]'][0].path, - resolutionPlaylistFile: files['payload[resolutionPlaylistFile]'][0].path - } - }, - - 'vod-audio-merge-transcoding': (payload: VODAudioMergeTranscodingSuccess, files) => { - return { - ...payload, - - videoFile: files['payload[videoFile]'][0].path - } - }, - - 'video-studio-transcoding': (payload: VideoStudioTranscodingSuccess, files) => { - return { - ...payload, - - videoFile: files['payload[videoFile]'][0].path - } - }, - - 'live-rtmp-hls-transcoding': () => ({}) -} - -async function postRunnerJobSuccess (req: express.Request, res: express.Response) { - const runnerJob = res.locals.runnerJob - const runner = runnerJob.Runner - const body: RunnerJobSuccessBody = req.body - - const resultPayload = jobSuccessPayloadBuilders[runnerJob.type](body.payload, req.files as UploadFiles) - - logger.info( - 'Remote runner %s is sending success result for job %s (%s)', runnerJob.Runner.name, runnerJob.uuid, runnerJob.type, - { resultPayload, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) } - ) - - const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) - await new RunnerJobHandler().complete({ runnerJob, resultPayload }) - - updateLastRunnerContact(req, runnerJob.Runner) - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -// --------------------------------------------------------------------------- -// Controllers for admins -// --------------------------------------------------------------------------- - -async function cancelRunnerJob (req: express.Request, res: express.Response) { - const runnerJob = res.locals.runnerJob - - logger.info('Cancelling job %s (%s)', runnerJob.uuid, runnerJob.type, lTags(runnerJob.uuid, runnerJob.type)) - - const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) - await new RunnerJobHandler().cancel({ runnerJob }) - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -async function deleteRunnerJob (req: express.Request, res: express.Response) { - const runnerJob = res.locals.runnerJob - - logger.info('Deleting job %s (%s)', runnerJob.uuid, runnerJob.type, lTags(runnerJob.uuid, runnerJob.type)) - - if (runnerJobCanBeCancelled(runnerJob)) { - const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) - await new RunnerJobHandler().cancel({ runnerJob }) - } - - await runnerJob.destroy() - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -async function listRunnerJobs (req: express.Request, res: express.Response) { - const query: ListRunnerJobsQuery = req.query - - const resultList = await RunnerJobModel.listForApi({ - start: query.start, - count: query.count, - sort: query.sort, - search: query.search, - stateOneOf: query.stateOneOf - }) - - return res.json({ - total: resultList.total, - data: resultList.data.map(d => d.toFormattedAdminJSON()) - }) -} diff --git a/server/controllers/api/runners/manage-runners.ts b/server/controllers/api/runners/manage-runners.ts deleted file mode 100644 index be7ebc0b3..000000000 --- a/server/controllers/api/runners/manage-runners.ts +++ /dev/null @@ -1,112 +0,0 @@ -import express from 'express' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { generateRunnerToken } from '@server/helpers/token-generator' -import { - apiRateLimiter, - asyncMiddleware, - authenticate, - ensureUserHasRight, - paginationValidator, - runnersSortValidator, - setDefaultPagination, - setDefaultSort -} from '@server/middlewares' -import { deleteRunnerValidator, getRunnerFromTokenValidator, registerRunnerValidator } from '@server/middlewares/validators/runners' -import { RunnerModel } from '@server/models/runner/runner' -import { HttpStatusCode, ListRunnersQuery, RegisterRunnerBody, UserRight } from '@shared/models' - -const lTags = loggerTagsFactory('api', 'runner') - -const manageRunnersRouter = express.Router() - -manageRunnersRouter.post('/register', - apiRateLimiter, - asyncMiddleware(registerRunnerValidator), - asyncMiddleware(registerRunner) -) -manageRunnersRouter.post('/unregister', - apiRateLimiter, - asyncMiddleware(getRunnerFromTokenValidator), - asyncMiddleware(unregisterRunner) -) - -manageRunnersRouter.delete('/:runnerId', - apiRateLimiter, - authenticate, - ensureUserHasRight(UserRight.MANAGE_RUNNERS), - asyncMiddleware(deleteRunnerValidator), - asyncMiddleware(deleteRunner) -) - -manageRunnersRouter.get('/', - apiRateLimiter, - authenticate, - ensureUserHasRight(UserRight.MANAGE_RUNNERS), - paginationValidator, - runnersSortValidator, - setDefaultSort, - setDefaultPagination, - asyncMiddleware(listRunners) -) - -// --------------------------------------------------------------------------- - -export { - manageRunnersRouter -} - -// --------------------------------------------------------------------------- - -async function registerRunner (req: express.Request, res: express.Response) { - const body: RegisterRunnerBody = req.body - - const runnerToken = generateRunnerToken() - - const runner = new RunnerModel({ - runnerToken, - name: body.name, - description: body.description, - lastContact: new Date(), - ip: req.ip, - runnerRegistrationTokenId: res.locals.runnerRegistrationToken.id - }) - - await runner.save() - - logger.info('Registered new runner %s', runner.name, { ...lTags(runner.name) }) - - return res.json({ id: runner.id, runnerToken }) -} -async function unregisterRunner (req: express.Request, res: express.Response) { - const runner = res.locals.runner - await runner.destroy() - - logger.info('Unregistered runner %s', runner.name, { ...lTags(runner.name) }) - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -async function deleteRunner (req: express.Request, res: express.Response) { - const runner = res.locals.runner - - await runner.destroy() - - logger.info('Deleted runner %s', runner.name, { ...lTags(runner.name) }) - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -async function listRunners (req: express.Request, res: express.Response) { - const query: ListRunnersQuery = req.query - - const resultList = await RunnerModel.listForApi({ - start: query.start, - count: query.count, - sort: query.sort - }) - - return res.json({ - total: resultList.total, - data: resultList.data.map(d => d.toFormattedJSON()) - }) -} diff --git a/server/controllers/api/runners/registration-tokens.ts b/server/controllers/api/runners/registration-tokens.ts deleted file mode 100644 index 117ff271b..000000000 --- a/server/controllers/api/runners/registration-tokens.ts +++ /dev/null @@ -1,91 +0,0 @@ -import express from 'express' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { generateRunnerRegistrationToken } from '@server/helpers/token-generator' -import { - apiRateLimiter, - asyncMiddleware, - authenticate, - ensureUserHasRight, - paginationValidator, - runnerRegistrationTokensSortValidator, - setDefaultPagination, - setDefaultSort -} from '@server/middlewares' -import { deleteRegistrationTokenValidator } from '@server/middlewares/validators/runners' -import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token' -import { HttpStatusCode, ListRunnerRegistrationTokensQuery, UserRight } from '@shared/models' - -const lTags = loggerTagsFactory('api', 'runner') - -const runnerRegistrationTokensRouter = express.Router() - -runnerRegistrationTokensRouter.post('/registration-tokens/generate', - apiRateLimiter, - authenticate, - ensureUserHasRight(UserRight.MANAGE_RUNNERS), - asyncMiddleware(generateRegistrationToken) -) - -runnerRegistrationTokensRouter.delete('/registration-tokens/:id', - apiRateLimiter, - authenticate, - ensureUserHasRight(UserRight.MANAGE_RUNNERS), - asyncMiddleware(deleteRegistrationTokenValidator), - asyncMiddleware(deleteRegistrationToken) -) - -runnerRegistrationTokensRouter.get('/registration-tokens', - apiRateLimiter, - authenticate, - ensureUserHasRight(UserRight.MANAGE_RUNNERS), - paginationValidator, - runnerRegistrationTokensSortValidator, - setDefaultSort, - setDefaultPagination, - asyncMiddleware(listRegistrationTokens) -) - -// --------------------------------------------------------------------------- - -export { - runnerRegistrationTokensRouter -} - -// --------------------------------------------------------------------------- - -async function generateRegistrationToken (req: express.Request, res: express.Response) { - logger.info('Generating new runner registration token.', lTags()) - - const registrationToken = new RunnerRegistrationTokenModel({ - registrationToken: generateRunnerRegistrationToken() - }) - - await registrationToken.save() - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -async function deleteRegistrationToken (req: express.Request, res: express.Response) { - logger.info('Removing runner registration token.', lTags()) - - const runnerRegistrationToken = res.locals.runnerRegistrationToken - - await runnerRegistrationToken.destroy() - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -async function listRegistrationTokens (req: express.Request, res: express.Response) { - const query: ListRunnerRegistrationTokensQuery = req.query - - const resultList = await RunnerRegistrationTokenModel.listForApi({ - start: query.start, - count: query.count, - sort: query.sort - }) - - return res.json({ - total: resultList.total, - data: resultList.data.map(d => d.toFormattedJSON()) - }) -} diff --git a/server/controllers/api/search/index.ts b/server/controllers/api/search/index.ts deleted file mode 100644 index 4d395161c..000000000 --- a/server/controllers/api/search/index.ts +++ /dev/null @@ -1,19 +0,0 @@ -import express from 'express' -import { apiRateLimiter } from '@server/middlewares' -import { searchChannelsRouter } from './search-video-channels' -import { searchPlaylistsRouter } from './search-video-playlists' -import { searchVideosRouter } from './search-videos' - -const searchRouter = express.Router() - -searchRouter.use(apiRateLimiter) - -searchRouter.use('/', searchVideosRouter) -searchRouter.use('/', searchChannelsRouter) -searchRouter.use('/', searchPlaylistsRouter) - -// --------------------------------------------------------------------------- - -export { - searchRouter -} diff --git a/server/controllers/api/search/search-video-channels.ts b/server/controllers/api/search/search-video-channels.ts deleted file mode 100644 index 1d2a9d235..000000000 --- a/server/controllers/api/search/search-video-channels.ts +++ /dev/null @@ -1,152 +0,0 @@ -import express from 'express' -import { sanitizeUrl } from '@server/helpers/core-utils' -import { pickSearchChannelQuery } from '@server/helpers/query' -import { doJSONRequest } from '@server/helpers/requests' -import { CONFIG } from '@server/initializers/config' -import { WEBSERVER } from '@server/initializers/constants' -import { findLatestAPRedirection } from '@server/lib/activitypub/activity' -import { Hooks } from '@server/lib/plugins/hooks' -import { buildMutedForSearchIndex, isSearchIndexSearch, isURISearch } from '@server/lib/search' -import { getServerActor } from '@server/models/application/application' -import { HttpStatusCode, ResultList, VideoChannel } from '@shared/models' -import { VideoChannelsSearchQueryAfterSanitize } from '../../../../shared/models/search' -import { isUserAbleToSearchRemoteURI } from '../../../helpers/express-utils' -import { logger } from '../../../helpers/logger' -import { getFormattedObjects } from '../../../helpers/utils' -import { getOrCreateAPActor, loadActorUrlOrGetFromWebfinger } from '../../../lib/activitypub/actors' -import { - asyncMiddleware, - openapiOperationDoc, - optionalAuthenticate, - paginationValidator, - setDefaultPagination, - setDefaultSearchSort, - videoChannelsListSearchValidator, - videoChannelsSearchSortValidator -} from '../../../middlewares' -import { VideoChannelModel } from '../../../models/video/video-channel' -import { MChannelAccountDefault } from '../../../types/models' -import { searchLocalUrl } from './shared' - -const searchChannelsRouter = express.Router() - -searchChannelsRouter.get('/video-channels', - openapiOperationDoc({ operationId: 'searchChannels' }), - paginationValidator, - setDefaultPagination, - videoChannelsSearchSortValidator, - setDefaultSearchSort, - optionalAuthenticate, - videoChannelsListSearchValidator, - asyncMiddleware(searchVideoChannels) -) - -// --------------------------------------------------------------------------- - -export { searchChannelsRouter } - -// --------------------------------------------------------------------------- - -function searchVideoChannels (req: express.Request, res: express.Response) { - const query = pickSearchChannelQuery(req.query) - const search = query.search || '' - - const parts = search.split('@') - - // Handle strings like @toto@example.com - if (parts.length === 3 && parts[0].length === 0) parts.shift() - const isWebfingerSearch = parts.length === 2 && parts.every(p => p && !p.includes(' ')) - - if (isURISearch(search) || isWebfingerSearch) return searchVideoChannelURI(search, res) - - // @username -> username to search in DB - if (search.startsWith('@')) query.search = search.replace(/^@/, '') - - if (isSearchIndexSearch(query)) { - return searchVideoChannelsIndex(query, res) - } - - return searchVideoChannelsDB(query, res) -} - -async function searchVideoChannelsIndex (query: VideoChannelsSearchQueryAfterSanitize, res: express.Response) { - const result = await buildMutedForSearchIndex(res) - - const body = await Hooks.wrapObject(Object.assign(query, result), 'filter:api.search.video-channels.index.list.params') - - const url = sanitizeUrl(CONFIG.SEARCH.SEARCH_INDEX.URL) + '/api/v1/search/video-channels' - - try { - logger.debug('Doing video channels search index request on %s.', url, { body }) - - const { body: searchIndexResult } = await doJSONRequest>(url, { method: 'POST', json: body }) - const jsonResult = await Hooks.wrapObject(searchIndexResult, 'filter:api.search.video-channels.index.list.result') - - return res.json(jsonResult) - } catch (err) { - logger.warn('Cannot use search index to make video channels search.', { err }) - - return res.fail({ - status: HttpStatusCode.INTERNAL_SERVER_ERROR_500, - message: 'Cannot use search index to make video channels search' - }) - } -} - -async function searchVideoChannelsDB (query: VideoChannelsSearchQueryAfterSanitize, res: express.Response) { - const serverActor = await getServerActor() - - const apiOptions = await Hooks.wrapObject({ - ...query, - - actorId: serverActor.id - }, 'filter:api.search.video-channels.local.list.params') - - const resultList = await Hooks.wrapPromiseFun( - VideoChannelModel.searchForApi, - apiOptions, - 'filter:api.search.video-channels.local.list.result' - ) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function searchVideoChannelURI (search: string, res: express.Response) { - let videoChannel: MChannelAccountDefault - let uri = search - - if (!isURISearch(search)) { - try { - uri = await loadActorUrlOrGetFromWebfinger(search) - } catch (err) { - logger.warn('Cannot load actor URL or get from webfinger.', { search, err }) - - return res.json({ total: 0, data: [] }) - } - } - - if (isUserAbleToSearchRemoteURI(res)) { - try { - const latestUri = await findLatestAPRedirection(uri) - - const actor = await getOrCreateAPActor(latestUri, 'all', true, true) - videoChannel = actor.VideoChannel - } catch (err) { - logger.info('Cannot search remote video channel %s.', uri, { err }) - } - } else { - videoChannel = await searchLocalUrl(sanitizeLocalUrl(uri), url => VideoChannelModel.loadByUrlAndPopulateAccount(url)) - } - - return res.json({ - total: videoChannel ? 1 : 0, - data: videoChannel ? [ videoChannel.toFormattedJSON() ] : [] - }) -} - -function sanitizeLocalUrl (url: string) { - if (!url) return '' - - // Handle alternative channel URLs - return url.replace(new RegExp('^' + WEBSERVER.URL + '/c/'), WEBSERVER.URL + '/video-channels/') -} diff --git a/server/controllers/api/search/search-video-playlists.ts b/server/controllers/api/search/search-video-playlists.ts deleted file mode 100644 index 97aeeaba9..000000000 --- a/server/controllers/api/search/search-video-playlists.ts +++ /dev/null @@ -1,131 +0,0 @@ -import express from 'express' -import { sanitizeUrl } from '@server/helpers/core-utils' -import { isUserAbleToSearchRemoteURI } from '@server/helpers/express-utils' -import { logger } from '@server/helpers/logger' -import { pickSearchPlaylistQuery } from '@server/helpers/query' -import { doJSONRequest } from '@server/helpers/requests' -import { getFormattedObjects } from '@server/helpers/utils' -import { CONFIG } from '@server/initializers/config' -import { WEBSERVER } from '@server/initializers/constants' -import { findLatestAPRedirection } from '@server/lib/activitypub/activity' -import { getOrCreateAPVideoPlaylist } from '@server/lib/activitypub/playlists/get' -import { Hooks } from '@server/lib/plugins/hooks' -import { buildMutedForSearchIndex, isSearchIndexSearch, isURISearch } from '@server/lib/search' -import { getServerActor } from '@server/models/application/application' -import { VideoPlaylistModel } from '@server/models/video/video-playlist' -import { MVideoPlaylistFullSummary } from '@server/types/models' -import { HttpStatusCode, ResultList, VideoPlaylist, VideoPlaylistsSearchQueryAfterSanitize } from '@shared/models' -import { - asyncMiddleware, - openapiOperationDoc, - optionalAuthenticate, - paginationValidator, - setDefaultPagination, - setDefaultSearchSort, - videoPlaylistsListSearchValidator, - videoPlaylistsSearchSortValidator -} from '../../../middlewares' -import { searchLocalUrl } from './shared' - -const searchPlaylistsRouter = express.Router() - -searchPlaylistsRouter.get('/video-playlists', - openapiOperationDoc({ operationId: 'searchPlaylists' }), - paginationValidator, - setDefaultPagination, - videoPlaylistsSearchSortValidator, - setDefaultSearchSort, - optionalAuthenticate, - videoPlaylistsListSearchValidator, - asyncMiddleware(searchVideoPlaylists) -) - -// --------------------------------------------------------------------------- - -export { searchPlaylistsRouter } - -// --------------------------------------------------------------------------- - -function searchVideoPlaylists (req: express.Request, res: express.Response) { - const query = pickSearchPlaylistQuery(req.query) - const search = query.search - - if (isURISearch(search)) return searchVideoPlaylistsURI(search, res) - - if (isSearchIndexSearch(query)) { - return searchVideoPlaylistsIndex(query, res) - } - - return searchVideoPlaylistsDB(query, res) -} - -async function searchVideoPlaylistsIndex (query: VideoPlaylistsSearchQueryAfterSanitize, res: express.Response) { - const result = await buildMutedForSearchIndex(res) - - const body = await Hooks.wrapObject(Object.assign(query, result), 'filter:api.search.video-playlists.index.list.params') - - const url = sanitizeUrl(CONFIG.SEARCH.SEARCH_INDEX.URL) + '/api/v1/search/video-playlists' - - try { - logger.debug('Doing video playlists search index request on %s.', url, { body }) - - const { body: searchIndexResult } = await doJSONRequest>(url, { method: 'POST', json: body }) - const jsonResult = await Hooks.wrapObject(searchIndexResult, 'filter:api.search.video-playlists.index.list.result') - - return res.json(jsonResult) - } catch (err) { - logger.warn('Cannot use search index to make video playlists search.', { err }) - - return res.fail({ - status: HttpStatusCode.INTERNAL_SERVER_ERROR_500, - message: 'Cannot use search index to make video playlists search' - }) - } -} - -async function searchVideoPlaylistsDB (query: VideoPlaylistsSearchQueryAfterSanitize, res: express.Response) { - const serverActor = await getServerActor() - - const apiOptions = await Hooks.wrapObject({ - ...query, - - followerActorId: serverActor.id - }, 'filter:api.search.video-playlists.local.list.params') - - const resultList = await Hooks.wrapPromiseFun( - VideoPlaylistModel.searchForApi, - apiOptions, - 'filter:api.search.video-playlists.local.list.result' - ) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function searchVideoPlaylistsURI (search: string, res: express.Response) { - let videoPlaylist: MVideoPlaylistFullSummary - - if (isUserAbleToSearchRemoteURI(res)) { - try { - const url = await findLatestAPRedirection(search) - - videoPlaylist = await getOrCreateAPVideoPlaylist(url) - } catch (err) { - logger.info('Cannot search remote video playlist %s.', search, { err }) - } - } else { - videoPlaylist = await searchLocalUrl(sanitizeLocalUrl(search), url => VideoPlaylistModel.loadByUrlWithAccountAndChannelSummary(url)) - } - - return res.json({ - total: videoPlaylist ? 1 : 0, - data: videoPlaylist ? [ videoPlaylist.toFormattedJSON() ] : [] - }) -} - -function sanitizeLocalUrl (url: string) { - if (!url) return '' - - // Handle alternative channel URLs - return url.replace(new RegExp('^' + WEBSERVER.URL + '/videos/watch/playlist/'), WEBSERVER.URL + '/video-playlists/') - .replace(new RegExp('^' + WEBSERVER.URL + '/w/p/'), WEBSERVER.URL + '/video-playlists/') -} diff --git a/server/controllers/api/search/search-videos.ts b/server/controllers/api/search/search-videos.ts deleted file mode 100644 index b33064335..000000000 --- a/server/controllers/api/search/search-videos.ts +++ /dev/null @@ -1,167 +0,0 @@ -import express from 'express' -import { sanitizeUrl } from '@server/helpers/core-utils' -import { pickSearchVideoQuery } from '@server/helpers/query' -import { doJSONRequest } from '@server/helpers/requests' -import { CONFIG } from '@server/initializers/config' -import { WEBSERVER } from '@server/initializers/constants' -import { findLatestAPRedirection } from '@server/lib/activitypub/activity' -import { getOrCreateAPVideo } from '@server/lib/activitypub/videos' -import { Hooks } from '@server/lib/plugins/hooks' -import { buildMutedForSearchIndex, isSearchIndexSearch, isURISearch } from '@server/lib/search' -import { getServerActor } from '@server/models/application/application' -import { HttpStatusCode, ResultList, Video } from '@shared/models' -import { VideosSearchQueryAfterSanitize } from '../../../../shared/models/search' -import { buildNSFWFilter, isUserAbleToSearchRemoteURI } from '../../../helpers/express-utils' -import { logger } from '../../../helpers/logger' -import { getFormattedObjects } from '../../../helpers/utils' -import { - asyncMiddleware, - commonVideosFiltersValidator, - openapiOperationDoc, - optionalAuthenticate, - paginationValidator, - setDefaultPagination, - setDefaultSearchSort, - videosSearchSortValidator, - videosSearchValidator -} from '../../../middlewares' -import { guessAdditionalAttributesFromQuery } from '../../../models/video/formatter' -import { VideoModel } from '../../../models/video/video' -import { MVideoAccountLightBlacklistAllFiles } from '../../../types/models' -import { searchLocalUrl } from './shared' - -const searchVideosRouter = express.Router() - -searchVideosRouter.get('/videos', - openapiOperationDoc({ operationId: 'searchVideos' }), - paginationValidator, - setDefaultPagination, - videosSearchSortValidator, - setDefaultSearchSort, - optionalAuthenticate, - commonVideosFiltersValidator, - videosSearchValidator, - asyncMiddleware(searchVideos) -) - -// --------------------------------------------------------------------------- - -export { searchVideosRouter } - -// --------------------------------------------------------------------------- - -function searchVideos (req: express.Request, res: express.Response) { - const query = pickSearchVideoQuery(req.query) - const search = query.search - - if (isURISearch(search)) { - return searchVideoURI(search, res) - } - - if (isSearchIndexSearch(query)) { - return searchVideosIndex(query, res) - } - - return searchVideosDB(query, res) -} - -async function searchVideosIndex (query: VideosSearchQueryAfterSanitize, res: express.Response) { - const result = await buildMutedForSearchIndex(res) - - let body = { ...query, ...result } - - // Use the default instance NSFW policy if not specified - if (!body.nsfw) { - const nsfwPolicy = res.locals.oauth - ? res.locals.oauth.token.User.nsfwPolicy - : CONFIG.INSTANCE.DEFAULT_NSFW_POLICY - - body.nsfw = nsfwPolicy === 'do_not_list' - ? 'false' - : 'both' - } - - body = await Hooks.wrapObject(body, 'filter:api.search.videos.index.list.params') - - const url = sanitizeUrl(CONFIG.SEARCH.SEARCH_INDEX.URL) + '/api/v1/search/videos' - - try { - logger.debug('Doing videos search index request on %s.', url, { body }) - - const { body: searchIndexResult } = await doJSONRequest>(url, { method: 'POST', json: body }) - const jsonResult = await Hooks.wrapObject(searchIndexResult, 'filter:api.search.videos.index.list.result') - - return res.json(jsonResult) - } catch (err) { - logger.warn('Cannot use search index to make video search.', { err }) - - return res.fail({ - status: HttpStatusCode.INTERNAL_SERVER_ERROR_500, - message: 'Cannot use search index to make video search' - }) - } -} - -async function searchVideosDB (query: VideosSearchQueryAfterSanitize, res: express.Response) { - const serverActor = await getServerActor() - - const apiOptions = await Hooks.wrapObject({ - ...query, - - displayOnlyForFollower: { - actorId: serverActor.id, - orLocalVideos: true - }, - - nsfw: buildNSFWFilter(res, query.nsfw), - user: res.locals.oauth - ? res.locals.oauth.token.User - : undefined - }, 'filter:api.search.videos.local.list.params') - - const resultList = await Hooks.wrapPromiseFun( - VideoModel.searchAndPopulateAccountAndServer, - apiOptions, - 'filter:api.search.videos.local.list.result' - ) - - return res.json(getFormattedObjects(resultList.data, resultList.total, guessAdditionalAttributesFromQuery(query))) -} - -async function searchVideoURI (url: string, res: express.Response) { - let video: MVideoAccountLightBlacklistAllFiles - - // Check if we can fetch a remote video with the URL - if (isUserAbleToSearchRemoteURI(res)) { - try { - const syncParam = { - rates: false, - shares: false, - comments: false, - refreshVideo: false - } - - const result = await getOrCreateAPVideo({ - videoObject: await findLatestAPRedirection(url), - syncParam - }) - video = result ? result.video : undefined - } catch (err) { - logger.info('Cannot search remote video %s.', url, { err }) - } - } else { - video = await searchLocalUrl(sanitizeLocalUrl(url), url => VideoModel.loadByUrlAndPopulateAccount(url)) - } - - return res.json({ - total: video ? 1 : 0, - data: video ? [ video.toFormattedJSON() ] : [] - }) -} - -function sanitizeLocalUrl (url: string) { - if (!url) return '' - - // Handle alternative video URLs - return url.replace(new RegExp('^' + WEBSERVER.URL + '/w/'), WEBSERVER.URL + '/videos/watch/') -} diff --git a/server/controllers/api/search/shared/index.ts b/server/controllers/api/search/shared/index.ts deleted file mode 100644 index 9c56149ef..000000000 --- a/server/controllers/api/search/shared/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './utils' diff --git a/server/controllers/api/server/contact.ts b/server/controllers/api/server/contact.ts deleted file mode 100644 index 56596bea5..000000000 --- a/server/controllers/api/server/contact.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { logger } from '@server/helpers/logger' -import express from 'express' -import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' -import { ContactForm } from '../../../../shared/models/server' -import { Emailer } from '../../../lib/emailer' -import { Redis } from '../../../lib/redis' -import { asyncMiddleware, contactAdministratorValidator } from '../../../middlewares' - -const contactRouter = express.Router() - -contactRouter.post('/contact', - asyncMiddleware(contactAdministratorValidator), - asyncMiddleware(contactAdministrator) -) - -async function contactAdministrator (req: express.Request, res: express.Response) { - const data = req.body as ContactForm - - Emailer.Instance.addContactFormJob(data.fromEmail, data.fromName, data.subject, data.body) - - try { - await Redis.Instance.setContactFormIp(req.ip) - } catch (err) { - logger.error(err) - } - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -// --------------------------------------------------------------------------- - -export { - contactRouter -} diff --git a/server/controllers/api/server/debug.ts b/server/controllers/api/server/debug.ts deleted file mode 100644 index f3792bfc8..000000000 --- a/server/controllers/api/server/debug.ts +++ /dev/null @@ -1,56 +0,0 @@ -import express from 'express' -import { InboxManager } from '@server/lib/activitypub/inbox-manager' -import { RemoveDanglingResumableUploadsScheduler } from '@server/lib/schedulers/remove-dangling-resumable-uploads-scheduler' -import { VideoViewsBufferScheduler } from '@server/lib/schedulers/video-views-buffer-scheduler' -import { VideoViewsManager } from '@server/lib/views/video-views-manager' -import { Debug, SendDebugCommand } from '@shared/models' -import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' -import { UserRight } from '../../../../shared/models/users' -import { authenticate, ensureUserHasRight } from '../../../middlewares' -import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler' -import { UpdateVideosScheduler } from '@server/lib/schedulers/update-videos-scheduler' - -const debugRouter = express.Router() - -debugRouter.get('/debug', - authenticate, - ensureUserHasRight(UserRight.MANAGE_DEBUG), - getDebug -) - -debugRouter.post('/debug/run-command', - authenticate, - ensureUserHasRight(UserRight.MANAGE_DEBUG), - runCommand -) - -// --------------------------------------------------------------------------- - -export { - debugRouter -} - -// --------------------------------------------------------------------------- - -function getDebug (req: express.Request, res: express.Response) { - return res.json({ - ip: req.ip, - activityPubMessagesWaiting: InboxManager.Instance.getActivityPubMessagesWaiting() - } as Debug) -} - -async function runCommand (req: express.Request, res: express.Response) { - const body: SendDebugCommand = req.body - - const processors: { [id in SendDebugCommand['command']]: () => Promise } = { - 'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(), - 'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(), - 'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(), - 'process-update-videos-scheduler': () => UpdateVideosScheduler.Instance.execute(), - 'process-video-channel-sync-latest': () => VideoChannelSyncLatestScheduler.Instance.execute() - } - - await processors[body.command]() - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} diff --git a/server/controllers/api/server/follows.ts b/server/controllers/api/server/follows.ts deleted file mode 100644 index 87828813a..000000000 --- a/server/controllers/api/server/follows.ts +++ /dev/null @@ -1,214 +0,0 @@ -import express from 'express' -import { getServerActor } from '@server/models/application/application' -import { ServerFollowCreate } from '@shared/models' -import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' -import { UserRight } from '../../../../shared/models/users' -import { logger } from '../../../helpers/logger' -import { getFormattedObjects } from '../../../helpers/utils' -import { SERVER_ACTOR_NAME } from '../../../initializers/constants' -import { sequelizeTypescript } from '../../../initializers/database' -import { autoFollowBackIfNeeded } from '../../../lib/activitypub/follow' -import { sendAccept, sendReject, sendUndoFollow } from '../../../lib/activitypub/send' -import { JobQueue } from '../../../lib/job-queue' -import { removeRedundanciesOfServer } from '../../../lib/redundancy' -import { - asyncMiddleware, - authenticate, - ensureUserHasRight, - paginationValidator, - setBodyHostsPort, - setDefaultPagination, - setDefaultSort -} from '../../../middlewares' -import { - acceptFollowerValidator, - followValidator, - getFollowerValidator, - instanceFollowersSortValidator, - instanceFollowingSortValidator, - listFollowsValidator, - rejectFollowerValidator, - removeFollowingValidator -} from '../../../middlewares/validators' -import { ActorFollowModel } from '../../../models/actor/actor-follow' - -const serverFollowsRouter = express.Router() -serverFollowsRouter.get('/following', - listFollowsValidator, - paginationValidator, - instanceFollowingSortValidator, - setDefaultSort, - setDefaultPagination, - asyncMiddleware(listFollowing) -) - -serverFollowsRouter.post('/following', - authenticate, - ensureUserHasRight(UserRight.MANAGE_SERVER_FOLLOW), - followValidator, - setBodyHostsPort, - asyncMiddleware(addFollow) -) - -serverFollowsRouter.delete('/following/:hostOrHandle', - authenticate, - ensureUserHasRight(UserRight.MANAGE_SERVER_FOLLOW), - asyncMiddleware(removeFollowingValidator), - asyncMiddleware(removeFollowing) -) - -serverFollowsRouter.get('/followers', - listFollowsValidator, - paginationValidator, - instanceFollowersSortValidator, - setDefaultSort, - setDefaultPagination, - asyncMiddleware(listFollowers) -) - -serverFollowsRouter.delete('/followers/:nameWithHost', - authenticate, - ensureUserHasRight(UserRight.MANAGE_SERVER_FOLLOW), - asyncMiddleware(getFollowerValidator), - asyncMiddleware(removeFollower) -) - -serverFollowsRouter.post('/followers/:nameWithHost/reject', - authenticate, - ensureUserHasRight(UserRight.MANAGE_SERVER_FOLLOW), - asyncMiddleware(getFollowerValidator), - rejectFollowerValidator, - asyncMiddleware(rejectFollower) -) - -serverFollowsRouter.post('/followers/:nameWithHost/accept', - authenticate, - ensureUserHasRight(UserRight.MANAGE_SERVER_FOLLOW), - asyncMiddleware(getFollowerValidator), - acceptFollowerValidator, - asyncMiddleware(acceptFollower) -) - -// --------------------------------------------------------------------------- - -export { - serverFollowsRouter -} - -// --------------------------------------------------------------------------- - -async function listFollowing (req: express.Request, res: express.Response) { - const serverActor = await getServerActor() - const resultList = await ActorFollowModel.listInstanceFollowingForApi({ - followerId: serverActor.id, - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - search: req.query.search, - actorType: req.query.actorType, - state: req.query.state - }) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function listFollowers (req: express.Request, res: express.Response) { - const serverActor = await getServerActor() - const resultList = await ActorFollowModel.listFollowersForApi({ - actorIds: [ serverActor.id ], - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - search: req.query.search, - actorType: req.query.actorType, - state: req.query.state - }) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function addFollow (req: express.Request, res: express.Response) { - const { hosts, handles } = req.body as ServerFollowCreate - const follower = await getServerActor() - - for (const host of hosts) { - const payload = { - host, - name: SERVER_ACTOR_NAME, - followerActorId: follower.id - } - - JobQueue.Instance.createJobAsync({ type: 'activitypub-follow', payload }) - } - - for (const handle of handles) { - const [ name, host ] = handle.split('@') - - const payload = { - host, - name, - followerActorId: follower.id - } - - JobQueue.Instance.createJobAsync({ type: 'activitypub-follow', payload }) - } - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function removeFollowing (req: express.Request, res: express.Response) { - const follow = res.locals.follow - - await sequelizeTypescript.transaction(async t => { - if (follow.state === 'accepted') sendUndoFollow(follow, t) - - // Disable redundancy on unfollowed instances - const server = follow.ActorFollowing.Server - server.redundancyAllowed = false - await server.save({ transaction: t }) - - // Async, could be long - removeRedundanciesOfServer(server.id) - .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, { err })) - - await follow.destroy({ transaction: t }) - }) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function rejectFollower (req: express.Request, res: express.Response) { - const follow = res.locals.follow - - follow.state = 'rejected' - await follow.save() - - sendReject(follow.url, follow.ActorFollower, follow.ActorFollowing) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function removeFollower (req: express.Request, res: express.Response) { - const follow = res.locals.follow - - if (follow.state === 'accepted' || follow.state === 'pending') { - sendReject(follow.url, follow.ActorFollower, follow.ActorFollowing) - } - - await follow.destroy() - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function acceptFollower (req: express.Request, res: express.Response) { - const follow = res.locals.follow - - sendAccept(follow) - - follow.state = 'accepted' - await follow.save() - - await autoFollowBackIfNeeded(follow) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} diff --git a/server/controllers/api/server/index.ts b/server/controllers/api/server/index.ts deleted file mode 100644 index 57f7d601c..000000000 --- a/server/controllers/api/server/index.ts +++ /dev/null @@ -1,27 +0,0 @@ -import express from 'express' -import { apiRateLimiter } from '@server/middlewares' -import { contactRouter } from './contact' -import { debugRouter } from './debug' -import { serverFollowsRouter } from './follows' -import { logsRouter } from './logs' -import { serverRedundancyRouter } from './redundancy' -import { serverBlocklistRouter } from './server-blocklist' -import { statsRouter } from './stats' - -const serverRouter = express.Router() - -serverRouter.use(apiRateLimiter) - -serverRouter.use('/', serverFollowsRouter) -serverRouter.use('/', serverRedundancyRouter) -serverRouter.use('/', statsRouter) -serverRouter.use('/', serverBlocklistRouter) -serverRouter.use('/', contactRouter) -serverRouter.use('/', logsRouter) -serverRouter.use('/', debugRouter) - -// --------------------------------------------------------------------------- - -export { - serverRouter -} diff --git a/server/controllers/api/server/logs.ts b/server/controllers/api/server/logs.ts deleted file mode 100644 index ed0aa6e8e..000000000 --- a/server/controllers/api/server/logs.ts +++ /dev/null @@ -1,203 +0,0 @@ -import express from 'express' -import { readdir, readFile } from 'fs-extra' -import { join } from 'path' -import { isArray } from '@server/helpers/custom-validators/misc' -import { logger, mtimeSortFilesDesc } from '@server/helpers/logger' -import { pick } from '@shared/core-utils' -import { ClientLogCreate, HttpStatusCode } from '@shared/models' -import { ServerLogLevel } from '../../../../shared/models/server/server-log-level.type' -import { UserRight } from '../../../../shared/models/users' -import { CONFIG } from '../../../initializers/config' -import { AUDIT_LOG_FILENAME, LOG_FILENAME, MAX_LOGS_OUTPUT_CHARACTERS } from '../../../initializers/constants' -import { asyncMiddleware, authenticate, buildRateLimiter, ensureUserHasRight, optionalAuthenticate } from '../../../middlewares' -import { createClientLogValidator, getAuditLogsValidator, getLogsValidator } from '../../../middlewares/validators/logs' - -const createClientLogRateLimiter = buildRateLimiter({ - windowMs: CONFIG.RATES_LIMIT.RECEIVE_CLIENT_LOG.WINDOW_MS, - max: CONFIG.RATES_LIMIT.RECEIVE_CLIENT_LOG.MAX -}) - -const logsRouter = express.Router() - -logsRouter.post('/logs/client', - createClientLogRateLimiter, - optionalAuthenticate, - createClientLogValidator, - createClientLog -) - -logsRouter.get('/logs', - authenticate, - ensureUserHasRight(UserRight.MANAGE_LOGS), - getLogsValidator, - asyncMiddleware(getLogs) -) - -logsRouter.get('/audit-logs', - authenticate, - ensureUserHasRight(UserRight.MANAGE_LOGS), - getAuditLogsValidator, - asyncMiddleware(getAuditLogs) -) - -// --------------------------------------------------------------------------- - -export { - logsRouter -} - -// --------------------------------------------------------------------------- - -function createClientLog (req: express.Request, res: express.Response) { - const logInfo = req.body as ClientLogCreate - - const meta = { - tags: [ 'client' ], - username: res.locals.oauth?.token?.User?.username, - - ...pick(logInfo, [ 'userAgent', 'stackTrace', 'meta', 'url' ]) - } - - logger.log(logInfo.level, `Client log: ${logInfo.message}`, meta) - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -const auditLogNameFilter = generateLogNameFilter(AUDIT_LOG_FILENAME) -async function getAuditLogs (req: express.Request, res: express.Response) { - const output = await generateOutput({ - startDateQuery: req.query.startDate, - endDateQuery: req.query.endDate, - level: 'audit', - nameFilter: auditLogNameFilter - }) - - return res.json(output).end() -} - -const logNameFilter = generateLogNameFilter(LOG_FILENAME) -async function getLogs (req: express.Request, res: express.Response) { - const output = await generateOutput({ - startDateQuery: req.query.startDate, - endDateQuery: req.query.endDate, - level: req.query.level || 'info', - tagsOneOf: req.query.tagsOneOf, - nameFilter: logNameFilter - }) - - return res.json(output) -} - -async function generateOutput (options: { - startDateQuery: string - endDateQuery?: string - - level: ServerLogLevel - nameFilter: RegExp - tagsOneOf?: string[] -}) { - const { startDateQuery, level, nameFilter } = options - - const tagsOneOf = Array.isArray(options.tagsOneOf) && options.tagsOneOf.length !== 0 - ? new Set(options.tagsOneOf) - : undefined - - const logFiles = await readdir(CONFIG.STORAGE.LOG_DIR) - const sortedLogFiles = await mtimeSortFilesDesc(logFiles, CONFIG.STORAGE.LOG_DIR) - let currentSize = 0 - - const startDate = new Date(startDateQuery) - const endDate = options.endDateQuery ? new Date(options.endDateQuery) : new Date() - - let output: string[] = [] - - for (const meta of sortedLogFiles) { - if (nameFilter.exec(meta.file) === null) continue - - const path = join(CONFIG.STORAGE.LOG_DIR, meta.file) - logger.debug('Opening %s to fetch logs.', path) - - const result = await getOutputFromFile({ path, startDate, endDate, level, currentSize, tagsOneOf }) - if (!result.output) break - - output = result.output.concat(output) - currentSize = result.currentSize - - if (currentSize > MAX_LOGS_OUTPUT_CHARACTERS || (result.logTime && result.logTime < startDate.getTime())) break - } - - return output -} - -async function getOutputFromFile (options: { - path: string - startDate: Date - endDate: Date - level: ServerLogLevel - currentSize: number - tagsOneOf: Set -}) { - const { path, startDate, endDate, level, tagsOneOf } = options - - const startTime = startDate.getTime() - const endTime = endDate.getTime() - let currentSize = options.currentSize - - let logTime: number - - const logsLevel: { [ id in ServerLogLevel ]: number } = { - audit: -1, - debug: 0, - info: 1, - warn: 2, - error: 3 - } - - const content = await readFile(path) - const lines = content.toString().split('\n') - const output: any[] = [] - - for (let i = lines.length - 1; i >= 0; i--) { - const line = lines[i] - let log: any - - try { - log = JSON.parse(line) - } catch { - // Maybe there a multiple \n at the end of the file - continue - } - - logTime = new Date(log.timestamp).getTime() - if ( - logTime >= startTime && - logTime <= endTime && - logsLevel[log.level] >= logsLevel[level] && - (!tagsOneOf || lineHasTag(log, tagsOneOf)) - ) { - output.push(log) - - currentSize += line.length - - if (currentSize > MAX_LOGS_OUTPUT_CHARACTERS) break - } else if (logTime < startTime) { - break - } - } - - return { currentSize, output: output.reverse(), logTime } -} - -function lineHasTag (line: { tags?: string }, tagsOneOf: Set) { - if (!isArray(line.tags)) return false - - for (const lineTag of line.tags) { - if (tagsOneOf.has(lineTag)) return true - } - - return false -} - -function generateLogNameFilter (baseName: string) { - return new RegExp('^' + baseName.replace(/\.log$/, '') + '\\d*.log$') -} diff --git a/server/controllers/api/server/redundancy.ts b/server/controllers/api/server/redundancy.ts deleted file mode 100644 index 94e187cd4..000000000 --- a/server/controllers/api/server/redundancy.ts +++ /dev/null @@ -1,116 +0,0 @@ -import express from 'express' -import { JobQueue } from '@server/lib/job-queue' -import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy' -import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' -import { UserRight } from '../../../../shared/models/users' -import { logger } from '../../../helpers/logger' -import { removeRedundanciesOfServer, removeVideoRedundancy } from '../../../lib/redundancy' -import { - asyncMiddleware, - authenticate, - ensureUserHasRight, - paginationValidator, - setDefaultPagination, - setDefaultVideoRedundanciesSort, - videoRedundanciesSortValidator -} from '../../../middlewares' -import { - addVideoRedundancyValidator, - listVideoRedundanciesValidator, - removeVideoRedundancyValidator, - updateServerRedundancyValidator -} from '../../../middlewares/validators/redundancy' - -const serverRedundancyRouter = express.Router() - -serverRedundancyRouter.put('/redundancy/:host', - authenticate, - ensureUserHasRight(UserRight.MANAGE_SERVER_FOLLOW), - asyncMiddleware(updateServerRedundancyValidator), - asyncMiddleware(updateRedundancy) -) - -serverRedundancyRouter.get('/redundancy/videos', - authenticate, - ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES), - listVideoRedundanciesValidator, - paginationValidator, - videoRedundanciesSortValidator, - setDefaultVideoRedundanciesSort, - setDefaultPagination, - asyncMiddleware(listVideoRedundancies) -) - -serverRedundancyRouter.post('/redundancy/videos', - authenticate, - ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES), - addVideoRedundancyValidator, - asyncMiddleware(addVideoRedundancy) -) - -serverRedundancyRouter.delete('/redundancy/videos/:redundancyId', - authenticate, - ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES), - removeVideoRedundancyValidator, - asyncMiddleware(removeVideoRedundancyController) -) - -// --------------------------------------------------------------------------- - -export { - serverRedundancyRouter -} - -// --------------------------------------------------------------------------- - -async function listVideoRedundancies (req: express.Request, res: express.Response) { - const resultList = await VideoRedundancyModel.listForApi({ - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - target: req.query.target, - strategy: req.query.strategy - }) - - const result = { - total: resultList.total, - data: resultList.data.map(r => VideoRedundancyModel.toFormattedJSONStatic(r)) - } - - return res.json(result) -} - -async function addVideoRedundancy (req: express.Request, res: express.Response) { - const payload = { - videoId: res.locals.onlyVideo.id - } - - await JobQueue.Instance.createJob({ - type: 'video-redundancy', - payload - }) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function removeVideoRedundancyController (req: express.Request, res: express.Response) { - await removeVideoRedundancy(res.locals.videoRedundancy) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function updateRedundancy (req: express.Request, res: express.Response) { - const server = res.locals.server - - server.redundancyAllowed = req.body.redundancyAllowed - - await server.save() - - if (server.redundancyAllowed !== true) { - // Async, could be long - removeRedundanciesOfServer(server.id) - .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, { err })) - } - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} diff --git a/server/controllers/api/server/server-blocklist.ts b/server/controllers/api/server/server-blocklist.ts deleted file mode 100644 index 740f95da3..000000000 --- a/server/controllers/api/server/server-blocklist.ts +++ /dev/null @@ -1,158 +0,0 @@ -import 'multer' -import express from 'express' -import { logger } from '@server/helpers/logger' -import { getServerActor } from '@server/models/application/application' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' -import { UserRight } from '../../../../shared/models/users' -import { getFormattedObjects } from '../../../helpers/utils' -import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../../../lib/blocklist' -import { - asyncMiddleware, - asyncRetryTransactionMiddleware, - authenticate, - ensureUserHasRight, - paginationValidator, - setDefaultPagination, - setDefaultSort -} from '../../../middlewares' -import { - accountsBlocklistSortValidator, - blockAccountValidator, - blockServerValidator, - serversBlocklistSortValidator, - unblockAccountByServerValidator, - unblockServerByServerValidator -} from '../../../middlewares/validators' -import { AccountBlocklistModel } from '../../../models/account/account-blocklist' -import { ServerBlocklistModel } from '../../../models/server/server-blocklist' - -const serverBlocklistRouter = express.Router() - -serverBlocklistRouter.get('/blocklist/accounts', - authenticate, - ensureUserHasRight(UserRight.MANAGE_ACCOUNTS_BLOCKLIST), - paginationValidator, - accountsBlocklistSortValidator, - setDefaultSort, - setDefaultPagination, - asyncMiddleware(listBlockedAccounts) -) - -serverBlocklistRouter.post('/blocklist/accounts', - authenticate, - ensureUserHasRight(UserRight.MANAGE_ACCOUNTS_BLOCKLIST), - asyncMiddleware(blockAccountValidator), - asyncRetryTransactionMiddleware(blockAccount) -) - -serverBlocklistRouter.delete('/blocklist/accounts/:accountName', - authenticate, - ensureUserHasRight(UserRight.MANAGE_ACCOUNTS_BLOCKLIST), - asyncMiddleware(unblockAccountByServerValidator), - asyncRetryTransactionMiddleware(unblockAccount) -) - -serverBlocklistRouter.get('/blocklist/servers', - authenticate, - ensureUserHasRight(UserRight.MANAGE_SERVERS_BLOCKLIST), - paginationValidator, - serversBlocklistSortValidator, - setDefaultSort, - setDefaultPagination, - asyncMiddleware(listBlockedServers) -) - -serverBlocklistRouter.post('/blocklist/servers', - authenticate, - ensureUserHasRight(UserRight.MANAGE_SERVERS_BLOCKLIST), - asyncMiddleware(blockServerValidator), - asyncRetryTransactionMiddleware(blockServer) -) - -serverBlocklistRouter.delete('/blocklist/servers/:host', - authenticate, - ensureUserHasRight(UserRight.MANAGE_SERVERS_BLOCKLIST), - asyncMiddleware(unblockServerByServerValidator), - asyncRetryTransactionMiddleware(unblockServer) -) - -export { - serverBlocklistRouter -} - -// --------------------------------------------------------------------------- - -async function listBlockedAccounts (req: express.Request, res: express.Response) { - const serverActor = await getServerActor() - - const resultList = await AccountBlocklistModel.listForApi({ - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - search: req.query.search, - accountId: serverActor.Account.id - }) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function blockAccount (req: express.Request, res: express.Response) { - const serverActor = await getServerActor() - const accountToBlock = res.locals.account - - await addAccountInBlocklist(serverActor.Account.id, accountToBlock.id) - - UserNotificationModel.removeNotificationsOf({ - id: accountToBlock.id, - type: 'account', - forUserId: null // For all users - }).catch(err => logger.error('Cannot remove notifications after an account mute.', { err })) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function unblockAccount (req: express.Request, res: express.Response) { - const accountBlock = res.locals.accountBlock - - await removeAccountFromBlocklist(accountBlock) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function listBlockedServers (req: express.Request, res: express.Response) { - const serverActor = await getServerActor() - - const resultList = await ServerBlocklistModel.listForApi({ - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - search: req.query.search, - accountId: serverActor.Account.id - }) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function blockServer (req: express.Request, res: express.Response) { - const serverActor = await getServerActor() - const serverToBlock = res.locals.server - - await addServerInBlocklist(serverActor.Account.id, serverToBlock.id) - - UserNotificationModel.removeNotificationsOf({ - id: serverToBlock.id, - type: 'server', - forUserId: null // For all users - }).catch(err => logger.error('Cannot remove notifications after a server mute.', { err })) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function unblockServer (req: express.Request, res: express.Response) { - const serverBlock = res.locals.serverBlock - - await removeServerFromBlocklist(serverBlock) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} diff --git a/server/controllers/api/server/stats.ts b/server/controllers/api/server/stats.ts deleted file mode 100644 index 2ab398f4d..000000000 --- a/server/controllers/api/server/stats.ts +++ /dev/null @@ -1,26 +0,0 @@ -import express from 'express' -import { StatsManager } from '@server/lib/stat-manager' -import { ROUTE_CACHE_LIFETIME } from '../../../initializers/constants' -import { asyncMiddleware } from '../../../middlewares' -import { cacheRoute } from '../../../middlewares/cache/cache' -import { Hooks } from '@server/lib/plugins/hooks' - -const statsRouter = express.Router() - -statsRouter.get('/stats', - cacheRoute(ROUTE_CACHE_LIFETIME.STATS), - asyncMiddleware(getStats) -) - -async function getStats (_req: express.Request, res: express.Response) { - let data = await StatsManager.Instance.getStats() - data = await Hooks.wrapObject(data, 'filter:api.server.stats.get.result') - - return res.json(data) -} - -// --------------------------------------------------------------------------- - -export { - statsRouter -} diff --git a/server/controllers/api/users/email-verification.ts b/server/controllers/api/users/email-verification.ts deleted file mode 100644 index 230aaa9af..000000000 --- a/server/controllers/api/users/email-verification.ts +++ /dev/null @@ -1,72 +0,0 @@ -import express from 'express' -import { HttpStatusCode } from '@shared/models' -import { CONFIG } from '../../../initializers/config' -import { sendVerifyRegistrationEmail, sendVerifyUserEmail } from '../../../lib/user' -import { asyncMiddleware, buildRateLimiter } from '../../../middlewares' -import { - registrationVerifyEmailValidator, - usersAskSendVerifyEmailValidator, - usersVerifyEmailValidator -} from '../../../middlewares/validators' - -const askSendEmailLimiter = buildRateLimiter({ - windowMs: CONFIG.RATES_LIMIT.ASK_SEND_EMAIL.WINDOW_MS, - max: CONFIG.RATES_LIMIT.ASK_SEND_EMAIL.MAX -}) - -const emailVerificationRouter = express.Router() - -emailVerificationRouter.post([ '/ask-send-verify-email', '/registrations/ask-send-verify-email' ], - askSendEmailLimiter, - asyncMiddleware(usersAskSendVerifyEmailValidator), - asyncMiddleware(reSendVerifyUserEmail) -) - -emailVerificationRouter.post('/:id/verify-email', - asyncMiddleware(usersVerifyEmailValidator), - asyncMiddleware(verifyUserEmail) -) - -emailVerificationRouter.post('/registrations/:registrationId/verify-email', - asyncMiddleware(registrationVerifyEmailValidator), - asyncMiddleware(verifyRegistrationEmail) -) - -// --------------------------------------------------------------------------- - -export { - emailVerificationRouter -} - -async function reSendVerifyUserEmail (req: express.Request, res: express.Response) { - const user = res.locals.user - const registration = res.locals.userRegistration - - if (user) await sendVerifyUserEmail(user) - else if (registration) await sendVerifyRegistrationEmail(registration) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function verifyUserEmail (req: express.Request, res: express.Response) { - const user = res.locals.user - user.emailVerified = true - - if (req.body.isPendingEmail === true) { - user.email = user.pendingEmail - user.pendingEmail = null - } - - await user.save() - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function verifyRegistrationEmail (req: express.Request, res: express.Response) { - const registration = res.locals.userRegistration - registration.emailVerified = true - - await registration.save() - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} diff --git a/server/controllers/api/users/index.ts b/server/controllers/api/users/index.ts deleted file mode 100644 index 5eac6fd0f..000000000 --- a/server/controllers/api/users/index.ts +++ /dev/null @@ -1,319 +0,0 @@ -import express from 'express' -import { tokensRouter } from '@server/controllers/api/users/token' -import { Hooks } from '@server/lib/plugins/hooks' -import { OAuthTokenModel } from '@server/models/oauth/oauth-token' -import { MUserAccountDefault } from '@server/types/models' -import { pick } from '@shared/core-utils' -import { HttpStatusCode, UserCreate, UserCreateResult, UserRight, UserUpdate } from '@shared/models' -import { auditLoggerFactory, getAuditIdFromRes, UserAuditView } from '../../../helpers/audit-logger' -import { logger } from '../../../helpers/logger' -import { generateRandomString, getFormattedObjects } from '../../../helpers/utils' -import { WEBSERVER } from '../../../initializers/constants' -import { sequelizeTypescript } from '../../../initializers/database' -import { Emailer } from '../../../lib/emailer' -import { Redis } from '../../../lib/redis' -import { buildUser, createUserAccountAndChannelAndPlaylist } from '../../../lib/user' -import { - adminUsersSortValidator, - apiRateLimiter, - asyncMiddleware, - asyncRetryTransactionMiddleware, - authenticate, - ensureUserHasRight, - paginationValidator, - setDefaultPagination, - setDefaultSort, - userAutocompleteValidator, - usersAddValidator, - usersGetValidator, - usersListValidator, - usersRemoveValidator, - usersUpdateValidator -} from '../../../middlewares' -import { - ensureCanModerateUser, - usersAskResetPasswordValidator, - usersBlockingValidator, - usersResetPasswordValidator -} from '../../../middlewares/validators' -import { UserModel } from '../../../models/user/user' -import { emailVerificationRouter } from './email-verification' -import { meRouter } from './me' -import { myAbusesRouter } from './my-abuses' -import { myBlocklistRouter } from './my-blocklist' -import { myVideosHistoryRouter } from './my-history' -import { myNotificationsRouter } from './my-notifications' -import { mySubscriptionsRouter } from './my-subscriptions' -import { myVideoPlaylistsRouter } from './my-video-playlists' -import { registrationsRouter } from './registrations' -import { twoFactorRouter } from './two-factor' - -const auditLogger = auditLoggerFactory('users') - -const usersRouter = express.Router() - -usersRouter.use(apiRateLimiter) - -usersRouter.use('/', emailVerificationRouter) -usersRouter.use('/', registrationsRouter) -usersRouter.use('/', twoFactorRouter) -usersRouter.use('/', tokensRouter) -usersRouter.use('/', myNotificationsRouter) -usersRouter.use('/', mySubscriptionsRouter) -usersRouter.use('/', myBlocklistRouter) -usersRouter.use('/', myVideosHistoryRouter) -usersRouter.use('/', myVideoPlaylistsRouter) -usersRouter.use('/', myAbusesRouter) -usersRouter.use('/', meRouter) - -usersRouter.get('/autocomplete', - userAutocompleteValidator, - asyncMiddleware(autocompleteUsers) -) - -usersRouter.get('/', - authenticate, - ensureUserHasRight(UserRight.MANAGE_USERS), - paginationValidator, - adminUsersSortValidator, - setDefaultSort, - setDefaultPagination, - usersListValidator, - asyncMiddleware(listUsers) -) - -usersRouter.post('/:id/block', - authenticate, - ensureUserHasRight(UserRight.MANAGE_USERS), - asyncMiddleware(usersBlockingValidator), - ensureCanModerateUser, - asyncMiddleware(blockUser) -) -usersRouter.post('/:id/unblock', - authenticate, - ensureUserHasRight(UserRight.MANAGE_USERS), - asyncMiddleware(usersBlockingValidator), - ensureCanModerateUser, - asyncMiddleware(unblockUser) -) - -usersRouter.get('/:id', - authenticate, - ensureUserHasRight(UserRight.MANAGE_USERS), - asyncMiddleware(usersGetValidator), - getUser -) - -usersRouter.post('/', - authenticate, - ensureUserHasRight(UserRight.MANAGE_USERS), - asyncMiddleware(usersAddValidator), - asyncRetryTransactionMiddleware(createUser) -) - -usersRouter.put('/:id', - authenticate, - ensureUserHasRight(UserRight.MANAGE_USERS), - asyncMiddleware(usersUpdateValidator), - ensureCanModerateUser, - asyncMiddleware(updateUser) -) - -usersRouter.delete('/:id', - authenticate, - ensureUserHasRight(UserRight.MANAGE_USERS), - asyncMiddleware(usersRemoveValidator), - ensureCanModerateUser, - asyncMiddleware(removeUser) -) - -usersRouter.post('/ask-reset-password', - asyncMiddleware(usersAskResetPasswordValidator), - asyncMiddleware(askResetUserPassword) -) - -usersRouter.post('/:id/reset-password', - asyncMiddleware(usersResetPasswordValidator), - asyncMiddleware(resetUserPassword) -) - -// --------------------------------------------------------------------------- - -export { - usersRouter -} - -// --------------------------------------------------------------------------- - -async function createUser (req: express.Request, res: express.Response) { - const body: UserCreate = req.body - - const userToCreate = buildUser({ - ...pick(body, [ 'username', 'password', 'email', 'role', 'videoQuota', 'videoQuotaDaily', 'adminFlags' ]), - - emailVerified: null - }) - - // NB: due to the validator usersAddValidator, password==='' can only be true if we can send the mail. - const createPassword = userToCreate.password === '' - if (createPassword) { - userToCreate.password = await generateRandomString(20) - } - - const { user, account, videoChannel } = await createUserAccountAndChannelAndPlaylist({ - userToCreate, - channelNames: body.channelName && { name: body.channelName, displayName: body.channelName } - }) - - auditLogger.create(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON())) - logger.info('User %s with its channel and account created.', body.username) - - if (createPassword) { - // this will send an email for newly created users, so then can set their first password. - logger.info('Sending to user %s a create password email', body.username) - const verificationString = await Redis.Instance.setCreatePasswordVerificationString(user.id) - const url = WEBSERVER.URL + '/reset-password?userId=' + user.id + '&verificationString=' + verificationString - Emailer.Instance.addPasswordCreateEmailJob(userToCreate.username, user.email, url) - } - - Hooks.runAction('action:api.user.created', { body, user, account, videoChannel, req, res }) - - return res.json({ - user: { - id: user.id, - account: { - id: account.id - } - } as UserCreateResult - }) -} - -async function unblockUser (req: express.Request, res: express.Response) { - const user = res.locals.user - - await changeUserBlock(res, user, false) - - Hooks.runAction('action:api.user.unblocked', { user, req, res }) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function blockUser (req: express.Request, res: express.Response) { - const user = res.locals.user - const reason = req.body.reason - - await changeUserBlock(res, user, true, reason) - - Hooks.runAction('action:api.user.blocked', { user, req, res }) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -function getUser (req: express.Request, res: express.Response) { - return res.json(res.locals.user.toFormattedJSON({ withAdminFlags: true })) -} - -async function autocompleteUsers (req: express.Request, res: express.Response) { - const resultList = await UserModel.autoComplete(req.query.search as string) - - return res.json(resultList) -} - -async function listUsers (req: express.Request, res: express.Response) { - const resultList = await UserModel.listForAdminApi({ - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - search: req.query.search, - blocked: req.query.blocked - }) - - return res.json(getFormattedObjects(resultList.data, resultList.total, { withAdminFlags: true })) -} - -async function removeUser (req: express.Request, res: express.Response) { - const user = res.locals.user - - auditLogger.delete(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON())) - - await sequelizeTypescript.transaction(async t => { - // Use a transaction to avoid inconsistencies with hooks (account/channel deletion & federation) - await user.destroy({ transaction: t }) - }) - - Hooks.runAction('action:api.user.deleted', { user, req, res }) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function updateUser (req: express.Request, res: express.Response) { - const body: UserUpdate = req.body - const userToUpdate = res.locals.user - const oldUserAuditView = new UserAuditView(userToUpdate.toFormattedJSON()) - const roleChanged = body.role !== undefined && body.role !== userToUpdate.role - - const keysToUpdate: (keyof UserUpdate)[] = [ - 'password', - 'email', - 'emailVerified', - 'videoQuota', - 'videoQuotaDaily', - 'role', - 'adminFlags', - 'pluginAuth' - ] - - for (const key of keysToUpdate) { - if (body[key] !== undefined) userToUpdate.set(key, body[key]) - } - - const user = await userToUpdate.save() - - // Destroy user token to refresh rights - if (roleChanged || body.password !== undefined) await OAuthTokenModel.deleteUserToken(userToUpdate.id) - - auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView) - - Hooks.runAction('action:api.user.updated', { user, req, res }) - - // Don't need to send this update to followers, these attributes are not federated - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function askResetUserPassword (req: express.Request, res: express.Response) { - const user = res.locals.user - - const verificationString = await Redis.Instance.setResetPasswordVerificationString(user.id) - const url = WEBSERVER.URL + '/reset-password?userId=' + user.id + '&verificationString=' + verificationString - Emailer.Instance.addPasswordResetEmailJob(user.username, user.email, url) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function resetUserPassword (req: express.Request, res: express.Response) { - const user = res.locals.user - user.password = req.body.password - - await user.save() - await Redis.Instance.removePasswordVerificationString(user.id) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function changeUserBlock (res: express.Response, user: MUserAccountDefault, block: boolean, reason?: string) { - const oldUserAuditView = new UserAuditView(user.toFormattedJSON()) - - user.blocked = block - user.blockedReason = reason || null - - await sequelizeTypescript.transaction(async t => { - await OAuthTokenModel.deleteUserToken(user.id, t) - - await user.save({ transaction: t }) - }) - - Emailer.Instance.addUserBlockJob(user, block, reason) - - auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView) -} diff --git a/server/controllers/api/users/me.ts b/server/controllers/api/users/me.ts deleted file mode 100644 index 26811136e..000000000 --- a/server/controllers/api/users/me.ts +++ /dev/null @@ -1,277 +0,0 @@ -import 'multer' -import express from 'express' -import { auditLoggerFactory, getAuditIdFromRes, UserAuditView } from '@server/helpers/audit-logger' -import { Hooks } from '@server/lib/plugins/hooks' -import { pick } from '@shared/core-utils' -import { ActorImageType, HttpStatusCode, UserUpdateMe, UserVideoQuota, UserVideoRate as FormattedUserVideoRate } from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { createReqFiles } from '../../../helpers/express-utils' -import { getFormattedObjects } from '../../../helpers/utils' -import { CONFIG } from '../../../initializers/config' -import { MIMETYPES } from '../../../initializers/constants' -import { sequelizeTypescript } from '../../../initializers/database' -import { sendUpdateActor } from '../../../lib/activitypub/send' -import { deleteLocalActorImageFile, updateLocalActorImageFiles } from '../../../lib/local-actor' -import { getOriginalVideoFileTotalDailyFromUser, getOriginalVideoFileTotalFromUser, sendVerifyUserEmail } from '../../../lib/user' -import { - asyncMiddleware, - asyncRetryTransactionMiddleware, - authenticate, - paginationValidator, - setDefaultPagination, - setDefaultSort, - setDefaultVideosSort, - usersUpdateMeValidator, - usersVideoRatingValidator -} from '../../../middlewares' -import { - deleteMeValidator, - getMyVideoImportsValidator, - usersVideosValidator, - videoImportsSortValidator, - videosSortValidator -} from '../../../middlewares/validators' -import { updateAvatarValidator } from '../../../middlewares/validators/actor-image' -import { AccountModel } from '../../../models/account/account' -import { AccountVideoRateModel } from '../../../models/account/account-video-rate' -import { UserModel } from '../../../models/user/user' -import { VideoModel } from '../../../models/video/video' -import { VideoImportModel } from '../../../models/video/video-import' - -const auditLogger = auditLoggerFactory('users') - -const reqAvatarFile = createReqFiles([ 'avatarfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT) - -const meRouter = express.Router() - -meRouter.get('/me', - authenticate, - asyncMiddleware(getUserInformation) -) -meRouter.delete('/me', - authenticate, - deleteMeValidator, - asyncMiddleware(deleteMe) -) - -meRouter.get('/me/video-quota-used', - authenticate, - asyncMiddleware(getUserVideoQuotaUsed) -) - -meRouter.get('/me/videos/imports', - authenticate, - paginationValidator, - videoImportsSortValidator, - setDefaultSort, - setDefaultPagination, - getMyVideoImportsValidator, - asyncMiddleware(getUserVideoImports) -) - -meRouter.get('/me/videos', - authenticate, - paginationValidator, - videosSortValidator, - setDefaultVideosSort, - setDefaultPagination, - asyncMiddleware(usersVideosValidator), - asyncMiddleware(getUserVideos) -) - -meRouter.get('/me/videos/:videoId/rating', - authenticate, - asyncMiddleware(usersVideoRatingValidator), - asyncMiddleware(getUserVideoRating) -) - -meRouter.put('/me', - authenticate, - asyncMiddleware(usersUpdateMeValidator), - asyncRetryTransactionMiddleware(updateMe) -) - -meRouter.post('/me/avatar/pick', - authenticate, - reqAvatarFile, - updateAvatarValidator, - asyncRetryTransactionMiddleware(updateMyAvatar) -) - -meRouter.delete('/me/avatar', - authenticate, - asyncRetryTransactionMiddleware(deleteMyAvatar) -) - -// --------------------------------------------------------------------------- - -export { - meRouter -} - -// --------------------------------------------------------------------------- - -async function getUserVideos (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.User - - const apiOptions = await Hooks.wrapObject({ - accountId: user.Account.id, - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - search: req.query.search, - channelId: res.locals.videoChannel?.id, - isLive: req.query.isLive - }, 'filter:api.user.me.videos.list.params') - - const resultList = await Hooks.wrapPromiseFun( - VideoModel.listUserVideosForApi, - apiOptions, - 'filter:api.user.me.videos.list.result' - ) - - const additionalAttributes = { - waitTranscoding: true, - state: true, - scheduledUpdate: true, - blacklistInfo: true - } - return res.json(getFormattedObjects(resultList.data, resultList.total, { additionalAttributes })) -} - -async function getUserVideoImports (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.User - const resultList = await VideoImportModel.listUserVideoImportsForApi({ - userId: user.id, - - ...pick(req.query, [ 'targetUrl', 'start', 'count', 'sort', 'search', 'videoChannelSyncId' ]) - }) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function getUserInformation (req: express.Request, res: express.Response) { - // We did not load channels in res.locals.user - const user = await UserModel.loadForMeAPI(res.locals.oauth.token.user.id) - - return res.json(user.toMeFormattedJSON()) -} - -async function getUserVideoQuotaUsed (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.user - const videoQuotaUsed = await getOriginalVideoFileTotalFromUser(user) - const videoQuotaUsedDaily = await getOriginalVideoFileTotalDailyFromUser(user) - - const data: UserVideoQuota = { - videoQuotaUsed, - videoQuotaUsedDaily - } - return res.json(data) -} - -async function getUserVideoRating (req: express.Request, res: express.Response) { - const videoId = res.locals.videoId.id - const accountId = +res.locals.oauth.token.User.Account.id - - const ratingObj = await AccountVideoRateModel.load(accountId, videoId, null) - const rating = ratingObj ? ratingObj.type : 'none' - - const json: FormattedUserVideoRate = { - videoId, - rating - } - return res.json(json) -} - -async function deleteMe (req: express.Request, res: express.Response) { - const user = await UserModel.loadByIdWithChannels(res.locals.oauth.token.User.id) - - auditLogger.delete(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON())) - - await user.destroy() - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function updateMe (req: express.Request, res: express.Response) { - const body: UserUpdateMe = req.body - let sendVerificationEmail = false - - const user = res.locals.oauth.token.user - - const keysToUpdate: (keyof UserUpdateMe & keyof AttributesOnly)[] = [ - 'password', - 'nsfwPolicy', - 'p2pEnabled', - 'autoPlayVideo', - 'autoPlayNextVideo', - 'autoPlayNextVideoPlaylist', - 'videosHistoryEnabled', - 'videoLanguages', - 'theme', - 'noInstanceConfigWarningModal', - 'noAccountSetupWarningModal', - 'noWelcomeModal', - 'emailPublic', - 'p2pEnabled' - ] - - for (const key of keysToUpdate) { - if (body[key] !== undefined) user.set(key, body[key]) - } - - if (body.email !== undefined) { - if (CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION) { - user.pendingEmail = body.email - sendVerificationEmail = true - } else { - user.email = body.email - } - } - - await sequelizeTypescript.transaction(async t => { - await user.save({ transaction: t }) - - if (body.displayName === undefined && body.description === undefined) return - - const userAccount = await AccountModel.load(user.Account.id, t) - - if (body.displayName !== undefined) userAccount.name = body.displayName - if (body.description !== undefined) userAccount.description = body.description - await userAccount.save({ transaction: t }) - - await sendUpdateActor(userAccount, t) - }) - - if (sendVerificationEmail === true) { - await sendVerifyUserEmail(user, true) - } - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function updateMyAvatar (req: express.Request, res: express.Response) { - const avatarPhysicalFile = req.files['avatarfile'][0] - const user = res.locals.oauth.token.user - - const userAccount = await AccountModel.load(user.Account.id) - - const avatars = await updateLocalActorImageFiles( - userAccount, - avatarPhysicalFile, - ActorImageType.AVATAR - ) - - return res.json({ - avatars: avatars.map(avatar => avatar.toFormattedJSON()) - }) -} - -async function deleteMyAvatar (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.user - - const userAccount = await AccountModel.load(user.Account.id) - await deleteLocalActorImageFile(userAccount, ActorImageType.AVATAR) - - return res.json({ avatars: [] }) -} diff --git a/server/controllers/api/users/my-abuses.ts b/server/controllers/api/users/my-abuses.ts deleted file mode 100644 index 103c3d332..000000000 --- a/server/controllers/api/users/my-abuses.ts +++ /dev/null @@ -1,48 +0,0 @@ -import express from 'express' -import { AbuseModel } from '@server/models/abuse/abuse' -import { - abuseListForUserValidator, - abusesSortValidator, - asyncMiddleware, - authenticate, - paginationValidator, - setDefaultPagination, - setDefaultSort -} from '../../../middlewares' - -const myAbusesRouter = express.Router() - -myAbusesRouter.get('/me/abuses', - authenticate, - paginationValidator, - abusesSortValidator, - setDefaultSort, - setDefaultPagination, - abuseListForUserValidator, - asyncMiddleware(listMyAbuses) -) - -// --------------------------------------------------------------------------- - -export { - myAbusesRouter -} - -// --------------------------------------------------------------------------- - -async function listMyAbuses (req: express.Request, res: express.Response) { - const resultList = await AbuseModel.listForUserApi({ - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - id: req.query.id, - search: req.query.search, - state: req.query.state, - user: res.locals.oauth.token.User - }) - - return res.json({ - total: resultList.total, - data: resultList.data.map(d => d.toFormattedUserJSON()) - }) -} diff --git a/server/controllers/api/users/my-blocklist.ts b/server/controllers/api/users/my-blocklist.ts deleted file mode 100644 index 0b56645cf..000000000 --- a/server/controllers/api/users/my-blocklist.ts +++ /dev/null @@ -1,149 +0,0 @@ -import 'multer' -import express from 'express' -import { logger } from '@server/helpers/logger' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' -import { getFormattedObjects } from '../../../helpers/utils' -import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../../../lib/blocklist' -import { - asyncMiddleware, - asyncRetryTransactionMiddleware, - authenticate, - paginationValidator, - setDefaultPagination, - setDefaultSort, - unblockAccountByAccountValidator -} from '../../../middlewares' -import { - accountsBlocklistSortValidator, - blockAccountValidator, - blockServerValidator, - serversBlocklistSortValidator, - unblockServerByAccountValidator -} from '../../../middlewares/validators' -import { AccountBlocklistModel } from '../../../models/account/account-blocklist' -import { ServerBlocklistModel } from '../../../models/server/server-blocklist' - -const myBlocklistRouter = express.Router() - -myBlocklistRouter.get('/me/blocklist/accounts', - authenticate, - paginationValidator, - accountsBlocklistSortValidator, - setDefaultSort, - setDefaultPagination, - asyncMiddleware(listBlockedAccounts) -) - -myBlocklistRouter.post('/me/blocklist/accounts', - authenticate, - asyncMiddleware(blockAccountValidator), - asyncRetryTransactionMiddleware(blockAccount) -) - -myBlocklistRouter.delete('/me/blocklist/accounts/:accountName', - authenticate, - asyncMiddleware(unblockAccountByAccountValidator), - asyncRetryTransactionMiddleware(unblockAccount) -) - -myBlocklistRouter.get('/me/blocklist/servers', - authenticate, - paginationValidator, - serversBlocklistSortValidator, - setDefaultSort, - setDefaultPagination, - asyncMiddleware(listBlockedServers) -) - -myBlocklistRouter.post('/me/blocklist/servers', - authenticate, - asyncMiddleware(blockServerValidator), - asyncRetryTransactionMiddleware(blockServer) -) - -myBlocklistRouter.delete('/me/blocklist/servers/:host', - authenticate, - asyncMiddleware(unblockServerByAccountValidator), - asyncRetryTransactionMiddleware(unblockServer) -) - -export { - myBlocklistRouter -} - -// --------------------------------------------------------------------------- - -async function listBlockedAccounts (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.User - - const resultList = await AccountBlocklistModel.listForApi({ - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - search: req.query.search, - accountId: user.Account.id - }) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function blockAccount (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.User - const accountToBlock = res.locals.account - - await addAccountInBlocklist(user.Account.id, accountToBlock.id) - - UserNotificationModel.removeNotificationsOf({ - id: accountToBlock.id, - type: 'account', - forUserId: user.id - }).catch(err => logger.error('Cannot remove notifications after an account mute.', { err })) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function unblockAccount (req: express.Request, res: express.Response) { - const accountBlock = res.locals.accountBlock - - await removeAccountFromBlocklist(accountBlock) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function listBlockedServers (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.User - - const resultList = await ServerBlocklistModel.listForApi({ - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - search: req.query.search, - accountId: user.Account.id - }) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function blockServer (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.User - const serverToBlock = res.locals.server - - await addServerInBlocklist(user.Account.id, serverToBlock.id) - - UserNotificationModel.removeNotificationsOf({ - id: serverToBlock.id, - type: 'server', - forUserId: user.id - }).catch(err => logger.error('Cannot remove notifications after a server mute.', { err })) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function unblockServer (req: express.Request, res: express.Response) { - const serverBlock = res.locals.serverBlock - - await removeServerFromBlocklist(serverBlock) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} diff --git a/server/controllers/api/users/my-history.ts b/server/controllers/api/users/my-history.ts deleted file mode 100644 index e6d3e86ac..000000000 --- a/server/controllers/api/users/my-history.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { forceNumber } from '@shared/core-utils' -import express from 'express' -import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' -import { getFormattedObjects } from '../../../helpers/utils' -import { sequelizeTypescript } from '../../../initializers/database' -import { - asyncMiddleware, - asyncRetryTransactionMiddleware, - authenticate, - paginationValidator, - setDefaultPagination, - userHistoryListValidator, - userHistoryRemoveAllValidator, - userHistoryRemoveElementValidator -} from '../../../middlewares' -import { UserVideoHistoryModel } from '../../../models/user/user-video-history' - -const myVideosHistoryRouter = express.Router() - -myVideosHistoryRouter.get('/me/history/videos', - authenticate, - paginationValidator, - setDefaultPagination, - userHistoryListValidator, - asyncMiddleware(listMyVideosHistory) -) - -myVideosHistoryRouter.delete('/me/history/videos/:videoId', - authenticate, - userHistoryRemoveElementValidator, - asyncMiddleware(removeUserHistoryElement) -) - -myVideosHistoryRouter.post('/me/history/videos/remove', - authenticate, - userHistoryRemoveAllValidator, - asyncRetryTransactionMiddleware(removeAllUserHistory) -) - -// --------------------------------------------------------------------------- - -export { - myVideosHistoryRouter -} - -// --------------------------------------------------------------------------- - -async function listMyVideosHistory (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.User - - const resultList = await UserVideoHistoryModel.listForApi(user, req.query.start, req.query.count, req.query.search) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function removeUserHistoryElement (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.User - - await UserVideoHistoryModel.removeUserHistoryElement(user, forceNumber(req.params.videoId)) - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -async function removeAllUserHistory (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.User - const beforeDate = req.body.beforeDate || null - - await sequelizeTypescript.transaction(t => { - return UserVideoHistoryModel.removeUserHistoryBefore(user, beforeDate, t) - }) - - return res.type('json') - .status(HttpStatusCode.NO_CONTENT_204) - .end() -} diff --git a/server/controllers/api/users/my-notifications.ts b/server/controllers/api/users/my-notifications.ts deleted file mode 100644 index 6014cdbbf..000000000 --- a/server/controllers/api/users/my-notifications.ts +++ /dev/null @@ -1,116 +0,0 @@ -import 'multer' -import express from 'express' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' -import { UserNotificationSetting } from '../../../../shared/models/users' -import { - asyncMiddleware, - asyncRetryTransactionMiddleware, - authenticate, - paginationValidator, - setDefaultPagination, - setDefaultSort, - userNotificationsSortValidator -} from '../../../middlewares' -import { - listUserNotificationsValidator, - markAsReadUserNotificationsValidator, - updateNotificationSettingsValidator -} from '../../../middlewares/validators/user-notifications' -import { UserNotificationSettingModel } from '../../../models/user/user-notification-setting' -import { meRouter } from './me' -import { getFormattedObjects } from '@server/helpers/utils' - -const myNotificationsRouter = express.Router() - -meRouter.put('/me/notification-settings', - authenticate, - updateNotificationSettingsValidator, - asyncRetryTransactionMiddleware(updateNotificationSettings) -) - -myNotificationsRouter.get('/me/notifications', - authenticate, - paginationValidator, - userNotificationsSortValidator, - setDefaultSort, - setDefaultPagination, - listUserNotificationsValidator, - asyncMiddleware(listUserNotifications) -) - -myNotificationsRouter.post('/me/notifications/read', - authenticate, - markAsReadUserNotificationsValidator, - asyncMiddleware(markAsReadUserNotifications) -) - -myNotificationsRouter.post('/me/notifications/read-all', - authenticate, - asyncMiddleware(markAsReadAllUserNotifications) -) - -export { - myNotificationsRouter -} - -// --------------------------------------------------------------------------- - -async function updateNotificationSettings (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.User - const body = req.body as UserNotificationSetting - - const query = { - where: { - userId: user.id - } - } - - const values: UserNotificationSetting = { - newVideoFromSubscription: body.newVideoFromSubscription, - newCommentOnMyVideo: body.newCommentOnMyVideo, - abuseAsModerator: body.abuseAsModerator, - videoAutoBlacklistAsModerator: body.videoAutoBlacklistAsModerator, - blacklistOnMyVideo: body.blacklistOnMyVideo, - myVideoPublished: body.myVideoPublished, - myVideoImportFinished: body.myVideoImportFinished, - newFollow: body.newFollow, - newUserRegistration: body.newUserRegistration, - commentMention: body.commentMention, - newInstanceFollower: body.newInstanceFollower, - autoInstanceFollowing: body.autoInstanceFollowing, - abuseNewMessage: body.abuseNewMessage, - abuseStateChange: body.abuseStateChange, - newPeerTubeVersion: body.newPeerTubeVersion, - newPluginVersion: body.newPluginVersion, - myVideoStudioEditionFinished: body.myVideoStudioEditionFinished - } - - await UserNotificationSettingModel.update(values, query) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function listUserNotifications (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.User - - const resultList = await UserNotificationModel.listForApi(user.id, req.query.start, req.query.count, req.query.sort, req.query.unread) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function markAsReadUserNotifications (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.User - - await UserNotificationModel.markAsRead(user.id, req.body.ids) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function markAsReadAllUserNotifications (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.User - - await UserNotificationModel.markAllAsRead(user.id) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} diff --git a/server/controllers/api/users/my-subscriptions.ts b/server/controllers/api/users/my-subscriptions.ts deleted file mode 100644 index c4360f59d..000000000 --- a/server/controllers/api/users/my-subscriptions.ts +++ /dev/null @@ -1,193 +0,0 @@ -import 'multer' -import express from 'express' -import { handlesToNameAndHost } from '@server/helpers/actors' -import { pickCommonVideoQuery } from '@server/helpers/query' -import { sendUndoFollow } from '@server/lib/activitypub/send' -import { Hooks } from '@server/lib/plugins/hooks' -import { VideoChannelModel } from '@server/models/video/video-channel' -import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' -import { buildNSFWFilter, getCountVideos } from '../../../helpers/express-utils' -import { getFormattedObjects } from '../../../helpers/utils' -import { sequelizeTypescript } from '../../../initializers/database' -import { JobQueue } from '../../../lib/job-queue' -import { - asyncMiddleware, - asyncRetryTransactionMiddleware, - authenticate, - commonVideosFiltersValidator, - paginationValidator, - setDefaultPagination, - setDefaultSort, - setDefaultVideosSort, - userSubscriptionAddValidator, - userSubscriptionGetValidator -} from '../../../middlewares' -import { - areSubscriptionsExistValidator, - userSubscriptionListValidator, - userSubscriptionsSortValidator, - videosSortValidator -} from '../../../middlewares/validators' -import { ActorFollowModel } from '../../../models/actor/actor-follow' -import { guessAdditionalAttributesFromQuery } from '../../../models/video/formatter' -import { VideoModel } from '../../../models/video/video' - -const mySubscriptionsRouter = express.Router() - -mySubscriptionsRouter.get('/me/subscriptions/videos', - authenticate, - paginationValidator, - videosSortValidator, - setDefaultVideosSort, - setDefaultPagination, - commonVideosFiltersValidator, - asyncMiddleware(getUserSubscriptionVideos) -) - -mySubscriptionsRouter.get('/me/subscriptions/exist', - authenticate, - areSubscriptionsExistValidator, - asyncMiddleware(areSubscriptionsExist) -) - -mySubscriptionsRouter.get('/me/subscriptions', - authenticate, - paginationValidator, - userSubscriptionsSortValidator, - setDefaultSort, - setDefaultPagination, - userSubscriptionListValidator, - asyncMiddleware(getUserSubscriptions) -) - -mySubscriptionsRouter.post('/me/subscriptions', - authenticate, - userSubscriptionAddValidator, - addUserSubscription -) - -mySubscriptionsRouter.get('/me/subscriptions/:uri', - authenticate, - userSubscriptionGetValidator, - asyncMiddleware(getUserSubscription) -) - -mySubscriptionsRouter.delete('/me/subscriptions/:uri', - authenticate, - userSubscriptionGetValidator, - asyncRetryTransactionMiddleware(deleteUserSubscription) -) - -// --------------------------------------------------------------------------- - -export { - mySubscriptionsRouter -} - -// --------------------------------------------------------------------------- - -async function areSubscriptionsExist (req: express.Request, res: express.Response) { - const uris = req.query.uris as string[] - const user = res.locals.oauth.token.User - - const sanitizedHandles = handlesToNameAndHost(uris) - - const results = await ActorFollowModel.listSubscriptionsOf(user.Account.Actor.id, sanitizedHandles) - - const existObject: { [id: string ]: boolean } = {} - for (const sanitizedHandle of sanitizedHandles) { - const obj = results.find(r => { - const server = r.ActorFollowing.Server - - return r.ActorFollowing.preferredUsername.toLowerCase() === sanitizedHandle.name.toLowerCase() && - ( - (!server && !sanitizedHandle.host) || - (server.host === sanitizedHandle.host) - ) - }) - - existObject[sanitizedHandle.handle] = obj !== undefined - } - - return res.json(existObject) -} - -function addUserSubscription (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.User - const [ name, host ] = req.body.uri.split('@') - - const payload = { - name, - host, - assertIsChannel: true, - followerActorId: user.Account.Actor.id - } - - JobQueue.Instance.createJobAsync({ type: 'activitypub-follow', payload }) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function getUserSubscription (req: express.Request, res: express.Response) { - const subscription = res.locals.subscription - const videoChannel = await VideoChannelModel.loadAndPopulateAccount(subscription.ActorFollowing.VideoChannel.id) - - return res.json(videoChannel.toFormattedJSON()) -} - -async function deleteUserSubscription (req: express.Request, res: express.Response) { - const subscription = res.locals.subscription - - await sequelizeTypescript.transaction(async t => { - if (subscription.state === 'accepted') { - sendUndoFollow(subscription, t) - } - - return subscription.destroy({ transaction: t }) - }) - - return res.type('json') - .status(HttpStatusCode.NO_CONTENT_204) - .end() -} - -async function getUserSubscriptions (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.User - const actorId = user.Account.Actor.id - - const resultList = await ActorFollowModel.listSubscriptionsForApi({ - actorId, - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - search: req.query.search - }) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function getUserSubscriptionVideos (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.User - const countVideos = getCountVideos(req) - const query = pickCommonVideoQuery(req.query) - - const apiOptions = await Hooks.wrapObject({ - ...query, - - displayOnlyForFollower: { - actorId: user.Account.Actor.id, - orLocalVideos: false - }, - nsfw: buildNSFWFilter(res, query.nsfw), - user, - countVideos - }, 'filter:api.user.me.subscription-videos.list.params') - - const resultList = await Hooks.wrapPromiseFun( - VideoModel.listForApi, - apiOptions, - 'filter:api.user.me.subscription-videos.list.result' - ) - - return res.json(getFormattedObjects(resultList.data, resultList.total, guessAdditionalAttributesFromQuery(query))) -} diff --git a/server/controllers/api/users/my-video-playlists.ts b/server/controllers/api/users/my-video-playlists.ts deleted file mode 100644 index fbdbb7e50..000000000 --- a/server/controllers/api/users/my-video-playlists.ts +++ /dev/null @@ -1,51 +0,0 @@ -import express from 'express' -import { forceNumber } from '@shared/core-utils' -import { uuidToShort } from '@shared/extra-utils' -import { VideosExistInPlaylists } from '../../../../shared/models/videos/playlist/video-exist-in-playlist.model' -import { asyncMiddleware, authenticate } from '../../../middlewares' -import { doVideosInPlaylistExistValidator } from '../../../middlewares/validators/videos/video-playlists' -import { VideoPlaylistModel } from '../../../models/video/video-playlist' - -const myVideoPlaylistsRouter = express.Router() - -myVideoPlaylistsRouter.get('/me/video-playlists/videos-exist', - authenticate, - doVideosInPlaylistExistValidator, - asyncMiddleware(doVideosInPlaylistExist) -) - -// --------------------------------------------------------------------------- - -export { - myVideoPlaylistsRouter -} - -// --------------------------------------------------------------------------- - -async function doVideosInPlaylistExist (req: express.Request, res: express.Response) { - const videoIds = req.query.videoIds.map(i => forceNumber(i)) - const user = res.locals.oauth.token.User - - const results = await VideoPlaylistModel.listPlaylistSummariesOf(user.Account.id, videoIds) - - const existObject: VideosExistInPlaylists = {} - - for (const videoId of videoIds) { - existObject[videoId] = [] - } - - for (const result of results) { - for (const element of result.VideoPlaylistElements) { - existObject[element.videoId].push({ - playlistElementId: element.id, - playlistId: result.id, - playlistDisplayName: result.name, - playlistShortUUID: uuidToShort(result.uuid), - startTimestamp: element.startTimestamp, - stopTimestamp: element.stopTimestamp - }) - } - } - - return res.json(existObject) -} diff --git a/server/controllers/api/users/registrations.ts b/server/controllers/api/users/registrations.ts deleted file mode 100644 index 5e213d6cc..000000000 --- a/server/controllers/api/users/registrations.ts +++ /dev/null @@ -1,249 +0,0 @@ -import express from 'express' -import { Emailer } from '@server/lib/emailer' -import { Hooks } from '@server/lib/plugins/hooks' -import { UserRegistrationModel } from '@server/models/user/user-registration' -import { pick } from '@shared/core-utils' -import { - HttpStatusCode, - UserRegister, - UserRegistrationRequest, - UserRegistrationState, - UserRegistrationUpdateState, - UserRight -} from '@shared/models' -import { auditLoggerFactory, UserAuditView } from '../../../helpers/audit-logger' -import { logger } from '../../../helpers/logger' -import { CONFIG } from '../../../initializers/config' -import { Notifier } from '../../../lib/notifier' -import { buildUser, createUserAccountAndChannelAndPlaylist, sendVerifyRegistrationEmail, sendVerifyUserEmail } from '../../../lib/user' -import { - acceptOrRejectRegistrationValidator, - asyncMiddleware, - asyncRetryTransactionMiddleware, - authenticate, - buildRateLimiter, - ensureUserHasRight, - ensureUserRegistrationAllowedFactory, - ensureUserRegistrationAllowedForIP, - getRegistrationValidator, - listRegistrationsValidator, - paginationValidator, - setDefaultPagination, - setDefaultSort, - userRegistrationsSortValidator, - usersDirectRegistrationValidator, - usersRequestRegistrationValidator -} from '../../../middlewares' - -const auditLogger = auditLoggerFactory('users') - -const registrationRateLimiter = buildRateLimiter({ - windowMs: CONFIG.RATES_LIMIT.SIGNUP.WINDOW_MS, - max: CONFIG.RATES_LIMIT.SIGNUP.MAX, - skipFailedRequests: true -}) - -const registrationsRouter = express.Router() - -registrationsRouter.post('/registrations/request', - registrationRateLimiter, - asyncMiddleware(ensureUserRegistrationAllowedFactory('request-registration')), - ensureUserRegistrationAllowedForIP, - asyncMiddleware(usersRequestRegistrationValidator), - asyncRetryTransactionMiddleware(requestRegistration) -) - -registrationsRouter.post('/registrations/:registrationId/accept', - authenticate, - ensureUserHasRight(UserRight.MANAGE_REGISTRATIONS), - asyncMiddleware(acceptOrRejectRegistrationValidator), - asyncRetryTransactionMiddleware(acceptRegistration) -) -registrationsRouter.post('/registrations/:registrationId/reject', - authenticate, - ensureUserHasRight(UserRight.MANAGE_REGISTRATIONS), - asyncMiddleware(acceptOrRejectRegistrationValidator), - asyncRetryTransactionMiddleware(rejectRegistration) -) - -registrationsRouter.delete('/registrations/:registrationId', - authenticate, - ensureUserHasRight(UserRight.MANAGE_REGISTRATIONS), - asyncMiddleware(getRegistrationValidator), - asyncRetryTransactionMiddleware(deleteRegistration) -) - -registrationsRouter.get('/registrations', - authenticate, - ensureUserHasRight(UserRight.MANAGE_REGISTRATIONS), - paginationValidator, - userRegistrationsSortValidator, - setDefaultSort, - setDefaultPagination, - listRegistrationsValidator, - asyncMiddleware(listRegistrations) -) - -registrationsRouter.post('/register', - registrationRateLimiter, - asyncMiddleware(ensureUserRegistrationAllowedFactory('direct-registration')), - ensureUserRegistrationAllowedForIP, - asyncMiddleware(usersDirectRegistrationValidator), - asyncRetryTransactionMiddleware(registerUser) -) - -// --------------------------------------------------------------------------- - -export { - registrationsRouter -} - -// --------------------------------------------------------------------------- - -async function requestRegistration (req: express.Request, res: express.Response) { - const body: UserRegistrationRequest = req.body - - const registration = new UserRegistrationModel({ - ...pick(body, [ 'username', 'password', 'email', 'registrationReason' ]), - - accountDisplayName: body.displayName, - channelDisplayName: body.channel?.displayName, - channelHandle: body.channel?.name, - - state: UserRegistrationState.PENDING, - - emailVerified: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION ? false : null - }) - - await registration.save() - - if (CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION) { - await sendVerifyRegistrationEmail(registration) - } - - Notifier.Instance.notifyOnNewRegistrationRequest(registration) - - Hooks.runAction('action:api.user.requested-registration', { body, registration, req, res }) - - return res.json(registration.toFormattedJSON()) -} - -// --------------------------------------------------------------------------- - -async function acceptRegistration (req: express.Request, res: express.Response) { - const registration = res.locals.userRegistration - const body: UserRegistrationUpdateState = req.body - - const userToCreate = buildUser({ - username: registration.username, - password: registration.password, - email: registration.email, - emailVerified: registration.emailVerified - }) - // We already encrypted password in registration model - userToCreate.skipPasswordEncryption = true - - // TODO: handle conflicts if someone else created a channel handle/user handle/user email between registration and approval - - const { user } = await createUserAccountAndChannelAndPlaylist({ - userToCreate, - userDisplayName: registration.accountDisplayName, - channelNames: registration.channelHandle && registration.channelDisplayName - ? { - name: registration.channelHandle, - displayName: registration.channelDisplayName - } - : undefined - }) - - registration.userId = user.id - registration.state = UserRegistrationState.ACCEPTED - registration.moderationResponse = body.moderationResponse - - await registration.save() - - logger.info('Registration of %s accepted', registration.username) - - if (body.preventEmailDelivery !== true) { - Emailer.Instance.addUserRegistrationRequestProcessedJob(registration) - } - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -async function rejectRegistration (req: express.Request, res: express.Response) { - const registration = res.locals.userRegistration - const body: UserRegistrationUpdateState = req.body - - registration.state = UserRegistrationState.REJECTED - registration.moderationResponse = body.moderationResponse - - await registration.save() - - if (body.preventEmailDelivery !== true) { - Emailer.Instance.addUserRegistrationRequestProcessedJob(registration) - } - - logger.info('Registration of %s rejected', registration.username) - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -// --------------------------------------------------------------------------- - -async function deleteRegistration (req: express.Request, res: express.Response) { - const registration = res.locals.userRegistration - - await registration.destroy() - - logger.info('Registration of %s deleted', registration.username) - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -// --------------------------------------------------------------------------- - -async function listRegistrations (req: express.Request, res: express.Response) { - const resultList = await UserRegistrationModel.listForApi({ - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - search: req.query.search - }) - - return res.json({ - total: resultList.total, - data: resultList.data.map(d => d.toFormattedJSON()) - }) -} - -// --------------------------------------------------------------------------- - -async function registerUser (req: express.Request, res: express.Response) { - const body: UserRegister = req.body - - const userToCreate = buildUser({ - ...pick(body, [ 'username', 'password', 'email' ]), - - emailVerified: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION ? false : null - }) - - const { user, account, videoChannel } = await createUserAccountAndChannelAndPlaylist({ - userToCreate, - userDisplayName: body.displayName || undefined, - channelNames: body.channel - }) - - auditLogger.create(body.username, new UserAuditView(user.toFormattedJSON())) - logger.info('User %s with its channel and account registered.', body.username) - - if (CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION) { - await sendVerifyUserEmail(user) - } - - Notifier.Instance.notifyOnNewDirectRegistration(user) - - Hooks.runAction('action:api.user.registered', { body, user, account, videoChannel, req, res }) - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} diff --git a/server/controllers/api/users/token.ts b/server/controllers/api/users/token.ts deleted file mode 100644 index c6afea67c..000000000 --- a/server/controllers/api/users/token.ts +++ /dev/null @@ -1,131 +0,0 @@ -import express from 'express' -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { OTP } from '@server/initializers/constants' -import { getAuthNameFromRefreshGrant, getBypassFromExternalAuth, getBypassFromPasswordGrant } from '@server/lib/auth/external-auth' -import { handleOAuthToken, MissingTwoFactorError } from '@server/lib/auth/oauth' -import { BypassLogin, revokeToken } from '@server/lib/auth/oauth-model' -import { Hooks } from '@server/lib/plugins/hooks' -import { asyncMiddleware, authenticate, buildRateLimiter, openapiOperationDoc } from '@server/middlewares' -import { buildUUID } from '@shared/extra-utils' -import { ScopedToken } from '@shared/models/users/user-scoped-token' - -const tokensRouter = express.Router() - -const loginRateLimiter = buildRateLimiter({ - windowMs: CONFIG.RATES_LIMIT.LOGIN.WINDOW_MS, - max: CONFIG.RATES_LIMIT.LOGIN.MAX -}) - -tokensRouter.post('/token', - loginRateLimiter, - openapiOperationDoc({ operationId: 'getOAuthToken' }), - asyncMiddleware(handleToken) -) - -tokensRouter.post('/revoke-token', - openapiOperationDoc({ operationId: 'revokeOAuthToken' }), - authenticate, - asyncMiddleware(handleTokenRevocation) -) - -tokensRouter.get('/scoped-tokens', - authenticate, - getScopedTokens -) - -tokensRouter.post('/scoped-tokens', - authenticate, - asyncMiddleware(renewScopedTokens) -) - -// --------------------------------------------------------------------------- - -export { - tokensRouter -} -// --------------------------------------------------------------------------- - -async function handleToken (req: express.Request, res: express.Response, next: express.NextFunction) { - const grantType = req.body.grant_type - - try { - const bypassLogin = await buildByPassLogin(req, grantType) - - const refreshTokenAuthName = grantType === 'refresh_token' - ? await getAuthNameFromRefreshGrant(req.body.refresh_token) - : undefined - - const options = { - refreshTokenAuthName, - bypassLogin - } - - const token = await handleOAuthToken(req, options) - - res.set('Cache-Control', 'no-store') - res.set('Pragma', 'no-cache') - - Hooks.runAction('action:api.user.oauth2-got-token', { username: token.user.username, ip: req.ip, req, res }) - - return res.json({ - token_type: 'Bearer', - - access_token: token.accessToken, - refresh_token: token.refreshToken, - - expires_in: token.accessTokenExpiresIn, - refresh_token_expires_in: token.refreshTokenExpiresIn - }) - } catch (err) { - logger.warn('Login error', { err }) - - if (err instanceof MissingTwoFactorError) { - res.set(OTP.HEADER_NAME, OTP.HEADER_REQUIRED_VALUE) - } - - return res.fail({ - status: err.code, - message: err.message, - type: err.name - }) - } -} - -async function handleTokenRevocation (req: express.Request, res: express.Response) { - const token = res.locals.oauth.token - - const result = await revokeToken(token, { req, explicitLogout: true }) - - return res.json(result) -} - -function getScopedTokens (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.user - - return res.json({ - feedToken: user.feedToken - } as ScopedToken) -} - -async function renewScopedTokens (req: express.Request, res: express.Response) { - const user = res.locals.oauth.token.user - - user.feedToken = buildUUID() - await user.save() - - return res.json({ - feedToken: user.feedToken - } as ScopedToken) -} - -async function buildByPassLogin (req: express.Request, grantType: string): Promise { - if (grantType !== 'password') return undefined - - if (req.body.externalAuthToken) { - // Consistency with the getBypassFromPasswordGrant promise - return getBypassFromExternalAuth(req.body.username, req.body.externalAuthToken) - } - - return getBypassFromPasswordGrant(req.body.username, req.body.password) -} diff --git a/server/controllers/api/users/two-factor.ts b/server/controllers/api/users/two-factor.ts deleted file mode 100644 index e6ae9e4dd..000000000 --- a/server/controllers/api/users/two-factor.ts +++ /dev/null @@ -1,95 +0,0 @@ -import express from 'express' -import { generateOTPSecret, isOTPValid } from '@server/helpers/otp' -import { encrypt } from '@server/helpers/peertube-crypto' -import { CONFIG } from '@server/initializers/config' -import { Redis } from '@server/lib/redis' -import { asyncMiddleware, authenticate, usersCheckCurrentPasswordFactory } from '@server/middlewares' -import { - confirmTwoFactorValidator, - disableTwoFactorValidator, - requestOrConfirmTwoFactorValidator -} from '@server/middlewares/validators/two-factor' -import { HttpStatusCode, TwoFactorEnableResult } from '@shared/models' - -const twoFactorRouter = express.Router() - -twoFactorRouter.post('/:id/two-factor/request', - authenticate, - asyncMiddleware(usersCheckCurrentPasswordFactory(req => req.params.id)), - asyncMiddleware(requestOrConfirmTwoFactorValidator), - asyncMiddleware(requestTwoFactor) -) - -twoFactorRouter.post('/:id/two-factor/confirm-request', - authenticate, - asyncMiddleware(requestOrConfirmTwoFactorValidator), - confirmTwoFactorValidator, - asyncMiddleware(confirmRequestTwoFactor) -) - -twoFactorRouter.post('/:id/two-factor/disable', - authenticate, - asyncMiddleware(usersCheckCurrentPasswordFactory(req => req.params.id)), - asyncMiddleware(disableTwoFactorValidator), - asyncMiddleware(disableTwoFactor) -) - -// --------------------------------------------------------------------------- - -export { - twoFactorRouter -} - -// --------------------------------------------------------------------------- - -async function requestTwoFactor (req: express.Request, res: express.Response) { - const user = res.locals.user - - const { secret, uri } = generateOTPSecret(user.email) - - const encryptedSecret = await encrypt(secret, CONFIG.SECRETS.PEERTUBE) - const requestToken = await Redis.Instance.setTwoFactorRequest(user.id, encryptedSecret) - - return res.json({ - otpRequest: { - requestToken, - secret, - uri - } - } as TwoFactorEnableResult) -} - -async function confirmRequestTwoFactor (req: express.Request, res: express.Response) { - const requestToken = req.body.requestToken - const otpToken = req.body.otpToken - const user = res.locals.user - - const encryptedSecret = await Redis.Instance.getTwoFactorRequestToken(user.id, requestToken) - if (!encryptedSecret) { - return res.fail({ - message: 'Invalid request token', - status: HttpStatusCode.FORBIDDEN_403 - }) - } - - if (await isOTPValid({ encryptedSecret, token: otpToken }) !== true) { - return res.fail({ - message: 'Invalid OTP token', - status: HttpStatusCode.FORBIDDEN_403 - }) - } - - user.otpSecret = encryptedSecret - await user.save() - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -async function disableTwoFactor (req: express.Request, res: express.Response) { - const user = res.locals.user - - user.otpSecret = null - await user.save() - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} diff --git a/server/controllers/api/video-channel-sync.ts b/server/controllers/api/video-channel-sync.ts deleted file mode 100644 index 6b52ac7dd..000000000 --- a/server/controllers/api/video-channel-sync.ts +++ /dev/null @@ -1,79 +0,0 @@ -import express from 'express' -import { auditLoggerFactory, getAuditIdFromRes, VideoChannelSyncAuditView } from '@server/helpers/audit-logger' -import { logger } from '@server/helpers/logger' -import { - apiRateLimiter, - asyncMiddleware, - asyncRetryTransactionMiddleware, - authenticate, - ensureCanManageChannelOrAccount, - ensureSyncExists, - ensureSyncIsEnabled, - videoChannelSyncValidator -} from '@server/middlewares' -import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' -import { MChannelSyncFormattable } from '@server/types/models' -import { HttpStatusCode, VideoChannelSyncState } from '@shared/models' - -const videoChannelSyncRouter = express.Router() -const auditLogger = auditLoggerFactory('channel-syncs') - -videoChannelSyncRouter.use(apiRateLimiter) - -videoChannelSyncRouter.post('/', - authenticate, - ensureSyncIsEnabled, - asyncMiddleware(videoChannelSyncValidator), - ensureCanManageChannelOrAccount, - asyncRetryTransactionMiddleware(createVideoChannelSync) -) - -videoChannelSyncRouter.delete('/:id', - authenticate, - asyncMiddleware(ensureSyncExists), - ensureCanManageChannelOrAccount, - asyncRetryTransactionMiddleware(removeVideoChannelSync) -) - -export { videoChannelSyncRouter } - -// --------------------------------------------------------------------------- - -async function createVideoChannelSync (req: express.Request, res: express.Response) { - const syncCreated: MChannelSyncFormattable = new VideoChannelSyncModel({ - externalChannelUrl: req.body.externalChannelUrl, - videoChannelId: req.body.videoChannelId, - state: VideoChannelSyncState.WAITING_FIRST_RUN - }) - - await syncCreated.save() - syncCreated.VideoChannel = res.locals.videoChannel - - auditLogger.create(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncCreated.toFormattedJSON())) - - logger.info( - 'Video synchronization for channel "%s" with external channel "%s" created.', - syncCreated.VideoChannel.name, - syncCreated.externalChannelUrl - ) - - return res.json({ - videoChannelSync: syncCreated.toFormattedJSON() - }) -} - -async function removeVideoChannelSync (req: express.Request, res: express.Response) { - const syncInstance = res.locals.videoChannelSync - - await syncInstance.destroy() - - auditLogger.delete(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncInstance.toFormattedJSON())) - - logger.info( - 'Video synchronization for channel "%s" with external channel "%s" deleted.', - syncInstance.VideoChannel.name, - syncInstance.externalChannelUrl - ) - - return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() -} diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts deleted file mode 100644 index 18de5bf6a..000000000 --- a/server/controllers/api/video-channel.ts +++ /dev/null @@ -1,431 +0,0 @@ -import express from 'express' -import { pickCommonVideoQuery } from '@server/helpers/query' -import { Hooks } from '@server/lib/plugins/hooks' -import { ActorFollowModel } from '@server/models/actor/actor-follow' -import { getServerActor } from '@server/models/application/application' -import { MChannelBannerAccountDefault } from '@server/types/models' -import { ActorImageType, HttpStatusCode, VideoChannelCreate, VideoChannelUpdate, VideosImportInChannelCreate } from '@shared/models' -import { auditLoggerFactory, getAuditIdFromRes, VideoChannelAuditView } from '../../helpers/audit-logger' -import { resetSequelizeInstance } from '../../helpers/database-utils' -import { buildNSFWFilter, createReqFiles, getCountVideos, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils' -import { logger } from '../../helpers/logger' -import { getFormattedObjects } from '../../helpers/utils' -import { MIMETYPES } from '../../initializers/constants' -import { sequelizeTypescript } from '../../initializers/database' -import { sendUpdateActor } from '../../lib/activitypub/send' -import { JobQueue } from '../../lib/job-queue' -import { deleteLocalActorImageFile, updateLocalActorImageFiles } from '../../lib/local-actor' -import { createLocalVideoChannel, federateAllVideosOfChannel } from '../../lib/video-channel' -import { - apiRateLimiter, - asyncMiddleware, - asyncRetryTransactionMiddleware, - authenticate, - commonVideosFiltersValidator, - ensureCanManageChannelOrAccount, - optionalAuthenticate, - paginationValidator, - setDefaultPagination, - setDefaultSort, - setDefaultVideosSort, - videoChannelsAddValidator, - videoChannelsRemoveValidator, - videoChannelsSortValidator, - videoChannelsUpdateValidator, - videoPlaylistsSortValidator -} from '../../middlewares' -import { - ensureChannelOwnerCanUpload, - ensureIsLocalChannel, - videoChannelImportVideosValidator, - videoChannelsFollowersSortValidator, - videoChannelsListValidator, - videoChannelsNameWithHostValidator, - videosSortValidator -} from '../../middlewares/validators' -import { updateAvatarValidator, updateBannerValidator } from '../../middlewares/validators/actor-image' -import { commonVideoPlaylistFiltersValidator } from '../../middlewares/validators/videos/video-playlists' -import { AccountModel } from '../../models/account/account' -import { guessAdditionalAttributesFromQuery } from '../../models/video/formatter' -import { VideoModel } from '../../models/video/video' -import { VideoChannelModel } from '../../models/video/video-channel' -import { VideoPlaylistModel } from '../../models/video/video-playlist' - -const auditLogger = auditLoggerFactory('channels') -const reqAvatarFile = createReqFiles([ 'avatarfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT) -const reqBannerFile = createReqFiles([ 'bannerfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT) - -const videoChannelRouter = express.Router() - -videoChannelRouter.use(apiRateLimiter) - -videoChannelRouter.get('/', - paginationValidator, - videoChannelsSortValidator, - setDefaultSort, - setDefaultPagination, - videoChannelsListValidator, - asyncMiddleware(listVideoChannels) -) - -videoChannelRouter.post('/', - authenticate, - asyncMiddleware(videoChannelsAddValidator), - asyncRetryTransactionMiddleware(addVideoChannel) -) - -videoChannelRouter.post('/:nameWithHost/avatar/pick', - authenticate, - reqAvatarFile, - asyncMiddleware(videoChannelsNameWithHostValidator), - ensureIsLocalChannel, - ensureCanManageChannelOrAccount, - updateAvatarValidator, - asyncMiddleware(updateVideoChannelAvatar) -) - -videoChannelRouter.post('/:nameWithHost/banner/pick', - authenticate, - reqBannerFile, - asyncMiddleware(videoChannelsNameWithHostValidator), - ensureIsLocalChannel, - ensureCanManageChannelOrAccount, - updateBannerValidator, - asyncMiddleware(updateVideoChannelBanner) -) - -videoChannelRouter.delete('/:nameWithHost/avatar', - authenticate, - asyncMiddleware(videoChannelsNameWithHostValidator), - ensureIsLocalChannel, - ensureCanManageChannelOrAccount, - asyncMiddleware(deleteVideoChannelAvatar) -) - -videoChannelRouter.delete('/:nameWithHost/banner', - authenticate, - asyncMiddleware(videoChannelsNameWithHostValidator), - ensureIsLocalChannel, - ensureCanManageChannelOrAccount, - asyncMiddleware(deleteVideoChannelBanner) -) - -videoChannelRouter.put('/:nameWithHost', - authenticate, - asyncMiddleware(videoChannelsNameWithHostValidator), - ensureIsLocalChannel, - ensureCanManageChannelOrAccount, - videoChannelsUpdateValidator, - asyncRetryTransactionMiddleware(updateVideoChannel) -) - -videoChannelRouter.delete('/:nameWithHost', - authenticate, - asyncMiddleware(videoChannelsNameWithHostValidator), - ensureIsLocalChannel, - ensureCanManageChannelOrAccount, - asyncMiddleware(videoChannelsRemoveValidator), - asyncRetryTransactionMiddleware(removeVideoChannel) -) - -videoChannelRouter.get('/:nameWithHost', - asyncMiddleware(videoChannelsNameWithHostValidator), - asyncMiddleware(getVideoChannel) -) - -videoChannelRouter.get('/:nameWithHost/video-playlists', - asyncMiddleware(videoChannelsNameWithHostValidator), - paginationValidator, - videoPlaylistsSortValidator, - setDefaultSort, - setDefaultPagination, - commonVideoPlaylistFiltersValidator, - asyncMiddleware(listVideoChannelPlaylists) -) - -videoChannelRouter.get('/:nameWithHost/videos', - asyncMiddleware(videoChannelsNameWithHostValidator), - paginationValidator, - videosSortValidator, - setDefaultVideosSort, - setDefaultPagination, - optionalAuthenticate, - commonVideosFiltersValidator, - asyncMiddleware(listVideoChannelVideos) -) - -videoChannelRouter.get('/:nameWithHost/followers', - authenticate, - asyncMiddleware(videoChannelsNameWithHostValidator), - ensureCanManageChannelOrAccount, - paginationValidator, - videoChannelsFollowersSortValidator, - setDefaultSort, - setDefaultPagination, - asyncMiddleware(listVideoChannelFollowers) -) - -videoChannelRouter.post('/:nameWithHost/import-videos', - authenticate, - asyncMiddleware(videoChannelsNameWithHostValidator), - asyncMiddleware(videoChannelImportVideosValidator), - ensureIsLocalChannel, - ensureCanManageChannelOrAccount, - asyncMiddleware(ensureChannelOwnerCanUpload), - asyncMiddleware(importVideosInChannel) -) - -// --------------------------------------------------------------------------- - -export { - videoChannelRouter -} - -// --------------------------------------------------------------------------- - -async function listVideoChannels (req: express.Request, res: express.Response) { - const serverActor = await getServerActor() - - const apiOptions = await Hooks.wrapObject({ - actorId: serverActor.id, - start: req.query.start, - count: req.query.count, - sort: req.query.sort - }, 'filter:api.video-channels.list.params') - - const resultList = await Hooks.wrapPromiseFun( - VideoChannelModel.listForApi, - apiOptions, - 'filter:api.video-channels.list.result' - ) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function updateVideoChannelBanner (req: express.Request, res: express.Response) { - const bannerPhysicalFile = req.files['bannerfile'][0] - const videoChannel = res.locals.videoChannel - const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannel.toFormattedJSON()) - - const banners = await updateLocalActorImageFiles(videoChannel, bannerPhysicalFile, ActorImageType.BANNER) - - auditLogger.update(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannel.toFormattedJSON()), oldVideoChannelAuditKeys) - - return res.json({ - banners: banners.map(b => b.toFormattedJSON()) - }) -} - -async function updateVideoChannelAvatar (req: express.Request, res: express.Response) { - const avatarPhysicalFile = req.files['avatarfile'][0] - const videoChannel = res.locals.videoChannel - const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannel.toFormattedJSON()) - - const avatars = await updateLocalActorImageFiles(videoChannel, avatarPhysicalFile, ActorImageType.AVATAR) - auditLogger.update(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannel.toFormattedJSON()), oldVideoChannelAuditKeys) - - return res.json({ - avatars: avatars.map(a => a.toFormattedJSON()) - }) -} - -async function deleteVideoChannelAvatar (req: express.Request, res: express.Response) { - const videoChannel = res.locals.videoChannel - - await deleteLocalActorImageFile(videoChannel, ActorImageType.AVATAR) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function deleteVideoChannelBanner (req: express.Request, res: express.Response) { - const videoChannel = res.locals.videoChannel - - await deleteLocalActorImageFile(videoChannel, ActorImageType.BANNER) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function addVideoChannel (req: express.Request, res: express.Response) { - const videoChannelInfo: VideoChannelCreate = req.body - - const videoChannelCreated = await sequelizeTypescript.transaction(async t => { - const account = await AccountModel.load(res.locals.oauth.token.User.Account.id, t) - - return createLocalVideoChannel(videoChannelInfo, account, t) - }) - - const payload = { actorId: videoChannelCreated.actorId } - await JobQueue.Instance.createJob({ type: 'actor-keys', payload }) - - auditLogger.create(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannelCreated.toFormattedJSON())) - logger.info('Video channel %s created.', videoChannelCreated.Actor.url) - - Hooks.runAction('action:api.video-channel.created', { videoChannel: videoChannelCreated, req, res }) - - return res.json({ - videoChannel: { - id: videoChannelCreated.id - } - }) -} - -async function updateVideoChannel (req: express.Request, res: express.Response) { - const videoChannelInstance = res.locals.videoChannel - const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannelInstance.toFormattedJSON()) - const videoChannelInfoToUpdate = req.body as VideoChannelUpdate - let doBulkVideoUpdate = false - - try { - await sequelizeTypescript.transaction(async t => { - if (videoChannelInfoToUpdate.displayName !== undefined) videoChannelInstance.name = videoChannelInfoToUpdate.displayName - if (videoChannelInfoToUpdate.description !== undefined) videoChannelInstance.description = videoChannelInfoToUpdate.description - - if (videoChannelInfoToUpdate.support !== undefined) { - const oldSupportField = videoChannelInstance.support - videoChannelInstance.support = videoChannelInfoToUpdate.support - - if (videoChannelInfoToUpdate.bulkVideosSupportUpdate === true && oldSupportField !== videoChannelInfoToUpdate.support) { - doBulkVideoUpdate = true - await VideoModel.bulkUpdateSupportField(videoChannelInstance, t) - } - } - - const videoChannelInstanceUpdated = await videoChannelInstance.save({ transaction: t }) as MChannelBannerAccountDefault - await sendUpdateActor(videoChannelInstanceUpdated, t) - - auditLogger.update( - getAuditIdFromRes(res), - new VideoChannelAuditView(videoChannelInstanceUpdated.toFormattedJSON()), - oldVideoChannelAuditKeys - ) - - Hooks.runAction('action:api.video-channel.updated', { videoChannel: videoChannelInstanceUpdated, req, res }) - - logger.info('Video channel %s updated.', videoChannelInstance.Actor.url) - }) - } catch (err) { - logger.debug('Cannot update the video channel.', { err }) - - // If the transaction is retried, sequelize will think the object has not changed - // So we need to restore the previous fields - await resetSequelizeInstance(videoChannelInstance) - - throw err - } - - res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() - - // Don't process in a transaction, and after the response because it could be long - if (doBulkVideoUpdate) { - await federateAllVideosOfChannel(videoChannelInstance) - } -} - -async function removeVideoChannel (req: express.Request, res: express.Response) { - const videoChannelInstance = res.locals.videoChannel - - await sequelizeTypescript.transaction(async t => { - await VideoPlaylistModel.resetPlaylistsOfChannel(videoChannelInstance.id, t) - - await videoChannelInstance.destroy({ transaction: t }) - - Hooks.runAction('action:api.video-channel.deleted', { videoChannel: videoChannelInstance, req, res }) - - auditLogger.delete(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannelInstance.toFormattedJSON())) - logger.info('Video channel %s deleted.', videoChannelInstance.Actor.url) - }) - - return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function getVideoChannel (req: express.Request, res: express.Response) { - const id = res.locals.videoChannel.id - const videoChannel = await Hooks.wrapObject(res.locals.videoChannel, 'filter:api.video-channel.get.result', { id }) - - if (videoChannel.isOutdated()) { - JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'actor', url: videoChannel.Actor.url } }) - } - - return res.json(videoChannel.toFormattedJSON()) -} - -async function listVideoChannelPlaylists (req: express.Request, res: express.Response) { - const serverActor = await getServerActor() - - const resultList = await VideoPlaylistModel.listForApi({ - followerActorId: serverActor.id, - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - videoChannelId: res.locals.videoChannel.id, - type: req.query.playlistType - }) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function listVideoChannelVideos (req: express.Request, res: express.Response) { - const serverActor = await getServerActor() - - const videoChannelInstance = res.locals.videoChannel - - const displayOnlyForFollower = isUserAbleToSearchRemoteURI(res) - ? null - : { - actorId: serverActor.id, - orLocalVideos: true - } - - const countVideos = getCountVideos(req) - const query = pickCommonVideoQuery(req.query) - - const apiOptions = await Hooks.wrapObject({ - ...query, - - displayOnlyForFollower, - nsfw: buildNSFWFilter(res, query.nsfw), - videoChannelId: videoChannelInstance.id, - user: res.locals.oauth ? res.locals.oauth.token.User : undefined, - countVideos - }, 'filter:api.video-channels.videos.list.params') - - const resultList = await Hooks.wrapPromiseFun( - VideoModel.listForApi, - apiOptions, - 'filter:api.video-channels.videos.list.result' - ) - - return res.json(getFormattedObjects(resultList.data, resultList.total, guessAdditionalAttributesFromQuery(query))) -} - -async function listVideoChannelFollowers (req: express.Request, res: express.Response) { - const channel = res.locals.videoChannel - - const resultList = await ActorFollowModel.listFollowersForApi({ - actorIds: [ channel.actorId ], - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - search: req.query.search, - state: 'accepted' - }) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function importVideosInChannel (req: express.Request, res: express.Response) { - const { externalChannelUrl } = req.body as VideosImportInChannelCreate - - await JobQueue.Instance.createJob({ - type: 'video-channel-import', - payload: { - externalChannelUrl, - videoChannelId: res.locals.videoChannel.id, - partOfChannelSyncId: res.locals.videoChannelSync?.id - } - }) - - logger.info('Video import job for channel "%s" with url "%s" created.', res.locals.videoChannel.name, externalChannelUrl) - - return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() -} diff --git a/server/controllers/api/video-playlist.ts b/server/controllers/api/video-playlist.ts deleted file mode 100644 index 73362e1e3..000000000 --- a/server/controllers/api/video-playlist.ts +++ /dev/null @@ -1,514 +0,0 @@ -import express from 'express' -import { scheduleRefreshIfNeeded } from '@server/lib/activitypub/playlists' -import { VideoMiniaturePermanentFileCache } from '@server/lib/files-cache' -import { Hooks } from '@server/lib/plugins/hooks' -import { getServerActor } from '@server/models/application/application' -import { MVideoPlaylistFull, MVideoPlaylistThumbnail, MVideoThumbnail } from '@server/types/models' -import { forceNumber } from '@shared/core-utils' -import { uuidToShort } from '@shared/extra-utils' -import { VideoPlaylistCreateResult, VideoPlaylistElementCreateResult } from '@shared/models' -import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' -import { VideoPlaylistCreate } from '../../../shared/models/videos/playlist/video-playlist-create.model' -import { VideoPlaylistElementCreate } from '../../../shared/models/videos/playlist/video-playlist-element-create.model' -import { VideoPlaylistElementUpdate } from '../../../shared/models/videos/playlist/video-playlist-element-update.model' -import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model' -import { VideoPlaylistReorder } from '../../../shared/models/videos/playlist/video-playlist-reorder.model' -import { VideoPlaylistUpdate } from '../../../shared/models/videos/playlist/video-playlist-update.model' -import { resetSequelizeInstance } from '../../helpers/database-utils' -import { createReqFiles } from '../../helpers/express-utils' -import { logger } from '../../helpers/logger' -import { getFormattedObjects } from '../../helpers/utils' -import { MIMETYPES, VIDEO_PLAYLIST_PRIVACIES } from '../../initializers/constants' -import { sequelizeTypescript } from '../../initializers/database' -import { sendCreateVideoPlaylist, sendDeleteVideoPlaylist, sendUpdateVideoPlaylist } from '../../lib/activitypub/send' -import { getLocalVideoPlaylistActivityPubUrl, getLocalVideoPlaylistElementActivityPubUrl } from '../../lib/activitypub/url' -import { updateLocalPlaylistMiniatureFromExisting } from '../../lib/thumbnail' -import { - apiRateLimiter, - asyncMiddleware, - asyncRetryTransactionMiddleware, - authenticate, - optionalAuthenticate, - paginationValidator, - setDefaultPagination, - setDefaultSort -} from '../../middlewares' -import { videoPlaylistsSortValidator } from '../../middlewares/validators' -import { - commonVideoPlaylistFiltersValidator, - videoPlaylistsAddValidator, - videoPlaylistsAddVideoValidator, - videoPlaylistsDeleteValidator, - videoPlaylistsGetValidator, - videoPlaylistsReorderVideosValidator, - videoPlaylistsUpdateOrRemoveVideoValidator, - videoPlaylistsUpdateValidator -} from '../../middlewares/validators/videos/video-playlists' -import { AccountModel } from '../../models/account/account' -import { VideoPlaylistModel } from '../../models/video/video-playlist' -import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element' - -const reqThumbnailFile = createReqFiles([ 'thumbnailfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT) - -const videoPlaylistRouter = express.Router() - -videoPlaylistRouter.use(apiRateLimiter) - -videoPlaylistRouter.get('/privacies', listVideoPlaylistPrivacies) - -videoPlaylistRouter.get('/', - paginationValidator, - videoPlaylistsSortValidator, - setDefaultSort, - setDefaultPagination, - commonVideoPlaylistFiltersValidator, - asyncMiddleware(listVideoPlaylists) -) - -videoPlaylistRouter.get('/:playlistId', - asyncMiddleware(videoPlaylistsGetValidator('summary')), - getVideoPlaylist -) - -videoPlaylistRouter.post('/', - authenticate, - reqThumbnailFile, - asyncMiddleware(videoPlaylistsAddValidator), - asyncRetryTransactionMiddleware(addVideoPlaylist) -) - -videoPlaylistRouter.put('/:playlistId', - authenticate, - reqThumbnailFile, - asyncMiddleware(videoPlaylistsUpdateValidator), - asyncRetryTransactionMiddleware(updateVideoPlaylist) -) - -videoPlaylistRouter.delete('/:playlistId', - authenticate, - asyncMiddleware(videoPlaylistsDeleteValidator), - asyncRetryTransactionMiddleware(removeVideoPlaylist) -) - -videoPlaylistRouter.get('/:playlistId/videos', - asyncMiddleware(videoPlaylistsGetValidator('summary')), - paginationValidator, - setDefaultPagination, - optionalAuthenticate, - asyncMiddleware(getVideoPlaylistVideos) -) - -videoPlaylistRouter.post('/:playlistId/videos', - authenticate, - asyncMiddleware(videoPlaylistsAddVideoValidator), - asyncRetryTransactionMiddleware(addVideoInPlaylist) -) - -videoPlaylistRouter.post('/:playlistId/videos/reorder', - authenticate, - asyncMiddleware(videoPlaylistsReorderVideosValidator), - asyncRetryTransactionMiddleware(reorderVideosPlaylist) -) - -videoPlaylistRouter.put('/:playlistId/videos/:playlistElementId', - authenticate, - asyncMiddleware(videoPlaylistsUpdateOrRemoveVideoValidator), - asyncRetryTransactionMiddleware(updateVideoPlaylistElement) -) - -videoPlaylistRouter.delete('/:playlistId/videos/:playlistElementId', - authenticate, - asyncMiddleware(videoPlaylistsUpdateOrRemoveVideoValidator), - asyncRetryTransactionMiddleware(removeVideoFromPlaylist) -) - -// --------------------------------------------------------------------------- - -export { - videoPlaylistRouter -} - -// --------------------------------------------------------------------------- - -function listVideoPlaylistPrivacies (req: express.Request, res: express.Response) { - res.json(VIDEO_PLAYLIST_PRIVACIES) -} - -async function listVideoPlaylists (req: express.Request, res: express.Response) { - const serverActor = await getServerActor() - const resultList = await VideoPlaylistModel.listForApi({ - followerActorId: serverActor.id, - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - type: req.query.playlistType - }) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -function getVideoPlaylist (req: express.Request, res: express.Response) { - const videoPlaylist = res.locals.videoPlaylistSummary - - scheduleRefreshIfNeeded(videoPlaylist) - - return res.json(videoPlaylist.toFormattedJSON()) -} - -async function addVideoPlaylist (req: express.Request, res: express.Response) { - const videoPlaylistInfo: VideoPlaylistCreate = req.body - const user = res.locals.oauth.token.User - - const videoPlaylist = new VideoPlaylistModel({ - name: videoPlaylistInfo.displayName, - description: videoPlaylistInfo.description, - privacy: videoPlaylistInfo.privacy || VideoPlaylistPrivacy.PRIVATE, - ownerAccountId: user.Account.id - }) as MVideoPlaylistFull - - videoPlaylist.url = getLocalVideoPlaylistActivityPubUrl(videoPlaylist) // We use the UUID, so set the URL after building the object - - if (videoPlaylistInfo.videoChannelId) { - const videoChannel = res.locals.videoChannel - - videoPlaylist.videoChannelId = videoChannel.id - videoPlaylist.VideoChannel = videoChannel - } - - const thumbnailField = req.files['thumbnailfile'] - const thumbnailModel = thumbnailField - ? await updateLocalPlaylistMiniatureFromExisting({ - inputPath: thumbnailField[0].path, - playlist: videoPlaylist, - automaticallyGenerated: false - }) - : undefined - - const videoPlaylistCreated = await sequelizeTypescript.transaction(async t => { - const videoPlaylistCreated = await videoPlaylist.save({ transaction: t }) as MVideoPlaylistFull - - if (thumbnailModel) { - thumbnailModel.automaticallyGenerated = false - await videoPlaylistCreated.setAndSaveThumbnail(thumbnailModel, t) - } - - // We need more attributes for the federation - videoPlaylistCreated.OwnerAccount = await AccountModel.load(user.Account.id, t) - await sendCreateVideoPlaylist(videoPlaylistCreated, t) - - return videoPlaylistCreated - }) - - logger.info('Video playlist with uuid %s created.', videoPlaylist.uuid) - - return res.json({ - videoPlaylist: { - id: videoPlaylistCreated.id, - shortUUID: uuidToShort(videoPlaylistCreated.uuid), - uuid: videoPlaylistCreated.uuid - } as VideoPlaylistCreateResult - }) -} - -async function updateVideoPlaylist (req: express.Request, res: express.Response) { - const videoPlaylistInstance = res.locals.videoPlaylistFull - const videoPlaylistInfoToUpdate = req.body as VideoPlaylistUpdate - - const wasPrivatePlaylist = videoPlaylistInstance.privacy === VideoPlaylistPrivacy.PRIVATE - const wasNotPrivatePlaylist = videoPlaylistInstance.privacy !== VideoPlaylistPrivacy.PRIVATE - - const thumbnailField = req.files['thumbnailfile'] - const thumbnailModel = thumbnailField - ? await updateLocalPlaylistMiniatureFromExisting({ - inputPath: thumbnailField[0].path, - playlist: videoPlaylistInstance, - automaticallyGenerated: false - }) - : undefined - - try { - await sequelizeTypescript.transaction(async t => { - const sequelizeOptions = { - transaction: t - } - - if (videoPlaylistInfoToUpdate.videoChannelId !== undefined) { - if (videoPlaylistInfoToUpdate.videoChannelId === null) { - videoPlaylistInstance.videoChannelId = null - } else { - const videoChannel = res.locals.videoChannel - - videoPlaylistInstance.videoChannelId = videoChannel.id - videoPlaylistInstance.VideoChannel = videoChannel - } - } - - if (videoPlaylistInfoToUpdate.displayName !== undefined) videoPlaylistInstance.name = videoPlaylistInfoToUpdate.displayName - if (videoPlaylistInfoToUpdate.description !== undefined) videoPlaylistInstance.description = videoPlaylistInfoToUpdate.description - - if (videoPlaylistInfoToUpdate.privacy !== undefined) { - videoPlaylistInstance.privacy = forceNumber(videoPlaylistInfoToUpdate.privacy) - - if (wasNotPrivatePlaylist === true && videoPlaylistInstance.privacy === VideoPlaylistPrivacy.PRIVATE) { - await sendDeleteVideoPlaylist(videoPlaylistInstance, t) - } - } - - const playlistUpdated = await videoPlaylistInstance.save(sequelizeOptions) - - if (thumbnailModel) { - thumbnailModel.automaticallyGenerated = false - await playlistUpdated.setAndSaveThumbnail(thumbnailModel, t) - } - - const isNewPlaylist = wasPrivatePlaylist && playlistUpdated.privacy !== VideoPlaylistPrivacy.PRIVATE - - if (isNewPlaylist) { - await sendCreateVideoPlaylist(playlistUpdated, t) - } else { - await sendUpdateVideoPlaylist(playlistUpdated, t) - } - - logger.info('Video playlist %s updated.', videoPlaylistInstance.uuid) - - return playlistUpdated - }) - } catch (err) { - logger.debug('Cannot update the video playlist.', { err }) - - // If the transaction is retried, sequelize will think the object has not changed - // So we need to restore the previous fields - await resetSequelizeInstance(videoPlaylistInstance) - - throw err - } - - return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function removeVideoPlaylist (req: express.Request, res: express.Response) { - const videoPlaylistInstance = res.locals.videoPlaylistSummary - - await sequelizeTypescript.transaction(async t => { - await videoPlaylistInstance.destroy({ transaction: t }) - - await sendDeleteVideoPlaylist(videoPlaylistInstance, t) - - logger.info('Video playlist %s deleted.', videoPlaylistInstance.uuid) - }) - - return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function addVideoInPlaylist (req: express.Request, res: express.Response) { - const body: VideoPlaylistElementCreate = req.body - const videoPlaylist = res.locals.videoPlaylistFull - const video = res.locals.onlyVideo - - const playlistElement = await sequelizeTypescript.transaction(async t => { - const position = await VideoPlaylistElementModel.getNextPositionOf(videoPlaylist.id, t) - - const playlistElement = await VideoPlaylistElementModel.create({ - position, - startTimestamp: body.startTimestamp || null, - stopTimestamp: body.stopTimestamp || null, - videoPlaylistId: videoPlaylist.id, - videoId: video.id - }, { transaction: t }) - - playlistElement.url = getLocalVideoPlaylistElementActivityPubUrl(videoPlaylist, playlistElement) - await playlistElement.save({ transaction: t }) - - videoPlaylist.changed('updatedAt', true) - await videoPlaylist.save({ transaction: t }) - - return playlistElement - }) - - // If the user did not set a thumbnail, automatically take the video thumbnail - if (videoPlaylist.hasThumbnail() === false || (videoPlaylist.hasGeneratedThumbnail() && playlistElement.position === 1)) { - await generateThumbnailForPlaylist(videoPlaylist, video) - } - - sendUpdateVideoPlaylist(videoPlaylist, undefined) - .catch(err => logger.error('Cannot send video playlist update.', { err })) - - logger.info('Video added in playlist %s at position %d.', videoPlaylist.uuid, playlistElement.position) - - Hooks.runAction('action:api.video-playlist-element.created', { playlistElement, req, res }) - - return res.json({ - videoPlaylistElement: { - id: playlistElement.id - } as VideoPlaylistElementCreateResult - }) -} - -async function updateVideoPlaylistElement (req: express.Request, res: express.Response) { - const body: VideoPlaylistElementUpdate = req.body - const videoPlaylist = res.locals.videoPlaylistFull - const videoPlaylistElement = res.locals.videoPlaylistElement - - const playlistElement: VideoPlaylistElementModel = await sequelizeTypescript.transaction(async t => { - if (body.startTimestamp !== undefined) videoPlaylistElement.startTimestamp = body.startTimestamp - if (body.stopTimestamp !== undefined) videoPlaylistElement.stopTimestamp = body.stopTimestamp - - const element = await videoPlaylistElement.save({ transaction: t }) - - videoPlaylist.changed('updatedAt', true) - await videoPlaylist.save({ transaction: t }) - - await sendUpdateVideoPlaylist(videoPlaylist, t) - - return element - }) - - logger.info('Element of position %d of playlist %s updated.', playlistElement.position, videoPlaylist.uuid) - - return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function removeVideoFromPlaylist (req: express.Request, res: express.Response) { - const videoPlaylistElement = res.locals.videoPlaylistElement - const videoPlaylist = res.locals.videoPlaylistFull - const positionToDelete = videoPlaylistElement.position - - await sequelizeTypescript.transaction(async t => { - await videoPlaylistElement.destroy({ transaction: t }) - - // Decrease position of the next elements - await VideoPlaylistElementModel.increasePositionOf(videoPlaylist.id, positionToDelete, -1, t) - - videoPlaylist.changed('updatedAt', true) - await videoPlaylist.save({ transaction: t }) - - logger.info('Video playlist element %d of playlist %s deleted.', videoPlaylistElement.position, videoPlaylist.uuid) - }) - - // Do we need to regenerate the default thumbnail? - if (positionToDelete === 1 && videoPlaylist.hasGeneratedThumbnail()) { - await regeneratePlaylistThumbnail(videoPlaylist) - } - - sendUpdateVideoPlaylist(videoPlaylist, undefined) - .catch(err => logger.error('Cannot send video playlist update.', { err })) - - return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function reorderVideosPlaylist (req: express.Request, res: express.Response) { - const videoPlaylist = res.locals.videoPlaylistFull - const body: VideoPlaylistReorder = req.body - - const start: number = body.startPosition - const insertAfter: number = body.insertAfterPosition - const reorderLength: number = body.reorderLength || 1 - - if (start === insertAfter) { - return res.status(HttpStatusCode.NO_CONTENT_204).end() - } - - // Example: if we reorder position 2 and insert after position 5 (so at position 6): # 1 2 3 4 5 6 7 8 9 - // * increase position when position > 5 # 1 2 3 4 5 7 8 9 10 - // * update position 2 -> position 6 # 1 3 4 5 6 7 8 9 10 - // * decrease position when position position > 2 # 1 2 3 4 5 6 7 8 9 - await sequelizeTypescript.transaction(async t => { - const newPosition = insertAfter + 1 - - // Add space after the position when we want to insert our reordered elements (increase) - await VideoPlaylistElementModel.increasePositionOf(videoPlaylist.id, newPosition, reorderLength, t) - - let oldPosition = start - - // We incremented the position of the elements we want to reorder - if (start >= newPosition) oldPosition += reorderLength - - const endOldPosition = oldPosition + reorderLength - 1 - // Insert our reordered elements in their place (update) - await VideoPlaylistElementModel.reassignPositionOf({ - videoPlaylistId: videoPlaylist.id, - firstPosition: oldPosition, - endPosition: endOldPosition, - newPosition, - transaction: t - }) - - // Decrease positions of elements after the old position of our ordered elements (decrease) - await VideoPlaylistElementModel.increasePositionOf(videoPlaylist.id, oldPosition, -reorderLength, t) - - videoPlaylist.changed('updatedAt', true) - await videoPlaylist.save({ transaction: t }) - - await sendUpdateVideoPlaylist(videoPlaylist, t) - }) - - // The first element changed - if ((start === 1 || insertAfter === 0) && videoPlaylist.hasGeneratedThumbnail()) { - await regeneratePlaylistThumbnail(videoPlaylist) - } - - logger.info( - 'Reordered playlist %s (inserted after position %d elements %d - %d).', - videoPlaylist.uuid, insertAfter, start, start + reorderLength - 1 - ) - - return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function getVideoPlaylistVideos (req: express.Request, res: express.Response) { - const videoPlaylistInstance = res.locals.videoPlaylistSummary - const user = res.locals.oauth ? res.locals.oauth.token.User : undefined - const server = await getServerActor() - - const apiOptions = await Hooks.wrapObject({ - start: req.query.start, - count: req.query.count, - videoPlaylistId: videoPlaylistInstance.id, - serverAccount: server.Account, - user - }, 'filter:api.video-playlist.videos.list.params') - - const resultList = await Hooks.wrapPromiseFun( - VideoPlaylistElementModel.listForApi, - apiOptions, - 'filter:api.video-playlist.videos.list.result' - ) - - const options = { accountId: user?.Account?.id } - return res.json(getFormattedObjects(resultList.data, resultList.total, options)) -} - -async function regeneratePlaylistThumbnail (videoPlaylist: MVideoPlaylistThumbnail) { - await videoPlaylist.Thumbnail.destroy() - videoPlaylist.Thumbnail = null - - const firstElement = await VideoPlaylistElementModel.loadFirstElementWithVideoThumbnail(videoPlaylist.id) - if (firstElement) await generateThumbnailForPlaylist(videoPlaylist, firstElement.Video) -} - -async function generateThumbnailForPlaylist (videoPlaylist: MVideoPlaylistThumbnail, video: MVideoThumbnail) { - logger.info('Generating default thumbnail to playlist %s.', videoPlaylist.url) - - const videoMiniature = video.getMiniature() - if (!videoMiniature) { - logger.info('Cannot generate thumbnail for playlist %s because video %s does not have any.', videoPlaylist.url, video.url) - return - } - - // Ensure the file is on disk - const videoMiniaturePermanentFileCache = new VideoMiniaturePermanentFileCache() - const inputPath = videoMiniature.isOwned() - ? videoMiniature.getPath() - : await videoMiniaturePermanentFileCache.downloadRemoteFile(videoMiniature) - - const thumbnailModel = await updateLocalPlaylistMiniatureFromExisting({ - inputPath, - playlist: videoPlaylist, - automaticallyGenerated: true, - keepOriginal: true - }) - - thumbnailModel.videoPlaylistId = videoPlaylist.id - - videoPlaylist.Thumbnail = await thumbnailModel.save() -} diff --git a/server/controllers/api/videos/blacklist.ts b/server/controllers/api/videos/blacklist.ts deleted file mode 100644 index 4103bb063..000000000 --- a/server/controllers/api/videos/blacklist.ts +++ /dev/null @@ -1,112 +0,0 @@ -import express from 'express' -import { blacklistVideo, unblacklistVideo } from '@server/lib/video-blacklist' -import { HttpStatusCode, UserRight, VideoBlacklistCreate } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { getFormattedObjects } from '../../../helpers/utils' -import { sequelizeTypescript } from '../../../initializers/database' -import { - asyncMiddleware, - authenticate, - blacklistSortValidator, - ensureUserHasRight, - openapiOperationDoc, - paginationValidator, - setBlacklistSort, - setDefaultPagination, - videosBlacklistAddValidator, - videosBlacklistFiltersValidator, - videosBlacklistRemoveValidator, - videosBlacklistUpdateValidator -} from '../../../middlewares' -import { VideoBlacklistModel } from '../../../models/video/video-blacklist' - -const blacklistRouter = express.Router() - -blacklistRouter.post('/:videoId/blacklist', - openapiOperationDoc({ operationId: 'addVideoBlock' }), - authenticate, - ensureUserHasRight(UserRight.MANAGE_VIDEO_BLACKLIST), - asyncMiddleware(videosBlacklistAddValidator), - asyncMiddleware(addVideoToBlacklistController) -) - -blacklistRouter.get('/blacklist', - openapiOperationDoc({ operationId: 'getVideoBlocks' }), - authenticate, - ensureUserHasRight(UserRight.MANAGE_VIDEO_BLACKLIST), - paginationValidator, - blacklistSortValidator, - setBlacklistSort, - setDefaultPagination, - videosBlacklistFiltersValidator, - asyncMiddleware(listBlacklist) -) - -blacklistRouter.put('/:videoId/blacklist', - authenticate, - ensureUserHasRight(UserRight.MANAGE_VIDEO_BLACKLIST), - asyncMiddleware(videosBlacklistUpdateValidator), - asyncMiddleware(updateVideoBlacklistController) -) - -blacklistRouter.delete('/:videoId/blacklist', - openapiOperationDoc({ operationId: 'delVideoBlock' }), - authenticate, - ensureUserHasRight(UserRight.MANAGE_VIDEO_BLACKLIST), - asyncMiddleware(videosBlacklistRemoveValidator), - asyncMiddleware(removeVideoFromBlacklistController) -) - -// --------------------------------------------------------------------------- - -export { - blacklistRouter -} - -// --------------------------------------------------------------------------- - -async function addVideoToBlacklistController (req: express.Request, res: express.Response) { - const videoInstance = res.locals.videoAll - const body: VideoBlacklistCreate = req.body - - await blacklistVideo(videoInstance, body) - - logger.info('Video %s blacklisted.', videoInstance.uuid) - - return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function updateVideoBlacklistController (req: express.Request, res: express.Response) { - const videoBlacklist = res.locals.videoBlacklist - - if (req.body.reason !== undefined) videoBlacklist.reason = req.body.reason - - await sequelizeTypescript.transaction(t => { - return videoBlacklist.save({ transaction: t }) - }) - - return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function listBlacklist (req: express.Request, res: express.Response) { - const resultList = await VideoBlacklistModel.listForApi({ - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - search: req.query.search, - type: req.query.type - }) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function removeVideoFromBlacklistController (req: express.Request, res: express.Response) { - const videoBlacklist = res.locals.videoBlacklist - const video = res.locals.videoAll - - await unblacklistVideo(videoBlacklist, video) - - logger.info('Video %s removed from blacklist.', video.uuid) - - return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() -} diff --git a/server/controllers/api/videos/captions.ts b/server/controllers/api/videos/captions.ts deleted file mode 100644 index 2b511a398..000000000 --- a/server/controllers/api/videos/captions.ts +++ /dev/null @@ -1,93 +0,0 @@ -import express from 'express' -import { Hooks } from '@server/lib/plugins/hooks' -import { MVideoCaption } from '@server/types/models' -import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' -import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils' -import { createReqFiles } from '../../../helpers/express-utils' -import { logger } from '../../../helpers/logger' -import { getFormattedObjects } from '../../../helpers/utils' -import { MIMETYPES } from '../../../initializers/constants' -import { sequelizeTypescript } from '../../../initializers/database' -import { federateVideoIfNeeded } from '../../../lib/activitypub/videos' -import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate } from '../../../middlewares' -import { addVideoCaptionValidator, deleteVideoCaptionValidator, listVideoCaptionsValidator } from '../../../middlewares/validators' -import { VideoCaptionModel } from '../../../models/video/video-caption' - -const reqVideoCaptionAdd = createReqFiles([ 'captionfile' ], MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT) - -const videoCaptionsRouter = express.Router() - -videoCaptionsRouter.get('/:videoId/captions', - asyncMiddleware(listVideoCaptionsValidator), - asyncMiddleware(listVideoCaptions) -) -videoCaptionsRouter.put('/:videoId/captions/:captionLanguage', - authenticate, - reqVideoCaptionAdd, - asyncMiddleware(addVideoCaptionValidator), - asyncRetryTransactionMiddleware(addVideoCaption) -) -videoCaptionsRouter.delete('/:videoId/captions/:captionLanguage', - authenticate, - asyncMiddleware(deleteVideoCaptionValidator), - asyncRetryTransactionMiddleware(deleteVideoCaption) -) - -// --------------------------------------------------------------------------- - -export { - videoCaptionsRouter -} - -// --------------------------------------------------------------------------- - -async function listVideoCaptions (req: express.Request, res: express.Response) { - const data = await VideoCaptionModel.listVideoCaptions(res.locals.onlyVideo.id) - - return res.json(getFormattedObjects(data, data.length)) -} - -async function addVideoCaption (req: express.Request, res: express.Response) { - const videoCaptionPhysicalFile = req.files['captionfile'][0] - const video = res.locals.videoAll - - const captionLanguage = req.params.captionLanguage - - const videoCaption = new VideoCaptionModel({ - videoId: video.id, - filename: VideoCaptionModel.generateCaptionName(captionLanguage), - language: captionLanguage - }) as MVideoCaption - - // Move physical file - await moveAndProcessCaptionFile(videoCaptionPhysicalFile, videoCaption) - - await sequelizeTypescript.transaction(async t => { - await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t) - - // Update video update - await federateVideoIfNeeded(video, false, t) - }) - - Hooks.runAction('action:api.video-caption.created', { caption: videoCaption, req, res }) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function deleteVideoCaption (req: express.Request, res: express.Response) { - const video = res.locals.videoAll - const videoCaption = res.locals.videoCaption - - await sequelizeTypescript.transaction(async t => { - await videoCaption.destroy({ transaction: t }) - - // Send video update - await federateVideoIfNeeded(video, false, t) - }) - - logger.info('Video caption %s of video %s deleted.', videoCaption.language, video.uuid) - - Hooks.runAction('action:api.video-caption.deleted', { caption: videoCaption, req, res }) - - return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() -} diff --git a/server/controllers/api/videos/comment.ts b/server/controllers/api/videos/comment.ts deleted file mode 100644 index 70ca21500..000000000 --- a/server/controllers/api/videos/comment.ts +++ /dev/null @@ -1,234 +0,0 @@ -import { MCommentFormattable } from '@server/types/models' -import express from 'express' - -import { ResultList, ThreadsResultList, UserRight, VideoCommentCreate } from '../../../../shared/models' -import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' -import { VideoCommentThreads } from '../../../../shared/models/videos/comment/video-comment.model' -import { auditLoggerFactory, CommentAuditView, getAuditIdFromRes } from '../../../helpers/audit-logger' -import { getFormattedObjects } from '../../../helpers/utils' -import { sequelizeTypescript } from '../../../initializers/database' -import { Notifier } from '../../../lib/notifier' -import { Hooks } from '../../../lib/plugins/hooks' -import { buildFormattedCommentTree, createVideoComment, removeComment } from '../../../lib/video-comment' -import { - asyncMiddleware, - asyncRetryTransactionMiddleware, - authenticate, - ensureUserHasRight, - optionalAuthenticate, - paginationValidator, - setDefaultPagination, - setDefaultSort -} from '../../../middlewares' -import { - addVideoCommentReplyValidator, - addVideoCommentThreadValidator, - listVideoCommentsValidator, - listVideoCommentThreadsValidator, - listVideoThreadCommentsValidator, - removeVideoCommentValidator, - videoCommentsValidator, - videoCommentThreadsSortValidator -} from '../../../middlewares/validators' -import { AccountModel } from '../../../models/account/account' -import { VideoCommentModel } from '../../../models/video/video-comment' - -const auditLogger = auditLoggerFactory('comments') -const videoCommentRouter = express.Router() - -videoCommentRouter.get('/:videoId/comment-threads', - paginationValidator, - videoCommentThreadsSortValidator, - setDefaultSort, - setDefaultPagination, - asyncMiddleware(listVideoCommentThreadsValidator), - optionalAuthenticate, - asyncMiddleware(listVideoThreads) -) -videoCommentRouter.get('/:videoId/comment-threads/:threadId', - asyncMiddleware(listVideoThreadCommentsValidator), - optionalAuthenticate, - asyncMiddleware(listVideoThreadComments) -) - -videoCommentRouter.post('/:videoId/comment-threads', - authenticate, - asyncMiddleware(addVideoCommentThreadValidator), - asyncRetryTransactionMiddleware(addVideoCommentThread) -) -videoCommentRouter.post('/:videoId/comments/:commentId', - authenticate, - asyncMiddleware(addVideoCommentReplyValidator), - asyncRetryTransactionMiddleware(addVideoCommentReply) -) -videoCommentRouter.delete('/:videoId/comments/:commentId', - authenticate, - asyncMiddleware(removeVideoCommentValidator), - asyncRetryTransactionMiddleware(removeVideoComment) -) - -videoCommentRouter.get('/comments', - authenticate, - ensureUserHasRight(UserRight.SEE_ALL_COMMENTS), - paginationValidator, - videoCommentsValidator, - setDefaultSort, - setDefaultPagination, - listVideoCommentsValidator, - asyncMiddleware(listComments) -) - -// --------------------------------------------------------------------------- - -export { - videoCommentRouter -} - -// --------------------------------------------------------------------------- - -async function listComments (req: express.Request, res: express.Response) { - const options = { - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - - isLocal: req.query.isLocal, - onLocalVideo: req.query.onLocalVideo, - search: req.query.search, - searchAccount: req.query.searchAccount, - searchVideo: req.query.searchVideo - } - - const resultList = await VideoCommentModel.listCommentsForApi(options) - - return res.json({ - total: resultList.total, - data: resultList.data.map(c => c.toFormattedAdminJSON()) - }) -} - -async function listVideoThreads (req: express.Request, res: express.Response) { - const video = res.locals.onlyVideo - const user = res.locals.oauth ? res.locals.oauth.token.User : undefined - - let resultList: ThreadsResultList - - if (video.commentsEnabled === true) { - const apiOptions = await Hooks.wrapObject({ - videoId: video.id, - isVideoOwned: video.isOwned(), - start: req.query.start, - count: req.query.count, - sort: req.query.sort, - user - }, 'filter:api.video-threads.list.params') - - resultList = await Hooks.wrapPromiseFun( - VideoCommentModel.listThreadsForApi, - apiOptions, - 'filter:api.video-threads.list.result' - ) - } else { - resultList = { - total: 0, - totalNotDeletedComments: 0, - data: [] - } - } - - return res.json({ - ...getFormattedObjects(resultList.data, resultList.total), - totalNotDeletedComments: resultList.totalNotDeletedComments - } as VideoCommentThreads) -} - -async function listVideoThreadComments (req: express.Request, res: express.Response) { - const video = res.locals.onlyVideo - const user = res.locals.oauth ? res.locals.oauth.token.User : undefined - - let resultList: ResultList - - if (video.commentsEnabled === true) { - const apiOptions = await Hooks.wrapObject({ - videoId: video.id, - threadId: res.locals.videoCommentThread.id, - user - }, 'filter:api.video-thread-comments.list.params') - - resultList = await Hooks.wrapPromiseFun( - VideoCommentModel.listThreadCommentsForApi, - apiOptions, - 'filter:api.video-thread-comments.list.result' - ) - } else { - resultList = { - total: 0, - data: [] - } - } - - if (resultList.data.length === 0) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'No comments were found' - }) - } - - return res.json(buildFormattedCommentTree(resultList)) -} - -async function addVideoCommentThread (req: express.Request, res: express.Response) { - const videoCommentInfo: VideoCommentCreate = req.body - - const comment = await sequelizeTypescript.transaction(async t => { - const account = await AccountModel.load(res.locals.oauth.token.User.Account.id, t) - - return createVideoComment({ - text: videoCommentInfo.text, - inReplyToComment: null, - video: res.locals.videoAll, - account - }, t) - }) - - Notifier.Instance.notifyOnNewComment(comment) - auditLogger.create(getAuditIdFromRes(res), new CommentAuditView(comment.toFormattedJSON())) - - Hooks.runAction('action:api.video-thread.created', { comment, req, res }) - - return res.json({ comment: comment.toFormattedJSON() }) -} - -async function addVideoCommentReply (req: express.Request, res: express.Response) { - const videoCommentInfo: VideoCommentCreate = req.body - - const comment = await sequelizeTypescript.transaction(async t => { - const account = await AccountModel.load(res.locals.oauth.token.User.Account.id, t) - - return createVideoComment({ - text: videoCommentInfo.text, - inReplyToComment: res.locals.videoCommentFull, - video: res.locals.videoAll, - account - }, t) - }) - - Notifier.Instance.notifyOnNewComment(comment) - auditLogger.create(getAuditIdFromRes(res), new CommentAuditView(comment.toFormattedJSON())) - - Hooks.runAction('action:api.video-comment-reply.created', { comment, req, res }) - - return res.json({ comment: comment.toFormattedJSON() }) -} - -async function removeVideoComment (req: express.Request, res: express.Response) { - const videoCommentInstance = res.locals.videoCommentFull - - await removeComment(videoCommentInstance, req, res) - - auditLogger.delete(getAuditIdFromRes(res), new CommentAuditView(videoCommentInstance.toFormattedJSON())) - - return res.type('json') - .status(HttpStatusCode.NO_CONTENT_204) - .end() -} diff --git a/server/controllers/api/videos/files.ts b/server/controllers/api/videos/files.ts deleted file mode 100644 index 67b60ff63..000000000 --- a/server/controllers/api/videos/files.ts +++ /dev/null @@ -1,122 +0,0 @@ -import express from 'express' -import toInt from 'validator/lib/toInt' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' -import { updatePlaylistAfterFileChange } from '@server/lib/hls' -import { removeAllWebVideoFiles, removeHLSFile, removeHLSPlaylist, removeWebVideoFile } from '@server/lib/video-file' -import { VideoFileModel } from '@server/models/video/video-file' -import { HttpStatusCode, UserRight } from '@shared/models' -import { - asyncMiddleware, - authenticate, - ensureUserHasRight, - videoFileMetadataGetValidator, - videoFilesDeleteHLSFileValidator, - videoFilesDeleteHLSValidator, - videoFilesDeleteWebVideoFileValidator, - videoFilesDeleteWebVideoValidator, - videosGetValidator -} from '../../../middlewares' - -const lTags = loggerTagsFactory('api', 'video') -const filesRouter = express.Router() - -filesRouter.get('/:id/metadata/:videoFileId', - asyncMiddleware(videosGetValidator), - asyncMiddleware(videoFileMetadataGetValidator), - asyncMiddleware(getVideoFileMetadata) -) - -filesRouter.delete('/:id/hls', - authenticate, - ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES), - asyncMiddleware(videoFilesDeleteHLSValidator), - asyncMiddleware(removeHLSPlaylistController) -) -filesRouter.delete('/:id/hls/:videoFileId', - authenticate, - ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES), - asyncMiddleware(videoFilesDeleteHLSFileValidator), - asyncMiddleware(removeHLSFileController) -) - -filesRouter.delete( - [ '/:id/webtorrent', '/:id/web-videos' ], // TODO: remove webtorrent in V7 - authenticate, - ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES), - asyncMiddleware(videoFilesDeleteWebVideoValidator), - asyncMiddleware(removeAllWebVideoFilesController) -) -filesRouter.delete( - [ '/:id/webtorrent/:videoFileId', '/:id/web-videos/:videoFileId' ], // TODO: remove webtorrent in V7 - authenticate, - ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES), - asyncMiddleware(videoFilesDeleteWebVideoFileValidator), - asyncMiddleware(removeWebVideoFileController) -) - -// --------------------------------------------------------------------------- - -export { - filesRouter -} - -// --------------------------------------------------------------------------- - -async function getVideoFileMetadata (req: express.Request, res: express.Response) { - const videoFile = await VideoFileModel.loadWithMetadata(toInt(req.params.videoFileId)) - - return res.json(videoFile.metadata) -} - -// --------------------------------------------------------------------------- - -async function removeHLSPlaylistController (req: express.Request, res: express.Response) { - const video = res.locals.videoAll - - logger.info('Deleting HLS playlist of %s.', video.url, lTags(video.uuid)) - await removeHLSPlaylist(video) - - await federateVideoIfNeeded(video, false, undefined) - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -async function removeHLSFileController (req: express.Request, res: express.Response) { - const video = res.locals.videoAll - const videoFileId = +req.params.videoFileId - - logger.info('Deleting HLS file %d of %s.', videoFileId, video.url, lTags(video.uuid)) - - const playlist = await removeHLSFile(video, videoFileId) - if (playlist) await updatePlaylistAfterFileChange(video, playlist) - - await federateVideoIfNeeded(video, false, undefined) - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -// --------------------------------------------------------------------------- - -async function removeAllWebVideoFilesController (req: express.Request, res: express.Response) { - const video = res.locals.videoAll - - logger.info('Deleting Web Video files of %s.', video.url, lTags(video.uuid)) - - await removeAllWebVideoFiles(video) - await federateVideoIfNeeded(video, false, undefined) - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -async function removeWebVideoFileController (req: express.Request, res: express.Response) { - const video = res.locals.videoAll - - const videoFileId = +req.params.videoFileId - logger.info('Deleting Web Video file %d of %s.', videoFileId, video.url, lTags(video.uuid)) - - await removeWebVideoFile(video, videoFileId) - await federateVideoIfNeeded(video, false, undefined) - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} diff --git a/server/controllers/api/videos/import.ts b/server/controllers/api/videos/import.ts deleted file mode 100644 index defe9efd4..000000000 --- a/server/controllers/api/videos/import.ts +++ /dev/null @@ -1,262 +0,0 @@ -import express from 'express' -import { move, readFile } from 'fs-extra' -import { decode } from 'magnet-uri' -import parseTorrent, { Instance } from 'parse-torrent' -import { join } from 'path' -import { buildYoutubeDLImport, buildVideoFromImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-pre-import' -import { MThumbnail, MVideoThumbnail } from '@server/types/models' -import { HttpStatusCode, ServerErrorCode, ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState } from '@shared/models' -import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger' -import { isArray } from '../../../helpers/custom-validators/misc' -import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils' -import { logger } from '../../../helpers/logger' -import { getSecureTorrentName } from '../../../helpers/utils' -import { CONFIG } from '../../../initializers/config' -import { MIMETYPES } from '../../../initializers/constants' -import { JobQueue } from '../../../lib/job-queue/job-queue' -import { updateLocalVideoMiniatureFromExisting } from '../../../lib/thumbnail' -import { - asyncMiddleware, - asyncRetryTransactionMiddleware, - authenticate, - videoImportAddValidator, - videoImportCancelValidator, - videoImportDeleteValidator -} from '../../../middlewares' - -const auditLogger = auditLoggerFactory('video-imports') -const videoImportsRouter = express.Router() - -const reqVideoFileImport = createReqFiles( - [ 'thumbnailfile', 'previewfile', 'torrentfile' ], - { ...MIMETYPES.TORRENT.MIMETYPE_EXT, ...MIMETYPES.IMAGE.MIMETYPE_EXT } -) - -videoImportsRouter.post('/imports', - authenticate, - reqVideoFileImport, - asyncMiddleware(videoImportAddValidator), - asyncRetryTransactionMiddleware(handleVideoImport) -) - -videoImportsRouter.post('/imports/:id/cancel', - authenticate, - asyncMiddleware(videoImportCancelValidator), - asyncRetryTransactionMiddleware(cancelVideoImport) -) - -videoImportsRouter.delete('/imports/:id', - authenticate, - asyncMiddleware(videoImportDeleteValidator), - asyncRetryTransactionMiddleware(deleteVideoImport) -) - -// --------------------------------------------------------------------------- - -export { - videoImportsRouter -} - -// --------------------------------------------------------------------------- - -async function deleteVideoImport (req: express.Request, res: express.Response) { - const videoImport = res.locals.videoImport - - await videoImport.destroy() - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -async function cancelVideoImport (req: express.Request, res: express.Response) { - const videoImport = res.locals.videoImport - - videoImport.state = VideoImportState.CANCELLED - await videoImport.save() - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -function handleVideoImport (req: express.Request, res: express.Response) { - if (req.body.targetUrl) return handleYoutubeDlImport(req, res) - - const file = req.files?.['torrentfile']?.[0] - if (req.body.magnetUri || file) return handleTorrentImport(req, res, file) -} - -async function handleTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { - const body: VideoImportCreate = req.body - const user = res.locals.oauth.token.User - - let videoName: string - let torrentName: string - let magnetUri: string - - if (torrentfile) { - const result = await processTorrentOrAbortRequest(req, res, torrentfile) - if (!result) return - - videoName = result.name - torrentName = result.torrentName - } else { - const result = processMagnetURI(body) - magnetUri = result.magnetUri - videoName = result.name - } - - const video = await buildVideoFromImport({ - channelId: res.locals.videoChannel.id, - importData: { name: videoName }, - importDataOverride: body, - importType: 'torrent' - }) - - const thumbnailModel = await processThumbnail(req, video) - const previewModel = await processPreview(req, video) - - const videoImport = await insertFromImportIntoDB({ - video, - thumbnailModel, - previewModel, - videoChannel: res.locals.videoChannel, - tags: body.tags || undefined, - user, - videoPasswords: body.videoPasswords, - videoImportAttributes: { - magnetUri, - torrentName, - state: VideoImportState.PENDING, - userId: user.id - } - }) - - const payload: VideoImportPayload = { - type: torrentfile - ? 'torrent-file' - : 'magnet-uri', - videoImportId: videoImport.id, - preventException: false - } - await JobQueue.Instance.createJob({ type: 'video-import', payload }) - - auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON())) - - return res.json(videoImport.toFormattedJSON()).end() -} - -function statusFromYtDlImportError (err: YoutubeDlImportError): number { - switch (err.code) { - case YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL: - return HttpStatusCode.FORBIDDEN_403 - - case YoutubeDlImportError.CODE.FETCH_ERROR: - return HttpStatusCode.BAD_REQUEST_400 - - default: - return HttpStatusCode.INTERNAL_SERVER_ERROR_500 - } -} - -async function handleYoutubeDlImport (req: express.Request, res: express.Response) { - const body: VideoImportCreate = req.body - const targetUrl = body.targetUrl - const user = res.locals.oauth.token.User - - try { - const { job, videoImport } = await buildYoutubeDLImport({ - targetUrl, - channel: res.locals.videoChannel, - importDataOverride: body, - thumbnailFilePath: req.files?.['thumbnailfile']?.[0].path, - previewFilePath: req.files?.['previewfile']?.[0].path, - user - }) - await JobQueue.Instance.createJob(job) - - auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON())) - - return res.json(videoImport.toFormattedJSON()).end() - } catch (err) { - logger.error('An error occurred while importing the video %s. ', targetUrl, { err }) - - return res.fail({ - message: err.message, - status: statusFromYtDlImportError(err), - data: { - targetUrl - } - }) - } -} - -async function processThumbnail (req: express.Request, video: MVideoThumbnail) { - const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined - if (thumbnailField) { - const thumbnailPhysicalFile = thumbnailField[0] - - return updateLocalVideoMiniatureFromExisting({ - inputPath: thumbnailPhysicalFile.path, - video, - type: ThumbnailType.MINIATURE, - automaticallyGenerated: false - }) - } - - return undefined -} - -async function processPreview (req: express.Request, video: MVideoThumbnail): Promise { - const previewField = req.files ? req.files['previewfile'] : undefined - if (previewField) { - const previewPhysicalFile = previewField[0] - - return updateLocalVideoMiniatureFromExisting({ - inputPath: previewPhysicalFile.path, - video, - type: ThumbnailType.PREVIEW, - automaticallyGenerated: false - }) - } - - return undefined -} - -async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { - const torrentName = torrentfile.originalname - - // Rename the torrent to a secured name - const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName)) - await move(torrentfile.path, newTorrentPath, { overwrite: true }) - torrentfile.path = newTorrentPath - - const buf = await readFile(torrentfile.path) - const parsedTorrent = parseTorrent(buf) as Instance - - if (parsedTorrent.files.length !== 1) { - cleanUpReqFiles(req) - - res.fail({ - type: ServerErrorCode.INCORRECT_FILES_IN_TORRENT, - message: 'Torrents with only 1 file are supported.' - }) - return undefined - } - - return { - name: extractNameFromArray(parsedTorrent.name), - torrentName - } -} - -function processMagnetURI (body: VideoImportCreate) { - const magnetUri = body.magnetUri - const parsed = decode(magnetUri) - - return { - name: extractNameFromArray(parsed.name), - magnetUri - } -} - -function extractNameFromArray (name: string | string[]) { - return isArray(name) ? name[0] : name -} diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts deleted file mode 100644 index 3cdd42289..000000000 --- a/server/controllers/api/videos/index.ts +++ /dev/null @@ -1,228 +0,0 @@ -import express from 'express' -import { pickCommonVideoQuery } from '@server/helpers/query' -import { doJSONRequest } from '@server/helpers/requests' -import { openapiOperationDoc } from '@server/middlewares/doc' -import { getServerActor } from '@server/models/application/application' -import { MVideoAccountLight } from '@server/types/models' -import { HttpStatusCode } from '../../../../shared/models' -import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' -import { buildNSFWFilter, getCountVideos } from '../../../helpers/express-utils' -import { logger } from '../../../helpers/logger' -import { getFormattedObjects } from '../../../helpers/utils' -import { REMOTE_SCHEME, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../initializers/constants' -import { sequelizeTypescript } from '../../../initializers/database' -import { JobQueue } from '../../../lib/job-queue' -import { Hooks } from '../../../lib/plugins/hooks' -import { - apiRateLimiter, - asyncMiddleware, - asyncRetryTransactionMiddleware, - authenticate, - checkVideoFollowConstraints, - commonVideosFiltersValidator, - optionalAuthenticate, - paginationValidator, - setDefaultPagination, - setDefaultVideosSort, - videosCustomGetValidator, - videosGetValidator, - videosRemoveValidator, - videosSortValidator -} from '../../../middlewares' -import { guessAdditionalAttributesFromQuery } from '../../../models/video/formatter' -import { VideoModel } from '../../../models/video/video' -import { blacklistRouter } from './blacklist' -import { videoCaptionsRouter } from './captions' -import { videoCommentRouter } from './comment' -import { filesRouter } from './files' -import { videoImportsRouter } from './import' -import { liveRouter } from './live' -import { ownershipVideoRouter } from './ownership' -import { videoPasswordRouter } from './passwords' -import { rateVideoRouter } from './rate' -import { videoSourceRouter } from './source' -import { statsRouter } from './stats' -import { storyboardRouter } from './storyboard' -import { studioRouter } from './studio' -import { tokenRouter } from './token' -import { transcodingRouter } from './transcoding' -import { updateRouter } from './update' -import { uploadRouter } from './upload' -import { viewRouter } from './view' - -const auditLogger = auditLoggerFactory('videos') -const videosRouter = express.Router() - -videosRouter.use(apiRateLimiter) - -videosRouter.use('/', blacklistRouter) -videosRouter.use('/', statsRouter) -videosRouter.use('/', rateVideoRouter) -videosRouter.use('/', videoCommentRouter) -videosRouter.use('/', studioRouter) -videosRouter.use('/', videoCaptionsRouter) -videosRouter.use('/', videoImportsRouter) -videosRouter.use('/', ownershipVideoRouter) -videosRouter.use('/', viewRouter) -videosRouter.use('/', liveRouter) -videosRouter.use('/', uploadRouter) -videosRouter.use('/', updateRouter) -videosRouter.use('/', filesRouter) -videosRouter.use('/', transcodingRouter) -videosRouter.use('/', tokenRouter) -videosRouter.use('/', videoPasswordRouter) -videosRouter.use('/', storyboardRouter) -videosRouter.use('/', videoSourceRouter) - -videosRouter.get('/categories', - openapiOperationDoc({ operationId: 'getCategories' }), - listVideoCategories -) -videosRouter.get('/licences', - openapiOperationDoc({ operationId: 'getLicences' }), - listVideoLicences -) -videosRouter.get('/languages', - openapiOperationDoc({ operationId: 'getLanguages' }), - listVideoLanguages -) -videosRouter.get('/privacies', - openapiOperationDoc({ operationId: 'getPrivacies' }), - listVideoPrivacies -) - -videosRouter.get('/', - openapiOperationDoc({ operationId: 'getVideos' }), - paginationValidator, - videosSortValidator, - setDefaultVideosSort, - setDefaultPagination, - optionalAuthenticate, - commonVideosFiltersValidator, - asyncMiddleware(listVideos) -) - -// TODO: remove, deprecated in 5.0 now we send the complete description in VideoDetails -videosRouter.get('/:id/description', - openapiOperationDoc({ operationId: 'getVideoDesc' }), - asyncMiddleware(videosGetValidator), - asyncMiddleware(getVideoDescription) -) - -videosRouter.get('/:id', - openapiOperationDoc({ operationId: 'getVideo' }), - optionalAuthenticate, - asyncMiddleware(videosCustomGetValidator('for-api')), - asyncMiddleware(checkVideoFollowConstraints), - asyncMiddleware(getVideo) -) - -videosRouter.delete('/:id', - openapiOperationDoc({ operationId: 'delVideo' }), - authenticate, - asyncMiddleware(videosRemoveValidator), - asyncRetryTransactionMiddleware(removeVideo) -) - -// --------------------------------------------------------------------------- - -export { - videosRouter -} - -// --------------------------------------------------------------------------- - -function listVideoCategories (_req: express.Request, res: express.Response) { - res.json(VIDEO_CATEGORIES) -} - -function listVideoLicences (_req: express.Request, res: express.Response) { - res.json(VIDEO_LICENCES) -} - -function listVideoLanguages (_req: express.Request, res: express.Response) { - res.json(VIDEO_LANGUAGES) -} - -function listVideoPrivacies (_req: express.Request, res: express.Response) { - res.json(VIDEO_PRIVACIES) -} - -async function getVideo (_req: express.Request, res: express.Response) { - const videoId = res.locals.videoAPI.id - const userId = res.locals.oauth?.token.User.id - - const video = await Hooks.wrapObject(res.locals.videoAPI, 'filter:api.video.get.result', { id: videoId, userId }) - - if (video.isOutdated()) { - JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'video', url: video.url } }) - } - - return res.json(video.toFormattedDetailsJSON()) -} - -async function getVideoDescription (req: express.Request, res: express.Response) { - const videoInstance = res.locals.videoAll - - const description = videoInstance.isOwned() - ? videoInstance.description - : await fetchRemoteVideoDescription(videoInstance) - - return res.json({ description }) -} - -async function listVideos (req: express.Request, res: express.Response) { - const serverActor = await getServerActor() - - const query = pickCommonVideoQuery(req.query) - const countVideos = getCountVideos(req) - - const apiOptions = await Hooks.wrapObject({ - ...query, - - displayOnlyForFollower: { - actorId: serverActor.id, - orLocalVideos: true - }, - nsfw: buildNSFWFilter(res, query.nsfw), - user: res.locals.oauth ? res.locals.oauth.token.User : undefined, - countVideos - }, 'filter:api.videos.list.params') - - const resultList = await Hooks.wrapPromiseFun( - VideoModel.listForApi, - apiOptions, - 'filter:api.videos.list.result' - ) - - return res.json(getFormattedObjects(resultList.data, resultList.total, guessAdditionalAttributesFromQuery(query))) -} - -async function removeVideo (req: express.Request, res: express.Response) { - const videoInstance = res.locals.videoAll - - await sequelizeTypescript.transaction(async t => { - await videoInstance.destroy({ transaction: t }) - }) - - auditLogger.delete(getAuditIdFromRes(res), new VideoAuditView(videoInstance.toFormattedDetailsJSON())) - logger.info('Video with name %s and uuid %s deleted.', videoInstance.name, videoInstance.uuid) - - Hooks.runAction('action:api.video.deleted', { video: videoInstance, req, res }) - - return res.type('json') - .status(HttpStatusCode.NO_CONTENT_204) - .end() -} - -// --------------------------------------------------------------------------- - -// FIXME: Should not exist, we rely on specific API -async function fetchRemoteVideoDescription (video: MVideoAccountLight) { - const host = video.VideoChannel.Account.Actor.Server.host - const path = video.getDescriptionAPIPath() - const url = REMOTE_SCHEME.HTTP + '://' + host + path - - const { body } = await doJSONRequest(url) - return body.description || '' -} diff --git a/server/controllers/api/videos/live.ts b/server/controllers/api/videos/live.ts deleted file mode 100644 index e19e8c652..000000000 --- a/server/controllers/api/videos/live.ts +++ /dev/null @@ -1,224 +0,0 @@ -import express from 'express' -import { exists } from '@server/helpers/custom-validators/misc' -import { createReqFiles } from '@server/helpers/express-utils' -import { getFormattedObjects } from '@server/helpers/utils' -import { ASSETS_PATH, MIMETYPES } from '@server/initializers/constants' -import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' -import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' -import { Hooks } from '@server/lib/plugins/hooks' -import { buildLocalVideoFromReq, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video' -import { - videoLiveAddValidator, - videoLiveFindReplaySessionValidator, - videoLiveGetValidator, - videoLiveListSessionsValidator, - videoLiveUpdateValidator -} from '@server/middlewares/validators/videos/video-live' -import { VideoLiveModel } from '@server/models/video/video-live' -import { VideoLiveSessionModel } from '@server/models/video/video-live-session' -import { MVideoDetails, MVideoFullLight, MVideoLive } from '@server/types/models' -import { buildUUID, uuidToShort } from '@shared/extra-utils' -import { HttpStatusCode, LiveVideoCreate, LiveVideoLatencyMode, LiveVideoUpdate, UserRight, VideoPrivacy, VideoState } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { sequelizeTypescript } from '../../../initializers/database' -import { updateLocalVideoMiniatureFromExisting } from '../../../lib/thumbnail' -import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, optionalAuthenticate } from '../../../middlewares' -import { VideoModel } from '../../../models/video/video' -import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting' -import { VideoPasswordModel } from '@server/models/video/video-password' - -const liveRouter = express.Router() - -const reqVideoFileLive = createReqFiles([ 'thumbnailfile', 'previewfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT) - -liveRouter.post('/live', - authenticate, - reqVideoFileLive, - asyncMiddleware(videoLiveAddValidator), - asyncRetryTransactionMiddleware(addLiveVideo) -) - -liveRouter.get('/live/:videoId/sessions', - authenticate, - asyncMiddleware(videoLiveGetValidator), - videoLiveListSessionsValidator, - asyncMiddleware(getLiveVideoSessions) -) - -liveRouter.get('/live/:videoId', - optionalAuthenticate, - asyncMiddleware(videoLiveGetValidator), - getLiveVideo -) - -liveRouter.put('/live/:videoId', - authenticate, - asyncMiddleware(videoLiveGetValidator), - videoLiveUpdateValidator, - asyncRetryTransactionMiddleware(updateLiveVideo) -) - -liveRouter.get('/:videoId/live-session', - asyncMiddleware(videoLiveFindReplaySessionValidator), - getLiveReplaySession -) - -// --------------------------------------------------------------------------- - -export { - liveRouter -} - -// --------------------------------------------------------------------------- - -function getLiveVideo (req: express.Request, res: express.Response) { - const videoLive = res.locals.videoLive - - return res.json(videoLive.toFormattedJSON(canSeePrivateLiveInformation(res))) -} - -function getLiveReplaySession (req: express.Request, res: express.Response) { - const session = res.locals.videoLiveSession - - return res.json(session.toFormattedJSON()) -} - -async function getLiveVideoSessions (req: express.Request, res: express.Response) { - const videoLive = res.locals.videoLive - - const data = await VideoLiveSessionModel.listSessionsOfLiveForAPI({ videoId: videoLive.videoId }) - - return res.json(getFormattedObjects(data, data.length)) -} - -function canSeePrivateLiveInformation (res: express.Response) { - const user = res.locals.oauth?.token.User - if (!user) return false - - if (user.hasRight(UserRight.GET_ANY_LIVE)) return true - - const video = res.locals.videoAll - return video.VideoChannel.Account.userId === user.id -} - -async function updateLiveVideo (req: express.Request, res: express.Response) { - const body: LiveVideoUpdate = req.body - - const video = res.locals.videoAll - const videoLive = res.locals.videoLive - - const newReplaySettingModel = await updateReplaySettings(videoLive, body) - if (newReplaySettingModel) videoLive.replaySettingId = newReplaySettingModel.id - else videoLive.replaySettingId = null - - if (exists(body.permanentLive)) videoLive.permanentLive = body.permanentLive - if (exists(body.latencyMode)) videoLive.latencyMode = body.latencyMode - - video.VideoLive = await videoLive.save() - - await federateVideoIfNeeded(video, false) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function updateReplaySettings (videoLive: MVideoLive, body: LiveVideoUpdate) { - if (exists(body.saveReplay)) videoLive.saveReplay = body.saveReplay - - // The live replay is not saved anymore, destroy the old model if it existed - if (!videoLive.saveReplay) { - if (videoLive.replaySettingId) { - await VideoLiveReplaySettingModel.removeSettings(videoLive.replaySettingId) - } - - return undefined - } - - const settingModel = videoLive.replaySettingId - ? await VideoLiveReplaySettingModel.load(videoLive.replaySettingId) - : new VideoLiveReplaySettingModel() - - if (exists(body.replaySettings.privacy)) settingModel.privacy = body.replaySettings.privacy - - return settingModel.save() -} - -async function addLiveVideo (req: express.Request, res: express.Response) { - const videoInfo: LiveVideoCreate = req.body - - // Prepare data so we don't block the transaction - let videoData = buildLocalVideoFromReq(videoInfo, res.locals.videoChannel.id) - videoData = await Hooks.wrapObject(videoData, 'filter:api.video.live.video-attribute.result') - - videoData.isLive = true - videoData.state = VideoState.WAITING_FOR_LIVE - videoData.duration = 0 - - const video = new VideoModel(videoData) as MVideoDetails - video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object - - const videoLive = new VideoLiveModel() - videoLive.saveReplay = videoInfo.saveReplay || false - videoLive.permanentLive = videoInfo.permanentLive || false - videoLive.latencyMode = videoInfo.latencyMode || LiveVideoLatencyMode.DEFAULT - videoLive.streamKey = buildUUID() - - const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ - video, - files: req.files, - fallback: type => { - return updateLocalVideoMiniatureFromExisting({ - inputPath: ASSETS_PATH.DEFAULT_LIVE_BACKGROUND, - video, - type, - automaticallyGenerated: true, - keepOriginal: true - }) - } - }) - - const { videoCreated } = await sequelizeTypescript.transaction(async t => { - const sequelizeOptions = { transaction: t } - - const videoCreated = await video.save(sequelizeOptions) as MVideoFullLight - - if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t) - if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t) - - // Do not forget to add video channel information to the created video - videoCreated.VideoChannel = res.locals.videoChannel - - if (videoLive.saveReplay) { - const replaySettings = new VideoLiveReplaySettingModel({ - privacy: videoInfo.replaySettings.privacy - }) - await replaySettings.save(sequelizeOptions) - - videoLive.replaySettingId = replaySettings.id - } - - videoLive.videoId = videoCreated.id - videoCreated.VideoLive = await videoLive.save(sequelizeOptions) - - await setVideoTags({ video, tags: videoInfo.tags, transaction: t }) - - await federateVideoIfNeeded(videoCreated, true, t) - - if (videoInfo.privacy === VideoPrivacy.PASSWORD_PROTECTED) { - await VideoPasswordModel.addPasswords(videoInfo.videoPasswords, video.id, t) - } - - logger.info('Video live %s with uuid %s created.', videoInfo.name, videoCreated.uuid) - - return { videoCreated } - }) - - Hooks.runAction('action:api.live-video.created', { video: videoCreated, req, res }) - - return res.json({ - video: { - id: videoCreated.id, - shortUUID: uuidToShort(videoCreated.uuid), - uuid: videoCreated.uuid - } - }) -} diff --git a/server/controllers/api/videos/ownership.ts b/server/controllers/api/videos/ownership.ts deleted file mode 100644 index 88355b289..000000000 --- a/server/controllers/api/videos/ownership.ts +++ /dev/null @@ -1,138 +0,0 @@ -import express from 'express' -import { MVideoFullLight } from '@server/types/models' -import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' -import { VideoChangeOwnershipStatus, VideoState } from '../../../../shared/models/videos' -import { logger } from '../../../helpers/logger' -import { getFormattedObjects } from '../../../helpers/utils' -import { sequelizeTypescript } from '../../../initializers/database' -import { sendUpdateVideo } from '../../../lib/activitypub/send' -import { changeVideoChannelShare } from '../../../lib/activitypub/share' -import { - asyncMiddleware, - asyncRetryTransactionMiddleware, - authenticate, - paginationValidator, - setDefaultPagination, - videosAcceptChangeOwnershipValidator, - videosChangeOwnershipValidator, - videosTerminateChangeOwnershipValidator -} from '../../../middlewares' -import { VideoModel } from '../../../models/video/video' -import { VideoChangeOwnershipModel } from '../../../models/video/video-change-ownership' -import { VideoChannelModel } from '../../../models/video/video-channel' - -const ownershipVideoRouter = express.Router() - -ownershipVideoRouter.post('/:videoId/give-ownership', - authenticate, - asyncMiddleware(videosChangeOwnershipValidator), - asyncRetryTransactionMiddleware(giveVideoOwnership) -) - -ownershipVideoRouter.get('/ownership', - authenticate, - paginationValidator, - setDefaultPagination, - asyncRetryTransactionMiddleware(listVideoOwnership) -) - -ownershipVideoRouter.post('/ownership/:id/accept', - authenticate, - asyncMiddleware(videosTerminateChangeOwnershipValidator), - asyncMiddleware(videosAcceptChangeOwnershipValidator), - asyncRetryTransactionMiddleware(acceptOwnership) -) - -ownershipVideoRouter.post('/ownership/:id/refuse', - authenticate, - asyncMiddleware(videosTerminateChangeOwnershipValidator), - asyncRetryTransactionMiddleware(refuseOwnership) -) - -// --------------------------------------------------------------------------- - -export { - ownershipVideoRouter -} - -// --------------------------------------------------------------------------- - -async function giveVideoOwnership (req: express.Request, res: express.Response) { - const videoInstance = res.locals.videoAll - const initiatorAccountId = res.locals.oauth.token.User.Account.id - const nextOwner = res.locals.nextOwner - - await sequelizeTypescript.transaction(t => { - return VideoChangeOwnershipModel.findOrCreate({ - where: { - initiatorAccountId, - nextOwnerAccountId: nextOwner.id, - videoId: videoInstance.id, - status: VideoChangeOwnershipStatus.WAITING - }, - defaults: { - initiatorAccountId, - nextOwnerAccountId: nextOwner.id, - videoId: videoInstance.id, - status: VideoChangeOwnershipStatus.WAITING - }, - transaction: t - }) - }) - - logger.info('Ownership change for video %s created.', videoInstance.name) - return res.type('json') - .status(HttpStatusCode.NO_CONTENT_204) - .end() -} - -async function listVideoOwnership (req: express.Request, res: express.Response) { - const currentAccountId = res.locals.oauth.token.User.Account.id - - const resultList = await VideoChangeOwnershipModel.listForApi( - currentAccountId, - req.query.start || 0, - req.query.count || 10, - req.query.sort || 'createdAt' - ) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -function acceptOwnership (req: express.Request, res: express.Response) { - return sequelizeTypescript.transaction(async t => { - const videoChangeOwnership = res.locals.videoChangeOwnership - const channel = res.locals.videoChannel - - // We need more attributes for federation - const targetVideo = await VideoModel.loadFull(videoChangeOwnership.Video.id, t) - - const oldVideoChannel = await VideoChannelModel.loadAndPopulateAccount(targetVideo.channelId, t) - - targetVideo.channelId = channel.id - - const targetVideoUpdated = await targetVideo.save({ transaction: t }) as MVideoFullLight - targetVideoUpdated.VideoChannel = channel - - if (targetVideoUpdated.hasPrivacyForFederation() && targetVideoUpdated.state === VideoState.PUBLISHED) { - await changeVideoChannelShare(targetVideoUpdated, oldVideoChannel, t) - await sendUpdateVideo(targetVideoUpdated, t, oldVideoChannel.Account.Actor) - } - - videoChangeOwnership.status = VideoChangeOwnershipStatus.ACCEPTED - await videoChangeOwnership.save({ transaction: t }) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() - }) -} - -function refuseOwnership (req: express.Request, res: express.Response) { - return sequelizeTypescript.transaction(async t => { - const videoChangeOwnership = res.locals.videoChangeOwnership - - videoChangeOwnership.status = VideoChangeOwnershipStatus.REFUSED - await videoChangeOwnership.save({ transaction: t }) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() - }) -} diff --git a/server/controllers/api/videos/passwords.ts b/server/controllers/api/videos/passwords.ts deleted file mode 100644 index d11cf5bcc..000000000 --- a/server/controllers/api/videos/passwords.ts +++ /dev/null @@ -1,105 +0,0 @@ -import express from 'express' - -import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' -import { getFormattedObjects } from '../../../helpers/utils' -import { - asyncMiddleware, - asyncRetryTransactionMiddleware, - authenticate, - setDefaultPagination, - setDefaultSort -} from '../../../middlewares' -import { - listVideoPasswordValidator, - paginationValidator, - removeVideoPasswordValidator, - updateVideoPasswordListValidator, - videoPasswordsSortValidator -} from '../../../middlewares/validators' -import { VideoPasswordModel } from '@server/models/video/video-password' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { Transaction } from 'sequelize' -import { getVideoWithAttributes } from '@server/helpers/video' - -const lTags = loggerTagsFactory('api', 'video') -const videoPasswordRouter = express.Router() - -videoPasswordRouter.get('/:videoId/passwords', - authenticate, - paginationValidator, - videoPasswordsSortValidator, - setDefaultSort, - setDefaultPagination, - asyncMiddleware(listVideoPasswordValidator), - asyncMiddleware(listVideoPasswords) -) - -videoPasswordRouter.put('/:videoId/passwords', - authenticate, - asyncMiddleware(updateVideoPasswordListValidator), - asyncMiddleware(updateVideoPasswordList) -) - -videoPasswordRouter.delete('/:videoId/passwords/:passwordId', - authenticate, - asyncMiddleware(removeVideoPasswordValidator), - asyncRetryTransactionMiddleware(removeVideoPassword) -) - -// --------------------------------------------------------------------------- - -export { - videoPasswordRouter -} - -// --------------------------------------------------------------------------- - -async function listVideoPasswords (req: express.Request, res: express.Response) { - const options = { - videoId: res.locals.videoAll.id, - start: req.query.start, - count: req.query.count, - sort: req.query.sort - } - - const resultList = await VideoPasswordModel.listPasswords(options) - - return res.json(getFormattedObjects(resultList.data, resultList.total)) -} - -async function updateVideoPasswordList (req: express.Request, res: express.Response) { - const videoInstance = getVideoWithAttributes(res) - const videoId = videoInstance.id - - const passwordArray = req.body.passwords as string[] - - await VideoPasswordModel.sequelize.transaction(async (t: Transaction) => { - await VideoPasswordModel.deleteAllPasswords(videoId, t) - await VideoPasswordModel.addPasswords(passwordArray, videoId, t) - }) - - logger.info( - `Video passwords for video with name %s and uuid %s have been updated`, - videoInstance.name, - videoInstance.uuid, - lTags(videoInstance.uuid) - ) - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -async function removeVideoPassword (req: express.Request, res: express.Response) { - const videoInstance = getVideoWithAttributes(res) - const password = res.locals.videoPassword - - await VideoPasswordModel.deletePassword(password.id) - logger.info( - 'Password with id %d of video named %s and uuid %s has been deleted.', - password.id, - videoInstance.name, - videoInstance.uuid, - lTags(videoInstance.uuid) - ) - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} diff --git a/server/controllers/api/videos/rate.ts b/server/controllers/api/videos/rate.ts deleted file mode 100644 index 6b26a8eee..000000000 --- a/server/controllers/api/videos/rate.ts +++ /dev/null @@ -1,87 +0,0 @@ -import express from 'express' -import { HttpStatusCode, UserVideoRateUpdate } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { VIDEO_RATE_TYPES } from '../../../initializers/constants' -import { sequelizeTypescript } from '../../../initializers/database' -import { getLocalRateUrl, sendVideoRateChange } from '../../../lib/activitypub/video-rates' -import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoUpdateRateValidator } from '../../../middlewares' -import { AccountModel } from '../../../models/account/account' -import { AccountVideoRateModel } from '../../../models/account/account-video-rate' - -const rateVideoRouter = express.Router() - -rateVideoRouter.put('/:id/rate', - authenticate, - asyncMiddleware(videoUpdateRateValidator), - asyncRetryTransactionMiddleware(rateVideo) -) - -// --------------------------------------------------------------------------- - -export { - rateVideoRouter -} - -// --------------------------------------------------------------------------- - -async function rateVideo (req: express.Request, res: express.Response) { - const body: UserVideoRateUpdate = req.body - const rateType = body.rating - const videoInstance = res.locals.videoAll - const userAccount = res.locals.oauth.token.User.Account - - await sequelizeTypescript.transaction(async t => { - const sequelizeOptions = { transaction: t } - - const accountInstance = await AccountModel.load(userAccount.id, t) - const previousRate = await AccountVideoRateModel.load(accountInstance.id, videoInstance.id, t) - - // Same rate, nothing do to - if (rateType === 'none' && !previousRate || previousRate?.type === rateType) return - - let likesToIncrement = 0 - let dislikesToIncrement = 0 - - if (rateType === VIDEO_RATE_TYPES.LIKE) likesToIncrement++ - else if (rateType === VIDEO_RATE_TYPES.DISLIKE) dislikesToIncrement++ - - // There was a previous rate, update it - if (previousRate) { - // We will remove the previous rate, so we will need to update the video count attribute - if (previousRate.type === 'like') likesToIncrement-- - else if (previousRate.type === 'dislike') dislikesToIncrement-- - - if (rateType === 'none') { // Destroy previous rate - await previousRate.destroy(sequelizeOptions) - } else { // Update previous rate - previousRate.type = rateType - previousRate.url = getLocalRateUrl(rateType, userAccount.Actor, videoInstance) - await previousRate.save(sequelizeOptions) - } - } else if (rateType !== 'none') { // There was not a previous rate, insert a new one if there is a rate - const query = { - accountId: accountInstance.id, - videoId: videoInstance.id, - type: rateType, - url: getLocalRateUrl(rateType, userAccount.Actor, videoInstance) - } - - await AccountVideoRateModel.create(query, sequelizeOptions) - } - - const incrementQuery = { - likes: likesToIncrement, - dislikes: dislikesToIncrement - } - - await videoInstance.increment(incrementQuery, sequelizeOptions) - - await sendVideoRateChange(accountInstance, videoInstance, likesToIncrement, dislikesToIncrement, t) - - logger.info('Account video rate for video %s of account %s updated.', videoInstance.name, accountInstance.name) - }) - - return res.type('json') - .status(HttpStatusCode.NO_CONTENT_204) - .end() -} diff --git a/server/controllers/api/videos/source.ts b/server/controllers/api/videos/source.ts deleted file mode 100644 index 75fe68b6c..000000000 --- a/server/controllers/api/videos/source.ts +++ /dev/null @@ -1,206 +0,0 @@ -import express from 'express' -import { move } from 'fs-extra' -import { sequelizeTypescript } from '@server/initializers/database' -import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue' -import { Hooks } from '@server/lib/plugins/hooks' -import { regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail' -import { uploadx } from '@server/lib/uploadx' -import { buildMoveToObjectStorageJob } from '@server/lib/video' -import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist' -import { buildNewFile } from '@server/lib/video-file' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { buildNextVideoState } from '@server/lib/video-state' -import { openapiOperationDoc } from '@server/middlewares/doc' -import { VideoModel } from '@server/models/video/video' -import { VideoSourceModel } from '@server/models/video/video-source' -import { MStreamingPlaylistFiles, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' -import { VideoState } from '@shared/models' -import { logger, loggerTagsFactory } from '../../../helpers/logger' -import { - asyncMiddleware, - authenticate, - replaceVideoSourceResumableInitValidator, - replaceVideoSourceResumableValidator, - videoSourceGetLatestValidator -} from '../../../middlewares' - -const lTags = loggerTagsFactory('api', 'video') - -const videoSourceRouter = express.Router() - -videoSourceRouter.get('/:id/source', - openapiOperationDoc({ operationId: 'getVideoSource' }), - authenticate, - asyncMiddleware(videoSourceGetLatestValidator), - getVideoLatestSource -) - -videoSourceRouter.post('/:id/source/replace-resumable', - authenticate, - asyncMiddleware(replaceVideoSourceResumableInitValidator), - (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end -) - -videoSourceRouter.delete('/:id/source/replace-resumable', - authenticate, - (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end -) - -videoSourceRouter.put('/:id/source/replace-resumable', - authenticate, - uploadx.upload, // uploadx doesn't next() before the file upload completes - asyncMiddleware(replaceVideoSourceResumableValidator), - asyncMiddleware(replaceVideoSourceResumable) -) - -// --------------------------------------------------------------------------- - -export { - videoSourceRouter -} - -// --------------------------------------------------------------------------- - -function getVideoLatestSource (req: express.Request, res: express.Response) { - return res.json(res.locals.videoSource.toFormattedJSON()) -} - -async function replaceVideoSourceResumable (req: express.Request, res: express.Response) { - const videoPhysicalFile = res.locals.updateVideoFileResumable - const user = res.locals.oauth.token.User - - const videoFile = await buildNewFile({ path: videoPhysicalFile.path, mode: 'web-video' }) - const originalFilename = videoPhysicalFile.originalname - - const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(res.locals.videoAll.uuid) - - try { - const destination = VideoPathManager.Instance.getFSVideoFileOutputPath(res.locals.videoAll, videoFile) - await move(videoPhysicalFile.path, destination) - - let oldWebVideoFiles: MVideoFile[] = [] - let oldStreamingPlaylists: MStreamingPlaylistFiles[] = [] - - const inputFileUpdatedAt = new Date() - - const video = await sequelizeTypescript.transaction(async transaction => { - const video = await VideoModel.loadFull(res.locals.videoAll.id, transaction) - - oldWebVideoFiles = video.VideoFiles - oldStreamingPlaylists = video.VideoStreamingPlaylists - - for (const file of video.VideoFiles) { - await file.destroy({ transaction }) - } - for (const playlist of oldStreamingPlaylists) { - await playlist.destroy({ transaction }) - } - - videoFile.videoId = video.id - await videoFile.save({ transaction }) - - video.VideoFiles = [ videoFile ] - video.VideoStreamingPlaylists = [] - - video.state = buildNextVideoState() - video.duration = videoPhysicalFile.duration - video.inputFileUpdatedAt = inputFileUpdatedAt - await video.save({ transaction }) - - await autoBlacklistVideoIfNeeded({ - video, - user, - isRemote: false, - isNew: false, - isNewFile: true, - transaction - }) - - return video - }) - - await removeOldFiles({ video, files: oldWebVideoFiles, playlists: oldStreamingPlaylists }) - - const source = await VideoSourceModel.create({ - filename: originalFilename, - videoId: video.id, - createdAt: inputFileUpdatedAt - }) - - await regenerateMiniaturesIfNeeded(video) - await video.VideoChannel.setAsUpdated() - await addVideoJobsAfterUpload(video, video.getMaxQualityFile()) - - logger.info('Replaced video file of video %s with uuid %s.', video.name, video.uuid, lTags(video.uuid)) - - Hooks.runAction('action:api.video.file-updated', { video, req, res }) - - return res.json(source.toFormattedJSON()) - } finally { - videoFileMutexReleaser() - } -} - -async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVideoFile) { - const jobs: (CreateJobArgument & CreateJobOptions)[] = [ - { - type: 'manage-video-torrent' as 'manage-video-torrent', - payload: { - videoId: video.id, - videoFileId: videoFile.id, - action: 'create' - } - }, - - { - type: 'generate-video-storyboard' as 'generate-video-storyboard', - payload: { - videoUUID: video.uuid, - // No need to federate, we process these jobs sequentially - federate: false - } - }, - - { - type: 'federate-video' as 'federate-video', - payload: { - videoUUID: video.uuid, - isNewVideo: false - } - } - ] - - if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { - jobs.push(await buildMoveToObjectStorageJob({ video, isNewVideo: false, previousVideoState: undefined })) - } - - if (video.state === VideoState.TO_TRANSCODE) { - jobs.push({ - type: 'transcoding-job-builder' as 'transcoding-job-builder', - payload: { - videoUUID: video.uuid, - optimizeJob: { - isNewVideo: false - } - } - }) - } - - return JobQueue.Instance.createSequentialJobFlow(...jobs) -} - -async function removeOldFiles (options: { - video: MVideo - files: MVideoFile[] - playlists: MStreamingPlaylistFiles[] -}) { - const { video, files, playlists } = options - - for (const file of files) { - await video.removeWebVideoFile(file) - } - - for (const playlist of playlists) { - await video.removeStreamingPlaylistFiles(playlist) - } -} diff --git a/server/controllers/api/videos/stats.ts b/server/controllers/api/videos/stats.ts deleted file mode 100644 index e79f01888..000000000 --- a/server/controllers/api/videos/stats.ts +++ /dev/null @@ -1,75 +0,0 @@ -import express from 'express' -import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' -import { VideoStatsOverallQuery, VideoStatsTimeserieMetric, VideoStatsTimeserieQuery } from '@shared/models' -import { - asyncMiddleware, - authenticate, - videoOverallStatsValidator, - videoRetentionStatsValidator, - videoTimeserieStatsValidator -} from '../../../middlewares' - -const statsRouter = express.Router() - -statsRouter.get('/:videoId/stats/overall', - authenticate, - asyncMiddleware(videoOverallStatsValidator), - asyncMiddleware(getOverallStats) -) - -statsRouter.get('/:videoId/stats/timeseries/:metric', - authenticate, - asyncMiddleware(videoTimeserieStatsValidator), - asyncMiddleware(getTimeserieStats) -) - -statsRouter.get('/:videoId/stats/retention', - authenticate, - asyncMiddleware(videoRetentionStatsValidator), - asyncMiddleware(getRetentionStats) -) - -// --------------------------------------------------------------------------- - -export { - statsRouter -} - -// --------------------------------------------------------------------------- - -async function getOverallStats (req: express.Request, res: express.Response) { - const video = res.locals.videoAll - const query = req.query as VideoStatsOverallQuery - - const stats = await LocalVideoViewerModel.getOverallStats({ - video, - startDate: query.startDate, - endDate: query.endDate - }) - - return res.json(stats) -} - -async function getRetentionStats (req: express.Request, res: express.Response) { - const video = res.locals.videoAll - - const stats = await LocalVideoViewerModel.getRetentionStats(video) - - return res.json(stats) -} - -async function getTimeserieStats (req: express.Request, res: express.Response) { - const video = res.locals.videoAll - const metric = req.params.metric as VideoStatsTimeserieMetric - - const query = req.query as VideoStatsTimeserieQuery - - const stats = await LocalVideoViewerModel.getTimeserieStats({ - video, - metric, - startDate: query.startDate ?? video.createdAt.toISOString(), - endDate: query.endDate ?? new Date().toISOString() - }) - - return res.json(stats) -} diff --git a/server/controllers/api/videos/storyboard.ts b/server/controllers/api/videos/storyboard.ts deleted file mode 100644 index 47a22011d..000000000 --- a/server/controllers/api/videos/storyboard.ts +++ /dev/null @@ -1,29 +0,0 @@ -import express from 'express' -import { getVideoWithAttributes } from '@server/helpers/video' -import { StoryboardModel } from '@server/models/video/storyboard' -import { asyncMiddleware, videosGetValidator } from '../../../middlewares' - -const storyboardRouter = express.Router() - -storyboardRouter.get('/:id/storyboards', - asyncMiddleware(videosGetValidator), - asyncMiddleware(listStoryboards) -) - -// --------------------------------------------------------------------------- - -export { - storyboardRouter -} - -// --------------------------------------------------------------------------- - -async function listStoryboards (req: express.Request, res: express.Response) { - const video = getVideoWithAttributes(res) - - const storyboards = await StoryboardModel.listStoryboardsOf(video) - - return res.json({ - storyboards: storyboards.map(s => s.toFormattedJSON()) - }) -} diff --git a/server/controllers/api/videos/studio.ts b/server/controllers/api/videos/studio.ts deleted file mode 100644 index 7c31dfd2b..000000000 --- a/server/controllers/api/videos/studio.ts +++ /dev/null @@ -1,143 +0,0 @@ -import Bluebird from 'bluebird' -import express from 'express' -import { move } from 'fs-extra' -import { basename } from 'path' -import { createAnyReqFiles } from '@server/helpers/express-utils' -import { MIMETYPES, VIDEO_FILTERS } from '@server/initializers/constants' -import { buildTaskFileFieldname, createVideoStudioJob, getStudioTaskFilePath, getTaskFileFromReq } from '@server/lib/video-studio' -import { - HttpStatusCode, - VideoState, - VideoStudioCreateEdition, - VideoStudioTask, - VideoStudioTaskCut, - VideoStudioTaskIntro, - VideoStudioTaskOutro, - VideoStudioTaskPayload, - VideoStudioTaskWatermark -} from '@shared/models' -import { asyncMiddleware, authenticate, videoStudioAddEditionValidator } from '../../../middlewares' - -const studioRouter = express.Router() - -const tasksFiles = createAnyReqFiles( - MIMETYPES.VIDEO.MIMETYPE_EXT, - (req: express.Request, file: Express.Multer.File, cb: (err: Error, result?: boolean) => void) => { - const body = req.body as VideoStudioCreateEdition - - // Fetch array element - const matches = file.fieldname.match(/tasks\[(\d+)\]/) - if (!matches) return cb(new Error('Cannot find array element indice for ' + file.fieldname)) - - const indice = parseInt(matches[1]) - const task = body.tasks[indice] - - if (!task) return cb(new Error('Cannot find array element of indice ' + indice + ' for ' + file.fieldname)) - - if ( - [ 'add-intro', 'add-outro', 'add-watermark' ].includes(task.name) && - file.fieldname === buildTaskFileFieldname(indice) - ) { - return cb(null, true) - } - - return cb(null, false) - } -) - -studioRouter.post('/:videoId/studio/edit', - authenticate, - tasksFiles, - asyncMiddleware(videoStudioAddEditionValidator), - asyncMiddleware(createEditionTasks) -) - -// --------------------------------------------------------------------------- - -export { - studioRouter -} - -// --------------------------------------------------------------------------- - -async function createEditionTasks (req: express.Request, res: express.Response) { - const files = req.files as Express.Multer.File[] - const body = req.body as VideoStudioCreateEdition - const video = res.locals.videoAll - - video.state = VideoState.TO_EDIT - await video.save() - - const payload = { - videoUUID: video.uuid, - tasks: await Bluebird.mapSeries(body.tasks, (t, i) => buildTaskPayload(t, i, files)) - } - - await createVideoStudioJob({ - user: res.locals.oauth.token.User, - payload, - video - }) - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} - -const taskPayloadBuilders: { - [id in VideoStudioTask['name']]: ( - task: VideoStudioTask, - indice?: number, - files?: Express.Multer.File[] - ) => Promise -} = { - 'add-intro': buildIntroOutroTask, - 'add-outro': buildIntroOutroTask, - 'cut': buildCutTask, - 'add-watermark': buildWatermarkTask -} - -function buildTaskPayload (task: VideoStudioTask, indice: number, files: Express.Multer.File[]): Promise { - return taskPayloadBuilders[task.name](task, indice, files) -} - -async function buildIntroOutroTask (task: VideoStudioTaskIntro | VideoStudioTaskOutro, indice: number, files: Express.Multer.File[]) { - const destination = await moveStudioFileToPersistentTMP(getTaskFileFromReq(files, indice).path) - - return { - name: task.name, - options: { - file: destination - } - } -} - -function buildCutTask (task: VideoStudioTaskCut) { - return Promise.resolve({ - name: task.name, - options: { - start: task.options.start, - end: task.options.end - } - }) -} - -async function buildWatermarkTask (task: VideoStudioTaskWatermark, indice: number, files: Express.Multer.File[]) { - const destination = await moveStudioFileToPersistentTMP(getTaskFileFromReq(files, indice).path) - - return { - name: task.name, - options: { - file: destination, - watermarkSizeRatio: VIDEO_FILTERS.WATERMARK.SIZE_RATIO, - horitonzalMarginRatio: VIDEO_FILTERS.WATERMARK.HORIZONTAL_MARGIN_RATIO, - verticalMarginRatio: VIDEO_FILTERS.WATERMARK.VERTICAL_MARGIN_RATIO - } - } -} - -async function moveStudioFileToPersistentTMP (file: string) { - const destination = getStudioTaskFilePath(basename(file)) - - await move(file, destination) - - return destination -} diff --git a/server/controllers/api/videos/token.ts b/server/controllers/api/videos/token.ts deleted file mode 100644 index e961ffd9e..000000000 --- a/server/controllers/api/videos/token.ts +++ /dev/null @@ -1,33 +0,0 @@ -import express from 'express' -import { VideoTokensManager } from '@server/lib/video-tokens-manager' -import { VideoPrivacy, VideoToken } from '@shared/models' -import { asyncMiddleware, optionalAuthenticate, videoFileTokenValidator, videosCustomGetValidator } from '../../../middlewares' - -const tokenRouter = express.Router() - -tokenRouter.post('/:id/token', - optionalAuthenticate, - asyncMiddleware(videosCustomGetValidator('only-video')), - videoFileTokenValidator, - generateToken -) - -// --------------------------------------------------------------------------- - -export { - tokenRouter -} - -// --------------------------------------------------------------------------- - -function generateToken (req: express.Request, res: express.Response) { - const video = res.locals.onlyVideo - - const files = video.privacy === VideoPrivacy.PASSWORD_PROTECTED - ? VideoTokensManager.Instance.createForPasswordProtectedVideo({ videoUUID: video.uuid }) - : VideoTokensManager.Instance.createForAuthUser({ videoUUID: video.uuid, user: res.locals.oauth.token.User }) - - return res.json({ - files - } as VideoToken) -} diff --git a/server/controllers/api/videos/transcoding.ts b/server/controllers/api/videos/transcoding.ts deleted file mode 100644 index c0b93742f..000000000 --- a/server/controllers/api/videos/transcoding.ts +++ /dev/null @@ -1,60 +0,0 @@ -import express from 'express' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { Hooks } from '@server/lib/plugins/hooks' -import { createTranscodingJobs } from '@server/lib/transcoding/create-transcoding-job' -import { computeResolutionsToTranscode } from '@server/lib/transcoding/transcoding-resolutions' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { HttpStatusCode, UserRight, VideoState, VideoTranscodingCreate } from '@shared/models' -import { asyncMiddleware, authenticate, createTranscodingValidator, ensureUserHasRight } from '../../../middlewares' - -const lTags = loggerTagsFactory('api', 'video') -const transcodingRouter = express.Router() - -transcodingRouter.post('/:videoId/transcoding', - authenticate, - ensureUserHasRight(UserRight.RUN_VIDEO_TRANSCODING), - asyncMiddleware(createTranscodingValidator), - asyncMiddleware(createTranscoding) -) - -// --------------------------------------------------------------------------- - -export { - transcodingRouter -} - -// --------------------------------------------------------------------------- - -async function createTranscoding (req: express.Request, res: express.Response) { - const video = res.locals.videoAll - logger.info('Creating %s transcoding job for %s.', req.body.transcodingType, video.url, lTags()) - - const body: VideoTranscodingCreate = req.body - - await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingTranscode') - - const { resolution: maxResolution, hasAudio } = await video.probeMaxQualityFile() - - const resolutions = await Hooks.wrapObject( - computeResolutionsToTranscode({ input: maxResolution, type: 'vod', includeInput: true, strictLower: false, hasAudio }), - 'filter:transcoding.manual.resolutions-to-transcode.result', - body - ) - - if (resolutions.length === 0) { - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) - } - - video.state = VideoState.TO_TRANSCODE - await video.save() - - await createTranscodingJobs({ - video, - resolutions, - transcodingType: body.transcodingType, - isNewVideo: false, - user: null // Don't specify priority since these transcoding jobs are fired by the admin - }) - - return res.sendStatus(HttpStatusCode.NO_CONTENT_204) -} diff --git a/server/controllers/api/videos/update.ts b/server/controllers/api/videos/update.ts deleted file mode 100644 index 1edc509dc..000000000 --- a/server/controllers/api/videos/update.ts +++ /dev/null @@ -1,210 +0,0 @@ -import express from 'express' -import { Transaction } from 'sequelize/types' -import { changeVideoChannelShare } from '@server/lib/activitypub/share' -import { addVideoJobsAfterUpdate, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video' -import { setVideoPrivacy } from '@server/lib/video-privacy' -import { openapiOperationDoc } from '@server/middlewares/doc' -import { FilteredModelAttributes } from '@server/types' -import { MVideoFullLight } from '@server/types/models' -import { forceNumber } from '@shared/core-utils' -import { HttpStatusCode, VideoPrivacy, VideoUpdate } from '@shared/models' -import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' -import { resetSequelizeInstance } from '../../../helpers/database-utils' -import { createReqFiles } from '../../../helpers/express-utils' -import { logger, loggerTagsFactory } from '../../../helpers/logger' -import { MIMETYPES } from '../../../initializers/constants' -import { sequelizeTypescript } from '../../../initializers/database' -import { Hooks } from '../../../lib/plugins/hooks' -import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist' -import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videosUpdateValidator } from '../../../middlewares' -import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' -import { VideoModel } from '../../../models/video/video' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { VideoPasswordModel } from '@server/models/video/video-password' -import { exists } from '@server/helpers/custom-validators/misc' - -const lTags = loggerTagsFactory('api', 'video') -const auditLogger = auditLoggerFactory('videos') -const updateRouter = express.Router() - -const reqVideoFileUpdate = createReqFiles([ 'thumbnailfile', 'previewfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT) - -updateRouter.put('/:id', - openapiOperationDoc({ operationId: 'putVideo' }), - authenticate, - reqVideoFileUpdate, - asyncMiddleware(videosUpdateValidator), - asyncRetryTransactionMiddleware(updateVideo) -) - -// --------------------------------------------------------------------------- - -export { - updateRouter -} - -// --------------------------------------------------------------------------- - -async function updateVideo (req: express.Request, res: express.Response) { - const videoFromReq = res.locals.videoAll - const oldVideoAuditView = new VideoAuditView(videoFromReq.toFormattedDetailsJSON()) - const videoInfoToUpdate: VideoUpdate = req.body - - const hadPrivacyForFederation = videoFromReq.hasPrivacyForFederation() - const oldPrivacy = videoFromReq.privacy - - const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ - video: videoFromReq, - files: req.files, - fallback: () => Promise.resolve(undefined), - automaticallyGenerated: false - }) - - const videoFileLockReleaser = await VideoPathManager.Instance.lockFiles(videoFromReq.uuid) - - try { - const { videoInstanceUpdated, isNewVideo } = await sequelizeTypescript.transaction(async t => { - // Refresh video since thumbnails to prevent concurrent updates - const video = await VideoModel.loadFull(videoFromReq.id, t) - - const oldVideoChannel = video.VideoChannel - - const keysToUpdate: (keyof VideoUpdate & FilteredModelAttributes)[] = [ - 'name', - 'category', - 'licence', - 'language', - 'nsfw', - 'waitTranscoding', - 'support', - 'description', - 'commentsEnabled', - 'downloadEnabled' - ] - - for (const key of keysToUpdate) { - if (videoInfoToUpdate[key] !== undefined) video.set(key, videoInfoToUpdate[key]) - } - - if (videoInfoToUpdate.originallyPublishedAt !== undefined && videoInfoToUpdate.originallyPublishedAt !== null) { - video.originallyPublishedAt = new Date(videoInfoToUpdate.originallyPublishedAt) - } - - // Privacy update? - let isNewVideo = false - if (videoInfoToUpdate.privacy !== undefined) { - isNewVideo = await updateVideoPrivacy({ videoInstance: video, videoInfoToUpdate, hadPrivacyForFederation, transaction: t }) - } - - // Force updatedAt attribute change - if (!video.changed()) { - await video.setAsRefreshed(t) - } - - const videoInstanceUpdated = await video.save({ transaction: t }) as MVideoFullLight - - // Thumbnail & preview updates? - if (thumbnailModel) await videoInstanceUpdated.addAndSaveThumbnail(thumbnailModel, t) - if (previewModel) await videoInstanceUpdated.addAndSaveThumbnail(previewModel, t) - - // Video tags update? - if (videoInfoToUpdate.tags !== undefined) { - await setVideoTags({ video: videoInstanceUpdated, tags: videoInfoToUpdate.tags, transaction: t }) - } - - // Video channel update? - if (res.locals.videoChannel && videoInstanceUpdated.channelId !== res.locals.videoChannel.id) { - await videoInstanceUpdated.$set('VideoChannel', res.locals.videoChannel, { transaction: t }) - videoInstanceUpdated.VideoChannel = res.locals.videoChannel - - if (hadPrivacyForFederation === true) { - await changeVideoChannelShare(videoInstanceUpdated, oldVideoChannel, t) - } - } - - // Schedule an update in the future? - await updateSchedule(videoInstanceUpdated, videoInfoToUpdate, t) - - await autoBlacklistVideoIfNeeded({ - video: videoInstanceUpdated, - user: res.locals.oauth.token.User, - isRemote: false, - isNew: false, - isNewFile: false, - transaction: t - }) - - auditLogger.update( - getAuditIdFromRes(res), - new VideoAuditView(videoInstanceUpdated.toFormattedDetailsJSON()), - oldVideoAuditView - ) - logger.info('Video with name %s and uuid %s updated.', video.name, video.uuid, lTags(video.uuid)) - - return { videoInstanceUpdated, isNewVideo } - }) - - Hooks.runAction('action:api.video.updated', { video: videoInstanceUpdated, body: req.body, req, res }) - - await addVideoJobsAfterUpdate({ - video: videoInstanceUpdated, - nameChanged: !!videoInfoToUpdate.name, - oldPrivacy, - isNewVideo - }) - } catch (err) { - // If the transaction is retried, sequelize will think the object has not changed - // So we need to restore the previous fields - await resetSequelizeInstance(videoFromReq) - - throw err - } finally { - videoFileLockReleaser() - } - - return res.type('json') - .status(HttpStatusCode.NO_CONTENT_204) - .end() -} - -async function updateVideoPrivacy (options: { - videoInstance: MVideoFullLight - videoInfoToUpdate: VideoUpdate - hadPrivacyForFederation: boolean - transaction: Transaction -}) { - const { videoInstance, videoInfoToUpdate, hadPrivacyForFederation, transaction } = options - const isNewVideo = videoInstance.isNewVideo(videoInfoToUpdate.privacy) - - const newPrivacy = forceNumber(videoInfoToUpdate.privacy) - setVideoPrivacy(videoInstance, newPrivacy) - - // Delete passwords if video is not anymore password protected - if (videoInstance.privacy === VideoPrivacy.PASSWORD_PROTECTED && newPrivacy !== VideoPrivacy.PASSWORD_PROTECTED) { - await VideoPasswordModel.deleteAllPasswords(videoInstance.id, transaction) - } - - if (newPrivacy === VideoPrivacy.PASSWORD_PROTECTED && exists(videoInfoToUpdate.videoPasswords)) { - await VideoPasswordModel.deleteAllPasswords(videoInstance.id, transaction) - await VideoPasswordModel.addPasswords(videoInfoToUpdate.videoPasswords, videoInstance.id, transaction) - } - - // Unfederate the video if the new privacy is not compatible with federation - if (hadPrivacyForFederation && !videoInstance.hasPrivacyForFederation()) { - await VideoModel.sendDelete(videoInstance, { transaction }) - } - - return isNewVideo -} - -function updateSchedule (videoInstance: MVideoFullLight, videoInfoToUpdate: VideoUpdate, transaction: Transaction) { - if (videoInfoToUpdate.scheduleUpdate) { - return ScheduleVideoUpdateModel.upsert({ - videoId: videoInstance.id, - updateAt: new Date(videoInfoToUpdate.scheduleUpdate.updateAt), - privacy: videoInfoToUpdate.scheduleUpdate.privacy || null - }, { transaction }) - } else if (videoInfoToUpdate.scheduleUpdate === null) { - return ScheduleVideoUpdateModel.deleteByVideoId(videoInstance.id, transaction) - } -} diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts deleted file mode 100644 index e520bf4b5..000000000 --- a/server/controllers/api/videos/upload.ts +++ /dev/null @@ -1,287 +0,0 @@ -import express from 'express' -import { move } from 'fs-extra' -import { basename } from 'path' -import { getResumableUploadPath } from '@server/helpers/upload' -import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' -import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue' -import { Redis } from '@server/lib/redis' -import { uploadx } from '@server/lib/uploadx' -import { buildLocalVideoFromReq, buildMoveToObjectStorageJob, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video' -import { buildNewFile } from '@server/lib/video-file' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { buildNextVideoState } from '@server/lib/video-state' -import { openapiOperationDoc } from '@server/middlewares/doc' -import { VideoPasswordModel } from '@server/models/video/video-password' -import { VideoSourceModel } from '@server/models/video/video-source' -import { MVideoFile, MVideoFullLight } from '@server/types/models' -import { uuidToShort } from '@shared/extra-utils' -import { HttpStatusCode, VideoCreate, VideoPrivacy, VideoState } from '@shared/models' -import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' -import { createReqFiles } from '../../../helpers/express-utils' -import { logger, loggerTagsFactory } from '../../../helpers/logger' -import { MIMETYPES } from '../../../initializers/constants' -import { sequelizeTypescript } from '../../../initializers/database' -import { Hooks } from '../../../lib/plugins/hooks' -import { generateLocalVideoMiniature } from '../../../lib/thumbnail' -import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist' -import { - asyncMiddleware, - asyncRetryTransactionMiddleware, - authenticate, - videosAddLegacyValidator, - videosAddResumableInitValidator, - videosAddResumableValidator -} from '../../../middlewares' -import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' -import { VideoModel } from '../../../models/video/video' - -const lTags = loggerTagsFactory('api', 'video') -const auditLogger = auditLoggerFactory('videos') -const uploadRouter = express.Router() - -const reqVideoFileAdd = createReqFiles( - [ 'videofile', 'thumbnailfile', 'previewfile' ], - { ...MIMETYPES.VIDEO.MIMETYPE_EXT, ...MIMETYPES.IMAGE.MIMETYPE_EXT } -) - -const reqVideoFileAddResumable = createReqFiles( - [ 'thumbnailfile', 'previewfile' ], - MIMETYPES.IMAGE.MIMETYPE_EXT, - getResumableUploadPath() -) - -uploadRouter.post('/upload', - openapiOperationDoc({ operationId: 'uploadLegacy' }), - authenticate, - reqVideoFileAdd, - asyncMiddleware(videosAddLegacyValidator), - asyncRetryTransactionMiddleware(addVideoLegacy) -) - -uploadRouter.post('/upload-resumable', - openapiOperationDoc({ operationId: 'uploadResumableInit' }), - authenticate, - reqVideoFileAddResumable, - asyncMiddleware(videosAddResumableInitValidator), - (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end -) - -uploadRouter.delete('/upload-resumable', - authenticate, - asyncMiddleware(deleteUploadResumableCache), - (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end -) - -uploadRouter.put('/upload-resumable', - openapiOperationDoc({ operationId: 'uploadResumable' }), - authenticate, - uploadx.upload, // uploadx doesn't next() before the file upload completes - asyncMiddleware(videosAddResumableValidator), - asyncMiddleware(addVideoResumable) -) - -// --------------------------------------------------------------------------- - -export { - uploadRouter -} - -// --------------------------------------------------------------------------- - -async function addVideoLegacy (req: express.Request, res: express.Response) { - // Uploading the video could be long - // Set timeout to 10 minutes, as Express's default is 2 minutes - req.setTimeout(1000 * 60 * 10, () => { - logger.error('Video upload has timed out.') - return res.fail({ - status: HttpStatusCode.REQUEST_TIMEOUT_408, - message: 'Video upload has timed out.' - }) - }) - - const videoPhysicalFile = req.files['videofile'][0] - const videoInfo: VideoCreate = req.body - const files = req.files - - const response = await addVideo({ req, res, videoPhysicalFile, videoInfo, files }) - - return res.json(response) -} - -async function addVideoResumable (req: express.Request, res: express.Response) { - const videoPhysicalFile = res.locals.uploadVideoFileResumable - const videoInfo = videoPhysicalFile.metadata - const files = { previewfile: videoInfo.previewfile, thumbnailfile: videoInfo.thumbnailfile } - - const response = await addVideo({ req, res, videoPhysicalFile, videoInfo, files }) - await Redis.Instance.setUploadSession(req.query.upload_id, response) - - return res.json(response) -} - -async function addVideo (options: { - req: express.Request - res: express.Response - videoPhysicalFile: express.VideoUploadFile - videoInfo: VideoCreate - files: express.UploadFiles -}) { - const { req, res, videoPhysicalFile, videoInfo, files } = options - const videoChannel = res.locals.videoChannel - const user = res.locals.oauth.token.User - - let videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id) - videoData = await Hooks.wrapObject(videoData, 'filter:api.video.upload.video-attribute.result') - - videoData.state = buildNextVideoState() - videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware - - const video = new VideoModel(videoData) as MVideoFullLight - video.VideoChannel = videoChannel - video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object - - const videoFile = await buildNewFile({ path: videoPhysicalFile.path, mode: 'web-video' }) - const originalFilename = videoPhysicalFile.originalname - - // Move physical file - const destination = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile) - await move(videoPhysicalFile.path, destination) - // This is important in case if there is another attempt in the retry process - videoPhysicalFile.filename = basename(destination) - videoPhysicalFile.path = destination - - const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ - video, - files, - fallback: type => generateLocalVideoMiniature({ video, videoFile, type }) - }) - - const { videoCreated } = await sequelizeTypescript.transaction(async t => { - const sequelizeOptions = { transaction: t } - - const videoCreated = await video.save(sequelizeOptions) as MVideoFullLight - - await videoCreated.addAndSaveThumbnail(thumbnailModel, t) - await videoCreated.addAndSaveThumbnail(previewModel, t) - - // Do not forget to add video channel information to the created video - videoCreated.VideoChannel = res.locals.videoChannel - - videoFile.videoId = video.id - await videoFile.save(sequelizeOptions) - - video.VideoFiles = [ videoFile ] - - await VideoSourceModel.create({ - filename: originalFilename, - videoId: video.id - }, { transaction: t }) - - await setVideoTags({ video, tags: videoInfo.tags, transaction: t }) - - // Schedule an update in the future? - if (videoInfo.scheduleUpdate) { - await ScheduleVideoUpdateModel.create({ - videoId: video.id, - updateAt: new Date(videoInfo.scheduleUpdate.updateAt), - privacy: videoInfo.scheduleUpdate.privacy || null - }, sequelizeOptions) - } - - await autoBlacklistVideoIfNeeded({ - video, - user, - isRemote: false, - isNew: true, - isNewFile: true, - transaction: t - }) - - if (videoInfo.privacy === VideoPrivacy.PASSWORD_PROTECTED) { - await VideoPasswordModel.addPasswords(videoInfo.videoPasswords, video.id, t) - } - - auditLogger.create(getAuditIdFromRes(res), new VideoAuditView(videoCreated.toFormattedDetailsJSON())) - logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoCreated.uuid, lTags(videoCreated.uuid)) - - return { videoCreated } - }) - - // Channel has a new content, set as updated - await videoCreated.VideoChannel.setAsUpdated() - - addVideoJobsAfterUpload(videoCreated, videoFile) - .catch(err => logger.error('Cannot build new video jobs of %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) })) - - Hooks.runAction('action:api.video.uploaded', { video: videoCreated, req, res }) - - return { - video: { - id: videoCreated.id, - shortUUID: uuidToShort(videoCreated.uuid), - uuid: videoCreated.uuid - } - } -} - -async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVideoFile) { - const jobs: (CreateJobArgument & CreateJobOptions)[] = [ - { - type: 'manage-video-torrent' as 'manage-video-torrent', - payload: { - videoId: video.id, - videoFileId: videoFile.id, - action: 'create' - } - }, - - { - type: 'generate-video-storyboard' as 'generate-video-storyboard', - payload: { - videoUUID: video.uuid, - // No need to federate, we process these jobs sequentially - federate: false - } - }, - - { - type: 'notify', - payload: { - action: 'new-video', - videoUUID: video.uuid - } - }, - - { - type: 'federate-video' as 'federate-video', - payload: { - videoUUID: video.uuid, - isNewVideo: true - } - } - ] - - if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { - jobs.push(await buildMoveToObjectStorageJob({ video, previousVideoState: undefined })) - } - - if (video.state === VideoState.TO_TRANSCODE) { - jobs.push({ - type: 'transcoding-job-builder' as 'transcoding-job-builder', - payload: { - videoUUID: video.uuid, - optimizeJob: { - isNewVideo: true - } - } - }) - } - - return JobQueue.Instance.createSequentialJobFlow(...jobs) -} - -async function deleteUploadResumableCache (req: express.Request, res: express.Response, next: express.NextFunction) { - await Redis.Instance.deleteUploadSession(req.query.upload_id) - - return next() -} diff --git a/server/controllers/api/videos/view.ts b/server/controllers/api/videos/view.ts deleted file mode 100644 index a747fa334..000000000 --- a/server/controllers/api/videos/view.ts +++ /dev/null @@ -1,60 +0,0 @@ -import express from 'express' -import { Hooks } from '@server/lib/plugins/hooks' -import { VideoViewsManager } from '@server/lib/views/video-views-manager' -import { MVideoId } from '@server/types/models' -import { HttpStatusCode, VideoView } from '@shared/models' -import { asyncMiddleware, methodsValidator, openapiOperationDoc, optionalAuthenticate, videoViewValidator } from '../../../middlewares' -import { UserVideoHistoryModel } from '../../../models/user/user-video-history' - -const viewRouter = express.Router() - -viewRouter.all( - [ '/:videoId/views', '/:videoId/watching' ], - openapiOperationDoc({ operationId: 'addView' }), - methodsValidator([ 'PUT', 'POST' ]), - optionalAuthenticate, - asyncMiddleware(videoViewValidator), - asyncMiddleware(viewVideo) -) - -// --------------------------------------------------------------------------- - -export { - viewRouter -} - -// --------------------------------------------------------------------------- - -async function viewVideo (req: express.Request, res: express.Response) { - const video = res.locals.onlyImmutableVideo - - const body = req.body as VideoView - - const ip = req.ip - const { successView } = await VideoViewsManager.Instance.processLocalView({ - video, - ip, - currentTime: body.currentTime, - viewEvent: body.viewEvent - }) - - if (successView) { - Hooks.runAction('action:api.video.viewed', { video, ip, req, res }) - } - - await updateUserHistoryIfNeeded(body, video, res) - - return res.status(HttpStatusCode.NO_CONTENT_204).end() -} - -async function updateUserHistoryIfNeeded (body: VideoView, video: MVideoId, res: express.Response) { - const user = res.locals.oauth?.token.User - if (!user) return - if (user.videosHistoryEnabled !== true) return - - await UserVideoHistoryModel.upsert({ - videoId: video.id, - userId: user.id, - currentTime: body.currentTime - }) -} diff --git a/server/controllers/client.ts b/server/controllers/client.ts deleted file mode 100644 index 2d0c49904..000000000 --- a/server/controllers/client.ts +++ /dev/null @@ -1,236 +0,0 @@ -import express from 'express' -import { constants, promises as fs } from 'fs' -import { readFile } from 'fs-extra' -import { join } from 'path' -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { Hooks } from '@server/lib/plugins/hooks' -import { root } from '@shared/core-utils' -import { buildFileLocale, getCompleteLocale, is18nLocale, LOCALE_FILES } from '@shared/core-utils/i18n' -import { HttpStatusCode } from '@shared/models' -import { STATIC_MAX_AGE } from '../initializers/constants' -import { ClientHtml, sendHTML, serveIndexHTML } from '../lib/client-html' -import { asyncMiddleware, buildRateLimiter, embedCSP } from '../middlewares' - -const clientsRouter = express.Router() - -const clientsRateLimiter = buildRateLimiter({ - windowMs: CONFIG.RATES_LIMIT.CLIENT.WINDOW_MS, - max: CONFIG.RATES_LIMIT.CLIENT.MAX -}) - -const distPath = join(root(), 'client', 'dist') -const testEmbedPath = join(distPath, 'standalone', 'videos', 'test-embed.html') - -// Special route that add OpenGraph and oEmbed tags -// Do not use a template engine for a so little thing -clientsRouter.use([ '/w/p/:id', '/videos/watch/playlist/:id' ], - clientsRateLimiter, - asyncMiddleware(generateWatchPlaylistHtmlPage) -) - -clientsRouter.use([ '/w/:id', '/videos/watch/:id' ], - clientsRateLimiter, - asyncMiddleware(generateWatchHtmlPage) -) - -clientsRouter.use([ '/accounts/:nameWithHost', '/a/:nameWithHost' ], - clientsRateLimiter, - asyncMiddleware(generateAccountHtmlPage) -) - -clientsRouter.use([ '/video-channels/:nameWithHost', '/c/:nameWithHost' ], - clientsRateLimiter, - asyncMiddleware(generateVideoChannelHtmlPage) -) - -clientsRouter.use('/@:nameWithHost', - clientsRateLimiter, - asyncMiddleware(generateActorHtmlPage) -) - -const embedMiddlewares = [ - clientsRateLimiter, - - CONFIG.CSP.ENABLED - ? embedCSP - : (req: express.Request, res: express.Response, next: express.NextFunction) => next(), - - // Set headers - (req: express.Request, res: express.Response, next: express.NextFunction) => { - res.removeHeader('X-Frame-Options') - - // Don't cache HTML file since it's an index to the immutable JS/CSS files - res.setHeader('Cache-Control', 'public, max-age=0') - - next() - }, - - asyncMiddleware(generateEmbedHtmlPage) -] - -clientsRouter.use('/videos/embed', ...embedMiddlewares) -clientsRouter.use('/video-playlists/embed', ...embedMiddlewares) - -const testEmbedController = (req: express.Request, res: express.Response) => res.sendFile(testEmbedPath) - -clientsRouter.use('/videos/test-embed', clientsRateLimiter, testEmbedController) -clientsRouter.use('/video-playlists/test-embed', clientsRateLimiter, testEmbedController) - -// Dynamic PWA manifest -clientsRouter.get('/manifest.webmanifest', clientsRateLimiter, asyncMiddleware(generateManifest)) - -// Static client overrides -// Must be consistent with static client overrides redirections in /support/nginx/peertube -const staticClientOverrides = [ - 'assets/images/logo.svg', - 'assets/images/favicon.png', - 'assets/images/icons/icon-36x36.png', - 'assets/images/icons/icon-48x48.png', - 'assets/images/icons/icon-72x72.png', - 'assets/images/icons/icon-96x96.png', - 'assets/images/icons/icon-144x144.png', - 'assets/images/icons/icon-192x192.png', - 'assets/images/icons/icon-512x512.png', - 'assets/images/default-playlist.jpg', - 'assets/images/default-avatar-account.png', - 'assets/images/default-avatar-account-48x48.png', - 'assets/images/default-avatar-video-channel.png', - 'assets/images/default-avatar-video-channel-48x48.png' -] - -for (const staticClientOverride of staticClientOverrides) { - const overridePhysicalPath = join(CONFIG.STORAGE.CLIENT_OVERRIDES_DIR, staticClientOverride) - clientsRouter.use(`/client/${staticClientOverride}`, asyncMiddleware(serveClientOverride(overridePhysicalPath))) -} - -clientsRouter.use('/client/locales/:locale/:file.json', serveServerTranslations) -clientsRouter.use('/client', express.static(distPath, { maxAge: STATIC_MAX_AGE.CLIENT })) - -// 404 for static files not found -clientsRouter.use('/client/*', (req: express.Request, res: express.Response) => { - res.status(HttpStatusCode.NOT_FOUND_404).end() -}) - -// Always serve index client page (the client is a single page application, let it handle routing) -// Try to provide the right language index.html -clientsRouter.use('/(:language)?', - clientsRateLimiter, - asyncMiddleware(serveIndexHTML) -) - -// --------------------------------------------------------------------------- - -export { - clientsRouter -} - -// --------------------------------------------------------------------------- - -function serveServerTranslations (req: express.Request, res: express.Response) { - const locale = req.params.locale - const file = req.params.file - - if (is18nLocale(locale) && LOCALE_FILES.includes(file)) { - const completeLocale = getCompleteLocale(locale) - const completeFileLocale = buildFileLocale(completeLocale) - - const path = join(__dirname, `../../../client/dist/locale/${file}.${completeFileLocale}.json`) - return res.sendFile(path, { maxAge: STATIC_MAX_AGE.SERVER }) - } - - return res.status(HttpStatusCode.NOT_FOUND_404).end() -} - -async function generateEmbedHtmlPage (req: express.Request, res: express.Response) { - const hookName = req.originalUrl.startsWith('/video-playlists/') - ? 'filter:html.embed.video-playlist.allowed.result' - : 'filter:html.embed.video.allowed.result' - - const allowParameters = { req } - - const allowedResult = await Hooks.wrapFun( - isEmbedAllowed, - allowParameters, - hookName - ) - - if (!allowedResult || allowedResult.allowed !== true) { - logger.info('Embed is not allowed.', { allowedResult }) - - return sendHTML(allowedResult?.html || '', res) - } - - const html = await ClientHtml.getEmbedHTML() - - return sendHTML(html, res) -} - -async function generateWatchHtmlPage (req: express.Request, res: express.Response) { - // Thread link is '/w/:videoId;threadId=:threadId' - // So to get the videoId we need to remove the last part - let videoId = req.params.id + '' - - const threadIdIndex = videoId.indexOf(';threadId') - if (threadIdIndex !== -1) videoId = videoId.substring(0, threadIdIndex) - - const html = await ClientHtml.getWatchHTMLPage(videoId, req, res) - - return sendHTML(html, res, true) -} - -async function generateWatchPlaylistHtmlPage (req: express.Request, res: express.Response) { - const html = await ClientHtml.getWatchPlaylistHTMLPage(req.params.id + '', req, res) - - return sendHTML(html, res, true) -} - -async function generateAccountHtmlPage (req: express.Request, res: express.Response) { - const html = await ClientHtml.getAccountHTMLPage(req.params.nameWithHost, req, res) - - return sendHTML(html, res, true) -} - -async function generateVideoChannelHtmlPage (req: express.Request, res: express.Response) { - const html = await ClientHtml.getVideoChannelHTMLPage(req.params.nameWithHost, req, res) - - return sendHTML(html, res, true) -} - -async function generateActorHtmlPage (req: express.Request, res: express.Response) { - const html = await ClientHtml.getActorHTMLPage(req.params.nameWithHost, req, res) - - return sendHTML(html, res, true) -} - -async function generateManifest (req: express.Request, res: express.Response) { - const manifestPhysicalPath = join(root(), 'client', 'dist', 'manifest.webmanifest') - const manifestJson = await readFile(manifestPhysicalPath, 'utf8') - const manifest = JSON.parse(manifestJson) - - manifest.name = CONFIG.INSTANCE.NAME - manifest.short_name = CONFIG.INSTANCE.NAME - manifest.description = CONFIG.INSTANCE.SHORT_DESCRIPTION - - res.json(manifest) -} - -function serveClientOverride (path: string) { - return async (req: express.Request, res: express.Response, next: express.NextFunction) => { - try { - await fs.access(path, constants.F_OK) - // Serve override client - res.sendFile(path, { maxAge: STATIC_MAX_AGE.SERVER }) - } catch { - // Serve dist client - next() - } - } -} - -type AllowedResult = { allowed: boolean, html?: string } -function isEmbedAllowed (_object: { - req: express.Request -}): AllowedResult { - return { allowed: true } -} diff --git a/server/controllers/download.ts b/server/controllers/download.ts deleted file mode 100644 index 4b94e34bd..000000000 --- a/server/controllers/download.ts +++ /dev/null @@ -1,213 +0,0 @@ -import cors from 'cors' -import express from 'express' -import { logger } from '@server/helpers/logger' -import { VideoTorrentsSimpleFileCache } from '@server/lib/files-cache' -import { generateHLSFilePresignedUrl, generateWebVideoPresignedUrl } from '@server/lib/object-storage' -import { Hooks } from '@server/lib/plugins/hooks' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' -import { forceNumber } from '@shared/core-utils' -import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@shared/models' -import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants' -import { asyncMiddleware, optionalAuthenticate, videosDownloadValidator } from '../middlewares' - -const downloadRouter = express.Router() - -downloadRouter.use(cors()) - -downloadRouter.use( - STATIC_DOWNLOAD_PATHS.TORRENTS + ':filename', - asyncMiddleware(downloadTorrent) -) - -downloadRouter.use( - STATIC_DOWNLOAD_PATHS.VIDEOS + ':id-:resolution([0-9]+).:extension', - optionalAuthenticate, - asyncMiddleware(videosDownloadValidator), - asyncMiddleware(downloadVideoFile) -) - -downloadRouter.use( - STATIC_DOWNLOAD_PATHS.HLS_VIDEOS + ':id-:resolution([0-9]+)-fragmented.:extension', - optionalAuthenticate, - asyncMiddleware(videosDownloadValidator), - asyncMiddleware(downloadHLSVideoFile) -) - -// --------------------------------------------------------------------------- - -export { - downloadRouter -} - -// --------------------------------------------------------------------------- - -async function downloadTorrent (req: express.Request, res: express.Response) { - const result = await VideoTorrentsSimpleFileCache.Instance.getFilePath(req.params.filename) - if (!result) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Torrent file not found' - }) - } - - const allowParameters = { - req, - res, - torrentPath: result.path, - downloadName: result.downloadName - } - - const allowedResult = await Hooks.wrapFun( - isTorrentDownloadAllowed, - allowParameters, - 'filter:api.download.torrent.allowed.result' - ) - - if (!checkAllowResult(res, allowParameters, allowedResult)) return - - return res.download(result.path, result.downloadName) -} - -async function downloadVideoFile (req: express.Request, res: express.Response) { - const video = res.locals.videoAll - - const videoFile = getVideoFile(req, video.VideoFiles) - if (!videoFile) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video file not found' - }) - } - - const allowParameters = { - req, - res, - video, - videoFile - } - - const allowedResult = await Hooks.wrapFun( - isVideoDownloadAllowed, - allowParameters, - 'filter:api.download.video.allowed.result' - ) - - if (!checkAllowResult(res, allowParameters, allowedResult)) return - - // Express uses basename on filename parameter - const videoName = video.name.replace(/[/\\]/g, '_') - const downloadFilename = `${videoName}-${videoFile.resolution}p${videoFile.extname}` - - if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { - return redirectToObjectStorage({ req, res, video, file: videoFile, downloadFilename }) - } - - await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), path => { - return res.download(path, downloadFilename) - }) -} - -async function downloadHLSVideoFile (req: express.Request, res: express.Response) { - const video = res.locals.videoAll - const streamingPlaylist = getHLSPlaylist(video) - if (!streamingPlaylist) return res.status(HttpStatusCode.NOT_FOUND_404).end - - const videoFile = getVideoFile(req, streamingPlaylist.VideoFiles) - if (!videoFile) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video file not found' - }) - } - - const allowParameters = { - req, - res, - video, - streamingPlaylist, - videoFile - } - - const allowedResult = await Hooks.wrapFun( - isVideoDownloadAllowed, - allowParameters, - 'filter:api.download.video.allowed.result' - ) - - if (!checkAllowResult(res, allowParameters, allowedResult)) return - - const downloadFilename = `${video.name}-${videoFile.resolution}p-${streamingPlaylist.getStringType()}${videoFile.extname}` - - if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { - return redirectToObjectStorage({ req, res, video, streamingPlaylist, file: videoFile, downloadFilename }) - } - - await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(streamingPlaylist), path => { - return res.download(path, downloadFilename) - }) -} - -function getVideoFile (req: express.Request, files: MVideoFile[]) { - const resolution = forceNumber(req.params.resolution) - return files.find(f => f.resolution === resolution) -} - -function getHLSPlaylist (video: MVideoFullLight) { - const playlist = video.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) - if (!playlist) return undefined - - return Object.assign(playlist, { Video: video }) -} - -type AllowedResult = { - allowed: boolean - errorMessage?: string -} - -function isTorrentDownloadAllowed (_object: { - torrentPath: string -}): AllowedResult { - return { allowed: true } -} - -function isVideoDownloadAllowed (_object: { - video: MVideo - videoFile: MVideoFile - streamingPlaylist?: MStreamingPlaylist -}): AllowedResult { - return { allowed: true } -} - -function checkAllowResult (res: express.Response, allowParameters: any, result?: AllowedResult) { - if (!result || result.allowed !== true) { - logger.info('Download is not allowed.', { result, allowParameters }) - - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: result?.errorMessage || 'Refused download' - }) - return false - } - - return true -} - -async function redirectToObjectStorage (options: { - req: express.Request - res: express.Response - video: MVideo - file: MVideoFile - streamingPlaylist?: MStreamingPlaylistVideo - downloadFilename: string -}) { - const { res, video, streamingPlaylist, file, downloadFilename } = options - - const url = streamingPlaylist - ? await generateHLSFilePresignedUrl({ streamingPlaylist, file, downloadFilename }) - : await generateWebVideoPresignedUrl({ file, downloadFilename }) - - logger.debug('Generating pre-signed URL %s for video %s', url, video.uuid) - - return res.redirect(url) -} diff --git a/server/controllers/feeds/comment-feeds.ts b/server/controllers/feeds/comment-feeds.ts deleted file mode 100644 index c013662ea..000000000 --- a/server/controllers/feeds/comment-feeds.ts +++ /dev/null @@ -1,96 +0,0 @@ -import express from 'express' -import { toSafeHtml } from '@server/helpers/markdown' -import { cacheRouteFactory } from '@server/middlewares' -import { CONFIG } from '../../initializers/config' -import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../../initializers/constants' -import { - asyncMiddleware, - feedsFormatValidator, - setFeedFormatContentType, - videoCommentsFeedsValidator, - feedsAccountOrChannelFiltersValidator -} from '../../middlewares' -import { VideoCommentModel } from '../../models/video/video-comment' -import { buildFeedMetadata, initFeed, sendFeed } from './shared' - -const commentFeedsRouter = express.Router() - -// --------------------------------------------------------------------------- - -const { middleware: cacheRouteMiddleware } = cacheRouteFactory({ - headerBlacklist: [ 'Content-Type' ] -}) - -// --------------------------------------------------------------------------- - -commentFeedsRouter.get('/video-comments.:format', - feedsFormatValidator, - setFeedFormatContentType, - cacheRouteMiddleware(ROUTE_CACHE_LIFETIME.FEEDS), - asyncMiddleware(feedsAccountOrChannelFiltersValidator), - asyncMiddleware(videoCommentsFeedsValidator), - asyncMiddleware(generateVideoCommentsFeed) -) - -// --------------------------------------------------------------------------- - -export { - commentFeedsRouter -} - -// --------------------------------------------------------------------------- - -async function generateVideoCommentsFeed (req: express.Request, res: express.Response) { - const start = 0 - const video = res.locals.videoAll - const account = res.locals.account - const videoChannel = res.locals.videoChannel - - const comments = await VideoCommentModel.listForFeed({ - start, - count: CONFIG.FEEDS.COMMENTS.COUNT, - videoId: video ? video.id : undefined, - accountId: account ? account.id : undefined, - videoChannelId: videoChannel ? videoChannel.id : undefined - }) - - const { name, description, imageUrl, link } = await buildFeedMetadata({ video, account, videoChannel }) - - const feed = initFeed({ - name, - description, - imageUrl, - isPodcast: false, - link, - resourceType: 'video-comments', - queryString: new URL(WEBSERVER.URL + req.originalUrl).search - }) - - // Adding video items to the feed, one at a time - for (const comment of comments) { - const localLink = WEBSERVER.URL + comment.getCommentStaticPath() - - let title = comment.Video.name - const author: { name: string, link: string }[] = [] - - if (comment.Account) { - title += ` - ${comment.Account.getDisplayName()}` - author.push({ - name: comment.Account.getDisplayName(), - link: comment.Account.Actor.url - }) - } - - feed.addItem({ - title, - id: localLink, - link: localLink, - content: toSafeHtml(comment.text), - author, - date: comment.createdAt - }) - } - - // Now the feed generation is done, let's send it! - return sendFeed(feed, req, res) -} diff --git a/server/controllers/feeds/index.ts b/server/controllers/feeds/index.ts deleted file mode 100644 index 19352318d..000000000 --- a/server/controllers/feeds/index.ts +++ /dev/null @@ -1,25 +0,0 @@ -import express from 'express' -import { CONFIG } from '@server/initializers/config' -import { buildRateLimiter } from '@server/middlewares' -import { commentFeedsRouter } from './comment-feeds' -import { videoFeedsRouter } from './video-feeds' -import { videoPodcastFeedsRouter } from './video-podcast-feeds' - -const feedsRouter = express.Router() - -const feedsRateLimiter = buildRateLimiter({ - windowMs: CONFIG.RATES_LIMIT.FEEDS.WINDOW_MS, - max: CONFIG.RATES_LIMIT.FEEDS.MAX -}) - -feedsRouter.use('/feeds', feedsRateLimiter) - -feedsRouter.use('/feeds', commentFeedsRouter) -feedsRouter.use('/feeds', videoFeedsRouter) -feedsRouter.use('/feeds', videoPodcastFeedsRouter) - -// --------------------------------------------------------------------------- - -export { - feedsRouter -} diff --git a/server/controllers/feeds/shared/common-feed-utils.ts b/server/controllers/feeds/shared/common-feed-utils.ts deleted file mode 100644 index 9e2f8adbb..000000000 --- a/server/controllers/feeds/shared/common-feed-utils.ts +++ /dev/null @@ -1,149 +0,0 @@ -import express from 'express' -import { Feed } from '@peertube/feed' -import { CustomTag, CustomXMLNS, Person } from '@peertube/feed/lib/typings' -import { mdToOneLinePlainText } from '@server/helpers/markdown' -import { CONFIG } from '@server/initializers/config' -import { WEBSERVER } from '@server/initializers/constants' -import { getBiggestActorImage } from '@server/lib/actor-image' -import { UserModel } from '@server/models/user/user' -import { MAccountDefault, MChannelBannerAccountDefault, MUser, MVideoFullLight } from '@server/types/models' -import { pick } from '@shared/core-utils' -import { ActorImageType } from '@shared/models' - -export function initFeed (parameters: { - name: string - description: string - imageUrl: string - isPodcast: boolean - link?: string - locked?: { isLocked: boolean, email: string } - author?: { - name: string - link: string - imageUrl: string - } - person?: Person[] - resourceType?: 'videos' | 'video-comments' - queryString?: string - medium?: string - stunServers?: string[] - trackers?: string[] - customXMLNS?: CustomXMLNS[] - customTags?: CustomTag[] -}) { - const webserverUrl = WEBSERVER.URL - const { name, description, link, imageUrl, isPodcast, resourceType, queryString, medium } = parameters - - return new Feed({ - title: name, - description: mdToOneLinePlainText(description), - // updated: TODO: somehowGetLatestUpdate, // optional, default = today - id: link || webserverUrl, - link: link || webserverUrl, - image: imageUrl, - favicon: webserverUrl + '/client/assets/images/favicon.png', - copyright: `All rights reserved, unless otherwise specified in the terms specified at ${webserverUrl}/about` + - ` and potential licenses granted by each content's rightholder.`, - generator: `Toraifōsu`, // ^.~ - medium: medium || 'video', - feedLinks: { - json: `${webserverUrl}/feeds/${resourceType}.json${queryString}`, - atom: `${webserverUrl}/feeds/${resourceType}.atom${queryString}`, - rss: isPodcast - ? `${webserverUrl}/feeds/podcast/videos.xml${queryString}` - : `${webserverUrl}/feeds/${resourceType}.xml${queryString}` - }, - - ...pick(parameters, [ 'stunServers', 'trackers', 'customXMLNS', 'customTags', 'author', 'person', 'locked' ]) - }) -} - -export function sendFeed (feed: Feed, req: express.Request, res: express.Response) { - const format = req.params.format - - if (format === 'atom' || format === 'atom1') { - return res.send(feed.atom1()).end() - } - - if (format === 'json' || format === 'json1') { - return res.send(feed.json1()).end() - } - - if (format === 'rss' || format === 'rss2') { - return res.send(feed.rss2()).end() - } - - // We're in the ambiguous '.xml' case and we look at the format query parameter - if (req.query.format === 'atom' || req.query.format === 'atom1') { - return res.send(feed.atom1()).end() - } - - return res.send(feed.rss2()).end() -} - -export async function buildFeedMetadata (options: { - videoChannel?: MChannelBannerAccountDefault - account?: MAccountDefault - video?: MVideoFullLight -}) { - const { video, videoChannel, account } = options - - let imageUrl = WEBSERVER.URL + '/client/assets/images/icons/icon-96x96.png' - let accountImageUrl: string - let name: string - let userName: string - let description: string - let email: string - let link: string - let accountLink: string - let user: MUser - - if (videoChannel) { - name = videoChannel.getDisplayName() - description = videoChannel.description - link = videoChannel.getClientUrl() - accountLink = videoChannel.Account.getClientUrl() - - if (videoChannel.Actor.hasImage(ActorImageType.AVATAR)) { - const videoChannelAvatar = getBiggestActorImage(videoChannel.Actor.Avatars) - imageUrl = WEBSERVER.URL + videoChannelAvatar.getStaticPath() - } - - if (videoChannel.Account.Actor.hasImage(ActorImageType.AVATAR)) { - const accountAvatar = getBiggestActorImage(videoChannel.Account.Actor.Avatars) - accountImageUrl = WEBSERVER.URL + accountAvatar.getStaticPath() - } - - user = await UserModel.loadById(videoChannel.Account.userId) - userName = videoChannel.Account.getDisplayName() - } else if (account) { - name = account.getDisplayName() - description = account.description - link = account.getClientUrl() - accountLink = link - - if (account.Actor.hasImage(ActorImageType.AVATAR)) { - const accountAvatar = getBiggestActorImage(account.Actor.Avatars) - imageUrl = WEBSERVER.URL + accountAvatar?.getStaticPath() - accountImageUrl = imageUrl - } - - user = await UserModel.loadById(account.userId) - } else if (video) { - name = video.name - description = video.description - link = video.url - } else { - name = CONFIG.INSTANCE.NAME - description = CONFIG.INSTANCE.DESCRIPTION - link = WEBSERVER.URL - } - - // If the user is local, has a verified email address, and allows it to be publicly displayed - // Return it so the owner can prove ownership of their feed - if (user && !user.pluginAuth && user.emailVerified && user.emailPublic) { - email = user.email - } - - return { name, userName, description, imageUrl, accountImageUrl, email, link, accountLink } -} diff --git a/server/controllers/feeds/shared/index.ts b/server/controllers/feeds/shared/index.ts deleted file mode 100644 index 0136c8477..000000000 --- a/server/controllers/feeds/shared/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './video-feed-utils' -export * from './common-feed-utils' diff --git a/server/controllers/feeds/shared/video-feed-utils.ts b/server/controllers/feeds/shared/video-feed-utils.ts deleted file mode 100644 index b154e04fa..000000000 --- a/server/controllers/feeds/shared/video-feed-utils.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { mdToOneLinePlainText, toSafeHtml } from '@server/helpers/markdown' -import { CONFIG } from '@server/initializers/config' -import { WEBSERVER } from '@server/initializers/constants' -import { getServerActor } from '@server/models/application/application' -import { getCategoryLabel } from '@server/models/video/formatter' -import { DisplayOnlyForFollowerOptions } from '@server/models/video/sql/video' -import { VideoModel } from '@server/models/video/video' -import { MThumbnail, MUserDefault } from '@server/types/models' -import { VideoInclude } from '@shared/models' - -export async function getVideosForFeeds (options: { - sort: string - nsfw: boolean - isLocal: boolean - include: VideoInclude - - accountId?: number - videoChannelId?: number - displayOnlyForFollower?: DisplayOnlyForFollowerOptions - user?: MUserDefault -}) { - const server = await getServerActor() - - const { data } = await VideoModel.listForApi({ - start: 0, - count: CONFIG.FEEDS.VIDEOS.COUNT, - displayOnlyForFollower: { - actorId: server.id, - orLocalVideos: true - }, - hasFiles: true, - countVideos: false, - - ...options - }) - - return data -} - -export function getCommonVideoFeedAttributes (video: VideoModel) { - const localLink = WEBSERVER.URL + video.getWatchStaticPath() - - const thumbnailModels: MThumbnail[] = [] - if (video.hasPreview()) thumbnailModels.push(video.getPreview()) - thumbnailModels.push(video.getMiniature()) - - return { - title: video.name, - link: localLink, - description: mdToOneLinePlainText(video.getTruncatedDescription()), - content: toSafeHtml(video.description), - - date: video.publishedAt, - nsfw: video.nsfw, - - category: video.category - ? [ { name: getCategoryLabel(video.category) } ] - : undefined, - - thumbnails: thumbnailModels.map(t => ({ - url: WEBSERVER.URL + t.getLocalStaticPath(), - width: t.width, - height: t.height - })) - } -} diff --git a/server/controllers/feeds/video-feeds.ts b/server/controllers/feeds/video-feeds.ts deleted file mode 100644 index e5941be40..000000000 --- a/server/controllers/feeds/video-feeds.ts +++ /dev/null @@ -1,189 +0,0 @@ -import express from 'express' -import { extname } from 'path' -import { Feed } from '@peertube/feed' -import { cacheRouteFactory } from '@server/middlewares' -import { VideoModel } from '@server/models/video/video' -import { VideoInclude } from '@shared/models' -import { buildNSFWFilter } from '../../helpers/express-utils' -import { MIMETYPES, ROUTE_CACHE_LIFETIME, WEBSERVER } from '../../initializers/constants' -import { - asyncMiddleware, - commonVideosFiltersValidator, - feedsFormatValidator, - setDefaultVideosSort, - setFeedFormatContentType, - feedsAccountOrChannelFiltersValidator, - videosSortValidator, - videoSubscriptionFeedsValidator -} from '../../middlewares' -import { buildFeedMetadata, getCommonVideoFeedAttributes, getVideosForFeeds, initFeed, sendFeed } from './shared' - -const videoFeedsRouter = express.Router() - -const { middleware: cacheRouteMiddleware } = cacheRouteFactory({ - headerBlacklist: [ 'Content-Type' ] -}) - -// --------------------------------------------------------------------------- - -videoFeedsRouter.get('/videos.:format', - videosSortValidator, - setDefaultVideosSort, - feedsFormatValidator, - setFeedFormatContentType, - cacheRouteMiddleware(ROUTE_CACHE_LIFETIME.FEEDS), - commonVideosFiltersValidator, - asyncMiddleware(feedsAccountOrChannelFiltersValidator), - asyncMiddleware(generateVideoFeed) -) - -videoFeedsRouter.get('/subscriptions.:format', - videosSortValidator, - setDefaultVideosSort, - feedsFormatValidator, - setFeedFormatContentType, - cacheRouteMiddleware(ROUTE_CACHE_LIFETIME.FEEDS), - commonVideosFiltersValidator, - asyncMiddleware(videoSubscriptionFeedsValidator), - asyncMiddleware(generateVideoFeedForSubscriptions) -) - -// --------------------------------------------------------------------------- - -export { - videoFeedsRouter -} - -// --------------------------------------------------------------------------- - -async function generateVideoFeed (req: express.Request, res: express.Response) { - const account = res.locals.account - const videoChannel = res.locals.videoChannel - - const { name, description, imageUrl, accountImageUrl, link, accountLink } = await buildFeedMetadata({ videoChannel, account }) - - const feed = initFeed({ - name, - description, - link, - isPodcast: false, - imageUrl, - author: { name, link: accountLink, imageUrl: accountImageUrl }, - resourceType: 'videos', - queryString: new URL(WEBSERVER.URL + req.url).search - }) - - const data = await getVideosForFeeds({ - sort: req.query.sort, - nsfw: buildNSFWFilter(res, req.query.nsfw), - isLocal: req.query.isLocal, - include: req.query.include | VideoInclude.FILES, - accountId: account?.id, - videoChannelId: videoChannel?.id - }) - - addVideosToFeed(feed, data) - - // Now the feed generation is done, let's send it! - return sendFeed(feed, req, res) -} - -async function generateVideoFeedForSubscriptions (req: express.Request, res: express.Response) { - const account = res.locals.account - const { name, description, imageUrl, link } = await buildFeedMetadata({ account }) - - const feed = initFeed({ - name, - description, - link, - isPodcast: false, - imageUrl, - resourceType: 'videos', - queryString: new URL(WEBSERVER.URL + req.url).search - }) - - const data = await getVideosForFeeds({ - sort: req.query.sort, - nsfw: buildNSFWFilter(res, req.query.nsfw), - isLocal: req.query.isLocal, - include: req.query.include | VideoInclude.FILES, - displayOnlyForFollower: { - actorId: res.locals.user.Account.Actor.id, - orLocalVideos: false - }, - user: res.locals.user - }) - - addVideosToFeed(feed, data) - - // Now the feed generation is done, let's send it! - return sendFeed(feed, req, res) -} - -// --------------------------------------------------------------------------- - -function addVideosToFeed (feed: Feed, videos: VideoModel[]) { - /** - * Adding video items to the feed object, one at a time - */ - for (const video of videos) { - const formattedVideoFiles = video.getFormattedAllVideoFilesJSON(false) - - const torrents = formattedVideoFiles.map(videoFile => ({ - title: video.name, - url: videoFile.torrentUrl, - size_in_bytes: videoFile.size - })) - - const videoFiles = formattedVideoFiles.map(videoFile => { - return { - type: MIMETYPES.VIDEO.EXT_MIMETYPE[extname(videoFile.fileUrl)], - medium: 'video', - height: videoFile.resolution.id, - fileSize: videoFile.size, - url: videoFile.fileUrl, - framerate: videoFile.fps, - duration: video.duration, - lang: video.language - } - }) - - feed.addItem({ - ...getCommonVideoFeedAttributes(video), - - id: WEBSERVER.URL + video.getWatchStaticPath(), - author: [ - { - name: video.VideoChannel.getDisplayName(), - link: video.VideoChannel.getClientUrl() - } - ], - torrents, - - // Enclosure - video: videoFiles.length !== 0 - ? { - url: videoFiles[0].url, - length: videoFiles[0].fileSize, - type: videoFiles[0].type - } - : undefined, - - // Media RSS - videos: videoFiles, - - embed: { - url: WEBSERVER.URL + video.getEmbedStaticPath(), - allowFullscreen: true - }, - player: { - url: WEBSERVER.URL + video.getWatchStaticPath() - }, - community: { - statistics: { - views: video.views - } - } - }) - } -} diff --git a/server/controllers/feeds/video-podcast-feeds.ts b/server/controllers/feeds/video-podcast-feeds.ts deleted file mode 100644 index fca82ba68..000000000 --- a/server/controllers/feeds/video-podcast-feeds.ts +++ /dev/null @@ -1,313 +0,0 @@ -import express from 'express' -import { extname } from 'path' -import { Feed } from '@peertube/feed' -import { CustomTag, CustomXMLNS, LiveItemStatus } from '@peertube/feed/lib/typings' -import { getBiggestActorImage } from '@server/lib/actor-image' -import { InternalEventEmitter } from '@server/lib/internal-event-emitter' -import { Hooks } from '@server/lib/plugins/hooks' -import { buildPodcastGroupsCache, cacheRouteFactory, videoFeedsPodcastSetCacheKey } from '@server/middlewares' -import { MVideo, MVideoCaptionVideo, MVideoFullLight } from '@server/types/models' -import { sortObjectComparator } from '@shared/core-utils' -import { ActorImageType, VideoFile, VideoInclude, VideoResolution, VideoState } from '@shared/models' -import { buildNSFWFilter } from '../../helpers/express-utils' -import { MIMETYPES, ROUTE_CACHE_LIFETIME, WEBSERVER } from '../../initializers/constants' -import { asyncMiddleware, setFeedPodcastContentType, videoFeedsPodcastValidator } from '../../middlewares' -import { VideoModel } from '../../models/video/video' -import { VideoCaptionModel } from '../../models/video/video-caption' -import { buildFeedMetadata, getCommonVideoFeedAttributes, getVideosForFeeds, initFeed } from './shared' - -const videoPodcastFeedsRouter = express.Router() - -// --------------------------------------------------------------------------- - -const { middleware: podcastCacheRouteMiddleware, instance: podcastApiCache } = cacheRouteFactory({ - headerBlacklist: [ 'Content-Type' ] -}) - -for (const event of ([ 'video-created', 'video-updated', 'video-deleted' ] as const)) { - InternalEventEmitter.Instance.on(event, ({ video }) => { - if (video.remote) return - - podcastApiCache.clearGroupSafe(buildPodcastGroupsCache({ channelId: video.channelId })) - }) -} - -for (const event of ([ 'channel-updated', 'channel-deleted' ] as const)) { - InternalEventEmitter.Instance.on(event, ({ channel }) => { - podcastApiCache.clearGroupSafe(buildPodcastGroupsCache({ channelId: channel.id })) - }) -} - -// --------------------------------------------------------------------------- - -videoPodcastFeedsRouter.get('/podcast/videos.xml', - setFeedPodcastContentType, - videoFeedsPodcastSetCacheKey, - podcastCacheRouteMiddleware(ROUTE_CACHE_LIFETIME.FEEDS), - asyncMiddleware(videoFeedsPodcastValidator), - asyncMiddleware(generateVideoPodcastFeed) -) - -// --------------------------------------------------------------------------- - -export { - videoPodcastFeedsRouter -} - -// --------------------------------------------------------------------------- - -async function generateVideoPodcastFeed (req: express.Request, res: express.Response) { - const videoChannel = res.locals.videoChannel - - const { name, userName, description, imageUrl, accountImageUrl, email, link, accountLink } = await buildFeedMetadata({ videoChannel }) - - const data = await getVideosForFeeds({ - sort: '-publishedAt', - nsfw: buildNSFWFilter(), - // Prevent podcast feeds from listing videos in other instances - // helps prevent duplicates when they are indexed -- only the author should control them - isLocal: true, - include: VideoInclude.FILES, - videoChannelId: videoChannel?.id - }) - - const customTags: CustomTag[] = await Hooks.wrapObject( - [], - 'filter:feed.podcast.channel.create-custom-tags.result', - { videoChannel } - ) - - const customXMLNS: CustomXMLNS[] = await Hooks.wrapObject( - [], - 'filter:feed.podcast.rss.create-custom-xmlns.result' - ) - - const feed = initFeed({ - name, - description, - link, - isPodcast: true, - imageUrl, - - locked: email - ? { isLocked: true, email } // Default to true because we have no way of offering a redirect yet - : undefined, - - person: [ { name: userName, href: accountLink, img: accountImageUrl } ], - resourceType: 'videos', - queryString: new URL(WEBSERVER.URL + req.url).search, - medium: 'video', - customXMLNS, - customTags - }) - - await addVideosToPodcastFeed(feed, data) - - // Now the feed generation is done, let's send it! - return res.send(feed.podcast()).end() -} - -type PodcastMedia = - { - type: string - length: number - bitrate: number - sources: { uri: string, contentType?: string }[] - title: string - language?: string - } | - { - sources: { uri: string }[] - type: string - title: string - } - -async function generatePodcastItem (options: { - video: VideoModel - liveItem: boolean - media: PodcastMedia[] -}) { - const { video, liveItem, media } = options - - const customTags: CustomTag[] = await Hooks.wrapObject( - [], - 'filter:feed.podcast.video.create-custom-tags.result', - { video, liveItem } - ) - - const account = video.VideoChannel.Account - - const author = { - name: account.getDisplayName(), - href: account.getClientUrl() - } - - const commonAttributes = getCommonVideoFeedAttributes(video) - const guid = liveItem - ? `${video.uuid}_${video.publishedAt.toISOString()}` - : commonAttributes.link - - let personImage: string - - if (account.Actor.hasImage(ActorImageType.AVATAR)) { - const avatar = getBiggestActorImage(account.Actor.Avatars) - personImage = WEBSERVER.URL + avatar.getStaticPath() - } - - return { - guid, - ...commonAttributes, - - trackers: video.getTrackerUrls(), - - author: [ author ], - person: [ - { - ...author, - - img: personImage - } - ], - - media, - - socialInteract: [ - { - uri: video.url, - protocol: 'activitypub', - accountUrl: account.getClientUrl() - } - ], - - customTags - } -} - -async function addVideosToPodcastFeed (feed: Feed, videos: VideoModel[]) { - const captionsGroup = await VideoCaptionModel.listCaptionsOfMultipleVideos(videos.map(v => v.id)) - - for (const video of videos) { - if (!video.isLive) { - await addVODPodcastItem({ feed, video, captionsGroup }) - } else if (video.isLive && video.state !== VideoState.LIVE_ENDED) { - await addLivePodcastItem({ feed, video }) - } - } -} - -async function addVODPodcastItem (options: { - feed: Feed - video: VideoModel - captionsGroup: { [ id: number ]: MVideoCaptionVideo[] } -}) { - const { feed, video, captionsGroup } = options - - const webVideos = video.getFormattedWebVideoFilesJSON(true) - .map(f => buildVODWebVideoFile(video, f)) - .sort(sortObjectComparator('bitrate', 'desc')) - - const streamingPlaylistFiles = buildVODStreamingPlaylists(video) - - // Order matters here, the first media URI will be the "default" - // So web videos are default if enabled - const media = [ ...webVideos, ...streamingPlaylistFiles ] - - const videoCaptions = buildVODCaptions(video, captionsGroup[video.id]) - const item = await generatePodcastItem({ video, liveItem: false, media }) - - feed.addPodcastItem({ ...item, subTitle: videoCaptions }) -} - -async function addLivePodcastItem (options: { - feed: Feed - video: VideoModel -}) { - const { feed, video } = options - - let status: LiveItemStatus - - switch (video.state) { - case VideoState.WAITING_FOR_LIVE: - status = LiveItemStatus.pending - break - case VideoState.PUBLISHED: - status = LiveItemStatus.live - break - } - - const item = await generatePodcastItem({ video, liveItem: true, media: buildLiveStreamingPlaylists(video) }) - - feed.addPodcastLiveItem({ ...item, status, start: video.updatedAt.toISOString() }) -} - -// --------------------------------------------------------------------------- - -function buildVODWebVideoFile (video: MVideo, videoFile: VideoFile) { - const isAudio = videoFile.resolution.id === VideoResolution.H_NOVIDEO - const type = isAudio - ? MIMETYPES.AUDIO.EXT_MIMETYPE[extname(videoFile.fileUrl)] - : MIMETYPES.VIDEO.EXT_MIMETYPE[extname(videoFile.fileUrl)] - - const sources = [ - { uri: videoFile.fileUrl }, - { uri: videoFile.torrentUrl, contentType: 'application/x-bittorrent' } - ] - - if (videoFile.magnetUri) { - sources.push({ uri: videoFile.magnetUri }) - } - - return { - type, - title: videoFile.resolution.label, - length: videoFile.size, - bitrate: videoFile.size / video.duration * 8, - language: video.language, - sources - } -} - -function buildVODStreamingPlaylists (video: MVideoFullLight) { - const hls = video.getHLSPlaylist() - if (!hls) return [] - - return [ - { - type: 'application/x-mpegURL', - title: 'HLS', - sources: [ - { uri: hls.getMasterPlaylistUrl(video) } - ], - language: video.language - } - ] -} - -function buildLiveStreamingPlaylists (video: MVideoFullLight) { - const hls = video.getHLSPlaylist() - - return [ - { - type: 'application/x-mpegURL', - title: `HLS live stream`, - sources: [ - { uri: hls.getMasterPlaylistUrl(video) } - ], - language: video.language - } - ] -} - -function buildVODCaptions (video: MVideo, videoCaptions: MVideoCaptionVideo[]) { - return videoCaptions.map(caption => { - const type = MIMETYPES.VIDEO_CAPTIONS.EXT_MIMETYPE[extname(caption.filename)] - if (!type) return null - - return { - url: caption.getFileUrl(video), - language: caption.language, - type, - rel: 'captions' - } - }).filter(c => c) -} diff --git a/server/controllers/index.ts b/server/controllers/index.ts deleted file mode 100644 index 8a647aff1..000000000 --- a/server/controllers/index.ts +++ /dev/null @@ -1,14 +0,0 @@ -export * from './activitypub' -export * from './api' -export * from './sitemap' -export * from './client' -export * from './download' -export * from './feeds' -export * from './lazy-static' -export * from './misc' -export * from './object-storage-proxy' -export * from './plugins' -export * from './services' -export * from './static' -export * from './tracker' -export * from './well-known' diff --git a/server/controllers/lazy-static.ts b/server/controllers/lazy-static.ts deleted file mode 100644 index dad30365c..000000000 --- a/server/controllers/lazy-static.ts +++ /dev/null @@ -1,128 +0,0 @@ -import cors from 'cors' -import express from 'express' -import { CONFIG } from '@server/initializers/config' -import { HttpStatusCode } from '../../shared/models/http/http-error-codes' -import { FILES_CACHE, LAZY_STATIC_PATHS, STATIC_MAX_AGE } from '../initializers/constants' -import { - AvatarPermanentFileCache, - VideoCaptionsSimpleFileCache, - VideoMiniaturePermanentFileCache, - VideoPreviewsSimpleFileCache, - VideoStoryboardsSimpleFileCache, - VideoTorrentsSimpleFileCache -} from '../lib/files-cache' -import { asyncMiddleware, handleStaticError } from '../middlewares' - -// --------------------------------------------------------------------------- -// Cache initializations -// --------------------------------------------------------------------------- - -VideoPreviewsSimpleFileCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, FILES_CACHE.PREVIEWS.MAX_AGE) -VideoCaptionsSimpleFileCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE) -VideoTorrentsSimpleFileCache.Instance.init(CONFIG.CACHE.TORRENTS.SIZE, FILES_CACHE.TORRENTS.MAX_AGE) -VideoStoryboardsSimpleFileCache.Instance.init(CONFIG.CACHE.STORYBOARDS.SIZE, FILES_CACHE.STORYBOARDS.MAX_AGE) - -// --------------------------------------------------------------------------- - -const lazyStaticRouter = express.Router() - -lazyStaticRouter.use(cors()) - -lazyStaticRouter.use( - LAZY_STATIC_PATHS.AVATARS + ':filename', - asyncMiddleware(getActorImage), - handleStaticError -) - -lazyStaticRouter.use( - LAZY_STATIC_PATHS.BANNERS + ':filename', - asyncMiddleware(getActorImage), - handleStaticError -) - -lazyStaticRouter.use( - LAZY_STATIC_PATHS.THUMBNAILS + ':filename', - asyncMiddleware(getThumbnail), - handleStaticError -) - -lazyStaticRouter.use( - LAZY_STATIC_PATHS.PREVIEWS + ':filename', - asyncMiddleware(getPreview), - handleStaticError -) - -lazyStaticRouter.use( - LAZY_STATIC_PATHS.STORYBOARDS + ':filename', - asyncMiddleware(getStoryboard), - handleStaticError -) - -lazyStaticRouter.use( - LAZY_STATIC_PATHS.VIDEO_CAPTIONS + ':filename', - asyncMiddleware(getVideoCaption), - handleStaticError -) - -lazyStaticRouter.use( - LAZY_STATIC_PATHS.TORRENTS + ':filename', - asyncMiddleware(getTorrent), - handleStaticError -) - -// --------------------------------------------------------------------------- - -export { - lazyStaticRouter, - getPreview, - getVideoCaption -} - -// --------------------------------------------------------------------------- -const avatarPermanentFileCache = new AvatarPermanentFileCache() - -function getActorImage (req: express.Request, res: express.Response, next: express.NextFunction) { - const filename = req.params.filename - - return avatarPermanentFileCache.lazyServe({ filename, res, next }) -} - -// --------------------------------------------------------------------------- -const videoMiniaturePermanentFileCache = new VideoMiniaturePermanentFileCache() - -function getThumbnail (req: express.Request, res: express.Response, next: express.NextFunction) { - const filename = req.params.filename - - return videoMiniaturePermanentFileCache.lazyServe({ filename, res, next }) -} - -// --------------------------------------------------------------------------- - -async function getPreview (req: express.Request, res: express.Response) { - const result = await VideoPreviewsSimpleFileCache.Instance.getFilePath(req.params.filename) - if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end() - - return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }) -} - -async function getStoryboard (req: express.Request, res: express.Response) { - const result = await VideoStoryboardsSimpleFileCache.Instance.getFilePath(req.params.filename) - if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end() - - return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }) -} - -async function getVideoCaption (req: express.Request, res: express.Response) { - const result = await VideoCaptionsSimpleFileCache.Instance.getFilePath(req.params.filename) - if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end() - - return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }) -} - -async function getTorrent (req: express.Request, res: express.Response) { - const result = await VideoTorrentsSimpleFileCache.Instance.getFilePath(req.params.filename) - if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end() - - // Torrents still use the old naming convention (video uuid + .torrent) - return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.SERVER }) -} diff --git a/server/controllers/misc.ts b/server/controllers/misc.ts deleted file mode 100644 index a7dfc7867..000000000 --- a/server/controllers/misc.ts +++ /dev/null @@ -1,210 +0,0 @@ -import cors from 'cors' -import express from 'express' -import { CONFIG, isEmailEnabled } from '@server/initializers/config' -import { serveIndexHTML } from '@server/lib/client-html' -import { ServerConfigManager } from '@server/lib/server-config-manager' -import { HttpStatusCode } from '@shared/models' -import { HttpNodeinfoDiasporaSoftwareNsSchema20 } from '../../shared/models/nodeinfo/nodeinfo.model' -import { CONSTRAINTS_FIELDS, DEFAULT_THEME_NAME, PEERTUBE_VERSION, ROUTE_CACHE_LIFETIME } from '../initializers/constants' -import { getThemeOrDefault } from '../lib/plugins/theme-utils' -import { apiRateLimiter, asyncMiddleware } from '../middlewares' -import { cacheRoute } from '../middlewares/cache/cache' -import { UserModel } from '../models/user/user' -import { VideoModel } from '../models/video/video' -import { VideoCommentModel } from '../models/video/video-comment' - -const miscRouter = express.Router() - -miscRouter.use(cors()) - -miscRouter.use('/nodeinfo/:version.json', - apiRateLimiter, - cacheRoute(ROUTE_CACHE_LIFETIME.NODEINFO), - asyncMiddleware(generateNodeinfo) -) - -// robots.txt service -miscRouter.get('/robots.txt', - apiRateLimiter, - cacheRoute(ROUTE_CACHE_LIFETIME.ROBOTS), - (_, res: express.Response) => { - res.type('text/plain') - - return res.send(CONFIG.INSTANCE.ROBOTS) - } -) - -miscRouter.all('/teapot', - apiRateLimiter, - getCup, - asyncMiddleware(serveIndexHTML) -) - -// security.txt service -miscRouter.get('/security.txt', - apiRateLimiter, - (_, res: express.Response) => { - return res.redirect(HttpStatusCode.MOVED_PERMANENTLY_301, '/.well-known/security.txt') - } -) - -// --------------------------------------------------------------------------- - -export { - miscRouter -} - -// --------------------------------------------------------------------------- - -async function generateNodeinfo (req: express.Request, res: express.Response) { - const { totalVideos } = await VideoModel.getStats() - const { totalLocalVideoComments } = await VideoCommentModel.getStats() - const { totalUsers, totalMonthlyActiveUsers, totalHalfYearActiveUsers } = await UserModel.getStats() - - if (!req.params.version || req.params.version !== '2.0') { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Nodeinfo schema version not handled' - }) - } - - const json = { - version: '2.0', - software: { - name: 'peertube', - version: PEERTUBE_VERSION - }, - protocols: [ - 'activitypub' - ], - services: { - inbound: [], - outbound: [ - 'atom1.0', - 'rss2.0' - ] - }, - openRegistrations: CONFIG.SIGNUP.ENABLED, - usage: { - users: { - total: totalUsers, - activeMonth: totalMonthlyActiveUsers, - activeHalfyear: totalHalfYearActiveUsers - }, - localPosts: totalVideos, - localComments: totalLocalVideoComments - }, - metadata: { - taxonomy: { - postsName: 'Videos' - }, - nodeName: CONFIG.INSTANCE.NAME, - nodeDescription: CONFIG.INSTANCE.SHORT_DESCRIPTION, - nodeConfig: { - search: { - remoteUri: { - users: CONFIG.SEARCH.REMOTE_URI.USERS, - anonymous: CONFIG.SEARCH.REMOTE_URI.ANONYMOUS - } - }, - plugin: { - registered: ServerConfigManager.Instance.getRegisteredPlugins() - }, - theme: { - registered: ServerConfigManager.Instance.getRegisteredThemes(), - default: getThemeOrDefault(CONFIG.THEME.DEFAULT, DEFAULT_THEME_NAME) - }, - email: { - enabled: isEmailEnabled() - }, - contactForm: { - enabled: CONFIG.CONTACT_FORM.ENABLED - }, - transcoding: { - hls: { - enabled: CONFIG.TRANSCODING.HLS.ENABLED - }, - web_videos: { - enabled: CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED - }, - enabledResolutions: ServerConfigManager.Instance.getEnabledResolutions('vod') - }, - live: { - enabled: CONFIG.LIVE.ENABLED, - transcoding: { - enabled: CONFIG.LIVE.TRANSCODING.ENABLED, - enabledResolutions: ServerConfigManager.Instance.getEnabledResolutions('live') - } - }, - import: { - videos: { - http: { - enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED - }, - torrent: { - enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED - } - } - }, - autoBlacklist: { - videos: { - ofUsers: { - enabled: CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED - } - } - }, - avatar: { - file: { - size: { - max: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max - }, - extensions: CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME - } - }, - video: { - image: { - extensions: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.EXTNAME, - size: { - max: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max - } - }, - file: { - extensions: CONSTRAINTS_FIELDS.VIDEOS.EXTNAME - } - }, - videoCaption: { - file: { - size: { - max: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max - }, - extensions: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.EXTNAME - } - }, - user: { - videoQuota: CONFIG.USER.VIDEO_QUOTA, - videoQuotaDaily: CONFIG.USER.VIDEO_QUOTA_DAILY - }, - trending: { - videos: { - intervalDays: CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS - } - }, - tracker: { - enabled: CONFIG.TRACKER.ENABLED - } - } - } - } as HttpNodeinfoDiasporaSoftwareNsSchema20 - - res.contentType('application/json; profile="http://nodeinfo.diaspora.software/ns/schema/2.0#"') - .send(json) - .end() -} - -function getCup (req: express.Request, res: express.Response, next: express.NextFunction) { - res.status(HttpStatusCode.I_AM_A_TEAPOT_418) - res.setHeader('Accept-Additions', 'Non-Dairy;1,Sugar;1') - res.setHeader('Safe', 'if-sepia-awake') - - return next() -} diff --git a/server/controllers/object-storage-proxy.ts b/server/controllers/object-storage-proxy.ts deleted file mode 100644 index d0c59bf93..000000000 --- a/server/controllers/object-storage-proxy.ts +++ /dev/null @@ -1,60 +0,0 @@ -import cors from 'cors' -import express from 'express' -import { OBJECT_STORAGE_PROXY_PATHS } from '@server/initializers/constants' -import { proxifyHLS, proxifyWebVideoFile } from '@server/lib/object-storage' -import { - asyncMiddleware, - ensureCanAccessPrivateVideoHLSFiles, - ensureCanAccessVideoPrivateWebVideoFiles, - ensurePrivateObjectStorageProxyIsEnabled, - optionalAuthenticate -} from '@server/middlewares' -import { doReinjectVideoFileToken } from './shared/m3u8-playlist' - -const objectStorageProxyRouter = express.Router() - -objectStorageProxyRouter.use(cors()) - -objectStorageProxyRouter.get( - [ OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEB_VIDEOS + ':filename', OBJECT_STORAGE_PROXY_PATHS.LEGACY_PRIVATE_WEB_VIDEOS + ':filename' ], - ensurePrivateObjectStorageProxyIsEnabled, - optionalAuthenticate, - asyncMiddleware(ensureCanAccessVideoPrivateWebVideoFiles), - asyncMiddleware(proxifyWebVideoController) -) - -objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:filename', - ensurePrivateObjectStorageProxyIsEnabled, - optionalAuthenticate, - asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles), - asyncMiddleware(proxifyHLSController) -) - -// --------------------------------------------------------------------------- - -export { - objectStorageProxyRouter -} - -function proxifyWebVideoController (req: express.Request, res: express.Response) { - const filename = req.params.filename - - return proxifyWebVideoFile({ req, res, filename }) -} - -function proxifyHLSController (req: express.Request, res: express.Response) { - const playlist = res.locals.videoStreamingPlaylist - const video = res.locals.onlyVideo - const filename = req.params.filename - - const reinjectVideoFileToken = filename.endsWith('.m3u8') && doReinjectVideoFileToken(req) - - return proxifyHLS({ - req, - res, - playlist, - video, - filename, - reinjectVideoFileToken - }) -} diff --git a/server/controllers/plugins.ts b/server/controllers/plugins.ts deleted file mode 100644 index f0491b16a..000000000 --- a/server/controllers/plugins.ts +++ /dev/null @@ -1,175 +0,0 @@ -import express from 'express' -import { join } from 'path' -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { buildRateLimiter } from '@server/middlewares' -import { optionalAuthenticate } from '@server/middlewares/auth' -import { getCompleteLocale, is18nLocale } from '../../shared/core-utils/i18n' -import { HttpStatusCode } from '../../shared/models/http/http-error-codes' -import { PluginType } from '../../shared/models/plugins/plugin.type' -import { isProdInstance } from '../helpers/core-utils' -import { PLUGIN_GLOBAL_CSS_PATH } from '../initializers/constants' -import { PluginManager, RegisteredPlugin } from '../lib/plugins/plugin-manager' -import { getExternalAuthValidator, getPluginValidator, pluginStaticDirectoryValidator } from '../middlewares/validators/plugins' -import { serveThemeCSSValidator } from '../middlewares/validators/themes' - -const sendFileOptions = { - maxAge: '30 days', - immutable: isProdInstance() -} - -const pluginsRouter = express.Router() - -const pluginsRateLimiter = buildRateLimiter({ - windowMs: CONFIG.RATES_LIMIT.PLUGINS.WINDOW_MS, - max: CONFIG.RATES_LIMIT.PLUGINS.MAX -}) - -pluginsRouter.get('/plugins/global.css', - pluginsRateLimiter, - servePluginGlobalCSS -) - -pluginsRouter.get('/plugins/translations/:locale.json', - pluginsRateLimiter, - getPluginTranslations -) - -pluginsRouter.get('/plugins/:pluginName/:pluginVersion/auth/:authName', - pluginsRateLimiter, - getPluginValidator(PluginType.PLUGIN), - getExternalAuthValidator, - handleAuthInPlugin -) - -pluginsRouter.get('/plugins/:pluginName/:pluginVersion/static/:staticEndpoint(*)', - pluginsRateLimiter, - getPluginValidator(PluginType.PLUGIN), - pluginStaticDirectoryValidator, - servePluginStaticDirectory -) - -pluginsRouter.get('/plugins/:pluginName/:pluginVersion/client-scripts/:staticEndpoint(*)', - pluginsRateLimiter, - getPluginValidator(PluginType.PLUGIN), - pluginStaticDirectoryValidator, - servePluginClientScripts -) - -pluginsRouter.use('/plugins/:pluginName/router', - pluginsRateLimiter, - getPluginValidator(PluginType.PLUGIN, false), - optionalAuthenticate, - servePluginCustomRoutes -) - -pluginsRouter.use('/plugins/:pluginName/:pluginVersion/router', - pluginsRateLimiter, - getPluginValidator(PluginType.PLUGIN), - optionalAuthenticate, - servePluginCustomRoutes -) - -pluginsRouter.get('/themes/:pluginName/:pluginVersion/static/:staticEndpoint(*)', - pluginsRateLimiter, - getPluginValidator(PluginType.THEME), - pluginStaticDirectoryValidator, - servePluginStaticDirectory -) - -pluginsRouter.get('/themes/:pluginName/:pluginVersion/client-scripts/:staticEndpoint(*)', - pluginsRateLimiter, - getPluginValidator(PluginType.THEME), - pluginStaticDirectoryValidator, - servePluginClientScripts -) - -pluginsRouter.get('/themes/:themeName/:themeVersion/css/:staticEndpoint(*)', - pluginsRateLimiter, - serveThemeCSSValidator, - serveThemeCSSDirectory -) - -// --------------------------------------------------------------------------- - -export { - pluginsRouter -} - -// --------------------------------------------------------------------------- - -function servePluginGlobalCSS (req: express.Request, res: express.Response) { - // Only cache requests that have a ?hash=... query param - const globalCSSOptions = req.query.hash - ? sendFileOptions - : {} - - return res.sendFile(PLUGIN_GLOBAL_CSS_PATH, globalCSSOptions) -} - -function getPluginTranslations (req: express.Request, res: express.Response) { - const locale = req.params.locale - - if (is18nLocale(locale)) { - const completeLocale = getCompleteLocale(locale) - const json = PluginManager.Instance.getTranslations(completeLocale) - - return res.json(json) - } - - return res.status(HttpStatusCode.NOT_FOUND_404).end() -} - -function servePluginStaticDirectory (req: express.Request, res: express.Response) { - const plugin: RegisteredPlugin = res.locals.registeredPlugin - const staticEndpoint = req.params.staticEndpoint - - const [ directory, ...file ] = staticEndpoint.split('/') - - const staticPath = plugin.staticDirs[directory] - if (!staticPath) return res.status(HttpStatusCode.NOT_FOUND_404).end() - - const filepath = file.join('/') - return res.sendFile(join(plugin.path, staticPath, filepath), sendFileOptions) -} - -function servePluginCustomRoutes (req: express.Request, res: express.Response, next: express.NextFunction) { - const plugin: RegisteredPlugin = res.locals.registeredPlugin - const router = PluginManager.Instance.getRouter(plugin.npmName) - - if (!router) return res.status(HttpStatusCode.NOT_FOUND_404).end() - - return router(req, res, next) -} - -function servePluginClientScripts (req: express.Request, res: express.Response) { - const plugin: RegisteredPlugin = res.locals.registeredPlugin - const staticEndpoint = req.params.staticEndpoint - - const file = plugin.clientScripts[staticEndpoint] - if (!file) return res.status(HttpStatusCode.NOT_FOUND_404).end() - - return res.sendFile(join(plugin.path, staticEndpoint), sendFileOptions) -} - -function serveThemeCSSDirectory (req: express.Request, res: express.Response) { - const plugin: RegisteredPlugin = res.locals.registeredPlugin - const staticEndpoint = req.params.staticEndpoint - - if (plugin.css.includes(staticEndpoint) === false) { - return res.status(HttpStatusCode.NOT_FOUND_404).end() - } - - return res.sendFile(join(plugin.path, staticEndpoint), sendFileOptions) -} - -function handleAuthInPlugin (req: express.Request, res: express.Response) { - const authOptions = res.locals.externalAuth - - try { - logger.debug('Forwarding auth plugin request in %s of plugin %s.', authOptions.authName, res.locals.registeredPlugin.npmName) - authOptions.onAuthRequest(req, res) - } catch (err) { - logger.error('Forward request error in auth %s of plugin %s.', authOptions.authName, res.locals.registeredPlugin.npmName, { err }) - } -} diff --git a/server/controllers/services.ts b/server/controllers/services.ts deleted file mode 100644 index 0fd63a30f..000000000 --- a/server/controllers/services.ts +++ /dev/null @@ -1,165 +0,0 @@ -import express from 'express' -import { MChannelSummary } from '@server/types/models' -import { escapeHTML } from '@shared/core-utils/renderer' -import { EMBED_SIZE, PREVIEWS_SIZE, THUMBNAILS_SIZE, WEBSERVER } from '../initializers/constants' -import { apiRateLimiter, asyncMiddleware, oembedValidator } from '../middlewares' -import { accountNameWithHostGetValidator } from '../middlewares/validators' -import { forceNumber } from '@shared/core-utils' - -const servicesRouter = express.Router() - -servicesRouter.use('/oembed', - apiRateLimiter, - asyncMiddleware(oembedValidator), - generateOEmbed -) -servicesRouter.use('/redirect/accounts/:accountName', - apiRateLimiter, - asyncMiddleware(accountNameWithHostGetValidator), - redirectToAccountUrl -) - -// --------------------------------------------------------------------------- - -export { - servicesRouter -} - -// --------------------------------------------------------------------------- - -function generateOEmbed (req: express.Request, res: express.Response) { - if (res.locals.videoAll) return generateVideoOEmbed(req, res) - - return generatePlaylistOEmbed(req, res) -} - -function generatePlaylistOEmbed (req: express.Request, res: express.Response) { - const playlist = res.locals.videoPlaylistSummary - - const json = buildOEmbed({ - channel: playlist.VideoChannel, - title: playlist.name, - embedPath: playlist.getEmbedStaticPath() + buildPlayerURLQuery(req.query.url), - previewPath: playlist.getThumbnailStaticPath(), - previewSize: THUMBNAILS_SIZE, - req - }) - - return res.json(json) -} - -function generateVideoOEmbed (req: express.Request, res: express.Response) { - const video = res.locals.videoAll - - const json = buildOEmbed({ - channel: video.VideoChannel, - title: video.name, - embedPath: video.getEmbedStaticPath() + buildPlayerURLQuery(req.query.url), - previewPath: video.getPreviewStaticPath(), - previewSize: PREVIEWS_SIZE, - req - }) - - return res.json(json) -} - -function buildPlayerURLQuery (inputQueryUrl: string) { - const allowedParameters = new Set([ - 'start', - 'stop', - 'loop', - 'autoplay', - 'muted', - 'controls', - 'controlBar', - 'title', - 'api', - 'warningTitle', - 'peertubeLink', - 'p2p', - 'subtitle', - 'bigPlayBackgroundColor', - 'mode', - 'foregroundColor' - ]) - - const params = new URLSearchParams() - - new URL(inputQueryUrl).searchParams.forEach((v, k) => { - if (allowedParameters.has(k)) { - params.append(k, v) - } - }) - - const stringQuery = params.toString() - if (!stringQuery) return '' - - return '?' + stringQuery -} - -function buildOEmbed (options: { - req: express.Request - title: string - channel: MChannelSummary - previewPath: string | null - embedPath: string - previewSize: { - height: number - width: number - } -}) { - const { req, previewSize, previewPath, title, channel, embedPath } = options - - const webserverUrl = WEBSERVER.URL - const maxHeight = forceNumber(req.query.maxheight) - const maxWidth = forceNumber(req.query.maxwidth) - - const embedUrl = webserverUrl + embedPath - const embedTitle = escapeHTML(title) - - let thumbnailUrl = previewPath - ? webserverUrl + previewPath - : undefined - - let embedWidth = EMBED_SIZE.width - if (maxWidth < embedWidth) embedWidth = maxWidth - - let embedHeight = EMBED_SIZE.height - if (maxHeight < embedHeight) embedHeight = maxHeight - - // Our thumbnail is too big for the consumer - if ( - (maxHeight !== undefined && maxHeight < previewSize.height) || - (maxWidth !== undefined && maxWidth < previewSize.width) - ) { - thumbnailUrl = undefined - } - - const html = `` - - const json: any = { - type: 'video', - version: '1.0', - html, - width: embedWidth, - height: embedHeight, - title, - author_name: channel.name, - author_url: channel.Actor.url, - provider_name: 'PeerTube', - provider_url: webserverUrl - } - - if (thumbnailUrl !== undefined) { - json.thumbnail_url = thumbnailUrl - json.thumbnail_width = previewSize.width - json.thumbnail_height = previewSize.height - } - - return json -} - -function redirectToAccountUrl (req: express.Request, res: express.Response, next: express.NextFunction) { - return res.redirect(res.locals.account.Actor.url) -} diff --git a/server/controllers/sitemap.ts b/server/controllers/sitemap.ts deleted file mode 100644 index 07f4c554e..000000000 --- a/server/controllers/sitemap.ts +++ /dev/null @@ -1,115 +0,0 @@ -import express from 'express' -import { truncate } from 'lodash' -import { ErrorLevel, SitemapStream, streamToPromise } from 'sitemap' -import { logger } from '@server/helpers/logger' -import { getServerActor } from '@server/models/application/application' -import { buildNSFWFilter } from '../helpers/express-utils' -import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants' -import { apiRateLimiter, asyncMiddleware } from '../middlewares' -import { cacheRoute } from '../middlewares/cache/cache' -import { AccountModel } from '../models/account/account' -import { VideoModel } from '../models/video/video' -import { VideoChannelModel } from '../models/video/video-channel' - -const sitemapRouter = express.Router() - -sitemapRouter.use('/sitemap.xml', - apiRateLimiter, - cacheRoute(ROUTE_CACHE_LIFETIME.SITEMAP), - asyncMiddleware(getSitemap) -) - -// --------------------------------------------------------------------------- - -export { - sitemapRouter -} - -// --------------------------------------------------------------------------- - -async function getSitemap (req: express.Request, res: express.Response) { - let urls = getSitemapBasicUrls() - - urls = urls.concat(await getSitemapLocalVideoUrls()) - urls = urls.concat(await getSitemapVideoChannelUrls()) - urls = urls.concat(await getSitemapAccountUrls()) - - const sitemapStream = new SitemapStream({ - hostname: WEBSERVER.URL, - errorHandler: (err: Error, level: ErrorLevel) => { - if (level === 'warn') { - logger.warn('Warning in sitemap generation.', { err }) - } else if (level === 'throw') { - logger.error('Error in sitemap generation.', { err }) - - throw err - } - } - }) - - for (const urlObj of urls) { - sitemapStream.write(urlObj) - } - sitemapStream.end() - - const xml = await streamToPromise(sitemapStream) - - res.header('Content-Type', 'application/xml') - res.send(xml) -} - -async function getSitemapVideoChannelUrls () { - const rows = await VideoChannelModel.listLocalsForSitemap('createdAt') - - return rows.map(channel => ({ - url: WEBSERVER.URL + '/video-channels/' + channel.Actor.preferredUsername - })) -} - -async function getSitemapAccountUrls () { - const rows = await AccountModel.listLocalsForSitemap('createdAt') - - return rows.map(channel => ({ - url: WEBSERVER.URL + '/accounts/' + channel.Actor.preferredUsername - })) -} - -async function getSitemapLocalVideoUrls () { - const serverActor = await getServerActor() - - const { data } = await VideoModel.listForApi({ - start: 0, - count: undefined, - sort: 'createdAt', - displayOnlyForFollower: { - actorId: serverActor.id, - orLocalVideos: true - }, - isLocal: true, - nsfw: buildNSFWFilter(), - countVideos: false - }) - - return data.map(v => ({ - url: WEBSERVER.URL + v.getWatchStaticPath(), - video: [ - { - // Sitemap title should be < 100 characters - title: truncate(v.name, { length: 100, omission: '...' }), - // Sitemap description should be < 2000 characters - description: truncate(v.description || v.name, { length: 2000, omission: '...' }), - player_loc: WEBSERVER.URL + v.getEmbedStaticPath(), - thumbnail_loc: WEBSERVER.URL + v.getMiniatureStaticPath() - } - ] - })) -} - -function getSitemapBasicUrls () { - const paths = [ - '/about/instance', - '/videos/local' - ] - - return paths.map(p => ({ url: WEBSERVER.URL + p })) -} diff --git a/server/controllers/static.ts b/server/controllers/static.ts deleted file mode 100644 index 97caa8292..000000000 --- a/server/controllers/static.ts +++ /dev/null @@ -1,116 +0,0 @@ -import cors from 'cors' -import express from 'express' -import { readFile } from 'fs-extra' -import { join } from 'path' -import { injectQueryToPlaylistUrls } from '@server/lib/hls' -import { - asyncMiddleware, - ensureCanAccessPrivateVideoHLSFiles, - ensureCanAccessVideoPrivateWebVideoFiles, - handleStaticError, - optionalAuthenticate -} from '@server/middlewares' -import { HttpStatusCode } from '@shared/models' -import { CONFIG } from '../initializers/config' -import { DIRECTORIES, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers/constants' -import { buildReinjectVideoFileTokenQuery, doReinjectVideoFileToken } from './shared/m3u8-playlist' - -const staticRouter = express.Router() - -// Cors is very important to let other servers access torrent and video files -staticRouter.use(cors()) - -// --------------------------------------------------------------------------- -// Web videos/Classic videos -// --------------------------------------------------------------------------- - -const privateWebVideoStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUIRE_AUTH === true - ? [ optionalAuthenticate, asyncMiddleware(ensureCanAccessVideoPrivateWebVideoFiles) ] - : [] - -staticRouter.use( - [ STATIC_PATHS.PRIVATE_WEB_VIDEOS, STATIC_PATHS.LEGACY_PRIVATE_WEB_VIDEOS ], - ...privateWebVideoStaticMiddlewares, - express.static(DIRECTORIES.VIDEOS.PRIVATE, { fallthrough: false }), - handleStaticError -) -staticRouter.use( - [ STATIC_PATHS.WEB_VIDEOS, STATIC_PATHS.LEGACY_WEB_VIDEOS ], - express.static(DIRECTORIES.VIDEOS.PUBLIC, { fallthrough: false }), - handleStaticError -) - -staticRouter.use( - STATIC_PATHS.REDUNDANCY, - express.static(CONFIG.STORAGE.REDUNDANCY_DIR, { fallthrough: false }), - handleStaticError -) - -// --------------------------------------------------------------------------- -// HLS -// --------------------------------------------------------------------------- - -const privateHLSStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUIRE_AUTH === true - ? [ optionalAuthenticate, asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles) ] - : [] - -staticRouter.use( - STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:playlistName.m3u8', - ...privateHLSStaticMiddlewares, - asyncMiddleware(servePrivateM3U8) -) - -staticRouter.use( - STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, - ...privateHLSStaticMiddlewares, - express.static(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, { fallthrough: false }), - handleStaticError -) -staticRouter.use( - STATIC_PATHS.STREAMING_PLAYLISTS.HLS, - express.static(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, { fallthrough: false }), - handleStaticError -) - -// FIXME: deprecated in v6, to remove -const thumbnailsPhysicalPath = CONFIG.STORAGE.THUMBNAILS_DIR -staticRouter.use( - STATIC_PATHS.THUMBNAILS, - express.static(thumbnailsPhysicalPath, { maxAge: STATIC_MAX_AGE.SERVER, fallthrough: false }), - handleStaticError -) - -// --------------------------------------------------------------------------- - -export { - staticRouter -} - -// --------------------------------------------------------------------------- - -async function servePrivateM3U8 (req: express.Request, res: express.Response) { - const path = join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, req.params.videoUUID, req.params.playlistName + '.m3u8') - const filename = req.params.playlistName + '.m3u8' - - let playlistContent: string - - try { - playlistContent = await readFile(path, 'utf-8') - } catch (err) { - if (err.message.includes('ENOENT')) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'File not found' - }) - } - - throw err - } - - // Inject token in playlist so players that cannot alter the HTTP request can still watch the video - const transformedContent = doReinjectVideoFileToken(req) - ? injectQueryToPlaylistUrls(playlistContent, buildReinjectVideoFileTokenQuery(req, filename.endsWith('master.m3u8'))) - : playlistContent - - return res.set('content-type', 'application/vnd.apple.mpegurl').send(transformedContent).end() -} diff --git a/server/controllers/tracker.ts b/server/controllers/tracker.ts deleted file mode 100644 index 9a8aa88bc..000000000 --- a/server/controllers/tracker.ts +++ /dev/null @@ -1,148 +0,0 @@ -import { Server as TrackerServer } from 'bittorrent-tracker' -import express from 'express' -import { createServer } from 'http' -import { LRUCache } from 'lru-cache' -import proxyAddr from 'proxy-addr' -import { WebSocketServer } from 'ws' -import { logger } from '../helpers/logger' -import { CONFIG } from '../initializers/config' -import { LRU_CACHE, TRACKER_RATE_LIMITS } from '../initializers/constants' -import { VideoFileModel } from '../models/video/video-file' -import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' - -const trackerRouter = express.Router() - -const blockedIPs = new LRUCache({ - max: LRU_CACHE.TRACKER_IPS.MAX_SIZE, - ttl: TRACKER_RATE_LIMITS.BLOCK_IP_LIFETIME -}) - -let peersIps = {} -let peersIpInfoHash = {} -runPeersChecker() - -const trackerServer = new TrackerServer({ - http: false, - udp: false, - ws: false, - filter: async function (infoHash, params, cb) { - if (CONFIG.TRACKER.ENABLED === false) { - return cb(new Error('Tracker is disabled on this instance.')) - } - - let ip: string - - if (params.type === 'ws') { - ip = params.ip - } else { - ip = params.httpReq.ip - } - - const key = ip + '-' + infoHash - - peersIps[ip] = peersIps[ip] ? peersIps[ip] + 1 : 1 - peersIpInfoHash[key] = peersIpInfoHash[key] ? peersIpInfoHash[key] + 1 : 1 - - if (CONFIG.TRACKER.REJECT_TOO_MANY_ANNOUNCES && peersIpInfoHash[key] > TRACKER_RATE_LIMITS.ANNOUNCES_PER_IP_PER_INFOHASH) { - return cb(new Error(`Too many requests (${peersIpInfoHash[key]} of ip ${ip} for torrent ${infoHash}`)) - } - - try { - if (CONFIG.TRACKER.PRIVATE === false) return cb() - - const videoFileExists = await VideoFileModel.doesInfohashExistCached(infoHash) - if (videoFileExists === true) return cb() - - const playlistExists = await VideoStreamingPlaylistModel.doesInfohashExistCached(infoHash) - if (playlistExists === true) return cb() - - cb(new Error(`Unknown infoHash ${infoHash} requested by ip ${ip}`)) - - // Close socket connection and block IP for a few time - if (params.type === 'ws') { - blockedIPs.set(ip, true) - - // setTimeout to wait filter response - setTimeout(() => params.socket.close(), 0) - } - } catch (err) { - logger.error('Error in tracker filter.', { err }) - return cb(err) - } - } -}) - -if (CONFIG.TRACKER.ENABLED !== false) { - trackerServer.on('error', function (err) { - logger.error('Error in tracker.', { err }) - }) - - trackerServer.on('warning', function (err) { - const message = err.message || '' - - if (CONFIG.LOG.LOG_TRACKER_UNKNOWN_INFOHASH === false && message.includes('Unknown infoHash')) { - return - } - - logger.warn('Warning in tracker.', { err }) - }) -} - -const onHttpRequest = trackerServer.onHttpRequest.bind(trackerServer) -trackerRouter.get('/tracker/announce', (req, res) => onHttpRequest(req, res, { action: 'announce' })) -trackerRouter.get('/tracker/scrape', (req, res) => onHttpRequest(req, res, { action: 'scrape' })) - -function createWebsocketTrackerServer (app: express.Application) { - const server = createServer(app) - const wss = new WebSocketServer({ noServer: true }) - - wss.on('connection', function (ws, req) { - ws['ip'] = proxyAddr(req, CONFIG.TRUST_PROXY) - - trackerServer.onWebSocketConnection(ws) - }) - - server.on('upgrade', (request: express.Request, socket, head) => { - if (request.url === '/tracker/socket') { - const ip = proxyAddr(request, CONFIG.TRUST_PROXY) - - if (blockedIPs.has(ip)) { - logger.debug('Blocking IP %s from tracker.', ip) - - socket.write('HTTP/1.1 403 Forbidden\r\n\r\n') - socket.destroy() - return - } - - return wss.handleUpgrade(request, socket, head, ws => wss.emit('connection', ws, request)) - } - - // Don't destroy socket, we have Socket.IO too - }) - - return { server, trackerServer } -} - -// --------------------------------------------------------------------------- - -export { - trackerRouter, - createWebsocketTrackerServer -} - -// --------------------------------------------------------------------------- - -function runPeersChecker () { - setInterval(() => { - logger.debug('Checking peers.') - - for (const ip of Object.keys(peersIpInfoHash)) { - if (peersIps[ip] > TRACKER_RATE_LIMITS.ANNOUNCES_PER_IP) { - logger.warn('Peer %s made abnormal requests (%d).', ip, peersIps[ip]) - } - } - - peersIpInfoHash = {} - peersIps = {} - }, TRACKER_RATE_LIMITS.INTERVAL) -} diff --git a/server/controllers/well-known.ts b/server/controllers/well-known.ts deleted file mode 100644 index 322cf6ea2..000000000 --- a/server/controllers/well-known.ts +++ /dev/null @@ -1,125 +0,0 @@ -import cors from 'cors' -import express from 'express' -import { join } from 'path' -import { asyncMiddleware, buildRateLimiter, handleStaticError, webfingerValidator } from '@server/middlewares' -import { root } from '@shared/core-utils' -import { CONFIG } from '../initializers/config' -import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants' -import { cacheRoute } from '../middlewares/cache/cache' - -const wellKnownRouter = express.Router() - -const wellKnownRateLimiter = buildRateLimiter({ - windowMs: CONFIG.RATES_LIMIT.WELL_KNOWN.WINDOW_MS, - max: CONFIG.RATES_LIMIT.WELL_KNOWN.MAX -}) - -wellKnownRouter.use(cors()) - -wellKnownRouter.get('/.well-known/webfinger', - wellKnownRateLimiter, - asyncMiddleware(webfingerValidator), - webfingerController -) - -wellKnownRouter.get('/.well-known/security.txt', - wellKnownRateLimiter, - cacheRoute(ROUTE_CACHE_LIFETIME.SECURITYTXT), - (_, res: express.Response) => { - res.type('text/plain') - return res.send(CONFIG.INSTANCE.SECURITYTXT + CONFIG.INSTANCE.SECURITYTXT_CONTACT) - } -) - -// nodeinfo service -wellKnownRouter.use('/.well-known/nodeinfo', - wellKnownRateLimiter, - cacheRoute(ROUTE_CACHE_LIFETIME.NODEINFO), - (_, res: express.Response) => { - return res.json({ - links: [ - { - rel: 'http://nodeinfo.diaspora.software/ns/schema/2.0', - href: WEBSERVER.URL + '/nodeinfo/2.0.json' - } - ] - }) - } -) - -// dnt-policy.txt service (see https://www.eff.org/dnt-policy) -wellKnownRouter.use('/.well-known/dnt-policy.txt', - wellKnownRateLimiter, - cacheRoute(ROUTE_CACHE_LIFETIME.DNT_POLICY), - (_, res: express.Response) => { - res.type('text/plain') - - return res.sendFile(join(root(), 'dist/server/static/dnt-policy/dnt-policy-1.0.txt')) - } -) - -// dnt service (see https://www.w3.org/TR/tracking-dnt/#status-resource) -wellKnownRouter.use('/.well-known/dnt/', - wellKnownRateLimiter, - (_, res: express.Response) => { - res.json({ tracking: 'N' }) - } -) - -wellKnownRouter.use('/.well-known/change-password', - wellKnownRateLimiter, - (_, res: express.Response) => { - res.redirect('/my-account/settings') - } -) - -wellKnownRouter.use('/.well-known/host-meta', - wellKnownRateLimiter, - (_, res: express.Response) => { - res.type('application/xml') - - const xml = '\n' + - '\n' + - ` \n` + - '' - - res.send(xml).end() - } -) - -wellKnownRouter.use('/.well-known/', - wellKnownRateLimiter, - cacheRoute(ROUTE_CACHE_LIFETIME.WELL_KNOWN), - express.static(CONFIG.STORAGE.WELL_KNOWN_DIR, { fallthrough: false }), - handleStaticError -) - -// --------------------------------------------------------------------------- - -export { - wellKnownRouter -} - -// --------------------------------------------------------------------------- - -function webfingerController (req: express.Request, res: express.Response) { - const actor = res.locals.actorUrl - - const json = { - subject: req.query.resource, - aliases: [ actor.url ], - links: [ - { - rel: 'self', - type: 'application/activity+json', - href: actor.url - }, - { - rel: 'http://ostatus.org/schema/1.0/subscribe', - template: WEBSERVER.URL + '/remote-interaction?uri={uri}' - } - ] - } - - return res.json(json) -} diff --git a/server/helpers/actors.ts b/server/helpers/actors.ts deleted file mode 100644 index c31fe6f8e..000000000 --- a/server/helpers/actors.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { WEBSERVER } from '@server/initializers/constants' - -function handleToNameAndHost (handle: string) { - let [ name, host ] = handle.split('@') - if (host === WEBSERVER.HOST) host = null - - return { name, host, handle } -} - -function handlesToNameAndHost (handles: string[]) { - return handles.map(h => handleToNameAndHost(h)) -} - -export { - handleToNameAndHost, - handlesToNameAndHost -} diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts deleted file mode 100644 index 7e8a03e8f..000000000 --- a/server/helpers/audit-logger.ts +++ /dev/null @@ -1,287 +0,0 @@ -import { diff } from 'deep-object-diff' -import express from 'express' -import flatten from 'flat' -import { chain } from 'lodash' -import { join } from 'path' -import { addColors, config, createLogger, format, transports } from 'winston' -import { AUDIT_LOG_FILENAME } from '@server/initializers/constants' -import { AdminAbuse, CustomConfig, User, VideoChannel, VideoChannelSync, VideoComment, VideoDetails, VideoImport } from '@shared/models' -import { CONFIG } from '../initializers/config' -import { jsonLoggerFormat, labelFormatter } from './logger' - -function getAuditIdFromRes (res: express.Response) { - return res.locals.oauth.token.User.username -} - -enum AUDIT_TYPE { - CREATE = 'create', - UPDATE = 'update', - DELETE = 'delete' -} - -const colors = config.npm.colors -colors.audit = config.npm.colors.info - -addColors(colors) - -const auditLogger = createLogger({ - levels: { audit: 0 }, - transports: [ - new transports.File({ - filename: join(CONFIG.STORAGE.LOG_DIR, AUDIT_LOG_FILENAME), - level: 'audit', - maxsize: 5242880, - maxFiles: 5, - format: format.combine( - format.timestamp(), - labelFormatter(), - format.splat(), - jsonLoggerFormat - ) - }) - ], - exitOnError: true -}) - -function auditLoggerWrapper (domain: string, user: string, action: AUDIT_TYPE, entity: EntityAuditView, oldEntity: EntityAuditView = null) { - let entityInfos: object - if (action === AUDIT_TYPE.UPDATE && oldEntity) { - const oldEntityKeys = oldEntity.toLogKeys() - const diffObject = diff(oldEntityKeys, entity.toLogKeys()) - const diffKeys = Object.entries(diffObject).reduce((newKeys, entry) => { - newKeys[`new-${entry[0]}`] = entry[1] - return newKeys - }, {}) - entityInfos = { ...oldEntityKeys, ...diffKeys } - } else { - entityInfos = { ...entity.toLogKeys() } - } - auditLogger.log('audit', JSON.stringify({ - user, - domain, - action, - ...entityInfos - })) -} - -function auditLoggerFactory (domain: string) { - return { - create (user: string, entity: EntityAuditView) { - auditLoggerWrapper(domain, user, AUDIT_TYPE.CREATE, entity) - }, - update (user: string, entity: EntityAuditView, oldEntity: EntityAuditView) { - auditLoggerWrapper(domain, user, AUDIT_TYPE.UPDATE, entity, oldEntity) - }, - delete (user: string, entity: EntityAuditView) { - auditLoggerWrapper(domain, user, AUDIT_TYPE.DELETE, entity) - } - } -} - -abstract class EntityAuditView { - constructor (private readonly keysToKeep: string[], private readonly prefix: string, private readonly entityInfos: object) { } - - toLogKeys (): object { - return chain(flatten(this.entityInfos, { delimiter: '-', safe: true })) - .pick(this.keysToKeep) - .mapKeys((_value, key) => `${this.prefix}-${key}`) - .value() - } -} - -const videoKeysToKeep = [ - 'tags', - 'uuid', - 'id', - 'uuid', - 'createdAt', - 'updatedAt', - 'publishedAt', - 'category', - 'licence', - 'language', - 'privacy', - 'description', - 'duration', - 'isLocal', - 'name', - 'thumbnailPath', - 'previewPath', - 'nsfw', - 'waitTranscoding', - 'account-id', - 'account-uuid', - 'account-name', - 'channel-id', - 'channel-uuid', - 'channel-name', - 'support', - 'commentsEnabled', - 'downloadEnabled' -] -class VideoAuditView extends EntityAuditView { - constructor (video: VideoDetails) { - super(videoKeysToKeep, 'video', video) - } -} - -const videoImportKeysToKeep = [ - 'id', - 'targetUrl', - 'video-name' -] -class VideoImportAuditView extends EntityAuditView { - constructor (videoImport: VideoImport) { - super(videoImportKeysToKeep, 'video-import', videoImport) - } -} - -const commentKeysToKeep = [ - 'id', - 'text', - 'threadId', - 'inReplyToCommentId', - 'videoId', - 'createdAt', - 'updatedAt', - 'totalReplies', - 'account-id', - 'account-uuid', - 'account-name' -] -class CommentAuditView extends EntityAuditView { - constructor (comment: VideoComment) { - super(commentKeysToKeep, 'comment', comment) - } -} - -const userKeysToKeep = [ - 'id', - 'username', - 'email', - 'nsfwPolicy', - 'autoPlayVideo', - 'role', - 'videoQuota', - 'createdAt', - 'account-id', - 'account-uuid', - 'account-name', - 'account-followingCount', - 'account-followersCount', - 'account-createdAt', - 'account-updatedAt', - 'account-avatar-path', - 'account-avatar-createdAt', - 'account-avatar-updatedAt', - 'account-displayName', - 'account-description', - 'videoChannels' -] -class UserAuditView extends EntityAuditView { - constructor (user: User) { - super(userKeysToKeep, 'user', user) - } -} - -const channelKeysToKeep = [ - 'id', - 'uuid', - 'name', - 'followingCount', - 'followersCount', - 'createdAt', - 'updatedAt', - 'avatar-path', - 'avatar-createdAt', - 'avatar-updatedAt', - 'displayName', - 'description', - 'support', - 'isLocal', - 'ownerAccount-id', - 'ownerAccount-uuid', - 'ownerAccount-name', - 'ownerAccount-displayedName' -] -class VideoChannelAuditView extends EntityAuditView { - constructor (channel: VideoChannel) { - super(channelKeysToKeep, 'channel', channel) - } -} - -const abuseKeysToKeep = [ - 'id', - 'reason', - 'reporterAccount', - 'createdAt' -] -class AbuseAuditView extends EntityAuditView { - constructor (abuse: AdminAbuse) { - super(abuseKeysToKeep, 'abuse', abuse) - } -} - -const customConfigKeysToKeep = [ - 'instance-name', - 'instance-shortDescription', - 'instance-description', - 'instance-terms', - 'instance-defaultClientRoute', - 'instance-defaultNSFWPolicy', - 'instance-customizations-javascript', - 'instance-customizations-css', - 'services-twitter-username', - 'services-twitter-whitelisted', - 'cache-previews-size', - 'cache-captions-size', - 'signup-enabled', - 'signup-limit', - 'signup-requiresEmailVerification', - 'admin-email', - 'user-videoQuota', - 'transcoding-enabled', - 'transcoding-threads', - 'transcoding-resolutions' -] -class CustomConfigAuditView extends EntityAuditView { - constructor (customConfig: CustomConfig) { - const infos: any = customConfig - const resolutionsDict = infos.transcoding.resolutions - const resolutionsArray = [] - - Object.entries(resolutionsDict) - .forEach(([ resolution, isEnabled ]) => { - if (isEnabled) resolutionsArray.push(resolution) - }) - - Object.assign({}, infos, { transcoding: { resolutions: resolutionsArray } }) - super(customConfigKeysToKeep, 'config', infos) - } -} - -const channelSyncKeysToKeep = [ - 'id', - 'externalChannelUrl', - 'channel-id', - 'channel-name' -] -class VideoChannelSyncAuditView extends EntityAuditView { - constructor (channelSync: VideoChannelSync) { - super(channelSyncKeysToKeep, 'channelSync', channelSync) - } -} - -export { - getAuditIdFromRes, - - auditLoggerFactory, - VideoImportAuditView, - VideoChannelAuditView, - CommentAuditView, - UserAuditView, - VideoAuditView, - AbuseAuditView, - CustomConfigAuditView, - VideoChannelSyncAuditView -} diff --git a/server/helpers/captions-utils.ts b/server/helpers/captions-utils.ts deleted file mode 100644 index f6e5b9784..000000000 --- a/server/helpers/captions-utils.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { createReadStream, createWriteStream, move, remove } from 'fs-extra' -import { join } from 'path' -import srt2vtt from 'srt-to-vtt' -import { Transform } from 'stream' -import { MVideoCaption } from '@server/types/models' -import { CONFIG } from '../initializers/config' -import { pipelinePromise } from './core-utils' - -async function moveAndProcessCaptionFile (physicalFile: { filename: string, path: string }, videoCaption: MVideoCaption) { - const videoCaptionsDir = CONFIG.STORAGE.CAPTIONS_DIR - const destination = join(videoCaptionsDir, videoCaption.filename) - - // Convert this srt file to vtt - if (physicalFile.path.endsWith('.srt')) { - await convertSrtToVtt(physicalFile.path, destination) - await remove(physicalFile.path) - } else if (physicalFile.path !== destination) { // Just move the vtt file - await move(physicalFile.path, destination, { overwrite: true }) - } - - // This is important in case if there is another attempt in the retry process - physicalFile.filename = videoCaption.filename - physicalFile.path = destination -} - -// --------------------------------------------------------------------------- - -export { - moveAndProcessCaptionFile -} - -// --------------------------------------------------------------------------- - -function convertSrtToVtt (source: string, destination: string) { - const fixVTT = new Transform({ - transform: (chunk, _encoding, cb) => { - let block: string = chunk.toString() - - block = block.replace(/(\d\d:\d\d:\d\d)(\s)/g, '$1.000$2') - .replace(/(\d\d:\d\d:\d\d),(\d)(\s)/g, '$1.00$2$3') - .replace(/(\d\d:\d\d:\d\d),(\d\d)(\s)/g, '$1.0$2$3') - - return cb(undefined, block) - } - }) - - return pipelinePromise( - createReadStream(source), - srt2vtt(), - fixVTT, - createWriteStream(destination) - ) -} diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts deleted file mode 100644 index 242c49e89..000000000 --- a/server/helpers/core-utils.ts +++ /dev/null @@ -1,315 +0,0 @@ -/* eslint-disable no-useless-call */ - -/* - Different from 'utils' because we don't import other PeerTube modules. - Useful to avoid circular dependencies. -*/ - -import { exec, ExecOptions } from 'child_process' -import { ED25519KeyPairOptions, generateKeyPair, randomBytes, RSAKeyPairOptions, scrypt } from 'crypto' -import { truncate } from 'lodash' -import { pipeline } from 'stream' -import { URL } from 'url' -import { promisify } from 'util' -import { promisify1, promisify2, promisify3 } from '@shared/core-utils' - -const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => { - if (!oldObject || typeof oldObject !== 'object') { - return valueConverter(oldObject) - } - - if (Array.isArray(oldObject)) { - return oldObject.map(e => objectConverter(e, keyConverter, valueConverter)) - } - - const newObject = {} - Object.keys(oldObject).forEach(oldKey => { - const newKey = keyConverter(oldKey) - newObject[newKey] = objectConverter(oldObject[oldKey], keyConverter, valueConverter) - }) - - return newObject -} - -function mapToJSON (map: Map) { - const obj: any = {} - - for (const [ k, v ] of map) { - obj[k] = v - } - - return obj -} - -// --------------------------------------------------------------------------- - -const timeTable = { - ms: 1, - second: 1000, - minute: 60000, - hour: 3600000, - day: 3600000 * 24, - week: 3600000 * 24 * 7, - month: 3600000 * 24 * 30 -} - -export function parseDurationToMs (duration: number | string): number { - if (duration === null) return null - if (typeof duration === 'number') return duration - if (!isNaN(+duration)) return +duration - - if (typeof duration === 'string') { - const split = duration.match(/^([\d.,]+)\s?(\w+)$/) - - if (split.length === 3) { - const len = parseFloat(split[1]) - let unit = split[2].replace(/s$/i, '').toLowerCase() - if (unit === 'm') { - unit = 'ms' - } - - return (len || 1) * (timeTable[unit] || 0) - } - } - - throw new Error(`Duration ${duration} could not be properly parsed`) -} - -export function parseBytes (value: string | number): number { - if (typeof value === 'number') return value - if (!isNaN(+value)) return +value - - const tgm = /^(\d+)\s*TB\s*(\d+)\s*GB\s*(\d+)\s*MB$/ - const tg = /^(\d+)\s*TB\s*(\d+)\s*GB$/ - const tm = /^(\d+)\s*TB\s*(\d+)\s*MB$/ - const gm = /^(\d+)\s*GB\s*(\d+)\s*MB$/ - const t = /^(\d+)\s*TB$/ - const g = /^(\d+)\s*GB$/ - const m = /^(\d+)\s*MB$/ - const b = /^(\d+)\s*B$/ - - let match: RegExpMatchArray - - if (value.match(tgm)) { - match = value.match(tgm) - return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 + - parseInt(match[2], 10) * 1024 * 1024 * 1024 + - parseInt(match[3], 10) * 1024 * 1024 - } - - if (value.match(tg)) { - match = value.match(tg) - return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 + - parseInt(match[2], 10) * 1024 * 1024 * 1024 - } - - if (value.match(tm)) { - match = value.match(tm) - return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 + - parseInt(match[2], 10) * 1024 * 1024 - } - - if (value.match(gm)) { - match = value.match(gm) - return parseInt(match[1], 10) * 1024 * 1024 * 1024 + - parseInt(match[2], 10) * 1024 * 1024 - } - - if (value.match(t)) { - match = value.match(t) - return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 - } - - if (value.match(g)) { - match = value.match(g) - return parseInt(match[1], 10) * 1024 * 1024 * 1024 - } - - if (value.match(m)) { - match = value.match(m) - return parseInt(match[1], 10) * 1024 * 1024 - } - - if (value.match(b)) { - match = value.match(b) - return parseInt(match[1], 10) * 1024 - } - - return parseInt(value, 10) -} - -// --------------------------------------------------------------------------- - -function sanitizeUrl (url: string) { - const urlObject = new URL(url) - - if (urlObject.protocol === 'https:' && urlObject.port === '443') { - urlObject.port = '' - } else if (urlObject.protocol === 'http:' && urlObject.port === '80') { - urlObject.port = '' - } - - return urlObject.href.replace(/\/$/, '') -} - -// Don't import remote scheme from constants because we are in core utils -function sanitizeHost (host: string, remoteScheme: string) { - const toRemove = remoteScheme === 'https' ? 443 : 80 - - return host.replace(new RegExp(`:${toRemove}$`), '') -} - -// --------------------------------------------------------------------------- - -function isTestInstance () { - return process.env.NODE_ENV === 'test' -} - -function isDevInstance () { - return process.env.NODE_ENV === 'dev' -} - -function isTestOrDevInstance () { - return isTestInstance() || isDevInstance() -} - -function isProdInstance () { - return process.env.NODE_ENV === 'production' -} - -function getAppNumber () { - return process.env.NODE_APP_INSTANCE || '' -} - -// --------------------------------------------------------------------------- - -// Consistent with .length, lodash truncate function is not -function peertubeTruncate (str: string, options: { length: number, separator?: RegExp, omission?: string }) { - const truncatedStr = truncate(str, options) - - // The truncated string is okay, we can return it - if (truncatedStr.length <= options.length) return truncatedStr - - // Lodash takes into account all UTF characters, whereas String.prototype.length does not: some characters have a length of 2 - // We always use the .length so we need to truncate more if needed - options.length -= truncatedStr.length - options.length - return truncate(str, options) -} - -function pageToStartAndCount (page: number, itemsPerPage: number) { - const start = (page - 1) * itemsPerPage - - return { start, count: itemsPerPage } -} - -// --------------------------------------------------------------------------- - -type SemVersion = { major: number, minor: number, patch: number } -function parseSemVersion (s: string) { - const parsed = s.match(/^v?(\d+)\.(\d+)\.(\d+)$/i) - - return { - major: parseInt(parsed[1]), - minor: parseInt(parsed[2]), - patch: parseInt(parsed[3]) - } as SemVersion -} - -// --------------------------------------------------------------------------- - -function execShell (command: string, options?: ExecOptions) { - return new Promise<{ err?: Error, stdout: string, stderr: string }>((res, rej) => { - exec(command, options, (err, stdout, stderr) => { - // eslint-disable-next-line prefer-promise-reject-errors - if (err) return rej({ err, stdout, stderr }) - - return res({ stdout, stderr }) - }) - }) -} - -// --------------------------------------------------------------------------- - -function generateRSAKeyPairPromise (size: number) { - return new Promise<{ publicKey: string, privateKey: string }>((res, rej) => { - const options: RSAKeyPairOptions<'pem', 'pem'> = { - modulusLength: size, - publicKeyEncoding: { - type: 'spki', - format: 'pem' - }, - privateKeyEncoding: { - type: 'pkcs1', - format: 'pem' - } - } - - generateKeyPair('rsa', options, (err, publicKey, privateKey) => { - if (err) return rej(err) - - return res({ publicKey, privateKey }) - }) - }) -} - -function generateED25519KeyPairPromise () { - return new Promise<{ publicKey: string, privateKey: string }>((res, rej) => { - const options: ED25519KeyPairOptions<'pem', 'pem'> = { - publicKeyEncoding: { - type: 'spki', - format: 'pem' - }, - privateKeyEncoding: { - type: 'pkcs8', - format: 'pem' - } - } - - generateKeyPair('ed25519', options, (err, publicKey, privateKey) => { - if (err) return rej(err) - - return res({ publicKey, privateKey }) - }) - }) -} - -// --------------------------------------------------------------------------- - -const randomBytesPromise = promisify1(randomBytes) -const scryptPromise = promisify3(scrypt) -const execPromise2 = promisify2(exec) -const execPromise = promisify1(exec) -const pipelinePromise = promisify(pipeline) - -// --------------------------------------------------------------------------- - -export { - isTestInstance, - isTestOrDevInstance, - isProdInstance, - getAppNumber, - - objectConverter, - mapToJSON, - - sanitizeUrl, - sanitizeHost, - - execShell, - - pageToStartAndCount, - peertubeTruncate, - - scryptPromise, - - randomBytesPromise, - - generateRSAKeyPairPromise, - generateED25519KeyPairPromise, - - execPromise2, - execPromise, - pipelinePromise, - - parseSemVersion -} diff --git a/server/helpers/custom-jsonld-signature.ts b/server/helpers/custom-jsonld-signature.ts deleted file mode 100644 index 3c706e372..000000000 --- a/server/helpers/custom-jsonld-signature.ts +++ /dev/null @@ -1,91 +0,0 @@ -import AsyncLRU from 'async-lru' -import { logger } from './logger' - -import jsonld = require('jsonld') - -const CACHE = { - 'https://w3id.org/security/v1': { - '@context': { - id: '@id', - type: '@type', - - dc: 'http://purl.org/dc/terms/', - sec: 'https://w3id.org/security#', - xsd: 'http://www.w3.org/2001/XMLSchema#', - - EcdsaKoblitzSignature2016: 'sec:EcdsaKoblitzSignature2016', - Ed25519Signature2018: 'sec:Ed25519Signature2018', - EncryptedMessage: 'sec:EncryptedMessage', - GraphSignature2012: 'sec:GraphSignature2012', - LinkedDataSignature2015: 'sec:LinkedDataSignature2015', - LinkedDataSignature2016: 'sec:LinkedDataSignature2016', - CryptographicKey: 'sec:Key', - - authenticationTag: 'sec:authenticationTag', - canonicalizationAlgorithm: 'sec:canonicalizationAlgorithm', - cipherAlgorithm: 'sec:cipherAlgorithm', - cipherData: 'sec:cipherData', - cipherKey: 'sec:cipherKey', - created: { '@id': 'dc:created', '@type': 'xsd:dateTime' }, - creator: { '@id': 'dc:creator', '@type': '@id' }, - digestAlgorithm: 'sec:digestAlgorithm', - digestValue: 'sec:digestValue', - domain: 'sec:domain', - encryptionKey: 'sec:encryptionKey', - expiration: { '@id': 'sec:expiration', '@type': 'xsd:dateTime' }, - expires: { '@id': 'sec:expiration', '@type': 'xsd:dateTime' }, - initializationVector: 'sec:initializationVector', - iterationCount: 'sec:iterationCount', - nonce: 'sec:nonce', - normalizationAlgorithm: 'sec:normalizationAlgorithm', - owner: { '@id': 'sec:owner', '@type': '@id' }, - password: 'sec:password', - privateKey: { '@id': 'sec:privateKey', '@type': '@id' }, - privateKeyPem: 'sec:privateKeyPem', - publicKey: { '@id': 'sec:publicKey', '@type': '@id' }, - publicKeyBase58: 'sec:publicKeyBase58', - publicKeyPem: 'sec:publicKeyPem', - publicKeyWif: 'sec:publicKeyWif', - publicKeyService: { '@id': 'sec:publicKeyService', '@type': '@id' }, - revoked: { '@id': 'sec:revoked', '@type': 'xsd:dateTime' }, - salt: 'sec:salt', - signature: 'sec:signature', - signatureAlgorithm: 'sec:signingAlgorithm', - signatureValue: 'sec:signatureValue' - } - } -} - -const nodeDocumentLoader = jsonld.documentLoaders.node() - -const lru = new AsyncLRU({ - max: 10, - load: (url, cb) => { - if (CACHE[url] !== undefined) { - logger.debug('Using cache for JSON-LD %s.', url) - - return cb(null, { - contextUrl: null, - document: CACHE[url], - documentUrl: url - }) - } - - nodeDocumentLoader(url) - .then(value => cb(null, value)) - .catch(err => cb(err)) - } -}) - -/* eslint-disable no-import-assign */ -jsonld.documentLoader = (url) => { - return new Promise((res, rej) => { - lru.get(url, (err, value) => { - if (err) return rej(err) - - return res(value) - }) - }) -} - -export { jsonld } diff --git a/server/helpers/custom-validators/abuses.ts b/server/helpers/custom-validators/abuses.ts deleted file mode 100644 index 94719641a..000000000 --- a/server/helpers/custom-validators/abuses.ts +++ /dev/null @@ -1,68 +0,0 @@ -import validator from 'validator' -import { abusePredefinedReasonsMap } from '@shared/core-utils/abuse' -import { AbuseCreate, AbuseFilter, AbusePredefinedReasonsString, AbuseVideoIs } from '@shared/models' -import { ABUSE_STATES, CONSTRAINTS_FIELDS } from '../../initializers/constants' -import { exists, isArray } from './misc' - -const ABUSES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.ABUSES -const ABUSE_MESSAGES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.ABUSE_MESSAGES - -function isAbuseReasonValid (value: string) { - return exists(value) && validator.isLength(value, ABUSES_CONSTRAINTS_FIELDS.REASON) -} - -function isAbusePredefinedReasonValid (value: AbusePredefinedReasonsString) { - return exists(value) && value in abusePredefinedReasonsMap -} - -function isAbuseFilterValid (value: AbuseFilter) { - return value === 'video' || value === 'comment' || value === 'account' -} - -function areAbusePredefinedReasonsValid (value: AbusePredefinedReasonsString[]) { - return exists(value) && isArray(value) && value.every(v => v in abusePredefinedReasonsMap) -} - -function isAbuseTimestampValid (value: number) { - return value === null || (exists(value) && validator.isInt('' + value, { min: 0 })) -} - -function isAbuseTimestampCoherent (endAt: number, { req }) { - const startAt = (req.body as AbuseCreate).video.startAt - - return exists(startAt) && endAt > startAt -} - -function isAbuseModerationCommentValid (value: string) { - return exists(value) && validator.isLength(value, ABUSES_CONSTRAINTS_FIELDS.MODERATION_COMMENT) -} - -function isAbuseStateValid (value: string) { - return exists(value) && ABUSE_STATES[value] !== undefined -} - -function isAbuseVideoIsValid (value: AbuseVideoIs) { - return exists(value) && ( - value === 'deleted' || - value === 'blacklisted' - ) -} - -function isAbuseMessageValid (value: string) { - return exists(value) && validator.isLength(value, ABUSE_MESSAGES_CONSTRAINTS_FIELDS.MESSAGE) -} - -// --------------------------------------------------------------------------- - -export { - isAbuseReasonValid, - isAbuseFilterValid, - isAbusePredefinedReasonValid, - isAbuseMessageValid, - areAbusePredefinedReasonsValid, - isAbuseTimestampValid, - isAbuseTimestampCoherent, - isAbuseModerationCommentValid, - isAbuseStateValid, - isAbuseVideoIsValid -} diff --git a/server/helpers/custom-validators/accounts.ts b/server/helpers/custom-validators/accounts.ts deleted file mode 100644 index f676669ea..000000000 --- a/server/helpers/custom-validators/accounts.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { isUserDescriptionValid, isUserUsernameValid } from './users' -import { exists } from './misc' - -function isAccountNameValid (value: string) { - return isUserUsernameValid(value) -} - -function isAccountIdValid (value: string) { - return exists(value) -} - -function isAccountDescriptionValid (value: string) { - return isUserDescriptionValid(value) -} - -// --------------------------------------------------------------------------- - -export { - isAccountIdValid, - isAccountDescriptionValid, - isAccountNameValid -} diff --git a/server/helpers/custom-validators/activitypub/activity.ts b/server/helpers/custom-validators/activitypub/activity.ts deleted file mode 100644 index 90a918523..000000000 --- a/server/helpers/custom-validators/activitypub/activity.ts +++ /dev/null @@ -1,151 +0,0 @@ -import validator from 'validator' -import { Activity, ActivityType } from '../../../../shared/models/activitypub' -import { isAbuseReasonValid } from '../abuses' -import { exists } from '../misc' -import { sanitizeAndCheckActorObject } from './actor' -import { isCacheFileObjectValid } from './cache-file' -import { isActivityPubUrlValid, isBaseActivityValid, isObjectValid } from './misc' -import { isPlaylistObjectValid } from './playlist' -import { sanitizeAndCheckVideoCommentObject } from './video-comments' -import { sanitizeAndCheckVideoTorrentObject } from './videos' -import { isWatchActionObjectValid } from './watch-action' - -function isRootActivityValid (activity: any) { - return isCollection(activity) || isActivity(activity) -} - -function isCollection (activity: any) { - return (activity.type === 'Collection' || activity.type === 'OrderedCollection') && - validator.isInt(activity.totalItems, { min: 0 }) && - Array.isArray(activity.items) -} - -function isActivity (activity: any) { - return isActivityPubUrlValid(activity.id) && - exists(activity.actor) && - (isActivityPubUrlValid(activity.actor) || isActivityPubUrlValid(activity.actor.id)) -} - -const activityCheckers: { [ P in ActivityType ]: (activity: Activity) => boolean } = { - Create: isCreateActivityValid, - Update: isUpdateActivityValid, - Delete: isDeleteActivityValid, - Follow: isFollowActivityValid, - Accept: isAcceptActivityValid, - Reject: isRejectActivityValid, - Announce: isAnnounceActivityValid, - Undo: isUndoActivityValid, - Like: isLikeActivityValid, - View: isViewActivityValid, - Flag: isFlagActivityValid, - Dislike: isDislikeActivityValid -} - -function isActivityValid (activity: any) { - const checker = activityCheckers[activity.type] - // Unknown activity type - if (!checker) return false - - return checker(activity) -} - -function isFlagActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Flag') && - isAbuseReasonValid(activity.content) && - isActivityPubUrlValid(activity.object) -} - -function isLikeActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Like') && - isObjectValid(activity.object) -} - -function isDislikeActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Dislike') && - isObjectValid(activity.object) -} - -function isAnnounceActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Announce') && - isObjectValid(activity.object) -} - -function isViewActivityValid (activity: any) { - return isBaseActivityValid(activity, 'View') && - isActivityPubUrlValid(activity.actor) && - isActivityPubUrlValid(activity.object) -} - -function isCreateActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Create') && - ( - isViewActivityValid(activity.object) || - isDislikeActivityValid(activity.object) || - isFlagActivityValid(activity.object) || - isPlaylistObjectValid(activity.object) || - isWatchActionObjectValid(activity.object) || - - isCacheFileObjectValid(activity.object) || - sanitizeAndCheckVideoCommentObject(activity.object) || - sanitizeAndCheckVideoTorrentObject(activity.object) - ) -} - -function isUpdateActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Update') && - ( - isCacheFileObjectValid(activity.object) || - isPlaylistObjectValid(activity.object) || - sanitizeAndCheckVideoTorrentObject(activity.object) || - sanitizeAndCheckActorObject(activity.object) - ) -} - -function isDeleteActivityValid (activity: any) { - // We don't really check objects - return isBaseActivityValid(activity, 'Delete') && - isObjectValid(activity.object) -} - -function isFollowActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Follow') && - isObjectValid(activity.object) -} - -function isAcceptActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Accept') -} - -function isRejectActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Reject') -} - -function isUndoActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Undo') && - ( - isFollowActivityValid(activity.object) || - isLikeActivityValid(activity.object) || - isDislikeActivityValid(activity.object) || - isAnnounceActivityValid(activity.object) || - isCreateActivityValid(activity.object) - ) -} - -// --------------------------------------------------------------------------- - -export { - isRootActivityValid, - isActivityValid, - isFlagActivityValid, - isLikeActivityValid, - isDislikeActivityValid, - isAnnounceActivityValid, - isViewActivityValid, - isCreateActivityValid, - isUpdateActivityValid, - isDeleteActivityValid, - isFollowActivityValid, - isAcceptActivityValid, - isRejectActivityValid, - isUndoActivityValid -} diff --git a/server/helpers/custom-validators/activitypub/actor.ts b/server/helpers/custom-validators/activitypub/actor.ts deleted file mode 100644 index f43c35b23..000000000 --- a/server/helpers/custom-validators/activitypub/actor.ts +++ /dev/null @@ -1,142 +0,0 @@ -import validator from 'validator' -import { CONSTRAINTS_FIELDS } from '../../../initializers/constants' -import { exists, isArray, isDateValid } from '../misc' -import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc' -import { isHostValid } from '../servers' -import { peertubeTruncate } from '@server/helpers/core-utils' - -function isActorEndpointsObjectValid (endpointObject: any) { - if (endpointObject?.sharedInbox) { - return isActivityPubUrlValid(endpointObject.sharedInbox) - } - - // Shared inbox is optional - return true -} - -function isActorPublicKeyObjectValid (publicKeyObject: any) { - return isActivityPubUrlValid(publicKeyObject.id) && - isActivityPubUrlValid(publicKeyObject.owner) && - isActorPublicKeyValid(publicKeyObject.publicKeyPem) -} - -function isActorTypeValid (type: string) { - return type === 'Person' || type === 'Application' || type === 'Group' || type === 'Service' || type === 'Organization' -} - -function isActorPublicKeyValid (publicKey: string) { - return exists(publicKey) && - typeof publicKey === 'string' && - publicKey.startsWith('-----BEGIN PUBLIC KEY-----') && - publicKey.includes('-----END PUBLIC KEY-----') && - validator.isLength(publicKey, CONSTRAINTS_FIELDS.ACTORS.PUBLIC_KEY) -} - -const actorNameAlphabet = '[ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789\\-_.:]' -const actorNameRegExp = new RegExp(`^${actorNameAlphabet}+$`) -function isActorPreferredUsernameValid (preferredUsername: string) { - return exists(preferredUsername) && validator.matches(preferredUsername, actorNameRegExp) -} - -function isActorPrivateKeyValid (privateKey: string) { - return exists(privateKey) && - typeof privateKey === 'string' && - (privateKey.startsWith('-----BEGIN RSA PRIVATE KEY-----') || privateKey.startsWith('-----BEGIN PRIVATE KEY-----')) && - // Sometimes there is a \n at the end, so just assert the string contains the end mark - (privateKey.includes('-----END RSA PRIVATE KEY-----') || privateKey.includes('-----END PRIVATE KEY-----')) && - validator.isLength(privateKey, CONSTRAINTS_FIELDS.ACTORS.PRIVATE_KEY) -} - -function isActorFollowingCountValid (value: string) { - return exists(value) && validator.isInt('' + value, { min: 0 }) -} - -function isActorFollowersCountValid (value: string) { - return exists(value) && validator.isInt('' + value, { min: 0 }) -} - -function isActorDeleteActivityValid (activity: any) { - return isBaseActivityValid(activity, 'Delete') -} - -function sanitizeAndCheckActorObject (actor: any) { - if (!isActorTypeValid(actor.type)) return false - - normalizeActor(actor) - - return exists(actor) && - isActivityPubUrlValid(actor.id) && - isActivityPubUrlValid(actor.inbox) && - isActorPreferredUsernameValid(actor.preferredUsername) && - isActivityPubUrlValid(actor.url) && - isActorPublicKeyObjectValid(actor.publicKey) && - isActorEndpointsObjectValid(actor.endpoints) && - - (!actor.outbox || isActivityPubUrlValid(actor.outbox)) && - (!actor.following || isActivityPubUrlValid(actor.following)) && - (!actor.followers || isActivityPubUrlValid(actor.followers)) && - - setValidAttributedTo(actor) && - setValidDescription(actor) && - // If this is a group (a channel), it should be attributed to an account - // In PeerTube we use this to attach a video channel to a specific account - (actor.type !== 'Group' || actor.attributedTo.length !== 0) -} - -function normalizeActor (actor: any) { - if (!actor) return - - if (!actor.url) { - actor.url = actor.id - } else if (typeof actor.url !== 'string') { - actor.url = actor.url.href || actor.url.url - } - - if (!isDateValid(actor.published)) actor.published = undefined - - if (actor.summary && typeof actor.summary === 'string') { - actor.summary = peertubeTruncate(actor.summary, { length: CONSTRAINTS_FIELDS.USERS.DESCRIPTION.max }) - - if (actor.summary.length < CONSTRAINTS_FIELDS.USERS.DESCRIPTION.min) { - actor.summary = null - } - } -} - -function isValidActorHandle (handle: string) { - if (!exists(handle)) return false - - const parts = handle.split('@') - if (parts.length !== 2) return false - - return isHostValid(parts[1]) -} - -function areValidActorHandles (handles: string[]) { - return isArray(handles) && handles.every(h => isValidActorHandle(h)) -} - -function setValidDescription (obj: any) { - if (!obj.summary) obj.summary = null - - return true -} - -// --------------------------------------------------------------------------- - -export { - normalizeActor, - actorNameAlphabet, - areValidActorHandles, - isActorEndpointsObjectValid, - isActorPublicKeyObjectValid, - isActorTypeValid, - isActorPublicKeyValid, - isActorPreferredUsernameValid, - isActorPrivateKeyValid, - isActorFollowingCountValid, - isActorFollowersCountValid, - isActorDeleteActivityValid, - sanitizeAndCheckActorObject, - isValidActorHandle -} diff --git a/server/helpers/custom-validators/activitypub/cache-file.ts b/server/helpers/custom-validators/activitypub/cache-file.ts deleted file mode 100644 index c5b3b4d9f..000000000 --- a/server/helpers/custom-validators/activitypub/cache-file.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { isActivityPubUrlValid } from './misc' -import { isRemoteVideoUrlValid } from './videos' -import { exists, isDateValid } from '../misc' -import { CacheFileObject } from '../../../../shared/models/activitypub/objects' - -function isCacheFileObjectValid (object: CacheFileObject) { - return exists(object) && - object.type === 'CacheFile' && - (object.expires === null || isDateValid(object.expires)) && - isActivityPubUrlValid(object.object) && - (isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url)) -} - -// --------------------------------------------------------------------------- - -export { - isCacheFileObjectValid -} - -// --------------------------------------------------------------------------- - -function isPlaylistRedundancyUrlValid (url: any) { - return url.type === 'Link' && - (url.mediaType || url.mimeType) === 'application/x-mpegURL' && - isActivityPubUrlValid(url.href) -} diff --git a/server/helpers/custom-validators/activitypub/misc.ts b/server/helpers/custom-validators/activitypub/misc.ts deleted file mode 100644 index 7df47cf15..000000000 --- a/server/helpers/custom-validators/activitypub/misc.ts +++ /dev/null @@ -1,76 +0,0 @@ -import validator from 'validator' -import { CONFIG } from '@server/initializers/config' -import { CONSTRAINTS_FIELDS } from '../../../initializers/constants' -import { exists } from '../misc' - -function isUrlValid (url: string) { - const isURLOptions = { - require_host: true, - require_tld: true, - require_protocol: true, - require_valid_protocol: true, - protocols: [ 'http', 'https' ] - } - - // We validate 'localhost', so we don't have the top level domain - if (CONFIG.WEBSERVER.HOSTNAME === 'localhost' || CONFIG.WEBSERVER.HOSTNAME === '127.0.0.1') { - isURLOptions.require_tld = false - } - - return exists(url) && validator.isURL('' + url, isURLOptions) -} - -function isActivityPubUrlValid (url: string) { - return isUrlValid(url) && validator.isLength('' + url, CONSTRAINTS_FIELDS.ACTORS.URL) -} - -function isBaseActivityValid (activity: any, type: string) { - return activity.type === type && - isActivityPubUrlValid(activity.id) && - isObjectValid(activity.actor) && - isUrlCollectionValid(activity.to) && - isUrlCollectionValid(activity.cc) -} - -function isUrlCollectionValid (collection: any) { - return collection === undefined || - (Array.isArray(collection) && collection.every(t => isActivityPubUrlValid(t))) -} - -function isObjectValid (object: any) { - return exists(object) && - ( - isActivityPubUrlValid(object) || isActivityPubUrlValid(object.id) - ) -} - -function setValidAttributedTo (obj: any) { - if (Array.isArray(obj.attributedTo) === false) { - obj.attributedTo = [] - return true - } - - obj.attributedTo = obj.attributedTo.filter(a => { - return isActivityPubUrlValid(a) || - ((a.type === 'Group' || a.type === 'Person') && isActivityPubUrlValid(a.id)) - }) - - return true -} - -function isActivityPubVideoDurationValid (value: string) { - // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration - return exists(value) && - typeof value === 'string' && - value.startsWith('PT') && - value.endsWith('S') -} - -export { - isUrlValid, - isActivityPubUrlValid, - isBaseActivityValid, - setValidAttributedTo, - isObjectValid, - isActivityPubVideoDurationValid -} diff --git a/server/helpers/custom-validators/activitypub/playlist.ts b/server/helpers/custom-validators/activitypub/playlist.ts deleted file mode 100644 index 49bcadcfd..000000000 --- a/server/helpers/custom-validators/activitypub/playlist.ts +++ /dev/null @@ -1,29 +0,0 @@ -import validator from 'validator' -import { PlaylistElementObject, PlaylistObject } from '@shared/models' -import { exists, isDateValid, isUUIDValid } from '../misc' -import { isVideoPlaylistNameValid } from '../video-playlists' -import { isActivityPubUrlValid } from './misc' - -function isPlaylistObjectValid (object: PlaylistObject) { - return exists(object) && - object.type === 'Playlist' && - validator.isInt(object.totalItems + '') && - isVideoPlaylistNameValid(object.name) && - isUUIDValid(object.uuid) && - isDateValid(object.published) && - isDateValid(object.updated) -} - -function isPlaylistElementObjectValid (object: PlaylistElementObject) { - return exists(object) && - object.type === 'PlaylistElement' && - validator.isInt(object.position + '') && - isActivityPubUrlValid(object.url) -} - -// --------------------------------------------------------------------------- - -export { - isPlaylistObjectValid, - isPlaylistElementObjectValid -} diff --git a/server/helpers/custom-validators/activitypub/signature.ts b/server/helpers/custom-validators/activitypub/signature.ts deleted file mode 100644 index cfb65361e..000000000 --- a/server/helpers/custom-validators/activitypub/signature.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { exists } from '../misc' -import { isActivityPubUrlValid } from './misc' - -function isSignatureTypeValid (signatureType: string) { - return exists(signatureType) && signatureType === 'RsaSignature2017' -} - -function isSignatureCreatorValid (signatureCreator: string) { - return exists(signatureCreator) && isActivityPubUrlValid(signatureCreator) -} - -function isSignatureValueValid (signatureValue: string) { - return exists(signatureValue) && signatureValue.length > 0 -} - -// --------------------------------------------------------------------------- - -export { - isSignatureTypeValid, - isSignatureCreatorValid, - isSignatureValueValid -} diff --git a/server/helpers/custom-validators/activitypub/video-comments.ts b/server/helpers/custom-validators/activitypub/video-comments.ts deleted file mode 100644 index ea852c491..000000000 --- a/server/helpers/custom-validators/activitypub/video-comments.ts +++ /dev/null @@ -1,59 +0,0 @@ -import validator from 'validator' -import { ACTIVITY_PUB } from '../../../initializers/constants' -import { exists, isArray, isDateValid } from '../misc' -import { isActivityPubUrlValid } from './misc' - -function sanitizeAndCheckVideoCommentObject (comment: any) { - if (!comment) return false - - if (!isCommentTypeValid(comment)) return false - - normalizeComment(comment) - - if (comment.type === 'Tombstone') { - return isActivityPubUrlValid(comment.id) && - isDateValid(comment.published) && - isDateValid(comment.deleted) && - isActivityPubUrlValid(comment.url) - } - - return isActivityPubUrlValid(comment.id) && - isCommentContentValid(comment.content) && - isActivityPubUrlValid(comment.inReplyTo) && - isDateValid(comment.published) && - isActivityPubUrlValid(comment.url) && - isArray(comment.to) && - ( - comment.to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 || - comment.cc.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 - ) // Only accept public comments -} - -// --------------------------------------------------------------------------- - -export { - sanitizeAndCheckVideoCommentObject -} - -// --------------------------------------------------------------------------- - -function isCommentContentValid (content: any) { - return exists(content) && validator.isLength('' + content, { min: 1 }) -} - -function normalizeComment (comment: any) { - if (!comment) return - - if (typeof comment.url !== 'string') { - if (typeof comment.url === 'object') comment.url = comment.url.href || comment.url.url - else comment.url = comment.id - } -} - -function isCommentTypeValid (comment: any): boolean { - if (comment.type === 'Note') return true - - if (comment.type === 'Tombstone' && comment.formerType === 'Note') return true - - return false -} diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts deleted file mode 100644 index 07e25b8ba..000000000 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ /dev/null @@ -1,241 +0,0 @@ -import validator from 'validator' -import { logger } from '@server/helpers/logger' -import { ActivityPubStoryboard, ActivityTrackerUrlObject, ActivityVideoFileMetadataUrlObject, VideoObject } from '@shared/models' -import { LiveVideoLatencyMode, VideoState } from '../../../../shared/models/videos' -import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers/constants' -import { peertubeTruncate } from '../../core-utils' -import { isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc' -import { isLiveLatencyModeValid } from '../video-lives' -import { - isVideoDescriptionValid, - isVideoDurationValid, - isVideoNameValid, - isVideoStateValid, - isVideoTagValid, - isVideoViewsValid -} from '../videos' -import { isActivityPubUrlValid, isActivityPubVideoDurationValid, isBaseActivityValid, setValidAttributedTo } from './misc' - -function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) { - return isBaseActivityValid(activity, 'Update') && - sanitizeAndCheckVideoTorrentObject(activity.object) -} - -function sanitizeAndCheckVideoTorrentObject (video: any) { - if (!video || video.type !== 'Video') return false - - if (!setValidRemoteTags(video)) { - logger.debug('Video has invalid tags', { video }) - return false - } - if (!setValidRemoteVideoUrls(video)) { - logger.debug('Video has invalid urls', { video }) - return false - } - if (!setRemoteVideoContent(video)) { - logger.debug('Video has invalid content', { video }) - return false - } - if (!setValidAttributedTo(video)) { - logger.debug('Video has invalid attributedTo', { video }) - return false - } - if (!setValidRemoteCaptions(video)) { - logger.debug('Video has invalid captions', { video }) - return false - } - if (!setValidRemoteIcon(video)) { - logger.debug('Video has invalid icons', { video }) - return false - } - if (!setValidStoryboard(video)) { - logger.debug('Video has invalid preview (storyboard)', { video }) - return false - } - - // Default attributes - if (!isVideoStateValid(video.state)) video.state = VideoState.PUBLISHED - if (!isBooleanValid(video.waitTranscoding)) video.waitTranscoding = false - if (!isBooleanValid(video.downloadEnabled)) video.downloadEnabled = true - if (!isBooleanValid(video.commentsEnabled)) video.commentsEnabled = false - if (!isBooleanValid(video.isLiveBroadcast)) video.isLiveBroadcast = false - if (!isBooleanValid(video.liveSaveReplay)) video.liveSaveReplay = false - if (!isBooleanValid(video.permanentLive)) video.permanentLive = false - if (!isLiveLatencyModeValid(video.latencyMode)) video.latencyMode = LiveVideoLatencyMode.DEFAULT - - return isActivityPubUrlValid(video.id) && - isVideoNameValid(video.name) && - isActivityPubVideoDurationValid(video.duration) && - isVideoDurationValid(video.duration.replace(/[^0-9]+/g, '')) && - isUUIDValid(video.uuid) && - (!video.category || isRemoteNumberIdentifierValid(video.category)) && - (!video.licence || isRemoteNumberIdentifierValid(video.licence)) && - (!video.language || isRemoteStringIdentifierValid(video.language)) && - isVideoViewsValid(video.views) && - isBooleanValid(video.sensitive) && - isDateValid(video.published) && - isDateValid(video.updated) && - (!video.originallyPublishedAt || isDateValid(video.originallyPublishedAt)) && - (!video.uploadDate || isDateValid(video.uploadDate)) && - (!video.content || isRemoteVideoContentValid(video.mediaType, video.content)) && - video.attributedTo.length !== 0 -} - -function isRemoteVideoUrlValid (url: any) { - return url.type === 'Link' && - // Video file link - ( - ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.includes(url.mediaType) && - isActivityPubUrlValid(url.href) && - validator.isInt(url.height + '', { min: 0 }) && - validator.isInt(url.size + '', { min: 0 }) && - (!url.fps || validator.isInt(url.fps + '', { min: -1 })) - ) || - // Torrent link - ( - ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.includes(url.mediaType) && - isActivityPubUrlValid(url.href) && - validator.isInt(url.height + '', { min: 0 }) - ) || - // Magnet link - ( - ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.includes(url.mediaType) && - validator.isLength(url.href, { min: 5 }) && - validator.isInt(url.height + '', { min: 0 }) - ) || - // HLS playlist link - ( - (url.mediaType || url.mimeType) === 'application/x-mpegURL' && - isActivityPubUrlValid(url.href) && - isArray(url.tag) - ) || - isAPVideoTrackerUrlObject(url) || - isAPVideoFileUrlMetadataObject(url) -} - -function isAPVideoFileUrlMetadataObject (url: any): url is ActivityVideoFileMetadataUrlObject { - return url && - url.type === 'Link' && - url.mediaType === 'application/json' && - isArray(url.rel) && url.rel.includes('metadata') -} - -function isAPVideoTrackerUrlObject (url: any): url is ActivityTrackerUrlObject { - return isArray(url.rel) && - url.rel.includes('tracker') && - isActivityPubUrlValid(url.href) -} - -// --------------------------------------------------------------------------- - -export { - sanitizeAndCheckVideoTorrentUpdateActivity, - isRemoteStringIdentifierValid, - sanitizeAndCheckVideoTorrentObject, - isRemoteVideoUrlValid, - isAPVideoFileUrlMetadataObject, - isAPVideoTrackerUrlObject -} - -// --------------------------------------------------------------------------- - -function setValidRemoteTags (video: any) { - if (Array.isArray(video.tag) === false) return false - - video.tag = video.tag.filter(t => { - return t.type === 'Hashtag' && - isVideoTagValid(t.name) - }) - - return true -} - -function setValidRemoteCaptions (video: any) { - if (!video.subtitleLanguage) video.subtitleLanguage = [] - - if (Array.isArray(video.subtitleLanguage) === false) return false - - video.subtitleLanguage = video.subtitleLanguage.filter(caption => { - if (!isActivityPubUrlValid(caption.url)) caption.url = null - - return isRemoteStringIdentifierValid(caption) - }) - - return true -} - -function isRemoteNumberIdentifierValid (data: any) { - return validator.isInt(data.identifier, { min: 0 }) -} - -function isRemoteStringIdentifierValid (data: any) { - return typeof data.identifier === 'string' -} - -function isRemoteVideoContentValid (mediaType: string, content: string) { - return mediaType === 'text/markdown' && isVideoDescriptionValid(content) -} - -function setValidRemoteIcon (video: any) { - if (video.icon && !isArray(video.icon)) video.icon = [ video.icon ] - if (!video.icon) video.icon = [] - - video.icon = video.icon.filter(icon => { - return icon.type === 'Image' && - isActivityPubUrlValid(icon.url) && - icon.mediaType === 'image/jpeg' && - validator.isInt(icon.width + '', { min: 0 }) && - validator.isInt(icon.height + '', { min: 0 }) - }) - - return video.icon.length !== 0 -} - -function setValidRemoteVideoUrls (video: any) { - if (Array.isArray(video.url) === false) return false - - video.url = video.url.filter(u => isRemoteVideoUrlValid(u)) - - return true -} - -function setRemoteVideoContent (video: any) { - if (video.content) { - video.content = peertubeTruncate(video.content, { length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max }) - } - - return true -} - -function setValidStoryboard (video: VideoObject) { - if (!video.preview) return true - if (!Array.isArray(video.preview)) return false - - video.preview = video.preview.filter(p => isStorybordValid(p)) - - return true -} - -function isStorybordValid (preview: ActivityPubStoryboard) { - if (!preview) return false - - if ( - preview.type !== 'Image' || - !isArray(preview.rel) || - !preview.rel.includes('storyboard') - ) { - return false - } - - preview.url = preview.url.filter(u => { - return u.mediaType === 'image/jpeg' && - isActivityPubUrlValid(u.href) && - validator.isInt(u.width + '', { min: 0 }) && - validator.isInt(u.height + '', { min: 0 }) && - validator.isInt(u.tileWidth + '', { min: 0 }) && - validator.isInt(u.tileHeight + '', { min: 0 }) && - isActivityPubVideoDurationValid(u.tileDuration) - }) - - return preview.url.length !== 0 -} diff --git a/server/helpers/custom-validators/activitypub/watch-action.ts b/server/helpers/custom-validators/activitypub/watch-action.ts deleted file mode 100644 index b9ffa63f6..000000000 --- a/server/helpers/custom-validators/activitypub/watch-action.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { WatchActionObject } from '@shared/models' -import { exists, isDateValid, isUUIDValid } from '../misc' -import { isVideoTimeValid } from '../video-view' -import { isActivityPubVideoDurationValid, isObjectValid } from './misc' - -function isWatchActionObjectValid (action: WatchActionObject) { - return exists(action) && - action.type === 'WatchAction' && - isObjectValid(action.id) && - isActivityPubVideoDurationValid(action.duration) && - isDateValid(action.startTime) && - isDateValid(action.endTime) && - isLocationValid(action.location) && - isUUIDValid(action.uuid) && - isObjectValid(action.object) && - isWatchSectionsValid(action.watchSections) -} - -// --------------------------------------------------------------------------- - -export { - isWatchActionObjectValid -} - -// --------------------------------------------------------------------------- - -function isLocationValid (location: any) { - if (!location) return true - - return typeof location === 'object' && typeof location.addressCountry === 'string' -} - -function isWatchSectionsValid (sections: WatchActionObject['watchSections']) { - return Array.isArray(sections) && sections.every(s => { - return isVideoTimeValid(s.startTimestamp) && isVideoTimeValid(s.endTimestamp) - }) -} diff --git a/server/helpers/custom-validators/actor-images.ts b/server/helpers/custom-validators/actor-images.ts deleted file mode 100644 index 89f5a2262..000000000 --- a/server/helpers/custom-validators/actor-images.ts +++ /dev/null @@ -1,24 +0,0 @@ - -import { UploadFilesForCheck } from 'express' -import { CONSTRAINTS_FIELDS } from '../../initializers/constants' -import { isFileValid } from './misc' - -const imageMimeTypes = CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME - .map(v => v.replace('.', '')) - .join('|') -const imageMimeTypesRegex = `image/(${imageMimeTypes})` - -function isActorImageFile (files: UploadFilesForCheck, fieldname: string) { - return isFileValid({ - files, - mimeTypeRegex: imageMimeTypesRegex, - field: fieldname, - maxSize: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max - }) -} - -// --------------------------------------------------------------------------- - -export { - isActorImageFile -} diff --git a/server/helpers/custom-validators/feeds.ts b/server/helpers/custom-validators/feeds.ts deleted file mode 100644 index fa35a7da6..000000000 --- a/server/helpers/custom-validators/feeds.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { exists } from './misc' - -function isValidRSSFeed (value: string) { - if (!exists(value)) return false - - const feedExtensions = [ - 'xml', - 'json', - 'json1', - 'rss', - 'rss2', - 'atom', - 'atom1' - ] - - return feedExtensions.includes(value) -} - -// --------------------------------------------------------------------------- - -export { - isValidRSSFeed -} diff --git a/server/helpers/custom-validators/follows.ts b/server/helpers/custom-validators/follows.ts deleted file mode 100644 index 0bec683c1..000000000 --- a/server/helpers/custom-validators/follows.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { exists, isArray } from './misc' -import { FollowState } from '@shared/models' - -function isFollowStateValid (value: FollowState) { - if (!exists(value)) return false - - return value === 'pending' || value === 'accepted' || value === 'rejected' -} - -function isRemoteHandleValid (value: string) { - if (!exists(value)) return false - if (typeof value !== 'string') return false - - return value.includes('@') -} - -function isEachUniqueHandleValid (handles: string[]) { - return isArray(handles) && - handles.every(handle => { - return isRemoteHandleValid(handle) && handles.indexOf(handle) === handles.lastIndexOf(handle) - }) -} - -// --------------------------------------------------------------------------- - -export { - isFollowStateValid, - isRemoteHandleValid, - isEachUniqueHandleValid -} diff --git a/server/helpers/custom-validators/jobs.ts b/server/helpers/custom-validators/jobs.ts deleted file mode 100644 index c168b3e91..000000000 --- a/server/helpers/custom-validators/jobs.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { JobState } from '../../../shared/models' -import { exists } from './misc' -import { jobTypes } from '@server/lib/job-queue/job-queue' - -const jobStates: JobState[] = [ 'active', 'completed', 'failed', 'waiting', 'delayed', 'paused', 'waiting-children' ] - -function isValidJobState (value: JobState) { - return exists(value) && jobStates.includes(value) -} - -function isValidJobType (value: any) { - return exists(value) && jobTypes.includes(value) -} - -// --------------------------------------------------------------------------- - -export { - jobStates, - isValidJobState, - isValidJobType -} diff --git a/server/helpers/custom-validators/logs.ts b/server/helpers/custom-validators/logs.ts deleted file mode 100644 index 215dbb0e1..000000000 --- a/server/helpers/custom-validators/logs.ts +++ /dev/null @@ -1,42 +0,0 @@ -import validator from 'validator' -import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' -import { ClientLogLevel, ServerLogLevel } from '@shared/models' -import { exists } from './misc' - -const serverLogLevels = new Set([ 'debug', 'info', 'warn', 'error' ]) -const clientLogLevels = new Set([ 'warn', 'error' ]) - -function isValidLogLevel (value: any) { - return exists(value) && serverLogLevels.has(value) -} - -function isValidClientLogMessage (value: any) { - return typeof value === 'string' && validator.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_MESSAGE) -} - -function isValidClientLogLevel (value: any) { - return exists(value) && clientLogLevels.has(value) -} - -function isValidClientLogStackTrace (value: any) { - return typeof value === 'string' && validator.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_STACK_TRACE) -} - -function isValidClientLogMeta (value: any) { - return typeof value === 'string' && validator.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_META) -} - -function isValidClientLogUserAgent (value: any) { - return typeof value === 'string' && validator.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_USER_AGENT) -} - -// --------------------------------------------------------------------------- - -export { - isValidLogLevel, - isValidClientLogMessage, - isValidClientLogStackTrace, - isValidClientLogMeta, - isValidClientLogLevel, - isValidClientLogUserAgent -} diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts deleted file mode 100644 index 937ae0632..000000000 --- a/server/helpers/custom-validators/misc.ts +++ /dev/null @@ -1,190 +0,0 @@ -import 'multer' -import { UploadFilesForCheck } from 'express' -import { sep } from 'path' -import validator from 'validator' -import { isShortUUID, shortToUUID } from '@shared/extra-utils' - -function exists (value: any) { - return value !== undefined && value !== null -} - -function isSafePath (p: string) { - return exists(p) && - (p + '').split(sep).every(part => { - return [ '..' ].includes(part) === false - }) -} - -function isSafeFilename (filename: string, extension?: string) { - const regex = extension - ? new RegExp(`^[a-z0-9-]+\\.${extension}$`) - : new RegExp(`^[a-z0-9-]+\\.[a-z0-9]{1,8}$`) - - return typeof filename === 'string' && !!filename.match(regex) -} - -function isSafePeerTubeFilenameWithoutExtension (filename: string) { - return filename.match(/^[a-z0-9-]+$/) -} - -function isArray (value: any): value is any[] { - return Array.isArray(value) -} - -function isNotEmptyIntArray (value: any) { - return Array.isArray(value) && value.every(v => validator.isInt('' + v)) && value.length !== 0 -} - -function isNotEmptyStringArray (value: any) { - return Array.isArray(value) && value.every(v => typeof v === 'string' && v.length !== 0) && value.length !== 0 -} - -function isArrayOf (value: any, validator: (value: any) => boolean) { - return isArray(value) && value.every(v => validator(v)) -} - -function isDateValid (value: string) { - return exists(value) && validator.isISO8601(value) -} - -function isIdValid (value: string) { - return exists(value) && validator.isInt('' + value) -} - -function isUUIDValid (value: string) { - return exists(value) && validator.isUUID('' + value, 4) -} - -function areUUIDsValid (values: string[]) { - return isArray(values) && values.every(v => isUUIDValid(v)) -} - -function isIdOrUUIDValid (value: string) { - return isIdValid(value) || isUUIDValid(value) -} - -function isBooleanValid (value: any) { - return typeof value === 'boolean' || (typeof value === 'string' && validator.isBoolean(value)) -} - -function isIntOrNull (value: any) { - return value === null || validator.isInt('' + value) -} - -// --------------------------------------------------------------------------- - -function isFileValid (options: { - files: UploadFilesForCheck - - maxSize: number | null - mimeTypeRegex: string | null - - field?: string - - optional?: boolean // Default false -}) { - const { files, mimeTypeRegex, field, maxSize, optional = false } = options - - // Should have files - if (!files) return optional - - const fileArray = isArray(files) - ? files - : files[field] - - if (!fileArray || !isArray(fileArray) || fileArray.length === 0) { - return optional - } - - // The file exists - const file = fileArray[0] - if (!file?.originalname) return false - - // Check size - if ((maxSize !== null) && file.size > maxSize) return false - - if (mimeTypeRegex === null) return true - - return checkMimetypeRegex(file.mimetype, mimeTypeRegex) -} - -function checkMimetypeRegex (fileMimeType: string, mimeTypeRegex: string) { - return new RegExp(`^${mimeTypeRegex}$`, 'i').test(fileMimeType) -} - -// --------------------------------------------------------------------------- - -function toCompleteUUID (value: string) { - if (isShortUUID(value)) { - try { - return shortToUUID(value) - } catch { - return '' - } - } - - return value -} - -function toCompleteUUIDs (values: string[]) { - return values.map(v => toCompleteUUID(v)) -} - -function toIntOrNull (value: string) { - const v = toValueOrNull(value) - - if (v === null || v === undefined) return v - if (typeof v === 'number') return v - - return validator.toInt('' + v) -} - -function toBooleanOrNull (value: any) { - const v = toValueOrNull(value) - - if (v === null || v === undefined) return v - if (typeof v === 'boolean') return v - - return validator.toBoolean('' + v) -} - -function toValueOrNull (value: string) { - if (value === 'null') return null - - return value -} - -function toIntArray (value: any) { - if (!value) return [] - if (isArray(value) === false) return [ validator.toInt(value) ] - - return value.map(v => validator.toInt(v)) -} - -// --------------------------------------------------------------------------- - -export { - exists, - isArrayOf, - isNotEmptyIntArray, - isArray, - isIntOrNull, - isIdValid, - isSafePath, - isNotEmptyStringArray, - isUUIDValid, - toCompleteUUIDs, - toCompleteUUID, - isIdOrUUIDValid, - isDateValid, - toValueOrNull, - toBooleanOrNull, - isBooleanValid, - toIntOrNull, - areUUIDsValid, - toIntArray, - isFileValid, - isSafePeerTubeFilenameWithoutExtension, - isSafeFilename, - checkMimetypeRegex -} diff --git a/server/helpers/custom-validators/plugins.ts b/server/helpers/custom-validators/plugins.ts deleted file mode 100644 index a20de0c4a..000000000 --- a/server/helpers/custom-validators/plugins.ts +++ /dev/null @@ -1,178 +0,0 @@ -import validator from 'validator' -import { PluginPackageJSON } from '../../../shared/models/plugins/plugin-package-json.model' -import { PluginType } from '../../../shared/models/plugins/plugin.type' -import { CONSTRAINTS_FIELDS } from '../../initializers/constants' -import { isUrlValid } from './activitypub/misc' -import { exists, isArray, isSafePath } from './misc' - -const PLUGINS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.PLUGINS - -function isPluginTypeValid (value: any) { - return exists(value) && - (value === PluginType.PLUGIN || value === PluginType.THEME) -} - -function isPluginNameValid (value: string) { - return exists(value) && - validator.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.NAME) && - validator.matches(value, /^[a-z-0-9]+$/) -} - -function isNpmPluginNameValid (value: string) { - return exists(value) && - validator.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.NAME) && - validator.matches(value, /^[a-z\-._0-9]+$/) && - (value.startsWith('peertube-plugin-') || value.startsWith('peertube-theme-')) -} - -function isPluginDescriptionValid (value: string) { - return exists(value) && validator.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.DESCRIPTION) -} - -function isPluginStableVersionValid (value: string) { - if (!exists(value)) return false - - const parts = (value + '').split('.') - - return parts.length === 3 && parts.every(p => validator.isInt(p)) -} - -function isPluginStableOrUnstableVersionValid (value: string) { - if (!exists(value)) return false - - // suffix is beta.x or alpha.x - const [ stable, suffix ] = value.split('-') - if (!isPluginStableVersionValid(stable)) return false - - const suffixRegex = /^(rc|alpha|beta)\.\d+$/ - if (suffix && !suffixRegex.test(suffix)) return false - - return true -} - -function isPluginEngineValid (engine: any) { - return exists(engine) && exists(engine.peertube) -} - -function isPluginHomepage (value: string) { - return exists(value) && (!value || isUrlValid(value)) -} - -function isPluginBugs (value: string) { - return exists(value) && (!value || isUrlValid(value)) -} - -function areStaticDirectoriesValid (staticDirs: any) { - if (!exists(staticDirs) || typeof staticDirs !== 'object') return false - - for (const key of Object.keys(staticDirs)) { - if (!isSafePath(staticDirs[key])) return false - } - - return true -} - -function areClientScriptsValid (clientScripts: any[]) { - return isArray(clientScripts) && - clientScripts.every(c => { - return isSafePath(c.script) && isArray(c.scopes) - }) -} - -function areTranslationPathsValid (translations: any) { - if (!exists(translations) || typeof translations !== 'object') return false - - for (const key of Object.keys(translations)) { - if (!isSafePath(translations[key])) return false - } - - return true -} - -function areCSSPathsValid (css: any[]) { - return isArray(css) && css.every(c => isSafePath(c)) -} - -function isThemeNameValid (name: string) { - return isPluginNameValid(name) -} - -function isPackageJSONValid (packageJSON: PluginPackageJSON, pluginType: PluginType) { - let result = true - const badFields: string[] = [] - - if (!isNpmPluginNameValid(packageJSON.name)) { - result = false - badFields.push('name') - } - - if (!isPluginDescriptionValid(packageJSON.description)) { - result = false - badFields.push('description') - } - - if (!isPluginEngineValid(packageJSON.engine)) { - result = false - badFields.push('engine') - } - - if (!isPluginHomepage(packageJSON.homepage)) { - result = false - badFields.push('homepage') - } - - if (!exists(packageJSON.author)) { - result = false - badFields.push('author') - } - - if (!isPluginBugs(packageJSON.bugs)) { - result = false - badFields.push('bugs') - } - - if (pluginType === PluginType.PLUGIN && !isSafePath(packageJSON.library)) { - result = false - badFields.push('library') - } - - if (!areStaticDirectoriesValid(packageJSON.staticDirs)) { - result = false - badFields.push('staticDirs') - } - - if (!areCSSPathsValid(packageJSON.css)) { - result = false - badFields.push('css') - } - - if (!areClientScriptsValid(packageJSON.clientScripts)) { - result = false - badFields.push('clientScripts') - } - - if (!areTranslationPathsValid(packageJSON.translations)) { - result = false - badFields.push('translations') - } - - return { result, badFields } -} - -function isLibraryCodeValid (library: any) { - return typeof library.register === 'function' && - typeof library.unregister === 'function' -} - -export { - isPluginTypeValid, - isPackageJSONValid, - isThemeNameValid, - isPluginHomepage, - isPluginStableVersionValid, - isPluginStableOrUnstableVersionValid, - isPluginNameValid, - isPluginDescriptionValid, - isLibraryCodeValid, - isNpmPluginNameValid -} diff --git a/server/helpers/custom-validators/runners/jobs.ts b/server/helpers/custom-validators/runners/jobs.ts deleted file mode 100644 index 6349e79ba..000000000 --- a/server/helpers/custom-validators/runners/jobs.ts +++ /dev/null @@ -1,197 +0,0 @@ -import { UploadFilesForCheck } from 'express' -import validator from 'validator' -import { CONSTRAINTS_FIELDS, RUNNER_JOB_STATES } from '@server/initializers/constants' -import { - LiveRTMPHLSTranscodingSuccess, - RunnerJobSuccessPayload, - RunnerJobType, - RunnerJobUpdatePayload, - VideoStudioTranscodingSuccess, - VODAudioMergeTranscodingSuccess, - VODHLSTranscodingSuccess, - VODWebVideoTranscodingSuccess -} from '@shared/models' -import { exists, isArray, isFileValid, isSafeFilename } from '../misc' - -const RUNNER_JOBS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNER_JOBS - -const runnerJobTypes = new Set([ 'vod-hls-transcoding', 'vod-web-video-transcoding', 'vod-audio-merge-transcoding' ]) -function isRunnerJobTypeValid (value: RunnerJobType) { - return runnerJobTypes.has(value) -} - -function isRunnerJobSuccessPayloadValid (value: RunnerJobSuccessPayload, type: RunnerJobType, files: UploadFilesForCheck) { - return isRunnerJobVODWebVideoResultPayloadValid(value as VODWebVideoTranscodingSuccess, type, files) || - isRunnerJobVODHLSResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) || - isRunnerJobVODAudioMergeResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) || - isRunnerJobLiveRTMPHLSResultPayloadValid(value as LiveRTMPHLSTranscodingSuccess, type) || - isRunnerJobVideoStudioResultPayloadValid(value as VideoStudioTranscodingSuccess, type, files) -} - -// --------------------------------------------------------------------------- - -function isRunnerJobProgressValid (value: string) { - return validator.isInt(value + '', RUNNER_JOBS_CONSTRAINTS_FIELDS.PROGRESS) -} - -function isRunnerJobUpdatePayloadValid (value: RunnerJobUpdatePayload, type: RunnerJobType, files: UploadFilesForCheck) { - return isRunnerJobVODWebVideoUpdatePayloadValid(value, type, files) || - isRunnerJobVODHLSUpdatePayloadValid(value, type, files) || - isRunnerJobVideoStudioUpdatePayloadValid(value, type, files) || - isRunnerJobVODAudioMergeUpdatePayloadValid(value, type, files) || - isRunnerJobLiveRTMPHLSUpdatePayloadValid(value, type, files) -} - -// --------------------------------------------------------------------------- - -function isRunnerJobTokenValid (value: string) { - return exists(value) && validator.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.TOKEN) -} - -function isRunnerJobAbortReasonValid (value: string) { - return validator.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.REASON) -} - -function isRunnerJobErrorMessageValid (value: string) { - return validator.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.ERROR_MESSAGE) -} - -function isRunnerJobStateValid (value: any) { - return exists(value) && RUNNER_JOB_STATES[value] !== undefined -} - -function isRunnerJobArrayOfStateValid (value: any) { - return isArray(value) && value.every(v => isRunnerJobStateValid(v)) -} - -// --------------------------------------------------------------------------- - -export { - isRunnerJobTypeValid, - isRunnerJobSuccessPayloadValid, - isRunnerJobUpdatePayloadValid, - isRunnerJobTokenValid, - isRunnerJobErrorMessageValid, - isRunnerJobProgressValid, - isRunnerJobAbortReasonValid, - isRunnerJobArrayOfStateValid, - isRunnerJobStateValid -} - -// --------------------------------------------------------------------------- - -function isRunnerJobVODWebVideoResultPayloadValid ( - _value: VODWebVideoTranscodingSuccess, - type: RunnerJobType, - files: UploadFilesForCheck -) { - return type === 'vod-web-video-transcoding' && - isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) -} - -function isRunnerJobVODHLSResultPayloadValid ( - _value: VODHLSTranscodingSuccess, - type: RunnerJobType, - files: UploadFilesForCheck -) { - return type === 'vod-hls-transcoding' && - isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) && - isFileValid({ files, field: 'payload[resolutionPlaylistFile]', mimeTypeRegex: null, maxSize: null }) -} - -function isRunnerJobVODAudioMergeResultPayloadValid ( - _value: VODAudioMergeTranscodingSuccess, - type: RunnerJobType, - files: UploadFilesForCheck -) { - return type === 'vod-audio-merge-transcoding' && - isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) -} - -function isRunnerJobLiveRTMPHLSResultPayloadValid ( - value: LiveRTMPHLSTranscodingSuccess, - type: RunnerJobType -) { - return type === 'live-rtmp-hls-transcoding' && (!value || (typeof value === 'object' && Object.keys(value).length === 0)) -} - -function isRunnerJobVideoStudioResultPayloadValid ( - _value: VideoStudioTranscodingSuccess, - type: RunnerJobType, - files: UploadFilesForCheck -) { - return type === 'video-studio-transcoding' && - isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) -} - -// --------------------------------------------------------------------------- - -function isRunnerJobVODWebVideoUpdatePayloadValid ( - value: RunnerJobUpdatePayload, - type: RunnerJobType, - _files: UploadFilesForCheck -) { - return type === 'vod-web-video-transcoding' && - (!value || (typeof value === 'object' && Object.keys(value).length === 0)) -} - -function isRunnerJobVODHLSUpdatePayloadValid ( - value: RunnerJobUpdatePayload, - type: RunnerJobType, - _files: UploadFilesForCheck -) { - return type === 'vod-hls-transcoding' && - (!value || (typeof value === 'object' && Object.keys(value).length === 0)) -} - -function isRunnerJobVODAudioMergeUpdatePayloadValid ( - value: RunnerJobUpdatePayload, - type: RunnerJobType, - _files: UploadFilesForCheck -) { - return type === 'vod-audio-merge-transcoding' && - (!value || (typeof value === 'object' && Object.keys(value).length === 0)) -} - -function isRunnerJobLiveRTMPHLSUpdatePayloadValid ( - value: RunnerJobUpdatePayload, - type: RunnerJobType, - files: UploadFilesForCheck -) { - let result = type === 'live-rtmp-hls-transcoding' && !!value && !!files - - result &&= isFileValid({ files, field: 'payload[masterPlaylistFile]', mimeTypeRegex: null, maxSize: null, optional: true }) - - result &&= isFileValid({ - files, - field: 'payload[resolutionPlaylistFile]', - mimeTypeRegex: null, - maxSize: null, - optional: !value.resolutionPlaylistFilename - }) - - if (files['payload[resolutionPlaylistFile]']) { - result &&= isSafeFilename(value.resolutionPlaylistFilename, 'm3u8') - } - - return result && - isSafeFilename(value.videoChunkFilename, 'ts') && - ( - ( - value.type === 'remove-chunk' - ) || - ( - value.type === 'add-chunk' && - isFileValid({ files, field: 'payload[videoChunkFile]', mimeTypeRegex: null, maxSize: null }) - ) - ) -} - -function isRunnerJobVideoStudioUpdatePayloadValid ( - value: RunnerJobUpdatePayload, - type: RunnerJobType, - _files: UploadFilesForCheck -) { - return type === 'video-studio-transcoding' && - (!value || (typeof value === 'object' && Object.keys(value).length === 0)) -} diff --git a/server/helpers/custom-validators/runners/runners.ts b/server/helpers/custom-validators/runners/runners.ts deleted file mode 100644 index 953fac3b5..000000000 --- a/server/helpers/custom-validators/runners/runners.ts +++ /dev/null @@ -1,30 +0,0 @@ -import validator from 'validator' -import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' -import { exists } from '../misc' - -const RUNNERS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNERS - -function isRunnerRegistrationTokenValid (value: string) { - return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.TOKEN) -} - -function isRunnerTokenValid (value: string) { - return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.TOKEN) -} - -function isRunnerNameValid (value: string) { - return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.NAME) -} - -function isRunnerDescriptionValid (value: string) { - return exists(value) && validator.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.DESCRIPTION) -} - -// --------------------------------------------------------------------------- - -export { - isRunnerRegistrationTokenValid, - isRunnerTokenValid, - isRunnerNameValid, - isRunnerDescriptionValid -} diff --git a/server/helpers/custom-validators/search.ts b/server/helpers/custom-validators/search.ts deleted file mode 100644 index 6dba5d14e..000000000 --- a/server/helpers/custom-validators/search.ts +++ /dev/null @@ -1,37 +0,0 @@ -import validator from 'validator' -import { SearchTargetType } from '@shared/models/search/search-target-query.model' -import { isArray, exists } from './misc' -import { CONFIG } from '@server/initializers/config' - -function isNumberArray (value: any) { - return isArray(value) && value.every(v => validator.isInt('' + v)) -} - -function isStringArray (value: any) { - return isArray(value) && value.every(v => typeof v === 'string') -} - -function isBooleanBothQueryValid (value: any) { - return value === 'true' || value === 'false' || value === 'both' -} - -function isSearchTargetValid (value: SearchTargetType) { - if (!exists(value)) return true - - const searchIndexConfig = CONFIG.SEARCH.SEARCH_INDEX - - if (value === 'local') return true - - if (value === 'search-index' && searchIndexConfig.ENABLED) return true - - return false -} - -// --------------------------------------------------------------------------- - -export { - isNumberArray, - isStringArray, - isBooleanBothQueryValid, - isSearchTargetValid -} diff --git a/server/helpers/custom-validators/servers.ts b/server/helpers/custom-validators/servers.ts deleted file mode 100644 index b2aa03b77..000000000 --- a/server/helpers/custom-validators/servers.ts +++ /dev/null @@ -1,42 +0,0 @@ -import validator from 'validator' -import { CONFIG } from '@server/initializers/config' -import { CONSTRAINTS_FIELDS } from '../../initializers/constants' -import { exists, isArray } from './misc' - -function isHostValid (host: string) { - const isURLOptions = { - require_host: true, - require_tld: true - } - - // We validate 'localhost', so we don't have the top level domain - if (CONFIG.WEBSERVER.HOSTNAME === 'localhost' || CONFIG.WEBSERVER.HOSTNAME === '127.0.0.1') { - isURLOptions.require_tld = false - } - - return exists(host) && validator.isURL(host, isURLOptions) && host.split('://').length === 1 -} - -function isEachUniqueHostValid (hosts: string[]) { - return isArray(hosts) && - hosts.every(host => { - return isHostValid(host) && hosts.indexOf(host) === hosts.lastIndexOf(host) - }) -} - -function isValidContactBody (value: any) { - return exists(value) && validator.isLength(value, CONSTRAINTS_FIELDS.CONTACT_FORM.BODY) -} - -function isValidContactFromName (value: any) { - return exists(value) && validator.isLength(value, CONSTRAINTS_FIELDS.CONTACT_FORM.FROM_NAME) -} - -// --------------------------------------------------------------------------- - -export { - isValidContactBody, - isValidContactFromName, - isEachUniqueHostValid, - isHostValid -} diff --git a/server/helpers/custom-validators/user-notifications.ts b/server/helpers/custom-validators/user-notifications.ts deleted file mode 100644 index 2de13ca09..000000000 --- a/server/helpers/custom-validators/user-notifications.ts +++ /dev/null @@ -1,23 +0,0 @@ -import validator from 'validator' -import { UserNotificationSettingValue } from '@shared/models' -import { exists } from './misc' - -function isUserNotificationTypeValid (value: any) { - return exists(value) && validator.isInt('' + value) -} - -function isUserNotificationSettingValid (value: any) { - return exists(value) && - validator.isInt('' + value) && - ( - value === UserNotificationSettingValue.NONE || - value === UserNotificationSettingValue.WEB || - value === UserNotificationSettingValue.EMAIL || - value === (UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL) - ) -} - -export { - isUserNotificationSettingValid, - isUserNotificationTypeValid -} diff --git a/server/helpers/custom-validators/user-registration.ts b/server/helpers/custom-validators/user-registration.ts deleted file mode 100644 index 9da0bb08a..000000000 --- a/server/helpers/custom-validators/user-registration.ts +++ /dev/null @@ -1,25 +0,0 @@ -import validator from 'validator' -import { CONSTRAINTS_FIELDS, USER_REGISTRATION_STATES } from '../../initializers/constants' -import { exists } from './misc' - -const USER_REGISTRATIONS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.USER_REGISTRATIONS - -function isRegistrationStateValid (value: string) { - return exists(value) && USER_REGISTRATION_STATES[value] !== undefined -} - -function isRegistrationModerationResponseValid (value: string) { - return exists(value) && validator.isLength(value, USER_REGISTRATIONS_CONSTRAINTS_FIELDS.MODERATOR_MESSAGE) -} - -function isRegistrationReasonValid (value: string) { - return exists(value) && validator.isLength(value, USER_REGISTRATIONS_CONSTRAINTS_FIELDS.REASON_MESSAGE) -} - -// --------------------------------------------------------------------------- - -export { - isRegistrationStateValid, - isRegistrationModerationResponseValid, - isRegistrationReasonValid -} diff --git a/server/helpers/custom-validators/users.ts b/server/helpers/custom-validators/users.ts deleted file mode 100644 index f02b3ba65..000000000 --- a/server/helpers/custom-validators/users.ts +++ /dev/null @@ -1,123 +0,0 @@ -import validator from 'validator' -import { UserRole } from '@shared/models' -import { isEmailEnabled } from '../../initializers/config' -import { CONSTRAINTS_FIELDS, NSFW_POLICY_TYPES } from '../../initializers/constants' -import { exists, isArray, isBooleanValid } from './misc' - -const USERS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.USERS - -function isUserPasswordValid (value: string) { - return validator.isLength(value, USERS_CONSTRAINTS_FIELDS.PASSWORD) -} - -function isUserPasswordValidOrEmpty (value: string) { - // Empty password is only possible if emailing is enabled. - if (value === '') return isEmailEnabled() - - return isUserPasswordValid(value) -} - -function isUserVideoQuotaValid (value: string) { - return exists(value) && validator.isInt(value + '', USERS_CONSTRAINTS_FIELDS.VIDEO_QUOTA) -} - -function isUserVideoQuotaDailyValid (value: string) { - return exists(value) && validator.isInt(value + '', USERS_CONSTRAINTS_FIELDS.VIDEO_QUOTA_DAILY) -} - -function isUserUsernameValid (value: string) { - return exists(value) && - validator.matches(value, new RegExp(`^[a-z0-9_]+([a-z0-9_.-]+[a-z0-9_]+)?$`)) && - validator.isLength(value, USERS_CONSTRAINTS_FIELDS.USERNAME) -} - -function isUserDisplayNameValid (value: string) { - return value === null || (exists(value) && validator.isLength(value, CONSTRAINTS_FIELDS.USERS.NAME)) -} - -function isUserDescriptionValid (value: string) { - return value === null || (exists(value) && validator.isLength(value, CONSTRAINTS_FIELDS.USERS.DESCRIPTION)) -} - -function isUserEmailVerifiedValid (value: any) { - return isBooleanValid(value) -} - -const nsfwPolicies = new Set(Object.values(NSFW_POLICY_TYPES)) -function isUserNSFWPolicyValid (value: any) { - return exists(value) && nsfwPolicies.has(value) -} - -function isUserP2PEnabledValid (value: any) { - return isBooleanValid(value) -} - -function isUserVideosHistoryEnabledValid (value: any) { - return isBooleanValid(value) -} - -function isUserAutoPlayVideoValid (value: any) { - return isBooleanValid(value) -} - -function isUserVideoLanguages (value: any) { - return value === null || (isArray(value) && value.length < CONSTRAINTS_FIELDS.USERS.VIDEO_LANGUAGES.max) -} - -function isUserAdminFlagsValid (value: any) { - return exists(value) && validator.isInt('' + value) -} - -function isUserBlockedValid (value: any) { - return isBooleanValid(value) -} - -function isUserAutoPlayNextVideoValid (value: any) { - return isBooleanValid(value) -} - -function isUserAutoPlayNextVideoPlaylistValid (value: any) { - return isBooleanValid(value) -} - -function isUserEmailPublicValid (value: any) { - return isBooleanValid(value) -} - -function isUserNoModal (value: any) { - return isBooleanValid(value) -} - -function isUserBlockedReasonValid (value: any) { - return value === null || (exists(value) && validator.isLength(value, CONSTRAINTS_FIELDS.USERS.BLOCKED_REASON)) -} - -function isUserRoleValid (value: any) { - return exists(value) && validator.isInt('' + value) && [ UserRole.ADMINISTRATOR, UserRole.MODERATOR, UserRole.USER ].includes(value) -} - -// --------------------------------------------------------------------------- - -export { - isUserVideosHistoryEnabledValid, - isUserBlockedValid, - isUserPasswordValid, - isUserPasswordValidOrEmpty, - isUserVideoLanguages, - isUserBlockedReasonValid, - isUserRoleValid, - isUserVideoQuotaValid, - isUserVideoQuotaDailyValid, - isUserUsernameValid, - isUserAdminFlagsValid, - isUserEmailVerifiedValid, - isUserNSFWPolicyValid, - isUserP2PEnabledValid, - isUserAutoPlayVideoValid, - isUserAutoPlayNextVideoValid, - isUserAutoPlayNextVideoPlaylistValid, - isUserDisplayNameValid, - isUserDescriptionValid, - isUserEmailPublicValid, - isUserNoModal -} diff --git a/server/helpers/custom-validators/video-blacklist.ts b/server/helpers/custom-validators/video-blacklist.ts deleted file mode 100644 index 34fcec38e..000000000 --- a/server/helpers/custom-validators/video-blacklist.ts +++ /dev/null @@ -1,22 +0,0 @@ -import validator from 'validator' -import { exists } from './misc' -import { CONSTRAINTS_FIELDS } from '../../initializers/constants' -import { VideoBlacklistType } from '../../../shared/models/videos' - -const VIDEO_BLACKLIST_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_BLACKLIST - -function isVideoBlacklistReasonValid (value: string) { - return value === null || validator.isLength(value, VIDEO_BLACKLIST_CONSTRAINTS_FIELDS.REASON) -} - -function isVideoBlacklistTypeValid (value: any) { - return exists(value) && - (value === VideoBlacklistType.AUTO_BEFORE_PUBLISHED || value === VideoBlacklistType.MANUAL) -} - -// --------------------------------------------------------------------------- - -export { - isVideoBlacklistReasonValid, - isVideoBlacklistTypeValid -} diff --git a/server/helpers/custom-validators/video-captions.ts b/server/helpers/custom-validators/video-captions.ts deleted file mode 100644 index 0e24655a0..000000000 --- a/server/helpers/custom-validators/video-captions.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { UploadFilesForCheck } from 'express' -import { readFile } from 'fs-extra' -import { getFileSize } from '@shared/extra-utils' -import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_LANGUAGES } from '../../initializers/constants' -import { logger } from '../logger' -import { exists, isFileValid } from './misc' - -function isVideoCaptionLanguageValid (value: any) { - return exists(value) && VIDEO_LANGUAGES[value] !== undefined -} - -// MacOS sends application/octet-stream -const videoCaptionTypesRegex = [ ...Object.keys(MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT), 'application/octet-stream' ] - .map(m => `(${m})`) - .join('|') - -function isVideoCaptionFile (files: UploadFilesForCheck, field: string) { - return isFileValid({ - files, - mimeTypeRegex: videoCaptionTypesRegex, - field, - maxSize: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max - }) -} - -async function isVTTFileValid (filePath: string) { - const size = await getFileSize(filePath) - const content = await readFile(filePath, 'utf8') - - logger.debug('Checking VTT file %s', filePath, { size, content }) - - if (size > CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max) return false - - return content?.startsWith('WEBVTT') -} - -// --------------------------------------------------------------------------- - -export { - isVideoCaptionFile, - isVTTFileValid, - isVideoCaptionLanguageValid -} diff --git a/server/helpers/custom-validators/video-channel-syncs.ts b/server/helpers/custom-validators/video-channel-syncs.ts deleted file mode 100644 index c5a9afa96..000000000 --- a/server/helpers/custom-validators/video-channel-syncs.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants' -import { exists } from './misc' - -export function isVideoChannelSyncStateValid (value: any) { - return exists(value) && VIDEO_CHANNEL_SYNC_STATE[value] !== undefined -} diff --git a/server/helpers/custom-validators/video-channels.ts b/server/helpers/custom-validators/video-channels.ts deleted file mode 100644 index 249083f39..000000000 --- a/server/helpers/custom-validators/video-channels.ts +++ /dev/null @@ -1,32 +0,0 @@ -import validator from 'validator' -import { CONSTRAINTS_FIELDS } from '../../initializers/constants' -import { exists } from './misc' -import { isUserUsernameValid } from './users' - -const VIDEO_CHANNELS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_CHANNELS - -function isVideoChannelUsernameValid (value: string) { - // Use the same constraints than user username - return isUserUsernameValid(value) -} - -function isVideoChannelDescriptionValid (value: string) { - return value === null || validator.isLength(value, VIDEO_CHANNELS_CONSTRAINTS_FIELDS.DESCRIPTION) -} - -function isVideoChannelDisplayNameValid (value: string) { - return exists(value) && validator.isLength(value, VIDEO_CHANNELS_CONSTRAINTS_FIELDS.NAME) -} - -function isVideoChannelSupportValid (value: string) { - return value === null || (exists(value) && validator.isLength(value, VIDEO_CHANNELS_CONSTRAINTS_FIELDS.SUPPORT)) -} - -// --------------------------------------------------------------------------- - -export { - isVideoChannelUsernameValid, - isVideoChannelDescriptionValid, - isVideoChannelDisplayNameValid, - isVideoChannelSupportValid -} diff --git a/server/helpers/custom-validators/video-comments.ts b/server/helpers/custom-validators/video-comments.ts deleted file mode 100644 index 94bdf237a..000000000 --- a/server/helpers/custom-validators/video-comments.ts +++ /dev/null @@ -1,14 +0,0 @@ -import validator from 'validator' -import { CONSTRAINTS_FIELDS } from '../../initializers/constants' - -const VIDEO_COMMENTS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_COMMENTS - -function isValidVideoCommentText (value: string) { - return value === null || validator.isLength(value, VIDEO_COMMENTS_CONSTRAINTS_FIELDS.TEXT) -} - -// --------------------------------------------------------------------------- - -export { - isValidVideoCommentText -} diff --git a/server/helpers/custom-validators/video-imports.ts b/server/helpers/custom-validators/video-imports.ts deleted file mode 100644 index da8962cb6..000000000 --- a/server/helpers/custom-validators/video-imports.ts +++ /dev/null @@ -1,46 +0,0 @@ -import 'multer' -import { UploadFilesForCheck } from 'express' -import validator from 'validator' -import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_IMPORT_STATES } from '../../initializers/constants' -import { exists, isFileValid } from './misc' - -function isVideoImportTargetUrlValid (url: string) { - const isURLOptions = { - require_host: true, - require_tld: true, - require_protocol: true, - require_valid_protocol: true, - protocols: [ 'http', 'https' ] - } - - return exists(url) && - validator.isURL('' + url, isURLOptions) && - validator.isLength('' + url, CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL) -} - -function isVideoImportStateValid (value: any) { - return exists(value) && VIDEO_IMPORT_STATES[value] !== undefined -} - -// MacOS sends application/octet-stream -const videoTorrentImportRegex = [ ...Object.keys(MIMETYPES.TORRENT.MIMETYPE_EXT), 'application/octet-stream' ] - .map(m => `(${m})`) - .join('|') - -function isVideoImportTorrentFile (files: UploadFilesForCheck) { - return isFileValid({ - files, - mimeTypeRegex: videoTorrentImportRegex, - field: 'torrentfile', - maxSize: CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_FILE.FILE_SIZE.max, - optional: true - }) -} - -// --------------------------------------------------------------------------- - -export { - isVideoImportStateValid, - isVideoImportTargetUrlValid, - isVideoImportTorrentFile -} diff --git a/server/helpers/custom-validators/video-lives.ts b/server/helpers/custom-validators/video-lives.ts deleted file mode 100644 index 69d08ae68..000000000 --- a/server/helpers/custom-validators/video-lives.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { LiveVideoLatencyMode } from '@shared/models' - -function isLiveLatencyModeValid (value: any) { - return [ LiveVideoLatencyMode.DEFAULT, LiveVideoLatencyMode.SMALL_LATENCY, LiveVideoLatencyMode.HIGH_LATENCY ].includes(value) -} - -// --------------------------------------------------------------------------- - -export { - isLiveLatencyModeValid -} diff --git a/server/helpers/custom-validators/video-ownership.ts b/server/helpers/custom-validators/video-ownership.ts deleted file mode 100644 index cf15b385a..000000000 --- a/server/helpers/custom-validators/video-ownership.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { Response } from 'express' -import { MUserId } from '@server/types/models' -import { MVideoChangeOwnershipFull } from '@server/types/models/video/video-change-ownership' -import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' - -function checkUserCanTerminateOwnershipChange (user: MUserId, videoChangeOwnership: MVideoChangeOwnershipFull, res: Response) { - if (videoChangeOwnership.NextOwner.userId === user.id) { - return true - } - - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Cannot terminate an ownership change of another user' - }) - return false -} - -export { - checkUserCanTerminateOwnershipChange -} diff --git a/server/helpers/custom-validators/video-playlists.ts b/server/helpers/custom-validators/video-playlists.ts deleted file mode 100644 index 180018fc5..000000000 --- a/server/helpers/custom-validators/video-playlists.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { exists } from './misc' -import validator from 'validator' -import { CONSTRAINTS_FIELDS, VIDEO_PLAYLIST_PRIVACIES, VIDEO_PLAYLIST_TYPES } from '../../initializers/constants' - -const PLAYLISTS_CONSTRAINT_FIELDS = CONSTRAINTS_FIELDS.VIDEO_PLAYLISTS - -function isVideoPlaylistNameValid (value: any) { - return exists(value) && validator.isLength(value, PLAYLISTS_CONSTRAINT_FIELDS.NAME) -} - -function isVideoPlaylistDescriptionValid (value: any) { - return value === null || (exists(value) && validator.isLength(value, PLAYLISTS_CONSTRAINT_FIELDS.DESCRIPTION)) -} - -function isVideoPlaylistPrivacyValid (value: number) { - return validator.isInt(value + '') && VIDEO_PLAYLIST_PRIVACIES[value] !== undefined -} - -function isVideoPlaylistTimestampValid (value: any) { - return value === null || (exists(value) && validator.isInt('' + value, { min: 0 })) -} - -function isVideoPlaylistTypeValid (value: any) { - return exists(value) && VIDEO_PLAYLIST_TYPES[value] !== undefined -} - -// --------------------------------------------------------------------------- - -export { - isVideoPlaylistNameValid, - isVideoPlaylistDescriptionValid, - isVideoPlaylistPrivacyValid, - isVideoPlaylistTimestampValid, - isVideoPlaylistTypeValid -} diff --git a/server/helpers/custom-validators/video-redundancies.ts b/server/helpers/custom-validators/video-redundancies.ts deleted file mode 100644 index 50a559c4f..000000000 --- a/server/helpers/custom-validators/video-redundancies.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { exists } from './misc' - -function isVideoRedundancyTarget (value: any) { - return exists(value) && - (value === 'my-videos' || value === 'remote-videos') -} - -// --------------------------------------------------------------------------- - -export { - isVideoRedundancyTarget -} diff --git a/server/helpers/custom-validators/video-stats.ts b/server/helpers/custom-validators/video-stats.ts deleted file mode 100644 index 1e22f0654..000000000 --- a/server/helpers/custom-validators/video-stats.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { VideoStatsTimeserieMetric } from '@shared/models' - -const validMetrics = new Set([ - 'viewers', - 'aggregateWatchTime' -]) - -function isValidStatTimeserieMetric (value: VideoStatsTimeserieMetric) { - return validMetrics.has(value) -} - -// --------------------------------------------------------------------------- - -export { - isValidStatTimeserieMetric -} diff --git a/server/helpers/custom-validators/video-studio.ts b/server/helpers/custom-validators/video-studio.ts deleted file mode 100644 index 68dfec8dd..000000000 --- a/server/helpers/custom-validators/video-studio.ts +++ /dev/null @@ -1,53 +0,0 @@ -import validator from 'validator' -import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' -import { buildTaskFileFieldname } from '@server/lib/video-studio' -import { VideoStudioTask } from '@shared/models' -import { isArray } from './misc' -import { isVideoFileMimeTypeValid, isVideoImageValid } from './videos' -import { forceNumber } from '@shared/core-utils' - -function isValidStudioTasksArray (tasks: any) { - if (!isArray(tasks)) return false - - return tasks.length >= CONSTRAINTS_FIELDS.VIDEO_STUDIO.TASKS.min && - tasks.length <= CONSTRAINTS_FIELDS.VIDEO_STUDIO.TASKS.max -} - -function isStudioCutTaskValid (task: VideoStudioTask) { - if (task.name !== 'cut') return false - if (!task.options) return false - - const { start, end } = task.options - if (!start && !end) return false - - if (start && !validator.isInt(start + '', CONSTRAINTS_FIELDS.VIDEO_STUDIO.CUT_TIME)) return false - if (end && !validator.isInt(end + '', CONSTRAINTS_FIELDS.VIDEO_STUDIO.CUT_TIME)) return false - - if (!start || !end) return true - - return forceNumber(start) < forceNumber(end) -} - -function isStudioTaskAddIntroOutroValid (task: VideoStudioTask, indice: number, files: Express.Multer.File[]) { - const file = files.find(f => f.fieldname === buildTaskFileFieldname(indice, 'file')) - - return (task.name === 'add-intro' || task.name === 'add-outro') && - file && isVideoFileMimeTypeValid([ file ], null) -} - -function isStudioTaskAddWatermarkValid (task: VideoStudioTask, indice: number, files: Express.Multer.File[]) { - const file = files.find(f => f.fieldname === buildTaskFileFieldname(indice, 'file')) - - return task.name === 'add-watermark' && - file && isVideoImageValid([ file ], null, true) -} - -// --------------------------------------------------------------------------- - -export { - isValidStudioTasksArray, - - isStudioCutTaskValid, - isStudioTaskAddIntroOutroValid, - isStudioTaskAddWatermarkValid -} diff --git a/server/helpers/custom-validators/video-transcoding.ts b/server/helpers/custom-validators/video-transcoding.ts deleted file mode 100644 index 220530de4..000000000 --- a/server/helpers/custom-validators/video-transcoding.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { exists } from './misc' - -function isValidCreateTranscodingType (value: any) { - return exists(value) && - (value === 'hls' || value === 'webtorrent' || value === 'web-video') // TODO: remove webtorrent in v7 -} - -// --------------------------------------------------------------------------- - -export { - isValidCreateTranscodingType -} diff --git a/server/helpers/custom-validators/video-view.ts b/server/helpers/custom-validators/video-view.ts deleted file mode 100644 index 091c92083..000000000 --- a/server/helpers/custom-validators/video-view.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { exists } from './misc' - -function isVideoTimeValid (value: number, videoDuration?: number) { - if (value < 0) return false - if (exists(videoDuration) && value > videoDuration) return false - - return true -} - -export { - isVideoTimeValid -} diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts deleted file mode 100644 index 00c6deed4..000000000 --- a/server/helpers/custom-validators/videos.ts +++ /dev/null @@ -1,218 +0,0 @@ -import { Request, Response, UploadFilesForCheck } from 'express' -import { decode as magnetUriDecode } from 'magnet-uri' -import validator from 'validator' -import { getVideoWithAttributes } from '@server/helpers/video' -import { HttpStatusCode, VideoInclude, VideoPrivacy, VideoRateType } from '@shared/models' -import { - CONSTRAINTS_FIELDS, - MIMETYPES, - VIDEO_CATEGORIES, - VIDEO_LICENCES, - VIDEO_LIVE, - VIDEO_PRIVACIES, - VIDEO_RATE_TYPES, - VIDEO_STATES -} from '../../initializers/constants' -import { exists, isArray, isDateValid, isFileValid } from './misc' - -const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS - -function isVideoIncludeValid (include: VideoInclude) { - return exists(include) && validator.isInt('' + include) -} - -function isVideoCategoryValid (value: any) { - return value === null || VIDEO_CATEGORIES[value] !== undefined -} - -function isVideoStateValid (value: any) { - return exists(value) && VIDEO_STATES[value] !== undefined -} - -function isVideoLicenceValid (value: any) { - return value === null || VIDEO_LICENCES[value] !== undefined -} - -function isVideoLanguageValid (value: any) { - return value === null || - (typeof value === 'string' && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.LANGUAGE)) -} - -function isVideoDurationValid (value: string) { - return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DURATION) -} - -function isVideoDescriptionValid (value: string) { - return value === null || (exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.DESCRIPTION)) -} - -function isVideoSupportValid (value: string) { - return value === null || (exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.SUPPORT)) -} - -function isVideoNameValid (value: string) { - return exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.NAME) -} - -function isVideoTagValid (tag: string) { - return exists(tag) && validator.isLength(tag, VIDEOS_CONSTRAINTS_FIELDS.TAG) -} - -function areVideoTagsValid (tags: string[]) { - return tags === null || ( - isArray(tags) && - validator.isInt(tags.length.toString(), VIDEOS_CONSTRAINTS_FIELDS.TAGS) && - tags.every(tag => isVideoTagValid(tag)) - ) -} - -function isVideoViewsValid (value: string) { - return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.VIEWS) -} - -const ratingTypes = new Set(Object.values(VIDEO_RATE_TYPES)) -function isVideoRatingTypeValid (value: string) { - return value === 'none' || ratingTypes.has(value as VideoRateType) -} - -function isVideoFileExtnameValid (value: string) { - return exists(value) && (value === VIDEO_LIVE.EXTENSION || MIMETYPES.VIDEO.EXT_MIMETYPE[value] !== undefined) -} - -function isVideoFileMimeTypeValid (files: UploadFilesForCheck, field = 'videofile') { - return isFileValid({ - files, - mimeTypeRegex: MIMETYPES.VIDEO.MIMETYPES_REGEX, - field, - maxSize: null - }) -} - -const videoImageTypes = CONSTRAINTS_FIELDS.VIDEOS.IMAGE.EXTNAME - .map(v => v.replace('.', '')) - .join('|') -const videoImageTypesRegex = `image/(${videoImageTypes})` - -function isVideoImageValid (files: UploadFilesForCheck, field: string, optional = true) { - return isFileValid({ - files, - mimeTypeRegex: videoImageTypesRegex, - field, - maxSize: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max, - optional - }) -} - -function isVideoPrivacyValid (value: number) { - return VIDEO_PRIVACIES[value] !== undefined -} - -function isVideoReplayPrivacyValid (value: number) { - return VIDEO_PRIVACIES[value] !== undefined && value !== VideoPrivacy.PASSWORD_PROTECTED -} - -function isScheduleVideoUpdatePrivacyValid (value: number) { - return value === VideoPrivacy.UNLISTED || value === VideoPrivacy.PUBLIC || value === VideoPrivacy.INTERNAL -} - -function isVideoOriginallyPublishedAtValid (value: string | null) { - return value === null || isDateValid(value) -} - -function isVideoFileInfoHashValid (value: string | null | undefined) { - return exists(value) && validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.INFO_HASH) -} - -function isVideoFileResolutionValid (value: string) { - return exists(value) && validator.isInt(value + '') -} - -function isVideoFPSResolutionValid (value: string) { - return value === null || validator.isInt(value + '') -} - -function isVideoFileSizeValid (value: string) { - return exists(value) && validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.FILE_SIZE) -} - -function isVideoMagnetUriValid (value: string) { - if (!exists(value)) return false - - const parsed = magnetUriDecode(value) - return parsed && isVideoFileInfoHashValid(parsed.infoHash) -} - -function isPasswordValid (password: string) { - return password.length >= CONSTRAINTS_FIELDS.VIDEO_PASSWORD.LENGTH.min && - password.length < CONSTRAINTS_FIELDS.VIDEO_PASSWORD.LENGTH.max -} - -function isValidPasswordProtectedPrivacy (req: Request, res: Response) { - const fail = (message: string) => { - res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message - }) - return false - } - - let privacy: VideoPrivacy - const video = getVideoWithAttributes(res) - - if (exists(req.body?.privacy)) privacy = req.body.privacy - else if (exists(video?.privacy)) privacy = video.privacy - - if (privacy !== VideoPrivacy.PASSWORD_PROTECTED) return true - - if (!exists(req.body.videoPasswords) && !exists(req.body.passwords)) return fail('Video passwords are missing.') - - const passwords = req.body.videoPasswords || req.body.passwords - - if (passwords.length === 0) return fail('At least one video password is required.') - - if (new Set(passwords).size !== passwords.length) return fail('Duplicate video passwords are not allowed.') - - for (const password of passwords) { - if (typeof password !== 'string') { - return fail('Video password should be a string.') - } - - if (!isPasswordValid(password)) { - return fail('Invalid video password. Password length should be at least 2 characters and no more than 100 characters.') - } - } - - return true -} - -// --------------------------------------------------------------------------- - -export { - isVideoCategoryValid, - isVideoLicenceValid, - isVideoLanguageValid, - isVideoDescriptionValid, - isVideoFileInfoHashValid, - isVideoNameValid, - areVideoTagsValid, - isVideoFPSResolutionValid, - isScheduleVideoUpdatePrivacyValid, - isVideoOriginallyPublishedAtValid, - isVideoMagnetUriValid, - isVideoStateValid, - isVideoIncludeValid, - isVideoViewsValid, - isVideoRatingTypeValid, - isVideoFileExtnameValid, - isVideoFileMimeTypeValid, - isVideoDurationValid, - isVideoTagValid, - isVideoPrivacyValid, - isVideoReplayPrivacyValid, - isVideoFileResolutionValid, - isVideoFileSizeValid, - isVideoImageValid, - isVideoSupportValid, - isPasswordValid, - isValidPasswordProtectedPrivacy -} diff --git a/server/helpers/custom-validators/webfinger.ts b/server/helpers/custom-validators/webfinger.ts deleted file mode 100644 index dd914341e..000000000 --- a/server/helpers/custom-validators/webfinger.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { REMOTE_SCHEME, WEBSERVER } from '../../initializers/constants' -import { sanitizeHost } from '../core-utils' -import { exists } from './misc' - -function isWebfingerLocalResourceValid (value: string) { - if (!exists(value)) return false - if (value.startsWith('acct:') === false) return false - - const actorWithHost = value.substr(5) - const actorParts = actorWithHost.split('@') - if (actorParts.length !== 2) return false - - const host = actorParts[1] - return sanitizeHost(host, REMOTE_SCHEME.HTTP) === WEBSERVER.HOST -} - -// --------------------------------------------------------------------------- - -export { - isWebfingerLocalResourceValid -} diff --git a/server/helpers/database-utils.ts b/server/helpers/database-utils.ts deleted file mode 100644 index b6ba7fd75..000000000 --- a/server/helpers/database-utils.ts +++ /dev/null @@ -1,121 +0,0 @@ -import retry from 'async/retry' -import Bluebird from 'bluebird' -import { Transaction } from 'sequelize' -import { Model } from 'sequelize-typescript' -import { sequelizeTypescript } from '@server/initializers/database' -import { logger } from './logger' - -function retryTransactionWrapper ( - functionToRetry: (arg1: A, arg2: B, arg3: C, arg4: D) => Promise, - arg1: A, - arg2: B, - arg3: C, - arg4: D, -): Promise - -function retryTransactionWrapper ( - functionToRetry: (arg1: A, arg2: B, arg3: C) => Promise, - arg1: A, - arg2: B, - arg3: C -): Promise - -function retryTransactionWrapper ( - functionToRetry: (arg1: A, arg2: B) => Promise, - arg1: A, - arg2: B -): Promise - -function retryTransactionWrapper ( - functionToRetry: (arg1: A) => Promise, - arg1: A -): Promise - -function retryTransactionWrapper ( - functionToRetry: () => Promise | Bluebird -): Promise - -function retryTransactionWrapper ( - functionToRetry: (...args: any[]) => Promise, - ...args: any[] -): Promise { - return transactionRetryer(callback => { - functionToRetry.apply(null, args) - .then((result: T) => callback(null, result)) - .catch(err => callback(err)) - }) - .catch(err => { - logger.warn(`Cannot execute ${functionToRetry.name} with many retries.`, { err }) - throw err - }) -} - -function transactionRetryer (func: (err: any, data: T) => any) { - return new Promise((res, rej) => { - retry( - { - times: 5, - - errorFilter: err => { - const willRetry = (err.name === 'SequelizeDatabaseError') - logger.debug('Maybe retrying the transaction function.', { willRetry, err, tags: [ 'sql', 'retry' ] }) - return willRetry - } - }, - func, - (err, data) => err ? rej(err) : res(data) - ) - }) -} - -function saveInTransactionWithRetries > (model: T) { - return retryTransactionWrapper(() => { - return sequelizeTypescript.transaction(async transaction => { - await model.save({ transaction }) - }) - }) -} - -// --------------------------------------------------------------------------- - -function resetSequelizeInstance (instance: Model) { - return instance.reload() -} - -function filterNonExistingModels ( - fromDatabase: T[], - newModels: T[] -) { - return fromDatabase.filter(f => !newModels.find(newModel => newModel.hasSameUniqueKeysThan(f))) -} - -function deleteAllModels > (models: T[], transaction: Transaction) { - return Promise.all(models.map(f => f.destroy({ transaction }))) -} - -// --------------------------------------------------------------------------- - -function runInReadCommittedTransaction (fn: (t: Transaction) => Promise) { - const options = { isolationLevel: Transaction.ISOLATION_LEVELS.READ_COMMITTED } - - return sequelizeTypescript.transaction(options, t => fn(t)) -} - -function afterCommitIfTransaction (t: Transaction, fn: Function) { - if (t) return t.afterCommit(() => fn()) - - return fn() -} - -// --------------------------------------------------------------------------- - -export { - resetSequelizeInstance, - retryTransactionWrapper, - transactionRetryer, - saveInTransactionWithRetries, - afterCommitIfTransaction, - filterNonExistingModels, - deleteAllModels, - runInReadCommittedTransaction -} diff --git a/server/helpers/decache.ts b/server/helpers/decache.ts deleted file mode 100644 index 6be446ff6..000000000 --- a/server/helpers/decache.ts +++ /dev/null @@ -1,78 +0,0 @@ -// Thanks: https://github.com/dwyl/decache -// We reuse this file to also uncache plugin base path - -import { extname } from 'path' - -function decachePlugin (libraryPath: string) { - const moduleName = find(libraryPath) - - if (!moduleName) return - - searchCache(moduleName, function (mod) { - delete require.cache[mod.id] - - removeCachedPath(mod.path) - }) -} - -function decacheModule (name: string) { - const moduleName = find(name) - - if (!moduleName) return - - searchCache(moduleName, function (mod) { - delete require.cache[mod.id] - - removeCachedPath(mod.path) - }) -} - -// --------------------------------------------------------------------------- - -export { - decacheModule, - decachePlugin -} - -// --------------------------------------------------------------------------- - -function find (moduleName: string) { - try { - return require.resolve(moduleName) - } catch { - return '' - } -} - -function searchCache (moduleName: string, callback: (current: NodeModule) => void) { - const resolvedModule = require.resolve(moduleName) - let mod: NodeModule - const visited = {} - - if (resolvedModule && ((mod = require.cache[resolvedModule]) !== undefined)) { - // Recursively go over the results - (function run (current) { - visited[current.id] = true - - current.children.forEach(function (child) { - if (extname(child.filename) !== '.node' && !visited[child.id]) { - run(child) - } - }) - - // Call the specified callback providing the - // found module - callback(current) - })(mod) - } -}; - -function removeCachedPath (pluginPath: string) { - const pathCache = (module.constructor as any)._pathCache as { [ id: string ]: string[] } - - Object.keys(pathCache).forEach(function (cacheKey) { - if (cacheKey.includes(pluginPath)) { - delete pathCache[cacheKey] - } - }) -} diff --git a/server/helpers/dns.ts b/server/helpers/dns.ts deleted file mode 100644 index da8b666c2..000000000 --- a/server/helpers/dns.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { lookup } from 'dns' -import { parse as parseIP } from 'ipaddr.js' - -function dnsLookupAll (hostname: string) { - return new Promise((res, rej) => { - lookup(hostname, { family: 0, all: true }, (err, adresses) => { - if (err) return rej(err) - - return res(adresses.map(a => a.address)) - }) - }) -} - -async function isResolvingToUnicastOnly (hostname: string) { - const addresses = await dnsLookupAll(hostname) - - for (const address of addresses) { - const parsed = parseIP(address) - - if (parsed.range() !== 'unicast') return false - } - - return true -} - -export { - dnsLookupAll, - isResolvingToUnicastOnly -} diff --git a/server/helpers/express-utils.ts b/server/helpers/express-utils.ts deleted file mode 100644 index 783097e55..000000000 --- a/server/helpers/express-utils.ts +++ /dev/null @@ -1,156 +0,0 @@ -import express, { RequestHandler } from 'express' -import multer, { diskStorage } from 'multer' -import { getLowercaseExtension } from '@shared/core-utils' -import { CONFIG } from '../initializers/config' -import { REMOTE_SCHEME } from '../initializers/constants' -import { isArray } from './custom-validators/misc' -import { logger } from './logger' -import { deleteFileAndCatch, generateRandomString } from './utils' -import { getExtFromMimetype } from './video' - -function buildNSFWFilter (res?: express.Response, paramNSFW?: string) { - if (paramNSFW === 'true') return true - if (paramNSFW === 'false') return false - if (paramNSFW === 'both') return undefined - - if (res?.locals.oauth) { - const user = res.locals.oauth.token.User - - // User does not want NSFW videos - if (user.nsfwPolicy === 'do_not_list') return false - - // Both - return undefined - } - - if (CONFIG.INSTANCE.DEFAULT_NSFW_POLICY === 'do_not_list') return false - - // Display all - return null -} - -function cleanUpReqFiles (req: express.Request) { - const filesObject = req.files - if (!filesObject) return - - if (isArray(filesObject)) { - filesObject.forEach(f => deleteFileAndCatch(f.path)) - return - } - - for (const key of Object.keys(filesObject)) { - const files = filesObject[key] - - files.forEach(f => deleteFileAndCatch(f.path)) - } -} - -function getHostWithPort (host: string) { - const splitted = host.split(':') - - // The port was not specified - if (splitted.length === 1) { - if (REMOTE_SCHEME.HTTP === 'https') return host + ':443' - - return host + ':80' - } - - return host -} - -function createReqFiles ( - fieldNames: string[], - mimeTypes: { [id: string]: string | string[] }, - destination = CONFIG.STORAGE.TMP_DIR -): RequestHandler { - const storage = diskStorage({ - destination: (req, file, cb) => { - cb(null, destination) - }, - - filename: (req, file, cb) => { - return generateReqFilename(file, mimeTypes, cb) - } - }) - - const fields: { name: string, maxCount: number }[] = [] - for (const fieldName of fieldNames) { - fields.push({ - name: fieldName, - maxCount: 1 - }) - } - - return multer({ storage }).fields(fields) -} - -function createAnyReqFiles ( - mimeTypes: { [id: string]: string | string[] }, - fileFilter: (req: express.Request, file: Express.Multer.File, cb: (err: Error, result: boolean) => void) => void -): RequestHandler { - const storage = diskStorage({ - destination: (req, file, cb) => { - cb(null, CONFIG.STORAGE.TMP_DIR) - }, - - filename: (req, file, cb) => { - return generateReqFilename(file, mimeTypes, cb) - } - }) - - return multer({ storage, fileFilter }).any() -} - -function isUserAbleToSearchRemoteURI (res: express.Response) { - const user = res.locals.oauth ? res.locals.oauth.token.User : undefined - - return CONFIG.SEARCH.REMOTE_URI.ANONYMOUS === true || - (CONFIG.SEARCH.REMOTE_URI.USERS === true && user !== undefined) -} - -function getCountVideos (req: express.Request) { - return req.query.skipCount !== true -} - -// --------------------------------------------------------------------------- - -export { - buildNSFWFilter, - getHostWithPort, - createAnyReqFiles, - isUserAbleToSearchRemoteURI, - createReqFiles, - cleanUpReqFiles, - getCountVideos -} - -// --------------------------------------------------------------------------- - -async function generateReqFilename ( - file: Express.Multer.File, - mimeTypes: { [id: string]: string | string[] }, - cb: (err: Error, name: string) => void -) { - let extension: string - const fileExtension = getLowercaseExtension(file.originalname) - const extensionFromMimetype = getExtFromMimetype(mimeTypes, file.mimetype) - - // Take the file extension if we don't understand the mime type - if (!extensionFromMimetype) { - extension = fileExtension - } else { - // Take the first available extension for this mimetype - extension = extensionFromMimetype - } - - let randomString = '' - - try { - randomString = await generateRandomString(16) - } catch (err) { - logger.error('Cannot generate random string for file name.', { err }) - randomString = 'fake-random-string' - } - - cb(null, randomString + extension) -} diff --git a/server/helpers/ffmpeg/codecs.ts b/server/helpers/ffmpeg/codecs.ts deleted file mode 100644 index 3bd7db396..000000000 --- a/server/helpers/ffmpeg/codecs.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { FfprobeData } from 'fluent-ffmpeg' -import { getAudioStream, getVideoStream } from '@shared/ffmpeg' -import { logger } from '../logger' -import { forceNumber } from '@shared/core-utils' - -export async function getVideoStreamCodec (path: string) { - const videoStream = await getVideoStream(path) - if (!videoStream) return '' - - const videoCodec = videoStream.codec_tag_string - - if (videoCodec === 'vp09') return 'vp09.00.50.08' - if (videoCodec === 'hev1') return 'hev1.1.6.L93.B0' - - const baseProfileMatrix = { - avc1: { - High: '6400', - Main: '4D40', - Baseline: '42E0' - }, - av01: { - High: '1', - Main: '0', - Professional: '2' - } - } - - let baseProfile = baseProfileMatrix[videoCodec][videoStream.profile] - if (!baseProfile) { - logger.warn('Cannot get video profile codec of %s.', path, { videoStream }) - baseProfile = baseProfileMatrix[videoCodec]['High'] // Fallback - } - - if (videoCodec === 'av01') { - let level = videoStream.level.toString() - if (level.length === 1) level = `0${level}` - - // Guess the tier indicator and bit depth - return `${videoCodec}.${baseProfile}.${level}M.08` - } - - let level = forceNumber(videoStream.level).toString(16) - if (level.length === 1) level = `0${level}` - - // Default, h264 codec - return `${videoCodec}.${baseProfile}${level}` -} - -export async function getAudioStreamCodec (path: string, existingProbe?: FfprobeData) { - const { audioStream } = await getAudioStream(path, existingProbe) - - if (!audioStream) return '' - - const audioCodecName = audioStream.codec_name - - if (audioCodecName === 'opus') return 'opus' - if (audioCodecName === 'vorbis') return 'vorbis' - if (audioCodecName === 'aac') return 'mp4a.40.2' - if (audioCodecName === 'mp3') return 'mp4a.40.34' - - logger.warn('Cannot get audio codec of %s.', path, { audioStream }) - - return 'mp4a.40.2' // Fallback -} diff --git a/server/helpers/ffmpeg/ffmpeg-image.ts b/server/helpers/ffmpeg/ffmpeg-image.ts deleted file mode 100644 index 0bb0ff2c0..000000000 --- a/server/helpers/ffmpeg/ffmpeg-image.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { FFmpegImage } from '@shared/ffmpeg' -import { getFFmpegCommandWrapperOptions } from './ffmpeg-options' - -export function processGIF (options: Parameters[0]) { - return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).processGIF(options) -} - -export function generateThumbnailFromVideo (options: Parameters[0]) { - return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).generateThumbnailFromVideo(options) -} - -export function convertWebPToJPG (options: Parameters[0]) { - return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).convertWebPToJPG(options) -} diff --git a/server/helpers/ffmpeg/ffmpeg-options.ts b/server/helpers/ffmpeg/ffmpeg-options.ts deleted file mode 100644 index 64d7c4179..000000000 --- a/server/helpers/ffmpeg/ffmpeg-options.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { FFMPEG_NICE } from '@server/initializers/constants' -import { FFmpegCommandWrapperOptions } from '@shared/ffmpeg' -import { AvailableEncoders } from '@shared/models' - -type CommandType = 'live' | 'vod' | 'thumbnail' - -export function getFFmpegCommandWrapperOptions (type: CommandType, availableEncoders?: AvailableEncoders): FFmpegCommandWrapperOptions { - return { - availableEncoders, - profile: getProfile(type), - - niceness: FFMPEG_NICE[type.toUpperCase()], - tmpDirectory: CONFIG.STORAGE.TMP_DIR, - threads: getThreads(type), - - logger: { - debug: logger.debug.bind(logger), - info: logger.info.bind(logger), - warn: logger.warn.bind(logger), - error: logger.error.bind(logger) - }, - lTags: { tags: [ 'ffmpeg' ] } - } -} - -// --------------------------------------------------------------------------- -// Private -// --------------------------------------------------------------------------- - -function getThreads (type: CommandType) { - if (type === 'live') return CONFIG.LIVE.TRANSCODING.THREADS - if (type === 'vod') return CONFIG.TRANSCODING.THREADS - - // Auto - return 0 -} - -function getProfile (type: CommandType) { - if (type === 'live') return CONFIG.LIVE.TRANSCODING.PROFILE - if (type === 'vod') return CONFIG.TRANSCODING.PROFILE - - return undefined -} diff --git a/server/helpers/ffmpeg/framerate.ts b/server/helpers/ffmpeg/framerate.ts deleted file mode 100644 index 18cb0e0e2..000000000 --- a/server/helpers/ffmpeg/framerate.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants' -import { VideoResolution } from '@shared/models' - -export function computeOutputFPS (options: { - inputFPS: number - resolution: VideoResolution -}) { - const { resolution } = options - - let fps = options.inputFPS - - if ( - // On small/medium resolutions, limit FPS - resolution !== undefined && - resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN && - fps > VIDEO_TRANSCODING_FPS.AVERAGE - ) { - // Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value - fps = getClosestFramerateStandard({ fps, type: 'STANDARD' }) - } - - // Hard FPS limits - if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = getClosestFramerateStandard({ fps, type: 'HD_STANDARD' }) - - if (fps < VIDEO_TRANSCODING_FPS.MIN) { - throw new Error(`Cannot compute FPS because ${fps} is lower than our minimum value ${VIDEO_TRANSCODING_FPS.MIN}`) - } - - return fps -} - -// --------------------------------------------------------------------------- -// Private -// --------------------------------------------------------------------------- - -function getClosestFramerateStandard (options: { - fps: number - type: 'HD_STANDARD' | 'STANDARD' -}) { - const { fps, type } = options - - return VIDEO_TRANSCODING_FPS[type].slice(0) - .sort((a, b) => fps % a - fps % b)[0] -} diff --git a/server/helpers/ffmpeg/index.ts b/server/helpers/ffmpeg/index.ts deleted file mode 100644 index bf1c73fb6..000000000 --- a/server/helpers/ffmpeg/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './codecs' -export * from './ffmpeg-image' -export * from './ffmpeg-options' -export * from './framerate' diff --git a/server/helpers/geo-ip.ts b/server/helpers/geo-ip.ts deleted file mode 100644 index 9e44d660f..000000000 --- a/server/helpers/geo-ip.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { pathExists, writeFile } from 'fs-extra' -import maxmind, { CountryResponse, Reader } from 'maxmind' -import { join } from 'path' -import { CONFIG } from '@server/initializers/config' -import { logger, loggerTagsFactory } from './logger' -import { isBinaryResponse, peertubeGot } from './requests' - -const lTags = loggerTagsFactory('geo-ip') - -const mmbdFilename = 'dbip-country-lite-latest.mmdb' -const mmdbPath = join(CONFIG.STORAGE.BIN_DIR, mmbdFilename) - -export class GeoIP { - private static instance: GeoIP - - private reader: Reader - - private constructor () { - } - - async safeCountryISOLookup (ip: string): Promise { - if (CONFIG.GEO_IP.ENABLED === false) return null - - await this.initReaderIfNeeded() - - try { - const result = this.reader.get(ip) - if (!result) return null - - return result.country.iso_code - } catch (err) { - logger.error('Cannot get country from IP.', { err }) - - return null - } - } - - async updateDatabase () { - if (CONFIG.GEO_IP.ENABLED === false) return - - const url = CONFIG.GEO_IP.COUNTRY.DATABASE_URL - - logger.info('Updating GeoIP database from %s.', url, lTags()) - - const gotOptions = { context: { bodyKBLimit: 200_000 }, responseType: 'buffer' as 'buffer' } - - try { - const gotResult = await peertubeGot(url, gotOptions) - - if (!isBinaryResponse(gotResult)) { - throw new Error('Not a binary response') - } - - await writeFile(mmdbPath, gotResult.body) - - // Reinit reader - this.reader = undefined - - logger.info('GeoIP database updated %s.', mmdbPath, lTags()) - } catch (err) { - logger.error('Cannot update GeoIP database from %s.', url, { err, ...lTags() }) - } - } - - private async initReaderIfNeeded () { - if (!this.reader) { - if (!await pathExists(mmdbPath)) { - await this.updateDatabase() - } - - this.reader = await maxmind.open(mmdbPath) - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/helpers/image-utils.ts b/server/helpers/image-utils.ts deleted file mode 100644 index 2a8bb6e6e..000000000 --- a/server/helpers/image-utils.ts +++ /dev/null @@ -1,179 +0,0 @@ -import { copy, readFile, remove, rename } from 'fs-extra' -import Jimp, { read as jimpRead } from 'jimp' -import { join } from 'path' -import { ColorActionName } from '@jimp/plugin-color' -import { getLowercaseExtension } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { convertWebPToJPG, generateThumbnailFromVideo, processGIF } from './ffmpeg' -import { logger, loggerTagsFactory } from './logger' - -const lTags = loggerTagsFactory('image-utils') - -function generateImageFilename (extension = '.jpg') { - return buildUUID() + extension -} - -async function processImage (options: { - path: string - destination: string - newSize: { width: number, height: number } - keepOriginal?: boolean // default false -}) { - const { path, destination, newSize, keepOriginal = false } = options - - const extension = getLowercaseExtension(path) - - if (path === destination) { - throw new Error('Jimp/FFmpeg needs an input path different that the output path.') - } - - logger.debug('Processing image %s to %s.', path, destination) - - // Use FFmpeg to process GIF - if (extension === '.gif') { - await processGIF({ path, destination, newSize }) - } else { - await jimpProcessor(path, destination, newSize, extension) - } - - if (keepOriginal !== true) await remove(path) -} - -async function generateImageFromVideoFile (options: { - fromPath: string - folder: string - imageName: string - size: { width: number, height: number } -}) { - const { fromPath, folder, imageName, size } = options - - const pendingImageName = 'pending-' + imageName - const pendingImagePath = join(folder, pendingImageName) - - try { - await generateThumbnailFromVideo({ fromPath, output: pendingImagePath }) - - const destination = join(folder, imageName) - await processImage({ path: pendingImagePath, destination, newSize: size }) - } catch (err) { - logger.error('Cannot generate image from video %s.', fromPath, { err, ...lTags() }) - - try { - await remove(pendingImagePath) - } catch (err) { - logger.debug('Cannot remove pending image path after generation error.', { err, ...lTags() }) - } - - throw err - } -} - -async function getImageSize (path: string) { - const inputBuffer = await readFile(path) - - const image = await jimpRead(inputBuffer) - - return { - width: image.getWidth(), - height: image.getHeight() - } -} - -// --------------------------------------------------------------------------- - -export { - generateImageFilename, - generateImageFromVideoFile, - - processImage, - - getImageSize -} - -// --------------------------------------------------------------------------- - -async function jimpProcessor (path: string, destination: string, newSize: { width: number, height: number }, inputExt: string) { - let sourceImage: Jimp - const inputBuffer = await readFile(path) - - try { - sourceImage = await jimpRead(inputBuffer) - } catch (err) { - logger.debug('Cannot read %s with jimp. Try to convert the image using ffmpeg first.', path, { err }) - - const newName = path + '.jpg' - await convertWebPToJPG({ path, destination: newName }) - await rename(newName, path) - - sourceImage = await jimpRead(path) - } - - await remove(destination) - - // Optimization if the source file has the appropriate size - const outputExt = getLowercaseExtension(destination) - if (skipProcessing({ sourceImage, newSize, imageBytes: inputBuffer.byteLength, inputExt, outputExt })) { - return copy(path, destination) - } - - await autoResize({ sourceImage, newSize, destination }) -} - -async function autoResize (options: { - sourceImage: Jimp - newSize: { width: number, height: number } - destination: string -}) { - const { sourceImage, newSize, destination } = options - - // Portrait mode targeting a landscape, apply some effect on the image - const sourceIsPortrait = sourceImage.getWidth() < sourceImage.getHeight() - const destIsPortraitOrSquare = newSize.width <= newSize.height - - removeExif(sourceImage) - - if (sourceIsPortrait && !destIsPortraitOrSquare) { - const baseImage = sourceImage.cloneQuiet().cover(newSize.width, newSize.height) - .color([ { apply: ColorActionName.SHADE, params: [ 50 ] } ]) - - const topImage = sourceImage.cloneQuiet().contain(newSize.width, newSize.height) - - return write(baseImage.blit(topImage, 0, 0), destination) - } - - return write(sourceImage.cover(newSize.width, newSize.height), destination) -} - -function write (image: Jimp, destination: string) { - return image.quality(80).writeAsync(destination) -} - -function skipProcessing (options: { - sourceImage: Jimp - newSize: { width: number, height: number } - imageBytes: number - inputExt: string - outputExt: string -}) { - const { sourceImage, newSize, imageBytes, inputExt, outputExt } = options - const { width, height } = newSize - - if (hasExif(sourceImage)) return false - if (sourceImage.getWidth() > width || sourceImage.getHeight() > height) return false - if (inputExt !== outputExt) return false - - const kB = 1000 - - if (height >= 1000) return imageBytes <= 200 * kB - if (height >= 500) return imageBytes <= 100 * kB - - return imageBytes <= 15 * kB -} - -function hasExif (image: Jimp) { - return !!(image.bitmap as any).exifBuffer -} - -function removeExif (image: Jimp) { - (image.bitmap as any).exifBuffer = null -} diff --git a/server/helpers/logger.ts b/server/helpers/logger.ts deleted file mode 100644 index 6649db40f..000000000 --- a/server/helpers/logger.ts +++ /dev/null @@ -1,208 +0,0 @@ -import { stat } from 'fs-extra' -import { join } from 'path' -import { format as sqlFormat } from 'sql-formatter' -import { createLogger, format, transports } from 'winston' -import { FileTransportOptions } from 'winston/lib/winston/transports' -import { context } from '@opentelemetry/api' -import { getSpanContext } from '@opentelemetry/api/build/src/trace/context-utils' -import { omit } from '@shared/core-utils' -import { CONFIG } from '../initializers/config' -import { LOG_FILENAME } from '../initializers/constants' - -const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT - -const consoleLoggerFormat = format.printf(info => { - let additionalInfos = JSON.stringify(getAdditionalInfo(info), removeCyclicValues(), 2) - - if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = '' - else additionalInfos = ' ' + additionalInfos - - if (info.sql) { - if (CONFIG.LOG.PRETTIFY_SQL) { - additionalInfos += '\n' + sqlFormat(info.sql, { - language: 'sql', - tabWidth: 2 - }) - } else { - additionalInfos += ' - ' + info.sql - } - } - - return `[${info.label}] ${info.timestamp} ${info.level}: ${info.message}${additionalInfos}` -}) - -const jsonLoggerFormat = format.printf(info => { - return JSON.stringify(info, removeCyclicValues()) -}) - -const timestampFormatter = format.timestamp({ - format: 'YYYY-MM-DD HH:mm:ss.SSS' -}) -const labelFormatter = (suffix?: string) => { - return format.label({ - label: suffix ? `${label} ${suffix}` : label - }) -} - -const fileLoggerOptions: FileTransportOptions = { - filename: join(CONFIG.STORAGE.LOG_DIR, LOG_FILENAME), - handleExceptions: true, - format: format.combine( - format.timestamp(), - jsonLoggerFormat - ) -} - -if (CONFIG.LOG.ROTATION.ENABLED) { - fileLoggerOptions.maxsize = CONFIG.LOG.ROTATION.MAX_FILE_SIZE - fileLoggerOptions.maxFiles = CONFIG.LOG.ROTATION.MAX_FILES -} - -function buildLogger (labelSuffix?: string) { - return createLogger({ - level: CONFIG.LOG.LEVEL, - defaultMeta: { - get traceId () { return getSpanContext(context.active())?.traceId }, - get spanId () { return getSpanContext(context.active())?.spanId }, - get traceFlags () { return getSpanContext(context.active())?.traceFlags } - }, - format: format.combine( - labelFormatter(labelSuffix), - format.splat() - ), - transports: [ - new transports.File(fileLoggerOptions), - new transports.Console({ - handleExceptions: true, - format: format.combine( - timestampFormatter, - format.colorize(), - consoleLoggerFormat - ) - }) - ], - exitOnError: true - }) -} - -const logger = buildLogger() - -// --------------------------------------------------------------------------- - -function bunyanLogFactory (level: string) { - return function (...params: any[]) { - let meta = null - let args = [].concat(params) - - if (arguments[0] instanceof Error) { - meta = arguments[0].toString() - args = Array.prototype.slice.call(arguments, 1) - args.push(meta) - } else if (typeof (args[0]) !== 'string') { - meta = arguments[0] - args = Array.prototype.slice.call(arguments, 1) - args.push(meta) - } - - logger[level].apply(logger, args) - } -} - -const bunyanLogger = { - level: () => { }, - trace: bunyanLogFactory('debug'), - debug: bunyanLogFactory('debug'), - verbose: bunyanLogFactory('debug'), - info: bunyanLogFactory('info'), - warn: bunyanLogFactory('warn'), - error: bunyanLogFactory('error'), - fatal: bunyanLogFactory('error') -} - -// --------------------------------------------------------------------------- - -type LoggerTagsFn = (...tags: string[]) => { tags: string[] } -function loggerTagsFactory (...defaultTags: string[]): LoggerTagsFn { - return (...tags: string[]) => { - return { tags: defaultTags.concat(tags) } - } -} - -// --------------------------------------------------------------------------- - -async function mtimeSortFilesDesc (files: string[], basePath: string) { - const promises = [] - const out: { file: string, mtime: number }[] = [] - - for (const file of files) { - const p = stat(basePath + '/' + file) - .then(stats => { - if (stats.isFile()) out.push({ file, mtime: stats.mtime.getTime() }) - }) - - promises.push(p) - } - - await Promise.all(promises) - - out.sort((a, b) => b.mtime - a.mtime) - - return out -} - -// --------------------------------------------------------------------------- - -export { - LoggerTagsFn, - - buildLogger, - timestampFormatter, - labelFormatter, - consoleLoggerFormat, - jsonLoggerFormat, - mtimeSortFilesDesc, - logger, - loggerTagsFactory, - bunyanLogger -} - -// --------------------------------------------------------------------------- - -function removeCyclicValues () { - const seen = new WeakSet() - - // Thanks: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Cyclic_object_value#Examples - return (key: string, value: any) => { - if (key === 'cert') return 'Replaced by the logger to avoid large log message' - - if (typeof value === 'object' && value !== null) { - if (seen.has(value)) return - - seen.add(value) - } - - if (value instanceof Set) { - return Array.from(value) - } - - if (value instanceof Map) { - return Array.from(value.entries()) - } - - if (value instanceof Error) { - const error = {} - - Object.getOwnPropertyNames(value).forEach(key => { error[key] = value[key] }) - - return error - } - - return value - } -} - -function getAdditionalInfo (info: any) { - const toOmit = [ 'label', 'timestamp', 'level', 'message', 'sql', 'tags' ] - - return omit(info, toOmit) -} diff --git a/server/helpers/markdown.ts b/server/helpers/markdown.ts deleted file mode 100644 index a20ac22d4..000000000 --- a/server/helpers/markdown.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { getDefaultSanitizeOptions, getTextOnlySanitizeOptions, TEXT_WITH_HTML_RULES } from '@shared/core-utils' - -const defaultSanitizeOptions = getDefaultSanitizeOptions() -const textOnlySanitizeOptions = getTextOnlySanitizeOptions() - -const sanitizeHtml = require('sanitize-html') -const markdownItEmoji = require('markdown-it-emoji/light') -const MarkdownItClass = require('markdown-it') - -const markdownItForSafeHtml = new MarkdownItClass('default', { linkify: true, breaks: true, html: true }) - .enable(TEXT_WITH_HTML_RULES) - .use(markdownItEmoji) - -const markdownItForPlainText = new MarkdownItClass('default', { linkify: false, breaks: true, html: false }) - .use(markdownItEmoji) - .use(plainTextPlugin) - -const toSafeHtml = (text: string) => { - if (!text) return '' - - // Restore line feed - const textWithLineFeed = text.replace(//g, '\r\n') - - // Convert possible markdown (emojis, emphasis and lists) to html - const html = markdownItForSafeHtml.render(textWithLineFeed) - - // Convert to safe Html - return sanitizeHtml(html, defaultSanitizeOptions) -} - -const mdToOneLinePlainText = (text: string) => { - if (!text) return '' - - markdownItForPlainText.render(text) - - // Convert to safe Html - return sanitizeHtml(markdownItForPlainText.plainText, textOnlySanitizeOptions) -} - -// --------------------------------------------------------------------------- - -export { - toSafeHtml, - mdToOneLinePlainText -} - -// --------------------------------------------------------------------------- - -// Thanks: https://github.com/wavesheep/markdown-it-plain-text -function plainTextPlugin (markdownIt: any) { - function plainTextRule (state: any) { - const text = scan(state.tokens) - - markdownIt.plainText = text - } - - function scan (tokens: any[]) { - let lastSeparator = '' - let text = '' - - function buildSeparator (token: any) { - if (token.type === 'list_item_close') { - lastSeparator = ', ' - } - - if (token.tag === 'br' || token.type === 'paragraph_close') { - lastSeparator = ' ' - } - } - - for (const token of tokens) { - buildSeparator(token) - - if (token.type !== 'inline') continue - - for (const child of token.children) { - buildSeparator(child) - - if (!child.content) continue - - text += lastSeparator + child.content - lastSeparator = '' - } - } - - return text - } - - markdownIt.core.ruler.push('plainText', plainTextRule) -} diff --git a/server/helpers/otp.ts b/server/helpers/otp.ts deleted file mode 100644 index a32cc9621..000000000 --- a/server/helpers/otp.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { Secret, TOTP } from 'otpauth' -import { CONFIG } from '@server/initializers/config' -import { WEBSERVER } from '@server/initializers/constants' -import { decrypt } from './peertube-crypto' - -async function isOTPValid (options: { - encryptedSecret: string - token: string -}) { - const { token, encryptedSecret } = options - - const secret = await decrypt(encryptedSecret, CONFIG.SECRETS.PEERTUBE) - - const totp = new TOTP({ - ...baseOTPOptions(), - - secret - }) - - const delta = totp.validate({ - token, - window: 1 - }) - - if (delta === null) return false - - return true -} - -function generateOTPSecret (email: string) { - const totp = new TOTP({ - ...baseOTPOptions(), - - label: email, - secret: new Secret() - }) - - return { - secret: totp.secret.base32, - uri: totp.toString() - } -} - -export { - isOTPValid, - generateOTPSecret -} - -// --------------------------------------------------------------------------- - -function baseOTPOptions () { - return { - issuer: WEBSERVER.HOST, - algorithm: 'SHA1', - digits: 6, - period: 30 - } -} diff --git a/server/helpers/peertube-crypto.ts b/server/helpers/peertube-crypto.ts deleted file mode 100644 index 95e78a904..000000000 --- a/server/helpers/peertube-crypto.ts +++ /dev/null @@ -1,208 +0,0 @@ -import { compare, genSalt, hash } from 'bcrypt' -import { createCipheriv, createDecipheriv, createSign, createVerify } from 'crypto' -import { Request } from 'express' -import { cloneDeep } from 'lodash' -import { promisify1, promisify2 } from '@shared/core-utils' -import { sha256 } from '@shared/extra-utils' -import { BCRYPT_SALT_SIZE, ENCRYPTION, HTTP_SIGNATURE, PRIVATE_RSA_KEY_SIZE } from '../initializers/constants' -import { MActor } from '../types/models' -import { generateRSAKeyPairPromise, randomBytesPromise, scryptPromise } from './core-utils' -import { jsonld } from './custom-jsonld-signature' -import { logger } from './logger' - -const bcryptComparePromise = promisify2(compare) -const bcryptGenSaltPromise = promisify1(genSalt) -const bcryptHashPromise = promisify2(hash) - -const httpSignature = require('@peertube/http-signature') - -function createPrivateAndPublicKeys () { - logger.info('Generating a RSA key...') - - return generateRSAKeyPairPromise(PRIVATE_RSA_KEY_SIZE) -} - -// --------------------------------------------------------------------------- -// User password checks -// --------------------------------------------------------------------------- - -function comparePassword (plainPassword: string, hashPassword: string) { - if (!plainPassword) return Promise.resolve(false) - - return bcryptComparePromise(plainPassword, hashPassword) -} - -async function cryptPassword (password: string) { - const salt = await bcryptGenSaltPromise(BCRYPT_SALT_SIZE) - - return bcryptHashPromise(password, salt) -} - -// --------------------------------------------------------------------------- -// HTTP Signature -// --------------------------------------------------------------------------- - -function isHTTPSignatureDigestValid (rawBody: Buffer, req: Request): boolean { - if (req.headers[HTTP_SIGNATURE.HEADER_NAME] && req.headers['digest']) { - return buildDigest(rawBody.toString()) === req.headers['digest'] - } - - return true -} - -function isHTTPSignatureVerified (httpSignatureParsed: any, actor: MActor): boolean { - return httpSignature.verifySignature(httpSignatureParsed, actor.publicKey) === true -} - -function parseHTTPSignature (req: Request, clockSkew?: number) { - const requiredHeaders = req.method === 'POST' - ? [ '(request-target)', 'host', 'digest' ] - : [ '(request-target)', 'host' ] - - const parsed = httpSignature.parse(req, { clockSkew, headers: requiredHeaders }) - - const parsedHeaders = parsed.params.headers - if (!parsedHeaders.includes('date') && !parsedHeaders.includes('(created)')) { - throw new Error(`date or (created) must be included in signature`) - } - - return parsed -} - -// --------------------------------------------------------------------------- -// JSONLD -// --------------------------------------------------------------------------- - -function isJsonLDSignatureVerified (fromActor: MActor, signedDocument: any): Promise { - if (signedDocument.signature.type === 'RsaSignature2017') { - return isJsonLDRSA2017Verified(fromActor, signedDocument) - } - - logger.warn('Unknown JSON LD signature %s.', signedDocument.signature.type, signedDocument) - - return Promise.resolve(false) -} - -// Backward compatibility with "other" implementations -async function isJsonLDRSA2017Verified (fromActor: MActor, signedDocument: any) { - const [ documentHash, optionsHash ] = await Promise.all([ - createDocWithoutSignatureHash(signedDocument), - createSignatureHash(signedDocument.signature) - ]) - - const toVerify = optionsHash + documentHash - - const verify = createVerify('RSA-SHA256') - verify.update(toVerify, 'utf8') - - return verify.verify(fromActor.publicKey, signedDocument.signature.signatureValue, 'base64') -} - -async function signJsonLDObject (byActor: MActor, data: T) { - const signature = { - type: 'RsaSignature2017', - creator: byActor.url, - created: new Date().toISOString() - } - - const [ documentHash, optionsHash ] = await Promise.all([ - createDocWithoutSignatureHash(data), - createSignatureHash(signature) - ]) - - const toSign = optionsHash + documentHash - - const sign = createSign('RSA-SHA256') - sign.update(toSign, 'utf8') - - const signatureValue = sign.sign(byActor.privateKey, 'base64') - Object.assign(signature, { signatureValue }) - - return Object.assign(data, { signature }) -} - -// --------------------------------------------------------------------------- - -function buildDigest (body: any) { - const rawBody = typeof body === 'string' ? body : JSON.stringify(body) - - return 'SHA-256=' + sha256(rawBody, 'base64') -} - -// --------------------------------------------------------------------------- -// Encryption -// --------------------------------------------------------------------------- - -async function encrypt (str: string, secret: string) { - const iv = await randomBytesPromise(ENCRYPTION.IV) - - const key = await scryptPromise(secret, ENCRYPTION.SALT, 32) - const cipher = createCipheriv(ENCRYPTION.ALGORITHM, key, iv) - - let encrypted = iv.toString(ENCRYPTION.ENCODING) + ':' - encrypted += cipher.update(str, 'utf8', ENCRYPTION.ENCODING) - encrypted += cipher.final(ENCRYPTION.ENCODING) - - return encrypted -} - -async function decrypt (encryptedArg: string, secret: string) { - const [ ivStr, encryptedStr ] = encryptedArg.split(':') - - const iv = Buffer.from(ivStr, 'hex') - const key = await scryptPromise(secret, ENCRYPTION.SALT, 32) - - const decipher = createDecipheriv(ENCRYPTION.ALGORITHM, key, iv) - - return decipher.update(encryptedStr, ENCRYPTION.ENCODING, 'utf8') + decipher.final('utf8') -} - -// --------------------------------------------------------------------------- - -export { - isHTTPSignatureDigestValid, - parseHTTPSignature, - isHTTPSignatureVerified, - buildDigest, - isJsonLDSignatureVerified, - comparePassword, - createPrivateAndPublicKeys, - cryptPassword, - signJsonLDObject, - - encrypt, - decrypt -} - -// --------------------------------------------------------------------------- - -function hashObject (obj: any): Promise { - return jsonld.promises.normalize(obj, { - safe: false, - algorithm: 'URDNA2015', - format: 'application/n-quads' - }).then(res => sha256(res)) -} - -function createSignatureHash (signature: any) { - const signatureCopy = cloneDeep(signature) - Object.assign(signatureCopy, { - '@context': [ - 'https://w3id.org/security/v1', - { RsaSignature2017: 'https://w3id.org/security#RsaSignature2017' } - ] - }) - - delete signatureCopy.type - delete signatureCopy.id - delete signatureCopy.signatureValue - - return hashObject(signatureCopy) -} - -function createDocWithoutSignatureHash (doc: any) { - const docWithoutSignature = cloneDeep(doc) - delete docWithoutSignature.signature - - return hashObject(docWithoutSignature) -} diff --git a/server/helpers/query.ts b/server/helpers/query.ts deleted file mode 100644 index c0f78368f..000000000 --- a/server/helpers/query.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { pick } from '@shared/core-utils' -import { - VideoChannelsSearchQueryAfterSanitize, - VideoPlaylistsSearchQueryAfterSanitize, - VideosCommonQueryAfterSanitize, - VideosSearchQueryAfterSanitize -} from '@shared/models' - -function pickCommonVideoQuery (query: VideosCommonQueryAfterSanitize) { - return pick(query, [ - 'start', - 'count', - 'sort', - 'nsfw', - 'isLive', - 'categoryOneOf', - 'licenceOneOf', - 'languageOneOf', - 'privacyOneOf', - 'tagsOneOf', - 'tagsAllOf', - 'isLocal', - 'include', - 'skipCount', - 'hasHLSFiles', - 'hasWebtorrentFiles', // TODO: Remove in v7 - 'hasWebVideoFiles', - 'search', - 'excludeAlreadyWatched' - ]) -} - -function pickSearchVideoQuery (query: VideosSearchQueryAfterSanitize) { - return { - ...pickCommonVideoQuery(query), - - ...pick(query, [ - 'searchTarget', - 'host', - 'startDate', - 'endDate', - 'originallyPublishedStartDate', - 'originallyPublishedEndDate', - 'durationMin', - 'durationMax', - 'uuids', - 'excludeAlreadyWatched' - ]) - } -} - -function pickSearchChannelQuery (query: VideoChannelsSearchQueryAfterSanitize) { - return pick(query, [ - 'searchTarget', - 'search', - 'start', - 'count', - 'sort', - 'host', - 'handles' - ]) -} - -function pickSearchPlaylistQuery (query: VideoPlaylistsSearchQueryAfterSanitize) { - return pick(query, [ - 'searchTarget', - 'search', - 'start', - 'count', - 'sort', - 'host', - 'uuids' - ]) -} - -export { - pickCommonVideoQuery, - pickSearchVideoQuery, - pickSearchPlaylistQuery, - pickSearchChannelQuery -} diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts deleted file mode 100644 index 1625d6e49..000000000 --- a/server/helpers/requests.ts +++ /dev/null @@ -1,246 +0,0 @@ -import { createWriteStream, remove } from 'fs-extra' -import got, { CancelableRequest, NormalizedOptions, Options as GotOptions, RequestError, Response } from 'got' -import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent' -import { ACTIVITY_PUB, BINARY_CONTENT_TYPES, PEERTUBE_VERSION, REQUEST_TIMEOUTS, WEBSERVER } from '../initializers/constants' -import { pipelinePromise } from './core-utils' -import { logger, loggerTagsFactory } from './logger' -import { getProxy, isProxyEnabled } from './proxy' - -const lTags = loggerTagsFactory('request') - -const httpSignature = require('@peertube/http-signature') - -export interface PeerTubeRequestError extends Error { - statusCode?: number - responseBody?: any - responseHeaders?: any - requestHeaders?: any -} - -type PeerTubeRequestOptions = { - timeout?: number - activityPub?: boolean - bodyKBLimit?: number // 1MB - - httpSignature?: { - algorithm: string - authorizationHeaderName: string - keyId: string - key: string - headers: string[] - } - - jsonResponse?: boolean - - followRedirect?: boolean -} & Pick - -const peertubeGot = got.extend({ - ...getAgent(), - - headers: { - 'user-agent': getUserAgent() - }, - - handlers: [ - (options, next) => { - const promiseOrStream = next(options) as CancelableRequest - const bodyKBLimit = options.context?.bodyKBLimit as number - if (!bodyKBLimit) throw new Error('No KB limit for this request') - - const bodyLimit = bodyKBLimit * 1000 - - /* eslint-disable @typescript-eslint/no-floating-promises */ - promiseOrStream.on('downloadProgress', progress => { - if (progress.transferred > bodyLimit && progress.percent !== 1) { - const message = `Exceeded the download limit of ${bodyLimit} B` - logger.warn(message, lTags()) - - // CancelableRequest - if (promiseOrStream.cancel) { - promiseOrStream.cancel() - return - } - - // Stream - (promiseOrStream as any).destroy() - } - }) - - return promiseOrStream - } - ], - - hooks: { - beforeRequest: [ - options => { - const headers = options.headers || {} - headers['host'] = options.url.host - }, - - options => { - const httpSignatureOptions = options.context?.httpSignature - - if (httpSignatureOptions) { - const method = options.method ?? 'GET' - const path = options.path ?? options.url.pathname - - if (!method || !path) { - throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`) - } - - httpSignature.signRequest({ - getHeader: function (header: string) { - const value = options.headers[header.toLowerCase()] - - if (!value) logger.warn('Unknown header requested by http-signature.', { headers: options.headers, header }) - return value - }, - - setHeader: function (header: string, value: string) { - options.headers[header] = value - }, - - method, - path - }, httpSignatureOptions) - } - } - ], - - beforeRetry: [ - (_options: NormalizedOptions, error: RequestError, retryCount: number) => { - logger.debug('Retrying request to %s.', error.request.requestUrl, { retryCount, error: buildRequestError(error), ...lTags() }) - } - ] - } -}) - -function doRequest (url: string, options: PeerTubeRequestOptions = {}) { - const gotOptions = buildGotOptions(options) - - return peertubeGot(url, gotOptions) - .catch(err => { throw buildRequestError(err) }) -} - -function doJSONRequest (url: string, options: PeerTubeRequestOptions = {}) { - const gotOptions = buildGotOptions(options) - - return peertubeGot(url, { ...gotOptions, responseType: 'json' }) - .catch(err => { throw buildRequestError(err) }) -} - -async function doRequestAndSaveToFile ( - url: string, - destPath: string, - options: PeerTubeRequestOptions = {} -) { - const gotOptions = buildGotOptions({ ...options, timeout: options.timeout ?? REQUEST_TIMEOUTS.FILE }) - - const outFile = createWriteStream(destPath) - - try { - await pipelinePromise( - peertubeGot.stream(url, gotOptions), - outFile - ) - } catch (err) { - remove(destPath) - .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err, ...lTags() })) - - throw buildRequestError(err) - } -} - -function getAgent () { - if (!isProxyEnabled()) return {} - - const proxy = getProxy() - - logger.info('Using proxy %s.', proxy, lTags()) - - const proxyAgentOptions = { - keepAlive: true, - keepAliveMsecs: 1000, - maxSockets: 256, - maxFreeSockets: 256, - scheduling: 'lifo' as 'lifo', - proxy - } - - return { - agent: { - http: new HttpProxyAgent(proxyAgentOptions), - https: new HttpsProxyAgent(proxyAgentOptions) - } - } -} - -function getUserAgent () { - return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})` -} - -function isBinaryResponse (result: Response) { - return BINARY_CONTENT_TYPES.has(result.headers['content-type']) -} - -// --------------------------------------------------------------------------- - -export { - PeerTubeRequestOptions, - - doRequest, - doJSONRequest, - doRequestAndSaveToFile, - isBinaryResponse, - getAgent, - peertubeGot -} - -// --------------------------------------------------------------------------- - -function buildGotOptions (options: PeerTubeRequestOptions) { - const { activityPub, bodyKBLimit = 1000 } = options - - const context = { bodyKBLimit, httpSignature: options.httpSignature } - - let headers = options.headers || {} - - if (!headers.date) { - headers = { ...headers, date: new Date().toUTCString() } - } - - if (activityPub && !headers.accept) { - headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER } - } - - return { - method: options.method, - dnsCache: true, - timeout: options.timeout ?? REQUEST_TIMEOUTS.DEFAULT, - json: options.json, - searchParams: options.searchParams, - followRedirect: options.followRedirect, - retry: 2, - headers, - context - } -} - -function buildRequestError (error: RequestError) { - const newError: PeerTubeRequestError = new Error(error.message) - newError.name = error.name - newError.stack = error.stack - - if (error.response) { - newError.responseBody = error.response.body - newError.responseHeaders = error.response.headers - newError.statusCode = error.response.statusCode - } - - if (error.options) { - newError.requestHeaders = error.options.headers - } - - return newError -} diff --git a/server/helpers/token-generator.ts b/server/helpers/token-generator.ts deleted file mode 100644 index 16313b818..000000000 --- a/server/helpers/token-generator.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { buildUUID } from '@shared/extra-utils' - -function generateRunnerRegistrationToken () { - return 'ptrrt-' + buildUUID() -} - -function generateRunnerToken () { - return 'ptrt-' + buildUUID() -} - -function generateRunnerJobToken () { - return 'ptrjt-' + buildUUID() -} - -export { - generateRunnerRegistrationToken, - generateRunnerToken, - generateRunnerJobToken -} diff --git a/server/helpers/upload.ts b/server/helpers/upload.ts deleted file mode 100644 index f5f476913..000000000 --- a/server/helpers/upload.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { join } from 'path' -import { DIRECTORIES } from '@server/initializers/constants' - -function getResumableUploadPath (filename?: string) { - if (filename) return join(DIRECTORIES.RESUMABLE_UPLOAD, filename) - - return DIRECTORIES.RESUMABLE_UPLOAD -} - -// --------------------------------------------------------------------------- - -export { - getResumableUploadPath -} diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts deleted file mode 100644 index 5a4fe4fdd..000000000 --- a/server/helpers/utils.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { remove } from 'fs-extra' -import { Instance as ParseTorrent } from 'parse-torrent' -import { join } from 'path' -import { sha256 } from '@shared/extra-utils' -import { ResultList } from '@shared/models' -import { CONFIG } from '../initializers/config' -import { randomBytesPromise } from './core-utils' -import { logger } from './logger' - -function deleteFileAndCatch (path: string) { - remove(path) - .catch(err => logger.error('Cannot delete the file %s asynchronously.', path, { err })) -} - -async function generateRandomString (size: number) { - const raw = await randomBytesPromise(size) - - return raw.toString('hex') -} - -interface FormattableToJSON { - toFormattedJSON (args?: U): V -} - -function getFormattedObjects> (objects: T[], objectsTotal: number, formattedArg?: U) { - const formattedObjects = objects.map(o => o.toFormattedJSON(formattedArg)) - - return { - total: objectsTotal, - data: formattedObjects - } as ResultList -} - -function generateVideoImportTmpPath (target: string | ParseTorrent, extension = '.mp4') { - const id = typeof target === 'string' - ? target - : target.infoHash - - const hash = sha256(id) - return join(CONFIG.STORAGE.TMP_DIR, `${hash}-import${extension}`) -} - -function getSecureTorrentName (originalName: string) { - return sha256(originalName) + '.torrent' -} - -/** - * From a filename like "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3.mp4", returns - * only the "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3" part. If the filename does - * not contain a UUID, returns null. - */ -function getUUIDFromFilename (filename: string) { - const regex = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/ - const result = filename.match(regex) - - if (!result || Array.isArray(result) === false) return null - - return result[0] -} - -// --------------------------------------------------------------------------- - -export { - deleteFileAndCatch, - generateRandomString, - getFormattedObjects, - getSecureTorrentName, - generateVideoImportTmpPath, - getUUIDFromFilename -} diff --git a/server/helpers/version.ts b/server/helpers/version.ts deleted file mode 100644 index 5b3bf59dd..000000000 --- a/server/helpers/version.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { execPromise, execPromise2 } from './core-utils' -import { logger } from './logger' - -async function getServerCommit () { - try { - const tag = await execPromise2( - '[ ! -d .git ] || git name-rev --name-only --tags --no-undefined HEAD 2>/dev/null || true', - { stdio: [ 0, 1, 2 ] } - ) - - if (tag) return tag.replace(/^v/, '') - } catch (err) { - logger.debug('Cannot get version from git tags.', { err }) - } - - try { - const version = await execPromise('[ ! -d .git ] || git rev-parse --short HEAD') - - if (version) return version.toString().trim() - } catch (err) { - logger.debug('Cannot get version from git HEAD.', { err }) - } - - return '' -} - -function getNodeABIVersion () { - const version = process.versions.modules - - return parseInt(version) -} - -export { - getServerCommit, - getNodeABIVersion -} diff --git a/server/helpers/video.ts b/server/helpers/video.ts deleted file mode 100644 index c688ef1e3..000000000 --- a/server/helpers/video.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { Response } from 'express' -import { CONFIG } from '@server/initializers/config' -import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models' -import { VideoPrivacy, VideoState } from '@shared/models' -import { forceNumber } from '@shared/core-utils' - -function getVideoWithAttributes (res: Response) { - return res.locals.videoAPI || res.locals.videoAll || res.locals.onlyVideo -} - -function extractVideo (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) { - return isStreamingPlaylist(videoOrPlaylist) - ? videoOrPlaylist.Video - : videoOrPlaylist -} - -function isPrivacyForFederation (privacy: VideoPrivacy) { - const castedPrivacy = forceNumber(privacy) - - return castedPrivacy === VideoPrivacy.PUBLIC || - (CONFIG.FEDERATION.VIDEOS.FEDERATE_UNLISTED === true && castedPrivacy === VideoPrivacy.UNLISTED) -} - -function isStateForFederation (state: VideoState) { - const castedState = forceNumber(state) - - return castedState === VideoState.PUBLISHED || castedState === VideoState.WAITING_FOR_LIVE || castedState === VideoState.LIVE_ENDED -} - -function getPrivaciesForFederation () { - return (CONFIG.FEDERATION.VIDEOS.FEDERATE_UNLISTED === true) - ? [ { privacy: VideoPrivacy.PUBLIC }, { privacy: VideoPrivacy.UNLISTED } ] - : [ { privacy: VideoPrivacy.PUBLIC } ] -} - -function getExtFromMimetype (mimeTypes: { [id: string]: string | string[] }, mimeType: string) { - const value = mimeTypes[mimeType] - - if (Array.isArray(value)) return value[0] - - return value -} - -export { - getVideoWithAttributes, - extractVideo, - getExtFromMimetype, - isStateForFederation, - isPrivacyForFederation, - getPrivaciesForFederation -} diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts deleted file mode 100644 index f33a7bccd..000000000 --- a/server/helpers/webtorrent.ts +++ /dev/null @@ -1,258 +0,0 @@ -import { decode, encode } from 'bencode' -import createTorrent from 'create-torrent' -import { createWriteStream, ensureDir, pathExists, readFile, remove, writeFile } from 'fs-extra' -import { encode as magnetUriEncode } from 'magnet-uri' -import parseTorrent from 'parse-torrent' -import { dirname, join } from 'path' -import { pipeline } from 'stream' -import WebTorrent, { Instance, TorrentFile } from 'webtorrent' -import { isArray } from '@server/helpers/custom-validators/misc' -import { WEBSERVER } from '@server/initializers/constants' -import { generateTorrentFileName } from '@server/lib/paths' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { MVideo } from '@server/types/models/video/video' -import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file' -import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist' -import { promisify2 } from '@shared/core-utils' -import { sha1 } from '@shared/extra-utils' -import { CONFIG } from '../initializers/config' -import { logger } from './logger' -import { generateVideoImportTmpPath } from './utils' -import { extractVideo } from './video' - -const createTorrentPromise = promisify2(createTorrent) - -async function downloadWebTorrentVideo (target: { uri: string, torrentName?: string }, timeout: number) { - const id = target.uri || target.torrentName - let timer - - const path = generateVideoImportTmpPath(id) - logger.info('Importing torrent video %s', id) - - const directoryPath = join(CONFIG.STORAGE.TMP_DIR, 'webtorrent') - await ensureDir(directoryPath) - - return new Promise((res, rej) => { - const webtorrent = new WebTorrent() - let file: TorrentFile - - const torrentId = target.uri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName) - - const options = { path: directoryPath } - const torrent = webtorrent.add(torrentId, options, torrent => { - if (torrent.files.length !== 1) { - if (timer) clearTimeout(timer) - - for (const file of torrent.files) { - deleteDownloadedFile({ directoryPath, filepath: file.path }) - } - - return safeWebtorrentDestroy(webtorrent, torrentId, undefined, target.torrentName) - .then(() => rej(new Error('Cannot import torrent ' + torrentId + ': there are multiple files in it'))) - } - - logger.debug('Got torrent from webtorrent %s.', id, { infoHash: torrent.infoHash }) - - file = torrent.files[0] - - // FIXME: avoid creating another stream when https://github.com/webtorrent/webtorrent/issues/1517 is fixed - const writeStream = createWriteStream(path) - writeStream.on('finish', () => { - if (timer) clearTimeout(timer) - - safeWebtorrentDestroy(webtorrent, torrentId, { directoryPath, filepath: file.path }, target.torrentName) - .then(() => res(path)) - .catch(err => logger.error('Cannot destroy webtorrent.', { err })) - }) - - pipeline( - file.createReadStream(), - writeStream, - err => { - if (err) rej(err) - } - ) - }) - - torrent.on('error', err => rej(err)) - - timer = setTimeout(() => { - const err = new Error('Webtorrent download timeout.') - - safeWebtorrentDestroy(webtorrent, torrentId, file ? { directoryPath, filepath: file.path } : undefined, target.torrentName) - .then(() => rej(err)) - .catch(destroyErr => { - logger.error('Cannot destroy webtorrent.', { err: destroyErr }) - rej(err) - }) - - }, timeout) - }) -} - -function createTorrentAndSetInfoHash (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { - return VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(videoOrPlaylist), videoPath => { - return createTorrentAndSetInfoHashFromPath(videoOrPlaylist, videoFile, videoPath) - }) -} - -async function createTorrentAndSetInfoHashFromPath ( - videoOrPlaylist: MVideo | MStreamingPlaylistVideo, - videoFile: MVideoFile, - filePath: string -) { - const video = extractVideo(videoOrPlaylist) - - const options = { - // Keep the extname, it's used by the client to stream the file inside a web browser - name: buildInfoName(video, videoFile), - createdBy: 'PeerTube', - announceList: buildAnnounceList(), - urlList: buildUrlList(video, videoFile) - } - - const torrentContent = await createTorrentPromise(filePath, options) - - const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution) - const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename) - logger.info('Creating torrent %s.', torrentPath) - - await writeFile(torrentPath, torrentContent) - - // Remove old torrent file if it existed - if (videoFile.hasTorrent()) { - await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)) - } - - const parsedTorrent = parseTorrent(torrentContent) - videoFile.infoHash = parsedTorrent.infoHash - videoFile.torrentFilename = torrentFilename -} - -async function updateTorrentMetadata (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { - const video = extractVideo(videoOrPlaylist) - - const oldTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) - - if (!await pathExists(oldTorrentPath)) { - logger.info('Do not update torrent metadata %s of video %s because the file does not exist anymore.', video.uuid, oldTorrentPath) - return - } - - const torrentContent = await readFile(oldTorrentPath) - const decoded = decode(torrentContent) - - decoded['announce-list'] = buildAnnounceList() - decoded.announce = decoded['announce-list'][0][0] - - decoded['url-list'] = buildUrlList(video, videoFile) - - decoded.info.name = buildInfoName(video, videoFile) - decoded['creation date'] = Math.ceil(Date.now() / 1000) - - const newTorrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution) - const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, newTorrentFilename) - - logger.info('Updating torrent metadata %s -> %s.', oldTorrentPath, newTorrentPath) - - await writeFile(newTorrentPath, encode(decoded)) - await remove(oldTorrentPath) - - videoFile.torrentFilename = newTorrentFilename - videoFile.infoHash = sha1(encode(decoded.info)) -} - -function generateMagnetUri ( - video: MVideo, - videoFile: MVideoFileRedundanciesOpt, - trackerUrls: string[] -) { - const xs = videoFile.getTorrentUrl() - const announce = trackerUrls - - let urlList = video.hasPrivateStaticPath() - ? [] - : [ videoFile.getFileUrl(video) ] - - const redundancies = videoFile.RedundancyVideos - if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl)) - - const magnetHash = { - xs, - announce, - urlList, - infoHash: videoFile.infoHash, - name: video.name - } - - return magnetUriEncode(magnetHash) -} - -// --------------------------------------------------------------------------- - -export { - createTorrentPromise, - updateTorrentMetadata, - - createTorrentAndSetInfoHash, - createTorrentAndSetInfoHashFromPath, - - generateMagnetUri, - downloadWebTorrentVideo -} - -// --------------------------------------------------------------------------- - -function safeWebtorrentDestroy ( - webtorrent: Instance, - torrentId: string, - downloadedFile?: { directoryPath: string, filepath: string }, - torrentName?: string -) { - return new Promise(res => { - webtorrent.destroy(err => { - // Delete torrent file - if (torrentName) { - logger.debug('Removing %s torrent after webtorrent download.', torrentId) - remove(torrentId) - .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err })) - } - - // Delete downloaded file - if (downloadedFile) deleteDownloadedFile(downloadedFile) - - if (err) logger.warn('Cannot destroy webtorrent in timeout.', { err }) - - return res() - }) - }) -} - -function deleteDownloadedFile (downloadedFile: { directoryPath: string, filepath: string }) { - // We want to delete the base directory - let pathToDelete = dirname(downloadedFile.filepath) - if (pathToDelete === '.') pathToDelete = downloadedFile.filepath - - const toRemovePath = join(downloadedFile.directoryPath, pathToDelete) - - logger.debug('Removing %s after webtorrent download.', toRemovePath) - remove(toRemovePath) - .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', toRemovePath, { err })) -} - -function buildAnnounceList () { - return [ - [ WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' ], - [ WEBSERVER.URL + '/tracker/announce' ] - ] -} - -function buildUrlList (video: MVideo, videoFile: MVideoFile) { - if (video.hasPrivateStaticPath()) return [] - - return [ videoFile.getFileUrl(video) ] -} - -function buildInfoName (video: MVideo, videoFile: MVideoFile) { - return `${video.name} ${videoFile.resolution}p${videoFile.extname}` -} diff --git a/server/helpers/youtube-dl/index.ts b/server/helpers/youtube-dl/index.ts deleted file mode 100644 index 6afc77dcf..000000000 --- a/server/helpers/youtube-dl/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './youtube-dl-cli' -export * from './youtube-dl-info-builder' -export * from './youtube-dl-wrapper' diff --git a/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/helpers/youtube-dl/youtube-dl-cli.ts deleted file mode 100644 index 765038cea..000000000 --- a/server/helpers/youtube-dl/youtube-dl-cli.ts +++ /dev/null @@ -1,259 +0,0 @@ -import execa from 'execa' -import { ensureDir, pathExists, writeFile } from 'fs-extra' -import { dirname, join } from 'path' -import { CONFIG } from '@server/initializers/config' -import { VideoResolution } from '@shared/models' -import { logger, loggerTagsFactory } from '../logger' -import { getProxy, isProxyEnabled } from '../proxy' -import { isBinaryResponse, peertubeGot } from '../requests' -import { OptionsOfBufferResponseBody } from 'got/dist/source' - -const lTags = loggerTagsFactory('youtube-dl') - -const youtubeDLBinaryPath = join(CONFIG.STORAGE.BIN_DIR, CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME) - -export class YoutubeDLCLI { - - static async safeGet () { - if (!await pathExists(youtubeDLBinaryPath)) { - await ensureDir(dirname(youtubeDLBinaryPath)) - - await this.updateYoutubeDLBinary() - } - - return new YoutubeDLCLI() - } - - static async updateYoutubeDLBinary () { - const url = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.URL - - logger.info('Updating youtubeDL binary from %s.', url, lTags()) - - const gotOptions: OptionsOfBufferResponseBody = { - context: { bodyKBLimit: 20_000 }, - responseType: 'buffer' as 'buffer' - } - - if (process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN) { - gotOptions.headers = { - authorization: 'Bearer ' + process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN - } - } - - try { - let gotResult = await peertubeGot(url, gotOptions) - - if (!isBinaryResponse(gotResult)) { - const json = JSON.parse(gotResult.body.toString()) - const latest = json.filter(release => release.prerelease === false)[0] - if (!latest) throw new Error('Cannot find latest release') - - const releaseName = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME - const releaseAsset = latest.assets.find(a => a.name === releaseName) - if (!releaseAsset) throw new Error(`Cannot find appropriate release with name ${releaseName} in release assets`) - - gotResult = await peertubeGot(releaseAsset.browser_download_url, gotOptions) - } - - if (!isBinaryResponse(gotResult)) { - throw new Error('Not a binary response') - } - - await writeFile(youtubeDLBinaryPath, gotResult.body) - - logger.info('youtube-dl updated %s.', youtubeDLBinaryPath, lTags()) - } catch (err) { - logger.error('Cannot update youtube-dl from %s.', url, { err, ...lTags() }) - } - } - - static getYoutubeDLVideoFormat (enabledResolutions: VideoResolution[], useBestFormat: boolean) { - /** - * list of format selectors in order or preference - * see https://github.com/ytdl-org/youtube-dl#format-selection - * - * case #1 asks for a mp4 using h264 (avc1) and the exact resolution in the hope - * of being able to do a "quick-transcode" - * case #2 is the first fallback. No "quick-transcode" means we can get anything else (like vp9) - * case #3 is the resolution-degraded equivalent of #1, and already a pretty safe fallback - * - * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499 - **/ - - let result: string[] = [] - - if (!useBestFormat) { - const resolution = enabledResolutions.length === 0 - ? VideoResolution.H_720P - : Math.max(...enabledResolutions) - - result = [ - `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1 - `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2 - `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]` // case # - ] - } - - return result.concat([ - 'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio', - 'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats - 'bestvideo[ext=mp4]+bestaudio[ext=m4a]', - 'best' // Ultimate fallback - ]).join('/') - } - - private constructor () { - - } - - download (options: { - url: string - format: string - output: string - processOptions: execa.NodeOptions - timeout?: number - additionalYoutubeDLArgs?: string[] - }) { - let args = options.additionalYoutubeDLArgs || [] - args = args.concat([ '--merge-output-format', 'mp4', '-f', options.format, '-o', options.output ]) - - return this.run({ - url: options.url, - processOptions: options.processOptions, - timeout: options.timeout, - args - }) - } - - async getInfo (options: { - url: string - format: string - processOptions: execa.NodeOptions - additionalYoutubeDLArgs?: string[] - }) { - const { url, format, additionalYoutubeDLArgs = [], processOptions } = options - - const completeArgs = additionalYoutubeDLArgs.concat([ '--dump-json', '-f', format ]) - - const data = await this.run({ url, args: completeArgs, processOptions }) - if (!data) return undefined - - const info = data.map(d => JSON.parse(d)) - - return info.length === 1 - ? info[0] - : info - } - - async getListInfo (options: { - url: string - latestVideosCount?: number - processOptions: execa.NodeOptions - }): Promise<{ upload_date: string, webpage_url: string }[]> { - const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ] - - if (CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME === 'yt-dlp') { - // Optimize listing videos only when using yt-dlp because it is bugged with youtube-dl when fetching a channel - additionalYoutubeDLArgs.push('--flat-playlist') - } - - if (options.latestVideosCount !== undefined) { - additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString()) - } - - const result = await this.getInfo({ - url: options.url, - format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false), - processOptions: options.processOptions, - additionalYoutubeDLArgs - }) - - if (!result) return result - if (!Array.isArray(result)) return [ result ] - - return result - } - - async getSubs (options: { - url: string - format: 'vtt' - processOptions: execa.NodeOptions - }) { - const { url, format, processOptions } = options - - const args = [ '--skip-download', '--all-subs', `--sub-format=${format}` ] - - const data = await this.run({ url, args, processOptions }) - const files: string[] = [] - - const skipString = '[info] Writing video subtitles to: ' - - for (let i = 0, len = data.length; i < len; i++) { - const line = data[i] - - if (line.indexOf(skipString) === 0) { - files.push(line.slice(skipString.length)) - } - } - - return files - } - - private async run (options: { - url: string - args: string[] - timeout?: number - processOptions: execa.NodeOptions - }) { - const { url, args, timeout, processOptions } = options - - let completeArgs = this.wrapWithProxyOptions(args) - completeArgs = this.wrapWithIPOptions(completeArgs) - completeArgs = this.wrapWithFFmpegOptions(completeArgs) - - const { PYTHON_PATH } = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE - const subProcess = execa(PYTHON_PATH, [ youtubeDLBinaryPath, ...completeArgs, url ], processOptions) - - if (timeout) { - setTimeout(() => subProcess.cancel(), timeout) - } - - const output = await subProcess - - logger.debug('Run youtube-dl command.', { command: output.command, ...lTags() }) - - return output.stdout - ? output.stdout.trim().split(/\r?\n/) - : undefined - } - - private wrapWithProxyOptions (args: string[]) { - if (isProxyEnabled()) { - logger.debug('Using proxy %s for YoutubeDL', getProxy(), lTags()) - - return [ '--proxy', getProxy() ].concat(args) - } - - return args - } - - private wrapWithIPOptions (args: string[]) { - if (CONFIG.IMPORT.VIDEOS.HTTP.FORCE_IPV4) { - logger.debug('Force ipv4 for YoutubeDL') - - return [ '--force-ipv4' ].concat(args) - } - - return args - } - - private wrapWithFFmpegOptions (args: string[]) { - if (process.env.FFMPEG_PATH) { - logger.debug('Using ffmpeg location %s for YoutubeDL', process.env.FFMPEG_PATH, lTags()) - - return [ '--ffmpeg-location', process.env.FFMPEG_PATH ].concat(args) - } - - return args - } -} diff --git a/server/helpers/youtube-dl/youtube-dl-info-builder.ts b/server/helpers/youtube-dl/youtube-dl-info-builder.ts deleted file mode 100644 index a74904e43..000000000 --- a/server/helpers/youtube-dl/youtube-dl-info-builder.ts +++ /dev/null @@ -1,205 +0,0 @@ -import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../../initializers/constants' -import { peertubeTruncate } from '../core-utils' -import { isUrlValid } from '../custom-validators/activitypub/misc' - -type YoutubeDLInfo = { - name?: string - description?: string - category?: number - language?: string - licence?: number - nsfw?: boolean - tags?: string[] - thumbnailUrl?: string - ext?: string - originallyPublishedAtWithoutTime?: Date - webpageUrl?: string - - urls?: string[] -} - -class YoutubeDLInfoBuilder { - private readonly info: any - - constructor (info: any) { - this.info = { ...info } - } - - getInfo () { - const obj = this.buildVideoInfo(this.normalizeObject(this.info)) - if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video' - - return obj - } - - private normalizeObject (obj: any) { - const newObj: any = {} - - for (const key of Object.keys(obj)) { - // Deprecated key - if (key === 'resolution') continue - - const value = obj[key] - - if (typeof value === 'string') { - newObj[key] = value.normalize() - } else { - newObj[key] = value - } - } - - return newObj - } - - private buildOriginallyPublishedAt (obj: any) { - let originallyPublishedAt: Date = null - - const uploadDateMatcher = /^(\d{4})(\d{2})(\d{2})$/.exec(obj.upload_date) - if (uploadDateMatcher) { - originallyPublishedAt = new Date() - originallyPublishedAt.setHours(0, 0, 0, 0) - - const year = parseInt(uploadDateMatcher[1], 10) - // Month starts from 0 - const month = parseInt(uploadDateMatcher[2], 10) - 1 - const day = parseInt(uploadDateMatcher[3], 10) - - originallyPublishedAt.setFullYear(year, month, day) - } - - return originallyPublishedAt - } - - private buildVideoInfo (obj: any): YoutubeDLInfo { - return { - name: this.titleTruncation(obj.title), - description: this.descriptionTruncation(obj.description), - category: this.getCategory(obj.categories), - licence: this.getLicence(obj.license), - language: this.getLanguage(obj.language), - nsfw: this.isNSFW(obj), - tags: this.getTags(obj.tags), - thumbnailUrl: obj.thumbnail || undefined, - urls: this.buildAvailableUrl(obj), - originallyPublishedAtWithoutTime: this.buildOriginallyPublishedAt(obj), - ext: obj.ext, - webpageUrl: obj.webpage_url - } - } - - private buildAvailableUrl (obj: any) { - const urls: string[] = [] - - if (obj.url) urls.push(obj.url) - if (obj.urls) { - if (Array.isArray(obj.urls)) urls.push(...obj.urls) - else urls.push(obj.urls) - } - - const formats = Array.isArray(obj.formats) - ? obj.formats - : [] - - for (const format of formats) { - if (!format.url) continue - - urls.push(format.url) - } - - const thumbnails = Array.isArray(obj.thumbnails) - ? obj.thumbnails - : [] - - for (const thumbnail of thumbnails) { - if (!thumbnail.url) continue - - urls.push(thumbnail.url) - } - - if (obj.thumbnail) urls.push(obj.thumbnail) - - for (const subtitleKey of Object.keys(obj.subtitles || {})) { - const subtitles = obj.subtitles[subtitleKey] - if (!Array.isArray(subtitles)) continue - - for (const subtitle of subtitles) { - if (!subtitle.url) continue - - urls.push(subtitle.url) - } - } - - return urls.filter(u => u && isUrlValid(u)) - } - - private titleTruncation (title: string) { - return peertubeTruncate(title, { - length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max, - separator: /,? +/, - omission: ' […]' - }) - } - - private descriptionTruncation (description: string) { - if (!description || description.length < CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.min) return undefined - - return peertubeTruncate(description, { - length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max, - separator: /,? +/, - omission: ' […]' - }) - } - - private isNSFW (info: any) { - return info?.age_limit >= 16 - } - - private getTags (tags: string[]) { - if (Array.isArray(tags) === false) return [] - - return tags - .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min) - .map(t => t.normalize()) - .slice(0, 5) - } - - private getLicence (licence: string) { - if (!licence) return undefined - - if (licence.includes('Creative Commons Attribution')) return 1 - - for (const key of Object.keys(VIDEO_LICENCES)) { - const peertubeLicence = VIDEO_LICENCES[key] - if (peertubeLicence.toLowerCase() === licence.toLowerCase()) return parseInt(key, 10) - } - - return undefined - } - - private getCategory (categories: string[]) { - if (!categories) return undefined - - const categoryString = categories[0] - if (!categoryString || typeof categoryString !== 'string') return undefined - - if (categoryString === 'News & Politics') return 11 - - for (const key of Object.keys(VIDEO_CATEGORIES)) { - const category = VIDEO_CATEGORIES[key] - if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10) - } - - return undefined - } - - private getLanguage (language: string) { - return VIDEO_LANGUAGES[language] ? language : undefined - } -} - -// --------------------------------------------------------------------------- - -export { - YoutubeDLInfo, - YoutubeDLInfoBuilder -} diff --git a/server/helpers/youtube-dl/youtube-dl-wrapper.ts b/server/helpers/youtube-dl/youtube-dl-wrapper.ts deleted file mode 100644 index ac3cd190e..000000000 --- a/server/helpers/youtube-dl/youtube-dl-wrapper.ts +++ /dev/null @@ -1,154 +0,0 @@ -import { move, pathExists, readdir, remove } from 'fs-extra' -import { dirname, join } from 'path' -import { inspect } from 'util' -import { CONFIG } from '@server/initializers/config' -import { isVideoFileExtnameValid } from '../custom-validators/videos' -import { logger, loggerTagsFactory } from '../logger' -import { generateVideoImportTmpPath } from '../utils' -import { YoutubeDLCLI } from './youtube-dl-cli' -import { YoutubeDLInfo, YoutubeDLInfoBuilder } from './youtube-dl-info-builder' - -const lTags = loggerTagsFactory('youtube-dl') - -export type YoutubeDLSubs = { - language: string - filename: string - path: string -}[] - -const processOptions = { - maxBuffer: 1024 * 1024 * 30 // 30MB -} - -class YoutubeDLWrapper { - - constructor ( - private readonly url: string, - private readonly enabledResolutions: number[], - private readonly useBestFormat: boolean - ) { - - } - - async getInfoForDownload (youtubeDLArgs: string[] = []): Promise { - const youtubeDL = await YoutubeDLCLI.safeGet() - - const info = await youtubeDL.getInfo({ - url: this.url, - format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions, this.useBestFormat), - additionalYoutubeDLArgs: youtubeDLArgs, - processOptions - }) - - if (!info) throw new Error(`YoutubeDL could not get info from ${this.url}`) - - if (info.is_live === true) throw new Error('Cannot download a live streaming.') - - const infoBuilder = new YoutubeDLInfoBuilder(info) - - return infoBuilder.getInfo() - } - - async getInfoForListImport (options: { - latestVideosCount?: number - }) { - const youtubeDL = await YoutubeDLCLI.safeGet() - - const list = await youtubeDL.getListInfo({ - url: this.url, - latestVideosCount: options.latestVideosCount, - processOptions - }) - - if (!Array.isArray(list)) throw new Error(`YoutubeDL could not get list info from ${this.url}: ${inspect(list)}`) - - return list.map(info => info.webpage_url) - } - - async getSubtitles (): Promise { - const cwd = CONFIG.STORAGE.TMP_DIR - - const youtubeDL = await YoutubeDLCLI.safeGet() - - const files = await youtubeDL.getSubs({ url: this.url, format: 'vtt', processOptions: { cwd } }) - if (!files) return [] - - logger.debug('Get subtitles from youtube dl.', { url: this.url, files, ...lTags() }) - - const subtitles = files.reduce((acc, filename) => { - const matched = filename.match(/\.([a-z]{2})(-[a-z]+)?\.(vtt|ttml)/i) - if (!matched?.[1]) return acc - - return [ - ...acc, - { - language: matched[1], - path: join(cwd, filename), - filename - } - ] - }, []) - - return subtitles - } - - async downloadVideo (fileExt: string, timeout: number): Promise { - // Leave empty the extension, youtube-dl will add it - const pathWithoutExtension = generateVideoImportTmpPath(this.url, '') - - logger.info('Importing youtubeDL video %s to %s', this.url, pathWithoutExtension, lTags()) - - const youtubeDL = await YoutubeDLCLI.safeGet() - - try { - await youtubeDL.download({ - url: this.url, - format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions, this.useBestFormat), - output: pathWithoutExtension, - timeout, - processOptions - }) - - // If youtube-dl did not guess an extension for our file, just use .mp4 as default - if (await pathExists(pathWithoutExtension)) { - await move(pathWithoutExtension, pathWithoutExtension + '.mp4') - } - - return this.guessVideoPathWithExtension(pathWithoutExtension, fileExt) - } catch (err) { - this.guessVideoPathWithExtension(pathWithoutExtension, fileExt) - .then(path => { - logger.debug('Error in youtube-dl import, deleting file %s.', path, { err, ...lTags() }) - - return remove(path) - }) - .catch(innerErr => logger.error('Cannot remove file in youtubeDL error.', { innerErr, ...lTags() })) - - throw err - } - } - - private async guessVideoPathWithExtension (tmpPath: string, sourceExt: string) { - if (!isVideoFileExtnameValid(sourceExt)) { - throw new Error('Invalid video extension ' + sourceExt) - } - - const extensions = [ sourceExt, '.mp4', '.mkv', '.webm' ] - - for (const extension of extensions) { - const path = tmpPath + extension - - if (await pathExists(path)) return path - } - - const directoryContent = await readdir(dirname(tmpPath)) - - throw new Error(`Cannot guess path of ${tmpPath}. Directory content: ${directoryContent.join(', ')}`) - } -} - -// --------------------------------------------------------------------------- - -export { - YoutubeDLWrapper -} diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts deleted file mode 100644 index 4281e2bb5..000000000 --- a/server/initializers/checker-after-init.ts +++ /dev/null @@ -1,325 +0,0 @@ -import config from 'config' -import { readFileSync, writeFileSync } from 'fs-extra' -import { URL } from 'url' -import { uniqify } from '@shared/core-utils' -import { getFFmpegVersion } from '@shared/ffmpeg' -import { VideoRedundancyConfigFilter } from '@shared/models/redundancy/video-redundancy-config-filter.type' -import { RecentlyAddedStrategy } from '../../shared/models/redundancy' -import { isProdInstance, parseBytes, parseSemVersion } from '../helpers/core-utils' -import { isArray } from '../helpers/custom-validators/misc' -import { logger } from '../helpers/logger' -import { ApplicationModel, getServerActor } from '../models/application/application' -import { OAuthClientModel } from '../models/oauth/oauth-client' -import { UserModel } from '../models/user/user' -import { CONFIG, getLocalConfigFilePath, isEmailEnabled, reloadConfig } from './config' -import { WEBSERVER } from './constants' - -async function checkActivityPubUrls () { - const actor = await getServerActor() - - const parsed = new URL(actor.url) - if (WEBSERVER.HOST !== parsed.host) { - const NODE_ENV = config.util.getEnv('NODE_ENV') - const NODE_CONFIG_DIR = config.util.getEnv('NODE_CONFIG_DIR') - - logger.warn( - 'It seems PeerTube was started (and created some data) with another domain name. ' + - 'This means you will not be able to federate! ' + - 'Please use %s %s npm run update-host to fix this.', - NODE_CONFIG_DIR ? `NODE_CONFIG_DIR=${NODE_CONFIG_DIR}` : '', - NODE_ENV ? `NODE_ENV=${NODE_ENV}` : '' - ) - } -} - -// Some checks on configuration files or throw if there is an error -function checkConfig () { - - const configFiles = config.util.getConfigSources().map(s => s.name).join(' -> ') - logger.info('Using following configuration file hierarchy: %s.', configFiles) - - checkRemovedConfigKeys() - - checkSecretsConfig() - checkEmailConfig() - checkNSFWPolicyConfig() - checkLocalRedundancyConfig() - checkRemoteRedundancyConfig() - checkStorageConfig() - checkTranscodingConfig() - checkImportConfig() - checkBroadcastMessageConfig() - checkSearchConfig() - checkLiveConfig() - checkObjectStorageConfig() - checkVideoStudioConfig() -} - -// We get db by param to not import it in this file (import orders) -async function clientsExist () { - const totalClients = await OAuthClientModel.countTotal() - - return totalClients !== 0 -} - -// We get db by param to not import it in this file (import orders) -async function usersExist () { - const totalUsers = await UserModel.countTotal() - - return totalUsers !== 0 -} - -// We get db by param to not import it in this file (import orders) -async function applicationExist () { - const totalApplication = await ApplicationModel.countTotal() - - return totalApplication !== 0 -} - -async function checkFFmpegVersion () { - const version = await getFFmpegVersion() - const { major, minor, patch } = parseSemVersion(version) - - if (major < 4 || (major === 4 && minor < 1)) { - logger.warn('Your ffmpeg version (%s) is outdated. PeerTube supports ffmpeg >= 4.1. Please upgrade ffmpeg.', version) - } - - if (major === 4 && minor === 4 && patch === 0) { - logger.warn('There is a bug in ffmpeg 4.4.0 with HLS videos. Please upgrade ffmpeg.') - } -} - -// --------------------------------------------------------------------------- - -export { - checkConfig, - clientsExist, - checkFFmpegVersion, - usersExist, - applicationExist, - checkActivityPubUrls -} - -// --------------------------------------------------------------------------- - -function checkRemovedConfigKeys () { - // Moved configuration keys - if (config.has('services.csp-logger')) { - logger.warn('services.csp-logger configuration has been renamed to csp.report_uri. Please update your configuration file.') - } - - if (config.has('transcoding.webtorrent.enabled')) { - const localConfigPath = getLocalConfigFilePath() - - const content = readFileSync(localConfigPath, { encoding: 'utf-8' }) - if (!content.includes('"webtorrent"')) { - throw new Error('Please rename transcoding.webtorrent.enabled key to transcoding.web_videos.enabled in your configuration file') - } - - try { - logger.info( - 'Replacing "transcoding.webtorrent.enabled" key to "transcoding.web_videos.enabled" in your local configuration ' + localConfigPath - ) - - writeFileSync(localConfigPath, content.replace('"webtorrent"', '"web_videos"'), { encoding: 'utf-8' }) - - reloadConfig() - } catch (err) { - logger.error('Cannot write new configuration to file ' + localConfigPath, { err }) - } - } -} - -function checkSecretsConfig () { - if (!CONFIG.SECRETS.PEERTUBE) { - throw new Error('secrets.peertube is missing in config. Generate one using `openssl rand -hex 32`') - } -} - -function checkEmailConfig () { - if (!isEmailEnabled()) { - if (CONFIG.SIGNUP.ENABLED && CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION) { - throw new Error('SMTP is not configured but you require signup email verification.') - } - - if (CONFIG.SIGNUP.ENABLED && CONFIG.SIGNUP.REQUIRES_APPROVAL) { - // eslint-disable-next-line max-len - logger.warn('SMTP is not configured but signup approval is enabled: PeerTube will not be able to send an email to the user upon acceptance/rejection of the registration request') - } - - if (CONFIG.CONTACT_FORM.ENABLED) { - logger.warn('SMTP is not configured so the contact form will not work.') - } - } -} - -function checkNSFWPolicyConfig () { - const defaultNSFWPolicy = CONFIG.INSTANCE.DEFAULT_NSFW_POLICY - - const available = [ 'do_not_list', 'blur', 'display' ] - if (available.includes(defaultNSFWPolicy) === false) { - throw new Error('NSFW policy setting should be ' + available.join(' or ') + ' instead of ' + defaultNSFWPolicy) - } -} - -function checkLocalRedundancyConfig () { - const redundancyVideos = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES - - if (isArray(redundancyVideos)) { - const available = [ 'most-views', 'trending', 'recently-added' ] - - for (const r of redundancyVideos) { - if (available.includes(r.strategy) === false) { - throw new Error('Videos redundancy should have ' + available.join(' or ') + ' strategy instead of ' + r.strategy) - } - - // Lifetime should not be < 10 hours - if (isProdInstance() && r.minLifetime < 1000 * 3600 * 10) { - throw new Error('Video redundancy minimum lifetime should be >= 10 hours for strategy ' + r.strategy) - } - } - - const filtered = uniqify(redundancyVideos.map(r => r.strategy)) - if (filtered.length !== redundancyVideos.length) { - throw new Error('Redundancy video entries should have unique strategies') - } - - const recentlyAddedStrategy = redundancyVideos.find(r => r.strategy === 'recently-added') as RecentlyAddedStrategy - if (recentlyAddedStrategy && isNaN(recentlyAddedStrategy.minViews)) { - throw new Error('Min views in recently added strategy is not a number') - } - } else { - throw new Error('Videos redundancy should be an array (you must uncomment lines containing - too)') - } -} - -function checkRemoteRedundancyConfig () { - const acceptFrom = CONFIG.REMOTE_REDUNDANCY.VIDEOS.ACCEPT_FROM - const acceptFromValues = new Set([ 'nobody', 'anybody', 'followings' ]) - - if (acceptFromValues.has(acceptFrom) === false) { - throw new Error('remote_redundancy.videos.accept_from has an incorrect value') - } -} - -function checkStorageConfig () { - // Check storage directory locations - if (isProdInstance()) { - const configStorage = config.get<{ [ name: string ]: string }>('storage') - - for (const key of Object.keys(configStorage)) { - if (configStorage[key].startsWith('storage/')) { - logger.warn( - 'Directory of %s should not be in the production directory of PeerTube. Please check your production configuration file.', - key - ) - } - } - } - - if (CONFIG.STORAGE.WEB_VIDEOS_DIR === CONFIG.STORAGE.REDUNDANCY_DIR) { - logger.warn('Redundancy directory should be different than the videos folder.') - } -} - -function checkTranscodingConfig () { - if (CONFIG.TRANSCODING.ENABLED) { - if (CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED === false && CONFIG.TRANSCODING.HLS.ENABLED === false) { - throw new Error('You need to enable at least Web Video transcoding or HLS transcoding.') - } - - if (CONFIG.TRANSCODING.CONCURRENCY <= 0) { - throw new Error('Transcoding concurrency should be > 0') - } - } - - if (CONFIG.IMPORT.VIDEOS.HTTP.ENABLED || CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED) { - if (CONFIG.IMPORT.VIDEOS.CONCURRENCY <= 0) { - throw new Error('Video import concurrency should be > 0') - } - } -} - -function checkImportConfig () { - if (CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED && !CONFIG.IMPORT.VIDEOS.HTTP) { - throw new Error('You need to enable HTTP import to allow synchronization') - } -} - -function checkBroadcastMessageConfig () { - if (CONFIG.BROADCAST_MESSAGE.ENABLED) { - const currentLevel = CONFIG.BROADCAST_MESSAGE.LEVEL - const available = [ 'info', 'warning', 'error' ] - - if (available.includes(currentLevel) === false) { - throw new Error('Broadcast message level should be ' + available.join(' or ') + ' instead of ' + currentLevel) - } - } -} - -function checkSearchConfig () { - if (CONFIG.SEARCH.SEARCH_INDEX.ENABLED === true) { - if (CONFIG.SEARCH.REMOTE_URI.USERS === false) { - throw new Error('You cannot enable search index without enabling remote URI search for users.') - } - } -} - -function checkLiveConfig () { - if (CONFIG.LIVE.ENABLED === true) { - if (CONFIG.LIVE.ALLOW_REPLAY === true && CONFIG.TRANSCODING.ENABLED === false) { - throw new Error('Live allow replay cannot be enabled if transcoding is not enabled.') - } - - if (CONFIG.LIVE.RTMP.ENABLED === false && CONFIG.LIVE.RTMPS.ENABLED === false) { - throw new Error('You must enable at least RTMP or RTMPS') - } - - if (CONFIG.LIVE.RTMPS.ENABLED) { - if (!CONFIG.LIVE.RTMPS.KEY_FILE) { - throw new Error('You must specify a key file to enable RTMPS') - } - - if (!CONFIG.LIVE.RTMPS.CERT_FILE) { - throw new Error('You must specify a cert file to enable RTMPS') - } - } - } -} - -function checkObjectStorageConfig () { - if (CONFIG.OBJECT_STORAGE.ENABLED === true) { - - if (!CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BUCKET_NAME) { - throw new Error('videos_bucket should be set when object storage support is enabled.') - } - - if (!CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME) { - throw new Error('streaming_playlists_bucket should be set when object storage support is enabled.') - } - - if ( - CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BUCKET_NAME === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME && - CONFIG.OBJECT_STORAGE.WEB_VIDEOS.PREFIX === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.PREFIX - ) { - if (CONFIG.OBJECT_STORAGE.WEB_VIDEOS.PREFIX === '') { - throw new Error('Object storage bucket prefixes should be set when the same bucket is used for both types of video.') - } - - throw new Error( - 'Object storage bucket prefixes should be set to different values when the same bucket is used for both types of video.' - ) - } - - if (CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART > parseBytes('250MB')) { - // eslint-disable-next-line max-len - logger.warn(`Object storage max upload part seems to have a big value (${CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART} bytes). Consider using a lower one (like 100MB).`) - } - } -} - -function checkVideoStudioConfig () { - if (CONFIG.VIDEO_STUDIO.ENABLED === true && CONFIG.TRANSCODING.ENABLED === false) { - throw new Error('Video studio cannot be enabled if transcoding is disabled') - } -} diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts deleted file mode 100644 index 0139ded4f..000000000 --- a/server/initializers/checker-before-init.ts +++ /dev/null @@ -1,162 +0,0 @@ -import { IConfig } from 'config' -import { promisify0 } from '@shared/core-utils' -import { parseSemVersion } from '../helpers/core-utils' -import { logger } from '../helpers/logger' - -// Special behaviour for config because we can reload it -const config: IConfig = require('config') - -// ONLY USE CORE MODULES IN THIS FILE! - -// Check the config files -function checkMissedConfig () { - const required = [ 'listen.port', 'listen.hostname', - 'webserver.https', 'webserver.hostname', 'webserver.port', - 'secrets.peertube', - 'trust_proxy', - 'oauth2.token_lifetime.access_token', 'oauth2.token_lifetime.refresh_token', - 'database.hostname', 'database.port', 'database.username', 'database.password', 'database.pool.max', - 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address', - 'email.body.signature', 'email.subject.prefix', - 'storage.avatars', 'storage.web_videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache', - 'storage.redundancy', 'storage.tmp', 'storage.streaming_playlists', 'storage.plugins', 'storage.well_known', - 'log.level', 'log.rotation.enabled', 'log.rotation.max_file_size', 'log.rotation.max_files', 'log.anonymize_ip', - 'log.log_ping_requests', 'log.log_tracker_unknown_infohash', 'log.prettify_sql', 'log.accept_client_log', - 'open_telemetry.metrics.enabled', 'open_telemetry.metrics.prometheus_exporter.hostname', - 'open_telemetry.metrics.prometheus_exporter.port', 'open_telemetry.tracing.enabled', 'open_telemetry.tracing.jaeger_exporter.endpoint', - 'open_telemetry.metrics.http_request_duration.enabled', - 'user.history.videos.enabled', 'user.video_quota', 'user.video_quota_daily', - 'video_channels.max_per_user', - 'csp.enabled', 'csp.report_only', 'csp.report_uri', - 'security.frameguard.enabled', 'security.powered_by_header.enabled', - 'cache.previews.size', 'cache.captions.size', 'cache.torrents.size', 'cache.storyboards.size', - 'admin.email', 'contact_form.enabled', - 'signup.enabled', 'signup.limit', 'signup.requires_approval', 'signup.requires_email_verification', 'signup.minimum_age', - 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist', - 'redundancy.videos.strategies', 'redundancy.videos.check_interval', - 'transcoding.enabled', 'transcoding.threads', 'transcoding.allow_additional_extensions', 'transcoding.web_videos.enabled', - 'transcoding.hls.enabled', 'transcoding.profile', 'transcoding.concurrency', - 'transcoding.resolutions.0p', 'transcoding.resolutions.144p', 'transcoding.resolutions.240p', 'transcoding.resolutions.360p', - 'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p', - 'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'transcoding.remote_runners.enabled', - 'video_studio.enabled', 'video_studio.remote_runners.enabled', - 'video_file.update.enabled', - 'remote_runners.stalled_jobs.vod', 'remote_runners.stalled_jobs.live', - 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout', - 'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user', - 'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization', - 'import.video_channel_synchronization.full_sync_videos_limit', - 'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days', - 'client.videos.miniature.display_author_avatar', - 'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth', - 'defaults.publish.download_enabled', 'defaults.publish.comments_enabled', 'defaults.publish.privacy', 'defaults.publish.licence', - 'instance.name', 'instance.short_description', 'instance.description', 'instance.terms', 'instance.default_client_route', - 'instance.is_nsfw', 'instance.default_nsfw_policy', 'instance.robots', 'instance.securitytxt', - 'services.twitter.username', 'services.twitter.whitelisted', - 'followers.instance.enabled', 'followers.instance.manual_approval', - 'tracker.enabled', 'tracker.private', 'tracker.reject_too_many_announces', - 'history.videos.max_age', 'views.videos.remote.max_age', 'views.videos.local_buffer_update_interval', 'views.videos.ip_view_expiration', - 'rates_limit.api.window', 'rates_limit.api.max', 'rates_limit.login.window', 'rates_limit.login.max', - 'rates_limit.signup.window', 'rates_limit.signup.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max', - 'rates_limit.receive_client_log.window', 'rates_limit.receive_client_log.max', 'rates_limit.plugins.window', 'rates_limit.plugins.max', - 'rates_limit.well_known.window', 'rates_limit.well_known.max', 'rates_limit.feeds.window', 'rates_limit.feeds.max', - 'rates_limit.activity_pub.window', 'rates_limit.activity_pub.max', 'rates_limit.client.window', 'rates_limit.client.max', - 'static_files.private_files_require_auth', - 'object_storage.enabled', 'object_storage.endpoint', 'object_storage.region', 'object_storage.upload_acl.public', - 'object_storage.upload_acl.private', 'object_storage.proxy.proxify_private_files', 'object_storage.credentials.access_key_id', - 'object_storage.credentials.secret_access_key', 'object_storage.max_upload_part', 'object_storage.streaming_playlists.bucket_name', - 'object_storage.streaming_playlists.prefix', 'object_storage.streaming_playlists.base_url', 'object_storage.web_videos.bucket_name', - 'object_storage.web_videos.prefix', 'object_storage.web_videos.base_url', - 'theme.default', - 'feeds.videos.count', 'feeds.comments.count', - 'geo_ip.enabled', 'geo_ip.country.database_url', - 'remote_redundancy.videos.accept_from', - 'federation.videos.federate_unlisted', 'federation.videos.cleanup_remote_interactions', - 'peertube.check_latest_version.enabled', 'peertube.check_latest_version.url', - 'search.remote_uri.users', 'search.remote_uri.anonymous', 'search.search_index.enabled', 'search.search_index.url', - 'search.search_index.disable_local_search', 'search.search_index.is_default_search', - 'live.enabled', 'live.allow_replay', 'live.latency_setting.enabled', 'live.max_duration', - 'live.max_user_lives', 'live.max_instance_lives', - 'live.rtmp.enabled', 'live.rtmp.port', 'live.rtmp.hostname', 'live.rtmp.public_hostname', - 'live.rtmps.enabled', 'live.rtmps.port', 'live.rtmps.hostname', 'live.rtmps.public_hostname', - 'live.rtmps.key_file', 'live.rtmps.cert_file', - 'live.transcoding.enabled', 'live.transcoding.threads', 'live.transcoding.profile', - 'live.transcoding.resolutions.144p', 'live.transcoding.resolutions.240p', 'live.transcoding.resolutions.360p', - 'live.transcoding.resolutions.480p', 'live.transcoding.resolutions.720p', 'live.transcoding.resolutions.1080p', - 'live.transcoding.resolutions.1440p', 'live.transcoding.resolutions.2160p', 'live.transcoding.always_transcode_original_resolution', - 'live.transcoding.remote_runners.enabled' - ] - - const requiredAlternatives = [ - [ // set - [ 'redis.hostname', 'redis.port' ], // alternative - [ 'redis.socket' ], - [ 'redis.sentinel.master_name', 'redis.sentinel.sentinels[0].hostname', 'redis.sentinel.sentinels[0].port' ] - ] - ] - const miss: string[] = [] - - for (const key of required) { - if (!config.has(key)) { - miss.push(key) - } - } - - const redundancyVideos = config.get('redundancy.videos.strategies') - - if (Array.isArray(redundancyVideos)) { - for (const r of redundancyVideos) { - if (!r.size) miss.push('redundancy.videos.strategies.size') - if (!r.min_lifetime) miss.push('redundancy.videos.strategies.min_lifetime') - } - } - - const missingAlternatives = requiredAlternatives.filter( - set => !set.find(alternative => !alternative.find(key => !config.has(key))) - ) - - missingAlternatives - .forEach(set => set[0].forEach(key => miss.push(key))) - - return miss -} - -// Check the available codecs -// We get CONFIG by param to not import it in this file (import orders) -async function checkFFmpeg (CONFIG: { TRANSCODING: { ENABLED: boolean } }) { - if (CONFIG.TRANSCODING.ENABLED === false) return undefined - - const Ffmpeg = require('fluent-ffmpeg') - const getAvailableCodecsPromise = promisify0(Ffmpeg.getAvailableCodecs) - const codecs = await getAvailableCodecsPromise() - const canEncode = [ 'libx264' ] - - for (const codec of canEncode) { - if (codecs[codec] === undefined) { - throw new Error('Unknown codec ' + codec + ' in FFmpeg.') - } - - if (codecs[codec].canEncode !== true) { - throw new Error('Unavailable encode codec ' + codec + ' in FFmpeg') - } - } -} - -function checkNodeVersion () { - const v = process.version - const { major } = parseSemVersion(v) - - logger.debug('Checking NodeJS version %s.', v) - - if (major <= 12) { - throw new Error('Your NodeJS version ' + v + ' is not supported. Please upgrade.') - } -} - -// --------------------------------------------------------------------------- - -export { - checkFFmpeg, - checkMissedConfig, - checkNodeVersion -} diff --git a/server/initializers/config.ts b/server/initializers/config.ts deleted file mode 100644 index 3e3b8ad1f..000000000 --- a/server/initializers/config.ts +++ /dev/null @@ -1,688 +0,0 @@ -import bytes from 'bytes' -import { IConfig } from 'config' -import { dirname, join } from 'path' -import { decacheModule } from '@server/helpers/decache' -import { VideoRedundancyConfigFilter } from '@shared/models/redundancy/video-redundancy-config-filter.type' -import { BroadcastMessageLevel } from '@shared/models/server' -import { buildPath, root } from '../../shared/core-utils' -import { VideoPrivacy, VideosRedundancyStrategy } from '../../shared/models' -import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' -import { parseBytes, parseDurationToMs } from '../helpers/core-utils' - -// Use a variable to reload the configuration if we need -let config: IConfig = require('config') - -const configChangedHandlers: Function[] = [] - -const CONFIG = { - CUSTOM_FILE: getLocalConfigFilePath(), - LISTEN: { - PORT: config.get('listen.port'), - HOSTNAME: config.get('listen.hostname') - }, - SECRETS: { - PEERTUBE: config.get('secrets.peertube') - }, - DATABASE: { - DBNAME: config.has('database.name') ? config.get('database.name') : 'peertube' + config.get('database.suffix'), - HOSTNAME: config.get('database.hostname'), - PORT: config.get('database.port'), - SSL: config.get('database.ssl'), - USERNAME: config.get('database.username'), - PASSWORD: config.get('database.password'), - POOL: { - MAX: config.get('database.pool.max') - } - }, - REDIS: { - HOSTNAME: config.has('redis.hostname') ? config.get('redis.hostname') : null, - PORT: config.has('redis.port') ? config.get('redis.port') : null, - SOCKET: config.has('redis.socket') ? config.get('redis.socket') : null, - AUTH: config.has('redis.auth') ? config.get('redis.auth') : null, - DB: config.has('redis.db') ? config.get('redis.db') : null, - SENTINEL: { - ENABLED: config.has('redis.sentinel.enabled') ? config.get('redis.sentinel.enabled') : false, - ENABLE_TLS: config.has('redis.sentinel.enable_tls') ? config.get('redis.sentinel.enable_tls') : false, - SENTINELS: config.has('redis.sentinel.sentinels') ? config.get<{ hostname: string, port: number }[]>('redis.sentinel.sentinels') : [], - MASTER_NAME: config.has('redis.sentinel.master_name') ? config.get('redis.sentinel.master_name') : null - } - }, - SMTP: { - TRANSPORT: config.has('smtp.transport') ? config.get('smtp.transport') : 'smtp', - SENDMAIL: config.has('smtp.sendmail') ? config.get('smtp.sendmail') : null, - HOSTNAME: config.get('smtp.hostname'), - PORT: config.get('smtp.port'), - USERNAME: config.get('smtp.username'), - PASSWORD: config.get('smtp.password'), - TLS: config.get('smtp.tls'), - DISABLE_STARTTLS: config.get('smtp.disable_starttls'), - CA_FILE: config.get('smtp.ca_file'), - FROM_ADDRESS: config.get('smtp.from_address') - }, - EMAIL: { - BODY: { - SIGNATURE: config.get('email.body.signature') - }, - SUBJECT: { - PREFIX: config.get('email.subject.prefix') + ' ' - } - }, - - CLIENT: { - VIDEOS: { - MINIATURE: { - get PREFER_AUTHOR_DISPLAY_NAME () { return config.get('client.videos.miniature.prefer_author_display_name') }, - get DISPLAY_AUTHOR_AVATAR () { return config.get('client.videos.miniature.display_author_avatar') } - }, - RESUMABLE_UPLOAD: { - get MAX_CHUNK_SIZE () { return parseBytes(config.get('client.videos.resumable_upload.max_chunk_size') || 0) } - } - }, - MENU: { - LOGIN: { - get REDIRECT_ON_SINGLE_EXTERNAL_AUTH () { return config.get('client.menu.login.redirect_on_single_external_auth') } - } - } - }, - - DEFAULTS: { - PUBLISH: { - DOWNLOAD_ENABLED: config.get('defaults.publish.download_enabled'), - COMMENTS_ENABLED: config.get('defaults.publish.comments_enabled'), - PRIVACY: config.get('defaults.publish.privacy'), - LICENCE: config.get('defaults.publish.licence') - }, - P2P: { - WEBAPP: { - ENABLED: config.get('defaults.p2p.webapp.enabled') - }, - EMBED: { - ENABLED: config.get('defaults.p2p.embed.enabled') - } - } - }, - - STORAGE: { - TMP_DIR: buildPath(config.get('storage.tmp')), - TMP_PERSISTENT_DIR: buildPath(config.get('storage.tmp_persistent')), - BIN_DIR: buildPath(config.get('storage.bin')), - ACTOR_IMAGES_DIR: buildPath(config.get('storage.avatars')), - LOG_DIR: buildPath(config.get('storage.logs')), - WEB_VIDEOS_DIR: buildPath(config.get('storage.web_videos')), - STREAMING_PLAYLISTS_DIR: buildPath(config.get('storage.streaming_playlists')), - REDUNDANCY_DIR: buildPath(config.get('storage.redundancy')), - THUMBNAILS_DIR: buildPath(config.get('storage.thumbnails')), - STORYBOARDS_DIR: buildPath(config.get('storage.storyboards')), - PREVIEWS_DIR: buildPath(config.get('storage.previews')), - CAPTIONS_DIR: buildPath(config.get('storage.captions')), - TORRENTS_DIR: buildPath(config.get('storage.torrents')), - CACHE_DIR: buildPath(config.get('storage.cache')), - PLUGINS_DIR: buildPath(config.get('storage.plugins')), - CLIENT_OVERRIDES_DIR: buildPath(config.get('storage.client_overrides')), - WELL_KNOWN_DIR: buildPath(config.get('storage.well_known')) - }, - STATIC_FILES: { - PRIVATE_FILES_REQUIRE_AUTH: config.get('static_files.private_files_require_auth') - }, - OBJECT_STORAGE: { - ENABLED: config.get('object_storage.enabled'), - MAX_UPLOAD_PART: bytes.parse(config.get('object_storage.max_upload_part')), - ENDPOINT: config.get('object_storage.endpoint'), - REGION: config.get('object_storage.region'), - UPLOAD_ACL: { - PUBLIC: config.get('object_storage.upload_acl.public'), - PRIVATE: config.get('object_storage.upload_acl.private') - }, - CREDENTIALS: { - ACCESS_KEY_ID: config.get('object_storage.credentials.access_key_id'), - SECRET_ACCESS_KEY: config.get('object_storage.credentials.secret_access_key') - }, - PROXY: { - PROXIFY_PRIVATE_FILES: config.get('object_storage.proxy.proxify_private_files') - }, - WEB_VIDEOS: { - BUCKET_NAME: config.get('object_storage.web_videos.bucket_name'), - PREFIX: config.get('object_storage.web_videos.prefix'), - BASE_URL: config.get('object_storage.web_videos.base_url') - }, - STREAMING_PLAYLISTS: { - BUCKET_NAME: config.get('object_storage.streaming_playlists.bucket_name'), - PREFIX: config.get('object_storage.streaming_playlists.prefix'), - BASE_URL: config.get('object_storage.streaming_playlists.base_url') - } - }, - WEBSERVER: { - SCHEME: config.get('webserver.https') === true ? 'https' : 'http', - WS: config.get('webserver.https') === true ? 'wss' : 'ws', - HOSTNAME: config.get('webserver.hostname'), - PORT: config.get('webserver.port') - }, - OAUTH2: { - TOKEN_LIFETIME: { - ACCESS_TOKEN: parseDurationToMs(config.get('oauth2.token_lifetime.access_token')), - REFRESH_TOKEN: parseDurationToMs(config.get('oauth2.token_lifetime.refresh_token')) - } - }, - RATES_LIMIT: { - API: { - WINDOW_MS: parseDurationToMs(config.get('rates_limit.api.window')), - MAX: config.get('rates_limit.api.max') - }, - SIGNUP: { - WINDOW_MS: parseDurationToMs(config.get('rates_limit.signup.window')), - MAX: config.get('rates_limit.signup.max') - }, - LOGIN: { - WINDOW_MS: parseDurationToMs(config.get('rates_limit.login.window')), - MAX: config.get('rates_limit.login.max') - }, - RECEIVE_CLIENT_LOG: { - WINDOW_MS: parseDurationToMs(config.get('rates_limit.receive_client_log.window')), - MAX: config.get('rates_limit.receive_client_log.max') - }, - ASK_SEND_EMAIL: { - WINDOW_MS: parseDurationToMs(config.get('rates_limit.ask_send_email.window')), - MAX: config.get('rates_limit.ask_send_email.max') - }, - PLUGINS: { - WINDOW_MS: parseDurationToMs(config.get('rates_limit.plugins.window')), - MAX: config.get('rates_limit.plugins.max') - }, - WELL_KNOWN: { - WINDOW_MS: parseDurationToMs(config.get('rates_limit.well_known.window')), - MAX: config.get('rates_limit.well_known.max') - }, - FEEDS: { - WINDOW_MS: parseDurationToMs(config.get('rates_limit.feeds.window')), - MAX: config.get('rates_limit.feeds.max') - }, - ACTIVITY_PUB: { - WINDOW_MS: parseDurationToMs(config.get('rates_limit.activity_pub.window')), - MAX: config.get('rates_limit.activity_pub.max') - }, - CLIENT: { - WINDOW_MS: parseDurationToMs(config.get('rates_limit.client.window')), - MAX: config.get('rates_limit.client.max') - } - }, - TRUST_PROXY: config.get('trust_proxy'), - LOG: { - LEVEL: config.get('log.level'), - ROTATION: { - ENABLED: config.get('log.rotation.enabled'), - MAX_FILE_SIZE: bytes.parse(config.get('log.rotation.max_file_size')), - MAX_FILES: config.get('log.rotation.max_files') - }, - ANONYMIZE_IP: config.get('log.anonymize_ip'), - LOG_PING_REQUESTS: config.get('log.log_ping_requests'), - LOG_TRACKER_UNKNOWN_INFOHASH: config.get('log.log_tracker_unknown_infohash'), - PRETTIFY_SQL: config.get('log.prettify_sql'), - ACCEPT_CLIENT_LOG: config.get('log.accept_client_log') - }, - OPEN_TELEMETRY: { - METRICS: { - ENABLED: config.get('open_telemetry.metrics.enabled'), - - HTTP_REQUEST_DURATION: { - ENABLED: config.get('open_telemetry.metrics.http_request_duration.enabled') - }, - - PROMETHEUS_EXPORTER: { - HOSTNAME: config.get('open_telemetry.metrics.prometheus_exporter.hostname'), - PORT: config.get('open_telemetry.metrics.prometheus_exporter.port') - } - }, - TRACING: { - ENABLED: config.get('open_telemetry.tracing.enabled'), - - JAEGER_EXPORTER: { - ENDPOINT: config.get('open_telemetry.tracing.jaeger_exporter.endpoint') - } - } - }, - TRENDING: { - VIDEOS: { - INTERVAL_DAYS: config.get('trending.videos.interval_days'), - ALGORITHMS: { - get ENABLED () { return config.get('trending.videos.algorithms.enabled') }, - get DEFAULT () { return config.get('trending.videos.algorithms.default') } - } - } - }, - REDUNDANCY: { - VIDEOS: { - CHECK_INTERVAL: parseDurationToMs(config.get('redundancy.videos.check_interval')), - STRATEGIES: buildVideosRedundancy(config.get('redundancy.videos.strategies')) - } - }, - REMOTE_REDUNDANCY: { - VIDEOS: { - ACCEPT_FROM: config.get('remote_redundancy.videos.accept_from') - } - }, - CSP: { - ENABLED: config.get('csp.enabled'), - REPORT_ONLY: config.get('csp.report_only'), - REPORT_URI: config.get('csp.report_uri') - }, - SECURITY: { - FRAMEGUARD: { - ENABLED: config.get('security.frameguard.enabled') - }, - POWERED_BY_HEADER: { - ENABLED: config.get('security.powered_by_header.enabled') - } - }, - TRACKER: { - ENABLED: config.get('tracker.enabled'), - PRIVATE: config.get('tracker.private'), - REJECT_TOO_MANY_ANNOUNCES: config.get('tracker.reject_too_many_announces') - }, - HISTORY: { - VIDEOS: { - MAX_AGE: parseDurationToMs(config.get('history.videos.max_age')) - } - }, - VIEWS: { - VIDEOS: { - REMOTE: { - MAX_AGE: parseDurationToMs(config.get('views.videos.remote.max_age')) - }, - LOCAL_BUFFER_UPDATE_INTERVAL: parseDurationToMs(config.get('views.videos.local_buffer_update_interval')), - IP_VIEW_EXPIRATION: parseDurationToMs(config.get('views.videos.ip_view_expiration')) - } - }, - GEO_IP: { - ENABLED: config.get('geo_ip.enabled'), - COUNTRY: { - DATABASE_URL: config.get('geo_ip.country.database_url') - } - }, - PLUGINS: { - INDEX: { - ENABLED: config.get('plugins.index.enabled'), - CHECK_LATEST_VERSIONS_INTERVAL: parseDurationToMs(config.get('plugins.index.check_latest_versions_interval')), - URL: config.get('plugins.index.url') - } - }, - FEDERATION: { - VIDEOS: { - FEDERATE_UNLISTED: config.get('federation.videos.federate_unlisted'), - CLEANUP_REMOTE_INTERACTIONS: config.get('federation.videos.cleanup_remote_interactions') - }, - SIGN_FEDERATED_FETCHES: config.get('federation.sign_federated_fetches') - }, - PEERTUBE: { - CHECK_LATEST_VERSION: { - ENABLED: config.get('peertube.check_latest_version.enabled'), - URL: config.get('peertube.check_latest_version.url') - } - }, - WEBADMIN: { - CONFIGURATION: { - EDITION: { - ALLOWED: config.get('webadmin.configuration.edition.allowed') - } - } - }, - FEEDS: { - VIDEOS: { - COUNT: config.get('feeds.videos.count') - }, - COMMENTS: { - COUNT: config.get('feeds.comments.count') - } - }, - REMOTE_RUNNERS: { - STALLED_JOBS: { - LIVE: parseDurationToMs(config.get('remote_runners.stalled_jobs.live')), - VOD: parseDurationToMs(config.get('remote_runners.stalled_jobs.vod')) - } - }, - ADMIN: { - get EMAIL () { return config.get('admin.email') } - }, - CONTACT_FORM: { - get ENABLED () { return config.get('contact_form.enabled') } - }, - SIGNUP: { - get ENABLED () { return config.get('signup.enabled') }, - get REQUIRES_APPROVAL () { return config.get('signup.requires_approval') }, - get LIMIT () { return config.get('signup.limit') }, - get REQUIRES_EMAIL_VERIFICATION () { return config.get('signup.requires_email_verification') }, - get MINIMUM_AGE () { return config.get('signup.minimum_age') }, - FILTERS: { - CIDR: { - get WHITELIST () { return config.get('signup.filters.cidr.whitelist') }, - get BLACKLIST () { return config.get('signup.filters.cidr.blacklist') } - } - } - }, - USER: { - HISTORY: { - VIDEOS: { - get ENABLED () { return config.get('user.history.videos.enabled') } - } - }, - get VIDEO_QUOTA () { return parseBytes(config.get('user.video_quota')) }, - get VIDEO_QUOTA_DAILY () { return parseBytes(config.get('user.video_quota_daily')) } - }, - VIDEO_CHANNELS: { - get MAX_PER_USER () { return config.get('video_channels.max_per_user') } - }, - TRANSCODING: { - get ENABLED () { return config.get('transcoding.enabled') }, - get ALLOW_ADDITIONAL_EXTENSIONS () { return config.get('transcoding.allow_additional_extensions') }, - get ALLOW_AUDIO_FILES () { return config.get('transcoding.allow_audio_files') }, - get THREADS () { return config.get('transcoding.threads') }, - get CONCURRENCY () { return config.get('transcoding.concurrency') }, - get PROFILE () { return config.get('transcoding.profile') }, - get ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION () { return config.get('transcoding.always_transcode_original_resolution') }, - RESOLUTIONS: { - get '0p' () { return config.get('transcoding.resolutions.0p') }, - get '144p' () { return config.get('transcoding.resolutions.144p') }, - get '240p' () { return config.get('transcoding.resolutions.240p') }, - get '360p' () { return config.get('transcoding.resolutions.360p') }, - get '480p' () { return config.get('transcoding.resolutions.480p') }, - get '720p' () { return config.get('transcoding.resolutions.720p') }, - get '1080p' () { return config.get('transcoding.resolutions.1080p') }, - get '1440p' () { return config.get('transcoding.resolutions.1440p') }, - get '2160p' () { return config.get('transcoding.resolutions.2160p') } - }, - HLS: { - get ENABLED () { return config.get('transcoding.hls.enabled') } - }, - WEB_VIDEOS: { - get ENABLED () { return config.get('transcoding.web_videos.enabled') } - }, - REMOTE_RUNNERS: { - get ENABLED () { return config.get('transcoding.remote_runners.enabled') } - } - }, - LIVE: { - get ENABLED () { return config.get('live.enabled') }, - - get MAX_DURATION () { return parseDurationToMs(config.get('live.max_duration')) }, - get MAX_INSTANCE_LIVES () { return config.get('live.max_instance_lives') }, - get MAX_USER_LIVES () { return config.get('live.max_user_lives') }, - - get ALLOW_REPLAY () { return config.get('live.allow_replay') }, - - LATENCY_SETTING: { - get ENABLED () { return config.get('live.latency_setting.enabled') } - }, - - RTMP: { - get ENABLED () { return config.get('live.rtmp.enabled') }, - get PORT () { return config.get('live.rtmp.port') }, - get HOSTNAME () { return config.get('live.rtmp.hostname') }, - get PUBLIC_HOSTNAME () { return config.get('live.rtmp.public_hostname') } - }, - - RTMPS: { - get ENABLED () { return config.get('live.rtmps.enabled') }, - get PORT () { return config.get('live.rtmps.port') }, - get HOSTNAME () { return config.get('live.rtmps.hostname') }, - get PUBLIC_HOSTNAME () { return config.get('live.rtmps.public_hostname') }, - get KEY_FILE () { return config.get('live.rtmps.key_file') }, - get CERT_FILE () { return config.get('live.rtmps.cert_file') } - }, - - TRANSCODING: { - get ENABLED () { return config.get('live.transcoding.enabled') }, - get THREADS () { return config.get('live.transcoding.threads') }, - get PROFILE () { return config.get('live.transcoding.profile') }, - - get ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION () { return config.get('live.transcoding.always_transcode_original_resolution') }, - - RESOLUTIONS: { - get '144p' () { return config.get('live.transcoding.resolutions.144p') }, - get '240p' () { return config.get('live.transcoding.resolutions.240p') }, - get '360p' () { return config.get('live.transcoding.resolutions.360p') }, - get '480p' () { return config.get('live.transcoding.resolutions.480p') }, - get '720p' () { return config.get('live.transcoding.resolutions.720p') }, - get '1080p' () { return config.get('live.transcoding.resolutions.1080p') }, - get '1440p' () { return config.get('live.transcoding.resolutions.1440p') }, - get '2160p' () { return config.get('live.transcoding.resolutions.2160p') } - }, - REMOTE_RUNNERS: { - get ENABLED () { return config.get('live.transcoding.remote_runners.enabled') } - } - } - }, - VIDEO_STUDIO: { - get ENABLED () { return config.get('video_studio.enabled') }, - REMOTE_RUNNERS: { - get ENABLED () { return config.get('video_studio.remote_runners.enabled') } - } - }, - VIDEO_FILE: { - UPDATE: { - get ENABLED () { return config.get('video_file.update.enabled') } - } - }, - IMPORT: { - VIDEOS: { - get CONCURRENCY () { return config.get('import.videos.concurrency') }, - get TIMEOUT () { return parseDurationToMs(config.get('import.videos.timeout')) }, - - HTTP: { - get ENABLED () { return config.get('import.videos.http.enabled') }, - - YOUTUBE_DL_RELEASE: { - get URL () { return config.get('import.videos.http.youtube_dl_release.url') }, - get NAME () { return config.get('import.videos.http.youtube_dl_release.name') }, - get PYTHON_PATH () { return config.get('import.videos.http.youtube_dl_release.python_path') } - }, - - get FORCE_IPV4 () { return config.get('import.videos.http.force_ipv4') } - }, - TORRENT: { - get ENABLED () { return config.get('import.videos.torrent.enabled') } - } - }, - VIDEO_CHANNEL_SYNCHRONIZATION: { - get ENABLED () { return config.get('import.video_channel_synchronization.enabled') }, - get MAX_PER_USER () { return config.get('import.video_channel_synchronization.max_per_user') }, - get CHECK_INTERVAL () { return parseDurationToMs(config.get('import.video_channel_synchronization.check_interval')) }, - get VIDEOS_LIMIT_PER_SYNCHRONIZATION () { - return config.get('import.video_channel_synchronization.videos_limit_per_synchronization') - }, - get FULL_SYNC_VIDEOS_LIMIT () { - return config.get('import.video_channel_synchronization.full_sync_videos_limit') - } - } - }, - AUTO_BLACKLIST: { - VIDEOS: { - OF_USERS: { - get ENABLED () { return config.get('auto_blacklist.videos.of_users.enabled') } - } - } - }, - CACHE: { - PREVIEWS: { - get SIZE () { return config.get('cache.previews.size') } - }, - VIDEO_CAPTIONS: { - get SIZE () { return config.get('cache.captions.size') } - }, - TORRENTS: { - get SIZE () { return config.get('cache.torrents.size') } - }, - STORYBOARDS: { - get SIZE () { return config.get('cache.storyboards.size') } - } - }, - INSTANCE: { - get NAME () { return config.get('instance.name') }, - get SHORT_DESCRIPTION () { return config.get('instance.short_description') }, - get DESCRIPTION () { return config.get('instance.description') }, - get TERMS () { return config.get('instance.terms') }, - get CODE_OF_CONDUCT () { return config.get('instance.code_of_conduct') }, - - get CREATION_REASON () { return config.get('instance.creation_reason') }, - - get MODERATION_INFORMATION () { return config.get('instance.moderation_information') }, - get ADMINISTRATOR () { return config.get('instance.administrator') }, - get MAINTENANCE_LIFETIME () { return config.get('instance.maintenance_lifetime') }, - get BUSINESS_MODEL () { return config.get('instance.business_model') }, - get HARDWARE_INFORMATION () { return config.get('instance.hardware_information') }, - - get LANGUAGES () { return config.get('instance.languages') || [] }, - get CATEGORIES () { return config.get('instance.categories') || [] }, - - get IS_NSFW () { return config.get('instance.is_nsfw') }, - get DEFAULT_NSFW_POLICY () { return config.get('instance.default_nsfw_policy') }, - - get DEFAULT_CLIENT_ROUTE () { return config.get('instance.default_client_route') }, - - CUSTOMIZATIONS: { - get JAVASCRIPT () { return config.get('instance.customizations.javascript') }, - get CSS () { return config.get('instance.customizations.css') } - }, - get ROBOTS () { return config.get('instance.robots') }, - get SECURITYTXT () { return config.get('instance.securitytxt') }, - get SECURITYTXT_CONTACT () { return config.get('admin.email') } - }, - SERVICES: { - TWITTER: { - get USERNAME () { return config.get('services.twitter.username') }, - get WHITELISTED () { return config.get('services.twitter.whitelisted') } - } - }, - FOLLOWERS: { - INSTANCE: { - get ENABLED () { return config.get('followers.instance.enabled') }, - get MANUAL_APPROVAL () { return config.get('followers.instance.manual_approval') } - } - }, - FOLLOWINGS: { - INSTANCE: { - AUTO_FOLLOW_BACK: { - get ENABLED () { - return config.get('followings.instance.auto_follow_back.enabled') - } - }, - AUTO_FOLLOW_INDEX: { - get ENABLED () { - return config.get('followings.instance.auto_follow_index.enabled') - }, - get INDEX_URL () { - return config.get('followings.instance.auto_follow_index.index_url') - } - } - } - }, - THEME: { - get DEFAULT () { return config.get('theme.default') } - }, - BROADCAST_MESSAGE: { - get ENABLED () { return config.get('broadcast_message.enabled') }, - get MESSAGE () { return config.get('broadcast_message.message') }, - get LEVEL () { return config.get('broadcast_message.level') }, - get DISMISSABLE () { return config.get('broadcast_message.dismissable') } - }, - SEARCH: { - REMOTE_URI: { - get USERS () { return config.get('search.remote_uri.users') }, - get ANONYMOUS () { return config.get('search.remote_uri.anonymous') } - }, - SEARCH_INDEX: { - get ENABLED () { return config.get('search.search_index.enabled') }, - get URL () { return config.get('search.search_index.url') }, - get DISABLE_LOCAL_SEARCH () { return config.get('search.search_index.disable_local_search') }, - get IS_DEFAULT_SEARCH () { return config.get('search.search_index.is_default_search') } - } - } - -} - -function registerConfigChangedHandler (fun: Function) { - configChangedHandlers.push(fun) -} - -function isEmailEnabled () { - if (CONFIG.SMTP.TRANSPORT === 'sendmail' && CONFIG.SMTP.SENDMAIL) return true - - if (CONFIG.SMTP.TRANSPORT === 'smtp' && CONFIG.SMTP.HOSTNAME && CONFIG.SMTP.PORT) return true - - return false -} - -function getLocalConfigFilePath () { - const localConfigDir = getLocalConfigDir() - - let filename = 'local' - if (process.env.NODE_ENV) filename += `-${process.env.NODE_ENV}` - if (process.env.NODE_APP_INSTANCE) filename += `-${process.env.NODE_APP_INSTANCE}` - - return join(localConfigDir, filename + '.json') -} - -// --------------------------------------------------------------------------- - -export { - CONFIG, - getLocalConfigFilePath, - registerConfigChangedHandler, - isEmailEnabled -} - -// --------------------------------------------------------------------------- - -function getLocalConfigDir () { - if (process.env.PEERTUBE_LOCAL_CONFIG) return process.env.PEERTUBE_LOCAL_CONFIG - - const configSources = config.util.getConfigSources() - if (configSources.length === 0) throw new Error('Invalid config source.') - - return dirname(configSources[0].name) -} - -function buildVideosRedundancy (objs: any[]): VideosRedundancyStrategy[] { - if (!objs) return [] - - if (!Array.isArray(objs)) return objs - - return objs.map(obj => { - return Object.assign({}, obj, { - minLifetime: parseDurationToMs(obj.min_lifetime), - size: bytes.parse(obj.size), - minViews: obj.min_views - }) - }) -} - -export function reloadConfig () { - - function getConfigDirectories () { - if (process.env.NODE_CONFIG_DIR) { - return process.env.NODE_CONFIG_DIR.split(':') - } - - return [ join(root(), 'config') ] - } - - function purge () { - const directories = getConfigDirectories() - - for (const fileName in require.cache) { - if (directories.some((dir) => fileName.includes(dir)) === false) { - continue - } - - delete require.cache[fileName] - } - - decacheModule('config') - } - - purge() - - config = require('config') - - for (const configChangedHandler of configChangedHandlers) { - configChangedHandler() - } -} diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts deleted file mode 100644 index de5f11f8f..000000000 --- a/server/initializers/constants.ts +++ /dev/null @@ -1,1396 +0,0 @@ -import { RepeatOptions } from 'bullmq' -import { Encoding, randomBytes } from 'crypto' -import { invert } from 'lodash' -import { join } from 'path' -import { randomInt, root } from '@shared/core-utils' -import { - AbuseState, - JobType, - RunnerJobState, - UserRegistrationState, - VideoChannelSyncState, - VideoImportState, - VideoPrivacy, - VideoRateType, - VideoResolution, - VideoState, - VideoTranscodingFPS -} from '../../shared/models' -import { ActivityPubActorType } from '../../shared/models/activitypub' -import { ActorImageType, FollowState } from '../../shared/models/actors' -import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' -import { VideoPlaylistPrivacy } from '../../shared/models/videos/playlist/video-playlist-privacy.model' -import { VideoPlaylistType } from '../../shared/models/videos/playlist/video-playlist-type.model' -// Do not use barrels, remain constants as independent as possible -import { isTestInstance, isTestOrDevInstance, parseDurationToMs, sanitizeHost, sanitizeUrl } from '../helpers/core-utils' -import { CONFIG, registerConfigChangedHandler } from './config' - -// --------------------------------------------------------------------------- - -const LAST_MIGRATION_VERSION = 800 - -// --------------------------------------------------------------------------- - -const API_VERSION = 'v1' -const PEERTUBE_VERSION: string = require(join(root(), 'package.json')).version - -const PAGINATION = { - GLOBAL: { - COUNT: { - DEFAULT: 15, - MAX: 100 - } - }, - OUTBOX: { - COUNT: { - MAX: 50 - } - } -} - -const WEBSERVER = { - URL: '', - HOST: '', - SCHEME: '', - WS: '', - HOSTNAME: '', - PORT: 0, - - RTMP_URL: '', - RTMPS_URL: '', - - RTMP_BASE_LIVE_URL: '', - RTMPS_BASE_LIVE_URL: '' -} - -// Sortable columns per schema -const SORTABLE_COLUMNS = { - ADMIN_USERS: [ 'id', 'username', 'videoQuotaUsed', 'createdAt', 'lastLoginDate', 'role' ], - USER_SUBSCRIPTIONS: [ 'id', 'createdAt' ], - ACCOUNTS: [ 'createdAt' ], - JOBS: [ 'createdAt' ], - VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ], - VIDEO_IMPORTS: [ 'createdAt' ], - VIDEO_CHANNEL_SYNCS: [ 'externalChannelUrl', 'videoChannel', 'createdAt', 'lastSyncAt', 'state' ], - - VIDEO_COMMENT_THREADS: [ 'createdAt', 'totalReplies' ], - VIDEO_COMMENTS: [ 'createdAt' ], - - VIDEO_PASSWORDS: [ 'createdAt' ], - - VIDEO_RATES: [ 'createdAt' ], - BLACKLISTS: [ 'id', 'name', 'duration', 'views', 'likes', 'dislikes', 'uuid', 'createdAt' ], - - INSTANCE_FOLLOWERS: [ 'createdAt', 'state', 'score' ], - INSTANCE_FOLLOWING: [ 'createdAt', 'redundancyAllowed', 'state' ], - ACCOUNT_FOLLOWERS: [ 'createdAt' ], - CHANNEL_FOLLOWERS: [ 'createdAt' ], - - USER_REGISTRATIONS: [ 'createdAt', 'state' ], - - RUNNERS: [ 'createdAt' ], - RUNNER_REGISTRATION_TOKENS: [ 'createdAt' ], - RUNNER_JOBS: [ 'updatedAt', 'createdAt', 'priority', 'state', 'progress' ], - - VIDEOS: [ 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes', 'trending', 'hot', 'best' ], - - // Don't forget to update peertube-search-index with the same values - VIDEOS_SEARCH: [ 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes', 'match' ], - VIDEO_CHANNELS_SEARCH: [ 'match', 'displayName', 'createdAt' ], - VIDEO_PLAYLISTS_SEARCH: [ 'match', 'displayName', 'createdAt' ], - - ABUSES: [ 'id', 'createdAt', 'state' ], - - ACCOUNTS_BLOCKLIST: [ 'createdAt' ], - SERVERS_BLOCKLIST: [ 'createdAt' ], - - USER_NOTIFICATIONS: [ 'createdAt', 'read' ], - - VIDEO_PLAYLISTS: [ 'name', 'displayName', 'createdAt', 'updatedAt' ], - - PLUGINS: [ 'name', 'createdAt', 'updatedAt' ], - - AVAILABLE_PLUGINS: [ 'npmName', 'popularity' ], - - VIDEO_REDUNDANCIES: [ 'name' ] -} - -const ROUTE_CACHE_LIFETIME = { - FEEDS: '15 minutes', - ROBOTS: '2 hours', - SITEMAP: '1 day', - SECURITYTXT: '2 hours', - NODEINFO: '10 minutes', - DNT_POLICY: '1 week', - ACTIVITY_PUB: { - VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example - }, - STATS: '4 hours', - WELL_KNOWN: '1 day' -} - -// --------------------------------------------------------------------------- - -// Number of points we add/remove after a successful/bad request -const ACTOR_FOLLOW_SCORE = { - PENALTY: -10, - BONUS: 10, - BASE: 1000, - MAX: 10000 -} - -const FOLLOW_STATES: { [ id: string ]: FollowState } = { - PENDING: 'pending', - ACCEPTED: 'accepted', - REJECTED: 'rejected' -} - -const REMOTE_SCHEME = { - HTTP: 'https', - WS: 'wss' -} - -// --------------------------------------------------------------------------- - -const JOB_ATTEMPTS: { [id in JobType]: number } = { - 'activitypub-http-broadcast': 1, - 'activitypub-http-broadcast-parallel': 1, - 'activitypub-http-unicast': 1, - 'activitypub-http-fetcher': 2, - 'activitypub-follow': 5, - 'activitypub-cleaner': 1, - 'video-file-import': 1, - 'video-transcoding': 1, - 'video-import': 1, - 'email': 5, - 'actor-keys': 3, - 'videos-views-stats': 1, - 'activitypub-refresher': 1, - 'video-redundancy': 1, - 'video-live-ending': 1, - 'video-studio-edition': 1, - 'manage-video-torrent': 1, - 'video-channel-import': 1, - 'after-video-channel-import': 1, - 'move-to-object-storage': 3, - 'transcoding-job-builder': 1, - 'generate-video-storyboard': 1, - 'notify': 1, - 'federate-video': 1 -} -// Excluded keys are jobs that can be configured by admins -const JOB_CONCURRENCY: { [id in Exclude]: number } = { - 'activitypub-http-broadcast': 1, - 'activitypub-http-broadcast-parallel': 30, - 'activitypub-http-unicast': 30, - 'activitypub-http-fetcher': 3, - 'activitypub-cleaner': 1, - 'activitypub-follow': 1, - 'video-file-import': 1, - 'email': 5, - 'actor-keys': 1, - 'videos-views-stats': 1, - 'activitypub-refresher': 1, - 'video-redundancy': 1, - 'video-live-ending': 10, - 'video-studio-edition': 1, - 'manage-video-torrent': 1, - 'move-to-object-storage': 1, - 'video-channel-import': 1, - 'after-video-channel-import': 1, - 'transcoding-job-builder': 1, - 'generate-video-storyboard': 1, - 'notify': 5, - 'federate-video': 3 -} -const JOB_TTL: { [id in JobType]: number } = { - 'activitypub-http-broadcast': 60000 * 10, // 10 minutes - 'activitypub-http-broadcast-parallel': 60000 * 10, // 10 minutes - 'activitypub-http-unicast': 60000 * 10, // 10 minutes - 'activitypub-http-fetcher': 1000 * 3600 * 10, // 10 hours - 'activitypub-follow': 60000 * 10, // 10 minutes - 'activitypub-cleaner': 1000 * 3600, // 1 hour - 'video-file-import': 1000 * 3600, // 1 hour - 'video-transcoding': 1000 * 3600 * 48, // 2 days, transcoding could be long - 'video-studio-edition': 1000 * 3600 * 10, // 10 hours - 'video-import': CONFIG.IMPORT.VIDEOS.TIMEOUT, - 'email': 60000 * 10, // 10 minutes - 'actor-keys': 60000 * 20, // 20 minutes - 'videos-views-stats': undefined, // Unlimited - 'activitypub-refresher': 60000 * 10, // 10 minutes - 'video-redundancy': 1000 * 3600 * 3, // 3 hours - 'video-live-ending': 1000 * 60 * 10, // 10 minutes - 'generate-video-storyboard': 1000 * 60 * 10, // 10 minutes - 'manage-video-torrent': 1000 * 3600 * 3, // 3 hours - 'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours - 'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours - 'after-video-channel-import': 60000 * 5, // 5 minutes - 'transcoding-job-builder': 60000, // 1 minute - 'notify': 60000 * 5, // 5 minutes - 'federate-video': 60000 * 5 // 5 minutes -} -const REPEAT_JOBS: { [ id in JobType ]?: RepeatOptions } = { - 'videos-views-stats': { - pattern: randomInt(1, 20) + ' * * * *' // Between 1-20 minutes past the hour - }, - 'activitypub-cleaner': { - pattern: '30 5 * * ' + randomInt(0, 7) // 1 time per week (random day) at 5:30 AM - } -} -const JOB_PRIORITY = { - TRANSCODING: 100, - VIDEO_STUDIO: 150 -} - -const JOB_REMOVAL_OPTIONS = { - COUNT: 10000, // Max jobs to store - - SUCCESS: { // Success jobs - 'DEFAULT': parseDurationToMs('2 days'), - - 'activitypub-http-broadcast-parallel': parseDurationToMs('10 minutes'), - 'activitypub-http-unicast': parseDurationToMs('1 hour'), - 'videos-views-stats': parseDurationToMs('3 hours'), - 'activitypub-refresher': parseDurationToMs('10 hours') - }, - - FAILURE: { // Failed job - DEFAULT: parseDurationToMs('7 days') - } -} - -const VIDEO_IMPORT_TIMEOUT = Math.floor(JOB_TTL['video-import'] * 0.9) - -const RUNNER_JOBS = { - MAX_FAILURES: 5, - LAST_CONTACT_UPDATE_INTERVAL: 30000 -} - -// --------------------------------------------------------------------------- - -const BROADCAST_CONCURRENCY = 30 // How many requests in parallel we do in activitypub-http-broadcast job -const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...) - -const AP_CLEANER = { - CONCURRENCY: 10, // How many requests in parallel we do in activitypub-cleaner job - UNAVAILABLE_TRESHOLD: 3, // How many attempts we do before removing an unavailable remote resource - PERIOD: parseDurationToMs('1 week') // /!\ Has to be sync with REPEAT_JOBS -} - -const REQUEST_TIMEOUTS = { - DEFAULT: 7000, // 7 seconds - FILE: 30000, // 30 seconds - REDUNDANCY: JOB_TTL['video-redundancy'] -} - -const SCHEDULER_INTERVALS_MS = { - RUNNER_JOB_WATCH_DOG: Math.min(CONFIG.REMOTE_RUNNERS.STALLED_JOBS.VOD, CONFIG.REMOTE_RUNNERS.STALLED_JOBS.LIVE), - ACTOR_FOLLOW_SCORES: 60000 * 60, // 1 hour - REMOVE_OLD_JOBS: 60000 * 60, // 1 hour - UPDATE_VIDEOS: 60000, // 1 minute - YOUTUBE_DL_UPDATE: 60000 * 60 * 24, // 1 day - GEO_IP_UPDATE: 60000 * 60 * 24, // 1 day - VIDEO_VIEWS_BUFFER_UPDATE: CONFIG.VIEWS.VIDEOS.LOCAL_BUFFER_UPDATE_INTERVAL, - CHECK_PLUGINS: CONFIG.PLUGINS.INDEX.CHECK_LATEST_VERSIONS_INTERVAL, - CHECK_PEERTUBE_VERSION: 60000 * 60 * 24, // 1 day - AUTO_FOLLOW_INDEX_INSTANCES: 60000 * 60 * 24, // 1 day - REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day - REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day - UPDATE_INBOX_STATS: 1000 * 60, // 1 minute - REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60, // 1 hour - CHANNEL_SYNC_CHECK_INTERVAL: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.CHECK_INTERVAL -} - -// --------------------------------------------------------------------------- - -const CONSTRAINTS_FIELDS = { - USERS: { - NAME: { min: 1, max: 120 }, // Length - DESCRIPTION: { min: 3, max: 1000 }, // Length - USERNAME: { min: 1, max: 50 }, // Length - PASSWORD: { min: 6, max: 255 }, // Length - VIDEO_QUOTA: { min: -1 }, - VIDEO_QUOTA_DAILY: { min: -1 }, - VIDEO_LANGUAGES: { max: 500 }, // Array length - BLOCKED_REASON: { min: 3, max: 250 } // Length - }, - ABUSES: { - REASON: { min: 2, max: 3000 }, // Length - MODERATION_COMMENT: { min: 2, max: 3000 } // Length - }, - ABUSE_MESSAGES: { - MESSAGE: { min: 2, max: 3000 } // Length - }, - USER_REGISTRATIONS: { - REASON_MESSAGE: { min: 2, max: 3000 }, // Length - MODERATOR_MESSAGE: { min: 2, max: 3000 } // Length - }, - VIDEO_BLACKLIST: { - REASON: { min: 2, max: 300 } // Length - }, - VIDEO_CHANNELS: { - NAME: { min: 1, max: 120 }, // Length - DESCRIPTION: { min: 3, max: 1000 }, // Length - SUPPORT: { min: 3, max: 1000 }, // Length - EXTERNAL_CHANNEL_URL: { min: 3, max: 2000 }, // Length - URL: { min: 3, max: 2000 } // Length - }, - VIDEO_CHANNEL_SYNCS: { - EXTERNAL_CHANNEL_URL: { min: 3, max: 2000 } // Length - }, - VIDEO_CAPTIONS: { - CAPTION_FILE: { - EXTNAME: [ '.vtt', '.srt' ], - FILE_SIZE: { - max: 20 * 1024 * 1024 // 20MB - } - } - }, - VIDEO_IMPORTS: { - URL: { min: 3, max: 2000 }, // Length - TORRENT_NAME: { min: 3, max: 255 }, // Length - TORRENT_FILE: { - EXTNAME: [ '.torrent' ], - FILE_SIZE: { - max: 1024 * 200 // 200 KB - } - } - }, - VIDEOS_REDUNDANCY: { - URL: { min: 3, max: 2000 } // Length - }, - VIDEO_RATES: { - URL: { min: 3, max: 2000 } // Length - }, - VIDEOS: { - NAME: { min: 3, max: 120 }, // Length - LANGUAGE: { min: 1, max: 10 }, // Length - TRUNCATED_DESCRIPTION: { min: 3, max: 250 }, // Length - DESCRIPTION: { min: 3, max: 10000 }, // Length - SUPPORT: { min: 3, max: 1000 }, // Length - IMAGE: { - EXTNAME: [ '.png', '.jpg', '.jpeg', '.webp' ], - FILE_SIZE: { - max: 4 * 1024 * 1024 // 4MB - } - }, - EXTNAME: [] as string[], - INFO_HASH: { min: 40, max: 40 }, // Length, info hash is 20 bytes length but we represent it in hexadecimal so 20 * 2 - DURATION: { min: 0 }, // Number - TAGS: { min: 0, max: 5 }, // Number of total tags - TAG: { min: 2, max: 30 }, // Length - VIEWS: { min: 0 }, - LIKES: { min: 0 }, - DISLIKES: { min: 0 }, - FILE_SIZE: { min: -1 }, - PARTIAL_UPLOAD_SIZE: { max: 50 * 1024 * 1024 * 1024 }, // 50GB - URL: { min: 3, max: 2000 } // Length - }, - VIDEO_PLAYLISTS: { - NAME: { min: 1, max: 120 }, // Length - DESCRIPTION: { min: 3, max: 1000 }, // Length - URL: { min: 3, max: 2000 }, // Length - IMAGE: { - EXTNAME: [ '.jpg', '.jpeg' ], - FILE_SIZE: { - max: 4 * 1024 * 1024 // 4MB - } - } - }, - ACTORS: { - PUBLIC_KEY: { min: 10, max: 5000 }, // Length - PRIVATE_KEY: { min: 10, max: 5000 }, // Length - URL: { min: 3, max: 2000 }, // Length - IMAGE: { - EXTNAME: [ '.png', '.jpeg', '.jpg', '.gif', '.webp' ], - FILE_SIZE: { - max: 4 * 1024 * 1024 // 4MB - } - } - }, - VIDEO_EVENTS: { - COUNT: { min: 0 } - }, - VIDEO_COMMENTS: { - TEXT: { min: 1, max: 10000 }, // Length - URL: { min: 3, max: 2000 } // Length - }, - VIDEO_SHARE: { - URL: { min: 3, max: 2000 } // Length - }, - CONTACT_FORM: { - FROM_NAME: { min: 1, max: 120 }, // Length - BODY: { min: 3, max: 5000 } // Length - }, - PLUGINS: { - NAME: { min: 1, max: 214 }, // Length - DESCRIPTION: { min: 1, max: 20000 } // Length - }, - COMMONS: { - URL: { min: 5, max: 2000 } // Length - }, - VIDEO_STUDIO: { - TASKS: { min: 1, max: 10 }, // Number of tasks - CUT_TIME: { min: 0 } // Value - }, - LOGS: { - CLIENT_MESSAGE: { min: 1, max: 1000 }, // Length - CLIENT_STACK_TRACE: { min: 1, max: 15000 }, // Length - CLIENT_META: { min: 1, max: 5000 }, // Length - CLIENT_USER_AGENT: { min: 1, max: 200 } // Length - }, - RUNNERS: { - TOKEN: { min: 1, max: 1000 }, // Length - NAME: { min: 1, max: 100 }, // Length - DESCRIPTION: { min: 1, max: 1000 } // Length - }, - RUNNER_JOBS: { - TOKEN: { min: 1, max: 1000 }, // Length - REASON: { min: 1, max: 5000 }, // Length - ERROR_MESSAGE: { min: 1, max: 5000 }, // Length - PROGRESS: { min: 0, max: 100 } // Value - }, - VIDEO_PASSWORD: { - LENGTH: { min: 2, max: 100 } - } -} - -const VIEW_LIFETIME = { - VIEW: CONFIG.VIEWS.VIDEOS.IP_VIEW_EXPIRATION, - VIEWER_COUNTER: 60000 * 2, // 2 minutes - VIEWER_STATS: 60000 * 60 // 1 hour -} - -const MAX_LOCAL_VIEWER_WATCH_SECTIONS = 100 - -let CONTACT_FORM_LIFETIME = 60000 * 60 // 1 hour - -const VIDEO_TRANSCODING_FPS: VideoTranscodingFPS = { - MIN: 1, - STANDARD: [ 24, 25, 30 ], - HD_STANDARD: [ 50, 60 ], - AUDIO_MERGE: 25, - AVERAGE: 30, - MAX: 60, - KEEP_ORIGIN_FPS_RESOLUTION_MIN: 720 // We keep the original FPS on high resolutions (720 minimum) -} - -const DEFAULT_AUDIO_RESOLUTION = VideoResolution.H_480P - -const VIDEO_RATE_TYPES: { [ id: string ]: VideoRateType } = { - LIKE: 'like', - DISLIKE: 'dislike' -} - -const FFMPEG_NICE = { - // parent process defaults to niceness = 0 - // reminder: lower = higher priority, max value is 19, lowest is -20 - LIVE: 5, // prioritize over VOD and THUMBNAIL - THUMBNAIL: 10, - VOD: 15 -} - -const VIDEO_CATEGORIES = { - 1: 'Music', - 2: 'Films', - 3: 'Vehicles', - 4: 'Art', - 5: 'Sports', - 6: 'Travels', - 7: 'Gaming', - 8: 'People', - 9: 'Comedy', - 10: 'Entertainment', - 11: 'News & Politics', - 12: 'How To', - 13: 'Education', - 14: 'Activism', - 15: 'Science & Technology', - 16: 'Animals', - 17: 'Kids', - 18: 'Food' -} - -// See https://creativecommons.org/licenses/?lang=en -const VIDEO_LICENCES = { - 1: 'Attribution', - 2: 'Attribution - Share Alike', - 3: 'Attribution - No Derivatives', - 4: 'Attribution - Non Commercial', - 5: 'Attribution - Non Commercial - Share Alike', - 6: 'Attribution - Non Commercial - No Derivatives', - 7: 'Public Domain Dedication' -} - -const VIDEO_LANGUAGES: { [id: string]: string } = {} - -const VIDEO_PRIVACIES: { [ id in VideoPrivacy ]: string } = { - [VideoPrivacy.PUBLIC]: 'Public', - [VideoPrivacy.UNLISTED]: 'Unlisted', - [VideoPrivacy.PRIVATE]: 'Private', - [VideoPrivacy.INTERNAL]: 'Internal', - [VideoPrivacy.PASSWORD_PROTECTED]: 'Password protected' -} - -const VIDEO_STATES: { [ id in VideoState ]: string } = { - [VideoState.PUBLISHED]: 'Published', - [VideoState.TO_TRANSCODE]: 'To transcode', - [VideoState.TO_IMPORT]: 'To import', - [VideoState.WAITING_FOR_LIVE]: 'Waiting for livestream', - [VideoState.LIVE_ENDED]: 'Livestream ended', - [VideoState.TO_MOVE_TO_EXTERNAL_STORAGE]: 'To move to an external storage', - [VideoState.TRANSCODING_FAILED]: 'Transcoding failed', - [VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED]: 'External storage move failed', - [VideoState.TO_EDIT]: 'To edit*' -} - -const VIDEO_IMPORT_STATES: { [ id in VideoImportState ]: string } = { - [VideoImportState.FAILED]: 'Failed', - [VideoImportState.PENDING]: 'Pending', - [VideoImportState.SUCCESS]: 'Success', - [VideoImportState.REJECTED]: 'Rejected', - [VideoImportState.CANCELLED]: 'Cancelled', - [VideoImportState.PROCESSING]: 'Processing' -} - -const VIDEO_CHANNEL_SYNC_STATE: { [ id in VideoChannelSyncState ]: string } = { - [VideoChannelSyncState.FAILED]: 'Failed', - [VideoChannelSyncState.SYNCED]: 'Synchronized', - [VideoChannelSyncState.PROCESSING]: 'Processing', - [VideoChannelSyncState.WAITING_FIRST_RUN]: 'Waiting first run' -} - -const ABUSE_STATES: { [ id in AbuseState ]: string } = { - [AbuseState.PENDING]: 'Pending', - [AbuseState.REJECTED]: 'Rejected', - [AbuseState.ACCEPTED]: 'Accepted' -} - -const USER_REGISTRATION_STATES: { [ id in UserRegistrationState ]: string } = { - [UserRegistrationState.PENDING]: 'Pending', - [UserRegistrationState.REJECTED]: 'Rejected', - [UserRegistrationState.ACCEPTED]: 'Accepted' -} - -const VIDEO_PLAYLIST_PRIVACIES: { [ id in VideoPlaylistPrivacy ]: string } = { - [VideoPlaylistPrivacy.PUBLIC]: 'Public', - [VideoPlaylistPrivacy.UNLISTED]: 'Unlisted', - [VideoPlaylistPrivacy.PRIVATE]: 'Private' -} - -const VIDEO_PLAYLIST_TYPES: { [ id in VideoPlaylistType ]: string } = { - [VideoPlaylistType.REGULAR]: 'Regular', - [VideoPlaylistType.WATCH_LATER]: 'Watch later' -} - -const RUNNER_JOB_STATES: { [ id in RunnerJobState ]: string } = { - [RunnerJobState.PROCESSING]: 'Processing', - [RunnerJobState.COMPLETED]: 'Completed', - [RunnerJobState.COMPLETING]: 'Completing', - [RunnerJobState.PENDING]: 'Pending', - [RunnerJobState.ERRORED]: 'Errored', - [RunnerJobState.WAITING_FOR_PARENT_JOB]: 'Waiting for parent job to finish', - [RunnerJobState.CANCELLED]: 'Cancelled', - [RunnerJobState.PARENT_ERRORED]: 'Parent job failed', - [RunnerJobState.PARENT_CANCELLED]: 'Parent job cancelled' -} - -const MIMETYPES = { - AUDIO: { - MIMETYPE_EXT: { - 'audio/mpeg': '.mp3', - 'audio/mp3': '.mp3', - - 'application/ogg': '.ogg', - 'audio/ogg': '.ogg', - - 'audio/x-ms-wma': '.wma', - 'audio/wav': '.wav', - 'audio/x-wav': '.wav', - - 'audio/x-flac': '.flac', - 'audio/flac': '.flac', - - 'audio/vnd.dlna.adts': '.aac', - 'audio/aac': '.aac', - - 'audio/m4a': '.m4a', - 'audio/mp4': '.m4a', - 'audio/x-m4a': '.m4a', - - 'audio/vnd.dolby.dd-raw': '.ac3', - 'audio/ac3': '.ac3' - }, - EXT_MIMETYPE: null as { [ id: string ]: string } - }, - VIDEO: { - MIMETYPE_EXT: null as { [ id: string ]: string | string[] }, - MIMETYPES_REGEX: null as string, - EXT_MIMETYPE: null as { [ id: string ]: string } - }, - IMAGE: { - MIMETYPE_EXT: { - 'image/png': '.png', - 'image/gif': '.gif', - 'image/webp': '.webp', - 'image/jpg': '.jpg', - 'image/jpeg': '.jpg' - }, - EXT_MIMETYPE: null as { [ id: string ]: string } - }, - VIDEO_CAPTIONS: { - MIMETYPE_EXT: { - 'text/vtt': '.vtt', - 'application/x-subrip': '.srt', - 'text/plain': '.srt' - }, - EXT_MIMETYPE: null as { [ id: string ]: string } - }, - TORRENT: { - MIMETYPE_EXT: { - 'application/x-bittorrent': '.torrent' - } - }, - M3U8: { - MIMETYPE_EXT: { - 'application/vnd.apple.mpegurl': '.m3u8' - } - } -} -MIMETYPES.AUDIO.EXT_MIMETYPE = invert(MIMETYPES.AUDIO.MIMETYPE_EXT) -MIMETYPES.IMAGE.EXT_MIMETYPE = invert(MIMETYPES.IMAGE.MIMETYPE_EXT) -MIMETYPES.VIDEO_CAPTIONS.EXT_MIMETYPE = invert(MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT) - -const BINARY_CONTENT_TYPES = new Set([ - 'binary/octet-stream', - 'application/octet-stream', - 'application/x-binary' -]) - -// --------------------------------------------------------------------------- - -const OVERVIEWS = { - VIDEOS: { - SAMPLE_THRESHOLD: 6, - SAMPLES_COUNT: 20 - } -} - -// --------------------------------------------------------------------------- - -const SERVER_ACTOR_NAME = 'peertube' - -const ACTIVITY_PUB = { - POTENTIAL_ACCEPT_HEADERS: [ - 'application/activity+json', - 'application/ld+json', - 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"' - ], - ACCEPT_HEADER: 'application/activity+json, application/ld+json', - PUBLIC: 'https://www.w3.org/ns/activitystreams#Public', - COLLECTION_ITEMS_PER_PAGE: 10, - FETCH_PAGE_LIMIT: 2000, - URL_MIME_TYPES: { - VIDEO: [] as string[], - TORRENT: [ 'application/x-bittorrent' ], - MAGNET: [ 'application/x-bittorrent;x-scheme-handler/magnet' ] - }, - MAX_RECURSION_COMMENTS: 100, - ACTOR_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 days - VIDEO_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 days - VIDEO_PLAYLIST_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2 // 2 days -} - -const ACTIVITY_PUB_ACTOR_TYPES: { [ id: string ]: ActivityPubActorType } = { - GROUP: 'Group', - PERSON: 'Person', - APPLICATION: 'Application', - ORGANIZATION: 'Organization', - SERVICE: 'Service' -} - -const HTTP_SIGNATURE = { - HEADER_NAME: 'signature', - ALGORITHM: 'rsa-sha256', - HEADERS_TO_SIGN_WITH_PAYLOAD: [ '(request-target)', 'host', 'date', 'digest' ], - HEADERS_TO_SIGN_WITHOUT_PAYLOAD: [ '(request-target)', 'host', 'date' ], - CLOCK_SKEW_SECONDS: 1800 -} - -// --------------------------------------------------------------------------- - -let PRIVATE_RSA_KEY_SIZE = 2048 - -// Password encryption -const BCRYPT_SALT_SIZE = 10 - -const ENCRYPTION = { - ALGORITHM: 'aes-256-cbc', - IV: 16, - SALT: 'peertube', - ENCODING: 'hex' as Encoding -} - -const USER_PASSWORD_RESET_LIFETIME = 60000 * 60 // 60 minutes -const USER_PASSWORD_CREATE_LIFETIME = 60000 * 60 * 24 * 7 // 7 days - -const TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME = 60000 * 10 // 10 minutes - -const EMAIL_VERIFY_LIFETIME = 60000 * 60 // 60 minutes - -const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = { - DO_NOT_LIST: 'do_not_list', - BLUR: 'blur', - DISPLAY: 'display' -} - -// --------------------------------------------------------------------------- - -// Express static paths (router) -const STATIC_PATHS = { - // TODO: deprecated in v6, to remove - THUMBNAILS: '/static/thumbnails/', - - // Need to keep this legacy path for previously generated torrents - LEGACY_WEB_VIDEOS: '/static/webseed/', - WEB_VIDEOS: '/static/web-videos/', - - // Need to keep this legacy path for previously generated torrents - LEGACY_PRIVATE_WEB_VIDEOS: '/static/webseed/private/', - PRIVATE_WEB_VIDEOS: '/static/web-videos/private/', - - REDUNDANCY: '/static/redundancy/', - - STREAMING_PLAYLISTS: { - HLS: '/static/streaming-playlists/hls', - PRIVATE_HLS: '/static/streaming-playlists/hls/private/' - } -} -const STATIC_DOWNLOAD_PATHS = { - TORRENTS: '/download/torrents/', - VIDEOS: '/download/videos/', - HLS_VIDEOS: '/download/streaming-playlists/hls/videos/' -} -const LAZY_STATIC_PATHS = { - THUMBNAILS: '/lazy-static/thumbnails/', - BANNERS: '/lazy-static/banners/', - AVATARS: '/lazy-static/avatars/', - PREVIEWS: '/lazy-static/previews/', - VIDEO_CAPTIONS: '/lazy-static/video-captions/', - TORRENTS: '/lazy-static/torrents/', - STORYBOARDS: '/lazy-static/storyboards/' -} -const OBJECT_STORAGE_PROXY_PATHS = { - // Need to keep this legacy path for previously generated torrents - LEGACY_PRIVATE_WEB_VIDEOS: '/object-storage-proxy/webseed/private/', - PRIVATE_WEB_VIDEOS: '/object-storage-proxy/web-videos/private/', - - STREAMING_PLAYLISTS: { - PRIVATE_HLS: '/object-storage-proxy/streaming-playlists/hls/private/' - } -} - -// Cache control -const STATIC_MAX_AGE = { - SERVER: '2h', - LAZY_SERVER: '2d', - CLIENT: '30d' -} - -// Videos thumbnail size -const THUMBNAILS_SIZE = { - width: 280, - height: 157, - minWidth: 150 -} -const PREVIEWS_SIZE = { - width: 850, - height: 480, - minWidth: 400 -} -const ACTOR_IMAGES_SIZE: { [key in ActorImageType]: { width: number, height: number }[] } = { - [ActorImageType.AVATAR]: [ - { - width: 120, - height: 120 - }, - { - width: 48, - height: 48 - } - ], - [ActorImageType.BANNER]: [ - { - width: 1920, - height: 317 // 6/1 ratio - } - ] -} - -const STORYBOARD = { - SPRITE_SIZE: { - width: 192, - height: 108 - }, - SPRITES_MAX_EDGE_COUNT: 10 -} - -const EMBED_SIZE = { - width: 560, - height: 315 -} - -// Sub folders of cache directory -const FILES_CACHE = { - PREVIEWS: { - DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'previews'), - MAX_AGE: 1000 * 3600 * 3 // 3 hours - }, - STORYBOARDS: { - DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'storyboards'), - MAX_AGE: 1000 * 3600 * 24 // 24 hours - }, - VIDEO_CAPTIONS: { - DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'video-captions'), - MAX_AGE: 1000 * 3600 * 3 // 3 hours - }, - TORRENTS: { - DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'torrents'), - MAX_AGE: 1000 * 3600 * 3 // 3 hours - } -} - -const LRU_CACHE = { - USER_TOKENS: { - MAX_SIZE: 1000 - }, - FILENAME_TO_PATH_PERMANENT_FILE_CACHE: { - MAX_SIZE: 1000 - }, - STATIC_VIDEO_FILES_RIGHTS_CHECK: { - MAX_SIZE: 5000, - TTL: parseDurationToMs('10 seconds') - }, - VIDEO_TOKENS: { - MAX_SIZE: 100_000, - TTL: parseDurationToMs('8 hours') - }, - TRACKER_IPS: { - MAX_SIZE: 100_000 - } -} - -const DIRECTORIES = { - RESUMABLE_UPLOAD: join(CONFIG.STORAGE.TMP_DIR, 'resumable-uploads'), - - HLS_STREAMING_PLAYLIST: { - PUBLIC: join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls'), - PRIVATE: join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls', 'private') - }, - - VIDEOS: { - PUBLIC: CONFIG.STORAGE.WEB_VIDEOS_DIR, - PRIVATE: join(CONFIG.STORAGE.WEB_VIDEOS_DIR, 'private') - }, - - HLS_REDUNDANCY: join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls') -} - -const RESUMABLE_UPLOAD_SESSION_LIFETIME = SCHEDULER_INTERVALS_MS.REMOVE_DANGLING_RESUMABLE_UPLOADS - -const VIDEO_LIVE = { - EXTENSION: '.ts', - CLEANUP_DELAY: 1000 * 60 * 5, // 5 minutes - SEGMENT_TIME_SECONDS: { - DEFAULT_LATENCY: 4, // 4 seconds - SMALL_LATENCY: 2 // 2 seconds - }, - SEGMENTS_LIST_SIZE: 15, // 15 maximum segments in live playlist - REPLAY_DIRECTORY: 'replay', - EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION: 4, - MAX_SOCKET_WAITING_DATA: 1024 * 1000 * 100, // 100MB - RTMP: { - CHUNK_SIZE: 60000, - GOP_CACHE: true, - PING: 60, - PING_TIMEOUT: 30, - BASE_PATH: 'live' - } -} - -const MEMOIZE_TTL = { - OVERVIEWS_SAMPLE: 1000 * 3600 * 4, // 4 hours - INFO_HASH_EXISTS: 1000 * 60, // 1 minute - VIDEO_DURATION: 1000 * 10, // 10 seconds - LIVE_ABLE_TO_UPLOAD: 1000 * 60, // 1 minute - LIVE_CHECK_SOCKET_HEALTH: 1000 * 60, // 1 minute - GET_STATS_FOR_OPEN_TELEMETRY_METRICS: 1000 * 60 // 1 minute -} - -const MEMOIZE_LENGTH = { - INFO_HASH_EXISTS: 200, - VIDEO_DURATION: 200 -} - -const WORKER_THREADS = { - DOWNLOAD_IMAGE: { - CONCURRENCY: 3, - MAX_THREADS: 1 - }, - PROCESS_IMAGE: { - CONCURRENCY: 1, - MAX_THREADS: 5 - } -} - -const REDUNDANCY = { - VIDEOS: { - RANDOMIZED_FACTOR: 5 - } -} - -const ACCEPT_HEADERS = [ 'html', 'application/json' ].concat(ACTIVITY_PUB.POTENTIAL_ACCEPT_HEADERS) -const OTP = { - HEADER_NAME: 'x-peertube-otp', - HEADER_REQUIRED_VALUE: 'required; app' -} - -const ASSETS_PATH = { - DEFAULT_AUDIO_BACKGROUND: join(root(), 'dist', 'server', 'assets', 'default-audio-background.jpg'), - DEFAULT_LIVE_BACKGROUND: join(root(), 'dist', 'server', 'assets', 'default-live-background.jpg') -} - -// --------------------------------------------------------------------------- - -const CUSTOM_HTML_TAG_COMMENTS = { - TITLE: '', - DESCRIPTION: '', - CUSTOM_CSS: '', - META_TAGS: '', - SERVER_CONFIG: '' -} - -const MAX_LOGS_OUTPUT_CHARACTERS = 10 * 1000 * 1000 -const LOG_FILENAME = 'peertube.log' -const AUDIT_LOG_FILENAME = 'peertube-audit.log' - -// --------------------------------------------------------------------------- - -const TRACKER_RATE_LIMITS = { - INTERVAL: 60000 * 5, // 5 minutes - ANNOUNCES_PER_IP_PER_INFOHASH: 15, // maximum announces per torrent in the interval - ANNOUNCES_PER_IP: 30, // maximum announces for all our torrents in the interval - BLOCK_IP_LIFETIME: parseDurationToMs('3 minutes') -} - -const P2P_MEDIA_LOADER_PEER_VERSION = 2 - -// --------------------------------------------------------------------------- - -const PLUGIN_GLOBAL_CSS_FILE_NAME = 'plugins-global.css' -const PLUGIN_GLOBAL_CSS_PATH = join(CONFIG.STORAGE.TMP_DIR, PLUGIN_GLOBAL_CSS_FILE_NAME) - -let PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME = 1000 * 60 * 5 // 5 minutes - -const DEFAULT_THEME_NAME = 'default' -const DEFAULT_USER_THEME_NAME = 'instance-default' - -// --------------------------------------------------------------------------- - -const SEARCH_INDEX = { - ROUTES: { - VIDEOS: '/api/v1/search/videos', - VIDEO_CHANNELS: '/api/v1/search/video-channels' - } -} - -// --------------------------------------------------------------------------- - -const STATS_TIMESERIE = { - MAX_DAYS: 365 * 10 // Around 10 years -} - -// --------------------------------------------------------------------------- - -// Special constants for a test instance -if (process.env.PRODUCTION_CONSTANTS !== 'true') { - if (isTestOrDevInstance()) { - PRIVATE_RSA_KEY_SIZE = 1024 - - ACTOR_FOLLOW_SCORE.BASE = 20 - - REMOTE_SCHEME.HTTP = 'http' - REMOTE_SCHEME.WS = 'ws' - - STATIC_MAX_AGE.SERVER = '0' - - SCHEDULER_INTERVALS_MS.ACTOR_FOLLOW_SCORES = 1000 - SCHEDULER_INTERVALS_MS.REMOVE_OLD_JOBS = 10000 - SCHEDULER_INTERVALS_MS.REMOVE_OLD_HISTORY = 5000 - SCHEDULER_INTERVALS_MS.REMOVE_OLD_VIEWS = 5000 - SCHEDULER_INTERVALS_MS.UPDATE_VIDEOS = 5000 - SCHEDULER_INTERVALS_MS.AUTO_FOLLOW_INDEX_INSTANCES = 5000 - SCHEDULER_INTERVALS_MS.UPDATE_INBOX_STATS = 5000 - SCHEDULER_INTERVALS_MS.CHECK_PEERTUBE_VERSION = 2000 - - REPEAT_JOBS['videos-views-stats'] = { every: 5000 } - - REPEAT_JOBS['activitypub-cleaner'] = { every: 5000 } - AP_CLEANER.PERIOD = 5000 - - REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1 - - CONTACT_FORM_LIFETIME = 1000 // 1 second - - JOB_ATTEMPTS['email'] = 1 - - FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000 - MEMOIZE_TTL.OVERVIEWS_SAMPLE = 3000 - MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD = 3000 - OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD = 2 - - PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME = 5000 - - JOB_REMOVAL_OPTIONS.SUCCESS['videos-views-stats'] = 10000 - } - - if (isTestInstance()) { - ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2 - ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 10 * 1000 // 10 seconds - ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL = 10 * 1000 // 10 seconds - ACTIVITY_PUB.VIDEO_PLAYLIST_REFRESH_INTERVAL = 10 * 1000 // 10 seconds - - CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max = 100 * 1024 // 100KB - CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max = 400 * 1024 // 400KB - - VIEW_LIFETIME.VIEWER_COUNTER = 1000 * 5 // 5 second - VIEW_LIFETIME.VIEWER_STATS = 1000 * 5 // 5 second - - VIDEO_LIVE.CLEANUP_DELAY = getIntEnv('PEERTUBE_TEST_CONSTANTS_VIDEO_LIVE_CLEANUP_DELAY') ?? 5000 - VIDEO_LIVE.SEGMENT_TIME_SECONDS.DEFAULT_LATENCY = 2 - VIDEO_LIVE.SEGMENT_TIME_SECONDS.SMALL_LATENCY = 1 - VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION = 1 - - RUNNER_JOBS.LAST_CONTACT_UPDATE_INTERVAL = 2000 - } -} - -updateWebserverUrls() -updateWebserverConfig() - -registerConfigChangedHandler(() => { - updateWebserverUrls() - updateWebserverConfig() -}) - -// --------------------------------------------------------------------------- - -const FILES_CONTENT_HASH = { - MANIFEST: generateContentHash(), - FAVICON: generateContentHash(), - LOGO: generateContentHash() -} - -// --------------------------------------------------------------------------- - -const VIDEO_FILTERS = { - WATERMARK: { - SIZE_RATIO: 1 / 10, - HORIZONTAL_MARGIN_RATIO: 1 / 20, - VERTICAL_MARGIN_RATIO: 1 / 20 - } -} - -// --------------------------------------------------------------------------- - -export { - WEBSERVER, - API_VERSION, - ENCRYPTION, - VIDEO_LIVE, - PEERTUBE_VERSION, - LAZY_STATIC_PATHS, - OBJECT_STORAGE_PROXY_PATHS, - SEARCH_INDEX, - DIRECTORIES, - RESUMABLE_UPLOAD_SESSION_LIFETIME, - RUNNER_JOB_STATES, - P2P_MEDIA_LOADER_PEER_VERSION, - STORYBOARD, - ACTOR_IMAGES_SIZE, - ACCEPT_HEADERS, - BCRYPT_SALT_SIZE, - TRACKER_RATE_LIMITS, - FILES_CACHE, - LOG_FILENAME, - CONSTRAINTS_FIELDS, - EMBED_SIZE, - REDUNDANCY, - JOB_CONCURRENCY, - JOB_ATTEMPTS, - AP_CLEANER, - LAST_MIGRATION_VERSION, - CUSTOM_HTML_TAG_COMMENTS, - STATS_TIMESERIE, - BROADCAST_CONCURRENCY, - AUDIT_LOG_FILENAME, - PAGINATION, - ACTOR_FOLLOW_SCORE, - PREVIEWS_SIZE, - REMOTE_SCHEME, - FOLLOW_STATES, - DEFAULT_USER_THEME_NAME, - SERVER_ACTOR_NAME, - TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME, - PLUGIN_GLOBAL_CSS_FILE_NAME, - PLUGIN_GLOBAL_CSS_PATH, - PRIVATE_RSA_KEY_SIZE, - VIDEO_FILTERS, - ROUTE_CACHE_LIFETIME, - SORTABLE_COLUMNS, - JOB_TTL, - DEFAULT_THEME_NAME, - NSFW_POLICY_TYPES, - STATIC_MAX_AGE, - STATIC_PATHS, - VIDEO_IMPORT_TIMEOUT, - VIDEO_PLAYLIST_TYPES, - MAX_LOGS_OUTPUT_CHARACTERS, - ACTIVITY_PUB, - ACTIVITY_PUB_ACTOR_TYPES, - THUMBNAILS_SIZE, - VIDEO_CATEGORIES, - MEMOIZE_LENGTH, - VIDEO_LANGUAGES, - VIDEO_PRIVACIES, - VIDEO_LICENCES, - VIDEO_STATES, - WORKER_THREADS, - VIDEO_RATE_TYPES, - JOB_PRIORITY, - VIDEO_TRANSCODING_FPS, - FFMPEG_NICE, - ABUSE_STATES, - USER_REGISTRATION_STATES, - LRU_CACHE, - REQUEST_TIMEOUTS, - RUNNER_JOBS, - MAX_LOCAL_VIEWER_WATCH_SECTIONS, - USER_PASSWORD_RESET_LIFETIME, - USER_PASSWORD_CREATE_LIFETIME, - MEMOIZE_TTL, - EMAIL_VERIFY_LIFETIME, - OVERVIEWS, - SCHEDULER_INTERVALS_MS, - REPEAT_JOBS, - STATIC_DOWNLOAD_PATHS, - MIMETYPES, - CRAWL_REQUEST_CONCURRENCY, - DEFAULT_AUDIO_RESOLUTION, - BINARY_CONTENT_TYPES, - JOB_REMOVAL_OPTIONS, - HTTP_SIGNATURE, - VIDEO_IMPORT_STATES, - VIDEO_CHANNEL_SYNC_STATE, - VIEW_LIFETIME, - CONTACT_FORM_LIFETIME, - VIDEO_PLAYLIST_PRIVACIES, - PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME, - ASSETS_PATH, - FILES_CONTENT_HASH, - OTP, - loadLanguages, - buildLanguages, - generateContentHash -} - -// --------------------------------------------------------------------------- - -function buildVideoMimetypeExt () { - const data = { - // streamable formats that warrant cross-browser compatibility - 'video/webm': '.webm', - // We'll add .ogg if additional extensions are enabled - // We could add .ogg here but since it could be an audio file, - // it would be confusing for users because PeerTube will refuse their file (based on the mimetype) - 'video/ogg': [ '.ogv' ], - 'video/mp4': '.mp4' - } - - if (CONFIG.TRANSCODING.ENABLED) { - if (CONFIG.TRANSCODING.ALLOW_ADDITIONAL_EXTENSIONS) { - data['video/ogg'].push('.ogg') - - Object.assign(data, { - 'video/x-matroska': '.mkv', - - // Developed by Apple - 'video/quicktime': [ '.mov', '.qt', '.mqv' ], // often used as output format by editing software - 'video/x-m4v': '.m4v', - 'video/m4v': '.m4v', - - // Developed by the Adobe Flash Platform - 'video/x-flv': '.flv', - 'video/x-f4v': '.f4v', // replacement for flv - - // Developed by Microsoft - 'video/x-ms-wmv': '.wmv', - 'video/x-msvideo': '.avi', - 'video/avi': '.avi', - - // Developed by 3GPP - // common video formats for cell phones - 'video/3gpp': [ '.3gp', '.3gpp' ], - 'video/3gpp2': [ '.3g2', '.3gpp2' ], - - // Developed by FFmpeg/Mplayer - 'application/x-nut': '.nut', - - // The standard video format used by many Sony and Panasonic HD camcorders. - // It is also used for storing high definition video on Blu-ray discs. - 'video/mp2t': '.mts', - 'video/vnd.dlna.mpeg-tts': '.mts', - - 'video/m2ts': '.m2ts', - - // Old formats reliant on MPEG-1/MPEG-2 - 'video/mpv': '.mpv', - 'video/mpeg2': '.m2v', - 'video/mpeg': [ '.m1v', '.mpg', '.mpe', '.mpeg', '.vob' ], - 'video/dvd': '.vob', - - // Could be anything - 'application/octet-stream': null, - 'application/mxf': '.mxf' // often used as exchange format by editing software - }) - } - - if (CONFIG.TRANSCODING.ALLOW_AUDIO_FILES) { - Object.assign(data, MIMETYPES.AUDIO.MIMETYPE_EXT) - } - } - - return data -} - -function updateWebserverUrls () { - WEBSERVER.URL = sanitizeUrl(CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT) - WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP) - WEBSERVER.WS = CONFIG.WEBSERVER.WS - - WEBSERVER.SCHEME = CONFIG.WEBSERVER.SCHEME - WEBSERVER.HOSTNAME = CONFIG.WEBSERVER.HOSTNAME - WEBSERVER.PORT = CONFIG.WEBSERVER.PORT - - const rtmpHostname = CONFIG.LIVE.RTMP.PUBLIC_HOSTNAME || CONFIG.WEBSERVER.HOSTNAME - const rtmpsHostname = CONFIG.LIVE.RTMPS.PUBLIC_HOSTNAME || CONFIG.WEBSERVER.HOSTNAME - - WEBSERVER.RTMP_URL = 'rtmp://' + rtmpHostname + ':' + CONFIG.LIVE.RTMP.PORT - WEBSERVER.RTMPS_URL = 'rtmps://' + rtmpsHostname + ':' + CONFIG.LIVE.RTMPS.PORT - - WEBSERVER.RTMP_BASE_LIVE_URL = WEBSERVER.RTMP_URL + '/' + VIDEO_LIVE.RTMP.BASE_PATH - WEBSERVER.RTMPS_BASE_LIVE_URL = WEBSERVER.RTMPS_URL + '/' + VIDEO_LIVE.RTMP.BASE_PATH -} - -function updateWebserverConfig () { - MIMETYPES.VIDEO.MIMETYPE_EXT = buildVideoMimetypeExt() - MIMETYPES.VIDEO.MIMETYPES_REGEX = buildMimetypesRegex(MIMETYPES.VIDEO.MIMETYPE_EXT) - - ACTIVITY_PUB.URL_MIME_TYPES.VIDEO = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) - - MIMETYPES.VIDEO.EXT_MIMETYPE = buildVideoExtMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT) - - CONSTRAINTS_FIELDS.VIDEOS.EXTNAME = Object.keys(MIMETYPES.VIDEO.EXT_MIMETYPE) -} - -function buildVideoExtMimetype (obj: { [ id: string ]: string | string[] }) { - const result: { [id: string]: string } = {} - - for (const mimetype of Object.keys(obj)) { - const value = obj[mimetype] - if (!value) continue - - const extensions = Array.isArray(value) ? value : [ value ] - - for (const extension of extensions) { - result[extension] = mimetype - } - } - - return result -} - -function buildMimetypesRegex (obj: { [id: string]: string | string[] }) { - return Object.keys(obj) - .map(m => `(${m})`) - .join('|') -} - -function loadLanguages () { - Object.assign(VIDEO_LANGUAGES, buildLanguages()) -} - -function buildLanguages () { - const iso639 = require('iso-639-3') - - const languages: { [id: string]: string } = {} - - const additionalLanguages = { - sgn: true, // Sign languages (macro language) - ase: true, // American sign language - asq: true, // Austrian sign language - sdl: true, // Arabian sign language - bfi: true, // British sign language - bzs: true, // Brazilian sign language - csl: true, // Chinese sign language - cse: true, // Czech sign language - dsl: true, // Danish sign language - fsl: true, // French sign language - gsg: true, // German sign language - pks: true, // Pakistan sign language - jsl: true, // Japanese sign language - sfs: true, // South African sign language - swl: true, // Swedish sign language - rsl: true, // Russian sign language - - kab: true, // Kabyle - - lat: true, // Latin - - epo: true, // Esperanto - tlh: true, // Klingon - jbo: true, // Lojban - avk: true, // Kotava - - zxx: true // No linguistic content (ISO-639-2) - } - - // Only add ISO639-1 languages and some sign languages (ISO639-3) - iso639 - .filter(l => { - return (l.iso6391 !== undefined && l.type === 'living') || - additionalLanguages[l.iso6393] === true - }) - .forEach(l => { languages[l.iso6391 || l.iso6393] = l.name }) - - // Override Occitan label - languages['oc'] = 'Occitan' - languages['el'] = 'Greek' - languages['tok'] = 'Toki Pona' - - // Chinese languages - languages['zh-Hans'] = 'Simplified Chinese' - languages['zh-Hant'] = 'Traditional Chinese' - - return languages -} - -function generateContentHash () { - return randomBytes(20).toString('hex') -} - -function getIntEnv (path: string) { - if (process.env[path]) return parseInt(process.env[path]) - - return undefined -} diff --git a/server/initializers/database.ts b/server/initializers/database.ts deleted file mode 100644 index bc120e398..000000000 --- a/server/initializers/database.ts +++ /dev/null @@ -1,233 +0,0 @@ -import { QueryTypes, Transaction } from 'sequelize' -import { Sequelize as SequelizeTypescript } from 'sequelize-typescript' -import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' -import { RunnerModel } from '@server/models/runner/runner' -import { RunnerJobModel } from '@server/models/runner/runner-job' -import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token' -import { TrackerModel } from '@server/models/server/tracker' -import { VideoTrackerModel } from '@server/models/server/video-tracker' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { UserRegistrationModel } from '@server/models/user/user-registration' -import { UserVideoHistoryModel } from '@server/models/user/user-video-history' -import { StoryboardModel } from '@server/models/video/storyboard' -import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting' -import { VideoLiveSessionModel } from '@server/models/video/video-live-session' -import { VideoSourceModel } from '@server/models/video/video-source' -import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' -import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section' -import { isTestOrDevInstance } from '../helpers/core-utils' -import { logger } from '../helpers/logger' -import { AbuseModel } from '../models/abuse/abuse' -import { AbuseMessageModel } from '../models/abuse/abuse-message' -import { VideoAbuseModel } from '../models/abuse/video-abuse' -import { VideoCommentAbuseModel } from '../models/abuse/video-comment-abuse' -import { AccountModel } from '../models/account/account' -import { AccountBlocklistModel } from '../models/account/account-blocklist' -import { AccountVideoRateModel } from '../models/account/account-video-rate' -import { ActorModel } from '../models/actor/actor' -import { ActorFollowModel } from '../models/actor/actor-follow' -import { ActorImageModel } from '../models/actor/actor-image' -import { ApplicationModel } from '../models/application/application' -import { OAuthClientModel } from '../models/oauth/oauth-client' -import { OAuthTokenModel } from '../models/oauth/oauth-token' -import { VideoRedundancyModel } from '../models/redundancy/video-redundancy' -import { PluginModel } from '../models/server/plugin' -import { ServerModel } from '../models/server/server' -import { ServerBlocklistModel } from '../models/server/server-blocklist' -import { UserNotificationSettingModel } from '../models/user/user-notification-setting' -import { ScheduleVideoUpdateModel } from '../models/video/schedule-video-update' -import { TagModel } from '../models/video/tag' -import { ThumbnailModel } from '../models/video/thumbnail' -import { VideoModel } from '../models/video/video' -import { VideoBlacklistModel } from '../models/video/video-blacklist' -import { VideoCaptionModel } from '../models/video/video-caption' -import { VideoChangeOwnershipModel } from '../models/video/video-change-ownership' -import { VideoChannelModel } from '../models/video/video-channel' -import { VideoCommentModel } from '../models/video/video-comment' -import { VideoFileModel } from '../models/video/video-file' -import { VideoImportModel } from '../models/video/video-import' -import { VideoLiveModel } from '../models/video/video-live' -import { VideoPlaylistModel } from '../models/video/video-playlist' -import { VideoPlaylistElementModel } from '../models/video/video-playlist-element' -import { VideoShareModel } from '../models/video/video-share' -import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' -import { VideoTagModel } from '../models/video/video-tag' -import { VideoViewModel } from '../models/view/video-view' -import { CONFIG } from './config' -import { VideoPasswordModel } from '@server/models/video/video-password' - -require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string - -const dbname = CONFIG.DATABASE.DBNAME -const username = CONFIG.DATABASE.USERNAME -const password = CONFIG.DATABASE.PASSWORD -const host = CONFIG.DATABASE.HOSTNAME -const port = CONFIG.DATABASE.PORT -const poolMax = CONFIG.DATABASE.POOL.MAX - -let dialectOptions: any = {} - -if (CONFIG.DATABASE.SSL) { - dialectOptions = { - ssl: { - rejectUnauthorized: false - } - } -} - -const sequelizeTypescript = new SequelizeTypescript({ - database: dbname, - dialect: 'postgres', - dialectOptions, - host, - port, - username, - password, - pool: { - max: poolMax - }, - benchmark: isTestOrDevInstance(), - isolationLevel: Transaction.ISOLATION_LEVELS.SERIALIZABLE, - logging: (message: string, benchmark: number) => { - if (process.env.NODE_DB_LOG === 'false') return - - let newMessage = 'Executed SQL request' - if (isTestOrDevInstance() === true && benchmark !== undefined) { - newMessage += ' in ' + benchmark + 'ms' - } - - logger.debug(newMessage, { sql: message, tags: [ 'sql' ] }) - } -}) - -function checkDatabaseConnectionOrDie () { - sequelizeTypescript.authenticate() - .then(() => logger.debug('Connection to PostgreSQL has been established successfully.')) - .catch(err => { - - logger.error('Unable to connect to PostgreSQL database.', { err }) - process.exit(-1) - }) -} - -async function initDatabaseModels (silent: boolean) { - sequelizeTypescript.addModels([ - ApplicationModel, - ActorModel, - ActorFollowModel, - ActorImageModel, - AccountModel, - OAuthClientModel, - OAuthTokenModel, - ServerModel, - TagModel, - AccountVideoRateModel, - UserModel, - AbuseMessageModel, - AbuseModel, - VideoCommentAbuseModel, - VideoAbuseModel, - VideoModel, - VideoChangeOwnershipModel, - VideoChannelModel, - VideoShareModel, - VideoFileModel, - VideoSourceModel, - VideoCaptionModel, - VideoBlacklistModel, - VideoTagModel, - VideoCommentModel, - ScheduleVideoUpdateModel, - VideoImportModel, - VideoViewModel, - VideoRedundancyModel, - UserVideoHistoryModel, - VideoLiveModel, - VideoLiveSessionModel, - VideoLiveReplaySettingModel, - AccountBlocklistModel, - ServerBlocklistModel, - UserNotificationModel, - UserNotificationSettingModel, - VideoStreamingPlaylistModel, - VideoPlaylistModel, - VideoPlaylistElementModel, - LocalVideoViewerModel, - LocalVideoViewerWatchSectionModel, - ThumbnailModel, - TrackerModel, - VideoTrackerModel, - PluginModel, - ActorCustomPageModel, - VideoJobInfoModel, - VideoChannelSyncModel, - UserRegistrationModel, - VideoPasswordModel, - RunnerRegistrationTokenModel, - RunnerModel, - RunnerJobModel, - StoryboardModel - ]) - - // Check extensions exist in the database - await checkPostgresExtensions() - - // Create custom PostgreSQL functions - await createFunctions() - - if (!silent) logger.info('Database %s is ready.', dbname) -} - -// --------------------------------------------------------------------------- - -export { - initDatabaseModels, - checkDatabaseConnectionOrDie, - sequelizeTypescript -} - -// --------------------------------------------------------------------------- - -async function checkPostgresExtensions () { - const promises = [ - checkPostgresExtension('pg_trgm'), - checkPostgresExtension('unaccent') - ] - - return Promise.all(promises) -} - -async function checkPostgresExtension (extension: string) { - const query = `SELECT 1 FROM pg_available_extensions WHERE name = '${extension}' AND installed_version IS NOT NULL;` - const options = { - type: QueryTypes.SELECT as QueryTypes.SELECT, - raw: true - } - - const res = await sequelizeTypescript.query(query, options) - - if (!res || res.length === 0) { - // Try to create the extension ourselves - try { - await sequelizeTypescript.query(`CREATE EXTENSION ${extension};`, { raw: true }) - - } catch { - const errorMessage = `You need to enable ${extension} extension in PostgreSQL. ` + - `You can do so by running 'CREATE EXTENSION ${extension};' as a PostgreSQL super user in ${CONFIG.DATABASE.DBNAME} database.` - throw new Error(errorMessage) - } - } -} - -function createFunctions () { - const query = `CREATE OR REPLACE FUNCTION immutable_unaccent(text) - RETURNS text AS -$func$ -SELECT public.unaccent('public.unaccent', $1::text) -$func$ LANGUAGE sql IMMUTABLE;` - - return sequelizeTypescript.query(query, { raw: true }) -} diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts deleted file mode 100644 index 2406a5936..000000000 --- a/server/initializers/installer.ts +++ /dev/null @@ -1,199 +0,0 @@ -import { ensureDir, readdir, remove } from 'fs-extra' -import passwordGenerator from 'password-generator' -import { join } from 'path' -import { isTestOrDevInstance } from '@server/helpers/core-utils' -import { generateRunnerRegistrationToken } from '@server/helpers/token-generator' -import { getNodeABIVersion } from '@server/helpers/version' -import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token' -import { UserRole } from '@shared/models' -import { logger } from '../helpers/logger' -import { buildUser, createApplicationActor, createUserAccountAndChannelAndPlaylist } from '../lib/user' -import { ApplicationModel } from '../models/application/application' -import { OAuthClientModel } from '../models/oauth/oauth-client' -import { applicationExist, clientsExist, usersExist } from './checker-after-init' -import { CONFIG } from './config' -import { DIRECTORIES, FILES_CACHE, LAST_MIGRATION_VERSION } from './constants' -import { sequelizeTypescript } from './database' - -async function installApplication () { - try { - await Promise.all([ - // Database related - sequelizeTypescript.sync() - .then(() => { - return Promise.all([ - createApplicationIfNotExist(), - createOAuthClientIfNotExist(), - createOAuthAdminIfNotExist(), - createRunnerRegistrationTokenIfNotExist() - ]) - }), - - // Directories - removeCacheAndTmpDirectories() - .then(() => createDirectoriesIfNotExist()) - ]) - } catch (err) { - logger.error('Cannot install application.', { err }) - process.exit(-1) - } -} - -// --------------------------------------------------------------------------- - -export { - installApplication -} - -// --------------------------------------------------------------------------- - -function removeCacheAndTmpDirectories () { - const cacheDirectories = Object.keys(FILES_CACHE) - .map(k => FILES_CACHE[k].DIRECTORY) - - const tasks: Promise[] = [] - - // Cache directories - for (const dir of cacheDirectories) { - tasks.push(removeDirectoryOrContent(dir)) - } - - tasks.push(removeDirectoryOrContent(CONFIG.STORAGE.TMP_DIR)) - - return Promise.all(tasks) -} - -async function removeDirectoryOrContent (dir: string) { - try { - await remove(dir) - } catch (err) { - logger.debug('Cannot remove directory %s. Removing content instead.', dir, { err }) - - const files = await readdir(dir) - - for (const file of files) { - await remove(join(dir, file)) - } - } -} - -function createDirectoriesIfNotExist () { - const storage = CONFIG.STORAGE - const cacheDirectories = Object.keys(FILES_CACHE) - .map(k => FILES_CACHE[k].DIRECTORY) - - const tasks: Promise[] = [] - for (const key of Object.keys(storage)) { - const dir = storage[key] - tasks.push(ensureDir(dir)) - } - - // Cache directories - for (const dir of cacheDirectories) { - tasks.push(ensureDir(dir)) - } - - tasks.push(ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE)) - tasks.push(ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC)) - tasks.push(ensureDir(DIRECTORIES.VIDEOS.PUBLIC)) - tasks.push(ensureDir(DIRECTORIES.VIDEOS.PRIVATE)) - - // Resumable upload directory - tasks.push(ensureDir(DIRECTORIES.RESUMABLE_UPLOAD)) - - return Promise.all(tasks) -} - -async function createOAuthClientIfNotExist () { - const exist = await clientsExist() - // Nothing to do, clients already exist - if (exist === true) return undefined - - logger.info('Creating a default OAuth Client.') - - const id = passwordGenerator(32, false, /[a-z0-9]/) - const secret = passwordGenerator(32, false, /[a-zA-Z0-9]/) - const client = new OAuthClientModel({ - clientId: id, - clientSecret: secret, - grants: [ 'password', 'refresh_token' ], - redirectUris: null - }) - - const createdClient = await client.save() - logger.info('Client id: ' + createdClient.clientId) - logger.info('Client secret: ' + createdClient.clientSecret) - - return undefined -} - -async function createOAuthAdminIfNotExist () { - const exist = await usersExist() - // Nothing to do, users already exist - if (exist === true) return undefined - - logger.info('Creating the administrator.') - - const username = 'root' - const role = UserRole.ADMINISTRATOR - const email = CONFIG.ADMIN.EMAIL - let validatePassword = true - let password = '' - - // Do not generate a random password for test and dev environments - if (isTestOrDevInstance()) { - password = 'test' - - if (process.env.NODE_APP_INSTANCE) { - password += process.env.NODE_APP_INSTANCE - } - - // Our password is weak so do not validate it - validatePassword = false - } else if (process.env.PT_INITIAL_ROOT_PASSWORD) { - password = process.env.PT_INITIAL_ROOT_PASSWORD - } else { - password = passwordGenerator(16, true) - } - - const user = buildUser({ - username, - email, - password, - role, - emailVerified: true, - videoQuota: -1, - videoQuotaDaily: -1 - }) - - await createUserAccountAndChannelAndPlaylist({ userToCreate: user, channelNames: undefined, validateUser: validatePassword }) - logger.info('Username: ' + username) - logger.info('User password: ' + password) -} - -async function createApplicationIfNotExist () { - const exist = await applicationExist() - // Nothing to do, application already exist - if (exist === true) return undefined - - logger.info('Creating application account.') - - const application = await ApplicationModel.create({ - migrationVersion: LAST_MIGRATION_VERSION, - nodeVersion: process.version, - nodeABIVersion: getNodeABIVersion() - }) - - return createApplicationActor(application.id) -} - -async function createRunnerRegistrationTokenIfNotExist () { - const total = await RunnerRegistrationTokenModel.countTotal() - if (total !== 0) return undefined - - const token = new RunnerRegistrationTokenModel({ - registrationToken: generateRunnerRegistrationToken() - }) - - await token.save() -} diff --git a/server/initializers/migrations/0530-playlist-multiple-video.ts b/server/initializers/migrations/0530-playlist-multiple-video.ts deleted file mode 100644 index 51a8c06b0..000000000 --- a/server/initializers/migrations/0530-playlist-multiple-video.ts +++ /dev/null @@ -1,46 +0,0 @@ -import * as Sequelize from 'sequelize' -import { WEBSERVER } from '../constants' - -async function up (utils: { - transaction: Sequelize.Transaction - queryInterface: Sequelize.QueryInterface - sequelize: Sequelize.Sequelize -}): Promise { - { - const field = { - type: Sequelize.STRING, - allowNull: true - } - await utils.queryInterface.changeColumn('videoPlaylistElement', 'url', field) - } - - { - await utils.sequelize.query('DROP INDEX IF EXISTS video_playlist_element_video_playlist_id_video_id;') - } - - { - const selectPlaylistUUID = 'SELECT "uuid" FROM "videoPlaylist" WHERE "id" = "videoPlaylistElement"."videoPlaylistId"' - const url = `'${WEBSERVER.URL}' || '/video-playlists/' || (${selectPlaylistUUID}) || '/videos/' || "videoPlaylistElement"."id"` - - const query = ` - UPDATE "videoPlaylistElement" SET "url" = ${url} WHERE id IN ( - SELECT "videoPlaylistElement"."id" FROM "videoPlaylistElement" - INNER JOIN "videoPlaylist" ON "videoPlaylist".id = "videoPlaylistElement"."videoPlaylistId" - INNER JOIN account ON account.id = "videoPlaylist"."ownerAccountId" - INNER JOIN actor ON actor.id = account."actorId" - WHERE actor."serverId" IS NULL - )` - - await utils.sequelize.query(query) - } - -} - -function down (options) { - throw new Error('Not implemented.') -} - -export { - up, - down -} diff --git a/server/initializers/migrations/0560-user-feed-token.ts b/server/initializers/migrations/0560-user-feed-token.ts deleted file mode 100644 index 4c85b04f7..000000000 --- a/server/initializers/migrations/0560-user-feed-token.ts +++ /dev/null @@ -1,51 +0,0 @@ -import * as Sequelize from 'sequelize' -import { buildUUID } from '@shared/extra-utils' - -async function up (utils: { - transaction: Sequelize.Transaction - queryInterface: Sequelize.QueryInterface - sequelize: Sequelize.Sequelize - db: any -}): Promise { - const q = utils.queryInterface - - { - // Create uuid column for users - const userFeedTokenUUID = { - type: Sequelize.UUID, - defaultValue: Sequelize.UUIDV4, - allowNull: true - } - await q.addColumn('user', 'feedToken', userFeedTokenUUID) - } - - // Set UUID to previous users - { - const query = 'SELECT * FROM "user" WHERE "feedToken" IS NULL' - const options = { type: Sequelize.QueryTypes.SELECT as Sequelize.QueryTypes.SELECT } - const users = await utils.sequelize.query(query, options) - - for (const user of users) { - const queryUpdate = `UPDATE "user" SET "feedToken" = '${buildUUID()}' WHERE id = ${user.id}` - await utils.sequelize.query(queryUpdate) - } - } - - { - const userFeedTokenUUID = { - type: Sequelize.UUID, - defaultValue: Sequelize.UUIDV4, - allowNull: false - } - await q.changeColumn('user', 'feedToken', userFeedTokenUUID) - } -} - -function down (options) { - throw new Error('Not implemented.') -} - -export { - up, - down -} diff --git a/server/initializers/migrations/0605-actor-missing-keys.ts b/server/initializers/migrations/0605-actor-missing-keys.ts deleted file mode 100644 index aa89a500c..000000000 --- a/server/initializers/migrations/0605-actor-missing-keys.ts +++ /dev/null @@ -1,33 +0,0 @@ -import * as Sequelize from 'sequelize' -import { generateRSAKeyPairPromise } from '../../helpers/core-utils' -import { PRIVATE_RSA_KEY_SIZE } from '../constants' - -async function up (utils: { - transaction: Sequelize.Transaction - queryInterface: Sequelize.QueryInterface - sequelize: Sequelize.Sequelize - db: any -}): Promise { - - { - const query = 'SELECT * FROM "actor" WHERE "serverId" IS NULL AND "publicKey" IS NULL' - const options = { type: Sequelize.QueryTypes.SELECT as Sequelize.QueryTypes.SELECT } - const actors = await utils.sequelize.query(query, options) - - for (const actor of actors) { - const { privateKey, publicKey } = await generateRSAKeyPairPromise(PRIVATE_RSA_KEY_SIZE) - - const queryUpdate = `UPDATE "actor" SET "publicKey" = '${publicKey}', "privateKey" = '${privateKey}' WHERE id = ${actor.id}` - await utils.sequelize.query(queryUpdate) - } - } -} - -function down (options) { - throw new Error('Not implemented.') -} - -export { - up, - down -} diff --git a/server/initializers/migrations/0660-object-storage.ts b/server/initializers/migrations/0660-object-storage.ts deleted file mode 100644 index 53cb89ce6..000000000 --- a/server/initializers/migrations/0660-object-storage.ts +++ /dev/null @@ -1,56 +0,0 @@ -import * as Sequelize from 'sequelize' -import { VideoStorage } from '@shared/models' - -async function up (utils: { - transaction: Sequelize.Transaction - queryInterface: Sequelize.QueryInterface - sequelize: Sequelize.Sequelize - db: any -}): Promise { - { - const query = ` - CREATE TABLE IF NOT EXISTS "videoJobInfo" ( - "id" serial, - "pendingMove" INTEGER NOT NULL, - "pendingTranscode" INTEGER NOT NULL, - "videoId" serial UNIQUE NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE, - "createdAt" timestamp WITH time zone NOT NULL, - "updatedAt" timestamp WITH time zone NOT NULL, - PRIMARY KEY ("id") - ); - ` - - await utils.sequelize.query(query) - } - - { - await utils.queryInterface.addColumn('videoFile', 'storage', { - type: Sequelize.INTEGER, - allowNull: true, - defaultValue: VideoStorage.FILE_SYSTEM - }) - await utils.queryInterface.changeColumn('videoFile', 'storage', { type: Sequelize.INTEGER, allowNull: false, defaultValue: null }) - } - - { - await utils.queryInterface.addColumn('videoStreamingPlaylist', 'storage', { - type: Sequelize.INTEGER, - allowNull: true, - defaultValue: VideoStorage.FILE_SYSTEM - }) - await utils.queryInterface.changeColumn('videoStreamingPlaylist', 'storage', { - type: Sequelize.INTEGER, - allowNull: false, - defaultValue: null - }) - } -} - -function down (options) { - throw new Error('Not implemented.') -} - -export { - up, - down -} diff --git a/server/initializers/migrations/0690-live-latency-mode.ts b/server/initializers/migrations/0690-live-latency-mode.ts deleted file mode 100644 index c31a61364..000000000 --- a/server/initializers/migrations/0690-live-latency-mode.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { LiveVideoLatencyMode } from '@shared/models' -import * as Sequelize from 'sequelize' - -async function up (utils: { - transaction: Sequelize.Transaction - queryInterface: Sequelize.QueryInterface - sequelize: Sequelize.Sequelize - db: any -}): Promise { - await utils.queryInterface.addColumn('videoLive', 'latencyMode', { - type: Sequelize.INTEGER, - defaultValue: null, - allowNull: true - }, { transaction: utils.transaction }) - - { - const query = `UPDATE "videoLive" SET "latencyMode" = ${LiveVideoLatencyMode.DEFAULT}` - await utils.sequelize.query(query, { type: Sequelize.QueryTypes.UPDATE, transaction: utils.transaction }) - } - - await utils.queryInterface.changeColumn('videoLive', 'latencyMode', { - type: Sequelize.INTEGER, - defaultValue: null, - allowNull: false - }, { transaction: utils.transaction }) -} - -function down () { - throw new Error('Not implemented.') -} - -export { - up, - down -} diff --git a/server/initializers/migrator.ts b/server/initializers/migrator.ts deleted file mode 100644 index 7ac20127e..000000000 --- a/server/initializers/migrator.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { readdir } from 'fs-extra' -import { join } from 'path' -import { QueryTypes } from 'sequelize' -import { logger } from '../helpers/logger' -import { LAST_MIGRATION_VERSION } from './constants' -import { sequelizeTypescript } from './database' - -async function migrate () { - const tables = await sequelizeTypescript.getQueryInterface().showAllTables() - - // No tables, we don't need to migrate anything - // The installer will do that - if (tables.length === 0) return - - let actualVersion: number | null = null - - const query = 'SELECT "migrationVersion" FROM "application"' - const options = { - type: QueryTypes.SELECT as QueryTypes.SELECT - } - - const rows = await sequelizeTypescript.query<{ migrationVersion: number }>(query, options) - if (rows?.[0]?.migrationVersion) { - actualVersion = rows[0].migrationVersion - } - - if (actualVersion === null) { - await sequelizeTypescript.query('INSERT INTO "application" ("migrationVersion") VALUES (0)') - actualVersion = 0 - } - - // No need migrations, abort - if (actualVersion >= LAST_MIGRATION_VERSION) return - - // If there are a new migration scripts - logger.info('Begin migrations.') - - const migrationScripts = await getMigrationScripts() - - for (const migrationScript of migrationScripts) { - try { - await executeMigration(actualVersion, migrationScript) - } catch (err) { - logger.error('Cannot execute migration %s.', migrationScript.version, { err }) - process.exit(-1) - } - } - - logger.info('Migrations finished. New migration version schema: %s', LAST_MIGRATION_VERSION) -} - -// --------------------------------------------------------------------------- - -export { - migrate -} - -// --------------------------------------------------------------------------- - -async function getMigrationScripts () { - const files = await readdir(join(__dirname, 'migrations')) - const filesToMigrate: { - version: string - script: string - }[] = [] - - files - .filter(file => file.endsWith('.js')) - .forEach(file => { - // Filename is something like 'version-blabla.js' - const version = file.split('-')[0] - filesToMigrate.push({ - version, - script: file - }) - }) - - return filesToMigrate -} - -async function executeMigration (actualVersion: number, entity: { version: string, script: string }) { - const versionScript = parseInt(entity.version, 10) - - // Do not execute old migration scripts - if (versionScript <= actualVersion) return undefined - - // Load the migration module and run it - const migrationScriptName = entity.script - logger.info('Executing %s migration script.', migrationScriptName) - - const migrationScript = require(join(__dirname, 'migrations', migrationScriptName)) - - return sequelizeTypescript.transaction(async t => { - const options = { - transaction: t, - queryInterface: sequelizeTypescript.getQueryInterface(), - sequelize: sequelizeTypescript - } - - await migrationScript.up(options) - - // Update the new migration version - await sequelizeTypescript.query('UPDATE "application" SET "migrationVersion" = ' + versionScript, { transaction: t }) - }) -} diff --git a/server/lib/activitypub/activity.ts b/server/lib/activitypub/activity.ts deleted file mode 100644 index 391bcd9c6..000000000 --- a/server/lib/activitypub/activity.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { doJSONRequest, PeerTubeRequestOptions } from '@server/helpers/requests' -import { CONFIG } from '@server/initializers/config' -import { ActivityObject, ActivityPubActor, ActivityType, APObjectId } from '@shared/models' -import { buildSignedRequestOptions } from './send' - -export function getAPId (object: string | { id: string }) { - if (typeof object === 'string') return object - - return object.id -} - -export function getActivityStreamDuration (duration: number) { - // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration - return 'PT' + duration + 'S' -} - -export function getDurationFromActivityStream (duration: string) { - return parseInt(duration.replace(/[^\d]+/, '')) -} - -// --------------------------------------------------------------------------- - -export function buildAvailableActivities (): ActivityType[] { - return [ - 'Create', - 'Update', - 'Delete', - 'Follow', - 'Accept', - 'Announce', - 'Undo', - 'Like', - 'Reject', - 'View', - 'Dislike', - 'Flag' - ] -} - -// --------------------------------------------------------------------------- - -export async function fetchAP (url: string, moreOptions: PeerTubeRequestOptions = {}) { - const options = { - activityPub: true, - - httpSignature: CONFIG.FEDERATION.SIGN_FEDERATED_FETCHES - ? await buildSignedRequestOptions({ hasPayload: false }) - : undefined, - - ...moreOptions - } - - return doJSONRequest(url, options) -} - -export async function fetchAPObjectIfNeeded (object: APObjectId) { - if (typeof object === 'string') { - const { body } = await fetchAP>(object) - - return body - } - - return object as Exclude -} - -export async function findLatestAPRedirection (url: string, iteration = 1) { - if (iteration > 10) throw new Error('Too much iterations to find final URL ' + url) - - const { headers } = await fetchAP(url, { followRedirect: false }) - - if (headers.location) return findLatestAPRedirection(headers.location, iteration + 1) - - return url -} diff --git a/server/lib/activitypub/actors/get.ts b/server/lib/activitypub/actors/get.ts deleted file mode 100644 index dd2bc9f03..000000000 --- a/server/lib/activitypub/actors/get.ts +++ /dev/null @@ -1,143 +0,0 @@ -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { logger } from '@server/helpers/logger' -import { JobQueue } from '@server/lib/job-queue' -import { ActorLoadByUrlType, loadActorByUrl } from '@server/lib/model-loaders' -import { MActor, MActorAccountChannelId, MActorAccountChannelIdActor, MActorAccountId, MActorFullActor } from '@server/types/models' -import { arrayify } from '@shared/core-utils' -import { ActivityPubActor, APObjectId } from '@shared/models' -import { fetchAPObjectIfNeeded, getAPId } from '../activity' -import { checkUrlsSameHost } from '../url' -import { refreshActorIfNeeded } from './refresh' -import { APActorCreator, fetchRemoteActor } from './shared' - -function getOrCreateAPActor ( - activityActor: string | ActivityPubActor, - fetchType: 'all', - recurseIfNeeded?: boolean, - updateCollections?: boolean -): Promise - -function getOrCreateAPActor ( - activityActor: string | ActivityPubActor, - fetchType?: 'association-ids', - recurseIfNeeded?: boolean, - updateCollections?: boolean -): Promise - -async function getOrCreateAPActor ( - activityActor: string | ActivityPubActor, - fetchType: ActorLoadByUrlType = 'association-ids', - recurseIfNeeded = true, - updateCollections = false -): Promise { - const actorUrl = getAPId(activityActor) - let actor = await loadActorFromDB(actorUrl, fetchType) - - let created = false - let accountPlaylistsUrl: string - - // We don't have this actor in our database, fetch it on remote - if (!actor) { - const { actorObject } = await fetchRemoteActor(actorUrl) - if (actorObject === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl) - - // actorUrl is just an alias/redirection, so process object id instead - if (actorObject.id !== actorUrl) return getOrCreateAPActor(actorObject, 'all', recurseIfNeeded, updateCollections) - - // Create the attributed to actor - // In PeerTube a video channel is owned by an account - let ownerActor: MActorFullActor - if (recurseIfNeeded === true && actorObject.type === 'Group') { - ownerActor = await getOrCreateAPOwner(actorObject, actorUrl) - } - - const creator = new APActorCreator(actorObject, ownerActor) - actor = await retryTransactionWrapper(creator.create.bind(creator)) - created = true - accountPlaylistsUrl = actorObject.playlists - } - - if (actor.Account) (actor as MActorAccountChannelIdActor).Account.Actor = actor - if (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor - - const { actor: actorRefreshed, refreshed } = await refreshActorIfNeeded({ actor, fetchedType: fetchType }) - if (!actorRefreshed) throw new Error('Actor ' + actor.url + ' does not exist anymore.') - - await scheduleOutboxFetchIfNeeded(actor, created, refreshed, updateCollections) - await schedulePlaylistFetchIfNeeded(actor, created, accountPlaylistsUrl) - - return actorRefreshed -} - -async function getOrCreateAPOwner (actorObject: ActivityPubActor, actorUrl: string) { - const accountAttributedTo = await findOwner(actorUrl, actorObject.attributedTo, 'Person') - if (!accountAttributedTo) { - throw new Error(`Cannot find account attributed to video channel ${actorUrl}`) - } - - try { - // Don't recurse another time - const recurseIfNeeded = false - return getOrCreateAPActor(accountAttributedTo, 'all', recurseIfNeeded) - } catch (err) { - logger.error('Cannot get or create account attributed to video channel ' + actorUrl) - throw new Error(err) - } -} - -async function findOwner (rootUrl: string, attributedTo: APObjectId[] | APObjectId, type: 'Person' | 'Group') { - for (const actorToCheck of arrayify(attributedTo)) { - const actorObject = await fetchAPObjectIfNeeded(getAPId(actorToCheck)) - - if (!actorObject) { - logger.warn('Unknown attributed to actor %s for owner %s', actorToCheck, rootUrl) - continue - } - - if (checkUrlsSameHost(actorObject.id, rootUrl) !== true) { - logger.warn(`Account attributed to ${actorObject.id} does not have the same host than owner actor url ${rootUrl}`) - continue - } - - if (actorObject.type === type) return actorObject - } - - return undefined -} - -// --------------------------------------------------------------------------- - -export { - getOrCreateAPOwner, - getOrCreateAPActor, - findOwner -} - -// --------------------------------------------------------------------------- - -async function loadActorFromDB (actorUrl: string, fetchType: ActorLoadByUrlType) { - let actor = await loadActorByUrl(actorUrl, fetchType) - - // Orphan actor (not associated to an account of channel) so recreate it - if (actor && (!actor.Account && !actor.VideoChannel)) { - await actor.destroy() - actor = null - } - - return actor -} - -async function scheduleOutboxFetchIfNeeded (actor: MActor, created: boolean, refreshed: boolean, updateCollections: boolean) { - if ((created === true || refreshed === true) && updateCollections === true) { - const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' } - await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) - } -} - -async function schedulePlaylistFetchIfNeeded (actor: MActorAccountId, created: boolean, accountPlaylistsUrl: string) { - // We created a new account: fetch the playlists - if (created === true && actor.Account && accountPlaylistsUrl) { - const payload = { uri: accountPlaylistsUrl, type: 'account-playlists' as 'account-playlists' } - await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) - } -} diff --git a/server/lib/activitypub/actors/image.ts b/server/lib/activitypub/actors/image.ts deleted file mode 100644 index e1d29af5b..000000000 --- a/server/lib/activitypub/actors/image.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { Transaction } from 'sequelize/types' -import { logger } from '@server/helpers/logger' -import { ActorImageModel } from '@server/models/actor/actor-image' -import { MActorImage, MActorImages } from '@server/types/models' -import { ActorImageType } from '@shared/models' - -type ImageInfo = { - name: string - fileUrl: string - height: number - width: number - onDisk?: boolean -} - -async function updateActorImages (actor: MActorImages, type: ActorImageType, imagesInfo: ImageInfo[], t: Transaction) { - const getAvatarsOrBanners = () => { - const result = type === ActorImageType.AVATAR - ? actor.Avatars - : actor.Banners - - return result || [] - } - - if (imagesInfo.length === 0) { - await deleteActorImages(actor, type, t) - } - - // Cleanup old images that did not have a width - for (const oldImageModel of getAvatarsOrBanners()) { - if (oldImageModel.width) continue - - await safeDeleteActorImage(actor, oldImageModel, type, t) - } - - for (const imageInfo of imagesInfo) { - const oldImageModel = getAvatarsOrBanners().find(i => imageInfo.width && i.width === imageInfo.width) - - if (oldImageModel) { - // Don't update the avatar if the file URL did not change - if (imageInfo.fileUrl && oldImageModel.fileUrl === imageInfo.fileUrl) { - continue - } - - await safeDeleteActorImage(actor, oldImageModel, type, t) - } - - const imageModel = await ActorImageModel.create({ - filename: imageInfo.name, - onDisk: imageInfo.onDisk ?? false, - fileUrl: imageInfo.fileUrl, - height: imageInfo.height, - width: imageInfo.width, - type, - actorId: actor.id - }, { transaction: t }) - - addActorImage(actor, type, imageModel) - } - - return actor -} - -async function deleteActorImages (actor: MActorImages, type: ActorImageType, t: Transaction) { - try { - const association = buildAssociationName(type) - - for (const image of actor[association]) { - await image.destroy({ transaction: t }) - } - - actor[association] = [] - } catch (err) { - logger.error('Cannot remove old image of actor %s.', actor.url, { err }) - } - - return actor -} - -async function safeDeleteActorImage (actor: MActorImages, toDelete: MActorImage, type: ActorImageType, t: Transaction) { - try { - await toDelete.destroy({ transaction: t }) - - const association = buildAssociationName(type) - actor[association] = actor[association].filter(image => image.id !== toDelete.id) - } catch (err) { - logger.error('Cannot remove old actor image of actor %s.', actor.url, { err }) - } -} - -// --------------------------------------------------------------------------- - -export { - ImageInfo, - - updateActorImages, - deleteActorImages -} - -// --------------------------------------------------------------------------- - -function addActorImage (actor: MActorImages, type: ActorImageType, imageModel: MActorImage) { - const association = buildAssociationName(type) - if (!actor[association]) actor[association] = [] - - actor[association].push(imageModel) -} - -function buildAssociationName (type: ActorImageType) { - return type === ActorImageType.AVATAR - ? 'Avatars' - : 'Banners' -} diff --git a/server/lib/activitypub/actors/index.ts b/server/lib/activitypub/actors/index.ts deleted file mode 100644 index 5ee2a6f1a..000000000 --- a/server/lib/activitypub/actors/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from './get' -export * from './image' -export * from './keys' -export * from './refresh' -export * from './updater' -export * from './webfinger' diff --git a/server/lib/activitypub/actors/keys.ts b/server/lib/activitypub/actors/keys.ts deleted file mode 100644 index c3d18abd8..000000000 --- a/server/lib/activitypub/actors/keys.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { createPrivateAndPublicKeys } from '@server/helpers/peertube-crypto' -import { MActor } from '@server/types/models' - -// Set account keys, this could be long so process after the account creation and do not block the client -async function generateAndSaveActorKeys (actor: T) { - const { publicKey, privateKey } = await createPrivateAndPublicKeys() - - actor.publicKey = publicKey - actor.privateKey = privateKey - - return actor.save() -} - -export { - generateAndSaveActorKeys -} diff --git a/server/lib/activitypub/actors/refresh.ts b/server/lib/activitypub/actors/refresh.ts deleted file mode 100644 index d15cb5e90..000000000 --- a/server/lib/activitypub/actors/refresh.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { CachePromiseFactory } from '@server/helpers/promise-cache' -import { PeerTubeRequestError } from '@server/helpers/requests' -import { ActorLoadByUrlType } from '@server/lib/model-loaders' -import { ActorModel } from '@server/models/actor/actor' -import { MActorAccountChannelId, MActorFull } from '@server/types/models' -import { HttpStatusCode } from '@shared/models' -import { fetchRemoteActor } from './shared' -import { APActorUpdater } from './updater' -import { getUrlFromWebfinger } from './webfinger' - -type RefreshResult = Promise<{ actor: T | MActorFull, refreshed: boolean }> - -type RefreshOptions = { - actor: T - fetchedType: ActorLoadByUrlType -} - -const promiseCache = new CachePromiseFactory(doRefresh, (options: RefreshOptions) => options.actor.url) - -function refreshActorIfNeeded (options: RefreshOptions): RefreshResult { - const actorArg = options.actor - if (!actorArg.isOutdated()) return Promise.resolve({ actor: actorArg, refreshed: false }) - - return promiseCache.run(options) -} - -export { - refreshActorIfNeeded -} - -// --------------------------------------------------------------------------- - -async function doRefresh (options: RefreshOptions): RefreshResult { - const { actor: actorArg, fetchedType } = options - - // We need more attributes - const actor = fetchedType === 'all' - ? actorArg as MActorFull - : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) - - const lTags = loggerTagsFactory('ap', 'actor', 'refresh', actor.url) - - logger.info('Refreshing actor %s.', actor.url, lTags()) - - try { - const actorUrl = await getActorUrl(actor) - const { actorObject } = await fetchRemoteActor(actorUrl) - - if (actorObject === undefined) { - logger.info('Cannot fetch remote actor %s in refresh actor.', actorUrl) - return { actor, refreshed: false } - } - - const updater = new APActorUpdater(actorObject, actor) - await updater.update() - - return { refreshed: true, actor } - } catch (err) { - if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { - logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url, lTags()) - - actor.Account - ? await actor.Account.destroy() - : await actor.VideoChannel.destroy() - - return { actor: undefined, refreshed: false } - } - - logger.info('Cannot refresh actor %s.', actor.url, { err, ...lTags() }) - return { actor, refreshed: false } - } -} - -function getActorUrl (actor: MActorFull) { - return getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) - .catch(err => { - logger.warn('Cannot get actor URL from webfinger, keeping the old one.', { err }) - return actor.url - }) -} diff --git a/server/lib/activitypub/actors/shared/creator.ts b/server/lib/activitypub/actors/shared/creator.ts deleted file mode 100644 index 500bc9912..000000000 --- a/server/lib/activitypub/actors/shared/creator.ts +++ /dev/null @@ -1,149 +0,0 @@ -import { Op, Transaction } from 'sequelize' -import { sequelizeTypescript } from '@server/initializers/database' -import { AccountModel } from '@server/models/account/account' -import { ActorModel } from '@server/models/actor/actor' -import { ServerModel } from '@server/models/server/server' -import { VideoChannelModel } from '@server/models/video/video-channel' -import { MAccount, MAccountDefault, MActor, MActorFullActor, MActorId, MActorImages, MChannel, MServer } from '@server/types/models' -import { ActivityPubActor, ActorImageType } from '@shared/models' -import { updateActorImages } from '../image' -import { getActorAttributesFromObject, getActorDisplayNameFromObject, getImagesInfoFromObject } from './object-to-model-attributes' -import { fetchActorFollowsCount } from './url-to-object' - -export class APActorCreator { - - constructor ( - private readonly actorObject: ActivityPubActor, - private readonly ownerActor?: MActorFullActor - ) { - - } - - async create (): Promise { - const { followersCount, followingCount } = await fetchActorFollowsCount(this.actorObject) - - const actorInstance = new ActorModel(getActorAttributesFromObject(this.actorObject, followersCount, followingCount)) - - return sequelizeTypescript.transaction(async t => { - const server = await this.setServer(actorInstance, t) - - const { actorCreated, created } = await this.saveActor(actorInstance, t) - - await this.setImageIfNeeded(actorCreated, ActorImageType.AVATAR, t) - await this.setImageIfNeeded(actorCreated, ActorImageType.BANNER, t) - - await this.tryToFixActorUrlIfNeeded(actorCreated, actorInstance, created, t) - - if (actorCreated.type === 'Person' || actorCreated.type === 'Application') { // Account or PeerTube instance - actorCreated.Account = await this.saveAccount(actorCreated, t) as MAccountDefault - actorCreated.Account.Actor = actorCreated - } - - if (actorCreated.type === 'Group') { // Video channel - const channel = await this.saveVideoChannel(actorCreated, t) - actorCreated.VideoChannel = Object.assign(channel, { Actor: actorCreated, Account: this.ownerActor.Account }) - } - - actorCreated.Server = server - - return actorCreated - }) - } - - private async setServer (actor: MActor, t: Transaction) { - const actorHost = new URL(actor.url).host - - const serverOptions = { - where: { - host: actorHost - }, - defaults: { - host: actorHost - }, - transaction: t - } - const [ server ] = await ServerModel.findOrCreate(serverOptions) - - // Save our new account in database - actor.serverId = server.id - - return server as MServer - } - - private async setImageIfNeeded (actor: MActor, type: ActorImageType, t: Transaction) { - const imagesInfo = getImagesInfoFromObject(this.actorObject, type) - if (imagesInfo.length === 0) return - - return updateActorImages(actor as MActorImages, type, imagesInfo, t) - } - - private async saveActor (actor: MActor, t: Transaction) { - // Force the actor creation using findOrCreate() instead of save() - // Sometimes Sequelize skips the save() when it thinks the instance already exists - // (which could be false in a retried query) - const [ actorCreated, created ] = await ActorModel.findOrCreate({ - defaults: actor.toJSON(), - where: { - [Op.or]: [ - { - url: actor.url - }, - { - serverId: actor.serverId, - preferredUsername: actor.preferredUsername - } - ] - }, - transaction: t - }) - - return { actorCreated, created } - } - - private async tryToFixActorUrlIfNeeded (actorCreated: MActor, newActor: MActor, created: boolean, t: Transaction) { - // Try to fix non HTTPS accounts of remote instances that fixed their URL afterwards - if (created !== true && actorCreated.url !== newActor.url) { - // Only fix http://example.com/account/djidane to https://example.com/account/djidane - if (actorCreated.url.replace(/^http:\/\//, '') !== newActor.url.replace(/^https:\/\//, '')) { - throw new Error(`Actor from DB with URL ${actorCreated.url} does not correspond to actor ${newActor.url}`) - } - - actorCreated.url = newActor.url - await actorCreated.save({ transaction: t }) - } - } - - private async saveAccount (actor: MActorId, t: Transaction) { - const [ accountCreated ] = await AccountModel.findOrCreate({ - defaults: { - name: getActorDisplayNameFromObject(this.actorObject), - description: this.actorObject.summary, - actorId: actor.id - }, - where: { - actorId: actor.id - }, - transaction: t - }) - - return accountCreated as MAccount - } - - private async saveVideoChannel (actor: MActorId, t: Transaction) { - const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({ - defaults: { - name: getActorDisplayNameFromObject(this.actorObject), - description: this.actorObject.summary, - support: this.actorObject.support, - actorId: actor.id, - accountId: this.ownerActor.Account.id - }, - where: { - actorId: actor.id - }, - transaction: t - }) - - return videoChannelCreated as MChannel - } -} diff --git a/server/lib/activitypub/actors/shared/index.ts b/server/lib/activitypub/actors/shared/index.ts deleted file mode 100644 index 52af1a8e1..000000000 --- a/server/lib/activitypub/actors/shared/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './creator' -export * from './object-to-model-attributes' -export * from './url-to-object' diff --git a/server/lib/activitypub/actors/shared/object-to-model-attributes.ts b/server/lib/activitypub/actors/shared/object-to-model-attributes.ts deleted file mode 100644 index 3ce332681..000000000 --- a/server/lib/activitypub/actors/shared/object-to-model-attributes.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc' -import { MIMETYPES } from '@server/initializers/constants' -import { ActorModel } from '@server/models/actor/actor' -import { FilteredModelAttributes } from '@server/types' -import { getLowercaseExtension } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { ActivityIconObject, ActivityPubActor, ActorImageType } from '@shared/models' - -function getActorAttributesFromObject ( - actorObject: ActivityPubActor, - followersCount: number, - followingCount: number -): FilteredModelAttributes { - return { - type: actorObject.type, - preferredUsername: actorObject.preferredUsername, - url: actorObject.id, - publicKey: actorObject.publicKey.publicKeyPem, - privateKey: null, - followersCount, - followingCount, - inboxUrl: actorObject.inbox, - outboxUrl: actorObject.outbox, - followersUrl: actorObject.followers, - followingUrl: actorObject.following, - - sharedInboxUrl: actorObject.endpoints?.sharedInbox - ? actorObject.endpoints.sharedInbox - : null - } -} - -function getImagesInfoFromObject (actorObject: ActivityPubActor, type: ActorImageType) { - const iconsOrImages = type === ActorImageType.AVATAR - ? actorObject.icon - : actorObject.image - - return normalizeIconOrImage(iconsOrImages) - .map(iconOrImage => { - const mimetypes = MIMETYPES.IMAGE - - if (iconOrImage.type !== 'Image' || !isActivityPubUrlValid(iconOrImage.url)) return undefined - - let extension: string - - if (iconOrImage.mediaType) { - extension = mimetypes.MIMETYPE_EXT[iconOrImage.mediaType] - } else { - const tmp = getLowercaseExtension(iconOrImage.url) - - if (mimetypes.EXT_MIMETYPE[tmp] !== undefined) extension = tmp - } - - if (!extension) return undefined - - return { - name: buildUUID() + extension, - fileUrl: iconOrImage.url, - height: iconOrImage.height, - width: iconOrImage.width, - type - } - }) - .filter(i => !!i) -} - -function getActorDisplayNameFromObject (actorObject: ActivityPubActor) { - return actorObject.name || actorObject.preferredUsername -} - -export { - getActorAttributesFromObject, - getImagesInfoFromObject, - getActorDisplayNameFromObject -} - -// --------------------------------------------------------------------------- - -function normalizeIconOrImage (icon: ActivityIconObject | ActivityIconObject[]): ActivityIconObject[] { - if (Array.isArray(icon)) return icon - if (icon) return [ icon ] - - return [] -} diff --git a/server/lib/activitypub/actors/shared/url-to-object.ts b/server/lib/activitypub/actors/shared/url-to-object.ts deleted file mode 100644 index 73766bd50..000000000 --- a/server/lib/activitypub/actors/shared/url-to-object.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { sanitizeAndCheckActorObject } from '@server/helpers/custom-validators/activitypub/actor' -import { logger } from '@server/helpers/logger' -import { ActivityPubActor, ActivityPubOrderedCollection } from '@shared/models' -import { fetchAP } from '../../activity' -import { checkUrlsSameHost } from '../../url' - -async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode: number, actorObject: ActivityPubActor }> { - logger.info('Fetching remote actor %s.', actorUrl) - - const { body, statusCode } = await fetchAP(actorUrl) - - if (sanitizeAndCheckActorObject(body) === false) { - logger.debug('Remote actor JSON is not valid.', { actorJSON: body }) - return { actorObject: undefined, statusCode } - } - - if (checkUrlsSameHost(body.id, actorUrl) !== true) { - logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, body.id) - return { actorObject: undefined, statusCode } - } - - return { - statusCode, - - actorObject: body - } -} - -async function fetchActorFollowsCount (actorObject: ActivityPubActor) { - let followersCount = 0 - let followingCount = 0 - - if (actorObject.followers) followersCount = await fetchActorTotalItems(actorObject.followers) - if (actorObject.following) followingCount = await fetchActorTotalItems(actorObject.following) - - return { followersCount, followingCount } -} - -// --------------------------------------------------------------------------- -export { - fetchActorFollowsCount, - fetchRemoteActor -} - -// --------------------------------------------------------------------------- - -async function fetchActorTotalItems (url: string) { - try { - const { body } = await fetchAP>(url) - - return body.totalItems || 0 - } catch (err) { - logger.info('Cannot fetch remote actor count %s.', url, { err }) - return 0 - } -} diff --git a/server/lib/activitypub/actors/updater.ts b/server/lib/activitypub/actors/updater.ts deleted file mode 100644 index 5a92e7a22..000000000 --- a/server/lib/activitypub/actors/updater.ts +++ /dev/null @@ -1,91 +0,0 @@ -import { resetSequelizeInstance, runInReadCommittedTransaction } from '@server/helpers/database-utils' -import { logger } from '@server/helpers/logger' -import { AccountModel } from '@server/models/account/account' -import { VideoChannelModel } from '@server/models/video/video-channel' -import { MAccount, MActor, MActorFull, MChannel } from '@server/types/models' -import { ActivityPubActor, ActorImageType } from '@shared/models' -import { getOrCreateAPOwner } from './get' -import { updateActorImages } from './image' -import { fetchActorFollowsCount } from './shared' -import { getImagesInfoFromObject } from './shared/object-to-model-attributes' - -export class APActorUpdater { - - private readonly accountOrChannel: MAccount | MChannel - - constructor ( - private readonly actorObject: ActivityPubActor, - private readonly actor: MActorFull - ) { - if (this.actorObject.type === 'Group') this.accountOrChannel = this.actor.VideoChannel - else this.accountOrChannel = this.actor.Account - } - - async update () { - const avatarsInfo = getImagesInfoFromObject(this.actorObject, ActorImageType.AVATAR) - const bannersInfo = getImagesInfoFromObject(this.actorObject, ActorImageType.BANNER) - - try { - await this.updateActorInstance(this.actor, this.actorObject) - - this.accountOrChannel.name = this.actorObject.name || this.actorObject.preferredUsername - this.accountOrChannel.description = this.actorObject.summary - - if (this.accountOrChannel instanceof VideoChannelModel) { - const owner = await getOrCreateAPOwner(this.actorObject, this.actorObject.url) - this.accountOrChannel.accountId = owner.Account.id - this.accountOrChannel.Account = owner.Account as AccountModel - - this.accountOrChannel.support = this.actorObject.support - } - - await runInReadCommittedTransaction(async t => { - await updateActorImages(this.actor, ActorImageType.BANNER, bannersInfo, t) - await updateActorImages(this.actor, ActorImageType.AVATAR, avatarsInfo, t) - }) - - await runInReadCommittedTransaction(async t => { - await this.actor.save({ transaction: t }) - await this.accountOrChannel.save({ transaction: t }) - }) - - logger.info('Remote account %s updated', this.actorObject.url) - } catch (err) { - if (this.actor !== undefined) { - await resetSequelizeInstance(this.actor) - } - - if (this.accountOrChannel !== undefined) { - await resetSequelizeInstance(this.accountOrChannel) - } - - // This is just a debug because we will retry the insert - logger.debug('Cannot update the remote account.', { err }) - throw err - } - } - - private async updateActorInstance (actorInstance: MActor, actorObject: ActivityPubActor) { - const { followersCount, followingCount } = await fetchActorFollowsCount(actorObject) - - actorInstance.type = actorObject.type - actorInstance.preferredUsername = actorObject.preferredUsername - actorInstance.url = actorObject.id - actorInstance.publicKey = actorObject.publicKey.publicKeyPem - actorInstance.followersCount = followersCount - actorInstance.followingCount = followingCount - actorInstance.inboxUrl = actorObject.inbox - actorInstance.outboxUrl = actorObject.outbox - actorInstance.followersUrl = actorObject.followers - actorInstance.followingUrl = actorObject.following - - if (actorObject.published) actorInstance.remoteCreatedAt = new Date(actorObject.published) - - if (actorObject.endpoints?.sharedInbox) { - actorInstance.sharedInboxUrl = actorObject.endpoints.sharedInbox - } - - // Force actor update - actorInstance.changed('updatedAt', true) - } -} diff --git a/server/lib/activitypub/actors/webfinger.ts b/server/lib/activitypub/actors/webfinger.ts deleted file mode 100644 index b20a724da..000000000 --- a/server/lib/activitypub/actors/webfinger.ts +++ /dev/null @@ -1,67 +0,0 @@ -import WebFinger from 'webfinger.js' -import { isProdInstance } from '@server/helpers/core-utils' -import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc' -import { REQUEST_TIMEOUTS, WEBSERVER } from '@server/initializers/constants' -import { ActorModel } from '@server/models/actor/actor' -import { MActorFull } from '@server/types/models' -import { WebFingerData } from '@shared/models' - -const webfinger = new WebFinger({ - webfist_fallback: false, - tls_only: isProdInstance(), - uri_fallback: false, - request_timeout: REQUEST_TIMEOUTS.DEFAULT -}) - -async function loadActorUrlOrGetFromWebfinger (uriArg: string) { - // Handle strings like @toto@example.com - const uri = uriArg.startsWith('@') ? uriArg.slice(1) : uriArg - - const [ name, host ] = uri.split('@') - let actor: MActorFull - - if (!host || host === WEBSERVER.HOST) { - actor = await ActorModel.loadLocalByName(name) - } else { - actor = await ActorModel.loadByNameAndHost(name, host) - } - - if (actor) return actor.url - - return getUrlFromWebfinger(uri) -} - -async function getUrlFromWebfinger (uri: string) { - const webfingerData: WebFingerData = await webfingerLookup(uri) - return getLinkOrThrow(webfingerData) -} - -// --------------------------------------------------------------------------- - -export { - getUrlFromWebfinger, - loadActorUrlOrGetFromWebfinger -} - -// --------------------------------------------------------------------------- - -function getLinkOrThrow (webfingerData: WebFingerData) { - if (Array.isArray(webfingerData.links) === false) throw new Error('WebFinger links is not an array.') - - const selfLink = webfingerData.links.find(l => l.rel === 'self') - if (selfLink === undefined || isActivityPubUrlValid(selfLink.href) === false) { - throw new Error('Cannot find self link or href is not a valid URL.') - } - - return selfLink.href -} - -function webfingerLookup (nameWithHost: string) { - return new Promise((res, rej) => { - webfinger.lookup(nameWithHost, (err, p) => { - if (err) return rej(err) - - return res(p.object) - }) - }) -} diff --git a/server/lib/activitypub/audience.ts b/server/lib/activitypub/audience.ts deleted file mode 100644 index 6f5491387..000000000 --- a/server/lib/activitypub/audience.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { ActivityAudience } from '../../../shared/models/activitypub' -import { ACTIVITY_PUB } from '../../initializers/constants' -import { MActorFollowersUrl } from '../../types/models' - -function getAudience (actorSender: MActorFollowersUrl, isPublic = true) { - return buildAudience([ actorSender.followersUrl ], isPublic) -} - -function buildAudience (followerUrls: string[], isPublic = true) { - let to: string[] = [] - let cc: string[] = [] - - if (isPublic) { - to = [ ACTIVITY_PUB.PUBLIC ] - cc = followerUrls - } else { // Unlisted - to = [] - cc = [] - } - - return { to, cc } -} - -function audiencify (object: T, audience: ActivityAudience) { - return { ...audience, ...object } -} - -// --------------------------------------------------------------------------- - -export { - buildAudience, - getAudience, - audiencify -} diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts deleted file mode 100644 index c3acd7112..000000000 --- a/server/lib/activitypub/cache-file.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { Transaction } from 'sequelize' -import { MActorId, MVideoRedundancy, MVideoWithAllFiles } from '@server/types/models' -import { CacheFileObject, VideoStreamingPlaylistType } from '@shared/models' -import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' - -async function createOrUpdateCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) { - const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t) - - if (redundancyModel) { - return updateCacheFile(cacheFileObject, redundancyModel, video, byActor, t) - } - - return createCacheFile(cacheFileObject, video, byActor, t) -} - -// --------------------------------------------------------------------------- - -export { - createOrUpdateCacheFile -} - -// --------------------------------------------------------------------------- - -function createCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) { - const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) - - return VideoRedundancyModel.create(attributes, { transaction: t }) -} - -function updateCacheFile ( - cacheFileObject: CacheFileObject, - redundancyModel: MVideoRedundancy, - video: MVideoWithAllFiles, - byActor: MActorId, - t: Transaction -) { - if (redundancyModel.actorId !== byActor.id) { - throw new Error('Cannot update redundancy ' + redundancyModel.url + ' of another actor.') - } - - const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) - - redundancyModel.expiresOn = attributes.expiresOn - redundancyModel.fileUrl = attributes.fileUrl - - return redundancyModel.save({ transaction: t }) -} - -function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId) { - - if (cacheFileObject.url.mediaType === 'application/x-mpegURL') { - const url = cacheFileObject.url - - const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS) - if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) - - return { - expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, - url: cacheFileObject.id, - fileUrl: url.href, - strategy: null, - videoStreamingPlaylistId: playlist.id, - actorId: byActor.id - } - } - - const url = cacheFileObject.url - const videoFile = video.VideoFiles.find(f => { - return f.resolution === url.height && f.fps === url.fps - }) - - if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`) - - return { - expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, - url: cacheFileObject.id, - fileUrl: url.href, - strategy: null, - videoFileId: videoFile.id, - actorId: byActor.id - } -} diff --git a/server/lib/activitypub/collection.ts b/server/lib/activitypub/collection.ts deleted file mode 100644 index a176cab51..000000000 --- a/server/lib/activitypub/collection.ts +++ /dev/null @@ -1,63 +0,0 @@ -import Bluebird from 'bluebird' -import validator from 'validator' -import { pageToStartAndCount } from '@server/helpers/core-utils' -import { ACTIVITY_PUB } from '@server/initializers/constants' -import { ResultList } from '@shared/models' -import { forceNumber } from '@shared/core-utils' - -type ActivityPubCollectionPaginationHandler = (start: number, count: number) => Bluebird> | Promise> - -async function activityPubCollectionPagination ( - baseUrl: string, - handler: ActivityPubCollectionPaginationHandler, - page?: any, - size = ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE -) { - if (!page || !validator.isInt(page)) { - // We just display the first page URL, we only need the total items - const result = await handler(0, 1) - - return { - id: baseUrl, - type: 'OrderedCollection', - totalItems: result.total, - first: result.data.length === 0 - ? undefined - : baseUrl + '?page=1' - } - } - - const { start, count } = pageToStartAndCount(page, size) - const result = await handler(start, count) - - let next: string | undefined - let prev: string | undefined - - // Assert page is a number - page = forceNumber(page) - - // There are more results - if (result.total > page * size) { - next = baseUrl + '?page=' + (page + 1) - } - - if (page > 1) { - prev = baseUrl + '?page=' + (page - 1) - } - - return { - id: baseUrl + '?page=' + page, - type: 'OrderedCollectionPage', - prev, - next, - partOf: baseUrl, - orderedItems: result.data, - totalItems: result.total - } -} - -// --------------------------------------------------------------------------- - -export { - activityPubCollectionPagination -} diff --git a/server/lib/activitypub/context.ts b/server/lib/activitypub/context.ts deleted file mode 100644 index 87eb498a3..000000000 --- a/server/lib/activitypub/context.ts +++ /dev/null @@ -1,212 +0,0 @@ -import { ContextType } from '@shared/models' -import { Hooks } from '../plugins/hooks' - -async function activityPubContextify (data: T, type: ContextType) { - return { ...await getContextData(type), ...data } -} - -// --------------------------------------------------------------------------- - -export { - getContextData, - activityPubContextify -} - -// --------------------------------------------------------------------------- - -type ContextValue = { [ id: string ]: (string | { '@type': string, '@id': string }) } - -const contextStore: { [ id in ContextType ]: (string | { [ id: string ]: string })[] } = { - Video: buildContext({ - Hashtag: 'as:Hashtag', - uuid: 'sc:identifier', - category: 'sc:category', - licence: 'sc:license', - subtitleLanguage: 'sc:subtitleLanguage', - sensitive: 'as:sensitive', - language: 'sc:inLanguage', - identifier: 'sc:identifier', - - isLiveBroadcast: 'sc:isLiveBroadcast', - liveSaveReplay: { - '@type': 'sc:Boolean', - '@id': 'pt:liveSaveReplay' - }, - permanentLive: { - '@type': 'sc:Boolean', - '@id': 'pt:permanentLive' - }, - latencyMode: { - '@type': 'sc:Number', - '@id': 'pt:latencyMode' - }, - - Infohash: 'pt:Infohash', - - tileWidth: { - '@type': 'sc:Number', - '@id': 'pt:tileWidth' - }, - tileHeight: { - '@type': 'sc:Number', - '@id': 'pt:tileHeight' - }, - tileDuration: { - '@type': 'sc:Number', - '@id': 'pt:tileDuration' - }, - - originallyPublishedAt: 'sc:datePublished', - - uploadDate: 'sc:uploadDate', - - views: { - '@type': 'sc:Number', - '@id': 'pt:views' - }, - state: { - '@type': 'sc:Number', - '@id': 'pt:state' - }, - size: { - '@type': 'sc:Number', - '@id': 'pt:size' - }, - fps: { - '@type': 'sc:Number', - '@id': 'pt:fps' - }, - commentsEnabled: { - '@type': 'sc:Boolean', - '@id': 'pt:commentsEnabled' - }, - downloadEnabled: { - '@type': 'sc:Boolean', - '@id': 'pt:downloadEnabled' - }, - waitTranscoding: { - '@type': 'sc:Boolean', - '@id': 'pt:waitTranscoding' - }, - support: { - '@type': 'sc:Text', - '@id': 'pt:support' - }, - likes: { - '@id': 'as:likes', - '@type': '@id' - }, - dislikes: { - '@id': 'as:dislikes', - '@type': '@id' - }, - shares: { - '@id': 'as:shares', - '@type': '@id' - }, - comments: { - '@id': 'as:comments', - '@type': '@id' - } - }), - - Playlist: buildContext({ - Playlist: 'pt:Playlist', - PlaylistElement: 'pt:PlaylistElement', - position: { - '@type': 'sc:Number', - '@id': 'pt:position' - }, - startTimestamp: { - '@type': 'sc:Number', - '@id': 'pt:startTimestamp' - }, - stopTimestamp: { - '@type': 'sc:Number', - '@id': 'pt:stopTimestamp' - }, - uuid: 'sc:identifier' - }), - - CacheFile: buildContext({ - expires: 'sc:expires', - CacheFile: 'pt:CacheFile' - }), - - Flag: buildContext({ - Hashtag: 'as:Hashtag' - }), - - Actor: buildContext({ - playlists: { - '@id': 'pt:playlists', - '@type': '@id' - }, - support: { - '@type': 'sc:Text', - '@id': 'pt:support' - }, - - // TODO: remove in a few versions, introduced in 4.2 - icons: 'as:icon' - }), - - WatchAction: buildContext({ - WatchAction: 'sc:WatchAction', - startTimestamp: { - '@type': 'sc:Number', - '@id': 'pt:startTimestamp' - }, - stopTimestamp: { - '@type': 'sc:Number', - '@id': 'pt:stopTimestamp' - }, - watchSection: { - '@type': 'sc:Number', - '@id': 'pt:stopTimestamp' - }, - uuid: 'sc:identifier' - }), - - Collection: buildContext(), - Follow: buildContext(), - Reject: buildContext(), - Accept: buildContext(), - View: buildContext(), - Announce: buildContext(), - Comment: buildContext(), - Delete: buildContext(), - Rate: buildContext() -} - -async function getContextData (type: ContextType) { - const contextData = await Hooks.wrapObject( - contextStore[type], - 'filter:activity-pub.activity.context.build.result' - ) - - return { '@context': contextData } -} - -function buildContext (contextValue?: ContextValue) { - const baseContext = [ - 'https://www.w3.org/ns/activitystreams', - 'https://w3id.org/security/v1', - { - RsaSignature2017: 'https://w3id.org/security#RsaSignature2017' - } - ] - - if (!contextValue) return baseContext - - return [ - ...baseContext, - - { - pt: 'https://joinpeertube.org/ns#', - sc: 'http://schema.org/', - - ...contextValue - } - ] -} diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts deleted file mode 100644 index b8348e8cf..000000000 --- a/server/lib/activitypub/crawl.ts +++ /dev/null @@ -1,58 +0,0 @@ -import Bluebird from 'bluebird' -import { URL } from 'url' -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' -import { logger } from '../../helpers/logger' -import { ACTIVITY_PUB, WEBSERVER } from '../../initializers/constants' -import { fetchAP } from './activity' - -type HandlerFunction = (items: T[]) => (Promise | Bluebird) -type CleanerFunction = (startedDate: Date) => Promise - -async function crawlCollectionPage (argUrl: string, handler: HandlerFunction, cleaner?: CleanerFunction) { - let url = argUrl - - logger.info('Crawling ActivityPub data on %s.', url) - - const startDate = new Date() - - const response = await fetchAP>(url) - const firstBody = response.body - - const limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT - let i = 0 - let nextLink = firstBody.first - while (nextLink && i < limit) { - let body: any - - if (typeof nextLink === 'string') { - // Don't crawl ourselves - const remoteHost = new URL(nextLink).host - if (remoteHost === WEBSERVER.HOST) continue - - url = nextLink - - const res = await fetchAP>(url) - body = res.body - } else { - // nextLink is already the object we want - body = nextLink - } - - nextLink = body.next - i++ - - if (Array.isArray(body.orderedItems)) { - const items = body.orderedItems - logger.info('Processing %i ActivityPub items for %s.', items.length, url) - - await handler(items) - } - } - - if (cleaner) await retryTransactionWrapper(cleaner, startDate) -} - -export { - crawlCollectionPage -} diff --git a/server/lib/activitypub/follow.ts b/server/lib/activitypub/follow.ts deleted file mode 100644 index f6e2a48fd..000000000 --- a/server/lib/activitypub/follow.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { Transaction } from 'sequelize' -import { getServerActor } from '@server/models/application/application' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { SERVER_ACTOR_NAME } from '../../initializers/constants' -import { ServerModel } from '../../models/server/server' -import { MActorFollowActors } from '../../types/models' -import { JobQueue } from '../job-queue' - -async function autoFollowBackIfNeeded (actorFollow: MActorFollowActors, transaction?: Transaction) { - if (!CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_BACK.ENABLED) return - - const follower = actorFollow.ActorFollower - - if (follower.type === 'Application' && follower.preferredUsername === SERVER_ACTOR_NAME) { - logger.info('Auto follow back %s.', follower.url) - - const me = await getServerActor() - - const server = await ServerModel.load(follower.serverId, transaction) - const host = server.host - - const payload = { - host, - name: SERVER_ACTOR_NAME, - followerActorId: me.id, - isAutoFollow: true - } - - JobQueue.Instance.createJobAsync({ type: 'activitypub-follow', payload }) - } -} - -// If we only have an host, use a default account handle -function getRemoteNameAndHost (handleOrHost: string) { - let name = SERVER_ACTOR_NAME - let host = handleOrHost - - const splitted = handleOrHost.split('@') - if (splitted.length === 2) { - name = splitted[0] - host = splitted[1] - } - - return { name, host } -} - -export { - autoFollowBackIfNeeded, - getRemoteNameAndHost -} diff --git a/server/lib/activitypub/inbox-manager.ts b/server/lib/activitypub/inbox-manager.ts deleted file mode 100644 index 27778cc9d..000000000 --- a/server/lib/activitypub/inbox-manager.ts +++ /dev/null @@ -1,47 +0,0 @@ -import PQueue from 'p-queue' -import { logger } from '@server/helpers/logger' -import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants' -import { MActorDefault, MActorSignature } from '@server/types/models' -import { Activity } from '@shared/models' -import { StatsManager } from '../stat-manager' -import { processActivities } from './process' - -class InboxManager { - - private static instance: InboxManager - private readonly inboxQueue: PQueue - - private constructor () { - this.inboxQueue = new PQueue({ concurrency: 1 }) - - setInterval(() => { - StatsManager.Instance.updateInboxWaiting(this.getActivityPubMessagesWaiting()) - }, SCHEDULER_INTERVALS_MS.UPDATE_INBOX_STATS) - } - - addInboxMessage (param: { - activities: Activity[] - signatureActor?: MActorSignature - inboxActor?: MActorDefault - }) { - this.inboxQueue.add(() => { - const options = { signatureActor: param.signatureActor, inboxActor: param.inboxActor } - - return processActivities(param.activities, options) - }).catch(err => logger.error('Error with inbox queue.', { err })) - } - - getActivityPubMessagesWaiting () { - return this.inboxQueue.size + this.inboxQueue.pending - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -// --------------------------------------------------------------------------- - -export { - InboxManager -} diff --git a/server/lib/activitypub/local-video-viewer.ts b/server/lib/activitypub/local-video-viewer.ts deleted file mode 100644 index bdd746791..000000000 --- a/server/lib/activitypub/local-video-viewer.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { Transaction } from 'sequelize' -import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' -import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section' -import { MVideo } from '@server/types/models' -import { WatchActionObject } from '@shared/models' -import { getDurationFromActivityStream } from './activity' - -async function createOrUpdateLocalVideoViewer (watchAction: WatchActionObject, video: MVideo, t: Transaction) { - const stats = await LocalVideoViewerModel.loadByUrl(watchAction.id) - if (stats) await stats.destroy({ transaction: t }) - - const localVideoViewer = await LocalVideoViewerModel.create({ - url: watchAction.id, - uuid: watchAction.uuid, - - watchTime: getDurationFromActivityStream(watchAction.duration), - - startDate: new Date(watchAction.startTime), - endDate: new Date(watchAction.endTime), - - country: watchAction.location - ? watchAction.location.addressCountry - : null, - - videoId: video.id - }, { transaction: t }) - - await LocalVideoViewerWatchSectionModel.bulkCreateSections({ - localVideoViewerId: localVideoViewer.id, - - watchSections: watchAction.watchSections.map(s => ({ - start: s.startTimestamp, - end: s.endTimestamp - })), - - transaction: t - }) -} - -// --------------------------------------------------------------------------- - -export { - createOrUpdateLocalVideoViewer -} diff --git a/server/lib/activitypub/outbox.ts b/server/lib/activitypub/outbox.ts deleted file mode 100644 index 5eef76871..000000000 --- a/server/lib/activitypub/outbox.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { ActorModel } from '@server/models/actor/actor' -import { getServerActor } from '@server/models/application/application' -import { JobQueue } from '../job-queue' - -async function addFetchOutboxJob (actor: Pick) { - // Don't fetch ourselves - const serverActor = await getServerActor() - if (serverActor.id === actor.id) { - logger.error('Cannot fetch our own outbox!') - return undefined - } - - const payload = { - uri: actor.outboxUrl, - type: 'activity' as 'activity' - } - - return JobQueue.Instance.createJobAsync({ type: 'activitypub-http-fetcher', payload }) -} - -export { - addFetchOutboxJob -} diff --git a/server/lib/activitypub/playlists/create-update.ts b/server/lib/activitypub/playlists/create-update.ts deleted file mode 100644 index b24299f29..000000000 --- a/server/lib/activitypub/playlists/create-update.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { map } from 'bluebird' -import { isArray } from '@server/helpers/custom-validators/misc' -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants' -import { sequelizeTypescript } from '@server/initializers/database' -import { updateRemotePlaylistMiniatureFromUrl } from '@server/lib/thumbnail' -import { VideoPlaylistModel } from '@server/models/video/video-playlist' -import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element' -import { FilteredModelAttributes } from '@server/types' -import { MThumbnail, MVideoPlaylist, MVideoPlaylistFull, MVideoPlaylistVideosLength } from '@server/types/models' -import { PlaylistObject } from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { getAPId } from '../activity' -import { getOrCreateAPActor } from '../actors' -import { crawlCollectionPage } from '../crawl' -import { getOrCreateAPVideo } from '../videos' -import { - fetchRemotePlaylistElement, - fetchRemoteVideoPlaylist, - playlistElementObjectToDBAttributes, - playlistObjectToDBAttributes -} from './shared' - -const lTags = loggerTagsFactory('ap', 'video-playlist') - -async function createAccountPlaylists (playlistUrls: string[]) { - await map(playlistUrls, async playlistUrl => { - try { - const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl) - if (exists === true) return - - const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl) - - if (playlistObject === undefined) { - throw new Error(`Cannot refresh remote playlist ${playlistUrl}: invalid body.`) - } - - return createOrUpdateVideoPlaylist(playlistObject) - } catch (err) { - logger.warn('Cannot add playlist element %s.', playlistUrl, { err, ...lTags(playlistUrl) }) - } - }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) -} - -async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, to?: string[]) { - const playlistAttributes = playlistObjectToDBAttributes(playlistObject, to || playlistObject.to) - - await setVideoChannel(playlistObject, playlistAttributes) - - const [ upsertPlaylist ] = await VideoPlaylistModel.upsert(playlistAttributes, { returning: true }) - - const playlistElementUrls = await fetchElementUrls(playlistObject) - - // Refetch playlist from DB since elements fetching could be long in time - const playlist = await VideoPlaylistModel.loadWithAccountAndChannel(upsertPlaylist.id, null) - - await updatePlaylistThumbnail(playlistObject, playlist) - - const elementsLength = await rebuildVideoPlaylistElements(playlistElementUrls, playlist) - playlist.setVideosLength(elementsLength) - - return playlist -} - -// --------------------------------------------------------------------------- - -export { - createAccountPlaylists, - createOrUpdateVideoPlaylist -} - -// --------------------------------------------------------------------------- - -async function setVideoChannel (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly) { - if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) { - throw new Error('Not attributed to for playlist object ' + getAPId(playlistObject)) - } - - const actor = await getOrCreateAPActor(getAPId(playlistObject.attributedTo[0]), 'all') - - if (!actor.VideoChannel) { - logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) }) - return - } - - playlistAttributes.videoChannelId = actor.VideoChannel.id - playlistAttributes.ownerAccountId = actor.VideoChannel.Account.id -} - -async function fetchElementUrls (playlistObject: PlaylistObject) { - let accItems: string[] = [] - await crawlCollectionPage(playlistObject.id, items => { - accItems = accItems.concat(items) - - return Promise.resolve() - }) - - return accItems -} - -async function updatePlaylistThumbnail (playlistObject: PlaylistObject, playlist: MVideoPlaylistFull) { - if (playlistObject.icon) { - let thumbnailModel: MThumbnail - - try { - thumbnailModel = await updateRemotePlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist }) - await playlist.setAndSaveThumbnail(thumbnailModel, undefined) - } catch (err) { - logger.warn('Cannot set thumbnail of %s.', playlistObject.id, { err, ...lTags(playlistObject.id, playlist.uuid, playlist.url) }) - - if (thumbnailModel) await thumbnailModel.removeThumbnail() - } - - return - } - - // Playlist does not have an icon, destroy existing one - if (playlist.hasThumbnail()) { - await playlist.Thumbnail.destroy() - playlist.Thumbnail = null - } -} - -async function rebuildVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) { - const elementsToCreate = await buildElementsDBAttributes(elementUrls, playlist) - - await retryTransactionWrapper(() => sequelizeTypescript.transaction(async t => { - await VideoPlaylistElementModel.deleteAllOf(playlist.id, t) - - for (const element of elementsToCreate) { - await VideoPlaylistElementModel.create(element, { transaction: t }) - } - })) - - logger.info('Rebuilt playlist %s with %s elements.', playlist.url, elementsToCreate.length, lTags(playlist.uuid, playlist.url)) - - return elementsToCreate.length -} - -async function buildElementsDBAttributes (elementUrls: string[], playlist: MVideoPlaylist) { - const elementsToCreate: FilteredModelAttributes[] = [] - - await map(elementUrls, async elementUrl => { - try { - const { elementObject } = await fetchRemotePlaylistElement(elementUrl) - - const { video } = await getOrCreateAPVideo({ videoObject: { id: elementObject.url }, fetchType: 'only-video' }) - - elementsToCreate.push(playlistElementObjectToDBAttributes(elementObject, playlist, video)) - } catch (err) { - logger.warn('Cannot add playlist element %s.', elementUrl, { err, ...lTags(playlist.uuid, playlist.url) }) - } - }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) - - return elementsToCreate -} diff --git a/server/lib/activitypub/playlists/get.ts b/server/lib/activitypub/playlists/get.ts deleted file mode 100644 index c34554d69..000000000 --- a/server/lib/activitypub/playlists/get.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { VideoPlaylistModel } from '@server/models/video/video-playlist' -import { MVideoPlaylistFullSummary } from '@server/types/models' -import { APObjectId } from '@shared/models' -import { getAPId } from '../activity' -import { createOrUpdateVideoPlaylist } from './create-update' -import { scheduleRefreshIfNeeded } from './refresh' -import { fetchRemoteVideoPlaylist } from './shared' - -async function getOrCreateAPVideoPlaylist (playlistObjectArg: APObjectId): Promise { - const playlistUrl = getAPId(playlistObjectArg) - - const playlistFromDatabase = await VideoPlaylistModel.loadByUrlWithAccountAndChannelSummary(playlistUrl) - - if (playlistFromDatabase) { - scheduleRefreshIfNeeded(playlistFromDatabase) - - return playlistFromDatabase - } - - const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl) - if (!playlistObject) throw new Error('Cannot fetch remote playlist with url: ' + playlistUrl) - - // playlistUrl is just an alias/redirection, so process object id instead - if (playlistObject.id !== playlistUrl) return getOrCreateAPVideoPlaylist(playlistObject) - - const playlistCreated = await createOrUpdateVideoPlaylist(playlistObject) - - return playlistCreated -} - -// --------------------------------------------------------------------------- - -export { - getOrCreateAPVideoPlaylist -} diff --git a/server/lib/activitypub/playlists/index.ts b/server/lib/activitypub/playlists/index.ts deleted file mode 100644 index e2470a674..000000000 --- a/server/lib/activitypub/playlists/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './get' -export * from './create-update' -export * from './refresh' diff --git a/server/lib/activitypub/playlists/refresh.ts b/server/lib/activitypub/playlists/refresh.ts deleted file mode 100644 index 33260ea02..000000000 --- a/server/lib/activitypub/playlists/refresh.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { PeerTubeRequestError } from '@server/helpers/requests' -import { JobQueue } from '@server/lib/job-queue' -import { MVideoPlaylist, MVideoPlaylistOwner } from '@server/types/models' -import { HttpStatusCode } from '@shared/models' -import { createOrUpdateVideoPlaylist } from './create-update' -import { fetchRemoteVideoPlaylist } from './shared' - -function scheduleRefreshIfNeeded (playlist: MVideoPlaylist) { - if (!playlist.isOutdated()) return - - JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'video-playlist', url: playlist.url } }) -} - -async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise { - if (!videoPlaylist.isOutdated()) return videoPlaylist - - const lTags = loggerTagsFactory('ap', 'video-playlist', 'refresh', videoPlaylist.uuid, videoPlaylist.url) - - logger.info('Refreshing playlist %s.', videoPlaylist.url, lTags()) - - try { - const { playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url) - - if (playlistObject === undefined) { - logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url, lTags()) - - await videoPlaylist.setAsRefreshed() - return videoPlaylist - } - - await createOrUpdateVideoPlaylist(playlistObject) - - return videoPlaylist - } catch (err) { - if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { - logger.info('Cannot refresh not existing playlist %s. Deleting it.', videoPlaylist.url, lTags()) - - await videoPlaylist.destroy() - return undefined - } - - logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err, ...lTags() }) - - await videoPlaylist.setAsRefreshed() - return videoPlaylist - } -} - -export { - scheduleRefreshIfNeeded, - refreshVideoPlaylistIfNeeded -} diff --git a/server/lib/activitypub/playlists/shared/index.ts b/server/lib/activitypub/playlists/shared/index.ts deleted file mode 100644 index a217f2291..000000000 --- a/server/lib/activitypub/playlists/shared/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './object-to-model-attributes' -export * from './url-to-object' diff --git a/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts b/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts deleted file mode 100644 index 753b5e660..000000000 --- a/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { ACTIVITY_PUB } from '@server/initializers/constants' -import { VideoPlaylistModel } from '@server/models/video/video-playlist' -import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element' -import { MVideoId, MVideoPlaylistId } from '@server/types/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { PlaylistElementObject, PlaylistObject, VideoPlaylistPrivacy } from '@shared/models' - -function playlistObjectToDBAttributes (playlistObject: PlaylistObject, to: string[]) { - const privacy = to.includes(ACTIVITY_PUB.PUBLIC) - ? VideoPlaylistPrivacy.PUBLIC - : VideoPlaylistPrivacy.UNLISTED - - return { - name: playlistObject.name, - description: playlistObject.content, - privacy, - url: playlistObject.id, - uuid: playlistObject.uuid, - ownerAccountId: null, - videoChannelId: null, - createdAt: new Date(playlistObject.published), - updatedAt: new Date(playlistObject.updated) - } as AttributesOnly -} - -function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) { - return { - position: elementObject.position, - url: elementObject.id, - startTimestamp: elementObject.startTimestamp || null, - stopTimestamp: elementObject.stopTimestamp || null, - videoPlaylistId: videoPlaylist.id, - videoId: video.id - } as AttributesOnly -} - -export { - playlistObjectToDBAttributes, - playlistElementObjectToDBAttributes -} diff --git a/server/lib/activitypub/playlists/shared/url-to-object.ts b/server/lib/activitypub/playlists/shared/url-to-object.ts deleted file mode 100644 index fd9fe5558..000000000 --- a/server/lib/activitypub/playlists/shared/url-to-object.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '@server/helpers/custom-validators/activitypub/playlist' -import { isArray } from '@server/helpers/custom-validators/misc' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { PlaylistElementObject, PlaylistObject } from '@shared/models' -import { fetchAP } from '../../activity' -import { checkUrlsSameHost } from '../../url' - -async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> { - const lTags = loggerTagsFactory('ap', 'video-playlist', playlistUrl) - - logger.info('Fetching remote playlist %s.', playlistUrl, lTags()) - - const { body, statusCode } = await fetchAP(playlistUrl) - - if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) { - logger.debug('Remote video playlist JSON is not valid.', { body, ...lTags() }) - return { statusCode, playlistObject: undefined } - } - - if (!isArray(body.to)) { - logger.debug('Remote video playlist JSON does not have a valid audience.', { body, ...lTags() }) - return { statusCode, playlistObject: undefined } - } - - return { statusCode, playlistObject: body } -} - -async function fetchRemotePlaylistElement (elementUrl: string): Promise<{ statusCode: number, elementObject: PlaylistElementObject }> { - const lTags = loggerTagsFactory('ap', 'video-playlist', 'element', elementUrl) - - logger.debug('Fetching remote playlist element %s.', elementUrl, lTags()) - - const { body, statusCode } = await fetchAP(elementUrl) - - if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in fetch playlist element ${elementUrl}`) - - if (checkUrlsSameHost(body.id, elementUrl) !== true) { - throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`) - } - - return { statusCode, elementObject: body } -} - -export { - fetchRemoteVideoPlaylist, - fetchRemotePlaylistElement -} diff --git a/server/lib/activitypub/process/index.ts b/server/lib/activitypub/process/index.ts deleted file mode 100644 index 5466739c1..000000000 --- a/server/lib/activitypub/process/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './process' diff --git a/server/lib/activitypub/process/process-accept.ts b/server/lib/activitypub/process/process-accept.ts deleted file mode 100644 index 077b01eda..000000000 --- a/server/lib/activitypub/process/process-accept.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { ActivityAccept } from '../../../../shared/models/activitypub' -import { ActorFollowModel } from '../../../models/actor/actor-follow' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActorDefault, MActorSignature } from '../../../types/models' -import { addFetchOutboxJob } from '../outbox' - -async function processAcceptActivity (options: APProcessorOptions) { - const { byActor: targetActor, inboxActor } = options - if (inboxActor === undefined) throw new Error('Need to accept on explicit inbox.') - - return processAccept(inboxActor, targetActor) -} - -// --------------------------------------------------------------------------- - -export { - processAcceptActivity -} - -// --------------------------------------------------------------------------- - -async function processAccept (actor: MActorDefault, targetActor: MActorSignature) { - const follow = await ActorFollowModel.loadByActorAndTarget(actor.id, targetActor.id) - if (!follow) throw new Error('Cannot find associated follow.') - - if (follow.state !== 'accepted') { - follow.state = 'accepted' - await follow.save() - - await addFetchOutboxJob(targetActor) - } -} diff --git a/server/lib/activitypub/process/process-announce.ts b/server/lib/activitypub/process/process-announce.ts deleted file mode 100644 index 9cc87ee27..000000000 --- a/server/lib/activitypub/process/process-announce.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { getAPId } from '@server/lib/activitypub/activity' -import { ActivityAnnounce } from '../../../../shared/models/activitypub' -import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { logger } from '../../../helpers/logger' -import { sequelizeTypescript } from '../../../initializers/database' -import { VideoShareModel } from '../../../models/video/video-share' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActorSignature, MVideoAccountLightBlacklistAllFiles } from '../../../types/models' -import { Notifier } from '../../notifier' -import { forwardVideoRelatedActivity } from '../send/shared/send-utils' -import { getOrCreateAPVideo } from '../videos' - -async function processAnnounceActivity (options: APProcessorOptions) { - const { activity, byActor: actorAnnouncer } = options - // Only notify if it is not from a fetcher job - const notify = options.fromFetch !== true - - // Announces on accounts are not supported - if (actorAnnouncer.type !== 'Application' && actorAnnouncer.type !== 'Group') return - - return retryTransactionWrapper(processVideoShare, actorAnnouncer, activity, notify) -} - -// --------------------------------------------------------------------------- - -export { - processAnnounceActivity -} - -// --------------------------------------------------------------------------- - -async function processVideoShare (actorAnnouncer: MActorSignature, activity: ActivityAnnounce, notify: boolean) { - const objectUri = getAPId(activity.object) - - let video: MVideoAccountLightBlacklistAllFiles - let videoCreated: boolean - - try { - const result = await getOrCreateAPVideo({ videoObject: objectUri }) - video = result.video - videoCreated = result.created - } catch (err) { - logger.debug('Cannot process share of %s. Maybe this is not a video object, so just skipping.', objectUri, { err }) - return - } - - await sequelizeTypescript.transaction(async t => { - // Add share entry - - const share = { - actorId: actorAnnouncer.id, - videoId: video.id, - url: activity.id - } - - const [ , created ] = await VideoShareModel.findOrCreate({ - where: { - url: activity.id - }, - defaults: share, - transaction: t - }) - - if (video.isOwned() && created === true) { - // Don't resend the activity to the sender - const exceptions = [ actorAnnouncer ] - - await forwardVideoRelatedActivity(activity, t, exceptions, video) - } - - return undefined - }) - - if (videoCreated && notify) Notifier.Instance.notifyOnNewVideoIfNeeded(video) -} diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts deleted file mode 100644 index 5f980de65..000000000 --- a/server/lib/activitypub/process/process-create.ts +++ /dev/null @@ -1,170 +0,0 @@ -import { isBlockedByServerOrAccount } from '@server/lib/blocklist' -import { isRedundancyAccepted } from '@server/lib/redundancy' -import { VideoModel } from '@server/models/video/video' -import { - AbuseObject, - ActivityCreate, - ActivityCreateObject, - ActivityObject, - CacheFileObject, - PlaylistObject, - VideoCommentObject, - VideoObject, - WatchActionObject -} from '@shared/models' -import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { logger } from '../../../helpers/logger' -import { sequelizeTypescript } from '../../../initializers/database' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../types/models' -import { Notifier } from '../../notifier' -import { fetchAPObjectIfNeeded } from '../activity' -import { createOrUpdateCacheFile } from '../cache-file' -import { createOrUpdateLocalVideoViewer } from '../local-video-viewer' -import { createOrUpdateVideoPlaylist } from '../playlists' -import { forwardVideoRelatedActivity } from '../send/shared/send-utils' -import { resolveThread } from '../video-comments' -import { getOrCreateAPVideo } from '../videos' - -async function processCreateActivity (options: APProcessorOptions>) { - const { activity, byActor } = options - - // Only notify if it is not from a fetcher job - const notify = options.fromFetch !== true - const activityObject = await fetchAPObjectIfNeeded>(activity.object) - const activityType = activityObject.type - - if (activityType === 'Video') { - return processCreateVideo(activityObject, notify) - } - - if (activityType === 'Note') { - // Comments will be fetched from videos - if (options.fromFetch) return - - return retryTransactionWrapper(processCreateVideoComment, activity, activityObject, byActor, notify) - } - - if (activityType === 'WatchAction') { - return retryTransactionWrapper(processCreateWatchAction, activityObject) - } - - if (activityType === 'CacheFile') { - return retryTransactionWrapper(processCreateCacheFile, activity, activityObject, byActor) - } - - if (activityType === 'Playlist') { - return retryTransactionWrapper(processCreatePlaylist, activity, activityObject, byActor) - } - - logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) - return Promise.resolve(undefined) -} - -// --------------------------------------------------------------------------- - -export { - processCreateActivity -} - -// --------------------------------------------------------------------------- - -async function processCreateVideo (videoToCreateData: VideoObject, notify: boolean) { - const syncParam = { rates: false, shares: false, comments: false, refreshVideo: false } - const { video, created } = await getOrCreateAPVideo({ videoObject: videoToCreateData, syncParam }) - - if (created && notify) Notifier.Instance.notifyOnNewVideoIfNeeded(video) - - return video -} - -async function processCreateCacheFile ( - activity: ActivityCreate, - cacheFile: CacheFileObject, - byActor: MActorSignature -) { - if (await isRedundancyAccepted(activity, byActor) !== true) return - - const { video } = await getOrCreateAPVideo({ videoObject: cacheFile.object }) - - await sequelizeTypescript.transaction(async t => { - return createOrUpdateCacheFile(cacheFile, video, byActor, t) - }) - - if (video.isOwned()) { - // Don't resend the activity to the sender - const exceptions = [ byActor ] - await forwardVideoRelatedActivity(activity, undefined, exceptions, video) - } -} - -async function processCreateWatchAction (watchAction: WatchActionObject) { - if (watchAction.actionStatus !== 'CompletedActionStatus') return - - const video = await VideoModel.loadByUrl(watchAction.object) - if (video.remote) return - - await sequelizeTypescript.transaction(async t => { - return createOrUpdateLocalVideoViewer(watchAction, video, t) - }) -} - -async function processCreateVideoComment ( - activity: ActivityCreate, - commentObject: VideoCommentObject, - byActor: MActorSignature, - notify: boolean -) { - const byAccount = byActor.Account - - if (!byAccount) throw new Error('Cannot create video comment with the non account actor ' + byActor.url) - - let video: MVideoAccountLightBlacklistAllFiles - let created: boolean - let comment: MCommentOwnerVideo - - try { - const resolveThreadResult = await resolveThread({ url: commentObject.id, isVideo: false }) - if (!resolveThreadResult) return // Comment not accepted - - video = resolveThreadResult.video - created = resolveThreadResult.commentCreated - comment = resolveThreadResult.comment - } catch (err) { - logger.debug( - 'Cannot process video comment because we could not resolve thread %s. Maybe it was not a video thread, so skip it.', - commentObject.inReplyTo, - { err } - ) - return - } - - // Try to not forward unwanted comments on our videos - if (video.isOwned()) { - if (await isBlockedByServerOrAccount(comment.Account, video.VideoChannel.Account)) { - logger.info('Skip comment forward from blocked account or server %s.', comment.Account.Actor.url) - return - } - - if (created === true) { - // Don't resend the activity to the sender - const exceptions = [ byActor ] - - await forwardVideoRelatedActivity(activity, undefined, exceptions, video) - } - } - - if (created && notify) Notifier.Instance.notifyOnNewComment(comment) -} - -async function processCreatePlaylist ( - activity: ActivityCreate, - playlistObject: PlaylistObject, - byActor: MActorSignature -) { - const byAccount = byActor.Account - - if (!byAccount) throw new Error('Cannot create video playlist with the non account actor ' + byActor.url) - - await createOrUpdateVideoPlaylist(playlistObject, activity.to) -} diff --git a/server/lib/activitypub/process/process-delete.ts b/server/lib/activitypub/process/process-delete.ts deleted file mode 100644 index ac0e7e235..000000000 --- a/server/lib/activitypub/process/process-delete.ts +++ /dev/null @@ -1,153 +0,0 @@ -import { ActivityDelete } from '../../../../shared/models/activitypub' -import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { logger } from '../../../helpers/logger' -import { sequelizeTypescript } from '../../../initializers/database' -import { ActorModel } from '../../../models/actor/actor' -import { VideoModel } from '../../../models/video/video' -import { VideoCommentModel } from '../../../models/video/video-comment' -import { VideoPlaylistModel } from '../../../models/video/video-playlist' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { - MAccountActor, - MActor, - MActorFull, - MActorSignature, - MChannelAccountActor, - MChannelActor, - MCommentOwnerVideo -} from '../../../types/models' -import { forwardVideoRelatedActivity } from '../send/shared/send-utils' - -async function processDeleteActivity (options: APProcessorOptions) { - const { activity, byActor } = options - - const objectUrl = typeof activity.object === 'string' ? activity.object : activity.object.id - - if (activity.actor === objectUrl) { - // We need more attributes (all the account and channel) - const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url) - - if (byActorFull.type === 'Person') { - if (!byActorFull.Account) throw new Error('Actor ' + byActorFull.url + ' is a person but we cannot find it in database.') - - const accountToDelete = byActorFull.Account as MAccountActor - accountToDelete.Actor = byActorFull - - return retryTransactionWrapper(processDeleteAccount, accountToDelete) - } else if (byActorFull.type === 'Group') { - if (!byActorFull.VideoChannel) throw new Error('Actor ' + byActorFull.url + ' is a group but we cannot find it in database.') - - const channelToDelete = byActorFull.VideoChannel as MChannelAccountActor & { Actor: MActorFull } - channelToDelete.Actor = byActorFull - return retryTransactionWrapper(processDeleteVideoChannel, channelToDelete) - } - } - - { - const videoCommentInstance = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(objectUrl) - if (videoCommentInstance) { - return retryTransactionWrapper(processDeleteVideoComment, byActor, videoCommentInstance, activity) - } - } - - { - const videoInstance = await VideoModel.loadByUrlAndPopulateAccount(objectUrl) - if (videoInstance) { - if (videoInstance.isOwned()) throw new Error(`Remote instance cannot delete owned video ${videoInstance.url}.`) - - return retryTransactionWrapper(processDeleteVideo, byActor, videoInstance) - } - } - - { - const videoPlaylist = await VideoPlaylistModel.loadByUrlAndPopulateAccount(objectUrl) - if (videoPlaylist) { - if (videoPlaylist.isOwned()) throw new Error(`Remote instance cannot delete owned playlist ${videoPlaylist.url}.`) - - return retryTransactionWrapper(processDeleteVideoPlaylist, byActor, videoPlaylist) - } - } - - return undefined -} - -// --------------------------------------------------------------------------- - -export { - processDeleteActivity -} - -// --------------------------------------------------------------------------- - -async function processDeleteVideo (actor: MActor, videoToDelete: VideoModel) { - logger.debug('Removing remote video "%s".', videoToDelete.uuid) - - await sequelizeTypescript.transaction(async t => { - if (videoToDelete.VideoChannel.Account.Actor.id !== actor.id) { - throw new Error('Account ' + actor.url + ' does not own video channel ' + videoToDelete.VideoChannel.Actor.url) - } - - await videoToDelete.destroy({ transaction: t }) - }) - - logger.info('Remote video with uuid %s removed.', videoToDelete.uuid) -} - -async function processDeleteVideoPlaylist (actor: MActor, playlistToDelete: VideoPlaylistModel) { - logger.debug('Removing remote video playlist "%s".', playlistToDelete.uuid) - - await sequelizeTypescript.transaction(async t => { - if (playlistToDelete.OwnerAccount.Actor.id !== actor.id) { - throw new Error('Account ' + actor.url + ' does not own video playlist ' + playlistToDelete.url) - } - - await playlistToDelete.destroy({ transaction: t }) - }) - - logger.info('Remote video playlist with uuid %s removed.', playlistToDelete.uuid) -} - -async function processDeleteAccount (accountToRemove: MAccountActor) { - logger.debug('Removing remote account "%s".', accountToRemove.Actor.url) - - await sequelizeTypescript.transaction(async t => { - await accountToRemove.destroy({ transaction: t }) - }) - - logger.info('Remote account %s removed.', accountToRemove.Actor.url) -} - -async function processDeleteVideoChannel (videoChannelToRemove: MChannelActor) { - logger.debug('Removing remote video channel "%s".', videoChannelToRemove.Actor.url) - - await sequelizeTypescript.transaction(async t => { - await videoChannelToRemove.destroy({ transaction: t }) - }) - - logger.info('Remote video channel %s removed.', videoChannelToRemove.Actor.url) -} - -function processDeleteVideoComment (byActor: MActorSignature, videoComment: MCommentOwnerVideo, activity: ActivityDelete) { - // Already deleted - if (videoComment.isDeleted()) return Promise.resolve() - - logger.debug('Removing remote video comment "%s".', videoComment.url) - - return sequelizeTypescript.transaction(async t => { - if (byActor.Account.id !== videoComment.Account.id && byActor.Account.id !== videoComment.Video.VideoChannel.accountId) { - throw new Error(`Account ${byActor.url} does not own video comment ${videoComment.url} or video ${videoComment.Video.url}`) - } - - videoComment.markAsDeleted() - - await videoComment.save({ transaction: t }) - - if (videoComment.Video.isOwned()) { - // Don't resend the activity to the sender - const exceptions = [ byActor ] - await forwardVideoRelatedActivity(activity, t, exceptions, videoComment.Video) - } - - logger.info('Remote video comment %s removed.', videoComment.url) - }) -} diff --git a/server/lib/activitypub/process/process-dislike.ts b/server/lib/activitypub/process/process-dislike.ts deleted file mode 100644 index 4e270f917..000000000 --- a/server/lib/activitypub/process/process-dislike.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { VideoModel } from '@server/models/video/video' -import { ActivityDislike } from '@shared/models' -import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { sequelizeTypescript } from '../../../initializers/database' -import { AccountVideoRateModel } from '../../../models/account/account-video-rate' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActorSignature } from '../../../types/models' -import { federateVideoIfNeeded, getOrCreateAPVideo } from '../videos' - -async function processDislikeActivity (options: APProcessorOptions) { - const { activity, byActor } = options - return retryTransactionWrapper(processDislike, activity, byActor) -} - -// --------------------------------------------------------------------------- - -export { - processDislikeActivity -} - -// --------------------------------------------------------------------------- - -async function processDislike (activity: ActivityDislike, byActor: MActorSignature) { - const dislikeObject = activity.object - const byAccount = byActor.Account - - if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) - - const { video: onlyVideo } = await getOrCreateAPVideo({ videoObject: dislikeObject, fetchType: 'only-video' }) - - // We don't care about dislikes of remote videos - if (!onlyVideo.isOwned()) return - - return sequelizeTypescript.transaction(async t => { - const video = await VideoModel.loadFull(onlyVideo.id, t) - - const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) - if (existingRate && existingRate.type === 'dislike') return - - await video.increment('dislikes', { transaction: t }) - video.dislikes++ - - if (existingRate && existingRate.type === 'like') { - await video.decrement('likes', { transaction: t }) - video.likes-- - } - - const rate = existingRate || new AccountVideoRateModel() - rate.type = 'dislike' - rate.videoId = video.id - rate.accountId = byAccount.id - rate.url = activity.id - - await rate.save({ transaction: t }) - - await federateVideoIfNeeded(video, false, t) - }) -} diff --git a/server/lib/activitypub/process/process-flag.ts b/server/lib/activitypub/process/process-flag.ts deleted file mode 100644 index bea285670..000000000 --- a/server/lib/activitypub/process/process-flag.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { createAccountAbuse, createVideoAbuse, createVideoCommentAbuse } from '@server/lib/moderation' -import { AccountModel } from '@server/models/account/account' -import { VideoModel } from '@server/models/video/video' -import { VideoCommentModel } from '@server/models/video/video-comment' -import { abusePredefinedReasonsMap } from '@shared/core-utils/abuse' -import { AbuseState, ActivityFlag } from '@shared/models' -import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { logger } from '../../../helpers/logger' -import { sequelizeTypescript } from '../../../initializers/database' -import { getAPId } from '../../../lib/activitypub/activity' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MAccountDefault, MActorSignature, MCommentOwnerVideo } from '../../../types/models' - -async function processFlagActivity (options: APProcessorOptions) { - const { activity, byActor } = options - - return retryTransactionWrapper(processCreateAbuse, activity, byActor) -} - -// --------------------------------------------------------------------------- - -export { - processFlagActivity -} - -// --------------------------------------------------------------------------- - -async function processCreateAbuse (flag: ActivityFlag, byActor: MActorSignature) { - const account = byActor.Account - if (!account) throw new Error('Cannot create abuse with the non account actor ' + byActor.url) - - const reporterAccount = await AccountModel.load(account.id) - - const objects = Array.isArray(flag.object) ? flag.object : [ flag.object ] - - const tags = Array.isArray(flag.tag) ? flag.tag : [] - const predefinedReasons = tags.map(tag => abusePredefinedReasonsMap[tag.name]) - .filter(v => !isNaN(v)) - - const startAt = flag.startAt - const endAt = flag.endAt - - for (const object of objects) { - try { - const uri = getAPId(object) - - logger.debug('Reporting remote abuse for object %s.', uri) - - await sequelizeTypescript.transaction(async t => { - const video = await VideoModel.loadByUrlAndPopulateAccount(uri, t) - let videoComment: MCommentOwnerVideo - let flaggedAccount: MAccountDefault - - if (!video) videoComment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(uri, t) - if (!videoComment) flaggedAccount = await AccountModel.loadByUrl(uri, t) - - if (!video && !videoComment && !flaggedAccount) { - logger.warn('Cannot flag unknown entity %s.', object) - return - } - - const baseAbuse = { - reporterAccountId: reporterAccount.id, - reason: flag.content, - state: AbuseState.PENDING, - predefinedReasons - } - - if (video) { - return createVideoAbuse({ - baseAbuse, - startAt, - endAt, - reporterAccount, - transaction: t, - videoInstance: video, - skipNotification: false - }) - } - - if (videoComment) { - return createVideoCommentAbuse({ - baseAbuse, - reporterAccount, - transaction: t, - commentInstance: videoComment, - skipNotification: false - }) - } - - return await createAccountAbuse({ - baseAbuse, - reporterAccount, - transaction: t, - accountInstance: flaggedAccount, - skipNotification: false - }) - }) - } catch (err) { - logger.debug('Cannot process report of %s', getAPId(object), { err }) - } - } -} diff --git a/server/lib/activitypub/process/process-follow.ts b/server/lib/activitypub/process/process-follow.ts deleted file mode 100644 index 7def753d5..000000000 --- a/server/lib/activitypub/process/process-follow.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { Transaction } from 'sequelize/types' -import { isBlockedByServerOrAccount } from '@server/lib/blocklist' -import { AccountModel } from '@server/models/account/account' -import { getServerActor } from '@server/models/application/application' -import { ActivityFollow } from '../../../../shared/models/activitypub' -import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { logger } from '../../../helpers/logger' -import { CONFIG } from '../../../initializers/config' -import { sequelizeTypescript } from '../../../initializers/database' -import { getAPId } from '../../../lib/activitypub/activity' -import { ActorModel } from '../../../models/actor/actor' -import { ActorFollowModel } from '../../../models/actor/actor-follow' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActorFollow, MActorFull, MActorId, MActorSignature } from '../../../types/models' -import { Notifier } from '../../notifier' -import { autoFollowBackIfNeeded } from '../follow' -import { sendAccept, sendReject } from '../send' - -async function processFollowActivity (options: APProcessorOptions) { - const { activity, byActor } = options - - const activityId = activity.id - const objectId = getAPId(activity.object) - - return retryTransactionWrapper(processFollow, byActor, activityId, objectId) -} - -// --------------------------------------------------------------------------- - -export { - processFollowActivity -} - -// --------------------------------------------------------------------------- - -async function processFollow (byActor: MActorSignature, activityId: string, targetActorURL: string) { - const { actorFollow, created, targetActor } = await sequelizeTypescript.transaction(async t => { - const targetActor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(targetActorURL, t) - - if (!targetActor) throw new Error('Unknown actor') - if (targetActor.isOwned() === false) throw new Error('This is not a local actor.') - - if (await rejectIfInstanceFollowDisabled(byActor, activityId, targetActor)) return { actorFollow: undefined } - if (await rejectIfMuted(byActor, activityId, targetActor)) return { actorFollow: undefined } - - const [ actorFollow, created ] = await ActorFollowModel.findOrCreateCustom({ - byActor, - targetActor, - activityId, - state: await isFollowingInstance(targetActor) && CONFIG.FOLLOWERS.INSTANCE.MANUAL_APPROVAL - ? 'pending' - : 'accepted', - transaction: t - }) - - if (rejectIfAlreadyRejected(actorFollow, byActor, activityId, targetActor)) return { actorFollow: undefined } - - await acceptIfNeeded(actorFollow, targetActor, t) - - await fixFollowURLIfNeeded(actorFollow, activityId, t) - - actorFollow.ActorFollower = byActor - actorFollow.ActorFollowing = targetActor - - // Target sends to actor he accepted the follow request - if (actorFollow.state === 'accepted') { - sendAccept(actorFollow) - - await autoFollowBackIfNeeded(actorFollow, t) - } - - return { actorFollow, created, targetActor } - }) - - // Rejected - if (!actorFollow) return - - if (created) { - const follower = await ActorModel.loadFull(byActor.id) - const actorFollowFull = Object.assign(actorFollow, { ActorFollowing: targetActor, ActorFollower: follower }) - - if (await isFollowingInstance(targetActor)) { - Notifier.Instance.notifyOfNewInstanceFollow(actorFollowFull) - } else { - Notifier.Instance.notifyOfNewUserFollow(actorFollowFull) - } - } - - logger.info('Actor %s is followed by actor %s.', targetActorURL, byActor.url) -} - -async function rejectIfInstanceFollowDisabled (byActor: MActorSignature, activityId: string, targetActor: MActorFull) { - if (await isFollowingInstance(targetActor) && CONFIG.FOLLOWERS.INSTANCE.ENABLED === false) { - logger.info('Rejecting %s because instance followers are disabled.', targetActor.url) - - sendReject(activityId, byActor, targetActor) - - return true - } - - return false -} - -async function rejectIfMuted (byActor: MActorSignature, activityId: string, targetActor: MActorFull) { - const followerAccount = await AccountModel.load(byActor.Account.id) - const followingAccountId = targetActor.Account - - if (followerAccount && await isBlockedByServerOrAccount(followerAccount, followingAccountId)) { - logger.info('Rejecting %s because follower is muted.', byActor.url) - - sendReject(activityId, byActor, targetActor) - - return true - } - - return false -} - -function rejectIfAlreadyRejected (actorFollow: MActorFollow, byActor: MActorSignature, activityId: string, targetActor: MActorFull) { - // Already rejected - if (actorFollow.state === 'rejected') { - logger.info('Rejecting %s because follow is already rejected.', byActor.url) - - sendReject(activityId, byActor, targetActor) - - return true - } - - return false -} - -async function acceptIfNeeded (actorFollow: MActorFollow, targetActor: MActorFull, transaction: Transaction) { - // Set the follow as accepted if the remote actor follows a channel or account - // Or if the instance automatically accepts followers - if (actorFollow.state === 'accepted') return - if (!await isFollowingInstance(targetActor)) return - if (CONFIG.FOLLOWERS.INSTANCE.MANUAL_APPROVAL === true && await isFollowingInstance(targetActor)) return - - actorFollow.state = 'accepted' - - await actorFollow.save({ transaction }) -} - -async function fixFollowURLIfNeeded (actorFollow: MActorFollow, activityId: string, transaction: Transaction) { - // Before PeerTube V3 we did not save the follow ID. Try to fix these old follows - if (!actorFollow.url) { - actorFollow.url = activityId - await actorFollow.save({ transaction }) - } -} - -async function isFollowingInstance (targetActor: MActorId) { - const serverActor = await getServerActor() - - return targetActor.id === serverActor.id -} diff --git a/server/lib/activitypub/process/process-like.ts b/server/lib/activitypub/process/process-like.ts deleted file mode 100644 index 580a05bcd..000000000 --- a/server/lib/activitypub/process/process-like.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { VideoModel } from '@server/models/video/video' -import { ActivityLike } from '../../../../shared/models/activitypub' -import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { sequelizeTypescript } from '../../../initializers/database' -import { getAPId } from '../../../lib/activitypub/activity' -import { AccountVideoRateModel } from '../../../models/account/account-video-rate' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActorSignature } from '../../../types/models' -import { federateVideoIfNeeded, getOrCreateAPVideo } from '../videos' - -async function processLikeActivity (options: APProcessorOptions) { - const { activity, byActor } = options - - return retryTransactionWrapper(processLikeVideo, byActor, activity) -} - -// --------------------------------------------------------------------------- - -export { - processLikeActivity -} - -// --------------------------------------------------------------------------- - -async function processLikeVideo (byActor: MActorSignature, activity: ActivityLike) { - const videoUrl = getAPId(activity.object) - - const byAccount = byActor.Account - if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) - - const { video: onlyVideo } = await getOrCreateAPVideo({ videoObject: videoUrl, fetchType: 'only-video' }) - - // We don't care about likes of remote videos - if (!onlyVideo.isOwned()) return - - return sequelizeTypescript.transaction(async t => { - const video = await VideoModel.loadFull(onlyVideo.id, t) - - const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) - if (existingRate && existingRate.type === 'like') return - - if (existingRate && existingRate.type === 'dislike') { - await video.decrement('dislikes', { transaction: t }) - video.dislikes-- - } - - await video.increment('likes', { transaction: t }) - video.likes++ - - const rate = existingRate || new AccountVideoRateModel() - rate.type = 'like' - rate.videoId = video.id - rate.accountId = byAccount.id - rate.url = activity.id - - await rate.save({ transaction: t }) - - await federateVideoIfNeeded(video, false, t) - }) -} diff --git a/server/lib/activitypub/process/process-reject.ts b/server/lib/activitypub/process/process-reject.ts deleted file mode 100644 index db7ff24d8..000000000 --- a/server/lib/activitypub/process/process-reject.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { ActivityReject } from '../../../../shared/models/activitypub/activity' -import { sequelizeTypescript } from '../../../initializers/database' -import { ActorFollowModel } from '../../../models/actor/actor-follow' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActor } from '../../../types/models' - -async function processRejectActivity (options: APProcessorOptions) { - const { byActor: targetActor, inboxActor } = options - if (inboxActor === undefined) throw new Error('Need to reject on explicit inbox.') - - return processReject(inboxActor, targetActor) -} - -// --------------------------------------------------------------------------- - -export { - processRejectActivity -} - -// --------------------------------------------------------------------------- - -async function processReject (follower: MActor, targetActor: MActor) { - return sequelizeTypescript.transaction(async t => { - const actorFollow = await ActorFollowModel.loadByActorAndTarget(follower.id, targetActor.id, t) - - if (!actorFollow) throw new Error(`'Unknown actor follow ${follower.id} -> ${targetActor.id}.`) - - actorFollow.state = 'rejected' - await actorFollow.save({ transaction: t }) - - return undefined - }) -} diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts deleted file mode 100644 index a9d8199de..000000000 --- a/server/lib/activitypub/process/process-undo.ts +++ /dev/null @@ -1,183 +0,0 @@ -import { VideoModel } from '@server/models/video/video' -import { - ActivityAnnounce, - ActivityCreate, - ActivityDislike, - ActivityFollow, - ActivityLike, - ActivityUndo, - ActivityUndoObject, - CacheFileObject -} from '../../../../shared/models/activitypub' -import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { logger } from '../../../helpers/logger' -import { sequelizeTypescript } from '../../../initializers/database' -import { AccountVideoRateModel } from '../../../models/account/account-video-rate' -import { ActorModel } from '../../../models/actor/actor' -import { ActorFollowModel } from '../../../models/actor/actor-follow' -import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' -import { VideoShareModel } from '../../../models/video/video-share' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActorSignature } from '../../../types/models' -import { fetchAPObjectIfNeeded } from '../activity' -import { forwardVideoRelatedActivity } from '../send/shared/send-utils' -import { federateVideoIfNeeded, getOrCreateAPVideo } from '../videos' - -async function processUndoActivity (options: APProcessorOptions>) { - const { activity, byActor } = options - const activityToUndo = activity.object - - if (activityToUndo.type === 'Like') { - return retryTransactionWrapper(processUndoLike, byActor, activity) - } - - if (activityToUndo.type === 'Create') { - const objectToUndo = await fetchAPObjectIfNeeded(activityToUndo.object) - - if (objectToUndo.type === 'CacheFile') { - return retryTransactionWrapper(processUndoCacheFile, byActor, activity, objectToUndo) - } - } - - if (activityToUndo.type === 'Dislike') { - return retryTransactionWrapper(processUndoDislike, byActor, activity) - } - - if (activityToUndo.type === 'Follow') { - return retryTransactionWrapper(processUndoFollow, byActor, activityToUndo) - } - - if (activityToUndo.type === 'Announce') { - return retryTransactionWrapper(processUndoAnnounce, byActor, activityToUndo) - } - - logger.warn('Unknown activity object type %s -> %s when undo activity.', activityToUndo.type, { activity: activity.id }) - - return undefined -} - -// --------------------------------------------------------------------------- - -export { - processUndoActivity -} - -// --------------------------------------------------------------------------- - -async function processUndoLike (byActor: MActorSignature, activity: ActivityUndo) { - const likeActivity = activity.object - - const { video: onlyVideo } = await getOrCreateAPVideo({ videoObject: likeActivity.object }) - // We don't care about likes of remote videos - if (!onlyVideo.isOwned()) return - - return sequelizeTypescript.transaction(async t => { - if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) - - const video = await VideoModel.loadFull(onlyVideo.id, t) - const rate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byActor.Account.id, video.id, likeActivity.id, t) - if (!rate || rate.type !== 'like') { - logger.warn('Unknown like by account %d for video %d.', byActor.Account.id, video.id) - return - } - - await rate.destroy({ transaction: t }) - await video.decrement('likes', { transaction: t }) - - video.likes-- - await federateVideoIfNeeded(video, false, t) - }) -} - -async function processUndoDislike (byActor: MActorSignature, activity: ActivityUndo) { - const dislikeActivity = activity.object - - const { video: onlyVideo } = await getOrCreateAPVideo({ videoObject: dislikeActivity.object }) - // We don't care about likes of remote videos - if (!onlyVideo.isOwned()) return - - return sequelizeTypescript.transaction(async t => { - if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) - - const video = await VideoModel.loadFull(onlyVideo.id, t) - const rate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byActor.Account.id, video.id, dislikeActivity.id, t) - if (!rate || rate.type !== 'dislike') { - logger.warn(`Unknown dislike by account %d for video %d.`, byActor.Account.id, video.id) - return - } - - await rate.destroy({ transaction: t }) - await video.decrement('dislikes', { transaction: t }) - video.dislikes-- - - await federateVideoIfNeeded(video, false, t) - }) -} - -// --------------------------------------------------------------------------- - -async function processUndoCacheFile ( - byActor: MActorSignature, - activity: ActivityUndo>, - cacheFileObject: CacheFileObject -) { - const { video } = await getOrCreateAPVideo({ videoObject: cacheFileObject.object }) - - return sequelizeTypescript.transaction(async t => { - const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t) - if (!cacheFile) { - logger.debug('Cannot undo unknown video cache %s.', cacheFileObject.id) - return - } - - if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.') - - await cacheFile.destroy({ transaction: t }) - - if (video.isOwned()) { - // Don't resend the activity to the sender - const exceptions = [ byActor ] - - await forwardVideoRelatedActivity(activity, t, exceptions, video) - } - }) -} - -function processUndoAnnounce (byActor: MActorSignature, announceActivity: ActivityAnnounce) { - return sequelizeTypescript.transaction(async t => { - const share = await VideoShareModel.loadByUrl(announceActivity.id, t) - if (!share) { - logger.warn('Unknown video share %d', announceActivity.id) - return - } - - if (share.actorId !== byActor.id) throw new Error(`${share.url} is not shared by ${byActor.url}.`) - - await share.destroy({ transaction: t }) - - if (share.Video.isOwned()) { - // Don't resend the activity to the sender - const exceptions = [ byActor ] - - await forwardVideoRelatedActivity(announceActivity, t, exceptions, share.Video) - } - }) -} - -// --------------------------------------------------------------------------- - -function processUndoFollow (follower: MActorSignature, followActivity: ActivityFollow) { - return sequelizeTypescript.transaction(async t => { - const following = await ActorModel.loadByUrlAndPopulateAccountAndChannel(followActivity.object, t) - const actorFollow = await ActorFollowModel.loadByActorAndTarget(follower.id, following.id, t) - - if (!actorFollow) { - logger.warn('Unknown actor follow %d -> %d.', follower.id, following.id) - return - } - - await actorFollow.destroy({ transaction: t }) - - return undefined - }) -} diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts deleted file mode 100644 index 304ed9de6..000000000 --- a/server/lib/activitypub/process/process-update.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { isRedundancyAccepted } from '@server/lib/redundancy' -import { ActivityUpdate, ActivityUpdateObject, CacheFileObject, VideoObject } from '../../../../shared/models/activitypub' -import { ActivityPubActor } from '../../../../shared/models/activitypub/activitypub-actor' -import { PlaylistObject } from '../../../../shared/models/activitypub/objects/playlist-object' -import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file' -import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos' -import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { logger } from '../../../helpers/logger' -import { sequelizeTypescript } from '../../../initializers/database' -import { ActorModel } from '../../../models/actor/actor' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActorFull, MActorSignature } from '../../../types/models' -import { fetchAPObjectIfNeeded } from '../activity' -import { APActorUpdater } from '../actors/updater' -import { createOrUpdateCacheFile } from '../cache-file' -import { createOrUpdateVideoPlaylist } from '../playlists' -import { forwardVideoRelatedActivity } from '../send/shared/send-utils' -import { APVideoUpdater, getOrCreateAPVideo } from '../videos' - -async function processUpdateActivity (options: APProcessorOptions>) { - const { activity, byActor } = options - - const object = await fetchAPObjectIfNeeded(activity.object) - const objectType = object.type - - if (objectType === 'Video') { - return retryTransactionWrapper(processUpdateVideo, activity) - } - - if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') { - // We need more attributes - const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url) - return retryTransactionWrapper(processUpdateActor, byActorFull, object) - } - - if (objectType === 'CacheFile') { - // We need more attributes - const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url) - return retryTransactionWrapper(processUpdateCacheFile, byActorFull, activity, object) - } - - if (objectType === 'Playlist') { - return retryTransactionWrapper(processUpdatePlaylist, byActor, activity, object) - } - - return undefined -} - -// --------------------------------------------------------------------------- - -export { - processUpdateActivity -} - -// --------------------------------------------------------------------------- - -async function processUpdateVideo (activity: ActivityUpdate) { - const videoObject = activity.object as VideoObject - - if (sanitizeAndCheckVideoTorrentObject(videoObject) === false) { - logger.debug('Video sent by update is not valid.', { videoObject }) - return undefined - } - - const { video, created } = await getOrCreateAPVideo({ - videoObject: videoObject.id, - allowRefresh: false, - fetchType: 'all' - }) - // We did not have this video, it has been created so no need to update - if (created) return - - const updater = new APVideoUpdater(videoObject, video) - return updater.update(activity.to) -} - -async function processUpdateCacheFile ( - byActor: MActorSignature, - activity: ActivityUpdate, - cacheFileObject: CacheFileObject -) { - if (await isRedundancyAccepted(activity, byActor) !== true) return - - if (!isCacheFileObjectValid(cacheFileObject)) { - logger.debug('Cache file object sent by update is not valid.', { cacheFileObject }) - return undefined - } - - const { video } = await getOrCreateAPVideo({ videoObject: cacheFileObject.object }) - - await sequelizeTypescript.transaction(async t => { - await createOrUpdateCacheFile(cacheFileObject, video, byActor, t) - }) - - if (video.isOwned()) { - // Don't resend the activity to the sender - const exceptions = [ byActor ] - - await forwardVideoRelatedActivity(activity, undefined, exceptions, video) - } -} - -async function processUpdateActor (actor: MActorFull, actorObject: ActivityPubActor) { - logger.debug('Updating remote account "%s".', actorObject.url) - - const updater = new APActorUpdater(actorObject, actor) - return updater.update() -} - -async function processUpdatePlaylist ( - byActor: MActorSignature, - activity: ActivityUpdate, - playlistObject: PlaylistObject -) { - const byAccount = byActor.Account - if (!byAccount) throw new Error('Cannot update video playlist with the non account actor ' + byActor.url) - - await createOrUpdateVideoPlaylist(playlistObject, activity.to) -} diff --git a/server/lib/activitypub/process/process-view.ts b/server/lib/activitypub/process/process-view.ts deleted file mode 100644 index e49506d82..000000000 --- a/server/lib/activitypub/process/process-view.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { VideoViewsManager } from '@server/lib/views/video-views-manager' -import { ActivityView } from '../../../../shared/models/activitypub' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActorSignature } from '../../../types/models' -import { forwardVideoRelatedActivity } from '../send/shared/send-utils' -import { getOrCreateAPVideo } from '../videos' - -async function processViewActivity (options: APProcessorOptions) { - const { activity, byActor } = options - - return processCreateView(activity, byActor) -} - -// --------------------------------------------------------------------------- - -export { - processViewActivity -} - -// --------------------------------------------------------------------------- - -async function processCreateView (activity: ActivityView, byActor: MActorSignature) { - const videoObject = activity.object - - const { video } = await getOrCreateAPVideo({ - videoObject, - fetchType: 'only-video', - allowRefresh: false - }) - - const viewerExpires = activity.expires - ? new Date(activity.expires) - : undefined - - await VideoViewsManager.Instance.processRemoteView({ video, viewerId: activity.id, viewerExpires }) - - if (video.isOwned()) { - // Forward the view but don't resend the activity to the sender - const exceptions = [ byActor ] - await forwardVideoRelatedActivity(activity, undefined, exceptions, video) - } -} diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts deleted file mode 100644 index 2bc3dce03..000000000 --- a/server/lib/activitypub/process/process.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { StatsManager } from '@server/lib/stat-manager' -import { Activity, ActivityType } from '../../../../shared/models/activitypub' -import { logger } from '../../../helpers/logger' -import { APProcessorOptions } from '../../../types/activitypub-processor.model' -import { MActorDefault, MActorSignature } from '../../../types/models' -import { getAPId } from '../activity' -import { getOrCreateAPActor } from '../actors' -import { checkUrlsSameHost } from '../url' -import { processAcceptActivity } from './process-accept' -import { processAnnounceActivity } from './process-announce' -import { processCreateActivity } from './process-create' -import { processDeleteActivity } from './process-delete' -import { processDislikeActivity } from './process-dislike' -import { processFlagActivity } from './process-flag' -import { processFollowActivity } from './process-follow' -import { processLikeActivity } from './process-like' -import { processRejectActivity } from './process-reject' -import { processUndoActivity } from './process-undo' -import { processUpdateActivity } from './process-update' -import { processViewActivity } from './process-view' - -const processActivity: { [ P in ActivityType ]: (options: APProcessorOptions) => Promise } = { - Create: processCreateActivity, - Update: processUpdateActivity, - Delete: processDeleteActivity, - Follow: processFollowActivity, - Accept: processAcceptActivity, - Reject: processRejectActivity, - Announce: processAnnounceActivity, - Undo: processUndoActivity, - Like: processLikeActivity, - Dislike: processDislikeActivity, - Flag: processFlagActivity, - View: processViewActivity -} - -async function processActivities ( - activities: Activity[], - options: { - signatureActor?: MActorSignature - inboxActor?: MActorDefault - outboxUrl?: string - fromFetch?: boolean - } = {} -) { - const { outboxUrl, signatureActor, inboxActor, fromFetch = false } = options - - const actorsCache: { [ url: string ]: MActorSignature } = {} - - for (const activity of activities) { - if (!signatureActor && [ 'Create', 'Announce', 'Like' ].includes(activity.type) === false) { - logger.error('Cannot process activity %s (type: %s) without the actor signature.', activity.id, activity.type) - continue - } - - const actorUrl = getAPId(activity.actor) - - // When we fetch remote data, we don't have signature - if (signatureActor && actorUrl !== signatureActor.url) { - logger.warn('Signature mismatch between %s and %s, skipping.', actorUrl, signatureActor.url) - continue - } - - if (outboxUrl && checkUrlsSameHost(outboxUrl, actorUrl) !== true) { - logger.warn('Host mismatch between outbox URL %s and actor URL %s, skipping.', outboxUrl, actorUrl) - continue - } - - const byActor = signatureActor || actorsCache[actorUrl] || await getOrCreateAPActor(actorUrl) - actorsCache[actorUrl] = byActor - - const activityProcessor = processActivity[activity.type] - if (activityProcessor === undefined) { - logger.warn('Unknown activity type %s.', activity.type, { activityId: activity.id }) - continue - } - - try { - await activityProcessor({ activity, byActor, inboxActor, fromFetch }) - - StatsManager.Instance.addInboxProcessedSuccess(activity.type) - } catch (err) { - logger.warn('Cannot process activity %s.', activity.type, { err }) - - StatsManager.Instance.addInboxProcessedError(activity.type) - } - } -} - -export { - processActivities -} diff --git a/server/lib/activitypub/send/http.ts b/server/lib/activitypub/send/http.ts deleted file mode 100644 index b461aa55d..000000000 --- a/server/lib/activitypub/send/http.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { buildDigest, signJsonLDObject } from '@server/helpers/peertube-crypto' -import { ACTIVITY_PUB, HTTP_SIGNATURE } from '@server/initializers/constants' -import { ActorModel } from '@server/models/actor/actor' -import { getServerActor } from '@server/models/application/application' -import { MActor } from '@server/types/models' -import { ContextType } from '@shared/models/activitypub/context' -import { activityPubContextify } from '../context' - -type Payload = { body: T, contextType: ContextType, signatureActorId?: number } - -async function computeBody ( - payload: Payload -): Promise { - let body = payload.body - - if (payload.signatureActorId) { - const actorSignature = await ActorModel.load(payload.signatureActorId) - if (!actorSignature) throw new Error('Unknown signature actor id.') - - body = await signAndContextify(actorSignature, payload.body, payload.contextType) - } - - return body -} - -async function buildSignedRequestOptions (options: { - signatureActorId?: number - hasPayload: boolean -}) { - let actor: MActor | null - - if (options.signatureActorId) { - actor = await ActorModel.load(options.signatureActorId) - if (!actor) throw new Error('Unknown signature actor id.') - } else { - // We need to sign the request, so use the server - actor = await getServerActor() - } - - const keyId = actor.url - return { - algorithm: HTTP_SIGNATURE.ALGORITHM, - authorizationHeaderName: HTTP_SIGNATURE.HEADER_NAME, - keyId, - key: actor.privateKey, - headers: options.hasPayload - ? HTTP_SIGNATURE.HEADERS_TO_SIGN_WITH_PAYLOAD - : HTTP_SIGNATURE.HEADERS_TO_SIGN_WITHOUT_PAYLOAD - } -} - -function buildGlobalHeaders (body: any) { - return { - 'digest': buildDigest(body), - 'content-type': 'application/activity+json', - 'accept': ACTIVITY_PUB.ACCEPT_HEADER - } -} - -async function signAndContextify (byActor: MActor, data: T, contextType: ContextType | null) { - const activity = contextType - ? await activityPubContextify(data, contextType) - : data - - return signJsonLDObject(byActor, activity) -} - -export { - buildGlobalHeaders, - computeBody, - buildSignedRequestOptions, - signAndContextify -} diff --git a/server/lib/activitypub/send/index.ts b/server/lib/activitypub/send/index.ts deleted file mode 100644 index 852ea2e74..000000000 --- a/server/lib/activitypub/send/index.ts +++ /dev/null @@ -1,10 +0,0 @@ -export * from './http' -export * from './send-accept' -export * from './send-announce' -export * from './send-create' -export * from './send-delete' -export * from './send-follow' -export * from './send-like' -export * from './send-reject' -export * from './send-undo' -export * from './send-update' diff --git a/server/lib/activitypub/send/send-accept.ts b/server/lib/activitypub/send/send-accept.ts deleted file mode 100644 index 4c9bcbb0b..000000000 --- a/server/lib/activitypub/send/send-accept.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { ActivityAccept, ActivityFollow } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { MActor, MActorFollowActors } from '../../../types/models' -import { getLocalActorFollowAcceptActivityPubUrl } from '../url' -import { buildFollowActivity } from './send-follow' -import { unicastTo } from './shared/send-utils' - -function sendAccept (actorFollow: MActorFollowActors) { - const follower = actorFollow.ActorFollower - const me = actorFollow.ActorFollowing - - if (!follower.serverId) { // This should never happen - logger.warn('Do not sending accept to local follower.') - return - } - - logger.info('Creating job to accept follower %s.', follower.url) - - const followData = buildFollowActivity(actorFollow.url, follower, me) - - const url = getLocalActorFollowAcceptActivityPubUrl(actorFollow) - const data = buildAcceptActivity(url, me, followData) - - return unicastTo({ - data, - byActor: me, - toActorUrl: follower.inboxUrl, - contextType: 'Accept' - }) -} - -// --------------------------------------------------------------------------- - -export { - sendAccept -} - -// --------------------------------------------------------------------------- - -function buildAcceptActivity (url: string, byActor: MActor, followActivityData: ActivityFollow): ActivityAccept { - return { - type: 'Accept', - id: url, - actor: byActor.url, - object: followActivityData - } -} diff --git a/server/lib/activitypub/send/send-announce.ts b/server/lib/activitypub/send/send-announce.ts deleted file mode 100644 index 6c078b047..000000000 --- a/server/lib/activitypub/send/send-announce.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { Transaction } from 'sequelize' -import { ActivityAnnounce, ActivityAudience } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { MActorLight, MVideo } from '../../../types/models' -import { MVideoShare } from '../../../types/models/video' -import { audiencify, getAudience } from '../audience' -import { getActorsInvolvedInVideo, getAudienceFromFollowersOf } from './shared' -import { broadcastToFollowers } from './shared/send-utils' - -async function buildAnnounceWithVideoAudience ( - byActor: MActorLight, - videoShare: MVideoShare, - video: MVideo, - t: Transaction -) { - const announcedObject = video.url - - const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) - const audience = getAudienceFromFollowersOf(actorsInvolvedInVideo) - - const activity = buildAnnounceActivity(videoShare.url, byActor, announcedObject, audience) - - return { activity, actorsInvolvedInVideo } -} - -async function sendVideoAnnounce (byActor: MActorLight, videoShare: MVideoShare, video: MVideo, transaction: Transaction) { - const { activity, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, transaction) - - logger.info('Creating job to send announce %s.', videoShare.url) - - return broadcastToFollowers({ - data: activity, - byActor, - toFollowersOf: actorsInvolvedInVideo, - transaction, - actorsException: [ byActor ], - contextType: 'Announce' - }) -} - -function buildAnnounceActivity (url: string, byActor: MActorLight, object: string, audience?: ActivityAudience): ActivityAnnounce { - if (!audience) audience = getAudience(byActor) - - return audiencify({ - type: 'Announce' as 'Announce', - id: url, - actor: byActor.url, - object - }, audience) -} - -// --------------------------------------------------------------------------- - -export { - sendVideoAnnounce, - buildAnnounceActivity, - buildAnnounceWithVideoAudience -} diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts deleted file mode 100644 index 2cd4db14d..000000000 --- a/server/lib/activitypub/send/send-create.ts +++ /dev/null @@ -1,226 +0,0 @@ -import { Transaction } from 'sequelize' -import { getServerActor } from '@server/models/application/application' -import { - ActivityAudience, - ActivityCreate, - ActivityCreateObject, - ContextType, - VideoCommentObject, - VideoPlaylistPrivacy, - VideoPrivacy -} from '@shared/models' -import { logger, loggerTagsFactory } from '../../../helpers/logger' -import { VideoCommentModel } from '../../../models/video/video-comment' -import { - MActorLight, - MCommentOwnerVideo, - MLocalVideoViewerWithWatchSections, - MVideoAccountLight, - MVideoAP, - MVideoPlaylistFull, - MVideoRedundancyFileVideo, - MVideoRedundancyStreamingPlaylistVideo -} from '../../../types/models' -import { audiencify, getAudience } from '../audience' -import { - broadcastToActors, - broadcastToFollowers, - getActorsInvolvedInVideo, - getAudienceFromFollowersOf, - getVideoCommentAudience, - sendVideoActivityToOrigin, - sendVideoRelatedActivity, - unicastTo -} from './shared' - -const lTags = loggerTagsFactory('ap', 'create') - -async function sendCreateVideo (video: MVideoAP, transaction: Transaction) { - if (!video.hasPrivacyForFederation()) return undefined - - logger.info('Creating job to send video creation of %s.', video.url, lTags(video.uuid)) - - const byActor = video.VideoChannel.Account.Actor - const videoObject = await video.toActivityPubObject() - - const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC) - const createActivity = buildCreateActivity(video.url, byActor, videoObject, audience) - - return broadcastToFollowers({ - data: createActivity, - byActor, - toFollowersOf: [ byActor ], - transaction, - contextType: 'Video' - }) -} - -async function sendCreateCacheFile ( - byActor: MActorLight, - video: MVideoAccountLight, - fileRedundancy: MVideoRedundancyStreamingPlaylistVideo | MVideoRedundancyFileVideo -) { - logger.info('Creating job to send file cache of %s.', fileRedundancy.url, lTags(video.uuid)) - - return sendVideoRelatedCreateActivity({ - byActor, - video, - url: fileRedundancy.url, - object: fileRedundancy.toActivityPubObject(), - contextType: 'CacheFile' - }) -} - -async function sendCreateWatchAction (stats: MLocalVideoViewerWithWatchSections, transaction: Transaction) { - logger.info('Creating job to send create watch action %s.', stats.url, lTags(stats.uuid)) - - const byActor = await getServerActor() - - const activityBuilder = (audience: ActivityAudience) => { - return buildCreateActivity(stats.url, byActor, stats.toActivityPubObject(), audience) - } - - return sendVideoActivityToOrigin(activityBuilder, { byActor, video: stats.Video, transaction, contextType: 'WatchAction' }) -} - -async function sendCreateVideoPlaylist (playlist: MVideoPlaylistFull, transaction: Transaction) { - if (playlist.privacy === VideoPlaylistPrivacy.PRIVATE) return undefined - - logger.info('Creating job to send create video playlist of %s.', playlist.url, lTags(playlist.uuid)) - - const byActor = playlist.OwnerAccount.Actor - const audience = getAudience(byActor, playlist.privacy === VideoPlaylistPrivacy.PUBLIC) - - const object = await playlist.toActivityPubObject(null, transaction) - const createActivity = buildCreateActivity(playlist.url, byActor, object, audience) - - const serverActor = await getServerActor() - const toFollowersOf = [ byActor, serverActor ] - - if (playlist.VideoChannel) toFollowersOf.push(playlist.VideoChannel.Actor) - - return broadcastToFollowers({ - data: createActivity, - byActor, - toFollowersOf, - transaction, - contextType: 'Playlist' - }) -} - -async function sendCreateVideoComment (comment: MCommentOwnerVideo, transaction: Transaction) { - logger.info('Creating job to send comment %s.', comment.url) - - const isOrigin = comment.Video.isOwned() - - const byActor = comment.Account.Actor - const threadParentComments = await VideoCommentModel.listThreadParentComments(comment, transaction) - const commentObject = comment.toActivityPubObject(threadParentComments) as VideoCommentObject - - const actorsInvolvedInComment = await getActorsInvolvedInVideo(comment.Video, transaction) - // Add the actor that commented too - actorsInvolvedInComment.push(byActor) - - const parentsCommentActors = threadParentComments.filter(c => !c.isDeleted()) - .map(c => c.Account.Actor) - - let audience: ActivityAudience - if (isOrigin) { - audience = getVideoCommentAudience(comment, threadParentComments, actorsInvolvedInComment, isOrigin) - } else { - audience = getAudienceFromFollowersOf(actorsInvolvedInComment.concat(parentsCommentActors)) - } - - const createActivity = buildCreateActivity(comment.url, byActor, commentObject, audience) - - // This was a reply, send it to the parent actors - const actorsException = [ byActor ] - await broadcastToActors({ - data: createActivity, - byActor, - toActors: parentsCommentActors, - transaction, - actorsException, - contextType: 'Comment' - }) - - // Broadcast to our followers - await broadcastToFollowers({ - data: createActivity, - byActor, - toFollowersOf: [ byActor ], - transaction, - contextType: 'Comment' - }) - - // Send to actors involved in the comment - if (isOrigin) { - return broadcastToFollowers({ - data: createActivity, - byActor, - toFollowersOf: actorsInvolvedInComment, - transaction, - actorsException, - contextType: 'Comment' - }) - } - - // Send to origin - return transaction.afterCommit(() => { - return unicastTo({ - data: createActivity, - byActor, - toActorUrl: comment.Video.VideoChannel.Account.Actor.getSharedInbox(), - contextType: 'Comment' - }) - }) -} - -function buildCreateActivity ( - url: string, - byActor: MActorLight, - object: T, - audience?: ActivityAudience -): ActivityCreate { - if (!audience) audience = getAudience(byActor) - - return audiencify( - { - type: 'Create' as 'Create', - id: url + '/activity', - actor: byActor.url, - object: typeof object === 'string' - ? object - : audiencify(object, audience) - }, - audience - ) -} - -// --------------------------------------------------------------------------- - -export { - sendCreateVideo, - buildCreateActivity, - sendCreateVideoComment, - sendCreateVideoPlaylist, - sendCreateCacheFile, - sendCreateWatchAction -} - -// --------------------------------------------------------------------------- - -async function sendVideoRelatedCreateActivity (options: { - byActor: MActorLight - video: MVideoAccountLight - url: string - object: any - contextType: ContextType - transaction?: Transaction -}) { - const activityBuilder = (audience: ActivityAudience) => { - return buildCreateActivity(options.url, options.byActor, options.object, audience) - } - - return sendVideoRelatedActivity(activityBuilder, options) -} diff --git a/server/lib/activitypub/send/send-delete.ts b/server/lib/activitypub/send/send-delete.ts deleted file mode 100644 index 0d85d9001..000000000 --- a/server/lib/activitypub/send/send-delete.ts +++ /dev/null @@ -1,158 +0,0 @@ -import { Transaction } from 'sequelize' -import { getServerActor } from '@server/models/application/application' -import { ActivityAudience, ActivityDelete } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { ActorModel } from '../../../models/actor/actor' -import { VideoCommentModel } from '../../../models/video/video-comment' -import { VideoShareModel } from '../../../models/video/video-share' -import { MActorUrl } from '../../../types/models' -import { MCommentOwnerVideo, MVideoAccountLight, MVideoPlaylistFullSummary } from '../../../types/models/video' -import { audiencify } from '../audience' -import { getDeleteActivityPubUrl } from '../url' -import { getActorsInvolvedInVideo, getVideoCommentAudience } from './shared' -import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './shared/send-utils' - -async function sendDeleteVideo (video: MVideoAccountLight, transaction: Transaction) { - logger.info('Creating job to broadcast delete of video %s.', video.url) - - const byActor = video.VideoChannel.Account.Actor - - const activityBuilder = (audience: ActivityAudience) => { - const url = getDeleteActivityPubUrl(video.url) - - return buildDeleteActivity(url, video.url, byActor, audience) - } - - return sendVideoRelatedActivity(activityBuilder, { byActor, video, contextType: 'Delete', transaction }) -} - -async function sendDeleteActor (byActor: ActorModel, transaction: Transaction) { - logger.info('Creating job to broadcast delete of actor %s.', byActor.url) - - const url = getDeleteActivityPubUrl(byActor.url) - const activity = buildDeleteActivity(url, byActor.url, byActor) - - const actorsInvolved = await VideoShareModel.loadActorsWhoSharedVideosOf(byActor.id, transaction) - - // In case the actor did not have any videos - const serverActor = await getServerActor() - actorsInvolved.push(serverActor) - - actorsInvolved.push(byActor) - - return broadcastToFollowers({ - data: activity, - byActor, - toFollowersOf: actorsInvolved, - contextType: 'Delete', - transaction - }) -} - -async function sendDeleteVideoComment (videoComment: MCommentOwnerVideo, transaction: Transaction) { - logger.info('Creating job to send delete of comment %s.', videoComment.url) - - const isVideoOrigin = videoComment.Video.isOwned() - - const url = getDeleteActivityPubUrl(videoComment.url) - const byActor = videoComment.isOwned() - ? videoComment.Account.Actor - : videoComment.Video.VideoChannel.Account.Actor - - const threadParentComments = await VideoCommentModel.listThreadParentComments(videoComment, transaction) - const threadParentCommentsFiltered = threadParentComments.filter(c => !c.isDeleted()) - - const actorsInvolvedInComment = await getActorsInvolvedInVideo(videoComment.Video, transaction) - actorsInvolvedInComment.push(byActor) // Add the actor that commented the video - - const audience = getVideoCommentAudience(videoComment, threadParentCommentsFiltered, actorsInvolvedInComment, isVideoOrigin) - const activity = buildDeleteActivity(url, videoComment.url, byActor, audience) - - // This was a reply, send it to the parent actors - const actorsException = [ byActor ] - await broadcastToActors({ - data: activity, - byActor, - toActors: threadParentCommentsFiltered.map(c => c.Account.Actor), - transaction, - contextType: 'Delete', - actorsException - }) - - // Broadcast to our followers - await broadcastToFollowers({ - data: activity, - byActor, - toFollowersOf: [ byActor ], - contextType: 'Delete', - transaction - }) - - // Send to actors involved in the comment - if (isVideoOrigin) { - return broadcastToFollowers({ - data: activity, - byActor, - toFollowersOf: actorsInvolvedInComment, - transaction, - contextType: 'Delete', - actorsException - }) - } - - // Send to origin - return transaction.afterCommit(() => { - return unicastTo({ - data: activity, - byActor, - toActorUrl: videoComment.Video.VideoChannel.Account.Actor.getSharedInbox(), - contextType: 'Delete' - }) - }) -} - -async function sendDeleteVideoPlaylist (videoPlaylist: MVideoPlaylistFullSummary, transaction: Transaction) { - logger.info('Creating job to send delete of playlist %s.', videoPlaylist.url) - - const byActor = videoPlaylist.OwnerAccount.Actor - - const url = getDeleteActivityPubUrl(videoPlaylist.url) - const activity = buildDeleteActivity(url, videoPlaylist.url, byActor) - - const serverActor = await getServerActor() - const toFollowersOf = [ byActor, serverActor ] - - if (videoPlaylist.VideoChannel) toFollowersOf.push(videoPlaylist.VideoChannel.Actor) - - return broadcastToFollowers({ - data: activity, - byActor, - toFollowersOf, - contextType: 'Delete', - transaction - }) -} - -// --------------------------------------------------------------------------- - -export { - sendDeleteVideo, - sendDeleteActor, - sendDeleteVideoComment, - sendDeleteVideoPlaylist -} - -// --------------------------------------------------------------------------- - -function buildDeleteActivity (url: string, object: string, byActor: MActorUrl, audience?: ActivityAudience): ActivityDelete { - const activity = { - type: 'Delete' as 'Delete', - id: url, - actor: byActor.url, - object - } - - if (audience) return audiencify(activity, audience) - - return activity -} diff --git a/server/lib/activitypub/send/send-dislike.ts b/server/lib/activitypub/send/send-dislike.ts deleted file mode 100644 index 959e74823..000000000 --- a/server/lib/activitypub/send/send-dislike.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { Transaction } from 'sequelize' -import { ActivityAudience, ActivityDislike } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { MActor, MActorAudience, MVideoAccountLight, MVideoUrl } from '../../../types/models' -import { audiencify, getAudience } from '../audience' -import { getVideoDislikeActivityPubUrlByLocalActor } from '../url' -import { sendVideoActivityToOrigin } from './shared/send-utils' - -function sendDislike (byActor: MActor, video: MVideoAccountLight, transaction: Transaction) { - logger.info('Creating job to dislike %s.', video.url) - - const activityBuilder = (audience: ActivityAudience) => { - const url = getVideoDislikeActivityPubUrlByLocalActor(byActor, video) - - return buildDislikeActivity(url, byActor, video, audience) - } - - return sendVideoActivityToOrigin(activityBuilder, { byActor, video, transaction, contextType: 'Rate' }) -} - -function buildDislikeActivity (url: string, byActor: MActorAudience, video: MVideoUrl, audience?: ActivityAudience): ActivityDislike { - if (!audience) audience = getAudience(byActor) - - return audiencify( - { - id: url, - type: 'Dislike' as 'Dislike', - actor: byActor.url, - object: video.url - }, - audience - ) -} - -// --------------------------------------------------------------------------- - -export { - sendDislike, - buildDislikeActivity -} diff --git a/server/lib/activitypub/send/send-flag.ts b/server/lib/activitypub/send/send-flag.ts deleted file mode 100644 index 138eb5adc..000000000 --- a/server/lib/activitypub/send/send-flag.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { Transaction } from 'sequelize' -import { ActivityAudience, ActivityFlag } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { MAbuseAP, MAccountLight, MActor } from '../../../types/models' -import { audiencify, getAudience } from '../audience' -import { getLocalAbuseActivityPubUrl } from '../url' -import { unicastTo } from './shared/send-utils' - -function sendAbuse (byActor: MActor, abuse: MAbuseAP, flaggedAccount: MAccountLight, t: Transaction) { - if (!flaggedAccount.Actor.serverId) return // Local user - - const url = getLocalAbuseActivityPubUrl(abuse) - - logger.info('Creating job to send abuse %s.', url) - - // Custom audience, we only send the abuse to the origin instance - const audience = { to: [ flaggedAccount.Actor.url ], cc: [] } - const flagActivity = buildFlagActivity(url, byActor, abuse, audience) - - return t.afterCommit(() => { - return unicastTo({ - data: flagActivity, - byActor, - toActorUrl: flaggedAccount.Actor.getSharedInbox(), - contextType: 'Flag' - }) - }) -} - -function buildFlagActivity (url: string, byActor: MActor, abuse: MAbuseAP, audience: ActivityAudience): ActivityFlag { - if (!audience) audience = getAudience(byActor) - - const activity = { id: url, actor: byActor.url, ...abuse.toActivityPubObject() } - - return audiencify(activity, audience) -} - -// --------------------------------------------------------------------------- - -export { - sendAbuse -} diff --git a/server/lib/activitypub/send/send-follow.ts b/server/lib/activitypub/send/send-follow.ts deleted file mode 100644 index 57501dadb..000000000 --- a/server/lib/activitypub/send/send-follow.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { Transaction } from 'sequelize' -import { ActivityFollow } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { MActor, MActorFollowActors } from '../../../types/models' -import { unicastTo } from './shared/send-utils' - -function sendFollow (actorFollow: MActorFollowActors, t: Transaction) { - const me = actorFollow.ActorFollower - const following = actorFollow.ActorFollowing - - // Same server as ours - if (!following.serverId) return - - logger.info('Creating job to send follow request to %s.', following.url) - - const data = buildFollowActivity(actorFollow.url, me, following) - - return t.afterCommit(() => { - return unicastTo({ data, byActor: me, toActorUrl: following.inboxUrl, contextType: 'Follow' }) - }) -} - -function buildFollowActivity (url: string, byActor: MActor, targetActor: MActor): ActivityFollow { - return { - type: 'Follow', - id: url, - actor: byActor.url, - object: targetActor.url - } -} - -// --------------------------------------------------------------------------- - -export { - sendFollow, - buildFollowActivity -} diff --git a/server/lib/activitypub/send/send-like.ts b/server/lib/activitypub/send/send-like.ts deleted file mode 100644 index 46c9fdec9..000000000 --- a/server/lib/activitypub/send/send-like.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { Transaction } from 'sequelize' -import { ActivityAudience, ActivityLike } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { MActor, MActorAudience, MVideoAccountLight, MVideoUrl } from '../../../types/models' -import { audiencify, getAudience } from '../audience' -import { getVideoLikeActivityPubUrlByLocalActor } from '../url' -import { sendVideoActivityToOrigin } from './shared/send-utils' - -function sendLike (byActor: MActor, video: MVideoAccountLight, transaction: Transaction) { - logger.info('Creating job to like %s.', video.url) - - const activityBuilder = (audience: ActivityAudience) => { - const url = getVideoLikeActivityPubUrlByLocalActor(byActor, video) - - return buildLikeActivity(url, byActor, video, audience) - } - - return sendVideoActivityToOrigin(activityBuilder, { byActor, video, transaction, contextType: 'Rate' }) -} - -function buildLikeActivity (url: string, byActor: MActorAudience, video: MVideoUrl, audience?: ActivityAudience): ActivityLike { - if (!audience) audience = getAudience(byActor) - - return audiencify( - { - id: url, - type: 'Like' as 'Like', - actor: byActor.url, - object: video.url - }, - audience - ) -} - -// --------------------------------------------------------------------------- - -export { - sendLike, - buildLikeActivity -} diff --git a/server/lib/activitypub/send/send-reject.ts b/server/lib/activitypub/send/send-reject.ts deleted file mode 100644 index a5f8c2ecf..000000000 --- a/server/lib/activitypub/send/send-reject.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { ActivityFollow, ActivityReject } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { MActor } from '../../../types/models' -import { getLocalActorFollowRejectActivityPubUrl } from '../url' -import { buildFollowActivity } from './send-follow' -import { unicastTo } from './shared/send-utils' - -function sendReject (followUrl: string, follower: MActor, following: MActor) { - if (!follower.serverId) { // This should never happen - logger.warn('Do not sending reject to local follower.') - return - } - - logger.info('Creating job to reject follower %s.', follower.url) - - const followData = buildFollowActivity(followUrl, follower, following) - - const url = getLocalActorFollowRejectActivityPubUrl() - const data = buildRejectActivity(url, following, followData) - - return unicastTo({ data, byActor: following, toActorUrl: follower.inboxUrl, contextType: 'Reject' }) -} - -// --------------------------------------------------------------------------- - -export { - sendReject -} - -// --------------------------------------------------------------------------- - -function buildRejectActivity (url: string, byActor: MActor, followActivityData: ActivityFollow): ActivityReject { - return { - type: 'Reject', - id: url, - actor: byActor.url, - object: followActivityData - } -} diff --git a/server/lib/activitypub/send/send-undo.ts b/server/lib/activitypub/send/send-undo.ts deleted file mode 100644 index b0b48c9c4..000000000 --- a/server/lib/activitypub/send/send-undo.ts +++ /dev/null @@ -1,172 +0,0 @@ -import { Transaction } from 'sequelize' -import { ActivityAudience, ActivityDislike, ActivityLike, ActivityUndo, ActivityUndoObject, ContextType } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { VideoModel } from '../../../models/video/video' -import { - MActor, - MActorAudience, - MActorFollowActors, - MActorLight, - MVideo, - MVideoAccountLight, - MVideoRedundancyVideo, - MVideoShare -} from '../../../types/models' -import { audiencify, getAudience } from '../audience' -import { getUndoActivityPubUrl, getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from '../url' -import { buildAnnounceWithVideoAudience } from './send-announce' -import { buildCreateActivity } from './send-create' -import { buildDislikeActivity } from './send-dislike' -import { buildFollowActivity } from './send-follow' -import { buildLikeActivity } from './send-like' -import { broadcastToFollowers, sendVideoActivityToOrigin, sendVideoRelatedActivity, unicastTo } from './shared/send-utils' - -function sendUndoFollow (actorFollow: MActorFollowActors, t: Transaction) { - const me = actorFollow.ActorFollower - const following = actorFollow.ActorFollowing - - // Same server as ours - if (!following.serverId) return - - logger.info('Creating job to send an unfollow request to %s.', following.url) - - const undoUrl = getUndoActivityPubUrl(actorFollow.url) - - const followActivity = buildFollowActivity(actorFollow.url, me, following) - const undoActivity = undoActivityData(undoUrl, me, followActivity) - - t.afterCommit(() => { - return unicastTo({ - data: undoActivity, - byActor: me, - toActorUrl: following.inboxUrl, - contextType: 'Follow' - }) - }) -} - -// --------------------------------------------------------------------------- - -async function sendUndoAnnounce (byActor: MActorLight, videoShare: MVideoShare, video: MVideo, transaction: Transaction) { - logger.info('Creating job to undo announce %s.', videoShare.url) - - const undoUrl = getUndoActivityPubUrl(videoShare.url) - - const { activity: announce, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, transaction) - const undoActivity = undoActivityData(undoUrl, byActor, announce) - - return broadcastToFollowers({ - data: undoActivity, - byActor, - toFollowersOf: actorsInvolvedInVideo, - transaction, - actorsException: [ byActor ], - contextType: 'Announce' - }) -} - -async function sendUndoCacheFile (byActor: MActor, redundancyModel: MVideoRedundancyVideo, transaction: Transaction) { - logger.info('Creating job to undo cache file %s.', redundancyModel.url) - - const associatedVideo = redundancyModel.getVideo() - if (!associatedVideo) { - logger.warn('Cannot send undo activity for redundancy %s: no video files associated.', redundancyModel.url) - return - } - - const video = await VideoModel.loadFull(associatedVideo.id) - const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) - - return sendUndoVideoRelatedActivity({ - byActor, - video, - url: redundancyModel.url, - activity: createActivity, - contextType: 'CacheFile', - transaction - }) -} - -// --------------------------------------------------------------------------- - -async function sendUndoLike (byActor: MActor, video: MVideoAccountLight, t: Transaction) { - logger.info('Creating job to undo a like of video %s.', video.url) - - const likeUrl = getVideoLikeActivityPubUrlByLocalActor(byActor, video) - const likeActivity = buildLikeActivity(likeUrl, byActor, video) - - return sendUndoVideoRateToOriginActivity({ byActor, video, url: likeUrl, activity: likeActivity, transaction: t }) -} - -async function sendUndoDislike (byActor: MActor, video: MVideoAccountLight, t: Transaction) { - logger.info('Creating job to undo a dislike of video %s.', video.url) - - const dislikeUrl = getVideoDislikeActivityPubUrlByLocalActor(byActor, video) - const dislikeActivity = buildDislikeActivity(dislikeUrl, byActor, video) - - return sendUndoVideoRateToOriginActivity({ byActor, video, url: dislikeUrl, activity: dislikeActivity, transaction: t }) -} - -// --------------------------------------------------------------------------- - -export { - sendUndoFollow, - sendUndoLike, - sendUndoDislike, - sendUndoAnnounce, - sendUndoCacheFile -} - -// --------------------------------------------------------------------------- - -function undoActivityData ( - url: string, - byActor: MActorAudience, - object: T, - audience?: ActivityAudience -): ActivityUndo { - if (!audience) audience = getAudience(byActor) - - return audiencify( - { - type: 'Undo' as 'Undo', - id: url, - actor: byActor.url, - object - }, - audience - ) -} - -async function sendUndoVideoRelatedActivity (options: { - byActor: MActor - video: MVideoAccountLight - url: string - activity: ActivityUndoObject - contextType: ContextType - transaction: Transaction -}) { - const activityBuilder = (audience: ActivityAudience) => { - const undoUrl = getUndoActivityPubUrl(options.url) - - return undoActivityData(undoUrl, options.byActor, options.activity, audience) - } - - return sendVideoRelatedActivity(activityBuilder, options) -} - -async function sendUndoVideoRateToOriginActivity (options: { - byActor: MActor - video: MVideoAccountLight - url: string - activity: ActivityLike | ActivityDislike - transaction: Transaction -}) { - const activityBuilder = (audience: ActivityAudience) => { - const undoUrl = getUndoActivityPubUrl(options.url) - - return undoActivityData(undoUrl, options.byActor, options.activity, audience) - } - - return sendVideoActivityToOrigin(activityBuilder, { ...options, contextType: 'Rate' }) -} diff --git a/server/lib/activitypub/send/send-update.ts b/server/lib/activitypub/send/send-update.ts deleted file mode 100644 index f3fb741c6..000000000 --- a/server/lib/activitypub/send/send-update.ts +++ /dev/null @@ -1,157 +0,0 @@ -import { Transaction } from 'sequelize' -import { getServerActor } from '@server/models/application/application' -import { ActivityAudience, ActivityUpdate, ActivityUpdateObject, VideoPlaylistPrivacy, VideoPrivacy } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { AccountModel } from '../../../models/account/account' -import { VideoModel } from '../../../models/video/video' -import { VideoShareModel } from '../../../models/video/video-share' -import { - MAccountDefault, - MActor, - MActorLight, - MChannelDefault, - MVideoAPLight, - MVideoPlaylistFull, - MVideoRedundancyVideo -} from '../../../types/models' -import { audiencify, getAudience } from '../audience' -import { getUpdateActivityPubUrl } from '../url' -import { getActorsInvolvedInVideo } from './shared' -import { broadcastToFollowers, sendVideoRelatedActivity } from './shared/send-utils' - -async function sendUpdateVideo (videoArg: MVideoAPLight, transaction: Transaction, overriddenByActor?: MActor) { - if (!videoArg.hasPrivacyForFederation()) return undefined - - const video = await videoArg.lightAPToFullAP(transaction) - - logger.info('Creating job to update video %s.', video.url) - - const byActor = overriddenByActor || video.VideoChannel.Account.Actor - - const url = getUpdateActivityPubUrl(video.url, video.updatedAt.toISOString()) - - const videoObject = await video.toActivityPubObject() - const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC) - - const updateActivity = buildUpdateActivity(url, byActor, videoObject, audience) - - const actorsInvolved = await getActorsInvolvedInVideo(video, transaction) - if (overriddenByActor) actorsInvolved.push(overriddenByActor) - - return broadcastToFollowers({ - data: updateActivity, - byActor, - toFollowersOf: actorsInvolved, - contextType: 'Video', - transaction - }) -} - -async function sendUpdateActor (accountOrChannel: MChannelDefault | MAccountDefault, transaction: Transaction) { - const byActor = accountOrChannel.Actor - - logger.info('Creating job to update actor %s.', byActor.url) - - const url = getUpdateActivityPubUrl(byActor.url, byActor.updatedAt.toISOString()) - const accountOrChannelObject = await (accountOrChannel as any).toActivityPubObject() // FIXME: typescript bug? - const audience = getAudience(byActor) - const updateActivity = buildUpdateActivity(url, byActor, accountOrChannelObject, audience) - - let actorsInvolved: MActor[] - if (accountOrChannel instanceof AccountModel) { - // Actors that shared my videos are involved too - actorsInvolved = await VideoShareModel.loadActorsWhoSharedVideosOf(byActor.id, transaction) - } else { - // Actors that shared videos of my channel are involved too - actorsInvolved = await VideoShareModel.loadActorsByVideoChannel(accountOrChannel.id, transaction) - } - - actorsInvolved.push(byActor) - - return broadcastToFollowers({ - data: updateActivity, - byActor, - toFollowersOf: actorsInvolved, - transaction, - contextType: 'Actor' - }) -} - -async function sendUpdateCacheFile (byActor: MActorLight, redundancyModel: MVideoRedundancyVideo) { - logger.info('Creating job to update cache file %s.', redundancyModel.url) - - const associatedVideo = redundancyModel.getVideo() - if (!associatedVideo) { - logger.warn('Cannot send update activity for redundancy %s: no video files associated.', redundancyModel.url) - return - } - - const video = await VideoModel.loadFull(associatedVideo.id) - - const activityBuilder = (audience: ActivityAudience) => { - const redundancyObject = redundancyModel.toActivityPubObject() - const url = getUpdateActivityPubUrl(redundancyModel.url, redundancyModel.updatedAt.toISOString()) - - return buildUpdateActivity(url, byActor, redundancyObject, audience) - } - - return sendVideoRelatedActivity(activityBuilder, { byActor, video, contextType: 'CacheFile' }) -} - -async function sendUpdateVideoPlaylist (videoPlaylist: MVideoPlaylistFull, transaction: Transaction) { - if (videoPlaylist.privacy === VideoPlaylistPrivacy.PRIVATE) return undefined - - const byActor = videoPlaylist.OwnerAccount.Actor - - logger.info('Creating job to update video playlist %s.', videoPlaylist.url) - - const url = getUpdateActivityPubUrl(videoPlaylist.url, videoPlaylist.updatedAt.toISOString()) - - const object = await videoPlaylist.toActivityPubObject(null, transaction) - const audience = getAudience(byActor, videoPlaylist.privacy === VideoPlaylistPrivacy.PUBLIC) - - const updateActivity = buildUpdateActivity(url, byActor, object, audience) - - const serverActor = await getServerActor() - const toFollowersOf = [ byActor, serverActor ] - - if (videoPlaylist.VideoChannel) toFollowersOf.push(videoPlaylist.VideoChannel.Actor) - - return broadcastToFollowers({ - data: updateActivity, - byActor, - toFollowersOf, - transaction, - contextType: 'Playlist' - }) -} - -// --------------------------------------------------------------------------- - -export { - sendUpdateActor, - sendUpdateVideo, - sendUpdateCacheFile, - sendUpdateVideoPlaylist -} - -// --------------------------------------------------------------------------- - -function buildUpdateActivity ( - url: string, - byActor: MActorLight, - object: ActivityUpdateObject, - audience?: ActivityAudience -): ActivityUpdate { - if (!audience) audience = getAudience(byActor) - - return audiencify( - { - type: 'Update' as 'Update', - id: url, - actor: byActor.url, - object: audiencify(object, audience) - }, - audience - ) -} diff --git a/server/lib/activitypub/send/send-view.ts b/server/lib/activitypub/send/send-view.ts deleted file mode 100644 index bf3451603..000000000 --- a/server/lib/activitypub/send/send-view.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { Transaction } from 'sequelize' -import { VideoViewsManager } from '@server/lib/views/video-views-manager' -import { MActorAudience, MActorLight, MVideoImmutable, MVideoUrl } from '@server/types/models' -import { ActivityAudience, ActivityView } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { audiencify, getAudience } from '../audience' -import { getLocalVideoViewActivityPubUrl } from '../url' -import { sendVideoRelatedActivity } from './shared/send-utils' - -type ViewType = 'view' | 'viewer' - -async function sendView (options: { - byActor: MActorLight - type: ViewType - video: MVideoImmutable - viewerIdentifier: string - transaction?: Transaction -}) { - const { byActor, type, video, viewerIdentifier, transaction } = options - - logger.info('Creating job to send %s of %s.', type, video.url) - - const activityBuilder = (audience: ActivityAudience) => { - const url = getLocalVideoViewActivityPubUrl(byActor, video, viewerIdentifier) - - return buildViewActivity({ url, byActor, video, audience, type }) - } - - return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction, contextType: 'View', parallelizable: true }) -} - -// --------------------------------------------------------------------------- - -export { - sendView -} - -// --------------------------------------------------------------------------- - -function buildViewActivity (options: { - url: string - byActor: MActorAudience - video: MVideoUrl - type: ViewType - audience?: ActivityAudience -}): ActivityView { - const { url, byActor, type, video, audience = getAudience(byActor) } = options - - return audiencify( - { - id: url, - type: 'View' as 'View', - actor: byActor.url, - object: video.url, - - expires: type === 'viewer' - ? new Date(VideoViewsManager.Instance.buildViewerExpireTime()).toISOString() - : undefined - }, - audience - ) -} diff --git a/server/lib/activitypub/send/shared/audience-utils.ts b/server/lib/activitypub/send/shared/audience-utils.ts deleted file mode 100644 index 2f6b0741d..000000000 --- a/server/lib/activitypub/send/shared/audience-utils.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { Transaction } from 'sequelize' -import { ACTIVITY_PUB } from '@server/initializers/constants' -import { ActorModel } from '@server/models/actor/actor' -import { VideoModel } from '@server/models/video/video' -import { VideoShareModel } from '@server/models/video/video-share' -import { MActorFollowersUrl, MActorUrl, MCommentOwner, MCommentOwnerVideo, MVideoId } from '@server/types/models' -import { ActivityAudience } from '@shared/models' - -function getOriginVideoAudience (accountActor: MActorUrl, actorsInvolvedInVideo: MActorFollowersUrl[] = []): ActivityAudience { - return { - to: [ accountActor.url ], - cc: actorsInvolvedInVideo.map(a => a.followersUrl) - } -} - -function getVideoCommentAudience ( - videoComment: MCommentOwnerVideo, - threadParentComments: MCommentOwner[], - actorsInvolvedInVideo: MActorFollowersUrl[], - isOrigin = false -): ActivityAudience { - const to = [ ACTIVITY_PUB.PUBLIC ] - const cc: string[] = [] - - // Owner of the video we comment - if (isOrigin === false) { - cc.push(videoComment.Video.VideoChannel.Account.Actor.url) - } - - // Followers of the poster - cc.push(videoComment.Account.Actor.followersUrl) - - // Send to actors we reply to - for (const parentComment of threadParentComments) { - if (parentComment.isDeleted()) continue - - cc.push(parentComment.Account.Actor.url) - } - - return { - to, - cc: cc.concat(actorsInvolvedInVideo.map(a => a.followersUrl)) - } -} - -function getAudienceFromFollowersOf (actorsInvolvedInObject: MActorFollowersUrl[]): ActivityAudience { - return { - to: [ ACTIVITY_PUB.PUBLIC ].concat(actorsInvolvedInObject.map(a => a.followersUrl)), - cc: [] - } -} - -async function getActorsInvolvedInVideo (video: MVideoId, t: Transaction) { - const actors = await VideoShareModel.listActorIdsAndFollowerUrlsByShare(video.id, t) - - const videoAll = video as VideoModel - - const videoActor = videoAll.VideoChannel?.Account - ? videoAll.VideoChannel.Account.Actor - : await ActorModel.loadAccountActorFollowerUrlByVideoId(video.id, t) - - actors.push(videoActor) - - return actors -} - -// --------------------------------------------------------------------------- - -export { - getOriginVideoAudience, - getActorsInvolvedInVideo, - getAudienceFromFollowersOf, - getVideoCommentAudience -} diff --git a/server/lib/activitypub/send/shared/index.ts b/server/lib/activitypub/send/shared/index.ts deleted file mode 100644 index bda579115..000000000 --- a/server/lib/activitypub/send/shared/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './audience-utils' -export * from './send-utils' diff --git a/server/lib/activitypub/send/shared/send-utils.ts b/server/lib/activitypub/send/shared/send-utils.ts deleted file mode 100644 index 2bc1ef8f5..000000000 --- a/server/lib/activitypub/send/shared/send-utils.ts +++ /dev/null @@ -1,291 +0,0 @@ -import { Transaction } from 'sequelize' -import { ActorFollowHealthCache } from '@server/lib/actor-follow-health-cache' -import { getServerActor } from '@server/models/application/application' -import { Activity, ActivityAudience, ActivitypubHttpBroadcastPayload } from '@shared/models' -import { ContextType } from '@shared/models/activitypub/context' -import { afterCommitIfTransaction } from '../../../../helpers/database-utils' -import { logger } from '../../../../helpers/logger' -import { ActorModel } from '../../../../models/actor/actor' -import { ActorFollowModel } from '../../../../models/actor/actor-follow' -import { MActor, MActorId, MActorLight, MActorWithInboxes, MVideoAccountLight, MVideoId, MVideoImmutable } from '../../../../types/models' -import { JobQueue } from '../../../job-queue' -import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getOriginVideoAudience } from './audience-utils' - -async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: { - byActor: MActorLight - video: MVideoImmutable | MVideoAccountLight - contextType: ContextType - parallelizable?: boolean - transaction?: Transaction -}) { - const { byActor, video, transaction, contextType, parallelizable } = options - - // Send to origin - if (video.isOwned() === false) { - return sendVideoActivityToOrigin(activityBuilder, options) - } - - const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, transaction) - - // Send to followers - const audience = getAudienceFromFollowersOf(actorsInvolvedInVideo) - const activity = activityBuilder(audience) - - const actorsException = [ byActor ] - - return broadcastToFollowers({ - data: activity, - byActor, - toFollowersOf: actorsInvolvedInVideo, - transaction, - actorsException, - parallelizable, - contextType - }) -} - -async function sendVideoActivityToOrigin (activityBuilder: (audience: ActivityAudience) => Activity, options: { - byActor: MActorLight - video: MVideoImmutable | MVideoAccountLight - contextType: ContextType - - actorsInvolvedInVideo?: MActorLight[] - transaction?: Transaction -}) { - const { byActor, video, actorsInvolvedInVideo, transaction, contextType } = options - - if (video.isOwned()) throw new Error('Cannot send activity to owned video origin ' + video.url) - - let accountActor: MActorLight = (video as MVideoAccountLight).VideoChannel?.Account?.Actor - if (!accountActor) accountActor = await ActorModel.loadAccountActorByVideoId(video.id, transaction) - - const audience = getOriginVideoAudience(accountActor, actorsInvolvedInVideo) - const activity = activityBuilder(audience) - - return afterCommitIfTransaction(transaction, () => { - return unicastTo({ - data: activity, - byActor, - toActorUrl: accountActor.getSharedInbox(), - contextType - }) - }) -} - -// --------------------------------------------------------------------------- - -async function forwardVideoRelatedActivity ( - activity: Activity, - t: Transaction, - followersException: MActorWithInboxes[], - video: MVideoId -) { - // Mastodon does not add our announces in audience, so we forward to them manually - const additionalActors = await getActorsInvolvedInVideo(video, t) - const additionalFollowerUrls = additionalActors.map(a => a.followersUrl) - - return forwardActivity(activity, t, followersException, additionalFollowerUrls) -} - -async function forwardActivity ( - activity: Activity, - t: Transaction, - followersException: MActorWithInboxes[] = [], - additionalFollowerUrls: string[] = [] -) { - logger.info('Forwarding activity %s.', activity.id) - - const to = activity.to || [] - const cc = activity.cc || [] - - const followersUrls = additionalFollowerUrls - for (const dest of to.concat(cc)) { - if (dest.endsWith('/followers')) { - followersUrls.push(dest) - } - } - - const toActorFollowers = await ActorModel.listByFollowersUrls(followersUrls, t) - const uris = await computeFollowerUris(toActorFollowers, followersException, t) - - if (uris.length === 0) { - logger.info('0 followers for %s, no forwarding.', toActorFollowers.map(a => a.id).join(', ')) - return undefined - } - - logger.debug('Creating forwarding job.', { uris }) - - const payload: ActivitypubHttpBroadcastPayload = { - uris, - body: activity, - contextType: null - } - return afterCommitIfTransaction(t, () => JobQueue.Instance.createJobAsync({ type: 'activitypub-http-broadcast', payload })) -} - -// --------------------------------------------------------------------------- - -async function broadcastToFollowers (options: { - data: any - byActor: MActorId - toFollowersOf: MActorId[] - transaction: Transaction - contextType: ContextType - - parallelizable?: boolean - actorsException?: MActorWithInboxes[] -}) { - const { data, byActor, toFollowersOf, transaction, contextType, actorsException = [], parallelizable } = options - - const uris = await computeFollowerUris(toFollowersOf, actorsException, transaction) - - return afterCommitIfTransaction(transaction, () => { - return broadcastTo({ - uris, - data, - byActor, - parallelizable, - contextType - }) - }) -} - -async function broadcastToActors (options: { - data: any - byActor: MActorId - toActors: MActor[] - transaction: Transaction - contextType: ContextType - actorsException?: MActorWithInboxes[] -}) { - const { data, byActor, toActors, transaction, contextType, actorsException = [] } = options - - const uris = await computeUris(toActors, actorsException) - - return afterCommitIfTransaction(transaction, () => { - return broadcastTo({ - uris, - data, - byActor, - contextType - }) - }) -} - -function broadcastTo (options: { - uris: string[] - data: any - byActor: MActorId - contextType: ContextType - parallelizable?: boolean // default to false -}) { - const { uris, data, byActor, contextType, parallelizable } = options - - if (uris.length === 0) return undefined - - const broadcastUris: string[] = [] - const unicastUris: string[] = [] - - // Bad URIs could be slow to respond, prefer to process them in a dedicated queue - for (const uri of uris) { - if (ActorFollowHealthCache.Instance.isBadInbox(uri)) { - unicastUris.push(uri) - } else { - broadcastUris.push(uri) - } - } - - logger.debug('Creating broadcast job.', { broadcastUris, unicastUris }) - - if (broadcastUris.length !== 0) { - const payload = { - uris: broadcastUris, - signatureActorId: byActor.id, - body: data, - contextType - } - - JobQueue.Instance.createJobAsync({ - type: parallelizable - ? 'activitypub-http-broadcast-parallel' - : 'activitypub-http-broadcast', - - payload - }) - } - - for (const unicastUri of unicastUris) { - const payload = { - uri: unicastUri, - signatureActorId: byActor.id, - body: data, - contextType - } - - JobQueue.Instance.createJobAsync({ type: 'activitypub-http-unicast', payload }) - } -} - -function unicastTo (options: { - data: any - byActor: MActorId - toActorUrl: string - contextType: ContextType -}) { - const { data, byActor, toActorUrl, contextType } = options - - logger.debug('Creating unicast job.', { uri: toActorUrl }) - - const payload = { - uri: toActorUrl, - signatureActorId: byActor.id, - body: data, - contextType - } - - JobQueue.Instance.createJobAsync({ type: 'activitypub-http-unicast', payload }) -} - -// --------------------------------------------------------------------------- - -export { - broadcastToFollowers, - unicastTo, - forwardActivity, - broadcastToActors, - sendVideoActivityToOrigin, - forwardVideoRelatedActivity, - sendVideoRelatedActivity -} - -// --------------------------------------------------------------------------- - -async function computeFollowerUris (toFollowersOf: MActorId[], actorsException: MActorWithInboxes[], t: Transaction) { - const toActorFollowerIds = toFollowersOf.map(a => a.id) - - const result = await ActorFollowModel.listAcceptedFollowerSharedInboxUrls(toActorFollowerIds, t) - const sharedInboxesException = await buildSharedInboxesException(actorsException) - - return result.data.filter(sharedInbox => sharedInboxesException.includes(sharedInbox) === false) -} - -async function computeUris (toActors: MActor[], actorsException: MActorWithInboxes[] = []) { - const serverActor = await getServerActor() - const targetUrls = toActors - .filter(a => a.id !== serverActor.id) // Don't send to ourselves - .map(a => a.getSharedInbox()) - - const toActorSharedInboxesSet = new Set(targetUrls) - - const sharedInboxesException = await buildSharedInboxesException(actorsException) - return Array.from(toActorSharedInboxesSet) - .filter(sharedInbox => sharedInboxesException.includes(sharedInbox) === false) -} - -async function buildSharedInboxesException (actorsException: MActorWithInboxes[]) { - const serverActor = await getServerActor() - - return actorsException - .map(f => f.getSharedInbox()) - .concat([ serverActor.sharedInboxUrl ]) -} diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts deleted file mode 100644 index 792a73f2a..000000000 --- a/server/lib/activitypub/share.ts +++ /dev/null @@ -1,120 +0,0 @@ -import { map } from 'bluebird' -import { Transaction } from 'sequelize' -import { getServerActor } from '@server/models/application/application' -import { logger, loggerTagsFactory } from '../../helpers/logger' -import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' -import { VideoShareModel } from '../../models/video/video-share' -import { MChannelActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../types/models/video' -import { fetchAP, getAPId } from './activity' -import { getOrCreateAPActor } from './actors' -import { sendUndoAnnounce, sendVideoAnnounce } from './send' -import { checkUrlsSameHost, getLocalVideoAnnounceActivityPubUrl } from './url' - -const lTags = loggerTagsFactory('share') - -async function shareVideoByServerAndChannel (video: MVideoAccountLight, t: Transaction) { - if (!video.hasPrivacyForFederation()) return undefined - - return Promise.all([ - shareByServer(video, t), - shareByVideoChannel(video, t) - ]) -} - -async function changeVideoChannelShare ( - video: MVideoAccountLight, - oldVideoChannel: MChannelActorLight, - t: Transaction -) { - logger.info( - 'Updating video channel of video %s: %s -> %s.', video.uuid, oldVideoChannel.name, video.VideoChannel.name, - lTags(video.uuid) - ) - - await undoShareByVideoChannel(video, oldVideoChannel, t) - - await shareByVideoChannel(video, t) -} - -async function addVideoShares (shareUrls: string[], video: MVideoId) { - await map(shareUrls, async shareUrl => { - try { - await addVideoShare(shareUrl, video) - } catch (err) { - logger.warn('Cannot add share %s.', shareUrl, { err }) - } - }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) -} - -export { - changeVideoChannelShare, - addVideoShares, - shareVideoByServerAndChannel -} - -// --------------------------------------------------------------------------- - -async function addVideoShare (shareUrl: string, video: MVideoId) { - const { body } = await fetchAP(shareUrl) - if (!body?.actor) throw new Error('Body or body actor is invalid') - - const actorUrl = getAPId(body.actor) - if (checkUrlsSameHost(shareUrl, actorUrl) !== true) { - throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`) - } - - const actor = await getOrCreateAPActor(actorUrl) - - const entry = { - actorId: actor.id, - videoId: video.id, - url: shareUrl - } - - await VideoShareModel.upsert(entry) -} - -async function shareByServer (video: MVideo, t: Transaction) { - const serverActor = await getServerActor() - - const serverShareUrl = getLocalVideoAnnounceActivityPubUrl(serverActor, video) - const [ serverShare ] = await VideoShareModel.findOrCreate({ - defaults: { - actorId: serverActor.id, - videoId: video.id, - url: serverShareUrl - }, - where: { - url: serverShareUrl - }, - transaction: t - }) - - return sendVideoAnnounce(serverActor, serverShare, video, t) -} - -async function shareByVideoChannel (video: MVideoAccountLight, t: Transaction) { - const videoChannelShareUrl = getLocalVideoAnnounceActivityPubUrl(video.VideoChannel.Actor, video) - const [ videoChannelShare ] = await VideoShareModel.findOrCreate({ - defaults: { - actorId: video.VideoChannel.actorId, - videoId: video.id, - url: videoChannelShareUrl - }, - where: { - url: videoChannelShareUrl - }, - transaction: t - }) - - return sendVideoAnnounce(video.VideoChannel.Actor, videoChannelShare, video, t) -} - -async function undoShareByVideoChannel (video: MVideo, oldVideoChannel: MChannelActorLight, t: Transaction) { - // Load old share - const oldShare = await VideoShareModel.load(oldVideoChannel.actorId, video.id, t) - if (!oldShare) return new Error('Cannot find old video channel share ' + oldVideoChannel.actorId + ' for video ' + video.id) - - await sendUndoAnnounce(oldVideoChannel.Actor, oldShare, video, t) - await oldShare.destroy({ transaction: t }) -} diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts deleted file mode 100644 index 5cdac71bf..000000000 --- a/server/lib/activitypub/url.ts +++ /dev/null @@ -1,177 +0,0 @@ -import { REMOTE_SCHEME, WEBSERVER } from '../../initializers/constants' -import { - MAbuseFull, - MAbuseId, - MActor, - MActorFollow, - MActorId, - MActorUrl, - MCommentId, - MLocalVideoViewer, - MVideoId, - MVideoPlaylistElement, - MVideoUrl, - MVideoUUID, - MVideoWithHost -} from '../../types/models' -import { MVideoFileVideoUUID } from '../../types/models/video/video-file' -import { MVideoPlaylist, MVideoPlaylistUUID } from '../../types/models/video/video-playlist' -import { MStreamingPlaylist } from '../../types/models/video/video-streaming-playlist' - -function getLocalVideoActivityPubUrl (video: MVideoUUID) { - return WEBSERVER.URL + '/videos/watch/' + video.uuid -} - -function getLocalVideoPlaylistActivityPubUrl (videoPlaylist: MVideoPlaylist) { - return WEBSERVER.URL + '/video-playlists/' + videoPlaylist.uuid -} - -function getLocalVideoPlaylistElementActivityPubUrl (videoPlaylist: MVideoPlaylistUUID, videoPlaylistElement: MVideoPlaylistElement) { - return WEBSERVER.URL + '/video-playlists/' + videoPlaylist.uuid + '/videos/' + videoPlaylistElement.id -} - -function getLocalVideoCacheFileActivityPubUrl (videoFile: MVideoFileVideoUUID) { - const suffixFPS = videoFile.fps && videoFile.fps !== -1 ? '-' + videoFile.fps : '' - - return `${WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` -} - -function getLocalVideoCacheStreamingPlaylistActivityPubUrl (video: MVideoUUID, playlist: MStreamingPlaylist) { - return `${WEBSERVER.URL}/redundancy/streaming-playlists/${playlist.getStringType()}/${video.uuid}` -} - -function getLocalVideoCommentActivityPubUrl (video: MVideoUUID, videoComment: MCommentId) { - return WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id -} - -function getLocalVideoChannelActivityPubUrl (videoChannelName: string) { - return WEBSERVER.URL + '/video-channels/' + videoChannelName -} - -function getLocalAccountActivityPubUrl (accountName: string) { - return WEBSERVER.URL + '/accounts/' + accountName -} - -function getLocalAbuseActivityPubUrl (abuse: MAbuseId) { - return WEBSERVER.URL + '/admin/abuses/' + abuse.id -} - -function getLocalVideoViewActivityPubUrl (byActor: MActorUrl, video: MVideoId, viewerIdentifier: string) { - return byActor.url + '/views/videos/' + video.id + '/' + viewerIdentifier -} - -function getLocalVideoViewerActivityPubUrl (stats: MLocalVideoViewer) { - return WEBSERVER.URL + '/videos/local-viewer/' + stats.uuid -} - -function getVideoLikeActivityPubUrlByLocalActor (byActor: MActorUrl, video: MVideoId) { - return byActor.url + '/likes/' + video.id -} - -function getVideoDislikeActivityPubUrlByLocalActor (byActor: MActorUrl, video: MVideoId) { - return byActor.url + '/dislikes/' + video.id -} - -function getLocalVideoSharesActivityPubUrl (video: MVideoUrl) { - return video.url + '/announces' -} - -function getLocalVideoCommentsActivityPubUrl (video: MVideoUrl) { - return video.url + '/comments' -} - -function getLocalVideoLikesActivityPubUrl (video: MVideoUrl) { - return video.url + '/likes' -} - -function getLocalVideoDislikesActivityPubUrl (video: MVideoUrl) { - return video.url + '/dislikes' -} - -function getLocalActorFollowActivityPubUrl (follower: MActor, following: MActorId) { - return follower.url + '/follows/' + following.id -} - -function getLocalActorFollowAcceptActivityPubUrl (actorFollow: MActorFollow) { - return WEBSERVER.URL + '/accepts/follows/' + actorFollow.id -} - -function getLocalActorFollowRejectActivityPubUrl () { - return WEBSERVER.URL + '/rejects/follows/' + new Date().toISOString() -} - -function getLocalVideoAnnounceActivityPubUrl (byActor: MActorId, video: MVideoUrl) { - return video.url + '/announces/' + byActor.id -} - -function getDeleteActivityPubUrl (originalUrl: string) { - return originalUrl + '/delete' -} - -function getUpdateActivityPubUrl (originalUrl: string, updatedAt: string) { - return originalUrl + '/updates/' + updatedAt -} - -function getUndoActivityPubUrl (originalUrl: string) { - return originalUrl + '/undo' -} - -// --------------------------------------------------------------------------- - -function getAbuseTargetUrl (abuse: MAbuseFull) { - return abuse.VideoAbuse?.Video?.url || - abuse.VideoCommentAbuse?.VideoComment?.url || - abuse.FlaggedAccount.Actor.url -} - -// --------------------------------------------------------------------------- - -function buildRemoteVideoBaseUrl (video: MVideoWithHost, path: string, scheme?: string) { - if (!scheme) scheme = REMOTE_SCHEME.HTTP - - const host = video.VideoChannel.Actor.Server.host - - return scheme + '://' + host + path -} - -// --------------------------------------------------------------------------- - -function checkUrlsSameHost (url1: string, url2: string) { - const idHost = new URL(url1).host - const actorHost = new URL(url2).host - - return idHost && actorHost && idHost.toLowerCase() === actorHost.toLowerCase() -} - -// --------------------------------------------------------------------------- - -export { - getLocalVideoActivityPubUrl, - getLocalVideoPlaylistActivityPubUrl, - getLocalVideoPlaylistElementActivityPubUrl, - getLocalVideoCacheFileActivityPubUrl, - getLocalVideoCacheStreamingPlaylistActivityPubUrl, - getLocalVideoCommentActivityPubUrl, - getLocalVideoChannelActivityPubUrl, - getLocalAccountActivityPubUrl, - getLocalAbuseActivityPubUrl, - getLocalActorFollowActivityPubUrl, - getLocalActorFollowAcceptActivityPubUrl, - getLocalVideoAnnounceActivityPubUrl, - getUpdateActivityPubUrl, - getUndoActivityPubUrl, - getVideoLikeActivityPubUrlByLocalActor, - getLocalVideoViewActivityPubUrl, - getVideoDislikeActivityPubUrlByLocalActor, - getLocalActorFollowRejectActivityPubUrl, - getDeleteActivityPubUrl, - getLocalVideoSharesActivityPubUrl, - getLocalVideoCommentsActivityPubUrl, - getLocalVideoLikesActivityPubUrl, - getLocalVideoDislikesActivityPubUrl, - getLocalVideoViewerActivityPubUrl, - - getAbuseTargetUrl, - checkUrlsSameHost, - buildRemoteVideoBaseUrl -} diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts deleted file mode 100644 index b861be5bd..000000000 --- a/server/lib/activitypub/video-comments.ts +++ /dev/null @@ -1,205 +0,0 @@ -import { map } from 'bluebird' - -import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments' -import { logger } from '../../helpers/logger' -import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' -import { VideoCommentModel } from '../../models/video/video-comment' -import { MComment, MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' -import { isRemoteVideoCommentAccepted } from '../moderation' -import { Hooks } from '../plugins/hooks' -import { fetchAP } from './activity' -import { getOrCreateAPActor } from './actors' -import { checkUrlsSameHost } from './url' -import { getOrCreateAPVideo } from './videos' - -type ResolveThreadParams = { - url: string - comments?: MCommentOwner[] - isVideo?: boolean - commentCreated?: boolean -} -type ResolveThreadResult = Promise<{ video: MVideoAccountLightBlacklistAllFiles, comment: MCommentOwnerVideo, commentCreated: boolean }> - -async function addVideoComments (commentUrls: string[]) { - return map(commentUrls, async commentUrl => { - try { - await resolveThread({ url: commentUrl, isVideo: false }) - } catch (err) { - logger.warn('Cannot resolve thread %s.', commentUrl, { err }) - } - }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) -} - -async function resolveThread (params: ResolveThreadParams): ResolveThreadResult { - const { url, isVideo } = params - - if (params.commentCreated === undefined) params.commentCreated = false - if (params.comments === undefined) params.comments = [] - - // If it is not a video, or if we don't know if it's a video, try to get the thread from DB - if (isVideo === false || isVideo === undefined) { - const result = await resolveCommentFromDB(params) - if (result) return result - } - - try { - // If it is a video, or if we don't know if it's a video - if (isVideo === true || isVideo === undefined) { - // Keep await so we catch the exception - return await tryToResolveThreadFromVideo(params) - } - } catch (err) { - logger.debug('Cannot resolve thread from video %s, maybe because it was not a video', url, { err }) - } - - return resolveRemoteParentComment(params) -} - -export { - addVideoComments, - resolveThread -} - -// --------------------------------------------------------------------------- - -async function resolveCommentFromDB (params: ResolveThreadParams) { - const { url, comments, commentCreated } = params - - const commentFromDatabase = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideoUrlAndAccount(url) - if (!commentFromDatabase) return undefined - - let parentComments = comments.concat([ commentFromDatabase ]) - - // Speed up things and resolve directly the thread - if (commentFromDatabase.InReplyToVideoComment) { - const data = await VideoCommentModel.listThreadParentComments(commentFromDatabase, undefined, 'DESC') - - parentComments = parentComments.concat(data) - } - - return resolveThread({ - url: commentFromDatabase.Video.url, - comments: parentComments, - isVideo: true, - commentCreated - }) -} - -async function tryToResolveThreadFromVideo (params: ResolveThreadParams) { - const { url, comments, commentCreated } = params - - // Maybe it's a reply to a video? - // If yes, it's done: we resolved all the thread - const syncParam = { rates: true, shares: true, comments: false, refreshVideo: false } - const { video } = await getOrCreateAPVideo({ videoObject: url, syncParam }) - - if (video.isOwned() && !video.hasPrivacyForFederation()) { - throw new Error('Cannot resolve thread of video with privacy that is not compatible with federation') - } - - let resultComment: MCommentOwnerVideo - if (comments.length !== 0) { - const firstReply = comments[comments.length - 1] as MCommentOwnerVideo - firstReply.inReplyToCommentId = null - firstReply.originCommentId = null - firstReply.videoId = video.id - firstReply.changed('updatedAt', true) - firstReply.Video = video - - if (await isRemoteCommentAccepted(firstReply) !== true) { - return undefined - } - - comments[comments.length - 1] = await firstReply.save() - - for (let i = comments.length - 2; i >= 0; i--) { - const comment = comments[i] as MCommentOwnerVideo - comment.originCommentId = firstReply.id - comment.inReplyToCommentId = comments[i + 1].id - comment.videoId = video.id - comment.changed('updatedAt', true) - comment.Video = video - - if (await isRemoteCommentAccepted(comment) !== true) { - return undefined - } - - comments[i] = await comment.save() - } - - resultComment = comments[0] as MCommentOwnerVideo - } - - return { video, comment: resultComment, commentCreated } -} - -async function resolveRemoteParentComment (params: ResolveThreadParams) { - const { url, comments } = params - - if (comments.length > ACTIVITY_PUB.MAX_RECURSION_COMMENTS) { - throw new Error('Recursion limit reached when resolving a thread') - } - - const { body } = await fetchAP(url) - - if (sanitizeAndCheckVideoCommentObject(body) === false) { - throw new Error(`Remote video comment JSON ${url} is not valid:` + JSON.stringify(body)) - } - - const actorUrl = body.attributedTo - if (!actorUrl && body.type !== 'Tombstone') throw new Error('Miss attributed to in comment') - - if (actorUrl && checkUrlsSameHost(url, actorUrl) !== true) { - throw new Error(`Actor url ${actorUrl} has not the same host than the comment url ${url}`) - } - - if (checkUrlsSameHost(body.id, url) !== true) { - throw new Error(`Comment url ${url} host is different from the AP object id ${body.id}`) - } - - const actor = actorUrl - ? await getOrCreateAPActor(actorUrl, 'all') - : null - - const comment = new VideoCommentModel({ - url: body.id, - text: body.content ? body.content : '', - videoId: null, - accountId: actor ? actor.Account.id : null, - inReplyToCommentId: null, - originCommentId: null, - createdAt: new Date(body.published), - updatedAt: new Date(body.updated), - deletedAt: body.deleted ? new Date(body.deleted) : null - }) as MCommentOwner - comment.Account = actor ? actor.Account : null - - return resolveThread({ - url: body.inReplyTo, - comments: comments.concat([ comment ]), - commentCreated: true - }) -} - -async function isRemoteCommentAccepted (comment: MComment) { - // Already created - if (comment.id) return true - - const acceptParameters = { - comment - } - - const acceptedResult = await Hooks.wrapFun( - isRemoteVideoCommentAccepted, - acceptParameters, - 'filter:activity-pub.remote-video-comment.create.accept.result' - ) - - if (!acceptedResult || acceptedResult.accepted !== true) { - logger.info('Refused to create a remote comment.', { acceptedResult, acceptParameters }) - - return false - } - - return true -} diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts deleted file mode 100644 index 2e7920f4e..000000000 --- a/server/lib/activitypub/video-rates.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { Transaction } from 'sequelize' -import { VideoRateType } from '../../../shared/models/videos' -import { MAccountActor, MActorUrl, MVideoAccountLight, MVideoFullLight, MVideoId } from '../../types/models' -import { sendLike, sendUndoDislike, sendUndoLike } from './send' -import { sendDislike } from './send/send-dislike' -import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from './url' -import { federateVideoIfNeeded } from './videos' - -async function sendVideoRateChange ( - account: MAccountActor, - video: MVideoFullLight, - likes: number, - dislikes: number, - t: Transaction -) { - if (video.isOwned()) return federateVideoIfNeeded(video, false, t) - - return sendVideoRateChangeToOrigin(account, video, likes, dislikes, t) -} - -function getLocalRateUrl (rateType: VideoRateType, actor: MActorUrl, video: MVideoId) { - return rateType === 'like' - ? getVideoLikeActivityPubUrlByLocalActor(actor, video) - : getVideoDislikeActivityPubUrlByLocalActor(actor, video) -} - -// --------------------------------------------------------------------------- - -export { - getLocalRateUrl, - sendVideoRateChange -} - -// --------------------------------------------------------------------------- - -async function sendVideoRateChangeToOrigin ( - account: MAccountActor, - video: MVideoAccountLight, - likes: number, - dislikes: number, - t: Transaction -) { - // Local video, we don't need to send like - if (video.isOwned()) return - - const actor = account.Actor - - // Keep the order: first we undo and then we create - - // Undo Like - if (likes < 0) await sendUndoLike(actor, video, t) - // Undo Dislike - if (dislikes < 0) await sendUndoDislike(actor, video, t) - - // Like - if (likes > 0) await sendLike(actor, video, t) - // Dislike - if (dislikes > 0) await sendDislike(actor, video, t) -} diff --git a/server/lib/activitypub/videos/federate.ts b/server/lib/activitypub/videos/federate.ts deleted file mode 100644 index d7e251153..000000000 --- a/server/lib/activitypub/videos/federate.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { Transaction } from 'sequelize/types' -import { MVideoAP, MVideoAPLight } from '@server/types/models' -import { sendCreateVideo, sendUpdateVideo } from '../send' -import { shareVideoByServerAndChannel } from '../share' - -async function federateVideoIfNeeded (videoArg: MVideoAPLight, isNewVideo: boolean, transaction?: Transaction) { - const video = videoArg as MVideoAP - - if ( - // Check this is not a blacklisted video, or unfederated blacklisted video - (video.isBlacklisted() === false || (isNewVideo === false && video.VideoBlacklist.unfederated === false)) && - // Check the video is public/unlisted and published - video.hasPrivacyForFederation() && video.hasStateForFederation() - ) { - const video = await videoArg.lightAPToFullAP(transaction) - - if (isNewVideo) { - // Now we'll add the video's meta data to our followers - await sendCreateVideo(video, transaction) - await shareVideoByServerAndChannel(video, transaction) - } else { - await sendUpdateVideo(video, transaction) - } - } -} - -export { - federateVideoIfNeeded -} diff --git a/server/lib/activitypub/videos/get.ts b/server/lib/activitypub/videos/get.ts deleted file mode 100644 index 288c506ee..000000000 --- a/server/lib/activitypub/videos/get.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { logger } from '@server/helpers/logger' -import { JobQueue } from '@server/lib/job-queue' -import { loadVideoByUrl, VideoLoadByUrlType } from '@server/lib/model-loaders' -import { MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models' -import { APObjectId } from '@shared/models' -import { getAPId } from '../activity' -import { refreshVideoIfNeeded } from './refresh' -import { APVideoCreator, fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared' - -type GetVideoResult = Promise<{ - video: T - created: boolean - autoBlacklisted?: boolean -}> - -type GetVideoParamAll = { - videoObject: APObjectId - syncParam?: SyncParam - fetchType?: 'all' - allowRefresh?: boolean -} - -type GetVideoParamImmutable = { - videoObject: APObjectId - syncParam?: SyncParam - fetchType: 'only-immutable-attributes' - allowRefresh: false -} - -type GetVideoParamOther = { - videoObject: APObjectId - syncParam?: SyncParam - fetchType?: 'all' | 'only-video' - allowRefresh?: boolean -} - -function getOrCreateAPVideo (options: GetVideoParamAll): GetVideoResult -function getOrCreateAPVideo (options: GetVideoParamImmutable): GetVideoResult -function getOrCreateAPVideo (options: GetVideoParamOther): GetVideoResult - -async function getOrCreateAPVideo ( - options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther -): GetVideoResult { - // Default params - const syncParam = options.syncParam || { rates: true, shares: true, comments: true, refreshVideo: false } - const fetchType = options.fetchType || 'all' - const allowRefresh = options.allowRefresh !== false - - // Get video url - const videoUrl = getAPId(options.videoObject) - let videoFromDatabase = await loadVideoByUrl(videoUrl, fetchType) - - if (videoFromDatabase) { - if (allowRefresh === true) { - // Typings ensure allowRefresh === false in only-immutable-attributes fetch type - videoFromDatabase = await scheduleRefresh(videoFromDatabase as MVideoThumbnail, fetchType, syncParam) - } - - return { video: videoFromDatabase, created: false } - } - - const { videoObject } = await fetchRemoteVideo(videoUrl) - if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl) - - // videoUrl is just an alias/rediraction, so process object id instead - if (videoObject.id !== videoUrl) return getOrCreateAPVideo({ ...options, fetchType: 'all', videoObject }) - - try { - const creator = new APVideoCreator(videoObject) - const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(creator.create.bind(creator)) - - await syncVideoExternalAttributes(videoCreated, videoObject, syncParam) - - return { video: videoCreated, created: true, autoBlacklisted } - } catch (err) { - // Maybe a concurrent getOrCreateAPVideo call created this video - if (err.name === 'SequelizeUniqueConstraintError') { - const alreadyCreatedVideo = await loadVideoByUrl(videoUrl, fetchType) - if (alreadyCreatedVideo) return { video: alreadyCreatedVideo, created: false } - - logger.error('Cannot create video %s because of SequelizeUniqueConstraintError error, but cannot find it in database.', videoUrl) - } - - throw err - } -} - -// --------------------------------------------------------------------------- - -export { - getOrCreateAPVideo -} - -// --------------------------------------------------------------------------- - -async function scheduleRefresh (video: MVideoThumbnail, fetchType: VideoLoadByUrlType, syncParam: SyncParam) { - if (!video.isOutdated()) return video - - const refreshOptions = { - video, - fetchedType: fetchType, - syncParam - } - - if (syncParam.refreshVideo === true) { - return refreshVideoIfNeeded(refreshOptions) - } - - await JobQueue.Instance.createJob({ - type: 'activitypub-refresher', - payload: { type: 'video', url: video.url } - }) - - return video -} diff --git a/server/lib/activitypub/videos/index.ts b/server/lib/activitypub/videos/index.ts deleted file mode 100644 index b22062598..000000000 --- a/server/lib/activitypub/videos/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './federate' -export * from './get' -export * from './refresh' -export * from './updater' diff --git a/server/lib/activitypub/videos/refresh.ts b/server/lib/activitypub/videos/refresh.ts deleted file mode 100644 index 9f952a218..000000000 --- a/server/lib/activitypub/videos/refresh.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { PeerTubeRequestError } from '@server/helpers/requests' -import { VideoLoadByUrlType } from '@server/lib/model-loaders' -import { VideoModel } from '@server/models/video/video' -import { MVideoAccountLightBlacklistAllFiles, MVideoThumbnail } from '@server/types/models' -import { HttpStatusCode } from '@shared/models' -import { ActorFollowHealthCache } from '../../actor-follow-health-cache' -import { fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared' -import { APVideoUpdater } from './updater' - -async function refreshVideoIfNeeded (options: { - video: MVideoThumbnail - fetchedType: VideoLoadByUrlType - syncParam: SyncParam -}): Promise { - if (!options.video.isOutdated()) return options.video - - // We need more attributes if the argument video was fetched with not enough joints - const video = options.fetchedType === 'all' - ? options.video as MVideoAccountLightBlacklistAllFiles - : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) - - const lTags = loggerTagsFactory('ap', 'video', 'refresh', video.uuid, video.url) - - logger.info('Refreshing video %s.', video.url, lTags()) - - try { - const { videoObject } = await fetchRemoteVideo(video.url) - - if (videoObject === undefined) { - logger.warn('Cannot refresh remote video %s: invalid body.', video.url, lTags()) - - await video.setAsRefreshed() - return video - } - - const videoUpdater = new APVideoUpdater(videoObject, video) - await videoUpdater.update() - - await syncVideoExternalAttributes(video, videoObject, options.syncParam) - - ActorFollowHealthCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId) - - return video - } catch (err) { - if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { - logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url, lTags()) - - // Video does not exist anymore - await video.destroy() - return undefined - } - - logger.warn('Cannot refresh video %s.', options.video.url, { err, ...lTags() }) - - ActorFollowHealthCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId) - - // Don't refresh in loop - await video.setAsRefreshed() - return video - } -} - -// --------------------------------------------------------------------------- - -export { - refreshVideoIfNeeded -} diff --git a/server/lib/activitypub/videos/shared/abstract-builder.ts b/server/lib/activitypub/videos/shared/abstract-builder.ts deleted file mode 100644 index 98c2f58eb..000000000 --- a/server/lib/activitypub/videos/shared/abstract-builder.ts +++ /dev/null @@ -1,190 +0,0 @@ -import { CreationAttributes, Transaction } from 'sequelize/types' -import { deleteAllModels, filterNonExistingModels } from '@server/helpers/database-utils' -import { logger, LoggerTagsFn } from '@server/helpers/logger' -import { updateRemoteVideoThumbnail } from '@server/lib/thumbnail' -import { setVideoTags } from '@server/lib/video' -import { StoryboardModel } from '@server/models/video/storyboard' -import { VideoCaptionModel } from '@server/models/video/video-caption' -import { VideoFileModel } from '@server/models/video/video-file' -import { VideoLiveModel } from '@server/models/video/video-live' -import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { - MStreamingPlaylistFiles, - MStreamingPlaylistFilesVideo, - MVideoCaption, - MVideoFile, - MVideoFullLight, - MVideoThumbnail -} from '@server/types/models' -import { ActivityTagObject, ThumbnailType, VideoObject, VideoStreamingPlaylistType } from '@shared/models' -import { findOwner, getOrCreateAPActor } from '../../actors' -import { - getCaptionAttributesFromObject, - getFileAttributesFromUrl, - getLiveAttributesFromObject, - getPreviewFromIcons, - getStoryboardAttributeFromObject, - getStreamingPlaylistAttributesFromObject, - getTagsFromObject, - getThumbnailFromIcons -} from './object-to-model-attributes' -import { getTrackerUrls, setVideoTrackers } from './trackers' - -export abstract class APVideoAbstractBuilder { - protected abstract videoObject: VideoObject - protected abstract lTags: LoggerTagsFn - - protected async getOrCreateVideoChannelFromVideoObject () { - const channel = await findOwner(this.videoObject.id, this.videoObject.attributedTo, 'Group') - if (!channel) throw new Error('Cannot find associated video channel to video ' + this.videoObject.url) - - return getOrCreateAPActor(channel.id, 'all') - } - - protected async setThumbnail (video: MVideoThumbnail, t?: Transaction) { - const miniatureIcon = getThumbnailFromIcons(this.videoObject) - if (!miniatureIcon) { - logger.warn('Cannot find thumbnail in video object', { object: this.videoObject }) - return undefined - } - - const miniatureModel = updateRemoteVideoThumbnail({ - fileUrl: miniatureIcon.url, - video, - type: ThumbnailType.MINIATURE, - size: miniatureIcon, - onDisk: false // Lazy download remote thumbnails - }) - - await video.addAndSaveThumbnail(miniatureModel, t) - } - - protected async setPreview (video: MVideoFullLight, t?: Transaction) { - const previewIcon = getPreviewFromIcons(this.videoObject) - if (!previewIcon) return - - const previewModel = updateRemoteVideoThumbnail({ - fileUrl: previewIcon.url, - video, - type: ThumbnailType.PREVIEW, - size: previewIcon, - onDisk: false // Lazy download remote previews - }) - - await video.addAndSaveThumbnail(previewModel, t) - } - - protected async setTags (video: MVideoFullLight, t: Transaction) { - const tags = getTagsFromObject(this.videoObject) - await setVideoTags({ video, tags, transaction: t }) - } - - protected async setTrackers (video: MVideoFullLight, t: Transaction) { - const trackers = getTrackerUrls(this.videoObject, video) - await setVideoTrackers({ video, trackers, transaction: t }) - } - - protected async insertOrReplaceCaptions (video: MVideoFullLight, t: Transaction) { - const existingCaptions = await VideoCaptionModel.listVideoCaptions(video.id, t) - - let captionsToCreate = getCaptionAttributesFromObject(video, this.videoObject) - .map(a => new VideoCaptionModel(a) as MVideoCaption) - - for (const existingCaption of existingCaptions) { - // Only keep captions that do not already exist - const filtered = captionsToCreate.filter(c => !c.isEqual(existingCaption)) - - // This caption already exists, we don't need to destroy and create it - if (filtered.length !== captionsToCreate.length) { - captionsToCreate = filtered - continue - } - - // Destroy this caption that does not exist anymore - await existingCaption.destroy({ transaction: t }) - } - - for (const captionToCreate of captionsToCreate) { - await captionToCreate.save({ transaction: t }) - } - } - - protected async insertOrReplaceStoryboard (video: MVideoFullLight, t: Transaction) { - const existingStoryboard = await StoryboardModel.loadByVideo(video.id, t) - if (existingStoryboard) await existingStoryboard.destroy({ transaction: t }) - - const storyboardAttributes = getStoryboardAttributeFromObject(video, this.videoObject) - if (!storyboardAttributes) return - - return StoryboardModel.create(storyboardAttributes, { transaction: t }) - } - - protected async insertOrReplaceLive (video: MVideoFullLight, transaction: Transaction) { - const attributes = getLiveAttributesFromObject(video, this.videoObject) - const [ videoLive ] = await VideoLiveModel.upsert(attributes, { transaction, returning: true }) - - video.VideoLive = videoLive - } - - protected async setWebVideoFiles (video: MVideoFullLight, t: Transaction) { - const videoFileAttributes = getFileAttributesFromUrl(video, this.videoObject.url) - const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a)) - - // Remove video files that do not exist anymore - await deleteAllModels(filterNonExistingModels(video.VideoFiles || [], newVideoFiles), t) - - // Update or add other one - const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t)) - video.VideoFiles = await Promise.all(upsertTasks) - } - - protected async setStreamingPlaylists (video: MVideoFullLight, t: Transaction) { - const streamingPlaylistAttributes = getStreamingPlaylistAttributesFromObject(video, this.videoObject) - const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a)) - - // Remove video playlists that do not exist anymore - await deleteAllModels(filterNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists), t) - - const oldPlaylists = video.VideoStreamingPlaylists - video.VideoStreamingPlaylists = [] - - for (const playlistAttributes of streamingPlaylistAttributes) { - const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlistAttributes, t) - streamingPlaylistModel.Video = video - - await this.setStreamingPlaylistFiles(oldPlaylists, streamingPlaylistModel, playlistAttributes.tagAPObject, t) - - video.VideoStreamingPlaylists.push(streamingPlaylistModel) - } - } - - private async insertOrReplaceStreamingPlaylist (attributes: CreationAttributes, t: Transaction) { - const [ streamingPlaylist ] = await VideoStreamingPlaylistModel.upsert(attributes, { returning: true, transaction: t }) - - return streamingPlaylist as MStreamingPlaylistFilesVideo - } - - private getStreamingPlaylistFiles (oldPlaylists: MStreamingPlaylistFiles[], type: VideoStreamingPlaylistType) { - const playlist = oldPlaylists.find(s => s.type === type) - if (!playlist) return [] - - return playlist.VideoFiles - } - - private async setStreamingPlaylistFiles ( - oldPlaylists: MStreamingPlaylistFiles[], - playlistModel: MStreamingPlaylistFilesVideo, - tagObjects: ActivityTagObject[], - t: Transaction - ) { - const oldStreamingPlaylistFiles = this.getStreamingPlaylistFiles(oldPlaylists || [], playlistModel.type) - - const newVideoFiles: MVideoFile[] = getFileAttributesFromUrl(playlistModel, tagObjects).map(a => new VideoFileModel(a)) - - await deleteAllModels(filterNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles), t) - - // Update or add other one - const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t)) - playlistModel.VideoFiles = await Promise.all(upsertTasks) - } -} diff --git a/server/lib/activitypub/videos/shared/creator.ts b/server/lib/activitypub/videos/shared/creator.ts deleted file mode 100644 index e44fd0d52..000000000 --- a/server/lib/activitypub/videos/shared/creator.ts +++ /dev/null @@ -1,65 +0,0 @@ - -import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' -import { sequelizeTypescript } from '@server/initializers/database' -import { Hooks } from '@server/lib/plugins/hooks' -import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist' -import { VideoModel } from '@server/models/video/video' -import { MVideoFullLight, MVideoThumbnail } from '@server/types/models' -import { VideoObject } from '@shared/models' -import { APVideoAbstractBuilder } from './abstract-builder' -import { getVideoAttributesFromObject } from './object-to-model-attributes' - -export class APVideoCreator extends APVideoAbstractBuilder { - protected lTags: LoggerTagsFn - - constructor (protected readonly videoObject: VideoObject) { - super() - - this.lTags = loggerTagsFactory('ap', 'video', 'create', this.videoObject.uuid, this.videoObject.id) - } - - async create () { - logger.debug('Adding remote video %s.', this.videoObject.id, this.lTags()) - - const channelActor = await this.getOrCreateVideoChannelFromVideoObject() - const channel = channelActor.VideoChannel - - const videoData = getVideoAttributesFromObject(channel, this.videoObject, this.videoObject.to) - const video = VideoModel.build({ ...videoData, likes: 0, dislikes: 0 }) as MVideoThumbnail - - const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => { - const videoCreated = await video.save({ transaction: t }) as MVideoFullLight - videoCreated.VideoChannel = channel - - await this.setThumbnail(videoCreated, t) - await this.setPreview(videoCreated, t) - await this.setWebVideoFiles(videoCreated, t) - await this.setStreamingPlaylists(videoCreated, t) - await this.setTags(videoCreated, t) - await this.setTrackers(videoCreated, t) - await this.insertOrReplaceCaptions(videoCreated, t) - await this.insertOrReplaceLive(videoCreated, t) - await this.insertOrReplaceStoryboard(videoCreated, t) - - // We added a video in this channel, set it as updated - await channel.setAsUpdated(t) - - const autoBlacklisted = await autoBlacklistVideoIfNeeded({ - video: videoCreated, - user: undefined, - isRemote: true, - isNew: true, - isNewFile: true, - transaction: t - }) - - logger.info('Remote video with uuid %s inserted.', this.videoObject.uuid, this.lTags()) - - Hooks.runAction('action:activity-pub.remote-video.created', { video: videoCreated, videoAPObject: this.videoObject }) - - return { autoBlacklisted, videoCreated } - }) - - return { autoBlacklisted, videoCreated } - } -} diff --git a/server/lib/activitypub/videos/shared/index.ts b/server/lib/activitypub/videos/shared/index.ts deleted file mode 100644 index 951403493..000000000 --- a/server/lib/activitypub/videos/shared/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from './abstract-builder' -export * from './creator' -export * from './object-to-model-attributes' -export * from './trackers' -export * from './url-to-object' -export * from './video-sync-attributes' diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts deleted file mode 100644 index 6cbe72e27..000000000 --- a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts +++ /dev/null @@ -1,285 +0,0 @@ -import { maxBy, minBy } from 'lodash' -import { decode as magnetUriDecode } from 'magnet-uri' -import { basename, extname } from 'path' -import { isAPVideoFileUrlMetadataObject } from '@server/helpers/custom-validators/activitypub/videos' -import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/videos' -import { logger } from '@server/helpers/logger' -import { getExtFromMimetype } from '@server/helpers/video' -import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants' -import { generateTorrentFileName } from '@server/lib/paths' -import { VideoCaptionModel } from '@server/models/video/video-caption' -import { VideoFileModel } from '@server/models/video/video-file' -import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { FilteredModelAttributes } from '@server/types' -import { isStreamingPlaylist, MChannelId, MStreamingPlaylistVideo, MVideo, MVideoId } from '@server/types/models' -import { - ActivityHashTagObject, - ActivityMagnetUrlObject, - ActivityPlaylistSegmentHashesObject, - ActivityPlaylistUrlObject, - ActivityTagObject, - ActivityUrlObject, - ActivityVideoUrlObject, - VideoObject, - VideoPrivacy, - VideoStreamingPlaylistType -} from '@shared/models' -import { getDurationFromActivityStream } from '../../activity' -import { isArray } from '@server/helpers/custom-validators/misc' -import { generateImageFilename } from '@server/helpers/image-utils' -import { arrayify } from '@shared/core-utils' - -function getThumbnailFromIcons (videoObject: VideoObject) { - let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth) - // Fallback if there are not valid icons - if (validIcons.length === 0) validIcons = videoObject.icon - - return minBy(validIcons, 'width') -} - -function getPreviewFromIcons (videoObject: VideoObject) { - const validIcons = videoObject.icon.filter(i => i.width > PREVIEWS_SIZE.minWidth) - - return maxBy(validIcons, 'width') -} - -function getTagsFromObject (videoObject: VideoObject) { - return videoObject.tag - .filter(isAPHashTagObject) - .map(t => t.name) -} - -function getFileAttributesFromUrl ( - videoOrPlaylist: MVideo | MStreamingPlaylistVideo, - urls: (ActivityTagObject | ActivityUrlObject)[] -) { - const fileUrls = urls.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] - - if (fileUrls.length === 0) return [] - - const attributes: FilteredModelAttributes[] = [] - for (const fileUrl of fileUrls) { - // Fetch associated magnet uri - const magnet = urls.filter(isAPMagnetUrlObject) - .find(u => u.height === fileUrl.height) - - if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) - - const parsed = magnetUriDecode(magnet.href) - if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) { - throw new Error('Cannot parse magnet URI ' + magnet.href) - } - - const torrentUrl = Array.isArray(parsed.xs) - ? parsed.xs[0] - : parsed.xs - - // Fetch associated metadata url, if any - const metadata = urls.filter(isAPVideoFileUrlMetadataObject) - .find(u => { - return u.height === fileUrl.height && - u.fps === fileUrl.fps && - u.rel.includes(fileUrl.mediaType) - }) - - const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType) - const resolution = fileUrl.height - const videoId = isStreamingPlaylist(videoOrPlaylist) ? null : videoOrPlaylist.id - const videoStreamingPlaylistId = isStreamingPlaylist(videoOrPlaylist) ? videoOrPlaylist.id : null - - const attribute = { - extname, - infoHash: parsed.infoHash, - resolution, - size: fileUrl.size, - fps: fileUrl.fps || -1, - metadataUrl: metadata?.href, - - // Use the name of the remote file because we don't proxify video file requests - filename: basename(fileUrl.href), - fileUrl: fileUrl.href, - - torrentUrl, - // Use our own torrent name since we proxify torrent requests - torrentFilename: generateTorrentFileName(videoOrPlaylist, resolution), - - // This is a video file owned by a video or by a streaming playlist - videoId, - videoStreamingPlaylistId - } - - attributes.push(attribute) - } - - return attributes -} - -function getStreamingPlaylistAttributesFromObject (video: MVideoId, videoObject: VideoObject) { - const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[] - if (playlistUrls.length === 0) return [] - - const attributes: (FilteredModelAttributes & { tagAPObject?: ActivityTagObject[] })[] = [] - for (const playlistUrlObject of playlistUrls) { - const segmentsSha256UrlObject = playlistUrlObject.tag.find(isAPPlaylistSegmentHashesUrlObject) - - const files: unknown[] = playlistUrlObject.tag.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] - - if (!segmentsSha256UrlObject) { - logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject }) - continue - } - - const attribute = { - type: VideoStreamingPlaylistType.HLS, - - playlistFilename: basename(playlistUrlObject.href), - playlistUrl: playlistUrlObject.href, - - segmentsSha256Filename: basename(segmentsSha256UrlObject.href), - segmentsSha256Url: segmentsSha256UrlObject.href, - - p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files), - p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, - videoId: video.id, - - tagAPObject: playlistUrlObject.tag - } - - attributes.push(attribute) - } - - return attributes -} - -function getLiveAttributesFromObject (video: MVideoId, videoObject: VideoObject) { - return { - saveReplay: videoObject.liveSaveReplay, - permanentLive: videoObject.permanentLive, - latencyMode: videoObject.latencyMode, - videoId: video.id - } -} - -function getCaptionAttributesFromObject (video: MVideoId, videoObject: VideoObject) { - return videoObject.subtitleLanguage.map(c => ({ - videoId: video.id, - filename: VideoCaptionModel.generateCaptionName(c.identifier), - language: c.identifier, - fileUrl: c.url - })) -} - -function getStoryboardAttributeFromObject (video: MVideoId, videoObject: VideoObject) { - if (!isArray(videoObject.preview)) return undefined - - const storyboard = videoObject.preview.find(p => p.rel.includes('storyboard')) - if (!storyboard) return undefined - - const url = arrayify(storyboard.url).find(u => u.mediaType === 'image/jpeg') - - return { - filename: generateImageFilename(extname(url.href)), - totalHeight: url.height, - totalWidth: url.width, - spriteHeight: url.tileHeight, - spriteWidth: url.tileWidth, - spriteDuration: getDurationFromActivityStream(url.tileDuration), - fileUrl: url.href, - videoId: video.id - } -} - -function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) { - const privacy = to.includes(ACTIVITY_PUB.PUBLIC) - ? VideoPrivacy.PUBLIC - : VideoPrivacy.UNLISTED - - const language = videoObject.language?.identifier - - const category = videoObject.category - ? parseInt(videoObject.category.identifier, 10) - : undefined - - const licence = videoObject.licence - ? parseInt(videoObject.licence.identifier, 10) - : undefined - - const description = videoObject.content || null - const support = videoObject.support || null - - return { - name: videoObject.name, - uuid: videoObject.uuid, - url: videoObject.id, - category, - licence, - language, - description, - support, - nsfw: videoObject.sensitive, - commentsEnabled: videoObject.commentsEnabled, - downloadEnabled: videoObject.downloadEnabled, - waitTranscoding: videoObject.waitTranscoding, - isLive: videoObject.isLiveBroadcast, - state: videoObject.state, - channelId: videoChannel.id, - duration: getDurationFromActivityStream(videoObject.duration), - createdAt: new Date(videoObject.published), - publishedAt: new Date(videoObject.published), - - originallyPublishedAt: videoObject.originallyPublishedAt - ? new Date(videoObject.originallyPublishedAt) - : null, - - inputFileUpdatedAt: videoObject.uploadDate - ? new Date(videoObject.uploadDate) - : null, - - updatedAt: new Date(videoObject.updated), - views: videoObject.views, - remote: true, - privacy - } -} - -// --------------------------------------------------------------------------- - -export { - getThumbnailFromIcons, - getPreviewFromIcons, - - getTagsFromObject, - - getFileAttributesFromUrl, - getStreamingPlaylistAttributesFromObject, - - getLiveAttributesFromObject, - getCaptionAttributesFromObject, - getStoryboardAttributeFromObject, - - getVideoAttributesFromObject -} - -// --------------------------------------------------------------------------- - -function isAPVideoUrlObject (url: any): url is ActivityVideoUrlObject { - const urlMediaType = url.mediaType - - return MIMETYPES.VIDEO.MIMETYPE_EXT[urlMediaType] && urlMediaType.startsWith('video/') -} - -function isAPStreamingPlaylistUrlObject (url: any): url is ActivityPlaylistUrlObject { - return url && url.mediaType === 'application/x-mpegURL' -} - -function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject { - return tag && tag.name === 'sha256' && tag.type === 'Link' && tag.mediaType === 'application/json' -} - -function isAPMagnetUrlObject (url: any): url is ActivityMagnetUrlObject { - return url && url.mediaType === 'application/x-bittorrent;x-scheme-handler/magnet' -} - -function isAPHashTagObject (url: any): url is ActivityHashTagObject { - return url && url.type === 'Hashtag' -} diff --git a/server/lib/activitypub/videos/shared/trackers.ts b/server/lib/activitypub/videos/shared/trackers.ts deleted file mode 100644 index 2418f45c2..000000000 --- a/server/lib/activitypub/videos/shared/trackers.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { Transaction } from 'sequelize/types' -import { isAPVideoTrackerUrlObject } from '@server/helpers/custom-validators/activitypub/videos' -import { isArray } from '@server/helpers/custom-validators/misc' -import { REMOTE_SCHEME } from '@server/initializers/constants' -import { TrackerModel } from '@server/models/server/tracker' -import { MVideo, MVideoWithHost } from '@server/types/models' -import { ActivityTrackerUrlObject, VideoObject } from '@shared/models' -import { buildRemoteVideoBaseUrl } from '../../url' - -function getTrackerUrls (object: VideoObject, video: MVideoWithHost) { - let wsFound = false - - const trackers = object.url.filter(u => isAPVideoTrackerUrlObject(u)) - .map((u: ActivityTrackerUrlObject) => { - if (isArray(u.rel) && u.rel.includes('websocket')) wsFound = true - - return u.href - }) - - if (wsFound) return trackers - - return [ - buildRemoteVideoBaseUrl(video, '/tracker/socket', REMOTE_SCHEME.WS), - buildRemoteVideoBaseUrl(video, '/tracker/announce') - ] -} - -async function setVideoTrackers (options: { - video: MVideo - trackers: string[] - transaction: Transaction -}) { - const { video, trackers, transaction } = options - - const trackerInstances = await TrackerModel.findOrCreateTrackers(trackers, transaction) - - await video.$set('Trackers', trackerInstances, { transaction }) -} - -export { - getTrackerUrls, - setVideoTrackers -} diff --git a/server/lib/activitypub/videos/shared/url-to-object.ts b/server/lib/activitypub/videos/shared/url-to-object.ts deleted file mode 100644 index 7fe008419..000000000 --- a/server/lib/activitypub/videos/shared/url-to-object.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { VideoObject } from '@shared/models' -import { fetchAP } from '../../activity' -import { checkUrlsSameHost } from '../../url' - -const lTags = loggerTagsFactory('ap', 'video') - -async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { - logger.info('Fetching remote video %s.', videoUrl, lTags(videoUrl)) - - const { statusCode, body } = await fetchAP(videoUrl) - - if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { - logger.debug('Remote video JSON is not valid.', { body, ...lTags(videoUrl) }) - - return { statusCode, videoObject: undefined } - } - - return { statusCode, videoObject: body } -} - -export { - fetchRemoteVideo -} diff --git a/server/lib/activitypub/videos/shared/video-sync-attributes.ts b/server/lib/activitypub/videos/shared/video-sync-attributes.ts deleted file mode 100644 index 7fb933559..000000000 --- a/server/lib/activitypub/videos/shared/video-sync-attributes.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { runInReadCommittedTransaction } from '@server/helpers/database-utils' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { JobQueue } from '@server/lib/job-queue' -import { VideoModel } from '@server/models/video/video' -import { VideoCommentModel } from '@server/models/video/video-comment' -import { VideoShareModel } from '@server/models/video/video-share' -import { MVideo } from '@server/types/models' -import { ActivitypubHttpFetcherPayload, ActivityPubOrderedCollection, VideoObject } from '@shared/models' -import { fetchAP } from '../../activity' -import { crawlCollectionPage } from '../../crawl' -import { addVideoShares } from '../../share' -import { addVideoComments } from '../../video-comments' - -const lTags = loggerTagsFactory('ap', 'video') - -type SyncParam = { - rates: boolean - shares: boolean - comments: boolean - refreshVideo?: boolean -} - -async function syncVideoExternalAttributes ( - video: MVideo, - fetchedVideo: VideoObject, - syncParam: Pick -) { - logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid) - - const ratePromise = updateVideoRates(video, fetchedVideo) - if (syncParam.rates) await ratePromise - - await syncShares(video, fetchedVideo, syncParam.shares) - - await syncComments(video, fetchedVideo, syncParam.comments) -} - -async function updateVideoRates (video: MVideo, fetchedVideo: VideoObject) { - const [ likes, dislikes ] = await Promise.all([ - getRatesCount('like', video, fetchedVideo), - getRatesCount('dislike', video, fetchedVideo) - ]) - - return runInReadCommittedTransaction(async t => { - await VideoModel.updateRatesOf(video.id, 'like', likes, t) - await VideoModel.updateRatesOf(video.id, 'dislike', dislikes, t) - }) -} - -// --------------------------------------------------------------------------- - -export { - SyncParam, - syncVideoExternalAttributes, - updateVideoRates -} - -// --------------------------------------------------------------------------- - -async function getRatesCount (type: 'like' | 'dislike', video: MVideo, fetchedVideo: VideoObject) { - const uri = type === 'like' - ? fetchedVideo.likes - : fetchedVideo.dislikes - - logger.info('Sync %s of video %s', type, video.url) - - const { body } = await fetchAP>(uri) - - if (isNaN(body.totalItems)) { - logger.error('Cannot sync %s of video %s, totalItems is not a number', type, video.url, { body }) - return - } - - return body.totalItems -} - -function syncShares (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) { - const uri = fetchedVideo.shares - - if (!isSync) { - return createJob({ uri, videoId: video.id, type: 'video-shares' }) - } - - const handler = items => addVideoShares(items, video) - const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate) - - return crawlCollectionPage(uri, handler, cleaner) - .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err, rootUrl: uri, ...lTags(video.uuid, video.url) })) -} - -function syncComments (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) { - const uri = fetchedVideo.comments - - if (!isSync) { - return createJob({ uri, videoId: video.id, type: 'video-comments' }) - } - - const handler = items => addVideoComments(items) - const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate) - - return crawlCollectionPage(uri, handler, cleaner) - .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err, rootUrl: uri, ...lTags(video.uuid, video.url) })) -} - -function createJob (payload: ActivitypubHttpFetcherPayload) { - return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) -} diff --git a/server/lib/activitypub/videos/updater.ts b/server/lib/activitypub/videos/updater.ts deleted file mode 100644 index acb087895..000000000 --- a/server/lib/activitypub/videos/updater.ts +++ /dev/null @@ -1,180 +0,0 @@ -import { Transaction } from 'sequelize/types' -import { resetSequelizeInstance, runInReadCommittedTransaction } from '@server/helpers/database-utils' -import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' -import { Notifier } from '@server/lib/notifier' -import { PeerTubeSocket } from '@server/lib/peertube-socket' -import { Hooks } from '@server/lib/plugins/hooks' -import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist' -import { VideoLiveModel } from '@server/models/video/video-live' -import { MActor, MChannelAccountLight, MChannelId, MVideoAccountLightBlacklistAllFiles, MVideoFullLight } from '@server/types/models' -import { VideoObject, VideoPrivacy } from '@shared/models' -import { APVideoAbstractBuilder, getVideoAttributesFromObject, updateVideoRates } from './shared' - -export class APVideoUpdater extends APVideoAbstractBuilder { - private readonly wasPrivateVideo: boolean - private readonly wasUnlistedVideo: boolean - - private readonly oldVideoChannel: MChannelAccountLight - - protected lTags: LoggerTagsFn - - constructor ( - protected readonly videoObject: VideoObject, - private readonly video: MVideoAccountLightBlacklistAllFiles - ) { - super() - - this.wasPrivateVideo = this.video.privacy === VideoPrivacy.PRIVATE - this.wasUnlistedVideo = this.video.privacy === VideoPrivacy.UNLISTED - - this.oldVideoChannel = this.video.VideoChannel - - this.lTags = loggerTagsFactory('ap', 'video', 'update', video.uuid, video.url) - } - - async update (overrideTo?: string[]) { - logger.debug( - 'Updating remote video "%s".', this.videoObject.uuid, - { videoObject: this.videoObject, ...this.lTags() } - ) - - const oldInputFileUpdatedAt = this.video.inputFileUpdatedAt - - try { - const channelActor = await this.getOrCreateVideoChannelFromVideoObject() - - const thumbnailModel = await this.setThumbnail(this.video) - - this.checkChannelUpdateOrThrow(channelActor) - - const videoUpdated = await this.updateVideo(channelActor.VideoChannel, undefined, overrideTo) - - if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel) - - await runInReadCommittedTransaction(async t => { - await this.setWebVideoFiles(videoUpdated, t) - await this.setStreamingPlaylists(videoUpdated, t) - }) - - await Promise.all([ - runInReadCommittedTransaction(t => this.setTags(videoUpdated, t)), - runInReadCommittedTransaction(t => this.setTrackers(videoUpdated, t)), - runInReadCommittedTransaction(t => this.setStoryboard(videoUpdated, t)), - runInReadCommittedTransaction(t => { - return Promise.all([ - this.setPreview(videoUpdated, t), - this.setThumbnail(videoUpdated, t) - ]) - }), - this.setOrDeleteLive(videoUpdated) - ]) - - await runInReadCommittedTransaction(t => this.setCaptions(videoUpdated, t)) - - await autoBlacklistVideoIfNeeded({ - video: videoUpdated, - user: undefined, - isRemote: true, - isNew: false, - isNewFile: oldInputFileUpdatedAt !== videoUpdated.inputFileUpdatedAt, - transaction: undefined - }) - - await updateVideoRates(videoUpdated, this.videoObject) - - // Notify our users? - if (this.wasPrivateVideo || this.wasUnlistedVideo) { - Notifier.Instance.notifyOnNewVideoIfNeeded(videoUpdated) - } - - if (videoUpdated.isLive) { - PeerTubeSocket.Instance.sendVideoLiveNewState(videoUpdated) - } - - Hooks.runAction('action:activity-pub.remote-video.updated', { video: videoUpdated, videoAPObject: this.videoObject }) - - logger.info('Remote video with uuid %s updated', this.videoObject.uuid, this.lTags()) - - return videoUpdated - } catch (err) { - await this.catchUpdateError(err) - } - } - - // Check we can update the channel: we trust the remote server - private checkChannelUpdateOrThrow (newChannelActor: MActor) { - if (!this.oldVideoChannel.Actor.serverId || !newChannelActor.serverId) { - throw new Error('Cannot check old channel/new channel validity because `serverId` is null') - } - - if (this.oldVideoChannel.Actor.serverId !== newChannelActor.serverId) { - throw new Error(`New channel ${newChannelActor.url} is not on the same server than new channel ${this.oldVideoChannel.Actor.url}`) - } - } - - private updateVideo (channel: MChannelId, transaction?: Transaction, overrideTo?: string[]) { - const to = overrideTo || this.videoObject.to - const videoData = getVideoAttributesFromObject(channel, this.videoObject, to) - this.video.name = videoData.name - this.video.uuid = videoData.uuid - this.video.url = videoData.url - this.video.category = videoData.category - this.video.licence = videoData.licence - this.video.language = videoData.language - this.video.description = videoData.description - this.video.support = videoData.support - this.video.nsfw = videoData.nsfw - this.video.commentsEnabled = videoData.commentsEnabled - this.video.downloadEnabled = videoData.downloadEnabled - this.video.waitTranscoding = videoData.waitTranscoding - this.video.state = videoData.state - this.video.duration = videoData.duration - this.video.createdAt = videoData.createdAt - this.video.publishedAt = videoData.publishedAt - this.video.originallyPublishedAt = videoData.originallyPublishedAt - this.video.inputFileUpdatedAt = videoData.inputFileUpdatedAt - this.video.privacy = videoData.privacy - this.video.channelId = videoData.channelId - this.video.views = videoData.views - this.video.isLive = videoData.isLive - - // Ensures we update the updatedAt attribute, even if main attributes did not change - this.video.changed('updatedAt', true) - - return this.video.save({ transaction }) as Promise - } - - private async setCaptions (videoUpdated: MVideoFullLight, t: Transaction) { - await this.insertOrReplaceCaptions(videoUpdated, t) - } - - private async setStoryboard (videoUpdated: MVideoFullLight, t: Transaction) { - await this.insertOrReplaceStoryboard(videoUpdated, t) - } - - private async setOrDeleteLive (videoUpdated: MVideoFullLight, transaction?: Transaction) { - if (!this.video.isLive) return - - if (this.video.isLive) return this.insertOrReplaceLive(videoUpdated, transaction) - - // Delete existing live if it exists - await VideoLiveModel.destroy({ - where: { - videoId: this.video.id - }, - transaction - }) - - videoUpdated.VideoLive = null - } - - private async catchUpdateError (err: Error) { - if (this.video !== undefined) { - await resetSequelizeInstance(this.video) - } - - // This is just a debug because we will retry the insert - logger.debug('Cannot update the remote video.', { err, ...this.lTags() }) - throw err - } -} diff --git a/server/lib/actor-follow-health-cache.ts b/server/lib/actor-follow-health-cache.ts deleted file mode 100644 index 34357a97a..000000000 --- a/server/lib/actor-follow-health-cache.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { ACTOR_FOLLOW_SCORE } from '../initializers/constants' -import { logger } from '../helpers/logger' - -// Cache follows scores, instead of writing them too often in database -// Keep data in memory, we don't really need Redis here as we don't really care to loose some scores -class ActorFollowHealthCache { - - private static instance: ActorFollowHealthCache - - private pendingFollowsScore: { [ url: string ]: number } = {} - - private pendingBadServer = new Set() - private pendingGoodServer = new Set() - - private readonly badInboxes = new Set() - - private constructor () {} - - static get Instance () { - return this.instance || (this.instance = new this()) - } - - updateActorFollowsHealth (goodInboxes: string[], badInboxes: string[]) { - this.badInboxes.clear() - - if (goodInboxes.length === 0 && badInboxes.length === 0) return - - logger.info( - 'Updating %d good actor follows and %d bad actor follows scores in cache.', - goodInboxes.length, badInboxes.length, { badInboxes } - ) - - for (const goodInbox of goodInboxes) { - if (this.pendingFollowsScore[goodInbox] === undefined) this.pendingFollowsScore[goodInbox] = 0 - - this.pendingFollowsScore[goodInbox] += ACTOR_FOLLOW_SCORE.BONUS - } - - for (const badInbox of badInboxes) { - if (this.pendingFollowsScore[badInbox] === undefined) this.pendingFollowsScore[badInbox] = 0 - - this.pendingFollowsScore[badInbox] += ACTOR_FOLLOW_SCORE.PENALTY - this.badInboxes.add(badInbox) - } - } - - isBadInbox (inboxUrl: string) { - return this.badInboxes.has(inboxUrl) - } - - addBadServerId (serverId: number) { - this.pendingBadServer.add(serverId) - } - - getBadFollowingServerIds () { - return Array.from(this.pendingBadServer) - } - - clearBadFollowingServerIds () { - this.pendingBadServer = new Set() - } - - addGoodServerId (serverId: number) { - this.pendingGoodServer.add(serverId) - } - - getGoodFollowingServerIds () { - return Array.from(this.pendingGoodServer) - } - - clearGoodFollowingServerIds () { - this.pendingGoodServer = new Set() - } - - getPendingFollowsScore () { - return this.pendingFollowsScore - } - - clearPendingFollowsScore () { - this.pendingFollowsScore = {} - } -} - -export { - ActorFollowHealthCache -} diff --git a/server/lib/actor-image.ts b/server/lib/actor-image.ts deleted file mode 100644 index e9bd148f6..000000000 --- a/server/lib/actor-image.ts +++ /dev/null @@ -1,14 +0,0 @@ -import maxBy from 'lodash/maxBy' - -function getBiggestActorImage (images: T[]) { - const image = maxBy(images, 'width') - - // If width is null, maxBy won't return a value - if (!image) return images[0] - - return image -} - -export { - getBiggestActorImage -} diff --git a/server/lib/auth/external-auth.ts b/server/lib/auth/external-auth.ts deleted file mode 100644 index bc5b74257..000000000 --- a/server/lib/auth/external-auth.ts +++ /dev/null @@ -1,231 +0,0 @@ - -import { - isUserAdminFlagsValid, - isUserDisplayNameValid, - isUserRoleValid, - isUserUsernameValid, - isUserVideoQuotaDailyValid, - isUserVideoQuotaValid -} from '@server/helpers/custom-validators/users' -import { logger } from '@server/helpers/logger' -import { generateRandomString } from '@server/helpers/utils' -import { PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME } from '@server/initializers/constants' -import { PluginManager } from '@server/lib/plugins/plugin-manager' -import { OAuthTokenModel } from '@server/models/oauth/oauth-token' -import { MUser } from '@server/types/models' -import { - RegisterServerAuthenticatedResult, - RegisterServerAuthPassOptions, - RegisterServerExternalAuthenticatedResult -} from '@server/types/plugins/register-server-auth.model' -import { UserAdminFlag, UserRole } from '@shared/models' -import { BypassLogin } from './oauth-model' - -export type ExternalUser = - Pick & - { displayName: string } - -// Token is the key, expiration date is the value -const authBypassTokens = new Map() - -async function onExternalUserAuthenticated (options: { - npmName: string - authName: string - authResult: RegisterServerExternalAuthenticatedResult -}) { - const { npmName, authName, authResult } = options - - if (!authResult.req || !authResult.res) { - logger.error('Cannot authenticate external user for auth %s of plugin %s: no req or res are provided.', authName, npmName) - return - } - - const { res } = authResult - - if (!isAuthResultValid(npmName, authName, authResult)) { - res.redirect('/login?externalAuthError=true') - return - } - - logger.info('Generating auth bypass token for %s in auth %s of plugin %s.', authResult.username, authName, npmName) - - const bypassToken = await generateRandomString(32) - - const expires = new Date() - expires.setTime(expires.getTime() + PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME) - - const user = buildUserResult(authResult) - authBypassTokens.set(bypassToken, { - expires, - user, - npmName, - authName, - userUpdater: authResult.userUpdater - }) - - // Cleanup expired tokens - const now = new Date() - for (const [ key, value ] of authBypassTokens) { - if (value.expires.getTime() < now.getTime()) { - authBypassTokens.delete(key) - } - } - - res.redirect(`/login?externalAuthToken=${bypassToken}&username=${user.username}`) -} - -async function getAuthNameFromRefreshGrant (refreshToken?: string) { - if (!refreshToken) return undefined - - const tokenModel = await OAuthTokenModel.loadByRefreshToken(refreshToken) - - return tokenModel?.authName -} - -async function getBypassFromPasswordGrant (username: string, password: string): Promise { - const plugins = PluginManager.Instance.getIdAndPassAuths() - const pluginAuths: { npmName?: string, registerAuthOptions: RegisterServerAuthPassOptions }[] = [] - - for (const plugin of plugins) { - const auths = plugin.idAndPassAuths - - for (const auth of auths) { - pluginAuths.push({ - npmName: plugin.npmName, - registerAuthOptions: auth - }) - } - } - - pluginAuths.sort((a, b) => { - const aWeight = a.registerAuthOptions.getWeight() - const bWeight = b.registerAuthOptions.getWeight() - - // DESC weight order - if (aWeight === bWeight) return 0 - if (aWeight < bWeight) return 1 - return -1 - }) - - const loginOptions = { - id: username, - password - } - - for (const pluginAuth of pluginAuths) { - const authOptions = pluginAuth.registerAuthOptions - const authName = authOptions.authName - const npmName = pluginAuth.npmName - - logger.debug( - 'Using auth method %s of plugin %s to login %s with weight %d.', - authName, npmName, loginOptions.id, authOptions.getWeight() - ) - - try { - const loginResult = await authOptions.login(loginOptions) - - if (!loginResult) continue - if (!isAuthResultValid(pluginAuth.npmName, authOptions.authName, loginResult)) continue - - logger.info( - 'Login success with auth method %s of plugin %s for %s.', - authName, npmName, loginOptions.id - ) - - return { - bypass: true, - pluginName: pluginAuth.npmName, - authName: authOptions.authName, - user: buildUserResult(loginResult), - userUpdater: loginResult.userUpdater - } - } catch (err) { - logger.error('Error in auth method %s of plugin %s', authOptions.authName, pluginAuth.npmName, { err }) - } - } - - return undefined -} - -function getBypassFromExternalAuth (username: string, externalAuthToken: string): BypassLogin { - const obj = authBypassTokens.get(externalAuthToken) - if (!obj) throw new Error('Cannot authenticate user with unknown bypass token') - - const { expires, user, authName, npmName } = obj - - const now = new Date() - if (now.getTime() > expires.getTime()) { - throw new Error('Cannot authenticate user with an expired external auth token') - } - - if (user.username !== username) { - throw new Error(`Cannot authenticate user ${user.username} with invalid username ${username}`) - } - - logger.info( - 'Auth success with external auth method %s of plugin %s for %s.', - authName, npmName, user.email - ) - - return { - bypass: true, - pluginName: npmName, - authName, - userUpdater: obj.userUpdater, - user - } -} - -function isAuthResultValid (npmName: string, authName: string, result: RegisterServerAuthenticatedResult) { - const returnError = (field: string) => { - logger.error('Auth method %s of plugin %s did not provide a valid %s.', authName, npmName, field, { [field]: result[field] }) - return false - } - - if (!isUserUsernameValid(result.username)) return returnError('username') - if (!result.email) return returnError('email') - - // Following fields are optional - if (result.role && !isUserRoleValid(result.role)) return returnError('role') - if (result.displayName && !isUserDisplayNameValid(result.displayName)) return returnError('displayName') - if (result.adminFlags && !isUserAdminFlagsValid(result.adminFlags)) return returnError('adminFlags') - if (result.videoQuota && !isUserVideoQuotaValid(result.videoQuota + '')) return returnError('videoQuota') - if (result.videoQuotaDaily && !isUserVideoQuotaDailyValid(result.videoQuotaDaily + '')) return returnError('videoQuotaDaily') - - if (result.userUpdater && typeof result.userUpdater !== 'function') { - logger.error('Auth method %s of plugin %s did not provide a valid user updater function.', authName, npmName) - return false - } - - return true -} - -function buildUserResult (pluginResult: RegisterServerAuthenticatedResult) { - return { - username: pluginResult.username, - email: pluginResult.email, - role: pluginResult.role ?? UserRole.USER, - displayName: pluginResult.displayName || pluginResult.username, - - adminFlags: pluginResult.adminFlags ?? UserAdminFlag.NONE, - - videoQuota: pluginResult.videoQuota, - videoQuotaDaily: pluginResult.videoQuotaDaily - } -} - -// --------------------------------------------------------------------------- - -export { - onExternalUserAuthenticated, - getBypassFromExternalAuth, - getAuthNameFromRefreshGrant, - getBypassFromPasswordGrant -} diff --git a/server/lib/auth/oauth-model.ts b/server/lib/auth/oauth-model.ts deleted file mode 100644 index d3a5eccd5..000000000 --- a/server/lib/auth/oauth-model.ts +++ /dev/null @@ -1,294 +0,0 @@ -import express from 'express' -import { AccessDeniedError } from '@node-oauth/oauth2-server' -import { PluginManager } from '@server/lib/plugins/plugin-manager' -import { AccountModel } from '@server/models/account/account' -import { AuthenticatedResultUpdaterFieldName, RegisterServerAuthenticatedResult } from '@server/types' -import { MOAuthClient } from '@server/types/models' -import { MOAuthTokenUser } from '@server/types/models/oauth/oauth-token' -import { MUser, MUserDefault } from '@server/types/models/user/user' -import { pick } from '@shared/core-utils' -import { AttributesOnly } from '@shared/typescript-utils' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { OAuthClientModel } from '../../models/oauth/oauth-client' -import { OAuthTokenModel } from '../../models/oauth/oauth-token' -import { UserModel } from '../../models/user/user' -import { findAvailableLocalActorName } from '../local-actor' -import { buildUser, createUserAccountAndChannelAndPlaylist } from '../user' -import { ExternalUser } from './external-auth' -import { TokensCache } from './tokens-cache' - -type TokenInfo = { - accessToken: string - refreshToken: string - accessTokenExpiresAt: Date - refreshTokenExpiresAt: Date -} - -export type BypassLogin = { - bypass: boolean - pluginName: string - authName?: string - user: ExternalUser - userUpdater: RegisterServerAuthenticatedResult['userUpdater'] -} - -async function getAccessToken (bearerToken: string) { - logger.debug('Getting access token.') - - if (!bearerToken) return undefined - - let tokenModel: MOAuthTokenUser - - if (TokensCache.Instance.hasToken(bearerToken)) { - tokenModel = TokensCache.Instance.getByToken(bearerToken) - } else { - tokenModel = await OAuthTokenModel.getByTokenAndPopulateUser(bearerToken) - - if (tokenModel) TokensCache.Instance.setToken(tokenModel) - } - - if (!tokenModel) return undefined - - if (tokenModel.User.pluginAuth) { - const valid = await PluginManager.Instance.isTokenValid(tokenModel, 'access') - - if (valid !== true) return undefined - } - - return tokenModel -} - -function getClient (clientId: string, clientSecret: string) { - logger.debug('Getting Client (clientId: ' + clientId + ', clientSecret: ' + clientSecret + ').') - - return OAuthClientModel.getByIdAndSecret(clientId, clientSecret) -} - -async function getRefreshToken (refreshToken: string) { - logger.debug('Getting RefreshToken (refreshToken: ' + refreshToken + ').') - - const tokenInfo = await OAuthTokenModel.getByRefreshTokenAndPopulateClient(refreshToken) - if (!tokenInfo) return undefined - - const tokenModel = tokenInfo.token - - if (tokenModel.User.pluginAuth) { - const valid = await PluginManager.Instance.isTokenValid(tokenModel, 'refresh') - - if (valid !== true) return undefined - } - - return tokenInfo -} - -async function getUser (usernameOrEmail?: string, password?: string, bypassLogin?: BypassLogin) { - // Special treatment coming from a plugin - if (bypassLogin && bypassLogin.bypass === true) { - logger.info('Bypassing oauth login by plugin %s.', bypassLogin.pluginName) - - let user = await UserModel.loadByEmail(bypassLogin.user.email) - - if (!user) user = await createUserFromExternal(bypassLogin.pluginName, bypassLogin.user) - else user = await updateUserFromExternal(user, bypassLogin.user, bypassLogin.userUpdater) - - // Cannot create a user - if (!user) throw new AccessDeniedError('Cannot create such user: an actor with that name already exists.') - - // If the user does not belongs to a plugin, it was created before its installation - // Then we just go through a regular login process - if (user.pluginAuth !== null) { - // This user does not belong to this plugin, skip it - if (user.pluginAuth !== bypassLogin.pluginName) { - logger.info( - 'Cannot bypass oauth login by plugin %s because %s has another plugin auth method (%s).', - bypassLogin.pluginName, bypassLogin.user.email, user.pluginAuth - ) - - return null - } - - checkUserValidityOrThrow(user) - - return user - } - } - - logger.debug('Getting User (username/email: ' + usernameOrEmail + ', password: ******).') - - const user = await UserModel.loadByUsernameOrEmail(usernameOrEmail) - - // If we don't find the user, or if the user belongs to a plugin - if (!user || user.pluginAuth !== null || !password) return null - - const passwordMatch = await user.isPasswordMatch(password) - if (passwordMatch !== true) return null - - checkUserValidityOrThrow(user) - - if (CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION && user.emailVerified === false) { - throw new AccessDeniedError('User email is not verified.') - } - - return user -} - -async function revokeToken ( - tokenInfo: { refreshToken: string }, - options: { - req?: express.Request - explicitLogout?: boolean - } = {} -): Promise<{ success: boolean, redirectUrl?: string }> { - const { req, explicitLogout } = options - - const token = await OAuthTokenModel.getByRefreshTokenAndPopulateUser(tokenInfo.refreshToken) - - if (token) { - let redirectUrl: string - - if (explicitLogout === true && token.User.pluginAuth && token.authName) { - redirectUrl = await PluginManager.Instance.onLogout(token.User.pluginAuth, token.authName, token.User, req) - } - - TokensCache.Instance.clearCacheByToken(token.accessToken) - - token.destroy() - .catch(err => logger.error('Cannot destroy token when revoking token.', { err })) - - return { success: true, redirectUrl } - } - - return { success: false } -} - -async function saveToken ( - token: TokenInfo, - client: MOAuthClient, - user: MUser, - options: { - refreshTokenAuthName?: string - bypassLogin?: BypassLogin - } = {} -) { - const { refreshTokenAuthName, bypassLogin } = options - let authName: string = null - - if (bypassLogin?.bypass === true) { - authName = bypassLogin.authName - } else if (refreshTokenAuthName) { - authName = refreshTokenAuthName - } - - logger.debug('Saving token ' + token.accessToken + ' for client ' + client.id + ' and user ' + user.id + '.') - - const tokenToCreate = { - accessToken: token.accessToken, - accessTokenExpiresAt: token.accessTokenExpiresAt, - refreshToken: token.refreshToken, - refreshTokenExpiresAt: token.refreshTokenExpiresAt, - authName, - oAuthClientId: client.id, - userId: user.id - } - - const tokenCreated = await OAuthTokenModel.create(tokenToCreate) - - user.lastLoginDate = new Date() - await user.save() - - return { - accessToken: tokenCreated.accessToken, - accessTokenExpiresAt: tokenCreated.accessTokenExpiresAt, - refreshToken: tokenCreated.refreshToken, - refreshTokenExpiresAt: tokenCreated.refreshTokenExpiresAt, - client, - user, - accessTokenExpiresIn: buildExpiresIn(tokenCreated.accessTokenExpiresAt), - refreshTokenExpiresIn: buildExpiresIn(tokenCreated.refreshTokenExpiresAt) - } -} - -export { - getAccessToken, - getClient, - getRefreshToken, - getUser, - revokeToken, - saveToken -} - -// --------------------------------------------------------------------------- - -async function createUserFromExternal (pluginAuth: string, userOptions: ExternalUser) { - const username = await findAvailableLocalActorName(userOptions.username) - - const userToCreate = buildUser({ - ...pick(userOptions, [ 'email', 'role', 'adminFlags', 'videoQuota', 'videoQuotaDaily' ]), - - username, - emailVerified: null, - password: null, - pluginAuth - }) - - const { user } = await createUserAccountAndChannelAndPlaylist({ - userToCreate, - userDisplayName: userOptions.displayName - }) - - return user -} - -async function updateUserFromExternal ( - user: MUserDefault, - userOptions: ExternalUser, - userUpdater: RegisterServerAuthenticatedResult['userUpdater'] -) { - if (!userUpdater) return user - - { - type UserAttributeKeys = keyof AttributesOnly - const mappingKeys: { [ id in UserAttributeKeys ]?: AuthenticatedResultUpdaterFieldName } = { - role: 'role', - adminFlags: 'adminFlags', - videoQuota: 'videoQuota', - videoQuotaDaily: 'videoQuotaDaily' - } - - for (const modelKey of Object.keys(mappingKeys)) { - const pluginOptionKey = mappingKeys[modelKey] - - const newValue = userUpdater({ fieldName: pluginOptionKey, currentValue: user[modelKey], newValue: userOptions[pluginOptionKey] }) - user.set(modelKey, newValue) - } - } - - { - type AccountAttributeKeys = keyof Partial> - const mappingKeys: { [ id in AccountAttributeKeys ]?: AuthenticatedResultUpdaterFieldName } = { - name: 'displayName' - } - - for (const modelKey of Object.keys(mappingKeys)) { - const optionKey = mappingKeys[modelKey] - - const newValue = userUpdater({ fieldName: optionKey, currentValue: user.Account[modelKey], newValue: userOptions[optionKey] }) - user.Account.set(modelKey, newValue) - } - } - - logger.debug('Updated user %s with plugin userUpdated function.', user.email, { user, userOptions }) - - user.Account = await user.Account.save() - - return user.save() -} - -function checkUserValidityOrThrow (user: MUser) { - if (user.blocked) throw new AccessDeniedError('User is blocked.') -} - -function buildExpiresIn (expiresAt: Date) { - return Math.floor((expiresAt.getTime() - new Date().getTime()) / 1000) -} diff --git a/server/lib/auth/oauth.ts b/server/lib/auth/oauth.ts deleted file mode 100644 index 887c4f7c9..000000000 --- a/server/lib/auth/oauth.ts +++ /dev/null @@ -1,223 +0,0 @@ -import express from 'express' -import OAuth2Server, { - InvalidClientError, - InvalidGrantError, - InvalidRequestError, - Request, - Response, - UnauthorizedClientError, - UnsupportedGrantTypeError -} from '@node-oauth/oauth2-server' -import { randomBytesPromise } from '@server/helpers/core-utils' -import { isOTPValid } from '@server/helpers/otp' -import { CONFIG } from '@server/initializers/config' -import { UserRegistrationModel } from '@server/models/user/user-registration' -import { MOAuthClient } from '@server/types/models' -import { sha1 } from '@shared/extra-utils' -import { HttpStatusCode, ServerErrorCode, UserRegistrationState } from '@shared/models' -import { OTP } from '../../initializers/constants' -import { BypassLogin, getClient, getRefreshToken, getUser, revokeToken, saveToken } from './oauth-model' - -class MissingTwoFactorError extends Error { - code = HttpStatusCode.UNAUTHORIZED_401 - name = ServerErrorCode.MISSING_TWO_FACTOR -} - -class InvalidTwoFactorError extends Error { - code = HttpStatusCode.BAD_REQUEST_400 - name = ServerErrorCode.INVALID_TWO_FACTOR -} - -class RegistrationWaitingForApproval extends Error { - code = HttpStatusCode.BAD_REQUEST_400 - name = ServerErrorCode.ACCOUNT_WAITING_FOR_APPROVAL -} - -class RegistrationApprovalRejected extends Error { - code = HttpStatusCode.BAD_REQUEST_400 - name = ServerErrorCode.ACCOUNT_APPROVAL_REJECTED -} - -/** - * - * Reimplement some functions of OAuth2Server to inject external auth methods - * - */ -const oAuthServer = new OAuth2Server({ - // Wants seconds - accessTokenLifetime: CONFIG.OAUTH2.TOKEN_LIFETIME.ACCESS_TOKEN / 1000, - refreshTokenLifetime: CONFIG.OAUTH2.TOKEN_LIFETIME.REFRESH_TOKEN / 1000, - - // See https://github.com/oauthjs/node-oauth2-server/wiki/Model-specification for the model specifications - model: require('./oauth-model') -}) - -// --------------------------------------------------------------------------- - -async function handleOAuthToken (req: express.Request, options: { refreshTokenAuthName?: string, bypassLogin?: BypassLogin }) { - const request = new Request(req) - const { refreshTokenAuthName, bypassLogin } = options - - if (request.method !== 'POST') { - throw new InvalidRequestError('Invalid request: method must be POST') - } - - if (!request.is([ 'application/x-www-form-urlencoded' ])) { - throw new InvalidRequestError('Invalid request: content must be application/x-www-form-urlencoded') - } - - const clientId = request.body.client_id - const clientSecret = request.body.client_secret - - if (!clientId || !clientSecret) { - throw new InvalidClientError('Invalid client: cannot retrieve client credentials') - } - - const client = await getClient(clientId, clientSecret) - if (!client) { - throw new InvalidClientError('Invalid client: client is invalid') - } - - const grantType = request.body.grant_type - if (!grantType) { - throw new InvalidRequestError('Missing parameter: `grant_type`') - } - - if (![ 'password', 'refresh_token' ].includes(grantType)) { - throw new UnsupportedGrantTypeError('Unsupported grant type: `grant_type` is invalid') - } - - if (!client.grants.includes(grantType)) { - throw new UnauthorizedClientError('Unauthorized client: `grant_type` is invalid') - } - - if (grantType === 'password') { - return handlePasswordGrant({ - request, - client, - bypassLogin - }) - } - - return handleRefreshGrant({ - request, - client, - refreshTokenAuthName - }) -} - -function handleOAuthAuthenticate ( - req: express.Request, - res: express.Response -) { - return oAuthServer.authenticate(new Request(req), new Response(res)) -} - -export { - MissingTwoFactorError, - InvalidTwoFactorError, - - handleOAuthToken, - handleOAuthAuthenticate -} - -// --------------------------------------------------------------------------- - -async function handlePasswordGrant (options: { - request: Request - client: MOAuthClient - bypassLogin?: BypassLogin -}) { - const { request, client, bypassLogin } = options - - if (!request.body.username) { - throw new InvalidRequestError('Missing parameter: `username`') - } - - if (!bypassLogin && !request.body.password) { - throw new InvalidRequestError('Missing parameter: `password`') - } - - const user = await getUser(request.body.username, request.body.password, bypassLogin) - if (!user) { - const registration = await UserRegistrationModel.loadByEmailOrUsername(request.body.username) - - if (registration?.state === UserRegistrationState.REJECTED) { - throw new RegistrationApprovalRejected('Registration approval for this account has been rejected') - } else if (registration?.state === UserRegistrationState.PENDING) { - throw new RegistrationWaitingForApproval('Registration for this account is awaiting approval') - } - - throw new InvalidGrantError('Invalid grant: user credentials are invalid') - } - - if (user.otpSecret) { - if (!request.headers[OTP.HEADER_NAME]) { - throw new MissingTwoFactorError('Missing two factor header') - } - - if (await isOTPValid({ encryptedSecret: user.otpSecret, token: request.headers[OTP.HEADER_NAME] }) !== true) { - throw new InvalidTwoFactorError('Invalid two factor header') - } - } - - const token = await buildToken() - - return saveToken(token, client, user, { bypassLogin }) -} - -async function handleRefreshGrant (options: { - request: Request - client: MOAuthClient - refreshTokenAuthName: string -}) { - const { request, client, refreshTokenAuthName } = options - - if (!request.body.refresh_token) { - throw new InvalidRequestError('Missing parameter: `refresh_token`') - } - - const refreshToken = await getRefreshToken(request.body.refresh_token) - - if (!refreshToken) { - throw new InvalidGrantError('Invalid grant: refresh token is invalid') - } - - if (refreshToken.client.id !== client.id) { - throw new InvalidGrantError('Invalid grant: refresh token is invalid') - } - - if (refreshToken.refreshTokenExpiresAt && refreshToken.refreshTokenExpiresAt < new Date()) { - throw new InvalidGrantError('Invalid grant: refresh token has expired') - } - - await revokeToken({ refreshToken: refreshToken.refreshToken }) - - const token = await buildToken() - - return saveToken(token, client, refreshToken.user, { refreshTokenAuthName }) -} - -function generateRandomToken () { - return randomBytesPromise(256) - .then(buffer => sha1(buffer)) -} - -function getTokenExpiresAt (type: 'access' | 'refresh') { - const lifetime = type === 'access' - ? CONFIG.OAUTH2.TOKEN_LIFETIME.ACCESS_TOKEN - : CONFIG.OAUTH2.TOKEN_LIFETIME.REFRESH_TOKEN - - return new Date(Date.now() + lifetime) -} - -async function buildToken () { - const [ accessToken, refreshToken ] = await Promise.all([ generateRandomToken(), generateRandomToken() ]) - - return { - accessToken, - refreshToken, - accessTokenExpiresAt: getTokenExpiresAt('access'), - refreshTokenExpiresAt: getTokenExpiresAt('refresh') - } -} diff --git a/server/lib/auth/tokens-cache.ts b/server/lib/auth/tokens-cache.ts deleted file mode 100644 index e7b12159b..000000000 --- a/server/lib/auth/tokens-cache.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { LRUCache } from 'lru-cache' -import { MOAuthTokenUser } from '@server/types/models' -import { LRU_CACHE } from '../../initializers/constants' - -export class TokensCache { - - private static instance: TokensCache - - private readonly accessTokenCache = new LRUCache({ max: LRU_CACHE.USER_TOKENS.MAX_SIZE }) - private readonly userHavingToken = new LRUCache({ max: LRU_CACHE.USER_TOKENS.MAX_SIZE }) - - private constructor () { } - - static get Instance () { - return this.instance || (this.instance = new this()) - } - - hasToken (token: string) { - return this.accessTokenCache.has(token) - } - - getByToken (token: string) { - return this.accessTokenCache.get(token) - } - - setToken (token: MOAuthTokenUser) { - this.accessTokenCache.set(token.accessToken, token) - this.userHavingToken.set(token.userId, token.accessToken) - } - - deleteUserToken (userId: number) { - this.clearCacheByUserId(userId) - } - - clearCacheByUserId (userId: number) { - const token = this.userHavingToken.get(userId) - - if (token !== undefined) { - this.accessTokenCache.delete(token) - this.userHavingToken.delete(userId) - } - } - - clearCacheByToken (token: string) { - const tokenModel = this.accessTokenCache.get(token) - - if (tokenModel !== undefined) { - this.userHavingToken.delete(tokenModel.userId) - this.accessTokenCache.delete(token) - } - } -} diff --git a/server/lib/blocklist.ts b/server/lib/blocklist.ts deleted file mode 100644 index 009e229ce..000000000 --- a/server/lib/blocklist.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { sequelizeTypescript } from '@server/initializers/database' -import { getServerActor } from '@server/models/application/application' -import { MAccountBlocklist, MAccountId, MAccountHost, MServerBlocklist } from '@server/types/models' -import { AccountBlocklistModel } from '../models/account/account-blocklist' -import { ServerBlocklistModel } from '../models/server/server-blocklist' - -function addAccountInBlocklist (byAccountId: number, targetAccountId: number) { - return sequelizeTypescript.transaction(async t => { - return AccountBlocklistModel.upsert({ - accountId: byAccountId, - targetAccountId - }, { transaction: t }) - }) -} - -function addServerInBlocklist (byAccountId: number, targetServerId: number) { - return sequelizeTypescript.transaction(async t => { - return ServerBlocklistModel.upsert({ - accountId: byAccountId, - targetServerId - }, { transaction: t }) - }) -} - -function removeAccountFromBlocklist (accountBlock: MAccountBlocklist) { - return sequelizeTypescript.transaction(async t => { - return accountBlock.destroy({ transaction: t }) - }) -} - -function removeServerFromBlocklist (serverBlock: MServerBlocklist) { - return sequelizeTypescript.transaction(async t => { - return serverBlock.destroy({ transaction: t }) - }) -} - -async function isBlockedByServerOrAccount (targetAccount: MAccountHost, userAccount?: MAccountId) { - const serverAccountId = (await getServerActor()).Account.id - const sourceAccounts = [ serverAccountId ] - - if (userAccount) sourceAccounts.push(userAccount.id) - - const accountMutedHash = await AccountBlocklistModel.isAccountMutedByAccounts(sourceAccounts, targetAccount.id) - if (accountMutedHash[serverAccountId] || (userAccount && accountMutedHash[userAccount.id])) { - return true - } - - const instanceMutedHash = await ServerBlocklistModel.isServerMutedByAccounts(sourceAccounts, targetAccount.Actor.serverId) - if (instanceMutedHash[serverAccountId] || (userAccount && instanceMutedHash[userAccount.id])) { - return true - } - - return false -} - -export { - addAccountInBlocklist, - addServerInBlocklist, - removeAccountFromBlocklist, - removeServerFromBlocklist, - isBlockedByServerOrAccount -} diff --git a/server/lib/client-html.ts b/server/lib/client-html.ts deleted file mode 100644 index 8e0c9e328..000000000 --- a/server/lib/client-html.ts +++ /dev/null @@ -1,623 +0,0 @@ -import express from 'express' -import { pathExists, readFile } from 'fs-extra' -import { truncate } from 'lodash' -import { join } from 'path' -import validator from 'validator' -import { isTestOrDevInstance } from '@server/helpers/core-utils' -import { toCompleteUUID } from '@server/helpers/custom-validators/misc' -import { mdToOneLinePlainText } from '@server/helpers/markdown' -import { ActorImageModel } from '@server/models/actor/actor-image' -import { root } from '@shared/core-utils' -import { escapeHTML } from '@shared/core-utils/renderer' -import { sha256 } from '@shared/extra-utils' -import { HTMLServerConfig } from '@shared/models' -import { buildFileLocale, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '../../shared/core-utils/i18n/i18n' -import { HttpStatusCode } from '../../shared/models/http/http-error-codes' -import { VideoPlaylistPrivacy, VideoPrivacy } from '../../shared/models/videos' -import { logger } from '../helpers/logger' -import { CONFIG } from '../initializers/config' -import { - ACCEPT_HEADERS, - CUSTOM_HTML_TAG_COMMENTS, - EMBED_SIZE, - FILES_CONTENT_HASH, - PLUGIN_GLOBAL_CSS_PATH, - WEBSERVER -} from '../initializers/constants' -import { AccountModel } from '../models/account/account' -import { VideoModel } from '../models/video/video' -import { VideoChannelModel } from '../models/video/video-channel' -import { VideoPlaylistModel } from '../models/video/video-playlist' -import { MAccountHost, MChannelHost, MVideo, MVideoPlaylist } from '../types/models' -import { getActivityStreamDuration } from './activitypub/activity' -import { getBiggestActorImage } from './actor-image' -import { Hooks } from './plugins/hooks' -import { ServerConfigManager } from './server-config-manager' -import { isVideoInPrivateDirectory } from './video-privacy' - -type Tags = { - ogType: string - twitterCard: 'player' | 'summary' | 'summary_large_image' - schemaType: string - - list?: { - numberOfItems: number - } - - escapedSiteName: string - escapedTitle: string - escapedTruncatedDescription: string - - url: string - originUrl: string - - disallowIndexation?: boolean - - embed?: { - url: string - createdAt: string - duration?: string - views?: number - } - - image: { - url: string - width?: number - height?: number - } -} - -type HookContext = { - video?: MVideo - playlist?: MVideoPlaylist -} - -class ClientHtml { - - private static htmlCache: { [path: string]: string } = {} - - static invalidCache () { - logger.info('Cleaning HTML cache.') - - ClientHtml.htmlCache = {} - } - - static async getDefaultHTMLPage (req: express.Request, res: express.Response, paramLang?: string) { - const html = paramLang - ? await ClientHtml.getIndexHTML(req, res, paramLang) - : await ClientHtml.getIndexHTML(req, res) - - let customHtml = ClientHtml.addTitleTag(html) - customHtml = ClientHtml.addDescriptionTag(customHtml) - - return customHtml - } - - static async getWatchHTMLPage (videoIdArg: string, req: express.Request, res: express.Response) { - const videoId = toCompleteUUID(videoIdArg) - - // Let Angular application handle errors - if (!validator.isInt(videoId) && !validator.isUUID(videoId, 4)) { - res.status(HttpStatusCode.NOT_FOUND_404) - return ClientHtml.getIndexHTML(req, res) - } - - const [ html, video ] = await Promise.all([ - ClientHtml.getIndexHTML(req, res), - VideoModel.loadWithBlacklist(videoId) - ]) - - // Let Angular application handle errors - if (!video || isVideoInPrivateDirectory(video.privacy) || video.VideoBlacklist) { - res.status(HttpStatusCode.NOT_FOUND_404) - return html - } - const escapedTruncatedDescription = buildEscapedTruncatedDescription(video.description) - - let customHtml = ClientHtml.addTitleTag(html, video.name) - customHtml = ClientHtml.addDescriptionTag(customHtml, escapedTruncatedDescription) - - const url = WEBSERVER.URL + video.getWatchStaticPath() - const originUrl = video.url - const title = video.name - const siteName = CONFIG.INSTANCE.NAME - - const image = { - url: WEBSERVER.URL + video.getPreviewStaticPath() - } - - const embed = { - url: WEBSERVER.URL + video.getEmbedStaticPath(), - createdAt: video.createdAt.toISOString(), - duration: getActivityStreamDuration(video.duration), - views: video.views - } - - const ogType = 'video' - const twitterCard = CONFIG.SERVICES.TWITTER.WHITELISTED ? 'player' : 'summary_large_image' - const schemaType = 'VideoObject' - - customHtml = await ClientHtml.addTags(customHtml, { - url, - originUrl, - escapedSiteName: escapeHTML(siteName), - escapedTitle: escapeHTML(title), - escapedTruncatedDescription, - disallowIndexation: video.privacy !== VideoPrivacy.PUBLIC, - image, - embed, - ogType, - twitterCard, - schemaType - }, { video }) - - return customHtml - } - - static async getWatchPlaylistHTMLPage (videoPlaylistIdArg: string, req: express.Request, res: express.Response) { - const videoPlaylistId = toCompleteUUID(videoPlaylistIdArg) - - // Let Angular application handle errors - if (!validator.isInt(videoPlaylistId) && !validator.isUUID(videoPlaylistId, 4)) { - res.status(HttpStatusCode.NOT_FOUND_404) - return ClientHtml.getIndexHTML(req, res) - } - - const [ html, videoPlaylist ] = await Promise.all([ - ClientHtml.getIndexHTML(req, res), - VideoPlaylistModel.loadWithAccountAndChannel(videoPlaylistId, null) - ]) - - // Let Angular application handle errors - if (!videoPlaylist || videoPlaylist.privacy === VideoPlaylistPrivacy.PRIVATE) { - res.status(HttpStatusCode.NOT_FOUND_404) - return html - } - - const escapedTruncatedDescription = buildEscapedTruncatedDescription(videoPlaylist.description) - - let customHtml = ClientHtml.addTitleTag(html, videoPlaylist.name) - customHtml = ClientHtml.addDescriptionTag(customHtml, escapedTruncatedDescription) - - const url = WEBSERVER.URL + videoPlaylist.getWatchStaticPath() - const originUrl = videoPlaylist.url - const title = videoPlaylist.name - const siteName = CONFIG.INSTANCE.NAME - - const image = { - url: videoPlaylist.getThumbnailUrl() - } - - const embed = { - url: WEBSERVER.URL + videoPlaylist.getEmbedStaticPath(), - createdAt: videoPlaylist.createdAt.toISOString() - } - - const list = { - numberOfItems: videoPlaylist.get('videosLength') as number - } - - const ogType = 'video' - const twitterCard = CONFIG.SERVICES.TWITTER.WHITELISTED ? 'player' : 'summary' - const schemaType = 'ItemList' - - customHtml = await ClientHtml.addTags(customHtml, { - url, - originUrl, - escapedSiteName: escapeHTML(siteName), - escapedTitle: escapeHTML(title), - escapedTruncatedDescription, - disallowIndexation: videoPlaylist.privacy !== VideoPlaylistPrivacy.PUBLIC, - embed, - image, - list, - ogType, - twitterCard, - schemaType - }, { playlist: videoPlaylist }) - - return customHtml - } - - static async getAccountHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) { - const accountModelPromise = AccountModel.loadByNameWithHost(nameWithHost) - return this.getAccountOrChannelHTMLPage(() => accountModelPromise, req, res) - } - - static async getVideoChannelHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) { - const videoChannelModelPromise = VideoChannelModel.loadByNameWithHostAndPopulateAccount(nameWithHost) - return this.getAccountOrChannelHTMLPage(() => videoChannelModelPromise, req, res) - } - - static async getActorHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) { - const [ account, channel ] = await Promise.all([ - AccountModel.loadByNameWithHost(nameWithHost), - VideoChannelModel.loadByNameWithHostAndPopulateAccount(nameWithHost) - ]) - - return this.getAccountOrChannelHTMLPage(() => Promise.resolve(account || channel), req, res) - } - - static async getEmbedHTML () { - const path = ClientHtml.getEmbedPath() - - // Disable HTML cache in dev mode because webpack can regenerate JS files - if (!isTestOrDevInstance() && ClientHtml.htmlCache[path]) { - return ClientHtml.htmlCache[path] - } - - const buffer = await readFile(path) - const serverConfig = await ServerConfigManager.Instance.getHTMLServerConfig() - - let html = buffer.toString() - html = await ClientHtml.addAsyncPluginCSS(html) - html = ClientHtml.addCustomCSS(html) - html = ClientHtml.addTitleTag(html) - html = ClientHtml.addDescriptionTag(html) - html = ClientHtml.addServerConfig(html, serverConfig) - - ClientHtml.htmlCache[path] = html - - return html - } - - private static async getAccountOrChannelHTMLPage ( - loader: () => Promise, - req: express.Request, - res: express.Response - ) { - const [ html, entity ] = await Promise.all([ - ClientHtml.getIndexHTML(req, res), - loader() - ]) - - // Let Angular application handle errors - if (!entity) { - res.status(HttpStatusCode.NOT_FOUND_404) - return ClientHtml.getIndexHTML(req, res) - } - - const escapedTruncatedDescription = buildEscapedTruncatedDescription(entity.description) - - let customHtml = ClientHtml.addTitleTag(html, entity.getDisplayName()) - customHtml = ClientHtml.addDescriptionTag(customHtml, escapedTruncatedDescription) - - const url = entity.getClientUrl() - const originUrl = entity.Actor.url - const siteName = CONFIG.INSTANCE.NAME - const title = entity.getDisplayName() - - const avatar = getBiggestActorImage(entity.Actor.Avatars) - const image = { - url: ActorImageModel.getImageUrl(avatar), - width: avatar?.width, - height: avatar?.height - } - - const ogType = 'website' - const twitterCard = 'summary' - const schemaType = 'ProfilePage' - - customHtml = await ClientHtml.addTags(customHtml, { - url, - originUrl, - escapedTitle: escapeHTML(title), - escapedSiteName: escapeHTML(siteName), - escapedTruncatedDescription, - image, - ogType, - twitterCard, - schemaType, - disallowIndexation: !entity.Actor.isOwned() - }, {}) - - return customHtml - } - - private static async getIndexHTML (req: express.Request, res: express.Response, paramLang?: string) { - const path = ClientHtml.getIndexPath(req, res, paramLang) - if (ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path] - - const buffer = await readFile(path) - const serverConfig = await ServerConfigManager.Instance.getHTMLServerConfig() - - let html = buffer.toString() - - html = ClientHtml.addManifestContentHash(html) - html = ClientHtml.addFaviconContentHash(html) - html = ClientHtml.addLogoContentHash(html) - html = ClientHtml.addCustomCSS(html) - html = ClientHtml.addServerConfig(html, serverConfig) - html = await ClientHtml.addAsyncPluginCSS(html) - - ClientHtml.htmlCache[path] = html - - return html - } - - private static getIndexPath (req: express.Request, res: express.Response, paramLang: string) { - let lang: string - - // Check param lang validity - if (paramLang && is18nLocale(paramLang)) { - lang = paramLang - - // Save locale in cookies - res.cookie('clientLanguage', lang, { - secure: WEBSERVER.SCHEME === 'https', - sameSite: 'none', - maxAge: 1000 * 3600 * 24 * 90 // 3 months - }) - - } else if (req.cookies.clientLanguage && is18nLocale(req.cookies.clientLanguage)) { - lang = req.cookies.clientLanguage - } else { - lang = req.acceptsLanguages(POSSIBLE_LOCALES) || getDefaultLocale() - } - - logger.debug( - 'Serving %s HTML language', buildFileLocale(lang), - { cookie: req.cookies?.clientLanguage, paramLang, acceptLanguage: req.headers['accept-language'] } - ) - - return join(root(), 'client', 'dist', buildFileLocale(lang), 'index.html') - } - - private static getEmbedPath () { - return join(root(), 'client', 'dist', 'standalone', 'videos', 'embed.html') - } - - private static addManifestContentHash (htmlStringPage: string) { - return htmlStringPage.replace('[manifestContentHash]', FILES_CONTENT_HASH.MANIFEST) - } - - private static addFaviconContentHash (htmlStringPage: string) { - return htmlStringPage.replace('[faviconContentHash]', FILES_CONTENT_HASH.FAVICON) - } - - private static addLogoContentHash (htmlStringPage: string) { - return htmlStringPage.replace('[logoContentHash]', FILES_CONTENT_HASH.LOGO) - } - - private static addTitleTag (htmlStringPage: string, title?: string) { - let text = title || CONFIG.INSTANCE.NAME - if (title) text += ` - ${CONFIG.INSTANCE.NAME}` - - const titleTag = `${escapeHTML(text)}` - - return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.TITLE, titleTag) - } - - private static addDescriptionTag (htmlStringPage: string, escapedTruncatedDescription?: string) { - const content = escapedTruncatedDescription || escapeHTML(CONFIG.INSTANCE.SHORT_DESCRIPTION) - const descriptionTag = `` - - return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.DESCRIPTION, descriptionTag) - } - - private static addCustomCSS (htmlStringPage: string) { - const styleTag = `` - - return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.CUSTOM_CSS, styleTag) - } - - private static addServerConfig (htmlStringPage: string, serverConfig: HTMLServerConfig) { - // Stringify the JSON object, and then stringify the string object so we can inject it into the HTML - const serverConfigString = JSON.stringify(JSON.stringify(serverConfig)) - const configScriptTag = `` - - return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.SERVER_CONFIG, configScriptTag) - } - - private static async addAsyncPluginCSS (htmlStringPage: string) { - if (!pathExists(PLUGIN_GLOBAL_CSS_PATH)) { - logger.info('Plugin Global CSS file is not available (generation may still be in progress), ignoring it.') - return htmlStringPage - } - - let globalCSSContent: Buffer - - try { - globalCSSContent = await readFile(PLUGIN_GLOBAL_CSS_PATH) - } catch (err) { - logger.error('Error retrieving the Plugin Global CSS file, ignoring it.', { err }) - return htmlStringPage - } - - if (globalCSSContent.byteLength === 0) return htmlStringPage - - const fileHash = sha256(globalCSSContent) - const linkTag = `` - - return htmlStringPage.replace('', linkTag + '') - } - - private static generateOpenGraphMetaTags (tags: Tags) { - const metaTags = { - 'og:type': tags.ogType, - 'og:site_name': tags.escapedSiteName, - 'og:title': tags.escapedTitle, - 'og:image': tags.image.url - } - - if (tags.image.width && tags.image.height) { - metaTags['og:image:width'] = tags.image.width - metaTags['og:image:height'] = tags.image.height - } - - metaTags['og:url'] = tags.url - metaTags['og:description'] = tags.escapedTruncatedDescription - - if (tags.embed) { - metaTags['og:video:url'] = tags.embed.url - metaTags['og:video:secure_url'] = tags.embed.url - metaTags['og:video:type'] = 'text/html' - metaTags['og:video:width'] = EMBED_SIZE.width - metaTags['og:video:height'] = EMBED_SIZE.height - } - - return metaTags - } - - private static generateStandardMetaTags (tags: Tags) { - return { - name: tags.escapedTitle, - description: tags.escapedTruncatedDescription, - image: tags.image.url - } - } - - private static generateTwitterCardMetaTags (tags: Tags) { - const metaTags = { - 'twitter:card': tags.twitterCard, - 'twitter:site': CONFIG.SERVICES.TWITTER.USERNAME, - 'twitter:title': tags.escapedTitle, - 'twitter:description': tags.escapedTruncatedDescription, - 'twitter:image': tags.image.url - } - - if (tags.image.width && tags.image.height) { - metaTags['twitter:image:width'] = tags.image.width - metaTags['twitter:image:height'] = tags.image.height - } - - if (tags.twitterCard === 'player') { - metaTags['twitter:player'] = tags.embed.url - metaTags['twitter:player:width'] = EMBED_SIZE.width - metaTags['twitter:player:height'] = EMBED_SIZE.height - } - - return metaTags - } - - private static async generateSchemaTags (tags: Tags, context: HookContext) { - const schema = { - '@context': 'http://schema.org', - '@type': tags.schemaType, - 'name': tags.escapedTitle, - 'description': tags.escapedTruncatedDescription, - 'image': tags.image.url, - 'url': tags.url - } - - if (tags.list) { - schema['numberOfItems'] = tags.list.numberOfItems - schema['thumbnailUrl'] = tags.image.url - } - - if (tags.embed) { - schema['embedUrl'] = tags.embed.url - schema['uploadDate'] = tags.embed.createdAt - - if (tags.embed.duration) schema['duration'] = tags.embed.duration - - schema['thumbnailUrl'] = tags.image.url - schema['contentUrl'] = tags.url - } - - return Hooks.wrapObject(schema, 'filter:html.client.json-ld.result', context) - } - - private static async addTags (htmlStringPage: string, tagsValues: Tags, context: HookContext) { - const openGraphMetaTags = this.generateOpenGraphMetaTags(tagsValues) - const standardMetaTags = this.generateStandardMetaTags(tagsValues) - const twitterCardMetaTags = this.generateTwitterCardMetaTags(tagsValues) - const schemaTags = await this.generateSchemaTags(tagsValues, context) - - const { url, escapedTitle, embed, originUrl, disallowIndexation } = tagsValues - - const oembedLinkTags: { type: string, href: string, escapedTitle: string }[] = [] - - if (embed) { - oembedLinkTags.push({ - type: 'application/json+oembed', - href: WEBSERVER.URL + '/services/oembed?url=' + encodeURIComponent(url), - escapedTitle - }) - } - - let tagsStr = '' - - // Opengraph - Object.keys(openGraphMetaTags).forEach(tagName => { - const tagValue = openGraphMetaTags[tagName] - - tagsStr += `` - }) - - // Standard - Object.keys(standardMetaTags).forEach(tagName => { - const tagValue = standardMetaTags[tagName] - - tagsStr += `` - }) - - // Twitter card - Object.keys(twitterCardMetaTags).forEach(tagName => { - const tagValue = twitterCardMetaTags[tagName] - - tagsStr += `` - }) - - // OEmbed - for (const oembedLinkTag of oembedLinkTags) { - tagsStr += `` - } - - // Schema.org - if (schemaTags) { - tagsStr += `` - } - - // SEO, use origin URL - tagsStr += `` - - if (disallowIndexation) { - tagsStr += `` - } - - return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.META_TAGS, tagsStr) - } -} - -function sendHTML (html: string, res: express.Response, localizedHTML: boolean = false) { - res.set('Content-Type', 'text/html; charset=UTF-8') - - if (localizedHTML) { - res.set('Vary', 'Accept-Language') - } - - return res.send(html) -} - -async function serveIndexHTML (req: express.Request, res: express.Response) { - if (req.accepts(ACCEPT_HEADERS) === 'html' || !req.headers.accept) { - try { - await generateHTMLPage(req, res, req.params.language) - return - } catch (err) { - logger.error('Cannot generate HTML page.', { err }) - return res.status(HttpStatusCode.INTERNAL_SERVER_ERROR_500).end() - } - } - - return res.status(HttpStatusCode.NOT_ACCEPTABLE_406).end() -} - -// --------------------------------------------------------------------------- - -export { - ClientHtml, - sendHTML, - serveIndexHTML -} - -async function generateHTMLPage (req: express.Request, res: express.Response, paramLang?: string) { - const html = await ClientHtml.getDefaultHTMLPage(req, res, paramLang) - - return sendHTML(html, res, true) -} - -function buildEscapedTruncatedDescription (description: string) { - return truncate(mdToOneLinePlainText(description), { length: 200 }) -} diff --git a/server/lib/emailer.ts b/server/lib/emailer.ts deleted file mode 100644 index f5c3e4745..000000000 --- a/server/lib/emailer.ts +++ /dev/null @@ -1,284 +0,0 @@ -import { readFileSync } from 'fs-extra' -import { merge } from 'lodash' -import { createTransport, Transporter } from 'nodemailer' -import { join } from 'path' -import { arrayify, root } from '@shared/core-utils' -import { EmailPayload, UserRegistrationState } from '@shared/models' -import { SendEmailDefaultOptions } from '../../shared/models/server/emailer.model' -import { isTestOrDevInstance } from '../helpers/core-utils' -import { bunyanLogger, logger } from '../helpers/logger' -import { CONFIG, isEmailEnabled } from '../initializers/config' -import { WEBSERVER } from '../initializers/constants' -import { MRegistration, MUser } from '../types/models' -import { JobQueue } from './job-queue' - -const Email = require('email-templates') - -class Emailer { - - private static instance: Emailer - private initialized = false - private transporter: Transporter - - private constructor () { - } - - init () { - // Already initialized - if (this.initialized === true) return - this.initialized = true - - if (!isEmailEnabled()) { - if (!isTestOrDevInstance()) { - logger.error('Cannot use SMTP server because of lack of configuration. PeerTube will not be able to send mails!') - } - - return - } - - if (CONFIG.SMTP.TRANSPORT === 'smtp') this.initSMTPTransport() - else if (CONFIG.SMTP.TRANSPORT === 'sendmail') this.initSendmailTransport() - } - - async checkConnection () { - if (!this.transporter || CONFIG.SMTP.TRANSPORT !== 'smtp') return - - logger.info('Testing SMTP server...') - - try { - const success = await this.transporter.verify() - if (success !== true) this.warnOnConnectionFailure() - - logger.info('Successfully connected to SMTP server.') - } catch (err) { - this.warnOnConnectionFailure(err) - } - } - - addPasswordResetEmailJob (username: string, to: string, resetPasswordUrl: string) { - const emailPayload: EmailPayload = { - template: 'password-reset', - to: [ to ], - subject: 'Reset your account password', - locals: { - username, - resetPasswordUrl, - - hideNotificationPreferencesLink: true - } - } - - return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) - } - - addPasswordCreateEmailJob (username: string, to: string, createPasswordUrl: string) { - const emailPayload: EmailPayload = { - template: 'password-create', - to: [ to ], - subject: 'Create your account password', - locals: { - username, - createPasswordUrl, - - hideNotificationPreferencesLink: true - } - } - - return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) - } - - addVerifyEmailJob (options: { - username: string - isRegistrationRequest: boolean - to: string - verifyEmailUrl: string - }) { - const { username, isRegistrationRequest, to, verifyEmailUrl } = options - - const emailPayload: EmailPayload = { - template: 'verify-email', - to: [ to ], - subject: `Verify your email on ${CONFIG.INSTANCE.NAME}`, - locals: { - username, - verifyEmailUrl, - isRegistrationRequest, - - hideNotificationPreferencesLink: true - } - } - - return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) - } - - addUserBlockJob (user: MUser, blocked: boolean, reason?: string) { - const reasonString = reason ? ` for the following reason: ${reason}` : '' - const blockedWord = blocked ? 'blocked' : 'unblocked' - - const to = user.email - const emailPayload: EmailPayload = { - to: [ to ], - subject: 'Account ' + blockedWord, - text: `Your account ${user.username} on ${CONFIG.INSTANCE.NAME} has been ${blockedWord}${reasonString}.` - } - - return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) - } - - addContactFormJob (fromEmail: string, fromName: string, subject: string, body: string) { - const emailPayload: EmailPayload = { - template: 'contact-form', - to: [ CONFIG.ADMIN.EMAIL ], - replyTo: `"${fromName}" <${fromEmail}>`, - subject: `(contact form) ${subject}`, - locals: { - fromName, - fromEmail, - body, - - // There are not notification preferences for the contact form - hideNotificationPreferencesLink: true - } - } - - return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) - } - - addUserRegistrationRequestProcessedJob (registration: MRegistration) { - let template: string - let subject: string - if (registration.state === UserRegistrationState.ACCEPTED) { - template = 'user-registration-request-accepted' - subject = `Your registration request for ${registration.username} has been accepted` - } else { - template = 'user-registration-request-rejected' - subject = `Your registration request for ${registration.username} has been rejected` - } - - const to = registration.email - const emailPayload: EmailPayload = { - to: [ to ], - template, - subject, - locals: { - username: registration.username, - moderationResponse: registration.moderationResponse, - loginLink: WEBSERVER.URL + '/login' - } - } - - return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) - } - - async sendMail (options: EmailPayload) { - if (!isEmailEnabled()) { - logger.info('Cannot send mail because SMTP is not configured.') - return - } - - const fromDisplayName = options.from - ? options.from - : CONFIG.INSTANCE.NAME - - const email = new Email({ - send: true, - htmlToText: { - selectors: [ - { selector: 'img', format: 'skip' }, - { selector: 'a', options: { hideLinkHrefIfSameAsText: true } } - ] - }, - message: { - from: `"${fromDisplayName}" <${CONFIG.SMTP.FROM_ADDRESS}>` - }, - transport: this.transporter, - views: { - root: join(root(), 'dist', 'server', 'lib', 'emails') - }, - subjectPrefix: CONFIG.EMAIL.SUBJECT.PREFIX - }) - - const toEmails = arrayify(options.to) - - for (const to of toEmails) { - const baseOptions: SendEmailDefaultOptions = { - template: 'common', - message: { - to, - from: options.from, - subject: options.subject, - replyTo: options.replyTo - }, - locals: { // default variables available in all templates - WEBSERVER, - EMAIL: CONFIG.EMAIL, - instanceName: CONFIG.INSTANCE.NAME, - text: options.text, - subject: options.subject - } - } - - // overridden/new variables given for a specific template in the payload - const sendOptions = merge(baseOptions, options) - - await email.send(sendOptions) - .then(res => logger.debug('Sent email.', { res })) - .catch(err => logger.error('Error in email sender.', { err })) - } - } - - private warnOnConnectionFailure (err?: Error) { - logger.error('Failed to connect to SMTP %s:%d.', CONFIG.SMTP.HOSTNAME, CONFIG.SMTP.PORT, { err }) - } - - private initSMTPTransport () { - logger.info('Using %s:%s as SMTP server.', CONFIG.SMTP.HOSTNAME, CONFIG.SMTP.PORT) - - let tls - if (CONFIG.SMTP.CA_FILE) { - tls = { - ca: [ readFileSync(CONFIG.SMTP.CA_FILE) ] - } - } - - let auth - if (CONFIG.SMTP.USERNAME && CONFIG.SMTP.PASSWORD) { - auth = { - user: CONFIG.SMTP.USERNAME, - pass: CONFIG.SMTP.PASSWORD - } - } - - this.transporter = createTransport({ - host: CONFIG.SMTP.HOSTNAME, - port: CONFIG.SMTP.PORT, - secure: CONFIG.SMTP.TLS, - debug: CONFIG.LOG.LEVEL === 'debug', - logger: bunyanLogger as any, - ignoreTLS: CONFIG.SMTP.DISABLE_STARTTLS, - tls, - auth - }) - } - - private initSendmailTransport () { - logger.info('Using sendmail to send emails') - - this.transporter = createTransport({ - sendmail: true, - newline: 'unix', - path: CONFIG.SMTP.SENDMAIL, - logger: bunyanLogger - }) - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -// --------------------------------------------------------------------------- - -export { - Emailer -} diff --git a/server/lib/files-cache/avatar-permanent-file-cache.ts b/server/lib/files-cache/avatar-permanent-file-cache.ts deleted file mode 100644 index 0c508b063..000000000 --- a/server/lib/files-cache/avatar-permanent-file-cache.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { CONFIG } from '@server/initializers/config' -import { ACTOR_IMAGES_SIZE } from '@server/initializers/constants' -import { ActorImageModel } from '@server/models/actor/actor-image' -import { MActorImage } from '@server/types/models' -import { AbstractPermanentFileCache } from './shared' - -export class AvatarPermanentFileCache extends AbstractPermanentFileCache { - - constructor () { - super(CONFIG.STORAGE.ACTOR_IMAGES_DIR) - } - - protected loadModel (filename: string) { - return ActorImageModel.loadByName(filename) - } - - protected getImageSize (image: MActorImage): { width: number, height: number } { - if (image.width && image.height) { - return { - height: image.height, - width: image.width - } - } - - return ACTOR_IMAGES_SIZE[image.type][0] - } -} diff --git a/server/lib/files-cache/index.ts b/server/lib/files-cache/index.ts deleted file mode 100644 index 5630a9b80..000000000 --- a/server/lib/files-cache/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from './avatar-permanent-file-cache' -export * from './video-miniature-permanent-file-cache' -export * from './video-captions-simple-file-cache' -export * from './video-previews-simple-file-cache' -export * from './video-storyboards-simple-file-cache' -export * from './video-torrents-simple-file-cache' diff --git a/server/lib/files-cache/shared/abstract-permanent-file-cache.ts b/server/lib/files-cache/shared/abstract-permanent-file-cache.ts deleted file mode 100644 index f990e9872..000000000 --- a/server/lib/files-cache/shared/abstract-permanent-file-cache.ts +++ /dev/null @@ -1,132 +0,0 @@ -import express from 'express' -import { LRUCache } from 'lru-cache' -import { Model } from 'sequelize' -import { logger } from '@server/helpers/logger' -import { CachePromise } from '@server/helpers/promise-cache' -import { LRU_CACHE, STATIC_MAX_AGE } from '@server/initializers/constants' -import { downloadImageFromWorker } from '@server/lib/worker/parent-process' -import { HttpStatusCode } from '@shared/models' - -type ImageModel = { - fileUrl: string - filename: string - onDisk: boolean - - isOwned (): boolean - getPath (): string - - save (): Promise -} - -export abstract class AbstractPermanentFileCache { - // Unsafe because it can return paths that do not exist anymore - private readonly filenameToPathUnsafeCache = new LRUCache({ - max: LRU_CACHE.FILENAME_TO_PATH_PERMANENT_FILE_CACHE.MAX_SIZE - }) - - protected abstract getImageSize (image: M): { width: number, height: number } - protected abstract loadModel (filename: string): Promise - - constructor (private readonly directory: string) { - - } - - async lazyServe (options: { - filename: string - res: express.Response - next: express.NextFunction - }) { - const { filename, res, next } = options - - if (this.filenameToPathUnsafeCache.has(filename)) { - return res.sendFile(this.filenameToPathUnsafeCache.get(filename), { maxAge: STATIC_MAX_AGE.SERVER }) - } - - const image = await this.lazyLoadIfNeeded(filename) - if (!image) return res.status(HttpStatusCode.NOT_FOUND_404).end() - - const path = image.getPath() - this.filenameToPathUnsafeCache.set(filename, path) - - return res.sendFile(path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }, (err: any) => { - if (!err) return - - this.onServeError({ err, image, next, filename }) - }) - } - - @CachePromise({ - keyBuilder: filename => filename - }) - private async lazyLoadIfNeeded (filename: string) { - const image = await this.loadModel(filename) - if (!image) return undefined - - if (image.onDisk === false) { - if (!image.fileUrl) return undefined - - try { - await this.downloadRemoteFile(image) - } catch (err) { - logger.warn('Cannot process remote image %s.', image.fileUrl, { err }) - - return undefined - } - } - - return image - } - - async downloadRemoteFile (image: M) { - logger.info('Download remote image %s lazily.', image.fileUrl) - - const destination = await this.downloadImage({ - filename: image.filename, - fileUrl: image.fileUrl, - size: this.getImageSize(image) - }) - - image.onDisk = true - image.save() - .catch(err => logger.error('Cannot save new image disk state.', { err })) - - return destination - } - - private onServeError (options: { - err: any - image: M - filename: string - next: express.NextFunction - }) { - const { err, image, filename, next } = options - - // It seems this actor image is not on the disk anymore - if (err.status === HttpStatusCode.NOT_FOUND_404 && !image.isOwned()) { - logger.error('Cannot lazy serve image %s.', filename, { err }) - - this.filenameToPathUnsafeCache.delete(filename) - - image.onDisk = false - image.save() - .catch(err => logger.error('Cannot save new image disk state.', { err })) - } - - return next(err) - } - - private downloadImage (options: { - fileUrl: string - filename: string - size: { width: number, height: number } - }) { - const downloaderOptions = { - url: options.fileUrl, - destDir: this.directory, - destName: options.filename, - size: options.size - } - - return downloadImageFromWorker(downloaderOptions) - } -} diff --git a/server/lib/files-cache/shared/abstract-simple-file-cache.ts b/server/lib/files-cache/shared/abstract-simple-file-cache.ts deleted file mode 100644 index 6fab322cd..000000000 --- a/server/lib/files-cache/shared/abstract-simple-file-cache.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { remove } from 'fs-extra' -import { logger } from '../../../helpers/logger' -import memoizee from 'memoizee' - -type GetFilePathResult = { isOwned: boolean, path: string, downloadName?: string } | undefined - -export abstract class AbstractSimpleFileCache { - - getFilePath: (params: T) => Promise - - abstract getFilePathImpl (params: T): Promise - - // Load and save the remote file, then return the local path from filesystem - protected abstract loadRemoteFile (key: string): Promise - - init (max: number, maxAge: number) { - this.getFilePath = memoizee(this.getFilePathImpl, { - maxAge, - max, - promise: true, - dispose: (result?: GetFilePathResult) => { - if (result && result.isOwned !== true) { - remove(result.path) - .then(() => logger.debug('%s removed from %s', result.path, this.constructor.name)) - .catch(err => logger.error('Cannot remove %s from cache %s.', result.path, this.constructor.name, { err })) - } - } - }) - } -} diff --git a/server/lib/files-cache/shared/index.ts b/server/lib/files-cache/shared/index.ts deleted file mode 100644 index 61c4aacc7..000000000 --- a/server/lib/files-cache/shared/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './abstract-permanent-file-cache' -export * from './abstract-simple-file-cache' diff --git a/server/lib/files-cache/video-captions-simple-file-cache.ts b/server/lib/files-cache/video-captions-simple-file-cache.ts deleted file mode 100644 index cbeeff732..000000000 --- a/server/lib/files-cache/video-captions-simple-file-cache.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { join } from 'path' -import { logger } from '@server/helpers/logger' -import { doRequestAndSaveToFile } from '@server/helpers/requests' -import { CONFIG } from '../../initializers/config' -import { FILES_CACHE } from '../../initializers/constants' -import { VideoModel } from '../../models/video/video' -import { VideoCaptionModel } from '../../models/video/video-caption' -import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache' - -class VideoCaptionsSimpleFileCache extends AbstractSimpleFileCache { - - private static instance: VideoCaptionsSimpleFileCache - - private constructor () { - super() - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } - - async getFilePathImpl (filename: string) { - const videoCaption = await VideoCaptionModel.loadWithVideoByFilename(filename) - if (!videoCaption) return undefined - - if (videoCaption.isOwned()) { - return { isOwned: true, path: join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.filename) } - } - - return this.loadRemoteFile(filename) - } - - // Key is the caption filename - protected async loadRemoteFile (key: string) { - const videoCaption = await VideoCaptionModel.loadWithVideoByFilename(key) - if (!videoCaption) return undefined - - if (videoCaption.isOwned()) throw new Error('Cannot load remote caption of owned video.') - - // Used to fetch the path - const video = await VideoModel.loadFull(videoCaption.videoId) - if (!video) return undefined - - const remoteUrl = videoCaption.getFileUrl(video) - const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.filename) - - try { - await doRequestAndSaveToFile(remoteUrl, destPath) - - return { isOwned: false, path: destPath } - } catch (err) { - logger.info('Cannot fetch remote caption file %s.', remoteUrl, { err }) - - return undefined - } - } -} - -export { - VideoCaptionsSimpleFileCache -} diff --git a/server/lib/files-cache/video-miniature-permanent-file-cache.ts b/server/lib/files-cache/video-miniature-permanent-file-cache.ts deleted file mode 100644 index 35d9466f7..000000000 --- a/server/lib/files-cache/video-miniature-permanent-file-cache.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { CONFIG } from '@server/initializers/config' -import { THUMBNAILS_SIZE } from '@server/initializers/constants' -import { ThumbnailModel } from '@server/models/video/thumbnail' -import { MThumbnail } from '@server/types/models' -import { ThumbnailType } from '@shared/models' -import { AbstractPermanentFileCache } from './shared' - -export class VideoMiniaturePermanentFileCache extends AbstractPermanentFileCache { - - constructor () { - super(CONFIG.STORAGE.THUMBNAILS_DIR) - } - - protected loadModel (filename: string) { - return ThumbnailModel.loadByFilename(filename, ThumbnailType.MINIATURE) - } - - protected getImageSize (image: MThumbnail): { width: number, height: number } { - if (image.width && image.height) { - return { - height: image.height, - width: image.width - } - } - - return THUMBNAILS_SIZE - } -} diff --git a/server/lib/files-cache/video-previews-simple-file-cache.ts b/server/lib/files-cache/video-previews-simple-file-cache.ts deleted file mode 100644 index a05e80e16..000000000 --- a/server/lib/files-cache/video-previews-simple-file-cache.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { join } from 'path' -import { FILES_CACHE } from '../../initializers/constants' -import { VideoModel } from '../../models/video/video' -import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache' -import { doRequestAndSaveToFile } from '@server/helpers/requests' -import { ThumbnailModel } from '@server/models/video/thumbnail' -import { ThumbnailType } from '@shared/models' -import { logger } from '@server/helpers/logger' - -class VideoPreviewsSimpleFileCache extends AbstractSimpleFileCache { - - private static instance: VideoPreviewsSimpleFileCache - - private constructor () { - super() - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } - - async getFilePathImpl (filename: string) { - const thumbnail = await ThumbnailModel.loadWithVideoByFilename(filename, ThumbnailType.PREVIEW) - if (!thumbnail) return undefined - - if (thumbnail.Video.isOwned()) return { isOwned: true, path: thumbnail.getPath() } - - return this.loadRemoteFile(thumbnail.Video.uuid) - } - - // Key is the video UUID - protected async loadRemoteFile (key: string) { - const video = await VideoModel.loadFull(key) - if (!video) return undefined - - if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.') - - const preview = video.getPreview() - const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, preview.filename) - const remoteUrl = preview.getOriginFileUrl(video) - - try { - await doRequestAndSaveToFile(remoteUrl, destPath) - - logger.debug('Fetched remote preview %s to %s.', remoteUrl, destPath) - - return { isOwned: false, path: destPath } - } catch (err) { - logger.info('Cannot fetch remote preview file %s.', remoteUrl, { err }) - - return undefined - } - } -} - -export { - VideoPreviewsSimpleFileCache -} diff --git a/server/lib/files-cache/video-storyboards-simple-file-cache.ts b/server/lib/files-cache/video-storyboards-simple-file-cache.ts deleted file mode 100644 index 4cd96e70c..000000000 --- a/server/lib/files-cache/video-storyboards-simple-file-cache.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { join } from 'path' -import { logger } from '@server/helpers/logger' -import { doRequestAndSaveToFile } from '@server/helpers/requests' -import { StoryboardModel } from '@server/models/video/storyboard' -import { FILES_CACHE } from '../../initializers/constants' -import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache' - -class VideoStoryboardsSimpleFileCache extends AbstractSimpleFileCache { - - private static instance: VideoStoryboardsSimpleFileCache - - private constructor () { - super() - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } - - async getFilePathImpl (filename: string) { - const storyboard = await StoryboardModel.loadWithVideoByFilename(filename) - if (!storyboard) return undefined - - if (storyboard.Video.isOwned()) return { isOwned: true, path: storyboard.getPath() } - - return this.loadRemoteFile(storyboard.filename) - } - - // Key is the storyboard filename - protected async loadRemoteFile (key: string) { - const storyboard = await StoryboardModel.loadWithVideoByFilename(key) - if (!storyboard) return undefined - - const destPath = join(FILES_CACHE.STORYBOARDS.DIRECTORY, storyboard.filename) - const remoteUrl = storyboard.getOriginFileUrl(storyboard.Video) - - try { - await doRequestAndSaveToFile(remoteUrl, destPath) - - logger.debug('Fetched remote storyboard %s to %s.', remoteUrl, destPath) - - return { isOwned: false, path: destPath } - } catch (err) { - logger.info('Cannot fetch remote storyboard file %s.', remoteUrl, { err }) - - return undefined - } - } -} - -export { - VideoStoryboardsSimpleFileCache -} diff --git a/server/lib/files-cache/video-torrents-simple-file-cache.ts b/server/lib/files-cache/video-torrents-simple-file-cache.ts deleted file mode 100644 index 8bcd0b9bf..000000000 --- a/server/lib/files-cache/video-torrents-simple-file-cache.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { join } from 'path' -import { logger } from '@server/helpers/logger' -import { doRequestAndSaveToFile } from '@server/helpers/requests' -import { VideoFileModel } from '@server/models/video/video-file' -import { MVideo, MVideoFile } from '@server/types/models' -import { CONFIG } from '../../initializers/config' -import { FILES_CACHE } from '../../initializers/constants' -import { VideoModel } from '../../models/video/video' -import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache' - -class VideoTorrentsSimpleFileCache extends AbstractSimpleFileCache { - - private static instance: VideoTorrentsSimpleFileCache - - private constructor () { - super() - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } - - async getFilePathImpl (filename: string) { - const file = await VideoFileModel.loadWithVideoOrPlaylistByTorrentFilename(filename) - if (!file) return undefined - - if (file.getVideo().isOwned()) { - const downloadName = this.buildDownloadName(file.getVideo(), file) - - return { isOwned: true, path: join(CONFIG.STORAGE.TORRENTS_DIR, file.torrentFilename), downloadName } - } - - return this.loadRemoteFile(filename) - } - - // Key is the torrent filename - protected async loadRemoteFile (key: string) { - const file = await VideoFileModel.loadWithVideoOrPlaylistByTorrentFilename(key) - if (!file) return undefined - - if (file.getVideo().isOwned()) throw new Error('Cannot load remote file of owned video.') - - // Used to fetch the path - const video = await VideoModel.loadFull(file.getVideo().id) - if (!video) return undefined - - const remoteUrl = file.getRemoteTorrentUrl(video) - const destPath = join(FILES_CACHE.TORRENTS.DIRECTORY, file.torrentFilename) - - try { - await doRequestAndSaveToFile(remoteUrl, destPath) - - const downloadName = this.buildDownloadName(video, file) - - return { isOwned: false, path: destPath, downloadName } - } catch (err) { - logger.info('Cannot fetch remote torrent file %s.', remoteUrl, { err }) - - return undefined - } - } - - private buildDownloadName (video: MVideo, file: MVideoFile) { - return `${video.name}-${file.resolution}p.torrent` - } -} - -export { - VideoTorrentsSimpleFileCache -} diff --git a/server/lib/hls.ts b/server/lib/hls.ts deleted file mode 100644 index 19044d7c2..000000000 --- a/server/lib/hls.ts +++ /dev/null @@ -1,285 +0,0 @@ -import { close, ensureDir, move, open, outputJSON, read, readFile, remove, stat, writeFile } from 'fs-extra' -import { flatten } from 'lodash' -import PQueue from 'p-queue' -import { basename, dirname, join } from 'path' -import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models' -import { uniqify, uuidRegex } from '@shared/core-utils' -import { sha256 } from '@shared/extra-utils' -import { getVideoStreamDimensionsInfo } from '@shared/ffmpeg' -import { VideoStorage } from '@shared/models' -import { getAudioStreamCodec, getVideoStreamCodec } from '../helpers/ffmpeg' -import { logger, loggerTagsFactory } from '../helpers/logger' -import { doRequest, doRequestAndSaveToFile } from '../helpers/requests' -import { generateRandomString } from '../helpers/utils' -import { CONFIG } from '../initializers/config' -import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers/constants' -import { sequelizeTypescript } from '../initializers/database' -import { VideoFileModel } from '../models/video/video-file' -import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' -import { storeHLSFileFromFilename } from './object-storage' -import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths' -import { VideoPathManager } from './video-path-manager' - -const lTags = loggerTagsFactory('hls') - -async function updateStreamingPlaylistsInfohashesIfNeeded () { - const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() - - // Use separate SQL queries, because we could have many videos to update - for (const playlist of playlistsToUpdate) { - await sequelizeTypescript.transaction(async t => { - const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t) - - playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles) - playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION - - await playlist.save({ transaction: t }) - }) - } -} - -async function updatePlaylistAfterFileChange (video: MVideo, playlist: MStreamingPlaylist) { - try { - let playlistWithFiles = await updateMasterHLSPlaylist(video, playlist) - playlistWithFiles = await updateSha256VODSegments(video, playlist) - - // Refresh playlist, operations can take some time - playlistWithFiles = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlist.id) - playlistWithFiles.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles) - await playlistWithFiles.save() - - video.setHLSPlaylist(playlistWithFiles) - } catch (err) { - logger.warn('Cannot update playlist after file change. Maybe due to concurrent transcoding', { err }) - } -} - -// --------------------------------------------------------------------------- - -// Avoid concurrency issues when updating streaming playlist files -const playlistFilesQueue = new PQueue({ concurrency: 1 }) - -function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist): Promise { - return playlistFilesQueue.add(async () => { - const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id) - - const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] - - for (const file of playlist.VideoFiles) { - const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) - - await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => { - const size = await getVideoStreamDimensionsInfo(videoFilePath) - - const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) - const resolution = `RESOLUTION=${size?.width || 0}x${size?.height || 0}` - - let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` - if (file.fps) line += ',FRAME-RATE=' + file.fps - - const codecs = await Promise.all([ - getVideoStreamCodec(videoFilePath), - getAudioStreamCodec(videoFilePath) - ]) - - line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` - - masterPlaylists.push(line) - masterPlaylists.push(playlistFilename) - }) - } - - if (playlist.playlistFilename) { - await video.removeStreamingPlaylistFile(playlist, playlist.playlistFilename) - } - playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive) - - const masterPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.playlistFilename) - await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') - - logger.info('Updating %s master playlist file of video %s', masterPlaylistPath, video.uuid, lTags(video.uuid)) - - if (playlist.storage === VideoStorage.OBJECT_STORAGE) { - playlist.playlistUrl = await storeHLSFileFromFilename(playlist, playlist.playlistFilename) - await remove(masterPlaylistPath) - } - - return playlist.save() - }) -} - -// --------------------------------------------------------------------------- - -function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingPlaylist): Promise { - return playlistFilesQueue.add(async () => { - const json: { [filename: string]: { [range: string]: string } } = {} - - const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id) - - // For all the resolutions available for this video - for (const file of playlist.VideoFiles) { - const rangeHashes: { [range: string]: string } = {} - const fileWithPlaylist = file.withVideoOrPlaylist(playlist) - - await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => { - - return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => { - const playlistContent = await readFile(resolutionPlaylistPath) - const ranges = getRangesFromPlaylist(playlistContent.toString()) - - const fd = await open(videoPath, 'r') - for (const range of ranges) { - const buf = Buffer.alloc(range.length) - await read(fd, buf, 0, range.length, range.offset) - - rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) - } - await close(fd) - - const videoFilename = file.filename - json[videoFilename] = rangeHashes - }) - }) - } - - if (playlist.segmentsSha256Filename) { - await video.removeStreamingPlaylistFile(playlist, playlist.segmentsSha256Filename) - } - playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive) - - const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) - await outputJSON(outputPath, json) - - if (playlist.storage === VideoStorage.OBJECT_STORAGE) { - playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlist, playlist.segmentsSha256Filename) - await remove(outputPath) - } - - return playlist.save() - }) -} - -// --------------------------------------------------------------------------- - -async function buildSha256Segment (segmentPath: string) { - const buf = await readFile(segmentPath) - return sha256(buf) -} - -function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number, bodyKBLimit: number) { - let timer - let remainingBodyKBLimit = bodyKBLimit - - logger.info('Importing HLS playlist %s', playlistUrl) - - return new Promise(async (res, rej) => { - const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10)) - - await ensureDir(tmpDirectory) - - timer = setTimeout(() => { - deleteTmpDirectory(tmpDirectory) - - return rej(new Error('HLS download timeout.')) - }, timeout) - - try { - // Fetch master playlist - const subPlaylistUrls = await fetchUniqUrls(playlistUrl) - - const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u)) - const fileUrls = uniqify(flatten(await Promise.all(subRequests))) - - logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls }) - - for (const fileUrl of fileUrls) { - const destPath = join(tmpDirectory, basename(fileUrl)) - - await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit, timeout: REQUEST_TIMEOUTS.REDUNDANCY }) - - const { size } = await stat(destPath) - remainingBodyKBLimit -= (size / 1000) - - logger.debug('Downloaded HLS playlist file %s with %d kB remained limit.', fileUrl, Math.floor(remainingBodyKBLimit)) - } - - clearTimeout(timer) - - await move(tmpDirectory, destinationDir, { overwrite: true }) - - return res() - } catch (err) { - deleteTmpDirectory(tmpDirectory) - - return rej(err) - } - }) - - function deleteTmpDirectory (directory: string) { - remove(directory) - .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) - } - - async function fetchUniqUrls (playlistUrl: string) { - const { body } = await doRequest(playlistUrl) - - if (!body) return [] - - const urls = body.split('\n') - .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4')) - .map(url => { - if (url.startsWith('http://') || url.startsWith('https://')) return url - - return `${dirname(playlistUrl)}/${url}` - }) - - return uniqify(urls) - } -} - -// --------------------------------------------------------------------------- - -async function renameVideoFileInPlaylist (playlistPath: string, newVideoFilename: string) { - const content = await readFile(playlistPath, 'utf8') - - const newContent = content.replace(new RegExp(`${uuidRegex}-\\d+-fragmented.mp4`, 'g'), newVideoFilename) - - await writeFile(playlistPath, newContent, 'utf8') -} - -// --------------------------------------------------------------------------- - -function injectQueryToPlaylistUrls (content: string, queryString: string) { - return content.replace(/\.(m3u8|ts|mp4)/gm, '.$1?' + queryString) -} - -// --------------------------------------------------------------------------- - -export { - updateMasterHLSPlaylist, - updateSha256VODSegments, - buildSha256Segment, - downloadPlaylistSegments, - updateStreamingPlaylistsInfohashesIfNeeded, - updatePlaylistAfterFileChange, - injectQueryToPlaylistUrls, - renameVideoFileInPlaylist -} - -// --------------------------------------------------------------------------- - -function getRangesFromPlaylist (playlistContent: string) { - const ranges: { offset: number, length: number }[] = [] - const lines = playlistContent.split('\n') - const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/ - - for (const line of lines) { - const captured = regex.exec(line) - - if (captured) { - ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) }) - } - } - - return ranges -} diff --git a/server/lib/internal-event-emitter.ts b/server/lib/internal-event-emitter.ts deleted file mode 100644 index 08b46a5c3..000000000 --- a/server/lib/internal-event-emitter.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { MChannel, MVideo } from '@server/types/models' -import { EventEmitter } from 'events' - -export interface PeerTubeInternalEvents { - 'video-created': (options: { video: MVideo }) => void - 'video-updated': (options: { video: MVideo }) => void - 'video-deleted': (options: { video: MVideo }) => void - - 'channel-created': (options: { channel: MChannel }) => void - 'channel-updated': (options: { channel: MChannel }) => void - 'channel-deleted': (options: { channel: MChannel }) => void -} - -declare interface InternalEventEmitter { - on( - event: U, listener: PeerTubeInternalEvents[U] - ): this - - emit( - event: U, ...args: Parameters - ): boolean -} - -class InternalEventEmitter extends EventEmitter { - - private static instance: InternalEventEmitter - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -export { - InternalEventEmitter -} diff --git a/server/lib/job-queue/handlers/activitypub-cleaner.ts b/server/lib/job-queue/handlers/activitypub-cleaner.ts deleted file mode 100644 index 6ee9e2429..000000000 --- a/server/lib/job-queue/handlers/activitypub-cleaner.ts +++ /dev/null @@ -1,202 +0,0 @@ -import { map } from 'bluebird' -import { Job } from 'bullmq' -import { - isAnnounceActivityValid, - isDislikeActivityValid, - isLikeActivityValid -} from '@server/helpers/custom-validators/activitypub/activity' -import { sanitizeAndCheckVideoCommentObject } from '@server/helpers/custom-validators/activitypub/video-comments' -import { PeerTubeRequestError } from '@server/helpers/requests' -import { AP_CLEANER } from '@server/initializers/constants' -import { fetchAP } from '@server/lib/activitypub/activity' -import { checkUrlsSameHost } from '@server/lib/activitypub/url' -import { Redis } from '@server/lib/redis' -import { VideoModel } from '@server/models/video/video' -import { VideoCommentModel } from '@server/models/video/video-comment' -import { VideoShareModel } from '@server/models/video/video-share' -import { HttpStatusCode } from '@shared/models' -import { logger, loggerTagsFactory } from '../../../helpers/logger' -import { AccountVideoRateModel } from '../../../models/account/account-video-rate' - -const lTags = loggerTagsFactory('ap-cleaner') - -// Job to clean remote interactions off local videos - -async function processActivityPubCleaner (_job: Job) { - logger.info('Processing ActivityPub cleaner.', lTags()) - - { - const rateUrls = await AccountVideoRateModel.listRemoteRateUrlsOfLocalVideos() - const { bodyValidator, deleter, updater } = rateOptionsFactory() - - await map(rateUrls, async rateUrl => { - // TODO: remove when https://github.com/mastodon/mastodon/issues/13571 is fixed - if (rateUrl.includes('#')) return - - const result = await updateObjectIfNeeded({ url: rateUrl, bodyValidator, updater, deleter }) - - if (result?.status === 'deleted') { - const { videoId, type } = result.data - - await VideoModel.syncLocalRates(videoId, type, undefined) - } - }, { concurrency: AP_CLEANER.CONCURRENCY }) - } - - { - const shareUrls = await VideoShareModel.listRemoteShareUrlsOfLocalVideos() - const { bodyValidator, deleter, updater } = shareOptionsFactory() - - await map(shareUrls, async shareUrl => { - await updateObjectIfNeeded({ url: shareUrl, bodyValidator, updater, deleter }) - }, { concurrency: AP_CLEANER.CONCURRENCY }) - } - - { - const commentUrls = await VideoCommentModel.listRemoteCommentUrlsOfLocalVideos() - const { bodyValidator, deleter, updater } = commentOptionsFactory() - - await map(commentUrls, async commentUrl => { - await updateObjectIfNeeded({ url: commentUrl, bodyValidator, updater, deleter }) - }, { concurrency: AP_CLEANER.CONCURRENCY }) - } -} - -// --------------------------------------------------------------------------- - -export { - processActivityPubCleaner -} - -// --------------------------------------------------------------------------- - -async function updateObjectIfNeeded (options: { - url: string - bodyValidator: (body: any) => boolean - updater: (url: string, newUrl: string) => Promise - deleter: (url: string) => Promise } -): Promise<{ data: T, status: 'deleted' | 'updated' } | null> { - const { url, bodyValidator, updater, deleter } = options - - const on404OrTombstone = async () => { - logger.info('Removing remote AP object %s.', url, lTags(url)) - const data = await deleter(url) - - return { status: 'deleted' as 'deleted', data } - } - - try { - const { body } = await fetchAP(url) - - // If not same id, check same host and update - if (!body?.id || !bodyValidator(body)) throw new Error(`Body or body id of ${url} is invalid`) - - if (body.type === 'Tombstone') { - return on404OrTombstone() - } - - const newUrl = body.id - if (newUrl !== url) { - if (checkUrlsSameHost(newUrl, url) !== true) { - throw new Error(`New url ${newUrl} has not the same host than old url ${url}`) - } - - logger.info('Updating remote AP object %s.', url, lTags(url)) - const data = await updater(url, newUrl) - - return { status: 'updated', data } - } - - return null - } catch (err) { - // Does not exist anymore, remove entry - if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { - return on404OrTombstone() - } - - logger.debug('Remote AP object %s is unavailable.', url, lTags(url)) - - const unavailability = await Redis.Instance.addAPUnavailability(url) - if (unavailability >= AP_CLEANER.UNAVAILABLE_TRESHOLD) { - logger.info('Removing unavailable AP resource %s.', url, lTags(url)) - return on404OrTombstone() - } - - return null - } -} - -function rateOptionsFactory () { - return { - bodyValidator: (body: any) => isLikeActivityValid(body) || isDislikeActivityValid(body), - - updater: async (url: string, newUrl: string) => { - const rate = await AccountVideoRateModel.loadByUrl(url, undefined) - rate.url = newUrl - - const videoId = rate.videoId - const type = rate.type - - await rate.save() - - return { videoId, type } - }, - - deleter: async (url) => { - const rate = await AccountVideoRateModel.loadByUrl(url, undefined) - - const videoId = rate.videoId - const type = rate.type - - await rate.destroy() - - return { videoId, type } - } - } -} - -function shareOptionsFactory () { - return { - bodyValidator: (body: any) => isAnnounceActivityValid(body), - - updater: async (url: string, newUrl: string) => { - const share = await VideoShareModel.loadByUrl(url, undefined) - share.url = newUrl - - await share.save() - - return undefined - }, - - deleter: async (url) => { - const share = await VideoShareModel.loadByUrl(url, undefined) - - await share.destroy() - - return undefined - } - } -} - -function commentOptionsFactory () { - return { - bodyValidator: (body: any) => sanitizeAndCheckVideoCommentObject(body), - - updater: async (url: string, newUrl: string) => { - const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(url) - comment.url = newUrl - - await comment.save() - - return undefined - }, - - deleter: async (url) => { - const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(url) - - await comment.destroy() - - return undefined - } - } -} diff --git a/server/lib/job-queue/handlers/activitypub-follow.ts b/server/lib/job-queue/handlers/activitypub-follow.ts deleted file mode 100644 index a68c32ba0..000000000 --- a/server/lib/job-queue/handlers/activitypub-follow.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { Job } from 'bullmq' -import { getLocalActorFollowActivityPubUrl } from '@server/lib/activitypub/url' -import { ActivitypubFollowPayload } from '@shared/models' -import { sanitizeHost } from '../../../helpers/core-utils' -import { retryTransactionWrapper } from '../../../helpers/database-utils' -import { logger } from '../../../helpers/logger' -import { REMOTE_SCHEME, WEBSERVER } from '../../../initializers/constants' -import { sequelizeTypescript } from '../../../initializers/database' -import { ActorModel } from '../../../models/actor/actor' -import { ActorFollowModel } from '../../../models/actor/actor-follow' -import { MActor, MActorFull } from '../../../types/models' -import { getOrCreateAPActor, loadActorUrlOrGetFromWebfinger } from '../../activitypub/actors' -import { sendFollow } from '../../activitypub/send' -import { Notifier } from '../../notifier' - -async function processActivityPubFollow (job: Job) { - const payload = job.data as ActivitypubFollowPayload - const host = payload.host - - logger.info('Processing ActivityPub follow in job %s.', job.id) - - let targetActor: MActorFull - if (!host || host === WEBSERVER.HOST) { - targetActor = await ActorModel.loadLocalByName(payload.name) - } else { - const sanitizedHost = sanitizeHost(host, REMOTE_SCHEME.HTTP) - const actorUrl = await loadActorUrlOrGetFromWebfinger(payload.name + '@' + sanitizedHost) - targetActor = await getOrCreateAPActor(actorUrl, 'all') - } - - if (payload.assertIsChannel && !targetActor.VideoChannel) { - logger.warn('Do not follow %s@%s because it is not a channel.', payload.name, host) - return - } - - const fromActor = await ActorModel.load(payload.followerActorId) - - return retryTransactionWrapper(follow, fromActor, targetActor, payload.isAutoFollow) -} -// --------------------------------------------------------------------------- - -export { - processActivityPubFollow -} - -// --------------------------------------------------------------------------- - -async function follow (fromActor: MActor, targetActor: MActorFull, isAutoFollow = false) { - if (fromActor.id === targetActor.id) { - throw new Error('Follower is the same as target actor.') - } - - // Same server, direct accept - const state = !fromActor.serverId && !targetActor.serverId ? 'accepted' : 'pending' - - const actorFollow = await sequelizeTypescript.transaction(async t => { - const [ actorFollow ] = await ActorFollowModel.findOrCreateCustom({ - byActor: fromActor, - state, - targetActor, - activityId: getLocalActorFollowActivityPubUrl(fromActor, targetActor), - transaction: t - }) - - // Send a notification to remote server if our follow is not already accepted - if (actorFollow.state !== 'accepted') sendFollow(actorFollow, t) - - return actorFollow - }) - - const followerFull = await ActorModel.loadFull(fromActor.id) - - const actorFollowFull = Object.assign(actorFollow, { - ActorFollowing: targetActor, - ActorFollower: followerFull - }) - - if (actorFollow.state === 'accepted') Notifier.Instance.notifyOfNewUserFollow(actorFollowFull) - if (isAutoFollow === true) Notifier.Instance.notifyOfAutoInstanceFollowing(actorFollowFull) - - return actorFollow -} diff --git a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts deleted file mode 100644 index 8904d086f..000000000 --- a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { Job } from 'bullmq' -import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from '@server/lib/activitypub/send' -import { ActorFollowHealthCache } from '@server/lib/actor-follow-health-cache' -import { parallelHTTPBroadcastFromWorker, sequentialHTTPBroadcastFromWorker } from '@server/lib/worker/parent-process' -import { ActivitypubHttpBroadcastPayload } from '@shared/models' -import { logger } from '../../../helpers/logger' - -// Prefer using a worker thread for HTTP requests because on high load we may have to sign many requests, which can be CPU intensive - -async function processActivityPubHttpSequentialBroadcast (job: Job) { - logger.info('Processing ActivityPub broadcast in job %s.', job.id) - - const requestOptions = await buildRequestOptions(job.data) - - const { badUrls, goodUrls } = await sequentialHTTPBroadcastFromWorker({ uris: job.data.uris, requestOptions }) - - return ActorFollowHealthCache.Instance.updateActorFollowsHealth(goodUrls, badUrls) -} - -async function processActivityPubParallelHttpBroadcast (job: Job) { - logger.info('Processing ActivityPub parallel broadcast in job %s.', job.id) - - const requestOptions = await buildRequestOptions(job.data) - - const { badUrls, goodUrls } = await parallelHTTPBroadcastFromWorker({ uris: job.data.uris, requestOptions }) - - return ActorFollowHealthCache.Instance.updateActorFollowsHealth(goodUrls, badUrls) -} - -// --------------------------------------------------------------------------- - -export { - processActivityPubHttpSequentialBroadcast, - processActivityPubParallelHttpBroadcast -} - -// --------------------------------------------------------------------------- - -async function buildRequestOptions (payload: ActivitypubHttpBroadcastPayload) { - const body = await computeBody(payload) - const httpSignatureOptions = await buildSignedRequestOptions({ signatureActorId: payload.signatureActorId, hasPayload: true }) - - return { - method: 'POST' as 'POST', - json: body, - httpSignature: httpSignatureOptions, - headers: buildGlobalHeaders(body) - } -} diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts deleted file mode 100644 index b6cb3c4a6..000000000 --- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { Job } from 'bullmq' -import { ActivitypubHttpFetcherPayload, FetchType } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { VideoModel } from '../../../models/video/video' -import { VideoCommentModel } from '../../../models/video/video-comment' -import { VideoShareModel } from '../../../models/video/video-share' -import { MVideoFullLight } from '../../../types/models' -import { crawlCollectionPage } from '../../activitypub/crawl' -import { createAccountPlaylists } from '../../activitypub/playlists' -import { processActivities } from '../../activitypub/process' -import { addVideoShares } from '../../activitypub/share' -import { addVideoComments } from '../../activitypub/video-comments' - -async function processActivityPubHttpFetcher (job: Job) { - logger.info('Processing ActivityPub fetcher in job %s.', job.id) - - const payload = job.data as ActivitypubHttpFetcherPayload - - let video: MVideoFullLight - if (payload.videoId) video = await VideoModel.loadFull(payload.videoId) - - const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise } = { - 'activity': items => processActivities(items, { outboxUrl: payload.uri, fromFetch: true }), - 'video-shares': items => addVideoShares(items, video), - 'video-comments': items => addVideoComments(items), - 'account-playlists': items => createAccountPlaylists(items) - } - - const cleanerType: { [ id in FetchType ]?: (crawlStartDate: Date) => Promise } = { - 'video-shares': crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate), - 'video-comments': crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate) - } - - return crawlCollectionPage(payload.uri, fetcherType[payload.type], cleanerType[payload.type]) -} - -// --------------------------------------------------------------------------- - -export { - processActivityPubHttpFetcher -} diff --git a/server/lib/job-queue/handlers/activitypub-http-unicast.ts b/server/lib/job-queue/handlers/activitypub-http-unicast.ts deleted file mode 100644 index 50fca3f94..000000000 --- a/server/lib/job-queue/handlers/activitypub-http-unicast.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { Job } from 'bullmq' -import { buildGlobalHeaders, buildSignedRequestOptions, computeBody } from '@server/lib/activitypub/send' -import { ActivitypubHttpUnicastPayload } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { doRequest } from '../../../helpers/requests' -import { ActorFollowHealthCache } from '../../actor-follow-health-cache' - -async function processActivityPubHttpUnicast (job: Job) { - logger.info('Processing ActivityPub unicast in job %s.', job.id) - - const payload = job.data as ActivitypubHttpUnicastPayload - const uri = payload.uri - - const body = await computeBody(payload) - const httpSignatureOptions = await buildSignedRequestOptions({ signatureActorId: payload.signatureActorId, hasPayload: true }) - - const options = { - method: 'POST' as 'POST', - json: body, - httpSignature: httpSignatureOptions, - headers: buildGlobalHeaders(body) - } - - try { - await doRequest(uri, options) - ActorFollowHealthCache.Instance.updateActorFollowsHealth([ uri ], []) - } catch (err) { - ActorFollowHealthCache.Instance.updateActorFollowsHealth([], [ uri ]) - - throw err - } -} - -// --------------------------------------------------------------------------- - -export { - processActivityPubHttpUnicast -} diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts deleted file mode 100644 index 706bf17fa..000000000 --- a/server/lib/job-queue/handlers/activitypub-refresher.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { Job } from 'bullmq' -import { refreshVideoPlaylistIfNeeded } from '@server/lib/activitypub/playlists' -import { refreshVideoIfNeeded } from '@server/lib/activitypub/videos' -import { loadVideoByUrl } from '@server/lib/model-loaders' -import { RefreshPayload } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { ActorModel } from '../../../models/actor/actor' -import { VideoPlaylistModel } from '../../../models/video/video-playlist' -import { refreshActorIfNeeded } from '../../activitypub/actors' - -async function refreshAPObject (job: Job) { - const payload = job.data as RefreshPayload - - logger.info('Processing AP refresher in job %s for %s.', job.id, payload.url) - - if (payload.type === 'video') return refreshVideo(payload.url) - if (payload.type === 'video-playlist') return refreshVideoPlaylist(payload.url) - if (payload.type === 'actor') return refreshActor(payload.url) -} - -// --------------------------------------------------------------------------- - -export { - refreshAPObject -} - -// --------------------------------------------------------------------------- - -async function refreshVideo (videoUrl: string) { - const fetchType = 'all' as 'all' - const syncParam = { rates: true, shares: true, comments: true } - - const videoFromDatabase = await loadVideoByUrl(videoUrl, fetchType) - if (videoFromDatabase) { - const refreshOptions = { - video: videoFromDatabase, - fetchedType: fetchType, - syncParam - } - - await refreshVideoIfNeeded(refreshOptions) - } -} - -async function refreshActor (actorUrl: string) { - const fetchType = 'all' as 'all' - const actor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorUrl) - - if (actor) { - await refreshActorIfNeeded({ actor, fetchedType: fetchType }) - } -} - -async function refreshVideoPlaylist (playlistUrl: string) { - const playlist = await VideoPlaylistModel.loadByUrlAndPopulateAccount(playlistUrl) - - if (playlist) { - await refreshVideoPlaylistIfNeeded(playlist) - } -} diff --git a/server/lib/job-queue/handlers/actor-keys.ts b/server/lib/job-queue/handlers/actor-keys.ts deleted file mode 100644 index 27a2d431b..000000000 --- a/server/lib/job-queue/handlers/actor-keys.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { Job } from 'bullmq' -import { generateAndSaveActorKeys } from '@server/lib/activitypub/actors' -import { ActorModel } from '@server/models/actor/actor' -import { ActorKeysPayload } from '@shared/models' -import { logger } from '../../../helpers/logger' - -async function processActorKeys (job: Job) { - const payload = job.data as ActorKeysPayload - logger.info('Processing actor keys in job %s.', job.id) - - const actor = await ActorModel.load(payload.actorId) - - await generateAndSaveActorKeys(actor) -} - -// --------------------------------------------------------------------------- - -export { - processActorKeys -} diff --git a/server/lib/job-queue/handlers/after-video-channel-import.ts b/server/lib/job-queue/handlers/after-video-channel-import.ts deleted file mode 100644 index ffdd8c5b5..000000000 --- a/server/lib/job-queue/handlers/after-video-channel-import.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { Job } from 'bullmq' -import { logger } from '@server/helpers/logger' -import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' -import { AfterVideoChannelImportPayload, VideoChannelSyncState, VideoImportPreventExceptionResult } from '@shared/models' - -export async function processAfterVideoChannelImport (job: Job) { - const payload = job.data as AfterVideoChannelImportPayload - if (!payload.channelSyncId) return - - logger.info('Processing after video channel import in job %s.', job.id) - - const sync = await VideoChannelSyncModel.loadWithChannel(payload.channelSyncId) - if (!sync) { - logger.error('Unknown sync id %d.', payload.channelSyncId) - return - } - - const childrenValues = await job.getChildrenValues() - - let errors = 0 - let successes = 0 - - for (const value of Object.values(childrenValues)) { - if (value.resultType === 'success') successes++ - else if (value.resultType === 'error') errors++ - } - - if (errors > 0) { - sync.state = VideoChannelSyncState.FAILED - logger.error(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" with failures.`, { errors, successes }) - } else { - sync.state = VideoChannelSyncState.SYNCED - logger.info(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" successfully.`, { successes }) - } - - await sync.save() -} diff --git a/server/lib/job-queue/handlers/email.ts b/server/lib/job-queue/handlers/email.ts deleted file mode 100644 index 567bcc076..000000000 --- a/server/lib/job-queue/handlers/email.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { Job } from 'bullmq' -import { EmailPayload } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { Emailer } from '../../emailer' - -async function processEmail (job: Job) { - const payload = job.data as EmailPayload - logger.info('Processing email in job %s.', job.id) - - return Emailer.Instance.sendMail(payload) -} - -// --------------------------------------------------------------------------- - -export { - processEmail -} diff --git a/server/lib/job-queue/handlers/federate-video.ts b/server/lib/job-queue/handlers/federate-video.ts deleted file mode 100644 index 6aac36741..000000000 --- a/server/lib/job-queue/handlers/federate-video.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { Job } from 'bullmq' -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { sequelizeTypescript } from '@server/initializers/database' -import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' -import { VideoModel } from '@server/models/video/video' -import { FederateVideoPayload } from '@shared/models' -import { logger } from '../../../helpers/logger' - -function processFederateVideo (job: Job) { - const payload = job.data as FederateVideoPayload - - logger.info('Processing video federation in job %s.', job.id) - - return retryTransactionWrapper(() => { - return sequelizeTypescript.transaction(async t => { - const video = await VideoModel.loadFull(payload.videoUUID, t) - if (!video) return - - return federateVideoIfNeeded(video, payload.isNewVideo, t) - }) - }) -} - -// --------------------------------------------------------------------------- - -export { - processFederateVideo -} diff --git a/server/lib/job-queue/handlers/generate-storyboard.ts b/server/lib/job-queue/handlers/generate-storyboard.ts deleted file mode 100644 index eea20274a..000000000 --- a/server/lib/job-queue/handlers/generate-storyboard.ts +++ /dev/null @@ -1,163 +0,0 @@ -import { Job } from 'bullmq' -import { join } from 'path' -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' -import { generateImageFilename, getImageSize } from '@server/helpers/image-utils' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { deleteFileAndCatch } from '@server/helpers/utils' -import { CONFIG } from '@server/initializers/config' -import { STORYBOARD } from '@server/initializers/constants' -import { sequelizeTypescript } from '@server/initializers/database' -import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { StoryboardModel } from '@server/models/video/storyboard' -import { VideoModel } from '@server/models/video/video' -import { MVideo } from '@server/types/models' -import { FFmpegImage, isAudioFile } from '@shared/ffmpeg' -import { GenerateStoryboardPayload } from '@shared/models' - -const lTagsBase = loggerTagsFactory('storyboard') - -async function processGenerateStoryboard (job: Job): Promise { - const payload = job.data as GenerateStoryboardPayload - const lTags = lTagsBase(payload.videoUUID) - - logger.info('Processing generate storyboard of %s in job %s.', payload.videoUUID, job.id, lTags) - - const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID) - - try { - const video = await VideoModel.loadFull(payload.videoUUID) - if (!video) { - logger.info('Video %s does not exist anymore, skipping storyboard generation.', payload.videoUUID, lTags) - return - } - - const inputFile = video.getMaxQualityFile() - - await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => { - const isAudio = await isAudioFile(videoPath) - - if (isAudio) { - logger.info('Do not generate a storyboard of %s since the video does not have a video stream', payload.videoUUID, lTags) - return - } - - const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')) - - const filename = generateImageFilename() - const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename) - - const totalSprites = buildTotalSprites(video) - if (totalSprites === 0) { - logger.info('Do not generate a storyboard of %s because the video is not long enough', payload.videoUUID, lTags) - return - } - - const spriteDuration = Math.round(video.duration / totalSprites) - - const spritesCount = findGridSize({ - toFind: totalSprites, - maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT - }) - - logger.debug( - 'Generating storyboard from video of %s to %s', video.uuid, destination, - { ...lTags, spritesCount, spriteDuration, videoDuration: video.duration } - ) - - await ffmpeg.generateStoryboardFromVideo({ - destination, - path: videoPath, - sprites: { - size: STORYBOARD.SPRITE_SIZE, - count: spritesCount, - duration: spriteDuration - } - }) - - const imageSize = await getImageSize(destination) - - await retryTransactionWrapper(() => { - return sequelizeTypescript.transaction(async transaction => { - const videoStillExists = await VideoModel.load(video.id, transaction) - if (!videoStillExists) { - logger.info('Video %s does not exist anymore, skipping storyboard generation.', payload.videoUUID, lTags) - deleteFileAndCatch(destination) - return - } - - const existing = await StoryboardModel.loadByVideo(video.id, transaction) - if (existing) await existing.destroy({ transaction }) - - await StoryboardModel.create({ - filename, - totalHeight: imageSize.height, - totalWidth: imageSize.width, - spriteHeight: STORYBOARD.SPRITE_SIZE.height, - spriteWidth: STORYBOARD.SPRITE_SIZE.width, - spriteDuration, - videoId: video.id - }, { transaction }) - - logger.info('Storyboard generation %s ended for video %s.', destination, video.uuid, lTags) - - if (payload.federate) { - await federateVideoIfNeeded(video, false, transaction) - } - }) - }) - }) - } finally { - inputFileMutexReleaser() - } -} - -// --------------------------------------------------------------------------- - -export { - processGenerateStoryboard -} - -function buildTotalSprites (video: MVideo) { - const maxSprites = STORYBOARD.SPRITE_SIZE.height * STORYBOARD.SPRITE_SIZE.width - const totalSprites = Math.min(Math.ceil(video.duration), maxSprites) - - // We can generate a single line - if (totalSprites <= STORYBOARD.SPRITES_MAX_EDGE_COUNT) return totalSprites - - return findGridFit(totalSprites, STORYBOARD.SPRITES_MAX_EDGE_COUNT) -} - -function findGridSize (options: { - toFind: number - maxEdgeCount: number -}) { - const { toFind, maxEdgeCount } = options - - for (let i = 1; i <= maxEdgeCount; i++) { - for (let j = i; j <= maxEdgeCount; j++) { - if (toFind === i * j) return { width: j, height: i } - } - } - - throw new Error(`Could not find grid size (to find: ${toFind}, max edge count: ${maxEdgeCount}`) -} - -function findGridFit (value: number, maxMultiplier: number) { - for (let i = value; i--; i > 0) { - if (!isPrimeWithin(i, maxMultiplier)) return i - } - - throw new Error('Could not find prime number below ' + value) -} - -function isPrimeWithin (value: number, maxMultiplier: number) { - if (value < 2) return false - - for (let i = 2, end = Math.min(Math.sqrt(value), maxMultiplier); i <= end; i++) { - if (value % i === 0 && value / i <= maxMultiplier) return false - } - - return true -} diff --git a/server/lib/job-queue/handlers/manage-video-torrent.ts b/server/lib/job-queue/handlers/manage-video-torrent.ts deleted file mode 100644 index edf52de0c..000000000 --- a/server/lib/job-queue/handlers/manage-video-torrent.ts +++ /dev/null @@ -1,110 +0,0 @@ -import { Job } from 'bullmq' -import { extractVideo } from '@server/helpers/video' -import { createTorrentAndSetInfoHash, updateTorrentMetadata } from '@server/helpers/webtorrent' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { VideoModel } from '@server/models/video/video' -import { VideoFileModel } from '@server/models/video/video-file' -import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { ManageVideoTorrentPayload } from '@shared/models' -import { logger } from '../../../helpers/logger' - -async function processManageVideoTorrent (job: Job) { - const payload = job.data as ManageVideoTorrentPayload - logger.info('Processing torrent in job %s.', job.id) - - if (payload.action === 'create') return doCreateAction(payload) - if (payload.action === 'update-metadata') return doUpdateMetadataAction(payload) -} - -// --------------------------------------------------------------------------- - -export { - processManageVideoTorrent -} - -// --------------------------------------------------------------------------- - -async function doCreateAction (payload: ManageVideoTorrentPayload & { action: 'create' }) { - const [ video, file ] = await Promise.all([ - loadVideoOrLog(payload.videoId), - loadFileOrLog(payload.videoFileId) - ]) - - if (!video || !file) return - - const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - await video.reload() - await file.reload() - - await createTorrentAndSetInfoHash(video, file) - - // Refresh videoFile because the createTorrentAndSetInfoHash could be long - const refreshedFile = await VideoFileModel.loadWithVideo(file.id) - // File does not exist anymore, remove the generated torrent - if (!refreshedFile) return file.removeTorrent() - - refreshedFile.infoHash = file.infoHash - refreshedFile.torrentFilename = file.torrentFilename - - await refreshedFile.save() - } finally { - fileMutexReleaser() - } -} - -async function doUpdateMetadataAction (payload: ManageVideoTorrentPayload & { action: 'update-metadata' }) { - const [ video, streamingPlaylist, file ] = await Promise.all([ - loadVideoOrLog(payload.videoId), - loadStreamingPlaylistOrLog(payload.streamingPlaylistId), - loadFileOrLog(payload.videoFileId) - ]) - - if ((!video && !streamingPlaylist) || !file) return - - const extractedVideo = extractVideo(video || streamingPlaylist) - const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(extractedVideo.uuid) - - try { - await updateTorrentMetadata(video || streamingPlaylist, file) - - await file.save() - } finally { - fileMutexReleaser() - } -} - -async function loadVideoOrLog (videoId: number) { - if (!videoId) return undefined - - const video = await VideoModel.load(videoId) - if (!video) { - logger.debug('Do not process torrent for video %d: does not exist anymore.', videoId) - } - - return video -} - -async function loadStreamingPlaylistOrLog (streamingPlaylistId: number) { - if (!streamingPlaylistId) return undefined - - const streamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(streamingPlaylistId) - if (!streamingPlaylist) { - logger.debug('Do not process torrent for streaming playlist %d: does not exist anymore.', streamingPlaylistId) - } - - return streamingPlaylist -} - -async function loadFileOrLog (videoFileId: number) { - if (!videoFileId) return undefined - - const file = await VideoFileModel.load(videoFileId) - - if (!file) { - logger.debug('Do not process torrent for file %d: does not exist anymore.', videoFileId) - } - - return file -} diff --git a/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/lib/job-queue/handlers/move-to-object-storage.ts deleted file mode 100644 index 9a99b6722..000000000 --- a/server/lib/job-queue/handlers/move-to-object-storage.ts +++ /dev/null @@ -1,159 +0,0 @@ -import { Job } from 'bullmq' -import { remove } from 'fs-extra' -import { join } from 'path' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { updateTorrentMetadata } from '@server/helpers/webtorrent' -import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants' -import { storeHLSFileFromFilename, storeWebVideoFile } from '@server/lib/object-storage' -import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state' -import { VideoModel } from '@server/models/video/video' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models' -import { MoveObjectStoragePayload, VideoState, VideoStorage } from '@shared/models' - -const lTagsBase = loggerTagsFactory('move-object-storage') - -export async function processMoveToObjectStorage (job: Job) { - const payload = job.data as MoveObjectStoragePayload - logger.info('Moving video %s in job %s.', payload.videoUUID, job.id) - - const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID) - - const video = await VideoModel.loadWithFiles(payload.videoUUID) - // No video, maybe deleted? - if (!video) { - logger.info('Can\'t process job %d, video does not exist.', job.id, lTagsBase(payload.videoUUID)) - fileMutexReleaser() - return undefined - } - - const lTags = lTagsBase(video.uuid, video.url) - - try { - if (video.VideoFiles) { - logger.debug('Moving %d web video files for video %s.', video.VideoFiles.length, video.uuid, lTags) - - await moveWebVideoFiles(video) - } - - if (video.VideoStreamingPlaylists) { - logger.debug('Moving HLS playlist of %s.', video.uuid) - - await moveHLSFiles(video) - } - - const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove') - if (pendingMove === 0) { - logger.info('Running cleanup after moving files to object storage (video %s in job %s)', video.uuid, job.id, lTags) - - await doAfterLastJob({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }) - } - } catch (err) { - await onMoveToObjectStorageFailure(job, err) - - throw err - } finally { - fileMutexReleaser() - } - - return payload.videoUUID -} - -export async function onMoveToObjectStorageFailure (job: Job, err: any) { - const payload = job.data as MoveObjectStoragePayload - - const video = await VideoModel.loadWithFiles(payload.videoUUID) - if (!video) return - - logger.error('Cannot move video %s to object storage.', video.url, { err, ...lTagsBase(video.uuid, video.url) }) - - await moveToFailedMoveToObjectStorageState(video) - await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove') -} - -// --------------------------------------------------------------------------- - -async function moveWebVideoFiles (video: MVideoWithAllFiles) { - for (const file of video.VideoFiles) { - if (file.storage !== VideoStorage.FILE_SYSTEM) continue - - const fileUrl = await storeWebVideoFile(video, file) - - const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file) - await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath }) - } -} - -async function moveHLSFiles (video: MVideoWithAllFiles) { - for (const playlist of video.VideoStreamingPlaylists) { - const playlistWithVideo = playlist.withVideo(video) - - for (const file of playlist.VideoFiles) { - if (file.storage !== VideoStorage.FILE_SYSTEM) continue - - // Resolution playlist - const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) - await storeHLSFileFromFilename(playlistWithVideo, playlistFilename) - - // Resolution fragmented file - const fileUrl = await storeHLSFileFromFilename(playlistWithVideo, file.filename) - - const oldPath = join(getHLSDirectory(video), file.filename) - - await onFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath }) - } - } -} - -async function doAfterLastJob (options: { - video: MVideoWithAllFiles - previousVideoState: VideoState - isNewVideo: boolean -}) { - const { video, previousVideoState, isNewVideo } = options - - for (const playlist of video.VideoStreamingPlaylists) { - if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue - - const playlistWithVideo = playlist.withVideo(video) - - // Master playlist - playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename) - // Sha256 segments file - playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename) - - playlist.storage = VideoStorage.OBJECT_STORAGE - - playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles) - playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION - - await playlist.save() - } - - // Remove empty hls video directory - if (video.VideoStreamingPlaylists) { - await remove(getHLSDirectory(video)) - } - - await moveToNextState({ video, previousVideoState, isNewVideo }) -} - -async function onFileMoved (options: { - videoOrPlaylist: MVideo | MStreamingPlaylistVideo - file: MVideoFile - fileUrl: string - oldPath: string -}) { - const { videoOrPlaylist, file, fileUrl, oldPath } = options - - file.fileUrl = fileUrl - file.storage = VideoStorage.OBJECT_STORAGE - - await updateTorrentMetadata(videoOrPlaylist, file) - await file.save() - - logger.debug('Removing %s because it\'s now on object storage', oldPath) - await remove(oldPath) -} diff --git a/server/lib/job-queue/handlers/notify.ts b/server/lib/job-queue/handlers/notify.ts deleted file mode 100644 index 83605396c..000000000 --- a/server/lib/job-queue/handlers/notify.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { Job } from 'bullmq' -import { Notifier } from '@server/lib/notifier' -import { VideoModel } from '@server/models/video/video' -import { NotifyPayload } from '@shared/models' -import { logger } from '../../../helpers/logger' - -async function processNotify (job: Job) { - const payload = job.data as NotifyPayload - logger.info('Processing %s notification in job %s.', payload.action, job.id) - - if (payload.action === 'new-video') return doNotifyNewVideo(payload) -} - -// --------------------------------------------------------------------------- - -export { - processNotify -} - -// --------------------------------------------------------------------------- - -async function doNotifyNewVideo (payload: NotifyPayload & { action: 'new-video' }) { - const refreshedVideo = await VideoModel.loadFull(payload.videoUUID) - if (!refreshedVideo) return - - Notifier.Instance.notifyOnNewVideoIfNeeded(refreshedVideo) -} diff --git a/server/lib/job-queue/handlers/transcoding-job-builder.ts b/server/lib/job-queue/handlers/transcoding-job-builder.ts deleted file mode 100644 index 8621b109f..000000000 --- a/server/lib/job-queue/handlers/transcoding-job-builder.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { Job } from 'bullmq' -import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job' -import { UserModel } from '@server/models/user/user' -import { VideoModel } from '@server/models/video/video' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { pick } from '@shared/core-utils' -import { TranscodingJobBuilderPayload } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { JobQueue } from '../job-queue' - -async function processTranscodingJobBuilder (job: Job) { - const payload = job.data as TranscodingJobBuilderPayload - - logger.info('Processing transcoding job builder in job %s.', job.id) - - if (payload.optimizeJob) { - const video = await VideoModel.loadFull(payload.videoUUID) - const user = await UserModel.loadByVideoId(video.id) - const videoFile = video.getMaxQualityFile() - - await createOptimizeOrMergeAudioJobs({ - ...pick(payload.optimizeJob, [ 'isNewVideo' ]), - - video, - videoFile, - user, - videoFileAlreadyLocked: false - }) - } - - for (const job of (payload.jobs || [])) { - await JobQueue.Instance.createJob(job) - - await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode') - } - - for (const sequentialJobs of (payload.sequentialJobs || [])) { - await JobQueue.Instance.createSequentialJobFlow(...sequentialJobs) - - await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode', sequentialJobs.filter(s => !!s).length) - } -} - -// --------------------------------------------------------------------------- - -export { - processTranscodingJobBuilder -} diff --git a/server/lib/job-queue/handlers/video-channel-import.ts b/server/lib/job-queue/handlers/video-channel-import.ts deleted file mode 100644 index 035f88e96..000000000 --- a/server/lib/job-queue/handlers/video-channel-import.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { Job } from 'bullmq' -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { synchronizeChannel } from '@server/lib/sync-channel' -import { VideoChannelModel } from '@server/models/video/video-channel' -import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' -import { MChannelSync } from '@server/types/models' -import { VideoChannelImportPayload } from '@shared/models' - -export async function processVideoChannelImport (job: Job) { - const payload = job.data as VideoChannelImportPayload - - logger.info('Processing video channel import in job %s.', job.id) - - // Channel import requires only http upload to be allowed - if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) { - throw new Error('Cannot import channel as the HTTP upload is disabled') - } - - if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) { - throw new Error('Cannot import channel as the synchronization is disabled') - } - - let channelSync: MChannelSync - if (payload.partOfChannelSyncId) { - channelSync = await VideoChannelSyncModel.loadWithChannel(payload.partOfChannelSyncId) - - if (!channelSync) { - throw new Error('Unlnown channel sync specified in videos channel import') - } - } - - const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId) - - logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `) - - await synchronizeChannel({ - channel: videoChannel, - externalChannelUrl: payload.externalChannelUrl, - channelSync, - videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.FULL_SYNC_VIDEOS_LIMIT - }) -} diff --git a/server/lib/job-queue/handlers/video-file-import.ts b/server/lib/job-queue/handlers/video-file-import.ts deleted file mode 100644 index d221e8968..000000000 --- a/server/lib/job-queue/handlers/video-file-import.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { Job } from 'bullmq' -import { copy, stat } from 'fs-extra' -import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' -import { CONFIG } from '@server/initializers/config' -import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' -import { generateWebVideoFilename } from '@server/lib/paths' -import { buildMoveToObjectStorageJob } from '@server/lib/video' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { VideoModel } from '@server/models/video/video' -import { VideoFileModel } from '@server/models/video/video-file' -import { MVideoFullLight } from '@server/types/models' -import { getLowercaseExtension } from '@shared/core-utils' -import { getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg' -import { VideoFileImportPayload, VideoStorage } from '@shared/models' -import { logger } from '../../../helpers/logger' -import { JobQueue } from '../job-queue' - -async function processVideoFileImport (job: Job) { - const payload = job.data as VideoFileImportPayload - logger.info('Processing video file import in job %s.', job.id) - - const video = await VideoModel.loadFull(payload.videoUUID) - // No video, maybe deleted? - if (!video) { - logger.info('Do not process job %d, video does not exist.', job.id) - return undefined - } - - await updateVideoFile(video, payload.filePath) - - if (CONFIG.OBJECT_STORAGE.ENABLED) { - await JobQueue.Instance.createJob(await buildMoveToObjectStorageJob({ video, previousVideoState: video.state })) - } else { - await federateVideoIfNeeded(video, false) - } - - return video -} - -// --------------------------------------------------------------------------- - -export { - processVideoFileImport -} - -// --------------------------------------------------------------------------- - -async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) { - const { resolution } = await getVideoStreamDimensionsInfo(inputFilePath) - const { size } = await stat(inputFilePath) - const fps = await getVideoStreamFPS(inputFilePath) - - const fileExt = getLowercaseExtension(inputFilePath) - - const currentVideoFile = video.VideoFiles.find(videoFile => videoFile.resolution === resolution) - - if (currentVideoFile) { - // Remove old file and old torrent - await video.removeWebVideoFile(currentVideoFile) - // Remove the old video file from the array - video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile) - - await currentVideoFile.destroy() - } - - const newVideoFile = new VideoFileModel({ - resolution, - extname: fileExt, - filename: generateWebVideoFilename(resolution, fileExt), - storage: VideoStorage.FILE_SYSTEM, - size, - fps, - videoId: video.id - }) - - const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile) - await copy(inputFilePath, outputPath) - - video.VideoFiles.push(newVideoFile) - await createTorrentAndSetInfoHash(video, newVideoFile) - - await newVideoFile.save() -} diff --git a/server/lib/job-queue/handlers/video-import.ts b/server/lib/job-queue/handlers/video-import.ts deleted file mode 100644 index e5cd258d6..000000000 --- a/server/lib/job-queue/handlers/video-import.ts +++ /dev/null @@ -1,344 +0,0 @@ -import { Job } from 'bullmq' -import { move, remove, stat } from 'fs-extra' -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { YoutubeDLWrapper } from '@server/helpers/youtube-dl' -import { CONFIG } from '@server/initializers/config' -import { isPostImportVideoAccepted } from '@server/lib/moderation' -import { generateWebVideoFilename } from '@server/lib/paths' -import { Hooks } from '@server/lib/plugins/hooks' -import { ServerConfigManager } from '@server/lib/server-config-manager' -import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job' -import { isAbleToUploadVideo } from '@server/lib/user' -import { buildMoveToObjectStorageJob } from '@server/lib/video' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { buildNextVideoState } from '@server/lib/video-state' -import { ThumbnailModel } from '@server/models/video/thumbnail' -import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' -import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import' -import { getLowercaseExtension } from '@shared/core-utils' -import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS, isAudioFile } from '@shared/ffmpeg' -import { - ThumbnailType, - VideoImportPayload, - VideoImportPreventExceptionResult, - VideoImportState, - VideoImportTorrentPayload, - VideoImportTorrentPayloadType, - VideoImportYoutubeDLPayload, - VideoImportYoutubeDLPayloadType, - VideoResolution, - VideoState -} from '@shared/models' -import { logger } from '../../../helpers/logger' -import { getSecureTorrentName } from '../../../helpers/utils' -import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent' -import { JOB_TTL } from '../../../initializers/constants' -import { sequelizeTypescript } from '../../../initializers/database' -import { VideoModel } from '../../../models/video/video' -import { VideoFileModel } from '../../../models/video/video-file' -import { VideoImportModel } from '../../../models/video/video-import' -import { federateVideoIfNeeded } from '../../activitypub/videos' -import { Notifier } from '../../notifier' -import { generateLocalVideoMiniature } from '../../thumbnail' -import { JobQueue } from '../job-queue' - -async function processVideoImport (job: Job): Promise { - const payload = job.data as VideoImportPayload - - const videoImport = await getVideoImportOrDie(payload) - if (videoImport.state === VideoImportState.CANCELLED) { - logger.info('Do not process import since it has been cancelled', { payload }) - return { resultType: 'success' } - } - - videoImport.state = VideoImportState.PROCESSING - await videoImport.save() - - try { - if (payload.type === 'youtube-dl') await processYoutubeDLImport(job, videoImport, payload) - if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') await processTorrentImport(job, videoImport, payload) - - return { resultType: 'success' } - } catch (err) { - if (!payload.preventException) throw err - - logger.warn('Catch error in video import to send value to parent job.', { payload, err }) - return { resultType: 'error' } - } -} - -// --------------------------------------------------------------------------- - -export { - processVideoImport -} - -// --------------------------------------------------------------------------- - -async function processTorrentImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportTorrentPayload) { - logger.info('Processing torrent video import in job %s.', job.id) - - const options = { type: payload.type, videoImportId: payload.videoImportId } - - const target = { - torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined, - uri: videoImport.magnetUri - } - return processFile(() => downloadWebTorrentVideo(target, JOB_TTL['video-import']), videoImport, options) -} - -async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportYoutubeDLPayload) { - logger.info('Processing youtubeDL video import in job %s.', job.id) - - const options = { type: payload.type, videoImportId: videoImport.id } - - const youtubeDL = new YoutubeDLWrapper( - videoImport.targetUrl, - ServerConfigManager.Instance.getEnabledResolutions('vod'), - CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION - ) - - return processFile( - () => youtubeDL.downloadVideo(payload.fileExt, JOB_TTL['video-import']), - videoImport, - options - ) -} - -async function getVideoImportOrDie (payload: VideoImportPayload) { - const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId) - if (!videoImport?.Video) { - throw new Error(`Cannot import video ${payload.videoImportId}: the video import or video linked to this import does not exist anymore.`) - } - - return videoImport -} - -type ProcessFileOptions = { - type: VideoImportYoutubeDLPayloadType | VideoImportTorrentPayloadType - videoImportId: number -} -async function processFile (downloader: () => Promise, videoImport: MVideoImportDefault, options: ProcessFileOptions) { - let tempVideoPath: string - let videoFile: VideoFileModel - - try { - // Download video from youtubeDL - tempVideoPath = await downloader() - - // Get information about this video - const stats = await stat(tempVideoPath) - const isAble = await isAbleToUploadVideo(videoImport.User.id, stats.size) - if (isAble === false) { - throw new Error('The user video quota is exceeded with this video to import.') - } - - const probe = await ffprobePromise(tempVideoPath) - - const { resolution } = await isAudioFile(tempVideoPath, probe) - ? { resolution: VideoResolution.H_NOVIDEO } - : await getVideoStreamDimensionsInfo(tempVideoPath, probe) - - const fps = await getVideoStreamFPS(tempVideoPath, probe) - const duration = await getVideoStreamDuration(tempVideoPath, probe) - - // Prepare video file object for creation in database - const fileExt = getLowercaseExtension(tempVideoPath) - const videoFileData = { - extname: fileExt, - resolution, - size: stats.size, - filename: generateWebVideoFilename(resolution, fileExt), - fps, - videoId: videoImport.videoId - } - videoFile = new VideoFileModel(videoFileData) - - const hookName = options.type === 'youtube-dl' - ? 'filter:api.video.post-import-url.accept.result' - : 'filter:api.video.post-import-torrent.accept.result' - - // Check we accept this video - const acceptParameters = { - videoImport, - video: videoImport.Video, - videoFilePath: tempVideoPath, - videoFile, - user: videoImport.User - } - const acceptedResult = await Hooks.wrapFun(isPostImportVideoAccepted, acceptParameters, hookName) - - if (acceptedResult.accepted !== true) { - logger.info('Refused imported video.', { acceptedResult, acceptParameters }) - - videoImport.state = VideoImportState.REJECTED - await videoImport.save() - - throw new Error(acceptedResult.errorMessage) - } - - // Video is accepted, resuming preparation - const videoFileLockReleaser = await VideoPathManager.Instance.lockFiles(videoImport.Video.uuid) - - try { - const videoImportWithFiles = await refreshVideoImportFromDB(videoImport, videoFile) - - // Move file - const videoDestFile = VideoPathManager.Instance.getFSVideoFileOutputPath(videoImportWithFiles.Video, videoFile) - await move(tempVideoPath, videoDestFile) - - tempVideoPath = null // This path is not used anymore - - let { - miniatureModel: thumbnailModel, - miniatureJSONSave: thumbnailSave - } = await generateMiniature(videoImportWithFiles, videoFile, ThumbnailType.MINIATURE) - - let { - miniatureModel: previewModel, - miniatureJSONSave: previewSave - } = await generateMiniature(videoImportWithFiles, videoFile, ThumbnailType.PREVIEW) - - // Create torrent - await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile) - - const videoFileSave = videoFile.toJSON() - - const { videoImportUpdated, video } = await retryTransactionWrapper(() => { - return sequelizeTypescript.transaction(async t => { - // Refresh video - const video = await VideoModel.load(videoImportWithFiles.videoId, t) - if (!video) throw new Error('Video linked to import ' + videoImportWithFiles.videoId + ' does not exist anymore.') - - await videoFile.save({ transaction: t }) - - // Update video DB object - video.duration = duration - video.state = buildNextVideoState(video.state) - await video.save({ transaction: t }) - - if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t) - if (previewModel) await video.addAndSaveThumbnail(previewModel, t) - - // Now we can federate the video (reload from database, we need more attributes) - const videoForFederation = await VideoModel.loadFull(video.uuid, t) - await federateVideoIfNeeded(videoForFederation, true, t) - - // Update video import object - videoImportWithFiles.state = VideoImportState.SUCCESS - const videoImportUpdated = await videoImportWithFiles.save({ transaction: t }) as MVideoImport - - logger.info('Video %s imported.', video.uuid) - - return { videoImportUpdated, video: videoForFederation } - }).catch(err => { - // Reset fields - if (thumbnailModel) thumbnailModel = new ThumbnailModel(thumbnailSave) - if (previewModel) previewModel = new ThumbnailModel(previewSave) - - videoFile = new VideoFileModel(videoFileSave) - - throw err - }) - }) - - await afterImportSuccess({ videoImport: videoImportUpdated, video, videoFile, user: videoImport.User, videoFileAlreadyLocked: true }) - } finally { - videoFileLockReleaser() - } - } catch (err) { - await onImportError(err, tempVideoPath, videoImport) - - throw err - } -} - -async function refreshVideoImportFromDB (videoImport: MVideoImportDefault, videoFile: MVideoFile): Promise { - // Refresh video, privacy may have changed - const video = await videoImport.Video.reload() - const videoWithFiles = Object.assign(video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] }) - - return Object.assign(videoImport, { Video: videoWithFiles }) -} - -async function generateMiniature (videoImportWithFiles: MVideoImportDefaultFiles, videoFile: MVideoFile, thumbnailType: ThumbnailType) { - // Generate miniature if the import did not created it - const needsMiniature = thumbnailType === ThumbnailType.MINIATURE - ? !videoImportWithFiles.Video.getMiniature() - : !videoImportWithFiles.Video.getPreview() - - if (!needsMiniature) { - return { - miniatureModel: null, - miniatureJSONSave: null - } - } - - const miniatureModel = await generateLocalVideoMiniature({ - video: videoImportWithFiles.Video, - videoFile, - type: thumbnailType - }) - const miniatureJSONSave = miniatureModel.toJSON() - - return { - miniatureModel, - miniatureJSONSave - } -} - -async function afterImportSuccess (options: { - videoImport: MVideoImport - video: MVideoFullLight - videoFile: MVideoFile - user: MUserId - videoFileAlreadyLocked: boolean -}) { - const { video, videoFile, videoImport, user, videoFileAlreadyLocked } = options - - Notifier.Instance.notifyOnFinishedVideoImport({ videoImport: Object.assign(videoImport, { Video: video }), success: true }) - - if (video.isBlacklisted()) { - const videoBlacklist = Object.assign(video.VideoBlacklist, { Video: video }) - - Notifier.Instance.notifyOnVideoAutoBlacklist(videoBlacklist) - } else { - Notifier.Instance.notifyOnNewVideoIfNeeded(video) - } - - // Generate the storyboard in the job queue, and don't forget to federate an update after - await JobQueue.Instance.createJob({ - type: 'generate-video-storyboard' as 'generate-video-storyboard', - payload: { - videoUUID: video.uuid, - federate: true - } - }) - - if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { - await JobQueue.Instance.createJob( - await buildMoveToObjectStorageJob({ video, previousVideoState: VideoState.TO_IMPORT }) - ) - return - } - - if (video.state === VideoState.TO_TRANSCODE) { // Create transcoding jobs? - await createOptimizeOrMergeAudioJobs({ video, videoFile, isNewVideo: true, user, videoFileAlreadyLocked }) - } -} - -async function onImportError (err: Error, tempVideoPath: string, videoImport: MVideoImportVideo) { - try { - if (tempVideoPath) await remove(tempVideoPath) - } catch (errUnlink) { - logger.warn('Cannot cleanup files after a video import error.', { err: errUnlink }) - } - - videoImport.error = err.message - if (videoImport.state !== VideoImportState.REJECTED) { - videoImport.state = VideoImportState.FAILED - } - await videoImport.save() - - Notifier.Instance.notifyOnFinishedVideoImport({ videoImport, success: false }) -} diff --git a/server/lib/job-queue/handlers/video-live-ending.ts b/server/lib/job-queue/handlers/video-live-ending.ts deleted file mode 100644 index 070d1d7a2..000000000 --- a/server/lib/job-queue/handlers/video-live-ending.ts +++ /dev/null @@ -1,279 +0,0 @@ -import { Job } from 'bullmq' -import { readdir, remove } from 'fs-extra' -import { join } from 'path' -import { peertubeTruncate } from '@server/helpers/core-utils' -import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' -import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' -import { federateVideoIfNeeded } from '@server/lib/activitypub/videos' -import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live' -import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths' -import { generateLocalVideoMiniature, regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail' -import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/hls-transcoding' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { moveToNextState } from '@server/lib/video-state' -import { VideoModel } from '@server/models/video/video' -import { VideoBlacklistModel } from '@server/models/video/video-blacklist' -import { VideoFileModel } from '@server/models/video/video-file' -import { VideoLiveModel } from '@server/models/video/video-live' -import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting' -import { VideoLiveSessionModel } from '@server/models/video/video-live-session' -import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@server/types/models' -import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg' -import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@shared/models' -import { logger, loggerTagsFactory } from '../../../helpers/logger' -import { JobQueue } from '../job-queue' - -const lTags = loggerTagsFactory('live', 'job') - -async function processVideoLiveEnding (job: Job) { - const payload = job.data as VideoLiveEndingPayload - - logger.info('Processing video live ending for %s.', payload.videoId, { payload, ...lTags() }) - - function logError () { - logger.warn('Video live %d does not exist anymore. Cannot process live ending.', payload.videoId, lTags()) - } - - const video = await VideoModel.load(payload.videoId) - const live = await VideoLiveModel.loadByVideoId(payload.videoId) - const liveSession = await VideoLiveSessionModel.load(payload.liveSessionId) - - if (!video || !live || !liveSession) { - logError() - return - } - - const permanentLive = live.permanentLive - - liveSession.endingProcessed = true - await liveSession.save() - - if (liveSession.saveReplay !== true) { - return cleanupLiveAndFederate({ permanentLive, video, streamingPlaylistId: payload.streamingPlaylistId }) - } - - if (permanentLive) { - await saveReplayToExternalVideo({ - liveVideo: video, - liveSession, - publishedAt: payload.publishedAt, - replayDirectory: payload.replayDirectory - }) - - return cleanupLiveAndFederate({ permanentLive, video, streamingPlaylistId: payload.streamingPlaylistId }) - } - - return replaceLiveByReplay({ - video, - liveSession, - live, - permanentLive, - replayDirectory: payload.replayDirectory - }) -} - -// --------------------------------------------------------------------------- - -export { - processVideoLiveEnding -} - -// --------------------------------------------------------------------------- - -async function saveReplayToExternalVideo (options: { - liveVideo: MVideo - liveSession: MVideoLiveSession - publishedAt: string - replayDirectory: string -}) { - const { liveVideo, liveSession, publishedAt, replayDirectory } = options - - const replaySettings = await VideoLiveReplaySettingModel.load(liveSession.replaySettingId) - - const videoNameSuffix = ` - ${new Date(publishedAt).toLocaleString()}` - const truncatedVideoName = peertubeTruncate(liveVideo.name, { - length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max - videoNameSuffix.length - }) - - const replayVideo = new VideoModel({ - name: truncatedVideoName + videoNameSuffix, - isLive: false, - state: VideoState.TO_TRANSCODE, - duration: 0, - - remote: liveVideo.remote, - category: liveVideo.category, - licence: liveVideo.licence, - language: liveVideo.language, - commentsEnabled: liveVideo.commentsEnabled, - downloadEnabled: liveVideo.downloadEnabled, - waitTranscoding: true, - nsfw: liveVideo.nsfw, - description: liveVideo.description, - support: liveVideo.support, - privacy: replaySettings.privacy, - channelId: liveVideo.channelId - }) as MVideoWithAllFiles - - replayVideo.Thumbnails = [] - replayVideo.VideoFiles = [] - replayVideo.VideoStreamingPlaylists = [] - - replayVideo.url = getLocalVideoActivityPubUrl(replayVideo) - - await replayVideo.save() - - liveSession.replayVideoId = replayVideo.id - await liveSession.save() - - // If live is blacklisted, also blacklist the replay - const blacklist = await VideoBlacklistModel.loadByVideoId(liveVideo.id) - if (blacklist) { - await VideoBlacklistModel.create({ - videoId: replayVideo.id, - unfederated: blacklist.unfederated, - reason: blacklist.reason, - type: blacklist.type - }) - } - - await assignReplayFilesToVideo({ video: replayVideo, replayDirectory }) - - await remove(replayDirectory) - - for (const type of [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ]) { - const image = await generateLocalVideoMiniature({ video: replayVideo, videoFile: replayVideo.getMaxQualityFile(), type }) - await replayVideo.addAndSaveThumbnail(image) - } - - await moveToNextState({ video: replayVideo, isNewVideo: true }) - - await createStoryboardJob(replayVideo) -} - -async function replaceLiveByReplay (options: { - video: MVideo - liveSession: MVideoLiveSession - live: MVideoLive - permanentLive: boolean - replayDirectory: string -}) { - const { video, liveSession, live, permanentLive, replayDirectory } = options - - const replaySettings = await VideoLiveReplaySettingModel.load(liveSession.replaySettingId) - const videoWithFiles = await VideoModel.loadFull(video.id) - const hlsPlaylist = videoWithFiles.getHLSPlaylist() - - await cleanupTMPLiveFiles(videoWithFiles, hlsPlaylist) - - await live.destroy() - - videoWithFiles.isLive = false - videoWithFiles.privacy = replaySettings.privacy - videoWithFiles.waitTranscoding = true - videoWithFiles.state = VideoState.TO_TRANSCODE - - await videoWithFiles.save() - - liveSession.replayVideoId = videoWithFiles.id - await liveSession.save() - - await VideoFileModel.removeHLSFilesOfVideoId(hlsPlaylist.id) - - // Reset playlist - hlsPlaylist.VideoFiles = [] - hlsPlaylist.playlistFilename = generateHLSMasterPlaylistFilename() - hlsPlaylist.segmentsSha256Filename = generateHlsSha256SegmentsFilename() - await hlsPlaylist.save() - - await assignReplayFilesToVideo({ video: videoWithFiles, replayDirectory }) - - // Should not happen in this function, but we keep the code if in the future we can replace the permanent live by a replay - if (permanentLive) { // Remove session replay - await remove(replayDirectory) - } else { // We won't stream again in this live, we can delete the base replay directory - await remove(getLiveReplayBaseDirectory(videoWithFiles)) - } - - // Regenerate the thumbnail & preview? - await regenerateMiniaturesIfNeeded(videoWithFiles) - - // We consider this is a new video - await moveToNextState({ video: videoWithFiles, isNewVideo: true }) - - await createStoryboardJob(videoWithFiles) -} - -async function assignReplayFilesToVideo (options: { - video: MVideo - replayDirectory: string -}) { - const { video, replayDirectory } = options - - const concatenatedTsFiles = await readdir(replayDirectory) - - for (const concatenatedTsFile of concatenatedTsFiles) { - const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) - await video.reload() - - const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile) - - const probe = await ffprobePromise(concatenatedTsFilePath) - const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe) - const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe) - const fps = await getVideoStreamFPS(concatenatedTsFilePath, probe) - - try { - await generateHlsPlaylistResolutionFromTS({ - video, - inputFileMutexReleaser, - concatenatedTsFilePath, - resolution, - fps, - isAAC: audioStream?.codec_name === 'aac' - }) - } catch (err) { - logger.error('Cannot generate HLS playlist resolution from TS files.', { err }) - } - - inputFileMutexReleaser() - } - - return video -} - -async function cleanupLiveAndFederate (options: { - video: MVideo - permanentLive: boolean - streamingPlaylistId: number -}) { - const { permanentLive, video, streamingPlaylistId } = options - - const streamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(streamingPlaylistId) - - if (streamingPlaylist) { - if (permanentLive) { - await cleanupAndDestroyPermanentLive(video, streamingPlaylist) - } else { - await cleanupUnsavedNormalLive(video, streamingPlaylist) - } - } - - try { - const fullVideo = await VideoModel.loadFull(video.id) - return federateVideoIfNeeded(fullVideo, false, undefined) - } catch (err) { - logger.warn('Cannot federate live after cleanup', { videoId: video.id, err }) - } -} - -function createStoryboardJob (video: MVideo) { - return JobQueue.Instance.createJob({ - type: 'generate-video-storyboard' as 'generate-video-storyboard', - payload: { - videoUUID: video.uuid, - federate: true - } - }) -} diff --git a/server/lib/job-queue/handlers/video-redundancy.ts b/server/lib/job-queue/handlers/video-redundancy.ts deleted file mode 100644 index bac99fdb7..000000000 --- a/server/lib/job-queue/handlers/video-redundancy.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { Job } from 'bullmq' -import { VideosRedundancyScheduler } from '@server/lib/schedulers/videos-redundancy-scheduler' -import { VideoRedundancyPayload } from '@shared/models' -import { logger } from '../../../helpers/logger' - -async function processVideoRedundancy (job: Job) { - const payload = job.data as VideoRedundancyPayload - logger.info('Processing video redundancy in job %s.', job.id) - - return VideosRedundancyScheduler.Instance.createManualRedundancy(payload.videoId) -} - -// --------------------------------------------------------------------------- - -export { - processVideoRedundancy -} diff --git a/server/lib/job-queue/handlers/video-studio-edition.ts b/server/lib/job-queue/handlers/video-studio-edition.ts deleted file mode 100644 index caf051bfa..000000000 --- a/server/lib/job-queue/handlers/video-studio-edition.ts +++ /dev/null @@ -1,180 +0,0 @@ -import { Job } from 'bullmq' -import { remove } from 'fs-extra' -import { join } from 'path' -import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' -import { CONFIG } from '@server/initializers/config' -import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles' -import { isAbleToUploadVideo } from '@server/lib/user' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { approximateIntroOutroAdditionalSize, onVideoStudioEnded, safeCleanupStudioTMPFiles } from '@server/lib/video-studio' -import { UserModel } from '@server/models/user/user' -import { VideoModel } from '@server/models/video/video' -import { MVideo, MVideoFullLight } from '@server/types/models' -import { pick } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { FFmpegEdition } from '@shared/ffmpeg' -import { - VideoStudioEditionPayload, - VideoStudioTask, - VideoStudioTaskCutPayload, - VideoStudioTaskIntroPayload, - VideoStudioTaskOutroPayload, - VideoStudioTaskPayload, - VideoStudioTaskWatermarkPayload -} from '@shared/models' -import { logger, loggerTagsFactory } from '../../../helpers/logger' - -const lTagsBase = loggerTagsFactory('video-studio') - -async function processVideoStudioEdition (job: Job) { - const payload = job.data as VideoStudioEditionPayload - const lTags = lTagsBase(payload.videoUUID) - - logger.info('Process video studio edition of %s in job %s.', payload.videoUUID, job.id, lTags) - - try { - const video = await VideoModel.loadFull(payload.videoUUID) - - // No video, maybe deleted? - if (!video) { - logger.info('Can\'t process job %d, video does not exist.', job.id, lTags) - - await safeCleanupStudioTMPFiles(payload.tasks) - return undefined - } - - await checkUserQuotaOrThrow(video, payload) - - const inputFile = video.getMaxQualityFile() - - const editionResultPath = await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async originalFilePath => { - let tmpInputFilePath: string - let outputPath: string - - for (const task of payload.tasks) { - const outputFilename = buildUUID() + inputFile.extname - outputPath = join(CONFIG.STORAGE.TMP_DIR, outputFilename) - - await processTask({ - inputPath: tmpInputFilePath ?? originalFilePath, - video, - outputPath, - task, - lTags - }) - - if (tmpInputFilePath) await remove(tmpInputFilePath) - - // For the next iteration - tmpInputFilePath = outputPath - } - - return outputPath - }) - - logger.info('Video edition ended for video %s.', video.uuid, lTags) - - await onVideoStudioEnded({ video, editionResultPath, tasks: payload.tasks }) - } catch (err) { - await safeCleanupStudioTMPFiles(payload.tasks) - - throw err - } -} - -// --------------------------------------------------------------------------- - -export { - processVideoStudioEdition -} - -// --------------------------------------------------------------------------- - -type TaskProcessorOptions = { - inputPath: string - outputPath: string - video: MVideo - task: T - lTags: { tags: string[] } -} - -const taskProcessors: { [id in VideoStudioTask['name']]: (options: TaskProcessorOptions) => Promise } = { - 'add-intro': processAddIntroOutro, - 'add-outro': processAddIntroOutro, - 'cut': processCut, - 'add-watermark': processAddWatermark -} - -async function processTask (options: TaskProcessorOptions) { - const { video, task, lTags } = options - - logger.info('Processing %s task for video %s.', task.name, video.uuid, { task, ...lTags }) - - const processor = taskProcessors[options.task.name] - if (!process) throw new Error('Unknown task ' + task.name) - - return processor(options) -} - -function processAddIntroOutro (options: TaskProcessorOptions) { - const { task, lTags } = options - - logger.debug('Will add intro/outro to the video.', { options, ...lTags }) - - return buildFFmpegEdition().addIntroOutro({ - ...pick(options, [ 'inputPath', 'outputPath' ]), - - introOutroPath: task.options.file, - type: task.name === 'add-intro' - ? 'intro' - : 'outro' - }) -} - -function processCut (options: TaskProcessorOptions) { - const { task, lTags } = options - - logger.debug('Will cut the video.', { options, ...lTags }) - - return buildFFmpegEdition().cutVideo({ - ...pick(options, [ 'inputPath', 'outputPath' ]), - - start: task.options.start, - end: task.options.end - }) -} - -function processAddWatermark (options: TaskProcessorOptions) { - const { task, lTags } = options - - logger.debug('Will add watermark to the video.', { options, ...lTags }) - - return buildFFmpegEdition().addWatermark({ - ...pick(options, [ 'inputPath', 'outputPath' ]), - - watermarkPath: task.options.file, - - videoFilters: { - watermarkSizeRatio: task.options.watermarkSizeRatio, - horitonzalMarginRatio: task.options.horitonzalMarginRatio, - verticalMarginRatio: task.options.verticalMarginRatio - } - }) -} - -// --------------------------------------------------------------------------- - -async function checkUserQuotaOrThrow (video: MVideoFullLight, payload: VideoStudioEditionPayload) { - const user = await UserModel.loadByVideoId(video.id) - - const filePathFinder = (i: number) => (payload.tasks[i] as VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload).options.file - - const additionalBytes = await approximateIntroOutroAdditionalSize(video, payload.tasks, filePathFinder) - if (await isAbleToUploadVideo(user.id, additionalBytes) === false) { - throw new Error('Quota exceeded for this user to edit the video') - } -} - -function buildFFmpegEdition () { - return new FFmpegEdition(getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders())) -} diff --git a/server/lib/job-queue/handlers/video-transcoding.ts b/server/lib/job-queue/handlers/video-transcoding.ts deleted file mode 100644 index 1c8f4fd9f..000000000 --- a/server/lib/job-queue/handlers/video-transcoding.ts +++ /dev/null @@ -1,150 +0,0 @@ -import { Job } from 'bullmq' -import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding' -import { generateHlsPlaylistResolution } from '@server/lib/transcoding/hls-transcoding' -import { mergeAudioVideofile, optimizeOriginalVideofile, transcodeNewWebVideoResolution } from '@server/lib/transcoding/web-transcoding' -import { removeAllWebVideoFiles } from '@server/lib/video-file' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { moveToFailedTranscodingState } from '@server/lib/video-state' -import { UserModel } from '@server/models/user/user' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { MUser, MUserId, MVideoFullLight } from '@server/types/models' -import { - HLSTranscodingPayload, - MergeAudioTranscodingPayload, - NewWebVideoResolutionTranscodingPayload, - OptimizeTranscodingPayload, - VideoTranscodingPayload -} from '@shared/models' -import { logger, loggerTagsFactory } from '../../../helpers/logger' -import { VideoModel } from '../../../models/video/video' - -type HandlerFunction = (job: Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise - -const handlers: { [ id in VideoTranscodingPayload['type'] ]: HandlerFunction } = { - 'new-resolution-to-hls': handleHLSJob, - 'new-resolution-to-web-video': handleNewWebVideoResolutionJob, - 'merge-audio-to-web-video': handleWebVideoMergeAudioJob, - 'optimize-to-web-video': handleWebVideoOptimizeJob -} - -const lTags = loggerTagsFactory('transcoding') - -async function processVideoTranscoding (job: Job) { - const payload = job.data as VideoTranscodingPayload - logger.info('Processing transcoding job %s.', job.id, lTags(payload.videoUUID)) - - const video = await VideoModel.loadFull(payload.videoUUID) - // No video, maybe deleted? - if (!video) { - logger.info('Do not process job %d, video does not exist.', job.id, lTags(payload.videoUUID)) - return undefined - } - - const user = await UserModel.loadByChannelActorId(video.VideoChannel.actorId) - - const handler = handlers[payload.type] - - if (!handler) { - await moveToFailedTranscodingState(video) - await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') - - throw new Error('Cannot find transcoding handler for ' + payload.type) - } - - try { - await handler(job, payload, video, user) - } catch (error) { - await moveToFailedTranscodingState(video) - - await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') - - throw error - } - - return video -} - -// --------------------------------------------------------------------------- - -export { - processVideoTranscoding -} - -// --------------------------------------------------------------------------- -// Job handlers -// --------------------------------------------------------------------------- - -async function handleWebVideoMergeAudioJob (job: Job, payload: MergeAudioTranscodingPayload, video: MVideoFullLight, user: MUserId) { - logger.info('Handling merge audio transcoding job for %s.', video.uuid, lTags(video.uuid), { payload }) - - await mergeAudioVideofile({ video, resolution: payload.resolution, fps: payload.fps, job }) - - logger.info('Merge audio transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload }) - - await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: !payload.hasChildren, video }) -} - -async function handleWebVideoOptimizeJob (job: Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) { - logger.info('Handling optimize transcoding job for %s.', video.uuid, lTags(video.uuid), { payload }) - - await optimizeOriginalVideofile({ video, inputVideoFile: video.getMaxQualityFile(), quickTranscode: payload.quickTranscode, job }) - - logger.info('Optimize transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload }) - - await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: !payload.hasChildren, video }) -} - -// --------------------------------------------------------------------------- - -async function handleNewWebVideoResolutionJob (job: Job, payload: NewWebVideoResolutionTranscodingPayload, video: MVideoFullLight) { - logger.info('Handling Web Video transcoding job for %s.', video.uuid, lTags(video.uuid), { payload }) - - await transcodeNewWebVideoResolution({ video, resolution: payload.resolution, fps: payload.fps, job }) - - logger.info('Web Video transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload }) - - await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video }) -} - -// --------------------------------------------------------------------------- - -async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, videoArg: MVideoFullLight) { - logger.info('Handling HLS transcoding job for %s.', videoArg.uuid, lTags(videoArg.uuid), { payload }) - - const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid) - let video: MVideoFullLight - - try { - video = await VideoModel.loadFull(videoArg.uuid) - - const videoFileInput = payload.copyCodecs - ? video.getWebVideoFile(payload.resolution) - : video.getMaxQualityFile() - - const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() - - await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => { - return generateHlsPlaylistResolution({ - video, - videoInputPath, - inputFileMutexReleaser, - resolution: payload.resolution, - fps: payload.fps, - copyCodecs: payload.copyCodecs, - job - }) - }) - } finally { - inputFileMutexReleaser() - } - - logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload }) - - if (payload.deleteWebVideoFiles === true) { - logger.info('Removing Web Video files of %s now we have a HLS version of it.', video.uuid, lTags(video.uuid)) - - await removeAllWebVideoFiles(video) - } - - await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video }) -} diff --git a/server/lib/job-queue/handlers/video-views-stats.ts b/server/lib/job-queue/handlers/video-views-stats.ts deleted file mode 100644 index c9aa218e5..000000000 --- a/server/lib/job-queue/handlers/video-views-stats.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { VideoViewModel } from '@server/models/view/video-view' -import { isTestOrDevInstance } from '../../../helpers/core-utils' -import { logger } from '../../../helpers/logger' -import { VideoModel } from '../../../models/video/video' -import { Redis } from '../../redis' - -async function processVideosViewsStats () { - const lastHour = new Date() - - // In test mode, we run this function multiple times per hour, so we don't want the values of the previous hour - if (!isTestOrDevInstance()) lastHour.setHours(lastHour.getHours() - 1) - - const hour = lastHour.getHours() - const startDate = lastHour.setMinutes(0, 0, 0) - const endDate = lastHour.setMinutes(59, 59, 999) - - const videoIds = await Redis.Instance.listVideosViewedForStats(hour) - if (videoIds.length === 0) return - - logger.info('Processing videos views stats in job for hour %d.', hour) - - for (const videoId of videoIds) { - try { - const views = await Redis.Instance.getVideoViewsStats(videoId, hour) - await Redis.Instance.deleteVideoViewsStats(videoId, hour) - - if (views) { - logger.debug('Adding %d views to video %d stats in hour %d.', views, videoId, hour) - - try { - const video = await VideoModel.load(videoId) - if (!video) { - logger.debug('Video %d does not exist anymore, skipping videos view stats.', videoId) - continue - } - - await VideoViewModel.create({ - startDate: new Date(startDate), - endDate: new Date(endDate), - views, - videoId - }) - } catch (err) { - logger.error('Cannot create video views stats for video %d in hour %d.', videoId, hour, { err }) - } - } - } catch (err) { - logger.error('Cannot update video views stats of video %d in hour %d.', videoId, hour, { err }) - } - } -} - -// --------------------------------------------------------------------------- - -export { - processVideosViewsStats -} diff --git a/server/lib/job-queue/index.ts b/server/lib/job-queue/index.ts deleted file mode 100644 index 57231e649..000000000 --- a/server/lib/job-queue/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './job-queue' diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts deleted file mode 100644 index 177bca285..000000000 --- a/server/lib/job-queue/job-queue.ts +++ /dev/null @@ -1,537 +0,0 @@ -import { - FlowJob, - FlowProducer, - Job, - JobsOptions, - Queue, - QueueEvents, - QueueEventsOptions, - QueueOptions, - Worker, - WorkerOptions -} from 'bullmq' -import { parseDurationToMs } from '@server/helpers/core-utils' -import { jobStates } from '@server/helpers/custom-validators/jobs' -import { CONFIG } from '@server/initializers/config' -import { processVideoRedundancy } from '@server/lib/job-queue/handlers/video-redundancy' -import { pick, timeoutPromise } from '@shared/core-utils' -import { - ActivitypubFollowPayload, - ActivitypubHttpBroadcastPayload, - ActivitypubHttpFetcherPayload, - ActivitypubHttpUnicastPayload, - ActorKeysPayload, - AfterVideoChannelImportPayload, - DeleteResumableUploadMetaFilePayload, - EmailPayload, - FederateVideoPayload, - GenerateStoryboardPayload, - JobState, - JobType, - ManageVideoTorrentPayload, - MoveObjectStoragePayload, - NotifyPayload, - RefreshPayload, - TranscodingJobBuilderPayload, - VideoChannelImportPayload, - VideoFileImportPayload, - VideoImportPayload, - VideoLiveEndingPayload, - VideoRedundancyPayload, - VideoStudioEditionPayload, - VideoTranscodingPayload -} from '../../../shared/models' -import { logger } from '../../helpers/logger' -import { JOB_ATTEMPTS, JOB_CONCURRENCY, JOB_REMOVAL_OPTIONS, JOB_TTL, REPEAT_JOBS, WEBSERVER } from '../../initializers/constants' -import { Hooks } from '../plugins/hooks' -import { Redis } from '../redis' -import { processActivityPubCleaner } from './handlers/activitypub-cleaner' -import { processActivityPubFollow } from './handlers/activitypub-follow' -import { processActivityPubHttpSequentialBroadcast, processActivityPubParallelHttpBroadcast } from './handlers/activitypub-http-broadcast' -import { processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' -import { processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' -import { refreshAPObject } from './handlers/activitypub-refresher' -import { processActorKeys } from './handlers/actor-keys' -import { processAfterVideoChannelImport } from './handlers/after-video-channel-import' -import { processEmail } from './handlers/email' -import { processFederateVideo } from './handlers/federate-video' -import { processManageVideoTorrent } from './handlers/manage-video-torrent' -import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage' -import { processNotify } from './handlers/notify' -import { processTranscodingJobBuilder } from './handlers/transcoding-job-builder' -import { processVideoChannelImport } from './handlers/video-channel-import' -import { processVideoFileImport } from './handlers/video-file-import' -import { processVideoImport } from './handlers/video-import' -import { processVideoLiveEnding } from './handlers/video-live-ending' -import { processVideoStudioEdition } from './handlers/video-studio-edition' -import { processVideoTranscoding } from './handlers/video-transcoding' -import { processVideosViewsStats } from './handlers/video-views-stats' -import { processGenerateStoryboard } from './handlers/generate-storyboard' - -export type CreateJobArgument = - { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | - { type: 'activitypub-http-broadcast-parallel', payload: ActivitypubHttpBroadcastPayload } | - { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } | - { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | - { type: 'activitypub-cleaner', payload: {} } | - { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | - { type: 'video-file-import', payload: VideoFileImportPayload } | - { type: 'video-transcoding', payload: VideoTranscodingPayload } | - { type: 'email', payload: EmailPayload } | - { type: 'transcoding-job-builder', payload: TranscodingJobBuilderPayload } | - { type: 'video-import', payload: VideoImportPayload } | - { type: 'activitypub-refresher', payload: RefreshPayload } | - { type: 'videos-views-stats', payload: {} } | - { type: 'video-live-ending', payload: VideoLiveEndingPayload } | - { type: 'actor-keys', payload: ActorKeysPayload } | - { type: 'video-redundancy', payload: VideoRedundancyPayload } | - { type: 'delete-resumable-upload-meta-file', payload: DeleteResumableUploadMetaFilePayload } | - { type: 'video-studio-edition', payload: VideoStudioEditionPayload } | - { type: 'manage-video-torrent', payload: ManageVideoTorrentPayload } | - { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | - { type: 'video-channel-import', payload: VideoChannelImportPayload } | - { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } | - { type: 'notify', payload: NotifyPayload } | - { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | - { type: 'federate-video', payload: FederateVideoPayload } | - { type: 'generate-video-storyboard', payload: GenerateStoryboardPayload } - -export type CreateJobOptions = { - delay?: number - priority?: number - failParentOnFailure?: boolean -} - -const handlers: { [id in JobType]: (job: Job) => Promise } = { - 'activitypub-cleaner': processActivityPubCleaner, - 'activitypub-follow': processActivityPubFollow, - 'activitypub-http-broadcast-parallel': processActivityPubParallelHttpBroadcast, - 'activitypub-http-broadcast': processActivityPubHttpSequentialBroadcast, - 'activitypub-http-fetcher': processActivityPubHttpFetcher, - 'activitypub-http-unicast': processActivityPubHttpUnicast, - 'activitypub-refresher': refreshAPObject, - 'actor-keys': processActorKeys, - 'after-video-channel-import': processAfterVideoChannelImport, - 'email': processEmail, - 'federate-video': processFederateVideo, - 'transcoding-job-builder': processTranscodingJobBuilder, - 'manage-video-torrent': processManageVideoTorrent, - 'move-to-object-storage': processMoveToObjectStorage, - 'notify': processNotify, - 'video-channel-import': processVideoChannelImport, - 'video-file-import': processVideoFileImport, - 'video-import': processVideoImport, - 'video-live-ending': processVideoLiveEnding, - 'video-redundancy': processVideoRedundancy, - 'video-studio-edition': processVideoStudioEdition, - 'video-transcoding': processVideoTranscoding, - 'videos-views-stats': processVideosViewsStats, - 'generate-video-storyboard': processGenerateStoryboard -} - -const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise } = { - 'move-to-object-storage': onMoveToObjectStorageFailure -} - -const jobTypes: JobType[] = [ - 'activitypub-cleaner', - 'activitypub-follow', - 'activitypub-http-broadcast-parallel', - 'activitypub-http-broadcast', - 'activitypub-http-fetcher', - 'activitypub-http-unicast', - 'activitypub-refresher', - 'actor-keys', - 'after-video-channel-import', - 'email', - 'federate-video', - 'generate-video-storyboard', - 'manage-video-torrent', - 'move-to-object-storage', - 'notify', - 'transcoding-job-builder', - 'video-channel-import', - 'video-file-import', - 'video-import', - 'video-live-ending', - 'video-redundancy', - 'video-studio-edition', - 'video-transcoding', - 'videos-views-stats' -] - -const silentFailure = new Set([ 'activitypub-http-unicast' ]) - -class JobQueue { - - private static instance: JobQueue - - private workers: { [id in JobType]?: Worker } = {} - private queues: { [id in JobType]?: Queue } = {} - private queueEvents: { [id in JobType]?: QueueEvents } = {} - - private flowProducer: FlowProducer - - private initialized = false - private jobRedisPrefix: string - - private constructor () { - } - - init () { - // Already initialized - if (this.initialized === true) return - this.initialized = true - - this.jobRedisPrefix = 'bull-' + WEBSERVER.HOST - - for (const handlerName of Object.keys(handlers)) { - this.buildWorker(handlerName) - this.buildQueue(handlerName) - this.buildQueueEvent(handlerName) - } - - this.flowProducer = new FlowProducer({ - connection: Redis.getRedisClientOptions('FlowProducer'), - prefix: this.jobRedisPrefix - }) - this.flowProducer.on('error', err => { logger.error('Error in flow producer', { err }) }) - - this.addRepeatableJobs() - } - - private buildWorker (handlerName: JobType) { - const workerOptions: WorkerOptions = { - autorun: false, - concurrency: this.getJobConcurrency(handlerName), - prefix: this.jobRedisPrefix, - connection: Redis.getRedisClientOptions('Worker'), - maxStalledCount: 10 - } - - const handler = function (job: Job) { - const timeout = JOB_TTL[handlerName] - const p = handlers[handlerName](job) - - if (!timeout) return p - - return timeoutPromise(p, timeout) - } - - const processor = async (jobArg: Job) => { - const job = await Hooks.wrapObject(jobArg, 'filter:job-queue.process.params', { type: handlerName }) - - return Hooks.wrapPromiseFun(handler, job, 'filter:job-queue.process.result') - } - - const worker = new Worker(handlerName, processor, workerOptions) - - worker.on('failed', (job, err) => { - const logLevel = silentFailure.has(handlerName) - ? 'debug' - : 'error' - - logger.log(logLevel, 'Cannot execute job %s in queue %s.', job.id, handlerName, { payload: job.data, err }) - - if (errorHandlers[job.name]) { - errorHandlers[job.name](job, err) - .catch(err => logger.error('Cannot run error handler for job failure %d in queue %s.', job.id, handlerName, { err })) - } - }) - - worker.on('error', err => { logger.error('Error in job worker %s.', handlerName, { err }) }) - - this.workers[handlerName] = worker - } - - private buildQueue (handlerName: JobType) { - const queueOptions: QueueOptions = { - connection: Redis.getRedisClientOptions('Queue'), - prefix: this.jobRedisPrefix - } - - const queue = new Queue(handlerName, queueOptions) - queue.on('error', err => { logger.error('Error in job queue %s.', handlerName, { err }) }) - - this.queues[handlerName] = queue - } - - private buildQueueEvent (handlerName: JobType) { - const queueEventsOptions: QueueEventsOptions = { - autorun: false, - connection: Redis.getRedisClientOptions('QueueEvent'), - prefix: this.jobRedisPrefix - } - - const queueEvents = new QueueEvents(handlerName, queueEventsOptions) - queueEvents.on('error', err => { logger.error('Error in job queue events %s.', handlerName, { err }) }) - - this.queueEvents[handlerName] = queueEvents - } - - // --------------------------------------------------------------------------- - - async terminate () { - const promises = Object.keys(this.workers) - .map(handlerName => { - const worker: Worker = this.workers[handlerName] - const queue: Queue = this.queues[handlerName] - const queueEvent: QueueEvents = this.queueEvents[handlerName] - - return Promise.all([ - worker.close(false), - queue.close(), - queueEvent.close() - ]) - }) - - return Promise.all(promises) - } - - start () { - const promises = Object.keys(this.workers) - .map(handlerName => { - const worker: Worker = this.workers[handlerName] - const queueEvent: QueueEvents = this.queueEvents[handlerName] - - return Promise.all([ - worker.run(), - queueEvent.run() - ]) - }) - - return Promise.all(promises) - } - - async pause () { - for (const handlerName of Object.keys(this.workers)) { - const worker: Worker = this.workers[handlerName] - - await worker.pause() - } - } - - resume () { - for (const handlerName of Object.keys(this.workers)) { - const worker: Worker = this.workers[handlerName] - - worker.resume() - } - } - - // --------------------------------------------------------------------------- - - createJobAsync (options: CreateJobArgument & CreateJobOptions): void { - this.createJob(options) - .catch(err => logger.error('Cannot create job.', { err, options })) - } - - createJob (options: CreateJobArgument & CreateJobOptions) { - const queue: Queue = this.queues[options.type] - if (queue === undefined) { - logger.error('Unknown queue %s: cannot create job.', options.type) - return - } - - const jobOptions = this.buildJobOptions(options.type as JobType, pick(options, [ 'priority', 'delay' ])) - - return queue.add('job', options.payload, jobOptions) - } - - createSequentialJobFlow (...jobs: ((CreateJobArgument & CreateJobOptions) | undefined)[]) { - let lastJob: FlowJob - - for (const job of jobs) { - if (!job) continue - - lastJob = { - ...this.buildJobFlowOption(job), - - children: lastJob - ? [ lastJob ] - : [] - } - } - - return this.flowProducer.add(lastJob) - } - - createJobWithChildren (parent: CreateJobArgument & CreateJobOptions, children: (CreateJobArgument & CreateJobOptions)[]) { - return this.flowProducer.add({ - ...this.buildJobFlowOption(parent), - - children: children.map(c => this.buildJobFlowOption(c)) - }) - } - - private buildJobFlowOption (job: CreateJobArgument & CreateJobOptions): FlowJob { - return { - name: 'job', - data: job.payload, - queueName: job.type, - opts: { - failParentOnFailure: true, - - ...this.buildJobOptions(job.type as JobType, pick(job, [ 'priority', 'delay', 'failParentOnFailure' ])) - } - } - } - - private buildJobOptions (type: JobType, options: CreateJobOptions = {}): JobsOptions { - return { - backoff: { delay: 60 * 1000, type: 'exponential' }, - attempts: JOB_ATTEMPTS[type], - priority: options.priority, - delay: options.delay, - - ...this.buildJobRemovalOptions(type) - } - } - - // --------------------------------------------------------------------------- - - async listForApi (options: { - state?: JobState - start: number - count: number - asc?: boolean - jobType: JobType - }): Promise { - const { state, start, count, asc, jobType } = options - - const states = this.buildStateFilter(state) - const filteredJobTypes = this.buildTypeFilter(jobType) - - let results: Job[] = [] - - for (const jobType of filteredJobTypes) { - const queue: Queue = this.queues[jobType] - - if (queue === undefined) { - logger.error('Unknown queue %s to list jobs.', jobType) - continue - } - - const jobs = await queue.getJobs(states, 0, start + count, asc) - results = results.concat(jobs) - } - - results.sort((j1: any, j2: any) => { - if (j1.timestamp < j2.timestamp) return -1 - else if (j1.timestamp === j2.timestamp) return 0 - - return 1 - }) - - if (asc === false) results.reverse() - - return results.slice(start, start + count) - } - - async count (state: JobState, jobType?: JobType): Promise { - const states = state ? [ state ] : jobStates - const filteredJobTypes = this.buildTypeFilter(jobType) - - let total = 0 - - for (const type of filteredJobTypes) { - const queue = this.queues[type] - if (queue === undefined) { - logger.error('Unknown queue %s to count jobs.', type) - continue - } - - const counts = await queue.getJobCounts() - - for (const s of states) { - total += counts[s] - } - } - - return total - } - - private buildStateFilter (state?: JobState) { - if (!state) return jobStates - - const states = [ state ] - - // Include parent if filtering on waiting - if (state === 'waiting') states.push('waiting-children') - - return states - } - - private buildTypeFilter (jobType?: JobType) { - if (!jobType) return jobTypes - - return jobTypes.filter(t => t === jobType) - } - - async getStats () { - const promises = jobTypes.map(async t => ({ jobType: t, counts: await this.queues[t].getJobCounts() })) - - return Promise.all(promises) - } - - // --------------------------------------------------------------------------- - - async removeOldJobs () { - for (const key of Object.keys(this.queues)) { - const queue: Queue = this.queues[key] - await queue.clean(parseDurationToMs('7 days'), 1000, 'completed') - await queue.clean(parseDurationToMs('7 days'), 1000, 'failed') - } - } - - private addRepeatableJobs () { - this.queues['videos-views-stats'].add('job', {}, { - repeat: REPEAT_JOBS['videos-views-stats'], - - ...this.buildJobRemovalOptions('videos-views-stats') - }).catch(err => logger.error('Cannot add repeatable job.', { err })) - - if (CONFIG.FEDERATION.VIDEOS.CLEANUP_REMOTE_INTERACTIONS) { - this.queues['activitypub-cleaner'].add('job', {}, { - repeat: REPEAT_JOBS['activitypub-cleaner'], - - ...this.buildJobRemovalOptions('activitypub-cleaner') - }).catch(err => logger.error('Cannot add repeatable job.', { err })) - } - } - - private getJobConcurrency (jobType: JobType) { - if (jobType === 'video-transcoding') return CONFIG.TRANSCODING.CONCURRENCY - if (jobType === 'video-import') return CONFIG.IMPORT.VIDEOS.CONCURRENCY - - return JOB_CONCURRENCY[jobType] - } - - private buildJobRemovalOptions (queueName: string) { - return { - removeOnComplete: { - // Wants seconds - age: (JOB_REMOVAL_OPTIONS.SUCCESS[queueName] || JOB_REMOVAL_OPTIONS.SUCCESS.DEFAULT) / 1000, - - count: JOB_REMOVAL_OPTIONS.COUNT - }, - removeOnFail: { - // Wants seconds - age: (JOB_REMOVAL_OPTIONS.FAILURE[queueName] || JOB_REMOVAL_OPTIONS.FAILURE.DEFAULT) / 1000, - - count: JOB_REMOVAL_OPTIONS.COUNT / 1000 - } - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -// --------------------------------------------------------------------------- - -export { - jobTypes, - JobQueue -} diff --git a/server/lib/live/index.ts b/server/lib/live/index.ts deleted file mode 100644 index 8b46800da..000000000 --- a/server/lib/live/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './live-manager' -export * from './live-quota-store' -export * from './live-segment-sha-store' -export * from './live-utils' diff --git a/server/lib/live/live-manager.ts b/server/lib/live/live-manager.ts deleted file mode 100644 index acb7af274..000000000 --- a/server/lib/live/live-manager.ts +++ /dev/null @@ -1,552 +0,0 @@ -import { readdir, readFile } from 'fs-extra' -import { createServer, Server } from 'net' -import { join } from 'path' -import { createServer as createServerTLS, Server as ServerTLS } from 'tls' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config' -import { VIDEO_LIVE, WEBSERVER } from '@server/initializers/constants' -import { sequelizeTypescript } from '@server/initializers/database' -import { RunnerJobModel } from '@server/models/runner/runner-job' -import { UserModel } from '@server/models/user/user' -import { VideoModel } from '@server/models/video/video' -import { VideoLiveModel } from '@server/models/video/video-live' -import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting' -import { VideoLiveSessionModel } from '@server/models/video/video-live-session' -import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { MVideo, MVideoLiveSession, MVideoLiveVideo, MVideoLiveVideoWithSetting } from '@server/types/models' -import { pick, wait } from '@shared/core-utils' -import { ffprobePromise, getVideoStreamBitrate, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream } from '@shared/ffmpeg' -import { LiveVideoError, VideoState } from '@shared/models' -import { federateVideoIfNeeded } from '../activitypub/videos' -import { JobQueue } from '../job-queue' -import { getLiveReplayBaseDirectory } from '../paths' -import { PeerTubeSocket } from '../peertube-socket' -import { Hooks } from '../plugins/hooks' -import { computeResolutionsToTranscode } from '../transcoding/transcoding-resolutions' -import { LiveQuotaStore } from './live-quota-store' -import { cleanupAndDestroyPermanentLive, getLiveSegmentTime } from './live-utils' -import { MuxingSession } from './shared' - -const NodeRtmpSession = require('node-media-server/src/node_rtmp_session') -const context = require('node-media-server/src/node_core_ctx') -const nodeMediaServerLogger = require('node-media-server/src/node_core_logger') - -// Disable node media server logs -nodeMediaServerLogger.setLogType(0) - -const config = { - rtmp: { - port: CONFIG.LIVE.RTMP.PORT, - chunk_size: VIDEO_LIVE.RTMP.CHUNK_SIZE, - gop_cache: VIDEO_LIVE.RTMP.GOP_CACHE, - ping: VIDEO_LIVE.RTMP.PING, - ping_timeout: VIDEO_LIVE.RTMP.PING_TIMEOUT - } -} - -const lTags = loggerTagsFactory('live') - -class LiveManager { - - private static instance: LiveManager - - private readonly muxingSessions = new Map() - private readonly videoSessions = new Map() - - private rtmpServer: Server - private rtmpsServer: ServerTLS - - private running = false - - private constructor () { - } - - init () { - const events = this.getContext().nodeEvent - events.on('postPublish', (sessionId: string, streamPath: string) => { - logger.debug('RTMP received stream', { id: sessionId, streamPath, ...lTags(sessionId) }) - - const splittedPath = streamPath.split('/') - if (splittedPath.length !== 3 || splittedPath[1] !== VIDEO_LIVE.RTMP.BASE_PATH) { - logger.warn('Live path is incorrect.', { streamPath, ...lTags(sessionId) }) - return this.abortSession(sessionId) - } - - const session = this.getContext().sessions.get(sessionId) - const inputLocalUrl = session.inputOriginLocalUrl + streamPath - const inputPublicUrl = session.inputOriginPublicUrl + streamPath - - this.handleSession({ sessionId, inputPublicUrl, inputLocalUrl, streamKey: splittedPath[2] }) - .catch(err => logger.error('Cannot handle sessions.', { err, ...lTags(sessionId) })) - }) - - events.on('donePublish', sessionId => { - logger.info('Live session ended.', { sessionId, ...lTags(sessionId) }) - - // Force session aborting, so we kill ffmpeg even if it still has data to process (slow CPU) - setTimeout(() => this.abortSession(sessionId), 2000) - }) - - registerConfigChangedHandler(() => { - if (!this.running && CONFIG.LIVE.ENABLED === true) { - this.run().catch(err => logger.error('Cannot run live server.', { err })) - return - } - - if (this.running && CONFIG.LIVE.ENABLED === false) { - this.stop() - } - }) - - // Cleanup broken lives, that were terminated by a server restart for example - this.handleBrokenLives() - .catch(err => logger.error('Cannot handle broken lives.', { err, ...lTags() })) - } - - async run () { - this.running = true - - if (CONFIG.LIVE.RTMP.ENABLED) { - logger.info('Running RTMP server on port %d', CONFIG.LIVE.RTMP.PORT, lTags()) - - this.rtmpServer = createServer(socket => { - const session = new NodeRtmpSession(config, socket) - - session.inputOriginLocalUrl = 'rtmp://127.0.0.1:' + CONFIG.LIVE.RTMP.PORT - session.inputOriginPublicUrl = WEBSERVER.RTMP_URL - session.run() - }) - - this.rtmpServer.on('error', err => { - logger.error('Cannot run RTMP server.', { err, ...lTags() }) - }) - - this.rtmpServer.listen(CONFIG.LIVE.RTMP.PORT, CONFIG.LIVE.RTMP.HOSTNAME) - } - - if (CONFIG.LIVE.RTMPS.ENABLED) { - logger.info('Running RTMPS server on port %d', CONFIG.LIVE.RTMPS.PORT, lTags()) - - const [ key, cert ] = await Promise.all([ - readFile(CONFIG.LIVE.RTMPS.KEY_FILE), - readFile(CONFIG.LIVE.RTMPS.CERT_FILE) - ]) - const serverOptions = { key, cert } - - this.rtmpsServer = createServerTLS(serverOptions, socket => { - const session = new NodeRtmpSession(config, socket) - - session.inputOriginLocalUrl = 'rtmps://127.0.0.1:' + CONFIG.LIVE.RTMPS.PORT - session.inputOriginPublicUrl = WEBSERVER.RTMPS_URL - session.run() - }) - - this.rtmpsServer.on('error', err => { - logger.error('Cannot run RTMPS server.', { err, ...lTags() }) - }) - - this.rtmpsServer.listen(CONFIG.LIVE.RTMPS.PORT, CONFIG.LIVE.RTMPS.HOSTNAME) - } - } - - stop () { - this.running = false - - if (this.rtmpServer) { - logger.info('Stopping RTMP server.', lTags()) - - this.rtmpServer.close() - this.rtmpServer = undefined - } - - if (this.rtmpsServer) { - logger.info('Stopping RTMPS server.', lTags()) - - this.rtmpsServer.close() - this.rtmpsServer = undefined - } - - // Sessions is an object - this.getContext().sessions.forEach((session: any) => { - if (session instanceof NodeRtmpSession) { - session.stop() - } - }) - } - - isRunning () { - return !!this.rtmpServer - } - - hasSession (sessionId: string) { - return this.getContext().sessions.has(sessionId) - } - - stopSessionOf (videoUUID: string, error: LiveVideoError | null) { - const sessionId = this.videoSessions.get(videoUUID) - if (!sessionId) { - logger.debug('No live session to stop for video %s', videoUUID, lTags(sessionId, videoUUID)) - return - } - - logger.info('Stopping live session of video %s', videoUUID, { error, ...lTags(sessionId, videoUUID) }) - - this.saveEndingSession(videoUUID, error) - .catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId, videoUUID) })) - - this.videoSessions.delete(videoUUID) - this.abortSession(sessionId) - } - - private getContext () { - return context - } - - private abortSession (sessionId: string) { - const session = this.getContext().sessions.get(sessionId) - if (session) { - session.stop() - this.getContext().sessions.delete(sessionId) - } - - const muxingSession = this.muxingSessions.get(sessionId) - if (muxingSession) { - // Muxing session will fire and event so we correctly cleanup the session - muxingSession.abort() - - this.muxingSessions.delete(sessionId) - } - } - - private async handleSession (options: { - sessionId: string - inputLocalUrl: string - inputPublicUrl: string - streamKey: string - }) { - const { inputLocalUrl, inputPublicUrl, sessionId, streamKey } = options - - const videoLive = await VideoLiveModel.loadByStreamKey(streamKey) - if (!videoLive) { - logger.warn('Unknown live video with stream key %s.', streamKey, lTags(sessionId)) - return this.abortSession(sessionId) - } - - const video = videoLive.Video - if (video.isBlacklisted()) { - logger.warn('Video is blacklisted. Refusing stream %s.', streamKey, lTags(sessionId, video.uuid)) - return this.abortSession(sessionId) - } - - if (this.videoSessions.has(video.uuid)) { - logger.warn('Video %s has already a live session. Refusing stream %s.', video.uuid, streamKey, lTags(sessionId, video.uuid)) - return this.abortSession(sessionId) - } - - // Cleanup old potential live (could happen with a permanent live) - const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id) - if (oldStreamingPlaylist) { - if (!videoLive.permanentLive) throw new Error('Found previous session in a non permanent live: ' + video.uuid) - - await cleanupAndDestroyPermanentLive(video, oldStreamingPlaylist) - } - - this.videoSessions.set(video.uuid, sessionId) - - const now = Date.now() - const probe = await ffprobePromise(inputLocalUrl) - - const [ { resolution, ratio }, fps, bitrate, hasAudio ] = await Promise.all([ - getVideoStreamDimensionsInfo(inputLocalUrl, probe), - getVideoStreamFPS(inputLocalUrl, probe), - getVideoStreamBitrate(inputLocalUrl, probe), - hasAudioStream(inputLocalUrl, probe) - ]) - - logger.info( - '%s probing took %d ms (bitrate: %d, fps: %d, resolution: %d)', - inputLocalUrl, Date.now() - now, bitrate, fps, resolution, lTags(sessionId, video.uuid) - ) - - const allResolutions = await Hooks.wrapObject( - this.buildAllResolutionsToTranscode(resolution, hasAudio), - 'filter:transcoding.auto.resolutions-to-transcode.result', - { video } - ) - - logger.info( - 'Handling live video of original resolution %d.', resolution, - { allResolutions, ...lTags(sessionId, video.uuid) } - ) - - return this.runMuxingSession({ - sessionId, - videoLive, - - inputLocalUrl, - inputPublicUrl, - fps, - bitrate, - ratio, - allResolutions, - hasAudio - }) - } - - private async runMuxingSession (options: { - sessionId: string - videoLive: MVideoLiveVideoWithSetting - - inputLocalUrl: string - inputPublicUrl: string - - fps: number - bitrate: number - ratio: number - allResolutions: number[] - hasAudio: boolean - }) { - const { sessionId, videoLive } = options - const videoUUID = videoLive.Video.uuid - const localLTags = lTags(sessionId, videoUUID) - - const liveSession = await this.saveStartingSession(videoLive) - - const user = await UserModel.loadByLiveId(videoLive.id) - LiveQuotaStore.Instance.addNewLive(user.id, sessionId) - - const muxingSession = new MuxingSession({ - context: this.getContext(), - sessionId, - videoLive, - user, - - ...pick(options, [ 'inputLocalUrl', 'inputPublicUrl', 'bitrate', 'ratio', 'fps', 'allResolutions', 'hasAudio' ]) - }) - - muxingSession.on('live-ready', () => this.publishAndFederateLive(videoLive, localLTags)) - - muxingSession.on('bad-socket-health', ({ videoUUID }) => { - logger.error( - 'Too much data in client socket stream (ffmpeg is too slow to transcode the video).' + - ' Stopping session of video %s.', videoUUID, - localLTags - ) - - this.stopSessionOf(videoUUID, LiveVideoError.BAD_SOCKET_HEALTH) - }) - - muxingSession.on('duration-exceeded', ({ videoUUID }) => { - logger.info('Stopping session of %s: max duration exceeded.', videoUUID, localLTags) - - this.stopSessionOf(videoUUID, LiveVideoError.DURATION_EXCEEDED) - }) - - muxingSession.on('quota-exceeded', ({ videoUUID }) => { - logger.info('Stopping session of %s: user quota exceeded.', videoUUID, localLTags) - - this.stopSessionOf(videoUUID, LiveVideoError.QUOTA_EXCEEDED) - }) - - muxingSession.on('transcoding-error', ({ videoUUID }) => { - this.stopSessionOf(videoUUID, LiveVideoError.FFMPEG_ERROR) - }) - - muxingSession.on('transcoding-end', ({ videoUUID }) => { - this.onMuxingFFmpegEnd(videoUUID, sessionId) - }) - - muxingSession.on('after-cleanup', ({ videoUUID }) => { - this.muxingSessions.delete(sessionId) - - LiveQuotaStore.Instance.removeLive(user.id, sessionId) - - muxingSession.destroy() - - return this.onAfterMuxingCleanup({ videoUUID, liveSession }) - .catch(err => logger.error('Error in end transmuxing.', { err, ...localLTags })) - }) - - this.muxingSessions.set(sessionId, muxingSession) - - muxingSession.runMuxing() - .catch(err => { - logger.error('Cannot run muxing.', { err, ...localLTags }) - this.abortSession(sessionId) - }) - } - - private async publishAndFederateLive (live: MVideoLiveVideo, localLTags: { tags: string[] }) { - const videoId = live.videoId - - try { - const video = await VideoModel.loadFull(videoId) - - logger.info('Will publish and federate live %s.', video.url, localLTags) - - video.state = VideoState.PUBLISHED - video.publishedAt = new Date() - await video.save() - - live.Video = video - - await wait(getLiveSegmentTime(live.latencyMode) * 1000 * VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION) - - try { - await federateVideoIfNeeded(video, false) - } catch (err) { - logger.error('Cannot federate live video %s.', video.url, { err, ...localLTags }) - } - - PeerTubeSocket.Instance.sendVideoLiveNewState(video) - - Hooks.runAction('action:live.video.state.updated', { video }) - } catch (err) { - logger.error('Cannot save/federate live video %d.', videoId, { err, ...localLTags }) - } - } - - private onMuxingFFmpegEnd (videoUUID: string, sessionId: string) { - // Session already cleaned up - if (!this.videoSessions.has(videoUUID)) return - - this.videoSessions.delete(videoUUID) - - this.saveEndingSession(videoUUID, null) - .catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId) })) - } - - private async onAfterMuxingCleanup (options: { - videoUUID: string - liveSession?: MVideoLiveSession - cleanupNow?: boolean // Default false - }) { - const { videoUUID, liveSession: liveSessionArg, cleanupNow = false } = options - - logger.debug('Live of video %s has been cleaned up. Moving to its next state.', videoUUID, lTags(videoUUID)) - - try { - const fullVideo = await VideoModel.loadFull(videoUUID) - if (!fullVideo) return - - const live = await VideoLiveModel.loadByVideoId(fullVideo.id) - - const liveSession = liveSessionArg ?? await VideoLiveSessionModel.findLatestSessionOf(fullVideo.id) - - // On server restart during a live - if (!liveSession.endDate) { - liveSession.endDate = new Date() - await liveSession.save() - } - - JobQueue.Instance.createJobAsync({ - type: 'video-live-ending', - payload: { - videoId: fullVideo.id, - - replayDirectory: live.saveReplay - ? await this.findReplayDirectory(fullVideo) - : undefined, - - liveSessionId: liveSession.id, - streamingPlaylistId: fullVideo.getHLSPlaylist()?.id, - - publishedAt: fullVideo.publishedAt.toISOString() - }, - - delay: cleanupNow - ? 0 - : VIDEO_LIVE.CLEANUP_DELAY - }) - - fullVideo.state = live.permanentLive - ? VideoState.WAITING_FOR_LIVE - : VideoState.LIVE_ENDED - - await fullVideo.save() - - PeerTubeSocket.Instance.sendVideoLiveNewState(fullVideo) - - await federateVideoIfNeeded(fullVideo, false) - - Hooks.runAction('action:live.video.state.updated', { video: fullVideo }) - } catch (err) { - logger.error('Cannot save/federate new video state of live streaming of video %s.', videoUUID, { err, ...lTags(videoUUID) }) - } - } - - private async handleBrokenLives () { - await RunnerJobModel.cancelAllJobs({ type: 'live-rtmp-hls-transcoding' }) - - const videoUUIDs = await VideoModel.listPublishedLiveUUIDs() - - for (const uuid of videoUUIDs) { - await this.onAfterMuxingCleanup({ videoUUID: uuid, cleanupNow: true }) - } - } - - private async findReplayDirectory (video: MVideo) { - const directory = getLiveReplayBaseDirectory(video) - const files = await readdir(directory) - - if (files.length === 0) return undefined - - return join(directory, files.sort().reverse()[0]) - } - - private buildAllResolutionsToTranscode (originResolution: number, hasAudio: boolean) { - const includeInput = CONFIG.LIVE.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION - - const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED - ? computeResolutionsToTranscode({ input: originResolution, type: 'live', includeInput, strictLower: false, hasAudio }) - : [] - - if (resolutionsEnabled.length === 0) { - return [ originResolution ] - } - - return resolutionsEnabled - } - - private async saveStartingSession (videoLive: MVideoLiveVideoWithSetting) { - const replaySettings = videoLive.saveReplay - ? new VideoLiveReplaySettingModel({ - privacy: videoLive.ReplaySetting.privacy - }) - : null - - return sequelizeTypescript.transaction(async t => { - if (videoLive.saveReplay) { - await replaySettings.save({ transaction: t }) - } - - return VideoLiveSessionModel.create({ - startDate: new Date(), - liveVideoId: videoLive.videoId, - saveReplay: videoLive.saveReplay, - replaySettingId: videoLive.saveReplay ? replaySettings.id : null, - endingProcessed: false - }, { transaction: t }) - }) - } - - private async saveEndingSession (videoUUID: string, error: LiveVideoError | null) { - const liveSession = await VideoLiveSessionModel.findCurrentSessionOf(videoUUID) - if (!liveSession) return - - liveSession.endDate = new Date() - liveSession.error = error - - return liveSession.save() - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -// --------------------------------------------------------------------------- - -export { - LiveManager -} diff --git a/server/lib/live/live-segment-sha-store.ts b/server/lib/live/live-segment-sha-store.ts deleted file mode 100644 index 8253c0274..000000000 --- a/server/lib/live/live-segment-sha-store.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { rename, writeJson } from 'fs-extra' -import PQueue from 'p-queue' -import { basename } from 'path' -import { mapToJSON } from '@server/helpers/core-utils' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { MStreamingPlaylistVideo } from '@server/types/models' -import { buildSha256Segment } from '../hls' -import { storeHLSFileFromPath } from '../object-storage' - -const lTags = loggerTagsFactory('live') - -class LiveSegmentShaStore { - - private readonly segmentsSha256 = new Map() - - private readonly videoUUID: string - - private readonly sha256Path: string - private readonly sha256PathTMP: string - - private readonly streamingPlaylist: MStreamingPlaylistVideo - private readonly sendToObjectStorage: boolean - private readonly writeQueue = new PQueue({ concurrency: 1 }) - - constructor (options: { - videoUUID: string - sha256Path: string - streamingPlaylist: MStreamingPlaylistVideo - sendToObjectStorage: boolean - }) { - this.videoUUID = options.videoUUID - - this.sha256Path = options.sha256Path - this.sha256PathTMP = options.sha256Path + '.tmp' - - this.streamingPlaylist = options.streamingPlaylist - this.sendToObjectStorage = options.sendToObjectStorage - } - - async addSegmentSha (segmentPath: string) { - logger.debug('Adding live sha segment %s.', segmentPath, lTags(this.videoUUID)) - - const shaResult = await buildSha256Segment(segmentPath) - - const segmentName = basename(segmentPath) - this.segmentsSha256.set(segmentName, shaResult) - - try { - await this.writeToDisk() - } catch (err) { - logger.error('Cannot write sha segments to disk.', { err }) - } - } - - async removeSegmentSha (segmentPath: string) { - const segmentName = basename(segmentPath) - - logger.debug('Removing live sha segment %s.', segmentPath, lTags(this.videoUUID)) - - if (!this.segmentsSha256.has(segmentName)) { - logger.warn( - 'Unknown segment in live segment hash store for video %s and segment %s.', - this.videoUUID, segmentPath, lTags(this.videoUUID) - ) - return - } - - this.segmentsSha256.delete(segmentName) - - await this.writeToDisk() - } - - private writeToDisk () { - return this.writeQueue.add(async () => { - logger.debug(`Writing segment sha JSON ${this.sha256Path} of ${this.videoUUID} on disk.`, lTags(this.videoUUID)) - - // Atomic write: use rename instead of move that is not atomic - await writeJson(this.sha256PathTMP, mapToJSON(this.segmentsSha256)) - await rename(this.sha256PathTMP, this.sha256Path) - - if (this.sendToObjectStorage) { - const url = await storeHLSFileFromPath(this.streamingPlaylist, this.sha256Path) - - if (this.streamingPlaylist.segmentsSha256Url !== url) { - this.streamingPlaylist.segmentsSha256Url = url - await this.streamingPlaylist.save() - } - } - }) - } -} - -export { - LiveSegmentShaStore -} diff --git a/server/lib/live/live-utils.ts b/server/lib/live/live-utils.ts deleted file mode 100644 index 3fb3ce1ce..000000000 --- a/server/lib/live/live-utils.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { pathExists, readdir, remove } from 'fs-extra' -import { basename, join } from 'path' -import { logger } from '@server/helpers/logger' -import { VIDEO_LIVE } from '@server/initializers/constants' -import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models' -import { LiveVideoLatencyMode, VideoStorage } from '@shared/models' -import { listHLSFileKeysOf, removeHLSFileObjectStorageByFullKey, removeHLSObjectStorage } from '../object-storage' -import { getLiveDirectory } from '../paths' - -function buildConcatenatedName (segmentOrPlaylistPath: string) { - const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) - - return 'concat-' + num[1] + '.ts' -} - -async function cleanupAndDestroyPermanentLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { - await cleanupTMPLiveFiles(video, streamingPlaylist) - - await streamingPlaylist.destroy() -} - -async function cleanupUnsavedNormalLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { - const hlsDirectory = getLiveDirectory(video) - - // We uploaded files to object storage too, remove them - if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { - await removeHLSObjectStorage(streamingPlaylist.withVideo(video)) - } - - await remove(hlsDirectory) - - await streamingPlaylist.destroy() -} - -async function cleanupTMPLiveFiles (video: MVideo, streamingPlaylist: MStreamingPlaylist) { - await cleanupTMPLiveFilesFromObjectStorage(streamingPlaylist.withVideo(video)) - - await cleanupTMPLiveFilesFromFilesystem(video) -} - -function getLiveSegmentTime (latencyMode: LiveVideoLatencyMode) { - if (latencyMode === LiveVideoLatencyMode.SMALL_LATENCY) { - return VIDEO_LIVE.SEGMENT_TIME_SECONDS.SMALL_LATENCY - } - - return VIDEO_LIVE.SEGMENT_TIME_SECONDS.DEFAULT_LATENCY -} - -export { - cleanupAndDestroyPermanentLive, - cleanupUnsavedNormalLive, - cleanupTMPLiveFiles, - getLiveSegmentTime, - buildConcatenatedName -} - -// --------------------------------------------------------------------------- - -function isTMPLiveFile (name: string) { - return name.endsWith('.ts') || - name.endsWith('.m3u8') || - name.endsWith('.json') || - name.endsWith('.mpd') || - name.endsWith('.m4s') || - name.endsWith('.tmp') -} - -async function cleanupTMPLiveFilesFromFilesystem (video: MVideo) { - const hlsDirectory = getLiveDirectory(video) - - if (!await pathExists(hlsDirectory)) return - - logger.info('Cleanup TMP live files from filesystem of %s.', hlsDirectory) - - const files = await readdir(hlsDirectory) - - for (const filename of files) { - if (isTMPLiveFile(filename)) { - const p = join(hlsDirectory, filename) - - remove(p) - .catch(err => logger.error('Cannot remove %s.', p, { err })) - } - } -} - -async function cleanupTMPLiveFilesFromObjectStorage (streamingPlaylist: MStreamingPlaylistVideo) { - if (streamingPlaylist.storage !== VideoStorage.OBJECT_STORAGE) return - - logger.info('Cleanup TMP live files from object storage for %s.', streamingPlaylist.Video.uuid) - - const keys = await listHLSFileKeysOf(streamingPlaylist) - - for (const key of keys) { - if (isTMPLiveFile(key)) { - await removeHLSFileObjectStorageByFullKey(key) - } - } -} diff --git a/server/lib/live/shared/index.ts b/server/lib/live/shared/index.ts deleted file mode 100644 index c4d1b59ec..000000000 --- a/server/lib/live/shared/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './muxing-session' diff --git a/server/lib/live/shared/muxing-session.ts b/server/lib/live/shared/muxing-session.ts deleted file mode 100644 index 02691b651..000000000 --- a/server/lib/live/shared/muxing-session.ts +++ /dev/null @@ -1,518 +0,0 @@ -import { mapSeries } from 'bluebird' -import { FSWatcher, watch } from 'chokidar' -import { EventEmitter } from 'events' -import { appendFile, ensureDir, readFile, stat } from 'fs-extra' -import PQueue from 'p-queue' -import { basename, join } from 'path' -import { computeOutputFPS } from '@server/helpers/ffmpeg' -import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE } from '@server/initializers/constants' -import { removeHLSFileObjectStorageByPath, storeHLSFileFromContent, storeHLSFileFromPath } from '@server/lib/object-storage' -import { VideoFileModel } from '@server/models/video/video-file' -import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models' -import { VideoStorage, VideoStreamingPlaylistType } from '@shared/models' -import { - generateHLSMasterPlaylistFilename, - generateHlsSha256SegmentsFilename, - getLiveDirectory, - getLiveReplayBaseDirectory -} from '../../paths' -import { isAbleToUploadVideo } from '../../user' -import { LiveQuotaStore } from '../live-quota-store' -import { LiveSegmentShaStore } from '../live-segment-sha-store' -import { buildConcatenatedName, getLiveSegmentTime } from '../live-utils' -import { AbstractTranscodingWrapper, FFmpegTranscodingWrapper, RemoteTranscodingWrapper } from './transcoding-wrapper' - -import memoizee = require('memoizee') -interface MuxingSessionEvents { - 'live-ready': (options: { videoUUID: string }) => void - - 'bad-socket-health': (options: { videoUUID: string }) => void - 'duration-exceeded': (options: { videoUUID: string }) => void - 'quota-exceeded': (options: { videoUUID: string }) => void - - 'transcoding-end': (options: { videoUUID: string }) => void - 'transcoding-error': (options: { videoUUID: string }) => void - - 'after-cleanup': (options: { videoUUID: string }) => void -} - -declare interface MuxingSession { - on( - event: U, listener: MuxingSessionEvents[U] - ): this - - emit( - event: U, ...args: Parameters - ): boolean -} - -class MuxingSession extends EventEmitter { - - private transcodingWrapper: AbstractTranscodingWrapper - - private readonly context: any - private readonly user: MUserId - private readonly sessionId: string - private readonly videoLive: MVideoLiveVideo - - private readonly inputLocalUrl: string - private readonly inputPublicUrl: string - - private readonly fps: number - private readonly allResolutions: number[] - - private readonly bitrate: number - private readonly ratio: number - - private readonly hasAudio: boolean - - private readonly videoUUID: string - private readonly saveReplay: boolean - - private readonly outDirectory: string - private readonly replayDirectory: string - - private readonly lTags: LoggerTagsFn - - // Path -> Queue - private readonly objectStorageSendQueues = new Map() - - private segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {} - - private streamingPlaylist: MStreamingPlaylistVideo - private liveSegmentShaStore: LiveSegmentShaStore - - private filesWatcher: FSWatcher - - private masterPlaylistCreated = false - private liveReady = false - - private aborted = false - - private readonly isAbleToUploadVideoWithCache = memoizee((userId: number) => { - return isAbleToUploadVideo(userId, 1000) - }, { maxAge: MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD }) - - private readonly hasClientSocketInBadHealthWithCache = memoizee((sessionId: string) => { - return this.hasClientSocketInBadHealth(sessionId) - }, { maxAge: MEMOIZE_TTL.LIVE_CHECK_SOCKET_HEALTH }) - - constructor (options: { - context: any - user: MUserId - sessionId: string - videoLive: MVideoLiveVideo - - inputLocalUrl: string - inputPublicUrl: string - - fps: number - bitrate: number - ratio: number - allResolutions: number[] - hasAudio: boolean - }) { - super() - - this.context = options.context - this.user = options.user - this.sessionId = options.sessionId - this.videoLive = options.videoLive - - this.inputLocalUrl = options.inputLocalUrl - this.inputPublicUrl = options.inputPublicUrl - - this.fps = options.fps - - this.bitrate = options.bitrate - this.ratio = options.ratio - - this.hasAudio = options.hasAudio - - this.allResolutions = options.allResolutions - - this.videoUUID = this.videoLive.Video.uuid - - this.saveReplay = this.videoLive.saveReplay - - this.outDirectory = getLiveDirectory(this.videoLive.Video) - this.replayDirectory = join(getLiveReplayBaseDirectory(this.videoLive.Video), new Date().toISOString()) - - this.lTags = loggerTagsFactory('live', this.sessionId, this.videoUUID) - } - - async runMuxing () { - this.streamingPlaylist = await this.createLivePlaylist() - - this.createLiveShaStore() - this.createFiles() - - await this.prepareDirectories() - - this.transcodingWrapper = this.buildTranscodingWrapper() - - this.transcodingWrapper.on('end', () => this.onTranscodedEnded()) - this.transcodingWrapper.on('error', () => this.onTranscodingError()) - - await this.transcodingWrapper.run() - - this.filesWatcher = watch(this.outDirectory, { depth: 0 }) - - this.watchMasterFile() - this.watchTSFiles() - } - - abort () { - if (!this.transcodingWrapper) return - - this.aborted = true - this.transcodingWrapper.abort() - } - - destroy () { - this.removeAllListeners() - this.isAbleToUploadVideoWithCache.clear() - this.hasClientSocketInBadHealthWithCache.clear() - } - - private watchMasterFile () { - this.filesWatcher.on('add', async path => { - if (path !== join(this.outDirectory, this.streamingPlaylist.playlistFilename)) return - if (this.masterPlaylistCreated === true) return - - try { - if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { - const masterContent = await readFile(path, 'utf-8') - logger.debug('Uploading live master playlist on object storage for %s', this.videoUUID, { masterContent, ...this.lTags() }) - - const url = await storeHLSFileFromContent(this.streamingPlaylist, this.streamingPlaylist.playlistFilename, masterContent) - - this.streamingPlaylist.playlistUrl = url - } - - this.streamingPlaylist.assignP2PMediaLoaderInfoHashes(this.videoLive.Video, this.allResolutions) - - await this.streamingPlaylist.save() - } catch (err) { - logger.error('Cannot update streaming playlist.', { err, ...this.lTags() }) - } - - this.masterPlaylistCreated = true - - logger.info('Master playlist file for %s has been created', this.videoUUID, this.lTags()) - }) - } - - private watchTSFiles () { - const startStreamDateTime = new Date().getTime() - - const addHandler = async (segmentPath: string) => { - if (segmentPath.endsWith('.ts') !== true) return - - logger.debug('Live add handler of TS file %s.', segmentPath, this.lTags()) - - const playlistId = this.getPlaylistIdFromTS(segmentPath) - - const segmentsToProcess = this.segmentsToProcessPerPlaylist[playlistId] || [] - this.processSegments(segmentsToProcess) - - this.segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ] - - if (this.hasClientSocketInBadHealthWithCache(this.sessionId)) { - this.emit('bad-socket-health', { videoUUID: this.videoUUID }) - return - } - - // Duration constraint check - if (this.isDurationConstraintValid(startStreamDateTime) !== true) { - this.emit('duration-exceeded', { videoUUID: this.videoUUID }) - return - } - - // Check user quota if the user enabled replay saving - if (await this.isQuotaExceeded(segmentPath) === true) { - this.emit('quota-exceeded', { videoUUID: this.videoUUID }) - } - } - - const deleteHandler = async (segmentPath: string) => { - if (segmentPath.endsWith('.ts') !== true) return - - logger.debug('Live delete handler of TS file %s.', segmentPath, this.lTags()) - - try { - await this.liveSegmentShaStore.removeSegmentSha(segmentPath) - } catch (err) { - logger.warn('Cannot remove segment sha %s from sha store', segmentPath, { err, ...this.lTags() }) - } - - if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { - try { - await removeHLSFileObjectStorageByPath(this.streamingPlaylist, segmentPath) - } catch (err) { - logger.error('Cannot remove segment %s from object storage', segmentPath, { err, ...this.lTags() }) - } - } - } - - this.filesWatcher.on('add', p => addHandler(p)) - this.filesWatcher.on('unlink', p => deleteHandler(p)) - } - - private async isQuotaExceeded (segmentPath: string) { - if (this.saveReplay !== true) return false - if (this.aborted) return false - - try { - const segmentStat = await stat(segmentPath) - - LiveQuotaStore.Instance.addQuotaTo(this.user.id, this.sessionId, segmentStat.size) - - const canUpload = await this.isAbleToUploadVideoWithCache(this.user.id) - - return canUpload !== true - } catch (err) { - logger.error('Cannot stat %s or check quota of %d.', segmentPath, this.user.id, { err, ...this.lTags() }) - } - } - - private createFiles () { - for (let i = 0; i < this.allResolutions.length; i++) { - const resolution = this.allResolutions[i] - - const file = new VideoFileModel({ - resolution, - size: -1, - extname: '.ts', - infoHash: null, - fps: this.fps, - storage: this.streamingPlaylist.storage, - videoStreamingPlaylistId: this.streamingPlaylist.id - }) - - VideoFileModel.customUpsert(file, 'streaming-playlist', null) - .catch(err => logger.error('Cannot create file for live streaming.', { err, ...this.lTags() })) - } - } - - private async prepareDirectories () { - await ensureDir(this.outDirectory) - - if (this.videoLive.saveReplay === true) { - await ensureDir(this.replayDirectory) - } - } - - private isDurationConstraintValid (streamingStartTime: number) { - const maxDuration = CONFIG.LIVE.MAX_DURATION - // No limit - if (maxDuration < 0) return true - - const now = new Date().getTime() - const max = streamingStartTime + maxDuration - - return now <= max - } - - private processSegments (segmentPaths: string[]) { - mapSeries(segmentPaths, previousSegment => this.processSegment(previousSegment)) - .catch(err => { - if (this.aborted) return - - logger.error('Cannot process segments', { err, ...this.lTags() }) - }) - } - - private async processSegment (segmentPath: string) { - // Add sha hash of previous segments, because ffmpeg should have finished generating them - await this.liveSegmentShaStore.addSegmentSha(segmentPath) - - if (this.saveReplay) { - await this.addSegmentToReplay(segmentPath) - } - - if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { - try { - await storeHLSFileFromPath(this.streamingPlaylist, segmentPath) - - await this.processM3U8ToObjectStorage(segmentPath) - } catch (err) { - logger.error('Cannot store TS segment %s in object storage', segmentPath, { err, ...this.lTags() }) - } - } - - // Master playlist and segment JSON file are created, live is ready - if (this.masterPlaylistCreated && !this.liveReady) { - this.liveReady = true - - this.emit('live-ready', { videoUUID: this.videoUUID }) - } - } - - private async processM3U8ToObjectStorage (segmentPath: string) { - const m3u8Path = join(this.outDirectory, this.getPlaylistNameFromTS(segmentPath)) - - logger.debug('Process M3U8 file %s.', m3u8Path, this.lTags()) - - const segmentName = basename(segmentPath) - - const playlistContent = await readFile(m3u8Path, 'utf-8') - // Remove new chunk references, that will be processed later - const filteredPlaylistContent = playlistContent.substring(0, playlistContent.lastIndexOf(segmentName) + segmentName.length) + '\n' - - try { - if (!this.objectStorageSendQueues.has(m3u8Path)) { - this.objectStorageSendQueues.set(m3u8Path, new PQueue({ concurrency: 1 })) - } - - const queue = this.objectStorageSendQueues.get(m3u8Path) - await queue.add(() => storeHLSFileFromContent(this.streamingPlaylist, m3u8Path, filteredPlaylistContent)) - } catch (err) { - logger.error('Cannot store in object storage m3u8 file %s', m3u8Path, { err, ...this.lTags() }) - } - } - - private onTranscodingError () { - this.emit('transcoding-error', ({ videoUUID: this.videoUUID })) - } - - private onTranscodedEnded () { - this.emit('transcoding-end', ({ videoUUID: this.videoUUID })) - - logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.inputLocalUrl, this.lTags()) - - setTimeout(() => { - // Wait latest segments generation, and close watchers - - const promise = this.filesWatcher?.close() || Promise.resolve() - promise - .then(() => { - // Process remaining segments hash - for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) { - this.processSegments(this.segmentsToProcessPerPlaylist[key]) - } - }) - .catch(err => { - logger.error( - 'Cannot close watchers of %s or process remaining hash segments.', this.outDirectory, - { err, ...this.lTags() } - ) - }) - - this.emit('after-cleanup', { videoUUID: this.videoUUID }) - }, 1000) - } - - private hasClientSocketInBadHealth (sessionId: string) { - const rtmpSession = this.context.sessions.get(sessionId) - - if (!rtmpSession) { - logger.warn('Cannot get session %s to check players socket health.', sessionId, this.lTags()) - return - } - - for (const playerSessionId of rtmpSession.players) { - const playerSession = this.context.sessions.get(playerSessionId) - - if (!playerSession) { - logger.error('Cannot get player session %s to check socket health.', playerSession, this.lTags()) - continue - } - - if (playerSession.socket.writableLength > VIDEO_LIVE.MAX_SOCKET_WAITING_DATA) { - return true - } - } - - return false - } - - private async addSegmentToReplay (segmentPath: string) { - const segmentName = basename(segmentPath) - const dest = join(this.replayDirectory, buildConcatenatedName(segmentName)) - - try { - const data = await readFile(segmentPath) - - await appendFile(dest, data) - } catch (err) { - logger.error('Cannot copy segment %s to replay directory.', segmentPath, { err, ...this.lTags() }) - } - } - - private async createLivePlaylist (): Promise { - const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(this.videoLive.Video) - - playlist.playlistFilename = generateHLSMasterPlaylistFilename(true) - playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(true) - - playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION - playlist.type = VideoStreamingPlaylistType.HLS - - playlist.storage = CONFIG.OBJECT_STORAGE.ENABLED - ? VideoStorage.OBJECT_STORAGE - : VideoStorage.FILE_SYSTEM - - return playlist.save() - } - - private createLiveShaStore () { - this.liveSegmentShaStore = new LiveSegmentShaStore({ - videoUUID: this.videoLive.Video.uuid, - sha256Path: join(this.outDirectory, this.streamingPlaylist.segmentsSha256Filename), - streamingPlaylist: this.streamingPlaylist, - sendToObjectStorage: CONFIG.OBJECT_STORAGE.ENABLED - }) - } - - private buildTranscodingWrapper () { - const options = { - streamingPlaylist: this.streamingPlaylist, - videoLive: this.videoLive, - - lTags: this.lTags, - - sessionId: this.sessionId, - inputLocalUrl: this.inputLocalUrl, - inputPublicUrl: this.inputPublicUrl, - - toTranscode: this.allResolutions.map(resolution => ({ - resolution, - fps: computeOutputFPS({ inputFPS: this.fps, resolution }) - })), - - fps: this.fps, - bitrate: this.bitrate, - ratio: this.ratio, - hasAudio: this.hasAudio, - - segmentListSize: VIDEO_LIVE.SEGMENTS_LIST_SIZE, - segmentDuration: getLiveSegmentTime(this.videoLive.latencyMode), - - outDirectory: this.outDirectory - } - - return CONFIG.LIVE.TRANSCODING.ENABLED && CONFIG.LIVE.TRANSCODING.REMOTE_RUNNERS.ENABLED - ? new RemoteTranscodingWrapper(options) - : new FFmpegTranscodingWrapper(options) - } - - private getPlaylistIdFromTS (segmentPath: string) { - const playlistIdMatcher = /^([\d+])-/ - - return basename(segmentPath).match(playlistIdMatcher)[1] - } - - private getPlaylistNameFromTS (segmentPath: string) { - return `${this.getPlaylistIdFromTS(segmentPath)}.m3u8` - } -} - -// --------------------------------------------------------------------------- - -export { - MuxingSession -} diff --git a/server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts b/server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts deleted file mode 100644 index 95168745d..000000000 --- a/server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts +++ /dev/null @@ -1,110 +0,0 @@ -import EventEmitter from 'events' -import { LoggerTagsFn } from '@server/helpers/logger' -import { MStreamingPlaylistVideo, MVideoLiveVideo } from '@server/types/models' -import { LiveVideoError } from '@shared/models' - -interface TranscodingWrapperEvents { - 'end': () => void - - 'error': (options: { err: Error }) => void -} - -declare interface AbstractTranscodingWrapper { - on( - event: U, listener: TranscodingWrapperEvents[U] - ): this - - emit( - event: U, ...args: Parameters - ): boolean -} - -interface AbstractTranscodingWrapperOptions { - streamingPlaylist: MStreamingPlaylistVideo - videoLive: MVideoLiveVideo - - lTags: LoggerTagsFn - - sessionId: string - inputLocalUrl: string - inputPublicUrl: string - - fps: number - toTranscode: { - resolution: number - fps: number - }[] - - bitrate: number - ratio: number - hasAudio: boolean - - segmentListSize: number - segmentDuration: number - - outDirectory: string -} - -abstract class AbstractTranscodingWrapper extends EventEmitter { - protected readonly videoLive: MVideoLiveVideo - - protected readonly toTranscode: { - resolution: number - fps: number - }[] - - protected readonly sessionId: string - protected readonly inputLocalUrl: string - protected readonly inputPublicUrl: string - - protected readonly fps: number - protected readonly bitrate: number - protected readonly ratio: number - protected readonly hasAudio: boolean - - protected readonly segmentListSize: number - protected readonly segmentDuration: number - - protected readonly videoUUID: string - - protected readonly outDirectory: string - - protected readonly lTags: LoggerTagsFn - - protected readonly streamingPlaylist: MStreamingPlaylistVideo - - constructor (options: AbstractTranscodingWrapperOptions) { - super() - - this.lTags = options.lTags - - this.videoLive = options.videoLive - this.videoUUID = options.videoLive.Video.uuid - this.streamingPlaylist = options.streamingPlaylist - - this.sessionId = options.sessionId - this.inputLocalUrl = options.inputLocalUrl - this.inputPublicUrl = options.inputPublicUrl - - this.fps = options.fps - this.toTranscode = options.toTranscode - - this.bitrate = options.bitrate - this.ratio = options.ratio - this.hasAudio = options.hasAudio - - this.segmentListSize = options.segmentListSize - this.segmentDuration = options.segmentDuration - - this.outDirectory = options.outDirectory - } - - abstract run (): Promise - - abstract abort (error?: LiveVideoError): void -} - -export { - AbstractTranscodingWrapper, - AbstractTranscodingWrapperOptions -} diff --git a/server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts b/server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts deleted file mode 100644 index c6ee8ebf1..000000000 --- a/server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts +++ /dev/null @@ -1,107 +0,0 @@ -import { FfmpegCommand } from 'fluent-ffmpeg' -import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { VIDEO_LIVE } from '@server/initializers/constants' -import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles' -import { FFmpegLive } from '@shared/ffmpeg' -import { getLiveSegmentTime } from '../../live-utils' -import { AbstractTranscodingWrapper } from './abstract-transcoding-wrapper' - -export class FFmpegTranscodingWrapper extends AbstractTranscodingWrapper { - private ffmpegCommand: FfmpegCommand - - private aborted = false - private errored = false - private ended = false - - async run () { - this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED - ? await this.buildFFmpegLive().getLiveTranscodingCommand({ - inputUrl: this.inputLocalUrl, - - outPath: this.outDirectory, - masterPlaylistName: this.streamingPlaylist.playlistFilename, - - segmentListSize: this.segmentListSize, - segmentDuration: this.segmentDuration, - - toTranscode: this.toTranscode, - - bitrate: this.bitrate, - ratio: this.ratio, - - hasAudio: this.hasAudio - }) - : this.buildFFmpegLive().getLiveMuxingCommand({ - inputUrl: this.inputLocalUrl, - outPath: this.outDirectory, - - masterPlaylistName: this.streamingPlaylist.playlistFilename, - - segmentListSize: VIDEO_LIVE.SEGMENTS_LIST_SIZE, - segmentDuration: getLiveSegmentTime(this.videoLive.latencyMode) - }) - - logger.info('Running local live muxing/transcoding for %s.', this.videoUUID, this.lTags()) - - let ffmpegShellCommand: string - this.ffmpegCommand.on('start', cmdline => { - ffmpegShellCommand = cmdline - - logger.debug('Running ffmpeg command for live', { ffmpegShellCommand, ...this.lTags() }) - }) - - this.ffmpegCommand.on('error', (err, stdout, stderr) => { - this.onFFmpegError({ err, stdout, stderr, ffmpegShellCommand }) - }) - - this.ffmpegCommand.on('end', () => { - this.onFFmpegEnded() - }) - - this.ffmpegCommand.run() - } - - abort () { - if (this.ended || this.errored || this.aborted) return - - logger.debug('Killing ffmpeg after live abort of ' + this.videoUUID, this.lTags()) - - this.ffmpegCommand.kill('SIGINT') - - this.aborted = true - this.emit('end') - } - - private onFFmpegError (options: { - err: any - stdout: string - stderr: string - ffmpegShellCommand: string - }) { - const { err, stdout, stderr, ffmpegShellCommand } = options - - // Don't care that we killed the ffmpeg process - if (err?.message?.includes('Exiting normally')) return - if (this.ended || this.errored || this.aborted) return - - logger.error('FFmpeg transcoding error.', { err, stdout, stderr, ffmpegShellCommand, ...this.lTags() }) - - this.errored = true - this.emit('error', { err }) - } - - private onFFmpegEnded () { - if (this.ended || this.errored || this.aborted) return - - logger.debug('Live ffmpeg transcoding ended for ' + this.videoUUID, this.lTags()) - - this.ended = true - this.emit('end') - } - - private buildFFmpegLive () { - return new FFmpegLive(getFFmpegCommandWrapperOptions('live', VideoTranscodingProfilesManager.Instance.getAvailableEncoders())) - } -} diff --git a/server/lib/live/shared/transcoding-wrapper/index.ts b/server/lib/live/shared/transcoding-wrapper/index.ts deleted file mode 100644 index ae28fa1ca..000000000 --- a/server/lib/live/shared/transcoding-wrapper/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './abstract-transcoding-wrapper' -export * from './ffmpeg-transcoding-wrapper' -export * from './remote-transcoding-wrapper' diff --git a/server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts b/server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts deleted file mode 100644 index 2aeeb31fb..000000000 --- a/server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { LiveRTMPHLSTranscodingJobHandler } from '@server/lib/runners' -import { AbstractTranscodingWrapper } from './abstract-transcoding-wrapper' - -export class RemoteTranscodingWrapper extends AbstractTranscodingWrapper { - async run () { - await new LiveRTMPHLSTranscodingJobHandler().create({ - rtmpUrl: this.inputPublicUrl, - sessionId: this.sessionId, - toTranscode: this.toTranscode, - video: this.videoLive.Video, - outputDirectory: this.outDirectory, - playlist: this.streamingPlaylist, - segmentListSize: this.segmentListSize, - segmentDuration: this.segmentDuration - }) - } - - abort () { - this.emit('end') - } -} diff --git a/server/lib/local-actor.ts b/server/lib/local-actor.ts deleted file mode 100644 index 611e6d0af..000000000 --- a/server/lib/local-actor.ts +++ /dev/null @@ -1,102 +0,0 @@ -import { remove } from 'fs-extra' -import { join } from 'path' -import { Transaction } from 'sequelize/types' -import { ActorModel } from '@server/models/actor/actor' -import { getLowercaseExtension } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { ActivityPubActorType, ActorImageType } from '@shared/models' -import { retryTransactionWrapper } from '../helpers/database-utils' -import { CONFIG } from '../initializers/config' -import { ACTOR_IMAGES_SIZE, WEBSERVER } from '../initializers/constants' -import { sequelizeTypescript } from '../initializers/database' -import { MAccountDefault, MActor, MChannelDefault } from '../types/models' -import { deleteActorImages, updateActorImages } from './activitypub/actors' -import { sendUpdateActor } from './activitypub/send' -import { processImageFromWorker } from './worker/parent-process' - -export function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string) { - return new ActorModel({ - type, - url, - preferredUsername, - publicKey: null, - privateKey: null, - followersCount: 0, - followingCount: 0, - inboxUrl: url + '/inbox', - outboxUrl: url + '/outbox', - sharedInboxUrl: WEBSERVER.URL + '/inbox', - followersUrl: url + '/followers', - followingUrl: url + '/following' - }) as MActor -} - -export async function updateLocalActorImageFiles ( - accountOrChannel: MAccountDefault | MChannelDefault, - imagePhysicalFile: Express.Multer.File, - type: ActorImageType -) { - const processImageSize = async (imageSize: { width: number, height: number }) => { - const extension = getLowercaseExtension(imagePhysicalFile.filename) - - const imageName = buildUUID() + extension - const destination = join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, imageName) - await processImageFromWorker({ path: imagePhysicalFile.path, destination, newSize: imageSize, keepOriginal: true }) - - return { - imageName, - imageSize - } - } - - const processedImages = await Promise.all(ACTOR_IMAGES_SIZE[type].map(processImageSize)) - await remove(imagePhysicalFile.path) - - return retryTransactionWrapper(() => sequelizeTypescript.transaction(async t => { - const actorImagesInfo = processedImages.map(({ imageName, imageSize }) => ({ - name: imageName, - fileUrl: null, - height: imageSize.height, - width: imageSize.width, - onDisk: true - })) - - const updatedActor = await updateActorImages(accountOrChannel.Actor, type, actorImagesInfo, t) - await updatedActor.save({ transaction: t }) - - await sendUpdateActor(accountOrChannel, t) - - return type === ActorImageType.AVATAR - ? updatedActor.Avatars - : updatedActor.Banners - })) -} - -export async function deleteLocalActorImageFile (accountOrChannel: MAccountDefault | MChannelDefault, type: ActorImageType) { - return retryTransactionWrapper(() => { - return sequelizeTypescript.transaction(async t => { - const updatedActor = await deleteActorImages(accountOrChannel.Actor, type, t) - await updatedActor.save({ transaction: t }) - - await sendUpdateActor(accountOrChannel, t) - - return updatedActor.Avatars - }) - }) -} - -// --------------------------------------------------------------------------- - -export async function findAvailableLocalActorName (baseActorName: string, transaction?: Transaction) { - let actor = await ActorModel.loadLocalByName(baseActorName, transaction) - if (!actor) return baseActorName - - for (let i = 1; i < 30; i++) { - const name = `${baseActorName}-${i}` - - actor = await ActorModel.loadLocalByName(name, transaction) - if (!actor) return name - } - - throw new Error('Cannot find available actor local name (too much iterations).') -} diff --git a/server/lib/model-loaders/actor.ts b/server/lib/model-loaders/actor.ts deleted file mode 100644 index 1355d8ee2..000000000 --- a/server/lib/model-loaders/actor.ts +++ /dev/null @@ -1,17 +0,0 @@ - -import { ActorModel } from '../../models/actor/actor' -import { MActorAccountChannelId, MActorFull } from '../../types/models' - -type ActorLoadByUrlType = 'all' | 'association-ids' - -function loadActorByUrl (url: string, fetchType: ActorLoadByUrlType): Promise { - if (fetchType === 'all') return ActorModel.loadByUrlAndPopulateAccountAndChannel(url) - - if (fetchType === 'association-ids') return ActorModel.loadByUrl(url) -} - -export { - ActorLoadByUrlType, - - loadActorByUrl -} diff --git a/server/lib/model-loaders/index.ts b/server/lib/model-loaders/index.ts deleted file mode 100644 index 9e5152cb2..000000000 --- a/server/lib/model-loaders/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './actor' -export * from './video' diff --git a/server/lib/model-loaders/video.ts b/server/lib/model-loaders/video.ts deleted file mode 100644 index 91057d405..000000000 --- a/server/lib/model-loaders/video.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { VideoModel } from '@server/models/video/video' -import { - MVideoAccountLightBlacklistAllFiles, - MVideoFormattableDetails, - MVideoFullLight, - MVideoId, - MVideoImmutable, - MVideoThumbnail -} from '@server/types/models' - -type VideoLoadType = 'for-api' | 'all' | 'only-video' | 'id' | 'none' | 'only-immutable-attributes' - -function loadVideo (id: number | string, fetchType: 'for-api', userId?: number): Promise -function loadVideo (id: number | string, fetchType: 'all', userId?: number): Promise -function loadVideo (id: number | string, fetchType: 'only-immutable-attributes'): Promise -function loadVideo (id: number | string, fetchType: 'only-video', userId?: number): Promise -function loadVideo (id: number | string, fetchType: 'id' | 'none', userId?: number): Promise -function loadVideo ( - id: number | string, - fetchType: VideoLoadType, - userId?: number -): Promise -function loadVideo ( - id: number | string, - fetchType: VideoLoadType, - userId?: number -): Promise { - - if (fetchType === 'for-api') return VideoModel.loadForGetAPI({ id, userId }) - - if (fetchType === 'all') return VideoModel.loadFull(id, undefined, userId) - - if (fetchType === 'only-immutable-attributes') return VideoModel.loadImmutableAttributes(id) - - if (fetchType === 'only-video') return VideoModel.load(id) - - if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id) -} - -type VideoLoadByUrlType = 'all' | 'only-video' | 'only-immutable-attributes' - -function loadVideoByUrl (url: string, fetchType: 'all'): Promise -function loadVideoByUrl (url: string, fetchType: 'only-immutable-attributes'): Promise -function loadVideoByUrl (url: string, fetchType: 'only-video'): Promise -function loadVideoByUrl ( - url: string, - fetchType: VideoLoadByUrlType -): Promise -function loadVideoByUrl ( - url: string, - fetchType: VideoLoadByUrlType -): Promise { - if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url) - - if (fetchType === 'only-immutable-attributes') return VideoModel.loadByUrlImmutableAttributes(url) - - if (fetchType === 'only-video') return VideoModel.loadByUrl(url) -} - -export { - VideoLoadType, - VideoLoadByUrlType, - - loadVideo, - loadVideoByUrl -} diff --git a/server/lib/moderation.ts b/server/lib/moderation.ts deleted file mode 100644 index db8284872..000000000 --- a/server/lib/moderation.ts +++ /dev/null @@ -1,258 +0,0 @@ -import express, { VideoUploadFile } from 'express' -import { PathLike } from 'fs-extra' -import { Transaction } from 'sequelize/types' -import { AbuseAuditView, auditLoggerFactory } from '@server/helpers/audit-logger' -import { afterCommitIfTransaction } from '@server/helpers/database-utils' -import { logger } from '@server/helpers/logger' -import { AbuseModel } from '@server/models/abuse/abuse' -import { VideoAbuseModel } from '@server/models/abuse/video-abuse' -import { VideoCommentAbuseModel } from '@server/models/abuse/video-comment-abuse' -import { VideoFileModel } from '@server/models/video/video-file' -import { FilteredModelAttributes } from '@server/types' -import { - MAbuseFull, - MAccountDefault, - MAccountLight, - MComment, - MCommentAbuseAccountVideo, - MCommentOwnerVideo, - MUser, - MVideoAbuseVideoFull, - MVideoAccountLightBlacklistAllFiles -} from '@server/types/models' -import { LiveVideoCreate, VideoCreate, VideoImportCreate } from '../../shared/models/videos' -import { VideoCommentCreate } from '../../shared/models/videos/comment' -import { UserModel } from '../models/user/user' -import { VideoModel } from '../models/video/video' -import { VideoCommentModel } from '../models/video/video-comment' -import { sendAbuse } from './activitypub/send/send-flag' -import { Notifier } from './notifier' - -export type AcceptResult = { - accepted: boolean - errorMessage?: string -} - -// --------------------------------------------------------------------------- - -// Stub function that can be filtered by plugins -function isLocalVideoFileAccepted (object: { - videoBody: VideoCreate - videoFile: VideoUploadFile - user: UserModel -}): AcceptResult { - return { accepted: true } -} - -// --------------------------------------------------------------------------- - -// Stub function that can be filtered by plugins -function isLocalLiveVideoAccepted (object: { - liveVideoBody: LiveVideoCreate - user: UserModel -}): AcceptResult { - return { accepted: true } -} - -// --------------------------------------------------------------------------- - -// Stub function that can be filtered by plugins -function isLocalVideoThreadAccepted (_object: { - req: express.Request - commentBody: VideoCommentCreate - video: VideoModel - user: UserModel -}): AcceptResult { - return { accepted: true } -} - -// Stub function that can be filtered by plugins -function isLocalVideoCommentReplyAccepted (_object: { - req: express.Request - commentBody: VideoCommentCreate - parentComment: VideoCommentModel - video: VideoModel - user: UserModel -}): AcceptResult { - return { accepted: true } -} - -// --------------------------------------------------------------------------- - -// Stub function that can be filtered by plugins -function isRemoteVideoCommentAccepted (_object: { - comment: MComment -}): AcceptResult { - return { accepted: true } -} - -// --------------------------------------------------------------------------- - -// Stub function that can be filtered by plugins -function isPreImportVideoAccepted (object: { - videoImportBody: VideoImportCreate - user: MUser -}): AcceptResult { - return { accepted: true } -} - -// Stub function that can be filtered by plugins -function isPostImportVideoAccepted (object: { - videoFilePath: PathLike - videoFile: VideoFileModel - user: MUser -}): AcceptResult { - return { accepted: true } -} - -// --------------------------------------------------------------------------- - -async function createVideoAbuse (options: { - baseAbuse: FilteredModelAttributes - videoInstance: MVideoAccountLightBlacklistAllFiles - startAt: number - endAt: number - transaction: Transaction - reporterAccount: MAccountDefault - skipNotification: boolean -}) { - const { baseAbuse, videoInstance, startAt, endAt, transaction, reporterAccount, skipNotification } = options - - const associateFun = async (abuseInstance: MAbuseFull) => { - const videoAbuseInstance: MVideoAbuseVideoFull = await VideoAbuseModel.create({ - abuseId: abuseInstance.id, - videoId: videoInstance.id, - startAt, - endAt - }, { transaction }) - - videoAbuseInstance.Video = videoInstance - abuseInstance.VideoAbuse = videoAbuseInstance - - return { isOwned: videoInstance.isOwned() } - } - - return createAbuse({ - base: baseAbuse, - reporterAccount, - flaggedAccount: videoInstance.VideoChannel.Account, - transaction, - skipNotification, - associateFun - }) -} - -function createVideoCommentAbuse (options: { - baseAbuse: FilteredModelAttributes - commentInstance: MCommentOwnerVideo - transaction: Transaction - reporterAccount: MAccountDefault - skipNotification: boolean -}) { - const { baseAbuse, commentInstance, transaction, reporterAccount, skipNotification } = options - - const associateFun = async (abuseInstance: MAbuseFull) => { - const commentAbuseInstance: MCommentAbuseAccountVideo = await VideoCommentAbuseModel.create({ - abuseId: abuseInstance.id, - videoCommentId: commentInstance.id - }, { transaction }) - - commentAbuseInstance.VideoComment = commentInstance - abuseInstance.VideoCommentAbuse = commentAbuseInstance - - return { isOwned: commentInstance.isOwned() } - } - - return createAbuse({ - base: baseAbuse, - reporterAccount, - flaggedAccount: commentInstance.Account, - transaction, - skipNotification, - associateFun - }) -} - -function createAccountAbuse (options: { - baseAbuse: FilteredModelAttributes - accountInstance: MAccountDefault - transaction: Transaction - reporterAccount: MAccountDefault - skipNotification: boolean -}) { - const { baseAbuse, accountInstance, transaction, reporterAccount, skipNotification } = options - - const associateFun = () => { - return Promise.resolve({ isOwned: accountInstance.isOwned() }) - } - - return createAbuse({ - base: baseAbuse, - reporterAccount, - flaggedAccount: accountInstance, - transaction, - skipNotification, - associateFun - }) -} - -// --------------------------------------------------------------------------- - -export { - isLocalLiveVideoAccepted, - - isLocalVideoFileAccepted, - isLocalVideoThreadAccepted, - isRemoteVideoCommentAccepted, - isLocalVideoCommentReplyAccepted, - isPreImportVideoAccepted, - isPostImportVideoAccepted, - - createAbuse, - createVideoAbuse, - createVideoCommentAbuse, - createAccountAbuse -} - -// --------------------------------------------------------------------------- - -async function createAbuse (options: { - base: FilteredModelAttributes - reporterAccount: MAccountDefault - flaggedAccount: MAccountLight - associateFun: (abuseInstance: MAbuseFull) => Promise<{ isOwned: boolean }> - skipNotification: boolean - transaction: Transaction -}) { - const { base, reporterAccount, flaggedAccount, associateFun, transaction, skipNotification } = options - const auditLogger = auditLoggerFactory('abuse') - - const abuseAttributes = Object.assign({}, base, { flaggedAccountId: flaggedAccount.id }) - const abuseInstance: MAbuseFull = await AbuseModel.create(abuseAttributes, { transaction }) - - abuseInstance.ReporterAccount = reporterAccount - abuseInstance.FlaggedAccount = flaggedAccount - - const { isOwned } = await associateFun(abuseInstance) - - if (isOwned === false) { - sendAbuse(reporterAccount.Actor, abuseInstance, abuseInstance.FlaggedAccount, transaction) - } - - const abuseJSON = abuseInstance.toFormattedAdminJSON() - auditLogger.create(reporterAccount.Actor.getIdentifier(), new AbuseAuditView(abuseJSON)) - - if (!skipNotification) { - afterCommitIfTransaction(transaction, () => { - Notifier.Instance.notifyOnNewAbuse({ - abuse: abuseJSON, - abuseInstance, - reporter: reporterAccount.Actor.getIdentifier() - }) - }) - } - - logger.info('Abuse report %d created.', abuseInstance.id) - - return abuseJSON -} diff --git a/server/lib/notifier/index.ts b/server/lib/notifier/index.ts deleted file mode 100644 index 5bc2f5f50..000000000 --- a/server/lib/notifier/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './notifier' diff --git a/server/lib/notifier/notifier.ts b/server/lib/notifier/notifier.ts deleted file mode 100644 index 920c55df0..000000000 --- a/server/lib/notifier/notifier.ts +++ /dev/null @@ -1,284 +0,0 @@ -import { MRegistration, MUser, MUserDefault } from '@server/types/models/user' -import { MVideoBlacklistLightVideo, MVideoBlacklistVideo } from '@server/types/models/video/video-blacklist' -import { UserNotificationSettingValue } from '../../../shared/models/users' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { MAbuseFull, MAbuseMessage, MActorFollowFull, MApplication, MPlugin } from '../../types/models' -import { MCommentOwnerVideo, MVideoAccountLight, MVideoFullLight } from '../../types/models/video' -import { JobQueue } from '../job-queue' -import { PeerTubeSocket } from '../peertube-socket' -import { Hooks } from '../plugins/hooks' -import { - AbstractNotification, - AbuseStateChangeForReporter, - AutoFollowForInstance, - CommentMention, - DirectRegistrationForModerators, - FollowForInstance, - FollowForUser, - ImportFinishedForOwner, - ImportFinishedForOwnerPayload, - NewAbuseForModerators, - NewAbuseMessageForModerators, - NewAbuseMessageForReporter, - NewAbusePayload, - NewAutoBlacklistForModerators, - NewBlacklistForOwner, - NewCommentForVideoOwner, - NewPeerTubeVersionForAdmins, - NewPluginVersionForAdmins, - NewVideoForSubscribers, - OwnedPublicationAfterAutoUnblacklist, - OwnedPublicationAfterScheduleUpdate, - OwnedPublicationAfterTranscoding, - RegistrationRequestForModerators, - StudioEditionFinishedForOwner, - UnblacklistForOwner -} from './shared' - -class Notifier { - - private readonly notificationModels = { - newVideo: [ NewVideoForSubscribers ], - publicationAfterTranscoding: [ OwnedPublicationAfterTranscoding ], - publicationAfterScheduleUpdate: [ OwnedPublicationAfterScheduleUpdate ], - publicationAfterAutoUnblacklist: [ OwnedPublicationAfterAutoUnblacklist ], - newComment: [ CommentMention, NewCommentForVideoOwner ], - newAbuse: [ NewAbuseForModerators ], - newBlacklist: [ NewBlacklistForOwner ], - unblacklist: [ UnblacklistForOwner ], - importFinished: [ ImportFinishedForOwner ], - directRegistration: [ DirectRegistrationForModerators ], - registrationRequest: [ RegistrationRequestForModerators ], - userFollow: [ FollowForUser ], - instanceFollow: [ FollowForInstance ], - autoInstanceFollow: [ AutoFollowForInstance ], - newAutoBlacklist: [ NewAutoBlacklistForModerators ], - abuseStateChange: [ AbuseStateChangeForReporter ], - newAbuseMessage: [ NewAbuseMessageForReporter, NewAbuseMessageForModerators ], - newPeertubeVersion: [ NewPeerTubeVersionForAdmins ], - newPluginVersion: [ NewPluginVersionForAdmins ], - videoStudioEditionFinished: [ StudioEditionFinishedForOwner ] - } - - private static instance: Notifier - - private constructor () { - } - - notifyOnNewVideoIfNeeded (video: MVideoAccountLight): void { - const models = this.notificationModels.newVideo - - this.sendNotifications(models, video) - .catch(err => logger.error('Cannot notify subscribers of new video %s.', video.url, { err })) - } - - notifyOnVideoPublishedAfterTranscoding (video: MVideoFullLight): void { - const models = this.notificationModels.publicationAfterTranscoding - - this.sendNotifications(models, video) - .catch(err => logger.error('Cannot notify owner that its video %s has been published after transcoding.', video.url, { err })) - } - - notifyOnVideoPublishedAfterScheduledUpdate (video: MVideoFullLight): void { - const models = this.notificationModels.publicationAfterScheduleUpdate - - this.sendNotifications(models, video) - .catch(err => logger.error('Cannot notify owner that its video %s has been published after scheduled update.', video.url, { err })) - } - - notifyOnVideoPublishedAfterRemovedFromAutoBlacklist (video: MVideoFullLight): void { - const models = this.notificationModels.publicationAfterAutoUnblacklist - - this.sendNotifications(models, video) - .catch(err => { - logger.error('Cannot notify owner that its video %s has been published after removed from auto-blacklist.', video.url, { err }) - }) - } - - notifyOnNewComment (comment: MCommentOwnerVideo): void { - const models = this.notificationModels.newComment - - this.sendNotifications(models, comment) - .catch(err => logger.error('Cannot notify of new comment.', comment.url, { err })) - } - - notifyOnNewAbuse (payload: NewAbusePayload): void { - const models = this.notificationModels.newAbuse - - this.sendNotifications(models, payload) - .catch(err => logger.error('Cannot notify of new abuse %d.', payload.abuseInstance.id, { err })) - } - - notifyOnVideoAutoBlacklist (videoBlacklist: MVideoBlacklistLightVideo): void { - const models = this.notificationModels.newAutoBlacklist - - this.sendNotifications(models, videoBlacklist) - .catch(err => logger.error('Cannot notify of auto-blacklist of video %s.', videoBlacklist.Video.url, { err })) - } - - notifyOnVideoBlacklist (videoBlacklist: MVideoBlacklistVideo): void { - const models = this.notificationModels.newBlacklist - - this.sendNotifications(models, videoBlacklist) - .catch(err => logger.error('Cannot notify video owner of new video blacklist of %s.', videoBlacklist.Video.url, { err })) - } - - notifyOnVideoUnblacklist (video: MVideoFullLight): void { - const models = this.notificationModels.unblacklist - - this.sendNotifications(models, video) - .catch(err => logger.error('Cannot notify video owner of unblacklist of %s.', video.url, { err })) - } - - notifyOnFinishedVideoImport (payload: ImportFinishedForOwnerPayload): void { - const models = this.notificationModels.importFinished - - this.sendNotifications(models, payload) - .catch(err => { - logger.error('Cannot notify owner that its video import %s is finished.', payload.videoImport.getTargetIdentifier(), { err }) - }) - } - - notifyOnNewDirectRegistration (user: MUserDefault): void { - const models = this.notificationModels.directRegistration - - this.sendNotifications(models, user) - .catch(err => logger.error('Cannot notify moderators of new user registration (%s).', user.username, { err })) - } - - notifyOnNewRegistrationRequest (registration: MRegistration): void { - const models = this.notificationModels.registrationRequest - - this.sendNotifications(models, registration) - .catch(err => logger.error('Cannot notify moderators of new registration request (%s).', registration.username, { err })) - } - - notifyOfNewUserFollow (actorFollow: MActorFollowFull): void { - const models = this.notificationModels.userFollow - - this.sendNotifications(models, actorFollow) - .catch(err => { - logger.error( - 'Cannot notify owner of channel %s of a new follow by %s.', - actorFollow.ActorFollowing.VideoChannel.getDisplayName(), - actorFollow.ActorFollower.Account.getDisplayName(), - { err } - ) - }) - } - - notifyOfNewInstanceFollow (actorFollow: MActorFollowFull): void { - const models = this.notificationModels.instanceFollow - - this.sendNotifications(models, actorFollow) - .catch(err => logger.error('Cannot notify administrators of new follower %s.', actorFollow.ActorFollower.url, { err })) - } - - notifyOfAutoInstanceFollowing (actorFollow: MActorFollowFull): void { - const models = this.notificationModels.autoInstanceFollow - - this.sendNotifications(models, actorFollow) - .catch(err => logger.error('Cannot notify administrators of auto instance following %s.', actorFollow.ActorFollowing.url, { err })) - } - - notifyOnAbuseStateChange (abuse: MAbuseFull): void { - const models = this.notificationModels.abuseStateChange - - this.sendNotifications(models, abuse) - .catch(err => logger.error('Cannot notify of abuse %d state change.', abuse.id, { err })) - } - - notifyOnAbuseMessage (abuse: MAbuseFull, message: MAbuseMessage): void { - const models = this.notificationModels.newAbuseMessage - - this.sendNotifications(models, { abuse, message }) - .catch(err => logger.error('Cannot notify on new abuse %d message.', abuse.id, { err })) - } - - notifyOfNewPeerTubeVersion (application: MApplication, latestVersion: string) { - const models = this.notificationModels.newPeertubeVersion - - this.sendNotifications(models, { application, latestVersion }) - .catch(err => logger.error('Cannot notify on new PeerTubeb version %s.', latestVersion, { err })) - } - - notifyOfNewPluginVersion (plugin: MPlugin) { - const models = this.notificationModels.newPluginVersion - - this.sendNotifications(models, plugin) - .catch(err => logger.error('Cannot notify on new plugin version %s.', plugin.name, { err })) - } - - notifyOfFinishedVideoStudioEdition (video: MVideoFullLight) { - const models = this.notificationModels.videoStudioEditionFinished - - this.sendNotifications(models, video) - .catch(err => logger.error('Cannot notify on finished studio edition %s.', video.url, { err })) - } - - private async notify (object: AbstractNotification) { - await object.prepare() - - const users = object.getTargetUsers() - - if (users.length === 0) return - if (await object.isDisabled()) return - - object.log() - - const toEmails: string[] = [] - - for (const user of users) { - const setting = object.getSetting(user) - - const webNotificationEnabled = this.isWebNotificationEnabled(setting) - const emailNotificationEnabled = this.isEmailEnabled(user, setting) - const notification = object.createNotification(user) - - if (webNotificationEnabled) { - await notification.save() - - PeerTubeSocket.Instance.sendNotification(user.id, notification) - } - - if (emailNotificationEnabled) { - toEmails.push(user.email) - } - - Hooks.runAction('action:notifier.notification.created', { webNotificationEnabled, emailNotificationEnabled, user, notification }) - } - - for (const to of toEmails) { - const payload = await object.createEmail(to) - JobQueue.Instance.createJobAsync({ type: 'email', payload }) - } - } - - private isEmailEnabled (user: MUser, value: UserNotificationSettingValue) { - if (CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION === true && user.emailVerified === false) return false - - return value & UserNotificationSettingValue.EMAIL - } - - private isWebNotificationEnabled (value: UserNotificationSettingValue) { - return value & UserNotificationSettingValue.WEB - } - - private async sendNotifications (models: (new (payload: T) => AbstractNotification)[], payload: T) { - for (const model of models) { - // eslint-disable-next-line new-cap - await this.notify(new model(payload)) - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -// --------------------------------------------------------------------------- - -export { - Notifier -} diff --git a/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts b/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts deleted file mode 100644 index 1dc1ccfc2..000000000 --- a/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { WEBSERVER } from '@server/initializers/constants' -import { AccountModel } from '@server/models/account/account' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MAbuseFull, MAbuseMessage, MAccountDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -type NewAbuseMessagePayload = { - abuse: MAbuseFull - message: MAbuseMessage -} - -export abstract class AbstractNewAbuseMessage extends AbstractNotification { - protected messageAccount: MAccountDefault - - async loadMessageAccount () { - this.messageAccount = await AccountModel.load(this.message.accountId) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.abuseNewMessage - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.ABUSE_NEW_MESSAGE, - userId: user.id, - abuseId: this.abuse.id - }) - notification.Abuse = this.abuse - - return notification - } - - protected createEmailFor (to: string, target: 'moderator' | 'reporter') { - const text = 'New message on report #' + this.abuse.id - const abuseUrl = target === 'moderator' - ? WEBSERVER.URL + '/admin/moderation/abuses/list?search=%23' + this.abuse.id - : WEBSERVER.URL + '/my-account/abuses?search=%23' + this.abuse.id - - const action = { - text: 'View report #' + this.abuse.id, - url: abuseUrl - } - - return { - template: 'abuse-new-message', - to, - subject: text, - locals: { - abuseId: this.abuse.id, - abuseUrl: action.url, - messageAccountName: this.messageAccount.getDisplayName(), - messageText: this.message.message, - action - } - } - } - - protected get abuse () { - return this.payload.abuse - } - - protected get message () { - return this.payload.message - } -} diff --git a/server/lib/notifier/shared/abuse/abuse-state-change-for-reporter.ts b/server/lib/notifier/shared/abuse/abuse-state-change-for-reporter.ts deleted file mode 100644 index 97e896c6a..000000000 --- a/server/lib/notifier/shared/abuse/abuse-state-change-for-reporter.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { WEBSERVER } from '@server/initializers/constants' -import { getAbuseTargetUrl } from '@server/lib/activitypub/url' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MAbuseFull, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { AbuseState, UserNotificationType } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class AbuseStateChangeForReporter extends AbstractNotification { - - private user: MUserDefault - - async prepare () { - const reporter = this.abuse.ReporterAccount - if (reporter.isOwned() !== true) return - - this.user = await UserModel.loadByAccountActorId(this.abuse.ReporterAccount.actorId) - } - - log () { - logger.info('Notifying reporter of abuse % of state change.', getAbuseTargetUrl(this.abuse)) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.abuseStateChange - } - - getTargetUsers () { - if (!this.user) return [] - - return [ this.user ] - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.ABUSE_STATE_CHANGE, - userId: user.id, - abuseId: this.abuse.id - }) - notification.Abuse = this.abuse - - return notification - } - - createEmail (to: string) { - const text = this.abuse.state === AbuseState.ACCEPTED - ? 'Report #' + this.abuse.id + ' has been accepted' - : 'Report #' + this.abuse.id + ' has been rejected' - - const abuseUrl = WEBSERVER.URL + '/my-account/abuses?search=%23' + this.abuse.id - - const action = { - text: 'View report #' + this.abuse.id, - url: abuseUrl - } - - return { - template: 'abuse-state-change', - to, - subject: text, - locals: { - action, - abuseId: this.abuse.id, - abuseUrl, - isAccepted: this.abuse.state === AbuseState.ACCEPTED - } - } - } - - private get abuse () { - return this.payload - } -} diff --git a/server/lib/notifier/shared/abuse/index.ts b/server/lib/notifier/shared/abuse/index.ts deleted file mode 100644 index 7b54c5591..000000000 --- a/server/lib/notifier/shared/abuse/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './abuse-state-change-for-reporter' -export * from './new-abuse-for-moderators' -export * from './new-abuse-message-for-reporter' -export * from './new-abuse-message-for-moderators' diff --git a/server/lib/notifier/shared/abuse/new-abuse-for-moderators.ts b/server/lib/notifier/shared/abuse/new-abuse-for-moderators.ts deleted file mode 100644 index 7d86fb55f..000000000 --- a/server/lib/notifier/shared/abuse/new-abuse-for-moderators.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { WEBSERVER } from '@server/initializers/constants' -import { getAbuseTargetUrl } from '@server/lib/activitypub/url' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MAbuseFull, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { UserAbuse, UserNotificationType, UserRight } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export type NewAbusePayload = { abuse: UserAbuse, abuseInstance: MAbuseFull, reporter: string } - -export class NewAbuseForModerators extends AbstractNotification { - private moderators: MUserDefault[] - - async prepare () { - this.moderators = await UserModel.listWithRight(UserRight.MANAGE_ABUSES) - } - - log () { - logger.info('Notifying %s user/moderators of new abuse %s.', this.moderators.length, getAbuseTargetUrl(this.payload.abuseInstance)) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.abuseAsModerator - } - - getTargetUsers () { - return this.moderators - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.NEW_ABUSE_FOR_MODERATORS, - userId: user.id, - abuseId: this.payload.abuseInstance.id - }) - notification.Abuse = this.payload.abuseInstance - - return notification - } - - createEmail (to: string) { - const abuseInstance = this.payload.abuseInstance - - if (abuseInstance.VideoAbuse) return this.createVideoAbuseEmail(to) - if (abuseInstance.VideoCommentAbuse) return this.createCommentAbuseEmail(to) - - return this.createAccountAbuseEmail(to) - } - - private createVideoAbuseEmail (to: string) { - const video = this.payload.abuseInstance.VideoAbuse.Video - const videoUrl = WEBSERVER.URL + video.getWatchStaticPath() - - return { - template: 'video-abuse-new', - to, - subject: `New video abuse report from ${this.payload.reporter}`, - locals: { - videoUrl, - isLocal: video.remote === false, - videoCreatedAt: new Date(video.createdAt).toLocaleString(), - videoPublishedAt: new Date(video.publishedAt).toLocaleString(), - videoName: video.name, - reason: this.payload.abuse.reason, - videoChannel: this.payload.abuse.video.channel, - reporter: this.payload.reporter, - action: this.buildEmailAction() - } - } - } - - private createCommentAbuseEmail (to: string) { - const comment = this.payload.abuseInstance.VideoCommentAbuse.VideoComment - const commentUrl = WEBSERVER.URL + comment.Video.getWatchStaticPath() + ';threadId=' + comment.getThreadId() - - return { - template: 'video-comment-abuse-new', - to, - subject: `New comment abuse report from ${this.payload.reporter}`, - locals: { - commentUrl, - videoName: comment.Video.name, - isLocal: comment.isOwned(), - commentCreatedAt: new Date(comment.createdAt).toLocaleString(), - reason: this.payload.abuse.reason, - flaggedAccount: this.payload.abuseInstance.FlaggedAccount.getDisplayName(), - reporter: this.payload.reporter, - action: this.buildEmailAction() - } - } - } - - private createAccountAbuseEmail (to: string) { - const account = this.payload.abuseInstance.FlaggedAccount - const accountUrl = account.getClientUrl() - - return { - template: 'account-abuse-new', - to, - subject: `New account abuse report from ${this.payload.reporter}`, - locals: { - accountUrl, - accountDisplayName: account.getDisplayName(), - isLocal: account.isOwned(), - reason: this.payload.abuse.reason, - reporter: this.payload.reporter, - action: this.buildEmailAction() - } - } - } - - private buildEmailAction () { - return { - text: 'View report #' + this.payload.abuseInstance.id, - url: WEBSERVER.URL + '/admin/moderation/abuses/list?search=%23' + this.payload.abuseInstance.id - } - } -} diff --git a/server/lib/notifier/shared/abuse/new-abuse-message-for-moderators.ts b/server/lib/notifier/shared/abuse/new-abuse-message-for-moderators.ts deleted file mode 100644 index 9d0629690..000000000 --- a/server/lib/notifier/shared/abuse/new-abuse-message-for-moderators.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { getAbuseTargetUrl } from '@server/lib/activitypub/url' -import { UserModel } from '@server/models/user/user' -import { MUserDefault } from '@server/types/models' -import { UserRight } from '@shared/models' -import { AbstractNewAbuseMessage } from './abstract-new-abuse-message' - -export class NewAbuseMessageForModerators extends AbstractNewAbuseMessage { - private moderators: MUserDefault[] - - async prepare () { - this.moderators = await UserModel.listWithRight(UserRight.MANAGE_ABUSES) - - // Don't notify my own message - this.moderators = this.moderators.filter(m => m.Account.id !== this.message.accountId) - if (this.moderators.length === 0) return - - await this.loadMessageAccount() - } - - log () { - logger.info('Notifying moderators of new abuse message on %s.', getAbuseTargetUrl(this.abuse)) - } - - getTargetUsers () { - return this.moderators - } - - createEmail (to: string) { - return this.createEmailFor(to, 'moderator') - } -} diff --git a/server/lib/notifier/shared/abuse/new-abuse-message-for-reporter.ts b/server/lib/notifier/shared/abuse/new-abuse-message-for-reporter.ts deleted file mode 100644 index c5bbb5447..000000000 --- a/server/lib/notifier/shared/abuse/new-abuse-message-for-reporter.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { getAbuseTargetUrl } from '@server/lib/activitypub/url' -import { UserModel } from '@server/models/user/user' -import { MUserDefault } from '@server/types/models' -import { AbstractNewAbuseMessage } from './abstract-new-abuse-message' - -export class NewAbuseMessageForReporter extends AbstractNewAbuseMessage { - private reporter: MUserDefault - - async prepare () { - // Only notify our users - if (this.abuse.ReporterAccount.isOwned() !== true) return - - await this.loadMessageAccount() - - const reporter = await UserModel.loadByAccountActorId(this.abuse.ReporterAccount.actorId) - // Don't notify my own message - if (reporter.Account.id === this.message.accountId) return - - this.reporter = reporter - } - - log () { - logger.info('Notifying reporter of new abuse message on %s.', getAbuseTargetUrl(this.abuse)) - } - - getTargetUsers () { - if (!this.reporter) return [] - - return [ this.reporter ] - } - - createEmail (to: string) { - return this.createEmailFor(to, 'reporter') - } -} diff --git a/server/lib/notifier/shared/blacklist/index.ts b/server/lib/notifier/shared/blacklist/index.ts deleted file mode 100644 index 2f98d88ae..000000000 --- a/server/lib/notifier/shared/blacklist/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './new-auto-blacklist-for-moderators' -export * from './new-blacklist-for-owner' -export * from './unblacklist-for-owner' diff --git a/server/lib/notifier/shared/blacklist/new-auto-blacklist-for-moderators.ts b/server/lib/notifier/shared/blacklist/new-auto-blacklist-for-moderators.ts deleted file mode 100644 index ad2cc00ea..000000000 --- a/server/lib/notifier/shared/blacklist/new-auto-blacklist-for-moderators.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { WEBSERVER } from '@server/initializers/constants' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { VideoChannelModel } from '@server/models/video/video-channel' -import { MUserDefault, MUserWithNotificationSetting, MVideoBlacklistLightVideo, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType, UserRight } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class NewAutoBlacklistForModerators extends AbstractNotification { - private moderators: MUserDefault[] - - async prepare () { - this.moderators = await UserModel.listWithRight(UserRight.MANAGE_VIDEO_BLACKLIST) - } - - log () { - logger.info('Notifying %s moderators of video auto-blacklist %s.', this.moderators.length, this.payload.Video.url) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.videoAutoBlacklistAsModerator - } - - getTargetUsers () { - return this.moderators - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.VIDEO_AUTO_BLACKLIST_FOR_MODERATORS, - userId: user.id, - videoBlacklistId: this.payload.id - }) - notification.VideoBlacklist = this.payload - - return notification - } - - async createEmail (to: string) { - const videoAutoBlacklistUrl = WEBSERVER.URL + '/admin/moderation/video-auto-blacklist/list' - const videoUrl = WEBSERVER.URL + this.payload.Video.getWatchStaticPath() - const channel = await VideoChannelModel.loadAndPopulateAccount(this.payload.Video.channelId) - - return { - template: 'video-auto-blacklist-new', - to, - subject: 'A new video is pending moderation', - locals: { - channel: channel.toFormattedSummaryJSON(), - videoUrl, - videoName: this.payload.Video.name, - action: { - text: 'Review autoblacklist', - url: videoAutoBlacklistUrl - } - } - } - } -} diff --git a/server/lib/notifier/shared/blacklist/new-blacklist-for-owner.ts b/server/lib/notifier/shared/blacklist/new-blacklist-for-owner.ts deleted file mode 100644 index 342b69ec7..000000000 --- a/server/lib/notifier/shared/blacklist/new-blacklist-for-owner.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { WEBSERVER } from '@server/initializers/constants' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MUserDefault, MUserWithNotificationSetting, MVideoBlacklistVideo, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class NewBlacklistForOwner extends AbstractNotification { - private user: MUserDefault - - async prepare () { - this.user = await UserModel.loadByVideoId(this.payload.videoId) - } - - log () { - logger.info('Notifying user %s that its video %s has been blacklisted.', this.user.username, this.payload.Video.url) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.blacklistOnMyVideo - } - - getTargetUsers () { - if (!this.user) return [] - - return [ this.user ] - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.BLACKLIST_ON_MY_VIDEO, - userId: user.id, - videoBlacklistId: this.payload.id - }) - notification.VideoBlacklist = this.payload - - return notification - } - - createEmail (to: string) { - const videoName = this.payload.Video.name - const videoUrl = WEBSERVER.URL + this.payload.Video.getWatchStaticPath() - - const reasonString = this.payload.reason ? ` for the following reason: ${this.payload.reason}` : '' - const blockedString = `Your video ${videoName} (${videoUrl} on ${CONFIG.INSTANCE.NAME} has been blacklisted${reasonString}.` - - return { - to, - subject: `Video ${videoName} blacklisted`, - text: blockedString, - locals: { - title: 'Your video was blacklisted' - } - } - } -} diff --git a/server/lib/notifier/shared/blacklist/unblacklist-for-owner.ts b/server/lib/notifier/shared/blacklist/unblacklist-for-owner.ts deleted file mode 100644 index e6f90e23c..000000000 --- a/server/lib/notifier/shared/blacklist/unblacklist-for-owner.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { WEBSERVER } from '@server/initializers/constants' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MUserDefault, MUserWithNotificationSetting, MVideoFullLight, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class UnblacklistForOwner extends AbstractNotification { - private user: MUserDefault - - async prepare () { - this.user = await UserModel.loadByVideoId(this.payload.id) - } - - log () { - logger.info('Notifying user %s that its video %s has been unblacklisted.', this.user.username, this.payload.url) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.blacklistOnMyVideo - } - - getTargetUsers () { - if (!this.user) return [] - - return [ this.user ] - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.UNBLACKLIST_ON_MY_VIDEO, - userId: user.id, - videoId: this.payload.id - }) - notification.Video = this.payload - - return notification - } - - createEmail (to: string) { - const video = this.payload - const videoUrl = WEBSERVER.URL + video.getWatchStaticPath() - - return { - to, - subject: `Video ${video.name} unblacklisted`, - text: `Your video "${video.name}" (${videoUrl}) on ${CONFIG.INSTANCE.NAME} has been unblacklisted.`, - locals: { - title: 'Your video was unblacklisted' - } - } - } -} diff --git a/server/lib/notifier/shared/comment/comment-mention.ts b/server/lib/notifier/shared/comment/comment-mention.ts deleted file mode 100644 index 3074e97db..000000000 --- a/server/lib/notifier/shared/comment/comment-mention.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { toSafeHtml } from '@server/helpers/markdown' -import { WEBSERVER } from '@server/initializers/constants' -import { AccountBlocklistModel } from '@server/models/account/account-blocklist' -import { getServerActor } from '@server/models/application/application' -import { ServerBlocklistModel } from '@server/models/server/server-blocklist' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { - MCommentOwnerVideo, - MUserDefault, - MUserNotifSettingAccount, - MUserWithNotificationSetting, - UserNotificationModelForApi -} from '@server/types/models' -import { UserNotificationSettingValue, UserNotificationType } from '@shared/models' -import { AbstractNotification } from '../common' - -export class CommentMention extends AbstractNotification { - private users: MUserDefault[] - - private serverAccountId: number - - private accountMutedHash: { [ id: number ]: boolean } - private instanceMutedHash: { [ id: number ]: boolean } - - async prepare () { - const extractedUsernames = this.payload.extractMentions() - logger.debug( - 'Extracted %d username from comment %s.', extractedUsernames.length, this.payload.url, - { usernames: extractedUsernames, text: this.payload.text } - ) - - this.users = await UserModel.listByUsernames(extractedUsernames) - - if (this.payload.Video.isOwned()) { - const userException = await UserModel.loadByVideoId(this.payload.videoId) - this.users = this.users.filter(u => u.id !== userException.id) - } - - // Don't notify if I mentioned myself - this.users = this.users.filter(u => u.Account.id !== this.payload.accountId) - - if (this.users.length === 0) return - - this.serverAccountId = (await getServerActor()).Account.id - - const sourceAccounts = this.users.map(u => u.Account.id).concat([ this.serverAccountId ]) - - this.accountMutedHash = await AccountBlocklistModel.isAccountMutedByAccounts(sourceAccounts, this.payload.accountId) - this.instanceMutedHash = await ServerBlocklistModel.isServerMutedByAccounts(sourceAccounts, this.payload.Account.Actor.serverId) - } - - log () { - logger.info('Notifying %d users of new comment %s.', this.users.length, this.payload.url) - } - - getSetting (user: MUserNotifSettingAccount) { - const accountId = user.Account.id - if ( - this.accountMutedHash[accountId] === true || this.instanceMutedHash[accountId] === true || - this.accountMutedHash[this.serverAccountId] === true || this.instanceMutedHash[this.serverAccountId] === true - ) { - return UserNotificationSettingValue.NONE - } - - return user.NotificationSetting.commentMention - } - - getTargetUsers () { - return this.users - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.COMMENT_MENTION, - userId: user.id, - commentId: this.payload.id - }) - notification.VideoComment = this.payload - - return notification - } - - createEmail (to: string) { - const comment = this.payload - - const accountName = comment.Account.getDisplayName() - const video = comment.Video - const videoUrl = WEBSERVER.URL + comment.Video.getWatchStaticPath() - const commentUrl = WEBSERVER.URL + comment.getCommentStaticPath() - const commentHtml = toSafeHtml(comment.text) - - return { - template: 'video-comment-mention', - to, - subject: 'Mention on video ' + video.name, - locals: { - comment, - commentHtml, - video, - videoUrl, - accountName, - action: { - text: 'View comment', - url: commentUrl - } - } - } - } -} diff --git a/server/lib/notifier/shared/comment/index.ts b/server/lib/notifier/shared/comment/index.ts deleted file mode 100644 index ae01a9646..000000000 --- a/server/lib/notifier/shared/comment/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './comment-mention' -export * from './new-comment-for-video-owner' diff --git a/server/lib/notifier/shared/comment/new-comment-for-video-owner.ts b/server/lib/notifier/shared/comment/new-comment-for-video-owner.ts deleted file mode 100644 index 4f96439a3..000000000 --- a/server/lib/notifier/shared/comment/new-comment-for-video-owner.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { toSafeHtml } from '@server/helpers/markdown' -import { WEBSERVER } from '@server/initializers/constants' -import { isBlockedByServerOrAccount } from '@server/lib/blocklist' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MCommentOwnerVideo, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class NewCommentForVideoOwner extends AbstractNotification { - private user: MUserDefault - - async prepare () { - this.user = await UserModel.loadByVideoId(this.payload.videoId) - } - - log () { - logger.info('Notifying owner of a video %s of new comment %s.', this.user.username, this.payload.url) - } - - isDisabled () { - if (this.payload.Video.isOwned() === false) return true - - // Not our user or user comments its own video - if (!this.user || this.payload.Account.userId === this.user.id) return true - - return isBlockedByServerOrAccount(this.payload.Account, this.user.Account) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.newCommentOnMyVideo - } - - getTargetUsers () { - if (!this.user) return [] - - return [ this.user ] - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.NEW_COMMENT_ON_MY_VIDEO, - userId: user.id, - commentId: this.payload.id - }) - notification.VideoComment = this.payload - - return notification - } - - createEmail (to: string) { - const video = this.payload.Video - const videoUrl = WEBSERVER.URL + this.payload.Video.getWatchStaticPath() - const commentUrl = WEBSERVER.URL + this.payload.getCommentStaticPath() - const commentHtml = toSafeHtml(this.payload.text) - - return { - template: 'video-comment-new', - to, - subject: 'New comment on your video ' + video.name, - locals: { - accountName: this.payload.Account.getDisplayName(), - accountUrl: this.payload.Account.Actor.url, - comment: this.payload, - commentHtml, - video, - videoUrl, - action: { - text: 'View comment', - url: commentUrl - } - } - } - } -} diff --git a/server/lib/notifier/shared/common/abstract-notification.ts b/server/lib/notifier/shared/common/abstract-notification.ts deleted file mode 100644 index 79403611e..000000000 --- a/server/lib/notifier/shared/common/abstract-notification.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { EmailPayload, UserNotificationSettingValue } from '@shared/models' - -export abstract class AbstractNotification { - - constructor (protected readonly payload: T) { - - } - - abstract prepare (): Promise - abstract log (): void - - abstract getSetting (user: U): UserNotificationSettingValue - abstract getTargetUsers (): U[] - - abstract createNotification (user: U): UserNotificationModelForApi - abstract createEmail (to: string): EmailPayload | Promise - - isDisabled (): boolean | Promise { - return false - } - -} diff --git a/server/lib/notifier/shared/common/index.ts b/server/lib/notifier/shared/common/index.ts deleted file mode 100644 index 0b2570278..000000000 --- a/server/lib/notifier/shared/common/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './abstract-notification' diff --git a/server/lib/notifier/shared/follow/auto-follow-for-instance.ts b/server/lib/notifier/shared/follow/auto-follow-for-instance.ts deleted file mode 100644 index ab9747ba8..000000000 --- a/server/lib/notifier/shared/follow/auto-follow-for-instance.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MActorFollowFull, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType, UserRight } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class AutoFollowForInstance extends AbstractNotification { - private admins: MUserDefault[] - - async prepare () { - this.admins = await UserModel.listWithRight(UserRight.MANAGE_SERVER_FOLLOW) - } - - log () { - logger.info('Notifying %d administrators of auto instance following: %s.', this.admins.length, this.actorFollow.ActorFollowing.url) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.autoInstanceFollowing - } - - getTargetUsers () { - return this.admins - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.AUTO_INSTANCE_FOLLOWING, - userId: user.id, - actorFollowId: this.actorFollow.id - }) - notification.ActorFollow = this.actorFollow - - return notification - } - - createEmail (to: string) { - const instanceUrl = this.actorFollow.ActorFollowing.url - - return { - to, - subject: 'Auto instance following', - text: `Your instance automatically followed a new instance: ${instanceUrl}.` - } - } - - private get actorFollow () { - return this.payload - } -} diff --git a/server/lib/notifier/shared/follow/follow-for-instance.ts b/server/lib/notifier/shared/follow/follow-for-instance.ts deleted file mode 100644 index 777a12ef4..000000000 --- a/server/lib/notifier/shared/follow/follow-for-instance.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { WEBSERVER } from '@server/initializers/constants' -import { isBlockedByServerOrAccount } from '@server/lib/blocklist' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MActorFollowFull, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType, UserRight } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class FollowForInstance extends AbstractNotification { - private admins: MUserDefault[] - - async prepare () { - this.admins = await UserModel.listWithRight(UserRight.MANAGE_SERVER_FOLLOW) - } - - isDisabled () { - const follower = Object.assign(this.actorFollow.ActorFollower.Account, { Actor: this.actorFollow.ActorFollower }) - - return isBlockedByServerOrAccount(follower) - } - - log () { - logger.info('Notifying %d administrators of new instance follower: %s.', this.admins.length, this.actorFollow.ActorFollower.url) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.newInstanceFollower - } - - getTargetUsers () { - return this.admins - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.NEW_INSTANCE_FOLLOWER, - userId: user.id, - actorFollowId: this.actorFollow.id - }) - notification.ActorFollow = this.actorFollow - - return notification - } - - createEmail (to: string) { - const awaitingApproval = this.actorFollow.state === 'pending' - ? ' awaiting manual approval.' - : '' - - return { - to, - subject: 'New instance follower', - text: `Your instance has a new follower: ${this.actorFollow.ActorFollower.url}${awaitingApproval}.`, - locals: { - title: 'New instance follower', - action: { - text: 'Review followers', - url: WEBSERVER.URL + '/admin/follows/followers-list' - } - } - } - } - - private get actorFollow () { - return this.payload - } -} diff --git a/server/lib/notifier/shared/follow/follow-for-user.ts b/server/lib/notifier/shared/follow/follow-for-user.ts deleted file mode 100644 index 697c82cdd..000000000 --- a/server/lib/notifier/shared/follow/follow-for-user.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { isBlockedByServerOrAccount } from '@server/lib/blocklist' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MActorFollowFull, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class FollowForUser extends AbstractNotification { - private followType: 'account' | 'channel' - private user: MUserDefault - - async prepare () { - // Account follows one of our account? - this.followType = 'channel' - this.user = await UserModel.loadByChannelActorId(this.actorFollow.ActorFollowing.id) - - // Account follows one of our channel? - if (!this.user) { - this.user = await UserModel.loadByAccountActorId(this.actorFollow.ActorFollowing.id) - this.followType = 'account' - } - } - - async isDisabled () { - if (this.payload.ActorFollowing.isOwned() === false) return true - - const followerAccount = this.actorFollow.ActorFollower.Account - const followerAccountWithActor = Object.assign(followerAccount, { Actor: this.actorFollow.ActorFollower }) - - return isBlockedByServerOrAccount(followerAccountWithActor, this.user.Account) - } - - log () { - logger.info('Notifying user %s of new follower: %s.', this.user.username, this.actorFollow.ActorFollower.Account.getDisplayName()) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.newFollow - } - - getTargetUsers () { - if (!this.user) return [] - - return [ this.user ] - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.NEW_FOLLOW, - userId: user.id, - actorFollowId: this.actorFollow.id - }) - notification.ActorFollow = this.actorFollow - - return notification - } - - createEmail (to: string) { - const following = this.actorFollow.ActorFollowing - const follower = this.actorFollow.ActorFollower - - const followingName = (following.VideoChannel || following.Account).getDisplayName() - - return { - template: 'follower-on-channel', - to, - subject: `New follower on your channel ${followingName}`, - locals: { - followerName: follower.Account.getDisplayName(), - followerUrl: follower.url, - followingName, - followingUrl: following.url, - followType: this.followType - } - } - } - - private get actorFollow () { - return this.payload - } -} diff --git a/server/lib/notifier/shared/follow/index.ts b/server/lib/notifier/shared/follow/index.ts deleted file mode 100644 index 27f5289d9..000000000 --- a/server/lib/notifier/shared/follow/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './auto-follow-for-instance' -export * from './follow-for-instance' -export * from './follow-for-user' diff --git a/server/lib/notifier/shared/index.ts b/server/lib/notifier/shared/index.ts deleted file mode 100644 index cc3ce8c7c..000000000 --- a/server/lib/notifier/shared/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export * from './abuse' -export * from './blacklist' -export * from './comment' -export * from './common' -export * from './follow' -export * from './instance' -export * from './video-publication' diff --git a/server/lib/notifier/shared/instance/direct-registration-for-moderators.ts b/server/lib/notifier/shared/instance/direct-registration-for-moderators.ts deleted file mode 100644 index 5044f2068..000000000 --- a/server/lib/notifier/shared/instance/direct-registration-for-moderators.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType, UserRight } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class DirectRegistrationForModerators extends AbstractNotification { - private moderators: MUserDefault[] - - async prepare () { - this.moderators = await UserModel.listWithRight(UserRight.MANAGE_USERS) - } - - log () { - logger.info('Notifying %s moderators of new user registration of %s.', this.moderators.length, this.payload.username) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.newUserRegistration - } - - getTargetUsers () { - return this.moderators - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.NEW_USER_REGISTRATION, - userId: user.id, - accountId: this.payload.Account.id - }) - notification.Account = this.payload.Account - - return notification - } - - createEmail (to: string) { - return { - template: 'user-registered', - to, - subject: `A new user registered on ${CONFIG.INSTANCE.NAME}: ${this.payload.username}`, - locals: { - user: this.payload - } - } - } -} diff --git a/server/lib/notifier/shared/instance/index.ts b/server/lib/notifier/shared/instance/index.ts deleted file mode 100644 index 8c75a8ee9..000000000 --- a/server/lib/notifier/shared/instance/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './new-peertube-version-for-admins' -export * from './new-plugin-version-for-admins' -export * from './direct-registration-for-moderators' -export * from './registration-request-for-moderators' diff --git a/server/lib/notifier/shared/instance/new-peertube-version-for-admins.ts b/server/lib/notifier/shared/instance/new-peertube-version-for-admins.ts deleted file mode 100644 index f5646c666..000000000 --- a/server/lib/notifier/shared/instance/new-peertube-version-for-admins.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MApplication, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType, UserRight } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export type NewPeerTubeVersionForAdminsPayload = { - application: MApplication - latestVersion: string -} - -export class NewPeerTubeVersionForAdmins extends AbstractNotification { - private admins: MUserDefault[] - - async prepare () { - // Use the debug right to know who is an administrator - this.admins = await UserModel.listWithRight(UserRight.MANAGE_DEBUG) - } - - log () { - logger.info('Notifying %s admins of new PeerTube version %s.', this.admins.length, this.payload.latestVersion) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.newPeerTubeVersion - } - - getTargetUsers () { - return this.admins - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.NEW_PEERTUBE_VERSION, - userId: user.id, - applicationId: this.payload.application.id - }) - notification.Application = this.payload.application - - return notification - } - - createEmail (to: string) { - return { - to, - template: 'peertube-version-new', - subject: `A new PeerTube version is available: ${this.payload.latestVersion}`, - locals: { - latestVersion: this.payload.latestVersion - } - } - } -} diff --git a/server/lib/notifier/shared/instance/new-plugin-version-for-admins.ts b/server/lib/notifier/shared/instance/new-plugin-version-for-admins.ts deleted file mode 100644 index 547c6726c..000000000 --- a/server/lib/notifier/shared/instance/new-plugin-version-for-admins.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { WEBSERVER } from '@server/initializers/constants' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MPlugin, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType, UserRight } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class NewPluginVersionForAdmins extends AbstractNotification { - private admins: MUserDefault[] - - async prepare () { - // Use the debug right to know who is an administrator - this.admins = await UserModel.listWithRight(UserRight.MANAGE_DEBUG) - } - - log () { - logger.info('Notifying %s admins of new PeerTube version %s.', this.admins.length, this.payload.latestVersion) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.newPluginVersion - } - - getTargetUsers () { - return this.admins - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.NEW_PLUGIN_VERSION, - userId: user.id, - pluginId: this.plugin.id - }) - notification.Plugin = this.plugin - - return notification - } - - createEmail (to: string) { - const pluginUrl = WEBSERVER.URL + '/admin/plugins/list-installed?pluginType=' + this.plugin.type - - return { - to, - template: 'plugin-version-new', - subject: `A new plugin/theme version is available: ${this.plugin.name}@${this.plugin.latestVersion}`, - locals: { - pluginName: this.plugin.name, - latestVersion: this.plugin.latestVersion, - pluginUrl - } - } - } - - private get plugin () { - return this.payload - } -} diff --git a/server/lib/notifier/shared/instance/registration-request-for-moderators.ts b/server/lib/notifier/shared/instance/registration-request-for-moderators.ts deleted file mode 100644 index 79920245a..000000000 --- a/server/lib/notifier/shared/instance/registration-request-for-moderators.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MRegistration, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType, UserRight } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class RegistrationRequestForModerators extends AbstractNotification { - private moderators: MUserDefault[] - - async prepare () { - this.moderators = await UserModel.listWithRight(UserRight.MANAGE_REGISTRATIONS) - } - - log () { - logger.info('Notifying %s moderators of new user registration request of %s.', this.moderators.length, this.payload.username) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.newUserRegistration - } - - getTargetUsers () { - return this.moderators - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.NEW_USER_REGISTRATION_REQUEST, - userId: user.id, - userRegistrationId: this.payload.id - }) - notification.UserRegistration = this.payload - - return notification - } - - createEmail (to: string) { - return { - template: 'user-registration-request', - to, - subject: `A new user wants to register: ${this.payload.username}`, - locals: { - registration: this.payload - } - } - } -} diff --git a/server/lib/notifier/shared/video-publication/abstract-owned-video-publication.ts b/server/lib/notifier/shared/video-publication/abstract-owned-video-publication.ts deleted file mode 100644 index a940cde69..000000000 --- a/server/lib/notifier/shared/video-publication/abstract-owned-video-publication.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { WEBSERVER } from '@server/initializers/constants' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MUserDefault, MUserWithNotificationSetting, MVideoFullLight, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export abstract class AbstractOwnedVideoPublication extends AbstractNotification { - protected user: MUserDefault - - async prepare () { - this.user = await UserModel.loadByVideoId(this.payload.id) - } - - log () { - logger.info('Notifying user %s of the publication of its video %s.', this.user.username, this.payload.url) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.myVideoPublished - } - - getTargetUsers () { - if (!this.user) return [] - - return [ this.user ] - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.MY_VIDEO_PUBLISHED, - userId: user.id, - videoId: this.payload.id - }) - notification.Video = this.payload - - return notification - } - - createEmail (to: string) { - const videoUrl = WEBSERVER.URL + this.payload.getWatchStaticPath() - - return { - to, - subject: `Your video ${this.payload.name} has been published`, - text: `Your video "${this.payload.name}" has been published.`, - locals: { - title: 'Your video is live', - action: { - text: 'View video', - url: videoUrl - } - } - } - } -} diff --git a/server/lib/notifier/shared/video-publication/import-finished-for-owner.ts b/server/lib/notifier/shared/video-publication/import-finished-for-owner.ts deleted file mode 100644 index 3bd64692f..000000000 --- a/server/lib/notifier/shared/video-publication/import-finished-for-owner.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { WEBSERVER } from '@server/initializers/constants' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MUserDefault, MUserWithNotificationSetting, MVideoImportVideo, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export type ImportFinishedForOwnerPayload = { - videoImport: MVideoImportVideo - success: boolean -} - -export class ImportFinishedForOwner extends AbstractNotification { - private user: MUserDefault - - async prepare () { - this.user = await UserModel.loadByVideoImportId(this.videoImport.id) - } - - log () { - logger.info('Notifying user %s its video import %s is finished.', this.user.username, this.videoImport.getTargetIdentifier()) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.myVideoImportFinished - } - - getTargetUsers () { - if (!this.user) return [] - - return [ this.user ] - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: this.payload.success - ? UserNotificationType.MY_VIDEO_IMPORT_SUCCESS - : UserNotificationType.MY_VIDEO_IMPORT_ERROR, - - userId: user.id, - videoImportId: this.videoImport.id - }) - notification.VideoImport = this.videoImport - - return notification - } - - createEmail (to: string) { - if (this.payload.success) return this.createSuccessEmail(to) - - return this.createFailEmail(to) - } - - private createSuccessEmail (to: string) { - const videoUrl = WEBSERVER.URL + this.videoImport.Video.getWatchStaticPath() - - return { - to, - subject: `Your video import ${this.videoImport.getTargetIdentifier()} is complete`, - text: `Your video "${this.videoImport.getTargetIdentifier()}" just finished importing.`, - locals: { - title: 'Import complete', - action: { - text: 'View video', - url: videoUrl - } - } - } - } - - private createFailEmail (to: string) { - const importUrl = WEBSERVER.URL + '/my-library/video-imports' - - const text = - `Your video import "${this.videoImport.getTargetIdentifier()}" encountered an error.` + - '\n\n' + - `See your videos import dashboard for more information: ${importUrl}.` - - return { - to, - subject: `Your video import "${this.videoImport.getTargetIdentifier()}" encountered an error`, - text, - locals: { - title: 'Import failed', - action: { - text: 'Review imports', - url: importUrl - } - } - } - } - - private get videoImport () { - return this.payload.videoImport - } -} diff --git a/server/lib/notifier/shared/video-publication/index.ts b/server/lib/notifier/shared/video-publication/index.ts deleted file mode 100644 index 5e92cb011..000000000 --- a/server/lib/notifier/shared/video-publication/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from './new-video-for-subscribers' -export * from './import-finished-for-owner' -export * from './owned-publication-after-auto-unblacklist' -export * from './owned-publication-after-schedule-update' -export * from './owned-publication-after-transcoding' -export * from './studio-edition-finished-for-owner' diff --git a/server/lib/notifier/shared/video-publication/new-video-for-subscribers.ts b/server/lib/notifier/shared/video-publication/new-video-for-subscribers.ts deleted file mode 100644 index df7a5561d..000000000 --- a/server/lib/notifier/shared/video-publication/new-video-for-subscribers.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { WEBSERVER } from '@server/initializers/constants' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MUserWithNotificationSetting, MVideoAccountLight, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType, VideoPrivacy, VideoState } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class NewVideoForSubscribers extends AbstractNotification { - private users: MUserWithNotificationSetting[] - - async prepare () { - // List all followers that are users - this.users = await UserModel.listUserSubscribersOf(this.payload.VideoChannel.actorId) - } - - log () { - logger.info('Notifying %d users of new video %s.', this.users.length, this.payload.url) - } - - isDisabled () { - return this.payload.privacy !== VideoPrivacy.PUBLIC || this.payload.state !== VideoState.PUBLISHED || this.payload.isBlacklisted() - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.newVideoFromSubscription - } - - getTargetUsers () { - return this.users - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.NEW_VIDEO_FROM_SUBSCRIPTION, - userId: user.id, - videoId: this.payload.id - }) - notification.Video = this.payload - - return notification - } - - createEmail (to: string) { - const channelName = this.payload.VideoChannel.getDisplayName() - const videoUrl = WEBSERVER.URL + this.payload.getWatchStaticPath() - - return { - to, - subject: channelName + ' just published a new video', - text: `Your subscription ${channelName} just published a new video: "${this.payload.name}".`, - locals: { - title: 'New content ', - action: { - text: 'View video', - url: videoUrl - } - } - } - } -} diff --git a/server/lib/notifier/shared/video-publication/owned-publication-after-auto-unblacklist.ts b/server/lib/notifier/shared/video-publication/owned-publication-after-auto-unblacklist.ts deleted file mode 100644 index 27d89a5c7..000000000 --- a/server/lib/notifier/shared/video-publication/owned-publication-after-auto-unblacklist.ts +++ /dev/null @@ -1,11 +0,0 @@ - -import { VideoState } from '@shared/models' -import { AbstractOwnedVideoPublication } from './abstract-owned-video-publication' - -export class OwnedPublicationAfterAutoUnblacklist extends AbstractOwnedVideoPublication { - - isDisabled () { - // Don't notify if video is still waiting for transcoding or scheduled update - return !!this.payload.ScheduleVideoUpdate || (this.payload.waitTranscoding && this.payload.state !== VideoState.PUBLISHED) - } -} diff --git a/server/lib/notifier/shared/video-publication/owned-publication-after-schedule-update.ts b/server/lib/notifier/shared/video-publication/owned-publication-after-schedule-update.ts deleted file mode 100644 index 2e253b358..000000000 --- a/server/lib/notifier/shared/video-publication/owned-publication-after-schedule-update.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { VideoState } from '@shared/models' -import { AbstractOwnedVideoPublication } from './abstract-owned-video-publication' - -export class OwnedPublicationAfterScheduleUpdate extends AbstractOwnedVideoPublication { - - isDisabled () { - // Don't notify if video is still blacklisted or waiting for transcoding - return !!this.payload.VideoBlacklist || (this.payload.waitTranscoding && this.payload.state !== VideoState.PUBLISHED) - } -} diff --git a/server/lib/notifier/shared/video-publication/owned-publication-after-transcoding.ts b/server/lib/notifier/shared/video-publication/owned-publication-after-transcoding.ts deleted file mode 100644 index 4fab1090f..000000000 --- a/server/lib/notifier/shared/video-publication/owned-publication-after-transcoding.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { AbstractOwnedVideoPublication } from './abstract-owned-video-publication' - -export class OwnedPublicationAfterTranscoding extends AbstractOwnedVideoPublication { - - isDisabled () { - // Don't notify if didn't wait for transcoding or video is still blacklisted/waiting for scheduled update - return !this.payload.waitTranscoding || !!this.payload.VideoBlacklist || !!this.payload.ScheduleVideoUpdate - } -} diff --git a/server/lib/notifier/shared/video-publication/studio-edition-finished-for-owner.ts b/server/lib/notifier/shared/video-publication/studio-edition-finished-for-owner.ts deleted file mode 100644 index f36399f05..000000000 --- a/server/lib/notifier/shared/video-publication/studio-edition-finished-for-owner.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { WEBSERVER } from '@server/initializers/constants' -import { UserModel } from '@server/models/user/user' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { MUserDefault, MUserWithNotificationSetting, MVideoFullLight, UserNotificationModelForApi } from '@server/types/models' -import { UserNotificationType } from '@shared/models' -import { AbstractNotification } from '../common/abstract-notification' - -export class StudioEditionFinishedForOwner extends AbstractNotification { - private user: MUserDefault - - async prepare () { - this.user = await UserModel.loadByVideoId(this.payload.id) - } - - log () { - logger.info('Notifying user %s its video studio edition %s is finished.', this.user.username, this.payload.url) - } - - getSetting (user: MUserWithNotificationSetting) { - return user.NotificationSetting.myVideoStudioEditionFinished - } - - getTargetUsers () { - if (!this.user) return [] - - return [ this.user ] - } - - createNotification (user: MUserWithNotificationSetting) { - const notification = UserNotificationModel.build({ - type: UserNotificationType.MY_VIDEO_STUDIO_EDITION_FINISHED, - userId: user.id, - videoId: this.payload.id - }) - notification.Video = this.payload - - return notification - } - - createEmail (to: string) { - const videoUrl = WEBSERVER.URL + this.payload.getWatchStaticPath() - - return { - to, - subject: `Edition of your video ${this.payload.name} has finished`, - text: `Edition of your video ${this.payload.name} has finished.`, - locals: { - title: 'Video edition has finished', - action: { - text: 'View video', - url: videoUrl - } - } - } - } -} diff --git a/server/lib/object-storage/index.ts b/server/lib/object-storage/index.ts deleted file mode 100644 index 3ad6cab63..000000000 --- a/server/lib/object-storage/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -export * from './keys' -export * from './proxy' -export * from './pre-signed-urls' -export * from './urls' -export * from './videos' diff --git a/server/lib/object-storage/keys.ts b/server/lib/object-storage/keys.ts deleted file mode 100644 index 6d2098298..000000000 --- a/server/lib/object-storage/keys.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { join } from 'path' -import { MStreamingPlaylistVideo } from '@server/types/models' - -function generateHLSObjectStorageKey (playlist: MStreamingPlaylistVideo, filename: string) { - return join(generateHLSObjectBaseStorageKey(playlist), filename) -} - -function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylistVideo) { - return join(playlist.getStringType(), playlist.Video.uuid) -} - -function generateWebVideoObjectStorageKey (filename: string) { - return filename -} - -export { - generateHLSObjectStorageKey, - generateHLSObjectBaseStorageKey, - generateWebVideoObjectStorageKey -} diff --git a/server/lib/object-storage/pre-signed-urls.ts b/server/lib/object-storage/pre-signed-urls.ts deleted file mode 100644 index caf149bb8..000000000 --- a/server/lib/object-storage/pre-signed-urls.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { GetObjectCommand } from '@aws-sdk/client-s3' -import { getSignedUrl } from '@aws-sdk/s3-request-presigner' -import { CONFIG } from '@server/initializers/config' -import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models' -import { generateHLSObjectStorageKey, generateWebVideoObjectStorageKey } from './keys' -import { buildKey, getClient } from './shared' -import { getHLSPublicFileUrl, getWebVideoPublicFileUrl } from './urls' - -export async function generateWebVideoPresignedUrl (options: { - file: MVideoFile - downloadFilename: string -}) { - const { file, downloadFilename } = options - - const key = generateWebVideoObjectStorageKey(file.filename) - - const command = new GetObjectCommand({ - Bucket: CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BUCKET_NAME, - Key: buildKey(key, CONFIG.OBJECT_STORAGE.WEB_VIDEOS), - ResponseContentDisposition: `attachment; filename=${downloadFilename}` - }) - - const url = await getSignedUrl(getClient(), command, { expiresIn: 3600 * 24 }) - - return getWebVideoPublicFileUrl(url) -} - -export async function generateHLSFilePresignedUrl (options: { - streamingPlaylist: MStreamingPlaylistVideo - file: MVideoFile - downloadFilename: string -}) { - const { streamingPlaylist, file, downloadFilename } = options - - const key = generateHLSObjectStorageKey(streamingPlaylist, file.filename) - - const command = new GetObjectCommand({ - Bucket: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME, - Key: buildKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS), - ResponseContentDisposition: `attachment; filename=${downloadFilename}` - }) - - const url = await getSignedUrl(getClient(), command, { expiresIn: 3600 * 24 }) - - return getHLSPublicFileUrl(url) -} diff --git a/server/lib/object-storage/proxy.ts b/server/lib/object-storage/proxy.ts deleted file mode 100644 index c09a0d1b0..000000000 --- a/server/lib/object-storage/proxy.ts +++ /dev/null @@ -1,97 +0,0 @@ -import express from 'express' -import { PassThrough, pipeline } from 'stream' -import { GetObjectCommandOutput } from '@aws-sdk/client-s3' -import { buildReinjectVideoFileTokenQuery } from '@server/controllers/shared/m3u8-playlist' -import { logger } from '@server/helpers/logger' -import { StreamReplacer } from '@server/helpers/stream-replacer' -import { MStreamingPlaylist, MVideo } from '@server/types/models' -import { HttpStatusCode } from '@shared/models' -import { injectQueryToPlaylistUrls } from '../hls' -import { getHLSFileReadStream, getWebVideoFileReadStream } from './videos' - -export async function proxifyWebVideoFile (options: { - req: express.Request - res: express.Response - filename: string -}) { - const { req, res, filename } = options - - logger.debug('Proxifying Web Video file %s from object storage.', filename) - - try { - const { response: s3Response, stream } = await getWebVideoFileReadStream({ - filename, - rangeHeader: req.header('range') - }) - - setS3Headers(res, s3Response) - - return stream.pipe(res) - } catch (err) { - return handleObjectStorageFailure(res, err) - } -} - -export async function proxifyHLS (options: { - req: express.Request - res: express.Response - playlist: MStreamingPlaylist - video: MVideo - filename: string - reinjectVideoFileToken: boolean -}) { - const { req, res, playlist, video, filename, reinjectVideoFileToken } = options - - logger.debug('Proxifying HLS file %s from object storage.', filename) - - try { - const { response: s3Response, stream } = await getHLSFileReadStream({ - playlist: playlist.withVideo(video), - filename, - rangeHeader: req.header('range') - }) - - setS3Headers(res, s3Response) - - const streamReplacer = reinjectVideoFileToken - ? new StreamReplacer(line => injectQueryToPlaylistUrls(line, buildReinjectVideoFileTokenQuery(req, filename.endsWith('master.m3u8')))) - : new PassThrough() - - return pipeline( - stream, - streamReplacer, - res, - err => { - if (!err) return - - handleObjectStorageFailure(res, err) - } - ) - } catch (err) { - return handleObjectStorageFailure(res, err) - } -} - -// --------------------------------------------------------------------------- -// Private -// --------------------------------------------------------------------------- - -function handleObjectStorageFailure (res: express.Response, err: Error) { - if (err.name === 'NoSuchKey') { - logger.debug('Could not find key in object storage to proxify private HLS video file.', { err }) - return res.sendStatus(HttpStatusCode.NOT_FOUND_404) - } - - return res.fail({ - status: HttpStatusCode.INTERNAL_SERVER_ERROR_500, - message: err.message, - type: err.name - }) -} - -function setS3Headers (res: express.Response, s3Response: GetObjectCommandOutput) { - if (s3Response.$metadata.httpStatusCode === HttpStatusCode.PARTIAL_CONTENT_206) { - res.setHeader('Content-Range', s3Response.ContentRange) - res.status(HttpStatusCode.PARTIAL_CONTENT_206) - } -} diff --git a/server/lib/object-storage/shared/client.ts b/server/lib/object-storage/shared/client.ts deleted file mode 100644 index d5cb074df..000000000 --- a/server/lib/object-storage/shared/client.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { S3Client } from '@aws-sdk/client-s3' -import { NodeHttpHandler } from '@aws-sdk/node-http-handler' -import { logger } from '@server/helpers/logger' -import { isProxyEnabled } from '@server/helpers/proxy' -import { getAgent } from '@server/helpers/requests' -import { CONFIG } from '@server/initializers/config' -import { lTags } from './logger' - -function getProxyRequestHandler () { - if (!isProxyEnabled()) return null - - const { agent } = getAgent() - - return new NodeHttpHandler({ - httpAgent: agent.http, - httpsAgent: agent.https - }) -} - -let endpointParsed: URL -function getEndpointParsed () { - if (endpointParsed) return endpointParsed - - endpointParsed = new URL(getEndpoint()) - - return endpointParsed -} - -let s3Client: S3Client -function getClient () { - if (s3Client) return s3Client - - const OBJECT_STORAGE = CONFIG.OBJECT_STORAGE - - s3Client = new S3Client({ - endpoint: getEndpoint(), - region: OBJECT_STORAGE.REGION, - credentials: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID - ? { - accessKeyId: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID, - secretAccessKey: OBJECT_STORAGE.CREDENTIALS.SECRET_ACCESS_KEY - } - : undefined, - requestHandler: getProxyRequestHandler() - }) - - logger.info('Initialized S3 client %s with region %s.', getEndpoint(), OBJECT_STORAGE.REGION, lTags()) - - return s3Client -} - -// --------------------------------------------------------------------------- - -export { - getEndpointParsed, - getClient -} - -// --------------------------------------------------------------------------- - -let endpoint: string -function getEndpoint () { - if (endpoint) return endpoint - - const endpointConfig = CONFIG.OBJECT_STORAGE.ENDPOINT - endpoint = endpointConfig.startsWith('http://') || endpointConfig.startsWith('https://') - ? CONFIG.OBJECT_STORAGE.ENDPOINT - : 'https://' + CONFIG.OBJECT_STORAGE.ENDPOINT - - return endpoint -} diff --git a/server/lib/object-storage/shared/index.ts b/server/lib/object-storage/shared/index.ts deleted file mode 100644 index 11e10aa9f..000000000 --- a/server/lib/object-storage/shared/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './client' -export * from './logger' -export * from './object-storage-helpers' diff --git a/server/lib/object-storage/shared/logger.ts b/server/lib/object-storage/shared/logger.ts deleted file mode 100644 index 8ab7cbd71..000000000 --- a/server/lib/object-storage/shared/logger.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { loggerTagsFactory } from '@server/helpers/logger' - -const lTags = loggerTagsFactory('object-storage') - -export { - lTags -} diff --git a/server/lib/object-storage/shared/object-storage-helpers.ts b/server/lib/object-storage/shared/object-storage-helpers.ts deleted file mode 100644 index 0d8878bd2..000000000 --- a/server/lib/object-storage/shared/object-storage-helpers.ts +++ /dev/null @@ -1,328 +0,0 @@ -import { map } from 'bluebird' -import { createReadStream, createWriteStream, ensureDir, ReadStream } from 'fs-extra' -import { dirname } from 'path' -import { Readable } from 'stream' -import { - _Object, - CompleteMultipartUploadCommandOutput, - DeleteObjectCommand, - GetObjectCommand, - ListObjectsV2Command, - PutObjectAclCommand, - PutObjectCommandInput, - S3Client -} from '@aws-sdk/client-s3' -import { Upload } from '@aws-sdk/lib-storage' -import { pipelinePromise } from '@server/helpers/core-utils' -import { isArray } from '@server/helpers/custom-validators/misc' -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { getInternalUrl } from '../urls' -import { getClient } from './client' -import { lTags } from './logger' - -type BucketInfo = { - BUCKET_NAME: string - PREFIX?: string -} - -async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) { - const s3Client = getClient() - - const commandPrefix = bucketInfo.PREFIX + prefix - const listCommand = new ListObjectsV2Command({ - Bucket: bucketInfo.BUCKET_NAME, - Prefix: commandPrefix - }) - - const listedObjects = await s3Client.send(listCommand) - - if (isArray(listedObjects.Contents) !== true) return [] - - return listedObjects.Contents.map(c => c.Key) -} - -// --------------------------------------------------------------------------- - -async function storeObject (options: { - inputPath: string - objectStorageKey: string - bucketInfo: BucketInfo - isPrivate: boolean -}): Promise { - const { inputPath, objectStorageKey, bucketInfo, isPrivate } = options - - logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) - - const fileStream = createReadStream(inputPath) - - return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo, isPrivate }) -} - -async function storeContent (options: { - content: string - inputPath: string - objectStorageKey: string - bucketInfo: BucketInfo - isPrivate: boolean -}): Promise { - const { content, objectStorageKey, bucketInfo, inputPath, isPrivate } = options - - logger.debug('Uploading %s content to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) - - return uploadToStorage({ objectStorageKey, content, bucketInfo, isPrivate }) -} - -// --------------------------------------------------------------------------- - -async function updateObjectACL (options: { - objectStorageKey: string - bucketInfo: BucketInfo - isPrivate: boolean -}) { - const { objectStorageKey, bucketInfo, isPrivate } = options - - const acl = getACL(isPrivate) - if (!acl) return - - const key = buildKey(objectStorageKey, bucketInfo) - - logger.debug('Updating ACL file %s in bucket %s', key, bucketInfo.BUCKET_NAME, lTags()) - - const command = new PutObjectAclCommand({ - Bucket: bucketInfo.BUCKET_NAME, - Key: key, - ACL: acl - }) - - await getClient().send(command) -} - -function updatePrefixACL (options: { - prefix: string - bucketInfo: BucketInfo - isPrivate: boolean -}) { - const { prefix, bucketInfo, isPrivate } = options - - const acl = getACL(isPrivate) - if (!acl) return - - logger.debug('Updating ACL of files in prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags()) - - return applyOnPrefix({ - prefix, - bucketInfo, - commandBuilder: obj => { - logger.debug('Updating ACL of %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags()) - - return new PutObjectAclCommand({ - Bucket: bucketInfo.BUCKET_NAME, - Key: obj.Key, - ACL: acl - }) - } - }) -} - -// --------------------------------------------------------------------------- - -function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) { - const key = buildKey(objectStorageKey, bucketInfo) - - return removeObjectByFullKey(key, bucketInfo) -} - -function removeObjectByFullKey (fullKey: string, bucketInfo: BucketInfo) { - logger.debug('Removing file %s in bucket %s', fullKey, bucketInfo.BUCKET_NAME, lTags()) - - const command = new DeleteObjectCommand({ - Bucket: bucketInfo.BUCKET_NAME, - Key: fullKey - }) - - return getClient().send(command) -} - -async function removePrefix (prefix: string, bucketInfo: BucketInfo) { - logger.debug('Removing prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags()) - - return applyOnPrefix({ - prefix, - bucketInfo, - commandBuilder: obj => { - logger.debug('Removing %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags()) - - return new DeleteObjectCommand({ - Bucket: bucketInfo.BUCKET_NAME, - Key: obj.Key - }) - } - }) -} - -// --------------------------------------------------------------------------- - -async function makeAvailable (options: { - key: string - destination: string - bucketInfo: BucketInfo -}) { - const { key, destination, bucketInfo } = options - - await ensureDir(dirname(options.destination)) - - const command = new GetObjectCommand({ - Bucket: bucketInfo.BUCKET_NAME, - Key: buildKey(key, bucketInfo) - }) - const response = await getClient().send(command) - - const file = createWriteStream(destination) - await pipelinePromise(response.Body as Readable, file) - - file.close() -} - -function buildKey (key: string, bucketInfo: BucketInfo) { - return bucketInfo.PREFIX + key -} - -// --------------------------------------------------------------------------- - -async function createObjectReadStream (options: { - key: string - bucketInfo: BucketInfo - rangeHeader: string -}) { - const { key, bucketInfo, rangeHeader } = options - - const command = new GetObjectCommand({ - Bucket: bucketInfo.BUCKET_NAME, - Key: buildKey(key, bucketInfo), - Range: rangeHeader - }) - - const response = await getClient().send(command) - - return { - response, - stream: response.Body as Readable - } -} - -// --------------------------------------------------------------------------- - -export { - BucketInfo, - buildKey, - - storeObject, - storeContent, - - removeObject, - removeObjectByFullKey, - removePrefix, - - makeAvailable, - - updateObjectACL, - updatePrefixACL, - - listKeysOfPrefix, - createObjectReadStream -} - -// --------------------------------------------------------------------------- - -async function uploadToStorage (options: { - content: ReadStream | string - objectStorageKey: string - bucketInfo: BucketInfo - isPrivate: boolean -}) { - const { content, objectStorageKey, bucketInfo, isPrivate } = options - - const input: PutObjectCommandInput = { - Body: content, - Bucket: bucketInfo.BUCKET_NAME, - Key: buildKey(objectStorageKey, bucketInfo) - } - - const acl = getACL(isPrivate) - if (acl) input.ACL = acl - - const parallelUploads3 = new Upload({ - client: getClient(), - queueSize: 4, - partSize: CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART, - - // `leavePartsOnError` must be set to `true` to avoid silently dropping failed parts - // More detailed explanation: - // https://github.com/aws/aws-sdk-js-v3/blob/v3.164.0/lib/lib-storage/src/Upload.ts#L274 - // https://github.com/aws/aws-sdk-js-v3/issues/2311#issuecomment-939413928 - leavePartsOnError: true, - params: input - }) - - const response = (await parallelUploads3.done()) as CompleteMultipartUploadCommandOutput - // Check is needed even if the HTTP status code is 200 OK - // For more information, see https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html - if (!response.Bucket) { - const message = `Error uploading ${objectStorageKey} to bucket ${bucketInfo.BUCKET_NAME}` - logger.error(message, { response, ...lTags() }) - throw new Error(message) - } - - logger.debug( - 'Completed %s%s in bucket %s', - bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, { ...lTags(), reseponseMetadata: response.$metadata } - ) - - return getInternalUrl(bucketInfo, objectStorageKey) -} - -async function applyOnPrefix (options: { - prefix: string - bucketInfo: BucketInfo - commandBuilder: (obj: _Object) => Parameters[0] - - continuationToken?: string -}) { - const { prefix, bucketInfo, commandBuilder, continuationToken } = options - - const s3Client = getClient() - - const commandPrefix = buildKey(prefix, bucketInfo) - const listCommand = new ListObjectsV2Command({ - Bucket: bucketInfo.BUCKET_NAME, - Prefix: commandPrefix, - ContinuationToken: continuationToken - }) - - const listedObjects = await s3Client.send(listCommand) - - if (isArray(listedObjects.Contents) !== true) { - const message = `Cannot apply function on ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.` - - logger.error(message, { response: listedObjects, ...lTags() }) - throw new Error(message) - } - - await map(listedObjects.Contents, object => { - const command = commandBuilder(object) - - return s3Client.send(command) - }, { concurrency: 10 }) - - // Repeat if not all objects could be listed at once (limit of 1000?) - if (listedObjects.IsTruncated) { - await applyOnPrefix({ ...options, continuationToken: listedObjects.ContinuationToken }) - } -} - -function getACL (isPrivate: boolean) { - return isPrivate - ? CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PRIVATE - : CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PUBLIC -} diff --git a/server/lib/object-storage/urls.ts b/server/lib/object-storage/urls.ts deleted file mode 100644 index 40619cd5a..000000000 --- a/server/lib/object-storage/urls.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { CONFIG } from '@server/initializers/config' -import { OBJECT_STORAGE_PROXY_PATHS, WEBSERVER } from '@server/initializers/constants' -import { MVideoUUID } from '@server/types/models' -import { BucketInfo, buildKey, getEndpointParsed } from './shared' - -function getInternalUrl (config: BucketInfo, keyWithoutPrefix: string) { - return getBaseUrl(config) + buildKey(keyWithoutPrefix, config) -} - -// --------------------------------------------------------------------------- - -function getWebVideoPublicFileUrl (fileUrl: string) { - const baseUrl = CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BASE_URL - if (!baseUrl) return fileUrl - - return replaceByBaseUrl(fileUrl, baseUrl) -} - -function getHLSPublicFileUrl (fileUrl: string) { - const baseUrl = CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BASE_URL - if (!baseUrl) return fileUrl - - return replaceByBaseUrl(fileUrl, baseUrl) -} - -// --------------------------------------------------------------------------- - -function getHLSPrivateFileUrl (video: MVideoUUID, filename: string) { - return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + video.uuid + `/${filename}` -} - -function getWebVideoPrivateFileUrl (filename: string) { - return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEB_VIDEOS + filename -} - -// --------------------------------------------------------------------------- - -export { - getInternalUrl, - - getWebVideoPublicFileUrl, - getHLSPublicFileUrl, - - getHLSPrivateFileUrl, - getWebVideoPrivateFileUrl, - - replaceByBaseUrl -} - -// --------------------------------------------------------------------------- - -function getBaseUrl (bucketInfo: BucketInfo, baseUrl?: string) { - if (baseUrl) return baseUrl - - return `${getEndpointParsed().protocol}//${bucketInfo.BUCKET_NAME}.${getEndpointParsed().host}/` -} - -const regex = new RegExp('https?://[^/]+') -function replaceByBaseUrl (fileUrl: string, baseUrl: string) { - if (!fileUrl) return fileUrl - - return fileUrl.replace(regex, baseUrl) -} diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts deleted file mode 100644 index 891e9ff76..000000000 --- a/server/lib/object-storage/videos.ts +++ /dev/null @@ -1,197 +0,0 @@ -import { basename, join } from 'path' -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/models' -import { getHLSDirectory } from '../paths' -import { VideoPathManager } from '../video-path-manager' -import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebVideoObjectStorageKey } from './keys' -import { - createObjectReadStream, - listKeysOfPrefix, - lTags, - makeAvailable, - removeObject, - removeObjectByFullKey, - removePrefix, - storeContent, - storeObject, - updateObjectACL, - updatePrefixACL -} from './shared' - -function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) { - return listKeysOfPrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) -} - -// --------------------------------------------------------------------------- - -function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename: string) { - return storeObject({ - inputPath: join(getHLSDirectory(playlist.Video), filename), - objectStorageKey: generateHLSObjectStorageKey(playlist, filename), - bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, - isPrivate: playlist.Video.hasPrivateStaticPath() - }) -} - -function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string) { - return storeObject({ - inputPath: path, - objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)), - bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, - isPrivate: playlist.Video.hasPrivateStaticPath() - }) -} - -function storeHLSFileFromContent (playlist: MStreamingPlaylistVideo, path: string, content: string) { - return storeContent({ - content, - inputPath: path, - objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)), - bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, - isPrivate: playlist.Video.hasPrivateStaticPath() - }) -} - -// --------------------------------------------------------------------------- - -function storeWebVideoFile (video: MVideo, file: MVideoFile) { - return storeObject({ - inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file), - objectStorageKey: generateWebVideoObjectStorageKey(file.filename), - bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS, - isPrivate: video.hasPrivateStaticPath() - }) -} - -// --------------------------------------------------------------------------- - -async function updateWebVideoFileACL (video: MVideo, file: MVideoFile) { - await updateObjectACL({ - objectStorageKey: generateWebVideoObjectStorageKey(file.filename), - bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS, - isPrivate: video.hasPrivateStaticPath() - }) -} - -async function updateHLSFilesACL (playlist: MStreamingPlaylistVideo) { - await updatePrefixACL({ - prefix: generateHLSObjectBaseStorageKey(playlist), - bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, - isPrivate: playlist.Video.hasPrivateStaticPath() - }) -} - -// --------------------------------------------------------------------------- - -function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) { - return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) -} - -function removeHLSFileObjectStorageByFilename (playlist: MStreamingPlaylistVideo, filename: string) { - return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) -} - -function removeHLSFileObjectStorageByPath (playlist: MStreamingPlaylistVideo, path: string) { - return removeObject(generateHLSObjectStorageKey(playlist, basename(path)), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) -} - -function removeHLSFileObjectStorageByFullKey (key: string) { - return removeObjectByFullKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) -} - -// --------------------------------------------------------------------------- - -function removeWebVideoObjectStorage (videoFile: MVideoFile) { - return removeObject(generateWebVideoObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.WEB_VIDEOS) -} - -// --------------------------------------------------------------------------- - -async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) { - const key = generateHLSObjectStorageKey(playlist, filename) - - logger.info('Fetching HLS file %s from object storage to %s.', key, destination, lTags()) - - await makeAvailable({ - key, - destination, - bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS - }) - - return destination -} - -async function makeWebVideoFileAvailable (filename: string, destination: string) { - const key = generateWebVideoObjectStorageKey(filename) - - logger.info('Fetching Web Video file %s from object storage to %s.', key, destination, lTags()) - - await makeAvailable({ - key, - destination, - bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS - }) - - return destination -} - -// --------------------------------------------------------------------------- - -function getWebVideoFileReadStream (options: { - filename: string - rangeHeader: string -}) { - const { filename, rangeHeader } = options - - const key = generateWebVideoObjectStorageKey(filename) - - return createObjectReadStream({ - key, - bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS, - rangeHeader - }) -} - -function getHLSFileReadStream (options: { - playlist: MStreamingPlaylistVideo - filename: string - rangeHeader: string -}) { - const { playlist, filename, rangeHeader } = options - - const key = generateHLSObjectStorageKey(playlist, filename) - - return createObjectReadStream({ - key, - bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, - rangeHeader - }) -} - -// --------------------------------------------------------------------------- - -export { - listHLSFileKeysOf, - - storeWebVideoFile, - storeHLSFileFromFilename, - storeHLSFileFromPath, - storeHLSFileFromContent, - - updateWebVideoFileACL, - updateHLSFilesACL, - - removeHLSObjectStorage, - removeHLSFileObjectStorageByFilename, - removeHLSFileObjectStorageByPath, - removeHLSFileObjectStorageByFullKey, - - removeWebVideoObjectStorage, - - makeWebVideoFileAvailable, - makeHLSFileAvailable, - - getWebVideoFileReadStream, - getHLSFileReadStream -} diff --git a/server/lib/opentelemetry/metric-helpers/index.ts b/server/lib/opentelemetry/metric-helpers/index.ts deleted file mode 100644 index 47b24a54f..000000000 --- a/server/lib/opentelemetry/metric-helpers/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export * from './bittorrent-tracker-observers-builder' -export * from './lives-observers-builder' -export * from './job-queue-observers-builder' -export * from './nodejs-observers-builder' -export * from './playback-metrics' -export * from './stats-observers-builder' -export * from './viewers-observers-builder' diff --git a/server/lib/opentelemetry/metric-helpers/job-queue-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/job-queue-observers-builder.ts deleted file mode 100644 index 56713ede8..000000000 --- a/server/lib/opentelemetry/metric-helpers/job-queue-observers-builder.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { Meter } from '@opentelemetry/api' -import { JobQueue } from '@server/lib/job-queue' - -export class JobQueueObserversBuilder { - - constructor (private readonly meter: Meter) { - - } - - buildObservers () { - this.meter.createObservableGauge('peertube_job_queue_total', { - description: 'Total jobs in the PeerTube job queue' - }).addCallback(async observableResult => { - const stats = await JobQueue.Instance.getStats() - - for (const { jobType, counts } of stats) { - for (const state of Object.keys(counts)) { - observableResult.observe(counts[state], { jobType, state }) - } - } - }) - } - -} diff --git a/server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts deleted file mode 100644 index 5effc18e1..000000000 --- a/server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Meter } from '@opentelemetry/api' -import { VideoModel } from '@server/models/video/video' - -export class LivesObserversBuilder { - - constructor (private readonly meter: Meter) { - - } - - buildObservers () { - this.meter.createObservableGauge('peertube_running_lives_total', { - description: 'Total running lives on the instance' - }).addCallback(async observableResult => { - const local = await VideoModel.countLives({ remote: false, mode: 'published' }) - const remote = await VideoModel.countLives({ remote: true, mode: 'published' }) - - observableResult.observe(local, { liveOrigin: 'local' }) - observableResult.observe(remote, { liveOrigin: 'remote' }) - }) - } -} diff --git a/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts deleted file mode 100644 index 8ed219e9e..000000000 --- a/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts +++ /dev/null @@ -1,202 +0,0 @@ -import { readdir } from 'fs-extra' -import { constants, NodeGCPerformanceDetail, PerformanceObserver } from 'perf_hooks' -import * as process from 'process' -import { Meter, ObservableResult } from '@opentelemetry/api' -import { ExplicitBucketHistogramAggregation } from '@opentelemetry/sdk-metrics' -import { View } from '@opentelemetry/sdk-metrics/build/src/view/View' -import { logger } from '@server/helpers/logger' - -// Thanks to https://github.com/siimon/prom-client -// We took their logic and adapter it for opentelemetry -// Try to keep consistency with their metric name/description so it's easier to process (grafana dashboard template etc) - -export class NodeJSObserversBuilder { - - constructor (private readonly meter: Meter) { - } - - static getViews () { - return [ - new View({ - aggregation: new ExplicitBucketHistogramAggregation([ 0.001, 0.01, 0.1, 1, 2, 5 ]), - instrumentName: 'nodejs_gc_duration_seconds' - }) - ] - } - - buildObservers () { - this.buildCPUObserver() - this.buildMemoryObserver() - - this.buildHandlesObserver() - this.buildFileDescriptorsObserver() - - this.buildGCObserver() - this.buildEventLoopLagObserver() - - this.buildLibUVActiveRequestsObserver() - this.buildActiveResourcesObserver() - } - - private buildCPUObserver () { - const cpuTotal = this.meter.createObservableCounter('process_cpu_seconds_total', { - description: 'Total user and system CPU time spent in seconds.' - }) - const cpuUser = this.meter.createObservableCounter('process_cpu_user_seconds_total', { - description: 'Total user CPU time spent in seconds.' - }) - const cpuSystem = this.meter.createObservableCounter('process_cpu_system_seconds_total', { - description: 'Total system CPU time spent in seconds.' - }) - - let lastCpuUsage = process.cpuUsage() - - this.meter.addBatchObservableCallback(observableResult => { - const cpuUsage = process.cpuUsage() - - const userUsageMicros = cpuUsage.user - lastCpuUsage.user - const systemUsageMicros = cpuUsage.system - lastCpuUsage.system - - lastCpuUsage = cpuUsage - - observableResult.observe(cpuTotal, (userUsageMicros + systemUsageMicros) / 1e6) - observableResult.observe(cpuUser, userUsageMicros / 1e6) - observableResult.observe(cpuSystem, systemUsageMicros / 1e6) - - }, [ cpuTotal, cpuUser, cpuSystem ]) - } - - private buildMemoryObserver () { - this.meter.createObservableGauge('nodejs_memory_usage_bytes', { - description: 'Memory' - }).addCallback(observableResult => { - const current = process.memoryUsage() - - observableResult.observe(current.heapTotal, { memoryType: 'heapTotal' }) - observableResult.observe(current.heapUsed, { memoryType: 'heapUsed' }) - observableResult.observe(current.arrayBuffers, { memoryType: 'arrayBuffers' }) - observableResult.observe(current.external, { memoryType: 'external' }) - observableResult.observe(current.rss, { memoryType: 'rss' }) - }) - } - - private buildHandlesObserver () { - if (typeof (process as any)._getActiveHandles !== 'function') return - - this.meter.createObservableGauge('nodejs_active_handles_total', { - description: 'Total number of active handles.' - }).addCallback(observableResult => { - const handles = (process as any)._getActiveHandles() - - observableResult.observe(handles.length) - }) - } - - private buildGCObserver () { - const kinds = { - [constants.NODE_PERFORMANCE_GC_MAJOR]: 'major', - [constants.NODE_PERFORMANCE_GC_MINOR]: 'minor', - [constants.NODE_PERFORMANCE_GC_INCREMENTAL]: 'incremental', - [constants.NODE_PERFORMANCE_GC_WEAKCB]: 'weakcb' - } - - const histogram = this.meter.createHistogram('nodejs_gc_duration_seconds', { - description: 'Garbage collection duration by kind, one of major, minor, incremental or weakcb' - }) - - const obs = new PerformanceObserver(list => { - const entry = list.getEntries()[0] - - // Node < 16 uses entry.kind - // Node >= 16 uses entry.detail.kind - // See: https://nodejs.org/docs/latest-v16.x/api/deprecations.html#deprecations_dep0152_extension_performanceentry_properties - const kind = entry.detail - ? kinds[(entry.detail as NodeGCPerformanceDetail).kind] - : kinds[(entry as any).kind] - - // Convert duration from milliseconds to seconds - histogram.record(entry.duration / 1000, { - kind - }) - }) - - obs.observe({ entryTypes: [ 'gc' ] }) - } - - private buildEventLoopLagObserver () { - const reportEventloopLag = (start: [ number, number ], observableResult: ObservableResult, res: () => void) => { - const delta = process.hrtime(start) - const nanosec = delta[0] * 1e9 + delta[1] - const seconds = nanosec / 1e9 - - observableResult.observe(seconds) - - res() - } - - this.meter.createObservableGauge('nodejs_eventloop_lag_seconds', { - description: 'Lag of event loop in seconds.' - }).addCallback(observableResult => { - return new Promise(res => { - const start = process.hrtime() - - setImmediate(reportEventloopLag, start, observableResult, res) - }) - }) - } - - private buildFileDescriptorsObserver () { - this.meter.createObservableGauge('process_open_fds', { - description: 'Number of open file descriptors.' - }).addCallback(async observableResult => { - try { - const fds = await readdir('/proc/self/fd') - observableResult.observe(fds.length - 1) - } catch (err) { - logger.debug('Cannot list file descriptors of current process for OpenTelemetry.', { err }) - } - }) - } - - private buildLibUVActiveRequestsObserver () { - if (typeof (process as any)._getActiveRequests !== 'function') return - - this.meter.createObservableGauge('nodejs_active_requests_total', { - description: 'Total number of active libuv requests.' - }).addCallback(observableResult => { - const requests = (process as any)._getActiveRequests() - - observableResult.observe(requests.length) - }) - } - - private buildActiveResourcesObserver () { - if (typeof (process as any).getActiveResourcesInfo !== 'function') return - - const grouped = this.meter.createObservableCounter('nodejs_active_resources', { - description: 'Number of active resources that are currently keeping the event loop alive, grouped by async resource type.' - }) - const total = this.meter.createObservableCounter('nodejs_active_resources_total', { - description: 'Total number of active resources.' - }) - - this.meter.addBatchObservableCallback(observableResult => { - const resources = (process as any).getActiveResourcesInfo() - - const data = {} - - for (let i = 0; i < resources.length; i++) { - const resource = resources[i] - - if (data[resource] === undefined) data[resource] = 0 - data[resource] += 1 - } - - for (const type of Object.keys(data)) { - observableResult.observe(grouped, data[type], { type }) - } - - observableResult.observe(total, resources.length) - }, [ grouped, total ]) - } -} diff --git a/server/lib/opentelemetry/metric-helpers/playback-metrics.ts b/server/lib/opentelemetry/metric-helpers/playback-metrics.ts deleted file mode 100644 index 1eb08b5a6..000000000 --- a/server/lib/opentelemetry/metric-helpers/playback-metrics.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { Counter, Meter } from '@opentelemetry/api' -import { MVideoImmutable } from '@server/types/models' -import { PlaybackMetricCreate } from '@shared/models' - -export class PlaybackMetrics { - private errorsCounter: Counter - private resolutionChangesCounter: Counter - - private downloadedBytesP2PCounter: Counter - private uploadedBytesP2PCounter: Counter - - private downloadedBytesHTTPCounter: Counter - - private peersP2PPeersGaugeBuffer: { - value: number - attributes: any - }[] = [] - - constructor (private readonly meter: Meter) { - - } - - buildCounters () { - this.errorsCounter = this.meter.createCounter('peertube_playback_errors_count', { - description: 'Errors collected from PeerTube player.' - }) - - this.resolutionChangesCounter = this.meter.createCounter('peertube_playback_resolution_changes_count', { - description: 'Resolution changes collected from PeerTube player.' - }) - - this.downloadedBytesHTTPCounter = this.meter.createCounter('peertube_playback_http_downloaded_bytes', { - description: 'Downloaded bytes with HTTP by PeerTube player.' - }) - this.downloadedBytesP2PCounter = this.meter.createCounter('peertube_playback_p2p_downloaded_bytes', { - description: 'Downloaded bytes with P2P by PeerTube player.' - }) - - this.uploadedBytesP2PCounter = this.meter.createCounter('peertube_playback_p2p_uploaded_bytes', { - description: 'Uploaded bytes with P2P by PeerTube player.' - }) - - this.meter.createObservableGauge('peertube_playback_p2p_peers', { - description: 'Total P2P peers connected to the PeerTube player.' - }).addCallback(observableResult => { - for (const gauge of this.peersP2PPeersGaugeBuffer) { - observableResult.observe(gauge.value, gauge.attributes) - } - - this.peersP2PPeersGaugeBuffer = [] - }) - } - - observe (video: MVideoImmutable, metrics: PlaybackMetricCreate) { - const attributes = { - videoOrigin: video.remote - ? 'remote' - : 'local', - - playerMode: metrics.playerMode, - - resolution: metrics.resolution + '', - fps: metrics.fps + '', - - p2pEnabled: metrics.p2pEnabled, - - videoUUID: video.uuid - } - - this.errorsCounter.add(metrics.errors, attributes) - this.resolutionChangesCounter.add(metrics.resolutionChanges, attributes) - - this.downloadedBytesHTTPCounter.add(metrics.downloadedBytesHTTP, attributes) - this.downloadedBytesP2PCounter.add(metrics.downloadedBytesP2P, attributes) - - this.uploadedBytesP2PCounter.add(metrics.uploadedBytesP2P, attributes) - - if (metrics.p2pPeers) { - this.peersP2PPeersGaugeBuffer.push({ - value: metrics.p2pPeers, - attributes - }) - } - } -} diff --git a/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts deleted file mode 100644 index 9f5f22e1b..000000000 --- a/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts +++ /dev/null @@ -1,186 +0,0 @@ -import memoizee from 'memoizee' -import { Meter } from '@opentelemetry/api' -import { MEMOIZE_TTL } from '@server/initializers/constants' -import { buildAvailableActivities } from '@server/lib/activitypub/activity' -import { StatsManager } from '@server/lib/stat-manager' - -export class StatsObserversBuilder { - - private readonly getInstanceStats = memoizee(() => { - return StatsManager.Instance.getStats() - }, { maxAge: MEMOIZE_TTL.GET_STATS_FOR_OPEN_TELEMETRY_METRICS }) - - constructor (private readonly meter: Meter) { - - } - - buildObservers () { - this.buildUserStatsObserver() - this.buildVideoStatsObserver() - this.buildCommentStatsObserver() - this.buildPlaylistStatsObserver() - this.buildChannelStatsObserver() - this.buildInstanceFollowsStatsObserver() - this.buildRedundancyStatsObserver() - this.buildActivityPubStatsObserver() - } - - private buildUserStatsObserver () { - this.meter.createObservableGauge('peertube_users_total', { - description: 'Total users on the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalUsers) - }) - - this.meter.createObservableGauge('peertube_active_users_total', { - description: 'Total active users on the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalDailyActiveUsers, { activeInterval: 'daily' }) - observableResult.observe(stats.totalWeeklyActiveUsers, { activeInterval: 'weekly' }) - observableResult.observe(stats.totalMonthlyActiveUsers, { activeInterval: 'monthly' }) - }) - } - - private buildChannelStatsObserver () { - this.meter.createObservableGauge('peertube_channels_total', { - description: 'Total channels on the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalLocalVideoChannels, { channelOrigin: 'local' }) - }) - - this.meter.createObservableGauge('peertube_active_channels_total', { - description: 'Total active channels on the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalLocalDailyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'daily' }) - observableResult.observe(stats.totalLocalWeeklyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'weekly' }) - observableResult.observe(stats.totalLocalMonthlyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'monthly' }) - }) - } - - private buildVideoStatsObserver () { - this.meter.createObservableGauge('peertube_videos_total', { - description: 'Total videos on the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalLocalVideos, { videoOrigin: 'local' }) - observableResult.observe(stats.totalVideos - stats.totalLocalVideos, { videoOrigin: 'remote' }) - }) - - this.meter.createObservableGauge('peertube_video_views_total', { - description: 'Total video views made on the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalLocalVideoViews, { viewOrigin: 'local' }) - }) - - this.meter.createObservableGauge('peertube_video_bytes_total', { - description: 'Total bytes of videos' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalLocalVideoFilesSize, { videoOrigin: 'local' }) - }) - } - - private buildCommentStatsObserver () { - this.meter.createObservableGauge('peertube_comments_total', { - description: 'Total comments on the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalLocalVideoComments, { accountOrigin: 'local' }) - }) - } - - private buildPlaylistStatsObserver () { - this.meter.createObservableGauge('peertube_playlists_total', { - description: 'Total playlists on the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalLocalPlaylists, { playlistOrigin: 'local' }) - }) - } - - private buildInstanceFollowsStatsObserver () { - this.meter.createObservableGauge('peertube_instance_followers_total', { - description: 'Total followers of the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalInstanceFollowers) - }) - - this.meter.createObservableGauge('peertube_instance_following_total', { - description: 'Total following of the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalInstanceFollowing) - }) - } - - private buildRedundancyStatsObserver () { - this.meter.createObservableGauge('peertube_redundancy_used_bytes_total', { - description: 'Total redundancy used of the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - for (const r of stats.videosRedundancy) { - observableResult.observe(r.totalUsed, { strategy: r.strategy }) - } - }) - - this.meter.createObservableGauge('peertube_redundancy_available_bytes_total', { - description: 'Total redundancy available of the instance' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - for (const r of stats.videosRedundancy) { - observableResult.observe(r.totalSize, { strategy: r.strategy }) - } - }) - } - - private buildActivityPubStatsObserver () { - const availableActivities = buildAvailableActivities() - - this.meter.createObservableGauge('peertube_ap_inbox_success_total', { - description: 'Total inbox messages processed with success' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - for (const type of availableActivities) { - observableResult.observe(stats[`totalActivityPub${type}MessagesSuccesses`], { activityType: type }) - } - }) - - this.meter.createObservableGauge('peertube_ap_inbox_error_total', { - description: 'Total inbox messages processed with error' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - for (const type of availableActivities) { - observableResult.observe(stats[`totalActivityPub${type}MessagesErrors`], { activityType: type }) - } - }) - - this.meter.createObservableGauge('peertube_ap_inbox_waiting_total', { - description: 'Total inbox messages waiting for being processed' - }).addCallback(async observableResult => { - const stats = await this.getInstanceStats() - - observableResult.observe(stats.totalActivityPubMessagesWaiting) - }) - } -} diff --git a/server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts b/server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts deleted file mode 100644 index c65f8ddae..000000000 --- a/server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { Meter } from '@opentelemetry/api' -import { VideoScope, ViewerScope } from '@server/lib/views/shared' -import { VideoViewsManager } from '@server/lib/views/video-views-manager' - -export class ViewersObserversBuilder { - - constructor (private readonly meter: Meter) { - - } - - buildObservers () { - this.meter.createObservableGauge('peertube_viewers_total', { - description: 'Total viewers on the instance' - }).addCallback(observableResult => { - for (const viewerScope of [ 'local', 'remote' ] as ViewerScope[]) { - for (const videoScope of [ 'local', 'remote' ] as VideoScope[]) { - const result = VideoViewsManager.Instance.getTotalViewers({ viewerScope, videoScope }) - - observableResult.observe(result, { viewerOrigin: viewerScope, videoOrigin: videoScope }) - } - } - }) - } -} diff --git a/server/lib/opentelemetry/metrics.ts b/server/lib/opentelemetry/metrics.ts deleted file mode 100644 index bffe00840..000000000 --- a/server/lib/opentelemetry/metrics.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { Application, Request, Response } from 'express' -import { Meter, metrics } from '@opentelemetry/api' -import { PrometheusExporter } from '@opentelemetry/exporter-prometheus' -import { MeterProvider } from '@opentelemetry/sdk-metrics' -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { MVideoImmutable } from '@server/types/models' -import { PlaybackMetricCreate } from '@shared/models' -import { - BittorrentTrackerObserversBuilder, - JobQueueObserversBuilder, - LivesObserversBuilder, - NodeJSObserversBuilder, - PlaybackMetrics, - StatsObserversBuilder, - ViewersObserversBuilder -} from './metric-helpers' - -class OpenTelemetryMetrics { - - private static instance: OpenTelemetryMetrics - - private meter: Meter - - private onRequestDuration: (req: Request, res: Response) => void - - private playbackMetrics: PlaybackMetrics - - private constructor () {} - - init (app: Application) { - if (CONFIG.OPEN_TELEMETRY.METRICS.ENABLED !== true) return - - app.use((req, res, next) => { - res.once('finish', () => { - if (!this.onRequestDuration) return - - this.onRequestDuration(req as Request, res as Response) - }) - - next() - }) - } - - registerMetrics (options: { trackerServer: any }) { - if (CONFIG.OPEN_TELEMETRY.METRICS.ENABLED !== true) return - - logger.info('Registering Open Telemetry metrics') - - const provider = new MeterProvider({ - views: [ - ...NodeJSObserversBuilder.getViews() - ] - }) - - provider.addMetricReader(new PrometheusExporter({ - host: CONFIG.OPEN_TELEMETRY.METRICS.PROMETHEUS_EXPORTER.HOSTNAME, - port: CONFIG.OPEN_TELEMETRY.METRICS.PROMETHEUS_EXPORTER.PORT - })) - - metrics.setGlobalMeterProvider(provider) - - this.meter = metrics.getMeter('default') - - if (CONFIG.OPEN_TELEMETRY.METRICS.HTTP_REQUEST_DURATION.ENABLED === true) { - this.buildRequestObserver() - } - - this.playbackMetrics = new PlaybackMetrics(this.meter) - this.playbackMetrics.buildCounters() - - const nodeJSObserversBuilder = new NodeJSObserversBuilder(this.meter) - nodeJSObserversBuilder.buildObservers() - - const jobQueueObserversBuilder = new JobQueueObserversBuilder(this.meter) - jobQueueObserversBuilder.buildObservers() - - const statsObserversBuilder = new StatsObserversBuilder(this.meter) - statsObserversBuilder.buildObservers() - - const livesObserversBuilder = new LivesObserversBuilder(this.meter) - livesObserversBuilder.buildObservers() - - const viewersObserversBuilder = new ViewersObserversBuilder(this.meter) - viewersObserversBuilder.buildObservers() - - const bittorrentTrackerObserversBuilder = new BittorrentTrackerObserversBuilder(this.meter, options.trackerServer) - bittorrentTrackerObserversBuilder.buildObservers() - } - - observePlaybackMetric (video: MVideoImmutable, metrics: PlaybackMetricCreate) { - this.playbackMetrics.observe(video, metrics) - } - - private buildRequestObserver () { - const requestDuration = this.meter.createHistogram('http_request_duration_ms', { - unit: 'milliseconds', - description: 'Duration of HTTP requests in ms' - }) - - this.onRequestDuration = (req: Request, res: Response) => { - const duration = Date.now() - res.locals.requestStart - - requestDuration.record(duration, { - path: this.buildRequestPath(req.originalUrl), - method: req.method, - statusCode: res.statusCode + '' - }) - } - } - - private buildRequestPath (path: string) { - return path.split('?')[0] - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -export { - OpenTelemetryMetrics -} diff --git a/server/lib/opentelemetry/tracing.ts b/server/lib/opentelemetry/tracing.ts deleted file mode 100644 index 9a81680b2..000000000 --- a/server/lib/opentelemetry/tracing.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { SequelizeInstrumentation } from 'opentelemetry-instrumentation-sequelize' -import { context, diag, DiagLogLevel, trace } from '@opentelemetry/api' -import { JaegerExporter } from '@opentelemetry/exporter-jaeger' -import { registerInstrumentations } from '@opentelemetry/instrumentation' -import { DnsInstrumentation } from '@opentelemetry/instrumentation-dns' -import { ExpressInstrumentation } from '@opentelemetry/instrumentation-express' -import FsInstrumentation from '@opentelemetry/instrumentation-fs' -import { HttpInstrumentation } from '@opentelemetry/instrumentation-http' -import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis' -import { PgInstrumentation } from '@opentelemetry/instrumentation-pg' -import { Resource } from '@opentelemetry/resources' -import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base' -import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node' -import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions' -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' - -const tracer = trace.getTracer('peertube') - -function registerOpentelemetryTracing () { - if (CONFIG.OPEN_TELEMETRY.TRACING.ENABLED !== true) return - - logger.info('Registering Open Telemetry tracing') - - const customLogger = (level: string) => { - return (message: string, ...args: unknown[]) => { - let fullMessage = message - - for (const arg of args) { - if (typeof arg === 'string') fullMessage += arg - else break - } - - logger[level](fullMessage) - } - } - - diag.setLogger({ - error: customLogger('error'), - warn: customLogger('warn'), - info: customLogger('info'), - debug: customLogger('debug'), - verbose: customLogger('verbose') - }, DiagLogLevel.INFO) - - const tracerProvider = new NodeTracerProvider({ - resource: new Resource({ - [SemanticResourceAttributes.SERVICE_NAME]: 'peertube' - }) - }) - - registerInstrumentations({ - tracerProvider, - instrumentations: [ - new PgInstrumentation({ - enhancedDatabaseReporting: true - }), - new DnsInstrumentation(), - new HttpInstrumentation(), - new ExpressInstrumentation(), - new IORedisInstrumentation({ - dbStatementSerializer: function (cmdName, cmdArgs) { - return [ cmdName, ...cmdArgs ].join(' ') - } - }), - new FsInstrumentation(), - new SequelizeInstrumentation() - ] - }) - - tracerProvider.addSpanProcessor( - new BatchSpanProcessor( - new JaegerExporter({ endpoint: CONFIG.OPEN_TELEMETRY.TRACING.JAEGER_EXPORTER.ENDPOINT }) - ) - ) - - tracerProvider.register() -} - -async function wrapWithSpanAndContext (spanName: string, cb: () => Promise) { - const span = tracer.startSpan(spanName) - const activeContext = trace.setSpan(context.active(), span) - - const result = await context.with(activeContext, () => cb()) - span.end() - - return result -} - -export { - registerOpentelemetryTracing, - tracer, - wrapWithSpanAndContext -} diff --git a/server/lib/paths.ts b/server/lib/paths.ts deleted file mode 100644 index db1cdede2..000000000 --- a/server/lib/paths.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { join } from 'path' -import { CONFIG } from '@server/initializers/config' -import { DIRECTORIES, VIDEO_LIVE } from '@server/initializers/constants' -import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' -import { removeFragmentedMP4Ext } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { isVideoInPrivateDirectory } from './video-privacy' - -// ################## Video file name ################## - -function generateWebVideoFilename (resolution: number, extname: string) { - return buildUUID() + '-' + resolution + extname -} - -function generateHLSVideoFilename (resolution: number) { - return `${buildUUID()}-${resolution}-fragmented.mp4` -} - -// ################## Streaming playlist ################## - -function getLiveDirectory (video: MVideo) { - return getHLSDirectory(video) -} - -function getLiveReplayBaseDirectory (video: MVideo) { - return join(getLiveDirectory(video), VIDEO_LIVE.REPLAY_DIRECTORY) -} - -function getHLSDirectory (video: MVideo) { - if (isVideoInPrivateDirectory(video.privacy)) { - return join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, video.uuid) - } - - return join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, video.uuid) -} - -function getHLSRedundancyDirectory (video: MVideoUUID) { - return join(DIRECTORIES.HLS_REDUNDANCY, video.uuid) -} - -function getHlsResolutionPlaylistFilename (videoFilename: string) { - // Video file name already contain resolution - return removeFragmentedMP4Ext(videoFilename) + '.m3u8' -} - -function generateHLSMasterPlaylistFilename (isLive = false) { - if (isLive) return 'master.m3u8' - - return buildUUID() + '-master.m3u8' -} - -function generateHlsSha256SegmentsFilename (isLive = false) { - if (isLive) return 'segments-sha256.json' - - return buildUUID() + '-segments-sha256.json' -} - -// ################## Torrents ################## - -function generateTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, resolution: number) { - const extension = '.torrent' - const uuid = buildUUID() - - if (isStreamingPlaylist(videoOrPlaylist)) { - return `${uuid}-${resolution}-${videoOrPlaylist.getStringType()}${extension}` - } - - return uuid + '-' + resolution + extension -} - -function getFSTorrentFilePath (videoFile: MVideoFile) { - return join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) -} - -// --------------------------------------------------------------------------- - -export { - generateHLSVideoFilename, - generateWebVideoFilename, - - generateTorrentFileName, - getFSTorrentFilePath, - - getHLSDirectory, - getLiveDirectory, - getLiveReplayBaseDirectory, - getHLSRedundancyDirectory, - - generateHLSMasterPlaylistFilename, - generateHlsSha256SegmentsFilename, - getHlsResolutionPlaylistFilename -} diff --git a/server/lib/peertube-socket.ts b/server/lib/peertube-socket.ts deleted file mode 100644 index 3e41a2def..000000000 --- a/server/lib/peertube-socket.ts +++ /dev/null @@ -1,129 +0,0 @@ -import { Server as HTTPServer } from 'http' -import { Namespace, Server as SocketServer, Socket } from 'socket.io' -import { isIdValid } from '@server/helpers/custom-validators/misc' -import { Debounce } from '@server/helpers/debounce' -import { MVideo, MVideoImmutable } from '@server/types/models' -import { MRunner } from '@server/types/models/runners' -import { UserNotificationModelForApi } from '@server/types/models/user' -import { LiveVideoEventPayload, LiveVideoEventType } from '@shared/models' -import { logger } from '../helpers/logger' -import { authenticateRunnerSocket, authenticateSocket } from '../middlewares' - -class PeerTubeSocket { - - private static instance: PeerTubeSocket - - private userNotificationSockets: { [ userId: number ]: Socket[] } = {} - private liveVideosNamespace: Namespace - private readonly runnerSockets = new Set() - - private constructor () {} - - init (server: HTTPServer) { - const io = new SocketServer(server) - - io.of('/user-notifications') - .use(authenticateSocket) - .on('connection', socket => { - const userId = socket.handshake.auth.user.id - - logger.debug('User %d connected to the notification system.', userId) - - if (!this.userNotificationSockets[userId]) this.userNotificationSockets[userId] = [] - - this.userNotificationSockets[userId].push(socket) - - socket.on('disconnect', () => { - logger.debug('User %d disconnected from SocketIO notifications.', userId) - - this.userNotificationSockets[userId] = this.userNotificationSockets[userId].filter(s => s !== socket) - }) - }) - - this.liveVideosNamespace = io.of('/live-videos') - .on('connection', socket => { - socket.on('subscribe', ({ videoId }) => { - if (!isIdValid(videoId)) return - - /* eslint-disable @typescript-eslint/no-floating-promises */ - socket.join(videoId) - }) - - socket.on('unsubscribe', ({ videoId }) => { - if (!isIdValid(videoId)) return - - /* eslint-disable @typescript-eslint/no-floating-promises */ - socket.leave(videoId) - }) - }) - - io.of('/runners') - .use(authenticateRunnerSocket) - .on('connection', socket => { - const runner: MRunner = socket.handshake.auth.runner - - logger.debug(`New runner "${runner.name}" connected to the notification system.`) - - this.runnerSockets.add(socket) - - socket.on('disconnect', () => { - logger.debug(`Runner "${runner.name}" disconnected from the notification system.`) - - this.runnerSockets.delete(socket) - }) - }) - } - - sendNotification (userId: number, notification: UserNotificationModelForApi) { - const sockets = this.userNotificationSockets[userId] - if (!sockets) return - - logger.debug('Sending user notification to user %d.', userId) - - const notificationMessage = notification.toFormattedJSON() - for (const socket of sockets) { - socket.emit('new-notification', notificationMessage) - } - } - - sendVideoLiveNewState (video: MVideo) { - const data: LiveVideoEventPayload = { state: video.state } - const type: LiveVideoEventType = 'state-change' - - logger.debug('Sending video live new state notification of %s.', video.url, { state: video.state }) - - this.liveVideosNamespace - .in(video.id) - .emit(type, data) - } - - sendVideoViewsUpdate (video: MVideoImmutable, numViewers: number) { - const data: LiveVideoEventPayload = { viewers: numViewers } - const type: LiveVideoEventType = 'views-change' - - logger.debug('Sending video live views update notification of %s.', video.url, { viewers: numViewers }) - - this.liveVideosNamespace - .in(video.id) - .emit(type, data) - } - - @Debounce({ timeoutMS: 1000 }) - sendAvailableJobsPingToRunners () { - logger.debug(`Sending available-jobs notification to ${this.runnerSockets.size} runner sockets`) - - for (const runners of this.runnerSockets) { - runners.emit('available-jobs') - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -// --------------------------------------------------------------------------- - -export { - PeerTubeSocket -} diff --git a/server/lib/plugins/hooks.ts b/server/lib/plugins/hooks.ts deleted file mode 100644 index 694527c12..000000000 --- a/server/lib/plugins/hooks.ts +++ /dev/null @@ -1,35 +0,0 @@ -import Bluebird from 'bluebird' -import { ServerActionHookName, ServerFilterHookName } from '../../../shared/models' -import { logger } from '../../helpers/logger' -import { PluginManager } from './plugin-manager' - -type PromiseFunction = (params: U) => Promise | Bluebird -type RawFunction = (params: U) => T - -// Helpers to run hooks -const Hooks = { - wrapObject: (result: T, hookName: U, context?: any) => { - return PluginManager.Instance.runHook(hookName, result, context) - }, - - wrapPromiseFun: async (fun: PromiseFunction, params: U, hookName: V) => { - const result = await fun(params) - - return PluginManager.Instance.runHook(hookName, result, params) - }, - - wrapFun: async (fun: RawFunction, params: U, hookName: V) => { - const result = fun(params) - - return PluginManager.Instance.runHook(hookName, result, params) - }, - - runAction: (hookName: U, params?: T) => { - PluginManager.Instance.runHook(hookName, undefined, params) - .catch(err => logger.error('Fatal hook error.', { err })) - } -} - -export { - Hooks -} diff --git a/server/lib/plugins/plugin-helpers-builder.ts b/server/lib/plugins/plugin-helpers-builder.ts deleted file mode 100644 index b4e3eece4..000000000 --- a/server/lib/plugins/plugin-helpers-builder.ts +++ /dev/null @@ -1,262 +0,0 @@ -import express from 'express' -import { Server } from 'http' -import { join } from 'path' -import { buildLogger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { WEBSERVER } from '@server/initializers/constants' -import { sequelizeTypescript } from '@server/initializers/database' -import { AccountModel } from '@server/models/account/account' -import { AccountBlocklistModel } from '@server/models/account/account-blocklist' -import { getServerActor } from '@server/models/application/application' -import { ServerModel } from '@server/models/server/server' -import { ServerBlocklistModel } from '@server/models/server/server-blocklist' -import { UserModel } from '@server/models/user/user' -import { VideoModel } from '@server/models/video/video' -import { VideoBlacklistModel } from '@server/models/video/video-blacklist' -import { MPlugin, MVideo, UserNotificationModelForApi } from '@server/types/models' -import { PeerTubeHelpers } from '@server/types/plugins' -import { ffprobePromise } from '@shared/ffmpeg' -import { VideoBlacklistCreate, VideoStorage } from '@shared/models' -import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist' -import { PeerTubeSocket } from '../peertube-socket' -import { ServerConfigManager } from '../server-config-manager' -import { blacklistVideo, unblacklistVideo } from '../video-blacklist' -import { VideoPathManager } from '../video-path-manager' - -function buildPluginHelpers (httpServer: Server, pluginModel: MPlugin, npmName: string): PeerTubeHelpers { - const logger = buildPluginLogger(npmName) - - const database = buildDatabaseHelpers() - const videos = buildVideosHelpers() - - const config = buildConfigHelpers() - - const server = buildServerHelpers(httpServer) - - const moderation = buildModerationHelpers() - - const plugin = buildPluginRelatedHelpers(pluginModel, npmName) - - const socket = buildSocketHelpers() - - const user = buildUserHelpers() - - return { - logger, - database, - videos, - config, - moderation, - plugin, - server, - socket, - user - } -} - -export { - buildPluginHelpers -} - -// --------------------------------------------------------------------------- - -function buildPluginLogger (npmName: string) { - return buildLogger(npmName) -} - -function buildDatabaseHelpers () { - return { - query: sequelizeTypescript.query.bind(sequelizeTypescript) - } -} - -function buildServerHelpers (httpServer: Server) { - return { - getHTTPServer: () => httpServer, - - getServerActor: () => getServerActor() - } -} - -function buildVideosHelpers () { - return { - loadByUrl: (url: string) => { - return VideoModel.loadByUrl(url) - }, - - loadByIdOrUUID: (id: number | string) => { - return VideoModel.load(id) - }, - - removeVideo: (id: number) => { - return sequelizeTypescript.transaction(async t => { - const video = await VideoModel.loadFull(id, t) - - await video.destroy({ transaction: t }) - }) - }, - - ffprobe: (path: string) => { - return ffprobePromise(path) - }, - - getFiles: async (id: number | string) => { - const video = await VideoModel.loadFull(id) - if (!video) return undefined - - const webVideoFiles = (video.VideoFiles || []).map(f => ({ - path: f.storage === VideoStorage.FILE_SYSTEM - ? VideoPathManager.Instance.getFSVideoFileOutputPath(video, f) - : null, - url: f.getFileUrl(video), - - resolution: f.resolution, - size: f.size, - fps: f.fps - })) - - const hls = video.getHLSPlaylist() - - const hlsVideoFiles = hls - ? (video.getHLSPlaylist().VideoFiles || []).map(f => { - return { - path: f.storage === VideoStorage.FILE_SYSTEM - ? VideoPathManager.Instance.getFSVideoFileOutputPath(hls, f) - : null, - url: f.getFileUrl(video), - resolution: f.resolution, - size: f.size, - fps: f.fps - } - }) - : [] - - const thumbnails = video.Thumbnails.map(t => ({ - type: t.type, - url: t.getOriginFileUrl(video), - path: t.getPath() - })) - - return { - webtorrent: { // TODO: remove in v7 - videoFiles: webVideoFiles - }, - - webVideo: { - videoFiles: webVideoFiles - }, - - hls: { - videoFiles: hlsVideoFiles - }, - - thumbnails - } - } - } -} - -function buildModerationHelpers () { - return { - blockServer: async (options: { byAccountId: number, hostToBlock: string }) => { - const serverToBlock = await ServerModel.loadOrCreateByHost(options.hostToBlock) - - await addServerInBlocklist(options.byAccountId, serverToBlock.id) - }, - - unblockServer: async (options: { byAccountId: number, hostToUnblock: string }) => { - const serverBlock = await ServerBlocklistModel.loadByAccountAndHost(options.byAccountId, options.hostToUnblock) - if (!serverBlock) return - - await removeServerFromBlocklist(serverBlock) - }, - - blockAccount: async (options: { byAccountId: number, handleToBlock: string }) => { - const accountToBlock = await AccountModel.loadByNameWithHost(options.handleToBlock) - if (!accountToBlock) return - - await addAccountInBlocklist(options.byAccountId, accountToBlock.id) - }, - - unblockAccount: async (options: { byAccountId: number, handleToUnblock: string }) => { - const targetAccount = await AccountModel.loadByNameWithHost(options.handleToUnblock) - if (!targetAccount) return - - const accountBlock = await AccountBlocklistModel.loadByAccountAndTarget(options.byAccountId, targetAccount.id) - if (!accountBlock) return - - await removeAccountFromBlocklist(accountBlock) - }, - - blacklistVideo: async (options: { videoIdOrUUID: number | string, createOptions: VideoBlacklistCreate }) => { - const video = await VideoModel.loadFull(options.videoIdOrUUID) - if (!video) return - - await blacklistVideo(video, options.createOptions) - }, - - unblacklistVideo: async (options: { videoIdOrUUID: number | string }) => { - const video = await VideoModel.loadFull(options.videoIdOrUUID) - if (!video) return - - const videoBlacklist = await VideoBlacklistModel.loadByVideoId(video.id) - if (!videoBlacklist) return - - await unblacklistVideo(videoBlacklist, video) - } - } -} - -function buildConfigHelpers () { - return { - getWebserverUrl () { - return WEBSERVER.URL - }, - - getServerListeningConfig () { - return { hostname: CONFIG.LISTEN.HOSTNAME, port: CONFIG.LISTEN.PORT } - }, - - getServerConfig () { - return ServerConfigManager.Instance.getServerConfig() - } - } -} - -function buildPluginRelatedHelpers (plugin: MPlugin, npmName: string) { - return { - getBaseStaticRoute: () => `/plugins/${plugin.name}/${plugin.version}/static/`, - - getBaseRouterRoute: () => `/plugins/${plugin.name}/${plugin.version}/router/`, - - getBaseWebSocketRoute: () => `/plugins/${plugin.name}/${plugin.version}/ws/`, - - getDataDirectoryPath: () => join(CONFIG.STORAGE.PLUGINS_DIR, 'data', npmName) - } -} - -function buildSocketHelpers () { - return { - sendNotification: (userId: number, notification: UserNotificationModelForApi) => { - PeerTubeSocket.Instance.sendNotification(userId, notification) - }, - sendVideoLiveNewState: (video: MVideo) => { - PeerTubeSocket.Instance.sendVideoLiveNewState(video) - } - } -} - -function buildUserHelpers () { - return { - loadById: (id: number) => { - return UserModel.loadByIdFull(id) - }, - - getAuthUser: (res: express.Response) => { - const user = res.locals.oauth?.token?.User || res.locals.videoFileToken?.user - if (!user) return undefined - - return UserModel.loadByIdFull(user.id) - } - } -} diff --git a/server/lib/plugins/plugin-index.ts b/server/lib/plugins/plugin-index.ts deleted file mode 100644 index 119cee8e0..000000000 --- a/server/lib/plugins/plugin-index.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { sanitizeUrl } from '@server/helpers/core-utils' -import { logger } from '@server/helpers/logger' -import { doJSONRequest } from '@server/helpers/requests' -import { CONFIG } from '@server/initializers/config' -import { PEERTUBE_VERSION } from '@server/initializers/constants' -import { PluginModel } from '@server/models/server/plugin' -import { - PeerTubePluginIndex, - PeertubePluginIndexList, - PeertubePluginLatestVersionRequest, - PeertubePluginLatestVersionResponse, - ResultList -} from '@shared/models' -import { PluginManager } from './plugin-manager' - -async function listAvailablePluginsFromIndex (options: PeertubePluginIndexList) { - const { start = 0, count = 20, search, sort = 'npmName', pluginType } = options - - const searchParams: PeertubePluginIndexList & Record = { - start, - count, - sort, - pluginType, - search, - currentPeerTubeEngine: options.currentPeerTubeEngine || PEERTUBE_VERSION - } - - const uri = CONFIG.PLUGINS.INDEX.URL + '/api/v1/plugins' - - try { - const { body } = await doJSONRequest(uri, { searchParams }) - - logger.debug('Got result from PeerTube index.', { body }) - - addInstanceInformation(body) - - return body as ResultList - } catch (err) { - logger.error('Cannot list available plugins from index %s.', uri, { err }) - return undefined - } -} - -function addInstanceInformation (result: ResultList) { - for (const d of result.data) { - d.installed = PluginManager.Instance.isRegistered(d.npmName) - d.name = PluginModel.normalizePluginName(d.npmName) - } - - return result -} - -async function getLatestPluginsVersion (npmNames: string[]): Promise { - const bodyRequest: PeertubePluginLatestVersionRequest = { - npmNames, - currentPeerTubeEngine: PEERTUBE_VERSION - } - - const uri = sanitizeUrl(CONFIG.PLUGINS.INDEX.URL) + '/api/v1/plugins/latest-version' - - const options = { - json: bodyRequest, - method: 'POST' as 'POST' - } - const { body } = await doJSONRequest(uri, options) - - return body -} - -async function getLatestPluginVersion (npmName: string) { - const results = await getLatestPluginsVersion([ npmName ]) - - if (Array.isArray(results) === false || results.length !== 1) { - logger.warn('Cannot get latest supported plugin version of %s.', npmName) - return undefined - } - - return results[0].latestVersion -} - -export { - listAvailablePluginsFromIndex, - getLatestPluginVersion, - getLatestPluginsVersion -} diff --git a/server/lib/plugins/plugin-manager.ts b/server/lib/plugins/plugin-manager.ts deleted file mode 100644 index 88c5b60d7..000000000 --- a/server/lib/plugins/plugin-manager.ts +++ /dev/null @@ -1,665 +0,0 @@ -import express from 'express' -import { createReadStream, createWriteStream } from 'fs' -import { ensureDir, outputFile, readJSON } from 'fs-extra' -import { Server } from 'http' -import { basename, join } from 'path' -import { decachePlugin } from '@server/helpers/decache' -import { ApplicationModel } from '@server/models/application/application' -import { MOAuthTokenUser, MUser } from '@server/types/models' -import { getCompleteLocale } from '@shared/core-utils' -import { - ClientScriptJSON, - PluginPackageJSON, - PluginTranslation, - PluginTranslationPathsJSON, - RegisterServerHookOptions -} from '@shared/models' -import { getHookType, internalRunHook } from '../../../shared/core-utils/plugins/hooks' -import { PluginType } from '../../../shared/models/plugins/plugin.type' -import { ServerHook, ServerHookName } from '../../../shared/models/plugins/server/server-hook.model' -import { isLibraryCodeValid, isPackageJSONValid } from '../../helpers/custom-validators/plugins' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { PLUGIN_GLOBAL_CSS_PATH } from '../../initializers/constants' -import { PluginModel } from '../../models/server/plugin' -import { PluginLibrary, RegisterServerAuthExternalOptions, RegisterServerAuthPassOptions, RegisterServerOptions } from '../../types/plugins' -import { ClientHtml } from '../client-html' -import { RegisterHelpers } from './register-helpers' -import { installNpmPlugin, installNpmPluginFromDisk, rebuildNativePlugins, removeNpmPlugin } from './yarn' - -export interface RegisteredPlugin { - npmName: string - name: string - version: string - description: string - peertubeEngine: string - - type: PluginType - - path: string - - staticDirs: { [name: string]: string } - clientScripts: { [name: string]: ClientScriptJSON } - - css: string[] - - // Only if this is a plugin - registerHelpers?: RegisterHelpers - unregister?: Function -} - -export interface HookInformationValue { - npmName: string - pluginName: string - handler: Function - priority: number -} - -type PluginLocalesTranslations = { - [locale: string]: PluginTranslation -} - -export class PluginManager implements ServerHook { - - private static instance: PluginManager - - private registeredPlugins: { [name: string]: RegisteredPlugin } = {} - - private hooks: { [name: string]: HookInformationValue[] } = {} - private translations: PluginLocalesTranslations = {} - - private server: Server - - private constructor () { - } - - init (server: Server) { - this.server = server - } - - registerWebSocketRouter () { - this.server.on('upgrade', (request, socket, head) => { - // Check if it's a plugin websocket connection - // No need to destroy the stream when we abort the request - // Other handlers in PeerTube will catch this upgrade event too (socket.io, tracker etc) - - const url = request.url - - const matched = url.match(`/plugins/([^/]+)/([^/]+/)?ws(/.*)`) - if (!matched) return - - const npmName = PluginModel.buildNpmName(matched[1], PluginType.PLUGIN) - const subRoute = matched[3] - - const result = this.getRegisteredPluginOrTheme(npmName) - if (!result) return - - const routes = result.registerHelpers.getWebSocketRoutes() - - const wss = routes.find(r => r.route.startsWith(subRoute)) - if (!wss) return - - try { - wss.handler(request, socket, head) - } catch (err) { - logger.error('Exception in plugin handler ' + npmName, { err }) - } - }) - } - - // ###################### Getters ###################### - - isRegistered (npmName: string) { - return !!this.getRegisteredPluginOrTheme(npmName) - } - - getRegisteredPluginOrTheme (npmName: string) { - return this.registeredPlugins[npmName] - } - - getRegisteredPluginByShortName (name: string) { - const npmName = PluginModel.buildNpmName(name, PluginType.PLUGIN) - const registered = this.getRegisteredPluginOrTheme(npmName) - - if (!registered || registered.type !== PluginType.PLUGIN) return undefined - - return registered - } - - getRegisteredThemeByShortName (name: string) { - const npmName = PluginModel.buildNpmName(name, PluginType.THEME) - const registered = this.getRegisteredPluginOrTheme(npmName) - - if (!registered || registered.type !== PluginType.THEME) return undefined - - return registered - } - - getRegisteredPlugins () { - return this.getRegisteredPluginsOrThemes(PluginType.PLUGIN) - } - - getRegisteredThemes () { - return this.getRegisteredPluginsOrThemes(PluginType.THEME) - } - - getIdAndPassAuths () { - return this.getRegisteredPlugins() - .map(p => ({ - npmName: p.npmName, - name: p.name, - version: p.version, - idAndPassAuths: p.registerHelpers.getIdAndPassAuths() - })) - .filter(v => v.idAndPassAuths.length !== 0) - } - - getExternalAuths () { - return this.getRegisteredPlugins() - .map(p => ({ - npmName: p.npmName, - name: p.name, - version: p.version, - externalAuths: p.registerHelpers.getExternalAuths() - })) - .filter(v => v.externalAuths.length !== 0) - } - - getRegisteredSettings (npmName: string) { - const result = this.getRegisteredPluginOrTheme(npmName) - if (!result || result.type !== PluginType.PLUGIN) return [] - - return result.registerHelpers.getSettings() - } - - getRouter (npmName: string) { - const result = this.getRegisteredPluginOrTheme(npmName) - if (!result || result.type !== PluginType.PLUGIN) return null - - return result.registerHelpers.getRouter() - } - - getTranslations (locale: string) { - return this.translations[locale] || {} - } - - async isTokenValid (token: MOAuthTokenUser, type: 'access' | 'refresh') { - const auth = this.getAuth(token.User.pluginAuth, token.authName) - if (!auth) return true - - if (auth.hookTokenValidity) { - try { - const { valid } = await auth.hookTokenValidity({ token, type }) - - if (valid === false) { - logger.info('Rejecting %s token validity from auth %s of plugin %s', type, token.authName, token.User.pluginAuth) - } - - return valid - } catch (err) { - logger.warn('Cannot run check token validity from auth %s of plugin %s.', token.authName, token.User.pluginAuth, { err }) - return true - } - } - - return true - } - - // ###################### External events ###################### - - async onLogout (npmName: string, authName: string, user: MUser, req: express.Request) { - const auth = this.getAuth(npmName, authName) - - if (auth?.onLogout) { - logger.info('Running onLogout function from auth %s of plugin %s', authName, npmName) - - try { - // Force await, in case or onLogout returns a promise - const result = await auth.onLogout(user, req) - - return typeof result === 'string' - ? result - : undefined - } catch (err) { - logger.warn('Cannot run onLogout function from auth %s of plugin %s.', authName, npmName, { err }) - } - } - - return undefined - } - - async onSettingsChanged (name: string, settings: any) { - const registered = this.getRegisteredPluginByShortName(name) - if (!registered) { - logger.error('Cannot find plugin %s to call on settings changed.', name) - } - - for (const cb of registered.registerHelpers.getOnSettingsChangedCallbacks()) { - try { - await cb(settings) - } catch (err) { - logger.error('Cannot run on settings changed callback for %s.', registered.npmName, { err }) - } - } - } - - // ###################### Hooks ###################### - - async runHook (hookName: ServerHookName, result?: T, params?: any): Promise { - if (!this.hooks[hookName]) return Promise.resolve(result) - - const hookType = getHookType(hookName) - - for (const hook of this.hooks[hookName]) { - logger.debug('Running hook %s of plugin %s.', hookName, hook.npmName) - - result = await internalRunHook({ - handler: hook.handler, - hookType, - result, - params, - onError: err => { logger.error('Cannot run hook %s of plugin %s.', hookName, hook.pluginName, { err }) } - }) - } - - return result - } - - // ###################### Registration ###################### - - async registerPluginsAndThemes () { - await this.resetCSSGlobalFile() - - const plugins = await PluginModel.listEnabledPluginsAndThemes() - - for (const plugin of plugins) { - try { - await this.registerPluginOrTheme(plugin) - } catch (err) { - // Try to unregister the plugin - try { - await this.unregister(PluginModel.buildNpmName(plugin.name, plugin.type)) - } catch { - // we don't care if we cannot unregister it - } - - logger.error('Cannot register plugin %s, skipping.', plugin.name, { err }) - } - } - - this.sortHooksByPriority() - } - - // Don't need the plugin type since themes cannot register server code - async unregister (npmName: string) { - logger.info('Unregister plugin %s.', npmName) - - const plugin = this.getRegisteredPluginOrTheme(npmName) - - if (!plugin) { - throw new Error(`Unknown plugin ${npmName} to unregister`) - } - - delete this.registeredPlugins[plugin.npmName] - - this.deleteTranslations(plugin.npmName) - - if (plugin.type === PluginType.PLUGIN) { - await plugin.unregister() - - // Remove hooks of this plugin - for (const key of Object.keys(this.hooks)) { - this.hooks[key] = this.hooks[key].filter(h => h.npmName !== npmName) - } - - const store = plugin.registerHelpers - store.reinitVideoConstants(plugin.npmName) - store.reinitTranscodingProfilesAndEncoders(plugin.npmName) - - logger.info('Regenerating registered plugin CSS to global file.') - await this.regeneratePluginGlobalCSS() - } - - ClientHtml.invalidCache() - } - - // ###################### Installation ###################### - - async install (options: { - toInstall: string - version?: string - fromDisk?: boolean // default false - register?: boolean // default true - }) { - const { toInstall, version, fromDisk = false, register = true } = options - - let plugin: PluginModel - let npmName: string - - logger.info('Installing plugin %s.', toInstall) - - try { - fromDisk - ? await installNpmPluginFromDisk(toInstall) - : await installNpmPlugin(toInstall, version) - - npmName = fromDisk ? basename(toInstall) : toInstall - const pluginType = PluginModel.getTypeFromNpmName(npmName) - const pluginName = PluginModel.normalizePluginName(npmName) - - const packageJSON = await this.getPackageJSON(pluginName, pluginType) - - this.sanitizeAndCheckPackageJSONOrThrow(packageJSON, pluginType); - - [ plugin ] = await PluginModel.upsert({ - name: pluginName, - description: packageJSON.description, - homepage: packageJSON.homepage, - type: pluginType, - version: packageJSON.version, - enabled: true, - uninstalled: false, - peertubeEngine: packageJSON.engine.peertube - }, { returning: true }) - - logger.info('Successful installation of plugin %s.', toInstall) - - if (register) { - await this.registerPluginOrTheme(plugin) - } - } catch (rootErr) { - logger.error('Cannot install plugin %s, removing it...', toInstall, { err: rootErr }) - - if (npmName) { - try { - await this.uninstall({ npmName }) - } catch (err) { - logger.error('Cannot uninstall plugin %s after failed installation.', toInstall, { err }) - - try { - await removeNpmPlugin(npmName) - } catch (err) { - logger.error('Cannot remove plugin %s after failed installation.', toInstall, { err }) - } - } - } - - throw rootErr - } - - return plugin - } - - async update (toUpdate: string, fromDisk = false) { - const npmName = fromDisk ? basename(toUpdate) : toUpdate - - logger.info('Updating plugin %s.', npmName) - - // Use the latest version from DB, to not upgrade to a version that does not support our PeerTube version - let version: string - if (!fromDisk) { - const plugin = await PluginModel.loadByNpmName(toUpdate) - version = plugin.latestVersion - } - - // Unregister old hooks - await this.unregister(npmName) - - return this.install({ toInstall: toUpdate, version, fromDisk }) - } - - async uninstall (options: { - npmName: string - unregister?: boolean // default true - }) { - const { npmName, unregister = true } = options - - logger.info('Uninstalling plugin %s.', npmName) - - if (unregister) { - try { - await this.unregister(npmName) - } catch (err) { - logger.warn('Cannot unregister plugin %s.', npmName, { err }) - } - } - - const plugin = await PluginModel.loadByNpmName(npmName) - if (!plugin || plugin.uninstalled === true) { - logger.error('Cannot uninstall plugin %s: it does not exist or is already uninstalled.', npmName) - return - } - - plugin.enabled = false - plugin.uninstalled = true - - await plugin.save() - - await removeNpmPlugin(npmName) - - logger.info('Plugin %s uninstalled.', npmName) - } - - async rebuildNativePluginsIfNeeded () { - if (!await ApplicationModel.nodeABIChanged()) return - - return rebuildNativePlugins() - } - - // ###################### Private register ###################### - - private async registerPluginOrTheme (plugin: PluginModel) { - const npmName = PluginModel.buildNpmName(plugin.name, plugin.type) - - logger.info('Registering plugin or theme %s.', npmName) - - const packageJSON = await this.getPackageJSON(plugin.name, plugin.type) - const pluginPath = this.getPluginPath(plugin.name, plugin.type) - - this.sanitizeAndCheckPackageJSONOrThrow(packageJSON, plugin.type) - - let library: PluginLibrary - let registerHelpers: RegisterHelpers - if (plugin.type === PluginType.PLUGIN) { - const result = await this.registerPlugin(plugin, pluginPath, packageJSON) - library = result.library - registerHelpers = result.registerStore - } - - const clientScripts: { [id: string]: ClientScriptJSON } = {} - for (const c of packageJSON.clientScripts) { - clientScripts[c.script] = c - } - - this.registeredPlugins[npmName] = { - npmName, - name: plugin.name, - type: plugin.type, - version: plugin.version, - description: plugin.description, - peertubeEngine: plugin.peertubeEngine, - path: pluginPath, - staticDirs: packageJSON.staticDirs, - clientScripts, - css: packageJSON.css, - registerHelpers: registerHelpers || undefined, - unregister: library ? library.unregister : undefined - } - - await this.addTranslations(plugin, npmName, packageJSON.translations) - - ClientHtml.invalidCache() - } - - private async registerPlugin (plugin: PluginModel, pluginPath: string, packageJSON: PluginPackageJSON) { - const npmName = PluginModel.buildNpmName(plugin.name, plugin.type) - - // Delete cache if needed - const modulePath = join(pluginPath, packageJSON.library) - decachePlugin(modulePath) - const library: PluginLibrary = require(modulePath) - - if (!isLibraryCodeValid(library)) { - throw new Error('Library code is not valid (miss register or unregister function)') - } - - const { registerOptions, registerStore } = this.getRegisterHelpers(npmName, plugin) - - await ensureDir(registerOptions.peertubeHelpers.plugin.getDataDirectoryPath()) - - await library.register(registerOptions) - - logger.info('Add plugin %s CSS to global file.', npmName) - - await this.addCSSToGlobalFile(pluginPath, packageJSON.css) - - return { library, registerStore } - } - - // ###################### Translations ###################### - - private async addTranslations (plugin: PluginModel, npmName: string, translationPaths: PluginTranslationPathsJSON) { - for (const locale of Object.keys(translationPaths)) { - const path = translationPaths[locale] - const json = await readJSON(join(this.getPluginPath(plugin.name, plugin.type), path)) - - const completeLocale = getCompleteLocale(locale) - - if (!this.translations[completeLocale]) this.translations[completeLocale] = {} - this.translations[completeLocale][npmName] = json - - logger.info('Added locale %s of plugin %s.', completeLocale, npmName) - } - } - - private deleteTranslations (npmName: string) { - for (const locale of Object.keys(this.translations)) { - delete this.translations[locale][npmName] - - logger.info('Deleted locale %s of plugin %s.', locale, npmName) - } - } - - // ###################### CSS ###################### - - private resetCSSGlobalFile () { - return outputFile(PLUGIN_GLOBAL_CSS_PATH, '') - } - - private async addCSSToGlobalFile (pluginPath: string, cssRelativePaths: string[]) { - for (const cssPath of cssRelativePaths) { - await this.concatFiles(join(pluginPath, cssPath), PLUGIN_GLOBAL_CSS_PATH) - } - } - - private concatFiles (input: string, output: string) { - return new Promise((res, rej) => { - const inputStream = createReadStream(input) - const outputStream = createWriteStream(output, { flags: 'a' }) - - inputStream.pipe(outputStream) - - inputStream.on('end', () => res()) - inputStream.on('error', err => rej(err)) - }) - } - - private async regeneratePluginGlobalCSS () { - await this.resetCSSGlobalFile() - - for (const plugin of this.getRegisteredPlugins()) { - await this.addCSSToGlobalFile(plugin.path, plugin.css) - } - } - - // ###################### Utils ###################### - - private sortHooksByPriority () { - for (const hookName of Object.keys(this.hooks)) { - this.hooks[hookName].sort((a, b) => { - return b.priority - a.priority - }) - } - } - - private getPackageJSON (pluginName: string, pluginType: PluginType) { - const pluginPath = join(this.getPluginPath(pluginName, pluginType), 'package.json') - - return readJSON(pluginPath) as Promise - } - - private getPluginPath (pluginName: string, pluginType: PluginType) { - const npmName = PluginModel.buildNpmName(pluginName, pluginType) - - return join(CONFIG.STORAGE.PLUGINS_DIR, 'node_modules', npmName) - } - - private getAuth (npmName: string, authName: string) { - const plugin = this.getRegisteredPluginOrTheme(npmName) - if (!plugin || plugin.type !== PluginType.PLUGIN) return null - - let auths: (RegisterServerAuthPassOptions | RegisterServerAuthExternalOptions)[] = plugin.registerHelpers.getIdAndPassAuths() - auths = auths.concat(plugin.registerHelpers.getExternalAuths()) - - return auths.find(a => a.authName === authName) - } - - // ###################### Private getters ###################### - - private getRegisteredPluginsOrThemes (type: PluginType) { - const plugins: RegisteredPlugin[] = [] - - for (const npmName of Object.keys(this.registeredPlugins)) { - const plugin = this.registeredPlugins[npmName] - if (plugin.type !== type) continue - - plugins.push(plugin) - } - - return plugins - } - - // ###################### Generate register helpers ###################### - - private getRegisterHelpers ( - npmName: string, - plugin: PluginModel - ): { registerStore: RegisterHelpers, registerOptions: RegisterServerOptions } { - const onHookAdded = (options: RegisterServerHookOptions) => { - if (!this.hooks[options.target]) this.hooks[options.target] = [] - - this.hooks[options.target].push({ - npmName, - pluginName: plugin.name, - handler: options.handler, - priority: options.priority || 0 - }) - } - - const registerHelpers = new RegisterHelpers(npmName, plugin, this.server, onHookAdded.bind(this)) - - return { - registerStore: registerHelpers, - registerOptions: registerHelpers.buildRegisterHelpers() - } - } - - private sanitizeAndCheckPackageJSONOrThrow (packageJSON: PluginPackageJSON, pluginType: PluginType) { - if (!packageJSON.staticDirs) packageJSON.staticDirs = {} - if (!packageJSON.css) packageJSON.css = [] - if (!packageJSON.clientScripts) packageJSON.clientScripts = [] - if (!packageJSON.translations) packageJSON.translations = {} - - const { result: packageJSONValid, badFields } = isPackageJSONValid(packageJSON, pluginType) - if (!packageJSONValid) { - const formattedFields = badFields.map(f => `"${f}"`) - .join(', ') - - throw new Error(`PackageJSON is invalid (invalid fields: ${formattedFields}).`) - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/plugins/register-helpers.ts b/server/lib/plugins/register-helpers.ts deleted file mode 100644 index 1aaef3606..000000000 --- a/server/lib/plugins/register-helpers.ts +++ /dev/null @@ -1,340 +0,0 @@ -import express from 'express' -import { Server } from 'http' -import { logger } from '@server/helpers/logger' -import { onExternalUserAuthenticated } from '@server/lib/auth/external-auth' -import { VideoConstantManagerFactory } from '@server/lib/plugins/video-constant-manager-factory' -import { PluginModel } from '@server/models/server/plugin' -import { - RegisterServerAuthExternalOptions, - RegisterServerAuthExternalResult, - RegisterServerAuthPassOptions, - RegisterServerExternalAuthenticatedResult, - RegisterServerOptions, - RegisterServerWebSocketRouteOptions -} from '@server/types/plugins' -import { - EncoderOptionsBuilder, - PluginSettingsManager, - PluginStorageManager, - RegisterServerHookOptions, - RegisterServerSettingOptions, - serverHookObject, - SettingsChangeCallback, - VideoPlaylistPrivacy, - VideoPrivacy -} from '@shared/models' -import { VideoTranscodingProfilesManager } from '../transcoding/default-transcoding-profiles' -import { buildPluginHelpers } from './plugin-helpers-builder' - -export class RegisterHelpers { - private readonly transcodingProfiles: { - [ npmName: string ]: { - type: 'vod' | 'live' - encoder: string - profile: string - }[] - } = {} - - private readonly transcodingEncoders: { - [ npmName: string ]: { - type: 'vod' | 'live' - streamType: 'audio' | 'video' - encoder: string - priority: number - }[] - } = {} - - private readonly settings: RegisterServerSettingOptions[] = [] - - private idAndPassAuths: RegisterServerAuthPassOptions[] = [] - private externalAuths: RegisterServerAuthExternalOptions[] = [] - - private readonly onSettingsChangeCallbacks: SettingsChangeCallback[] = [] - - private readonly webSocketRoutes: RegisterServerWebSocketRouteOptions[] = [] - - private readonly router: express.Router - private readonly videoConstantManagerFactory: VideoConstantManagerFactory - - constructor ( - private readonly npmName: string, - private readonly plugin: PluginModel, - private readonly server: Server, - private readonly onHookAdded: (options: RegisterServerHookOptions) => void - ) { - this.router = express.Router() - this.videoConstantManagerFactory = new VideoConstantManagerFactory(this.npmName) - } - - buildRegisterHelpers (): RegisterServerOptions { - const registerHook = this.buildRegisterHook() - const registerSetting = this.buildRegisterSetting() - - const getRouter = this.buildGetRouter() - const registerWebSocketRoute = this.buildRegisterWebSocketRoute() - - const settingsManager = this.buildSettingsManager() - const storageManager = this.buildStorageManager() - - const videoLanguageManager = this.videoConstantManagerFactory.createVideoConstantManager('language') - - const videoLicenceManager = this.videoConstantManagerFactory.createVideoConstantManager('licence') - const videoCategoryManager = this.videoConstantManagerFactory.createVideoConstantManager('category') - - const videoPrivacyManager = this.videoConstantManagerFactory.createVideoConstantManager('privacy') - const playlistPrivacyManager = this.videoConstantManagerFactory.createVideoConstantManager('playlistPrivacy') - - const transcodingManager = this.buildTranscodingManager() - - const registerIdAndPassAuth = this.buildRegisterIdAndPassAuth() - const registerExternalAuth = this.buildRegisterExternalAuth() - const unregisterIdAndPassAuth = this.buildUnregisterIdAndPassAuth() - const unregisterExternalAuth = this.buildUnregisterExternalAuth() - - const peertubeHelpers = buildPluginHelpers(this.server, this.plugin, this.npmName) - - return { - registerHook, - registerSetting, - - getRouter, - registerWebSocketRoute, - - settingsManager, - storageManager, - - videoLanguageManager: { - ...videoLanguageManager, - /** @deprecated use `addConstant` instead **/ - addLanguage: videoLanguageManager.addConstant, - /** @deprecated use `deleteConstant` instead **/ - deleteLanguage: videoLanguageManager.deleteConstant - }, - videoCategoryManager: { - ...videoCategoryManager, - /** @deprecated use `addConstant` instead **/ - addCategory: videoCategoryManager.addConstant, - /** @deprecated use `deleteConstant` instead **/ - deleteCategory: videoCategoryManager.deleteConstant - }, - videoLicenceManager: { - ...videoLicenceManager, - /** @deprecated use `addConstant` instead **/ - addLicence: videoLicenceManager.addConstant, - /** @deprecated use `deleteConstant` instead **/ - deleteLicence: videoLicenceManager.deleteConstant - }, - - videoPrivacyManager: { - ...videoPrivacyManager, - /** @deprecated use `deleteConstant` instead **/ - deletePrivacy: videoPrivacyManager.deleteConstant - }, - playlistPrivacyManager: { - ...playlistPrivacyManager, - /** @deprecated use `deleteConstant` instead **/ - deletePlaylistPrivacy: playlistPrivacyManager.deleteConstant - }, - - transcodingManager, - - registerIdAndPassAuth, - registerExternalAuth, - unregisterIdAndPassAuth, - unregisterExternalAuth, - - peertubeHelpers - } - } - - reinitVideoConstants (npmName: string) { - this.videoConstantManagerFactory.resetVideoConstants(npmName) - } - - reinitTranscodingProfilesAndEncoders (npmName: string) { - const profiles = this.transcodingProfiles[npmName] - if (Array.isArray(profiles)) { - for (const profile of profiles) { - VideoTranscodingProfilesManager.Instance.removeProfile(profile) - } - } - - const encoders = this.transcodingEncoders[npmName] - if (Array.isArray(encoders)) { - for (const o of encoders) { - VideoTranscodingProfilesManager.Instance.removeEncoderPriority(o.type, o.streamType, o.encoder, o.priority) - } - } - } - - getSettings () { - return this.settings - } - - getRouter () { - return this.router - } - - getIdAndPassAuths () { - return this.idAndPassAuths - } - - getExternalAuths () { - return this.externalAuths - } - - getOnSettingsChangedCallbacks () { - return this.onSettingsChangeCallbacks - } - - getWebSocketRoutes () { - return this.webSocketRoutes - } - - private buildGetRouter () { - return () => this.router - } - - private buildRegisterWebSocketRoute () { - return (options: RegisterServerWebSocketRouteOptions) => { - this.webSocketRoutes.push(options) - } - } - - private buildRegisterSetting () { - return (options: RegisterServerSettingOptions) => { - this.settings.push(options) - } - } - - private buildRegisterHook () { - return (options: RegisterServerHookOptions) => { - if (serverHookObject[options.target] !== true) { - logger.warn('Unknown hook %s of plugin %s. Skipping.', options.target, this.npmName) - return - } - - return this.onHookAdded(options) - } - } - - private buildRegisterIdAndPassAuth () { - return (options: RegisterServerAuthPassOptions) => { - if (!options.authName || typeof options.getWeight !== 'function' || typeof options.login !== 'function') { - logger.error('Cannot register auth plugin %s: authName, getWeight or login are not valid.', this.npmName, { options }) - return - } - - this.idAndPassAuths.push(options) - } - } - - private buildRegisterExternalAuth () { - const self = this - - return (options: RegisterServerAuthExternalOptions) => { - if (!options.authName || typeof options.authDisplayName !== 'function' || typeof options.onAuthRequest !== 'function') { - logger.error('Cannot register auth plugin %s: authName, authDisplayName or onAuthRequest are not valid.', this.npmName, { options }) - return - } - - this.externalAuths.push(options) - - return { - userAuthenticated (result: RegisterServerExternalAuthenticatedResult): void { - onExternalUserAuthenticated({ - npmName: self.npmName, - authName: options.authName, - authResult: result - }).catch(err => { - logger.error('Cannot execute onExternalUserAuthenticated.', { npmName: self.npmName, authName: options.authName, err }) - }) - } - } as RegisterServerAuthExternalResult - } - } - - private buildUnregisterExternalAuth () { - return (authName: string) => { - this.externalAuths = this.externalAuths.filter(a => a.authName !== authName) - } - } - - private buildUnregisterIdAndPassAuth () { - return (authName: string) => { - this.idAndPassAuths = this.idAndPassAuths.filter(a => a.authName !== authName) - } - } - - private buildSettingsManager (): PluginSettingsManager { - return { - getSetting: (name: string) => PluginModel.getSetting(this.plugin.name, this.plugin.type, name, this.settings), - - getSettings: (names: string[]) => PluginModel.getSettings(this.plugin.name, this.plugin.type, names, this.settings), - - setSetting: (name: string, value: string) => PluginModel.setSetting(this.plugin.name, this.plugin.type, name, value), - - onSettingsChange: (cb: SettingsChangeCallback) => this.onSettingsChangeCallbacks.push(cb) - } - } - - private buildStorageManager (): PluginStorageManager { - return { - getData: (key: string) => PluginModel.getData(this.plugin.name, this.plugin.type, key), - - storeData: (key: string, data: any) => PluginModel.storeData(this.plugin.name, this.plugin.type, key, data) - } - } - - private buildTranscodingManager () { - const self = this - - function addProfile (type: 'live' | 'vod', encoder: string, profile: string, builder: EncoderOptionsBuilder) { - if (profile === 'default') { - logger.error('A plugin cannot add a default live transcoding profile') - return false - } - - VideoTranscodingProfilesManager.Instance.addProfile({ - type, - encoder, - profile, - builder - }) - - if (!self.transcodingProfiles[self.npmName]) self.transcodingProfiles[self.npmName] = [] - self.transcodingProfiles[self.npmName].push({ type, encoder, profile }) - - return true - } - - function addEncoderPriority (type: 'live' | 'vod', streamType: 'audio' | 'video', encoder: string, priority: number) { - VideoTranscodingProfilesManager.Instance.addEncoderPriority(type, streamType, encoder, priority) - - if (!self.transcodingEncoders[self.npmName]) self.transcodingEncoders[self.npmName] = [] - self.transcodingEncoders[self.npmName].push({ type, streamType, encoder, priority }) - } - - return { - addLiveProfile (encoder: string, profile: string, builder: EncoderOptionsBuilder) { - return addProfile('live', encoder, profile, builder) - }, - - addVODProfile (encoder: string, profile: string, builder: EncoderOptionsBuilder) { - return addProfile('vod', encoder, profile, builder) - }, - - addLiveEncoderPriority (streamType: 'audio' | 'video', encoder: string, priority: number) { - return addEncoderPriority('live', streamType, encoder, priority) - }, - - addVODEncoderPriority (streamType: 'audio' | 'video', encoder: string, priority: number) { - return addEncoderPriority('vod', streamType, encoder, priority) - }, - - removeAllProfilesAndEncoderPriorities () { - return self.reinitTranscodingProfilesAndEncoders(self.npmName) - } - } - } -} diff --git a/server/lib/plugins/theme-utils.ts b/server/lib/plugins/theme-utils.ts deleted file mode 100644 index 76c671f1c..000000000 --- a/server/lib/plugins/theme-utils.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { DEFAULT_THEME_NAME, DEFAULT_USER_THEME_NAME } from '../../initializers/constants' -import { PluginManager } from './plugin-manager' -import { CONFIG } from '../../initializers/config' - -function getThemeOrDefault (name: string, defaultTheme: string) { - if (isThemeRegistered(name)) return name - - // Fallback to admin default theme - if (name !== CONFIG.THEME.DEFAULT) return getThemeOrDefault(CONFIG.THEME.DEFAULT, DEFAULT_THEME_NAME) - - return defaultTheme -} - -function isThemeRegistered (name: string) { - if (name === DEFAULT_THEME_NAME || name === DEFAULT_USER_THEME_NAME) return true - - return !!PluginManager.Instance.getRegisteredThemes() - .find(r => r.name === name) -} - -export { - getThemeOrDefault, - isThemeRegistered -} diff --git a/server/lib/plugins/video-constant-manager-factory.ts b/server/lib/plugins/video-constant-manager-factory.ts deleted file mode 100644 index 5f7edfbe2..000000000 --- a/server/lib/plugins/video-constant-manager-factory.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { - VIDEO_CATEGORIES, - VIDEO_LANGUAGES, - VIDEO_LICENCES, - VIDEO_PLAYLIST_PRIVACIES, - VIDEO_PRIVACIES -} from '@server/initializers/constants' -import { ConstantManager } from '@shared/models/plugins/server/plugin-constant-manager.model' - -type AlterableVideoConstant = 'language' | 'licence' | 'category' | 'privacy' | 'playlistPrivacy' -type VideoConstant = Record - -type UpdatedVideoConstant = { - [name in AlterableVideoConstant]: { - [ npmName: string]: { - added: VideoConstant[] - deleted: VideoConstant[] - } - } -} - -const constantsHash: { [key in AlterableVideoConstant]: VideoConstant } = { - language: VIDEO_LANGUAGES, - licence: VIDEO_LICENCES, - category: VIDEO_CATEGORIES, - privacy: VIDEO_PRIVACIES, - playlistPrivacy: VIDEO_PLAYLIST_PRIVACIES -} - -export class VideoConstantManagerFactory { - private readonly updatedVideoConstants: UpdatedVideoConstant = { - playlistPrivacy: { }, - privacy: { }, - language: { }, - licence: { }, - category: { } - } - - constructor ( - private readonly npmName: string - ) {} - - public resetVideoConstants (npmName: string) { - const types: AlterableVideoConstant[] = [ 'language', 'licence', 'category', 'privacy', 'playlistPrivacy' ] - for (const type of types) { - this.resetConstants({ npmName, type }) - } - } - - private resetConstants (parameters: { npmName: string, type: AlterableVideoConstant }) { - const { npmName, type } = parameters - const updatedConstants = this.updatedVideoConstants[type][npmName] - - if (!updatedConstants) return - - for (const added of updatedConstants.added) { - delete constantsHash[type][added.key] - } - - for (const deleted of updatedConstants.deleted) { - constantsHash[type][deleted.key] = deleted.label - } - - delete this.updatedVideoConstants[type][npmName] - } - - public createVideoConstantManager(type: AlterableVideoConstant): ConstantManager { - const { npmName } = this - return { - addConstant: (key: K, label: string) => this.addConstant({ npmName, type, key, label }), - deleteConstant: (key: K) => this.deleteConstant({ npmName, type, key }), - getConstantValue: (key: K) => constantsHash[type][key], - getConstants: () => constantsHash[type] as Record, - resetConstants: () => this.resetConstants({ npmName, type }) - } - } - - private addConstant (parameters: { - npmName: string - type: AlterableVideoConstant - key: T - label: string - }) { - const { npmName, type, key, label } = parameters - const obj = constantsHash[type] - - if (obj[key]) { - logger.warn('Cannot add %s %s by plugin %s: key already exists.', type, npmName, key) - return false - } - - if (!this.updatedVideoConstants[type][npmName]) { - this.updatedVideoConstants[type][npmName] = { - added: [], - deleted: [] - } - } - - this.updatedVideoConstants[type][npmName].added.push({ key, label } as VideoConstant) - obj[key] = label - - return true - } - - private deleteConstant (parameters: { - npmName: string - type: AlterableVideoConstant - key: T - }) { - const { npmName, type, key } = parameters - const obj = constantsHash[type] - - if (!obj[key]) { - logger.warn('Cannot delete %s by plugin %s: key %s does not exist.', type, npmName, key) - return false - } - - if (!this.updatedVideoConstants[type][npmName]) { - this.updatedVideoConstants[type][npmName] = { - added: [], - deleted: [] - } - } - - const updatedConstants = this.updatedVideoConstants[type][npmName] - - const alreadyAdded = updatedConstants.added.find(a => a.key === key) - if (alreadyAdded) { - updatedConstants.added.filter(a => a.key !== key) - } else if (obj[key]) { - updatedConstants.deleted.push({ key, label: obj[key] } as VideoConstant) - } - - delete obj[key] - - return true - } -} diff --git a/server/lib/plugins/yarn.ts b/server/lib/plugins/yarn.ts deleted file mode 100644 index 9cf6ec9e9..000000000 --- a/server/lib/plugins/yarn.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { outputJSON, pathExists } from 'fs-extra' -import { join } from 'path' -import { execShell } from '../../helpers/core-utils' -import { isNpmPluginNameValid, isPluginStableOrUnstableVersionValid } from '../../helpers/custom-validators/plugins' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { getLatestPluginVersion } from './plugin-index' - -async function installNpmPlugin (npmName: string, versionArg?: string) { - // Security check - checkNpmPluginNameOrThrow(npmName) - if (versionArg) checkPluginVersionOrThrow(versionArg) - - const version = versionArg || await getLatestPluginVersion(npmName) - - let toInstall = npmName - if (version) toInstall += `@${version}` - - const { stdout } = await execYarn('add ' + toInstall) - - logger.debug('Added a yarn package.', { yarnStdout: stdout }) -} - -async function installNpmPluginFromDisk (path: string) { - await execYarn('add file:' + path) -} - -async function removeNpmPlugin (name: string) { - checkNpmPluginNameOrThrow(name) - - await execYarn('remove ' + name) -} - -async function rebuildNativePlugins () { - await execYarn('install --pure-lockfile') -} - -// ############################################################################ - -export { - installNpmPlugin, - installNpmPluginFromDisk, - rebuildNativePlugins, - removeNpmPlugin -} - -// ############################################################################ - -async function execYarn (command: string) { - try { - const pluginDirectory = CONFIG.STORAGE.PLUGINS_DIR - const pluginPackageJSON = join(pluginDirectory, 'package.json') - - // Create empty package.json file if needed - if (!await pathExists(pluginPackageJSON)) { - await outputJSON(pluginPackageJSON, {}) - } - - return execShell(`yarn ${command}`, { cwd: pluginDirectory }) - } catch (result) { - logger.error('Cannot exec yarn.', { command, err: result.err, stderr: result.stderr }) - - throw result.err - } -} - -function checkNpmPluginNameOrThrow (name: string) { - if (!isNpmPluginNameValid(name)) throw new Error('Invalid NPM plugin name to install') -} - -function checkPluginVersionOrThrow (name: string) { - if (!isPluginStableOrUnstableVersionValid(name)) throw new Error('Invalid NPM plugin version to install') -} diff --git a/server/lib/redis.ts b/server/lib/redis.ts deleted file mode 100644 index 48d9986b5..000000000 --- a/server/lib/redis.ts +++ /dev/null @@ -1,465 +0,0 @@ -import IoRedis, { RedisOptions } from 'ioredis' -import { exists } from '@server/helpers/custom-validators/misc' -import { sha256 } from '@shared/extra-utils' -import { logger } from '../helpers/logger' -import { generateRandomString } from '../helpers/utils' -import { CONFIG } from '../initializers/config' -import { - AP_CLEANER, - CONTACT_FORM_LIFETIME, - RESUMABLE_UPLOAD_SESSION_LIFETIME, - TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME, - EMAIL_VERIFY_LIFETIME, - USER_PASSWORD_CREATE_LIFETIME, - USER_PASSWORD_RESET_LIFETIME, - VIEW_LIFETIME, - WEBSERVER -} from '../initializers/constants' - -class Redis { - - private static instance: Redis - private initialized = false - private connected = false - private client: IoRedis - private prefix: string - - private constructor () { - } - - init () { - // Already initialized - if (this.initialized === true) return - this.initialized = true - - const redisMode = CONFIG.REDIS.SENTINEL.ENABLED ? 'sentinel' : 'standalone' - logger.info('Connecting to redis ' + redisMode + '...') - - this.client = new IoRedis(Redis.getRedisClientOptions('', { enableAutoPipelining: true })) - this.client.on('error', err => logger.error('Redis failed to connect', { err })) - this.client.on('connect', () => { - logger.info('Connected to redis.') - - this.connected = true - }) - this.client.on('reconnecting', (ms) => { - logger.error(`Reconnecting to redis in ${ms}.`) - }) - this.client.on('close', () => { - logger.error('Connection to redis has closed.') - this.connected = false - }) - - this.client.on('end', () => { - logger.error('Connection to redis has closed and no more reconnects will be done.') - }) - - this.prefix = 'redis-' + WEBSERVER.HOST + '-' - } - - static getRedisClientOptions (name?: string, options: RedisOptions = {}): RedisOptions { - const connectionName = [ 'PeerTube', name ].join('') - const connectTimeout = 20000 // Could be slow since node use sync call to compile PeerTube - - if (CONFIG.REDIS.SENTINEL.ENABLED) { - return { - connectionName, - connectTimeout, - enableTLSForSentinelMode: CONFIG.REDIS.SENTINEL.ENABLE_TLS, - sentinelPassword: CONFIG.REDIS.AUTH, - sentinels: CONFIG.REDIS.SENTINEL.SENTINELS, - name: CONFIG.REDIS.SENTINEL.MASTER_NAME, - ...options - } - } - - return { - connectionName, - connectTimeout, - password: CONFIG.REDIS.AUTH, - db: CONFIG.REDIS.DB, - host: CONFIG.REDIS.HOSTNAME, - port: CONFIG.REDIS.PORT, - path: CONFIG.REDIS.SOCKET, - showFriendlyErrorStack: true, - ...options - } - } - - getClient () { - return this.client - } - - getPrefix () { - return this.prefix - } - - isConnected () { - return this.connected - } - - /* ************ Forgot password ************ */ - - async setResetPasswordVerificationString (userId: number) { - const generatedString = await generateRandomString(32) - - await this.setValue(this.generateResetPasswordKey(userId), generatedString, USER_PASSWORD_RESET_LIFETIME) - - return generatedString - } - - async setCreatePasswordVerificationString (userId: number) { - const generatedString = await generateRandomString(32) - - await this.setValue(this.generateResetPasswordKey(userId), generatedString, USER_PASSWORD_CREATE_LIFETIME) - - return generatedString - } - - async removePasswordVerificationString (userId: number) { - return this.removeValue(this.generateResetPasswordKey(userId)) - } - - async getResetPasswordVerificationString (userId: number) { - return this.getValue(this.generateResetPasswordKey(userId)) - } - - /* ************ Two factor auth request ************ */ - - async setTwoFactorRequest (userId: number, otpSecret: string) { - const requestToken = await generateRandomString(32) - - await this.setValue(this.generateTwoFactorRequestKey(userId, requestToken), otpSecret, TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME) - - return requestToken - } - - async getTwoFactorRequestToken (userId: number, requestToken: string) { - return this.getValue(this.generateTwoFactorRequestKey(userId, requestToken)) - } - - /* ************ Email verification ************ */ - - async setUserVerifyEmailVerificationString (userId: number) { - const generatedString = await generateRandomString(32) - - await this.setValue(this.generateUserVerifyEmailKey(userId), generatedString, EMAIL_VERIFY_LIFETIME) - - return generatedString - } - - async getUserVerifyEmailLink (userId: number) { - return this.getValue(this.generateUserVerifyEmailKey(userId)) - } - - async setRegistrationVerifyEmailVerificationString (registrationId: number) { - const generatedString = await generateRandomString(32) - - await this.setValue(this.generateRegistrationVerifyEmailKey(registrationId), generatedString, EMAIL_VERIFY_LIFETIME) - - return generatedString - } - - async getRegistrationVerifyEmailLink (registrationId: number) { - return this.getValue(this.generateRegistrationVerifyEmailKey(registrationId)) - } - - /* ************ Contact form per IP ************ */ - - async setContactFormIp (ip: string) { - return this.setValue(this.generateContactFormKey(ip), '1', CONTACT_FORM_LIFETIME) - } - - async doesContactFormIpExist (ip: string) { - return this.exists(this.generateContactFormKey(ip)) - } - - /* ************ Views per IP ************ */ - - setIPVideoView (ip: string, videoUUID: string) { - return this.setValue(this.generateIPViewKey(ip, videoUUID), '1', VIEW_LIFETIME.VIEW) - } - - async doesVideoIPViewExist (ip: string, videoUUID: string) { - return this.exists(this.generateIPViewKey(ip, videoUUID)) - } - - /* ************ Video views stats ************ */ - - addVideoViewStats (videoId: number) { - const { videoKey, setKey } = this.generateVideoViewStatsKeys({ videoId }) - - return Promise.all([ - this.addToSet(setKey, videoId.toString()), - this.increment(videoKey) - ]) - } - - async getVideoViewsStats (videoId: number, hour: number) { - const { videoKey } = this.generateVideoViewStatsKeys({ videoId, hour }) - - const valueString = await this.getValue(videoKey) - const valueInt = parseInt(valueString, 10) - - if (isNaN(valueInt)) { - logger.error('Cannot get videos views stats of video %d in hour %d: views number is NaN (%s).', videoId, hour, valueString) - return undefined - } - - return valueInt - } - - async listVideosViewedForStats (hour: number) { - const { setKey } = this.generateVideoViewStatsKeys({ hour }) - - const stringIds = await this.getSet(setKey) - return stringIds.map(s => parseInt(s, 10)) - } - - deleteVideoViewsStats (videoId: number, hour: number) { - const { setKey, videoKey } = this.generateVideoViewStatsKeys({ videoId, hour }) - - return Promise.all([ - this.deleteFromSet(setKey, videoId.toString()), - this.deleteKey(videoKey) - ]) - } - - /* ************ Local video views buffer ************ */ - - addLocalVideoView (videoId: number) { - const { videoKey, setKey } = this.generateLocalVideoViewsKeys(videoId) - - return Promise.all([ - this.addToSet(setKey, videoId.toString()), - this.increment(videoKey) - ]) - } - - async getLocalVideoViews (videoId: number) { - const { videoKey } = this.generateLocalVideoViewsKeys(videoId) - - const valueString = await this.getValue(videoKey) - const valueInt = parseInt(valueString, 10) - - if (isNaN(valueInt)) { - logger.error('Cannot get videos views of video %d: views number is NaN (%s).', videoId, valueString) - return undefined - } - - return valueInt - } - - async listLocalVideosViewed () { - const { setKey } = this.generateLocalVideoViewsKeys() - - const stringIds = await this.getSet(setKey) - return stringIds.map(s => parseInt(s, 10)) - } - - deleteLocalVideoViews (videoId: number) { - const { setKey, videoKey } = this.generateLocalVideoViewsKeys(videoId) - - return Promise.all([ - this.deleteFromSet(setKey, videoId.toString()), - this.deleteKey(videoKey) - ]) - } - - /* ************ Video viewers stats ************ */ - - getLocalVideoViewer (options: { - key?: string - // Or - ip?: string - videoId?: number - }) { - if (options.key) return this.getObject(options.key) - - const { viewerKey } = this.generateLocalVideoViewerKeys(options.ip, options.videoId) - - return this.getObject(viewerKey) - } - - setLocalVideoViewer (ip: string, videoId: number, object: any) { - const { setKey, viewerKey } = this.generateLocalVideoViewerKeys(ip, videoId) - - return Promise.all([ - this.addToSet(setKey, viewerKey), - this.setObject(viewerKey, object) - ]) - } - - listLocalVideoViewerKeys () { - const { setKey } = this.generateLocalVideoViewerKeys() - - return this.getSet(setKey) - } - - deleteLocalVideoViewersKeys (key: string) { - const { setKey } = this.generateLocalVideoViewerKeys() - - return Promise.all([ - this.deleteFromSet(setKey, key), - this.deleteKey(key) - ]) - } - - /* ************ Resumable uploads final responses ************ */ - - setUploadSession (uploadId: string, response?: { video: { id: number, shortUUID: string, uuid: string } }) { - return this.setValue( - 'resumable-upload-' + uploadId, - response - ? JSON.stringify(response) - : '', - RESUMABLE_UPLOAD_SESSION_LIFETIME - ) - } - - doesUploadSessionExist (uploadId: string) { - return this.exists('resumable-upload-' + uploadId) - } - - async getUploadSession (uploadId: string) { - const value = await this.getValue('resumable-upload-' + uploadId) - - return value - ? JSON.parse(value) as { video: { id: number, shortUUID: string, uuid: string } } - : undefined - } - - deleteUploadSession (uploadId: string) { - return this.deleteKey('resumable-upload-' + uploadId) - } - - /* ************ AP resource unavailability ************ */ - - async addAPUnavailability (url: string) { - const key = this.generateAPUnavailabilityKey(url) - - const value = await this.increment(key) - await this.setExpiration(key, AP_CLEANER.PERIOD * 2) - - return value - } - - /* ************ Keys generation ************ */ - - private generateLocalVideoViewsKeys (videoId: number): { setKey: string, videoKey: string } - private generateLocalVideoViewsKeys (): { setKey: string } - private generateLocalVideoViewsKeys (videoId?: number) { - return { setKey: `local-video-views-buffer`, videoKey: `local-video-views-buffer-${videoId}` } - } - - private generateLocalVideoViewerKeys (ip: string, videoId: number): { setKey: string, viewerKey: string } - private generateLocalVideoViewerKeys (): { setKey: string } - private generateLocalVideoViewerKeys (ip?: string, videoId?: number) { - return { setKey: `local-video-viewer-stats-keys`, viewerKey: `local-video-viewer-stats-${ip}-${videoId}` } - } - - private generateVideoViewStatsKeys (options: { videoId?: number, hour?: number }) { - const hour = exists(options.hour) - ? options.hour - : new Date().getHours() - - return { setKey: `videos-view-h${hour}`, videoKey: `video-view-${options.videoId}-h${hour}` } - } - - private generateResetPasswordKey (userId: number) { - return 'reset-password-' + userId - } - - private generateTwoFactorRequestKey (userId: number, token: string) { - return 'two-factor-request-' + userId + '-' + token - } - - private generateUserVerifyEmailKey (userId: number) { - return 'verify-email-user-' + userId - } - - private generateRegistrationVerifyEmailKey (registrationId: number) { - return 'verify-email-registration-' + registrationId - } - - private generateIPViewKey (ip: string, videoUUID: string) { - return `views-${videoUUID}-${ip}` - } - - private generateContactFormKey (ip: string) { - return 'contact-form-' + ip - } - - private generateAPUnavailabilityKey (url: string) { - return 'ap-unavailability-' + sha256(url) - } - - /* ************ Redis helpers ************ */ - - private getValue (key: string) { - return this.client.get(this.prefix + key) - } - - private getSet (key: string) { - return this.client.smembers(this.prefix + key) - } - - private addToSet (key: string, value: string) { - return this.client.sadd(this.prefix + key, value) - } - - private deleteFromSet (key: string, value: string) { - return this.client.srem(this.prefix + key, value) - } - - private deleteKey (key: string) { - return this.client.del(this.prefix + key) - } - - private async getObject (key: string) { - const value = await this.getValue(key) - if (!value) return null - - return JSON.parse(value) - } - - private setObject (key: string, value: { [ id: string ]: number | string }, expirationMilliseconds?: number) { - return this.setValue(key, JSON.stringify(value), expirationMilliseconds) - } - - private async setValue (key: string, value: string, expirationMilliseconds?: number) { - const result = expirationMilliseconds !== undefined - ? await this.client.set(this.prefix + key, value, 'PX', expirationMilliseconds) - : await this.client.set(this.prefix + key, value) - - if (result !== 'OK') throw new Error('Redis set result is not OK.') - } - - private removeValue (key: string) { - return this.client.del(this.prefix + key) - } - - private increment (key: string) { - return this.client.incr(this.prefix + key) - } - - private async exists (key: string) { - const result = await this.client.exists(this.prefix + key) - - return result !== 0 - } - - private setExpiration (key: string, ms: number) { - return this.client.expire(this.prefix + key, ms / 1000) - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -// --------------------------------------------------------------------------- - -export { - Redis -} diff --git a/server/lib/redundancy.ts b/server/lib/redundancy.ts deleted file mode 100644 index 2613d01be..000000000 --- a/server/lib/redundancy.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { Transaction } from 'sequelize' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { ActorFollowModel } from '@server/models/actor/actor-follow' -import { getServerActor } from '@server/models/application/application' -import { MActorSignature, MVideoRedundancyVideo } from '@server/types/models' -import { Activity } from '@shared/models' -import { VideoRedundancyModel } from '../models/redundancy/video-redundancy' -import { sendUndoCacheFile } from './activitypub/send' - -const lTags = loggerTagsFactory('redundancy') - -async function removeVideoRedundancy (videoRedundancy: MVideoRedundancyVideo, t?: Transaction) { - const serverActor = await getServerActor() - - // Local cache, send undo to remote instances - if (videoRedundancy.actorId === serverActor.id) await sendUndoCacheFile(serverActor, videoRedundancy, t) - - await videoRedundancy.destroy({ transaction: t }) -} - -async function removeRedundanciesOfServer (serverId: number) { - const redundancies = await VideoRedundancyModel.listLocalOfServer(serverId) - - for (const redundancy of redundancies) { - await removeVideoRedundancy(redundancy) - } -} - -async function isRedundancyAccepted (activity: Activity, byActor: MActorSignature) { - const configAcceptFrom = CONFIG.REMOTE_REDUNDANCY.VIDEOS.ACCEPT_FROM - if (configAcceptFrom === 'nobody') { - logger.info('Do not accept remote redundancy %s due instance accept policy.', activity.id, lTags()) - return false - } - - if (configAcceptFrom === 'followings') { - const serverActor = await getServerActor() - const allowed = await ActorFollowModel.isFollowedBy(byActor.id, serverActor.id) - - if (allowed !== true) { - logger.info( - 'Do not accept remote redundancy %s because actor %s is not followed by our instance.', - activity.id, byActor.url, lTags() - ) - return false - } - } - - return true -} - -// --------------------------------------------------------------------------- - -export { - isRedundancyAccepted, - removeRedundanciesOfServer, - removeVideoRedundancy -} diff --git a/server/lib/runners/index.ts b/server/lib/runners/index.ts deleted file mode 100644 index a737c7b59..000000000 --- a/server/lib/runners/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './job-handlers' -export * from './runner' -export * from './runner-urls' diff --git a/server/lib/runners/job-handlers/abstract-job-handler.ts b/server/lib/runners/job-handlers/abstract-job-handler.ts deleted file mode 100644 index 329977de1..000000000 --- a/server/lib/runners/job-handlers/abstract-job-handler.ts +++ /dev/null @@ -1,269 +0,0 @@ -import { throttle } from 'lodash' -import { saveInTransactionWithRetries } from '@server/helpers/database-utils' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { RUNNER_JOBS } from '@server/initializers/constants' -import { sequelizeTypescript } from '@server/initializers/database' -import { PeerTubeSocket } from '@server/lib/peertube-socket' -import { RunnerJobModel } from '@server/models/runner/runner-job' -import { setAsUpdated } from '@server/models/shared' -import { MRunnerJob } from '@server/types/models/runners' -import { pick } from '@shared/core-utils' -import { - RunnerJobLiveRTMPHLSTranscodingPayload, - RunnerJobLiveRTMPHLSTranscodingPrivatePayload, - RunnerJobState, - RunnerJobStudioTranscodingPayload, - RunnerJobSuccessPayload, - RunnerJobType, - RunnerJobUpdatePayload, - RunnerJobVideoStudioTranscodingPrivatePayload, - RunnerJobVODAudioMergeTranscodingPayload, - RunnerJobVODAudioMergeTranscodingPrivatePayload, - RunnerJobVODHLSTranscodingPayload, - RunnerJobVODHLSTranscodingPrivatePayload, - RunnerJobVODWebVideoTranscodingPayload, - RunnerJobVODWebVideoTranscodingPrivatePayload -} from '@shared/models' - -type CreateRunnerJobArg = - { - type: Extract - payload: RunnerJobVODWebVideoTranscodingPayload - privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload - } | - { - type: Extract - payload: RunnerJobVODHLSTranscodingPayload - privatePayload: RunnerJobVODHLSTranscodingPrivatePayload - } | - { - type: Extract - payload: RunnerJobVODAudioMergeTranscodingPayload - privatePayload: RunnerJobVODAudioMergeTranscodingPrivatePayload - } | - { - type: Extract - payload: RunnerJobLiveRTMPHLSTranscodingPayload - privatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload - } | - { - type: Extract - payload: RunnerJobStudioTranscodingPayload - privatePayload: RunnerJobVideoStudioTranscodingPrivatePayload - } - -export abstract class AbstractJobHandler { - - protected readonly lTags = loggerTagsFactory('runner') - - static setJobAsUpdatedThrottled = throttle(setAsUpdated, 2000) - - // --------------------------------------------------------------------------- - - abstract create (options: C): Promise - - protected async createRunnerJob (options: CreateRunnerJobArg & { - jobUUID: string - priority: number - dependsOnRunnerJob?: MRunnerJob - }): Promise { - const { priority, dependsOnRunnerJob } = options - - logger.debug('Creating runner job', { options, ...this.lTags(options.type) }) - - const runnerJob = new RunnerJobModel({ - ...pick(options, [ 'type', 'payload', 'privatePayload' ]), - - uuid: options.jobUUID, - - state: dependsOnRunnerJob - ? RunnerJobState.WAITING_FOR_PARENT_JOB - : RunnerJobState.PENDING, - - dependsOnRunnerJobId: dependsOnRunnerJob?.id, - - priority - }) - - const job = await sequelizeTypescript.transaction(async transaction => { - return runnerJob.save({ transaction }) - }) - - if (runnerJob.state === RunnerJobState.PENDING) { - PeerTubeSocket.Instance.sendAvailableJobsPingToRunners() - } - - return job - } - - // --------------------------------------------------------------------------- - - protected abstract specificUpdate (options: { - runnerJob: MRunnerJob - updatePayload?: U - }): Promise | void - - async update (options: { - runnerJob: MRunnerJob - progress?: number - updatePayload?: U - }) { - const { runnerJob, progress } = options - - await this.specificUpdate(options) - - if (progress) runnerJob.progress = progress - - if (!runnerJob.changed()) { - try { - await AbstractJobHandler.setJobAsUpdatedThrottled({ sequelize: sequelizeTypescript, table: 'runnerJob', id: runnerJob.id }) - } catch (err) { - logger.warn('Cannot set remote job as updated', { err, ...this.lTags(runnerJob.id, runnerJob.type) }) - } - - return - } - - await saveInTransactionWithRetries(runnerJob) - } - - // --------------------------------------------------------------------------- - - async complete (options: { - runnerJob: MRunnerJob - resultPayload: S - }) { - const { runnerJob } = options - - runnerJob.state = RunnerJobState.COMPLETING - await saveInTransactionWithRetries(runnerJob) - - try { - await this.specificComplete(options) - - runnerJob.state = RunnerJobState.COMPLETED - } catch (err) { - logger.error('Cannot complete runner job', { err, ...this.lTags(runnerJob.id, runnerJob.type) }) - - runnerJob.state = RunnerJobState.ERRORED - runnerJob.error = err.message - } - - runnerJob.progress = null - runnerJob.finishedAt = new Date() - - await saveInTransactionWithRetries(runnerJob) - - const [ affectedCount ] = await RunnerJobModel.updateDependantJobsOf(runnerJob) - - if (affectedCount !== 0) PeerTubeSocket.Instance.sendAvailableJobsPingToRunners() - } - - protected abstract specificComplete (options: { - runnerJob: MRunnerJob - resultPayload: S - }): Promise | void - - // --------------------------------------------------------------------------- - - async cancel (options: { - runnerJob: MRunnerJob - fromParent?: boolean - }) { - const { runnerJob, fromParent } = options - - await this.specificCancel(options) - - const cancelState = fromParent - ? RunnerJobState.PARENT_CANCELLED - : RunnerJobState.CANCELLED - - runnerJob.setToErrorOrCancel(cancelState) - - await saveInTransactionWithRetries(runnerJob) - - const children = await RunnerJobModel.listChildrenOf(runnerJob) - for (const child of children) { - logger.info(`Cancelling child job ${child.uuid} of ${runnerJob.uuid} because of parent cancel`, this.lTags(child.uuid)) - - await this.cancel({ runnerJob: child, fromParent: true }) - } - } - - protected abstract specificCancel (options: { - runnerJob: MRunnerJob - }): Promise | void - - // --------------------------------------------------------------------------- - - protected abstract isAbortSupported (): boolean - - async abort (options: { - runnerJob: MRunnerJob - }) { - const { runnerJob } = options - - if (this.isAbortSupported() !== true) { - return this.error({ runnerJob, message: 'Job has been aborted but it is not supported by this job type' }) - } - - await this.specificAbort(options) - - runnerJob.resetToPending() - - await saveInTransactionWithRetries(runnerJob) - } - - protected setAbortState (runnerJob: MRunnerJob) { - runnerJob.resetToPending() - } - - protected abstract specificAbort (options: { - runnerJob: MRunnerJob - }): Promise | void - - // --------------------------------------------------------------------------- - - async error (options: { - runnerJob: MRunnerJob - message: string - fromParent?: boolean - }) { - const { runnerJob, message, fromParent } = options - - const errorState = fromParent - ? RunnerJobState.PARENT_ERRORED - : RunnerJobState.ERRORED - - const nextState = errorState === RunnerJobState.ERRORED && this.isAbortSupported() && runnerJob.failures < RUNNER_JOBS.MAX_FAILURES - ? RunnerJobState.PENDING - : errorState - - await this.specificError({ ...options, nextState }) - - if (nextState === errorState) { - runnerJob.setToErrorOrCancel(nextState) - runnerJob.error = message - } else { - runnerJob.resetToPending() - } - - await saveInTransactionWithRetries(runnerJob) - - if (runnerJob.state === errorState) { - const children = await RunnerJobModel.listChildrenOf(runnerJob) - - for (const child of children) { - logger.info(`Erroring child job ${child.uuid} of ${runnerJob.uuid} because of parent error`, this.lTags(child.uuid)) - - await this.error({ runnerJob: child, message: 'Parent error', fromParent: true }) - } - } - } - - protected abstract specificError (options: { - runnerJob: MRunnerJob - message: string - nextState: RunnerJobState - }): Promise | void -} diff --git a/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts b/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts deleted file mode 100644 index f425828d9..000000000 --- a/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts +++ /dev/null @@ -1,66 +0,0 @@ - -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { logger } from '@server/helpers/logger' -import { moveToFailedTranscodingState, moveToNextState } from '@server/lib/video-state' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { MRunnerJob } from '@server/types/models/runners' -import { RunnerJobState, RunnerJobSuccessPayload, RunnerJobUpdatePayload, RunnerJobVODPrivatePayload } from '@shared/models' -import { AbstractJobHandler } from './abstract-job-handler' -import { loadTranscodingRunnerVideo } from './shared' - -// eslint-disable-next-line max-len -export abstract class AbstractVODTranscodingJobHandler extends AbstractJobHandler { - - protected isAbortSupported () { - return true - } - - protected specificUpdate (_options: { - runnerJob: MRunnerJob - }) { - // empty - } - - protected specificAbort (_options: { - runnerJob: MRunnerJob - }) { - // empty - } - - protected async specificError (options: { - runnerJob: MRunnerJob - nextState: RunnerJobState - }) { - if (options.nextState !== RunnerJobState.ERRORED) return - - const video = await loadTranscodingRunnerVideo(options.runnerJob, this.lTags) - if (!video) return - - await moveToFailedTranscodingState(video) - - await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') - } - - protected async specificCancel (options: { - runnerJob: MRunnerJob - }) { - const { runnerJob } = options - - const video = await loadTranscodingRunnerVideo(options.runnerJob, this.lTags) - if (!video) return - - const pending = await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') - - logger.debug(`Pending transcode decreased to ${pending} after cancel`, this.lTags(video.uuid)) - - if (pending === 0) { - logger.info( - `All transcoding jobs of ${video.uuid} have been processed or canceled, moving it to its next state`, - this.lTags(video.uuid) - ) - - const privatePayload = runnerJob.privatePayload as RunnerJobVODPrivatePayload - await retryTransactionWrapper(moveToNextState, { video, isNewVideo: privatePayload.isNewVideo }) - } - } -} diff --git a/server/lib/runners/job-handlers/index.ts b/server/lib/runners/job-handlers/index.ts deleted file mode 100644 index 40ad2f97a..000000000 --- a/server/lib/runners/job-handlers/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export * from './abstract-job-handler' -export * from './live-rtmp-hls-transcoding-job-handler' -export * from './runner-job-handlers' -export * from './video-studio-transcoding-job-handler' -export * from './vod-audio-merge-transcoding-job-handler' -export * from './vod-hls-transcoding-job-handler' -export * from './vod-web-video-transcoding-job-handler' diff --git a/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts b/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts deleted file mode 100644 index 6b2894f8c..000000000 --- a/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts +++ /dev/null @@ -1,173 +0,0 @@ -import { move, remove } from 'fs-extra' -import { join } from 'path' -import { logger } from '@server/helpers/logger' -import { JOB_PRIORITY } from '@server/initializers/constants' -import { LiveManager } from '@server/lib/live' -import { MStreamingPlaylist, MVideo } from '@server/types/models' -import { MRunnerJob } from '@server/types/models/runners' -import { buildUUID } from '@shared/extra-utils' -import { - LiveRTMPHLSTranscodingSuccess, - LiveRTMPHLSTranscodingUpdatePayload, - LiveVideoError, - RunnerJobLiveRTMPHLSTranscodingPayload, - RunnerJobLiveRTMPHLSTranscodingPrivatePayload, - RunnerJobState -} from '@shared/models' -import { AbstractJobHandler } from './abstract-job-handler' - -type CreateOptions = { - video: MVideo - playlist: MStreamingPlaylist - - sessionId: string - rtmpUrl: string - - toTranscode: { - resolution: number - fps: number - }[] - - segmentListSize: number - segmentDuration: number - - outputDirectory: string -} - -// eslint-disable-next-line max-len -export class LiveRTMPHLSTranscodingJobHandler extends AbstractJobHandler { - - async create (options: CreateOptions) { - const { video, rtmpUrl, toTranscode, playlist, segmentDuration, segmentListSize, outputDirectory, sessionId } = options - - const jobUUID = buildUUID() - const payload: RunnerJobLiveRTMPHLSTranscodingPayload = { - input: { - rtmpUrl - }, - output: { - toTranscode, - segmentListSize, - segmentDuration - } - } - - const privatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload = { - videoUUID: video.uuid, - masterPlaylistName: playlist.playlistFilename, - sessionId, - outputDirectory - } - - const job = await this.createRunnerJob({ - type: 'live-rtmp-hls-transcoding', - jobUUID, - payload, - privatePayload, - priority: JOB_PRIORITY.TRANSCODING - }) - - return job - } - - // --------------------------------------------------------------------------- - - protected async specificUpdate (options: { - runnerJob: MRunnerJob - updatePayload: LiveRTMPHLSTranscodingUpdatePayload - }) { - const { runnerJob, updatePayload } = options - - const privatePayload = runnerJob.privatePayload as RunnerJobLiveRTMPHLSTranscodingPrivatePayload - const outputDirectory = privatePayload.outputDirectory - const videoUUID = privatePayload.videoUUID - - // Always process the chunk first before moving m3u8 that references this chunk - if (updatePayload.type === 'add-chunk') { - await move( - updatePayload.videoChunkFile as string, - join(outputDirectory, updatePayload.videoChunkFilename), - { overwrite: true } - ) - } else if (updatePayload.type === 'remove-chunk') { - await remove(join(outputDirectory, updatePayload.videoChunkFilename)) - } - - if (updatePayload.resolutionPlaylistFile && updatePayload.resolutionPlaylistFilename) { - await move( - updatePayload.resolutionPlaylistFile as string, - join(outputDirectory, updatePayload.resolutionPlaylistFilename), - { overwrite: true } - ) - } - - if (updatePayload.masterPlaylistFile) { - await move(updatePayload.masterPlaylistFile as string, join(outputDirectory, privatePayload.masterPlaylistName), { overwrite: true }) - } - - logger.debug( - 'Runner live RTMP to HLS job %s for %s updated.', - runnerJob.uuid, videoUUID, { updatePayload, ...this.lTags(videoUUID, runnerJob.uuid) } - ) - } - - // --------------------------------------------------------------------------- - - protected specificComplete (options: { - runnerJob: MRunnerJob - }) { - return this.stopLive({ - runnerJob: options.runnerJob, - type: 'ended' - }) - } - - // --------------------------------------------------------------------------- - - protected isAbortSupported () { - return false - } - - protected specificAbort () { - throw new Error('Not implemented') - } - - protected specificError (options: { - runnerJob: MRunnerJob - nextState: RunnerJobState - }) { - return this.stopLive({ - runnerJob: options.runnerJob, - type: 'errored' - }) - } - - protected specificCancel (options: { - runnerJob: MRunnerJob - }) { - return this.stopLive({ - runnerJob: options.runnerJob, - type: 'cancelled' - }) - } - - private stopLive (options: { - runnerJob: MRunnerJob - type: 'ended' | 'errored' | 'cancelled' - }) { - const { runnerJob, type } = options - - const privatePayload = runnerJob.privatePayload as RunnerJobLiveRTMPHLSTranscodingPrivatePayload - const videoUUID = privatePayload.videoUUID - - const errorType = { - ended: null, - errored: LiveVideoError.RUNNER_JOB_ERROR, - cancelled: LiveVideoError.RUNNER_JOB_CANCEL - } - - LiveManager.Instance.stopSessionOf(privatePayload.videoUUID, errorType[type]) - - logger.info('Runner live RTMP to HLS job %s for video %s %s.', runnerJob.uuid, videoUUID, type, this.lTags(runnerJob.uuid, videoUUID)) - } -} diff --git a/server/lib/runners/job-handlers/runner-job-handlers.ts b/server/lib/runners/job-handlers/runner-job-handlers.ts deleted file mode 100644 index 85551c365..000000000 --- a/server/lib/runners/job-handlers/runner-job-handlers.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { MRunnerJob } from '@server/types/models/runners' -import { RunnerJobSuccessPayload, RunnerJobType, RunnerJobUpdatePayload } from '@shared/models' -import { AbstractJobHandler } from './abstract-job-handler' -import { LiveRTMPHLSTranscodingJobHandler } from './live-rtmp-hls-transcoding-job-handler' -import { VideoStudioTranscodingJobHandler } from './video-studio-transcoding-job-handler' -import { VODAudioMergeTranscodingJobHandler } from './vod-audio-merge-transcoding-job-handler' -import { VODHLSTranscodingJobHandler } from './vod-hls-transcoding-job-handler' -import { VODWebVideoTranscodingJobHandler } from './vod-web-video-transcoding-job-handler' - -const processors: Record AbstractJobHandler> = { - 'vod-web-video-transcoding': VODWebVideoTranscodingJobHandler, - 'vod-hls-transcoding': VODHLSTranscodingJobHandler, - 'vod-audio-merge-transcoding': VODAudioMergeTranscodingJobHandler, - 'live-rtmp-hls-transcoding': LiveRTMPHLSTranscodingJobHandler, - 'video-studio-transcoding': VideoStudioTranscodingJobHandler -} - -export function getRunnerJobHandlerClass (job: MRunnerJob) { - return processors[job.type] -} diff --git a/server/lib/runners/job-handlers/shared/index.ts b/server/lib/runners/job-handlers/shared/index.ts deleted file mode 100644 index 348273ae2..000000000 --- a/server/lib/runners/job-handlers/shared/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './vod-helpers' diff --git a/server/lib/runners/job-handlers/shared/vod-helpers.ts b/server/lib/runners/job-handlers/shared/vod-helpers.ts deleted file mode 100644 index 1a2ad02ca..000000000 --- a/server/lib/runners/job-handlers/shared/vod-helpers.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { move } from 'fs-extra' -import { dirname, join } from 'path' -import { logger, LoggerTagsFn } from '@server/helpers/logger' -import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding' -import { onWebVideoFileTranscoding } from '@server/lib/transcoding/web-transcoding' -import { buildNewFile } from '@server/lib/video-file' -import { VideoModel } from '@server/models/video/video' -import { MVideoFullLight } from '@server/types/models' -import { MRunnerJob } from '@server/types/models/runners' -import { RunnerJobVODAudioMergeTranscodingPrivatePayload, RunnerJobVODWebVideoTranscodingPrivatePayload } from '@shared/models' - -export async function onVODWebVideoOrAudioMergeTranscodingJob (options: { - video: MVideoFullLight - videoFilePath: string - privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload | RunnerJobVODAudioMergeTranscodingPrivatePayload -}) { - const { video, videoFilePath, privatePayload } = options - - const videoFile = await buildNewFile({ path: videoFilePath, mode: 'web-video' }) - videoFile.videoId = video.id - - const newVideoFilePath = join(dirname(videoFilePath), videoFile.filename) - await move(videoFilePath, newVideoFilePath) - - await onWebVideoFileTranscoding({ - video, - videoFile, - videoOutputPath: newVideoFilePath - }) - - await onTranscodingEnded({ isNewVideo: privatePayload.isNewVideo, moveVideoToNextState: true, video }) -} - -export async function loadTranscodingRunnerVideo (runnerJob: MRunnerJob, lTags: LoggerTagsFn) { - const videoUUID = runnerJob.privatePayload.videoUUID - - const video = await VideoModel.loadFull(videoUUID) - if (!video) { - logger.info('Video %s does not exist anymore after transcoding runner job.', videoUUID, lTags(videoUUID)) - return undefined - } - - return video -} diff --git a/server/lib/runners/job-handlers/video-studio-transcoding-job-handler.ts b/server/lib/runners/job-handlers/video-studio-transcoding-job-handler.ts deleted file mode 100644 index f604382b7..000000000 --- a/server/lib/runners/job-handlers/video-studio-transcoding-job-handler.ts +++ /dev/null @@ -1,157 +0,0 @@ - -import { basename } from 'path' -import { logger } from '@server/helpers/logger' -import { onVideoStudioEnded, safeCleanupStudioTMPFiles } from '@server/lib/video-studio' -import { MVideo } from '@server/types/models' -import { MRunnerJob } from '@server/types/models/runners' -import { buildUUID } from '@shared/extra-utils' -import { - isVideoStudioTaskIntro, - isVideoStudioTaskOutro, - isVideoStudioTaskWatermark, - RunnerJobState, - RunnerJobUpdatePayload, - RunnerJobStudioTranscodingPayload, - RunnerJobVideoStudioTranscodingPrivatePayload, - VideoStudioTranscodingSuccess, - VideoState, - VideoStudioTaskPayload -} from '@shared/models' -import { generateRunnerEditionTranscodingVideoInputFileUrl, generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls' -import { AbstractJobHandler } from './abstract-job-handler' -import { loadTranscodingRunnerVideo } from './shared' - -type CreateOptions = { - video: MVideo - tasks: VideoStudioTaskPayload[] - priority: number -} - -// eslint-disable-next-line max-len -export class VideoStudioTranscodingJobHandler extends AbstractJobHandler { - - async create (options: CreateOptions) { - const { video, priority, tasks } = options - - const jobUUID = buildUUID() - const payload: RunnerJobStudioTranscodingPayload = { - input: { - videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid) - }, - tasks: tasks.map(t => { - if (isVideoStudioTaskIntro(t) || isVideoStudioTaskOutro(t)) { - return { - ...t, - - options: { - ...t.options, - - file: generateRunnerEditionTranscodingVideoInputFileUrl(jobUUID, video.uuid, basename(t.options.file)) - } - } - } - - if (isVideoStudioTaskWatermark(t)) { - return { - ...t, - - options: { - ...t.options, - - file: generateRunnerEditionTranscodingVideoInputFileUrl(jobUUID, video.uuid, basename(t.options.file)) - } - } - } - - return t - }) - } - - const privatePayload: RunnerJobVideoStudioTranscodingPrivatePayload = { - videoUUID: video.uuid, - originalTasks: tasks - } - - const job = await this.createRunnerJob({ - type: 'video-studio-transcoding', - jobUUID, - payload, - privatePayload, - priority - }) - - return job - } - - // --------------------------------------------------------------------------- - - protected isAbortSupported () { - return true - } - - protected specificUpdate (_options: { - runnerJob: MRunnerJob - }) { - // empty - } - - protected specificAbort (_options: { - runnerJob: MRunnerJob - }) { - // empty - } - - protected async specificComplete (options: { - runnerJob: MRunnerJob - resultPayload: VideoStudioTranscodingSuccess - }) { - const { runnerJob, resultPayload } = options - const privatePayload = runnerJob.privatePayload as RunnerJobVideoStudioTranscodingPrivatePayload - - const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) - if (!video) { - await safeCleanupStudioTMPFiles(privatePayload.originalTasks) - - } - - const videoFilePath = resultPayload.videoFile as string - - await onVideoStudioEnded({ video, editionResultPath: videoFilePath, tasks: privatePayload.originalTasks }) - - logger.info( - 'Runner video edition transcoding job %s for %s ended.', - runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid) - ) - } - - protected specificError (options: { - runnerJob: MRunnerJob - nextState: RunnerJobState - }) { - if (options.nextState === RunnerJobState.ERRORED) { - return this.specificErrorOrCancel(options) - } - - return Promise.resolve() - } - - protected specificCancel (options: { - runnerJob: MRunnerJob - }) { - return this.specificErrorOrCancel(options) - } - - private async specificErrorOrCancel (options: { - runnerJob: MRunnerJob - }) { - const { runnerJob } = options - - const payload = runnerJob.privatePayload as RunnerJobVideoStudioTranscodingPrivatePayload - await safeCleanupStudioTMPFiles(payload.originalTasks) - - const video = await loadTranscodingRunnerVideo(options.runnerJob, this.lTags) - if (!video) return - - return video.setNewState(VideoState.PUBLISHED, false, undefined) - } -} diff --git a/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts deleted file mode 100644 index 137a94535..000000000 --- a/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { MVideo } from '@server/types/models' -import { MRunnerJob } from '@server/types/models/runners' -import { pick } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { getVideoStreamDuration } from '@shared/ffmpeg' -import { - RunnerJobUpdatePayload, - RunnerJobVODAudioMergeTranscodingPayload, - RunnerJobVODWebVideoTranscodingPrivatePayload, - VODAudioMergeTranscodingSuccess -} from '@shared/models' -import { generateRunnerTranscodingVideoInputFileUrl, generateRunnerTranscodingVideoPreviewFileUrl } from '../runner-urls' -import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler' -import { loadTranscodingRunnerVideo, onVODWebVideoOrAudioMergeTranscodingJob } from './shared' - -type CreateOptions = { - video: MVideo - isNewVideo: boolean - resolution: number - fps: number - priority: number - dependsOnRunnerJob?: MRunnerJob -} - -// eslint-disable-next-line max-len -export class VODAudioMergeTranscodingJobHandler extends AbstractVODTranscodingJobHandler { - - async create (options: CreateOptions) { - const { video, resolution, fps, priority, dependsOnRunnerJob } = options - - const jobUUID = buildUUID() - const payload: RunnerJobVODAudioMergeTranscodingPayload = { - input: { - audioFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid), - previewFileUrl: generateRunnerTranscodingVideoPreviewFileUrl(jobUUID, video.uuid) - }, - output: { - resolution, - fps - } - } - - const privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload = { - ...pick(options, [ 'isNewVideo' ]), - - videoUUID: video.uuid - } - - const job = await this.createRunnerJob({ - type: 'vod-audio-merge-transcoding', - jobUUID, - payload, - privatePayload, - priority, - dependsOnRunnerJob - }) - - await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode') - - return job - } - - // --------------------------------------------------------------------------- - - protected async specificComplete (options: { - runnerJob: MRunnerJob - resultPayload: VODAudioMergeTranscodingSuccess - }) { - const { runnerJob, resultPayload } = options - const privatePayload = runnerJob.privatePayload as RunnerJobVODWebVideoTranscodingPrivatePayload - - const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) - if (!video) return - - const videoFilePath = resultPayload.videoFile as string - - // ffmpeg generated a new video file, so update the video duration - // See https://trac.ffmpeg.org/ticket/5456 - video.duration = await getVideoStreamDuration(videoFilePath) - await video.save() - - // We can remove the old audio file - const oldAudioFile = video.VideoFiles[0] - await video.removeWebVideoFile(oldAudioFile) - await oldAudioFile.destroy() - video.VideoFiles = [] - - await onVODWebVideoOrAudioMergeTranscodingJob({ video, videoFilePath, privatePayload }) - - logger.info( - 'Runner VOD audio merge transcoding job %s for %s ended.', - runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid) - ) - } -} diff --git a/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts deleted file mode 100644 index 02845952c..000000000 --- a/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { move } from 'fs-extra' -import { dirname, join } from 'path' -import { logger } from '@server/helpers/logger' -import { renameVideoFileInPlaylist } from '@server/lib/hls' -import { getHlsResolutionPlaylistFilename } from '@server/lib/paths' -import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding' -import { onHLSVideoFileTranscoding } from '@server/lib/transcoding/hls-transcoding' -import { buildNewFile, removeAllWebVideoFiles } from '@server/lib/video-file' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { MVideo } from '@server/types/models' -import { MRunnerJob } from '@server/types/models/runners' -import { pick } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { - RunnerJobUpdatePayload, - RunnerJobVODHLSTranscodingPayload, - RunnerJobVODHLSTranscodingPrivatePayload, - VODHLSTranscodingSuccess -} from '@shared/models' -import { generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls' -import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler' -import { loadTranscodingRunnerVideo } from './shared' - -type CreateOptions = { - video: MVideo - isNewVideo: boolean - deleteWebVideoFiles: boolean - resolution: number - fps: number - priority: number - dependsOnRunnerJob?: MRunnerJob -} - -// eslint-disable-next-line max-len -export class VODHLSTranscodingJobHandler extends AbstractVODTranscodingJobHandler { - - async create (options: CreateOptions) { - const { video, resolution, fps, dependsOnRunnerJob, priority } = options - - const jobUUID = buildUUID() - - const payload: RunnerJobVODHLSTranscodingPayload = { - input: { - videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid) - }, - output: { - resolution, - fps - } - } - - const privatePayload: RunnerJobVODHLSTranscodingPrivatePayload = { - ...pick(options, [ 'isNewVideo', 'deleteWebVideoFiles' ]), - - videoUUID: video.uuid - } - - const job = await this.createRunnerJob({ - type: 'vod-hls-transcoding', - jobUUID, - payload, - privatePayload, - priority, - dependsOnRunnerJob - }) - - await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode') - - return job - } - - // --------------------------------------------------------------------------- - - protected async specificComplete (options: { - runnerJob: MRunnerJob - resultPayload: VODHLSTranscodingSuccess - }) { - const { runnerJob, resultPayload } = options - const privatePayload = runnerJob.privatePayload as RunnerJobVODHLSTranscodingPrivatePayload - - const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) - if (!video) return - - const videoFilePath = resultPayload.videoFile as string - const resolutionPlaylistFilePath = resultPayload.resolutionPlaylistFile as string - - const videoFile = await buildNewFile({ path: videoFilePath, mode: 'hls' }) - const newVideoFilePath = join(dirname(videoFilePath), videoFile.filename) - await move(videoFilePath, newVideoFilePath) - - const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFile.filename) - const newResolutionPlaylistFilePath = join(dirname(resolutionPlaylistFilePath), resolutionPlaylistFilename) - await move(resolutionPlaylistFilePath, newResolutionPlaylistFilePath) - - await renameVideoFileInPlaylist(newResolutionPlaylistFilePath, videoFile.filename) - - await onHLSVideoFileTranscoding({ - video, - videoFile, - m3u8OutputPath: newResolutionPlaylistFilePath, - videoOutputPath: newVideoFilePath - }) - - await onTranscodingEnded({ isNewVideo: privatePayload.isNewVideo, moveVideoToNextState: true, video }) - - if (privatePayload.deleteWebVideoFiles === true) { - logger.info('Removing web video files of %s now we have a HLS version of it.', video.uuid, this.lTags(video.uuid)) - - await removeAllWebVideoFiles(video) - } - - logger.info('Runner VOD HLS job %s for %s ended.', runnerJob.uuid, video.uuid, this.lTags(runnerJob.uuid, video.uuid)) - } -} diff --git a/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts b/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts deleted file mode 100644 index 9ee8ab88e..000000000 --- a/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { MVideo } from '@server/types/models' -import { MRunnerJob } from '@server/types/models/runners' -import { pick } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { - RunnerJobUpdatePayload, - RunnerJobVODWebVideoTranscodingPayload, - RunnerJobVODWebVideoTranscodingPrivatePayload, - VODWebVideoTranscodingSuccess -} from '@shared/models' -import { generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls' -import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler' -import { loadTranscodingRunnerVideo, onVODWebVideoOrAudioMergeTranscodingJob } from './shared' - -type CreateOptions = { - video: MVideo - isNewVideo: boolean - resolution: number - fps: number - priority: number - dependsOnRunnerJob?: MRunnerJob -} - -// eslint-disable-next-line max-len -export class VODWebVideoTranscodingJobHandler extends AbstractVODTranscodingJobHandler { - - async create (options: CreateOptions) { - const { video, resolution, fps, priority, dependsOnRunnerJob } = options - - const jobUUID = buildUUID() - const payload: RunnerJobVODWebVideoTranscodingPayload = { - input: { - videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid) - }, - output: { - resolution, - fps - } - } - - const privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload = { - ...pick(options, [ 'isNewVideo' ]), - - videoUUID: video.uuid - } - - const job = await this.createRunnerJob({ - type: 'vod-web-video-transcoding', - jobUUID, - payload, - privatePayload, - dependsOnRunnerJob, - priority - }) - - await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode') - - return job - } - - // --------------------------------------------------------------------------- - - protected async specificComplete (options: { - runnerJob: MRunnerJob - resultPayload: VODWebVideoTranscodingSuccess - }) { - const { runnerJob, resultPayload } = options - const privatePayload = runnerJob.privatePayload as RunnerJobVODWebVideoTranscodingPrivatePayload - - const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) - if (!video) return - - const videoFilePath = resultPayload.videoFile as string - - await onVODWebVideoOrAudioMergeTranscodingJob({ video, videoFilePath, privatePayload }) - - logger.info( - 'Runner VOD web video transcoding job %s for %s ended.', - runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid) - ) - } -} diff --git a/server/lib/runners/runner-urls.ts b/server/lib/runners/runner-urls.ts deleted file mode 100644 index a27060b33..000000000 --- a/server/lib/runners/runner-urls.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { WEBSERVER } from '@server/initializers/constants' - -export function generateRunnerTranscodingVideoInputFileUrl (jobUUID: string, videoUUID: string) { - return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/max-quality' -} - -export function generateRunnerTranscodingVideoPreviewFileUrl (jobUUID: string, videoUUID: string) { - return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/previews/max-quality' -} - -export function generateRunnerEditionTranscodingVideoInputFileUrl (jobUUID: string, videoUUID: string, filename: string) { - return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/studio/task-files/' + filename -} diff --git a/server/lib/runners/runner.ts b/server/lib/runners/runner.ts deleted file mode 100644 index 947fdb3f0..000000000 --- a/server/lib/runners/runner.ts +++ /dev/null @@ -1,49 +0,0 @@ -import express from 'express' -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { sequelizeTypescript } from '@server/initializers/database' -import { MRunner, MRunnerJob } from '@server/types/models/runners' -import { RUNNER_JOBS } from '@server/initializers/constants' -import { RunnerJobState } from '@shared/models' - -const lTags = loggerTagsFactory('runner') - -const updatingRunner = new Set() - -function updateLastRunnerContact (req: express.Request, runner: MRunner) { - const now = new Date() - - // Don't update last runner contact too often - if (now.getTime() - runner.lastContact.getTime() < RUNNER_JOBS.LAST_CONTACT_UPDATE_INTERVAL) return - if (updatingRunner.has(runner.id)) return - - updatingRunner.add(runner.id) - - runner.lastContact = now - runner.ip = req.ip - - logger.debug('Updating last runner contact for %s', runner.name, lTags(runner.name)) - - retryTransactionWrapper(() => { - return sequelizeTypescript.transaction(async transaction => { - return runner.save({ transaction }) - }) - }) - .catch(err => logger.error('Cannot update last runner contact for %s', runner.name, { err, ...lTags(runner.name) })) - .finally(() => updatingRunner.delete(runner.id)) -} - -function runnerJobCanBeCancelled (runnerJob: MRunnerJob) { - const allowedStates = new Set([ - RunnerJobState.PENDING, - RunnerJobState.PROCESSING, - RunnerJobState.WAITING_FOR_PARENT_JOB - ]) - - return allowedStates.has(runnerJob.state) -} - -export { - updateLastRunnerContact, - runnerJobCanBeCancelled -} diff --git a/server/lib/schedulers/abstract-scheduler.ts b/server/lib/schedulers/abstract-scheduler.ts deleted file mode 100644 index f3d51a22e..000000000 --- a/server/lib/schedulers/abstract-scheduler.ts +++ /dev/null @@ -1,35 +0,0 @@ -import Bluebird from 'bluebird' -import { logger } from '../../helpers/logger' - -export abstract class AbstractScheduler { - - protected abstract schedulerIntervalMs: number - - private interval: NodeJS.Timer - private isRunning = false - - enable () { - if (!this.schedulerIntervalMs) throw new Error('Interval is not correctly set.') - - this.interval = setInterval(() => this.execute(), this.schedulerIntervalMs) - } - - disable () { - clearInterval(this.interval) - } - - async execute () { - if (this.isRunning === true) return - this.isRunning = true - - try { - await this.internalExecute() - } catch (err) { - logger.error('Cannot execute %s scheduler.', this.constructor.name, { err }) - } finally { - this.isRunning = false - } - } - - protected abstract internalExecute (): Promise | Bluebird -} diff --git a/server/lib/schedulers/actor-follow-scheduler.ts b/server/lib/schedulers/actor-follow-scheduler.ts deleted file mode 100644 index e1c56c135..000000000 --- a/server/lib/schedulers/actor-follow-scheduler.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { isTestOrDevInstance } from '../../helpers/core-utils' -import { logger } from '../../helpers/logger' -import { ACTOR_FOLLOW_SCORE, SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { ActorFollowModel } from '../../models/actor/actor-follow' -import { ActorFollowHealthCache } from '../actor-follow-health-cache' -import { AbstractScheduler } from './abstract-scheduler' - -export class ActorFollowScheduler extends AbstractScheduler { - - private static instance: AbstractScheduler - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.ACTOR_FOLLOW_SCORES - - private constructor () { - super() - } - - protected async internalExecute () { - await this.processPendingScores() - - await this.removeBadActorFollows() - } - - private async processPendingScores () { - const pendingScores = ActorFollowHealthCache.Instance.getPendingFollowsScore() - const badServerIds = ActorFollowHealthCache.Instance.getBadFollowingServerIds() - const goodServerIds = ActorFollowHealthCache.Instance.getGoodFollowingServerIds() - - ActorFollowHealthCache.Instance.clearPendingFollowsScore() - ActorFollowHealthCache.Instance.clearBadFollowingServerIds() - ActorFollowHealthCache.Instance.clearGoodFollowingServerIds() - - for (const inbox of Object.keys(pendingScores)) { - await ActorFollowModel.updateScore(inbox, pendingScores[inbox]) - } - - await ActorFollowModel.updateScoreByFollowingServers(badServerIds, ACTOR_FOLLOW_SCORE.PENALTY) - await ActorFollowModel.updateScoreByFollowingServers(goodServerIds, ACTOR_FOLLOW_SCORE.BONUS) - } - - private async removeBadActorFollows () { - if (!isTestOrDevInstance()) logger.info('Removing bad actor follows (scheduler).') - - try { - await ActorFollowModel.removeBadActorFollows() - } catch (err) { - logger.error('Error in bad actor follows scheduler.', { err }) - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/auto-follow-index-instances.ts b/server/lib/schedulers/auto-follow-index-instances.ts deleted file mode 100644 index 956ece749..000000000 --- a/server/lib/schedulers/auto-follow-index-instances.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { chunk } from 'lodash' -import { doJSONRequest } from '@server/helpers/requests' -import { JobQueue } from '@server/lib/job-queue' -import { ActorFollowModel } from '@server/models/actor/actor-follow' -import { getServerActor } from '@server/models/application/application' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { SCHEDULER_INTERVALS_MS, SERVER_ACTOR_NAME } from '../../initializers/constants' -import { AbstractScheduler } from './abstract-scheduler' - -export class AutoFollowIndexInstances extends AbstractScheduler { - - private static instance: AbstractScheduler - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.AUTO_FOLLOW_INDEX_INSTANCES - - private lastCheck: Date - - private constructor () { - super() - } - - protected async internalExecute () { - return this.autoFollow() - } - - private async autoFollow () { - if (CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.ENABLED === false) return - - const indexUrl = CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.INDEX_URL - - logger.info('Auto follow instances of index %s.', indexUrl) - - try { - const serverActor = await getServerActor() - - const searchParams = { count: 1000 } - if (this.lastCheck) Object.assign(searchParams, { since: this.lastCheck.toISOString() }) - - this.lastCheck = new Date() - - const { body } = await doJSONRequest(indexUrl, { searchParams }) - if (!body.data || Array.isArray(body.data) === false) { - logger.error('Cannot auto follow instances of index %s. Please check the auto follow URL.', indexUrl, { body }) - return - } - - const hosts: string[] = body.data.map(o => o.host) - const chunks = chunk(hosts, 20) - - for (const chunk of chunks) { - const unfollowedHosts = await ActorFollowModel.keepUnfollowedInstance(chunk) - - for (const unfollowedHost of unfollowedHosts) { - const payload = { - host: unfollowedHost, - name: SERVER_ACTOR_NAME, - followerActorId: serverActor.id, - isAutoFollow: true - } - - JobQueue.Instance.createJobAsync({ type: 'activitypub-follow', payload }) - } - } - - } catch (err) { - logger.error('Cannot auto follow hosts of index %s.', indexUrl, { err }) - } - - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/geo-ip-update-scheduler.ts b/server/lib/schedulers/geo-ip-update-scheduler.ts deleted file mode 100644 index b06f5a9b5..000000000 --- a/server/lib/schedulers/geo-ip-update-scheduler.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { GeoIP } from '@server/helpers/geo-ip' -import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { AbstractScheduler } from './abstract-scheduler' - -export class GeoIPUpdateScheduler extends AbstractScheduler { - - private static instance: AbstractScheduler - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.GEO_IP_UPDATE - - private constructor () { - super() - } - - protected internalExecute () { - return GeoIP.Instance.updateDatabase() - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/peertube-version-check-scheduler.ts b/server/lib/schedulers/peertube-version-check-scheduler.ts deleted file mode 100644 index bc38ed49f..000000000 --- a/server/lib/schedulers/peertube-version-check-scheduler.ts +++ /dev/null @@ -1,55 +0,0 @@ - -import { doJSONRequest } from '@server/helpers/requests' -import { ApplicationModel } from '@server/models/application/application' -import { compareSemVer } from '@shared/core-utils' -import { JoinPeerTubeVersions } from '@shared/models' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { PEERTUBE_VERSION, SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { Notifier } from '../notifier' -import { AbstractScheduler } from './abstract-scheduler' - -export class PeerTubeVersionCheckScheduler extends AbstractScheduler { - - private static instance: AbstractScheduler - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHECK_PEERTUBE_VERSION - - private constructor () { - super() - } - - protected async internalExecute () { - return this.checkLatestVersion() - } - - private async checkLatestVersion () { - if (CONFIG.PEERTUBE.CHECK_LATEST_VERSION.ENABLED === false) return - - logger.info('Checking latest PeerTube version.') - - const { body } = await doJSONRequest(CONFIG.PEERTUBE.CHECK_LATEST_VERSION.URL) - - if (!body?.peertube?.latestVersion) { - logger.warn('Cannot check latest PeerTube version: body is invalid.', { body }) - return - } - - const latestVersion = body.peertube.latestVersion - const application = await ApplicationModel.load() - - // Already checked this version - if (application.latestPeerTubeVersion === latestVersion) return - - if (compareSemVer(PEERTUBE_VERSION, latestVersion) < 0) { - application.latestPeerTubeVersion = latestVersion - await application.save() - - Notifier.Instance.notifyOfNewPeerTubeVersion(application, latestVersion) - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/plugins-check-scheduler.ts b/server/lib/schedulers/plugins-check-scheduler.ts deleted file mode 100644 index 820c01693..000000000 --- a/server/lib/schedulers/plugins-check-scheduler.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { chunk } from 'lodash' -import { compareSemVer } from '@shared/core-utils' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { PluginModel } from '../../models/server/plugin' -import { Notifier } from '../notifier' -import { getLatestPluginsVersion } from '../plugins/plugin-index' -import { AbstractScheduler } from './abstract-scheduler' - -export class PluginsCheckScheduler extends AbstractScheduler { - - private static instance: AbstractScheduler - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHECK_PLUGINS - - private constructor () { - super() - } - - protected async internalExecute () { - return this.checkLatestPluginsVersion() - } - - private async checkLatestPluginsVersion () { - if (CONFIG.PLUGINS.INDEX.ENABLED === false) return - - logger.info('Checking latest plugins version.') - - const plugins = await PluginModel.listInstalled() - - // Process 10 plugins in 1 HTTP request - const chunks = chunk(plugins, 10) - for (const chunk of chunks) { - // Find plugins according to their npm name - const pluginIndex: { [npmName: string]: PluginModel } = {} - for (const plugin of chunk) { - pluginIndex[PluginModel.buildNpmName(plugin.name, plugin.type)] = plugin - } - - const npmNames = Object.keys(pluginIndex) - - try { - const results = await getLatestPluginsVersion(npmNames) - - for (const result of results) { - const plugin = pluginIndex[result.npmName] - if (!result.latestVersion) continue - - if ( - !plugin.latestVersion || - (plugin.latestVersion !== result.latestVersion && compareSemVer(plugin.latestVersion, result.latestVersion) < 0) - ) { - plugin.latestVersion = result.latestVersion - await plugin.save() - - // Notify if there is an higher plugin version available - if (compareSemVer(plugin.version, result.latestVersion) < 0) { - Notifier.Instance.notifyOfNewPluginVersion(plugin) - } - - logger.info('Plugin %s has a new latest version %s.', result.npmName, plugin.latestVersion) - } - } - } catch (err) { - logger.error('Cannot get latest plugins version.', { npmNames, err }) - } - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts b/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts deleted file mode 100644 index 61e93eafa..000000000 --- a/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts +++ /dev/null @@ -1,40 +0,0 @@ - -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants' -import { uploadx } from '../uploadx' -import { AbstractScheduler } from './abstract-scheduler' - -const lTags = loggerTagsFactory('scheduler', 'resumable-upload', 'cleaner') - -export class RemoveDanglingResumableUploadsScheduler extends AbstractScheduler { - - private static instance: AbstractScheduler - private lastExecutionTimeMs: number - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.REMOVE_DANGLING_RESUMABLE_UPLOADS - - private constructor () { - super() - - this.lastExecutionTimeMs = new Date().getTime() - } - - protected async internalExecute () { - logger.debug('Removing dangling resumable uploads', lTags()) - - const now = new Date().getTime() - - try { - // Remove files that were not updated since the last execution - await uploadx.storage.purge(now - this.lastExecutionTimeMs) - } catch (error) { - logger.error('Failed to handle file during resumable video upload folder cleanup', { error, ...lTags() }) - } finally { - this.lastExecutionTimeMs = now - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/remove-old-history-scheduler.ts b/server/lib/schedulers/remove-old-history-scheduler.ts deleted file mode 100644 index 34b160799..000000000 --- a/server/lib/schedulers/remove-old-history-scheduler.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { logger } from '../../helpers/logger' -import { AbstractScheduler } from './abstract-scheduler' -import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { UserVideoHistoryModel } from '../../models/user/user-video-history' -import { CONFIG } from '../../initializers/config' - -export class RemoveOldHistoryScheduler extends AbstractScheduler { - - private static instance: AbstractScheduler - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.REMOVE_OLD_HISTORY - - private constructor () { - super() - } - - protected internalExecute () { - if (CONFIG.HISTORY.VIDEOS.MAX_AGE === -1) return - - logger.info('Removing old videos history.') - - const now = new Date() - const beforeDate = new Date(now.getTime() - CONFIG.HISTORY.VIDEOS.MAX_AGE).toISOString() - - return UserVideoHistoryModel.removeOldHistory(beforeDate) - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/remove-old-views-scheduler.ts b/server/lib/schedulers/remove-old-views-scheduler.ts deleted file mode 100644 index 8bc53a045..000000000 --- a/server/lib/schedulers/remove-old-views-scheduler.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { VideoViewModel } from '@server/models/view/video-view' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { AbstractScheduler } from './abstract-scheduler' - -export class RemoveOldViewsScheduler extends AbstractScheduler { - - private static instance: AbstractScheduler - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.REMOVE_OLD_VIEWS - - private constructor () { - super() - } - - protected internalExecute () { - if (CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE === -1) return - - logger.info('Removing old videos views.') - - const now = new Date() - const beforeDate = new Date(now.getTime() - CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE).toISOString() - - return VideoViewModel.removeOldRemoteViewsHistory(beforeDate) - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/runner-job-watch-dog-scheduler.ts b/server/lib/schedulers/runner-job-watch-dog-scheduler.ts deleted file mode 100644 index f7a26d2bc..000000000 --- a/server/lib/schedulers/runner-job-watch-dog-scheduler.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { CONFIG } from '@server/initializers/config' -import { RunnerJobModel } from '@server/models/runner/runner-job' -import { logger, loggerTagsFactory } from '../../helpers/logger' -import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { getRunnerJobHandlerClass } from '../runners' -import { AbstractScheduler } from './abstract-scheduler' - -const lTags = loggerTagsFactory('runner') - -export class RunnerJobWatchDogScheduler extends AbstractScheduler { - - private static instance: AbstractScheduler - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.RUNNER_JOB_WATCH_DOG - - private constructor () { - super() - } - - protected async internalExecute () { - const vodStalledJobs = await RunnerJobModel.listStalledJobs({ - staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.VOD, - types: [ 'vod-audio-merge-transcoding', 'vod-hls-transcoding', 'vod-web-video-transcoding' ] - }) - - const liveStalledJobs = await RunnerJobModel.listStalledJobs({ - staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.LIVE, - types: [ 'live-rtmp-hls-transcoding' ] - }) - - for (const stalled of [ ...vodStalledJobs, ...liveStalledJobs ]) { - logger.info('Abort stalled runner job %s (%s)', stalled.uuid, stalled.type, lTags(stalled.uuid, stalled.type)) - - const Handler = getRunnerJobHandlerClass(stalled) - await new Handler().abort({ runnerJob: stalled }) - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts deleted file mode 100644 index e38685c04..000000000 --- a/server/lib/schedulers/update-videos-scheduler.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { VideoModel } from '@server/models/video/video' -import { MScheduleVideoUpdate } from '@server/types/models' -import { VideoPrivacy, VideoState } from '@shared/models' -import { logger } from '../../helpers/logger' -import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { sequelizeTypescript } from '../../initializers/database' -import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update' -import { Notifier } from '../notifier' -import { addVideoJobsAfterUpdate } from '../video' -import { VideoPathManager } from '../video-path-manager' -import { setVideoPrivacy } from '../video-privacy' -import { AbstractScheduler } from './abstract-scheduler' - -export class UpdateVideosScheduler extends AbstractScheduler { - - private static instance: AbstractScheduler - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.UPDATE_VIDEOS - - private constructor () { - super() - } - - protected async internalExecute () { - return this.updateVideos() - } - - private async updateVideos () { - if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined - - const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate() - - for (const schedule of schedules) { - const videoOnly = await VideoModel.load(schedule.videoId) - const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoOnly.uuid) - - try { - const { video, published } = await this.updateAVideo(schedule) - - if (published) Notifier.Instance.notifyOnVideoPublishedAfterScheduledUpdate(video) - } catch (err) { - logger.error('Cannot update video', { err }) - } - - mutexReleaser() - } - } - - private async updateAVideo (schedule: MScheduleVideoUpdate) { - let oldPrivacy: VideoPrivacy - let isNewVideo: boolean - let published = false - - const video = await sequelizeTypescript.transaction(async t => { - const video = await VideoModel.loadFull(schedule.videoId, t) - if (video.state === VideoState.TO_TRANSCODE) return null - - logger.info('Executing scheduled video update on %s.', video.uuid) - - if (schedule.privacy) { - isNewVideo = video.isNewVideo(schedule.privacy) - oldPrivacy = video.privacy - - setVideoPrivacy(video, schedule.privacy) - await video.save({ transaction: t }) - - if (oldPrivacy === VideoPrivacy.PRIVATE) { - published = true - } - } - - await schedule.destroy({ transaction: t }) - - return video - }) - - if (!video) { - return { video, published: false } - } - - await addVideoJobsAfterUpdate({ video, oldPrivacy, isNewVideo, nameChanged: false }) - - return { video, published } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts b/server/lib/schedulers/video-channel-sync-latest-scheduler.ts deleted file mode 100644 index efb957fac..000000000 --- a/server/lib/schedulers/video-channel-sync-latest-scheduler.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { VideoChannelModel } from '@server/models/video/video-channel' -import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' -import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { synchronizeChannel } from '../sync-channel' -import { AbstractScheduler } from './abstract-scheduler' - -export class VideoChannelSyncLatestScheduler extends AbstractScheduler { - private static instance: AbstractScheduler - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHANNEL_SYNC_CHECK_INTERVAL - - private constructor () { - super() - } - - protected async internalExecute () { - if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) { - logger.debug('Discard channels synchronization as the feature is disabled') - return - } - - logger.info('Checking channels to synchronize') - - const channelSyncs = await VideoChannelSyncModel.listSyncs() - - for (const sync of channelSyncs) { - const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId) - - logger.info( - 'Creating video import jobs for "%s" sync with external channel "%s"', - channel.Actor.preferredUsername, sync.externalChannelUrl - ) - - const onlyAfter = sync.lastSyncAt || sync.createdAt - - await synchronizeChannel({ - channel, - externalChannelUrl: sync.externalChannelUrl, - videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION, - channelSync: sync, - onlyAfter - }) - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/video-views-buffer-scheduler.ts b/server/lib/schedulers/video-views-buffer-scheduler.ts deleted file mode 100644 index 244a88b14..000000000 --- a/server/lib/schedulers/video-views-buffer-scheduler.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { VideoModel } from '@server/models/video/video' -import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { federateVideoIfNeeded } from '../activitypub/videos' -import { Redis } from '../redis' -import { AbstractScheduler } from './abstract-scheduler' - -const lTags = loggerTagsFactory('views') - -export class VideoViewsBufferScheduler extends AbstractScheduler { - - private static instance: AbstractScheduler - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.VIDEO_VIEWS_BUFFER_UPDATE - - private constructor () { - super() - } - - protected async internalExecute () { - const videoIds = await Redis.Instance.listLocalVideosViewed() - if (videoIds.length === 0) return - - for (const videoId of videoIds) { - try { - const views = await Redis.Instance.getLocalVideoViews(videoId) - await Redis.Instance.deleteLocalVideoViews(videoId) - - const video = await VideoModel.loadFull(videoId) - if (!video) { - logger.debug('Video %d does not exist anymore, skipping videos view addition.', videoId, lTags()) - continue - } - - logger.info('Processing local video %s views buffer.', video.uuid, lTags(video.uuid)) - - // If this is a remote video, the origin instance will send us an update - await VideoModel.incrementViews(videoId, views) - - // Send video update - video.views += views - await federateVideoIfNeeded(video, false) - } catch (err) { - logger.error('Cannot process local video views buffer of video %d.', videoId, { err, ...lTags() }) - } - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts deleted file mode 100644 index 91625ccb5..000000000 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ /dev/null @@ -1,375 +0,0 @@ -import { move } from 'fs-extra' -import { join } from 'path' -import { getServerActor } from '@server/models/application/application' -import { VideoModel } from '@server/models/video/video' -import { - MStreamingPlaylistFiles, - MVideoAccountLight, - MVideoFile, - MVideoFileVideo, - MVideoRedundancyFileVideo, - MVideoRedundancyStreamingPlaylistVideo, - MVideoRedundancyVideo, - MVideoWithAllFiles -} from '@server/types/models' -import { VideosRedundancyStrategy } from '../../../shared/models/redundancy' -import { logger, loggerTagsFactory } from '../../helpers/logger' -import { downloadWebTorrentVideo } from '../../helpers/webtorrent' -import { CONFIG } from '../../initializers/config' -import { DIRECTORIES, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants' -import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' -import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send' -import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url' -import { getOrCreateAPVideo } from '../activitypub/videos' -import { downloadPlaylistSegments } from '../hls' -import { removeVideoRedundancy } from '../redundancy' -import { generateHLSRedundancyUrl, generateWebVideoRedundancyUrl } from '../video-urls' -import { AbstractScheduler } from './abstract-scheduler' - -const lTags = loggerTagsFactory('redundancy') - -type CandidateToDuplicate = { - redundancy: VideosRedundancyStrategy - video: MVideoWithAllFiles - files: MVideoFile[] - streamingPlaylists: MStreamingPlaylistFiles[] -} - -function isMVideoRedundancyFileVideo ( - o: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo -): o is MVideoRedundancyFileVideo { - return !!(o as MVideoRedundancyFileVideo).VideoFile -} - -export class VideosRedundancyScheduler extends AbstractScheduler { - - private static instance: VideosRedundancyScheduler - - protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL - - private constructor () { - super() - } - - async createManualRedundancy (videoId: number) { - const videoToDuplicate = await VideoModel.loadWithFiles(videoId) - - if (!videoToDuplicate) { - logger.warn('Video to manually duplicate %d does not exist anymore.', videoId, lTags()) - return - } - - return this.createVideoRedundancies({ - video: videoToDuplicate, - redundancy: null, - files: videoToDuplicate.VideoFiles, - streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists - }) - } - - protected async internalExecute () { - for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { - logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy, lTags()) - - try { - const videoToDuplicate = await this.findVideoToDuplicate(redundancyConfig) - if (!videoToDuplicate) continue - - const candidateToDuplicate = { - video: videoToDuplicate, - redundancy: redundancyConfig, - files: videoToDuplicate.VideoFiles, - streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists - } - - await this.purgeCacheIfNeeded(candidateToDuplicate) - - if (await this.isTooHeavy(candidateToDuplicate)) { - logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url, lTags(videoToDuplicate.uuid)) - continue - } - - logger.info( - 'Will duplicate video %s in redundancy scheduler "%s".', - videoToDuplicate.url, redundancyConfig.strategy, lTags(videoToDuplicate.uuid) - ) - - await this.createVideoRedundancies(candidateToDuplicate) - } catch (err) { - logger.error('Cannot run videos redundancy %s.', redundancyConfig.strategy, { err, ...lTags() }) - } - } - - await this.extendsLocalExpiration() - - await this.purgeRemoteExpired() - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } - - private async extendsLocalExpiration () { - const expired = await VideoRedundancyModel.listLocalExpired() - - for (const redundancyModel of expired) { - try { - const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) - - // If the admin disabled the redundancy, remove this redundancy instead of extending it - if (!redundancyConfig) { - logger.info( - 'Destroying redundancy %s because the redundancy %s does not exist anymore.', - redundancyModel.url, redundancyModel.strategy - ) - - await removeVideoRedundancy(redundancyModel) - continue - } - - const { totalUsed } = await VideoRedundancyModel.getStats(redundancyConfig.strategy) - - // If the admin decreased the cache size, remove this redundancy instead of extending it - if (totalUsed > redundancyConfig.size) { - logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy) - - await removeVideoRedundancy(redundancyModel) - continue - } - - await this.extendsRedundancy(redundancyModel) - } catch (err) { - logger.error( - 'Cannot extend or remove expiration of %s video from our redundancy system.', - this.buildEntryLogId(redundancyModel), { err, ...lTags(redundancyModel.getVideoUUID()) } - ) - } - } - } - - private async extendsRedundancy (redundancyModel: MVideoRedundancyVideo) { - const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) - // Redundancy strategy disabled, remove our redundancy instead of extending expiration - if (!redundancy) { - await removeVideoRedundancy(redundancyModel) - return - } - - await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime) - } - - private async purgeRemoteExpired () { - const expired = await VideoRedundancyModel.listRemoteExpired() - - for (const redundancyModel of expired) { - try { - await removeVideoRedundancy(redundancyModel) - } catch (err) { - logger.error( - 'Cannot remove redundancy %s from our redundancy system.', - this.buildEntryLogId(redundancyModel), lTags(redundancyModel.getVideoUUID()) - ) - } - } - } - - private findVideoToDuplicate (cache: VideosRedundancyStrategy) { - if (cache.strategy === 'most-views') { - return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) - } - - if (cache.strategy === 'trending') { - return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) - } - - if (cache.strategy === 'recently-added') { - const minViews = cache.minViews - return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews) - } - } - - private async createVideoRedundancies (data: CandidateToDuplicate) { - const video = await this.loadAndRefreshVideo(data.video.url) - - if (!video) { - logger.info('Video %s we want to duplicate does not existing anymore, skipping.', data.video.url, lTags(data.video.uuid)) - - return - } - - for (const file of data.files) { - const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) - if (existingRedundancy) { - await this.extendsRedundancy(existingRedundancy) - - continue - } - - await this.createVideoFileRedundancy(data.redundancy, video, file) - } - - for (const streamingPlaylist of data.streamingPlaylists) { - const existingRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(streamingPlaylist.id) - if (existingRedundancy) { - await this.extendsRedundancy(existingRedundancy) - - continue - } - - await this.createStreamingPlaylistRedundancy(data.redundancy, video, streamingPlaylist) - } - } - - private async createVideoFileRedundancy (redundancy: VideosRedundancyStrategy | null, video: MVideoAccountLight, fileArg: MVideoFile) { - let strategy = 'manual' - let expiresOn: Date = null - - if (redundancy) { - strategy = redundancy.strategy - expiresOn = this.buildNewExpiration(redundancy.minLifetime) - } - - const file = fileArg as MVideoFileVideo - file.Video = video - - const serverActor = await getServerActor() - - logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy, lTags(video.uuid)) - - const tmpPath = await downloadWebTorrentVideo({ uri: file.torrentUrl }, VIDEO_IMPORT_TIMEOUT) - - const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, file.filename) - await move(tmpPath, destPath, { overwrite: true }) - - const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({ - expiresOn, - url: getLocalVideoCacheFileActivityPubUrl(file), - fileUrl: generateWebVideoRedundancyUrl(file), - strategy, - videoFileId: file.id, - actorId: serverActor.id - }) - - createdModel.VideoFile = file - - await sendCreateCacheFile(serverActor, video, createdModel) - - logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url, lTags(video.uuid)) - } - - private async createStreamingPlaylistRedundancy ( - redundancy: VideosRedundancyStrategy, - video: MVideoAccountLight, - playlistArg: MStreamingPlaylistFiles - ) { - let strategy = 'manual' - let expiresOn: Date = null - - if (redundancy) { - strategy = redundancy.strategy - expiresOn = this.buildNewExpiration(redundancy.minLifetime) - } - - const playlist = Object.assign(playlistArg, { Video: video }) - const serverActor = await getServerActor() - - logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy, lTags(video.uuid)) - - const destDirectory = join(DIRECTORIES.HLS_REDUNDANCY, video.uuid) - const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video) - - const maxSizeKB = this.getTotalFileSizes([], [ playlist ]) / 1000 - const toleranceKB = maxSizeKB + ((5 * maxSizeKB) / 100) // 5% more tolerance - await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT, toleranceKB) - - const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ - expiresOn, - url: getLocalVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), - fileUrl: generateHLSRedundancyUrl(video, playlistArg), - strategy, - videoStreamingPlaylistId: playlist.id, - actorId: serverActor.id - }) - - createdModel.VideoStreamingPlaylist = playlist - - await sendCreateCacheFile(serverActor, video, createdModel) - - logger.info('Duplicated playlist %s -> %s.', masterPlaylistUrl, createdModel.url, lTags(video.uuid)) - } - - private async extendsExpirationOf (redundancy: MVideoRedundancyVideo, expiresAfterMs: number) { - logger.info('Extending expiration of %s.', redundancy.url, lTags(redundancy.getVideoUUID())) - - const serverActor = await getServerActor() - - redundancy.expiresOn = this.buildNewExpiration(expiresAfterMs) - await redundancy.save() - - await sendUpdateCacheFile(serverActor, redundancy) - } - - private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) { - while (await this.isTooHeavy(candidateToDuplicate)) { - const redundancy = candidateToDuplicate.redundancy - const toDelete = await VideoRedundancyModel.loadOldestLocalExpired(redundancy.strategy, redundancy.minLifetime) - if (!toDelete) return - - const videoId = toDelete.VideoFile - ? toDelete.VideoFile.videoId - : toDelete.VideoStreamingPlaylist.videoId - - const redundancies = await VideoRedundancyModel.listLocalByVideoId(videoId) - - for (const redundancy of redundancies) { - await removeVideoRedundancy(redundancy) - } - } - } - - private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) { - const maxSize = candidateToDuplicate.redundancy.size - - const { totalUsed: alreadyUsed } = await VideoRedundancyModel.getStats(candidateToDuplicate.redundancy.strategy) - - const videoSize = this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists) - const willUse = alreadyUsed + videoSize - - logger.debug('Checking candidate size.', { maxSize, alreadyUsed, videoSize, willUse, ...lTags(candidateToDuplicate.video.uuid) }) - - return willUse > maxSize - } - - private buildNewExpiration (expiresAfterMs: number) { - return new Date(Date.now() + expiresAfterMs) - } - - private buildEntryLogId (object: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo) { - if (isMVideoRedundancyFileVideo(object)) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` - - return `${object.VideoStreamingPlaylist.getMasterPlaylistUrl(object.VideoStreamingPlaylist.Video)}` - } - - private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]): number { - const fileReducer = (previous: number, current: MVideoFile) => previous + current.size - - let allFiles = files - for (const p of playlists) { - allFiles = allFiles.concat(p.VideoFiles) - } - - return allFiles.reduce(fileReducer, 0) - } - - private async loadAndRefreshVideo (videoUrl: string) { - // We need more attributes and check if the video still exists - const getVideoOptions = { - videoObject: videoUrl, - syncParam: { rates: false, shares: false, comments: false, refreshVideo: true }, - fetchType: 'all' as 'all' - } - const { video } = await getOrCreateAPVideo(getVideoOptions) - - return video - } -} diff --git a/server/lib/schedulers/youtube-dl-update-scheduler.ts b/server/lib/schedulers/youtube-dl-update-scheduler.ts deleted file mode 100644 index 1ee4ae1b2..000000000 --- a/server/lib/schedulers/youtube-dl-update-scheduler.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { YoutubeDLCLI } from '@server/helpers/youtube-dl' -import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants' -import { AbstractScheduler } from './abstract-scheduler' - -export class YoutubeDlUpdateScheduler extends AbstractScheduler { - - private static instance: AbstractScheduler - - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.YOUTUBE_DL_UPDATE - - private constructor () { - super() - } - - protected internalExecute () { - return YoutubeDLCLI.updateYoutubeDLBinary() - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/search.ts b/server/lib/search.ts deleted file mode 100644 index b3363fbec..000000000 --- a/server/lib/search.ts +++ /dev/null @@ -1,49 +0,0 @@ -import express from 'express' -import { CONFIG } from '@server/initializers/config' -import { AccountBlocklistModel } from '@server/models/account/account-blocklist' -import { getServerActor } from '@server/models/application/application' -import { ServerBlocklistModel } from '@server/models/server/server-blocklist' -import { SearchTargetQuery } from '@shared/models' - -function isSearchIndexSearch (query: SearchTargetQuery) { - if (query.searchTarget === 'search-index') return true - - const searchIndexConfig = CONFIG.SEARCH.SEARCH_INDEX - - if (searchIndexConfig.ENABLED !== true) return false - - if (searchIndexConfig.IS_DEFAULT_SEARCH && !query.searchTarget) return true - - return false -} - -async function buildMutedForSearchIndex (res: express.Response) { - const serverActor = await getServerActor() - const accountIds = [ serverActor.Account.id ] - - if (res.locals.oauth) { - accountIds.push(res.locals.oauth.token.User.Account.id) - } - - const [ blockedHosts, blockedAccounts ] = await Promise.all([ - ServerBlocklistModel.listHostsBlockedBy(accountIds), - AccountBlocklistModel.listHandlesBlockedBy(accountIds) - ]) - - return { - blockedHosts, - blockedAccounts - } -} - -function isURISearch (search: string) { - if (!search) return false - - return search.startsWith('http://') || search.startsWith('https://') -} - -export { - isSearchIndexSearch, - buildMutedForSearchIndex, - isURISearch -} diff --git a/server/lib/server-config-manager.ts b/server/lib/server-config-manager.ts deleted file mode 100644 index beb5d4d82..000000000 --- a/server/lib/server-config-manager.ts +++ /dev/null @@ -1,384 +0,0 @@ -import { getServerCommit } from '@server/helpers/version' -import { CONFIG, isEmailEnabled } from '@server/initializers/config' -import { CONSTRAINTS_FIELDS, DEFAULT_THEME_NAME, PEERTUBE_VERSION } from '@server/initializers/constants' -import { isSignupAllowed, isSignupAllowedForCurrentIP } from '@server/lib/signup' -import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' -import { PluginModel } from '@server/models/server/plugin' -import { HTMLServerConfig, RegisteredExternalAuthConfig, RegisteredIdAndPassAuthConfig, ServerConfig } from '@shared/models' -import { Hooks } from './plugins/hooks' -import { PluginManager } from './plugins/plugin-manager' -import { getThemeOrDefault } from './plugins/theme-utils' -import { VideoTranscodingProfilesManager } from './transcoding/default-transcoding-profiles' - -/** - * - * Used to send the server config to clients (using REST/API or plugins API) - * We need a singleton class to manage config state depending on external events (to build menu entries etc) - * - */ - -class ServerConfigManager { - - private static instance: ServerConfigManager - - private serverCommit: string - - private homepageEnabled = false - - private constructor () {} - - async init () { - const instanceHomepage = await ActorCustomPageModel.loadInstanceHomepage() - - this.updateHomepageState(instanceHomepage?.content) - } - - updateHomepageState (content: string) { - this.homepageEnabled = !!content - } - - async getHTMLServerConfig (): Promise { - if (this.serverCommit === undefined) this.serverCommit = await getServerCommit() - - const defaultTheme = getThemeOrDefault(CONFIG.THEME.DEFAULT, DEFAULT_THEME_NAME) - - return { - client: { - videos: { - miniature: { - displayAuthorAvatar: CONFIG.CLIENT.VIDEOS.MINIATURE.DISPLAY_AUTHOR_AVATAR, - preferAuthorDisplayName: CONFIG.CLIENT.VIDEOS.MINIATURE.PREFER_AUTHOR_DISPLAY_NAME - }, - resumableUpload: { - maxChunkSize: CONFIG.CLIENT.VIDEOS.RESUMABLE_UPLOAD.MAX_CHUNK_SIZE - } - }, - menu: { - login: { - redirectOnSingleExternalAuth: CONFIG.CLIENT.MENU.LOGIN.REDIRECT_ON_SINGLE_EXTERNAL_AUTH - } - } - }, - - defaults: { - publish: { - downloadEnabled: CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED, - commentsEnabled: CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED, - privacy: CONFIG.DEFAULTS.PUBLISH.PRIVACY, - licence: CONFIG.DEFAULTS.PUBLISH.LICENCE - }, - p2p: { - webapp: { - enabled: CONFIG.DEFAULTS.P2P.WEBAPP.ENABLED - }, - embed: { - enabled: CONFIG.DEFAULTS.P2P.EMBED.ENABLED - } - } - }, - - webadmin: { - configuration: { - edition: { - allowed: CONFIG.WEBADMIN.CONFIGURATION.EDITION.ALLOWED - } - } - }, - - instance: { - name: CONFIG.INSTANCE.NAME, - shortDescription: CONFIG.INSTANCE.SHORT_DESCRIPTION, - isNSFW: CONFIG.INSTANCE.IS_NSFW, - defaultNSFWPolicy: CONFIG.INSTANCE.DEFAULT_NSFW_POLICY, - defaultClientRoute: CONFIG.INSTANCE.DEFAULT_CLIENT_ROUTE, - customizations: { - javascript: CONFIG.INSTANCE.CUSTOMIZATIONS.JAVASCRIPT, - css: CONFIG.INSTANCE.CUSTOMIZATIONS.CSS - } - }, - search: { - remoteUri: { - users: CONFIG.SEARCH.REMOTE_URI.USERS, - anonymous: CONFIG.SEARCH.REMOTE_URI.ANONYMOUS - }, - searchIndex: { - enabled: CONFIG.SEARCH.SEARCH_INDEX.ENABLED, - url: CONFIG.SEARCH.SEARCH_INDEX.URL, - disableLocalSearch: CONFIG.SEARCH.SEARCH_INDEX.DISABLE_LOCAL_SEARCH, - isDefaultSearch: CONFIG.SEARCH.SEARCH_INDEX.IS_DEFAULT_SEARCH - } - }, - plugin: { - registered: this.getRegisteredPlugins(), - registeredExternalAuths: this.getExternalAuthsPlugins(), - registeredIdAndPassAuths: this.getIdAndPassAuthPlugins() - }, - theme: { - registered: this.getRegisteredThemes(), - default: defaultTheme - }, - email: { - enabled: isEmailEnabled() - }, - contactForm: { - enabled: CONFIG.CONTACT_FORM.ENABLED - }, - serverVersion: PEERTUBE_VERSION, - serverCommit: this.serverCommit, - transcoding: { - remoteRunners: { - enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED - }, - hls: { - enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.HLS.ENABLED - }, - web_videos: { - enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED - }, - enabledResolutions: this.getEnabledResolutions('vod'), - profile: CONFIG.TRANSCODING.PROFILE, - availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('vod') - }, - live: { - enabled: CONFIG.LIVE.ENABLED, - - allowReplay: CONFIG.LIVE.ALLOW_REPLAY, - latencySetting: { - enabled: CONFIG.LIVE.LATENCY_SETTING.ENABLED - }, - - maxDuration: CONFIG.LIVE.MAX_DURATION, - maxInstanceLives: CONFIG.LIVE.MAX_INSTANCE_LIVES, - maxUserLives: CONFIG.LIVE.MAX_USER_LIVES, - - transcoding: { - enabled: CONFIG.LIVE.TRANSCODING.ENABLED, - remoteRunners: { - enabled: CONFIG.LIVE.TRANSCODING.ENABLED && CONFIG.LIVE.TRANSCODING.REMOTE_RUNNERS.ENABLED - }, - enabledResolutions: this.getEnabledResolutions('live'), - profile: CONFIG.LIVE.TRANSCODING.PROFILE, - availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('live') - }, - - rtmp: { - port: CONFIG.LIVE.RTMP.PORT - } - }, - videoStudio: { - enabled: CONFIG.VIDEO_STUDIO.ENABLED, - remoteRunners: { - enabled: CONFIG.VIDEO_STUDIO.REMOTE_RUNNERS.ENABLED - } - }, - videoFile: { - update: { - enabled: CONFIG.VIDEO_FILE.UPDATE.ENABLED - } - }, - import: { - videos: { - http: { - enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED - }, - torrent: { - enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED - } - }, - videoChannelSynchronization: { - enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED - } - }, - autoBlacklist: { - videos: { - ofUsers: { - enabled: CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED - } - } - }, - avatar: { - file: { - size: { - max: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max - }, - extensions: CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME - } - }, - banner: { - file: { - size: { - max: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max - }, - extensions: CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME - } - }, - video: { - image: { - extensions: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.EXTNAME, - size: { - max: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max - } - }, - file: { - extensions: CONSTRAINTS_FIELDS.VIDEOS.EXTNAME - } - }, - videoCaption: { - file: { - size: { - max: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max - }, - extensions: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.EXTNAME - } - }, - user: { - videoQuota: CONFIG.USER.VIDEO_QUOTA, - videoQuotaDaily: CONFIG.USER.VIDEO_QUOTA_DAILY - }, - videoChannels: { - maxPerUser: CONFIG.VIDEO_CHANNELS.MAX_PER_USER - }, - trending: { - videos: { - intervalDays: CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS, - algorithms: { - enabled: CONFIG.TRENDING.VIDEOS.ALGORITHMS.ENABLED, - default: CONFIG.TRENDING.VIDEOS.ALGORITHMS.DEFAULT - } - } - }, - tracker: { - enabled: CONFIG.TRACKER.ENABLED - }, - - followings: { - instance: { - autoFollowIndex: { - indexUrl: CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.INDEX_URL - } - } - }, - - broadcastMessage: { - enabled: CONFIG.BROADCAST_MESSAGE.ENABLED, - message: CONFIG.BROADCAST_MESSAGE.MESSAGE, - level: CONFIG.BROADCAST_MESSAGE.LEVEL, - dismissable: CONFIG.BROADCAST_MESSAGE.DISMISSABLE - }, - - homepage: { - enabled: this.homepageEnabled - } - } - } - - async getServerConfig (ip?: string): Promise { - const { allowed } = await Hooks.wrapPromiseFun( - isSignupAllowed, - - { - ip, - signupMode: CONFIG.SIGNUP.REQUIRES_APPROVAL - ? 'request-registration' - : 'direct-registration' - }, - - CONFIG.SIGNUP.REQUIRES_APPROVAL - ? 'filter:api.user.request-signup.allowed.result' - : 'filter:api.user.signup.allowed.result' - ) - - const allowedForCurrentIP = isSignupAllowedForCurrentIP(ip) - - const signup = { - allowed, - allowedForCurrentIP, - minimumAge: CONFIG.SIGNUP.MINIMUM_AGE, - requiresApproval: CONFIG.SIGNUP.REQUIRES_APPROVAL, - requiresEmailVerification: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION - } - - const htmlConfig = await this.getHTMLServerConfig() - - return { ...htmlConfig, signup } - } - - getRegisteredThemes () { - return PluginManager.Instance.getRegisteredThemes() - .map(t => ({ - npmName: PluginModel.buildNpmName(t.name, t.type), - name: t.name, - version: t.version, - description: t.description, - css: t.css, - clientScripts: t.clientScripts - })) - } - - getRegisteredPlugins () { - return PluginManager.Instance.getRegisteredPlugins() - .map(p => ({ - npmName: PluginModel.buildNpmName(p.name, p.type), - name: p.name, - version: p.version, - description: p.description, - clientScripts: p.clientScripts - })) - } - - getEnabledResolutions (type: 'vod' | 'live') { - const transcoding = type === 'vod' - ? CONFIG.TRANSCODING - : CONFIG.LIVE.TRANSCODING - - return Object.keys(transcoding.RESOLUTIONS) - .filter(key => transcoding.ENABLED && transcoding.RESOLUTIONS[key] === true) - .map(r => parseInt(r, 10)) - } - - private getIdAndPassAuthPlugins () { - const result: RegisteredIdAndPassAuthConfig[] = [] - - for (const p of PluginManager.Instance.getIdAndPassAuths()) { - for (const auth of p.idAndPassAuths) { - result.push({ - npmName: p.npmName, - name: p.name, - version: p.version, - authName: auth.authName, - weight: auth.getWeight() - }) - } - } - - return result - } - - private getExternalAuthsPlugins () { - const result: RegisteredExternalAuthConfig[] = [] - - for (const p of PluginManager.Instance.getExternalAuths()) { - for (const auth of p.externalAuths) { - result.push({ - npmName: p.npmName, - name: p.name, - version: p.version, - authName: auth.authName, - authDisplayName: auth.authDisplayName() - }) - } - } - - return result - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -// --------------------------------------------------------------------------- - -export { - ServerConfigManager -} diff --git a/server/lib/signup.ts b/server/lib/signup.ts deleted file mode 100644 index 6702c22cb..000000000 --- a/server/lib/signup.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { IPv4, IPv6, parse, subnetMatch } from 'ipaddr.js' -import { CONFIG } from '../initializers/config' -import { UserModel } from '../models/user/user' - -const isCidr = require('is-cidr') - -export type SignupMode = 'direct-registration' | 'request-registration' - -async function isSignupAllowed (options: { - signupMode: SignupMode - - ip: string // For plugins - body?: any -}): Promise<{ allowed: boolean, errorMessage?: string }> { - const { signupMode } = options - - if (CONFIG.SIGNUP.ENABLED === false) { - return { allowed: false, errorMessage: 'User registration is not allowed' } - } - - if (signupMode === 'direct-registration' && CONFIG.SIGNUP.REQUIRES_APPROVAL === true) { - return { allowed: false, errorMessage: 'User registration requires approval' } - } - - // No limit and signup is enabled - if (CONFIG.SIGNUP.LIMIT === -1) { - return { allowed: true } - } - - const totalUsers = await UserModel.countTotal() - - return { allowed: totalUsers < CONFIG.SIGNUP.LIMIT, errorMessage: 'User limit is reached on this instance' } -} - -function isSignupAllowedForCurrentIP (ip: string) { - if (!ip) return false - - const addr = parse(ip) - const excludeList = [ 'blacklist' ] - let matched = '' - - // if there is a valid, non-empty whitelist, we exclude all unknown addresses too - if (CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr(cidr)).length > 0) { - excludeList.push('unknown') - } - - if (addr.kind() === 'ipv4') { - const addrV4 = IPv4.parse(ip) - const rangeList = { - whitelist: CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr.v4(cidr)) - .map(cidr => IPv4.parseCIDR(cidr)), - blacklist: CONFIG.SIGNUP.FILTERS.CIDR.BLACKLIST.filter(cidr => isCidr.v4(cidr)) - .map(cidr => IPv4.parseCIDR(cidr)) - } - matched = subnetMatch(addrV4, rangeList, 'unknown') - } else if (addr.kind() === 'ipv6') { - const addrV6 = IPv6.parse(ip) - const rangeList = { - whitelist: CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr.v6(cidr)) - .map(cidr => IPv6.parseCIDR(cidr)), - blacklist: CONFIG.SIGNUP.FILTERS.CIDR.BLACKLIST.filter(cidr => isCidr.v6(cidr)) - .map(cidr => IPv6.parseCIDR(cidr)) - } - matched = subnetMatch(addrV6, rangeList, 'unknown') - } - - return !excludeList.includes(matched) -} - -// --------------------------------------------------------------------------- - -export { - isSignupAllowed, - isSignupAllowedForCurrentIP -} diff --git a/server/lib/stat-manager.ts b/server/lib/stat-manager.ts deleted file mode 100644 index 0516e7f1a..000000000 --- a/server/lib/stat-manager.ts +++ /dev/null @@ -1,182 +0,0 @@ -import { mapSeries } from 'bluebird' -import { CONFIG } from '@server/initializers/config' -import { ActorFollowModel } from '@server/models/actor/actor-follow' -import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy' -import { UserModel } from '@server/models/user/user' -import { VideoModel } from '@server/models/video/video' -import { VideoChannelModel } from '@server/models/video/video-channel' -import { VideoCommentModel } from '@server/models/video/video-comment' -import { VideoFileModel } from '@server/models/video/video-file' -import { VideoPlaylistModel } from '@server/models/video/video-playlist' -import { ActivityType, ServerStats, VideoRedundancyStrategyWithManual } from '@shared/models' - -class StatsManager { - - private static instance: StatsManager - - private readonly instanceStartDate = new Date() - - private readonly inboxMessages = { - processed: 0, - errors: 0, - successes: 0, - waiting: 0, - errorsPerType: this.buildAPPerType(), - successesPerType: this.buildAPPerType() - } - - private constructor () {} - - updateInboxWaiting (inboxMessagesWaiting: number) { - this.inboxMessages.waiting = inboxMessagesWaiting - } - - addInboxProcessedSuccess (type: ActivityType) { - this.inboxMessages.processed++ - this.inboxMessages.successes++ - this.inboxMessages.successesPerType[type]++ - } - - addInboxProcessedError (type: ActivityType) { - this.inboxMessages.processed++ - this.inboxMessages.errors++ - this.inboxMessages.errorsPerType[type]++ - } - - async getStats () { - const { totalLocalVideos, totalLocalVideoViews, totalVideos } = await VideoModel.getStats() - const { totalLocalVideoComments, totalVideoComments } = await VideoCommentModel.getStats() - const { totalUsers, totalDailyActiveUsers, totalWeeklyActiveUsers, totalMonthlyActiveUsers } = await UserModel.getStats() - const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats() - const { totalLocalVideoFilesSize } = await VideoFileModel.getStats() - const { - totalLocalVideoChannels, - totalLocalDailyActiveVideoChannels, - totalLocalWeeklyActiveVideoChannels, - totalLocalMonthlyActiveVideoChannels - } = await VideoChannelModel.getStats() - const { totalLocalPlaylists } = await VideoPlaylistModel.getStats() - - const videosRedundancyStats = await this.buildRedundancyStats() - - const data: ServerStats = { - totalUsers, - totalDailyActiveUsers, - totalWeeklyActiveUsers, - totalMonthlyActiveUsers, - - totalLocalVideos, - totalLocalVideoViews, - totalLocalVideoComments, - totalLocalVideoFilesSize, - - totalVideos, - totalVideoComments, - - totalLocalVideoChannels, - totalLocalDailyActiveVideoChannels, - totalLocalWeeklyActiveVideoChannels, - totalLocalMonthlyActiveVideoChannels, - - totalLocalPlaylists, - - totalInstanceFollowers, - totalInstanceFollowing, - - videosRedundancy: videosRedundancyStats, - - ...this.buildAPStats() - } - - return data - } - - private buildActivityPubMessagesProcessedPerSecond () { - const now = new Date() - const startedSeconds = (now.getTime() - this.instanceStartDate.getTime()) / 1000 - - return this.inboxMessages.processed / startedSeconds - } - - private buildRedundancyStats () { - const strategies = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES - .map(r => ({ - strategy: r.strategy as VideoRedundancyStrategyWithManual, - size: r.size - })) - - strategies.push({ strategy: 'manual', size: null }) - - return mapSeries(strategies, r => { - return VideoRedundancyModel.getStats(r.strategy) - .then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size })) - }) - } - - private buildAPPerType () { - return { - Create: 0, - Update: 0, - Delete: 0, - Follow: 0, - Accept: 0, - Reject: 0, - Announce: 0, - Undo: 0, - Like: 0, - Dislike: 0, - Flag: 0, - View: 0 - } - } - - private buildAPStats () { - return { - totalActivityPubMessagesProcessed: this.inboxMessages.processed, - - totalActivityPubMessagesSuccesses: this.inboxMessages.successes, - - // Dirty, but simpler and with type checking - totalActivityPubCreateMessagesSuccesses: this.inboxMessages.successesPerType.Create, - totalActivityPubUpdateMessagesSuccesses: this.inboxMessages.successesPerType.Update, - totalActivityPubDeleteMessagesSuccesses: this.inboxMessages.successesPerType.Delete, - totalActivityPubFollowMessagesSuccesses: this.inboxMessages.successesPerType.Follow, - totalActivityPubAcceptMessagesSuccesses: this.inboxMessages.successesPerType.Accept, - totalActivityPubRejectMessagesSuccesses: this.inboxMessages.successesPerType.Reject, - totalActivityPubAnnounceMessagesSuccesses: this.inboxMessages.successesPerType.Announce, - totalActivityPubUndoMessagesSuccesses: this.inboxMessages.successesPerType.Undo, - totalActivityPubLikeMessagesSuccesses: this.inboxMessages.successesPerType.Like, - totalActivityPubDislikeMessagesSuccesses: this.inboxMessages.successesPerType.Dislike, - totalActivityPubFlagMessagesSuccesses: this.inboxMessages.successesPerType.Flag, - totalActivityPubViewMessagesSuccesses: this.inboxMessages.successesPerType.View, - - totalActivityPubCreateMessagesErrors: this.inboxMessages.errorsPerType.Create, - totalActivityPubUpdateMessagesErrors: this.inboxMessages.errorsPerType.Update, - totalActivityPubDeleteMessagesErrors: this.inboxMessages.errorsPerType.Delete, - totalActivityPubFollowMessagesErrors: this.inboxMessages.errorsPerType.Follow, - totalActivityPubAcceptMessagesErrors: this.inboxMessages.errorsPerType.Accept, - totalActivityPubRejectMessagesErrors: this.inboxMessages.errorsPerType.Reject, - totalActivityPubAnnounceMessagesErrors: this.inboxMessages.errorsPerType.Announce, - totalActivityPubUndoMessagesErrors: this.inboxMessages.errorsPerType.Undo, - totalActivityPubLikeMessagesErrors: this.inboxMessages.errorsPerType.Like, - totalActivityPubDislikeMessagesErrors: this.inboxMessages.errorsPerType.Dislike, - totalActivityPubFlagMessagesErrors: this.inboxMessages.errorsPerType.Flag, - totalActivityPubViewMessagesErrors: this.inboxMessages.errorsPerType.View, - - totalActivityPubMessagesErrors: this.inboxMessages.errors, - - activityPubMessagesProcessedPerSecond: this.buildActivityPubMessagesProcessedPerSecond(), - totalActivityPubMessagesWaiting: this.inboxMessages.waiting - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -// --------------------------------------------------------------------------- - -export { - StatsManager -} diff --git a/server/lib/sync-channel.ts b/server/lib/sync-channel.ts deleted file mode 100644 index 3a805a943..000000000 --- a/server/lib/sync-channel.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { logger } from '@server/helpers/logger' -import { YoutubeDLWrapper } from '@server/helpers/youtube-dl' -import { CONFIG } from '@server/initializers/config' -import { buildYoutubeDLImport } from '@server/lib/video-pre-import' -import { UserModel } from '@server/models/user/user' -import { VideoImportModel } from '@server/models/video/video-import' -import { MChannel, MChannelAccountDefault, MChannelSync } from '@server/types/models' -import { VideoChannelSyncState, VideoPrivacy } from '@shared/models' -import { CreateJobArgument, JobQueue } from './job-queue' -import { ServerConfigManager } from './server-config-manager' - -export async function synchronizeChannel (options: { - channel: MChannelAccountDefault - externalChannelUrl: string - videosCountLimit: number - channelSync?: MChannelSync - onlyAfter?: Date -}) { - const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options - - if (channelSync) { - channelSync.state = VideoChannelSyncState.PROCESSING - channelSync.lastSyncAt = new Date() - await channelSync.save() - } - - try { - const user = await UserModel.loadByChannelActorId(channel.actorId) - const youtubeDL = new YoutubeDLWrapper( - externalChannelUrl, - ServerConfigManager.Instance.getEnabledResolutions('vod'), - CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION - ) - - const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit }) - - logger.info( - 'Fetched %d candidate URLs for sync channel %s.', - targetUrls.length, channel.Actor.preferredUsername, { targetUrls } - ) - - if (targetUrls.length === 0) { - if (channelSync) { - channelSync.state = VideoChannelSyncState.SYNCED - await channelSync.save() - } - - return - } - - const children: CreateJobArgument[] = [] - - for (const targetUrl of targetUrls) { - if (await skipImport(channel, targetUrl, onlyAfter)) continue - - const { job } = await buildYoutubeDLImport({ - user, - channel, - targetUrl, - channelSync, - importDataOverride: { - privacy: VideoPrivacy.PUBLIC - } - }) - - children.push(job) - } - - // Will update the channel sync status - const parent: CreateJobArgument = { - type: 'after-video-channel-import', - payload: { - channelSyncId: channelSync?.id - } - } - - await JobQueue.Instance.createJobWithChildren(parent, children) - } catch (err) { - logger.error(`Failed to import ${externalChannelUrl} in channel ${channel.name}`, { err }) - channelSync.state = VideoChannelSyncState.FAILED - await channelSync.save() - } -} - -// --------------------------------------------------------------------------- - -async function skipImport (channel: MChannel, targetUrl: string, onlyAfter?: Date) { - if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) { - logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', targetUrl, channel.name) - return true - } - - if (onlyAfter) { - const youtubeDL = new YoutubeDLWrapper( - targetUrl, - ServerConfigManager.Instance.getEnabledResolutions('vod'), - CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION - ) - - const videoInfo = await youtubeDL.getInfoForDownload() - - const onlyAfterWithoutTime = new Date(onlyAfter) - onlyAfterWithoutTime.setHours(0, 0, 0, 0) - - if (videoInfo.originallyPublishedAtWithoutTime.getTime() < onlyAfterWithoutTime.getTime()) { - return true - } - } - - return false -} diff --git a/server/lib/thumbnail.ts b/server/lib/thumbnail.ts deleted file mode 100644 index 0b98da14f..000000000 --- a/server/lib/thumbnail.ts +++ /dev/null @@ -1,327 +0,0 @@ -import { join } from 'path' -import { ThumbnailType } from '@shared/models' -import { generateImageFilename, generateImageFromVideoFile } from '../helpers/image-utils' -import { CONFIG } from '../initializers/config' -import { ASSETS_PATH, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '../initializers/constants' -import { ThumbnailModel } from '../models/video/thumbnail' -import { MVideoFile, MVideoThumbnail, MVideoUUID, MVideoWithAllFiles } from '../types/models' -import { MThumbnail } from '../types/models/video/thumbnail' -import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist' -import { VideoPathManager } from './video-path-manager' -import { downloadImageFromWorker, processImageFromWorker } from './worker/parent-process' - -type ImageSize = { height?: number, width?: number } - -function updateLocalPlaylistMiniatureFromExisting (options: { - inputPath: string - playlist: MVideoPlaylistThumbnail - automaticallyGenerated: boolean - keepOriginal?: boolean // default to false - size?: ImageSize -}) { - const { inputPath, playlist, automaticallyGenerated, keepOriginal = false, size } = options - const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size) - const type = ThumbnailType.MINIATURE - - const thumbnailCreator = () => { - return processImageFromWorker({ path: inputPath, destination: outputPath, newSize: { width, height }, keepOriginal }) - } - - return updateThumbnailFromFunction({ - thumbnailCreator, - filename, - height, - width, - type, - automaticallyGenerated, - onDisk: true, - existingThumbnail - }) -} - -function updateRemotePlaylistMiniatureFromUrl (options: { - downloadUrl: string - playlist: MVideoPlaylistThumbnail - size?: ImageSize -}) { - const { downloadUrl, playlist, size } = options - const { filename, basePath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size) - const type = ThumbnailType.MINIATURE - - // Only save the file URL if it is a remote playlist - const fileUrl = playlist.isOwned() - ? null - : downloadUrl - - const thumbnailCreator = () => { - return downloadImageFromWorker({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } }) - } - - return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl, onDisk: true }) -} - -function updateLocalVideoMiniatureFromExisting (options: { - inputPath: string - video: MVideoThumbnail - type: ThumbnailType - automaticallyGenerated: boolean - size?: ImageSize - keepOriginal?: boolean // default to false -}) { - const { inputPath, video, type, automaticallyGenerated, size, keepOriginal = false } = options - - const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) - - const thumbnailCreator = () => { - return processImageFromWorker({ path: inputPath, destination: outputPath, newSize: { width, height }, keepOriginal }) - } - - return updateThumbnailFromFunction({ - thumbnailCreator, - filename, - height, - width, - type, - automaticallyGenerated, - existingThumbnail, - onDisk: true - }) -} - -function generateLocalVideoMiniature (options: { - video: MVideoThumbnail - videoFile: MVideoFile - type: ThumbnailType -}) { - const { video, videoFile, type } = options - - return VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), input => { - const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type) - - const thumbnailCreator = videoFile.isAudio() - ? () => processImageFromWorker({ - path: ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, - destination: outputPath, - newSize: { width, height }, - keepOriginal: true - }) - : () => generateImageFromVideoFile({ - fromPath: input, - folder: basePath, - imageName: filename, - size: { height, width } - }) - - return updateThumbnailFromFunction({ - thumbnailCreator, - filename, - height, - width, - type, - automaticallyGenerated: true, - onDisk: true, - existingThumbnail - }) - }) -} - -// --------------------------------------------------------------------------- - -function updateLocalVideoMiniatureFromUrl (options: { - downloadUrl: string - video: MVideoThumbnail - type: ThumbnailType - size?: ImageSize -}) { - const { downloadUrl, video, type, size } = options - const { filename: updatedFilename, basePath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) - - // Only save the file URL if it is a remote video - const fileUrl = video.isOwned() - ? null - : downloadUrl - - const thumbnailUrlChanged = hasThumbnailUrlChanged(existingThumbnail, downloadUrl, video) - - // Do not change the thumbnail filename if the file did not change - const filename = thumbnailUrlChanged - ? updatedFilename - : existingThumbnail.filename - - const thumbnailCreator = () => { - if (thumbnailUrlChanged) { - return downloadImageFromWorker({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } }) - } - - return Promise.resolve() - } - - return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl, onDisk: true }) -} - -function updateRemoteVideoThumbnail (options: { - fileUrl: string - video: MVideoThumbnail - type: ThumbnailType - size: ImageSize - onDisk: boolean -}) { - const { fileUrl, video, type, size, onDisk } = options - const { filename: generatedFilename, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) - - const thumbnail = existingThumbnail || new ThumbnailModel() - - // Do not change the thumbnail filename if the file did not change - if (hasThumbnailUrlChanged(existingThumbnail, fileUrl, video)) { - thumbnail.filename = generatedFilename - } - - thumbnail.height = height - thumbnail.width = width - thumbnail.type = type - thumbnail.fileUrl = fileUrl - thumbnail.onDisk = onDisk - - return thumbnail -} - -// --------------------------------------------------------------------------- - -async function regenerateMiniaturesIfNeeded (video: MVideoWithAllFiles) { - if (video.getMiniature().automaticallyGenerated === true) { - const miniature = await generateLocalVideoMiniature({ - video, - videoFile: video.getMaxQualityFile(), - type: ThumbnailType.MINIATURE - }) - await video.addAndSaveThumbnail(miniature) - } - - if (video.getPreview().automaticallyGenerated === true) { - const preview = await generateLocalVideoMiniature({ - video, - videoFile: video.getMaxQualityFile(), - type: ThumbnailType.PREVIEW - }) - await video.addAndSaveThumbnail(preview) - } -} - -// --------------------------------------------------------------------------- - -export { - generateLocalVideoMiniature, - regenerateMiniaturesIfNeeded, - updateLocalVideoMiniatureFromUrl, - updateLocalVideoMiniatureFromExisting, - updateRemoteVideoThumbnail, - updateRemotePlaylistMiniatureFromUrl, - updateLocalPlaylistMiniatureFromExisting -} - -// --------------------------------------------------------------------------- -// Private -// --------------------------------------------------------------------------- - -function hasThumbnailUrlChanged (existingThumbnail: MThumbnail, downloadUrl: string, video: MVideoUUID) { - const existingUrl = existingThumbnail - ? existingThumbnail.fileUrl - : null - - // If the thumbnail URL did not change and has a unique filename (introduced in 3.1), avoid thumbnail processing - return !existingUrl || existingUrl !== downloadUrl || downloadUrl.endsWith(`${video.uuid}.jpg`) -} - -function buildMetadataFromPlaylist (playlist: MVideoPlaylistThumbnail, size: ImageSize) { - const filename = playlist.generateThumbnailName() - const basePath = CONFIG.STORAGE.THUMBNAILS_DIR - - return { - filename, - basePath, - existingThumbnail: playlist.Thumbnail, - outputPath: join(basePath, filename), - height: size ? size.height : THUMBNAILS_SIZE.height, - width: size ? size.width : THUMBNAILS_SIZE.width - } -} - -function buildMetadataFromVideo (video: MVideoThumbnail, type: ThumbnailType, size?: ImageSize) { - const existingThumbnail = Array.isArray(video.Thumbnails) - ? video.Thumbnails.find(t => t.type === type) - : undefined - - if (type === ThumbnailType.MINIATURE) { - const filename = generateImageFilename() - const basePath = CONFIG.STORAGE.THUMBNAILS_DIR - - return { - filename, - basePath, - existingThumbnail, - outputPath: join(basePath, filename), - height: size ? size.height : THUMBNAILS_SIZE.height, - width: size ? size.width : THUMBNAILS_SIZE.width - } - } - - if (type === ThumbnailType.PREVIEW) { - const filename = generateImageFilename() - const basePath = CONFIG.STORAGE.PREVIEWS_DIR - - return { - filename, - basePath, - existingThumbnail, - outputPath: join(basePath, filename), - height: size ? size.height : PREVIEWS_SIZE.height, - width: size ? size.width : PREVIEWS_SIZE.width - } - } - - return undefined -} - -async function updateThumbnailFromFunction (parameters: { - thumbnailCreator: () => Promise - filename: string - height: number - width: number - type: ThumbnailType - onDisk: boolean - automaticallyGenerated?: boolean - fileUrl?: string - existingThumbnail?: MThumbnail -}) { - const { - thumbnailCreator, - filename, - width, - height, - type, - existingThumbnail, - onDisk, - automaticallyGenerated = null, - fileUrl = null - } = parameters - - const oldFilename = existingThumbnail && existingThumbnail.filename !== filename - ? existingThumbnail.filename - : undefined - - const thumbnail: MThumbnail = existingThumbnail || new ThumbnailModel() - - thumbnail.filename = filename - thumbnail.height = height - thumbnail.width = width - thumbnail.type = type - thumbnail.fileUrl = fileUrl - thumbnail.automaticallyGenerated = automaticallyGenerated - thumbnail.onDisk = onDisk - - if (oldFilename) thumbnail.previousThumbnailFilename = oldFilename - - await thumbnailCreator() - - return thumbnail -} diff --git a/server/lib/timeserie.ts b/server/lib/timeserie.ts deleted file mode 100644 index 08b12129a..000000000 --- a/server/lib/timeserie.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { logger } from '@server/helpers/logger' - -function buildGroupByAndBoundaries (startDateString: string, endDateString: string) { - const startDate = new Date(startDateString) - const endDate = new Date(endDateString) - - const groupInterval = buildGroupInterval(startDate, endDate) - - logger.debug('Found "%s" group interval.', groupInterval, { startDate, endDate }) - - // Remove parts of the date we don't need - if (groupInterval.endsWith(' month') || groupInterval.endsWith(' months')) { - startDate.setDate(1) - startDate.setHours(0, 0, 0, 0) - } else if (groupInterval.endsWith(' day') || groupInterval.endsWith(' days')) { - startDate.setHours(0, 0, 0, 0) - } else if (groupInterval.endsWith(' hour') || groupInterval.endsWith(' hours')) { - startDate.setMinutes(0, 0, 0) - } else { - startDate.setSeconds(0, 0) - } - - return { - groupInterval, - startDate, - endDate - } -} - -// --------------------------------------------------------------------------- - -export { - buildGroupByAndBoundaries -} - -// --------------------------------------------------------------------------- - -function buildGroupInterval (startDate: Date, endDate: Date): string { - const aYear = 31536000 - const aMonth = 2678400 - const aDay = 86400 - const anHour = 3600 - const aMinute = 60 - - const diffSeconds = (endDate.getTime() - startDate.getTime()) / 1000 - - if (diffSeconds >= 6 * aYear) return '6 months' - if (diffSeconds >= 2 * aYear) return '1 month' - if (diffSeconds >= 6 * aMonth) return '7 days' - if (diffSeconds >= 2 * aMonth) return '2 days' - - if (diffSeconds >= 15 * aDay) return '1 day' - if (diffSeconds >= 8 * aDay) return '12 hours' - if (diffSeconds >= 4 * aDay) return '6 hours' - - if (diffSeconds >= 15 * anHour) return '1 hour' - - if (diffSeconds >= 180 * aMinute) return '10 minutes' - - return '1 minute' -} diff --git a/server/lib/transcoding/create-transcoding-job.ts b/server/lib/transcoding/create-transcoding-job.ts deleted file mode 100644 index d78e68b87..000000000 --- a/server/lib/transcoding/create-transcoding-job.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { CONFIG } from '@server/initializers/config' -import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' -import { TranscodingJobQueueBuilder, TranscodingRunnerJobBuilder } from './shared' - -export function createOptimizeOrMergeAudioJobs (options: { - video: MVideoFullLight - videoFile: MVideoFile - isNewVideo: boolean - user: MUserId - videoFileAlreadyLocked: boolean -}) { - return getJobBuilder().createOptimizeOrMergeAudioJobs(options) -} - -// --------------------------------------------------------------------------- - -export function createTranscodingJobs (options: { - transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7 - video: MVideoFullLight - resolutions: number[] - isNewVideo: boolean - user: MUserId -}) { - return getJobBuilder().createTranscodingJobs(options) -} - -// --------------------------------------------------------------------------- -// Private -// --------------------------------------------------------------------------- - -function getJobBuilder () { - if (CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED === true) { - return new TranscodingRunnerJobBuilder() - } - - return new TranscodingJobQueueBuilder() -} diff --git a/server/lib/transcoding/default-transcoding-profiles.ts b/server/lib/transcoding/default-transcoding-profiles.ts deleted file mode 100644 index 8f8fdd026..000000000 --- a/server/lib/transcoding/default-transcoding-profiles.ts +++ /dev/null @@ -1,143 +0,0 @@ - -import { logger } from '@server/helpers/logger' -import { FFmpegCommandWrapper, getDefaultAvailableEncoders } from '@shared/ffmpeg' -import { AvailableEncoders, EncoderOptionsBuilder } from '@shared/models' - -// --------------------------------------------------------------------------- -// Profile manager to get and change default profiles -// --------------------------------------------------------------------------- - -class VideoTranscodingProfilesManager { - private static instance: VideoTranscodingProfilesManager - - // 1 === less priority - private readonly encodersPriorities = { - vod: this.buildDefaultEncodersPriorities(), - live: this.buildDefaultEncodersPriorities() - } - - private readonly availableEncoders = getDefaultAvailableEncoders() - - private availableProfiles = { - vod: [] as string[], - live: [] as string[] - } - - private constructor () { - this.buildAvailableProfiles() - } - - getAvailableEncoders (): AvailableEncoders { - return { - available: this.availableEncoders, - encodersToTry: { - vod: { - video: this.getEncodersByPriority('vod', 'video'), - audio: this.getEncodersByPriority('vod', 'audio') - }, - live: { - video: this.getEncodersByPriority('live', 'video'), - audio: this.getEncodersByPriority('live', 'audio') - } - } - } - } - - getAvailableProfiles (type: 'vod' | 'live') { - return this.availableProfiles[type] - } - - addProfile (options: { - type: 'vod' | 'live' - encoder: string - profile: string - builder: EncoderOptionsBuilder - }) { - const { type, encoder, profile, builder } = options - - const encoders = this.availableEncoders[type] - - if (!encoders[encoder]) encoders[encoder] = {} - encoders[encoder][profile] = builder - - this.buildAvailableProfiles() - } - - removeProfile (options: { - type: 'vod' | 'live' - encoder: string - profile: string - }) { - const { type, encoder, profile } = options - - delete this.availableEncoders[type][encoder][profile] - this.buildAvailableProfiles() - } - - addEncoderPriority (type: 'vod' | 'live', streamType: 'audio' | 'video', encoder: string, priority: number) { - this.encodersPriorities[type][streamType].push({ name: encoder, priority }) - - FFmpegCommandWrapper.resetSupportedEncoders() - } - - removeEncoderPriority (type: 'vod' | 'live', streamType: 'audio' | 'video', encoder: string, priority: number) { - this.encodersPriorities[type][streamType] = this.encodersPriorities[type][streamType] - .filter(o => o.name !== encoder && o.priority !== priority) - - FFmpegCommandWrapper.resetSupportedEncoders() - } - - private getEncodersByPriority (type: 'vod' | 'live', streamType: 'audio' | 'video') { - return this.encodersPriorities[type][streamType] - .sort((e1, e2) => { - if (e1.priority > e2.priority) return -1 - else if (e1.priority === e2.priority) return 0 - - return 1 - }) - .map(e => e.name) - } - - private buildAvailableProfiles () { - for (const type of [ 'vod', 'live' ]) { - const result = new Set() - - const encoders = this.availableEncoders[type] - - for (const encoderName of Object.keys(encoders)) { - for (const profile of Object.keys(encoders[encoderName])) { - result.add(profile) - } - } - - this.availableProfiles[type] = Array.from(result) - } - - logger.debug('Available transcoding profiles built.', { availableProfiles: this.availableProfiles }) - } - - private buildDefaultEncodersPriorities () { - return { - video: [ - { name: 'libx264', priority: 100 } - ], - - // Try the first one, if not available try the second one etc - audio: [ - // we favor VBR, if a good AAC encoder is available - { name: 'libfdk_aac', priority: 200 }, - { name: 'aac', priority: 100 } - ] - } - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -// --------------------------------------------------------------------------- - -export { - VideoTranscodingProfilesManager -} diff --git a/server/lib/transcoding/ended-transcoding.ts b/server/lib/transcoding/ended-transcoding.ts deleted file mode 100644 index d31674ede..000000000 --- a/server/lib/transcoding/ended-transcoding.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { MVideo } from '@server/types/models' -import { moveToNextState } from '../video-state' - -export async function onTranscodingEnded (options: { - video: MVideo - isNewVideo: boolean - moveVideoToNextState: boolean -}) { - const { video, isNewVideo, moveVideoToNextState } = options - - await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') - - if (moveVideoToNextState) { - await retryTransactionWrapper(moveToNextState, { video, isNewVideo }) - } -} diff --git a/server/lib/transcoding/hls-transcoding.ts b/server/lib/transcoding/hls-transcoding.ts deleted file mode 100644 index 2c325d9ee..000000000 --- a/server/lib/transcoding/hls-transcoding.ts +++ /dev/null @@ -1,180 +0,0 @@ -import { MutexInterface } from 'async-mutex' -import { Job } from 'bullmq' -import { ensureDir, move, stat } from 'fs-extra' -import { basename, extname as extnameUtil, join } from 'path' -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' -import { sequelizeTypescript } from '@server/initializers/database' -import { MVideo, MVideoFile } from '@server/types/models' -import { pick } from '@shared/core-utils' -import { getVideoStreamDuration, getVideoStreamFPS } from '@shared/ffmpeg' -import { VideoResolution } from '@shared/models' -import { CONFIG } from '../../initializers/config' -import { VideoFileModel } from '../../models/video/video-file' -import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' -import { updatePlaylistAfterFileChange } from '../hls' -import { generateHLSVideoFilename, getHlsResolutionPlaylistFilename } from '../paths' -import { buildFileMetadata } from '../video-file' -import { VideoPathManager } from '../video-path-manager' -import { buildFFmpegVOD } from './shared' - -// Concat TS segments from a live video to a fragmented mp4 HLS playlist -export async function generateHlsPlaylistResolutionFromTS (options: { - video: MVideo - concatenatedTsFilePath: string - resolution: VideoResolution - fps: number - isAAC: boolean - inputFileMutexReleaser: MutexInterface.Releaser -}) { - return generateHlsPlaylistCommon({ - type: 'hls-from-ts' as 'hls-from-ts', - inputPath: options.concatenatedTsFilePath, - - ...pick(options, [ 'video', 'resolution', 'fps', 'inputFileMutexReleaser', 'isAAC' ]) - }) -} - -// Generate an HLS playlist from an input file, and update the master playlist -export function generateHlsPlaylistResolution (options: { - video: MVideo - videoInputPath: string - resolution: VideoResolution - fps: number - copyCodecs: boolean - inputFileMutexReleaser: MutexInterface.Releaser - job?: Job -}) { - return generateHlsPlaylistCommon({ - type: 'hls' as 'hls', - inputPath: options.videoInputPath, - - ...pick(options, [ 'video', 'resolution', 'fps', 'copyCodecs', 'inputFileMutexReleaser', 'job' ]) - }) -} - -export async function onHLSVideoFileTranscoding (options: { - video: MVideo - videoFile: MVideoFile - videoOutputPath: string - m3u8OutputPath: string -}) { - const { video, videoFile, videoOutputPath, m3u8OutputPath } = options - - // Create or update the playlist - const playlist = await retryTransactionWrapper(() => { - return sequelizeTypescript.transaction(async transaction => { - return VideoStreamingPlaylistModel.loadOrGenerate(video, transaction) - }) - }) - videoFile.videoStreamingPlaylistId = playlist.id - - const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - await video.reload() - - const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, videoFile) - await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video)) - - // Move playlist file - const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, basename(m3u8OutputPath)) - await move(m3u8OutputPath, resolutionPlaylistPath, { overwrite: true }) - // Move video file - await move(videoOutputPath, videoFilePath, { overwrite: true }) - - // Update video duration if it was not set (in case of a live for example) - if (!video.duration) { - video.duration = await getVideoStreamDuration(videoFilePath) - await video.save() - } - - const stats = await stat(videoFilePath) - - videoFile.size = stats.size - videoFile.fps = await getVideoStreamFPS(videoFilePath) - videoFile.metadata = await buildFileMetadata(videoFilePath) - - await createTorrentAndSetInfoHash(playlist, videoFile) - - const oldFile = await VideoFileModel.loadHLSFile({ - playlistId: playlist.id, - fps: videoFile.fps, - resolution: videoFile.resolution - }) - - if (oldFile) { - await video.removeStreamingPlaylistVideoFile(playlist, oldFile) - await oldFile.destroy() - } - - const savedVideoFile = await VideoFileModel.customUpsert(videoFile, 'streaming-playlist', undefined) - - await updatePlaylistAfterFileChange(video, playlist) - - return { resolutionPlaylistPath, videoFile: savedVideoFile } - } finally { - mutexReleaser() - } -} - -// --------------------------------------------------------------------------- - -async function generateHlsPlaylistCommon (options: { - type: 'hls' | 'hls-from-ts' - video: MVideo - inputPath: string - - resolution: VideoResolution - fps: number - - inputFileMutexReleaser: MutexInterface.Releaser - - copyCodecs?: boolean - isAAC?: boolean - - job?: Job -}) { - const { type, video, inputPath, resolution, fps, copyCodecs, isAAC, job, inputFileMutexReleaser } = options - const transcodeDirectory = CONFIG.STORAGE.TMP_DIR - - const videoTranscodedBasePath = join(transcodeDirectory, type) - await ensureDir(videoTranscodedBasePath) - - const videoFilename = generateHLSVideoFilename(resolution) - const videoOutputPath = join(videoTranscodedBasePath, videoFilename) - - const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFilename) - const m3u8OutputPath = join(videoTranscodedBasePath, resolutionPlaylistFilename) - - const transcodeOptions = { - type, - - inputPath, - outputPath: m3u8OutputPath, - - resolution, - fps, - copyCodecs, - - isAAC, - - inputFileMutexReleaser, - - hlsPlaylist: { - videoFilename - } - } - - await buildFFmpegVOD(job).transcode(transcodeOptions) - - const newVideoFile = new VideoFileModel({ - resolution, - extname: extnameUtil(videoFilename), - size: 0, - filename: videoFilename, - fps: -1 - }) - - await onHLSVideoFileTranscoding({ video, videoFile: newVideoFile, videoOutputPath, m3u8OutputPath }) -} diff --git a/server/lib/transcoding/shared/ffmpeg-builder.ts b/server/lib/transcoding/shared/ffmpeg-builder.ts deleted file mode 100644 index 441445ec4..000000000 --- a/server/lib/transcoding/shared/ffmpeg-builder.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Job } from 'bullmq' -import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg' -import { logger } from '@server/helpers/logger' -import { FFmpegVOD } from '@shared/ffmpeg' -import { VideoTranscodingProfilesManager } from '../default-transcoding-profiles' - -export function buildFFmpegVOD (job?: Job) { - return new FFmpegVOD({ - ...getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders()), - - updateJobProgress: progress => { - if (!job) return - - job.updateProgress(progress) - .catch(err => logger.error('Cannot update ffmpeg job progress', { err })) - } - }) -} diff --git a/server/lib/transcoding/shared/index.ts b/server/lib/transcoding/shared/index.ts deleted file mode 100644 index f0b45bcbb..000000000 --- a/server/lib/transcoding/shared/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './job-builders' -export * from './ffmpeg-builder' diff --git a/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts b/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts deleted file mode 100644 index 15fc814ae..000000000 --- a/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts +++ /dev/null @@ -1,21 +0,0 @@ - -import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models' - -export abstract class AbstractJobBuilder { - - abstract createOptimizeOrMergeAudioJobs (options: { - video: MVideoFullLight - videoFile: MVideoFile - isNewVideo: boolean - user: MUserId - videoFileAlreadyLocked: boolean - }): Promise - - abstract createTranscodingJobs (options: { - transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7 - video: MVideoFullLight - resolutions: number[] - isNewVideo: boolean - user: MUserId | null - }): Promise -} diff --git a/server/lib/transcoding/shared/job-builders/index.ts b/server/lib/transcoding/shared/job-builders/index.ts deleted file mode 100644 index 9b1c82adf..000000000 --- a/server/lib/transcoding/shared/job-builders/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './transcoding-job-queue-builder' -export * from './transcoding-runner-job-builder' diff --git a/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts deleted file mode 100644 index 0505c2b2f..000000000 --- a/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts +++ /dev/null @@ -1,322 +0,0 @@ -import Bluebird from 'bluebird' -import { computeOutputFPS } from '@server/helpers/ffmpeg' -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { DEFAULT_AUDIO_RESOLUTION, VIDEO_TRANSCODING_FPS } from '@server/initializers/constants' -import { CreateJobArgument, JobQueue } from '@server/lib/job-queue' -import { Hooks } from '@server/lib/plugins/hooks' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models' -import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg' -import { - HLSTranscodingPayload, - MergeAudioTranscodingPayload, - NewWebVideoResolutionTranscodingPayload, - OptimizeTranscodingPayload, - VideoTranscodingPayload -} from '@shared/models' -import { getTranscodingJobPriority } from '../../transcoding-priority' -import { canDoQuickTranscode } from '../../transcoding-quick-transcode' -import { buildOriginalFileResolution, computeResolutionsToTranscode } from '../../transcoding-resolutions' -import { AbstractJobBuilder } from './abstract-job-builder' - -export class TranscodingJobQueueBuilder extends AbstractJobBuilder { - - async createOptimizeOrMergeAudioJobs (options: { - video: MVideoFullLight - videoFile: MVideoFile - isNewVideo: boolean - user: MUserId - videoFileAlreadyLocked: boolean - }) { - const { video, videoFile, isNewVideo, user, videoFileAlreadyLocked } = options - - let mergeOrOptimizePayload: MergeAudioTranscodingPayload | OptimizeTranscodingPayload - let nextTranscodingSequentialJobPayloads: (NewWebVideoResolutionTranscodingPayload | HLSTranscodingPayload)[][] = [] - - const mutexReleaser = videoFileAlreadyLocked - ? () => {} - : await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - await video.reload() - await videoFile.reload() - - await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), async videoFilePath => { - const probe = await ffprobePromise(videoFilePath) - - const { resolution } = await getVideoStreamDimensionsInfo(videoFilePath, probe) - const hasAudio = await hasAudioStream(videoFilePath, probe) - const quickTranscode = await canDoQuickTranscode(videoFilePath, probe) - const inputFPS = videoFile.isAudio() - ? VIDEO_TRANSCODING_FPS.AUDIO_MERGE // The first transcoding job will transcode to this FPS value - : await getVideoStreamFPS(videoFilePath, probe) - - const maxResolution = await isAudioFile(videoFilePath, probe) - ? DEFAULT_AUDIO_RESOLUTION - : buildOriginalFileResolution(resolution) - - if (CONFIG.TRANSCODING.HLS.ENABLED === true) { - nextTranscodingSequentialJobPayloads.push([ - this.buildHLSJobPayload({ - deleteWebVideoFiles: CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED === false, - - // We had some issues with a web video quick transcoded while producing a HLS version of it - copyCodecs: !quickTranscode, - - resolution: maxResolution, - fps: computeOutputFPS({ inputFPS, resolution: maxResolution }), - videoUUID: video.uuid, - isNewVideo - }) - ]) - } - - const lowerResolutionJobPayloads = await this.buildLowerResolutionJobPayloads({ - video, - inputVideoResolution: maxResolution, - inputVideoFPS: inputFPS, - hasAudio, - isNewVideo - }) - - nextTranscodingSequentialJobPayloads = [ ...nextTranscodingSequentialJobPayloads, ...lowerResolutionJobPayloads ] - - const hasChildren = nextTranscodingSequentialJobPayloads.length !== 0 - mergeOrOptimizePayload = videoFile.isAudio() - ? this.buildMergeAudioPayload({ videoUUID: video.uuid, isNewVideo, hasChildren }) - : this.buildOptimizePayload({ videoUUID: video.uuid, isNewVideo, quickTranscode, hasChildren }) - }) - } finally { - mutexReleaser() - } - - const nextTranscodingSequentialJobs = await Bluebird.mapSeries(nextTranscodingSequentialJobPayloads, payloads => { - return Bluebird.mapSeries(payloads, payload => { - return this.buildTranscodingJob({ payload, user }) - }) - }) - - const transcodingJobBuilderJob: CreateJobArgument = { - type: 'transcoding-job-builder', - payload: { - videoUUID: video.uuid, - sequentialJobs: nextTranscodingSequentialJobs - } - } - - const mergeOrOptimizeJob = await this.buildTranscodingJob({ payload: mergeOrOptimizePayload, user }) - - await JobQueue.Instance.createSequentialJobFlow(...[ mergeOrOptimizeJob, transcodingJobBuilderJob ]) - - await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode') - } - - // --------------------------------------------------------------------------- - - async createTranscodingJobs (options: { - transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7 - video: MVideoFullLight - resolutions: number[] - isNewVideo: boolean - user: MUserId | null - }) { - const { video, transcodingType, resolutions, isNewVideo } = options - - const maxResolution = Math.max(...resolutions) - const childrenResolutions = resolutions.filter(r => r !== maxResolution) - - logger.info('Manually creating transcoding jobs for %s.', transcodingType, { childrenResolutions, maxResolution }) - - const { fps: inputFPS } = await video.probeMaxQualityFile() - - const children = childrenResolutions.map(resolution => { - const fps = computeOutputFPS({ inputFPS, resolution }) - - if (transcodingType === 'hls') { - return this.buildHLSJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo }) - } - - if (transcodingType === 'webtorrent' || transcodingType === 'web-video') { - return this.buildWebVideoJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo }) - } - - throw new Error('Unknown transcoding type') - }) - - const fps = computeOutputFPS({ inputFPS, resolution: maxResolution }) - - const parent = transcodingType === 'hls' - ? this.buildHLSJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo }) - : this.buildWebVideoJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo }) - - // Process the last resolution after the other ones to prevent concurrency issue - // Because low resolutions use the biggest one as ffmpeg input - await this.createTranscodingJobsWithChildren({ videoUUID: video.uuid, parent, children, user: null }) - } - - // --------------------------------------------------------------------------- - - private async createTranscodingJobsWithChildren (options: { - videoUUID: string - parent: (HLSTranscodingPayload | NewWebVideoResolutionTranscodingPayload) - children: (HLSTranscodingPayload | NewWebVideoResolutionTranscodingPayload)[] - user: MUserId | null - }) { - const { videoUUID, parent, children, user } = options - - const parentJob = await this.buildTranscodingJob({ payload: parent, user }) - const childrenJobs = await Bluebird.mapSeries(children, c => this.buildTranscodingJob({ payload: c, user })) - - await JobQueue.Instance.createJobWithChildren(parentJob, childrenJobs) - - await VideoJobInfoModel.increaseOrCreate(videoUUID, 'pendingTranscode', 1 + children.length) - } - - private async buildTranscodingJob (options: { - payload: VideoTranscodingPayload - user: MUserId | null // null means we don't want priority - }) { - const { user, payload } = options - - return { - type: 'video-transcoding' as 'video-transcoding', - priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: undefined }), - payload - } - } - - private async buildLowerResolutionJobPayloads (options: { - video: MVideoWithFileThumbnail - inputVideoResolution: number - inputVideoFPS: number - hasAudio: boolean - isNewVideo: boolean - }) { - const { video, inputVideoResolution, inputVideoFPS, isNewVideo, hasAudio } = options - - // Create transcoding jobs if there are enabled resolutions - const resolutionsEnabled = await Hooks.wrapObject( - computeResolutionsToTranscode({ input: inputVideoResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }), - 'filter:transcoding.auto.resolutions-to-transcode.result', - options - ) - - const sequentialPayloads: (NewWebVideoResolutionTranscodingPayload | HLSTranscodingPayload)[][] = [] - - for (const resolution of resolutionsEnabled) { - const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution }) - - if (CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED) { - const payloads: (NewWebVideoResolutionTranscodingPayload | HLSTranscodingPayload)[] = [ - this.buildWebVideoJobPayload({ - videoUUID: video.uuid, - resolution, - fps, - isNewVideo - }) - ] - - // Create a subsequent job to create HLS resolution that will just copy web video codecs - if (CONFIG.TRANSCODING.HLS.ENABLED) { - payloads.push( - this.buildHLSJobPayload({ - videoUUID: video.uuid, - resolution, - fps, - isNewVideo, - copyCodecs: true - }) - ) - } - - sequentialPayloads.push(payloads) - } else if (CONFIG.TRANSCODING.HLS.ENABLED) { - sequentialPayloads.push([ - this.buildHLSJobPayload({ - videoUUID: video.uuid, - resolution, - fps, - copyCodecs: false, - isNewVideo - }) - ]) - } - } - - return sequentialPayloads - } - - private buildHLSJobPayload (options: { - videoUUID: string - resolution: number - fps: number - isNewVideo: boolean - deleteWebVideoFiles?: boolean // default false - copyCodecs?: boolean // default false - }): HLSTranscodingPayload { - const { videoUUID, resolution, fps, isNewVideo, deleteWebVideoFiles = false, copyCodecs = false } = options - - return { - type: 'new-resolution-to-hls', - videoUUID, - resolution, - fps, - copyCodecs, - isNewVideo, - deleteWebVideoFiles - } - } - - private buildWebVideoJobPayload (options: { - videoUUID: string - resolution: number - fps: number - isNewVideo: boolean - }): NewWebVideoResolutionTranscodingPayload { - const { videoUUID, resolution, fps, isNewVideo } = options - - return { - type: 'new-resolution-to-web-video', - videoUUID, - isNewVideo, - resolution, - fps - } - } - - private buildMergeAudioPayload (options: { - videoUUID: string - isNewVideo: boolean - hasChildren: boolean - }): MergeAudioTranscodingPayload { - const { videoUUID, isNewVideo, hasChildren } = options - - return { - type: 'merge-audio-to-web-video', - resolution: DEFAULT_AUDIO_RESOLUTION, - fps: VIDEO_TRANSCODING_FPS.AUDIO_MERGE, - videoUUID, - isNewVideo, - hasChildren - } - } - - private buildOptimizePayload (options: { - videoUUID: string - quickTranscode: boolean - isNewVideo: boolean - hasChildren: boolean - }): OptimizeTranscodingPayload { - const { videoUUID, quickTranscode, isNewVideo, hasChildren } = options - - return { - type: 'optimize-to-web-video', - videoUUID, - isNewVideo, - hasChildren, - quickTranscode - } - } -} diff --git a/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts deleted file mode 100644 index f0671bd7a..000000000 --- a/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts +++ /dev/null @@ -1,196 +0,0 @@ -import { computeOutputFPS } from '@server/helpers/ffmpeg' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { DEFAULT_AUDIO_RESOLUTION, VIDEO_TRANSCODING_FPS } from '@server/initializers/constants' -import { Hooks } from '@server/lib/plugins/hooks' -import { VODAudioMergeTranscodingJobHandler, VODHLSTranscodingJobHandler, VODWebVideoTranscodingJobHandler } from '@server/lib/runners' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models' -import { MRunnerJob } from '@server/types/models/runners' -import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg' -import { getTranscodingJobPriority } from '../../transcoding-priority' -import { computeResolutionsToTranscode } from '../../transcoding-resolutions' -import { AbstractJobBuilder } from './abstract-job-builder' - -/** - * - * Class to build transcoding job in the local job queue - * - */ - -const lTags = loggerTagsFactory('transcoding') - -export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { - - async createOptimizeOrMergeAudioJobs (options: { - video: MVideoFullLight - videoFile: MVideoFile - isNewVideo: boolean - user: MUserId - videoFileAlreadyLocked: boolean - }) { - const { video, videoFile, isNewVideo, user, videoFileAlreadyLocked } = options - - const mutexReleaser = videoFileAlreadyLocked - ? () => {} - : await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - await video.reload() - await videoFile.reload() - - await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), async videoFilePath => { - const probe = await ffprobePromise(videoFilePath) - - const { resolution } = await getVideoStreamDimensionsInfo(videoFilePath, probe) - const hasAudio = await hasAudioStream(videoFilePath, probe) - const inputFPS = videoFile.isAudio() - ? VIDEO_TRANSCODING_FPS.AUDIO_MERGE // The first transcoding job will transcode to this FPS value - : await getVideoStreamFPS(videoFilePath, probe) - - const maxResolution = await isAudioFile(videoFilePath, probe) - ? DEFAULT_AUDIO_RESOLUTION - : resolution - - const fps = computeOutputFPS({ inputFPS, resolution: maxResolution }) - const priority = await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) - - const mainRunnerJob = videoFile.isAudio() - ? await new VODAudioMergeTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority }) - : await new VODWebVideoTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority }) - - if (CONFIG.TRANSCODING.HLS.ENABLED === true) { - await new VODHLSTranscodingJobHandler().create({ - video, - deleteWebVideoFiles: CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED === false, - resolution: maxResolution, - fps, - isNewVideo, - dependsOnRunnerJob: mainRunnerJob, - priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) - }) - } - - await this.buildLowerResolutionJobPayloads({ - video, - inputVideoResolution: maxResolution, - inputVideoFPS: inputFPS, - hasAudio, - isNewVideo, - mainRunnerJob, - user - }) - }) - } finally { - mutexReleaser() - } - } - - // --------------------------------------------------------------------------- - - async createTranscodingJobs (options: { - transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7 - video: MVideoFullLight - resolutions: number[] - isNewVideo: boolean - user: MUserId | null - }) { - const { video, transcodingType, resolutions, isNewVideo, user } = options - - const maxResolution = Math.max(...resolutions) - const { fps: inputFPS } = await video.probeMaxQualityFile() - const maxFPS = computeOutputFPS({ inputFPS, resolution: maxResolution }) - const priority = await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) - - const childrenResolutions = resolutions.filter(r => r !== maxResolution) - - logger.info('Manually creating transcoding jobs for %s.', transcodingType, { childrenResolutions, maxResolution }) - - // Process the last resolution before the other ones to prevent concurrency issue - // Because low resolutions use the biggest one as ffmpeg input - const mainJob = transcodingType === 'hls' - // eslint-disable-next-line max-len - ? await new VODHLSTranscodingJobHandler().create({ video, resolution: maxResolution, fps: maxFPS, isNewVideo, deleteWebVideoFiles: false, priority }) - : await new VODWebVideoTranscodingJobHandler().create({ video, resolution: maxResolution, fps: maxFPS, isNewVideo, priority }) - - for (const resolution of childrenResolutions) { - const dependsOnRunnerJob = mainJob - const fps = computeOutputFPS({ inputFPS, resolution }) - - if (transcodingType === 'hls') { - await new VODHLSTranscodingJobHandler().create({ - video, - resolution, - fps, - isNewVideo, - deleteWebVideoFiles: false, - dependsOnRunnerJob, - priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) - }) - continue - } - - if (transcodingType === 'webtorrent' || transcodingType === 'web-video') { - await new VODWebVideoTranscodingJobHandler().create({ - video, - resolution, - fps, - isNewVideo, - dependsOnRunnerJob, - priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) - }) - continue - } - - throw new Error('Unknown transcoding type') - } - } - - private async buildLowerResolutionJobPayloads (options: { - mainRunnerJob: MRunnerJob - video: MVideoWithFileThumbnail - inputVideoResolution: number - inputVideoFPS: number - hasAudio: boolean - isNewVideo: boolean - user: MUserId - }) { - const { video, inputVideoResolution, inputVideoFPS, isNewVideo, hasAudio, mainRunnerJob, user } = options - - // Create transcoding jobs if there are enabled resolutions - const resolutionsEnabled = await Hooks.wrapObject( - computeResolutionsToTranscode({ input: inputVideoResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }), - 'filter:transcoding.auto.resolutions-to-transcode.result', - options - ) - - logger.debug('Lower resolutions build for %s.', video.uuid, { resolutionsEnabled, ...lTags(video.uuid) }) - - for (const resolution of resolutionsEnabled) { - const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution }) - - if (CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED) { - await new VODWebVideoTranscodingJobHandler().create({ - video, - resolution, - fps, - isNewVideo, - dependsOnRunnerJob: mainRunnerJob, - priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) - }) - } - - if (CONFIG.TRANSCODING.HLS.ENABLED) { - await new VODHLSTranscodingJobHandler().create({ - video, - resolution, - fps, - isNewVideo, - deleteWebVideoFiles: false, - dependsOnRunnerJob: mainRunnerJob, - priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) - }) - } - } - } -} diff --git a/server/lib/transcoding/transcoding-priority.ts b/server/lib/transcoding/transcoding-priority.ts deleted file mode 100644 index 82ab6f2f1..000000000 --- a/server/lib/transcoding/transcoding-priority.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { JOB_PRIORITY } from '@server/initializers/constants' -import { VideoModel } from '@server/models/video/video' -import { MUserId } from '@server/types/models' - -export async function getTranscodingJobPriority (options: { - user: MUserId - fallback: number - type: 'vod' | 'studio' -}) { - const { user, fallback, type } = options - - if (!user) return fallback - - const now = new Date() - const lastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7) - - const videoUploadedByUser = await VideoModel.countVideosUploadedByUserSince(user.id, lastWeek) - - const base = type === 'vod' - ? JOB_PRIORITY.TRANSCODING - : JOB_PRIORITY.VIDEO_STUDIO - - return base + videoUploadedByUser -} diff --git a/server/lib/transcoding/transcoding-quick-transcode.ts b/server/lib/transcoding/transcoding-quick-transcode.ts deleted file mode 100644 index 53f12cd06..000000000 --- a/server/lib/transcoding/transcoding-quick-transcode.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { FfprobeData } from 'fluent-ffmpeg' -import { CONFIG } from '@server/initializers/config' -import { canDoQuickAudioTranscode, canDoQuickVideoTranscode, ffprobePromise } from '@shared/ffmpeg' - -export async function canDoQuickTranscode (path: string, existingProbe?: FfprobeData): Promise { - if (CONFIG.TRANSCODING.PROFILE !== 'default') return false - - const probe = existingProbe || await ffprobePromise(path) - - return await canDoQuickVideoTranscode(path, probe) && - await canDoQuickAudioTranscode(path, probe) -} diff --git a/server/lib/transcoding/transcoding-resolutions.ts b/server/lib/transcoding/transcoding-resolutions.ts deleted file mode 100644 index 9a6bf5722..000000000 --- a/server/lib/transcoding/transcoding-resolutions.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { CONFIG } from '@server/initializers/config' -import { toEven } from '@shared/core-utils' -import { VideoResolution } from '@shared/models' - -export function buildOriginalFileResolution (inputResolution: number) { - if (CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION === true) { - return toEven(inputResolution) - } - - const resolutions = computeResolutionsToTranscode({ - input: inputResolution, - type: 'vod', - includeInput: false, - strictLower: false, - // We don't really care about the audio resolution in this context - hasAudio: true - }) - - if (resolutions.length === 0) { - return toEven(inputResolution) - } - - return Math.max(...resolutions) -} - -export function computeResolutionsToTranscode (options: { - input: number - type: 'vod' | 'live' - includeInput: boolean - strictLower: boolean - hasAudio: boolean -}) { - const { input, type, includeInput, strictLower, hasAudio } = options - - const configResolutions = type === 'vod' - ? CONFIG.TRANSCODING.RESOLUTIONS - : CONFIG.LIVE.TRANSCODING.RESOLUTIONS - - const resolutionsEnabled = new Set() - - // Put in the order we want to proceed jobs - const availableResolutions: VideoResolution[] = [ - VideoResolution.H_NOVIDEO, - VideoResolution.H_480P, - VideoResolution.H_360P, - VideoResolution.H_720P, - VideoResolution.H_240P, - VideoResolution.H_144P, - VideoResolution.H_1080P, - VideoResolution.H_1440P, - VideoResolution.H_4K - ] - - for (const resolution of availableResolutions) { - // Resolution not enabled - if (configResolutions[resolution + 'p'] !== true) continue - // Too big resolution for input file - if (input < resolution) continue - // We only want lower resolutions than input file - if (strictLower && input === resolution) continue - // Audio resolutio but no audio in the video - if (resolution === VideoResolution.H_NOVIDEO && !hasAudio) continue - - resolutionsEnabled.add(resolution) - } - - if (includeInput) { - // Always use an even resolution to avoid issues with ffmpeg - resolutionsEnabled.add(toEven(input)) - } - - return Array.from(resolutionsEnabled) -} diff --git a/server/lib/transcoding/web-transcoding.ts b/server/lib/transcoding/web-transcoding.ts deleted file mode 100644 index f92d457a0..000000000 --- a/server/lib/transcoding/web-transcoding.ts +++ /dev/null @@ -1,263 +0,0 @@ -import { Job } from 'bullmq' -import { copyFile, move, remove, stat } from 'fs-extra' -import { basename, join } from 'path' -import { computeOutputFPS } from '@server/helpers/ffmpeg' -import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' -import { VideoModel } from '@server/models/video/video' -import { MVideoFile, MVideoFullLight } from '@server/types/models' -import { ffprobePromise, getVideoStreamDuration, getVideoStreamFPS, TranscodeVODOptionsType } from '@shared/ffmpeg' -import { VideoResolution, VideoStorage } from '@shared/models' -import { CONFIG } from '../../initializers/config' -import { VideoFileModel } from '../../models/video/video-file' -import { JobQueue } from '../job-queue' -import { generateWebVideoFilename } from '../paths' -import { buildFileMetadata } from '../video-file' -import { VideoPathManager } from '../video-path-manager' -import { buildFFmpegVOD } from './shared' -import { buildOriginalFileResolution } from './transcoding-resolutions' - -// Optimize the original video file and replace it. The resolution is not changed. -export async function optimizeOriginalVideofile (options: { - video: MVideoFullLight - inputVideoFile: MVideoFile - quickTranscode: boolean - job: Job -}) { - const { video, inputVideoFile, quickTranscode, job } = options - - const transcodeDirectory = CONFIG.STORAGE.TMP_DIR - const newExtname = '.mp4' - - // Will be released by our transcodeVOD function once ffmpeg is ran - const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - await video.reload() - await inputVideoFile.reload() - - const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video) - - const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async videoInputPath => { - const videoOutputPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) - - const transcodeType: TranscodeVODOptionsType = quickTranscode - ? 'quick-transcode' - : 'video' - - const resolution = buildOriginalFileResolution(inputVideoFile.resolution) - const fps = computeOutputFPS({ inputFPS: inputVideoFile.fps, resolution }) - - // Could be very long! - await buildFFmpegVOD(job).transcode({ - type: transcodeType, - - inputPath: videoInputPath, - outputPath: videoOutputPath, - - inputFileMutexReleaser, - - resolution, - fps - }) - - // Important to do this before getVideoFilename() to take in account the new filename - inputVideoFile.resolution = resolution - inputVideoFile.extname = newExtname - inputVideoFile.filename = generateWebVideoFilename(resolution, newExtname) - inputVideoFile.storage = VideoStorage.FILE_SYSTEM - - const { videoFile } = await onWebVideoFileTranscoding({ - video, - videoFile: inputVideoFile, - videoOutputPath - }) - - await remove(videoInputPath) - - return { transcodeType, videoFile } - }) - - return result - } finally { - inputFileMutexReleaser() - } -} - -// Transcode the original video file to a lower resolution compatible with web browsers -export async function transcodeNewWebVideoResolution (options: { - video: MVideoFullLight - resolution: VideoResolution - fps: number - job: Job -}) { - const { video: videoArg, resolution, fps, job } = options - - const transcodeDirectory = CONFIG.STORAGE.TMP_DIR - const newExtname = '.mp4' - - const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid) - - try { - const video = await VideoModel.loadFull(videoArg.uuid) - const file = video.getMaxQualityFile().withVideoOrPlaylist(video) - - const result = await VideoPathManager.Instance.makeAvailableVideoFile(file, async videoInputPath => { - const newVideoFile = new VideoFileModel({ - resolution, - extname: newExtname, - filename: generateWebVideoFilename(resolution, newExtname), - size: 0, - videoId: video.id - }) - - const videoOutputPath = join(transcodeDirectory, newVideoFile.filename) - - const transcodeOptions = { - type: 'video' as 'video', - - inputPath: videoInputPath, - outputPath: videoOutputPath, - - inputFileMutexReleaser, - - resolution, - fps - } - - await buildFFmpegVOD(job).transcode(transcodeOptions) - - return onWebVideoFileTranscoding({ video, videoFile: newVideoFile, videoOutputPath }) - }) - - return result - } finally { - inputFileMutexReleaser() - } -} - -// Merge an image with an audio file to create a video -export async function mergeAudioVideofile (options: { - video: MVideoFullLight - resolution: VideoResolution - fps: number - job: Job -}) { - const { video: videoArg, resolution, fps, job } = options - - const transcodeDirectory = CONFIG.STORAGE.TMP_DIR - const newExtname = '.mp4' - - const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid) - - try { - const video = await VideoModel.loadFull(videoArg.uuid) - const inputVideoFile = video.getMinQualityFile() - - const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video) - - const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async audioInputPath => { - const videoOutputPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) - - // If the user updates the video preview during transcoding - const previewPath = video.getPreview().getPath() - const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath)) - await copyFile(previewPath, tmpPreviewPath) - - const transcodeOptions = { - type: 'merge-audio' as 'merge-audio', - - inputPath: tmpPreviewPath, - outputPath: videoOutputPath, - - inputFileMutexReleaser, - - audioPath: audioInputPath, - resolution, - fps - } - - try { - await buildFFmpegVOD(job).transcode(transcodeOptions) - - await remove(audioInputPath) - await remove(tmpPreviewPath) - } catch (err) { - await remove(tmpPreviewPath) - throw err - } - - // Important to do this before getVideoFilename() to take in account the new file extension - inputVideoFile.extname = newExtname - inputVideoFile.resolution = resolution - inputVideoFile.filename = generateWebVideoFilename(inputVideoFile.resolution, newExtname) - - // ffmpeg generated a new video file, so update the video duration - // See https://trac.ffmpeg.org/ticket/5456 - video.duration = await getVideoStreamDuration(videoOutputPath) - await video.save() - - return onWebVideoFileTranscoding({ - video, - videoFile: inputVideoFile, - videoOutputPath, - wasAudioFile: true - }) - }) - - return result - } finally { - inputFileMutexReleaser() - } -} - -export async function onWebVideoFileTranscoding (options: { - video: MVideoFullLight - videoFile: MVideoFile - videoOutputPath: string - wasAudioFile?: boolean // default false -}) { - const { video, videoFile, videoOutputPath, wasAudioFile } = options - - const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - await video.reload() - - const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile) - - const stats = await stat(videoOutputPath) - - const probe = await ffprobePromise(videoOutputPath) - const fps = await getVideoStreamFPS(videoOutputPath, probe) - const metadata = await buildFileMetadata(videoOutputPath, probe) - - await move(videoOutputPath, outputPath, { overwrite: true }) - - videoFile.size = stats.size - videoFile.fps = fps - videoFile.metadata = metadata - - await createTorrentAndSetInfoHash(video, videoFile) - - const oldFile = await VideoFileModel.loadWebVideoFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) - if (oldFile) await video.removeWebVideoFile(oldFile) - - await VideoFileModel.customUpsert(videoFile, 'video', undefined) - video.VideoFiles = await video.$get('VideoFiles') - - if (wasAudioFile) { - await JobQueue.Instance.createJob({ - type: 'generate-video-storyboard' as 'generate-video-storyboard', - payload: { - videoUUID: video.uuid, - // No need to federate, we process these jobs sequentially - federate: false - } - }) - } - - return { video, videoFile } - } finally { - mutexReleaser() - } -} diff --git a/server/lib/uploadx.ts b/server/lib/uploadx.ts deleted file mode 100644 index c7e0eb414..000000000 --- a/server/lib/uploadx.ts +++ /dev/null @@ -1,37 +0,0 @@ -import express from 'express' -import { buildLogger } from '@server/helpers/logger' -import { getResumableUploadPath } from '@server/helpers/upload' -import { CONFIG } from '@server/initializers/config' -import { LogLevel, Uploadx } from '@uploadx/core' -import { extname } from 'path' - -const logger = buildLogger('uploadx') - -const uploadx = new Uploadx({ - directory: getResumableUploadPath(), - - expiration: { maxAge: undefined, rolling: true }, - - // Could be big with thumbnails/previews - maxMetadataSize: '10MB', - - logger: { - logLevel: CONFIG.LOG.LEVEL as LogLevel, - debug: logger.debug.bind(logger), - info: logger.info.bind(logger), - warn: logger.warn.bind(logger), - error: logger.error.bind(logger) - }, - - userIdentifier: (_, res: express.Response) => { - if (!res.locals.oauth) return undefined - - return res.locals.oauth.token.user.id + '' - }, - - filename: file => `${file.userId}-${file.id}${extname(file.metadata.filename)}` -}) - -export { - uploadx -} diff --git a/server/lib/user.ts b/server/lib/user.ts deleted file mode 100644 index 56995cca3..000000000 --- a/server/lib/user.ts +++ /dev/null @@ -1,301 +0,0 @@ -import { Transaction } from 'sequelize/types' -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { UserModel } from '@server/models/user/user' -import { MActorDefault } from '@server/types/models/actor' -import { ActivityPubActorType } from '../../shared/models/activitypub' -import { UserAdminFlag, UserNotificationSetting, UserNotificationSettingValue, UserRole } from '../../shared/models/users' -import { SERVER_ACTOR_NAME, WEBSERVER } from '../initializers/constants' -import { sequelizeTypescript } from '../initializers/database' -import { AccountModel } from '../models/account/account' -import { UserNotificationSettingModel } from '../models/user/user-notification-setting' -import { MAccountDefault, MChannelActor } from '../types/models' -import { MRegistration, MUser, MUserDefault, MUserId } from '../types/models/user' -import { generateAndSaveActorKeys } from './activitypub/actors' -import { getLocalAccountActivityPubUrl } from './activitypub/url' -import { Emailer } from './emailer' -import { LiveQuotaStore } from './live/live-quota-store' -import { buildActorInstance, findAvailableLocalActorName } from './local-actor' -import { Redis } from './redis' -import { createLocalVideoChannel } from './video-channel' -import { createWatchLaterPlaylist } from './video-playlist' - -type ChannelNames = { name: string, displayName: string } - -function buildUser (options: { - username: string - password: string - email: string - - role?: UserRole // Default to UserRole.User - adminFlags?: UserAdminFlag // Default to UserAdminFlag.NONE - - emailVerified: boolean | null - - videoQuota?: number // Default to CONFIG.USER.VIDEO_QUOTA - videoQuotaDaily?: number // Default to CONFIG.USER.VIDEO_QUOTA_DAILY - - pluginAuth?: string -}): MUser { - const { - username, - password, - email, - role = UserRole.USER, - emailVerified, - videoQuota = CONFIG.USER.VIDEO_QUOTA, - videoQuotaDaily = CONFIG.USER.VIDEO_QUOTA_DAILY, - adminFlags = UserAdminFlag.NONE, - pluginAuth - } = options - - return new UserModel({ - username, - password, - email, - - nsfwPolicy: CONFIG.INSTANCE.DEFAULT_NSFW_POLICY, - p2pEnabled: CONFIG.DEFAULTS.P2P.WEBAPP.ENABLED, - videosHistoryEnabled: CONFIG.USER.HISTORY.VIDEOS.ENABLED, - - autoPlayVideo: true, - - role, - emailVerified, - adminFlags, - - videoQuota, - videoQuotaDaily, - - pluginAuth - }) -} - -// --------------------------------------------------------------------------- - -async function createUserAccountAndChannelAndPlaylist (parameters: { - userToCreate: MUser - userDisplayName?: string - channelNames?: ChannelNames - validateUser?: boolean -}): Promise<{ user: MUserDefault, account: MAccountDefault, videoChannel: MChannelActor }> { - const { userToCreate, userDisplayName, channelNames, validateUser = true } = parameters - - const { user, account, videoChannel } = await sequelizeTypescript.transaction(async t => { - const userOptions = { - transaction: t, - validate: validateUser - } - - const userCreated: MUserDefault = await userToCreate.save(userOptions) - userCreated.NotificationSetting = await createDefaultUserNotificationSettings(userCreated, t) - - const accountCreated = await createLocalAccountWithoutKeys({ - name: userCreated.username, - displayName: userDisplayName, - userId: userCreated.id, - applicationId: null, - t - }) - userCreated.Account = accountCreated - - const channelAttributes = await buildChannelAttributes({ user: userCreated, transaction: t, channelNames }) - const videoChannel = await createLocalVideoChannel(channelAttributes, accountCreated, t) - - const videoPlaylist = await createWatchLaterPlaylist(accountCreated, t) - - return { user: userCreated, account: accountCreated, videoChannel, videoPlaylist } - }) - - const [ accountActorWithKeys, channelActorWithKeys ] = await Promise.all([ - generateAndSaveActorKeys(account.Actor), - generateAndSaveActorKeys(videoChannel.Actor) - ]) - - account.Actor = accountActorWithKeys - videoChannel.Actor = channelActorWithKeys - - return { user, account, videoChannel } -} - -async function createLocalAccountWithoutKeys (parameters: { - name: string - displayName?: string - userId: number | null - applicationId: number | null - t: Transaction | undefined - type?: ActivityPubActorType -}) { - const { name, displayName, userId, applicationId, t, type = 'Person' } = parameters - const url = getLocalAccountActivityPubUrl(name) - - const actorInstance = buildActorInstance(type, url, name) - const actorInstanceCreated: MActorDefault = await actorInstance.save({ transaction: t }) - - const accountInstance = new AccountModel({ - name: displayName || name, - userId, - applicationId, - actorId: actorInstanceCreated.id - }) - - const accountInstanceCreated: MAccountDefault = await accountInstance.save({ transaction: t }) - accountInstanceCreated.Actor = actorInstanceCreated - - return accountInstanceCreated -} - -async function createApplicationActor (applicationId: number) { - const accountCreated = await createLocalAccountWithoutKeys({ - name: SERVER_ACTOR_NAME, - userId: null, - applicationId, - t: undefined, - type: 'Application' - }) - - accountCreated.Actor = await generateAndSaveActorKeys(accountCreated.Actor) - - return accountCreated -} - -// --------------------------------------------------------------------------- - -async function sendVerifyUserEmail (user: MUser, isPendingEmail = false) { - const verificationString = await Redis.Instance.setUserVerifyEmailVerificationString(user.id) - let verifyEmailUrl = `${WEBSERVER.URL}/verify-account/email?userId=${user.id}&verificationString=${verificationString}` - - if (isPendingEmail) verifyEmailUrl += '&isPendingEmail=true' - - const to = isPendingEmail - ? user.pendingEmail - : user.email - - const username = user.username - - Emailer.Instance.addVerifyEmailJob({ username, to, verifyEmailUrl, isRegistrationRequest: false }) -} - -async function sendVerifyRegistrationEmail (registration: MRegistration) { - const verificationString = await Redis.Instance.setRegistrationVerifyEmailVerificationString(registration.id) - const verifyEmailUrl = `${WEBSERVER.URL}/verify-account/email?registrationId=${registration.id}&verificationString=${verificationString}` - - const to = registration.email - const username = registration.username - - Emailer.Instance.addVerifyEmailJob({ username, to, verifyEmailUrl, isRegistrationRequest: true }) -} - -// --------------------------------------------------------------------------- - -async function getOriginalVideoFileTotalFromUser (user: MUserId) { - // Don't use sequelize because we need to use a sub query - const query = UserModel.generateUserQuotaBaseSQL({ - withSelect: true, - whereUserId: '$userId', - daily: false - }) - - const base = await UserModel.getTotalRawQuery(query, user.id) - - return base + LiveQuotaStore.Instance.getLiveQuotaOf(user.id) -} - -// Returns cumulative size of all video files uploaded in the last 24 hours. -async function getOriginalVideoFileTotalDailyFromUser (user: MUserId) { - // Don't use sequelize because we need to use a sub query - const query = UserModel.generateUserQuotaBaseSQL({ - withSelect: true, - whereUserId: '$userId', - daily: true - }) - - const base = await UserModel.getTotalRawQuery(query, user.id) - - return base + LiveQuotaStore.Instance.getLiveQuotaOf(user.id) -} - -async function isAbleToUploadVideo (userId: number, newVideoSize: number) { - const user = await UserModel.loadById(userId) - - if (user.videoQuota === -1 && user.videoQuotaDaily === -1) return Promise.resolve(true) - - const [ totalBytes, totalBytesDaily ] = await Promise.all([ - getOriginalVideoFileTotalFromUser(user), - getOriginalVideoFileTotalDailyFromUser(user) - ]) - - const uploadedTotal = newVideoSize + totalBytes - const uploadedDaily = newVideoSize + totalBytesDaily - - logger.debug( - 'Check user %d quota to upload another video.', userId, - { totalBytes, totalBytesDaily, videoQuota: user.videoQuota, videoQuotaDaily: user.videoQuotaDaily, newVideoSize } - ) - - if (user.videoQuotaDaily === -1) return uploadedTotal < user.videoQuota - if (user.videoQuota === -1) return uploadedDaily < user.videoQuotaDaily - - return uploadedTotal < user.videoQuota && uploadedDaily < user.videoQuotaDaily -} - -// --------------------------------------------------------------------------- - -export { - getOriginalVideoFileTotalFromUser, - getOriginalVideoFileTotalDailyFromUser, - createApplicationActor, - createUserAccountAndChannelAndPlaylist, - createLocalAccountWithoutKeys, - - sendVerifyUserEmail, - sendVerifyRegistrationEmail, - - isAbleToUploadVideo, - buildUser -} - -// --------------------------------------------------------------------------- - -function createDefaultUserNotificationSettings (user: MUserId, t: Transaction | undefined) { - const values: UserNotificationSetting & { userId: number } = { - userId: user.id, - newVideoFromSubscription: UserNotificationSettingValue.WEB, - newCommentOnMyVideo: UserNotificationSettingValue.WEB, - myVideoImportFinished: UserNotificationSettingValue.WEB, - myVideoPublished: UserNotificationSettingValue.WEB, - abuseAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - videoAutoBlacklistAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - blacklistOnMyVideo: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - newUserRegistration: UserNotificationSettingValue.WEB, - commentMention: UserNotificationSettingValue.WEB, - newFollow: UserNotificationSettingValue.WEB, - newInstanceFollower: UserNotificationSettingValue.WEB, - abuseNewMessage: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - abuseStateChange: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - autoInstanceFollowing: UserNotificationSettingValue.WEB, - newPeerTubeVersion: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - newPluginVersion: UserNotificationSettingValue.WEB, - myVideoStudioEditionFinished: UserNotificationSettingValue.WEB - } - - return UserNotificationSettingModel.create(values, { transaction: t }) -} - -async function buildChannelAttributes (options: { - user: MUser - transaction?: Transaction - channelNames?: ChannelNames -}) { - const { user, transaction, channelNames } = options - - if (channelNames) return channelNames - - const channelName = await findAvailableLocalActorName(user.username + '_channel', transaction) - const videoChannelDisplayName = `Main ${user.username} channel` - - return { - name: channelName, - displayName: videoChannelDisplayName - } -} diff --git a/server/lib/video-blacklist.ts b/server/lib/video-blacklist.ts deleted file mode 100644 index d5664a1b9..000000000 --- a/server/lib/video-blacklist.ts +++ /dev/null @@ -1,145 +0,0 @@ -import { Transaction } from 'sequelize' -import { afterCommitIfTransaction } from '@server/helpers/database-utils' -import { sequelizeTypescript } from '@server/initializers/database' -import { - MUser, - MVideoAccountLight, - MVideoBlacklist, - MVideoBlacklistVideo, - MVideoFullLight, - MVideoWithBlacklistLight -} from '@server/types/models' -import { LiveVideoError, UserRight, VideoBlacklistCreate, VideoBlacklistType } from '../../shared/models' -import { UserAdminFlag } from '../../shared/models/users/user-flag.model' -import { logger, loggerTagsFactory } from '../helpers/logger' -import { CONFIG } from '../initializers/config' -import { VideoBlacklistModel } from '../models/video/video-blacklist' -import { sendDeleteVideo } from './activitypub/send' -import { federateVideoIfNeeded } from './activitypub/videos' -import { LiveManager } from './live/live-manager' -import { Notifier } from './notifier' -import { Hooks } from './plugins/hooks' - -const lTags = loggerTagsFactory('blacklist') - -async function autoBlacklistVideoIfNeeded (parameters: { - video: MVideoWithBlacklistLight - user?: MUser - isRemote: boolean - isNew: boolean - isNewFile: boolean - notify?: boolean - transaction?: Transaction -}) { - const { video, user, isRemote, isNew, isNewFile, notify = true, transaction } = parameters - const doAutoBlacklist = await Hooks.wrapFun( - autoBlacklistNeeded, - { video, user, isRemote, isNew, isNewFile }, - 'filter:video.auto-blacklist.result' - ) - - if (!doAutoBlacklist) return false - - const videoBlacklistToCreate = { - videoId: video.id, - unfederated: true, - reason: 'Auto-blacklisted. Moderator review required.', - type: VideoBlacklistType.AUTO_BEFORE_PUBLISHED - } - const [ videoBlacklist ] = await VideoBlacklistModel.findOrCreate({ - where: { - videoId: video.id - }, - defaults: videoBlacklistToCreate, - transaction - }) - video.VideoBlacklist = videoBlacklist - - videoBlacklist.Video = video - - if (notify) { - afterCommitIfTransaction(transaction, () => { - Notifier.Instance.notifyOnVideoAutoBlacklist(videoBlacklist) - }) - } - - logger.info('Video %s auto-blacklisted.', video.uuid, lTags(video.uuid)) - - return true -} - -async function blacklistVideo (videoInstance: MVideoAccountLight, options: VideoBlacklistCreate) { - const blacklist: MVideoBlacklistVideo = await VideoBlacklistModel.create({ - videoId: videoInstance.id, - unfederated: options.unfederate === true, - reason: options.reason, - type: VideoBlacklistType.MANUAL - }) - blacklist.Video = videoInstance - - if (options.unfederate === true) { - await sendDeleteVideo(videoInstance, undefined) - } - - if (videoInstance.isLive) { - LiveManager.Instance.stopSessionOf(videoInstance.uuid, LiveVideoError.BLACKLISTED) - } - - Notifier.Instance.notifyOnVideoBlacklist(blacklist) -} - -async function unblacklistVideo (videoBlacklist: MVideoBlacklist, video: MVideoFullLight) { - const videoBlacklistType = await sequelizeTypescript.transaction(async t => { - const unfederated = videoBlacklist.unfederated - const videoBlacklistType = videoBlacklist.type - - await videoBlacklist.destroy({ transaction: t }) - video.VideoBlacklist = undefined - - // Re federate the video - if (unfederated === true) { - await federateVideoIfNeeded(video, true, t) - } - - return videoBlacklistType - }) - - Notifier.Instance.notifyOnVideoUnblacklist(video) - - if (videoBlacklistType === VideoBlacklistType.AUTO_BEFORE_PUBLISHED) { - Notifier.Instance.notifyOnVideoPublishedAfterRemovedFromAutoBlacklist(video) - - // Delete on object so new video notifications will send - delete video.VideoBlacklist - Notifier.Instance.notifyOnNewVideoIfNeeded(video) - } -} - -// --------------------------------------------------------------------------- - -export { - autoBlacklistVideoIfNeeded, - blacklistVideo, - unblacklistVideo -} - -// --------------------------------------------------------------------------- - -function autoBlacklistNeeded (parameters: { - video: MVideoWithBlacklistLight - isRemote: boolean - isNew: boolean - isNewFile: boolean - user?: MUser -}) { - const { user, video, isRemote, isNew, isNewFile } = parameters - - // Already blacklisted - if (video.VideoBlacklist) return false - if (!CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED || !user) return false - if (isRemote || (isNew === false && isNewFile === false)) return false - - if (user.hasRight(UserRight.MANAGE_VIDEO_BLACKLIST) || user.hasAdminFlag(UserAdminFlag.BYPASS_VIDEO_AUTO_BLACKLIST)) return false - - return true -} diff --git a/server/lib/video-channel.ts b/server/lib/video-channel.ts deleted file mode 100644 index 8322c9ad2..000000000 --- a/server/lib/video-channel.ts +++ /dev/null @@ -1,50 +0,0 @@ -import * as Sequelize from 'sequelize' -import { VideoChannelCreate } from '../../shared/models' -import { VideoModel } from '../models/video/video' -import { VideoChannelModel } from '../models/video/video-channel' -import { MAccountId, MChannelId } from '../types/models' -import { getLocalVideoChannelActivityPubUrl } from './activitypub/url' -import { federateVideoIfNeeded } from './activitypub/videos' -import { buildActorInstance } from './local-actor' - -async function createLocalVideoChannel (videoChannelInfo: VideoChannelCreate, account: MAccountId, t: Sequelize.Transaction) { - const url = getLocalVideoChannelActivityPubUrl(videoChannelInfo.name) - const actorInstance = buildActorInstance('Group', url, videoChannelInfo.name) - - const actorInstanceCreated = await actorInstance.save({ transaction: t }) - - const videoChannelData = { - name: videoChannelInfo.displayName, - description: videoChannelInfo.description, - support: videoChannelInfo.support, - accountId: account.id, - actorId: actorInstanceCreated.id - } - - const videoChannel = new VideoChannelModel(videoChannelData) - - const options = { transaction: t } - const videoChannelCreated = await videoChannel.save(options) - - videoChannelCreated.Actor = actorInstanceCreated - - // No need to send this empty video channel to followers - return videoChannelCreated -} - -async function federateAllVideosOfChannel (videoChannel: MChannelId) { - const videoIds = await VideoModel.getAllIdsFromChannel(videoChannel) - - for (const videoId of videoIds) { - const video = await VideoModel.loadFull(videoId) - - await federateVideoIfNeeded(video, false) - } -} - -// --------------------------------------------------------------------------- - -export { - createLocalVideoChannel, - federateAllVideosOfChannel -} diff --git a/server/lib/video-comment.ts b/server/lib/video-comment.ts deleted file mode 100644 index 6eb865f7f..000000000 --- a/server/lib/video-comment.ts +++ /dev/null @@ -1,116 +0,0 @@ -import express from 'express' -import { cloneDeep } from 'lodash' -import * as Sequelize from 'sequelize' -import { logger } from '@server/helpers/logger' -import { sequelizeTypescript } from '@server/initializers/database' -import { ResultList } from '../../shared/models' -import { VideoCommentThreadTree } from '../../shared/models/videos/comment/video-comment.model' -import { VideoCommentModel } from '../models/video/video-comment' -import { - MAccountDefault, - MComment, - MCommentFormattable, - MCommentOwnerVideo, - MCommentOwnerVideoReply, - MVideoFullLight -} from '../types/models' -import { sendCreateVideoComment, sendDeleteVideoComment } from './activitypub/send' -import { getLocalVideoCommentActivityPubUrl } from './activitypub/url' -import { Hooks } from './plugins/hooks' - -async function removeComment (commentArg: MComment, req: express.Request, res: express.Response) { - let videoCommentInstanceBefore: MCommentOwnerVideo - - await sequelizeTypescript.transaction(async t => { - const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(commentArg.url, t) - - videoCommentInstanceBefore = cloneDeep(comment) - - if (comment.isOwned() || comment.Video.isOwned()) { - await sendDeleteVideoComment(comment, t) - } - - comment.markAsDeleted() - - await comment.save({ transaction: t }) - - logger.info('Video comment %d deleted.', comment.id) - }) - - Hooks.runAction('action:api.video-comment.deleted', { comment: videoCommentInstanceBefore, req, res }) -} - -async function createVideoComment (obj: { - text: string - inReplyToComment: MComment | null - video: MVideoFullLight - account: MAccountDefault -}, t: Sequelize.Transaction) { - let originCommentId: number | null = null - let inReplyToCommentId: number | null = null - - if (obj.inReplyToComment && obj.inReplyToComment !== null) { - originCommentId = obj.inReplyToComment.originCommentId || obj.inReplyToComment.id - inReplyToCommentId = obj.inReplyToComment.id - } - - const comment = await VideoCommentModel.create({ - text: obj.text, - originCommentId, - inReplyToCommentId, - videoId: obj.video.id, - accountId: obj.account.id, - url: new Date().toISOString() - }, { transaction: t, validate: false }) - - comment.url = getLocalVideoCommentActivityPubUrl(obj.video, comment) - - const savedComment: MCommentOwnerVideoReply = await comment.save({ transaction: t }) - savedComment.InReplyToVideoComment = obj.inReplyToComment - savedComment.Video = obj.video - savedComment.Account = obj.account - - await sendCreateVideoComment(savedComment, t) - - return savedComment -} - -function buildFormattedCommentTree (resultList: ResultList): VideoCommentThreadTree { - // Comments are sorted by id ASC - const comments = resultList.data - - const comment = comments.shift() - const thread: VideoCommentThreadTree = { - comment: comment.toFormattedJSON(), - children: [] - } - const idx = { - [comment.id]: thread - } - - while (comments.length !== 0) { - const childComment = comments.shift() - - const childCommentThread: VideoCommentThreadTree = { - comment: childComment.toFormattedJSON(), - children: [] - } - - const parentCommentThread = idx[childComment.inReplyToCommentId] - // Maybe the parent comment was blocked by the admin/user - if (!parentCommentThread) continue - - parentCommentThread.children.push(childCommentThread) - idx[childComment.id] = childCommentThread - } - - return thread -} - -// --------------------------------------------------------------------------- - -export { - removeComment, - createVideoComment, - buildFormattedCommentTree -} diff --git a/server/lib/video-file.ts b/server/lib/video-file.ts deleted file mode 100644 index 46af67ccd..000000000 --- a/server/lib/video-file.ts +++ /dev/null @@ -1,145 +0,0 @@ -import { FfprobeData } from 'fluent-ffmpeg' -import { logger } from '@server/helpers/logger' -import { VideoFileModel } from '@server/models/video/video-file' -import { MVideoWithAllFiles } from '@server/types/models' -import { getLowercaseExtension } from '@shared/core-utils' -import { getFileSize } from '@shared/extra-utils' -import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, isAudioFile } from '@shared/ffmpeg' -import { VideoFileMetadata, VideoResolution } from '@shared/models' -import { lTags } from './object-storage/shared' -import { generateHLSVideoFilename, generateWebVideoFilename } from './paths' -import { VideoPathManager } from './video-path-manager' - -async function buildNewFile (options: { - path: string - mode: 'web-video' | 'hls' -}) { - const { path, mode } = options - - const probe = await ffprobePromise(path) - const size = await getFileSize(path) - - const videoFile = new VideoFileModel({ - extname: getLowercaseExtension(path), - size, - metadata: await buildFileMetadata(path, probe) - }) - - if (await isAudioFile(path, probe)) { - videoFile.resolution = VideoResolution.H_NOVIDEO - } else { - videoFile.fps = await getVideoStreamFPS(path, probe) - videoFile.resolution = (await getVideoStreamDimensionsInfo(path, probe)).resolution - } - - videoFile.filename = mode === 'web-video' - ? generateWebVideoFilename(videoFile.resolution, videoFile.extname) - : generateHLSVideoFilename(videoFile.resolution) - - return videoFile -} - -// --------------------------------------------------------------------------- - -async function removeHLSPlaylist (video: MVideoWithAllFiles) { - const hls = video.getHLSPlaylist() - if (!hls) return - - const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - await video.removeStreamingPlaylistFiles(hls) - await hls.destroy() - - video.VideoStreamingPlaylists = video.VideoStreamingPlaylists.filter(p => p.id !== hls.id) - } finally { - videoFileMutexReleaser() - } -} - -async function removeHLSFile (video: MVideoWithAllFiles, fileToDeleteId: number) { - logger.info('Deleting HLS file %d of %s.', fileToDeleteId, video.url, lTags(video.uuid)) - - const hls = video.getHLSPlaylist() - const files = hls.VideoFiles - - if (files.length === 1) { - await removeHLSPlaylist(video) - return undefined - } - - const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - const toDelete = files.find(f => f.id === fileToDeleteId) - await video.removeStreamingPlaylistVideoFile(video.getHLSPlaylist(), toDelete) - await toDelete.destroy() - - hls.VideoFiles = hls.VideoFiles.filter(f => f.id !== toDelete.id) - } finally { - videoFileMutexReleaser() - } - - return hls -} - -// --------------------------------------------------------------------------- - -async function removeAllWebVideoFiles (video: MVideoWithAllFiles) { - const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - for (const file of video.VideoFiles) { - await video.removeWebVideoFile(file) - await file.destroy() - } - - video.VideoFiles = [] - } finally { - videoFileMutexReleaser() - } - - return video -} - -async function removeWebVideoFile (video: MVideoWithAllFiles, fileToDeleteId: number) { - const files = video.VideoFiles - - if (files.length === 1) { - return removeAllWebVideoFiles(video) - } - - const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) - try { - const toDelete = files.find(f => f.id === fileToDeleteId) - await video.removeWebVideoFile(toDelete) - await toDelete.destroy() - - video.VideoFiles = files.filter(f => f.id !== toDelete.id) - } finally { - videoFileMutexReleaser() - } - - return video -} - -// --------------------------------------------------------------------------- - -async function buildFileMetadata (path: string, existingProbe?: FfprobeData) { - const metadata = existingProbe || await ffprobePromise(path) - - return new VideoFileMetadata(metadata) -} - -// --------------------------------------------------------------------------- - -export { - buildNewFile, - - removeHLSPlaylist, - removeHLSFile, - removeAllWebVideoFiles, - removeWebVideoFile, - - buildFileMetadata -} diff --git a/server/lib/video-path-manager.ts b/server/lib/video-path-manager.ts deleted file mode 100644 index 133544bb2..000000000 --- a/server/lib/video-path-manager.ts +++ /dev/null @@ -1,174 +0,0 @@ -import { Mutex } from 'async-mutex' -import { remove } from 'fs-extra' -import { extname, join } from 'path' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { extractVideo } from '@server/helpers/video' -import { CONFIG } from '@server/initializers/config' -import { DIRECTORIES } from '@server/initializers/constants' -import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '@server/types/models' -import { buildUUID } from '@shared/extra-utils' -import { VideoStorage } from '@shared/models' -import { makeHLSFileAvailable, makeWebVideoFileAvailable } from './object-storage' -import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths' -import { isVideoInPrivateDirectory } from './video-privacy' - -type MakeAvailableCB = (path: string) => Promise | T - -const lTags = loggerTagsFactory('video-path-manager') - -class VideoPathManager { - - private static instance: VideoPathManager - - // Key is a video UUID - private readonly videoFileMutexStore = new Map() - - private constructor () {} - - getFSHLSOutputPath (video: MVideo, filename?: string) { - const base = getHLSDirectory(video) - if (!filename) return base - - return join(base, filename) - } - - getFSRedundancyVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { - if (videoFile.isHLS()) { - const video = extractVideo(videoOrPlaylist) - - return join(getHLSRedundancyDirectory(video), videoFile.filename) - } - - return join(CONFIG.STORAGE.REDUNDANCY_DIR, videoFile.filename) - } - - getFSVideoFileOutputPath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { - const video = extractVideo(videoOrPlaylist) - - if (videoFile.isHLS()) { - return join(getHLSDirectory(video), videoFile.filename) - } - - if (isVideoInPrivateDirectory(video.privacy)) { - return join(DIRECTORIES.VIDEOS.PRIVATE, videoFile.filename) - } - - return join(DIRECTORIES.VIDEOS.PUBLIC, videoFile.filename) - } - - async makeAvailableVideoFile (videoFile: MVideoFileVideo | MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB) { - if (videoFile.storage === VideoStorage.FILE_SYSTEM) { - return this.makeAvailableFactory( - () => this.getFSVideoFileOutputPath(videoFile.getVideoOrStreamingPlaylist(), videoFile), - false, - cb - ) - } - - const destination = this.buildTMPDestination(videoFile.filename) - - if (videoFile.isHLS()) { - const playlist = (videoFile as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist - - return this.makeAvailableFactory( - () => makeHLSFileAvailable(playlist, videoFile.filename, destination), - true, - cb - ) - } - - return this.makeAvailableFactory( - () => makeWebVideoFileAvailable(videoFile.filename, destination), - true, - cb - ) - } - - async makeAvailableResolutionPlaylistFile (videoFile: MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB) { - const filename = getHlsResolutionPlaylistFilename(videoFile.filename) - - if (videoFile.storage === VideoStorage.FILE_SYSTEM) { - return this.makeAvailableFactory( - () => join(getHLSDirectory(videoFile.getVideo()), filename), - false, - cb - ) - } - - const playlist = videoFile.VideoStreamingPlaylist - return this.makeAvailableFactory( - () => makeHLSFileAvailable(playlist, filename, this.buildTMPDestination(filename)), - true, - cb - ) - } - - async makeAvailablePlaylistFile (playlist: MStreamingPlaylistVideo, filename: string, cb: MakeAvailableCB) { - if (playlist.storage === VideoStorage.FILE_SYSTEM) { - return this.makeAvailableFactory( - () => join(getHLSDirectory(playlist.Video), filename), - false, - cb - ) - } - - return this.makeAvailableFactory( - () => makeHLSFileAvailable(playlist, filename, this.buildTMPDestination(filename)), - true, - cb - ) - } - - async lockFiles (videoUUID: string) { - if (!this.videoFileMutexStore.has(videoUUID)) { - this.videoFileMutexStore.set(videoUUID, new Mutex()) - } - - const mutex = this.videoFileMutexStore.get(videoUUID) - const releaser = await mutex.acquire() - - logger.debug('Locked files of %s.', videoUUID, lTags(videoUUID)) - - return releaser - } - - unlockFiles (videoUUID: string) { - const mutex = this.videoFileMutexStore.get(videoUUID) - - mutex.release() - - logger.debug('Released lockfiles of %s.', videoUUID, lTags(videoUUID)) - } - - private async makeAvailableFactory (method: () => Promise | string, clean: boolean, cb: MakeAvailableCB) { - let result: T - - const destination = await method() - - try { - result = await cb(destination) - } catch (err) { - if (destination && clean) await remove(destination) - throw err - } - - if (clean) await remove(destination) - - return result - } - - private buildTMPDestination (filename: string) { - return join(CONFIG.STORAGE.TMP_DIR, buildUUID() + extname(filename)) - - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -// --------------------------------------------------------------------------- - -export { - VideoPathManager -} diff --git a/server/lib/video-playlist.ts b/server/lib/video-playlist.ts deleted file mode 100644 index a1af2e1af..000000000 --- a/server/lib/video-playlist.ts +++ /dev/null @@ -1,30 +0,0 @@ -import * as Sequelize from 'sequelize' -import { VideoPlaylistPrivacy } from '../../shared/models/videos/playlist/video-playlist-privacy.model' -import { VideoPlaylistType } from '../../shared/models/videos/playlist/video-playlist-type.model' -import { VideoPlaylistModel } from '../models/video/video-playlist' -import { MAccount } from '../types/models' -import { MVideoPlaylistOwner } from '../types/models/video/video-playlist' -import { getLocalVideoPlaylistActivityPubUrl } from './activitypub/url' - -async function createWatchLaterPlaylist (account: MAccount, t: Sequelize.Transaction) { - const videoPlaylist: MVideoPlaylistOwner = new VideoPlaylistModel({ - name: 'Watch later', - privacy: VideoPlaylistPrivacy.PRIVATE, - type: VideoPlaylistType.WATCH_LATER, - ownerAccountId: account.id - }) - - videoPlaylist.url = getLocalVideoPlaylistActivityPubUrl(videoPlaylist) // We use the UUID, so set the URL after building the object - - await videoPlaylist.save({ transaction: t }) - - videoPlaylist.OwnerAccount = account - - return videoPlaylist -} - -// --------------------------------------------------------------------------- - -export { - createWatchLaterPlaylist -} diff --git a/server/lib/video-pre-import.ts b/server/lib/video-pre-import.ts deleted file mode 100644 index fcb9f77d7..000000000 --- a/server/lib/video-pre-import.ts +++ /dev/null @@ -1,323 +0,0 @@ -import { remove } from 'fs-extra' -import { moveAndProcessCaptionFile } from '@server/helpers/captions-utils' -import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions' -import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos' -import { isResolvingToUnicastOnly } from '@server/helpers/dns' -import { logger } from '@server/helpers/logger' -import { YoutubeDLInfo, YoutubeDLWrapper } from '@server/helpers/youtube-dl' -import { CONFIG } from '@server/initializers/config' -import { sequelizeTypescript } from '@server/initializers/database' -import { Hooks } from '@server/lib/plugins/hooks' -import { ServerConfigManager } from '@server/lib/server-config-manager' -import { setVideoTags } from '@server/lib/video' -import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist' -import { VideoModel } from '@server/models/video/video' -import { VideoCaptionModel } from '@server/models/video/video-caption' -import { VideoImportModel } from '@server/models/video/video-import' -import { FilteredModelAttributes } from '@server/types' -import { - MChannelAccountDefault, - MChannelSync, - MThumbnail, - MUser, - MVideoAccountDefault, - MVideoCaption, - MVideoImportFormattable, - MVideoTag, - MVideoThumbnail, - MVideoWithBlacklistLight -} from '@server/types/models' -import { ThumbnailType, VideoImportCreate, VideoImportPayload, VideoImportState, VideoPrivacy, VideoState } from '@shared/models' -import { getLocalVideoActivityPubUrl } from './activitypub/url' -import { updateLocalVideoMiniatureFromExisting, updateLocalVideoMiniatureFromUrl } from './thumbnail' -import { VideoPasswordModel } from '@server/models/video/video-password' - -class YoutubeDlImportError extends Error { - code: YoutubeDlImportError.CODE - cause?: Error // Property to remove once ES2022 is used - constructor ({ message, code }) { - super(message) - this.code = code - } - - static fromError (err: Error, code: YoutubeDlImportError.CODE, message?: string) { - const ytDlErr = new this({ message: message ?? err.message, code }) - ytDlErr.cause = err - ytDlErr.stack = err.stack // Useless once ES2022 is used - return ytDlErr - } -} - -namespace YoutubeDlImportError { - export enum CODE { - FETCH_ERROR, - NOT_ONLY_UNICAST_URL - } -} - -// --------------------------------------------------------------------------- - -async function insertFromImportIntoDB (parameters: { - video: MVideoThumbnail - thumbnailModel: MThumbnail - previewModel: MThumbnail - videoChannel: MChannelAccountDefault - tags: string[] - videoImportAttributes: FilteredModelAttributes - user: MUser - videoPasswords?: string[] -}): Promise { - const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user, videoPasswords } = parameters - - const videoImport = await sequelizeTypescript.transaction(async t => { - const sequelizeOptions = { transaction: t } - - // Save video object in database - const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag) - videoCreated.VideoChannel = videoChannel - - if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t) - if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t) - - if (videoCreated.privacy === VideoPrivacy.PASSWORD_PROTECTED) { - await VideoPasswordModel.addPasswords(videoPasswords, video.id, t) - } - - await autoBlacklistVideoIfNeeded({ - video: videoCreated, - user, - notify: false, - isRemote: false, - isNew: true, - isNewFile: true, - transaction: t - }) - - await setVideoTags({ video: videoCreated, tags, transaction: t }) - - // Create video import object in database - const videoImport = await VideoImportModel.create( - Object.assign({ videoId: videoCreated.id }, videoImportAttributes), - sequelizeOptions - ) as MVideoImportFormattable - videoImport.Video = videoCreated - - return videoImport - }) - - return videoImport -} - -async function buildVideoFromImport ({ channelId, importData, importDataOverride, importType }: { - channelId: number - importData: YoutubeDLInfo - importDataOverride?: Partial - importType: 'url' | 'torrent' -}): Promise { - let videoData = { - name: importDataOverride?.name || importData.name || 'Unknown name', - remote: false, - category: importDataOverride?.category || importData.category, - licence: importDataOverride?.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE, - language: importDataOverride?.language || importData.language, - commentsEnabled: importDataOverride?.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED, - downloadEnabled: importDataOverride?.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED, - waitTranscoding: importDataOverride?.waitTranscoding ?? true, - state: VideoState.TO_IMPORT, - nsfw: importDataOverride?.nsfw || importData.nsfw || false, - description: importDataOverride?.description || importData.description, - support: importDataOverride?.support || null, - privacy: importDataOverride?.privacy || VideoPrivacy.PRIVATE, - duration: 0, // duration will be set by the import job - channelId, - originallyPublishedAt: importDataOverride?.originallyPublishedAt - ? new Date(importDataOverride?.originallyPublishedAt) - : importData.originallyPublishedAtWithoutTime - } - - videoData = await Hooks.wrapObject( - videoData, - importType === 'url' - ? 'filter:api.video.import-url.video-attribute.result' - : 'filter:api.video.import-torrent.video-attribute.result' - ) - - const video = new VideoModel(videoData) - video.url = getLocalVideoActivityPubUrl(video) - - return video -} - -async function buildYoutubeDLImport (options: { - targetUrl: string - channel: MChannelAccountDefault - user: MUser - channelSync?: MChannelSync - importDataOverride?: Partial - thumbnailFilePath?: string - previewFilePath?: string -}) { - const { targetUrl, channel, channelSync, importDataOverride, thumbnailFilePath, previewFilePath, user } = options - - const youtubeDL = new YoutubeDLWrapper( - targetUrl, - ServerConfigManager.Instance.getEnabledResolutions('vod'), - CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION - ) - - // Get video infos - let youtubeDLInfo: YoutubeDLInfo - try { - youtubeDLInfo = await youtubeDL.getInfoForDownload() - } catch (err) { - throw YoutubeDlImportError.fromError( - err, YoutubeDlImportError.CODE.FETCH_ERROR, `Cannot fetch information from import for URL ${targetUrl}` - ) - } - - if (!await hasUnicastURLsOnly(youtubeDLInfo)) { - throw new YoutubeDlImportError({ - message: 'Cannot use non unicast IP as targetUrl.', - code: YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL - }) - } - - const video = await buildVideoFromImport({ - channelId: channel.id, - importData: youtubeDLInfo, - importDataOverride, - importType: 'url' - }) - - const thumbnailModel = await forgeThumbnail({ - inputPath: thumbnailFilePath, - downloadUrl: youtubeDLInfo.thumbnailUrl, - video, - type: ThumbnailType.MINIATURE - }) - - const previewModel = await forgeThumbnail({ - inputPath: previewFilePath, - downloadUrl: youtubeDLInfo.thumbnailUrl, - video, - type: ThumbnailType.PREVIEW - }) - - const videoImport = await insertFromImportIntoDB({ - video, - thumbnailModel, - previewModel, - videoChannel: channel, - tags: importDataOverride?.tags || youtubeDLInfo.tags, - user, - videoImportAttributes: { - targetUrl, - state: VideoImportState.PENDING, - userId: user.id, - videoChannelSyncId: channelSync?.id - }, - videoPasswords: importDataOverride.videoPasswords - }) - - // Get video subtitles - await processYoutubeSubtitles(youtubeDL, targetUrl, video.id) - - let fileExt = `.${youtubeDLInfo.ext}` - if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4' - - const payload: VideoImportPayload = { - type: 'youtube-dl' as 'youtube-dl', - videoImportId: videoImport.id, - fileExt, - // If part of a sync process, there is a parent job that will aggregate children results - preventException: !!channelSync - } - - return { - videoImport, - job: { type: 'video-import' as 'video-import', payload } - } -} - -// --------------------------------------------------------------------------- - -export { - buildYoutubeDLImport, - YoutubeDlImportError, - insertFromImportIntoDB, - buildVideoFromImport -} - -// --------------------------------------------------------------------------- - -async function forgeThumbnail ({ inputPath, video, downloadUrl, type }: { - inputPath?: string - downloadUrl?: string - video: MVideoThumbnail - type: ThumbnailType -}): Promise { - if (inputPath) { - return updateLocalVideoMiniatureFromExisting({ - inputPath, - video, - type, - automaticallyGenerated: false - }) - } - - if (downloadUrl) { - try { - return await updateLocalVideoMiniatureFromUrl({ downloadUrl, video, type }) - } catch (err) { - logger.warn('Cannot process thumbnail %s from youtube-dl.', downloadUrl, { err }) - } - } - - return null -} - -async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) { - try { - const subtitles = await youtubeDL.getSubtitles() - - logger.info('Found %s subtitles candidates from youtube-dl import %s.', subtitles.length, targetUrl) - - for (const subtitle of subtitles) { - if (!await isVTTFileValid(subtitle.path)) { - logger.info('%s is not a valid youtube-dl subtitle, skipping', subtitle.path) - await remove(subtitle.path) - continue - } - - const videoCaption = new VideoCaptionModel({ - videoId, - language: subtitle.language, - filename: VideoCaptionModel.generateCaptionName(subtitle.language) - }) as MVideoCaption - - // Move physical file - await moveAndProcessCaptionFile(subtitle, videoCaption) - - await sequelizeTypescript.transaction(async t => { - await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t) - }) - - logger.info('Added %s youtube-dl subtitle', subtitle.path) - } - } catch (err) { - logger.warn('Cannot get video subtitles.', { err }) - } -} - -async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) { - const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname) - const uniqHosts = new Set(hosts) - - for (const h of uniqHosts) { - if (await isResolvingToUnicastOnly(h) !== true) { - return false - } - } - - return true -} diff --git a/server/lib/video-privacy.ts b/server/lib/video-privacy.ts deleted file mode 100644 index 5dd4d9781..000000000 --- a/server/lib/video-privacy.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { move } from 'fs-extra' -import { join } from 'path' -import { logger } from '@server/helpers/logger' -import { DIRECTORIES } from '@server/initializers/constants' -import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' -import { VideoPrivacy, VideoStorage } from '@shared/models' -import { updateHLSFilesACL, updateWebVideoFileACL } from './object-storage' - -const validPrivacySet = new Set([ - VideoPrivacy.PRIVATE, - VideoPrivacy.INTERNAL, - VideoPrivacy.PASSWORD_PROTECTED -]) - -function setVideoPrivacy (video: MVideo, newPrivacy: VideoPrivacy) { - if (video.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) { - video.publishedAt = new Date() - } - - video.privacy = newPrivacy -} - -function isVideoInPrivateDirectory (privacy) { - return validPrivacySet.has(privacy) -} - -function isVideoInPublicDirectory (privacy: VideoPrivacy) { - return !isVideoInPrivateDirectory(privacy) -} - -async function moveFilesIfPrivacyChanged (video: MVideoFullLight, oldPrivacy: VideoPrivacy) { - // Now public, previously private - if (isVideoInPublicDirectory(video.privacy) && isVideoInPrivateDirectory(oldPrivacy)) { - await moveFiles({ type: 'private-to-public', video }) - - return true - } - - // Now private, previously public - if (isVideoInPrivateDirectory(video.privacy) && isVideoInPublicDirectory(oldPrivacy)) { - await moveFiles({ type: 'public-to-private', video }) - - return true - } - - return false -} - -export { - setVideoPrivacy, - - isVideoInPrivateDirectory, - isVideoInPublicDirectory, - - moveFilesIfPrivacyChanged -} - -// --------------------------------------------------------------------------- - -type MoveType = 'private-to-public' | 'public-to-private' - -async function moveFiles (options: { - type: MoveType - video: MVideoFullLight -}) { - const { type, video } = options - - for (const file of video.VideoFiles) { - if (file.storage === VideoStorage.FILE_SYSTEM) { - await moveWebVideoFileOnFS(type, video, file) - } else { - await updateWebVideoFileACL(video, file) - } - } - - const hls = video.getHLSPlaylist() - - if (hls) { - if (hls.storage === VideoStorage.FILE_SYSTEM) { - await moveHLSFilesOnFS(type, video) - } else { - await updateHLSFilesACL(hls) - } - } -} - -async function moveWebVideoFileOnFS (type: MoveType, video: MVideo, file: MVideoFile) { - const directories = getWebVideoDirectories(type) - - const source = join(directories.old, file.filename) - const destination = join(directories.new, file.filename) - - try { - logger.info('Moving web video files of %s after privacy change (%s -> %s).', video.uuid, source, destination) - - await move(source, destination) - } catch (err) { - logger.error('Cannot move web video file %s to %s after privacy change', source, destination, { err }) - } -} - -function getWebVideoDirectories (moveType: MoveType) { - if (moveType === 'private-to-public') { - return { old: DIRECTORIES.VIDEOS.PRIVATE, new: DIRECTORIES.VIDEOS.PUBLIC } - } - - return { old: DIRECTORIES.VIDEOS.PUBLIC, new: DIRECTORIES.VIDEOS.PRIVATE } -} - -// --------------------------------------------------------------------------- - -async function moveHLSFilesOnFS (type: MoveType, video: MVideo) { - const directories = getHLSDirectories(type) - - const source = join(directories.old, video.uuid) - const destination = join(directories.new, video.uuid) - - try { - logger.info('Moving HLS files of %s after privacy change (%s -> %s).', video.uuid, source, destination) - - await move(source, destination) - } catch (err) { - logger.error('Cannot move HLS file %s to %s after privacy change', source, destination, { err }) - } -} - -function getHLSDirectories (moveType: MoveType) { - if (moveType === 'private-to-public') { - return { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC } - } - - return { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE } -} diff --git a/server/lib/video-state.ts b/server/lib/video-state.ts deleted file mode 100644 index 893725d85..000000000 --- a/server/lib/video-state.ts +++ /dev/null @@ -1,154 +0,0 @@ -import { Transaction } from 'sequelize' -import { retryTransactionWrapper } from '@server/helpers/database-utils' -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { sequelizeTypescript } from '@server/initializers/database' -import { VideoModel } from '@server/models/video/video' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { MVideo, MVideoFullLight, MVideoUUID } from '@server/types/models' -import { VideoState } from '@shared/models' -import { federateVideoIfNeeded } from './activitypub/videos' -import { JobQueue } from './job-queue' -import { Notifier } from './notifier' -import { buildMoveToObjectStorageJob } from './video' - -function buildNextVideoState (currentState?: VideoState) { - if (currentState === VideoState.PUBLISHED) { - throw new Error('Video is already in its final state') - } - - if ( - currentState !== VideoState.TO_EDIT && - currentState !== VideoState.TO_TRANSCODE && - currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE && - CONFIG.TRANSCODING.ENABLED - ) { - return VideoState.TO_TRANSCODE - } - - if ( - currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE && - CONFIG.OBJECT_STORAGE.ENABLED - ) { - return VideoState.TO_MOVE_TO_EXTERNAL_STORAGE - } - - return VideoState.PUBLISHED -} - -function moveToNextState (options: { - video: MVideoUUID - previousVideoState?: VideoState - isNewVideo?: boolean // Default true -}) { - const { video, previousVideoState, isNewVideo = true } = options - - return retryTransactionWrapper(() => { - return sequelizeTypescript.transaction(async t => { - // Maybe the video changed in database, refresh it - const videoDatabase = await VideoModel.loadFull(video.uuid, t) - // Video does not exist anymore - if (!videoDatabase) return undefined - - // Already in its final state - if (videoDatabase.state === VideoState.PUBLISHED) { - return federateVideoIfNeeded(videoDatabase, false, t) - } - - const newState = buildNextVideoState(videoDatabase.state) - - if (newState === VideoState.PUBLISHED) { - return moveToPublishedState({ video: videoDatabase, previousVideoState, isNewVideo, transaction: t }) - } - - if (newState === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { - return moveToExternalStorageState({ video: videoDatabase, isNewVideo, transaction: t }) - } - }) - }) -} - -async function moveToExternalStorageState (options: { - video: MVideoFullLight - isNewVideo: boolean - transaction: Transaction -}) { - const { video, isNewVideo, transaction } = options - - const videoJobInfo = await VideoJobInfoModel.load(video.id, transaction) - const pendingTranscode = videoJobInfo?.pendingTranscode || 0 - - // We want to wait all transcoding jobs before moving the video on an external storage - if (pendingTranscode !== 0) return false - - const previousVideoState = video.state - - if (video.state !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { - await video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE, isNewVideo, transaction) - } - - logger.info('Creating external storage move job for video %s.', video.uuid, { tags: [ video.uuid ] }) - - try { - await JobQueue.Instance.createJob(await buildMoveToObjectStorageJob({ video, previousVideoState, isNewVideo })) - - return true - } catch (err) { - logger.error('Cannot add move to object storage job', { err }) - - return false - } -} - -function moveToFailedTranscodingState (video: MVideo) { - if (video.state === VideoState.TRANSCODING_FAILED) return - - return video.setNewState(VideoState.TRANSCODING_FAILED, false, undefined) -} - -function moveToFailedMoveToObjectStorageState (video: MVideo) { - if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED) return - - return video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED, false, undefined) -} - -// --------------------------------------------------------------------------- - -export { - buildNextVideoState, - moveToExternalStorageState, - moveToFailedTranscodingState, - moveToFailedMoveToObjectStorageState, - moveToNextState -} - -// --------------------------------------------------------------------------- - -async function moveToPublishedState (options: { - video: MVideoFullLight - isNewVideo: boolean - transaction: Transaction - previousVideoState?: VideoState -}) { - const { video, isNewVideo, transaction, previousVideoState } = options - const previousState = previousVideoState ?? video.state - - logger.info('Publishing video %s.', video.uuid, { isNewVideo, previousState, tags: [ video.uuid ] }) - - await video.setNewState(VideoState.PUBLISHED, isNewVideo, transaction) - - await federateVideoIfNeeded(video, isNewVideo, transaction) - - if (previousState === VideoState.TO_EDIT) { - Notifier.Instance.notifyOfFinishedVideoStudioEdition(video) - return - } - - if (isNewVideo) { - Notifier.Instance.notifyOnNewVideoIfNeeded(video) - - if (previousState === VideoState.TO_TRANSCODE) { - Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(video) - } - } -} diff --git a/server/lib/video-studio.ts b/server/lib/video-studio.ts deleted file mode 100644 index f549a7084..000000000 --- a/server/lib/video-studio.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { move, remove } from 'fs-extra' -import { join } from 'path' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { createTorrentAndSetInfoHashFromPath } from '@server/helpers/webtorrent' -import { CONFIG } from '@server/initializers/config' -import { UserModel } from '@server/models/user/user' -import { MUser, MVideo, MVideoFile, MVideoFullLight, MVideoWithAllFiles } from '@server/types/models' -import { getVideoStreamDuration } from '@shared/ffmpeg' -import { VideoStudioEditionPayload, VideoStudioTask, VideoStudioTaskPayload } from '@shared/models' -import { federateVideoIfNeeded } from './activitypub/videos' -import { JobQueue } from './job-queue' -import { VideoStudioTranscodingJobHandler } from './runners' -import { createOptimizeOrMergeAudioJobs } from './transcoding/create-transcoding-job' -import { getTranscodingJobPriority } from './transcoding/transcoding-priority' -import { buildNewFile, removeHLSPlaylist, removeWebVideoFile } from './video-file' -import { VideoPathManager } from './video-path-manager' - -const lTags = loggerTagsFactory('video-studio') - -export function buildTaskFileFieldname (indice: number, fieldName = 'file') { - return `tasks[${indice}][options][${fieldName}]` -} - -export function getTaskFileFromReq (files: Express.Multer.File[], indice: number, fieldName = 'file') { - return files.find(f => f.fieldname === buildTaskFileFieldname(indice, fieldName)) -} - -export function getStudioTaskFilePath (filename: string) { - return join(CONFIG.STORAGE.TMP_PERSISTENT_DIR, filename) -} - -export async function safeCleanupStudioTMPFiles (tasks: VideoStudioTaskPayload[]) { - logger.info('Removing studio task files', { tasks, ...lTags() }) - - for (const task of tasks) { - try { - if (task.name === 'add-intro' || task.name === 'add-outro') { - await remove(task.options.file) - } else if (task.name === 'add-watermark') { - await remove(task.options.file) - } - } catch (err) { - logger.error('Cannot remove studio file', { err }) - } - } -} - -// --------------------------------------------------------------------------- - -export async function approximateIntroOutroAdditionalSize ( - video: MVideoFullLight, - tasks: VideoStudioTask[], - fileFinder: (i: number) => string -) { - let additionalDuration = 0 - - for (let i = 0; i < tasks.length; i++) { - const task = tasks[i] - - if (task.name !== 'add-intro' && task.name !== 'add-outro') continue - - const filePath = fileFinder(i) - additionalDuration += await getVideoStreamDuration(filePath) - } - - return (video.getMaxQualityFile().size / video.duration) * additionalDuration -} - -// --------------------------------------------------------------------------- - -export async function createVideoStudioJob (options: { - video: MVideo - user: MUser - payload: VideoStudioEditionPayload -}) { - const { video, user, payload } = options - - const priority = await getTranscodingJobPriority({ user, type: 'studio', fallback: 0 }) - - if (CONFIG.VIDEO_STUDIO.REMOTE_RUNNERS.ENABLED) { - await new VideoStudioTranscodingJobHandler().create({ video, tasks: payload.tasks, priority }) - return - } - - await JobQueue.Instance.createJob({ type: 'video-studio-edition', payload, priority }) -} - -export async function onVideoStudioEnded (options: { - editionResultPath: string - tasks: VideoStudioTaskPayload[] - video: MVideoFullLight -}) { - const { video, tasks, editionResultPath } = options - - const newFile = await buildNewFile({ path: editionResultPath, mode: 'web-video' }) - newFile.videoId = video.id - - const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newFile) - await move(editionResultPath, outputPath) - - await safeCleanupStudioTMPFiles(tasks) - - await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath) - await removeAllFiles(video, newFile) - - await newFile.save() - - video.duration = await getVideoStreamDuration(outputPath) - await video.save() - - await federateVideoIfNeeded(video, false, undefined) - - const user = await UserModel.loadByVideoId(video.id) - - await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user, videoFileAlreadyLocked: false }) -} - -// --------------------------------------------------------------------------- -// Private -// --------------------------------------------------------------------------- - -async function removeAllFiles (video: MVideoWithAllFiles, webVideoFileException: MVideoFile) { - await removeHLSPlaylist(video) - - for (const file of video.VideoFiles) { - if (file.id === webVideoFileException.id) continue - - await removeWebVideoFile(video, file.id) - } -} diff --git a/server/lib/video-tokens-manager.ts b/server/lib/video-tokens-manager.ts deleted file mode 100644 index e28e55cf7..000000000 --- a/server/lib/video-tokens-manager.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { LRUCache } from 'lru-cache' -import { LRU_CACHE } from '@server/initializers/constants' -import { MUserAccountUrl } from '@server/types/models' -import { pick } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' - -// --------------------------------------------------------------------------- -// Create temporary tokens that can be used as URL query parameters to access video static files -// --------------------------------------------------------------------------- - -class VideoTokensManager { - - private static instance: VideoTokensManager - - private readonly lruCache = new LRUCache({ - max: LRU_CACHE.VIDEO_TOKENS.MAX_SIZE, - ttl: LRU_CACHE.VIDEO_TOKENS.TTL - }) - - private constructor () {} - - createForAuthUser (options: { - user: MUserAccountUrl - videoUUID: string - }) { - const { token, expires } = this.generateVideoToken() - - this.lruCache.set(token, pick(options, [ 'user', 'videoUUID' ])) - - return { token, expires } - } - - createForPasswordProtectedVideo (options: { - videoUUID: string - }) { - const { token, expires } = this.generateVideoToken() - - this.lruCache.set(token, pick(options, [ 'videoUUID' ])) - - return { token, expires } - } - - hasToken (options: { - token: string - videoUUID: string - }) { - const value = this.lruCache.get(options.token) - if (!value) return false - - return value.videoUUID === options.videoUUID - } - - getUserFromToken (options: { - token: string - }) { - const value = this.lruCache.get(options.token) - if (!value) return undefined - - return value.user - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } - - private generateVideoToken () { - const token = buildUUID() - const expires = new Date(new Date().getTime() + LRU_CACHE.VIDEO_TOKENS.TTL) - - return { token, expires } - } -} - -// --------------------------------------------------------------------------- - -export { - VideoTokensManager -} diff --git a/server/lib/video-urls.ts b/server/lib/video-urls.ts deleted file mode 100644 index 0597488ad..000000000 --- a/server/lib/video-urls.ts +++ /dev/null @@ -1,31 +0,0 @@ - -import { STATIC_PATHS, WEBSERVER } from '@server/initializers/constants' -import { MStreamingPlaylist, MVideo, MVideoFile, MVideoUUID } from '@server/types/models' - -// ################## Redundancy ################## - -function generateHLSRedundancyUrl (video: MVideo, playlist: MStreamingPlaylist) { - // Base URL used by our HLS player - return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + playlist.getStringType() + '/' + video.uuid -} - -function generateWebVideoRedundancyUrl (file: MVideoFile) { - return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + file.filename -} - -// ################## Meta data ################## - -function getLocalVideoFileMetadataUrl (video: MVideoUUID, videoFile: MVideoFile) { - const path = '/api/v1/videos/' - - return WEBSERVER.URL + path + video.uuid + '/metadata/' + videoFile.id -} - -// --------------------------------------------------------------------------- - -export { - getLocalVideoFileMetadataUrl, - - generateWebVideoRedundancyUrl, - generateHLSRedundancyUrl -} diff --git a/server/lib/video.ts b/server/lib/video.ts deleted file mode 100644 index 362c861a5..000000000 --- a/server/lib/video.ts +++ /dev/null @@ -1,189 +0,0 @@ -import { UploadFiles } from 'express' -import memoizee from 'memoizee' -import { Transaction } from 'sequelize/types' -import { CONFIG } from '@server/initializers/config' -import { MEMOIZE_LENGTH, MEMOIZE_TTL } from '@server/initializers/constants' -import { TagModel } from '@server/models/video/tag' -import { VideoModel } from '@server/models/video/video' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { FilteredModelAttributes } from '@server/types' -import { MThumbnail, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models' -import { ManageVideoTorrentPayload, ThumbnailType, VideoCreate, VideoPrivacy, VideoState } from '@shared/models' -import { CreateJobArgument, JobQueue } from './job-queue/job-queue' -import { updateLocalVideoMiniatureFromExisting } from './thumbnail' -import { moveFilesIfPrivacyChanged } from './video-privacy' - -function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes { - return { - name: videoInfo.name, - remote: false, - category: videoInfo.category, - licence: videoInfo.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE, - language: videoInfo.language, - commentsEnabled: videoInfo.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED, - downloadEnabled: videoInfo.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED, - waitTranscoding: videoInfo.waitTranscoding || false, - nsfw: videoInfo.nsfw || false, - description: videoInfo.description, - support: videoInfo.support, - privacy: videoInfo.privacy || VideoPrivacy.PRIVATE, - channelId, - originallyPublishedAt: videoInfo.originallyPublishedAt - ? new Date(videoInfo.originallyPublishedAt) - : null - } -} - -async function buildVideoThumbnailsFromReq (options: { - video: MVideoThumbnail - files: UploadFiles - fallback: (type: ThumbnailType) => Promise - automaticallyGenerated?: boolean -}) { - const { video, files, fallback, automaticallyGenerated } = options - - const promises = [ - { - type: ThumbnailType.MINIATURE, - fieldName: 'thumbnailfile' - }, - { - type: ThumbnailType.PREVIEW, - fieldName: 'previewfile' - } - ].map(p => { - const fields = files?.[p.fieldName] - - if (fields) { - return updateLocalVideoMiniatureFromExisting({ - inputPath: fields[0].path, - video, - type: p.type, - automaticallyGenerated: automaticallyGenerated || false - }) - } - - return fallback(p.type) - }) - - return Promise.all(promises) -} - -// --------------------------------------------------------------------------- - -async function setVideoTags (options: { - video: MVideoTag - tags: string[] - transaction?: Transaction -}) { - const { video, tags, transaction } = options - - const internalTags = tags || [] - const tagInstances = await TagModel.findOrCreateTags(internalTags, transaction) - - await video.$set('Tags', tagInstances, { transaction }) - video.Tags = tagInstances -} - -// --------------------------------------------------------------------------- - -async function buildMoveToObjectStorageJob (options: { - video: MVideoUUID - previousVideoState: VideoState - isNewVideo?: boolean // Default true -}) { - const { video, previousVideoState, isNewVideo = true } = options - - await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingMove') - - return { - type: 'move-to-object-storage' as 'move-to-object-storage', - payload: { - videoUUID: video.uuid, - isNewVideo, - previousVideoState - } - } -} - -// --------------------------------------------------------------------------- - -async function getVideoDuration (videoId: number | string) { - const video = await VideoModel.load(videoId) - - const duration = video.isLive - ? undefined - : video.duration - - return { duration, isLive: video.isLive } -} - -const getCachedVideoDuration = memoizee(getVideoDuration, { - promise: true, - max: MEMOIZE_LENGTH.VIDEO_DURATION, - maxAge: MEMOIZE_TTL.VIDEO_DURATION -}) - -// --------------------------------------------------------------------------- - -async function addVideoJobsAfterUpdate (options: { - video: MVideoFullLight - isNewVideo: boolean - - nameChanged: boolean - oldPrivacy: VideoPrivacy -}) { - const { video, nameChanged, oldPrivacy, isNewVideo } = options - const jobs: CreateJobArgument[] = [] - - const filePathChanged = await moveFilesIfPrivacyChanged(video, oldPrivacy) - - if (!video.isLive && (nameChanged || filePathChanged)) { - for (const file of (video.VideoFiles || [])) { - const payload: ManageVideoTorrentPayload = { action: 'update-metadata', videoId: video.id, videoFileId: file.id } - - jobs.push({ type: 'manage-video-torrent', payload }) - } - - const hls = video.getHLSPlaylist() - - for (const file of (hls?.VideoFiles || [])) { - const payload: ManageVideoTorrentPayload = { action: 'update-metadata', streamingPlaylistId: hls.id, videoFileId: file.id } - - jobs.push({ type: 'manage-video-torrent', payload }) - } - } - - jobs.push({ - type: 'federate-video', - payload: { - videoUUID: video.uuid, - isNewVideo - } - }) - - const wasConfidentialVideo = new Set([ VideoPrivacy.PRIVATE, VideoPrivacy.UNLISTED, VideoPrivacy.INTERNAL ]).has(oldPrivacy) - - if (wasConfidentialVideo) { - jobs.push({ - type: 'notify', - payload: { - action: 'new-video', - videoUUID: video.uuid - } - }) - } - - return JobQueue.Instance.createSequentialJobFlow(...jobs) -} - -// --------------------------------------------------------------------------- - -export { - buildLocalVideoFromReq, - buildVideoThumbnailsFromReq, - setVideoTags, - buildMoveToObjectStorageJob, - addVideoJobsAfterUpdate, - getCachedVideoDuration -} diff --git a/server/lib/views/shared/index.ts b/server/lib/views/shared/index.ts deleted file mode 100644 index 139471183..000000000 --- a/server/lib/views/shared/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './video-viewer-counters' -export * from './video-viewer-stats' -export * from './video-views' diff --git a/server/lib/views/shared/video-viewer-counters.ts b/server/lib/views/shared/video-viewer-counters.ts deleted file mode 100644 index f5b83130e..000000000 --- a/server/lib/views/shared/video-viewer-counters.ts +++ /dev/null @@ -1,198 +0,0 @@ -import { isTestOrDevInstance } from '@server/helpers/core-utils' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { VIEW_LIFETIME } from '@server/initializers/constants' -import { sendView } from '@server/lib/activitypub/send/send-view' -import { PeerTubeSocket } from '@server/lib/peertube-socket' -import { getServerActor } from '@server/models/application/application' -import { VideoModel } from '@server/models/video/video' -import { MVideo, MVideoImmutable } from '@server/types/models' -import { buildUUID, sha256 } from '@shared/extra-utils' - -const lTags = loggerTagsFactory('views') - -export type ViewerScope = 'local' | 'remote' -export type VideoScope = 'local' | 'remote' - -type Viewer = { - expires: number - id: string - viewerScope: ViewerScope - videoScope: VideoScope - lastFederation?: number -} - -export class VideoViewerCounters { - - // expires is new Date().getTime() - private readonly viewersPerVideo = new Map() - private readonly idToViewer = new Map() - - private readonly salt = buildUUID() - - private processingViewerCounters = false - - constructor () { - setInterval(() => this.cleanViewerCounters(), VIEW_LIFETIME.VIEWER_COUNTER) - } - - // --------------------------------------------------------------------------- - - async addLocalViewer (options: { - video: MVideoImmutable - ip: string - }) { - const { video, ip } = options - - logger.debug('Adding local viewer to video viewers counter %s.', video.uuid, { ...lTags(video.uuid) }) - - const viewerId = this.generateViewerId(ip, video.uuid) - const viewer = this.idToViewer.get(viewerId) - - if (viewer) { - viewer.expires = this.buildViewerExpireTime() - await this.federateViewerIfNeeded(video, viewer) - - return false - } - - const newViewer = await this.addViewerToVideo({ viewerId, video, viewerScope: 'local' }) - await this.federateViewerIfNeeded(video, newViewer) - - return true - } - - async addRemoteViewer (options: { - video: MVideo - viewerId: string - viewerExpires: Date - }) { - const { video, viewerExpires, viewerId } = options - - logger.debug('Adding remote viewer to video %s.', video.uuid, { ...lTags(video.uuid) }) - - await this.addViewerToVideo({ video, viewerExpires, viewerId, viewerScope: 'remote' }) - - return true - } - - // --------------------------------------------------------------------------- - - getTotalViewers (options: { - viewerScope: ViewerScope - videoScope: VideoScope - }) { - let total = 0 - - for (const viewers of this.viewersPerVideo.values()) { - total += viewers.filter(v => v.viewerScope === options.viewerScope && v.videoScope === options.videoScope).length - } - - return total - } - - getViewers (video: MVideo) { - const viewers = this.viewersPerVideo.get(video.id) - if (!viewers) return 0 - - return viewers.length - } - - buildViewerExpireTime () { - return new Date().getTime() + VIEW_LIFETIME.VIEWER_COUNTER - } - - // --------------------------------------------------------------------------- - - private async addViewerToVideo (options: { - video: MVideoImmutable - viewerId: string - viewerScope: ViewerScope - viewerExpires?: Date - }) { - const { video, viewerExpires, viewerId, viewerScope } = options - - let watchers = this.viewersPerVideo.get(video.id) - - if (!watchers) { - watchers = [] - this.viewersPerVideo.set(video.id, watchers) - } - - const expires = viewerExpires - ? viewerExpires.getTime() - : this.buildViewerExpireTime() - - const videoScope: VideoScope = video.remote - ? 'remote' - : 'local' - - const viewer = { id: viewerId, expires, videoScope, viewerScope } - watchers.push(viewer) - - this.idToViewer.set(viewerId, viewer) - - await this.notifyClients(video.id, watchers.length) - - return viewer - } - - private async cleanViewerCounters () { - if (this.processingViewerCounters) return - this.processingViewerCounters = true - - if (!isTestOrDevInstance()) logger.info('Cleaning video viewers.', lTags()) - - try { - for (const videoId of this.viewersPerVideo.keys()) { - const notBefore = new Date().getTime() - - const viewers = this.viewersPerVideo.get(videoId) - - // Only keep not expired viewers - const newViewers: Viewer[] = [] - - // Filter new viewers - for (const viewer of viewers) { - if (viewer.expires > notBefore) { - newViewers.push(viewer) - } else { - this.idToViewer.delete(viewer.id) - } - } - - if (newViewers.length === 0) this.viewersPerVideo.delete(videoId) - else this.viewersPerVideo.set(videoId, newViewers) - - await this.notifyClients(videoId, newViewers.length) - } - } catch (err) { - logger.error('Error in video clean viewers scheduler.', { err, ...lTags() }) - } - - this.processingViewerCounters = false - } - - private async notifyClients (videoId: string | number, viewersLength: number) { - const video = await VideoModel.loadImmutableAttributes(videoId) - if (!video) return - - PeerTubeSocket.Instance.sendVideoViewsUpdate(video, viewersLength) - - logger.debug('Video viewers update for %s is %d.', video.url, viewersLength, lTags()) - } - - private generateViewerId (ip: string, videoUUID: string) { - return sha256(this.salt + '-' + ip + '-' + videoUUID) - } - - private async federateViewerIfNeeded (video: MVideoImmutable, viewer: Viewer) { - // Federate the viewer if it's been a "long" time we did not - const now = new Date().getTime() - const federationLimit = now - (VIEW_LIFETIME.VIEWER_COUNTER * 0.75) - - if (viewer.lastFederation && viewer.lastFederation > federationLimit) return - - await sendView({ byActor: await getServerActor(), video, type: 'viewer', viewerIdentifier: viewer.id }) - viewer.lastFederation = now - } -} diff --git a/server/lib/views/shared/video-viewer-stats.ts b/server/lib/views/shared/video-viewer-stats.ts deleted file mode 100644 index ebd963e59..000000000 --- a/server/lib/views/shared/video-viewer-stats.ts +++ /dev/null @@ -1,196 +0,0 @@ -import { Transaction } from 'sequelize/types' -import { isTestOrDevInstance } from '@server/helpers/core-utils' -import { GeoIP } from '@server/helpers/geo-ip' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { MAX_LOCAL_VIEWER_WATCH_SECTIONS, VIEW_LIFETIME } from '@server/initializers/constants' -import { sequelizeTypescript } from '@server/initializers/database' -import { sendCreateWatchAction } from '@server/lib/activitypub/send' -import { getLocalVideoViewerActivityPubUrl } from '@server/lib/activitypub/url' -import { Redis } from '@server/lib/redis' -import { VideoModel } from '@server/models/video/video' -import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' -import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section' -import { MVideo, MVideoImmutable } from '@server/types/models' -import { VideoViewEvent } from '@shared/models' - -const lTags = loggerTagsFactory('views') - -type LocalViewerStats = { - firstUpdated: number // Date.getTime() - lastUpdated: number // Date.getTime() - - watchSections: { - start: number - end: number - }[] - - watchTime: number - - country: string - - videoId: number -} - -export class VideoViewerStats { - private processingViewersStats = false - - constructor () { - setInterval(() => this.processViewerStats(), VIEW_LIFETIME.VIEWER_STATS) - } - - // --------------------------------------------------------------------------- - - async addLocalViewer (options: { - video: MVideoImmutable - currentTime: number - ip: string - viewEvent?: VideoViewEvent - }) { - const { video, ip, viewEvent, currentTime } = options - - logger.debug('Adding local viewer to video stats %s.', video.uuid, { currentTime, viewEvent, ...lTags(video.uuid) }) - - return this.updateLocalViewerStats({ video, viewEvent, currentTime, ip }) - } - - // --------------------------------------------------------------------------- - - async getWatchTime (videoId: number, ip: string) { - const stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ ip, videoId }) - - return stats?.watchTime || 0 - } - - // --------------------------------------------------------------------------- - - private async updateLocalViewerStats (options: { - video: MVideoImmutable - ip: string - currentTime: number - viewEvent?: VideoViewEvent - }) { - const { video, ip, viewEvent, currentTime } = options - const nowMs = new Date().getTime() - - let stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ ip, videoId: video.id }) - - if (stats && stats.watchSections.length >= MAX_LOCAL_VIEWER_WATCH_SECTIONS) { - logger.warn('Too much watch section to store for a viewer, skipping this one', { currentTime, viewEvent, ...lTags(video.uuid) }) - return - } - - if (!stats) { - const country = await GeoIP.Instance.safeCountryISOLookup(ip) - - stats = { - firstUpdated: nowMs, - lastUpdated: nowMs, - - watchSections: [], - - watchTime: 0, - - country, - videoId: video.id - } - } - - stats.lastUpdated = nowMs - - if (viewEvent === 'seek' || stats.watchSections.length === 0) { - stats.watchSections.push({ - start: currentTime, - end: currentTime - }) - } else { - const lastSection = stats.watchSections[stats.watchSections.length - 1] - - if (lastSection.start > currentTime) { - logger.debug('Invalid end watch section %d. Last start record was at %d. Starting a new section.', currentTime, lastSection.start) - - stats.watchSections.push({ - start: currentTime, - end: currentTime - }) - } else { - lastSection.end = currentTime - } - } - - stats.watchTime = this.buildWatchTimeFromSections(stats.watchSections) - - logger.debug('Set local video viewer stats for video %s.', video.uuid, { stats, ...lTags(video.uuid) }) - - await Redis.Instance.setLocalVideoViewer(ip, video.id, stats) - } - - async processViewerStats () { - if (this.processingViewersStats) return - this.processingViewersStats = true - - if (!isTestOrDevInstance()) logger.info('Processing viewer statistics.', lTags()) - - const now = new Date().getTime() - - try { - const allKeys = await Redis.Instance.listLocalVideoViewerKeys() - - for (const key of allKeys) { - const stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ key }) - - // Process expired stats - if (stats.lastUpdated > now - VIEW_LIFETIME.VIEWER_STATS) { - continue - } - - try { - await sequelizeTypescript.transaction(async t => { - const video = await VideoModel.load(stats.videoId, t) - if (!video) return - - const statsModel = await this.saveViewerStats(video, stats, t) - - if (video.remote) { - await sendCreateWatchAction(statsModel, t) - } - }) - - await Redis.Instance.deleteLocalVideoViewersKeys(key) - } catch (err) { - logger.error('Cannot process viewer stats for Redis key %s.', key, { err, ...lTags() }) - } - } - } catch (err) { - logger.error('Error in video save viewers stats scheduler.', { err, ...lTags() }) - } - - this.processingViewersStats = false - } - - private async saveViewerStats (video: MVideo, stats: LocalViewerStats, transaction: Transaction) { - const statsModel = new LocalVideoViewerModel({ - startDate: new Date(stats.firstUpdated), - endDate: new Date(stats.lastUpdated), - watchTime: stats.watchTime, - country: stats.country, - videoId: video.id - }) - - statsModel.url = getLocalVideoViewerActivityPubUrl(statsModel) - statsModel.Video = video as VideoModel - - await statsModel.save({ transaction }) - - statsModel.WatchSections = await LocalVideoViewerWatchSectionModel.bulkCreateSections({ - localVideoViewerId: statsModel.id, - watchSections: stats.watchSections, - transaction - }) - - return statsModel - } - - private buildWatchTimeFromSections (sections: { start: number, end: number }[]) { - return sections.reduce((p, current) => p + (current.end - current.start), 0) - } -} diff --git a/server/lib/views/shared/video-views.ts b/server/lib/views/shared/video-views.ts deleted file mode 100644 index e563287e1..000000000 --- a/server/lib/views/shared/video-views.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { sendView } from '@server/lib/activitypub/send/send-view' -import { getCachedVideoDuration } from '@server/lib/video' -import { getServerActor } from '@server/models/application/application' -import { MVideo, MVideoImmutable } from '@server/types/models' -import { buildUUID } from '@shared/extra-utils' -import { Redis } from '../../redis' - -const lTags = loggerTagsFactory('views') - -export class VideoViews { - - async addLocalView (options: { - video: MVideoImmutable - ip: string - watchTime: number - }) { - const { video, ip, watchTime } = options - - logger.debug('Adding local view to video %s.', video.uuid, { watchTime, ...lTags(video.uuid) }) - - if (!await this.hasEnoughWatchTime(video, watchTime)) return false - - const viewExists = await Redis.Instance.doesVideoIPViewExist(ip, video.uuid) - if (viewExists) return false - - await Redis.Instance.setIPVideoView(ip, video.uuid) - - await this.addView(video) - - await sendView({ byActor: await getServerActor(), video, type: 'view', viewerIdentifier: buildUUID() }) - - return true - } - - async addRemoteView (options: { - video: MVideo - }) { - const { video } = options - - logger.debug('Adding remote view to video %s.', video.uuid, { ...lTags(video.uuid) }) - - await this.addView(video) - - return true - } - - // --------------------------------------------------------------------------- - - private async addView (video: MVideoImmutable) { - const promises: Promise[] = [] - - if (video.isOwned()) { - promises.push(Redis.Instance.addLocalVideoView(video.id)) - } - - promises.push(Redis.Instance.addVideoViewStats(video.id)) - - await Promise.all(promises) - } - - private async hasEnoughWatchTime (video: MVideoImmutable, watchTime: number) { - const { duration, isLive } = await getCachedVideoDuration(video.id) - - if (isLive || duration >= 30) return watchTime >= 30 - - // Check more than 50% of the video is watched - return duration / watchTime < 2 - } -} diff --git a/server/lib/views/video-views-manager.ts b/server/lib/views/video-views-manager.ts deleted file mode 100644 index c088dad5e..000000000 --- a/server/lib/views/video-views-manager.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { MVideo, MVideoImmutable } from '@server/types/models' -import { VideoViewEvent } from '@shared/models' -import { VideoScope, VideoViewerCounters, VideoViewerStats, VideoViews, ViewerScope } from './shared' - -/** - * If processing a local view: - * - We update viewer information (segments watched, watch time etc) - * - We add +1 to video viewers counter if this is a new viewer - * - We add +1 to video views counter if this is a new view and if the user watched enough seconds - * - We send AP message to notify about this viewer and this view - * - We update last video time for the user if authenticated - * - * If processing a remote view: - * - We add +1 to video viewers counter - * - We add +1 to video views counter - * - * A viewer is a someone that watched one or multiple sections of a video - * A viewer that watched only a few seconds of a video may not increment the video views counter - * Viewers statistics are sent to origin instance using the `WatchAction` ActivityPub object - * - */ - -const lTags = loggerTagsFactory('views') - -export class VideoViewsManager { - - private static instance: VideoViewsManager - - private videoViewerStats: VideoViewerStats - private videoViewerCounters: VideoViewerCounters - private videoViews: VideoViews - - private constructor () { - } - - init () { - this.videoViewerStats = new VideoViewerStats() - this.videoViewerCounters = new VideoViewerCounters() - this.videoViews = new VideoViews() - } - - async processLocalView (options: { - video: MVideoImmutable - currentTime: number - ip: string | null - viewEvent?: VideoViewEvent - }) { - const { video, ip, viewEvent, currentTime } = options - - logger.debug('Processing local view for %s and ip %s.', video.url, ip, lTags()) - - await this.videoViewerStats.addLocalViewer({ video, ip, viewEvent, currentTime }) - - const successViewer = await this.videoViewerCounters.addLocalViewer({ video, ip }) - - // Do it after added local viewer to fetch updated information - const watchTime = await this.videoViewerStats.getWatchTime(video.id, ip) - - const successView = await this.videoViews.addLocalView({ video, watchTime, ip }) - - return { successView, successViewer } - } - - async processRemoteView (options: { - video: MVideo - viewerId: string | null - viewerExpires?: Date - }) { - const { video, viewerId, viewerExpires } = options - - logger.debug('Processing remote view for %s.', video.url, { viewerExpires, viewerId, ...lTags() }) - - if (viewerExpires) await this.videoViewerCounters.addRemoteViewer({ video, viewerId, viewerExpires }) - else await this.videoViews.addRemoteView({ video }) - } - - getViewers (video: MVideo) { - return this.videoViewerCounters.getViewers(video) - } - - getTotalViewers (options: { - viewerScope: ViewerScope - videoScope: VideoScope - }) { - return this.videoViewerCounters.getTotalViewers(options) - } - - buildViewerExpireTime () { - return this.videoViewerCounters.buildViewerExpireTime() - } - - processViewerStats () { - return this.videoViewerStats.processViewerStats() - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} diff --git a/server/lib/worker/parent-process.ts b/server/lib/worker/parent-process.ts deleted file mode 100644 index 48b6c682b..000000000 --- a/server/lib/worker/parent-process.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { join } from 'path' -import Piscina from 'piscina' -import { processImage } from '@server/helpers/image-utils' -import { JOB_CONCURRENCY, WORKER_THREADS } from '@server/initializers/constants' -import { httpBroadcast } from './workers/http-broadcast' -import { downloadImage } from './workers/image-downloader' - -let downloadImageWorker: Piscina - -function downloadImageFromWorker (options: Parameters[0]): Promise> { - if (!downloadImageWorker) { - downloadImageWorker = new Piscina({ - filename: join(__dirname, 'workers', 'image-downloader.js'), - concurrentTasksPerWorker: WORKER_THREADS.DOWNLOAD_IMAGE.CONCURRENCY, - maxThreads: WORKER_THREADS.DOWNLOAD_IMAGE.MAX_THREADS - }) - } - - return downloadImageWorker.run(options) -} - -// --------------------------------------------------------------------------- - -let processImageWorker: Piscina - -function processImageFromWorker (options: Parameters[0]): Promise> { - if (!processImageWorker) { - processImageWorker = new Piscina({ - filename: join(__dirname, 'workers', 'image-processor.js'), - concurrentTasksPerWorker: WORKER_THREADS.PROCESS_IMAGE.CONCURRENCY, - maxThreads: WORKER_THREADS.PROCESS_IMAGE.MAX_THREADS - }) - } - - return processImageWorker.run(options) -} - -// --------------------------------------------------------------------------- - -let parallelHTTPBroadcastWorker: Piscina - -function parallelHTTPBroadcastFromWorker (options: Parameters[0]): Promise> { - if (!parallelHTTPBroadcastWorker) { - parallelHTTPBroadcastWorker = new Piscina({ - filename: join(__dirname, 'workers', 'http-broadcast.js'), - // Keep it sync with job concurrency so the worker will accept all the requests sent by the parallelized jobs - concurrentTasksPerWorker: JOB_CONCURRENCY['activitypub-http-broadcast-parallel'], - maxThreads: 1 - }) - } - - return parallelHTTPBroadcastWorker.run(options) -} - -// --------------------------------------------------------------------------- - -let sequentialHTTPBroadcastWorker: Piscina - -function sequentialHTTPBroadcastFromWorker (options: Parameters[0]): Promise> { - if (!sequentialHTTPBroadcastWorker) { - sequentialHTTPBroadcastWorker = new Piscina({ - filename: join(__dirname, 'workers', 'http-broadcast.js'), - // Keep it sync with job concurrency so the worker will accept all the requests sent by the parallelized jobs - concurrentTasksPerWorker: JOB_CONCURRENCY['activitypub-http-broadcast'], - maxThreads: 1 - }) - } - - return sequentialHTTPBroadcastWorker.run(options) -} - -export { - downloadImageFromWorker, - processImageFromWorker, - parallelHTTPBroadcastFromWorker, - sequentialHTTPBroadcastFromWorker -} diff --git a/server/lib/worker/workers/http-broadcast.ts b/server/lib/worker/workers/http-broadcast.ts deleted file mode 100644 index 8c9c6b8ca..000000000 --- a/server/lib/worker/workers/http-broadcast.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { map } from 'bluebird' -import { logger } from '@server/helpers/logger' -import { doRequest, PeerTubeRequestOptions } from '@server/helpers/requests' -import { BROADCAST_CONCURRENCY } from '@server/initializers/constants' - -async function httpBroadcast (payload: { - uris: string[] - requestOptions: PeerTubeRequestOptions -}) { - const { uris, requestOptions } = payload - - const badUrls: string[] = [] - const goodUrls: string[] = [] - - await map(uris, async uri => { - try { - await doRequest(uri, requestOptions) - goodUrls.push(uri) - } catch (err) { - logger.debug('HTTP broadcast to %s failed.', uri, { err }) - badUrls.push(uri) - } - }, { concurrency: BROADCAST_CONCURRENCY }) - - return { goodUrls, badUrls } -} - -module.exports = httpBroadcast - -export { - httpBroadcast -} diff --git a/server/lib/worker/workers/image-downloader.ts b/server/lib/worker/workers/image-downloader.ts deleted file mode 100644 index 209594589..000000000 --- a/server/lib/worker/workers/image-downloader.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { remove } from 'fs-extra' -import { join } from 'path' -import { processImage } from '@server/helpers/image-utils' -import { doRequestAndSaveToFile } from '@server/helpers/requests' -import { CONFIG } from '@server/initializers/config' - -async function downloadImage (options: { - url: string - destDir: string - destName: string - size: { width: number, height: number } -}) { - const { url, destDir, destName, size } = options - - const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName) - await doRequestAndSaveToFile(url, tmpPath) - - const destPath = join(destDir, destName) - - try { - await processImage({ path: tmpPath, destination: destPath, newSize: size }) - } catch (err) { - await remove(tmpPath) - - throw err - } - - return destPath -} - -module.exports = downloadImage - -export { - downloadImage -} diff --git a/server/lib/worker/workers/image-processor.ts b/server/lib/worker/workers/image-processor.ts deleted file mode 100644 index 0ab41a5a0..000000000 --- a/server/lib/worker/workers/image-processor.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { processImage } from '@server/helpers/image-utils' - -module.exports = processImage - -export { - processImage -} diff --git a/server/middlewares/activitypub.ts b/server/middlewares/activitypub.ts deleted file mode 100644 index 261b9f690..000000000 --- a/server/middlewares/activitypub.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { NextFunction, Request, Response } from 'express' -import { isActorDeleteActivityValid } from '@server/helpers/custom-validators/activitypub/actor' -import { getAPId } from '@server/lib/activitypub/activity' -import { wrapWithSpanAndContext } from '@server/lib/opentelemetry/tracing' -import { ActivityDelete, ActivityPubSignature, HttpStatusCode } from '@shared/models' -import { logger } from '../helpers/logger' -import { isHTTPSignatureVerified, isJsonLDSignatureVerified, parseHTTPSignature } from '../helpers/peertube-crypto' -import { ACCEPT_HEADERS, ACTIVITY_PUB, HTTP_SIGNATURE } from '../initializers/constants' -import { getOrCreateAPActor, loadActorUrlOrGetFromWebfinger } from '../lib/activitypub/actors' - -async function checkSignature (req: Request, res: Response, next: NextFunction) { - try { - const httpSignatureChecked = await checkHttpSignature(req, res) - if (httpSignatureChecked !== true) return - - const actor = res.locals.signature.actor - - // Forwarded activity - const bodyActor = req.body.actor - const bodyActorId = getAPId(bodyActor) - if (bodyActorId && bodyActorId !== actor.url) { - const jsonLDSignatureChecked = await checkJsonLDSignature(req, res) - if (jsonLDSignatureChecked !== true) return - } - - return next() - } catch (err) { - const activity: ActivityDelete = req.body - if (isActorDeleteActivityValid(activity) && activity.object === activity.actor) { - logger.debug('Handling signature error on actor delete activity', { err }) - return res.status(HttpStatusCode.NO_CONTENT_204).end() - } - - logger.warn('Error in ActivityPub signature checker.', { err }) - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'ActivityPub signature could not be checked' - }) - } -} - -function executeIfActivityPub (req: Request, res: Response, next: NextFunction) { - const accepted = req.accepts(ACCEPT_HEADERS) - if (accepted === false || ACTIVITY_PUB.POTENTIAL_ACCEPT_HEADERS.includes(accepted) === false) { - // Bypass this route - return next('route') - } - - logger.debug('ActivityPub request for %s.', req.url) - - return next() -} - -// --------------------------------------------------------------------------- - -export { - checkSignature, - executeIfActivityPub, - checkHttpSignature -} - -// --------------------------------------------------------------------------- - -async function checkHttpSignature (req: Request, res: Response) { - return wrapWithSpanAndContext('peertube.activitypub.checkHTTPSignature', async () => { - // FIXME: compatibility with http-signature < v1.3 - const sig = req.headers[HTTP_SIGNATURE.HEADER_NAME] as string - if (sig && sig.startsWith('Signature ') === true) req.headers[HTTP_SIGNATURE.HEADER_NAME] = sig.replace(/^Signature /, '') - - let parsed: any - - try { - parsed = parseHTTPSignature(req, HTTP_SIGNATURE.CLOCK_SKEW_SECONDS) - } catch (err) { - logger.warn('Invalid signature because of exception in signature parser', { reqBody: req.body, err }) - - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: err.message - }) - return false - } - - const keyId = parsed.keyId - if (!keyId) { - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Invalid key ID', - data: { - keyId - } - }) - return false - } - - logger.debug('Checking HTTP signature of actor %s...', keyId) - - let [ actorUrl ] = keyId.split('#') - if (actorUrl.startsWith('acct:')) { - actorUrl = await loadActorUrlOrGetFromWebfinger(actorUrl.replace(/^acct:/, '')) - } - - const actor = await getOrCreateAPActor(actorUrl) - - const verified = isHTTPSignatureVerified(parsed, actor) - if (verified !== true) { - logger.warn('Signature from %s is invalid', actorUrl, { parsed }) - - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Invalid signature', - data: { - actorUrl - } - }) - return false - } - - res.locals.signature = { actor } - return true - }) -} - -async function checkJsonLDSignature (req: Request, res: Response) { - return wrapWithSpanAndContext('peertube.activitypub.JSONLDSignature', async () => { - const signatureObject: ActivityPubSignature = req.body.signature - - if (!signatureObject?.creator) { - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Object and creator signature do not match' - }) - return false - } - - const [ creator ] = signatureObject.creator.split('#') - - logger.debug('Checking JsonLD signature of actor %s...', creator) - - const actor = await getOrCreateAPActor(creator) - const verified = await isJsonLDSignatureVerified(actor, req.body) - - if (verified !== true) { - logger.warn('Signature not verified.', req.body) - - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Signature could not be verified' - }) - return false - } - - res.locals.signature = { actor } - return true - }) -} diff --git a/server/middlewares/async.ts b/server/middlewares/async.ts deleted file mode 100644 index 7e131257d..000000000 --- a/server/middlewares/async.ts +++ /dev/null @@ -1,44 +0,0 @@ -import Bluebird from 'bluebird' -import { NextFunction, Request, RequestHandler, Response } from 'express' -import { ValidationChain } from 'express-validator' -import { ExpressPromiseHandler } from '@server/types/express-handler' -import { retryTransactionWrapper } from '../helpers/database-utils' - -// Syntactic sugar to avoid try/catch in express controllers/middlewares - -export type RequestPromiseHandler = ValidationChain | ExpressPromiseHandler - -function asyncMiddleware (fun: RequestPromiseHandler | RequestPromiseHandler[]) { - return (req: Request, res: Response, next: NextFunction) => { - if (Array.isArray(fun) === true) { - return Bluebird.each(fun as RequestPromiseHandler[], f => { - return new Promise((resolve, reject) => { - return asyncMiddleware(f)(req, res, err => { - if (err) return reject(err) - - return resolve() - }) - }) - }).then(() => next()) - .catch(err => next(err)) - } - - return Promise.resolve((fun as RequestHandler)(req, res, next)) - .catch(err => next(err)) - } -} - -function asyncRetryTransactionMiddleware (fun: (req: Request, res: Response, next: NextFunction) => Promise) { - return (req: Request, res: Response, next: NextFunction) => { - return Promise.resolve( - retryTransactionWrapper(fun, req, res, next) - ).catch(err => next(err)) - } -} - -// --------------------------------------------------------------------------- - -export { - asyncMiddleware, - asyncRetryTransactionMiddleware -} diff --git a/server/middlewares/auth.ts b/server/middlewares/auth.ts deleted file mode 100644 index 39a7b2998..000000000 --- a/server/middlewares/auth.ts +++ /dev/null @@ -1,113 +0,0 @@ -import express from 'express' -import { Socket } from 'socket.io' -import { getAccessToken } from '@server/lib/auth/oauth-model' -import { RunnerModel } from '@server/models/runner/runner' -import { HttpStatusCode } from '../../shared/models/http/http-error-codes' -import { logger } from '../helpers/logger' -import { handleOAuthAuthenticate } from '../lib/auth/oauth' -import { ServerErrorCode } from '@shared/models' - -function authenticate (req: express.Request, res: express.Response, next: express.NextFunction) { - handleOAuthAuthenticate(req, res) - .then((token: any) => { - res.locals.oauth = { token } - res.locals.authenticated = true - - return next() - }) - .catch(err => { - logger.info('Cannot authenticate.', { err }) - - return res.fail({ - status: err.status, - message: 'Token is invalid', - type: err.name - }) - }) -} - -function authenticateSocket (socket: Socket, next: (err?: any) => void) { - const accessToken = socket.handshake.query['accessToken'] - - logger.debug('Checking access token in runner.') - - if (!accessToken) return next(new Error('No access token provided')) - if (typeof accessToken !== 'string') return next(new Error('Access token is invalid')) - - getAccessToken(accessToken) - .then(tokenDB => { - const now = new Date() - - if (!tokenDB || tokenDB.accessTokenExpiresAt < now || tokenDB.refreshTokenExpiresAt < now) { - return next(new Error('Invalid access token.')) - } - - socket.handshake.auth.user = tokenDB.User - - return next() - }) - .catch(err => logger.error('Cannot get access token.', { err })) -} - -function authenticatePromise (options: { - req: express.Request - res: express.Response - errorMessage?: string - errorStatus?: HttpStatusCode - errorType?: ServerErrorCode -}) { - const { req, res, errorMessage = 'Not authenticated', errorStatus = HttpStatusCode.UNAUTHORIZED_401, errorType } = options - return new Promise(resolve => { - // Already authenticated? (or tried to) - if (res.locals.oauth?.token.User) return resolve() - - if (res.locals.authenticated === false) { - return res.fail({ - status: errorStatus, - type: errorType, - message: errorMessage - }) - } - - authenticate(req, res, () => resolve()) - }) -} - -function optionalAuthenticate (req: express.Request, res: express.Response, next: express.NextFunction) { - if (req.header('authorization')) return authenticate(req, res, next) - - res.locals.authenticated = false - - return next() -} - -// --------------------------------------------------------------------------- - -function authenticateRunnerSocket (socket: Socket, next: (err?: any) => void) { - const runnerToken = socket.handshake.auth['runnerToken'] - - logger.debug('Checking runner token in socket.') - - if (!runnerToken) return next(new Error('No runner token provided')) - if (typeof runnerToken !== 'string') return next(new Error('Runner token is invalid')) - - RunnerModel.loadByToken(runnerToken) - .then(runner => { - if (!runner) return next(new Error('Invalid runner token.')) - - socket.handshake.auth.runner = runner - - return next() - }) - .catch(err => logger.error('Cannot get runner token.', { err })) -} - -// --------------------------------------------------------------------------- - -export { - authenticate, - authenticateSocket, - authenticatePromise, - optionalAuthenticate, - authenticateRunnerSocket -} diff --git a/server/middlewares/cache/cache.ts b/server/middlewares/cache/cache.ts deleted file mode 100644 index 6041c76c3..000000000 --- a/server/middlewares/cache/cache.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' -import { ApiCache, APICacheOptions } from './shared' - -const defaultOptions: APICacheOptions = { - excludeStatus: [ - HttpStatusCode.FORBIDDEN_403, - HttpStatusCode.NOT_FOUND_404 - ] -} - -function cacheRoute (duration: string) { - const instance = new ApiCache(defaultOptions) - - return instance.buildMiddleware(duration) -} - -function cacheRouteFactory (options: APICacheOptions) { - const instance = new ApiCache({ ...defaultOptions, ...options }) - - return { instance, middleware: instance.buildMiddleware.bind(instance) } -} - -// --------------------------------------------------------------------------- - -function buildPodcastGroupsCache (options: { - channelId: number -}) { - return 'podcast-feed-' + options.channelId -} - -// --------------------------------------------------------------------------- - -export { - cacheRoute, - cacheRouteFactory, - - buildPodcastGroupsCache -} diff --git a/server/middlewares/cache/index.ts b/server/middlewares/cache/index.ts deleted file mode 100644 index 79b512828..000000000 --- a/server/middlewares/cache/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './cache' diff --git a/server/middlewares/cache/shared/api-cache.ts b/server/middlewares/cache/shared/api-cache.ts deleted file mode 100644 index b50b7dce4..000000000 --- a/server/middlewares/cache/shared/api-cache.ts +++ /dev/null @@ -1,314 +0,0 @@ -// Thanks: https://github.com/kwhitley/apicache -// We duplicated the library because it is unmaintened and prevent us to upgrade to recent NodeJS versions - -import express from 'express' -import { OutgoingHttpHeaders } from 'http' -import { isTestInstance, parseDurationToMs } from '@server/helpers/core-utils' -import { logger } from '@server/helpers/logger' -import { Redis } from '@server/lib/redis' -import { asyncMiddleware } from '@server/middlewares' -import { HttpStatusCode } from '@shared/models' - -export interface APICacheOptions { - headerBlacklist?: string[] - excludeStatus?: HttpStatusCode[] -} - -interface CacheObject { - status: number - headers: OutgoingHttpHeaders - data: any - encoding: BufferEncoding - timestamp: number -} - -export class ApiCache { - - private readonly options: APICacheOptions - private readonly timers: { [ id: string ]: NodeJS.Timeout } = {} - - private readonly index = { - groups: [] as string[], - all: [] as string[] - } - - // Cache keys per group - private groups: { [groupIndex: string]: string[] } = {} - - private readonly seed: number - - constructor (options: APICacheOptions) { - this.seed = new Date().getTime() - - this.options = { - headerBlacklist: [], - excludeStatus: [], - - ...options - } - } - - buildMiddleware (strDuration: string) { - const duration = parseDurationToMs(strDuration) - - return asyncMiddleware( - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - const key = this.getCacheKey(req) - const redis = Redis.Instance.getClient() - - if (!Redis.Instance.isConnected()) return this.makeResponseCacheable(res, next, key, duration) - - try { - const obj = await redis.hgetall(key) - if (obj?.response) { - return this.sendCachedResponse(req, res, JSON.parse(obj.response), duration) - } - - return this.makeResponseCacheable(res, next, key, duration) - } catch (err) { - return this.makeResponseCacheable(res, next, key, duration) - } - } - ) - } - - clearGroupSafe (group: string) { - const run = async () => { - const cacheKeys = this.groups[group] - if (!cacheKeys) return - - for (const key of cacheKeys) { - try { - await this.clear(key) - } catch (err) { - logger.error('Cannot clear ' + key, { err }) - } - } - - delete this.groups[group] - } - - void run() - } - - private getCacheKey (req: express.Request) { - return Redis.Instance.getPrefix() + 'api-cache-' + this.seed + '-' + req.originalUrl - } - - private shouldCacheResponse (response: express.Response) { - if (!response) return false - if (this.options.excludeStatus.includes(response.statusCode)) return false - - return true - } - - private addIndexEntries (key: string, res: express.Response) { - this.index.all.unshift(key) - - const groups = res.locals.apicacheGroups || [] - - for (const group of groups) { - if (!this.groups[group]) this.groups[group] = [] - - this.groups[group].push(key) - } - } - - private filterBlacklistedHeaders (headers: OutgoingHttpHeaders) { - return Object.keys(headers) - .filter(key => !this.options.headerBlacklist.includes(key)) - .reduce((acc, header) => { - acc[header] = headers[header] - - return acc - }, {}) - } - - private createCacheObject (status: number, headers: OutgoingHttpHeaders, data: any, encoding: BufferEncoding) { - return { - status, - headers: this.filterBlacklistedHeaders(headers), - data, - encoding, - - // Seconds since epoch, used to properly decrement max-age headers in cached responses. - timestamp: new Date().getTime() / 1000 - } as CacheObject - } - - private async cacheResponse (key: string, value: object, duration: number) { - const redis = Redis.Instance.getClient() - - if (Redis.Instance.isConnected()) { - await Promise.all([ - redis.hset(key, 'response', JSON.stringify(value)), - redis.hset(key, 'duration', duration + ''), - redis.expire(key, duration / 1000) - ]) - } - - // add automatic cache clearing from duration, includes max limit on setTimeout - this.timers[key] = setTimeout(() => { - this.clear(key) - .catch(err => logger.error('Cannot clear Redis key %s.', key, { err })) - }, Math.min(duration, 2147483647)) - } - - private accumulateContent (res: express.Response, content: any) { - if (!content) return - - if (typeof content === 'string') { - res.locals.apicache.content = (res.locals.apicache.content || '') + content - return - } - - if (Buffer.isBuffer(content)) { - let oldContent = res.locals.apicache.content - - if (typeof oldContent === 'string') { - oldContent = Buffer.from(oldContent) - } - - if (!oldContent) { - oldContent = Buffer.alloc(0) - } - - res.locals.apicache.content = Buffer.concat( - [ oldContent, content ], - oldContent.length + content.length - ) - - return - } - - res.locals.apicache.content = content - } - - private makeResponseCacheable (res: express.Response, next: express.NextFunction, key: string, duration: number) { - const self = this - - res.locals.apicache = { - write: res.write, - writeHead: res.writeHead, - end: res.end, - cacheable: true, - content: undefined, - headers: undefined - } - - // Patch express - res.writeHead = function () { - if (self.shouldCacheResponse(res)) { - res.setHeader('cache-control', 'max-age=' + (duration / 1000).toFixed(0)) - } else { - res.setHeader('cache-control', 'no-cache, no-store, must-revalidate') - } - - res.locals.apicache.headers = Object.assign({}, res.getHeaders()) - return res.locals.apicache.writeHead.apply(this, arguments as any) - } - - res.write = function (chunk: any) { - self.accumulateContent(res, chunk) - return res.locals.apicache.write.apply(this, arguments as any) - } - - res.end = function (content: any, encoding: BufferEncoding) { - if (self.shouldCacheResponse(res)) { - self.accumulateContent(res, content) - - if (res.locals.apicache.cacheable && res.locals.apicache.content) { - self.addIndexEntries(key, res) - - const headers = res.locals.apicache.headers || res.getHeaders() - const cacheObject = self.createCacheObject( - res.statusCode, - headers, - res.locals.apicache.content, - encoding - ) - self.cacheResponse(key, cacheObject, duration) - .catch(err => logger.error('Cannot cache response', { err })) - } - } - - res.locals.apicache.end.apply(this, arguments as any) - } as any - - next() - } - - private sendCachedResponse (request: express.Request, response: express.Response, cacheObject: CacheObject, duration: number) { - const headers = response.getHeaders() - - if (isTestInstance()) { - Object.assign(headers, { - 'x-api-cache-cached': 'true' - }) - } - - Object.assign(headers, this.filterBlacklistedHeaders(cacheObject.headers || {}), { - // Set properly decremented max-age header - // This ensures that max-age is in sync with the cache expiration - 'cache-control': - 'max-age=' + - Math.max( - 0, - (duration / 1000 - (new Date().getTime() / 1000 - cacheObject.timestamp)) - ).toFixed(0) - }) - - // unstringify buffers - let data = cacheObject.data - if (data && data.type === 'Buffer') { - data = typeof data.data === 'number' - ? Buffer.alloc(data.data) - : Buffer.from(data.data) - } - - // Test Etag against If-None-Match for 304 - const cachedEtag = cacheObject.headers.etag - const requestEtag = request.headers['if-none-match'] - - if (requestEtag && cachedEtag === requestEtag) { - response.writeHead(304, headers) - return response.end() - } - - response.writeHead(cacheObject.status || 200, headers) - - return response.end(data, cacheObject.encoding) - } - - private async clear (target: string) { - const redis = Redis.Instance.getClient() - - if (target) { - clearTimeout(this.timers[target]) - delete this.timers[target] - - try { - await redis.del(target) - } catch (err) { - logger.error('Cannot delete %s in redis cache.', target, { err }) - } - - this.index.all = this.index.all.filter(key => key !== target) - } else { - for (const key of this.index.all) { - clearTimeout(this.timers[key]) - delete this.timers[key] - - try { - await redis.del(key) - } catch (err) { - logger.error('Cannot delete %s in redis cache.', key, { err }) - } - } - - this.index.all = [] - } - - return this.index - } -} diff --git a/server/middlewares/cache/shared/index.ts b/server/middlewares/cache/shared/index.ts deleted file mode 100644 index c707eaf7a..000000000 --- a/server/middlewares/cache/shared/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './api-cache' diff --git a/server/middlewares/csp.ts b/server/middlewares/csp.ts deleted file mode 100644 index e2a75a17e..000000000 --- a/server/middlewares/csp.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { contentSecurityPolicy } from 'helmet' -import { CONFIG } from '../initializers/config' - -const baseDirectives = Object.assign({}, - { - defaultSrc: [ '\'none\'' ], // by default, not specifying default-src = '*' - connectSrc: [ '*', 'data:' ], - mediaSrc: [ '\'self\'', 'https:', 'blob:' ], - fontSrc: [ '\'self\'', 'data:' ], - imgSrc: [ '\'self\'', 'data:', 'blob:' ], - scriptSrc: [ '\'self\' \'unsafe-inline\' \'unsafe-eval\'', 'blob:' ], - styleSrc: [ '\'self\' \'unsafe-inline\'' ], - objectSrc: [ '\'none\'' ], // only define to allow plugins, else let defaultSrc 'none' block it - formAction: [ '\'self\'' ], - frameAncestors: [ '\'none\'' ], - baseUri: [ '\'self\'' ], - manifestSrc: [ '\'self\'' ], - frameSrc: [ '\'self\'' ], // instead of deprecated child-src / self because of test-embed - workerSrc: [ '\'self\'', 'blob:' ] // instead of deprecated child-src - }, - CONFIG.CSP.REPORT_URI ? { reportUri: CONFIG.CSP.REPORT_URI } : {}, - CONFIG.WEBSERVER.SCHEME === 'https' ? { upgradeInsecureRequests: [] } : {} -) - -const baseCSP = contentSecurityPolicy({ - directives: baseDirectives, - reportOnly: CONFIG.CSP.REPORT_ONLY -}) - -const embedCSP = contentSecurityPolicy({ - directives: Object.assign({}, baseDirectives, { frameAncestors: [ '*' ] }), - reportOnly: CONFIG.CSP.REPORT_ONLY -}) - -// --------------------------------------------------------------------------- - -export { - baseCSP, - embedCSP -} diff --git a/server/middlewares/error.ts b/server/middlewares/error.ts deleted file mode 100644 index 94762e355..000000000 --- a/server/middlewares/error.ts +++ /dev/null @@ -1,63 +0,0 @@ -import express from 'express' -import { ProblemDocument, ProblemDocumentExtension } from 'http-problem-details' -import { logger } from '@server/helpers/logger' -import { HttpStatusCode } from '@shared/models' - -function apiFailMiddleware (req: express.Request, res: express.Response, next: express.NextFunction) { - res.fail = options => { - const { status = HttpStatusCode.BAD_REQUEST_400, message, title, type, data, instance, tags } = options - - const extension = new ProblemDocumentExtension({ - ...data, - - docs: res.locals.docUrl, - code: type, - - // For <= 3.2 compatibility - error: message - }) - - res.status(status) - - if (!res.headersSent) { - res.setHeader('Content-Type', 'application/problem+json') - } - - const json = new ProblemDocument({ - status, - title, - instance, - - detail: message, - - type: type - ? `https://docs.joinpeertube.org/api-rest-reference.html#section/Errors/${type}` - : undefined - }, extension) - - logger.debug('Bad HTTP request.', { json, tags }) - - res.json(json) - } - - if (next) next() -} - -function handleStaticError (err: any, req: express.Request, res: express.Response, next: express.NextFunction) { - const message = err.message || '' - - if (message.includes('ENOENT')) { - return res.fail({ - status: err.status || HttpStatusCode.INTERNAL_SERVER_ERROR_500, - message: err.message, - type: err.name - }) - } - - return next(err) -} - -export { - apiFailMiddleware, - handleStaticError -} diff --git a/server/middlewares/index.ts b/server/middlewares/index.ts deleted file mode 100644 index b40f864ce..000000000 --- a/server/middlewares/index.ts +++ /dev/null @@ -1,15 +0,0 @@ -export * from './validators' -export * from './cache' -export * from './activitypub' -export * from './async' -export * from './auth' -export * from './pagination' -export * from './rate-limiter' -export * from './robots' -export * from './servers' -export * from './sort' -export * from './user-right' -export * from './dnt' -export * from './error' -export * from './doc' -export * from './csp' diff --git a/server/middlewares/pagination.ts b/server/middlewares/pagination.ts deleted file mode 100644 index 17e43f743..000000000 --- a/server/middlewares/pagination.ts +++ /dev/null @@ -1,19 +0,0 @@ -import express from 'express' -import { forceNumber } from '@shared/core-utils' -import { PAGINATION } from '../initializers/constants' - -function setDefaultPagination (req: express.Request, res: express.Response, next: express.NextFunction) { - if (!req.query.start) req.query.start = 0 - else req.query.start = forceNumber(req.query.start) - - if (!req.query.count) req.query.count = PAGINATION.GLOBAL.COUNT.DEFAULT - else req.query.count = forceNumber(req.query.count) - - return next() -} - -// --------------------------------------------------------------------------- - -export { - setDefaultPagination -} diff --git a/server/middlewares/rate-limiter.ts b/server/middlewares/rate-limiter.ts deleted file mode 100644 index 143d43632..000000000 --- a/server/middlewares/rate-limiter.ts +++ /dev/null @@ -1,59 +0,0 @@ -import express from 'express' -import RateLimit, { Options as RateLimitHandlerOptions } from 'express-rate-limit' -import { CONFIG } from '@server/initializers/config' -import { RunnerModel } from '@server/models/runner/runner' -import { UserRole } from '@shared/models' -import { optionalAuthenticate } from './auth' - -const whitelistRoles = new Set([ UserRole.ADMINISTRATOR, UserRole.MODERATOR ]) - -export function buildRateLimiter (options: { - windowMs: number - max: number - skipFailedRequests?: boolean -}) { - return RateLimit({ - windowMs: options.windowMs, - max: options.max, - skipFailedRequests: options.skipFailedRequests, - - handler: (req, res, next, options) => { - // Bypass rate limit for registered runners - if (req.body?.runnerToken) { - return RunnerModel.loadByToken(req.body.runnerToken) - .then(runner => { - if (runner) return next() - - return sendRateLimited(res, options) - }) - } - - // Bypass rate limit for admins/moderators - return optionalAuthenticate(req, res, () => { - if (res.locals.authenticated === true && whitelistRoles.has(res.locals.oauth.token.User.role)) { - return next() - } - - return sendRateLimited(res, options) - }) - } - }) -} - -export const apiRateLimiter = buildRateLimiter({ - windowMs: CONFIG.RATES_LIMIT.API.WINDOW_MS, - max: CONFIG.RATES_LIMIT.API.MAX -}) - -export const activityPubRateLimiter = buildRateLimiter({ - windowMs: CONFIG.RATES_LIMIT.ACTIVITY_PUB.WINDOW_MS, - max: CONFIG.RATES_LIMIT.ACTIVITY_PUB.MAX -}) - -// --------------------------------------------------------------------------- -// Private -// --------------------------------------------------------------------------- - -function sendRateLimited (res: express.Response, options: RateLimitHandlerOptions) { - return res.status(options.statusCode).send(options.message) -} diff --git a/server/middlewares/servers.ts b/server/middlewares/servers.ts deleted file mode 100644 index ebfa03e6c..000000000 --- a/server/middlewares/servers.ts +++ /dev/null @@ -1,29 +0,0 @@ -import express from 'express' -import { HttpStatusCode } from '../../shared/models/http/http-error-codes' -import { getHostWithPort } from '../helpers/express-utils' - -function setBodyHostsPort (req: express.Request, res: express.Response, next: express.NextFunction) { - if (!req.body.hosts) return next() - - for (let i = 0; i < req.body.hosts.length; i++) { - const hostWithPort = getHostWithPort(req.body.hosts[i]) - - // Problem with the url parsing? - if (hostWithPort === null) { - return res.fail({ - status: HttpStatusCode.INTERNAL_SERVER_ERROR_500, - message: 'Could not parse hosts' - }) - } - - req.body.hosts[i] = hostWithPort - } - - return next() -} - -// --------------------------------------------------------------------------- - -export { - setBodyHostsPort -} diff --git a/server/middlewares/user-right.ts b/server/middlewares/user-right.ts deleted file mode 100644 index 7d53e8341..000000000 --- a/server/middlewares/user-right.ts +++ /dev/null @@ -1,26 +0,0 @@ -import express from 'express' -import { HttpStatusCode, UserRight } from '@shared/models' -import { logger } from '../helpers/logger' - -function ensureUserHasRight (userRight: UserRight) { - return function (req: express.Request, res: express.Response, next: express.NextFunction) { - const user = res.locals.oauth.token.user - if (user.hasRight(userRight) === false) { - const message = `User ${user.username} does not have right ${userRight} to access to ${req.path}.` - logger.info(message) - - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message - }) - } - - return next() - } -} - -// --------------------------------------------------------------------------- - -export { - ensureUserHasRight -} diff --git a/server/middlewares/validators/abuse.ts b/server/middlewares/validators/abuse.ts deleted file mode 100644 index 70bae1775..000000000 --- a/server/middlewares/validators/abuse.ts +++ /dev/null @@ -1,255 +0,0 @@ -import express from 'express' -import { body, param, query } from 'express-validator' -import { - areAbusePredefinedReasonsValid, - isAbuseFilterValid, - isAbuseMessageValid, - isAbuseModerationCommentValid, - isAbusePredefinedReasonValid, - isAbuseReasonValid, - isAbuseStateValid, - isAbuseTimestampCoherent, - isAbuseTimestampValid, - isAbuseVideoIsValid -} from '@server/helpers/custom-validators/abuses' -import { exists, isIdOrUUIDValid, isIdValid, toCompleteUUID, toIntOrNull } from '@server/helpers/custom-validators/misc' -import { logger } from '@server/helpers/logger' -import { AbuseMessageModel } from '@server/models/abuse/abuse-message' -import { AbuseCreate, UserRight } from '@shared/models' -import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' -import { areValidationErrors, doesAbuseExist, doesAccountIdExist, doesCommentIdExist, doesVideoExist } from './shared' -import { forceNumber } from '@shared/core-utils' - -const abuseReportValidator = [ - body('account.id') - .optional() - .custom(isIdValid), - - body('video.id') - .optional() - .customSanitizer(toCompleteUUID) - .custom(isIdOrUUIDValid), - body('video.startAt') - .optional() - .customSanitizer(toIntOrNull) - .custom(isAbuseTimestampValid), - body('video.endAt') - .optional() - .customSanitizer(toIntOrNull) - .custom(isAbuseTimestampValid) - .bail() - .custom(isAbuseTimestampCoherent) - .withMessage('Should have a startAt timestamp beginning before endAt'), - - body('comment.id') - .optional() - .custom(isIdValid), - - body('reason') - .custom(isAbuseReasonValid), - - body('predefinedReasons') - .optional() - .custom(areAbusePredefinedReasonsValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const body: AbuseCreate = req.body - - if (body.video?.id && !await doesVideoExist(body.video.id, res)) return - if (body.account?.id && !await doesAccountIdExist(body.account.id, res)) return - if (body.comment?.id && !await doesCommentIdExist(body.comment.id, res)) return - - if (!body.video?.id && !body.account?.id && !body.comment?.id) { - res.fail({ message: 'video id or account id or comment id is required.' }) - return - } - - return next() - } -] - -const abuseGetValidator = [ - param('id') - .custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesAbuseExist(req.params.id, res)) return - - return next() - } -] - -const abuseUpdateValidator = [ - param('id') - .custom(isIdValid), - - body('state') - .optional() - .custom(isAbuseStateValid), - body('moderationComment') - .optional() - .custom(isAbuseModerationCommentValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesAbuseExist(req.params.id, res)) return - - return next() - } -] - -const abuseListForAdminsValidator = [ - query('id') - .optional() - .custom(isIdValid), - query('filter') - .optional() - .custom(isAbuseFilterValid), - query('predefinedReason') - .optional() - .custom(isAbusePredefinedReasonValid), - query('search') - .optional() - .custom(exists), - query('state') - .optional() - .custom(isAbuseStateValid), - query('videoIs') - .optional() - .custom(isAbuseVideoIsValid), - query('searchReporter') - .optional() - .custom(exists), - query('searchReportee') - .optional() - .custom(exists), - query('searchVideo') - .optional() - .custom(exists), - query('searchVideoChannel') - .optional() - .custom(exists), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const abuseListForUserValidator = [ - query('id') - .optional() - .custom(isIdValid), - - query('search') - .optional() - .custom(exists), - - query('state') - .optional() - .custom(isAbuseStateValid), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const getAbuseValidator = [ - param('id') - .custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesAbuseExist(req.params.id, res)) return - - const user = res.locals.oauth.token.user - const abuse = res.locals.abuse - - if (user.hasRight(UserRight.MANAGE_ABUSES) !== true && abuse.reporterAccountId !== user.Account.id) { - const message = `User ${user.username} does not have right to get abuse ${abuse.id}` - logger.warn(message) - - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message - }) - } - - return next() - } -] - -const checkAbuseValidForMessagesValidator = [ - (req: express.Request, res: express.Response, next: express.NextFunction) => { - const abuse = res.locals.abuse - if (abuse.ReporterAccount.isOwned() === false) { - return res.fail({ message: 'This abuse was created by a user of your instance.' }) - } - - return next() - } -] - -const addAbuseMessageValidator = [ - body('message') - .custom(isAbuseMessageValid), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const deleteAbuseMessageValidator = [ - param('messageId') - .custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const user = res.locals.oauth.token.user - const abuse = res.locals.abuse - - const messageId = forceNumber(req.params.messageId) - const abuseMessage = await AbuseMessageModel.loadByIdAndAbuseId(messageId, abuse.id) - - if (!abuseMessage) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Abuse message not found' - }) - } - - if (user.hasRight(UserRight.MANAGE_ABUSES) !== true && abuseMessage.accountId !== user.Account.id) { - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Cannot delete this abuse message' - }) - } - - res.locals.abuseMessage = abuseMessage - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - abuseListForAdminsValidator, - abuseReportValidator, - abuseGetValidator, - addAbuseMessageValidator, - checkAbuseValidForMessagesValidator, - abuseUpdateValidator, - deleteAbuseMessageValidator, - abuseListForUserValidator, - getAbuseValidator -} diff --git a/server/middlewares/validators/account.ts b/server/middlewares/validators/account.ts deleted file mode 100644 index 551f67d61..000000000 --- a/server/middlewares/validators/account.ts +++ /dev/null @@ -1,35 +0,0 @@ -import express from 'express' -import { param } from 'express-validator' -import { isAccountNameValid } from '../../helpers/custom-validators/accounts' -import { areValidationErrors, doesAccountNameWithHostExist, doesLocalAccountNameExist } from './shared' - -const localAccountValidator = [ - param('name') - .custom(isAccountNameValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesLocalAccountNameExist(req.params.name, res)) return - - return next() - } -] - -const accountNameWithHostGetValidator = [ - param('accountName') - .exists(), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesAccountNameWithHostExist(req.params.accountName, res)) return - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - localAccountValidator, - accountNameWithHostGetValidator -} diff --git a/server/middlewares/validators/activitypub/activity.ts b/server/middlewares/validators/activitypub/activity.ts deleted file mode 100644 index e296b8be7..000000000 --- a/server/middlewares/validators/activitypub/activity.ts +++ /dev/null @@ -1,29 +0,0 @@ -import express from 'express' -import { getServerActor } from '@server/models/application/application' -import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' -import { isRootActivityValid } from '../../../helpers/custom-validators/activitypub/activity' -import { logger } from '../../../helpers/logger' - -async function activityPubValidator (req: express.Request, res: express.Response, next: express.NextFunction) { - logger.debug('Checking activity pub parameters') - - if (!isRootActivityValid(req.body)) { - logger.warn('Incorrect activity parameters.', { activity: req.body }) - return res.fail({ message: 'Incorrect activity' }) - } - - const serverActor = await getServerActor() - const remoteActor = res.locals.signature.actor - if (serverActor.id === remoteActor.id || remoteActor.serverId === null) { - logger.error('Receiving request in INBOX by ourselves!', req.body) - return res.status(HttpStatusCode.CONFLICT_409).end() - } - - return next() -} - -// --------------------------------------------------------------------------- - -export { - activityPubValidator -} diff --git a/server/middlewares/validators/activitypub/index.ts b/server/middlewares/validators/activitypub/index.ts deleted file mode 100644 index 159338d26..000000000 --- a/server/middlewares/validators/activitypub/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './activity' -export * from './signature' -export * from './pagination' diff --git a/server/middlewares/validators/activitypub/pagination.ts b/server/middlewares/validators/activitypub/pagination.ts deleted file mode 100644 index 1259e4fef..000000000 --- a/server/middlewares/validators/activitypub/pagination.ts +++ /dev/null @@ -1,25 +0,0 @@ -import express from 'express' -import { query } from 'express-validator' -import { PAGINATION } from '@server/initializers/constants' -import { areValidationErrors } from '../shared' - -const apPaginationValidator = [ - query('page') - .optional() - .isInt({ min: 1 }), - query('size') - .optional() - .isInt({ min: 0, max: PAGINATION.OUTBOX.COUNT.MAX }).withMessage(`Should have a valid page size (max: ${PAGINATION.OUTBOX.COUNT.MAX})`), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - apPaginationValidator -} diff --git a/server/middlewares/validators/activitypub/signature.ts b/server/middlewares/validators/activitypub/signature.ts deleted file mode 100644 index 998d0c0c4..000000000 --- a/server/middlewares/validators/activitypub/signature.ts +++ /dev/null @@ -1,39 +0,0 @@ -import express from 'express' -import { body } from 'express-validator' -import { - isSignatureCreatorValid, - isSignatureTypeValid, - isSignatureValueValid -} from '../../../helpers/custom-validators/activitypub/signature' -import { isDateValid } from '../../../helpers/custom-validators/misc' -import { logger } from '../../../helpers/logger' -import { areValidationErrors } from '../shared' - -const signatureValidator = [ - body('signature.type') - .optional() - .custom(isSignatureTypeValid), - body('signature.created') - .optional() - .custom(isDateValid).withMessage('Should have a signature created date that conforms to ISO 8601'), - body('signature.creator') - .optional() - .custom(isSignatureCreatorValid), - body('signature.signatureValue') - .optional() - .custom(isSignatureValueValid), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - logger.debug('Checking Linked Data Signature parameter', { parameters: { signature: req.body.signature } }) - - if (areValidationErrors(req, res, { omitLog: true })) return - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - signatureValidator -} diff --git a/server/middlewares/validators/actor-image.ts b/server/middlewares/validators/actor-image.ts deleted file mode 100644 index 9dcf5e871..000000000 --- a/server/middlewares/validators/actor-image.ts +++ /dev/null @@ -1,27 +0,0 @@ -import express from 'express' -import { body } from 'express-validator' -import { isActorImageFile } from '@server/helpers/custom-validators/actor-images' -import { cleanUpReqFiles } from '../../helpers/express-utils' -import { CONSTRAINTS_FIELDS } from '../../initializers/constants' -import { areValidationErrors } from './shared' - -const updateActorImageValidatorFactory = (fieldname: string) => ([ - body(fieldname).custom((value, { req }) => isActorImageFile(req.files, fieldname)).withMessage( - 'This file is not supported or too large. Please, make sure it is of the following type : ' + - CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME.join(', ') - ), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return cleanUpReqFiles(req) - - return next() - } -]) - -const updateAvatarValidator = updateActorImageValidatorFactory('avatarfile') -const updateBannerValidator = updateActorImageValidatorFactory('bannerfile') - -export { - updateAvatarValidator, - updateBannerValidator -} diff --git a/server/middlewares/validators/blocklist.ts b/server/middlewares/validators/blocklist.ts deleted file mode 100644 index 8ec6cb01d..000000000 --- a/server/middlewares/validators/blocklist.ts +++ /dev/null @@ -1,179 +0,0 @@ -import express from 'express' -import { body, param, query } from 'express-validator' -import { areValidActorHandles } from '@server/helpers/custom-validators/activitypub/actor' -import { getServerActor } from '@server/models/application/application' -import { arrayify } from '@shared/core-utils' -import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' -import { isEachUniqueHostValid, isHostValid } from '../../helpers/custom-validators/servers' -import { WEBSERVER } from '../../initializers/constants' -import { AccountBlocklistModel } from '../../models/account/account-blocklist' -import { ServerModel } from '../../models/server/server' -import { ServerBlocklistModel } from '../../models/server/server-blocklist' -import { areValidationErrors, doesAccountNameWithHostExist } from './shared' - -const blockAccountValidator = [ - body('accountName') - .exists(), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesAccountNameWithHostExist(req.body.accountName, res)) return - - const user = res.locals.oauth.token.User - const accountToBlock = res.locals.account - - if (user.Account.id === accountToBlock.id) { - res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: 'You cannot block yourself.' - }) - return - } - - return next() - } -] - -const unblockAccountByAccountValidator = [ - param('accountName') - .exists(), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesAccountNameWithHostExist(req.params.accountName, res)) return - - const user = res.locals.oauth.token.User - const targetAccount = res.locals.account - if (!await doesUnblockAccountExist(user.Account.id, targetAccount.id, res)) return - - return next() - } -] - -const unblockAccountByServerValidator = [ - param('accountName') - .exists(), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesAccountNameWithHostExist(req.params.accountName, res)) return - - const serverActor = await getServerActor() - const targetAccount = res.locals.account - if (!await doesUnblockAccountExist(serverActor.Account.id, targetAccount.id, res)) return - - return next() - } -] - -const blockServerValidator = [ - body('host') - .custom(isHostValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const host: string = req.body.host - - if (host === WEBSERVER.HOST) { - return res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: 'You cannot block your own server.' - }) - } - - const server = await ServerModel.loadOrCreateByHost(host) - - res.locals.server = server - - return next() - } -] - -const unblockServerByAccountValidator = [ - param('host') - .custom(isHostValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const user = res.locals.oauth.token.User - if (!await doesUnblockServerExist(user.Account.id, req.params.host, res)) return - - return next() - } -] - -const unblockServerByServerValidator = [ - param('host') - .custom(isHostValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const serverActor = await getServerActor() - if (!await doesUnblockServerExist(serverActor.Account.id, req.params.host, res)) return - - return next() - } -] - -const blocklistStatusValidator = [ - query('hosts') - .optional() - .customSanitizer(arrayify) - .custom(isEachUniqueHostValid).withMessage('Should have a valid hosts array'), - - query('accounts') - .optional() - .customSanitizer(arrayify) - .custom(areValidActorHandles).withMessage('Should have a valid accounts array'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - blockServerValidator, - blockAccountValidator, - unblockAccountByAccountValidator, - unblockServerByAccountValidator, - unblockAccountByServerValidator, - unblockServerByServerValidator, - blocklistStatusValidator -} - -// --------------------------------------------------------------------------- - -async function doesUnblockAccountExist (accountId: number, targetAccountId: number, res: express.Response) { - const accountBlock = await AccountBlocklistModel.loadByAccountAndTarget(accountId, targetAccountId) - if (!accountBlock) { - res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Account block entry not found.' - }) - return false - } - - res.locals.accountBlock = accountBlock - return true -} - -async function doesUnblockServerExist (accountId: number, host: string, res: express.Response) { - const serverBlock = await ServerBlocklistModel.loadByAccountAndHost(accountId, host) - if (!serverBlock) { - res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Server block entry not found.' - }) - return false - } - - res.locals.serverBlock = serverBlock - return true -} diff --git a/server/middlewares/validators/bulk.ts b/server/middlewares/validators/bulk.ts deleted file mode 100644 index a1cea8032..000000000 --- a/server/middlewares/validators/bulk.ts +++ /dev/null @@ -1,38 +0,0 @@ -import express from 'express' -import { body } from 'express-validator' -import { isBulkRemoveCommentsOfScopeValid } from '@server/helpers/custom-validators/bulk' -import { HttpStatusCode, UserRight } from '@shared/models' -import { BulkRemoveCommentsOfBody } from '@shared/models/bulk/bulk-remove-comments-of-body.model' -import { areValidationErrors, doesAccountNameWithHostExist } from './shared' - -const bulkRemoveCommentsOfValidator = [ - body('accountName') - .exists(), - body('scope') - .custom(isBulkRemoveCommentsOfScopeValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesAccountNameWithHostExist(req.body.accountName, res)) return - - const user = res.locals.oauth.token.User - const body = req.body as BulkRemoveCommentsOfBody - - if (body.scope === 'instance' && user.hasRight(UserRight.REMOVE_ANY_VIDEO_COMMENT) !== true) { - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'User cannot remove any comments of this instance.' - }) - } - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - bulkRemoveCommentsOfValidator -} - -// --------------------------------------------------------------------------- diff --git a/server/middlewares/validators/config.ts b/server/middlewares/validators/config.ts deleted file mode 100644 index 7790025e4..000000000 --- a/server/middlewares/validators/config.ts +++ /dev/null @@ -1,194 +0,0 @@ -import express from 'express' -import { body } from 'express-validator' -import { isIntOrNull } from '@server/helpers/custom-validators/misc' -import { CONFIG, isEmailEnabled } from '@server/initializers/config' -import { HttpStatusCode } from '@shared/models/http/http-error-codes' -import { CustomConfig } from '../../../shared/models/server/custom-config.model' -import { isThemeNameValid } from '../../helpers/custom-validators/plugins' -import { isUserNSFWPolicyValid, isUserVideoQuotaDailyValid, isUserVideoQuotaValid } from '../../helpers/custom-validators/users' -import { isThemeRegistered } from '../../lib/plugins/theme-utils' -import { areValidationErrors } from './shared' - -const customConfigUpdateValidator = [ - body('instance.name').exists(), - body('instance.shortDescription').exists(), - body('instance.description').exists(), - body('instance.terms').exists(), - body('instance.defaultNSFWPolicy').custom(isUserNSFWPolicyValid), - body('instance.defaultClientRoute').exists(), - body('instance.customizations.css').exists(), - body('instance.customizations.javascript').exists(), - - body('services.twitter.username').exists(), - body('services.twitter.whitelisted').isBoolean(), - - body('cache.previews.size').isInt(), - body('cache.captions.size').isInt(), - body('cache.torrents.size').isInt(), - body('cache.storyboards.size').isInt(), - - body('signup.enabled').isBoolean(), - body('signup.limit').isInt(), - body('signup.requiresEmailVerification').isBoolean(), - body('signup.requiresApproval').isBoolean(), - body('signup.minimumAge').isInt(), - - body('admin.email').isEmail(), - body('contactForm.enabled').isBoolean(), - - body('user.history.videos.enabled').isBoolean(), - body('user.videoQuota').custom(isUserVideoQuotaValid), - body('user.videoQuotaDaily').custom(isUserVideoQuotaDailyValid), - - body('videoChannels.maxPerUser').isInt(), - - body('transcoding.enabled').isBoolean(), - body('transcoding.allowAdditionalExtensions').isBoolean(), - body('transcoding.threads').isInt(), - body('transcoding.concurrency').isInt({ min: 1 }), - body('transcoding.resolutions.0p').isBoolean(), - body('transcoding.resolutions.144p').isBoolean(), - body('transcoding.resolutions.240p').isBoolean(), - body('transcoding.resolutions.360p').isBoolean(), - body('transcoding.resolutions.480p').isBoolean(), - body('transcoding.resolutions.720p').isBoolean(), - body('transcoding.resolutions.1080p').isBoolean(), - body('transcoding.resolutions.1440p').isBoolean(), - body('transcoding.resolutions.2160p').isBoolean(), - body('transcoding.remoteRunners.enabled').isBoolean(), - - body('transcoding.alwaysTranscodeOriginalResolution').isBoolean(), - - body('transcoding.webVideos.enabled').isBoolean(), - body('transcoding.hls.enabled').isBoolean(), - - body('videoStudio.enabled').isBoolean(), - body('videoStudio.remoteRunners.enabled').isBoolean(), - - body('videoFile.update.enabled').isBoolean(), - - body('import.videos.concurrency').isInt({ min: 0 }), - body('import.videos.http.enabled').isBoolean(), - body('import.videos.torrent.enabled').isBoolean(), - - body('import.videoChannelSynchronization.enabled').isBoolean(), - - body('trending.videos.algorithms.default').exists(), - body('trending.videos.algorithms.enabled').exists(), - - body('followers.instance.enabled').isBoolean(), - body('followers.instance.manualApproval').isBoolean(), - - body('theme.default').custom(v => isThemeNameValid(v) && isThemeRegistered(v)), - - body('broadcastMessage.enabled').isBoolean(), - body('broadcastMessage.message').exists(), - body('broadcastMessage.level').exists(), - body('broadcastMessage.dismissable').isBoolean(), - - body('live.enabled').isBoolean(), - body('live.allowReplay').isBoolean(), - body('live.maxDuration').isInt(), - body('live.maxInstanceLives').custom(isIntOrNull), - body('live.maxUserLives').custom(isIntOrNull), - body('live.transcoding.enabled').isBoolean(), - body('live.transcoding.threads').isInt(), - body('live.transcoding.resolutions.144p').isBoolean(), - body('live.transcoding.resolutions.240p').isBoolean(), - body('live.transcoding.resolutions.360p').isBoolean(), - body('live.transcoding.resolutions.480p').isBoolean(), - body('live.transcoding.resolutions.720p').isBoolean(), - body('live.transcoding.resolutions.1080p').isBoolean(), - body('live.transcoding.resolutions.1440p').isBoolean(), - body('live.transcoding.resolutions.2160p').isBoolean(), - body('live.transcoding.alwaysTranscodeOriginalResolution').isBoolean(), - body('live.transcoding.remoteRunners.enabled').isBoolean(), - - body('search.remoteUri.users').isBoolean(), - body('search.remoteUri.anonymous').isBoolean(), - body('search.searchIndex.enabled').isBoolean(), - body('search.searchIndex.url').exists(), - body('search.searchIndex.disableLocalSearch').isBoolean(), - body('search.searchIndex.isDefaultSearch').isBoolean(), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!checkInvalidConfigIfEmailDisabled(req.body, res)) return - if (!checkInvalidTranscodingConfig(req.body, res)) return - if (!checkInvalidSynchronizationConfig(req.body, res)) return - if (!checkInvalidLiveConfig(req.body, res)) return - if (!checkInvalidVideoStudioConfig(req.body, res)) return - - return next() - } -] - -function ensureConfigIsEditable (req: express.Request, res: express.Response, next: express.NextFunction) { - if (!CONFIG.WEBADMIN.CONFIGURATION.EDITION.ALLOWED) { - return res.fail({ - status: HttpStatusCode.METHOD_NOT_ALLOWED_405, - message: 'Server configuration is static and cannot be edited' - }) - } - - return next() -} - -// --------------------------------------------------------------------------- - -export { - customConfigUpdateValidator, - ensureConfigIsEditable -} - -function checkInvalidConfigIfEmailDisabled (customConfig: CustomConfig, res: express.Response) { - if (isEmailEnabled()) return true - - if (customConfig.signup.requiresEmailVerification === true) { - res.fail({ message: 'SMTP is not configured but you require signup email verification.' }) - return false - } - - return true -} - -function checkInvalidTranscodingConfig (customConfig: CustomConfig, res: express.Response) { - if (customConfig.transcoding.enabled === false) return true - - if (customConfig.transcoding.webVideos.enabled === false && customConfig.transcoding.hls.enabled === false) { - res.fail({ message: 'You need to enable at least web_videos transcoding or hls transcoding' }) - return false - } - - return true -} - -function checkInvalidSynchronizationConfig (customConfig: CustomConfig, res: express.Response) { - if (customConfig.import.videoChannelSynchronization.enabled && !customConfig.import.videos.http.enabled) { - res.fail({ message: 'You need to enable HTTP video import in order to enable channel synchronization' }) - return false - } - return true -} - -function checkInvalidLiveConfig (customConfig: CustomConfig, res: express.Response) { - if (customConfig.live.enabled === false) return true - - if (customConfig.live.allowReplay === true && customConfig.transcoding.enabled === false) { - res.fail({ message: 'You cannot allow live replay if transcoding is not enabled' }) - return false - } - - return true -} - -function checkInvalidVideoStudioConfig (customConfig: CustomConfig, res: express.Response) { - if (customConfig.videoStudio.enabled === false) return true - - if (customConfig.videoStudio.enabled === true && customConfig.transcoding.enabled === false) { - res.fail({ message: 'You cannot enable video studio if transcoding is not enabled' }) - return false - } - - return true -} diff --git a/server/middlewares/validators/feeds.ts b/server/middlewares/validators/feeds.ts deleted file mode 100644 index 72804a259..000000000 --- a/server/middlewares/validators/feeds.ts +++ /dev/null @@ -1,178 +0,0 @@ -import express from 'express' -import { param, query } from 'express-validator' -import { HttpStatusCode } from '@shared/models' -import { isValidRSSFeed } from '../../helpers/custom-validators/feeds' -import { exists, isIdOrUUIDValid, isIdValid, toCompleteUUID } from '../../helpers/custom-validators/misc' -import { buildPodcastGroupsCache } from '../cache' -import { - areValidationErrors, - checkCanSeeVideo, - doesAccountIdExist, - doesAccountNameWithHostExist, - doesUserFeedTokenCorrespond, - doesVideoChannelIdExist, - doesVideoChannelNameWithHostExist, - doesVideoExist -} from './shared' - -const feedsFormatValidator = [ - param('format') - .optional() - .custom(isValidRSSFeed).withMessage('Should have a valid format (rss, atom, json)'), - query('format') - .optional() - .custom(isValidRSSFeed).withMessage('Should have a valid format (rss, atom, json)'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -function setFeedFormatContentType (req: express.Request, res: express.Response, next: express.NextFunction) { - const format = req.query.format || req.params.format || 'rss' - - let acceptableContentTypes: string[] - if (format === 'atom' || format === 'atom1') { - acceptableContentTypes = [ 'application/atom+xml', 'application/xml', 'text/xml' ] - } else if (format === 'json' || format === 'json1') { - acceptableContentTypes = [ 'application/json' ] - } else if (format === 'rss' || format === 'rss2') { - acceptableContentTypes = [ 'application/rss+xml', 'application/xml', 'text/xml' ] - } else { - acceptableContentTypes = [ 'application/xml', 'text/xml' ] - } - - return feedContentTypeResponse(req, res, next, acceptableContentTypes) -} - -function setFeedPodcastContentType (req: express.Request, res: express.Response, next: express.NextFunction) { - const acceptableContentTypes = [ 'application/rss+xml', 'application/xml', 'text/xml' ] - - return feedContentTypeResponse(req, res, next, acceptableContentTypes) -} - -function feedContentTypeResponse ( - req: express.Request, - res: express.Response, - next: express.NextFunction, - acceptableContentTypes: string[] -) { - if (req.accepts(acceptableContentTypes)) { - res.set('Content-Type', req.accepts(acceptableContentTypes) as string) - } else { - return res.fail({ - status: HttpStatusCode.NOT_ACCEPTABLE_406, - message: `You should accept at least one of the following content-types: ${acceptableContentTypes.join(', ')}` - }) - } - - return next() -} - -// --------------------------------------------------------------------------- - -const feedsAccountOrChannelFiltersValidator = [ - query('accountId') - .optional() - .custom(isIdValid), - - query('accountName') - .optional(), - - query('videoChannelId') - .optional() - .custom(isIdValid), - - query('videoChannelName') - .optional(), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - if (req.query.accountId && !await doesAccountIdExist(req.query.accountId, res)) return - if (req.query.videoChannelId && !await doesVideoChannelIdExist(req.query.videoChannelId, res)) return - if (req.query.accountName && !await doesAccountNameWithHostExist(req.query.accountName, res)) return - if (req.query.videoChannelName && !await doesVideoChannelNameWithHostExist(req.query.videoChannelName, res)) return - - return next() - } -] - -// --------------------------------------------------------------------------- - -const videoFeedsPodcastValidator = [ - query('videoChannelId') - .custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoChannelIdExist(req.query.videoChannelId, res)) return - - return next() - } -] - -const videoFeedsPodcastSetCacheKey = [ - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (req.query.videoChannelId) { - res.locals.apicacheGroups = [ buildPodcastGroupsCache({ channelId: req.query.videoChannelId }) ] - } - - return next() - } -] -// --------------------------------------------------------------------------- - -const videoSubscriptionFeedsValidator = [ - query('accountId') - .custom(isIdValid), - - query('token') - .custom(exists), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - if (!await doesAccountIdExist(req.query.accountId, res)) return - if (!await doesUserFeedTokenCorrespond(res.locals.account.userId, req.query.token, res)) return - - return next() - } -] - -const videoCommentsFeedsValidator = [ - query('videoId') - .optional() - .customSanitizer(toCompleteUUID) - .custom(isIdOrUUIDValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - if (req.query.videoId && (req.query.videoChannelId || req.query.videoChannelName)) { - return res.fail({ message: 'videoId cannot be mixed with a channel filter' }) - } - - if (req.query.videoId) { - if (!await doesVideoExist(req.query.videoId, res)) return - if (!await checkCanSeeVideo({ req, res, paramId: req.query.videoId, video: res.locals.videoAll })) return - } - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - feedsFormatValidator, - setFeedFormatContentType, - setFeedPodcastContentType, - feedsAccountOrChannelFiltersValidator, - videoFeedsPodcastValidator, - videoSubscriptionFeedsValidator, - videoFeedsPodcastSetCacheKey, - videoCommentsFeedsValidator -} diff --git a/server/middlewares/validators/follows.ts b/server/middlewares/validators/follows.ts deleted file mode 100644 index be98a4c04..000000000 --- a/server/middlewares/validators/follows.ts +++ /dev/null @@ -1,158 +0,0 @@ -import express from 'express' -import { body, param, query } from 'express-validator' -import { isProdInstance } from '@server/helpers/core-utils' -import { isEachUniqueHandleValid, isFollowStateValid, isRemoteHandleValid } from '@server/helpers/custom-validators/follows' -import { loadActorUrlOrGetFromWebfinger } from '@server/lib/activitypub/actors' -import { getRemoteNameAndHost } from '@server/lib/activitypub/follow' -import { getServerActor } from '@server/models/application/application' -import { MActorFollowActorsDefault } from '@server/types/models' -import { ServerFollowCreate } from '@shared/models' -import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' -import { isActorTypeValid, isValidActorHandle } from '../../helpers/custom-validators/activitypub/actor' -import { isEachUniqueHostValid, isHostValid } from '../../helpers/custom-validators/servers' -import { logger } from '../../helpers/logger' -import { WEBSERVER } from '../../initializers/constants' -import { ActorModel } from '../../models/actor/actor' -import { ActorFollowModel } from '../../models/actor/actor-follow' -import { areValidationErrors } from './shared' - -const listFollowsValidator = [ - query('state') - .optional() - .custom(isFollowStateValid), - query('actorType') - .optional() - .custom(isActorTypeValid), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const followValidator = [ - body('hosts') - .toArray() - .custom(isEachUniqueHostValid).withMessage('Should have an array of unique hosts'), - - body('handles') - .toArray() - .custom(isEachUniqueHandleValid).withMessage('Should have an array of handles'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - // Force https if the administrator wants to follow remote actors - if (isProdInstance() && WEBSERVER.SCHEME === 'http') { - return res - .status(HttpStatusCode.INTERNAL_SERVER_ERROR_500) - .json({ - error: 'Cannot follow on a non HTTPS web server.' - }) - } - - if (areValidationErrors(req, res)) return - - const body: ServerFollowCreate = req.body - if (body.hosts.length === 0 && body.handles.length === 0) { - - return res - .status(HttpStatusCode.BAD_REQUEST_400) - .json({ - error: 'You must provide at least one handle or one host.' - }) - } - - return next() - } -] - -const removeFollowingValidator = [ - param('hostOrHandle') - .custom(value => isHostValid(value) || isRemoteHandleValid(value)), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const serverActor = await getServerActor() - - const { name, host } = getRemoteNameAndHost(req.params.hostOrHandle) - const follow = await ActorFollowModel.loadByActorAndTargetNameAndHostForAPI({ - actorId: serverActor.id, - targetName: name, - targetHost: host - }) - - if (!follow) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: `Follow ${req.params.hostOrHandle} not found.` - }) - } - - res.locals.follow = follow - return next() - } -] - -const getFollowerValidator = [ - param('nameWithHost') - .custom(isValidActorHandle), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - let follow: MActorFollowActorsDefault - try { - const actorUrl = await loadActorUrlOrGetFromWebfinger(req.params.nameWithHost) - const actor = await ActorModel.loadByUrl(actorUrl) - - const serverActor = await getServerActor() - follow = await ActorFollowModel.loadByActorAndTarget(actor.id, serverActor.id) - } catch (err) { - logger.warn('Cannot get actor from handle.', { handle: req.params.nameWithHost, err }) - } - - if (!follow) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: `Follower ${req.params.nameWithHost} not found.` - }) - } - - res.locals.follow = follow - return next() - } -] - -const acceptFollowerValidator = [ - (req: express.Request, res: express.Response, next: express.NextFunction) => { - const follow = res.locals.follow - if (follow.state !== 'pending' && follow.state !== 'rejected') { - return res.fail({ message: 'Follow is not in pending/rejected state.' }) - } - - return next() - } -] - -const rejectFollowerValidator = [ - (req: express.Request, res: express.Response, next: express.NextFunction) => { - const follow = res.locals.follow - if (follow.state !== 'pending' && follow.state !== 'accepted') { - return res.fail({ message: 'Follow is not in pending/accepted state.' }) - } - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - followValidator, - removeFollowingValidator, - getFollowerValidator, - acceptFollowerValidator, - rejectFollowerValidator, - listFollowsValidator -} diff --git a/server/middlewares/validators/index.ts b/server/middlewares/validators/index.ts deleted file mode 100644 index 1d0964667..000000000 --- a/server/middlewares/validators/index.ts +++ /dev/null @@ -1,31 +0,0 @@ -export * from './abuse' -export * from './account' -export * from './activitypub' -export * from './actor-image' -export * from './blocklist' -export * from './bulk' -export * from './config' -export * from './express' -export * from './feeds' -export * from './follows' -export * from './jobs' -export * from './logs' -export * from './metrics' -export * from './object-storage-proxy' -export * from './oembed' -export * from './pagination' -export * from './plugins' -export * from './redundancy' -export * from './search' -export * from './server' -export * from './sort' -export * from './static' -export * from './themes' -export * from './user-email-verification' -export * from './user-history' -export * from './user-notifications' -export * from './user-registrations' -export * from './user-subscriptions' -export * from './users' -export * from './videos' -export * from './webfinger' diff --git a/server/middlewares/validators/jobs.ts b/server/middlewares/validators/jobs.ts deleted file mode 100644 index e5008adc3..000000000 --- a/server/middlewares/validators/jobs.ts +++ /dev/null @@ -1,29 +0,0 @@ -import express from 'express' -import { param, query } from 'express-validator' -import { isValidJobState, isValidJobType } from '../../helpers/custom-validators/jobs' -import { loggerTagsFactory } from '../../helpers/logger' -import { areValidationErrors } from './shared' - -const lTags = loggerTagsFactory('validators', 'jobs') - -const listJobsValidator = [ - param('state') - .optional() - .custom(isValidJobState), - - query('jobType') - .optional() - .custom(isValidJobType), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res, lTags())) return - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - listJobsValidator -} diff --git a/server/middlewares/validators/logs.ts b/server/middlewares/validators/logs.ts deleted file mode 100644 index 2d828bb42..000000000 --- a/server/middlewares/validators/logs.ts +++ /dev/null @@ -1,93 +0,0 @@ -import express from 'express' -import { body, query } from 'express-validator' -import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc' -import { isStringArray } from '@server/helpers/custom-validators/search' -import { CONFIG } from '@server/initializers/config' -import { arrayify } from '@shared/core-utils' -import { HttpStatusCode } from '@shared/models' -import { - isValidClientLogLevel, - isValidClientLogMessage, - isValidClientLogMeta, - isValidClientLogStackTrace, - isValidClientLogUserAgent, - isValidLogLevel -} from '../../helpers/custom-validators/logs' -import { isDateValid } from '../../helpers/custom-validators/misc' -import { areValidationErrors } from './shared' - -const createClientLogValidator = [ - body('message') - .custom(isValidClientLogMessage), - - body('url') - .custom(isUrlValid), - - body('level') - .custom(isValidClientLogLevel), - - body('stackTrace') - .optional() - .custom(isValidClientLogStackTrace), - - body('meta') - .optional() - .custom(isValidClientLogMeta), - - body('userAgent') - .optional() - .custom(isValidClientLogUserAgent), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (CONFIG.LOG.ACCEPT_CLIENT_LOG !== true) { - return res.sendStatus(HttpStatusCode.FORBIDDEN_403) - } - - if (areValidationErrors(req, res)) return - - return next() - } -] - -const getLogsValidator = [ - query('startDate') - .custom(isDateValid).withMessage('Should have a start date that conforms to ISO 8601'), - query('level') - .optional() - .custom(isValidLogLevel), - query('tagsOneOf') - .optional() - .customSanitizer(arrayify) - .custom(isStringArray).withMessage('Should have a valid one of tags array'), - query('endDate') - .optional() - .custom(isDateValid).withMessage('Should have an end date that conforms to ISO 8601'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const getAuditLogsValidator = [ - query('startDate') - .custom(isDateValid).withMessage('Should have a start date that conforms to ISO 8601'), - query('endDate') - .optional() - .custom(isDateValid).withMessage('Should have a end date that conforms to ISO 8601'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - getLogsValidator, - getAuditLogsValidator, - createClientLogValidator -} diff --git a/server/middlewares/validators/metrics.ts b/server/middlewares/validators/metrics.ts deleted file mode 100644 index 986b30a19..000000000 --- a/server/middlewares/validators/metrics.ts +++ /dev/null @@ -1,60 +0,0 @@ -import express from 'express' -import { body } from 'express-validator' -import { isValidPlayerMode } from '@server/helpers/custom-validators/metrics' -import { isIdOrUUIDValid, toCompleteUUID } from '@server/helpers/custom-validators/misc' -import { CONFIG } from '@server/initializers/config' -import { HttpStatusCode, PlaybackMetricCreate } from '@shared/models' -import { areValidationErrors, doesVideoExist } from './shared' - -const addPlaybackMetricValidator = [ - body('resolution') - .isInt({ min: 0 }), - body('fps') - .optional() - .isInt({ min: 0 }), - - body('p2pPeers') - .optional() - .isInt({ min: 0 }), - - body('p2pEnabled') - .isBoolean(), - - body('playerMode') - .custom(isValidPlayerMode), - - body('resolutionChanges') - .isInt({ min: 0 }), - - body('errors') - .isInt({ min: 0 }), - - body('downloadedBytesP2P') - .isInt({ min: 0 }), - body('downloadedBytesHTTP') - .isInt({ min: 0 }), - - body('uploadedBytesP2P') - .isInt({ min: 0 }), - - body('videoId') - .customSanitizer(toCompleteUUID) - .custom(isIdOrUUIDValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (!CONFIG.OPEN_TELEMETRY.METRICS.ENABLED) return res.sendStatus(HttpStatusCode.NO_CONTENT_204) - - const body: PlaybackMetricCreate = req.body - - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(body.videoId, res, 'only-immutable-attributes')) return - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - addPlaybackMetricValidator -} diff --git a/server/middlewares/validators/object-storage-proxy.ts b/server/middlewares/validators/object-storage-proxy.ts deleted file mode 100644 index bbd77f262..000000000 --- a/server/middlewares/validators/object-storage-proxy.ts +++ /dev/null @@ -1,20 +0,0 @@ -import express from 'express' -import { CONFIG } from '@server/initializers/config' -import { HttpStatusCode } from '@shared/models' - -const ensurePrivateObjectStorageProxyIsEnabled = [ - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES !== true) { - return res.fail({ - message: 'Private object storage proxy is not enabled', - status: HttpStatusCode.BAD_REQUEST_400 - }) - } - - return next() - } -] - -export { - ensurePrivateObjectStorageProxyIsEnabled -} diff --git a/server/middlewares/validators/oembed.ts b/server/middlewares/validators/oembed.ts deleted file mode 100644 index ef9a227a0..000000000 --- a/server/middlewares/validators/oembed.ts +++ /dev/null @@ -1,158 +0,0 @@ -import express from 'express' -import { query } from 'express-validator' -import { join } from 'path' -import { loadVideo } from '@server/lib/model-loaders' -import { VideoPlaylistModel } from '@server/models/video/video-playlist' -import { VideoPlaylistPrivacy, VideoPrivacy } from '@shared/models' -import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' -import { isTestOrDevInstance } from '../../helpers/core-utils' -import { isIdOrUUIDValid, isUUIDValid, toCompleteUUID } from '../../helpers/custom-validators/misc' -import { WEBSERVER } from '../../initializers/constants' -import { areValidationErrors } from './shared' - -const playlistPaths = [ - join('videos', 'watch', 'playlist'), - join('w', 'p') -] - -const videoPaths = [ - join('videos', 'watch'), - 'w' -] - -function buildUrls (paths: string[]) { - return paths.map(p => WEBSERVER.SCHEME + '://' + join(WEBSERVER.HOST, p) + '/') -} - -const startPlaylistURLs = buildUrls(playlistPaths) -const startVideoURLs = buildUrls(videoPaths) - -const isURLOptions = { - require_host: true, - require_tld: true -} - -// We validate 'localhost', so we don't have the top level domain -if (isTestOrDevInstance()) { - isURLOptions.require_tld = false -} - -const oembedValidator = [ - query('url') - .isURL(isURLOptions), - query('maxwidth') - .optional() - .isInt(), - query('maxheight') - .optional() - .isInt(), - query('format') - .optional() - .isIn([ 'xml', 'json' ]), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - if (req.query.format !== undefined && req.query.format !== 'json') { - return res.fail({ - status: HttpStatusCode.NOT_IMPLEMENTED_501, - message: 'Requested format is not implemented on server.', - data: { - format: req.query.format - } - }) - } - - const url = req.query.url as string - - let urlPath: string - - try { - urlPath = new URL(url).pathname - } catch (err) { - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: err.message, - data: { - url - } - }) - } - - const isPlaylist = startPlaylistURLs.some(u => url.startsWith(u)) - const isVideo = isPlaylist ? false : startVideoURLs.some(u => url.startsWith(u)) - - const startIsOk = isVideo || isPlaylist - - const parts = urlPath.split('/') - - if (startIsOk === false || parts.length === 0) { - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Invalid url.', - data: { - url - } - }) - } - - const elementId = toCompleteUUID(parts.pop()) - if (isIdOrUUIDValid(elementId) === false) { - return res.fail({ message: 'Invalid video or playlist id.' }) - } - - if (isVideo) { - const video = await loadVideo(elementId, 'all') - - if (!video) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video not found' - }) - } - - if ( - video.privacy === VideoPrivacy.PUBLIC || - (video.privacy === VideoPrivacy.UNLISTED && isUUIDValid(elementId) === true) - ) { - res.locals.videoAll = video - return next() - } - - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Video is not publicly available' - }) - } - - // Is playlist - - const videoPlaylist = await VideoPlaylistModel.loadWithAccountAndChannelSummary(elementId, undefined) - if (!videoPlaylist) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video playlist not found' - }) - } - - if ( - videoPlaylist.privacy === VideoPlaylistPrivacy.PUBLIC || - (videoPlaylist.privacy === VideoPlaylistPrivacy.UNLISTED && isUUIDValid(elementId)) - ) { - res.locals.videoPlaylistSummary = videoPlaylist - return next() - } - - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Playlist is not public' - }) - } - -] - -// --------------------------------------------------------------------------- - -export { - oembedValidator -} diff --git a/server/middlewares/validators/pagination.ts b/server/middlewares/validators/pagination.ts deleted file mode 100644 index 79ddbbf18..000000000 --- a/server/middlewares/validators/pagination.ts +++ /dev/null @@ -1,30 +0,0 @@ -import express from 'express' -import { query } from 'express-validator' -import { PAGINATION } from '@server/initializers/constants' -import { areValidationErrors } from './shared' - -const paginationValidator = paginationValidatorBuilder() - -function paginationValidatorBuilder (tags: string[] = []) { - return [ - query('start') - .optional() - .isInt({ min: 0 }), - query('count') - .optional() - .isInt({ min: 0, max: PAGINATION.GLOBAL.COUNT.MAX }).withMessage(`Should have a number count (max: ${PAGINATION.GLOBAL.COUNT.MAX})`), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res, { tags })) return - - return next() - } - ] -} - -// --------------------------------------------------------------------------- - -export { - paginationValidator, - paginationValidatorBuilder -} diff --git a/server/middlewares/validators/plugins.ts b/server/middlewares/validators/plugins.ts deleted file mode 100644 index 64bef2648..000000000 --- a/server/middlewares/validators/plugins.ts +++ /dev/null @@ -1,218 +0,0 @@ -import express from 'express' -import { body, param, query, ValidationChain } from 'express-validator' -import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' -import { PluginType } from '../../../shared/models/plugins/plugin.type' -import { InstallOrUpdatePlugin } from '../../../shared/models/plugins/server/api/install-plugin.model' -import { exists, isBooleanValid, isSafePath, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc' -import { - isNpmPluginNameValid, - isPluginNameValid, - isPluginStableOrUnstableVersionValid, - isPluginTypeValid -} from '../../helpers/custom-validators/plugins' -import { CONFIG } from '../../initializers/config' -import { PluginManager } from '../../lib/plugins/plugin-manager' -import { PluginModel } from '../../models/server/plugin' -import { areValidationErrors } from './shared' - -const getPluginValidator = (pluginType: PluginType, withVersion = true) => { - const validators: (ValidationChain | express.Handler)[] = [ - param('pluginName') - .custom(isPluginNameValid) - ] - - if (withVersion) { - validators.push( - param('pluginVersion') - .custom(isPluginStableOrUnstableVersionValid) - ) - } - - return validators.concat([ - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const npmName = PluginModel.buildNpmName(req.params.pluginName, pluginType) - const plugin = PluginManager.Instance.getRegisteredPluginOrTheme(npmName) - - if (!plugin) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'No plugin found named ' + npmName - }) - } - if (withVersion && plugin.version !== req.params.pluginVersion) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'No plugin found named ' + npmName + ' with version ' + req.params.pluginVersion - }) - } - - res.locals.registeredPlugin = plugin - - return next() - } - ]) -} - -const getExternalAuthValidator = [ - param('authName') - .custom(exists), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const plugin = res.locals.registeredPlugin - if (!plugin.registerHelpers) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'No registered helpers were found for this plugin' - }) - } - - const externalAuth = plugin.registerHelpers.getExternalAuths().find(a => a.authName === req.params.authName) - if (!externalAuth) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'No external auths were found for this plugin' - }) - } - - res.locals.externalAuth = externalAuth - - return next() - } -] - -const pluginStaticDirectoryValidator = [ - param('staticEndpoint') - .custom(isSafePath), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const listPluginsValidator = [ - query('pluginType') - .optional() - .customSanitizer(toIntOrNull) - .custom(isPluginTypeValid), - query('uninstalled') - .optional() - .customSanitizer(toBooleanOrNull) - .custom(isBooleanValid), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const installOrUpdatePluginValidator = [ - body('npmName') - .optional() - .custom(isNpmPluginNameValid), - body('pluginVersion') - .optional() - .custom(isPluginStableOrUnstableVersionValid), - body('path') - .optional() - .custom(isSafePath), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const body: InstallOrUpdatePlugin = req.body - if (!body.path && !body.npmName) { - return res.fail({ message: 'Should have either a npmName or a path' }) - } - if (body.pluginVersion && !body.npmName) { - return res.fail({ message: 'Should have a npmName when specifying a pluginVersion' }) - } - - return next() - } -] - -const uninstallPluginValidator = [ - body('npmName') - .custom(isNpmPluginNameValid), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const existingPluginValidator = [ - param('npmName') - .custom(isNpmPluginNameValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const plugin = await PluginModel.loadByNpmName(req.params.npmName) - if (!plugin) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Plugin not found' - }) - } - - res.locals.plugin = plugin - return next() - } -] - -const updatePluginSettingsValidator = [ - body('settings') - .exists(), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const listAvailablePluginsValidator = [ - query('search') - .optional() - .exists(), - query('pluginType') - .optional() - .customSanitizer(toIntOrNull) - .custom(isPluginTypeValid), - query('currentPeerTubeEngine') - .optional() - .custom(isPluginStableOrUnstableVersionValid), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - if (CONFIG.PLUGINS.INDEX.ENABLED === false) { - return res.fail({ message: 'Plugin index is not enabled' }) - } - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - pluginStaticDirectoryValidator, - getPluginValidator, - updatePluginSettingsValidator, - uninstallPluginValidator, - listAvailablePluginsValidator, - existingPluginValidator, - installOrUpdatePluginValidator, - listPluginsValidator, - getExternalAuthValidator -} diff --git a/server/middlewares/validators/redundancy.ts b/server/middlewares/validators/redundancy.ts deleted file mode 100644 index c80f9b728..000000000 --- a/server/middlewares/validators/redundancy.ts +++ /dev/null @@ -1,198 +0,0 @@ -import express from 'express' -import { body, param, query } from 'express-validator' -import { isVideoRedundancyTarget } from '@server/helpers/custom-validators/video-redundancies' -import { forceNumber } from '@shared/core-utils' -import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' -import { - exists, - isBooleanValid, - isIdOrUUIDValid, - isIdValid, - toBooleanOrNull, - toCompleteUUID, - toIntOrNull -} from '../../helpers/custom-validators/misc' -import { isHostValid } from '../../helpers/custom-validators/servers' -import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy' -import { ServerModel } from '../../models/server/server' -import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from './shared' - -const videoFileRedundancyGetValidator = [ - isValidVideoIdParam('videoId'), - - param('resolution') - .customSanitizer(toIntOrNull) - .custom(exists), - param('fps') - .optional() - .customSanitizer(toIntOrNull) - .custom(exists), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.videoId, res)) return - - const video = res.locals.videoAll - - const paramResolution = req.params.resolution as unknown as number // We casted to int above - const paramFPS = req.params.fps as unknown as number // We casted to int above - - const videoFile = video.VideoFiles.find(f => { - return f.resolution === paramResolution && (!req.params.fps || paramFPS) - }) - - if (!videoFile) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video file not found.' - }) - } - res.locals.videoFile = videoFile - - const videoRedundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id) - if (!videoRedundancy) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video redundancy not found.' - }) - } - res.locals.videoRedundancy = videoRedundancy - - return next() - } -] - -const videoPlaylistRedundancyGetValidator = [ - isValidVideoIdParam('videoId'), - - param('streamingPlaylistType') - .customSanitizer(toIntOrNull) - .custom(exists), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.videoId, res)) return - - const video = res.locals.videoAll - - const paramPlaylistType = req.params.streamingPlaylistType as unknown as number // We casted to int above - const videoStreamingPlaylist = video.VideoStreamingPlaylists.find(p => p.type === paramPlaylistType) - - if (!videoStreamingPlaylist) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video playlist not found.' - }) - } - res.locals.videoStreamingPlaylist = videoStreamingPlaylist - - const videoRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(videoStreamingPlaylist.id) - if (!videoRedundancy) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video redundancy not found.' - }) - } - res.locals.videoRedundancy = videoRedundancy - - return next() - } -] - -const updateServerRedundancyValidator = [ - param('host') - .custom(isHostValid), - - body('redundancyAllowed') - .customSanitizer(toBooleanOrNull) - .custom(isBooleanValid).withMessage('Should have a valid redundancyAllowed boolean'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const server = await ServerModel.loadByHost(req.params.host) - - if (!server) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: `Server ${req.params.host} not found.` - }) - } - - res.locals.server = server - return next() - } -] - -const listVideoRedundanciesValidator = [ - query('target') - .custom(isVideoRedundancyTarget), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const addVideoRedundancyValidator = [ - body('videoId') - .customSanitizer(toCompleteUUID) - .custom(isIdOrUUIDValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - if (!await doesVideoExist(req.body.videoId, res, 'only-video')) return - - if (res.locals.onlyVideo.remote === false) { - return res.fail({ message: 'Cannot create a redundancy on a local video' }) - } - - if (res.locals.onlyVideo.isLive) { - return res.fail({ message: 'Cannot create a redundancy of a live video' }) - } - - const alreadyExists = await VideoRedundancyModel.isLocalByVideoUUIDExists(res.locals.onlyVideo.uuid) - if (alreadyExists) { - return res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: 'This video is already duplicated by your instance.' - }) - } - - return next() - } -] - -const removeVideoRedundancyValidator = [ - param('redundancyId') - .custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const redundancy = await VideoRedundancyModel.loadByIdWithVideo(forceNumber(req.params.redundancyId)) - if (!redundancy) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video redundancy not found' - }) - } - - res.locals.videoRedundancy = redundancy - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - videoFileRedundancyGetValidator, - videoPlaylistRedundancyGetValidator, - updateServerRedundancyValidator, - listVideoRedundanciesValidator, - addVideoRedundancyValidator, - removeVideoRedundancyValidator -} diff --git a/server/middlewares/validators/runners/index.ts b/server/middlewares/validators/runners/index.ts deleted file mode 100644 index 9a9629a80..000000000 --- a/server/middlewares/validators/runners/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './jobs' -export * from './registration-token' -export * from './runners' diff --git a/server/middlewares/validators/runners/job-files.ts b/server/middlewares/validators/runners/job-files.ts deleted file mode 100644 index 57c27fcfe..000000000 --- a/server/middlewares/validators/runners/job-files.ts +++ /dev/null @@ -1,60 +0,0 @@ -import express from 'express' -import { param } from 'express-validator' -import { basename } from 'path' -import { isSafeFilename } from '@server/helpers/custom-validators/misc' -import { hasVideoStudioTaskFile, HttpStatusCode, RunnerJobStudioTranscodingPayload } from '@shared/models' -import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared' - -const tags = [ 'runner' ] - -export const runnerJobGetVideoTranscodingFileValidator = [ - isValidVideoIdParam('videoId'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - if (!await doesVideoExist(req.params.videoId, res, 'all')) return - - const runnerJob = res.locals.runnerJob - - if (runnerJob.privatePayload.videoUUID !== res.locals.videoAll.uuid) { - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Job is not associated to this video', - tags: [ ...tags, res.locals.videoAll.uuid ] - }) - } - - return next() - } -] - -export const runnerJobGetVideoStudioTaskFileValidator = [ - param('filename').custom(v => isSafeFilename(v)), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const filename = req.params.filename - - const payload = res.locals.runnerJob.payload as RunnerJobStudioTranscodingPayload - - const found = Array.isArray(payload?.tasks) && payload.tasks.some(t => { - if (hasVideoStudioTaskFile(t)) { - return basename(t.options.file) === filename - } - - return false - }) - - if (!found) { - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'File is not associated to this edition task', - tags: [ ...tags, res.locals.videoAll.uuid ] - }) - } - - return next() - } -] diff --git a/server/middlewares/validators/runners/jobs.ts b/server/middlewares/validators/runners/jobs.ts deleted file mode 100644 index 62f9340a5..000000000 --- a/server/middlewares/validators/runners/jobs.ts +++ /dev/null @@ -1,216 +0,0 @@ -import express from 'express' -import { body, param, query } from 'express-validator' -import { exists, isUUIDValid } from '@server/helpers/custom-validators/misc' -import { - isRunnerJobAbortReasonValid, - isRunnerJobArrayOfStateValid, - isRunnerJobErrorMessageValid, - isRunnerJobProgressValid, - isRunnerJobSuccessPayloadValid, - isRunnerJobTokenValid, - isRunnerJobUpdatePayloadValid -} from '@server/helpers/custom-validators/runners/jobs' -import { isRunnerTokenValid } from '@server/helpers/custom-validators/runners/runners' -import { cleanUpReqFiles } from '@server/helpers/express-utils' -import { LiveManager } from '@server/lib/live' -import { runnerJobCanBeCancelled } from '@server/lib/runners' -import { RunnerJobModel } from '@server/models/runner/runner-job' -import { arrayify } from '@shared/core-utils' -import { - HttpStatusCode, - RunnerJobLiveRTMPHLSTranscodingPrivatePayload, - RunnerJobState, - RunnerJobSuccessBody, - RunnerJobUpdateBody, - ServerErrorCode -} from '@shared/models' -import { areValidationErrors } from '../shared' - -const tags = [ 'runner' ] - -export const acceptRunnerJobValidator = [ - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (res.locals.runnerJob.state !== RunnerJobState.PENDING) { - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'This runner job is not in pending state', - tags - }) - } - - return next() - } -] - -export const abortRunnerJobValidator = [ - body('reason').custom(isRunnerJobAbortReasonValid), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res, { tags })) return - - return next() - } -] - -export const updateRunnerJobValidator = [ - body('progress').optional().custom(isRunnerJobProgressValid), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res, { tags })) return cleanUpReqFiles(req) - - const body = req.body as RunnerJobUpdateBody - const job = res.locals.runnerJob - - if (isRunnerJobUpdatePayloadValid(body.payload, job.type, req.files) !== true) { - cleanUpReqFiles(req) - - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Payload is invalid', - tags - }) - } - - if (res.locals.runnerJob.type === 'live-rtmp-hls-transcoding') { - const privatePayload = job.privatePayload as RunnerJobLiveRTMPHLSTranscodingPrivatePayload - - if (!LiveManager.Instance.hasSession(privatePayload.sessionId)) { - cleanUpReqFiles(req) - - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - type: ServerErrorCode.RUNNER_JOB_NOT_IN_PROCESSING_STATE, - message: 'Session of this live ended', - tags - }) - } - } - - return next() - } -] - -export const errorRunnerJobValidator = [ - body('message').custom(isRunnerJobErrorMessageValid), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res, { tags })) return - - return next() - } -] - -export const successRunnerJobValidator = [ - (req: express.Request, res: express.Response, next: express.NextFunction) => { - const body = req.body as RunnerJobSuccessBody - - if (isRunnerJobSuccessPayloadValid(body.payload, res.locals.runnerJob.type, req.files) !== true) { - cleanUpReqFiles(req) - - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Payload is invalid', - tags - }) - } - - return next() - } -] - -export const cancelRunnerJobValidator = [ - (req: express.Request, res: express.Response, next: express.NextFunction) => { - const runnerJob = res.locals.runnerJob - - if (runnerJobCanBeCancelled(runnerJob) !== true) { - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Cannot cancel this job that is not in "pending", "processing" or "waiting for parent job" state', - tags - }) - } - - return next() - } -] - -export const listRunnerJobsValidator = [ - query('search') - .optional() - .custom(exists), - - query('stateOneOf') - .optional() - .customSanitizer(arrayify) - .custom(isRunnerJobArrayOfStateValid), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - return next() - } -] - -export const runnerJobGetValidator = [ - param('jobUUID').custom(isUUIDValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res, { tags })) return - - const runnerJob = await RunnerJobModel.loadWithRunner(req.params.jobUUID) - - if (!runnerJob) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Unknown runner job', - tags - }) - } - - res.locals.runnerJob = runnerJob - - return next() - } -] - -export function jobOfRunnerGetValidatorFactory (allowedStates: RunnerJobState[]) { - return [ - param('jobUUID').custom(isUUIDValid), - - body('runnerToken').custom(isRunnerTokenValid), - body('jobToken').custom(isRunnerJobTokenValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res, { tags })) return cleanUpReqFiles(req) - - const runnerJob = await RunnerJobModel.loadByRunnerAndJobTokensWithRunner({ - uuid: req.params.jobUUID, - runnerToken: req.body.runnerToken, - jobToken: req.body.jobToken - }) - - if (!runnerJob) { - cleanUpReqFiles(req) - - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Unknown runner job', - tags - }) - } - - if (!allowedStates.includes(runnerJob.state)) { - cleanUpReqFiles(req) - - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - type: ServerErrorCode.RUNNER_JOB_NOT_IN_PROCESSING_STATE, - message: 'Job is not in "processing" state', - tags - }) - } - - res.locals.runnerJob = runnerJob - - return next() - } - ] -} diff --git a/server/middlewares/validators/runners/registration-token.ts b/server/middlewares/validators/runners/registration-token.ts deleted file mode 100644 index cc31d4a7e..000000000 --- a/server/middlewares/validators/runners/registration-token.ts +++ /dev/null @@ -1,37 +0,0 @@ -import express from 'express' -import { param } from 'express-validator' -import { isIdValid } from '@server/helpers/custom-validators/misc' -import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token' -import { forceNumber } from '@shared/core-utils' -import { HttpStatusCode } from '@shared/models' -import { areValidationErrors } from '../shared/utils' - -const tags = [ 'runner' ] - -const deleteRegistrationTokenValidator = [ - param('id').custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res, { tags })) return - - const registrationToken = await RunnerRegistrationTokenModel.load(forceNumber(req.params.id)) - - if (!registrationToken) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Registration token not found', - tags - }) - } - - res.locals.runnerRegistrationToken = registrationToken - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - deleteRegistrationTokenValidator -} diff --git a/server/middlewares/validators/runners/runners.ts b/server/middlewares/validators/runners/runners.ts deleted file mode 100644 index 4d4d79b4c..000000000 --- a/server/middlewares/validators/runners/runners.ts +++ /dev/null @@ -1,104 +0,0 @@ -import express from 'express' -import { body, param } from 'express-validator' -import { isIdValid } from '@server/helpers/custom-validators/misc' -import { - isRunnerDescriptionValid, - isRunnerNameValid, - isRunnerRegistrationTokenValid, - isRunnerTokenValid -} from '@server/helpers/custom-validators/runners/runners' -import { RunnerModel } from '@server/models/runner/runner' -import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token' -import { forceNumber } from '@shared/core-utils' -import { HttpStatusCode, RegisterRunnerBody, ServerErrorCode } from '@shared/models' -import { areValidationErrors } from '../shared/utils' - -const tags = [ 'runner' ] - -const registerRunnerValidator = [ - body('registrationToken').custom(isRunnerRegistrationTokenValid), - body('name').custom(isRunnerNameValid), - body('description').optional().custom(isRunnerDescriptionValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res, { tags })) return - - const body: RegisterRunnerBody = req.body - - const runnerRegistrationToken = await RunnerRegistrationTokenModel.loadByRegistrationToken(body.registrationToken) - - if (!runnerRegistrationToken) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Registration token is invalid', - tags - }) - } - - const existing = await RunnerModel.loadByName(body.name) - if (existing) { - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'This runner name already exists on this instance', - tags - }) - } - - res.locals.runnerRegistrationToken = runnerRegistrationToken - - return next() - } -] - -const deleteRunnerValidator = [ - param('runnerId').custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res, { tags })) return - - const runner = await RunnerModel.load(forceNumber(req.params.runnerId)) - - if (!runner) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Runner not found', - tags - }) - } - - res.locals.runner = runner - - return next() - } -] - -const getRunnerFromTokenValidator = [ - body('runnerToken').custom(isRunnerTokenValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res, { tags })) return - - const runner = await RunnerModel.loadByToken(req.body.runnerToken) - - if (!runner) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Unknown runner token', - type: ServerErrorCode.UNKNOWN_RUNNER_TOKEN, - tags - }) - } - - res.locals.runner = runner - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - registerRunnerValidator, - deleteRunnerValidator, - getRunnerFromTokenValidator -} diff --git a/server/middlewares/validators/search.ts b/server/middlewares/validators/search.ts deleted file mode 100644 index a63fd0893..000000000 --- a/server/middlewares/validators/search.ts +++ /dev/null @@ -1,112 +0,0 @@ -import express from 'express' -import { query } from 'express-validator' -import { isSearchTargetValid } from '@server/helpers/custom-validators/search' -import { isHostValid } from '@server/helpers/custom-validators/servers' -import { areUUIDsValid, isDateValid, isNotEmptyStringArray, toCompleteUUIDs } from '../../helpers/custom-validators/misc' -import { areValidationErrors } from './shared' - -const videosSearchValidator = [ - query('search') - .optional() - .not().isEmpty(), - - query('host') - .optional() - .custom(isHostValid), - - query('startDate') - .optional() - .custom(isDateValid).withMessage('Should have a start date that conforms to ISO 8601'), - query('endDate') - .optional() - .custom(isDateValid).withMessage('Should have a end date that conforms to ISO 8601'), - - query('originallyPublishedStartDate') - .optional() - .custom(isDateValid).withMessage('Should have a published start date that conforms to ISO 8601'), - query('originallyPublishedEndDate') - .optional() - .custom(isDateValid).withMessage('Should have a published end date that conforms to ISO 8601'), - - query('durationMin') - .optional() - .isInt(), - query('durationMax') - .optional() - .isInt(), - - query('uuids') - .optional() - .toArray() - .customSanitizer(toCompleteUUIDs) - .custom(areUUIDsValid).withMessage('Should have valid array of uuid'), - - query('searchTarget') - .optional() - .custom(isSearchTargetValid), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const videoChannelsListSearchValidator = [ - query('search') - .optional() - .not().isEmpty(), - - query('host') - .optional() - .custom(isHostValid), - - query('searchTarget') - .optional() - .custom(isSearchTargetValid), - - query('handles') - .optional() - .toArray() - .custom(isNotEmptyStringArray).withMessage('Should have valid array of handles'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const videoPlaylistsListSearchValidator = [ - query('search') - .optional() - .not().isEmpty(), - - query('host') - .optional() - .custom(isHostValid), - - query('searchTarget') - .optional() - .custom(isSearchTargetValid), - - query('uuids') - .optional() - .toArray() - .customSanitizer(toCompleteUUIDs) - .custom(areUUIDsValid).withMessage('Should have valid array of uuid'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - videosSearchValidator, - videoChannelsListSearchValidator, - videoPlaylistsListSearchValidator -} diff --git a/server/middlewares/validators/server.ts b/server/middlewares/validators/server.ts deleted file mode 100644 index 7d37ae229..000000000 --- a/server/middlewares/validators/server.ts +++ /dev/null @@ -1,75 +0,0 @@ -import express from 'express' -import { body } from 'express-validator' -import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' -import { isHostValid, isValidContactBody } from '../../helpers/custom-validators/servers' -import { isUserDisplayNameValid } from '../../helpers/custom-validators/users' -import { logger } from '../../helpers/logger' -import { CONFIG, isEmailEnabled } from '../../initializers/config' -import { Redis } from '../../lib/redis' -import { ServerModel } from '../../models/server/server' -import { areValidationErrors } from './shared' - -const serverGetValidator = [ - body('host').custom(isHostValid).withMessage('Should have a valid host'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const server = await ServerModel.loadByHost(req.body.host) - if (!server) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Server host not found.' - }) - } - - res.locals.server = server - - return next() - } -] - -const contactAdministratorValidator = [ - body('fromName') - .custom(isUserDisplayNameValid), - body('fromEmail') - .isEmail(), - body('body') - .custom(isValidContactBody), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - if (CONFIG.CONTACT_FORM.ENABLED === false) { - return res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: 'Contact form is not enabled on this instance.' - }) - } - - if (isEmailEnabled() === false) { - return res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: 'SMTP is not configured on this instance.' - }) - } - - if (await Redis.Instance.doesContactFormIpExist(req.ip)) { - logger.info('Refusing a contact form by %s: already sent one recently.', req.ip) - - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'You already sent a contact form recently.' - }) - } - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - serverGetValidator, - contactAdministratorValidator -} diff --git a/server/middlewares/validators/shared/abuses.ts b/server/middlewares/validators/shared/abuses.ts deleted file mode 100644 index 2c988f9ec..000000000 --- a/server/middlewares/validators/shared/abuses.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { Response } from 'express' -import { AbuseModel } from '@server/models/abuse/abuse' -import { HttpStatusCode } from '@shared/models' -import { forceNumber } from '@shared/core-utils' - -async function doesAbuseExist (abuseId: number | string, res: Response) { - const abuse = await AbuseModel.loadByIdWithReporter(forceNumber(abuseId)) - - if (!abuse) { - res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Abuse not found' - }) - - return false - } - - res.locals.abuse = abuse - return true -} - -// --------------------------------------------------------------------------- - -export { - doesAbuseExist -} diff --git a/server/middlewares/validators/shared/accounts.ts b/server/middlewares/validators/shared/accounts.ts deleted file mode 100644 index 72b0e235e..000000000 --- a/server/middlewares/validators/shared/accounts.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { Response } from 'express' -import { AccountModel } from '@server/models/account/account' -import { UserModel } from '@server/models/user/user' -import { MAccountDefault } from '@server/types/models' -import { forceNumber } from '@shared/core-utils' -import { HttpStatusCode } from '@shared/models' - -function doesAccountIdExist (id: number | string, res: Response, sendNotFound = true) { - const promise = AccountModel.load(forceNumber(id)) - - return doesAccountExist(promise, res, sendNotFound) -} - -function doesLocalAccountNameExist (name: string, res: Response, sendNotFound = true) { - const promise = AccountModel.loadLocalByName(name) - - return doesAccountExist(promise, res, sendNotFound) -} - -function doesAccountNameWithHostExist (nameWithDomain: string, res: Response, sendNotFound = true) { - const promise = AccountModel.loadByNameWithHost(nameWithDomain) - - return doesAccountExist(promise, res, sendNotFound) -} - -async function doesAccountExist (p: Promise, res: Response, sendNotFound: boolean) { - const account = await p - - if (!account) { - if (sendNotFound === true) { - res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Account not found' - }) - } - return false - } - - res.locals.account = account - return true -} - -async function doesUserFeedTokenCorrespond (id: number, token: string, res: Response) { - const user = await UserModel.loadByIdWithChannels(forceNumber(id)) - - if (token !== user.feedToken) { - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'User and token mismatch' - }) - return false - } - - res.locals.user = user - return true -} - -// --------------------------------------------------------------------------- - -export { - doesAccountIdExist, - doesLocalAccountNameExist, - doesAccountNameWithHostExist, - doesAccountExist, - doesUserFeedTokenCorrespond -} diff --git a/server/middlewares/validators/shared/index.ts b/server/middlewares/validators/shared/index.ts deleted file mode 100644 index e5cff2dda..000000000 --- a/server/middlewares/validators/shared/index.ts +++ /dev/null @@ -1,14 +0,0 @@ -export * from './abuses' -export * from './accounts' -export * from './users' -export * from './utils' -export * from './video-blacklists' -export * from './video-captions' -export * from './video-channels' -export * from './video-channel-syncs' -export * from './video-comments' -export * from './video-imports' -export * from './video-ownerships' -export * from './video-playlists' -export * from './video-passwords' -export * from './videos' diff --git a/server/middlewares/validators/shared/user-registrations.ts b/server/middlewares/validators/shared/user-registrations.ts deleted file mode 100644 index dbc7dda06..000000000 --- a/server/middlewares/validators/shared/user-registrations.ts +++ /dev/null @@ -1,60 +0,0 @@ -import express from 'express' -import { UserRegistrationModel } from '@server/models/user/user-registration' -import { MRegistration } from '@server/types/models' -import { forceNumber, pick } from '@shared/core-utils' -import { HttpStatusCode } from '@shared/models' - -function checkRegistrationIdExist (idArg: number | string, res: express.Response) { - const id = forceNumber(idArg) - return checkRegistrationExist(() => UserRegistrationModel.load(id), res) -} - -function checkRegistrationEmailExist (email: string, res: express.Response, abortResponse = true) { - return checkRegistrationExist(() => UserRegistrationModel.loadByEmail(email), res, abortResponse) -} - -async function checkRegistrationHandlesDoNotAlreadyExist (options: { - username: string - channelHandle: string - email: string - res: express.Response -}) { - const { res } = options - - const registration = await UserRegistrationModel.loadByEmailOrHandle(pick(options, [ 'username', 'email', 'channelHandle' ])) - - if (registration) { - res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: 'Registration with this username, channel name or email already exists.' - }) - return false - } - - return true -} - -async function checkRegistrationExist (finder: () => Promise, res: express.Response, abortResponse = true) { - const registration = await finder() - - if (!registration) { - if (abortResponse === true) { - res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'User not found' - }) - } - - return false - } - - res.locals.userRegistration = registration - return true -} - -export { - checkRegistrationIdExist, - checkRegistrationEmailExist, - checkRegistrationHandlesDoNotAlreadyExist, - checkRegistrationExist -} diff --git a/server/middlewares/validators/shared/users.ts b/server/middlewares/validators/shared/users.ts deleted file mode 100644 index 030adc9f7..000000000 --- a/server/middlewares/validators/shared/users.ts +++ /dev/null @@ -1,63 +0,0 @@ -import express from 'express' -import { ActorModel } from '@server/models/actor/actor' -import { UserModel } from '@server/models/user/user' -import { MUserDefault } from '@server/types/models' -import { forceNumber } from '@shared/core-utils' -import { HttpStatusCode } from '@shared/models' - -function checkUserIdExist (idArg: number | string, res: express.Response, withStats = false) { - const id = forceNumber(idArg) - return checkUserExist(() => UserModel.loadByIdWithChannels(id, withStats), res) -} - -function checkUserEmailExist (email: string, res: express.Response, abortResponse = true) { - return checkUserExist(() => UserModel.loadByEmail(email), res, abortResponse) -} - -async function checkUserNameOrEmailDoNotAlreadyExist (username: string, email: string, res: express.Response) { - const user = await UserModel.loadByUsernameOrEmail(username, email) - - if (user) { - res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: 'User with this username or email already exists.' - }) - return false - } - - const actor = await ActorModel.loadLocalByName(username) - if (actor) { - res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: 'Another actor (account/channel) with this name on this instance already exists or has already existed.' - }) - return false - } - - return true -} - -async function checkUserExist (finder: () => Promise, res: express.Response, abortResponse = true) { - const user = await finder() - - if (!user) { - if (abortResponse === true) { - res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'User not found' - }) - } - - return false - } - - res.locals.user = user - return true -} - -export { - checkUserIdExist, - checkUserEmailExist, - checkUserNameOrEmailDoNotAlreadyExist, - checkUserExist -} diff --git a/server/middlewares/validators/shared/utils.ts b/server/middlewares/validators/shared/utils.ts deleted file mode 100644 index f39128fdd..000000000 --- a/server/middlewares/validators/shared/utils.ts +++ /dev/null @@ -1,69 +0,0 @@ -import express from 'express' -import { param, validationResult } from 'express-validator' -import { isIdOrUUIDValid, toCompleteUUID } from '@server/helpers/custom-validators/misc' -import { logger } from '../../../helpers/logger' - -function areValidationErrors ( - req: express.Request, - res: express.Response, - options: { - omitLog?: boolean - omitBodyLog?: boolean - tags?: string[] - } = {}) { - const { omitLog = false, omitBodyLog = false, tags = [] } = options - - if (!omitLog) { - logger.debug( - 'Checking %s - %s parameters', - req.method, req.originalUrl, - { - body: omitBodyLog - ? 'omitted' - : req.body, - params: req.params, - query: req.query, - files: req.files, - tags - } - ) - } - - const errors = validationResult(req) - - if (!errors.isEmpty()) { - logger.warn('Incorrect request parameters', { path: req.originalUrl, err: errors.mapped() }) - - res.fail({ - message: 'Incorrect request parameters: ' + Object.keys(errors.mapped()).join(', '), - instance: req.originalUrl, - data: { - 'invalid-params': errors.mapped() - } - }) - - return true - } - - return false -} - -function isValidVideoIdParam (paramName: string) { - return param(paramName) - .customSanitizer(toCompleteUUID) - .custom(isIdOrUUIDValid).withMessage('Should have a valid video id (id, short UUID or UUID)') -} - -function isValidPlaylistIdParam (paramName: string) { - return param(paramName) - .customSanitizer(toCompleteUUID) - .custom(isIdOrUUIDValid).withMessage('Should have a valid playlist id (id, short UUID or UUID)') -} - -// --------------------------------------------------------------------------- - -export { - areValidationErrors, - isValidVideoIdParam, - isValidPlaylistIdParam -} diff --git a/server/middlewares/validators/shared/video-blacklists.ts b/server/middlewares/validators/shared/video-blacklists.ts deleted file mode 100644 index f85b39b23..000000000 --- a/server/middlewares/validators/shared/video-blacklists.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { Response } from 'express' -import { VideoBlacklistModel } from '@server/models/video/video-blacklist' -import { HttpStatusCode } from '@shared/models' - -async function doesVideoBlacklistExist (videoId: number, res: Response) { - const videoBlacklist = await VideoBlacklistModel.loadByVideoId(videoId) - - if (videoBlacklist === null) { - res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Blacklisted video not found' - }) - return false - } - - res.locals.videoBlacklist = videoBlacklist - return true -} - -// --------------------------------------------------------------------------- - -export { - doesVideoBlacklistExist -} diff --git a/server/middlewares/validators/shared/video-captions.ts b/server/middlewares/validators/shared/video-captions.ts deleted file mode 100644 index 831b366ea..000000000 --- a/server/middlewares/validators/shared/video-captions.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { Response } from 'express' -import { VideoCaptionModel } from '@server/models/video/video-caption' -import { MVideoId } from '@server/types/models' -import { HttpStatusCode } from '@shared/models' - -async function doesVideoCaptionExist (video: MVideoId, language: string, res: Response) { - const videoCaption = await VideoCaptionModel.loadByVideoIdAndLanguage(video.id, language) - - if (!videoCaption) { - res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video caption not found' - }) - return false - } - - res.locals.videoCaption = videoCaption - return true -} - -// --------------------------------------------------------------------------- - -export { - doesVideoCaptionExist -} diff --git a/server/middlewares/validators/shared/video-channel-syncs.ts b/server/middlewares/validators/shared/video-channel-syncs.ts deleted file mode 100644 index a6e51eb97..000000000 --- a/server/middlewares/validators/shared/video-channel-syncs.ts +++ /dev/null @@ -1,24 +0,0 @@ -import express from 'express' -import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' -import { HttpStatusCode } from '@shared/models' - -async function doesVideoChannelSyncIdExist (id: number, res: express.Response) { - const sync = await VideoChannelSyncModel.loadWithChannel(+id) - - if (!sync) { - res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video channel sync not found' - }) - return false - } - - res.locals.videoChannelSync = sync - return true -} - -// --------------------------------------------------------------------------- - -export { - doesVideoChannelSyncIdExist -} diff --git a/server/middlewares/validators/shared/video-channels.ts b/server/middlewares/validators/shared/video-channels.ts deleted file mode 100644 index bed9f5dbe..000000000 --- a/server/middlewares/validators/shared/video-channels.ts +++ /dev/null @@ -1,36 +0,0 @@ -import express from 'express' -import { VideoChannelModel } from '@server/models/video/video-channel' -import { MChannelBannerAccountDefault } from '@server/types/models' -import { HttpStatusCode } from '@shared/models' - -async function doesVideoChannelIdExist (id: number, res: express.Response) { - const videoChannel = await VideoChannelModel.loadAndPopulateAccount(+id) - - return processVideoChannelExist(videoChannel, res) -} - -async function doesVideoChannelNameWithHostExist (nameWithDomain: string, res: express.Response) { - const videoChannel = await VideoChannelModel.loadByNameWithHostAndPopulateAccount(nameWithDomain) - - return processVideoChannelExist(videoChannel, res) -} - -// --------------------------------------------------------------------------- - -export { - doesVideoChannelIdExist, - doesVideoChannelNameWithHostExist -} - -function processVideoChannelExist (videoChannel: MChannelBannerAccountDefault, res: express.Response) { - if (!videoChannel) { - res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video channel not found' - }) - return false - } - - res.locals.videoChannel = videoChannel - return true -} diff --git a/server/middlewares/validators/shared/video-comments.ts b/server/middlewares/validators/shared/video-comments.ts deleted file mode 100644 index 0961b3ec9..000000000 --- a/server/middlewares/validators/shared/video-comments.ts +++ /dev/null @@ -1,80 +0,0 @@ -import express from 'express' -import { VideoCommentModel } from '@server/models/video/video-comment' -import { MVideoId } from '@server/types/models' -import { forceNumber } from '@shared/core-utils' -import { HttpStatusCode, ServerErrorCode } from '@shared/models' - -async function doesVideoCommentThreadExist (idArg: number | string, video: MVideoId, res: express.Response) { - const id = forceNumber(idArg) - const videoComment = await VideoCommentModel.loadById(id) - - if (!videoComment) { - res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video comment thread not found' - }) - return false - } - - if (videoComment.videoId !== video.id) { - res.fail({ - type: ServerErrorCode.COMMENT_NOT_ASSOCIATED_TO_VIDEO, - message: 'Video comment is not associated to this video.' - }) - return false - } - - if (videoComment.inReplyToCommentId !== null) { - res.fail({ message: 'Video comment is not a thread.' }) - return false - } - - res.locals.videoCommentThread = videoComment - return true -} - -async function doesVideoCommentExist (idArg: number | string, video: MVideoId, res: express.Response) { - const id = forceNumber(idArg) - const videoComment = await VideoCommentModel.loadByIdAndPopulateVideoAndAccountAndReply(id) - - if (!videoComment) { - res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video comment thread not found' - }) - return false - } - - if (videoComment.videoId !== video.id) { - res.fail({ - type: ServerErrorCode.COMMENT_NOT_ASSOCIATED_TO_VIDEO, - message: 'Video comment is not associated to this video.' - }) - return false - } - - res.locals.videoCommentFull = videoComment - return true -} - -async function doesCommentIdExist (idArg: number | string, res: express.Response) { - const id = forceNumber(idArg) - const videoComment = await VideoCommentModel.loadByIdAndPopulateVideoAndAccountAndReply(id) - - if (!videoComment) { - res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video comment thread not found' - }) - return false - } - - res.locals.videoCommentFull = videoComment - return true -} - -export { - doesVideoCommentThreadExist, - doesVideoCommentExist, - doesCommentIdExist -} diff --git a/server/middlewares/validators/shared/video-imports.ts b/server/middlewares/validators/shared/video-imports.ts deleted file mode 100644 index 69fda4b32..000000000 --- a/server/middlewares/validators/shared/video-imports.ts +++ /dev/null @@ -1,22 +0,0 @@ -import express from 'express' -import { VideoImportModel } from '@server/models/video/video-import' -import { HttpStatusCode } from '@shared/models' - -async function doesVideoImportExist (id: number, res: express.Response) { - const videoImport = await VideoImportModel.loadAndPopulateVideo(id) - - if (!videoImport) { - res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video import not found' - }) - return false - } - - res.locals.videoImport = videoImport - return true -} - -export { - doesVideoImportExist -} diff --git a/server/middlewares/validators/shared/video-ownerships.ts b/server/middlewares/validators/shared/video-ownerships.ts deleted file mode 100644 index 33ac9c8b6..000000000 --- a/server/middlewares/validators/shared/video-ownerships.ts +++ /dev/null @@ -1,25 +0,0 @@ -import express from 'express' -import { VideoChangeOwnershipModel } from '@server/models/video/video-change-ownership' -import { forceNumber } from '@shared/core-utils' -import { HttpStatusCode } from '@shared/models' - -async function doesChangeVideoOwnershipExist (idArg: number | string, res: express.Response) { - const id = forceNumber(idArg) - const videoChangeOwnership = await VideoChangeOwnershipModel.load(id) - - if (!videoChangeOwnership) { - res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video change ownership not found' - }) - return false - } - - res.locals.videoChangeOwnership = videoChangeOwnership - - return true -} - -export { - doesChangeVideoOwnershipExist -} diff --git a/server/middlewares/validators/shared/video-passwords.ts b/server/middlewares/validators/shared/video-passwords.ts deleted file mode 100644 index efcc95dc4..000000000 --- a/server/middlewares/validators/shared/video-passwords.ts +++ /dev/null @@ -1,80 +0,0 @@ -import express from 'express' -import { HttpStatusCode, UserRight, VideoPrivacy } from '@shared/models' -import { forceNumber } from '@shared/core-utils' -import { VideoPasswordModel } from '@server/models/video/video-password' -import { header } from 'express-validator' -import { getVideoWithAttributes } from '@server/helpers/video' - -function isValidVideoPasswordHeader () { - return header('x-peertube-video-password') - .optional() - .isString() -} - -function checkVideoIsPasswordProtected (res: express.Response) { - const video = getVideoWithAttributes(res) - if (video.privacy !== VideoPrivacy.PASSWORD_PROTECTED) { - res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Video is not password protected' - }) - return false - } - - return true -} - -async function doesVideoPasswordExist (idArg: number | string, res: express.Response) { - const video = getVideoWithAttributes(res) - const id = forceNumber(idArg) - const videoPassword = await VideoPasswordModel.loadByIdAndVideo({ id, videoId: video.id }) - - if (!videoPassword) { - res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video password not found' - }) - return false - } - - res.locals.videoPassword = videoPassword - - return true -} - -async function isVideoPasswordDeletable (res: express.Response) { - const user = res.locals.oauth.token.User - const userAccount = user.Account - const video = res.locals.videoAll - - // Check if the user who did the request is able to delete the video passwords - if ( - user.hasRight(UserRight.UPDATE_ANY_VIDEO) === false && // Not a moderator - video.VideoChannel.accountId !== userAccount.id // Not the video owner - ) { - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Cannot remove passwords of another user\'s video' - }) - return false - } - - const passwordCount = await VideoPasswordModel.countByVideoId(video.id) - - if (passwordCount <= 1) { - res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Cannot delete the last password of the protected video' - }) - return false - } - - return true -} - -export { - isValidVideoPasswordHeader, - checkVideoIsPasswordProtected as isVideoPasswordProtected, - doesVideoPasswordExist, - isVideoPasswordDeletable -} diff --git a/server/middlewares/validators/shared/video-playlists.ts b/server/middlewares/validators/shared/video-playlists.ts deleted file mode 100644 index 4342fe552..000000000 --- a/server/middlewares/validators/shared/video-playlists.ts +++ /dev/null @@ -1,39 +0,0 @@ -import express from 'express' -import { VideoPlaylistModel } from '@server/models/video/video-playlist' -import { MVideoPlaylist } from '@server/types/models' -import { HttpStatusCode } from '@shared/models' - -export type VideoPlaylistFetchType = 'summary' | 'all' -async function doesVideoPlaylistExist (id: number | string, res: express.Response, fetchType: VideoPlaylistFetchType = 'summary') { - if (fetchType === 'summary') { - const videoPlaylist = await VideoPlaylistModel.loadWithAccountAndChannelSummary(id, undefined) - res.locals.videoPlaylistSummary = videoPlaylist - - return handleVideoPlaylist(videoPlaylist, res) - } - - const videoPlaylist = await VideoPlaylistModel.loadWithAccountAndChannel(id, undefined) - res.locals.videoPlaylistFull = videoPlaylist - - return handleVideoPlaylist(videoPlaylist, res) -} - -// --------------------------------------------------------------------------- - -export { - doesVideoPlaylistExist -} - -// --------------------------------------------------------------------------- - -function handleVideoPlaylist (videoPlaylist: MVideoPlaylist, res: express.Response) { - if (!videoPlaylist) { - res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video playlist not found' - }) - return false - } - - return true -} diff --git a/server/middlewares/validators/shared/videos.ts b/server/middlewares/validators/shared/videos.ts deleted file mode 100644 index 9a7497007..000000000 --- a/server/middlewares/validators/shared/videos.ts +++ /dev/null @@ -1,311 +0,0 @@ -import { Request, Response } from 'express' -import { loadVideo, VideoLoadType } from '@server/lib/model-loaders' -import { isAbleToUploadVideo } from '@server/lib/user' -import { VideoTokensManager } from '@server/lib/video-tokens-manager' -import { authenticatePromise } from '@server/middlewares/auth' -import { VideoModel } from '@server/models/video/video' -import { VideoChannelModel } from '@server/models/video/video-channel' -import { VideoFileModel } from '@server/models/video/video-file' -import { - MUser, - MUserAccountId, - MUserId, - MVideo, - MVideoAccountLight, - MVideoFormattableDetails, - MVideoFullLight, - MVideoId, - MVideoImmutable, - MVideoThumbnail, - MVideoWithRights -} from '@server/types/models' -import { HttpStatusCode, ServerErrorCode, UserRight, VideoPrivacy } from '@shared/models' -import { VideoPasswordModel } from '@server/models/video/video-password' -import { exists } from '@server/helpers/custom-validators/misc' - -async function doesVideoExist (id: number | string, res: Response, fetchType: VideoLoadType = 'all') { - const userId = res.locals.oauth ? res.locals.oauth.token.User.id : undefined - - const video = await loadVideo(id, fetchType, userId) - - if (!video) { - res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video not found' - }) - - return false - } - - switch (fetchType) { - case 'for-api': - res.locals.videoAPI = video as MVideoFormattableDetails - break - - case 'all': - res.locals.videoAll = video as MVideoFullLight - break - - case 'only-immutable-attributes': - res.locals.onlyImmutableVideo = video as MVideoImmutable - break - - case 'id': - res.locals.videoId = video as MVideoId - break - - case 'only-video': - res.locals.onlyVideo = video as MVideoThumbnail - break - } - - return true -} - -// --------------------------------------------------------------------------- - -async function doesVideoFileOfVideoExist (id: number, videoIdOrUUID: number | string, res: Response) { - if (!await VideoFileModel.doesVideoExistForVideoFile(id, videoIdOrUUID)) { - res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'VideoFile matching Video not found' - }) - return false - } - - return true -} - -// --------------------------------------------------------------------------- - -async function doesVideoChannelOfAccountExist (channelId: number, user: MUserAccountId, res: Response) { - const videoChannel = await VideoChannelModel.loadAndPopulateAccount(channelId) - - if (videoChannel === null) { - res.fail({ message: 'Unknown video "video channel" for this instance.' }) - return false - } - - // Don't check account id if the user can update any video - if (user.hasRight(UserRight.UPDATE_ANY_VIDEO) === true) { - res.locals.videoChannel = videoChannel - return true - } - - if (videoChannel.Account.id !== user.Account.id) { - res.fail({ - message: 'Unknown video "video channel" for this account.' - }) - return false - } - - res.locals.videoChannel = videoChannel - return true -} - -// --------------------------------------------------------------------------- - -async function checkCanSeeVideo (options: { - req: Request - res: Response - paramId: string - video: MVideo -}) { - const { req, res, video, paramId } = options - - if (video.requiresUserAuth({ urlParamId: paramId, checkBlacklist: true })) { - return checkCanSeeUserAuthVideo({ req, res, video }) - } - - if (video.privacy === VideoPrivacy.PASSWORD_PROTECTED) { - return checkCanSeePasswordProtectedVideo({ req, res, video }) - } - - if (video.privacy === VideoPrivacy.UNLISTED || video.privacy === VideoPrivacy.PUBLIC) { - return true - } - - throw new Error('Unknown video privacy when checking video right ' + video.url) -} - -async function checkCanSeeUserAuthVideo (options: { - req: Request - res: Response - video: MVideoId | MVideoWithRights -}) { - const { req, res, video } = options - - const fail = () => { - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Cannot fetch information of private/internal/blocked video' - }) - - return false - } - - await authenticatePromise({ req, res }) - - const user = res.locals.oauth?.token.User - if (!user) return fail() - - const videoWithRights = await getVideoWithRights(video as MVideoWithRights) - - const privacy = videoWithRights.privacy - - if (privacy === VideoPrivacy.INTERNAL) { - // We know we have a user - return true - } - - if (videoWithRights.isBlacklisted()) { - if (canUserAccessVideo(user, videoWithRights, UserRight.MANAGE_VIDEO_BLACKLIST)) return true - - return fail() - } - - if (privacy === VideoPrivacy.PRIVATE || privacy === VideoPrivacy.UNLISTED) { - if (canUserAccessVideo(user, videoWithRights, UserRight.SEE_ALL_VIDEOS)) return true - - return fail() - } - - // Should not happen - return fail() -} - -async function checkCanSeePasswordProtectedVideo (options: { - req: Request - res: Response - video: MVideo -}) { - const { req, res, video } = options - - const videoWithRights = await getVideoWithRights(video as MVideoWithRights) - - const videoPassword = req.header('x-peertube-video-password') - - if (!exists(videoPassword)) { - const errorMessage = 'Please provide a password to access this password protected video' - const errorType = ServerErrorCode.VIDEO_REQUIRES_PASSWORD - - if (req.header('authorization')) { - await authenticatePromise({ req, res, errorMessage, errorStatus: HttpStatusCode.FORBIDDEN_403, errorType }) - const user = res.locals.oauth?.token.User - - if (canUserAccessVideo(user, videoWithRights, UserRight.SEE_ALL_VIDEOS)) return true - } - - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - type: errorType, - message: errorMessage - }) - return false - } - - if (await VideoPasswordModel.isACorrectPassword({ videoId: video.id, password: videoPassword })) return true - - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - type: ServerErrorCode.INCORRECT_VIDEO_PASSWORD, - message: 'Incorrect video password. Access to the video is denied.' - }) - - return false -} - -function canUserAccessVideo (user: MUser, video: MVideoWithRights | MVideoAccountLight, right: UserRight) { - const isOwnedByUser = video.VideoChannel.Account.userId === user.id - - return isOwnedByUser || user.hasRight(right) -} - -async function getVideoWithRights (video: MVideoWithRights): Promise { - return video.VideoChannel?.Account?.userId - ? video - : VideoModel.loadFull(video.id) -} - -// --------------------------------------------------------------------------- - -async function checkCanAccessVideoStaticFiles (options: { - video: MVideo - req: Request - res: Response - paramId: string -}) { - const { video, req, res } = options - - if (res.locals.oauth?.token.User || exists(req.header('x-peertube-video-password'))) { - return checkCanSeeVideo(options) - } - - const videoFileToken = req.query.videoFileToken - if (videoFileToken && VideoTokensManager.Instance.hasToken({ token: videoFileToken, videoUUID: video.uuid })) { - const user = VideoTokensManager.Instance.getUserFromToken({ token: videoFileToken }) - - res.locals.videoFileToken = { user } - return true - } - - if (!video.hasPrivateStaticPath()) return true - - res.sendStatus(HttpStatusCode.FORBIDDEN_403) - return false -} - -// --------------------------------------------------------------------------- - -function checkUserCanManageVideo (user: MUser, video: MVideoAccountLight, right: UserRight, res: Response, onlyOwned = true) { - // Retrieve the user who did the request - if (onlyOwned && video.isOwned() === false) { - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Cannot manage a video of another server.' - }) - return false - } - - // Check if the user can delete the video - // The user can delete it if he has the right - // Or if s/he is the video's account - const account = video.VideoChannel.Account - if (user.hasRight(right) === false && account.userId !== user.id) { - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Cannot manage a video of another user.' - }) - return false - } - - return true -} - -// --------------------------------------------------------------------------- - -async function checkUserQuota (user: MUserId, videoFileSize: number, res: Response) { - if (await isAbleToUploadVideo(user.id, videoFileSize) === false) { - res.fail({ - status: HttpStatusCode.PAYLOAD_TOO_LARGE_413, - message: 'The user video quota is exceeded with this video.', - type: ServerErrorCode.QUOTA_REACHED - }) - return false - } - - return true -} - -// --------------------------------------------------------------------------- - -export { - doesVideoChannelOfAccountExist, - doesVideoExist, - doesVideoFileOfVideoExist, - - checkCanAccessVideoStaticFiles, - checkUserCanManageVideo, - checkCanSeeVideo, - checkUserQuota -} diff --git a/server/middlewares/validators/sort.ts b/server/middlewares/validators/sort.ts deleted file mode 100644 index 07d6cba82..000000000 --- a/server/middlewares/validators/sort.ts +++ /dev/null @@ -1,66 +0,0 @@ -import express from 'express' -import { query } from 'express-validator' -import { SORTABLE_COLUMNS } from '../../initializers/constants' -import { areValidationErrors } from './shared' - -export const adminUsersSortValidator = checkSortFactory(SORTABLE_COLUMNS.ADMIN_USERS) -export const accountsSortValidator = checkSortFactory(SORTABLE_COLUMNS.ACCOUNTS) -export const jobsSortValidator = checkSortFactory(SORTABLE_COLUMNS.JOBS, [ 'jobs' ]) -export const abusesSortValidator = checkSortFactory(SORTABLE_COLUMNS.ABUSES) -export const videosSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEOS) -export const videoImportsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_IMPORTS) -export const videosSearchSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEOS_SEARCH) -export const videoChannelsSearchSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_CHANNELS_SEARCH) -export const videoPlaylistsSearchSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_PLAYLISTS_SEARCH) -export const videoCommentsValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_COMMENTS) -export const videoCommentThreadsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_COMMENT_THREADS) -export const videoRatesSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_RATES) -export const blacklistSortValidator = checkSortFactory(SORTABLE_COLUMNS.BLACKLISTS) -export const videoChannelsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_CHANNELS) -export const instanceFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.INSTANCE_FOLLOWERS) -export const instanceFollowingSortValidator = checkSortFactory(SORTABLE_COLUMNS.INSTANCE_FOLLOWING) -export const userSubscriptionsSortValidator = checkSortFactory(SORTABLE_COLUMNS.USER_SUBSCRIPTIONS) -export const accountsBlocklistSortValidator = checkSortFactory(SORTABLE_COLUMNS.ACCOUNTS_BLOCKLIST) -export const serversBlocklistSortValidator = checkSortFactory(SORTABLE_COLUMNS.SERVERS_BLOCKLIST) -export const userNotificationsSortValidator = checkSortFactory(SORTABLE_COLUMNS.USER_NOTIFICATIONS) -export const videoPlaylistsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_PLAYLISTS) -export const pluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.PLUGINS) -export const availablePluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.AVAILABLE_PLUGINS) -export const videoRedundanciesSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_REDUNDANCIES) -export const videoChannelSyncsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_CHANNEL_SYNCS) -export const videoPasswordsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_PASSWORDS) - -export const accountsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.ACCOUNT_FOLLOWERS) -export const videoChannelsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.CHANNEL_FOLLOWERS) - -export const userRegistrationsSortValidator = checkSortFactory(SORTABLE_COLUMNS.USER_REGISTRATIONS) - -export const runnersSortValidator = checkSortFactory(SORTABLE_COLUMNS.RUNNERS) -export const runnerRegistrationTokensSortValidator = checkSortFactory(SORTABLE_COLUMNS.RUNNER_REGISTRATION_TOKENS) -export const runnerJobsSortValidator = checkSortFactory(SORTABLE_COLUMNS.RUNNER_JOBS) - -// --------------------------------------------------------------------------- - -function checkSortFactory (columns: string[], tags: string[] = []) { - return checkSort(createSortableColumns(columns), tags) -} - -function checkSort (sortableColumns: string[], tags: string[] = []) { - return [ - query('sort') - .optional() - .isIn(sortableColumns), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res, { tags })) return - - return next() - } - ] -} - -function createSortableColumns (sortableColumns: string[]) { - const sortableColumnDesc = sortableColumns.map(sortableColumn => '-' + sortableColumn) - - return sortableColumns.concat(sortableColumnDesc) -} diff --git a/server/middlewares/validators/static.ts b/server/middlewares/validators/static.ts deleted file mode 100644 index 86cc0a8d7..000000000 --- a/server/middlewares/validators/static.ts +++ /dev/null @@ -1,184 +0,0 @@ -import express from 'express' -import { query } from 'express-validator' -import { LRUCache } from 'lru-cache' -import { basename, dirname } from 'path' -import { exists, isSafePeerTubeFilenameWithoutExtension, isUUIDValid, toBooleanOrNull } from '@server/helpers/custom-validators/misc' -import { logger } from '@server/helpers/logger' -import { LRU_CACHE } from '@server/initializers/constants' -import { VideoModel } from '@server/models/video/video' -import { VideoFileModel } from '@server/models/video/video-file' -import { MStreamingPlaylist, MVideoFile, MVideoThumbnail } from '@server/types/models' -import { HttpStatusCode } from '@shared/models' -import { areValidationErrors, checkCanAccessVideoStaticFiles, isValidVideoPasswordHeader } from './shared' - -type LRUValue = { - allowed: boolean - video?: MVideoThumbnail - file?: MVideoFile - playlist?: MStreamingPlaylist } - -const staticFileTokenBypass = new LRUCache({ - max: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.MAX_SIZE, - ttl: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.TTL -}) - -const ensureCanAccessVideoPrivateWebVideoFiles = [ - query('videoFileToken').optional().custom(exists), - - isValidVideoPasswordHeader(), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const token = extractTokenOrDie(req, res) - if (!token) return - - const cacheKey = token + '-' + req.originalUrl - - if (staticFileTokenBypass.has(cacheKey)) { - const { allowed, file, video } = staticFileTokenBypass.get(cacheKey) - - if (allowed === true) { - res.locals.onlyVideo = video - res.locals.videoFile = file - - return next() - } - - return res.sendStatus(HttpStatusCode.FORBIDDEN_403) - } - - const result = await isWebVideoAllowed(req, res) - - staticFileTokenBypass.set(cacheKey, result) - - if (result.allowed !== true) return - - res.locals.onlyVideo = result.video - res.locals.videoFile = result.file - - return next() - } -] - -const ensureCanAccessPrivateVideoHLSFiles = [ - query('videoFileToken') - .optional() - .custom(exists), - - query('reinjectVideoFileToken') - .optional() - .customSanitizer(toBooleanOrNull) - .isBoolean().withMessage('Should be a valid reinjectVideoFileToken boolean'), - - query('playlistName') - .optional() - .customSanitizer(isSafePeerTubeFilenameWithoutExtension), - - isValidVideoPasswordHeader(), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const videoUUID = basename(dirname(req.originalUrl)) - - if (!isUUIDValid(videoUUID)) { - logger.debug('Path does not contain valid video UUID to serve static file %s', req.originalUrl) - - return res.sendStatus(HttpStatusCode.FORBIDDEN_403) - } - - const token = extractTokenOrDie(req, res) - if (!token) return - - const cacheKey = token + '-' + videoUUID - - if (staticFileTokenBypass.has(cacheKey)) { - const { allowed, file, playlist, video } = staticFileTokenBypass.get(cacheKey) - - if (allowed === true) { - res.locals.onlyVideo = video - res.locals.videoFile = file - res.locals.videoStreamingPlaylist = playlist - - return next() - } - - return res.sendStatus(HttpStatusCode.FORBIDDEN_403) - } - - const result = await isHLSAllowed(req, res, videoUUID) - - staticFileTokenBypass.set(cacheKey, result) - - if (result.allowed !== true) return - - res.locals.onlyVideo = result.video - res.locals.videoFile = result.file - res.locals.videoStreamingPlaylist = result.playlist - - return next() - } -] - -export { - ensureCanAccessVideoPrivateWebVideoFiles, - ensureCanAccessPrivateVideoHLSFiles -} - -// --------------------------------------------------------------------------- - -async function isWebVideoAllowed (req: express.Request, res: express.Response) { - const filename = basename(req.path) - - const file = await VideoFileModel.loadWithVideoByFilename(filename) - if (!file) { - logger.debug('Unknown static file %s to serve', req.originalUrl, { filename }) - - res.sendStatus(HttpStatusCode.FORBIDDEN_403) - return { allowed: false } - } - - const video = await VideoModel.load(file.getVideo().id) - - return { - file, - video, - allowed: await checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid }) - } -} - -async function isHLSAllowed (req: express.Request, res: express.Response, videoUUID: string) { - const filename = basename(req.path) - - const video = await VideoModel.loadWithFiles(videoUUID) - - if (!video) { - logger.debug('Unknown static file %s to serve', req.originalUrl, { videoUUID }) - - res.sendStatus(HttpStatusCode.FORBIDDEN_403) - return { allowed: false } - } - - const file = await VideoFileModel.loadByFilename(filename) - - return { - file, - video, - playlist: video.getHLSPlaylist(), - allowed: await checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid }) - } -} - -function extractTokenOrDie (req: express.Request, res: express.Response) { - const token = req.header('x-peertube-video-password') || req.query.videoFileToken || res.locals.oauth?.token.accessToken - - if (!token) { - return res.fail({ - message: 'Video password header, video file token query parameter and bearer token are all missing', // - status: HttpStatusCode.FORBIDDEN_403 - }) - } - - return token -} diff --git a/server/middlewares/validators/themes.ts b/server/middlewares/validators/themes.ts deleted file mode 100644 index 080b3e096..000000000 --- a/server/middlewares/validators/themes.ts +++ /dev/null @@ -1,46 +0,0 @@ -import express from 'express' -import { param } from 'express-validator' -import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' -import { isSafePath } from '../../helpers/custom-validators/misc' -import { isPluginNameValid, isPluginStableOrUnstableVersionValid } from '../../helpers/custom-validators/plugins' -import { PluginManager } from '../../lib/plugins/plugin-manager' -import { areValidationErrors } from './shared' - -const serveThemeCSSValidator = [ - param('themeName') - .custom(isPluginNameValid), - param('themeVersion') - .custom(isPluginStableOrUnstableVersionValid), - param('staticEndpoint') - .custom(isSafePath), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const theme = PluginManager.Instance.getRegisteredThemeByShortName(req.params.themeName) - - if (!theme || theme.version !== req.params.themeVersion) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'No theme named ' + req.params.themeName + ' was found with version ' + req.params.themeVersion - }) - } - - if (theme.css.includes(req.params.staticEndpoint) === false) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'No static endpoint was found for this theme' - }) - } - - res.locals.registeredPlugin = theme - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - serveThemeCSSValidator -} diff --git a/server/middlewares/validators/two-factor.ts b/server/middlewares/validators/two-factor.ts deleted file mode 100644 index 106b579b5..000000000 --- a/server/middlewares/validators/two-factor.ts +++ /dev/null @@ -1,81 +0,0 @@ -import express from 'express' -import { body, param } from 'express-validator' -import { HttpStatusCode, UserRight } from '@shared/models' -import { exists, isIdValid } from '../../helpers/custom-validators/misc' -import { areValidationErrors, checkUserIdExist } from './shared' - -const requestOrConfirmTwoFactorValidator = [ - param('id').custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - if (!await checkCanEnableOrDisableTwoFactor(req.params.id, res)) return - - if (res.locals.user.otpSecret) { - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: `Two factor is already enabled.` - }) - } - - return next() - } -] - -const confirmTwoFactorValidator = [ - body('requestToken').custom(exists), - body('otpToken').custom(exists), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const disableTwoFactorValidator = [ - param('id').custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - if (!await checkCanEnableOrDisableTwoFactor(req.params.id, res)) return - - if (!res.locals.user.otpSecret) { - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: `Two factor is already disabled.` - }) - } - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - requestOrConfirmTwoFactorValidator, - confirmTwoFactorValidator, - disableTwoFactorValidator -} - -// --------------------------------------------------------------------------- - -async function checkCanEnableOrDisableTwoFactor (userId: number | string, res: express.Response) { - const authUser = res.locals.oauth.token.user - - if (!await checkUserIdExist(userId, res)) return - - if (res.locals.user.id !== authUser.id && authUser.hasRight(UserRight.MANAGE_USERS) !== true) { - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: `User ${authUser.username} does not have right to change two factor setting of this user.` - }) - - return false - } - - return true -} diff --git a/server/middlewares/validators/user-email-verification.ts b/server/middlewares/validators/user-email-verification.ts deleted file mode 100644 index 74702a8f5..000000000 --- a/server/middlewares/validators/user-email-verification.ts +++ /dev/null @@ -1,94 +0,0 @@ -import express from 'express' -import { body, param } from 'express-validator' -import { toBooleanOrNull } from '@server/helpers/custom-validators/misc' -import { HttpStatusCode } from '@shared/models' -import { logger } from '../../helpers/logger' -import { Redis } from '../../lib/redis' -import { areValidationErrors, checkUserEmailExist, checkUserIdExist } from './shared' -import { checkRegistrationEmailExist, checkRegistrationIdExist } from './shared/user-registrations' - -const usersAskSendVerifyEmailValidator = [ - body('email').isEmail().not().isEmpty().withMessage('Should have a valid email'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const [ userExists, registrationExists ] = await Promise.all([ - checkUserEmailExist(req.body.email, res, false), - checkRegistrationEmailExist(req.body.email, res, false) - ]) - - if (!userExists && !registrationExists) { - logger.debug('User or registration with email %s does not exist (asking verify email).', req.body.email) - // Do not leak our emails - return res.status(HttpStatusCode.NO_CONTENT_204).end() - } - - if (res.locals.user?.pluginAuth) { - return res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: 'Cannot ask verification email of a user that uses a plugin authentication.' - }) - } - - return next() - } -] - -const usersVerifyEmailValidator = [ - param('id') - .isInt().not().isEmpty().withMessage('Should have a valid id'), - - body('verificationString') - .not().isEmpty().withMessage('Should have a valid verification string'), - body('isPendingEmail') - .optional() - .customSanitizer(toBooleanOrNull), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await checkUserIdExist(req.params.id, res)) return - - const user = res.locals.user - const redisVerificationString = await Redis.Instance.getUserVerifyEmailLink(user.id) - - if (redisVerificationString !== req.body.verificationString) { - return res.fail({ status: HttpStatusCode.FORBIDDEN_403, message: 'Invalid verification string.' }) - } - - return next() - } -] - -// --------------------------------------------------------------------------- - -const registrationVerifyEmailValidator = [ - param('registrationId') - .isInt().not().isEmpty().withMessage('Should have a valid registrationId'), - - body('verificationString') - .not().isEmpty().withMessage('Should have a valid verification string'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await checkRegistrationIdExist(req.params.registrationId, res)) return - - const registration = res.locals.userRegistration - const redisVerificationString = await Redis.Instance.getRegistrationVerifyEmailLink(registration.id) - - if (redisVerificationString !== req.body.verificationString) { - return res.fail({ status: HttpStatusCode.FORBIDDEN_403, message: 'Invalid verification string.' }) - } - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - usersAskSendVerifyEmailValidator, - usersVerifyEmailValidator, - - registrationVerifyEmailValidator -} diff --git a/server/middlewares/validators/user-history.ts b/server/middlewares/validators/user-history.ts deleted file mode 100644 index f2dae3134..000000000 --- a/server/middlewares/validators/user-history.ts +++ /dev/null @@ -1,47 +0,0 @@ -import express from 'express' -import { body, param, query } from 'express-validator' -import { exists, isDateValid, isIdValid } from '../../helpers/custom-validators/misc' -import { areValidationErrors } from './shared' - -const userHistoryListValidator = [ - query('search') - .optional() - .custom(exists), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const userHistoryRemoveAllValidator = [ - body('beforeDate') - .optional() - .custom(isDateValid).withMessage('Should have a before date that conforms to ISO 8601'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const userHistoryRemoveElementValidator = [ - param('videoId') - .custom(isIdValid), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - userHistoryListValidator, - userHistoryRemoveElementValidator, - userHistoryRemoveAllValidator -} diff --git a/server/middlewares/validators/user-notifications.ts b/server/middlewares/validators/user-notifications.ts deleted file mode 100644 index 8d70dcdd2..000000000 --- a/server/middlewares/validators/user-notifications.ts +++ /dev/null @@ -1,71 +0,0 @@ -import express from 'express' -import { body, query } from 'express-validator' -import { isNotEmptyIntArray, toBooleanOrNull } from '../../helpers/custom-validators/misc' -import { isUserNotificationSettingValid } from '../../helpers/custom-validators/user-notifications' -import { areValidationErrors } from './shared' - -const listUserNotificationsValidator = [ - query('unread') - .optional() - .customSanitizer(toBooleanOrNull) - .isBoolean().withMessage('Should have a valid unread boolean'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const updateNotificationSettingsValidator = [ - body('newVideoFromSubscription') - .custom(isUserNotificationSettingValid), - body('newCommentOnMyVideo') - .custom(isUserNotificationSettingValid), - body('abuseAsModerator') - .custom(isUserNotificationSettingValid), - body('videoAutoBlacklistAsModerator') - .custom(isUserNotificationSettingValid), - body('blacklistOnMyVideo') - .custom(isUserNotificationSettingValid), - body('myVideoImportFinished') - .custom(isUserNotificationSettingValid), - body('myVideoPublished') - .custom(isUserNotificationSettingValid), - body('commentMention') - .custom(isUserNotificationSettingValid), - body('newFollow') - .custom(isUserNotificationSettingValid), - body('newUserRegistration') - .custom(isUserNotificationSettingValid), - body('newInstanceFollower') - .custom(isUserNotificationSettingValid), - body('autoInstanceFollowing') - .custom(isUserNotificationSettingValid), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const markAsReadUserNotificationsValidator = [ - body('ids') - .optional() - .custom(isNotEmptyIntArray).withMessage('Should have a valid array of notification ids'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - listUserNotificationsValidator, - updateNotificationSettingsValidator, - markAsReadUserNotificationsValidator -} diff --git a/server/middlewares/validators/user-registrations.ts b/server/middlewares/validators/user-registrations.ts deleted file mode 100644 index 47397391b..000000000 --- a/server/middlewares/validators/user-registrations.ts +++ /dev/null @@ -1,208 +0,0 @@ -import express from 'express' -import { body, param, query, ValidationChain } from 'express-validator' -import { exists, isBooleanValid, isIdValid, toBooleanOrNull } from '@server/helpers/custom-validators/misc' -import { isRegistrationModerationResponseValid, isRegistrationReasonValid } from '@server/helpers/custom-validators/user-registration' -import { CONFIG } from '@server/initializers/config' -import { Hooks } from '@server/lib/plugins/hooks' -import { HttpStatusCode, UserRegister, UserRegistrationRequest, UserRegistrationState } from '@shared/models' -import { isUserDisplayNameValid, isUserPasswordValid, isUserUsernameValid } from '../../helpers/custom-validators/users' -import { isVideoChannelDisplayNameValid, isVideoChannelUsernameValid } from '../../helpers/custom-validators/video-channels' -import { isSignupAllowed, isSignupAllowedForCurrentIP, SignupMode } from '../../lib/signup' -import { ActorModel } from '../../models/actor/actor' -import { areValidationErrors, checkUserNameOrEmailDoNotAlreadyExist } from './shared' -import { checkRegistrationHandlesDoNotAlreadyExist, checkRegistrationIdExist } from './shared/user-registrations' - -const usersDirectRegistrationValidator = usersCommonRegistrationValidatorFactory() - -const usersRequestRegistrationValidator = [ - ...usersCommonRegistrationValidatorFactory([ - body('registrationReason') - .custom(isRegistrationReasonValid) - ]), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - const body: UserRegistrationRequest = req.body - - if (CONFIG.SIGNUP.REQUIRES_APPROVAL !== true) { - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Signup approval is not enabled on this instance' - }) - } - - const options = { username: body.username, email: body.email, channelHandle: body.channel?.name, res } - if (!await checkRegistrationHandlesDoNotAlreadyExist(options)) return - - return next() - } -] - -// --------------------------------------------------------------------------- - -function ensureUserRegistrationAllowedFactory (signupMode: SignupMode) { - return async (req: express.Request, res: express.Response, next: express.NextFunction) => { - const allowedParams = { - body: req.body, - ip: req.ip, - signupMode - } - - const allowedResult = await Hooks.wrapPromiseFun( - isSignupAllowed, - allowedParams, - - signupMode === 'direct-registration' - ? 'filter:api.user.signup.allowed.result' - : 'filter:api.user.request-signup.allowed.result' - ) - - if (allowedResult.allowed === false) { - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: allowedResult.errorMessage || 'User registration is not allowed' - }) - } - - return next() - } -} - -const ensureUserRegistrationAllowedForIP = [ - (req: express.Request, res: express.Response, next: express.NextFunction) => { - const allowed = isSignupAllowedForCurrentIP(req.ip) - - if (allowed === false) { - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'You are not on a network authorized for registration.' - }) - } - - return next() - } -] - -// --------------------------------------------------------------------------- - -const acceptOrRejectRegistrationValidator = [ - param('registrationId') - .custom(isIdValid), - - body('moderationResponse') - .custom(isRegistrationModerationResponseValid), - - body('preventEmailDelivery') - .optional() - .customSanitizer(toBooleanOrNull) - .custom(isBooleanValid).withMessage('Should have preventEmailDelivery boolean'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await checkRegistrationIdExist(req.params.registrationId, res)) return - - if (res.locals.userRegistration.state !== UserRegistrationState.PENDING) { - return res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: 'This registration is already accepted or rejected.' - }) - } - - return next() - } -] - -// --------------------------------------------------------------------------- - -const getRegistrationValidator = [ - param('registrationId') - .custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await checkRegistrationIdExist(req.params.registrationId, res)) return - - return next() - } -] - -// --------------------------------------------------------------------------- - -const listRegistrationsValidator = [ - query('search') - .optional() - .custom(exists), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - usersDirectRegistrationValidator, - usersRequestRegistrationValidator, - - ensureUserRegistrationAllowedFactory, - ensureUserRegistrationAllowedForIP, - - getRegistrationValidator, - listRegistrationsValidator, - - acceptOrRejectRegistrationValidator -} - -// --------------------------------------------------------------------------- - -function usersCommonRegistrationValidatorFactory (additionalValidationChain: ValidationChain[] = []) { - return [ - body('username') - .custom(isUserUsernameValid), - body('password') - .custom(isUserPasswordValid), - body('email') - .isEmail(), - body('displayName') - .optional() - .custom(isUserDisplayNameValid), - - body('channel.name') - .optional() - .custom(isVideoChannelUsernameValid), - body('channel.displayName') - .optional() - .custom(isVideoChannelDisplayNameValid), - - ...additionalValidationChain, - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res, { omitBodyLog: true })) return - - const body: UserRegister | UserRegistrationRequest = req.body - - if (!await checkUserNameOrEmailDoNotAlreadyExist(body.username, body.email, res)) return - - if (body.channel) { - if (!body.channel.name || !body.channel.displayName) { - return res.fail({ message: 'Channel is optional but if you specify it, channel.name and channel.displayName are required.' }) - } - - if (body.channel.name === body.username) { - return res.fail({ message: 'Channel name cannot be the same as user username.' }) - } - - const existing = await ActorModel.loadLocalByName(body.channel.name) - if (existing) { - return res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: `Channel with name ${body.channel.name} already exists.` - }) - } - } - - return next() - } - ] -} diff --git a/server/middlewares/validators/user-subscriptions.ts b/server/middlewares/validators/user-subscriptions.ts deleted file mode 100644 index 68d83add5..000000000 --- a/server/middlewares/validators/user-subscriptions.ts +++ /dev/null @@ -1,111 +0,0 @@ -import express from 'express' -import { body, param, query } from 'express-validator' -import { arrayify } from '@shared/core-utils' -import { FollowState } from '@shared/models' -import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' -import { areValidActorHandles, isValidActorHandle } from '../../helpers/custom-validators/activitypub/actor' -import { WEBSERVER } from '../../initializers/constants' -import { ActorFollowModel } from '../../models/actor/actor-follow' -import { areValidationErrors } from './shared' - -const userSubscriptionListValidator = [ - query('search') - .optional() - .not().isEmpty(), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const userSubscriptionAddValidator = [ - body('uri') - .custom(isValidActorHandle).withMessage('Should have a valid URI to follow (username@domain)'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const areSubscriptionsExistValidator = [ - query('uris') - .customSanitizer(arrayify) - .custom(areValidActorHandles).withMessage('Should have a valid array of URIs'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const userSubscriptionGetValidator = [ - param('uri') - .custom(isValidActorHandle), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesSubscriptionExist({ uri: req.params.uri, res, state: 'accepted' })) return - - return next() - } -] - -const userSubscriptionDeleteValidator = [ - param('uri') - .custom(isValidActorHandle), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesSubscriptionExist({ uri: req.params.uri, res })) return - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - areSubscriptionsExistValidator, - userSubscriptionListValidator, - userSubscriptionAddValidator, - userSubscriptionGetValidator, - userSubscriptionDeleteValidator -} - -// --------------------------------------------------------------------------- - -async function doesSubscriptionExist (options: { - uri: string - res: express.Response - state?: FollowState -}) { - const { uri, res, state } = options - - let [ name, host ] = uri.split('@') - if (host === WEBSERVER.HOST) host = null - - const user = res.locals.oauth.token.User - const subscription = await ActorFollowModel.loadByActorAndTargetNameAndHostForAPI({ - actorId: user.Account.Actor.id, - targetName: name, - targetHost: host, - state - }) - - if (!subscription?.ActorFollowing.VideoChannel) { - res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: `Subscription ${uri} not found.` - }) - return false - } - - res.locals.subscription = subscription - - return true -} diff --git a/server/middlewares/validators/users.ts b/server/middlewares/validators/users.ts deleted file mode 100644 index 3d311b15b..000000000 --- a/server/middlewares/validators/users.ts +++ /dev/null @@ -1,489 +0,0 @@ -import express from 'express' -import { body, param, query } from 'express-validator' -import { forceNumber } from '@shared/core-utils' -import { HttpStatusCode, UserRight, UserRole } from '@shared/models' -import { exists, isBooleanValid, isIdValid, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc' -import { isThemeNameValid } from '../../helpers/custom-validators/plugins' -import { - isUserAdminFlagsValid, - isUserAutoPlayNextVideoValid, - isUserAutoPlayVideoValid, - isUserBlockedReasonValid, - isUserDescriptionValid, - isUserDisplayNameValid, - isUserEmailPublicValid, - isUserNoModal, - isUserNSFWPolicyValid, - isUserP2PEnabledValid, - isUserPasswordValid, - isUserPasswordValidOrEmpty, - isUserRoleValid, - isUserUsernameValid, - isUserVideoLanguages, - isUserVideoQuotaDailyValid, - isUserVideoQuotaValid, - isUserVideosHistoryEnabledValid -} from '../../helpers/custom-validators/users' -import { isVideoChannelUsernameValid } from '../../helpers/custom-validators/video-channels' -import { logger } from '../../helpers/logger' -import { isThemeRegistered } from '../../lib/plugins/theme-utils' -import { Redis } from '../../lib/redis' -import { ActorModel } from '../../models/actor/actor' -import { - areValidationErrors, - checkUserEmailExist, - checkUserIdExist, - checkUserNameOrEmailDoNotAlreadyExist, - doesVideoChannelIdExist, - doesVideoExist, - isValidVideoIdParam -} from './shared' - -const usersListValidator = [ - query('blocked') - .optional() - .customSanitizer(toBooleanOrNull) - .isBoolean().withMessage('Should be a valid blocked boolean'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const usersAddValidator = [ - body('username') - .custom(isUserUsernameValid) - .withMessage('Should have a valid username (lowercase alphanumeric characters)'), - body('password') - .custom(isUserPasswordValidOrEmpty), - body('email') - .isEmail(), - - body('channelName') - .optional() - .custom(isVideoChannelUsernameValid), - - body('videoQuota') - .optional() - .custom(isUserVideoQuotaValid), - - body('videoQuotaDaily') - .optional() - .custom(isUserVideoQuotaDailyValid), - - body('role') - .customSanitizer(toIntOrNull) - .custom(isUserRoleValid), - - body('adminFlags') - .optional() - .custom(isUserAdminFlagsValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res, { omitBodyLog: true })) return - if (!await checkUserNameOrEmailDoNotAlreadyExist(req.body.username, req.body.email, res)) return - - const authUser = res.locals.oauth.token.User - if (authUser.role !== UserRole.ADMINISTRATOR && req.body.role !== UserRole.USER) { - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'You can only create users (and not administrators or moderators)' - }) - } - - if (req.body.channelName) { - if (req.body.channelName === req.body.username) { - return res.fail({ message: 'Channel name cannot be the same as user username.' }) - } - - const existing = await ActorModel.loadLocalByName(req.body.channelName) - if (existing) { - return res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: `Channel with name ${req.body.channelName} already exists.` - }) - } - } - - return next() - } -] - -const usersRemoveValidator = [ - param('id') - .custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await checkUserIdExist(req.params.id, res)) return - - const user = res.locals.user - if (user.username === 'root') { - return res.fail({ message: 'Cannot remove the root user' }) - } - - return next() - } -] - -const usersBlockingValidator = [ - param('id') - .custom(isIdValid), - body('reason') - .optional() - .custom(isUserBlockedReasonValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await checkUserIdExist(req.params.id, res)) return - - const user = res.locals.user - if (user.username === 'root') { - return res.fail({ message: 'Cannot block the root user' }) - } - - return next() - } -] - -const deleteMeValidator = [ - (req: express.Request, res: express.Response, next: express.NextFunction) => { - const user = res.locals.oauth.token.User - if (user.username === 'root') { - return res.fail({ message: 'You cannot delete your root account.' }) - } - - return next() - } -] - -const usersUpdateValidator = [ - param('id').custom(isIdValid), - - body('password') - .optional() - .custom(isUserPasswordValid), - body('email') - .optional() - .isEmail(), - body('emailVerified') - .optional() - .isBoolean(), - body('videoQuota') - .optional() - .custom(isUserVideoQuotaValid), - body('videoQuotaDaily') - .optional() - .custom(isUserVideoQuotaDailyValid), - body('pluginAuth') - .optional() - .exists(), - body('role') - .optional() - .customSanitizer(toIntOrNull) - .custom(isUserRoleValid), - body('adminFlags') - .optional() - .custom(isUserAdminFlagsValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res, { omitBodyLog: true })) return - if (!await checkUserIdExist(req.params.id, res)) return - - const user = res.locals.user - if (user.username === 'root' && req.body.role !== undefined && user.role !== req.body.role) { - return res.fail({ message: 'Cannot change root role.' }) - } - - return next() - } -] - -const usersUpdateMeValidator = [ - body('displayName') - .optional() - .custom(isUserDisplayNameValid), - body('description') - .optional() - .custom(isUserDescriptionValid), - body('currentPassword') - .optional() - .custom(isUserPasswordValid), - body('password') - .optional() - .custom(isUserPasswordValid), - body('emailPublic') - .optional() - .custom(isUserEmailPublicValid), - body('email') - .optional() - .isEmail(), - body('nsfwPolicy') - .optional() - .custom(isUserNSFWPolicyValid), - body('autoPlayVideo') - .optional() - .custom(isUserAutoPlayVideoValid), - body('p2pEnabled') - .optional() - .custom(isUserP2PEnabledValid).withMessage('Should have a valid p2p enabled boolean'), - body('videoLanguages') - .optional() - .custom(isUserVideoLanguages), - body('videosHistoryEnabled') - .optional() - .custom(isUserVideosHistoryEnabledValid).withMessage('Should have a valid videos history enabled boolean'), - body('theme') - .optional() - .custom(v => isThemeNameValid(v) && isThemeRegistered(v)), - - body('noInstanceConfigWarningModal') - .optional() - .custom(v => isUserNoModal(v)).withMessage('Should have a valid noInstanceConfigWarningModal boolean'), - body('noWelcomeModal') - .optional() - .custom(v => isUserNoModal(v)).withMessage('Should have a valid noWelcomeModal boolean'), - body('noAccountSetupWarningModal') - .optional() - .custom(v => isUserNoModal(v)).withMessage('Should have a valid noAccountSetupWarningModal boolean'), - - body('autoPlayNextVideo') - .optional() - .custom(v => isUserAutoPlayNextVideoValid(v)).withMessage('Should have a valid autoPlayNextVideo boolean'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - const user = res.locals.oauth.token.User - - if (req.body.password || req.body.email) { - if (user.pluginAuth !== null) { - return res.fail({ message: 'You cannot update your email or password that is associated with an external auth system.' }) - } - - if (!req.body.currentPassword) { - return res.fail({ message: 'currentPassword parameter is missing.' }) - } - - if (await user.isPasswordMatch(req.body.currentPassword) !== true) { - return res.fail({ - status: HttpStatusCode.UNAUTHORIZED_401, - message: 'currentPassword is invalid.' - }) - } - } - - if (areValidationErrors(req, res, { omitBodyLog: true })) return - - return next() - } -] - -const usersGetValidator = [ - param('id') - .custom(isIdValid), - query('withStats') - .optional() - .isBoolean().withMessage('Should have a valid withStats boolean'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await checkUserIdExist(req.params.id, res, req.query.withStats)) return - - return next() - } -] - -const usersVideoRatingValidator = [ - isValidVideoIdParam('videoId'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.videoId, res, 'id')) return - - return next() - } -] - -const usersVideosValidator = [ - query('isLive') - .optional() - .customSanitizer(toBooleanOrNull) - .custom(isBooleanValid).withMessage('Should have a valid isLive boolean'), - - query('channelId') - .optional() - .customSanitizer(toIntOrNull) - .custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - if (req.query.channelId && !await doesVideoChannelIdExist(req.query.channelId, res)) return - - return next() - } -] - -const usersAskResetPasswordValidator = [ - body('email') - .isEmail(), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const exists = await checkUserEmailExist(req.body.email, res, false) - if (!exists) { - logger.debug('User with email %s does not exist (asking reset password).', req.body.email) - // Do not leak our emails - return res.status(HttpStatusCode.NO_CONTENT_204).end() - } - - if (res.locals.user.pluginAuth) { - return res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: 'Cannot recover password of a user that uses a plugin authentication.' - }) - } - - return next() - } -] - -const usersResetPasswordValidator = [ - param('id') - .custom(isIdValid), - body('verificationString') - .not().isEmpty(), - body('password') - .custom(isUserPasswordValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await checkUserIdExist(req.params.id, res)) return - - const user = res.locals.user - const redisVerificationString = await Redis.Instance.getResetPasswordVerificationString(user.id) - - if (redisVerificationString !== req.body.verificationString) { - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Invalid verification string.' - }) - } - - return next() - } -] - -const usersCheckCurrentPasswordFactory = (targetUserIdGetter: (req: express.Request) => number | string) => { - return [ - body('currentPassword').optional().custom(exists), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const user = res.locals.oauth.token.User - const isAdminOrModerator = user.role === UserRole.ADMINISTRATOR || user.role === UserRole.MODERATOR - const targetUserId = forceNumber(targetUserIdGetter(req)) - - // Admin/moderator action on another user, skip the password check - if (isAdminOrModerator && targetUserId !== user.id) { - return next() - } - - if (!req.body.currentPassword) { - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'currentPassword is missing' - }) - } - - if (await user.isPasswordMatch(req.body.currentPassword) !== true) { - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'currentPassword is invalid.' - }) - } - - return next() - } - ] -} - -const userAutocompleteValidator = [ - param('search') - .isString() - .not().isEmpty() -] - -const ensureAuthUserOwnsAccountValidator = [ - (req: express.Request, res: express.Response, next: express.NextFunction) => { - const user = res.locals.oauth.token.User - - if (res.locals.account.id !== user.Account.id) { - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Only owner of this account can access this resource.' - }) - } - - return next() - } -] - -const ensureCanManageChannelOrAccount = [ - (req: express.Request, res: express.Response, next: express.NextFunction) => { - const user = res.locals.oauth.token.user - const account = res.locals.videoChannel?.Account ?? res.locals.account - const isUserOwner = account.userId === user.id - - if (!isUserOwner && user.hasRight(UserRight.MANAGE_ANY_VIDEO_CHANNEL) === false) { - const message = `User ${user.username} does not have right this channel or account.` - - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message - }) - } - - return next() - } -] - -const ensureCanModerateUser = [ - (req: express.Request, res: express.Response, next: express.NextFunction) => { - const authUser = res.locals.oauth.token.User - const onUser = res.locals.user - - if (authUser.role === UserRole.ADMINISTRATOR) return next() - if (authUser.role === UserRole.MODERATOR && onUser.role === UserRole.USER) return next() - - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'A moderator can only manage users.' - }) - } -] - -// --------------------------------------------------------------------------- - -export { - usersListValidator, - usersAddValidator, - deleteMeValidator, - usersBlockingValidator, - usersRemoveValidator, - usersUpdateValidator, - usersUpdateMeValidator, - usersVideoRatingValidator, - usersCheckCurrentPasswordFactory, - usersGetValidator, - usersVideosValidator, - usersAskResetPasswordValidator, - usersResetPasswordValidator, - userAutocompleteValidator, - ensureAuthUserOwnsAccountValidator, - ensureCanModerateUser, - ensureCanManageChannelOrAccount -} diff --git a/server/middlewares/validators/videos/index.ts b/server/middlewares/validators/videos/index.ts deleted file mode 100644 index 8c6fc49b1..000000000 --- a/server/middlewares/validators/videos/index.ts +++ /dev/null @@ -1,19 +0,0 @@ -export * from './video-blacklist' -export * from './video-captions' -export * from './video-channel-sync' -export * from './video-channels' -export * from './video-comments' -export * from './video-files' -export * from './video-imports' -export * from './video-live' -export * from './video-ownership-changes' -export * from './video-passwords' -export * from './video-rates' -export * from './video-shares' -export * from './video-source' -export * from './video-stats' -export * from './video-studio' -export * from './video-token' -export * from './video-transcoding' -export * from './video-view' -export * from './videos' diff --git a/server/middlewares/validators/videos/shared/index.ts b/server/middlewares/validators/videos/shared/index.ts deleted file mode 100644 index eb11dcc6a..000000000 --- a/server/middlewares/validators/videos/shared/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './upload' -export * from './video-validators' diff --git a/server/middlewares/validators/videos/shared/upload.ts b/server/middlewares/validators/videos/shared/upload.ts deleted file mode 100644 index ea0dddc3c..000000000 --- a/server/middlewares/validators/videos/shared/upload.ts +++ /dev/null @@ -1,39 +0,0 @@ -import express from 'express' -import { logger } from '@server/helpers/logger' -import { getVideoStreamDuration } from '@shared/ffmpeg' -import { HttpStatusCode } from '@shared/models' - -export async function addDurationToVideoFileIfNeeded (options: { - res: express.Response - videoFile: { path: string, duration?: number } - middlewareName: string -}) { - const { res, middlewareName, videoFile } = options - - try { - if (!videoFile.duration) await addDurationToVideo(videoFile) - } catch (err) { - logger.error('Invalid input file in ' + middlewareName, { err }) - - res.fail({ - status: HttpStatusCode.UNPROCESSABLE_ENTITY_422, - message: 'Video file unreadable.' - }) - return false - } - - return true -} - -// --------------------------------------------------------------------------- -// Private -// --------------------------------------------------------------------------- - -async function addDurationToVideo (videoFile: { path: string, duration?: number }) { - const duration = await getVideoStreamDuration(videoFile.path) - - // FFmpeg may not be able to guess video duration - // For example with m2v files: https://trac.ffmpeg.org/ticket/9726#comment:2 - if (isNaN(duration)) videoFile.duration = 0 - else videoFile.duration = duration -} diff --git a/server/middlewares/validators/videos/shared/video-validators.ts b/server/middlewares/validators/videos/shared/video-validators.ts deleted file mode 100644 index 95e4fef11..000000000 --- a/server/middlewares/validators/videos/shared/video-validators.ts +++ /dev/null @@ -1,100 +0,0 @@ -import express from 'express' -import { isVideoFileMimeTypeValid, isVideoFileSizeValid } from '@server/helpers/custom-validators/videos' -import { logger } from '@server/helpers/logger' -import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' -import { isLocalVideoFileAccepted } from '@server/lib/moderation' -import { Hooks } from '@server/lib/plugins/hooks' -import { MUserAccountId, MVideo } from '@server/types/models' -import { HttpStatusCode, ServerErrorCode, ServerFilterHookName, VideoState } from '@shared/models' -import { checkUserQuota } from '../../shared' - -export async function commonVideoFileChecks (options: { - res: express.Response - user: MUserAccountId - videoFileSize: number - files: express.UploadFilesForCheck -}): Promise { - const { res, user, videoFileSize, files } = options - - if (!isVideoFileMimeTypeValid(files)) { - res.fail({ - status: HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415, - message: 'This file is not supported. Please, make sure it is of the following type: ' + - CONSTRAINTS_FIELDS.VIDEOS.EXTNAME.join(', ') - }) - return false - } - - if (!isVideoFileSizeValid(videoFileSize.toString())) { - res.fail({ - status: HttpStatusCode.PAYLOAD_TOO_LARGE_413, - message: 'This file is too large. It exceeds the maximum file size authorized.', - type: ServerErrorCode.MAX_FILE_SIZE_REACHED - }) - return false - } - - if (await checkUserQuota(user, videoFileSize, res) === false) return false - - return true -} - -export async function isVideoFileAccepted (options: { - req: express.Request - res: express.Response - videoFile: express.VideoUploadFile - hook: Extract -}) { - const { req, res, videoFile, hook } = options - - // Check we accept this video - const acceptParameters = { - videoBody: req.body, - videoFile, - user: res.locals.oauth.token.User - } - const acceptedResult = await Hooks.wrapFun(isLocalVideoFileAccepted, acceptParameters, hook) - - if (!acceptedResult || acceptedResult.accepted !== true) { - logger.info('Refused local video file.', { acceptedResult, acceptParameters }) - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: acceptedResult.errorMessage || 'Refused local video file' - }) - return false - } - - return true -} - -export function checkVideoFileCanBeEdited (video: MVideo, res: express.Response) { - if (video.isLive) { - res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Cannot edit a live video' - }) - - return false - } - - if (video.state === VideoState.TO_TRANSCODE || video.state === VideoState.TO_EDIT) { - res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: 'Cannot edit video that is already waiting for transcoding/edition' - }) - - return false - } - - const validStates = new Set([ VideoState.PUBLISHED, VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED, VideoState.TRANSCODING_FAILED ]) - if (!validStates.has(video.state)) { - res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Video state is not compatible with edition' - }) - - return false - } - - return true -} diff --git a/server/middlewares/validators/videos/video-blacklist.ts b/server/middlewares/validators/videos/video-blacklist.ts deleted file mode 100644 index 6b9aea07c..000000000 --- a/server/middlewares/validators/videos/video-blacklist.ts +++ /dev/null @@ -1,87 +0,0 @@ -import express from 'express' -import { body, query } from 'express-validator' -import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' -import { isBooleanValid, toBooleanOrNull, toIntOrNull } from '../../../helpers/custom-validators/misc' -import { isVideoBlacklistReasonValid, isVideoBlacklistTypeValid } from '../../../helpers/custom-validators/video-blacklist' -import { areValidationErrors, doesVideoBlacklistExist, doesVideoExist, isValidVideoIdParam } from '../shared' - -const videosBlacklistRemoveValidator = [ - isValidVideoIdParam('videoId'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.videoId, res)) return - if (!await doesVideoBlacklistExist(res.locals.videoAll.id, res)) return - - return next() - } -] - -const videosBlacklistAddValidator = [ - isValidVideoIdParam('videoId'), - - body('unfederate') - .optional() - .customSanitizer(toBooleanOrNull) - .custom(isBooleanValid).withMessage('Should have a valid unfederate boolean'), - body('reason') - .optional() - .custom(isVideoBlacklistReasonValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.videoId, res)) return - - const video = res.locals.videoAll - if (req.body.unfederate === true && video.remote === true) { - return res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: 'You cannot unfederate a remote video.' - }) - } - - return next() - } -] - -const videosBlacklistUpdateValidator = [ - isValidVideoIdParam('videoId'), - - body('reason') - .optional() - .custom(isVideoBlacklistReasonValid).withMessage('Should have a valid reason'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.videoId, res)) return - if (!await doesVideoBlacklistExist(res.locals.videoAll.id, res)) return - - return next() - } -] - -const videosBlacklistFiltersValidator = [ - query('type') - .optional() - .customSanitizer(toIntOrNull) - .custom(isVideoBlacklistTypeValid).withMessage('Should have a valid video blacklist type attribute'), - query('search') - .optional() - .not() - .isEmpty().withMessage('Should have a valid search'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - videosBlacklistAddValidator, - videosBlacklistRemoveValidator, - videosBlacklistUpdateValidator, - videosBlacklistFiltersValidator -} diff --git a/server/middlewares/validators/videos/video-captions.ts b/server/middlewares/validators/videos/video-captions.ts deleted file mode 100644 index 077a58d2e..000000000 --- a/server/middlewares/validators/videos/video-captions.ts +++ /dev/null @@ -1,83 +0,0 @@ -import express from 'express' -import { body, param } from 'express-validator' -import { UserRight } from '@shared/models' -import { isVideoCaptionFile, isVideoCaptionLanguageValid } from '../../../helpers/custom-validators/video-captions' -import { cleanUpReqFiles } from '../../../helpers/express-utils' -import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../../initializers/constants' -import { - areValidationErrors, - checkCanSeeVideo, - checkUserCanManageVideo, - doesVideoCaptionExist, - doesVideoExist, - isValidVideoIdParam, - isValidVideoPasswordHeader -} from '../shared' - -const addVideoCaptionValidator = [ - isValidVideoIdParam('videoId'), - - param('captionLanguage') - .custom(isVideoCaptionLanguageValid).not().isEmpty(), - - body('captionfile') - .custom((_, { req }) => isVideoCaptionFile(req.files, 'captionfile')) - .withMessage( - 'This caption file is not supported or too large. ' + - `Please, make sure it is under ${CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max} bytes ` + - 'and one of the following mimetypes: ' + - Object.keys(MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT).map(key => `${key} (${MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT[key]})`).join(', ') - ), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return cleanUpReqFiles(req) - if (!await doesVideoExist(req.params.videoId, res)) return cleanUpReqFiles(req) - - // Check if the user who did the request is able to update the video - const user = res.locals.oauth.token.User - if (!checkUserCanManageVideo(user, res.locals.videoAll, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req) - - return next() - } -] - -const deleteVideoCaptionValidator = [ - isValidVideoIdParam('videoId'), - - param('captionLanguage') - .custom(isVideoCaptionLanguageValid).not().isEmpty().withMessage('Should have a valid caption language'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.videoId, res)) return - if (!await doesVideoCaptionExist(res.locals.videoAll, req.params.captionLanguage, res)) return - - // Check if the user who did the request is able to update the video - const user = res.locals.oauth.token.User - if (!checkUserCanManageVideo(user, res.locals.videoAll, UserRight.UPDATE_ANY_VIDEO, res)) return - - return next() - } -] - -const listVideoCaptionsValidator = [ - isValidVideoIdParam('videoId'), - - isValidVideoPasswordHeader(), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.videoId, res, 'only-video')) return - - const video = res.locals.onlyVideo - if (!await checkCanSeeVideo({ req, res, video, paramId: req.params.videoId })) return - - return next() - } -] - -export { - addVideoCaptionValidator, - listVideoCaptionsValidator, - deleteVideoCaptionValidator -} diff --git a/server/middlewares/validators/videos/video-channel-sync.ts b/server/middlewares/validators/videos/video-channel-sync.ts deleted file mode 100644 index 7e5b12471..000000000 --- a/server/middlewares/validators/videos/video-channel-sync.ts +++ /dev/null @@ -1,56 +0,0 @@ -import * as express from 'express' -import { body, param } from 'express-validator' -import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc' -import { CONFIG } from '@server/initializers/config' -import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' -import { HttpStatusCode, VideoChannelSyncCreate } from '@shared/models' -import { areValidationErrors, doesVideoChannelIdExist } from '../shared' -import { doesVideoChannelSyncIdExist } from '../shared/video-channel-syncs' - -export const ensureSyncIsEnabled = (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) { - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Synchronization is impossible as video channel synchronization is not enabled on the server' - }) - } - - return next() -} - -export const videoChannelSyncValidator = [ - body('externalChannelUrl') - .custom(isUrlValid), - - body('videoChannelId') - .isInt(), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const body: VideoChannelSyncCreate = req.body - if (!await doesVideoChannelIdExist(body.videoChannelId, res)) return - - const count = await VideoChannelSyncModel.countByAccount(res.locals.videoChannel.accountId) - if (count >= CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER) { - return res.fail({ - message: `You cannot create more than ${CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER} channel synchronizations` - }) - } - - return next() - } -] - -export const ensureSyncExists = [ - param('id').exists().isInt().withMessage('Should have an sync id'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - if (!await doesVideoChannelSyncIdExist(+req.params.id, res)) return - if (!await doesVideoChannelIdExist(res.locals.videoChannelSync.videoChannelId, res)) return - - return next() - } -] diff --git a/server/middlewares/validators/videos/video-channels.ts b/server/middlewares/validators/videos/video-channels.ts deleted file mode 100644 index ca6b57003..000000000 --- a/server/middlewares/validators/videos/video-channels.ts +++ /dev/null @@ -1,194 +0,0 @@ -import express from 'express' -import { body, param, query } from 'express-validator' -import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc' -import { CONFIG } from '@server/initializers/config' -import { MChannelAccountDefault } from '@server/types/models' -import { VideosImportInChannelCreate } from '@shared/models' -import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' -import { isBooleanValid, isIdValid, toBooleanOrNull } from '../../../helpers/custom-validators/misc' -import { - isVideoChannelDescriptionValid, - isVideoChannelDisplayNameValid, - isVideoChannelSupportValid, - isVideoChannelUsernameValid -} from '../../../helpers/custom-validators/video-channels' -import { ActorModel } from '../../../models/actor/actor' -import { VideoChannelModel } from '../../../models/video/video-channel' -import { areValidationErrors, checkUserQuota, doesVideoChannelNameWithHostExist } from '../shared' -import { doesVideoChannelSyncIdExist } from '../shared/video-channel-syncs' - -export const videoChannelsAddValidator = [ - body('name') - .custom(isVideoChannelUsernameValid), - body('displayName') - .custom(isVideoChannelDisplayNameValid), - body('description') - .optional() - .custom(isVideoChannelDescriptionValid), - body('support') - .optional() - .custom(isVideoChannelSupportValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const actor = await ActorModel.loadLocalByName(req.body.name) - if (actor) { - res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: 'Another actor (account/channel) with this name on this instance already exists or has already existed.' - }) - return false - } - - const count = await VideoChannelModel.countByAccount(res.locals.oauth.token.User.Account.id) - if (count >= CONFIG.VIDEO_CHANNELS.MAX_PER_USER) { - res.fail({ message: `You cannot create more than ${CONFIG.VIDEO_CHANNELS.MAX_PER_USER} channels` }) - return false - } - - return next() - } -] - -export const videoChannelsUpdateValidator = [ - param('nameWithHost') - .exists(), - - body('displayName') - .optional() - .custom(isVideoChannelDisplayNameValid), - body('description') - .optional() - .custom(isVideoChannelDescriptionValid), - body('support') - .optional() - .custom(isVideoChannelSupportValid), - body('bulkVideosSupportUpdate') - .optional() - .custom(isBooleanValid).withMessage('Should have a valid bulkVideosSupportUpdate boolean field'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -export const videoChannelsRemoveValidator = [ - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (!await checkVideoChannelIsNotTheLastOne(res.locals.videoChannel, res)) return - - return next() - } -] - -export const videoChannelsNameWithHostValidator = [ - param('nameWithHost') - .exists(), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - if (!await doesVideoChannelNameWithHostExist(req.params.nameWithHost, res)) return - - return next() - } -] - -export const ensureIsLocalChannel = [ - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (res.locals.videoChannel.Actor.isOwned() === false) { - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'This channel is not owned.' - }) - } - - return next() - } -] - -export const ensureChannelOwnerCanUpload = [ - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - const channel = res.locals.videoChannel - const user = { id: channel.Account.userId } - - if (!await checkUserQuota(user, 1, res)) return - - next() - } -] - -export const videoChannelStatsValidator = [ - query('withStats') - .optional() - .customSanitizer(toBooleanOrNull) - .custom(isBooleanValid).withMessage('Should have a valid stats flag boolean'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - return next() - } -] - -export const videoChannelsListValidator = [ - query('search') - .optional() - .not().isEmpty(), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -export const videoChannelImportVideosValidator = [ - body('externalChannelUrl') - .custom(isUrlValid), - - body('videoChannelSyncId') - .optional() - .custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const body: VideosImportInChannelCreate = req.body - - if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) { - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Channel import is impossible as video upload via HTTP is not enabled on the server' - }) - } - - if (body.videoChannelSyncId && !await doesVideoChannelSyncIdExist(body.videoChannelSyncId, res)) return - - if (res.locals.videoChannelSync && res.locals.videoChannelSync.videoChannelId !== res.locals.videoChannel.id) { - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'This channel sync is not owned by this channel' - }) - } - - return next() - } -] - -// --------------------------------------------------------------------------- - -async function checkVideoChannelIsNotTheLastOne (videoChannel: MChannelAccountDefault, res: express.Response) { - const count = await VideoChannelModel.countByAccount(videoChannel.Account.id) - - if (count <= 1) { - res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: 'Cannot remove the last channel of this user' - }) - return false - } - - return true -} diff --git a/server/middlewares/validators/videos/video-comments.ts b/server/middlewares/validators/videos/video-comments.ts deleted file mode 100644 index 70689b02e..000000000 --- a/server/middlewares/validators/videos/video-comments.ts +++ /dev/null @@ -1,249 +0,0 @@ -import express from 'express' -import { body, param, query } from 'express-validator' -import { MUserAccountUrl } from '@server/types/models' -import { HttpStatusCode, UserRight } from '@shared/models' -import { exists, isBooleanValid, isIdValid, toBooleanOrNull } from '../../../helpers/custom-validators/misc' -import { isValidVideoCommentText } from '../../../helpers/custom-validators/video-comments' -import { logger } from '../../../helpers/logger' -import { AcceptResult, isLocalVideoCommentReplyAccepted, isLocalVideoThreadAccepted } from '../../../lib/moderation' -import { Hooks } from '../../../lib/plugins/hooks' -import { MCommentOwnerVideoReply, MVideo, MVideoFullLight } from '../../../types/models/video' -import { - areValidationErrors, - checkCanSeeVideo, - doesVideoCommentExist, - doesVideoCommentThreadExist, - doesVideoExist, - isValidVideoIdParam, - isValidVideoPasswordHeader -} from '../shared' - -const listVideoCommentsValidator = [ - query('isLocal') - .optional() - .customSanitizer(toBooleanOrNull) - .custom(isBooleanValid) - .withMessage('Should have a valid isLocal boolean'), - - query('onLocalVideo') - .optional() - .customSanitizer(toBooleanOrNull) - .custom(isBooleanValid) - .withMessage('Should have a valid onLocalVideo boolean'), - - query('search') - .optional() - .custom(exists), - - query('searchAccount') - .optional() - .custom(exists), - - query('searchVideo') - .optional() - .custom(exists), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const listVideoCommentThreadsValidator = [ - isValidVideoIdParam('videoId'), - isValidVideoPasswordHeader(), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.videoId, res, 'only-video')) return - - if (!await checkCanSeeVideo({ req, res, paramId: req.params.videoId, video: res.locals.onlyVideo })) return - - return next() - } -] - -const listVideoThreadCommentsValidator = [ - isValidVideoIdParam('videoId'), - - param('threadId') - .custom(isIdValid), - isValidVideoPasswordHeader(), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.videoId, res, 'only-video')) return - if (!await doesVideoCommentThreadExist(req.params.threadId, res.locals.onlyVideo, res)) return - - if (!await checkCanSeeVideo({ req, res, paramId: req.params.videoId, video: res.locals.onlyVideo })) return - - return next() - } -] - -const addVideoCommentThreadValidator = [ - isValidVideoIdParam('videoId'), - - body('text') - .custom(isValidVideoCommentText), - isValidVideoPasswordHeader(), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.videoId, res)) return - - if (!await checkCanSeeVideo({ req, res, paramId: req.params.videoId, video: res.locals.videoAll })) return - - if (!isVideoCommentsEnabled(res.locals.videoAll, res)) return - if (!await isVideoCommentAccepted(req, res, res.locals.videoAll, false)) return - - return next() - } -] - -const addVideoCommentReplyValidator = [ - isValidVideoIdParam('videoId'), - - param('commentId').custom(isIdValid), - isValidVideoPasswordHeader(), - - body('text').custom(isValidVideoCommentText), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.videoId, res)) return - - if (!await checkCanSeeVideo({ req, res, paramId: req.params.videoId, video: res.locals.videoAll })) return - - if (!isVideoCommentsEnabled(res.locals.videoAll, res)) return - if (!await doesVideoCommentExist(req.params.commentId, res.locals.videoAll, res)) return - if (!await isVideoCommentAccepted(req, res, res.locals.videoAll, true)) return - - return next() - } -] - -const videoCommentGetValidator = [ - isValidVideoIdParam('videoId'), - - param('commentId') - .custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.videoId, res, 'id')) return - if (!await doesVideoCommentExist(req.params.commentId, res.locals.videoId, res)) return - - return next() - } -] - -const removeVideoCommentValidator = [ - isValidVideoIdParam('videoId'), - - param('commentId') - .custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.videoId, res)) return - if (!await doesVideoCommentExist(req.params.commentId, res.locals.videoAll, res)) return - - // Check if the user who did the request is able to delete the video - if (!checkUserCanDeleteVideoComment(res.locals.oauth.token.User, res.locals.videoCommentFull, res)) return - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - listVideoCommentThreadsValidator, - listVideoThreadCommentsValidator, - addVideoCommentThreadValidator, - listVideoCommentsValidator, - addVideoCommentReplyValidator, - videoCommentGetValidator, - removeVideoCommentValidator -} - -// --------------------------------------------------------------------------- - -function isVideoCommentsEnabled (video: MVideo, res: express.Response) { - if (video.commentsEnabled !== true) { - res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: 'Video comments are disabled for this video.' - }) - return false - } - - return true -} - -function checkUserCanDeleteVideoComment (user: MUserAccountUrl, videoComment: MCommentOwnerVideoReply, res: express.Response) { - if (videoComment.isDeleted()) { - res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: 'This comment is already deleted' - }) - return false - } - - const userAccount = user.Account - - if ( - user.hasRight(UserRight.REMOVE_ANY_VIDEO_COMMENT) === false && // Not a moderator - videoComment.accountId !== userAccount.id && // Not the comment owner - videoComment.Video.VideoChannel.accountId !== userAccount.id // Not the video owner - ) { - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Cannot remove video comment of another user' - }) - return false - } - - return true -} - -async function isVideoCommentAccepted (req: express.Request, res: express.Response, video: MVideoFullLight, isReply: boolean) { - const acceptParameters = { - video, - commentBody: req.body, - user: res.locals.oauth.token.User, - req - } - - let acceptedResult: AcceptResult - - if (isReply) { - const acceptReplyParameters = Object.assign(acceptParameters, { parentComment: res.locals.videoCommentFull }) - - acceptedResult = await Hooks.wrapFun( - isLocalVideoCommentReplyAccepted, - acceptReplyParameters, - 'filter:api.video-comment-reply.create.accept.result' - ) - } else { - acceptedResult = await Hooks.wrapFun( - isLocalVideoThreadAccepted, - acceptParameters, - 'filter:api.video-thread.create.accept.result' - ) - } - - if (!acceptedResult || acceptedResult.accepted !== true) { - logger.info('Refused local comment.', { acceptedResult, acceptParameters }) - - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: acceptedResult?.errorMessage || 'Comment has been rejected.' - }) - return false - } - - return true -} diff --git a/server/middlewares/validators/videos/video-files.ts b/server/middlewares/validators/videos/video-files.ts deleted file mode 100644 index 6c0ecda42..000000000 --- a/server/middlewares/validators/videos/video-files.ts +++ /dev/null @@ -1,163 +0,0 @@ -import express from 'express' -import { param } from 'express-validator' -import { isIdValid } from '@server/helpers/custom-validators/misc' -import { MVideo } from '@server/types/models' -import { HttpStatusCode } from '@shared/models' -import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared' - -const videoFilesDeleteWebVideoValidator = [ - isValidVideoIdParam('id'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.id, res)) return - - const video = res.locals.videoAll - - if (!checkLocalVideo(video, res)) return - - if (!video.hasWebVideoFiles()) { - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'This video does not have Web Video files' - }) - } - - if (!video.getHLSPlaylist()) { - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Cannot delete Web Video files since this video does not have HLS playlist' - }) - } - - return next() - } -] - -const videoFilesDeleteWebVideoFileValidator = [ - isValidVideoIdParam('id'), - - param('videoFileId') - .custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.id, res)) return - - const video = res.locals.videoAll - - if (!checkLocalVideo(video, res)) return - - const files = video.VideoFiles - if (!files.find(f => f.id === +req.params.videoFileId)) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'This video does not have this Web Video file id' - }) - } - - if (files.length === 1 && !video.getHLSPlaylist()) { - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Cannot delete Web Video files since this video does not have HLS playlist' - }) - } - - return next() - } -] - -// --------------------------------------------------------------------------- - -const videoFilesDeleteHLSValidator = [ - isValidVideoIdParam('id'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.id, res)) return - - const video = res.locals.videoAll - - if (!checkLocalVideo(video, res)) return - - if (!video.getHLSPlaylist()) { - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'This video does not have HLS files' - }) - } - - if (!video.hasWebVideoFiles()) { - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Cannot delete HLS playlist since this video does not have Web Video files' - }) - } - - return next() - } -] - -const videoFilesDeleteHLSFileValidator = [ - isValidVideoIdParam('id'), - - param('videoFileId') - .custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.id, res)) return - - const video = res.locals.videoAll - - if (!checkLocalVideo(video, res)) return - - if (!video.getHLSPlaylist()) { - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'This video does not have HLS files' - }) - } - - const hlsFiles = video.getHLSPlaylist().VideoFiles - if (!hlsFiles.find(f => f.id === +req.params.videoFileId)) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'This HLS playlist does not have this file id' - }) - } - - // Last file to delete - if (hlsFiles.length === 1 && !video.hasWebVideoFiles()) { - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Cannot delete last HLS playlist file since this video does not have Web Video files' - }) - } - - return next() - } -] - -export { - videoFilesDeleteWebVideoValidator, - videoFilesDeleteWebVideoFileValidator, - - videoFilesDeleteHLSValidator, - videoFilesDeleteHLSFileValidator -} - -// --------------------------------------------------------------------------- - -function checkLocalVideo (video: MVideo, res: express.Response) { - if (video.remote) { - res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Cannot delete files of remote video' - }) - - return false - } - - return true -} diff --git a/server/middlewares/validators/videos/video-imports.ts b/server/middlewares/validators/videos/video-imports.ts deleted file mode 100644 index a1cb65b70..000000000 --- a/server/middlewares/validators/videos/video-imports.ts +++ /dev/null @@ -1,209 +0,0 @@ -import express from 'express' -import { body, param, query } from 'express-validator' -import { isResolvingToUnicastOnly } from '@server/helpers/dns' -import { isPreImportVideoAccepted } from '@server/lib/moderation' -import { Hooks } from '@server/lib/plugins/hooks' -import { MUserAccountId, MVideoImport } from '@server/types/models' -import { forceNumber } from '@shared/core-utils' -import { HttpStatusCode, UserRight, VideoImportState } from '@shared/models' -import { VideoImportCreate } from '@shared/models/videos/import/video-import-create.model' -import { isIdValid, toIntOrNull } from '../../../helpers/custom-validators/misc' -import { isVideoImportTargetUrlValid, isVideoImportTorrentFile } from '../../../helpers/custom-validators/video-imports' -import { - isValidPasswordProtectedPrivacy, - isVideoMagnetUriValid, - isVideoNameValid -} from '../../../helpers/custom-validators/videos' -import { cleanUpReqFiles } from '../../../helpers/express-utils' -import { logger } from '../../../helpers/logger' -import { CONFIG } from '../../../initializers/config' -import { CONSTRAINTS_FIELDS } from '../../../initializers/constants' -import { areValidationErrors, doesVideoChannelOfAccountExist, doesVideoImportExist } from '../shared' -import { getCommonVideoEditAttributes } from './videos' - -const videoImportAddValidator = getCommonVideoEditAttributes().concat([ - body('channelId') - .customSanitizer(toIntOrNull) - .custom(isIdValid), - body('targetUrl') - .optional() - .custom(isVideoImportTargetUrlValid), - body('magnetUri') - .optional() - .custom(isVideoMagnetUriValid), - body('torrentfile') - .custom((value, { req }) => isVideoImportTorrentFile(req.files)) - .withMessage( - 'This torrent file is not supported or too large. Please, make sure it is of the following type: ' + - CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_FILE.EXTNAME.join(', ') - ), - body('name') - .optional() - .custom(isVideoNameValid).withMessage( - `Should have a video name between ${CONSTRAINTS_FIELDS.VIDEOS.NAME.min} and ${CONSTRAINTS_FIELDS.VIDEOS.NAME.max} characters long` - ), - body('videoPasswords') - .optional() - .isArray() - .withMessage('Video passwords should be an array.'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - const user = res.locals.oauth.token.User - const torrentFile = req.files?.['torrentfile'] ? req.files['torrentfile'][0] : undefined - - if (areValidationErrors(req, res)) return cleanUpReqFiles(req) - - if (!isValidPasswordProtectedPrivacy(req, res)) return cleanUpReqFiles(req) - - if (CONFIG.IMPORT.VIDEOS.HTTP.ENABLED !== true && req.body.targetUrl) { - cleanUpReqFiles(req) - - return res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: 'HTTP import is not enabled on this instance.' - }) - } - - if (CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED !== true && (req.body.magnetUri || torrentFile)) { - cleanUpReqFiles(req) - - return res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: 'Torrent/magnet URI import is not enabled on this instance.' - }) - } - - if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) - - // Check we have at least 1 required param - if (!req.body.targetUrl && !req.body.magnetUri && !torrentFile) { - cleanUpReqFiles(req) - - return res.fail({ message: 'Should have a magnetUri or a targetUrl or a torrent file.' }) - } - - if (req.body.targetUrl) { - const hostname = new URL(req.body.targetUrl).hostname - - if (await isResolvingToUnicastOnly(hostname) !== true) { - cleanUpReqFiles(req) - - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Cannot use non unicast IP as targetUrl.' - }) - } - } - - if (!await isImportAccepted(req, res)) return cleanUpReqFiles(req) - - return next() - } -]) - -const getMyVideoImportsValidator = [ - query('videoChannelSyncId') - .optional() - .custom(isIdValid), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const videoImportDeleteValidator = [ - param('id') - .custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - if (!await doesVideoImportExist(parseInt(req.params.id), res)) return - if (!checkUserCanManageImport(res.locals.oauth.token.user, res.locals.videoImport, res)) return - - if (res.locals.videoImport.state === VideoImportState.PENDING) { - return res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: 'Cannot delete a pending video import. Cancel it or wait for the end of the import first.' - }) - } - - return next() - } -] - -const videoImportCancelValidator = [ - param('id') - .custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - if (!await doesVideoImportExist(forceNumber(req.params.id), res)) return - if (!checkUserCanManageImport(res.locals.oauth.token.user, res.locals.videoImport, res)) return - - if (res.locals.videoImport.state !== VideoImportState.PENDING) { - return res.fail({ - status: HttpStatusCode.CONFLICT_409, - message: 'Cannot cancel a non pending video import.' - }) - } - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - videoImportAddValidator, - videoImportCancelValidator, - videoImportDeleteValidator, - getMyVideoImportsValidator -} - -// --------------------------------------------------------------------------- - -async function isImportAccepted (req: express.Request, res: express.Response) { - const body: VideoImportCreate = req.body - const hookName = body.targetUrl - ? 'filter:api.video.pre-import-url.accept.result' - : 'filter:api.video.pre-import-torrent.accept.result' - - // Check we accept this video - const acceptParameters = { - videoImportBody: body, - user: res.locals.oauth.token.User - } - const acceptedResult = await Hooks.wrapFun( - isPreImportVideoAccepted, - acceptParameters, - hookName - ) - - if (!acceptedResult || acceptedResult.accepted !== true) { - logger.info('Refused to import video.', { acceptedResult, acceptParameters }) - - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: acceptedResult.errorMessage || 'Refused to import video' - }) - return false - } - - return true -} - -function checkUserCanManageImport (user: MUserAccountId, videoImport: MVideoImport, res: express.Response) { - if (user.hasRight(UserRight.MANAGE_VIDEO_IMPORTS) === false && videoImport.userId !== user.id) { - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Cannot manage video import of another user' - }) - return false - } - - return true -} diff --git a/server/middlewares/validators/videos/video-live.ts b/server/middlewares/validators/videos/video-live.ts deleted file mode 100644 index ec69a3011..000000000 --- a/server/middlewares/validators/videos/video-live.ts +++ /dev/null @@ -1,342 +0,0 @@ -import express from 'express' -import { body } from 'express-validator' -import { isLiveLatencyModeValid } from '@server/helpers/custom-validators/video-lives' -import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' -import { isLocalLiveVideoAccepted } from '@server/lib/moderation' -import { Hooks } from '@server/lib/plugins/hooks' -import { VideoModel } from '@server/models/video/video' -import { VideoLiveModel } from '@server/models/video/video-live' -import { VideoLiveSessionModel } from '@server/models/video/video-live-session' -import { - HttpStatusCode, - LiveVideoCreate, - LiveVideoLatencyMode, - LiveVideoUpdate, - ServerErrorCode, - UserRight, - VideoState -} from '@shared/models' -import { exists, isBooleanValid, isIdValid, toBooleanOrNull, toIntOrNull } from '../../../helpers/custom-validators/misc' -import { isValidPasswordProtectedPrivacy, isVideoNameValid, isVideoReplayPrivacyValid } from '../../../helpers/custom-validators/videos' -import { cleanUpReqFiles } from '../../../helpers/express-utils' -import { logger } from '../../../helpers/logger' -import { CONFIG } from '../../../initializers/config' -import { - areValidationErrors, - checkUserCanManageVideo, - doesVideoChannelOfAccountExist, - doesVideoExist, - isValidVideoIdParam -} from '../shared' -import { getCommonVideoEditAttributes } from './videos' - -const videoLiveGetValidator = [ - isValidVideoIdParam('videoId'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.videoId, res, 'all')) return - - const videoLive = await VideoLiveModel.loadByVideoId(res.locals.videoAll.id) - if (!videoLive) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Live video not found' - }) - } - - res.locals.videoLive = videoLive - - return next() - } -] - -const videoLiveAddValidator = getCommonVideoEditAttributes().concat([ - body('channelId') - .customSanitizer(toIntOrNull) - .custom(isIdValid), - - body('name') - .custom(isVideoNameValid).withMessage( - `Should have a video name between ${CONSTRAINTS_FIELDS.VIDEOS.NAME.min} and ${CONSTRAINTS_FIELDS.VIDEOS.NAME.max} characters long` - ), - - body('saveReplay') - .optional() - .customSanitizer(toBooleanOrNull) - .custom(isBooleanValid).withMessage('Should have a valid saveReplay boolean'), - - body('replaySettings.privacy') - .optional() - .customSanitizer(toIntOrNull) - .custom(isVideoReplayPrivacyValid), - - body('permanentLive') - .optional() - .customSanitizer(toBooleanOrNull) - .custom(isBooleanValid).withMessage('Should have a valid permanentLive boolean'), - - body('latencyMode') - .optional() - .customSanitizer(toIntOrNull) - .custom(isLiveLatencyModeValid), - - body('videoPasswords') - .optional() - .isArray() - .withMessage('Video passwords should be an array.'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return cleanUpReqFiles(req) - - if (!isValidPasswordProtectedPrivacy(req, res)) return cleanUpReqFiles(req) - - if (CONFIG.LIVE.ENABLED !== true) { - cleanUpReqFiles(req) - - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Live is not enabled on this instance', - type: ServerErrorCode.LIVE_NOT_ENABLED - }) - } - - const body: LiveVideoCreate = req.body - - if (hasValidSaveReplay(body) !== true) { - cleanUpReqFiles(req) - - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Saving live replay is not enabled on this instance', - type: ServerErrorCode.LIVE_NOT_ALLOWING_REPLAY - }) - } - - if (hasValidLatencyMode(body) !== true) { - cleanUpReqFiles(req) - - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Custom latency mode is not allowed by this instance' - }) - } - - if (body.saveReplay && !body.replaySettings?.privacy) { - cleanUpReqFiles(req) - - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Live replay is enabled but privacy replay setting is missing' - }) - } - - const user = res.locals.oauth.token.User - if (!await doesVideoChannelOfAccountExist(body.channelId, user, res)) return cleanUpReqFiles(req) - - if (CONFIG.LIVE.MAX_INSTANCE_LIVES !== -1) { - const totalInstanceLives = await VideoModel.countLives({ remote: false, mode: 'not-ended' }) - - if (totalInstanceLives >= CONFIG.LIVE.MAX_INSTANCE_LIVES) { - cleanUpReqFiles(req) - - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Cannot create this live because the max instance lives limit is reached.', - type: ServerErrorCode.MAX_INSTANCE_LIVES_LIMIT_REACHED - }) - } - } - - if (CONFIG.LIVE.MAX_USER_LIVES !== -1) { - const totalUserLives = await VideoModel.countLivesOfAccount(user.Account.id) - - if (totalUserLives >= CONFIG.LIVE.MAX_USER_LIVES) { - cleanUpReqFiles(req) - - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Cannot create this live because the max user lives limit is reached.', - type: ServerErrorCode.MAX_USER_LIVES_LIMIT_REACHED - }) - } - } - - if (!await isLiveVideoAccepted(req, res)) return cleanUpReqFiles(req) - - return next() - } -]) - -const videoLiveUpdateValidator = [ - body('saveReplay') - .optional() - .customSanitizer(toBooleanOrNull) - .custom(isBooleanValid).withMessage('Should have a valid saveReplay boolean'), - - body('replaySettings.privacy') - .optional() - .customSanitizer(toIntOrNull) - .custom(isVideoReplayPrivacyValid), - - body('latencyMode') - .optional() - .customSanitizer(toIntOrNull) - .custom(isLiveLatencyModeValid), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const body: LiveVideoUpdate = req.body - - if (hasValidSaveReplay(body) !== true) { - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Saving live replay is not allowed by this instance' - }) - } - - if (hasValidLatencyMode(body) !== true) { - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Custom latency mode is not allowed by this instance' - }) - } - - if (!checkLiveSettingsReplayConsistency({ res, body })) return - - if (res.locals.videoAll.state !== VideoState.WAITING_FOR_LIVE) { - return res.fail({ message: 'Cannot update a live that has already started' }) - } - - // Check the user can manage the live - const user = res.locals.oauth.token.User - if (!checkUserCanManageVideo(user, res.locals.videoAll, UserRight.GET_ANY_LIVE, res)) return - - return next() - } -] - -const videoLiveListSessionsValidator = [ - (req: express.Request, res: express.Response, next: express.NextFunction) => { - // Check the user can manage the live - const user = res.locals.oauth.token.User - if (!checkUserCanManageVideo(user, res.locals.videoAll, UserRight.GET_ANY_LIVE, res)) return - - return next() - } -] - -const videoLiveFindReplaySessionValidator = [ - isValidVideoIdParam('videoId'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.videoId, res, 'id')) return - - const session = await VideoLiveSessionModel.findSessionOfReplay(res.locals.videoId.id) - if (!session) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'No live replay found' - }) - } - - res.locals.videoLiveSession = session - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - videoLiveAddValidator, - videoLiveUpdateValidator, - videoLiveListSessionsValidator, - videoLiveFindReplaySessionValidator, - videoLiveGetValidator -} - -// --------------------------------------------------------------------------- - -async function isLiveVideoAccepted (req: express.Request, res: express.Response) { - // Check we accept this video - const acceptParameters = { - liveVideoBody: req.body, - user: res.locals.oauth.token.User - } - const acceptedResult = await Hooks.wrapFun( - isLocalLiveVideoAccepted, - acceptParameters, - 'filter:api.live-video.create.accept.result' - ) - - if (!acceptedResult || acceptedResult.accepted !== true) { - logger.info('Refused local live video.', { acceptedResult, acceptParameters }) - - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: acceptedResult.errorMessage || 'Refused local live video' - }) - return false - } - - return true -} - -function hasValidSaveReplay (body: LiveVideoUpdate | LiveVideoCreate) { - if (CONFIG.LIVE.ALLOW_REPLAY !== true && body.saveReplay === true) return false - - return true -} - -function hasValidLatencyMode (body: LiveVideoUpdate | LiveVideoCreate) { - if ( - CONFIG.LIVE.LATENCY_SETTING.ENABLED !== true && - exists(body.latencyMode) && - body.latencyMode !== LiveVideoLatencyMode.DEFAULT - ) return false - - return true -} - -function checkLiveSettingsReplayConsistency (options: { - res: express.Response - body: LiveVideoUpdate -}) { - const { res, body } = options - - // We now save replays of this live, so replay settings are mandatory - if (res.locals.videoLive.saveReplay !== true && body.saveReplay === true) { - - if (!exists(body.replaySettings)) { - res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Replay settings are missing now the live replay is saved' - }) - return false - } - - if (!exists(body.replaySettings.privacy)) { - res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Privacy replay setting is missing now the live replay is saved' - }) - return false - } - } - - // Save replay was and is not enabled, so send an error the user if it specified replay settings - if ((!exists(body.saveReplay) && res.locals.videoLive.saveReplay === false) || body.saveReplay === false) { - if (exists(body.replaySettings)) { - res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Cannot save replay settings since live replay is not enabled' - }) - return false - } - } - - return true -} diff --git a/server/middlewares/validators/videos/video-ownership-changes.ts b/server/middlewares/validators/videos/video-ownership-changes.ts deleted file mode 100644 index 3eca78c25..000000000 --- a/server/middlewares/validators/videos/video-ownership-changes.ts +++ /dev/null @@ -1,107 +0,0 @@ -import express from 'express' -import { param } from 'express-validator' -import { isIdValid } from '@server/helpers/custom-validators/misc' -import { checkUserCanTerminateOwnershipChange } from '@server/helpers/custom-validators/video-ownership' -import { AccountModel } from '@server/models/account/account' -import { MVideoWithAllFiles } from '@server/types/models' -import { HttpStatusCode, UserRight, VideoChangeOwnershipAccept, VideoChangeOwnershipStatus, VideoState } from '@shared/models' -import { - areValidationErrors, - checkUserCanManageVideo, - checkUserQuota, - doesChangeVideoOwnershipExist, - doesVideoChannelOfAccountExist, - doesVideoExist, - isValidVideoIdParam -} from '../shared' - -const videosChangeOwnershipValidator = [ - isValidVideoIdParam('videoId'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.videoId, res)) return - - // Check if the user who did the request is able to change the ownership of the video - if (!checkUserCanManageVideo(res.locals.oauth.token.User, res.locals.videoAll, UserRight.CHANGE_VIDEO_OWNERSHIP, res)) return - - const nextOwner = await AccountModel.loadLocalByName(req.body.username) - if (!nextOwner) { - res.fail({ message: 'Changing video ownership to a remote account is not supported yet' }) - return - } - - res.locals.nextOwner = nextOwner - return next() - } -] - -const videosTerminateChangeOwnershipValidator = [ - param('id') - .custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesChangeVideoOwnershipExist(req.params.id, res)) return - - // Check if the user who did the request is able to change the ownership of the video - if (!checkUserCanTerminateOwnershipChange(res.locals.oauth.token.User, res.locals.videoChangeOwnership, res)) return - - const videoChangeOwnership = res.locals.videoChangeOwnership - - if (videoChangeOwnership.status !== VideoChangeOwnershipStatus.WAITING) { - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Ownership already accepted or refused' - }) - return - } - - return next() - } -] - -const videosAcceptChangeOwnershipValidator = [ - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - const body = req.body as VideoChangeOwnershipAccept - if (!await doesVideoChannelOfAccountExist(body.channelId, res.locals.oauth.token.User, res)) return - - const videoChangeOwnership = res.locals.videoChangeOwnership - - const video = videoChangeOwnership.Video - - if (!await checkCanAccept(video, res)) return - - return next() - } -] - -export { - videosChangeOwnershipValidator, - videosTerminateChangeOwnershipValidator, - videosAcceptChangeOwnershipValidator -} - -// --------------------------------------------------------------------------- - -async function checkCanAccept (video: MVideoWithAllFiles, res: express.Response): Promise { - if (video.isLive) { - - if (video.state !== VideoState.WAITING_FOR_LIVE) { - res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'You can accept an ownership change of a published live.' - }) - - return false - } - - return true - } - - const user = res.locals.oauth.token.User - - if (!await checkUserQuota(user, video.getMaxQualityFile().size, res)) return false - - return true -} diff --git a/server/middlewares/validators/videos/video-passwords.ts b/server/middlewares/validators/videos/video-passwords.ts deleted file mode 100644 index 200e496f6..000000000 --- a/server/middlewares/validators/videos/video-passwords.ts +++ /dev/null @@ -1,77 +0,0 @@ -import express from 'express' -import { - areValidationErrors, - doesVideoExist, - isVideoPasswordProtected, - isValidVideoIdParam, - doesVideoPasswordExist, - isVideoPasswordDeletable, - checkUserCanManageVideo -} from '../shared' -import { body, param } from 'express-validator' -import { isIdValid } from '@server/helpers/custom-validators/misc' -import { isValidPasswordProtectedPrivacy } from '@server/helpers/custom-validators/videos' -import { UserRight } from '@shared/models' - -const listVideoPasswordValidator = [ - isValidVideoIdParam('videoId'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - if (!await doesVideoExist(req.params.videoId, res)) return - if (!isVideoPasswordProtected(res)) return - - // Check if the user who did the request is able to access video password list - const user = res.locals.oauth.token.User - if (!checkUserCanManageVideo(user, res.locals.videoAll, UserRight.SEE_ALL_VIDEOS, res)) return - - return next() - } -] - -const updateVideoPasswordListValidator = [ - body('passwords') - .optional() - .isArray() - .withMessage('Video passwords should be an array.'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - if (!await doesVideoExist(req.params.videoId, res)) return - if (!isValidPasswordProtectedPrivacy(req, res)) return - - // Check if the user who did the request is able to update video passwords - const user = res.locals.oauth.token.User - if (!checkUserCanManageVideo(user, res.locals.videoAll, UserRight.UPDATE_ANY_VIDEO, res)) return - - return next() - } -] - -const removeVideoPasswordValidator = [ - isValidVideoIdParam('videoId'), - - param('passwordId') - .custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - if (!await doesVideoExist(req.params.videoId, res)) return - if (!isVideoPasswordProtected(res)) return - if (!await doesVideoPasswordExist(req.params.passwordId, res)) return - if (!await isVideoPasswordDeletable(res)) return - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - listVideoPasswordValidator, - updateVideoPasswordListValidator, - removeVideoPasswordValidator -} diff --git a/server/middlewares/validators/videos/video-playlists.ts b/server/middlewares/validators/videos/video-playlists.ts deleted file mode 100644 index 95a5ba63a..000000000 --- a/server/middlewares/validators/videos/video-playlists.ts +++ /dev/null @@ -1,419 +0,0 @@ -import express from 'express' -import { body, param, query, ValidationChain } from 'express-validator' -import { ExpressPromiseHandler } from '@server/types/express-handler' -import { MUserAccountId } from '@server/types/models' -import { forceNumber } from '@shared/core-utils' -import { - HttpStatusCode, - UserRight, - VideoPlaylistCreate, - VideoPlaylistPrivacy, - VideoPlaylistType, - VideoPlaylistUpdate -} from '@shared/models' -import { - isArrayOf, - isIdOrUUIDValid, - isIdValid, - isUUIDValid, - toCompleteUUID, - toIntArray, - toIntOrNull, - toValueOrNull -} from '../../../helpers/custom-validators/misc' -import { - isVideoPlaylistDescriptionValid, - isVideoPlaylistNameValid, - isVideoPlaylistPrivacyValid, - isVideoPlaylistTimestampValid, - isVideoPlaylistTypeValid -} from '../../../helpers/custom-validators/video-playlists' -import { isVideoImageValid } from '../../../helpers/custom-validators/videos' -import { cleanUpReqFiles } from '../../../helpers/express-utils' -import { CONSTRAINTS_FIELDS } from '../../../initializers/constants' -import { VideoPlaylistElementModel } from '../../../models/video/video-playlist-element' -import { MVideoPlaylist } from '../../../types/models/video/video-playlist' -import { authenticatePromise } from '../../auth' -import { - areValidationErrors, - doesVideoChannelIdExist, - doesVideoExist, - doesVideoPlaylistExist, - isValidPlaylistIdParam, - VideoPlaylistFetchType -} from '../shared' - -const videoPlaylistsAddValidator = getCommonPlaylistEditAttributes().concat([ - body('displayName') - .custom(isVideoPlaylistNameValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return cleanUpReqFiles(req) - - const body: VideoPlaylistCreate = req.body - if (body.videoChannelId && !await doesVideoChannelIdExist(body.videoChannelId, res)) return cleanUpReqFiles(req) - - if ( - !body.videoChannelId && - (body.privacy === VideoPlaylistPrivacy.PUBLIC || body.privacy === VideoPlaylistPrivacy.UNLISTED) - ) { - cleanUpReqFiles(req) - - return res.fail({ message: 'Cannot set "public" or "unlisted" a playlist that is not assigned to a channel.' }) - } - - return next() - } -]) - -const videoPlaylistsUpdateValidator = getCommonPlaylistEditAttributes().concat([ - isValidPlaylistIdParam('playlistId'), - - body('displayName') - .optional() - .custom(isVideoPlaylistNameValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return cleanUpReqFiles(req) - - if (!await doesVideoPlaylistExist(req.params.playlistId, res, 'all')) return cleanUpReqFiles(req) - - const videoPlaylist = getPlaylist(res) - - if (!checkUserCanManageVideoPlaylist(res.locals.oauth.token.User, videoPlaylist, UserRight.REMOVE_ANY_VIDEO_PLAYLIST, res)) { - return cleanUpReqFiles(req) - } - - const body: VideoPlaylistUpdate = req.body - - const newPrivacy = body.privacy || videoPlaylist.privacy - if (newPrivacy === VideoPlaylistPrivacy.PUBLIC && - ( - (!videoPlaylist.videoChannelId && !body.videoChannelId) || - body.videoChannelId === null - ) - ) { - cleanUpReqFiles(req) - - return res.fail({ message: 'Cannot set "public" a playlist that is not assigned to a channel.' }) - } - - if (videoPlaylist.type === VideoPlaylistType.WATCH_LATER) { - cleanUpReqFiles(req) - - return res.fail({ message: 'Cannot update a watch later playlist.' }) - } - - if (body.videoChannelId && !await doesVideoChannelIdExist(body.videoChannelId, res)) return cleanUpReqFiles(req) - - return next() - } -]) - -const videoPlaylistsDeleteValidator = [ - isValidPlaylistIdParam('playlistId'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - if (!await doesVideoPlaylistExist(req.params.playlistId, res)) return - - const videoPlaylist = getPlaylist(res) - if (videoPlaylist.type === VideoPlaylistType.WATCH_LATER) { - return res.fail({ message: 'Cannot delete a watch later playlist.' }) - } - - if (!checkUserCanManageVideoPlaylist(res.locals.oauth.token.User, videoPlaylist, UserRight.REMOVE_ANY_VIDEO_PLAYLIST, res)) { - return - } - - return next() - } -] - -const videoPlaylistsGetValidator = (fetchType: VideoPlaylistFetchType) => { - return [ - isValidPlaylistIdParam('playlistId'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - if (!await doesVideoPlaylistExist(req.params.playlistId, res, fetchType)) return - - const videoPlaylist = res.locals.videoPlaylistFull || res.locals.videoPlaylistSummary - - // Playlist is unlisted, check we used the uuid to fetch it - if (videoPlaylist.privacy === VideoPlaylistPrivacy.UNLISTED) { - if (isUUIDValid(req.params.playlistId)) return next() - - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Playlist not found' - }) - } - - if (videoPlaylist.privacy === VideoPlaylistPrivacy.PRIVATE) { - await authenticatePromise({ req, res }) - - const user = res.locals.oauth ? res.locals.oauth.token.User : null - - if ( - !user || - (videoPlaylist.OwnerAccount.id !== user.Account.id && !user.hasRight(UserRight.UPDATE_ANY_VIDEO_PLAYLIST)) - ) { - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Cannot get this private video playlist.' - }) - } - - return next() - } - - return next() - } - ] -} - -const videoPlaylistsSearchValidator = [ - query('search') - .optional() - .not().isEmpty(), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const videoPlaylistsAddVideoValidator = [ - isValidPlaylistIdParam('playlistId'), - - body('videoId') - .customSanitizer(toCompleteUUID) - .custom(isIdOrUUIDValid).withMessage('Should have a valid video id/uuid/short uuid'), - body('startTimestamp') - .optional() - .custom(isVideoPlaylistTimestampValid), - body('stopTimestamp') - .optional() - .custom(isVideoPlaylistTimestampValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - if (!await doesVideoPlaylistExist(req.params.playlistId, res, 'all')) return - if (!await doesVideoExist(req.body.videoId, res, 'only-video')) return - - const videoPlaylist = getPlaylist(res) - - if (!checkUserCanManageVideoPlaylist(res.locals.oauth.token.User, videoPlaylist, UserRight.UPDATE_ANY_VIDEO_PLAYLIST, res)) { - return - } - - return next() - } -] - -const videoPlaylistsUpdateOrRemoveVideoValidator = [ - isValidPlaylistIdParam('playlistId'), - param('playlistElementId') - .customSanitizer(toCompleteUUID) - .custom(isIdValid).withMessage('Should have an element id/uuid/short uuid'), - body('startTimestamp') - .optional() - .custom(isVideoPlaylistTimestampValid), - body('stopTimestamp') - .optional() - .custom(isVideoPlaylistTimestampValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - if (!await doesVideoPlaylistExist(req.params.playlistId, res, 'all')) return - - const videoPlaylist = getPlaylist(res) - - const videoPlaylistElement = await VideoPlaylistElementModel.loadById(req.params.playlistElementId) - if (!videoPlaylistElement) { - res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video playlist element not found' - }) - return - } - res.locals.videoPlaylistElement = videoPlaylistElement - - if (!checkUserCanManageVideoPlaylist(res.locals.oauth.token.User, videoPlaylist, UserRight.UPDATE_ANY_VIDEO_PLAYLIST, res)) return - - return next() - } -] - -const videoPlaylistElementAPGetValidator = [ - isValidPlaylistIdParam('playlistId'), - param('playlistElementId') - .custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const playlistElementId = forceNumber(req.params.playlistElementId) - const playlistId = req.params.playlistId - - const videoPlaylistElement = await VideoPlaylistElementModel.loadByPlaylistAndElementIdForAP(playlistId, playlistElementId) - if (!videoPlaylistElement) { - res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video playlist element not found' - }) - return - } - - if (videoPlaylistElement.VideoPlaylist.privacy === VideoPlaylistPrivacy.PRIVATE) { - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Cannot get this private video playlist.' - }) - } - - res.locals.videoPlaylistElementAP = videoPlaylistElement - - return next() - } -] - -const videoPlaylistsReorderVideosValidator = [ - isValidPlaylistIdParam('playlistId'), - - body('startPosition') - .isInt({ min: 1 }), - body('insertAfterPosition') - .isInt({ min: 0 }), - body('reorderLength') - .optional() - .isInt({ min: 1 }), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - if (!await doesVideoPlaylistExist(req.params.playlistId, res, 'all')) return - - const videoPlaylist = getPlaylist(res) - if (!checkUserCanManageVideoPlaylist(res.locals.oauth.token.User, videoPlaylist, UserRight.UPDATE_ANY_VIDEO_PLAYLIST, res)) return - - const nextPosition = await VideoPlaylistElementModel.getNextPositionOf(videoPlaylist.id) - const startPosition: number = req.body.startPosition - const insertAfterPosition: number = req.body.insertAfterPosition - const reorderLength: number = req.body.reorderLength - - if (startPosition >= nextPosition || insertAfterPosition >= nextPosition) { - res.fail({ message: `Start position or insert after position exceed the playlist limits (max: ${nextPosition - 1})` }) - return - } - - if (reorderLength && reorderLength + startPosition > nextPosition) { - res.fail({ message: `Reorder length with this start position exceeds the playlist limits (max: ${nextPosition - startPosition})` }) - return - } - - return next() - } -] - -const commonVideoPlaylistFiltersValidator = [ - query('playlistType') - .optional() - .custom(isVideoPlaylistTypeValid), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -const doVideosInPlaylistExistValidator = [ - query('videoIds') - .customSanitizer(toIntArray) - .custom(v => isArrayOf(v, isIdValid)).withMessage('Should have a valid video ids array'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - videoPlaylistsAddValidator, - videoPlaylistsUpdateValidator, - videoPlaylistsDeleteValidator, - videoPlaylistsGetValidator, - videoPlaylistsSearchValidator, - - videoPlaylistsAddVideoValidator, - videoPlaylistsUpdateOrRemoveVideoValidator, - videoPlaylistsReorderVideosValidator, - - videoPlaylistElementAPGetValidator, - - commonVideoPlaylistFiltersValidator, - - doVideosInPlaylistExistValidator -} - -// --------------------------------------------------------------------------- - -function getCommonPlaylistEditAttributes () { - return [ - body('thumbnailfile') - .custom((value, { req }) => isVideoImageValid(req.files, 'thumbnailfile')) - .withMessage( - 'This thumbnail file is not supported or too large. Please, make sure it is of the following type: ' + - CONSTRAINTS_FIELDS.VIDEO_PLAYLISTS.IMAGE.EXTNAME.join(', ') - ), - - body('description') - .optional() - .customSanitizer(toValueOrNull) - .custom(isVideoPlaylistDescriptionValid), - body('privacy') - .optional() - .customSanitizer(toIntOrNull) - .custom(isVideoPlaylistPrivacyValid), - body('videoChannelId') - .optional() - .customSanitizer(toIntOrNull) - ] as (ValidationChain | ExpressPromiseHandler)[] -} - -function checkUserCanManageVideoPlaylist (user: MUserAccountId, videoPlaylist: MVideoPlaylist, right: UserRight, res: express.Response) { - if (videoPlaylist.isOwned() === false) { - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Cannot manage video playlist of another server.' - }) - return false - } - - // Check if the user can manage the video playlist - // The user can delete it if s/he is an admin - // Or if s/he is the video playlist's owner - if (user.hasRight(right) === false && videoPlaylist.ownerAccountId !== user.Account.id) { - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Cannot manage video playlist of another user' - }) - return false - } - - return true -} - -function getPlaylist (res: express.Response) { - return res.locals.videoPlaylistFull || res.locals.videoPlaylistSummary -} diff --git a/server/middlewares/validators/videos/video-rates.ts b/server/middlewares/validators/videos/video-rates.ts deleted file mode 100644 index c837b047b..000000000 --- a/server/middlewares/validators/videos/video-rates.ts +++ /dev/null @@ -1,72 +0,0 @@ -import express from 'express' -import { body, param, query } from 'express-validator' -import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' -import { VideoRateType } from '../../../../shared/models/videos' -import { isAccountNameValid } from '../../../helpers/custom-validators/accounts' -import { isIdValid } from '../../../helpers/custom-validators/misc' -import { isRatingValid } from '../../../helpers/custom-validators/video-rates' -import { isVideoRatingTypeValid } from '../../../helpers/custom-validators/videos' -import { AccountVideoRateModel } from '../../../models/account/account-video-rate' -import { areValidationErrors, checkCanSeeVideo, doesVideoExist, isValidVideoIdParam, isValidVideoPasswordHeader } from '../shared' - -const videoUpdateRateValidator = [ - isValidVideoIdParam('id'), - - body('rating') - .custom(isVideoRatingTypeValid), - isValidVideoPasswordHeader(), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.id, res)) return - - if (!await checkCanSeeVideo({ req, res, paramId: req.params.id, video: res.locals.videoAll })) return - - return next() - } -] - -const getAccountVideoRateValidatorFactory = function (rateType: VideoRateType) { - return [ - param('name') - .custom(isAccountNameValid), - param('videoId') - .custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const rate = await AccountVideoRateModel.loadLocalAndPopulateVideo(rateType, req.params.name, +req.params.videoId) - if (!rate) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video rate not found' - }) - } - - res.locals.accountVideoRate = rate - - return next() - } - ] -} - -const videoRatingValidator = [ - query('rating') - .optional() - .custom(isRatingValid).withMessage('Value must be one of "like" or "dislike"'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - videoUpdateRateValidator, - getAccountVideoRateValidatorFactory, - videoRatingValidator -} diff --git a/server/middlewares/validators/videos/video-shares.ts b/server/middlewares/validators/videos/video-shares.ts deleted file mode 100644 index c234de6ed..000000000 --- a/server/middlewares/validators/videos/video-shares.ts +++ /dev/null @@ -1,35 +0,0 @@ -import express from 'express' -import { param } from 'express-validator' -import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' -import { isIdValid } from '../../../helpers/custom-validators/misc' -import { VideoShareModel } from '../../../models/video/video-share' -import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared' - -const videosShareValidator = [ - isValidVideoIdParam('id'), - - param('actorId') - .custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.id, res)) return - - const video = res.locals.videoAll - - const share = await VideoShareModel.load(req.params.actorId, video.id) - if (!share) { - return res.status(HttpStatusCode.NOT_FOUND_404) - .end() - } - - res.locals.videoShare = share - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - videosShareValidator -} diff --git a/server/middlewares/validators/videos/video-source.ts b/server/middlewares/validators/videos/video-source.ts deleted file mode 100644 index bbccb58b0..000000000 --- a/server/middlewares/validators/videos/video-source.ts +++ /dev/null @@ -1,130 +0,0 @@ -import express from 'express' -import { body, header } from 'express-validator' -import { getResumableUploadPath } from '@server/helpers/upload' -import { getVideoWithAttributes } from '@server/helpers/video' -import { CONFIG } from '@server/initializers/config' -import { uploadx } from '@server/lib/uploadx' -import { VideoSourceModel } from '@server/models/video/video-source' -import { MVideoFullLight } from '@server/types/models' -import { HttpStatusCode, UserRight } from '@shared/models' -import { Metadata as UploadXMetadata } from '@uploadx/core' -import { logger } from '../../../helpers/logger' -import { areValidationErrors, checkUserCanManageVideo, doesVideoExist, isValidVideoIdParam } from '../shared' -import { addDurationToVideoFileIfNeeded, checkVideoFileCanBeEdited, commonVideoFileChecks, isVideoFileAccepted } from './shared' - -export const videoSourceGetLatestValidator = [ - isValidVideoIdParam('id'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.id, res, 'all')) return - - const video = getVideoWithAttributes(res) as MVideoFullLight - - const user = res.locals.oauth.token.User - if (!checkUserCanManageVideo(user, video, UserRight.UPDATE_ANY_VIDEO, res)) return - - res.locals.videoSource = await VideoSourceModel.loadLatest(video.id) - - if (!res.locals.videoSource) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Video source not found' - }) - } - - return next() - } -] - -export const replaceVideoSourceResumableValidator = [ - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - const body: express.CustomUploadXFile = req.body - const file = { ...body, duration: undefined, path: getResumableUploadPath(body.name), filename: body.metadata.filename } - const cleanup = () => uploadx.storage.delete(file).catch(err => logger.error('Cannot delete the file %s', file.name, { err })) - - if (!await checkCanUpdateVideoFile({ req, res })) { - return cleanup() - } - - if (!await addDurationToVideoFileIfNeeded({ videoFile: file, res, middlewareName: 'updateVideoFileResumableValidator' })) { - return cleanup() - } - - if (!await isVideoFileAccepted({ req, res, videoFile: file, hook: 'filter:api.video.update-file.accept.result' })) { - return cleanup() - } - - res.locals.updateVideoFileResumable = { ...file, originalname: file.filename } - - return next() - } -] - -export const replaceVideoSourceResumableInitValidator = [ - body('filename') - .exists(), - - header('x-upload-content-length') - .isNumeric() - .exists() - .withMessage('Should specify the file length'), - header('x-upload-content-type') - .isString() - .exists() - .withMessage('Should specify the file mimetype'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - const user = res.locals.oauth.token.User - - logger.debug('Checking updateVideoFileResumableInitValidator parameters and headers', { - parameters: req.body, - headers: req.headers - }) - - if (areValidationErrors(req, res, { omitLog: true })) return - - if (!await checkCanUpdateVideoFile({ req, res })) return - - const videoFileMetadata = { - mimetype: req.headers['x-upload-content-type'] as string, - size: +req.headers['x-upload-content-length'], - originalname: req.body.filename - } - - const files = { videofile: [ videoFileMetadata ] } - if (await commonVideoFileChecks({ res, user, videoFileSize: videoFileMetadata.size, files }) === false) return - - return next() - } -] - -// --------------------------------------------------------------------------- -// Private -// --------------------------------------------------------------------------- - -async function checkCanUpdateVideoFile (options: { - req: express.Request - res: express.Response -}) { - const { req, res } = options - - if (!CONFIG.VIDEO_FILE.UPDATE.ENABLED) { - res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Updating the file of an existing video is not allowed on this instance' - }) - return false - } - - if (!await doesVideoExist(req.params.id, res)) return false - - const user = res.locals.oauth.token.User - const video = res.locals.videoAll - - if (!checkUserCanManageVideo(user, video, UserRight.UPDATE_ANY_VIDEO, res)) return false - - if (!checkVideoFileCanBeEdited(video, res)) return false - - return true -} diff --git a/server/middlewares/validators/videos/video-stats.ts b/server/middlewares/validators/videos/video-stats.ts deleted file mode 100644 index a79526d39..000000000 --- a/server/middlewares/validators/videos/video-stats.ts +++ /dev/null @@ -1,108 +0,0 @@ -import express from 'express' -import { param, query } from 'express-validator' -import { isDateValid } from '@server/helpers/custom-validators/misc' -import { isValidStatTimeserieMetric } from '@server/helpers/custom-validators/video-stats' -import { STATS_TIMESERIE } from '@server/initializers/constants' -import { HttpStatusCode, UserRight, VideoStatsTimeserieQuery } from '@shared/models' -import { areValidationErrors, checkUserCanManageVideo, doesVideoExist, isValidVideoIdParam } from '../shared' - -const videoOverallStatsValidator = [ - isValidVideoIdParam('videoId'), - - query('startDate') - .optional() - .custom(isDateValid), - - query('endDate') - .optional() - .custom(isDateValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await commonStatsCheck(req, res)) return - - return next() - } -] - -const videoRetentionStatsValidator = [ - isValidVideoIdParam('videoId'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await commonStatsCheck(req, res)) return - - if (res.locals.videoAll.isLive) { - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Cannot get retention stats of live video' - }) - } - - return next() - } -] - -const videoTimeserieStatsValidator = [ - isValidVideoIdParam('videoId'), - - param('metric') - .custom(isValidStatTimeserieMetric), - - query('startDate') - .optional() - .custom(isDateValid), - - query('endDate') - .optional() - .custom(isDateValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await commonStatsCheck(req, res)) return - - const query: VideoStatsTimeserieQuery = req.query - if ( - (query.startDate && !query.endDate) || - (!query.startDate && query.endDate) - ) { - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Both start date and end date should be defined if one of them is specified' - }) - } - - if (query.startDate && getIntervalByDays(query.startDate, query.endDate) > STATS_TIMESERIE.MAX_DAYS) { - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Star date and end date interval is too big' - }) - } - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - videoOverallStatsValidator, - videoTimeserieStatsValidator, - videoRetentionStatsValidator -} - -// --------------------------------------------------------------------------- - -async function commonStatsCheck (req: express.Request, res: express.Response) { - if (!await doesVideoExist(req.params.videoId, res, 'all')) return false - if (!checkUserCanManageVideo(res.locals.oauth.token.User, res.locals.videoAll, UserRight.SEE_ALL_VIDEOS, res)) return false - - return true -} - -function getIntervalByDays (startDateString: string, endDateString: string) { - const startDate = new Date(startDateString) - const endDate = new Date(endDateString) - - return (endDate.getTime() - startDate.getTime()) / 1000 / 86400 -} diff --git a/server/middlewares/validators/videos/video-studio.ts b/server/middlewares/validators/videos/video-studio.ts deleted file mode 100644 index a375af60a..000000000 --- a/server/middlewares/validators/videos/video-studio.ts +++ /dev/null @@ -1,105 +0,0 @@ -import express from 'express' -import { body, param } from 'express-validator' -import { isIdOrUUIDValid } from '@server/helpers/custom-validators/misc' -import { - isStudioCutTaskValid, - isStudioTaskAddIntroOutroValid, - isStudioTaskAddWatermarkValid, - isValidStudioTasksArray -} from '@server/helpers/custom-validators/video-studio' -import { cleanUpReqFiles } from '@server/helpers/express-utils' -import { CONFIG } from '@server/initializers/config' -import { approximateIntroOutroAdditionalSize, getTaskFileFromReq } from '@server/lib/video-studio' -import { isAudioFile } from '@shared/ffmpeg' -import { HttpStatusCode, UserRight, VideoStudioCreateEdition, VideoStudioTask } from '@shared/models' -import { areValidationErrors, checkUserCanManageVideo, checkUserQuota, doesVideoExist } from '../shared' -import { checkVideoFileCanBeEdited } from './shared' - -const videoStudioAddEditionValidator = [ - param('videoId') - .custom(isIdOrUUIDValid).withMessage('Should have a valid video id/uuid/short uuid'), - - body('tasks') - .custom(isValidStudioTasksArray).withMessage('Should have a valid array of tasks'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (CONFIG.VIDEO_STUDIO.ENABLED !== true) { - res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Video studio is disabled on this instance' - }) - - return cleanUpReqFiles(req) - } - - if (areValidationErrors(req, res)) return cleanUpReqFiles(req) - - const body: VideoStudioCreateEdition = req.body - const files = req.files as Express.Multer.File[] - - for (let i = 0; i < body.tasks.length; i++) { - const task = body.tasks[i] - - if (!checkTask(req, task, i)) { - res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: `Task ${task.name} is invalid` - }) - - return cleanUpReqFiles(req) - } - - if (task.name === 'add-intro' || task.name === 'add-outro') { - const filePath = getTaskFileFromReq(files, i).path - - // Our concat filter needs a video stream - if (await isAudioFile(filePath)) { - res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: `Task ${task.name} is invalid: file does not contain a video stream` - }) - - return cleanUpReqFiles(req) - } - } - } - - if (!await doesVideoExist(req.params.videoId, res)) return cleanUpReqFiles(req) - - const video = res.locals.videoAll - if (!checkVideoFileCanBeEdited(video, res)) return cleanUpReqFiles(req) - - const user = res.locals.oauth.token.User - if (!checkUserCanManageVideo(user, video, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req) - - // Try to make an approximation of bytes added by the intro/outro - const additionalBytes = await approximateIntroOutroAdditionalSize(video, body.tasks, i => getTaskFileFromReq(files, i).path) - if (await checkUserQuota(user, additionalBytes, res) === false) return cleanUpReqFiles(req) - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - videoStudioAddEditionValidator -} - -// --------------------------------------------------------------------------- - -const taskCheckers: { - [id in VideoStudioTask['name']]: (task: VideoStudioTask, indice?: number, files?: Express.Multer.File[]) => boolean -} = { - 'cut': isStudioCutTaskValid, - 'add-intro': isStudioTaskAddIntroOutroValid, - 'add-outro': isStudioTaskAddIntroOutroValid, - 'add-watermark': isStudioTaskAddWatermarkValid -} - -function checkTask (req: express.Request, task: VideoStudioTask, indice?: number) { - const checker = taskCheckers[task.name] - if (!checker) return false - - return checker(task, indice, req.files as Express.Multer.File[]) -} diff --git a/server/middlewares/validators/videos/video-token.ts b/server/middlewares/validators/videos/video-token.ts deleted file mode 100644 index d4253e21d..000000000 --- a/server/middlewares/validators/videos/video-token.ts +++ /dev/null @@ -1,24 +0,0 @@ -import express from 'express' -import { VideoPrivacy } from '../../../../shared/models/videos' -import { HttpStatusCode } from '@shared/models' -import { exists } from '@server/helpers/custom-validators/misc' - -const videoFileTokenValidator = [ - (req: express.Request, res: express.Response, next: express.NextFunction) => { - const video = res.locals.onlyVideo - if (video.privacy !== VideoPrivacy.PASSWORD_PROTECTED && !exists(res.locals.oauth.token.User)) { - return res.fail({ - status: HttpStatusCode.UNAUTHORIZED_401, - message: 'Not authenticated' - }) - } - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - videoFileTokenValidator -} diff --git a/server/middlewares/validators/videos/video-transcoding.ts b/server/middlewares/validators/videos/video-transcoding.ts deleted file mode 100644 index 2f99ff42c..000000000 --- a/server/middlewares/validators/videos/video-transcoding.ts +++ /dev/null @@ -1,61 +0,0 @@ -import express from 'express' -import { body } from 'express-validator' -import { isBooleanValid, toBooleanOrNull } from '@server/helpers/custom-validators/misc' -import { isValidCreateTranscodingType } from '@server/helpers/custom-validators/video-transcoding' -import { CONFIG } from '@server/initializers/config' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' -import { HttpStatusCode, ServerErrorCode, VideoTranscodingCreate } from '@shared/models' -import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared' - -const createTranscodingValidator = [ - isValidVideoIdParam('videoId'), - - body('transcodingType') - .custom(isValidCreateTranscodingType), - - body('forceTranscoding') - .optional() - .customSanitizer(toBooleanOrNull) - .custom(isBooleanValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.videoId, res, 'all')) return - - const video = res.locals.videoAll - - if (video.remote) { - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Cannot run transcoding job on a remote video' - }) - } - - if (CONFIG.TRANSCODING.ENABLED !== true) { - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Cannot run transcoding job because transcoding is disabled on this instance' - }) - } - - const body = req.body as VideoTranscodingCreate - if (body.forceTranscoding === true) return next() - - const info = await VideoJobInfoModel.load(video.id) - if (info && info.pendingTranscode > 0) { - return res.fail({ - status: HttpStatusCode.CONFLICT_409, - type: ServerErrorCode.VIDEO_ALREADY_BEING_TRANSCODED, - message: 'This video is already being transcoded' - }) - } - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - createTranscodingValidator -} diff --git a/server/middlewares/validators/videos/video-view.ts b/server/middlewares/validators/videos/video-view.ts deleted file mode 100644 index a2f61f4ba..000000000 --- a/server/middlewares/validators/videos/video-view.ts +++ /dev/null @@ -1,61 +0,0 @@ -import express from 'express' -import { body, param } from 'express-validator' -import { isVideoTimeValid } from '@server/helpers/custom-validators/video-view' -import { getCachedVideoDuration } from '@server/lib/video' -import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' -import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' -import { isIdValid, isIntOrNull, toIntOrNull } from '../../../helpers/custom-validators/misc' -import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared' - -const getVideoLocalViewerValidator = [ - param('localViewerId') - .custom(isIdValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const localViewer = await LocalVideoViewerModel.loadFullById(+req.params.localViewerId) - if (!localViewer) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Local viewer not found' - }) - } - - res.locals.localViewerFull = localViewer - - return next() - } -] - -const videoViewValidator = [ - isValidVideoIdParam('videoId'), - - body('currentTime') - .customSanitizer(toIntOrNull) - .custom(isIntOrNull), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.videoId, res, 'only-immutable-attributes')) return - - const video = res.locals.onlyImmutableVideo - const { duration } = await getCachedVideoDuration(video.id) - - if (!isVideoTimeValid(req.body.currentTime, duration)) { - return res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Current time is invalid' - }) - } - - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - videoViewValidator, - getVideoLocalViewerValidator -} diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts deleted file mode 100644 index 5a49779ed..000000000 --- a/server/middlewares/validators/videos/videos.ts +++ /dev/null @@ -1,575 +0,0 @@ -import express from 'express' -import { body, header, param, query, ValidationChain } from 'express-validator' -import { isTestInstance } from '@server/helpers/core-utils' -import { getResumableUploadPath } from '@server/helpers/upload' -import { Redis } from '@server/lib/redis' -import { uploadx } from '@server/lib/uploadx' -import { getServerActor } from '@server/models/application/application' -import { ExpressPromiseHandler } from '@server/types/express-handler' -import { MUserAccountId, MVideoFullLight } from '@server/types/models' -import { arrayify } from '@shared/core-utils' -import { HttpStatusCode, ServerErrorCode, UserRight, VideoState } from '@shared/models' -import { - exists, - isBooleanValid, - isDateValid, - isFileValid, - isIdValid, - toBooleanOrNull, - toIntOrNull, - toValueOrNull -} from '../../../helpers/custom-validators/misc' -import { isBooleanBothQueryValid, isNumberArray, isStringArray } from '../../../helpers/custom-validators/search' -import { - areVideoTagsValid, - isScheduleVideoUpdatePrivacyValid, - isValidPasswordProtectedPrivacy, - isVideoCategoryValid, - isVideoDescriptionValid, - isVideoImageValid, - isVideoIncludeValid, - isVideoLanguageValid, - isVideoLicenceValid, - isVideoNameValid, - isVideoOriginallyPublishedAtValid, - isVideoPrivacyValid, - isVideoSupportValid -} from '../../../helpers/custom-validators/videos' -import { cleanUpReqFiles } from '../../../helpers/express-utils' -import { logger } from '../../../helpers/logger' -import { getVideoWithAttributes } from '../../../helpers/video' -import { CONFIG } from '../../../initializers/config' -import { CONSTRAINTS_FIELDS, OVERVIEWS } from '../../../initializers/constants' -import { VideoModel } from '../../../models/video/video' -import { - areValidationErrors, - checkCanAccessVideoStaticFiles, - checkCanSeeVideo, - checkUserCanManageVideo, - doesVideoChannelOfAccountExist, - doesVideoExist, - doesVideoFileOfVideoExist, - isValidVideoIdParam, - isValidVideoPasswordHeader -} from '../shared' -import { addDurationToVideoFileIfNeeded, commonVideoFileChecks, isVideoFileAccepted } from './shared' - -const videosAddLegacyValidator = getCommonVideoEditAttributes().concat([ - body('videofile') - .custom((_, { req }) => isFileValid({ files: req.files, field: 'videofile', mimeTypeRegex: null, maxSize: null })) - .withMessage('Should have a file'), - body('name') - .trim() - .custom(isVideoNameValid).withMessage( - `Should have a video name between ${CONSTRAINTS_FIELDS.VIDEOS.NAME.min} and ${CONSTRAINTS_FIELDS.VIDEOS.NAME.max} characters long` - ), - body('channelId') - .customSanitizer(toIntOrNull) - .custom(isIdValid), - body('videoPasswords') - .optional() - .isArray() - .withMessage('Video passwords should be an array.'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return cleanUpReqFiles(req) - - const videoFile: express.VideoUploadFile = req.files['videofile'][0] - const user = res.locals.oauth.token.User - - if ( - !await commonVideoChecksPass({ req, res, user, videoFileSize: videoFile.size, files: req.files }) || - !isValidPasswordProtectedPrivacy(req, res) || - !await addDurationToVideoFileIfNeeded({ videoFile, res, middlewareName: 'videosAddvideosAddLegacyValidatorResumableValidator' }) || - !await isVideoFileAccepted({ req, res, videoFile, hook: 'filter:api.video.upload.accept.result' }) - ) { - return cleanUpReqFiles(req) - } - - return next() - } -]) - -/** - * Gets called after the last PUT request - */ -const videosAddResumableValidator = [ - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - const user = res.locals.oauth.token.User - const body: express.CustomUploadXFile = req.body - const file = { ...body, duration: undefined, path: getResumableUploadPath(body.name), filename: body.metadata.filename } - const cleanup = () => uploadx.storage.delete(file).catch(err => logger.error('Cannot delete the file %s', file.name, { err })) - - const uploadId = req.query.upload_id - const sessionExists = await Redis.Instance.doesUploadSessionExist(uploadId) - - if (sessionExists) { - const sessionResponse = await Redis.Instance.getUploadSession(uploadId) - - if (!sessionResponse) { - res.setHeader('Retry-After', 300) // ask to retry after 5 min, knowing the upload_id is kept for up to 15 min after completion - - return res.fail({ - status: HttpStatusCode.SERVICE_UNAVAILABLE_503, - message: 'The upload is already being processed' - }) - } - - const videoStillExists = await VideoModel.load(sessionResponse.video.id) - - if (videoStillExists) { - if (isTestInstance()) { - res.setHeader('x-resumable-upload-cached', 'true') - } - - return res.json(sessionResponse) - } - } - - await Redis.Instance.setUploadSession(uploadId) - - if (!await doesVideoChannelOfAccountExist(file.metadata.channelId, user, res)) return cleanup() - if (!await addDurationToVideoFileIfNeeded({ videoFile: file, res, middlewareName: 'videosAddResumableValidator' })) return cleanup() - if (!await isVideoFileAccepted({ req, res, videoFile: file, hook: 'filter:api.video.upload.accept.result' })) return cleanup() - - res.locals.uploadVideoFileResumable = { ...file, originalname: file.filename } - - return next() - } -] - -/** - * File is created in POST initialisation, and its body is saved as a 'metadata' field is saved by uploadx for later use. - * see https://github.com/kukhariev/node-uploadx/blob/dc9fb4a8ac5a6f481902588e93062f591ec6ef03/packages/core/src/handlers/uploadx.ts - * - * Uploadx doesn't use next() until the upload completes, so this middleware has to be placed before uploadx - * see https://github.com/kukhariev/node-uploadx/blob/dc9fb4a8ac5a6f481902588e93062f591ec6ef03/packages/core/src/handlers/base-handler.ts - * - */ -const videosAddResumableInitValidator = getCommonVideoEditAttributes().concat([ - body('filename') - .exists(), - body('name') - .trim() - .custom(isVideoNameValid).withMessage( - `Should have a video name between ${CONSTRAINTS_FIELDS.VIDEOS.NAME.min} and ${CONSTRAINTS_FIELDS.VIDEOS.NAME.max} characters long` - ), - body('channelId') - .customSanitizer(toIntOrNull) - .custom(isIdValid), - body('videoPasswords') - .optional() - .isArray() - .withMessage('Video passwords should be an array.'), - - header('x-upload-content-length') - .isNumeric() - .exists() - .withMessage('Should specify the file length'), - header('x-upload-content-type') - .isString() - .exists() - .withMessage('Should specify the file mimetype'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - const videoFileMetadata = { - mimetype: req.headers['x-upload-content-type'] as string, - size: +req.headers['x-upload-content-length'], - originalname: req.body.filename - } - - const user = res.locals.oauth.token.User - const cleanup = () => cleanUpReqFiles(req) - - logger.debug('Checking videosAddResumableInitValidator parameters and headers', { - parameters: req.body, - headers: req.headers, - files: req.files - }) - - if (areValidationErrors(req, res, { omitLog: true })) return cleanup() - - const files = { videofile: [ videoFileMetadata ] } - if (!await commonVideoChecksPass({ req, res, user, videoFileSize: videoFileMetadata.size, files })) return cleanup() - - if (!isValidPasswordProtectedPrivacy(req, res)) return cleanup() - - // Multer required unsetting the Content-Type, now we can set it for node-uploadx - req.headers['content-type'] = 'application/json; charset=utf-8' - - // Place thumbnail/previewfile in metadata so that uploadx saves it in .META - if (req.files?.['previewfile']) req.body.previewfile = req.files['previewfile'] - if (req.files?.['thumbnailfile']) req.body.thumbnailfile = req.files['thumbnailfile'] - - return next() - } -]) - -const videosUpdateValidator = getCommonVideoEditAttributes().concat([ - isValidVideoIdParam('id'), - - body('name') - .optional() - .trim() - .custom(isVideoNameValid).withMessage( - `Should have a video name between ${CONSTRAINTS_FIELDS.VIDEOS.NAME.min} and ${CONSTRAINTS_FIELDS.VIDEOS.NAME.max} characters long` - ), - body('channelId') - .optional() - .customSanitizer(toIntOrNull) - .custom(isIdValid), - body('videoPasswords') - .optional() - .isArray() - .withMessage('Video passwords should be an array.'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return cleanUpReqFiles(req) - if (areErrorsInScheduleUpdate(req, res)) return cleanUpReqFiles(req) - if (!await doesVideoExist(req.params.id, res)) return cleanUpReqFiles(req) - - if (!isValidPasswordProtectedPrivacy(req, res)) return cleanUpReqFiles(req) - - const video = getVideoWithAttributes(res) - if (video.isLive && video.privacy !== req.body.privacy && video.state !== VideoState.WAITING_FOR_LIVE) { - return res.fail({ message: 'Cannot update privacy of a live that has already started' }) - } - - // Check if the user who did the request is able to update the video - const user = res.locals.oauth.token.User - if (!checkUserCanManageVideo(user, res.locals.videoAll, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req) - - if (req.body.channelId && !await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) - - return next() - } -]) - -async function checkVideoFollowConstraints (req: express.Request, res: express.Response, next: express.NextFunction) { - const video = getVideoWithAttributes(res) - - // Anybody can watch local videos - if (video.isOwned() === true) return next() - - // Logged user - if (res.locals.oauth) { - // Users can search or watch remote videos - if (CONFIG.SEARCH.REMOTE_URI.USERS === true) return next() - } - - // Anybody can search or watch remote videos - if (CONFIG.SEARCH.REMOTE_URI.ANONYMOUS === true) return next() - - // Check our instance follows an actor that shared this video - const serverActor = await getServerActor() - if (await VideoModel.checkVideoHasInstanceFollow(video.id, serverActor.id) === true) return next() - - return res.fail({ - status: HttpStatusCode.FORBIDDEN_403, - message: 'Cannot get this video regarding follow constraints', - type: ServerErrorCode.DOES_NOT_RESPECT_FOLLOW_CONSTRAINTS, - data: { - originUrl: video.url - } - }) -} - -const videosCustomGetValidator = (fetchType: 'for-api' | 'all' | 'only-video' | 'only-immutable-attributes') => { - return [ - isValidVideoIdParam('id'), - - isValidVideoPasswordHeader(), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.id, res, fetchType)) return - - // Controllers does not need to check video rights - if (fetchType === 'only-immutable-attributes') return next() - - const video = getVideoWithAttributes(res) as MVideoFullLight - - if (!await checkCanSeeVideo({ req, res, video, paramId: req.params.id })) return - - return next() - } - ] -} - -const videosGetValidator = videosCustomGetValidator('all') - -const videoFileMetadataGetValidator = getCommonVideoEditAttributes().concat([ - isValidVideoIdParam('id'), - - param('videoFileId') - .custom(isIdValid).not().isEmpty().withMessage('Should have a valid videoFileId'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoFileOfVideoExist(+req.params.videoFileId, req.params.id, res)) return - - return next() - } -]) - -const videosDownloadValidator = [ - isValidVideoIdParam('id'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.id, res, 'all')) return - - const video = getVideoWithAttributes(res) - - if (!await checkCanAccessVideoStaticFiles({ req, res, video, paramId: req.params.id })) return - - return next() - } -] - -const videosRemoveValidator = [ - isValidVideoIdParam('id'), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.id, res)) return - - // Check if the user who did the request is able to delete the video - if (!checkUserCanManageVideo(res.locals.oauth.token.User, res.locals.videoAll, UserRight.REMOVE_ANY_VIDEO, res)) return - - return next() - } -] - -const videosOverviewValidator = [ - query('page') - .optional() - .isInt({ min: 1, max: OVERVIEWS.VIDEOS.SAMPLES_COUNT }), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - return next() - } -] - -function getCommonVideoEditAttributes () { - return [ - body('thumbnailfile') - .custom((value, { req }) => isVideoImageValid(req.files, 'thumbnailfile')).withMessage( - 'This thumbnail file is not supported or too large. Please, make sure it is of the following type: ' + - CONSTRAINTS_FIELDS.VIDEOS.IMAGE.EXTNAME.join(', ') - ), - body('previewfile') - .custom((value, { req }) => isVideoImageValid(req.files, 'previewfile')).withMessage( - 'This preview file is not supported or too large. Please, make sure it is of the following type: ' + - CONSTRAINTS_FIELDS.VIDEOS.IMAGE.EXTNAME.join(', ') - ), - - body('category') - .optional() - .customSanitizer(toIntOrNull) - .custom(isVideoCategoryValid), - body('licence') - .optional() - .customSanitizer(toIntOrNull) - .custom(isVideoLicenceValid), - body('language') - .optional() - .customSanitizer(toValueOrNull) - .custom(isVideoLanguageValid), - body('nsfw') - .optional() - .customSanitizer(toBooleanOrNull) - .custom(isBooleanValid).withMessage('Should have a valid nsfw boolean'), - body('waitTranscoding') - .optional() - .customSanitizer(toBooleanOrNull) - .custom(isBooleanValid).withMessage('Should have a valid waitTranscoding boolean'), - body('privacy') - .optional() - .customSanitizer(toIntOrNull) - .custom(isVideoPrivacyValid), - body('description') - .optional() - .customSanitizer(toValueOrNull) - .custom(isVideoDescriptionValid), - body('support') - .optional() - .customSanitizer(toValueOrNull) - .custom(isVideoSupportValid), - body('tags') - .optional() - .customSanitizer(toValueOrNull) - .custom(areVideoTagsValid) - .withMessage( - `Should have an array of up to ${CONSTRAINTS_FIELDS.VIDEOS.TAGS.max} tags between ` + - `${CONSTRAINTS_FIELDS.VIDEOS.TAG.min} and ${CONSTRAINTS_FIELDS.VIDEOS.TAG.max} characters each` - ), - body('commentsEnabled') - .optional() - .customSanitizer(toBooleanOrNull) - .custom(isBooleanValid).withMessage('Should have commentsEnabled boolean'), - body('downloadEnabled') - .optional() - .customSanitizer(toBooleanOrNull) - .custom(isBooleanValid).withMessage('Should have downloadEnabled boolean'), - body('originallyPublishedAt') - .optional() - .customSanitizer(toValueOrNull) - .custom(isVideoOriginallyPublishedAtValid), - body('scheduleUpdate') - .optional() - .customSanitizer(toValueOrNull), - body('scheduleUpdate.updateAt') - .optional() - .custom(isDateValid).withMessage('Should have a schedule update date that conforms to ISO 8601'), - body('scheduleUpdate.privacy') - .optional() - .customSanitizer(toIntOrNull) - .custom(isScheduleVideoUpdatePrivacyValid) - ] as (ValidationChain | ExpressPromiseHandler)[] -} - -const commonVideosFiltersValidator = [ - query('categoryOneOf') - .optional() - .customSanitizer(arrayify) - .custom(isNumberArray).withMessage('Should have a valid categoryOneOf array'), - query('licenceOneOf') - .optional() - .customSanitizer(arrayify) - .custom(isNumberArray).withMessage('Should have a valid licenceOneOf array'), - query('languageOneOf') - .optional() - .customSanitizer(arrayify) - .custom(isStringArray).withMessage('Should have a valid languageOneOf array'), - query('privacyOneOf') - .optional() - .customSanitizer(arrayify) - .custom(isNumberArray).withMessage('Should have a valid privacyOneOf array'), - query('tagsOneOf') - .optional() - .customSanitizer(arrayify) - .custom(isStringArray).withMessage('Should have a valid tagsOneOf array'), - query('tagsAllOf') - .optional() - .customSanitizer(arrayify) - .custom(isStringArray).withMessage('Should have a valid tagsAllOf array'), - query('nsfw') - .optional() - .custom(isBooleanBothQueryValid), - query('isLive') - .optional() - .customSanitizer(toBooleanOrNull) - .custom(isBooleanValid).withMessage('Should have a valid isLive boolean'), - query('include') - .optional() - .custom(isVideoIncludeValid), - query('isLocal') - .optional() - .customSanitizer(toBooleanOrNull) - .custom(isBooleanValid).withMessage('Should have a valid isLocal boolean'), - query('hasHLSFiles') - .optional() - .customSanitizer(toBooleanOrNull) - .custom(isBooleanValid).withMessage('Should have a valid hasHLSFiles boolean'), - query('hasWebtorrentFiles') // TODO: remove in v7 - .optional() - .customSanitizer(toBooleanOrNull) - .custom(isBooleanValid).withMessage('Should have a valid hasWebtorrentFiles boolean'), - query('hasWebVideoFiles') - .optional() - .customSanitizer(toBooleanOrNull) - .custom(isBooleanValid).withMessage('Should have a valid hasWebVideoFiles boolean'), - query('skipCount') - .optional() - .customSanitizer(toBooleanOrNull) - .custom(isBooleanValid).withMessage('Should have a valid skipCount boolean'), - query('search') - .optional() - .custom(exists), - query('excludeAlreadyWatched') - .optional() - .customSanitizer(toBooleanOrNull) - .isBoolean().withMessage('Should be a valid excludeAlreadyWatched boolean'), - - (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - const user = res.locals.oauth?.token.User - - if ((!user || user.hasRight(UserRight.SEE_ALL_VIDEOS) !== true)) { - if (req.query.include || req.query.privacyOneOf) { - return res.fail({ - status: HttpStatusCode.UNAUTHORIZED_401, - message: 'You are not allowed to see all videos.' - }) - } - } - - if (!user && exists(req.query.excludeAlreadyWatched)) { - res.fail({ - status: HttpStatusCode.BAD_REQUEST_400, - message: 'Cannot use excludeAlreadyWatched parameter when auth token is not provided' - }) - return false - } - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - videosAddLegacyValidator, - videosAddResumableValidator, - videosAddResumableInitValidator, - - videosUpdateValidator, - videosGetValidator, - videoFileMetadataGetValidator, - videosDownloadValidator, - checkVideoFollowConstraints, - videosCustomGetValidator, - videosRemoveValidator, - - getCommonVideoEditAttributes, - - commonVideosFiltersValidator, - - videosOverviewValidator -} - -// --------------------------------------------------------------------------- - -function areErrorsInScheduleUpdate (req: express.Request, res: express.Response) { - if (req.body.scheduleUpdate) { - if (!req.body.scheduleUpdate.updateAt) { - logger.warn('Invalid parameters: scheduleUpdate.updateAt is mandatory.') - - res.fail({ message: 'Schedule update at is mandatory.' }) - return true - } - } - - return false -} - -async function commonVideoChecksPass (options: { - req: express.Request - res: express.Response - user: MUserAccountId - videoFileSize: number - files: express.UploadFilesForCheck -}): Promise { - const { req, res, user } = options - - if (areErrorsInScheduleUpdate(req, res)) return false - - if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return false - - if (!await commonVideoFileChecks(options)) return false - - return true -} diff --git a/server/middlewares/validators/webfinger.ts b/server/middlewares/validators/webfinger.ts deleted file mode 100644 index dcfba99fa..000000000 --- a/server/middlewares/validators/webfinger.ts +++ /dev/null @@ -1,37 +0,0 @@ -import express from 'express' -import { query } from 'express-validator' -import { HttpStatusCode } from '../../../shared/models/http/http-error-codes' -import { isWebfingerLocalResourceValid } from '../../helpers/custom-validators/webfinger' -import { getHostWithPort } from '../../helpers/express-utils' -import { ActorModel } from '../../models/actor/actor' -import { areValidationErrors } from './shared' - -const webfingerValidator = [ - query('resource') - .custom(isWebfingerLocalResourceValid), - - async (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (areValidationErrors(req, res)) return - - // Remove 'acct:' from the beginning of the string - const nameWithHost = getHostWithPort(req.query.resource.substr(5)) - const [ name ] = nameWithHost.split('@') - - const actor = await ActorModel.loadLocalUrlByName(name) - if (!actor) { - return res.fail({ - status: HttpStatusCode.NOT_FOUND_404, - message: 'Actor not found' - }) - } - - res.locals.actorUrl = actor - return next() - } -] - -// --------------------------------------------------------------------------- - -export { - webfingerValidator -} diff --git a/server/models/abuse/abuse-message.ts b/server/models/abuse/abuse-message.ts deleted file mode 100644 index 14a5bffa2..000000000 --- a/server/models/abuse/abuse-message.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { FindOptions } from 'sequelize' -import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { isAbuseMessageValid } from '@server/helpers/custom-validators/abuses' -import { MAbuseMessage, MAbuseMessageFormattable } from '@server/types/models' -import { AbuseMessage } from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { AccountModel, ScopeNames as AccountScopeNames } from '../account/account' -import { getSort, throwIfNotValid } from '../shared' -import { AbuseModel } from './abuse' - -@Table({ - tableName: 'abuseMessage', - indexes: [ - { - fields: [ 'abuseId' ] - }, - { - fields: [ 'accountId' ] - } - ] -}) -export class AbuseMessageModel extends Model>> { - - @AllowNull(false) - @Is('AbuseMessage', value => throwIfNotValid(value, isAbuseMessageValid, 'message')) - @Column(DataType.TEXT) - message: string - - @AllowNull(false) - @Column - byModerator: boolean - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => AccountModel) - @Column - accountId: number - - @BelongsTo(() => AccountModel, { - foreignKey: { - name: 'accountId', - allowNull: true - }, - onDelete: 'set null' - }) - Account: AccountModel - - @ForeignKey(() => AbuseModel) - @Column - abuseId: number - - @BelongsTo(() => AbuseModel, { - foreignKey: { - name: 'abuseId', - allowNull: false - }, - onDelete: 'cascade' - }) - Abuse: AbuseModel - - static listForApi (abuseId: number) { - const getQuery = (forCount: boolean) => { - const query: FindOptions = { - where: { abuseId }, - order: getSort('createdAt') - } - - if (forCount !== true) { - query.include = [ - { - model: AccountModel.scope(AccountScopeNames.SUMMARY), - required: false - } - ] - } - - return query - } - - return Promise.all([ - AbuseMessageModel.count(getQuery(true)), - AbuseMessageModel.findAll(getQuery(false)) - ]).then(([ total, data ]) => ({ total, data })) - } - - static loadByIdAndAbuseId (messageId: number, abuseId: number): Promise { - return AbuseMessageModel.findOne({ - where: { - id: messageId, - abuseId - } - }) - } - - toFormattedJSON (this: MAbuseMessageFormattable): AbuseMessage { - const account = this.Account - ? this.Account.toFormattedSummaryJSON() - : null - - return { - id: this.id, - createdAt: this.createdAt, - - byModerator: this.byModerator, - message: this.message, - - account - } - } -} diff --git a/server/models/abuse/abuse.ts b/server/models/abuse/abuse.ts deleted file mode 100644 index 4ce40bf2f..000000000 --- a/server/models/abuse/abuse.ts +++ /dev/null @@ -1,624 +0,0 @@ -import { invert } from 'lodash' -import { literal, Op, QueryTypes } from 'sequelize' -import { - AllowNull, - BelongsTo, - Column, - CreatedAt, - DataType, - Default, - ForeignKey, - HasOne, - Is, - Model, - Scopes, - Table, - UpdatedAt -} from 'sequelize-typescript' -import { isAbuseModerationCommentValid, isAbuseReasonValid, isAbuseStateValid } from '@server/helpers/custom-validators/abuses' -import { abusePredefinedReasonsMap } from '@shared/core-utils' -import { - AbuseFilter, - AbuseObject, - AbusePredefinedReasons, - AbusePredefinedReasonsString, - AbuseState, - AbuseVideoIs, - AdminAbuse, - AdminVideoAbuse, - AdminVideoCommentAbuse, - UserAbuse, - UserVideoAbuse -} from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { ABUSE_STATES, CONSTRAINTS_FIELDS } from '../../initializers/constants' -import { MAbuseAdminFormattable, MAbuseAP, MAbuseFull, MAbuseReporter, MAbuseUserFormattable, MUserAccountId } from '../../types/models' -import { AccountModel, ScopeNames as AccountScopeNames, SummaryOptions as AccountSummaryOptions } from '../account/account' -import { getSort, throwIfNotValid } from '../shared' -import { ThumbnailModel } from '../video/thumbnail' -import { ScopeNames as VideoScopeNames, VideoModel } from '../video/video' -import { VideoBlacklistModel } from '../video/video-blacklist' -import { ScopeNames as VideoChannelScopeNames, SummaryOptions as ChannelSummaryOptions, VideoChannelModel } from '../video/video-channel' -import { ScopeNames as CommentScopeNames, VideoCommentModel } from '../video/video-comment' -import { buildAbuseListQuery, BuildAbusesQueryOptions } from './sql/abuse-query-builder' -import { VideoAbuseModel } from './video-abuse' -import { VideoCommentAbuseModel } from './video-comment-abuse' - -export enum ScopeNames { - FOR_API = 'FOR_API' -} - -@Scopes(() => ({ - [ScopeNames.FOR_API]: () => { - return { - attributes: { - include: [ - [ - literal( - '(' + - 'SELECT count(*) ' + - 'FROM "abuseMessage" ' + - 'WHERE "abuseId" = "AbuseModel"."id"' + - ')' - ), - 'countMessages' - ], - [ - // we don't care about this count for deleted videos, so there are not included - literal( - '(' + - 'SELECT count(*) ' + - 'FROM "videoAbuse" ' + - 'WHERE "videoId" = "VideoAbuse"."videoId" AND "videoId" IS NOT NULL' + - ')' - ), - 'countReportsForVideo' - ], - [ - // we don't care about this count for deleted videos, so there are not included - literal( - '(' + - 'SELECT t.nth ' + - 'FROM ( ' + - 'SELECT id, ' + - 'row_number() OVER (PARTITION BY "videoId" ORDER BY "createdAt") AS nth ' + - 'FROM "videoAbuse" ' + - ') t ' + - 'WHERE t.id = "VideoAbuse".id AND t.id IS NOT NULL' + - ')' - ), - 'nthReportForVideo' - ], - [ - literal( - '(' + - 'SELECT count("abuse"."id") ' + - 'FROM "abuse" ' + - 'WHERE "abuse"."reporterAccountId" = "AbuseModel"."reporterAccountId"' + - ')' - ), - 'countReportsForReporter' - ], - [ - literal( - '(' + - 'SELECT count("abuse"."id") ' + - 'FROM "abuse" ' + - 'WHERE "abuse"."flaggedAccountId" = "AbuseModel"."flaggedAccountId"' + - ')' - ), - 'countReportsForReportee' - ] - ] - }, - include: [ - { - model: AccountModel.scope({ - method: [ - AccountScopeNames.SUMMARY, - { actorRequired: false } as AccountSummaryOptions - ] - }), - as: 'ReporterAccount' - }, - { - model: AccountModel.scope({ - method: [ - AccountScopeNames.SUMMARY, - { actorRequired: false } as AccountSummaryOptions - ] - }), - as: 'FlaggedAccount' - }, - { - model: VideoCommentAbuseModel.unscoped(), - include: [ - { - model: VideoCommentModel.unscoped(), - include: [ - { - model: VideoModel.unscoped(), - attributes: [ 'name', 'id', 'uuid' ] - } - ] - } - ] - }, - { - model: VideoAbuseModel.unscoped(), - include: [ - { - attributes: [ 'id', 'uuid', 'name', 'nsfw' ], - model: VideoModel.unscoped(), - include: [ - { - attributes: [ 'filename', 'fileUrl', 'type' ], - model: ThumbnailModel - }, - { - model: VideoChannelModel.scope({ - method: [ - VideoChannelScopeNames.SUMMARY, - { withAccount: false, actorRequired: false } as ChannelSummaryOptions - ] - }), - required: false - }, - { - attributes: [ 'id', 'reason', 'unfederated' ], - required: false, - model: VideoBlacklistModel - } - ] - } - ] - } - ] - } - } -})) -@Table({ - tableName: 'abuse', - indexes: [ - { - fields: [ 'reporterAccountId' ] - }, - { - fields: [ 'flaggedAccountId' ] - } - ] -}) -export class AbuseModel extends Model>> { - - @AllowNull(false) - @Default(null) - @Is('AbuseReason', value => throwIfNotValid(value, isAbuseReasonValid, 'reason')) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.ABUSES.REASON.max)) - reason: string - - @AllowNull(false) - @Default(null) - @Is('AbuseState', value => throwIfNotValid(value, isAbuseStateValid, 'state')) - @Column - state: AbuseState - - @AllowNull(true) - @Default(null) - @Is('AbuseModerationComment', value => throwIfNotValid(value, isAbuseModerationCommentValid, 'moderationComment', true)) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.ABUSES.MODERATION_COMMENT.max)) - moderationComment: string - - @AllowNull(true) - @Default(null) - @Column(DataType.ARRAY(DataType.INTEGER)) - predefinedReasons: AbusePredefinedReasons[] - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => AccountModel) - @Column - reporterAccountId: number - - @BelongsTo(() => AccountModel, { - foreignKey: { - name: 'reporterAccountId', - allowNull: true - }, - as: 'ReporterAccount', - onDelete: 'set null' - }) - ReporterAccount: AccountModel - - @ForeignKey(() => AccountModel) - @Column - flaggedAccountId: number - - @BelongsTo(() => AccountModel, { - foreignKey: { - name: 'flaggedAccountId', - allowNull: true - }, - as: 'FlaggedAccount', - onDelete: 'set null' - }) - FlaggedAccount: AccountModel - - @HasOne(() => VideoCommentAbuseModel, { - foreignKey: { - name: 'abuseId', - allowNull: false - }, - onDelete: 'cascade' - }) - VideoCommentAbuse: VideoCommentAbuseModel - - @HasOne(() => VideoAbuseModel, { - foreignKey: { - name: 'abuseId', - allowNull: false - }, - onDelete: 'cascade' - }) - VideoAbuse: VideoAbuseModel - - static loadByIdWithReporter (id: number): Promise { - const query = { - where: { - id - }, - include: [ - { - model: AccountModel, - as: 'ReporterAccount' - } - ] - } - - return AbuseModel.findOne(query) - } - - static loadFull (id: number): Promise { - const query = { - where: { - id - }, - include: [ - { - model: AccountModel.scope(AccountScopeNames.SUMMARY), - required: false, - as: 'ReporterAccount' - }, - { - model: AccountModel.scope(AccountScopeNames.SUMMARY), - as: 'FlaggedAccount' - }, - { - model: VideoAbuseModel, - required: false, - include: [ - { - model: VideoModel.scope([ VideoScopeNames.WITH_ACCOUNT_DETAILS ]) - } - ] - }, - { - model: VideoCommentAbuseModel, - required: false, - include: [ - { - model: VideoCommentModel.scope([ - CommentScopeNames.WITH_ACCOUNT - ]), - include: [ - { - model: VideoModel - } - ] - } - ] - } - ] - } - - return AbuseModel.findOne(query) - } - - static async listForAdminApi (parameters: { - start: number - count: number - sort: string - - filter?: AbuseFilter - - serverAccountId: number - user?: MUserAccountId - - id?: number - predefinedReason?: AbusePredefinedReasonsString - state?: AbuseState - videoIs?: AbuseVideoIs - - search?: string - searchReporter?: string - searchReportee?: string - searchVideo?: string - searchVideoChannel?: string - }) { - const { - start, - count, - sort, - search, - user, - serverAccountId, - state, - videoIs, - predefinedReason, - searchReportee, - searchVideo, - filter, - searchVideoChannel, - searchReporter, - id - } = parameters - - const userAccountId = user ? user.Account.id : undefined - const predefinedReasonId = predefinedReason ? abusePredefinedReasonsMap[predefinedReason] : undefined - - const queryOptions: BuildAbusesQueryOptions = { - start, - count, - sort, - id, - filter, - predefinedReasonId, - search, - state, - videoIs, - searchReportee, - searchVideo, - searchVideoChannel, - searchReporter, - serverAccountId, - userAccountId - } - - const [ total, data ] = await Promise.all([ - AbuseModel.internalCountForApi(queryOptions), - AbuseModel.internalListForApi(queryOptions) - ]) - - return { total, data } - } - - static async listForUserApi (parameters: { - user: MUserAccountId - - start: number - count: number - sort: string - - id?: number - search?: string - state?: AbuseState - }) { - const { - start, - count, - sort, - search, - user, - state, - id - } = parameters - - const queryOptions: BuildAbusesQueryOptions = { - start, - count, - sort, - id, - search, - state, - reporterAccountId: user.Account.id - } - - const [ total, data ] = await Promise.all([ - AbuseModel.internalCountForApi(queryOptions), - AbuseModel.internalListForApi(queryOptions) - ]) - - return { total, data } - } - - buildBaseVideoCommentAbuse (this: MAbuseUserFormattable) { - // Associated video comment could have been destroyed if the video has been deleted - if (!this.VideoCommentAbuse?.VideoComment) return null - - const entity = this.VideoCommentAbuse.VideoComment - - return { - id: entity.id, - threadId: entity.getThreadId(), - - text: entity.text ?? '', - - deleted: entity.isDeleted(), - - video: { - id: entity.Video.id, - name: entity.Video.name, - uuid: entity.Video.uuid - } - } - } - - buildBaseVideoAbuse (this: MAbuseUserFormattable): UserVideoAbuse { - if (!this.VideoAbuse) return null - - const abuseModel = this.VideoAbuse - const entity = abuseModel.Video || abuseModel.deletedVideo - - return { - id: entity.id, - uuid: entity.uuid, - name: entity.name, - nsfw: entity.nsfw, - - startAt: abuseModel.startAt, - endAt: abuseModel.endAt, - - deleted: !abuseModel.Video, - blacklisted: abuseModel.Video?.isBlacklisted() || false, - thumbnailPath: abuseModel.Video?.getMiniatureStaticPath(), - - channel: abuseModel.Video?.VideoChannel.toFormattedJSON() || abuseModel.deletedVideo?.channel - } - } - - buildBaseAbuse (this: MAbuseUserFormattable, countMessages: number): UserAbuse { - const predefinedReasons = AbuseModel.getPredefinedReasonsStrings(this.predefinedReasons) - - return { - id: this.id, - reason: this.reason, - predefinedReasons, - - flaggedAccount: this.FlaggedAccount - ? this.FlaggedAccount.toFormattedJSON() - : null, - - state: { - id: this.state, - label: AbuseModel.getStateLabel(this.state) - }, - - countMessages, - - createdAt: this.createdAt, - updatedAt: this.updatedAt - } - } - - toFormattedAdminJSON (this: MAbuseAdminFormattable): AdminAbuse { - const countReportsForVideo = this.get('countReportsForVideo') as number - const nthReportForVideo = this.get('nthReportForVideo') as number - - const countReportsForReporter = this.get('countReportsForReporter') as number - const countReportsForReportee = this.get('countReportsForReportee') as number - - const countMessages = this.get('countMessages') as number - - const baseVideo = this.buildBaseVideoAbuse() - const video: AdminVideoAbuse = baseVideo - ? Object.assign(baseVideo, { - countReports: countReportsForVideo, - nthReport: nthReportForVideo - }) - : null - - const comment: AdminVideoCommentAbuse = this.buildBaseVideoCommentAbuse() - - const abuse = this.buildBaseAbuse(countMessages || 0) - - return Object.assign(abuse, { - video, - comment, - - moderationComment: this.moderationComment, - - reporterAccount: this.ReporterAccount - ? this.ReporterAccount.toFormattedJSON() - : null, - - countReportsForReporter: (countReportsForReporter || 0), - countReportsForReportee: (countReportsForReportee || 0) - }) - } - - toFormattedUserJSON (this: MAbuseUserFormattable): UserAbuse { - const countMessages = this.get('countMessages') as number - - const video = this.buildBaseVideoAbuse() - const comment = this.buildBaseVideoCommentAbuse() - const abuse = this.buildBaseAbuse(countMessages || 0) - - return Object.assign(abuse, { - video, - comment - }) - } - - toActivityPubObject (this: MAbuseAP): AbuseObject { - const predefinedReasons = AbuseModel.getPredefinedReasonsStrings(this.predefinedReasons) - - const object = this.VideoAbuse?.Video?.url || this.VideoCommentAbuse?.VideoComment?.url || this.FlaggedAccount.Actor.url - - const startAt = this.VideoAbuse?.startAt - const endAt = this.VideoAbuse?.endAt - - return { - type: 'Flag' as 'Flag', - content: this.reason, - mediaType: 'text/markdown', - object, - tag: predefinedReasons.map(r => ({ - type: 'Hashtag' as 'Hashtag', - name: r - })), - startAt, - endAt - } - } - - private static async internalCountForApi (parameters: BuildAbusesQueryOptions) { - const { query, replacements } = buildAbuseListQuery(parameters, 'count') - const options = { - type: QueryTypes.SELECT as QueryTypes.SELECT, - replacements - } - - const [ { total } ] = await AbuseModel.sequelize.query<{ total: string }>(query, options) - if (total === null) return 0 - - return parseInt(total, 10) - } - - private static async internalListForApi (parameters: BuildAbusesQueryOptions) { - const { query, replacements } = buildAbuseListQuery(parameters, 'id') - const options = { - type: QueryTypes.SELECT as QueryTypes.SELECT, - replacements - } - - const rows = await AbuseModel.sequelize.query<{ id: string }>(query, options) - const ids = rows.map(r => r.id) - - if (ids.length === 0) return [] - - return AbuseModel.scope(ScopeNames.FOR_API) - .findAll({ - order: getSort(parameters.sort), - where: { - id: { - [Op.in]: ids - } - } - }) - } - - private static getStateLabel (id: number) { - return ABUSE_STATES[id] || 'Unknown' - } - - private static getPredefinedReasonsStrings (predefinedReasons: AbusePredefinedReasons[]): AbusePredefinedReasonsString[] { - const invertedPredefinedReasons = invert(abusePredefinedReasonsMap) - - return (predefinedReasons || []) - .map(r => invertedPredefinedReasons[r] as AbusePredefinedReasonsString) - .filter(v => !!v) - } -} diff --git a/server/models/abuse/sql/abuse-query-builder.ts b/server/models/abuse/sql/abuse-query-builder.ts deleted file mode 100644 index 282d4541a..000000000 --- a/server/models/abuse/sql/abuse-query-builder.ts +++ /dev/null @@ -1,167 +0,0 @@ - -import { exists } from '@server/helpers/custom-validators/misc' -import { forceNumber } from '@shared/core-utils' -import { AbuseFilter, AbuseState, AbuseVideoIs } from '@shared/models' -import { buildBlockedAccountSQL, buildSortDirectionAndField } from '../../shared' - -export type BuildAbusesQueryOptions = { - start: number - count: number - sort: string - - // search - search?: string - searchReporter?: string - searchReportee?: string - - // video related - searchVideo?: string - searchVideoChannel?: string - videoIs?: AbuseVideoIs - - // filters - id?: number - predefinedReasonId?: number - filter?: AbuseFilter - - state?: AbuseState - - // accountIds - serverAccountId?: number - userAccountId?: number - - reporterAccountId?: number -} - -function buildAbuseListQuery (options: BuildAbusesQueryOptions, type: 'count' | 'id') { - const whereAnd: string[] = [] - const replacements: any = {} - - const joins = [ - 'LEFT JOIN "videoAbuse" ON "videoAbuse"."abuseId" = "abuse"."id"', - 'LEFT JOIN "video" ON "videoAbuse"."videoId" = "video"."id"', - 'LEFT JOIN "videoBlacklist" ON "videoBlacklist"."videoId" = "video"."id"', - 'LEFT JOIN "videoChannel" ON "video"."channelId" = "videoChannel"."id"', - 'LEFT JOIN "account" "reporterAccount" ON "reporterAccount"."id" = "abuse"."reporterAccountId"', - 'LEFT JOIN "account" "flaggedAccount" ON "flaggedAccount"."id" = "abuse"."flaggedAccountId"', - 'LEFT JOIN "commentAbuse" ON "commentAbuse"."abuseId" = "abuse"."id"', - 'LEFT JOIN "videoComment" ON "commentAbuse"."videoCommentId" = "videoComment"."id"' - ] - - if (options.serverAccountId || options.userAccountId) { - whereAnd.push( - '"abuse"."reporterAccountId" IS NULL OR ' + - '"abuse"."reporterAccountId" NOT IN (' + buildBlockedAccountSQL([ options.serverAccountId, options.userAccountId ]) + ')' - ) - } - - if (options.reporterAccountId) { - whereAnd.push('"abuse"."reporterAccountId" = :reporterAccountId') - replacements.reporterAccountId = options.reporterAccountId - } - - if (options.search) { - const searchWhereOr = [ - '"video"."name" ILIKE :search', - '"videoChannel"."name" ILIKE :search', - `"videoAbuse"."deletedVideo"->>'name' ILIKE :search`, - `"videoAbuse"."deletedVideo"->'channel'->>'displayName' ILIKE :search`, - '"reporterAccount"."name" ILIKE :search', - '"flaggedAccount"."name" ILIKE :search' - ] - - replacements.search = `%${options.search}%` - whereAnd.push('(' + searchWhereOr.join(' OR ') + ')') - } - - if (options.searchVideo) { - whereAnd.push('"video"."name" ILIKE :searchVideo') - replacements.searchVideo = `%${options.searchVideo}%` - } - - if (options.searchVideoChannel) { - whereAnd.push('"videoChannel"."name" ILIKE :searchVideoChannel') - replacements.searchVideoChannel = `%${options.searchVideoChannel}%` - } - - if (options.id) { - whereAnd.push('"abuse"."id" = :id') - replacements.id = options.id - } - - if (options.state) { - whereAnd.push('"abuse"."state" = :state') - replacements.state = options.state - } - - if (options.videoIs === 'deleted') { - whereAnd.push('"videoAbuse"."deletedVideo" IS NOT NULL') - } else if (options.videoIs === 'blacklisted') { - whereAnd.push('"videoBlacklist"."id" IS NOT NULL') - } - - if (options.predefinedReasonId) { - whereAnd.push(':predefinedReasonId = ANY("abuse"."predefinedReasons")') - replacements.predefinedReasonId = options.predefinedReasonId - } - - if (options.filter === 'video') { - whereAnd.push('"videoAbuse"."id" IS NOT NULL') - } else if (options.filter === 'comment') { - whereAnd.push('"commentAbuse"."id" IS NOT NULL') - } else if (options.filter === 'account') { - whereAnd.push('"videoAbuse"."id" IS NULL AND "commentAbuse"."id" IS NULL') - } - - if (options.searchReporter) { - whereAnd.push('"reporterAccount"."name" ILIKE :searchReporter') - replacements.searchReporter = `%${options.searchReporter}%` - } - - if (options.searchReportee) { - whereAnd.push('"flaggedAccount"."name" ILIKE :searchReportee') - replacements.searchReportee = `%${options.searchReportee}%` - } - - const prefix = type === 'count' - ? 'SELECT COUNT("abuse"."id") AS "total"' - : 'SELECT "abuse"."id" ' - - let suffix = '' - if (type !== 'count') { - - if (options.sort) { - const order = buildAbuseOrder(options.sort) - suffix += `${order} ` - } - - if (exists(options.count)) { - const count = forceNumber(options.count) - suffix += `LIMIT ${count} ` - } - - if (exists(options.start)) { - const start = forceNumber(options.start) - suffix += `OFFSET ${start} ` - } - } - - const where = whereAnd.length !== 0 - ? `WHERE ${whereAnd.join(' AND ')}` - : '' - - return { - query: `${prefix} FROM "abuse" ${joins.join(' ')} ${where} ${suffix}`, - replacements - } -} - -function buildAbuseOrder (value: string) { - const { direction, field } = buildSortDirectionAndField(value) - - return `ORDER BY "abuse"."${field}" ${direction}` -} - -export { - buildAbuseListQuery -} diff --git a/server/models/abuse/video-abuse.ts b/server/models/abuse/video-abuse.ts deleted file mode 100644 index 773a9ebba..000000000 --- a/server/models/abuse/video-abuse.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { AttributesOnly } from '@shared/typescript-utils' -import { VideoDetails } from '@shared/models' -import { VideoModel } from '../video/video' -import { AbuseModel } from './abuse' - -@Table({ - tableName: 'videoAbuse', - indexes: [ - { - fields: [ 'abuseId' ] - }, - { - fields: [ 'videoId' ] - } - ] -}) -export class VideoAbuseModel extends Model>> { - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @AllowNull(true) - @Default(null) - @Column - startAt: number - - @AllowNull(true) - @Default(null) - @Column - endAt: number - - @AllowNull(true) - @Default(null) - @Column(DataType.JSONB) - deletedVideo: VideoDetails - - @ForeignKey(() => AbuseModel) - @Column - abuseId: number - - @BelongsTo(() => AbuseModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade' - }) - Abuse: AbuseModel - - @ForeignKey(() => VideoModel) - @Column - videoId: number - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'set null' - }) - Video: VideoModel -} diff --git a/server/models/abuse/video-comment-abuse.ts b/server/models/abuse/video-comment-abuse.ts deleted file mode 100644 index 337aaaa58..000000000 --- a/server/models/abuse/video-comment-abuse.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { BelongsTo, Column, CreatedAt, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { AttributesOnly } from '@shared/typescript-utils' -import { VideoCommentModel } from '../video/video-comment' -import { AbuseModel } from './abuse' - -@Table({ - tableName: 'commentAbuse', - indexes: [ - { - fields: [ 'abuseId' ] - }, - { - fields: [ 'videoCommentId' ] - } - ] -}) -export class VideoCommentAbuseModel extends Model>> { - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => AbuseModel) - @Column - abuseId: number - - @BelongsTo(() => AbuseModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade' - }) - Abuse: AbuseModel - - @ForeignKey(() => VideoCommentModel) - @Column - videoCommentId: number - - @BelongsTo(() => VideoCommentModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'set null' - }) - VideoComment: VideoCommentModel -} diff --git a/server/models/account/account-blocklist.ts b/server/models/account/account-blocklist.ts deleted file mode 100644 index f6212ff6e..000000000 --- a/server/models/account/account-blocklist.ts +++ /dev/null @@ -1,236 +0,0 @@ -import { FindOptions, Op, QueryTypes } from 'sequelize' -import { BelongsTo, Column, CreatedAt, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { handlesToNameAndHost } from '@server/helpers/actors' -import { MAccountBlocklist, MAccountBlocklistFormattable } from '@server/types/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { AccountBlock } from '../../../shared/models' -import { ActorModel } from '../actor/actor' -import { ServerModel } from '../server/server' -import { createSafeIn, getSort, searchAttribute } from '../shared' -import { AccountModel } from './account' - -@Table({ - tableName: 'accountBlocklist', - indexes: [ - { - fields: [ 'accountId', 'targetAccountId' ], - unique: true - }, - { - fields: [ 'targetAccountId' ] - } - ] -}) -export class AccountBlocklistModel extends Model>> { - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => AccountModel) - @Column - accountId: number - - @BelongsTo(() => AccountModel, { - foreignKey: { - name: 'accountId', - allowNull: false - }, - as: 'ByAccount', - onDelete: 'CASCADE' - }) - ByAccount: AccountModel - - @ForeignKey(() => AccountModel) - @Column - targetAccountId: number - - @BelongsTo(() => AccountModel, { - foreignKey: { - name: 'targetAccountId', - allowNull: false - }, - as: 'BlockedAccount', - onDelete: 'CASCADE' - }) - BlockedAccount: AccountModel - - static isAccountMutedByAccounts (accountIds: number[], targetAccountId: number) { - const query = { - attributes: [ 'accountId', 'id' ], - where: { - accountId: { - [Op.in]: accountIds - }, - targetAccountId - }, - raw: true - } - - return AccountBlocklistModel.unscoped() - .findAll(query) - .then(rows => { - const result: { [accountId: number]: boolean } = {} - - for (const accountId of accountIds) { - result[accountId] = !!rows.find(r => r.accountId === accountId) - } - - return result - }) - } - - static loadByAccountAndTarget (accountId: number, targetAccountId: number): Promise { - const query = { - where: { - accountId, - targetAccountId - } - } - - return AccountBlocklistModel.findOne(query) - } - - static listForApi (parameters: { - start: number - count: number - sort: string - search?: string - accountId: number - }) { - const { start, count, sort, search, accountId } = parameters - - const getQuery = (forCount: boolean) => { - const query: FindOptions = { - offset: start, - limit: count, - order: getSort(sort), - where: { accountId } - } - - if (search) { - Object.assign(query.where, { - [Op.or]: [ - searchAttribute(search, '$BlockedAccount.name$'), - searchAttribute(search, '$BlockedAccount.Actor.url$') - ] - }) - } - - if (forCount !== true) { - query.include = [ - { - model: AccountModel, - required: true, - as: 'ByAccount' - }, - { - model: AccountModel, - required: true, - as: 'BlockedAccount' - } - ] - } else if (search) { // We need some joins when counting with search - query.include = [ - { - model: AccountModel.unscoped(), - required: true, - as: 'BlockedAccount', - include: [ - { - model: ActorModel.unscoped(), - required: true - } - ] - } - ] - } - - return query - } - - return Promise.all([ - AccountBlocklistModel.count(getQuery(true)), - AccountBlocklistModel.findAll(getQuery(false)) - ]).then(([ total, data ]) => ({ total, data })) - } - - static listHandlesBlockedBy (accountIds: number[]): Promise { - const query = { - attributes: [ 'id' ], - where: { - accountId: { - [Op.in]: accountIds - } - }, - include: [ - { - attributes: [ 'id' ], - model: AccountModel.unscoped(), - required: true, - as: 'BlockedAccount', - include: [ - { - attributes: [ 'preferredUsername' ], - model: ActorModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'host' ], - model: ServerModel.unscoped(), - required: true - } - ] - } - ] - } - ] - } - - return AccountBlocklistModel.findAll(query) - .then(entries => entries.map(e => `${e.BlockedAccount.Actor.preferredUsername}@${e.BlockedAccount.Actor.Server.host}`)) - } - - static getBlockStatus (byAccountIds: number[], handles: string[]): Promise<{ name: string, host: string, accountId: number }[]> { - const sanitizedHandles = handlesToNameAndHost(handles) - - const localHandles = sanitizedHandles.filter(h => !h.host) - .map(h => h.name) - - const remoteHandles = sanitizedHandles.filter(h => !!h.host) - .map(h => ([ h.name, h.host ])) - - const handlesWhere: string[] = [] - - if (localHandles.length !== 0) { - handlesWhere.push(`("actor"."preferredUsername" IN (:localHandles) AND "server"."id" IS NULL)`) - } - - if (remoteHandles.length !== 0) { - handlesWhere.push(`(("actor"."preferredUsername", "server"."host") IN (:remoteHandles))`) - } - - const rawQuery = `SELECT "accountBlocklist"."accountId", "actor"."preferredUsername" AS "name", "server"."host" ` + - `FROM "accountBlocklist" ` + - `INNER JOIN "account" ON "account"."id" = "accountBlocklist"."targetAccountId" ` + - `INNER JOIN "actor" ON "actor"."id" = "account"."actorId" ` + - `LEFT JOIN "server" ON "server"."id" = "actor"."serverId" ` + - `WHERE "accountBlocklist"."accountId" IN (${createSafeIn(AccountBlocklistModel.sequelize, byAccountIds)}) ` + - `AND (${handlesWhere.join(' OR ')})` - - return AccountBlocklistModel.sequelize.query(rawQuery, { - type: QueryTypes.SELECT as QueryTypes.SELECT, - replacements: { byAccountIds, localHandles, remoteHandles } - }) - } - - toFormattedJSON (this: MAccountBlocklistFormattable): AccountBlock { - return { - byAccount: this.ByAccount.toFormattedJSON(), - blockedAccount: this.BlockedAccount.toFormattedJSON(), - createdAt: this.createdAt - } - } -} diff --git a/server/models/account/account-video-rate.ts b/server/models/account/account-video-rate.ts deleted file mode 100644 index 18ff07d53..000000000 --- a/server/models/account/account-video-rate.ts +++ /dev/null @@ -1,259 +0,0 @@ -import { FindOptions, Op, QueryTypes, Transaction } from 'sequelize' -import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { - MAccountVideoRate, - MAccountVideoRateAccountUrl, - MAccountVideoRateAccountVideo, - MAccountVideoRateFormattable -} from '@server/types/models' -import { AccountVideoRate, VideoRateType } from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' -import { CONSTRAINTS_FIELDS, VIDEO_RATE_TYPES } from '../../initializers/constants' -import { ActorModel } from '../actor/actor' -import { getSort, throwIfNotValid } from '../shared' -import { VideoModel } from '../video/video' -import { ScopeNames as VideoChannelScopeNames, SummaryOptions, VideoChannelModel } from '../video/video-channel' -import { AccountModel } from './account' - -/* - Account rates per video. -*/ -@Table({ - tableName: 'accountVideoRate', - indexes: [ - { - fields: [ 'videoId', 'accountId' ], - unique: true - }, - { - fields: [ 'videoId' ] - }, - { - fields: [ 'accountId' ] - }, - { - fields: [ 'videoId', 'type' ] - }, - { - fields: [ 'url' ], - unique: true - } - ] -}) -export class AccountVideoRateModel extends Model>> { - - @AllowNull(false) - @Column(DataType.ENUM(...Object.values(VIDEO_RATE_TYPES))) - type: VideoRateType - - @AllowNull(false) - @Is('AccountVideoRateUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url')) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_RATES.URL.max)) - url: string - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => VideoModel) - @Column - videoId: number - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'CASCADE' - }) - Video: VideoModel - - @ForeignKey(() => AccountModel) - @Column - accountId: number - - @BelongsTo(() => AccountModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'CASCADE' - }) - Account: AccountModel - - static load (accountId: number, videoId: number, transaction?: Transaction): Promise { - const options: FindOptions = { - where: { - accountId, - videoId - } - } - if (transaction) options.transaction = transaction - - return AccountVideoRateModel.findOne(options) - } - - static loadByAccountAndVideoOrUrl (accountId: number, videoId: number, url: string, t?: Transaction): Promise { - const options: FindOptions = { - where: { - [Op.or]: [ - { - accountId, - videoId - }, - { - url - } - ] - } - } - if (t) options.transaction = t - - return AccountVideoRateModel.findOne(options) - } - - static listByAccountForApi (options: { - start: number - count: number - sort: string - type?: string - accountId: number - }) { - const getQuery = (forCount: boolean) => { - const query: FindOptions = { - offset: options.start, - limit: options.count, - order: getSort(options.sort), - where: { - accountId: options.accountId - } - } - - if (options.type) query.where['type'] = options.type - - if (forCount !== true) { - query.include = [ - { - model: VideoModel, - required: true, - include: [ - { - model: VideoChannelModel.scope({ method: [ VideoChannelScopeNames.SUMMARY, { withAccount: true } as SummaryOptions ] }), - required: true - } - ] - } - ] - } - - return query - } - - return Promise.all([ - AccountVideoRateModel.count(getQuery(true)), - AccountVideoRateModel.findAll(getQuery(false)) - ]).then(([ total, data ]) => ({ total, data })) - } - - static listRemoteRateUrlsOfLocalVideos () { - const query = `SELECT "accountVideoRate".url FROM "accountVideoRate" ` + - `INNER JOIN account ON account.id = "accountVideoRate"."accountId" ` + - `INNER JOIN actor ON actor.id = account."actorId" AND actor."serverId" IS NOT NULL ` + - `INNER JOIN video ON video.id = "accountVideoRate"."videoId" AND video.remote IS FALSE` - - return AccountVideoRateModel.sequelize.query<{ url: string }>(query, { - type: QueryTypes.SELECT, - raw: true - }).then(rows => rows.map(r => r.url)) - } - - static loadLocalAndPopulateVideo ( - rateType: VideoRateType, - accountName: string, - videoId: number, - t?: Transaction - ): Promise { - const options: FindOptions = { - where: { - videoId, - type: rateType - }, - include: [ - { - model: AccountModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'id', 'url', 'followersUrl', 'preferredUsername' ], - model: ActorModel.unscoped(), - required: true, - where: { - [Op.and]: [ - ActorModel.wherePreferredUsername(accountName), - { serverId: null } - ] - } - } - ] - }, - { - model: VideoModel.unscoped(), - required: true - } - ] - } - if (t) options.transaction = t - - return AccountVideoRateModel.findOne(options) - } - - static loadByUrl (url: string, transaction: Transaction) { - const options: FindOptions = { - where: { - url - } - } - if (transaction) options.transaction = transaction - - return AccountVideoRateModel.findOne(options) - } - - static listAndCountAccountUrlsByVideoId (rateType: VideoRateType, videoId: number, start: number, count: number, t?: Transaction) { - const query = { - offset: start, - limit: count, - where: { - videoId, - type: rateType - }, - transaction: t, - include: [ - { - attributes: [ 'actorId' ], - model: AccountModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'url' ], - model: ActorModel.unscoped(), - required: true - } - ] - } - ] - } - - return Promise.all([ - AccountVideoRateModel.count(query), - AccountVideoRateModel.findAll(query) - ]).then(([ total, data ]) => ({ total, data })) - } - - toFormattedJSON (this: MAccountVideoRateFormattable): AccountVideoRate { - return { - video: this.Video.toFormattedJSON(), - rating: this.type - } - } -} diff --git a/server/models/account/account.ts b/server/models/account/account.ts deleted file mode 100644 index 8593f2f28..000000000 --- a/server/models/account/account.ts +++ /dev/null @@ -1,468 +0,0 @@ -import { FindOptions, Includeable, IncludeOptions, Op, Transaction, WhereOptions } from 'sequelize' -import { - AllowNull, - BeforeDestroy, - BelongsTo, - Column, - CreatedAt, - DataType, - Default, - DefaultScope, - ForeignKey, - HasMany, - Is, - Model, - Scopes, - Table, - UpdatedAt -} from 'sequelize-typescript' -import { ModelCache } from '@server/models/shared/model-cache' -import { AttributesOnly } from '@shared/typescript-utils' -import { Account, AccountSummary } from '../../../shared/models/actors' -import { isAccountDescriptionValid } from '../../helpers/custom-validators/accounts' -import { CONSTRAINTS_FIELDS, SERVER_ACTOR_NAME, WEBSERVER } from '../../initializers/constants' -import { sendDeleteActor } from '../../lib/activitypub/send/send-delete' -import { - MAccount, - MAccountActor, - MAccountAP, - MAccountDefault, - MAccountFormattable, - MAccountHost, - MAccountSummaryFormattable, - MChannelHost -} from '../../types/models' -import { ActorModel } from '../actor/actor' -import { ActorFollowModel } from '../actor/actor-follow' -import { ActorImageModel } from '../actor/actor-image' -import { ApplicationModel } from '../application/application' -import { ServerModel } from '../server/server' -import { ServerBlocklistModel } from '../server/server-blocklist' -import { buildSQLAttributes, getSort, throwIfNotValid } from '../shared' -import { UserModel } from '../user/user' -import { VideoModel } from '../video/video' -import { VideoChannelModel } from '../video/video-channel' -import { VideoCommentModel } from '../video/video-comment' -import { VideoPlaylistModel } from '../video/video-playlist' -import { AccountBlocklistModel } from './account-blocklist' - -export enum ScopeNames { - SUMMARY = 'SUMMARY' -} - -export type SummaryOptions = { - actorRequired?: boolean // Default: true - whereActor?: WhereOptions - whereServer?: WhereOptions - withAccountBlockerIds?: number[] - forCount?: boolean -} - -@DefaultScope(() => ({ - include: [ - { - model: ActorModel, // Default scope includes avatar and server - required: true - } - ] -})) -@Scopes(() => ({ - [ScopeNames.SUMMARY]: (options: SummaryOptions = {}) => { - const serverInclude: IncludeOptions = { - attributes: [ 'host' ], - model: ServerModel.unscoped(), - required: !!options.whereServer, - where: options.whereServer - } - - const actorInclude: Includeable = { - attributes: [ 'id', 'preferredUsername', 'url', 'serverId' ], - model: ActorModel.unscoped(), - required: options.actorRequired ?? true, - where: options.whereActor, - include: [ serverInclude ] - } - - if (options.forCount !== true) { - actorInclude.include.push({ - model: ActorImageModel, - as: 'Avatars', - required: false - }) - } - - const queryInclude: Includeable[] = [ - actorInclude - ] - - const query: FindOptions = { - attributes: [ 'id', 'name', 'actorId' ] - } - - if (options.withAccountBlockerIds) { - queryInclude.push({ - attributes: [ 'id' ], - model: AccountBlocklistModel.unscoped(), - as: 'BlockedBy', - required: false, - where: { - accountId: { - [Op.in]: options.withAccountBlockerIds - } - } - }) - - serverInclude.include = [ - { - attributes: [ 'id' ], - model: ServerBlocklistModel.unscoped(), - required: false, - where: { - accountId: { - [Op.in]: options.withAccountBlockerIds - } - } - } - ] - } - - query.include = queryInclude - - return query - } -})) -@Table({ - tableName: 'account', - indexes: [ - { - fields: [ 'actorId' ], - unique: true - }, - { - fields: [ 'applicationId' ] - }, - { - fields: [ 'userId' ] - } - ] -}) -export class AccountModel extends Model>> { - - @AllowNull(false) - @Column - name: string - - @AllowNull(true) - @Default(null) - @Is('AccountDescription', value => throwIfNotValid(value, isAccountDescriptionValid, 'description', true)) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.USERS.DESCRIPTION.max)) - description: string - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => ActorModel) - @Column - actorId: number - - @BelongsTo(() => ActorModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade' - }) - Actor: ActorModel - - @ForeignKey(() => UserModel) - @Column - userId: number - - @BelongsTo(() => UserModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'cascade' - }) - User: UserModel - - @ForeignKey(() => ApplicationModel) - @Column - applicationId: number - - @BelongsTo(() => ApplicationModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'cascade' - }) - Application: ApplicationModel - - @HasMany(() => VideoChannelModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade', - hooks: true - }) - VideoChannels: VideoChannelModel[] - - @HasMany(() => VideoPlaylistModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade', - hooks: true - }) - VideoPlaylists: VideoPlaylistModel[] - - @HasMany(() => VideoCommentModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'cascade', - hooks: true - }) - VideoComments: VideoCommentModel[] - - @HasMany(() => AccountBlocklistModel, { - foreignKey: { - name: 'targetAccountId', - allowNull: false - }, - as: 'BlockedBy', - onDelete: 'CASCADE' - }) - BlockedBy: AccountBlocklistModel[] - - @BeforeDestroy - static async sendDeleteIfOwned (instance: AccountModel, options) { - if (!instance.Actor) { - instance.Actor = await instance.$get('Actor', { transaction: options.transaction }) - } - - await ActorFollowModel.removeFollowsOf(instance.Actor.id, options.transaction) - - if (instance.isOwned()) { - return sendDeleteActor(instance.Actor, options.transaction) - } - - return undefined - } - - // --------------------------------------------------------------------------- - - static getSQLAttributes (tableName: string, aliasPrefix = '') { - return buildSQLAttributes({ - model: this, - tableName, - aliasPrefix - }) - } - - // --------------------------------------------------------------------------- - - static load (id: number, transaction?: Transaction): Promise { - return AccountModel.findByPk(id, { transaction }) - } - - static loadByNameWithHost (nameWithHost: string): Promise { - const [ accountName, host ] = nameWithHost.split('@') - - if (!host || host === WEBSERVER.HOST) return AccountModel.loadLocalByName(accountName) - - return AccountModel.loadByNameAndHost(accountName, host) - } - - static loadLocalByName (name: string): Promise { - const fun = () => { - const query = { - where: { - [Op.or]: [ - { - userId: { - [Op.ne]: null - } - }, - { - applicationId: { - [Op.ne]: null - } - } - ] - }, - include: [ - { - model: ActorModel, - required: true, - where: ActorModel.wherePreferredUsername(name) - } - ] - } - - return AccountModel.findOne(query) - } - - return ModelCache.Instance.doCache({ - cacheType: 'local-account-name', - key: name, - fun, - // The server actor never change, so we can easily cache it - whitelist: () => name === SERVER_ACTOR_NAME - }) - } - - static loadByNameAndHost (name: string, host: string): Promise { - const query = { - include: [ - { - model: ActorModel, - required: true, - where: ActorModel.wherePreferredUsername(name), - include: [ - { - model: ServerModel, - required: true, - where: { - host - } - } - ] - } - ] - } - - return AccountModel.findOne(query) - } - - static loadByUrl (url: string, transaction?: Transaction): Promise { - const query = { - include: [ - { - model: ActorModel, - required: true, - where: { - url - } - } - ], - transaction - } - - return AccountModel.findOne(query) - } - - static listForApi (start: number, count: number, sort: string) { - const query = { - offset: start, - limit: count, - order: getSort(sort) - } - - return Promise.all([ - AccountModel.count(), - AccountModel.findAll(query) - ]).then(([ total, data ]) => ({ total, data })) - } - - static loadAccountIdFromVideo (videoId: number): Promise { - const query = { - include: [ - { - attributes: [ 'id', 'accountId' ], - model: VideoChannelModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'id', 'channelId' ], - model: VideoModel.unscoped(), - where: { - id: videoId - } - } - ] - } - ] - } - - return AccountModel.findOne(query) - } - - static listLocalsForSitemap (sort: string): Promise { - const query = { - attributes: [ ], - offset: 0, - order: getSort(sort), - include: [ - { - attributes: [ 'preferredUsername', 'serverId' ], - model: ActorModel.unscoped(), - where: { - serverId: null - } - } - ] - } - - return AccountModel - .unscoped() - .findAll(query) - } - - toFormattedJSON (this: MAccountFormattable): Account { - return { - ...this.Actor.toFormattedJSON(), - - id: this.id, - displayName: this.getDisplayName(), - description: this.description, - updatedAt: this.updatedAt, - userId: this.userId ?? undefined - } - } - - toFormattedSummaryJSON (this: MAccountSummaryFormattable): AccountSummary { - const actor = this.Actor.toFormattedSummaryJSON() - - return { - id: this.id, - displayName: this.getDisplayName(), - - name: actor.name, - url: actor.url, - host: actor.host, - avatars: actor.avatars - } - } - - async toActivityPubObject (this: MAccountAP) { - const obj = await this.Actor.toActivityPubObject(this.name) - - return Object.assign(obj, { - summary: this.description - }) - } - - isOwned () { - return this.Actor.isOwned() - } - - isOutdated () { - return this.Actor.isOutdated() - } - - getDisplayName () { - return this.name - } - - // Avoid error when running this method on MAccount... | MChannel... - getClientUrl (this: MAccountHost | MChannelHost) { - return WEBSERVER.URL + '/a/' + this.Actor.getIdentifier() - } - - isBlocked () { - return this.BlockedBy && this.BlockedBy.length !== 0 - } -} diff --git a/server/models/account/actor-custom-page.ts b/server/models/account/actor-custom-page.ts deleted file mode 100644 index 893023181..000000000 --- a/server/models/account/actor-custom-page.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { CustomPage } from '@shared/models' -import { ActorModel } from '../actor/actor' -import { getServerActor } from '../application/application' - -@Table({ - tableName: 'actorCustomPage', - indexes: [ - { - fields: [ 'actorId', 'type' ], - unique: true - } - ] -}) -export class ActorCustomPageModel extends Model { - - @AllowNull(true) - @Column(DataType.TEXT) - content: string - - @AllowNull(false) - @Column - type: 'homepage' - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => ActorModel) - @Column - actorId: number - - @BelongsTo(() => ActorModel, { - foreignKey: { - name: 'actorId', - allowNull: false - }, - onDelete: 'cascade' - }) - Actor: ActorModel - - static async updateInstanceHomepage (content: string) { - const serverActor = await getServerActor() - - return ActorCustomPageModel.upsert({ - content, - actorId: serverActor.id, - type: 'homepage' - }) - } - - static async loadInstanceHomepage () { - const serverActor = await getServerActor() - - return ActorCustomPageModel.findOne({ - where: { - actorId: serverActor.id - } - }) - } - - toFormattedJSON (): CustomPage { - return { - content: this.content - } - } -} diff --git a/server/models/actor/actor-follow.ts b/server/models/actor/actor-follow.ts deleted file mode 100644 index 71ce9fa6f..000000000 --- a/server/models/actor/actor-follow.ts +++ /dev/null @@ -1,662 +0,0 @@ -import { difference } from 'lodash' -import { Attributes, FindOptions, Includeable, IncludeOptions, Op, QueryTypes, Transaction, WhereAttributeHash } from 'sequelize' -import { - AfterCreate, - AfterDestroy, - AfterUpdate, - AllowNull, - BelongsTo, - Column, - CreatedAt, - DataType, - Default, - ForeignKey, - Is, - IsInt, - Max, - Model, - Table, - UpdatedAt -} from 'sequelize-typescript' -import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc' -import { afterCommitIfTransaction } from '@server/helpers/database-utils' -import { getServerActor } from '@server/models/application/application' -import { - MActor, - MActorFollowActors, - MActorFollowActorsDefault, - MActorFollowActorsDefaultSubscription, - MActorFollowFollowingHost, - MActorFollowFormattable, - MActorFollowSubscriptions -} from '@server/types/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { FollowState } from '../../../shared/models/actors' -import { ActorFollow } from '../../../shared/models/actors/follow.model' -import { logger } from '../../helpers/logger' -import { ACTOR_FOLLOW_SCORE, CONSTRAINTS_FIELDS, FOLLOW_STATES, SERVER_ACTOR_NAME, SORTABLE_COLUMNS } from '../../initializers/constants' -import { AccountModel } from '../account/account' -import { ServerModel } from '../server/server' -import { buildSQLAttributes, createSafeIn, getSort, searchAttribute, throwIfNotValid } from '../shared' -import { doesExist } from '../shared/query' -import { VideoChannelModel } from '../video/video-channel' -import { ActorModel, unusedActorAttributesForAPI } from './actor' -import { InstanceListFollowersQueryBuilder, ListFollowersOptions } from './sql/instance-list-followers-query-builder' -import { InstanceListFollowingQueryBuilder, ListFollowingOptions } from './sql/instance-list-following-query-builder' - -@Table({ - tableName: 'actorFollow', - indexes: [ - { - fields: [ 'actorId' ] - }, - { - fields: [ 'targetActorId' ] - }, - { - fields: [ 'actorId', 'targetActorId' ], - unique: true - }, - { - fields: [ 'score' ] - }, - { - fields: [ 'url' ], - unique: true - } - ] -}) -export class ActorFollowModel extends Model>> { - - @AllowNull(false) - @Column(DataType.ENUM(...Object.values(FOLLOW_STATES))) - state: FollowState - - @AllowNull(false) - @Default(ACTOR_FOLLOW_SCORE.BASE) - @IsInt - @Max(ACTOR_FOLLOW_SCORE.MAX) - @Column - score: number - - // Allow null because we added this column in PeerTube v3, and don't want to generate fake URLs of remote follows - @AllowNull(true) - @Is('ActorFollowUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url')) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.COMMONS.URL.max)) - url: string - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => ActorModel) - @Column - actorId: number - - @BelongsTo(() => ActorModel, { - foreignKey: { - name: 'actorId', - allowNull: false - }, - as: 'ActorFollower', - onDelete: 'CASCADE' - }) - ActorFollower: ActorModel - - @ForeignKey(() => ActorModel) - @Column - targetActorId: number - - @BelongsTo(() => ActorModel, { - foreignKey: { - name: 'targetActorId', - allowNull: false - }, - as: 'ActorFollowing', - onDelete: 'CASCADE' - }) - ActorFollowing: ActorModel - - @AfterCreate - @AfterUpdate - static incrementFollowerAndFollowingCount (instance: ActorFollowModel, options: any) { - return afterCommitIfTransaction(options.transaction, () => { - return Promise.all([ - ActorModel.rebuildFollowsCount(instance.actorId, 'following'), - ActorModel.rebuildFollowsCount(instance.targetActorId, 'followers') - ]) - }) - } - - @AfterDestroy - static decrementFollowerAndFollowingCount (instance: ActorFollowModel, options: any) { - return afterCommitIfTransaction(options.transaction, () => { - return Promise.all([ - ActorModel.rebuildFollowsCount(instance.actorId, 'following'), - ActorModel.rebuildFollowsCount(instance.targetActorId, 'followers') - ]) - }) - } - - // --------------------------------------------------------------------------- - - static getSQLAttributes (tableName: string, aliasPrefix = '') { - return buildSQLAttributes({ - model: this, - tableName, - aliasPrefix - }) - } - - // --------------------------------------------------------------------------- - - /* - * @deprecated Use `findOrCreateCustom` instead - */ - static findOrCreate (): any { - throw new Error('Must not be called') - } - - // findOrCreate has issues with actor follow hooks - static async findOrCreateCustom (options: { - byActor: MActor - targetActor: MActor - activityId: string - state: FollowState - transaction: Transaction - }): Promise<[ MActorFollowActors, boolean ]> { - const { byActor, targetActor, activityId, state, transaction } = options - - let created = false - let actorFollow: MActorFollowActors = await ActorFollowModel.loadByActorAndTarget(byActor.id, targetActor.id, transaction) - - if (!actorFollow) { - created = true - - actorFollow = await ActorFollowModel.create({ - actorId: byActor.id, - targetActorId: targetActor.id, - url: activityId, - - state - }, { transaction }) - - actorFollow.ActorFollowing = targetActor - actorFollow.ActorFollower = byActor - } - - return [ actorFollow, created ] - } - - static removeFollowsOf (actorId: number, t?: Transaction) { - const query = { - where: { - [Op.or]: [ - { - actorId - }, - { - targetActorId: actorId - } - ] - }, - transaction: t - } - - return ActorFollowModel.destroy(query) - } - - // Remove actor follows with a score of 0 (too many requests where they were unreachable) - static async removeBadActorFollows () { - const actorFollows = await ActorFollowModel.listBadActorFollows() - - const actorFollowsRemovePromises = actorFollows.map(actorFollow => actorFollow.destroy()) - await Promise.all(actorFollowsRemovePromises) - - const numberOfActorFollowsRemoved = actorFollows.length - - if (numberOfActorFollowsRemoved) logger.info('Removed bad %d actor follows.', numberOfActorFollowsRemoved) - } - - static isFollowedBy (actorId: number, followerActorId: number) { - const query = `SELECT 1 FROM "actorFollow" ` + - `WHERE "actorId" = $followerActorId AND "targetActorId" = $actorId AND "state" = 'accepted' ` + - `LIMIT 1` - - return doesExist(this.sequelize, query, { actorId, followerActorId }) - } - - static loadByActorAndTarget (actorId: number, targetActorId: number, t?: Transaction): Promise { - const query = { - where: { - actorId, - targetActorId - }, - include: [ - { - model: ActorModel, - required: true, - as: 'ActorFollower' - }, - { - model: ActorModel, - required: true, - as: 'ActorFollowing' - } - ], - transaction: t - } - - return ActorFollowModel.findOne(query) - } - - static loadByActorAndTargetNameAndHostForAPI (options: { - actorId: number - targetName: string - targetHost: string - state?: FollowState - transaction?: Transaction - }): Promise { - const { actorId, targetHost, targetName, state, transaction } = options - - const actorFollowingPartInclude: IncludeOptions = { - model: ActorModel, - required: true, - as: 'ActorFollowing', - where: ActorModel.wherePreferredUsername(targetName), - include: [ - { - model: VideoChannelModel.unscoped(), - required: false - } - ] - } - - if (targetHost === null) { - actorFollowingPartInclude.where['serverId'] = null - } else { - actorFollowingPartInclude.include.push({ - model: ServerModel, - required: true, - where: { - host: targetHost - } - }) - } - - const where: WhereAttributeHash> = { actorId } - if (state) where.state = state - - const query: FindOptions> = { - where, - include: [ - actorFollowingPartInclude, - { - model: ActorModel, - required: true, - as: 'ActorFollower' - } - ], - transaction - } - - return ActorFollowModel.findOne(query) - } - - static listSubscriptionsOf (actorId: number, targets: { name: string, host?: string }[]): Promise { - const whereTab = targets - .map(t => { - if (t.host) { - return { - [Op.and]: [ - ActorModel.wherePreferredUsername(t.name), - { $host$: t.host } - ] - } - } - - return { - [Op.and]: [ - ActorModel.wherePreferredUsername(t.name), - { $serverId$: null } - ] - } - }) - - const query = { - attributes: [ 'id' ], - where: { - [Op.and]: [ - { - [Op.or]: whereTab - }, - { - state: 'accepted', - actorId - } - ] - }, - include: [ - { - attributes: [ 'preferredUsername' ], - model: ActorModel.unscoped(), - required: true, - as: 'ActorFollowing', - include: [ - { - attributes: [ 'host' ], - model: ServerModel.unscoped(), - required: false - } - ] - } - ] - } - - return ActorFollowModel.findAll(query) - } - - static listInstanceFollowingForApi (options: ListFollowingOptions) { - return Promise.all([ - new InstanceListFollowingQueryBuilder(this.sequelize, options).countFollowing(), - new InstanceListFollowingQueryBuilder(this.sequelize, options).listFollowing() - ]).then(([ total, data ]) => ({ total, data })) - } - - static listFollowersForApi (options: ListFollowersOptions) { - return Promise.all([ - new InstanceListFollowersQueryBuilder(this.sequelize, options).countFollowers(), - new InstanceListFollowersQueryBuilder(this.sequelize, options).listFollowers() - ]).then(([ total, data ]) => ({ total, data })) - } - - static listSubscriptionsForApi (options: { - actorId: number - start: number - count: number - sort: string - search?: string - }) { - const { actorId, start, count, sort } = options - const where = { - state: 'accepted', - actorId - } - - if (options.search) { - Object.assign(where, { - [Op.or]: [ - searchAttribute(options.search, '$ActorFollowing.preferredUsername$'), - searchAttribute(options.search, '$ActorFollowing.VideoChannel.name$') - ] - }) - } - - const getQuery = (forCount: boolean) => { - let channelInclude: Includeable[] = [] - - if (forCount !== true) { - channelInclude = [ - { - attributes: { - exclude: unusedActorAttributesForAPI - }, - model: ActorModel, - required: true - }, - { - model: AccountModel.unscoped(), - required: true, - include: [ - { - attributes: { - exclude: unusedActorAttributesForAPI - }, - model: ActorModel, - required: true - } - ] - } - ] - } - - return { - attributes: forCount === true - ? [] - : SORTABLE_COLUMNS.USER_SUBSCRIPTIONS, - distinct: true, - offset: start, - limit: count, - order: getSort(sort), - where, - include: [ - { - attributes: [ 'id' ], - model: ActorModel.unscoped(), - as: 'ActorFollowing', - required: true, - include: [ - { - model: VideoChannelModel.unscoped(), - required: true, - include: channelInclude - } - ] - } - ] - } - } - - return Promise.all([ - ActorFollowModel.count(getQuery(true)), - ActorFollowModel.findAll(getQuery(false)) - ]).then(([ total, rows ]) => ({ - total, - data: rows.map(r => r.ActorFollowing.VideoChannel) - })) - } - - static async keepUnfollowedInstance (hosts: string[]) { - const followerId = (await getServerActor()).id - - const query = { - attributes: [ 'id' ], - where: { - actorId: followerId - }, - include: [ - { - attributes: [ 'id' ], - model: ActorModel.unscoped(), - required: true, - as: 'ActorFollowing', - where: { - preferredUsername: SERVER_ACTOR_NAME - }, - include: [ - { - attributes: [ 'host' ], - model: ServerModel.unscoped(), - required: true, - where: { - host: { - [Op.in]: hosts - } - } - } - ] - } - ] - } - - const res = await ActorFollowModel.findAll(query) - const followedHosts = res.map(row => row.ActorFollowing.Server.host) - - return difference(hosts, followedHosts) - } - - static listAcceptedFollowerUrlsForAP (actorIds: number[], t: Transaction, start?: number, count?: number) { - return ActorFollowModel.createListAcceptedFollowForApiQuery('followers', actorIds, t, start, count) - } - - static listAcceptedFollowerSharedInboxUrls (actorIds: number[], t: Transaction) { - return ActorFollowModel.createListAcceptedFollowForApiQuery( - 'followers', - actorIds, - t, - undefined, - undefined, - 'sharedInboxUrl', - true - ) - } - - static listAcceptedFollowingUrlsForApi (actorIds: number[], t: Transaction, start?: number, count?: number) { - return ActorFollowModel.createListAcceptedFollowForApiQuery('following', actorIds, t, start, count) - } - - static async getStats () { - const serverActor = await getServerActor() - - const totalInstanceFollowing = await ActorFollowModel.count({ - where: { - actorId: serverActor.id, - state: 'accepted' - } - }) - - const totalInstanceFollowers = await ActorFollowModel.count({ - where: { - targetActorId: serverActor.id, - state: 'accepted' - } - }) - - return { - totalInstanceFollowing, - totalInstanceFollowers - } - } - - static updateScore (inboxUrl: string, value: number, t?: Transaction) { - const query = `UPDATE "actorFollow" SET "score" = LEAST("score" + ${value}, ${ACTOR_FOLLOW_SCORE.MAX}) ` + - 'WHERE id IN (' + - 'SELECT "actorFollow"."id" FROM "actorFollow" ' + - 'INNER JOIN "actor" ON "actor"."id" = "actorFollow"."actorId" ' + - `WHERE "actor"."inboxUrl" = '${inboxUrl}' OR "actor"."sharedInboxUrl" = '${inboxUrl}'` + - ')' - - const options = { - type: QueryTypes.BULKUPDATE, - transaction: t - } - - return ActorFollowModel.sequelize.query(query, options) - } - - static async updateScoreByFollowingServers (serverIds: number[], value: number, t?: Transaction) { - if (serverIds.length === 0) return - - const me = await getServerActor() - const serverIdsString = createSafeIn(ActorFollowModel.sequelize, serverIds) - - const query = `UPDATE "actorFollow" SET "score" = LEAST("score" + ${value}, ${ACTOR_FOLLOW_SCORE.MAX}) ` + - 'WHERE id IN (' + - 'SELECT "actorFollow"."id" FROM "actorFollow" ' + - 'INNER JOIN "actor" ON "actor"."id" = "actorFollow"."targetActorId" ' + - `WHERE "actorFollow"."actorId" = ${me.Account.actorId} ` + // I'm the follower - `AND "actor"."serverId" IN (${serverIdsString})` + // Criteria on followings - ')' - - const options = { - type: QueryTypes.BULKUPDATE, - transaction: t - } - - return ActorFollowModel.sequelize.query(query, options) - } - - private static async createListAcceptedFollowForApiQuery ( - type: 'followers' | 'following', - actorIds: number[], - t: Transaction, - start?: number, - count?: number, - columnUrl = 'url', - distinct = false - ) { - let firstJoin: string - let secondJoin: string - - if (type === 'followers') { - firstJoin = 'targetActorId' - secondJoin = 'actorId' - } else { - firstJoin = 'actorId' - secondJoin = 'targetActorId' - } - - const selections: string[] = [] - if (distinct === true) selections.push(`DISTINCT("Follows"."${columnUrl}") AS "selectionUrl"`) - else selections.push(`"Follows"."${columnUrl}" AS "selectionUrl"`) - - selections.push('COUNT(*) AS "total"') - - const tasks: Promise[] = [] - - for (const selection of selections) { - let query = 'SELECT ' + selection + ' FROM "actor" ' + - 'INNER JOIN "actorFollow" ON "actorFollow"."' + firstJoin + '" = "actor"."id" ' + - 'INNER JOIN "actor" AS "Follows" ON "actorFollow"."' + secondJoin + '" = "Follows"."id" ' + - `WHERE "actor"."id" = ANY ($actorIds) AND "actorFollow"."state" = 'accepted' AND "Follows"."${columnUrl}" IS NOT NULL ` - - if (count !== undefined) query += 'LIMIT ' + count - if (start !== undefined) query += ' OFFSET ' + start - - const options = { - bind: { actorIds }, - type: QueryTypes.SELECT, - transaction: t - } - tasks.push(ActorFollowModel.sequelize.query(query, options)) - } - - const [ followers, [ dataTotal ] ] = await Promise.all(tasks) - const urls: string[] = followers.map(f => f.selectionUrl) - - return { - data: urls, - total: dataTotal ? parseInt(dataTotal.total, 10) : 0 - } - } - - private static listBadActorFollows () { - const query = { - where: { - score: { - [Op.lte]: 0 - } - }, - logging: false - } - - return ActorFollowModel.findAll(query) - } - - toFormattedJSON (this: MActorFollowFormattable): ActorFollow { - const follower = this.ActorFollower.toFormattedJSON() - const following = this.ActorFollowing.toFormattedJSON() - - return { - id: this.id, - follower, - following, - score: this.score, - state: this.state, - createdAt: this.createdAt, - updatedAt: this.updatedAt - } - } -} diff --git a/server/models/actor/actor-image.ts b/server/models/actor/actor-image.ts deleted file mode 100644 index 51085a16d..000000000 --- a/server/models/actor/actor-image.ts +++ /dev/null @@ -1,171 +0,0 @@ -import { remove } from 'fs-extra' -import { join } from 'path' -import { - AfterDestroy, - AllowNull, - BelongsTo, - Column, - CreatedAt, - Default, - ForeignKey, - Is, - Model, - Table, - UpdatedAt -} from 'sequelize-typescript' -import { MActorImage, MActorImageFormattable } from '@server/types/models' -import { getLowercaseExtension } from '@shared/core-utils' -import { ActivityIconObject, ActorImageType } from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { ActorImage } from '../../../shared/models/actors/actor-image.model' -import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { LAZY_STATIC_PATHS, MIMETYPES, WEBSERVER } from '../../initializers/constants' -import { buildSQLAttributes, throwIfNotValid } from '../shared' -import { ActorModel } from './actor' - -@Table({ - tableName: 'actorImage', - indexes: [ - { - fields: [ 'filename' ], - unique: true - }, - { - fields: [ 'actorId', 'type', 'width' ], - unique: true - } - ] -}) -export class ActorImageModel extends Model>> { - - @AllowNull(false) - @Column - filename: string - - @AllowNull(true) - @Default(null) - @Column - height: number - - @AllowNull(true) - @Default(null) - @Column - width: number - - @AllowNull(true) - @Is('ActorImageFileUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'fileUrl', true)) - @Column - fileUrl: string - - @AllowNull(false) - @Column - onDisk: boolean - - @AllowNull(false) - @Column - type: ActorImageType - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => ActorModel) - @Column - actorId: number - - @BelongsTo(() => ActorModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'CASCADE' - }) - Actor: ActorModel - - @AfterDestroy - static removeFilesAndSendDelete (instance: ActorImageModel) { - logger.info('Removing actor image file %s.', instance.filename) - - // Don't block the transaction - instance.removeImage() - .catch(err => logger.error('Cannot remove actor image file %s.', instance.filename, { err })) - } - - // --------------------------------------------------------------------------- - - static getSQLAttributes (tableName: string, aliasPrefix = '') { - return buildSQLAttributes({ - model: this, - tableName, - aliasPrefix - }) - } - - // --------------------------------------------------------------------------- - - static loadByName (filename: string) { - const query = { - where: { - filename - } - } - - return ActorImageModel.findOne(query) - } - - static getImageUrl (image: MActorImage) { - if (!image) return undefined - - return WEBSERVER.URL + image.getStaticPath() - } - - toFormattedJSON (this: MActorImageFormattable): ActorImage { - return { - width: this.width, - path: this.getStaticPath(), - createdAt: this.createdAt, - updatedAt: this.updatedAt - } - } - - toActivityPubObject (): ActivityIconObject { - const extension = getLowercaseExtension(this.filename) - - return { - type: 'Image', - mediaType: MIMETYPES.IMAGE.EXT_MIMETYPE[extension], - height: this.height, - width: this.width, - url: ActorImageModel.getImageUrl(this) - } - } - - getStaticPath () { - switch (this.type) { - case ActorImageType.AVATAR: - return join(LAZY_STATIC_PATHS.AVATARS, this.filename) - - case ActorImageType.BANNER: - return join(LAZY_STATIC_PATHS.BANNERS, this.filename) - - default: - throw new Error('Unknown actor image type: ' + this.type) - } - } - - getPath () { - return join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, this.filename) - } - - removeImage () { - const imagePath = join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, this.filename) - return remove(imagePath) - } - - isOwned () { - return !this.fileUrl - } -} diff --git a/server/models/actor/actor.ts b/server/models/actor/actor.ts deleted file mode 100644 index e2e85f3d6..000000000 --- a/server/models/actor/actor.ts +++ /dev/null @@ -1,686 +0,0 @@ -import { col, fn, literal, Op, QueryTypes, Transaction, where } from 'sequelize' -import { - AllowNull, - BelongsTo, - Column, - CreatedAt, - DataType, - DefaultScope, - ForeignKey, - HasMany, - HasOne, - Is, - Model, - Scopes, - Table, - UpdatedAt -} from 'sequelize-typescript' -import { activityPubContextify } from '@server/lib/activitypub/context' -import { getBiggestActorImage } from '@server/lib/actor-image' -import { ModelCache } from '@server/models/shared/model-cache' -import { forceNumber, getLowercaseExtension } from '@shared/core-utils' -import { ActivityIconObject, ActivityPubActorType, ActorImageType } from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { - isActorFollowersCountValid, - isActorFollowingCountValid, - isActorPreferredUsernameValid, - isActorPrivateKeyValid, - isActorPublicKeyValid -} from '../../helpers/custom-validators/activitypub/actor' -import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' -import { - ACTIVITY_PUB, - ACTIVITY_PUB_ACTOR_TYPES, - CONSTRAINTS_FIELDS, - MIMETYPES, - SERVER_ACTOR_NAME, - WEBSERVER -} from '../../initializers/constants' -import { - MActor, - MActorAccountChannelId, - MActorAPAccount, - MActorAPChannel, - MActorFollowersUrl, - MActorFormattable, - MActorFull, - MActorHost, - MActorHostOnly, - MActorId, - MActorSummaryFormattable, - MActorUrl, - MActorWithInboxes -} from '../../types/models' -import { AccountModel } from '../account/account' -import { getServerActor } from '../application/application' -import { ServerModel } from '../server/server' -import { buildSQLAttributes, isOutdated, throwIfNotValid } from '../shared' -import { VideoModel } from '../video/video' -import { VideoChannelModel } from '../video/video-channel' -import { ActorFollowModel } from './actor-follow' -import { ActorImageModel } from './actor-image' - -enum ScopeNames { - FULL = 'FULL' -} - -export const unusedActorAttributesForAPI: (keyof AttributesOnly)[] = [ - 'publicKey', - 'privateKey', - 'inboxUrl', - 'outboxUrl', - 'sharedInboxUrl', - 'followersUrl', - 'followingUrl' -] - -@DefaultScope(() => ({ - include: [ - { - model: ServerModel, - required: false - }, - { - model: ActorImageModel, - as: 'Avatars', - required: false - } - ] -})) -@Scopes(() => ({ - [ScopeNames.FULL]: { - include: [ - { - model: AccountModel.unscoped(), - required: false - }, - { - model: VideoChannelModel.unscoped(), - required: false, - include: [ - { - model: AccountModel, - required: true - } - ] - }, - { - model: ServerModel, - required: false - }, - { - model: ActorImageModel, - as: 'Avatars', - required: false - }, - { - model: ActorImageModel, - as: 'Banners', - required: false - } - ] - } -})) -@Table({ - tableName: 'actor', - indexes: [ - { - fields: [ 'url' ], - unique: true - }, - { - fields: [ fn('lower', col('preferredUsername')), 'serverId' ], - name: 'actor_preferred_username_lower_server_id', - unique: true, - where: { - serverId: { - [Op.ne]: null - } - } - }, - { - fields: [ fn('lower', col('preferredUsername')) ], - name: 'actor_preferred_username_lower', - unique: true, - where: { - serverId: null - } - }, - { - fields: [ 'inboxUrl', 'sharedInboxUrl' ] - }, - { - fields: [ 'sharedInboxUrl' ] - }, - { - fields: [ 'serverId' ] - }, - { - fields: [ 'followersUrl' ] - } - ] -}) -export class ActorModel extends Model>> { - - @AllowNull(false) - @Column(DataType.ENUM(...Object.values(ACTIVITY_PUB_ACTOR_TYPES))) - type: ActivityPubActorType - - @AllowNull(false) - @Is('ActorPreferredUsername', value => throwIfNotValid(value, isActorPreferredUsernameValid, 'actor preferred username')) - @Column - preferredUsername: string - - @AllowNull(false) - @Is('ActorUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url')) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.ACTORS.URL.max)) - url: string - - @AllowNull(true) - @Is('ActorPublicKey', value => throwIfNotValid(value, isActorPublicKeyValid, 'public key', true)) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.ACTORS.PUBLIC_KEY.max)) - publicKey: string - - @AllowNull(true) - @Is('ActorPublicKey', value => throwIfNotValid(value, isActorPrivateKeyValid, 'private key', true)) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.ACTORS.PRIVATE_KEY.max)) - privateKey: string - - @AllowNull(false) - @Is('ActorFollowersCount', value => throwIfNotValid(value, isActorFollowersCountValid, 'followers count')) - @Column - followersCount: number - - @AllowNull(false) - @Is('ActorFollowersCount', value => throwIfNotValid(value, isActorFollowingCountValid, 'following count')) - @Column - followingCount: number - - @AllowNull(false) - @Is('ActorInboxUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'inbox url')) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.ACTORS.URL.max)) - inboxUrl: string - - @AllowNull(true) - @Is('ActorOutboxUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'outbox url', true)) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.ACTORS.URL.max)) - outboxUrl: string - - @AllowNull(true) - @Is('ActorSharedInboxUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'shared inbox url', true)) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.ACTORS.URL.max)) - sharedInboxUrl: string - - @AllowNull(true) - @Is('ActorFollowersUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'followers url', true)) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.ACTORS.URL.max)) - followersUrl: string - - @AllowNull(true) - @Is('ActorFollowingUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'following url', true)) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.ACTORS.URL.max)) - followingUrl: string - - @AllowNull(true) - @Column - remoteCreatedAt: Date - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @HasMany(() => ActorImageModel, { - as: 'Avatars', - onDelete: 'cascade', - hooks: true, - foreignKey: { - allowNull: false - }, - scope: { - type: ActorImageType.AVATAR - } - }) - Avatars: ActorImageModel[] - - @HasMany(() => ActorImageModel, { - as: 'Banners', - onDelete: 'cascade', - hooks: true, - foreignKey: { - allowNull: false - }, - scope: { - type: ActorImageType.BANNER - } - }) - Banners: ActorImageModel[] - - @HasMany(() => ActorFollowModel, { - foreignKey: { - name: 'actorId', - allowNull: false - }, - as: 'ActorFollowings', - onDelete: 'cascade' - }) - ActorFollowing: ActorFollowModel[] - - @HasMany(() => ActorFollowModel, { - foreignKey: { - name: 'targetActorId', - allowNull: false - }, - as: 'ActorFollowers', - onDelete: 'cascade' - }) - ActorFollowers: ActorFollowModel[] - - @ForeignKey(() => ServerModel) - @Column - serverId: number - - @BelongsTo(() => ServerModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'cascade' - }) - Server: ServerModel - - @HasOne(() => AccountModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'cascade', - hooks: true - }) - Account: AccountModel - - @HasOne(() => VideoChannelModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'cascade', - hooks: true - }) - VideoChannel: VideoChannelModel - - // --------------------------------------------------------------------------- - - static getSQLAttributes (tableName: string, aliasPrefix = '') { - return buildSQLAttributes({ - model: this, - tableName, - aliasPrefix - }) - } - - static getSQLAPIAttributes (tableName: string, aliasPrefix = '') { - return buildSQLAttributes({ - model: this, - tableName, - aliasPrefix, - excludeAttributes: unusedActorAttributesForAPI - }) - } - - // --------------------------------------------------------------------------- - - static wherePreferredUsername (preferredUsername: string, colName = 'preferredUsername') { - return where(fn('lower', col(colName)), preferredUsername.toLowerCase()) - } - - // --------------------------------------------------------------------------- - - static async load (id: number): Promise { - const actorServer = await getServerActor() - if (id === actorServer.id) return actorServer - - return ActorModel.unscoped().findByPk(id) - } - - static loadFull (id: number): Promise { - return ActorModel.scope(ScopeNames.FULL).findByPk(id) - } - - static loadAccountActorFollowerUrlByVideoId (videoId: number, transaction: Transaction) { - const query = `SELECT "actor"."id" AS "id", "actor"."followersUrl" AS "followersUrl" ` + - `FROM "actor" ` + - `INNER JOIN "account" ON "actor"."id" = "account"."actorId" ` + - `INNER JOIN "videoChannel" ON "videoChannel"."accountId" = "account"."id" ` + - `INNER JOIN "video" ON "video"."channelId" = "videoChannel"."id" AND "video"."id" = :videoId` - - const options = { - type: QueryTypes.SELECT as QueryTypes.SELECT, - replacements: { videoId }, - plain: true as true, - transaction - } - - return ActorModel.sequelize.query(query, options) - } - - static listByFollowersUrls (followersUrls: string[], transaction?: Transaction): Promise { - const query = { - where: { - followersUrl: { - [Op.in]: followersUrls - } - }, - transaction - } - - return ActorModel.scope(ScopeNames.FULL).findAll(query) - } - - static loadLocalByName (preferredUsername: string, transaction?: Transaction): Promise { - const fun = () => { - const query = { - where: { - [Op.and]: [ - this.wherePreferredUsername(preferredUsername, '"ActorModel"."preferredUsername"'), - { - serverId: null - } - ] - }, - transaction - } - - return ActorModel.scope(ScopeNames.FULL).findOne(query) - } - - return ModelCache.Instance.doCache({ - cacheType: 'local-actor-name', - key: preferredUsername, - // The server actor never change, so we can easily cache it - whitelist: () => preferredUsername === SERVER_ACTOR_NAME, - fun - }) - } - - static loadLocalUrlByName (preferredUsername: string, transaction?: Transaction): Promise { - const fun = () => { - const query = { - attributes: [ 'url' ], - where: { - [Op.and]: [ - this.wherePreferredUsername(preferredUsername), - { - serverId: null - } - ] - }, - transaction - } - - return ActorModel.unscoped().findOne(query) - } - - return ModelCache.Instance.doCache({ - cacheType: 'local-actor-url', - key: preferredUsername, - // The server actor never change, so we can easily cache it - whitelist: () => preferredUsername === SERVER_ACTOR_NAME, - fun - }) - } - - static loadByNameAndHost (preferredUsername: string, host: string): Promise { - const query = { - where: this.wherePreferredUsername(preferredUsername, '"ActorModel"."preferredUsername"'), - include: [ - { - model: ServerModel, - required: true, - where: { - host - } - } - ] - } - - return ActorModel.scope(ScopeNames.FULL).findOne(query) - } - - static loadByUrl (url: string, transaction?: Transaction): Promise { - const query = { - where: { - url - }, - transaction, - include: [ - { - attributes: [ 'id' ], - model: AccountModel.unscoped(), - required: false - }, - { - attributes: [ 'id' ], - model: VideoChannelModel.unscoped(), - required: false - } - ] - } - - return ActorModel.unscoped().findOne(query) - } - - static loadByUrlAndPopulateAccountAndChannel (url: string, transaction?: Transaction): Promise { - const query = { - where: { - url - }, - transaction - } - - return ActorModel.scope(ScopeNames.FULL).findOne(query) - } - - static rebuildFollowsCount (ofId: number, type: 'followers' | 'following', transaction?: Transaction) { - const sanitizedOfId = forceNumber(ofId) - const where = { id: sanitizedOfId } - - let columnToUpdate: string - let columnOfCount: string - - if (type === 'followers') { - columnToUpdate = 'followersCount' - columnOfCount = 'targetActorId' - } else { - columnToUpdate = 'followingCount' - columnOfCount = 'actorId' - } - - return ActorModel.update({ - [columnToUpdate]: literal(`(SELECT COUNT(*) FROM "actorFollow" WHERE "${columnOfCount}" = ${sanitizedOfId} AND "state" = 'accepted')`) - }, { where, transaction }) - } - - static loadAccountActorByVideoId (videoId: number, transaction: Transaction): Promise { - const query = { - include: [ - { - attributes: [ 'id' ], - model: AccountModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'id', 'accountId' ], - model: VideoChannelModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'id', 'channelId' ], - model: VideoModel.unscoped(), - where: { - id: videoId - } - } - ] - } - ] - } - ], - transaction - } - - return ActorModel.unscoped().findOne(query) - } - - getSharedInbox (this: MActorWithInboxes) { - return this.sharedInboxUrl || this.inboxUrl - } - - toFormattedSummaryJSON (this: MActorSummaryFormattable) { - return { - url: this.url, - name: this.preferredUsername, - host: this.getHost(), - avatars: (this.Avatars || []).map(a => a.toFormattedJSON()) - } - } - - toFormattedJSON (this: MActorFormattable) { - return { - ...this.toFormattedSummaryJSON(), - - id: this.id, - hostRedundancyAllowed: this.getRedundancyAllowed(), - followingCount: this.followingCount, - followersCount: this.followersCount, - createdAt: this.getCreatedAt(), - - banners: (this.Banners || []).map(b => b.toFormattedJSON()) - } - } - - toActivityPubObject (this: MActorAPChannel | MActorAPAccount, name: string) { - let icon: ActivityIconObject[] - let image: ActivityIconObject - - if (this.hasImage(ActorImageType.AVATAR)) { - icon = this.Avatars.map(a => a.toActivityPubObject()) - } - - if (this.hasImage(ActorImageType.BANNER)) { - const banner = getBiggestActorImage((this as MActorAPChannel).Banners) - const extension = getLowercaseExtension(banner.filename) - - image = { - type: 'Image', - mediaType: MIMETYPES.IMAGE.EXT_MIMETYPE[extension], - height: banner.height, - width: banner.width, - url: ActorImageModel.getImageUrl(banner) - } - } - - const json = { - type: this.type, - id: this.url, - following: this.getFollowingUrl(), - followers: this.getFollowersUrl(), - playlists: this.getPlaylistsUrl(), - inbox: this.inboxUrl, - outbox: this.outboxUrl, - preferredUsername: this.preferredUsername, - url: this.url, - name, - endpoints: { - sharedInbox: this.sharedInboxUrl - }, - publicKey: { - id: this.getPublicKeyUrl(), - owner: this.url, - publicKeyPem: this.publicKey - }, - published: this.getCreatedAt().toISOString(), - - icon, - - image - } - - return activityPubContextify(json, 'Actor') - } - - getFollowerSharedInboxUrls (t: Transaction) { - const query = { - attributes: [ 'sharedInboxUrl' ], - include: [ - { - attribute: [], - model: ActorFollowModel.unscoped(), - required: true, - as: 'ActorFollowing', - where: { - state: 'accepted', - targetActorId: this.id - } - } - ], - transaction: t - } - - return ActorModel.findAll(query) - .then(accounts => accounts.map(a => a.sharedInboxUrl)) - } - - getFollowingUrl () { - return this.url + '/following' - } - - getFollowersUrl () { - return this.url + '/followers' - } - - getPlaylistsUrl () { - return this.url + '/playlists' - } - - getPublicKeyUrl () { - return this.url + '#main-key' - } - - isOwned () { - return this.serverId === null - } - - getWebfingerUrl (this: MActorHost) { - return 'acct:' + this.preferredUsername + '@' + this.getHost() - } - - getIdentifier (this: MActorHost) { - return this.Server ? `${this.preferredUsername}@${this.Server.host}` : this.preferredUsername - } - - getHost (this: MActorHostOnly) { - return this.Server ? this.Server.host : WEBSERVER.HOST - } - - getRedundancyAllowed () { - return this.Server ? this.Server.redundancyAllowed : false - } - - hasImage (type: ActorImageType) { - const images = type === ActorImageType.AVATAR - ? this.Avatars - : this.Banners - - return Array.isArray(images) && images.length !== 0 - } - - isOutdated () { - if (this.isOwned()) return false - - return isOutdated(this, ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL) - } - - getCreatedAt (this: MActorAPChannel | MActorAPAccount | MActorFormattable) { - return this.remoteCreatedAt || this.createdAt - } -} diff --git a/server/models/actor/sql/instance-list-followers-query-builder.ts b/server/models/actor/sql/instance-list-followers-query-builder.ts deleted file mode 100644 index 34ce29b5d..000000000 --- a/server/models/actor/sql/instance-list-followers-query-builder.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { Sequelize } from 'sequelize' -import { ModelBuilder } from '@server/models/shared' -import { MActorFollowActorsDefault } from '@server/types/models' -import { ActivityPubActorType, FollowState } from '@shared/models' -import { parseRowCountResult } from '../../shared' -import { InstanceListFollowsQueryBuilder } from './shared/instance-list-follows-query-builder' - -export interface ListFollowersOptions { - actorIds: number[] - start: number - count: number - sort: string - state?: FollowState - actorType?: ActivityPubActorType - search?: string -} - -export class InstanceListFollowersQueryBuilder extends InstanceListFollowsQueryBuilder { - - constructor ( - protected readonly sequelize: Sequelize, - protected readonly options: ListFollowersOptions - ) { - super(sequelize, options) - } - - async listFollowers () { - this.buildListQuery() - - const results = await this.runQuery({ nest: true }) - const modelBuilder = new ModelBuilder(this.sequelize) - - return modelBuilder.createModels(results, 'ActorFollow') - } - - async countFollowers () { - this.buildCountQuery() - - const result = await this.runQuery() - - return parseRowCountResult(result) - } - - protected getWhere () { - let where = 'WHERE "ActorFollowing"."id" IN (:actorIds) ' - this.replacements.actorIds = this.options.actorIds - - if (this.options.state) { - where += 'AND "ActorFollowModel"."state" = :state ' - this.replacements.state = this.options.state - } - - if (this.options.search) { - const escapedLikeSearch = this.sequelize.escape('%' + this.options.search + '%') - - where += `AND (` + - `"ActorFollower->Server"."host" ILIKE ${escapedLikeSearch} ` + - `OR "ActorFollower"."preferredUsername" ILIKE ${escapedLikeSearch} ` + - `)` - } - - if (this.options.actorType) { - where += `AND "ActorFollower"."type" = :actorType ` - this.replacements.actorType = this.options.actorType - } - - return where - } -} diff --git a/server/models/actor/sql/instance-list-following-query-builder.ts b/server/models/actor/sql/instance-list-following-query-builder.ts deleted file mode 100644 index 77b4e3dce..000000000 --- a/server/models/actor/sql/instance-list-following-query-builder.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { Sequelize } from 'sequelize' -import { ModelBuilder } from '@server/models/shared' -import { MActorFollowActorsDefault } from '@server/types/models' -import { ActivityPubActorType, FollowState } from '@shared/models' -import { parseRowCountResult } from '../../shared' -import { InstanceListFollowsQueryBuilder } from './shared/instance-list-follows-query-builder' - -export interface ListFollowingOptions { - followerId: number - start: number - count: number - sort: string - state?: FollowState - actorType?: ActivityPubActorType - search?: string -} - -export class InstanceListFollowingQueryBuilder extends InstanceListFollowsQueryBuilder { - - constructor ( - protected readonly sequelize: Sequelize, - protected readonly options: ListFollowingOptions - ) { - super(sequelize, options) - } - - async listFollowing () { - this.buildListQuery() - - const results = await this.runQuery({ nest: true }) - const modelBuilder = new ModelBuilder(this.sequelize) - - return modelBuilder.createModels(results, 'ActorFollow') - } - - async countFollowing () { - this.buildCountQuery() - - const result = await this.runQuery() - - return parseRowCountResult(result) - } - - protected getWhere () { - let where = 'WHERE "ActorFollowModel"."actorId" = :followerId ' - this.replacements.followerId = this.options.followerId - - if (this.options.state) { - where += 'AND "ActorFollowModel"."state" = :state ' - this.replacements.state = this.options.state - } - - if (this.options.search) { - const escapedLikeSearch = this.sequelize.escape('%' + this.options.search + '%') - - where += `AND (` + - `"ActorFollowing->Server"."host" ILIKE ${escapedLikeSearch} ` + - `OR "ActorFollowing"."preferredUsername" ILIKE ${escapedLikeSearch} ` + - `)` - } - - if (this.options.actorType) { - where += `AND "ActorFollowing"."type" = :actorType ` - this.replacements.actorType = this.options.actorType - } - - return where - } -} diff --git a/server/models/actor/sql/shared/actor-follow-table-attributes.ts b/server/models/actor/sql/shared/actor-follow-table-attributes.ts deleted file mode 100644 index 4431aa6d1..000000000 --- a/server/models/actor/sql/shared/actor-follow-table-attributes.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { Memoize } from '@server/helpers/memoize' -import { ServerModel } from '@server/models/server/server' -import { ActorModel } from '../../actor' -import { ActorFollowModel } from '../../actor-follow' -import { ActorImageModel } from '../../actor-image' - -export class ActorFollowTableAttributes { - - @Memoize() - getFollowAttributes () { - return ActorFollowModel.getSQLAttributes('ActorFollowModel').join(', ') - } - - @Memoize() - getActorAttributes (actorTableName: string) { - return ActorModel.getSQLAttributes(actorTableName, `${actorTableName}.`).join(', ') - } - - @Memoize() - getServerAttributes (actorTableName: string) { - return ServerModel.getSQLAttributes(`${actorTableName}->Server`, `${actorTableName}.Server.`).join(', ') - } - - @Memoize() - getAvatarAttributes (actorTableName: string) { - return ActorImageModel.getSQLAttributes(`${actorTableName}->Avatars`, `${actorTableName}.Avatars.`).join(', ') - } -} diff --git a/server/models/actor/sql/shared/instance-list-follows-query-builder.ts b/server/models/actor/sql/shared/instance-list-follows-query-builder.ts deleted file mode 100644 index d9593e48b..000000000 --- a/server/models/actor/sql/shared/instance-list-follows-query-builder.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { Sequelize } from 'sequelize' -import { AbstractRunQuery } from '@server/models/shared' -import { ActorImageType } from '@shared/models' -import { getInstanceFollowsSort } from '../../../shared' -import { ActorFollowTableAttributes } from './actor-follow-table-attributes' - -type BaseOptions = { - sort: string - count: number - start: number -} - -export abstract class InstanceListFollowsQueryBuilder extends AbstractRunQuery { - protected readonly tableAttributes = new ActorFollowTableAttributes() - - protected innerQuery: string - - constructor ( - protected readonly sequelize: Sequelize, - protected readonly options: T - ) { - super(sequelize) - } - - protected abstract getWhere (): string - - protected getJoins () { - return 'INNER JOIN "actor" "ActorFollower" ON "ActorFollower"."id" = "ActorFollowModel"."actorId" ' + - 'INNER JOIN "actor" "ActorFollowing" ON "ActorFollowing"."id" = "ActorFollowModel"."targetActorId" ' - } - - protected getServerJoin (actorName: string) { - return `LEFT JOIN "server" "${actorName}->Server" ON "${actorName}"."serverId" = "${actorName}->Server"."id" ` - } - - protected getAvatarsJoin (actorName: string) { - return `LEFT JOIN "actorImage" "${actorName}->Avatars" ON "${actorName}.id" = "${actorName}->Avatars"."actorId" ` + - `AND "${actorName}->Avatars"."type" = ${ActorImageType.AVATAR}` - } - - private buildInnerQuery () { - this.innerQuery = `${this.getInnerSelect()} ` + - `FROM "actorFollow" AS "ActorFollowModel" ` + - `${this.getJoins()} ` + - `${this.getServerJoin('ActorFollowing')} ` + - `${this.getServerJoin('ActorFollower')} ` + - `${this.getWhere()} ` + - `${this.getOrder()} ` + - `LIMIT :limit OFFSET :offset ` - - this.replacements.limit = this.options.count - this.replacements.offset = this.options.start - } - - protected buildListQuery () { - this.buildInnerQuery() - - this.query = `${this.getSelect()} ` + - `FROM (${this.innerQuery}) AS "ActorFollowModel" ` + - `${this.getAvatarsJoin('ActorFollower')} ` + - `${this.getAvatarsJoin('ActorFollowing')} ` + - `${this.getOrder()}` - } - - protected buildCountQuery () { - this.query = `SELECT COUNT(*) AS "total" ` + - `FROM "actorFollow" AS "ActorFollowModel" ` + - `${this.getJoins()} ` + - `${this.getServerJoin('ActorFollowing')} ` + - `${this.getServerJoin('ActorFollower')} ` + - `${this.getWhere()}` - } - - private getInnerSelect () { - return this.buildSelect([ - this.tableAttributes.getFollowAttributes(), - this.tableAttributes.getActorAttributes('ActorFollower'), - this.tableAttributes.getActorAttributes('ActorFollowing'), - this.tableAttributes.getServerAttributes('ActorFollower'), - this.tableAttributes.getServerAttributes('ActorFollowing') - ]) - } - - private getSelect () { - return this.buildSelect([ - '"ActorFollowModel".*', - this.tableAttributes.getAvatarAttributes('ActorFollower'), - this.tableAttributes.getAvatarAttributes('ActorFollowing') - ]) - } - - private getOrder () { - const orders = getInstanceFollowsSort(this.options.sort) - - return 'ORDER BY ' + orders.map(o => `"${o[0]}" ${o[1]}`).join(', ') - } -} diff --git a/server/models/application/application.ts b/server/models/application/application.ts deleted file mode 100644 index c51ceb245..000000000 --- a/server/models/application/application.ts +++ /dev/null @@ -1,79 +0,0 @@ -import memoizee from 'memoizee' -import { AllowNull, Column, Default, DefaultScope, HasOne, IsInt, Model, Table } from 'sequelize-typescript' -import { getNodeABIVersion } from '@server/helpers/version' -import { AttributesOnly } from '@shared/typescript-utils' -import { AccountModel } from '../account/account' - -export const getServerActor = memoizee(async function () { - const application = await ApplicationModel.load() - if (!application) throw Error('Could not load Application from database.') - - const actor = application.Account.Actor - actor.Account = application.Account - - return actor -}, { promise: true }) - -@DefaultScope(() => ({ - include: [ - { - model: AccountModel, - required: true - } - ] -})) -@Table({ - tableName: 'application', - timestamps: false -}) -export class ApplicationModel extends Model>> { - - @AllowNull(false) - @Default(0) - @IsInt - @Column - migrationVersion: number - - @AllowNull(true) - @Column - latestPeerTubeVersion: string - - @AllowNull(false) - @Column - nodeVersion: string - - @AllowNull(false) - @Column - nodeABIVersion: number - - @HasOne(() => AccountModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'cascade' - }) - Account: AccountModel - - static countTotal () { - return ApplicationModel.count() - } - - static load () { - return ApplicationModel.findOne() - } - - static async nodeABIChanged () { - const application = await this.load() - - return application.nodeABIVersion !== getNodeABIVersion() - } - - static async updateNodeVersions () { - const application = await this.load() - - application.nodeABIVersion = getNodeABIVersion() - application.nodeVersion = process.version - - await application.save() - } -} diff --git a/server/models/migrations.ts b/server/models/migrations.ts deleted file mode 100644 index 6c11332a1..000000000 --- a/server/models/migrations.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { ModelAttributeColumnOptions } from 'sequelize' - -declare namespace Migration { - interface Boolean extends ModelAttributeColumnOptions { - defaultValue: boolean | null - } - - interface String extends ModelAttributeColumnOptions { - defaultValue: string | null - } - - interface Integer extends ModelAttributeColumnOptions { - defaultValue: number | null - } - - interface BigInteger extends ModelAttributeColumnOptions { - defaultValue: number | null - } - - interface UUID extends ModelAttributeColumnOptions { - defaultValue: null - } -} - -export { - Migration -} diff --git a/server/models/oauth/oauth-client.ts b/server/models/oauth/oauth-client.ts deleted file mode 100644 index 457e84613..000000000 --- a/server/models/oauth/oauth-client.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { AllowNull, Column, CreatedAt, DataType, HasMany, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { AttributesOnly } from '@shared/typescript-utils' -import { OAuthTokenModel } from './oauth-token' - -@Table({ - tableName: 'oAuthClient', - indexes: [ - { - fields: [ 'clientId' ], - unique: true - }, - { - fields: [ 'clientId', 'clientSecret' ], - unique: true - } - ] -}) -export class OAuthClientModel extends Model>> { - - @AllowNull(false) - @Column - clientId: string - - @AllowNull(false) - @Column - clientSecret: string - - @Column(DataType.ARRAY(DataType.STRING)) - grants: string[] - - @Column(DataType.ARRAY(DataType.STRING)) - redirectUris: string[] - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @HasMany(() => OAuthTokenModel, { - onDelete: 'cascade' - }) - OAuthTokens: OAuthTokenModel[] - - static countTotal () { - return OAuthClientModel.count() - } - - static loadFirstClient () { - return OAuthClientModel.findOne() - } - - static getByIdAndSecret (clientId: string, clientSecret: string) { - const query = { - where: { - clientId, - clientSecret - } - } - - return OAuthClientModel.findOne(query) - } -} diff --git a/server/models/oauth/oauth-token.ts b/server/models/oauth/oauth-token.ts deleted file mode 100644 index f72423190..000000000 --- a/server/models/oauth/oauth-token.ts +++ /dev/null @@ -1,220 +0,0 @@ -import { Transaction } from 'sequelize' -import { - AfterDestroy, - AfterUpdate, - AllowNull, - BelongsTo, - Column, - CreatedAt, - ForeignKey, - Model, - Scopes, - Table, - UpdatedAt -} from 'sequelize-typescript' -import { TokensCache } from '@server/lib/auth/tokens-cache' -import { MUserAccountId } from '@server/types/models' -import { MOAuthTokenUser } from '@server/types/models/oauth/oauth-token' -import { AttributesOnly } from '@shared/typescript-utils' -import { logger } from '../../helpers/logger' -import { AccountModel } from '../account/account' -import { ActorModel } from '../actor/actor' -import { UserModel } from '../user/user' -import { OAuthClientModel } from './oauth-client' - -export type OAuthTokenInfo = { - refreshToken: string - refreshTokenExpiresAt: Date - client: { - id: number - } - user: MUserAccountId - token: MOAuthTokenUser -} - -enum ScopeNames { - WITH_USER = 'WITH_USER' -} - -@Scopes(() => ({ - [ScopeNames.WITH_USER]: { - include: [ - { - model: UserModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'id' ], - model: AccountModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'id', 'url' ], - model: ActorModel.unscoped(), - required: true - } - ] - } - ] - } - ] - } -})) -@Table({ - tableName: 'oAuthToken', - indexes: [ - { - fields: [ 'refreshToken' ], - unique: true - }, - { - fields: [ 'accessToken' ], - unique: true - }, - { - fields: [ 'userId' ] - }, - { - fields: [ 'oAuthClientId' ] - } - ] -}) -export class OAuthTokenModel extends Model>> { - - @AllowNull(false) - @Column - accessToken: string - - @AllowNull(false) - @Column - accessTokenExpiresAt: Date - - @AllowNull(false) - @Column - refreshToken: string - - @AllowNull(false) - @Column - refreshTokenExpiresAt: Date - - @Column - authName: string - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => UserModel) - @Column - userId: number - - @BelongsTo(() => UserModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade' - }) - User: UserModel - - @ForeignKey(() => OAuthClientModel) - @Column - oAuthClientId: number - - @BelongsTo(() => OAuthClientModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade' - }) - OAuthClients: OAuthClientModel[] - - @AfterUpdate - @AfterDestroy - static removeTokenCache (token: OAuthTokenModel) { - return TokensCache.Instance.clearCacheByToken(token.accessToken) - } - - static loadByRefreshToken (refreshToken: string) { - const query = { - where: { refreshToken } - } - - return OAuthTokenModel.findOne(query) - } - - static getByRefreshTokenAndPopulateClient (refreshToken: string) { - const query = { - where: { - refreshToken - }, - include: [ OAuthClientModel ] - } - - return OAuthTokenModel.scope(ScopeNames.WITH_USER) - .findOne(query) - .then(token => { - if (!token) return null - - return { - refreshToken: token.refreshToken, - refreshTokenExpiresAt: token.refreshTokenExpiresAt, - client: { - id: token.oAuthClientId - }, - user: token.User, - token - } as OAuthTokenInfo - }) - .catch(err => { - logger.error('getRefreshToken error.', { err }) - throw err - }) - } - - static getByTokenAndPopulateUser (bearerToken: string): Promise { - const query = { - where: { - accessToken: bearerToken - } - } - - return OAuthTokenModel.scope(ScopeNames.WITH_USER) - .findOne(query) - .then(token => { - if (!token) return null - - return Object.assign(token, { user: token.User }) - }) - } - - static getByRefreshTokenAndPopulateUser (refreshToken: string): Promise { - const query = { - where: { - refreshToken - } - } - - return OAuthTokenModel.scope(ScopeNames.WITH_USER) - .findOne(query) - .then(token => { - if (!token) return undefined - - return Object.assign(token, { user: token.User }) - }) - } - - static deleteUserToken (userId: number, t?: Transaction) { - TokensCache.Instance.deleteUserToken(userId) - - const query = { - where: { - userId - }, - transaction: t - } - - return OAuthTokenModel.destroy(query) - } -} diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts deleted file mode 100644 index cebf47dfd..000000000 --- a/server/models/redundancy/video-redundancy.ts +++ /dev/null @@ -1,793 +0,0 @@ -import { sample } from 'lodash' -import { literal, Op, QueryTypes, Transaction, WhereOptions } from 'sequelize' -import { - AllowNull, - BeforeDestroy, - BelongsTo, - Column, - CreatedAt, - DataType, - ForeignKey, - Is, - Model, - Scopes, - Table, - UpdatedAt -} from 'sequelize-typescript' -import { getServerActor } from '@server/models/application/application' -import { MActor, MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models' -import { - CacheFileObject, - FileRedundancyInformation, - StreamingPlaylistRedundancyInformation, - VideoPrivacy, - VideoRedundanciesTarget, - VideoRedundancy, - VideoRedundancyStrategy, - VideoRedundancyStrategyWithManual -} from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { isTestInstance } from '../../helpers/core-utils' -import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants' -import { ActorModel } from '../actor/actor' -import { ServerModel } from '../server/server' -import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../shared' -import { ScheduleVideoUpdateModel } from '../video/schedule-video-update' -import { VideoModel } from '../video/video' -import { VideoChannelModel } from '../video/video-channel' -import { VideoFileModel } from '../video/video-file' -import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist' - -export enum ScopeNames { - WITH_VIDEO = 'WITH_VIDEO' -} - -@Scopes(() => ({ - [ScopeNames.WITH_VIDEO]: { - include: [ - { - model: VideoFileModel, - required: false, - include: [ - { - model: VideoModel, - required: true - } - ] - }, - { - model: VideoStreamingPlaylistModel, - required: false, - include: [ - { - model: VideoModel, - required: true - } - ] - } - ] - } -})) - -@Table({ - tableName: 'videoRedundancy', - indexes: [ - { - fields: [ 'videoFileId' ] - }, - { - fields: [ 'actorId' ] - }, - { - fields: [ 'expiresOn' ] - }, - { - fields: [ 'url' ], - unique: true - } - ] -}) -export class VideoRedundancyModel extends Model>> { - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @AllowNull(true) - @Column - expiresOn: Date - - @AllowNull(false) - @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl')) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max)) - fileUrl: string - - @AllowNull(false) - @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url')) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max)) - url: string - - @AllowNull(true) - @Column - strategy: string // Only used by us - - @ForeignKey(() => VideoFileModel) - @Column - videoFileId: number - - @BelongsTo(() => VideoFileModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'cascade' - }) - VideoFile: VideoFileModel - - @ForeignKey(() => VideoStreamingPlaylistModel) - @Column - videoStreamingPlaylistId: number - - @BelongsTo(() => VideoStreamingPlaylistModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'cascade' - }) - VideoStreamingPlaylist: VideoStreamingPlaylistModel - - @ForeignKey(() => ActorModel) - @Column - actorId: number - - @BelongsTo(() => ActorModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade' - }) - Actor: ActorModel - - @BeforeDestroy - static async removeFile (instance: VideoRedundancyModel) { - if (!instance.isOwned()) return - - if (instance.videoFileId) { - const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId) - - const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}` - logger.info('Removing duplicated video file %s.', logIdentifier) - - videoFile.Video.removeWebVideoFile(videoFile, true) - .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err })) - } - - if (instance.videoStreamingPlaylistId) { - const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId) - - const videoUUID = videoStreamingPlaylist.Video.uuid - logger.info('Removing duplicated video streaming playlist %s.', videoUUID) - - videoStreamingPlaylist.Video.removeStreamingPlaylistFiles(videoStreamingPlaylist, true) - .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err })) - } - - return undefined - } - - static async loadLocalByFileId (videoFileId: number): Promise { - const actor = await getServerActor() - - const query = { - where: { - actorId: actor.id, - videoFileId - } - } - - return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) - } - - static async listLocalByVideoId (videoId: number): Promise { - const actor = await getServerActor() - - const queryStreamingPlaylist = { - where: { - actorId: actor.id - }, - include: [ - { - model: VideoStreamingPlaylistModel.unscoped(), - required: true, - include: [ - { - model: VideoModel.unscoped(), - required: true, - where: { - id: videoId - } - } - ] - } - ] - } - - const queryFiles = { - where: { - actorId: actor.id - }, - include: [ - { - model: VideoFileModel, - required: true, - include: [ - { - model: VideoModel, - required: true, - where: { - id: videoId - } - } - ] - } - ] - } - - return Promise.all([ - VideoRedundancyModel.findAll(queryStreamingPlaylist), - VideoRedundancyModel.findAll(queryFiles) - ]).then(([ r1, r2 ]) => r1.concat(r2)) - } - - static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise { - const actor = await getServerActor() - - const query = { - where: { - actorId: actor.id, - videoStreamingPlaylistId - } - } - - return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) - } - - static loadByIdWithVideo (id: number, transaction?: Transaction): Promise { - const query = { - where: { id }, - transaction - } - - return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) - } - - static loadByUrl (url: string, transaction?: Transaction): Promise { - const query = { - where: { - url - }, - transaction - } - - return VideoRedundancyModel.findOne(query) - } - - static async isLocalByVideoUUIDExists (uuid: string) { - const actor = await getServerActor() - - const query = { - raw: true, - attributes: [ 'id' ], - where: { - actorId: actor.id - }, - include: [ - { - attributes: [], - model: VideoFileModel, - required: true, - include: [ - { - attributes: [], - model: VideoModel, - required: true, - where: { - uuid - } - } - ] - } - ] - } - - return VideoRedundancyModel.findOne(query) - .then(r => !!r) - } - - static async getVideoSample (p: Promise) { - const rows = await p - if (rows.length === 0) return undefined - - const ids = rows.map(r => r.id) - const id = sample(ids) - - return VideoModel.loadWithFiles(id, undefined, !isTestInstance()) - } - - static async findMostViewToDuplicate (randomizedFactor: number) { - const peertubeActor = await getServerActor() - - // On VideoModel! - const query = { - attributes: [ 'id', 'views' ], - limit: randomizedFactor, - order: getVideoSort('-views'), - where: { - privacy: VideoPrivacy.PUBLIC, - isLive: false, - ...this.buildVideoIdsForDuplication(peertubeActor) - }, - include: [ - VideoRedundancyModel.buildServerRedundancyInclude() - ] - } - - return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) - } - - static async findTrendingToDuplicate (randomizedFactor: number) { - const peertubeActor = await getServerActor() - - // On VideoModel! - const query = { - attributes: [ 'id', 'views' ], - subQuery: false, - group: 'VideoModel.id', - limit: randomizedFactor, - order: getVideoSort('-trending'), - where: { - privacy: VideoPrivacy.PUBLIC, - isLive: false, - ...this.buildVideoIdsForDuplication(peertubeActor) - }, - include: [ - VideoRedundancyModel.buildServerRedundancyInclude(), - - VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS) - ] - } - - return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) - } - - static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) { - const peertubeActor = await getServerActor() - - // On VideoModel! - const query = { - attributes: [ 'id', 'publishedAt' ], - limit: randomizedFactor, - order: getVideoSort('-publishedAt'), - where: { - privacy: VideoPrivacy.PUBLIC, - isLive: false, - views: { - [Op.gte]: minViews - }, - ...this.buildVideoIdsForDuplication(peertubeActor) - }, - include: [ - VideoRedundancyModel.buildServerRedundancyInclude(), - - // Required by publishedAt sort - { - model: ScheduleVideoUpdateModel.unscoped(), - required: false - } - ] - } - - return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) - } - - static async loadOldestLocalExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number): Promise { - const expiredDate = new Date() - expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs) - - const actor = await getServerActor() - - const query = { - where: { - actorId: actor.id, - strategy, - createdAt: { - [Op.lt]: expiredDate - } - } - } - - return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query) - } - - static async listLocalExpired (): Promise { - const actor = await getServerActor() - - const query = { - where: { - actorId: actor.id, - expiresOn: { - [Op.lt]: new Date() - } - } - } - - return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query) - } - - static async listRemoteExpired () { - const actor = await getServerActor() - - const query = { - where: { - actorId: { - [Op.ne]: actor.id - }, - expiresOn: { - [Op.lt]: new Date(), - [Op.ne]: null - } - } - } - - return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query) - } - - static async listLocalOfServer (serverId: number) { - const actor = await getServerActor() - const buildVideoInclude = () => ({ - model: VideoModel, - required: true, - include: [ - { - attributes: [], - model: VideoChannelModel.unscoped(), - required: true, - include: [ - { - attributes: [], - model: ActorModel.unscoped(), - required: true, - where: { - serverId - } - } - ] - } - ] - }) - - const query = { - where: { - [Op.and]: [ - { - actorId: actor.id - }, - { - [Op.or]: [ - { - '$VideoStreamingPlaylist.id$': { - [Op.ne]: null - } - }, - { - '$VideoFile.id$': { - [Op.ne]: null - } - } - ] - } - ] - }, - include: [ - { - model: VideoFileModel.unscoped(), - required: false, - include: [ buildVideoInclude() ] - }, - { - model: VideoStreamingPlaylistModel.unscoped(), - required: false, - include: [ buildVideoInclude() ] - } - ] - } - - return VideoRedundancyModel.findAll(query) - } - - static listForApi (options: { - start: number - count: number - sort: string - target: VideoRedundanciesTarget - strategy?: string - }) { - const { start, count, sort, target, strategy } = options - const redundancyWhere: WhereOptions = {} - const videosWhere: WhereOptions = {} - let redundancySqlSuffix = '' - - if (target === 'my-videos') { - Object.assign(videosWhere, { remote: false }) - } else if (target === 'remote-videos') { - Object.assign(videosWhere, { remote: true }) - Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } }) - redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL' - } - - if (strategy) { - Object.assign(redundancyWhere, { strategy }) - } - - const videoFilterWhere = { - [Op.and]: [ - { - [Op.or]: [ - { - id: { - [Op.in]: literal( - '(' + - 'SELECT "videoId" FROM "videoFile" ' + - 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' + - redundancySqlSuffix + - ')' - ) - } - }, - { - id: { - [Op.in]: literal( - '(' + - 'select "videoId" FROM "videoStreamingPlaylist" ' + - 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' + - redundancySqlSuffix + - ')' - ) - } - } - ] - }, - - videosWhere - ] - } - - // /!\ On video model /!\ - const findOptions = { - offset: start, - limit: count, - order: getSort(sort), - include: [ - { - required: false, - model: VideoFileModel, - include: [ - { - model: VideoRedundancyModel.unscoped(), - required: false, - where: redundancyWhere - } - ] - }, - { - required: false, - model: VideoStreamingPlaylistModel.unscoped(), - include: [ - { - model: VideoRedundancyModel.unscoped(), - required: false, - where: redundancyWhere - }, - { - model: VideoFileModel, - required: false - } - ] - } - ], - where: videoFilterWhere - } - - // /!\ On video model /!\ - const countOptions = { - where: videoFilterWhere - } - - return Promise.all([ - VideoModel.findAll(findOptions), - - VideoModel.count(countOptions) - ]).then(([ data, total ]) => ({ total, data })) - } - - static async getStats (strategy: VideoRedundancyStrategyWithManual) { - const actor = await getServerActor() - - const sql = `WITH "tmp" AS ` + - `(` + - `SELECT "videoFile"."size" AS "videoFileSize", "videoStreamingFile"."size" AS "videoStreamingFileSize", ` + - `"videoFile"."videoId" AS "videoFileVideoId", "videoStreamingPlaylist"."videoId" AS "videoStreamingVideoId"` + - `FROM "videoRedundancy" AS "videoRedundancy" ` + - `LEFT JOIN "videoFile" AS "videoFile" ON "videoRedundancy"."videoFileId" = "videoFile"."id" ` + - `LEFT JOIN "videoStreamingPlaylist" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ` + - `LEFT JOIN "videoFile" AS "videoStreamingFile" ` + - `ON "videoStreamingPlaylist"."id" = "videoStreamingFile"."videoStreamingPlaylistId" ` + - `WHERE "videoRedundancy"."strategy" = :strategy AND "videoRedundancy"."actorId" = :actorId` + - `), ` + - `"videoIds" AS (` + - `SELECT "videoFileVideoId" AS "videoId" FROM "tmp" ` + - `UNION SELECT "videoStreamingVideoId" AS "videoId" FROM "tmp" ` + - `) ` + - `SELECT ` + - `COALESCE(SUM("videoFileSize"), '0') + COALESCE(SUM("videoStreamingFileSize"), '0') AS "totalUsed", ` + - `(SELECT COUNT("videoIds"."videoId") FROM "videoIds") AS "totalVideos", ` + - `COUNT(*) AS "totalVideoFiles" ` + - `FROM "tmp"` - - return VideoRedundancyModel.sequelize.query(sql, { - replacements: { strategy, actorId: actor.id }, - type: QueryTypes.SELECT - }).then(([ row ]) => ({ - totalUsed: parseAggregateResult(row.totalUsed), - totalVideos: row.totalVideos, - totalVideoFiles: row.totalVideoFiles - })) - } - - static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy { - const filesRedundancies: FileRedundancyInformation[] = [] - const streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = [] - - for (const file of video.VideoFiles) { - for (const redundancy of file.RedundancyVideos) { - filesRedundancies.push({ - id: redundancy.id, - fileUrl: redundancy.fileUrl, - strategy: redundancy.strategy, - createdAt: redundancy.createdAt, - updatedAt: redundancy.updatedAt, - expiresOn: redundancy.expiresOn, - size: file.size - }) - } - } - - for (const playlist of video.VideoStreamingPlaylists) { - const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0) - - for (const redundancy of playlist.RedundancyVideos) { - streamingPlaylistsRedundancies.push({ - id: redundancy.id, - fileUrl: redundancy.fileUrl, - strategy: redundancy.strategy, - createdAt: redundancy.createdAt, - updatedAt: redundancy.updatedAt, - expiresOn: redundancy.expiresOn, - size - }) - } - } - - return { - id: video.id, - name: video.name, - url: video.url, - uuid: video.uuid, - - redundancies: { - files: filesRedundancies, - streamingPlaylists: streamingPlaylistsRedundancies - } - } - } - - getVideo () { - if (this.VideoFile?.Video) return this.VideoFile.Video - - if (this.VideoStreamingPlaylist?.Video) return this.VideoStreamingPlaylist.Video - - return undefined - } - - getVideoUUID () { - const video = this.getVideo() - if (!video) return undefined - - return video.uuid - } - - isOwned () { - return !!this.strategy - } - - toActivityPubObject (this: MVideoRedundancyAP): CacheFileObject { - if (this.VideoStreamingPlaylist) { - return { - id: this.url, - type: 'CacheFile' as 'CacheFile', - object: this.VideoStreamingPlaylist.Video.url, - expires: this.expiresOn ? this.expiresOn.toISOString() : null, - url: { - type: 'Link', - mediaType: 'application/x-mpegURL', - href: this.fileUrl - } - } - } - - return { - id: this.url, - type: 'CacheFile' as 'CacheFile', - object: this.VideoFile.Video.url, - expires: this.expiresOn ? this.expiresOn.toISOString() : null, - url: { - type: 'Link', - mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[this.VideoFile.extname] as any, - href: this.fileUrl, - height: this.VideoFile.resolution, - size: this.VideoFile.size, - fps: this.VideoFile.fps - } - } - } - - // Don't include video files we already duplicated - private static buildVideoIdsForDuplication (peertubeActor: MActor) { - const notIn = literal( - '(' + - `SELECT "videoFile"."videoId" AS "videoId" FROM "videoRedundancy" ` + - `INNER JOIN "videoFile" ON "videoFile"."id" = "videoRedundancy"."videoFileId" ` + - `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` + - `UNION ` + - `SELECT "videoStreamingPlaylist"."videoId" AS "videoId" FROM "videoRedundancy" ` + - `INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoRedundancy"."videoStreamingPlaylistId" ` + - `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` + - ')' - ) - - return { - id: { - [Op.notIn]: notIn - } - } - } - - private static buildServerRedundancyInclude () { - return { - attributes: [], - model: VideoChannelModel.unscoped(), - required: true, - include: [ - { - attributes: [], - model: ActorModel.unscoped(), - required: true, - include: [ - { - attributes: [], - model: ServerModel.unscoped(), - required: true, - where: { - redundancyAllowed: true - } - } - ] - } - ] - } - } -} diff --git a/server/models/runner/runner-job.ts b/server/models/runner/runner-job.ts deleted file mode 100644 index f2ffd6a84..000000000 --- a/server/models/runner/runner-job.ts +++ /dev/null @@ -1,357 +0,0 @@ -import { Op, Transaction } from 'sequelize' -import { - AllowNull, - BelongsTo, - Column, - CreatedAt, - DataType, - Default, - ForeignKey, - IsUUID, - Model, - Scopes, - Table, - UpdatedAt -} from 'sequelize-typescript' -import { isArray, isUUIDValid } from '@server/helpers/custom-validators/misc' -import { CONSTRAINTS_FIELDS, RUNNER_JOB_STATES } from '@server/initializers/constants' -import { MRunnerJob, MRunnerJobRunner, MRunnerJobRunnerParent } from '@server/types/models/runners' -import { RunnerJob, RunnerJobAdmin, RunnerJobPayload, RunnerJobPrivatePayload, RunnerJobState, RunnerJobType } from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { getSort, searchAttribute } from '../shared' -import { RunnerModel } from './runner' - -enum ScopeNames { - WITH_RUNNER = 'WITH_RUNNER', - WITH_PARENT = 'WITH_PARENT' -} - -@Scopes(() => ({ - [ScopeNames.WITH_RUNNER]: { - include: [ - { - model: RunnerModel.unscoped(), - required: false - } - ] - }, - [ScopeNames.WITH_PARENT]: { - include: [ - { - model: RunnerJobModel.unscoped(), - required: false - } - ] - } -})) -@Table({ - tableName: 'runnerJob', - indexes: [ - { - fields: [ 'uuid' ], - unique: true - }, - { - fields: [ 'processingJobToken' ], - unique: true - }, - { - fields: [ 'runnerId' ] - } - ] -}) -export class RunnerJobModel extends Model>> { - - @AllowNull(false) - @IsUUID(4) - @Column(DataType.UUID) - uuid: string - - @AllowNull(false) - @Column - type: RunnerJobType - - @AllowNull(false) - @Column(DataType.JSONB) - payload: RunnerJobPayload - - @AllowNull(false) - @Column(DataType.JSONB) - privatePayload: RunnerJobPrivatePayload - - @AllowNull(false) - @Column - state: RunnerJobState - - @AllowNull(false) - @Default(0) - @Column - failures: number - - @AllowNull(true) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.RUNNER_JOBS.ERROR_MESSAGE.max)) - error: string - - // Less has priority - @AllowNull(false) - @Column - priority: number - - // Used to fetch the appropriate job when the runner wants to post the result - @AllowNull(true) - @Column - processingJobToken: string - - @AllowNull(true) - @Column - progress: number - - @AllowNull(true) - @Column - startedAt: Date - - @AllowNull(true) - @Column - finishedAt: Date - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => RunnerJobModel) - @Column - dependsOnRunnerJobId: number - - @BelongsTo(() => RunnerJobModel, { - foreignKey: { - name: 'dependsOnRunnerJobId', - allowNull: true - }, - onDelete: 'cascade' - }) - DependsOnRunnerJob: RunnerJobModel - - @ForeignKey(() => RunnerModel) - @Column - runnerId: number - - @BelongsTo(() => RunnerModel, { - foreignKey: { - name: 'runnerId', - allowNull: true - }, - onDelete: 'SET NULL' - }) - Runner: RunnerModel - - // --------------------------------------------------------------------------- - - static loadWithRunner (uuid: string) { - const query = { - where: { uuid } - } - - return RunnerJobModel.scope(ScopeNames.WITH_RUNNER).findOne(query) - } - - static loadByRunnerAndJobTokensWithRunner (options: { - uuid: string - runnerToken: string - jobToken: string - }) { - const { uuid, runnerToken, jobToken } = options - - const query = { - where: { - uuid, - processingJobToken: jobToken - }, - include: { - model: RunnerModel.unscoped(), - required: true, - where: { - runnerToken - } - } - } - - return RunnerJobModel.findOne(query) - } - - static listAvailableJobs () { - const query = { - limit: 10, - order: getSort('priority'), - where: { - state: RunnerJobState.PENDING - } - } - - return RunnerJobModel.findAll(query) - } - - static listStalledJobs (options: { - staleTimeMS: number - types: RunnerJobType[] - }) { - const before = new Date(Date.now() - options.staleTimeMS) - - return RunnerJobModel.findAll({ - where: { - type: { - [Op.in]: options.types - }, - state: RunnerJobState.PROCESSING, - updatedAt: { - [Op.lt]: before - } - } - }) - } - - static listChildrenOf (job: MRunnerJob, transaction?: Transaction) { - const query = { - where: { - dependsOnRunnerJobId: job.id - }, - transaction - } - - return RunnerJobModel.findAll(query) - } - - static listForApi (options: { - start: number - count: number - sort: string - search?: string - stateOneOf?: RunnerJobState[] - }) { - const { start, count, sort, search, stateOneOf } = options - - const query = { - offset: start, - limit: count, - order: getSort(sort), - where: [] - } - - if (search) { - if (isUUIDValid(search)) { - query.where.push({ uuid: search }) - } else { - query.where.push({ - [Op.or]: [ - searchAttribute(search, 'type'), - searchAttribute(search, '$Runner.name$') - ] - }) - } - } - - if (isArray(stateOneOf) && stateOneOf.length !== 0) { - query.where.push({ - state: { - [Op.in]: stateOneOf - } - }) - } - - return Promise.all([ - RunnerJobModel.scope([ ScopeNames.WITH_RUNNER ]).count(query), - RunnerJobModel.scope([ ScopeNames.WITH_RUNNER, ScopeNames.WITH_PARENT ]).findAll(query) - ]).then(([ total, data ]) => ({ total, data })) - } - - static updateDependantJobsOf (runnerJob: MRunnerJob) { - const where = { - dependsOnRunnerJobId: runnerJob.id - } - - return RunnerJobModel.update({ state: RunnerJobState.PENDING }, { where }) - } - - static cancelAllJobs (options: { type: RunnerJobType }) { - const where = { - type: options.type - } - - return RunnerJobModel.update({ state: RunnerJobState.CANCELLED }, { where }) - } - - // --------------------------------------------------------------------------- - - resetToPending () { - this.state = RunnerJobState.PENDING - this.processingJobToken = null - this.progress = null - this.startedAt = null - this.runnerId = null - } - - setToErrorOrCancel ( - state: RunnerJobState.PARENT_ERRORED | RunnerJobState.ERRORED | RunnerJobState.CANCELLED | RunnerJobState.PARENT_CANCELLED - ) { - this.state = state - this.processingJobToken = null - this.finishedAt = new Date() - } - - toFormattedJSON (this: MRunnerJobRunnerParent): RunnerJob { - const runner = this.Runner - ? { - id: this.Runner.id, - name: this.Runner.name, - description: this.Runner.description - } - : null - - const parent = this.DependsOnRunnerJob - ? { - id: this.DependsOnRunnerJob.id, - uuid: this.DependsOnRunnerJob.uuid, - type: this.DependsOnRunnerJob.type, - state: { - id: this.DependsOnRunnerJob.state, - label: RUNNER_JOB_STATES[this.DependsOnRunnerJob.state] - } - } - : undefined - - return { - uuid: this.uuid, - type: this.type, - - state: { - id: this.state, - label: RUNNER_JOB_STATES[this.state] - }, - - progress: this.progress, - priority: this.priority, - failures: this.failures, - error: this.error, - - payload: this.payload, - - startedAt: this.startedAt?.toISOString(), - finishedAt: this.finishedAt?.toISOString(), - - createdAt: this.createdAt.toISOString(), - updatedAt: this.updatedAt.toISOString(), - - parent, - runner - } - } - - toFormattedAdminJSON (this: MRunnerJobRunnerParent): RunnerJobAdmin { - return { - ...this.toFormattedJSON(), - - privatePayload: this.privatePayload - } - } -} diff --git a/server/models/runner/runner-registration-token.ts b/server/models/runner/runner-registration-token.ts deleted file mode 100644 index b2ae6c9eb..000000000 --- a/server/models/runner/runner-registration-token.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { FindOptions, literal } from 'sequelize' -import { AllowNull, Column, CreatedAt, HasMany, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { MRunnerRegistrationToken } from '@server/types/models/runners' -import { RunnerRegistrationToken } from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { getSort } from '../shared' -import { RunnerModel } from './runner' - -/** - * - * Tokens used by PeerTube runners to register themselves to the PeerTube instance - * - */ - -@Table({ - tableName: 'runnerRegistrationToken', - indexes: [ - { - fields: [ 'registrationToken' ], - unique: true - } - ] -}) -export class RunnerRegistrationTokenModel extends Model>> { - - @AllowNull(false) - @Column - registrationToken: string - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @HasMany(() => RunnerModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'cascade' - }) - Runners: RunnerModel[] - - static load (id: number) { - return RunnerRegistrationTokenModel.findByPk(id) - } - - static loadByRegistrationToken (registrationToken: string) { - const query = { - where: { registrationToken } - } - - return RunnerRegistrationTokenModel.findOne(query) - } - - static countTotal () { - return RunnerRegistrationTokenModel.unscoped().count() - } - - static listForApi (options: { - start: number - count: number - sort: string - }) { - const { start, count, sort } = options - - const query: FindOptions = { - attributes: { - include: [ - [ - literal('(SELECT COUNT(*) FROM "runner" WHERE "runner"."runnerRegistrationTokenId" = "RunnerRegistrationTokenModel"."id")'), - 'registeredRunnersCount' - ] - ] - }, - offset: start, - limit: count, - order: getSort(sort) - } - - return Promise.all([ - RunnerRegistrationTokenModel.count(query), - RunnerRegistrationTokenModel.findAll(query) - ]).then(([ total, data ]) => ({ total, data })) - } - - // --------------------------------------------------------------------------- - - toFormattedJSON (this: MRunnerRegistrationToken): RunnerRegistrationToken { - const registeredRunnersCount = this.get('registeredRunnersCount') as number - - return { - id: this.id, - - registrationToken: this.registrationToken, - - createdAt: this.createdAt, - updatedAt: this.updatedAt, - - registeredRunnersCount - } - } -} diff --git a/server/models/runner/runner.ts b/server/models/runner/runner.ts deleted file mode 100644 index 4d07707d8..000000000 --- a/server/models/runner/runner.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { FindOptions } from 'sequelize' -import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { MRunner } from '@server/types/models/runners' -import { Runner } from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { getSort } from '../shared' -import { RunnerRegistrationTokenModel } from './runner-registration-token' -import { CONSTRAINTS_FIELDS } from '@server/initializers/constants' - -@Table({ - tableName: 'runner', - indexes: [ - { - fields: [ 'runnerToken' ], - unique: true - }, - { - fields: [ 'runnerRegistrationTokenId' ] - }, - { - fields: [ 'name' ], - unique: true - } - ] -}) -export class RunnerModel extends Model>> { - - // Used to identify the appropriate runner when it uses the runner REST API - @AllowNull(false) - @Column - runnerToken: string - - @AllowNull(false) - @Column - name: string - - @AllowNull(true) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.RUNNERS.DESCRIPTION.max)) - description: string - - @AllowNull(false) - @Column - lastContact: Date - - @AllowNull(false) - @Column - ip: string - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => RunnerRegistrationTokenModel) - @Column - runnerRegistrationTokenId: number - - @BelongsTo(() => RunnerRegistrationTokenModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade' - }) - RunnerRegistrationToken: RunnerRegistrationTokenModel - - // --------------------------------------------------------------------------- - - static load (id: number) { - return RunnerModel.findByPk(id) - } - - static loadByToken (runnerToken: string) { - const query = { - where: { runnerToken } - } - - return RunnerModel.findOne(query) - } - - static loadByName (name: string) { - const query = { - where: { name } - } - - return RunnerModel.findOne(query) - } - - static listForApi (options: { - start: number - count: number - sort: string - }) { - const { start, count, sort } = options - - const query: FindOptions = { - offset: start, - limit: count, - order: getSort(sort) - } - - return Promise.all([ - RunnerModel.count(query), - RunnerModel.findAll(query) - ]).then(([ total, data ]) => ({ total, data })) - } - - // --------------------------------------------------------------------------- - - toFormattedJSON (this: MRunner): Runner { - return { - id: this.id, - - name: this.name, - description: this.description, - - ip: this.ip, - lastContact: this.lastContact, - - createdAt: this.createdAt, - updatedAt: this.updatedAt - } - } -} diff --git a/server/models/server/plugin.ts b/server/models/server/plugin.ts deleted file mode 100644 index 9948c9f7a..000000000 --- a/server/models/server/plugin.ts +++ /dev/null @@ -1,305 +0,0 @@ -import { FindAndCountOptions, json, QueryTypes } from 'sequelize' -import { AllowNull, Column, CreatedAt, DataType, DefaultScope, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { MPlugin, MPluginFormattable } from '@server/types/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { PeerTubePlugin, PluginType, RegisterServerSettingOptions, SettingEntries, SettingValue } from '../../../shared/models' -import { - isPluginDescriptionValid, - isPluginHomepage, - isPluginNameValid, - isPluginStableOrUnstableVersionValid, - isPluginStableVersionValid, - isPluginTypeValid -} from '../../helpers/custom-validators/plugins' -import { getSort, throwIfNotValid } from '../shared' - -@DefaultScope(() => ({ - attributes: { - exclude: [ 'storage' ] - } -})) - -@Table({ - tableName: 'plugin', - indexes: [ - { - fields: [ 'name', 'type' ], - unique: true - } - ] -}) -export class PluginModel extends Model>> { - - @AllowNull(false) - @Is('PluginName', value => throwIfNotValid(value, isPluginNameValid, 'name')) - @Column - name: string - - @AllowNull(false) - @Is('PluginType', value => throwIfNotValid(value, isPluginTypeValid, 'type')) - @Column - type: number - - @AllowNull(false) - @Is('PluginVersion', value => throwIfNotValid(value, isPluginStableOrUnstableVersionValid, 'version')) - @Column - version: string - - @AllowNull(true) - @Is('PluginLatestVersion', value => throwIfNotValid(value, isPluginStableVersionValid, 'version')) - @Column - latestVersion: string - - @AllowNull(false) - @Column - enabled: boolean - - @AllowNull(false) - @Column - uninstalled: boolean - - @AllowNull(false) - @Column - peertubeEngine: string - - @AllowNull(true) - @Is('PluginDescription', value => throwIfNotValid(value, isPluginDescriptionValid, 'description')) - @Column - description: string - - @AllowNull(false) - @Is('PluginHomepage', value => throwIfNotValid(value, isPluginHomepage, 'homepage')) - @Column - homepage: string - - @AllowNull(true) - @Column(DataType.JSONB) - settings: any - - @AllowNull(true) - @Column(DataType.JSONB) - storage: any - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - static listEnabledPluginsAndThemes (): Promise { - const query = { - where: { - enabled: true, - uninstalled: false - } - } - - return PluginModel.findAll(query) - } - - static loadByNpmName (npmName: string): Promise { - const name = this.normalizePluginName(npmName) - const type = this.getTypeFromNpmName(npmName) - - const query = { - where: { - name, - type - } - } - - return PluginModel.findOne(query) - } - - static getSetting (pluginName: string, pluginType: PluginType, settingName: string, registeredSettings: RegisterServerSettingOptions[]) { - const query = { - attributes: [ 'settings' ], - where: { - name: pluginName, - type: pluginType - } - } - - return PluginModel.findOne(query) - .then(p => { - if (!p?.settings || p.settings === undefined) { - const registered = registeredSettings.find(s => s.name === settingName) - if (!registered || registered.default === undefined) return undefined - - return registered.default - } - - return p.settings[settingName] - }) - } - - static getSettings ( - pluginName: string, - pluginType: PluginType, - settingNames: string[], - registeredSettings: RegisterServerSettingOptions[] - ) { - const query = { - attributes: [ 'settings' ], - where: { - name: pluginName, - type: pluginType - } - } - - return PluginModel.findOne(query) - .then(p => { - const result: SettingEntries = {} - - for (const name of settingNames) { - if (!p?.settings || p.settings[name] === undefined) { - const registered = registeredSettings.find(s => s.name === name) - - if (registered?.default !== undefined) { - result[name] = registered.default - } - } else { - result[name] = p.settings[name] - } - } - - return result - }) - } - - static setSetting (pluginName: string, pluginType: PluginType, settingName: string, settingValue: SettingValue) { - const query = { - where: { - name: pluginName, - type: pluginType - } - } - - const toSave = { - [`settings.${settingName}`]: settingValue - } - - return PluginModel.update(toSave, query) - .then(() => undefined) - } - - static getData (pluginName: string, pluginType: PluginType, key: string) { - const query = { - raw: true, - attributes: [ [ json('storage.' + key), 'value' ] as any ], // FIXME: typings - where: { - name: pluginName, - type: pluginType - } - } - - return PluginModel.findOne(query) - .then((c: any) => { - if (!c) return undefined - const value = c.value - - try { - return JSON.parse(value) - } catch { - return value - } - }) - } - - static storeData (pluginName: string, pluginType: PluginType, key: string, data: any) { - const query = 'UPDATE "plugin" SET "storage" = jsonb_set(coalesce("storage", \'{}\'), :key, :data::jsonb) ' + - 'WHERE "name" = :pluginName AND "type" = :pluginType' - - const jsonPath = '{' + key + '}' - - const options = { - replacements: { pluginName, pluginType, key: jsonPath, data: JSON.stringify(data) }, - type: QueryTypes.UPDATE - } - - return PluginModel.sequelize.query(query, options) - .then(() => undefined) - } - - static listForApi (options: { - pluginType?: PluginType - uninstalled?: boolean - start: number - count: number - sort: string - }) { - const { uninstalled = false } = options - const query: FindAndCountOptions = { - offset: options.start, - limit: options.count, - order: getSort(options.sort), - where: { - uninstalled - } - } - - if (options.pluginType) query.where['type'] = options.pluginType - - return Promise.all([ - PluginModel.count(query), - PluginModel.findAll(query) - ]).then(([ total, data ]) => ({ total, data })) - } - - static listInstalled (): Promise { - const query = { - where: { - uninstalled: false - } - } - - return PluginModel.findAll(query) - } - - static normalizePluginName (npmName: string) { - return npmName.replace(/^peertube-((theme)|(plugin))-/, '') - } - - static getTypeFromNpmName (npmName: string) { - return npmName.startsWith('peertube-plugin-') - ? PluginType.PLUGIN - : PluginType.THEME - } - - static buildNpmName (name: string, type: PluginType) { - if (type === PluginType.THEME) return 'peertube-theme-' + name - - return 'peertube-plugin-' + name - } - - getPublicSettings (registeredSettings: RegisterServerSettingOptions[]) { - const result: SettingEntries = {} - const settings = this.settings || {} - - for (const r of registeredSettings) { - if (r.private !== false) continue - - result[r.name] = settings[r.name] ?? r.default ?? null - } - - return result - } - - toFormattedJSON (this: MPluginFormattable): PeerTubePlugin { - return { - name: this.name, - type: this.type, - version: this.version, - latestVersion: this.latestVersion, - enabled: this.enabled, - uninstalled: this.uninstalled, - peertubeEngine: this.peertubeEngine, - description: this.description, - homepage: this.homepage, - settings: this.settings, - createdAt: this.createdAt, - updatedAt: this.updatedAt - } - } - -} diff --git a/server/models/server/server-blocklist.ts b/server/models/server/server-blocklist.ts deleted file mode 100644 index 3d755fe4a..000000000 --- a/server/models/server/server-blocklist.ts +++ /dev/null @@ -1,190 +0,0 @@ -import { Op, QueryTypes } from 'sequelize' -import { BelongsTo, Column, CreatedAt, ForeignKey, Model, Scopes, Table, UpdatedAt } from 'sequelize-typescript' -import { MServerBlocklist, MServerBlocklistAccountServer, MServerBlocklistFormattable } from '@server/types/models' -import { ServerBlock } from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { AccountModel } from '../account/account' -import { createSafeIn, getSort, searchAttribute } from '../shared' -import { ServerModel } from './server' - -enum ScopeNames { - WITH_ACCOUNT = 'WITH_ACCOUNT', - WITH_SERVER = 'WITH_SERVER' -} - -@Scopes(() => ({ - [ScopeNames.WITH_ACCOUNT]: { - include: [ - { - model: AccountModel, - required: true - } - ] - }, - [ScopeNames.WITH_SERVER]: { - include: [ - { - model: ServerModel, - required: true - } - ] - } -})) - -@Table({ - tableName: 'serverBlocklist', - indexes: [ - { - fields: [ 'accountId', 'targetServerId' ], - unique: true - }, - { - fields: [ 'targetServerId' ] - } - ] -}) -export class ServerBlocklistModel extends Model>> { - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => AccountModel) - @Column - accountId: number - - @BelongsTo(() => AccountModel, { - foreignKey: { - name: 'accountId', - allowNull: false - }, - onDelete: 'CASCADE' - }) - ByAccount: AccountModel - - @ForeignKey(() => ServerModel) - @Column - targetServerId: number - - @BelongsTo(() => ServerModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'CASCADE' - }) - BlockedServer: ServerModel - - static isServerMutedByAccounts (accountIds: number[], targetServerId: number) { - const query = { - attributes: [ 'accountId', 'id' ], - where: { - accountId: { - [Op.in]: accountIds - }, - targetServerId - }, - raw: true - } - - return ServerBlocklistModel.unscoped() - .findAll(query) - .then(rows => { - const result: { [accountId: number]: boolean } = {} - - for (const accountId of accountIds) { - result[accountId] = !!rows.find(r => r.accountId === accountId) - } - - return result - }) - } - - static loadByAccountAndHost (accountId: number, host: string): Promise { - const query = { - where: { - accountId - }, - include: [ - { - model: ServerModel, - where: { - host - }, - required: true - } - ] - } - - return ServerBlocklistModel.findOne(query) - } - - static listHostsBlockedBy (accountIds: number[]): Promise { - const query = { - attributes: [ ], - where: { - accountId: { - [Op.in]: accountIds - } - }, - include: [ - { - attributes: [ 'host' ], - model: ServerModel.unscoped(), - required: true - } - ] - } - - return ServerBlocklistModel.findAll(query) - .then(entries => entries.map(e => e.BlockedServer.host)) - } - - static getBlockStatus (byAccountIds: number[], hosts: string[]): Promise<{ host: string, accountId: number }[]> { - const rawQuery = `SELECT "server"."host", "serverBlocklist"."accountId" ` + - `FROM "serverBlocklist" ` + - `INNER JOIN "server" ON "server"."id" = "serverBlocklist"."targetServerId" ` + - `WHERE "server"."host" IN (:hosts) ` + - `AND "serverBlocklist"."accountId" IN (${createSafeIn(ServerBlocklistModel.sequelize, byAccountIds)})` - - return ServerBlocklistModel.sequelize.query(rawQuery, { - type: QueryTypes.SELECT as QueryTypes.SELECT, - replacements: { hosts } - }) - } - - static listForApi (parameters: { - start: number - count: number - sort: string - search?: string - accountId: number - }) { - const { start, count, sort, search, accountId } = parameters - - const query = { - offset: start, - limit: count, - order: getSort(sort), - where: { - accountId, - - ...searchAttribute(search, '$BlockedServer.host$') - } - } - - return Promise.all([ - ServerBlocklistModel.scope(ScopeNames.WITH_SERVER).count(query), - ServerBlocklistModel.scope([ ScopeNames.WITH_ACCOUNT, ScopeNames.WITH_SERVER ]).findAll(query) - ]).then(([ total, data ]) => ({ total, data })) - } - - toFormattedJSON (this: MServerBlocklistFormattable): ServerBlock { - return { - byAccount: this.ByAccount.toFormattedJSON(), - blockedServer: this.BlockedServer.toFormattedJSON(), - createdAt: this.createdAt - } - } -} diff --git a/server/models/server/server.ts b/server/models/server/server.ts deleted file mode 100644 index a5e05f460..000000000 --- a/server/models/server/server.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { Transaction } from 'sequelize' -import { AllowNull, Column, CreatedAt, Default, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { MServer, MServerFormattable } from '@server/types/models/server' -import { AttributesOnly } from '@shared/typescript-utils' -import { isHostValid } from '../../helpers/custom-validators/servers' -import { ActorModel } from '../actor/actor' -import { buildSQLAttributes, throwIfNotValid } from '../shared' -import { ServerBlocklistModel } from './server-blocklist' - -@Table({ - tableName: 'server', - indexes: [ - { - fields: [ 'host' ], - unique: true - } - ] -}) -export class ServerModel extends Model>> { - - @AllowNull(false) - @Is('Host', value => throwIfNotValid(value, isHostValid, 'valid host')) - @Column - host: string - - @AllowNull(false) - @Default(false) - @Column - redundancyAllowed: boolean - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @HasMany(() => ActorModel, { - foreignKey: { - name: 'serverId', - allowNull: true - }, - onDelete: 'CASCADE', - hooks: true - }) - Actors: ActorModel[] - - @HasMany(() => ServerBlocklistModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'CASCADE' - }) - BlockedBy: ServerBlocklistModel[] - - // --------------------------------------------------------------------------- - - static getSQLAttributes (tableName: string, aliasPrefix = '') { - return buildSQLAttributes({ - model: this, - tableName, - aliasPrefix - }) - } - - // --------------------------------------------------------------------------- - - static load (id: number, transaction?: Transaction): Promise { - const query = { - where: { - id - }, - transaction - } - - return ServerModel.findOne(query) - } - - static loadByHost (host: string): Promise { - const query = { - where: { - host - } - } - - return ServerModel.findOne(query) - } - - static async loadOrCreateByHost (host: string) { - let server = await ServerModel.loadByHost(host) - if (!server) server = await ServerModel.create({ host }) - - return server - } - - isBlocked () { - return this.BlockedBy && this.BlockedBy.length !== 0 - } - - toFormattedJSON (this: MServerFormattable) { - return { - host: this.host - } - } -} diff --git a/server/models/server/tracker.ts b/server/models/server/tracker.ts deleted file mode 100644 index ee087c4a3..000000000 --- a/server/models/server/tracker.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { AllowNull, BelongsToMany, Column, CreatedAt, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { Transaction } from 'sequelize/types' -import { MTracker } from '@server/types/models/server/tracker' -import { AttributesOnly } from '@shared/typescript-utils' -import { VideoModel } from '../video/video' -import { VideoTrackerModel } from './video-tracker' - -@Table({ - tableName: 'tracker', - indexes: [ - { - fields: [ 'url' ], - unique: true - } - ] -}) -export class TrackerModel extends Model>> { - - @AllowNull(false) - @Column - url: string - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @BelongsToMany(() => VideoModel, { - foreignKey: 'trackerId', - through: () => VideoTrackerModel, - onDelete: 'CASCADE' - }) - Videos: VideoModel[] - - static listUrlsByVideoId (videoId: number) { - const query = { - include: [ - { - attributes: [ 'id' ], - model: VideoModel.unscoped(), - required: true, - where: { id: videoId } - } - ] - } - - return TrackerModel.findAll(query) - .then(rows => rows.map(rows => rows.url)) - } - - static findOrCreateTrackers (trackers: string[], transaction: Transaction): Promise { - if (trackers === null) return Promise.resolve([]) - - const tasks: Promise[] = [] - trackers.forEach(tracker => { - const query = { - where: { - url: tracker - }, - defaults: { - url: tracker - }, - transaction - } - - const promise = TrackerModel.findOrCreate(query) - .then(([ trackerInstance ]) => trackerInstance) - tasks.push(promise) - }) - - return Promise.all(tasks) - } -} diff --git a/server/models/server/video-tracker.ts b/server/models/server/video-tracker.ts deleted file mode 100644 index f14f3bd7d..000000000 --- a/server/models/server/video-tracker.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { Column, CreatedAt, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { AttributesOnly } from '@shared/typescript-utils' -import { VideoModel } from '../video/video' -import { TrackerModel } from './tracker' - -@Table({ - tableName: 'videoTracker', - indexes: [ - { - fields: [ 'videoId' ] - }, - { - fields: [ 'trackerId' ] - } - ] -}) -export class VideoTrackerModel extends Model>> { - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => VideoModel) - @Column - videoId: number - - @ForeignKey(() => TrackerModel) - @Column - trackerId: number -} diff --git a/server/models/shared/index.ts b/server/models/shared/index.ts deleted file mode 100644 index 5a7621e4d..000000000 --- a/server/models/shared/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -export * from './abstract-run-query' -export * from './model-builder' -export * from './model-cache' -export * from './query' -export * from './sequelize-helpers' -export * from './sort' -export * from './sql' -export * from './update' diff --git a/server/models/shared/model-builder.ts b/server/models/shared/model-builder.ts deleted file mode 100644 index 07f7c4038..000000000 --- a/server/models/shared/model-builder.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { isPlainObject } from 'lodash' -import { Model as SequelizeModel, ModelStatic, Sequelize } from 'sequelize' -import { logger } from '@server/helpers/logger' - -/** - * - * Build Sequelize models from sequelize raw query (that must use { nest: true } options) - * - * In order to sequelize to correctly build the JSON this class will ingest, - * the columns selected in the raw query should be in the following form: - * * All tables must be Pascal Cased (for example "VideoChannel") - * * Root table must end with `Model` (for example "VideoCommentModel") - * * Joined tables must contain the origin table name + '->JoinedTable'. For example: - * * "Actor" is joined to "Account": "Actor" table must be renamed "Account->Actor" - * * "Account->Actor" is joined to "Server": "Server" table must be renamed to "Account->Actor->Server" - * * Selected columns must be renamed to contain the JSON path: - * * "videoComment"."id": "VideoCommentModel"."id" - * * "Account"."Actor"."Server"."id": "Account.Actor.Server.id" - * * All tables must contain the row id - */ - -export class ModelBuilder { - private readonly modelRegistry = new Map() - - constructor (private readonly sequelize: Sequelize) { - - } - - createModels (jsonArray: any[], baseModelName: string): T[] { - const result: T[] = [] - - for (const json of jsonArray) { - const { created, model } = this.createModel(json, baseModelName, json.id + '.' + baseModelName) - - if (created) result.push(model) - } - - return result - } - - private createModel (json: any, modelName: string, keyPath: string) { - if (!json.id) return { created: false, model: null } - - const { created, model } = this.createOrFindModel(json, modelName, keyPath) - - for (const key of Object.keys(json)) { - const value = json[key] - if (!value) continue - - // Child model - if (isPlainObject(value)) { - const { created, model: subModel } = this.createModel(value, key, keyPath + '.' + json.id + '.' + key) - if (!created || !subModel) continue - - const Model = this.findModelBuilder(modelName) - const association = Model.associations[key] - - if (!association) { - logger.error('Cannot find association %s of model %s', key, modelName, { associations: Object.keys(Model.associations) }) - continue - } - - if (association.isMultiAssociation) { - if (!Array.isArray(model[key])) model[key] = [] - - model[key].push(subModel) - } else { - model[key] = subModel - } - } - } - - return { created, model } - } - - private createOrFindModel (json: any, modelName: string, keyPath: string) { - const registryKey = this.getModelRegistryKey(json, keyPath) - if (this.modelRegistry.has(registryKey)) { - return { - created: false, - model: this.modelRegistry.get(registryKey) - } - } - - const Model = this.findModelBuilder(modelName) - - if (!Model) { - logger.error( - 'Cannot build model %s that does not exist', this.buildSequelizeModelName(modelName), - { existing: this.sequelize.modelManager.all.map(m => m.name) } - ) - return { created: false, model: null } - } - - const model = Model.build(json, { raw: true, isNewRecord: false }) - - this.modelRegistry.set(registryKey, model) - - return { created: true, model } - } - - private findModelBuilder (modelName: string) { - return this.sequelize.modelManager.getModel(this.buildSequelizeModelName(modelName)) as ModelStatic - } - - private buildSequelizeModelName (modelName: string) { - if (modelName === 'Avatars') return 'ActorImageModel' - if (modelName === 'ActorFollowing') return 'ActorModel' - if (modelName === 'ActorFollower') return 'ActorModel' - if (modelName === 'FlaggedAccount') return 'AccountModel' - - return modelName + 'Model' - } - - private getModelRegistryKey (json: any, keyPath: string) { - return keyPath + json.id - } -} diff --git a/server/models/shared/model-cache.ts b/server/models/shared/model-cache.ts deleted file mode 100644 index 3651267e7..000000000 --- a/server/models/shared/model-cache.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { Model } from 'sequelize-typescript' -import { logger } from '@server/helpers/logger' - -type ModelCacheType = - 'local-account-name' - | 'local-actor-name' - | 'local-actor-url' - | 'load-video-immutable-id' - | 'load-video-immutable-url' - -type DeleteKey = - 'video' - -class ModelCache { - - private static instance: ModelCache - - private readonly localCache: { [id in ModelCacheType]: Map } = { - 'local-account-name': new Map(), - 'local-actor-name': new Map(), - 'local-actor-url': new Map(), - 'load-video-immutable-id': new Map(), - 'load-video-immutable-url': new Map() - } - - private readonly deleteIds: { - [deleteKey in DeleteKey]: Map - } = { - video: new Map() - } - - private constructor () { - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } - - doCache (options: { - cacheType: ModelCacheType - key: string - fun: () => Promise - whitelist?: () => boolean - deleteKey?: DeleteKey - }) { - const { cacheType, key, fun, whitelist, deleteKey } = options - - if (whitelist && whitelist() !== true) return fun() - - const cache = this.localCache[cacheType] - - if (cache.has(key)) { - logger.debug('Model cache hit for %s -> %s.', cacheType, key) - return Promise.resolve(cache.get(key)) - } - - return fun().then(m => { - if (!m) return m - - if (!whitelist || whitelist()) cache.set(key, m) - - if (deleteKey) { - const map = this.deleteIds[deleteKey] - if (!map.has(m.id)) map.set(m.id, []) - - const a = map.get(m.id) - a.push({ cacheType, key }) - } - - return m - }) - } - - invalidateCache (deleteKey: DeleteKey, modelId: number) { - const map = this.deleteIds[deleteKey] - - if (!map.has(modelId)) return - - for (const toDelete of map.get(modelId)) { - logger.debug('Removing %s -> %d of model cache %s -> %s.', deleteKey, modelId, toDelete.cacheType, toDelete.key) - this.localCache[toDelete.cacheType].delete(toDelete.key) - } - - map.delete(modelId) - } -} - -export { - ModelCache -} diff --git a/server/models/shared/query.ts b/server/models/shared/query.ts deleted file mode 100644 index 934acc21f..000000000 --- a/server/models/shared/query.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { BindOrReplacements, Op, QueryTypes, Sequelize } from 'sequelize' -import validator from 'validator' -import { forceNumber } from '@shared/core-utils' - -function doesExist (sequelize: Sequelize, query: string, bind?: BindOrReplacements) { - const options = { - type: QueryTypes.SELECT as QueryTypes.SELECT, - bind, - raw: true - } - - return sequelize.query(query, options) - .then(results => results.length === 1) -} - -function createSimilarityAttribute (col: string, value: string) { - return Sequelize.fn( - 'similarity', - - searchTrigramNormalizeCol(col), - - searchTrigramNormalizeValue(value) - ) -} - -function buildWhereIdOrUUID (id: number | string) { - return validator.isInt('' + id) ? { id } : { uuid: id } -} - -function parseAggregateResult (result: any) { - if (!result) return 0 - - const total = forceNumber(result) - if (isNaN(total)) return 0 - - return total -} - -function parseRowCountResult (result: any) { - if (result.length !== 0) return result[0].total - - return 0 -} - -function createSafeIn (sequelize: Sequelize, toEscape: (string | number)[], additionalUnescaped: string[] = []) { - return toEscape.map(t => { - return t === null - ? null - : sequelize.escape('' + t) - }).concat(additionalUnescaped).join(', ') -} - -function searchAttribute (sourceField?: string, targetField?: string) { - if (!sourceField) return {} - - return { - [targetField]: { - // FIXME: ts error - [Op.iLike as any]: `%${sourceField}%` - } - } -} - -export { - doesExist, - createSimilarityAttribute, - buildWhereIdOrUUID, - parseAggregateResult, - parseRowCountResult, - createSafeIn, - searchAttribute -} - -// --------------------------------------------------------------------------- - -function searchTrigramNormalizeValue (value: string) { - return Sequelize.fn('lower', Sequelize.fn('immutable_unaccent', value)) -} - -function searchTrigramNormalizeCol (col: string) { - return Sequelize.fn('lower', Sequelize.fn('immutable_unaccent', Sequelize.col(col))) -} diff --git a/server/models/shared/sql.ts b/server/models/shared/sql.ts deleted file mode 100644 index 5aaeb49f0..000000000 --- a/server/models/shared/sql.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { literal, Model, ModelStatic } from 'sequelize' -import { forceNumber } from '@shared/core-utils' -import { AttributesOnly } from '@shared/typescript-utils' - -function buildLocalAccountIdsIn () { - return literal( - '(SELECT "account"."id" FROM "account" INNER JOIN "actor" ON "actor"."id" = "account"."actorId" AND "actor"."serverId" IS NULL)' - ) -} - -function buildLocalActorIdsIn () { - return literal( - '(SELECT "actor"."id" FROM "actor" WHERE "actor"."serverId" IS NULL)' - ) -} - -function buildBlockedAccountSQL (blockerIds: number[]) { - const blockerIdsString = blockerIds.join(', ') - - return 'SELECT "targetAccountId" AS "id" FROM "accountBlocklist" WHERE "accountId" IN (' + blockerIdsString + ')' + - ' UNION ' + - 'SELECT "account"."id" AS "id" FROM account INNER JOIN "actor" ON account."actorId" = actor.id ' + - 'INNER JOIN "serverBlocklist" ON "actor"."serverId" = "serverBlocklist"."targetServerId" ' + - 'WHERE "serverBlocklist"."accountId" IN (' + blockerIdsString + ')' -} - -function buildServerIdsFollowedBy (actorId: any) { - const actorIdNumber = forceNumber(actorId) - - return '(' + - 'SELECT "actor"."serverId" FROM "actorFollow" ' + - 'INNER JOIN "actor" ON actor.id = "actorFollow"."targetActorId" ' + - 'WHERE "actorFollow"."actorId" = ' + actorIdNumber + - ')' -} - -function buildSQLAttributes (options: { - model: ModelStatic - tableName: string - - excludeAttributes?: Exclude, symbol>[] - aliasPrefix?: string -}) { - const { model, tableName, aliasPrefix, excludeAttributes } = options - - const attributes = Object.keys(model.getAttributes()) as Exclude, symbol>[] - - return attributes - .filter(a => { - if (!excludeAttributes) return true - if (excludeAttributes.includes(a)) return false - - return true - }) - .map(a => { - return `"${tableName}"."${a}" AS "${aliasPrefix || ''}${a}"` - }) -} - -// --------------------------------------------------------------------------- - -export { - buildSQLAttributes, - buildBlockedAccountSQL, - buildServerIdsFollowedBy, - buildLocalAccountIdsIn, - buildLocalActorIdsIn -} diff --git a/server/models/user/sql/user-notitication-list-query-builder.ts b/server/models/user/sql/user-notitication-list-query-builder.ts deleted file mode 100644 index 7b29807a3..000000000 --- a/server/models/user/sql/user-notitication-list-query-builder.ts +++ /dev/null @@ -1,273 +0,0 @@ -import { Sequelize } from 'sequelize' -import { AbstractRunQuery, ModelBuilder } from '@server/models/shared' -import { UserNotificationModelForApi } from '@server/types/models' -import { ActorImageType } from '@shared/models' -import { getSort } from '../../shared' - -export interface ListNotificationsOptions { - userId: number - unread?: boolean - sort: string - offset: number - limit: number -} - -export class UserNotificationListQueryBuilder extends AbstractRunQuery { - private innerQuery: string - - constructor ( - protected readonly sequelize: Sequelize, - private readonly options: ListNotificationsOptions - ) { - super(sequelize) - } - - async listNotifications () { - this.buildQuery() - - const results = await this.runQuery({ nest: true }) - const modelBuilder = new ModelBuilder(this.sequelize) - - return modelBuilder.createModels(results, 'UserNotification') - } - - private buildInnerQuery () { - this.innerQuery = `SELECT * FROM "userNotification" AS "UserNotificationModel" ` + - `${this.getWhere()} ` + - `${this.getOrder()} ` + - `LIMIT :limit OFFSET :offset ` - - this.replacements.limit = this.options.limit - this.replacements.offset = this.options.offset - } - - private buildQuery () { - this.buildInnerQuery() - - this.query = ` - ${this.getSelect()} - FROM (${this.innerQuery}) "UserNotificationModel" - ${this.getJoins()} - ${this.getOrder()}` - } - - private getWhere () { - let base = '"UserNotificationModel"."userId" = :userId ' - this.replacements.userId = this.options.userId - - if (this.options.unread === true) { - base += 'AND "UserNotificationModel"."read" IS FALSE ' - } else if (this.options.unread === false) { - base += 'AND "UserNotificationModel"."read" IS TRUE ' - } - - return `WHERE ${base}` - } - - private getOrder () { - const orders = getSort(this.options.sort) - - return 'ORDER BY ' + orders.map(o => `"UserNotificationModel"."${o[0]}" ${o[1]}`).join(', ') - } - - private getSelect () { - return `SELECT - "UserNotificationModel"."id", - "UserNotificationModel"."type", - "UserNotificationModel"."read", - "UserNotificationModel"."createdAt", - "UserNotificationModel"."updatedAt", - "Video"."id" AS "Video.id", - "Video"."uuid" AS "Video.uuid", - "Video"."name" AS "Video.name", - "Video->VideoChannel"."id" AS "Video.VideoChannel.id", - "Video->VideoChannel"."name" AS "Video.VideoChannel.name", - "Video->VideoChannel->Actor"."id" AS "Video.VideoChannel.Actor.id", - "Video->VideoChannel->Actor"."preferredUsername" AS "Video.VideoChannel.Actor.preferredUsername", - "Video->VideoChannel->Actor->Avatars"."id" AS "Video.VideoChannel.Actor.Avatars.id", - "Video->VideoChannel->Actor->Avatars"."width" AS "Video.VideoChannel.Actor.Avatars.width", - "Video->VideoChannel->Actor->Avatars"."type" AS "Video.VideoChannel.Actor.Avatars.type", - "Video->VideoChannel->Actor->Avatars"."filename" AS "Video.VideoChannel.Actor.Avatars.filename", - "Video->VideoChannel->Actor->Server"."id" AS "Video.VideoChannel.Actor.Server.id", - "Video->VideoChannel->Actor->Server"."host" AS "Video.VideoChannel.Actor.Server.host", - "VideoComment"."id" AS "VideoComment.id", - "VideoComment"."originCommentId" AS "VideoComment.originCommentId", - "VideoComment->Account"."id" AS "VideoComment.Account.id", - "VideoComment->Account"."name" AS "VideoComment.Account.name", - "VideoComment->Account->Actor"."id" AS "VideoComment.Account.Actor.id", - "VideoComment->Account->Actor"."preferredUsername" AS "VideoComment.Account.Actor.preferredUsername", - "VideoComment->Account->Actor->Avatars"."id" AS "VideoComment.Account.Actor.Avatars.id", - "VideoComment->Account->Actor->Avatars"."width" AS "VideoComment.Account.Actor.Avatars.width", - "VideoComment->Account->Actor->Avatars"."type" AS "VideoComment.Account.Actor.Avatars.type", - "VideoComment->Account->Actor->Avatars"."filename" AS "VideoComment.Account.Actor.Avatars.filename", - "VideoComment->Account->Actor->Server"."id" AS "VideoComment.Account.Actor.Server.id", - "VideoComment->Account->Actor->Server"."host" AS "VideoComment.Account.Actor.Server.host", - "VideoComment->Video"."id" AS "VideoComment.Video.id", - "VideoComment->Video"."uuid" AS "VideoComment.Video.uuid", - "VideoComment->Video"."name" AS "VideoComment.Video.name", - "Abuse"."id" AS "Abuse.id", - "Abuse"."state" AS "Abuse.state", - "Abuse->VideoAbuse"."id" AS "Abuse.VideoAbuse.id", - "Abuse->VideoAbuse->Video"."id" AS "Abuse.VideoAbuse.Video.id", - "Abuse->VideoAbuse->Video"."uuid" AS "Abuse.VideoAbuse.Video.uuid", - "Abuse->VideoAbuse->Video"."name" AS "Abuse.VideoAbuse.Video.name", - "Abuse->VideoCommentAbuse"."id" AS "Abuse.VideoCommentAbuse.id", - "Abuse->VideoCommentAbuse->VideoComment"."id" AS "Abuse.VideoCommentAbuse.VideoComment.id", - "Abuse->VideoCommentAbuse->VideoComment"."originCommentId" AS "Abuse.VideoCommentAbuse.VideoComment.originCommentId", - "Abuse->VideoCommentAbuse->VideoComment->Video"."id" AS "Abuse.VideoCommentAbuse.VideoComment.Video.id", - "Abuse->VideoCommentAbuse->VideoComment->Video"."name" AS "Abuse.VideoCommentAbuse.VideoComment.Video.name", - "Abuse->VideoCommentAbuse->VideoComment->Video"."uuid" AS "Abuse.VideoCommentAbuse.VideoComment.Video.uuid", - "Abuse->FlaggedAccount"."id" AS "Abuse.FlaggedAccount.id", - "Abuse->FlaggedAccount"."name" AS "Abuse.FlaggedAccount.name", - "Abuse->FlaggedAccount"."description" AS "Abuse.FlaggedAccount.description", - "Abuse->FlaggedAccount"."actorId" AS "Abuse.FlaggedAccount.actorId", - "Abuse->FlaggedAccount"."userId" AS "Abuse.FlaggedAccount.userId", - "Abuse->FlaggedAccount"."applicationId" AS "Abuse.FlaggedAccount.applicationId", - "Abuse->FlaggedAccount"."createdAt" AS "Abuse.FlaggedAccount.createdAt", - "Abuse->FlaggedAccount"."updatedAt" AS "Abuse.FlaggedAccount.updatedAt", - "Abuse->FlaggedAccount->Actor"."id" AS "Abuse.FlaggedAccount.Actor.id", - "Abuse->FlaggedAccount->Actor"."preferredUsername" AS "Abuse.FlaggedAccount.Actor.preferredUsername", - "Abuse->FlaggedAccount->Actor->Avatars"."id" AS "Abuse.FlaggedAccount.Actor.Avatars.id", - "Abuse->FlaggedAccount->Actor->Avatars"."width" AS "Abuse.FlaggedAccount.Actor.Avatars.width", - "Abuse->FlaggedAccount->Actor->Avatars"."type" AS "Abuse.FlaggedAccount.Actor.Avatars.type", - "Abuse->FlaggedAccount->Actor->Avatars"."filename" AS "Abuse.FlaggedAccount.Actor.Avatars.filename", - "Abuse->FlaggedAccount->Actor->Server"."id" AS "Abuse.FlaggedAccount.Actor.Server.id", - "Abuse->FlaggedAccount->Actor->Server"."host" AS "Abuse.FlaggedAccount.Actor.Server.host", - "VideoBlacklist"."id" AS "VideoBlacklist.id", - "VideoBlacklist->Video"."id" AS "VideoBlacklist.Video.id", - "VideoBlacklist->Video"."uuid" AS "VideoBlacklist.Video.uuid", - "VideoBlacklist->Video"."name" AS "VideoBlacklist.Video.name", - "VideoImport"."id" AS "VideoImport.id", - "VideoImport"."magnetUri" AS "VideoImport.magnetUri", - "VideoImport"."targetUrl" AS "VideoImport.targetUrl", - "VideoImport"."torrentName" AS "VideoImport.torrentName", - "VideoImport->Video"."id" AS "VideoImport.Video.id", - "VideoImport->Video"."uuid" AS "VideoImport.Video.uuid", - "VideoImport->Video"."name" AS "VideoImport.Video.name", - "Plugin"."id" AS "Plugin.id", - "Plugin"."name" AS "Plugin.name", - "Plugin"."type" AS "Plugin.type", - "Plugin"."latestVersion" AS "Plugin.latestVersion", - "Application"."id" AS "Application.id", - "Application"."latestPeerTubeVersion" AS "Application.latestPeerTubeVersion", - "ActorFollow"."id" AS "ActorFollow.id", - "ActorFollow"."state" AS "ActorFollow.state", - "ActorFollow->ActorFollower"."id" AS "ActorFollow.ActorFollower.id", - "ActorFollow->ActorFollower"."preferredUsername" AS "ActorFollow.ActorFollower.preferredUsername", - "ActorFollow->ActorFollower->Account"."id" AS "ActorFollow.ActorFollower.Account.id", - "ActorFollow->ActorFollower->Account"."name" AS "ActorFollow.ActorFollower.Account.name", - "ActorFollow->ActorFollower->Avatars"."id" AS "ActorFollow.ActorFollower.Avatars.id", - "ActorFollow->ActorFollower->Avatars"."width" AS "ActorFollow.ActorFollower.Avatars.width", - "ActorFollow->ActorFollower->Avatars"."type" AS "ActorFollow.ActorFollower.Avatars.type", - "ActorFollow->ActorFollower->Avatars"."filename" AS "ActorFollow.ActorFollower.Avatars.filename", - "ActorFollow->ActorFollower->Server"."id" AS "ActorFollow.ActorFollower.Server.id", - "ActorFollow->ActorFollower->Server"."host" AS "ActorFollow.ActorFollower.Server.host", - "ActorFollow->ActorFollowing"."id" AS "ActorFollow.ActorFollowing.id", - "ActorFollow->ActorFollowing"."preferredUsername" AS "ActorFollow.ActorFollowing.preferredUsername", - "ActorFollow->ActorFollowing"."type" AS "ActorFollow.ActorFollowing.type", - "ActorFollow->ActorFollowing->VideoChannel"."id" AS "ActorFollow.ActorFollowing.VideoChannel.id", - "ActorFollow->ActorFollowing->VideoChannel"."name" AS "ActorFollow.ActorFollowing.VideoChannel.name", - "ActorFollow->ActorFollowing->Account"."id" AS "ActorFollow.ActorFollowing.Account.id", - "ActorFollow->ActorFollowing->Account"."name" AS "ActorFollow.ActorFollowing.Account.name", - "ActorFollow->ActorFollowing->Server"."id" AS "ActorFollow.ActorFollowing.Server.id", - "ActorFollow->ActorFollowing->Server"."host" AS "ActorFollow.ActorFollowing.Server.host", - "Account"."id" AS "Account.id", - "Account"."name" AS "Account.name", - "Account->Actor"."id" AS "Account.Actor.id", - "Account->Actor"."preferredUsername" AS "Account.Actor.preferredUsername", - "Account->Actor->Avatars"."id" AS "Account.Actor.Avatars.id", - "Account->Actor->Avatars"."width" AS "Account.Actor.Avatars.width", - "Account->Actor->Avatars"."type" AS "Account.Actor.Avatars.type", - "Account->Actor->Avatars"."filename" AS "Account.Actor.Avatars.filename", - "Account->Actor->Server"."id" AS "Account.Actor.Server.id", - "Account->Actor->Server"."host" AS "Account.Actor.Server.host", - "UserRegistration"."id" AS "UserRegistration.id", - "UserRegistration"."username" AS "UserRegistration.username"` - } - - private getJoins () { - return ` - LEFT JOIN ( - "video" AS "Video" - INNER JOIN "videoChannel" AS "Video->VideoChannel" ON "Video"."channelId" = "Video->VideoChannel"."id" - INNER JOIN "actor" AS "Video->VideoChannel->Actor" ON "Video->VideoChannel"."actorId" = "Video->VideoChannel->Actor"."id" - LEFT JOIN "actorImage" AS "Video->VideoChannel->Actor->Avatars" - ON "Video->VideoChannel->Actor"."id" = "Video->VideoChannel->Actor->Avatars"."actorId" - AND "Video->VideoChannel->Actor->Avatars"."type" = ${ActorImageType.AVATAR} - LEFT JOIN "server" AS "Video->VideoChannel->Actor->Server" - ON "Video->VideoChannel->Actor"."serverId" = "Video->VideoChannel->Actor->Server"."id" - ) ON "UserNotificationModel"."videoId" = "Video"."id" - - LEFT JOIN ( - "videoComment" AS "VideoComment" - INNER JOIN "account" AS "VideoComment->Account" ON "VideoComment"."accountId" = "VideoComment->Account"."id" - INNER JOIN "actor" AS "VideoComment->Account->Actor" ON "VideoComment->Account"."actorId" = "VideoComment->Account->Actor"."id" - LEFT JOIN "actorImage" AS "VideoComment->Account->Actor->Avatars" - ON "VideoComment->Account->Actor"."id" = "VideoComment->Account->Actor->Avatars"."actorId" - AND "VideoComment->Account->Actor->Avatars"."type" = ${ActorImageType.AVATAR} - LEFT JOIN "server" AS "VideoComment->Account->Actor->Server" - ON "VideoComment->Account->Actor"."serverId" = "VideoComment->Account->Actor->Server"."id" - INNER JOIN "video" AS "VideoComment->Video" ON "VideoComment"."videoId" = "VideoComment->Video"."id" - ) ON "UserNotificationModel"."commentId" = "VideoComment"."id" - - LEFT JOIN "abuse" AS "Abuse" ON "UserNotificationModel"."abuseId" = "Abuse"."id" - LEFT JOIN "videoAbuse" AS "Abuse->VideoAbuse" ON "Abuse"."id" = "Abuse->VideoAbuse"."abuseId" - LEFT JOIN "video" AS "Abuse->VideoAbuse->Video" ON "Abuse->VideoAbuse"."videoId" = "Abuse->VideoAbuse->Video"."id" - LEFT JOIN "commentAbuse" AS "Abuse->VideoCommentAbuse" ON "Abuse"."id" = "Abuse->VideoCommentAbuse"."abuseId" - LEFT JOIN "videoComment" AS "Abuse->VideoCommentAbuse->VideoComment" - ON "Abuse->VideoCommentAbuse"."videoCommentId" = "Abuse->VideoCommentAbuse->VideoComment"."id" - LEFT JOIN "video" AS "Abuse->VideoCommentAbuse->VideoComment->Video" - ON "Abuse->VideoCommentAbuse->VideoComment"."videoId" = "Abuse->VideoCommentAbuse->VideoComment->Video"."id" - LEFT JOIN ( - "account" AS "Abuse->FlaggedAccount" - INNER JOIN "actor" AS "Abuse->FlaggedAccount->Actor" ON "Abuse->FlaggedAccount"."actorId" = "Abuse->FlaggedAccount->Actor"."id" - LEFT JOIN "actorImage" AS "Abuse->FlaggedAccount->Actor->Avatars" - ON "Abuse->FlaggedAccount->Actor"."id" = "Abuse->FlaggedAccount->Actor->Avatars"."actorId" - AND "Abuse->FlaggedAccount->Actor->Avatars"."type" = ${ActorImageType.AVATAR} - LEFT JOIN "server" AS "Abuse->FlaggedAccount->Actor->Server" - ON "Abuse->FlaggedAccount->Actor"."serverId" = "Abuse->FlaggedAccount->Actor->Server"."id" - ) ON "Abuse"."flaggedAccountId" = "Abuse->FlaggedAccount"."id" - - LEFT JOIN ( - "videoBlacklist" AS "VideoBlacklist" - INNER JOIN "video" AS "VideoBlacklist->Video" ON "VideoBlacklist"."videoId" = "VideoBlacklist->Video"."id" - ) ON "UserNotificationModel"."videoBlacklistId" = "VideoBlacklist"."id" - - LEFT JOIN "videoImport" AS "VideoImport" ON "UserNotificationModel"."videoImportId" = "VideoImport"."id" - LEFT JOIN "video" AS "VideoImport->Video" ON "VideoImport"."videoId" = "VideoImport->Video"."id" - - LEFT JOIN "plugin" AS "Plugin" ON "UserNotificationModel"."pluginId" = "Plugin"."id" - - LEFT JOIN "application" AS "Application" ON "UserNotificationModel"."applicationId" = "Application"."id" - - LEFT JOIN ( - "actorFollow" AS "ActorFollow" - INNER JOIN "actor" AS "ActorFollow->ActorFollower" ON "ActorFollow"."actorId" = "ActorFollow->ActorFollower"."id" - INNER JOIN "account" AS "ActorFollow->ActorFollower->Account" - ON "ActorFollow->ActorFollower"."id" = "ActorFollow->ActorFollower->Account"."actorId" - LEFT JOIN "actorImage" AS "ActorFollow->ActorFollower->Avatars" - ON "ActorFollow->ActorFollower"."id" = "ActorFollow->ActorFollower->Avatars"."actorId" - AND "ActorFollow->ActorFollower->Avatars"."type" = ${ActorImageType.AVATAR} - LEFT JOIN "server" AS "ActorFollow->ActorFollower->Server" - ON "ActorFollow->ActorFollower"."serverId" = "ActorFollow->ActorFollower->Server"."id" - INNER JOIN "actor" AS "ActorFollow->ActorFollowing" ON "ActorFollow"."targetActorId" = "ActorFollow->ActorFollowing"."id" - LEFT JOIN "videoChannel" AS "ActorFollow->ActorFollowing->VideoChannel" - ON "ActorFollow->ActorFollowing"."id" = "ActorFollow->ActorFollowing->VideoChannel"."actorId" - LEFT JOIN "account" AS "ActorFollow->ActorFollowing->Account" - ON "ActorFollow->ActorFollowing"."id" = "ActorFollow->ActorFollowing->Account"."actorId" - LEFT JOIN "server" AS "ActorFollow->ActorFollowing->Server" - ON "ActorFollow->ActorFollowing"."serverId" = "ActorFollow->ActorFollowing->Server"."id" - ) ON "UserNotificationModel"."actorFollowId" = "ActorFollow"."id" - - LEFT JOIN ( - "account" AS "Account" - INNER JOIN "actor" AS "Account->Actor" ON "Account"."actorId" = "Account->Actor"."id" - LEFT JOIN "actorImage" AS "Account->Actor->Avatars" - ON "Account->Actor"."id" = "Account->Actor->Avatars"."actorId" - AND "Account->Actor->Avatars"."type" = ${ActorImageType.AVATAR} - LEFT JOIN "server" AS "Account->Actor->Server" ON "Account->Actor"."serverId" = "Account->Actor->Server"."id" - ) ON "UserNotificationModel"."accountId" = "Account"."id" - - LEFT JOIN "userRegistration" as "UserRegistration" ON "UserNotificationModel"."userRegistrationId" = "UserRegistration"."id"` - } -} diff --git a/server/models/user/user-notification-setting.ts b/server/models/user/user-notification-setting.ts deleted file mode 100644 index 394494c0c..000000000 --- a/server/models/user/user-notification-setting.ts +++ /dev/null @@ -1,232 +0,0 @@ -import { - AfterDestroy, - AfterUpdate, - AllowNull, - BelongsTo, - Column, - CreatedAt, - Default, - ForeignKey, - Is, - Model, - Table, - UpdatedAt -} from 'sequelize-typescript' -import { TokensCache } from '@server/lib/auth/tokens-cache' -import { MNotificationSettingFormattable } from '@server/types/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { UserNotificationSetting, UserNotificationSettingValue } from '../../../shared/models/users/user-notification-setting.model' -import { isUserNotificationSettingValid } from '../../helpers/custom-validators/user-notifications' -import { throwIfNotValid } from '../shared' -import { UserModel } from './user' - -@Table({ - tableName: 'userNotificationSetting', - indexes: [ - { - fields: [ 'userId' ], - unique: true - } - ] -}) -export class UserNotificationSettingModel extends Model>> { - - @AllowNull(false) - @Default(null) - @Is( - 'UserNotificationSettingNewVideoFromSubscription', - value => throwIfNotValid(value, isUserNotificationSettingValid, 'newVideoFromSubscription') - ) - @Column - newVideoFromSubscription: UserNotificationSettingValue - - @AllowNull(false) - @Default(null) - @Is( - 'UserNotificationSettingNewCommentOnMyVideo', - value => throwIfNotValid(value, isUserNotificationSettingValid, 'newCommentOnMyVideo') - ) - @Column - newCommentOnMyVideo: UserNotificationSettingValue - - @AllowNull(false) - @Default(null) - @Is( - 'UserNotificationSettingAbuseAsModerator', - value => throwIfNotValid(value, isUserNotificationSettingValid, 'abuseAsModerator') - ) - @Column - abuseAsModerator: UserNotificationSettingValue - - @AllowNull(false) - @Default(null) - @Is( - 'UserNotificationSettingVideoAutoBlacklistAsModerator', - value => throwIfNotValid(value, isUserNotificationSettingValid, 'videoAutoBlacklistAsModerator') - ) - @Column - videoAutoBlacklistAsModerator: UserNotificationSettingValue - - @AllowNull(false) - @Default(null) - @Is( - 'UserNotificationSettingBlacklistOnMyVideo', - value => throwIfNotValid(value, isUserNotificationSettingValid, 'blacklistOnMyVideo') - ) - @Column - blacklistOnMyVideo: UserNotificationSettingValue - - @AllowNull(false) - @Default(null) - @Is( - 'UserNotificationSettingMyVideoPublished', - value => throwIfNotValid(value, isUserNotificationSettingValid, 'myVideoPublished') - ) - @Column - myVideoPublished: UserNotificationSettingValue - - @AllowNull(false) - @Default(null) - @Is( - 'UserNotificationSettingMyVideoImportFinished', - value => throwIfNotValid(value, isUserNotificationSettingValid, 'myVideoImportFinished') - ) - @Column - myVideoImportFinished: UserNotificationSettingValue - - @AllowNull(false) - @Default(null) - @Is( - 'UserNotificationSettingNewUserRegistration', - value => throwIfNotValid(value, isUserNotificationSettingValid, 'newUserRegistration') - ) - @Column - newUserRegistration: UserNotificationSettingValue - - @AllowNull(false) - @Default(null) - @Is( - 'UserNotificationSettingNewInstanceFollower', - value => throwIfNotValid(value, isUserNotificationSettingValid, 'newInstanceFollower') - ) - @Column - newInstanceFollower: UserNotificationSettingValue - - @AllowNull(false) - @Default(null) - @Is( - 'UserNotificationSettingNewInstanceFollower', - value => throwIfNotValid(value, isUserNotificationSettingValid, 'autoInstanceFollowing') - ) - @Column - autoInstanceFollowing: UserNotificationSettingValue - - @AllowNull(false) - @Default(null) - @Is( - 'UserNotificationSettingNewFollow', - value => throwIfNotValid(value, isUserNotificationSettingValid, 'newFollow') - ) - @Column - newFollow: UserNotificationSettingValue - - @AllowNull(false) - @Default(null) - @Is( - 'UserNotificationSettingCommentMention', - value => throwIfNotValid(value, isUserNotificationSettingValid, 'commentMention') - ) - @Column - commentMention: UserNotificationSettingValue - - @AllowNull(false) - @Default(null) - @Is( - 'UserNotificationSettingAbuseStateChange', - value => throwIfNotValid(value, isUserNotificationSettingValid, 'abuseStateChange') - ) - @Column - abuseStateChange: UserNotificationSettingValue - - @AllowNull(false) - @Default(null) - @Is( - 'UserNotificationSettingAbuseNewMessage', - value => throwIfNotValid(value, isUserNotificationSettingValid, 'abuseNewMessage') - ) - @Column - abuseNewMessage: UserNotificationSettingValue - - @AllowNull(false) - @Default(null) - @Is( - 'UserNotificationSettingNewPeerTubeVersion', - value => throwIfNotValid(value, isUserNotificationSettingValid, 'newPeerTubeVersion') - ) - @Column - newPeerTubeVersion: UserNotificationSettingValue - - @AllowNull(false) - @Default(null) - @Is( - 'UserNotificationSettingNewPeerPluginVersion', - value => throwIfNotValid(value, isUserNotificationSettingValid, 'newPluginVersion') - ) - @Column - newPluginVersion: UserNotificationSettingValue - - @AllowNull(false) - @Default(null) - @Is( - 'UserNotificationSettingMyVideoStudioEditionFinished', - value => throwIfNotValid(value, isUserNotificationSettingValid, 'myVideoStudioEditionFinished') - ) - @Column - myVideoStudioEditionFinished: UserNotificationSettingValue - - @ForeignKey(() => UserModel) - @Column - userId: number - - @BelongsTo(() => UserModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade' - }) - User: UserModel - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @AfterUpdate - @AfterDestroy - static removeTokenCache (instance: UserNotificationSettingModel) { - return TokensCache.Instance.clearCacheByUserId(instance.userId) - } - - toFormattedJSON (this: MNotificationSettingFormattable): UserNotificationSetting { - return { - newCommentOnMyVideo: this.newCommentOnMyVideo, - newVideoFromSubscription: this.newVideoFromSubscription, - abuseAsModerator: this.abuseAsModerator, - videoAutoBlacklistAsModerator: this.videoAutoBlacklistAsModerator, - blacklistOnMyVideo: this.blacklistOnMyVideo, - myVideoPublished: this.myVideoPublished, - myVideoImportFinished: this.myVideoImportFinished, - newUserRegistration: this.newUserRegistration, - commentMention: this.commentMention, - newFollow: this.newFollow, - newInstanceFollower: this.newInstanceFollower, - autoInstanceFollowing: this.autoInstanceFollowing, - abuseNewMessage: this.abuseNewMessage, - abuseStateChange: this.abuseStateChange, - newPeerTubeVersion: this.newPeerTubeVersion, - myVideoStudioEditionFinished: this.myVideoStudioEditionFinished, - newPluginVersion: this.newPluginVersion - } - } -} diff --git a/server/models/user/user-notification.ts b/server/models/user/user-notification.ts deleted file mode 100644 index 667ee7f5f..000000000 --- a/server/models/user/user-notification.ts +++ /dev/null @@ -1,534 +0,0 @@ -import { ModelIndexesOptions, Op, WhereOptions } from 'sequelize' -import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { getBiggestActorImage } from '@server/lib/actor-image' -import { UserNotificationIncludes, UserNotificationModelForApi } from '@server/types/models/user' -import { forceNumber } from '@shared/core-utils' -import { uuidToShort } from '@shared/extra-utils' -import { UserNotification, UserNotificationType } from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { isBooleanValid } from '../../helpers/custom-validators/misc' -import { isUserNotificationTypeValid } from '../../helpers/custom-validators/user-notifications' -import { AbuseModel } from '../abuse/abuse' -import { AccountModel } from '../account/account' -import { ActorFollowModel } from '../actor/actor-follow' -import { ApplicationModel } from '../application/application' -import { PluginModel } from '../server/plugin' -import { throwIfNotValid } from '../shared' -import { VideoModel } from '../video/video' -import { VideoBlacklistModel } from '../video/video-blacklist' -import { VideoCommentModel } from '../video/video-comment' -import { VideoImportModel } from '../video/video-import' -import { UserNotificationListQueryBuilder } from './sql/user-notitication-list-query-builder' -import { UserModel } from './user' -import { UserRegistrationModel } from './user-registration' - -@Table({ - tableName: 'userNotification', - indexes: [ - { - fields: [ 'userId' ] - }, - { - fields: [ 'videoId' ], - where: { - videoId: { - [Op.ne]: null - } - } - }, - { - fields: [ 'commentId' ], - where: { - commentId: { - [Op.ne]: null - } - } - }, - { - fields: [ 'abuseId' ], - where: { - abuseId: { - [Op.ne]: null - } - } - }, - { - fields: [ 'videoBlacklistId' ], - where: { - videoBlacklistId: { - [Op.ne]: null - } - } - }, - { - fields: [ 'videoImportId' ], - where: { - videoImportId: { - [Op.ne]: null - } - } - }, - { - fields: [ 'accountId' ], - where: { - accountId: { - [Op.ne]: null - } - } - }, - { - fields: [ 'actorFollowId' ], - where: { - actorFollowId: { - [Op.ne]: null - } - } - }, - { - fields: [ 'pluginId' ], - where: { - pluginId: { - [Op.ne]: null - } - } - }, - { - fields: [ 'applicationId' ], - where: { - applicationId: { - [Op.ne]: null - } - } - }, - { - fields: [ 'userRegistrationId' ], - where: { - userRegistrationId: { - [Op.ne]: null - } - } - } - ] as (ModelIndexesOptions & { where?: WhereOptions })[] -}) -export class UserNotificationModel extends Model>> { - - @AllowNull(false) - @Default(null) - @Is('UserNotificationType', value => throwIfNotValid(value, isUserNotificationTypeValid, 'type')) - @Column - type: UserNotificationType - - @AllowNull(false) - @Default(false) - @Is('UserNotificationRead', value => throwIfNotValid(value, isBooleanValid, 'read')) - @Column - read: boolean - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => UserModel) - @Column - userId: number - - @BelongsTo(() => UserModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade' - }) - User: UserModel - - @ForeignKey(() => VideoModel) - @Column - videoId: number - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'cascade' - }) - Video: VideoModel - - @ForeignKey(() => VideoCommentModel) - @Column - commentId: number - - @BelongsTo(() => VideoCommentModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'cascade' - }) - VideoComment: VideoCommentModel - - @ForeignKey(() => AbuseModel) - @Column - abuseId: number - - @BelongsTo(() => AbuseModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'cascade' - }) - Abuse: AbuseModel - - @ForeignKey(() => VideoBlacklistModel) - @Column - videoBlacklistId: number - - @BelongsTo(() => VideoBlacklistModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'cascade' - }) - VideoBlacklist: VideoBlacklistModel - - @ForeignKey(() => VideoImportModel) - @Column - videoImportId: number - - @BelongsTo(() => VideoImportModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'cascade' - }) - VideoImport: VideoImportModel - - @ForeignKey(() => AccountModel) - @Column - accountId: number - - @BelongsTo(() => AccountModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'cascade' - }) - Account: AccountModel - - @ForeignKey(() => ActorFollowModel) - @Column - actorFollowId: number - - @BelongsTo(() => ActorFollowModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'cascade' - }) - ActorFollow: ActorFollowModel - - @ForeignKey(() => PluginModel) - @Column - pluginId: number - - @BelongsTo(() => PluginModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'cascade' - }) - Plugin: PluginModel - - @ForeignKey(() => ApplicationModel) - @Column - applicationId: number - - @BelongsTo(() => ApplicationModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'cascade' - }) - Application: ApplicationModel - - @ForeignKey(() => UserRegistrationModel) - @Column - userRegistrationId: number - - @BelongsTo(() => UserRegistrationModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'cascade' - }) - UserRegistration: UserRegistrationModel - - static listForApi (userId: number, start: number, count: number, sort: string, unread?: boolean) { - const where = { userId } - - const query = { - userId, - unread, - offset: start, - limit: count, - sort, - where - } - - if (unread !== undefined) query.where['read'] = !unread - - return Promise.all([ - UserNotificationModel.count({ where }) - .then(count => count || 0), - - count === 0 - ? [] as UserNotificationModelForApi[] - : new UserNotificationListQueryBuilder(this.sequelize, query).listNotifications() - ]).then(([ total, data ]) => ({ total, data })) - } - - static markAsRead (userId: number, notificationIds: number[]) { - const query = { - where: { - userId, - id: { - [Op.in]: notificationIds - } - } - } - - return UserNotificationModel.update({ read: true }, query) - } - - static markAllAsRead (userId: number) { - const query = { where: { userId } } - - return UserNotificationModel.update({ read: true }, query) - } - - static removeNotificationsOf (options: { id: number, type: 'account' | 'server', forUserId?: number }) { - const id = forceNumber(options.id) - - function buildAccountWhereQuery (base: string) { - const whereSuffix = options.forUserId - ? ` AND "userNotification"."userId" = ${options.forUserId}` - : '' - - if (options.type === 'account') { - return base + - ` WHERE "account"."id" = ${id} ${whereSuffix}` - } - - return base + - ` WHERE "actor"."serverId" = ${id} ${whereSuffix}` - } - - const queries = [ - buildAccountWhereQuery( - `SELECT "userNotification"."id" FROM "userNotification" ` + - `INNER JOIN "account" ON "userNotification"."accountId" = "account"."id" ` + - `INNER JOIN actor ON "actor"."id" = "account"."actorId" ` - ), - - // Remove notifications from muted accounts that followed ours - buildAccountWhereQuery( - `SELECT "userNotification"."id" FROM "userNotification" ` + - `INNER JOIN "actorFollow" ON "actorFollow".id = "userNotification"."actorFollowId" ` + - `INNER JOIN actor ON actor.id = "actorFollow"."actorId" ` + - `INNER JOIN account ON account."actorId" = actor.id ` - ), - - // Remove notifications from muted accounts that commented something - buildAccountWhereQuery( - `SELECT "userNotification"."id" FROM "userNotification" ` + - `INNER JOIN "actorFollow" ON "actorFollow".id = "userNotification"."actorFollowId" ` + - `INNER JOIN actor ON actor.id = "actorFollow"."actorId" ` + - `INNER JOIN account ON account."actorId" = actor.id ` - ), - - buildAccountWhereQuery( - `SELECT "userNotification"."id" FROM "userNotification" ` + - `INNER JOIN "videoComment" ON "videoComment".id = "userNotification"."commentId" ` + - `INNER JOIN account ON account.id = "videoComment"."accountId" ` + - `INNER JOIN actor ON "actor"."id" = "account"."actorId" ` - ) - ] - - const query = `DELETE FROM "userNotification" WHERE id IN (${queries.join(' UNION ')})` - - return UserNotificationModel.sequelize.query(query) - } - - toFormattedJSON (this: UserNotificationModelForApi): UserNotification { - const video = this.Video - ? { - ...this.formatVideo(this.Video), - - channel: this.formatActor(this.Video.VideoChannel) - } - : undefined - - const videoImport = this.VideoImport - ? { - id: this.VideoImport.id, - video: this.VideoImport.Video - ? this.formatVideo(this.VideoImport.Video) - : undefined, - torrentName: this.VideoImport.torrentName, - magnetUri: this.VideoImport.magnetUri, - targetUrl: this.VideoImport.targetUrl - } - : undefined - - const comment = this.VideoComment - ? { - id: this.VideoComment.id, - threadId: this.VideoComment.getThreadId(), - account: this.formatActor(this.VideoComment.Account), - video: this.formatVideo(this.VideoComment.Video) - } - : undefined - - const abuse = this.Abuse ? this.formatAbuse(this.Abuse) : undefined - - const videoBlacklist = this.VideoBlacklist - ? { - id: this.VideoBlacklist.id, - video: this.formatVideo(this.VideoBlacklist.Video) - } - : undefined - - const account = this.Account ? this.formatActor(this.Account) : undefined - - const actorFollowingType = { - Application: 'instance' as 'instance', - Group: 'channel' as 'channel', - Person: 'account' as 'account' - } - const actorFollow = this.ActorFollow - ? { - id: this.ActorFollow.id, - state: this.ActorFollow.state, - follower: { - id: this.ActorFollow.ActorFollower.Account.id, - displayName: this.ActorFollow.ActorFollower.Account.getDisplayName(), - name: this.ActorFollow.ActorFollower.preferredUsername, - host: this.ActorFollow.ActorFollower.getHost(), - - ...this.formatAvatars(this.ActorFollow.ActorFollower.Avatars) - }, - following: { - type: actorFollowingType[this.ActorFollow.ActorFollowing.type], - displayName: (this.ActorFollow.ActorFollowing.VideoChannel || this.ActorFollow.ActorFollowing.Account).getDisplayName(), - name: this.ActorFollow.ActorFollowing.preferredUsername, - host: this.ActorFollow.ActorFollowing.getHost() - } - } - : undefined - - const plugin = this.Plugin - ? { - name: this.Plugin.name, - type: this.Plugin.type, - latestVersion: this.Plugin.latestVersion - } - : undefined - - const peertube = this.Application - ? { latestVersion: this.Application.latestPeerTubeVersion } - : undefined - - const registration = this.UserRegistration - ? { id: this.UserRegistration.id, username: this.UserRegistration.username } - : undefined - - return { - id: this.id, - type: this.type, - read: this.read, - video, - videoImport, - comment, - abuse, - videoBlacklist, - account, - actorFollow, - plugin, - peertube, - registration, - createdAt: this.createdAt.toISOString(), - updatedAt: this.updatedAt.toISOString() - } - } - - formatVideo (video: UserNotificationIncludes.VideoInclude) { - return { - id: video.id, - uuid: video.uuid, - shortUUID: uuidToShort(video.uuid), - name: video.name - } - } - - formatAbuse (abuse: UserNotificationIncludes.AbuseInclude) { - const commentAbuse = abuse.VideoCommentAbuse?.VideoComment - ? { - threadId: abuse.VideoCommentAbuse.VideoComment.getThreadId(), - - video: abuse.VideoCommentAbuse.VideoComment.Video - ? { - id: abuse.VideoCommentAbuse.VideoComment.Video.id, - name: abuse.VideoCommentAbuse.VideoComment.Video.name, - shortUUID: uuidToShort(abuse.VideoCommentAbuse.VideoComment.Video.uuid), - uuid: abuse.VideoCommentAbuse.VideoComment.Video.uuid - } - : undefined - } - : undefined - - const videoAbuse = abuse.VideoAbuse?.Video - ? this.formatVideo(abuse.VideoAbuse.Video) - : undefined - - const accountAbuse = (!commentAbuse && !videoAbuse && abuse.FlaggedAccount) - ? this.formatActor(abuse.FlaggedAccount) - : undefined - - return { - id: abuse.id, - state: abuse.state, - video: videoAbuse, - comment: commentAbuse, - account: accountAbuse - } - } - - formatActor ( - accountOrChannel: UserNotificationIncludes.AccountIncludeActor | UserNotificationIncludes.VideoChannelIncludeActor - ) { - return { - id: accountOrChannel.id, - displayName: accountOrChannel.getDisplayName(), - name: accountOrChannel.Actor.preferredUsername, - host: accountOrChannel.Actor.getHost(), - - ...this.formatAvatars(accountOrChannel.Actor.Avatars) - } - } - - formatAvatars (avatars: UserNotificationIncludes.ActorImageInclude[]) { - if (!avatars || avatars.length === 0) return { avatar: undefined, avatars: [] } - - return { - avatar: this.formatAvatar(getBiggestActorImage(avatars)), - - avatars: avatars.map(a => this.formatAvatar(a)) - } - } - - formatAvatar (a: UserNotificationIncludes.ActorImageInclude) { - return { - path: a.getStaticPath(), - width: a.width - } - } -} diff --git a/server/models/user/user-registration.ts b/server/models/user/user-registration.ts deleted file mode 100644 index adda3cc7e..000000000 --- a/server/models/user/user-registration.ts +++ /dev/null @@ -1,259 +0,0 @@ -import { FindOptions, Op, WhereOptions } from 'sequelize' -import { - AllowNull, - BeforeCreate, - BelongsTo, - Column, - CreatedAt, - DataType, - ForeignKey, - Is, - IsEmail, - Model, - Table, - UpdatedAt -} from 'sequelize-typescript' -import { - isRegistrationModerationResponseValid, - isRegistrationReasonValid, - isRegistrationStateValid -} from '@server/helpers/custom-validators/user-registration' -import { isVideoChannelDisplayNameValid } from '@server/helpers/custom-validators/video-channels' -import { cryptPassword } from '@server/helpers/peertube-crypto' -import { USER_REGISTRATION_STATES } from '@server/initializers/constants' -import { MRegistration, MRegistrationFormattable } from '@server/types/models' -import { UserRegistration, UserRegistrationState } from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { isUserDisplayNameValid, isUserEmailVerifiedValid, isUserPasswordValid } from '../../helpers/custom-validators/users' -import { getSort, throwIfNotValid } from '../shared' -import { UserModel } from './user' - -@Table({ - tableName: 'userRegistration', - indexes: [ - { - fields: [ 'username' ], - unique: true - }, - { - fields: [ 'email' ], - unique: true - }, - { - fields: [ 'channelHandle' ], - unique: true - }, - { - fields: [ 'userId' ], - unique: true - } - ] -}) -export class UserRegistrationModel extends Model>> { - - @AllowNull(false) - @Is('RegistrationState', value => throwIfNotValid(value, isRegistrationStateValid, 'state')) - @Column - state: UserRegistrationState - - @AllowNull(false) - @Is('RegistrationReason', value => throwIfNotValid(value, isRegistrationReasonValid, 'registration reason')) - @Column(DataType.TEXT) - registrationReason: string - - @AllowNull(true) - @Is('RegistrationModerationResponse', value => throwIfNotValid(value, isRegistrationModerationResponseValid, 'moderation response', true)) - @Column(DataType.TEXT) - moderationResponse: string - - @AllowNull(true) - @Is('RegistrationPassword', value => throwIfNotValid(value, isUserPasswordValid, 'registration password', true)) - @Column - password: string - - @AllowNull(false) - @Column - username: string - - @AllowNull(false) - @IsEmail - @Column(DataType.STRING(400)) - email: string - - @AllowNull(true) - @Is('RegistrationEmailVerified', value => throwIfNotValid(value, isUserEmailVerifiedValid, 'email verified boolean', true)) - @Column - emailVerified: boolean - - @AllowNull(true) - @Is('RegistrationAccountDisplayName', value => throwIfNotValid(value, isUserDisplayNameValid, 'account display name', true)) - @Column - accountDisplayName: string - - @AllowNull(true) - @Is('ChannelHandle', value => throwIfNotValid(value, isVideoChannelDisplayNameValid, 'channel handle', true)) - @Column - channelHandle: string - - @AllowNull(true) - @Is('ChannelDisplayName', value => throwIfNotValid(value, isVideoChannelDisplayNameValid, 'channel display name', true)) - @Column - channelDisplayName: string - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => UserModel) - @Column - userId: number - - @BelongsTo(() => UserModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'SET NULL' - }) - User: UserModel - - @BeforeCreate - static async cryptPasswordIfNeeded (instance: UserRegistrationModel) { - instance.password = await cryptPassword(instance.password) - } - - static load (id: number): Promise { - return UserRegistrationModel.findByPk(id) - } - - static loadByEmail (email: string): Promise { - const query = { - where: { email } - } - - return UserRegistrationModel.findOne(query) - } - - static loadByEmailOrUsername (emailOrUsername: string): Promise { - const query = { - where: { - [Op.or]: [ - { email: emailOrUsername }, - { username: emailOrUsername } - ] - } - } - - return UserRegistrationModel.findOne(query) - } - - static loadByEmailOrHandle (options: { - email: string - username: string - channelHandle?: string - }): Promise { - const { email, username, channelHandle } = options - - let or: WhereOptions = [ - { email }, - { channelHandle: username }, - { username } - ] - - if (channelHandle) { - or = or.concat([ - { username: channelHandle }, - { channelHandle } - ]) - } - - const query = { - where: { - [Op.or]: or - } - } - - return UserRegistrationModel.findOne(query) - } - - // --------------------------------------------------------------------------- - - static listForApi (options: { - start: number - count: number - sort: string - search?: string - }) { - const { start, count, sort, search } = options - - const where: WhereOptions = {} - - if (search) { - Object.assign(where, { - [Op.or]: [ - { - email: { - [Op.iLike]: '%' + search + '%' - } - }, - { - username: { - [Op.iLike]: '%' + search + '%' - } - } - ] - }) - } - - const query: FindOptions = { - offset: start, - limit: count, - order: getSort(sort), - where, - include: [ - { - model: UserModel.unscoped(), - required: false - } - ] - } - - return Promise.all([ - UserRegistrationModel.count(query), - UserRegistrationModel.findAll(query) - ]).then(([ total, data ]) => ({ total, data })) - } - - // --------------------------------------------------------------------------- - - toFormattedJSON (this: MRegistrationFormattable): UserRegistration { - return { - id: this.id, - - state: { - id: this.state, - label: USER_REGISTRATION_STATES[this.state] - }, - - registrationReason: this.registrationReason, - moderationResponse: this.moderationResponse, - - username: this.username, - email: this.email, - emailVerified: this.emailVerified, - - accountDisplayName: this.accountDisplayName, - - channelHandle: this.channelHandle, - channelDisplayName: this.channelDisplayName, - - createdAt: this.createdAt, - updatedAt: this.updatedAt, - - user: this.User - ? { id: this.User.id } - : null - } - } -} diff --git a/server/models/user/user-video-history.ts b/server/models/user/user-video-history.ts deleted file mode 100644 index f4d0889a1..000000000 --- a/server/models/user/user-video-history.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { DestroyOptions, Op, Transaction } from 'sequelize' -import { AllowNull, BelongsTo, Column, CreatedAt, ForeignKey, IsInt, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { MUserAccountId, MUserId } from '@server/types/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { VideoModel } from '../video/video' -import { UserModel } from './user' - -@Table({ - tableName: 'userVideoHistory', - indexes: [ - { - fields: [ 'userId', 'videoId' ], - unique: true - }, - { - fields: [ 'userId' ] - }, - { - fields: [ 'videoId' ] - } - ] -}) -export class UserVideoHistoryModel extends Model>> { - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @AllowNull(false) - @IsInt - @Column - currentTime: number - - @ForeignKey(() => VideoModel) - @Column - videoId: number - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'CASCADE' - }) - Video: VideoModel - - @ForeignKey(() => UserModel) - @Column - userId: number - - @BelongsTo(() => UserModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'CASCADE' - }) - User: UserModel - - static listForApi (user: MUserAccountId, start: number, count: number, search?: string) { - return VideoModel.listForApi({ - start, - count, - search, - sort: '-"userVideoHistory"."updatedAt"', - nsfw: null, // All - displayOnlyForFollower: null, - user, - historyOfUser: user - }) - } - - static removeUserHistoryElement (user: MUserId, videoId: number) { - const query: DestroyOptions = { - where: { - userId: user.id, - videoId - } - } - - return UserVideoHistoryModel.destroy(query) - } - - static removeUserHistoryBefore (user: MUserId, beforeDate: string, t: Transaction) { - const query: DestroyOptions = { - where: { - userId: user.id - }, - transaction: t - } - - if (beforeDate) { - query.where['updatedAt'] = { - [Op.lt]: beforeDate - } - } - - return UserVideoHistoryModel.destroy(query) - } - - static removeOldHistory (beforeDate: string) { - const query: DestroyOptions = { - where: { - updatedAt: { - [Op.lt]: beforeDate - } - } - } - - return UserVideoHistoryModel.destroy(query) - } -} diff --git a/server/models/user/user.ts b/server/models/user/user.ts deleted file mode 100644 index ff6328d48..000000000 --- a/server/models/user/user.ts +++ /dev/null @@ -1,983 +0,0 @@ -import { col, FindOptions, fn, literal, Op, QueryTypes, where, WhereOptions } from 'sequelize' -import { - AfterDestroy, - AfterUpdate, - AllowNull, - BeforeCreate, - BeforeUpdate, - Column, - CreatedAt, - DataType, - Default, - DefaultScope, - HasMany, - HasOne, - Is, - IsEmail, - IsUUID, - Model, - Scopes, - Table, - UpdatedAt -} from 'sequelize-typescript' -import { TokensCache } from '@server/lib/auth/tokens-cache' -import { LiveQuotaStore } from '@server/lib/live' -import { - MMyUserFormattable, - MUser, - MUserDefault, - MUserFormattable, - MUserNotifSettingChannelDefault, - MUserWithNotificationSetting -} from '@server/types/models' -import { forceNumber } from '@shared/core-utils' -import { AttributesOnly } from '@shared/typescript-utils' -import { hasUserRight, USER_ROLE_LABELS } from '../../../shared/core-utils/users' -import { AbuseState, MyUser, UserRight, VideoPlaylistType } from '../../../shared/models' -import { User, UserRole } from '../../../shared/models/users' -import { UserAdminFlag } from '../../../shared/models/users/user-flag.model' -import { NSFWPolicyType } from '../../../shared/models/videos/nsfw-policy.type' -import { isThemeNameValid } from '../../helpers/custom-validators/plugins' -import { - isUserAdminFlagsValid, - isUserAutoPlayNextVideoPlaylistValid, - isUserAutoPlayNextVideoValid, - isUserAutoPlayVideoValid, - isUserBlockedReasonValid, - isUserBlockedValid, - isUserEmailVerifiedValid, - isUserNoModal, - isUserNSFWPolicyValid, - isUserP2PEnabledValid, - isUserPasswordValid, - isUserRoleValid, - isUserVideoLanguages, - isUserVideoQuotaDailyValid, - isUserVideoQuotaValid, - isUserVideosHistoryEnabledValid -} from '../../helpers/custom-validators/users' -import { comparePassword, cryptPassword } from '../../helpers/peertube-crypto' -import { DEFAULT_USER_THEME_NAME, NSFW_POLICY_TYPES } from '../../initializers/constants' -import { getThemeOrDefault } from '../../lib/plugins/theme-utils' -import { AccountModel } from '../account/account' -import { ActorModel } from '../actor/actor' -import { ActorFollowModel } from '../actor/actor-follow' -import { ActorImageModel } from '../actor/actor-image' -import { OAuthTokenModel } from '../oauth/oauth-token' -import { getAdminUsersSort, throwIfNotValid } from '../shared' -import { VideoModel } from '../video/video' -import { VideoChannelModel } from '../video/video-channel' -import { VideoImportModel } from '../video/video-import' -import { VideoLiveModel } from '../video/video-live' -import { VideoPlaylistModel } from '../video/video-playlist' -import { UserNotificationSettingModel } from './user-notification-setting' - -enum ScopeNames { - FOR_ME_API = 'FOR_ME_API', - WITH_VIDEOCHANNELS = 'WITH_VIDEOCHANNELS', - WITH_QUOTA = 'WITH_QUOTA', - WITH_STATS = 'WITH_STATS' -} - -@DefaultScope(() => ({ - include: [ - { - model: AccountModel, - required: true - }, - { - model: UserNotificationSettingModel, - required: true - } - ] -})) -@Scopes(() => ({ - [ScopeNames.FOR_ME_API]: { - include: [ - { - model: AccountModel, - include: [ - { - model: VideoChannelModel.unscoped(), - include: [ - { - model: ActorModel, - required: true, - include: [ - { - model: ActorImageModel, - as: 'Banners', - required: false - } - ] - } - ] - }, - { - attributes: [ 'id', 'name', 'type' ], - model: VideoPlaylistModel.unscoped(), - required: true, - where: { - type: { - [Op.ne]: VideoPlaylistType.REGULAR - } - } - } - ] - }, - { - model: UserNotificationSettingModel, - required: true - } - ] - }, - [ScopeNames.WITH_VIDEOCHANNELS]: { - include: [ - { - model: AccountModel, - include: [ - { - model: VideoChannelModel - }, - { - attributes: [ 'id', 'name', 'type' ], - model: VideoPlaylistModel.unscoped(), - required: true, - where: { - type: { - [Op.ne]: VideoPlaylistType.REGULAR - } - } - } - ] - } - ] - }, - [ScopeNames.WITH_QUOTA]: { - attributes: { - include: [ - [ - literal( - '(' + - UserModel.generateUserQuotaBaseSQL({ - withSelect: false, - whereUserId: '"UserModel"."id"', - daily: false - }) + - ')' - ), - 'videoQuotaUsed' - ], - [ - literal( - '(' + - UserModel.generateUserQuotaBaseSQL({ - withSelect: false, - whereUserId: '"UserModel"."id"', - daily: true - }) + - ')' - ), - 'videoQuotaUsedDaily' - ] - ] - } - }, - [ScopeNames.WITH_STATS]: { - attributes: { - include: [ - [ - literal( - '(' + - 'SELECT COUNT("video"."id") ' + - 'FROM "video" ' + - 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' + - 'INNER JOIN "account" ON "account"."id" = "videoChannel"."accountId" ' + - 'WHERE "account"."userId" = "UserModel"."id"' + - ')' - ), - 'videosCount' - ], - [ - literal( - '(' + - `SELECT concat_ws(':', "abuses", "acceptedAbuses") ` + - 'FROM (' + - 'SELECT COUNT("abuse"."id") AS "abuses", ' + - `COUNT("abuse"."id") FILTER (WHERE "abuse"."state" = ${AbuseState.ACCEPTED}) AS "acceptedAbuses" ` + - 'FROM "abuse" ' + - 'INNER JOIN "account" ON "account"."id" = "abuse"."flaggedAccountId" ' + - 'WHERE "account"."userId" = "UserModel"."id"' + - ') t' + - ')' - ), - 'abusesCount' - ], - [ - literal( - '(' + - 'SELECT COUNT("abuse"."id") ' + - 'FROM "abuse" ' + - 'INNER JOIN "account" ON "account"."id" = "abuse"."reporterAccountId" ' + - 'WHERE "account"."userId" = "UserModel"."id"' + - ')' - ), - 'abusesCreatedCount' - ], - [ - literal( - '(' + - 'SELECT COUNT("videoComment"."id") ' + - 'FROM "videoComment" ' + - 'INNER JOIN "account" ON "account"."id" = "videoComment"."accountId" ' + - 'WHERE "account"."userId" = "UserModel"."id"' + - ')' - ), - 'videoCommentsCount' - ] - ] - } - } -})) -@Table({ - tableName: 'user', - indexes: [ - { - fields: [ 'username' ], - unique: true - }, - { - fields: [ 'email' ], - unique: true - } - ] -}) -export class UserModel extends Model>> { - - @AllowNull(true) - @Is('UserPassword', value => throwIfNotValid(value, isUserPasswordValid, 'user password', true)) - @Column - password: string - - @AllowNull(false) - @Column - username: string - - @AllowNull(false) - @IsEmail - @Column(DataType.STRING(400)) - email: string - - @AllowNull(true) - @IsEmail - @Column(DataType.STRING(400)) - pendingEmail: string - - @AllowNull(true) - @Default(null) - @Is('UserEmailVerified', value => throwIfNotValid(value, isUserEmailVerifiedValid, 'email verified boolean', true)) - @Column - emailVerified: boolean - - @AllowNull(false) - @Is('UserNSFWPolicy', value => throwIfNotValid(value, isUserNSFWPolicyValid, 'NSFW policy')) - @Column(DataType.ENUM(...Object.values(NSFW_POLICY_TYPES))) - nsfwPolicy: NSFWPolicyType - - @AllowNull(false) - @Is('p2pEnabled', value => throwIfNotValid(value, isUserP2PEnabledValid, 'P2P enabled')) - @Column - p2pEnabled: boolean - - @AllowNull(false) - @Default(true) - @Is('UserVideosHistoryEnabled', value => throwIfNotValid(value, isUserVideosHistoryEnabledValid, 'Videos history enabled')) - @Column - videosHistoryEnabled: boolean - - @AllowNull(false) - @Default(true) - @Is('UserAutoPlayVideo', value => throwIfNotValid(value, isUserAutoPlayVideoValid, 'auto play video boolean')) - @Column - autoPlayVideo: boolean - - @AllowNull(false) - @Default(false) - @Is('UserAutoPlayNextVideo', value => throwIfNotValid(value, isUserAutoPlayNextVideoValid, 'auto play next video boolean')) - @Column - autoPlayNextVideo: boolean - - @AllowNull(false) - @Default(true) - @Is( - 'UserAutoPlayNextVideoPlaylist', - value => throwIfNotValid(value, isUserAutoPlayNextVideoPlaylistValid, 'auto play next video for playlists boolean') - ) - @Column - autoPlayNextVideoPlaylist: boolean - - @AllowNull(true) - @Default(null) - @Is('UserVideoLanguages', value => throwIfNotValid(value, isUserVideoLanguages, 'video languages')) - @Column(DataType.ARRAY(DataType.STRING)) - videoLanguages: string[] - - @AllowNull(false) - @Default(UserAdminFlag.NONE) - @Is('UserAdminFlags', value => throwIfNotValid(value, isUserAdminFlagsValid, 'user admin flags')) - @Column - adminFlags?: UserAdminFlag - - @AllowNull(false) - @Default(false) - @Is('UserBlocked', value => throwIfNotValid(value, isUserBlockedValid, 'blocked boolean')) - @Column - blocked: boolean - - @AllowNull(true) - @Default(null) - @Is('UserBlockedReason', value => throwIfNotValid(value, isUserBlockedReasonValid, 'blocked reason', true)) - @Column - blockedReason: string - - @AllowNull(false) - @Is('UserRole', value => throwIfNotValid(value, isUserRoleValid, 'role')) - @Column - role: number - - @AllowNull(false) - @Is('UserVideoQuota', value => throwIfNotValid(value, isUserVideoQuotaValid, 'video quota')) - @Column(DataType.BIGINT) - videoQuota: number - - @AllowNull(false) - @Is('UserVideoQuotaDaily', value => throwIfNotValid(value, isUserVideoQuotaDailyValid, 'video quota daily')) - @Column(DataType.BIGINT) - videoQuotaDaily: number - - @AllowNull(false) - @Default(DEFAULT_USER_THEME_NAME) - @Is('UserTheme', value => throwIfNotValid(value, isThemeNameValid, 'theme')) - @Column - theme: string - - @AllowNull(false) - @Default(false) - @Is( - 'UserNoInstanceConfigWarningModal', - value => throwIfNotValid(value, isUserNoModal, 'no instance config warning modal') - ) - @Column - noInstanceConfigWarningModal: boolean - - @AllowNull(false) - @Default(false) - @Is( - 'UserNoWelcomeModal', - value => throwIfNotValid(value, isUserNoModal, 'no welcome modal') - ) - @Column - noWelcomeModal: boolean - - @AllowNull(false) - @Default(false) - @Is( - 'UserNoAccountSetupWarningModal', - value => throwIfNotValid(value, isUserNoModal, 'no account setup warning modal') - ) - @Column - noAccountSetupWarningModal: boolean - - @AllowNull(true) - @Default(null) - @Column - pluginAuth: string - - @AllowNull(false) - @Default(DataType.UUIDV4) - @IsUUID(4) - @Column(DataType.UUID) - feedToken: string - - @AllowNull(true) - @Default(null) - @Column - lastLoginDate: Date - - @AllowNull(false) - @Default(false) - @Column - emailPublic: boolean - - @AllowNull(true) - @Default(null) - @Column - otpSecret: string - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @HasOne(() => AccountModel, { - foreignKey: 'userId', - onDelete: 'cascade', - hooks: true - }) - Account: AccountModel - - @HasOne(() => UserNotificationSettingModel, { - foreignKey: 'userId', - onDelete: 'cascade', - hooks: true - }) - NotificationSetting: UserNotificationSettingModel - - @HasMany(() => VideoImportModel, { - foreignKey: 'userId', - onDelete: 'cascade' - }) - VideoImports: VideoImportModel[] - - @HasMany(() => OAuthTokenModel, { - foreignKey: 'userId', - onDelete: 'cascade' - }) - OAuthTokens: OAuthTokenModel[] - - // Used if we already set an encrypted password in user model - skipPasswordEncryption = false - - @BeforeCreate - @BeforeUpdate - static async cryptPasswordIfNeeded (instance: UserModel) { - if (instance.skipPasswordEncryption) return - if (!instance.changed('password')) return - if (!instance.password) return - - instance.password = await cryptPassword(instance.password) - } - - @AfterUpdate - @AfterDestroy - static removeTokenCache (instance: UserModel) { - return TokensCache.Instance.clearCacheByUserId(instance.id) - } - - static countTotal () { - return UserModel.unscoped().count() - } - - static listForAdminApi (parameters: { - start: number - count: number - sort: string - search?: string - blocked?: boolean - }) { - const { start, count, sort, search, blocked } = parameters - const where: WhereOptions = {} - - if (search) { - Object.assign(where, { - [Op.or]: [ - { - email: { - [Op.iLike]: '%' + search + '%' - } - }, - { - username: { - [Op.iLike]: '%' + search + '%' - } - } - ] - }) - } - - if (blocked !== undefined) { - Object.assign(where, { blocked }) - } - - const query: FindOptions = { - offset: start, - limit: count, - order: getAdminUsersSort(sort), - where - } - - return Promise.all([ - UserModel.unscoped().count(query), - UserModel.scope([ 'defaultScope', ScopeNames.WITH_QUOTA ]).findAll(query) - ]).then(([ total, data ]) => ({ total, data })) - } - - static listWithRight (right: UserRight): Promise { - const roles = Object.keys(USER_ROLE_LABELS) - .map(k => parseInt(k, 10) as UserRole) - .filter(role => hasUserRight(role, right)) - - const query = { - where: { - role: { - [Op.in]: roles - } - } - } - - return UserModel.findAll(query) - } - - static listUserSubscribersOf (actorId: number): Promise { - const query = { - include: [ - { - model: UserNotificationSettingModel.unscoped(), - required: true - }, - { - attributes: [ 'userId' ], - model: AccountModel.unscoped(), - required: true, - include: [ - { - attributes: [], - model: ActorModel.unscoped(), - required: true, - where: { - serverId: null - }, - include: [ - { - attributes: [], - as: 'ActorFollowings', - model: ActorFollowModel.unscoped(), - required: true, - where: { - state: 'accepted', - targetActorId: actorId - } - } - ] - } - ] - } - ] - } - - return UserModel.unscoped().findAll(query) - } - - static listByUsernames (usernames: string[]): Promise { - const query = { - where: { - username: usernames - } - } - - return UserModel.findAll(query) - } - - static loadById (id: number): Promise { - return UserModel.unscoped().findByPk(id) - } - - static loadByIdFull (id: number): Promise { - return UserModel.findByPk(id) - } - - static loadByIdWithChannels (id: number, withStats = false): Promise { - const scopes = [ - ScopeNames.WITH_VIDEOCHANNELS - ] - - if (withStats) { - scopes.push(ScopeNames.WITH_QUOTA) - scopes.push(ScopeNames.WITH_STATS) - } - - return UserModel.scope(scopes).findByPk(id) - } - - static loadByUsername (username: string): Promise { - const query = { - where: { - username - } - } - - return UserModel.findOne(query) - } - - static loadForMeAPI (id: number): Promise { - const query = { - where: { - id - } - } - - return UserModel.scope(ScopeNames.FOR_ME_API).findOne(query) - } - - static loadByEmail (email: string): Promise { - const query = { - where: { - email - } - } - - return UserModel.findOne(query) - } - - static loadByUsernameOrEmail (username: string, email?: string): Promise { - if (!email) email = username - - const query = { - where: { - [Op.or]: [ - where(fn('lower', col('username')), fn('lower', username) as any), - - { email } - ] - } - } - - return UserModel.findOne(query) - } - - static loadByVideoId (videoId: number): Promise { - const query = { - include: [ - { - required: true, - attributes: [ 'id' ], - model: AccountModel.unscoped(), - include: [ - { - required: true, - attributes: [ 'id' ], - model: VideoChannelModel.unscoped(), - include: [ - { - required: true, - attributes: [ 'id' ], - model: VideoModel.unscoped(), - where: { - id: videoId - } - } - ] - } - ] - } - ] - } - - return UserModel.findOne(query) - } - - static loadByVideoImportId (videoImportId: number): Promise { - const query = { - include: [ - { - required: true, - attributes: [ 'id' ], - model: VideoImportModel.unscoped(), - where: { - id: videoImportId - } - } - ] - } - - return UserModel.findOne(query) - } - - static loadByChannelActorId (videoChannelActorId: number): Promise { - const query = { - include: [ - { - required: true, - attributes: [ 'id' ], - model: AccountModel.unscoped(), - include: [ - { - required: true, - attributes: [ 'id' ], - model: VideoChannelModel.unscoped(), - where: { - actorId: videoChannelActorId - } - } - ] - } - ] - } - - return UserModel.findOne(query) - } - - static loadByAccountActorId (accountActorId: number): Promise { - const query = { - include: [ - { - required: true, - attributes: [ 'id' ], - model: AccountModel.unscoped(), - where: { - actorId: accountActorId - } - } - ] - } - - return UserModel.findOne(query) - } - - static loadByLiveId (liveId: number): Promise { - const query = { - include: [ - { - attributes: [ 'id' ], - model: AccountModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'id' ], - model: VideoChannelModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'id' ], - model: VideoModel.unscoped(), - required: true, - include: [ - { - attributes: [], - model: VideoLiveModel.unscoped(), - required: true, - where: { - id: liveId - } - } - ] - } - ] - } - ] - } - ] - } - - return UserModel.unscoped().findOne(query) - } - - static generateUserQuotaBaseSQL (options: { - whereUserId: '$userId' | '"UserModel"."id"' - withSelect: boolean - daily: boolean - }) { - const andWhere = options.daily === true - ? 'AND "video"."createdAt" > now() - interval \'24 hours\'' - : '' - - const videoChannelJoin = 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' + - 'INNER JOIN "account" ON "videoChannel"."accountId" = "account"."id" ' + - `WHERE "account"."userId" = ${options.whereUserId} ${andWhere}` - - const webVideoFiles = 'SELECT "videoFile"."size" AS "size", "video"."id" AS "videoId" FROM "videoFile" ' + - 'INNER JOIN "video" ON "videoFile"."videoId" = "video"."id" AND "video"."isLive" IS FALSE ' + - videoChannelJoin - - const hlsFiles = 'SELECT "videoFile"."size" AS "size", "video"."id" AS "videoId" FROM "videoFile" ' + - 'INNER JOIN "videoStreamingPlaylist" ON "videoFile"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id ' + - 'INNER JOIN "video" ON "videoStreamingPlaylist"."videoId" = "video"."id" AND "video"."isLive" IS FALSE ' + - videoChannelJoin - - return 'SELECT COALESCE(SUM("size"), 0) AS "total" ' + - 'FROM (' + - `SELECT MAX("t1"."size") AS "size" FROM (${webVideoFiles} UNION ${hlsFiles}) t1 ` + - 'GROUP BY "t1"."videoId"' + - ') t2' - } - - static getTotalRawQuery (query: string, userId: number) { - const options = { - bind: { userId }, - type: QueryTypes.SELECT as QueryTypes.SELECT - } - - return UserModel.sequelize.query<{ total: string }>(query, options) - .then(([ { total } ]) => { - if (total === null) return 0 - - return parseInt(total, 10) - }) - } - - static async getStats () { - function getActiveUsers (days: number) { - const query = { - where: { - [Op.and]: [ - literal(`"lastLoginDate" > NOW() - INTERVAL '${days}d'`) - ] - } - } - - return UserModel.unscoped().count(query) - } - - const totalUsers = await UserModel.unscoped().count() - const totalDailyActiveUsers = await getActiveUsers(1) - const totalWeeklyActiveUsers = await getActiveUsers(7) - const totalMonthlyActiveUsers = await getActiveUsers(30) - const totalHalfYearActiveUsers = await getActiveUsers(180) - - return { - totalUsers, - totalDailyActiveUsers, - totalWeeklyActiveUsers, - totalMonthlyActiveUsers, - totalHalfYearActiveUsers - } - } - - static autoComplete (search: string) { - const query = { - where: { - username: { - [Op.like]: `%${search}%` - } - }, - limit: 10 - } - - return UserModel.findAll(query) - .then(u => u.map(u => u.username)) - } - - hasRight (right: UserRight) { - return hasUserRight(this.role, right) - } - - hasAdminFlag (flag: UserAdminFlag) { - return this.adminFlags & flag - } - - isPasswordMatch (password: string) { - return comparePassword(password, this.password) - } - - toFormattedJSON (this: MUserFormattable, parameters: { withAdminFlags?: boolean } = {}): User { - const videoQuotaUsed = this.get('videoQuotaUsed') - const videoQuotaUsedDaily = this.get('videoQuotaUsedDaily') - const videosCount = this.get('videosCount') - const [ abusesCount, abusesAcceptedCount ] = (this.get('abusesCount') as string || ':').split(':') - const abusesCreatedCount = this.get('abusesCreatedCount') - const videoCommentsCount = this.get('videoCommentsCount') - - const json: User = { - id: this.id, - username: this.username, - email: this.email, - theme: getThemeOrDefault(this.theme, DEFAULT_USER_THEME_NAME), - - pendingEmail: this.pendingEmail, - emailPublic: this.emailPublic, - emailVerified: this.emailVerified, - - nsfwPolicy: this.nsfwPolicy, - - p2pEnabled: this.p2pEnabled, - - videosHistoryEnabled: this.videosHistoryEnabled, - autoPlayVideo: this.autoPlayVideo, - autoPlayNextVideo: this.autoPlayNextVideo, - autoPlayNextVideoPlaylist: this.autoPlayNextVideoPlaylist, - videoLanguages: this.videoLanguages, - - role: { - id: this.role, - label: USER_ROLE_LABELS[this.role] - }, - - videoQuota: this.videoQuota, - videoQuotaDaily: this.videoQuotaDaily, - - videoQuotaUsed: videoQuotaUsed !== undefined - ? forceNumber(videoQuotaUsed) + LiveQuotaStore.Instance.getLiveQuotaOf(this.id) - : undefined, - - videoQuotaUsedDaily: videoQuotaUsedDaily !== undefined - ? forceNumber(videoQuotaUsedDaily) + LiveQuotaStore.Instance.getLiveQuotaOf(this.id) - : undefined, - - videosCount: videosCount !== undefined - ? forceNumber(videosCount) - : undefined, - abusesCount: abusesCount - ? forceNumber(abusesCount) - : undefined, - abusesAcceptedCount: abusesAcceptedCount - ? forceNumber(abusesAcceptedCount) - : undefined, - abusesCreatedCount: abusesCreatedCount !== undefined - ? forceNumber(abusesCreatedCount) - : undefined, - videoCommentsCount: videoCommentsCount !== undefined - ? forceNumber(videoCommentsCount) - : undefined, - - noInstanceConfigWarningModal: this.noInstanceConfigWarningModal, - noWelcomeModal: this.noWelcomeModal, - noAccountSetupWarningModal: this.noAccountSetupWarningModal, - - blocked: this.blocked, - blockedReason: this.blockedReason, - - account: this.Account.toFormattedJSON(), - - notificationSettings: this.NotificationSetting - ? this.NotificationSetting.toFormattedJSON() - : undefined, - - videoChannels: [], - - createdAt: this.createdAt, - - pluginAuth: this.pluginAuth, - - lastLoginDate: this.lastLoginDate, - - twoFactorEnabled: !!this.otpSecret - } - - if (parameters.withAdminFlags) { - Object.assign(json, { adminFlags: this.adminFlags }) - } - - if (Array.isArray(this.Account.VideoChannels) === true) { - json.videoChannels = this.Account.VideoChannels - .map(c => c.toFormattedJSON()) - .sort((v1, v2) => { - if (v1.createdAt < v2.createdAt) return -1 - if (v1.createdAt === v2.createdAt) return 0 - - return 1 - }) - } - - return json - } - - toMeFormattedJSON (this: MMyUserFormattable): MyUser { - const formatted = this.toFormattedJSON({ withAdminFlags: true }) - - const specialPlaylists = this.Account.VideoPlaylists - .map(p => ({ id: p.id, name: p.name, type: p.type })) - - return Object.assign(formatted, { specialPlaylists }) - } -} diff --git a/server/models/video/formatter/index.ts b/server/models/video/formatter/index.ts deleted file mode 100644 index 77b406559..000000000 --- a/server/models/video/formatter/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './video-activity-pub-format' -export * from './video-api-format' diff --git a/server/models/video/formatter/shared/index.ts b/server/models/video/formatter/shared/index.ts deleted file mode 100644 index d558fa7d6..000000000 --- a/server/models/video/formatter/shared/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './video-format-utils' diff --git a/server/models/video/formatter/shared/video-format-utils.ts b/server/models/video/formatter/shared/video-format-utils.ts deleted file mode 100644 index df3bbdf1c..000000000 --- a/server/models/video/formatter/shared/video-format-utils.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { MVideoFile } from '@server/types/models' - -export function sortByResolutionDesc (fileA: MVideoFile, fileB: MVideoFile) { - if (fileA.resolution < fileB.resolution) return 1 - if (fileA.resolution === fileB.resolution) return 0 - return -1 -} diff --git a/server/models/video/formatter/video-activity-pub-format.ts b/server/models/video/formatter/video-activity-pub-format.ts deleted file mode 100644 index 694c66c33..000000000 --- a/server/models/video/formatter/video-activity-pub-format.ts +++ /dev/null @@ -1,296 +0,0 @@ -import { isArray } from '@server/helpers/custom-validators/misc' -import { generateMagnetUri } from '@server/helpers/webtorrent' -import { getActivityStreamDuration } from '@server/lib/activitypub/activity' -import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls' -import { - ActivityIconObject, - ActivityPlaylistUrlObject, - ActivityPubStoryboard, - ActivityTagObject, - ActivityTrackerUrlObject, - ActivityUrlObject, - VideoObject -} from '@shared/models' -import { MIMETYPES, WEBSERVER } from '../../../initializers/constants' -import { - getLocalVideoCommentsActivityPubUrl, - getLocalVideoDislikesActivityPubUrl, - getLocalVideoLikesActivityPubUrl, - getLocalVideoSharesActivityPubUrl -} from '../../../lib/activitypub/url' -import { MStreamingPlaylistFiles, MUserId, MVideo, MVideoAP, MVideoFile } from '../../../types/models' -import { VideoCaptionModel } from '../video-caption' -import { sortByResolutionDesc } from './shared' -import { getCategoryLabel, getLanguageLabel, getLicenceLabel } from './video-api-format' - -export function videoModelToActivityPubObject (video: MVideoAP): VideoObject { - const language = video.language - ? { identifier: video.language, name: getLanguageLabel(video.language) } - : undefined - - const category = video.category - ? { identifier: video.category + '', name: getCategoryLabel(video.category) } - : undefined - - const licence = video.licence - ? { identifier: video.licence + '', name: getLicenceLabel(video.licence) } - : undefined - - const url: ActivityUrlObject[] = [ - // HTML url should be the first element in the array so Mastodon correctly displays the embed - { - type: 'Link', - mediaType: 'text/html', - href: WEBSERVER.URL + '/videos/watch/' + video.uuid - } as ActivityUrlObject, - - ...buildVideoFileUrls({ video, files: video.VideoFiles }), - - ...buildStreamingPlaylistUrls(video), - - ...buildTrackerUrls(video) - ] - - return { - type: 'Video' as 'Video', - id: video.url, - name: video.name, - duration: getActivityStreamDuration(video.duration), - uuid: video.uuid, - category, - licence, - language, - views: video.views, - sensitive: video.nsfw, - waitTranscoding: video.waitTranscoding, - - state: video.state, - commentsEnabled: video.commentsEnabled, - downloadEnabled: video.downloadEnabled, - published: video.publishedAt.toISOString(), - - originallyPublishedAt: video.originallyPublishedAt - ? video.originallyPublishedAt.toISOString() - : null, - - updated: video.updatedAt.toISOString(), - - uploadDate: video.inputFileUpdatedAt?.toISOString(), - - tag: buildTags(video), - - mediaType: 'text/markdown', - content: video.description, - support: video.support, - - subtitleLanguage: buildSubtitleLanguage(video), - - icon: buildIcon(video), - - preview: buildPreviewAPAttribute(video), - - url, - - likes: getLocalVideoLikesActivityPubUrl(video), - dislikes: getLocalVideoDislikesActivityPubUrl(video), - shares: getLocalVideoSharesActivityPubUrl(video), - comments: getLocalVideoCommentsActivityPubUrl(video), - - attributedTo: [ - { - type: 'Person', - id: video.VideoChannel.Account.Actor.url - }, - { - type: 'Group', - id: video.VideoChannel.Actor.url - } - ], - - ...buildLiveAPAttributes(video) - } -} - -// --------------------------------------------------------------------------- -// Private -// --------------------------------------------------------------------------- - -function buildLiveAPAttributes (video: MVideoAP) { - if (!video.isLive) { - return { - isLiveBroadcast: false, - liveSaveReplay: null, - permanentLive: null, - latencyMode: null - } - } - - return { - isLiveBroadcast: true, - liveSaveReplay: video.VideoLive.saveReplay, - permanentLive: video.VideoLive.permanentLive, - latencyMode: video.VideoLive.latencyMode - } -} - -function buildPreviewAPAttribute (video: MVideoAP): ActivityPubStoryboard[] { - if (!video.Storyboard) return undefined - - const storyboard = video.Storyboard - - return [ - { - type: 'Image', - rel: [ 'storyboard' ], - url: [ - { - mediaType: 'image/jpeg', - - href: storyboard.getOriginFileUrl(video), - - width: storyboard.totalWidth, - height: storyboard.totalHeight, - - tileWidth: storyboard.spriteWidth, - tileHeight: storyboard.spriteHeight, - tileDuration: getActivityStreamDuration(storyboard.spriteDuration) - } - ] - } - ] -} - -function buildVideoFileUrls (options: { - video: MVideo - files: MVideoFile[] - user?: MUserId -}): ActivityUrlObject[] { - const { video, files } = options - - if (!isArray(files)) return [] - - const urls: ActivityUrlObject[] = [] - - const trackerUrls = video.getTrackerUrls() - const sortedFiles = files - .filter(f => !f.isLive()) - .sort(sortByResolutionDesc) - - for (const file of sortedFiles) { - urls.push({ - type: 'Link', - mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[file.extname] as any, - href: file.getFileUrl(video), - height: file.resolution, - size: file.size, - fps: file.fps - }) - - urls.push({ - type: 'Link', - rel: [ 'metadata', MIMETYPES.VIDEO.EXT_MIMETYPE[file.extname] ], - mediaType: 'application/json' as 'application/json', - href: getLocalVideoFileMetadataUrl(video, file), - height: file.resolution, - fps: file.fps - }) - - if (file.hasTorrent()) { - urls.push({ - type: 'Link', - mediaType: 'application/x-bittorrent' as 'application/x-bittorrent', - href: file.getTorrentUrl(), - height: file.resolution - }) - - urls.push({ - type: 'Link', - mediaType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet', - href: generateMagnetUri(video, file, trackerUrls), - height: file.resolution - }) - } - } - - return urls -} - -// --------------------------------------------------------------------------- - -function buildStreamingPlaylistUrls (video: MVideoAP): ActivityPlaylistUrlObject[] { - if (!isArray(video.VideoStreamingPlaylists)) return [] - - return video.VideoStreamingPlaylists - .map(playlist => ({ - type: 'Link', - mediaType: 'application/x-mpegURL' as 'application/x-mpegURL', - href: playlist.getMasterPlaylistUrl(video), - tag: buildStreamingPlaylistTags(video, playlist) - })) -} - -function buildStreamingPlaylistTags (video: MVideoAP, playlist: MStreamingPlaylistFiles) { - return [ - ...playlist.p2pMediaLoaderInfohashes.map(i => ({ type: 'Infohash' as 'Infohash', name: i })), - - { - type: 'Link', - name: 'sha256', - mediaType: 'application/json' as 'application/json', - href: playlist.getSha256SegmentsUrl(video) - }, - - ...buildVideoFileUrls({ video, files: playlist.VideoFiles }) - ] as ActivityTagObject[] -} - -// --------------------------------------------------------------------------- - -function buildTrackerUrls (video: MVideoAP): ActivityTrackerUrlObject[] { - return video.getTrackerUrls() - .map(trackerUrl => { - const rel2 = trackerUrl.startsWith('http') - ? 'http' - : 'websocket' - - return { - type: 'Link', - name: `tracker-${rel2}`, - rel: [ 'tracker', rel2 ], - href: trackerUrl - } - }) -} - -// --------------------------------------------------------------------------- - -function buildTags (video: MVideoAP) { - if (!isArray(video.Tags)) return [] - - return video.Tags.map(t => ({ - type: 'Hashtag' as 'Hashtag', - name: t.name - })) -} - -function buildIcon (video: MVideoAP): ActivityIconObject[] { - return [ video.getMiniature(), video.getPreview() ] - .map(i => ({ - type: 'Image', - url: i.getOriginFileUrl(video), - mediaType: 'image/jpeg', - width: i.width, - height: i.height - })) -} - -function buildSubtitleLanguage (video: MVideoAP) { - if (!isArray(video.VideoCaptions)) return [] - - return video.VideoCaptions - .map(caption => ({ - identifier: caption.language, - name: VideoCaptionModel.getLanguageLabel(caption.language), - url: caption.getFileUrl(video) - })) -} diff --git a/server/models/video/formatter/video-api-format.ts b/server/models/video/formatter/video-api-format.ts deleted file mode 100644 index 7a58f5d3c..000000000 --- a/server/models/video/formatter/video-api-format.ts +++ /dev/null @@ -1,305 +0,0 @@ -import { generateMagnetUri } from '@server/helpers/webtorrent' -import { tracer } from '@server/lib/opentelemetry/tracing' -import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls' -import { VideoViewsManager } from '@server/lib/views/video-views-manager' -import { uuidToShort } from '@shared/extra-utils' -import { - Video, - VideoAdditionalAttributes, - VideoDetails, - VideoFile, - VideoInclude, - VideosCommonQueryAfterSanitize, - VideoStreamingPlaylist -} from '@shared/models' -import { isArray } from '../../../helpers/custom-validators/misc' -import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES, VIDEO_STATES } from '../../../initializers/constants' -import { MServer, MStreamingPlaylistRedundanciesOpt, MVideoFormattable, MVideoFormattableDetails } from '../../../types/models' -import { MVideoFileRedundanciesOpt } from '../../../types/models/video/video-file' -import { sortByResolutionDesc } from './shared' - -export type VideoFormattingJSONOptions = { - completeDescription?: boolean - - additionalAttributes?: { - state?: boolean - waitTranscoding?: boolean - scheduledUpdate?: boolean - blacklistInfo?: boolean - files?: boolean - blockedOwner?: boolean - } -} - -export function guessAdditionalAttributesFromQuery (query: VideosCommonQueryAfterSanitize): VideoFormattingJSONOptions { - if (!query?.include) return {} - - return { - additionalAttributes: { - state: !!(query.include & VideoInclude.NOT_PUBLISHED_STATE), - waitTranscoding: !!(query.include & VideoInclude.NOT_PUBLISHED_STATE), - scheduledUpdate: !!(query.include & VideoInclude.NOT_PUBLISHED_STATE), - blacklistInfo: !!(query.include & VideoInclude.BLACKLISTED), - files: !!(query.include & VideoInclude.FILES), - blockedOwner: !!(query.include & VideoInclude.BLOCKED_OWNER) - } - } -} - -// --------------------------------------------------------------------------- - -export function videoModelToFormattedJSON (video: MVideoFormattable, options: VideoFormattingJSONOptions = {}): Video { - const span = tracer.startSpan('peertube.VideoModel.toFormattedJSON') - - const userHistory = isArray(video.UserVideoHistories) - ? video.UserVideoHistories[0] - : undefined - - const videoObject: Video = { - id: video.id, - uuid: video.uuid, - shortUUID: uuidToShort(video.uuid), - - url: video.url, - - name: video.name, - category: { - id: video.category, - label: getCategoryLabel(video.category) - }, - licence: { - id: video.licence, - label: getLicenceLabel(video.licence) - }, - language: { - id: video.language, - label: getLanguageLabel(video.language) - }, - privacy: { - id: video.privacy, - label: getPrivacyLabel(video.privacy) - }, - nsfw: video.nsfw, - - truncatedDescription: video.getTruncatedDescription(), - description: options && options.completeDescription === true - ? video.description - : video.getTruncatedDescription(), - - isLocal: video.isOwned(), - duration: video.duration, - - views: video.views, - viewers: VideoViewsManager.Instance.getViewers(video), - - likes: video.likes, - dislikes: video.dislikes, - thumbnailPath: video.getMiniatureStaticPath(), - previewPath: video.getPreviewStaticPath(), - embedPath: video.getEmbedStaticPath(), - createdAt: video.createdAt, - updatedAt: video.updatedAt, - publishedAt: video.publishedAt, - originallyPublishedAt: video.originallyPublishedAt, - - isLive: video.isLive, - - account: video.VideoChannel.Account.toFormattedSummaryJSON(), - channel: video.VideoChannel.toFormattedSummaryJSON(), - - userHistory: userHistory - ? { currentTime: userHistory.currentTime } - : undefined, - - // Can be added by external plugins - pluginData: (video as any).pluginData, - - ...buildAdditionalAttributes(video, options) - } - - span.end() - - return videoObject -} - -export function videoModelToFormattedDetailsJSON (video: MVideoFormattableDetails): VideoDetails { - const span = tracer.startSpan('peertube.VideoModel.toFormattedDetailsJSON') - - const videoJSON = video.toFormattedJSON({ - completeDescription: true, - additionalAttributes: { - scheduledUpdate: true, - blacklistInfo: true, - files: true - } - }) as Video & Required> - - const tags = video.Tags - ? video.Tags.map(t => t.name) - : [] - - const detailsJSON = { - ...videoJSON, - - support: video.support, - descriptionPath: video.getDescriptionAPIPath(), - channel: video.VideoChannel.toFormattedJSON(), - account: video.VideoChannel.Account.toFormattedJSON(), - tags, - commentsEnabled: video.commentsEnabled, - downloadEnabled: video.downloadEnabled, - waitTranscoding: video.waitTranscoding, - inputFileUpdatedAt: video.inputFileUpdatedAt, - state: { - id: video.state, - label: getStateLabel(video.state) - }, - - trackerUrls: video.getTrackerUrls() - } - - span.end() - - return detailsJSON -} - -export function streamingPlaylistsModelToFormattedJSON ( - video: MVideoFormattable, - playlists: MStreamingPlaylistRedundanciesOpt[] -): VideoStreamingPlaylist[] { - if (isArray(playlists) === false) return [] - - return playlists - .map(playlist => ({ - id: playlist.id, - type: playlist.type, - - playlistUrl: playlist.getMasterPlaylistUrl(video), - segmentsSha256Url: playlist.getSha256SegmentsUrl(video), - - redundancies: isArray(playlist.RedundancyVideos) - ? playlist.RedundancyVideos.map(r => ({ baseUrl: r.fileUrl })) - : [], - - files: videoFilesModelToFormattedJSON(video, playlist.VideoFiles) - })) -} - -export function videoFilesModelToFormattedJSON ( - video: MVideoFormattable, - videoFiles: MVideoFileRedundanciesOpt[], - options: { - includeMagnet?: boolean // default true - } = {} -): VideoFile[] { - const { includeMagnet = true } = options - - if (isArray(videoFiles) === false) return [] - - const trackerUrls = includeMagnet - ? video.getTrackerUrls() - : [] - - return videoFiles - .filter(f => !f.isLive()) - .sort(sortByResolutionDesc) - .map(videoFile => { - return { - id: videoFile.id, - - resolution: { - id: videoFile.resolution, - label: videoFile.resolution === 0 - ? 'Audio' - : `${videoFile.resolution}p` - }, - - magnetUri: includeMagnet && videoFile.hasTorrent() - ? generateMagnetUri(video, videoFile, trackerUrls) - : undefined, - - size: videoFile.size, - fps: videoFile.fps, - - torrentUrl: videoFile.getTorrentUrl(), - torrentDownloadUrl: videoFile.getTorrentDownloadUrl(), - - fileUrl: videoFile.getFileUrl(video), - fileDownloadUrl: videoFile.getFileDownloadUrl(video), - - metadataUrl: videoFile.metadataUrl ?? getLocalVideoFileMetadataUrl(video, videoFile) - } - }) -} - -// --------------------------------------------------------------------------- - -export function getCategoryLabel (id: number) { - return VIDEO_CATEGORIES[id] || 'Unknown' -} - -export function getLicenceLabel (id: number) { - return VIDEO_LICENCES[id] || 'Unknown' -} - -export function getLanguageLabel (id: string) { - return VIDEO_LANGUAGES[id] || 'Unknown' -} - -export function getPrivacyLabel (id: number) { - return VIDEO_PRIVACIES[id] || 'Unknown' -} - -export function getStateLabel (id: number) { - return VIDEO_STATES[id] || 'Unknown' -} - -// --------------------------------------------------------------------------- -// Private -// --------------------------------------------------------------------------- - -function buildAdditionalAttributes (video: MVideoFormattable, options: VideoFormattingJSONOptions) { - const add = options.additionalAttributes - - const result: Partial = {} - - if (add?.state === true) { - result.state = { - id: video.state, - label: getStateLabel(video.state) - } - } - - if (add?.waitTranscoding === true) { - result.waitTranscoding = video.waitTranscoding - } - - if (add?.scheduledUpdate === true && video.ScheduleVideoUpdate) { - result.scheduledUpdate = { - updateAt: video.ScheduleVideoUpdate.updateAt, - privacy: video.ScheduleVideoUpdate.privacy || undefined - } - } - - if (add?.blacklistInfo === true) { - result.blacklisted = !!video.VideoBlacklist - result.blacklistedReason = - video.VideoBlacklist - ? video.VideoBlacklist.reason - : null - } - - if (add?.blockedOwner === true) { - result.blockedOwner = video.VideoChannel.Account.isBlocked() - - const server = video.VideoChannel.Account.Actor.Server as MServer - result.blockedServer = !!(server?.isBlocked()) - } - - if (add?.files === true) { - result.streamingPlaylists = streamingPlaylistsModelToFormattedJSON(video, video.VideoStreamingPlaylists) - result.files = videoFilesModelToFormattedJSON(video, video.VideoFiles) - } - - return result -} diff --git a/server/models/video/schedule-video-update.ts b/server/models/video/schedule-video-update.ts deleted file mode 100644 index b3cf26966..000000000 --- a/server/models/video/schedule-video-update.ts +++ /dev/null @@ -1,95 +0,0 @@ -import { Op, Transaction } from 'sequelize' -import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { MScheduleVideoUpdateFormattable, MScheduleVideoUpdate } from '@server/types/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { VideoPrivacy } from '../../../shared/models/videos' -import { VideoModel } from './video' - -@Table({ - tableName: 'scheduleVideoUpdate', - indexes: [ - { - fields: [ 'videoId' ], - unique: true - }, - { - fields: [ 'updateAt' ] - } - ] -}) -export class ScheduleVideoUpdateModel extends Model>> { - - @AllowNull(false) - @Default(null) - @Column - updateAt: Date - - @AllowNull(true) - @Default(null) - @Column - privacy: VideoPrivacy.PUBLIC | VideoPrivacy.UNLISTED | VideoPrivacy.INTERNAL - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => VideoModel) - @Column - videoId: number - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade' - }) - Video: VideoModel - - static areVideosToUpdate () { - const query = { - logging: false, - attributes: [ 'id' ], - where: { - updateAt: { - [Op.lte]: new Date() - } - } - } - - return ScheduleVideoUpdateModel.findOne(query) - .then(res => !!res) - } - - static listVideosToUpdate (transaction?: Transaction) { - const query = { - where: { - updateAt: { - [Op.lte]: new Date() - } - }, - transaction - } - - return ScheduleVideoUpdateModel.findAll(query) - } - - static deleteByVideoId (videoId: number, t: Transaction) { - const query = { - where: { - videoId - }, - transaction: t - } - - return ScheduleVideoUpdateModel.destroy(query) - } - - toFormattedJSON (this: MScheduleVideoUpdateFormattable) { - return { - updateAt: this.updateAt, - privacy: this.privacy || undefined - } - } -} diff --git a/server/models/video/sql/comment/video-comment-list-query-builder.ts b/server/models/video/sql/comment/video-comment-list-query-builder.ts deleted file mode 100644 index a7eed22a1..000000000 --- a/server/models/video/sql/comment/video-comment-list-query-builder.ts +++ /dev/null @@ -1,400 +0,0 @@ -import { Model, Sequelize, Transaction } from 'sequelize' -import { AbstractRunQuery, ModelBuilder } from '@server/models/shared' -import { ActorImageType, VideoPrivacy } from '@shared/models' -import { createSafeIn, getSort, parseRowCountResult } from '../../../shared' -import { VideoCommentTableAttributes } from './video-comment-table-attributes' - -export interface ListVideoCommentsOptions { - selectType: 'api' | 'feed' | 'comment-only' - - start?: number - count?: number - sort?: string - - videoId?: number - threadId?: number - accountId?: number - videoChannelId?: number - - blockerAccountIds?: number[] - - isThread?: boolean - notDeleted?: boolean - isLocal?: boolean - onLocalVideo?: boolean - onPublicVideo?: boolean - videoAccountOwnerId?: boolean - - search?: string - searchAccount?: string - searchVideo?: string - - includeReplyCounters?: boolean - - transaction?: Transaction -} - -export class VideoCommentListQueryBuilder extends AbstractRunQuery { - private readonly tableAttributes = new VideoCommentTableAttributes() - - private innerQuery: string - - private select = '' - private joins = '' - - private innerSelect = '' - private innerJoins = '' - private innerLateralJoins = '' - private innerWhere = '' - - private readonly built = { - cte: false, - accountJoin: false, - videoJoin: false, - videoChannelJoin: false, - avatarJoin: false - } - - constructor ( - protected readonly sequelize: Sequelize, - private readonly options: ListVideoCommentsOptions - ) { - super(sequelize) - - if (this.options.includeReplyCounters && !this.options.videoId) { - throw new Error('Cannot include reply counters without videoId') - } - } - - async listComments () { - this.buildListQuery() - - const results = await this.runQuery({ nest: true, transaction: this.options.transaction }) - const modelBuilder = new ModelBuilder(this.sequelize) - - return modelBuilder.createModels(results, 'VideoComment') - } - - async countComments () { - this.buildCountQuery() - - const result = await this.runQuery({ transaction: this.options.transaction }) - - return parseRowCountResult(result) - } - - // --------------------------------------------------------------------------- - - private buildListQuery () { - this.buildInnerListQuery() - this.buildListSelect() - - this.query = `${this.select} ` + - `FROM (${this.innerQuery}) AS "VideoCommentModel" ` + - `${this.joins} ` + - `${this.getOrder()}` - } - - private buildInnerListQuery () { - this.buildWhere() - this.buildInnerListSelect() - - this.innerQuery = `${this.innerSelect} ` + - `FROM "videoComment" AS "VideoCommentModel" ` + - `${this.innerJoins} ` + - `${this.innerLateralJoins} ` + - `${this.innerWhere} ` + - `${this.getOrder()} ` + - `${this.getInnerLimit()}` - } - - // --------------------------------------------------------------------------- - - private buildCountQuery () { - this.buildWhere() - - this.query = `SELECT COUNT(*) AS "total" ` + - `FROM "videoComment" AS "VideoCommentModel" ` + - `${this.innerJoins} ` + - `${this.innerWhere}` - } - - // --------------------------------------------------------------------------- - - private buildWhere () { - let where: string[] = [] - - if (this.options.videoId) { - this.replacements.videoId = this.options.videoId - - where.push('"VideoCommentModel"."videoId" = :videoId') - } - - if (this.options.threadId) { - this.replacements.threadId = this.options.threadId - - where.push('("VideoCommentModel"."id" = :threadId OR "VideoCommentModel"."originCommentId" = :threadId)') - } - - if (this.options.accountId) { - this.replacements.accountId = this.options.accountId - - where.push('"VideoCommentModel"."accountId" = :accountId') - } - - if (this.options.videoChannelId) { - this.buildVideoChannelJoin() - - this.replacements.videoChannelId = this.options.videoChannelId - - where.push('"Account->VideoChannel"."id" = :videoChannelId') - } - - if (this.options.blockerAccountIds) { - this.buildVideoChannelJoin() - - where = where.concat(this.getBlockWhere('VideoCommentModel', 'Video->VideoChannel')) - } - - if (this.options.isThread === true) { - where.push('"VideoCommentModel"."inReplyToCommentId" IS NULL') - } - - if (this.options.notDeleted === true) { - where.push('"VideoCommentModel"."deletedAt" IS NULL') - } - - if (this.options.isLocal === true) { - this.buildAccountJoin() - - where.push('"Account->Actor"."serverId" IS NULL') - } else if (this.options.isLocal === false) { - this.buildAccountJoin() - - where.push('"Account->Actor"."serverId" IS NOT NULL') - } - - if (this.options.onLocalVideo === true) { - this.buildVideoJoin() - - where.push('"Video"."remote" IS FALSE') - } else if (this.options.onLocalVideo === false) { - this.buildVideoJoin() - - where.push('"Video"."remote" IS TRUE') - } - - if (this.options.onPublicVideo === true) { - this.buildVideoJoin() - - where.push(`"Video"."privacy" = ${VideoPrivacy.PUBLIC}`) - } - - if (this.options.videoAccountOwnerId) { - this.buildVideoChannelJoin() - - this.replacements.videoAccountOwnerId = this.options.videoAccountOwnerId - - where.push(`"Video->VideoChannel"."accountId" = :videoAccountOwnerId`) - } - - if (this.options.search) { - this.buildVideoJoin() - this.buildAccountJoin() - - const escapedLikeSearch = this.sequelize.escape('%' + this.options.search + '%') - - where.push( - `(` + - `"VideoCommentModel"."text" ILIKE ${escapedLikeSearch} OR ` + - `"Account->Actor"."preferredUsername" ILIKE ${escapedLikeSearch} OR ` + - `"Account"."name" ILIKE ${escapedLikeSearch} OR ` + - `"Video"."name" ILIKE ${escapedLikeSearch} ` + - `)` - ) - } - - if (this.options.searchAccount) { - this.buildAccountJoin() - - const escapedLikeSearch = this.sequelize.escape('%' + this.options.searchAccount + '%') - - where.push( - `(` + - `"Account->Actor"."preferredUsername" ILIKE ${escapedLikeSearch} OR ` + - `"Account"."name" ILIKE ${escapedLikeSearch} ` + - `)` - ) - } - - if (this.options.searchVideo) { - this.buildVideoJoin() - - const escapedLikeSearch = this.sequelize.escape('%' + this.options.searchVideo + '%') - - where.push(`"Video"."name" ILIKE ${escapedLikeSearch}`) - } - - if (where.length !== 0) { - this.innerWhere = `WHERE ${where.join(' AND ')}` - } - } - - private buildAccountJoin () { - if (this.built.accountJoin) return - - this.innerJoins += ' LEFT JOIN "account" "Account" ON "Account"."id" = "VideoCommentModel"."accountId" ' + - 'LEFT JOIN "actor" "Account->Actor" ON "Account->Actor"."id" = "Account"."actorId" ' + - 'LEFT JOIN "server" "Account->Actor->Server" ON "Account->Actor"."serverId" = "Account->Actor->Server"."id" ' - - this.built.accountJoin = true - } - - private buildVideoJoin () { - if (this.built.videoJoin) return - - this.innerJoins += ' LEFT JOIN "video" "Video" ON "Video"."id" = "VideoCommentModel"."videoId" ' - - this.built.videoJoin = true - } - - private buildVideoChannelJoin () { - if (this.built.videoChannelJoin) return - - this.buildVideoJoin() - - this.innerJoins += ' LEFT JOIN "videoChannel" "Video->VideoChannel" ON "Video"."channelId" = "Video->VideoChannel"."id" ' - - this.built.videoChannelJoin = true - } - - private buildAvatarsJoin () { - if (this.options.selectType !== 'api' && this.options.selectType !== 'feed') return '' - if (this.built.avatarJoin) return - - this.joins += `LEFT JOIN "actorImage" "Account->Actor->Avatars" ` + - `ON "VideoCommentModel"."Account.Actor.id" = "Account->Actor->Avatars"."actorId" ` + - `AND "Account->Actor->Avatars"."type" = ${ActorImageType.AVATAR}` - - this.built.avatarJoin = true - } - - // --------------------------------------------------------------------------- - - private buildListSelect () { - const toSelect = [ '"VideoCommentModel".*' ] - - if (this.options.selectType === 'api' || this.options.selectType === 'feed') { - this.buildAvatarsJoin() - - toSelect.push(this.tableAttributes.getAvatarAttributes()) - } - - this.select = this.buildSelect(toSelect) - } - - private buildInnerListSelect () { - let toSelect = [ this.tableAttributes.getVideoCommentAttributes() ] - - if (this.options.selectType === 'api' || this.options.selectType === 'feed') { - this.buildAccountJoin() - this.buildVideoJoin() - - toSelect = toSelect.concat([ - this.tableAttributes.getVideoAttributes(), - this.tableAttributes.getAccountAttributes(), - this.tableAttributes.getActorAttributes(), - this.tableAttributes.getServerAttributes() - ]) - } - - if (this.options.includeReplyCounters === true) { - this.buildTotalRepliesSelect() - this.buildAuthorTotalRepliesSelect() - - toSelect.push('"totalRepliesFromVideoAuthor"."count" AS "totalRepliesFromVideoAuthor"') - toSelect.push('"totalReplies"."count" AS "totalReplies"') - } - - this.innerSelect = this.buildSelect(toSelect) - } - - // --------------------------------------------------------------------------- - - private getBlockWhere (commentTableName: string, channelTableName: string) { - const where: string[] = [] - - const blockerIdsString = createSafeIn( - this.sequelize, - this.options.blockerAccountIds, - [ `"${channelTableName}"."accountId"` ] - ) - - where.push( - `NOT EXISTS (` + - `SELECT 1 FROM "accountBlocklist" ` + - `WHERE "targetAccountId" = "${commentTableName}"."accountId" ` + - `AND "accountId" IN (${blockerIdsString})` + - `)` - ) - - where.push( - `NOT EXISTS (` + - `SELECT 1 FROM "account" ` + - `INNER JOIN "actor" ON account."actorId" = actor.id ` + - `INNER JOIN "serverBlocklist" ON "actor"."serverId" = "serverBlocklist"."targetServerId" ` + - `WHERE "account"."id" = "${commentTableName}"."accountId" ` + - `AND "serverBlocklist"."accountId" IN (${blockerIdsString})` + - `)` - ) - - return where - } - - // --------------------------------------------------------------------------- - - private buildTotalRepliesSelect () { - const blockWhereString = this.getBlockWhere('replies', 'videoChannel').join(' AND ') - - // Help the planner by providing videoId that should filter out many comments - this.replacements.videoId = this.options.videoId - - this.innerLateralJoins += `LEFT JOIN LATERAL (` + - `SELECT COUNT("replies"."id") AS "count" FROM "videoComment" AS "replies" ` + - `INNER JOIN "video" ON "video"."id" = "replies"."videoId" AND "replies"."videoId" = :videoId ` + - `LEFT JOIN "videoChannel" ON "video"."channelId" = "videoChannel"."id" ` + - `WHERE "replies"."originCommentId" = "VideoCommentModel"."id" ` + - `AND "deletedAt" IS NULL ` + - `AND ${blockWhereString} ` + - `) "totalReplies" ON TRUE ` - } - - private buildAuthorTotalRepliesSelect () { - // Help the planner by providing videoId that should filter out many comments - this.replacements.videoId = this.options.videoId - - this.innerLateralJoins += `LEFT JOIN LATERAL (` + - `SELECT COUNT("replies"."id") AS "count" FROM "videoComment" AS "replies" ` + - `INNER JOIN "video" ON "video"."id" = "replies"."videoId" AND "replies"."videoId" = :videoId ` + - `INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ` + - `WHERE "replies"."originCommentId" = "VideoCommentModel"."id" AND "replies"."accountId" = "videoChannel"."accountId"` + - `) "totalRepliesFromVideoAuthor" ON TRUE ` - } - - private getOrder () { - if (!this.options.sort) return '' - - const orders = getSort(this.options.sort) - - return 'ORDER BY ' + orders.map(o => `"${o[0]}" ${o[1]}`).join(', ') - } - - private getInnerLimit () { - if (!this.options.count) return '' - - this.replacements.limit = this.options.count - this.replacements.offset = this.options.start || 0 - - return `LIMIT :limit OFFSET :offset ` - } -} diff --git a/server/models/video/sql/comment/video-comment-table-attributes.ts b/server/models/video/sql/comment/video-comment-table-attributes.ts deleted file mode 100644 index 87f8750c1..000000000 --- a/server/models/video/sql/comment/video-comment-table-attributes.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { Memoize } from '@server/helpers/memoize' -import { AccountModel } from '@server/models/account/account' -import { ActorModel } from '@server/models/actor/actor' -import { ActorImageModel } from '@server/models/actor/actor-image' -import { ServerModel } from '@server/models/server/server' -import { VideoCommentModel } from '../../video-comment' - -export class VideoCommentTableAttributes { - - @Memoize() - getVideoCommentAttributes () { - return VideoCommentModel.getSQLAttributes('VideoCommentModel').join(', ') - } - - @Memoize() - getAccountAttributes () { - return AccountModel.getSQLAttributes('Account', 'Account.').join(', ') - } - - @Memoize() - getVideoAttributes () { - return [ - `"Video"."id" AS "Video.id"`, - `"Video"."uuid" AS "Video.uuid"`, - `"Video"."name" AS "Video.name"` - ].join(', ') - } - - @Memoize() - getActorAttributes () { - return ActorModel.getSQLAPIAttributes('Account->Actor', `Account.Actor.`).join(', ') - } - - @Memoize() - getServerAttributes () { - return ServerModel.getSQLAttributes('Account->Actor->Server', `Account.Actor.Server.`).join(', ') - } - - @Memoize() - getAvatarAttributes () { - return ActorImageModel.getSQLAttributes('Account->Actor->Avatars', 'Account.Actor.Avatars.').join(', ') - } -} diff --git a/server/models/video/sql/video/index.ts b/server/models/video/sql/video/index.ts deleted file mode 100644 index e9132d5e1..000000000 --- a/server/models/video/sql/video/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './video-model-get-query-builder' -export * from './videos-id-list-query-builder' -export * from './videos-model-list-query-builder' diff --git a/server/models/video/sql/video/shared/abstract-video-query-builder.ts b/server/models/video/sql/video/shared/abstract-video-query-builder.ts deleted file mode 100644 index 56a00aa0c..000000000 --- a/server/models/video/sql/video/shared/abstract-video-query-builder.ts +++ /dev/null @@ -1,340 +0,0 @@ -import { Sequelize } from 'sequelize' -import validator from 'validator' -import { MUserAccountId } from '@server/types/models' -import { ActorImageType } from '@shared/models' -import { AbstractRunQuery } from '../../../../shared/abstract-run-query' -import { createSafeIn } from '../../../../shared' -import { VideoTableAttributes } from './video-table-attributes' - -/** - * - * Abstract builder to create SQL query and fetch video models - * - */ - -export class AbstractVideoQueryBuilder extends AbstractRunQuery { - protected attributes: { [key: string]: string } = {} - - protected joins = '' - protected where: string - - protected tables: VideoTableAttributes - - constructor ( - protected readonly sequelize: Sequelize, - protected readonly mode: 'list' | 'get' - ) { - super(sequelize) - - this.tables = new VideoTableAttributes(this.mode) - } - - protected buildSelect () { - return 'SELECT ' + Object.keys(this.attributes).map(key => { - const value = this.attributes[key] - if (value) return `${key} AS ${value}` - - return key - }).join(', ') - } - - protected includeChannels () { - this.addJoin('INNER JOIN "videoChannel" AS "VideoChannel" ON "video"."channelId" = "VideoChannel"."id"') - this.addJoin('INNER JOIN "actor" AS "VideoChannel->Actor" ON "VideoChannel"."actorId" = "VideoChannel->Actor"."id"') - - this.addJoin( - 'LEFT OUTER JOIN "server" AS "VideoChannel->Actor->Server" ON "VideoChannel->Actor"."serverId" = "VideoChannel->Actor->Server"."id"' - ) - - this.addJoin( - 'LEFT OUTER JOIN "actorImage" AS "VideoChannel->Actor->Avatars" ' + - 'ON "VideoChannel->Actor"."id" = "VideoChannel->Actor->Avatars"."actorId" ' + - `AND "VideoChannel->Actor->Avatars"."type" = ${ActorImageType.AVATAR}` - ) - - this.attributes = { - ...this.attributes, - - ...this.buildAttributesObject('VideoChannel', this.tables.getChannelAttributes()), - ...this.buildActorInclude('VideoChannel->Actor'), - ...this.buildAvatarInclude('VideoChannel->Actor->Avatars'), - ...this.buildServerInclude('VideoChannel->Actor->Server') - } - } - - protected includeAccounts () { - this.addJoin('INNER JOIN "account" AS "VideoChannel->Account" ON "VideoChannel"."accountId" = "VideoChannel->Account"."id"') - this.addJoin( - 'INNER JOIN "actor" AS "VideoChannel->Account->Actor" ON "VideoChannel->Account"."actorId" = "VideoChannel->Account->Actor"."id"' - ) - - this.addJoin( - 'LEFT OUTER JOIN "server" AS "VideoChannel->Account->Actor->Server" ' + - 'ON "VideoChannel->Account->Actor"."serverId" = "VideoChannel->Account->Actor->Server"."id"' - ) - - this.addJoin( - 'LEFT OUTER JOIN "actorImage" AS "VideoChannel->Account->Actor->Avatars" ' + - 'ON "VideoChannel->Account"."actorId"= "VideoChannel->Account->Actor->Avatars"."actorId" ' + - `AND "VideoChannel->Account->Actor->Avatars"."type" = ${ActorImageType.AVATAR}` - ) - - this.attributes = { - ...this.attributes, - - ...this.buildAttributesObject('VideoChannel->Account', this.tables.getAccountAttributes()), - ...this.buildActorInclude('VideoChannel->Account->Actor'), - ...this.buildAvatarInclude('VideoChannel->Account->Actor->Avatars'), - ...this.buildServerInclude('VideoChannel->Account->Actor->Server') - } - } - - protected includeOwnerUser () { - this.addJoin('INNER JOIN "videoChannel" AS "VideoChannel" ON "video"."channelId" = "VideoChannel"."id"') - this.addJoin('INNER JOIN "account" AS "VideoChannel->Account" ON "VideoChannel"."accountId" = "VideoChannel->Account"."id"') - - this.attributes = { - ...this.attributes, - - ...this.buildAttributesObject('VideoChannel', this.tables.getChannelAttributes()), - ...this.buildAttributesObject('VideoChannel->Account', this.tables.getUserAccountAttributes()) - } - } - - protected includeThumbnails () { - this.addJoin('LEFT OUTER JOIN "thumbnail" AS "Thumbnails" ON "video"."id" = "Thumbnails"."videoId"') - - this.attributes = { - ...this.attributes, - - ...this.buildAttributesObject('Thumbnails', this.tables.getThumbnailAttributes()) - } - } - - protected includeWebVideoFiles () { - this.addJoin('LEFT JOIN "videoFile" AS "VideoFiles" ON "VideoFiles"."videoId" = "video"."id"') - - this.attributes = { - ...this.attributes, - - ...this.buildAttributesObject('VideoFiles', this.tables.getFileAttributes()) - } - } - - protected includeStreamingPlaylistFiles () { - this.addJoin( - 'LEFT JOIN "videoStreamingPlaylist" AS "VideoStreamingPlaylists" ON "VideoStreamingPlaylists"."videoId" = "video"."id"' - ) - - this.addJoin( - 'LEFT JOIN "videoFile" AS "VideoStreamingPlaylists->VideoFiles" ' + - 'ON "VideoStreamingPlaylists->VideoFiles"."videoStreamingPlaylistId" = "VideoStreamingPlaylists"."id"' - ) - - this.attributes = { - ...this.attributes, - - ...this.buildAttributesObject('VideoStreamingPlaylists', this.tables.getStreamingPlaylistAttributes()), - ...this.buildAttributesObject('VideoStreamingPlaylists->VideoFiles', this.tables.getFileAttributes()) - } - } - - protected includeUserHistory (userId: number) { - this.addJoin( - 'LEFT OUTER JOIN "userVideoHistory" ' + - 'ON "video"."id" = "userVideoHistory"."videoId" AND "userVideoHistory"."userId" = :userVideoHistoryId' - ) - - this.replacements.userVideoHistoryId = userId - - this.attributes = { - ...this.attributes, - - ...this.buildAttributesObject('userVideoHistory', this.tables.getUserHistoryAttributes()) - } - } - - protected includePlaylist (playlistId: number) { - this.addJoin( - 'INNER JOIN "videoPlaylistElement" as "VideoPlaylistElement" ON "videoPlaylistElement"."videoId" = "video"."id" ' + - 'AND "VideoPlaylistElement"."videoPlaylistId" = :videoPlaylistId' - ) - - this.replacements.videoPlaylistId = playlistId - - this.attributes = { - ...this.attributes, - - ...this.buildAttributesObject('VideoPlaylistElement', this.tables.getPlaylistAttributes()) - } - } - - protected includeTags () { - this.addJoin( - 'LEFT OUTER JOIN (' + - '"videoTag" AS "Tags->VideoTagModel" INNER JOIN "tag" AS "Tags" ON "Tags"."id" = "Tags->VideoTagModel"."tagId"' + - ') ' + - 'ON "video"."id" = "Tags->VideoTagModel"."videoId"' - ) - - this.attributes = { - ...this.attributes, - - ...this.buildAttributesObject('Tags', this.tables.getTagAttributes()), - ...this.buildAttributesObject('Tags->VideoTagModel', this.tables.getVideoTagAttributes()) - } - } - - protected includeBlacklisted () { - this.addJoin( - 'LEFT OUTER JOIN "videoBlacklist" AS "VideoBlacklist" ON "video"."id" = "VideoBlacklist"."videoId"' - ) - - this.attributes = { - ...this.attributes, - - ...this.buildAttributesObject('VideoBlacklist', this.tables.getBlacklistedAttributes()) - } - } - - protected includeBlockedOwnerAndServer (serverAccountId: number, user?: MUserAccountId) { - const blockerIds = [ serverAccountId ] - if (user) blockerIds.push(user.Account.id) - - const inClause = createSafeIn(this.sequelize, blockerIds) - - this.addJoin( - 'LEFT JOIN "accountBlocklist" AS "VideoChannel->Account->AccountBlocklist" ' + - 'ON "VideoChannel->Account"."id" = "VideoChannel->Account->AccountBlocklist"."targetAccountId" ' + - 'AND "VideoChannel->Account->AccountBlocklist"."accountId" IN (' + inClause + ')' - ) - - this.addJoin( - 'LEFT JOIN "serverBlocklist" AS "VideoChannel->Account->Actor->Server->ServerBlocklist" ' + - 'ON "VideoChannel->Account->Actor->Server->ServerBlocklist"."targetServerId" = "VideoChannel->Account->Actor"."serverId" ' + - 'AND "VideoChannel->Account->Actor->Server->ServerBlocklist"."accountId" IN (' + inClause + ') ' - ) - - this.attributes = { - ...this.attributes, - - ...this.buildAttributesObject('VideoChannel->Account->AccountBlocklist', this.tables.getBlocklistAttributes()), - ...this.buildAttributesObject('VideoChannel->Account->Actor->Server->ServerBlocklist', this.tables.getBlocklistAttributes()) - } - } - - protected includeScheduleUpdate () { - this.addJoin( - 'LEFT OUTER JOIN "scheduleVideoUpdate" AS "ScheduleVideoUpdate" ON "video"."id" = "ScheduleVideoUpdate"."videoId"' - ) - - this.attributes = { - ...this.attributes, - - ...this.buildAttributesObject('ScheduleVideoUpdate', this.tables.getScheduleUpdateAttributes()) - } - } - - protected includeLive () { - this.addJoin( - 'LEFT OUTER JOIN "videoLive" AS "VideoLive" ON "video"."id" = "VideoLive"."videoId"' - ) - - this.attributes = { - ...this.attributes, - - ...this.buildAttributesObject('VideoLive', this.tables.getLiveAttributes()) - } - } - - protected includeTrackers () { - this.addJoin( - 'LEFT OUTER JOIN (' + - '"videoTracker" AS "Trackers->VideoTrackerModel" ' + - 'INNER JOIN "tracker" AS "Trackers" ON "Trackers"."id" = "Trackers->VideoTrackerModel"."trackerId"' + - ') ON "video"."id" = "Trackers->VideoTrackerModel"."videoId"' - ) - - this.attributes = { - ...this.attributes, - - ...this.buildAttributesObject('Trackers', this.tables.getTrackerAttributes()), - ...this.buildAttributesObject('Trackers->VideoTrackerModel', this.tables.getVideoTrackerAttributes()) - } - } - - protected includeWebVideoRedundancies () { - this.addJoin( - 'LEFT OUTER JOIN "videoRedundancy" AS "VideoFiles->RedundancyVideos" ON ' + - '"VideoFiles"."id" = "VideoFiles->RedundancyVideos"."videoFileId"' - ) - - this.attributes = { - ...this.attributes, - - ...this.buildAttributesObject('VideoFiles->RedundancyVideos', this.tables.getRedundancyAttributes()) - } - } - - protected includeStreamingPlaylistRedundancies () { - this.addJoin( - 'LEFT OUTER JOIN "videoRedundancy" AS "VideoStreamingPlaylists->RedundancyVideos" ' + - 'ON "VideoStreamingPlaylists"."id" = "VideoStreamingPlaylists->RedundancyVideos"."videoStreamingPlaylistId"' - ) - - this.attributes = { - ...this.attributes, - - ...this.buildAttributesObject('VideoStreamingPlaylists->RedundancyVideos', this.tables.getRedundancyAttributes()) - } - } - - protected buildActorInclude (prefixKey: string) { - return this.buildAttributesObject(prefixKey, this.tables.getActorAttributes()) - } - - protected buildAvatarInclude (prefixKey: string) { - return this.buildAttributesObject(prefixKey, this.tables.getAvatarAttributes()) - } - - protected buildServerInclude (prefixKey: string) { - return this.buildAttributesObject(prefixKey, this.tables.getServerAttributes()) - } - - protected buildAttributesObject (prefixKey: string, attributeKeys: string[]) { - const result: { [id: string]: string } = {} - - const prefixValue = prefixKey.replace(/->/g, '.') - - for (const attribute of attributeKeys) { - result[`"${prefixKey}"."${attribute}"`] = `"${prefixValue}.${attribute}"` - } - - return result - } - - protected whereId (options: { ids?: number[], id?: string | number, url?: string }) { - if (options.ids) { - this.where = `WHERE "video"."id" IN (${createSafeIn(this.sequelize, options.ids)})` - return - } - - if (options.url) { - this.where = 'WHERE "video"."url" = :videoUrl' - this.replacements.videoUrl = options.url - return - } - - if (validator.isInt('' + options.id)) { - this.where = 'WHERE "video".id = :videoId' - } else { - this.where = 'WHERE uuid = :videoId' - } - - this.replacements.videoId = options.id - } - - protected addJoin (join: string) { - this.joins += join + ' ' - } -} diff --git a/server/models/video/sql/video/shared/video-file-query-builder.ts b/server/models/video/sql/video/shared/video-file-query-builder.ts deleted file mode 100644 index 196b72b43..000000000 --- a/server/models/video/sql/video/shared/video-file-query-builder.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { Sequelize, Transaction } from 'sequelize' -import { AbstractVideoQueryBuilder } from './abstract-video-query-builder' - -export type FileQueryOptions = { - id?: string | number - url?: string - - includeRedundancy: boolean - - transaction?: Transaction - - logging?: boolean -} - -/** - * - * Fetch files (web videos and streaming playlist) according to a video - * - */ - -export class VideoFileQueryBuilder extends AbstractVideoQueryBuilder { - protected attributes: { [key: string]: string } - - constructor (protected readonly sequelize: Sequelize) { - super(sequelize, 'get') - } - - queryWebVideos (options: FileQueryOptions) { - this.buildWebVideoFilesQuery(options) - - return this.runQuery(options) - } - - queryStreamingPlaylistVideos (options: FileQueryOptions) { - this.buildVideoStreamingPlaylistFilesQuery(options) - - return this.runQuery(options) - } - - private buildWebVideoFilesQuery (options: FileQueryOptions) { - this.attributes = { - '"video"."id"': '' - } - - this.includeWebVideoFiles() - - if (options.includeRedundancy) { - this.includeWebVideoRedundancies() - } - - this.whereId(options) - - this.query = this.buildQuery() - } - - private buildVideoStreamingPlaylistFilesQuery (options: FileQueryOptions) { - this.attributes = { - '"video"."id"': '' - } - - this.includeStreamingPlaylistFiles() - - if (options.includeRedundancy) { - this.includeStreamingPlaylistRedundancies() - } - - this.whereId(options) - - this.query = this.buildQuery() - } - - private buildQuery () { - return `${this.buildSelect()} FROM "video" ${this.joins} ${this.where}` - } -} diff --git a/server/models/video/sql/video/shared/video-model-builder.ts b/server/models/video/sql/video/shared/video-model-builder.ts deleted file mode 100644 index 740aa842f..000000000 --- a/server/models/video/sql/video/shared/video-model-builder.ts +++ /dev/null @@ -1,408 +0,0 @@ - -import { AccountModel } from '@server/models/account/account' -import { AccountBlocklistModel } from '@server/models/account/account-blocklist' -import { ActorModel } from '@server/models/actor/actor' -import { ActorImageModel } from '@server/models/actor/actor-image' -import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy' -import { ServerModel } from '@server/models/server/server' -import { ServerBlocklistModel } from '@server/models/server/server-blocklist' -import { TrackerModel } from '@server/models/server/tracker' -import { UserVideoHistoryModel } from '@server/models/user/user-video-history' -import { VideoInclude } from '@shared/models' -import { ScheduleVideoUpdateModel } from '../../../schedule-video-update' -import { TagModel } from '../../../tag' -import { ThumbnailModel } from '../../../thumbnail' -import { VideoModel } from '../../../video' -import { VideoBlacklistModel } from '../../../video-blacklist' -import { VideoChannelModel } from '../../../video-channel' -import { VideoFileModel } from '../../../video-file' -import { VideoLiveModel } from '../../../video-live' -import { VideoStreamingPlaylistModel } from '../../../video-streaming-playlist' -import { VideoTableAttributes } from './video-table-attributes' - -type SQLRow = { [id: string]: string | number } - -/** - * - * Build video models from SQL rows - * - */ - -export class VideoModelBuilder { - private videosMemo: { [ id: number ]: VideoModel } - private videoStreamingPlaylistMemo: { [ id: number ]: VideoStreamingPlaylistModel } - private videoFileMemo: { [ id: number ]: VideoFileModel } - - private thumbnailsDone: Set - private actorImagesDone: Set - private historyDone: Set - private blacklistDone: Set - private accountBlocklistDone: Set - private serverBlocklistDone: Set - private liveDone: Set - private redundancyDone: Set - private scheduleVideoUpdateDone: Set - - private trackersDone: Set - private tagsDone: Set - - private videos: VideoModel[] - - private readonly buildOpts = { raw: true, isNewRecord: false } - - constructor ( - private readonly mode: 'get' | 'list', - private readonly tables: VideoTableAttributes - ) { - - } - - buildVideosFromRows (options: { - rows: SQLRow[] - include?: VideoInclude - rowsWebVideoFiles?: SQLRow[] - rowsStreamingPlaylist?: SQLRow[] - }) { - const { rows, rowsWebVideoFiles, rowsStreamingPlaylist, include } = options - - this.reinit() - - for (const row of rows) { - this.buildVideoAndAccount(row) - - const videoModel = this.videosMemo[row.id as number] - - this.setUserHistory(row, videoModel) - this.addThumbnail(row, videoModel) - - const channelActor = videoModel.VideoChannel?.Actor - if (channelActor) { - this.addActorAvatar(row, 'VideoChannel.Actor', channelActor) - } - - const accountActor = videoModel.VideoChannel?.Account?.Actor - if (accountActor) { - this.addActorAvatar(row, 'VideoChannel.Account.Actor', accountActor) - } - - if (!rowsWebVideoFiles) { - this.addWebVideoFile(row, videoModel) - } - - if (!rowsStreamingPlaylist) { - this.addStreamingPlaylist(row, videoModel) - this.addStreamingPlaylistFile(row) - } - - if (this.mode === 'get') { - this.addTag(row, videoModel) - this.addTracker(row, videoModel) - this.setBlacklisted(row, videoModel) - this.setScheduleVideoUpdate(row, videoModel) - this.setLive(row, videoModel) - } else { - if (include & VideoInclude.BLACKLISTED) { - this.setBlacklisted(row, videoModel) - } - - if (include & VideoInclude.BLOCKED_OWNER) { - this.setBlockedOwner(row, videoModel) - this.setBlockedServer(row, videoModel) - } - } - } - - this.grabSeparateWebVideoFiles(rowsWebVideoFiles) - this.grabSeparateStreamingPlaylistFiles(rowsStreamingPlaylist) - - return this.videos - } - - private reinit () { - this.videosMemo = {} - this.videoStreamingPlaylistMemo = {} - this.videoFileMemo = {} - - this.thumbnailsDone = new Set() - this.actorImagesDone = new Set() - this.historyDone = new Set() - this.blacklistDone = new Set() - this.liveDone = new Set() - this.redundancyDone = new Set() - this.scheduleVideoUpdateDone = new Set() - - this.accountBlocklistDone = new Set() - this.serverBlocklistDone = new Set() - - this.trackersDone = new Set() - this.tagsDone = new Set() - - this.videos = [] - } - - private grabSeparateWebVideoFiles (rowsWebVideoFiles?: SQLRow[]) { - if (!rowsWebVideoFiles) return - - for (const row of rowsWebVideoFiles) { - const id = row['VideoFiles.id'] - if (!id) continue - - const videoModel = this.videosMemo[row.id] - this.addWebVideoFile(row, videoModel) - this.addRedundancy(row, 'VideoFiles', this.videoFileMemo[id]) - } - } - - private grabSeparateStreamingPlaylistFiles (rowsStreamingPlaylist?: SQLRow[]) { - if (!rowsStreamingPlaylist) return - - for (const row of rowsStreamingPlaylist) { - const id = row['VideoStreamingPlaylists.id'] - if (!id) continue - - const videoModel = this.videosMemo[row.id] - - this.addStreamingPlaylist(row, videoModel) - this.addStreamingPlaylistFile(row) - this.addRedundancy(row, 'VideoStreamingPlaylists', this.videoStreamingPlaylistMemo[id]) - } - } - - private buildVideoAndAccount (row: SQLRow) { - if (this.videosMemo[row.id]) return - - const videoModel = new VideoModel(this.grab(row, this.tables.getVideoAttributes(), ''), this.buildOpts) - - videoModel.UserVideoHistories = [] - videoModel.Thumbnails = [] - videoModel.VideoFiles = [] - videoModel.VideoStreamingPlaylists = [] - videoModel.Tags = [] - videoModel.Trackers = [] - - this.buildAccount(row, videoModel) - - this.videosMemo[row.id] = videoModel - - // Keep rows order - this.videos.push(videoModel) - } - - private buildAccount (row: SQLRow, videoModel: VideoModel) { - const id = row['VideoChannel.Account.id'] - if (!id) return - - const channelModel = new VideoChannelModel(this.grab(row, this.tables.getChannelAttributes(), 'VideoChannel'), this.buildOpts) - channelModel.Actor = this.buildActor(row, 'VideoChannel') - - const accountModel = new AccountModel(this.grab(row, this.tables.getAccountAttributes(), 'VideoChannel.Account'), this.buildOpts) - accountModel.Actor = this.buildActor(row, 'VideoChannel.Account') - - accountModel.BlockedBy = [] - - channelModel.Account = accountModel - - videoModel.VideoChannel = channelModel - } - - private buildActor (row: SQLRow, prefix: string) { - const actorPrefix = `${prefix}.Actor` - const serverPrefix = `${actorPrefix}.Server` - - const serverModel = row[`${serverPrefix}.id`] !== null - ? new ServerModel(this.grab(row, this.tables.getServerAttributes(), serverPrefix), this.buildOpts) - : null - - if (serverModel) serverModel.BlockedBy = [] - - const actorModel = new ActorModel(this.grab(row, this.tables.getActorAttributes(), actorPrefix), this.buildOpts) - actorModel.Server = serverModel - actorModel.Avatars = [] - - return actorModel - } - - private setUserHistory (row: SQLRow, videoModel: VideoModel) { - const id = row['userVideoHistory.id'] - if (!id || this.historyDone.has(id)) return - - const attributes = this.grab(row, this.tables.getUserHistoryAttributes(), 'userVideoHistory') - const historyModel = new UserVideoHistoryModel(attributes, this.buildOpts) - videoModel.UserVideoHistories.push(historyModel) - - this.historyDone.add(id) - } - - private addActorAvatar (row: SQLRow, actorPrefix: string, actor: ActorModel) { - const avatarPrefix = `${actorPrefix}.Avatars` - const id = row[`${avatarPrefix}.id`] - const key = `${row.id}${id}` - - if (!id || this.actorImagesDone.has(key)) return - - const attributes = this.grab(row, this.tables.getAvatarAttributes(), avatarPrefix) - const avatarModel = new ActorImageModel(attributes, this.buildOpts) - actor.Avatars.push(avatarModel) - - this.actorImagesDone.add(key) - } - - private addThumbnail (row: SQLRow, videoModel: VideoModel) { - const id = row['Thumbnails.id'] - if (!id || this.thumbnailsDone.has(id)) return - - const attributes = this.grab(row, this.tables.getThumbnailAttributes(), 'Thumbnails') - const thumbnailModel = new ThumbnailModel(attributes, this.buildOpts) - videoModel.Thumbnails.push(thumbnailModel) - - this.thumbnailsDone.add(id) - } - - private addWebVideoFile (row: SQLRow, videoModel: VideoModel) { - const id = row['VideoFiles.id'] - if (!id || this.videoFileMemo[id]) return - - const attributes = this.grab(row, this.tables.getFileAttributes(), 'VideoFiles') - const videoFileModel = new VideoFileModel(attributes, this.buildOpts) - videoModel.VideoFiles.push(videoFileModel) - - this.videoFileMemo[id] = videoFileModel - } - - private addStreamingPlaylist (row: SQLRow, videoModel: VideoModel) { - const id = row['VideoStreamingPlaylists.id'] - if (!id || this.videoStreamingPlaylistMemo[id]) return - - const attributes = this.grab(row, this.tables.getStreamingPlaylistAttributes(), 'VideoStreamingPlaylists') - const streamingPlaylist = new VideoStreamingPlaylistModel(attributes, this.buildOpts) - streamingPlaylist.VideoFiles = [] - - videoModel.VideoStreamingPlaylists.push(streamingPlaylist) - - this.videoStreamingPlaylistMemo[id] = streamingPlaylist - } - - private addStreamingPlaylistFile (row: SQLRow) { - const id = row['VideoStreamingPlaylists.VideoFiles.id'] - if (!id || this.videoFileMemo[id]) return - - const streamingPlaylist = this.videoStreamingPlaylistMemo[row['VideoStreamingPlaylists.id']] - - const attributes = this.grab(row, this.tables.getFileAttributes(), 'VideoStreamingPlaylists.VideoFiles') - const videoFileModel = new VideoFileModel(attributes, this.buildOpts) - streamingPlaylist.VideoFiles.push(videoFileModel) - - this.videoFileMemo[id] = videoFileModel - } - - private addRedundancy (row: SQLRow, prefix: string, to: VideoFileModel | VideoStreamingPlaylistModel) { - if (!to.RedundancyVideos) to.RedundancyVideos = [] - - const redundancyPrefix = `${prefix}.RedundancyVideos` - const id = row[`${redundancyPrefix}.id`] - - if (!id || this.redundancyDone.has(id)) return - - const attributes = this.grab(row, this.tables.getRedundancyAttributes(), redundancyPrefix) - const redundancyModel = new VideoRedundancyModel(attributes, this.buildOpts) - to.RedundancyVideos.push(redundancyModel) - - this.redundancyDone.add(id) - } - - private addTag (row: SQLRow, videoModel: VideoModel) { - if (!row['Tags.name']) return - - const key = `${row['Tags.VideoTagModel.videoId']}-${row['Tags.VideoTagModel.tagId']}` - if (this.tagsDone.has(key)) return - - const attributes = this.grab(row, this.tables.getTagAttributes(), 'Tags') - const tagModel = new TagModel(attributes, this.buildOpts) - videoModel.Tags.push(tagModel) - - this.tagsDone.add(key) - } - - private addTracker (row: SQLRow, videoModel: VideoModel) { - if (!row['Trackers.id']) return - - const key = `${row['Trackers.VideoTrackerModel.videoId']}-${row['Trackers.VideoTrackerModel.trackerId']}` - if (this.trackersDone.has(key)) return - - const attributes = this.grab(row, this.tables.getTrackerAttributes(), 'Trackers') - const trackerModel = new TrackerModel(attributes, this.buildOpts) - videoModel.Trackers.push(trackerModel) - - this.trackersDone.add(key) - } - - private setBlacklisted (row: SQLRow, videoModel: VideoModel) { - const id = row['VideoBlacklist.id'] - if (!id || this.blacklistDone.has(id)) return - - const attributes = this.grab(row, this.tables.getBlacklistedAttributes(), 'VideoBlacklist') - videoModel.VideoBlacklist = new VideoBlacklistModel(attributes, this.buildOpts) - - this.blacklistDone.add(id) - } - - private setBlockedOwner (row: SQLRow, videoModel: VideoModel) { - const id = row['VideoChannel.Account.AccountBlocklist.id'] - if (!id) return - - const key = `${videoModel.id}-${id}` - if (this.accountBlocklistDone.has(key)) return - - const attributes = this.grab(row, this.tables.getBlocklistAttributes(), 'VideoChannel.Account.AccountBlocklist') - videoModel.VideoChannel.Account.BlockedBy.push(new AccountBlocklistModel(attributes, this.buildOpts)) - - this.accountBlocklistDone.add(key) - } - - private setBlockedServer (row: SQLRow, videoModel: VideoModel) { - const id = row['VideoChannel.Account.Actor.Server.ServerBlocklist.id'] - if (!id || this.serverBlocklistDone.has(id)) return - - const key = `${videoModel.id}-${id}` - if (this.serverBlocklistDone.has(key)) return - - const attributes = this.grab(row, this.tables.getBlocklistAttributes(), 'VideoChannel.Account.Actor.Server.ServerBlocklist') - videoModel.VideoChannel.Account.Actor.Server.BlockedBy.push(new ServerBlocklistModel(attributes, this.buildOpts)) - - this.serverBlocklistDone.add(key) - } - - private setScheduleVideoUpdate (row: SQLRow, videoModel: VideoModel) { - const id = row['ScheduleVideoUpdate.id'] - if (!id || this.scheduleVideoUpdateDone.has(id)) return - - const attributes = this.grab(row, this.tables.getScheduleUpdateAttributes(), 'ScheduleVideoUpdate') - videoModel.ScheduleVideoUpdate = new ScheduleVideoUpdateModel(attributes, this.buildOpts) - - this.scheduleVideoUpdateDone.add(id) - } - - private setLive (row: SQLRow, videoModel: VideoModel) { - const id = row['VideoLive.id'] - if (!id || this.liveDone.has(id)) return - - const attributes = this.grab(row, this.tables.getLiveAttributes(), 'VideoLive') - videoModel.VideoLive = new VideoLiveModel(attributes, this.buildOpts) - - this.liveDone.add(id) - } - - private grab (row: SQLRow, attributes: string[], prefix: string) { - const result: { [ id: string ]: string | number } = {} - - for (const a of attributes) { - const key = prefix - ? prefix + '.' + a - : a - - result[a] = row[key] - } - - return result - } -} diff --git a/server/models/video/sql/video/video-model-get-query-builder.ts b/server/models/video/sql/video/video-model-get-query-builder.ts deleted file mode 100644 index 3f43d4d92..000000000 --- a/server/models/video/sql/video/video-model-get-query-builder.ts +++ /dev/null @@ -1,189 +0,0 @@ -import { Sequelize, Transaction } from 'sequelize' -import { pick } from '@shared/core-utils' -import { AbstractVideoQueryBuilder } from './shared/abstract-video-query-builder' -import { VideoFileQueryBuilder } from './shared/video-file-query-builder' -import { VideoModelBuilder } from './shared/video-model-builder' -import { VideoTableAttributes } from './shared/video-table-attributes' - -/** - * - * Build a GET SQL query, fetch rows and create the video model - * - */ - -export type GetType = - 'api' | - 'full' | - 'account-blacklist-files' | - 'all-files' | - 'thumbnails' | - 'thumbnails-blacklist' | - 'id' | - 'blacklist-rights' - -export type BuildVideoGetQueryOptions = { - id?: number | string - url?: string - - type: GetType - - userId?: number - transaction?: Transaction - - logging?: boolean -} - -export class VideoModelGetQueryBuilder { - videoQueryBuilder: VideosModelGetQuerySubBuilder - webVideoFilesQueryBuilder: VideoFileQueryBuilder - streamingPlaylistFilesQueryBuilder: VideoFileQueryBuilder - - private readonly videoModelBuilder: VideoModelBuilder - - private static readonly videoFilesInclude = new Set([ 'api', 'full', 'account-blacklist-files', 'all-files' ]) - - constructor (protected readonly sequelize: Sequelize) { - this.videoQueryBuilder = new VideosModelGetQuerySubBuilder(sequelize) - this.webVideoFilesQueryBuilder = new VideoFileQueryBuilder(sequelize) - this.streamingPlaylistFilesQueryBuilder = new VideoFileQueryBuilder(sequelize) - - this.videoModelBuilder = new VideoModelBuilder('get', new VideoTableAttributes('get')) - } - - async queryVideo (options: BuildVideoGetQueryOptions) { - const fileQueryOptions = { - ...pick(options, [ 'id', 'url', 'transaction', 'logging' ]), - - includeRedundancy: this.shouldIncludeRedundancies(options) - } - - const [ videoRows, webVideoFilesRows, streamingPlaylistFilesRows ] = await Promise.all([ - this.videoQueryBuilder.queryVideos(options), - - VideoModelGetQueryBuilder.videoFilesInclude.has(options.type) - ? this.webVideoFilesQueryBuilder.queryWebVideos(fileQueryOptions) - : Promise.resolve(undefined), - - VideoModelGetQueryBuilder.videoFilesInclude.has(options.type) - ? this.streamingPlaylistFilesQueryBuilder.queryStreamingPlaylistVideos(fileQueryOptions) - : Promise.resolve(undefined) - ]) - - const videos = this.videoModelBuilder.buildVideosFromRows({ - rows: videoRows, - rowsWebVideoFiles: webVideoFilesRows, - rowsStreamingPlaylist: streamingPlaylistFilesRows - }) - - if (videos.length > 1) { - throw new Error('Video results is more than 1') - } - - if (videos.length === 0) return null - - return videos[0] - } - - private shouldIncludeRedundancies (options: BuildVideoGetQueryOptions) { - return options.type === 'api' - } -} - -export class VideosModelGetQuerySubBuilder extends AbstractVideoQueryBuilder { - protected attributes: { [key: string]: string } - - protected webVideoFilesQuery: string - protected streamingPlaylistFilesQuery: string - - private static readonly trackersInclude = new Set([ 'api' ]) - private static readonly liveInclude = new Set([ 'api', 'full' ]) - private static readonly scheduleUpdateInclude = new Set([ 'api', 'full' ]) - private static readonly tagsInclude = new Set([ 'api', 'full' ]) - private static readonly userHistoryInclude = new Set([ 'api', 'full' ]) - private static readonly accountInclude = new Set([ 'api', 'full', 'account-blacklist-files' ]) - private static readonly ownerUserInclude = new Set([ 'blacklist-rights' ]) - - private static readonly blacklistedInclude = new Set([ - 'api', - 'full', - 'account-blacklist-files', - 'thumbnails-blacklist', - 'blacklist-rights' - ]) - - private static readonly thumbnailsInclude = new Set([ - 'api', - 'full', - 'account-blacklist-files', - 'all-files', - 'thumbnails', - 'thumbnails-blacklist' - ]) - - constructor (protected readonly sequelize: Sequelize) { - super(sequelize, 'get') - } - - queryVideos (options: BuildVideoGetQueryOptions) { - this.buildMainGetQuery(options) - - return this.runQuery(options) - } - - private buildMainGetQuery (options: BuildVideoGetQueryOptions) { - this.attributes = { - '"video".*': '' - } - - if (VideosModelGetQuerySubBuilder.thumbnailsInclude.has(options.type)) { - this.includeThumbnails() - } - - if (VideosModelGetQuerySubBuilder.blacklistedInclude.has(options.type)) { - this.includeBlacklisted() - } - - if (VideosModelGetQuerySubBuilder.accountInclude.has(options.type)) { - this.includeChannels() - this.includeAccounts() - } - - if (VideosModelGetQuerySubBuilder.tagsInclude.has(options.type)) { - this.includeTags() - } - - if (VideosModelGetQuerySubBuilder.scheduleUpdateInclude.has(options.type)) { - this.includeScheduleUpdate() - } - - if (VideosModelGetQuerySubBuilder.liveInclude.has(options.type)) { - this.includeLive() - } - - if (options.userId && VideosModelGetQuerySubBuilder.userHistoryInclude.has(options.type)) { - this.includeUserHistory(options.userId) - } - - if (VideosModelGetQuerySubBuilder.ownerUserInclude.has(options.type)) { - this.includeOwnerUser() - } - - if (VideosModelGetQuerySubBuilder.trackersInclude.has(options.type)) { - this.includeTrackers() - } - - this.whereId(options) - - this.query = this.buildQuery(options) - } - - private buildQuery (options: BuildVideoGetQueryOptions) { - const order = VideosModelGetQuerySubBuilder.tagsInclude.has(options.type) - ? 'ORDER BY "Tags"."name" ASC' - : '' - - const from = `SELECT * FROM "video" ${this.where} LIMIT 1` - - return `${this.buildSelect()} FROM (${from}) AS "video" ${this.joins} ${order}` - } -} diff --git a/server/models/video/sql/video/videos-id-list-query-builder.ts b/server/models/video/sql/video/videos-id-list-query-builder.ts deleted file mode 100644 index 7f2376102..000000000 --- a/server/models/video/sql/video/videos-id-list-query-builder.ts +++ /dev/null @@ -1,728 +0,0 @@ -import { Sequelize, Transaction } from 'sequelize' -import validator from 'validator' -import { exists } from '@server/helpers/custom-validators/misc' -import { WEBSERVER } from '@server/initializers/constants' -import { buildSortDirectionAndField } from '@server/models/shared' -import { MUserAccountId, MUserId } from '@server/types/models' -import { forceNumber } from '@shared/core-utils' -import { VideoInclude, VideoPrivacy, VideoState } from '@shared/models' -import { createSafeIn, parseRowCountResult } from '../../../shared' -import { AbstractRunQuery } from '../../../shared/abstract-run-query' - -/** - * - * Build videos list SQL query to fetch rows - * - */ - -export type DisplayOnlyForFollowerOptions = { - actorId: number - orLocalVideos: boolean -} - -export type BuildVideosListQueryOptions = { - attributes?: string[] - - serverAccountIdForBlock: number - - displayOnlyForFollower: DisplayOnlyForFollowerOptions - - count: number - start: number - sort: string - - nsfw?: boolean - host?: string - isLive?: boolean - isLocal?: boolean - include?: VideoInclude - - categoryOneOf?: number[] - licenceOneOf?: number[] - languageOneOf?: string[] - tagsOneOf?: string[] - tagsAllOf?: string[] - privacyOneOf?: VideoPrivacy[] - - uuids?: string[] - - hasFiles?: boolean - hasHLSFiles?: boolean - - hasWebVideoFiles?: boolean - hasWebtorrentFiles?: boolean // TODO: Remove in v7 - - accountId?: number - videoChannelId?: number - - videoPlaylistId?: number - - trendingAlgorithm?: string // best, hot, or any other algorithm implemented - trendingDays?: number - - user?: MUserAccountId - historyOfUser?: MUserId - - startDate?: string // ISO 8601 - endDate?: string // ISO 8601 - originallyPublishedStartDate?: string - originallyPublishedEndDate?: string - - durationMin?: number // seconds - durationMax?: number // seconds - - search?: string - - isCount?: boolean - - group?: string - having?: string - - transaction?: Transaction - logging?: boolean - - excludeAlreadyWatched?: boolean -} - -export class VideosIdListQueryBuilder extends AbstractRunQuery { - protected replacements: any = {} - - private attributes: string[] - private joins: string[] = [] - - private readonly and: string[] = [] - - private readonly cte: string[] = [] - - private group = '' - private having = '' - - private sort = '' - private limit = '' - private offset = '' - - constructor (protected readonly sequelize: Sequelize) { - super(sequelize) - } - - queryVideoIds (options: BuildVideosListQueryOptions) { - this.buildIdsListQuery(options) - - return this.runQuery() - } - - countVideoIds (countOptions: BuildVideosListQueryOptions): Promise { - this.buildIdsListQuery(countOptions) - - return this.runQuery().then(rows => parseRowCountResult(rows)) - } - - getQuery (options: BuildVideosListQueryOptions) { - this.buildIdsListQuery(options) - - return { query: this.query, sort: this.sort, replacements: this.replacements } - } - - private buildIdsListQuery (options: BuildVideosListQueryOptions) { - this.attributes = options.attributes || [ '"video"."id"' ] - - if (options.group) this.group = options.group - if (options.having) this.having = options.having - - this.joins = this.joins.concat([ - 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId"', - 'INNER JOIN "account" ON "account"."id" = "videoChannel"."accountId"', - 'INNER JOIN "actor" "accountActor" ON "account"."actorId" = "accountActor"."id"' - ]) - - if (!(options.include & VideoInclude.BLACKLISTED)) { - this.whereNotBlacklisted() - } - - if (options.serverAccountIdForBlock && !(options.include & VideoInclude.BLOCKED_OWNER)) { - this.whereNotBlocked(options.serverAccountIdForBlock, options.user) - } - - // Only list published videos - if (!(options.include & VideoInclude.NOT_PUBLISHED_STATE)) { - this.whereStateAvailable() - } - - if (options.videoPlaylistId) { - this.joinPlaylist(options.videoPlaylistId) - } - - if (exists(options.isLocal)) { - this.whereLocal(options.isLocal) - } - - if (options.host) { - this.whereHost(options.host) - } - - if (options.accountId) { - this.whereAccountId(options.accountId) - } - - if (options.videoChannelId) { - this.whereChannelId(options.videoChannelId) - } - - if (options.displayOnlyForFollower) { - this.whereFollowerActorId(options.displayOnlyForFollower) - } - - if (options.hasFiles === true) { - this.whereFileExists() - } - - if (exists(options.hasWebtorrentFiles)) { - this.whereWebVideoFileExists(options.hasWebtorrentFiles) - } else if (exists(options.hasWebVideoFiles)) { - this.whereWebVideoFileExists(options.hasWebVideoFiles) - } - - if (exists(options.hasHLSFiles)) { - this.whereHLSFileExists(options.hasHLSFiles) - } - - if (options.tagsOneOf) { - this.whereTagsOneOf(options.tagsOneOf) - } - - if (options.tagsAllOf) { - this.whereTagsAllOf(options.tagsAllOf) - } - - if (options.privacyOneOf) { - this.wherePrivacyOneOf(options.privacyOneOf) - } else { - // Only list videos with the appropriate privacy - this.wherePrivacyAvailable(options.user) - } - - if (options.uuids) { - this.whereUUIDs(options.uuids) - } - - if (options.nsfw === true) { - this.whereNSFW() - } else if (options.nsfw === false) { - this.whereSFW() - } - - if (options.isLive === true) { - this.whereLive() - } else if (options.isLive === false) { - this.whereVOD() - } - - if (options.categoryOneOf) { - this.whereCategoryOneOf(options.categoryOneOf) - } - - if (options.licenceOneOf) { - this.whereLicenceOneOf(options.licenceOneOf) - } - - if (options.languageOneOf) { - this.whereLanguageOneOf(options.languageOneOf) - } - - // We don't exclude results in this so if we do a count we don't need to add this complex clause - if (options.isCount !== true) { - if (options.trendingDays) { - this.groupForTrending(options.trendingDays) - } else if ([ 'best', 'hot' ].includes(options.trendingAlgorithm)) { - this.groupForHotOrBest(options.trendingAlgorithm, options.user) - } - } - - if (options.historyOfUser) { - this.joinHistory(options.historyOfUser.id) - } - - if (options.startDate) { - this.whereStartDate(options.startDate) - } - - if (options.endDate) { - this.whereEndDate(options.endDate) - } - - if (options.originallyPublishedStartDate) { - this.whereOriginallyPublishedStartDate(options.originallyPublishedStartDate) - } - - if (options.originallyPublishedEndDate) { - this.whereOriginallyPublishedEndDate(options.originallyPublishedEndDate) - } - - if (options.durationMin) { - this.whereDurationMin(options.durationMin) - } - - if (options.durationMax) { - this.whereDurationMax(options.durationMax) - } - - if (options.excludeAlreadyWatched) { - if (exists(options.user.id)) { - this.whereExcludeAlreadyWatched(options.user.id) - } else { - throw new Error('Cannot use excludeAlreadyWatched parameter when auth token is not provided') - } - } - - this.whereSearch(options.search) - - if (options.isCount === true) { - this.setCountAttribute() - } else { - if (exists(options.sort)) { - this.setSort(options.sort) - } - - if (exists(options.count)) { - this.setLimit(options.count) - } - - if (exists(options.start)) { - this.setOffset(options.start) - } - } - - const cteString = this.cte.length !== 0 - ? `WITH ${this.cte.join(', ')} ` - : '' - - this.query = cteString + - 'SELECT ' + this.attributes.join(', ') + ' ' + - 'FROM "video" ' + this.joins.join(' ') + ' ' + - 'WHERE ' + this.and.join(' AND ') + ' ' + - this.group + ' ' + - this.having + ' ' + - this.sort + ' ' + - this.limit + ' ' + - this.offset - } - - private setCountAttribute () { - this.attributes = [ 'COUNT(*) as "total"' ] - } - - private joinHistory (userId: number) { - this.joins.push('INNER JOIN "userVideoHistory" ON "video"."id" = "userVideoHistory"."videoId"') - - this.and.push('"userVideoHistory"."userId" = :historyOfUser') - - this.replacements.historyOfUser = userId - } - - private joinPlaylist (playlistId: number) { - this.joins.push( - 'INNER JOIN "videoPlaylistElement" "video"."id" = "videoPlaylistElement"."videoId" ' + - 'AND "videoPlaylistElement"."videoPlaylistId" = :videoPlaylistId' - ) - - this.replacements.videoPlaylistId = playlistId - } - - private whereStateAvailable () { - this.and.push( - `("video"."state" = ${VideoState.PUBLISHED} OR ` + - `("video"."state" = ${VideoState.TO_TRANSCODE} AND "video"."waitTranscoding" IS false))` - ) - } - - private wherePrivacyAvailable (user?: MUserAccountId) { - if (user) { - this.and.push( - `("video"."privacy" = ${VideoPrivacy.PUBLIC} OR "video"."privacy" = ${VideoPrivacy.INTERNAL})` - ) - } else { // Or only public videos - this.and.push( - `"video"."privacy" = ${VideoPrivacy.PUBLIC}` - ) - } - } - - private whereLocal (isLocal: boolean) { - const isRemote = isLocal ? 'FALSE' : 'TRUE' - - this.and.push('"video"."remote" IS ' + isRemote) - } - - private whereHost (host: string) { - // Local instance - if (host === WEBSERVER.HOST) { - this.and.push('"accountActor"."serverId" IS NULL') - return - } - - this.joins.push('INNER JOIN "server" ON "server"."id" = "accountActor"."serverId"') - - this.and.push('"server"."host" = :host') - this.replacements.host = host - } - - private whereAccountId (accountId: number) { - this.and.push('"account"."id" = :accountId') - this.replacements.accountId = accountId - } - - private whereChannelId (channelId: number) { - this.and.push('"videoChannel"."id" = :videoChannelId') - this.replacements.videoChannelId = channelId - } - - private whereFollowerActorId (options: { actorId: number, orLocalVideos: boolean }) { - let query = - '(' + - ' EXISTS (' + // Videos shared by actors we follow - ' SELECT 1 FROM "videoShare" ' + - ' INNER JOIN "actorFollow" "actorFollowShare" ON "actorFollowShare"."targetActorId" = "videoShare"."actorId" ' + - ' AND "actorFollowShare"."actorId" = :followerActorId AND "actorFollowShare"."state" = \'accepted\' ' + - ' WHERE "videoShare"."videoId" = "video"."id"' + - ' )' + - ' OR' + - ' EXISTS (' + // Videos published by channels or accounts we follow - ' SELECT 1 from "actorFollow" ' + - ' WHERE ("actorFollow"."targetActorId" = "account"."actorId" OR "actorFollow"."targetActorId" = "videoChannel"."actorId") ' + - ' AND "actorFollow"."actorId" = :followerActorId ' + - ' AND "actorFollow"."state" = \'accepted\'' + - ' )' - - if (options.orLocalVideos) { - query += ' OR "video"."remote" IS FALSE' - } - - query += ')' - - this.and.push(query) - this.replacements.followerActorId = options.actorId - } - - private whereFileExists () { - this.and.push(`(${this.buildWebVideoFileExistsQuery(true)} OR ${this.buildHLSFileExistsQuery(true)})`) - } - - private whereWebVideoFileExists (exists: boolean) { - this.and.push(this.buildWebVideoFileExistsQuery(exists)) - } - - private whereHLSFileExists (exists: boolean) { - this.and.push(this.buildHLSFileExistsQuery(exists)) - } - - private buildWebVideoFileExistsQuery (exists: boolean) { - const prefix = exists ? '' : 'NOT ' - - return prefix + 'EXISTS (SELECT 1 FROM "videoFile" WHERE "videoFile"."videoId" = "video"."id")' - } - - private buildHLSFileExistsQuery (exists: boolean) { - const prefix = exists ? '' : 'NOT ' - - return prefix + 'EXISTS (' + - ' SELECT 1 FROM "videoStreamingPlaylist" ' + - ' INNER JOIN "videoFile" ON "videoFile"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ' + - ' WHERE "videoStreamingPlaylist"."videoId" = "video"."id"' + - ')' - } - - private whereTagsOneOf (tagsOneOf: string[]) { - const tagsOneOfLower = tagsOneOf.map(t => t.toLowerCase()) - - this.and.push( - 'EXISTS (' + - ' SELECT 1 FROM "videoTag" ' + - ' INNER JOIN "tag" ON "tag"."id" = "videoTag"."tagId" ' + - ' WHERE lower("tag"."name") IN (' + createSafeIn(this.sequelize, tagsOneOfLower) + ') ' + - ' AND "video"."id" = "videoTag"."videoId"' + - ')' - ) - } - - private whereTagsAllOf (tagsAllOf: string[]) { - const tagsAllOfLower = tagsAllOf.map(t => t.toLowerCase()) - - this.and.push( - 'EXISTS (' + - ' SELECT 1 FROM "videoTag" ' + - ' INNER JOIN "tag" ON "tag"."id" = "videoTag"."tagId" ' + - ' WHERE lower("tag"."name") IN (' + createSafeIn(this.sequelize, tagsAllOfLower) + ') ' + - ' AND "video"."id" = "videoTag"."videoId" ' + - ' GROUP BY "videoTag"."videoId" HAVING COUNT(*) = ' + tagsAllOfLower.length + - ')' - ) - } - - private wherePrivacyOneOf (privacyOneOf: VideoPrivacy[]) { - this.and.push('"video"."privacy" IN (:privacyOneOf)') - this.replacements.privacyOneOf = privacyOneOf - } - - private whereUUIDs (uuids: string[]) { - this.and.push('"video"."uuid" IN (' + createSafeIn(this.sequelize, uuids) + ')') - } - - private whereCategoryOneOf (categoryOneOf: number[]) { - this.and.push('"video"."category" IN (:categoryOneOf)') - this.replacements.categoryOneOf = categoryOneOf - } - - private whereLicenceOneOf (licenceOneOf: number[]) { - this.and.push('"video"."licence" IN (:licenceOneOf)') - this.replacements.licenceOneOf = licenceOneOf - } - - private whereLanguageOneOf (languageOneOf: string[]) { - const languages = languageOneOf.filter(l => l && l !== '_unknown') - const languagesQueryParts: string[] = [] - - if (languages.length !== 0) { - languagesQueryParts.push('"video"."language" IN (:languageOneOf)') - this.replacements.languageOneOf = languages - - languagesQueryParts.push( - 'EXISTS (' + - ' SELECT 1 FROM "videoCaption" WHERE "videoCaption"."language" ' + - ' IN (' + createSafeIn(this.sequelize, languages) + ') AND ' + - ' "videoCaption"."videoId" = "video"."id"' + - ')' - ) - } - - if (languageOneOf.includes('_unknown')) { - languagesQueryParts.push('"video"."language" IS NULL') - } - - if (languagesQueryParts.length !== 0) { - this.and.push('(' + languagesQueryParts.join(' OR ') + ')') - } - } - - private whereNSFW () { - this.and.push('"video"."nsfw" IS TRUE') - } - - private whereSFW () { - this.and.push('"video"."nsfw" IS FALSE') - } - - private whereLive () { - this.and.push('"video"."isLive" IS TRUE') - } - - private whereVOD () { - this.and.push('"video"."isLive" IS FALSE') - } - - private whereNotBlocked (serverAccountId: number, user?: MUserAccountId) { - const blockerIds = [ serverAccountId ] - if (user) blockerIds.push(user.Account.id) - - const inClause = createSafeIn(this.sequelize, blockerIds) - - this.and.push( - 'NOT EXISTS (' + - ' SELECT 1 FROM "accountBlocklist" ' + - ' WHERE "accountBlocklist"."accountId" IN (' + inClause + ') ' + - ' AND "accountBlocklist"."targetAccountId" = "account"."id" ' + - ')' + - 'AND NOT EXISTS (' + - ' SELECT 1 FROM "serverBlocklist" WHERE "serverBlocklist"."accountId" IN (' + inClause + ') ' + - ' AND "serverBlocklist"."targetServerId" = "accountActor"."serverId"' + - ')' - ) - } - - private whereSearch (search?: string) { - if (!search) { - this.attributes.push('0 as similarity') - return - } - - const escapedSearch = this.sequelize.escape(search) - const escapedLikeSearch = this.sequelize.escape('%' + search + '%') - - this.cte.push( - '"trigramSearch" AS (' + - ' SELECT "video"."id", ' + - ` similarity(lower(immutable_unaccent("video"."name")), lower(immutable_unaccent(${escapedSearch}))) as similarity ` + - ' FROM "video" ' + - ' WHERE lower(immutable_unaccent("video"."name")) % lower(immutable_unaccent(' + escapedSearch + ')) OR ' + - ' lower(immutable_unaccent("video"."name")) LIKE lower(immutable_unaccent(' + escapedLikeSearch + '))' + - ')' - ) - - this.joins.push('LEFT JOIN "trigramSearch" ON "video"."id" = "trigramSearch"."id"') - - let base = '(' + - ' "trigramSearch"."id" IS NOT NULL OR ' + - ' EXISTS (' + - ' SELECT 1 FROM "videoTag" ' + - ' INNER JOIN "tag" ON "tag"."id" = "videoTag"."tagId" ' + - ` WHERE lower("tag"."name") = lower(${escapedSearch}) ` + - ' AND "video"."id" = "videoTag"."videoId"' + - ' )' - - if (validator.isUUID(search)) { - base += ` OR "video"."uuid" = ${escapedSearch}` - } - - base += ')' - - this.and.push(base) - this.attributes.push(`COALESCE("trigramSearch"."similarity", 0) as similarity`) - } - - private whereNotBlacklisted () { - this.and.push('"video"."id" NOT IN (SELECT "videoBlacklist"."videoId" FROM "videoBlacklist")') - } - - private whereStartDate (startDate: string) { - this.and.push('"video"."publishedAt" >= :startDate') - this.replacements.startDate = startDate - } - - private whereEndDate (endDate: string) { - this.and.push('"video"."publishedAt" <= :endDate') - this.replacements.endDate = endDate - } - - private whereOriginallyPublishedStartDate (startDate: string) { - this.and.push('"video"."originallyPublishedAt" >= :originallyPublishedStartDate') - this.replacements.originallyPublishedStartDate = startDate - } - - private whereOriginallyPublishedEndDate (endDate: string) { - this.and.push('"video"."originallyPublishedAt" <= :originallyPublishedEndDate') - this.replacements.originallyPublishedEndDate = endDate - } - - private whereDurationMin (durationMin: number) { - this.and.push('"video"."duration" >= :durationMin') - this.replacements.durationMin = durationMin - } - - private whereDurationMax (durationMax: number) { - this.and.push('"video"."duration" <= :durationMax') - this.replacements.durationMax = durationMax - } - - private whereExcludeAlreadyWatched (userId: number) { - this.and.push( - 'NOT EXISTS (' + - ' SELECT 1' + - ' FROM "userVideoHistory"' + - ' WHERE "video"."id" = "userVideoHistory"."videoId"' + - ' AND "userVideoHistory"."userId" = :excludeAlreadyWatchedUserId' + - ')' - ) - this.replacements.excludeAlreadyWatchedUserId = userId - } - - private groupForTrending (trendingDays: number) { - const viewsGteDate = new Date(new Date().getTime() - (24 * 3600 * 1000) * trendingDays) - - this.joins.push('LEFT JOIN "videoView" ON "video"."id" = "videoView"."videoId" AND "videoView"."startDate" >= :viewsGteDate') - this.replacements.viewsGteDate = viewsGteDate - - this.attributes.push('COALESCE(SUM("videoView"."views"), 0) AS "score"') - - this.group = 'GROUP BY "video"."id"' - } - - private groupForHotOrBest (trendingAlgorithm: string, user?: MUserAccountId) { - /** - * "Hotness" is a measure based on absolute view/comment/like/dislike numbers, - * with fixed weights only applied to their log values. - * - * This algorithm gives little chance for an old video to have a good score, - * for which recent spikes in interactions could be a sign of "hotness" and - * justify a better score. However there are multiple ways to achieve that - * goal, which is left for later. Yes, this is a TODO :) - * - * notes: - * - weights and base score are in number of half-days. - * - all comments are counted, regardless of being written by the video author or not - * see https://github.com/reddit-archive/reddit/blob/master/r2/r2/lib/db/_sorts.pyx#L47-L58 - * - we have less interactions than on reddit, so multiply weights by an arbitrary factor - */ - const weights = { - like: 3 * 50, - dislike: -3 * 50, - view: Math.floor((1 / 3) * 50), - comment: 2 * 50, // a comment takes more time than a like to do, but can be done multiple times - history: -2 * 50 - } - - this.joins.push('LEFT JOIN "videoComment" ON "video"."id" = "videoComment"."videoId"') - - let attribute = - `LOG(GREATEST(1, "video"."likes" - 1)) * ${weights.like} ` + // likes (+) - `+ LOG(GREATEST(1, "video"."dislikes" - 1)) * ${weights.dislike} ` + // dislikes (-) - `+ LOG("video"."views" + 1) * ${weights.view} ` + // views (+) - `+ LOG(GREATEST(1, COUNT(DISTINCT "videoComment"."id"))) * ${weights.comment} ` + // comments (+) - '+ (SELECT (EXTRACT(epoch FROM "video"."publishedAt") - 1446156582) / 47000) ' // base score (in number of half-days) - - if (trendingAlgorithm === 'best' && user) { - this.joins.push( - 'LEFT JOIN "userVideoHistory" ON "video"."id" = "userVideoHistory"."videoId" AND "userVideoHistory"."userId" = :bestUser' - ) - this.replacements.bestUser = user.id - - attribute += `+ POWER(COUNT(DISTINCT "userVideoHistory"."id"), 2.0) * ${weights.history} ` - } - - attribute += 'AS "score"' - this.attributes.push(attribute) - - this.group = 'GROUP BY "video"."id"' - } - - private setSort (sort: string) { - if (sort === '-originallyPublishedAt' || sort === 'originallyPublishedAt') { - this.attributes.push('COALESCE("video"."originallyPublishedAt", "video"."publishedAt") AS "publishedAtForOrder"') - } - - this.sort = this.buildOrder(sort) - } - - private buildOrder (value: string) { - const { direction, field } = buildSortDirectionAndField(value) - if (field.match(/^[a-zA-Z."]+$/) === null) throw new Error('Invalid sort column ' + field) - - if (field.toLowerCase() === 'random') return 'ORDER BY RANDOM()' - - if ([ 'trending', 'hot', 'best' ].includes(field.toLowerCase())) { // Sort by aggregation - return `ORDER BY "score" ${direction}, "video"."views" ${direction}` - } - - let firstSort: string - - if (field.toLowerCase() === 'match') { // Search - firstSort = '"similarity"' - } else if (field === 'originallyPublishedAt') { - firstSort = '"publishedAtForOrder"' - } else if (field.includes('.')) { - firstSort = field - } else { - firstSort = `"video"."${field}"` - } - - return `ORDER BY ${firstSort} ${direction}, "video"."id" ASC` - } - - private setLimit (countArg: number) { - const count = forceNumber(countArg) - this.limit = `LIMIT ${count}` - } - - private setOffset (startArg: number) { - const start = forceNumber(startArg) - this.offset = `OFFSET ${start}` - } -} diff --git a/server/models/video/sql/video/videos-model-list-query-builder.ts b/server/models/video/sql/video/videos-model-list-query-builder.ts deleted file mode 100644 index b73dc28cd..000000000 --- a/server/models/video/sql/video/videos-model-list-query-builder.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { Sequelize } from 'sequelize' -import { pick } from '@shared/core-utils' -import { VideoInclude } from '@shared/models' -import { AbstractVideoQueryBuilder } from './shared/abstract-video-query-builder' -import { VideoFileQueryBuilder } from './shared/video-file-query-builder' -import { VideoModelBuilder } from './shared/video-model-builder' -import { BuildVideosListQueryOptions, VideosIdListQueryBuilder } from './videos-id-list-query-builder' - -/** - * - * Build videos list SQL query and create video models - * - */ - -export class VideosModelListQueryBuilder extends AbstractVideoQueryBuilder { - protected attributes: { [key: string]: string } - - private innerQuery: string - private innerSort: string - - webVideoFilesQueryBuilder: VideoFileQueryBuilder - streamingPlaylistFilesQueryBuilder: VideoFileQueryBuilder - - private readonly videoModelBuilder: VideoModelBuilder - - constructor (protected readonly sequelize: Sequelize) { - super(sequelize, 'list') - - this.videoModelBuilder = new VideoModelBuilder(this.mode, this.tables) - this.webVideoFilesQueryBuilder = new VideoFileQueryBuilder(sequelize) - this.streamingPlaylistFilesQueryBuilder = new VideoFileQueryBuilder(sequelize) - } - - async queryVideos (options: BuildVideosListQueryOptions) { - this.buildInnerQuery(options) - this.buildMainQuery(options) - - const rows = await this.runQuery() - - if (options.include & VideoInclude.FILES) { - const videoIds = Array.from(new Set(rows.map(r => r.id))) - - if (videoIds.length !== 0) { - const fileQueryOptions = { - ...pick(options, [ 'transaction', 'logging' ]), - - ids: videoIds, - includeRedundancy: false - } - - const [ rowsWebVideoFiles, rowsStreamingPlaylist ] = await Promise.all([ - this.webVideoFilesQueryBuilder.queryWebVideos(fileQueryOptions), - this.streamingPlaylistFilesQueryBuilder.queryStreamingPlaylistVideos(fileQueryOptions) - ]) - - return this.videoModelBuilder.buildVideosFromRows({ rows, include: options.include, rowsStreamingPlaylist, rowsWebVideoFiles }) - } - } - - return this.videoModelBuilder.buildVideosFromRows({ rows, include: options.include }) - } - - private buildInnerQuery (options: BuildVideosListQueryOptions) { - const idsQueryBuilder = new VideosIdListQueryBuilder(this.sequelize) - const { query, sort, replacements } = idsQueryBuilder.getQuery(options) - - this.replacements = replacements - this.innerQuery = query - this.innerSort = sort - } - - private buildMainQuery (options: BuildVideosListQueryOptions) { - this.attributes = { - '"video".*': '' - } - - this.addJoin('INNER JOIN "video" ON "tmp"."id" = "video"."id"') - - this.includeChannels() - this.includeAccounts() - this.includeThumbnails() - - if (options.user) { - this.includeUserHistory(options.user.id) - } - - if (options.videoPlaylistId) { - this.includePlaylist(options.videoPlaylistId) - } - - if (options.include & VideoInclude.BLACKLISTED) { - this.includeBlacklisted() - } - - if (options.include & VideoInclude.BLOCKED_OWNER) { - this.includeBlockedOwnerAndServer(options.serverAccountIdForBlock, options.user) - } - - const select = this.buildSelect() - - this.query = `${select} FROM (${this.innerQuery}) AS "tmp" ${this.joins} ${this.innerSort}` - } -} diff --git a/server/models/video/storyboard.ts b/server/models/video/storyboard.ts deleted file mode 100644 index 1c3c6d850..000000000 --- a/server/models/video/storyboard.ts +++ /dev/null @@ -1,169 +0,0 @@ -import { remove } from 'fs-extra' -import { join } from 'path' -import { AfterDestroy, AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { CONFIG } from '@server/initializers/config' -import { MStoryboard, MStoryboardVideo, MVideo } from '@server/types/models' -import { Storyboard } from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { logger } from '../../helpers/logger' -import { CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, WEBSERVER } from '../../initializers/constants' -import { VideoModel } from './video' -import { Transaction } from 'sequelize' - -@Table({ - tableName: 'storyboard', - indexes: [ - { - fields: [ 'videoId' ], - unique: true - }, - { - fields: [ 'filename' ], - unique: true - } - ] -}) -export class StoryboardModel extends Model>> { - - @AllowNull(false) - @Column - filename: string - - @AllowNull(false) - @Column - totalHeight: number - - @AllowNull(false) - @Column - totalWidth: number - - @AllowNull(false) - @Column - spriteHeight: number - - @AllowNull(false) - @Column - spriteWidth: number - - @AllowNull(false) - @Column - spriteDuration: number - - @AllowNull(true) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.COMMONS.URL.max)) - fileUrl: string - - @ForeignKey(() => VideoModel) - @Column - videoId: number - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'CASCADE' - }) - Video: VideoModel - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @AfterDestroy - static removeInstanceFile (instance: StoryboardModel) { - logger.info('Removing storyboard file %s.', instance.filename) - - // Don't block the transaction - instance.removeFile() - .catch(err => logger.error('Cannot remove storyboard file %s.', instance.filename, { err })) - } - - static loadByVideo (videoId: number, transaction?: Transaction): Promise { - const query = { - where: { - videoId - }, - transaction - } - - return StoryboardModel.findOne(query) - } - - static loadByFilename (filename: string): Promise { - const query = { - where: { - filename - } - } - - return StoryboardModel.findOne(query) - } - - static loadWithVideoByFilename (filename: string): Promise { - const query = { - where: { - filename - }, - include: [ - { - model: VideoModel.unscoped(), - required: true - } - ] - } - - return StoryboardModel.findOne(query) - } - - // --------------------------------------------------------------------------- - - static async listStoryboardsOf (video: MVideo): Promise { - const query = { - where: { - videoId: video.id - } - } - - const storyboards = await StoryboardModel.findAll(query) - - return storyboards.map(s => Object.assign(s, { Video: video })) - } - - // --------------------------------------------------------------------------- - - getOriginFileUrl (video: MVideo) { - if (video.isOwned()) { - return WEBSERVER.URL + this.getLocalStaticPath() - } - - return this.fileUrl - } - - getLocalStaticPath () { - return LAZY_STATIC_PATHS.STORYBOARDS + this.filename - } - - getPath () { - return join(CONFIG.STORAGE.STORYBOARDS_DIR, this.filename) - } - - removeFile () { - return remove(this.getPath()) - } - - toFormattedJSON (this: MStoryboardVideo): Storyboard { - return { - storyboardPath: this.getLocalStaticPath(), - - totalHeight: this.totalHeight, - totalWidth: this.totalWidth, - - spriteWidth: this.spriteWidth, - spriteHeight: this.spriteHeight, - - spriteDuration: this.spriteDuration - } - } -} diff --git a/server/models/video/tag.ts b/server/models/video/tag.ts deleted file mode 100644 index cebde3755..000000000 --- a/server/models/video/tag.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { col, fn, QueryTypes, Transaction } from 'sequelize' -import { AllowNull, BelongsToMany, Column, CreatedAt, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { MTag } from '@server/types/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { VideoPrivacy, VideoState } from '../../../shared/models/videos' -import { isVideoTagValid } from '../../helpers/custom-validators/videos' -import { throwIfNotValid } from '../shared' -import { VideoModel } from './video' -import { VideoTagModel } from './video-tag' - -@Table({ - tableName: 'tag', - timestamps: false, - indexes: [ - { - fields: [ 'name' ], - unique: true - }, - { - name: 'tag_lower_name', - fields: [ fn('lower', col('name')) ] - } - ] -}) -export class TagModel extends Model>> { - - @AllowNull(false) - @Is('VideoTag', value => throwIfNotValid(value, isVideoTagValid, 'tag')) - @Column - name: string - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @BelongsToMany(() => VideoModel, { - foreignKey: 'tagId', - through: () => VideoTagModel, - onDelete: 'CASCADE' - }) - Videos: VideoModel[] - - static findOrCreateTags (tags: string[], transaction: Transaction): Promise { - if (tags === null) return Promise.resolve([]) - - const uniqueTags = new Set(tags) - - const tasks = Array.from(uniqueTags).map(tag => { - const query = { - where: { - name: tag - }, - defaults: { - name: tag - }, - transaction - } - - return TagModel.findOrCreate(query) - .then(([ tagInstance ]) => tagInstance) - }) - - return Promise.all(tasks) - } - - // threshold corresponds to how many video the field should have to be returned - static getRandomSamples (threshold: number, count: number): Promise { - const query = 'SELECT tag.name FROM tag ' + - 'INNER JOIN "videoTag" ON "videoTag"."tagId" = tag.id ' + - 'INNER JOIN video ON video.id = "videoTag"."videoId" ' + - 'WHERE video.privacy = $videoPrivacy AND video.state = $videoState ' + - 'GROUP BY tag.name HAVING COUNT(tag.name) >= $threshold ' + - 'ORDER BY random() ' + - 'LIMIT $count' - - const options = { - bind: { threshold, count, videoPrivacy: VideoPrivacy.PUBLIC, videoState: VideoState.PUBLISHED }, - type: QueryTypes.SELECT as QueryTypes.SELECT - } - - return TagModel.sequelize.query<{ name: string }>(query, options) - .then(data => data.map(d => d.name)) - } -} diff --git a/server/models/video/thumbnail.ts b/server/models/video/thumbnail.ts deleted file mode 100644 index 1722acdb4..000000000 --- a/server/models/video/thumbnail.ts +++ /dev/null @@ -1,208 +0,0 @@ -import { remove } from 'fs-extra' -import { join } from 'path' -import { - AfterDestroy, - AllowNull, - BeforeCreate, - BeforeUpdate, - BelongsTo, - Column, - CreatedAt, - DataType, - Default, - ForeignKey, - Model, - Table, - UpdatedAt -} from 'sequelize-typescript' -import { afterCommitIfTransaction } from '@server/helpers/database-utils' -import { MThumbnail, MThumbnailVideo, MVideo } from '@server/types/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, WEBSERVER } from '../../initializers/constants' -import { VideoModel } from './video' -import { VideoPlaylistModel } from './video-playlist' - -@Table({ - tableName: 'thumbnail', - indexes: [ - { - fields: [ 'videoId' ] - }, - { - fields: [ 'videoPlaylistId' ], - unique: true - }, - { - fields: [ 'filename', 'type' ], - unique: true - } - ] -}) -export class ThumbnailModel extends Model>> { - - @AllowNull(false) - @Column - filename: string - - @AllowNull(true) - @Default(null) - @Column - height: number - - @AllowNull(true) - @Default(null) - @Column - width: number - - @AllowNull(false) - @Column - type: ThumbnailType - - @AllowNull(true) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.COMMONS.URL.max)) - fileUrl: string - - @AllowNull(true) - @Column - automaticallyGenerated: boolean - - @AllowNull(false) - @Column - onDisk: boolean - - @ForeignKey(() => VideoModel) - @Column - videoId: number - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'CASCADE' - }) - Video: VideoModel - - @ForeignKey(() => VideoPlaylistModel) - @Column - videoPlaylistId: number - - @BelongsTo(() => VideoPlaylistModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'CASCADE' - }) - VideoPlaylist: VideoPlaylistModel - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - // If this thumbnail replaced existing one, track the old name - previousThumbnailFilename: string - - private static readonly types: { [ id in ThumbnailType ]: { label: string, directory: string, staticPath: string } } = { - [ThumbnailType.MINIATURE]: { - label: 'miniature', - directory: CONFIG.STORAGE.THUMBNAILS_DIR, - staticPath: LAZY_STATIC_PATHS.THUMBNAILS - }, - [ThumbnailType.PREVIEW]: { - label: 'preview', - directory: CONFIG.STORAGE.PREVIEWS_DIR, - staticPath: LAZY_STATIC_PATHS.PREVIEWS - } - } - - @BeforeCreate - @BeforeUpdate - static removeOldFile (instance: ThumbnailModel, options) { - return afterCommitIfTransaction(options.transaction, () => instance.removePreviousFilenameIfNeeded()) - } - - @AfterDestroy - static removeFiles (instance: ThumbnailModel) { - logger.info('Removing %s file %s.', ThumbnailModel.types[instance.type].label, instance.filename) - - // Don't block the transaction - instance.removeThumbnail() - .catch(err => logger.error('Cannot remove thumbnail file %s.', instance.filename, { err })) - } - - static loadByFilename (filename: string, thumbnailType: ThumbnailType): Promise { - const query = { - where: { - filename, - type: thumbnailType - } - } - - return ThumbnailModel.findOne(query) - } - - static loadWithVideoByFilename (filename: string, thumbnailType: ThumbnailType): Promise { - const query = { - where: { - filename, - type: thumbnailType - }, - include: [ - { - model: VideoModel.unscoped(), - required: true - } - ] - } - - return ThumbnailModel.findOne(query) - } - - static buildPath (type: ThumbnailType, filename: string) { - const directory = ThumbnailModel.types[type].directory - - return join(directory, filename) - } - - getOriginFileUrl (video: MVideo) { - const staticPath = ThumbnailModel.types[this.type].staticPath + this.filename - - if (video.isOwned()) return WEBSERVER.URL + staticPath - - return this.fileUrl - } - - getLocalStaticPath () { - return ThumbnailModel.types[this.type].staticPath + this.filename - } - - getPath () { - return ThumbnailModel.buildPath(this.type, this.filename) - } - - getPreviousPath () { - return ThumbnailModel.buildPath(this.type, this.previousThumbnailFilename) - } - - removeThumbnail () { - return remove(this.getPath()) - } - - removePreviousFilenameIfNeeded () { - if (!this.previousThumbnailFilename) return - - const previousPath = this.getPreviousPath() - remove(previousPath) - .catch(err => logger.error('Cannot remove previous thumbnail file %s.', previousPath, { err })) - - this.previousThumbnailFilename = undefined - } - - isOwned () { - return !this.fileUrl - } -} diff --git a/server/models/video/video-blacklist.ts b/server/models/video/video-blacklist.ts deleted file mode 100644 index 9247d0e2b..000000000 --- a/server/models/video/video-blacklist.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { FindOptions } from 'sequelize' -import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { MVideoBlacklist, MVideoBlacklistFormattable } from '@server/types/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { VideoBlacklist, VideoBlacklistType } from '../../../shared/models/videos' -import { isVideoBlacklistReasonValid, isVideoBlacklistTypeValid } from '../../helpers/custom-validators/video-blacklist' -import { CONSTRAINTS_FIELDS } from '../../initializers/constants' -import { getBlacklistSort, searchAttribute, throwIfNotValid } from '../shared' -import { ThumbnailModel } from './thumbnail' -import { VideoModel } from './video' -import { ScopeNames as VideoChannelScopeNames, SummaryOptions, VideoChannelModel } from './video-channel' - -@Table({ - tableName: 'videoBlacklist', - indexes: [ - { - fields: [ 'videoId' ], - unique: true - } - ] -}) -export class VideoBlacklistModel extends Model>> { - - @AllowNull(true) - @Is('VideoBlacklistReason', value => throwIfNotValid(value, isVideoBlacklistReasonValid, 'reason', true)) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_BLACKLIST.REASON.max)) - reason: string - - @AllowNull(false) - @Column - unfederated: boolean - - @AllowNull(false) - @Default(null) - @Is('VideoBlacklistType', value => throwIfNotValid(value, isVideoBlacklistTypeValid, 'type')) - @Column - type: VideoBlacklistType - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => VideoModel) - @Column - videoId: number - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade' - }) - Video: VideoModel - - static listForApi (parameters: { - start: number - count: number - sort: string - search?: string - type?: VideoBlacklistType - }) { - const { start, count, sort, search, type } = parameters - - function buildBaseQuery (): FindOptions { - return { - offset: start, - limit: count, - order: getBlacklistSort(sort) - } - } - - const countQuery = buildBaseQuery() - - const findQuery = buildBaseQuery() - findQuery.include = [ - { - model: VideoModel, - required: true, - where: searchAttribute(search, 'name'), - include: [ - { - model: VideoChannelModel.scope({ method: [ VideoChannelScopeNames.SUMMARY, { withAccount: true } as SummaryOptions ] }), - required: true - }, - { - model: ThumbnailModel, - attributes: [ 'type', 'filename' ], - required: false - } - ] - } - ] - - if (type) { - countQuery.where = { type } - findQuery.where = { type } - } - - return Promise.all([ - VideoBlacklistModel.count(countQuery), - VideoBlacklistModel.findAll(findQuery) - ]).then(([ count, rows ]) => { - return { - data: rows, - total: count - } - }) - } - - static loadByVideoId (id: number): Promise { - const query = { - where: { - videoId: id - } - } - - return VideoBlacklistModel.findOne(query) - } - - toFormattedJSON (this: MVideoBlacklistFormattable): VideoBlacklist { - return { - id: this.id, - createdAt: this.createdAt, - updatedAt: this.updatedAt, - reason: this.reason, - unfederated: this.unfederated, - type: this.type, - - video: this.Video.toFormattedJSON() - } - } -} diff --git a/server/models/video/video-caption.ts b/server/models/video/video-caption.ts deleted file mode 100644 index dd4cefd65..000000000 --- a/server/models/video/video-caption.ts +++ /dev/null @@ -1,247 +0,0 @@ -import { remove } from 'fs-extra' -import { join } from 'path' -import { Op, OrderItem, Transaction } from 'sequelize' -import { - AllowNull, - BeforeDestroy, - BelongsTo, - Column, - CreatedAt, - DataType, - ForeignKey, - Is, - Model, - Scopes, - Table, - UpdatedAt -} from 'sequelize-typescript' -import { MVideo, MVideoCaption, MVideoCaptionFormattable, MVideoCaptionLanguageUrl, MVideoCaptionVideo } from '@server/types/models' -import { buildUUID } from '@shared/extra-utils' -import { AttributesOnly } from '@shared/typescript-utils' -import { VideoCaption } from '../../../shared/models/videos/caption/video-caption.model' -import { isVideoCaptionLanguageValid } from '../../helpers/custom-validators/video-captions' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, VIDEO_LANGUAGES, WEBSERVER } from '../../initializers/constants' -import { buildWhereIdOrUUID, throwIfNotValid } from '../shared' -import { VideoModel } from './video' - -export enum ScopeNames { - WITH_VIDEO_UUID_AND_REMOTE = 'WITH_VIDEO_UUID_AND_REMOTE' -} - -@Scopes(() => ({ - [ScopeNames.WITH_VIDEO_UUID_AND_REMOTE]: { - include: [ - { - attributes: [ 'id', 'uuid', 'remote' ], - model: VideoModel.unscoped(), - required: true - } - ] - } -})) - -@Table({ - tableName: 'videoCaption', - indexes: [ - { - fields: [ 'filename' ], - unique: true - }, - { - fields: [ 'videoId' ] - }, - { - fields: [ 'videoId', 'language' ], - unique: true - } - ] -}) -export class VideoCaptionModel extends Model>> { - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @AllowNull(false) - @Is('VideoCaptionLanguage', value => throwIfNotValid(value, isVideoCaptionLanguageValid, 'language')) - @Column - language: string - - @AllowNull(false) - @Column - filename: string - - @AllowNull(true) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.COMMONS.URL.max)) - fileUrl: string - - @ForeignKey(() => VideoModel) - @Column - videoId: number - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'CASCADE' - }) - Video: VideoModel - - @BeforeDestroy - static async removeFiles (instance: VideoCaptionModel, options) { - if (!instance.Video) { - instance.Video = await instance.$get('Video', { transaction: options.transaction }) - } - - if (instance.isOwned()) { - logger.info('Removing caption %s.', instance.filename) - - try { - await instance.removeCaptionFile() - } catch (err) { - logger.error('Cannot remove caption file %s.', instance.filename) - } - } - - return undefined - } - - static loadByVideoIdAndLanguage (videoId: string | number, language: string, transaction?: Transaction): Promise { - const videoInclude = { - model: VideoModel.unscoped(), - attributes: [ 'id', 'remote', 'uuid' ], - where: buildWhereIdOrUUID(videoId) - } - - const query = { - where: { - language - }, - include: [ - videoInclude - ], - transaction - } - - return VideoCaptionModel.findOne(query) - } - - static loadWithVideoByFilename (filename: string): Promise { - const query = { - where: { - filename - }, - include: [ - { - model: VideoModel.unscoped(), - attributes: [ 'id', 'remote', 'uuid' ] - } - ] - } - - return VideoCaptionModel.findOne(query) - } - - static async insertOrReplaceLanguage (caption: MVideoCaption, transaction: Transaction) { - const existing = await VideoCaptionModel.loadByVideoIdAndLanguage(caption.videoId, caption.language, transaction) - - // Delete existing file - if (existing) await existing.destroy({ transaction }) - - return caption.save({ transaction }) - } - - static listVideoCaptions (videoId: number, transaction?: Transaction): Promise { - const query = { - order: [ [ 'language', 'ASC' ] ] as OrderItem[], - where: { - videoId - }, - transaction - } - - return VideoCaptionModel.scope(ScopeNames.WITH_VIDEO_UUID_AND_REMOTE).findAll(query) - } - - static async listCaptionsOfMultipleVideos (videoIds: number[], transaction?: Transaction) { - const query = { - order: [ [ 'language', 'ASC' ] ] as OrderItem[], - where: { - videoId: { - [Op.in]: videoIds - } - }, - transaction - } - - const captions = await VideoCaptionModel.scope(ScopeNames.WITH_VIDEO_UUID_AND_REMOTE).findAll(query) - const result: { [ id: number ]: MVideoCaptionVideo[] } = {} - - for (const id of videoIds) { - result[id] = [] - } - - for (const caption of captions) { - result[caption.videoId].push(caption) - } - - return result - } - - static getLanguageLabel (language: string) { - return VIDEO_LANGUAGES[language] || 'Unknown' - } - - static deleteAllCaptionsOfRemoteVideo (videoId: number, transaction: Transaction) { - const query = { - where: { - videoId - }, - transaction - } - - return VideoCaptionModel.destroy(query) - } - - static generateCaptionName (language: string) { - return `${buildUUID()}-${language}.vtt` - } - - isOwned () { - return this.Video.remote === false - } - - toFormattedJSON (this: MVideoCaptionFormattable): VideoCaption { - return { - language: { - id: this.language, - label: VideoCaptionModel.getLanguageLabel(this.language) - }, - captionPath: this.getCaptionStaticPath(), - updatedAt: this.updatedAt.toISOString() - } - } - - getCaptionStaticPath (this: MVideoCaptionLanguageUrl) { - return join(LAZY_STATIC_PATHS.VIDEO_CAPTIONS, this.filename) - } - - removeCaptionFile (this: MVideoCaption) { - return remove(CONFIG.STORAGE.CAPTIONS_DIR + this.filename) - } - - getFileUrl (this: MVideoCaptionLanguageUrl, video: MVideo) { - if (video.isOwned()) return WEBSERVER.URL + this.getCaptionStaticPath() - - return this.fileUrl - } - - isEqual (this: MVideoCaption, other: MVideoCaption) { - if (this.fileUrl) return this.fileUrl === other.fileUrl - - return this.filename === other.filename - } -} diff --git a/server/models/video/video-change-ownership.ts b/server/models/video/video-change-ownership.ts deleted file mode 100644 index 26f072f4f..000000000 --- a/server/models/video/video-change-ownership.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { AllowNull, BelongsTo, Column, CreatedAt, ForeignKey, Model, Scopes, Table, UpdatedAt } from 'sequelize-typescript' -import { MVideoChangeOwnershipFormattable, MVideoChangeOwnershipFull } from '@server/types/models/video/video-change-ownership' -import { AttributesOnly } from '@shared/typescript-utils' -import { VideoChangeOwnership, VideoChangeOwnershipStatus } from '../../../shared/models/videos' -import { AccountModel } from '../account/account' -import { getSort } from '../shared' -import { ScopeNames as VideoScopeNames, VideoModel } from './video' - -enum ScopeNames { - WITH_ACCOUNTS = 'WITH_ACCOUNTS', - WITH_VIDEO = 'WITH_VIDEO' -} - -@Table({ - tableName: 'videoChangeOwnership', - indexes: [ - { - fields: [ 'videoId' ] - }, - { - fields: [ 'initiatorAccountId' ] - }, - { - fields: [ 'nextOwnerAccountId' ] - } - ] -}) -@Scopes(() => ({ - [ScopeNames.WITH_ACCOUNTS]: { - include: [ - { - model: AccountModel, - as: 'Initiator', - required: true - }, - { - model: AccountModel, - as: 'NextOwner', - required: true - } - ] - }, - [ScopeNames.WITH_VIDEO]: { - include: [ - { - model: VideoModel.scope([ - VideoScopeNames.WITH_THUMBNAILS, - VideoScopeNames.WITH_WEB_VIDEO_FILES, - VideoScopeNames.WITH_STREAMING_PLAYLISTS, - VideoScopeNames.WITH_ACCOUNT_DETAILS - ]), - required: true - } - ] - } -})) -export class VideoChangeOwnershipModel extends Model>> { - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @AllowNull(false) - @Column - status: VideoChangeOwnershipStatus - - @ForeignKey(() => AccountModel) - @Column - initiatorAccountId: number - - @BelongsTo(() => AccountModel, { - foreignKey: { - name: 'initiatorAccountId', - allowNull: false - }, - onDelete: 'cascade' - }) - Initiator: AccountModel - - @ForeignKey(() => AccountModel) - @Column - nextOwnerAccountId: number - - @BelongsTo(() => AccountModel, { - foreignKey: { - name: 'nextOwnerAccountId', - allowNull: false - }, - onDelete: 'cascade' - }) - NextOwner: AccountModel - - @ForeignKey(() => VideoModel) - @Column - videoId: number - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade' - }) - Video: VideoModel - - static listForApi (nextOwnerId: number, start: number, count: number, sort: string) { - const query = { - offset: start, - limit: count, - order: getSort(sort), - where: { - nextOwnerAccountId: nextOwnerId - } - } - - return Promise.all([ - VideoChangeOwnershipModel.scope(ScopeNames.WITH_ACCOUNTS).count(query), - VideoChangeOwnershipModel.scope([ ScopeNames.WITH_ACCOUNTS, ScopeNames.WITH_VIDEO ]).findAll(query) - ]).then(([ count, rows ]) => ({ total: count, data: rows })) - } - - static load (id: number): Promise { - return VideoChangeOwnershipModel.scope([ ScopeNames.WITH_ACCOUNTS, ScopeNames.WITH_VIDEO ]) - .findByPk(id) - } - - toFormattedJSON (this: MVideoChangeOwnershipFormattable): VideoChangeOwnership { - return { - id: this.id, - status: this.status, - initiatorAccount: this.Initiator.toFormattedJSON(), - nextOwnerAccount: this.NextOwner.toFormattedJSON(), - video: this.Video.toFormattedJSON(), - createdAt: this.createdAt - } - } -} diff --git a/server/models/video/video-channel-sync.ts b/server/models/video/video-channel-sync.ts deleted file mode 100644 index a4cbf51f5..000000000 --- a/server/models/video/video-channel-sync.ts +++ /dev/null @@ -1,176 +0,0 @@ -import { Op } from 'sequelize' -import { - AllowNull, - BelongsTo, - Column, - CreatedAt, - DataType, - Default, - DefaultScope, - ForeignKey, - Is, - Model, - Table, - UpdatedAt -} from 'sequelize-typescript' -import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc' -import { isVideoChannelSyncStateValid } from '@server/helpers/custom-validators/video-channel-syncs' -import { CONSTRAINTS_FIELDS, VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants' -import { MChannelSync, MChannelSyncChannel, MChannelSyncFormattable } from '@server/types/models' -import { VideoChannelSync, VideoChannelSyncState } from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { AccountModel } from '../account/account' -import { UserModel } from '../user/user' -import { getChannelSyncSort, throwIfNotValid } from '../shared' -import { VideoChannelModel } from './video-channel' - -@DefaultScope(() => ({ - include: [ - { - model: VideoChannelModel, // Default scope includes avatar and server - required: true - } - ] -})) -@Table({ - tableName: 'videoChannelSync', - indexes: [ - { - fields: [ 'videoChannelId' ] - } - ] -}) -export class VideoChannelSyncModel extends Model>> { - - @AllowNull(false) - @Default(null) - @Is('VideoChannelExternalChannelUrl', value => throwIfNotValid(value, isUrlValid, 'externalChannelUrl', true)) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_CHANNEL_SYNCS.EXTERNAL_CHANNEL_URL.max)) - externalChannelUrl: string - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => VideoChannelModel) - @Column - videoChannelId: number - - @BelongsTo(() => VideoChannelModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade' - }) - VideoChannel: VideoChannelModel - - @AllowNull(false) - @Default(VideoChannelSyncState.WAITING_FIRST_RUN) - @Is('VideoChannelSyncState', value => throwIfNotValid(value, isVideoChannelSyncStateValid, 'state')) - @Column - state: VideoChannelSyncState - - @AllowNull(true) - @Column(DataType.DATE) - lastSyncAt: Date - - static listByAccountForAPI (options: { - accountId: number - start: number - count: number - sort: string - }) { - const getQuery = (forCount: boolean) => { - const videoChannelModel = forCount - ? VideoChannelModel.unscoped() - : VideoChannelModel - - return { - offset: options.start, - limit: options.count, - order: getChannelSyncSort(options.sort), - include: [ - { - model: videoChannelModel, - required: true, - where: { - accountId: options.accountId - } - } - ] - } - } - - return Promise.all([ - VideoChannelSyncModel.unscoped().count(getQuery(true)), - VideoChannelSyncModel.unscoped().findAll(getQuery(false)) - ]).then(([ total, data ]) => ({ total, data })) - } - - static countByAccount (accountId: number) { - const query = { - include: [ - { - model: VideoChannelModel.unscoped(), - required: true, - where: { - accountId - } - } - ] - } - - return VideoChannelSyncModel.unscoped().count(query) - } - - static loadWithChannel (id: number): Promise { - return VideoChannelSyncModel.findByPk(id) - } - - static async listSyncs (): Promise { - const query = { - include: [ - { - model: VideoChannelModel.unscoped(), - required: true, - include: [ - { - model: AccountModel.unscoped(), - required: true, - include: [ { - attributes: [], - model: UserModel.unscoped(), - required: true, - where: { - videoQuota: { - [Op.ne]: 0 - }, - videoQuotaDaily: { - [Op.ne]: 0 - } - } - } ] - } - ] - } - ] - } - return VideoChannelSyncModel.unscoped().findAll(query) - } - - toFormattedJSON (this: MChannelSyncFormattable): VideoChannelSync { - return { - id: this.id, - state: { - id: this.state, - label: VIDEO_CHANNEL_SYNC_STATE[this.state] - }, - externalChannelUrl: this.externalChannelUrl, - createdAt: this.createdAt.toISOString(), - channel: this.VideoChannel.toFormattedSummaryJSON(), - lastSyncAt: this.lastSyncAt?.toISOString() - } - } -} diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts deleted file mode 100644 index 2c38850d7..000000000 --- a/server/models/video/video-channel.ts +++ /dev/null @@ -1,860 +0,0 @@ -import { FindOptions, Includeable, literal, Op, QueryTypes, ScopeOptions, Transaction, WhereOptions } from 'sequelize' -import { - AfterCreate, - AfterDestroy, - AfterUpdate, - AllowNull, - BeforeDestroy, - BelongsTo, - Column, - CreatedAt, - DataType, - Default, - DefaultScope, - ForeignKey, - HasMany, - Is, - Model, - Scopes, - Sequelize, - Table, - UpdatedAt -} from 'sequelize-typescript' -import { CONFIG } from '@server/initializers/config' -import { InternalEventEmitter } from '@server/lib/internal-event-emitter' -import { MAccountHost } from '@server/types/models' -import { forceNumber, pick } from '@shared/core-utils' -import { AttributesOnly } from '@shared/typescript-utils' -import { ActivityPubActor } from '../../../shared/models/activitypub' -import { VideoChannel, VideoChannelSummary } from '../../../shared/models/videos' -import { - isVideoChannelDescriptionValid, - isVideoChannelDisplayNameValid, - isVideoChannelSupportValid -} from '../../helpers/custom-validators/video-channels' -import { CONSTRAINTS_FIELDS, WEBSERVER } from '../../initializers/constants' -import { sendDeleteActor } from '../../lib/activitypub/send' -import { - MChannel, - MChannelActor, - MChannelAP, - MChannelBannerAccountDefault, - MChannelFormattable, - MChannelHost, - MChannelSummaryFormattable -} from '../../types/models/video' -import { AccountModel, ScopeNames as AccountModelScopeNames, SummaryOptions as AccountSummaryOptions } from '../account/account' -import { ActorModel, unusedActorAttributesForAPI } from '../actor/actor' -import { ActorFollowModel } from '../actor/actor-follow' -import { ActorImageModel } from '../actor/actor-image' -import { ServerModel } from '../server/server' -import { - buildServerIdsFollowedBy, - buildTrigramSearchIndex, - createSimilarityAttribute, - getSort, - setAsUpdated, - throwIfNotValid -} from '../shared' -import { VideoModel } from './video' -import { VideoPlaylistModel } from './video-playlist' - -export enum ScopeNames { - FOR_API = 'FOR_API', - SUMMARY = 'SUMMARY', - WITH_ACCOUNT = 'WITH_ACCOUNT', - WITH_ACTOR = 'WITH_ACTOR', - WITH_ACTOR_BANNER = 'WITH_ACTOR_BANNER', - WITH_VIDEOS = 'WITH_VIDEOS', - WITH_STATS = 'WITH_STATS' -} - -type AvailableForListOptions = { - actorId: number - search?: string - host?: string - handles?: string[] - forCount?: boolean -} - -type AvailableWithStatsOptions = { - daysPrior: number -} - -export type SummaryOptions = { - actorRequired?: boolean // Default: true - withAccount?: boolean // Default: false - withAccountBlockerIds?: number[] -} - -@DefaultScope(() => ({ - include: [ - { - model: ActorModel, - required: true - } - ] -})) -@Scopes(() => ({ - [ScopeNames.FOR_API]: (options: AvailableForListOptions) => { - // Only list local channels OR channels that are on an instance followed by actorId - const inQueryInstanceFollow = buildServerIdsFollowedBy(options.actorId) - - const whereActorAnd: WhereOptions[] = [ - { - [Op.or]: [ - { - serverId: null - }, - { - serverId: { - [Op.in]: Sequelize.literal(inQueryInstanceFollow) - } - } - ] - } - ] - - let serverRequired = false - let whereServer: WhereOptions - - if (options.host && options.host !== WEBSERVER.HOST) { - serverRequired = true - whereServer = { host: options.host } - } - - if (options.host === WEBSERVER.HOST) { - whereActorAnd.push({ - serverId: null - }) - } - - if (Array.isArray(options.handles) && options.handles.length !== 0) { - const or: string[] = [] - - for (const handle of options.handles || []) { - const [ preferredUsername, host ] = handle.split('@') - - const sanitizedPreferredUsername = VideoChannelModel.sequelize.escape(preferredUsername.toLowerCase()) - const sanitizedHost = VideoChannelModel.sequelize.escape(host) - - if (!host || host === WEBSERVER.HOST) { - or.push(`(LOWER("preferredUsername") = ${sanitizedPreferredUsername} AND "serverId" IS NULL)`) - } else { - or.push( - `(` + - `LOWER("preferredUsername") = ${sanitizedPreferredUsername} ` + - `AND "host" = ${sanitizedHost}` + - `)` - ) - } - } - - whereActorAnd.push({ - id: { - [Op.in]: literal(`(SELECT "actor".id FROM actor LEFT JOIN server on server.id = actor."serverId" WHERE ${or.join(' OR ')})`) - } - }) - } - - const channelActorInclude: Includeable[] = [] - const accountActorInclude: Includeable[] = [] - - if (options.forCount !== true) { - accountActorInclude.push({ - model: ServerModel, - required: false - }) - - accountActorInclude.push({ - model: ActorImageModel, - as: 'Avatars', - required: false - }) - - channelActorInclude.push({ - model: ActorImageModel, - as: 'Avatars', - required: false - }) - - channelActorInclude.push({ - model: ActorImageModel, - as: 'Banners', - required: false - }) - } - - if (options.forCount !== true || serverRequired) { - channelActorInclude.push({ - model: ServerModel, - duplicating: false, - required: serverRequired, - where: whereServer - }) - } - - return { - include: [ - { - attributes: { - exclude: unusedActorAttributesForAPI - }, - model: ActorModel.unscoped(), - where: { - [Op.and]: whereActorAnd - }, - include: channelActorInclude - }, - { - model: AccountModel.unscoped(), - required: true, - include: [ - { - attributes: { - exclude: unusedActorAttributesForAPI - }, - model: ActorModel.unscoped(), - required: true, - include: accountActorInclude - } - ] - } - ] - } - }, - [ScopeNames.SUMMARY]: (options: SummaryOptions = {}) => { - const include: Includeable[] = [ - { - attributes: [ 'id', 'preferredUsername', 'url', 'serverId' ], - model: ActorModel.unscoped(), - required: options.actorRequired ?? true, - include: [ - { - attributes: [ 'host' ], - model: ServerModel.unscoped(), - required: false - }, - { - model: ActorImageModel, - as: 'Avatars', - required: false - } - ] - } - ] - - const base: FindOptions = { - attributes: [ 'id', 'name', 'description', 'actorId' ] - } - - if (options.withAccount === true) { - include.push({ - model: AccountModel.scope({ - method: [ AccountModelScopeNames.SUMMARY, { withAccountBlockerIds: options.withAccountBlockerIds } as AccountSummaryOptions ] - }), - required: true - }) - } - - base.include = include - - return base - }, - [ScopeNames.WITH_ACCOUNT]: { - include: [ - { - model: AccountModel, - required: true - } - ] - }, - [ScopeNames.WITH_ACTOR]: { - include: [ - ActorModel - ] - }, - [ScopeNames.WITH_ACTOR_BANNER]: { - include: [ - { - model: ActorModel, - include: [ - { - model: ActorImageModel, - required: false, - as: 'Banners' - } - ] - } - ] - }, - [ScopeNames.WITH_VIDEOS]: { - include: [ - VideoModel - ] - }, - [ScopeNames.WITH_STATS]: (options: AvailableWithStatsOptions = { daysPrior: 30 }) => { - const daysPrior = forceNumber(options.daysPrior) - - return { - attributes: { - include: [ - [ - literal('(SELECT COUNT(*) FROM "video" WHERE "channelId" = "VideoChannelModel"."id")'), - 'videosCount' - ], - [ - literal( - '(' + - `SELECT string_agg(concat_ws('|', t.day, t.views), ',') ` + - 'FROM ( ' + - 'WITH ' + - 'days AS ( ' + - `SELECT generate_series(date_trunc('day', now()) - '${daysPrior} day'::interval, ` + - `date_trunc('day', now()), '1 day'::interval) AS day ` + - ') ' + - 'SELECT days.day AS day, COALESCE(SUM("videoView".views), 0) AS views ' + - 'FROM days ' + - 'LEFT JOIN (' + - '"videoView" INNER JOIN "video" ON "videoView"."videoId" = "video"."id" ' + - 'AND "video"."channelId" = "VideoChannelModel"."id"' + - `) ON date_trunc('day', "videoView"."startDate") = date_trunc('day', days.day) ` + - 'GROUP BY day ' + - 'ORDER BY day ' + - ') t' + - ')' - ), - 'viewsPerDay' - ], - [ - literal( - '(' + - 'SELECT COALESCE(SUM("video".views), 0) AS totalViews ' + - 'FROM "video" ' + - 'WHERE "video"."channelId" = "VideoChannelModel"."id"' + - ')' - ), - 'totalViews' - ] - ] - } - } - } -})) -@Table({ - tableName: 'videoChannel', - indexes: [ - buildTrigramSearchIndex('video_channel_name_trigram', 'name'), - - { - fields: [ 'accountId' ] - }, - { - fields: [ 'actorId' ] - } - ] -}) -export class VideoChannelModel extends Model>> { - - @AllowNull(false) - @Is('VideoChannelName', value => throwIfNotValid(value, isVideoChannelDisplayNameValid, 'name')) - @Column - name: string - - @AllowNull(true) - @Default(null) - @Is('VideoChannelDescription', value => throwIfNotValid(value, isVideoChannelDescriptionValid, 'description', true)) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_CHANNELS.DESCRIPTION.max)) - description: string - - @AllowNull(true) - @Default(null) - @Is('VideoChannelSupport', value => throwIfNotValid(value, isVideoChannelSupportValid, 'support', true)) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_CHANNELS.SUPPORT.max)) - support: string - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => ActorModel) - @Column - actorId: number - - @BelongsTo(() => ActorModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade' - }) - Actor: ActorModel - - @ForeignKey(() => AccountModel) - @Column - accountId: number - - @BelongsTo(() => AccountModel, { - foreignKey: { - allowNull: false - } - }) - Account: AccountModel - - @HasMany(() => VideoModel, { - foreignKey: { - name: 'channelId', - allowNull: false - }, - onDelete: 'CASCADE', - hooks: true - }) - Videos: VideoModel[] - - @HasMany(() => VideoPlaylistModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'CASCADE', - hooks: true - }) - VideoPlaylists: VideoPlaylistModel[] - - @AfterCreate - static notifyCreate (channel: MChannel) { - InternalEventEmitter.Instance.emit('channel-created', { channel }) - } - - @AfterUpdate - static notifyUpdate (channel: MChannel) { - InternalEventEmitter.Instance.emit('channel-updated', { channel }) - } - - @AfterDestroy - static notifyDestroy (channel: MChannel) { - InternalEventEmitter.Instance.emit('channel-deleted', { channel }) - } - - @BeforeDestroy - static async sendDeleteIfOwned (instance: VideoChannelModel, options) { - if (!instance.Actor) { - instance.Actor = await instance.$get('Actor', { transaction: options.transaction }) - } - - await ActorFollowModel.removeFollowsOf(instance.Actor.id, options.transaction) - - if (instance.Actor.isOwned()) { - return sendDeleteActor(instance.Actor, options.transaction) - } - - return undefined - } - - static countByAccount (accountId: number) { - const query = { - where: { - accountId - } - } - - return VideoChannelModel.unscoped().count(query) - } - - static async getStats () { - - function getLocalVideoChannelStats (days?: number) { - const options = { - type: QueryTypes.SELECT as QueryTypes.SELECT, - raw: true - } - - const videoJoin = days - ? `INNER JOIN "video" AS "Videos" ON "VideoChannelModel"."id" = "Videos"."channelId" ` + - `AND ("Videos"."publishedAt" > Now() - interval '${days}d')` - : '' - - const query = ` - SELECT COUNT(DISTINCT("VideoChannelModel"."id")) AS "count" - FROM "videoChannel" AS "VideoChannelModel" - ${videoJoin} - INNER JOIN "account" AS "Account" ON "VideoChannelModel"."accountId" = "Account"."id" - INNER JOIN "actor" AS "Account->Actor" ON "Account"."actorId" = "Account->Actor"."id" - AND "Account->Actor"."serverId" IS NULL` - - return VideoChannelModel.sequelize.query<{ count: string }>(query, options) - .then(r => parseInt(r[0].count, 10)) - } - - const totalLocalVideoChannels = await getLocalVideoChannelStats() - const totalLocalDailyActiveVideoChannels = await getLocalVideoChannelStats(1) - const totalLocalWeeklyActiveVideoChannels = await getLocalVideoChannelStats(7) - const totalLocalMonthlyActiveVideoChannels = await getLocalVideoChannelStats(30) - const totalLocalHalfYearActiveVideoChannels = await getLocalVideoChannelStats(180) - - return { - totalLocalVideoChannels, - totalLocalDailyActiveVideoChannels, - totalLocalWeeklyActiveVideoChannels, - totalLocalMonthlyActiveVideoChannels, - totalLocalHalfYearActiveVideoChannels - } - } - - static listLocalsForSitemap (sort: string): Promise { - const query = { - attributes: [ ], - offset: 0, - order: getSort(sort), - include: [ - { - attributes: [ 'preferredUsername', 'serverId' ], - model: ActorModel.unscoped(), - where: { - serverId: null - } - } - ] - } - - return VideoChannelModel - .unscoped() - .findAll(query) - } - - static listForApi (parameters: Pick & { - start: number - count: number - sort: string - }) { - const { actorId } = parameters - - const query = { - offset: parameters.start, - limit: parameters.count, - order: getSort(parameters.sort) - } - - const getScope = (forCount: boolean) => { - return { method: [ ScopeNames.FOR_API, { actorId, forCount } as AvailableForListOptions ] } - } - - return Promise.all([ - VideoChannelModel.scope(getScope(true)).count(), - VideoChannelModel.scope(getScope(false)).findAll(query) - ]).then(([ total, data ]) => ({ total, data })) - } - - static searchForApi (options: Pick & { - start: number - count: number - sort: string - }) { - let attributesInclude: any[] = [ literal('0 as similarity') ] - let where: WhereOptions - - if (options.search) { - const escapedSearch = VideoChannelModel.sequelize.escape(options.search) - const escapedLikeSearch = VideoChannelModel.sequelize.escape('%' + options.search + '%') - attributesInclude = [ createSimilarityAttribute('VideoChannelModel.name', options.search) ] - - where = { - [Op.or]: [ - Sequelize.literal( - 'lower(immutable_unaccent("VideoChannelModel"."name")) % lower(immutable_unaccent(' + escapedSearch + '))' - ), - Sequelize.literal( - 'lower(immutable_unaccent("VideoChannelModel"."name")) LIKE lower(immutable_unaccent(' + escapedLikeSearch + '))' - ) - ] - } - } - - const query = { - attributes: { - include: attributesInclude - }, - offset: options.start, - limit: options.count, - order: getSort(options.sort), - where - } - - const getScope = (forCount: boolean) => { - return { - method: [ - ScopeNames.FOR_API, { - ...pick(options, [ 'actorId', 'host', 'handles' ]), - - forCount - } as AvailableForListOptions - ] - } - } - - return Promise.all([ - VideoChannelModel.scope(getScope(true)).count(query), - VideoChannelModel.scope(getScope(false)).findAll(query) - ]).then(([ total, data ]) => ({ total, data })) - } - - static listByAccountForAPI (options: { - accountId: number - start: number - count: number - sort: string - withStats?: boolean - search?: string - }) { - const escapedSearch = VideoModel.sequelize.escape(options.search) - const escapedLikeSearch = VideoModel.sequelize.escape('%' + options.search + '%') - const where = options.search - ? { - [Op.or]: [ - Sequelize.literal( - 'lower(immutable_unaccent("VideoChannelModel"."name")) % lower(immutable_unaccent(' + escapedSearch + '))' - ), - Sequelize.literal( - 'lower(immutable_unaccent("VideoChannelModel"."name")) LIKE lower(immutable_unaccent(' + escapedLikeSearch + '))' - ) - ] - } - : null - - const getQuery = (forCount: boolean) => { - const accountModel = forCount - ? AccountModel.unscoped() - : AccountModel - - return { - offset: options.start, - limit: options.count, - order: getSort(options.sort), - include: [ - { - model: accountModel, - where: { - id: options.accountId - }, - required: true - } - ], - where - } - } - - const findScopes: string | ScopeOptions | (string | ScopeOptions)[] = [ ScopeNames.WITH_ACTOR_BANNER ] - - if (options.withStats === true) { - findScopes.push({ - method: [ ScopeNames.WITH_STATS, { daysPrior: 30 } as AvailableWithStatsOptions ] - }) - } - - return Promise.all([ - VideoChannelModel.unscoped().count(getQuery(true)), - VideoChannelModel.scope(findScopes).findAll(getQuery(false)) - ]).then(([ total, data ]) => ({ total, data })) - } - - static listAllByAccount (accountId: number): Promise { - const query = { - limit: CONFIG.VIDEO_CHANNELS.MAX_PER_USER, - include: [ - { - attributes: [], - model: AccountModel.unscoped(), - where: { - id: accountId - }, - required: true - } - ] - } - - return VideoChannelModel.findAll(query) - } - - static loadAndPopulateAccount (id: number, transaction?: Transaction): Promise { - return VideoChannelModel.unscoped() - .scope([ ScopeNames.WITH_ACTOR_BANNER, ScopeNames.WITH_ACCOUNT ]) - .findByPk(id, { transaction }) - } - - static loadByUrlAndPopulateAccount (url: string): Promise { - const query = { - include: [ - { - model: ActorModel, - required: true, - where: { - url - }, - include: [ - { - model: ActorImageModel, - required: false, - as: 'Banners' - } - ] - } - ] - } - - return VideoChannelModel - .scope([ ScopeNames.WITH_ACCOUNT ]) - .findOne(query) - } - - static loadByNameWithHostAndPopulateAccount (nameWithHost: string) { - const [ name, host ] = nameWithHost.split('@') - - if (!host || host === WEBSERVER.HOST) return VideoChannelModel.loadLocalByNameAndPopulateAccount(name) - - return VideoChannelModel.loadByNameAndHostAndPopulateAccount(name, host) - } - - static loadLocalByNameAndPopulateAccount (name: string): Promise { - const query = { - include: [ - { - model: ActorModel, - required: true, - where: { - [Op.and]: [ - ActorModel.wherePreferredUsername(name, 'Actor.preferredUsername'), - { serverId: null } - ] - }, - include: [ - { - model: ActorImageModel, - required: false, - as: 'Banners' - } - ] - } - ] - } - - return VideoChannelModel.unscoped() - .scope([ ScopeNames.WITH_ACCOUNT ]) - .findOne(query) - } - - static loadByNameAndHostAndPopulateAccount (name: string, host: string): Promise { - const query = { - include: [ - { - model: ActorModel, - required: true, - where: ActorModel.wherePreferredUsername(name, 'Actor.preferredUsername'), - include: [ - { - model: ServerModel, - required: true, - where: { host } - }, - { - model: ActorImageModel, - required: false, - as: 'Banners' - } - ] - } - ] - } - - return VideoChannelModel.unscoped() - .scope([ ScopeNames.WITH_ACCOUNT ]) - .findOne(query) - } - - toFormattedSummaryJSON (this: MChannelSummaryFormattable): VideoChannelSummary { - const actor = this.Actor.toFormattedSummaryJSON() - - return { - id: this.id, - name: actor.name, - displayName: this.getDisplayName(), - url: actor.url, - host: actor.host, - avatars: actor.avatars - } - } - - toFormattedJSON (this: MChannelFormattable): VideoChannel { - const viewsPerDayString = this.get('viewsPerDay') as string - const videosCount = this.get('videosCount') as number - - let viewsPerDay: { date: Date, views: number }[] - - if (viewsPerDayString) { - viewsPerDay = viewsPerDayString.split(',') - .map(v => { - const [ dateString, amount ] = v.split('|') - - return { - date: new Date(dateString), - views: +amount - } - }) - } - - const totalViews = this.get('totalViews') as number - - const actor = this.Actor.toFormattedJSON() - const videoChannel = { - id: this.id, - displayName: this.getDisplayName(), - description: this.description, - support: this.support, - isLocal: this.Actor.isOwned(), - updatedAt: this.updatedAt, - - ownerAccount: undefined, - - videosCount, - viewsPerDay, - totalViews, - - avatars: actor.avatars - } - - if (this.Account) videoChannel.ownerAccount = this.Account.toFormattedJSON() - - return Object.assign(actor, videoChannel) - } - - async toActivityPubObject (this: MChannelAP): Promise { - const obj = await this.Actor.toActivityPubObject(this.name) - - return Object.assign(obj, { - summary: this.description, - support: this.support, - attributedTo: [ - { - type: 'Person' as 'Person', - id: this.Account.Actor.url - } - ] - }) - } - - // Avoid error when running this method on MAccount... | MChannel... - getClientUrl (this: MAccountHost | MChannelHost) { - return WEBSERVER.URL + '/c/' + this.Actor.getIdentifier() - } - - getDisplayName () { - return this.name - } - - isOutdated () { - return this.Actor.isOutdated() - } - - setAsUpdated (transaction?: Transaction) { - return setAsUpdated({ sequelize: this.sequelize, table: 'videoChannel', id: this.id, transaction }) - } -} diff --git a/server/models/video/video-comment.ts b/server/models/video/video-comment.ts deleted file mode 100644 index ff5142809..000000000 --- a/server/models/video/video-comment.ts +++ /dev/null @@ -1,683 +0,0 @@ -import { FindOptions, Op, Order, QueryTypes, Sequelize, Transaction } from 'sequelize' -import { - AllowNull, - BelongsTo, - Column, - CreatedAt, - DataType, - ForeignKey, - HasMany, - Is, - Model, - Scopes, - Table, - UpdatedAt -} from 'sequelize-typescript' -import { getServerActor } from '@server/models/application/application' -import { MAccount, MAccountId, MUserAccountId } from '@server/types/models' -import { pick, uniqify } from '@shared/core-utils' -import { AttributesOnly } from '@shared/typescript-utils' -import { ActivityTagObject, ActivityTombstoneObject } from '../../../shared/models/activitypub/objects/common-objects' -import { VideoCommentObject } from '../../../shared/models/activitypub/objects/video-comment-object' -import { VideoComment, VideoCommentAdmin } from '../../../shared/models/videos/comment/video-comment.model' -import { actorNameAlphabet } from '../../helpers/custom-validators/activitypub/actor' -import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' -import { regexpCapture } from '../../helpers/regexp' -import { CONSTRAINTS_FIELDS, WEBSERVER } from '../../initializers/constants' -import { - MComment, - MCommentAdminFormattable, - MCommentAP, - MCommentFormattable, - MCommentId, - MCommentOwner, - MCommentOwnerReplyVideoLight, - MCommentOwnerVideo, - MCommentOwnerVideoFeed, - MCommentOwnerVideoReply, - MVideoImmutable -} from '../../types/models/video' -import { VideoCommentAbuseModel } from '../abuse/video-comment-abuse' -import { AccountModel } from '../account/account' -import { ActorModel } from '../actor/actor' -import { buildLocalAccountIdsIn, buildSQLAttributes, throwIfNotValid } from '../shared' -import { ListVideoCommentsOptions, VideoCommentListQueryBuilder } from './sql/comment/video-comment-list-query-builder' -import { VideoModel } from './video' -import { VideoChannelModel } from './video-channel' - -export enum ScopeNames { - WITH_ACCOUNT = 'WITH_ACCOUNT', - WITH_IN_REPLY_TO = 'WITH_IN_REPLY_TO', - WITH_VIDEO = 'WITH_VIDEO' -} - -@Scopes(() => ({ - [ScopeNames.WITH_ACCOUNT]: { - include: [ - { - model: AccountModel - } - ] - }, - [ScopeNames.WITH_IN_REPLY_TO]: { - include: [ - { - model: VideoCommentModel, - as: 'InReplyToVideoComment' - } - ] - }, - [ScopeNames.WITH_VIDEO]: { - include: [ - { - model: VideoModel, - required: true, - include: [ - { - model: VideoChannelModel, - required: true, - include: [ - { - model: AccountModel, - required: true - } - ] - } - ] - } - ] - } -})) -@Table({ - tableName: 'videoComment', - indexes: [ - { - fields: [ 'videoId' ] - }, - { - fields: [ 'videoId', 'originCommentId' ] - }, - { - fields: [ 'url' ], - unique: true - }, - { - fields: [ 'accountId' ] - }, - { - fields: [ - { name: 'createdAt', order: 'DESC' } - ] - } - ] -}) -export class VideoCommentModel extends Model>> { - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @AllowNull(true) - @Column(DataType.DATE) - deletedAt: Date - - @AllowNull(false) - @Is('VideoCommentUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url')) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max)) - url: string - - @AllowNull(false) - @Column(DataType.TEXT) - text: string - - @ForeignKey(() => VideoCommentModel) - @Column - originCommentId: number - - @BelongsTo(() => VideoCommentModel, { - foreignKey: { - name: 'originCommentId', - allowNull: true - }, - as: 'OriginVideoComment', - onDelete: 'CASCADE' - }) - OriginVideoComment: VideoCommentModel - - @ForeignKey(() => VideoCommentModel) - @Column - inReplyToCommentId: number - - @BelongsTo(() => VideoCommentModel, { - foreignKey: { - name: 'inReplyToCommentId', - allowNull: true - }, - as: 'InReplyToVideoComment', - onDelete: 'CASCADE' - }) - InReplyToVideoComment: VideoCommentModel | null - - @ForeignKey(() => VideoModel) - @Column - videoId: number - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'CASCADE' - }) - Video: VideoModel - - @ForeignKey(() => AccountModel) - @Column - accountId: number - - @BelongsTo(() => AccountModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'CASCADE' - }) - Account: AccountModel - - @HasMany(() => VideoCommentAbuseModel, { - foreignKey: { - name: 'videoCommentId', - allowNull: true - }, - onDelete: 'set null' - }) - CommentAbuses: VideoCommentAbuseModel[] - - // --------------------------------------------------------------------------- - - static getSQLAttributes (tableName: string, aliasPrefix = '') { - return buildSQLAttributes({ - model: this, - tableName, - aliasPrefix - }) - } - - // --------------------------------------------------------------------------- - - static loadById (id: number, t?: Transaction): Promise { - const query: FindOptions = { - where: { - id - } - } - - if (t !== undefined) query.transaction = t - - return VideoCommentModel.findOne(query) - } - - static loadByIdAndPopulateVideoAndAccountAndReply (id: number, t?: Transaction): Promise { - const query: FindOptions = { - where: { - id - } - } - - if (t !== undefined) query.transaction = t - - return VideoCommentModel - .scope([ ScopeNames.WITH_VIDEO, ScopeNames.WITH_ACCOUNT, ScopeNames.WITH_IN_REPLY_TO ]) - .findOne(query) - } - - static loadByUrlAndPopulateAccountAndVideo (url: string, t?: Transaction): Promise { - const query: FindOptions = { - where: { - url - } - } - - if (t !== undefined) query.transaction = t - - return VideoCommentModel.scope([ ScopeNames.WITH_ACCOUNT, ScopeNames.WITH_VIDEO ]).findOne(query) - } - - static loadByUrlAndPopulateReplyAndVideoUrlAndAccount (url: string, t?: Transaction): Promise { - const query: FindOptions = { - where: { - url - }, - include: [ - { - attributes: [ 'id', 'url' ], - model: VideoModel.unscoped() - } - ] - } - - if (t !== undefined) query.transaction = t - - return VideoCommentModel.scope([ ScopeNames.WITH_IN_REPLY_TO, ScopeNames.WITH_ACCOUNT ]).findOne(query) - } - - static listCommentsForApi (parameters: { - start: number - count: number - sort: string - - onLocalVideo?: boolean - isLocal?: boolean - search?: string - searchAccount?: string - searchVideo?: string - }) { - const queryOptions: ListVideoCommentsOptions = { - ...pick(parameters, [ 'start', 'count', 'sort', 'isLocal', 'search', 'searchVideo', 'searchAccount', 'onLocalVideo' ]), - - selectType: 'api', - notDeleted: true - } - - return Promise.all([ - new VideoCommentListQueryBuilder(VideoCommentModel.sequelize, queryOptions).listComments(), - new VideoCommentListQueryBuilder(VideoCommentModel.sequelize, queryOptions).countComments() - ]).then(([ rows, count ]) => { - return { total: count, data: rows } - }) - } - - static async listThreadsForApi (parameters: { - videoId: number - isVideoOwned: boolean - start: number - count: number - sort: string - user?: MUserAccountId - }) { - const { videoId, user } = parameters - - const blockerAccountIds = await VideoCommentModel.buildBlockerAccountIds({ user }) - - const commonOptions: ListVideoCommentsOptions = { - selectType: 'api', - videoId, - blockerAccountIds - } - - const listOptions: ListVideoCommentsOptions = { - ...commonOptions, - ...pick(parameters, [ 'sort', 'start', 'count' ]), - - isThread: true, - includeReplyCounters: true - } - - const countOptions: ListVideoCommentsOptions = { - ...commonOptions, - - isThread: true - } - - const notDeletedCountOptions: ListVideoCommentsOptions = { - ...commonOptions, - - notDeleted: true - } - - return Promise.all([ - new VideoCommentListQueryBuilder(VideoCommentModel.sequelize, listOptions).listComments(), - new VideoCommentListQueryBuilder(VideoCommentModel.sequelize, countOptions).countComments(), - new VideoCommentListQueryBuilder(VideoCommentModel.sequelize, notDeletedCountOptions).countComments() - ]).then(([ rows, count, totalNotDeletedComments ]) => { - return { total: count, data: rows, totalNotDeletedComments } - }) - } - - static async listThreadCommentsForApi (parameters: { - videoId: number - threadId: number - user?: MUserAccountId - }) { - const { user } = parameters - - const blockerAccountIds = await VideoCommentModel.buildBlockerAccountIds({ user }) - - const queryOptions: ListVideoCommentsOptions = { - ...pick(parameters, [ 'videoId', 'threadId' ]), - - selectType: 'api', - sort: 'createdAt', - - blockerAccountIds, - includeReplyCounters: true - } - - return Promise.all([ - new VideoCommentListQueryBuilder(VideoCommentModel.sequelize, queryOptions).listComments(), - new VideoCommentListQueryBuilder(VideoCommentModel.sequelize, queryOptions).countComments() - ]).then(([ rows, count ]) => { - return { total: count, data: rows } - }) - } - - static listThreadParentComments (comment: MCommentId, t: Transaction, order: 'ASC' | 'DESC' = 'ASC'): Promise { - const query = { - order: [ [ 'createdAt', order ] ] as Order, - where: { - id: { - [Op.in]: Sequelize.literal('(' + - 'WITH RECURSIVE children (id, "inReplyToCommentId") AS ( ' + - `SELECT id, "inReplyToCommentId" FROM "videoComment" WHERE id = ${comment.id} ` + - 'UNION ' + - 'SELECT "parent"."id", "parent"."inReplyToCommentId" FROM "videoComment" "parent" ' + - 'INNER JOIN "children" ON "children"."inReplyToCommentId" = "parent"."id"' + - ') ' + - 'SELECT id FROM children' + - ')'), - [Op.ne]: comment.id - } - }, - transaction: t - } - - return VideoCommentModel - .scope([ ScopeNames.WITH_ACCOUNT ]) - .findAll(query) - } - - static async listAndCountByVideoForAP (parameters: { - video: MVideoImmutable - start: number - count: number - }) { - const { video } = parameters - - const blockerAccountIds = await VideoCommentModel.buildBlockerAccountIds({ user: null }) - - const queryOptions: ListVideoCommentsOptions = { - ...pick(parameters, [ 'start', 'count' ]), - - selectType: 'comment-only', - videoId: video.id, - sort: 'createdAt', - - blockerAccountIds - } - - return Promise.all([ - new VideoCommentListQueryBuilder(VideoCommentModel.sequelize, queryOptions).listComments(), - new VideoCommentListQueryBuilder(VideoCommentModel.sequelize, queryOptions).countComments() - ]).then(([ rows, count ]) => { - return { total: count, data: rows } - }) - } - - static async listForFeed (parameters: { - start: number - count: number - videoId?: number - accountId?: number - videoChannelId?: number - }) { - const blockerAccountIds = await VideoCommentModel.buildBlockerAccountIds({ user: null }) - - const queryOptions: ListVideoCommentsOptions = { - ...pick(parameters, [ 'start', 'count', 'accountId', 'videoId', 'videoChannelId' ]), - - selectType: 'feed', - - sort: '-createdAt', - onPublicVideo: true, - notDeleted: true, - - blockerAccountIds - } - - return new VideoCommentListQueryBuilder(VideoCommentModel.sequelize, queryOptions).listComments() - } - - static listForBulkDelete (ofAccount: MAccount, filter: { onVideosOfAccount?: MAccountId } = {}) { - const queryOptions: ListVideoCommentsOptions = { - selectType: 'comment-only', - - accountId: ofAccount.id, - videoAccountOwnerId: filter.onVideosOfAccount?.id, - - notDeleted: true, - count: 5000 - } - - return new VideoCommentListQueryBuilder(VideoCommentModel.sequelize, queryOptions).listComments() - } - - static async getStats () { - const totalLocalVideoComments = await VideoCommentModel.count({ - include: [ - { - model: AccountModel.unscoped(), - required: true, - include: [ - { - model: ActorModel.unscoped(), - required: true, - where: { - serverId: null - } - } - ] - } - ] - }) - const totalVideoComments = await VideoCommentModel.count() - - return { - totalLocalVideoComments, - totalVideoComments - } - } - - static listRemoteCommentUrlsOfLocalVideos () { - const query = `SELECT "videoComment".url FROM "videoComment" ` + - `INNER JOIN account ON account.id = "videoComment"."accountId" ` + - `INNER JOIN actor ON actor.id = "account"."actorId" AND actor."serverId" IS NOT NULL ` + - `INNER JOIN video ON video.id = "videoComment"."videoId" AND video.remote IS FALSE` - - return VideoCommentModel.sequelize.query<{ url: string }>(query, { - type: QueryTypes.SELECT, - raw: true - }).then(rows => rows.map(r => r.url)) - } - - static cleanOldCommentsOf (videoId: number, beforeUpdatedAt: Date) { - const query = { - where: { - updatedAt: { - [Op.lt]: beforeUpdatedAt - }, - videoId, - accountId: { - [Op.notIn]: buildLocalAccountIdsIn() - }, - // Do not delete Tombstones - deletedAt: null - } - } - - return VideoCommentModel.destroy(query) - } - - getCommentStaticPath () { - return this.Video.getWatchStaticPath() + ';threadId=' + this.getThreadId() - } - - getThreadId (): number { - return this.originCommentId || this.id - } - - isOwned () { - if (!this.Account) return false - - return this.Account.isOwned() - } - - markAsDeleted () { - this.text = '' - this.deletedAt = new Date() - this.accountId = null - } - - isDeleted () { - return this.deletedAt !== null - } - - extractMentions () { - let result: string[] = [] - - const localMention = `@(${actorNameAlphabet}+)` - const remoteMention = `${localMention}@${WEBSERVER.HOST}` - - const mentionRegex = this.isOwned() - ? '(?:(?:' + remoteMention + ')|(?:' + localMention + '))' // Include local mentions? - : '(?:' + remoteMention + ')' - - const firstMentionRegex = new RegExp(`^${mentionRegex} `, 'g') - const endMentionRegex = new RegExp(` ${mentionRegex}$`, 'g') - const remoteMentionsRegex = new RegExp(' ' + remoteMention + ' ', 'g') - - result = result.concat( - regexpCapture(this.text, firstMentionRegex) - .map(([ , username1, username2 ]) => username1 || username2), - - regexpCapture(this.text, endMentionRegex) - .map(([ , username1, username2 ]) => username1 || username2), - - regexpCapture(this.text, remoteMentionsRegex) - .map(([ , username ]) => username) - ) - - // Include local mentions - if (this.isOwned()) { - const localMentionsRegex = new RegExp(' ' + localMention + ' ', 'g') - - result = result.concat( - regexpCapture(this.text, localMentionsRegex) - .map(([ , username ]) => username) - ) - } - - return uniqify(result) - } - - toFormattedJSON (this: MCommentFormattable) { - return { - id: this.id, - url: this.url, - text: this.text, - - threadId: this.getThreadId(), - inReplyToCommentId: this.inReplyToCommentId || null, - videoId: this.videoId, - - createdAt: this.createdAt, - updatedAt: this.updatedAt, - deletedAt: this.deletedAt, - - isDeleted: this.isDeleted(), - - totalRepliesFromVideoAuthor: this.get('totalRepliesFromVideoAuthor') || 0, - totalReplies: this.get('totalReplies') || 0, - - account: this.Account - ? this.Account.toFormattedJSON() - : null - } as VideoComment - } - - toFormattedAdminJSON (this: MCommentAdminFormattable) { - return { - id: this.id, - url: this.url, - text: this.text, - - threadId: this.getThreadId(), - inReplyToCommentId: this.inReplyToCommentId || null, - videoId: this.videoId, - - createdAt: this.createdAt, - updatedAt: this.updatedAt, - - video: { - id: this.Video.id, - uuid: this.Video.uuid, - name: this.Video.name - }, - - account: this.Account - ? this.Account.toFormattedJSON() - : null - } as VideoCommentAdmin - } - - toActivityPubObject (this: MCommentAP, threadParentComments: MCommentOwner[]): VideoCommentObject | ActivityTombstoneObject { - let inReplyTo: string - // New thread, so in AS we reply to the video - if (this.inReplyToCommentId === null) { - inReplyTo = this.Video.url - } else { - inReplyTo = this.InReplyToVideoComment.url - } - - if (this.isDeleted()) { - return { - id: this.url, - type: 'Tombstone', - formerType: 'Note', - inReplyTo, - published: this.createdAt.toISOString(), - updated: this.updatedAt.toISOString(), - deleted: this.deletedAt.toISOString() - } - } - - const tag: ActivityTagObject[] = [] - for (const parentComment of threadParentComments) { - if (!parentComment.Account) continue - - const actor = parentComment.Account.Actor - - tag.push({ - type: 'Mention', - href: actor.url, - name: `@${actor.preferredUsername}@${actor.getHost()}` - }) - } - - return { - type: 'Note' as 'Note', - id: this.url, - - content: this.text, - mediaType: 'text/markdown', - - inReplyTo, - updated: this.updatedAt.toISOString(), - published: this.createdAt.toISOString(), - url: this.url, - attributedTo: this.Account.Actor.url, - tag - } - } - - private static async buildBlockerAccountIds (options: { - user: MUserAccountId - }): Promise { - const { user } = options - - const serverActor = await getServerActor() - const blockerAccountIds = [ serverActor.Account.id ] - - if (user) blockerAccountIds.push(user.Account.id) - - return blockerAccountIds - } -} diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts deleted file mode 100644 index ee34ad2ff..000000000 --- a/server/models/video/video-file.ts +++ /dev/null @@ -1,635 +0,0 @@ -import { remove } from 'fs-extra' -import memoizee from 'memoizee' -import { join } from 'path' -import { FindOptions, Op, Transaction, WhereOptions } from 'sequelize' -import { - AllowNull, - BelongsTo, - Column, - CreatedAt, - DataType, - Default, - DefaultScope, - ForeignKey, - HasMany, - Is, - Model, - Scopes, - Table, - UpdatedAt -} from 'sequelize-typescript' -import validator from 'validator' -import { logger } from '@server/helpers/logger' -import { extractVideo } from '@server/helpers/video' -import { CONFIG } from '@server/initializers/config' -import { buildRemoteVideoBaseUrl } from '@server/lib/activitypub/url' -import { - getHLSPrivateFileUrl, - getHLSPublicFileUrl, - getWebVideoPrivateFileUrl, - getWebVideoPublicFileUrl -} from '@server/lib/object-storage' -import { getFSTorrentFilePath } from '@server/lib/paths' -import { isVideoInPrivateDirectory } from '@server/lib/video-privacy' -import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models' -import { VideoResolution, VideoStorage } from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { - isVideoFileExtnameValid, - isVideoFileInfoHashValid, - isVideoFileResolutionValid, - isVideoFileSizeValid, - isVideoFPSResolutionValid -} from '../../helpers/custom-validators/videos' -import { - LAZY_STATIC_PATHS, - MEMOIZE_LENGTH, - MEMOIZE_TTL, - STATIC_DOWNLOAD_PATHS, - STATIC_PATHS, - WEBSERVER -} from '../../initializers/constants' -import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../types/models/video/video-file' -import { VideoRedundancyModel } from '../redundancy/video-redundancy' -import { doesExist, parseAggregateResult, throwIfNotValid } from '../shared' -import { VideoModel } from './video' -import { VideoStreamingPlaylistModel } from './video-streaming-playlist' - -export enum ScopeNames { - WITH_VIDEO = 'WITH_VIDEO', - WITH_METADATA = 'WITH_METADATA', - WITH_VIDEO_OR_PLAYLIST = 'WITH_VIDEO_OR_PLAYLIST' -} - -@DefaultScope(() => ({ - attributes: { - exclude: [ 'metadata' ] - } -})) -@Scopes(() => ({ - [ScopeNames.WITH_VIDEO]: { - include: [ - { - model: VideoModel.unscoped(), - required: true - } - ] - }, - [ScopeNames.WITH_VIDEO_OR_PLAYLIST]: (options: { whereVideo?: WhereOptions } = {}) => { - return { - include: [ - { - model: VideoModel.unscoped(), - required: false, - where: options.whereVideo - }, - { - model: VideoStreamingPlaylistModel.unscoped(), - required: false, - include: [ - { - model: VideoModel.unscoped(), - required: true, - where: options.whereVideo - } - ] - } - ] - } - }, - [ScopeNames.WITH_METADATA]: { - attributes: { - include: [ 'metadata' ] - } - } -})) -@Table({ - tableName: 'videoFile', - indexes: [ - { - fields: [ 'videoId' ], - where: { - videoId: { - [Op.ne]: null - } - } - }, - { - fields: [ 'videoStreamingPlaylistId' ], - where: { - videoStreamingPlaylistId: { - [Op.ne]: null - } - } - }, - - { - fields: [ 'infoHash' ] - }, - - { - fields: [ 'torrentFilename' ], - unique: true - }, - - { - fields: [ 'filename' ], - unique: true - }, - - { - fields: [ 'videoId', 'resolution', 'fps' ], - unique: true, - where: { - videoId: { - [Op.ne]: null - } - } - }, - { - fields: [ 'videoStreamingPlaylistId', 'resolution', 'fps' ], - unique: true, - where: { - videoStreamingPlaylistId: { - [Op.ne]: null - } - } - } - ] -}) -export class VideoFileModel extends Model>> { - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @AllowNull(false) - @Is('VideoFileResolution', value => throwIfNotValid(value, isVideoFileResolutionValid, 'resolution')) - @Column - resolution: number - - @AllowNull(false) - @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileSizeValid, 'size')) - @Column(DataType.BIGINT) - size: number - - @AllowNull(false) - @Is('VideoFileExtname', value => throwIfNotValid(value, isVideoFileExtnameValid, 'extname')) - @Column - extname: string - - @AllowNull(true) - @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash', true)) - @Column - infoHash: string - - @AllowNull(false) - @Default(-1) - @Is('VideoFileFPS', value => throwIfNotValid(value, isVideoFPSResolutionValid, 'fps')) - @Column - fps: number - - @AllowNull(true) - @Column(DataType.JSONB) - metadata: any - - @AllowNull(true) - @Column - metadataUrl: string - - // Could be null for remote files - @AllowNull(true) - @Column - fileUrl: string - - // Could be null for live files - @AllowNull(true) - @Column - filename: string - - // Could be null for remote files - @AllowNull(true) - @Column - torrentUrl: string - - // Could be null for live files - @AllowNull(true) - @Column - torrentFilename: string - - @ForeignKey(() => VideoModel) - @Column - videoId: number - - @AllowNull(false) - @Default(VideoStorage.FILE_SYSTEM) - @Column - storage: VideoStorage - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'CASCADE' - }) - Video: VideoModel - - @ForeignKey(() => VideoStreamingPlaylistModel) - @Column - videoStreamingPlaylistId: number - - @BelongsTo(() => VideoStreamingPlaylistModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'CASCADE' - }) - VideoStreamingPlaylist: VideoStreamingPlaylistModel - - @HasMany(() => VideoRedundancyModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'CASCADE', - hooks: true - }) - RedundancyVideos: VideoRedundancyModel[] - - static doesInfohashExistCached = memoizee(VideoFileModel.doesInfohashExist, { - promise: true, - max: MEMOIZE_LENGTH.INFO_HASH_EXISTS, - maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS - }) - - static doesInfohashExist (infoHash: string) { - const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1' - - return doesExist(this.sequelize, query, { infoHash }) - } - - static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) { - const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID) - - return !!videoFile - } - - static async doesOwnedTorrentFileExist (filename: string) { - const query = 'SELECT 1 FROM "videoFile" ' + - 'LEFT JOIN "video" "webvideo" ON "webvideo"."id" = "videoFile"."videoId" AND "webvideo"."remote" IS FALSE ' + - 'LEFT JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ' + - 'LEFT JOIN "video" "hlsVideo" ON "hlsVideo"."id" = "videoStreamingPlaylist"."videoId" AND "hlsVideo"."remote" IS FALSE ' + - 'WHERE "torrentFilename" = $filename AND ("hlsVideo"."id" IS NOT NULL OR "webvideo"."id" IS NOT NULL) LIMIT 1' - - return doesExist(this.sequelize, query, { filename }) - } - - static async doesOwnedWebVideoFileExist (filename: string) { - const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' + - `WHERE "filename" = $filename AND "storage" = ${VideoStorage.FILE_SYSTEM} LIMIT 1` - - return doesExist(this.sequelize, query, { filename }) - } - - static loadByFilename (filename: string) { - const query = { - where: { - filename - } - } - - return VideoFileModel.findOne(query) - } - - static loadWithVideoByFilename (filename: string): Promise { - const query = { - where: { - filename - } - } - - return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query) - } - - static loadWithVideoOrPlaylistByTorrentFilename (filename: string) { - const query = { - where: { - torrentFilename: filename - } - } - - return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query) - } - - static load (id: number): Promise { - return VideoFileModel.findByPk(id) - } - - static loadWithMetadata (id: number) { - return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id) - } - - static loadWithVideo (id: number) { - return VideoFileModel.scope(ScopeNames.WITH_VIDEO).findByPk(id) - } - - static loadWithVideoOrPlaylist (id: number, videoIdOrUUID: number | string) { - const whereVideo = validator.isUUID(videoIdOrUUID + '') - ? { uuid: videoIdOrUUID } - : { id: videoIdOrUUID } - - const options = { - where: { - id - } - } - - return VideoFileModel.scope({ method: [ ScopeNames.WITH_VIDEO_OR_PLAYLIST, whereVideo ] }) - .findOne(options) - .then(file => { - // We used `required: false` so check we have at least a video or a streaming playlist - if (!file.Video && !file.VideoStreamingPlaylist) return null - - return file - }) - } - - static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Transaction) { - const query = { - include: [ - { - model: VideoModel.unscoped(), - required: true, - include: [ - { - model: VideoStreamingPlaylistModel.unscoped(), - required: true, - where: { - id: streamingPlaylistId - } - } - ] - } - ], - transaction - } - - return VideoFileModel.findAll(query) - } - - static getStats () { - const webVideoFilesQuery: FindOptions = { - include: [ - { - attributes: [], - required: true, - model: VideoModel.unscoped(), - where: { - remote: false - } - } - ] - } - - const hlsFilesQuery: FindOptions = { - include: [ - { - attributes: [], - required: true, - model: VideoStreamingPlaylistModel.unscoped(), - include: [ - { - attributes: [], - model: VideoModel.unscoped(), - required: true, - where: { - remote: false - } - } - ] - } - ] - } - - return Promise.all([ - VideoFileModel.aggregate('size', 'SUM', webVideoFilesQuery), - VideoFileModel.aggregate('size', 'SUM', hlsFilesQuery) - ]).then(([ webVideoResult, hlsResult ]) => ({ - totalLocalVideoFilesSize: parseAggregateResult(webVideoResult) + parseAggregateResult(hlsResult) - })) - } - - // Redefine upsert because sequelize does not use an appropriate where clause in the update query with 2 unique indexes - static async customUpsert ( - videoFile: MVideoFile, - mode: 'streaming-playlist' | 'video', - transaction: Transaction - ) { - const baseFind = { - fps: videoFile.fps, - resolution: videoFile.resolution, - transaction - } - - const element = mode === 'streaming-playlist' - ? await VideoFileModel.loadHLSFile({ ...baseFind, playlistId: videoFile.videoStreamingPlaylistId }) - : await VideoFileModel.loadWebVideoFile({ ...baseFind, videoId: videoFile.videoId }) - - if (!element) return videoFile.save({ transaction }) - - for (const k of Object.keys(videoFile.toJSON())) { - element.set(k, videoFile[k]) - } - - return element.save({ transaction }) - } - - static async loadWebVideoFile (options: { - videoId: number - fps: number - resolution: number - transaction?: Transaction - }) { - const where = { - fps: options.fps, - resolution: options.resolution, - videoId: options.videoId - } - - return VideoFileModel.findOne({ where, transaction: options.transaction }) - } - - static async loadHLSFile (options: { - playlistId: number - fps: number - resolution: number - transaction?: Transaction - }) { - const where = { - fps: options.fps, - resolution: options.resolution, - videoStreamingPlaylistId: options.playlistId - } - - return VideoFileModel.findOne({ where, transaction: options.transaction }) - } - - static removeHLSFilesOfVideoId (videoStreamingPlaylistId: number) { - const options = { - where: { videoStreamingPlaylistId } - } - - return VideoFileModel.destroy(options) - } - - hasTorrent () { - return this.infoHash && this.torrentFilename - } - - getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo { - if (this.videoId || (this as MVideoFileVideo).Video) return (this as MVideoFileVideo).Video - - return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist - } - - getVideo (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo { - return extractVideo(this.getVideoOrStreamingPlaylist()) - } - - isAudio () { - return this.resolution === VideoResolution.H_NOVIDEO - } - - isLive () { - return this.size === -1 - } - - isHLS () { - return !!this.videoStreamingPlaylistId - } - - // --------------------------------------------------------------------------- - - getObjectStorageUrl (video: MVideo) { - if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) { - return this.getPrivateObjectStorageUrl(video) - } - - return this.getPublicObjectStorageUrl() - } - - private getPrivateObjectStorageUrl (video: MVideo) { - if (this.isHLS()) { - return getHLSPrivateFileUrl(video, this.filename) - } - - return getWebVideoPrivateFileUrl(this.filename) - } - - private getPublicObjectStorageUrl () { - if (this.isHLS()) { - return getHLSPublicFileUrl(this.fileUrl) - } - - return getWebVideoPublicFileUrl(this.fileUrl) - } - - // --------------------------------------------------------------------------- - - getFileUrl (video: MVideo) { - if (video.isOwned()) { - if (this.storage === VideoStorage.OBJECT_STORAGE) { - return this.getObjectStorageUrl(video) - } - - return WEBSERVER.URL + this.getFileStaticPath(video) - } - - return this.fileUrl - } - - // --------------------------------------------------------------------------- - - getFileStaticPath (video: MVideo) { - if (this.isHLS()) return this.getHLSFileStaticPath(video) - - return this.getWebVideoFileStaticPath(video) - } - - private getWebVideoFileStaticPath (video: MVideo) { - if (isVideoInPrivateDirectory(video.privacy)) { - return join(STATIC_PATHS.PRIVATE_WEB_VIDEOS, this.filename) - } - - return join(STATIC_PATHS.WEB_VIDEOS, this.filename) - } - - private getHLSFileStaticPath (video: MVideo) { - if (isVideoInPrivateDirectory(video.privacy)) { - return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.filename) - } - - return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename) - } - - // --------------------------------------------------------------------------- - - getFileDownloadUrl (video: MVideoWithHost) { - const path = this.isHLS() - ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`) - : join(STATIC_DOWNLOAD_PATHS.VIDEOS, `${video.uuid}-${this.resolution}${this.extname}`) - - if (video.isOwned()) return WEBSERVER.URL + path - - // FIXME: don't guess remote URL - return buildRemoteVideoBaseUrl(video, path) - } - - getRemoteTorrentUrl (video: MVideo) { - if (video.isOwned()) throw new Error(`Video ${video.url} is not a remote video`) - - return this.torrentUrl - } - - // We proxify torrent requests so use a local URL - getTorrentUrl () { - if (!this.torrentFilename) return null - - return WEBSERVER.URL + this.getTorrentStaticPath() - } - - getTorrentStaticPath () { - if (!this.torrentFilename) return null - - return join(LAZY_STATIC_PATHS.TORRENTS, this.torrentFilename) - } - - getTorrentDownloadUrl () { - if (!this.torrentFilename) return null - - return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.TORRENTS, this.torrentFilename) - } - - removeTorrent () { - if (!this.torrentFilename) return null - - const torrentPath = getFSTorrentFilePath(this) - return remove(torrentPath) - .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) - } - - hasSameUniqueKeysThan (other: MVideoFile) { - return this.fps === other.fps && - this.resolution === other.resolution && - ( - (this.videoId !== null && this.videoId === other.videoId) || - (this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId) - ) - } - - withVideoOrPlaylist (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) { - if (isStreamingPlaylist(videoOrPlaylist)) return Object.assign(this, { VideoStreamingPlaylist: videoOrPlaylist }) - - return Object.assign(this, { Video: videoOrPlaylist }) - } -} diff --git a/server/models/video/video-import.ts b/server/models/video/video-import.ts deleted file mode 100644 index c040e0fda..000000000 --- a/server/models/video/video-import.ts +++ /dev/null @@ -1,267 +0,0 @@ -import { IncludeOptions, Op, WhereOptions } from 'sequelize' -import { - AfterUpdate, - AllowNull, - BelongsTo, - Column, - CreatedAt, - DataType, - Default, - DefaultScope, - ForeignKey, - Is, - Model, - Table, - UpdatedAt -} from 'sequelize-typescript' -import { afterCommitIfTransaction } from '@server/helpers/database-utils' -import { MVideoImportDefault, MVideoImportFormattable } from '@server/types/models/video/video-import' -import { VideoImport, VideoImportState } from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { isVideoImportStateValid, isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports' -import { isVideoMagnetUriValid } from '../../helpers/custom-validators/videos' -import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers/constants' -import { UserModel } from '../user/user' -import { getSort, searchAttribute, throwIfNotValid } from '../shared' -import { ScopeNames as VideoModelScopeNames, VideoModel } from './video' -import { VideoChannelSyncModel } from './video-channel-sync' - -const defaultVideoScope = () => { - return VideoModel.scope([ - VideoModelScopeNames.WITH_ACCOUNT_DETAILS, - VideoModelScopeNames.WITH_TAGS, - VideoModelScopeNames.WITH_THUMBNAILS - ]) -} - -@DefaultScope(() => ({ - include: [ - { - model: UserModel.unscoped(), - required: true - }, - { - model: defaultVideoScope(), - required: false - }, - { - model: VideoChannelSyncModel.unscoped(), - required: false - } - ] -})) - -@Table({ - tableName: 'videoImport', - indexes: [ - { - fields: [ 'videoId' ], - unique: true - }, - { - fields: [ 'userId' ] - } - ] -}) -export class VideoImportModel extends Model>> { - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @AllowNull(true) - @Default(null) - @Is('VideoImportTargetUrl', value => throwIfNotValid(value, isVideoImportTargetUrlValid, 'targetUrl', true)) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max)) - targetUrl: string - - @AllowNull(true) - @Default(null) - @Is('VideoImportMagnetUri', value => throwIfNotValid(value, isVideoMagnetUriValid, 'magnetUri', true)) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max)) // Use the same constraints than URLs - magnetUri: string - - @AllowNull(true) - @Default(null) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_NAME.max)) - torrentName: string - - @AllowNull(false) - @Default(null) - @Is('VideoImportState', value => throwIfNotValid(value, isVideoImportStateValid, 'state')) - @Column - state: VideoImportState - - @AllowNull(true) - @Default(null) - @Column(DataType.TEXT) - error: string - - @ForeignKey(() => UserModel) - @Column - userId: number - - @BelongsTo(() => UserModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade' - }) - User: UserModel - - @ForeignKey(() => VideoModel) - @Column - videoId: number - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'set null' - }) - Video: VideoModel - - @ForeignKey(() => VideoChannelSyncModel) - @Column - videoChannelSyncId: number - - @BelongsTo(() => VideoChannelSyncModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'set null' - }) - VideoChannelSync: VideoChannelSyncModel - - @AfterUpdate - static deleteVideoIfFailed (instance: VideoImportModel, options) { - if (instance.state === VideoImportState.FAILED) { - return afterCommitIfTransaction(options.transaction, () => instance.Video.destroy()) - } - - return undefined - } - - static loadAndPopulateVideo (id: number): Promise { - return VideoImportModel.findByPk(id) - } - - static listUserVideoImportsForApi (options: { - userId: number - start: number - count: number - sort: string - - search?: string - targetUrl?: string - videoChannelSyncId?: number - }) { - const { userId, start, count, sort, targetUrl, videoChannelSyncId, search } = options - - const where: WhereOptions = { userId } - const include: IncludeOptions[] = [ - { - attributes: [ 'id' ], - model: UserModel.unscoped(), // FIXME: Without this, sequelize try to COUNT(DISTINCT(*)) which is an invalid SQL query - required: true - }, - { - model: VideoChannelSyncModel.unscoped(), - required: false - } - ] - - if (targetUrl) where['targetUrl'] = targetUrl - if (videoChannelSyncId) where['videoChannelSyncId'] = videoChannelSyncId - - if (search) { - include.push({ - model: defaultVideoScope(), - required: true, - where: searchAttribute(search, 'name') - }) - } else { - include.push({ - model: defaultVideoScope(), - required: false - }) - } - - const query = { - distinct: true, - include, - offset: start, - limit: count, - order: getSort(sort), - where - } - - return Promise.all([ - VideoImportModel.unscoped().count(query), - VideoImportModel.findAll(query) - ]).then(([ total, data ]) => ({ total, data })) - } - - static async urlAlreadyImported (channelId: number, targetUrl: string): Promise { - const element = await VideoImportModel.unscoped().findOne({ - where: { - targetUrl, - state: { - [Op.in]: [ VideoImportState.PENDING, VideoImportState.PROCESSING, VideoImportState.SUCCESS ] - } - }, - include: [ - { - model: VideoModel, - required: true, - where: { - channelId - } - } - ] - }) - - return !!element - } - - getTargetIdentifier () { - return this.targetUrl || this.magnetUri || this.torrentName - } - - toFormattedJSON (this: MVideoImportFormattable): VideoImport { - const videoFormatOptions = { - completeDescription: true, - additionalAttributes: { state: true, waitTranscoding: true, scheduledUpdate: true } - } - const video = this.Video - ? Object.assign(this.Video.toFormattedJSON(videoFormatOptions), { tags: this.Video.Tags.map(t => t.name) }) - : undefined - - const videoChannelSync = this.VideoChannelSync - ? { id: this.VideoChannelSync.id, externalChannelUrl: this.VideoChannelSync.externalChannelUrl } - : undefined - - return { - id: this.id, - - targetUrl: this.targetUrl, - magnetUri: this.magnetUri, - torrentName: this.torrentName, - - state: { - id: this.state, - label: VideoImportModel.getStateLabel(this.state) - }, - error: this.error, - updatedAt: this.updatedAt.toISOString(), - createdAt: this.createdAt.toISOString(), - video, - videoChannelSync - } - } - - private static getStateLabel (id: number) { - return VIDEO_IMPORT_STATES[id] || 'Unknown' - } -} diff --git a/server/models/video/video-job-info.ts b/server/models/video/video-job-info.ts deleted file mode 100644 index 5845b8c74..000000000 --- a/server/models/video/video-job-info.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { Op, QueryTypes, Transaction } from 'sequelize' -import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, IsInt, Model, Table, Unique, UpdatedAt } from 'sequelize-typescript' -import { forceNumber } from '@shared/core-utils' -import { AttributesOnly } from '@shared/typescript-utils' -import { VideoModel } from './video' - -export type VideoJobInfoColumnType = 'pendingMove' | 'pendingTranscode' - -@Table({ - tableName: 'videoJobInfo', - indexes: [ - { - fields: [ 'videoId' ], - where: { - videoId: { - [Op.ne]: null - } - } - } - ] -}) - -export class VideoJobInfoModel extends Model>> { - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @AllowNull(false) - @Default(0) - @IsInt - @Column - pendingMove: number - - @AllowNull(false) - @Default(0) - @IsInt - @Column - pendingTranscode: number - - @ForeignKey(() => VideoModel) - @Unique - @Column - videoId: number - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade' - }) - Video: VideoModel - - static load (videoId: number, transaction?: Transaction) { - const where = { - videoId - } - - return VideoJobInfoModel.findOne({ where, transaction }) - } - - static async increaseOrCreate (videoUUID: string, column: VideoJobInfoColumnType, amountArg = 1): Promise { - const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } } - const amount = forceNumber(amountArg) - - const [ result ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(` - INSERT INTO "videoJobInfo" ("videoId", "${column}", "createdAt", "updatedAt") - SELECT - "video"."id" AS "videoId", ${amount}, NOW(), NOW() - FROM - "video" - WHERE - "video"."uuid" = $videoUUID - ON CONFLICT ("videoId") DO UPDATE - SET - "${column}" = "videoJobInfo"."${column}" + ${amount}, - "updatedAt" = NOW() - RETURNING - "${column}" - `, options) - - return result[column] - } - - static async decrease (videoUUID: string, column: VideoJobInfoColumnType): Promise { - const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } } - - const result = await VideoJobInfoModel.sequelize.query(` - UPDATE - "videoJobInfo" - SET - "${column}" = "videoJobInfo"."${column}" - 1, - "updatedAt" = NOW() - FROM "video" - WHERE - "video"."id" = "videoJobInfo"."videoId" AND "video"."uuid" = $videoUUID - RETURNING - "${column}"; - `, options) - - if (result.length === 0) return undefined - - return result[0][column] - } - - static async abortAllTasks (videoUUID: string, column: VideoJobInfoColumnType): Promise { - const options = { type: QueryTypes.UPDATE as QueryTypes.UPDATE, bind: { videoUUID } } - - await VideoJobInfoModel.sequelize.query(` - UPDATE - "videoJobInfo" - SET - "${column}" = 0, - "updatedAt" = NOW() - FROM "video" - WHERE - "video"."id" = "videoJobInfo"."videoId" AND "video"."uuid" = $videoUUID - `, options) - } -} diff --git a/server/models/video/video-live-replay-setting.ts b/server/models/video/video-live-replay-setting.ts deleted file mode 100644 index 1c824dfa2..000000000 --- a/server/models/video/video-live-replay-setting.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { isVideoPrivacyValid } from '@server/helpers/custom-validators/videos' -import { MLiveReplaySetting } from '@server/types/models/video/video-live-replay-setting' -import { VideoPrivacy } from '@shared/models/videos/video-privacy.enum' -import { Transaction } from 'sequelize' -import { AllowNull, Column, CreatedAt, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { throwIfNotValid } from '../shared/sequelize-helpers' - -@Table({ - tableName: 'videoLiveReplaySetting' -}) -export class VideoLiveReplaySettingModel extends Model { - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @AllowNull(false) - @Is('VideoPrivacy', value => throwIfNotValid(value, isVideoPrivacyValid, 'privacy')) - @Column - privacy: VideoPrivacy - - static load (id: number, transaction?: Transaction): Promise { - return VideoLiveReplaySettingModel.findOne({ - where: { id }, - transaction - }) - } - - static removeSettings (id: number) { - return VideoLiveReplaySettingModel.destroy({ - where: { id } - }) - } - - toFormattedJSON () { - return { - privacy: this.privacy - } - } -} diff --git a/server/models/video/video-live-session.ts b/server/models/video/video-live-session.ts deleted file mode 100644 index 9426f5d11..000000000 --- a/server/models/video/video-live-session.ts +++ /dev/null @@ -1,217 +0,0 @@ -import { FindOptions } from 'sequelize' -import { - AllowNull, - BeforeDestroy, - BelongsTo, - Column, - CreatedAt, - DataType, - ForeignKey, - Model, - Scopes, - Table, - UpdatedAt -} from 'sequelize-typescript' -import { MVideoLiveSession, MVideoLiveSessionReplay } from '@server/types/models' -import { uuidToShort } from '@shared/extra-utils' -import { LiveVideoError, LiveVideoSession } from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { VideoModel } from './video' -import { VideoLiveReplaySettingModel } from './video-live-replay-setting' - -export enum ScopeNames { - WITH_REPLAY = 'WITH_REPLAY' -} - -@Scopes(() => ({ - [ScopeNames.WITH_REPLAY]: { - include: [ - { - model: VideoModel.unscoped(), - as: 'ReplayVideo', - required: false - }, - { - model: VideoLiveReplaySettingModel, - required: false - } - ] - } -})) -@Table({ - tableName: 'videoLiveSession', - indexes: [ - { - fields: [ 'replayVideoId' ], - unique: true - }, - { - fields: [ 'liveVideoId' ] - }, - { - fields: [ 'replaySettingId' ], - unique: true - } - ] -}) -export class VideoLiveSessionModel extends Model>> { - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @AllowNull(false) - @Column(DataType.DATE) - startDate: Date - - @AllowNull(true) - @Column(DataType.DATE) - endDate: Date - - @AllowNull(true) - @Column - error: LiveVideoError - - @AllowNull(false) - @Column - saveReplay: boolean - - @AllowNull(false) - @Column - endingProcessed: boolean - - @ForeignKey(() => VideoModel) - @Column - replayVideoId: number - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: true, - name: 'replayVideoId' - }, - as: 'ReplayVideo', - onDelete: 'set null' - }) - ReplayVideo: VideoModel - - @ForeignKey(() => VideoModel) - @Column - liveVideoId: number - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: true, - name: 'liveVideoId' - }, - as: 'LiveVideo', - onDelete: 'set null' - }) - LiveVideo: VideoModel - - @ForeignKey(() => VideoLiveReplaySettingModel) - @Column - replaySettingId: number - - @BelongsTo(() => VideoLiveReplaySettingModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'set null' - }) - ReplaySetting: VideoLiveReplaySettingModel - - @BeforeDestroy - static deleteReplaySetting (instance: VideoLiveSessionModel) { - return VideoLiveReplaySettingModel.destroy({ - where: { - id: instance.replaySettingId - } - }) - } - - static load (id: number): Promise { - return VideoLiveSessionModel.findOne({ - where: { id } - }) - } - - static findSessionOfReplay (replayVideoId: number) { - const query = { - where: { - replayVideoId - } - } - - return VideoLiveSessionModel.scope(ScopeNames.WITH_REPLAY).findOne(query) - } - - static findCurrentSessionOf (videoUUID: string) { - return VideoLiveSessionModel.findOne({ - where: { - endDate: null - }, - include: [ - { - model: VideoModel.unscoped(), - as: 'LiveVideo', - required: true, - where: { - uuid: videoUUID - } - } - ], - order: [ [ 'startDate', 'DESC' ] ] - }) - } - - static findLatestSessionOf (videoId: number) { - return VideoLiveSessionModel.findOne({ - where: { - liveVideoId: videoId - }, - order: [ [ 'startDate', 'DESC' ] ] - }) - } - - static listSessionsOfLiveForAPI (options: { videoId: number }) { - const { videoId } = options - - const query: FindOptions> = { - where: { - liveVideoId: videoId - }, - order: [ [ 'startDate', 'ASC' ] ] - } - - return VideoLiveSessionModel.scope(ScopeNames.WITH_REPLAY).findAll(query) - } - - toFormattedJSON (this: MVideoLiveSessionReplay): LiveVideoSession { - const replayVideo = this.ReplayVideo - ? { - id: this.ReplayVideo.id, - uuid: this.ReplayVideo.uuid, - shortUUID: uuidToShort(this.ReplayVideo.uuid) - } - : undefined - - const replaySettings = this.replaySettingId - ? this.ReplaySetting.toFormattedJSON() - : undefined - - return { - id: this.id, - startDate: this.startDate.toISOString(), - endDate: this.endDate - ? this.endDate.toISOString() - : null, - endingProcessed: this.endingProcessed, - saveReplay: this.saveReplay, - replaySettings, - replayVideo, - error: this.error - } - } -} diff --git a/server/models/video/video-live.ts b/server/models/video/video-live.ts deleted file mode 100644 index ca1118641..000000000 --- a/server/models/video/video-live.ts +++ /dev/null @@ -1,184 +0,0 @@ -import { Transaction } from 'sequelize' -import { - AllowNull, - BeforeDestroy, - BelongsTo, - Column, - CreatedAt, - DataType, - DefaultScope, - ForeignKey, - Model, - Table, - UpdatedAt -} from 'sequelize-typescript' -import { CONFIG } from '@server/initializers/config' -import { WEBSERVER } from '@server/initializers/constants' -import { MVideoLive, MVideoLiveVideoWithSetting } from '@server/types/models' -import { LiveVideo, LiveVideoLatencyMode, VideoState } from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { VideoModel } from './video' -import { VideoBlacklistModel } from './video-blacklist' -import { VideoLiveReplaySettingModel } from './video-live-replay-setting' - -@DefaultScope(() => ({ - include: [ - { - model: VideoModel, - required: true, - include: [ - { - model: VideoBlacklistModel, - required: false - } - ] - }, - { - model: VideoLiveReplaySettingModel, - required: false - } - ] -})) -@Table({ - tableName: 'videoLive', - indexes: [ - { - fields: [ 'videoId' ], - unique: true - }, - { - fields: [ 'replaySettingId' ], - unique: true - } - ] -}) -export class VideoLiveModel extends Model>> { - - @AllowNull(true) - @Column(DataType.STRING) - streamKey: string - - @AllowNull(false) - @Column - saveReplay: boolean - - @AllowNull(false) - @Column - permanentLive: boolean - - @AllowNull(false) - @Column - latencyMode: LiveVideoLatencyMode - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => VideoModel) - @Column - videoId: number - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade' - }) - Video: VideoModel - - @ForeignKey(() => VideoLiveReplaySettingModel) - @Column - replaySettingId: number - - @BelongsTo(() => VideoLiveReplaySettingModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'set null' - }) - ReplaySetting: VideoLiveReplaySettingModel - - @BeforeDestroy - static deleteReplaySetting (instance: VideoLiveModel, options: { transaction: Transaction }) { - return VideoLiveReplaySettingModel.destroy({ - where: { - id: instance.replaySettingId - }, - transaction: options.transaction - }) - } - - static loadByStreamKey (streamKey: string) { - const query = { - where: { - streamKey - }, - include: [ - { - model: VideoModel.unscoped(), - required: true, - where: { - state: VideoState.WAITING_FOR_LIVE - }, - include: [ - { - model: VideoBlacklistModel.unscoped(), - required: false - } - ] - }, - { - model: VideoLiveReplaySettingModel.unscoped(), - required: false - } - ] - } - - return VideoLiveModel.findOne(query) - } - - static loadByVideoId (videoId: number) { - const query = { - where: { - videoId - } - } - - return VideoLiveModel.findOne(query) - } - - toFormattedJSON (canSeePrivateInformation: boolean): LiveVideo { - let privateInformation: Pick | {} = {} - - // If we don't have a stream key, it means this is a remote live so we don't specify the rtmp URL - // We also display these private information only to the live owne/moderators - if (this.streamKey && canSeePrivateInformation === true) { - privateInformation = { - streamKey: this.streamKey, - - rtmpUrl: CONFIG.LIVE.RTMP.ENABLED - ? WEBSERVER.RTMP_BASE_LIVE_URL - : null, - - rtmpsUrl: CONFIG.LIVE.RTMPS.ENABLED - ? WEBSERVER.RTMPS_BASE_LIVE_URL - : null - } - } - - const replaySettings = this.replaySettingId - ? this.ReplaySetting.toFormattedJSON() - : undefined - - return { - ...privateInformation, - - permanentLive: this.permanentLive, - saveReplay: this.saveReplay, - replaySettings, - latencyMode: this.latencyMode - } - } -} diff --git a/server/models/video/video-password.ts b/server/models/video/video-password.ts deleted file mode 100644 index 648366c3b..000000000 --- a/server/models/video/video-password.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { AllowNull, BelongsTo, Column, CreatedAt, DefaultScope, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { VideoModel } from './video' -import { AttributesOnly } from '@shared/typescript-utils' -import { ResultList, VideoPassword } from '@shared/models' -import { getSort, throwIfNotValid } from '../shared' -import { FindOptions, Transaction } from 'sequelize' -import { MVideoPassword } from '@server/types/models' -import { isPasswordValid } from '@server/helpers/custom-validators/videos' -import { pick } from '@shared/core-utils' - -@DefaultScope(() => ({ - include: [ - { - model: VideoModel.unscoped(), - required: true - } - ] -})) -@Table({ - tableName: 'videoPassword', - indexes: [ - { - fields: [ 'videoId', 'password' ], - unique: true - } - ] -}) -export class VideoPasswordModel extends Model>> { - - @AllowNull(false) - @Is('VideoPassword', value => throwIfNotValid(value, isPasswordValid, 'videoPassword')) - @Column - password: string - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => VideoModel) - @Column - videoId: number - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade' - }) - Video: VideoModel - - static async countByVideoId (videoId: number, t?: Transaction) { - const query: FindOptions = { - where: { - videoId - }, - transaction: t - } - - return VideoPasswordModel.count(query) - } - - static async loadByIdAndVideo (options: { id: number, videoId: number, t?: Transaction }): Promise { - const { id, videoId, t } = options - const query: FindOptions = { - where: { - id, - videoId - }, - transaction: t - } - - return VideoPasswordModel.findOne(query) - } - - static async listPasswords (options: { - start: number - count: number - sort: string - videoId: number - }): Promise> { - const { start, count, sort, videoId } = options - - const { count: total, rows: data } = await VideoPasswordModel.findAndCountAll({ - where: { videoId }, - order: getSort(sort), - offset: start, - limit: count - }) - - return { total, data } - } - - static async addPasswords (passwords: string[], videoId: number, transaction?: Transaction): Promise { - for (const password of passwords) { - await VideoPasswordModel.create({ - password, - videoId - }, { transaction }) - } - } - - static async deleteAllPasswords (videoId: number, transaction?: Transaction) { - await VideoPasswordModel.destroy({ - where: { videoId }, - transaction - }) - } - - static async deletePassword (passwordId: number, transaction?: Transaction) { - await VideoPasswordModel.destroy({ - where: { id: passwordId }, - transaction - }) - } - - static async isACorrectPassword (options: { - videoId: number - password: string - }) { - const query = { - where: pick(options, [ 'videoId', 'password' ]) - } - return VideoPasswordModel.findOne(query) - } - - toFormattedJSON (): VideoPassword { - return { - id: this.id, - password: this.password, - videoId: this.videoId, - createdAt: this.createdAt, - updatedAt: this.updatedAt - } - } -} diff --git a/server/models/video/video-playlist-element.ts b/server/models/video/video-playlist-element.ts deleted file mode 100644 index 61ae6b9fe..000000000 --- a/server/models/video/video-playlist-element.ts +++ /dev/null @@ -1,370 +0,0 @@ -import { AggregateOptions, Op, ScopeOptions, Sequelize, Transaction } from 'sequelize' -import { - AllowNull, - BelongsTo, - Column, - CreatedAt, - DataType, - Default, - ForeignKey, - Is, - IsInt, - Min, - Model, - Table, - UpdatedAt -} from 'sequelize-typescript' -import validator from 'validator' -import { MUserAccountId } from '@server/types/models' -import { - MVideoPlaylistElement, - MVideoPlaylistElementAP, - MVideoPlaylistElementFormattable, - MVideoPlaylistElementVideoUrlPlaylistPrivacy, - MVideoPlaylistVideoThumbnail -} from '@server/types/models/video/video-playlist-element' -import { forceNumber } from '@shared/core-utils' -import { AttributesOnly } from '@shared/typescript-utils' -import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object' -import { VideoPrivacy } from '../../../shared/models/videos' -import { VideoPlaylistElement, VideoPlaylistElementType } from '../../../shared/models/videos/playlist/video-playlist-element.model' -import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' -import { CONSTRAINTS_FIELDS } from '../../initializers/constants' -import { AccountModel } from '../account/account' -import { getSort, throwIfNotValid } from '../shared' -import { ForAPIOptions, ScopeNames as VideoScopeNames, VideoModel } from './video' -import { VideoPlaylistModel } from './video-playlist' - -@Table({ - tableName: 'videoPlaylistElement', - indexes: [ - { - fields: [ 'videoPlaylistId' ] - }, - { - fields: [ 'videoId' ] - }, - { - fields: [ 'url' ], - unique: true - } - ] -}) -export class VideoPlaylistElementModel extends Model>> { - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @AllowNull(true) - @Is('VideoPlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url', true)) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_PLAYLISTS.URL.max)) - url: string - - @AllowNull(false) - @Default(1) - @IsInt - @Min(1) - @Column - position: number - - @AllowNull(true) - @IsInt - @Min(0) - @Column - startTimestamp: number - - @AllowNull(true) - @IsInt - @Min(0) - @Column - stopTimestamp: number - - @ForeignKey(() => VideoPlaylistModel) - @Column - videoPlaylistId: number - - @BelongsTo(() => VideoPlaylistModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'CASCADE' - }) - VideoPlaylist: VideoPlaylistModel - - @ForeignKey(() => VideoModel) - @Column - videoId: number - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'set null' - }) - Video: VideoModel - - static deleteAllOf (videoPlaylistId: number, transaction?: Transaction) { - const query = { - where: { - videoPlaylistId - }, - transaction - } - - return VideoPlaylistElementModel.destroy(query) - } - - static listForApi (options: { - start: number - count: number - videoPlaylistId: number - serverAccount: AccountModel - user?: MUserAccountId - }) { - const accountIds = [ options.serverAccount.id ] - const videoScope: (ScopeOptions | string)[] = [ - VideoScopeNames.WITH_BLACKLISTED - ] - - if (options.user) { - accountIds.push(options.user.Account.id) - videoScope.push({ method: [ VideoScopeNames.WITH_USER_HISTORY, options.user.id ] }) - } - - const forApiOptions: ForAPIOptions = { withAccountBlockerIds: accountIds } - videoScope.push({ - method: [ - VideoScopeNames.FOR_API, forApiOptions - ] - }) - - const findQuery = { - offset: options.start, - limit: options.count, - order: getSort('position'), - where: { - videoPlaylistId: options.videoPlaylistId - }, - include: [ - { - model: VideoModel.scope(videoScope), - required: false - } - ] - } - - const countQuery = { - where: { - videoPlaylistId: options.videoPlaylistId - } - } - - return Promise.all([ - VideoPlaylistElementModel.count(countQuery), - VideoPlaylistElementModel.findAll(findQuery) - ]).then(([ total, data ]) => ({ total, data })) - } - - static loadByPlaylistAndVideo (videoPlaylistId: number, videoId: number): Promise { - const query = { - where: { - videoPlaylistId, - videoId - } - } - - return VideoPlaylistElementModel.findOne(query) - } - - static loadById (playlistElementId: number | string): Promise { - return VideoPlaylistElementModel.findByPk(playlistElementId) - } - - static loadByPlaylistAndElementIdForAP ( - playlistId: number | string, - playlistElementId: number - ): Promise { - const playlistWhere = validator.isUUID('' + playlistId) - ? { uuid: playlistId } - : { id: playlistId } - - const query = { - include: [ - { - attributes: [ 'privacy' ], - model: VideoPlaylistModel.unscoped(), - where: playlistWhere - }, - { - attributes: [ 'url' ], - model: VideoModel.unscoped() - } - ], - where: { - id: playlistElementId - } - } - - return VideoPlaylistElementModel.findOne(query) - } - - static listUrlsOfForAP (videoPlaylistId: number, start: number, count: number, t?: Transaction) { - const getQuery = (forCount: boolean) => { - return { - attributes: forCount - ? [] - : [ 'url' ], - offset: start, - limit: count, - order: getSort('position'), - where: { - videoPlaylistId - }, - transaction: t - } - } - - return Promise.all([ - VideoPlaylistElementModel.count(getQuery(true)), - VideoPlaylistElementModel.findAll(getQuery(false)) - ]).then(([ total, rows ]) => ({ - total, - data: rows.map(e => e.url) - })) - } - - static loadFirstElementWithVideoThumbnail (videoPlaylistId: number): Promise { - const query = { - order: getSort('position'), - where: { - videoPlaylistId - }, - include: [ - { - model: VideoModel.scope(VideoScopeNames.WITH_THUMBNAILS), - required: true - } - ] - } - - return VideoPlaylistElementModel - .findOne(query) - } - - static getNextPositionOf (videoPlaylistId: number, transaction?: Transaction) { - const query: AggregateOptions = { - where: { - videoPlaylistId - }, - transaction - } - - return VideoPlaylistElementModel.max('position', query) - .then(position => position ? position + 1 : 1) - } - - static reassignPositionOf (options: { - videoPlaylistId: number - firstPosition: number - endPosition: number - newPosition: number - transaction?: Transaction - }) { - const { videoPlaylistId, firstPosition, endPosition, newPosition, transaction } = options - - const query = { - where: { - videoPlaylistId, - position: { - [Op.gte]: firstPosition, - [Op.lte]: endPosition - } - }, - transaction, - validate: false // We use a literal to update the position - } - - const positionQuery = Sequelize.literal(`${forceNumber(newPosition)} + "position" - ${forceNumber(firstPosition)}`) - return VideoPlaylistElementModel.update({ position: positionQuery }, query) - } - - static increasePositionOf ( - videoPlaylistId: number, - fromPosition: number, - by = 1, - transaction?: Transaction - ) { - const query = { - where: { - videoPlaylistId, - position: { - [Op.gte]: fromPosition - } - }, - transaction - } - - return VideoPlaylistElementModel.increment({ position: by }, query) - } - - toFormattedJSON ( - this: MVideoPlaylistElementFormattable, - options: { accountId?: number } = {} - ): VideoPlaylistElement { - return { - id: this.id, - position: this.position, - startTimestamp: this.startTimestamp, - stopTimestamp: this.stopTimestamp, - - type: this.getType(options.accountId), - - video: this.getVideoElement(options.accountId) - } - } - - getType (this: MVideoPlaylistElementFormattable, accountId?: number) { - const video = this.Video - - if (!video) return VideoPlaylistElementType.DELETED - - // Owned video, don't filter it - if (accountId && video.VideoChannel.Account.id === accountId) return VideoPlaylistElementType.REGULAR - - // Internal video? - if (video.privacy === VideoPrivacy.INTERNAL && accountId) return VideoPlaylistElementType.REGULAR - - // Private, internal and password protected videos cannot be read without appropriate access (ownership, internal) - if (new Set([ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL, VideoPrivacy.PASSWORD_PROTECTED ]).has(video.privacy)) { - return VideoPlaylistElementType.PRIVATE - } - - if (video.isBlacklisted() || video.isBlocked()) return VideoPlaylistElementType.UNAVAILABLE - - return VideoPlaylistElementType.REGULAR - } - - getVideoElement (this: MVideoPlaylistElementFormattable, accountId?: number) { - if (!this.Video) return null - if (this.getType(accountId) !== VideoPlaylistElementType.REGULAR) return null - - return this.Video.toFormattedJSON() - } - - toActivityPubObject (this: MVideoPlaylistElementAP): PlaylistElementObject { - const base: PlaylistElementObject = { - id: this.url, - type: 'PlaylistElement', - - url: this.Video?.url || null, - position: this.position - } - - if (this.startTimestamp) base.startTimestamp = this.startTimestamp - if (this.stopTimestamp) base.stopTimestamp = this.stopTimestamp - - return base - } -} diff --git a/server/models/video/video-playlist.ts b/server/models/video/video-playlist.ts deleted file mode 100644 index 15999d409..000000000 --- a/server/models/video/video-playlist.ts +++ /dev/null @@ -1,725 +0,0 @@ -import { join } from 'path' -import { FindOptions, Includeable, literal, Op, ScopeOptions, Sequelize, Transaction, WhereOptions } from 'sequelize' -import { - AllowNull, - BelongsTo, - Column, - CreatedAt, - DataType, - Default, - ForeignKey, - HasMany, - HasOne, - Is, - IsUUID, - Model, - Scopes, - Table, - UpdatedAt -} from 'sequelize-typescript' -import { activityPubCollectionPagination } from '@server/lib/activitypub/collection' -import { MAccountId, MChannelId } from '@server/types/models' -import { buildPlaylistEmbedPath, buildPlaylistWatchPath, pick } from '@shared/core-utils' -import { buildUUID, uuidToShort } from '@shared/extra-utils' -import { ActivityIconObject, PlaylistObject, VideoPlaylist, VideoPlaylistPrivacy, VideoPlaylistType } from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' -import { - isVideoPlaylistDescriptionValid, - isVideoPlaylistNameValid, - isVideoPlaylistPrivacyValid -} from '../../helpers/custom-validators/video-playlists' -import { - ACTIVITY_PUB, - CONSTRAINTS_FIELDS, - LAZY_STATIC_PATHS, - THUMBNAILS_SIZE, - VIDEO_PLAYLIST_PRIVACIES, - VIDEO_PLAYLIST_TYPES, - WEBSERVER -} from '../../initializers/constants' -import { MThumbnail } from '../../types/models/video/thumbnail' -import { - MVideoPlaylistAccountThumbnail, - MVideoPlaylistAP, - MVideoPlaylistFormattable, - MVideoPlaylistFull, - MVideoPlaylistFullSummary, - MVideoPlaylistSummaryWithElements -} from '../../types/models/video/video-playlist' -import { AccountModel, ScopeNames as AccountScopeNames, SummaryOptions } from '../account/account' -import { ActorModel } from '../actor/actor' -import { - buildServerIdsFollowedBy, - buildTrigramSearchIndex, - buildWhereIdOrUUID, - createSimilarityAttribute, - getPlaylistSort, - isOutdated, - setAsUpdated, - throwIfNotValid -} from '../shared' -import { ThumbnailModel } from './thumbnail' -import { ScopeNames as VideoChannelScopeNames, VideoChannelModel } from './video-channel' -import { VideoPlaylistElementModel } from './video-playlist-element' - -enum ScopeNames { - AVAILABLE_FOR_LIST = 'AVAILABLE_FOR_LIST', - WITH_VIDEOS_LENGTH = 'WITH_VIDEOS_LENGTH', - WITH_ACCOUNT_AND_CHANNEL_SUMMARY = 'WITH_ACCOUNT_AND_CHANNEL_SUMMARY', - WITH_ACCOUNT = 'WITH_ACCOUNT', - WITH_THUMBNAIL = 'WITH_THUMBNAIL', - WITH_ACCOUNT_AND_CHANNEL = 'WITH_ACCOUNT_AND_CHANNEL' -} - -type AvailableForListOptions = { - followerActorId?: number - type?: VideoPlaylistType - accountId?: number - videoChannelId?: number - listMyPlaylists?: boolean - search?: string - host?: string - uuids?: string[] - withVideos?: boolean - forCount?: boolean -} - -function getVideoLengthSelect () { - return 'SELECT COUNT("id") FROM "videoPlaylistElement" WHERE "videoPlaylistId" = "VideoPlaylistModel"."id"' -} - -@Scopes(() => ({ - [ScopeNames.WITH_THUMBNAIL]: { - include: [ - { - model: ThumbnailModel, - required: false - } - ] - }, - [ScopeNames.WITH_VIDEOS_LENGTH]: { - attributes: { - include: [ - [ - literal(`(${getVideoLengthSelect()})`), - 'videosLength' - ] - ] - } - } as FindOptions, - [ScopeNames.WITH_ACCOUNT]: { - include: [ - { - model: AccountModel, - required: true - } - ] - }, - [ScopeNames.WITH_ACCOUNT_AND_CHANNEL_SUMMARY]: { - include: [ - { - model: AccountModel.scope(AccountScopeNames.SUMMARY), - required: true - }, - { - model: VideoChannelModel.scope(VideoChannelScopeNames.SUMMARY), - required: false - } - ] - }, - [ScopeNames.WITH_ACCOUNT_AND_CHANNEL]: { - include: [ - { - model: AccountModel, - required: true - }, - { - model: VideoChannelModel, - required: false - } - ] - }, - [ScopeNames.AVAILABLE_FOR_LIST]: (options: AvailableForListOptions) => { - const whereAnd: WhereOptions[] = [] - - const whereServer = options.host && options.host !== WEBSERVER.HOST - ? { host: options.host } - : undefined - - let whereActor: WhereOptions = {} - - if (options.host === WEBSERVER.HOST) { - whereActor = { - [Op.and]: [ { serverId: null } ] - } - } - - if (options.listMyPlaylists !== true) { - whereAnd.push({ - privacy: VideoPlaylistPrivacy.PUBLIC - }) - - // Only list local playlists - const whereActorOr: WhereOptions[] = [ - { - serverId: null - } - ] - - // … OR playlists that are on an instance followed by actorId - if (options.followerActorId) { - const inQueryInstanceFollow = buildServerIdsFollowedBy(options.followerActorId) - - whereActorOr.push({ - serverId: { - [Op.in]: literal(inQueryInstanceFollow) - } - }) - } - - Object.assign(whereActor, { [Op.or]: whereActorOr }) - } - - if (options.accountId) { - whereAnd.push({ - ownerAccountId: options.accountId - }) - } - - if (options.videoChannelId) { - whereAnd.push({ - videoChannelId: options.videoChannelId - }) - } - - if (options.type) { - whereAnd.push({ - type: options.type - }) - } - - if (options.uuids) { - whereAnd.push({ - uuid: { - [Op.in]: options.uuids - } - }) - } - - if (options.withVideos === true) { - whereAnd.push( - literal(`(${getVideoLengthSelect()}) != 0`) - ) - } - - let attributesInclude: any[] = [ literal('0 as similarity') ] - - if (options.search) { - const escapedSearch = VideoPlaylistModel.sequelize.escape(options.search) - const escapedLikeSearch = VideoPlaylistModel.sequelize.escape('%' + options.search + '%') - attributesInclude = [ createSimilarityAttribute('VideoPlaylistModel.name', options.search) ] - - whereAnd.push({ - [Op.or]: [ - Sequelize.literal( - 'lower(immutable_unaccent("VideoPlaylistModel"."name")) % lower(immutable_unaccent(' + escapedSearch + '))' - ), - Sequelize.literal( - 'lower(immutable_unaccent("VideoPlaylistModel"."name")) LIKE lower(immutable_unaccent(' + escapedLikeSearch + '))' - ) - ] - }) - } - - const where = { - [Op.and]: whereAnd - } - - const include: Includeable[] = [ - { - model: AccountModel.scope({ - method: [ AccountScopeNames.SUMMARY, { whereActor, whereServer, forCount: options.forCount } as SummaryOptions ] - }), - required: true - } - ] - - if (options.forCount !== true) { - include.push({ - model: VideoChannelModel.scope(VideoChannelScopeNames.SUMMARY), - required: false - }) - } - - return { - attributes: { - include: attributesInclude - }, - where, - include - } as FindOptions - } -})) - -@Table({ - tableName: 'videoPlaylist', - indexes: [ - buildTrigramSearchIndex('video_playlist_name_trigram', 'name'), - - { - fields: [ 'ownerAccountId' ] - }, - { - fields: [ 'videoChannelId' ] - }, - { - fields: [ 'url' ], - unique: true - } - ] -}) -export class VideoPlaylistModel extends Model>> { - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @AllowNull(false) - @Is('VideoPlaylistName', value => throwIfNotValid(value, isVideoPlaylistNameValid, 'name')) - @Column - name: string - - @AllowNull(true) - @Is('VideoPlaylistDescription', value => throwIfNotValid(value, isVideoPlaylistDescriptionValid, 'description', true)) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_PLAYLISTS.DESCRIPTION.max)) - description: string - - @AllowNull(false) - @Is('VideoPlaylistPrivacy', value => throwIfNotValid(value, isVideoPlaylistPrivacyValid, 'privacy')) - @Column - privacy: VideoPlaylistPrivacy - - @AllowNull(false) - @Is('VideoPlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url')) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_PLAYLISTS.URL.max)) - url: string - - @AllowNull(false) - @Default(DataType.UUIDV4) - @IsUUID(4) - @Column(DataType.UUID) - uuid: string - - @AllowNull(false) - @Default(VideoPlaylistType.REGULAR) - @Column - type: VideoPlaylistType - - @ForeignKey(() => AccountModel) - @Column - ownerAccountId: number - - @BelongsTo(() => AccountModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'CASCADE' - }) - OwnerAccount: AccountModel - - @ForeignKey(() => VideoChannelModel) - @Column - videoChannelId: number - - @BelongsTo(() => VideoChannelModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'CASCADE' - }) - VideoChannel: VideoChannelModel - - @HasMany(() => VideoPlaylistElementModel, { - foreignKey: { - name: 'videoPlaylistId', - allowNull: false - }, - onDelete: 'CASCADE' - }) - VideoPlaylistElements: VideoPlaylistElementModel[] - - @HasOne(() => ThumbnailModel, { - foreignKey: { - name: 'videoPlaylistId', - allowNull: true - }, - onDelete: 'CASCADE', - hooks: true - }) - Thumbnail: ThumbnailModel - - static listForApi (options: AvailableForListOptions & { - start: number - count: number - sort: string - }) { - const query = { - offset: options.start, - limit: options.count, - order: getPlaylistSort(options.sort) - } - - const commonAvailableForListOptions = pick(options, [ - 'type', - 'followerActorId', - 'accountId', - 'videoChannelId', - 'listMyPlaylists', - 'search', - 'host', - 'uuids' - ]) - - const scopesFind: (string | ScopeOptions)[] = [ - { - method: [ - ScopeNames.AVAILABLE_FOR_LIST, - { - ...commonAvailableForListOptions, - - withVideos: options.withVideos || false - } as AvailableForListOptions - ] - }, - ScopeNames.WITH_VIDEOS_LENGTH, - ScopeNames.WITH_THUMBNAIL - ] - - const scopesCount: (string | ScopeOptions)[] = [ - { - method: [ - ScopeNames.AVAILABLE_FOR_LIST, - - { - ...commonAvailableForListOptions, - - withVideos: options.withVideos || false, - forCount: true - } as AvailableForListOptions - ] - }, - ScopeNames.WITH_VIDEOS_LENGTH - ] - - return Promise.all([ - VideoPlaylistModel.scope(scopesCount).count(), - VideoPlaylistModel.scope(scopesFind).findAll(query) - ]).then(([ count, rows ]) => ({ total: count, data: rows })) - } - - static searchForApi (options: Pick & { - start: number - count: number - sort: string - }) { - return VideoPlaylistModel.listForApi({ - ...options, - - type: VideoPlaylistType.REGULAR, - listMyPlaylists: false, - withVideos: true - }) - } - - static listPublicUrlsOfForAP (options: { account?: MAccountId, channel?: MChannelId }, start: number, count: number) { - const where = { - privacy: VideoPlaylistPrivacy.PUBLIC - } - - if (options.account) { - Object.assign(where, { ownerAccountId: options.account.id }) - } - - if (options.channel) { - Object.assign(where, { videoChannelId: options.channel.id }) - } - - const getQuery = (forCount: boolean) => { - return { - attributes: forCount === true - ? [] - : [ 'url' ], - offset: start, - limit: count, - where - } - } - - return Promise.all([ - VideoPlaylistModel.count(getQuery(true)), - VideoPlaylistModel.findAll(getQuery(false)) - ]).then(([ total, rows ]) => ({ - total, - data: rows.map(p => p.url) - })) - } - - static listPlaylistSummariesOf (accountId: number, videoIds: number[]): Promise { - const query = { - attributes: [ 'id', 'name', 'uuid' ], - where: { - ownerAccountId: accountId - }, - include: [ - { - attributes: [ 'id', 'videoId', 'startTimestamp', 'stopTimestamp' ], - model: VideoPlaylistElementModel.unscoped(), - where: { - videoId: { - [Op.in]: videoIds - } - }, - required: true - } - ] - } - - return VideoPlaylistModel.findAll(query) - } - - static doesPlaylistExist (url: string) { - const query = { - attributes: [ 'id' ], - where: { - url - } - } - - return VideoPlaylistModel - .findOne(query) - .then(e => !!e) - } - - static loadWithAccountAndChannelSummary (id: number | string, transaction: Transaction): Promise { - const where = buildWhereIdOrUUID(id) - - const query = { - where, - transaction - } - - return VideoPlaylistModel - .scope([ ScopeNames.WITH_ACCOUNT_AND_CHANNEL_SUMMARY, ScopeNames.WITH_VIDEOS_LENGTH, ScopeNames.WITH_THUMBNAIL ]) - .findOne(query) - } - - static loadWithAccountAndChannel (id: number | string, transaction: Transaction): Promise { - const where = buildWhereIdOrUUID(id) - - const query = { - where, - transaction - } - - return VideoPlaylistModel - .scope([ ScopeNames.WITH_ACCOUNT_AND_CHANNEL, ScopeNames.WITH_VIDEOS_LENGTH, ScopeNames.WITH_THUMBNAIL ]) - .findOne(query) - } - - static loadByUrlAndPopulateAccount (url: string): Promise { - const query = { - where: { - url - } - } - - return VideoPlaylistModel.scope([ ScopeNames.WITH_ACCOUNT, ScopeNames.WITH_THUMBNAIL ]).findOne(query) - } - - static loadByUrlWithAccountAndChannelSummary (url: string): Promise { - const query = { - where: { - url - } - } - - return VideoPlaylistModel - .scope([ ScopeNames.WITH_ACCOUNT_AND_CHANNEL_SUMMARY, ScopeNames.WITH_VIDEOS_LENGTH, ScopeNames.WITH_THUMBNAIL ]) - .findOne(query) - } - - static getPrivacyLabel (privacy: VideoPlaylistPrivacy) { - return VIDEO_PLAYLIST_PRIVACIES[privacy] || 'Unknown' - } - - static getTypeLabel (type: VideoPlaylistType) { - return VIDEO_PLAYLIST_TYPES[type] || 'Unknown' - } - - static resetPlaylistsOfChannel (videoChannelId: number, transaction: Transaction) { - const query = { - where: { - videoChannelId - }, - transaction - } - - return VideoPlaylistModel.update({ privacy: VideoPlaylistPrivacy.PRIVATE, videoChannelId: null }, query) - } - - async setAndSaveThumbnail (thumbnail: MThumbnail, t: Transaction) { - thumbnail.videoPlaylistId = this.id - - this.Thumbnail = await thumbnail.save({ transaction: t }) - } - - hasThumbnail () { - return !!this.Thumbnail - } - - hasGeneratedThumbnail () { - return this.hasThumbnail() && this.Thumbnail.automaticallyGenerated === true - } - - generateThumbnailName () { - const extension = '.jpg' - - return 'playlist-' + buildUUID() + extension - } - - getThumbnailUrl () { - if (!this.hasThumbnail()) return null - - return WEBSERVER.URL + LAZY_STATIC_PATHS.THUMBNAILS + this.Thumbnail.filename - } - - getThumbnailStaticPath () { - if (!this.hasThumbnail()) return null - - return join(LAZY_STATIC_PATHS.THUMBNAILS, this.Thumbnail.filename) - } - - getWatchStaticPath () { - return buildPlaylistWatchPath({ shortUUID: uuidToShort(this.uuid) }) - } - - getEmbedStaticPath () { - return buildPlaylistEmbedPath(this) - } - - static async getStats () { - const totalLocalPlaylists = await VideoPlaylistModel.count({ - include: [ - { - model: AccountModel.unscoped(), - required: true, - include: [ - { - model: ActorModel.unscoped(), - required: true, - where: { - serverId: null - } - } - ] - } - ], - where: { - privacy: VideoPlaylistPrivacy.PUBLIC - } - }) - - return { - totalLocalPlaylists - } - } - - setAsRefreshed () { - return setAsUpdated({ sequelize: this.sequelize, table: 'videoPlaylist', id: this.id }) - } - - setVideosLength (videosLength: number) { - this.set('videosLength' as any, videosLength, { raw: true }) - } - - isOwned () { - return this.OwnerAccount.isOwned() - } - - isOutdated () { - if (this.isOwned()) return false - - return isOutdated(this, ACTIVITY_PUB.VIDEO_PLAYLIST_REFRESH_INTERVAL) - } - - toFormattedJSON (this: MVideoPlaylistFormattable): VideoPlaylist { - return { - id: this.id, - uuid: this.uuid, - shortUUID: uuidToShort(this.uuid), - - isLocal: this.isOwned(), - - url: this.url, - - displayName: this.name, - description: this.description, - privacy: { - id: this.privacy, - label: VideoPlaylistModel.getPrivacyLabel(this.privacy) - }, - - thumbnailPath: this.getThumbnailStaticPath(), - embedPath: this.getEmbedStaticPath(), - - type: { - id: this.type, - label: VideoPlaylistModel.getTypeLabel(this.type) - }, - - videosLength: this.get('videosLength') as number, - - createdAt: this.createdAt, - updatedAt: this.updatedAt, - - ownerAccount: this.OwnerAccount.toFormattedSummaryJSON(), - videoChannel: this.VideoChannel - ? this.VideoChannel.toFormattedSummaryJSON() - : null - } - } - - toActivityPubObject (this: MVideoPlaylistAP, page: number, t: Transaction): Promise { - const handler = (start: number, count: number) => { - return VideoPlaylistElementModel.listUrlsOfForAP(this.id, start, count, t) - } - - let icon: ActivityIconObject - if (this.hasThumbnail()) { - icon = { - type: 'Image' as 'Image', - url: this.getThumbnailUrl(), - mediaType: 'image/jpeg' as 'image/jpeg', - width: THUMBNAILS_SIZE.width, - height: THUMBNAILS_SIZE.height - } - } - - return activityPubCollectionPagination(this.url, handler, page) - .then(o => { - return Object.assign(o, { - type: 'Playlist' as 'Playlist', - name: this.name, - content: this.description, - mediaType: 'text/markdown' as 'text/markdown', - uuid: this.uuid, - published: this.createdAt.toISOString(), - updated: this.updatedAt.toISOString(), - attributedTo: this.VideoChannel ? [ this.VideoChannel.Actor.url ] : [], - icon - }) - }) - } -} diff --git a/server/models/video/video-share.ts b/server/models/video/video-share.ts deleted file mode 100644 index b4de2b20f..000000000 --- a/server/models/video/video-share.ts +++ /dev/null @@ -1,216 +0,0 @@ -import { literal, Op, QueryTypes, Transaction } from 'sequelize' -import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Is, Model, Scopes, Table, UpdatedAt } from 'sequelize-typescript' -import { forceNumber } from '@shared/core-utils' -import { AttributesOnly } from '@shared/typescript-utils' -import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' -import { CONSTRAINTS_FIELDS } from '../../initializers/constants' -import { MActorDefault, MActorFollowersUrl, MActorId } from '../../types/models' -import { MVideoShareActor, MVideoShareFull } from '../../types/models/video' -import { ActorModel } from '../actor/actor' -import { buildLocalActorIdsIn, throwIfNotValid } from '../shared' -import { VideoModel } from './video' - -enum ScopeNames { - FULL = 'FULL', - WITH_ACTOR = 'WITH_ACTOR' -} - -@Scopes(() => ({ - [ScopeNames.FULL]: { - include: [ - { - model: ActorModel, - required: true - }, - { - model: VideoModel, - required: true - } - ] - }, - [ScopeNames.WITH_ACTOR]: { - include: [ - { - model: ActorModel, - required: true - } - ] - } -})) -@Table({ - tableName: 'videoShare', - indexes: [ - { - fields: [ 'actorId' ] - }, - { - fields: [ 'videoId' ] - }, - { - fields: [ 'url' ], - unique: true - } - ] -}) -export class VideoShareModel extends Model>> { - - @AllowNull(false) - @Is('VideoShareUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url')) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_SHARE.URL.max)) - url: string - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => ActorModel) - @Column - actorId: number - - @BelongsTo(() => ActorModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade' - }) - Actor: ActorModel - - @ForeignKey(() => VideoModel) - @Column - videoId: number - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade' - }) - Video: VideoModel - - static load (actorId: number | string, videoId: number | string, t?: Transaction): Promise { - return VideoShareModel.scope(ScopeNames.WITH_ACTOR).findOne({ - where: { - actorId, - videoId - }, - transaction: t - }) - } - - static loadByUrl (url: string, t: Transaction): Promise { - return VideoShareModel.scope(ScopeNames.FULL).findOne({ - where: { - url - }, - transaction: t - }) - } - - static listActorIdsAndFollowerUrlsByShare (videoId: number, t: Transaction) { - const query = `SELECT "actor"."id" AS "id", "actor"."followersUrl" AS "followersUrl" ` + - `FROM "videoShare" ` + - `INNER JOIN "actor" ON "actor"."id" = "videoShare"."actorId" ` + - `WHERE "videoShare"."videoId" = :videoId` - - const options = { - type: QueryTypes.SELECT as QueryTypes.SELECT, - replacements: { videoId }, - transaction: t - } - - return VideoShareModel.sequelize.query(query, options) - } - - static loadActorsWhoSharedVideosOf (actorOwnerId: number, t: Transaction): Promise { - const safeOwnerId = forceNumber(actorOwnerId) - - // /!\ On actor model - const query = { - where: { - [Op.and]: [ - literal( - `EXISTS (` + - ` SELECT 1 FROM "videoShare" ` + - ` INNER JOIN "video" ON "videoShare"."videoId" = "video"."id" ` + - ` INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ` + - ` INNER JOIN "account" ON "account"."id" = "videoChannel"."accountId" ` + - ` WHERE "videoShare"."actorId" = "ActorModel"."id" AND "account"."actorId" = ${safeOwnerId} ` + - ` LIMIT 1` + - `)` - ) - ] - }, - transaction: t - } - - return ActorModel.findAll(query) - } - - static loadActorsByVideoChannel (videoChannelId: number, t: Transaction): Promise { - const safeChannelId = forceNumber(videoChannelId) - - // /!\ On actor model - const query = { - where: { - [Op.and]: [ - literal( - `EXISTS (` + - ` SELECT 1 FROM "videoShare" ` + - ` INNER JOIN "video" ON "videoShare"."videoId" = "video"."id" ` + - ` WHERE "videoShare"."actorId" = "ActorModel"."id" AND "video"."channelId" = ${safeChannelId} ` + - ` LIMIT 1` + - `)` - ) - ] - }, - transaction: t - } - - return ActorModel.findAll(query) - } - - static listAndCountByVideoId (videoId: number, start: number, count: number, t?: Transaction) { - const query = { - offset: start, - limit: count, - where: { - videoId - }, - transaction: t - } - - return Promise.all([ - VideoShareModel.count(query), - VideoShareModel.findAll(query) - ]).then(([ total, data ]) => ({ total, data })) - } - - static listRemoteShareUrlsOfLocalVideos () { - const query = `SELECT "videoShare".url FROM "videoShare" ` + - `INNER JOIN actor ON actor.id = "videoShare"."actorId" AND actor."serverId" IS NOT NULL ` + - `INNER JOIN video ON video.id = "videoShare"."videoId" AND video.remote IS FALSE` - - return VideoShareModel.sequelize.query<{ url: string }>(query, { - type: QueryTypes.SELECT, - raw: true - }).then(rows => rows.map(r => r.url)) - } - - static cleanOldSharesOf (videoId: number, beforeUpdatedAt: Date) { - const query = { - where: { - updatedAt: { - [Op.lt]: beforeUpdatedAt - }, - videoId, - actorId: { - [Op.notIn]: buildLocalActorIdsIn() - } - } - } - - return VideoShareModel.destroy(query) - } -} diff --git a/server/models/video/video-source.ts b/server/models/video/video-source.ts deleted file mode 100644 index 1b6868b85..000000000 --- a/server/models/video/video-source.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { Transaction } from 'sequelize' -import { AllowNull, BelongsTo, Column, CreatedAt, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { VideoSource } from '@shared/models/videos/video-source' -import { AttributesOnly } from '@shared/typescript-utils' -import { getSort } from '../shared' -import { VideoModel } from './video' - -@Table({ - tableName: 'videoSource', - indexes: [ - { - fields: [ 'videoId' ] - }, - { - fields: [ { name: 'createdAt', order: 'DESC' } ] - } - ] -}) -export class VideoSourceModel extends Model>> { - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @AllowNull(false) - @Column - filename: string - - @ForeignKey(() => VideoModel) - @Column - videoId: number - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade' - }) - Video: VideoModel - - static loadLatest (videoId: number, transaction?: Transaction) { - return VideoSourceModel.findOne({ - where: { videoId }, - order: getSort('-createdAt'), - transaction - }) - } - - toFormattedJSON (): VideoSource { - return { - filename: this.filename, - createdAt: this.createdAt.toISOString() - } - } -} diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts deleted file mode 100644 index a85c79c9f..000000000 --- a/server/models/video/video-streaming-playlist.ts +++ /dev/null @@ -1,328 +0,0 @@ -import memoizee from 'memoizee' -import { join } from 'path' -import { Op, Transaction } from 'sequelize' -import { - AllowNull, - BelongsTo, - Column, - CreatedAt, - DataType, - Default, - ForeignKey, - HasMany, - Is, - Model, - Table, - UpdatedAt -} from 'sequelize-typescript' -import { CONFIG } from '@server/initializers/config' -import { getHLSPrivateFileUrl, getHLSPublicFileUrl } from '@server/lib/object-storage' -import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '@server/lib/paths' -import { isVideoInPrivateDirectory } from '@server/lib/video-privacy' -import { VideoFileModel } from '@server/models/video/video-file' -import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models' -import { sha1 } from '@shared/extra-utils' -import { VideoStorage } from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' -import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' -import { isArrayOf } from '../../helpers/custom-validators/misc' -import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' -import { - CONSTRAINTS_FIELDS, - MEMOIZE_LENGTH, - MEMOIZE_TTL, - P2P_MEDIA_LOADER_PEER_VERSION, - STATIC_PATHS, - WEBSERVER -} from '../../initializers/constants' -import { VideoRedundancyModel } from '../redundancy/video-redundancy' -import { doesExist, throwIfNotValid } from '../shared' -import { VideoModel } from './video' - -@Table({ - tableName: 'videoStreamingPlaylist', - indexes: [ - { - fields: [ 'videoId' ] - }, - { - fields: [ 'videoId', 'type' ], - unique: true - }, - { - fields: [ 'p2pMediaLoaderInfohashes' ], - using: 'gin' - } - ] -}) -export class VideoStreamingPlaylistModel extends Model>> { - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @AllowNull(false) - @Column - type: VideoStreamingPlaylistType - - @AllowNull(false) - @Column - playlistFilename: string - - @AllowNull(true) - @Is('PlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'playlist url', true)) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max)) - playlistUrl: string - - @AllowNull(false) - @Is('VideoStreamingPlaylistInfoHashes', value => throwIfNotValid(value, v => isArrayOf(v, isVideoFileInfoHashValid), 'info hashes')) - @Column(DataType.ARRAY(DataType.STRING)) - p2pMediaLoaderInfohashes: string[] - - @AllowNull(false) - @Column - p2pMediaLoaderPeerVersion: number - - @AllowNull(false) - @Column - segmentsSha256Filename: string - - @AllowNull(true) - @Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url', true)) - @Column - segmentsSha256Url: string - - @ForeignKey(() => VideoModel) - @Column - videoId: number - - @AllowNull(false) - @Default(VideoStorage.FILE_SYSTEM) - @Column - storage: VideoStorage - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'CASCADE' - }) - Video: VideoModel - - @HasMany(() => VideoFileModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'CASCADE' - }) - VideoFiles: VideoFileModel[] - - @HasMany(() => VideoRedundancyModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'CASCADE', - hooks: true - }) - RedundancyVideos: VideoRedundancyModel[] - - static doesInfohashExistCached = memoizee(VideoStreamingPlaylistModel.doesInfohashExist, { - promise: true, - max: MEMOIZE_LENGTH.INFO_HASH_EXISTS, - maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS - }) - - static doesInfohashExist (infoHash: string) { - const query = 'SELECT 1 FROM "videoStreamingPlaylist" WHERE $infoHash = ANY("p2pMediaLoaderInfohashes") LIMIT 1' - - return doesExist(this.sequelize, query, { infoHash }) - } - - static buildP2PMediaLoaderInfoHashes (playlistUrl: string, files: unknown[]) { - const hashes: string[] = [] - - // https://github.com/Novage/p2p-media-loader/blob/master/p2p-media-loader-core/lib/p2p-media-manager.ts#L115 - for (let i = 0; i < files.length; i++) { - hashes.push(sha1(`${P2P_MEDIA_LOADER_PEER_VERSION}${playlistUrl}+V${i}`)) - } - - return hashes - } - - static listByIncorrectPeerVersion () { - const query = { - where: { - p2pMediaLoaderPeerVersion: { - [Op.ne]: P2P_MEDIA_LOADER_PEER_VERSION - } - }, - include: [ - { - model: VideoModel.unscoped(), - required: true - } - ] - } - - return VideoStreamingPlaylistModel.findAll(query) - } - - static loadWithVideoAndFiles (id: number) { - const options = { - include: [ - { - model: VideoModel.unscoped(), - required: true - }, - { - model: VideoFileModel.unscoped() - } - ] - } - - return VideoStreamingPlaylistModel.findByPk(id, options) - } - - static loadWithVideo (id: number) { - const options = { - include: [ - { - model: VideoModel.unscoped(), - required: true - } - ] - } - - return VideoStreamingPlaylistModel.findByPk(id, options) - } - - static loadHLSPlaylistByVideo (videoId: number, transaction?: Transaction): Promise { - const options = { - where: { - type: VideoStreamingPlaylistType.HLS, - videoId - }, - transaction - } - - return VideoStreamingPlaylistModel.findOne(options) - } - - static async loadOrGenerate (video: MVideo, transaction?: Transaction) { - let playlist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id, transaction) - - if (!playlist) { - playlist = new VideoStreamingPlaylistModel({ - p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, - type: VideoStreamingPlaylistType.HLS, - storage: VideoStorage.FILE_SYSTEM, - p2pMediaLoaderInfohashes: [], - playlistFilename: generateHLSMasterPlaylistFilename(video.isLive), - segmentsSha256Filename: generateHlsSha256SegmentsFilename(video.isLive), - videoId: video.id - }) - - await playlist.save({ transaction }) - } - - return Object.assign(playlist, { Video: video }) - } - - static doesOwnedHLSPlaylistExist (videoUUID: string) { - const query = `SELECT 1 FROM "videoStreamingPlaylist" ` + - `INNER JOIN "video" ON "video"."id" = "videoStreamingPlaylist"."videoId" ` + - `AND "video"."remote" IS FALSE AND "video"."uuid" = $videoUUID ` + - `AND "storage" = ${VideoStorage.FILE_SYSTEM} LIMIT 1` - - return doesExist(this.sequelize, query, { videoUUID }) - } - - assignP2PMediaLoaderInfoHashes (video: MVideo, files: unknown[]) { - const masterPlaylistUrl = this.getMasterPlaylistUrl(video) - - this.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(masterPlaylistUrl, files) - } - - // --------------------------------------------------------------------------- - - getMasterPlaylistUrl (video: MVideo) { - if (video.isOwned()) { - if (this.storage === VideoStorage.OBJECT_STORAGE) { - return this.getMasterPlaylistObjectStorageUrl(video) - } - - return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video) - } - - return this.playlistUrl - } - - private getMasterPlaylistObjectStorageUrl (video: MVideo) { - if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) { - return getHLSPrivateFileUrl(video, this.playlistFilename) - } - - return getHLSPublicFileUrl(this.playlistUrl) - } - - // --------------------------------------------------------------------------- - - getSha256SegmentsUrl (video: MVideo) { - if (video.isOwned()) { - if (this.storage === VideoStorage.OBJECT_STORAGE) { - return this.getSha256SegmentsObjectStorageUrl(video) - } - - return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video) - } - - return this.segmentsSha256Url - } - - private getSha256SegmentsObjectStorageUrl (video: MVideo) { - if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) { - return getHLSPrivateFileUrl(video, this.segmentsSha256Filename) - } - - return getHLSPublicFileUrl(this.segmentsSha256Url) - } - - // --------------------------------------------------------------------------- - - getStringType () { - if (this.type === VideoStreamingPlaylistType.HLS) return 'hls' - - return 'unknown' - } - - getTrackerUrls (baseUrlHttp: string, baseUrlWs: string) { - return [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] - } - - hasSameUniqueKeysThan (other: MStreamingPlaylist) { - return this.type === other.type && - this.videoId === other.videoId - } - - withVideo (video: MVideo) { - return Object.assign(this, { Video: video }) - } - - private getMasterPlaylistStaticPath (video: MVideo) { - if (isVideoInPrivateDirectory(video.privacy)) { - return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.playlistFilename) - } - - return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.playlistFilename) - } - - private getSha256SegmentsStaticPath (video: MVideo) { - if (isVideoInPrivateDirectory(video.privacy)) { - return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.segmentsSha256Filename) - } - - return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.segmentsSha256Filename) - } -} diff --git a/server/models/video/video-tag.ts b/server/models/video/video-tag.ts deleted file mode 100644 index 7e880c968..000000000 --- a/server/models/video/video-tag.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { Column, CreatedAt, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' -import { AttributesOnly } from '@shared/typescript-utils' -import { TagModel } from './tag' -import { VideoModel } from './video' - -@Table({ - tableName: 'videoTag', - indexes: [ - { - fields: [ 'videoId' ] - }, - { - fields: [ 'tagId' ] - } - ] -}) -export class VideoTagModel extends Model>> { - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @ForeignKey(() => VideoModel) - @Column - videoId: number - - @ForeignKey(() => TagModel) - @Column - tagId: number -} diff --git a/server/models/video/video.ts b/server/models/video/video.ts deleted file mode 100644 index 73308182d..000000000 --- a/server/models/video/video.ts +++ /dev/null @@ -1,2047 +0,0 @@ -import Bluebird from 'bluebird' -import { remove } from 'fs-extra' -import { maxBy, minBy } from 'lodash' -import { FindOptions, Includeable, IncludeOptions, Op, QueryTypes, ScopeOptions, Sequelize, Transaction, WhereOptions } from 'sequelize' -import { - AfterCreate, - AfterDestroy, - AfterUpdate, - AllowNull, - BeforeDestroy, - BelongsTo, - BelongsToMany, - Column, - CreatedAt, - DataType, - Default, - ForeignKey, - HasMany, - HasOne, - Is, - IsInt, - IsUUID, - Min, - Model, - Scopes, - Table, - UpdatedAt -} from 'sequelize-typescript' -import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video' -import { InternalEventEmitter } from '@server/lib/internal-event-emitter' -import { LiveManager } from '@server/lib/live/live-manager' -import { removeHLSFileObjectStorageByFilename, removeHLSObjectStorage, removeWebVideoObjectStorage } from '@server/lib/object-storage' -import { tracer } from '@server/lib/opentelemetry/tracing' -import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' -import { Hooks } from '@server/lib/plugins/hooks' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { isVideoInPrivateDirectory } from '@server/lib/video-privacy' -import { getServerActor } from '@server/models/application/application' -import { ModelCache } from '@server/models/shared/model-cache' -import { buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils' -import { uuidToShort } from '@shared/extra-utils' -import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream } from '@shared/ffmpeg' -import { - ResultList, - ThumbnailType, - UserRight, - Video, - VideoDetails, - VideoFile, - VideoInclude, - VideoObject, - VideoPrivacy, - VideoRateType, - VideoState, - VideoStorage, - VideoStreamingPlaylistType -} from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { peertubeTruncate } from '../../helpers/core-utils' -import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' -import { exists, isArray, isBooleanValid, isUUIDValid } from '../../helpers/custom-validators/misc' -import { - isVideoDescriptionValid, - isVideoDurationValid, - isVideoNameValid, - isVideoPrivacyValid, - isVideoStateValid, - isVideoSupportValid -} from '../../helpers/custom-validators/videos' -import { logger } from '../../helpers/logger' -import { CONFIG } from '../../initializers/config' -import { ACTIVITY_PUB, API_VERSION, CONSTRAINTS_FIELDS, WEBSERVER } from '../../initializers/constants' -import { sendDeleteVideo } from '../../lib/activitypub/send' -import { - MChannel, - MChannelAccountDefault, - MChannelId, - MStoryboard, - MStreamingPlaylist, - MStreamingPlaylistFilesVideo, - MUserAccountId, - MUserId, - MVideo, - MVideoAccountLight, - MVideoAccountLightBlacklistAllFiles, - MVideoAP, - MVideoAPLight, - MVideoCaptionLanguageUrl, - MVideoDetails, - MVideoFileVideo, - MVideoFormattable, - MVideoFormattableDetails, - MVideoForUser, - MVideoFullLight, - MVideoId, - MVideoImmutable, - MVideoThumbnail, - MVideoThumbnailBlacklist, - MVideoWithAllFiles, - MVideoWithFile -} from '../../types/models' -import { MThumbnail } from '../../types/models/video/thumbnail' -import { MVideoFile, MVideoFileStreamingPlaylistVideo } from '../../types/models/video/video-file' -import { VideoAbuseModel } from '../abuse/video-abuse' -import { AccountModel } from '../account/account' -import { AccountVideoRateModel } from '../account/account-video-rate' -import { ActorModel } from '../actor/actor' -import { ActorImageModel } from '../actor/actor-image' -import { VideoRedundancyModel } from '../redundancy/video-redundancy' -import { ServerModel } from '../server/server' -import { TrackerModel } from '../server/tracker' -import { VideoTrackerModel } from '../server/video-tracker' -import { buildTrigramSearchIndex, buildWhereIdOrUUID, getVideoSort, isOutdated, setAsUpdated, throwIfNotValid } from '../shared' -import { UserModel } from '../user/user' -import { UserVideoHistoryModel } from '../user/user-video-history' -import { VideoViewModel } from '../view/video-view' -import { videoModelToActivityPubObject } from './formatter/video-activity-pub-format' -import { - videoFilesModelToFormattedJSON, - VideoFormattingJSONOptions, - videoModelToFormattedDetailsJSON, - videoModelToFormattedJSON -} from './formatter/video-api-format' -import { ScheduleVideoUpdateModel } from './schedule-video-update' -import { - BuildVideosListQueryOptions, - DisplayOnlyForFollowerOptions, - VideoModelGetQueryBuilder, - VideosIdListQueryBuilder, - VideosModelListQueryBuilder -} from './sql/video' -import { StoryboardModel } from './storyboard' -import { TagModel } from './tag' -import { ThumbnailModel } from './thumbnail' -import { VideoBlacklistModel } from './video-blacklist' -import { VideoCaptionModel } from './video-caption' -import { ScopeNames as VideoChannelScopeNames, SummaryOptions, VideoChannelModel } from './video-channel' -import { VideoCommentModel } from './video-comment' -import { VideoFileModel } from './video-file' -import { VideoImportModel } from './video-import' -import { VideoJobInfoModel } from './video-job-info' -import { VideoLiveModel } from './video-live' -import { VideoPasswordModel } from './video-password' -import { VideoPlaylistElementModel } from './video-playlist-element' -import { VideoShareModel } from './video-share' -import { VideoSourceModel } from './video-source' -import { VideoStreamingPlaylistModel } from './video-streaming-playlist' -import { VideoTagModel } from './video-tag' - -export enum ScopeNames { - FOR_API = 'FOR_API', - WITH_ACCOUNT_DETAILS = 'WITH_ACCOUNT_DETAILS', - WITH_TAGS = 'WITH_TAGS', - WITH_WEB_VIDEO_FILES = 'WITH_WEB_VIDEO_FILES', - WITH_SCHEDULED_UPDATE = 'WITH_SCHEDULED_UPDATE', - WITH_BLACKLISTED = 'WITH_BLACKLISTED', - WITH_STREAMING_PLAYLISTS = 'WITH_STREAMING_PLAYLISTS', - WITH_IMMUTABLE_ATTRIBUTES = 'WITH_IMMUTABLE_ATTRIBUTES', - WITH_USER_HISTORY = 'WITH_USER_HISTORY', - WITH_THUMBNAILS = 'WITH_THUMBNAILS' -} - -export type ForAPIOptions = { - ids?: number[] - - videoPlaylistId?: number - - withAccountBlockerIds?: number[] -} - -@Scopes(() => ({ - [ScopeNames.WITH_IMMUTABLE_ATTRIBUTES]: { - attributes: [ 'id', 'url', 'uuid', 'remote' ] - }, - [ScopeNames.FOR_API]: (options: ForAPIOptions) => { - const include: Includeable[] = [ - { - model: VideoChannelModel.scope({ - method: [ - VideoChannelScopeNames.SUMMARY, { - withAccount: true, - withAccountBlockerIds: options.withAccountBlockerIds - } as SummaryOptions - ] - }), - required: true - }, - { - attributes: [ 'type', 'filename' ], - model: ThumbnailModel, - required: false - } - ] - - const query: FindOptions = {} - - if (options.ids) { - query.where = { - id: { - [Op.in]: options.ids - } - } - } - - if (options.videoPlaylistId) { - include.push({ - model: VideoPlaylistElementModel.unscoped(), - required: true, - where: { - videoPlaylistId: options.videoPlaylistId - } - }) - } - - query.include = include - - return query - }, - [ScopeNames.WITH_THUMBNAILS]: { - include: [ - { - model: ThumbnailModel, - required: false - } - ] - }, - [ScopeNames.WITH_ACCOUNT_DETAILS]: { - include: [ - { - model: VideoChannelModel.unscoped(), - required: true, - include: [ - { - attributes: { - exclude: [ 'privateKey', 'publicKey' ] - }, - model: ActorModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'host' ], - model: ServerModel.unscoped(), - required: false - }, - { - model: ActorImageModel, - as: 'Avatars', - required: false - } - ] - }, - { - model: AccountModel.unscoped(), - required: true, - include: [ - { - model: ActorModel.unscoped(), - attributes: { - exclude: [ 'privateKey', 'publicKey' ] - }, - required: true, - include: [ - { - attributes: [ 'host' ], - model: ServerModel.unscoped(), - required: false - }, - { - model: ActorImageModel, - as: 'Avatars', - required: false - } - ] - } - ] - } - ] - } - ] - }, - [ScopeNames.WITH_TAGS]: { - include: [ TagModel ] - }, - [ScopeNames.WITH_BLACKLISTED]: { - include: [ - { - attributes: [ 'id', 'reason', 'unfederated' ], - model: VideoBlacklistModel, - required: false - } - ] - }, - [ScopeNames.WITH_WEB_VIDEO_FILES]: (withRedundancies = false) => { - let subInclude: any[] = [] - - if (withRedundancies === true) { - subInclude = [ - { - attributes: [ 'fileUrl' ], - model: VideoRedundancyModel.unscoped(), - required: false - } - ] - } - - return { - include: [ - { - model: VideoFileModel, - separate: true, - required: false, - include: subInclude - } - ] - } - }, - [ScopeNames.WITH_STREAMING_PLAYLISTS]: (withRedundancies = false) => { - const subInclude: IncludeOptions[] = [ - { - model: VideoFileModel, - required: false - } - ] - - if (withRedundancies === true) { - subInclude.push({ - attributes: [ 'fileUrl' ], - model: VideoRedundancyModel.unscoped(), - required: false - }) - } - - return { - include: [ - { - model: VideoStreamingPlaylistModel.unscoped(), - required: false, - separate: true, - include: subInclude - } - ] - } - }, - [ScopeNames.WITH_SCHEDULED_UPDATE]: { - include: [ - { - model: ScheduleVideoUpdateModel.unscoped(), - required: false - } - ] - }, - [ScopeNames.WITH_USER_HISTORY]: (userId: number) => { - return { - include: [ - { - attributes: [ 'currentTime' ], - model: UserVideoHistoryModel.unscoped(), - required: false, - where: { - userId - } - } - ] - } - } -})) -@Table({ - tableName: 'video', - indexes: [ - buildTrigramSearchIndex('video_name_trigram', 'name'), - - { fields: [ 'createdAt' ] }, - { - fields: [ - { name: 'publishedAt', order: 'DESC' }, - { name: 'id', order: 'ASC' } - ] - }, - { fields: [ 'duration' ] }, - { - fields: [ - { name: 'views', order: 'DESC' }, - { name: 'id', order: 'ASC' } - ] - }, - { fields: [ 'channelId' ] }, - { - fields: [ 'originallyPublishedAt' ], - where: { - originallyPublishedAt: { - [Op.ne]: null - } - } - }, - { - fields: [ 'category' ], // We don't care videos with an unknown category - where: { - category: { - [Op.ne]: null - } - } - }, - { - fields: [ 'licence' ], // We don't care videos with an unknown licence - where: { - licence: { - [Op.ne]: null - } - } - }, - { - fields: [ 'language' ], // We don't care videos with an unknown language - where: { - language: { - [Op.ne]: null - } - } - }, - { - fields: [ 'nsfw' ], // Most of the videos are not NSFW - where: { - nsfw: true - } - }, - { - fields: [ 'remote' ], // Only index local videos - where: { - remote: false - } - }, - { - fields: [ 'uuid' ], - unique: true - }, - { - fields: [ 'url' ], - unique: true - } - ] -}) -export class VideoModel extends Model>> { - - @AllowNull(false) - @Default(DataType.UUIDV4) - @IsUUID(4) - @Column(DataType.UUID) - uuid: string - - @AllowNull(false) - @Is('VideoName', value => throwIfNotValid(value, isVideoNameValid, 'name')) - @Column - name: string - - @AllowNull(true) - @Default(null) - @Column - category: number - - @AllowNull(true) - @Default(null) - @Column - licence: number - - @AllowNull(true) - @Default(null) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.LANGUAGE.max)) - language: string - - @AllowNull(false) - @Is('VideoPrivacy', value => throwIfNotValid(value, isVideoPrivacyValid, 'privacy')) - @Column - privacy: VideoPrivacy - - @AllowNull(false) - @Is('VideoNSFW', value => throwIfNotValid(value, isBooleanValid, 'NSFW boolean')) - @Column - nsfw: boolean - - @AllowNull(true) - @Default(null) - @Is('VideoDescription', value => throwIfNotValid(value, isVideoDescriptionValid, 'description', true)) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max)) - description: string - - @AllowNull(true) - @Default(null) - @Is('VideoSupport', value => throwIfNotValid(value, isVideoSupportValid, 'support', true)) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.SUPPORT.max)) - support: string - - @AllowNull(false) - @Is('VideoDuration', value => throwIfNotValid(value, isVideoDurationValid, 'duration')) - @Column - duration: number - - @AllowNull(false) - @Default(0) - @IsInt - @Min(0) - @Column - views: number - - @AllowNull(false) - @Default(0) - @IsInt - @Min(0) - @Column - likes: number - - @AllowNull(false) - @Default(0) - @IsInt - @Min(0) - @Column - dislikes: number - - @AllowNull(false) - @Column - remote: boolean - - @AllowNull(false) - @Default(false) - @Column - isLive: boolean - - @AllowNull(false) - @Is('VideoUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url')) - @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max)) - url: string - - @AllowNull(false) - @Column - commentsEnabled: boolean - - @AllowNull(false) - @Column - downloadEnabled: boolean - - @AllowNull(false) - @Column - waitTranscoding: boolean - - @AllowNull(false) - @Default(null) - @Is('VideoState', value => throwIfNotValid(value, isVideoStateValid, 'state')) - @Column - state: VideoState - - // We already have the information in videoSource table for local videos, but we prefer to normalize it for performance - // And also to store the info from remote instances - @AllowNull(true) - @Column - inputFileUpdatedAt: Date - - @CreatedAt - createdAt: Date - - @UpdatedAt - updatedAt: Date - - @AllowNull(false) - @Default(DataType.NOW) - @Column - publishedAt: Date - - @AllowNull(true) - @Default(null) - @Column - originallyPublishedAt: Date - - @ForeignKey(() => VideoChannelModel) - @Column - channelId: number - - @BelongsTo(() => VideoChannelModel, { - foreignKey: { - allowNull: true - }, - onDelete: 'cascade' - }) - VideoChannel: VideoChannelModel - - @BelongsToMany(() => TagModel, { - foreignKey: 'videoId', - through: () => VideoTagModel, - onDelete: 'CASCADE' - }) - Tags: TagModel[] - - @BelongsToMany(() => TrackerModel, { - foreignKey: 'videoId', - through: () => VideoTrackerModel, - onDelete: 'CASCADE' - }) - Trackers: TrackerModel[] - - @HasMany(() => ThumbnailModel, { - foreignKey: { - name: 'videoId', - allowNull: true - }, - hooks: true, - onDelete: 'cascade' - }) - Thumbnails: ThumbnailModel[] - - @HasMany(() => VideoPlaylistElementModel, { - foreignKey: { - name: 'videoId', - allowNull: true - }, - onDelete: 'set null' - }) - VideoPlaylistElements: VideoPlaylistElementModel[] - - @HasOne(() => VideoSourceModel, { - foreignKey: { - name: 'videoId', - allowNull: false - }, - onDelete: 'CASCADE' - }) - VideoSource: VideoSourceModel - - @HasMany(() => VideoAbuseModel, { - foreignKey: { - name: 'videoId', - allowNull: true - }, - onDelete: 'set null' - }) - VideoAbuses: VideoAbuseModel[] - - @HasMany(() => VideoFileModel, { - foreignKey: { - name: 'videoId', - allowNull: true - }, - hooks: true, - onDelete: 'cascade' - }) - VideoFiles: VideoFileModel[] - - @HasMany(() => VideoStreamingPlaylistModel, { - foreignKey: { - name: 'videoId', - allowNull: false - }, - hooks: true, - onDelete: 'cascade' - }) - VideoStreamingPlaylists: VideoStreamingPlaylistModel[] - - @HasMany(() => VideoShareModel, { - foreignKey: { - name: 'videoId', - allowNull: false - }, - onDelete: 'cascade' - }) - VideoShares: VideoShareModel[] - - @HasMany(() => AccountVideoRateModel, { - foreignKey: { - name: 'videoId', - allowNull: false - }, - onDelete: 'cascade' - }) - AccountVideoRates: AccountVideoRateModel[] - - @HasMany(() => VideoCommentModel, { - foreignKey: { - name: 'videoId', - allowNull: false - }, - onDelete: 'cascade', - hooks: true - }) - VideoComments: VideoCommentModel[] - - @HasMany(() => VideoViewModel, { - foreignKey: { - name: 'videoId', - allowNull: false - }, - onDelete: 'cascade' - }) - VideoViews: VideoViewModel[] - - @HasMany(() => UserVideoHistoryModel, { - foreignKey: { - name: 'videoId', - allowNull: false - }, - onDelete: 'cascade' - }) - UserVideoHistories: UserVideoHistoryModel[] - - @HasOne(() => ScheduleVideoUpdateModel, { - foreignKey: { - name: 'videoId', - allowNull: false - }, - onDelete: 'cascade' - }) - ScheduleVideoUpdate: ScheduleVideoUpdateModel - - @HasOne(() => VideoBlacklistModel, { - foreignKey: { - name: 'videoId', - allowNull: false - }, - onDelete: 'cascade' - }) - VideoBlacklist: VideoBlacklistModel - - @HasOne(() => VideoLiveModel, { - foreignKey: { - name: 'videoId', - allowNull: false - }, - hooks: true, - onDelete: 'cascade' - }) - VideoLive: VideoLiveModel - - @HasOne(() => VideoImportModel, { - foreignKey: { - name: 'videoId', - allowNull: true - }, - onDelete: 'set null' - }) - VideoImport: VideoImportModel - - @HasMany(() => VideoCaptionModel, { - foreignKey: { - name: 'videoId', - allowNull: false - }, - onDelete: 'cascade', - hooks: true, - ['separate' as any]: true - }) - VideoCaptions: VideoCaptionModel[] - - @HasMany(() => VideoPasswordModel, { - foreignKey: { - name: 'videoId', - allowNull: false - }, - onDelete: 'cascade' - }) - VideoPasswords: VideoPasswordModel[] - - @HasOne(() => VideoJobInfoModel, { - foreignKey: { - name: 'videoId', - allowNull: false - }, - onDelete: 'cascade' - }) - VideoJobInfo: VideoJobInfoModel - - @HasOne(() => StoryboardModel, { - foreignKey: { - name: 'videoId', - allowNull: false - }, - onDelete: 'cascade', - hooks: true - }) - Storyboard: StoryboardModel - - @AfterCreate - static notifyCreate (video: MVideo) { - InternalEventEmitter.Instance.emit('video-created', { video }) - } - - @AfterUpdate - static notifyUpdate (video: MVideo) { - InternalEventEmitter.Instance.emit('video-updated', { video }) - } - - @AfterDestroy - static notifyDestroy (video: MVideo) { - InternalEventEmitter.Instance.emit('video-deleted', { video }) - } - - @BeforeDestroy - static async sendDelete (instance: MVideoAccountLight, options: { transaction: Transaction }) { - if (!instance.isOwned()) return undefined - - // Lazy load channels - if (!instance.VideoChannel) { - instance.VideoChannel = await instance.$get('VideoChannel', { - include: [ - ActorModel, - AccountModel - ], - transaction: options.transaction - }) as MChannelAccountDefault - } - - return sendDeleteVideo(instance, options.transaction) - } - - @BeforeDestroy - static async removeFiles (instance: VideoModel, options) { - const tasks: Promise[] = [] - - logger.info('Removing files of video %s.', instance.url) - - if (instance.isOwned()) { - if (!Array.isArray(instance.VideoFiles)) { - instance.VideoFiles = await instance.$get('VideoFiles', { transaction: options.transaction }) - } - - // Remove physical files and torrents - instance.VideoFiles.forEach(file => { - tasks.push(instance.removeWebVideoFile(file)) - }) - - // Remove playlists file - if (!Array.isArray(instance.VideoStreamingPlaylists)) { - instance.VideoStreamingPlaylists = await instance.$get('VideoStreamingPlaylists', { transaction: options.transaction }) - } - - for (const p of instance.VideoStreamingPlaylists) { - tasks.push(instance.removeStreamingPlaylistFiles(p)) - } - } - - // Do not wait video deletion because we could be in a transaction - Promise.all(tasks) - .then(() => logger.info('Removed files of video %s.', instance.url)) - .catch(err => logger.error('Some errors when removing files of video %s in before destroy hook.', instance.uuid, { err })) - - return undefined - } - - @BeforeDestroy - static stopLiveIfNeeded (instance: VideoModel) { - if (!instance.isLive) return - - logger.info('Stopping live of video %s after video deletion.', instance.uuid) - - LiveManager.Instance.stopSessionOf(instance.uuid, null) - } - - @BeforeDestroy - static invalidateCache (instance: VideoModel) { - ModelCache.Instance.invalidateCache('video', instance.id) - } - - @BeforeDestroy - static async saveEssentialDataToAbuses (instance: VideoModel, options) { - const tasks: Promise[] = [] - - if (!Array.isArray(instance.VideoAbuses)) { - instance.VideoAbuses = await instance.$get('VideoAbuses', { transaction: options.transaction }) - - if (instance.VideoAbuses.length === 0) return undefined - } - - logger.info('Saving video abuses details of video %s.', instance.url) - - if (!instance.Trackers) instance.Trackers = await instance.$get('Trackers', { transaction: options.transaction }) - const details = instance.toFormattedDetailsJSON() - - for (const abuse of instance.VideoAbuses) { - abuse.deletedVideo = details - tasks.push(abuse.save({ transaction: options.transaction })) - } - - await Promise.all(tasks) - } - - static listLocalIds (): Promise { - const query = { - attributes: [ 'id' ], - raw: true, - where: { - remote: false - } - } - - return VideoModel.findAll(query) - .then(rows => rows.map(r => r.id)) - } - - static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) { - function getRawQuery (select: string) { - const queryVideo = 'SELECT ' + select + ' FROM "video" AS "Video" ' + - 'INNER JOIN "videoChannel" AS "VideoChannel" ON "VideoChannel"."id" = "Video"."channelId" ' + - 'INNER JOIN "account" AS "Account" ON "Account"."id" = "VideoChannel"."accountId" ' + - 'WHERE "Account"."actorId" = ' + actorId - const queryVideoShare = 'SELECT ' + select + ' FROM "videoShare" AS "VideoShare" ' + - 'INNER JOIN "video" AS "Video" ON "Video"."id" = "VideoShare"."videoId" ' + - 'WHERE "VideoShare"."actorId" = ' + actorId - - return `(${queryVideo}) UNION (${queryVideoShare})` - } - - const rawQuery = getRawQuery('"Video"."id"') - const rawCountQuery = getRawQuery('COUNT("Video"."id") as "total"') - - const query = { - distinct: true, - offset: start, - limit: count, - order: getVideoSort('-createdAt', [ 'Tags', 'name', 'ASC' ]), - where: { - id: { - [Op.in]: Sequelize.literal('(' + rawQuery + ')') - }, - [Op.or]: getPrivaciesForFederation() - }, - include: [ - { - attributes: [ 'filename', 'language', 'fileUrl' ], - model: VideoCaptionModel.unscoped(), - required: false - }, - { - model: StoryboardModel.unscoped(), - required: false - }, - { - attributes: [ 'id', 'url' ], - model: VideoShareModel.unscoped(), - required: false, - // We only want videos shared by this actor - where: { - [Op.and]: [ - { - id: { - [Op.not]: null - } - }, - { - actorId - } - ] - }, - include: [ - { - attributes: [ 'id', 'url' ], - model: ActorModel.unscoped() - } - ] - }, - { - model: VideoChannelModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'name' ], - model: AccountModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'id', 'url', 'followersUrl' ], - model: ActorModel.unscoped(), - required: true - } - ] - }, - { - attributes: [ 'id', 'url', 'followersUrl' ], - model: ActorModel.unscoped(), - required: true - } - ] - }, - { - model: VideoStreamingPlaylistModel.unscoped(), - required: false, - include: [ - { - model: VideoFileModel, - required: false - } - ] - }, - VideoLiveModel.unscoped(), - VideoFileModel, - TagModel - ] - } - - return Bluebird.all([ - VideoModel.scope(ScopeNames.WITH_THUMBNAILS).findAll(query), - VideoModel.sequelize.query<{ total: string }>(rawCountQuery, { type: QueryTypes.SELECT }) - ]).then(([ rows, totals ]) => { - // totals: totalVideos + totalVideoShares - let totalVideos = 0 - let totalVideoShares = 0 - if (totals[0]) totalVideos = parseInt(totals[0].total, 10) - if (totals[1]) totalVideoShares = parseInt(totals[1].total, 10) - - const total = totalVideos + totalVideoShares - return { - data: rows, - total - } - }) - } - - static async listPublishedLiveUUIDs () { - const options = { - attributes: [ 'uuid' ], - where: { - isLive: true, - remote: false, - state: VideoState.PUBLISHED - } - } - - const result = await VideoModel.findAll(options) - - return result.map(v => v.uuid) - } - - static listUserVideosForApi (options: { - accountId: number - start: number - count: number - sort: string - - channelId?: number - isLive?: boolean - search?: string - }) { - const { accountId, channelId, start, count, sort, search, isLive } = options - - function buildBaseQuery (forCount: boolean): FindOptions { - const where: WhereOptions = {} - - if (search) { - where.name = { - [Op.iLike]: '%' + search + '%' - } - } - - if (exists(isLive)) { - where.isLive = isLive - } - - const channelWhere = channelId - ? { id: channelId } - : {} - - const baseQuery = { - offset: start, - limit: count, - where, - order: getVideoSort(sort), - include: [ - { - model: forCount - ? VideoChannelModel.unscoped() - : VideoChannelModel, - required: true, - where: channelWhere, - include: [ - { - model: forCount - ? AccountModel.unscoped() - : AccountModel, - where: { - id: accountId - }, - required: true - } - ] - } - ] - } - - return baseQuery - } - - const countQuery = buildBaseQuery(true) - const findQuery = buildBaseQuery(false) - - const findScopes: (string | ScopeOptions)[] = [ - ScopeNames.WITH_SCHEDULED_UPDATE, - ScopeNames.WITH_BLACKLISTED, - ScopeNames.WITH_THUMBNAILS - ] - - return Promise.all([ - VideoModel.count(countQuery), - VideoModel.scope(findScopes).findAll(findQuery) - ]).then(([ count, rows ]) => { - return { - data: rows, - total: count - } - }) - } - - static async listForApi (options: { - start: number - count: number - sort: string - - nsfw: boolean - isLive?: boolean - isLocal?: boolean - include?: VideoInclude - - hasFiles?: boolean // default false - - hasWebtorrentFiles?: boolean // TODO: remove in v7 - hasWebVideoFiles?: boolean - - hasHLSFiles?: boolean - - categoryOneOf?: number[] - licenceOneOf?: number[] - languageOneOf?: string[] - tagsOneOf?: string[] - tagsAllOf?: string[] - privacyOneOf?: VideoPrivacy[] - - accountId?: number - videoChannelId?: number - - displayOnlyForFollower: DisplayOnlyForFollowerOptions | null - - videoPlaylistId?: number - - trendingDays?: number - - user?: MUserAccountId - historyOfUser?: MUserId - - countVideos?: boolean - - search?: string - - excludeAlreadyWatched?: boolean - }) { - VideoModel.throwIfPrivateIncludeWithoutUser(options.include, options.user) - VideoModel.throwIfPrivacyOneOfWithoutUser(options.privacyOneOf, options.user) - - const trendingDays = options.sort.endsWith('trending') - ? CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS - : undefined - - let trendingAlgorithm: string - if (options.sort.endsWith('hot')) trendingAlgorithm = 'hot' - if (options.sort.endsWith('best')) trendingAlgorithm = 'best' - - const serverActor = await getServerActor() - - const queryOptions = { - ...pick(options, [ - 'start', - 'count', - 'sort', - 'nsfw', - 'isLive', - 'categoryOneOf', - 'licenceOneOf', - 'languageOneOf', - 'tagsOneOf', - 'tagsAllOf', - 'privacyOneOf', - 'isLocal', - 'include', - 'displayOnlyForFollower', - 'hasFiles', - 'accountId', - 'videoChannelId', - 'videoPlaylistId', - 'user', - 'historyOfUser', - 'hasHLSFiles', - 'hasWebtorrentFiles', - 'hasWebVideoFiles', - 'search', - 'excludeAlreadyWatched' - ]), - - serverAccountIdForBlock: serverActor.Account.id, - trendingDays, - trendingAlgorithm - } - - return VideoModel.getAvailableForApi(queryOptions, options.countVideos) - } - - static async searchAndPopulateAccountAndServer (options: { - start: number - count: number - sort: string - - nsfw?: boolean - isLive?: boolean - isLocal?: boolean - include?: VideoInclude - - categoryOneOf?: number[] - licenceOneOf?: number[] - languageOneOf?: string[] - tagsOneOf?: string[] - tagsAllOf?: string[] - privacyOneOf?: VideoPrivacy[] - - displayOnlyForFollower: DisplayOnlyForFollowerOptions | null - - user?: MUserAccountId - - hasWebtorrentFiles?: boolean // TODO: remove in v7 - hasWebVideoFiles?: boolean - - hasHLSFiles?: boolean - - search?: string - - host?: string - startDate?: string // ISO 8601 - endDate?: string // ISO 8601 - originallyPublishedStartDate?: string - originallyPublishedEndDate?: string - - durationMin?: number // seconds - durationMax?: number // seconds - uuids?: string[] - - excludeAlreadyWatched?: boolean - }) { - VideoModel.throwIfPrivateIncludeWithoutUser(options.include, options.user) - VideoModel.throwIfPrivacyOneOfWithoutUser(options.privacyOneOf, options.user) - - const serverActor = await getServerActor() - - const queryOptions = { - ...pick(options, [ - 'include', - 'nsfw', - 'isLive', - 'categoryOneOf', - 'licenceOneOf', - 'languageOneOf', - 'tagsOneOf', - 'tagsAllOf', - 'privacyOneOf', - 'user', - 'isLocal', - 'host', - 'start', - 'count', - 'sort', - 'startDate', - 'endDate', - 'originallyPublishedStartDate', - 'originallyPublishedEndDate', - 'durationMin', - 'durationMax', - 'hasHLSFiles', - 'hasWebtorrentFiles', - 'hasWebVideoFiles', - 'uuids', - 'search', - 'displayOnlyForFollower', - 'excludeAlreadyWatched' - ]), - serverAccountIdForBlock: serverActor.Account.id - } - - return VideoModel.getAvailableForApi(queryOptions) - } - - static countLives (options: { - remote: boolean - mode: 'published' | 'not-ended' - }) { - const query = { - where: { - remote: options.remote, - isLive: true, - state: options.mode === 'not-ended' - ? { [Op.ne]: VideoState.LIVE_ENDED } - : { [Op.eq]: VideoState.PUBLISHED } - } - } - - return VideoModel.count(query) - } - - static countVideosUploadedByUserSince (userId: number, since: Date) { - const options = { - include: [ - { - model: VideoChannelModel.unscoped(), - required: true, - include: [ - { - model: AccountModel.unscoped(), - required: true, - include: [ - { - model: UserModel.unscoped(), - required: true, - where: { - id: userId - } - } - ] - } - ] - } - ], - where: { - createdAt: { - [Op.gte]: since - } - } - } - - return VideoModel.unscoped().count(options) - } - - static countLivesOfAccount (accountId: number) { - const options = { - where: { - remote: false, - isLive: true, - state: { - [Op.ne]: VideoState.LIVE_ENDED - } - }, - include: [ - { - required: true, - model: VideoChannelModel.unscoped(), - where: { - accountId - } - } - ] - } - - return VideoModel.count(options) - } - - static load (id: number | string, transaction?: Transaction): Promise { - const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize) - - return queryBuilder.queryVideo({ id, transaction, type: 'thumbnails' }) - } - - static loadWithBlacklist (id: number | string, transaction?: Transaction): Promise { - const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize) - - return queryBuilder.queryVideo({ id, transaction, type: 'thumbnails-blacklist' }) - } - - static loadImmutableAttributes (id: number | string, t?: Transaction): Promise { - const fun = () => { - const query = { - where: buildWhereIdOrUUID(id), - transaction: t - } - - return VideoModel.scope(ScopeNames.WITH_IMMUTABLE_ATTRIBUTES).findOne(query) - } - - return ModelCache.Instance.doCache({ - cacheType: 'load-video-immutable-id', - key: '' + id, - deleteKey: 'video', - fun - }) - } - - static loadByUrlImmutableAttributes (url: string, transaction?: Transaction): Promise { - const fun = () => { - const query: FindOptions = { - where: { - url - }, - transaction - } - - return VideoModel.scope(ScopeNames.WITH_IMMUTABLE_ATTRIBUTES).findOne(query) - } - - return ModelCache.Instance.doCache({ - cacheType: 'load-video-immutable-url', - key: url, - deleteKey: 'video', - fun - }) - } - - static loadOnlyId (id: number | string, transaction?: Transaction): Promise { - const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize) - - return queryBuilder.queryVideo({ id, transaction, type: 'id' }) - } - - static loadWithFiles (id: number | string, transaction?: Transaction, logging?: boolean): Promise { - const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize) - - return queryBuilder.queryVideo({ id, transaction, type: 'all-files', logging }) - } - - static loadByUrl (url: string, transaction?: Transaction): Promise { - const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize) - - return queryBuilder.queryVideo({ url, transaction, type: 'thumbnails' }) - } - - static loadByUrlAndPopulateAccount (url: string, transaction?: Transaction): Promise { - const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize) - - return queryBuilder.queryVideo({ url, transaction, type: 'account-blacklist-files' }) - } - - static loadFull (id: number | string, t?: Transaction, userId?: number): Promise { - const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize) - - return queryBuilder.queryVideo({ id, transaction: t, type: 'full', userId }) - } - - static loadForGetAPI (parameters: { - id: number | string - transaction?: Transaction - userId?: number - }): Promise { - const { id, transaction, userId } = parameters - const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize) - - return queryBuilder.queryVideo({ id, transaction, type: 'api', userId }) - } - - static async getStats () { - const serverActor = await getServerActor() - - let totalLocalVideoViews = await VideoModel.sum('views', { - where: { - remote: false - } - }) - - // Sequelize could return null... - if (!totalLocalVideoViews) totalLocalVideoViews = 0 - - const baseOptions = { - start: 0, - count: 0, - sort: '-publishedAt', - nsfw: null, - displayOnlyForFollower: { - actorId: serverActor.id, - orLocalVideos: true - } - } - - const { total: totalLocalVideos } = await VideoModel.listForApi({ - ...baseOptions, - - isLocal: true - }) - - const { total: totalVideos } = await VideoModel.listForApi(baseOptions) - - return { - totalLocalVideos, - totalLocalVideoViews, - totalVideos - } - } - - static incrementViews (id: number, views: number) { - return VideoModel.increment('views', { - by: views, - where: { - id - } - }) - } - - static updateRatesOf (videoId: number, type: VideoRateType, count: number, t: Transaction) { - const field = type === 'like' - ? 'likes' - : 'dislikes' - - const rawQuery = `UPDATE "video" SET "${field}" = :count WHERE "video"."id" = :videoId` - - return AccountVideoRateModel.sequelize.query(rawQuery, { - transaction: t, - replacements: { videoId, rateType: type, count }, - type: QueryTypes.UPDATE - }) - } - - static syncLocalRates (videoId: number, type: VideoRateType, t: Transaction) { - const field = type === 'like' - ? 'likes' - : 'dislikes' - - const rawQuery = `UPDATE "video" SET "${field}" = ` + - '(' + - 'SELECT COUNT(id) FROM "accountVideoRate" WHERE "accountVideoRate"."videoId" = "video"."id" AND type = :rateType' + - ') ' + - 'WHERE "video"."id" = :videoId' - - return AccountVideoRateModel.sequelize.query(rawQuery, { - transaction: t, - replacements: { videoId, rateType: type }, - type: QueryTypes.UPDATE - }) - } - - static checkVideoHasInstanceFollow (videoId: number, followerActorId: number) { - // Instances only share videos - const query = 'SELECT 1 FROM "videoShare" ' + - 'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "videoShare"."actorId" ' + - 'WHERE "actorFollow"."actorId" = $followerActorId AND "actorFollow"."state" = \'accepted\' AND "videoShare"."videoId" = $videoId ' + - 'UNION ' + - 'SELECT 1 FROM "video" ' + - 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' + - 'INNER JOIN "account" ON "account"."id" = "videoChannel"."accountId" ' + - 'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "account"."actorId" ' + - 'WHERE "actorFollow"."actorId" = $followerActorId AND "actorFollow"."state" = \'accepted\' AND "video"."id" = $videoId ' + - 'LIMIT 1' - - const options = { - type: QueryTypes.SELECT as QueryTypes.SELECT, - bind: { followerActorId, videoId }, - raw: true - } - - return VideoModel.sequelize.query(query, options) - .then(results => results.length === 1) - } - - static bulkUpdateSupportField (ofChannel: MChannel, t: Transaction) { - const options = { - where: { - channelId: ofChannel.id - }, - transaction: t - } - - return VideoModel.update({ support: ofChannel.support }, options) - } - - static getAllIdsFromChannel (videoChannel: MChannelId): Promise { - const query = { - attributes: [ 'id' ], - where: { - channelId: videoChannel.id - } - } - - return VideoModel.findAll(query) - .then(videos => videos.map(v => v.id)) - } - - // threshold corresponds to how many video the field should have to be returned - static async getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) { - const serverActor = await getServerActor() - - const queryOptions: BuildVideosListQueryOptions = { - attributes: [ `"${field}"` ], - group: `GROUP BY "${field}"`, - having: `HAVING COUNT("${field}") >= ${threshold}`, - start: 0, - sort: 'random', - count, - serverAccountIdForBlock: serverActor.Account.id, - displayOnlyForFollower: { - actorId: serverActor.id, - orLocalVideos: true - } - } - - const queryBuilder = new VideosIdListQueryBuilder(VideoModel.sequelize) - - return queryBuilder.queryVideoIds(queryOptions) - .then(rows => rows.map(r => r[field])) - } - - static buildTrendingQuery (trendingDays: number) { - return { - attributes: [], - subQuery: false, - model: VideoViewModel, - required: false, - where: { - startDate: { - // FIXME: ts error - [Op.gte as any]: new Date(new Date().getTime() - (24 * 3600 * 1000) * trendingDays) - } - } - } - } - - private static async getAvailableForApi ( - options: BuildVideosListQueryOptions, - countVideos = true - ): Promise> { - const span = tracer.startSpan('peertube.VideoModel.getAvailableForApi') - - function getCount () { - if (countVideos !== true) return Promise.resolve(undefined) - - const countOptions = Object.assign({}, options, { isCount: true }) - const queryBuilder = new VideosIdListQueryBuilder(VideoModel.sequelize) - - return queryBuilder.countVideoIds(countOptions) - } - - function getModels () { - if (options.count === 0) return Promise.resolve([]) - - const queryBuilder = new VideosModelListQueryBuilder(VideoModel.sequelize) - - return queryBuilder.queryVideos(options) - } - - const [ count, rows ] = await Promise.all([ getCount(), getModels() ]) - - span.end() - - return { - data: rows, - total: count - } - } - - private static throwIfPrivateIncludeWithoutUser (include: VideoInclude, user: MUserAccountId) { - if (VideoModel.isPrivateInclude(include) && !user?.hasRight(UserRight.SEE_ALL_VIDEOS)) { - throw new Error('Try to include protected videos but user cannot see all videos') - } - } - - private static throwIfPrivacyOneOfWithoutUser (privacyOneOf: VideoPrivacy[], user: MUserAccountId) { - if (privacyOneOf && !user?.hasRight(UserRight.SEE_ALL_VIDEOS)) { - throw new Error('Try to choose video privacies but user cannot see all videos') - } - } - - private static isPrivateInclude (include: VideoInclude) { - return include & VideoInclude.BLACKLISTED || - include & VideoInclude.BLOCKED_OWNER || - include & VideoInclude.NOT_PUBLISHED_STATE - } - - isBlacklisted () { - return !!this.VideoBlacklist - } - - isBlocked () { - return this.VideoChannel.Account.Actor.Server?.isBlocked() || this.VideoChannel.Account.isBlocked() - } - - getQualityFileBy (this: T, fun: (files: MVideoFile[], it: (file: MVideoFile) => number) => MVideoFile) { - const files = this.getAllFiles() - const file = fun(files, file => file.resolution) - if (!file) return undefined - - if (file.videoId) { - return Object.assign(file, { Video: this }) - } - - if (file.videoStreamingPlaylistId) { - const streamingPlaylistWithVideo = Object.assign(this.VideoStreamingPlaylists[0], { Video: this }) - - return Object.assign(file, { VideoStreamingPlaylist: streamingPlaylistWithVideo }) - } - - throw new Error('File is not associated to a video of a playlist') - } - - getMaxQualityFile (this: T): MVideoFileVideo | MVideoFileStreamingPlaylistVideo { - return this.getQualityFileBy(maxBy) - } - - getMinQualityFile (this: T): MVideoFileVideo | MVideoFileStreamingPlaylistVideo { - return this.getQualityFileBy(minBy) - } - - getWebVideoFile (this: T, resolution: number): MVideoFileVideo { - if (Array.isArray(this.VideoFiles) === false) return undefined - - const file = this.VideoFiles.find(f => f.resolution === resolution) - if (!file) return undefined - - return Object.assign(file, { Video: this }) - } - - hasWebVideoFiles () { - return Array.isArray(this.VideoFiles) === true && this.VideoFiles.length !== 0 - } - - async addAndSaveThumbnail (thumbnail: MThumbnail, transaction?: Transaction) { - thumbnail.videoId = this.id - - const savedThumbnail = await thumbnail.save({ transaction }) - - if (Array.isArray(this.Thumbnails) === false) this.Thumbnails = [] - - this.Thumbnails = this.Thumbnails.filter(t => t.id !== savedThumbnail.id) - this.Thumbnails.push(savedThumbnail) - } - - getMiniature () { - if (Array.isArray(this.Thumbnails) === false) return undefined - - return this.Thumbnails.find(t => t.type === ThumbnailType.MINIATURE) - } - - hasPreview () { - return !!this.getPreview() - } - - getPreview () { - if (Array.isArray(this.Thumbnails) === false) return undefined - - return this.Thumbnails.find(t => t.type === ThumbnailType.PREVIEW) - } - - isOwned () { - return this.remote === false - } - - getWatchStaticPath () { - return buildVideoWatchPath({ shortUUID: uuidToShort(this.uuid) }) - } - - getEmbedStaticPath () { - return buildVideoEmbedPath(this) - } - - getMiniatureStaticPath () { - const thumbnail = this.getMiniature() - if (!thumbnail) return null - - return thumbnail.getLocalStaticPath() - } - - getPreviewStaticPath () { - const preview = this.getPreview() - if (!preview) return null - - return preview.getLocalStaticPath() - } - - toFormattedJSON (this: MVideoFormattable, options?: VideoFormattingJSONOptions): Video { - return videoModelToFormattedJSON(this, options) - } - - toFormattedDetailsJSON (this: MVideoFormattableDetails): VideoDetails { - return videoModelToFormattedDetailsJSON(this) - } - - getFormattedWebVideoFilesJSON (includeMagnet = true): VideoFile[] { - return videoFilesModelToFormattedJSON(this, this.VideoFiles, { includeMagnet }) - } - - getFormattedHLSVideoFilesJSON (includeMagnet = true): VideoFile[] { - let acc: VideoFile[] = [] - - for (const p of this.VideoStreamingPlaylists) { - acc = acc.concat(videoFilesModelToFormattedJSON(this, p.VideoFiles, { includeMagnet })) - } - - return acc - } - - getFormattedAllVideoFilesJSON (includeMagnet = true): VideoFile[] { - let files: VideoFile[] = [] - - if (Array.isArray(this.VideoFiles)) { - files = files.concat(this.getFormattedWebVideoFilesJSON(includeMagnet)) - } - - if (Array.isArray(this.VideoStreamingPlaylists)) { - files = files.concat(this.getFormattedHLSVideoFilesJSON(includeMagnet)) - } - - return files - } - - toActivityPubObject (this: MVideoAP): Promise { - return Hooks.wrapObject( - videoModelToActivityPubObject(this), - 'filter:activity-pub.video.json-ld.build.result', - { video: this } - ) - } - - async lightAPToFullAP (this: MVideoAPLight, transaction: Transaction): Promise { - const videoAP = this as MVideoAP - - const getCaptions = () => { - if (isArray(videoAP.VideoCaptions)) return videoAP.VideoCaptions - - return this.$get('VideoCaptions', { - attributes: [ 'filename', 'language', 'fileUrl' ], - transaction - }) as Promise - } - - const getStoryboard = () => { - if (videoAP.Storyboard) return videoAP.Storyboard - - return this.$get('Storyboard', { transaction }) as Promise - } - - const [ captions, storyboard ] = await Promise.all([ getCaptions(), getStoryboard() ]) - - return Object.assign(this, { - VideoCaptions: captions, - Storyboard: storyboard - }) - } - - getTruncatedDescription () { - if (!this.description) return null - - const maxLength = CONSTRAINTS_FIELDS.VIDEOS.TRUNCATED_DESCRIPTION.max - return peertubeTruncate(this.description, { length: maxLength }) - } - - getAllFiles () { - let files: MVideoFile[] = [] - - if (Array.isArray(this.VideoFiles)) { - files = files.concat(this.VideoFiles) - } - - if (Array.isArray(this.VideoStreamingPlaylists)) { - for (const p of this.VideoStreamingPlaylists) { - if (Array.isArray(p.VideoFiles)) { - files = files.concat(p.VideoFiles) - } - } - } - - return files - } - - probeMaxQualityFile () { - const file = this.getMaxQualityFile() - const videoOrPlaylist = file.getVideoOrStreamingPlaylist() - - return VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(videoOrPlaylist), async originalFilePath => { - const probe = await ffprobePromise(originalFilePath) - - const { audioStream } = await getAudioStream(originalFilePath, probe) - const hasAudio = await hasAudioStream(originalFilePath, probe) - const fps = await getVideoStreamFPS(originalFilePath, probe) - - return { - audioStream, - hasAudio, - fps, - - ...await getVideoStreamDimensionsInfo(originalFilePath, probe) - } - }) - } - - getDescriptionAPIPath () { - return `/api/${API_VERSION}/videos/${this.uuid}/description` - } - - getHLSPlaylist (): MStreamingPlaylistFilesVideo { - if (!this.VideoStreamingPlaylists) return undefined - - const playlist = this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) - if (!playlist) return undefined - - return playlist.withVideo(this) - } - - setHLSPlaylist (playlist: MStreamingPlaylist) { - const toAdd = [ playlist ] as [ VideoStreamingPlaylistModel ] - - if (Array.isArray(this.VideoStreamingPlaylists) === false || this.VideoStreamingPlaylists.length === 0) { - this.VideoStreamingPlaylists = toAdd - return - } - - this.VideoStreamingPlaylists = this.VideoStreamingPlaylists - .filter(s => s.type !== VideoStreamingPlaylistType.HLS) - .concat(toAdd) - } - - removeWebVideoFile (videoFile: MVideoFile, isRedundancy = false) { - const filePath = isRedundancy - ? VideoPathManager.Instance.getFSRedundancyVideoFilePath(this, videoFile) - : VideoPathManager.Instance.getFSVideoFileOutputPath(this, videoFile) - - const promises: Promise[] = [ remove(filePath) ] - if (!isRedundancy) promises.push(videoFile.removeTorrent()) - - if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { - promises.push(removeWebVideoObjectStorage(videoFile)) - } - - return Promise.all(promises) - } - - async removeStreamingPlaylistFiles (streamingPlaylist: MStreamingPlaylist, isRedundancy = false) { - const directoryPath = isRedundancy - ? getHLSRedundancyDirectory(this) - : getHLSDirectory(this) - - await remove(directoryPath) - - if (isRedundancy !== true) { - const streamingPlaylistWithFiles = streamingPlaylist as MStreamingPlaylistFilesVideo - streamingPlaylistWithFiles.Video = this - - if (!Array.isArray(streamingPlaylistWithFiles.VideoFiles)) { - streamingPlaylistWithFiles.VideoFiles = await streamingPlaylistWithFiles.$get('VideoFiles') - } - - // Remove physical files and torrents - await Promise.all( - streamingPlaylistWithFiles.VideoFiles.map(file => file.removeTorrent()) - ) - - if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { - await removeHLSObjectStorage(streamingPlaylist.withVideo(this)) - } - } - } - - async removeStreamingPlaylistVideoFile (streamingPlaylist: MStreamingPlaylist, videoFile: MVideoFile) { - const filePath = VideoPathManager.Instance.getFSHLSOutputPath(this, videoFile.filename) - await videoFile.removeTorrent() - await remove(filePath) - - const resolutionFilename = getHlsResolutionPlaylistFilename(videoFile.filename) - await remove(VideoPathManager.Instance.getFSHLSOutputPath(this, resolutionFilename)) - - if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { - await removeHLSFileObjectStorageByFilename(streamingPlaylist.withVideo(this), videoFile.filename) - await removeHLSFileObjectStorageByFilename(streamingPlaylist.withVideo(this), resolutionFilename) - } - } - - async removeStreamingPlaylistFile (streamingPlaylist: MStreamingPlaylist, filename: string) { - const filePath = VideoPathManager.Instance.getFSHLSOutputPath(this, filename) - await remove(filePath) - - if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { - await removeHLSFileObjectStorageByFilename(streamingPlaylist.withVideo(this), filename) - } - } - - isOutdated () { - if (this.isOwned()) return false - - return isOutdated(this, ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL) - } - - hasPrivacyForFederation () { - return isPrivacyForFederation(this.privacy) - } - - hasStateForFederation () { - return isStateForFederation(this.state) - } - - isNewVideo (newPrivacy: VideoPrivacy) { - return this.hasPrivacyForFederation() === false && isPrivacyForFederation(newPrivacy) === true - } - - setAsRefreshed (transaction?: Transaction) { - return setAsUpdated({ sequelize: this.sequelize, table: 'video', id: this.id, transaction }) - } - - // --------------------------------------------------------------------------- - - requiresUserAuth (options: { - urlParamId: string - checkBlacklist: boolean - }) { - const { urlParamId, checkBlacklist } = options - - if (this.privacy === VideoPrivacy.PRIVATE || this.privacy === VideoPrivacy.INTERNAL) { - return true - } - - if (this.privacy === VideoPrivacy.UNLISTED) { - if (urlParamId && !isUUIDValid(urlParamId)) return true - - return false - } - - if (checkBlacklist && this.VideoBlacklist) return true - - if (this.privacy === VideoPrivacy.PUBLIC || this.privacy === VideoPrivacy.PASSWORD_PROTECTED) { - return false - } - - throw new Error(`Unknown video privacy ${this.privacy} to know if the video requires auth`) - } - - hasPrivateStaticPath () { - return isVideoInPrivateDirectory(this.privacy) - } - - // --------------------------------------------------------------------------- - - async setNewState (newState: VideoState, isNewVideo: boolean, transaction: Transaction) { - if (this.state === newState) throw new Error('Cannot use same state ' + newState) - - this.state = newState - - if (this.state === VideoState.PUBLISHED && isNewVideo) { - this.publishedAt = new Date() - } - - await this.save({ transaction }) - } - - getBandwidthBits (this: MVideo, videoFile: MVideoFile) { - if (!this.duration) return videoFile.size - - return Math.ceil((videoFile.size * 8) / this.duration) - } - - getTrackerUrls () { - if (this.isOwned()) { - return [ - WEBSERVER.URL + '/tracker/announce', - WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' - ] - } - - return this.Trackers.map(t => t.url) - } -} diff --git a/server/models/view/local-video-viewer-watch-section.ts b/server/models/view/local-video-viewer-watch-section.ts deleted file mode 100644 index e29bb7847..000000000 --- a/server/models/view/local-video-viewer-watch-section.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { Transaction } from 'sequelize' -import { AllowNull, BelongsTo, Column, CreatedAt, ForeignKey, Model, Table } from 'sequelize-typescript' -import { MLocalVideoViewerWatchSection } from '@server/types/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { LocalVideoViewerModel } from './local-video-viewer' - -@Table({ - tableName: 'localVideoViewerWatchSection', - updatedAt: false, - indexes: [ - { - fields: [ 'localVideoViewerId' ] - } - ] -}) -export class LocalVideoViewerWatchSectionModel extends Model>> { - @CreatedAt - createdAt: Date - - @AllowNull(false) - @Column - watchStart: number - - @AllowNull(false) - @Column - watchEnd: number - - @ForeignKey(() => LocalVideoViewerModel) - @Column - localVideoViewerId: number - - @BelongsTo(() => LocalVideoViewerModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'CASCADE' - }) - LocalVideoViewer: LocalVideoViewerModel - - static async bulkCreateSections (options: { - localVideoViewerId: number - watchSections: { - start: number - end: number - }[] - transaction?: Transaction - }) { - const { localVideoViewerId, watchSections, transaction } = options - const models: MLocalVideoViewerWatchSection[] = [] - - for (const section of watchSections) { - const model = await this.create({ - watchStart: section.start, - watchEnd: section.end, - localVideoViewerId - }, { transaction }) - - models.push(model) - } - - return models - } -} diff --git a/server/models/view/local-video-viewer.ts b/server/models/view/local-video-viewer.ts deleted file mode 100644 index c7ac51a03..000000000 --- a/server/models/view/local-video-viewer.ts +++ /dev/null @@ -1,368 +0,0 @@ -import { QueryTypes } from 'sequelize' -import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, HasMany, IsUUID, Model, Table } from 'sequelize-typescript' -import { getActivityStreamDuration } from '@server/lib/activitypub/activity' -import { buildGroupByAndBoundaries } from '@server/lib/timeserie' -import { MLocalVideoViewer, MLocalVideoViewerWithWatchSections, MVideo } from '@server/types/models' -import { VideoStatsOverall, VideoStatsRetention, VideoStatsTimeserie, VideoStatsTimeserieMetric, WatchActionObject } from '@shared/models' -import { AttributesOnly } from '@shared/typescript-utils' -import { VideoModel } from '../video/video' -import { LocalVideoViewerWatchSectionModel } from './local-video-viewer-watch-section' - -/** - * - * Aggregate viewers of local videos only to display statistics to video owners - * A viewer is a user that watched one or multiple sections of a specific video inside a time window - * - */ - -@Table({ - tableName: 'localVideoViewer', - updatedAt: false, - indexes: [ - { - fields: [ 'videoId' ] - }, - { - fields: [ 'url' ], - unique: true - } - ] -}) -export class LocalVideoViewerModel extends Model>> { - @CreatedAt - createdAt: Date - - @AllowNull(false) - @Column(DataType.DATE) - startDate: Date - - @AllowNull(false) - @Column(DataType.DATE) - endDate: Date - - @AllowNull(false) - @Column - watchTime: number - - @AllowNull(true) - @Column - country: string - - @AllowNull(false) - @Default(DataType.UUIDV4) - @IsUUID(4) - @Column(DataType.UUID) - uuid: string - - @AllowNull(false) - @Column - url: string - - @ForeignKey(() => VideoModel) - @Column - videoId: number - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'CASCADE' - }) - Video: VideoModel - - @HasMany(() => LocalVideoViewerWatchSectionModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'cascade' - }) - WatchSections: LocalVideoViewerWatchSectionModel[] - - static loadByUrl (url: string): Promise { - return this.findOne({ - where: { - url - } - }) - } - - static loadFullById (id: number): Promise { - return this.findOne({ - include: [ - { - model: VideoModel.unscoped(), - required: true - }, - { - model: LocalVideoViewerWatchSectionModel.unscoped(), - required: true - } - ], - where: { - id - } - }) - } - - static async getOverallStats (options: { - video: MVideo - startDate?: string - endDate?: string - }): Promise { - const { video, startDate, endDate } = options - - const queryOptions = { - type: QueryTypes.SELECT as QueryTypes.SELECT, - replacements: { videoId: video.id } as any - } - - if (startDate) queryOptions.replacements.startDate = startDate - if (endDate) queryOptions.replacements.endDate = endDate - - const buildTotalViewersPromise = () => { - let totalViewersDateWhere = '' - - if (startDate) totalViewersDateWhere += ' AND "localVideoViewer"."endDate" >= :startDate' - if (endDate) totalViewersDateWhere += ' AND "localVideoViewer"."startDate" <= :endDate' - - const totalViewersQuery = `SELECT ` + - `COUNT("localVideoViewer"."id") AS "totalViewers" ` + - `FROM "localVideoViewer" ` + - `WHERE "videoId" = :videoId ${totalViewersDateWhere}` - - return LocalVideoViewerModel.sequelize.query(totalViewersQuery, queryOptions) - } - - const buildWatchTimePromise = () => { - let watchTimeDateWhere = '' - - // We know this where is not exact - // But we prefer to take into account only watch section that started and ended **in** the interval - if (startDate) watchTimeDateWhere += ' AND "localVideoViewer"."startDate" >= :startDate' - if (endDate) watchTimeDateWhere += ' AND "localVideoViewer"."endDate" <= :endDate' - - const watchTimeQuery = `SELECT ` + - `SUM("localVideoViewer"."watchTime") AS "totalWatchTime", ` + - `AVG("localVideoViewer"."watchTime") AS "averageWatchTime" ` + - `FROM "localVideoViewer" ` + - `WHERE "videoId" = :videoId ${watchTimeDateWhere}` - - return LocalVideoViewerModel.sequelize.query(watchTimeQuery, queryOptions) - } - - const buildWatchPeakPromise = () => { - let watchPeakDateWhereStart = '' - let watchPeakDateWhereEnd = '' - - if (startDate) { - watchPeakDateWhereStart += ' AND "localVideoViewer"."startDate" >= :startDate' - watchPeakDateWhereEnd += ' AND "localVideoViewer"."endDate" >= :startDate' - } - - if (endDate) { - watchPeakDateWhereStart += ' AND "localVideoViewer"."startDate" <= :endDate' - watchPeakDateWhereEnd += ' AND "localVideoViewer"."endDate" <= :endDate' - } - - // Add viewers that were already here, before our start date - const beforeWatchersQuery = startDate - // eslint-disable-next-line max-len - ? `SELECT COUNT(*) AS "total" FROM "localVideoViewer" WHERE "localVideoViewer"."startDate" < :startDate AND "localVideoViewer"."endDate" >= :startDate` - : `SELECT 0 AS "total"` - - const watchPeakQuery = `WITH - "beforeWatchers" AS (${beforeWatchersQuery}), - "watchPeakValues" AS ( - SELECT "startDate" AS "dateBreakpoint", 1 AS "inc" - FROM "localVideoViewer" - WHERE "videoId" = :videoId ${watchPeakDateWhereStart} - UNION ALL - SELECT "endDate" AS "dateBreakpoint", -1 AS "inc" - FROM "localVideoViewer" - WHERE "videoId" = :videoId ${watchPeakDateWhereEnd} - ) - SELECT "dateBreakpoint", "concurrent" - FROM ( - SELECT "dateBreakpoint", SUM(SUM("inc")) OVER (ORDER BY "dateBreakpoint") + (SELECT "total" FROM "beforeWatchers") AS "concurrent" - FROM "watchPeakValues" - GROUP BY "dateBreakpoint" - ) tmp - ORDER BY "concurrent" DESC - FETCH FIRST 1 ROW ONLY` - - return LocalVideoViewerModel.sequelize.query(watchPeakQuery, queryOptions) - } - - const buildCountriesPromise = () => { - let countryDateWhere = '' - - if (startDate) countryDateWhere += ' AND "localVideoViewer"."endDate" >= :startDate' - if (endDate) countryDateWhere += ' AND "localVideoViewer"."startDate" <= :endDate' - - const countriesQuery = `SELECT country, COUNT(country) as viewers ` + - `FROM "localVideoViewer" ` + - `WHERE "videoId" = :videoId AND country IS NOT NULL ${countryDateWhere} ` + - `GROUP BY country ` + - `ORDER BY viewers DESC` - - return LocalVideoViewerModel.sequelize.query(countriesQuery, queryOptions) - } - - const [ rowsTotalViewers, rowsWatchTime, rowsWatchPeak, rowsCountries ] = await Promise.all([ - buildTotalViewersPromise(), - buildWatchTimePromise(), - buildWatchPeakPromise(), - buildCountriesPromise() - ]) - - const viewersPeak = rowsWatchPeak.length !== 0 - ? parseInt(rowsWatchPeak[0].concurrent) || 0 - : 0 - - return { - totalWatchTime: rowsWatchTime.length !== 0 - ? Math.round(rowsWatchTime[0].totalWatchTime) || 0 - : 0, - averageWatchTime: rowsWatchTime.length !== 0 - ? Math.round(rowsWatchTime[0].averageWatchTime) || 0 - : 0, - - totalViewers: rowsTotalViewers.length !== 0 - ? Math.round(rowsTotalViewers[0].totalViewers) || 0 - : 0, - - viewersPeak, - viewersPeakDate: rowsWatchPeak.length !== 0 && viewersPeak !== 0 - ? rowsWatchPeak[0].dateBreakpoint || null - : null, - - countries: rowsCountries.map(r => ({ - isoCode: r.country, - viewers: r.viewers - })) - } - } - - static async getRetentionStats (video: MVideo): Promise { - const step = Math.max(Math.round(video.duration / 100), 1) - - const query = `WITH "total" AS (SELECT COUNT(*) AS viewers FROM "localVideoViewer" WHERE "videoId" = :videoId) ` + - `SELECT serie AS "second", ` + - `(COUNT("localVideoViewer".id)::float / (SELECT GREATEST("total"."viewers", 1) FROM "total")) AS "retention" ` + - `FROM generate_series(0, ${video.duration}, ${step}) serie ` + - `LEFT JOIN "localVideoViewer" ON "localVideoViewer"."videoId" = :videoId ` + - `AND EXISTS (` + - `SELECT 1 FROM "localVideoViewerWatchSection" ` + - `WHERE "localVideoViewer"."id" = "localVideoViewerWatchSection"."localVideoViewerId" ` + - `AND serie >= "localVideoViewerWatchSection"."watchStart" ` + - `AND serie <= "localVideoViewerWatchSection"."watchEnd"` + - `)` + - `GROUP BY serie ` + - `ORDER BY serie ASC` - - const queryOptions = { - type: QueryTypes.SELECT as QueryTypes.SELECT, - replacements: { videoId: video.id } - } - - const rows = await LocalVideoViewerModel.sequelize.query(query, queryOptions) - - return { - data: rows.map(r => ({ - second: r.second, - retentionPercent: parseFloat(r.retention) * 100 - })) - } - } - - static async getTimeserieStats (options: { - video: MVideo - metric: VideoStatsTimeserieMetric - startDate: string - endDate: string - }): Promise { - const { video, metric } = options - - const { groupInterval, startDate, endDate } = buildGroupByAndBoundaries(options.startDate, options.endDate) - - const selectMetrics: { [ id in VideoStatsTimeserieMetric ]: string } = { - viewers: 'COUNT("localVideoViewer"."id")', - aggregateWatchTime: 'SUM("localVideoViewer"."watchTime")' - } - - const intervalWhere: { [ id in VideoStatsTimeserieMetric ]: string } = { - // Viewer is still in the interval. Overlap algorithm - viewers: '"localVideoViewer"."startDate" <= "intervals"."endDate" ' + - 'AND "localVideoViewer"."endDate" >= "intervals"."startDate"', - - // We do an aggregation, so only sum things once. Arbitrary we use the end date for that purpose - aggregateWatchTime: '"localVideoViewer"."endDate" >= "intervals"."startDate" ' + - 'AND "localVideoViewer"."endDate" <= "intervals"."endDate"' - } - - const query = `WITH "intervals" AS ( - SELECT - "time" AS "startDate", "time" + :groupInterval::interval as "endDate" - FROM - generate_series(:startDate::timestamptz, :endDate::timestamptz, :groupInterval::interval) serie("time") - ) - SELECT "intervals"."startDate" as "date", COALESCE(${selectMetrics[metric]}, 0) AS value - FROM - intervals - LEFT JOIN "localVideoViewer" ON "localVideoViewer"."videoId" = :videoId - AND ${intervalWhere[metric]} - GROUP BY - "intervals"."startDate" - ORDER BY - "intervals"."startDate"` - - const queryOptions = { - type: QueryTypes.SELECT as QueryTypes.SELECT, - replacements: { - startDate, - endDate, - groupInterval, - videoId: video.id - } - } - - const rows = await LocalVideoViewerModel.sequelize.query(query, queryOptions) - - return { - groupInterval, - data: rows.map(r => ({ - date: r.date, - value: parseInt(r.value) - })) - } - } - - toActivityPubObject (this: MLocalVideoViewerWithWatchSections): WatchActionObject { - const location = this.country - ? { - location: { - addressCountry: this.country - } - } - : {} - - return { - id: this.url, - type: 'WatchAction', - duration: getActivityStreamDuration(this.watchTime), - startTime: this.startDate.toISOString(), - endTime: this.endDate.toISOString(), - - object: this.Video.url, - uuid: this.uuid, - actionStatus: 'CompletedActionStatus', - - watchSections: this.WatchSections.map(w => ({ - startTimestamp: w.watchStart, - endTimestamp: w.watchEnd - })), - - ...location - } - } -} diff --git a/server/models/view/video-view.ts b/server/models/view/video-view.ts deleted file mode 100644 index 1504a364e..000000000 --- a/server/models/view/video-view.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { literal, Op } from 'sequelize' -import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table } from 'sequelize-typescript' -import { AttributesOnly } from '@shared/typescript-utils' -import { VideoModel } from '../video/video' - -/** - * - * Aggregate views of all videos federated with our instance - * Mainly used by the trending/hot algorithms - * - */ - -@Table({ - tableName: 'videoView', - updatedAt: false, - indexes: [ - { - fields: [ 'videoId' ] - }, - { - fields: [ 'startDate' ] - } - ] -}) -export class VideoViewModel extends Model>> { - @CreatedAt - createdAt: Date - - @AllowNull(false) - @Column(DataType.DATE) - startDate: Date - - @AllowNull(false) - @Column(DataType.DATE) - endDate: Date - - @AllowNull(false) - @Column - views: number - - @ForeignKey(() => VideoModel) - @Column - videoId: number - - @BelongsTo(() => VideoModel, { - foreignKey: { - allowNull: false - }, - onDelete: 'CASCADE' - }) - Video: VideoModel - - static removeOldRemoteViewsHistory (beforeDate: string) { - const query = { - where: { - startDate: { - [Op.lt]: beforeDate - }, - videoId: { - [Op.in]: literal('(SELECT "id" FROM "video" WHERE "remote" IS TRUE)') - } - } - } - - return VideoViewModel.destroy(query) - } -} diff --git a/server/package.json b/server/package.json new file mode 100644 index 000000000..2bf62ff85 --- /dev/null +++ b/server/package.json @@ -0,0 +1,12 @@ +{ + "name": "@peertube/peertube-server", + "private": true, + "version": "0.0.0", + "files": [ "dist" ], + "type": "module", + "exports": { + "./*": "./dist/*" + }, + "devDependencies": {}, + "dependencies": {} +} diff --git a/server/scripts/create-generate-storyboard-job.ts b/server/scripts/create-generate-storyboard-job.ts new file mode 100644 index 000000000..1f70e4d15 --- /dev/null +++ b/server/scripts/create-generate-storyboard-job.ts @@ -0,0 +1,85 @@ +import { program } from 'commander' +import { toCompleteUUID } from '@server/helpers/custom-validators/misc.js' +import { initDatabaseModels } from '@server/initializers/database.js' +import { JobQueue } from '@server/lib/job-queue/index.js' +import { StoryboardModel } from '@server/models/video/storyboard.js' +import { VideoModel } from '@server/models/video/video.js' + +program + .description('Generate videos storyboard') + .option('-v, --video [videoUUID]', 'Generate the storyboard of a specific video') + .option('-a, --all-videos', 'Generate missing storyboards of local videos') + .parse(process.argv) + +const options = program.opts() + +if (!options['video'] && !options['allVideos']) { + console.error('You need to choose videos for storyboard generation.') + process.exit(-1) +} + +run() + .then(() => process.exit(0)) + .catch(err => { + console.error(err) + process.exit(-1) + }) + +async function run () { + await initDatabaseModels(true) + + JobQueue.Instance.init() + + let ids: number[] = [] + + if (options['video']) { + const video = await VideoModel.load(toCompleteUUID(options['video'])) + + if (!video) { + console.error('Unknown video ' + options['video']) + process.exit(-1) + } + + if (video.remote === true) { + console.error('Cannot process a remote video') + process.exit(-1) + } + + if (video.isLive) { + console.error('Cannot process live video') + process.exit(-1) + } + + ids.push(video.id) + } else { + ids = await listLocalMissingStoryboards() + } + + for (const id of ids) { + const videoFull = await VideoModel.load(id) + + if (videoFull.isLive) continue + + await JobQueue.Instance.createJob({ + type: 'generate-video-storyboard', + payload: { + videoUUID: videoFull.uuid, + federate: true + } + }) + + console.log(`Created generate-storyboard job for ${videoFull.name}.`) + } +} + +async function listLocalMissingStoryboards () { + const ids = await VideoModel.listLocalIds() + const results: number[] = [] + + for (const id of ids) { + const storyboard = await StoryboardModel.loadByVideo(id) + if (!storyboard) results.push(id) + } + + return results +} diff --git a/server/scripts/create-import-video-file-job.ts b/server/scripts/create-import-video-file-job.ts new file mode 100644 index 000000000..33a45fba1 --- /dev/null +++ b/server/scripts/create-import-video-file-job.ts @@ -0,0 +1,50 @@ +import { program } from 'commander' +import { resolve } from 'path' +import { isUUIDValid, toCompleteUUID } from '@server/helpers/custom-validators/misc.js' +import { initDatabaseModels } from '../server/initializers/database.js' +import { JobQueue } from '../server/lib/job-queue/index.js' +import { VideoModel } from '../server/models/video/video.js' + +program + .option('-v, --video [videoUUID]', 'Video UUID') + .option('-i, --import [videoFile]', 'Video file') + .description('Import a video file to replace an already uploaded file or to add a new resolution') + .parse(process.argv) + +const options = program.opts() + +if (options.video === undefined || options.import === undefined) { + console.error('All parameters are mandatory.') + process.exit(-1) +} + +run() + .then(() => process.exit(0)) + .catch(err => { + console.error(err) + process.exit(-1) + }) + +async function run () { + await initDatabaseModels(true) + + const uuid = toCompleteUUID(options.video) + + if (isUUIDValid(uuid) === false) { + console.error('%s is not a valid video UUID.', options.video) + return + } + + const video = await VideoModel.load(uuid) + if (!video) throw new Error('Video not found.') + if (video.isOwned() === false) throw new Error('Cannot import files of a non owned video.') + + const dataInput = { + videoUUID: video.uuid, + filePath: resolve(options.import) + } + + JobQueue.Instance.init() + await JobQueue.Instance.createJob({ type: 'video-file-import', payload: dataInput }) + console.log('Import job for video %s created.', video.uuid) +} diff --git a/server/scripts/create-move-video-storage-job.ts b/server/scripts/create-move-video-storage-job.ts new file mode 100644 index 000000000..a615d1f44 --- /dev/null +++ b/server/scripts/create-move-video-storage-job.ts @@ -0,0 +1,99 @@ +import { program } from 'commander' +import { toCompleteUUID } from '@server/helpers/custom-validators/misc.js' +import { CONFIG } from '@server/initializers/config.js' +import { initDatabaseModels } from '@server/initializers/database.js' +import { JobQueue } from '@server/lib/job-queue/index.js' +import { moveToExternalStorageState } from '@server/lib/video-state.js' +import { VideoModel } from '@server/models/video/video.js' +import { VideoState, VideoStorage } from '@peertube/peertube-models' + +program + .description('Move videos to another storage.') + .option('-o, --to-object-storage', 'Move videos in object storage') + .option('-v, --video [videoUUID]', 'Move a specific video') + .option('-a, --all-videos', 'Migrate all videos') + .parse(process.argv) + +const options = program.opts() + +if (!options['toObjectStorage']) { + console.error('You need to choose where to send video files.') + process.exit(-1) +} + +if (!options['video'] && !options['allVideos']) { + console.error('You need to choose which videos to move.') + process.exit(-1) +} + +if (options['toObjectStorage'] && !CONFIG.OBJECT_STORAGE.ENABLED) { + console.error('Object storage is not enabled on this instance.') + process.exit(-1) +} + +run() + .then(() => process.exit(0)) + .catch(err => { + console.error(err) + process.exit(-1) + }) + +async function run () { + await initDatabaseModels(true) + + JobQueue.Instance.init() + + let ids: number[] = [] + + if (options['video']) { + const video = await VideoModel.load(toCompleteUUID(options['video'])) + + if (!video) { + console.error('Unknown video ' + options['video']) + process.exit(-1) + } + + if (video.remote === true) { + console.error('Cannot process a remote video') + process.exit(-1) + } + + if (video.isLive) { + console.error('Cannot process live video') + process.exit(-1) + } + + if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { + console.error('This video is already being moved to external storage') + process.exit(-1) + } + + ids.push(video.id) + } else { + ids = await VideoModel.listLocalIds() + } + + for (const id of ids) { + const videoFull = await VideoModel.loadFull(id) + + if (videoFull.isLive) continue + + const files = videoFull.VideoFiles || [] + const hls = videoFull.getHLSPlaylist() + + if (files.some(f => f.storage === VideoStorage.FILE_SYSTEM) || hls?.storage === VideoStorage.FILE_SYSTEM) { + console.log('Processing video %s.', videoFull.name) + + const success = await moveToExternalStorageState({ video: videoFull, isNewVideo: false, transaction: undefined }) + + if (!success) { + console.error( + 'Cannot create move job for %s: job creation may have failed or there may be pending transcoding jobs for this video', + videoFull.name + ) + } + } + + console.log(`Created move-to-object-storage job for ${videoFull.name}.`) + } +} diff --git a/server/scripts/migrations/peertube-4.0.ts b/server/scripts/migrations/peertube-4.0.ts new file mode 100644 index 000000000..619c1da71 --- /dev/null +++ b/server/scripts/migrations/peertube-4.0.ts @@ -0,0 +1,110 @@ +import Bluebird from 'bluebird' +import { move } from 'fs-extra/esm' +import { readFile, writeFile } from 'fs/promises' +import { join } from 'path' +import { initDatabaseModels } from '@server/initializers/database.js' +import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js' +import { JobQueue } from '@server/lib/job-queue/index.js' +import { + generateHLSMasterPlaylistFilename, + generateHlsSha256SegmentsFilename, + getHlsResolutionPlaylistFilename +} from '@server/lib/paths.js' +import { VideoPathManager } from '@server/lib/video-path-manager.js' +import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js' +import { VideoModel } from '@server/models/video/video.js' + +run() + .then(() => process.exit(0)) + .catch(err => { + console.error(err) + process.exit(-1) + + }) + +async function run () { + console.log('Migrate old HLS paths to new format.') + + await initDatabaseModels(true) + + JobQueue.Instance.init() + + const ids = await VideoModel.listLocalIds() + + await Bluebird.map(ids, async id => { + try { + await processVideo(id) + } catch (err) { + console.error('Cannot process video %s.', { err }) + } + }, { concurrency: 5 }) + + console.log('Migration finished!') +} + +async function processVideo (videoId: number) { + const video = await VideoModel.loadWithFiles(videoId) + + const hls = video.getHLSPlaylist() + if (video.isLive || !hls || hls.playlistFilename !== 'master.m3u8' || hls.VideoFiles.length === 0) { + return + } + + console.log(`Renaming HLS playlist files of video ${video.name}.`) + + const playlist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id) + const hlsDirPath = VideoPathManager.Instance.getFSHLSOutputPath(video) + + const masterPlaylistPath = join(hlsDirPath, playlist.playlistFilename) + let masterPlaylistContent = await readFile(masterPlaylistPath, 'utf8') + + for (const videoFile of hls.VideoFiles) { + const srcName = `${videoFile.resolution}.m3u8` + const dstName = getHlsResolutionPlaylistFilename(videoFile.filename) + + const src = join(hlsDirPath, srcName) + const dst = join(hlsDirPath, dstName) + + try { + await move(src, dst) + + masterPlaylistContent = masterPlaylistContent.replace(new RegExp('^' + srcName + '$', 'm'), dstName) + } catch (err) { + console.error('Cannot move video file %s to %s.', src, dst, err) + } + } + + await writeFile(masterPlaylistPath, masterPlaylistContent) + + if (playlist.segmentsSha256Filename === 'segments-sha256.json') { + try { + const newName = generateHlsSha256SegmentsFilename(video.isLive) + + const dst = join(hlsDirPath, newName) + await move(join(hlsDirPath, playlist.segmentsSha256Filename), dst) + playlist.segmentsSha256Filename = newName + } catch (err) { + console.error(`Cannot rename ${video.name} segments-sha256.json file to a new name`, err) + } + } + + if (playlist.playlistFilename === 'master.m3u8') { + try { + const newName = generateHLSMasterPlaylistFilename(video.isLive) + + const dst = join(hlsDirPath, newName) + await move(join(hlsDirPath, playlist.playlistFilename), dst) + playlist.playlistFilename = newName + } catch (err) { + console.error(`Cannot rename ${video.name} master.m3u8 file to a new name`, err) + } + } + + // Everything worked, we can save the playlist now + await playlist.save() + + const allVideo = await VideoModel.loadFull(video.id) + await federateVideoIfNeeded(allVideo, false) + + console.log(`Successfully moved HLS files of ${video.name}.`) +} diff --git a/server/scripts/migrations/peertube-4.2.ts b/server/scripts/migrations/peertube-4.2.ts new file mode 100644 index 000000000..6c89ee39e --- /dev/null +++ b/server/scripts/migrations/peertube-4.2.ts @@ -0,0 +1,123 @@ +import { ActorImageType } from '@peertube/peertube-models' +import { buildUUID, getLowercaseExtension } from '@peertube/peertube-node-utils' +import { getImageSize, processImage } from '@server/helpers/image-utils.js' +import { CONFIG } from '@server/initializers/config.js' +import { ACTOR_IMAGES_SIZE } from '@server/initializers/constants.js' +import { initDatabaseModels } from '@server/initializers/database.js' +import { updateActorImages } from '@server/lib/activitypub/actors/index.js' +import { sendUpdateActor } from '@server/lib/activitypub/send/index.js' +import { getBiggestActorImage } from '@server/lib/actor-image.js' +import { JobQueue } from '@server/lib/job-queue/index.js' +import { AccountModel } from '@server/models/account/account.js' +import { ActorModel } from '@server/models/actor/actor.js' +import { VideoChannelModel } from '@server/models/video/video-channel.js' +import { MAccountDefault, MActorDefault, MChannelDefault } from '@server/types/models/index.js' +import minBy from 'lodash-es/minBy.js' +import { join } from 'path' + +run() + .then(() => process.exit(0)) + .catch(err => { + console.error(err) + process.exit(-1) + }) + +async function run () { + console.log('Generate avatar miniatures from existing avatars.') + + await initDatabaseModels(true) + JobQueue.Instance.init() + + const accounts: AccountModel[] = await AccountModel.findAll({ + include: [ + { + model: ActorModel, + required: true, + where: { + serverId: null + } + }, + { + model: VideoChannelModel, + include: [ + { + model: AccountModel + } + ] + } + ] + }) + + for (const account of accounts) { + try { + await fillAvatarSizeIfNeeded(account) + await generateSmallerAvatarIfNeeded(account) + } catch (err) { + console.error(`Cannot process account avatar ${account.name}`, err) + } + + for (const videoChannel of account.VideoChannels) { + try { + await fillAvatarSizeIfNeeded(videoChannel) + await generateSmallerAvatarIfNeeded(videoChannel) + } catch (err) { + console.error(`Cannot process channel avatar ${videoChannel.name}`, err) + } + } + } + + console.log('Generation finished!') +} + +async function fillAvatarSizeIfNeeded (accountOrChannel: MAccountDefault | MChannelDefault) { + const avatars = accountOrChannel.Actor.Avatars + + for (const avatar of avatars) { + if (avatar.width && avatar.height) continue + + console.log('Filling size of avatars of %s.', accountOrChannel.name) + + const { width, height } = await getImageSize(join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, avatar.filename)) + avatar.width = width + avatar.height = height + + await avatar.save() + } +} + +async function generateSmallerAvatarIfNeeded (accountOrChannel: MAccountDefault | MChannelDefault) { + const avatars = accountOrChannel.Actor.Avatars + if (avatars.length !== 1) { + return + } + + console.log(`Processing ${accountOrChannel.name}.`) + + await generateSmallerAvatar(accountOrChannel.Actor) + accountOrChannel.Actor = Object.assign(accountOrChannel.Actor, { Server: null }) + + return sendUpdateActor(accountOrChannel, undefined) +} + +async function generateSmallerAvatar (actor: MActorDefault) { + const bigAvatar = getBiggestActorImage(actor.Avatars) + + const imageSize = minBy(ACTOR_IMAGES_SIZE[ActorImageType.AVATAR], 'width') + const sourceFilename = bigAvatar.filename + + const newImageName = buildUUID() + getLowercaseExtension(sourceFilename) + const source = join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, sourceFilename) + const destination = join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, newImageName) + + await processImage({ path: source, destination, newSize: imageSize, keepOriginal: true }) + + const actorImageInfo = { + name: newImageName, + fileUrl: null, + height: imageSize.height, + width: imageSize.width, + onDisk: true + } + + await updateActorImages(actor, ActorImageType.AVATAR, [ actorImageInfo ], undefined) +} diff --git a/server/scripts/migrations/peertube-5.0.ts b/server/scripts/migrations/peertube-5.0.ts new file mode 100644 index 000000000..6139abd08 --- /dev/null +++ b/server/scripts/migrations/peertube-5.0.ts @@ -0,0 +1,71 @@ +import { ensureDir } from 'fs-extra/esm' +import { Op } from 'sequelize' +import { updateTorrentMetadata } from '@server/helpers/webtorrent.js' +import { DIRECTORIES } from '@server/initializers/constants.js' +import { moveFilesIfPrivacyChanged } from '@server/lib/video-privacy.js' +import { VideoModel } from '@server/models/video/video.js' +import { MVideoFullLight } from '@server/types/models/index.js' +import { VideoPrivacy } from '@peertube/peertube-models' +import { initDatabaseModels } from '@server/initializers/database.js' + +run() + .then(() => process.exit(0)) + .catch(err => { + console.error(err) + process.exit(-1) + }) + +async function run () { + console.log('Moving private video files in dedicated folders.') + + await ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE) + await ensureDir(DIRECTORIES.VIDEOS.PRIVATE) + + await initDatabaseModels(true) + + const videos = await VideoModel.unscoped().findAll({ + attributes: [ 'uuid' ], + where: { + privacy: { + [Op.in]: [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ] + } + } + }) + + for (const { uuid } of videos) { + try { + console.log('Moving files of video %s.', uuid) + + const video = await VideoModel.loadFull(uuid) + + try { + await moveFilesIfPrivacyChanged(video, VideoPrivacy.PUBLIC) + } catch (err) { + console.error('Cannot move files of video %s.', uuid, err) + } + + try { + await updateTorrents(video) + } catch (err) { + console.error('Cannot regenerate torrents of video %s.', uuid, err) + } + } catch (err) { + console.error('Cannot process video %s.', uuid, err) + } + } +} + +async function updateTorrents (video: MVideoFullLight) { + for (const file of video.VideoFiles) { + await updateTorrentMetadata(video, file) + + await file.save() + } + + const playlist = video.getHLSPlaylist() + for (const file of (playlist?.VideoFiles || [])) { + await updateTorrentMetadata(playlist, file) + + await file.save() + } +} diff --git a/server/scripts/parse-log.ts b/server/scripts/parse-log.ts new file mode 100755 index 000000000..e80c0d927 --- /dev/null +++ b/server/scripts/parse-log.ts @@ -0,0 +1,161 @@ +import { program } from 'commander' +import { createReadStream } from 'fs' +import { readdir } from 'fs/promises' +import { join } from 'path' +import { stdin } from 'process' +import { createInterface } from 'readline' +import { format as sqlFormat } from 'sql-formatter' +import { inspect } from 'util' +import * as winston from 'winston' +import { labelFormatter, mtimeSortFilesDesc } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' + +program + .option('-l, --level [level]', 'Level log (debug/info/warn/error)') + .option('-f, --files [file...]', 'Files to parse. If not provided, the script will parse the latest log file from config)') + .option('-t, --tags [tags...]', 'Display only lines with these tags') + .option('-nt, --not-tags [tags...]', 'Donrt display lines containing these tags') + .parse(process.argv) + +const options = program.opts() + +const excludedKeys = { + level: true, + message: true, + splat: true, + timestamp: true, + tags: true, + label: true, + sql: true +} +function keysExcluder (key, value) { + return excludedKeys[key] === true ? undefined : value +} + +const loggerFormat = winston.format.printf((info) => { + let additionalInfos = JSON.stringify(info, keysExcluder, 2) + if (additionalInfos === '{}') additionalInfos = '' + else additionalInfos = ' ' + additionalInfos + + if (info.sql) { + if (CONFIG.LOG.PRETTIFY_SQL) { + additionalInfos += '\n' + sqlFormat(info.sql, { + language: 'sql', + tabWidth: 2 + }) + } else { + additionalInfos += ' - ' + info.sql + } + } + + return `[${info.label}] ${toTimeFormat(info.timestamp)} ${info.level}: ${info.message}${additionalInfos}` +}) + +const logger = winston.createLogger({ + transports: [ + new winston.transports.Console({ + level: options.level || 'debug', + stderrLevels: [], + format: winston.format.combine( + winston.format.splat(), + labelFormatter(), + winston.format.colorize(), + loggerFormat + ) + }) + ], + exitOnError: true +}) + +const logLevels = { + error: logger.error.bind(logger), + warn: logger.warn.bind(logger), + info: logger.info.bind(logger), + debug: logger.debug.bind(logger) +} + +run() + .then(() => process.exit(0)) + .catch(err => console.error(err)) + +async function run () { + const files = await getFiles() + + for (const file of files) { + if (file === 'peertube-audit.log') continue + + await readFile(file) + } +} + +function readFile (file: string) { + console.log('Opening %s.', file) + + const stream = file === '-' ? stdin : createReadStream(file) + + const rl = createInterface({ + input: stream + }) + + return new Promise(res => { + rl.on('line', line => { + try { + const log = JSON.parse(line) + if (options.tags && !containsTags(log.tags, options.tags)) { + return + } + + if (options.notTags && containsTags(log.tags, options.notTags)) { + return + } + + // Don't know why but loggerFormat does not remove splat key + Object.assign(log, { splat: undefined }) + + logLevels[log.level](log) + } catch (err) { + console.error('Cannot parse line.', inspect(line)) + throw err + } + }) + + stream.once('end', () => res()) + }) +} + +// Thanks: https://stackoverflow.com/a/37014317 +async function getNewestFile (files: string[], basePath: string) { + const sorted = await mtimeSortFilesDesc(files, basePath) + + return (sorted.length > 0) ? sorted[0].file : '' +} + +async function getFiles () { + if (options.files) return options.files + + const logFiles = await readdir(CONFIG.STORAGE.LOG_DIR) + + const filename = await getNewestFile(logFiles, CONFIG.STORAGE.LOG_DIR) + return [ join(CONFIG.STORAGE.LOG_DIR, filename) ] +} + +function toTimeFormat (time: string) { + const timestamp = Date.parse(time) + + if (isNaN(timestamp) === true) return 'Unknown date' + + const d = new Date(timestamp) + return d.toLocaleString() + `.${d.getMilliseconds()}` +} + +function containsTags (loggerTags: string[], optionsTags: string[]) { + if (!loggerTags) return false + + for (const lt of loggerTags) { + for (const ot of optionsTags) { + if (lt === ot) return true + } + } + + return false +} diff --git a/server/scripts/plugin/install.ts b/server/scripts/plugin/install.ts new file mode 100755 index 000000000..3b13120ff --- /dev/null +++ b/server/scripts/plugin/install.ts @@ -0,0 +1,41 @@ +import { program } from 'commander' +import { isAbsolute } from 'path' +import { initDatabaseModels } from '../../server/initializers/database.js' +import { PluginManager } from '../../server/lib/plugins/plugin-manager.js' + +program + .option('-n, --npm-name [npmName]', 'Plugin to install') + .option('-v, --plugin-version [pluginVersion]', 'Plugin version to install') + .option('-p, --plugin-path [pluginPath]', 'Path of the plugin you want to install') + .parse(process.argv) + +const options = program.opts() + +if (!options.npmName && !options.pluginPath) { + console.error('You need to specify a plugin name with the desired version, or a plugin path.') + process.exit(-1) +} + +if (options.pluginPath && !isAbsolute(options.pluginPath)) { + console.error('Plugin path should be absolute.') + process.exit(-1) +} + +run() + .then(() => process.exit(0)) + .catch(err => { + console.error(err) + process.exit(-1) + }) + +async function run () { + await initDatabaseModels(true) + + const toInstall = options.npmName || options.pluginPath + await PluginManager.Instance.install({ + toInstall, + version: options.pluginVersion, + fromDisk: !!options.pluginPath, + register: false + }) +} diff --git a/server/scripts/plugin/uninstall.ts b/server/scripts/plugin/uninstall.ts new file mode 100755 index 000000000..baf0422c4 --- /dev/null +++ b/server/scripts/plugin/uninstall.ts @@ -0,0 +1,29 @@ +import { program } from 'commander' +import { initDatabaseModels } from '@server/initializers/database.js' +import { PluginManager } from '@server/lib/plugins/plugin-manager.js' + +program + .option('-n, --npm-name [npmName]', 'Package name to install') + .parse(process.argv) + +const options = program.opts() + +if (!options.npmName) { + console.error('You need to specify the plugin name.') + process.exit(-1) +} + +run() + .then(() => process.exit(0)) + .catch(err => { + console.error(err) + process.exit(-1) + }) + +async function run () { + + await initDatabaseModels(true) + + const toUninstall = options.npmName + await PluginManager.Instance.uninstall({ npmName: toUninstall, unregister: false }) +} diff --git a/server/scripts/prune-storage.ts b/server/scripts/prune-storage.ts new file mode 100755 index 000000000..9309724b9 --- /dev/null +++ b/server/scripts/prune-storage.ts @@ -0,0 +1,187 @@ +import Bluebird from 'bluebird' +import { remove } from 'fs-extra/esm' +import { readdir, stat } from 'fs/promises' +import { basename, join } from 'path' +import prompt from 'prompt' +import { uniqify } from '@peertube/peertube-core-utils' +import { ThumbnailType, ThumbnailType_Type } from '@peertube/peertube-models' +import { DIRECTORIES } from '@server/initializers/constants.js' +import { VideoFileModel } from '@server/models/video/video-file.js' +import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js' +import { getUUIDFromFilename } from '../server/helpers/utils.js' +import { CONFIG } from '../server/initializers/config.js' +import { initDatabaseModels } from '../server/initializers/database.js' +import { ActorImageModel } from '../server/models/actor/actor-image.js' +import { VideoRedundancyModel } from '../server/models/redundancy/video-redundancy.js' +import { ThumbnailModel } from '../server/models/video/thumbnail.js' +import { VideoModel } from '../server/models/video/video.js' + +run() + .then(() => process.exit(0)) + .catch(err => { + console.error(err) + process.exit(-1) + }) + +async function run () { + const dirs = Object.values(CONFIG.STORAGE) + + if (uniqify(dirs).length !== dirs.length) { + console.error('Cannot prune storage because you put multiple storage keys in the same directory.') + process.exit(0) + } + + await initDatabaseModels(true) + + let toDelete: string[] = [] + + console.log('Detecting files to remove, it could take a while...') + + toDelete = toDelete.concat( + await pruneDirectory(DIRECTORIES.VIDEOS.PUBLIC, doesWebVideoFileExist()), + await pruneDirectory(DIRECTORIES.VIDEOS.PRIVATE, doesWebVideoFileExist()), + + await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, doesHLSPlaylistExist()), + await pruneDirectory(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, doesHLSPlaylistExist()), + + await pruneDirectory(CONFIG.STORAGE.TORRENTS_DIR, doesTorrentFileExist()), + + await pruneDirectory(CONFIG.STORAGE.REDUNDANCY_DIR, doesRedundancyExist), + + await pruneDirectory(CONFIG.STORAGE.PREVIEWS_DIR, doesThumbnailExist(true, ThumbnailType.PREVIEW)), + await pruneDirectory(CONFIG.STORAGE.THUMBNAILS_DIR, doesThumbnailExist(false, ThumbnailType.MINIATURE)), + + await pruneDirectory(CONFIG.STORAGE.ACTOR_IMAGES_DIR, doesActorImageExist) + ) + + const tmpFiles = await readdir(CONFIG.STORAGE.TMP_DIR) + toDelete = toDelete.concat(tmpFiles.map(t => join(CONFIG.STORAGE.TMP_DIR, t))) + + if (toDelete.length === 0) { + console.log('No files to delete.') + return + } + + console.log('Will delete %d files:\n\n%s\n\n', toDelete.length, toDelete.join('\n')) + + const res = await askConfirmation() + if (res === true) { + console.log('Processing delete...\n') + + for (const path of toDelete) { + await remove(path) + } + + console.log('Done!') + } else { + console.log('Exiting without deleting files.') + } +} + +type ExistFun = (file: string) => Promise | boolean +async function pruneDirectory (directory: string, existFun: ExistFun) { + const files = await readdir(directory) + + const toDelete: string[] = [] + await Bluebird.map(files, async file => { + const filePath = join(directory, file) + + if (await existFun(filePath) !== true) { + toDelete.push(filePath) + } + }, { concurrency: 20 }) + + return toDelete +} + +function doesWebVideoFileExist () { + return (filePath: string) => { + // Don't delete private directory + if (filePath === DIRECTORIES.VIDEOS.PRIVATE) return true + + return VideoFileModel.doesOwnedWebVideoFileExist(basename(filePath)) + } +} + +function doesHLSPlaylistExist () { + return (hlsPath: string) => { + // Don't delete private directory + if (hlsPath === DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE) return true + + return VideoStreamingPlaylistModel.doesOwnedHLSPlaylistExist(basename(hlsPath)) + } +} + +function doesTorrentFileExist () { + return (filePath: string) => VideoFileModel.doesOwnedTorrentFileExist(basename(filePath)) +} + +function doesThumbnailExist (keepOnlyOwned: boolean, type: ThumbnailType_Type) { + return async (filePath: string) => { + const thumbnail = await ThumbnailModel.loadByFilename(basename(filePath), type) + if (!thumbnail) return false + + if (keepOnlyOwned) { + const video = await VideoModel.load(thumbnail.videoId) + if (video.isOwned() === false) return false + } + + return true + } +} + +async function doesActorImageExist (filePath: string) { + const image = await ActorImageModel.loadByName(basename(filePath)) + + return !!image +} + +async function doesRedundancyExist (filePath: string) { + const isPlaylist = (await stat(filePath)).isDirectory() + + if (isPlaylist) { + // Don't delete HLS redundancy directory + if (filePath === DIRECTORIES.HLS_REDUNDANCY) return true + + const uuid = getUUIDFromFilename(filePath) + const video = await VideoModel.loadWithFiles(uuid) + if (!video) return false + + const p = video.getHLSPlaylist() + if (!p) return false + + const redundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(p.id) + return !!redundancy + } + + const file = await VideoFileModel.loadByFilename(basename(filePath)) + if (!file) return false + + const redundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) + return !!redundancy +} + +async function askConfirmation () { + return new Promise((res, rej) => { + prompt.start() + + const schema = { + properties: { + confirm: { + type: 'string', + description: 'These following unused files can be deleted, but please check your backups first (bugs happen).' + + ' Notice PeerTube must have been stopped when your ran this script.' + + ' Can we delete these files?', + default: 'n', + required: true + } + } + } + + prompt.get(schema, function (err, result) { + if (err) return rej(err) + + return res(result.confirm?.match(/y/) !== null) + }) + }) +} diff --git a/server/scripts/regenerate-thumbnails.ts b/server/scripts/regenerate-thumbnails.ts new file mode 100644 index 000000000..d4346ce40 --- /dev/null +++ b/server/scripts/regenerate-thumbnails.ts @@ -0,0 +1,64 @@ +import Bluebird from 'bluebird' +import { program } from 'commander' +import { pathExists, remove } from 'fs-extra/esm' +import { generateImageFilename, processImage } from '@server/helpers/image-utils.js' +import { THUMBNAILS_SIZE } from '@server/initializers/constants.js' +import { initDatabaseModels } from '@server/initializers/database.js' +import { VideoModel } from '@server/models/video/video.js' + +program + .description('Regenerate local thumbnails using preview files') + .parse(process.argv) + +run() + .then(() => process.exit(0)) + .catch(err => console.error(err)) + +async function run () { + await initDatabaseModels(true) + + const ids = await VideoModel.listLocalIds() + + await Bluebird.map(ids, id => { + return processVideo(id) + .catch(err => console.error('Cannot process video %d.', id, err)) + }, { concurrency: 20 }) +} + +async function processVideo (id: number) { + const video = await VideoModel.loadWithFiles(id) + + console.log('Processing video %s.', video.name) + + const thumbnail = video.getMiniature() + const preview = video.getPreview() + + const previewPath = preview.getPath() + + if (!await pathExists(previewPath)) { + throw new Error(`Preview ${previewPath} does not exist on disk`) + } + + const size = { + width: THUMBNAILS_SIZE.width, + height: THUMBNAILS_SIZE.height + } + + const oldPath = thumbnail.getPath() + + // Update thumbnail + thumbnail.filename = generateImageFilename() + thumbnail.width = size.width + thumbnail.height = size.height + + const thumbnailPath = thumbnail.getPath() + await processImage({ path: previewPath, destination: thumbnailPath, newSize: size, keepOriginal: true }) + + // Save new attributes + await thumbnail.save() + + // Remove old thumbnail + await remove(oldPath) + + // Don't federate, remote instances will refresh the thumbnails after a while +} diff --git a/server/scripts/reset-password.ts b/server/scripts/reset-password.ts new file mode 100755 index 000000000..96e301ba9 --- /dev/null +++ b/server/scripts/reset-password.ts @@ -0,0 +1,58 @@ +import { program } from 'commander' +import readline from 'readline' +import { Writable } from 'stream' +import { isUserPasswordValid } from '@server/helpers/custom-validators/users.js' +import { initDatabaseModels } from '@server/initializers/database.js' +import { UserModel } from '@server/models/user/user.js' + +program + .option('-u, --user [user]', 'User') + .parse(process.argv) + +const options = program.opts() + +if (options.user === undefined) { + console.error('All parameters are mandatory.') + process.exit(-1) +} + +initDatabaseModels(true) + .then(() => { + return UserModel.loadByUsername(options.user) + }) + .then(user => { + if (!user) { + console.error('Unknown user.') + process.exit(-1) + } + + const mutableStdout = new Writable({ + write: function (_chunk, _encoding, callback) { + callback() + } + }) + const rl = readline.createInterface({ + input: process.stdin, + output: mutableStdout, + terminal: true + }) + + console.log('New password?') + rl.on('line', function (password) { + if (!isUserPasswordValid(password)) { + console.error('New password is invalid.') + process.exit(-1) + } + + user.password = password + + user.save() + .then(() => console.log('User password updated.')) + .catch(err => console.error(err)) + .finally(() => process.exit(0)) + }) + }) + .catch(err => { + console.error(err) + process.exit(-1) + }) diff --git a/server/scripts/update-host.ts b/server/scripts/update-host.ts new file mode 100755 index 000000000..52ac4947a --- /dev/null +++ b/server/scripts/update-host.ts @@ -0,0 +1,140 @@ +import { updateTorrentMetadata } from '@server/helpers/webtorrent.js' +import { getServerActor } from '@server/models/application/application.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { initDatabaseModels } from '@server/initializers/database.js' +import { + getLocalAccountActivityPubUrl, + getLocalVideoActivityPubUrl, + getLocalVideoAnnounceActivityPubUrl, + getLocalVideoChannelActivityPubUrl, + getLocalVideoCommentActivityPubUrl +} from '@server/lib/activitypub/url.js' +import { AccountModel } from '@server/models/account/account.js' +import { ActorFollowModel } from '@server/models/actor/actor-follow.js' +import { ActorModel } from '@server/models/actor/actor.js' +import { VideoChannelModel } from '@server/models/video/video-channel.js' +import { VideoCommentModel } from '@server/models/video/video-comment.js' +import { VideoShareModel } from '@server/models/video/video-share.js' +import { VideoModel } from '@server/models/video/video.js' + +run() + .then(() => process.exit(0)) + .catch(err => { + console.error(err) + process.exit(-1) + }) + +async function run () { + await initDatabaseModels(true) + + const serverAccount = await getServerActor() + + { + const res = await ActorFollowModel.listAcceptedFollowingUrlsForApi([ serverAccount.id ], undefined) + const hasFollowing = res.total > 0 + + if (hasFollowing === true) { + throw new Error('Cannot update host because you follow other servers!') + } + } + + console.log('Updating actors.') + + const actors: ActorModel[] = await ActorModel.unscoped().findAll({ + include: [ + { + model: VideoChannelModel.unscoped(), + required: false + }, + { + model: AccountModel.unscoped(), + required: false + } + ] + }) + for (const actor of actors) { + if (actor.isOwned() === false) continue + + console.log('Updating actor ' + actor.url) + + const newUrl = actor.Account + ? getLocalAccountActivityPubUrl(actor.preferredUsername) + : getLocalVideoChannelActivityPubUrl(actor.preferredUsername) + + actor.url = newUrl + actor.inboxUrl = newUrl + '/inbox' + actor.outboxUrl = newUrl + '/outbox' + actor.sharedInboxUrl = WEBSERVER.URL + '/inbox' + actor.followersUrl = newUrl + '/followers' + actor.followingUrl = newUrl + '/following' + + await actor.save() + } + + console.log('Updating video shares.') + + const videoShares: VideoShareModel[] = await VideoShareModel.findAll({ + include: [ VideoModel.unscoped(), ActorModel.unscoped() ] + }) + for (const videoShare of videoShares) { + if (videoShare.Video.isOwned() === false) continue + + console.log('Updating video share ' + videoShare.url) + + videoShare.url = getLocalVideoAnnounceActivityPubUrl(videoShare.Actor, videoShare.Video) + await videoShare.save() + } + + console.log('Updating video comments.') + const videoComments: VideoCommentModel[] = await VideoCommentModel.findAll({ + include: [ + { + model: VideoModel.unscoped() + }, + { + model: AccountModel.unscoped(), + include: [ + { + model: ActorModel.unscoped() + } + ] + } + ] + }) + for (const comment of videoComments) { + if (comment.isOwned() === false) continue + + console.log('Updating comment ' + comment.url) + + comment.url = getLocalVideoCommentActivityPubUrl(comment.Video, comment) + await comment.save() + } + + console.log('Updating video and torrent files.') + + const ids = await VideoModel.listLocalIds() + for (const id of ids) { + const video = await VideoModel.loadFull(id) + + console.log('Updating video ' + video.uuid) + + video.url = getLocalVideoActivityPubUrl(video) + await video.save() + + for (const file of video.VideoFiles) { + console.log('Updating torrent file %s of video %s.', file.resolution, video.uuid) + await updateTorrentMetadata(video, file) + + await file.save() + } + + const playlist = video.getHLSPlaylist() + for (const file of (playlist?.VideoFiles || [])) { + console.log('Updating fragmented torrent file %s of video %s.', file.resolution, video.uuid) + + await updateTorrentMetadata(playlist, file) + + await file.save() + } + } +} diff --git a/scripts/upgrade.sh b/server/scripts/upgrade.sh similarity index 100% rename from scripts/upgrade.sh rename to server/scripts/upgrade.sh diff --git a/server/server.ts b/server/server.ts new file mode 100644 index 000000000..7f9c68ad9 --- /dev/null +++ b/server/server.ts @@ -0,0 +1,376 @@ +import { registerOpentelemetryTracing } from '@server/lib/opentelemetry/tracing.js' +await registerOpentelemetryTracing() + +process.title = 'peertube' + +// ----------- Core checker ----------- +import { checkMissedConfig, checkFFmpeg, checkNodeVersion } from './server/initializers/checker-before-init.js' + +// Do not use barrels because we don't want to load all modules here (we need to initialize database first) +import { CONFIG } from './server/initializers/config.js' +import { API_VERSION, WEBSERVER, loadLanguages } from './server/initializers/constants.js' +import { logger } from './server/helpers/logger.js' + +const missed = checkMissedConfig() +if (missed.length !== 0) { + logger.error('Your configuration files miss keys: ' + missed) + process.exit(-1) +} + +checkFFmpeg(CONFIG) + .catch(err => { + logger.error('Error in ffmpeg check.', { err }) + process.exit(-1) + }) + +try { + checkNodeVersion() +} catch (err) { + logger.error('Error in NodeJS check.', { err }) + process.exit(-1) +} + +import { checkConfig, checkActivityPubUrls, checkFFmpegVersion } from './server/initializers/checker-after-init.js' + +try { + checkConfig() +} catch (err) { + logger.error('Config error.', { err }) + process.exit(-1) +} + +// ----------- Database ----------- + +// Initialize database and models +import { initDatabaseModels, checkDatabaseConnectionOrDie } from './server/initializers/database.js' +checkDatabaseConnectionOrDie() + +import { migrate } from './server/initializers/migrator.js' +migrate() + .then(() => initDatabaseModels(false)) + .then(() => startApplication()) + .catch(err => { + logger.error('Cannot start application.', { err }) + process.exit(-1) + }) + +// ----------- Initialize ----------- +loadLanguages() + .catch(err => logger.error('Cannot load languages', { err })) + +// Express configuration +import express from 'express' +import morgan, { token } from 'morgan' +import cors from 'cors' +import cookieParser from 'cookie-parser' +import { frameguard } from 'helmet' +import { parse } from 'useragent' +import anonymize from 'ip-anonymize' +import { program as cli } from 'commander' + +const app = express().disable('x-powered-by') + +// Trust our proxy (IP forwarding...) +app.set('trust proxy', CONFIG.TRUST_PROXY) + +app.use((_req, res, next) => { + // OpenTelemetry + res.locals.requestStart = Date.now() + + if (CONFIG.SECURITY.POWERED_BY_HEADER.ENABLED === true) { + res.setHeader('x-powered-by', 'PeerTube') + } + + return next() +}) + +// Security middleware +import { baseCSP } from './server/middlewares/csp.js' + +if (CONFIG.CSP.ENABLED) { + app.use(baseCSP) +} + +if (CONFIG.SECURITY.FRAMEGUARD.ENABLED) { + app.use(frameguard({ + action: 'deny' // we only allow it for /videos/embed, see server/controllers/client.ts + })) +} + +// ----------- PeerTube modules ----------- +import { installApplication } from './server/initializers/installer.js' +import { Emailer } from './server/lib/emailer.js' +import { JobQueue } from './server/lib/job-queue/index.js' +import { + activityPubRouter, + apiRouter, + miscRouter, + clientsRouter, + feedsRouter, + staticRouter, + wellKnownRouter, + lazyStaticRouter, + servicesRouter, + objectStorageProxyRouter, + pluginsRouter, + trackerRouter, + createWebsocketTrackerServer, + sitemapRouter, + downloadRouter +} from './server/controllers/index.js' +import { advertiseDoNotTrack } from './server/middlewares/dnt.js' +import { apiFailMiddleware } from './server/middlewares/error.js' +import { Redis } from './server/lib/redis.js' +import { ActorFollowScheduler } from './server/lib/schedulers/actor-follow-scheduler.js' +import { RemoveOldViewsScheduler } from './server/lib/schedulers/remove-old-views-scheduler.js' +import { UpdateVideosScheduler } from './server/lib/schedulers/update-videos-scheduler.js' +import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-update-scheduler.js' +import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler.js' +import { RemoveOldHistoryScheduler } from './server/lib/schedulers/remove-old-history-scheduler.js' +import { AutoFollowIndexInstances } from './server/lib/schedulers/auto-follow-index-instances.js' +import { RemoveDanglingResumableUploadsScheduler } from './server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.js' +import { VideoViewsBufferScheduler } from './server/lib/schedulers/video-views-buffer-scheduler.js' +import { GeoIPUpdateScheduler } from './server/lib/schedulers/geo-ip-update-scheduler.js' +import { RunnerJobWatchDogScheduler } from './server/lib/schedulers/runner-job-watch-dog-scheduler.js' +import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto.js' +import { PeerTubeSocket } from './server/lib/peertube-socket.js' +import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls.js' +import { PluginsCheckScheduler } from './server/lib/schedulers/plugins-check-scheduler.js' +import { PeerTubeVersionCheckScheduler } from './server/lib/schedulers/peertube-version-check-scheduler.js' +import { Hooks } from './server/lib/plugins/hooks.js' +import { PluginManager } from './server/lib/plugins/plugin-manager.js' +import { LiveManager } from './server/lib/live/index.js' +import { HttpStatusCode } from '@peertube/peertube-models' +import { ServerConfigManager } from '@server/lib/server-config-manager.js' +import { VideoViewsManager } from '@server/lib/views/video-views-manager.js' +import { isTestOrDevInstance } from '@peertube/peertube-node-utils' +import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics.js' +import { ApplicationModel } from '@server/models/application/application.js' +import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler.js' + +// ----------- Command line ----------- + +cli + .option('--no-client', 'Start PeerTube without client interface') + .option('--no-plugins', 'Start PeerTube without plugins/themes enabled') + .option('--benchmark-startup', 'Automatically stop server when initialized') + .parse(process.argv) + +// ----------- App ----------- + +// Enable CORS for develop +if (isTestOrDevInstance()) { + app.use(cors({ + origin: '*', + exposedHeaders: 'Retry-After', + credentials: true + })) +} + +// For the logger +token('remote-addr', (req: express.Request) => { + if (CONFIG.LOG.ANONYMIZE_IP === true || req.get('DNT') === '1') { + return anonymize(req.ip, 16, 16) + } + + return req.ip +}) +token('user-agent', (req: express.Request) => { + if (req.get('DNT') === '1') { + return parse(req.get('user-agent')).family + } + + return req.get('user-agent') +}) +app.use(morgan('combined', { + stream: { + write: (str: string) => logger.info(str.trim(), { tags: [ 'http' ] }) + }, + skip: req => CONFIG.LOG.LOG_PING_REQUESTS === false && req.originalUrl === '/api/v1/ping' +})) + +// Add .fail() helper to response +app.use(apiFailMiddleware) + +// For body requests +app.use(express.urlencoded({ extended: false })) +app.use(express.json({ + type: [ 'application/json', 'application/*+json' ], + limit: '500kb', + verify: (req: express.Request, res: express.Response, buf: Buffer) => { + const valid = isHTTPSignatureDigestValid(buf, req) + + if (valid !== true) { + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Invalid digest' + }) + } + } +})) + +// Cookies +app.use(cookieParser()) + +// W3C DNT Tracking Status +app.use(advertiseDoNotTrack) + +// ----------- Open Telemetry ----------- + +OpenTelemetryMetrics.Instance.init(app) + +// ----------- Views, routes and static files ----------- + +app.use('/api/' + API_VERSION, apiRouter) + +// Services (oembed...) +app.use('/services', servicesRouter) + +// Plugins & themes +app.use('/', pluginsRouter) + +app.use('/', activityPubRouter) +app.use('/', feedsRouter) +app.use('/', trackerRouter) +app.use('/', sitemapRouter) + +// Static files +app.use('/', staticRouter) +app.use('/', wellKnownRouter) +app.use('/', miscRouter) +app.use('/', downloadRouter) +app.use('/', lazyStaticRouter) +app.use('/', objectStorageProxyRouter) + +// Client files, last valid routes! +const cliOptions = cli.opts<{ client: boolean, plugins: boolean }>() +if (cliOptions.client) app.use('/', clientsRouter) + +// ----------- Errors ----------- + +// Catch unmatched routes +app.use((_req, res: express.Response) => { + res.status(HttpStatusCode.NOT_FOUND_404).end() +}) + +// Catch thrown errors +app.use((err, _req, res: express.Response, _next) => { + // Format error to be logged + let error = 'Unknown error.' + if (err) { + error = err.stack || err.message || err + } + + // Handling Sequelize error traces + const sql = err?.parent ? err.parent.sql : undefined + + // Help us to debug SequelizeConnectionAcquireTimeoutError errors + const activeRequests = err?.name === 'SequelizeConnectionAcquireTimeoutError' && typeof (process as any)._getActiveRequests !== 'function' + ? (process as any)._getActiveRequests() + : undefined + + logger.error('Error in controller.', { err: error, sql, activeRequests }) + + return res.fail({ + status: err.status || HttpStatusCode.INTERNAL_SERVER_ERROR_500, + message: err.message, + type: err.name + }) +}) + +const { server, trackerServer } = createWebsocketTrackerServer(app) + +// ----------- Run ----------- + +async function startApplication () { + const port = CONFIG.LISTEN.PORT + const hostname = CONFIG.LISTEN.HOSTNAME + + await installApplication() + + // Check activity pub urls are valid + checkActivityPubUrls() + .catch(err => { + logger.error('Error in ActivityPub URLs checker.', { err }) + process.exit(-1) + }) + + checkFFmpegVersion() + .catch(err => logger.error('Cannot check ffmpeg version', { err })) + + Redis.Instance.init() + Emailer.Instance.init() + + await Promise.all([ + Emailer.Instance.checkConnection(), + JobQueue.Instance.init(), + ServerConfigManager.Instance.init() + ]) + + // Enable Schedulers + ActorFollowScheduler.Instance.enable() + UpdateVideosScheduler.Instance.enable() + YoutubeDlUpdateScheduler.Instance.enable() + VideosRedundancyScheduler.Instance.enable() + RemoveOldHistoryScheduler.Instance.enable() + RemoveOldViewsScheduler.Instance.enable() + PluginsCheckScheduler.Instance.enable() + PeerTubeVersionCheckScheduler.Instance.enable() + AutoFollowIndexInstances.Instance.enable() + RemoveDanglingResumableUploadsScheduler.Instance.enable() + VideoChannelSyncLatestScheduler.Instance.enable() + VideoViewsBufferScheduler.Instance.enable() + GeoIPUpdateScheduler.Instance.enable() + RunnerJobWatchDogScheduler.Instance.enable() + + OpenTelemetryMetrics.Instance.registerMetrics({ trackerServer }) + + PluginManager.Instance.init(server) + // Before PeerTubeSocket init + PluginManager.Instance.registerWebSocketRouter() + + PeerTubeSocket.Instance.init(server) + VideoViewsManager.Instance.init() + + updateStreamingPlaylistsInfohashesIfNeeded() + .catch(err => logger.error('Cannot update streaming playlist infohashes.', { err })) + + LiveManager.Instance.init() + if (CONFIG.LIVE.ENABLED) await LiveManager.Instance.run() + + // Make server listening + server.listen(port, hostname, async () => { + if (cliOptions.plugins) { + try { + await PluginManager.Instance.rebuildNativePluginsIfNeeded() + + await PluginManager.Instance.registerPluginsAndThemes() + } catch (err) { + logger.error('Cannot register plugins and themes.', { err }) + } + } + + ApplicationModel.updateNodeVersions() + .catch(err => logger.error('Cannot update node versions.', { err })) + + JobQueue.Instance.start() + .catch(err => { + logger.error('Cannot start job queue.', { err }) + process.exit(-1) + }) + + logger.info('HTTP server listening on %s:%d', hostname, port) + logger.info('Web server: %s', WEBSERVER.URL) + + Hooks.runAction('action:application.listening') + + if (cliOptions['benchmarkStartup']) process.exit(0) + }) + + process.on('exit', () => { + JobQueue.Instance.terminate() + .catch(err => logger.error('Cannot terminate job queue.', { err })) + }) + + process.on('SIGINT', () => process.exit(0)) +} diff --git a/server/assets/default-audio-background.jpg b/server/server/assets/default-audio-background.jpg similarity index 100% rename from server/assets/default-audio-background.jpg rename to server/server/assets/default-audio-background.jpg diff --git a/server/assets/default-live-background.jpg b/server/server/assets/default-live-background.jpg similarity index 100% rename from server/assets/default-live-background.jpg rename to server/server/assets/default-live-background.jpg diff --git a/server/server/controllers/activitypub/client.ts b/server/server/controllers/activitypub/client.ts new file mode 100644 index 000000000..5d5e43bf5 --- /dev/null +++ b/server/server/controllers/activitypub/client.ts @@ -0,0 +1,485 @@ +import cors from 'cors' +import express from 'express' +import { VideoCommentObject, VideoPlaylistPrivacy, VideoPrivacy, VideoRateType } from '@peertube/peertube-models' +import { activityPubCollectionPagination } from '@server/lib/activitypub/collection.js' +import { getContextFilter } from '@server/lib/activitypub/context.js' +import { getServerActor } from '@server/models/application/application.js' +import { MAccountId, MActorId, MChannelId, MVideoId } from '@server/types/models/index.js' +import { activityPubContextify } from '../../helpers/activity-pub-utils.js' +import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../../initializers/constants.js' +import { audiencify, getAudience } from '../../lib/activitypub/audience.js' +import { buildAnnounceWithVideoAudience, buildLikeActivity } from '../../lib/activitypub/send/index.js' +import { buildCreateActivity } from '../../lib/activitypub/send/send-create.js' +import { buildDislikeActivity } from '../../lib/activitypub/send/send-dislike.js' +import { + getLocalVideoCommentsActivityPubUrl, + getLocalVideoDislikesActivityPubUrl, + getLocalVideoLikesActivityPubUrl, + getLocalVideoSharesActivityPubUrl +} from '../../lib/activitypub/url.js' +import { cacheRoute } from '../../middlewares/cache/cache.js' +import { + activityPubRateLimiter, + asyncMiddleware, + ensureIsLocalChannel, + executeIfActivityPub, + localAccountValidator, + videoChannelsNameWithHostValidator, + videosCustomGetValidator, + videosShareValidator +} from '../../middlewares/index.js' +import { + getAccountVideoRateValidatorFactory, + getVideoLocalViewerValidator, + videoCommentGetValidator +} from '../../middlewares/validators/index.js' +import { videoFileRedundancyGetValidator, videoPlaylistRedundancyGetValidator } from '../../middlewares/validators/redundancy.js' +import { videoPlaylistElementAPGetValidator, videoPlaylistsGetValidator } from '../../middlewares/validators/videos/video-playlists.js' +import { AccountVideoRateModel } from '../../models/account/account-video-rate.js' +import { AccountModel } from '../../models/account/account.js' +import { ActorFollowModel } from '../../models/actor/actor-follow.js' +import { VideoCommentModel } from '../../models/video/video-comment.js' +import { VideoPlaylistModel } from '../../models/video/video-playlist.js' +import { VideoShareModel } from '../../models/video/video-share.js' +import { activityPubResponse } from './utils.js' + +const activityPubClientRouter = express.Router() +activityPubClientRouter.use(cors()) + +// Intercept ActivityPub client requests + +activityPubClientRouter.get( + [ '/accounts?/:name', '/accounts?/:name/video-channels', '/a/:name', '/a/:name/video-channels' ], + executeIfActivityPub, + activityPubRateLimiter, + asyncMiddleware(localAccountValidator), + asyncMiddleware(accountController) +) +activityPubClientRouter.get('/accounts?/:name/followers', + executeIfActivityPub, + activityPubRateLimiter, + asyncMiddleware(localAccountValidator), + asyncMiddleware(accountFollowersController) +) +activityPubClientRouter.get('/accounts?/:name/following', + executeIfActivityPub, + activityPubRateLimiter, + asyncMiddleware(localAccountValidator), + asyncMiddleware(accountFollowingController) +) +activityPubClientRouter.get('/accounts?/:name/playlists', + executeIfActivityPub, + activityPubRateLimiter, + asyncMiddleware(localAccountValidator), + asyncMiddleware(accountPlaylistsController) +) +activityPubClientRouter.get('/accounts?/:name/likes/:videoId', + executeIfActivityPub, + activityPubRateLimiter, + cacheRoute(ROUTE_CACHE_LIFETIME.ACTIVITY_PUB.VIDEOS), + asyncMiddleware(getAccountVideoRateValidatorFactory('like')), + asyncMiddleware(getAccountVideoRateFactory('like')) +) +activityPubClientRouter.get('/accounts?/:name/dislikes/:videoId', + executeIfActivityPub, + activityPubRateLimiter, + cacheRoute(ROUTE_CACHE_LIFETIME.ACTIVITY_PUB.VIDEOS), + asyncMiddleware(getAccountVideoRateValidatorFactory('dislike')), + asyncMiddleware(getAccountVideoRateFactory('dislike')) +) + +activityPubClientRouter.get( + [ '/videos/watch/:id', '/w/:id' ], + executeIfActivityPub, + activityPubRateLimiter, + cacheRoute(ROUTE_CACHE_LIFETIME.ACTIVITY_PUB.VIDEOS), + asyncMiddleware(videosCustomGetValidator('all')), + asyncMiddleware(videoController) +) +activityPubClientRouter.get('/videos/watch/:id/activity', + executeIfActivityPub, + activityPubRateLimiter, + asyncMiddleware(videosCustomGetValidator('all')), + asyncMiddleware(videoController) +) +activityPubClientRouter.get('/videos/watch/:id/announces', + executeIfActivityPub, + activityPubRateLimiter, + asyncMiddleware(videosCustomGetValidator('only-immutable-attributes')), + asyncMiddleware(videoAnnouncesController) +) +activityPubClientRouter.get('/videos/watch/:id/announces/:actorId', + executeIfActivityPub, + activityPubRateLimiter, + asyncMiddleware(videosShareValidator), + asyncMiddleware(videoAnnounceController) +) +activityPubClientRouter.get('/videos/watch/:id/likes', + executeIfActivityPub, + activityPubRateLimiter, + asyncMiddleware(videosCustomGetValidator('only-immutable-attributes')), + asyncMiddleware(videoLikesController) +) +activityPubClientRouter.get('/videos/watch/:id/dislikes', + executeIfActivityPub, + activityPubRateLimiter, + asyncMiddleware(videosCustomGetValidator('only-immutable-attributes')), + asyncMiddleware(videoDislikesController) +) +activityPubClientRouter.get('/videos/watch/:id/comments', + executeIfActivityPub, + activityPubRateLimiter, + asyncMiddleware(videosCustomGetValidator('only-immutable-attributes')), + asyncMiddleware(videoCommentsController) +) +activityPubClientRouter.get('/videos/watch/:videoId/comments/:commentId', + executeIfActivityPub, + activityPubRateLimiter, + asyncMiddleware(videoCommentGetValidator), + asyncMiddleware(videoCommentController) +) +activityPubClientRouter.get('/videos/watch/:videoId/comments/:commentId/activity', + executeIfActivityPub, + activityPubRateLimiter, + asyncMiddleware(videoCommentGetValidator), + asyncMiddleware(videoCommentController) +) + +activityPubClientRouter.get( + [ '/video-channels/:nameWithHost', '/video-channels/:nameWithHost/videos', '/c/:nameWithHost', '/c/:nameWithHost/videos' ], + executeIfActivityPub, + activityPubRateLimiter, + asyncMiddleware(videoChannelsNameWithHostValidator), + ensureIsLocalChannel, + asyncMiddleware(videoChannelController) +) +activityPubClientRouter.get('/video-channels/:nameWithHost/followers', + executeIfActivityPub, + activityPubRateLimiter, + asyncMiddleware(videoChannelsNameWithHostValidator), + ensureIsLocalChannel, + asyncMiddleware(videoChannelFollowersController) +) +activityPubClientRouter.get('/video-channels/:nameWithHost/following', + executeIfActivityPub, + activityPubRateLimiter, + asyncMiddleware(videoChannelsNameWithHostValidator), + ensureIsLocalChannel, + asyncMiddleware(videoChannelFollowingController) +) +activityPubClientRouter.get('/video-channels/:nameWithHost/playlists', + executeIfActivityPub, + activityPubRateLimiter, + asyncMiddleware(videoChannelsNameWithHostValidator), + ensureIsLocalChannel, + asyncMiddleware(videoChannelPlaylistsController) +) + +activityPubClientRouter.get('/redundancy/videos/:videoId/:resolution([0-9]+)(-:fps([0-9]+))?', + executeIfActivityPub, + activityPubRateLimiter, + asyncMiddleware(videoFileRedundancyGetValidator), + asyncMiddleware(videoRedundancyController) +) +activityPubClientRouter.get('/redundancy/streaming-playlists/:streamingPlaylistType/:videoId', + executeIfActivityPub, + activityPubRateLimiter, + asyncMiddleware(videoPlaylistRedundancyGetValidator), + asyncMiddleware(videoRedundancyController) +) + +activityPubClientRouter.get( + [ '/video-playlists/:playlistId', '/videos/watch/playlist/:playlistId', '/w/p/:playlistId' ], + executeIfActivityPub, + activityPubRateLimiter, + asyncMiddleware(videoPlaylistsGetValidator('all')), + asyncMiddleware(videoPlaylistController) +) +activityPubClientRouter.get('/video-playlists/:playlistId/videos/:playlistElementId', + executeIfActivityPub, + activityPubRateLimiter, + asyncMiddleware(videoPlaylistElementAPGetValidator), + asyncMiddleware(videoPlaylistElementController) +) + +activityPubClientRouter.get('/videos/local-viewer/:localViewerId', + executeIfActivityPub, + activityPubRateLimiter, + asyncMiddleware(getVideoLocalViewerValidator), + asyncMiddleware(getVideoLocalViewerController) +) + +// --------------------------------------------------------------------------- + +export { + activityPubClientRouter +} + +// --------------------------------------------------------------------------- + +async function accountController (req: express.Request, res: express.Response) { + const account = res.locals.account + + return activityPubResponse(activityPubContextify(await account.toActivityPubObject(), 'Actor', getContextFilter()), res) +} + +async function accountFollowersController (req: express.Request, res: express.Response) { + const account = res.locals.account + const activityPubResult = await actorFollowers(req, account.Actor) + + return activityPubResponse(activityPubContextify(activityPubResult, 'Collection', getContextFilter()), res) +} + +async function accountFollowingController (req: express.Request, res: express.Response) { + const account = res.locals.account + const activityPubResult = await actorFollowing(req, account.Actor) + + return activityPubResponse(activityPubContextify(activityPubResult, 'Collection', getContextFilter()), res) +} + +async function accountPlaylistsController (req: express.Request, res: express.Response) { + const account = res.locals.account + const activityPubResult = await actorPlaylists(req, { account }) + + return activityPubResponse(activityPubContextify(activityPubResult, 'Collection', getContextFilter()), res) +} + +async function videoChannelPlaylistsController (req: express.Request, res: express.Response) { + const channel = res.locals.videoChannel + const activityPubResult = await actorPlaylists(req, { channel }) + + return activityPubResponse(activityPubContextify(activityPubResult, 'Collection', getContextFilter()), res) +} + +function getAccountVideoRateFactory (rateType: VideoRateType) { + return (req: express.Request, res: express.Response) => { + const accountVideoRate = res.locals.accountVideoRate + + const byActor = accountVideoRate.Account.Actor + const APObject = rateType === 'like' + ? buildLikeActivity(accountVideoRate.url, byActor, accountVideoRate.Video) + : buildDislikeActivity(accountVideoRate.url, byActor, accountVideoRate.Video) + + return activityPubResponse(activityPubContextify(APObject, 'Rate', getContextFilter()), res) + } +} + +async function videoController (req: express.Request, res: express.Response) { + const video = res.locals.videoAll + + if (redirectIfNotOwned(video.url, res)) return + + // We need captions to render AP object + const videoAP = await video.lightAPToFullAP(undefined) + + const audience = getAudience(videoAP.VideoChannel.Account.Actor, videoAP.privacy === VideoPrivacy.PUBLIC) + const videoObject = audiencify(await videoAP.toActivityPubObject(), audience) + + if (req.path.endsWith('/activity')) { + const data = buildCreateActivity(videoAP.url, video.VideoChannel.Account.Actor, videoObject, audience) + return activityPubResponse(activityPubContextify(data, 'Video', getContextFilter()), res) + } + + return activityPubResponse(activityPubContextify(videoObject, 'Video', getContextFilter()), res) +} + +async function videoAnnounceController (req: express.Request, res: express.Response) { + const share = res.locals.videoShare + + if (redirectIfNotOwned(share.url, res)) return + + const { activity } = await buildAnnounceWithVideoAudience(share.Actor, share, res.locals.videoAll, undefined) + + return activityPubResponse(activityPubContextify(activity, 'Announce', getContextFilter()), res) +} + +async function videoAnnouncesController (req: express.Request, res: express.Response) { + const video = res.locals.onlyImmutableVideo + + if (redirectIfNotOwned(video.url, res)) return + + const handler = async (start: number, count: number) => { + const result = await VideoShareModel.listAndCountByVideoId(video.id, start, count) + return { + total: result.total, + data: result.data.map(r => r.url) + } + } + const json = await activityPubCollectionPagination(getLocalVideoSharesActivityPubUrl(video), handler, req.query.page) + + return activityPubResponse(activityPubContextify(json, 'Collection', getContextFilter()), res) +} + +async function videoLikesController (req: express.Request, res: express.Response) { + const video = res.locals.onlyImmutableVideo + + if (redirectIfNotOwned(video.url, res)) return + + const json = await videoRates(req, 'like', video, getLocalVideoLikesActivityPubUrl(video)) + + return activityPubResponse(activityPubContextify(json, 'Collection', getContextFilter()), res) +} + +async function videoDislikesController (req: express.Request, res: express.Response) { + const video = res.locals.onlyImmutableVideo + + if (redirectIfNotOwned(video.url, res)) return + + const json = await videoRates(req, 'dislike', video, getLocalVideoDislikesActivityPubUrl(video)) + + return activityPubResponse(activityPubContextify(json, 'Collection', getContextFilter()), res) +} + +async function videoCommentsController (req: express.Request, res: express.Response) { + const video = res.locals.onlyImmutableVideo + + if (redirectIfNotOwned(video.url, res)) return + + const handler = async (start: number, count: number) => { + const result = await VideoCommentModel.listAndCountByVideoForAP({ video, start, count }) + + return { + total: result.total, + data: result.data.map(r => r.url) + } + } + const json = await activityPubCollectionPagination(getLocalVideoCommentsActivityPubUrl(video), handler, req.query.page) + + return activityPubResponse(activityPubContextify(json, 'Collection', getContextFilter()), res) +} + +async function videoChannelController (req: express.Request, res: express.Response) { + const videoChannel = res.locals.videoChannel + + return activityPubResponse(activityPubContextify(await videoChannel.toActivityPubObject(), 'Actor', getContextFilter()), res) +} + +async function videoChannelFollowersController (req: express.Request, res: express.Response) { + const videoChannel = res.locals.videoChannel + const activityPubResult = await actorFollowers(req, videoChannel.Actor) + + return activityPubResponse(activityPubContextify(activityPubResult, 'Collection', getContextFilter()), res) +} + +async function videoChannelFollowingController (req: express.Request, res: express.Response) { + const videoChannel = res.locals.videoChannel + const activityPubResult = await actorFollowing(req, videoChannel.Actor) + + return activityPubResponse(activityPubContextify(activityPubResult, 'Collection', getContextFilter()), res) +} + +async function videoCommentController (req: express.Request, res: express.Response) { + const videoComment = res.locals.videoCommentFull + + if (redirectIfNotOwned(videoComment.url, res)) return + + const threadParentComments = await VideoCommentModel.listThreadParentComments(videoComment, undefined) + const isPublic = true // Comments are always public + let videoCommentObject = videoComment.toActivityPubObject(threadParentComments) + + if (videoComment.Account) { + const audience = getAudience(videoComment.Account.Actor, isPublic) + videoCommentObject = audiencify(videoCommentObject, audience) + + if (req.path.endsWith('/activity')) { + const data = buildCreateActivity(videoComment.url, videoComment.Account.Actor, videoCommentObject as VideoCommentObject, audience) + return activityPubResponse(activityPubContextify(data, 'Comment', getContextFilter()), res) + } + } + + return activityPubResponse(activityPubContextify(videoCommentObject, 'Comment', getContextFilter()), res) +} + +async function videoRedundancyController (req: express.Request, res: express.Response) { + const videoRedundancy = res.locals.videoRedundancy + + if (redirectIfNotOwned(videoRedundancy.url, res)) return + + const serverActor = await getServerActor() + + const audience = getAudience(serverActor) + const object = audiencify(videoRedundancy.toActivityPubObject(), audience) + + if (req.path.endsWith('/activity')) { + const data = buildCreateActivity(videoRedundancy.url, serverActor, object, audience) + return activityPubResponse(activityPubContextify(data, 'CacheFile', getContextFilter()), res) + } + + return activityPubResponse(activityPubContextify(object, 'CacheFile', getContextFilter()), res) +} + +async function videoPlaylistController (req: express.Request, res: express.Response) { + const playlist = res.locals.videoPlaylistFull + + if (redirectIfNotOwned(playlist.url, res)) return + + // We need more attributes + playlist.OwnerAccount = await AccountModel.load(playlist.ownerAccountId) + + const json = await playlist.toActivityPubObject(req.query.page, null) + const audience = getAudience(playlist.OwnerAccount.Actor, playlist.privacy === VideoPlaylistPrivacy.PUBLIC) + const object = audiencify(json, audience) + + return activityPubResponse(activityPubContextify(object, 'Playlist', getContextFilter()), res) +} + +function videoPlaylistElementController (req: express.Request, res: express.Response) { + const videoPlaylistElement = res.locals.videoPlaylistElementAP + + if (redirectIfNotOwned(videoPlaylistElement.url, res)) return + + const json = videoPlaylistElement.toActivityPubObject() + return activityPubResponse(activityPubContextify(json, 'Playlist', getContextFilter()), res) +} + +function getVideoLocalViewerController (req: express.Request, res: express.Response) { + const localViewer = res.locals.localViewerFull + + return activityPubResponse(activityPubContextify(localViewer.toActivityPubObject(), 'WatchAction', getContextFilter()), res) +} + +// --------------------------------------------------------------------------- + +function actorFollowing (req: express.Request, actor: MActorId) { + const handler = (start: number, count: number) => { + return ActorFollowModel.listAcceptedFollowingUrlsForApi([ actor.id ], undefined, start, count) + } + + return activityPubCollectionPagination(WEBSERVER.URL + req.path, handler, req.query.page) +} + +function actorFollowers (req: express.Request, actor: MActorId) { + const handler = (start: number, count: number) => { + return ActorFollowModel.listAcceptedFollowerUrlsForAP([ actor.id ], undefined, start, count) + } + + return activityPubCollectionPagination(WEBSERVER.URL + req.path, handler, req.query.page) +} + +function actorPlaylists (req: express.Request, options: { account: MAccountId } | { channel: MChannelId }) { + const handler = (start: number, count: number) => { + return VideoPlaylistModel.listPublicUrlsOfForAP(options, start, count) + } + + return activityPubCollectionPagination(WEBSERVER.URL + req.path, handler, req.query.page) +} + +function videoRates (req: express.Request, rateType: VideoRateType, video: MVideoId, url: string) { + const handler = async (start: number, count: number) => { + const result = await AccountVideoRateModel.listAndCountAccountUrlsByVideoId(rateType, video.id, start, count) + return { + total: result.total, + data: result.data.map(r => r.url) + } + } + return activityPubCollectionPagination(url, handler, req.query.page) +} + +function redirectIfNotOwned (url: string, res: express.Response) { + if (url.startsWith(WEBSERVER.URL) === false) { + res.redirect(url) + return true + } + + return false +} diff --git a/server/server/controllers/activitypub/inbox.ts b/server/server/controllers/activitypub/inbox.ts new file mode 100644 index 000000000..6473cea06 --- /dev/null +++ b/server/server/controllers/activitypub/inbox.ts @@ -0,0 +1,84 @@ +import express from 'express' +import { Activity, ActivityPubCollection, ActivityPubOrderedCollection, HttpStatusCode, RootActivity } from '@peertube/peertube-models' +import { InboxManager } from '@server/lib/activitypub/inbox-manager.js' +import { isActivityValid } from '../../helpers/custom-validators/activitypub/activity.js' +import { logger } from '../../helpers/logger.js' +import { + activityPubRateLimiter, + asyncMiddleware, + checkSignature, + ensureIsLocalChannel, + localAccountValidator, + signatureValidator, + videoChannelsNameWithHostValidator +} from '../../middlewares/index.js' +import { activityPubValidator } from '../../middlewares/validators/activitypub/activity.js' + +const inboxRouter = express.Router() + +inboxRouter.post('/inbox', + activityPubRateLimiter, + signatureValidator, + asyncMiddleware(checkSignature), + asyncMiddleware(activityPubValidator), + inboxController +) + +inboxRouter.post('/accounts/:name/inbox', + activityPubRateLimiter, + signatureValidator, + asyncMiddleware(checkSignature), + asyncMiddleware(localAccountValidator), + asyncMiddleware(activityPubValidator), + inboxController +) + +inboxRouter.post('/video-channels/:nameWithHost/inbox', + activityPubRateLimiter, + signatureValidator, + asyncMiddleware(checkSignature), + asyncMiddleware(videoChannelsNameWithHostValidator), + ensureIsLocalChannel, + asyncMiddleware(activityPubValidator), + inboxController +) + +// --------------------------------------------------------------------------- + +export { + inboxRouter +} + +// --------------------------------------------------------------------------- + +function inboxController (req: express.Request, res: express.Response) { + const rootActivity: RootActivity = req.body + let activities: Activity[] + + if ([ 'Collection', 'CollectionPage' ].includes(rootActivity.type)) { + activities = (rootActivity as ActivityPubCollection).items + } else if ([ 'OrderedCollection', 'OrderedCollectionPage' ].includes(rootActivity.type)) { + activities = (rootActivity as ActivityPubOrderedCollection).orderedItems + } else { + activities = [ rootActivity as Activity ] + } + + // Only keep activities we are able to process + logger.debug('Filtering %d activities...', activities.length) + activities = activities.filter(a => isActivityValid(a)) + logger.debug('We keep %d activities.', activities.length, { activities }) + + const accountOrChannel = res.locals.account || res.locals.videoChannel + + logger.info('Receiving inbox requests for %d activities by %s.', activities.length, res.locals.signature.actor.url) + + InboxManager.Instance.addInboxMessage({ + activities, + signatureActor: res.locals.signature.actor, + inboxActor: accountOrChannel + ? accountOrChannel.Actor + : undefined + }) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} diff --git a/server/server/controllers/activitypub/index.ts b/server/server/controllers/activitypub/index.ts new file mode 100644 index 000000000..7057457ee --- /dev/null +++ b/server/server/controllers/activitypub/index.ts @@ -0,0 +1,17 @@ +import express from 'express' + +import { activityPubClientRouter } from './client.js' +import { inboxRouter } from './inbox.js' +import { outboxRouter } from './outbox.js' + +const activityPubRouter = express.Router() + +activityPubRouter.use('/', inboxRouter) +activityPubRouter.use('/', outboxRouter) +activityPubRouter.use('/', activityPubClientRouter) + +// --------------------------------------------------------------------------- + +export { + activityPubRouter +} diff --git a/server/server/controllers/activitypub/outbox.ts b/server/server/controllers/activitypub/outbox.ts new file mode 100644 index 000000000..7714f8400 --- /dev/null +++ b/server/server/controllers/activitypub/outbox.ts @@ -0,0 +1,86 @@ +import express from 'express' +import { Activity, VideoPrivacy } from '@peertube/peertube-models' +import { activityPubContextify } from '@server/helpers/activity-pub-utils.js' +import { activityPubCollectionPagination } from '@server/lib/activitypub/collection.js' +import { getContextFilter } from '@server/lib/activitypub/context.js' +import { MActorLight } from '@server/types/models/index.js' +import { logger } from '../../helpers/logger.js' +import { buildAudience } from '../../lib/activitypub/audience.js' +import { buildAnnounceActivity, buildCreateActivity } from '../../lib/activitypub/send/index.js' +import { + activityPubRateLimiter, + asyncMiddleware, + ensureIsLocalChannel, + localAccountValidator, + videoChannelsNameWithHostValidator +} from '../../middlewares/index.js' +import { apPaginationValidator } from '../../middlewares/validators/activitypub/index.js' +import { VideoModel } from '../../models/video/video.js' +import { activityPubResponse } from './utils.js' + +const outboxRouter = express.Router() + +outboxRouter.get('/accounts/:name/outbox', + activityPubRateLimiter, + apPaginationValidator, + localAccountValidator, + asyncMiddleware(outboxController) +) + +outboxRouter.get('/video-channels/:nameWithHost/outbox', + activityPubRateLimiter, + apPaginationValidator, + asyncMiddleware(videoChannelsNameWithHostValidator), + ensureIsLocalChannel, + asyncMiddleware(outboxController) +) + +// --------------------------------------------------------------------------- + +export { + outboxRouter +} + +// --------------------------------------------------------------------------- + +async function outboxController (req: express.Request, res: express.Response) { + const accountOrVideoChannel = res.locals.account || res.locals.videoChannel + const actor = accountOrVideoChannel.Actor + const actorOutboxUrl = actor.url + '/outbox' + + logger.info('Receiving outbox request for %s.', actorOutboxUrl) + + const handler = (start: number, count: number) => buildActivities(actor, start, count) + const json = await activityPubCollectionPagination(actorOutboxUrl, handler, req.query.page, req.query.size) + + return activityPubResponse(activityPubContextify(json, 'Collection', getContextFilter()), res) +} + +async function buildActivities (actor: MActorLight, start: number, count: number) { + const data = await VideoModel.listAllAndSharedByActorForOutbox(actor.id, start, count) + const activities: Activity[] = [] + + for (const video of data.data) { + const byActor = video.VideoChannel.Account.Actor + const createActivityAudience = buildAudience([ byActor.followersUrl ], video.privacy === VideoPrivacy.PUBLIC) + + // This is a shared video + if (video.VideoShares !== undefined && video.VideoShares.length !== 0) { + const videoShare = video.VideoShares[0] + const announceActivity = buildAnnounceActivity(videoShare.url, actor, video.url, createActivityAudience) + + activities.push(announceActivity) + } else { + // FIXME: only use the video URL to reduce load. Breaks compat with PeerTube < 6.0.0 + const videoObject = await video.toActivityPubObject() + const createActivity = buildCreateActivity(video.url, byActor, videoObject, createActivityAudience) + + activities.push(createActivity) + } + } + + return { + data: activities, + total: data.total + } +} diff --git a/server/controllers/activitypub/utils.ts b/server/server/controllers/activitypub/utils.ts similarity index 100% rename from server/controllers/activitypub/utils.ts rename to server/server/controllers/activitypub/utils.ts diff --git a/server/server/controllers/api/abuse.ts b/server/server/controllers/api/abuse.ts new file mode 100644 index 000000000..78fd514c6 --- /dev/null +++ b/server/server/controllers/api/abuse.ts @@ -0,0 +1,259 @@ +import express from 'express' +import { logger } from '@server/helpers/logger.js' +import { createAccountAbuse, createVideoAbuse, createVideoCommentAbuse } from '@server/lib/moderation.js' +import { Notifier } from '@server/lib/notifier/index.js' +import { AbuseMessageModel } from '@server/models/abuse/abuse-message.js' +import { AbuseModel } from '@server/models/abuse/abuse.js' +import { getServerActor } from '@server/models/application/application.js' +import { abusePredefinedReasonsMap } from '@peertube/peertube-core-utils' +import { AbuseCreate, AbuseState, HttpStatusCode, UserRight } from '@peertube/peertube-models' +import { getFormattedObjects } from '../../helpers/utils.js' +import { sequelizeTypescript } from '../../initializers/database.js' +import { + abuseGetValidator, + abuseListForAdminsValidator, + abuseReportValidator, + abusesSortValidator, + abuseUpdateValidator, + addAbuseMessageValidator, + apiRateLimiter, + asyncMiddleware, + asyncRetryTransactionMiddleware, + authenticate, + checkAbuseValidForMessagesValidator, + deleteAbuseMessageValidator, + ensureUserHasRight, + getAbuseValidator, + openapiOperationDoc, + paginationValidator, + setDefaultPagination, + setDefaultSort +} from '../../middlewares/index.js' +import { AccountModel } from '../../models/account/account.js' + +const abuseRouter = express.Router() + +abuseRouter.use(apiRateLimiter) + +abuseRouter.get('/', + openapiOperationDoc({ operationId: 'getAbuses' }), + authenticate, + ensureUserHasRight(UserRight.MANAGE_ABUSES), + paginationValidator, + abusesSortValidator, + setDefaultSort, + setDefaultPagination, + abuseListForAdminsValidator, + asyncMiddleware(listAbusesForAdmins) +) +abuseRouter.put('/:id', + authenticate, + ensureUserHasRight(UserRight.MANAGE_ABUSES), + asyncMiddleware(abuseUpdateValidator), + asyncRetryTransactionMiddleware(updateAbuse) +) +abuseRouter.post('/', + authenticate, + asyncMiddleware(abuseReportValidator), + asyncRetryTransactionMiddleware(reportAbuse) +) +abuseRouter.delete('/:id', + authenticate, + ensureUserHasRight(UserRight.MANAGE_ABUSES), + asyncMiddleware(abuseGetValidator), + asyncRetryTransactionMiddleware(deleteAbuse) +) + +abuseRouter.get('/:id/messages', + authenticate, + asyncMiddleware(getAbuseValidator), + checkAbuseValidForMessagesValidator, + asyncRetryTransactionMiddleware(listAbuseMessages) +) + +abuseRouter.post('/:id/messages', + authenticate, + asyncMiddleware(getAbuseValidator), + checkAbuseValidForMessagesValidator, + addAbuseMessageValidator, + asyncRetryTransactionMiddleware(addAbuseMessage) +) + +abuseRouter.delete('/:id/messages/:messageId', + authenticate, + asyncMiddleware(getAbuseValidator), + checkAbuseValidForMessagesValidator, + asyncMiddleware(deleteAbuseMessageValidator), + asyncRetryTransactionMiddleware(deleteAbuseMessage) +) + +// --------------------------------------------------------------------------- + +export { + abuseRouter +} + +// --------------------------------------------------------------------------- + +async function listAbusesForAdmins (req: express.Request, res: express.Response) { + const user = res.locals.oauth.token.user + const serverActor = await getServerActor() + + const resultList = await AbuseModel.listForAdminApi({ + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + id: req.query.id, + filter: req.query.filter, + predefinedReason: req.query.predefinedReason, + search: req.query.search, + state: req.query.state, + videoIs: req.query.videoIs, + searchReporter: req.query.searchReporter, + searchReportee: req.query.searchReportee, + searchVideo: req.query.searchVideo, + searchVideoChannel: req.query.searchVideoChannel, + serverAccountId: serverActor.Account.id, + user + }) + + return res.json({ + total: resultList.total, + data: resultList.data.map(d => d.toFormattedAdminJSON()) + }) +} + +async function updateAbuse (req: express.Request, res: express.Response) { + const abuse = res.locals.abuse + let stateUpdated = false + + if (req.body.moderationComment !== undefined) abuse.moderationComment = req.body.moderationComment + + if (req.body.state !== undefined) { + abuse.state = req.body.state + stateUpdated = true + } + + await sequelizeTypescript.transaction(t => { + return abuse.save({ transaction: t }) + }) + + if (stateUpdated === true) { + AbuseModel.loadFull(abuse.id) + .then(abuseFull => Notifier.Instance.notifyOnAbuseStateChange(abuseFull)) + .catch(err => logger.error('Cannot notify on abuse state change', { err })) + } + + // Do not send the delete to other instances, we updated OUR copy of this abuse + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function deleteAbuse (req: express.Request, res: express.Response) { + const abuse = res.locals.abuse + + await sequelizeTypescript.transaction(t => { + return abuse.destroy({ transaction: t }) + }) + + // Do not send the delete to other instances, we delete OUR copy of this abuse + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function reportAbuse (req: express.Request, res: express.Response) { + const videoInstance = res.locals.videoAll + const commentInstance = res.locals.videoCommentFull + const accountInstance = res.locals.account + + const body: AbuseCreate = req.body + + const { id } = await sequelizeTypescript.transaction(async t => { + const user = res.locals.oauth.token.User + // Don't send abuse notification if reporter is an admin/moderator + const skipNotification = user.hasRight(UserRight.MANAGE_ABUSES) + + const reporterAccount = await AccountModel.load(user.Account.id, t) + const predefinedReasons = body.predefinedReasons?.map(r => abusePredefinedReasonsMap[r]) + + const baseAbuse = { + reporterAccountId: reporterAccount.id, + reason: body.reason, + state: AbuseState.PENDING, + predefinedReasons + } + + if (body.video) { + return createVideoAbuse({ + baseAbuse, + videoInstance, + reporterAccount, + transaction: t, + startAt: body.video.startAt, + endAt: body.video.endAt, + skipNotification + }) + } + + if (body.comment) { + return createVideoCommentAbuse({ + baseAbuse, + commentInstance, + reporterAccount, + transaction: t, + skipNotification + }) + } + + // Account report + return createAccountAbuse({ + baseAbuse, + accountInstance, + reporterAccount, + transaction: t, + skipNotification + }) + }) + + return res.json({ abuse: { id } }) +} + +async function listAbuseMessages (req: express.Request, res: express.Response) { + const abuse = res.locals.abuse + + const resultList = await AbuseMessageModel.listForApi(abuse.id) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function addAbuseMessage (req: express.Request, res: express.Response) { + const abuse = res.locals.abuse + const user = res.locals.oauth.token.user + + const abuseMessage = await AbuseMessageModel.create({ + message: req.body.message, + byModerator: abuse.reporterAccountId !== user.Account.id, + accountId: user.Account.id, + abuseId: abuse.id + }) + + AbuseModel.loadFull(abuse.id) + .then(abuseFull => Notifier.Instance.notifyOnAbuseMessage(abuseFull, abuseMessage)) + .catch(err => logger.error('Cannot notify on new abuse message', { err })) + + return res.json({ + abuseMessage: { + id: abuseMessage.id + } + }) +} + +async function deleteAbuseMessage (req: express.Request, res: express.Response) { + const abuseMessage = res.locals.abuseMessage + + await sequelizeTypescript.transaction(t => { + return abuseMessage.destroy({ transaction: t }) + }) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} diff --git a/server/server/controllers/api/accounts.ts b/server/server/controllers/api/accounts.ts new file mode 100644 index 000000000..b5fd4d133 --- /dev/null +++ b/server/server/controllers/api/accounts.ts @@ -0,0 +1,266 @@ +import express from 'express' +import { pickCommonVideoQuery } from '@server/helpers/query.js' +import { ActorFollowModel } from '@server/models/actor/actor-follow.js' +import { getServerActor } from '@server/models/application/application.js' +import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync.js' +import { buildNSFWFilter, getCountVideos, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils.js' +import { getFormattedObjects } from '../../helpers/utils.js' +import { JobQueue } from '../../lib/job-queue/index.js' +import { Hooks } from '../../lib/plugins/hooks.js' +import { + apiRateLimiter, + asyncMiddleware, + authenticate, + commonVideosFiltersValidator, + optionalAuthenticate, + paginationValidator, + setDefaultPagination, + setDefaultSort, + setDefaultVideosSort, + videoPlaylistsSortValidator, + videoRatesSortValidator, + videoRatingValidator +} from '../../middlewares/index.js' +import { + accountNameWithHostGetValidator, + accountsFollowersSortValidator, + accountsSortValidator, + ensureAuthUserOwnsAccountValidator, + ensureCanManageChannelOrAccount, + videoChannelsSortValidator, + videoChannelStatsValidator, + videoChannelSyncsSortValidator, + videosSortValidator +} from '../../middlewares/validators/index.js' +import { commonVideoPlaylistFiltersValidator, videoPlaylistsSearchValidator } from '../../middlewares/validators/videos/video-playlists.js' +import { AccountModel } from '../../models/account/account.js' +import { AccountVideoRateModel } from '../../models/account/account-video-rate.js' +import { guessAdditionalAttributesFromQuery } from '../../models/video/formatter/index.js' +import { VideoModel } from '../../models/video/video.js' +import { VideoChannelModel } from '../../models/video/video-channel.js' +import { VideoPlaylistModel } from '../../models/video/video-playlist.js' + +const accountsRouter = express.Router() + +accountsRouter.use(apiRateLimiter) + +accountsRouter.get('/', + paginationValidator, + accountsSortValidator, + setDefaultSort, + setDefaultPagination, + asyncMiddleware(listAccounts) +) + +accountsRouter.get('/:accountName', + asyncMiddleware(accountNameWithHostGetValidator), + getAccount +) + +accountsRouter.get('/:accountName/videos', + asyncMiddleware(accountNameWithHostGetValidator), + paginationValidator, + videosSortValidator, + setDefaultVideosSort, + setDefaultPagination, + optionalAuthenticate, + commonVideosFiltersValidator, + asyncMiddleware(listAccountVideos) +) + +accountsRouter.get('/:accountName/video-channels', + asyncMiddleware(accountNameWithHostGetValidator), + videoChannelStatsValidator, + paginationValidator, + videoChannelsSortValidator, + setDefaultSort, + setDefaultPagination, + asyncMiddleware(listAccountChannels) +) + +accountsRouter.get('/:accountName/video-channel-syncs', + authenticate, + asyncMiddleware(accountNameWithHostGetValidator), + ensureCanManageChannelOrAccount, + paginationValidator, + videoChannelSyncsSortValidator, + setDefaultSort, + setDefaultPagination, + asyncMiddleware(listAccountChannelsSync) +) + +accountsRouter.get('/:accountName/video-playlists', + optionalAuthenticate, + asyncMiddleware(accountNameWithHostGetValidator), + paginationValidator, + videoPlaylistsSortValidator, + setDefaultSort, + setDefaultPagination, + commonVideoPlaylistFiltersValidator, + videoPlaylistsSearchValidator, + asyncMiddleware(listAccountPlaylists) +) + +accountsRouter.get('/:accountName/ratings', + authenticate, + asyncMiddleware(accountNameWithHostGetValidator), + ensureAuthUserOwnsAccountValidator, + paginationValidator, + videoRatesSortValidator, + setDefaultSort, + setDefaultPagination, + videoRatingValidator, + asyncMiddleware(listAccountRatings) +) + +accountsRouter.get('/:accountName/followers', + authenticate, + asyncMiddleware(accountNameWithHostGetValidator), + ensureAuthUserOwnsAccountValidator, + paginationValidator, + accountsFollowersSortValidator, + setDefaultSort, + setDefaultPagination, + asyncMiddleware(listAccountFollowers) +) + +// --------------------------------------------------------------------------- + +export { + accountsRouter +} + +// --------------------------------------------------------------------------- + +function getAccount (req: express.Request, res: express.Response) { + const account = res.locals.account + + if (account.isOutdated()) { + JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'actor', url: account.Actor.url } }) + } + + return res.json(account.toFormattedJSON()) +} + +async function listAccounts (req: express.Request, res: express.Response) { + const resultList = await AccountModel.listForApi(req.query.start, req.query.count, req.query.sort) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function listAccountChannels (req: express.Request, res: express.Response) { + const options = { + accountId: res.locals.account.id, + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + withStats: req.query.withStats, + search: req.query.search + } + + const resultList = await VideoChannelModel.listByAccountForAPI(options) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function listAccountChannelsSync (req: express.Request, res: express.Response) { + const options = { + accountId: res.locals.account.id, + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + search: req.query.search + } + + const resultList = await VideoChannelSyncModel.listByAccountForAPI(options) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function listAccountPlaylists (req: express.Request, res: express.Response) { + const serverActor = await getServerActor() + + // Allow users to see their private/unlisted video playlists + let listMyPlaylists = false + if (res.locals.oauth && res.locals.oauth.token.User.Account.id === res.locals.account.id) { + listMyPlaylists = true + } + + const resultList = await VideoPlaylistModel.listForApi({ + search: req.query.search, + followerActorId: serverActor.id, + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + accountId: res.locals.account.id, + listMyPlaylists, + type: req.query.playlistType + }) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function listAccountVideos (req: express.Request, res: express.Response) { + const serverActor = await getServerActor() + + const account = res.locals.account + + const displayOnlyForFollower = isUserAbleToSearchRemoteURI(res) + ? null + : { + actorId: serverActor.id, + orLocalVideos: true + } + + const countVideos = getCountVideos(req) + const query = pickCommonVideoQuery(req.query) + + const apiOptions = await Hooks.wrapObject({ + ...query, + + displayOnlyForFollower, + nsfw: buildNSFWFilter(res, query.nsfw), + accountId: account.id, + user: res.locals.oauth ? res.locals.oauth.token.User : undefined, + countVideos + }, 'filter:api.accounts.videos.list.params') + + const resultList = await Hooks.wrapPromiseFun( + VideoModel.listForApi, + apiOptions, + 'filter:api.accounts.videos.list.result' + ) + + return res.json(getFormattedObjects(resultList.data, resultList.total, guessAdditionalAttributesFromQuery(query))) +} + +async function listAccountRatings (req: express.Request, res: express.Response) { + const account = res.locals.account + + const resultList = await AccountVideoRateModel.listByAccountForApi({ + accountId: account.id, + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + type: req.query.rating + }) + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function listAccountFollowers (req: express.Request, res: express.Response) { + const account = res.locals.account + + const channels = await VideoChannelModel.listAllByAccount(account.id) + const actorIds = [ account.actorId ].concat(channels.map(c => c.actorId)) + + const resultList = await ActorFollowModel.listFollowersForApi({ + actorIds, + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + search: req.query.search, + state: 'accepted' + }) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} diff --git a/server/server/controllers/api/blocklist.ts b/server/server/controllers/api/blocklist.ts new file mode 100644 index 000000000..08049431d --- /dev/null +++ b/server/server/controllers/api/blocklist.ts @@ -0,0 +1,110 @@ +import express from 'express' +import { handleToNameAndHost } from '@server/helpers/actors.js' +import { logger } from '@server/helpers/logger.js' +import { AccountBlocklistModel } from '@server/models/account/account-blocklist.js' +import { getServerActor } from '@server/models/application/application.js' +import { ServerBlocklistModel } from '@server/models/server/server-blocklist.js' +import { MActorAccountId, MUserAccountId } from '@server/types/models/index.js' +import { BlockStatus } from '@peertube/peertube-models' +import { apiRateLimiter, asyncMiddleware, blocklistStatusValidator, optionalAuthenticate } from '../../middlewares/index.js' + +const blocklistRouter = express.Router() + +blocklistRouter.use(apiRateLimiter) + +blocklistRouter.get('/status', + optionalAuthenticate, + blocklistStatusValidator, + asyncMiddleware(getBlocklistStatus) +) + +// --------------------------------------------------------------------------- + +export { + blocklistRouter +} + +// --------------------------------------------------------------------------- + +async function getBlocklistStatus (req: express.Request, res: express.Response) { + const hosts = req.query.hosts as string[] + const accounts = req.query.accounts as string[] + const user = res.locals.oauth?.token.User + + const serverActor = await getServerActor() + + const byAccountIds = [ serverActor.Account.id ] + if (user) byAccountIds.push(user.Account.id) + + const status: BlockStatus = { + accounts: {}, + hosts: {} + } + + const baseOptions = { + byAccountIds, + user, + serverActor, + status + } + + await Promise.all([ + populateServerBlocklistStatus({ ...baseOptions, hosts }), + populateAccountBlocklistStatus({ ...baseOptions, accounts }) + ]) + + return res.json(status) +} + +async function populateServerBlocklistStatus (options: { + byAccountIds: number[] + user?: MUserAccountId + serverActor: MActorAccountId + hosts: string[] + status: BlockStatus +}) { + const { byAccountIds, user, serverActor, hosts, status } = options + + if (!hosts || hosts.length === 0) return + + const serverBlocklistStatus = await ServerBlocklistModel.getBlockStatus(byAccountIds, hosts) + + logger.debug('Got server blocklist status.', { serverBlocklistStatus, byAccountIds, hosts }) + + for (const host of hosts) { + const block = serverBlocklistStatus.find(b => b.host === host) + + status.hosts[host] = getStatus(block, serverActor, user) + } +} + +async function populateAccountBlocklistStatus (options: { + byAccountIds: number[] + user?: MUserAccountId + serverActor: MActorAccountId + accounts: string[] + status: BlockStatus +}) { + const { byAccountIds, user, serverActor, accounts, status } = options + + if (!accounts || accounts.length === 0) return + + const accountBlocklistStatus = await AccountBlocklistModel.getBlockStatus(byAccountIds, accounts) + + logger.debug('Got account blocklist status.', { accountBlocklistStatus, byAccountIds, accounts }) + + for (const account of accounts) { + const sanitizedHandle = handleToNameAndHost(account) + + const block = accountBlocklistStatus.find(b => b.name === sanitizedHandle.name && b.host === sanitizedHandle.host) + + status.accounts[sanitizedHandle.handle] = getStatus(block, serverActor, user) + } +} + +function getStatus (block: { accountId: number }, serverActor: MActorAccountId, user?: MUserAccountId) { + return { + blockedByServer: !!(block && block.accountId === serverActor.Account.id), + blockedByUser: !!(block && user && block.accountId === user.Account.id) + } +} diff --git a/server/server/controllers/api/bulk.ts b/server/server/controllers/api/bulk.ts new file mode 100644 index 000000000..829482b4a --- /dev/null +++ b/server/server/controllers/api/bulk.ts @@ -0,0 +1,43 @@ +import express from 'express' +import { BulkRemoveCommentsOfBody, HttpStatusCode } from '@peertube/peertube-models' +import { removeComment } from '@server/lib/video-comment.js' +import { bulkRemoveCommentsOfValidator } from '@server/middlewares/validators/bulk.js' +import { VideoCommentModel } from '@server/models/video/video-comment.js' +import { apiRateLimiter, asyncMiddleware, authenticate } from '../../middlewares/index.js' + +const bulkRouter = express.Router() + +bulkRouter.use(apiRateLimiter) + +bulkRouter.post('/remove-comments-of', + authenticate, + asyncMiddleware(bulkRemoveCommentsOfValidator), + asyncMiddleware(bulkRemoveCommentsOf) +) + +// --------------------------------------------------------------------------- + +export { + bulkRouter +} + +// --------------------------------------------------------------------------- + +async function bulkRemoveCommentsOf (req: express.Request, res: express.Response) { + const account = res.locals.account + const body = req.body as BulkRemoveCommentsOfBody + const user = res.locals.oauth.token.User + + const filter = body.scope === 'my-videos' + ? { onVideosOfAccount: user.Account } + : {} + + const comments = await VideoCommentModel.listForBulkDelete(account, filter) + + // Don't wait result + res.status(HttpStatusCode.NO_CONTENT_204).end() + + for (const comment of comments) { + await removeComment(comment, req, res) + } +} diff --git a/server/server/controllers/api/config.ts b/server/server/controllers/api/config.ts new file mode 100644 index 000000000..58469e97c --- /dev/null +++ b/server/server/controllers/api/config.ts @@ -0,0 +1,377 @@ +import express from 'express' +import { remove, writeJSON } from 'fs-extra/esm' +import snakeCase from 'lodash-es/snakeCase.js' +import validator from 'validator' +import { ServerConfigManager } from '@server/lib/server-config-manager.js' +import { About, CustomConfig, UserRight } from '@peertube/peertube-models' +import { auditLoggerFactory, CustomConfigAuditView, getAuditIdFromRes } from '../../helpers/audit-logger.js' +import { objectConverter } from '../../helpers/core-utils.js' +import { CONFIG, reloadConfig } from '../../initializers/config.js' +import { ClientHtml } from '../../lib/client-html.js' +import { apiRateLimiter, asyncMiddleware, authenticate, ensureUserHasRight, openapiOperationDoc } from '../../middlewares/index.js' +import { customConfigUpdateValidator, ensureConfigIsEditable } from '../../middlewares/validators/config.js' + +const configRouter = express.Router() + +configRouter.use(apiRateLimiter) + +const auditLogger = auditLoggerFactory('config') + +configRouter.get('/', + openapiOperationDoc({ operationId: 'getConfig' }), + asyncMiddleware(getConfig) +) + +configRouter.get('/about', + openapiOperationDoc({ operationId: 'getAbout' }), + getAbout +) + +configRouter.get('/custom', + openapiOperationDoc({ operationId: 'getCustomConfig' }), + authenticate, + ensureUserHasRight(UserRight.MANAGE_CONFIGURATION), + getCustomConfig +) + +configRouter.put('/custom', + openapiOperationDoc({ operationId: 'putCustomConfig' }), + authenticate, + ensureUserHasRight(UserRight.MANAGE_CONFIGURATION), + ensureConfigIsEditable, + customConfigUpdateValidator, + asyncMiddleware(updateCustomConfig) +) + +configRouter.delete('/custom', + openapiOperationDoc({ operationId: 'delCustomConfig' }), + authenticate, + ensureUserHasRight(UserRight.MANAGE_CONFIGURATION), + ensureConfigIsEditable, + asyncMiddleware(deleteCustomConfig) +) + +async function getConfig (req: express.Request, res: express.Response) { + const json = await ServerConfigManager.Instance.getServerConfig(req.ip) + + return res.json(json) +} + +function getAbout (req: express.Request, res: express.Response) { + const about: About = { + instance: { + name: CONFIG.INSTANCE.NAME, + shortDescription: CONFIG.INSTANCE.SHORT_DESCRIPTION, + description: CONFIG.INSTANCE.DESCRIPTION, + terms: CONFIG.INSTANCE.TERMS, + codeOfConduct: CONFIG.INSTANCE.CODE_OF_CONDUCT, + + hardwareInformation: CONFIG.INSTANCE.HARDWARE_INFORMATION, + + creationReason: CONFIG.INSTANCE.CREATION_REASON, + moderationInformation: CONFIG.INSTANCE.MODERATION_INFORMATION, + administrator: CONFIG.INSTANCE.ADMINISTRATOR, + maintenanceLifetime: CONFIG.INSTANCE.MAINTENANCE_LIFETIME, + businessModel: CONFIG.INSTANCE.BUSINESS_MODEL, + + languages: CONFIG.INSTANCE.LANGUAGES, + categories: CONFIG.INSTANCE.CATEGORIES + } + } + + return res.json(about) +} + +function getCustomConfig (req: express.Request, res: express.Response) { + const data = customConfig() + + return res.json(data) +} + +async function deleteCustomConfig (req: express.Request, res: express.Response) { + await remove(CONFIG.CUSTOM_FILE) + + auditLogger.delete(getAuditIdFromRes(res), new CustomConfigAuditView(customConfig())) + + await reloadConfig() + ClientHtml.invalidCache() + + const data = customConfig() + + return res.json(data) +} + +async function updateCustomConfig (req: express.Request, res: express.Response) { + const oldCustomConfigAuditKeys = new CustomConfigAuditView(customConfig()) + + // camelCase to snake_case key + Force number conversion + const toUpdateJSON = convertCustomConfigBody(req.body) + + await writeJSON(CONFIG.CUSTOM_FILE, toUpdateJSON, { spaces: 2 }) + + await reloadConfig() + ClientHtml.invalidCache() + + const data = customConfig() + + auditLogger.update( + getAuditIdFromRes(res), + new CustomConfigAuditView(data), + oldCustomConfigAuditKeys + ) + + return res.json(data) +} + +// --------------------------------------------------------------------------- + +export { + configRouter +} + +// --------------------------------------------------------------------------- + +function customConfig (): CustomConfig { + return { + instance: { + name: CONFIG.INSTANCE.NAME, + shortDescription: CONFIG.INSTANCE.SHORT_DESCRIPTION, + description: CONFIG.INSTANCE.DESCRIPTION, + terms: CONFIG.INSTANCE.TERMS, + codeOfConduct: CONFIG.INSTANCE.CODE_OF_CONDUCT, + + creationReason: CONFIG.INSTANCE.CREATION_REASON, + moderationInformation: CONFIG.INSTANCE.MODERATION_INFORMATION, + administrator: CONFIG.INSTANCE.ADMINISTRATOR, + maintenanceLifetime: CONFIG.INSTANCE.MAINTENANCE_LIFETIME, + businessModel: CONFIG.INSTANCE.BUSINESS_MODEL, + hardwareInformation: CONFIG.INSTANCE.HARDWARE_INFORMATION, + + languages: CONFIG.INSTANCE.LANGUAGES, + categories: CONFIG.INSTANCE.CATEGORIES, + + isNSFW: CONFIG.INSTANCE.IS_NSFW, + defaultNSFWPolicy: CONFIG.INSTANCE.DEFAULT_NSFW_POLICY, + + defaultClientRoute: CONFIG.INSTANCE.DEFAULT_CLIENT_ROUTE, + + customizations: { + css: CONFIG.INSTANCE.CUSTOMIZATIONS.CSS, + javascript: CONFIG.INSTANCE.CUSTOMIZATIONS.JAVASCRIPT + } + }, + theme: { + default: CONFIG.THEME.DEFAULT + }, + services: { + twitter: { + username: CONFIG.SERVICES.TWITTER.USERNAME, + whitelisted: CONFIG.SERVICES.TWITTER.WHITELISTED + } + }, + client: { + videos: { + miniature: { + preferAuthorDisplayName: CONFIG.CLIENT.VIDEOS.MINIATURE.PREFER_AUTHOR_DISPLAY_NAME + } + }, + menu: { + login: { + redirectOnSingleExternalAuth: CONFIG.CLIENT.MENU.LOGIN.REDIRECT_ON_SINGLE_EXTERNAL_AUTH + } + } + }, + cache: { + previews: { + size: CONFIG.CACHE.PREVIEWS.SIZE + }, + captions: { + size: CONFIG.CACHE.VIDEO_CAPTIONS.SIZE + }, + torrents: { + size: CONFIG.CACHE.TORRENTS.SIZE + }, + storyboards: { + size: CONFIG.CACHE.STORYBOARDS.SIZE + } + }, + signup: { + enabled: CONFIG.SIGNUP.ENABLED, + limit: CONFIG.SIGNUP.LIMIT, + requiresApproval: CONFIG.SIGNUP.REQUIRES_APPROVAL, + requiresEmailVerification: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION, + minimumAge: CONFIG.SIGNUP.MINIMUM_AGE + }, + admin: { + email: CONFIG.ADMIN.EMAIL + }, + contactForm: { + enabled: CONFIG.CONTACT_FORM.ENABLED + }, + user: { + history: { + videos: { + enabled: CONFIG.USER.HISTORY.VIDEOS.ENABLED + } + }, + videoQuota: CONFIG.USER.VIDEO_QUOTA, + videoQuotaDaily: CONFIG.USER.VIDEO_QUOTA_DAILY + }, + videoChannels: { + maxPerUser: CONFIG.VIDEO_CHANNELS.MAX_PER_USER + }, + transcoding: { + enabled: CONFIG.TRANSCODING.ENABLED, + remoteRunners: { + enabled: CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED + }, + allowAdditionalExtensions: CONFIG.TRANSCODING.ALLOW_ADDITIONAL_EXTENSIONS, + allowAudioFiles: CONFIG.TRANSCODING.ALLOW_AUDIO_FILES, + threads: CONFIG.TRANSCODING.THREADS, + concurrency: CONFIG.TRANSCODING.CONCURRENCY, + profile: CONFIG.TRANSCODING.PROFILE, + resolutions: { + '0p': CONFIG.TRANSCODING.RESOLUTIONS['0p'], + '144p': CONFIG.TRANSCODING.RESOLUTIONS['144p'], + '240p': CONFIG.TRANSCODING.RESOLUTIONS['240p'], + '360p': CONFIG.TRANSCODING.RESOLUTIONS['360p'], + '480p': CONFIG.TRANSCODING.RESOLUTIONS['480p'], + '720p': CONFIG.TRANSCODING.RESOLUTIONS['720p'], + '1080p': CONFIG.TRANSCODING.RESOLUTIONS['1080p'], + '1440p': CONFIG.TRANSCODING.RESOLUTIONS['1440p'], + '2160p': CONFIG.TRANSCODING.RESOLUTIONS['2160p'] + }, + alwaysTranscodeOriginalResolution: CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION, + webVideos: { + enabled: CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED + }, + hls: { + enabled: CONFIG.TRANSCODING.HLS.ENABLED + } + }, + live: { + enabled: CONFIG.LIVE.ENABLED, + allowReplay: CONFIG.LIVE.ALLOW_REPLAY, + latencySetting: { + enabled: CONFIG.LIVE.LATENCY_SETTING.ENABLED + }, + maxDuration: CONFIG.LIVE.MAX_DURATION, + maxInstanceLives: CONFIG.LIVE.MAX_INSTANCE_LIVES, + maxUserLives: CONFIG.LIVE.MAX_USER_LIVES, + transcoding: { + enabled: CONFIG.LIVE.TRANSCODING.ENABLED, + remoteRunners: { + enabled: CONFIG.LIVE.TRANSCODING.REMOTE_RUNNERS.ENABLED + }, + threads: CONFIG.LIVE.TRANSCODING.THREADS, + profile: CONFIG.LIVE.TRANSCODING.PROFILE, + resolutions: { + '144p': CONFIG.LIVE.TRANSCODING.RESOLUTIONS['144p'], + '240p': CONFIG.LIVE.TRANSCODING.RESOLUTIONS['240p'], + '360p': CONFIG.LIVE.TRANSCODING.RESOLUTIONS['360p'], + '480p': CONFIG.LIVE.TRANSCODING.RESOLUTIONS['480p'], + '720p': CONFIG.LIVE.TRANSCODING.RESOLUTIONS['720p'], + '1080p': CONFIG.LIVE.TRANSCODING.RESOLUTIONS['1080p'], + '1440p': CONFIG.LIVE.TRANSCODING.RESOLUTIONS['1440p'], + '2160p': CONFIG.LIVE.TRANSCODING.RESOLUTIONS['2160p'] + }, + alwaysTranscodeOriginalResolution: CONFIG.LIVE.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION + } + }, + videoStudio: { + enabled: CONFIG.VIDEO_STUDIO.ENABLED, + remoteRunners: { + enabled: CONFIG.VIDEO_STUDIO.REMOTE_RUNNERS.ENABLED + } + }, + videoFile: { + update: { + enabled: CONFIG.VIDEO_FILE.UPDATE.ENABLED + } + }, + import: { + videos: { + concurrency: CONFIG.IMPORT.VIDEOS.CONCURRENCY, + http: { + enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED + }, + torrent: { + enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED + } + }, + videoChannelSynchronization: { + enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED, + maxPerUser: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER + } + }, + trending: { + videos: { + algorithms: { + enabled: CONFIG.TRENDING.VIDEOS.ALGORITHMS.ENABLED, + default: CONFIG.TRENDING.VIDEOS.ALGORITHMS.DEFAULT + } + } + }, + autoBlacklist: { + videos: { + ofUsers: { + enabled: CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED + } + } + }, + followers: { + instance: { + enabled: CONFIG.FOLLOWERS.INSTANCE.ENABLED, + manualApproval: CONFIG.FOLLOWERS.INSTANCE.MANUAL_APPROVAL + } + }, + followings: { + instance: { + autoFollowBack: { + enabled: CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_BACK.ENABLED + }, + + autoFollowIndex: { + enabled: CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.ENABLED, + indexUrl: CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.INDEX_URL + } + } + }, + broadcastMessage: { + enabled: CONFIG.BROADCAST_MESSAGE.ENABLED, + message: CONFIG.BROADCAST_MESSAGE.MESSAGE, + level: CONFIG.BROADCAST_MESSAGE.LEVEL, + dismissable: CONFIG.BROADCAST_MESSAGE.DISMISSABLE + }, + search: { + remoteUri: { + users: CONFIG.SEARCH.REMOTE_URI.USERS, + anonymous: CONFIG.SEARCH.REMOTE_URI.ANONYMOUS + }, + searchIndex: { + enabled: CONFIG.SEARCH.SEARCH_INDEX.ENABLED, + url: CONFIG.SEARCH.SEARCH_INDEX.URL, + disableLocalSearch: CONFIG.SEARCH.SEARCH_INDEX.DISABLE_LOCAL_SEARCH, + isDefaultSearch: CONFIG.SEARCH.SEARCH_INDEX.IS_DEFAULT_SEARCH + } + } + } +} + +function convertCustomConfigBody (body: CustomConfig) { + function keyConverter (k: string) { + // Transcoding resolutions exception + if (/^\d{3,4}p$/.exec(k)) return k + if (k === '0p') return k + + return snakeCase(k) + } + + function valueConverter (v: any) { + if (validator.default.isNumeric(v + '')) return parseInt('' + v, 10) + + return v + } + + return objectConverter(body, keyConverter, valueConverter) +} diff --git a/server/server/controllers/api/custom-page.ts b/server/server/controllers/api/custom-page.ts new file mode 100644 index 000000000..0be8d46c6 --- /dev/null +++ b/server/server/controllers/api/custom-page.ts @@ -0,0 +1,48 @@ +import express from 'express' +import { ServerConfigManager } from '@server/lib/server-config-manager.js' +import { ActorCustomPageModel } from '@server/models/account/actor-custom-page.js' +import { HttpStatusCode, UserRight } from '@peertube/peertube-models' +import { apiRateLimiter, asyncMiddleware, authenticate, ensureUserHasRight } from '../../middlewares/index.js' + +const customPageRouter = express.Router() + +customPageRouter.use(apiRateLimiter) + +customPageRouter.get('/homepage/instance', + asyncMiddleware(getInstanceHomepage) +) + +customPageRouter.put('/homepage/instance', + authenticate, + ensureUserHasRight(UserRight.MANAGE_INSTANCE_CUSTOM_PAGE), + asyncMiddleware(updateInstanceHomepage) +) + +// --------------------------------------------------------------------------- + +export { + customPageRouter +} + +// --------------------------------------------------------------------------- + +async function getInstanceHomepage (req: express.Request, res: express.Response) { + const page = await ActorCustomPageModel.loadInstanceHomepage() + if (!page) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Instance homepage could not be found' + }) + } + + return res.json(page.toFormattedJSON()) +} + +async function updateInstanceHomepage (req: express.Request, res: express.Response) { + const content = req.body.content + + await ActorCustomPageModel.updateInstanceHomepage(content) + ServerConfigManager.Instance.updateHomepageState(content) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} diff --git a/server/server/controllers/api/index.ts b/server/server/controllers/api/index.ts new file mode 100644 index 000000000..cb5d59802 --- /dev/null +++ b/server/server/controllers/api/index.ts @@ -0,0 +1,73 @@ +import cors from 'cors' +import express from 'express' +import { logger } from '@server/helpers/logger.js' +import { HttpStatusCode } from '@peertube/peertube-models' +import { abuseRouter } from './abuse.js' +import { accountsRouter } from './accounts.js' +import { blocklistRouter } from './blocklist.js' +import { bulkRouter } from './bulk.js' +import { configRouter } from './config.js' +import { customPageRouter } from './custom-page.js' +import { jobsRouter } from './jobs.js' +import { metricsRouter } from './metrics.js' +import { oauthClientsRouter } from './oauth-clients.js' +import { overviewsRouter } from './overviews.js' +import { pluginRouter } from './plugins.js' +import { runnersRouter } from './runners/index.js' +import { searchRouter } from './search/index.js' +import { serverRouter } from './server/index.js' +import { usersRouter } from './users/index.js' +import { videoChannelSyncRouter } from './video-channel-sync.js' +import { videoChannelRouter } from './video-channel.js' +import { videoPlaylistRouter } from './video-playlist.js' +import { videosRouter } from './videos/index.js' + +const apiRouter = express.Router() + +apiRouter.use(cors({ + origin: '*', + exposedHeaders: 'Retry-After', + credentials: true +})) + +apiRouter.use('/server', serverRouter) +apiRouter.use('/abuses', abuseRouter) +apiRouter.use('/bulk', bulkRouter) +apiRouter.use('/oauth-clients', oauthClientsRouter) +apiRouter.use('/config', configRouter) +apiRouter.use('/users', usersRouter) +apiRouter.use('/accounts', accountsRouter) +apiRouter.use('/video-channels', videoChannelRouter) +apiRouter.use('/video-channel-syncs', videoChannelSyncRouter) +apiRouter.use('/video-playlists', videoPlaylistRouter) +apiRouter.use('/videos', videosRouter) +apiRouter.use('/jobs', jobsRouter) +apiRouter.use('/metrics', metricsRouter) +apiRouter.use('/search', searchRouter) +apiRouter.use('/overviews', overviewsRouter) +apiRouter.use('/plugins', pluginRouter) +apiRouter.use('/custom-pages', customPageRouter) +apiRouter.use('/blocklist', blocklistRouter) +apiRouter.use('/runners', runnersRouter) + +// apiRouter.use(apiRateLimiter) +apiRouter.use('/ping', pong) +apiRouter.use('/*', badRequest) + +// --------------------------------------------------------------------------- + +export { apiRouter } + +// --------------------------------------------------------------------------- + +function pong (req: express.Request, res: express.Response) { + return res.send('pong').status(HttpStatusCode.OK_200).end() +} + +function badRequest (req: express.Request, res: express.Response) { + logger.debug(`API express handler not found: bad PeerTube request for ${req.method} - ${req.originalUrl}`) + + return res.type('json') + .status(HttpStatusCode.BAD_REQUEST_400) + .end() +} diff --git a/server/server/controllers/api/jobs.ts b/server/server/controllers/api/jobs.ts new file mode 100644 index 000000000..9a86d2a1e --- /dev/null +++ b/server/server/controllers/api/jobs.ts @@ -0,0 +1,109 @@ +import { Job as BullJob } from 'bullmq' +import express from 'express' +import { HttpStatusCode, Job, JobState, JobType, ResultList, UserRight } from '@peertube/peertube-models' +import { isArray } from '../../helpers/custom-validators/misc.js' +import { JobQueue } from '../../lib/job-queue/index.js' +import { + apiRateLimiter, + asyncMiddleware, + authenticate, + ensureUserHasRight, + jobsSortValidator, + openapiOperationDoc, + paginationValidatorBuilder, + setDefaultPagination, + setDefaultSort +} from '../../middlewares/index.js' +import { listJobsValidator } from '../../middlewares/validators/jobs.js' + +const jobsRouter = express.Router() + +jobsRouter.use(apiRateLimiter) + +jobsRouter.post('/pause', + authenticate, + ensureUserHasRight(UserRight.MANAGE_JOBS), + asyncMiddleware(pauseJobQueue) +) + +jobsRouter.post('/resume', + authenticate, + ensureUserHasRight(UserRight.MANAGE_JOBS), + resumeJobQueue +) + +jobsRouter.get('/:state?', + openapiOperationDoc({ operationId: 'getJobs' }), + authenticate, + ensureUserHasRight(UserRight.MANAGE_JOBS), + paginationValidatorBuilder([ 'jobs' ]), + jobsSortValidator, + setDefaultSort, + setDefaultPagination, + listJobsValidator, + asyncMiddleware(listJobs) +) + +// --------------------------------------------------------------------------- + +export { + jobsRouter +} + +// --------------------------------------------------------------------------- + +async function pauseJobQueue (req: express.Request, res: express.Response) { + await JobQueue.Instance.pause() + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +function resumeJobQueue (req: express.Request, res: express.Response) { + JobQueue.Instance.resume() + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +async function listJobs (req: express.Request, res: express.Response) { + const state = req.params.state as JobState + const asc = req.query.sort === 'createdAt' + const jobType = req.query.jobType + + const jobs = await JobQueue.Instance.listForApi({ + state, + start: req.query.start, + count: req.query.count, + asc, + jobType + }) + const total = await JobQueue.Instance.count(state, jobType) + + const result: ResultList = { + total, + data: await Promise.all(jobs.map(j => formatJob(j, state))) + } + + return res.json(result) +} + +async function formatJob (job: BullJob, state?: JobState): Promise { + const error = isArray(job.stacktrace) && job.stacktrace.length !== 0 + ? job.stacktrace[0] + : null + + return { + id: job.id, + state: state || await job.getState(), + type: job.queueName as JobType, + data: job.data, + parent: job.parent + ? { id: job.parent.id } + : undefined, + progress: job.progress as number, + priority: job.opts.priority, + error, + createdAt: new Date(job.timestamp), + finishedOn: new Date(job.finishedOn), + processedOn: new Date(job.processedOn) + } +} diff --git a/server/server/controllers/api/metrics.ts b/server/server/controllers/api/metrics.ts new file mode 100644 index 000000000..42299817f --- /dev/null +++ b/server/server/controllers/api/metrics.ts @@ -0,0 +1,34 @@ +import express from 'express' +import { CONFIG } from '@server/initializers/config.js' +import { OpenTelemetryMetrics } from '@server/lib/opentelemetry/metrics.js' +import { HttpStatusCode, PlaybackMetricCreate } from '@peertube/peertube-models' +import { addPlaybackMetricValidator, apiRateLimiter, asyncMiddleware } from '../../middlewares/index.js' + +const metricsRouter = express.Router() + +metricsRouter.use(apiRateLimiter) + +metricsRouter.post('/playback', + asyncMiddleware(addPlaybackMetricValidator), + addPlaybackMetric +) + +// --------------------------------------------------------------------------- + +export { + metricsRouter +} + +// --------------------------------------------------------------------------- + +function addPlaybackMetric (req: express.Request, res: express.Response) { + if (!CONFIG.OPEN_TELEMETRY.METRICS.ENABLED) { + return res.sendStatus(HttpStatusCode.FORBIDDEN_403) + } + + const body: PlaybackMetricCreate = req.body + + OpenTelemetryMetrics.Instance.observePlaybackMetric(res.locals.onlyImmutableVideo, body) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} diff --git a/server/server/controllers/api/oauth-clients.ts b/server/server/controllers/api/oauth-clients.ts new file mode 100644 index 000000000..a9ed9e330 --- /dev/null +++ b/server/server/controllers/api/oauth-clients.ts @@ -0,0 +1,54 @@ +import express from 'express' +import { HttpStatusCode, OAuthClientLocal } from '@peertube/peertube-models' +import { isTestOrDevInstance } from '@peertube/peertube-node-utils' +import { OAuthClientModel } from '@server/models/oauth/oauth-client.js' +import { logger } from '../../helpers/logger.js' +import { CONFIG } from '../../initializers/config.js' +import { apiRateLimiter, asyncMiddleware, openapiOperationDoc } from '../../middlewares/index.js' + +const oauthClientsRouter = express.Router() + +oauthClientsRouter.use(apiRateLimiter) + +oauthClientsRouter.get('/local', + openapiOperationDoc({ operationId: 'getOAuthClient' }), + asyncMiddleware(getLocalClient) +) + +// Get the client credentials for the PeerTube front end +async function getLocalClient (req: express.Request, res: express.Response, next: express.NextFunction) { + const serverHostname = CONFIG.WEBSERVER.HOSTNAME + const serverPort = CONFIG.WEBSERVER.PORT + let headerHostShouldBe = serverHostname + if (serverPort !== 80 && serverPort !== 443) { + headerHostShouldBe += ':' + serverPort + } + + // Don't make this check if this is a test instance + if (!isTestOrDevInstance() && req.get('host') !== headerHostShouldBe) { + logger.info( + 'Getting client tokens for host %s is forbidden (expected %s).', req.get('host'), headerHostShouldBe, + { webserverConfig: CONFIG.WEBSERVER } + ) + + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: `Getting client tokens for host ${req.get('host')} is forbidden` + }) + } + + const client = await OAuthClientModel.loadFirstClient() + if (!client) throw new Error('No client available.') + + const json: OAuthClientLocal = { + client_id: client.clientId, + client_secret: client.clientSecret + } + return res.json(json) +} + +// --------------------------------------------------------------------------- + +export { + oauthClientsRouter +} diff --git a/server/server/controllers/api/overviews.ts b/server/server/controllers/api/overviews.ts new file mode 100644 index 000000000..a34669662 --- /dev/null +++ b/server/server/controllers/api/overviews.ts @@ -0,0 +1,139 @@ +import express from 'express' +import memoizee from 'memoizee' +import { logger } from '@server/helpers/logger.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { getServerActor } from '@server/models/application/application.js' +import { VideoModel } from '@server/models/video/video.js' +import { CategoryOverview, ChannelOverview, TagOverview, VideosOverview } from '@peertube/peertube-models' +import { buildNSFWFilter } from '../../helpers/express-utils.js' +import { MEMOIZE_TTL, OVERVIEWS } from '../../initializers/constants.js' +import { apiRateLimiter, asyncMiddleware, optionalAuthenticate, videosOverviewValidator } from '../../middlewares/index.js' +import { TagModel } from '../../models/video/tag.js' + +const overviewsRouter = express.Router() + +overviewsRouter.use(apiRateLimiter) + +overviewsRouter.get('/videos', + videosOverviewValidator, + optionalAuthenticate, + asyncMiddleware(getVideosOverview) +) + +// --------------------------------------------------------------------------- + +export { overviewsRouter } + +// --------------------------------------------------------------------------- + +const buildSamples = memoizee(async function () { + const [ categories, channels, tags ] = await Promise.all([ + VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT), + VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT), + TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT) + ]) + + const result = { categories, channels, tags } + + logger.debug('Building samples for overview endpoint.', { result }) + + return result +}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE }) + +// This endpoint could be quite long, but we cache it +async function getVideosOverview (req: express.Request, res: express.Response) { + const attributes = await buildSamples() + + const page = req.query.page || 1 + const index = page - 1 + + const categories: CategoryOverview[] = [] + const channels: ChannelOverview[] = [] + const tags: TagOverview[] = [] + + await Promise.all([ + getVideosByCategory(attributes.categories, index, res, categories), + getVideosByChannel(attributes.channels, index, res, channels), + getVideosByTag(attributes.tags, index, res, tags) + ]) + + const result: VideosOverview = { + categories, + channels, + tags + } + + return res.json(result) +} + +async function getVideosByTag (tagsSample: string[], index: number, res: express.Response, acc: TagOverview[]) { + if (tagsSample.length <= index) return + + const tag = tagsSample[index] + const videos = await getVideos(res, { tagsOneOf: [ tag ] }) + + if (videos.length === 0) return + + acc.push({ + tag, + videos + }) +} + +async function getVideosByCategory (categoriesSample: number[], index: number, res: express.Response, acc: CategoryOverview[]) { + if (categoriesSample.length <= index) return + + const category = categoriesSample[index] + const videos = await getVideos(res, { categoryOneOf: [ category ] }) + + if (videos.length === 0) return + + acc.push({ + category: videos[0].category, + videos + }) +} + +async function getVideosByChannel (channelsSample: number[], index: number, res: express.Response, acc: ChannelOverview[]) { + if (channelsSample.length <= index) return + + const channelId = channelsSample[index] + const videos = await getVideos(res, { videoChannelId: channelId }) + + if (videos.length === 0) return + + acc.push({ + channel: videos[0].channel, + videos + }) +} + +async function getVideos ( + res: express.Response, + where: { videoChannelId?: number, tagsOneOf?: string[], categoryOneOf?: number[] } +) { + const serverActor = await getServerActor() + + const query = await Hooks.wrapObject({ + start: 0, + count: 12, + sort: '-createdAt', + displayOnlyForFollower: { + actorId: serverActor.id, + orLocalVideos: true + }, + nsfw: buildNSFWFilter(res), + user: res.locals.oauth ? res.locals.oauth.token.User : undefined, + countVideos: false, + + ...where + }, 'filter:api.overviews.videos.list.params') + + const { data } = await Hooks.wrapPromiseFun( + VideoModel.listForApi, + query, + 'filter:api.overviews.videos.list.result' + ) + + return data.map(d => d.toFormattedJSON()) +} diff --git a/server/server/controllers/api/plugins.ts b/server/server/controllers/api/plugins.ts new file mode 100644 index 000000000..85e458c9e --- /dev/null +++ b/server/server/controllers/api/plugins.ts @@ -0,0 +1,230 @@ +import express from 'express' +import { logger } from '@server/helpers/logger.js' +import { getFormattedObjects } from '@server/helpers/utils.js' +import { listAvailablePluginsFromIndex } from '@server/lib/plugins/plugin-index.js' +import { PluginManager } from '@server/lib/plugins/plugin-manager.js' +import { + apiRateLimiter, + asyncMiddleware, + authenticate, + availablePluginsSortValidator, + ensureUserHasRight, + openapiOperationDoc, + paginationValidator, + pluginsSortValidator, + setDefaultPagination, + setDefaultSort +} from '@server/middlewares/index.js' +import { + existingPluginValidator, + installOrUpdatePluginValidator, + listAvailablePluginsValidator, + listPluginsValidator, + uninstallPluginValidator, + updatePluginSettingsValidator +} from '@server/middlewares/validators/plugins.js' +import { PluginModel } from '@server/models/server/plugin.js' +import { + HttpStatusCode, + InstallOrUpdatePlugin, + ManagePlugin, + PeertubePluginIndexList, + PublicServerSetting, + RegisteredServerSettings, + UserRight +} from '@peertube/peertube-models' + +const pluginRouter = express.Router() + +pluginRouter.use(apiRateLimiter) + +pluginRouter.get('/available', + openapiOperationDoc({ operationId: 'getAvailablePlugins' }), + authenticate, + ensureUserHasRight(UserRight.MANAGE_PLUGINS), + listAvailablePluginsValidator, + paginationValidator, + availablePluginsSortValidator, + setDefaultSort, + setDefaultPagination, + asyncMiddleware(listAvailablePlugins) +) + +pluginRouter.get('/', + openapiOperationDoc({ operationId: 'getPlugins' }), + authenticate, + ensureUserHasRight(UserRight.MANAGE_PLUGINS), + listPluginsValidator, + paginationValidator, + pluginsSortValidator, + setDefaultSort, + setDefaultPagination, + asyncMiddleware(listPlugins) +) + +pluginRouter.get('/:npmName/registered-settings', + authenticate, + ensureUserHasRight(UserRight.MANAGE_PLUGINS), + asyncMiddleware(existingPluginValidator), + getPluginRegisteredSettings +) + +pluginRouter.get('/:npmName/public-settings', + asyncMiddleware(existingPluginValidator), + getPublicPluginSettings +) + +pluginRouter.put('/:npmName/settings', + authenticate, + ensureUserHasRight(UserRight.MANAGE_PLUGINS), + updatePluginSettingsValidator, + asyncMiddleware(existingPluginValidator), + asyncMiddleware(updatePluginSettings) +) + +pluginRouter.get('/:npmName', + authenticate, + ensureUserHasRight(UserRight.MANAGE_PLUGINS), + asyncMiddleware(existingPluginValidator), + getPlugin +) + +pluginRouter.post('/install', + openapiOperationDoc({ operationId: 'addPlugin' }), + authenticate, + ensureUserHasRight(UserRight.MANAGE_PLUGINS), + installOrUpdatePluginValidator, + asyncMiddleware(installPlugin) +) + +pluginRouter.post('/update', + openapiOperationDoc({ operationId: 'updatePlugin' }), + authenticate, + ensureUserHasRight(UserRight.MANAGE_PLUGINS), + installOrUpdatePluginValidator, + asyncMiddleware(updatePlugin) +) + +pluginRouter.post('/uninstall', + openapiOperationDoc({ operationId: 'uninstallPlugin' }), + authenticate, + ensureUserHasRight(UserRight.MANAGE_PLUGINS), + uninstallPluginValidator, + asyncMiddleware(uninstallPlugin) +) + +// --------------------------------------------------------------------------- + +export { + pluginRouter +} + +// --------------------------------------------------------------------------- + +async function listPlugins (req: express.Request, res: express.Response) { + const pluginType = req.query.pluginType + const uninstalled = req.query.uninstalled + + const resultList = await PluginModel.listForApi({ + pluginType, + uninstalled, + start: req.query.start, + count: req.query.count, + sort: req.query.sort + }) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +function getPlugin (req: express.Request, res: express.Response) { + const plugin = res.locals.plugin + + return res.json(plugin.toFormattedJSON()) +} + +async function installPlugin (req: express.Request, res: express.Response) { + const body: InstallOrUpdatePlugin = req.body + + const fromDisk = !!body.path + const toInstall = body.npmName || body.path + + const pluginVersion = body.pluginVersion && body.npmName + ? body.pluginVersion + : undefined + + try { + const plugin = await PluginManager.Instance.install({ toInstall, version: pluginVersion, fromDisk }) + + return res.json(plugin.toFormattedJSON()) + } catch (err) { + logger.warn('Cannot install plugin %s.', toInstall, { err }) + return res.fail({ message: 'Cannot install plugin ' + toInstall }) + } +} + +async function updatePlugin (req: express.Request, res: express.Response) { + const body: InstallOrUpdatePlugin = req.body + + const fromDisk = !!body.path + const toUpdate = body.npmName || body.path + try { + const plugin = await PluginManager.Instance.update(toUpdate, fromDisk) + + return res.json(plugin.toFormattedJSON()) + } catch (err) { + logger.warn('Cannot update plugin %s.', toUpdate, { err }) + return res.fail({ message: 'Cannot update plugin ' + toUpdate }) + } +} + +async function uninstallPlugin (req: express.Request, res: express.Response) { + const body: ManagePlugin = req.body + + await PluginManager.Instance.uninstall({ npmName: body.npmName }) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +function getPublicPluginSettings (req: express.Request, res: express.Response) { + const plugin = res.locals.plugin + const registeredSettings = PluginManager.Instance.getRegisteredSettings(req.params.npmName) + const publicSettings = plugin.getPublicSettings(registeredSettings) + + const json: PublicServerSetting = { publicSettings } + + return res.json(json) +} + +function getPluginRegisteredSettings (req: express.Request, res: express.Response) { + const registeredSettings = PluginManager.Instance.getRegisteredSettings(req.params.npmName) + + const json: RegisteredServerSettings = { registeredSettings } + + return res.json(json) +} + +async function updatePluginSettings (req: express.Request, res: express.Response) { + const plugin = res.locals.plugin + + plugin.settings = req.body.settings + await plugin.save() + + await PluginManager.Instance.onSettingsChanged(plugin.name, plugin.settings) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function listAvailablePlugins (req: express.Request, res: express.Response) { + const query: PeertubePluginIndexList = req.query + + const resultList = await listAvailablePluginsFromIndex(query) + + if (!resultList) { + return res.fail({ + status: HttpStatusCode.SERVICE_UNAVAILABLE_503, + message: 'Plugin index unavailable. Please retry later' + }) + } + + return res.json(resultList) +} diff --git a/server/server/controllers/api/runners/index.ts b/server/server/controllers/api/runners/index.ts new file mode 100644 index 000000000..b5cec3d1a --- /dev/null +++ b/server/server/controllers/api/runners/index.ts @@ -0,0 +1,20 @@ +import express from 'express' +import { runnerJobsRouter } from './jobs.js' +import { runnerJobFilesRouter } from './jobs-files.js' +import { manageRunnersRouter } from './manage-runners.js' +import { runnerRegistrationTokensRouter } from './registration-tokens.js' + +const runnersRouter = express.Router() + +// No api route limiter here, they are defined in child routers + +runnersRouter.use('/', manageRunnersRouter) +runnersRouter.use('/', runnerJobsRouter) +runnersRouter.use('/', runnerJobFilesRouter) +runnersRouter.use('/', runnerRegistrationTokensRouter) + +// --------------------------------------------------------------------------- + +export { + runnersRouter +} diff --git a/server/server/controllers/api/runners/jobs-files.ts b/server/server/controllers/api/runners/jobs-files.ts new file mode 100644 index 000000000..91a7302b8 --- /dev/null +++ b/server/server/controllers/api/runners/jobs-files.ts @@ -0,0 +1,112 @@ +import express from 'express' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { proxifyHLS, proxifyWebVideoFile } from '@server/lib/object-storage/index.js' +import { VideoPathManager } from '@server/lib/video-path-manager.js' +import { getStudioTaskFilePath } from '@server/lib/video-studio.js' +import { apiRateLimiter, asyncMiddleware } from '@server/middlewares/index.js' +import { jobOfRunnerGetValidatorFactory } from '@server/middlewares/validators/runners/index.js' +import { + runnerJobGetVideoStudioTaskFileValidator, + runnerJobGetVideoTranscodingFileValidator +} from '@server/middlewares/validators/runners/job-files.js' +import { RunnerJobState, VideoStorage } from '@peertube/peertube-models' + +const lTags = loggerTagsFactory('api', 'runner') + +const runnerJobFilesRouter = express.Router() + +runnerJobFilesRouter.post('/jobs/:jobUUID/files/videos/:videoId/max-quality', + apiRateLimiter, + asyncMiddleware(jobOfRunnerGetValidatorFactory([ RunnerJobState.PROCESSING ])), + asyncMiddleware(runnerJobGetVideoTranscodingFileValidator), + asyncMiddleware(getMaxQualityVideoFile) +) + +runnerJobFilesRouter.post('/jobs/:jobUUID/files/videos/:videoId/previews/max-quality', + apiRateLimiter, + asyncMiddleware(jobOfRunnerGetValidatorFactory([ RunnerJobState.PROCESSING ])), + asyncMiddleware(runnerJobGetVideoTranscodingFileValidator), + getMaxQualityVideoPreview +) + +runnerJobFilesRouter.post('/jobs/:jobUUID/files/videos/:videoId/studio/task-files/:filename', + apiRateLimiter, + asyncMiddleware(jobOfRunnerGetValidatorFactory([ RunnerJobState.PROCESSING ])), + asyncMiddleware(runnerJobGetVideoTranscodingFileValidator), + runnerJobGetVideoStudioTaskFileValidator, + getVideoStudioTaskFile +) + +// --------------------------------------------------------------------------- + +export { + runnerJobFilesRouter +} + +// --------------------------------------------------------------------------- + +async function getMaxQualityVideoFile (req: express.Request, res: express.Response) { + const runnerJob = res.locals.runnerJob + const runner = runnerJob.Runner + const video = res.locals.videoAll + + logger.info( + 'Get max quality file of video %s of job %s for runner %s', video.uuid, runnerJob.uuid, runner.name, + lTags(runner.name, runnerJob.id, runnerJob.type) + ) + + const file = video.getMaxQualityFile() + + if (file.storage === VideoStorage.OBJECT_STORAGE) { + if (file.isHLS()) { + return proxifyHLS({ + req, + res, + filename: file.filename, + playlist: video.getHLSPlaylist(), + reinjectVideoFileToken: false, + video + }) + } + + // Web video + return proxifyWebVideoFile({ + req, + res, + filename: file.filename + }) + } + + return VideoPathManager.Instance.makeAvailableVideoFile(file, videoPath => { + return res.sendFile(videoPath) + }) +} + +function getMaxQualityVideoPreview (req: express.Request, res: express.Response) { + const runnerJob = res.locals.runnerJob + const runner = runnerJob.Runner + const video = res.locals.videoAll + + logger.info( + 'Get max quality preview file of video %s of job %s for runner %s', video.uuid, runnerJob.uuid, runner.name, + lTags(runner.name, runnerJob.id, runnerJob.type) + ) + + const file = video.getPreview() + + return res.sendFile(file.getPath()) +} + +function getVideoStudioTaskFile (req: express.Request, res: express.Response) { + const runnerJob = res.locals.runnerJob + const runner = runnerJob.Runner + const video = res.locals.videoAll + const filename = req.params.filename + + logger.info( + 'Get video studio task file %s of video %s of job %s for runner %s', filename, video.uuid, runnerJob.uuid, runner.name, + lTags(runner.name, runnerJob.id, runnerJob.type) + ) + + return res.sendFile(getStudioTaskFilePath(filename)) +} diff --git a/server/server/controllers/api/runners/jobs.ts b/server/server/controllers/api/runners/jobs.ts new file mode 100644 index 000000000..c7aada7c2 --- /dev/null +++ b/server/server/controllers/api/runners/jobs.ts @@ -0,0 +1,416 @@ +import express, { UploadFiles } from 'express' +import { retryTransactionWrapper } from '@server/helpers/database-utils.js' +import { createReqFiles } from '@server/helpers/express-utils.js' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { generateRunnerJobToken } from '@server/helpers/token-generator.js' +import { MIMETYPES } from '@server/initializers/constants.js' +import { sequelizeTypescript } from '@server/initializers/database.js' +import { getRunnerJobHandlerClass, runnerJobCanBeCancelled, updateLastRunnerContact } from '@server/lib/runners/index.js' +import { + apiRateLimiter, + asyncMiddleware, + authenticate, + ensureUserHasRight, + paginationValidator, + runnerJobsSortValidator, + setDefaultPagination, + setDefaultSort +} from '@server/middlewares/index.js' +import { + abortRunnerJobValidator, + acceptRunnerJobValidator, + cancelRunnerJobValidator, + errorRunnerJobValidator, + getRunnerFromTokenValidator, + jobOfRunnerGetValidatorFactory, + listRunnerJobsValidator, + runnerJobGetValidator, + successRunnerJobValidator, + updateRunnerJobValidator +} from '@server/middlewares/validators/runners/index.js' +import { RunnerModel } from '@server/models/runner/runner.js' +import { RunnerJobModel } from '@server/models/runner/runner-job.js' +import { + AbortRunnerJobBody, + AcceptRunnerJobResult, + ErrorRunnerJobBody, + HttpStatusCode, + ListRunnerJobsQuery, + LiveRTMPHLSTranscodingUpdatePayload, + RequestRunnerJobResult, + RunnerJobState, + RunnerJobSuccessBody, + RunnerJobSuccessPayload, + RunnerJobType, + RunnerJobUpdateBody, + RunnerJobUpdatePayload, + ServerErrorCode, + UserRight, + VideoStudioTranscodingSuccess, + VODAudioMergeTranscodingSuccess, + VODHLSTranscodingSuccess, + VODWebVideoTranscodingSuccess +} from '@peertube/peertube-models' + +const postRunnerJobSuccessVideoFiles = createReqFiles( + [ 'payload[videoFile]', 'payload[resolutionPlaylistFile]' ], + { ...MIMETYPES.VIDEO.MIMETYPE_EXT, ...MIMETYPES.M3U8.MIMETYPE_EXT } +) + +const runnerJobUpdateVideoFiles = createReqFiles( + [ 'payload[videoChunkFile]', 'payload[resolutionPlaylistFile]', 'payload[masterPlaylistFile]' ], + { ...MIMETYPES.VIDEO.MIMETYPE_EXT, ...MIMETYPES.M3U8.MIMETYPE_EXT } +) + +const lTags = loggerTagsFactory('api', 'runner') + +const runnerJobsRouter = express.Router() + +// --------------------------------------------------------------------------- +// Controllers for runners +// --------------------------------------------------------------------------- + +runnerJobsRouter.post('/jobs/request', + apiRateLimiter, + asyncMiddleware(getRunnerFromTokenValidator), + asyncMiddleware(requestRunnerJob) +) + +runnerJobsRouter.post('/jobs/:jobUUID/accept', + apiRateLimiter, + asyncMiddleware(runnerJobGetValidator), + acceptRunnerJobValidator, + asyncMiddleware(getRunnerFromTokenValidator), + asyncMiddleware(acceptRunnerJob) +) + +runnerJobsRouter.post('/jobs/:jobUUID/abort', + apiRateLimiter, + asyncMiddleware(jobOfRunnerGetValidatorFactory([ RunnerJobState.PROCESSING ])), + abortRunnerJobValidator, + asyncMiddleware(abortRunnerJob) +) + +runnerJobsRouter.post('/jobs/:jobUUID/update', + runnerJobUpdateVideoFiles, + apiRateLimiter, // Has to be after multer middleware to parse runner token + asyncMiddleware(jobOfRunnerGetValidatorFactory([ RunnerJobState.PROCESSING, RunnerJobState.COMPLETING, RunnerJobState.COMPLETED ])), + updateRunnerJobValidator, + asyncMiddleware(updateRunnerJobController) +) + +runnerJobsRouter.post('/jobs/:jobUUID/error', + asyncMiddleware(jobOfRunnerGetValidatorFactory([ RunnerJobState.PROCESSING ])), + errorRunnerJobValidator, + asyncMiddleware(errorRunnerJob) +) + +runnerJobsRouter.post('/jobs/:jobUUID/success', + postRunnerJobSuccessVideoFiles, + apiRateLimiter, // Has to be after multer middleware to parse runner token + asyncMiddleware(jobOfRunnerGetValidatorFactory([ RunnerJobState.PROCESSING ])), + successRunnerJobValidator, + asyncMiddleware(postRunnerJobSuccess) +) + +// --------------------------------------------------------------------------- +// Controllers for admins +// --------------------------------------------------------------------------- + +runnerJobsRouter.post('/jobs/:jobUUID/cancel', + authenticate, + ensureUserHasRight(UserRight.MANAGE_RUNNERS), + asyncMiddleware(runnerJobGetValidator), + cancelRunnerJobValidator, + asyncMiddleware(cancelRunnerJob) +) + +runnerJobsRouter.get('/jobs', + authenticate, + ensureUserHasRight(UserRight.MANAGE_RUNNERS), + paginationValidator, + runnerJobsSortValidator, + setDefaultSort, + setDefaultPagination, + listRunnerJobsValidator, + asyncMiddleware(listRunnerJobs) +) + +runnerJobsRouter.delete('/jobs/:jobUUID', + authenticate, + ensureUserHasRight(UserRight.MANAGE_RUNNERS), + asyncMiddleware(runnerJobGetValidator), + asyncMiddleware(deleteRunnerJob) +) + +// --------------------------------------------------------------------------- + +export { + runnerJobsRouter +} + +// --------------------------------------------------------------------------- + +// --------------------------------------------------------------------------- +// Controllers for runners +// --------------------------------------------------------------------------- + +async function requestRunnerJob (req: express.Request, res: express.Response) { + const runner = res.locals.runner + const availableJobs = await RunnerJobModel.listAvailableJobs() + + logger.debug('Runner %s requests for a job.', runner.name, { availableJobs, ...lTags(runner.name) }) + + const result: RequestRunnerJobResult = { + availableJobs: availableJobs.map(j => ({ + uuid: j.uuid, + type: j.type, + payload: j.payload + })) + } + + updateLastRunnerContact(req, runner) + + return res.json(result) +} + +async function acceptRunnerJob (req: express.Request, res: express.Response) { + const runner = res.locals.runner + const runnerJob = res.locals.runnerJob + + const newRunnerJob = await retryTransactionWrapper(() => { + return sequelizeTypescript.transaction(async transaction => { + await runnerJob.reload({ transaction }) + + if (runnerJob.state !== RunnerJobState.PENDING) { + res.fail({ + type: ServerErrorCode.RUNNER_JOB_NOT_IN_PENDING_STATE, + message: 'This job is not in pending state anymore', + status: HttpStatusCode.CONFLICT_409 + }) + + return undefined + } + + runnerJob.state = RunnerJobState.PROCESSING + runnerJob.processingJobToken = generateRunnerJobToken() + runnerJob.startedAt = new Date() + runnerJob.runnerId = runner.id + + return runnerJob.save({ transaction }) + }) + }) + if (!newRunnerJob) return + + newRunnerJob.Runner = runner as RunnerModel + + const result: AcceptRunnerJobResult = { + job: { + ...newRunnerJob.toFormattedJSON(), + + jobToken: newRunnerJob.processingJobToken + } + } + + updateLastRunnerContact(req, runner) + + logger.info( + 'Remote runner %s has accepted job %s (%s)', runner.name, runnerJob.uuid, runnerJob.type, + lTags(runner.name, runnerJob.uuid, runnerJob.type) + ) + + return res.json(result) +} + +async function abortRunnerJob (req: express.Request, res: express.Response) { + const runnerJob = res.locals.runnerJob + const runner = runnerJob.Runner + const body: AbortRunnerJobBody = req.body + + logger.info( + 'Remote runner %s is aborting job %s (%s)', runner.name, runnerJob.uuid, runnerJob.type, + { reason: body.reason, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) } + ) + + const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) + await new RunnerJobHandler().abort({ runnerJob }) + + updateLastRunnerContact(req, runnerJob.Runner) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +async function errorRunnerJob (req: express.Request, res: express.Response) { + const runnerJob = res.locals.runnerJob + const runner = runnerJob.Runner + const body: ErrorRunnerJobBody = req.body + + runnerJob.failures += 1 + + logger.error( + 'Remote runner %s had an error with job %s (%s)', runner.name, runnerJob.uuid, runnerJob.type, + { errorMessage: body.message, totalFailures: runnerJob.failures, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) } + ) + + const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) + await new RunnerJobHandler().error({ runnerJob, message: body.message }) + + updateLastRunnerContact(req, runnerJob.Runner) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +// --------------------------------------------------------------------------- + +const jobUpdateBuilders: { + [id in RunnerJobType]?: (payload: RunnerJobUpdatePayload, files?: UploadFiles) => RunnerJobUpdatePayload +} = { + 'live-rtmp-hls-transcoding': (payload: LiveRTMPHLSTranscodingUpdatePayload, files) => { + return { + ...payload, + + masterPlaylistFile: files['payload[masterPlaylistFile]']?.[0].path, + resolutionPlaylistFile: files['payload[resolutionPlaylistFile]']?.[0].path, + videoChunkFile: files['payload[videoChunkFile]']?.[0].path + } + } +} + +async function updateRunnerJobController (req: express.Request, res: express.Response) { + const runnerJob = res.locals.runnerJob + const runner = runnerJob.Runner + const body: RunnerJobUpdateBody = req.body + + if (runnerJob.state === RunnerJobState.COMPLETING || runnerJob.state === RunnerJobState.COMPLETED) { + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) + } + + const payloadBuilder = jobUpdateBuilders[runnerJob.type] + const updatePayload = payloadBuilder + ? payloadBuilder(body.payload, req.files as UploadFiles) + : undefined + + logger.debug( + 'Remote runner %s is updating job %s (%s)', runnerJob.Runner.name, runnerJob.uuid, runnerJob.type, + { body, updatePayload, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) } + ) + + const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) + await new RunnerJobHandler().update({ + runnerJob, + progress: req.body.progress, + updatePayload + }) + + updateLastRunnerContact(req, runnerJob.Runner) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +// --------------------------------------------------------------------------- + +const jobSuccessPayloadBuilders: { + [id in RunnerJobType]: (payload: RunnerJobSuccessPayload, files?: UploadFiles) => RunnerJobSuccessPayload +} = { + 'vod-web-video-transcoding': (payload: VODWebVideoTranscodingSuccess, files) => { + return { + ...payload, + + videoFile: files['payload[videoFile]'][0].path + } + }, + + 'vod-hls-transcoding': (payload: VODHLSTranscodingSuccess, files) => { + return { + ...payload, + + videoFile: files['payload[videoFile]'][0].path, + resolutionPlaylistFile: files['payload[resolutionPlaylistFile]'][0].path + } + }, + + 'vod-audio-merge-transcoding': (payload: VODAudioMergeTranscodingSuccess, files) => { + return { + ...payload, + + videoFile: files['payload[videoFile]'][0].path + } + }, + + 'video-studio-transcoding': (payload: VideoStudioTranscodingSuccess, files) => { + return { + ...payload, + + videoFile: files['payload[videoFile]'][0].path + } + }, + + 'live-rtmp-hls-transcoding': () => ({}) +} + +async function postRunnerJobSuccess (req: express.Request, res: express.Response) { + const runnerJob = res.locals.runnerJob + const runner = runnerJob.Runner + const body: RunnerJobSuccessBody = req.body + + const resultPayload = jobSuccessPayloadBuilders[runnerJob.type](body.payload, req.files as UploadFiles) + + logger.info( + 'Remote runner %s is sending success result for job %s (%s)', runnerJob.Runner.name, runnerJob.uuid, runnerJob.type, + { resultPayload, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) } + ) + + const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) + await new RunnerJobHandler().complete({ runnerJob, resultPayload }) + + updateLastRunnerContact(req, runnerJob.Runner) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +// --------------------------------------------------------------------------- +// Controllers for admins +// --------------------------------------------------------------------------- + +async function cancelRunnerJob (req: express.Request, res: express.Response) { + const runnerJob = res.locals.runnerJob + + logger.info('Cancelling job %s (%s)', runnerJob.uuid, runnerJob.type, lTags(runnerJob.uuid, runnerJob.type)) + + const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) + await new RunnerJobHandler().cancel({ runnerJob }) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +async function deleteRunnerJob (req: express.Request, res: express.Response) { + const runnerJob = res.locals.runnerJob + + logger.info('Deleting job %s (%s)', runnerJob.uuid, runnerJob.type, lTags(runnerJob.uuid, runnerJob.type)) + + if (runnerJobCanBeCancelled(runnerJob)) { + const RunnerJobHandler = getRunnerJobHandlerClass(runnerJob) + await new RunnerJobHandler().cancel({ runnerJob }) + } + + await runnerJob.destroy() + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +async function listRunnerJobs (req: express.Request, res: express.Response) { + const query: ListRunnerJobsQuery = req.query + + const resultList = await RunnerJobModel.listForApi({ + start: query.start, + count: query.count, + sort: query.sort, + search: query.search, + stateOneOf: query.stateOneOf + }) + + return res.json({ + total: resultList.total, + data: resultList.data.map(d => d.toFormattedAdminJSON()) + }) +} diff --git a/server/server/controllers/api/runners/manage-runners.ts b/server/server/controllers/api/runners/manage-runners.ts new file mode 100644 index 000000000..fb2199bea --- /dev/null +++ b/server/server/controllers/api/runners/manage-runners.ts @@ -0,0 +1,116 @@ +import express from 'express' +import { HttpStatusCode, ListRunnersQuery, RegisterRunnerBody, UserRight } from '@peertube/peertube-models' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { generateRunnerToken } from '@server/helpers/token-generator.js' +import { + apiRateLimiter, + asyncMiddleware, + authenticate, + ensureUserHasRight, + paginationValidator, + runnersSortValidator, + setDefaultPagination, + setDefaultSort +} from '@server/middlewares/index.js' +import { + deleteRunnerValidator, + getRunnerFromTokenValidator, + registerRunnerValidator +} from '@server/middlewares/validators/runners/index.js' +import { RunnerModel } from '@server/models/runner/runner.js' + +const lTags = loggerTagsFactory('api', 'runner') + +const manageRunnersRouter = express.Router() + +manageRunnersRouter.post('/register', + apiRateLimiter, + asyncMiddleware(registerRunnerValidator), + asyncMiddleware(registerRunner) +) +manageRunnersRouter.post('/unregister', + apiRateLimiter, + asyncMiddleware(getRunnerFromTokenValidator), + asyncMiddleware(unregisterRunner) +) + +manageRunnersRouter.delete('/:runnerId', + apiRateLimiter, + authenticate, + ensureUserHasRight(UserRight.MANAGE_RUNNERS), + asyncMiddleware(deleteRunnerValidator), + asyncMiddleware(deleteRunner) +) + +manageRunnersRouter.get('/', + apiRateLimiter, + authenticate, + ensureUserHasRight(UserRight.MANAGE_RUNNERS), + paginationValidator, + runnersSortValidator, + setDefaultSort, + setDefaultPagination, + asyncMiddleware(listRunners) +) + +// --------------------------------------------------------------------------- + +export { + manageRunnersRouter +} + +// --------------------------------------------------------------------------- + +async function registerRunner (req: express.Request, res: express.Response) { + const body: RegisterRunnerBody = req.body + + const runnerToken = generateRunnerToken() + + const runner = new RunnerModel({ + runnerToken, + name: body.name, + description: body.description, + lastContact: new Date(), + ip: req.ip, + runnerRegistrationTokenId: res.locals.runnerRegistrationToken.id + }) + + await runner.save() + + logger.info('Registered new runner %s', runner.name, { ...lTags(runner.name) }) + + return res.json({ id: runner.id, runnerToken }) +} +async function unregisterRunner (req: express.Request, res: express.Response) { + const runner = res.locals.runner + await runner.destroy() + + logger.info('Unregistered runner %s', runner.name, { ...lTags(runner.name) }) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +async function deleteRunner (req: express.Request, res: express.Response) { + const runner = res.locals.runner + + await runner.destroy() + + logger.info('Deleted runner %s', runner.name, { ...lTags(runner.name) }) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +async function listRunners (req: express.Request, res: express.Response) { + const query: ListRunnersQuery = req.query + + const resultList = await RunnerModel.listForApi({ + start: query.start, + count: query.count, + sort: query.sort + }) + + return res.json({ + total: resultList.total, + data: resultList.data.map(d => d.toFormattedJSON()) + }) +} diff --git a/server/server/controllers/api/runners/registration-tokens.ts b/server/server/controllers/api/runners/registration-tokens.ts new file mode 100644 index 000000000..462c489bb --- /dev/null +++ b/server/server/controllers/api/runners/registration-tokens.ts @@ -0,0 +1,91 @@ +import express from 'express' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { generateRunnerRegistrationToken } from '@server/helpers/token-generator.js' +import { + apiRateLimiter, + asyncMiddleware, + authenticate, + ensureUserHasRight, + paginationValidator, + runnerRegistrationTokensSortValidator, + setDefaultPagination, + setDefaultSort +} from '@server/middlewares/index.js' +import { deleteRegistrationTokenValidator } from '@server/middlewares/validators/runners/index.js' +import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token.js' +import { HttpStatusCode, ListRunnerRegistrationTokensQuery, UserRight } from '@peertube/peertube-models' + +const lTags = loggerTagsFactory('api', 'runner') + +const runnerRegistrationTokensRouter = express.Router() + +runnerRegistrationTokensRouter.post('/registration-tokens/generate', + apiRateLimiter, + authenticate, + ensureUserHasRight(UserRight.MANAGE_RUNNERS), + asyncMiddleware(generateRegistrationToken) +) + +runnerRegistrationTokensRouter.delete('/registration-tokens/:id', + apiRateLimiter, + authenticate, + ensureUserHasRight(UserRight.MANAGE_RUNNERS), + asyncMiddleware(deleteRegistrationTokenValidator), + asyncMiddleware(deleteRegistrationToken) +) + +runnerRegistrationTokensRouter.get('/registration-tokens', + apiRateLimiter, + authenticate, + ensureUserHasRight(UserRight.MANAGE_RUNNERS), + paginationValidator, + runnerRegistrationTokensSortValidator, + setDefaultSort, + setDefaultPagination, + asyncMiddleware(listRegistrationTokens) +) + +// --------------------------------------------------------------------------- + +export { + runnerRegistrationTokensRouter +} + +// --------------------------------------------------------------------------- + +async function generateRegistrationToken (req: express.Request, res: express.Response) { + logger.info('Generating new runner registration token.', lTags()) + + const registrationToken = new RunnerRegistrationTokenModel({ + registrationToken: generateRunnerRegistrationToken() + }) + + await registrationToken.save() + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +async function deleteRegistrationToken (req: express.Request, res: express.Response) { + logger.info('Removing runner registration token.', lTags()) + + const runnerRegistrationToken = res.locals.runnerRegistrationToken + + await runnerRegistrationToken.destroy() + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +async function listRegistrationTokens (req: express.Request, res: express.Response) { + const query: ListRunnerRegistrationTokensQuery = req.query + + const resultList = await RunnerRegistrationTokenModel.listForApi({ + start: query.start, + count: query.count, + sort: query.sort + }) + + return res.json({ + total: resultList.total, + data: resultList.data.map(d => d.toFormattedJSON()) + }) +} diff --git a/server/server/controllers/api/search/index.ts b/server/server/controllers/api/search/index.ts new file mode 100644 index 000000000..f1a1f493f --- /dev/null +++ b/server/server/controllers/api/search/index.ts @@ -0,0 +1,19 @@ +import express from 'express' +import { apiRateLimiter } from '@server/middlewares/index.js' +import { searchChannelsRouter } from './search-video-channels.js' +import { searchPlaylistsRouter } from './search-video-playlists.js' +import { searchVideosRouter } from './search-videos.js' + +const searchRouter = express.Router() + +searchRouter.use(apiRateLimiter) + +searchRouter.use('/', searchVideosRouter) +searchRouter.use('/', searchChannelsRouter) +searchRouter.use('/', searchPlaylistsRouter) + +// --------------------------------------------------------------------------- + +export { + searchRouter +} diff --git a/server/server/controllers/api/search/search-video-channels.ts b/server/server/controllers/api/search/search-video-channels.ts new file mode 100644 index 000000000..cccc3b3dc --- /dev/null +++ b/server/server/controllers/api/search/search-video-channels.ts @@ -0,0 +1,151 @@ +import express from 'express' +import { sanitizeUrl } from '@server/helpers/core-utils.js' +import { pickSearchChannelQuery } from '@server/helpers/query.js' +import { doJSONRequest } from '@server/helpers/requests.js' +import { CONFIG } from '@server/initializers/config.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { findLatestAPRedirection } from '@server/lib/activitypub/activity.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { buildMutedForSearchIndex, isSearchIndexSearch, isURISearch } from '@server/lib/search.js' +import { getServerActor } from '@server/models/application/application.js' +import { HttpStatusCode, ResultList, VideoChannel, VideoChannelsSearchQueryAfterSanitize } from '@peertube/peertube-models' +import { isUserAbleToSearchRemoteURI } from '../../../helpers/express-utils.js' +import { logger } from '../../../helpers/logger.js' +import { getFormattedObjects } from '../../../helpers/utils.js' +import { getOrCreateAPActor, loadActorUrlOrGetFromWebfinger } from '../../../lib/activitypub/actors/index.js' +import { + asyncMiddleware, + openapiOperationDoc, + optionalAuthenticate, + paginationValidator, + setDefaultPagination, + setDefaultSearchSort, + videoChannelsListSearchValidator, + videoChannelsSearchSortValidator +} from '../../../middlewares/index.js' +import { VideoChannelModel } from '../../../models/video/video-channel.js' +import { MChannelAccountDefault } from '../../../types/models/index.js' +import { searchLocalUrl } from './shared/index.js' + +const searchChannelsRouter = express.Router() + +searchChannelsRouter.get('/video-channels', + openapiOperationDoc({ operationId: 'searchChannels' }), + paginationValidator, + setDefaultPagination, + videoChannelsSearchSortValidator, + setDefaultSearchSort, + optionalAuthenticate, + videoChannelsListSearchValidator, + asyncMiddleware(searchVideoChannels) +) + +// --------------------------------------------------------------------------- + +export { searchChannelsRouter } + +// --------------------------------------------------------------------------- + +function searchVideoChannels (req: express.Request, res: express.Response) { + const query = pickSearchChannelQuery(req.query) + const search = query.search || '' + + const parts = search.split('@') + + // Handle strings like @toto@example.com + if (parts.length === 3 && parts[0].length === 0) parts.shift() + const isWebfingerSearch = parts.length === 2 && parts.every(p => p && !p.includes(' ')) + + if (isURISearch(search) || isWebfingerSearch) return searchVideoChannelURI(search, res) + + // @username -> username to search in DB + if (search.startsWith('@')) query.search = search.replace(/^@/, '') + + if (isSearchIndexSearch(query)) { + return searchVideoChannelsIndex(query, res) + } + + return searchVideoChannelsDB(query, res) +} + +async function searchVideoChannelsIndex (query: VideoChannelsSearchQueryAfterSanitize, res: express.Response) { + const result = await buildMutedForSearchIndex(res) + + const body = await Hooks.wrapObject(Object.assign(query, result), 'filter:api.search.video-channels.index.list.params') + + const url = sanitizeUrl(CONFIG.SEARCH.SEARCH_INDEX.URL) + '/api/v1/search/video-channels' + + try { + logger.debug('Doing video channels search index request on %s.', url, { body }) + + const { body: searchIndexResult } = await doJSONRequest>(url, { method: 'POST', json: body }) + const jsonResult = await Hooks.wrapObject(searchIndexResult, 'filter:api.search.video-channels.index.list.result') + + return res.json(jsonResult) + } catch (err) { + logger.warn('Cannot use search index to make video channels search.', { err }) + + return res.fail({ + status: HttpStatusCode.INTERNAL_SERVER_ERROR_500, + message: 'Cannot use search index to make video channels search' + }) + } +} + +async function searchVideoChannelsDB (query: VideoChannelsSearchQueryAfterSanitize, res: express.Response) { + const serverActor = await getServerActor() + + const apiOptions = await Hooks.wrapObject({ + ...query, + + actorId: serverActor.id + }, 'filter:api.search.video-channels.local.list.params') + + const resultList = await Hooks.wrapPromiseFun( + VideoChannelModel.searchForApi, + apiOptions, + 'filter:api.search.video-channels.local.list.result' + ) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function searchVideoChannelURI (search: string, res: express.Response) { + let videoChannel: MChannelAccountDefault + let uri = search + + if (!isURISearch(search)) { + try { + uri = await loadActorUrlOrGetFromWebfinger(search) + } catch (err) { + logger.warn('Cannot load actor URL or get from webfinger.', { search, err }) + + return res.json({ total: 0, data: [] }) + } + } + + if (isUserAbleToSearchRemoteURI(res)) { + try { + const latestUri = await findLatestAPRedirection(uri) + + const actor = await getOrCreateAPActor(latestUri, 'all', true, true) + videoChannel = actor.VideoChannel + } catch (err) { + logger.info('Cannot search remote video channel %s.', uri, { err }) + } + } else { + videoChannel = await searchLocalUrl(sanitizeLocalUrl(uri), url => VideoChannelModel.loadByUrlAndPopulateAccount(url)) + } + + return res.json({ + total: videoChannel ? 1 : 0, + data: videoChannel ? [ videoChannel.toFormattedJSON() ] : [] + }) +} + +function sanitizeLocalUrl (url: string) { + if (!url) return '' + + // Handle alternative channel URLs + return url.replace(new RegExp('^' + WEBSERVER.URL + '/c/'), WEBSERVER.URL + '/video-channels/') +} diff --git a/server/server/controllers/api/search/search-video-playlists.ts b/server/server/controllers/api/search/search-video-playlists.ts new file mode 100644 index 000000000..4c0f98cf2 --- /dev/null +++ b/server/server/controllers/api/search/search-video-playlists.ts @@ -0,0 +1,131 @@ +import express from 'express' +import { sanitizeUrl } from '@server/helpers/core-utils.js' +import { isUserAbleToSearchRemoteURI } from '@server/helpers/express-utils.js' +import { logger } from '@server/helpers/logger.js' +import { pickSearchPlaylistQuery } from '@server/helpers/query.js' +import { doJSONRequest } from '@server/helpers/requests.js' +import { getFormattedObjects } from '@server/helpers/utils.js' +import { CONFIG } from '@server/initializers/config.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { findLatestAPRedirection } from '@server/lib/activitypub/activity.js' +import { getOrCreateAPVideoPlaylist } from '@server/lib/activitypub/playlists/get.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { buildMutedForSearchIndex, isSearchIndexSearch, isURISearch } from '@server/lib/search.js' +import { getServerActor } from '@server/models/application/application.js' +import { VideoPlaylistModel } from '@server/models/video/video-playlist.js' +import { MVideoPlaylistFullSummary } from '@server/types/models/index.js' +import { HttpStatusCode, ResultList, VideoPlaylist, VideoPlaylistsSearchQueryAfterSanitize } from '@peertube/peertube-models' +import { + asyncMiddleware, + openapiOperationDoc, + optionalAuthenticate, + paginationValidator, + setDefaultPagination, + setDefaultSearchSort, + videoPlaylistsListSearchValidator, + videoPlaylistsSearchSortValidator +} from '../../../middlewares/index.js' +import { searchLocalUrl } from './shared/index.js' + +const searchPlaylistsRouter = express.Router() + +searchPlaylistsRouter.get('/video-playlists', + openapiOperationDoc({ operationId: 'searchPlaylists' }), + paginationValidator, + setDefaultPagination, + videoPlaylistsSearchSortValidator, + setDefaultSearchSort, + optionalAuthenticate, + videoPlaylistsListSearchValidator, + asyncMiddleware(searchVideoPlaylists) +) + +// --------------------------------------------------------------------------- + +export { searchPlaylistsRouter } + +// --------------------------------------------------------------------------- + +function searchVideoPlaylists (req: express.Request, res: express.Response) { + const query = pickSearchPlaylistQuery(req.query) + const search = query.search + + if (isURISearch(search)) return searchVideoPlaylistsURI(search, res) + + if (isSearchIndexSearch(query)) { + return searchVideoPlaylistsIndex(query, res) + } + + return searchVideoPlaylistsDB(query, res) +} + +async function searchVideoPlaylistsIndex (query: VideoPlaylistsSearchQueryAfterSanitize, res: express.Response) { + const result = await buildMutedForSearchIndex(res) + + const body = await Hooks.wrapObject(Object.assign(query, result), 'filter:api.search.video-playlists.index.list.params') + + const url = sanitizeUrl(CONFIG.SEARCH.SEARCH_INDEX.URL) + '/api/v1/search/video-playlists' + + try { + logger.debug('Doing video playlists search index request on %s.', url, { body }) + + const { body: searchIndexResult } = await doJSONRequest>(url, { method: 'POST', json: body }) + const jsonResult = await Hooks.wrapObject(searchIndexResult, 'filter:api.search.video-playlists.index.list.result') + + return res.json(jsonResult) + } catch (err) { + logger.warn('Cannot use search index to make video playlists search.', { err }) + + return res.fail({ + status: HttpStatusCode.INTERNAL_SERVER_ERROR_500, + message: 'Cannot use search index to make video playlists search' + }) + } +} + +async function searchVideoPlaylistsDB (query: VideoPlaylistsSearchQueryAfterSanitize, res: express.Response) { + const serverActor = await getServerActor() + + const apiOptions = await Hooks.wrapObject({ + ...query, + + followerActorId: serverActor.id + }, 'filter:api.search.video-playlists.local.list.params') + + const resultList = await Hooks.wrapPromiseFun( + VideoPlaylistModel.searchForApi, + apiOptions, + 'filter:api.search.video-playlists.local.list.result' + ) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function searchVideoPlaylistsURI (search: string, res: express.Response) { + let videoPlaylist: MVideoPlaylistFullSummary + + if (isUserAbleToSearchRemoteURI(res)) { + try { + const url = await findLatestAPRedirection(search) + + videoPlaylist = await getOrCreateAPVideoPlaylist(url) + } catch (err) { + logger.info('Cannot search remote video playlist %s.', search, { err }) + } + } else { + videoPlaylist = await searchLocalUrl(sanitizeLocalUrl(search), url => VideoPlaylistModel.loadByUrlWithAccountAndChannelSummary(url)) + } + + return res.json({ + total: videoPlaylist ? 1 : 0, + data: videoPlaylist ? [ videoPlaylist.toFormattedJSON() ] : [] + }) +} + +function sanitizeLocalUrl (url: string) { + if (!url) return '' + + // Handle alternative channel URLs + return url.replace(new RegExp('^' + WEBSERVER.URL + '/videos/watch/playlist/'), WEBSERVER.URL + '/video-playlists/') + .replace(new RegExp('^' + WEBSERVER.URL + '/w/p/'), WEBSERVER.URL + '/video-playlists/') +} diff --git a/server/server/controllers/api/search/search-videos.ts b/server/server/controllers/api/search/search-videos.ts new file mode 100644 index 000000000..a6700ed7d --- /dev/null +++ b/server/server/controllers/api/search/search-videos.ts @@ -0,0 +1,166 @@ +import express from 'express' +import { sanitizeUrl } from '@server/helpers/core-utils.js' +import { pickSearchVideoQuery } from '@server/helpers/query.js' +import { doJSONRequest } from '@server/helpers/requests.js' +import { CONFIG } from '@server/initializers/config.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { findLatestAPRedirection } from '@server/lib/activitypub/activity.js' +import { getOrCreateAPVideo } from '@server/lib/activitypub/videos/index.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { buildMutedForSearchIndex, isSearchIndexSearch, isURISearch } from '@server/lib/search.js' +import { getServerActor } from '@server/models/application/application.js' +import { HttpStatusCode, ResultList, Video, VideosSearchQueryAfterSanitize } from '@peertube/peertube-models' +import { buildNSFWFilter, isUserAbleToSearchRemoteURI } from '../../../helpers/express-utils.js' +import { logger } from '../../../helpers/logger.js' +import { getFormattedObjects } from '../../../helpers/utils.js' +import { + asyncMiddleware, + commonVideosFiltersValidator, + openapiOperationDoc, + optionalAuthenticate, + paginationValidator, + setDefaultPagination, + setDefaultSearchSort, + videosSearchSortValidator, + videosSearchValidator +} from '../../../middlewares/index.js' +import { guessAdditionalAttributesFromQuery } from '../../../models/video/formatter/index.js' +import { VideoModel } from '../../../models/video/video.js' +import { MVideoAccountLightBlacklistAllFiles } from '../../../types/models/index.js' +import { searchLocalUrl } from './shared/index.js' + +const searchVideosRouter = express.Router() + +searchVideosRouter.get('/videos', + openapiOperationDoc({ operationId: 'searchVideos' }), + paginationValidator, + setDefaultPagination, + videosSearchSortValidator, + setDefaultSearchSort, + optionalAuthenticate, + commonVideosFiltersValidator, + videosSearchValidator, + asyncMiddleware(searchVideos) +) + +// --------------------------------------------------------------------------- + +export { searchVideosRouter } + +// --------------------------------------------------------------------------- + +function searchVideos (req: express.Request, res: express.Response) { + const query = pickSearchVideoQuery(req.query) + const search = query.search + + if (isURISearch(search)) { + return searchVideoURI(search, res) + } + + if (isSearchIndexSearch(query)) { + return searchVideosIndex(query, res) + } + + return searchVideosDB(query, res) +} + +async function searchVideosIndex (query: VideosSearchQueryAfterSanitize, res: express.Response) { + const result = await buildMutedForSearchIndex(res) + + let body = { ...query, ...result } + + // Use the default instance NSFW policy if not specified + if (!body.nsfw) { + const nsfwPolicy = res.locals.oauth + ? res.locals.oauth.token.User.nsfwPolicy + : CONFIG.INSTANCE.DEFAULT_NSFW_POLICY + + body.nsfw = nsfwPolicy === 'do_not_list' + ? 'false' + : 'both' + } + + body = await Hooks.wrapObject(body, 'filter:api.search.videos.index.list.params') + + const url = sanitizeUrl(CONFIG.SEARCH.SEARCH_INDEX.URL) + '/api/v1/search/videos' + + try { + logger.debug('Doing videos search index request on %s.', url, { body }) + + const { body: searchIndexResult } = await doJSONRequest>(url, { method: 'POST', json: body }) + const jsonResult = await Hooks.wrapObject(searchIndexResult, 'filter:api.search.videos.index.list.result') + + return res.json(jsonResult) + } catch (err) { + logger.warn('Cannot use search index to make video search.', { err }) + + return res.fail({ + status: HttpStatusCode.INTERNAL_SERVER_ERROR_500, + message: 'Cannot use search index to make video search' + }) + } +} + +async function searchVideosDB (query: VideosSearchQueryAfterSanitize, res: express.Response) { + const serverActor = await getServerActor() + + const apiOptions = await Hooks.wrapObject({ + ...query, + + displayOnlyForFollower: { + actorId: serverActor.id, + orLocalVideos: true + }, + + nsfw: buildNSFWFilter(res, query.nsfw), + user: res.locals.oauth + ? res.locals.oauth.token.User + : undefined + }, 'filter:api.search.videos.local.list.params') + + const resultList = await Hooks.wrapPromiseFun( + VideoModel.searchAndPopulateAccountAndServer, + apiOptions, + 'filter:api.search.videos.local.list.result' + ) + + return res.json(getFormattedObjects(resultList.data, resultList.total, guessAdditionalAttributesFromQuery(query))) +} + +async function searchVideoURI (url: string, res: express.Response) { + let video: MVideoAccountLightBlacklistAllFiles + + // Check if we can fetch a remote video with the URL + if (isUserAbleToSearchRemoteURI(res)) { + try { + const syncParam = { + rates: false, + shares: false, + comments: false, + refreshVideo: false + } + + const result = await getOrCreateAPVideo({ + videoObject: await findLatestAPRedirection(url), + syncParam + }) + video = result ? result.video : undefined + } catch (err) { + logger.info('Cannot search remote video %s.', url, { err }) + } + } else { + video = await searchLocalUrl(sanitizeLocalUrl(url), url => VideoModel.loadByUrlAndPopulateAccount(url)) + } + + return res.json({ + total: video ? 1 : 0, + data: video ? [ video.toFormattedJSON() ] : [] + }) +} + +function sanitizeLocalUrl (url: string) { + if (!url) return '' + + // Handle alternative video URLs + return url.replace(new RegExp('^' + WEBSERVER.URL + '/w/'), WEBSERVER.URL + '/videos/watch/') +} diff --git a/server/server/controllers/api/search/shared/index.ts b/server/server/controllers/api/search/shared/index.ts new file mode 100644 index 000000000..84814912c --- /dev/null +++ b/server/server/controllers/api/search/shared/index.ts @@ -0,0 +1 @@ +export * from './utils.js' diff --git a/server/controllers/api/search/shared/utils.ts b/server/server/controllers/api/search/shared/utils.ts similarity index 100% rename from server/controllers/api/search/shared/utils.ts rename to server/server/controllers/api/search/shared/utils.ts diff --git a/server/server/controllers/api/server/contact.ts b/server/server/controllers/api/server/contact.ts new file mode 100644 index 000000000..53e73fa2b --- /dev/null +++ b/server/server/controllers/api/server/contact.ts @@ -0,0 +1,33 @@ +import express from 'express' +import { ContactForm, HttpStatusCode } from '@peertube/peertube-models' +import { logger } from '@server/helpers/logger.js' +import { Emailer } from '../../../lib/emailer.js' +import { Redis } from '../../../lib/redis.js' +import { asyncMiddleware, contactAdministratorValidator } from '../../../middlewares/index.js' + +const contactRouter = express.Router() + +contactRouter.post('/contact', + asyncMiddleware(contactAdministratorValidator), + asyncMiddleware(contactAdministrator) +) + +async function contactAdministrator (req: express.Request, res: express.Response) { + const data = req.body as ContactForm + + Emailer.Instance.addContactFormJob(data.fromEmail, data.fromName, data.subject, data.body) + + try { + await Redis.Instance.setContactFormIp(req.ip) + } catch (err) { + logger.error(err) + } + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +// --------------------------------------------------------------------------- + +export { + contactRouter +} diff --git a/server/server/controllers/api/server/debug.ts b/server/server/controllers/api/server/debug.ts new file mode 100644 index 000000000..338a1214f --- /dev/null +++ b/server/server/controllers/api/server/debug.ts @@ -0,0 +1,54 @@ +import express from 'express' +import { Debug, HttpStatusCode, SendDebugCommand, UserRight } from '@peertube/peertube-models' +import { InboxManager } from '@server/lib/activitypub/inbox-manager.js' +import { RemoveDanglingResumableUploadsScheduler } from '@server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.js' +import { UpdateVideosScheduler } from '@server/lib/schedulers/update-videos-scheduler.js' +import { VideoChannelSyncLatestScheduler } from '@server/lib/schedulers/video-channel-sync-latest-scheduler.js' +import { VideoViewsBufferScheduler } from '@server/lib/schedulers/video-views-buffer-scheduler.js' +import { VideoViewsManager } from '@server/lib/views/video-views-manager.js' +import { authenticate, ensureUserHasRight } from '../../../middlewares/index.js' + +const debugRouter = express.Router() + +debugRouter.get('/debug', + authenticate, + ensureUserHasRight(UserRight.MANAGE_DEBUG), + getDebug +) + +debugRouter.post('/debug/run-command', + authenticate, + ensureUserHasRight(UserRight.MANAGE_DEBUG), + runCommand +) + +// --------------------------------------------------------------------------- + +export { + debugRouter +} + +// --------------------------------------------------------------------------- + +function getDebug (req: express.Request, res: express.Response) { + return res.json({ + ip: req.ip, + activityPubMessagesWaiting: InboxManager.Instance.getActivityPubMessagesWaiting() + } as Debug) +} + +async function runCommand (req: express.Request, res: express.Response) { + const body: SendDebugCommand = req.body + + const processors: { [id in SendDebugCommand['command']]: () => Promise } = { + 'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(), + 'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(), + 'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats(), + 'process-update-videos-scheduler': () => UpdateVideosScheduler.Instance.execute(), + 'process-video-channel-sync-latest': () => VideoChannelSyncLatestScheduler.Instance.execute() + } + + await processors[body.command]() + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} diff --git a/server/server/controllers/api/server/follows.ts b/server/server/controllers/api/server/follows.ts new file mode 100644 index 000000000..d7f84053a --- /dev/null +++ b/server/server/controllers/api/server/follows.ts @@ -0,0 +1,212 @@ +import express from 'express' +import { HttpStatusCode, ServerFollowCreate, UserRight } from '@peertube/peertube-models' +import { getServerActor } from '@server/models/application/application.js' +import { logger } from '../../../helpers/logger.js' +import { getFormattedObjects } from '../../../helpers/utils.js' +import { SERVER_ACTOR_NAME } from '../../../initializers/constants.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { autoFollowBackIfNeeded } from '../../../lib/activitypub/follow.js' +import { sendAccept, sendReject, sendUndoFollow } from '../../../lib/activitypub/send/index.js' +import { JobQueue } from '../../../lib/job-queue/index.js' +import { removeRedundanciesOfServer } from '../../../lib/redundancy.js' +import { + asyncMiddleware, + authenticate, + ensureUserHasRight, + paginationValidator, + setBodyHostsPort, + setDefaultPagination, + setDefaultSort +} from '../../../middlewares/index.js' +import { + acceptFollowerValidator, + followValidator, + getFollowerValidator, + instanceFollowersSortValidator, + instanceFollowingSortValidator, + listFollowsValidator, + rejectFollowerValidator, + removeFollowingValidator +} from '../../../middlewares/validators/index.js' +import { ActorFollowModel } from '../../../models/actor/actor-follow.js' + +const serverFollowsRouter = express.Router() +serverFollowsRouter.get('/following', + listFollowsValidator, + paginationValidator, + instanceFollowingSortValidator, + setDefaultSort, + setDefaultPagination, + asyncMiddleware(listFollowing) +) + +serverFollowsRouter.post('/following', + authenticate, + ensureUserHasRight(UserRight.MANAGE_SERVER_FOLLOW), + followValidator, + setBodyHostsPort, + asyncMiddleware(addFollow) +) + +serverFollowsRouter.delete('/following/:hostOrHandle', + authenticate, + ensureUserHasRight(UserRight.MANAGE_SERVER_FOLLOW), + asyncMiddleware(removeFollowingValidator), + asyncMiddleware(removeFollowing) +) + +serverFollowsRouter.get('/followers', + listFollowsValidator, + paginationValidator, + instanceFollowersSortValidator, + setDefaultSort, + setDefaultPagination, + asyncMiddleware(listFollowers) +) + +serverFollowsRouter.delete('/followers/:nameWithHost', + authenticate, + ensureUserHasRight(UserRight.MANAGE_SERVER_FOLLOW), + asyncMiddleware(getFollowerValidator), + asyncMiddleware(removeFollower) +) + +serverFollowsRouter.post('/followers/:nameWithHost/reject', + authenticate, + ensureUserHasRight(UserRight.MANAGE_SERVER_FOLLOW), + asyncMiddleware(getFollowerValidator), + rejectFollowerValidator, + asyncMiddleware(rejectFollower) +) + +serverFollowsRouter.post('/followers/:nameWithHost/accept', + authenticate, + ensureUserHasRight(UserRight.MANAGE_SERVER_FOLLOW), + asyncMiddleware(getFollowerValidator), + acceptFollowerValidator, + asyncMiddleware(acceptFollower) +) + +// --------------------------------------------------------------------------- + +export { + serverFollowsRouter +} + +// --------------------------------------------------------------------------- + +async function listFollowing (req: express.Request, res: express.Response) { + const serverActor = await getServerActor() + const resultList = await ActorFollowModel.listInstanceFollowingForApi({ + followerId: serverActor.id, + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + search: req.query.search, + actorType: req.query.actorType, + state: req.query.state + }) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function listFollowers (req: express.Request, res: express.Response) { + const serverActor = await getServerActor() + const resultList = await ActorFollowModel.listFollowersForApi({ + actorIds: [ serverActor.id ], + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + search: req.query.search, + actorType: req.query.actorType, + state: req.query.state + }) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function addFollow (req: express.Request, res: express.Response) { + const { hosts, handles } = req.body as ServerFollowCreate + const follower = await getServerActor() + + for (const host of hosts) { + const payload = { + host, + name: SERVER_ACTOR_NAME, + followerActorId: follower.id + } + + JobQueue.Instance.createJobAsync({ type: 'activitypub-follow', payload }) + } + + for (const handle of handles) { + const [ name, host ] = handle.split('@') + + const payload = { + host, + name, + followerActorId: follower.id + } + + JobQueue.Instance.createJobAsync({ type: 'activitypub-follow', payload }) + } + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function removeFollowing (req: express.Request, res: express.Response) { + const follow = res.locals.follow + + await sequelizeTypescript.transaction(async t => { + if (follow.state === 'accepted') sendUndoFollow(follow, t) + + // Disable redundancy on unfollowed instances + const server = follow.ActorFollowing.Server + server.redundancyAllowed = false + await server.save({ transaction: t }) + + // Async, could be long + removeRedundanciesOfServer(server.id) + .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, { err })) + + await follow.destroy({ transaction: t }) + }) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function rejectFollower (req: express.Request, res: express.Response) { + const follow = res.locals.follow + + follow.state = 'rejected' + await follow.save() + + sendReject(follow.url, follow.ActorFollower, follow.ActorFollowing) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function removeFollower (req: express.Request, res: express.Response) { + const follow = res.locals.follow + + if (follow.state === 'accepted' || follow.state === 'pending') { + sendReject(follow.url, follow.ActorFollower, follow.ActorFollowing) + } + + await follow.destroy() + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function acceptFollower (req: express.Request, res: express.Response) { + const follow = res.locals.follow + + sendAccept(follow) + + follow.state = 'accepted' + await follow.save() + + await autoFollowBackIfNeeded(follow) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} diff --git a/server/server/controllers/api/server/index.ts b/server/server/controllers/api/server/index.ts new file mode 100644 index 000000000..e3df8889b --- /dev/null +++ b/server/server/controllers/api/server/index.ts @@ -0,0 +1,27 @@ +import express from 'express' +import { apiRateLimiter } from '@server/middlewares/index.js' +import { contactRouter } from './contact.js' +import { debugRouter } from './debug.js' +import { serverFollowsRouter } from './follows.js' +import { logsRouter } from './logs.js' +import { serverRedundancyRouter } from './redundancy.js' +import { serverBlocklistRouter } from './server-blocklist.js' +import { statsRouter } from './stats.js' + +const serverRouter = express.Router() + +serverRouter.use(apiRateLimiter) + +serverRouter.use('/', serverFollowsRouter) +serverRouter.use('/', serverRedundancyRouter) +serverRouter.use('/', statsRouter) +serverRouter.use('/', serverBlocklistRouter) +serverRouter.use('/', contactRouter) +serverRouter.use('/', logsRouter) +serverRouter.use('/', debugRouter) + +// --------------------------------------------------------------------------- + +export { + serverRouter +} diff --git a/server/server/controllers/api/server/logs.ts b/server/server/controllers/api/server/logs.ts new file mode 100644 index 000000000..1ebf8053c --- /dev/null +++ b/server/server/controllers/api/server/logs.ts @@ -0,0 +1,201 @@ +import express from 'express' +import { readdir, readFile } from 'fs/promises' +import { join } from 'path' +import { pick } from '@peertube/peertube-core-utils' +import { ClientLogCreate, HttpStatusCode, ServerLogLevel, UserRight } from '@peertube/peertube-models' +import { isArray } from '@server/helpers/custom-validators/misc.js' +import { logger, mtimeSortFilesDesc } from '@server/helpers/logger.js' +import { CONFIG } from '../../../initializers/config.js' +import { AUDIT_LOG_FILENAME, LOG_FILENAME, MAX_LOGS_OUTPUT_CHARACTERS } from '../../../initializers/constants.js' +import { asyncMiddleware, authenticate, buildRateLimiter, ensureUserHasRight, optionalAuthenticate } from '../../../middlewares/index.js' +import { createClientLogValidator, getAuditLogsValidator, getLogsValidator } from '../../../middlewares/validators/logs.js' + +const createClientLogRateLimiter = buildRateLimiter({ + windowMs: CONFIG.RATES_LIMIT.RECEIVE_CLIENT_LOG.WINDOW_MS, + max: CONFIG.RATES_LIMIT.RECEIVE_CLIENT_LOG.MAX +}) + +const logsRouter = express.Router() + +logsRouter.post('/logs/client', + createClientLogRateLimiter, + optionalAuthenticate, + createClientLogValidator, + createClientLog +) + +logsRouter.get('/logs', + authenticate, + ensureUserHasRight(UserRight.MANAGE_LOGS), + getLogsValidator, + asyncMiddleware(getLogs) +) + +logsRouter.get('/audit-logs', + authenticate, + ensureUserHasRight(UserRight.MANAGE_LOGS), + getAuditLogsValidator, + asyncMiddleware(getAuditLogs) +) + +// --------------------------------------------------------------------------- + +export { + logsRouter +} + +// --------------------------------------------------------------------------- + +function createClientLog (req: express.Request, res: express.Response) { + const logInfo = req.body as ClientLogCreate + + const meta = { + tags: [ 'client' ], + username: res.locals.oauth?.token?.User?.username, + + ...pick(logInfo, [ 'userAgent', 'stackTrace', 'meta', 'url' ]) + } + + logger.log(logInfo.level, `Client log: ${logInfo.message}`, meta) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +const auditLogNameFilter = generateLogNameFilter(AUDIT_LOG_FILENAME) +async function getAuditLogs (req: express.Request, res: express.Response) { + const output = await generateOutput({ + startDateQuery: req.query.startDate, + endDateQuery: req.query.endDate, + level: 'audit', + nameFilter: auditLogNameFilter + }) + + return res.json(output).end() +} + +const logNameFilter = generateLogNameFilter(LOG_FILENAME) +async function getLogs (req: express.Request, res: express.Response) { + const output = await generateOutput({ + startDateQuery: req.query.startDate, + endDateQuery: req.query.endDate, + level: req.query.level || 'info', + tagsOneOf: req.query.tagsOneOf, + nameFilter: logNameFilter + }) + + return res.json(output) +} + +async function generateOutput (options: { + startDateQuery: string + endDateQuery?: string + + level: ServerLogLevel + nameFilter: RegExp + tagsOneOf?: string[] +}) { + const { startDateQuery, level, nameFilter } = options + + const tagsOneOf = Array.isArray(options.tagsOneOf) && options.tagsOneOf.length !== 0 + ? new Set(options.tagsOneOf) + : undefined + + const logFiles = await readdir(CONFIG.STORAGE.LOG_DIR) + const sortedLogFiles = await mtimeSortFilesDesc(logFiles, CONFIG.STORAGE.LOG_DIR) + let currentSize = 0 + + const startDate = new Date(startDateQuery) + const endDate = options.endDateQuery ? new Date(options.endDateQuery) : new Date() + + let output: string[] = [] + + for (const meta of sortedLogFiles) { + if (nameFilter.exec(meta.file) === null) continue + + const path = join(CONFIG.STORAGE.LOG_DIR, meta.file) + logger.debug('Opening %s to fetch logs.', path) + + const result = await getOutputFromFile({ path, startDate, endDate, level, currentSize, tagsOneOf }) + if (!result.output) break + + output = result.output.concat(output) + currentSize = result.currentSize + + if (currentSize > MAX_LOGS_OUTPUT_CHARACTERS || (result.logTime && result.logTime < startDate.getTime())) break + } + + return output +} + +async function getOutputFromFile (options: { + path: string + startDate: Date + endDate: Date + level: ServerLogLevel + currentSize: number + tagsOneOf: Set +}) { + const { path, startDate, endDate, level, tagsOneOf } = options + + const startTime = startDate.getTime() + const endTime = endDate.getTime() + let currentSize = options.currentSize + + let logTime: number + + const logsLevel: { [ id in ServerLogLevel ]: number } = { + audit: -1, + debug: 0, + info: 1, + warn: 2, + error: 3 + } + + const content = await readFile(path) + const lines = content.toString().split('\n') + const output: any[] = [] + + for (let i = lines.length - 1; i >= 0; i--) { + const line = lines[i] + let log: any + + try { + log = JSON.parse(line) + } catch { + // Maybe there a multiple \n at the end of the file + continue + } + + logTime = new Date(log.timestamp).getTime() + if ( + logTime >= startTime && + logTime <= endTime && + logsLevel[log.level] >= logsLevel[level] && + (!tagsOneOf || lineHasTag(log, tagsOneOf)) + ) { + output.push(log) + + currentSize += line.length + + if (currentSize > MAX_LOGS_OUTPUT_CHARACTERS) break + } else if (logTime < startTime) { + break + } + } + + return { currentSize, output: output.reverse(), logTime } +} + +function lineHasTag (line: { tags?: string }, tagsOneOf: Set) { + if (!isArray(line.tags)) return false + + for (const lineTag of line.tags) { + if (tagsOneOf.has(lineTag)) return true + } + + return false +} + +function generateLogNameFilter (baseName: string) { + return new RegExp('^' + baseName.replace(/\.log$/, '') + '\\d*.log$') +} diff --git a/server/server/controllers/api/server/redundancy.ts b/server/server/controllers/api/server/redundancy.ts new file mode 100644 index 000000000..e16f85593 --- /dev/null +++ b/server/server/controllers/api/server/redundancy.ts @@ -0,0 +1,115 @@ +import express from 'express' +import { HttpStatusCode, UserRight } from '@peertube/peertube-models' +import { JobQueue } from '@server/lib/job-queue/index.js' +import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy.js' +import { logger } from '../../../helpers/logger.js' +import { removeRedundanciesOfServer, removeVideoRedundancy } from '../../../lib/redundancy.js' +import { + asyncMiddleware, + authenticate, + ensureUserHasRight, + paginationValidator, + setDefaultPagination, + setDefaultVideoRedundanciesSort, + videoRedundanciesSortValidator +} from '../../../middlewares/index.js' +import { + addVideoRedundancyValidator, + listVideoRedundanciesValidator, + removeVideoRedundancyValidator, + updateServerRedundancyValidator +} from '../../../middlewares/validators/redundancy.js' + +const serverRedundancyRouter = express.Router() + +serverRedundancyRouter.put('/redundancy/:host', + authenticate, + ensureUserHasRight(UserRight.MANAGE_SERVER_FOLLOW), + asyncMiddleware(updateServerRedundancyValidator), + asyncMiddleware(updateRedundancy) +) + +serverRedundancyRouter.get('/redundancy/videos', + authenticate, + ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES), + listVideoRedundanciesValidator, + paginationValidator, + videoRedundanciesSortValidator, + setDefaultVideoRedundanciesSort, + setDefaultPagination, + asyncMiddleware(listVideoRedundancies) +) + +serverRedundancyRouter.post('/redundancy/videos', + authenticate, + ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES), + addVideoRedundancyValidator, + asyncMiddleware(addVideoRedundancy) +) + +serverRedundancyRouter.delete('/redundancy/videos/:redundancyId', + authenticate, + ensureUserHasRight(UserRight.MANAGE_VIDEOS_REDUNDANCIES), + removeVideoRedundancyValidator, + asyncMiddleware(removeVideoRedundancyController) +) + +// --------------------------------------------------------------------------- + +export { + serverRedundancyRouter +} + +// --------------------------------------------------------------------------- + +async function listVideoRedundancies (req: express.Request, res: express.Response) { + const resultList = await VideoRedundancyModel.listForApi({ + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + target: req.query.target, + strategy: req.query.strategy + }) + + const result = { + total: resultList.total, + data: resultList.data.map(r => VideoRedundancyModel.toFormattedJSONStatic(r)) + } + + return res.json(result) +} + +async function addVideoRedundancy (req: express.Request, res: express.Response) { + const payload = { + videoId: res.locals.onlyVideo.id + } + + await JobQueue.Instance.createJob({ + type: 'video-redundancy', + payload + }) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function removeVideoRedundancyController (req: express.Request, res: express.Response) { + await removeVideoRedundancy(res.locals.videoRedundancy) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function updateRedundancy (req: express.Request, res: express.Response) { + const server = res.locals.server + + server.redundancyAllowed = req.body.redundancyAllowed + + await server.save() + + if (server.redundancyAllowed !== true) { + // Async, could be long + removeRedundanciesOfServer(server.id) + .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, { err })) + } + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} diff --git a/server/server/controllers/api/server/server-blocklist.ts b/server/server/controllers/api/server/server-blocklist.ts new file mode 100644 index 000000000..9ca7ec03e --- /dev/null +++ b/server/server/controllers/api/server/server-blocklist.ts @@ -0,0 +1,162 @@ +import 'multer' +import express from 'express' +import { HttpStatusCode, UserRight } from '@peertube/peertube-models' +import { logger } from '@server/helpers/logger.js' +import { getServerActor } from '@server/models/application/application.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { getFormattedObjects } from '../../../helpers/utils.js' +import { + addAccountInBlocklist, + addServerInBlocklist, + removeAccountFromBlocklist, + removeServerFromBlocklist +} from '../../../lib/blocklist.js' +import { + asyncMiddleware, + asyncRetryTransactionMiddleware, + authenticate, + ensureUserHasRight, + paginationValidator, + setDefaultPagination, + setDefaultSort +} from '../../../middlewares/index.js' +import { + accountsBlocklistSortValidator, + blockAccountValidator, + blockServerValidator, + serversBlocklistSortValidator, + unblockAccountByServerValidator, + unblockServerByServerValidator +} from '../../../middlewares/validators/index.js' +import { AccountBlocklistModel } from '../../../models/account/account-blocklist.js' +import { ServerBlocklistModel } from '../../../models/server/server-blocklist.js' + +const serverBlocklistRouter = express.Router() + +serverBlocklistRouter.get('/blocklist/accounts', + authenticate, + ensureUserHasRight(UserRight.MANAGE_ACCOUNTS_BLOCKLIST), + paginationValidator, + accountsBlocklistSortValidator, + setDefaultSort, + setDefaultPagination, + asyncMiddleware(listBlockedAccounts) +) + +serverBlocklistRouter.post('/blocklist/accounts', + authenticate, + ensureUserHasRight(UserRight.MANAGE_ACCOUNTS_BLOCKLIST), + asyncMiddleware(blockAccountValidator), + asyncRetryTransactionMiddleware(blockAccount) +) + +serverBlocklistRouter.delete('/blocklist/accounts/:accountName', + authenticate, + ensureUserHasRight(UserRight.MANAGE_ACCOUNTS_BLOCKLIST), + asyncMiddleware(unblockAccountByServerValidator), + asyncRetryTransactionMiddleware(unblockAccount) +) + +serverBlocklistRouter.get('/blocklist/servers', + authenticate, + ensureUserHasRight(UserRight.MANAGE_SERVERS_BLOCKLIST), + paginationValidator, + serversBlocklistSortValidator, + setDefaultSort, + setDefaultPagination, + asyncMiddleware(listBlockedServers) +) + +serverBlocklistRouter.post('/blocklist/servers', + authenticate, + ensureUserHasRight(UserRight.MANAGE_SERVERS_BLOCKLIST), + asyncMiddleware(blockServerValidator), + asyncRetryTransactionMiddleware(blockServer) +) + +serverBlocklistRouter.delete('/blocklist/servers/:host', + authenticate, + ensureUserHasRight(UserRight.MANAGE_SERVERS_BLOCKLIST), + asyncMiddleware(unblockServerByServerValidator), + asyncRetryTransactionMiddleware(unblockServer) +) + +export { + serverBlocklistRouter +} + +// --------------------------------------------------------------------------- + +async function listBlockedAccounts (req: express.Request, res: express.Response) { + const serverActor = await getServerActor() + + const resultList = await AccountBlocklistModel.listForApi({ + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + search: req.query.search, + accountId: serverActor.Account.id + }) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function blockAccount (req: express.Request, res: express.Response) { + const serverActor = await getServerActor() + const accountToBlock = res.locals.account + + await addAccountInBlocklist(serverActor.Account.id, accountToBlock.id) + + UserNotificationModel.removeNotificationsOf({ + id: accountToBlock.id, + type: 'account', + forUserId: null // For all users + }).catch(err => logger.error('Cannot remove notifications after an account mute.', { err })) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function unblockAccount (req: express.Request, res: express.Response) { + const accountBlock = res.locals.accountBlock + + await removeAccountFromBlocklist(accountBlock) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function listBlockedServers (req: express.Request, res: express.Response) { + const serverActor = await getServerActor() + + const resultList = await ServerBlocklistModel.listForApi({ + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + search: req.query.search, + accountId: serverActor.Account.id + }) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function blockServer (req: express.Request, res: express.Response) { + const serverActor = await getServerActor() + const serverToBlock = res.locals.server + + await addServerInBlocklist(serverActor.Account.id, serverToBlock.id) + + UserNotificationModel.removeNotificationsOf({ + id: serverToBlock.id, + type: 'server', + forUserId: null // For all users + }).catch(err => logger.error('Cannot remove notifications after a server mute.', { err })) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function unblockServer (req: express.Request, res: express.Response) { + const serverBlock = res.locals.serverBlock + + await removeServerFromBlocklist(serverBlock) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} diff --git a/server/server/controllers/api/server/stats.ts b/server/server/controllers/api/server/stats.ts new file mode 100644 index 000000000..5a5f70f02 --- /dev/null +++ b/server/server/controllers/api/server/stats.ts @@ -0,0 +1,26 @@ +import express from 'express' +import { StatsManager } from '@server/lib/stat-manager.js' +import { ROUTE_CACHE_LIFETIME } from '../../../initializers/constants.js' +import { asyncMiddleware } from '../../../middlewares/index.js' +import { cacheRoute } from '../../../middlewares/cache/cache.js' +import { Hooks } from '@server/lib/plugins/hooks.js' + +const statsRouter = express.Router() + +statsRouter.get('/stats', + cacheRoute(ROUTE_CACHE_LIFETIME.STATS), + asyncMiddleware(getStats) +) + +async function getStats (_req: express.Request, res: express.Response) { + let data = await StatsManager.Instance.getStats() + data = await Hooks.wrapObject(data, 'filter:api.server.stats.get.result') + + return res.json(data) +} + +// --------------------------------------------------------------------------- + +export { + statsRouter +} diff --git a/server/server/controllers/api/users/email-verification.ts b/server/server/controllers/api/users/email-verification.ts new file mode 100644 index 000000000..ef9dc30b6 --- /dev/null +++ b/server/server/controllers/api/users/email-verification.ts @@ -0,0 +1,72 @@ +import express from 'express' +import { HttpStatusCode } from '@peertube/peertube-models' +import { CONFIG } from '../../../initializers/config.js' +import { sendVerifyRegistrationEmail, sendVerifyUserEmail } from '../../../lib/user.js' +import { asyncMiddleware, buildRateLimiter } from '../../../middlewares/index.js' +import { + registrationVerifyEmailValidator, + usersAskSendVerifyEmailValidator, + usersVerifyEmailValidator +} from '../../../middlewares/validators/index.js' + +const askSendEmailLimiter = buildRateLimiter({ + windowMs: CONFIG.RATES_LIMIT.ASK_SEND_EMAIL.WINDOW_MS, + max: CONFIG.RATES_LIMIT.ASK_SEND_EMAIL.MAX +}) + +const emailVerificationRouter = express.Router() + +emailVerificationRouter.post([ '/ask-send-verify-email', '/registrations/ask-send-verify-email' ], + askSendEmailLimiter, + asyncMiddleware(usersAskSendVerifyEmailValidator), + asyncMiddleware(reSendVerifyUserEmail) +) + +emailVerificationRouter.post('/:id/verify-email', + asyncMiddleware(usersVerifyEmailValidator), + asyncMiddleware(verifyUserEmail) +) + +emailVerificationRouter.post('/registrations/:registrationId/verify-email', + asyncMiddleware(registrationVerifyEmailValidator), + asyncMiddleware(verifyRegistrationEmail) +) + +// --------------------------------------------------------------------------- + +export { + emailVerificationRouter +} + +async function reSendVerifyUserEmail (req: express.Request, res: express.Response) { + const user = res.locals.user + const registration = res.locals.userRegistration + + if (user) await sendVerifyUserEmail(user) + else if (registration) await sendVerifyRegistrationEmail(registration) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function verifyUserEmail (req: express.Request, res: express.Response) { + const user = res.locals.user + user.emailVerified = true + + if (req.body.isPendingEmail === true) { + user.email = user.pendingEmail + user.pendingEmail = null + } + + await user.save() + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function verifyRegistrationEmail (req: express.Request, res: express.Response) { + const registration = res.locals.userRegistration + registration.emailVerified = true + + await registration.save() + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} diff --git a/server/server/controllers/api/users/index.ts b/server/server/controllers/api/users/index.ts new file mode 100644 index 000000000..e5290fe25 --- /dev/null +++ b/server/server/controllers/api/users/index.ts @@ -0,0 +1,319 @@ +import express from 'express' +import { tokensRouter } from '@server/controllers/api/users/token.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { OAuthTokenModel } from '@server/models/oauth/oauth-token.js' +import { MUserAccountDefault } from '@server/types/models/index.js' +import { pick } from '@peertube/peertube-core-utils' +import { HttpStatusCode, UserCreate, UserCreateResult, UserRight, UserUpdate } from '@peertube/peertube-models' +import { auditLoggerFactory, getAuditIdFromRes, UserAuditView } from '../../../helpers/audit-logger.js' +import { logger } from '../../../helpers/logger.js' +import { generateRandomString, getFormattedObjects } from '../../../helpers/utils.js' +import { WEBSERVER } from '../../../initializers/constants.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { Emailer } from '../../../lib/emailer.js' +import { Redis } from '../../../lib/redis.js' +import { buildUser, createUserAccountAndChannelAndPlaylist } from '../../../lib/user.js' +import { + adminUsersSortValidator, + apiRateLimiter, + asyncMiddleware, + asyncRetryTransactionMiddleware, + authenticate, + ensureUserHasRight, + paginationValidator, + setDefaultPagination, + setDefaultSort, + userAutocompleteValidator, + usersAddValidator, + usersGetValidator, + usersListValidator, + usersRemoveValidator, + usersUpdateValidator +} from '../../../middlewares/index.js' +import { + ensureCanModerateUser, + usersAskResetPasswordValidator, + usersBlockingValidator, + usersResetPasswordValidator +} from '../../../middlewares/validators/index.js' +import { UserModel } from '../../../models/user/user.js' +import { emailVerificationRouter } from './email-verification.js' +import { meRouter } from './me.js' +import { myAbusesRouter } from './my-abuses.js' +import { myBlocklistRouter } from './my-blocklist.js' +import { myVideosHistoryRouter } from './my-history.js' +import { myNotificationsRouter } from './my-notifications.js' +import { mySubscriptionsRouter } from './my-subscriptions.js' +import { myVideoPlaylistsRouter } from './my-video-playlists.js' +import { registrationsRouter } from './registrations.js' +import { twoFactorRouter } from './two-factor.js' + +const auditLogger = auditLoggerFactory('users') + +const usersRouter = express.Router() + +usersRouter.use(apiRateLimiter) + +usersRouter.use('/', emailVerificationRouter) +usersRouter.use('/', registrationsRouter) +usersRouter.use('/', twoFactorRouter) +usersRouter.use('/', tokensRouter) +usersRouter.use('/', myNotificationsRouter) +usersRouter.use('/', mySubscriptionsRouter) +usersRouter.use('/', myBlocklistRouter) +usersRouter.use('/', myVideosHistoryRouter) +usersRouter.use('/', myVideoPlaylistsRouter) +usersRouter.use('/', myAbusesRouter) +usersRouter.use('/', meRouter) + +usersRouter.get('/autocomplete', + userAutocompleteValidator, + asyncMiddleware(autocompleteUsers) +) + +usersRouter.get('/', + authenticate, + ensureUserHasRight(UserRight.MANAGE_USERS), + paginationValidator, + adminUsersSortValidator, + setDefaultSort, + setDefaultPagination, + usersListValidator, + asyncMiddleware(listUsers) +) + +usersRouter.post('/:id/block', + authenticate, + ensureUserHasRight(UserRight.MANAGE_USERS), + asyncMiddleware(usersBlockingValidator), + ensureCanModerateUser, + asyncMiddleware(blockUser) +) +usersRouter.post('/:id/unblock', + authenticate, + ensureUserHasRight(UserRight.MANAGE_USERS), + asyncMiddleware(usersBlockingValidator), + ensureCanModerateUser, + asyncMiddleware(unblockUser) +) + +usersRouter.get('/:id', + authenticate, + ensureUserHasRight(UserRight.MANAGE_USERS), + asyncMiddleware(usersGetValidator), + getUser +) + +usersRouter.post('/', + authenticate, + ensureUserHasRight(UserRight.MANAGE_USERS), + asyncMiddleware(usersAddValidator), + asyncRetryTransactionMiddleware(createUser) +) + +usersRouter.put('/:id', + authenticate, + ensureUserHasRight(UserRight.MANAGE_USERS), + asyncMiddleware(usersUpdateValidator), + ensureCanModerateUser, + asyncMiddleware(updateUser) +) + +usersRouter.delete('/:id', + authenticate, + ensureUserHasRight(UserRight.MANAGE_USERS), + asyncMiddleware(usersRemoveValidator), + ensureCanModerateUser, + asyncMiddleware(removeUser) +) + +usersRouter.post('/ask-reset-password', + asyncMiddleware(usersAskResetPasswordValidator), + asyncMiddleware(askResetUserPassword) +) + +usersRouter.post('/:id/reset-password', + asyncMiddleware(usersResetPasswordValidator), + asyncMiddleware(resetUserPassword) +) + +// --------------------------------------------------------------------------- + +export { + usersRouter +} + +// --------------------------------------------------------------------------- + +async function createUser (req: express.Request, res: express.Response) { + const body: UserCreate = req.body + + const userToCreate = buildUser({ + ...pick(body, [ 'username', 'password', 'email', 'role', 'videoQuota', 'videoQuotaDaily', 'adminFlags' ]), + + emailVerified: null + }) + + // NB: due to the validator usersAddValidator, password==='' can only be true if we can send the mail. + const createPassword = userToCreate.password === '' + if (createPassword) { + userToCreate.password = await generateRandomString(20) + } + + const { user, account, videoChannel } = await createUserAccountAndChannelAndPlaylist({ + userToCreate, + channelNames: body.channelName && { name: body.channelName, displayName: body.channelName } + }) + + auditLogger.create(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON())) + logger.info('User %s with its channel and account created.', body.username) + + if (createPassword) { + // this will send an email for newly created users, so then can set their first password. + logger.info('Sending to user %s a create password email', body.username) + const verificationString = await Redis.Instance.setCreatePasswordVerificationString(user.id) + const url = WEBSERVER.URL + '/reset-password?userId=' + user.id + '&verificationString=' + verificationString + Emailer.Instance.addPasswordCreateEmailJob(userToCreate.username, user.email, url) + } + + Hooks.runAction('action:api.user.created', { body, user, account, videoChannel, req, res }) + + return res.json({ + user: { + id: user.id, + account: { + id: account.id + } + } as UserCreateResult + }) +} + +async function unblockUser (req: express.Request, res: express.Response) { + const user = res.locals.user + + await changeUserBlock(res, user, false) + + Hooks.runAction('action:api.user.unblocked', { user, req, res }) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function blockUser (req: express.Request, res: express.Response) { + const user = res.locals.user + const reason = req.body.reason + + await changeUserBlock(res, user, true, reason) + + Hooks.runAction('action:api.user.blocked', { user, req, res }) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +function getUser (req: express.Request, res: express.Response) { + return res.json(res.locals.user.toFormattedJSON({ withAdminFlags: true })) +} + +async function autocompleteUsers (req: express.Request, res: express.Response) { + const resultList = await UserModel.autoComplete(req.query.search as string) + + return res.json(resultList) +} + +async function listUsers (req: express.Request, res: express.Response) { + const resultList = await UserModel.listForAdminApi({ + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + search: req.query.search, + blocked: req.query.blocked + }) + + return res.json(getFormattedObjects(resultList.data, resultList.total, { withAdminFlags: true })) +} + +async function removeUser (req: express.Request, res: express.Response) { + const user = res.locals.user + + auditLogger.delete(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON())) + + await sequelizeTypescript.transaction(async t => { + // Use a transaction to avoid inconsistencies with hooks (account/channel deletion & federation) + await user.destroy({ transaction: t }) + }) + + Hooks.runAction('action:api.user.deleted', { user, req, res }) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function updateUser (req: express.Request, res: express.Response) { + const body: UserUpdate = req.body + const userToUpdate = res.locals.user + const oldUserAuditView = new UserAuditView(userToUpdate.toFormattedJSON()) + const roleChanged = body.role !== undefined && body.role !== userToUpdate.role + + const keysToUpdate: (keyof UserUpdate)[] = [ + 'password', + 'email', + 'emailVerified', + 'videoQuota', + 'videoQuotaDaily', + 'role', + 'adminFlags', + 'pluginAuth' + ] + + for (const key of keysToUpdate) { + if (body[key] !== undefined) userToUpdate.set(key, body[key]) + } + + const user = await userToUpdate.save() + + // Destroy user token to refresh rights + if (roleChanged || body.password !== undefined) await OAuthTokenModel.deleteUserToken(userToUpdate.id) + + auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView) + + Hooks.runAction('action:api.user.updated', { user, req, res }) + + // Don't need to send this update to followers, these attributes are not federated + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function askResetUserPassword (req: express.Request, res: express.Response) { + const user = res.locals.user + + const verificationString = await Redis.Instance.setResetPasswordVerificationString(user.id) + const url = WEBSERVER.URL + '/reset-password?userId=' + user.id + '&verificationString=' + verificationString + Emailer.Instance.addPasswordResetEmailJob(user.username, user.email, url) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function resetUserPassword (req: express.Request, res: express.Response) { + const user = res.locals.user + user.password = req.body.password + + await user.save() + await Redis.Instance.removePasswordVerificationString(user.id) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function changeUserBlock (res: express.Response, user: MUserAccountDefault, block: boolean, reason?: string) { + const oldUserAuditView = new UserAuditView(user.toFormattedJSON()) + + user.blocked = block + user.blockedReason = reason || null + + await sequelizeTypescript.transaction(async t => { + await OAuthTokenModel.deleteUserToken(user.id, t) + + await user.save({ transaction: t }) + }) + + Emailer.Instance.addUserBlockJob(user, block, reason) + + auditLogger.update(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON()), oldUserAuditView) +} diff --git a/server/server/controllers/api/users/me.ts b/server/server/controllers/api/users/me.ts new file mode 100644 index 000000000..69edd342d --- /dev/null +++ b/server/server/controllers/api/users/me.ts @@ -0,0 +1,283 @@ +import 'multer' +import express from 'express' +import { pick } from '@peertube/peertube-core-utils' +import { + ActorImageType, + HttpStatusCode, + UserUpdateMe, + UserVideoQuota, + UserVideoRate as FormattedUserVideoRate +} from '@peertube/peertube-models' +import { auditLoggerFactory, getAuditIdFromRes, UserAuditView } from '@server/helpers/audit-logger.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { createReqFiles } from '../../../helpers/express-utils.js' +import { getFormattedObjects } from '../../../helpers/utils.js' +import { CONFIG } from '../../../initializers/config.js' +import { MIMETYPES } from '../../../initializers/constants.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { sendUpdateActor } from '../../../lib/activitypub/send/index.js' +import { deleteLocalActorImageFile, updateLocalActorImageFiles } from '../../../lib/local-actor.js' +import { getOriginalVideoFileTotalDailyFromUser, getOriginalVideoFileTotalFromUser, sendVerifyUserEmail } from '../../../lib/user.js' +import { + asyncMiddleware, + asyncRetryTransactionMiddleware, + authenticate, + paginationValidator, + setDefaultPagination, + setDefaultSort, + setDefaultVideosSort, + usersUpdateMeValidator, + usersVideoRatingValidator +} from '../../../middlewares/index.js' +import { updateAvatarValidator } from '../../../middlewares/validators/actor-image.js' +import { + deleteMeValidator, + getMyVideoImportsValidator, + usersVideosValidator, + videoImportsSortValidator, + videosSortValidator +} from '../../../middlewares/validators/index.js' +import { AccountVideoRateModel } from '../../../models/account/account-video-rate.js' +import { AccountModel } from '../../../models/account/account.js' +import { UserModel } from '../../../models/user/user.js' +import { VideoImportModel } from '../../../models/video/video-import.js' +import { VideoModel } from '../../../models/video/video.js' + +const auditLogger = auditLoggerFactory('users') + +const reqAvatarFile = createReqFiles([ 'avatarfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT) + +const meRouter = express.Router() + +meRouter.get('/me', + authenticate, + asyncMiddleware(getUserInformation) +) +meRouter.delete('/me', + authenticate, + deleteMeValidator, + asyncMiddleware(deleteMe) +) + +meRouter.get('/me/video-quota-used', + authenticate, + asyncMiddleware(getUserVideoQuotaUsed) +) + +meRouter.get('/me/videos/imports', + authenticate, + paginationValidator, + videoImportsSortValidator, + setDefaultSort, + setDefaultPagination, + getMyVideoImportsValidator, + asyncMiddleware(getUserVideoImports) +) + +meRouter.get('/me/videos', + authenticate, + paginationValidator, + videosSortValidator, + setDefaultVideosSort, + setDefaultPagination, + asyncMiddleware(usersVideosValidator), + asyncMiddleware(getUserVideos) +) + +meRouter.get('/me/videos/:videoId/rating', + authenticate, + asyncMiddleware(usersVideoRatingValidator), + asyncMiddleware(getUserVideoRating) +) + +meRouter.put('/me', + authenticate, + asyncMiddleware(usersUpdateMeValidator), + asyncRetryTransactionMiddleware(updateMe) +) + +meRouter.post('/me/avatar/pick', + authenticate, + reqAvatarFile, + updateAvatarValidator, + asyncRetryTransactionMiddleware(updateMyAvatar) +) + +meRouter.delete('/me/avatar', + authenticate, + asyncRetryTransactionMiddleware(deleteMyAvatar) +) + +// --------------------------------------------------------------------------- + +export { + meRouter +} + +// --------------------------------------------------------------------------- + +async function getUserVideos (req: express.Request, res: express.Response) { + const user = res.locals.oauth.token.User + + const apiOptions = await Hooks.wrapObject({ + accountId: user.Account.id, + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + search: req.query.search, + channelId: res.locals.videoChannel?.id, + isLive: req.query.isLive + }, 'filter:api.user.me.videos.list.params') + + const resultList = await Hooks.wrapPromiseFun( + VideoModel.listUserVideosForApi, + apiOptions, + 'filter:api.user.me.videos.list.result' + ) + + const additionalAttributes = { + waitTranscoding: true, + state: true, + scheduledUpdate: true, + blacklistInfo: true + } + return res.json(getFormattedObjects(resultList.data, resultList.total, { additionalAttributes })) +} + +async function getUserVideoImports (req: express.Request, res: express.Response) { + const user = res.locals.oauth.token.User + const resultList = await VideoImportModel.listUserVideoImportsForApi({ + userId: user.id, + + ...pick(req.query, [ 'targetUrl', 'start', 'count', 'sort', 'search', 'videoChannelSyncId' ]) + }) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function getUserInformation (req: express.Request, res: express.Response) { + // We did not load channels in res.locals.user + const user = await UserModel.loadForMeAPI(res.locals.oauth.token.user.id) + + return res.json(user.toMeFormattedJSON()) +} + +async function getUserVideoQuotaUsed (req: express.Request, res: express.Response) { + const user = res.locals.oauth.token.user + const videoQuotaUsed = await getOriginalVideoFileTotalFromUser(user) + const videoQuotaUsedDaily = await getOriginalVideoFileTotalDailyFromUser(user) + + const data: UserVideoQuota = { + videoQuotaUsed, + videoQuotaUsedDaily + } + return res.json(data) +} + +async function getUserVideoRating (req: express.Request, res: express.Response) { + const videoId = res.locals.videoId.id + const accountId = +res.locals.oauth.token.User.Account.id + + const ratingObj = await AccountVideoRateModel.load(accountId, videoId, null) + const rating = ratingObj ? ratingObj.type : 'none' + + const json: FormattedUserVideoRate = { + videoId, + rating + } + return res.json(json) +} + +async function deleteMe (req: express.Request, res: express.Response) { + const user = await UserModel.loadByIdWithChannels(res.locals.oauth.token.User.id) + + auditLogger.delete(getAuditIdFromRes(res), new UserAuditView(user.toFormattedJSON())) + + await user.destroy() + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function updateMe (req: express.Request, res: express.Response) { + const body: UserUpdateMe = req.body + let sendVerificationEmail = false + + const user = res.locals.oauth.token.user + + const keysToUpdate: (keyof UserUpdateMe & keyof AttributesOnly)[] = [ + 'password', + 'nsfwPolicy', + 'p2pEnabled', + 'autoPlayVideo', + 'autoPlayNextVideo', + 'autoPlayNextVideoPlaylist', + 'videosHistoryEnabled', + 'videoLanguages', + 'theme', + 'noInstanceConfigWarningModal', + 'noAccountSetupWarningModal', + 'noWelcomeModal', + 'emailPublic', + 'p2pEnabled' + ] + + for (const key of keysToUpdate) { + if (body[key] !== undefined) user.set(key, body[key]) + } + + if (body.email !== undefined) { + if (CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION) { + user.pendingEmail = body.email + sendVerificationEmail = true + } else { + user.email = body.email + } + } + + await sequelizeTypescript.transaction(async t => { + await user.save({ transaction: t }) + + if (body.displayName === undefined && body.description === undefined) return + + const userAccount = await AccountModel.load(user.Account.id, t) + + if (body.displayName !== undefined) userAccount.name = body.displayName + if (body.description !== undefined) userAccount.description = body.description + await userAccount.save({ transaction: t }) + + await sendUpdateActor(userAccount, t) + }) + + if (sendVerificationEmail === true) { + await sendVerifyUserEmail(user, true) + } + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function updateMyAvatar (req: express.Request, res: express.Response) { + const avatarPhysicalFile = req.files['avatarfile'][0] + const user = res.locals.oauth.token.user + + const userAccount = await AccountModel.load(user.Account.id) + + const avatars = await updateLocalActorImageFiles( + userAccount, + avatarPhysicalFile, + ActorImageType.AVATAR + ) + + return res.json({ + avatars: avatars.map(avatar => avatar.toFormattedJSON()) + }) +} + +async function deleteMyAvatar (req: express.Request, res: express.Response) { + const user = res.locals.oauth.token.user + + const userAccount = await AccountModel.load(user.Account.id) + await deleteLocalActorImageFile(userAccount, ActorImageType.AVATAR) + + return res.json({ avatars: [] }) +} diff --git a/server/server/controllers/api/users/my-abuses.ts b/server/server/controllers/api/users/my-abuses.ts new file mode 100644 index 000000000..510d5528f --- /dev/null +++ b/server/server/controllers/api/users/my-abuses.ts @@ -0,0 +1,48 @@ +import express from 'express' +import { AbuseModel } from '@server/models/abuse/abuse.js' +import { + abuseListForUserValidator, + abusesSortValidator, + asyncMiddleware, + authenticate, + paginationValidator, + setDefaultPagination, + setDefaultSort +} from '../../../middlewares/index.js' + +const myAbusesRouter = express.Router() + +myAbusesRouter.get('/me/abuses', + authenticate, + paginationValidator, + abusesSortValidator, + setDefaultSort, + setDefaultPagination, + abuseListForUserValidator, + asyncMiddleware(listMyAbuses) +) + +// --------------------------------------------------------------------------- + +export { + myAbusesRouter +} + +// --------------------------------------------------------------------------- + +async function listMyAbuses (req: express.Request, res: express.Response) { + const resultList = await AbuseModel.listForUserApi({ + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + id: req.query.id, + search: req.query.search, + state: req.query.state, + user: res.locals.oauth.token.User + }) + + return res.json({ + total: resultList.total, + data: resultList.data.map(d => d.toFormattedUserJSON()) + }) +} diff --git a/server/server/controllers/api/users/my-blocklist.ts b/server/server/controllers/api/users/my-blocklist.ts new file mode 100644 index 000000000..46a988ea6 --- /dev/null +++ b/server/server/controllers/api/users/my-blocklist.ts @@ -0,0 +1,154 @@ +import 'multer' +import express from 'express' +import { HttpStatusCode } from '@peertube/peertube-models' +import { logger } from '@server/helpers/logger.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { getFormattedObjects } from '../../../helpers/utils.js' +import { + addAccountInBlocklist, + addServerInBlocklist, + removeAccountFromBlocklist, + removeServerFromBlocklist +} from '../../../lib/blocklist.js' +import { + asyncMiddleware, + asyncRetryTransactionMiddleware, + authenticate, + paginationValidator, + setDefaultPagination, + setDefaultSort, + unblockAccountByAccountValidator +} from '../../../middlewares/index.js' +import { + accountsBlocklistSortValidator, + blockAccountValidator, + blockServerValidator, + serversBlocklistSortValidator, + unblockServerByAccountValidator +} from '../../../middlewares/validators/index.js' +import { AccountBlocklistModel } from '../../../models/account/account-blocklist.js' +import { ServerBlocklistModel } from '../../../models/server/server-blocklist.js' + +const myBlocklistRouter = express.Router() + +myBlocklistRouter.get('/me/blocklist/accounts', + authenticate, + paginationValidator, + accountsBlocklistSortValidator, + setDefaultSort, + setDefaultPagination, + asyncMiddleware(listBlockedAccounts) +) + +myBlocklistRouter.post('/me/blocklist/accounts', + authenticate, + asyncMiddleware(blockAccountValidator), + asyncRetryTransactionMiddleware(blockAccount) +) + +myBlocklistRouter.delete('/me/blocklist/accounts/:accountName', + authenticate, + asyncMiddleware(unblockAccountByAccountValidator), + asyncRetryTransactionMiddleware(unblockAccount) +) + +myBlocklistRouter.get('/me/blocklist/servers', + authenticate, + paginationValidator, + serversBlocklistSortValidator, + setDefaultSort, + setDefaultPagination, + asyncMiddleware(listBlockedServers) +) + +myBlocklistRouter.post('/me/blocklist/servers', + authenticate, + asyncMiddleware(blockServerValidator), + asyncRetryTransactionMiddleware(blockServer) +) + +myBlocklistRouter.delete('/me/blocklist/servers/:host', + authenticate, + asyncMiddleware(unblockServerByAccountValidator), + asyncRetryTransactionMiddleware(unblockServer) +) + +export { + myBlocklistRouter +} + +// --------------------------------------------------------------------------- + +async function listBlockedAccounts (req: express.Request, res: express.Response) { + const user = res.locals.oauth.token.User + + const resultList = await AccountBlocklistModel.listForApi({ + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + search: req.query.search, + accountId: user.Account.id + }) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function blockAccount (req: express.Request, res: express.Response) { + const user = res.locals.oauth.token.User + const accountToBlock = res.locals.account + + await addAccountInBlocklist(user.Account.id, accountToBlock.id) + + UserNotificationModel.removeNotificationsOf({ + id: accountToBlock.id, + type: 'account', + forUserId: user.id + }).catch(err => logger.error('Cannot remove notifications after an account mute.', { err })) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function unblockAccount (req: express.Request, res: express.Response) { + const accountBlock = res.locals.accountBlock + + await removeAccountFromBlocklist(accountBlock) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function listBlockedServers (req: express.Request, res: express.Response) { + const user = res.locals.oauth.token.User + + const resultList = await ServerBlocklistModel.listForApi({ + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + search: req.query.search, + accountId: user.Account.id + }) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function blockServer (req: express.Request, res: express.Response) { + const user = res.locals.oauth.token.User + const serverToBlock = res.locals.server + + await addServerInBlocklist(user.Account.id, serverToBlock.id) + + UserNotificationModel.removeNotificationsOf({ + id: serverToBlock.id, + type: 'server', + forUserId: user.id + }).catch(err => logger.error('Cannot remove notifications after a server mute.', { err })) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function unblockServer (req: express.Request, res: express.Response) { + const serverBlock = res.locals.serverBlock + + await removeServerFromBlocklist(serverBlock) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} diff --git a/server/server/controllers/api/users/my-history.ts b/server/server/controllers/api/users/my-history.ts new file mode 100644 index 000000000..c2106e4e5 --- /dev/null +++ b/server/server/controllers/api/users/my-history.ts @@ -0,0 +1,75 @@ +import express from 'express' +import { forceNumber } from '@peertube/peertube-core-utils' +import { HttpStatusCode } from '@peertube/peertube-models' +import { getFormattedObjects } from '../../../helpers/utils.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { + asyncMiddleware, + asyncRetryTransactionMiddleware, + authenticate, + paginationValidator, + setDefaultPagination, + userHistoryListValidator, + userHistoryRemoveAllValidator, + userHistoryRemoveElementValidator +} from '../../../middlewares/index.js' +import { UserVideoHistoryModel } from '../../../models/user/user-video-history.js' + +const myVideosHistoryRouter = express.Router() + +myVideosHistoryRouter.get('/me/history/videos', + authenticate, + paginationValidator, + setDefaultPagination, + userHistoryListValidator, + asyncMiddleware(listMyVideosHistory) +) + +myVideosHistoryRouter.delete('/me/history/videos/:videoId', + authenticate, + userHistoryRemoveElementValidator, + asyncMiddleware(removeUserHistoryElement) +) + +myVideosHistoryRouter.post('/me/history/videos/remove', + authenticate, + userHistoryRemoveAllValidator, + asyncRetryTransactionMiddleware(removeAllUserHistory) +) + +// --------------------------------------------------------------------------- + +export { + myVideosHistoryRouter +} + +// --------------------------------------------------------------------------- + +async function listMyVideosHistory (req: express.Request, res: express.Response) { + const user = res.locals.oauth.token.User + + const resultList = await UserVideoHistoryModel.listForApi(user, req.query.start, req.query.count, req.query.search) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function removeUserHistoryElement (req: express.Request, res: express.Response) { + const user = res.locals.oauth.token.User + + await UserVideoHistoryModel.removeUserHistoryElement(user, forceNumber(req.params.videoId)) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +async function removeAllUserHistory (req: express.Request, res: express.Response) { + const user = res.locals.oauth.token.User + const beforeDate = req.body.beforeDate || null + + await sequelizeTypescript.transaction(t => { + return UserVideoHistoryModel.removeUserHistoryBefore(user, beforeDate, t) + }) + + return res.type('json') + .status(HttpStatusCode.NO_CONTENT_204) + .end() +} diff --git a/server/server/controllers/api/users/my-notifications.ts b/server/server/controllers/api/users/my-notifications.ts new file mode 100644 index 000000000..c0172a452 --- /dev/null +++ b/server/server/controllers/api/users/my-notifications.ts @@ -0,0 +1,115 @@ +import 'multer' +import express from 'express' +import { HttpStatusCode, UserNotificationSetting } from '@peertube/peertube-models' +import { getFormattedObjects } from '@server/helpers/utils.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { + asyncMiddleware, + asyncRetryTransactionMiddleware, + authenticate, + paginationValidator, + setDefaultPagination, + setDefaultSort, + userNotificationsSortValidator +} from '../../../middlewares/index.js' +import { + listUserNotificationsValidator, + markAsReadUserNotificationsValidator, + updateNotificationSettingsValidator +} from '../../../middlewares/validators/user-notifications.js' +import { UserNotificationSettingModel } from '../../../models/user/user-notification-setting.js' +import { meRouter } from './me.js' + +const myNotificationsRouter = express.Router() + +meRouter.put('/me/notification-settings', + authenticate, + updateNotificationSettingsValidator, + asyncRetryTransactionMiddleware(updateNotificationSettings) +) + +myNotificationsRouter.get('/me/notifications', + authenticate, + paginationValidator, + userNotificationsSortValidator, + setDefaultSort, + setDefaultPagination, + listUserNotificationsValidator, + asyncMiddleware(listUserNotifications) +) + +myNotificationsRouter.post('/me/notifications/read', + authenticate, + markAsReadUserNotificationsValidator, + asyncMiddleware(markAsReadUserNotifications) +) + +myNotificationsRouter.post('/me/notifications/read-all', + authenticate, + asyncMiddleware(markAsReadAllUserNotifications) +) + +export { + myNotificationsRouter +} + +// --------------------------------------------------------------------------- + +async function updateNotificationSettings (req: express.Request, res: express.Response) { + const user = res.locals.oauth.token.User + const body = req.body as UserNotificationSetting + + const query = { + where: { + userId: user.id + } + } + + const values: UserNotificationSetting = { + newVideoFromSubscription: body.newVideoFromSubscription, + newCommentOnMyVideo: body.newCommentOnMyVideo, + abuseAsModerator: body.abuseAsModerator, + videoAutoBlacklistAsModerator: body.videoAutoBlacklistAsModerator, + blacklistOnMyVideo: body.blacklistOnMyVideo, + myVideoPublished: body.myVideoPublished, + myVideoImportFinished: body.myVideoImportFinished, + newFollow: body.newFollow, + newUserRegistration: body.newUserRegistration, + commentMention: body.commentMention, + newInstanceFollower: body.newInstanceFollower, + autoInstanceFollowing: body.autoInstanceFollowing, + abuseNewMessage: body.abuseNewMessage, + abuseStateChange: body.abuseStateChange, + newPeerTubeVersion: body.newPeerTubeVersion, + newPluginVersion: body.newPluginVersion, + myVideoStudioEditionFinished: body.myVideoStudioEditionFinished + } + + await UserNotificationSettingModel.update(values, query) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function listUserNotifications (req: express.Request, res: express.Response) { + const user = res.locals.oauth.token.User + + const resultList = await UserNotificationModel.listForApi(user.id, req.query.start, req.query.count, req.query.sort, req.query.unread) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function markAsReadUserNotifications (req: express.Request, res: express.Response) { + const user = res.locals.oauth.token.User + + await UserNotificationModel.markAsRead(user.id, req.body.ids) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function markAsReadAllUserNotifications (req: express.Request, res: express.Response) { + const user = res.locals.oauth.token.User + + await UserNotificationModel.markAllAsRead(user.id) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} diff --git a/server/server/controllers/api/users/my-subscriptions.ts b/server/server/controllers/api/users/my-subscriptions.ts new file mode 100644 index 000000000..f2010950e --- /dev/null +++ b/server/server/controllers/api/users/my-subscriptions.ts @@ -0,0 +1,193 @@ +import 'multer' +import express from 'express' +import { HttpStatusCode } from '@peertube/peertube-models' +import { handlesToNameAndHost } from '@server/helpers/actors.js' +import { pickCommonVideoQuery } from '@server/helpers/query.js' +import { sendUndoFollow } from '@server/lib/activitypub/send/index.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { VideoChannelModel } from '@server/models/video/video-channel.js' +import { buildNSFWFilter, getCountVideos } from '../../../helpers/express-utils.js' +import { getFormattedObjects } from '../../../helpers/utils.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { JobQueue } from '../../../lib/job-queue/index.js' +import { + asyncMiddleware, + asyncRetryTransactionMiddleware, + authenticate, + commonVideosFiltersValidator, + paginationValidator, + setDefaultPagination, + setDefaultSort, + setDefaultVideosSort, + userSubscriptionAddValidator, + userSubscriptionGetValidator +} from '../../../middlewares/index.js' +import { + areSubscriptionsExistValidator, + userSubscriptionListValidator, + userSubscriptionsSortValidator, + videosSortValidator +} from '../../../middlewares/validators/index.js' +import { ActorFollowModel } from '../../../models/actor/actor-follow.js' +import { guessAdditionalAttributesFromQuery } from '../../../models/video/formatter/index.js' +import { VideoModel } from '../../../models/video/video.js' + +const mySubscriptionsRouter = express.Router() + +mySubscriptionsRouter.get('/me/subscriptions/videos', + authenticate, + paginationValidator, + videosSortValidator, + setDefaultVideosSort, + setDefaultPagination, + commonVideosFiltersValidator, + asyncMiddleware(getUserSubscriptionVideos) +) + +mySubscriptionsRouter.get('/me/subscriptions/exist', + authenticate, + areSubscriptionsExistValidator, + asyncMiddleware(areSubscriptionsExist) +) + +mySubscriptionsRouter.get('/me/subscriptions', + authenticate, + paginationValidator, + userSubscriptionsSortValidator, + setDefaultSort, + setDefaultPagination, + userSubscriptionListValidator, + asyncMiddleware(getUserSubscriptions) +) + +mySubscriptionsRouter.post('/me/subscriptions', + authenticate, + userSubscriptionAddValidator, + addUserSubscription +) + +mySubscriptionsRouter.get('/me/subscriptions/:uri', + authenticate, + userSubscriptionGetValidator, + asyncMiddleware(getUserSubscription) +) + +mySubscriptionsRouter.delete('/me/subscriptions/:uri', + authenticate, + userSubscriptionGetValidator, + asyncRetryTransactionMiddleware(deleteUserSubscription) +) + +// --------------------------------------------------------------------------- + +export { + mySubscriptionsRouter +} + +// --------------------------------------------------------------------------- + +async function areSubscriptionsExist (req: express.Request, res: express.Response) { + const uris = req.query.uris as string[] + const user = res.locals.oauth.token.User + + const sanitizedHandles = handlesToNameAndHost(uris) + + const results = await ActorFollowModel.listSubscriptionsOf(user.Account.Actor.id, sanitizedHandles) + + const existObject: { [id: string ]: boolean } = {} + for (const sanitizedHandle of sanitizedHandles) { + const obj = results.find(r => { + const server = r.ActorFollowing.Server + + return r.ActorFollowing.preferredUsername.toLowerCase() === sanitizedHandle.name.toLowerCase() && + ( + (!server && !sanitizedHandle.host) || + (server.host === sanitizedHandle.host) + ) + }) + + existObject[sanitizedHandle.handle] = obj !== undefined + } + + return res.json(existObject) +} + +function addUserSubscription (req: express.Request, res: express.Response) { + const user = res.locals.oauth.token.User + const [ name, host ] = req.body.uri.split('@') + + const payload = { + name, + host, + assertIsChannel: true, + followerActorId: user.Account.Actor.id + } + + JobQueue.Instance.createJobAsync({ type: 'activitypub-follow', payload }) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function getUserSubscription (req: express.Request, res: express.Response) { + const subscription = res.locals.subscription + const videoChannel = await VideoChannelModel.loadAndPopulateAccount(subscription.ActorFollowing.VideoChannel.id) + + return res.json(videoChannel.toFormattedJSON()) +} + +async function deleteUserSubscription (req: express.Request, res: express.Response) { + const subscription = res.locals.subscription + + await sequelizeTypescript.transaction(async t => { + if (subscription.state === 'accepted') { + sendUndoFollow(subscription, t) + } + + return subscription.destroy({ transaction: t }) + }) + + return res.type('json') + .status(HttpStatusCode.NO_CONTENT_204) + .end() +} + +async function getUserSubscriptions (req: express.Request, res: express.Response) { + const user = res.locals.oauth.token.User + const actorId = user.Account.Actor.id + + const resultList = await ActorFollowModel.listSubscriptionsForApi({ + actorId, + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + search: req.query.search + }) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function getUserSubscriptionVideos (req: express.Request, res: express.Response) { + const user = res.locals.oauth.token.User + const countVideos = getCountVideos(req) + const query = pickCommonVideoQuery(req.query) + + const apiOptions = await Hooks.wrapObject({ + ...query, + + displayOnlyForFollower: { + actorId: user.Account.Actor.id, + orLocalVideos: false + }, + nsfw: buildNSFWFilter(res, query.nsfw), + user, + countVideos + }, 'filter:api.user.me.subscription-videos.list.params') + + const resultList = await Hooks.wrapPromiseFun( + VideoModel.listForApi, + apiOptions, + 'filter:api.user.me.subscription-videos.list.result' + ) + + return res.json(getFormattedObjects(resultList.data, resultList.total, guessAdditionalAttributesFromQuery(query))) +} diff --git a/server/server/controllers/api/users/my-video-playlists.ts b/server/server/controllers/api/users/my-video-playlists.ts new file mode 100644 index 000000000..a72ea103d --- /dev/null +++ b/server/server/controllers/api/users/my-video-playlists.ts @@ -0,0 +1,51 @@ +import express from 'express' +import { forceNumber } from '@peertube/peertube-core-utils' +import { VideosExistInPlaylists } from '@peertube/peertube-models' +import { uuidToShort } from '@peertube/peertube-node-utils' +import { asyncMiddleware, authenticate } from '../../../middlewares/index.js' +import { doVideosInPlaylistExistValidator } from '../../../middlewares/validators/videos/video-playlists.js' +import { VideoPlaylistModel } from '../../../models/video/video-playlist.js' + +const myVideoPlaylistsRouter = express.Router() + +myVideoPlaylistsRouter.get('/me/video-playlists/videos-exist', + authenticate, + doVideosInPlaylistExistValidator, + asyncMiddleware(doVideosInPlaylistExist) +) + +// --------------------------------------------------------------------------- + +export { + myVideoPlaylistsRouter +} + +// --------------------------------------------------------------------------- + +async function doVideosInPlaylistExist (req: express.Request, res: express.Response) { + const videoIds = req.query.videoIds.map(i => forceNumber(i)) + const user = res.locals.oauth.token.User + + const results = await VideoPlaylistModel.listPlaylistSummariesOf(user.Account.id, videoIds) + + const existObject: VideosExistInPlaylists = {} + + for (const videoId of videoIds) { + existObject[videoId] = [] + } + + for (const result of results) { + for (const element of result.VideoPlaylistElements) { + existObject[element.videoId].push({ + playlistElementId: element.id, + playlistId: result.id, + playlistDisplayName: result.name, + playlistShortUUID: uuidToShort(result.uuid), + startTimestamp: element.startTimestamp, + stopTimestamp: element.stopTimestamp + }) + } + } + + return res.json(existObject) +} diff --git a/server/server/controllers/api/users/registrations.ts b/server/server/controllers/api/users/registrations.ts new file mode 100644 index 000000000..6c2a309e8 --- /dev/null +++ b/server/server/controllers/api/users/registrations.ts @@ -0,0 +1,249 @@ +import express from 'express' +import { Emailer } from '@server/lib/emailer.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { UserRegistrationModel } from '@server/models/user/user-registration.js' +import { pick } from '@peertube/peertube-core-utils' +import { + HttpStatusCode, + UserRegister, + UserRegistrationRequest, + UserRegistrationState, + UserRegistrationUpdateState, + UserRight +} from '@peertube/peertube-models' +import { auditLoggerFactory, UserAuditView } from '../../../helpers/audit-logger.js' +import { logger } from '../../../helpers/logger.js' +import { CONFIG } from '../../../initializers/config.js' +import { Notifier } from '../../../lib/notifier/index.js' +import { buildUser, createUserAccountAndChannelAndPlaylist, sendVerifyRegistrationEmail, sendVerifyUserEmail } from '../../../lib/user.js' +import { + acceptOrRejectRegistrationValidator, + asyncMiddleware, + asyncRetryTransactionMiddleware, + authenticate, + buildRateLimiter, + ensureUserHasRight, + ensureUserRegistrationAllowedFactory, + ensureUserRegistrationAllowedForIP, + getRegistrationValidator, + listRegistrationsValidator, + paginationValidator, + setDefaultPagination, + setDefaultSort, + userRegistrationsSortValidator, + usersDirectRegistrationValidator, + usersRequestRegistrationValidator +} from '../../../middlewares/index.js' + +const auditLogger = auditLoggerFactory('users') + +const registrationRateLimiter = buildRateLimiter({ + windowMs: CONFIG.RATES_LIMIT.SIGNUP.WINDOW_MS, + max: CONFIG.RATES_LIMIT.SIGNUP.MAX, + skipFailedRequests: true +}) + +const registrationsRouter = express.Router() + +registrationsRouter.post('/registrations/request', + registrationRateLimiter, + asyncMiddleware(ensureUserRegistrationAllowedFactory('request-registration')), + ensureUserRegistrationAllowedForIP, + asyncMiddleware(usersRequestRegistrationValidator), + asyncRetryTransactionMiddleware(requestRegistration) +) + +registrationsRouter.post('/registrations/:registrationId/accept', + authenticate, + ensureUserHasRight(UserRight.MANAGE_REGISTRATIONS), + asyncMiddleware(acceptOrRejectRegistrationValidator), + asyncRetryTransactionMiddleware(acceptRegistration) +) +registrationsRouter.post('/registrations/:registrationId/reject', + authenticate, + ensureUserHasRight(UserRight.MANAGE_REGISTRATIONS), + asyncMiddleware(acceptOrRejectRegistrationValidator), + asyncRetryTransactionMiddleware(rejectRegistration) +) + +registrationsRouter.delete('/registrations/:registrationId', + authenticate, + ensureUserHasRight(UserRight.MANAGE_REGISTRATIONS), + asyncMiddleware(getRegistrationValidator), + asyncRetryTransactionMiddleware(deleteRegistration) +) + +registrationsRouter.get('/registrations', + authenticate, + ensureUserHasRight(UserRight.MANAGE_REGISTRATIONS), + paginationValidator, + userRegistrationsSortValidator, + setDefaultSort, + setDefaultPagination, + listRegistrationsValidator, + asyncMiddleware(listRegistrations) +) + +registrationsRouter.post('/register', + registrationRateLimiter, + asyncMiddleware(ensureUserRegistrationAllowedFactory('direct-registration')), + ensureUserRegistrationAllowedForIP, + asyncMiddleware(usersDirectRegistrationValidator), + asyncRetryTransactionMiddleware(registerUser) +) + +// --------------------------------------------------------------------------- + +export { + registrationsRouter +} + +// --------------------------------------------------------------------------- + +async function requestRegistration (req: express.Request, res: express.Response) { + const body: UserRegistrationRequest = req.body + + const registration = new UserRegistrationModel({ + ...pick(body, [ 'username', 'password', 'email', 'registrationReason' ]), + + accountDisplayName: body.displayName, + channelDisplayName: body.channel?.displayName, + channelHandle: body.channel?.name, + + state: UserRegistrationState.PENDING, + + emailVerified: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION ? false : null + }) + + await registration.save() + + if (CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION) { + await sendVerifyRegistrationEmail(registration) + } + + Notifier.Instance.notifyOnNewRegistrationRequest(registration) + + Hooks.runAction('action:api.user.requested-registration', { body, registration, req, res }) + + return res.json(registration.toFormattedJSON()) +} + +// --------------------------------------------------------------------------- + +async function acceptRegistration (req: express.Request, res: express.Response) { + const registration = res.locals.userRegistration + const body: UserRegistrationUpdateState = req.body + + const userToCreate = buildUser({ + username: registration.username, + password: registration.password, + email: registration.email, + emailVerified: registration.emailVerified + }) + // We already encrypted password in registration model + userToCreate.skipPasswordEncryption = true + + // TODO: handle conflicts if someone else created a channel handle/user handle/user email between registration and approval + + const { user } = await createUserAccountAndChannelAndPlaylist({ + userToCreate, + userDisplayName: registration.accountDisplayName, + channelNames: registration.channelHandle && registration.channelDisplayName + ? { + name: registration.channelHandle, + displayName: registration.channelDisplayName + } + : undefined + }) + + registration.userId = user.id + registration.state = UserRegistrationState.ACCEPTED + registration.moderationResponse = body.moderationResponse + + await registration.save() + + logger.info('Registration of %s accepted', registration.username) + + if (body.preventEmailDelivery !== true) { + Emailer.Instance.addUserRegistrationRequestProcessedJob(registration) + } + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +async function rejectRegistration (req: express.Request, res: express.Response) { + const registration = res.locals.userRegistration + const body: UserRegistrationUpdateState = req.body + + registration.state = UserRegistrationState.REJECTED + registration.moderationResponse = body.moderationResponse + + await registration.save() + + if (body.preventEmailDelivery !== true) { + Emailer.Instance.addUserRegistrationRequestProcessedJob(registration) + } + + logger.info('Registration of %s rejected', registration.username) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +// --------------------------------------------------------------------------- + +async function deleteRegistration (req: express.Request, res: express.Response) { + const registration = res.locals.userRegistration + + await registration.destroy() + + logger.info('Registration of %s deleted', registration.username) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +// --------------------------------------------------------------------------- + +async function listRegistrations (req: express.Request, res: express.Response) { + const resultList = await UserRegistrationModel.listForApi({ + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + search: req.query.search + }) + + return res.json({ + total: resultList.total, + data: resultList.data.map(d => d.toFormattedJSON()) + }) +} + +// --------------------------------------------------------------------------- + +async function registerUser (req: express.Request, res: express.Response) { + const body: UserRegister = req.body + + const userToCreate = buildUser({ + ...pick(body, [ 'username', 'password', 'email' ]), + + emailVerified: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION ? false : null + }) + + const { user, account, videoChannel } = await createUserAccountAndChannelAndPlaylist({ + userToCreate, + userDisplayName: body.displayName || undefined, + channelNames: body.channel + }) + + auditLogger.create(body.username, new UserAuditView(user.toFormattedJSON())) + logger.info('User %s with its channel and account registered.', body.username) + + if (CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION) { + await sendVerifyUserEmail(user) + } + + Notifier.Instance.notifyOnNewDirectRegistration(user) + + Hooks.runAction('action:api.user.registered', { body, user, account, videoChannel, req, res }) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} diff --git a/server/server/controllers/api/users/token.ts b/server/server/controllers/api/users/token.ts new file mode 100644 index 000000000..a3f18f6e1 --- /dev/null +++ b/server/server/controllers/api/users/token.ts @@ -0,0 +1,131 @@ +import express from 'express' +import { ScopedToken } from '@peertube/peertube-models' +import { logger } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' +import { OTP } from '@server/initializers/constants.js' +import { getAuthNameFromRefreshGrant, getBypassFromExternalAuth, getBypassFromPasswordGrant } from '@server/lib/auth/external-auth.js' +import { BypassLogin, revokeToken } from '@server/lib/auth/oauth-model.js' +import { handleOAuthToken, MissingTwoFactorError } from '@server/lib/auth/oauth.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { asyncMiddleware, authenticate, buildRateLimiter, openapiOperationDoc } from '@server/middlewares/index.js' +import { buildUUID } from '@peertube/peertube-node-utils' + +const tokensRouter = express.Router() + +const loginRateLimiter = buildRateLimiter({ + windowMs: CONFIG.RATES_LIMIT.LOGIN.WINDOW_MS, + max: CONFIG.RATES_LIMIT.LOGIN.MAX +}) + +tokensRouter.post('/token', + loginRateLimiter, + openapiOperationDoc({ operationId: 'getOAuthToken' }), + asyncMiddleware(handleToken) +) + +tokensRouter.post('/revoke-token', + openapiOperationDoc({ operationId: 'revokeOAuthToken' }), + authenticate, + asyncMiddleware(handleTokenRevocation) +) + +tokensRouter.get('/scoped-tokens', + authenticate, + getScopedTokens +) + +tokensRouter.post('/scoped-tokens', + authenticate, + asyncMiddleware(renewScopedTokens) +) + +// --------------------------------------------------------------------------- + +export { + tokensRouter +} +// --------------------------------------------------------------------------- + +async function handleToken (req: express.Request, res: express.Response, next: express.NextFunction) { + const grantType = req.body.grant_type + + try { + const bypassLogin = await buildByPassLogin(req, grantType) + + const refreshTokenAuthName = grantType === 'refresh_token' + ? await getAuthNameFromRefreshGrant(req.body.refresh_token) + : undefined + + const options = { + refreshTokenAuthName, + bypassLogin + } + + const token = await handleOAuthToken(req, options) + + res.set('Cache-Control', 'no-store') + res.set('Pragma', 'no-cache') + + Hooks.runAction('action:api.user.oauth2-got-token', { username: token.user.username, ip: req.ip, req, res }) + + return res.json({ + token_type: 'Bearer', + + access_token: token.accessToken, + refresh_token: token.refreshToken, + + expires_in: token.accessTokenExpiresIn, + refresh_token_expires_in: token.refreshTokenExpiresIn + }) + } catch (err) { + logger.warn('Login error', { err }) + + if (err instanceof MissingTwoFactorError) { + res.set(OTP.HEADER_NAME, OTP.HEADER_REQUIRED_VALUE) + } + + return res.fail({ + status: err.code, + message: err.message, + type: err.name + }) + } +} + +async function handleTokenRevocation (req: express.Request, res: express.Response) { + const token = res.locals.oauth.token + + const result = await revokeToken(token, { req, explicitLogout: true }) + + return res.json(result) +} + +function getScopedTokens (req: express.Request, res: express.Response) { + const user = res.locals.oauth.token.user + + return res.json({ + feedToken: user.feedToken + } as ScopedToken) +} + +async function renewScopedTokens (req: express.Request, res: express.Response) { + const user = res.locals.oauth.token.user + + user.feedToken = buildUUID() + await user.save() + + return res.json({ + feedToken: user.feedToken + } as ScopedToken) +} + +async function buildByPassLogin (req: express.Request, grantType: string): Promise { + if (grantType !== 'password') return undefined + + if (req.body.externalAuthToken) { + // Consistency with the getBypassFromPasswordGrant promise + return getBypassFromExternalAuth(req.body.username, req.body.externalAuthToken) + } + + return getBypassFromPasswordGrant(req.body.username, req.body.password) +} diff --git a/server/server/controllers/api/users/two-factor.ts b/server/server/controllers/api/users/two-factor.ts new file mode 100644 index 000000000..f3dba3a53 --- /dev/null +++ b/server/server/controllers/api/users/two-factor.ts @@ -0,0 +1,95 @@ +import express from 'express' +import { generateOTPSecret, isOTPValid } from '@server/helpers/otp.js' +import { encrypt } from '@server/helpers/peertube-crypto.js' +import { CONFIG } from '@server/initializers/config.js' +import { Redis } from '@server/lib/redis.js' +import { asyncMiddleware, authenticate, usersCheckCurrentPasswordFactory } from '@server/middlewares/index.js' +import { + confirmTwoFactorValidator, + disableTwoFactorValidator, + requestOrConfirmTwoFactorValidator +} from '@server/middlewares/validators/two-factor.js' +import { HttpStatusCode, TwoFactorEnableResult } from '@peertube/peertube-models' + +const twoFactorRouter = express.Router() + +twoFactorRouter.post('/:id/two-factor/request', + authenticate, + asyncMiddleware(usersCheckCurrentPasswordFactory(req => req.params.id)), + asyncMiddleware(requestOrConfirmTwoFactorValidator), + asyncMiddleware(requestTwoFactor) +) + +twoFactorRouter.post('/:id/two-factor/confirm-request', + authenticate, + asyncMiddleware(requestOrConfirmTwoFactorValidator), + confirmTwoFactorValidator, + asyncMiddleware(confirmRequestTwoFactor) +) + +twoFactorRouter.post('/:id/two-factor/disable', + authenticate, + asyncMiddleware(usersCheckCurrentPasswordFactory(req => req.params.id)), + asyncMiddleware(disableTwoFactorValidator), + asyncMiddleware(disableTwoFactor) +) + +// --------------------------------------------------------------------------- + +export { + twoFactorRouter +} + +// --------------------------------------------------------------------------- + +async function requestTwoFactor (req: express.Request, res: express.Response) { + const user = res.locals.user + + const { secret, uri } = generateOTPSecret(user.email) + + const encryptedSecret = await encrypt(secret, CONFIG.SECRETS.PEERTUBE) + const requestToken = await Redis.Instance.setTwoFactorRequest(user.id, encryptedSecret) + + return res.json({ + otpRequest: { + requestToken, + secret, + uri + } + } as TwoFactorEnableResult) +} + +async function confirmRequestTwoFactor (req: express.Request, res: express.Response) { + const requestToken = req.body.requestToken + const otpToken = req.body.otpToken + const user = res.locals.user + + const encryptedSecret = await Redis.Instance.getTwoFactorRequestToken(user.id, requestToken) + if (!encryptedSecret) { + return res.fail({ + message: 'Invalid request token', + status: HttpStatusCode.FORBIDDEN_403 + }) + } + + if (await isOTPValid({ encryptedSecret, token: otpToken }) !== true) { + return res.fail({ + message: 'Invalid OTP token', + status: HttpStatusCode.FORBIDDEN_403 + }) + } + + user.otpSecret = encryptedSecret + await user.save() + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +async function disableTwoFactor (req: express.Request, res: express.Response) { + const user = res.locals.user + + user.otpSecret = null + await user.save() + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} diff --git a/server/server/controllers/api/video-channel-sync.ts b/server/server/controllers/api/video-channel-sync.ts new file mode 100644 index 000000000..fb1f05f7c --- /dev/null +++ b/server/server/controllers/api/video-channel-sync.ts @@ -0,0 +1,79 @@ +import express from 'express' +import { auditLoggerFactory, getAuditIdFromRes, VideoChannelSyncAuditView } from '@server/helpers/audit-logger.js' +import { logger } from '@server/helpers/logger.js' +import { + apiRateLimiter, + asyncMiddleware, + asyncRetryTransactionMiddleware, + authenticate, + ensureCanManageChannelOrAccount, + ensureSyncExists, + ensureSyncIsEnabled, + videoChannelSyncValidator +} from '@server/middlewares/index.js' +import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync.js' +import { MChannelSyncFormattable } from '@server/types/models/index.js' +import { HttpStatusCode, VideoChannelSyncState } from '@peertube/peertube-models' + +const videoChannelSyncRouter = express.Router() +const auditLogger = auditLoggerFactory('channel-syncs') + +videoChannelSyncRouter.use(apiRateLimiter) + +videoChannelSyncRouter.post('/', + authenticate, + ensureSyncIsEnabled, + asyncMiddleware(videoChannelSyncValidator), + ensureCanManageChannelOrAccount, + asyncRetryTransactionMiddleware(createVideoChannelSync) +) + +videoChannelSyncRouter.delete('/:id', + authenticate, + asyncMiddleware(ensureSyncExists), + ensureCanManageChannelOrAccount, + asyncRetryTransactionMiddleware(removeVideoChannelSync) +) + +export { videoChannelSyncRouter } + +// --------------------------------------------------------------------------- + +async function createVideoChannelSync (req: express.Request, res: express.Response) { + const syncCreated: MChannelSyncFormattable = new VideoChannelSyncModel({ + externalChannelUrl: req.body.externalChannelUrl, + videoChannelId: req.body.videoChannelId, + state: VideoChannelSyncState.WAITING_FIRST_RUN + }) + + await syncCreated.save() + syncCreated.VideoChannel = res.locals.videoChannel + + auditLogger.create(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncCreated.toFormattedJSON())) + + logger.info( + 'Video synchronization for channel "%s" with external channel "%s" created.', + syncCreated.VideoChannel.name, + syncCreated.externalChannelUrl + ) + + return res.json({ + videoChannelSync: syncCreated.toFormattedJSON() + }) +} + +async function removeVideoChannelSync (req: express.Request, res: express.Response) { + const syncInstance = res.locals.videoChannelSync + + await syncInstance.destroy() + + auditLogger.delete(getAuditIdFromRes(res), new VideoChannelSyncAuditView(syncInstance.toFormattedJSON())) + + logger.info( + 'Video synchronization for channel "%s" with external channel "%s" deleted.', + syncInstance.VideoChannel.name, + syncInstance.externalChannelUrl + ) + + return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() +} diff --git a/server/server/controllers/api/video-channel.ts b/server/server/controllers/api/video-channel.ts new file mode 100644 index 000000000..4add1a637 --- /dev/null +++ b/server/server/controllers/api/video-channel.ts @@ -0,0 +1,437 @@ +import express from 'express' +import { + ActorImageType, + HttpStatusCode, + VideoChannelCreate, + VideoChannelUpdate, + VideosImportInChannelCreate +} from '@peertube/peertube-models' +import { pickCommonVideoQuery } from '@server/helpers/query.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { ActorFollowModel } from '@server/models/actor/actor-follow.js' +import { getServerActor } from '@server/models/application/application.js' +import { MChannelBannerAccountDefault } from '@server/types/models/index.js' +import { auditLoggerFactory, getAuditIdFromRes, VideoChannelAuditView } from '../../helpers/audit-logger.js' +import { resetSequelizeInstance } from '../../helpers/database-utils.js' +import { buildNSFWFilter, createReqFiles, getCountVideos, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils.js' +import { logger } from '../../helpers/logger.js' +import { getFormattedObjects } from '../../helpers/utils.js' +import { MIMETYPES } from '../../initializers/constants.js' +import { sequelizeTypescript } from '../../initializers/database.js' +import { sendUpdateActor } from '../../lib/activitypub/send/index.js' +import { JobQueue } from '../../lib/job-queue/index.js' +import { deleteLocalActorImageFile, updateLocalActorImageFiles } from '../../lib/local-actor.js' +import { createLocalVideoChannel, federateAllVideosOfChannel } from '../../lib/video-channel.js' +import { + apiRateLimiter, + asyncMiddleware, + asyncRetryTransactionMiddleware, + authenticate, + commonVideosFiltersValidator, + ensureCanManageChannelOrAccount, + optionalAuthenticate, + paginationValidator, + setDefaultPagination, + setDefaultSort, + setDefaultVideosSort, + videoChannelsAddValidator, + videoChannelsRemoveValidator, + videoChannelsSortValidator, + videoChannelsUpdateValidator, + videoPlaylistsSortValidator +} from '../../middlewares/index.js' +import { updateAvatarValidator, updateBannerValidator } from '../../middlewares/validators/actor-image.js' +import { + ensureChannelOwnerCanUpload, + ensureIsLocalChannel, + videoChannelImportVideosValidator, + videoChannelsFollowersSortValidator, + videoChannelsListValidator, + videoChannelsNameWithHostValidator, + videosSortValidator +} from '../../middlewares/validators/index.js' +import { commonVideoPlaylistFiltersValidator } from '../../middlewares/validators/videos/video-playlists.js' +import { AccountModel } from '../../models/account/account.js' +import { guessAdditionalAttributesFromQuery } from '../../models/video/formatter/index.js' +import { VideoChannelModel } from '../../models/video/video-channel.js' +import { VideoPlaylistModel } from '../../models/video/video-playlist.js' +import { VideoModel } from '../../models/video/video.js' + +const auditLogger = auditLoggerFactory('channels') +const reqAvatarFile = createReqFiles([ 'avatarfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT) +const reqBannerFile = createReqFiles([ 'bannerfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT) + +const videoChannelRouter = express.Router() + +videoChannelRouter.use(apiRateLimiter) + +videoChannelRouter.get('/', + paginationValidator, + videoChannelsSortValidator, + setDefaultSort, + setDefaultPagination, + videoChannelsListValidator, + asyncMiddleware(listVideoChannels) +) + +videoChannelRouter.post('/', + authenticate, + asyncMiddleware(videoChannelsAddValidator), + asyncRetryTransactionMiddleware(addVideoChannel) +) + +videoChannelRouter.post('/:nameWithHost/avatar/pick', + authenticate, + reqAvatarFile, + asyncMiddleware(videoChannelsNameWithHostValidator), + ensureIsLocalChannel, + ensureCanManageChannelOrAccount, + updateAvatarValidator, + asyncMiddleware(updateVideoChannelAvatar) +) + +videoChannelRouter.post('/:nameWithHost/banner/pick', + authenticate, + reqBannerFile, + asyncMiddleware(videoChannelsNameWithHostValidator), + ensureIsLocalChannel, + ensureCanManageChannelOrAccount, + updateBannerValidator, + asyncMiddleware(updateVideoChannelBanner) +) + +videoChannelRouter.delete('/:nameWithHost/avatar', + authenticate, + asyncMiddleware(videoChannelsNameWithHostValidator), + ensureIsLocalChannel, + ensureCanManageChannelOrAccount, + asyncMiddleware(deleteVideoChannelAvatar) +) + +videoChannelRouter.delete('/:nameWithHost/banner', + authenticate, + asyncMiddleware(videoChannelsNameWithHostValidator), + ensureIsLocalChannel, + ensureCanManageChannelOrAccount, + asyncMiddleware(deleteVideoChannelBanner) +) + +videoChannelRouter.put('/:nameWithHost', + authenticate, + asyncMiddleware(videoChannelsNameWithHostValidator), + ensureIsLocalChannel, + ensureCanManageChannelOrAccount, + videoChannelsUpdateValidator, + asyncRetryTransactionMiddleware(updateVideoChannel) +) + +videoChannelRouter.delete('/:nameWithHost', + authenticate, + asyncMiddleware(videoChannelsNameWithHostValidator), + ensureIsLocalChannel, + ensureCanManageChannelOrAccount, + asyncMiddleware(videoChannelsRemoveValidator), + asyncRetryTransactionMiddleware(removeVideoChannel) +) + +videoChannelRouter.get('/:nameWithHost', + asyncMiddleware(videoChannelsNameWithHostValidator), + asyncMiddleware(getVideoChannel) +) + +videoChannelRouter.get('/:nameWithHost/video-playlists', + asyncMiddleware(videoChannelsNameWithHostValidator), + paginationValidator, + videoPlaylistsSortValidator, + setDefaultSort, + setDefaultPagination, + commonVideoPlaylistFiltersValidator, + asyncMiddleware(listVideoChannelPlaylists) +) + +videoChannelRouter.get('/:nameWithHost/videos', + asyncMiddleware(videoChannelsNameWithHostValidator), + paginationValidator, + videosSortValidator, + setDefaultVideosSort, + setDefaultPagination, + optionalAuthenticate, + commonVideosFiltersValidator, + asyncMiddleware(listVideoChannelVideos) +) + +videoChannelRouter.get('/:nameWithHost/followers', + authenticate, + asyncMiddleware(videoChannelsNameWithHostValidator), + ensureCanManageChannelOrAccount, + paginationValidator, + videoChannelsFollowersSortValidator, + setDefaultSort, + setDefaultPagination, + asyncMiddleware(listVideoChannelFollowers) +) + +videoChannelRouter.post('/:nameWithHost/import-videos', + authenticate, + asyncMiddleware(videoChannelsNameWithHostValidator), + asyncMiddleware(videoChannelImportVideosValidator), + ensureIsLocalChannel, + ensureCanManageChannelOrAccount, + asyncMiddleware(ensureChannelOwnerCanUpload), + asyncMiddleware(importVideosInChannel) +) + +// --------------------------------------------------------------------------- + +export { + videoChannelRouter +} + +// --------------------------------------------------------------------------- + +async function listVideoChannels (req: express.Request, res: express.Response) { + const serverActor = await getServerActor() + + const apiOptions = await Hooks.wrapObject({ + actorId: serverActor.id, + start: req.query.start, + count: req.query.count, + sort: req.query.sort + }, 'filter:api.video-channels.list.params') + + const resultList = await Hooks.wrapPromiseFun( + VideoChannelModel.listForApi, + apiOptions, + 'filter:api.video-channels.list.result' + ) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function updateVideoChannelBanner (req: express.Request, res: express.Response) { + const bannerPhysicalFile = req.files['bannerfile'][0] + const videoChannel = res.locals.videoChannel + const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannel.toFormattedJSON()) + + const banners = await updateLocalActorImageFiles(videoChannel, bannerPhysicalFile, ActorImageType.BANNER) + + auditLogger.update(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannel.toFormattedJSON()), oldVideoChannelAuditKeys) + + return res.json({ + banners: banners.map(b => b.toFormattedJSON()) + }) +} + +async function updateVideoChannelAvatar (req: express.Request, res: express.Response) { + const avatarPhysicalFile = req.files['avatarfile'][0] + const videoChannel = res.locals.videoChannel + const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannel.toFormattedJSON()) + + const avatars = await updateLocalActorImageFiles(videoChannel, avatarPhysicalFile, ActorImageType.AVATAR) + auditLogger.update(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannel.toFormattedJSON()), oldVideoChannelAuditKeys) + + return res.json({ + avatars: avatars.map(a => a.toFormattedJSON()) + }) +} + +async function deleteVideoChannelAvatar (req: express.Request, res: express.Response) { + const videoChannel = res.locals.videoChannel + + await deleteLocalActorImageFile(videoChannel, ActorImageType.AVATAR) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function deleteVideoChannelBanner (req: express.Request, res: express.Response) { + const videoChannel = res.locals.videoChannel + + await deleteLocalActorImageFile(videoChannel, ActorImageType.BANNER) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function addVideoChannel (req: express.Request, res: express.Response) { + const videoChannelInfo: VideoChannelCreate = req.body + + const videoChannelCreated = await sequelizeTypescript.transaction(async t => { + const account = await AccountModel.load(res.locals.oauth.token.User.Account.id, t) + + return createLocalVideoChannel(videoChannelInfo, account, t) + }) + + const payload = { actorId: videoChannelCreated.actorId } + await JobQueue.Instance.createJob({ type: 'actor-keys', payload }) + + auditLogger.create(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannelCreated.toFormattedJSON())) + logger.info('Video channel %s created.', videoChannelCreated.Actor.url) + + Hooks.runAction('action:api.video-channel.created', { videoChannel: videoChannelCreated, req, res }) + + return res.json({ + videoChannel: { + id: videoChannelCreated.id + } + }) +} + +async function updateVideoChannel (req: express.Request, res: express.Response) { + const videoChannelInstance = res.locals.videoChannel + const oldVideoChannelAuditKeys = new VideoChannelAuditView(videoChannelInstance.toFormattedJSON()) + const videoChannelInfoToUpdate = req.body as VideoChannelUpdate + let doBulkVideoUpdate = false + + try { + await sequelizeTypescript.transaction(async t => { + if (videoChannelInfoToUpdate.displayName !== undefined) videoChannelInstance.name = videoChannelInfoToUpdate.displayName + if (videoChannelInfoToUpdate.description !== undefined) videoChannelInstance.description = videoChannelInfoToUpdate.description + + if (videoChannelInfoToUpdate.support !== undefined) { + const oldSupportField = videoChannelInstance.support + videoChannelInstance.support = videoChannelInfoToUpdate.support + + if (videoChannelInfoToUpdate.bulkVideosSupportUpdate === true && oldSupportField !== videoChannelInfoToUpdate.support) { + doBulkVideoUpdate = true + await VideoModel.bulkUpdateSupportField(videoChannelInstance, t) + } + } + + const videoChannelInstanceUpdated = await videoChannelInstance.save({ transaction: t }) as MChannelBannerAccountDefault + await sendUpdateActor(videoChannelInstanceUpdated, t) + + auditLogger.update( + getAuditIdFromRes(res), + new VideoChannelAuditView(videoChannelInstanceUpdated.toFormattedJSON()), + oldVideoChannelAuditKeys + ) + + Hooks.runAction('action:api.video-channel.updated', { videoChannel: videoChannelInstanceUpdated, req, res }) + + logger.info('Video channel %s updated.', videoChannelInstance.Actor.url) + }) + } catch (err) { + logger.debug('Cannot update the video channel.', { err }) + + // If the transaction is retried, sequelize will think the object has not changed + // So we need to restore the previous fields + await resetSequelizeInstance(videoChannelInstance) + + throw err + } + + res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() + + // Don't process in a transaction, and after the response because it could be long + if (doBulkVideoUpdate) { + await federateAllVideosOfChannel(videoChannelInstance) + } +} + +async function removeVideoChannel (req: express.Request, res: express.Response) { + const videoChannelInstance = res.locals.videoChannel + + await sequelizeTypescript.transaction(async t => { + await VideoPlaylistModel.resetPlaylistsOfChannel(videoChannelInstance.id, t) + + await videoChannelInstance.destroy({ transaction: t }) + + Hooks.runAction('action:api.video-channel.deleted', { videoChannel: videoChannelInstance, req, res }) + + auditLogger.delete(getAuditIdFromRes(res), new VideoChannelAuditView(videoChannelInstance.toFormattedJSON())) + logger.info('Video channel %s deleted.', videoChannelInstance.Actor.url) + }) + + return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function getVideoChannel (req: express.Request, res: express.Response) { + const id = res.locals.videoChannel.id + const videoChannel = await Hooks.wrapObject(res.locals.videoChannel, 'filter:api.video-channel.get.result', { id }) + + if (videoChannel.isOutdated()) { + JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'actor', url: videoChannel.Actor.url } }) + } + + return res.json(videoChannel.toFormattedJSON()) +} + +async function listVideoChannelPlaylists (req: express.Request, res: express.Response) { + const serverActor = await getServerActor() + + const resultList = await VideoPlaylistModel.listForApi({ + followerActorId: serverActor.id, + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + videoChannelId: res.locals.videoChannel.id, + type: req.query.playlistType + }) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function listVideoChannelVideos (req: express.Request, res: express.Response) { + const serverActor = await getServerActor() + + const videoChannelInstance = res.locals.videoChannel + + const displayOnlyForFollower = isUserAbleToSearchRemoteURI(res) + ? null + : { + actorId: serverActor.id, + orLocalVideos: true + } + + const countVideos = getCountVideos(req) + const query = pickCommonVideoQuery(req.query) + + const apiOptions = await Hooks.wrapObject({ + ...query, + + displayOnlyForFollower, + nsfw: buildNSFWFilter(res, query.nsfw), + videoChannelId: videoChannelInstance.id, + user: res.locals.oauth ? res.locals.oauth.token.User : undefined, + countVideos + }, 'filter:api.video-channels.videos.list.params') + + const resultList = await Hooks.wrapPromiseFun( + VideoModel.listForApi, + apiOptions, + 'filter:api.video-channels.videos.list.result' + ) + + return res.json(getFormattedObjects(resultList.data, resultList.total, guessAdditionalAttributesFromQuery(query))) +} + +async function listVideoChannelFollowers (req: express.Request, res: express.Response) { + const channel = res.locals.videoChannel + + const resultList = await ActorFollowModel.listFollowersForApi({ + actorIds: [ channel.actorId ], + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + search: req.query.search, + state: 'accepted' + }) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function importVideosInChannel (req: express.Request, res: express.Response) { + const { externalChannelUrl } = req.body as VideosImportInChannelCreate + + await JobQueue.Instance.createJob({ + type: 'video-channel-import', + payload: { + externalChannelUrl, + videoChannelId: res.locals.videoChannel.id, + partOfChannelSyncId: res.locals.videoChannelSync?.id + } + }) + + logger.info('Video import job for channel "%s" with url "%s" created.', res.locals.videoChannel.name, externalChannelUrl) + + return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() +} diff --git a/server/server/controllers/api/video-playlist.ts b/server/server/controllers/api/video-playlist.ts new file mode 100644 index 000000000..64207f50d --- /dev/null +++ b/server/server/controllers/api/video-playlist.ts @@ -0,0 +1,518 @@ +import express from 'express' +import { forceNumber } from '@peertube/peertube-core-utils' +import { + HttpStatusCode, + VideoPlaylistCreate, + VideoPlaylistCreateResult, + VideoPlaylistElementCreate, + VideoPlaylistElementCreateResult, + VideoPlaylistElementUpdate, + VideoPlaylistPrivacy, + VideoPlaylistPrivacyType, + VideoPlaylistReorder, + VideoPlaylistUpdate +} from '@peertube/peertube-models' +import { scheduleRefreshIfNeeded } from '@server/lib/activitypub/playlists/index.js' +import { VideoMiniaturePermanentFileCache } from '@server/lib/files-cache/index.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { getServerActor } from '@server/models/application/application.js' +import { MVideoPlaylistFull, MVideoPlaylistThumbnail, MVideoThumbnail } from '@server/types/models/index.js' +import { uuidToShort } from '@peertube/peertube-node-utils' +import { resetSequelizeInstance } from '../../helpers/database-utils.js' +import { createReqFiles } from '../../helpers/express-utils.js' +import { logger } from '../../helpers/logger.js' +import { getFormattedObjects } from '../../helpers/utils.js' +import { MIMETYPES, VIDEO_PLAYLIST_PRIVACIES } from '../../initializers/constants.js' +import { sequelizeTypescript } from '../../initializers/database.js' +import { sendCreateVideoPlaylist, sendDeleteVideoPlaylist, sendUpdateVideoPlaylist } from '../../lib/activitypub/send/index.js' +import { getLocalVideoPlaylistActivityPubUrl, getLocalVideoPlaylistElementActivityPubUrl } from '../../lib/activitypub/url.js' +import { updateLocalPlaylistMiniatureFromExisting } from '../../lib/thumbnail.js' +import { + apiRateLimiter, + asyncMiddleware, + asyncRetryTransactionMiddleware, + authenticate, + optionalAuthenticate, + paginationValidator, + setDefaultPagination, + setDefaultSort +} from '../../middlewares/index.js' +import { videoPlaylistsSortValidator } from '../../middlewares/validators/index.js' +import { + commonVideoPlaylistFiltersValidator, + videoPlaylistsAddValidator, + videoPlaylistsAddVideoValidator, + videoPlaylistsDeleteValidator, + videoPlaylistsGetValidator, + videoPlaylistsReorderVideosValidator, + videoPlaylistsUpdateOrRemoveVideoValidator, + videoPlaylistsUpdateValidator +} from '../../middlewares/validators/videos/video-playlists.js' +import { AccountModel } from '../../models/account/account.js' +import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element.js' +import { VideoPlaylistModel } from '../../models/video/video-playlist.js' + +const reqThumbnailFile = createReqFiles([ 'thumbnailfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT) + +const videoPlaylistRouter = express.Router() + +videoPlaylistRouter.use(apiRateLimiter) + +videoPlaylistRouter.get('/privacies', listVideoPlaylistPrivacies) + +videoPlaylistRouter.get('/', + paginationValidator, + videoPlaylistsSortValidator, + setDefaultSort, + setDefaultPagination, + commonVideoPlaylistFiltersValidator, + asyncMiddleware(listVideoPlaylists) +) + +videoPlaylistRouter.get('/:playlistId', + asyncMiddleware(videoPlaylistsGetValidator('summary')), + getVideoPlaylist +) + +videoPlaylistRouter.post('/', + authenticate, + reqThumbnailFile, + asyncMiddleware(videoPlaylistsAddValidator), + asyncRetryTransactionMiddleware(addVideoPlaylist) +) + +videoPlaylistRouter.put('/:playlistId', + authenticate, + reqThumbnailFile, + asyncMiddleware(videoPlaylistsUpdateValidator), + asyncRetryTransactionMiddleware(updateVideoPlaylist) +) + +videoPlaylistRouter.delete('/:playlistId', + authenticate, + asyncMiddleware(videoPlaylistsDeleteValidator), + asyncRetryTransactionMiddleware(removeVideoPlaylist) +) + +videoPlaylistRouter.get('/:playlistId/videos', + asyncMiddleware(videoPlaylistsGetValidator('summary')), + paginationValidator, + setDefaultPagination, + optionalAuthenticate, + asyncMiddleware(getVideoPlaylistVideos) +) + +videoPlaylistRouter.post('/:playlistId/videos', + authenticate, + asyncMiddleware(videoPlaylistsAddVideoValidator), + asyncRetryTransactionMiddleware(addVideoInPlaylist) +) + +videoPlaylistRouter.post('/:playlistId/videos/reorder', + authenticate, + asyncMiddleware(videoPlaylistsReorderVideosValidator), + asyncRetryTransactionMiddleware(reorderVideosPlaylist) +) + +videoPlaylistRouter.put('/:playlistId/videos/:playlistElementId', + authenticate, + asyncMiddleware(videoPlaylistsUpdateOrRemoveVideoValidator), + asyncRetryTransactionMiddleware(updateVideoPlaylistElement) +) + +videoPlaylistRouter.delete('/:playlistId/videos/:playlistElementId', + authenticate, + asyncMiddleware(videoPlaylistsUpdateOrRemoveVideoValidator), + asyncRetryTransactionMiddleware(removeVideoFromPlaylist) +) + +// --------------------------------------------------------------------------- + +export { + videoPlaylistRouter +} + +// --------------------------------------------------------------------------- + +function listVideoPlaylistPrivacies (req: express.Request, res: express.Response) { + res.json(VIDEO_PLAYLIST_PRIVACIES) +} + +async function listVideoPlaylists (req: express.Request, res: express.Response) { + const serverActor = await getServerActor() + const resultList = await VideoPlaylistModel.listForApi({ + followerActorId: serverActor.id, + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + type: req.query.playlistType + }) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +function getVideoPlaylist (req: express.Request, res: express.Response) { + const videoPlaylist = res.locals.videoPlaylistSummary + + scheduleRefreshIfNeeded(videoPlaylist) + + return res.json(videoPlaylist.toFormattedJSON()) +} + +async function addVideoPlaylist (req: express.Request, res: express.Response) { + const videoPlaylistInfo: VideoPlaylistCreate = req.body + const user = res.locals.oauth.token.User + + const videoPlaylist = new VideoPlaylistModel({ + name: videoPlaylistInfo.displayName, + description: videoPlaylistInfo.description, + privacy: videoPlaylistInfo.privacy || VideoPlaylistPrivacy.PRIVATE, + ownerAccountId: user.Account.id + }) as MVideoPlaylistFull + + videoPlaylist.url = getLocalVideoPlaylistActivityPubUrl(videoPlaylist) // We use the UUID, so set the URL after building the object + + if (videoPlaylistInfo.videoChannelId) { + const videoChannel = res.locals.videoChannel + + videoPlaylist.videoChannelId = videoChannel.id + videoPlaylist.VideoChannel = videoChannel + } + + const thumbnailField = req.files['thumbnailfile'] + const thumbnailModel = thumbnailField + ? await updateLocalPlaylistMiniatureFromExisting({ + inputPath: thumbnailField[0].path, + playlist: videoPlaylist, + automaticallyGenerated: false + }) + : undefined + + const videoPlaylistCreated = await sequelizeTypescript.transaction(async t => { + const videoPlaylistCreated = await videoPlaylist.save({ transaction: t }) as MVideoPlaylistFull + + if (thumbnailModel) { + thumbnailModel.automaticallyGenerated = false + await videoPlaylistCreated.setAndSaveThumbnail(thumbnailModel, t) + } + + // We need more attributes for the federation + videoPlaylistCreated.OwnerAccount = await AccountModel.load(user.Account.id, t) + await sendCreateVideoPlaylist(videoPlaylistCreated, t) + + return videoPlaylistCreated + }) + + logger.info('Video playlist with uuid %s created.', videoPlaylist.uuid) + + return res.json({ + videoPlaylist: { + id: videoPlaylistCreated.id, + shortUUID: uuidToShort(videoPlaylistCreated.uuid), + uuid: videoPlaylistCreated.uuid + } as VideoPlaylistCreateResult + }) +} + +async function updateVideoPlaylist (req: express.Request, res: express.Response) { + const videoPlaylistInstance = res.locals.videoPlaylistFull + const videoPlaylistInfoToUpdate = req.body as VideoPlaylistUpdate + + const wasPrivatePlaylist = videoPlaylistInstance.privacy === VideoPlaylistPrivacy.PRIVATE + const wasNotPrivatePlaylist = videoPlaylistInstance.privacy !== VideoPlaylistPrivacy.PRIVATE + + const thumbnailField = req.files['thumbnailfile'] + const thumbnailModel = thumbnailField + ? await updateLocalPlaylistMiniatureFromExisting({ + inputPath: thumbnailField[0].path, + playlist: videoPlaylistInstance, + automaticallyGenerated: false + }) + : undefined + + try { + await sequelizeTypescript.transaction(async t => { + const sequelizeOptions = { + transaction: t + } + + if (videoPlaylistInfoToUpdate.videoChannelId !== undefined) { + if (videoPlaylistInfoToUpdate.videoChannelId === null) { + videoPlaylistInstance.videoChannelId = null + } else { + const videoChannel = res.locals.videoChannel + + videoPlaylistInstance.videoChannelId = videoChannel.id + videoPlaylistInstance.VideoChannel = videoChannel + } + } + + if (videoPlaylistInfoToUpdate.displayName !== undefined) videoPlaylistInstance.name = videoPlaylistInfoToUpdate.displayName + if (videoPlaylistInfoToUpdate.description !== undefined) videoPlaylistInstance.description = videoPlaylistInfoToUpdate.description + + if (videoPlaylistInfoToUpdate.privacy !== undefined) { + videoPlaylistInstance.privacy = forceNumber(videoPlaylistInfoToUpdate.privacy) as VideoPlaylistPrivacyType + + if (wasNotPrivatePlaylist === true && videoPlaylistInstance.privacy === VideoPlaylistPrivacy.PRIVATE) { + await sendDeleteVideoPlaylist(videoPlaylistInstance, t) + } + } + + const playlistUpdated = await videoPlaylistInstance.save(sequelizeOptions) + + if (thumbnailModel) { + thumbnailModel.automaticallyGenerated = false + await playlistUpdated.setAndSaveThumbnail(thumbnailModel, t) + } + + const isNewPlaylist = wasPrivatePlaylist && playlistUpdated.privacy !== VideoPlaylistPrivacy.PRIVATE + + if (isNewPlaylist) { + await sendCreateVideoPlaylist(playlistUpdated, t) + } else { + await sendUpdateVideoPlaylist(playlistUpdated, t) + } + + logger.info('Video playlist %s updated.', videoPlaylistInstance.uuid) + + return playlistUpdated + }) + } catch (err) { + logger.debug('Cannot update the video playlist.', { err }) + + // If the transaction is retried, sequelize will think the object has not changed + // So we need to restore the previous fields + await resetSequelizeInstance(videoPlaylistInstance) + + throw err + } + + return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function removeVideoPlaylist (req: express.Request, res: express.Response) { + const videoPlaylistInstance = res.locals.videoPlaylistSummary + + await sequelizeTypescript.transaction(async t => { + await videoPlaylistInstance.destroy({ transaction: t }) + + await sendDeleteVideoPlaylist(videoPlaylistInstance, t) + + logger.info('Video playlist %s deleted.', videoPlaylistInstance.uuid) + }) + + return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function addVideoInPlaylist (req: express.Request, res: express.Response) { + const body: VideoPlaylistElementCreate = req.body + const videoPlaylist = res.locals.videoPlaylistFull + const video = res.locals.onlyVideo + + const playlistElement = await sequelizeTypescript.transaction(async t => { + const position = await VideoPlaylistElementModel.getNextPositionOf(videoPlaylist.id, t) + + const playlistElement = await VideoPlaylistElementModel.create({ + position, + startTimestamp: body.startTimestamp || null, + stopTimestamp: body.stopTimestamp || null, + videoPlaylistId: videoPlaylist.id, + videoId: video.id + }, { transaction: t }) + + playlistElement.url = getLocalVideoPlaylistElementActivityPubUrl(videoPlaylist, playlistElement) + await playlistElement.save({ transaction: t }) + + videoPlaylist.changed('updatedAt', true) + await videoPlaylist.save({ transaction: t }) + + return playlistElement + }) + + // If the user did not set a thumbnail, automatically take the video thumbnail + if (videoPlaylist.hasThumbnail() === false || (videoPlaylist.hasGeneratedThumbnail() && playlistElement.position === 1)) { + await generateThumbnailForPlaylist(videoPlaylist, video) + } + + sendUpdateVideoPlaylist(videoPlaylist, undefined) + .catch(err => logger.error('Cannot send video playlist update.', { err })) + + logger.info('Video added in playlist %s at position %d.', videoPlaylist.uuid, playlistElement.position) + + Hooks.runAction('action:api.video-playlist-element.created', { playlistElement, req, res }) + + return res.json({ + videoPlaylistElement: { + id: playlistElement.id + } as VideoPlaylistElementCreateResult + }) +} + +async function updateVideoPlaylistElement (req: express.Request, res: express.Response) { + const body: VideoPlaylistElementUpdate = req.body + const videoPlaylist = res.locals.videoPlaylistFull + const videoPlaylistElement = res.locals.videoPlaylistElement + + const playlistElement: VideoPlaylistElementModel = await sequelizeTypescript.transaction(async t => { + if (body.startTimestamp !== undefined) videoPlaylistElement.startTimestamp = body.startTimestamp + if (body.stopTimestamp !== undefined) videoPlaylistElement.stopTimestamp = body.stopTimestamp + + const element = await videoPlaylistElement.save({ transaction: t }) + + videoPlaylist.changed('updatedAt', true) + await videoPlaylist.save({ transaction: t }) + + await sendUpdateVideoPlaylist(videoPlaylist, t) + + return element + }) + + logger.info('Element of position %d of playlist %s updated.', playlistElement.position, videoPlaylist.uuid) + + return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function removeVideoFromPlaylist (req: express.Request, res: express.Response) { + const videoPlaylistElement = res.locals.videoPlaylistElement + const videoPlaylist = res.locals.videoPlaylistFull + const positionToDelete = videoPlaylistElement.position + + await sequelizeTypescript.transaction(async t => { + await videoPlaylistElement.destroy({ transaction: t }) + + // Decrease position of the next elements + await VideoPlaylistElementModel.increasePositionOf(videoPlaylist.id, positionToDelete, -1, t) + + videoPlaylist.changed('updatedAt', true) + await videoPlaylist.save({ transaction: t }) + + logger.info('Video playlist element %d of playlist %s deleted.', videoPlaylistElement.position, videoPlaylist.uuid) + }) + + // Do we need to regenerate the default thumbnail? + if (positionToDelete === 1 && videoPlaylist.hasGeneratedThumbnail()) { + await regeneratePlaylistThumbnail(videoPlaylist) + } + + sendUpdateVideoPlaylist(videoPlaylist, undefined) + .catch(err => logger.error('Cannot send video playlist update.', { err })) + + return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function reorderVideosPlaylist (req: express.Request, res: express.Response) { + const videoPlaylist = res.locals.videoPlaylistFull + const body: VideoPlaylistReorder = req.body + + const start: number = body.startPosition + const insertAfter: number = body.insertAfterPosition + const reorderLength: number = body.reorderLength || 1 + + if (start === insertAfter) { + return res.status(HttpStatusCode.NO_CONTENT_204).end() + } + + // Example: if we reorder position 2 and insert after position 5 (so at position 6): # 1 2 3 4 5 6 7 8 9 + // * increase position when position > 5 # 1 2 3 4 5 7 8 9 10 + // * update position 2 -> position 6 # 1 3 4 5 6 7 8 9 10 + // * decrease position when position position > 2 # 1 2 3 4 5 6 7 8 9 + await sequelizeTypescript.transaction(async t => { + const newPosition = insertAfter + 1 + + // Add space after the position when we want to insert our reordered elements (increase) + await VideoPlaylistElementModel.increasePositionOf(videoPlaylist.id, newPosition, reorderLength, t) + + let oldPosition = start + + // We incremented the position of the elements we want to reorder + if (start >= newPosition) oldPosition += reorderLength + + const endOldPosition = oldPosition + reorderLength - 1 + // Insert our reordered elements in their place (update) + await VideoPlaylistElementModel.reassignPositionOf({ + videoPlaylistId: videoPlaylist.id, + firstPosition: oldPosition, + endPosition: endOldPosition, + newPosition, + transaction: t + }) + + // Decrease positions of elements after the old position of our ordered elements (decrease) + await VideoPlaylistElementModel.increasePositionOf(videoPlaylist.id, oldPosition, -reorderLength, t) + + videoPlaylist.changed('updatedAt', true) + await videoPlaylist.save({ transaction: t }) + + await sendUpdateVideoPlaylist(videoPlaylist, t) + }) + + // The first element changed + if ((start === 1 || insertAfter === 0) && videoPlaylist.hasGeneratedThumbnail()) { + await regeneratePlaylistThumbnail(videoPlaylist) + } + + logger.info( + 'Reordered playlist %s (inserted after position %d elements %d - %d).', + videoPlaylist.uuid, insertAfter, start, start + reorderLength - 1 + ) + + return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function getVideoPlaylistVideos (req: express.Request, res: express.Response) { + const videoPlaylistInstance = res.locals.videoPlaylistSummary + const user = res.locals.oauth ? res.locals.oauth.token.User : undefined + const server = await getServerActor() + + const apiOptions = await Hooks.wrapObject({ + start: req.query.start, + count: req.query.count, + videoPlaylistId: videoPlaylistInstance.id, + serverAccount: server.Account, + user + }, 'filter:api.video-playlist.videos.list.params') + + const resultList = await Hooks.wrapPromiseFun( + VideoPlaylistElementModel.listForApi, + apiOptions, + 'filter:api.video-playlist.videos.list.result' + ) + + const options = { accountId: user?.Account?.id } + return res.json(getFormattedObjects(resultList.data, resultList.total, options)) +} + +async function regeneratePlaylistThumbnail (videoPlaylist: MVideoPlaylistThumbnail) { + await videoPlaylist.Thumbnail.destroy() + videoPlaylist.Thumbnail = null + + const firstElement = await VideoPlaylistElementModel.loadFirstElementWithVideoThumbnail(videoPlaylist.id) + if (firstElement) await generateThumbnailForPlaylist(videoPlaylist, firstElement.Video) +} + +async function generateThumbnailForPlaylist (videoPlaylist: MVideoPlaylistThumbnail, video: MVideoThumbnail) { + logger.info('Generating default thumbnail to playlist %s.', videoPlaylist.url) + + const videoMiniature = video.getMiniature() + if (!videoMiniature) { + logger.info('Cannot generate thumbnail for playlist %s because video %s does not have any.', videoPlaylist.url, video.url) + return + } + + // Ensure the file is on disk + const videoMiniaturePermanentFileCache = new VideoMiniaturePermanentFileCache() + const inputPath = videoMiniature.isOwned() + ? videoMiniature.getPath() + : await videoMiniaturePermanentFileCache.downloadRemoteFile(videoMiniature) + + const thumbnailModel = await updateLocalPlaylistMiniatureFromExisting({ + inputPath, + playlist: videoPlaylist, + automaticallyGenerated: true, + keepOriginal: true + }) + + thumbnailModel.videoPlaylistId = videoPlaylist.id + + videoPlaylist.Thumbnail = await thumbnailModel.save() +} diff --git a/server/server/controllers/api/videos/blacklist.ts b/server/server/controllers/api/videos/blacklist.ts new file mode 100644 index 000000000..1facbc5d3 --- /dev/null +++ b/server/server/controllers/api/videos/blacklist.ts @@ -0,0 +1,112 @@ +import express from 'express' +import { blacklistVideo, unblacklistVideo } from '@server/lib/video-blacklist.js' +import { HttpStatusCode, UserRight, VideoBlacklistCreate } from '@peertube/peertube-models' +import { logger } from '../../../helpers/logger.js' +import { getFormattedObjects } from '../../../helpers/utils.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { + asyncMiddleware, + authenticate, + blacklistSortValidator, + ensureUserHasRight, + openapiOperationDoc, + paginationValidator, + setBlacklistSort, + setDefaultPagination, + videosBlacklistAddValidator, + videosBlacklistFiltersValidator, + videosBlacklistRemoveValidator, + videosBlacklistUpdateValidator +} from '../../../middlewares/index.js' +import { VideoBlacklistModel } from '../../../models/video/video-blacklist.js' + +const blacklistRouter = express.Router() + +blacklistRouter.post('/:videoId/blacklist', + openapiOperationDoc({ operationId: 'addVideoBlock' }), + authenticate, + ensureUserHasRight(UserRight.MANAGE_VIDEO_BLACKLIST), + asyncMiddleware(videosBlacklistAddValidator), + asyncMiddleware(addVideoToBlacklistController) +) + +blacklistRouter.get('/blacklist', + openapiOperationDoc({ operationId: 'getVideoBlocks' }), + authenticate, + ensureUserHasRight(UserRight.MANAGE_VIDEO_BLACKLIST), + paginationValidator, + blacklistSortValidator, + setBlacklistSort, + setDefaultPagination, + videosBlacklistFiltersValidator, + asyncMiddleware(listBlacklist) +) + +blacklistRouter.put('/:videoId/blacklist', + authenticate, + ensureUserHasRight(UserRight.MANAGE_VIDEO_BLACKLIST), + asyncMiddleware(videosBlacklistUpdateValidator), + asyncMiddleware(updateVideoBlacklistController) +) + +blacklistRouter.delete('/:videoId/blacklist', + openapiOperationDoc({ operationId: 'delVideoBlock' }), + authenticate, + ensureUserHasRight(UserRight.MANAGE_VIDEO_BLACKLIST), + asyncMiddleware(videosBlacklistRemoveValidator), + asyncMiddleware(removeVideoFromBlacklistController) +) + +// --------------------------------------------------------------------------- + +export { + blacklistRouter +} + +// --------------------------------------------------------------------------- + +async function addVideoToBlacklistController (req: express.Request, res: express.Response) { + const videoInstance = res.locals.videoAll + const body: VideoBlacklistCreate = req.body + + await blacklistVideo(videoInstance, body) + + logger.info('Video %s blacklisted.', videoInstance.uuid) + + return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function updateVideoBlacklistController (req: express.Request, res: express.Response) { + const videoBlacklist = res.locals.videoBlacklist + + if (req.body.reason !== undefined) videoBlacklist.reason = req.body.reason + + await sequelizeTypescript.transaction(t => { + return videoBlacklist.save({ transaction: t }) + }) + + return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function listBlacklist (req: express.Request, res: express.Response) { + const resultList = await VideoBlacklistModel.listForApi({ + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + search: req.query.search, + type: req.query.type + }) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function removeVideoFromBlacklistController (req: express.Request, res: express.Response) { + const videoBlacklist = res.locals.videoBlacklist + const video = res.locals.videoAll + + await unblacklistVideo(videoBlacklist, video) + + logger.info('Video %s removed from blacklist.', video.uuid) + + return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() +} diff --git a/server/server/controllers/api/videos/captions.ts b/server/server/controllers/api/videos/captions.ts new file mode 100644 index 000000000..b8e7149c6 --- /dev/null +++ b/server/server/controllers/api/videos/captions.ts @@ -0,0 +1,93 @@ +import express from 'express' +import { HttpStatusCode } from '@peertube/peertube-models' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { MVideoCaption } from '@server/types/models/index.js' +import { moveAndProcessCaptionFile } from '../../../helpers/captions-utils.js' +import { createReqFiles } from '../../../helpers/express-utils.js' +import { logger } from '../../../helpers/logger.js' +import { getFormattedObjects } from '../../../helpers/utils.js' +import { MIMETYPES } from '../../../initializers/constants.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { federateVideoIfNeeded } from '../../../lib/activitypub/videos/index.js' +import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate } from '../../../middlewares/index.js' +import { addVideoCaptionValidator, deleteVideoCaptionValidator, listVideoCaptionsValidator } from '../../../middlewares/validators/index.js' +import { VideoCaptionModel } from '../../../models/video/video-caption.js' + +const reqVideoCaptionAdd = createReqFiles([ 'captionfile' ], MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT) + +const videoCaptionsRouter = express.Router() + +videoCaptionsRouter.get('/:videoId/captions', + asyncMiddleware(listVideoCaptionsValidator), + asyncMiddleware(listVideoCaptions) +) +videoCaptionsRouter.put('/:videoId/captions/:captionLanguage', + authenticate, + reqVideoCaptionAdd, + asyncMiddleware(addVideoCaptionValidator), + asyncRetryTransactionMiddleware(addVideoCaption) +) +videoCaptionsRouter.delete('/:videoId/captions/:captionLanguage', + authenticate, + asyncMiddleware(deleteVideoCaptionValidator), + asyncRetryTransactionMiddleware(deleteVideoCaption) +) + +// --------------------------------------------------------------------------- + +export { + videoCaptionsRouter +} + +// --------------------------------------------------------------------------- + +async function listVideoCaptions (req: express.Request, res: express.Response) { + const data = await VideoCaptionModel.listVideoCaptions(res.locals.onlyVideo.id) + + return res.json(getFormattedObjects(data, data.length)) +} + +async function addVideoCaption (req: express.Request, res: express.Response) { + const videoCaptionPhysicalFile = req.files['captionfile'][0] + const video = res.locals.videoAll + + const captionLanguage = req.params.captionLanguage + + const videoCaption = new VideoCaptionModel({ + videoId: video.id, + filename: VideoCaptionModel.generateCaptionName(captionLanguage), + language: captionLanguage + }) as MVideoCaption + + // Move physical file + await moveAndProcessCaptionFile(videoCaptionPhysicalFile, videoCaption) + + await sequelizeTypescript.transaction(async t => { + await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t) + + // Update video update + await federateVideoIfNeeded(video, false, t) + }) + + Hooks.runAction('action:api.video-caption.created', { caption: videoCaption, req, res }) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function deleteVideoCaption (req: express.Request, res: express.Response) { + const video = res.locals.videoAll + const videoCaption = res.locals.videoCaption + + await sequelizeTypescript.transaction(async t => { + await videoCaption.destroy({ transaction: t }) + + // Send video update + await federateVideoIfNeeded(video, false, t) + }) + + logger.info('Video caption %s of video %s deleted.', videoCaption.language, video.uuid) + + Hooks.runAction('action:api.video-caption.deleted', { caption: videoCaption, req, res }) + + return res.type('json').status(HttpStatusCode.NO_CONTENT_204).end() +} diff --git a/server/server/controllers/api/videos/comment.ts b/server/server/controllers/api/videos/comment.ts new file mode 100644 index 000000000..ae8994382 --- /dev/null +++ b/server/server/controllers/api/videos/comment.ts @@ -0,0 +1,238 @@ +import express from 'express' +import { + HttpStatusCode, + ResultList, + ThreadsResultList, + UserRight, + VideoCommentCreate, + VideoCommentThreads +} from '@peertube/peertube-models' +import { MCommentFormattable } from '@server/types/models/index.js' +import { auditLoggerFactory, CommentAuditView, getAuditIdFromRes } from '../../../helpers/audit-logger.js' +import { getFormattedObjects } from '../../../helpers/utils.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { Notifier } from '../../../lib/notifier/index.js' +import { Hooks } from '../../../lib/plugins/hooks.js' +import { buildFormattedCommentTree, createVideoComment, removeComment } from '../../../lib/video-comment.js' +import { + asyncMiddleware, + asyncRetryTransactionMiddleware, + authenticate, + ensureUserHasRight, + optionalAuthenticate, + paginationValidator, + setDefaultPagination, + setDefaultSort +} from '../../../middlewares/index.js' +import { + addVideoCommentReplyValidator, + addVideoCommentThreadValidator, + listVideoCommentsValidator, + listVideoCommentThreadsValidator, + listVideoThreadCommentsValidator, + removeVideoCommentValidator, + videoCommentsValidator, + videoCommentThreadsSortValidator +} from '../../../middlewares/validators/index.js' +import { AccountModel } from '../../../models/account/account.js' +import { VideoCommentModel } from '../../../models/video/video-comment.js' + +const auditLogger = auditLoggerFactory('comments') +const videoCommentRouter = express.Router() + +videoCommentRouter.get('/:videoId/comment-threads', + paginationValidator, + videoCommentThreadsSortValidator, + setDefaultSort, + setDefaultPagination, + asyncMiddleware(listVideoCommentThreadsValidator), + optionalAuthenticate, + asyncMiddleware(listVideoThreads) +) +videoCommentRouter.get('/:videoId/comment-threads/:threadId', + asyncMiddleware(listVideoThreadCommentsValidator), + optionalAuthenticate, + asyncMiddleware(listVideoThreadComments) +) + +videoCommentRouter.post('/:videoId/comment-threads', + authenticate, + asyncMiddleware(addVideoCommentThreadValidator), + asyncRetryTransactionMiddleware(addVideoCommentThread) +) +videoCommentRouter.post('/:videoId/comments/:commentId', + authenticate, + asyncMiddleware(addVideoCommentReplyValidator), + asyncRetryTransactionMiddleware(addVideoCommentReply) +) +videoCommentRouter.delete('/:videoId/comments/:commentId', + authenticate, + asyncMiddleware(removeVideoCommentValidator), + asyncRetryTransactionMiddleware(removeVideoComment) +) + +videoCommentRouter.get('/comments', + authenticate, + ensureUserHasRight(UserRight.SEE_ALL_COMMENTS), + paginationValidator, + videoCommentsValidator, + setDefaultSort, + setDefaultPagination, + listVideoCommentsValidator, + asyncMiddleware(listComments) +) + +// --------------------------------------------------------------------------- + +export { + videoCommentRouter +} + +// --------------------------------------------------------------------------- + +async function listComments (req: express.Request, res: express.Response) { + const options = { + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + + isLocal: req.query.isLocal, + onLocalVideo: req.query.onLocalVideo, + search: req.query.search, + searchAccount: req.query.searchAccount, + searchVideo: req.query.searchVideo + } + + const resultList = await VideoCommentModel.listCommentsForApi(options) + + return res.json({ + total: resultList.total, + data: resultList.data.map(c => c.toFormattedAdminJSON()) + }) +} + +async function listVideoThreads (req: express.Request, res: express.Response) { + const video = res.locals.onlyVideo + const user = res.locals.oauth ? res.locals.oauth.token.User : undefined + + let resultList: ThreadsResultList + + if (video.commentsEnabled === true) { + const apiOptions = await Hooks.wrapObject({ + videoId: video.id, + isVideoOwned: video.isOwned(), + start: req.query.start, + count: req.query.count, + sort: req.query.sort, + user + }, 'filter:api.video-threads.list.params') + + resultList = await Hooks.wrapPromiseFun( + VideoCommentModel.listThreadsForApi, + apiOptions, + 'filter:api.video-threads.list.result' + ) + } else { + resultList = { + total: 0, + totalNotDeletedComments: 0, + data: [] + } + } + + return res.json({ + ...getFormattedObjects(resultList.data, resultList.total), + totalNotDeletedComments: resultList.totalNotDeletedComments + } as VideoCommentThreads) +} + +async function listVideoThreadComments (req: express.Request, res: express.Response) { + const video = res.locals.onlyVideo + const user = res.locals.oauth ? res.locals.oauth.token.User : undefined + + let resultList: ResultList + + if (video.commentsEnabled === true) { + const apiOptions = await Hooks.wrapObject({ + videoId: video.id, + threadId: res.locals.videoCommentThread.id, + user + }, 'filter:api.video-thread-comments.list.params') + + resultList = await Hooks.wrapPromiseFun( + VideoCommentModel.listThreadCommentsForApi, + apiOptions, + 'filter:api.video-thread-comments.list.result' + ) + } else { + resultList = { + total: 0, + data: [] + } + } + + if (resultList.data.length === 0) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'No comments were found' + }) + } + + return res.json(buildFormattedCommentTree(resultList)) +} + +async function addVideoCommentThread (req: express.Request, res: express.Response) { + const videoCommentInfo: VideoCommentCreate = req.body + + const comment = await sequelizeTypescript.transaction(async t => { + const account = await AccountModel.load(res.locals.oauth.token.User.Account.id, t) + + return createVideoComment({ + text: videoCommentInfo.text, + inReplyToComment: null, + video: res.locals.videoAll, + account + }, t) + }) + + Notifier.Instance.notifyOnNewComment(comment) + auditLogger.create(getAuditIdFromRes(res), new CommentAuditView(comment.toFormattedJSON())) + + Hooks.runAction('action:api.video-thread.created', { comment, req, res }) + + return res.json({ comment: comment.toFormattedJSON() }) +} + +async function addVideoCommentReply (req: express.Request, res: express.Response) { + const videoCommentInfo: VideoCommentCreate = req.body + + const comment = await sequelizeTypescript.transaction(async t => { + const account = await AccountModel.load(res.locals.oauth.token.User.Account.id, t) + + return createVideoComment({ + text: videoCommentInfo.text, + inReplyToComment: res.locals.videoCommentFull, + video: res.locals.videoAll, + account + }, t) + }) + + Notifier.Instance.notifyOnNewComment(comment) + auditLogger.create(getAuditIdFromRes(res), new CommentAuditView(comment.toFormattedJSON())) + + Hooks.runAction('action:api.video-comment-reply.created', { comment, req, res }) + + return res.json({ comment: comment.toFormattedJSON() }) +} + +async function removeVideoComment (req: express.Request, res: express.Response) { + const videoCommentInstance = res.locals.videoCommentFull + + await removeComment(videoCommentInstance, req, res) + + auditLogger.delete(getAuditIdFromRes(res), new CommentAuditView(videoCommentInstance.toFormattedJSON())) + + return res.type('json') + .status(HttpStatusCode.NO_CONTENT_204) + .end() +} diff --git a/server/server/controllers/api/videos/files.ts b/server/server/controllers/api/videos/files.ts new file mode 100644 index 000000000..c62c85c54 --- /dev/null +++ b/server/server/controllers/api/videos/files.ts @@ -0,0 +1,122 @@ +import express from 'express' +import validator from 'validator' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js' +import { updatePlaylistAfterFileChange } from '@server/lib/hls.js' +import { removeAllWebVideoFiles, removeHLSFile, removeHLSPlaylist, removeWebVideoFile } from '@server/lib/video-file.js' +import { VideoFileModel } from '@server/models/video/video-file.js' +import { HttpStatusCode, UserRight } from '@peertube/peertube-models' +import { + asyncMiddleware, + authenticate, + ensureUserHasRight, + videoFileMetadataGetValidator, + videoFilesDeleteHLSFileValidator, + videoFilesDeleteHLSValidator, + videoFilesDeleteWebVideoFileValidator, + videoFilesDeleteWebVideoValidator, + videosGetValidator +} from '../../../middlewares/index.js' + +const lTags = loggerTagsFactory('api', 'video') +const filesRouter = express.Router() + +filesRouter.get('/:id/metadata/:videoFileId', + asyncMiddleware(videosGetValidator), + asyncMiddleware(videoFileMetadataGetValidator), + asyncMiddleware(getVideoFileMetadata) +) + +filesRouter.delete('/:id/hls', + authenticate, + ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES), + asyncMiddleware(videoFilesDeleteHLSValidator), + asyncMiddleware(removeHLSPlaylistController) +) +filesRouter.delete('/:id/hls/:videoFileId', + authenticate, + ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES), + asyncMiddleware(videoFilesDeleteHLSFileValidator), + asyncMiddleware(removeHLSFileController) +) + +filesRouter.delete( + [ '/:id/webtorrent', '/:id/web-videos' ], // TODO: remove webtorrent in V7 + authenticate, + ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES), + asyncMiddleware(videoFilesDeleteWebVideoValidator), + asyncMiddleware(removeAllWebVideoFilesController) +) +filesRouter.delete( + [ '/:id/webtorrent/:videoFileId', '/:id/web-videos/:videoFileId' ], // TODO: remove webtorrent in V7 + authenticate, + ensureUserHasRight(UserRight.MANAGE_VIDEO_FILES), + asyncMiddleware(videoFilesDeleteWebVideoFileValidator), + asyncMiddleware(removeWebVideoFileController) +) + +// --------------------------------------------------------------------------- + +export { + filesRouter +} + +// --------------------------------------------------------------------------- + +async function getVideoFileMetadata (req: express.Request, res: express.Response) { + const videoFile = await VideoFileModel.loadWithMetadata(validator.default.toInt(req.params.videoFileId)) + + return res.json(videoFile.metadata) +} + +// --------------------------------------------------------------------------- + +async function removeHLSPlaylistController (req: express.Request, res: express.Response) { + const video = res.locals.videoAll + + logger.info('Deleting HLS playlist of %s.', video.url, lTags(video.uuid)) + await removeHLSPlaylist(video) + + await federateVideoIfNeeded(video, false, undefined) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +async function removeHLSFileController (req: express.Request, res: express.Response) { + const video = res.locals.videoAll + const videoFileId = +req.params.videoFileId + + logger.info('Deleting HLS file %d of %s.', videoFileId, video.url, lTags(video.uuid)) + + const playlist = await removeHLSFile(video, videoFileId) + if (playlist) await updatePlaylistAfterFileChange(video, playlist) + + await federateVideoIfNeeded(video, false, undefined) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +// --------------------------------------------------------------------------- + +async function removeAllWebVideoFilesController (req: express.Request, res: express.Response) { + const video = res.locals.videoAll + + logger.info('Deleting Web Video files of %s.', video.url, lTags(video.uuid)) + + await removeAllWebVideoFiles(video) + await federateVideoIfNeeded(video, false, undefined) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +async function removeWebVideoFileController (req: express.Request, res: express.Response) { + const video = res.locals.videoAll + + const videoFileId = +req.params.videoFileId + logger.info('Deleting Web Video file %d of %s.', videoFileId, video.url, lTags(video.uuid)) + + await removeWebVideoFile(video, videoFileId) + await federateVideoIfNeeded(video, false, undefined) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} diff --git a/server/server/controllers/api/videos/import.ts b/server/server/controllers/api/videos/import.ts new file mode 100644 index 000000000..185215cc9 --- /dev/null +++ b/server/server/controllers/api/videos/import.ts @@ -0,0 +1,270 @@ +import express from 'express' +import { move } from 'fs-extra/esm' +import { readFile } from 'fs/promises' +import { decode } from 'magnet-uri' +import parseTorrent, { Instance } from 'parse-torrent' +import { join } from 'path' +import { buildVideoFromImport, buildYoutubeDLImport, insertFromImportIntoDB, YoutubeDlImportError } from '@server/lib/video-pre-import.js' +import { MThumbnail, MVideoThumbnail } from '@server/types/models/index.js' +import { + HttpStatusCode, + ServerErrorCode, + ThumbnailType, + VideoImportCreate, + VideoImportPayload, + VideoImportState +} from '@peertube/peertube-models' +import { auditLoggerFactory, getAuditIdFromRes, VideoImportAuditView } from '../../../helpers/audit-logger.js' +import { isArray } from '../../../helpers/custom-validators/misc.js' +import { cleanUpReqFiles, createReqFiles } from '../../../helpers/express-utils.js' +import { logger } from '../../../helpers/logger.js' +import { getSecureTorrentName } from '../../../helpers/utils.js' +import { CONFIG } from '../../../initializers/config.js' +import { MIMETYPES } from '../../../initializers/constants.js' +import { JobQueue } from '../../../lib/job-queue/job-queue.js' +import { updateLocalVideoMiniatureFromExisting } from '../../../lib/thumbnail.js' +import { + asyncMiddleware, + asyncRetryTransactionMiddleware, + authenticate, + videoImportAddValidator, + videoImportCancelValidator, + videoImportDeleteValidator +} from '../../../middlewares/index.js' + +const auditLogger = auditLoggerFactory('video-imports') +const videoImportsRouter = express.Router() + +const reqVideoFileImport = createReqFiles( + [ 'thumbnailfile', 'previewfile', 'torrentfile' ], + { ...MIMETYPES.TORRENT.MIMETYPE_EXT, ...MIMETYPES.IMAGE.MIMETYPE_EXT } +) + +videoImportsRouter.post('/imports', + authenticate, + reqVideoFileImport, + asyncMiddleware(videoImportAddValidator), + asyncRetryTransactionMiddleware(handleVideoImport) +) + +videoImportsRouter.post('/imports/:id/cancel', + authenticate, + asyncMiddleware(videoImportCancelValidator), + asyncRetryTransactionMiddleware(cancelVideoImport) +) + +videoImportsRouter.delete('/imports/:id', + authenticate, + asyncMiddleware(videoImportDeleteValidator), + asyncRetryTransactionMiddleware(deleteVideoImport) +) + +// --------------------------------------------------------------------------- + +export { + videoImportsRouter +} + +// --------------------------------------------------------------------------- + +async function deleteVideoImport (req: express.Request, res: express.Response) { + const videoImport = res.locals.videoImport + + await videoImport.destroy() + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +async function cancelVideoImport (req: express.Request, res: express.Response) { + const videoImport = res.locals.videoImport + + videoImport.state = VideoImportState.CANCELLED + await videoImport.save() + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +function handleVideoImport (req: express.Request, res: express.Response) { + if (req.body.targetUrl) return handleYoutubeDlImport(req, res) + + const file = req.files?.['torrentfile']?.[0] + if (req.body.magnetUri || file) return handleTorrentImport(req, res, file) +} + +async function handleTorrentImport (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { + const body: VideoImportCreate = req.body + const user = res.locals.oauth.token.User + + let videoName: string + let torrentName: string + let magnetUri: string + + if (torrentfile) { + const result = await processTorrentOrAbortRequest(req, res, torrentfile) + if (!result) return + + videoName = result.name + torrentName = result.torrentName + } else { + const result = processMagnetURI(body) + magnetUri = result.magnetUri + videoName = result.name + } + + const video = await buildVideoFromImport({ + channelId: res.locals.videoChannel.id, + importData: { name: videoName }, + importDataOverride: body, + importType: 'torrent' + }) + + const thumbnailModel = await processThumbnail(req, video) + const previewModel = await processPreview(req, video) + + const videoImport = await insertFromImportIntoDB({ + video, + thumbnailModel, + previewModel, + videoChannel: res.locals.videoChannel, + tags: body.tags || undefined, + user, + videoPasswords: body.videoPasswords, + videoImportAttributes: { + magnetUri, + torrentName, + state: VideoImportState.PENDING, + userId: user.id + } + }) + + const payload: VideoImportPayload = { + type: torrentfile + ? 'torrent-file' + : 'magnet-uri', + videoImportId: videoImport.id, + preventException: false + } + await JobQueue.Instance.createJob({ type: 'video-import', payload }) + + auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON())) + + return res.json(videoImport.toFormattedJSON()).end() +} + +function statusFromYtDlImportError (err: YoutubeDlImportError): number { + switch (err.code) { + case YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL: + return HttpStatusCode.FORBIDDEN_403 + + case YoutubeDlImportError.CODE.FETCH_ERROR: + return HttpStatusCode.BAD_REQUEST_400 + + default: + return HttpStatusCode.INTERNAL_SERVER_ERROR_500 + } +} + +async function handleYoutubeDlImport (req: express.Request, res: express.Response) { + const body: VideoImportCreate = req.body + const targetUrl = body.targetUrl + const user = res.locals.oauth.token.User + + try { + const { job, videoImport } = await buildYoutubeDLImport({ + targetUrl, + channel: res.locals.videoChannel, + importDataOverride: body, + thumbnailFilePath: req.files?.['thumbnailfile']?.[0].path, + previewFilePath: req.files?.['previewfile']?.[0].path, + user + }) + await JobQueue.Instance.createJob(job) + + auditLogger.create(getAuditIdFromRes(res), new VideoImportAuditView(videoImport.toFormattedJSON())) + + return res.json(videoImport.toFormattedJSON()).end() + } catch (err) { + logger.error('An error occurred while importing the video %s. ', targetUrl, { err }) + + return res.fail({ + message: err.message, + status: statusFromYtDlImportError(err), + data: { + targetUrl + } + }) + } +} + +async function processThumbnail (req: express.Request, video: MVideoThumbnail) { + const thumbnailField = req.files ? req.files['thumbnailfile'] : undefined + if (thumbnailField) { + const thumbnailPhysicalFile = thumbnailField[0] + + return updateLocalVideoMiniatureFromExisting({ + inputPath: thumbnailPhysicalFile.path, + video, + type: ThumbnailType.MINIATURE, + automaticallyGenerated: false + }) + } + + return undefined +} + +async function processPreview (req: express.Request, video: MVideoThumbnail): Promise { + const previewField = req.files ? req.files['previewfile'] : undefined + if (previewField) { + const previewPhysicalFile = previewField[0] + + return updateLocalVideoMiniatureFromExisting({ + inputPath: previewPhysicalFile.path, + video, + type: ThumbnailType.PREVIEW, + automaticallyGenerated: false + }) + } + + return undefined +} + +async function processTorrentOrAbortRequest (req: express.Request, res: express.Response, torrentfile: Express.Multer.File) { + const torrentName = torrentfile.originalname + + // Rename the torrent to a secured name + const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, getSecureTorrentName(torrentName)) + await move(torrentfile.path, newTorrentPath, { overwrite: true }) + torrentfile.path = newTorrentPath + + const buf = await readFile(torrentfile.path) + const parsedTorrent = parseTorrent(buf) as Instance + + if (parsedTorrent.files.length !== 1) { + cleanUpReqFiles(req) + + res.fail({ + type: ServerErrorCode.INCORRECT_FILES_IN_TORRENT, + message: 'Torrents with only 1 file are supported.' + }) + return undefined + } + + return { + name: extractNameFromArray(parsedTorrent.name), + torrentName + } +} + +function processMagnetURI (body: VideoImportCreate) { + const magnetUri = body.magnetUri + const parsed = decode(magnetUri) + + return { + name: extractNameFromArray(parsed.name), + magnetUri + } +} + +function extractNameFromArray (name: string | string[]) { + return isArray(name) ? name[0] : name +} diff --git a/server/server/controllers/api/videos/index.ts b/server/server/controllers/api/videos/index.ts new file mode 100644 index 000000000..f8e3d9cb5 --- /dev/null +++ b/server/server/controllers/api/videos/index.ts @@ -0,0 +1,228 @@ +import express from 'express' +import { HttpStatusCode } from '@peertube/peertube-models' +import { pickCommonVideoQuery } from '@server/helpers/query.js' +import { doJSONRequest } from '@server/helpers/requests.js' +import { openapiOperationDoc } from '@server/middlewares/doc.js' +import { getServerActor } from '@server/models/application/application.js' +import { MVideoAccountLight } from '@server/types/models/index.js' +import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger.js' +import { buildNSFWFilter, getCountVideos } from '../../../helpers/express-utils.js' +import { logger } from '../../../helpers/logger.js' +import { getFormattedObjects } from '../../../helpers/utils.js' +import { REMOTE_SCHEME, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../initializers/constants.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { JobQueue } from '../../../lib/job-queue/index.js' +import { Hooks } from '../../../lib/plugins/hooks.js' +import { + apiRateLimiter, + asyncMiddleware, + asyncRetryTransactionMiddleware, + authenticate, + checkVideoFollowConstraints, + commonVideosFiltersValidator, + optionalAuthenticate, + paginationValidator, + setDefaultPagination, + setDefaultVideosSort, + videosCustomGetValidator, + videosGetValidator, + videosRemoveValidator, + videosSortValidator +} from '../../../middlewares/index.js' +import { guessAdditionalAttributesFromQuery } from '../../../models/video/formatter/index.js' +import { VideoModel } from '../../../models/video/video.js' +import { blacklistRouter } from './blacklist.js' +import { videoCaptionsRouter } from './captions.js' +import { videoCommentRouter } from './comment.js' +import { filesRouter } from './files.js' +import { videoImportsRouter } from './import.js' +import { liveRouter } from './live.js' +import { ownershipVideoRouter } from './ownership.js' +import { videoPasswordRouter } from './passwords.js' +import { rateVideoRouter } from './rate.js' +import { videoSourceRouter } from './source.js' +import { statsRouter } from './stats.js' +import { storyboardRouter } from './storyboard.js' +import { studioRouter } from './studio.js' +import { tokenRouter } from './token.js' +import { transcodingRouter } from './transcoding.js' +import { updateRouter } from './update.js' +import { uploadRouter } from './upload.js' +import { viewRouter } from './view.js' + +const auditLogger = auditLoggerFactory('videos') +const videosRouter = express.Router() + +videosRouter.use(apiRateLimiter) + +videosRouter.use('/', blacklistRouter) +videosRouter.use('/', statsRouter) +videosRouter.use('/', rateVideoRouter) +videosRouter.use('/', videoCommentRouter) +videosRouter.use('/', studioRouter) +videosRouter.use('/', videoCaptionsRouter) +videosRouter.use('/', videoImportsRouter) +videosRouter.use('/', ownershipVideoRouter) +videosRouter.use('/', viewRouter) +videosRouter.use('/', liveRouter) +videosRouter.use('/', uploadRouter) +videosRouter.use('/', updateRouter) +videosRouter.use('/', filesRouter) +videosRouter.use('/', transcodingRouter) +videosRouter.use('/', tokenRouter) +videosRouter.use('/', videoPasswordRouter) +videosRouter.use('/', storyboardRouter) +videosRouter.use('/', videoSourceRouter) + +videosRouter.get('/categories', + openapiOperationDoc({ operationId: 'getCategories' }), + listVideoCategories +) +videosRouter.get('/licences', + openapiOperationDoc({ operationId: 'getLicences' }), + listVideoLicences +) +videosRouter.get('/languages', + openapiOperationDoc({ operationId: 'getLanguages' }), + listVideoLanguages +) +videosRouter.get('/privacies', + openapiOperationDoc({ operationId: 'getPrivacies' }), + listVideoPrivacies +) + +videosRouter.get('/', + openapiOperationDoc({ operationId: 'getVideos' }), + paginationValidator, + videosSortValidator, + setDefaultVideosSort, + setDefaultPagination, + optionalAuthenticate, + commonVideosFiltersValidator, + asyncMiddleware(listVideos) +) + +// TODO: remove, deprecated in 5.0 now we send the complete description in VideoDetails +videosRouter.get('/:id/description', + openapiOperationDoc({ operationId: 'getVideoDesc' }), + asyncMiddleware(videosGetValidator), + asyncMiddleware(getVideoDescription) +) + +videosRouter.get('/:id', + openapiOperationDoc({ operationId: 'getVideo' }), + optionalAuthenticate, + asyncMiddleware(videosCustomGetValidator('for-api')), + asyncMiddleware(checkVideoFollowConstraints), + asyncMiddleware(getVideo) +) + +videosRouter.delete('/:id', + openapiOperationDoc({ operationId: 'delVideo' }), + authenticate, + asyncMiddleware(videosRemoveValidator), + asyncRetryTransactionMiddleware(removeVideo) +) + +// --------------------------------------------------------------------------- + +export { + videosRouter +} + +// --------------------------------------------------------------------------- + +function listVideoCategories (_req: express.Request, res: express.Response) { + res.json(VIDEO_CATEGORIES) +} + +function listVideoLicences (_req: express.Request, res: express.Response) { + res.json(VIDEO_LICENCES) +} + +function listVideoLanguages (_req: express.Request, res: express.Response) { + res.json(VIDEO_LANGUAGES) +} + +function listVideoPrivacies (_req: express.Request, res: express.Response) { + res.json(VIDEO_PRIVACIES) +} + +async function getVideo (_req: express.Request, res: express.Response) { + const videoId = res.locals.videoAPI.id + const userId = res.locals.oauth?.token.User.id + + const video = await Hooks.wrapObject(res.locals.videoAPI, 'filter:api.video.get.result', { id: videoId, userId }) + + if (video.isOutdated()) { + JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'video', url: video.url } }) + } + + return res.json(video.toFormattedDetailsJSON()) +} + +async function getVideoDescription (req: express.Request, res: express.Response) { + const videoInstance = res.locals.videoAll + + const description = videoInstance.isOwned() + ? videoInstance.description + : await fetchRemoteVideoDescription(videoInstance) + + return res.json({ description }) +} + +async function listVideos (req: express.Request, res: express.Response) { + const serverActor = await getServerActor() + + const query = pickCommonVideoQuery(req.query) + const countVideos = getCountVideos(req) + + const apiOptions = await Hooks.wrapObject({ + ...query, + + displayOnlyForFollower: { + actorId: serverActor.id, + orLocalVideos: true + }, + nsfw: buildNSFWFilter(res, query.nsfw), + user: res.locals.oauth ? res.locals.oauth.token.User : undefined, + countVideos + }, 'filter:api.videos.list.params') + + const resultList = await Hooks.wrapPromiseFun( + VideoModel.listForApi, + apiOptions, + 'filter:api.videos.list.result' + ) + + return res.json(getFormattedObjects(resultList.data, resultList.total, guessAdditionalAttributesFromQuery(query))) +} + +async function removeVideo (req: express.Request, res: express.Response) { + const videoInstance = res.locals.videoAll + + await sequelizeTypescript.transaction(async t => { + await videoInstance.destroy({ transaction: t }) + }) + + auditLogger.delete(getAuditIdFromRes(res), new VideoAuditView(videoInstance.toFormattedDetailsJSON())) + logger.info('Video with name %s and uuid %s deleted.', videoInstance.name, videoInstance.uuid) + + Hooks.runAction('action:api.video.deleted', { video: videoInstance, req, res }) + + return res.type('json') + .status(HttpStatusCode.NO_CONTENT_204) + .end() +} + +// --------------------------------------------------------------------------- + +// FIXME: Should not exist, we rely on specific API +async function fetchRemoteVideoDescription (video: MVideoAccountLight) { + const host = video.VideoChannel.Account.Actor.Server.host + const path = video.getDescriptionAPIPath() + const url = REMOTE_SCHEME.HTTP + '://' + host + path + + const { body } = await doJSONRequest(url) + return body.description || '' +} diff --git a/server/server/controllers/api/videos/live.ts b/server/server/controllers/api/videos/live.ts new file mode 100644 index 000000000..84cb3d51b --- /dev/null +++ b/server/server/controllers/api/videos/live.ts @@ -0,0 +1,232 @@ +import express from 'express' +import { + HttpStatusCode, + LiveVideoCreate, + LiveVideoLatencyMode, + LiveVideoUpdate, + UserRight, + VideoPrivacy, + VideoState +} from '@peertube/peertube-models' +import { exists } from '@server/helpers/custom-validators/misc.js' +import { createReqFiles } from '@server/helpers/express-utils.js' +import { getFormattedObjects } from '@server/helpers/utils.js' +import { ASSETS_PATH, MIMETYPES } from '@server/initializers/constants.js' +import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url.js' +import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { buildLocalVideoFromReq, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video.js' +import { + videoLiveAddValidator, + videoLiveFindReplaySessionValidator, + videoLiveGetValidator, + videoLiveListSessionsValidator, + videoLiveUpdateValidator +} from '@server/middlewares/validators/videos/video-live.js' +import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting.js' +import { VideoLiveSessionModel } from '@server/models/video/video-live-session.js' +import { VideoLiveModel } from '@server/models/video/video-live.js' +import { VideoPasswordModel } from '@server/models/video/video-password.js' +import { MVideoDetails, MVideoFullLight, MVideoLive } from '@server/types/models/index.js' +import { buildUUID, uuidToShort } from '@peertube/peertube-node-utils' +import { logger } from '../../../helpers/logger.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { updateLocalVideoMiniatureFromExisting } from '../../../lib/thumbnail.js' +import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, optionalAuthenticate } from '../../../middlewares/index.js' +import { VideoModel } from '../../../models/video/video.js' + +const liveRouter = express.Router() + +const reqVideoFileLive = createReqFiles([ 'thumbnailfile', 'previewfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT) + +liveRouter.post('/live', + authenticate, + reqVideoFileLive, + asyncMiddleware(videoLiveAddValidator), + asyncRetryTransactionMiddleware(addLiveVideo) +) + +liveRouter.get('/live/:videoId/sessions', + authenticate, + asyncMiddleware(videoLiveGetValidator), + videoLiveListSessionsValidator, + asyncMiddleware(getLiveVideoSessions) +) + +liveRouter.get('/live/:videoId', + optionalAuthenticate, + asyncMiddleware(videoLiveGetValidator), + getLiveVideo +) + +liveRouter.put('/live/:videoId', + authenticate, + asyncMiddleware(videoLiveGetValidator), + videoLiveUpdateValidator, + asyncRetryTransactionMiddleware(updateLiveVideo) +) + +liveRouter.get('/:videoId/live-session', + asyncMiddleware(videoLiveFindReplaySessionValidator), + getLiveReplaySession +) + +// --------------------------------------------------------------------------- + +export { + liveRouter +} + +// --------------------------------------------------------------------------- + +function getLiveVideo (req: express.Request, res: express.Response) { + const videoLive = res.locals.videoLive + + return res.json(videoLive.toFormattedJSON(canSeePrivateLiveInformation(res))) +} + +function getLiveReplaySession (req: express.Request, res: express.Response) { + const session = res.locals.videoLiveSession + + return res.json(session.toFormattedJSON()) +} + +async function getLiveVideoSessions (req: express.Request, res: express.Response) { + const videoLive = res.locals.videoLive + + const data = await VideoLiveSessionModel.listSessionsOfLiveForAPI({ videoId: videoLive.videoId }) + + return res.json(getFormattedObjects(data, data.length)) +} + +function canSeePrivateLiveInformation (res: express.Response) { + const user = res.locals.oauth?.token.User + if (!user) return false + + if (user.hasRight(UserRight.GET_ANY_LIVE)) return true + + const video = res.locals.videoAll + return video.VideoChannel.Account.userId === user.id +} + +async function updateLiveVideo (req: express.Request, res: express.Response) { + const body: LiveVideoUpdate = req.body + + const video = res.locals.videoAll + const videoLive = res.locals.videoLive + + const newReplaySettingModel = await updateReplaySettings(videoLive, body) + if (newReplaySettingModel) videoLive.replaySettingId = newReplaySettingModel.id + else videoLive.replaySettingId = null + + if (exists(body.permanentLive)) videoLive.permanentLive = body.permanentLive + if (exists(body.latencyMode)) videoLive.latencyMode = body.latencyMode + + video.VideoLive = await videoLive.save() + + await federateVideoIfNeeded(video, false) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function updateReplaySettings (videoLive: MVideoLive, body: LiveVideoUpdate) { + if (exists(body.saveReplay)) videoLive.saveReplay = body.saveReplay + + // The live replay is not saved anymore, destroy the old model if it existed + if (!videoLive.saveReplay) { + if (videoLive.replaySettingId) { + await VideoLiveReplaySettingModel.removeSettings(videoLive.replaySettingId) + } + + return undefined + } + + const settingModel = videoLive.replaySettingId + ? await VideoLiveReplaySettingModel.load(videoLive.replaySettingId) + : new VideoLiveReplaySettingModel() + + if (exists(body.replaySettings.privacy)) settingModel.privacy = body.replaySettings.privacy + + return settingModel.save() +} + +async function addLiveVideo (req: express.Request, res: express.Response) { + const videoInfo: LiveVideoCreate = req.body + + // Prepare data so we don't block the transaction + let videoData = buildLocalVideoFromReq(videoInfo, res.locals.videoChannel.id) + videoData = await Hooks.wrapObject(videoData, 'filter:api.video.live.video-attribute.result') + + videoData.isLive = true + videoData.state = VideoState.WAITING_FOR_LIVE + videoData.duration = 0 + + const video = new VideoModel(videoData) as MVideoDetails + video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object + + const videoLive = new VideoLiveModel() + videoLive.saveReplay = videoInfo.saveReplay || false + videoLive.permanentLive = videoInfo.permanentLive || false + videoLive.latencyMode = videoInfo.latencyMode || LiveVideoLatencyMode.DEFAULT + videoLive.streamKey = buildUUID() + + const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ + video, + files: req.files, + fallback: type => { + return updateLocalVideoMiniatureFromExisting({ + inputPath: ASSETS_PATH.DEFAULT_LIVE_BACKGROUND, + video, + type, + automaticallyGenerated: true, + keepOriginal: true + }) + } + }) + + const { videoCreated } = await sequelizeTypescript.transaction(async t => { + const sequelizeOptions = { transaction: t } + + const videoCreated = await video.save(sequelizeOptions) as MVideoFullLight + + if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t) + if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t) + + // Do not forget to add video channel information to the created video + videoCreated.VideoChannel = res.locals.videoChannel + + if (videoLive.saveReplay) { + const replaySettings = new VideoLiveReplaySettingModel({ + privacy: videoInfo.replaySettings.privacy + }) + await replaySettings.save(sequelizeOptions) + + videoLive.replaySettingId = replaySettings.id + } + + videoLive.videoId = videoCreated.id + videoCreated.VideoLive = await videoLive.save(sequelizeOptions) + + await setVideoTags({ video, tags: videoInfo.tags, transaction: t }) + + await federateVideoIfNeeded(videoCreated, true, t) + + if (videoInfo.privacy === VideoPrivacy.PASSWORD_PROTECTED) { + await VideoPasswordModel.addPasswords(videoInfo.videoPasswords, video.id, t) + } + + logger.info('Video live %s with uuid %s created.', videoInfo.name, videoCreated.uuid) + + return { videoCreated } + }) + + Hooks.runAction('action:api.live-video.created', { video: videoCreated, req, res }) + + return res.json({ + video: { + id: videoCreated.id, + shortUUID: uuidToShort(videoCreated.uuid), + uuid: videoCreated.uuid + } + }) +} diff --git a/server/server/controllers/api/videos/ownership.ts b/server/server/controllers/api/videos/ownership.ts new file mode 100644 index 000000000..ceb3a2739 --- /dev/null +++ b/server/server/controllers/api/videos/ownership.ts @@ -0,0 +1,137 @@ +import express from 'express' +import { HttpStatusCode, VideoChangeOwnershipStatus, VideoState } from '@peertube/peertube-models' +import { MVideoFullLight } from '@server/types/models/index.js' +import { logger } from '../../../helpers/logger.js' +import { getFormattedObjects } from '../../../helpers/utils.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { sendUpdateVideo } from '../../../lib/activitypub/send/index.js' +import { changeVideoChannelShare } from '../../../lib/activitypub/share.js' +import { + asyncMiddleware, + asyncRetryTransactionMiddleware, + authenticate, + paginationValidator, + setDefaultPagination, + videosAcceptChangeOwnershipValidator, + videosChangeOwnershipValidator, + videosTerminateChangeOwnershipValidator +} from '../../../middlewares/index.js' +import { VideoChangeOwnershipModel } from '../../../models/video/video-change-ownership.js' +import { VideoChannelModel } from '../../../models/video/video-channel.js' +import { VideoModel } from '../../../models/video/video.js' + +const ownershipVideoRouter = express.Router() + +ownershipVideoRouter.post('/:videoId/give-ownership', + authenticate, + asyncMiddleware(videosChangeOwnershipValidator), + asyncRetryTransactionMiddleware(giveVideoOwnership) +) + +ownershipVideoRouter.get('/ownership', + authenticate, + paginationValidator, + setDefaultPagination, + asyncRetryTransactionMiddleware(listVideoOwnership) +) + +ownershipVideoRouter.post('/ownership/:id/accept', + authenticate, + asyncMiddleware(videosTerminateChangeOwnershipValidator), + asyncMiddleware(videosAcceptChangeOwnershipValidator), + asyncRetryTransactionMiddleware(acceptOwnership) +) + +ownershipVideoRouter.post('/ownership/:id/refuse', + authenticate, + asyncMiddleware(videosTerminateChangeOwnershipValidator), + asyncRetryTransactionMiddleware(refuseOwnership) +) + +// --------------------------------------------------------------------------- + +export { + ownershipVideoRouter +} + +// --------------------------------------------------------------------------- + +async function giveVideoOwnership (req: express.Request, res: express.Response) { + const videoInstance = res.locals.videoAll + const initiatorAccountId = res.locals.oauth.token.User.Account.id + const nextOwner = res.locals.nextOwner + + await sequelizeTypescript.transaction(t => { + return VideoChangeOwnershipModel.findOrCreate({ + where: { + initiatorAccountId, + nextOwnerAccountId: nextOwner.id, + videoId: videoInstance.id, + status: VideoChangeOwnershipStatus.WAITING + }, + defaults: { + initiatorAccountId, + nextOwnerAccountId: nextOwner.id, + videoId: videoInstance.id, + status: VideoChangeOwnershipStatus.WAITING + }, + transaction: t + }) + }) + + logger.info('Ownership change for video %s created.', videoInstance.name) + return res.type('json') + .status(HttpStatusCode.NO_CONTENT_204) + .end() +} + +async function listVideoOwnership (req: express.Request, res: express.Response) { + const currentAccountId = res.locals.oauth.token.User.Account.id + + const resultList = await VideoChangeOwnershipModel.listForApi( + currentAccountId, + req.query.start || 0, + req.query.count || 10, + req.query.sort || 'createdAt' + ) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +function acceptOwnership (req: express.Request, res: express.Response) { + return sequelizeTypescript.transaction(async t => { + const videoChangeOwnership = res.locals.videoChangeOwnership + const channel = res.locals.videoChannel + + // We need more attributes for federation + const targetVideo = await VideoModel.loadFull(videoChangeOwnership.Video.id, t) + + const oldVideoChannel = await VideoChannelModel.loadAndPopulateAccount(targetVideo.channelId, t) + + targetVideo.channelId = channel.id + + const targetVideoUpdated = await targetVideo.save({ transaction: t }) as MVideoFullLight + targetVideoUpdated.VideoChannel = channel + + if (targetVideoUpdated.hasPrivacyForFederation() && targetVideoUpdated.state === VideoState.PUBLISHED) { + await changeVideoChannelShare(targetVideoUpdated, oldVideoChannel, t) + await sendUpdateVideo(targetVideoUpdated, t, oldVideoChannel.Account.Actor) + } + + videoChangeOwnership.status = VideoChangeOwnershipStatus.ACCEPTED + await videoChangeOwnership.save({ transaction: t }) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() + }) +} + +function refuseOwnership (req: express.Request, res: express.Response) { + return sequelizeTypescript.transaction(async t => { + const videoChangeOwnership = res.locals.videoChangeOwnership + + videoChangeOwnership.status = VideoChangeOwnershipStatus.REFUSED + await videoChangeOwnership.save({ transaction: t }) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() + }) +} diff --git a/server/server/controllers/api/videos/passwords.ts b/server/server/controllers/api/videos/passwords.ts new file mode 100644 index 000000000..bc356e8ac --- /dev/null +++ b/server/server/controllers/api/videos/passwords.ts @@ -0,0 +1,104 @@ +import express from 'express' +import { Transaction } from 'sequelize' +import { HttpStatusCode } from '@peertube/peertube-models' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { getVideoWithAttributes } from '@server/helpers/video.js' +import { VideoPasswordModel } from '@server/models/video/video-password.js' +import { getFormattedObjects } from '../../../helpers/utils.js' +import { + asyncMiddleware, + asyncRetryTransactionMiddleware, + authenticate, + setDefaultPagination, + setDefaultSort +} from '../../../middlewares/index.js' +import { + listVideoPasswordValidator, + paginationValidator, + removeVideoPasswordValidator, + updateVideoPasswordListValidator, + videoPasswordsSortValidator +} from '../../../middlewares/validators/index.js' + +const lTags = loggerTagsFactory('api', 'video') +const videoPasswordRouter = express.Router() + +videoPasswordRouter.get('/:videoId/passwords', + authenticate, + paginationValidator, + videoPasswordsSortValidator, + setDefaultSort, + setDefaultPagination, + asyncMiddleware(listVideoPasswordValidator), + asyncMiddleware(listVideoPasswords) +) + +videoPasswordRouter.put('/:videoId/passwords', + authenticate, + asyncMiddleware(updateVideoPasswordListValidator), + asyncMiddleware(updateVideoPasswordList) +) + +videoPasswordRouter.delete('/:videoId/passwords/:passwordId', + authenticate, + asyncMiddleware(removeVideoPasswordValidator), + asyncRetryTransactionMiddleware(removeVideoPassword) +) + +// --------------------------------------------------------------------------- + +export { + videoPasswordRouter +} + +// --------------------------------------------------------------------------- + +async function listVideoPasswords (req: express.Request, res: express.Response) { + const options = { + videoId: res.locals.videoAll.id, + start: req.query.start, + count: req.query.count, + sort: req.query.sort + } + + const resultList = await VideoPasswordModel.listPasswords(options) + + return res.json(getFormattedObjects(resultList.data, resultList.total)) +} + +async function updateVideoPasswordList (req: express.Request, res: express.Response) { + const videoInstance = getVideoWithAttributes(res) + const videoId = videoInstance.id + + const passwordArray = req.body.passwords as string[] + + await VideoPasswordModel.sequelize.transaction(async (t: Transaction) => { + await VideoPasswordModel.deleteAllPasswords(videoId, t) + await VideoPasswordModel.addPasswords(passwordArray, videoId, t) + }) + + logger.info( + `Video passwords for video with name %s and uuid %s have been updated`, + videoInstance.name, + videoInstance.uuid, + lTags(videoInstance.uuid) + ) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +async function removeVideoPassword (req: express.Request, res: express.Response) { + const videoInstance = getVideoWithAttributes(res) + const password = res.locals.videoPassword + + await VideoPasswordModel.deletePassword(password.id) + logger.info( + 'Password with id %d of video named %s and uuid %s has been deleted.', + password.id, + videoInstance.name, + videoInstance.uuid, + lTags(videoInstance.uuid) + ) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} diff --git a/server/server/controllers/api/videos/rate.ts b/server/server/controllers/api/videos/rate.ts new file mode 100644 index 000000000..b0c4a328d --- /dev/null +++ b/server/server/controllers/api/videos/rate.ts @@ -0,0 +1,87 @@ +import express from 'express' +import { HttpStatusCode, UserVideoRateUpdate } from '@peertube/peertube-models' +import { logger } from '../../../helpers/logger.js' +import { VIDEO_RATE_TYPES } from '../../../initializers/constants.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { getLocalRateUrl, sendVideoRateChange } from '../../../lib/activitypub/video-rates.js' +import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoUpdateRateValidator } from '../../../middlewares/index.js' +import { AccountModel } from '../../../models/account/account.js' +import { AccountVideoRateModel } from '../../../models/account/account-video-rate.js' + +const rateVideoRouter = express.Router() + +rateVideoRouter.put('/:id/rate', + authenticate, + asyncMiddleware(videoUpdateRateValidator), + asyncRetryTransactionMiddleware(rateVideo) +) + +// --------------------------------------------------------------------------- + +export { + rateVideoRouter +} + +// --------------------------------------------------------------------------- + +async function rateVideo (req: express.Request, res: express.Response) { + const body: UserVideoRateUpdate = req.body + const rateType = body.rating + const videoInstance = res.locals.videoAll + const userAccount = res.locals.oauth.token.User.Account + + await sequelizeTypescript.transaction(async t => { + const sequelizeOptions = { transaction: t } + + const accountInstance = await AccountModel.load(userAccount.id, t) + const previousRate = await AccountVideoRateModel.load(accountInstance.id, videoInstance.id, t) + + // Same rate, nothing do to + if (rateType === 'none' && !previousRate || previousRate?.type === rateType) return + + let likesToIncrement = 0 + let dislikesToIncrement = 0 + + if (rateType === VIDEO_RATE_TYPES.LIKE) likesToIncrement++ + else if (rateType === VIDEO_RATE_TYPES.DISLIKE) dislikesToIncrement++ + + // There was a previous rate, update it + if (previousRate) { + // We will remove the previous rate, so we will need to update the video count attribute + if (previousRate.type === 'like') likesToIncrement-- + else if (previousRate.type === 'dislike') dislikesToIncrement-- + + if (rateType === 'none') { // Destroy previous rate + await previousRate.destroy(sequelizeOptions) + } else { // Update previous rate + previousRate.type = rateType + previousRate.url = getLocalRateUrl(rateType, userAccount.Actor, videoInstance) + await previousRate.save(sequelizeOptions) + } + } else if (rateType !== 'none') { // There was not a previous rate, insert a new one if there is a rate + const query = { + accountId: accountInstance.id, + videoId: videoInstance.id, + type: rateType, + url: getLocalRateUrl(rateType, userAccount.Actor, videoInstance) + } + + await AccountVideoRateModel.create(query, sequelizeOptions) + } + + const incrementQuery = { + likes: likesToIncrement, + dislikes: dislikesToIncrement + } + + await videoInstance.increment(incrementQuery, sequelizeOptions) + + await sendVideoRateChange(accountInstance, videoInstance, likesToIncrement, dislikesToIncrement, t) + + logger.info('Account video rate for video %s of account %s updated.', videoInstance.name, accountInstance.name) + }) + + return res.type('json') + .status(HttpStatusCode.NO_CONTENT_204) + .end() +} diff --git a/server/server/controllers/api/videos/source.ts b/server/server/controllers/api/videos/source.ts new file mode 100644 index 000000000..d66062842 --- /dev/null +++ b/server/server/controllers/api/videos/source.ts @@ -0,0 +1,206 @@ +import express from 'express' +import { move } from 'fs-extra/esm' +import { sequelizeTypescript } from '@server/initializers/database.js' +import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue/index.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail.js' +import { uploadx } from '@server/lib/uploadx.js' +import { buildMoveToObjectStorageJob } from '@server/lib/video.js' +import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js' +import { buildNewFile } from '@server/lib/video-file.js' +import { VideoPathManager } from '@server/lib/video-path-manager.js' +import { buildNextVideoState } from '@server/lib/video-state.js' +import { openapiOperationDoc } from '@server/middlewares/doc.js' +import { VideoModel } from '@server/models/video/video.js' +import { VideoSourceModel } from '@server/models/video/video-source.js' +import { MStreamingPlaylistFiles, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models/index.js' +import { VideoState } from '@peertube/peertube-models' +import { logger, loggerTagsFactory } from '../../../helpers/logger.js' +import { + asyncMiddleware, + authenticate, + replaceVideoSourceResumableInitValidator, + replaceVideoSourceResumableValidator, + videoSourceGetLatestValidator +} from '../../../middlewares/index.js' + +const lTags = loggerTagsFactory('api', 'video') + +const videoSourceRouter = express.Router() + +videoSourceRouter.get('/:id/source', + openapiOperationDoc({ operationId: 'getVideoSource' }), + authenticate, + asyncMiddleware(videoSourceGetLatestValidator), + getVideoLatestSource +) + +videoSourceRouter.post('/:id/source/replace-resumable', + authenticate, + asyncMiddleware(replaceVideoSourceResumableInitValidator), + (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end +) + +videoSourceRouter.delete('/:id/source/replace-resumable', + authenticate, + (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end +) + +videoSourceRouter.put('/:id/source/replace-resumable', + authenticate, + uploadx.upload, // uploadx doesn't next() before the file upload completes + asyncMiddleware(replaceVideoSourceResumableValidator), + asyncMiddleware(replaceVideoSourceResumable) +) + +// --------------------------------------------------------------------------- + +export { + videoSourceRouter +} + +// --------------------------------------------------------------------------- + +function getVideoLatestSource (req: express.Request, res: express.Response) { + return res.json(res.locals.videoSource.toFormattedJSON()) +} + +async function replaceVideoSourceResumable (req: express.Request, res: express.Response) { + const videoPhysicalFile = res.locals.updateVideoFileResumable + const user = res.locals.oauth.token.User + + const videoFile = await buildNewFile({ path: videoPhysicalFile.path, mode: 'web-video' }) + const originalFilename = videoPhysicalFile.originalname + + const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(res.locals.videoAll.uuid) + + try { + const destination = VideoPathManager.Instance.getFSVideoFileOutputPath(res.locals.videoAll, videoFile) + await move(videoPhysicalFile.path, destination) + + let oldWebVideoFiles: MVideoFile[] = [] + let oldStreamingPlaylists: MStreamingPlaylistFiles[] = [] + + const inputFileUpdatedAt = new Date() + + const video = await sequelizeTypescript.transaction(async transaction => { + const video = await VideoModel.loadFull(res.locals.videoAll.id, transaction) + + oldWebVideoFiles = video.VideoFiles + oldStreamingPlaylists = video.VideoStreamingPlaylists + + for (const file of video.VideoFiles) { + await file.destroy({ transaction }) + } + for (const playlist of oldStreamingPlaylists) { + await playlist.destroy({ transaction }) + } + + videoFile.videoId = video.id + await videoFile.save({ transaction }) + + video.VideoFiles = [ videoFile ] + video.VideoStreamingPlaylists = [] + + video.state = buildNextVideoState() + video.duration = videoPhysicalFile.duration + video.inputFileUpdatedAt = inputFileUpdatedAt + await video.save({ transaction }) + + await autoBlacklistVideoIfNeeded({ + video, + user, + isRemote: false, + isNew: false, + isNewFile: true, + transaction + }) + + return video + }) + + await removeOldFiles({ video, files: oldWebVideoFiles, playlists: oldStreamingPlaylists }) + + const source = await VideoSourceModel.create({ + filename: originalFilename, + videoId: video.id, + createdAt: inputFileUpdatedAt + }) + + await regenerateMiniaturesIfNeeded(video) + await video.VideoChannel.setAsUpdated() + await addVideoJobsAfterUpload(video, video.getMaxQualityFile()) + + logger.info('Replaced video file of video %s with uuid %s.', video.name, video.uuid, lTags(video.uuid)) + + Hooks.runAction('action:api.video.file-updated', { video, req, res }) + + return res.json(source.toFormattedJSON()) + } finally { + videoFileMutexReleaser() + } +} + +async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVideoFile) { + const jobs: (CreateJobArgument & CreateJobOptions)[] = [ + { + type: 'manage-video-torrent' as 'manage-video-torrent', + payload: { + videoId: video.id, + videoFileId: videoFile.id, + action: 'create' + } + }, + + { + type: 'generate-video-storyboard' as 'generate-video-storyboard', + payload: { + videoUUID: video.uuid, + // No need to federate, we process these jobs sequentially + federate: false + } + }, + + { + type: 'federate-video' as 'federate-video', + payload: { + videoUUID: video.uuid, + isNewVideo: false + } + } + ] + + if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { + jobs.push(await buildMoveToObjectStorageJob({ video, isNewVideo: false, previousVideoState: undefined })) + } + + if (video.state === VideoState.TO_TRANSCODE) { + jobs.push({ + type: 'transcoding-job-builder' as 'transcoding-job-builder', + payload: { + videoUUID: video.uuid, + optimizeJob: { + isNewVideo: false + } + } + }) + } + + return JobQueue.Instance.createSequentialJobFlow(...jobs) +} + +async function removeOldFiles (options: { + video: MVideo + files: MVideoFile[] + playlists: MStreamingPlaylistFiles[] +}) { + const { video, files, playlists } = options + + for (const file of files) { + await video.removeWebVideoFile(file) + } + + for (const playlist of playlists) { + await video.removeStreamingPlaylistFiles(playlist) + } +} diff --git a/server/server/controllers/api/videos/stats.ts b/server/server/controllers/api/videos/stats.ts new file mode 100644 index 000000000..8b3182e49 --- /dev/null +++ b/server/server/controllers/api/videos/stats.ts @@ -0,0 +1,75 @@ +import express from 'express' +import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer.js' +import { VideoStatsOverallQuery, VideoStatsTimeserieMetric, VideoStatsTimeserieQuery } from '@peertube/peertube-models' +import { + asyncMiddleware, + authenticate, + videoOverallStatsValidator, + videoRetentionStatsValidator, + videoTimeserieStatsValidator +} from '../../../middlewares/index.js' + +const statsRouter = express.Router() + +statsRouter.get('/:videoId/stats/overall', + authenticate, + asyncMiddleware(videoOverallStatsValidator), + asyncMiddleware(getOverallStats) +) + +statsRouter.get('/:videoId/stats/timeseries/:metric', + authenticate, + asyncMiddleware(videoTimeserieStatsValidator), + asyncMiddleware(getTimeserieStats) +) + +statsRouter.get('/:videoId/stats/retention', + authenticate, + asyncMiddleware(videoRetentionStatsValidator), + asyncMiddleware(getRetentionStats) +) + +// --------------------------------------------------------------------------- + +export { + statsRouter +} + +// --------------------------------------------------------------------------- + +async function getOverallStats (req: express.Request, res: express.Response) { + const video = res.locals.videoAll + const query = req.query as VideoStatsOverallQuery + + const stats = await LocalVideoViewerModel.getOverallStats({ + video, + startDate: query.startDate, + endDate: query.endDate + }) + + return res.json(stats) +} + +async function getRetentionStats (req: express.Request, res: express.Response) { + const video = res.locals.videoAll + + const stats = await LocalVideoViewerModel.getRetentionStats(video) + + return res.json(stats) +} + +async function getTimeserieStats (req: express.Request, res: express.Response) { + const video = res.locals.videoAll + const metric = req.params.metric as VideoStatsTimeserieMetric + + const query = req.query as VideoStatsTimeserieQuery + + const stats = await LocalVideoViewerModel.getTimeserieStats({ + video, + metric, + startDate: query.startDate ?? video.createdAt.toISOString(), + endDate: query.endDate ?? new Date().toISOString() + }) + + return res.json(stats) +} diff --git a/server/server/controllers/api/videos/storyboard.ts b/server/server/controllers/api/videos/storyboard.ts new file mode 100644 index 000000000..92e580141 --- /dev/null +++ b/server/server/controllers/api/videos/storyboard.ts @@ -0,0 +1,29 @@ +import express from 'express' +import { getVideoWithAttributes } from '@server/helpers/video.js' +import { StoryboardModel } from '@server/models/video/storyboard.js' +import { asyncMiddleware, videosGetValidator } from '../../../middlewares/index.js' + +const storyboardRouter = express.Router() + +storyboardRouter.get('/:id/storyboards', + asyncMiddleware(videosGetValidator), + asyncMiddleware(listStoryboards) +) + +// --------------------------------------------------------------------------- + +export { + storyboardRouter +} + +// --------------------------------------------------------------------------- + +async function listStoryboards (req: express.Request, res: express.Response) { + const video = getVideoWithAttributes(res) + + const storyboards = await StoryboardModel.listStoryboardsOf(video) + + return res.json({ + storyboards: storyboards.map(s => s.toFormattedJSON()) + }) +} diff --git a/server/server/controllers/api/videos/studio.ts b/server/server/controllers/api/videos/studio.ts new file mode 100644 index 000000000..642bd26ed --- /dev/null +++ b/server/server/controllers/api/videos/studio.ts @@ -0,0 +1,143 @@ +import Bluebird from 'bluebird' +import express from 'express' +import { move } from 'fs-extra/esm' +import { basename } from 'path' +import { createAnyReqFiles } from '@server/helpers/express-utils.js' +import { MIMETYPES, VIDEO_FILTERS } from '@server/initializers/constants.js' +import { buildTaskFileFieldname, createVideoStudioJob, getStudioTaskFilePath, getTaskFileFromReq } from '@server/lib/video-studio.js' +import { + HttpStatusCode, + VideoState, + VideoStudioCreateEdition, + VideoStudioTask, + VideoStudioTaskCut, + VideoStudioTaskIntro, + VideoStudioTaskOutro, + VideoStudioTaskPayload, + VideoStudioTaskWatermark +} from '@peertube/peertube-models' +import { asyncMiddleware, authenticate, videoStudioAddEditionValidator } from '../../../middlewares/index.js' + +const studioRouter = express.Router() + +const tasksFiles = createAnyReqFiles( + MIMETYPES.VIDEO.MIMETYPE_EXT, + (req: express.Request, file: Express.Multer.File, cb: (err: Error, result?: boolean) => void) => { + const body = req.body as VideoStudioCreateEdition + + // Fetch array element + const matches = file.fieldname.match(/tasks\[(\d+)\]/) + if (!matches) return cb(new Error('Cannot find array element indice for ' + file.fieldname)) + + const indice = parseInt(matches[1]) + const task = body.tasks[indice] + + if (!task) return cb(new Error('Cannot find array element of indice ' + indice + ' for ' + file.fieldname)) + + if ( + [ 'add-intro', 'add-outro', 'add-watermark' ].includes(task.name) && + file.fieldname === buildTaskFileFieldname(indice) + ) { + return cb(null, true) + } + + return cb(null, false) + } +) + +studioRouter.post('/:videoId/studio/edit', + authenticate, + tasksFiles, + asyncMiddleware(videoStudioAddEditionValidator), + asyncMiddleware(createEditionTasks) +) + +// --------------------------------------------------------------------------- + +export { + studioRouter +} + +// --------------------------------------------------------------------------- + +async function createEditionTasks (req: express.Request, res: express.Response) { + const files = req.files as Express.Multer.File[] + const body = req.body as VideoStudioCreateEdition + const video = res.locals.videoAll + + video.state = VideoState.TO_EDIT + await video.save() + + const payload = { + videoUUID: video.uuid, + tasks: await Bluebird.mapSeries(body.tasks, (t, i) => buildTaskPayload(t, i, files)) + } + + await createVideoStudioJob({ + user: res.locals.oauth.token.User, + payload, + video + }) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} + +const taskPayloadBuilders: { + [id in VideoStudioTask['name']]: ( + task: VideoStudioTask, + indice?: number, + files?: Express.Multer.File[] + ) => Promise +} = { + 'add-intro': buildIntroOutroTask, + 'add-outro': buildIntroOutroTask, + 'cut': buildCutTask, + 'add-watermark': buildWatermarkTask +} + +function buildTaskPayload (task: VideoStudioTask, indice: number, files: Express.Multer.File[]): Promise { + return taskPayloadBuilders[task.name](task, indice, files) +} + +async function buildIntroOutroTask (task: VideoStudioTaskIntro | VideoStudioTaskOutro, indice: number, files: Express.Multer.File[]) { + const destination = await moveStudioFileToPersistentTMP(getTaskFileFromReq(files, indice).path) + + return { + name: task.name, + options: { + file: destination + } + } +} + +function buildCutTask (task: VideoStudioTaskCut) { + return Promise.resolve({ + name: task.name, + options: { + start: task.options.start, + end: task.options.end + } + }) +} + +async function buildWatermarkTask (task: VideoStudioTaskWatermark, indice: number, files: Express.Multer.File[]) { + const destination = await moveStudioFileToPersistentTMP(getTaskFileFromReq(files, indice).path) + + return { + name: task.name, + options: { + file: destination, + watermarkSizeRatio: VIDEO_FILTERS.WATERMARK.SIZE_RATIO, + horitonzalMarginRatio: VIDEO_FILTERS.WATERMARK.HORIZONTAL_MARGIN_RATIO, + verticalMarginRatio: VIDEO_FILTERS.WATERMARK.VERTICAL_MARGIN_RATIO + } + } +} + +async function moveStudioFileToPersistentTMP (file: string) { + const destination = getStudioTaskFilePath(basename(file)) + + await move(file, destination) + + return destination +} diff --git a/server/server/controllers/api/videos/token.ts b/server/server/controllers/api/videos/token.ts new file mode 100644 index 000000000..9892518aa --- /dev/null +++ b/server/server/controllers/api/videos/token.ts @@ -0,0 +1,33 @@ +import express from 'express' +import { VideoTokensManager } from '@server/lib/video-tokens-manager.js' +import { VideoPrivacy, VideoToken } from '@peertube/peertube-models' +import { asyncMiddleware, optionalAuthenticate, videoFileTokenValidator, videosCustomGetValidator } from '../../../middlewares/index.js' + +const tokenRouter = express.Router() + +tokenRouter.post('/:id/token', + optionalAuthenticate, + asyncMiddleware(videosCustomGetValidator('only-video')), + videoFileTokenValidator, + generateToken +) + +// --------------------------------------------------------------------------- + +export { + tokenRouter +} + +// --------------------------------------------------------------------------- + +function generateToken (req: express.Request, res: express.Response) { + const video = res.locals.onlyVideo + + const files = video.privacy === VideoPrivacy.PASSWORD_PROTECTED + ? VideoTokensManager.Instance.createForPasswordProtectedVideo({ videoUUID: video.uuid }) + : VideoTokensManager.Instance.createForAuthUser({ videoUUID: video.uuid, user: res.locals.oauth.token.User }) + + return res.json({ + files + } as VideoToken) +} diff --git a/server/server/controllers/api/videos/transcoding.ts b/server/server/controllers/api/videos/transcoding.ts new file mode 100644 index 000000000..5a5eb6dd1 --- /dev/null +++ b/server/server/controllers/api/videos/transcoding.ts @@ -0,0 +1,60 @@ +import express from 'express' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { createTranscodingJobs } from '@server/lib/transcoding/create-transcoding-job.js' +import { computeResolutionsToTranscode } from '@server/lib/transcoding/transcoding-resolutions.js' +import { VideoJobInfoModel } from '@server/models/video/video-job-info.js' +import { HttpStatusCode, UserRight, VideoState, VideoTranscodingCreate } from '@peertube/peertube-models' +import { asyncMiddleware, authenticate, createTranscodingValidator, ensureUserHasRight } from '../../../middlewares/index.js' + +const lTags = loggerTagsFactory('api', 'video') +const transcodingRouter = express.Router() + +transcodingRouter.post('/:videoId/transcoding', + authenticate, + ensureUserHasRight(UserRight.RUN_VIDEO_TRANSCODING), + asyncMiddleware(createTranscodingValidator), + asyncMiddleware(createTranscoding) +) + +// --------------------------------------------------------------------------- + +export { + transcodingRouter +} + +// --------------------------------------------------------------------------- + +async function createTranscoding (req: express.Request, res: express.Response) { + const video = res.locals.videoAll + logger.info('Creating %s transcoding job for %s.', req.body.transcodingType, video.url, lTags()) + + const body: VideoTranscodingCreate = req.body + + await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingTranscode') + + const { resolution: maxResolution, hasAudio } = await video.probeMaxQualityFile() + + const resolutions = await Hooks.wrapObject( + computeResolutionsToTranscode({ input: maxResolution, type: 'vod', includeInput: true, strictLower: false, hasAudio }), + 'filter:transcoding.manual.resolutions-to-transcode.result', + body + ) + + if (resolutions.length === 0) { + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) + } + + video.state = VideoState.TO_TRANSCODE + await video.save() + + await createTranscodingJobs({ + video, + resolutions, + transcodingType: body.transcodingType, + isNewVideo: false, + user: null // Don't specify priority since these transcoding jobs are fired by the admin + }) + + return res.sendStatus(HttpStatusCode.NO_CONTENT_204) +} diff --git a/server/server/controllers/api/videos/update.ts b/server/server/controllers/api/videos/update.ts new file mode 100644 index 000000000..1c8cecba1 --- /dev/null +++ b/server/server/controllers/api/videos/update.ts @@ -0,0 +1,210 @@ +import express from 'express' +import { Transaction } from 'sequelize' +import { forceNumber } from '@peertube/peertube-core-utils' +import { HttpStatusCode, VideoPrivacy, VideoPrivacyType, VideoUpdate } from '@peertube/peertube-models' +import { exists } from '@server/helpers/custom-validators/misc.js' +import { changeVideoChannelShare } from '@server/lib/activitypub/share.js' +import { VideoPathManager } from '@server/lib/video-path-manager.js' +import { setVideoPrivacy } from '@server/lib/video-privacy.js' +import { addVideoJobsAfterUpdate, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video.js' +import { openapiOperationDoc } from '@server/middlewares/doc.js' +import { VideoPasswordModel } from '@server/models/video/video-password.js' +import { FilteredModelAttributes } from '@server/types/index.js' +import { MVideoFullLight } from '@server/types/models/index.js' +import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger.js' +import { resetSequelizeInstance } from '../../../helpers/database-utils.js' +import { createReqFiles } from '../../../helpers/express-utils.js' +import { logger, loggerTagsFactory } from '../../../helpers/logger.js' +import { MIMETYPES } from '../../../initializers/constants.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { Hooks } from '../../../lib/plugins/hooks.js' +import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist.js' +import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videosUpdateValidator } from '../../../middlewares/index.js' +import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update.js' +import { VideoModel } from '../../../models/video/video.js' + +const lTags = loggerTagsFactory('api', 'video') +const auditLogger = auditLoggerFactory('videos') +const updateRouter = express.Router() + +const reqVideoFileUpdate = createReqFiles([ 'thumbnailfile', 'previewfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT) + +updateRouter.put('/:id', + openapiOperationDoc({ operationId: 'putVideo' }), + authenticate, + reqVideoFileUpdate, + asyncMiddleware(videosUpdateValidator), + asyncRetryTransactionMiddleware(updateVideo) +) + +// --------------------------------------------------------------------------- + +export { + updateRouter +} + +// --------------------------------------------------------------------------- + +async function updateVideo (req: express.Request, res: express.Response) { + const videoFromReq = res.locals.videoAll + const oldVideoAuditView = new VideoAuditView(videoFromReq.toFormattedDetailsJSON()) + const videoInfoToUpdate: VideoUpdate = req.body + + const hadPrivacyForFederation = videoFromReq.hasPrivacyForFederation() + const oldPrivacy = videoFromReq.privacy + + const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ + video: videoFromReq, + files: req.files, + fallback: () => Promise.resolve(undefined), + automaticallyGenerated: false + }) + + const videoFileLockReleaser = await VideoPathManager.Instance.lockFiles(videoFromReq.uuid) + + try { + const { videoInstanceUpdated, isNewVideo } = await sequelizeTypescript.transaction(async t => { + // Refresh video since thumbnails to prevent concurrent updates + const video = await VideoModel.loadFull(videoFromReq.id, t) + + const oldVideoChannel = video.VideoChannel + + const keysToUpdate: (keyof VideoUpdate & FilteredModelAttributes)[] = [ + 'name', + 'category', + 'licence', + 'language', + 'nsfw', + 'waitTranscoding', + 'support', + 'description', + 'commentsEnabled', + 'downloadEnabled' + ] + + for (const key of keysToUpdate) { + if (videoInfoToUpdate[key] !== undefined) video.set(key, videoInfoToUpdate[key]) + } + + if (videoInfoToUpdate.originallyPublishedAt !== undefined && videoInfoToUpdate.originallyPublishedAt !== null) { + video.originallyPublishedAt = new Date(videoInfoToUpdate.originallyPublishedAt) + } + + // Privacy update? + let isNewVideo = false + if (videoInfoToUpdate.privacy !== undefined) { + isNewVideo = await updateVideoPrivacy({ videoInstance: video, videoInfoToUpdate, hadPrivacyForFederation, transaction: t }) + } + + // Force updatedAt attribute change + if (!video.changed()) { + await video.setAsRefreshed(t) + } + + const videoInstanceUpdated = await video.save({ transaction: t }) as MVideoFullLight + + // Thumbnail & preview updates? + if (thumbnailModel) await videoInstanceUpdated.addAndSaveThumbnail(thumbnailModel, t) + if (previewModel) await videoInstanceUpdated.addAndSaveThumbnail(previewModel, t) + + // Video tags update? + if (videoInfoToUpdate.tags !== undefined) { + await setVideoTags({ video: videoInstanceUpdated, tags: videoInfoToUpdate.tags, transaction: t }) + } + + // Video channel update? + if (res.locals.videoChannel && videoInstanceUpdated.channelId !== res.locals.videoChannel.id) { + await videoInstanceUpdated.$set('VideoChannel', res.locals.videoChannel, { transaction: t }) + videoInstanceUpdated.VideoChannel = res.locals.videoChannel + + if (hadPrivacyForFederation === true) { + await changeVideoChannelShare(videoInstanceUpdated, oldVideoChannel, t) + } + } + + // Schedule an update in the future? + await updateSchedule(videoInstanceUpdated, videoInfoToUpdate, t) + + await autoBlacklistVideoIfNeeded({ + video: videoInstanceUpdated, + user: res.locals.oauth.token.User, + isRemote: false, + isNew: false, + isNewFile: false, + transaction: t + }) + + auditLogger.update( + getAuditIdFromRes(res), + new VideoAuditView(videoInstanceUpdated.toFormattedDetailsJSON()), + oldVideoAuditView + ) + logger.info('Video with name %s and uuid %s updated.', video.name, video.uuid, lTags(video.uuid)) + + return { videoInstanceUpdated, isNewVideo } + }) + + Hooks.runAction('action:api.video.updated', { video: videoInstanceUpdated, body: req.body, req, res }) + + await addVideoJobsAfterUpdate({ + video: videoInstanceUpdated, + nameChanged: !!videoInfoToUpdate.name, + oldPrivacy, + isNewVideo + }) + } catch (err) { + // If the transaction is retried, sequelize will think the object has not changed + // So we need to restore the previous fields + await resetSequelizeInstance(videoFromReq) + + throw err + } finally { + videoFileLockReleaser() + } + + return res.type('json') + .status(HttpStatusCode.NO_CONTENT_204) + .end() +} + +async function updateVideoPrivacy (options: { + videoInstance: MVideoFullLight + videoInfoToUpdate: VideoUpdate + hadPrivacyForFederation: boolean + transaction: Transaction +}) { + const { videoInstance, videoInfoToUpdate, hadPrivacyForFederation, transaction } = options + const isNewVideo = videoInstance.isNewVideo(videoInfoToUpdate.privacy) + + const newPrivacy = forceNumber(videoInfoToUpdate.privacy) as VideoPrivacyType + setVideoPrivacy(videoInstance, newPrivacy) + + // Delete passwords if video is not anymore password protected + if (videoInstance.privacy === VideoPrivacy.PASSWORD_PROTECTED && newPrivacy !== VideoPrivacy.PASSWORD_PROTECTED) { + await VideoPasswordModel.deleteAllPasswords(videoInstance.id, transaction) + } + + if (newPrivacy === VideoPrivacy.PASSWORD_PROTECTED && exists(videoInfoToUpdate.videoPasswords)) { + await VideoPasswordModel.deleteAllPasswords(videoInstance.id, transaction) + await VideoPasswordModel.addPasswords(videoInfoToUpdate.videoPasswords, videoInstance.id, transaction) + } + + // Unfederate the video if the new privacy is not compatible with federation + if (hadPrivacyForFederation && !videoInstance.hasPrivacyForFederation()) { + await VideoModel.sendDelete(videoInstance, { transaction }) + } + + return isNewVideo +} + +function updateSchedule (videoInstance: MVideoFullLight, videoInfoToUpdate: VideoUpdate, transaction: Transaction) { + if (videoInfoToUpdate.scheduleUpdate) { + return ScheduleVideoUpdateModel.upsert({ + videoId: videoInstance.id, + updateAt: new Date(videoInfoToUpdate.scheduleUpdate.updateAt), + privacy: videoInfoToUpdate.scheduleUpdate.privacy || null + }, { transaction }) + } else if (videoInfoToUpdate.scheduleUpdate === null) { + return ScheduleVideoUpdateModel.deleteByVideoId(videoInstance.id, transaction) + } +} diff --git a/server/server/controllers/api/videos/upload.ts b/server/server/controllers/api/videos/upload.ts new file mode 100644 index 000000000..47f06e336 --- /dev/null +++ b/server/server/controllers/api/videos/upload.ts @@ -0,0 +1,287 @@ +import express from 'express' +import { move } from 'fs-extra/esm' +import { basename } from 'path' +import { getResumableUploadPath } from '@server/helpers/upload.js' +import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url.js' +import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue/index.js' +import { Redis } from '@server/lib/redis.js' +import { uploadx } from '@server/lib/uploadx.js' +import { buildLocalVideoFromReq, buildMoveToObjectStorageJob, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video.js' +import { buildNewFile } from '@server/lib/video-file.js' +import { VideoPathManager } from '@server/lib/video-path-manager.js' +import { buildNextVideoState } from '@server/lib/video-state.js' +import { openapiOperationDoc } from '@server/middlewares/doc.js' +import { VideoPasswordModel } from '@server/models/video/video-password.js' +import { VideoSourceModel } from '@server/models/video/video-source.js' +import { MVideoFile, MVideoFullLight } from '@server/types/models/index.js' +import { uuidToShort } from '@peertube/peertube-node-utils' +import { HttpStatusCode, VideoCreate, VideoPrivacy, VideoState } from '@peertube/peertube-models' +import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger.js' +import { createReqFiles } from '../../../helpers/express-utils.js' +import { logger, loggerTagsFactory } from '../../../helpers/logger.js' +import { MIMETYPES } from '../../../initializers/constants.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { Hooks } from '../../../lib/plugins/hooks.js' +import { generateLocalVideoMiniature } from '../../../lib/thumbnail.js' +import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist.js' +import { + asyncMiddleware, + asyncRetryTransactionMiddleware, + authenticate, + videosAddLegacyValidator, + videosAddResumableInitValidator, + videosAddResumableValidator +} from '../../../middlewares/index.js' +import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update.js' +import { VideoModel } from '../../../models/video/video.js' + +const lTags = loggerTagsFactory('api', 'video') +const auditLogger = auditLoggerFactory('videos') +const uploadRouter = express.Router() + +const reqVideoFileAdd = createReqFiles( + [ 'videofile', 'thumbnailfile', 'previewfile' ], + { ...MIMETYPES.VIDEO.MIMETYPE_EXT, ...MIMETYPES.IMAGE.MIMETYPE_EXT } +) + +const reqVideoFileAddResumable = createReqFiles( + [ 'thumbnailfile', 'previewfile' ], + MIMETYPES.IMAGE.MIMETYPE_EXT, + getResumableUploadPath() +) + +uploadRouter.post('/upload', + openapiOperationDoc({ operationId: 'uploadLegacy' }), + authenticate, + reqVideoFileAdd, + asyncMiddleware(videosAddLegacyValidator), + asyncRetryTransactionMiddleware(addVideoLegacy) +) + +uploadRouter.post('/upload-resumable', + openapiOperationDoc({ operationId: 'uploadResumableInit' }), + authenticate, + reqVideoFileAddResumable, + asyncMiddleware(videosAddResumableInitValidator), + (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end +) + +uploadRouter.delete('/upload-resumable', + authenticate, + asyncMiddleware(deleteUploadResumableCache), + (req, res) => uploadx.upload(req, res) // Prevent next() call, explicitely tell to uploadx it's the end +) + +uploadRouter.put('/upload-resumable', + openapiOperationDoc({ operationId: 'uploadResumable' }), + authenticate, + uploadx.upload, // uploadx doesn't next() before the file upload completes + asyncMiddleware(videosAddResumableValidator), + asyncMiddleware(addVideoResumable) +) + +// --------------------------------------------------------------------------- + +export { + uploadRouter +} + +// --------------------------------------------------------------------------- + +async function addVideoLegacy (req: express.Request, res: express.Response) { + // Uploading the video could be long + // Set timeout to 10 minutes, as Express's default is 2 minutes + req.setTimeout(1000 * 60 * 10, () => { + logger.error('Video upload has timed out.') + return res.fail({ + status: HttpStatusCode.REQUEST_TIMEOUT_408, + message: 'Video upload has timed out.' + }) + }) + + const videoPhysicalFile = req.files['videofile'][0] + const videoInfo: VideoCreate = req.body + const files = req.files + + const response = await addVideo({ req, res, videoPhysicalFile, videoInfo, files }) + + return res.json(response) +} + +async function addVideoResumable (req: express.Request, res: express.Response) { + const videoPhysicalFile = res.locals.uploadVideoFileResumable + const videoInfo = videoPhysicalFile.metadata + const files = { previewfile: videoInfo.previewfile, thumbnailfile: videoInfo.thumbnailfile } + + const response = await addVideo({ req, res, videoPhysicalFile, videoInfo, files }) + await Redis.Instance.setUploadSession(req.query.upload_id, response) + + return res.json(response) +} + +async function addVideo (options: { + req: express.Request + res: express.Response + videoPhysicalFile: express.VideoUploadFile + videoInfo: VideoCreate + files: express.UploadFiles +}) { + const { req, res, videoPhysicalFile, videoInfo, files } = options + const videoChannel = res.locals.videoChannel + const user = res.locals.oauth.token.User + + let videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id) + videoData = await Hooks.wrapObject(videoData, 'filter:api.video.upload.video-attribute.result') + + videoData.state = buildNextVideoState() + videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware + + const video = new VideoModel(videoData) as MVideoFullLight + video.VideoChannel = videoChannel + video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object + + const videoFile = await buildNewFile({ path: videoPhysicalFile.path, mode: 'web-video' }) + const originalFilename = videoPhysicalFile.originalname + + // Move physical file + const destination = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile) + await move(videoPhysicalFile.path, destination) + // This is important in case if there is another attempt in the retry process + videoPhysicalFile.filename = basename(destination) + videoPhysicalFile.path = destination + + const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ + video, + files, + fallback: type => generateLocalVideoMiniature({ video, videoFile, type }) + }) + + const { videoCreated } = await sequelizeTypescript.transaction(async t => { + const sequelizeOptions = { transaction: t } + + const videoCreated = await video.save(sequelizeOptions) as MVideoFullLight + + await videoCreated.addAndSaveThumbnail(thumbnailModel, t) + await videoCreated.addAndSaveThumbnail(previewModel, t) + + // Do not forget to add video channel information to the created video + videoCreated.VideoChannel = res.locals.videoChannel + + videoFile.videoId = video.id + await videoFile.save(sequelizeOptions) + + video.VideoFiles = [ videoFile ] + + await VideoSourceModel.create({ + filename: originalFilename, + videoId: video.id + }, { transaction: t }) + + await setVideoTags({ video, tags: videoInfo.tags, transaction: t }) + + // Schedule an update in the future? + if (videoInfo.scheduleUpdate) { + await ScheduleVideoUpdateModel.create({ + videoId: video.id, + updateAt: new Date(videoInfo.scheduleUpdate.updateAt), + privacy: videoInfo.scheduleUpdate.privacy || null + }, sequelizeOptions) + } + + await autoBlacklistVideoIfNeeded({ + video, + user, + isRemote: false, + isNew: true, + isNewFile: true, + transaction: t + }) + + if (videoInfo.privacy === VideoPrivacy.PASSWORD_PROTECTED) { + await VideoPasswordModel.addPasswords(videoInfo.videoPasswords, video.id, t) + } + + auditLogger.create(getAuditIdFromRes(res), new VideoAuditView(videoCreated.toFormattedDetailsJSON())) + logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoCreated.uuid, lTags(videoCreated.uuid)) + + return { videoCreated } + }) + + // Channel has a new content, set as updated + await videoCreated.VideoChannel.setAsUpdated() + + addVideoJobsAfterUpload(videoCreated, videoFile) + .catch(err => logger.error('Cannot build new video jobs of %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) })) + + Hooks.runAction('action:api.video.uploaded', { video: videoCreated, req, res }) + + return { + video: { + id: videoCreated.id, + shortUUID: uuidToShort(videoCreated.uuid), + uuid: videoCreated.uuid + } + } +} + +async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVideoFile) { + const jobs: (CreateJobArgument & CreateJobOptions)[] = [ + { + type: 'manage-video-torrent' as 'manage-video-torrent', + payload: { + videoId: video.id, + videoFileId: videoFile.id, + action: 'create' + } + }, + + { + type: 'generate-video-storyboard' as 'generate-video-storyboard', + payload: { + videoUUID: video.uuid, + // No need to federate, we process these jobs sequentially + federate: false + } + }, + + { + type: 'notify', + payload: { + action: 'new-video', + videoUUID: video.uuid + } + }, + + { + type: 'federate-video' as 'federate-video', + payload: { + videoUUID: video.uuid, + isNewVideo: true + } + } + ] + + if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { + jobs.push(await buildMoveToObjectStorageJob({ video, previousVideoState: undefined })) + } + + if (video.state === VideoState.TO_TRANSCODE) { + jobs.push({ + type: 'transcoding-job-builder' as 'transcoding-job-builder', + payload: { + videoUUID: video.uuid, + optimizeJob: { + isNewVideo: true + } + } + }) + } + + return JobQueue.Instance.createSequentialJobFlow(...jobs) +} + +async function deleteUploadResumableCache (req: express.Request, res: express.Response, next: express.NextFunction) { + await Redis.Instance.deleteUploadSession(req.query.upload_id) + + return next() +} diff --git a/server/server/controllers/api/videos/view.ts b/server/server/controllers/api/videos/view.ts new file mode 100644 index 000000000..cc0534753 --- /dev/null +++ b/server/server/controllers/api/videos/view.ts @@ -0,0 +1,66 @@ +import express from 'express' +import { HttpStatusCode, VideoView } from '@peertube/peertube-models' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { VideoViewsManager } from '@server/lib/views/video-views-manager.js' +import { MVideoId } from '@server/types/models/index.js' +import { + asyncMiddleware, + methodsValidator, + openapiOperationDoc, + optionalAuthenticate, + videoViewValidator +} from '../../../middlewares/index.js' +import { UserVideoHistoryModel } from '../../../models/user/user-video-history.js' + +const viewRouter = express.Router() + +viewRouter.all( + [ '/:videoId/views', '/:videoId/watching' ], + openapiOperationDoc({ operationId: 'addView' }), + methodsValidator([ 'PUT', 'POST' ]), + optionalAuthenticate, + asyncMiddleware(videoViewValidator), + asyncMiddleware(viewVideo) +) + +// --------------------------------------------------------------------------- + +export { + viewRouter +} + +// --------------------------------------------------------------------------- + +async function viewVideo (req: express.Request, res: express.Response) { + const video = res.locals.onlyImmutableVideo + + const body = req.body as VideoView + + const ip = req.ip + const { successView } = await VideoViewsManager.Instance.processLocalView({ + video, + ip, + currentTime: body.currentTime, + viewEvent: body.viewEvent + }) + + if (successView) { + Hooks.runAction('action:api.video.viewed', { video, ip, req, res }) + } + + await updateUserHistoryIfNeeded(body, video, res) + + return res.status(HttpStatusCode.NO_CONTENT_204).end() +} + +async function updateUserHistoryIfNeeded (body: VideoView, video: MVideoId, res: express.Response) { + const user = res.locals.oauth?.token.User + if (!user) return + if (user.videosHistoryEnabled !== true) return + + await UserVideoHistoryModel.upsert({ + videoId: video.id, + userId: user.id, + currentTime: body.currentTime + }) +} diff --git a/server/server/controllers/client.ts b/server/server/controllers/client.ts new file mode 100644 index 000000000..a790859c7 --- /dev/null +++ b/server/server/controllers/client.ts @@ -0,0 +1,236 @@ +import express from 'express' +import { constants, promises as fs } from 'fs' +import { readFile } from 'fs/promises' +import { join } from 'path' +import { buildFileLocale, getCompleteLocale, is18nLocale, LOCALE_FILES } from '@peertube/peertube-core-utils' +import { HttpStatusCode } from '@peertube/peertube-models' +import { logger } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { currentDir, root } from '@peertube/peertube-node-utils' +import { STATIC_MAX_AGE } from '../initializers/constants.js' +import { ClientHtml, sendHTML, serveIndexHTML } from '../lib/client-html.js' +import { asyncMiddleware, buildRateLimiter, embedCSP } from '../middlewares/index.js' + +const clientsRouter = express.Router() + +const clientsRateLimiter = buildRateLimiter({ + windowMs: CONFIG.RATES_LIMIT.CLIENT.WINDOW_MS, + max: CONFIG.RATES_LIMIT.CLIENT.MAX +}) + +const distPath = join(root(), 'client', 'dist') +const testEmbedPath = join(distPath, 'standalone', 'videos', 'test-embed.html') + +// Special route that add OpenGraph and oEmbed tags +// Do not use a template engine for a so little thing +clientsRouter.use([ '/w/p/:id', '/videos/watch/playlist/:id' ], + clientsRateLimiter, + asyncMiddleware(generateWatchPlaylistHtmlPage) +) + +clientsRouter.use([ '/w/:id', '/videos/watch/:id' ], + clientsRateLimiter, + asyncMiddleware(generateWatchHtmlPage) +) + +clientsRouter.use([ '/accounts/:nameWithHost', '/a/:nameWithHost' ], + clientsRateLimiter, + asyncMiddleware(generateAccountHtmlPage) +) + +clientsRouter.use([ '/video-channels/:nameWithHost', '/c/:nameWithHost' ], + clientsRateLimiter, + asyncMiddleware(generateVideoChannelHtmlPage) +) + +clientsRouter.use('/@:nameWithHost', + clientsRateLimiter, + asyncMiddleware(generateActorHtmlPage) +) + +const embedMiddlewares = [ + clientsRateLimiter, + + CONFIG.CSP.ENABLED + ? embedCSP + : (req: express.Request, res: express.Response, next: express.NextFunction) => next(), + + // Set headers + (req: express.Request, res: express.Response, next: express.NextFunction) => { + res.removeHeader('X-Frame-Options') + + // Don't cache HTML file since it's an index to the immutable JS/CSS files + res.setHeader('Cache-Control', 'public, max-age=0') + + next() + }, + + asyncMiddleware(generateEmbedHtmlPage) +] + +clientsRouter.use('/videos/embed', ...embedMiddlewares) +clientsRouter.use('/video-playlists/embed', ...embedMiddlewares) + +const testEmbedController = (req: express.Request, res: express.Response) => res.sendFile(testEmbedPath) + +clientsRouter.use('/videos/test-embed', clientsRateLimiter, testEmbedController) +clientsRouter.use('/video-playlists/test-embed', clientsRateLimiter, testEmbedController) + +// Dynamic PWA manifest +clientsRouter.get('/manifest.webmanifest', clientsRateLimiter, asyncMiddleware(generateManifest)) + +// Static client overrides +// Must be consistent with static client overrides redirections in /support/nginx/peertube +const staticClientOverrides = [ + 'assets/images/logo.svg', + 'assets/images/favicon.png', + 'assets/images/icons/icon-36x36.png', + 'assets/images/icons/icon-48x48.png', + 'assets/images/icons/icon-72x72.png', + 'assets/images/icons/icon-96x96.png', + 'assets/images/icons/icon-144x144.png', + 'assets/images/icons/icon-192x192.png', + 'assets/images/icons/icon-512x512.png', + 'assets/images/default-playlist.jpg', + 'assets/images/default-avatar-account.png', + 'assets/images/default-avatar-account-48x48.png', + 'assets/images/default-avatar-video-channel.png', + 'assets/images/default-avatar-video-channel-48x48.png' +] + +for (const staticClientOverride of staticClientOverrides) { + const overridePhysicalPath = join(CONFIG.STORAGE.CLIENT_OVERRIDES_DIR, staticClientOverride) + clientsRouter.use(`/client/${staticClientOverride}`, asyncMiddleware(serveClientOverride(overridePhysicalPath))) +} + +clientsRouter.use('/client/locales/:locale/:file.json', serveServerTranslations) +clientsRouter.use('/client', express.static(distPath, { maxAge: STATIC_MAX_AGE.CLIENT })) + +// 404 for static files not found +clientsRouter.use('/client/*', (req: express.Request, res: express.Response) => { + res.status(HttpStatusCode.NOT_FOUND_404).end() +}) + +// Always serve index client page (the client is a single page application, let it handle routing) +// Try to provide the right language index.html +clientsRouter.use('/(:language)?', + clientsRateLimiter, + asyncMiddleware(serveIndexHTML) +) + +// --------------------------------------------------------------------------- + +export { + clientsRouter +} + +// --------------------------------------------------------------------------- + +function serveServerTranslations (req: express.Request, res: express.Response) { + const locale = req.params.locale + const file = req.params.file + + if (is18nLocale(locale) && LOCALE_FILES.includes(file)) { + const completeLocale = getCompleteLocale(locale) + const completeFileLocale = buildFileLocale(completeLocale) + + const path = join(currentDir(import.meta.url), `../../../client/dist/locale/${file}.${completeFileLocale}.json`) + return res.sendFile(path, { maxAge: STATIC_MAX_AGE.SERVER }) + } + + return res.status(HttpStatusCode.NOT_FOUND_404).end() +} + +async function generateEmbedHtmlPage (req: express.Request, res: express.Response) { + const hookName = req.originalUrl.startsWith('/video-playlists/') + ? 'filter:html.embed.video-playlist.allowed.result' + : 'filter:html.embed.video.allowed.result' + + const allowParameters = { req } + + const allowedResult = await Hooks.wrapFun( + isEmbedAllowed, + allowParameters, + hookName + ) + + if (!allowedResult || allowedResult.allowed !== true) { + logger.info('Embed is not allowed.', { allowedResult }) + + return sendHTML(allowedResult?.html || '', res) + } + + const html = await ClientHtml.getEmbedHTML() + + return sendHTML(html, res) +} + +async function generateWatchHtmlPage (req: express.Request, res: express.Response) { + // Thread link is '/w/:videoId;threadId=:threadId' + // So to get the videoId we need to remove the last part + let videoId = req.params.id + '' + + const threadIdIndex = videoId.indexOf(';threadId') + if (threadIdIndex !== -1) videoId = videoId.substring(0, threadIdIndex) + + const html = await ClientHtml.getWatchHTMLPage(videoId, req, res) + + return sendHTML(html, res, true) +} + +async function generateWatchPlaylistHtmlPage (req: express.Request, res: express.Response) { + const html = await ClientHtml.getWatchPlaylistHTMLPage(req.params.id + '', req, res) + + return sendHTML(html, res, true) +} + +async function generateAccountHtmlPage (req: express.Request, res: express.Response) { + const html = await ClientHtml.getAccountHTMLPage(req.params.nameWithHost, req, res) + + return sendHTML(html, res, true) +} + +async function generateVideoChannelHtmlPage (req: express.Request, res: express.Response) { + const html = await ClientHtml.getVideoChannelHTMLPage(req.params.nameWithHost, req, res) + + return sendHTML(html, res, true) +} + +async function generateActorHtmlPage (req: express.Request, res: express.Response) { + const html = await ClientHtml.getActorHTMLPage(req.params.nameWithHost, req, res) + + return sendHTML(html, res, true) +} + +async function generateManifest (req: express.Request, res: express.Response) { + const manifestPhysicalPath = join(root(), 'client', 'dist', 'manifest.webmanifest') + const manifestJson = await readFile(manifestPhysicalPath, 'utf8') + const manifest = JSON.parse(manifestJson) + + manifest.name = CONFIG.INSTANCE.NAME + manifest.short_name = CONFIG.INSTANCE.NAME + manifest.description = CONFIG.INSTANCE.SHORT_DESCRIPTION + + res.json(manifest) +} + +function serveClientOverride (path: string) { + return async (req: express.Request, res: express.Response, next: express.NextFunction) => { + try { + await fs.access(path, constants.F_OK) + // Serve override client + res.sendFile(path, { maxAge: STATIC_MAX_AGE.SERVER }) + } catch { + // Serve dist client + next() + } + } +} + +type AllowedResult = { allowed: boolean, html?: string } +function isEmbedAllowed (_object: { + req: express.Request +}): AllowedResult { + return { allowed: true } +} diff --git a/server/server/controllers/download.ts b/server/server/controllers/download.ts new file mode 100644 index 000000000..51b7b3af5 --- /dev/null +++ b/server/server/controllers/download.ts @@ -0,0 +1,213 @@ +import cors from 'cors' +import express from 'express' +import { logger } from '@server/helpers/logger.js' +import { VideoTorrentsSimpleFileCache } from '@server/lib/files-cache/index.js' +import { generateHLSFilePresignedUrl, generateWebVideoPresignedUrl } from '@server/lib/object-storage/index.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { VideoPathManager } from '@server/lib/video-path-manager.js' +import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models/index.js' +import { forceNumber } from '@peertube/peertube-core-utils' +import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@peertube/peertube-models' +import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants.js' +import { asyncMiddleware, optionalAuthenticate, videosDownloadValidator } from '../middlewares/index.js' + +const downloadRouter = express.Router() + +downloadRouter.use(cors()) + +downloadRouter.use( + STATIC_DOWNLOAD_PATHS.TORRENTS + ':filename', + asyncMiddleware(downloadTorrent) +) + +downloadRouter.use( + STATIC_DOWNLOAD_PATHS.VIDEOS + ':id-:resolution([0-9]+).:extension', + optionalAuthenticate, + asyncMiddleware(videosDownloadValidator), + asyncMiddleware(downloadVideoFile) +) + +downloadRouter.use( + STATIC_DOWNLOAD_PATHS.HLS_VIDEOS + ':id-:resolution([0-9]+)-fragmented.:extension', + optionalAuthenticate, + asyncMiddleware(videosDownloadValidator), + asyncMiddleware(downloadHLSVideoFile) +) + +// --------------------------------------------------------------------------- + +export { + downloadRouter +} + +// --------------------------------------------------------------------------- + +async function downloadTorrent (req: express.Request, res: express.Response) { + const result = await VideoTorrentsSimpleFileCache.Instance.getFilePath(req.params.filename) + if (!result) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Torrent file not found' + }) + } + + const allowParameters = { + req, + res, + torrentPath: result.path, + downloadName: result.downloadName + } + + const allowedResult = await Hooks.wrapFun( + isTorrentDownloadAllowed, + allowParameters, + 'filter:api.download.torrent.allowed.result' + ) + + if (!checkAllowResult(res, allowParameters, allowedResult)) return + + return res.download(result.path, result.downloadName) +} + +async function downloadVideoFile (req: express.Request, res: express.Response) { + const video = res.locals.videoAll + + const videoFile = getVideoFile(req, video.VideoFiles) + if (!videoFile) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video file not found' + }) + } + + const allowParameters = { + req, + res, + video, + videoFile + } + + const allowedResult = await Hooks.wrapFun( + isVideoDownloadAllowed, + allowParameters, + 'filter:api.download.video.allowed.result' + ) + + if (!checkAllowResult(res, allowParameters, allowedResult)) return + + // Express uses basename on filename parameter + const videoName = video.name.replace(/[/\\]/g, '_') + const downloadFilename = `${videoName}-${videoFile.resolution}p${videoFile.extname}` + + if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { + return redirectToObjectStorage({ req, res, video, file: videoFile, downloadFilename }) + } + + await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), path => { + return res.download(path, downloadFilename) + }) +} + +async function downloadHLSVideoFile (req: express.Request, res: express.Response) { + const video = res.locals.videoAll + const streamingPlaylist = getHLSPlaylist(video) + if (!streamingPlaylist) return res.status(HttpStatusCode.NOT_FOUND_404).end + + const videoFile = getVideoFile(req, streamingPlaylist.VideoFiles) + if (!videoFile) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video file not found' + }) + } + + const allowParameters = { + req, + res, + video, + streamingPlaylist, + videoFile + } + + const allowedResult = await Hooks.wrapFun( + isVideoDownloadAllowed, + allowParameters, + 'filter:api.download.video.allowed.result' + ) + + if (!checkAllowResult(res, allowParameters, allowedResult)) return + + const downloadFilename = `${video.name}-${videoFile.resolution}p-${streamingPlaylist.getStringType()}${videoFile.extname}` + + if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { + return redirectToObjectStorage({ req, res, video, streamingPlaylist, file: videoFile, downloadFilename }) + } + + await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(streamingPlaylist), path => { + return res.download(path, downloadFilename) + }) +} + +function getVideoFile (req: express.Request, files: MVideoFile[]) { + const resolution = forceNumber(req.params.resolution) + return files.find(f => f.resolution === resolution) +} + +function getHLSPlaylist (video: MVideoFullLight) { + const playlist = video.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) + if (!playlist) return undefined + + return Object.assign(playlist, { Video: video }) +} + +type AllowedResult = { + allowed: boolean + errorMessage?: string +} + +function isTorrentDownloadAllowed (_object: { + torrentPath: string +}): AllowedResult { + return { allowed: true } +} + +function isVideoDownloadAllowed (_object: { + video: MVideo + videoFile: MVideoFile + streamingPlaylist?: MStreamingPlaylist +}): AllowedResult { + return { allowed: true } +} + +function checkAllowResult (res: express.Response, allowParameters: any, result?: AllowedResult) { + if (!result || result.allowed !== true) { + logger.info('Download is not allowed.', { result, allowParameters }) + + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: result?.errorMessage || 'Refused download' + }) + return false + } + + return true +} + +async function redirectToObjectStorage (options: { + req: express.Request + res: express.Response + video: MVideo + file: MVideoFile + streamingPlaylist?: MStreamingPlaylistVideo + downloadFilename: string +}) { + const { res, video, streamingPlaylist, file, downloadFilename } = options + + const url = streamingPlaylist + ? await generateHLSFilePresignedUrl({ streamingPlaylist, file, downloadFilename }) + : await generateWebVideoPresignedUrl({ file, downloadFilename }) + + logger.debug('Generating pre-signed URL %s for video %s', url, video.uuid) + + return res.redirect(url) +} diff --git a/server/server/controllers/feeds/comment-feeds.ts b/server/server/controllers/feeds/comment-feeds.ts new file mode 100644 index 000000000..4178f198e --- /dev/null +++ b/server/server/controllers/feeds/comment-feeds.ts @@ -0,0 +1,96 @@ +import express from 'express' +import { toSafeHtml } from '@server/helpers/markdown.js' +import { cacheRouteFactory } from '@server/middlewares/index.js' +import { CONFIG } from '../../initializers/config.js' +import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../../initializers/constants.js' +import { + asyncMiddleware, + feedsFormatValidator, + setFeedFormatContentType, + videoCommentsFeedsValidator, + feedsAccountOrChannelFiltersValidator +} from '../../middlewares/index.js' +import { VideoCommentModel } from '../../models/video/video-comment.js' +import { buildFeedMetadata, initFeed, sendFeed } from './shared/index.js' + +const commentFeedsRouter = express.Router() + +// --------------------------------------------------------------------------- + +const { middleware: cacheRouteMiddleware } = cacheRouteFactory({ + headerBlacklist: [ 'Content-Type' ] +}) + +// --------------------------------------------------------------------------- + +commentFeedsRouter.get('/video-comments.:format', + feedsFormatValidator, + setFeedFormatContentType, + cacheRouteMiddleware(ROUTE_CACHE_LIFETIME.FEEDS), + asyncMiddleware(feedsAccountOrChannelFiltersValidator), + asyncMiddleware(videoCommentsFeedsValidator), + asyncMiddleware(generateVideoCommentsFeed) +) + +// --------------------------------------------------------------------------- + +export { + commentFeedsRouter +} + +// --------------------------------------------------------------------------- + +async function generateVideoCommentsFeed (req: express.Request, res: express.Response) { + const start = 0 + const video = res.locals.videoAll + const account = res.locals.account + const videoChannel = res.locals.videoChannel + + const comments = await VideoCommentModel.listForFeed({ + start, + count: CONFIG.FEEDS.COMMENTS.COUNT, + videoId: video ? video.id : undefined, + accountId: account ? account.id : undefined, + videoChannelId: videoChannel ? videoChannel.id : undefined + }) + + const { name, description, imageUrl, link } = await buildFeedMetadata({ video, account, videoChannel }) + + const feed = initFeed({ + name, + description, + imageUrl, + isPodcast: false, + link, + resourceType: 'video-comments', + queryString: new URL(WEBSERVER.URL + req.originalUrl).search + }) + + // Adding video items to the feed, one at a time + for (const comment of comments) { + const localLink = WEBSERVER.URL + comment.getCommentStaticPath() + + let title = comment.Video.name + const author: { name: string, link: string }[] = [] + + if (comment.Account) { + title += ` - ${comment.Account.getDisplayName()}` + author.push({ + name: comment.Account.getDisplayName(), + link: comment.Account.Actor.url + }) + } + + feed.addItem({ + title, + id: localLink, + link: localLink, + content: toSafeHtml(comment.text), + author, + date: comment.createdAt + }) + } + + // Now the feed generation is done, let's send it! + return sendFeed(feed, req, res) +} diff --git a/server/server/controllers/feeds/index.ts b/server/server/controllers/feeds/index.ts new file mode 100644 index 000000000..b7210372f --- /dev/null +++ b/server/server/controllers/feeds/index.ts @@ -0,0 +1,25 @@ +import express from 'express' +import { CONFIG } from '@server/initializers/config.js' +import { buildRateLimiter } from '@server/middlewares/index.js' +import { commentFeedsRouter } from './comment-feeds.js' +import { videoFeedsRouter } from './video-feeds.js' +import { videoPodcastFeedsRouter } from './video-podcast-feeds.js' + +const feedsRouter = express.Router() + +const feedsRateLimiter = buildRateLimiter({ + windowMs: CONFIG.RATES_LIMIT.FEEDS.WINDOW_MS, + max: CONFIG.RATES_LIMIT.FEEDS.MAX +}) + +feedsRouter.use('/feeds', feedsRateLimiter) + +feedsRouter.use('/feeds', commentFeedsRouter) +feedsRouter.use('/feeds', videoFeedsRouter) +feedsRouter.use('/feeds', videoPodcastFeedsRouter) + +// --------------------------------------------------------------------------- + +export { + feedsRouter +} diff --git a/server/server/controllers/feeds/shared/common-feed-utils.ts b/server/server/controllers/feeds/shared/common-feed-utils.ts new file mode 100644 index 000000000..af9cafb5d --- /dev/null +++ b/server/server/controllers/feeds/shared/common-feed-utils.ts @@ -0,0 +1,149 @@ +import express from 'express' +import { Feed } from '@peertube/feed' +import { CustomTag, CustomXMLNS, Person } from '@peertube/feed/lib/typings/index.js' +import { mdToOneLinePlainText } from '@server/helpers/markdown.js' +import { CONFIG } from '@server/initializers/config.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { getBiggestActorImage } from '@server/lib/actor-image.js' +import { UserModel } from '@server/models/user/user.js' +import { MAccountDefault, MChannelBannerAccountDefault, MUser, MVideoFullLight } from '@server/types/models/index.js' +import { pick } from '@peertube/peertube-core-utils' +import { ActorImageType } from '@peertube/peertube-models' + +export function initFeed (parameters: { + name: string + description: string + imageUrl: string + isPodcast: boolean + link?: string + locked?: { isLocked: boolean, email: string } + author?: { + name: string + link: string + imageUrl: string + } + person?: Person[] + resourceType?: 'videos' | 'video-comments' + queryString?: string + medium?: string + stunServers?: string[] + trackers?: string[] + customXMLNS?: CustomXMLNS[] + customTags?: CustomTag[] +}) { + const webserverUrl = WEBSERVER.URL + const { name, description, link, imageUrl, isPodcast, resourceType, queryString, medium } = parameters + + return new Feed({ + title: name, + description: mdToOneLinePlainText(description), + // updated: TODO: somehowGetLatestUpdate, // optional, default = today + id: link || webserverUrl, + link: link || webserverUrl, + image: imageUrl, + favicon: webserverUrl + '/client/assets/images/favicon.png', + copyright: `All rights reserved, unless otherwise specified in the terms specified at ${webserverUrl}/about` + + ` and potential licenses granted by each content's rightholder.`, + generator: `Toraifōsu`, // ^.~ + medium: medium || 'video', + feedLinks: { + json: `${webserverUrl}/feeds/${resourceType}.json${queryString}`, + atom: `${webserverUrl}/feeds/${resourceType}.atom${queryString}`, + rss: isPodcast + ? `${webserverUrl}/feeds/podcast/videos.xml${queryString}` + : `${webserverUrl}/feeds/${resourceType}.xml${queryString}` + }, + + ...pick(parameters, [ 'stunServers', 'trackers', 'customXMLNS', 'customTags', 'author', 'person', 'locked' ]) + }) +} + +export function sendFeed (feed: Feed, req: express.Request, res: express.Response) { + const format = req.params.format + + if (format === 'atom' || format === 'atom1') { + return res.send(feed.atom1()).end() + } + + if (format === 'json' || format === 'json1') { + return res.send(feed.json1()).end() + } + + if (format === 'rss' || format === 'rss2') { + return res.send(feed.rss2()).end() + } + + // We're in the ambiguous '.xml' case and we look at the format query parameter + if (req.query.format === 'atom' || req.query.format === 'atom1') { + return res.send(feed.atom1()).end() + } + + return res.send(feed.rss2()).end() +} + +export async function buildFeedMetadata (options: { + videoChannel?: MChannelBannerAccountDefault + account?: MAccountDefault + video?: MVideoFullLight +}) { + const { video, videoChannel, account } = options + + let imageUrl = WEBSERVER.URL + '/client/assets/images/icons/icon-96x96.png' + let accountImageUrl: string + let name: string + let userName: string + let description: string + let email: string + let link: string + let accountLink: string + let user: MUser + + if (videoChannel) { + name = videoChannel.getDisplayName() + description = videoChannel.description + link = videoChannel.getClientUrl() + accountLink = videoChannel.Account.getClientUrl() + + if (videoChannel.Actor.hasImage(ActorImageType.AVATAR)) { + const videoChannelAvatar = getBiggestActorImage(videoChannel.Actor.Avatars) + imageUrl = WEBSERVER.URL + videoChannelAvatar.getStaticPath() + } + + if (videoChannel.Account.Actor.hasImage(ActorImageType.AVATAR)) { + const accountAvatar = getBiggestActorImage(videoChannel.Account.Actor.Avatars) + accountImageUrl = WEBSERVER.URL + accountAvatar.getStaticPath() + } + + user = await UserModel.loadById(videoChannel.Account.userId) + userName = videoChannel.Account.getDisplayName() + } else if (account) { + name = account.getDisplayName() + description = account.description + link = account.getClientUrl() + accountLink = link + + if (account.Actor.hasImage(ActorImageType.AVATAR)) { + const accountAvatar = getBiggestActorImage(account.Actor.Avatars) + imageUrl = WEBSERVER.URL + accountAvatar?.getStaticPath() + accountImageUrl = imageUrl + } + + user = await UserModel.loadById(account.userId) + } else if (video) { + name = video.name + description = video.description + link = video.url + } else { + name = CONFIG.INSTANCE.NAME + description = CONFIG.INSTANCE.DESCRIPTION + link = WEBSERVER.URL + } + + // If the user is local, has a verified email address, and allows it to be publicly displayed + // Return it so the owner can prove ownership of their feed + if (user && !user.pluginAuth && user.emailVerified && user.emailPublic) { + email = user.email + } + + return { name, userName, description, imageUrl, accountImageUrl, email, link, accountLink } +} diff --git a/server/server/controllers/feeds/shared/index.ts b/server/server/controllers/feeds/shared/index.ts new file mode 100644 index 000000000..b36a7265c --- /dev/null +++ b/server/server/controllers/feeds/shared/index.ts @@ -0,0 +1,2 @@ +export * from './video-feed-utils.js' +export * from './common-feed-utils.js' diff --git a/server/server/controllers/feeds/shared/video-feed-utils.ts b/server/server/controllers/feeds/shared/video-feed-utils.ts new file mode 100644 index 000000000..b6306c214 --- /dev/null +++ b/server/server/controllers/feeds/shared/video-feed-utils.ts @@ -0,0 +1,66 @@ +import { VideoIncludeType } from '@peertube/peertube-models' +import { mdToOneLinePlainText, toSafeHtml } from '@server/helpers/markdown.js' +import { CONFIG } from '@server/initializers/config.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { getServerActor } from '@server/models/application/application.js' +import { getCategoryLabel } from '@server/models/video/formatter/index.js' +import { DisplayOnlyForFollowerOptions } from '@server/models/video/sql/video/index.js' +import { VideoModel } from '@server/models/video/video.js' +import { MThumbnail, MUserDefault } from '@server/types/models/index.js' + +export async function getVideosForFeeds (options: { + sort: string + nsfw: boolean + isLocal: boolean + include: VideoIncludeType + + accountId?: number + videoChannelId?: number + displayOnlyForFollower?: DisplayOnlyForFollowerOptions + user?: MUserDefault +}) { + const server = await getServerActor() + + const { data } = await VideoModel.listForApi({ + start: 0, + count: CONFIG.FEEDS.VIDEOS.COUNT, + displayOnlyForFollower: { + actorId: server.id, + orLocalVideos: true + }, + hasFiles: true, + countVideos: false, + + ...options + }) + + return data +} + +export function getCommonVideoFeedAttributes (video: VideoModel) { + const localLink = WEBSERVER.URL + video.getWatchStaticPath() + + const thumbnailModels: MThumbnail[] = [] + if (video.hasPreview()) thumbnailModels.push(video.getPreview()) + thumbnailModels.push(video.getMiniature()) + + return { + title: video.name, + link: localLink, + description: mdToOneLinePlainText(video.getTruncatedDescription()), + content: toSafeHtml(video.description), + + date: video.publishedAt, + nsfw: video.nsfw, + + category: video.category + ? [ { name: getCategoryLabel(video.category) } ] + : undefined, + + thumbnails: thumbnailModels.map(t => ({ + url: WEBSERVER.URL + t.getLocalStaticPath(), + width: t.width, + height: t.height + })) + } +} diff --git a/server/server/controllers/feeds/video-feeds.ts b/server/server/controllers/feeds/video-feeds.ts new file mode 100644 index 000000000..fc0e6e0f7 --- /dev/null +++ b/server/server/controllers/feeds/video-feeds.ts @@ -0,0 +1,189 @@ +import express from 'express' +import { extname } from 'path' +import { Feed } from '@peertube/feed' +import { cacheRouteFactory } from '@server/middlewares/index.js' +import { VideoModel } from '@server/models/video/video.js' +import { VideoInclude } from '@peertube/peertube-models' +import { buildNSFWFilter } from '../../helpers/express-utils.js' +import { MIMETYPES, ROUTE_CACHE_LIFETIME, WEBSERVER } from '../../initializers/constants.js' +import { + asyncMiddleware, + commonVideosFiltersValidator, + feedsFormatValidator, + setDefaultVideosSort, + setFeedFormatContentType, + feedsAccountOrChannelFiltersValidator, + videosSortValidator, + videoSubscriptionFeedsValidator +} from '../../middlewares/index.js' +import { buildFeedMetadata, getCommonVideoFeedAttributes, getVideosForFeeds, initFeed, sendFeed } from './shared/index.js' + +const videoFeedsRouter = express.Router() + +const { middleware: cacheRouteMiddleware } = cacheRouteFactory({ + headerBlacklist: [ 'Content-Type' ] +}) + +// --------------------------------------------------------------------------- + +videoFeedsRouter.get('/videos.:format', + videosSortValidator, + setDefaultVideosSort, + feedsFormatValidator, + setFeedFormatContentType, + cacheRouteMiddleware(ROUTE_CACHE_LIFETIME.FEEDS), + commonVideosFiltersValidator, + asyncMiddleware(feedsAccountOrChannelFiltersValidator), + asyncMiddleware(generateVideoFeed) +) + +videoFeedsRouter.get('/subscriptions.:format', + videosSortValidator, + setDefaultVideosSort, + feedsFormatValidator, + setFeedFormatContentType, + cacheRouteMiddleware(ROUTE_CACHE_LIFETIME.FEEDS), + commonVideosFiltersValidator, + asyncMiddleware(videoSubscriptionFeedsValidator), + asyncMiddleware(generateVideoFeedForSubscriptions) +) + +// --------------------------------------------------------------------------- + +export { + videoFeedsRouter +} + +// --------------------------------------------------------------------------- + +async function generateVideoFeed (req: express.Request, res: express.Response) { + const account = res.locals.account + const videoChannel = res.locals.videoChannel + + const { name, description, imageUrl, accountImageUrl, link, accountLink } = await buildFeedMetadata({ videoChannel, account }) + + const feed = initFeed({ + name, + description, + link, + isPodcast: false, + imageUrl, + author: { name, link: accountLink, imageUrl: accountImageUrl }, + resourceType: 'videos', + queryString: new URL(WEBSERVER.URL + req.url).search + }) + + const data = await getVideosForFeeds({ + sort: req.query.sort, + nsfw: buildNSFWFilter(res, req.query.nsfw), + isLocal: req.query.isLocal, + include: req.query.include | VideoInclude.FILES, + accountId: account?.id, + videoChannelId: videoChannel?.id + }) + + addVideosToFeed(feed, data) + + // Now the feed generation is done, let's send it! + return sendFeed(feed, req, res) +} + +async function generateVideoFeedForSubscriptions (req: express.Request, res: express.Response) { + const account = res.locals.account + const { name, description, imageUrl, link } = await buildFeedMetadata({ account }) + + const feed = initFeed({ + name, + description, + link, + isPodcast: false, + imageUrl, + resourceType: 'videos', + queryString: new URL(WEBSERVER.URL + req.url).search + }) + + const data = await getVideosForFeeds({ + sort: req.query.sort, + nsfw: buildNSFWFilter(res, req.query.nsfw), + isLocal: req.query.isLocal, + include: req.query.include | VideoInclude.FILES, + displayOnlyForFollower: { + actorId: res.locals.user.Account.Actor.id, + orLocalVideos: false + }, + user: res.locals.user + }) + + addVideosToFeed(feed, data) + + // Now the feed generation is done, let's send it! + return sendFeed(feed, req, res) +} + +// --------------------------------------------------------------------------- + +function addVideosToFeed (feed: Feed, videos: VideoModel[]) { + /** + * Adding video items to the feed object, one at a time + */ + for (const video of videos) { + const formattedVideoFiles = video.getFormattedAllVideoFilesJSON(false) + + const torrents = formattedVideoFiles.map(videoFile => ({ + title: video.name, + url: videoFile.torrentUrl, + size_in_bytes: videoFile.size + })) + + const videoFiles = formattedVideoFiles.map(videoFile => { + return { + type: MIMETYPES.VIDEO.EXT_MIMETYPE[extname(videoFile.fileUrl)], + medium: 'video', + height: videoFile.resolution.id, + fileSize: videoFile.size, + url: videoFile.fileUrl, + framerate: videoFile.fps, + duration: video.duration, + lang: video.language + } + }) + + feed.addItem({ + ...getCommonVideoFeedAttributes(video), + + id: WEBSERVER.URL + video.getWatchStaticPath(), + author: [ + { + name: video.VideoChannel.getDisplayName(), + link: video.VideoChannel.getClientUrl() + } + ], + torrents, + + // Enclosure + video: videoFiles.length !== 0 + ? { + url: videoFiles[0].url, + length: videoFiles[0].fileSize, + type: videoFiles[0].type + } + : undefined, + + // Media RSS + videos: videoFiles, + + embed: { + url: WEBSERVER.URL + video.getEmbedStaticPath(), + allowFullscreen: true + }, + player: { + url: WEBSERVER.URL + video.getWatchStaticPath() + }, + community: { + statistics: { + views: video.views + } + } + }) + } +} diff --git a/server/server/controllers/feeds/video-podcast-feeds.ts b/server/server/controllers/feeds/video-podcast-feeds.ts new file mode 100644 index 000000000..84d5acadd --- /dev/null +++ b/server/server/controllers/feeds/video-podcast-feeds.ts @@ -0,0 +1,313 @@ +import express from 'express' +import { extname } from 'path' +import { Feed } from '@peertube/feed' +import { CustomTag, CustomXMLNS, LiveItemStatus } from '@peertube/feed/lib/typings/index.js' +import { getBiggestActorImage } from '@server/lib/actor-image.js' +import { InternalEventEmitter } from '@server/lib/internal-event-emitter.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { buildPodcastGroupsCache, cacheRouteFactory, videoFeedsPodcastSetCacheKey } from '@server/middlewares/index.js' +import { MVideo, MVideoCaptionVideo, MVideoFullLight } from '@server/types/models/index.js' +import { sortObjectComparator } from '@peertube/peertube-core-utils' +import { ActorImageType, VideoFile, VideoInclude, VideoResolution, VideoState } from '@peertube/peertube-models' +import { buildNSFWFilter } from '../../helpers/express-utils.js' +import { MIMETYPES, ROUTE_CACHE_LIFETIME, WEBSERVER } from '../../initializers/constants.js' +import { asyncMiddleware, setFeedPodcastContentType, videoFeedsPodcastValidator } from '../../middlewares/index.js' +import { VideoModel } from '../../models/video/video.js' +import { VideoCaptionModel } from '../../models/video/video-caption.js' +import { buildFeedMetadata, getCommonVideoFeedAttributes, getVideosForFeeds, initFeed } from './shared/index.js' + +const videoPodcastFeedsRouter = express.Router() + +// --------------------------------------------------------------------------- + +const { middleware: podcastCacheRouteMiddleware, instance: podcastApiCache } = cacheRouteFactory({ + headerBlacklist: [ 'Content-Type' ] +}) + +for (const event of ([ 'video-created', 'video-updated', 'video-deleted' ] as const)) { + InternalEventEmitter.Instance.on(event, ({ video }) => { + if (video.remote) return + + podcastApiCache.clearGroupSafe(buildPodcastGroupsCache({ channelId: video.channelId })) + }) +} + +for (const event of ([ 'channel-updated', 'channel-deleted' ] as const)) { + InternalEventEmitter.Instance.on(event, ({ channel }) => { + podcastApiCache.clearGroupSafe(buildPodcastGroupsCache({ channelId: channel.id })) + }) +} + +// --------------------------------------------------------------------------- + +videoPodcastFeedsRouter.get('/podcast/videos.xml', + setFeedPodcastContentType, + videoFeedsPodcastSetCacheKey, + podcastCacheRouteMiddleware(ROUTE_CACHE_LIFETIME.FEEDS), + asyncMiddleware(videoFeedsPodcastValidator), + asyncMiddleware(generateVideoPodcastFeed) +) + +// --------------------------------------------------------------------------- + +export { + videoPodcastFeedsRouter +} + +// --------------------------------------------------------------------------- + +async function generateVideoPodcastFeed (req: express.Request, res: express.Response) { + const videoChannel = res.locals.videoChannel + + const { name, userName, description, imageUrl, accountImageUrl, email, link, accountLink } = await buildFeedMetadata({ videoChannel }) + + const data = await getVideosForFeeds({ + sort: '-publishedAt', + nsfw: buildNSFWFilter(), + // Prevent podcast feeds from listing videos in other instances + // helps prevent duplicates when they are indexed -- only the author should control them + isLocal: true, + include: VideoInclude.FILES, + videoChannelId: videoChannel?.id + }) + + const customTags: CustomTag[] = await Hooks.wrapObject( + [], + 'filter:feed.podcast.channel.create-custom-tags.result', + { videoChannel } + ) + + const customXMLNS: CustomXMLNS[] = await Hooks.wrapObject( + [], + 'filter:feed.podcast.rss.create-custom-xmlns.result' + ) + + const feed = initFeed({ + name, + description, + link, + isPodcast: true, + imageUrl, + + locked: email + ? { isLocked: true, email } // Default to true because we have no way of offering a redirect yet + : undefined, + + person: [ { name: userName, href: accountLink, img: accountImageUrl } ], + resourceType: 'videos', + queryString: new URL(WEBSERVER.URL + req.url).search, + medium: 'video', + customXMLNS, + customTags + }) + + await addVideosToPodcastFeed(feed, data) + + // Now the feed generation is done, let's send it! + return res.send(feed.podcast()).end() +} + +type PodcastMedia = + { + type: string + length: number + bitrate: number + sources: { uri: string, contentType?: string }[] + title: string + language?: string + } | + { + sources: { uri: string }[] + type: string + title: string + } + +async function generatePodcastItem (options: { + video: VideoModel + liveItem: boolean + media: PodcastMedia[] +}) { + const { video, liveItem, media } = options + + const customTags: CustomTag[] = await Hooks.wrapObject( + [], + 'filter:feed.podcast.video.create-custom-tags.result', + { video, liveItem } + ) + + const account = video.VideoChannel.Account + + const author = { + name: account.getDisplayName(), + href: account.getClientUrl() + } + + const commonAttributes = getCommonVideoFeedAttributes(video) + const guid = liveItem + ? `${video.uuid}_${video.publishedAt.toISOString()}` + : commonAttributes.link + + let personImage: string + + if (account.Actor.hasImage(ActorImageType.AVATAR)) { + const avatar = getBiggestActorImage(account.Actor.Avatars) + personImage = WEBSERVER.URL + avatar.getStaticPath() + } + + return { + guid, + ...commonAttributes, + + trackers: video.getTrackerUrls(), + + author: [ author ], + person: [ + { + ...author, + + img: personImage + } + ], + + media, + + socialInteract: [ + { + uri: video.url, + protocol: 'activitypub', + accountUrl: account.getClientUrl() + } + ], + + customTags + } +} + +async function addVideosToPodcastFeed (feed: Feed, videos: VideoModel[]) { + const captionsGroup = await VideoCaptionModel.listCaptionsOfMultipleVideos(videos.map(v => v.id)) + + for (const video of videos) { + if (!video.isLive) { + await addVODPodcastItem({ feed, video, captionsGroup }) + } else if (video.isLive && video.state !== VideoState.LIVE_ENDED) { + await addLivePodcastItem({ feed, video }) + } + } +} + +async function addVODPodcastItem (options: { + feed: Feed + video: VideoModel + captionsGroup: { [ id: number ]: MVideoCaptionVideo[] } +}) { + const { feed, video, captionsGroup } = options + + const webVideos = video.getFormattedWebVideoFilesJSON(true) + .map(f => buildVODWebVideoFile(video, f)) + .sort(sortObjectComparator('bitrate', 'desc')) + + const streamingPlaylistFiles = buildVODStreamingPlaylists(video) + + // Order matters here, the first media URI will be the "default" + // So web videos are default if enabled + const media = [ ...webVideos, ...streamingPlaylistFiles ] + + const videoCaptions = buildVODCaptions(video, captionsGroup[video.id]) + const item = await generatePodcastItem({ video, liveItem: false, media }) + + feed.addPodcastItem({ ...item, subTitle: videoCaptions }) +} + +async function addLivePodcastItem (options: { + feed: Feed + video: VideoModel +}) { + const { feed, video } = options + + let status: LiveItemStatus + + switch (video.state) { + case VideoState.WAITING_FOR_LIVE: + status = LiveItemStatus.pending + break + case VideoState.PUBLISHED: + status = LiveItemStatus.live + break + } + + const item = await generatePodcastItem({ video, liveItem: true, media: buildLiveStreamingPlaylists(video) }) + + feed.addPodcastLiveItem({ ...item, status, start: video.updatedAt.toISOString() }) +} + +// --------------------------------------------------------------------------- + +function buildVODWebVideoFile (video: MVideo, videoFile: VideoFile) { + const isAudio = videoFile.resolution.id === VideoResolution.H_NOVIDEO + const type = isAudio + ? MIMETYPES.AUDIO.EXT_MIMETYPE[extname(videoFile.fileUrl)] + : MIMETYPES.VIDEO.EXT_MIMETYPE[extname(videoFile.fileUrl)] + + const sources = [ + { uri: videoFile.fileUrl }, + { uri: videoFile.torrentUrl, contentType: 'application/x-bittorrent' } + ] + + if (videoFile.magnetUri) { + sources.push({ uri: videoFile.magnetUri }) + } + + return { + type, + title: videoFile.resolution.label, + length: videoFile.size, + bitrate: videoFile.size / video.duration * 8, + language: video.language, + sources + } +} + +function buildVODStreamingPlaylists (video: MVideoFullLight) { + const hls = video.getHLSPlaylist() + if (!hls) return [] + + return [ + { + type: 'application/x-mpegURL', + title: 'HLS', + sources: [ + { uri: hls.getMasterPlaylistUrl(video) } + ], + language: video.language + } + ] +} + +function buildLiveStreamingPlaylists (video: MVideoFullLight) { + const hls = video.getHLSPlaylist() + + return [ + { + type: 'application/x-mpegURL', + title: `HLS live stream`, + sources: [ + { uri: hls.getMasterPlaylistUrl(video) } + ], + language: video.language + } + ] +} + +function buildVODCaptions (video: MVideo, videoCaptions: MVideoCaptionVideo[]) { + return videoCaptions.map(caption => { + const type = MIMETYPES.VIDEO_CAPTIONS.EXT_MIMETYPE[extname(caption.filename)] + if (!type) return null + + return { + url: caption.getFileUrl(video), + language: caption.language, + type, + rel: 'captions' + } + }).filter(c => c) +} diff --git a/server/server/controllers/index.ts b/server/server/controllers/index.ts new file mode 100644 index 000000000..db326c486 --- /dev/null +++ b/server/server/controllers/index.ts @@ -0,0 +1,14 @@ +export * from './activitypub/index.js' +export * from './api/index.js' +export * from './sitemap.js' +export * from './client.js' +export * from './download.js' +export * from './feeds/index.js' +export * from './lazy-static.js' +export * from './misc.js' +export * from './object-storage-proxy.js' +export * from './plugins.js' +export * from './services.js' +export * from './static.js' +export * from './tracker.js' +export * from './well-known.js' diff --git a/server/server/controllers/lazy-static.ts b/server/server/controllers/lazy-static.ts new file mode 100644 index 000000000..69aa549a7 --- /dev/null +++ b/server/server/controllers/lazy-static.ts @@ -0,0 +1,128 @@ +import cors from 'cors' +import express from 'express' +import { HttpStatusCode } from '@peertube/peertube-models' +import { CONFIG } from '@server/initializers/config.js' +import { FILES_CACHE, LAZY_STATIC_PATHS, STATIC_MAX_AGE } from '../initializers/constants.js' +import { + AvatarPermanentFileCache, + VideoCaptionsSimpleFileCache, + VideoMiniaturePermanentFileCache, + VideoPreviewsSimpleFileCache, + VideoStoryboardsSimpleFileCache, + VideoTorrentsSimpleFileCache +} from '../lib/files-cache/index.js' +import { asyncMiddleware, handleStaticError } from '../middlewares/index.js' + +// --------------------------------------------------------------------------- +// Cache initializations +// --------------------------------------------------------------------------- + +VideoPreviewsSimpleFileCache.Instance.init(CONFIG.CACHE.PREVIEWS.SIZE, FILES_CACHE.PREVIEWS.MAX_AGE) +VideoCaptionsSimpleFileCache.Instance.init(CONFIG.CACHE.VIDEO_CAPTIONS.SIZE, FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE) +VideoTorrentsSimpleFileCache.Instance.init(CONFIG.CACHE.TORRENTS.SIZE, FILES_CACHE.TORRENTS.MAX_AGE) +VideoStoryboardsSimpleFileCache.Instance.init(CONFIG.CACHE.STORYBOARDS.SIZE, FILES_CACHE.STORYBOARDS.MAX_AGE) + +// --------------------------------------------------------------------------- + +const lazyStaticRouter = express.Router() + +lazyStaticRouter.use(cors()) + +lazyStaticRouter.use( + LAZY_STATIC_PATHS.AVATARS + ':filename', + asyncMiddleware(getActorImage), + handleStaticError +) + +lazyStaticRouter.use( + LAZY_STATIC_PATHS.BANNERS + ':filename', + asyncMiddleware(getActorImage), + handleStaticError +) + +lazyStaticRouter.use( + LAZY_STATIC_PATHS.THUMBNAILS + ':filename', + asyncMiddleware(getThumbnail), + handleStaticError +) + +lazyStaticRouter.use( + LAZY_STATIC_PATHS.PREVIEWS + ':filename', + asyncMiddleware(getPreview), + handleStaticError +) + +lazyStaticRouter.use( + LAZY_STATIC_PATHS.STORYBOARDS + ':filename', + asyncMiddleware(getStoryboard), + handleStaticError +) + +lazyStaticRouter.use( + LAZY_STATIC_PATHS.VIDEO_CAPTIONS + ':filename', + asyncMiddleware(getVideoCaption), + handleStaticError +) + +lazyStaticRouter.use( + LAZY_STATIC_PATHS.TORRENTS + ':filename', + asyncMiddleware(getTorrent), + handleStaticError +) + +// --------------------------------------------------------------------------- + +export { + lazyStaticRouter, + getPreview, + getVideoCaption +} + +// --------------------------------------------------------------------------- +const avatarPermanentFileCache = new AvatarPermanentFileCache() + +function getActorImage (req: express.Request, res: express.Response, next: express.NextFunction) { + const filename = req.params.filename + + return avatarPermanentFileCache.lazyServe({ filename, res, next }) +} + +// --------------------------------------------------------------------------- +const videoMiniaturePermanentFileCache = new VideoMiniaturePermanentFileCache() + +function getThumbnail (req: express.Request, res: express.Response, next: express.NextFunction) { + const filename = req.params.filename + + return videoMiniaturePermanentFileCache.lazyServe({ filename, res, next }) +} + +// --------------------------------------------------------------------------- + +async function getPreview (req: express.Request, res: express.Response) { + const result = await VideoPreviewsSimpleFileCache.Instance.getFilePath(req.params.filename) + if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end() + + return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }) +} + +async function getStoryboard (req: express.Request, res: express.Response) { + const result = await VideoStoryboardsSimpleFileCache.Instance.getFilePath(req.params.filename) + if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end() + + return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }) +} + +async function getVideoCaption (req: express.Request, res: express.Response) { + const result = await VideoCaptionsSimpleFileCache.Instance.getFilePath(req.params.filename) + if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end() + + return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }) +} + +async function getTorrent (req: express.Request, res: express.Response) { + const result = await VideoTorrentsSimpleFileCache.Instance.getFilePath(req.params.filename) + if (!result) return res.status(HttpStatusCode.NOT_FOUND_404).end() + + // Torrents still use the old naming convention (video uuid + .torrent) + return res.sendFile(result.path, { maxAge: STATIC_MAX_AGE.SERVER }) +} diff --git a/server/server/controllers/misc.ts b/server/server/controllers/misc.ts new file mode 100644 index 000000000..cf204e965 --- /dev/null +++ b/server/server/controllers/misc.ts @@ -0,0 +1,209 @@ +import cors from 'cors' +import express from 'express' +import { HttpNodeinfoDiasporaSoftwareNsSchema20, HttpStatusCode } from '@peertube/peertube-models' +import { CONFIG, isEmailEnabled } from '@server/initializers/config.js' +import { serveIndexHTML } from '@server/lib/client-html.js' +import { ServerConfigManager } from '@server/lib/server-config-manager.js' +import { CONSTRAINTS_FIELDS, DEFAULT_THEME_NAME, PEERTUBE_VERSION, ROUTE_CACHE_LIFETIME } from '../initializers/constants.js' +import { getThemeOrDefault } from '../lib/plugins/theme-utils.js' +import { cacheRoute } from '../middlewares/cache/cache.js' +import { apiRateLimiter, asyncMiddleware } from '../middlewares/index.js' +import { UserModel } from '../models/user/user.js' +import { VideoCommentModel } from '../models/video/video-comment.js' +import { VideoModel } from '../models/video/video.js' + +const miscRouter = express.Router() + +miscRouter.use(cors()) + +miscRouter.use('/nodeinfo/:version.json', + apiRateLimiter, + cacheRoute(ROUTE_CACHE_LIFETIME.NODEINFO), + asyncMiddleware(generateNodeinfo) +) + +// robots.txt service +miscRouter.get('/robots.txt', + apiRateLimiter, + cacheRoute(ROUTE_CACHE_LIFETIME.ROBOTS), + (_, res: express.Response) => { + res.type('text/plain') + + return res.send(CONFIG.INSTANCE.ROBOTS) + } +) + +miscRouter.all('/teapot', + apiRateLimiter, + getCup, + asyncMiddleware(serveIndexHTML) +) + +// security.txt service +miscRouter.get('/security.txt', + apiRateLimiter, + (_, res: express.Response) => { + return res.redirect(HttpStatusCode.MOVED_PERMANENTLY_301, '/.well-known/security.txt') + } +) + +// --------------------------------------------------------------------------- + +export { + miscRouter +} + +// --------------------------------------------------------------------------- + +async function generateNodeinfo (req: express.Request, res: express.Response) { + const { totalVideos } = await VideoModel.getStats() + const { totalLocalVideoComments } = await VideoCommentModel.getStats() + const { totalUsers, totalMonthlyActiveUsers, totalHalfYearActiveUsers } = await UserModel.getStats() + + if (!req.params.version || req.params.version !== '2.0') { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Nodeinfo schema version not handled' + }) + } + + const json = { + version: '2.0', + software: { + name: 'peertube', + version: PEERTUBE_VERSION + }, + protocols: [ + 'activitypub' + ], + services: { + inbound: [], + outbound: [ + 'atom1.0', + 'rss2.0' + ] + }, + openRegistrations: CONFIG.SIGNUP.ENABLED, + usage: { + users: { + total: totalUsers, + activeMonth: totalMonthlyActiveUsers, + activeHalfyear: totalHalfYearActiveUsers + }, + localPosts: totalVideos, + localComments: totalLocalVideoComments + }, + metadata: { + taxonomy: { + postsName: 'Videos' + }, + nodeName: CONFIG.INSTANCE.NAME, + nodeDescription: CONFIG.INSTANCE.SHORT_DESCRIPTION, + nodeConfig: { + search: { + remoteUri: { + users: CONFIG.SEARCH.REMOTE_URI.USERS, + anonymous: CONFIG.SEARCH.REMOTE_URI.ANONYMOUS + } + }, + plugin: { + registered: ServerConfigManager.Instance.getRegisteredPlugins() + }, + theme: { + registered: ServerConfigManager.Instance.getRegisteredThemes(), + default: getThemeOrDefault(CONFIG.THEME.DEFAULT, DEFAULT_THEME_NAME) + }, + email: { + enabled: isEmailEnabled() + }, + contactForm: { + enabled: CONFIG.CONTACT_FORM.ENABLED + }, + transcoding: { + hls: { + enabled: CONFIG.TRANSCODING.HLS.ENABLED + }, + web_videos: { + enabled: CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED + }, + enabledResolutions: ServerConfigManager.Instance.getEnabledResolutions('vod') + }, + live: { + enabled: CONFIG.LIVE.ENABLED, + transcoding: { + enabled: CONFIG.LIVE.TRANSCODING.ENABLED, + enabledResolutions: ServerConfigManager.Instance.getEnabledResolutions('live') + } + }, + import: { + videos: { + http: { + enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED + }, + torrent: { + enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED + } + } + }, + autoBlacklist: { + videos: { + ofUsers: { + enabled: CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED + } + } + }, + avatar: { + file: { + size: { + max: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max + }, + extensions: CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME + } + }, + video: { + image: { + extensions: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.EXTNAME, + size: { + max: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max + } + }, + file: { + extensions: CONSTRAINTS_FIELDS.VIDEOS.EXTNAME + } + }, + videoCaption: { + file: { + size: { + max: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max + }, + extensions: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.EXTNAME + } + }, + user: { + videoQuota: CONFIG.USER.VIDEO_QUOTA, + videoQuotaDaily: CONFIG.USER.VIDEO_QUOTA_DAILY + }, + trending: { + videos: { + intervalDays: CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS + } + }, + tracker: { + enabled: CONFIG.TRACKER.ENABLED + } + } + } + } as HttpNodeinfoDiasporaSoftwareNsSchema20 + + res.contentType('application/json; profile="http://nodeinfo.diaspora.software/ns/schema/2.0#"') + .send(json) + .end() +} + +function getCup (req: express.Request, res: express.Response, next: express.NextFunction) { + res.status(HttpStatusCode.I_AM_A_TEAPOT_418) + res.setHeader('Accept-Additions', 'Non-Dairy;1,Sugar;1') + res.setHeader('Safe', 'if-sepia-awake') + + return next() +} diff --git a/server/server/controllers/object-storage-proxy.ts b/server/server/controllers/object-storage-proxy.ts new file mode 100644 index 000000000..957f46198 --- /dev/null +++ b/server/server/controllers/object-storage-proxy.ts @@ -0,0 +1,60 @@ +import cors from 'cors' +import express from 'express' +import { OBJECT_STORAGE_PROXY_PATHS } from '@server/initializers/constants.js' +import { proxifyHLS, proxifyWebVideoFile } from '@server/lib/object-storage/index.js' +import { + asyncMiddleware, + ensureCanAccessPrivateVideoHLSFiles, + ensureCanAccessVideoPrivateWebVideoFiles, + ensurePrivateObjectStorageProxyIsEnabled, + optionalAuthenticate +} from '@server/middlewares/index.js' +import { doReinjectVideoFileToken } from './shared/m3u8-playlist.js' + +const objectStorageProxyRouter = express.Router() + +objectStorageProxyRouter.use(cors()) + +objectStorageProxyRouter.get( + [ OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEB_VIDEOS + ':filename', OBJECT_STORAGE_PROXY_PATHS.LEGACY_PRIVATE_WEB_VIDEOS + ':filename' ], + ensurePrivateObjectStorageProxyIsEnabled, + optionalAuthenticate, + asyncMiddleware(ensureCanAccessVideoPrivateWebVideoFiles), + asyncMiddleware(proxifyWebVideoController) +) + +objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:filename', + ensurePrivateObjectStorageProxyIsEnabled, + optionalAuthenticate, + asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles), + asyncMiddleware(proxifyHLSController) +) + +// --------------------------------------------------------------------------- + +export { + objectStorageProxyRouter +} + +function proxifyWebVideoController (req: express.Request, res: express.Response) { + const filename = req.params.filename + + return proxifyWebVideoFile({ req, res, filename }) +} + +function proxifyHLSController (req: express.Request, res: express.Response) { + const playlist = res.locals.videoStreamingPlaylist + const video = res.locals.onlyVideo + const filename = req.params.filename + + const reinjectVideoFileToken = filename.endsWith('.m3u8') && doReinjectVideoFileToken(req) + + return proxifyHLS({ + req, + res, + playlist, + video, + filename, + reinjectVideoFileToken + }) +} diff --git a/server/server/controllers/plugins.ts b/server/server/controllers/plugins.ts new file mode 100644 index 000000000..1bcecce0e --- /dev/null +++ b/server/server/controllers/plugins.ts @@ -0,0 +1,174 @@ +import express from 'express' +import { join } from 'path' +import { getCompleteLocale, is18nLocale } from '@peertube/peertube-core-utils' +import { HttpStatusCode, PluginType } from '@peertube/peertube-models' +import { isProdInstance } from '@peertube/peertube-node-utils' +import { logger } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' +import { optionalAuthenticate } from '@server/middlewares/auth.js' +import { buildRateLimiter } from '@server/middlewares/index.js' +import { PLUGIN_GLOBAL_CSS_PATH } from '../initializers/constants.js' +import { PluginManager, RegisteredPlugin } from '../lib/plugins/plugin-manager.js' +import { getExternalAuthValidator, getPluginValidator, pluginStaticDirectoryValidator } from '../middlewares/validators/plugins.js' +import { serveThemeCSSValidator } from '../middlewares/validators/themes.js' + +const sendFileOptions = { + maxAge: '30 days', + immutable: isProdInstance() +} + +const pluginsRouter = express.Router() + +const pluginsRateLimiter = buildRateLimiter({ + windowMs: CONFIG.RATES_LIMIT.PLUGINS.WINDOW_MS, + max: CONFIG.RATES_LIMIT.PLUGINS.MAX +}) + +pluginsRouter.get('/plugins/global.css', + pluginsRateLimiter, + servePluginGlobalCSS +) + +pluginsRouter.get('/plugins/translations/:locale.json', + pluginsRateLimiter, + getPluginTranslations +) + +pluginsRouter.get('/plugins/:pluginName/:pluginVersion/auth/:authName', + pluginsRateLimiter, + getPluginValidator(PluginType.PLUGIN), + getExternalAuthValidator, + handleAuthInPlugin +) + +pluginsRouter.get('/plugins/:pluginName/:pluginVersion/static/:staticEndpoint(*)', + pluginsRateLimiter, + getPluginValidator(PluginType.PLUGIN), + pluginStaticDirectoryValidator, + servePluginStaticDirectory +) + +pluginsRouter.get('/plugins/:pluginName/:pluginVersion/client-scripts/:staticEndpoint(*)', + pluginsRateLimiter, + getPluginValidator(PluginType.PLUGIN), + pluginStaticDirectoryValidator, + servePluginClientScripts +) + +pluginsRouter.use('/plugins/:pluginName/router', + pluginsRateLimiter, + getPluginValidator(PluginType.PLUGIN, false), + optionalAuthenticate, + servePluginCustomRoutes +) + +pluginsRouter.use('/plugins/:pluginName/:pluginVersion/router', + pluginsRateLimiter, + getPluginValidator(PluginType.PLUGIN), + optionalAuthenticate, + servePluginCustomRoutes +) + +pluginsRouter.get('/themes/:pluginName/:pluginVersion/static/:staticEndpoint(*)', + pluginsRateLimiter, + getPluginValidator(PluginType.THEME), + pluginStaticDirectoryValidator, + servePluginStaticDirectory +) + +pluginsRouter.get('/themes/:pluginName/:pluginVersion/client-scripts/:staticEndpoint(*)', + pluginsRateLimiter, + getPluginValidator(PluginType.THEME), + pluginStaticDirectoryValidator, + servePluginClientScripts +) + +pluginsRouter.get('/themes/:themeName/:themeVersion/css/:staticEndpoint(*)', + pluginsRateLimiter, + serveThemeCSSValidator, + serveThemeCSSDirectory +) + +// --------------------------------------------------------------------------- + +export { + pluginsRouter +} + +// --------------------------------------------------------------------------- + +function servePluginGlobalCSS (req: express.Request, res: express.Response) { + // Only cache requests that have a ?hash=... query param + const globalCSSOptions = req.query.hash + ? sendFileOptions + : {} + + return res.sendFile(PLUGIN_GLOBAL_CSS_PATH, globalCSSOptions) +} + +function getPluginTranslations (req: express.Request, res: express.Response) { + const locale = req.params.locale + + if (is18nLocale(locale)) { + const completeLocale = getCompleteLocale(locale) + const json = PluginManager.Instance.getTranslations(completeLocale) + + return res.json(json) + } + + return res.status(HttpStatusCode.NOT_FOUND_404).end() +} + +function servePluginStaticDirectory (req: express.Request, res: express.Response) { + const plugin: RegisteredPlugin = res.locals.registeredPlugin + const staticEndpoint = req.params.staticEndpoint + + const [ directory, ...file ] = staticEndpoint.split('/') + + const staticPath = plugin.staticDirs[directory] + if (!staticPath) return res.status(HttpStatusCode.NOT_FOUND_404).end() + + const filepath = file.join('/') + return res.sendFile(join(plugin.path, staticPath, filepath), sendFileOptions) +} + +function servePluginCustomRoutes (req: express.Request, res: express.Response, next: express.NextFunction) { + const plugin: RegisteredPlugin = res.locals.registeredPlugin + const router = PluginManager.Instance.getRouter(plugin.npmName) + + if (!router) return res.status(HttpStatusCode.NOT_FOUND_404).end() + + return router(req, res, next) +} + +function servePluginClientScripts (req: express.Request, res: express.Response) { + const plugin: RegisteredPlugin = res.locals.registeredPlugin + const staticEndpoint = req.params.staticEndpoint + + const file = plugin.clientScripts[staticEndpoint] + if (!file) return res.status(HttpStatusCode.NOT_FOUND_404).end() + + return res.sendFile(join(plugin.path, staticEndpoint), sendFileOptions) +} + +function serveThemeCSSDirectory (req: express.Request, res: express.Response) { + const plugin: RegisteredPlugin = res.locals.registeredPlugin + const staticEndpoint = req.params.staticEndpoint + + if (plugin.css.includes(staticEndpoint) === false) { + return res.status(HttpStatusCode.NOT_FOUND_404).end() + } + + return res.sendFile(join(plugin.path, staticEndpoint), sendFileOptions) +} + +function handleAuthInPlugin (req: express.Request, res: express.Response) { + const authOptions = res.locals.externalAuth + + try { + logger.debug('Forwarding auth plugin request in %s of plugin %s.', authOptions.authName, res.locals.registeredPlugin.npmName) + authOptions.onAuthRequest(req, res) + } catch (err) { + logger.error('Forward request error in auth %s of plugin %s.', authOptions.authName, res.locals.registeredPlugin.npmName, { err }) + } +} diff --git a/server/server/controllers/services.ts b/server/server/controllers/services.ts new file mode 100644 index 000000000..b59f0f337 --- /dev/null +++ b/server/server/controllers/services.ts @@ -0,0 +1,164 @@ +import express from 'express' +import { escapeHTML, forceNumber } from '@peertube/peertube-core-utils' +import { MChannelSummary } from '@server/types/models/index.js' +import { EMBED_SIZE, PREVIEWS_SIZE, THUMBNAILS_SIZE, WEBSERVER } from '../initializers/constants.js' +import { apiRateLimiter, asyncMiddleware, oembedValidator } from '../middlewares/index.js' +import { accountNameWithHostGetValidator } from '../middlewares/validators/index.js' + +const servicesRouter = express.Router() + +servicesRouter.use('/oembed', + apiRateLimiter, + asyncMiddleware(oembedValidator), + generateOEmbed +) +servicesRouter.use('/redirect/accounts/:accountName', + apiRateLimiter, + asyncMiddleware(accountNameWithHostGetValidator), + redirectToAccountUrl +) + +// --------------------------------------------------------------------------- + +export { + servicesRouter +} + +// --------------------------------------------------------------------------- + +function generateOEmbed (req: express.Request, res: express.Response) { + if (res.locals.videoAll) return generateVideoOEmbed(req, res) + + return generatePlaylistOEmbed(req, res) +} + +function generatePlaylistOEmbed (req: express.Request, res: express.Response) { + const playlist = res.locals.videoPlaylistSummary + + const json = buildOEmbed({ + channel: playlist.VideoChannel, + title: playlist.name, + embedPath: playlist.getEmbedStaticPath() + buildPlayerURLQuery(req.query.url), + previewPath: playlist.getThumbnailStaticPath(), + previewSize: THUMBNAILS_SIZE, + req + }) + + return res.json(json) +} + +function generateVideoOEmbed (req: express.Request, res: express.Response) { + const video = res.locals.videoAll + + const json = buildOEmbed({ + channel: video.VideoChannel, + title: video.name, + embedPath: video.getEmbedStaticPath() + buildPlayerURLQuery(req.query.url), + previewPath: video.getPreviewStaticPath(), + previewSize: PREVIEWS_SIZE, + req + }) + + return res.json(json) +} + +function buildPlayerURLQuery (inputQueryUrl: string) { + const allowedParameters = new Set([ + 'start', + 'stop', + 'loop', + 'autoplay', + 'muted', + 'controls', + 'controlBar', + 'title', + 'api', + 'warningTitle', + 'peertubeLink', + 'p2p', + 'subtitle', + 'bigPlayBackgroundColor', + 'mode', + 'foregroundColor' + ]) + + const params = new URLSearchParams() + + new URL(inputQueryUrl).searchParams.forEach((v, k) => { + if (allowedParameters.has(k)) { + params.append(k, v) + } + }) + + const stringQuery = params.toString() + if (!stringQuery) return '' + + return '?' + stringQuery +} + +function buildOEmbed (options: { + req: express.Request + title: string + channel: MChannelSummary + previewPath: string | null + embedPath: string + previewSize: { + height: number + width: number + } +}) { + const { req, previewSize, previewPath, title, channel, embedPath } = options + + const webserverUrl = WEBSERVER.URL + const maxHeight = forceNumber(req.query.maxheight) + const maxWidth = forceNumber(req.query.maxwidth) + + const embedUrl = webserverUrl + embedPath + const embedTitle = escapeHTML(title) + + let thumbnailUrl = previewPath + ? webserverUrl + previewPath + : undefined + + let embedWidth = EMBED_SIZE.width + if (maxWidth < embedWidth) embedWidth = maxWidth + + let embedHeight = EMBED_SIZE.height + if (maxHeight < embedHeight) embedHeight = maxHeight + + // Our thumbnail is too big for the consumer + if ( + (maxHeight !== undefined && maxHeight < previewSize.height) || + (maxWidth !== undefined && maxWidth < previewSize.width) + ) { + thumbnailUrl = undefined + } + + const html = `` + + const json: any = { + type: 'video', + version: '1.0', + html, + width: embedWidth, + height: embedHeight, + title, + author_name: channel.name, + author_url: channel.Actor.url, + provider_name: 'PeerTube', + provider_url: webserverUrl + } + + if (thumbnailUrl !== undefined) { + json.thumbnail_url = thumbnailUrl + json.thumbnail_width = previewSize.width + json.thumbnail_height = previewSize.height + } + + return json +} + +function redirectToAccountUrl (req: express.Request, res: express.Response, next: express.NextFunction) { + return res.redirect(res.locals.account.Actor.url) +} diff --git a/server/controllers/shared/m3u8-playlist.ts b/server/server/controllers/shared/m3u8-playlist.ts similarity index 100% rename from server/controllers/shared/m3u8-playlist.ts rename to server/server/controllers/shared/m3u8-playlist.ts diff --git a/server/server/controllers/sitemap.ts b/server/server/controllers/sitemap.ts new file mode 100644 index 000000000..69aea6167 --- /dev/null +++ b/server/server/controllers/sitemap.ts @@ -0,0 +1,115 @@ +import express from 'express' +import truncate from 'lodash-es/truncate.js' +import { ErrorLevel, SitemapStream, streamToPromise } from 'sitemap' +import { logger } from '@server/helpers/logger.js' +import { getServerActor } from '@server/models/application/application.js' +import { buildNSFWFilter } from '../helpers/express-utils.js' +import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants.js' +import { apiRateLimiter, asyncMiddleware } from '../middlewares/index.js' +import { cacheRoute } from '../middlewares/cache/cache.js' +import { AccountModel } from '../models/account/account.js' +import { VideoModel } from '../models/video/video.js' +import { VideoChannelModel } from '../models/video/video-channel.js' + +const sitemapRouter = express.Router() + +sitemapRouter.use('/sitemap.xml', + apiRateLimiter, + cacheRoute(ROUTE_CACHE_LIFETIME.SITEMAP), + asyncMiddleware(getSitemap) +) + +// --------------------------------------------------------------------------- + +export { + sitemapRouter +} + +// --------------------------------------------------------------------------- + +async function getSitemap (req: express.Request, res: express.Response) { + let urls = getSitemapBasicUrls() + + urls = urls.concat(await getSitemapLocalVideoUrls()) + urls = urls.concat(await getSitemapVideoChannelUrls()) + urls = urls.concat(await getSitemapAccountUrls()) + + const sitemapStream = new SitemapStream({ + hostname: WEBSERVER.URL, + errorHandler: (err: Error, level: ErrorLevel) => { + if (level === 'warn') { + logger.warn('Warning in sitemap generation.', { err }) + } else if (level === 'throw') { + logger.error('Error in sitemap generation.', { err }) + + throw err + } + } + }) + + for (const urlObj of urls) { + sitemapStream.write(urlObj) + } + sitemapStream.end() + + const xml = await streamToPromise(sitemapStream) + + res.header('Content-Type', 'application/xml') + res.send(xml) +} + +async function getSitemapVideoChannelUrls () { + const rows = await VideoChannelModel.listLocalsForSitemap('createdAt') + + return rows.map(channel => ({ + url: WEBSERVER.URL + '/video-channels/' + channel.Actor.preferredUsername + })) +} + +async function getSitemapAccountUrls () { + const rows = await AccountModel.listLocalsForSitemap('createdAt') + + return rows.map(channel => ({ + url: WEBSERVER.URL + '/accounts/' + channel.Actor.preferredUsername + })) +} + +async function getSitemapLocalVideoUrls () { + const serverActor = await getServerActor() + + const { data } = await VideoModel.listForApi({ + start: 0, + count: undefined, + sort: 'createdAt', + displayOnlyForFollower: { + actorId: serverActor.id, + orLocalVideos: true + }, + isLocal: true, + nsfw: buildNSFWFilter(), + countVideos: false + }) + + return data.map(v => ({ + url: WEBSERVER.URL + v.getWatchStaticPath(), + video: [ + { + // Sitemap title should be < 100 characters + title: truncate(v.name, { length: 100, omission: '...' }), + // Sitemap description should be < 2000 characters + description: truncate(v.description || v.name, { length: 2000, omission: '...' }), + player_loc: WEBSERVER.URL + v.getEmbedStaticPath(), + thumbnail_loc: WEBSERVER.URL + v.getMiniatureStaticPath() + } + ] + })) +} + +function getSitemapBasicUrls () { + const paths = [ + '/about/instance', + '/videos/local' + ] + + return paths.map(p => ({ url: WEBSERVER.URL + p })) +} diff --git a/server/server/controllers/static.ts b/server/server/controllers/static.ts new file mode 100644 index 000000000..06807a9d5 --- /dev/null +++ b/server/server/controllers/static.ts @@ -0,0 +1,116 @@ +import cors from 'cors' +import express from 'express' +import { readFile } from 'fs/promises' +import { join } from 'path' +import { injectQueryToPlaylistUrls } from '@server/lib/hls.js' +import { + asyncMiddleware, + ensureCanAccessPrivateVideoHLSFiles, + ensureCanAccessVideoPrivateWebVideoFiles, + handleStaticError, + optionalAuthenticate +} from '@server/middlewares/index.js' +import { HttpStatusCode } from '@peertube/peertube-models' +import { CONFIG } from '../initializers/config.js' +import { DIRECTORIES, STATIC_MAX_AGE, STATIC_PATHS } from '../initializers/constants.js' +import { buildReinjectVideoFileTokenQuery, doReinjectVideoFileToken } from './shared/m3u8-playlist.js' + +const staticRouter = express.Router() + +// Cors is very important to let other servers access torrent and video files +staticRouter.use(cors()) + +// --------------------------------------------------------------------------- +// Web videos/Classic videos +// --------------------------------------------------------------------------- + +const privateWebVideoStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUIRE_AUTH === true + ? [ optionalAuthenticate, asyncMiddleware(ensureCanAccessVideoPrivateWebVideoFiles) ] + : [] + +staticRouter.use( + [ STATIC_PATHS.PRIVATE_WEB_VIDEOS, STATIC_PATHS.LEGACY_PRIVATE_WEB_VIDEOS ], + ...privateWebVideoStaticMiddlewares, + express.static(DIRECTORIES.VIDEOS.PRIVATE, { fallthrough: false }), + handleStaticError +) +staticRouter.use( + [ STATIC_PATHS.WEB_VIDEOS, STATIC_PATHS.LEGACY_WEB_VIDEOS ], + express.static(DIRECTORIES.VIDEOS.PUBLIC, { fallthrough: false }), + handleStaticError +) + +staticRouter.use( + STATIC_PATHS.REDUNDANCY, + express.static(CONFIG.STORAGE.REDUNDANCY_DIR, { fallthrough: false }), + handleStaticError +) + +// --------------------------------------------------------------------------- +// HLS +// --------------------------------------------------------------------------- + +const privateHLSStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUIRE_AUTH === true + ? [ optionalAuthenticate, asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles) ] + : [] + +staticRouter.use( + STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:playlistName.m3u8', + ...privateHLSStaticMiddlewares, + asyncMiddleware(servePrivateM3U8) +) + +staticRouter.use( + STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, + ...privateHLSStaticMiddlewares, + express.static(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, { fallthrough: false }), + handleStaticError +) +staticRouter.use( + STATIC_PATHS.STREAMING_PLAYLISTS.HLS, + express.static(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, { fallthrough: false }), + handleStaticError +) + +// FIXME: deprecated in v6, to remove +const thumbnailsPhysicalPath = CONFIG.STORAGE.THUMBNAILS_DIR +staticRouter.use( + STATIC_PATHS.THUMBNAILS, + express.static(thumbnailsPhysicalPath, { maxAge: STATIC_MAX_AGE.SERVER, fallthrough: false }), + handleStaticError +) + +// --------------------------------------------------------------------------- + +export { + staticRouter +} + +// --------------------------------------------------------------------------- + +async function servePrivateM3U8 (req: express.Request, res: express.Response) { + const path = join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, req.params.videoUUID, req.params.playlistName + '.m3u8') + const filename = req.params.playlistName + '.m3u8' + + let playlistContent: string + + try { + playlistContent = await readFile(path, 'utf-8') + } catch (err) { + if (err.message.includes('ENOENT')) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'File not found' + }) + } + + throw err + } + + // Inject token in playlist so players that cannot alter the HTTP request can still watch the video + const transformedContent = doReinjectVideoFileToken(req) + ? injectQueryToPlaylistUrls(playlistContent, buildReinjectVideoFileTokenQuery(req, filename.endsWith('master.m3u8'))) + : playlistContent + + return res.set('content-type', 'application/vnd.apple.mpegurl').send(transformedContent).end() +} diff --git a/server/server/controllers/tracker.ts b/server/server/controllers/tracker.ts new file mode 100644 index 000000000..e8d29f976 --- /dev/null +++ b/server/server/controllers/tracker.ts @@ -0,0 +1,148 @@ +import { Server as TrackerServer } from 'bittorrent-tracker' +import express from 'express' +import { createServer } from 'http' +import { LRUCache } from 'lru-cache' +import proxyAddr from 'proxy-addr' +import { WebSocketServer } from 'ws' +import { logger } from '../helpers/logger.js' +import { CONFIG } from '../initializers/config.js' +import { LRU_CACHE, TRACKER_RATE_LIMITS } from '../initializers/constants.js' +import { VideoFileModel } from '../models/video/video-file.js' +import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist.js' + +const trackerRouter = express.Router() + +const blockedIPs = new LRUCache({ + max: LRU_CACHE.TRACKER_IPS.MAX_SIZE, + ttl: TRACKER_RATE_LIMITS.BLOCK_IP_LIFETIME +}) + +let peersIps = {} +let peersIpInfoHash = {} +runPeersChecker() + +const trackerServer = new TrackerServer({ + http: false, + udp: false, + ws: false, + filter: async function (infoHash, params, cb) { + if (CONFIG.TRACKER.ENABLED === false) { + return cb(new Error('Tracker is disabled on this instance.')) + } + + let ip: string + + if (params.type === 'ws') { + ip = params.ip + } else { + ip = params.httpReq.ip + } + + const key = ip + '-' + infoHash + + peersIps[ip] = peersIps[ip] ? peersIps[ip] + 1 : 1 + peersIpInfoHash[key] = peersIpInfoHash[key] ? peersIpInfoHash[key] + 1 : 1 + + if (CONFIG.TRACKER.REJECT_TOO_MANY_ANNOUNCES && peersIpInfoHash[key] > TRACKER_RATE_LIMITS.ANNOUNCES_PER_IP_PER_INFOHASH) { + return cb(new Error(`Too many requests (${peersIpInfoHash[key]} of ip ${ip} for torrent ${infoHash}`)) + } + + try { + if (CONFIG.TRACKER.PRIVATE === false) return cb() + + const videoFileExists = await VideoFileModel.doesInfohashExistCached(infoHash) + if (videoFileExists === true) return cb() + + const playlistExists = await VideoStreamingPlaylistModel.doesInfohashExistCached(infoHash) + if (playlistExists === true) return cb() + + cb(new Error(`Unknown infoHash ${infoHash} requested by ip ${ip}`)) + + // Close socket connection and block IP for a few time + if (params.type === 'ws') { + blockedIPs.set(ip, true) + + // setTimeout to wait filter response + setTimeout(() => params.socket.close(), 0) + } + } catch (err) { + logger.error('Error in tracker filter.', { err }) + return cb(err) + } + } +}) + +if (CONFIG.TRACKER.ENABLED !== false) { + trackerServer.on('error', function (err) { + logger.error('Error in tracker.', { err }) + }) + + trackerServer.on('warning', function (err) { + const message = err.message || '' + + if (CONFIG.LOG.LOG_TRACKER_UNKNOWN_INFOHASH === false && message.includes('Unknown infoHash')) { + return + } + + logger.warn('Warning in tracker.', { err }) + }) +} + +const onHttpRequest = trackerServer.onHttpRequest.bind(trackerServer) +trackerRouter.get('/tracker/announce', (req, res) => onHttpRequest(req, res, { action: 'announce' })) +trackerRouter.get('/tracker/scrape', (req, res) => onHttpRequest(req, res, { action: 'scrape' })) + +function createWebsocketTrackerServer (app: express.Application) { + const server = createServer(app) + const wss = new WebSocketServer({ noServer: true }) + + wss.on('connection', function (ws, req) { + ws['ip'] = proxyAddr(req, CONFIG.TRUST_PROXY) + + trackerServer.onWebSocketConnection(ws) + }) + + server.on('upgrade', (request: express.Request, socket, head) => { + if (request.url === '/tracker/socket') { + const ip = proxyAddr(request, CONFIG.TRUST_PROXY) + + if (blockedIPs.has(ip)) { + logger.debug('Blocking IP %s from tracker.', ip) + + socket.write('HTTP/1.1 403 Forbidden\r\n\r\n') + socket.destroy() + return + } + + return wss.handleUpgrade(request, socket, head, ws => wss.emit('connection', ws, request)) + } + + // Don't destroy socket, we have Socket.IO too + }) + + return { server, trackerServer } +} + +// --------------------------------------------------------------------------- + +export { + trackerRouter, + createWebsocketTrackerServer +} + +// --------------------------------------------------------------------------- + +function runPeersChecker () { + setInterval(() => { + logger.debug('Checking peers.') + + for (const ip of Object.keys(peersIpInfoHash)) { + if (peersIps[ip] > TRACKER_RATE_LIMITS.ANNOUNCES_PER_IP) { + logger.warn('Peer %s made abnormal requests (%d).', ip, peersIps[ip]) + } + } + + peersIpInfoHash = {} + peersIps = {} + }, TRACKER_RATE_LIMITS.INTERVAL) +} diff --git a/server/server/controllers/well-known.ts b/server/server/controllers/well-known.ts new file mode 100644 index 000000000..b6bcd79ef --- /dev/null +++ b/server/server/controllers/well-known.ts @@ -0,0 +1,125 @@ +import cors from 'cors' +import express from 'express' +import { join } from 'path' +import { asyncMiddleware, buildRateLimiter, handleStaticError, webfingerValidator } from '@server/middlewares/index.js' +import { root } from '@peertube/peertube-node-utils' +import { CONFIG } from '../initializers/config.js' +import { ROUTE_CACHE_LIFETIME, WEBSERVER } from '../initializers/constants.js' +import { cacheRoute } from '../middlewares/cache/cache.js' + +const wellKnownRouter = express.Router() + +const wellKnownRateLimiter = buildRateLimiter({ + windowMs: CONFIG.RATES_LIMIT.WELL_KNOWN.WINDOW_MS, + max: CONFIG.RATES_LIMIT.WELL_KNOWN.MAX +}) + +wellKnownRouter.use(cors()) + +wellKnownRouter.get('/.well-known/webfinger', + wellKnownRateLimiter, + asyncMiddleware(webfingerValidator), + webfingerController +) + +wellKnownRouter.get('/.well-known/security.txt', + wellKnownRateLimiter, + cacheRoute(ROUTE_CACHE_LIFETIME.SECURITYTXT), + (_, res: express.Response) => { + res.type('text/plain') + return res.send(CONFIG.INSTANCE.SECURITYTXT + CONFIG.INSTANCE.SECURITYTXT_CONTACT) + } +) + +// nodeinfo service +wellKnownRouter.use('/.well-known/nodeinfo', + wellKnownRateLimiter, + cacheRoute(ROUTE_CACHE_LIFETIME.NODEINFO), + (_, res: express.Response) => { + return res.json({ + links: [ + { + rel: 'http://nodeinfo.diaspora.software/ns/schema/2.0', + href: WEBSERVER.URL + '/nodeinfo/2.0.json' + } + ] + }) + } +) + +// dnt-policy.txt service (see https://www.eff.org/dnt-policy) +wellKnownRouter.use('/.well-known/dnt-policy.txt', + wellKnownRateLimiter, + cacheRoute(ROUTE_CACHE_LIFETIME.DNT_POLICY), + (_, res: express.Response) => { + res.type('text/plain') + + return res.sendFile(join(root(), 'dist/server/static/dnt-policy/dnt-policy-1.0.txt')) + } +) + +// dnt service (see https://www.w3.org/TR/tracking-dnt/#status-resource) +wellKnownRouter.use('/.well-known/dnt/', + wellKnownRateLimiter, + (_, res: express.Response) => { + res.json({ tracking: 'N' }) + } +) + +wellKnownRouter.use('/.well-known/change-password', + wellKnownRateLimiter, + (_, res: express.Response) => { + res.redirect('/my-account/settings') + } +) + +wellKnownRouter.use('/.well-known/host-meta', + wellKnownRateLimiter, + (_, res: express.Response) => { + res.type('application/xml') + + const xml = '\n' + + '\n' + + ` \n` + + '' + + res.send(xml).end() + } +) + +wellKnownRouter.use('/.well-known/', + wellKnownRateLimiter, + cacheRoute(ROUTE_CACHE_LIFETIME.WELL_KNOWN), + express.static(CONFIG.STORAGE.WELL_KNOWN_DIR, { fallthrough: false }), + handleStaticError +) + +// --------------------------------------------------------------------------- + +export { + wellKnownRouter +} + +// --------------------------------------------------------------------------- + +function webfingerController (req: express.Request, res: express.Response) { + const actor = res.locals.actorUrl + + const json = { + subject: req.query.resource, + aliases: [ actor.url ], + links: [ + { + rel: 'self', + type: 'application/activity+json', + href: actor.url + }, + { + rel: 'http://ostatus.org/schema/1.0/subscribe', + template: WEBSERVER.URL + '/remote-interaction?uri={uri}' + } + ] + } + + return res.json(json) +} diff --git a/server/server/helpers/activity-pub-utils.ts b/server/server/helpers/activity-pub-utils.ts new file mode 100644 index 000000000..acc5c304b --- /dev/null +++ b/server/server/helpers/activity-pub-utils.ts @@ -0,0 +1,230 @@ +import { ContextType } from '@peertube/peertube-models' +import { ACTIVITY_PUB } from '@server/initializers/constants.js' +import { buildDigest, signJsonLDObject } from './peertube-crypto.js' + +type ContextFilter = (arg: T) => Promise + +export function buildGlobalHTTPHeaders (body: any) { + return { + 'digest': buildDigest(body), + 'content-type': 'application/activity+json', + 'accept': ACTIVITY_PUB.ACCEPT_HEADER + } +} + +export async function activityPubContextify (data: T, type: ContextType, contextFilter: ContextFilter) { + return { ...await getContextData(type, contextFilter), ...data } +} + +export async function signAndContextify ( + byActor: { url: string, privateKey: string }, + data: T, + contextType: ContextType | null, + contextFilter: ContextFilter +) { + const activity = contextType + ? await activityPubContextify(data, contextType, contextFilter) + : data + + return signJsonLDObject(byActor, activity) +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +type ContextValue = { [ id: string ]: (string | { '@type': string, '@id': string }) } + +const contextStore: { [ id in ContextType ]: (string | { [ id: string ]: string })[] } = { + Video: buildContext({ + Hashtag: 'as:Hashtag', + uuid: 'sc:identifier', + category: 'sc:category', + licence: 'sc:license', + subtitleLanguage: 'sc:subtitleLanguage', + sensitive: 'as:sensitive', + language: 'sc:inLanguage', + identifier: 'sc:identifier', + + isLiveBroadcast: 'sc:isLiveBroadcast', + liveSaveReplay: { + '@type': 'sc:Boolean', + '@id': 'pt:liveSaveReplay' + }, + permanentLive: { + '@type': 'sc:Boolean', + '@id': 'pt:permanentLive' + }, + latencyMode: { + '@type': 'sc:Number', + '@id': 'pt:latencyMode' + }, + + Infohash: 'pt:Infohash', + + tileWidth: { + '@type': 'sc:Number', + '@id': 'pt:tileWidth' + }, + tileHeight: { + '@type': 'sc:Number', + '@id': 'pt:tileHeight' + }, + tileDuration: { + '@type': 'sc:Number', + '@id': 'pt:tileDuration' + }, + + originallyPublishedAt: 'sc:datePublished', + + uploadDate: 'sc:uploadDate', + + views: { + '@type': 'sc:Number', + '@id': 'pt:views' + }, + state: { + '@type': 'sc:Number', + '@id': 'pt:state' + }, + size: { + '@type': 'sc:Number', + '@id': 'pt:size' + }, + fps: { + '@type': 'sc:Number', + '@id': 'pt:fps' + }, + commentsEnabled: { + '@type': 'sc:Boolean', + '@id': 'pt:commentsEnabled' + }, + downloadEnabled: { + '@type': 'sc:Boolean', + '@id': 'pt:downloadEnabled' + }, + waitTranscoding: { + '@type': 'sc:Boolean', + '@id': 'pt:waitTranscoding' + }, + support: { + '@type': 'sc:Text', + '@id': 'pt:support' + }, + likes: { + '@id': 'as:likes', + '@type': '@id' + }, + dislikes: { + '@id': 'as:dislikes', + '@type': '@id' + }, + shares: { + '@id': 'as:shares', + '@type': '@id' + }, + comments: { + '@id': 'as:comments', + '@type': '@id' + } + }), + + Playlist: buildContext({ + Playlist: 'pt:Playlist', + PlaylistElement: 'pt:PlaylistElement', + position: { + '@type': 'sc:Number', + '@id': 'pt:position' + }, + startTimestamp: { + '@type': 'sc:Number', + '@id': 'pt:startTimestamp' + }, + stopTimestamp: { + '@type': 'sc:Number', + '@id': 'pt:stopTimestamp' + }, + uuid: 'sc:identifier' + }), + + CacheFile: buildContext({ + expires: 'sc:expires', + CacheFile: 'pt:CacheFile' + }), + + Flag: buildContext({ + Hashtag: 'as:Hashtag' + }), + + Actor: buildContext({ + playlists: { + '@id': 'pt:playlists', + '@type': '@id' + }, + support: { + '@type': 'sc:Text', + '@id': 'pt:support' + }, + + // TODO: remove in a few versions, introduced in 4.2 + icons: 'as:icon' + }), + + WatchAction: buildContext({ + WatchAction: 'sc:WatchAction', + startTimestamp: { + '@type': 'sc:Number', + '@id': 'pt:startTimestamp' + }, + stopTimestamp: { + '@type': 'sc:Number', + '@id': 'pt:stopTimestamp' + }, + watchSection: { + '@type': 'sc:Number', + '@id': 'pt:stopTimestamp' + }, + uuid: 'sc:identifier' + }), + + Collection: buildContext(), + Follow: buildContext(), + Reject: buildContext(), + Accept: buildContext(), + View: buildContext(), + Announce: buildContext(), + Comment: buildContext(), + Delete: buildContext(), + Rate: buildContext() +} + +async function getContextData (type: ContextType, contextFilter: ContextFilter) { + const contextData = contextFilter + ? await contextFilter(contextStore[type]) + : contextStore[type] + + return { '@context': contextData } +} + +function buildContext (contextValue?: ContextValue) { + const baseContext = [ + 'https://www.w3.org/ns/activitystreams', + 'https://w3id.org/security/v1', + { + RsaSignature2017: 'https://w3id.org/security#RsaSignature2017' + } + ] + + if (!contextValue) return baseContext + + return [ + ...baseContext, + + { + pt: 'https://joinpeertube.org/ns#', + sc: 'http://schema.org/', + + ...contextValue + } + ] +} diff --git a/server/server/helpers/actors.ts b/server/server/helpers/actors.ts new file mode 100644 index 000000000..5be5c9ee7 --- /dev/null +++ b/server/server/helpers/actors.ts @@ -0,0 +1,17 @@ +import { WEBSERVER } from '@server/initializers/constants.js' + +function handleToNameAndHost (handle: string) { + let [ name, host ] = handle.split('@') + if (host === WEBSERVER.HOST) host = null + + return { name, host, handle } +} + +function handlesToNameAndHost (handles: string[]) { + return handles.map(h => handleToNameAndHost(h)) +} + +export { + handleToNameAndHost, + handlesToNameAndHost +} diff --git a/server/server/helpers/audit-logger.ts b/server/server/helpers/audit-logger.ts new file mode 100644 index 000000000..bf79ced01 --- /dev/null +++ b/server/server/helpers/audit-logger.ts @@ -0,0 +1,298 @@ +import { + AdminAbuse, + CustomConfig, + User, + VideoChannel, + VideoChannelSync, + VideoComment, + VideoDetails, + VideoImport +} from '@peertube/peertube-models' +import { AUDIT_LOG_FILENAME } from '@server/initializers/constants.js' +import { diff } from 'deep-object-diff' +import express from 'express' +import flatten from 'flat' +import { join } from 'path' +import { addColors, config, createLogger, format, transports } from 'winston' +import { CONFIG } from '../initializers/config.js' +import { jsonLoggerFormat, labelFormatter } from './logger.js' + +function getAuditIdFromRes (res: express.Response) { + return res.locals.oauth.token.User.username +} + +enum AUDIT_TYPE { + CREATE = 'create', + UPDATE = 'update', + DELETE = 'delete' +} + +const colors = config.npm.colors +colors.audit = config.npm.colors.info + +addColors(colors) + +const auditLogger = createLogger({ + levels: { audit: 0 }, + transports: [ + new transports.File({ + filename: join(CONFIG.STORAGE.LOG_DIR, AUDIT_LOG_FILENAME), + level: 'audit', + maxsize: 5242880, + maxFiles: 5, + format: format.combine( + format.timestamp(), + labelFormatter(), + format.splat(), + jsonLoggerFormat + ) + }) + ], + exitOnError: true +}) + +function auditLoggerWrapper (domain: string, user: string, action: AUDIT_TYPE, entity: EntityAuditView, oldEntity: EntityAuditView = null) { + let entityInfos: object + + if (action === AUDIT_TYPE.UPDATE && oldEntity) { + const oldEntityKeys = oldEntity.toLogKeys() + const diffObject = diff(oldEntityKeys, entity.toLogKeys()) + const diffKeys = Object.entries(diffObject).reduce((newKeys, entry) => { + newKeys[`new-${entry[0]}`] = entry[1] + return newKeys + }, {}) + entityInfos = { ...oldEntityKeys, ...diffKeys } + } else { + entityInfos = { ...entity.toLogKeys() } + } + + auditLogger.log('audit', JSON.stringify({ + user, + domain, + action, + ...entityInfos + })) +} + +function auditLoggerFactory (domain: string) { + return { + create (user: string, entity: EntityAuditView) { + auditLoggerWrapper(domain, user, AUDIT_TYPE.CREATE, entity) + }, + update (user: string, entity: EntityAuditView, oldEntity: EntityAuditView) { + auditLoggerWrapper(domain, user, AUDIT_TYPE.UPDATE, entity, oldEntity) + }, + delete (user: string, entity: EntityAuditView) { + auditLoggerWrapper(domain, user, AUDIT_TYPE.DELETE, entity) + } + } +} + +abstract class EntityAuditView { + constructor (private readonly keysToKeep: Set, private readonly prefix: string, private readonly entityInfos: object) { } + + toLogKeys (): object { + const obj = flatten(this.entityInfos, { delimiter: '-', safe: true }) + + return Object.keys(obj) + .filter(key => this.keysToKeep.has(key)) + .reduce((p, k) => ({ ...p, [`${this.prefix}-${k}`]: obj[k] }), {}) + } +} + +const videoKeysToKeep = new Set([ + 'tags', + 'uuid', + 'id', + 'uuid', + 'createdAt', + 'updatedAt', + 'publishedAt', + 'category', + 'licence', + 'language', + 'privacy', + 'description', + 'duration', + 'isLocal', + 'name', + 'thumbnailPath', + 'previewPath', + 'nsfw', + 'waitTranscoding', + 'account-id', + 'account-uuid', + 'account-name', + 'channel-id', + 'channel-uuid', + 'channel-name', + 'support', + 'commentsEnabled', + 'downloadEnabled' +]) +class VideoAuditView extends EntityAuditView { + constructor (video: VideoDetails) { + super(videoKeysToKeep, 'video', video) + } +} + +const videoImportKeysToKeep = new Set([ + 'id', + 'targetUrl', + 'video-name' +]) +class VideoImportAuditView extends EntityAuditView { + constructor (videoImport: VideoImport) { + super(videoImportKeysToKeep, 'video-import', videoImport) + } +} + +const commentKeysToKeep = new Set([ + 'id', + 'text', + 'threadId', + 'inReplyToCommentId', + 'videoId', + 'createdAt', + 'updatedAt', + 'totalReplies', + 'account-id', + 'account-uuid', + 'account-name' +]) +class CommentAuditView extends EntityAuditView { + constructor (comment: VideoComment) { + super(commentKeysToKeep, 'comment', comment) + } +} + +const userKeysToKeep = new Set([ + 'id', + 'username', + 'email', + 'nsfwPolicy', + 'autoPlayVideo', + 'role', + 'videoQuota', + 'createdAt', + 'account-id', + 'account-uuid', + 'account-name', + 'account-followingCount', + 'account-followersCount', + 'account-createdAt', + 'account-updatedAt', + 'account-avatar-path', + 'account-avatar-createdAt', + 'account-avatar-updatedAt', + 'account-displayName', + 'account-description', + 'videoChannels' +]) +class UserAuditView extends EntityAuditView { + constructor (user: User) { + super(userKeysToKeep, 'user', user) + } +} + +const channelKeysToKeep = new Set([ + 'id', + 'uuid', + 'name', + 'followingCount', + 'followersCount', + 'createdAt', + 'updatedAt', + 'avatar-path', + 'avatar-createdAt', + 'avatar-updatedAt', + 'displayName', + 'description', + 'support', + 'isLocal', + 'ownerAccount-id', + 'ownerAccount-uuid', + 'ownerAccount-name', + 'ownerAccount-displayedName' +]) +class VideoChannelAuditView extends EntityAuditView { + constructor (channel: VideoChannel) { + super(channelKeysToKeep, 'channel', channel) + } +} + +const abuseKeysToKeep = new Set([ + 'id', + 'reason', + 'reporterAccount', + 'createdAt' +]) +class AbuseAuditView extends EntityAuditView { + constructor (abuse: AdminAbuse) { + super(abuseKeysToKeep, 'abuse', abuse) + } +} + +const customConfigKeysToKeep = new Set([ + 'instance-name', + 'instance-shortDescription', + 'instance-description', + 'instance-terms', + 'instance-defaultClientRoute', + 'instance-defaultNSFWPolicy', + 'instance-customizations-javascript', + 'instance-customizations-css', + 'services-twitter-username', + 'services-twitter-whitelisted', + 'cache-previews-size', + 'cache-captions-size', + 'signup-enabled', + 'signup-limit', + 'signup-requiresEmailVerification', + 'admin-email', + 'user-videoQuota', + 'transcoding-enabled', + 'transcoding-threads', + 'transcoding-resolutions' +]) +class CustomConfigAuditView extends EntityAuditView { + constructor (customConfig: CustomConfig) { + const infos: any = customConfig + const resolutionsDict = infos.transcoding.resolutions + const resolutionsArray = [] + + Object.entries(resolutionsDict) + .forEach(([ resolution, isEnabled ]) => { + if (isEnabled) resolutionsArray.push(resolution) + }) + + Object.assign({}, infos, { transcoding: { resolutions: resolutionsArray } }) + super(customConfigKeysToKeep, 'config', infos) + } +} + +const channelSyncKeysToKeep = new Set([ + 'id', + 'externalChannelUrl', + 'channel-id', + 'channel-name' +]) +class VideoChannelSyncAuditView extends EntityAuditView { + constructor (channelSync: VideoChannelSync) { + super(channelSyncKeysToKeep, 'channelSync', channelSync) + } +} + +export { + getAuditIdFromRes, + + auditLoggerFactory, + VideoImportAuditView, + VideoChannelAuditView, + CommentAuditView, + UserAuditView, + VideoAuditView, + AbuseAuditView, + CustomConfigAuditView, + VideoChannelSyncAuditView +} diff --git a/server/server/helpers/captions-utils.ts b/server/server/helpers/captions-utils.ts new file mode 100644 index 000000000..f165cb447 --- /dev/null +++ b/server/server/helpers/captions-utils.ts @@ -0,0 +1,55 @@ +import { createReadStream, createWriteStream } from 'fs' +import { move, remove } from 'fs-extra/esm' +import { join } from 'path' +import { Transform } from 'stream' +import { MVideoCaption } from '@server/types/models/index.js' +import { CONFIG } from '../initializers/config.js' +import { pipelinePromise } from './core-utils.js' + +async function moveAndProcessCaptionFile (physicalFile: { filename: string, path: string }, videoCaption: MVideoCaption) { + const videoCaptionsDir = CONFIG.STORAGE.CAPTIONS_DIR + const destination = join(videoCaptionsDir, videoCaption.filename) + + // Convert this srt file to vtt + if (physicalFile.path.endsWith('.srt')) { + await convertSrtToVtt(physicalFile.path, destination) + await remove(physicalFile.path) + } else if (physicalFile.path !== destination) { // Just move the vtt file + await move(physicalFile.path, destination, { overwrite: true }) + } + + // This is important in case if there is another attempt in the retry process + physicalFile.filename = videoCaption.filename + physicalFile.path = destination +} + +// --------------------------------------------------------------------------- + +export { + moveAndProcessCaptionFile +} + +// --------------------------------------------------------------------------- + +async function convertSrtToVtt (source: string, destination: string) { + const fixVTT = new Transform({ + transform: (chunk, _encoding, cb) => { + let block: string = chunk.toString() + + block = block.replace(/(\d\d:\d\d:\d\d)(\s)/g, '$1.000$2') + .replace(/(\d\d:\d\d:\d\d),(\d)(\s)/g, '$1.00$2$3') + .replace(/(\d\d:\d\d:\d\d),(\d\d)(\s)/g, '$1.0$2$3') + + return cb(undefined, block) + } + }) + + const srt2vtt = await import('srt-to-vtt') + + return pipelinePromise( + createReadStream(source), + srt2vtt.default(), + fixVTT, + createWriteStream(destination) + ) +} diff --git a/server/server/helpers/core-utils.ts b/server/server/helpers/core-utils.ts new file mode 100644 index 000000000..6dc09d317 --- /dev/null +++ b/server/server/helpers/core-utils.ts @@ -0,0 +1,288 @@ +/* eslint-disable no-useless-call */ + +/* + Different from 'utils' because we don't import other PeerTube modules. + Useful to avoid circular dependencies. +*/ + +import { promisify1, promisify2, promisify3 } from '@peertube/peertube-core-utils' +import { exec, ExecOptions } from 'child_process' +import { ED25519KeyPairOptions, generateKeyPair, randomBytes, RSAKeyPairOptions, scrypt } from 'crypto' +import truncate from 'lodash-es/truncate.js' +import { pipeline } from 'stream' +import { URL } from 'url' +import { promisify } from 'util' + +const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => { + if (!oldObject || typeof oldObject !== 'object') { + return valueConverter(oldObject) + } + + if (Array.isArray(oldObject)) { + return oldObject.map(e => objectConverter(e, keyConverter, valueConverter)) + } + + const newObject = {} + Object.keys(oldObject).forEach(oldKey => { + const newKey = keyConverter(oldKey) + newObject[newKey] = objectConverter(oldObject[oldKey], keyConverter, valueConverter) + }) + + return newObject +} + +function mapToJSON (map: Map) { + const obj: any = {} + + for (const [ k, v ] of map) { + obj[k] = v + } + + return obj +} + +// --------------------------------------------------------------------------- + +const timeTable = { + ms: 1, + second: 1000, + minute: 60000, + hour: 3600000, + day: 3600000 * 24, + week: 3600000 * 24 * 7, + month: 3600000 * 24 * 30 +} + +export function parseDurationToMs (duration: number | string): number { + if (duration === null) return null + if (typeof duration === 'number') return duration + if (!isNaN(+duration)) return +duration + + if (typeof duration === 'string') { + const split = duration.match(/^([\d.,]+)\s?(\w+)$/) + + if (split.length === 3) { + const len = parseFloat(split[1]) + let unit = split[2].replace(/s$/i, '').toLowerCase() + if (unit === 'm') { + unit = 'ms' + } + + return (len || 1) * (timeTable[unit] || 0) + } + } + + throw new Error(`Duration ${duration} could not be properly parsed`) +} + +export function parseBytes (value: string | number): number { + if (typeof value === 'number') return value + if (!isNaN(+value)) return +value + + const tgm = /^(\d+)\s*TB\s*(\d+)\s*GB\s*(\d+)\s*MB$/ + const tg = /^(\d+)\s*TB\s*(\d+)\s*GB$/ + const tm = /^(\d+)\s*TB\s*(\d+)\s*MB$/ + const gm = /^(\d+)\s*GB\s*(\d+)\s*MB$/ + const t = /^(\d+)\s*TB$/ + const g = /^(\d+)\s*GB$/ + const m = /^(\d+)\s*MB$/ + const b = /^(\d+)\s*B$/ + + let match: RegExpMatchArray + + if (value.match(tgm)) { + match = value.match(tgm) + return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 + + parseInt(match[2], 10) * 1024 * 1024 * 1024 + + parseInt(match[3], 10) * 1024 * 1024 + } + + if (value.match(tg)) { + match = value.match(tg) + return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 + + parseInt(match[2], 10) * 1024 * 1024 * 1024 + } + + if (value.match(tm)) { + match = value.match(tm) + return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 + + parseInt(match[2], 10) * 1024 * 1024 + } + + if (value.match(gm)) { + match = value.match(gm) + return parseInt(match[1], 10) * 1024 * 1024 * 1024 + + parseInt(match[2], 10) * 1024 * 1024 + } + + if (value.match(t)) { + match = value.match(t) + return parseInt(match[1], 10) * 1024 * 1024 * 1024 * 1024 + } + + if (value.match(g)) { + match = value.match(g) + return parseInt(match[1], 10) * 1024 * 1024 * 1024 + } + + if (value.match(m)) { + match = value.match(m) + return parseInt(match[1], 10) * 1024 * 1024 + } + + if (value.match(b)) { + match = value.match(b) + return parseInt(match[1], 10) * 1024 + } + + return parseInt(value, 10) +} + +// --------------------------------------------------------------------------- + +function sanitizeUrl (url: string) { + const urlObject = new URL(url) + + if (urlObject.protocol === 'https:' && urlObject.port === '443') { + urlObject.port = '' + } else if (urlObject.protocol === 'http:' && urlObject.port === '80') { + urlObject.port = '' + } + + return urlObject.href.replace(/\/$/, '') +} + +// Don't import remote scheme from constants because we are in core utils +function sanitizeHost (host: string, remoteScheme: string) { + const toRemove = remoteScheme === 'https' ? 443 : 80 + + return host.replace(new RegExp(`:${toRemove}$`), '') +} + +// --------------------------------------------------------------------------- + +// Consistent with .length, lodash truncate function is not +function peertubeTruncate (str: string, options: { length: number, separator?: RegExp, omission?: string }) { + const truncatedStr = truncate(str, options) + + // The truncated string is okay, we can return it + if (truncatedStr.length <= options.length) return truncatedStr + + // Lodash takes into account all UTF characters, whereas String.prototype.length does not: some characters have a length of 2 + // We always use the .length so we need to truncate more if needed + options.length -= truncatedStr.length - options.length + return truncate(str, options) +} + +function pageToStartAndCount (page: number, itemsPerPage: number) { + const start = (page - 1) * itemsPerPage + + return { start, count: itemsPerPage } +} + +// --------------------------------------------------------------------------- + +type SemVersion = { major: number, minor: number, patch: number } +function parseSemVersion (s: string) { + const parsed = s.match(/^v?(\d+)\.(\d+)\.(\d+)$/i) + + return { + major: parseInt(parsed[1]), + minor: parseInt(parsed[2]), + patch: parseInt(parsed[3]) + } as SemVersion +} + +// --------------------------------------------------------------------------- + +function execShell (command: string, options?: ExecOptions) { + return new Promise<{ err?: Error, stdout: string, stderr: string }>((res, rej) => { + exec(command, options, (err, stdout, stderr) => { + // eslint-disable-next-line prefer-promise-reject-errors + if (err) return rej({ err, stdout, stderr }) + + return res({ stdout, stderr }) + }) + }) +} + +// --------------------------------------------------------------------------- + +function generateRSAKeyPairPromise (size: number) { + return new Promise<{ publicKey: string, privateKey: string }>((res, rej) => { + const options: RSAKeyPairOptions<'pem', 'pem'> = { + modulusLength: size, + publicKeyEncoding: { + type: 'spki', + format: 'pem' + }, + privateKeyEncoding: { + type: 'pkcs1', + format: 'pem' + } + } + + generateKeyPair('rsa', options, (err, publicKey, privateKey) => { + if (err) return rej(err) + + return res({ publicKey, privateKey }) + }) + }) +} + +function generateED25519KeyPairPromise () { + return new Promise<{ publicKey: string, privateKey: string }>((res, rej) => { + const options: ED25519KeyPairOptions<'pem', 'pem'> = { + publicKeyEncoding: { + type: 'spki', + format: 'pem' + }, + privateKeyEncoding: { + type: 'pkcs8', + format: 'pem' + } + } + + generateKeyPair('ed25519', options, (err, publicKey, privateKey) => { + if (err) return rej(err) + + return res({ publicKey, privateKey }) + }) + }) +} + +// --------------------------------------------------------------------------- + +const randomBytesPromise = promisify1(randomBytes) +const scryptPromise = promisify3(scrypt) +const execPromise2 = promisify2(exec) +const execPromise = promisify1(exec) +const pipelinePromise = promisify(pipeline) + +// --------------------------------------------------------------------------- + +export { + objectConverter, + mapToJSON, + + sanitizeUrl, + sanitizeHost, + + execShell, + + pageToStartAndCount, + peertubeTruncate, + + scryptPromise, + + randomBytesPromise, + + generateRSAKeyPairPromise, + generateED25519KeyPairPromise, + + execPromise2, + execPromise, + pipelinePromise, + + parseSemVersion +} diff --git a/server/server/helpers/custom-jsonld-signature.ts b/server/server/helpers/custom-jsonld-signature.ts new file mode 100644 index 000000000..0bbe682da --- /dev/null +++ b/server/server/helpers/custom-jsonld-signature.ts @@ -0,0 +1,90 @@ +import AsyncLRU from 'async-lru' +import jsonld from 'jsonld' +import { logger } from './logger.js' + +const CACHE = { + 'https://w3id.org/security/v1': { + '@context': { + id: '@id', + type: '@type', + + dc: 'http://purl.org/dc/terms/', + sec: 'https://w3id.org/security#', + xsd: 'http://www.w3.org/2001/XMLSchema#', + + EcdsaKoblitzSignature2016: 'sec:EcdsaKoblitzSignature2016', + Ed25519Signature2018: 'sec:Ed25519Signature2018', + EncryptedMessage: 'sec:EncryptedMessage', + GraphSignature2012: 'sec:GraphSignature2012', + LinkedDataSignature2015: 'sec:LinkedDataSignature2015', + LinkedDataSignature2016: 'sec:LinkedDataSignature2016', + CryptographicKey: 'sec:Key', + + authenticationTag: 'sec:authenticationTag', + canonicalizationAlgorithm: 'sec:canonicalizationAlgorithm', + cipherAlgorithm: 'sec:cipherAlgorithm', + cipherData: 'sec:cipherData', + cipherKey: 'sec:cipherKey', + created: { '@id': 'dc:created', '@type': 'xsd:dateTime' }, + creator: { '@id': 'dc:creator', '@type': '@id' }, + digestAlgorithm: 'sec:digestAlgorithm', + digestValue: 'sec:digestValue', + domain: 'sec:domain', + encryptionKey: 'sec:encryptionKey', + expiration: { '@id': 'sec:expiration', '@type': 'xsd:dateTime' }, + expires: { '@id': 'sec:expiration', '@type': 'xsd:dateTime' }, + initializationVector: 'sec:initializationVector', + iterationCount: 'sec:iterationCount', + nonce: 'sec:nonce', + normalizationAlgorithm: 'sec:normalizationAlgorithm', + owner: { '@id': 'sec:owner', '@type': '@id' }, + password: 'sec:password', + privateKey: { '@id': 'sec:privateKey', '@type': '@id' }, + privateKeyPem: 'sec:privateKeyPem', + publicKey: { '@id': 'sec:publicKey', '@type': '@id' }, + publicKeyBase58: 'sec:publicKeyBase58', + publicKeyPem: 'sec:publicKeyPem', + publicKeyWif: 'sec:publicKeyWif', + publicKeyService: { '@id': 'sec:publicKeyService', '@type': '@id' }, + revoked: { '@id': 'sec:revoked', '@type': 'xsd:dateTime' }, + salt: 'sec:salt', + signature: 'sec:signature', + signatureAlgorithm: 'sec:signingAlgorithm', + signatureValue: 'sec:signatureValue' + } + } +} + +const nodeDocumentLoader = (jsonld as any).documentLoaders.node() + +const lru = new AsyncLRU({ + max: 10, + load: (url, cb) => { + if (CACHE[url] !== undefined) { + logger.debug('Using cache for JSON-LD %s.', url) + + return cb(null, { + contextUrl: null, + document: CACHE[url], + documentUrl: url + }) + } + + nodeDocumentLoader(url) + .then(value => cb(null, value)) + .catch(err => cb(err)) + } +}); + +/* eslint-disable no-import-assign */ +(jsonld as any).documentLoader = (url) => { + return new Promise((res, rej) => { + lru.get(url, (err, value) => { + if (err) return rej(err) + + return res(value) + }) + }) +} + +export { jsonld } diff --git a/server/server/helpers/custom-validators/abuses.ts b/server/server/helpers/custom-validators/abuses.ts new file mode 100644 index 000000000..4c4ae8bf7 --- /dev/null +++ b/server/server/helpers/custom-validators/abuses.ts @@ -0,0 +1,68 @@ +import validator from 'validator' +import { abusePredefinedReasonsMap } from '@peertube/peertube-core-utils' +import { AbuseCreate, AbuseFilter, AbusePredefinedReasonsString, AbuseVideoIs } from '@peertube/peertube-models' +import { ABUSE_STATES, CONSTRAINTS_FIELDS } from '../../initializers/constants.js' +import { exists, isArray } from './misc.js' + +const ABUSES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.ABUSES +const ABUSE_MESSAGES_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.ABUSE_MESSAGES + +function isAbuseReasonValid (value: string) { + return exists(value) && validator.default.isLength(value, ABUSES_CONSTRAINTS_FIELDS.REASON) +} + +function isAbusePredefinedReasonValid (value: AbusePredefinedReasonsString) { + return exists(value) && value in abusePredefinedReasonsMap +} + +function isAbuseFilterValid (value: AbuseFilter) { + return value === 'video' || value === 'comment' || value === 'account' +} + +function areAbusePredefinedReasonsValid (value: AbusePredefinedReasonsString[]) { + return exists(value) && isArray(value) && value.every(v => v in abusePredefinedReasonsMap) +} + +function isAbuseTimestampValid (value: number) { + return value === null || (exists(value) && validator.default.isInt('' + value, { min: 0 })) +} + +function isAbuseTimestampCoherent (endAt: number, { req }) { + const startAt = (req.body as AbuseCreate).video.startAt + + return exists(startAt) && endAt > startAt +} + +function isAbuseModerationCommentValid (value: string) { + return exists(value) && validator.default.isLength(value, ABUSES_CONSTRAINTS_FIELDS.MODERATION_COMMENT) +} + +function isAbuseStateValid (value: string) { + return exists(value) && ABUSE_STATES[value] !== undefined +} + +function isAbuseVideoIsValid (value: AbuseVideoIs) { + return exists(value) && ( + value === 'deleted' || + value === 'blacklisted' + ) +} + +function isAbuseMessageValid (value: string) { + return exists(value) && validator.default.isLength(value, ABUSE_MESSAGES_CONSTRAINTS_FIELDS.MESSAGE) +} + +// --------------------------------------------------------------------------- + +export { + isAbuseReasonValid, + isAbuseFilterValid, + isAbusePredefinedReasonValid, + isAbuseMessageValid, + areAbusePredefinedReasonsValid, + isAbuseTimestampValid, + isAbuseTimestampCoherent, + isAbuseModerationCommentValid, + isAbuseStateValid, + isAbuseVideoIsValid +} diff --git a/server/server/helpers/custom-validators/accounts.ts b/server/server/helpers/custom-validators/accounts.ts new file mode 100644 index 000000000..48fa1594e --- /dev/null +++ b/server/server/helpers/custom-validators/accounts.ts @@ -0,0 +1,22 @@ +import { isUserDescriptionValid, isUserUsernameValid } from './users.js' +import { exists } from './misc.js' + +function isAccountNameValid (value: string) { + return isUserUsernameValid(value) +} + +function isAccountIdValid (value: string) { + return exists(value) +} + +function isAccountDescriptionValid (value: string) { + return isUserDescriptionValid(value) +} + +// --------------------------------------------------------------------------- + +export { + isAccountIdValid, + isAccountDescriptionValid, + isAccountNameValid +} diff --git a/server/server/helpers/custom-validators/activitypub/activity.ts b/server/server/helpers/custom-validators/activitypub/activity.ts new file mode 100644 index 000000000..05399ebdf --- /dev/null +++ b/server/server/helpers/custom-validators/activitypub/activity.ts @@ -0,0 +1,151 @@ +import validator from 'validator' +import { Activity, ActivityType } from '@peertube/peertube-models' +import { isAbuseReasonValid } from '../abuses.js' +import { exists } from '../misc.js' +import { sanitizeAndCheckActorObject } from './actor.js' +import { isCacheFileObjectValid } from './cache-file.js' +import { isActivityPubUrlValid, isBaseActivityValid, isObjectValid } from './misc.js' +import { isPlaylistObjectValid } from './playlist.js' +import { sanitizeAndCheckVideoCommentObject } from './video-comments.js' +import { sanitizeAndCheckVideoTorrentObject } from './videos.js' +import { isWatchActionObjectValid } from './watch-action.js' + +function isRootActivityValid (activity: any) { + return isCollection(activity) || isActivity(activity) +} + +function isCollection (activity: any) { + return (activity.type === 'Collection' || activity.type === 'OrderedCollection') && + validator.default.isInt(activity.totalItems, { min: 0 }) && + Array.isArray(activity.items) +} + +function isActivity (activity: any) { + return isActivityPubUrlValid(activity.id) && + exists(activity.actor) && + (isActivityPubUrlValid(activity.actor) || isActivityPubUrlValid(activity.actor.id)) +} + +const activityCheckers: { [ P in ActivityType ]: (activity: Activity) => boolean } = { + Create: isCreateActivityValid, + Update: isUpdateActivityValid, + Delete: isDeleteActivityValid, + Follow: isFollowActivityValid, + Accept: isAcceptActivityValid, + Reject: isRejectActivityValid, + Announce: isAnnounceActivityValid, + Undo: isUndoActivityValid, + Like: isLikeActivityValid, + View: isViewActivityValid, + Flag: isFlagActivityValid, + Dislike: isDislikeActivityValid +} + +function isActivityValid (activity: any) { + const checker = activityCheckers[activity.type] + // Unknown activity type + if (!checker) return false + + return checker(activity) +} + +function isFlagActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Flag') && + isAbuseReasonValid(activity.content) && + isActivityPubUrlValid(activity.object) +} + +function isLikeActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Like') && + isObjectValid(activity.object) +} + +function isDislikeActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Dislike') && + isObjectValid(activity.object) +} + +function isAnnounceActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Announce') && + isObjectValid(activity.object) +} + +function isViewActivityValid (activity: any) { + return isBaseActivityValid(activity, 'View') && + isActivityPubUrlValid(activity.actor) && + isActivityPubUrlValid(activity.object) +} + +function isCreateActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Create') && + ( + isViewActivityValid(activity.object) || + isDislikeActivityValid(activity.object) || + isFlagActivityValid(activity.object) || + isPlaylistObjectValid(activity.object) || + isWatchActionObjectValid(activity.object) || + + isCacheFileObjectValid(activity.object) || + sanitizeAndCheckVideoCommentObject(activity.object) || + sanitizeAndCheckVideoTorrentObject(activity.object) + ) +} + +function isUpdateActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Update') && + ( + isCacheFileObjectValid(activity.object) || + isPlaylistObjectValid(activity.object) || + sanitizeAndCheckVideoTorrentObject(activity.object) || + sanitizeAndCheckActorObject(activity.object) + ) +} + +function isDeleteActivityValid (activity: any) { + // We don't really check objects + return isBaseActivityValid(activity, 'Delete') && + isObjectValid(activity.object) +} + +function isFollowActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Follow') && + isObjectValid(activity.object) +} + +function isAcceptActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Accept') +} + +function isRejectActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Reject') +} + +function isUndoActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Undo') && + ( + isFollowActivityValid(activity.object) || + isLikeActivityValid(activity.object) || + isDislikeActivityValid(activity.object) || + isAnnounceActivityValid(activity.object) || + isCreateActivityValid(activity.object) + ) +} + +// --------------------------------------------------------------------------- + +export { + isRootActivityValid, + isActivityValid, + isFlagActivityValid, + isLikeActivityValid, + isDislikeActivityValid, + isAnnounceActivityValid, + isViewActivityValid, + isCreateActivityValid, + isUpdateActivityValid, + isDeleteActivityValid, + isFollowActivityValid, + isAcceptActivityValid, + isRejectActivityValid, + isUndoActivityValid +} diff --git a/server/server/helpers/custom-validators/activitypub/actor.ts b/server/server/helpers/custom-validators/activitypub/actor.ts new file mode 100644 index 000000000..71ddbc323 --- /dev/null +++ b/server/server/helpers/custom-validators/activitypub/actor.ts @@ -0,0 +1,142 @@ +import validator from 'validator' +import { CONSTRAINTS_FIELDS } from '../../../initializers/constants.js' +import { exists, isArray, isDateValid } from '../misc.js' +import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc.js' +import { isHostValid } from '../servers.js' +import { peertubeTruncate } from '@server/helpers/core-utils.js' + +function isActorEndpointsObjectValid (endpointObject: any) { + if (endpointObject?.sharedInbox) { + return isActivityPubUrlValid(endpointObject.sharedInbox) + } + + // Shared inbox is optional + return true +} + +function isActorPublicKeyObjectValid (publicKeyObject: any) { + return isActivityPubUrlValid(publicKeyObject.id) && + isActivityPubUrlValid(publicKeyObject.owner) && + isActorPublicKeyValid(publicKeyObject.publicKeyPem) +} + +function isActorTypeValid (type: string) { + return type === 'Person' || type === 'Application' || type === 'Group' || type === 'Service' || type === 'Organization' +} + +function isActorPublicKeyValid (publicKey: string) { + return exists(publicKey) && + typeof publicKey === 'string' && + publicKey.startsWith('-----BEGIN PUBLIC KEY-----') && + publicKey.includes('-----END PUBLIC KEY-----') && + validator.default.isLength(publicKey, CONSTRAINTS_FIELDS.ACTORS.PUBLIC_KEY) +} + +const actorNameAlphabet = '[ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789\\-_.:]' +const actorNameRegExp = new RegExp(`^${actorNameAlphabet}+$`) +function isActorPreferredUsernameValid (preferredUsername: string) { + return exists(preferredUsername) && validator.default.matches(preferredUsername, actorNameRegExp) +} + +function isActorPrivateKeyValid (privateKey: string) { + return exists(privateKey) && + typeof privateKey === 'string' && + (privateKey.startsWith('-----BEGIN RSA PRIVATE KEY-----') || privateKey.startsWith('-----BEGIN PRIVATE KEY-----')) && + // Sometimes there is a \n at the end, so just assert the string contains the end mark + (privateKey.includes('-----END RSA PRIVATE KEY-----') || privateKey.includes('-----END PRIVATE KEY-----')) && + validator.default.isLength(privateKey, CONSTRAINTS_FIELDS.ACTORS.PRIVATE_KEY) +} + +function isActorFollowingCountValid (value: string) { + return exists(value) && validator.default.isInt('' + value, { min: 0 }) +} + +function isActorFollowersCountValid (value: string) { + return exists(value) && validator.default.isInt('' + value, { min: 0 }) +} + +function isActorDeleteActivityValid (activity: any) { + return isBaseActivityValid(activity, 'Delete') +} + +function sanitizeAndCheckActorObject (actor: any) { + if (!isActorTypeValid(actor.type)) return false + + normalizeActor(actor) + + return exists(actor) && + isActivityPubUrlValid(actor.id) && + isActivityPubUrlValid(actor.inbox) && + isActorPreferredUsernameValid(actor.preferredUsername) && + isActivityPubUrlValid(actor.url) && + isActorPublicKeyObjectValid(actor.publicKey) && + isActorEndpointsObjectValid(actor.endpoints) && + + (!actor.outbox || isActivityPubUrlValid(actor.outbox)) && + (!actor.following || isActivityPubUrlValid(actor.following)) && + (!actor.followers || isActivityPubUrlValid(actor.followers)) && + + setValidAttributedTo(actor) && + setValidDescription(actor) && + // If this is a group (a channel), it should be attributed to an account + // In PeerTube we use this to attach a video channel to a specific account + (actor.type !== 'Group' || actor.attributedTo.length !== 0) +} + +function normalizeActor (actor: any) { + if (!actor) return + + if (!actor.url) { + actor.url = actor.id + } else if (typeof actor.url !== 'string') { + actor.url = actor.url.href || actor.url.url + } + + if (!isDateValid(actor.published)) actor.published = undefined + + if (actor.summary && typeof actor.summary === 'string') { + actor.summary = peertubeTruncate(actor.summary, { length: CONSTRAINTS_FIELDS.USERS.DESCRIPTION.max }) + + if (actor.summary.length < CONSTRAINTS_FIELDS.USERS.DESCRIPTION.min) { + actor.summary = null + } + } +} + +function isValidActorHandle (handle: string) { + if (!exists(handle)) return false + + const parts = handle.split('@') + if (parts.length !== 2) return false + + return isHostValid(parts[1]) +} + +function areValidActorHandles (handles: string[]) { + return isArray(handles) && handles.every(h => isValidActorHandle(h)) +} + +function setValidDescription (obj: any) { + if (!obj.summary) obj.summary = null + + return true +} + +// --------------------------------------------------------------------------- + +export { + normalizeActor, + actorNameAlphabet, + areValidActorHandles, + isActorEndpointsObjectValid, + isActorPublicKeyObjectValid, + isActorTypeValid, + isActorPublicKeyValid, + isActorPreferredUsernameValid, + isActorPrivateKeyValid, + isActorFollowingCountValid, + isActorFollowersCountValid, + isActorDeleteActivityValid, + sanitizeAndCheckActorObject, + isValidActorHandle +} diff --git a/server/server/helpers/custom-validators/activitypub/cache-file.ts b/server/server/helpers/custom-validators/activitypub/cache-file.ts new file mode 100644 index 000000000..2b1408eb7 --- /dev/null +++ b/server/server/helpers/custom-validators/activitypub/cache-file.ts @@ -0,0 +1,26 @@ +import { CacheFileObject } from '@peertube/peertube-models' +import { exists, isDateValid } from '../misc.js' +import { isActivityPubUrlValid } from './misc.js' +import { isRemoteVideoUrlValid } from './videos.js' + +function isCacheFileObjectValid (object: CacheFileObject) { + return exists(object) && + object.type === 'CacheFile' && + (object.expires === null || isDateValid(object.expires)) && + isActivityPubUrlValid(object.object) && + (isRemoteVideoUrlValid(object.url) || isPlaylistRedundancyUrlValid(object.url)) +} + +// --------------------------------------------------------------------------- + +export { + isCacheFileObjectValid +} + +// --------------------------------------------------------------------------- + +function isPlaylistRedundancyUrlValid (url: any) { + return url.type === 'Link' && + (url.mediaType || url.mimeType) === 'application/x-mpegURL' && + isActivityPubUrlValid(url.href) +} diff --git a/server/server/helpers/custom-validators/activitypub/misc.ts b/server/server/helpers/custom-validators/activitypub/misc.ts new file mode 100644 index 000000000..0daa824f6 --- /dev/null +++ b/server/server/helpers/custom-validators/activitypub/misc.ts @@ -0,0 +1,76 @@ +import validator from 'validator' +import { CONFIG } from '@server/initializers/config.js' +import { CONSTRAINTS_FIELDS } from '../../../initializers/constants.js' +import { exists } from '../misc.js' + +function isUrlValid (url: string) { + const isURLOptions = { + require_host: true, + require_tld: true, + require_protocol: true, + require_valid_protocol: true, + protocols: [ 'http', 'https' ] + } + + // We validate 'localhost', so we don't have the top level domain + if (CONFIG.WEBSERVER.HOSTNAME === 'localhost' || CONFIG.WEBSERVER.HOSTNAME === '127.0.0.1') { + isURLOptions.require_tld = false + } + + return exists(url) && validator.default.isURL('' + url, isURLOptions) +} + +function isActivityPubUrlValid (url: string) { + return isUrlValid(url) && validator.default.isLength('' + url, CONSTRAINTS_FIELDS.ACTORS.URL) +} + +function isBaseActivityValid (activity: any, type: string) { + return activity.type === type && + isActivityPubUrlValid(activity.id) && + isObjectValid(activity.actor) && + isUrlCollectionValid(activity.to) && + isUrlCollectionValid(activity.cc) +} + +function isUrlCollectionValid (collection: any) { + return collection === undefined || + (Array.isArray(collection) && collection.every(t => isActivityPubUrlValid(t))) +} + +function isObjectValid (object: any) { + return exists(object) && + ( + isActivityPubUrlValid(object) || isActivityPubUrlValid(object.id) + ) +} + +function setValidAttributedTo (obj: any) { + if (Array.isArray(obj.attributedTo) === false) { + obj.attributedTo = [] + return true + } + + obj.attributedTo = obj.attributedTo.filter(a => { + return isActivityPubUrlValid(a) || + ((a.type === 'Group' || a.type === 'Person') && isActivityPubUrlValid(a.id)) + }) + + return true +} + +function isActivityPubVideoDurationValid (value: string) { + // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration + return exists(value) && + typeof value === 'string' && + value.startsWith('PT') && + value.endsWith('S') +} + +export { + isUrlValid, + isActivityPubUrlValid, + isBaseActivityValid, + setValidAttributedTo, + isObjectValid, + isActivityPubVideoDurationValid +} diff --git a/server/server/helpers/custom-validators/activitypub/playlist.ts b/server/server/helpers/custom-validators/activitypub/playlist.ts new file mode 100644 index 000000000..7c338992b --- /dev/null +++ b/server/server/helpers/custom-validators/activitypub/playlist.ts @@ -0,0 +1,29 @@ +import validator from 'validator' +import { PlaylistElementObject, PlaylistObject } from '@peertube/peertube-models' +import { exists, isDateValid, isUUIDValid } from '../misc.js' +import { isVideoPlaylistNameValid } from '../video-playlists.js' +import { isActivityPubUrlValid } from './misc.js' + +function isPlaylistObjectValid (object: PlaylistObject) { + return exists(object) && + object.type === 'Playlist' && + validator.default.isInt(object.totalItems + '') && + isVideoPlaylistNameValid(object.name) && + isUUIDValid(object.uuid) && + isDateValid(object.published) && + isDateValid(object.updated) +} + +function isPlaylistElementObjectValid (object: PlaylistElementObject) { + return exists(object) && + object.type === 'PlaylistElement' && + validator.default.isInt(object.position + '') && + isActivityPubUrlValid(object.url) +} + +// --------------------------------------------------------------------------- + +export { + isPlaylistObjectValid, + isPlaylistElementObjectValid +} diff --git a/server/server/helpers/custom-validators/activitypub/signature.ts b/server/server/helpers/custom-validators/activitypub/signature.ts new file mode 100644 index 000000000..2ce67875d --- /dev/null +++ b/server/server/helpers/custom-validators/activitypub/signature.ts @@ -0,0 +1,22 @@ +import { exists } from '../misc.js' +import { isActivityPubUrlValid } from './misc.js' + +function isSignatureTypeValid (signatureType: string) { + return exists(signatureType) && signatureType === 'RsaSignature2017' +} + +function isSignatureCreatorValid (signatureCreator: string) { + return exists(signatureCreator) && isActivityPubUrlValid(signatureCreator) +} + +function isSignatureValueValid (signatureValue: string) { + return exists(signatureValue) && signatureValue.length > 0 +} + +// --------------------------------------------------------------------------- + +export { + isSignatureTypeValid, + isSignatureCreatorValid, + isSignatureValueValid +} diff --git a/server/server/helpers/custom-validators/activitypub/video-comments.ts b/server/server/helpers/custom-validators/activitypub/video-comments.ts new file mode 100644 index 000000000..a024476dc --- /dev/null +++ b/server/server/helpers/custom-validators/activitypub/video-comments.ts @@ -0,0 +1,59 @@ +import validator from 'validator' +import { ACTIVITY_PUB } from '../../../initializers/constants.js' +import { exists, isArray, isDateValid } from '../misc.js' +import { isActivityPubUrlValid } from './misc.js' + +function sanitizeAndCheckVideoCommentObject (comment: any) { + if (!comment) return false + + if (!isCommentTypeValid(comment)) return false + + normalizeComment(comment) + + if (comment.type === 'Tombstone') { + return isActivityPubUrlValid(comment.id) && + isDateValid(comment.published) && + isDateValid(comment.deleted) && + isActivityPubUrlValid(comment.url) + } + + return isActivityPubUrlValid(comment.id) && + isCommentContentValid(comment.content) && + isActivityPubUrlValid(comment.inReplyTo) && + isDateValid(comment.published) && + isActivityPubUrlValid(comment.url) && + isArray(comment.to) && + ( + comment.to.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 || + comment.cc.indexOf(ACTIVITY_PUB.PUBLIC) !== -1 + ) // Only accept public comments +} + +// --------------------------------------------------------------------------- + +export { + sanitizeAndCheckVideoCommentObject +} + +// --------------------------------------------------------------------------- + +function isCommentContentValid (content: any) { + return exists(content) && validator.default.isLength('' + content, { min: 1 }) +} + +function normalizeComment (comment: any) { + if (!comment) return + + if (typeof comment.url !== 'string') { + if (typeof comment.url === 'object') comment.url = comment.url.href || comment.url.url + else comment.url = comment.id + } +} + +function isCommentTypeValid (comment: any): boolean { + if (comment.type === 'Note') return true + + if (comment.type === 'Tombstone' && comment.formerType === 'Note') return true + + return false +} diff --git a/server/server/helpers/custom-validators/activitypub/videos.ts b/server/server/helpers/custom-validators/activitypub/videos.ts new file mode 100644 index 000000000..b7298c474 --- /dev/null +++ b/server/server/helpers/custom-validators/activitypub/videos.ts @@ -0,0 +1,247 @@ +import validator from 'validator' +import { + ActivityPubStoryboard, + ActivityTrackerUrlObject, + ActivityVideoFileMetadataUrlObject, + LiveVideoLatencyMode, + VideoObject, + VideoState +} from '@peertube/peertube-models' +import { logger } from '@server/helpers/logger.js' +import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers/constants.js' +import { peertubeTruncate } from '../../core-utils.js' +import { isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc.js' +import { isLiveLatencyModeValid } from '../video-lives.js' +import { + isVideoDescriptionValid, + isVideoDurationValid, + isVideoNameValid, + isVideoStateValid, + isVideoTagValid, + isVideoViewsValid +} from '../videos.js' +import { isActivityPubUrlValid, isActivityPubVideoDurationValid, isBaseActivityValid, setValidAttributedTo } from './misc.js' + +function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) { + return isBaseActivityValid(activity, 'Update') && + sanitizeAndCheckVideoTorrentObject(activity.object) +} + +function sanitizeAndCheckVideoTorrentObject (video: any) { + if (!video || video.type !== 'Video') return false + + if (!setValidRemoteTags(video)) { + logger.debug('Video has invalid tags', { video }) + return false + } + if (!setValidRemoteVideoUrls(video)) { + logger.debug('Video has invalid urls', { video }) + return false + } + if (!setRemoteVideoContent(video)) { + logger.debug('Video has invalid content', { video }) + return false + } + if (!setValidAttributedTo(video)) { + logger.debug('Video has invalid attributedTo', { video }) + return false + } + if (!setValidRemoteCaptions(video)) { + logger.debug('Video has invalid captions', { video }) + return false + } + if (!setValidRemoteIcon(video)) { + logger.debug('Video has invalid icons', { video }) + return false + } + if (!setValidStoryboard(video)) { + logger.debug('Video has invalid preview (storyboard)', { video }) + return false + } + + // Default attributes + if (!isVideoStateValid(video.state)) video.state = VideoState.PUBLISHED + if (!isBooleanValid(video.waitTranscoding)) video.waitTranscoding = false + if (!isBooleanValid(video.downloadEnabled)) video.downloadEnabled = true + if (!isBooleanValid(video.commentsEnabled)) video.commentsEnabled = false + if (!isBooleanValid(video.isLiveBroadcast)) video.isLiveBroadcast = false + if (!isBooleanValid(video.liveSaveReplay)) video.liveSaveReplay = false + if (!isBooleanValid(video.permanentLive)) video.permanentLive = false + if (!isLiveLatencyModeValid(video.latencyMode)) video.latencyMode = LiveVideoLatencyMode.DEFAULT + + return isActivityPubUrlValid(video.id) && + isVideoNameValid(video.name) && + isActivityPubVideoDurationValid(video.duration) && + isVideoDurationValid(video.duration.replace(/[^0-9]+/g, '')) && + isUUIDValid(video.uuid) && + (!video.category || isRemoteNumberIdentifierValid(video.category)) && + (!video.licence || isRemoteNumberIdentifierValid(video.licence)) && + (!video.language || isRemoteStringIdentifierValid(video.language)) && + isVideoViewsValid(video.views) && + isBooleanValid(video.sensitive) && + isDateValid(video.published) && + isDateValid(video.updated) && + (!video.originallyPublishedAt || isDateValid(video.originallyPublishedAt)) && + (!video.uploadDate || isDateValid(video.uploadDate)) && + (!video.content || isRemoteVideoContentValid(video.mediaType, video.content)) && + video.attributedTo.length !== 0 +} + +function isRemoteVideoUrlValid (url: any) { + return url.type === 'Link' && + // Video file link + ( + ACTIVITY_PUB.URL_MIME_TYPES.VIDEO.includes(url.mediaType) && + isActivityPubUrlValid(url.href) && + validator.default.isInt(url.height + '', { min: 0 }) && + validator.default.isInt(url.size + '', { min: 0 }) && + (!url.fps || validator.default.isInt(url.fps + '', { min: -1 })) + ) || + // Torrent link + ( + ACTIVITY_PUB.URL_MIME_TYPES.TORRENT.includes(url.mediaType) && + isActivityPubUrlValid(url.href) && + validator.default.isInt(url.height + '', { min: 0 }) + ) || + // Magnet link + ( + ACTIVITY_PUB.URL_MIME_TYPES.MAGNET.includes(url.mediaType) && + validator.default.isLength(url.href, { min: 5 }) && + validator.default.isInt(url.height + '', { min: 0 }) + ) || + // HLS playlist link + ( + (url.mediaType || url.mimeType) === 'application/x-mpegURL' && + isActivityPubUrlValid(url.href) && + isArray(url.tag) + ) || + isAPVideoTrackerUrlObject(url) || + isAPVideoFileUrlMetadataObject(url) +} + +function isAPVideoFileUrlMetadataObject (url: any): url is ActivityVideoFileMetadataUrlObject { + return url && + url.type === 'Link' && + url.mediaType === 'application/json' && + isArray(url.rel) && url.rel.includes('metadata') +} + +function isAPVideoTrackerUrlObject (url: any): url is ActivityTrackerUrlObject { + return isArray(url.rel) && + url.rel.includes('tracker') && + isActivityPubUrlValid(url.href) +} + +// --------------------------------------------------------------------------- + +export { + sanitizeAndCheckVideoTorrentUpdateActivity, + isRemoteStringIdentifierValid, + sanitizeAndCheckVideoTorrentObject, + isRemoteVideoUrlValid, + isAPVideoFileUrlMetadataObject, + isAPVideoTrackerUrlObject +} + +// --------------------------------------------------------------------------- + +function setValidRemoteTags (video: any) { + if (Array.isArray(video.tag) === false) return false + + video.tag = video.tag.filter(t => { + return t.type === 'Hashtag' && + isVideoTagValid(t.name) + }) + + return true +} + +function setValidRemoteCaptions (video: any) { + if (!video.subtitleLanguage) video.subtitleLanguage = [] + + if (Array.isArray(video.subtitleLanguage) === false) return false + + video.subtitleLanguage = video.subtitleLanguage.filter(caption => { + if (!isActivityPubUrlValid(caption.url)) caption.url = null + + return isRemoteStringIdentifierValid(caption) + }) + + return true +} + +function isRemoteNumberIdentifierValid (data: any) { + return validator.default.isInt(data.identifier, { min: 0 }) +} + +function isRemoteStringIdentifierValid (data: any) { + return typeof data.identifier === 'string' +} + +function isRemoteVideoContentValid (mediaType: string, content: string) { + return mediaType === 'text/markdown' && isVideoDescriptionValid(content) +} + +function setValidRemoteIcon (video: any) { + if (video.icon && !isArray(video.icon)) video.icon = [ video.icon ] + if (!video.icon) video.icon = [] + + video.icon = video.icon.filter(icon => { + return icon.type === 'Image' && + isActivityPubUrlValid(icon.url) && + icon.mediaType === 'image/jpeg' && + validator.default.isInt(icon.width + '', { min: 0 }) && + validator.default.isInt(icon.height + '', { min: 0 }) + }) + + return video.icon.length !== 0 +} + +function setValidRemoteVideoUrls (video: any) { + if (Array.isArray(video.url) === false) return false + + video.url = video.url.filter(u => isRemoteVideoUrlValid(u)) + + return true +} + +function setRemoteVideoContent (video: any) { + if (video.content) { + video.content = peertubeTruncate(video.content, { length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max }) + } + + return true +} + +function setValidStoryboard (video: VideoObject) { + if (!video.preview) return true + if (!Array.isArray(video.preview)) return false + + video.preview = video.preview.filter(p => isStorybordValid(p)) + + return true +} + +function isStorybordValid (preview: ActivityPubStoryboard) { + if (!preview) return false + + if ( + preview.type !== 'Image' || + !isArray(preview.rel) || + !preview.rel.includes('storyboard') + ) { + return false + } + + preview.url = preview.url.filter(u => { + return u.mediaType === 'image/jpeg' && + isActivityPubUrlValid(u.href) && + validator.default.isInt(u.width + '', { min: 0 }) && + validator.default.isInt(u.height + '', { min: 0 }) && + validator.default.isInt(u.tileWidth + '', { min: 0 }) && + validator.default.isInt(u.tileHeight + '', { min: 0 }) && + isActivityPubVideoDurationValid(u.tileDuration) + }) + + return preview.url.length !== 0 +} diff --git a/server/server/helpers/custom-validators/activitypub/watch-action.ts b/server/server/helpers/custom-validators/activitypub/watch-action.ts new file mode 100644 index 000000000..426aa3805 --- /dev/null +++ b/server/server/helpers/custom-validators/activitypub/watch-action.ts @@ -0,0 +1,37 @@ +import { WatchActionObject } from '@peertube/peertube-models' +import { exists, isDateValid, isUUIDValid } from '../misc.js' +import { isVideoTimeValid } from '../video-view.js' +import { isActivityPubVideoDurationValid, isObjectValid } from './misc.js' + +function isWatchActionObjectValid (action: WatchActionObject) { + return exists(action) && + action.type === 'WatchAction' && + isObjectValid(action.id) && + isActivityPubVideoDurationValid(action.duration) && + isDateValid(action.startTime) && + isDateValid(action.endTime) && + isLocationValid(action.location) && + isUUIDValid(action.uuid) && + isObjectValid(action.object) && + isWatchSectionsValid(action.watchSections) +} + +// --------------------------------------------------------------------------- + +export { + isWatchActionObjectValid +} + +// --------------------------------------------------------------------------- + +function isLocationValid (location: any) { + if (!location) return true + + return typeof location === 'object' && typeof location.addressCountry === 'string' +} + +function isWatchSectionsValid (sections: WatchActionObject['watchSections']) { + return Array.isArray(sections) && sections.every(s => { + return isVideoTimeValid(s.startTimestamp) && isVideoTimeValid(s.endTimestamp) + }) +} diff --git a/server/server/helpers/custom-validators/actor-images.ts b/server/server/helpers/custom-validators/actor-images.ts new file mode 100644 index 000000000..5d30003c5 --- /dev/null +++ b/server/server/helpers/custom-validators/actor-images.ts @@ -0,0 +1,24 @@ + +import { UploadFilesForCheck } from 'express' +import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js' +import { isFileValid } from './misc.js' + +const imageMimeTypes = CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME + .map(v => v.replace('.', '')) + .join('|') +const imageMimeTypesRegex = `image/(${imageMimeTypes})` + +function isActorImageFile (files: UploadFilesForCheck, fieldname: string) { + return isFileValid({ + files, + mimeTypeRegex: imageMimeTypesRegex, + field: fieldname, + maxSize: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max + }) +} + +// --------------------------------------------------------------------------- + +export { + isActorImageFile +} diff --git a/server/helpers/custom-validators/bulk.ts b/server/server/helpers/custom-validators/bulk.ts similarity index 100% rename from server/helpers/custom-validators/bulk.ts rename to server/server/helpers/custom-validators/bulk.ts diff --git a/server/server/helpers/custom-validators/feeds.ts b/server/server/helpers/custom-validators/feeds.ts new file mode 100644 index 000000000..c597b889f --- /dev/null +++ b/server/server/helpers/custom-validators/feeds.ts @@ -0,0 +1,23 @@ +import { exists } from './misc.js' + +function isValidRSSFeed (value: string) { + if (!exists(value)) return false + + const feedExtensions = [ + 'xml', + 'json', + 'json1', + 'rss', + 'rss2', + 'atom', + 'atom1' + ] + + return feedExtensions.includes(value) +} + +// --------------------------------------------------------------------------- + +export { + isValidRSSFeed +} diff --git a/server/server/helpers/custom-validators/follows.ts b/server/server/helpers/custom-validators/follows.ts new file mode 100644 index 000000000..8e354dc25 --- /dev/null +++ b/server/server/helpers/custom-validators/follows.ts @@ -0,0 +1,30 @@ +import { exists, isArray } from './misc.js' +import { FollowState } from '@peertube/peertube-models' + +function isFollowStateValid (value: FollowState) { + if (!exists(value)) return false + + return value === 'pending' || value === 'accepted' || value === 'rejected' +} + +function isRemoteHandleValid (value: string) { + if (!exists(value)) return false + if (typeof value !== 'string') return false + + return value.includes('@') +} + +function isEachUniqueHandleValid (handles: string[]) { + return isArray(handles) && + handles.every(handle => { + return isRemoteHandleValid(handle) && handles.indexOf(handle) === handles.lastIndexOf(handle) + }) +} + +// --------------------------------------------------------------------------- + +export { + isFollowStateValid, + isRemoteHandleValid, + isEachUniqueHandleValid +} diff --git a/server/server/helpers/custom-validators/jobs.ts b/server/server/helpers/custom-validators/jobs.ts new file mode 100644 index 000000000..1da32c76a --- /dev/null +++ b/server/server/helpers/custom-validators/jobs.ts @@ -0,0 +1,21 @@ +import { JobState } from '@peertube/peertube-models' +import { jobTypes } from '@server/lib/job-queue/job-queue.js' +import { exists } from './misc.js' + +const jobStates: JobState[] = [ 'active', 'completed', 'failed', 'waiting', 'delayed', 'paused', 'waiting-children' ] + +function isValidJobState (value: JobState) { + return exists(value) && jobStates.includes(value) +} + +function isValidJobType (value: any) { + return exists(value) && jobTypes.includes(value) +} + +// --------------------------------------------------------------------------- + +export { + jobStates, + isValidJobState, + isValidJobType +} diff --git a/server/server/helpers/custom-validators/logs.ts b/server/server/helpers/custom-validators/logs.ts new file mode 100644 index 000000000..989ba09e5 --- /dev/null +++ b/server/server/helpers/custom-validators/logs.ts @@ -0,0 +1,42 @@ +import validator from 'validator' +import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js' +import { ClientLogLevel, ServerLogLevel } from '@peertube/peertube-models' +import { exists } from './misc.js' + +const serverLogLevels = new Set([ 'debug', 'info', 'warn', 'error' ]) +const clientLogLevels = new Set([ 'warn', 'error' ]) + +function isValidLogLevel (value: any) { + return exists(value) && serverLogLevels.has(value) +} + +function isValidClientLogMessage (value: any) { + return typeof value === 'string' && validator.default.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_MESSAGE) +} + +function isValidClientLogLevel (value: any) { + return exists(value) && clientLogLevels.has(value) +} + +function isValidClientLogStackTrace (value: any) { + return typeof value === 'string' && validator.default.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_STACK_TRACE) +} + +function isValidClientLogMeta (value: any) { + return typeof value === 'string' && validator.default.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_META) +} + +function isValidClientLogUserAgent (value: any) { + return typeof value === 'string' && validator.default.isLength(value, CONSTRAINTS_FIELDS.LOGS.CLIENT_USER_AGENT) +} + +// --------------------------------------------------------------------------- + +export { + isValidLogLevel, + isValidClientLogMessage, + isValidClientLogStackTrace, + isValidClientLogMeta, + isValidClientLogLevel, + isValidClientLogUserAgent +} diff --git a/server/helpers/custom-validators/metrics.ts b/server/server/helpers/custom-validators/metrics.ts similarity index 100% rename from server/helpers/custom-validators/metrics.ts rename to server/server/helpers/custom-validators/metrics.ts diff --git a/server/server/helpers/custom-validators/misc.ts b/server/server/helpers/custom-validators/misc.ts new file mode 100644 index 000000000..0be9ce2a3 --- /dev/null +++ b/server/server/helpers/custom-validators/misc.ts @@ -0,0 +1,190 @@ +import 'multer' +import { UploadFilesForCheck } from 'express' +import { sep } from 'path' +import validator from 'validator' +import { isShortUUID, shortToUUID } from '@peertube/peertube-node-utils' + +function exists (value: any) { + return value !== undefined && value !== null +} + +function isSafePath (p: string) { + return exists(p) && + (p + '').split(sep).every(part => { + return [ '..' ].includes(part) === false + }) +} + +function isSafeFilename (filename: string, extension?: string) { + const regex = extension + ? new RegExp(`^[a-z0-9-]+\\.${extension}$`) + : new RegExp(`^[a-z0-9-]+\\.[a-z0-9]{1,8}$`) + + return typeof filename === 'string' && !!filename.match(regex) +} + +function isSafePeerTubeFilenameWithoutExtension (filename: string) { + return filename.match(/^[a-z0-9-]+$/) +} + +function isArray (value: any): value is any[] { + return Array.isArray(value) +} + +function isNotEmptyIntArray (value: any) { + return Array.isArray(value) && value.every(v => validator.default.isInt('' + v)) && value.length !== 0 +} + +function isNotEmptyStringArray (value: any) { + return Array.isArray(value) && value.every(v => typeof v === 'string' && v.length !== 0) && value.length !== 0 +} + +function isArrayOf (value: any, validator: (value: any) => boolean) { + return isArray(value) && value.every(v => validator(v)) +} + +function isDateValid (value: string) { + return exists(value) && validator.default.isISO8601(value) +} + +function isIdValid (value: string) { + return exists(value) && validator.default.isInt('' + value) +} + +function isUUIDValid (value: string) { + return exists(value) && validator.default.isUUID('' + value, 4) +} + +function areUUIDsValid (values: string[]) { + return isArray(values) && values.every(v => isUUIDValid(v)) +} + +function isIdOrUUIDValid (value: string) { + return isIdValid(value) || isUUIDValid(value) +} + +function isBooleanValid (value: any) { + return typeof value === 'boolean' || (typeof value === 'string' && validator.default.isBoolean(value)) +} + +function isIntOrNull (value: any) { + return value === null || validator.default.isInt('' + value) +} + +// --------------------------------------------------------------------------- + +function isFileValid (options: { + files: UploadFilesForCheck + + maxSize: number | null + mimeTypeRegex: string | null + + field?: string + + optional?: boolean // Default false +}) { + const { files, mimeTypeRegex, field, maxSize, optional = false } = options + + // Should have files + if (!files) return optional + + const fileArray = isArray(files) + ? files + : files[field] + + if (!fileArray || !isArray(fileArray) || fileArray.length === 0) { + return optional + } + + // The file exists + const file = fileArray[0] + if (!file?.originalname) return false + + // Check size + if ((maxSize !== null) && file.size > maxSize) return false + + if (mimeTypeRegex === null) return true + + return checkMimetypeRegex(file.mimetype, mimeTypeRegex) +} + +function checkMimetypeRegex (fileMimeType: string, mimeTypeRegex: string) { + return new RegExp(`^${mimeTypeRegex}$`, 'i').test(fileMimeType) +} + +// --------------------------------------------------------------------------- + +function toCompleteUUID (value: string) { + if (isShortUUID(value)) { + try { + return shortToUUID(value) + } catch { + return '' + } + } + + return value +} + +function toCompleteUUIDs (values: string[]) { + return values.map(v => toCompleteUUID(v)) +} + +function toIntOrNull (value: string) { + const v = toValueOrNull(value) + + if (v === null || v === undefined) return v + if (typeof v === 'number') return v + + return validator.default.toInt('' + v) +} + +function toBooleanOrNull (value: any) { + const v = toValueOrNull(value) + + if (v === null || v === undefined) return v + if (typeof v === 'boolean') return v + + return validator.default.toBoolean('' + v) +} + +function toValueOrNull (value: string) { + if (value === 'null') return null + + return value +} + +function toIntArray (value: any) { + if (!value) return [] + if (isArray(value) === false) return [ validator.default.toInt(value) ] + + return value.map(v => validator.default.toInt(v)) +} + +// --------------------------------------------------------------------------- + +export { + exists, + isArrayOf, + isNotEmptyIntArray, + isArray, + isIntOrNull, + isIdValid, + isSafePath, + isNotEmptyStringArray, + isUUIDValid, + toCompleteUUIDs, + toCompleteUUID, + isIdOrUUIDValid, + isDateValid, + toValueOrNull, + toBooleanOrNull, + isBooleanValid, + toIntOrNull, + areUUIDsValid, + toIntArray, + isFileValid, + isSafePeerTubeFilenameWithoutExtension, + isSafeFilename, + checkMimetypeRegex +} diff --git a/server/server/helpers/custom-validators/plugins.ts b/server/server/helpers/custom-validators/plugins.ts new file mode 100644 index 000000000..875482f53 --- /dev/null +++ b/server/server/helpers/custom-validators/plugins.ts @@ -0,0 +1,177 @@ +import validator from 'validator' +import { PluginPackageJSON, PluginType, PluginType_Type } from '@peertube/peertube-models' +import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js' +import { isUrlValid } from './activitypub/misc.js' +import { exists, isArray, isSafePath } from './misc.js' + +const PLUGINS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.PLUGINS + +function isPluginTypeValid (value: any) { + return exists(value) && + (value === PluginType.PLUGIN || value === PluginType.THEME) +} + +function isPluginNameValid (value: string) { + return exists(value) && + validator.default.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.NAME) && + validator.default.matches(value, /^[a-z-0-9]+$/) +} + +function isNpmPluginNameValid (value: string) { + return exists(value) && + validator.default.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.NAME) && + validator.default.matches(value, /^[a-z\-._0-9]+$/) && + (value.startsWith('peertube-plugin-') || value.startsWith('peertube-theme-')) +} + +function isPluginDescriptionValid (value: string) { + return exists(value) && validator.default.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.DESCRIPTION) +} + +function isPluginStableVersionValid (value: string) { + if (!exists(value)) return false + + const parts = (value + '').split('.') + + return parts.length === 3 && parts.every(p => validator.default.isInt(p)) +} + +function isPluginStableOrUnstableVersionValid (value: string) { + if (!exists(value)) return false + + // suffix is beta.x or alpha.x + const [ stable, suffix ] = value.split('-') + if (!isPluginStableVersionValid(stable)) return false + + const suffixRegex = /^(rc|alpha|beta)\.\d+$/ + if (suffix && !suffixRegex.test(suffix)) return false + + return true +} + +function isPluginEngineValid (engine: any) { + return exists(engine) && exists(engine.peertube) +} + +function isPluginHomepage (value: string) { + return exists(value) && (!value || isUrlValid(value)) +} + +function isPluginBugs (value: string) { + return exists(value) && (!value || isUrlValid(value)) +} + +function areStaticDirectoriesValid (staticDirs: any) { + if (!exists(staticDirs) || typeof staticDirs !== 'object') return false + + for (const key of Object.keys(staticDirs)) { + if (!isSafePath(staticDirs[key])) return false + } + + return true +} + +function areClientScriptsValid (clientScripts: any[]) { + return isArray(clientScripts) && + clientScripts.every(c => { + return isSafePath(c.script) && isArray(c.scopes) + }) +} + +function areTranslationPathsValid (translations: any) { + if (!exists(translations) || typeof translations !== 'object') return false + + for (const key of Object.keys(translations)) { + if (!isSafePath(translations[key])) return false + } + + return true +} + +function areCSSPathsValid (css: any[]) { + return isArray(css) && css.every(c => isSafePath(c)) +} + +function isThemeNameValid (name: string) { + return isPluginNameValid(name) +} + +function isPackageJSONValid (packageJSON: PluginPackageJSON, pluginType: PluginType_Type) { + let result = true + const badFields: string[] = [] + + if (!isNpmPluginNameValid(packageJSON.name)) { + result = false + badFields.push('name') + } + + if (!isPluginDescriptionValid(packageJSON.description)) { + result = false + badFields.push('description') + } + + if (!isPluginEngineValid(packageJSON.engine)) { + result = false + badFields.push('engine') + } + + if (!isPluginHomepage(packageJSON.homepage)) { + result = false + badFields.push('homepage') + } + + if (!exists(packageJSON.author)) { + result = false + badFields.push('author') + } + + if (!isPluginBugs(packageJSON.bugs)) { + result = false + badFields.push('bugs') + } + + if (pluginType === PluginType.PLUGIN && !isSafePath(packageJSON.library)) { + result = false + badFields.push('library') + } + + if (!areStaticDirectoriesValid(packageJSON.staticDirs)) { + result = false + badFields.push('staticDirs') + } + + if (!areCSSPathsValid(packageJSON.css)) { + result = false + badFields.push('css') + } + + if (!areClientScriptsValid(packageJSON.clientScripts)) { + result = false + badFields.push('clientScripts') + } + + if (!areTranslationPathsValid(packageJSON.translations)) { + result = false + badFields.push('translations') + } + + return { result, badFields } +} + +function isLibraryCodeValid (library: any) { + return typeof library.register === 'function' && + typeof library.unregister === 'function' +} + +export { + isPluginTypeValid, + isPackageJSONValid, + isThemeNameValid, + isPluginHomepage, + isPluginStableVersionValid, + isPluginStableOrUnstableVersionValid, + isPluginNameValid, + isPluginDescriptionValid, + isLibraryCodeValid, + isNpmPluginNameValid +} diff --git a/server/server/helpers/custom-validators/runners/jobs.ts b/server/server/helpers/custom-validators/runners/jobs.ts new file mode 100644 index 000000000..78c6c2e1e --- /dev/null +++ b/server/server/helpers/custom-validators/runners/jobs.ts @@ -0,0 +1,197 @@ +import { UploadFilesForCheck } from 'express' +import validator from 'validator' +import { CONSTRAINTS_FIELDS, RUNNER_JOB_STATES } from '@server/initializers/constants.js' +import { + LiveRTMPHLSTranscodingSuccess, + RunnerJobSuccessPayload, + RunnerJobType, + RunnerJobUpdatePayload, + VideoStudioTranscodingSuccess, + VODAudioMergeTranscodingSuccess, + VODHLSTranscodingSuccess, + VODWebVideoTranscodingSuccess +} from '@peertube/peertube-models' +import { exists, isArray, isFileValid, isSafeFilename } from '../misc.js' + +const RUNNER_JOBS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNER_JOBS + +const runnerJobTypes = new Set([ 'vod-hls-transcoding', 'vod-web-video-transcoding', 'vod-audio-merge-transcoding' ]) +function isRunnerJobTypeValid (value: RunnerJobType) { + return runnerJobTypes.has(value) +} + +function isRunnerJobSuccessPayloadValid (value: RunnerJobSuccessPayload, type: RunnerJobType, files: UploadFilesForCheck) { + return isRunnerJobVODWebVideoResultPayloadValid(value as VODWebVideoTranscodingSuccess, type, files) || + isRunnerJobVODHLSResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) || + isRunnerJobVODAudioMergeResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) || + isRunnerJobLiveRTMPHLSResultPayloadValid(value as LiveRTMPHLSTranscodingSuccess, type) || + isRunnerJobVideoStudioResultPayloadValid(value as VideoStudioTranscodingSuccess, type, files) +} + +// --------------------------------------------------------------------------- + +function isRunnerJobProgressValid (value: string) { + return validator.default.isInt(value + '', RUNNER_JOBS_CONSTRAINTS_FIELDS.PROGRESS) +} + +function isRunnerJobUpdatePayloadValid (value: RunnerJobUpdatePayload, type: RunnerJobType, files: UploadFilesForCheck) { + return isRunnerJobVODWebVideoUpdatePayloadValid(value, type, files) || + isRunnerJobVODHLSUpdatePayloadValid(value, type, files) || + isRunnerJobVideoStudioUpdatePayloadValid(value, type, files) || + isRunnerJobVODAudioMergeUpdatePayloadValid(value, type, files) || + isRunnerJobLiveRTMPHLSUpdatePayloadValid(value, type, files) +} + +// --------------------------------------------------------------------------- + +function isRunnerJobTokenValid (value: string) { + return exists(value) && validator.default.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.TOKEN) +} + +function isRunnerJobAbortReasonValid (value: string) { + return validator.default.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.REASON) +} + +function isRunnerJobErrorMessageValid (value: string) { + return validator.default.isLength(value, RUNNER_JOBS_CONSTRAINTS_FIELDS.ERROR_MESSAGE) +} + +function isRunnerJobStateValid (value: any) { + return exists(value) && RUNNER_JOB_STATES[value] !== undefined +} + +function isRunnerJobArrayOfStateValid (value: any) { + return isArray(value) && value.every(v => isRunnerJobStateValid(v)) +} + +// --------------------------------------------------------------------------- + +export { + isRunnerJobTypeValid, + isRunnerJobSuccessPayloadValid, + isRunnerJobUpdatePayloadValid, + isRunnerJobTokenValid, + isRunnerJobErrorMessageValid, + isRunnerJobProgressValid, + isRunnerJobAbortReasonValid, + isRunnerJobArrayOfStateValid, + isRunnerJobStateValid +} + +// --------------------------------------------------------------------------- + +function isRunnerJobVODWebVideoResultPayloadValid ( + _value: VODWebVideoTranscodingSuccess, + type: RunnerJobType, + files: UploadFilesForCheck +) { + return type === 'vod-web-video-transcoding' && + isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) +} + +function isRunnerJobVODHLSResultPayloadValid ( + _value: VODHLSTranscodingSuccess, + type: RunnerJobType, + files: UploadFilesForCheck +) { + return type === 'vod-hls-transcoding' && + isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) && + isFileValid({ files, field: 'payload[resolutionPlaylistFile]', mimeTypeRegex: null, maxSize: null }) +} + +function isRunnerJobVODAudioMergeResultPayloadValid ( + _value: VODAudioMergeTranscodingSuccess, + type: RunnerJobType, + files: UploadFilesForCheck +) { + return type === 'vod-audio-merge-transcoding' && + isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) +} + +function isRunnerJobLiveRTMPHLSResultPayloadValid ( + value: LiveRTMPHLSTranscodingSuccess, + type: RunnerJobType +) { + return type === 'live-rtmp-hls-transcoding' && (!value || (typeof value === 'object' && Object.keys(value).length === 0)) +} + +function isRunnerJobVideoStudioResultPayloadValid ( + _value: VideoStudioTranscodingSuccess, + type: RunnerJobType, + files: UploadFilesForCheck +) { + return type === 'video-studio-transcoding' && + isFileValid({ files, field: 'payload[videoFile]', mimeTypeRegex: null, maxSize: null }) +} + +// --------------------------------------------------------------------------- + +function isRunnerJobVODWebVideoUpdatePayloadValid ( + value: RunnerJobUpdatePayload, + type: RunnerJobType, + _files: UploadFilesForCheck +) { + return type === 'vod-web-video-transcoding' && + (!value || (typeof value === 'object' && Object.keys(value).length === 0)) +} + +function isRunnerJobVODHLSUpdatePayloadValid ( + value: RunnerJobUpdatePayload, + type: RunnerJobType, + _files: UploadFilesForCheck +) { + return type === 'vod-hls-transcoding' && + (!value || (typeof value === 'object' && Object.keys(value).length === 0)) +} + +function isRunnerJobVODAudioMergeUpdatePayloadValid ( + value: RunnerJobUpdatePayload, + type: RunnerJobType, + _files: UploadFilesForCheck +) { + return type === 'vod-audio-merge-transcoding' && + (!value || (typeof value === 'object' && Object.keys(value).length === 0)) +} + +function isRunnerJobLiveRTMPHLSUpdatePayloadValid ( + value: RunnerJobUpdatePayload, + type: RunnerJobType, + files: UploadFilesForCheck +) { + let result = type === 'live-rtmp-hls-transcoding' && !!value && !!files + + result &&= isFileValid({ files, field: 'payload[masterPlaylistFile]', mimeTypeRegex: null, maxSize: null, optional: true }) + + result &&= isFileValid({ + files, + field: 'payload[resolutionPlaylistFile]', + mimeTypeRegex: null, + maxSize: null, + optional: !value.resolutionPlaylistFilename + }) + + if (files['payload[resolutionPlaylistFile]']) { + result &&= isSafeFilename(value.resolutionPlaylistFilename, 'm3u8') + } + + return result && + isSafeFilename(value.videoChunkFilename, 'ts') && + ( + ( + value.type === 'remove-chunk' + ) || + ( + value.type === 'add-chunk' && + isFileValid({ files, field: 'payload[videoChunkFile]', mimeTypeRegex: null, maxSize: null }) + ) + ) +} + +function isRunnerJobVideoStudioUpdatePayloadValid ( + value: RunnerJobUpdatePayload, + type: RunnerJobType, + _files: UploadFilesForCheck +) { + return type === 'video-studio-transcoding' && + (!value || (typeof value === 'object' && Object.keys(value).length === 0)) +} diff --git a/server/server/helpers/custom-validators/runners/runners.ts b/server/server/helpers/custom-validators/runners/runners.ts new file mode 100644 index 000000000..821622a2b --- /dev/null +++ b/server/server/helpers/custom-validators/runners/runners.ts @@ -0,0 +1,30 @@ +import validator from 'validator' +import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js' +import { exists } from '../misc.js' + +const RUNNERS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNERS + +function isRunnerRegistrationTokenValid (value: string) { + return exists(value) && validator.default.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.TOKEN) +} + +function isRunnerTokenValid (value: string) { + return exists(value) && validator.default.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.TOKEN) +} + +function isRunnerNameValid (value: string) { + return exists(value) && validator.default.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.NAME) +} + +function isRunnerDescriptionValid (value: string) { + return exists(value) && validator.default.isLength(value, RUNNERS_CONSTRAINTS_FIELDS.DESCRIPTION) +} + +// --------------------------------------------------------------------------- + +export { + isRunnerRegistrationTokenValid, + isRunnerTokenValid, + isRunnerNameValid, + isRunnerDescriptionValid +} diff --git a/server/server/helpers/custom-validators/search.ts b/server/server/helpers/custom-validators/search.ts new file mode 100644 index 000000000..cb950c8dd --- /dev/null +++ b/server/server/helpers/custom-validators/search.ts @@ -0,0 +1,37 @@ +import validator from 'validator' +import { SearchTargetType } from '@peertube/peertube-models' +import { CONFIG } from '@server/initializers/config.js' +import { exists, isArray } from './misc.js' + +function isNumberArray (value: any) { + return isArray(value) && value.every(v => validator.default.isInt('' + v)) +} + +function isStringArray (value: any) { + return isArray(value) && value.every(v => typeof v === 'string') +} + +function isBooleanBothQueryValid (value: any) { + return value === 'true' || value === 'false' || value === 'both' +} + +function isSearchTargetValid (value: SearchTargetType) { + if (!exists(value)) return true + + const searchIndexConfig = CONFIG.SEARCH.SEARCH_INDEX + + if (value === 'local') return true + + if (value === 'search-index' && searchIndexConfig.ENABLED) return true + + return false +} + +// --------------------------------------------------------------------------- + +export { + isNumberArray, + isStringArray, + isBooleanBothQueryValid, + isSearchTargetValid +} diff --git a/server/server/helpers/custom-validators/servers.ts b/server/server/helpers/custom-validators/servers.ts new file mode 100644 index 000000000..5f884e911 --- /dev/null +++ b/server/server/helpers/custom-validators/servers.ts @@ -0,0 +1,42 @@ +import validator from 'validator' +import { CONFIG } from '@server/initializers/config.js' +import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js' +import { exists, isArray } from './misc.js' + +function isHostValid (host: string) { + const isURLOptions = { + require_host: true, + require_tld: true + } + + // We validate 'localhost', so we don't have the top level domain + if (CONFIG.WEBSERVER.HOSTNAME === 'localhost' || CONFIG.WEBSERVER.HOSTNAME === '127.0.0.1') { + isURLOptions.require_tld = false + } + + return exists(host) && validator.default.isURL(host, isURLOptions) && host.split('://').length === 1 +} + +function isEachUniqueHostValid (hosts: string[]) { + return isArray(hosts) && + hosts.every(host => { + return isHostValid(host) && hosts.indexOf(host) === hosts.lastIndexOf(host) + }) +} + +function isValidContactBody (value: any) { + return exists(value) && validator.default.isLength(value, CONSTRAINTS_FIELDS.CONTACT_FORM.BODY) +} + +function isValidContactFromName (value: any) { + return exists(value) && validator.default.isLength(value, CONSTRAINTS_FIELDS.CONTACT_FORM.FROM_NAME) +} + +// --------------------------------------------------------------------------- + +export { + isValidContactBody, + isValidContactFromName, + isEachUniqueHostValid, + isHostValid +} diff --git a/server/server/helpers/custom-validators/user-notifications.ts b/server/server/helpers/custom-validators/user-notifications.ts new file mode 100644 index 000000000..151f96522 --- /dev/null +++ b/server/server/helpers/custom-validators/user-notifications.ts @@ -0,0 +1,23 @@ +import validator from 'validator' +import { UserNotificationSettingValue } from '@peertube/peertube-models' +import { exists } from './misc.js' + +function isUserNotificationTypeValid (value: any) { + return exists(value) && validator.default.isInt('' + value) +} + +function isUserNotificationSettingValid (value: any) { + return exists(value) && + validator.default.isInt('' + value) && + ( + value === UserNotificationSettingValue.NONE || + value === UserNotificationSettingValue.WEB || + value === UserNotificationSettingValue.EMAIL || + value === (UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL) + ) +} + +export { + isUserNotificationSettingValid, + isUserNotificationTypeValid +} diff --git a/server/server/helpers/custom-validators/user-registration.ts b/server/server/helpers/custom-validators/user-registration.ts new file mode 100644 index 000000000..7d8c8e592 --- /dev/null +++ b/server/server/helpers/custom-validators/user-registration.ts @@ -0,0 +1,25 @@ +import validator from 'validator' +import { CONSTRAINTS_FIELDS, USER_REGISTRATION_STATES } from '../../initializers/constants.js' +import { exists } from './misc.js' + +const USER_REGISTRATIONS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.USER_REGISTRATIONS + +function isRegistrationStateValid (value: string) { + return exists(value) && USER_REGISTRATION_STATES[value] !== undefined +} + +function isRegistrationModerationResponseValid (value: string) { + return exists(value) && validator.default.isLength(value, USER_REGISTRATIONS_CONSTRAINTS_FIELDS.MODERATOR_MESSAGE) +} + +function isRegistrationReasonValid (value: string) { + return exists(value) && validator.default.isLength(value, USER_REGISTRATIONS_CONSTRAINTS_FIELDS.REASON_MESSAGE) +} + +// --------------------------------------------------------------------------- + +export { + isRegistrationStateValid, + isRegistrationModerationResponseValid, + isRegistrationReasonValid +} diff --git a/server/server/helpers/custom-validators/users.ts b/server/server/helpers/custom-validators/users.ts new file mode 100644 index 000000000..84c49ff66 --- /dev/null +++ b/server/server/helpers/custom-validators/users.ts @@ -0,0 +1,125 @@ +import validator from 'validator' +import { UserRole } from '@peertube/peertube-models' +import { isEmailEnabled } from '../../initializers/config.js' +import { CONSTRAINTS_FIELDS, NSFW_POLICY_TYPES } from '../../initializers/constants.js' +import { exists, isArray, isBooleanValid } from './misc.js' + +const USERS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.USERS + +function isUserPasswordValid (value: string) { + return validator.default.isLength(value, USERS_CONSTRAINTS_FIELDS.PASSWORD) +} + +function isUserPasswordValidOrEmpty (value: string) { + // Empty password is only possible if emailing is enabled. + if (value === '') return isEmailEnabled() + + return isUserPasswordValid(value) +} + +function isUserVideoQuotaValid (value: string) { + return exists(value) && validator.default.isInt(value + '', USERS_CONSTRAINTS_FIELDS.VIDEO_QUOTA) +} + +function isUserVideoQuotaDailyValid (value: string) { + return exists(value) && validator.default.isInt(value + '', USERS_CONSTRAINTS_FIELDS.VIDEO_QUOTA_DAILY) +} + +function isUserUsernameValid (value: string) { + return exists(value) && + validator.default.matches(value, new RegExp(`^[a-z0-9_]+([a-z0-9_.-]+[a-z0-9_]+)?$`)) && + validator.default.isLength(value, USERS_CONSTRAINTS_FIELDS.USERNAME) +} + +function isUserDisplayNameValid (value: string) { + return value === null || (exists(value) && validator.default.isLength(value, CONSTRAINTS_FIELDS.USERS.NAME)) +} + +function isUserDescriptionValid (value: string) { + return value === null || (exists(value) && validator.default.isLength(value, CONSTRAINTS_FIELDS.USERS.DESCRIPTION)) +} + +function isUserEmailVerifiedValid (value: any) { + return isBooleanValid(value) +} + +const nsfwPolicies = new Set(Object.values(NSFW_POLICY_TYPES)) +function isUserNSFWPolicyValid (value: any) { + return exists(value) && nsfwPolicies.has(value) +} + +function isUserP2PEnabledValid (value: any) { + return isBooleanValid(value) +} + +function isUserVideosHistoryEnabledValid (value: any) { + return isBooleanValid(value) +} + +function isUserAutoPlayVideoValid (value: any) { + return isBooleanValid(value) +} + +function isUserVideoLanguages (value: any) { + return value === null || (isArray(value) && value.length < CONSTRAINTS_FIELDS.USERS.VIDEO_LANGUAGES.max) +} + +function isUserAdminFlagsValid (value: any) { + return exists(value) && validator.default.isInt('' + value) +} + +function isUserBlockedValid (value: any) { + return isBooleanValid(value) +} + +function isUserAutoPlayNextVideoValid (value: any) { + return isBooleanValid(value) +} + +function isUserAutoPlayNextVideoPlaylistValid (value: any) { + return isBooleanValid(value) +} + +function isUserEmailPublicValid (value: any) { + return isBooleanValid(value) +} + +function isUserNoModal (value: any) { + return isBooleanValid(value) +} + +function isUserBlockedReasonValid (value: any) { + return value === null || (exists(value) && validator.default.isLength(value, CONSTRAINTS_FIELDS.USERS.BLOCKED_REASON)) +} + +function isUserRoleValid (value: any) { + return exists(value) && + validator.default.isInt('' + value) && + [ UserRole.ADMINISTRATOR, UserRole.MODERATOR, UserRole.USER ].includes(value) +} + +// --------------------------------------------------------------------------- + +export { + isUserVideosHistoryEnabledValid, + isUserBlockedValid, + isUserPasswordValid, + isUserPasswordValidOrEmpty, + isUserVideoLanguages, + isUserBlockedReasonValid, + isUserRoleValid, + isUserVideoQuotaValid, + isUserVideoQuotaDailyValid, + isUserUsernameValid, + isUserAdminFlagsValid, + isUserEmailVerifiedValid, + isUserNSFWPolicyValid, + isUserP2PEnabledValid, + isUserAutoPlayVideoValid, + isUserAutoPlayNextVideoValid, + isUserAutoPlayNextVideoPlaylistValid, + isUserDisplayNameValid, + isUserDescriptionValid, + isUserEmailPublicValid, + isUserNoModal +} diff --git a/server/server/helpers/custom-validators/video-blacklist.ts b/server/server/helpers/custom-validators/video-blacklist.ts new file mode 100644 index 000000000..15a13ae5f --- /dev/null +++ b/server/server/helpers/custom-validators/video-blacklist.ts @@ -0,0 +1,22 @@ +import validator from 'validator' +import { VideoBlacklistType } from '@peertube/peertube-models' +import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js' +import { exists } from './misc.js' + +const VIDEO_BLACKLIST_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_BLACKLIST + +function isVideoBlacklistReasonValid (value: string) { + return value === null || validator.default.isLength(value, VIDEO_BLACKLIST_CONSTRAINTS_FIELDS.REASON) +} + +function isVideoBlacklistTypeValid (value: any) { + return exists(value) && + (value === VideoBlacklistType.AUTO_BEFORE_PUBLISHED || value === VideoBlacklistType.MANUAL) +} + +// --------------------------------------------------------------------------- + +export { + isVideoBlacklistReasonValid, + isVideoBlacklistTypeValid +} diff --git a/server/server/helpers/custom-validators/video-captions.ts b/server/server/helpers/custom-validators/video-captions.ts new file mode 100644 index 000000000..dcd9fc2c1 --- /dev/null +++ b/server/server/helpers/custom-validators/video-captions.ts @@ -0,0 +1,43 @@ +import { UploadFilesForCheck } from 'express' +import { readFile } from 'fs/promises' +import { getFileSize } from '@peertube/peertube-node-utils' +import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_LANGUAGES } from '../../initializers/constants.js' +import { logger } from '../logger.js' +import { exists, isFileValid } from './misc.js' + +function isVideoCaptionLanguageValid (value: any) { + return exists(value) && VIDEO_LANGUAGES[value] !== undefined +} + +// MacOS sends application/octet-stream +const videoCaptionTypesRegex = [ ...Object.keys(MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT), 'application/octet-stream' ] + .map(m => `(${m})`) + .join('|') + +function isVideoCaptionFile (files: UploadFilesForCheck, field: string) { + return isFileValid({ + files, + mimeTypeRegex: videoCaptionTypesRegex, + field, + maxSize: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max + }) +} + +async function isVTTFileValid (filePath: string) { + const size = await getFileSize(filePath) + const content = await readFile(filePath, 'utf8') + + logger.debug('Checking VTT file %s', filePath, { size, content }) + + if (size > CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max) return false + + return content?.startsWith('WEBVTT') +} + +// --------------------------------------------------------------------------- + +export { + isVideoCaptionFile, + isVTTFileValid, + isVideoCaptionLanguageValid +} diff --git a/server/server/helpers/custom-validators/video-channel-syncs.ts b/server/server/helpers/custom-validators/video-channel-syncs.ts new file mode 100644 index 000000000..a07d8ed52 --- /dev/null +++ b/server/server/helpers/custom-validators/video-channel-syncs.ts @@ -0,0 +1,6 @@ +import { VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants.js' +import { exists } from './misc.js' + +export function isVideoChannelSyncStateValid (value: any) { + return exists(value) && VIDEO_CHANNEL_SYNC_STATE[value] !== undefined +} diff --git a/server/server/helpers/custom-validators/video-channels.ts b/server/server/helpers/custom-validators/video-channels.ts new file mode 100644 index 000000000..2ea3afd2d --- /dev/null +++ b/server/server/helpers/custom-validators/video-channels.ts @@ -0,0 +1,32 @@ +import validator from 'validator' +import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js' +import { exists } from './misc.js' +import { isUserUsernameValid } from './users.js' + +const VIDEO_CHANNELS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_CHANNELS + +function isVideoChannelUsernameValid (value: string) { + // Use the same constraints than user username + return isUserUsernameValid(value) +} + +function isVideoChannelDescriptionValid (value: string) { + return value === null || validator.default.isLength(value, VIDEO_CHANNELS_CONSTRAINTS_FIELDS.DESCRIPTION) +} + +function isVideoChannelDisplayNameValid (value: string) { + return exists(value) && validator.default.isLength(value, VIDEO_CHANNELS_CONSTRAINTS_FIELDS.NAME) +} + +function isVideoChannelSupportValid (value: string) { + return value === null || (exists(value) && validator.default.isLength(value, VIDEO_CHANNELS_CONSTRAINTS_FIELDS.SUPPORT)) +} + +// --------------------------------------------------------------------------- + +export { + isVideoChannelUsernameValid, + isVideoChannelDescriptionValid, + isVideoChannelDisplayNameValid, + isVideoChannelSupportValid +} diff --git a/server/server/helpers/custom-validators/video-comments.ts b/server/server/helpers/custom-validators/video-comments.ts new file mode 100644 index 000000000..104583663 --- /dev/null +++ b/server/server/helpers/custom-validators/video-comments.ts @@ -0,0 +1,14 @@ +import validator from 'validator' +import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js' + +const VIDEO_COMMENTS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEO_COMMENTS + +function isValidVideoCommentText (value: string) { + return value === null || validator.default.isLength(value, VIDEO_COMMENTS_CONSTRAINTS_FIELDS.TEXT) +} + +// --------------------------------------------------------------------------- + +export { + isValidVideoCommentText +} diff --git a/server/server/helpers/custom-validators/video-imports.ts b/server/server/helpers/custom-validators/video-imports.ts new file mode 100644 index 000000000..7d6bba932 --- /dev/null +++ b/server/server/helpers/custom-validators/video-imports.ts @@ -0,0 +1,46 @@ +import 'multer' +import { UploadFilesForCheck } from 'express' +import validator from 'validator' +import { CONSTRAINTS_FIELDS, MIMETYPES, VIDEO_IMPORT_STATES } from '../../initializers/constants.js' +import { exists, isFileValid } from './misc.js' + +function isVideoImportTargetUrlValid (url: string) { + const isURLOptions = { + require_host: true, + require_tld: true, + require_protocol: true, + require_valid_protocol: true, + protocols: [ 'http', 'https' ] + } + + return exists(url) && + validator.default.isURL('' + url, isURLOptions) && + validator.default.isLength('' + url, CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL) +} + +function isVideoImportStateValid (value: any) { + return exists(value) && VIDEO_IMPORT_STATES[value] !== undefined +} + +// MacOS sends application/octet-stream +const videoTorrentImportRegex = [ ...Object.keys(MIMETYPES.TORRENT.MIMETYPE_EXT), 'application/octet-stream' ] + .map(m => `(${m})`) + .join('|') + +function isVideoImportTorrentFile (files: UploadFilesForCheck) { + return isFileValid({ + files, + mimeTypeRegex: videoTorrentImportRegex, + field: 'torrentfile', + maxSize: CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_FILE.FILE_SIZE.max, + optional: true + }) +} + +// --------------------------------------------------------------------------- + +export { + isVideoImportStateValid, + isVideoImportTargetUrlValid, + isVideoImportTorrentFile +} diff --git a/server/server/helpers/custom-validators/video-lives.ts b/server/server/helpers/custom-validators/video-lives.ts new file mode 100644 index 000000000..1a9268113 --- /dev/null +++ b/server/server/helpers/custom-validators/video-lives.ts @@ -0,0 +1,11 @@ +import { LiveVideoLatencyMode } from '@peertube/peertube-models' + +function isLiveLatencyModeValid (value: any) { + return [ LiveVideoLatencyMode.DEFAULT, LiveVideoLatencyMode.SMALL_LATENCY, LiveVideoLatencyMode.HIGH_LATENCY ].includes(value) +} + +// --------------------------------------------------------------------------- + +export { + isLiveLatencyModeValid +} diff --git a/server/server/helpers/custom-validators/video-ownership.ts b/server/server/helpers/custom-validators/video-ownership.ts new file mode 100644 index 000000000..5961ed97c --- /dev/null +++ b/server/server/helpers/custom-validators/video-ownership.ts @@ -0,0 +1,20 @@ +import { Response } from 'express' +import { HttpStatusCode } from '@peertube/peertube-models' +import { MUserId } from '@server/types/models/index.js' +import { MVideoChangeOwnershipFull } from '@server/types/models/video/video-change-ownership.js' + +function checkUserCanTerminateOwnershipChange (user: MUserId, videoChangeOwnership: MVideoChangeOwnershipFull, res: Response) { + if (videoChangeOwnership.NextOwner.userId === user.id) { + return true + } + + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Cannot terminate an ownership change of another user' + }) + return false +} + +export { + checkUserCanTerminateOwnershipChange +} diff --git a/server/server/helpers/custom-validators/video-playlists.ts b/server/server/helpers/custom-validators/video-playlists.ts new file mode 100644 index 000000000..f3b30bf50 --- /dev/null +++ b/server/server/helpers/custom-validators/video-playlists.ts @@ -0,0 +1,35 @@ +import { exists } from './misc.js' +import validator from 'validator' +import { CONSTRAINTS_FIELDS, VIDEO_PLAYLIST_PRIVACIES, VIDEO_PLAYLIST_TYPES } from '../../initializers/constants.js' + +const PLAYLISTS_CONSTRAINT_FIELDS = CONSTRAINTS_FIELDS.VIDEO_PLAYLISTS + +function isVideoPlaylistNameValid (value: any) { + return exists(value) && validator.default.isLength(value, PLAYLISTS_CONSTRAINT_FIELDS.NAME) +} + +function isVideoPlaylistDescriptionValid (value: any) { + return value === null || (exists(value) && validator.default.isLength(value, PLAYLISTS_CONSTRAINT_FIELDS.DESCRIPTION)) +} + +function isVideoPlaylistPrivacyValid (value: number) { + return validator.default.isInt(value + '') && VIDEO_PLAYLIST_PRIVACIES[value] !== undefined +} + +function isVideoPlaylistTimestampValid (value: any) { + return value === null || (exists(value) && validator.default.isInt('' + value, { min: 0 })) +} + +function isVideoPlaylistTypeValid (value: any) { + return exists(value) && VIDEO_PLAYLIST_TYPES[value] !== undefined +} + +// --------------------------------------------------------------------------- + +export { + isVideoPlaylistNameValid, + isVideoPlaylistDescriptionValid, + isVideoPlaylistPrivacyValid, + isVideoPlaylistTimestampValid, + isVideoPlaylistTypeValid +} diff --git a/server/helpers/custom-validators/video-rates.ts b/server/server/helpers/custom-validators/video-rates.ts similarity index 100% rename from server/helpers/custom-validators/video-rates.ts rename to server/server/helpers/custom-validators/video-rates.ts diff --git a/server/server/helpers/custom-validators/video-redundancies.ts b/server/server/helpers/custom-validators/video-redundancies.ts new file mode 100644 index 000000000..91ab35f4c --- /dev/null +++ b/server/server/helpers/custom-validators/video-redundancies.ts @@ -0,0 +1,12 @@ +import { exists } from './misc.js' + +function isVideoRedundancyTarget (value: any) { + return exists(value) && + (value === 'my-videos' || value === 'remote-videos') +} + +// --------------------------------------------------------------------------- + +export { + isVideoRedundancyTarget +} diff --git a/server/server/helpers/custom-validators/video-stats.ts b/server/server/helpers/custom-validators/video-stats.ts new file mode 100644 index 000000000..7bd30c8e2 --- /dev/null +++ b/server/server/helpers/custom-validators/video-stats.ts @@ -0,0 +1,16 @@ +import { VideoStatsTimeserieMetric } from '@peertube/peertube-models' + +const validMetrics = new Set([ + 'viewers', + 'aggregateWatchTime' +]) + +function isValidStatTimeserieMetric (value: VideoStatsTimeserieMetric) { + return validMetrics.has(value) +} + +// --------------------------------------------------------------------------- + +export { + isValidStatTimeserieMetric +} diff --git a/server/server/helpers/custom-validators/video-studio.ts b/server/server/helpers/custom-validators/video-studio.ts new file mode 100644 index 000000000..de9467320 --- /dev/null +++ b/server/server/helpers/custom-validators/video-studio.ts @@ -0,0 +1,53 @@ +import validator from 'validator' +import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js' +import { buildTaskFileFieldname } from '@server/lib/video-studio.js' +import { VideoStudioTask } from '@peertube/peertube-models' +import { isArray } from './misc.js' +import { isVideoFileMimeTypeValid, isVideoImageValid } from './videos.js' +import { forceNumber } from '@peertube/peertube-core-utils' + +function isValidStudioTasksArray (tasks: any) { + if (!isArray(tasks)) return false + + return tasks.length >= CONSTRAINTS_FIELDS.VIDEO_STUDIO.TASKS.min && + tasks.length <= CONSTRAINTS_FIELDS.VIDEO_STUDIO.TASKS.max +} + +function isStudioCutTaskValid (task: VideoStudioTask) { + if (task.name !== 'cut') return false + if (!task.options) return false + + const { start, end } = task.options + if (!start && !end) return false + + if (start && !validator.default.isInt(start + '', CONSTRAINTS_FIELDS.VIDEO_STUDIO.CUT_TIME)) return false + if (end && !validator.default.isInt(end + '', CONSTRAINTS_FIELDS.VIDEO_STUDIO.CUT_TIME)) return false + + if (!start || !end) return true + + return forceNumber(start) < forceNumber(end) +} + +function isStudioTaskAddIntroOutroValid (task: VideoStudioTask, indice: number, files: Express.Multer.File[]) { + const file = files.find(f => f.fieldname === buildTaskFileFieldname(indice, 'file')) + + return (task.name === 'add-intro' || task.name === 'add-outro') && + file && isVideoFileMimeTypeValid([ file ], null) +} + +function isStudioTaskAddWatermarkValid (task: VideoStudioTask, indice: number, files: Express.Multer.File[]) { + const file = files.find(f => f.fieldname === buildTaskFileFieldname(indice, 'file')) + + return task.name === 'add-watermark' && + file && isVideoImageValid([ file ], null, true) +} + +// --------------------------------------------------------------------------- + +export { + isValidStudioTasksArray, + + isStudioCutTaskValid, + isStudioTaskAddIntroOutroValid, + isStudioTaskAddWatermarkValid +} diff --git a/server/server/helpers/custom-validators/video-transcoding.ts b/server/server/helpers/custom-validators/video-transcoding.ts new file mode 100644 index 000000000..aa09d273e --- /dev/null +++ b/server/server/helpers/custom-validators/video-transcoding.ts @@ -0,0 +1,12 @@ +import { exists } from './misc.js' + +function isValidCreateTranscodingType (value: any) { + return exists(value) && + (value === 'hls' || value === 'webtorrent' || value === 'web-video') // TODO: remove webtorrent in v7 +} + +// --------------------------------------------------------------------------- + +export { + isValidCreateTranscodingType +} diff --git a/server/server/helpers/custom-validators/video-view.ts b/server/server/helpers/custom-validators/video-view.ts new file mode 100644 index 000000000..2edd47ec9 --- /dev/null +++ b/server/server/helpers/custom-validators/video-view.ts @@ -0,0 +1,12 @@ +import { exists } from './misc.js' + +function isVideoTimeValid (value: number, videoDuration?: number) { + if (value < 0) return false + if (exists(videoDuration) && value > videoDuration) return false + + return true +} + +export { + isVideoTimeValid +} diff --git a/server/server/helpers/custom-validators/videos.ts b/server/server/helpers/custom-validators/videos.ts new file mode 100644 index 000000000..0dede6c85 --- /dev/null +++ b/server/server/helpers/custom-validators/videos.ts @@ -0,0 +1,218 @@ +import { Request, Response, UploadFilesForCheck } from 'express' +import { decode as magnetUriDecode } from 'magnet-uri' +import validator from 'validator' +import { HttpStatusCode, VideoIncludeType, VideoPrivacy, VideoPrivacyType, VideoRateType } from '@peertube/peertube-models' +import { getVideoWithAttributes } from '@server/helpers/video.js' +import { + CONSTRAINTS_FIELDS, + MIMETYPES, + VIDEO_CATEGORIES, + VIDEO_LICENCES, + VIDEO_LIVE, + VIDEO_PRIVACIES, + VIDEO_RATE_TYPES, + VIDEO_STATES +} from '../../initializers/constants.js' +import { exists, isArray, isDateValid, isFileValid } from './misc.js' + +const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS + +function isVideoIncludeValid (include: VideoIncludeType) { + return exists(include) && validator.default.isInt('' + include) +} + +function isVideoCategoryValid (value: any) { + return value === null || VIDEO_CATEGORIES[value] !== undefined +} + +function isVideoStateValid (value: any) { + return exists(value) && VIDEO_STATES[value] !== undefined +} + +function isVideoLicenceValid (value: any) { + return value === null || VIDEO_LICENCES[value] !== undefined +} + +function isVideoLanguageValid (value: any) { + return value === null || + (typeof value === 'string' && validator.default.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.LANGUAGE)) +} + +function isVideoDurationValid (value: string) { + return exists(value) && validator.default.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DURATION) +} + +function isVideoDescriptionValid (value: string) { + return value === null || (exists(value) && validator.default.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.DESCRIPTION)) +} + +function isVideoSupportValid (value: string) { + return value === null || (exists(value) && validator.default.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.SUPPORT)) +} + +function isVideoNameValid (value: string) { + return exists(value) && validator.default.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.NAME) +} + +function isVideoTagValid (tag: string) { + return exists(tag) && validator.default.isLength(tag, VIDEOS_CONSTRAINTS_FIELDS.TAG) +} + +function areVideoTagsValid (tags: string[]) { + return tags === null || ( + isArray(tags) && + validator.default.isInt(tags.length.toString(), VIDEOS_CONSTRAINTS_FIELDS.TAGS) && + tags.every(tag => isVideoTagValid(tag)) + ) +} + +function isVideoViewsValid (value: string) { + return exists(value) && validator.default.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.VIEWS) +} + +const ratingTypes = new Set(Object.values(VIDEO_RATE_TYPES)) +function isVideoRatingTypeValid (value: string) { + return value === 'none' || ratingTypes.has(value as VideoRateType) +} + +function isVideoFileExtnameValid (value: string) { + return exists(value) && (value === VIDEO_LIVE.EXTENSION || MIMETYPES.VIDEO.EXT_MIMETYPE[value] !== undefined) +} + +function isVideoFileMimeTypeValid (files: UploadFilesForCheck, field = 'videofile') { + return isFileValid({ + files, + mimeTypeRegex: MIMETYPES.VIDEO.MIMETYPES_REGEX, + field, + maxSize: null + }) +} + +const videoImageTypes = CONSTRAINTS_FIELDS.VIDEOS.IMAGE.EXTNAME + .map(v => v.replace('.', '')) + .join('|') +const videoImageTypesRegex = `image/(${videoImageTypes})` + +function isVideoImageValid (files: UploadFilesForCheck, field: string, optional = true) { + return isFileValid({ + files, + mimeTypeRegex: videoImageTypesRegex, + field, + maxSize: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max, + optional + }) +} + +function isVideoPrivacyValid (value: number) { + return VIDEO_PRIVACIES[value] !== undefined +} + +function isVideoReplayPrivacyValid (value: number) { + return VIDEO_PRIVACIES[value] !== undefined && value !== VideoPrivacy.PASSWORD_PROTECTED +} + +function isScheduleVideoUpdatePrivacyValid (value: number) { + return value === VideoPrivacy.UNLISTED || value === VideoPrivacy.PUBLIC || value === VideoPrivacy.INTERNAL +} + +function isVideoOriginallyPublishedAtValid (value: string | null) { + return value === null || isDateValid(value) +} + +function isVideoFileInfoHashValid (value: string | null | undefined) { + return exists(value) && validator.default.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.INFO_HASH) +} + +function isVideoFileResolutionValid (value: string) { + return exists(value) && validator.default.isInt(value + '') +} + +function isVideoFPSResolutionValid (value: string) { + return value === null || validator.default.isInt(value + '') +} + +function isVideoFileSizeValid (value: string) { + return exists(value) && validator.default.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.FILE_SIZE) +} + +function isVideoMagnetUriValid (value: string) { + if (!exists(value)) return false + + const parsed = magnetUriDecode(value) + return parsed && isVideoFileInfoHashValid(parsed.infoHash) +} + +function isPasswordValid (password: string) { + return password.length >= CONSTRAINTS_FIELDS.VIDEO_PASSWORD.LENGTH.min && + password.length < CONSTRAINTS_FIELDS.VIDEO_PASSWORD.LENGTH.max +} + +function isValidPasswordProtectedPrivacy (req: Request, res: Response) { + const fail = (message: string) => { + res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message + }) + return false + } + + let privacy: VideoPrivacyType + const video = getVideoWithAttributes(res) + + if (exists(req.body?.privacy)) privacy = req.body.privacy + else if (exists(video?.privacy)) privacy = video.privacy + + if (privacy !== VideoPrivacy.PASSWORD_PROTECTED) return true + + if (!exists(req.body.videoPasswords) && !exists(req.body.passwords)) return fail('Video passwords are missing.') + + const passwords = req.body.videoPasswords || req.body.passwords + + if (passwords.length === 0) return fail('At least one video password is required.') + + if (new Set(passwords).size !== passwords.length) return fail('Duplicate video passwords are not allowed.') + + for (const password of passwords) { + if (typeof password !== 'string') { + return fail('Video password should be a string.') + } + + if (!isPasswordValid(password)) { + return fail('Invalid video password. Password length should be at least 2 characters and no more than 100 characters.') + } + } + + return true +} + +// --------------------------------------------------------------------------- + +export { + isVideoCategoryValid, + isVideoLicenceValid, + isVideoLanguageValid, + isVideoDescriptionValid, + isVideoFileInfoHashValid, + isVideoNameValid, + areVideoTagsValid, + isVideoFPSResolutionValid, + isScheduleVideoUpdatePrivacyValid, + isVideoOriginallyPublishedAtValid, + isVideoMagnetUriValid, + isVideoStateValid, + isVideoIncludeValid, + isVideoViewsValid, + isVideoRatingTypeValid, + isVideoFileExtnameValid, + isVideoFileMimeTypeValid, + isVideoDurationValid, + isVideoTagValid, + isVideoPrivacyValid, + isVideoReplayPrivacyValid, + isVideoFileResolutionValid, + isVideoFileSizeValid, + isVideoImageValid, + isVideoSupportValid, + isPasswordValid, + isValidPasswordProtectedPrivacy +} diff --git a/server/server/helpers/custom-validators/webfinger.ts b/server/server/helpers/custom-validators/webfinger.ts new file mode 100644 index 000000000..ce10c2247 --- /dev/null +++ b/server/server/helpers/custom-validators/webfinger.ts @@ -0,0 +1,21 @@ +import { REMOTE_SCHEME, WEBSERVER } from '../../initializers/constants.js' +import { sanitizeHost } from '../core-utils.js' +import { exists } from './misc.js' + +function isWebfingerLocalResourceValid (value: string) { + if (!exists(value)) return false + if (value.startsWith('acct:') === false) return false + + const actorWithHost = value.substr(5) + const actorParts = actorWithHost.split('@') + if (actorParts.length !== 2) return false + + const host = actorParts[1] + return sanitizeHost(host, REMOTE_SCHEME.HTTP) === WEBSERVER.HOST +} + +// --------------------------------------------------------------------------- + +export { + isWebfingerLocalResourceValid +} diff --git a/server/server/helpers/database-utils.ts b/server/server/helpers/database-utils.ts new file mode 100644 index 000000000..b7276bb84 --- /dev/null +++ b/server/server/helpers/database-utils.ts @@ -0,0 +1,121 @@ +import retry from 'async/retry.js' +import Bluebird from 'bluebird' +import { Transaction } from 'sequelize' +import { Model } from 'sequelize-typescript' +import { sequelizeTypescript } from '@server/initializers/database.js' +import { logger } from './logger.js' + +function retryTransactionWrapper ( + functionToRetry: (arg1: A, arg2: B, arg3: C, arg4: D) => Promise, + arg1: A, + arg2: B, + arg3: C, + arg4: D, +): Promise + +function retryTransactionWrapper ( + functionToRetry: (arg1: A, arg2: B, arg3: C) => Promise, + arg1: A, + arg2: B, + arg3: C +): Promise + +function retryTransactionWrapper ( + functionToRetry: (arg1: A, arg2: B) => Promise, + arg1: A, + arg2: B +): Promise + +function retryTransactionWrapper ( + functionToRetry: (arg1: A) => Promise, + arg1: A +): Promise + +function retryTransactionWrapper ( + functionToRetry: () => Promise | Bluebird +): Promise + +function retryTransactionWrapper ( + functionToRetry: (...args: any[]) => Promise, + ...args: any[] +): Promise { + return transactionRetryer(callback => { + functionToRetry.apply(null, args) + .then((result: T) => callback(null, result)) + .catch(err => callback(err)) + }) + .catch(err => { + logger.warn(`Cannot execute ${functionToRetry.name} with many retries.`, { err }) + throw err + }) +} + +function transactionRetryer (func: (err: any, data: T) => any) { + return new Promise((res, rej) => { + retry( + { + times: 5, + + errorFilter: err => { + const willRetry = (err.name === 'SequelizeDatabaseError') + logger.debug('Maybe retrying the transaction function.', { willRetry, err, tags: [ 'sql', 'retry' ] }) + return willRetry + } + }, + func, + (err, data) => err ? rej(err) : res(data) + ) + }) +} + +function saveInTransactionWithRetries > (model: T) { + return retryTransactionWrapper(() => { + return sequelizeTypescript.transaction(async transaction => { + await model.save({ transaction }) + }) + }) +} + +// --------------------------------------------------------------------------- + +function resetSequelizeInstance (instance: Model) { + return instance.reload() +} + +function filterNonExistingModels ( + fromDatabase: T[], + newModels: T[] +) { + return fromDatabase.filter(f => !newModels.find(newModel => newModel.hasSameUniqueKeysThan(f))) +} + +function deleteAllModels > (models: T[], transaction: Transaction) { + return Promise.all(models.map(f => f.destroy({ transaction }))) +} + +// --------------------------------------------------------------------------- + +function runInReadCommittedTransaction (fn: (t: Transaction) => Promise) { + const options = { isolationLevel: Transaction.ISOLATION_LEVELS.READ_COMMITTED } + + return sequelizeTypescript.transaction(options, t => fn(t)) +} + +function afterCommitIfTransaction (t: Transaction, fn: Function) { + if (t) return t.afterCommit(() => fn()) + + return fn() +} + +// --------------------------------------------------------------------------- + +export { + resetSequelizeInstance, + retryTransactionWrapper, + transactionRetryer, + saveInTransactionWithRetries, + afterCommitIfTransaction, + filterNonExistingModels, + deleteAllModels, + runInReadCommittedTransaction +} diff --git a/server/helpers/debounce.ts b/server/server/helpers/debounce.ts similarity index 100% rename from server/helpers/debounce.ts rename to server/server/helpers/debounce.ts diff --git a/server/server/helpers/decache.ts b/server/server/helpers/decache.ts new file mode 100644 index 000000000..cef3d639c --- /dev/null +++ b/server/server/helpers/decache.ts @@ -0,0 +1,79 @@ +// Thanks: https://github.com/dwyl/decache +// We reuse this file to also uncache plugin base path + +import { Module } from 'module' +import { extname } from 'path' + +function decachePlugin (require: NodeRequire, libraryPath: string) { + const moduleName = find(require, libraryPath) + + if (!moduleName) return + + searchCache(require, moduleName, function (mod) { + delete require.cache[mod.id] + + removeCachedPath(mod.path) + }) +} + +function decacheModule (require: NodeRequire, name: string) { + const moduleName = find(require, name) + + if (!moduleName) return + + searchCache(require, moduleName, function (mod) { + delete require.cache[mod.id] + + removeCachedPath(mod.path) + }) +} + +// --------------------------------------------------------------------------- + +export { + decacheModule, + decachePlugin +} + +// --------------------------------------------------------------------------- + +function find (require: NodeRequire, moduleName: string) { + try { + return require.resolve(moduleName) + } catch { + return '' + } +} + +function searchCache (require: NodeRequire, moduleName: string, callback: (current: NodeModule) => void) { + const resolvedModule = require.resolve(moduleName) + let mod: NodeModule + const visited = {} + + if (resolvedModule && ((mod = require.cache[resolvedModule]) !== undefined)) { + // Recursively go over the results + (function run (current) { + visited[current.id] = true + + current.children.forEach(function (child) { + if (extname(child.filename) !== '.node' && !visited[child.id]) { + run(child) + } + }) + + // Call the specified callback providing the + // found module + callback(current) + })(mod) + } +}; + +function removeCachedPath (pluginPath: string) { + const pathCache = (Module as any)._pathCache as { [ id: string ]: string[] } + + Object.keys(pathCache).forEach(function (cacheKey) { + if (cacheKey.includes(pluginPath)) { + delete pathCache[cacheKey] + } + }) +} diff --git a/server/server/helpers/dns.ts b/server/server/helpers/dns.ts new file mode 100644 index 000000000..52a0654d9 --- /dev/null +++ b/server/server/helpers/dns.ts @@ -0,0 +1,29 @@ +import { lookup } from 'dns' +import ipaddr from 'ipaddr.js' + +function dnsLookupAll (hostname: string) { + return new Promise((res, rej) => { + lookup(hostname, { family: 0, all: true }, (err, adresses) => { + if (err) return rej(err) + + return res(adresses.map(a => a.address)) + }) + }) +} + +async function isResolvingToUnicastOnly (hostname: string) { + const addresses = await dnsLookupAll(hostname) + + for (const address of addresses) { + const parsed = ipaddr.parse(address) + + if (parsed.range() !== 'unicast') return false + } + + return true +} + +export { + dnsLookupAll, + isResolvingToUnicastOnly +} diff --git a/server/server/helpers/express-utils.ts b/server/server/helpers/express-utils.ts new file mode 100644 index 000000000..773cad2b2 --- /dev/null +++ b/server/server/helpers/express-utils.ts @@ -0,0 +1,156 @@ +import express, { RequestHandler } from 'express' +import multer, { diskStorage } from 'multer' +import { getLowercaseExtension } from '@peertube/peertube-node-utils' +import { CONFIG } from '../initializers/config.js' +import { REMOTE_SCHEME } from '../initializers/constants.js' +import { isArray } from './custom-validators/misc.js' +import { logger } from './logger.js' +import { deleteFileAndCatch, generateRandomString } from './utils.js' +import { getExtFromMimetype } from './video.js' + +function buildNSFWFilter (res?: express.Response, paramNSFW?: string) { + if (paramNSFW === 'true') return true + if (paramNSFW === 'false') return false + if (paramNSFW === 'both') return undefined + + if (res?.locals.oauth) { + const user = res.locals.oauth.token.User + + // User does not want NSFW videos + if (user.nsfwPolicy === 'do_not_list') return false + + // Both + return undefined + } + + if (CONFIG.INSTANCE.DEFAULT_NSFW_POLICY === 'do_not_list') return false + + // Display all + return null +} + +function cleanUpReqFiles (req: express.Request) { + const filesObject = req.files + if (!filesObject) return + + if (isArray(filesObject)) { + filesObject.forEach(f => deleteFileAndCatch(f.path)) + return + } + + for (const key of Object.keys(filesObject)) { + const files = filesObject[key] + + files.forEach(f => deleteFileAndCatch(f.path)) + } +} + +function getHostWithPort (host: string) { + const splitted = host.split(':') + + // The port was not specified + if (splitted.length === 1) { + if (REMOTE_SCHEME.HTTP === 'https') return host + ':443' + + return host + ':80' + } + + return host +} + +function createReqFiles ( + fieldNames: string[], + mimeTypes: { [id: string]: string | string[] }, + destination = CONFIG.STORAGE.TMP_DIR +): RequestHandler { + const storage = diskStorage({ + destination: (req, file, cb) => { + cb(null, destination) + }, + + filename: (req, file, cb) => { + return generateReqFilename(file, mimeTypes, cb) + } + }) + + const fields: { name: string, maxCount: number }[] = [] + for (const fieldName of fieldNames) { + fields.push({ + name: fieldName, + maxCount: 1 + }) + } + + return multer({ storage }).fields(fields) +} + +function createAnyReqFiles ( + mimeTypes: { [id: string]: string | string[] }, + fileFilter: (req: express.Request, file: Express.Multer.File, cb: (err: Error, result: boolean) => void) => void +): RequestHandler { + const storage = diskStorage({ + destination: (req, file, cb) => { + cb(null, CONFIG.STORAGE.TMP_DIR) + }, + + filename: (req, file, cb) => { + return generateReqFilename(file, mimeTypes, cb) + } + }) + + return multer({ storage, fileFilter }).any() +} + +function isUserAbleToSearchRemoteURI (res: express.Response) { + const user = res.locals.oauth ? res.locals.oauth.token.User : undefined + + return CONFIG.SEARCH.REMOTE_URI.ANONYMOUS === true || + (CONFIG.SEARCH.REMOTE_URI.USERS === true && user !== undefined) +} + +function getCountVideos (req: express.Request) { + return req.query.skipCount !== true +} + +// --------------------------------------------------------------------------- + +export { + buildNSFWFilter, + getHostWithPort, + createAnyReqFiles, + isUserAbleToSearchRemoteURI, + createReqFiles, + cleanUpReqFiles, + getCountVideos +} + +// --------------------------------------------------------------------------- + +async function generateReqFilename ( + file: Express.Multer.File, + mimeTypes: { [id: string]: string | string[] }, + cb: (err: Error, name: string) => void +) { + let extension: string + const fileExtension = getLowercaseExtension(file.originalname) + const extensionFromMimetype = getExtFromMimetype(mimeTypes, file.mimetype) + + // Take the file extension if we don't understand the mime type + if (!extensionFromMimetype) { + extension = fileExtension + } else { + // Take the first available extension for this mimetype + extension = extensionFromMimetype + } + + let randomString = '' + + try { + randomString = await generateRandomString(16) + } catch (err) { + logger.error('Cannot generate random string for file name.', { err }) + randomString = 'fake-random-string' + } + + cb(null, randomString + extension) +} diff --git a/server/server/helpers/ffmpeg/codecs.ts b/server/server/helpers/ffmpeg/codecs.ts new file mode 100644 index 000000000..ff98b8f99 --- /dev/null +++ b/server/server/helpers/ffmpeg/codecs.ts @@ -0,0 +1,64 @@ +import { FfprobeData } from 'fluent-ffmpeg' +import { getAudioStream, getVideoStream } from '@peertube/peertube-ffmpeg' +import { logger } from '../logger.js' +import { forceNumber } from '@peertube/peertube-core-utils' + +export async function getVideoStreamCodec (path: string) { + const videoStream = await getVideoStream(path) + if (!videoStream) return '' + + const videoCodec = videoStream.codec_tag_string + + if (videoCodec === 'vp09') return 'vp09.00.50.08' + if (videoCodec === 'hev1') return 'hev1.1.6.L93.B0' + + const baseProfileMatrix = { + avc1: { + High: '6400', + Main: '4D40', + Baseline: '42E0' + }, + av01: { + High: '1', + Main: '0', + Professional: '2' + } + } + + let baseProfile = baseProfileMatrix[videoCodec][videoStream.profile] + if (!baseProfile) { + logger.warn('Cannot get video profile codec of %s.', path, { videoStream }) + baseProfile = baseProfileMatrix[videoCodec]['High'] // Fallback + } + + if (videoCodec === 'av01') { + let level = videoStream.level.toString() + if (level.length === 1) level = `0${level}` + + // Guess the tier indicator and bit depth + return `${videoCodec}.${baseProfile}.${level}M.08` + } + + let level = forceNumber(videoStream.level).toString(16) + if (level.length === 1) level = `0${level}` + + // Default, h264 codec + return `${videoCodec}.${baseProfile}${level}` +} + +export async function getAudioStreamCodec (path: string, existingProbe?: FfprobeData) { + const { audioStream } = await getAudioStream(path, existingProbe) + + if (!audioStream) return '' + + const audioCodecName = audioStream.codec_name + + if (audioCodecName === 'opus') return 'opus' + if (audioCodecName === 'vorbis') return 'vorbis' + if (audioCodecName === 'aac') return 'mp4a.40.2' + if (audioCodecName === 'mp3') return 'mp4a.40.34' + + logger.warn('Cannot get audio codec of %s.', path, { audioStream }) + + return 'mp4a.40.2' // Fallback +} diff --git a/server/server/helpers/ffmpeg/ffmpeg-image.ts b/server/server/helpers/ffmpeg/ffmpeg-image.ts new file mode 100644 index 000000000..1f6c6a3c3 --- /dev/null +++ b/server/server/helpers/ffmpeg/ffmpeg-image.ts @@ -0,0 +1,14 @@ +import { FFmpegImage } from '@peertube/peertube-ffmpeg' +import { getFFmpegCommandWrapperOptions } from './ffmpeg-options.js' + +export function processGIF (options: Parameters[0]) { + return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).processGIF(options) +} + +export function generateThumbnailFromVideo (options: Parameters[0]) { + return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).generateThumbnailFromVideo(options) +} + +export function convertWebPToJPG (options: Parameters[0]) { + return new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')).convertWebPToJPG(options) +} diff --git a/server/server/helpers/ffmpeg/ffmpeg-options.ts b/server/server/helpers/ffmpeg/ffmpeg-options.ts new file mode 100644 index 000000000..ec1b91f86 --- /dev/null +++ b/server/server/helpers/ffmpeg/ffmpeg-options.ts @@ -0,0 +1,45 @@ +import { logger } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' +import { FFMPEG_NICE } from '@server/initializers/constants.js' +import { FFmpegCommandWrapperOptions } from '@peertube/peertube-ffmpeg' +import { AvailableEncoders } from '@peertube/peertube-models' + +type CommandType = 'live' | 'vod' | 'thumbnail' + +export function getFFmpegCommandWrapperOptions (type: CommandType, availableEncoders?: AvailableEncoders): FFmpegCommandWrapperOptions { + return { + availableEncoders, + profile: getProfile(type), + + niceness: FFMPEG_NICE[type.toUpperCase()], + tmpDirectory: CONFIG.STORAGE.TMP_DIR, + threads: getThreads(type), + + logger: { + debug: logger.debug.bind(logger), + info: logger.info.bind(logger), + warn: logger.warn.bind(logger), + error: logger.error.bind(logger) + }, + lTags: { tags: [ 'ffmpeg' ] } + } +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +function getThreads (type: CommandType) { + if (type === 'live') return CONFIG.LIVE.TRANSCODING.THREADS + if (type === 'vod') return CONFIG.TRANSCODING.THREADS + + // Auto + return 0 +} + +function getProfile (type: CommandType) { + if (type === 'live') return CONFIG.LIVE.TRANSCODING.PROFILE + if (type === 'vod') return CONFIG.TRANSCODING.PROFILE + + return undefined +} diff --git a/server/server/helpers/ffmpeg/framerate.ts b/server/server/helpers/ffmpeg/framerate.ts new file mode 100644 index 000000000..8aef6f021 --- /dev/null +++ b/server/server/helpers/ffmpeg/framerate.ts @@ -0,0 +1,43 @@ +import { VIDEO_TRANSCODING_FPS } from '@server/initializers/constants.js' + +export function computeOutputFPS (options: { + inputFPS: number + resolution: number +}) { + const { resolution } = options + + let fps = options.inputFPS + + if ( + // On small/medium resolutions, limit FPS + resolution !== undefined && + resolution < VIDEO_TRANSCODING_FPS.KEEP_ORIGIN_FPS_RESOLUTION_MIN && + fps > VIDEO_TRANSCODING_FPS.AVERAGE + ) { + // Get closest standard framerate by modulo: downsampling has to be done to a divisor of the nominal fps value + fps = getClosestFramerateStandard({ fps, type: 'STANDARD' }) + } + + // Hard FPS limits + if (fps > VIDEO_TRANSCODING_FPS.MAX) fps = getClosestFramerateStandard({ fps, type: 'HD_STANDARD' }) + + if (fps < VIDEO_TRANSCODING_FPS.MIN) { + throw new Error(`Cannot compute FPS because ${fps} is lower than our minimum value ${VIDEO_TRANSCODING_FPS.MIN}`) + } + + return fps +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +function getClosestFramerateStandard (options: { + fps: number + type: 'HD_STANDARD' | 'STANDARD' +}) { + const { fps, type } = options + + return VIDEO_TRANSCODING_FPS[type].slice(0) + .sort((a, b) => fps % a - fps % b)[0] +} diff --git a/server/server/helpers/ffmpeg/index.ts b/server/server/helpers/ffmpeg/index.ts new file mode 100644 index 000000000..69864926e --- /dev/null +++ b/server/server/helpers/ffmpeg/index.ts @@ -0,0 +1,4 @@ +export * from './codecs.js' +export * from './ffmpeg-image.js' +export * from './ffmpeg-options.js' +export * from './framerate.js' diff --git a/server/server/helpers/geo-ip.ts b/server/server/helpers/geo-ip.ts new file mode 100644 index 000000000..6a9cd2124 --- /dev/null +++ b/server/server/helpers/geo-ip.ts @@ -0,0 +1,79 @@ +import { pathExists } from 'fs-extra/esm' +import { writeFile } from 'fs/promises' +import maxmind, { CountryResponse, Reader } from 'maxmind' +import { join } from 'path' +import { CONFIG } from '@server/initializers/config.js' +import { logger, loggerTagsFactory } from './logger.js' +import { isBinaryResponse, peertubeGot } from './requests.js' + +const lTags = loggerTagsFactory('geo-ip') + +const mmbdFilename = 'dbip-country-lite-latest.mmdb' +const mmdbPath = join(CONFIG.STORAGE.BIN_DIR, mmbdFilename) + +export class GeoIP { + private static instance: GeoIP + + private reader: Reader + + private constructor () { + } + + async safeCountryISOLookup (ip: string): Promise { + if (CONFIG.GEO_IP.ENABLED === false) return null + + await this.initReaderIfNeeded() + + try { + const result = this.reader.get(ip) + if (!result) return null + + return result.country.iso_code + } catch (err) { + logger.error('Cannot get country from IP.', { err }) + + return null + } + } + + async updateDatabase () { + if (CONFIG.GEO_IP.ENABLED === false) return + + const url = CONFIG.GEO_IP.COUNTRY.DATABASE_URL + + logger.info('Updating GeoIP database from %s.', url, lTags()) + + const gotOptions = { context: { bodyKBLimit: 200_000 }, responseType: 'buffer' as 'buffer' } + + try { + const gotResult = await peertubeGot(url, gotOptions) + + if (!isBinaryResponse(gotResult)) { + throw new Error('Not a binary response') + } + + await writeFile(mmdbPath, gotResult.body) + + // Reinit reader + this.reader = undefined + + logger.info('GeoIP database updated %s.', mmdbPath, lTags()) + } catch (err) { + logger.error('Cannot update GeoIP database from %s.', url, { err, ...lTags() }) + } + } + + private async initReaderIfNeeded () { + if (!this.reader) { + if (!await pathExists(mmdbPath)) { + await this.updateDatabase() + } + + this.reader = await maxmind.open(mmdbPath) + } + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/server/helpers/image-utils.ts b/server/server/helpers/image-utils.ts new file mode 100644 index 000000000..7c732235d --- /dev/null +++ b/server/server/helpers/image-utils.ts @@ -0,0 +1,184 @@ +import { copy, remove } from 'fs-extra/esm' +import { readFile, rename } from 'fs/promises' +import { join } from 'path' +import { ColorActionName } from '@jimp/plugin-color' +import { buildUUID, getLowercaseExtension } from '@peertube/peertube-node-utils' +import { convertWebPToJPG, generateThumbnailFromVideo, processGIF } from './ffmpeg/index.js' +import { logger, loggerTagsFactory } from './logger.js' + +import type Jimp from 'jimp' + +const lTags = loggerTagsFactory('image-utils') + +function generateImageFilename (extension = '.jpg') { + return buildUUID() + extension +} + +async function processImage (options: { + path: string + destination: string + newSize: { width: number, height: number } + keepOriginal?: boolean // default false +}) { + const { path, destination, newSize, keepOriginal = false } = options + + const extension = getLowercaseExtension(path) + + if (path === destination) { + throw new Error('Jimp/FFmpeg needs an input path different that the output path.') + } + + logger.debug('Processing image %s to %s.', path, destination) + + // Use FFmpeg to process GIF + if (extension === '.gif') { + await processGIF({ path, destination, newSize }) + } else { + await jimpProcessor(path, destination, newSize, extension) + } + + if (keepOriginal !== true) await remove(path) +} + +async function generateImageFromVideoFile (options: { + fromPath: string + folder: string + imageName: string + size: { width: number, height: number } +}) { + const { fromPath, folder, imageName, size } = options + + const pendingImageName = 'pending-' + imageName + const pendingImagePath = join(folder, pendingImageName) + + try { + await generateThumbnailFromVideo({ fromPath, output: pendingImagePath }) + + const destination = join(folder, imageName) + await processImage({ path: pendingImagePath, destination, newSize: size }) + } catch (err) { + logger.error('Cannot generate image from video %s.', fromPath, { err, ...lTags() }) + + try { + await remove(pendingImagePath) + } catch (err) { + logger.debug('Cannot remove pending image path after generation error.', { err, ...lTags() }) + } + + throw err + } +} + +async function getImageSize (path: string) { + const inputBuffer = await readFile(path) + + const Jimp = await import('jimp') + + const image = await Jimp.default.read(inputBuffer) + + return { + width: image.getWidth(), + height: image.getHeight() + } +} + +// --------------------------------------------------------------------------- + +export { + generateImageFilename, + generateImageFromVideoFile, + + processImage, + + getImageSize +} + +// --------------------------------------------------------------------------- + +async function jimpProcessor (path: string, destination: string, newSize: { width: number, height: number }, inputExt: string) { + let sourceImage: Jimp + const inputBuffer = await readFile(path) + + const Jimp = await import('jimp') + + try { + sourceImage = await Jimp.default.read(inputBuffer) + } catch (err) { + logger.debug('Cannot read %s with jimp. Try to convert the image using ffmpeg first.', path, { err }) + + const newName = path + '.jpg' + await convertWebPToJPG({ path, destination: newName }) + await rename(newName, path) + + sourceImage = await Jimp.default.read(path) + } + + await remove(destination) + + // Optimization if the source file has the appropriate size + const outputExt = getLowercaseExtension(destination) + if (skipProcessing({ sourceImage, newSize, imageBytes: inputBuffer.byteLength, inputExt, outputExt })) { + return copy(path, destination) + } + + await autoResize({ sourceImage, newSize, destination }) +} + +async function autoResize (options: { + sourceImage: Jimp + newSize: { width: number, height: number } + destination: string +}) { + const { sourceImage, newSize, destination } = options + + // Portrait mode targeting a landscape, apply some effect on the image + const sourceIsPortrait = sourceImage.getWidth() < sourceImage.getHeight() + const destIsPortraitOrSquare = newSize.width <= newSize.height + + removeExif(sourceImage) + + if (sourceIsPortrait && !destIsPortraitOrSquare) { + const baseImage = sourceImage.cloneQuiet().cover(newSize.width, newSize.height) + .color([ { apply: ColorActionName.SHADE, params: [ 50 ] } ]) + + const topImage = sourceImage.cloneQuiet().contain(newSize.width, newSize.height) + + return write(baseImage.blit(topImage, 0, 0), destination) + } + + return write(sourceImage.cover(newSize.width, newSize.height), destination) +} + +function write (image: Jimp, destination: string) { + return image.quality(80).writeAsync(destination) +} + +function skipProcessing (options: { + sourceImage: Jimp + newSize: { width: number, height: number } + imageBytes: number + inputExt: string + outputExt: string +}) { + const { sourceImage, newSize, imageBytes, inputExt, outputExt } = options + const { width, height } = newSize + + if (hasExif(sourceImage)) return false + if (sourceImage.getWidth() > width || sourceImage.getHeight() > height) return false + if (inputExt !== outputExt) return false + + const kB = 1000 + + if (height >= 1000) return imageBytes <= 200 * kB + if (height >= 500) return imageBytes <= 100 * kB + + return imageBytes <= 15 * kB +} + +function hasExif (image: Jimp) { + return !!(image.bitmap as any).exifBuffer +} + +function removeExif (image: Jimp) { + (image.bitmap as any).exifBuffer = null +} diff --git a/server/server/helpers/logger.ts b/server/server/helpers/logger.ts new file mode 100644 index 000000000..1379d4864 --- /dev/null +++ b/server/server/helpers/logger.ts @@ -0,0 +1,208 @@ +import { stat } from 'fs/promises' +import { join } from 'path' +import { format as sqlFormat } from 'sql-formatter' +import { createLogger, format, transports } from 'winston' +import { FileTransportOptions } from 'winston/lib/winston/transports' +import { context } from '@opentelemetry/api' +import { getSpanContext } from '@opentelemetry/api/build/src/trace/context-utils.js' +import { omit } from '@peertube/peertube-core-utils' +import { CONFIG } from '../initializers/config.js' +import { LOG_FILENAME } from '../initializers/constants.js' + +const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT + +const consoleLoggerFormat = format.printf(info => { + let additionalInfos = JSON.stringify(getAdditionalInfo(info), removeCyclicValues(), 2) + + if (additionalInfos === undefined || additionalInfos === '{}') additionalInfos = '' + else additionalInfos = ' ' + additionalInfos + + if (info.sql) { + if (CONFIG.LOG.PRETTIFY_SQL) { + additionalInfos += '\n' + sqlFormat(info.sql, { + language: 'sql', + tabWidth: 2 + }) + } else { + additionalInfos += ' - ' + info.sql + } + } + + return `[${info.label}] ${info.timestamp} ${info.level}: ${info.message}${additionalInfos}` +}) + +const jsonLoggerFormat = format.printf(info => { + return JSON.stringify(info, removeCyclicValues()) +}) + +const timestampFormatter = format.timestamp({ + format: 'YYYY-MM-DD HH:mm:ss.SSS' +}) +const labelFormatter = (suffix?: string) => { + return format.label({ + label: suffix ? `${label} ${suffix}` : label + }) +} + +const fileLoggerOptions: FileTransportOptions = { + filename: join(CONFIG.STORAGE.LOG_DIR, LOG_FILENAME), + handleExceptions: true, + format: format.combine( + format.timestamp(), + jsonLoggerFormat + ) +} + +if (CONFIG.LOG.ROTATION.ENABLED) { + fileLoggerOptions.maxsize = CONFIG.LOG.ROTATION.MAX_FILE_SIZE + fileLoggerOptions.maxFiles = CONFIG.LOG.ROTATION.MAX_FILES +} + +function buildLogger (labelSuffix?: string) { + return createLogger({ + level: CONFIG.LOG.LEVEL, + defaultMeta: { + get traceId () { return getSpanContext(context.active())?.traceId }, + get spanId () { return getSpanContext(context.active())?.spanId }, + get traceFlags () { return getSpanContext(context.active())?.traceFlags } + }, + format: format.combine( + labelFormatter(labelSuffix), + format.splat() + ), + transports: [ + new transports.File(fileLoggerOptions), + new transports.Console({ + handleExceptions: true, + format: format.combine( + timestampFormatter, + format.colorize(), + consoleLoggerFormat + ) + }) + ], + exitOnError: true + }) +} + +const logger = buildLogger() + +// --------------------------------------------------------------------------- + +function bunyanLogFactory (level: string) { + return function (...params: any[]) { + let meta = null + let args = [].concat(params) + + if (arguments[0] instanceof Error) { + meta = arguments[0].toString() + args = Array.prototype.slice.call(arguments, 1) + args.push(meta) + } else if (typeof (args[0]) !== 'string') { + meta = arguments[0] + args = Array.prototype.slice.call(arguments, 1) + args.push(meta) + } + + logger[level].apply(logger, args) + } +} + +const bunyanLogger = { + level: () => { }, + trace: bunyanLogFactory('debug'), + debug: bunyanLogFactory('debug'), + verbose: bunyanLogFactory('debug'), + info: bunyanLogFactory('info'), + warn: bunyanLogFactory('warn'), + error: bunyanLogFactory('error'), + fatal: bunyanLogFactory('error') +} + +// --------------------------------------------------------------------------- + +type LoggerTagsFn = (...tags: string[]) => { tags: string[] } +function loggerTagsFactory (...defaultTags: string[]): LoggerTagsFn { + return (...tags: string[]) => { + return { tags: defaultTags.concat(tags) } + } +} + +// --------------------------------------------------------------------------- + +async function mtimeSortFilesDesc (files: string[], basePath: string) { + const promises = [] + const out: { file: string, mtime: number }[] = [] + + for (const file of files) { + const p = stat(basePath + '/' + file) + .then(stats => { + if (stats.isFile()) out.push({ file, mtime: stats.mtime.getTime() }) + }) + + promises.push(p) + } + + await Promise.all(promises) + + out.sort((a, b) => b.mtime - a.mtime) + + return out +} + +// --------------------------------------------------------------------------- + +export { + type LoggerTagsFn, + + buildLogger, + timestampFormatter, + labelFormatter, + consoleLoggerFormat, + jsonLoggerFormat, + mtimeSortFilesDesc, + logger, + loggerTagsFactory, + bunyanLogger +} + +// --------------------------------------------------------------------------- + +function removeCyclicValues () { + const seen = new WeakSet() + + // Thanks: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Cyclic_object_value#Examples + return (key: string, value: any) => { + if (key === 'cert') return 'Replaced by the logger to avoid large log message' + + if (typeof value === 'object' && value !== null) { + if (seen.has(value)) return + + seen.add(value) + } + + if (value instanceof Set) { + return Array.from(value) + } + + if (value instanceof Map) { + return Array.from(value.entries()) + } + + if (value instanceof Error) { + const error = {} + + Object.getOwnPropertyNames(value).forEach(key => { error[key] = value[key] }) + + return error + } + + return value + } +} + +function getAdditionalInfo (info: any) { + const toOmit = [ 'label', 'timestamp', 'level', 'message', 'sql', 'tags' ] + + return omit(info, toOmit) +} diff --git a/server/server/helpers/markdown.ts b/server/server/helpers/markdown.ts new file mode 100644 index 000000000..e38e55502 --- /dev/null +++ b/server/server/helpers/markdown.ts @@ -0,0 +1,89 @@ +import MarkdownItClass from 'markdown-it' +import markdownItEmoji from 'markdown-it-emoji/light.js' +import sanitizeHtml from 'sanitize-html' +import { getDefaultSanitizeOptions, getTextOnlySanitizeOptions, TEXT_WITH_HTML_RULES } from '@peertube/peertube-core-utils' + +const defaultSanitizeOptions = getDefaultSanitizeOptions() +const textOnlySanitizeOptions = getTextOnlySanitizeOptions() + +const markdownItForSafeHtml = new MarkdownItClass('default', { linkify: true, breaks: true, html: true }) + .enable(TEXT_WITH_HTML_RULES) + .use(markdownItEmoji) + +const markdownItForPlainText = new MarkdownItClass('default', { linkify: false, breaks: true, html: false }) + .use(markdownItEmoji) + .use(plainTextPlugin) + +const toSafeHtml = (text: string) => { + if (!text) return '' + + // Restore line feed + const textWithLineFeed = text.replace(//g, '\r\n') + + // Convert possible markdown (emojis, emphasis and lists) to html + const html = markdownItForSafeHtml.render(textWithLineFeed) + + // Convert to safe Html + return sanitizeHtml(html, defaultSanitizeOptions) +} + +const mdToOneLinePlainText = (text: string) => { + if (!text) return '' + + markdownItForPlainText.render(text) + + // Convert to safe Html + return sanitizeHtml(markdownItForPlainText.plainText, textOnlySanitizeOptions) +} + +// --------------------------------------------------------------------------- + +export { + toSafeHtml, + mdToOneLinePlainText +} + +// --------------------------------------------------------------------------- + +// Thanks: https://github.com/wavesheep/markdown-it-plain-text +function plainTextPlugin (markdownIt: any) { + function plainTextRule (state: any) { + const text = scan(state.tokens) + + markdownIt.plainText = text + } + + function scan (tokens: any[]) { + let lastSeparator = '' + let text = '' + + function buildSeparator (token: any) { + if (token.type === 'list_item_close') { + lastSeparator = ', ' + } + + if (token.tag === 'br' || token.type === 'paragraph_close') { + lastSeparator = ' ' + } + } + + for (const token of tokens) { + buildSeparator(token) + + if (token.type !== 'inline') continue + + for (const child of token.children) { + buildSeparator(child) + + if (!child.content) continue + + text += lastSeparator + child.content + lastSeparator = '' + } + } + + return text + } + + markdownIt.core.ruler.push('plainText', plainTextRule) +} diff --git a/server/helpers/memoize.ts b/server/server/helpers/memoize.ts similarity index 100% rename from server/helpers/memoize.ts rename to server/server/helpers/memoize.ts diff --git a/server/server/helpers/mentions.ts b/server/server/helpers/mentions.ts new file mode 100644 index 000000000..238e468ea --- /dev/null +++ b/server/server/helpers/mentions.ts @@ -0,0 +1,42 @@ +import { uniqify } from '@peertube/peertube-core-utils' +import { WEBSERVER } from '@server/initializers/constants.js' +import { actorNameAlphabet } from './custom-validators/activitypub/actor.js' +import { regexpCapture } from './regexp.js' + +export function extractMentions (text: string, isOwned: boolean) { + let result: string[] = [] + + const localMention = `@(${actorNameAlphabet}+)` + const remoteMention = `${localMention}@${WEBSERVER.HOST}` + + const mentionRegex = isOwned + ? '(?:(?:' + remoteMention + ')|(?:' + localMention + '))' // Include local mentions? + : '(?:' + remoteMention + ')' + + const firstMentionRegex = new RegExp(`^${mentionRegex} `, 'g') + const endMentionRegex = new RegExp(` ${mentionRegex}$`, 'g') + const remoteMentionsRegex = new RegExp(' ' + remoteMention + ' ', 'g') + + result = result.concat( + regexpCapture(text, firstMentionRegex) + .map(([ , username1, username2 ]) => username1 || username2), + + regexpCapture(text, endMentionRegex) + .map(([ , username1, username2 ]) => username1 || username2), + + regexpCapture(text, remoteMentionsRegex) + .map(([ , username ]) => username) + ) + + // Include local mentions + if (isOwned) { + const localMentionsRegex = new RegExp(' ' + localMention + ' ', 'g') + + result = result.concat( + regexpCapture(text, localMentionsRegex) + .map(([ , username ]) => username) + ) + } + + return uniqify(result) +} diff --git a/server/server/helpers/otp.ts b/server/server/helpers/otp.ts new file mode 100644 index 000000000..673a043f6 --- /dev/null +++ b/server/server/helpers/otp.ts @@ -0,0 +1,58 @@ +import { Secret, TOTP } from 'otpauth' +import { CONFIG } from '@server/initializers/config.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { decrypt } from './peertube-crypto.js' + +async function isOTPValid (options: { + encryptedSecret: string + token: string +}) { + const { token, encryptedSecret } = options + + const secret = await decrypt(encryptedSecret, CONFIG.SECRETS.PEERTUBE) + + const totp = new TOTP({ + ...baseOTPOptions(), + + secret + }) + + const delta = totp.validate({ + token, + window: 1 + }) + + if (delta === null) return false + + return true +} + +function generateOTPSecret (email: string) { + const totp = new TOTP({ + ...baseOTPOptions(), + + label: email, + secret: new Secret() + }) + + return { + secret: totp.secret.base32, + uri: totp.toString() + } +} + +export { + isOTPValid, + generateOTPSecret +} + +// --------------------------------------------------------------------------- + +function baseOTPOptions () { + return { + issuer: WEBSERVER.HOST, + algorithm: 'SHA1', + digits: 6, + period: 30 + } +} diff --git a/server/server/helpers/peertube-crypto.ts b/server/server/helpers/peertube-crypto.ts new file mode 100644 index 000000000..45f2da6bf --- /dev/null +++ b/server/server/helpers/peertube-crypto.ts @@ -0,0 +1,208 @@ +import httpSignature from '@peertube/http-signature' +import { sha256 } from '@peertube/peertube-node-utils' +import { createCipheriv, createDecipheriv, createSign, createVerify } from 'crypto' +import { Request } from 'express' +import cloneDeep from 'lodash-es/cloneDeep.js' +import { BCRYPT_SALT_SIZE, ENCRYPTION, HTTP_SIGNATURE, PRIVATE_RSA_KEY_SIZE } from '../initializers/constants.js' +import { MActor } from '../types/models/index.js' +import { generateRSAKeyPairPromise, randomBytesPromise, scryptPromise } from './core-utils.js' +import { logger } from './logger.js' + +function createPrivateAndPublicKeys () { + logger.info('Generating a RSA key...') + + return generateRSAKeyPairPromise(PRIVATE_RSA_KEY_SIZE) +} + +// --------------------------------------------------------------------------- +// User password checks +// --------------------------------------------------------------------------- + +async function comparePassword (plainPassword: string, hashPassword: string) { + if (!plainPassword) return false + + const { compare } = await import('bcrypt') + + return compare(plainPassword, hashPassword) +} + +async function cryptPassword (password: string) { + const { genSalt, hash } = await import('bcrypt') + + const salt = await genSalt(BCRYPT_SALT_SIZE) + + return hash(password, salt) +} + +// --------------------------------------------------------------------------- +// HTTP Signature +// --------------------------------------------------------------------------- + +function isHTTPSignatureDigestValid (rawBody: Buffer, req: Request): boolean { + if (req.headers[HTTP_SIGNATURE.HEADER_NAME] && req.headers['digest']) { + return buildDigest(rawBody.toString()) === req.headers['digest'] + } + + return true +} + +function isHTTPSignatureVerified (httpSignatureParsed: any, actor: MActor): boolean { + return httpSignature.verifySignature(httpSignatureParsed, actor.publicKey) === true +} + +function parseHTTPSignature (req: Request, clockSkew?: number) { + const requiredHeaders = req.method === 'POST' + ? [ '(request-target)', 'host', 'digest' ] + : [ '(request-target)', 'host' ] + + const parsed = httpSignature.parse(req, { clockSkew, headers: requiredHeaders }) + + const parsedHeaders = parsed.params.headers + if (!parsedHeaders.includes('date') && !parsedHeaders.includes('(created)')) { + throw new Error(`date or (created) must be included in signature`) + } + + return parsed +} + +// --------------------------------------------------------------------------- +// JSONLD +// --------------------------------------------------------------------------- + +function isJsonLDSignatureVerified (fromActor: MActor, signedDocument: any): Promise { + if (signedDocument.signature.type === 'RsaSignature2017') { + return isJsonLDRSA2017Verified(fromActor, signedDocument) + } + + logger.warn('Unknown JSON LD signature %s.', signedDocument.signature.type, signedDocument) + + return Promise.resolve(false) +} + +// Backward compatibility with "other" implementations +async function isJsonLDRSA2017Verified (fromActor: MActor, signedDocument: any) { + const [ documentHash, optionsHash ] = await Promise.all([ + createDocWithoutSignatureHash(signedDocument), + createSignatureHash(signedDocument.signature) + ]) + + const toVerify = optionsHash + documentHash + + const verify = createVerify('RSA-SHA256') + verify.update(toVerify, 'utf8') + + return verify.verify(fromActor.publicKey, signedDocument.signature.signatureValue, 'base64') +} + +async function signJsonLDObject (byActor: { url: string, privateKey: string }, data: T) { + const signature = { + type: 'RsaSignature2017', + creator: byActor.url, + created: new Date().toISOString() + } + + const [ documentHash, optionsHash ] = await Promise.all([ + createDocWithoutSignatureHash(data), + createSignatureHash(signature) + ]) + + const toSign = optionsHash + documentHash + + const sign = createSign('RSA-SHA256') + sign.update(toSign, 'utf8') + + const signatureValue = sign.sign(byActor.privateKey, 'base64') + Object.assign(signature, { signatureValue }) + + return Object.assign(data, { signature }) +} + +// --------------------------------------------------------------------------- + +function buildDigest (body: any) { + const rawBody = typeof body === 'string' ? body : JSON.stringify(body) + + return 'SHA-256=' + sha256(rawBody, 'base64') +} + +// --------------------------------------------------------------------------- +// Encryption +// --------------------------------------------------------------------------- + +async function encrypt (str: string, secret: string) { + const iv = await randomBytesPromise(ENCRYPTION.IV) + + const key = await scryptPromise(secret, ENCRYPTION.SALT, 32) + const cipher = createCipheriv(ENCRYPTION.ALGORITHM, key, iv) + + let encrypted = iv.toString(ENCRYPTION.ENCODING) + ':' + encrypted += cipher.update(str, 'utf8', ENCRYPTION.ENCODING) + encrypted += cipher.final(ENCRYPTION.ENCODING) + + return encrypted +} + +async function decrypt (encryptedArg: string, secret: string) { + const [ ivStr, encryptedStr ] = encryptedArg.split(':') + + const iv = Buffer.from(ivStr, 'hex') + const key = await scryptPromise(secret, ENCRYPTION.SALT, 32) + + const decipher = createDecipheriv(ENCRYPTION.ALGORITHM, key, iv) + + return decipher.update(encryptedStr, ENCRYPTION.ENCODING, 'utf8') + decipher.final('utf8') +} + +// --------------------------------------------------------------------------- + +export { + isHTTPSignatureDigestValid, + parseHTTPSignature, + isHTTPSignatureVerified, + buildDigest, + isJsonLDSignatureVerified, + comparePassword, + createPrivateAndPublicKeys, + cryptPassword, + signJsonLDObject, + + encrypt, + decrypt +} + +// --------------------------------------------------------------------------- + +async function hashObject (obj: any): Promise { + const { jsonld } = await import('./custom-jsonld-signature.js') + + const res = await (jsonld as any).promises.normalize(obj, { + safe: false, + algorithm: 'URDNA2015', + format: 'application/n-quads' + }) + + return sha256(res) +} + +function createSignatureHash (signature: any) { + const signatureCopy = cloneDeep(signature) + Object.assign(signatureCopy, { + '@context': [ + 'https://w3id.org/security/v1', + { RsaSignature2017: 'https://w3id.org/security#RsaSignature2017' } + ] + }) + + delete signatureCopy.type + delete signatureCopy.id + delete signatureCopy.signatureValue + + return hashObject(signatureCopy) +} + +function createDocWithoutSignatureHash (doc: any) { + const docWithoutSignature = cloneDeep(doc) + delete docWithoutSignature.signature + + return hashObject(docWithoutSignature) +} diff --git a/server/helpers/promise-cache.ts b/server/server/helpers/promise-cache.ts similarity index 100% rename from server/helpers/promise-cache.ts rename to server/server/helpers/promise-cache.ts diff --git a/server/helpers/proxy.ts b/server/server/helpers/proxy.ts similarity index 100% rename from server/helpers/proxy.ts rename to server/server/helpers/proxy.ts diff --git a/server/server/helpers/query.ts b/server/server/helpers/query.ts new file mode 100644 index 000000000..7bd63f40e --- /dev/null +++ b/server/server/helpers/query.ts @@ -0,0 +1,81 @@ +import { pick } from '@peertube/peertube-core-utils' +import { + VideoChannelsSearchQueryAfterSanitize, + VideoPlaylistsSearchQueryAfterSanitize, + VideosCommonQueryAfterSanitize, + VideosSearchQueryAfterSanitize +} from '@peertube/peertube-models' + +function pickCommonVideoQuery (query: VideosCommonQueryAfterSanitize) { + return pick(query, [ + 'start', + 'count', + 'sort', + 'nsfw', + 'isLive', + 'categoryOneOf', + 'licenceOneOf', + 'languageOneOf', + 'privacyOneOf', + 'tagsOneOf', + 'tagsAllOf', + 'isLocal', + 'include', + 'skipCount', + 'hasHLSFiles', + 'hasWebtorrentFiles', // TODO: Remove in v7 + 'hasWebVideoFiles', + 'search', + 'excludeAlreadyWatched' + ]) +} + +function pickSearchVideoQuery (query: VideosSearchQueryAfterSanitize) { + return { + ...pickCommonVideoQuery(query), + + ...pick(query, [ + 'searchTarget', + 'host', + 'startDate', + 'endDate', + 'originallyPublishedStartDate', + 'originallyPublishedEndDate', + 'durationMin', + 'durationMax', + 'uuids', + 'excludeAlreadyWatched' + ]) + } +} + +function pickSearchChannelQuery (query: VideoChannelsSearchQueryAfterSanitize) { + return pick(query, [ + 'searchTarget', + 'search', + 'start', + 'count', + 'sort', + 'host', + 'handles' + ]) +} + +function pickSearchPlaylistQuery (query: VideoPlaylistsSearchQueryAfterSanitize) { + return pick(query, [ + 'searchTarget', + 'search', + 'start', + 'count', + 'sort', + 'host', + 'uuids' + ]) +} + +export { + pickCommonVideoQuery, + pickSearchVideoQuery, + pickSearchPlaylistQuery, + pickSearchChannelQuery +} diff --git a/server/helpers/regexp.ts b/server/server/helpers/regexp.ts similarity index 100% rename from server/helpers/regexp.ts rename to server/server/helpers/regexp.ts diff --git a/server/server/helpers/requests.ts b/server/server/helpers/requests.ts new file mode 100644 index 000000000..c7dd1172b --- /dev/null +++ b/server/server/helpers/requests.ts @@ -0,0 +1,258 @@ +import httpSignature from '@peertube/http-signature' +import { createWriteStream } from 'fs' +import { remove } from 'fs-extra/esm' +import got, { CancelableRequest, OptionsInit, OptionsOfTextResponseBody, OptionsOfUnknownResponseBody, RequestError, Response } from 'got' +import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent' +import { ACTIVITY_PUB, BINARY_CONTENT_TYPES, PEERTUBE_VERSION, REQUEST_TIMEOUTS, WEBSERVER } from '../initializers/constants.js' +import { pipelinePromise } from './core-utils.js' +import { logger, loggerTagsFactory } from './logger.js' +import { getProxy, isProxyEnabled } from './proxy.js' + +const lTags = loggerTagsFactory('request') + +export interface PeerTubeRequestError extends Error { + statusCode?: number + responseBody?: any + responseHeaders?: any + requestHeaders?: any +} + +type PeerTubeRequestOptions = { + timeout?: number + activityPub?: boolean + bodyKBLimit?: number // 1MB + + httpSignature?: { + algorithm: string + authorizationHeaderName: string + keyId: string + key: string + headers: string[] + } + + jsonResponse?: boolean + + followRedirect?: boolean +} & Pick + +const peertubeGot = got.extend({ + ...getAgent(), + + headers: { + 'user-agent': getUserAgent() + }, + + handlers: [ + (options, next) => { + const promiseOrStream = next(options) as CancelableRequest + const bodyKBLimit = options.context?.bodyKBLimit as number + if (!bodyKBLimit) throw new Error('No KB limit for this request') + + const bodyLimit = bodyKBLimit * 1000 + + /* eslint-disable @typescript-eslint/no-floating-promises */ + promiseOrStream.on('downloadProgress', progress => { + if (progress.transferred > bodyLimit && progress.percent !== 1) { + const message = `Exceeded the download limit of ${bodyLimit} B` + logger.warn(message, lTags()) + + // CancelableRequest + if (promiseOrStream.cancel) { + promiseOrStream.cancel() + return + } + + // Stream + (promiseOrStream as any).destroy() + } + }) + + return promiseOrStream + } + ], + + hooks: { + beforeRequest: [ + options => { + const headers = options.headers || {} + headers['host'] = buildUrl(options.url).host + }, + + options => { + const httpSignatureOptions = options.context?.httpSignature + + if (httpSignatureOptions) { + const method = options.method ?? 'GET' + const path = buildUrl(options.url).pathname + + if (!method || !path) { + throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`) + } + + httpSignature.signRequest({ + getHeader: function (header: string) { + const value = options.headers[header.toLowerCase()] + + if (!value) logger.warn('Unknown header requested by http-signature.', { headers: options.headers, header }) + return value + }, + + setHeader: function (header: string, value: string) { + options.headers[header] = value + }, + + method, + path + }, httpSignatureOptions) + } + } + ], + + beforeRetry: [ + (error: RequestError, retryCount: number) => { + logger.debug('Retrying request to %s.', error.request.requestUrl, { retryCount, error: buildRequestError(error), ...lTags() }) + } + ] + } +}) + +function doRequest (url: string, options: PeerTubeRequestOptions = {}) { + const gotOptions = buildGotOptions(options) as OptionsOfTextResponseBody + + return peertubeGot(url, gotOptions) + .catch(err => { throw buildRequestError(err) }) +} + +function doJSONRequest (url: string, options: PeerTubeRequestOptions = {}) { + const gotOptions = buildGotOptions(options) + + return peertubeGot(url, { ...gotOptions, responseType: 'json' }) + .catch(err => { throw buildRequestError(err) }) +} + +async function doRequestAndSaveToFile ( + url: string, + destPath: string, + options: PeerTubeRequestOptions = {} +) { + const gotOptions = buildGotOptions({ ...options, timeout: options.timeout ?? REQUEST_TIMEOUTS.FILE }) + + const outFile = createWriteStream(destPath) + + try { + await pipelinePromise( + peertubeGot.stream(url, { ...gotOptions, isStream: true }), + outFile + ) + } catch (err) { + remove(destPath) + .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err, ...lTags() })) + + throw buildRequestError(err) + } +} + +function getAgent () { + if (!isProxyEnabled()) return {} + + const proxy = getProxy() + + logger.info('Using proxy %s.', proxy, lTags()) + + const proxyAgentOptions = { + keepAlive: true, + keepAliveMsecs: 1000, + maxSockets: 256, + maxFreeSockets: 256, + scheduling: 'lifo' as 'lifo', + proxy + } + + return { + agent: { + http: new HttpProxyAgent(proxyAgentOptions), + https: new HttpsProxyAgent(proxyAgentOptions) + } + } +} + +function getUserAgent () { + return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})` +} + +function isBinaryResponse (result: Response) { + return BINARY_CONTENT_TYPES.has(result.headers['content-type']) +} + +// --------------------------------------------------------------------------- + +export { + type PeerTubeRequestOptions, + + doRequest, + doJSONRequest, + doRequestAndSaveToFile, + isBinaryResponse, + getAgent, + peertubeGot +} + +// --------------------------------------------------------------------------- + +function buildGotOptions (options: PeerTubeRequestOptions): OptionsOfUnknownResponseBody { + const { activityPub, bodyKBLimit = 1000 } = options + + const context = { bodyKBLimit, httpSignature: options.httpSignature } + + let headers = options.headers || {} + + if (!headers.date) { + headers = { ...headers, date: new Date().toUTCString() } + } + + if (activityPub && !headers.accept) { + headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER } + } + + return { + method: options.method, + dnsCache: true, + timeout: { + request: options.timeout ?? REQUEST_TIMEOUTS.DEFAULT + }, + json: options.json, + searchParams: options.searchParams, + followRedirect: options.followRedirect, + retry: { + limit: 2 + }, + headers, + context + } +} + +function buildRequestError (error: RequestError) { + const newError: PeerTubeRequestError = new Error(error.message) + newError.name = error.name + newError.stack = error.stack + + if (error.response) { + newError.responseBody = error.response.body + newError.responseHeaders = error.response.headers + newError.statusCode = error.response.statusCode + } + + if (error.options) { + newError.requestHeaders = error.options.headers + } + + return newError +} + +function buildUrl (url: string | URL) { + if (typeof url === 'string') { + return new URL(url) + } + + return url +} diff --git a/server/helpers/stream-replacer.ts b/server/server/helpers/stream-replacer.ts similarity index 100% rename from server/helpers/stream-replacer.ts rename to server/server/helpers/stream-replacer.ts diff --git a/server/server/helpers/token-generator.ts b/server/server/helpers/token-generator.ts new file mode 100644 index 000000000..c7d52c41a --- /dev/null +++ b/server/server/helpers/token-generator.ts @@ -0,0 +1,19 @@ +import { buildUUID } from '@peertube/peertube-node-utils' + +function generateRunnerRegistrationToken () { + return 'ptrrt-' + buildUUID() +} + +function generateRunnerToken () { + return 'ptrt-' + buildUUID() +} + +function generateRunnerJobToken () { + return 'ptrjt-' + buildUUID() +} + +export { + generateRunnerRegistrationToken, + generateRunnerToken, + generateRunnerJobToken +} diff --git a/server/server/helpers/upload.ts b/server/server/helpers/upload.ts new file mode 100644 index 000000000..1a182a1f6 --- /dev/null +++ b/server/server/helpers/upload.ts @@ -0,0 +1,14 @@ +import { join } from 'path' +import { DIRECTORIES } from '@server/initializers/constants.js' + +function getResumableUploadPath (filename?: string) { + if (filename) return join(DIRECTORIES.RESUMABLE_UPLOAD, filename) + + return DIRECTORIES.RESUMABLE_UPLOAD +} + +// --------------------------------------------------------------------------- + +export { + getResumableUploadPath +} diff --git a/server/server/helpers/utils.ts b/server/server/helpers/utils.ts new file mode 100644 index 000000000..850286b61 --- /dev/null +++ b/server/server/helpers/utils.ts @@ -0,0 +1,70 @@ +import { remove } from 'fs-extra/esm' +import { Instance as ParseTorrent } from 'parse-torrent' +import { join } from 'path' +import { sha256 } from '@peertube/peertube-node-utils' +import { ResultList } from '@peertube/peertube-models' +import { CONFIG } from '../initializers/config.js' +import { randomBytesPromise } from './core-utils.js' +import { logger } from './logger.js' + +function deleteFileAndCatch (path: string) { + remove(path) + .catch(err => logger.error('Cannot delete the file %s asynchronously.', path, { err })) +} + +async function generateRandomString (size: number) { + const raw = await randomBytesPromise(size) + + return raw.toString('hex') +} + +interface FormattableToJSON { + toFormattedJSON (args?: U): V +} + +function getFormattedObjects> (objects: T[], objectsTotal: number, formattedArg?: U) { + const formattedObjects = objects.map(o => o.toFormattedJSON(formattedArg)) + + return { + total: objectsTotal, + data: formattedObjects + } as ResultList +} + +function generateVideoImportTmpPath (target: string | ParseTorrent, extension = '.mp4') { + const id = typeof target === 'string' + ? target + : target.infoHash + + const hash = sha256(id) + return join(CONFIG.STORAGE.TMP_DIR, `${hash}-import${extension}`) +} + +function getSecureTorrentName (originalName: string) { + return sha256(originalName) + '.torrent' +} + +/** + * From a filename like "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3.mp4", returns + * only the "ede4cba5-742b-46fa-a388-9a6eb3a3aeb3" part. If the filename does + * not contain a UUID, returns null. + */ +function getUUIDFromFilename (filename: string) { + const regex = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/ + const result = filename.match(regex) + + if (!result || Array.isArray(result) === false) return null + + return result[0] +} + +// --------------------------------------------------------------------------- + +export { + deleteFileAndCatch, + generateRandomString, + getFormattedObjects, + getSecureTorrentName, + generateVideoImportTmpPath, + getUUIDFromFilename +} diff --git a/server/server/helpers/version.ts b/server/server/helpers/version.ts new file mode 100644 index 000000000..2608a36b7 --- /dev/null +++ b/server/server/helpers/version.ts @@ -0,0 +1,36 @@ +import { execPromise, execPromise2 } from './core-utils.js' +import { logger } from './logger.js' + +async function getServerCommit () { + try { + const tag = await execPromise2( + '[ ! -d .git ] || git name-rev --name-only --tags --no-undefined HEAD 2>/dev/null || true', + { stdio: [ 0, 1, 2 ] } + ) + + if (tag) return tag.replace(/^v/, '') + } catch (err) { + logger.debug('Cannot get version from git tags.', { err }) + } + + try { + const version = await execPromise('[ ! -d .git ] || git rev-parse --short HEAD') + + if (version) return version.toString().trim() + } catch (err) { + logger.debug('Cannot get version from git HEAD.', { err }) + } + + return '' +} + +function getNodeABIVersion () { + const version = process.versions.modules + + return parseInt(version) +} + +export { + getServerCommit, + getNodeABIVersion +} diff --git a/server/server/helpers/video.ts b/server/server/helpers/video.ts new file mode 100644 index 000000000..dc56a6697 --- /dev/null +++ b/server/server/helpers/video.ts @@ -0,0 +1,51 @@ +import { Response } from 'express' +import { forceNumber } from '@peertube/peertube-core-utils' +import { VideoPrivacy, VideoPrivacyType, VideoState, VideoStateType } from '@peertube/peertube-models' +import { CONFIG } from '@server/initializers/config.js' +import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models/index.js' + +function getVideoWithAttributes (res: Response) { + return res.locals.videoAPI || res.locals.videoAll || res.locals.onlyVideo +} + +function extractVideo (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) { + return isStreamingPlaylist(videoOrPlaylist) + ? videoOrPlaylist.Video + : videoOrPlaylist +} + +function isPrivacyForFederation (privacy: VideoPrivacyType) { + const castedPrivacy = forceNumber(privacy) + + return castedPrivacy === VideoPrivacy.PUBLIC || + (CONFIG.FEDERATION.VIDEOS.FEDERATE_UNLISTED === true && castedPrivacy === VideoPrivacy.UNLISTED) +} + +function isStateForFederation (state: VideoStateType) { + const castedState = forceNumber(state) + + return castedState === VideoState.PUBLISHED || castedState === VideoState.WAITING_FOR_LIVE || castedState === VideoState.LIVE_ENDED +} + +function getPrivaciesForFederation () { + return (CONFIG.FEDERATION.VIDEOS.FEDERATE_UNLISTED === true) + ? [ { privacy: VideoPrivacy.PUBLIC }, { privacy: VideoPrivacy.UNLISTED } ] + : [ { privacy: VideoPrivacy.PUBLIC } ] +} + +function getExtFromMimetype (mimeTypes: { [id: string]: string | string[] }, mimeType: string) { + const value = mimeTypes[mimeType] + + if (Array.isArray(value)) return value[0] + + return value +} + +export { + getVideoWithAttributes, + extractVideo, + getExtFromMimetype, + isStateForFederation, + isPrivacyForFederation, + getPrivaciesForFederation +} diff --git a/server/server/helpers/webtorrent.ts b/server/server/helpers/webtorrent.ts new file mode 100644 index 000000000..29062dae3 --- /dev/null +++ b/server/server/helpers/webtorrent.ts @@ -0,0 +1,263 @@ +import bencode from 'bencode' +import createTorrent from 'create-torrent' +import { createWriteStream } from 'fs' +import { ensureDir, pathExists, remove } from 'fs-extra/esm' +import { readFile, writeFile } from 'fs/promises' +import { encode as magnetUriEncode } from 'magnet-uri' +import parseTorrent from 'parse-torrent' +import { dirname, join } from 'path' +import { pipeline } from 'stream' +import { promisify2 } from '@peertube/peertube-core-utils' +import { isArray } from '@server/helpers/custom-validators/misc.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { generateTorrentFileName } from '@server/lib/paths.js' +import { VideoPathManager } from '@server/lib/video-path-manager.js' +import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file.js' +import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist.js' +import { MVideo } from '@server/types/models/video/video.js' +import { sha1 } from '@peertube/peertube-node-utils' +import { CONFIG } from '../initializers/config.js' +import { logger } from './logger.js' +import { generateVideoImportTmpPath } from './utils.js' +import { extractVideo } from './video.js' + +import type { Instance, TorrentFile } from 'webtorrent' + +const createTorrentPromise = promisify2(createTorrent) + +async function downloadWebTorrentVideo (target: { uri: string, torrentName?: string }, timeout: number) { + const id = target.uri || target.torrentName + let timer + + const path = generateVideoImportTmpPath(id) + logger.info('Importing torrent video %s', id) + + const directoryPath = join(CONFIG.STORAGE.TMP_DIR, 'webtorrent') + await ensureDir(directoryPath) + + // eslint-disable-next-line new-cap + const webtorrent = new (await import('webtorrent')).default() + + return new Promise((res, rej) => { + let file: TorrentFile + + const torrentId = target.uri || join(CONFIG.STORAGE.TORRENTS_DIR, target.torrentName) + + const options = { path: directoryPath } + const torrent = webtorrent.add(torrentId, options, torrent => { + if (torrent.files.length !== 1) { + if (timer) clearTimeout(timer) + + for (const file of torrent.files) { + deleteDownloadedFile({ directoryPath, filepath: file.path }) + } + + return safeWebtorrentDestroy(webtorrent, torrentId, undefined, target.torrentName) + .then(() => rej(new Error('Cannot import torrent ' + torrentId + ': there are multiple files in it'))) + } + + logger.debug('Got torrent from webtorrent %s.', id, { infoHash: torrent.infoHash }) + + file = torrent.files[0] + + // FIXME: avoid creating another stream when https://github.com/webtorrent/webtorrent/issues/1517 is fixed + const writeStream = createWriteStream(path) + writeStream.on('finish', () => { + if (timer) clearTimeout(timer) + + safeWebtorrentDestroy(webtorrent, torrentId, { directoryPath, filepath: file.path }, target.torrentName) + .then(() => res(path)) + .catch(err => logger.error('Cannot destroy webtorrent.', { err })) + }) + + pipeline( + file.createReadStream(), + writeStream, + err => { + if (err) rej(err) + } + ) + }) + + torrent.on('error', err => rej(err)) + + timer = setTimeout(() => { + const err = new Error('Webtorrent download timeout.') + + safeWebtorrentDestroy(webtorrent, torrentId, file ? { directoryPath, filepath: file.path } : undefined, target.torrentName) + .then(() => rej(err)) + .catch(destroyErr => { + logger.error('Cannot destroy webtorrent.', { err: destroyErr }) + rej(err) + }) + + }, timeout) + }) +} + +function createTorrentAndSetInfoHash (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { + return VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(videoOrPlaylist), videoPath => { + return createTorrentAndSetInfoHashFromPath(videoOrPlaylist, videoFile, videoPath) + }) +} + +async function createTorrentAndSetInfoHashFromPath ( + videoOrPlaylist: MVideo | MStreamingPlaylistVideo, + videoFile: MVideoFile, + filePath: string +) { + const video = extractVideo(videoOrPlaylist) + + const options = { + // Keep the extname, it's used by the client to stream the file inside a web browser + name: buildInfoName(video, videoFile), + createdBy: 'PeerTube', + announceList: buildAnnounceList(), + urlList: buildUrlList(video, videoFile) + } + + const torrentContent = await createTorrentPromise(filePath, options) + + const torrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution) + const torrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, torrentFilename) + logger.info('Creating torrent %s.', torrentPath) + + await writeFile(torrentPath, torrentContent) + + // Remove old torrent file if it existed + if (videoFile.hasTorrent()) { + await remove(join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename)) + } + + const parsedTorrent = parseTorrent(torrentContent) + videoFile.infoHash = parsedTorrent.infoHash + videoFile.torrentFilename = torrentFilename +} + +async function updateTorrentMetadata (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { + const video = extractVideo(videoOrPlaylist) + + const oldTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) + + if (!await pathExists(oldTorrentPath)) { + logger.info('Do not update torrent metadata %s of video %s because the file does not exist anymore.', video.uuid, oldTorrentPath) + return + } + + const torrentContent = await readFile(oldTorrentPath) + const decoded = bencode.decode(torrentContent) + + decoded['announce-list'] = buildAnnounceList() + decoded.announce = decoded['announce-list'][0][0] + + decoded['url-list'] = buildUrlList(video, videoFile) + + decoded.info.name = buildInfoName(video, videoFile) + decoded['creation date'] = Math.ceil(Date.now() / 1000) + + const newTorrentFilename = generateTorrentFileName(videoOrPlaylist, videoFile.resolution) + const newTorrentPath = join(CONFIG.STORAGE.TORRENTS_DIR, newTorrentFilename) + + logger.info('Updating torrent metadata %s -> %s.', oldTorrentPath, newTorrentPath) + + await writeFile(newTorrentPath, bencode.encode(decoded)) + await remove(oldTorrentPath) + + videoFile.torrentFilename = newTorrentFilename + videoFile.infoHash = sha1(bencode.encode(decoded.info)) +} + +function generateMagnetUri ( + video: MVideo, + videoFile: MVideoFileRedundanciesOpt, + trackerUrls: string[] +) { + const xs = videoFile.getTorrentUrl() + const announce = trackerUrls + + let urlList = video.hasPrivateStaticPath() + ? [] + : [ videoFile.getFileUrl(video) ] + + const redundancies = videoFile.RedundancyVideos + if (isArray(redundancies)) urlList = urlList.concat(redundancies.map(r => r.fileUrl)) + + const magnetHash = { + xs, + announce, + urlList, + infoHash: videoFile.infoHash, + name: video.name + } + + return magnetUriEncode(magnetHash) +} + +// --------------------------------------------------------------------------- + +export { + createTorrentPromise, + updateTorrentMetadata, + + createTorrentAndSetInfoHash, + createTorrentAndSetInfoHashFromPath, + + generateMagnetUri, + downloadWebTorrentVideo +} + +// --------------------------------------------------------------------------- + +function safeWebtorrentDestroy ( + webtorrent: Instance, + torrentId: string, + downloadedFile?: { directoryPath: string, filepath: string }, + torrentName?: string +) { + return new Promise(res => { + webtorrent.destroy(err => { + // Delete torrent file + if (torrentName) { + logger.debug('Removing %s torrent after webtorrent download.', torrentId) + remove(torrentId) + .catch(err => logger.error('Cannot remove torrent %s in webtorrent download.', torrentId, { err })) + } + + // Delete downloaded file + if (downloadedFile) deleteDownloadedFile(downloadedFile) + + if (err) logger.warn('Cannot destroy webtorrent in timeout.', { err }) + + return res() + }) + }) +} + +function deleteDownloadedFile (downloadedFile: { directoryPath: string, filepath: string }) { + // We want to delete the base directory + let pathToDelete = dirname(downloadedFile.filepath) + if (pathToDelete === '.') pathToDelete = downloadedFile.filepath + + const toRemovePath = join(downloadedFile.directoryPath, pathToDelete) + + logger.debug('Removing %s after webtorrent download.', toRemovePath) + remove(toRemovePath) + .catch(err => logger.error('Cannot remove torrent file %s in webtorrent download.', toRemovePath, { err })) +} + +function buildAnnounceList () { + return [ + [ WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' ], + [ WEBSERVER.URL + '/tracker/announce' ] + ] +} + +function buildUrlList (video: MVideo, videoFile: MVideoFile) { + if (video.hasPrivateStaticPath()) return [] + + return [ videoFile.getFileUrl(video) ] +} + +function buildInfoName (video: MVideo, videoFile: MVideoFile) { + return `${video.name} ${videoFile.resolution}p${videoFile.extname}` +} diff --git a/server/server/helpers/youtube-dl/index.ts b/server/server/helpers/youtube-dl/index.ts new file mode 100644 index 000000000..29a07d93f --- /dev/null +++ b/server/server/helpers/youtube-dl/index.ts @@ -0,0 +1,3 @@ +export * from './youtube-dl-cli.js' +export * from './youtube-dl-info-builder.js' +export * from './youtube-dl-wrapper.js' diff --git a/server/server/helpers/youtube-dl/youtube-dl-cli.ts b/server/server/helpers/youtube-dl/youtube-dl-cli.ts new file mode 100644 index 000000000..007f061c6 --- /dev/null +++ b/server/server/helpers/youtube-dl/youtube-dl-cli.ts @@ -0,0 +1,260 @@ +import execa from 'execa' +import { ensureDir, pathExists } from 'fs-extra/esm' +import { writeFile } from 'fs/promises' +import { OptionsOfBufferResponseBody } from 'got' +import { dirname, join } from 'path' +import { VideoResolution, VideoResolutionType } from '@peertube/peertube-models' +import { CONFIG } from '@server/initializers/config.js' +import { logger, loggerTagsFactory } from '../logger.js' +import { getProxy, isProxyEnabled } from '../proxy.js' +import { isBinaryResponse, peertubeGot } from '../requests.js' + +const lTags = loggerTagsFactory('youtube-dl') + +const youtubeDLBinaryPath = join(CONFIG.STORAGE.BIN_DIR, CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME) + +export class YoutubeDLCLI { + + static async safeGet () { + if (!await pathExists(youtubeDLBinaryPath)) { + await ensureDir(dirname(youtubeDLBinaryPath)) + + await this.updateYoutubeDLBinary() + } + + return new YoutubeDLCLI() + } + + static async updateYoutubeDLBinary () { + const url = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.URL + + logger.info('Updating youtubeDL binary from %s.', url, lTags()) + + const gotOptions: OptionsOfBufferResponseBody = { + context: { bodyKBLimit: 20_000 }, + responseType: 'buffer' as 'buffer' + } + + if (process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN) { + gotOptions.headers = { + authorization: 'Bearer ' + process.env.YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN + } + } + + try { + let gotResult = await peertubeGot(url, gotOptions) + + if (!isBinaryResponse(gotResult)) { + const json = JSON.parse(gotResult.body.toString()) + const latest = json.filter(release => release.prerelease === false)[0] + if (!latest) throw new Error('Cannot find latest release') + + const releaseName = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME + const releaseAsset = latest.assets.find(a => a.name === releaseName) + if (!releaseAsset) throw new Error(`Cannot find appropriate release with name ${releaseName} in release assets`) + + gotResult = await peertubeGot(releaseAsset.browser_download_url, gotOptions) + } + + if (!isBinaryResponse(gotResult)) { + throw new Error('Not a binary response') + } + + await writeFile(youtubeDLBinaryPath, gotResult.body) + + logger.info('youtube-dl updated %s.', youtubeDLBinaryPath, lTags()) + } catch (err) { + logger.error('Cannot update youtube-dl from %s.', url, { err, ...lTags() }) + } + } + + static getYoutubeDLVideoFormat (enabledResolutions: VideoResolutionType[], useBestFormat: boolean) { + /** + * list of format selectors in order or preference + * see https://github.com/ytdl-org/youtube-dl#format-selection + * + * case #1 asks for a mp4 using h264 (avc1) and the exact resolution in the hope + * of being able to do a "quick-transcode" + * case #2 is the first fallback. No "quick-transcode" means we can get anything else (like vp9) + * case #3 is the resolution-degraded equivalent of #1, and already a pretty safe fallback + * + * in any case we avoid AV1, see https://github.com/Chocobozzz/PeerTube/issues/3499 + **/ + + let result: string[] = [] + + if (!useBestFormat) { + const resolution = enabledResolutions.length === 0 + ? VideoResolution.H_720P + : Math.max(...enabledResolutions) + + result = [ + `bestvideo[vcodec^=avc1][height=${resolution}]+bestaudio[ext=m4a]`, // case #1 + `bestvideo[vcodec!*=av01][vcodec!*=vp9.2][height=${resolution}]+bestaudio`, // case #2 + `bestvideo[vcodec^=avc1][height<=${resolution}]+bestaudio[ext=m4a]` // case # + ] + } + + return result.concat([ + 'bestvideo[vcodec!*=av01][vcodec!*=vp9.2]+bestaudio', + 'best[vcodec!*=av01][vcodec!*=vp9.2]', // case fallback for known formats + 'bestvideo[ext=mp4]+bestaudio[ext=m4a]', + 'best' // Ultimate fallback + ]).join('/') + } + + private constructor () { + + } + + download (options: { + url: string + format: string + output: string + processOptions: execa.NodeOptions + timeout?: number + additionalYoutubeDLArgs?: string[] + }) { + let args = options.additionalYoutubeDLArgs || [] + args = args.concat([ '--merge-output-format', 'mp4', '-f', options.format, '-o', options.output ]) + + return this.run({ + url: options.url, + processOptions: options.processOptions, + timeout: options.timeout, + args + }) + } + + async getInfo (options: { + url: string + format: string + processOptions: execa.NodeOptions + additionalYoutubeDLArgs?: string[] + }) { + const { url, format, additionalYoutubeDLArgs = [], processOptions } = options + + const completeArgs = additionalYoutubeDLArgs.concat([ '--dump-json', '-f', format ]) + + const data = await this.run({ url, args: completeArgs, processOptions }) + if (!data) return undefined + + const info = data.map(d => JSON.parse(d)) + + return info.length === 1 + ? info[0] + : info + } + + async getListInfo (options: { + url: string + latestVideosCount?: number + processOptions: execa.NodeOptions + }): Promise<{ upload_date: string, webpage_url: string }[]> { + const additionalYoutubeDLArgs = [ '--skip-download', '--playlist-reverse' ] + + if (CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE.NAME === 'yt-dlp') { + // Optimize listing videos only when using yt-dlp because it is bugged with youtube-dl when fetching a channel + additionalYoutubeDLArgs.push('--flat-playlist') + } + + if (options.latestVideosCount !== undefined) { + additionalYoutubeDLArgs.push('--playlist-end', options.latestVideosCount.toString()) + } + + const result = await this.getInfo({ + url: options.url, + format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false), + processOptions: options.processOptions, + additionalYoutubeDLArgs + }) + + if (!result) return result + if (!Array.isArray(result)) return [ result ] + + return result + } + + async getSubs (options: { + url: string + format: 'vtt' + processOptions: execa.NodeOptions + }) { + const { url, format, processOptions } = options + + const args = [ '--skip-download', '--all-subs', `--sub-format=${format}` ] + + const data = await this.run({ url, args, processOptions }) + const files: string[] = [] + + const skipString = '[info] Writing video subtitles to: ' + + for (let i = 0, len = data.length; i < len; i++) { + const line = data[i] + + if (line.indexOf(skipString) === 0) { + files.push(line.slice(skipString.length)) + } + } + + return files + } + + private async run (options: { + url: string + args: string[] + timeout?: number + processOptions: execa.NodeOptions + }) { + const { url, args, timeout, processOptions } = options + + let completeArgs = this.wrapWithProxyOptions(args) + completeArgs = this.wrapWithIPOptions(completeArgs) + completeArgs = this.wrapWithFFmpegOptions(completeArgs) + + const { PYTHON_PATH } = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE + const subProcess = execa(PYTHON_PATH, [ youtubeDLBinaryPath, ...completeArgs, url ], processOptions) + + if (timeout) { + setTimeout(() => subProcess.cancel(), timeout) + } + + const output = await subProcess + + logger.debug('Run youtube-dl command.', { command: output.command, ...lTags() }) + + return output.stdout + ? output.stdout.trim().split(/\r?\n/) + : undefined + } + + private wrapWithProxyOptions (args: string[]) { + if (isProxyEnabled()) { + logger.debug('Using proxy %s for YoutubeDL', getProxy(), lTags()) + + return [ '--proxy', getProxy() ].concat(args) + } + + return args + } + + private wrapWithIPOptions (args: string[]) { + if (CONFIG.IMPORT.VIDEOS.HTTP.FORCE_IPV4) { + logger.debug('Force ipv4 for YoutubeDL') + + return [ '--force-ipv4' ].concat(args) + } + + return args + } + + private wrapWithFFmpegOptions (args: string[]) { + if (process.env.FFMPEG_PATH) { + logger.debug('Using ffmpeg location %s for YoutubeDL', process.env.FFMPEG_PATH, lTags()) + + return [ '--ffmpeg-location', process.env.FFMPEG_PATH ].concat(args) + } + + return args + } +} diff --git a/server/server/helpers/youtube-dl/youtube-dl-info-builder.ts b/server/server/helpers/youtube-dl/youtube-dl-info-builder.ts new file mode 100644 index 000000000..0287f6183 --- /dev/null +++ b/server/server/helpers/youtube-dl/youtube-dl-info-builder.ts @@ -0,0 +1,198 @@ +import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../../initializers/constants.js' +import { peertubeTruncate } from '../core-utils.js' +import { isUrlValid } from '../custom-validators/activitypub/misc.js' + +export type YoutubeDLInfo = { + name?: string + description?: string + category?: number + language?: string + licence?: number + nsfw?: boolean + tags?: string[] + thumbnailUrl?: string + ext?: string + originallyPublishedAtWithoutTime?: Date + webpageUrl?: string + + urls?: string[] +} + +export class YoutubeDLInfoBuilder { + private readonly info: any + + constructor (info: any) { + this.info = { ...info } + } + + getInfo () { + const obj = this.buildVideoInfo(this.normalizeObject(this.info)) + if (obj.name && obj.name.length < CONSTRAINTS_FIELDS.VIDEOS.NAME.min) obj.name += ' video' + + return obj + } + + private normalizeObject (obj: any) { + const newObj: any = {} + + for (const key of Object.keys(obj)) { + // Deprecated key + if (key === 'resolution') continue + + const value = obj[key] + + if (typeof value === 'string') { + newObj[key] = value.normalize() + } else { + newObj[key] = value + } + } + + return newObj + } + + private buildOriginallyPublishedAt (obj: any) { + let originallyPublishedAt: Date = null + + const uploadDateMatcher = /^(\d{4})(\d{2})(\d{2})$/.exec(obj.upload_date) + if (uploadDateMatcher) { + originallyPublishedAt = new Date() + originallyPublishedAt.setHours(0, 0, 0, 0) + + const year = parseInt(uploadDateMatcher[1], 10) + // Month starts from 0 + const month = parseInt(uploadDateMatcher[2], 10) - 1 + const day = parseInt(uploadDateMatcher[3], 10) + + originallyPublishedAt.setFullYear(year, month, day) + } + + return originallyPublishedAt + } + + private buildVideoInfo (obj: any): YoutubeDLInfo { + return { + name: this.titleTruncation(obj.title), + description: this.descriptionTruncation(obj.description), + category: this.getCategory(obj.categories), + licence: this.getLicence(obj.license), + language: this.getLanguage(obj.language), + nsfw: this.isNSFW(obj), + tags: this.getTags(obj.tags), + thumbnailUrl: obj.thumbnail || undefined, + urls: this.buildAvailableUrl(obj), + originallyPublishedAtWithoutTime: this.buildOriginallyPublishedAt(obj), + ext: obj.ext, + webpageUrl: obj.webpage_url + } + } + + private buildAvailableUrl (obj: any) { + const urls: string[] = [] + + if (obj.url) urls.push(obj.url) + if (obj.urls) { + if (Array.isArray(obj.urls)) urls.push(...obj.urls) + else urls.push(obj.urls) + } + + const formats = Array.isArray(obj.formats) + ? obj.formats + : [] + + for (const format of formats) { + if (!format.url) continue + + urls.push(format.url) + } + + const thumbnails = Array.isArray(obj.thumbnails) + ? obj.thumbnails + : [] + + for (const thumbnail of thumbnails) { + if (!thumbnail.url) continue + + urls.push(thumbnail.url) + } + + if (obj.thumbnail) urls.push(obj.thumbnail) + + for (const subtitleKey of Object.keys(obj.subtitles || {})) { + const subtitles = obj.subtitles[subtitleKey] + if (!Array.isArray(subtitles)) continue + + for (const subtitle of subtitles) { + if (!subtitle.url) continue + + urls.push(subtitle.url) + } + } + + return urls.filter(u => u && isUrlValid(u)) + } + + private titleTruncation (title: string) { + return peertubeTruncate(title, { + length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max, + separator: /,? +/, + omission: ' […]' + }) + } + + private descriptionTruncation (description: string) { + if (!description || description.length < CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.min) return undefined + + return peertubeTruncate(description, { + length: CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max, + separator: /,? +/, + omission: ' […]' + }) + } + + private isNSFW (info: any) { + return info?.age_limit >= 16 + } + + private getTags (tags: string[]) { + if (Array.isArray(tags) === false) return [] + + return tags + .filter(t => t.length < CONSTRAINTS_FIELDS.VIDEOS.TAG.max && t.length > CONSTRAINTS_FIELDS.VIDEOS.TAG.min) + .map(t => t.normalize()) + .slice(0, 5) + } + + private getLicence (licence: string) { + if (!licence) return undefined + + if (licence.includes('Creative Commons Attribution')) return 1 + + for (const key of Object.keys(VIDEO_LICENCES)) { + const peertubeLicence = VIDEO_LICENCES[key] + if (peertubeLicence.toLowerCase() === licence.toLowerCase()) return parseInt(key, 10) + } + + return undefined + } + + private getCategory (categories: string[]) { + if (!categories) return undefined + + const categoryString = categories[0] + if (!categoryString || typeof categoryString !== 'string') return undefined + + if (categoryString === 'News & Politics') return 11 + + for (const key of Object.keys(VIDEO_CATEGORIES)) { + const category = VIDEO_CATEGORIES[key] + if (categoryString.toLowerCase() === category.toLowerCase()) return parseInt(key, 10) + } + + return undefined + } + + private getLanguage (language: string) { + return VIDEO_LANGUAGES[language] ? language : undefined + } +} diff --git a/server/server/helpers/youtube-dl/youtube-dl-wrapper.ts b/server/server/helpers/youtube-dl/youtube-dl-wrapper.ts new file mode 100644 index 000000000..d5acb371a --- /dev/null +++ b/server/server/helpers/youtube-dl/youtube-dl-wrapper.ts @@ -0,0 +1,156 @@ +import { move, pathExists, remove } from 'fs-extra/esm' +import { readdir } from 'fs/promises' +import { dirname, join } from 'path' +import { inspect } from 'util' +import { VideoResolutionType } from '@peertube/peertube-models' +import { CONFIG } from '@server/initializers/config.js' +import { isVideoFileExtnameValid } from '../custom-validators/videos.js' +import { logger, loggerTagsFactory } from '../logger.js' +import { generateVideoImportTmpPath } from '../utils.js' +import { YoutubeDLCLI } from './youtube-dl-cli.js' +import { YoutubeDLInfo, YoutubeDLInfoBuilder } from './youtube-dl-info-builder.js' + +const lTags = loggerTagsFactory('youtube-dl') + +export type YoutubeDLSubs = { + language: string + filename: string + path: string +}[] + +const processOptions = { + maxBuffer: 1024 * 1024 * 30 // 30MB +} + +class YoutubeDLWrapper { + + constructor ( + private readonly url: string, + private readonly enabledResolutions: VideoResolutionType[], + private readonly useBestFormat: boolean + ) { + + } + + async getInfoForDownload (youtubeDLArgs: string[] = []): Promise { + const youtubeDL = await YoutubeDLCLI.safeGet() + + const info = await youtubeDL.getInfo({ + url: this.url, + format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions, this.useBestFormat), + additionalYoutubeDLArgs: youtubeDLArgs, + processOptions + }) + + if (!info) throw new Error(`YoutubeDL could not get info from ${this.url}`) + + if (info.is_live === true) throw new Error('Cannot download a live streaming.') + + const infoBuilder = new YoutubeDLInfoBuilder(info) + + return infoBuilder.getInfo() + } + + async getInfoForListImport (options: { + latestVideosCount?: number + }) { + const youtubeDL = await YoutubeDLCLI.safeGet() + + const list = await youtubeDL.getListInfo({ + url: this.url, + latestVideosCount: options.latestVideosCount, + processOptions + }) + + if (!Array.isArray(list)) throw new Error(`YoutubeDL could not get list info from ${this.url}: ${inspect(list)}`) + + return list.map(info => info.webpage_url) + } + + async getSubtitles (): Promise { + const cwd = CONFIG.STORAGE.TMP_DIR + + const youtubeDL = await YoutubeDLCLI.safeGet() + + const files = await youtubeDL.getSubs({ url: this.url, format: 'vtt', processOptions: { cwd } }) + if (!files) return [] + + logger.debug('Get subtitles from youtube dl.', { url: this.url, files, ...lTags() }) + + const subtitles = files.reduce((acc, filename) => { + const matched = filename.match(/\.([a-z]{2})(-[a-z]+)?\.(vtt|ttml)/i) + if (!matched?.[1]) return acc + + return [ + ...acc, + { + language: matched[1], + path: join(cwd, filename), + filename + } + ] + }, []) + + return subtitles + } + + async downloadVideo (fileExt: string, timeout: number): Promise { + // Leave empty the extension, youtube-dl will add it + const pathWithoutExtension = generateVideoImportTmpPath(this.url, '') + + logger.info('Importing youtubeDL video %s to %s', this.url, pathWithoutExtension, lTags()) + + const youtubeDL = await YoutubeDLCLI.safeGet() + + try { + await youtubeDL.download({ + url: this.url, + format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions, this.useBestFormat), + output: pathWithoutExtension, + timeout, + processOptions + }) + + // If youtube-dl did not guess an extension for our file, just use .mp4 as default + if (await pathExists(pathWithoutExtension)) { + await move(pathWithoutExtension, pathWithoutExtension + '.mp4') + } + + return this.guessVideoPathWithExtension(pathWithoutExtension, fileExt) + } catch (err) { + this.guessVideoPathWithExtension(pathWithoutExtension, fileExt) + .then(path => { + logger.debug('Error in youtube-dl import, deleting file %s.', path, { err, ...lTags() }) + + return remove(path) + }) + .catch(innerErr => logger.error('Cannot remove file in youtubeDL error.', { innerErr, ...lTags() })) + + throw err + } + } + + private async guessVideoPathWithExtension (tmpPath: string, sourceExt: string) { + if (!isVideoFileExtnameValid(sourceExt)) { + throw new Error('Invalid video extension ' + sourceExt) + } + + const extensions = [ sourceExt, '.mp4', '.mkv', '.webm' ] + + for (const extension of extensions) { + const path = tmpPath + extension + + if (await pathExists(path)) return path + } + + const directoryContent = await readdir(dirname(tmpPath)) + + throw new Error(`Cannot guess path of ${tmpPath}. Directory content: ${directoryContent.join(', ')}`) + } +} + +// --------------------------------------------------------------------------- + +export { + YoutubeDLWrapper +} diff --git a/server/server/initializers/checker-after-init.ts b/server/server/initializers/checker-after-init.ts new file mode 100644 index 000000000..afcf6176b --- /dev/null +++ b/server/server/initializers/checker-after-init.ts @@ -0,0 +1,326 @@ +import config from 'config' +import { readFileSync, writeFileSync } from 'fs' +import { URL } from 'url' +import { uniqify } from '@peertube/peertube-core-utils' +import { getFFmpegVersion } from '@peertube/peertube-ffmpeg' +import { RecentlyAddedStrategy, VideoRedundancyConfigFilter } from '@peertube/peertube-models' +import { isProdInstance } from '@peertube/peertube-node-utils' +import { parseBytes, parseSemVersion } from '../helpers/core-utils.js' +import { isArray } from '../helpers/custom-validators/misc.js' +import { logger } from '../helpers/logger.js' +import { ApplicationModel, getServerActor } from '../models/application/application.js' +import { OAuthClientModel } from '../models/oauth/oauth-client.js' +import { UserModel } from '../models/user/user.js' +import { CONFIG, getLocalConfigFilePath, isEmailEnabled, reloadConfig } from './config.js' +import { WEBSERVER } from './constants.js' + +async function checkActivityPubUrls () { + const actor = await getServerActor() + + const parsed = new URL(actor.url) + if (WEBSERVER.HOST !== parsed.host) { + const NODE_ENV = config.util.getEnv('NODE_ENV') + const NODE_CONFIG_DIR = config.util.getEnv('NODE_CONFIG_DIR') + + logger.warn( + 'It seems PeerTube was started (and created some data) with another domain name. ' + + 'This means you will not be able to federate! ' + + 'Please use %s %s npm run update-host to fix this.', + NODE_CONFIG_DIR ? `NODE_CONFIG_DIR=${NODE_CONFIG_DIR}` : '', + NODE_ENV ? `NODE_ENV=${NODE_ENV}` : '' + ) + } +} + +// Some checks on configuration files or throw if there is an error +function checkConfig () { + + const configFiles = config.util.getConfigSources().map(s => s.name).join(' -> ') + logger.info('Using following configuration file hierarchy: %s.', configFiles) + + checkRemovedConfigKeys() + + checkSecretsConfig() + checkEmailConfig() + checkNSFWPolicyConfig() + checkLocalRedundancyConfig() + checkRemoteRedundancyConfig() + checkStorageConfig() + checkTranscodingConfig() + checkImportConfig() + checkBroadcastMessageConfig() + checkSearchConfig() + checkLiveConfig() + checkObjectStorageConfig() + checkVideoStudioConfig() +} + +// We get db by param to not import it in this file (import orders) +async function clientsExist () { + const totalClients = await OAuthClientModel.countTotal() + + return totalClients !== 0 +} + +// We get db by param to not import it in this file (import orders) +async function usersExist () { + const totalUsers = await UserModel.countTotal() + + return totalUsers !== 0 +} + +// We get db by param to not import it in this file (import orders) +async function applicationExist () { + const totalApplication = await ApplicationModel.countTotal() + + return totalApplication !== 0 +} + +async function checkFFmpegVersion () { + const version = await getFFmpegVersion() + const { major, minor, patch } = parseSemVersion(version) + + if (major < 4 || (major === 4 && minor < 1)) { + logger.warn('Your ffmpeg version (%s) is outdated. PeerTube supports ffmpeg >= 4.1. Please upgrade ffmpeg.', version) + } + + if (major === 4 && minor === 4 && patch === 0) { + logger.warn('There is a bug in ffmpeg 4.4.0 with HLS videos. Please upgrade ffmpeg.') + } +} + +// --------------------------------------------------------------------------- + +export { + checkConfig, + clientsExist, + checkFFmpegVersion, + usersExist, + applicationExist, + checkActivityPubUrls +} + +// --------------------------------------------------------------------------- + +function checkRemovedConfigKeys () { + // Moved configuration keys + if (config.has('services.csp-logger')) { + logger.warn('services.csp-logger configuration has been renamed to csp.report_uri. Please update your configuration file.') + } + + if (config.has('transcoding.webtorrent.enabled')) { + const localConfigPath = getLocalConfigFilePath() + + const content = readFileSync(localConfigPath, { encoding: 'utf-8' }) + if (!content.includes('"webtorrent"')) { + throw new Error('Please rename transcoding.webtorrent.enabled key to transcoding.web_videos.enabled in your configuration file') + } + + try { + logger.info( + 'Replacing "transcoding.webtorrent.enabled" key to "transcoding.web_videos.enabled" in your local configuration ' + localConfigPath + ) + + writeFileSync(localConfigPath, content.replace('"webtorrent"', '"web_videos"'), { encoding: 'utf-8' }) + + reloadConfig() + .catch(err => logger.error('Cannot reload configuration', { err })) + } catch (err) { + logger.error('Cannot write new configuration to file ' + localConfigPath, { err }) + } + } +} + +function checkSecretsConfig () { + if (!CONFIG.SECRETS.PEERTUBE) { + throw new Error('secrets.peertube is missing in config. Generate one using `openssl rand -hex 32`') + } +} + +function checkEmailConfig () { + if (!isEmailEnabled()) { + if (CONFIG.SIGNUP.ENABLED && CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION) { + throw new Error('SMTP is not configured but you require signup email verification.') + } + + if (CONFIG.SIGNUP.ENABLED && CONFIG.SIGNUP.REQUIRES_APPROVAL) { + // eslint-disable-next-line max-len + logger.warn('SMTP is not configured but signup approval is enabled: PeerTube will not be able to send an email to the user upon acceptance/rejection of the registration request') + } + + if (CONFIG.CONTACT_FORM.ENABLED) { + logger.warn('SMTP is not configured so the contact form will not work.') + } + } +} + +function checkNSFWPolicyConfig () { + const defaultNSFWPolicy = CONFIG.INSTANCE.DEFAULT_NSFW_POLICY + + const available = [ 'do_not_list', 'blur', 'display' ] + if (available.includes(defaultNSFWPolicy) === false) { + throw new Error('NSFW policy setting should be ' + available.join(' or ') + ' instead of ' + defaultNSFWPolicy) + } +} + +function checkLocalRedundancyConfig () { + const redundancyVideos = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES + + if (isArray(redundancyVideos)) { + const available = [ 'most-views', 'trending', 'recently-added' ] + + for (const r of redundancyVideos) { + if (available.includes(r.strategy) === false) { + throw new Error('Videos redundancy should have ' + available.join(' or ') + ' strategy instead of ' + r.strategy) + } + + // Lifetime should not be < 10 hours + if (isProdInstance() && r.minLifetime < 1000 * 3600 * 10) { + throw new Error('Video redundancy minimum lifetime should be >= 10 hours for strategy ' + r.strategy) + } + } + + const filtered = uniqify(redundancyVideos.map(r => r.strategy)) + if (filtered.length !== redundancyVideos.length) { + throw new Error('Redundancy video entries should have unique strategies') + } + + const recentlyAddedStrategy = redundancyVideos.find(r => r.strategy === 'recently-added') as RecentlyAddedStrategy + if (recentlyAddedStrategy && isNaN(recentlyAddedStrategy.minViews)) { + throw new Error('Min views in recently added strategy is not a number') + } + } else { + throw new Error('Videos redundancy should be an array (you must uncomment lines containing - too)') + } +} + +function checkRemoteRedundancyConfig () { + const acceptFrom = CONFIG.REMOTE_REDUNDANCY.VIDEOS.ACCEPT_FROM + const acceptFromValues = new Set([ 'nobody', 'anybody', 'followings' ]) + + if (acceptFromValues.has(acceptFrom) === false) { + throw new Error('remote_redundancy.videos.accept_from has an incorrect value') + } +} + +function checkStorageConfig () { + // Check storage directory locations + if (isProdInstance()) { + const configStorage = config.get<{ [ name: string ]: string }>('storage') + + for (const key of Object.keys(configStorage)) { + if (configStorage[key].startsWith('storage/')) { + logger.warn( + 'Directory of %s should not be in the production directory of PeerTube. Please check your production configuration file.', + key + ) + } + } + } + + if (CONFIG.STORAGE.WEB_VIDEOS_DIR === CONFIG.STORAGE.REDUNDANCY_DIR) { + logger.warn('Redundancy directory should be different than the videos folder.') + } +} + +function checkTranscodingConfig () { + if (CONFIG.TRANSCODING.ENABLED) { + if (CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED === false && CONFIG.TRANSCODING.HLS.ENABLED === false) { + throw new Error('You need to enable at least Web Video transcoding or HLS transcoding.') + } + + if (CONFIG.TRANSCODING.CONCURRENCY <= 0) { + throw new Error('Transcoding concurrency should be > 0') + } + } + + if (CONFIG.IMPORT.VIDEOS.HTTP.ENABLED || CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED) { + if (CONFIG.IMPORT.VIDEOS.CONCURRENCY <= 0) { + throw new Error('Video import concurrency should be > 0') + } + } +} + +function checkImportConfig () { + if (CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED && !CONFIG.IMPORT.VIDEOS.HTTP) { + throw new Error('You need to enable HTTP import to allow synchronization') + } +} + +function checkBroadcastMessageConfig () { + if (CONFIG.BROADCAST_MESSAGE.ENABLED) { + const currentLevel = CONFIG.BROADCAST_MESSAGE.LEVEL + const available = [ 'info', 'warning', 'error' ] + + if (available.includes(currentLevel) === false) { + throw new Error('Broadcast message level should be ' + available.join(' or ') + ' instead of ' + currentLevel) + } + } +} + +function checkSearchConfig () { + if (CONFIG.SEARCH.SEARCH_INDEX.ENABLED === true) { + if (CONFIG.SEARCH.REMOTE_URI.USERS === false) { + throw new Error('You cannot enable search index without enabling remote URI search for users.') + } + } +} + +function checkLiveConfig () { + if (CONFIG.LIVE.ENABLED === true) { + if (CONFIG.LIVE.ALLOW_REPLAY === true && CONFIG.TRANSCODING.ENABLED === false) { + throw new Error('Live allow replay cannot be enabled if transcoding is not enabled.') + } + + if (CONFIG.LIVE.RTMP.ENABLED === false && CONFIG.LIVE.RTMPS.ENABLED === false) { + throw new Error('You must enable at least RTMP or RTMPS') + } + + if (CONFIG.LIVE.RTMPS.ENABLED) { + if (!CONFIG.LIVE.RTMPS.KEY_FILE) { + throw new Error('You must specify a key file to enable RTMPS') + } + + if (!CONFIG.LIVE.RTMPS.CERT_FILE) { + throw new Error('You must specify a cert file to enable RTMPS') + } + } + } +} + +function checkObjectStorageConfig () { + if (CONFIG.OBJECT_STORAGE.ENABLED === true) { + + if (!CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BUCKET_NAME) { + throw new Error('videos_bucket should be set when object storage support is enabled.') + } + + if (!CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME) { + throw new Error('streaming_playlists_bucket should be set when object storage support is enabled.') + } + + if ( + CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BUCKET_NAME === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME && + CONFIG.OBJECT_STORAGE.WEB_VIDEOS.PREFIX === CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.PREFIX + ) { + if (CONFIG.OBJECT_STORAGE.WEB_VIDEOS.PREFIX === '') { + throw new Error('Object storage bucket prefixes should be set when the same bucket is used for both types of video.') + } + + throw new Error( + 'Object storage bucket prefixes should be set to different values when the same bucket is used for both types of video.' + ) + } + + if (CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART > parseBytes('250MB')) { + // eslint-disable-next-line max-len + logger.warn(`Object storage max upload part seems to have a big value (${CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART} bytes). Consider using a lower one (like 100MB).`) + } + } +} + +function checkVideoStudioConfig () { + if (CONFIG.VIDEO_STUDIO.ENABLED === true && CONFIG.TRANSCODING.ENABLED === false) { + throw new Error('Video studio cannot be enabled if transcoding is disabled') + } +} diff --git a/server/server/initializers/checker-before-init.ts b/server/server/initializers/checker-before-init.ts new file mode 100644 index 000000000..f33da0914 --- /dev/null +++ b/server/server/initializers/checker-before-init.ts @@ -0,0 +1,159 @@ +import config from 'config' +import { promisify0 } from '@peertube/peertube-core-utils' +import { parseSemVersion } from '../helpers/core-utils.js' +import { logger } from '../helpers/logger.js' + +// ONLY USE CORE MODULES IN THIS FILE! + +// Check the config files +function checkMissedConfig () { + const required = [ 'listen.port', 'listen.hostname', + 'webserver.https', 'webserver.hostname', 'webserver.port', + 'secrets.peertube', + 'trust_proxy', + 'oauth2.token_lifetime.access_token', 'oauth2.token_lifetime.refresh_token', + 'database.hostname', 'database.port', 'database.username', 'database.password', 'database.pool.max', + 'smtp.hostname', 'smtp.port', 'smtp.username', 'smtp.password', 'smtp.tls', 'smtp.from_address', + 'email.body.signature', 'email.subject.prefix', + 'storage.avatars', 'storage.web_videos', 'storage.logs', 'storage.previews', 'storage.thumbnails', 'storage.torrents', 'storage.cache', + 'storage.redundancy', 'storage.tmp', 'storage.streaming_playlists', 'storage.plugins', 'storage.well_known', + 'log.level', 'log.rotation.enabled', 'log.rotation.max_file_size', 'log.rotation.max_files', 'log.anonymize_ip', + 'log.log_ping_requests', 'log.log_tracker_unknown_infohash', 'log.prettify_sql', 'log.accept_client_log', + 'open_telemetry.metrics.enabled', 'open_telemetry.metrics.prometheus_exporter.hostname', + 'open_telemetry.metrics.prometheus_exporter.port', 'open_telemetry.tracing.enabled', 'open_telemetry.tracing.jaeger_exporter.endpoint', + 'open_telemetry.metrics.http_request_duration.enabled', + 'user.history.videos.enabled', 'user.video_quota', 'user.video_quota_daily', + 'video_channels.max_per_user', + 'csp.enabled', 'csp.report_only', 'csp.report_uri', + 'security.frameguard.enabled', 'security.powered_by_header.enabled', + 'cache.previews.size', 'cache.captions.size', 'cache.torrents.size', 'cache.storyboards.size', + 'admin.email', 'contact_form.enabled', + 'signup.enabled', 'signup.limit', 'signup.requires_approval', 'signup.requires_email_verification', 'signup.minimum_age', + 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist', + 'redundancy.videos.strategies', 'redundancy.videos.check_interval', + 'transcoding.enabled', 'transcoding.threads', 'transcoding.allow_additional_extensions', 'transcoding.web_videos.enabled', + 'transcoding.hls.enabled', 'transcoding.profile', 'transcoding.concurrency', + 'transcoding.resolutions.0p', 'transcoding.resolutions.144p', 'transcoding.resolutions.240p', 'transcoding.resolutions.360p', + 'transcoding.resolutions.480p', 'transcoding.resolutions.720p', 'transcoding.resolutions.1080p', 'transcoding.resolutions.1440p', + 'transcoding.resolutions.2160p', 'transcoding.always_transcode_original_resolution', 'transcoding.remote_runners.enabled', + 'video_studio.enabled', 'video_studio.remote_runners.enabled', + 'video_file.update.enabled', + 'remote_runners.stalled_jobs.vod', 'remote_runners.stalled_jobs.live', + 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout', + 'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user', + 'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization', + 'import.video_channel_synchronization.full_sync_videos_limit', + 'auto_blacklist.videos.of_users.enabled', 'trending.videos.interval_days', + 'client.videos.miniature.display_author_avatar', + 'client.videos.miniature.prefer_author_display_name', 'client.menu.login.redirect_on_single_external_auth', + 'defaults.publish.download_enabled', 'defaults.publish.comments_enabled', 'defaults.publish.privacy', 'defaults.publish.licence', + 'instance.name', 'instance.short_description', 'instance.description', 'instance.terms', 'instance.default_client_route', + 'instance.is_nsfw', 'instance.default_nsfw_policy', 'instance.robots', 'instance.securitytxt', + 'services.twitter.username', 'services.twitter.whitelisted', + 'followers.instance.enabled', 'followers.instance.manual_approval', + 'tracker.enabled', 'tracker.private', 'tracker.reject_too_many_announces', + 'history.videos.max_age', 'views.videos.remote.max_age', 'views.videos.local_buffer_update_interval', 'views.videos.ip_view_expiration', + 'rates_limit.api.window', 'rates_limit.api.max', 'rates_limit.login.window', 'rates_limit.login.max', + 'rates_limit.signup.window', 'rates_limit.signup.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max', + 'rates_limit.receive_client_log.window', 'rates_limit.receive_client_log.max', 'rates_limit.plugins.window', 'rates_limit.plugins.max', + 'rates_limit.well_known.window', 'rates_limit.well_known.max', 'rates_limit.feeds.window', 'rates_limit.feeds.max', + 'rates_limit.activity_pub.window', 'rates_limit.activity_pub.max', 'rates_limit.client.window', 'rates_limit.client.max', + 'static_files.private_files_require_auth', + 'object_storage.enabled', 'object_storage.endpoint', 'object_storage.region', 'object_storage.upload_acl.public', + 'object_storage.upload_acl.private', 'object_storage.proxy.proxify_private_files', 'object_storage.credentials.access_key_id', + 'object_storage.credentials.secret_access_key', 'object_storage.max_upload_part', 'object_storage.streaming_playlists.bucket_name', + 'object_storage.streaming_playlists.prefix', 'object_storage.streaming_playlists.base_url', 'object_storage.web_videos.bucket_name', + 'object_storage.web_videos.prefix', 'object_storage.web_videos.base_url', + 'theme.default', + 'feeds.videos.count', 'feeds.comments.count', + 'geo_ip.enabled', 'geo_ip.country.database_url', + 'remote_redundancy.videos.accept_from', + 'federation.videos.federate_unlisted', 'federation.videos.cleanup_remote_interactions', + 'peertube.check_latest_version.enabled', 'peertube.check_latest_version.url', + 'search.remote_uri.users', 'search.remote_uri.anonymous', 'search.search_index.enabled', 'search.search_index.url', + 'search.search_index.disable_local_search', 'search.search_index.is_default_search', + 'live.enabled', 'live.allow_replay', 'live.latency_setting.enabled', 'live.max_duration', + 'live.max_user_lives', 'live.max_instance_lives', + 'live.rtmp.enabled', 'live.rtmp.port', 'live.rtmp.hostname', 'live.rtmp.public_hostname', + 'live.rtmps.enabled', 'live.rtmps.port', 'live.rtmps.hostname', 'live.rtmps.public_hostname', + 'live.rtmps.key_file', 'live.rtmps.cert_file', + 'live.transcoding.enabled', 'live.transcoding.threads', 'live.transcoding.profile', + 'live.transcoding.resolutions.144p', 'live.transcoding.resolutions.240p', 'live.transcoding.resolutions.360p', + 'live.transcoding.resolutions.480p', 'live.transcoding.resolutions.720p', 'live.transcoding.resolutions.1080p', + 'live.transcoding.resolutions.1440p', 'live.transcoding.resolutions.2160p', 'live.transcoding.always_transcode_original_resolution', + 'live.transcoding.remote_runners.enabled' + ] + + const requiredAlternatives = [ + [ // set + [ 'redis.hostname', 'redis.port' ], // alternative + [ 'redis.socket' ], + [ 'redis.sentinel.master_name', 'redis.sentinel.sentinels[0].hostname', 'redis.sentinel.sentinels[0].port' ] + ] + ] + const miss: string[] = [] + + for (const key of required) { + if (!config.has(key)) { + miss.push(key) + } + } + + const redundancyVideos = config.get('redundancy.videos.strategies') + + if (Array.isArray(redundancyVideos)) { + for (const r of redundancyVideos) { + if (!r.size) miss.push('redundancy.videos.strategies.size') + if (!r.min_lifetime) miss.push('redundancy.videos.strategies.min_lifetime') + } + } + + const missingAlternatives = requiredAlternatives.filter( + set => !set.find(alternative => !alternative.find(key => !config.has(key))) + ) + + missingAlternatives + .forEach(set => set[0].forEach(key => miss.push(key))) + + return miss +} + +// Check the available codecs +// We get CONFIG by param to not import it in this file (import orders) +async function checkFFmpeg (CONFIG: { TRANSCODING: { ENABLED: boolean } }) { + if (CONFIG.TRANSCODING.ENABLED === false) return undefined + + const Ffmpeg = (await import('fluent-ffmpeg')).default + const getAvailableCodecsPromise = promisify0(Ffmpeg.getAvailableCodecs) + const codecs = await getAvailableCodecsPromise() + const canEncode = [ 'libx264' ] + + for (const codec of canEncode) { + if (codecs[codec] === undefined) { + throw new Error('Unknown codec ' + codec + ' in FFmpeg.') + } + + if (codecs[codec].canEncode !== true) { + throw new Error('Unavailable encode codec ' + codec + ' in FFmpeg') + } + } +} + +function checkNodeVersion () { + const v = process.version + const { major } = parseSemVersion(v) + + logger.debug('Checking NodeJS version %s.', v) + + if (major <= 12) { + throw new Error('Your NodeJS version ' + v + ' is not supported. Please upgrade.') + } +} + +// --------------------------------------------------------------------------- + +export { + checkFFmpeg, + checkMissedConfig, + checkNodeVersion +} diff --git a/server/server/initializers/config.ts b/server/server/initializers/config.ts new file mode 100644 index 000000000..dd196ea0c --- /dev/null +++ b/server/server/initializers/config.ts @@ -0,0 +1,699 @@ +import bytes from 'bytes' +import { IConfig } from 'config' +import { createRequire } from 'module' +import { dirname, join } from 'path' +import { + BroadcastMessageLevel, + NSFWPolicyType, + VideoPrivacyType, + VideoRedundancyConfigFilter, + VideosRedundancyStrategy +} from '@peertube/peertube-models' +import { decacheModule } from '@server/helpers/decache.js' +import { buildPath, root } from '@peertube/peertube-node-utils' +import { parseBytes, parseDurationToMs } from '../helpers/core-utils.js' + +const require = createRequire(import.meta.url) +let config: IConfig = require('config') + +const configChangedHandlers: Function[] = [] + +const CONFIG = { + CUSTOM_FILE: getLocalConfigFilePath(), + LISTEN: { + PORT: config.get('listen.port'), + HOSTNAME: config.get('listen.hostname') + }, + SECRETS: { + PEERTUBE: config.get('secrets.peertube') + }, + DATABASE: { + DBNAME: config.has('database.name') ? config.get('database.name') : 'peertube' + config.get('database.suffix'), + HOSTNAME: config.get('database.hostname'), + PORT: config.get('database.port'), + SSL: config.get('database.ssl'), + USERNAME: config.get('database.username'), + PASSWORD: config.get('database.password'), + POOL: { + MAX: config.get('database.pool.max') + } + }, + REDIS: { + HOSTNAME: config.has('redis.hostname') ? config.get('redis.hostname') : null, + PORT: config.has('redis.port') ? config.get('redis.port') : null, + SOCKET: config.has('redis.socket') ? config.get('redis.socket') : null, + AUTH: config.has('redis.auth') ? config.get('redis.auth') : null, + DB: config.has('redis.db') ? config.get('redis.db') : null, + SENTINEL: { + ENABLED: config.has('redis.sentinel.enabled') ? config.get('redis.sentinel.enabled') : false, + ENABLE_TLS: config.has('redis.sentinel.enable_tls') ? config.get('redis.sentinel.enable_tls') : false, + SENTINELS: config.has('redis.sentinel.sentinels') ? config.get<{ hostname: string, port: number }[]>('redis.sentinel.sentinels') : [], + MASTER_NAME: config.has('redis.sentinel.master_name') ? config.get('redis.sentinel.master_name') : null + } + }, + SMTP: { + TRANSPORT: config.has('smtp.transport') ? config.get('smtp.transport') : 'smtp', + SENDMAIL: config.has('smtp.sendmail') ? config.get('smtp.sendmail') : null, + HOSTNAME: config.get('smtp.hostname'), + PORT: config.get('smtp.port'), + USERNAME: config.get('smtp.username'), + PASSWORD: config.get('smtp.password'), + TLS: config.get('smtp.tls'), + DISABLE_STARTTLS: config.get('smtp.disable_starttls'), + CA_FILE: config.get('smtp.ca_file'), + FROM_ADDRESS: config.get('smtp.from_address') + }, + EMAIL: { + BODY: { + SIGNATURE: config.get('email.body.signature') + }, + SUBJECT: { + PREFIX: config.get('email.subject.prefix') + ' ' + } + }, + + CLIENT: { + VIDEOS: { + MINIATURE: { + get PREFER_AUTHOR_DISPLAY_NAME () { return config.get('client.videos.miniature.prefer_author_display_name') }, + get DISPLAY_AUTHOR_AVATAR () { return config.get('client.videos.miniature.display_author_avatar') } + }, + RESUMABLE_UPLOAD: { + get MAX_CHUNK_SIZE () { return parseBytes(config.get('client.videos.resumable_upload.max_chunk_size') || 0) } + } + }, + MENU: { + LOGIN: { + get REDIRECT_ON_SINGLE_EXTERNAL_AUTH () { return config.get('client.menu.login.redirect_on_single_external_auth') } + } + } + }, + + DEFAULTS: { + PUBLISH: { + DOWNLOAD_ENABLED: config.get('defaults.publish.download_enabled'), + COMMENTS_ENABLED: config.get('defaults.publish.comments_enabled'), + PRIVACY: config.get('defaults.publish.privacy'), + LICENCE: config.get('defaults.publish.licence') + }, + P2P: { + WEBAPP: { + ENABLED: config.get('defaults.p2p.webapp.enabled') + }, + EMBED: { + ENABLED: config.get('defaults.p2p.embed.enabled') + } + } + }, + + STORAGE: { + TMP_DIR: buildPath(config.get('storage.tmp')), + TMP_PERSISTENT_DIR: buildPath(config.get('storage.tmp_persistent')), + BIN_DIR: buildPath(config.get('storage.bin')), + ACTOR_IMAGES_DIR: buildPath(config.get('storage.avatars')), + LOG_DIR: buildPath(config.get('storage.logs')), + WEB_VIDEOS_DIR: buildPath(config.get('storage.web_videos')), + STREAMING_PLAYLISTS_DIR: buildPath(config.get('storage.streaming_playlists')), + REDUNDANCY_DIR: buildPath(config.get('storage.redundancy')), + THUMBNAILS_DIR: buildPath(config.get('storage.thumbnails')), + STORYBOARDS_DIR: buildPath(config.get('storage.storyboards')), + PREVIEWS_DIR: buildPath(config.get('storage.previews')), + CAPTIONS_DIR: buildPath(config.get('storage.captions')), + TORRENTS_DIR: buildPath(config.get('storage.torrents')), + CACHE_DIR: buildPath(config.get('storage.cache')), + PLUGINS_DIR: buildPath(config.get('storage.plugins')), + CLIENT_OVERRIDES_DIR: buildPath(config.get('storage.client_overrides')), + WELL_KNOWN_DIR: buildPath(config.get('storage.well_known')) + }, + STATIC_FILES: { + PRIVATE_FILES_REQUIRE_AUTH: config.get('static_files.private_files_require_auth') + }, + OBJECT_STORAGE: { + ENABLED: config.get('object_storage.enabled'), + MAX_UPLOAD_PART: bytes.parse(config.get('object_storage.max_upload_part')), + ENDPOINT: config.get('object_storage.endpoint'), + REGION: config.get('object_storage.region'), + UPLOAD_ACL: { + PUBLIC: config.get('object_storage.upload_acl.public'), + PRIVATE: config.get('object_storage.upload_acl.private') + }, + CREDENTIALS: { + ACCESS_KEY_ID: config.get('object_storage.credentials.access_key_id'), + SECRET_ACCESS_KEY: config.get('object_storage.credentials.secret_access_key') + }, + PROXY: { + PROXIFY_PRIVATE_FILES: config.get('object_storage.proxy.proxify_private_files') + }, + WEB_VIDEOS: { + BUCKET_NAME: config.get('object_storage.web_videos.bucket_name'), + PREFIX: config.get('object_storage.web_videos.prefix'), + BASE_URL: config.get('object_storage.web_videos.base_url') + }, + STREAMING_PLAYLISTS: { + BUCKET_NAME: config.get('object_storage.streaming_playlists.bucket_name'), + PREFIX: config.get('object_storage.streaming_playlists.prefix'), + BASE_URL: config.get('object_storage.streaming_playlists.base_url') + } + }, + WEBSERVER: { + SCHEME: config.get('webserver.https') === true ? 'https' : 'http', + WS: config.get('webserver.https') === true ? 'wss' : 'ws', + HOSTNAME: config.get('webserver.hostname'), + PORT: config.get('webserver.port') + }, + OAUTH2: { + TOKEN_LIFETIME: { + ACCESS_TOKEN: parseDurationToMs(config.get('oauth2.token_lifetime.access_token')), + REFRESH_TOKEN: parseDurationToMs(config.get('oauth2.token_lifetime.refresh_token')) + } + }, + RATES_LIMIT: { + API: { + WINDOW_MS: parseDurationToMs(config.get('rates_limit.api.window')), + MAX: config.get('rates_limit.api.max') + }, + SIGNUP: { + WINDOW_MS: parseDurationToMs(config.get('rates_limit.signup.window')), + MAX: config.get('rates_limit.signup.max') + }, + LOGIN: { + WINDOW_MS: parseDurationToMs(config.get('rates_limit.login.window')), + MAX: config.get('rates_limit.login.max') + }, + RECEIVE_CLIENT_LOG: { + WINDOW_MS: parseDurationToMs(config.get('rates_limit.receive_client_log.window')), + MAX: config.get('rates_limit.receive_client_log.max') + }, + ASK_SEND_EMAIL: { + WINDOW_MS: parseDurationToMs(config.get('rates_limit.ask_send_email.window')), + MAX: config.get('rates_limit.ask_send_email.max') + }, + PLUGINS: { + WINDOW_MS: parseDurationToMs(config.get('rates_limit.plugins.window')), + MAX: config.get('rates_limit.plugins.max') + }, + WELL_KNOWN: { + WINDOW_MS: parseDurationToMs(config.get('rates_limit.well_known.window')), + MAX: config.get('rates_limit.well_known.max') + }, + FEEDS: { + WINDOW_MS: parseDurationToMs(config.get('rates_limit.feeds.window')), + MAX: config.get('rates_limit.feeds.max') + }, + ACTIVITY_PUB: { + WINDOW_MS: parseDurationToMs(config.get('rates_limit.activity_pub.window')), + MAX: config.get('rates_limit.activity_pub.max') + }, + CLIENT: { + WINDOW_MS: parseDurationToMs(config.get('rates_limit.client.window')), + MAX: config.get('rates_limit.client.max') + } + }, + TRUST_PROXY: config.get('trust_proxy'), + LOG: { + LEVEL: config.get('log.level'), + ROTATION: { + ENABLED: config.get('log.rotation.enabled'), + MAX_FILE_SIZE: bytes.parse(config.get('log.rotation.max_file_size')), + MAX_FILES: config.get('log.rotation.max_files') + }, + ANONYMIZE_IP: config.get('log.anonymize_ip'), + LOG_PING_REQUESTS: config.get('log.log_ping_requests'), + LOG_TRACKER_UNKNOWN_INFOHASH: config.get('log.log_tracker_unknown_infohash'), + PRETTIFY_SQL: config.get('log.prettify_sql'), + ACCEPT_CLIENT_LOG: config.get('log.accept_client_log') + }, + OPEN_TELEMETRY: { + METRICS: { + ENABLED: config.get('open_telemetry.metrics.enabled'), + + HTTP_REQUEST_DURATION: { + ENABLED: config.get('open_telemetry.metrics.http_request_duration.enabled') + }, + + PROMETHEUS_EXPORTER: { + HOSTNAME: config.get('open_telemetry.metrics.prometheus_exporter.hostname'), + PORT: config.get('open_telemetry.metrics.prometheus_exporter.port') + } + }, + TRACING: { + ENABLED: config.get('open_telemetry.tracing.enabled'), + + JAEGER_EXPORTER: { + ENDPOINT: config.get('open_telemetry.tracing.jaeger_exporter.endpoint') + } + } + }, + TRENDING: { + VIDEOS: { + INTERVAL_DAYS: config.get('trending.videos.interval_days'), + ALGORITHMS: { + get ENABLED () { return config.get('trending.videos.algorithms.enabled') }, + get DEFAULT () { return config.get('trending.videos.algorithms.default') } + } + } + }, + REDUNDANCY: { + VIDEOS: { + CHECK_INTERVAL: parseDurationToMs(config.get('redundancy.videos.check_interval')), + STRATEGIES: buildVideosRedundancy(config.get('redundancy.videos.strategies')) + } + }, + REMOTE_REDUNDANCY: { + VIDEOS: { + ACCEPT_FROM: config.get('remote_redundancy.videos.accept_from') + } + }, + CSP: { + ENABLED: config.get('csp.enabled'), + REPORT_ONLY: config.get('csp.report_only'), + REPORT_URI: config.get('csp.report_uri') + }, + SECURITY: { + FRAMEGUARD: { + ENABLED: config.get('security.frameguard.enabled') + }, + POWERED_BY_HEADER: { + ENABLED: config.get('security.powered_by_header.enabled') + } + }, + TRACKER: { + ENABLED: config.get('tracker.enabled'), + PRIVATE: config.get('tracker.private'), + REJECT_TOO_MANY_ANNOUNCES: config.get('tracker.reject_too_many_announces') + }, + HISTORY: { + VIDEOS: { + MAX_AGE: parseDurationToMs(config.get('history.videos.max_age')) + } + }, + VIEWS: { + VIDEOS: { + REMOTE: { + MAX_AGE: parseDurationToMs(config.get('views.videos.remote.max_age')) + }, + LOCAL_BUFFER_UPDATE_INTERVAL: parseDurationToMs(config.get('views.videos.local_buffer_update_interval')), + IP_VIEW_EXPIRATION: parseDurationToMs(config.get('views.videos.ip_view_expiration')) + } + }, + GEO_IP: { + ENABLED: config.get('geo_ip.enabled'), + COUNTRY: { + DATABASE_URL: config.get('geo_ip.country.database_url') + } + }, + PLUGINS: { + INDEX: { + ENABLED: config.get('plugins.index.enabled'), + CHECK_LATEST_VERSIONS_INTERVAL: parseDurationToMs(config.get('plugins.index.check_latest_versions_interval')), + URL: config.get('plugins.index.url') + } + }, + FEDERATION: { + VIDEOS: { + FEDERATE_UNLISTED: config.get('federation.videos.federate_unlisted'), + CLEANUP_REMOTE_INTERACTIONS: config.get('federation.videos.cleanup_remote_interactions') + }, + SIGN_FEDERATED_FETCHES: config.get('federation.sign_federated_fetches') + }, + PEERTUBE: { + CHECK_LATEST_VERSION: { + ENABLED: config.get('peertube.check_latest_version.enabled'), + URL: config.get('peertube.check_latest_version.url') + } + }, + WEBADMIN: { + CONFIGURATION: { + EDITION: { + ALLOWED: config.get('webadmin.configuration.edition.allowed') + } + } + }, + FEEDS: { + VIDEOS: { + COUNT: config.get('feeds.videos.count') + }, + COMMENTS: { + COUNT: config.get('feeds.comments.count') + } + }, + REMOTE_RUNNERS: { + STALLED_JOBS: { + LIVE: parseDurationToMs(config.get('remote_runners.stalled_jobs.live')), + VOD: parseDurationToMs(config.get('remote_runners.stalled_jobs.vod')) + } + }, + ADMIN: { + get EMAIL () { return config.get('admin.email') } + }, + CONTACT_FORM: { + get ENABLED () { return config.get('contact_form.enabled') } + }, + SIGNUP: { + get ENABLED () { return config.get('signup.enabled') }, + get REQUIRES_APPROVAL () { return config.get('signup.requires_approval') }, + get LIMIT () { return config.get('signup.limit') }, + get REQUIRES_EMAIL_VERIFICATION () { return config.get('signup.requires_email_verification') }, + get MINIMUM_AGE () { return config.get('signup.minimum_age') }, + FILTERS: { + CIDR: { + get WHITELIST () { return config.get('signup.filters.cidr.whitelist') }, + get BLACKLIST () { return config.get('signup.filters.cidr.blacklist') } + } + } + }, + USER: { + HISTORY: { + VIDEOS: { + get ENABLED () { return config.get('user.history.videos.enabled') } + } + }, + get VIDEO_QUOTA () { return parseBytes(config.get('user.video_quota')) }, + get VIDEO_QUOTA_DAILY () { return parseBytes(config.get('user.video_quota_daily')) } + }, + VIDEO_CHANNELS: { + get MAX_PER_USER () { return config.get('video_channels.max_per_user') } + }, + TRANSCODING: { + get ENABLED () { return config.get('transcoding.enabled') }, + get ALLOW_ADDITIONAL_EXTENSIONS () { return config.get('transcoding.allow_additional_extensions') }, + get ALLOW_AUDIO_FILES () { return config.get('transcoding.allow_audio_files') }, + get THREADS () { return config.get('transcoding.threads') }, + get CONCURRENCY () { return config.get('transcoding.concurrency') }, + get PROFILE () { return config.get('transcoding.profile') }, + get ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION () { return config.get('transcoding.always_transcode_original_resolution') }, + RESOLUTIONS: { + get '0p' () { return config.get('transcoding.resolutions.0p') }, + get '144p' () { return config.get('transcoding.resolutions.144p') }, + get '240p' () { return config.get('transcoding.resolutions.240p') }, + get '360p' () { return config.get('transcoding.resolutions.360p') }, + get '480p' () { return config.get('transcoding.resolutions.480p') }, + get '720p' () { return config.get('transcoding.resolutions.720p') }, + get '1080p' () { return config.get('transcoding.resolutions.1080p') }, + get '1440p' () { return config.get('transcoding.resolutions.1440p') }, + get '2160p' () { return config.get('transcoding.resolutions.2160p') } + }, + HLS: { + get ENABLED () { return config.get('transcoding.hls.enabled') } + }, + WEB_VIDEOS: { + get ENABLED () { return config.get('transcoding.web_videos.enabled') } + }, + REMOTE_RUNNERS: { + get ENABLED () { return config.get('transcoding.remote_runners.enabled') } + } + }, + LIVE: { + get ENABLED () { return config.get('live.enabled') }, + + get MAX_DURATION () { return parseDurationToMs(config.get('live.max_duration')) }, + get MAX_INSTANCE_LIVES () { return config.get('live.max_instance_lives') }, + get MAX_USER_LIVES () { return config.get('live.max_user_lives') }, + + get ALLOW_REPLAY () { return config.get('live.allow_replay') }, + + LATENCY_SETTING: { + get ENABLED () { return config.get('live.latency_setting.enabled') } + }, + + RTMP: { + get ENABLED () { return config.get('live.rtmp.enabled') }, + get PORT () { return config.get('live.rtmp.port') }, + get HOSTNAME () { return config.get('live.rtmp.hostname') }, + get PUBLIC_HOSTNAME () { return config.get('live.rtmp.public_hostname') } + }, + + RTMPS: { + get ENABLED () { return config.get('live.rtmps.enabled') }, + get PORT () { return config.get('live.rtmps.port') }, + get HOSTNAME () { return config.get('live.rtmps.hostname') }, + get PUBLIC_HOSTNAME () { return config.get('live.rtmps.public_hostname') }, + get KEY_FILE () { return config.get('live.rtmps.key_file') }, + get CERT_FILE () { return config.get('live.rtmps.cert_file') } + }, + + TRANSCODING: { + get ENABLED () { return config.get('live.transcoding.enabled') }, + get THREADS () { return config.get('live.transcoding.threads') }, + get PROFILE () { return config.get('live.transcoding.profile') }, + + get ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION () { return config.get('live.transcoding.always_transcode_original_resolution') }, + + RESOLUTIONS: { + get '144p' () { return config.get('live.transcoding.resolutions.144p') }, + get '240p' () { return config.get('live.transcoding.resolutions.240p') }, + get '360p' () { return config.get('live.transcoding.resolutions.360p') }, + get '480p' () { return config.get('live.transcoding.resolutions.480p') }, + get '720p' () { return config.get('live.transcoding.resolutions.720p') }, + get '1080p' () { return config.get('live.transcoding.resolutions.1080p') }, + get '1440p' () { return config.get('live.transcoding.resolutions.1440p') }, + get '2160p' () { return config.get('live.transcoding.resolutions.2160p') } + }, + REMOTE_RUNNERS: { + get ENABLED () { return config.get('live.transcoding.remote_runners.enabled') } + } + } + }, + VIDEO_STUDIO: { + get ENABLED () { return config.get('video_studio.enabled') }, + REMOTE_RUNNERS: { + get ENABLED () { return config.get('video_studio.remote_runners.enabled') } + } + }, + VIDEO_FILE: { + UPDATE: { + get ENABLED () { return config.get('video_file.update.enabled') } + } + }, + IMPORT: { + VIDEOS: { + get CONCURRENCY () { return config.get('import.videos.concurrency') }, + get TIMEOUT () { return parseDurationToMs(config.get('import.videos.timeout')) }, + + HTTP: { + get ENABLED () { return config.get('import.videos.http.enabled') }, + + YOUTUBE_DL_RELEASE: { + get URL () { return config.get('import.videos.http.youtube_dl_release.url') }, + get NAME () { return config.get('import.videos.http.youtube_dl_release.name') }, + get PYTHON_PATH () { return config.get('import.videos.http.youtube_dl_release.python_path') } + }, + + get FORCE_IPV4 () { return config.get('import.videos.http.force_ipv4') } + }, + TORRENT: { + get ENABLED () { return config.get('import.videos.torrent.enabled') } + } + }, + VIDEO_CHANNEL_SYNCHRONIZATION: { + get ENABLED () { return config.get('import.video_channel_synchronization.enabled') }, + get MAX_PER_USER () { return config.get('import.video_channel_synchronization.max_per_user') }, + get CHECK_INTERVAL () { return parseDurationToMs(config.get('import.video_channel_synchronization.check_interval')) }, + get VIDEOS_LIMIT_PER_SYNCHRONIZATION () { + return config.get('import.video_channel_synchronization.videos_limit_per_synchronization') + }, + get FULL_SYNC_VIDEOS_LIMIT () { + return config.get('import.video_channel_synchronization.full_sync_videos_limit') + } + } + }, + AUTO_BLACKLIST: { + VIDEOS: { + OF_USERS: { + get ENABLED () { return config.get('auto_blacklist.videos.of_users.enabled') } + } + } + }, + CACHE: { + PREVIEWS: { + get SIZE () { return config.get('cache.previews.size') } + }, + VIDEO_CAPTIONS: { + get SIZE () { return config.get('cache.captions.size') } + }, + TORRENTS: { + get SIZE () { return config.get('cache.torrents.size') } + }, + STORYBOARDS: { + get SIZE () { return config.get('cache.storyboards.size') } + } + }, + INSTANCE: { + get NAME () { return config.get('instance.name') }, + get SHORT_DESCRIPTION () { return config.get('instance.short_description') }, + get DESCRIPTION () { return config.get('instance.description') }, + get TERMS () { return config.get('instance.terms') }, + get CODE_OF_CONDUCT () { return config.get('instance.code_of_conduct') }, + + get CREATION_REASON () { return config.get('instance.creation_reason') }, + + get MODERATION_INFORMATION () { return config.get('instance.moderation_information') }, + get ADMINISTRATOR () { return config.get('instance.administrator') }, + get MAINTENANCE_LIFETIME () { return config.get('instance.maintenance_lifetime') }, + get BUSINESS_MODEL () { return config.get('instance.business_model') }, + get HARDWARE_INFORMATION () { return config.get('instance.hardware_information') }, + + get LANGUAGES () { return config.get('instance.languages') || [] }, + get CATEGORIES () { return config.get('instance.categories') || [] }, + + get IS_NSFW () { return config.get('instance.is_nsfw') }, + get DEFAULT_NSFW_POLICY () { return config.get('instance.default_nsfw_policy') }, + + get DEFAULT_CLIENT_ROUTE () { return config.get('instance.default_client_route') }, + + CUSTOMIZATIONS: { + get JAVASCRIPT () { return config.get('instance.customizations.javascript') }, + get CSS () { return config.get('instance.customizations.css') } + }, + get ROBOTS () { return config.get('instance.robots') }, + get SECURITYTXT () { return config.get('instance.securitytxt') }, + get SECURITYTXT_CONTACT () { return config.get('admin.email') } + }, + SERVICES: { + TWITTER: { + get USERNAME () { return config.get('services.twitter.username') }, + get WHITELISTED () { return config.get('services.twitter.whitelisted') } + } + }, + FOLLOWERS: { + INSTANCE: { + get ENABLED () { return config.get('followers.instance.enabled') }, + get MANUAL_APPROVAL () { return config.get('followers.instance.manual_approval') } + } + }, + FOLLOWINGS: { + INSTANCE: { + AUTO_FOLLOW_BACK: { + get ENABLED () { + return config.get('followings.instance.auto_follow_back.enabled') + } + }, + AUTO_FOLLOW_INDEX: { + get ENABLED () { + return config.get('followings.instance.auto_follow_index.enabled') + }, + get INDEX_URL () { + return config.get('followings.instance.auto_follow_index.index_url') + } + } + } + }, + THEME: { + get DEFAULT () { return config.get('theme.default') } + }, + BROADCAST_MESSAGE: { + get ENABLED () { return config.get('broadcast_message.enabled') }, + get MESSAGE () { return config.get('broadcast_message.message') }, + get LEVEL () { return config.get('broadcast_message.level') }, + get DISMISSABLE () { return config.get('broadcast_message.dismissable') } + }, + SEARCH: { + REMOTE_URI: { + get USERS () { return config.get('search.remote_uri.users') }, + get ANONYMOUS () { return config.get('search.remote_uri.anonymous') } + }, + SEARCH_INDEX: { + get ENABLED () { return config.get('search.search_index.enabled') }, + get URL () { return config.get('search.search_index.url') }, + get DISABLE_LOCAL_SEARCH () { return config.get('search.search_index.disable_local_search') }, + get IS_DEFAULT_SEARCH () { return config.get('search.search_index.is_default_search') } + } + } + +} + +function registerConfigChangedHandler (fun: Function) { + configChangedHandlers.push(fun) +} + +function isEmailEnabled () { + if (CONFIG.SMTP.TRANSPORT === 'sendmail' && CONFIG.SMTP.SENDMAIL) return true + + if (CONFIG.SMTP.TRANSPORT === 'smtp' && CONFIG.SMTP.HOSTNAME && CONFIG.SMTP.PORT) return true + + return false +} + +function getLocalConfigFilePath () { + const localConfigDir = getLocalConfigDir() + + let filename = 'local' + if (process.env.NODE_ENV) filename += `-${process.env.NODE_ENV}` + if (process.env.NODE_APP_INSTANCE) filename += `-${process.env.NODE_APP_INSTANCE}` + + return join(localConfigDir, filename + '.json') +} + +function getConfigModule () { + return config +} + +// --------------------------------------------------------------------------- + +export { + CONFIG, + getConfigModule, + getLocalConfigFilePath, + registerConfigChangedHandler, + isEmailEnabled +} + +// --------------------------------------------------------------------------- + +function getLocalConfigDir () { + if (process.env.PEERTUBE_LOCAL_CONFIG) return process.env.PEERTUBE_LOCAL_CONFIG + + const configSources = config.util.getConfigSources() + if (configSources.length === 0) throw new Error('Invalid config source.') + + return dirname(configSources[0].name) +} + +function buildVideosRedundancy (objs: any[]): VideosRedundancyStrategy[] { + if (!objs) return [] + + if (!Array.isArray(objs)) return objs + + return objs.map(obj => { + return Object.assign({}, obj, { + minLifetime: parseDurationToMs(obj.min_lifetime), + size: bytes.parse(obj.size), + minViews: obj.min_views + }) + }) +} + +export function reloadConfig () { + + function getConfigDirectories () { + if (process.env.NODE_CONFIG_DIR) { + return process.env.NODE_CONFIG_DIR.split(':') + } + + return [ join(root(), 'config') ] + } + + function purge () { + const directories = getConfigDirectories() + + for (const fileName in require.cache) { + if (directories.some((dir) => fileName.includes(dir)) === false) { + continue + } + + delete require.cache[fileName] + } + + decacheModule(require, 'config') + } + + purge() + + config = require('config') + + for (const configChangedHandler of configChangedHandlers) { + configChangedHandler() + } + + return Promise.resolve() +} diff --git a/server/server/initializers/constants.ts b/server/server/initializers/constants.ts new file mode 100644 index 000000000..34392dbc8 --- /dev/null +++ b/server/server/initializers/constants.ts @@ -0,0 +1,1411 @@ +import { randomInt } from '@peertube/peertube-core-utils' +import { + AbuseState, + AbuseStateType, + ActivityPubActorType, + ActorImageType, + ActorImageType_Type, + FollowState, + JobType, + NSFWPolicyType, + RunnerJobState, + RunnerJobStateType, + UserRegistrationState, + UserRegistrationStateType, + VideoChannelSyncState, + VideoChannelSyncStateType, + VideoImportState, + VideoImportStateType, + VideoPlaylistPrivacy, + VideoPlaylistPrivacyType, + VideoPlaylistType, + VideoPlaylistType_Type, + VideoPrivacy, + VideoPrivacyType, + VideoRateType, + VideoResolution, + VideoState, + VideoStateType, + VideoTranscodingFPS +} from '@peertube/peertube-models' +import { isTestInstance, isTestOrDevInstance, root } from '@peertube/peertube-node-utils' +import { RepeatOptions } from 'bullmq' +import { Encoding, randomBytes } from 'crypto' +import { readJsonSync } from 'fs-extra/esm' +import invert from 'lodash-es/invert.js' +import { join } from 'path' +// Do not use barrels, remain constants as independent as possible +import { parseDurationToMs, sanitizeHost, sanitizeUrl } from '../helpers/core-utils.js' +import { CONFIG, registerConfigChangedHandler } from './config.js' + +// --------------------------------------------------------------------------- + +const LAST_MIGRATION_VERSION = 800 + +// --------------------------------------------------------------------------- + +const API_VERSION = 'v1' +const PEERTUBE_VERSION: string = readJsonSync(join(root(), 'package.json')).version + +const PAGINATION = { + GLOBAL: { + COUNT: { + DEFAULT: 15, + MAX: 100 + } + }, + OUTBOX: { + COUNT: { + MAX: 50 + } + } +} + +const WEBSERVER = { + URL: '', + HOST: '', + SCHEME: '', + WS: '', + HOSTNAME: '', + PORT: 0, + + RTMP_URL: '', + RTMPS_URL: '', + + RTMP_BASE_LIVE_URL: '', + RTMPS_BASE_LIVE_URL: '' +} + +// Sortable columns per schema +const SORTABLE_COLUMNS = { + ADMIN_USERS: [ 'id', 'username', 'videoQuotaUsed', 'createdAt', 'lastLoginDate', 'role' ], + USER_SUBSCRIPTIONS: [ 'id', 'createdAt' ], + ACCOUNTS: [ 'createdAt' ], + JOBS: [ 'createdAt' ], + VIDEO_CHANNELS: [ 'id', 'name', 'updatedAt', 'createdAt' ], + VIDEO_IMPORTS: [ 'createdAt' ], + VIDEO_CHANNEL_SYNCS: [ 'externalChannelUrl', 'videoChannel', 'createdAt', 'lastSyncAt', 'state' ], + + VIDEO_COMMENT_THREADS: [ 'createdAt', 'totalReplies' ], + VIDEO_COMMENTS: [ 'createdAt' ], + + VIDEO_PASSWORDS: [ 'createdAt' ], + + VIDEO_RATES: [ 'createdAt' ], + BLACKLISTS: [ 'id', 'name', 'duration', 'views', 'likes', 'dislikes', 'uuid', 'createdAt' ], + + INSTANCE_FOLLOWERS: [ 'createdAt', 'state', 'score' ], + INSTANCE_FOLLOWING: [ 'createdAt', 'redundancyAllowed', 'state' ], + ACCOUNT_FOLLOWERS: [ 'createdAt' ], + CHANNEL_FOLLOWERS: [ 'createdAt' ], + + USER_REGISTRATIONS: [ 'createdAt', 'state' ], + + RUNNERS: [ 'createdAt' ], + RUNNER_REGISTRATION_TOKENS: [ 'createdAt' ], + RUNNER_JOBS: [ 'updatedAt', 'createdAt', 'priority', 'state', 'progress' ], + + VIDEOS: [ 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes', 'trending', 'hot', 'best' ], + + // Don't forget to update peertube-search-index with the same values + VIDEOS_SEARCH: [ 'name', 'duration', 'createdAt', 'publishedAt', 'originallyPublishedAt', 'views', 'likes', 'match' ], + VIDEO_CHANNELS_SEARCH: [ 'match', 'displayName', 'createdAt' ], + VIDEO_PLAYLISTS_SEARCH: [ 'match', 'displayName', 'createdAt' ], + + ABUSES: [ 'id', 'createdAt', 'state' ], + + ACCOUNTS_BLOCKLIST: [ 'createdAt' ], + SERVERS_BLOCKLIST: [ 'createdAt' ], + + USER_NOTIFICATIONS: [ 'createdAt', 'read' ], + + VIDEO_PLAYLISTS: [ 'name', 'displayName', 'createdAt', 'updatedAt' ], + + PLUGINS: [ 'name', 'createdAt', 'updatedAt' ], + + AVAILABLE_PLUGINS: [ 'npmName', 'popularity' ], + + VIDEO_REDUNDANCIES: [ 'name' ] +} + +const ROUTE_CACHE_LIFETIME = { + FEEDS: '15 minutes', + ROBOTS: '2 hours', + SITEMAP: '1 day', + SECURITYTXT: '2 hours', + NODEINFO: '10 minutes', + DNT_POLICY: '1 week', + ACTIVITY_PUB: { + VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example + }, + STATS: '4 hours', + WELL_KNOWN: '1 day' +} + +// --------------------------------------------------------------------------- + +// Number of points we add/remove after a successful/bad request +const ACTOR_FOLLOW_SCORE = { + PENALTY: -10, + BONUS: 10, + BASE: 1000, + MAX: 10000 +} + +const FOLLOW_STATES: { [ id: string ]: FollowState } = { + PENDING: 'pending', + ACCEPTED: 'accepted', + REJECTED: 'rejected' +} + +const REMOTE_SCHEME = { + HTTP: 'https', + WS: 'wss' +} + +// --------------------------------------------------------------------------- + +const JOB_ATTEMPTS: { [id in JobType]: number } = { + 'activitypub-http-broadcast': 1, + 'activitypub-http-broadcast-parallel': 1, + 'activitypub-http-unicast': 1, + 'activitypub-http-fetcher': 2, + 'activitypub-follow': 5, + 'activitypub-cleaner': 1, + 'video-file-import': 1, + 'video-transcoding': 1, + 'video-import': 1, + 'email': 5, + 'actor-keys': 3, + 'videos-views-stats': 1, + 'activitypub-refresher': 1, + 'video-redundancy': 1, + 'video-live-ending': 1, + 'video-studio-edition': 1, + 'manage-video-torrent': 1, + 'video-channel-import': 1, + 'after-video-channel-import': 1, + 'move-to-object-storage': 3, + 'transcoding-job-builder': 1, + 'generate-video-storyboard': 1, + 'notify': 1, + 'federate-video': 1 +} +// Excluded keys are jobs that can be configured by admins +const JOB_CONCURRENCY: { [id in Exclude]: number } = { + 'activitypub-http-broadcast': 1, + 'activitypub-http-broadcast-parallel': 30, + 'activitypub-http-unicast': 30, + 'activitypub-http-fetcher': 3, + 'activitypub-cleaner': 1, + 'activitypub-follow': 1, + 'video-file-import': 1, + 'email': 5, + 'actor-keys': 1, + 'videos-views-stats': 1, + 'activitypub-refresher': 1, + 'video-redundancy': 1, + 'video-live-ending': 10, + 'video-studio-edition': 1, + 'manage-video-torrent': 1, + 'move-to-object-storage': 1, + 'video-channel-import': 1, + 'after-video-channel-import': 1, + 'transcoding-job-builder': 1, + 'generate-video-storyboard': 1, + 'notify': 5, + 'federate-video': 3 +} +const JOB_TTL: { [id in JobType]: number } = { + 'activitypub-http-broadcast': 60000 * 10, // 10 minutes + 'activitypub-http-broadcast-parallel': 60000 * 10, // 10 minutes + 'activitypub-http-unicast': 60000 * 10, // 10 minutes + 'activitypub-http-fetcher': 1000 * 3600 * 10, // 10 hours + 'activitypub-follow': 60000 * 10, // 10 minutes + 'activitypub-cleaner': 1000 * 3600, // 1 hour + 'video-file-import': 1000 * 3600, // 1 hour + 'video-transcoding': 1000 * 3600 * 48, // 2 days, transcoding could be long + 'video-studio-edition': 1000 * 3600 * 10, // 10 hours + 'video-import': CONFIG.IMPORT.VIDEOS.TIMEOUT, + 'email': 60000 * 10, // 10 minutes + 'actor-keys': 60000 * 20, // 20 minutes + 'videos-views-stats': undefined, // Unlimited + 'activitypub-refresher': 60000 * 10, // 10 minutes + 'video-redundancy': 1000 * 3600 * 3, // 3 hours + 'video-live-ending': 1000 * 60 * 10, // 10 minutes + 'generate-video-storyboard': 1000 * 60 * 10, // 10 minutes + 'manage-video-torrent': 1000 * 3600 * 3, // 3 hours + 'move-to-object-storage': 1000 * 60 * 60 * 3, // 3 hours + 'video-channel-import': 1000 * 60 * 60 * 4, // 4 hours + 'after-video-channel-import': 60000 * 5, // 5 minutes + 'transcoding-job-builder': 60000, // 1 minute + 'notify': 60000 * 5, // 5 minutes + 'federate-video': 60000 * 5 // 5 minutes +} +const REPEAT_JOBS: { [ id in JobType ]?: RepeatOptions } = { + 'videos-views-stats': { + pattern: randomInt(1, 20) + ' * * * *' // Between 1-20 minutes past the hour + }, + 'activitypub-cleaner': { + pattern: '30 5 * * ' + randomInt(0, 7) // 1 time per week (random day) at 5:30 AM + } +} +const JOB_PRIORITY = { + TRANSCODING: 100, + VIDEO_STUDIO: 150 +} + +const JOB_REMOVAL_OPTIONS = { + COUNT: 10000, // Max jobs to store + + SUCCESS: { // Success jobs + 'DEFAULT': parseDurationToMs('2 days'), + + 'activitypub-http-broadcast-parallel': parseDurationToMs('10 minutes'), + 'activitypub-http-unicast': parseDurationToMs('1 hour'), + 'videos-views-stats': parseDurationToMs('3 hours'), + 'activitypub-refresher': parseDurationToMs('10 hours') + }, + + FAILURE: { // Failed job + DEFAULT: parseDurationToMs('7 days') + } +} + +const VIDEO_IMPORT_TIMEOUT = Math.floor(JOB_TTL['video-import'] * 0.9) + +const RUNNER_JOBS = { + MAX_FAILURES: 5, + LAST_CONTACT_UPDATE_INTERVAL: 30000 +} + +// --------------------------------------------------------------------------- + +const BROADCAST_CONCURRENCY = 30 // How many requests in parallel we do in activitypub-http-broadcast job +const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...) + +const AP_CLEANER = { + CONCURRENCY: 10, // How many requests in parallel we do in activitypub-cleaner job + UNAVAILABLE_TRESHOLD: 3, // How many attempts we do before removing an unavailable remote resource + PERIOD: parseDurationToMs('1 week') // /!\ Has to be sync with REPEAT_JOBS +} + +const REQUEST_TIMEOUTS = { + DEFAULT: 7000, // 7 seconds + FILE: 30000, // 30 seconds + REDUNDANCY: JOB_TTL['video-redundancy'] +} + +const SCHEDULER_INTERVALS_MS = { + RUNNER_JOB_WATCH_DOG: Math.min(CONFIG.REMOTE_RUNNERS.STALLED_JOBS.VOD, CONFIG.REMOTE_RUNNERS.STALLED_JOBS.LIVE), + ACTOR_FOLLOW_SCORES: 60000 * 60, // 1 hour + REMOVE_OLD_JOBS: 60000 * 60, // 1 hour + UPDATE_VIDEOS: 60000, // 1 minute + YOUTUBE_DL_UPDATE: 60000 * 60 * 24, // 1 day + GEO_IP_UPDATE: 60000 * 60 * 24, // 1 day + VIDEO_VIEWS_BUFFER_UPDATE: CONFIG.VIEWS.VIDEOS.LOCAL_BUFFER_UPDATE_INTERVAL, + CHECK_PLUGINS: CONFIG.PLUGINS.INDEX.CHECK_LATEST_VERSIONS_INTERVAL, + CHECK_PEERTUBE_VERSION: 60000 * 60 * 24, // 1 day + AUTO_FOLLOW_INDEX_INSTANCES: 60000 * 60 * 24, // 1 day + REMOVE_OLD_VIEWS: 60000 * 60 * 24, // 1 day + REMOVE_OLD_HISTORY: 60000 * 60 * 24, // 1 day + UPDATE_INBOX_STATS: 1000 * 60, // 1 minute + REMOVE_DANGLING_RESUMABLE_UPLOADS: 60000 * 60, // 1 hour + CHANNEL_SYNC_CHECK_INTERVAL: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.CHECK_INTERVAL +} + +// --------------------------------------------------------------------------- + +const CONSTRAINTS_FIELDS = { + USERS: { + NAME: { min: 1, max: 120 }, // Length + DESCRIPTION: { min: 3, max: 1000 }, // Length + USERNAME: { min: 1, max: 50 }, // Length + PASSWORD: { min: 6, max: 255 }, // Length + VIDEO_QUOTA: { min: -1 }, + VIDEO_QUOTA_DAILY: { min: -1 }, + VIDEO_LANGUAGES: { max: 500 }, // Array length + BLOCKED_REASON: { min: 3, max: 250 } // Length + }, + ABUSES: { + REASON: { min: 2, max: 3000 }, // Length + MODERATION_COMMENT: { min: 2, max: 3000 } // Length + }, + ABUSE_MESSAGES: { + MESSAGE: { min: 2, max: 3000 } // Length + }, + USER_REGISTRATIONS: { + REASON_MESSAGE: { min: 2, max: 3000 }, // Length + MODERATOR_MESSAGE: { min: 2, max: 3000 } // Length + }, + VIDEO_BLACKLIST: { + REASON: { min: 2, max: 300 } // Length + }, + VIDEO_CHANNELS: { + NAME: { min: 1, max: 120 }, // Length + DESCRIPTION: { min: 3, max: 1000 }, // Length + SUPPORT: { min: 3, max: 1000 }, // Length + EXTERNAL_CHANNEL_URL: { min: 3, max: 2000 }, // Length + URL: { min: 3, max: 2000 } // Length + }, + VIDEO_CHANNEL_SYNCS: { + EXTERNAL_CHANNEL_URL: { min: 3, max: 2000 } // Length + }, + VIDEO_CAPTIONS: { + CAPTION_FILE: { + EXTNAME: [ '.vtt', '.srt' ], + FILE_SIZE: { + max: 20 * 1024 * 1024 // 20MB + } + } + }, + VIDEO_IMPORTS: { + URL: { min: 3, max: 2000 }, // Length + TORRENT_NAME: { min: 3, max: 255 }, // Length + TORRENT_FILE: { + EXTNAME: [ '.torrent' ], + FILE_SIZE: { + max: 1024 * 200 // 200 KB + } + } + }, + VIDEOS_REDUNDANCY: { + URL: { min: 3, max: 2000 } // Length + }, + VIDEO_RATES: { + URL: { min: 3, max: 2000 } // Length + }, + VIDEOS: { + NAME: { min: 3, max: 120 }, // Length + LANGUAGE: { min: 1, max: 10 }, // Length + TRUNCATED_DESCRIPTION: { min: 3, max: 250 }, // Length + DESCRIPTION: { min: 3, max: 10000 }, // Length + SUPPORT: { min: 3, max: 1000 }, // Length + IMAGE: { + EXTNAME: [ '.png', '.jpg', '.jpeg', '.webp' ], + FILE_SIZE: { + max: 4 * 1024 * 1024 // 4MB + } + }, + EXTNAME: [] as string[], + INFO_HASH: { min: 40, max: 40 }, // Length, info hash is 20 bytes length but we represent it in hexadecimal so 20 * 2 + DURATION: { min: 0 }, // Number + TAGS: { min: 0, max: 5 }, // Number of total tags + TAG: { min: 2, max: 30 }, // Length + VIEWS: { min: 0 }, + LIKES: { min: 0 }, + DISLIKES: { min: 0 }, + FILE_SIZE: { min: -1 }, + PARTIAL_UPLOAD_SIZE: { max: 50 * 1024 * 1024 * 1024 }, // 50GB + URL: { min: 3, max: 2000 } // Length + }, + VIDEO_PLAYLISTS: { + NAME: { min: 1, max: 120 }, // Length + DESCRIPTION: { min: 3, max: 1000 }, // Length + URL: { min: 3, max: 2000 }, // Length + IMAGE: { + EXTNAME: [ '.jpg', '.jpeg' ], + FILE_SIZE: { + max: 4 * 1024 * 1024 // 4MB + } + } + }, + ACTORS: { + PUBLIC_KEY: { min: 10, max: 5000 }, // Length + PRIVATE_KEY: { min: 10, max: 5000 }, // Length + URL: { min: 3, max: 2000 }, // Length + IMAGE: { + EXTNAME: [ '.png', '.jpeg', '.jpg', '.gif', '.webp' ], + FILE_SIZE: { + max: 4 * 1024 * 1024 // 4MB + } + } + }, + VIDEO_EVENTS: { + COUNT: { min: 0 } + }, + VIDEO_COMMENTS: { + TEXT: { min: 1, max: 10000 }, // Length + URL: { min: 3, max: 2000 } // Length + }, + VIDEO_SHARE: { + URL: { min: 3, max: 2000 } // Length + }, + CONTACT_FORM: { + FROM_NAME: { min: 1, max: 120 }, // Length + BODY: { min: 3, max: 5000 } // Length + }, + PLUGINS: { + NAME: { min: 1, max: 214 }, // Length + DESCRIPTION: { min: 1, max: 20000 } // Length + }, + COMMONS: { + URL: { min: 5, max: 2000 } // Length + }, + VIDEO_STUDIO: { + TASKS: { min: 1, max: 10 }, // Number of tasks + CUT_TIME: { min: 0 } // Value + }, + LOGS: { + CLIENT_MESSAGE: { min: 1, max: 1000 }, // Length + CLIENT_STACK_TRACE: { min: 1, max: 15000 }, // Length + CLIENT_META: { min: 1, max: 5000 }, // Length + CLIENT_USER_AGENT: { min: 1, max: 200 } // Length + }, + RUNNERS: { + TOKEN: { min: 1, max: 1000 }, // Length + NAME: { min: 1, max: 100 }, // Length + DESCRIPTION: { min: 1, max: 1000 } // Length + }, + RUNNER_JOBS: { + TOKEN: { min: 1, max: 1000 }, // Length + REASON: { min: 1, max: 5000 }, // Length + ERROR_MESSAGE: { min: 1, max: 5000 }, // Length + PROGRESS: { min: 0, max: 100 } // Value + }, + VIDEO_PASSWORD: { + LENGTH: { min: 2, max: 100 } + } +} + +const VIEW_LIFETIME = { + VIEW: CONFIG.VIEWS.VIDEOS.IP_VIEW_EXPIRATION, + VIEWER_COUNTER: 60000 * 2, // 2 minutes + VIEWER_STATS: 60000 * 60 // 1 hour +} + +const MAX_LOCAL_VIEWER_WATCH_SECTIONS = 100 + +let CONTACT_FORM_LIFETIME = 60000 * 60 // 1 hour + +const VIDEO_TRANSCODING_FPS: VideoTranscodingFPS = { + MIN: 1, + STANDARD: [ 24, 25, 30 ], + HD_STANDARD: [ 50, 60 ], + AUDIO_MERGE: 25, + AVERAGE: 30, + MAX: 60, + KEEP_ORIGIN_FPS_RESOLUTION_MIN: 720 // We keep the original FPS on high resolutions (720 minimum) +} + +const DEFAULT_AUDIO_RESOLUTION = VideoResolution.H_480P + +const VIDEO_RATE_TYPES: { [ id: string ]: VideoRateType } = { + LIKE: 'like', + DISLIKE: 'dislike' +} + +const FFMPEG_NICE = { + // parent process defaults to niceness = 0 + // reminder: lower = higher priority, max value is 19, lowest is -20 + LIVE: 5, // prioritize over VOD and THUMBNAIL + THUMBNAIL: 10, + VOD: 15 +} + +const VIDEO_CATEGORIES = { + 1: 'Music', + 2: 'Films', + 3: 'Vehicles', + 4: 'Art', + 5: 'Sports', + 6: 'Travels', + 7: 'Gaming', + 8: 'People', + 9: 'Comedy', + 10: 'Entertainment', + 11: 'News & Politics', + 12: 'How To', + 13: 'Education', + 14: 'Activism', + 15: 'Science & Technology', + 16: 'Animals', + 17: 'Kids', + 18: 'Food' +} + +// See https://creativecommons.org/licenses/?lang=en +const VIDEO_LICENCES = { + 1: 'Attribution', + 2: 'Attribution - Share Alike', + 3: 'Attribution - No Derivatives', + 4: 'Attribution - Non Commercial', + 5: 'Attribution - Non Commercial - Share Alike', + 6: 'Attribution - Non Commercial - No Derivatives', + 7: 'Public Domain Dedication' +} + +const VIDEO_LANGUAGES: { [id: string]: string } = {} + +const VIDEO_PRIVACIES: { [ id in VideoPrivacyType ]: string } = { + [VideoPrivacy.PUBLIC]: 'Public', + [VideoPrivacy.UNLISTED]: 'Unlisted', + [VideoPrivacy.PRIVATE]: 'Private', + [VideoPrivacy.INTERNAL]: 'Internal', + [VideoPrivacy.PASSWORD_PROTECTED]: 'Password protected' +} + +const VIDEO_STATES: { [ id in VideoStateType ]: string } = { + [VideoState.PUBLISHED]: 'Published', + [VideoState.TO_TRANSCODE]: 'To transcode', + [VideoState.TO_IMPORT]: 'To import', + [VideoState.WAITING_FOR_LIVE]: 'Waiting for livestream', + [VideoState.LIVE_ENDED]: 'Livestream ended', + [VideoState.TO_MOVE_TO_EXTERNAL_STORAGE]: 'To move to an external storage', + [VideoState.TRANSCODING_FAILED]: 'Transcoding failed', + [VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED]: 'External storage move failed', + [VideoState.TO_EDIT]: 'To edit*' +} + +const VIDEO_IMPORT_STATES: { [ id in VideoImportStateType ]: string } = { + [VideoImportState.FAILED]: 'Failed', + [VideoImportState.PENDING]: 'Pending', + [VideoImportState.SUCCESS]: 'Success', + [VideoImportState.REJECTED]: 'Rejected', + [VideoImportState.CANCELLED]: 'Cancelled', + [VideoImportState.PROCESSING]: 'Processing' +} + +const VIDEO_CHANNEL_SYNC_STATE: { [ id in VideoChannelSyncStateType ]: string } = { + [VideoChannelSyncState.FAILED]: 'Failed', + [VideoChannelSyncState.SYNCED]: 'Synchronized', + [VideoChannelSyncState.PROCESSING]: 'Processing', + [VideoChannelSyncState.WAITING_FIRST_RUN]: 'Waiting first run' +} + +const ABUSE_STATES: { [ id in AbuseStateType ]: string } = { + [AbuseState.PENDING]: 'Pending', + [AbuseState.REJECTED]: 'Rejected', + [AbuseState.ACCEPTED]: 'Accepted' +} + +const USER_REGISTRATION_STATES: { [ id in UserRegistrationStateType ]: string } = { + [UserRegistrationState.PENDING]: 'Pending', + [UserRegistrationState.REJECTED]: 'Rejected', + [UserRegistrationState.ACCEPTED]: 'Accepted' +} + +const VIDEO_PLAYLIST_PRIVACIES: { [ id in VideoPlaylistPrivacyType ]: string } = { + [VideoPlaylistPrivacy.PUBLIC]: 'Public', + [VideoPlaylistPrivacy.UNLISTED]: 'Unlisted', + [VideoPlaylistPrivacy.PRIVATE]: 'Private' +} + +const VIDEO_PLAYLIST_TYPES: { [ id in VideoPlaylistType_Type ]: string } = { + [VideoPlaylistType.REGULAR]: 'Regular', + [VideoPlaylistType.WATCH_LATER]: 'Watch later' +} + +const RUNNER_JOB_STATES: { [ id in RunnerJobStateType ]: string } = { + [RunnerJobState.PROCESSING]: 'Processing', + [RunnerJobState.COMPLETED]: 'Completed', + [RunnerJobState.COMPLETING]: 'Completing', + [RunnerJobState.PENDING]: 'Pending', + [RunnerJobState.ERRORED]: 'Errored', + [RunnerJobState.WAITING_FOR_PARENT_JOB]: 'Waiting for parent job to finish', + [RunnerJobState.CANCELLED]: 'Cancelled', + [RunnerJobState.PARENT_ERRORED]: 'Parent job failed', + [RunnerJobState.PARENT_CANCELLED]: 'Parent job cancelled' +} + +const MIMETYPES = { + AUDIO: { + MIMETYPE_EXT: { + 'audio/mpeg': '.mp3', + 'audio/mp3': '.mp3', + + 'application/ogg': '.ogg', + 'audio/ogg': '.ogg', + + 'audio/x-ms-wma': '.wma', + 'audio/wav': '.wav', + 'audio/x-wav': '.wav', + + 'audio/x-flac': '.flac', + 'audio/flac': '.flac', + + 'audio/vnd.dlna.adts': '.aac', + 'audio/aac': '.aac', + + 'audio/m4a': '.m4a', + 'audio/mp4': '.m4a', + 'audio/x-m4a': '.m4a', + + 'audio/vnd.dolby.dd-raw': '.ac3', + 'audio/ac3': '.ac3' + }, + EXT_MIMETYPE: null as { [ id: string ]: string } + }, + VIDEO: { + MIMETYPE_EXT: null as { [ id: string ]: string | string[] }, + MIMETYPES_REGEX: null as string, + EXT_MIMETYPE: null as { [ id: string ]: string } + }, + IMAGE: { + MIMETYPE_EXT: { + 'image/png': '.png', + 'image/gif': '.gif', + 'image/webp': '.webp', + 'image/jpg': '.jpg', + 'image/jpeg': '.jpg' + }, + EXT_MIMETYPE: null as { [ id: string ]: string } + }, + VIDEO_CAPTIONS: { + MIMETYPE_EXT: { + 'text/vtt': '.vtt', + 'application/x-subrip': '.srt', + 'text/plain': '.srt' + }, + EXT_MIMETYPE: null as { [ id: string ]: string } + }, + TORRENT: { + MIMETYPE_EXT: { + 'application/x-bittorrent': '.torrent' + } + }, + M3U8: { + MIMETYPE_EXT: { + 'application/vnd.apple.mpegurl': '.m3u8' + } + } +} +MIMETYPES.AUDIO.EXT_MIMETYPE = invert(MIMETYPES.AUDIO.MIMETYPE_EXT) +MIMETYPES.IMAGE.EXT_MIMETYPE = invert(MIMETYPES.IMAGE.MIMETYPE_EXT) +MIMETYPES.VIDEO_CAPTIONS.EXT_MIMETYPE = invert(MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT) + +const BINARY_CONTENT_TYPES = new Set([ + 'binary/octet-stream', + 'application/octet-stream', + 'application/x-binary' +]) + +// --------------------------------------------------------------------------- + +const OVERVIEWS = { + VIDEOS: { + SAMPLE_THRESHOLD: 6, + SAMPLES_COUNT: 20 + } +} + +// --------------------------------------------------------------------------- + +const SERVER_ACTOR_NAME = 'peertube' + +const ACTIVITY_PUB = { + POTENTIAL_ACCEPT_HEADERS: [ + 'application/activity+json', + 'application/ld+json', + 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"' + ], + ACCEPT_HEADER: 'application/activity+json, application/ld+json', + PUBLIC: 'https://www.w3.org/ns/activitystreams#Public', + COLLECTION_ITEMS_PER_PAGE: 10, + FETCH_PAGE_LIMIT: 2000, + URL_MIME_TYPES: { + VIDEO: [] as string[], + TORRENT: [ 'application/x-bittorrent' ], + MAGNET: [ 'application/x-bittorrent;x-scheme-handler/magnet' ] + }, + MAX_RECURSION_COMMENTS: 100, + ACTOR_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 days + VIDEO_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2, // 2 days + VIDEO_PLAYLIST_REFRESH_INTERVAL: 3600 * 24 * 1000 * 2 // 2 days +} + +const ACTIVITY_PUB_ACTOR_TYPES: { [ id: string ]: ActivityPubActorType } = { + GROUP: 'Group', + PERSON: 'Person', + APPLICATION: 'Application', + ORGANIZATION: 'Organization', + SERVICE: 'Service' +} + +const HTTP_SIGNATURE = { + HEADER_NAME: 'signature', + ALGORITHM: 'rsa-sha256', + HEADERS_TO_SIGN_WITH_PAYLOAD: [ '(request-target)', 'host', 'date', 'digest' ], + HEADERS_TO_SIGN_WITHOUT_PAYLOAD: [ '(request-target)', 'host', 'date' ], + CLOCK_SKEW_SECONDS: 1800 +} + +// --------------------------------------------------------------------------- + +let PRIVATE_RSA_KEY_SIZE = 2048 + +// Password encryption +const BCRYPT_SALT_SIZE = 10 + +const ENCRYPTION = { + ALGORITHM: 'aes-256-cbc', + IV: 16, + SALT: 'peertube', + ENCODING: 'hex' as Encoding +} + +const USER_PASSWORD_RESET_LIFETIME = 60000 * 60 // 60 minutes +const USER_PASSWORD_CREATE_LIFETIME = 60000 * 60 * 24 * 7 // 7 days + +const TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME = 60000 * 10 // 10 minutes + +const EMAIL_VERIFY_LIFETIME = 60000 * 60 // 60 minutes + +const NSFW_POLICY_TYPES: { [ id: string ]: NSFWPolicyType } = { + DO_NOT_LIST: 'do_not_list', + BLUR: 'blur', + DISPLAY: 'display' +} + +// --------------------------------------------------------------------------- + +// Express static paths (router) +const STATIC_PATHS = { + // TODO: deprecated in v6, to remove + THUMBNAILS: '/static/thumbnails/', + + // Need to keep this legacy path for previously generated torrents + LEGACY_WEB_VIDEOS: '/static/webseed/', + WEB_VIDEOS: '/static/web-videos/', + + // Need to keep this legacy path for previously generated torrents + LEGACY_PRIVATE_WEB_VIDEOS: '/static/webseed/private/', + PRIVATE_WEB_VIDEOS: '/static/web-videos/private/', + + REDUNDANCY: '/static/redundancy/', + + STREAMING_PLAYLISTS: { + HLS: '/static/streaming-playlists/hls', + PRIVATE_HLS: '/static/streaming-playlists/hls/private/' + } +} +const STATIC_DOWNLOAD_PATHS = { + TORRENTS: '/download/torrents/', + VIDEOS: '/download/videos/', + HLS_VIDEOS: '/download/streaming-playlists/hls/videos/' +} +const LAZY_STATIC_PATHS = { + THUMBNAILS: '/lazy-static/thumbnails/', + BANNERS: '/lazy-static/banners/', + AVATARS: '/lazy-static/avatars/', + PREVIEWS: '/lazy-static/previews/', + VIDEO_CAPTIONS: '/lazy-static/video-captions/', + TORRENTS: '/lazy-static/torrents/', + STORYBOARDS: '/lazy-static/storyboards/' +} +const OBJECT_STORAGE_PROXY_PATHS = { + // Need to keep this legacy path for previously generated torrents + LEGACY_PRIVATE_WEB_VIDEOS: '/object-storage-proxy/webseed/private/', + PRIVATE_WEB_VIDEOS: '/object-storage-proxy/web-videos/private/', + + STREAMING_PLAYLISTS: { + PRIVATE_HLS: '/object-storage-proxy/streaming-playlists/hls/private/' + } +} + +// Cache control +const STATIC_MAX_AGE = { + SERVER: '2h', + LAZY_SERVER: '2d', + CLIENT: '30d' +} + +// Videos thumbnail size +const THUMBNAILS_SIZE = { + width: 280, + height: 157, + minWidth: 150 +} +const PREVIEWS_SIZE = { + width: 850, + height: 480, + minWidth: 400 +} +const ACTOR_IMAGES_SIZE: { [key in ActorImageType_Type]: { width: number, height: number }[] } = { + [ActorImageType.AVATAR]: [ + { + width: 120, + height: 120 + }, + { + width: 48, + height: 48 + } + ], + [ActorImageType.BANNER]: [ + { + width: 1920, + height: 317 // 6/1 ratio + } + ] +} + +const STORYBOARD = { + SPRITE_SIZE: { + width: 192, + height: 108 + }, + SPRITES_MAX_EDGE_COUNT: 10 +} + +const EMBED_SIZE = { + width: 560, + height: 315 +} + +// Sub folders of cache directory +const FILES_CACHE = { + PREVIEWS: { + DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'previews'), + MAX_AGE: 1000 * 3600 * 3 // 3 hours + }, + STORYBOARDS: { + DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'storyboards'), + MAX_AGE: 1000 * 3600 * 24 // 24 hours + }, + VIDEO_CAPTIONS: { + DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'video-captions'), + MAX_AGE: 1000 * 3600 * 3 // 3 hours + }, + TORRENTS: { + DIRECTORY: join(CONFIG.STORAGE.CACHE_DIR, 'torrents'), + MAX_AGE: 1000 * 3600 * 3 // 3 hours + } +} + +const LRU_CACHE = { + USER_TOKENS: { + MAX_SIZE: 1000 + }, + FILENAME_TO_PATH_PERMANENT_FILE_CACHE: { + MAX_SIZE: 1000 + }, + STATIC_VIDEO_FILES_RIGHTS_CHECK: { + MAX_SIZE: 5000, + TTL: parseDurationToMs('10 seconds') + }, + VIDEO_TOKENS: { + MAX_SIZE: 100_000, + TTL: parseDurationToMs('8 hours') + }, + TRACKER_IPS: { + MAX_SIZE: 100_000 + } +} + +const DIRECTORIES = { + RESUMABLE_UPLOAD: join(CONFIG.STORAGE.TMP_DIR, 'resumable-uploads'), + + HLS_STREAMING_PLAYLIST: { + PUBLIC: join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls'), + PRIVATE: join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls', 'private') + }, + + VIDEOS: { + PUBLIC: CONFIG.STORAGE.WEB_VIDEOS_DIR, + PRIVATE: join(CONFIG.STORAGE.WEB_VIDEOS_DIR, 'private') + }, + + HLS_REDUNDANCY: join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls') +} + +const RESUMABLE_UPLOAD_SESSION_LIFETIME = SCHEDULER_INTERVALS_MS.REMOVE_DANGLING_RESUMABLE_UPLOADS + +const VIDEO_LIVE = { + EXTENSION: '.ts', + CLEANUP_DELAY: 1000 * 60 * 5, // 5 minutes + SEGMENT_TIME_SECONDS: { + DEFAULT_LATENCY: 4, // 4 seconds + SMALL_LATENCY: 2 // 2 seconds + }, + SEGMENTS_LIST_SIZE: 15, // 15 maximum segments in live playlist + REPLAY_DIRECTORY: 'replay', + EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION: 4, + MAX_SOCKET_WAITING_DATA: 1024 * 1000 * 100, // 100MB + RTMP: { + CHUNK_SIZE: 60000, + GOP_CACHE: true, + PING: 60, + PING_TIMEOUT: 30, + BASE_PATH: 'live' + } +} + +const MEMOIZE_TTL = { + OVERVIEWS_SAMPLE: 1000 * 3600 * 4, // 4 hours + INFO_HASH_EXISTS: 1000 * 60, // 1 minute + VIDEO_DURATION: 1000 * 10, // 10 seconds + LIVE_ABLE_TO_UPLOAD: 1000 * 60, // 1 minute + LIVE_CHECK_SOCKET_HEALTH: 1000 * 60, // 1 minute + GET_STATS_FOR_OPEN_TELEMETRY_METRICS: 1000 * 60 // 1 minute +} + +const MEMOIZE_LENGTH = { + INFO_HASH_EXISTS: 200, + VIDEO_DURATION: 200 +} + +const WORKER_THREADS = { + DOWNLOAD_IMAGE: { + CONCURRENCY: 3, + MAX_THREADS: 1 + }, + PROCESS_IMAGE: { + CONCURRENCY: 1, + MAX_THREADS: 5 + } +} + +const REDUNDANCY = { + VIDEOS: { + RANDOMIZED_FACTOR: 5 + } +} + +const ACCEPT_HEADERS = [ 'html', 'application/json' ].concat(ACTIVITY_PUB.POTENTIAL_ACCEPT_HEADERS) +const OTP = { + HEADER_NAME: 'x-peertube-otp', + HEADER_REQUIRED_VALUE: 'required; app' +} + +const ASSETS_PATH = { + DEFAULT_AUDIO_BACKGROUND: join(root(), 'dist', 'server', 'assets', 'default-audio-background.jpg'), + DEFAULT_LIVE_BACKGROUND: join(root(), 'dist', 'server', 'assets', 'default-live-background.jpg') +} + +// --------------------------------------------------------------------------- + +const CUSTOM_HTML_TAG_COMMENTS = { + TITLE: '', + DESCRIPTION: '', + CUSTOM_CSS: '', + META_TAGS: '', + SERVER_CONFIG: '' +} + +const MAX_LOGS_OUTPUT_CHARACTERS = 10 * 1000 * 1000 +const LOG_FILENAME = 'peertube.log' +const AUDIT_LOG_FILENAME = 'peertube-audit.log' + +// --------------------------------------------------------------------------- + +const TRACKER_RATE_LIMITS = { + INTERVAL: 60000 * 5, // 5 minutes + ANNOUNCES_PER_IP_PER_INFOHASH: 15, // maximum announces per torrent in the interval + ANNOUNCES_PER_IP: 30, // maximum announces for all our torrents in the interval + BLOCK_IP_LIFETIME: parseDurationToMs('3 minutes') +} + +const P2P_MEDIA_LOADER_PEER_VERSION = 2 + +// --------------------------------------------------------------------------- + +const PLUGIN_GLOBAL_CSS_FILE_NAME = 'plugins-global.css' +const PLUGIN_GLOBAL_CSS_PATH = join(CONFIG.STORAGE.TMP_DIR, PLUGIN_GLOBAL_CSS_FILE_NAME) + +let PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME = 1000 * 60 * 5 // 5 minutes + +const DEFAULT_THEME_NAME = 'default' +const DEFAULT_USER_THEME_NAME = 'instance-default' + +// --------------------------------------------------------------------------- + +const SEARCH_INDEX = { + ROUTES: { + VIDEOS: '/api/v1/search/videos', + VIDEO_CHANNELS: '/api/v1/search/video-channels' + } +} + +// --------------------------------------------------------------------------- + +const STATS_TIMESERIE = { + MAX_DAYS: 365 * 10 // Around 10 years +} + +// --------------------------------------------------------------------------- + +// Special constants for a test instance +if (process.env.PRODUCTION_CONSTANTS !== 'true') { + if (isTestOrDevInstance()) { + PRIVATE_RSA_KEY_SIZE = 1024 + + ACTOR_FOLLOW_SCORE.BASE = 20 + + REMOTE_SCHEME.HTTP = 'http' + REMOTE_SCHEME.WS = 'ws' + + STATIC_MAX_AGE.SERVER = '0' + + SCHEDULER_INTERVALS_MS.ACTOR_FOLLOW_SCORES = 1000 + SCHEDULER_INTERVALS_MS.REMOVE_OLD_JOBS = 10000 + SCHEDULER_INTERVALS_MS.REMOVE_OLD_HISTORY = 5000 + SCHEDULER_INTERVALS_MS.REMOVE_OLD_VIEWS = 5000 + SCHEDULER_INTERVALS_MS.UPDATE_VIDEOS = 5000 + SCHEDULER_INTERVALS_MS.AUTO_FOLLOW_INDEX_INSTANCES = 5000 + SCHEDULER_INTERVALS_MS.UPDATE_INBOX_STATS = 5000 + SCHEDULER_INTERVALS_MS.CHECK_PEERTUBE_VERSION = 2000 + + REPEAT_JOBS['videos-views-stats'] = { every: 5000 } + + REPEAT_JOBS['activitypub-cleaner'] = { every: 5000 } + AP_CLEANER.PERIOD = 5000 + + REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1 + + CONTACT_FORM_LIFETIME = 1000 // 1 second + + JOB_ATTEMPTS['email'] = 1 + + FILES_CACHE.VIDEO_CAPTIONS.MAX_AGE = 3000 + MEMOIZE_TTL.OVERVIEWS_SAMPLE = 3000 + MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD = 3000 + OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD = 2 + + PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME = 5000 + + JOB_REMOVAL_OPTIONS.SUCCESS['videos-views-stats'] = 10000 + } + + if (isTestInstance()) { + ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2 + ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 10 * 1000 // 10 seconds + ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL = 10 * 1000 // 10 seconds + ACTIVITY_PUB.VIDEO_PLAYLIST_REFRESH_INTERVAL = 10 * 1000 // 10 seconds + + CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max = 100 * 1024 // 100KB + CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max = 400 * 1024 // 400KB + + VIEW_LIFETIME.VIEWER_COUNTER = 1000 * 5 // 5 second + VIEW_LIFETIME.VIEWER_STATS = 1000 * 5 // 5 second + + VIDEO_LIVE.CLEANUP_DELAY = getIntEnv('PEERTUBE_TEST_CONSTANTS_VIDEO_LIVE_CLEANUP_DELAY') ?? 5000 + VIDEO_LIVE.SEGMENT_TIME_SECONDS.DEFAULT_LATENCY = 2 + VIDEO_LIVE.SEGMENT_TIME_SECONDS.SMALL_LATENCY = 1 + VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION = 1 + + RUNNER_JOBS.LAST_CONTACT_UPDATE_INTERVAL = 2000 + } +} + +updateWebserverUrls() +updateWebserverConfig() + +registerConfigChangedHandler(() => { + updateWebserverUrls() + updateWebserverConfig() +}) + +// --------------------------------------------------------------------------- + +const FILES_CONTENT_HASH = { + MANIFEST: generateContentHash(), + FAVICON: generateContentHash(), + LOGO: generateContentHash() +} + +// --------------------------------------------------------------------------- + +const VIDEO_FILTERS = { + WATERMARK: { + SIZE_RATIO: 1 / 10, + HORIZONTAL_MARGIN_RATIO: 1 / 20, + VERTICAL_MARGIN_RATIO: 1 / 20 + } +} + +// --------------------------------------------------------------------------- + +export { + WEBSERVER, + API_VERSION, + ENCRYPTION, + VIDEO_LIVE, + PEERTUBE_VERSION, + LAZY_STATIC_PATHS, + OBJECT_STORAGE_PROXY_PATHS, + SEARCH_INDEX, + DIRECTORIES, + RESUMABLE_UPLOAD_SESSION_LIFETIME, + RUNNER_JOB_STATES, + P2P_MEDIA_LOADER_PEER_VERSION, + STORYBOARD, + ACTOR_IMAGES_SIZE, + ACCEPT_HEADERS, + BCRYPT_SALT_SIZE, + TRACKER_RATE_LIMITS, + FILES_CACHE, + LOG_FILENAME, + CONSTRAINTS_FIELDS, + EMBED_SIZE, + REDUNDANCY, + JOB_CONCURRENCY, + JOB_ATTEMPTS, + AP_CLEANER, + LAST_MIGRATION_VERSION, + CUSTOM_HTML_TAG_COMMENTS, + STATS_TIMESERIE, + BROADCAST_CONCURRENCY, + AUDIT_LOG_FILENAME, + PAGINATION, + ACTOR_FOLLOW_SCORE, + PREVIEWS_SIZE, + REMOTE_SCHEME, + FOLLOW_STATES, + DEFAULT_USER_THEME_NAME, + SERVER_ACTOR_NAME, + TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME, + PLUGIN_GLOBAL_CSS_FILE_NAME, + PLUGIN_GLOBAL_CSS_PATH, + PRIVATE_RSA_KEY_SIZE, + VIDEO_FILTERS, + ROUTE_CACHE_LIFETIME, + SORTABLE_COLUMNS, + JOB_TTL, + DEFAULT_THEME_NAME, + NSFW_POLICY_TYPES, + STATIC_MAX_AGE, + STATIC_PATHS, + VIDEO_IMPORT_TIMEOUT, + VIDEO_PLAYLIST_TYPES, + MAX_LOGS_OUTPUT_CHARACTERS, + ACTIVITY_PUB, + ACTIVITY_PUB_ACTOR_TYPES, + THUMBNAILS_SIZE, + VIDEO_CATEGORIES, + MEMOIZE_LENGTH, + VIDEO_LANGUAGES, + VIDEO_PRIVACIES, + VIDEO_LICENCES, + VIDEO_STATES, + WORKER_THREADS, + VIDEO_RATE_TYPES, + JOB_PRIORITY, + VIDEO_TRANSCODING_FPS, + FFMPEG_NICE, + ABUSE_STATES, + USER_REGISTRATION_STATES, + LRU_CACHE, + REQUEST_TIMEOUTS, + RUNNER_JOBS, + MAX_LOCAL_VIEWER_WATCH_SECTIONS, + USER_PASSWORD_RESET_LIFETIME, + USER_PASSWORD_CREATE_LIFETIME, + MEMOIZE_TTL, + EMAIL_VERIFY_LIFETIME, + OVERVIEWS, + SCHEDULER_INTERVALS_MS, + REPEAT_JOBS, + STATIC_DOWNLOAD_PATHS, + MIMETYPES, + CRAWL_REQUEST_CONCURRENCY, + DEFAULT_AUDIO_RESOLUTION, + BINARY_CONTENT_TYPES, + JOB_REMOVAL_OPTIONS, + HTTP_SIGNATURE, + VIDEO_IMPORT_STATES, + VIDEO_CHANNEL_SYNC_STATE, + VIEW_LIFETIME, + CONTACT_FORM_LIFETIME, + VIDEO_PLAYLIST_PRIVACIES, + PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME, + ASSETS_PATH, + FILES_CONTENT_HASH, + OTP, + loadLanguages, + buildLanguages, + generateContentHash +} + +// --------------------------------------------------------------------------- + +function buildVideoMimetypeExt () { + const data = { + // streamable formats that warrant cross-browser compatibility + 'video/webm': '.webm', + // We'll add .ogg if additional extensions are enabled + // We could add .ogg here but since it could be an audio file, + // it would be confusing for users because PeerTube will refuse their file (based on the mimetype) + 'video/ogg': [ '.ogv' ], + 'video/mp4': '.mp4' + } + + if (CONFIG.TRANSCODING.ENABLED) { + if (CONFIG.TRANSCODING.ALLOW_ADDITIONAL_EXTENSIONS) { + data['video/ogg'].push('.ogg') + + Object.assign(data, { + 'video/x-matroska': '.mkv', + + // Developed by Apple + 'video/quicktime': [ '.mov', '.qt', '.mqv' ], // often used as output format by editing software + 'video/x-m4v': '.m4v', + 'video/m4v': '.m4v', + + // Developed by the Adobe Flash Platform + 'video/x-flv': '.flv', + 'video/x-f4v': '.f4v', // replacement for flv + + // Developed by Microsoft + 'video/x-ms-wmv': '.wmv', + 'video/x-msvideo': '.avi', + 'video/avi': '.avi', + + // Developed by 3GPP + // common video formats for cell phones + 'video/3gpp': [ '.3gp', '.3gpp' ], + 'video/3gpp2': [ '.3g2', '.3gpp2' ], + + // Developed by FFmpeg/Mplayer + 'application/x-nut': '.nut', + + // The standard video format used by many Sony and Panasonic HD camcorders. + // It is also used for storing high definition video on Blu-ray discs. + 'video/mp2t': '.mts', + 'video/vnd.dlna.mpeg-tts': '.mts', + + 'video/m2ts': '.m2ts', + + // Old formats reliant on MPEG-1/MPEG-2 + 'video/mpv': '.mpv', + 'video/mpeg2': '.m2v', + 'video/mpeg': [ '.m1v', '.mpg', '.mpe', '.mpeg', '.vob' ], + 'video/dvd': '.vob', + + // Could be anything + 'application/octet-stream': null, + 'application/mxf': '.mxf' // often used as exchange format by editing software + }) + } + + if (CONFIG.TRANSCODING.ALLOW_AUDIO_FILES) { + Object.assign(data, MIMETYPES.AUDIO.MIMETYPE_EXT) + } + } + + return data +} + +function updateWebserverUrls () { + WEBSERVER.URL = sanitizeUrl(CONFIG.WEBSERVER.SCHEME + '://' + CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT) + WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP) + WEBSERVER.WS = CONFIG.WEBSERVER.WS + + WEBSERVER.SCHEME = CONFIG.WEBSERVER.SCHEME + WEBSERVER.HOSTNAME = CONFIG.WEBSERVER.HOSTNAME + WEBSERVER.PORT = CONFIG.WEBSERVER.PORT + + const rtmpHostname = CONFIG.LIVE.RTMP.PUBLIC_HOSTNAME || CONFIG.WEBSERVER.HOSTNAME + const rtmpsHostname = CONFIG.LIVE.RTMPS.PUBLIC_HOSTNAME || CONFIG.WEBSERVER.HOSTNAME + + WEBSERVER.RTMP_URL = 'rtmp://' + rtmpHostname + ':' + CONFIG.LIVE.RTMP.PORT + WEBSERVER.RTMPS_URL = 'rtmps://' + rtmpsHostname + ':' + CONFIG.LIVE.RTMPS.PORT + + WEBSERVER.RTMP_BASE_LIVE_URL = WEBSERVER.RTMP_URL + '/' + VIDEO_LIVE.RTMP.BASE_PATH + WEBSERVER.RTMPS_BASE_LIVE_URL = WEBSERVER.RTMPS_URL + '/' + VIDEO_LIVE.RTMP.BASE_PATH +} + +function updateWebserverConfig () { + MIMETYPES.VIDEO.MIMETYPE_EXT = buildVideoMimetypeExt() + MIMETYPES.VIDEO.MIMETYPES_REGEX = buildMimetypesRegex(MIMETYPES.VIDEO.MIMETYPE_EXT) + + ACTIVITY_PUB.URL_MIME_TYPES.VIDEO = Object.keys(MIMETYPES.VIDEO.MIMETYPE_EXT) + + MIMETYPES.VIDEO.EXT_MIMETYPE = buildVideoExtMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT) + + CONSTRAINTS_FIELDS.VIDEOS.EXTNAME = Object.keys(MIMETYPES.VIDEO.EXT_MIMETYPE) +} + +function buildVideoExtMimetype (obj: { [ id: string ]: string | string[] }) { + const result: { [id: string]: string } = {} + + for (const mimetype of Object.keys(obj)) { + const value = obj[mimetype] + if (!value) continue + + const extensions = Array.isArray(value) ? value : [ value ] + + for (const extension of extensions) { + result[extension] = mimetype + } + } + + return result +} + +function buildMimetypesRegex (obj: { [id: string]: string | string[] }) { + return Object.keys(obj) + .map(m => `(${m})`) + .join('|') +} + +async function loadLanguages () { + if (Object.keys(VIDEO_LANGUAGES).length !== 0) return + + Object.assign(VIDEO_LANGUAGES, await buildLanguages()) +} + +async function buildLanguages () { + const { iso6393 } = await import('iso-639-3') + + const languages: { [id: string]: string } = {} + + const additionalLanguages = { + sgn: true, // Sign languages (macro language) + ase: true, // American sign language + asq: true, // Austrian sign language + sdl: true, // Arabian sign language + bfi: true, // British sign language + bzs: true, // Brazilian sign language + csl: true, // Chinese sign language + cse: true, // Czech sign language + dsl: true, // Danish sign language + fsl: true, // French sign language + gsg: true, // German sign language + pks: true, // Pakistan sign language + jsl: true, // Japanese sign language + sfs: true, // South African sign language + swl: true, // Swedish sign language + rsl: true, // Russian sign language + + kab: true, // Kabyle + + lat: true, // Latin + + epo: true, // Esperanto + tlh: true, // Klingon + jbo: true, // Lojban + avk: true, // Kotava + + zxx: true // No linguistic content (ISO-639-2) + } + + // Only add ISO639-1 languages and some sign languages (ISO639-3) + iso6393 + .filter(l => { + return (l.iso6391 !== undefined && l.type === 'living') || + additionalLanguages[l.iso6393] === true + }) + .forEach(l => { languages[l.iso6391 || l.iso6393] = l.name }) + + // Override Occitan label + languages['oc'] = 'Occitan' + languages['el'] = 'Greek' + languages['tok'] = 'Toki Pona' + + // Chinese languages + languages['zh-Hans'] = 'Simplified Chinese' + languages['zh-Hant'] = 'Traditional Chinese' + + return languages +} + +function generateContentHash () { + return randomBytes(20).toString('hex') +} + +function getIntEnv (path: string) { + if (process.env[path]) return parseInt(process.env[path]) + + return undefined +} diff --git a/server/server/initializers/database.ts b/server/server/initializers/database.ts new file mode 100644 index 000000000..fe399a633 --- /dev/null +++ b/server/server/initializers/database.ts @@ -0,0 +1,234 @@ +import pg from 'pg' +import { QueryTypes, Transaction } from 'sequelize' +import { Sequelize as SequelizeTypescript } from 'sequelize-typescript' +import { isTestOrDevInstance } from '@peertube/peertube-node-utils' +import { ActorCustomPageModel } from '@server/models/account/actor-custom-page.js' +import { RunnerJobModel } from '@server/models/runner/runner-job.js' +import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token.js' +import { RunnerModel } from '@server/models/runner/runner.js' +import { TrackerModel } from '@server/models/server/tracker.js' +import { VideoTrackerModel } from '@server/models/server/video-tracker.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { UserRegistrationModel } from '@server/models/user/user-registration.js' +import { UserVideoHistoryModel } from '@server/models/user/user-video-history.js' +import { UserModel } from '@server/models/user/user.js' +import { StoryboardModel } from '@server/models/video/storyboard.js' +import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync.js' +import { VideoJobInfoModel } from '@server/models/video/video-job-info.js' +import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting.js' +import { VideoLiveSessionModel } from '@server/models/video/video-live-session.js' +import { VideoPasswordModel } from '@server/models/video/video-password.js' +import { VideoSourceModel } from '@server/models/video/video-source.js' +import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section.js' +import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer.js' +import { logger } from '../helpers/logger.js' +import { AbuseMessageModel } from '../models/abuse/abuse-message.js' +import { AbuseModel } from '../models/abuse/abuse.js' +import { VideoAbuseModel } from '../models/abuse/video-abuse.js' +import { VideoCommentAbuseModel } from '../models/abuse/video-comment-abuse.js' +import { AccountBlocklistModel } from '../models/account/account-blocklist.js' +import { AccountVideoRateModel } from '../models/account/account-video-rate.js' +import { AccountModel } from '../models/account/account.js' +import { ActorFollowModel } from '../models/actor/actor-follow.js' +import { ActorImageModel } from '../models/actor/actor-image.js' +import { ActorModel } from '../models/actor/actor.js' +import { ApplicationModel } from '../models/application/application.js' +import { OAuthClientModel } from '../models/oauth/oauth-client.js' +import { OAuthTokenModel } from '../models/oauth/oauth-token.js' +import { VideoRedundancyModel } from '../models/redundancy/video-redundancy.js' +import { PluginModel } from '../models/server/plugin.js' +import { ServerBlocklistModel } from '../models/server/server-blocklist.js' +import { ServerModel } from '../models/server/server.js' +import { UserNotificationSettingModel } from '../models/user/user-notification-setting.js' +import { ScheduleVideoUpdateModel } from '../models/video/schedule-video-update.js' +import { TagModel } from '../models/video/tag.js' +import { ThumbnailModel } from '../models/video/thumbnail.js' +import { VideoBlacklistModel } from '../models/video/video-blacklist.js' +import { VideoCaptionModel } from '../models/video/video-caption.js' +import { VideoChangeOwnershipModel } from '../models/video/video-change-ownership.js' +import { VideoChannelModel } from '../models/video/video-channel.js' +import { VideoCommentModel } from '../models/video/video-comment.js' +import { VideoFileModel } from '../models/video/video-file.js' +import { VideoImportModel } from '../models/video/video-import.js' +import { VideoLiveModel } from '../models/video/video-live.js' +import { VideoPlaylistElementModel } from '../models/video/video-playlist-element.js' +import { VideoPlaylistModel } from '../models/video/video-playlist.js' +import { VideoShareModel } from '../models/video/video-share.js' +import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist.js' +import { VideoTagModel } from '../models/video/video-tag.js' +import { VideoModel } from '../models/video/video.js' +import { VideoViewModel } from '../models/view/video-view.js' +import { CONFIG } from './config.js' + +pg.defaults.parseInt8 = true // Avoid BIGINT to be converted to string + +const dbname = CONFIG.DATABASE.DBNAME +const username = CONFIG.DATABASE.USERNAME +const password = CONFIG.DATABASE.PASSWORD +const host = CONFIG.DATABASE.HOSTNAME +const port = CONFIG.DATABASE.PORT +const poolMax = CONFIG.DATABASE.POOL.MAX + +let dialectOptions: any = {} + +if (CONFIG.DATABASE.SSL) { + dialectOptions = { + ssl: { + rejectUnauthorized: false + } + } +} + +const sequelizeTypescript = new SequelizeTypescript({ + database: dbname, + dialect: 'postgres', + dialectOptions, + host, + port, + username, + password, + pool: { + max: poolMax + }, + benchmark: isTestOrDevInstance(), + isolationLevel: Transaction.ISOLATION_LEVELS.SERIALIZABLE, + logging: (message: string, benchmark: number) => { + if (process.env.NODE_DB_LOG === 'false') return + + let newMessage = 'Executed SQL request' + if (isTestOrDevInstance() === true && benchmark !== undefined) { + newMessage += ' in ' + benchmark + 'ms' + } + + logger.debug(newMessage, { sql: message, tags: [ 'sql' ] }) + } +}) + +function checkDatabaseConnectionOrDie () { + sequelizeTypescript.authenticate() + .then(() => logger.debug('Connection to PostgreSQL has been established successfully.')) + .catch(err => { + + logger.error('Unable to connect to PostgreSQL database.', { err }) + process.exit(-1) + }) +} + +async function initDatabaseModels (silent: boolean) { + sequelizeTypescript.addModels([ + ApplicationModel, + ActorModel, + ActorFollowModel, + ActorImageModel, + AccountModel, + OAuthClientModel, + OAuthTokenModel, + ServerModel, + TagModel, + AccountVideoRateModel, + UserModel, + AbuseMessageModel, + AbuseModel, + VideoCommentAbuseModel, + VideoAbuseModel, + VideoModel, + VideoChangeOwnershipModel, + VideoChannelModel, + VideoShareModel, + VideoFileModel, + VideoSourceModel, + VideoCaptionModel, + VideoBlacklistModel, + VideoTagModel, + VideoCommentModel, + ScheduleVideoUpdateModel, + VideoImportModel, + VideoViewModel, + VideoRedundancyModel, + UserVideoHistoryModel, + VideoLiveModel, + VideoLiveSessionModel, + VideoLiveReplaySettingModel, + AccountBlocklistModel, + ServerBlocklistModel, + UserNotificationModel, + UserNotificationSettingModel, + VideoStreamingPlaylistModel, + VideoPlaylistModel, + VideoPlaylistElementModel, + LocalVideoViewerModel, + LocalVideoViewerWatchSectionModel, + ThumbnailModel, + TrackerModel, + VideoTrackerModel, + PluginModel, + ActorCustomPageModel, + VideoJobInfoModel, + VideoChannelSyncModel, + UserRegistrationModel, + VideoPasswordModel, + RunnerRegistrationTokenModel, + RunnerModel, + RunnerJobModel, + StoryboardModel + ]) + + // Check extensions exist in the database + await checkPostgresExtensions() + + // Create custom PostgreSQL functions + await createFunctions() + + if (!silent) logger.info('Database %s is ready.', dbname) +} + +// --------------------------------------------------------------------------- + +export { + initDatabaseModels, + checkDatabaseConnectionOrDie, + sequelizeTypescript +} + +// --------------------------------------------------------------------------- + +async function checkPostgresExtensions () { + const promises = [ + checkPostgresExtension('pg_trgm'), + checkPostgresExtension('unaccent') + ] + + return Promise.all(promises) +} + +async function checkPostgresExtension (extension: string) { + const query = `SELECT 1 FROM pg_available_extensions WHERE name = '${extension}' AND installed_version IS NOT NULL;` + const options = { + type: QueryTypes.SELECT as QueryTypes.SELECT, + raw: true + } + + const res = await sequelizeTypescript.query(query, options) + + if (!res || res.length === 0) { + // Try to create the extension ourselves + try { + await sequelizeTypescript.query(`CREATE EXTENSION ${extension};`, { raw: true }) + + } catch { + const errorMessage = `You need to enable ${extension} extension in PostgreSQL. ` + + `You can do so by running 'CREATE EXTENSION ${extension};' as a PostgreSQL super user in ${CONFIG.DATABASE.DBNAME} database.` + throw new Error(errorMessage) + } + } +} + +function createFunctions () { + const query = `CREATE OR REPLACE FUNCTION immutable_unaccent(text) + RETURNS text AS +$func$ +SELECT public.unaccent('public.unaccent', $1::text) +$func$ LANGUAGE sql IMMUTABLE;` + + return sequelizeTypescript.query(query, { raw: true }) +} diff --git a/server/server/initializers/installer.ts b/server/server/initializers/installer.ts new file mode 100644 index 000000000..ff25f1909 --- /dev/null +++ b/server/server/initializers/installer.ts @@ -0,0 +1,200 @@ +import { ensureDir, remove } from 'fs-extra/esm' +import { readdir } from 'fs/promises' +import passwordGenerator from 'password-generator' +import { join } from 'path' +import { UserRole } from '@peertube/peertube-models' +import { isTestOrDevInstance } from '@peertube/peertube-node-utils' +import { generateRunnerRegistrationToken } from '@server/helpers/token-generator.js' +import { getNodeABIVersion } from '@server/helpers/version.js' +import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token.js' +import { logger } from '../helpers/logger.js' +import { buildUser, createApplicationActor, createUserAccountAndChannelAndPlaylist } from '../lib/user.js' +import { ApplicationModel } from '../models/application/application.js' +import { OAuthClientModel } from '../models/oauth/oauth-client.js' +import { applicationExist, clientsExist, usersExist } from './checker-after-init.js' +import { CONFIG } from './config.js' +import { DIRECTORIES, FILES_CACHE, LAST_MIGRATION_VERSION } from './constants.js' +import { sequelizeTypescript } from './database.js' + +async function installApplication () { + try { + await Promise.all([ + // Database related + sequelizeTypescript.sync() + .then(() => { + return Promise.all([ + createApplicationIfNotExist(), + createOAuthClientIfNotExist(), + createOAuthAdminIfNotExist(), + createRunnerRegistrationTokenIfNotExist() + ]) + }), + + // Directories + removeCacheAndTmpDirectories() + .then(() => createDirectoriesIfNotExist()) + ]) + } catch (err) { + logger.error('Cannot install application.', { err }) + process.exit(-1) + } +} + +// --------------------------------------------------------------------------- + +export { + installApplication +} + +// --------------------------------------------------------------------------- + +function removeCacheAndTmpDirectories () { + const cacheDirectories = Object.keys(FILES_CACHE) + .map(k => FILES_CACHE[k].DIRECTORY) + + const tasks: Promise[] = [] + + // Cache directories + for (const dir of cacheDirectories) { + tasks.push(removeDirectoryOrContent(dir)) + } + + tasks.push(removeDirectoryOrContent(CONFIG.STORAGE.TMP_DIR)) + + return Promise.all(tasks) +} + +async function removeDirectoryOrContent (dir: string) { + try { + await remove(dir) + } catch (err) { + logger.debug('Cannot remove directory %s. Removing content instead.', dir, { err }) + + const files = await readdir(dir) + + for (const file of files) { + await remove(join(dir, file)) + } + } +} + +function createDirectoriesIfNotExist () { + const storage = CONFIG.STORAGE + const cacheDirectories = Object.keys(FILES_CACHE) + .map(k => FILES_CACHE[k].DIRECTORY) + + const tasks: Promise[] = [] + for (const key of Object.keys(storage)) { + const dir = storage[key] + tasks.push(ensureDir(dir)) + } + + // Cache directories + for (const dir of cacheDirectories) { + tasks.push(ensureDir(dir)) + } + + tasks.push(ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE)) + tasks.push(ensureDir(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC)) + tasks.push(ensureDir(DIRECTORIES.VIDEOS.PUBLIC)) + tasks.push(ensureDir(DIRECTORIES.VIDEOS.PRIVATE)) + + // Resumable upload directory + tasks.push(ensureDir(DIRECTORIES.RESUMABLE_UPLOAD)) + + return Promise.all(tasks) +} + +async function createOAuthClientIfNotExist () { + const exist = await clientsExist() + // Nothing to do, clients already exist + if (exist === true) return undefined + + logger.info('Creating a default OAuth Client.') + + const id = passwordGenerator(32, false, /[a-z0-9]/) + const secret = passwordGenerator(32, false, /[a-zA-Z0-9]/) + const client = new OAuthClientModel({ + clientId: id, + clientSecret: secret, + grants: [ 'password', 'refresh_token' ], + redirectUris: null + }) + + const createdClient = await client.save() + logger.info('Client id: ' + createdClient.clientId) + logger.info('Client secret: ' + createdClient.clientSecret) + + return undefined +} + +async function createOAuthAdminIfNotExist () { + const exist = await usersExist() + // Nothing to do, users already exist + if (exist === true) return undefined + + logger.info('Creating the administrator.') + + const username = 'root' + const role = UserRole.ADMINISTRATOR + const email = CONFIG.ADMIN.EMAIL + let validatePassword = true + let password = '' + + // Do not generate a random password for test and dev environments + if (isTestOrDevInstance()) { + password = 'test' + + if (process.env.NODE_APP_INSTANCE) { + password += process.env.NODE_APP_INSTANCE + } + + // Our password is weak so do not validate it + validatePassword = false + } else if (process.env.PT_INITIAL_ROOT_PASSWORD) { + password = process.env.PT_INITIAL_ROOT_PASSWORD + } else { + password = passwordGenerator(16, true) + } + + const user = buildUser({ + username, + email, + password, + role, + emailVerified: true, + videoQuota: -1, + videoQuotaDaily: -1 + }) + + await createUserAccountAndChannelAndPlaylist({ userToCreate: user, channelNames: undefined, validateUser: validatePassword }) + logger.info('Username: ' + username) + logger.info('User password: ' + password) +} + +async function createApplicationIfNotExist () { + const exist = await applicationExist() + // Nothing to do, application already exist + if (exist === true) return undefined + + logger.info('Creating application account.') + + const application = await ApplicationModel.create({ + migrationVersion: LAST_MIGRATION_VERSION, + nodeVersion: process.version, + nodeABIVersion: getNodeABIVersion() + }) + + return createApplicationActor(application.id) +} + +async function createRunnerRegistrationTokenIfNotExist () { + const total = await RunnerRegistrationTokenModel.countTotal() + if (total !== 0) return undefined + + const token = new RunnerRegistrationTokenModel({ + registrationToken: generateRunnerRegistrationToken() + }) + + await token.save() +} diff --git a/server/initializers/migrations/0505-user-last-login-date.ts b/server/server/initializers/migrations/0505-user-last-login-date.ts similarity index 100% rename from server/initializers/migrations/0505-user-last-login-date.ts rename to server/server/initializers/migrations/0505-user-last-login-date.ts diff --git a/server/initializers/migrations/0510-video-file-metadata.ts b/server/server/initializers/migrations/0510-video-file-metadata.ts similarity index 100% rename from server/initializers/migrations/0510-video-file-metadata.ts rename to server/server/initializers/migrations/0510-video-file-metadata.ts diff --git a/server/initializers/migrations/0515-video-abuse-reason-timestamps.ts b/server/server/initializers/migrations/0515-video-abuse-reason-timestamps.ts similarity index 100% rename from server/initializers/migrations/0515-video-abuse-reason-timestamps.ts rename to server/server/initializers/migrations/0515-video-abuse-reason-timestamps.ts diff --git a/server/initializers/migrations/0520-abuses-split.ts b/server/server/initializers/migrations/0520-abuses-split.ts similarity index 100% rename from server/initializers/migrations/0520-abuses-split.ts rename to server/server/initializers/migrations/0520-abuses-split.ts diff --git a/server/initializers/migrations/0525-abuse-messages.ts b/server/server/initializers/migrations/0525-abuse-messages.ts similarity index 100% rename from server/initializers/migrations/0525-abuse-messages.ts rename to server/server/initializers/migrations/0525-abuse-messages.ts diff --git a/server/server/initializers/migrations/0530-playlist-multiple-video.ts b/server/server/initializers/migrations/0530-playlist-multiple-video.ts new file mode 100644 index 000000000..a53b93fd7 --- /dev/null +++ b/server/server/initializers/migrations/0530-playlist-multiple-video.ts @@ -0,0 +1,46 @@ +import * as Sequelize from 'sequelize' +import { WEBSERVER } from '../constants.js' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize +}): Promise { + { + const field = { + type: Sequelize.STRING, + allowNull: true + } + await utils.queryInterface.changeColumn('videoPlaylistElement', 'url', field) + } + + { + await utils.sequelize.query('DROP INDEX IF EXISTS video_playlist_element_video_playlist_id_video_id;') + } + + { + const selectPlaylistUUID = 'SELECT "uuid" FROM "videoPlaylist" WHERE "id" = "videoPlaylistElement"."videoPlaylistId"' + const url = `'${WEBSERVER.URL}' || '/video-playlists/' || (${selectPlaylistUUID}) || '/videos/' || "videoPlaylistElement"."id"` + + const query = ` + UPDATE "videoPlaylistElement" SET "url" = ${url} WHERE id IN ( + SELECT "videoPlaylistElement"."id" FROM "videoPlaylistElement" + INNER JOIN "videoPlaylist" ON "videoPlaylist".id = "videoPlaylistElement"."videoPlaylistId" + INNER JOIN account ON account.id = "videoPlaylist"."ownerAccountId" + INNER JOIN actor ON actor.id = account."actorId" + WHERE actor."serverId" IS NULL + )` + + await utils.sequelize.query(query) + } + +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/initializers/migrations/0535-video-live.ts b/server/server/initializers/migrations/0535-video-live.ts similarity index 100% rename from server/initializers/migrations/0535-video-live.ts rename to server/server/initializers/migrations/0535-video-live.ts diff --git a/server/initializers/migrations/0540-video-file-infohash.ts b/server/server/initializers/migrations/0540-video-file-infohash.ts similarity index 100% rename from server/initializers/migrations/0540-video-file-infohash.ts rename to server/server/initializers/migrations/0540-video-file-infohash.ts diff --git a/server/initializers/migrations/0545-video-live-save-replay.ts b/server/server/initializers/migrations/0545-video-live-save-replay.ts similarity index 100% rename from server/initializers/migrations/0545-video-live-save-replay.ts rename to server/server/initializers/migrations/0545-video-live-save-replay.ts diff --git a/server/initializers/migrations/0550-actor-follow-cleanup.ts b/server/server/initializers/migrations/0550-actor-follow-cleanup.ts similarity index 100% rename from server/initializers/migrations/0550-actor-follow-cleanup.ts rename to server/server/initializers/migrations/0550-actor-follow-cleanup.ts diff --git a/server/initializers/migrations/0555-actor-follow-url.ts b/server/server/initializers/migrations/0555-actor-follow-url.ts similarity index 100% rename from server/initializers/migrations/0555-actor-follow-url.ts rename to server/server/initializers/migrations/0555-actor-follow-url.ts diff --git a/server/server/initializers/migrations/0560-user-feed-token.ts b/server/server/initializers/migrations/0560-user-feed-token.ts new file mode 100644 index 000000000..f2ec9c91d --- /dev/null +++ b/server/server/initializers/migrations/0560-user-feed-token.ts @@ -0,0 +1,51 @@ +import * as Sequelize from 'sequelize' +import { buildUUID } from '@peertube/peertube-node-utils' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize + db: any +}): Promise { + const q = utils.queryInterface + + { + // Create uuid column for users + const userFeedTokenUUID = { + type: Sequelize.UUID, + defaultValue: Sequelize.UUIDV4, + allowNull: true + } + await q.addColumn('user', 'feedToken', userFeedTokenUUID) + } + + // Set UUID to previous users + { + const query = 'SELECT * FROM "user" WHERE "feedToken" IS NULL' + const options = { type: Sequelize.QueryTypes.SELECT as Sequelize.QueryTypes.SELECT } + const users = await utils.sequelize.query(query, options) + + for (const user of users) { + const queryUpdate = `UPDATE "user" SET "feedToken" = '${buildUUID()}' WHERE id = ${user.id}` + await utils.sequelize.query(queryUpdate) + } + } + + { + const userFeedTokenUUID = { + type: Sequelize.UUID, + defaultValue: Sequelize.UUIDV4, + allowNull: false + } + await q.changeColumn('user', 'feedToken', userFeedTokenUUID) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/initializers/migrations/0565-actor-follow-local-url.ts b/server/server/initializers/migrations/0565-actor-follow-local-url.ts similarity index 100% rename from server/initializers/migrations/0565-actor-follow-local-url.ts rename to server/server/initializers/migrations/0565-actor-follow-local-url.ts diff --git a/server/initializers/migrations/0570-permanent-live.ts b/server/server/initializers/migrations/0570-permanent-live.ts similarity index 100% rename from server/initializers/migrations/0570-permanent-live.ts rename to server/server/initializers/migrations/0570-permanent-live.ts diff --git a/server/initializers/migrations/0575-duplicate-thumbnail.ts b/server/server/initializers/migrations/0575-duplicate-thumbnail.ts similarity index 100% rename from server/initializers/migrations/0575-duplicate-thumbnail.ts rename to server/server/initializers/migrations/0575-duplicate-thumbnail.ts diff --git a/server/initializers/migrations/0580-caption-filename.ts b/server/server/initializers/migrations/0580-caption-filename.ts similarity index 100% rename from server/initializers/migrations/0580-caption-filename.ts rename to server/server/initializers/migrations/0580-caption-filename.ts diff --git a/server/initializers/migrations/0585-video-file-names.ts b/server/server/initializers/migrations/0585-video-file-names.ts similarity index 100% rename from server/initializers/migrations/0585-video-file-names.ts rename to server/server/initializers/migrations/0585-video-file-names.ts diff --git a/server/initializers/migrations/0590-trackers.ts b/server/server/initializers/migrations/0590-trackers.ts similarity index 100% rename from server/initializers/migrations/0590-trackers.ts rename to server/server/initializers/migrations/0590-trackers.ts diff --git a/server/initializers/migrations/0595-remote-url.ts b/server/server/initializers/migrations/0595-remote-url.ts similarity index 100% rename from server/initializers/migrations/0595-remote-url.ts rename to server/server/initializers/migrations/0595-remote-url.ts diff --git a/server/initializers/migrations/0600-duplicate-video-files.ts b/server/server/initializers/migrations/0600-duplicate-video-files.ts similarity index 100% rename from server/initializers/migrations/0600-duplicate-video-files.ts rename to server/server/initializers/migrations/0600-duplicate-video-files.ts diff --git a/server/server/initializers/migrations/0605-actor-missing-keys.ts b/server/server/initializers/migrations/0605-actor-missing-keys.ts new file mode 100644 index 000000000..3ef95ea22 --- /dev/null +++ b/server/server/initializers/migrations/0605-actor-missing-keys.ts @@ -0,0 +1,33 @@ +import * as Sequelize from 'sequelize' +import { generateRSAKeyPairPromise } from '../../helpers/core-utils.js' +import { PRIVATE_RSA_KEY_SIZE } from '../constants.js' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize + db: any +}): Promise { + + { + const query = 'SELECT * FROM "actor" WHERE "serverId" IS NULL AND "publicKey" IS NULL' + const options = { type: Sequelize.QueryTypes.SELECT as Sequelize.QueryTypes.SELECT } + const actors = await utils.sequelize.query(query, options) + + for (const actor of actors) { + const { privateKey, publicKey } = await generateRSAKeyPairPromise(PRIVATE_RSA_KEY_SIZE) + + const queryUpdate = `UPDATE "actor" SET "publicKey" = '${publicKey}', "privateKey" = '${privateKey}' WHERE id = ${actor.id}` + await utils.sequelize.query(queryUpdate) + } + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/initializers/migrations/0610-views-index copy.ts b/server/server/initializers/migrations/0610-views-index copy.ts similarity index 100% rename from server/initializers/migrations/0610-views-index copy.ts rename to server/server/initializers/migrations/0610-views-index copy.ts diff --git a/server/initializers/migrations/0612-captions-unique.ts b/server/server/initializers/migrations/0612-captions-unique.ts similarity index 100% rename from server/initializers/migrations/0612-captions-unique.ts rename to server/server/initializers/migrations/0612-captions-unique.ts diff --git a/server/initializers/migrations/0615-latest-versions-notification-settings.ts b/server/server/initializers/migrations/0615-latest-versions-notification-settings.ts similarity index 100% rename from server/initializers/migrations/0615-latest-versions-notification-settings.ts rename to server/server/initializers/migrations/0615-latest-versions-notification-settings.ts diff --git a/server/initializers/migrations/0620-latest-versions-application.ts b/server/server/initializers/migrations/0620-latest-versions-application.ts similarity index 100% rename from server/initializers/migrations/0620-latest-versions-application.ts rename to server/server/initializers/migrations/0620-latest-versions-application.ts diff --git a/server/initializers/migrations/0625-latest-versions-notification.ts b/server/server/initializers/migrations/0625-latest-versions-notification.ts similarity index 100% rename from server/initializers/migrations/0625-latest-versions-notification.ts rename to server/server/initializers/migrations/0625-latest-versions-notification.ts diff --git a/server/initializers/migrations/0630-banner.ts b/server/server/initializers/migrations/0630-banner.ts similarity index 100% rename from server/initializers/migrations/0630-banner.ts rename to server/server/initializers/migrations/0630-banner.ts diff --git a/server/initializers/migrations/0635-actor-image-size.ts b/server/server/initializers/migrations/0635-actor-image-size.ts similarity index 100% rename from server/initializers/migrations/0635-actor-image-size.ts rename to server/server/initializers/migrations/0635-actor-image-size.ts diff --git a/server/initializers/migrations/0640-unique-keys.ts b/server/server/initializers/migrations/0640-unique-keys.ts similarity index 100% rename from server/initializers/migrations/0640-unique-keys.ts rename to server/server/initializers/migrations/0640-unique-keys.ts diff --git a/server/initializers/migrations/0645-actor-remote-creation-date.ts b/server/server/initializers/migrations/0645-actor-remote-creation-date.ts similarity index 100% rename from server/initializers/migrations/0645-actor-remote-creation-date.ts rename to server/server/initializers/migrations/0645-actor-remote-creation-date.ts diff --git a/server/initializers/migrations/0650-actor-custom-pages.ts b/server/server/initializers/migrations/0650-actor-custom-pages.ts similarity index 100% rename from server/initializers/migrations/0650-actor-custom-pages.ts rename to server/server/initializers/migrations/0650-actor-custom-pages.ts diff --git a/server/initializers/migrations/0655-streaming-playlist-filenames.ts b/server/server/initializers/migrations/0655-streaming-playlist-filenames.ts similarity index 100% rename from server/initializers/migrations/0655-streaming-playlist-filenames.ts rename to server/server/initializers/migrations/0655-streaming-playlist-filenames.ts diff --git a/server/server/initializers/migrations/0660-object-storage.ts b/server/server/initializers/migrations/0660-object-storage.ts new file mode 100644 index 000000000..c9dc7780e --- /dev/null +++ b/server/server/initializers/migrations/0660-object-storage.ts @@ -0,0 +1,56 @@ +import * as Sequelize from 'sequelize' +import { VideoStorage } from '@peertube/peertube-models' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize + db: any +}): Promise { + { + const query = ` + CREATE TABLE IF NOT EXISTS "videoJobInfo" ( + "id" serial, + "pendingMove" INTEGER NOT NULL, + "pendingTranscode" INTEGER NOT NULL, + "videoId" serial UNIQUE NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + "createdAt" timestamp WITH time zone NOT NULL, + "updatedAt" timestamp WITH time zone NOT NULL, + PRIMARY KEY ("id") + ); + ` + + await utils.sequelize.query(query) + } + + { + await utils.queryInterface.addColumn('videoFile', 'storage', { + type: Sequelize.INTEGER, + allowNull: true, + defaultValue: VideoStorage.FILE_SYSTEM + }) + await utils.queryInterface.changeColumn('videoFile', 'storage', { type: Sequelize.INTEGER, allowNull: false, defaultValue: null }) + } + + { + await utils.queryInterface.addColumn('videoStreamingPlaylist', 'storage', { + type: Sequelize.INTEGER, + allowNull: true, + defaultValue: VideoStorage.FILE_SYSTEM + }) + await utils.queryInterface.changeColumn('videoStreamingPlaylist', 'storage', { + type: Sequelize.INTEGER, + allowNull: false, + defaultValue: null + }) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/initializers/migrations/0665-no-account-warning-modal.ts b/server/server/initializers/migrations/0665-no-account-warning-modal.ts similarity index 100% rename from server/initializers/migrations/0665-no-account-warning-modal.ts rename to server/server/initializers/migrations/0665-no-account-warning-modal.ts diff --git a/server/initializers/migrations/0670-pending-job-default.ts b/server/server/initializers/migrations/0670-pending-job-default.ts similarity index 100% rename from server/initializers/migrations/0670-pending-job-default.ts rename to server/server/initializers/migrations/0670-pending-job-default.ts diff --git a/server/initializers/migrations/0675-p2p-enabled.ts b/server/server/initializers/migrations/0675-p2p-enabled.ts similarity index 100% rename from server/initializers/migrations/0675-p2p-enabled.ts rename to server/server/initializers/migrations/0675-p2p-enabled.ts diff --git a/server/initializers/migrations/0680-files-storage-default.ts b/server/server/initializers/migrations/0680-files-storage-default.ts similarity index 100% rename from server/initializers/migrations/0680-files-storage-default.ts rename to server/server/initializers/migrations/0680-files-storage-default.ts diff --git a/server/initializers/migrations/0685-multiple-actor-images.ts b/server/server/initializers/migrations/0685-multiple-actor-images.ts similarity index 100% rename from server/initializers/migrations/0685-multiple-actor-images.ts rename to server/server/initializers/migrations/0685-multiple-actor-images.ts diff --git a/server/server/initializers/migrations/0690-live-latency-mode.ts b/server/server/initializers/migrations/0690-live-latency-mode.ts new file mode 100644 index 000000000..b7f50cf4b --- /dev/null +++ b/server/server/initializers/migrations/0690-live-latency-mode.ts @@ -0,0 +1,35 @@ +import { LiveVideoLatencyMode } from '@peertube/peertube-models' +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize + db: any +}): Promise { + await utils.queryInterface.addColumn('videoLive', 'latencyMode', { + type: Sequelize.INTEGER, + defaultValue: null, + allowNull: true + }, { transaction: utils.transaction }) + + { + const query = `UPDATE "videoLive" SET "latencyMode" = ${LiveVideoLatencyMode.DEFAULT}` + await utils.sequelize.query(query, { type: Sequelize.QueryTypes.UPDATE, transaction: utils.transaction }) + } + + await utils.queryInterface.changeColumn('videoLive', 'latencyMode', { + type: Sequelize.INTEGER, + defaultValue: null, + allowNull: false + }, { transaction: utils.transaction }) +} + +function down () { + throw new Error('Not implemented.') +} + +export { + up, + down +} diff --git a/server/initializers/migrations/0695-remove-remote-rates.ts b/server/server/initializers/migrations/0695-remove-remote-rates.ts similarity index 100% rename from server/initializers/migrations/0695-remove-remote-rates.ts rename to server/server/initializers/migrations/0695-remove-remote-rates.ts diff --git a/server/initializers/migrations/0700-edition-finished-notification.ts b/server/server/initializers/migrations/0700-edition-finished-notification.ts similarity index 100% rename from server/initializers/migrations/0700-edition-finished-notification.ts rename to server/server/initializers/migrations/0700-edition-finished-notification.ts diff --git a/server/initializers/migrations/0705-local-video-viewers.ts b/server/server/initializers/migrations/0705-local-video-viewers.ts similarity index 100% rename from server/initializers/migrations/0705-local-video-viewers.ts rename to server/server/initializers/migrations/0705-local-video-viewers.ts diff --git a/server/initializers/migrations/0710-live-sessions.ts b/server/server/initializers/migrations/0710-live-sessions.ts similarity index 100% rename from server/initializers/migrations/0710-live-sessions.ts rename to server/server/initializers/migrations/0710-live-sessions.ts diff --git a/server/initializers/migrations/0715-video-source.ts b/server/server/initializers/migrations/0715-video-source.ts similarity index 100% rename from server/initializers/migrations/0715-video-source.ts rename to server/server/initializers/migrations/0715-video-source.ts diff --git a/server/initializers/migrations/0720-session-ending-processed.ts b/server/server/initializers/migrations/0720-session-ending-processed.ts similarity index 100% rename from server/initializers/migrations/0720-session-ending-processed.ts rename to server/server/initializers/migrations/0720-session-ending-processed.ts diff --git a/server/initializers/migrations/0725-node-version.ts b/server/server/initializers/migrations/0725-node-version.ts similarity index 100% rename from server/initializers/migrations/0725-node-version.ts rename to server/server/initializers/migrations/0725-node-version.ts diff --git a/server/initializers/migrations/0730-video-channel-sync.ts b/server/server/initializers/migrations/0730-video-channel-sync.ts similarity index 100% rename from server/initializers/migrations/0730-video-channel-sync.ts rename to server/server/initializers/migrations/0730-video-channel-sync.ts diff --git a/server/initializers/migrations/0735-video-channel-sync-import-foreign-key.ts b/server/server/initializers/migrations/0735-video-channel-sync-import-foreign-key.ts similarity index 100% rename from server/initializers/migrations/0735-video-channel-sync-import-foreign-key.ts rename to server/server/initializers/migrations/0735-video-channel-sync-import-foreign-key.ts diff --git a/server/initializers/migrations/0740-fix-old-enums.ts b/server/server/initializers/migrations/0740-fix-old-enums.ts similarity index 100% rename from server/initializers/migrations/0740-fix-old-enums.ts rename to server/server/initializers/migrations/0740-fix-old-enums.ts diff --git a/server/initializers/migrations/0745-user-otp.ts b/server/server/initializers/migrations/0745-user-otp.ts similarity index 100% rename from server/initializers/migrations/0745-user-otp.ts rename to server/server/initializers/migrations/0745-user-otp.ts diff --git a/server/initializers/migrations/0750-user-registration.ts b/server/server/initializers/migrations/0750-user-registration.ts similarity index 100% rename from server/initializers/migrations/0750-user-registration.ts rename to server/server/initializers/migrations/0750-user-registration.ts diff --git a/server/initializers/migrations/0755-unique-viewer-url.ts b/server/server/initializers/migrations/0755-unique-viewer-url.ts similarity index 100% rename from server/initializers/migrations/0755-unique-viewer-url.ts rename to server/server/initializers/migrations/0755-unique-viewer-url.ts diff --git a/server/initializers/migrations/0760-video-live-replay-setting.ts b/server/server/initializers/migrations/0760-video-live-replay-setting.ts similarity index 100% rename from server/initializers/migrations/0760-video-live-replay-setting.ts rename to server/server/initializers/migrations/0760-video-live-replay-setting.ts diff --git a/server/initializers/migrations/0765-remote-transcoding.ts b/server/server/initializers/migrations/0765-remote-transcoding.ts similarity index 100% rename from server/initializers/migrations/0765-remote-transcoding.ts rename to server/server/initializers/migrations/0765-remote-transcoding.ts diff --git a/server/initializers/migrations/0770-actor-preferred-username.ts b/server/server/initializers/migrations/0770-actor-preferred-username.ts similarity index 100% rename from server/initializers/migrations/0770-actor-preferred-username.ts rename to server/server/initializers/migrations/0770-actor-preferred-username.ts diff --git a/server/initializers/migrations/0775-add-user-is-email-public.ts b/server/server/initializers/migrations/0775-add-user-is-email-public.ts similarity index 100% rename from server/initializers/migrations/0775-add-user-is-email-public.ts rename to server/server/initializers/migrations/0775-add-user-is-email-public.ts diff --git a/server/initializers/migrations/0780-notification-registration.ts b/server/server/initializers/migrations/0780-notification-registration.ts similarity index 100% rename from server/initializers/migrations/0780-notification-registration.ts rename to server/server/initializers/migrations/0780-notification-registration.ts diff --git a/server/initializers/migrations/0785-video-password-protection.ts b/server/server/initializers/migrations/0785-video-password-protection.ts similarity index 100% rename from server/initializers/migrations/0785-video-password-protection.ts rename to server/server/initializers/migrations/0785-video-password-protection.ts diff --git a/server/initializers/migrations/0790-thumbnail-disk.ts b/server/server/initializers/migrations/0790-thumbnail-disk.ts similarity index 100% rename from server/initializers/migrations/0790-thumbnail-disk.ts rename to server/server/initializers/migrations/0790-thumbnail-disk.ts diff --git a/server/initializers/migrations/0795-duplicate-runner-name.ts b/server/server/initializers/migrations/0795-duplicate-runner-name.ts similarity index 100% rename from server/initializers/migrations/0795-duplicate-runner-name.ts rename to server/server/initializers/migrations/0795-duplicate-runner-name.ts diff --git a/server/initializers/migrations/0800-video-replace-file.ts b/server/server/initializers/migrations/0800-video-replace-file.ts similarity index 100% rename from server/initializers/migrations/0800-video-replace-file.ts rename to server/server/initializers/migrations/0800-video-replace-file.ts diff --git a/server/server/initializers/migrator.ts b/server/server/initializers/migrator.ts new file mode 100644 index 000000000..a6494fb16 --- /dev/null +++ b/server/server/initializers/migrator.ts @@ -0,0 +1,106 @@ +import { readdir } from 'fs/promises' +import { join } from 'path' +import { QueryTypes } from 'sequelize' +import { currentDir } from '@peertube/peertube-node-utils' +import { logger } from '../helpers/logger.js' +import { LAST_MIGRATION_VERSION } from './constants.js' +import { sequelizeTypescript } from './database.js' + +async function migrate () { + const tables = await sequelizeTypescript.getQueryInterface().showAllTables() + + // No tables, we don't need to migrate anything + // The installer will do that + if (tables.length === 0) return + + let actualVersion: number | null = null + + const query = 'SELECT "migrationVersion" FROM "application"' + const options = { + type: QueryTypes.SELECT as QueryTypes.SELECT + } + + const rows = await sequelizeTypescript.query<{ migrationVersion: number }>(query, options) + if (rows?.[0]?.migrationVersion) { + actualVersion = rows[0].migrationVersion + } + + if (actualVersion === null) { + await sequelizeTypescript.query('INSERT INTO "application" ("migrationVersion") VALUES (0)') + actualVersion = 0 + } + + // No need migrations, abort + if (actualVersion >= LAST_MIGRATION_VERSION) return + + // If there are a new migration scripts + logger.info('Begin migrations.') + + const migrationScripts = await getMigrationScripts() + + for (const migrationScript of migrationScripts) { + try { + await executeMigration(actualVersion, migrationScript) + } catch (err) { + logger.error('Cannot execute migration %s.', migrationScript.version, { err }) + process.exit(-1) + } + } + + logger.info('Migrations finished. New migration version schema: %s', LAST_MIGRATION_VERSION) +} + +// --------------------------------------------------------------------------- + +export { + migrate +} + +// --------------------------------------------------------------------------- + +async function getMigrationScripts () { + const files = await readdir(join(currentDir(import.meta.url), 'migrations')) + const filesToMigrate: { + version: string + script: string + }[] = [] + + files + .filter(file => file.endsWith('.js')) + .forEach(file => { + // Filename is something like 'version-blabla.js' + const version = file.split('-')[0] + filesToMigrate.push({ + version, + script: file + }) + }) + + return filesToMigrate +} + +async function executeMigration (actualVersion: number, entity: { version: string, script: string }) { + const versionScript = parseInt(entity.version, 10) + + // Do not execute old migration scripts + if (versionScript <= actualVersion) return undefined + + // Load the migration module and run it + const migrationScriptName = entity.script + logger.info('Executing %s migration script.', migrationScriptName) + + const migrationScript = await import(join(currentDir(import.meta.url), 'migrations', migrationScriptName)) + + return sequelizeTypescript.transaction(async t => { + const options = { + transaction: t, + queryInterface: sequelizeTypescript.getQueryInterface(), + sequelize: sequelizeTypescript + } + + await migrationScript.up(options) + + // Update the new migration version + await sequelizeTypescript.query('UPDATE "application" SET "migrationVersion" = ' + versionScript, { transaction: t }) + }) +} diff --git a/server/server/lib/activitypub/activity.ts b/server/server/lib/activitypub/activity.ts new file mode 100644 index 000000000..ea4b3bd63 --- /dev/null +++ b/server/server/lib/activitypub/activity.ts @@ -0,0 +1,74 @@ +import { doJSONRequest, PeerTubeRequestOptions } from '@server/helpers/requests.js' +import { CONFIG } from '@server/initializers/config.js' +import { ActivityObject, ActivityPubActor, ActivityType, APObjectId } from '@peertube/peertube-models' +import { buildSignedRequestOptions } from './send/index.js' + +export function getAPId (object: string | { id: string }) { + if (typeof object === 'string') return object + + return object.id +} + +export function getActivityStreamDuration (duration: number) { + // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration + return 'PT' + duration + 'S' +} + +export function getDurationFromActivityStream (duration: string) { + return parseInt(duration.replace(/[^\d]+/, '')) +} + +// --------------------------------------------------------------------------- + +export function buildAvailableActivities (): ActivityType[] { + return [ + 'Create', + 'Update', + 'Delete', + 'Follow', + 'Accept', + 'Announce', + 'Undo', + 'Like', + 'Reject', + 'View', + 'Dislike', + 'Flag' + ] +} + +// --------------------------------------------------------------------------- + +export async function fetchAP (url: string, moreOptions: PeerTubeRequestOptions = {}) { + const options = { + activityPub: true, + + httpSignature: CONFIG.FEDERATION.SIGN_FEDERATED_FETCHES + ? await buildSignedRequestOptions({ hasPayload: false }) + : undefined, + + ...moreOptions + } + + return doJSONRequest(url, options) +} + +export async function fetchAPObjectIfNeeded (object: APObjectId) { + if (typeof object === 'string') { + const { body } = await fetchAP>(object) + + return body + } + + return object as Exclude +} + +export async function findLatestAPRedirection (url: string, iteration = 1) { + if (iteration > 10) throw new Error('Too much iterations to find final URL ' + url) + + const { headers } = await fetchAP(url, { followRedirect: false }) + + if (headers.location) return findLatestAPRedirection(headers.location, iteration + 1) + + return url +} diff --git a/server/server/lib/activitypub/actors/get.ts b/server/server/lib/activitypub/actors/get.ts new file mode 100644 index 000000000..d301aa8f1 --- /dev/null +++ b/server/server/lib/activitypub/actors/get.ts @@ -0,0 +1,149 @@ +import { arrayify } from '@peertube/peertube-core-utils' +import { ActivityPubActor, APObjectId } from '@peertube/peertube-models' +import { retryTransactionWrapper } from '@server/helpers/database-utils.js' +import { logger } from '@server/helpers/logger.js' +import { JobQueue } from '@server/lib/job-queue/index.js' +import { ActorLoadByUrlType, loadActorByUrl } from '@server/lib/model-loaders/index.js' +import { + MActor, + MActorAccountChannelId, + MActorAccountChannelIdActor, + MActorAccountId, + MActorFullActor +} from '@server/types/models/index.js' +import { fetchAPObjectIfNeeded, getAPId } from '../activity.js' +import { checkUrlsSameHost } from '../url.js' +import { refreshActorIfNeeded } from './refresh.js' +import { APActorCreator, fetchRemoteActor } from './shared/index.js' + +function getOrCreateAPActor ( + activityActor: string | ActivityPubActor, + fetchType: 'all', + recurseIfNeeded?: boolean, + updateCollections?: boolean +): Promise + +function getOrCreateAPActor ( + activityActor: string | ActivityPubActor, + fetchType?: 'association-ids', + recurseIfNeeded?: boolean, + updateCollections?: boolean +): Promise + +async function getOrCreateAPActor ( + activityActor: string | ActivityPubActor, + fetchType: ActorLoadByUrlType = 'association-ids', + recurseIfNeeded = true, + updateCollections = false +): Promise { + const actorUrl = getAPId(activityActor) + let actor = await loadActorFromDB(actorUrl, fetchType) + + let created = false + let accountPlaylistsUrl: string + + // We don't have this actor in our database, fetch it on remote + if (!actor) { + const { actorObject } = await fetchRemoteActor(actorUrl) + if (actorObject === undefined) throw new Error('Cannot fetch remote actor ' + actorUrl) + + // actorUrl is just an alias/redirection, so process object id instead + if (actorObject.id !== actorUrl) return getOrCreateAPActor(actorObject, 'all', recurseIfNeeded, updateCollections) + + // Create the attributed to actor + // In PeerTube a video channel is owned by an account + let ownerActor: MActorFullActor + if (recurseIfNeeded === true && actorObject.type === 'Group') { + ownerActor = await getOrCreateAPOwner(actorObject, actorUrl) + } + + const creator = new APActorCreator(actorObject, ownerActor) + actor = await retryTransactionWrapper(creator.create.bind(creator)) + created = true + accountPlaylistsUrl = actorObject.playlists + } + + if (actor.Account) (actor as MActorAccountChannelIdActor).Account.Actor = actor + if (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor + + const { actor: actorRefreshed, refreshed } = await refreshActorIfNeeded({ actor, fetchedType: fetchType }) + if (!actorRefreshed) throw new Error('Actor ' + actor.url + ' does not exist anymore.') + + await scheduleOutboxFetchIfNeeded(actor, created, refreshed, updateCollections) + await schedulePlaylistFetchIfNeeded(actor, created, accountPlaylistsUrl) + + return actorRefreshed +} + +async function getOrCreateAPOwner (actorObject: ActivityPubActor, actorUrl: string) { + const accountAttributedTo = await findOwner(actorUrl, actorObject.attributedTo, 'Person') + if (!accountAttributedTo) { + throw new Error(`Cannot find account attributed to video channel ${actorUrl}`) + } + + try { + // Don't recurse another time + const recurseIfNeeded = false + return getOrCreateAPActor(accountAttributedTo, 'all', recurseIfNeeded) + } catch (err) { + logger.error('Cannot get or create account attributed to video channel ' + actorUrl) + throw new Error(err) + } +} + +async function findOwner (rootUrl: string, attributedTo: APObjectId[] | APObjectId, type: 'Person' | 'Group') { + for (const actorToCheck of arrayify(attributedTo)) { + const actorObject = await fetchAPObjectIfNeeded(getAPId(actorToCheck)) + + if (!actorObject) { + logger.warn('Unknown attributed to actor %s for owner %s', actorToCheck, rootUrl) + continue + } + + if (checkUrlsSameHost(actorObject.id, rootUrl) !== true) { + logger.warn(`Account attributed to ${actorObject.id} does not have the same host than owner actor url ${rootUrl}`) + continue + } + + if (actorObject.type === type) return actorObject + } + + return undefined +} + +// --------------------------------------------------------------------------- + +export { + getOrCreateAPOwner, + getOrCreateAPActor, + findOwner +} + +// --------------------------------------------------------------------------- + +async function loadActorFromDB (actorUrl: string, fetchType: ActorLoadByUrlType) { + let actor = await loadActorByUrl(actorUrl, fetchType) + + // Orphan actor (not associated to an account of channel) so recreate it + if (actor && (!actor.Account && !actor.VideoChannel)) { + await actor.destroy() + actor = null + } + + return actor +} + +async function scheduleOutboxFetchIfNeeded (actor: MActor, created: boolean, refreshed: boolean, updateCollections: boolean) { + if ((created === true || refreshed === true) && updateCollections === true) { + const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' } + await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) + } +} + +async function schedulePlaylistFetchIfNeeded (actor: MActorAccountId, created: boolean, accountPlaylistsUrl: string) { + // We created a new account: fetch the playlists + if (created === true && actor.Account && accountPlaylistsUrl) { + const payload = { uri: accountPlaylistsUrl, type: 'account-playlists' as 'account-playlists' } + await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) + } +} diff --git a/server/server/lib/activitypub/actors/image.ts b/server/server/lib/activitypub/actors/image.ts new file mode 100644 index 000000000..290f967b7 --- /dev/null +++ b/server/server/lib/activitypub/actors/image.ts @@ -0,0 +1,112 @@ +import { ActorImageType, ActorImageType_Type } from '@peertube/peertube-models' +import { logger } from '@server/helpers/logger.js' +import { ActorImageModel } from '@server/models/actor/actor-image.js' +import { MActorImage, MActorImages } from '@server/types/models/index.js' +import { Transaction } from 'sequelize' + +type ImageInfo = { + name: string + fileUrl: string + height: number + width: number + onDisk?: boolean +} + +async function updateActorImages (actor: MActorImages, type: ActorImageType_Type, imagesInfo: ImageInfo[], t: Transaction) { + const getAvatarsOrBanners = () => { + const result = type === ActorImageType.AVATAR + ? actor.Avatars + : actor.Banners + + return result || [] + } + + if (imagesInfo.length === 0) { + await deleteActorImages(actor, type, t) + } + + // Cleanup old images that did not have a width + for (const oldImageModel of getAvatarsOrBanners()) { + if (oldImageModel.width) continue + + await safeDeleteActorImage(actor, oldImageModel, type, t) + } + + for (const imageInfo of imagesInfo) { + const oldImageModel = getAvatarsOrBanners().find(i => imageInfo.width && i.width === imageInfo.width) + + if (oldImageModel) { + // Don't update the avatar if the file URL did not change + if (imageInfo.fileUrl && oldImageModel.fileUrl === imageInfo.fileUrl) { + continue + } + + await safeDeleteActorImage(actor, oldImageModel, type, t) + } + + const imageModel = await ActorImageModel.create({ + filename: imageInfo.name, + onDisk: imageInfo.onDisk ?? false, + fileUrl: imageInfo.fileUrl, + height: imageInfo.height, + width: imageInfo.width, + type, + actorId: actor.id + }, { transaction: t }) + + addActorImage(actor, type, imageModel) + } + + return actor +} + +async function deleteActorImages (actor: MActorImages, type: ActorImageType_Type, t: Transaction) { + try { + const association = buildAssociationName(type) + + for (const image of actor[association]) { + await image.destroy({ transaction: t }) + } + + actor[association] = [] + } catch (err) { + logger.error('Cannot remove old image of actor %s.', actor.url, { err }) + } + + return actor +} + +async function safeDeleteActorImage (actor: MActorImages, toDelete: MActorImage, type: ActorImageType_Type, t: Transaction) { + try { + await toDelete.destroy({ transaction: t }) + + const association = buildAssociationName(type) + actor[association] = actor[association].filter(image => image.id !== toDelete.id) + } catch (err) { + logger.error('Cannot remove old actor image of actor %s.', actor.url, { err }) + } +} + +// --------------------------------------------------------------------------- + +export { + type ImageInfo, + + updateActorImages, + deleteActorImages +} + +// --------------------------------------------------------------------------- + +function addActorImage (actor: MActorImages, type: ActorImageType_Type, imageModel: MActorImage) { + const association = buildAssociationName(type) + if (!actor[association]) actor[association] = [] + + actor[association].push(imageModel) +} + +function buildAssociationName (type: ActorImageType_Type) { + return type === ActorImageType.AVATAR + ? 'Avatars' + : 'Banners' +} diff --git a/server/server/lib/activitypub/actors/index.ts b/server/server/lib/activitypub/actors/index.ts new file mode 100644 index 000000000..9062e1394 --- /dev/null +++ b/server/server/lib/activitypub/actors/index.ts @@ -0,0 +1,6 @@ +export * from './get.js' +export * from './image.js' +export * from './keys.js' +export * from './refresh.js' +export * from './updater.js' +export * from './webfinger.js' diff --git a/server/server/lib/activitypub/actors/keys.ts b/server/server/lib/activitypub/actors/keys.ts new file mode 100644 index 000000000..bf7973756 --- /dev/null +++ b/server/server/lib/activitypub/actors/keys.ts @@ -0,0 +1,16 @@ +import { createPrivateAndPublicKeys } from '@server/helpers/peertube-crypto.js' +import { MActor } from '@server/types/models/index.js' + +// Set account keys, this could be long so process after the account creation and do not block the client +async function generateAndSaveActorKeys (actor: T) { + const { publicKey, privateKey } = await createPrivateAndPublicKeys() + + actor.publicKey = publicKey + actor.privateKey = privateKey + + return actor.save() +} + +export { + generateAndSaveActorKeys +} diff --git a/server/server/lib/activitypub/actors/refresh.ts b/server/server/lib/activitypub/actors/refresh.ts new file mode 100644 index 000000000..87f017ae8 --- /dev/null +++ b/server/server/lib/activitypub/actors/refresh.ts @@ -0,0 +1,81 @@ +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { CachePromiseFactory } from '@server/helpers/promise-cache.js' +import { PeerTubeRequestError } from '@server/helpers/requests.js' +import { ActorLoadByUrlType } from '@server/lib/model-loaders/index.js' +import { ActorModel } from '@server/models/actor/actor.js' +import { MActorAccountChannelId, MActorFull } from '@server/types/models/index.js' +import { HttpStatusCode } from '@peertube/peertube-models' +import { fetchRemoteActor } from './shared/index.js' +import { APActorUpdater } from './updater.js' +import { getUrlFromWebfinger } from './webfinger.js' + +type RefreshResult = Promise<{ actor: T | MActorFull, refreshed: boolean }> + +type RefreshOptions = { + actor: T + fetchedType: ActorLoadByUrlType +} + +const promiseCache = new CachePromiseFactory(doRefresh, (options: RefreshOptions) => options.actor.url) + +function refreshActorIfNeeded (options: RefreshOptions): RefreshResult { + const actorArg = options.actor + if (!actorArg.isOutdated()) return Promise.resolve({ actor: actorArg, refreshed: false }) + + return promiseCache.run(options) +} + +export { + refreshActorIfNeeded +} + +// --------------------------------------------------------------------------- + +async function doRefresh (options: RefreshOptions): RefreshResult { + const { actor: actorArg, fetchedType } = options + + // We need more attributes + const actor = fetchedType === 'all' + ? actorArg as MActorFull + : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) + + const lTags = loggerTagsFactory('ap', 'actor', 'refresh', actor.url) + + logger.info('Refreshing actor %s.', actor.url, lTags()) + + try { + const actorUrl = await getActorUrl(actor) + const { actorObject } = await fetchRemoteActor(actorUrl) + + if (actorObject === undefined) { + logger.info('Cannot fetch remote actor %s in refresh actor.', actorUrl) + return { actor, refreshed: false } + } + + const updater = new APActorUpdater(actorObject, actor) + await updater.update() + + return { refreshed: true, actor } + } catch (err) { + if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { + logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url, lTags()) + + actor.Account + ? await actor.Account.destroy() + : await actor.VideoChannel.destroy() + + return { actor: undefined, refreshed: false } + } + + logger.info('Cannot refresh actor %s.', actor.url, { err, ...lTags() }) + return { actor, refreshed: false } + } +} + +function getActorUrl (actor: MActorFull) { + return getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) + .catch(err => { + logger.warn('Cannot get actor URL from webfinger, keeping the old one.', { err }) + return actor.url + }) +} diff --git a/server/server/lib/activitypub/actors/shared/creator.ts b/server/server/lib/activitypub/actors/shared/creator.ts new file mode 100644 index 000000000..426f918fb --- /dev/null +++ b/server/server/lib/activitypub/actors/shared/creator.ts @@ -0,0 +1,158 @@ +import { Op, Transaction } from 'sequelize' +import { ActivityPubActor, ActorImageType, ActorImageType_Type } from '@peertube/peertube-models' +import { sequelizeTypescript } from '@server/initializers/database.js' +import { AccountModel } from '@server/models/account/account.js' +import { ActorModel } from '@server/models/actor/actor.js' +import { ServerModel } from '@server/models/server/server.js' +import { VideoChannelModel } from '@server/models/video/video-channel.js' +import { + MAccount, + MAccountDefault, + MActor, + MActorFullActor, + MActorId, + MActorImages, + MChannel, + MServer +} from '@server/types/models/index.js' +import { updateActorImages } from '../image.js' +import { getActorAttributesFromObject, getActorDisplayNameFromObject, getImagesInfoFromObject } from './object-to-model-attributes.js' +import { fetchActorFollowsCount } from './url-to-object.js' + +export class APActorCreator { + + constructor ( + private readonly actorObject: ActivityPubActor, + private readonly ownerActor?: MActorFullActor + ) { + + } + + async create (): Promise { + const { followersCount, followingCount } = await fetchActorFollowsCount(this.actorObject) + + const actorInstance = new ActorModel(getActorAttributesFromObject(this.actorObject, followersCount, followingCount)) + + return sequelizeTypescript.transaction(async t => { + const server = await this.setServer(actorInstance, t) + + const { actorCreated, created } = await this.saveActor(actorInstance, t) + + await this.setImageIfNeeded(actorCreated, ActorImageType.AVATAR, t) + await this.setImageIfNeeded(actorCreated, ActorImageType.BANNER, t) + + await this.tryToFixActorUrlIfNeeded(actorCreated, actorInstance, created, t) + + if (actorCreated.type === 'Person' || actorCreated.type === 'Application') { // Account or PeerTube instance + actorCreated.Account = await this.saveAccount(actorCreated, t) as MAccountDefault + actorCreated.Account.Actor = actorCreated + } + + if (actorCreated.type === 'Group') { // Video channel + const channel = await this.saveVideoChannel(actorCreated, t) + actorCreated.VideoChannel = Object.assign(channel, { Actor: actorCreated, Account: this.ownerActor.Account }) + } + + actorCreated.Server = server + + return actorCreated + }) + } + + private async setServer (actor: MActor, t: Transaction) { + const actorHost = new URL(actor.url).host + + const serverOptions = { + where: { + host: actorHost + }, + defaults: { + host: actorHost + }, + transaction: t + } + const [ server ] = await ServerModel.findOrCreate(serverOptions) + + // Save our new account in database + actor.serverId = server.id + + return server as MServer + } + + private async setImageIfNeeded (actor: MActor, type: ActorImageType_Type, t: Transaction) { + const imagesInfo = getImagesInfoFromObject(this.actorObject, type) + if (imagesInfo.length === 0) return + + return updateActorImages(actor as MActorImages, type, imagesInfo, t) + } + + private async saveActor (actor: MActor, t: Transaction) { + // Force the actor creation using findOrCreate() instead of save() + // Sometimes Sequelize skips the save() when it thinks the instance already exists + // (which could be false in a retried query) + const [ actorCreated, created ] = await ActorModel.findOrCreate({ + defaults: actor.toJSON(), + where: { + [Op.or]: [ + { + url: actor.url + }, + { + serverId: actor.serverId, + preferredUsername: actor.preferredUsername + } + ] + }, + transaction: t + }) + + return { actorCreated, created } + } + + private async tryToFixActorUrlIfNeeded (actorCreated: MActor, newActor: MActor, created: boolean, t: Transaction) { + // Try to fix non HTTPS accounts of remote instances that fixed their URL afterwards + if (created !== true && actorCreated.url !== newActor.url) { + // Only fix http://example.com/account/djidane to https://example.com/account/djidane + if (actorCreated.url.replace(/^http:\/\//, '') !== newActor.url.replace(/^https:\/\//, '')) { + throw new Error(`Actor from DB with URL ${actorCreated.url} does not correspond to actor ${newActor.url}`) + } + + actorCreated.url = newActor.url + await actorCreated.save({ transaction: t }) + } + } + + private async saveAccount (actor: MActorId, t: Transaction) { + const [ accountCreated ] = await AccountModel.findOrCreate({ + defaults: { + name: getActorDisplayNameFromObject(this.actorObject), + description: this.actorObject.summary, + actorId: actor.id + }, + where: { + actorId: actor.id + }, + transaction: t + }) + + return accountCreated as MAccount + } + + private async saveVideoChannel (actor: MActorId, t: Transaction) { + const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({ + defaults: { + name: getActorDisplayNameFromObject(this.actorObject), + description: this.actorObject.summary, + support: this.actorObject.support, + actorId: actor.id, + accountId: this.ownerActor.Account.id + }, + where: { + actorId: actor.id + }, + transaction: t + }) + + return videoChannelCreated as MChannel + } +} diff --git a/server/server/lib/activitypub/actors/shared/index.ts b/server/server/lib/activitypub/actors/shared/index.ts new file mode 100644 index 000000000..d213a7986 --- /dev/null +++ b/server/server/lib/activitypub/actors/shared/index.ts @@ -0,0 +1,3 @@ +export * from './creator.js' +export * from './object-to-model-attributes.js' +export * from './url-to-object.js' diff --git a/server/server/lib/activitypub/actors/shared/object-to-model-attributes.ts b/server/server/lib/activitypub/actors/shared/object-to-model-attributes.ts new file mode 100644 index 000000000..987c48085 --- /dev/null +++ b/server/server/lib/activitypub/actors/shared/object-to-model-attributes.ts @@ -0,0 +1,83 @@ +import { ActivityIconObject, ActivityPubActor, ActorImageType, ActorImageType_Type } from '@peertube/peertube-models' +import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js' +import { MIMETYPES } from '@server/initializers/constants.js' +import { ActorModel } from '@server/models/actor/actor.js' +import { FilteredModelAttributes } from '@server/types/index.js' +import { buildUUID, getLowercaseExtension } from '@peertube/peertube-node-utils' + +function getActorAttributesFromObject ( + actorObject: ActivityPubActor, + followersCount: number, + followingCount: number +): FilteredModelAttributes { + return { + type: actorObject.type, + preferredUsername: actorObject.preferredUsername, + url: actorObject.id, + publicKey: actorObject.publicKey.publicKeyPem, + privateKey: null, + followersCount, + followingCount, + inboxUrl: actorObject.inbox, + outboxUrl: actorObject.outbox, + followersUrl: actorObject.followers, + followingUrl: actorObject.following, + + sharedInboxUrl: actorObject.endpoints?.sharedInbox + ? actorObject.endpoints.sharedInbox + : null + } +} + +function getImagesInfoFromObject (actorObject: ActivityPubActor, type: ActorImageType_Type) { + const iconsOrImages = type === ActorImageType.AVATAR + ? actorObject.icon + : actorObject.image + + return normalizeIconOrImage(iconsOrImages) + .map(iconOrImage => { + const mimetypes = MIMETYPES.IMAGE + + if (iconOrImage.type !== 'Image' || !isActivityPubUrlValid(iconOrImage.url)) return undefined + + let extension: string + + if (iconOrImage.mediaType) { + extension = mimetypes.MIMETYPE_EXT[iconOrImage.mediaType] + } else { + const tmp = getLowercaseExtension(iconOrImage.url) + + if (mimetypes.EXT_MIMETYPE[tmp] !== undefined) extension = tmp + } + + if (!extension) return undefined + + return { + name: buildUUID() + extension, + fileUrl: iconOrImage.url, + height: iconOrImage.height, + width: iconOrImage.width, + type + } + }) + .filter(i => !!i) +} + +function getActorDisplayNameFromObject (actorObject: ActivityPubActor) { + return actorObject.name || actorObject.preferredUsername +} + +export { + getActorAttributesFromObject, + getImagesInfoFromObject, + getActorDisplayNameFromObject +} + +// --------------------------------------------------------------------------- + +function normalizeIconOrImage (icon: ActivityIconObject | ActivityIconObject[]): ActivityIconObject[] { + if (Array.isArray(icon)) return icon + if (icon) return [ icon ] + + return [] +} diff --git a/server/server/lib/activitypub/actors/shared/url-to-object.ts b/server/server/lib/activitypub/actors/shared/url-to-object.ts new file mode 100644 index 000000000..6dc04e396 --- /dev/null +++ b/server/server/lib/activitypub/actors/shared/url-to-object.ts @@ -0,0 +1,56 @@ +import { sanitizeAndCheckActorObject } from '@server/helpers/custom-validators/activitypub/actor.js' +import { logger } from '@server/helpers/logger.js' +import { ActivityPubActor, ActivityPubOrderedCollection } from '@peertube/peertube-models' +import { fetchAP } from '../../activity.js' +import { checkUrlsSameHost } from '../../url.js' + +async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode: number, actorObject: ActivityPubActor }> { + logger.info('Fetching remote actor %s.', actorUrl) + + const { body, statusCode } = await fetchAP(actorUrl) + + if (sanitizeAndCheckActorObject(body) === false) { + logger.debug('Remote actor JSON is not valid.', { actorJSON: body }) + return { actorObject: undefined, statusCode } + } + + if (checkUrlsSameHost(body.id, actorUrl) !== true) { + logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, body.id) + return { actorObject: undefined, statusCode } + } + + return { + statusCode, + + actorObject: body + } +} + +async function fetchActorFollowsCount (actorObject: ActivityPubActor) { + let followersCount = 0 + let followingCount = 0 + + if (actorObject.followers) followersCount = await fetchActorTotalItems(actorObject.followers) + if (actorObject.following) followingCount = await fetchActorTotalItems(actorObject.following) + + return { followersCount, followingCount } +} + +// --------------------------------------------------------------------------- +export { + fetchActorFollowsCount, + fetchRemoteActor +} + +// --------------------------------------------------------------------------- + +async function fetchActorTotalItems (url: string) { + try { + const { body } = await fetchAP>(url) + + return body.totalItems || 0 + } catch (err) { + logger.info('Cannot fetch remote actor count %s.', url, { err }) + return 0 + } +} diff --git a/server/server/lib/activitypub/actors/updater.ts b/server/server/lib/activitypub/actors/updater.ts new file mode 100644 index 000000000..81953611a --- /dev/null +++ b/server/server/lib/activitypub/actors/updater.ts @@ -0,0 +1,91 @@ +import { resetSequelizeInstance, runInReadCommittedTransaction } from '@server/helpers/database-utils.js' +import { logger } from '@server/helpers/logger.js' +import { AccountModel } from '@server/models/account/account.js' +import { VideoChannelModel } from '@server/models/video/video-channel.js' +import { MAccount, MActor, MActorFull, MChannel } from '@server/types/models/index.js' +import { ActivityPubActor, ActorImageType } from '@peertube/peertube-models' +import { getOrCreateAPOwner } from './get.js' +import { updateActorImages } from './image.js' +import { fetchActorFollowsCount } from './shared/index.js' +import { getImagesInfoFromObject } from './shared/object-to-model-attributes.js' + +export class APActorUpdater { + + private readonly accountOrChannel: MAccount | MChannel + + constructor ( + private readonly actorObject: ActivityPubActor, + private readonly actor: MActorFull + ) { + if (this.actorObject.type === 'Group') this.accountOrChannel = this.actor.VideoChannel + else this.accountOrChannel = this.actor.Account + } + + async update () { + const avatarsInfo = getImagesInfoFromObject(this.actorObject, ActorImageType.AVATAR) + const bannersInfo = getImagesInfoFromObject(this.actorObject, ActorImageType.BANNER) + + try { + await this.updateActorInstance(this.actor, this.actorObject) + + this.accountOrChannel.name = this.actorObject.name || this.actorObject.preferredUsername + this.accountOrChannel.description = this.actorObject.summary + + if (this.accountOrChannel instanceof VideoChannelModel) { + const owner = await getOrCreateAPOwner(this.actorObject, this.actorObject.url) + this.accountOrChannel.accountId = owner.Account.id + this.accountOrChannel.Account = owner.Account as AccountModel + + this.accountOrChannel.support = this.actorObject.support + } + + await runInReadCommittedTransaction(async t => { + await updateActorImages(this.actor, ActorImageType.BANNER, bannersInfo, t) + await updateActorImages(this.actor, ActorImageType.AVATAR, avatarsInfo, t) + }) + + await runInReadCommittedTransaction(async t => { + await this.actor.save({ transaction: t }) + await this.accountOrChannel.save({ transaction: t }) + }) + + logger.info('Remote account %s updated', this.actorObject.url) + } catch (err) { + if (this.actor !== undefined) { + await resetSequelizeInstance(this.actor) + } + + if (this.accountOrChannel !== undefined) { + await resetSequelizeInstance(this.accountOrChannel) + } + + // This is just a debug because we will retry the insert + logger.debug('Cannot update the remote account.', { err }) + throw err + } + } + + private async updateActorInstance (actorInstance: MActor, actorObject: ActivityPubActor) { + const { followersCount, followingCount } = await fetchActorFollowsCount(actorObject) + + actorInstance.type = actorObject.type + actorInstance.preferredUsername = actorObject.preferredUsername + actorInstance.url = actorObject.id + actorInstance.publicKey = actorObject.publicKey.publicKeyPem + actorInstance.followersCount = followersCount + actorInstance.followingCount = followingCount + actorInstance.inboxUrl = actorObject.inbox + actorInstance.outboxUrl = actorObject.outbox + actorInstance.followersUrl = actorObject.followers + actorInstance.followingUrl = actorObject.following + + if (actorObject.published) actorInstance.remoteCreatedAt = new Date(actorObject.published) + + if (actorObject.endpoints?.sharedInbox) { + actorInstance.sharedInboxUrl = actorObject.endpoints.sharedInbox + } + + // Force actor update + actorInstance.changed('updatedAt', true) + } +} diff --git a/server/server/lib/activitypub/actors/webfinger.ts b/server/server/lib/activitypub/actors/webfinger.ts new file mode 100644 index 000000000..21d148a61 --- /dev/null +++ b/server/server/lib/activitypub/actors/webfinger.ts @@ -0,0 +1,67 @@ +import WebFinger from 'webfinger.js' +import { WebFingerData } from '@peertube/peertube-models' +import { isProdInstance } from '@peertube/peertube-node-utils' +import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js' +import { REQUEST_TIMEOUTS, WEBSERVER } from '@server/initializers/constants.js' +import { ActorModel } from '@server/models/actor/actor.js' +import { MActorFull } from '@server/types/models/index.js' + +const webfinger = new WebFinger({ + webfist_fallback: false, + tls_only: isProdInstance(), + uri_fallback: false, + request_timeout: REQUEST_TIMEOUTS.DEFAULT +}) + +async function loadActorUrlOrGetFromWebfinger (uriArg: string) { + // Handle strings like @toto@example.com + const uri = uriArg.startsWith('@') ? uriArg.slice(1) : uriArg + + const [ name, host ] = uri.split('@') + let actor: MActorFull + + if (!host || host === WEBSERVER.HOST) { + actor = await ActorModel.loadLocalByName(name) + } else { + actor = await ActorModel.loadByNameAndHost(name, host) + } + + if (actor) return actor.url + + return getUrlFromWebfinger(uri) +} + +async function getUrlFromWebfinger (uri: string) { + const webfingerData: WebFingerData = await webfingerLookup(uri) + return getLinkOrThrow(webfingerData) +} + +// --------------------------------------------------------------------------- + +export { + getUrlFromWebfinger, + loadActorUrlOrGetFromWebfinger +} + +// --------------------------------------------------------------------------- + +function getLinkOrThrow (webfingerData: WebFingerData) { + if (Array.isArray(webfingerData.links) === false) throw new Error('WebFinger links is not an array.') + + const selfLink = webfingerData.links.find(l => l.rel === 'self') + if (selfLink === undefined || isActivityPubUrlValid(selfLink.href) === false) { + throw new Error('Cannot find self link or href is not a valid URL.') + } + + return selfLink.href +} + +function webfingerLookup (nameWithHost: string) { + return new Promise((res, rej) => { + webfinger.lookup(nameWithHost, (err, p) => { + if (err) return rej(err) + + return res(p.object) + }) + }) +} diff --git a/server/server/lib/activitypub/audience.ts b/server/server/lib/activitypub/audience.ts new file mode 100644 index 000000000..197acdb85 --- /dev/null +++ b/server/server/lib/activitypub/audience.ts @@ -0,0 +1,34 @@ +import { ActivityAudience } from '@peertube/peertube-models' +import { ACTIVITY_PUB } from '../../initializers/constants.js' +import { MActorFollowersUrl } from '../../types/models/index.js' + +function getAudience (actorSender: MActorFollowersUrl, isPublic = true) { + return buildAudience([ actorSender.followersUrl ], isPublic) +} + +function buildAudience (followerUrls: string[], isPublic = true) { + let to: string[] = [] + let cc: string[] = [] + + if (isPublic) { + to = [ ACTIVITY_PUB.PUBLIC ] + cc = followerUrls + } else { // Unlisted + to = [] + cc = [] + } + + return { to, cc } +} + +function audiencify (object: T, audience: ActivityAudience) { + return { ...audience, ...object } +} + +// --------------------------------------------------------------------------- + +export { + buildAudience, + getAudience, + audiencify +} diff --git a/server/server/lib/activitypub/cache-file.ts b/server/server/lib/activitypub/cache-file.ts new file mode 100644 index 000000000..ef8bbace8 --- /dev/null +++ b/server/server/lib/activitypub/cache-file.ts @@ -0,0 +1,82 @@ +import { Transaction } from 'sequelize' +import { MActorId, MVideoRedundancy, MVideoWithAllFiles } from '@server/types/models/index.js' +import { CacheFileObject, VideoStreamingPlaylistType } from '@peertube/peertube-models' +import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy.js' + +async function createOrUpdateCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) { + const redundancyModel = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t) + + if (redundancyModel) { + return updateCacheFile(cacheFileObject, redundancyModel, video, byActor, t) + } + + return createCacheFile(cacheFileObject, video, byActor, t) +} + +// --------------------------------------------------------------------------- + +export { + createOrUpdateCacheFile +} + +// --------------------------------------------------------------------------- + +function createCacheFile (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId, t: Transaction) { + const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) + + return VideoRedundancyModel.create(attributes, { transaction: t }) +} + +function updateCacheFile ( + cacheFileObject: CacheFileObject, + redundancyModel: MVideoRedundancy, + video: MVideoWithAllFiles, + byActor: MActorId, + t: Transaction +) { + if (redundancyModel.actorId !== byActor.id) { + throw new Error('Cannot update redundancy ' + redundancyModel.url + ' of another actor.') + } + + const attributes = cacheFileActivityObjectToDBAttributes(cacheFileObject, video, byActor) + + redundancyModel.expiresOn = attributes.expiresOn + redundancyModel.fileUrl = attributes.fileUrl + + return redundancyModel.save({ transaction: t }) +} + +function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject, video: MVideoWithAllFiles, byActor: MActorId) { + + if (cacheFileObject.url.mediaType === 'application/x-mpegURL') { + const url = cacheFileObject.url + + const playlist = video.VideoStreamingPlaylists.find(t => t.type === VideoStreamingPlaylistType.HLS) + if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url) + + return { + expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, + url: cacheFileObject.id, + fileUrl: url.href, + strategy: null, + videoStreamingPlaylistId: playlist.id, + actorId: byActor.id + } + } + + const url = cacheFileObject.url + const videoFile = video.VideoFiles.find(f => { + return f.resolution === url.height && f.fps === url.fps + }) + + if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`) + + return { + expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null, + url: cacheFileObject.id, + fileUrl: url.href, + strategy: null, + videoFileId: videoFile.id, + actorId: byActor.id + } +} diff --git a/server/server/lib/activitypub/collection.ts b/server/server/lib/activitypub/collection.ts new file mode 100644 index 000000000..280c3a5e9 --- /dev/null +++ b/server/server/lib/activitypub/collection.ts @@ -0,0 +1,63 @@ +import Bluebird from 'bluebird' +import validator from 'validator' +import { pageToStartAndCount } from '@server/helpers/core-utils.js' +import { ACTIVITY_PUB } from '@server/initializers/constants.js' +import { ResultList } from '@peertube/peertube-models' +import { forceNumber } from '@peertube/peertube-core-utils' + +type ActivityPubCollectionPaginationHandler = (start: number, count: number) => Bluebird> | Promise> + +async function activityPubCollectionPagination ( + baseUrl: string, + handler: ActivityPubCollectionPaginationHandler, + page?: any, + size = ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE +) { + if (!page || !validator.default.isInt(page)) { + // We just display the first page URL, we only need the total items + const result = await handler(0, 1) + + return { + id: baseUrl, + type: 'OrderedCollection', + totalItems: result.total, + first: result.data.length === 0 + ? undefined + : baseUrl + '?page=1' + } + } + + const { start, count } = pageToStartAndCount(page, size) + const result = await handler(start, count) + + let next: string | undefined + let prev: string | undefined + + // Assert page is a number + page = forceNumber(page) + + // There are more results + if (result.total > page * size) { + next = baseUrl + '?page=' + (page + 1) + } + + if (page > 1) { + prev = baseUrl + '?page=' + (page - 1) + } + + return { + id: baseUrl + '?page=' + page, + type: 'OrderedCollectionPage', + prev, + next, + partOf: baseUrl, + orderedItems: result.data, + totalItems: result.total + } +} + +// --------------------------------------------------------------------------- + +export { + activityPubCollectionPagination +} diff --git a/server/server/lib/activitypub/context.ts b/server/server/lib/activitypub/context.ts new file mode 100644 index 000000000..8de565976 --- /dev/null +++ b/server/server/lib/activitypub/context.ts @@ -0,0 +1,10 @@ +import { Hooks } from '../plugins/hooks.js' + +export function getContextFilter () { + return (contextData: T) => { + return Hooks.wrapObject( + contextData, + 'filter:activity-pub.activity.context.build.result' + ) + } +} diff --git a/server/server/lib/activitypub/crawl.ts b/server/server/lib/activitypub/crawl.ts new file mode 100644 index 000000000..bc773279a --- /dev/null +++ b/server/server/lib/activitypub/crawl.ts @@ -0,0 +1,58 @@ +import Bluebird from 'bluebird' +import { URL } from 'url' +import { ActivityPubOrderedCollection } from '@peertube/peertube-models' +import { retryTransactionWrapper } from '@server/helpers/database-utils.js' +import { logger } from '../../helpers/logger.js' +import { ACTIVITY_PUB, WEBSERVER } from '../../initializers/constants.js' +import { fetchAP } from './activity.js' + +type HandlerFunction = (items: T[]) => (Promise | Bluebird) +type CleanerFunction = (startedDate: Date) => Promise + +async function crawlCollectionPage (argUrl: string, handler: HandlerFunction, cleaner?: CleanerFunction) { + let url = argUrl + + logger.info('Crawling ActivityPub data on %s.', url) + + const startDate = new Date() + + const response = await fetchAP>(url) + const firstBody = response.body + + const limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT + let i = 0 + let nextLink = firstBody.first + while (nextLink && i < limit) { + let body: any + + if (typeof nextLink === 'string') { + // Don't crawl ourselves + const remoteHost = new URL(nextLink).host + if (remoteHost === WEBSERVER.HOST) continue + + url = nextLink + + const res = await fetchAP>(url) + body = res.body + } else { + // nextLink is already the object we want + body = nextLink + } + + nextLink = body.next + i++ + + if (Array.isArray(body.orderedItems)) { + const items = body.orderedItems + logger.info('Processing %i ActivityPub items for %s.', items.length, url) + + await handler(items) + } + } + + if (cleaner) await retryTransactionWrapper(cleaner, startDate) +} + +export { + crawlCollectionPage +} diff --git a/server/server/lib/activitypub/follow.ts b/server/server/lib/activitypub/follow.ts new file mode 100644 index 000000000..d6c60602b --- /dev/null +++ b/server/server/lib/activitypub/follow.ts @@ -0,0 +1,51 @@ +import { Transaction } from 'sequelize' +import { getServerActor } from '@server/models/application/application.js' +import { logger } from '../../helpers/logger.js' +import { CONFIG } from '../../initializers/config.js' +import { SERVER_ACTOR_NAME } from '../../initializers/constants.js' +import { ServerModel } from '../../models/server/server.js' +import { MActorFollowActors } from '../../types/models/index.js' +import { JobQueue } from '../job-queue/index.js' + +async function autoFollowBackIfNeeded (actorFollow: MActorFollowActors, transaction?: Transaction) { + if (!CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_BACK.ENABLED) return + + const follower = actorFollow.ActorFollower + + if (follower.type === 'Application' && follower.preferredUsername === SERVER_ACTOR_NAME) { + logger.info('Auto follow back %s.', follower.url) + + const me = await getServerActor() + + const server = await ServerModel.load(follower.serverId, transaction) + const host = server.host + + const payload = { + host, + name: SERVER_ACTOR_NAME, + followerActorId: me.id, + isAutoFollow: true + } + + JobQueue.Instance.createJobAsync({ type: 'activitypub-follow', payload }) + } +} + +// If we only have an host, use a default account handle +function getRemoteNameAndHost (handleOrHost: string) { + let name = SERVER_ACTOR_NAME + let host = handleOrHost + + const splitted = handleOrHost.split('@') + if (splitted.length === 2) { + name = splitted[0] + host = splitted[1] + } + + return { name, host } +} + +export { + autoFollowBackIfNeeded, + getRemoteNameAndHost +} diff --git a/server/server/lib/activitypub/inbox-manager.ts b/server/server/lib/activitypub/inbox-manager.ts new file mode 100644 index 000000000..ecf95d1ee --- /dev/null +++ b/server/server/lib/activitypub/inbox-manager.ts @@ -0,0 +1,47 @@ +import PQueue from 'p-queue' +import { logger } from '@server/helpers/logger.js' +import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants.js' +import { MActorDefault, MActorSignature } from '@server/types/models/index.js' +import { Activity } from '@peertube/peertube-models' +import { StatsManager } from '../stat-manager.js' +import { processActivities } from './process/index.js' + +class InboxManager { + + private static instance: InboxManager + private readonly inboxQueue: PQueue + + private constructor () { + this.inboxQueue = new PQueue({ concurrency: 1 }) + + setInterval(() => { + StatsManager.Instance.updateInboxWaiting(this.getActivityPubMessagesWaiting()) + }, SCHEDULER_INTERVALS_MS.UPDATE_INBOX_STATS) + } + + addInboxMessage (param: { + activities: Activity[] + signatureActor?: MActorSignature + inboxActor?: MActorDefault + }) { + this.inboxQueue.add(() => { + const options = { signatureActor: param.signatureActor, inboxActor: param.inboxActor } + + return processActivities(param.activities, options) + }).catch(err => logger.error('Error with inbox queue.', { err })) + } + + getActivityPubMessagesWaiting () { + return this.inboxQueue.size + this.inboxQueue.pending + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} + +// --------------------------------------------------------------------------- + +export { + InboxManager +} diff --git a/server/server/lib/activitypub/local-video-viewer.ts b/server/server/lib/activitypub/local-video-viewer.ts new file mode 100644 index 000000000..accd0c894 --- /dev/null +++ b/server/server/lib/activitypub/local-video-viewer.ts @@ -0,0 +1,44 @@ +import { Transaction } from 'sequelize' +import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer.js' +import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section.js' +import { MVideo } from '@server/types/models/index.js' +import { WatchActionObject } from '@peertube/peertube-models' +import { getDurationFromActivityStream } from './activity.js' + +async function createOrUpdateLocalVideoViewer (watchAction: WatchActionObject, video: MVideo, t: Transaction) { + const stats = await LocalVideoViewerModel.loadByUrl(watchAction.id) + if (stats) await stats.destroy({ transaction: t }) + + const localVideoViewer = await LocalVideoViewerModel.create({ + url: watchAction.id, + uuid: watchAction.uuid, + + watchTime: getDurationFromActivityStream(watchAction.duration), + + startDate: new Date(watchAction.startTime), + endDate: new Date(watchAction.endTime), + + country: watchAction.location + ? watchAction.location.addressCountry + : null, + + videoId: video.id + }, { transaction: t }) + + await LocalVideoViewerWatchSectionModel.bulkCreateSections({ + localVideoViewerId: localVideoViewer.id, + + watchSections: watchAction.watchSections.map(s => ({ + start: s.startTimestamp, + end: s.endTimestamp + })), + + transaction: t + }) +} + +// --------------------------------------------------------------------------- + +export { + createOrUpdateLocalVideoViewer +} diff --git a/server/server/lib/activitypub/outbox.ts b/server/server/lib/activitypub/outbox.ts new file mode 100644 index 000000000..8e3cf6815 --- /dev/null +++ b/server/server/lib/activitypub/outbox.ts @@ -0,0 +1,24 @@ +import { logger } from '@server/helpers/logger.js' +import { ActorModel } from '@server/models/actor/actor.js' +import { getServerActor } from '@server/models/application/application.js' +import { JobQueue } from '../job-queue/index.js' + +async function addFetchOutboxJob (actor: Pick) { + // Don't fetch ourselves + const serverActor = await getServerActor() + if (serverActor.id === actor.id) { + logger.error('Cannot fetch our own outbox!') + return undefined + } + + const payload = { + uri: actor.outboxUrl, + type: 'activity' as 'activity' + } + + return JobQueue.Instance.createJobAsync({ type: 'activitypub-http-fetcher', payload }) +} + +export { + addFetchOutboxJob +} diff --git a/server/server/lib/activitypub/playlists/create-update.ts b/server/server/lib/activitypub/playlists/create-update.ts new file mode 100644 index 000000000..05fef5df1 --- /dev/null +++ b/server/server/lib/activitypub/playlists/create-update.ts @@ -0,0 +1,157 @@ +import Bluebird from 'bluebird' +import { isArray } from '@server/helpers/custom-validators/misc.js' +import { retryTransactionWrapper } from '@server/helpers/database-utils.js' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { CRAWL_REQUEST_CONCURRENCY } from '@server/initializers/constants.js' +import { sequelizeTypescript } from '@server/initializers/database.js' +import { updateRemotePlaylistMiniatureFromUrl } from '@server/lib/thumbnail.js' +import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element.js' +import { VideoPlaylistModel } from '@server/models/video/video-playlist.js' +import { FilteredModelAttributes } from '@server/types/index.js' +import { MThumbnail, MVideoPlaylist, MVideoPlaylistFull, MVideoPlaylistVideosLength } from '@server/types/models/index.js' +import { PlaylistObject } from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { getAPId } from '../activity.js' +import { getOrCreateAPActor } from '../actors/index.js' +import { crawlCollectionPage } from '../crawl.js' +import { getOrCreateAPVideo } from '../videos/index.js' +import { + fetchRemotePlaylistElement, + fetchRemoteVideoPlaylist, + playlistElementObjectToDBAttributes, + playlistObjectToDBAttributes +} from './shared/index.js' + +const lTags = loggerTagsFactory('ap', 'video-playlist') + +async function createAccountPlaylists (playlistUrls: string[]) { + await Bluebird.map(playlistUrls, async playlistUrl => { + try { + const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl) + if (exists === true) return + + const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl) + + if (playlistObject === undefined) { + throw new Error(`Cannot refresh remote playlist ${playlistUrl}: invalid body.`) + } + + return createOrUpdateVideoPlaylist(playlistObject) + } catch (err) { + logger.warn('Cannot add playlist element %s.', playlistUrl, { err, ...lTags(playlistUrl) }) + } + }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) +} + +async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, to?: string[]) { + const playlistAttributes = playlistObjectToDBAttributes(playlistObject, to || playlistObject.to) + + await setVideoChannel(playlistObject, playlistAttributes) + + const [ upsertPlaylist ] = await VideoPlaylistModel.upsert(playlistAttributes, { returning: true }) + + const playlistElementUrls = await fetchElementUrls(playlistObject) + + // Refetch playlist from DB since elements fetching could be long in time + const playlist = await VideoPlaylistModel.loadWithAccountAndChannel(upsertPlaylist.id, null) + + await updatePlaylistThumbnail(playlistObject, playlist) + + const elementsLength = await rebuildVideoPlaylistElements(playlistElementUrls, playlist) + playlist.setVideosLength(elementsLength) + + return playlist +} + +// --------------------------------------------------------------------------- + +export { + createAccountPlaylists, + createOrUpdateVideoPlaylist +} + +// --------------------------------------------------------------------------- + +async function setVideoChannel (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly) { + if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) { + throw new Error('Not attributed to for playlist object ' + getAPId(playlistObject)) + } + + const actor = await getOrCreateAPActor(getAPId(playlistObject.attributedTo[0]), 'all') + + if (!actor.VideoChannel) { + logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) }) + return + } + + playlistAttributes.videoChannelId = actor.VideoChannel.id + playlistAttributes.ownerAccountId = actor.VideoChannel.Account.id +} + +async function fetchElementUrls (playlistObject: PlaylistObject) { + let accItems: string[] = [] + await crawlCollectionPage(playlistObject.id, items => { + accItems = accItems.concat(items) + + return Promise.resolve() + }) + + return accItems +} + +async function updatePlaylistThumbnail (playlistObject: PlaylistObject, playlist: MVideoPlaylistFull) { + if (playlistObject.icon) { + let thumbnailModel: MThumbnail + + try { + thumbnailModel = await updateRemotePlaylistMiniatureFromUrl({ downloadUrl: playlistObject.icon.url, playlist }) + await playlist.setAndSaveThumbnail(thumbnailModel, undefined) + } catch (err) { + logger.warn('Cannot set thumbnail of %s.', playlistObject.id, { err, ...lTags(playlistObject.id, playlist.uuid, playlist.url) }) + + if (thumbnailModel) await thumbnailModel.removeThumbnail() + } + + return + } + + // Playlist does not have an icon, destroy existing one + if (playlist.hasThumbnail()) { + await playlist.Thumbnail.destroy() + playlist.Thumbnail = null + } +} + +async function rebuildVideoPlaylistElements (elementUrls: string[], playlist: MVideoPlaylist) { + const elementsToCreate = await buildElementsDBAttributes(elementUrls, playlist) + + await retryTransactionWrapper(() => sequelizeTypescript.transaction(async t => { + await VideoPlaylistElementModel.deleteAllOf(playlist.id, t) + + for (const element of elementsToCreate) { + await VideoPlaylistElementModel.create(element, { transaction: t }) + } + })) + + logger.info('Rebuilt playlist %s with %s elements.', playlist.url, elementsToCreate.length, lTags(playlist.uuid, playlist.url)) + + return elementsToCreate.length +} + +async function buildElementsDBAttributes (elementUrls: string[], playlist: MVideoPlaylist) { + const elementsToCreate: FilteredModelAttributes[] = [] + + await Bluebird.map(elementUrls, async elementUrl => { + try { + const { elementObject } = await fetchRemotePlaylistElement(elementUrl) + + const { video } = await getOrCreateAPVideo({ videoObject: { id: elementObject.url }, fetchType: 'only-video' }) + + elementsToCreate.push(playlistElementObjectToDBAttributes(elementObject, playlist, video)) + } catch (err) { + logger.warn('Cannot add playlist element %s.', elementUrl, { err, ...lTags(playlist.uuid, playlist.url) }) + } + }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) + + return elementsToCreate +} diff --git a/server/server/lib/activitypub/playlists/get.ts b/server/server/lib/activitypub/playlists/get.ts new file mode 100644 index 000000000..645ab5cdb --- /dev/null +++ b/server/server/lib/activitypub/playlists/get.ts @@ -0,0 +1,35 @@ +import { VideoPlaylistModel } from '@server/models/video/video-playlist.js' +import { MVideoPlaylistFullSummary } from '@server/types/models/index.js' +import { APObjectId } from '@peertube/peertube-models' +import { getAPId } from '../activity.js' +import { createOrUpdateVideoPlaylist } from './create-update.js' +import { scheduleRefreshIfNeeded } from './refresh.js' +import { fetchRemoteVideoPlaylist } from './shared/index.js' + +async function getOrCreateAPVideoPlaylist (playlistObjectArg: APObjectId): Promise { + const playlistUrl = getAPId(playlistObjectArg) + + const playlistFromDatabase = await VideoPlaylistModel.loadByUrlWithAccountAndChannelSummary(playlistUrl) + + if (playlistFromDatabase) { + scheduleRefreshIfNeeded(playlistFromDatabase) + + return playlistFromDatabase + } + + const { playlistObject } = await fetchRemoteVideoPlaylist(playlistUrl) + if (!playlistObject) throw new Error('Cannot fetch remote playlist with url: ' + playlistUrl) + + // playlistUrl is just an alias/redirection, so process object id instead + if (playlistObject.id !== playlistUrl) return getOrCreateAPVideoPlaylist(playlistObject) + + const playlistCreated = await createOrUpdateVideoPlaylist(playlistObject) + + return playlistCreated +} + +// --------------------------------------------------------------------------- + +export { + getOrCreateAPVideoPlaylist +} diff --git a/server/server/lib/activitypub/playlists/index.ts b/server/server/lib/activitypub/playlists/index.ts new file mode 100644 index 000000000..5c0864ab4 --- /dev/null +++ b/server/server/lib/activitypub/playlists/index.ts @@ -0,0 +1,3 @@ +export * from './get.js' +export * from './create-update.js' +export * from './refresh.js' diff --git a/server/server/lib/activitypub/playlists/refresh.ts b/server/server/lib/activitypub/playlists/refresh.ts new file mode 100644 index 000000000..033a5b64a --- /dev/null +++ b/server/server/lib/activitypub/playlists/refresh.ts @@ -0,0 +1,53 @@ +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { PeerTubeRequestError } from '@server/helpers/requests.js' +import { JobQueue } from '@server/lib/job-queue/index.js' +import { MVideoPlaylist, MVideoPlaylistOwner } from '@server/types/models/index.js' +import { HttpStatusCode } from '@peertube/peertube-models' +import { createOrUpdateVideoPlaylist } from './create-update.js' +import { fetchRemoteVideoPlaylist } from './shared/index.js' + +function scheduleRefreshIfNeeded (playlist: MVideoPlaylist) { + if (!playlist.isOutdated()) return + + JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'video-playlist', url: playlist.url } }) +} + +async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise { + if (!videoPlaylist.isOutdated()) return videoPlaylist + + const lTags = loggerTagsFactory('ap', 'video-playlist', 'refresh', videoPlaylist.uuid, videoPlaylist.url) + + logger.info('Refreshing playlist %s.', videoPlaylist.url, lTags()) + + try { + const { playlistObject } = await fetchRemoteVideoPlaylist(videoPlaylist.url) + + if (playlistObject === undefined) { + logger.warn('Cannot refresh remote playlist %s: invalid body.', videoPlaylist.url, lTags()) + + await videoPlaylist.setAsRefreshed() + return videoPlaylist + } + + await createOrUpdateVideoPlaylist(playlistObject) + + return videoPlaylist + } catch (err) { + if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { + logger.info('Cannot refresh not existing playlist %s. Deleting it.', videoPlaylist.url, lTags()) + + await videoPlaylist.destroy() + return undefined + } + + logger.warn('Cannot refresh video playlist %s.', videoPlaylist.url, { err, ...lTags() }) + + await videoPlaylist.setAsRefreshed() + return videoPlaylist + } +} + +export { + scheduleRefreshIfNeeded, + refreshVideoPlaylistIfNeeded +} diff --git a/server/server/lib/activitypub/playlists/shared/index.ts b/server/server/lib/activitypub/playlists/shared/index.ts new file mode 100644 index 000000000..af333ce18 --- /dev/null +++ b/server/server/lib/activitypub/playlists/shared/index.ts @@ -0,0 +1,2 @@ +export * from './object-to-model-attributes.js' +export * from './url-to-object.js' diff --git a/server/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts b/server/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts new file mode 100644 index 000000000..1964e04df --- /dev/null +++ b/server/server/lib/activitypub/playlists/shared/object-to-model-attributes.ts @@ -0,0 +1,40 @@ +import { ACTIVITY_PUB } from '@server/initializers/constants.js' +import { VideoPlaylistModel } from '@server/models/video/video-playlist.js' +import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element.js' +import { MVideoId, MVideoPlaylistId } from '@server/types/models/index.js' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { PlaylistElementObject, PlaylistObject, VideoPlaylistPrivacy } from '@peertube/peertube-models' + +function playlistObjectToDBAttributes (playlistObject: PlaylistObject, to: string[]) { + const privacy = to.includes(ACTIVITY_PUB.PUBLIC) + ? VideoPlaylistPrivacy.PUBLIC + : VideoPlaylistPrivacy.UNLISTED + + return { + name: playlistObject.name, + description: playlistObject.content, + privacy, + url: playlistObject.id, + uuid: playlistObject.uuid, + ownerAccountId: null, + videoChannelId: null, + createdAt: new Date(playlistObject.published), + updatedAt: new Date(playlistObject.updated) + } as AttributesOnly +} + +function playlistElementObjectToDBAttributes (elementObject: PlaylistElementObject, videoPlaylist: MVideoPlaylistId, video: MVideoId) { + return { + position: elementObject.position, + url: elementObject.id, + startTimestamp: elementObject.startTimestamp || null, + stopTimestamp: elementObject.stopTimestamp || null, + videoPlaylistId: videoPlaylist.id, + videoId: video.id + } as AttributesOnly +} + +export { + playlistObjectToDBAttributes, + playlistElementObjectToDBAttributes +} diff --git a/server/server/lib/activitypub/playlists/shared/url-to-object.ts b/server/server/lib/activitypub/playlists/shared/url-to-object.ts new file mode 100644 index 000000000..3e6adbcb0 --- /dev/null +++ b/server/server/lib/activitypub/playlists/shared/url-to-object.ts @@ -0,0 +1,47 @@ +import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '@server/helpers/custom-validators/activitypub/playlist.js' +import { isArray } from '@server/helpers/custom-validators/misc.js' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { PlaylistElementObject, PlaylistObject } from '@peertube/peertube-models' +import { fetchAP } from '../../activity.js' +import { checkUrlsSameHost } from '../../url.js' + +async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> { + const lTags = loggerTagsFactory('ap', 'video-playlist', playlistUrl) + + logger.info('Fetching remote playlist %s.', playlistUrl, lTags()) + + const { body, statusCode } = await fetchAP(playlistUrl) + + if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) { + logger.debug('Remote video playlist JSON is not valid.', { body, ...lTags() }) + return { statusCode, playlistObject: undefined } + } + + if (!isArray(body.to)) { + logger.debug('Remote video playlist JSON does not have a valid audience.', { body, ...lTags() }) + return { statusCode, playlistObject: undefined } + } + + return { statusCode, playlistObject: body } +} + +async function fetchRemotePlaylistElement (elementUrl: string): Promise<{ statusCode: number, elementObject: PlaylistElementObject }> { + const lTags = loggerTagsFactory('ap', 'video-playlist', 'element', elementUrl) + + logger.debug('Fetching remote playlist element %s.', elementUrl, lTags()) + + const { body, statusCode } = await fetchAP(elementUrl) + + if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in fetch playlist element ${elementUrl}`) + + if (checkUrlsSameHost(body.id, elementUrl) !== true) { + throw new Error(`Playlist element url ${elementUrl} host is different from the AP object id ${body.id}`) + } + + return { statusCode, elementObject: body } +} + +export { + fetchRemoteVideoPlaylist, + fetchRemotePlaylistElement +} diff --git a/server/server/lib/activitypub/process/index.ts b/server/server/lib/activitypub/process/index.ts new file mode 100644 index 000000000..47160b9cb --- /dev/null +++ b/server/server/lib/activitypub/process/index.ts @@ -0,0 +1 @@ +export * from './process.js' diff --git a/server/server/lib/activitypub/process/process-accept.ts b/server/server/lib/activitypub/process/process-accept.ts new file mode 100644 index 000000000..11092caaa --- /dev/null +++ b/server/server/lib/activitypub/process/process-accept.ts @@ -0,0 +1,32 @@ +import { ActivityAccept } from '@peertube/peertube-models' +import { ActorFollowModel } from '../../../models/actor/actor-follow.js' +import { APProcessorOptions } from '../../../types/activitypub-processor.model.js' +import { MActorDefault, MActorSignature } from '../../../types/models/index.js' +import { addFetchOutboxJob } from '../outbox.js' + +async function processAcceptActivity (options: APProcessorOptions) { + const { byActor: targetActor, inboxActor } = options + if (inboxActor === undefined) throw new Error('Need to accept on explicit inbox.') + + return processAccept(inboxActor, targetActor) +} + +// --------------------------------------------------------------------------- + +export { + processAcceptActivity +} + +// --------------------------------------------------------------------------- + +async function processAccept (actor: MActorDefault, targetActor: MActorSignature) { + const follow = await ActorFollowModel.loadByActorAndTarget(actor.id, targetActor.id) + if (!follow) throw new Error('Cannot find associated follow.') + + if (follow.state !== 'accepted') { + follow.state = 'accepted' + await follow.save() + + await addFetchOutboxJob(targetActor) + } +} diff --git a/server/server/lib/activitypub/process/process-announce.ts b/server/server/lib/activitypub/process/process-announce.ts new file mode 100644 index 000000000..ed178f73c --- /dev/null +++ b/server/server/lib/activitypub/process/process-announce.ts @@ -0,0 +1,75 @@ +import { ActivityAnnounce } from '@peertube/peertube-models' +import { getAPId } from '@server/lib/activitypub/activity.js' +import { retryTransactionWrapper } from '../../../helpers/database-utils.js' +import { logger } from '../../../helpers/logger.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { VideoShareModel } from '../../../models/video/video-share.js' +import { APProcessorOptions } from '../../../types/activitypub-processor.model.js' +import { MActorSignature, MVideoAccountLightBlacklistAllFiles } from '../../../types/models/index.js' +import { Notifier } from '../../notifier/index.js' +import { forwardVideoRelatedActivity } from '../send/shared/send-utils.js' +import { getOrCreateAPVideo } from '../videos/index.js' + +async function processAnnounceActivity (options: APProcessorOptions) { + const { activity, byActor: actorAnnouncer } = options + // Only notify if it is not from a fetcher job + const notify = options.fromFetch !== true + + // Announces on accounts are not supported + if (actorAnnouncer.type !== 'Application' && actorAnnouncer.type !== 'Group') return + + return retryTransactionWrapper(processVideoShare, actorAnnouncer, activity, notify) +} + +// --------------------------------------------------------------------------- + +export { + processAnnounceActivity +} + +// --------------------------------------------------------------------------- + +async function processVideoShare (actorAnnouncer: MActorSignature, activity: ActivityAnnounce, notify: boolean) { + const objectUri = getAPId(activity.object) + + let video: MVideoAccountLightBlacklistAllFiles + let videoCreated: boolean + + try { + const result = await getOrCreateAPVideo({ videoObject: objectUri }) + video = result.video + videoCreated = result.created + } catch (err) { + logger.debug('Cannot process share of %s. Maybe this is not a video object, so just skipping.', objectUri, { err }) + return + } + + await sequelizeTypescript.transaction(async t => { + // Add share entry + + const share = { + actorId: actorAnnouncer.id, + videoId: video.id, + url: activity.id + } + + const [ , created ] = await VideoShareModel.findOrCreate({ + where: { + url: activity.id + }, + defaults: share, + transaction: t + }) + + if (video.isOwned() && created === true) { + // Don't resend the activity to the sender + const exceptions = [ actorAnnouncer ] + + await forwardVideoRelatedActivity(activity, t, exceptions, video) + } + + return undefined + }) + + if (videoCreated && notify) Notifier.Instance.notifyOnNewVideoIfNeeded(video) +} diff --git a/server/server/lib/activitypub/process/process-create.ts b/server/server/lib/activitypub/process/process-create.ts new file mode 100644 index 000000000..7366fe935 --- /dev/null +++ b/server/server/lib/activitypub/process/process-create.ts @@ -0,0 +1,170 @@ +import { isBlockedByServerOrAccount } from '@server/lib/blocklist.js' +import { isRedundancyAccepted } from '@server/lib/redundancy.js' +import { VideoModel } from '@server/models/video/video.js' +import { + AbuseObject, + ActivityCreate, + ActivityCreateObject, + ActivityObject, + CacheFileObject, + PlaylistObject, + VideoCommentObject, + VideoObject, + WatchActionObject +} from '@peertube/peertube-models' +import { retryTransactionWrapper } from '../../../helpers/database-utils.js' +import { logger } from '../../../helpers/logger.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { APProcessorOptions } from '../../../types/activitypub-processor.model.js' +import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../types/models/index.js' +import { Notifier } from '../../notifier/index.js' +import { fetchAPObjectIfNeeded } from '../activity.js' +import { createOrUpdateCacheFile } from '../cache-file.js' +import { createOrUpdateLocalVideoViewer } from '../local-video-viewer.js' +import { createOrUpdateVideoPlaylist } from '../playlists/index.js' +import { forwardVideoRelatedActivity } from '../send/shared/send-utils.js' +import { resolveThread } from '../video-comments.js' +import { getOrCreateAPVideo } from '../videos/index.js' + +async function processCreateActivity (options: APProcessorOptions>) { + const { activity, byActor } = options + + // Only notify if it is not from a fetcher job + const notify = options.fromFetch !== true + const activityObject = await fetchAPObjectIfNeeded>(activity.object) + const activityType = activityObject.type + + if (activityType === 'Video') { + return processCreateVideo(activityObject, notify) + } + + if (activityType === 'Note') { + // Comments will be fetched from videos + if (options.fromFetch) return + + return retryTransactionWrapper(processCreateVideoComment, activity, activityObject, byActor, notify) + } + + if (activityType === 'WatchAction') { + return retryTransactionWrapper(processCreateWatchAction, activityObject) + } + + if (activityType === 'CacheFile') { + return retryTransactionWrapper(processCreateCacheFile, activity, activityObject, byActor) + } + + if (activityType === 'Playlist') { + return retryTransactionWrapper(processCreatePlaylist, activity, activityObject, byActor) + } + + logger.warn('Unknown activity object type %s when creating activity.', activityType, { activity: activity.id }) + return Promise.resolve(undefined) +} + +// --------------------------------------------------------------------------- + +export { + processCreateActivity +} + +// --------------------------------------------------------------------------- + +async function processCreateVideo (videoToCreateData: VideoObject, notify: boolean) { + const syncParam = { rates: false, shares: false, comments: false, refreshVideo: false } + const { video, created } = await getOrCreateAPVideo({ videoObject: videoToCreateData, syncParam }) + + if (created && notify) Notifier.Instance.notifyOnNewVideoIfNeeded(video) + + return video +} + +async function processCreateCacheFile ( + activity: ActivityCreate, + cacheFile: CacheFileObject, + byActor: MActorSignature +) { + if (await isRedundancyAccepted(activity, byActor) !== true) return + + const { video } = await getOrCreateAPVideo({ videoObject: cacheFile.object }) + + await sequelizeTypescript.transaction(async t => { + return createOrUpdateCacheFile(cacheFile, video, byActor, t) + }) + + if (video.isOwned()) { + // Don't resend the activity to the sender + const exceptions = [ byActor ] + await forwardVideoRelatedActivity(activity, undefined, exceptions, video) + } +} + +async function processCreateWatchAction (watchAction: WatchActionObject) { + if (watchAction.actionStatus !== 'CompletedActionStatus') return + + const video = await VideoModel.loadByUrl(watchAction.object) + if (video.remote) return + + await sequelizeTypescript.transaction(async t => { + return createOrUpdateLocalVideoViewer(watchAction, video, t) + }) +} + +async function processCreateVideoComment ( + activity: ActivityCreate, + commentObject: VideoCommentObject, + byActor: MActorSignature, + notify: boolean +) { + const byAccount = byActor.Account + + if (!byAccount) throw new Error('Cannot create video comment with the non account actor ' + byActor.url) + + let video: MVideoAccountLightBlacklistAllFiles + let created: boolean + let comment: MCommentOwnerVideo + + try { + const resolveThreadResult = await resolveThread({ url: commentObject.id, isVideo: false }) + if (!resolveThreadResult) return // Comment not accepted + + video = resolveThreadResult.video + created = resolveThreadResult.commentCreated + comment = resolveThreadResult.comment + } catch (err) { + logger.debug( + 'Cannot process video comment because we could not resolve thread %s. Maybe it was not a video thread, so skip it.', + commentObject.inReplyTo, + { err } + ) + return + } + + // Try to not forward unwanted comments on our videos + if (video.isOwned()) { + if (await isBlockedByServerOrAccount(comment.Account, video.VideoChannel.Account)) { + logger.info('Skip comment forward from blocked account or server %s.', comment.Account.Actor.url) + return + } + + if (created === true) { + // Don't resend the activity to the sender + const exceptions = [ byActor ] + + await forwardVideoRelatedActivity(activity, undefined, exceptions, video) + } + } + + if (created && notify) Notifier.Instance.notifyOnNewComment(comment) +} + +async function processCreatePlaylist ( + activity: ActivityCreate, + playlistObject: PlaylistObject, + byActor: MActorSignature +) { + const byAccount = byActor.Account + + if (!byAccount) throw new Error('Cannot create video playlist with the non account actor ' + byActor.url) + + await createOrUpdateVideoPlaylist(playlistObject, activity.to) +} diff --git a/server/server/lib/activitypub/process/process-delete.ts b/server/server/lib/activitypub/process/process-delete.ts new file mode 100644 index 000000000..932e4cf94 --- /dev/null +++ b/server/server/lib/activitypub/process/process-delete.ts @@ -0,0 +1,153 @@ +import { ActivityDelete } from '@peertube/peertube-models' +import { retryTransactionWrapper } from '../../../helpers/database-utils.js' +import { logger } from '../../../helpers/logger.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { ActorModel } from '../../../models/actor/actor.js' +import { VideoCommentModel } from '../../../models/video/video-comment.js' +import { VideoPlaylistModel } from '../../../models/video/video-playlist.js' +import { VideoModel } from '../../../models/video/video.js' +import { APProcessorOptions } from '../../../types/activitypub-processor.model.js' +import { + MAccountActor, + MActor, + MActorFull, + MActorSignature, + MChannelAccountActor, + MChannelActor, + MCommentOwnerVideo +} from '../../../types/models/index.js' +import { forwardVideoRelatedActivity } from '../send/shared/send-utils.js' + +async function processDeleteActivity (options: APProcessorOptions) { + const { activity, byActor } = options + + const objectUrl = typeof activity.object === 'string' ? activity.object : activity.object.id + + if (activity.actor === objectUrl) { + // We need more attributes (all the account and channel) + const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url) + + if (byActorFull.type === 'Person') { + if (!byActorFull.Account) throw new Error('Actor ' + byActorFull.url + ' is a person but we cannot find it in database.') + + const accountToDelete = byActorFull.Account as MAccountActor + accountToDelete.Actor = byActorFull + + return retryTransactionWrapper(processDeleteAccount, accountToDelete) + } else if (byActorFull.type === 'Group') { + if (!byActorFull.VideoChannel) throw new Error('Actor ' + byActorFull.url + ' is a group but we cannot find it in database.') + + const channelToDelete = byActorFull.VideoChannel as MChannelAccountActor & { Actor: MActorFull } + channelToDelete.Actor = byActorFull + return retryTransactionWrapper(processDeleteVideoChannel, channelToDelete) + } + } + + { + const videoCommentInstance = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(objectUrl) + if (videoCommentInstance) { + return retryTransactionWrapper(processDeleteVideoComment, byActor, videoCommentInstance, activity) + } + } + + { + const videoInstance = await VideoModel.loadByUrlAndPopulateAccount(objectUrl) + if (videoInstance) { + if (videoInstance.isOwned()) throw new Error(`Remote instance cannot delete owned video ${videoInstance.url}.`) + + return retryTransactionWrapper(processDeleteVideo, byActor, videoInstance) + } + } + + { + const videoPlaylist = await VideoPlaylistModel.loadByUrlAndPopulateAccount(objectUrl) + if (videoPlaylist) { + if (videoPlaylist.isOwned()) throw new Error(`Remote instance cannot delete owned playlist ${videoPlaylist.url}.`) + + return retryTransactionWrapper(processDeleteVideoPlaylist, byActor, videoPlaylist) + } + } + + return undefined +} + +// --------------------------------------------------------------------------- + +export { + processDeleteActivity +} + +// --------------------------------------------------------------------------- + +async function processDeleteVideo (actor: MActor, videoToDelete: VideoModel) { + logger.debug('Removing remote video "%s".', videoToDelete.uuid) + + await sequelizeTypescript.transaction(async t => { + if (videoToDelete.VideoChannel.Account.Actor.id !== actor.id) { + throw new Error('Account ' + actor.url + ' does not own video channel ' + videoToDelete.VideoChannel.Actor.url) + } + + await videoToDelete.destroy({ transaction: t }) + }) + + logger.info('Remote video with uuid %s removed.', videoToDelete.uuid) +} + +async function processDeleteVideoPlaylist (actor: MActor, playlistToDelete: VideoPlaylistModel) { + logger.debug('Removing remote video playlist "%s".', playlistToDelete.uuid) + + await sequelizeTypescript.transaction(async t => { + if (playlistToDelete.OwnerAccount.Actor.id !== actor.id) { + throw new Error('Account ' + actor.url + ' does not own video playlist ' + playlistToDelete.url) + } + + await playlistToDelete.destroy({ transaction: t }) + }) + + logger.info('Remote video playlist with uuid %s removed.', playlistToDelete.uuid) +} + +async function processDeleteAccount (accountToRemove: MAccountActor) { + logger.debug('Removing remote account "%s".', accountToRemove.Actor.url) + + await sequelizeTypescript.transaction(async t => { + await accountToRemove.destroy({ transaction: t }) + }) + + logger.info('Remote account %s removed.', accountToRemove.Actor.url) +} + +async function processDeleteVideoChannel (videoChannelToRemove: MChannelActor) { + logger.debug('Removing remote video channel "%s".', videoChannelToRemove.Actor.url) + + await sequelizeTypescript.transaction(async t => { + await videoChannelToRemove.destroy({ transaction: t }) + }) + + logger.info('Remote video channel %s removed.', videoChannelToRemove.Actor.url) +} + +function processDeleteVideoComment (byActor: MActorSignature, videoComment: MCommentOwnerVideo, activity: ActivityDelete) { + // Already deleted + if (videoComment.isDeleted()) return Promise.resolve() + + logger.debug('Removing remote video comment "%s".', videoComment.url) + + return sequelizeTypescript.transaction(async t => { + if (byActor.Account.id !== videoComment.Account.id && byActor.Account.id !== videoComment.Video.VideoChannel.accountId) { + throw new Error(`Account ${byActor.url} does not own video comment ${videoComment.url} or video ${videoComment.Video.url}`) + } + + videoComment.markAsDeleted() + + await videoComment.save({ transaction: t }) + + if (videoComment.Video.isOwned()) { + // Don't resend the activity to the sender + const exceptions = [ byActor ] + await forwardVideoRelatedActivity(activity, t, exceptions, videoComment.Video) + } + + logger.info('Remote video comment %s removed.', videoComment.url) + }) +} diff --git a/server/server/lib/activitypub/process/process-dislike.ts b/server/server/lib/activitypub/process/process-dislike.ts new file mode 100644 index 000000000..8040cad93 --- /dev/null +++ b/server/server/lib/activitypub/process/process-dislike.ts @@ -0,0 +1,58 @@ +import { VideoModel } from '@server/models/video/video.js' +import { ActivityDislike } from '@peertube/peertube-models' +import { retryTransactionWrapper } from '../../../helpers/database-utils.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { AccountVideoRateModel } from '../../../models/account/account-video-rate.js' +import { APProcessorOptions } from '../../../types/activitypub-processor.model.js' +import { MActorSignature } from '../../../types/models/index.js' +import { federateVideoIfNeeded, getOrCreateAPVideo } from '../videos/index.js' + +async function processDislikeActivity (options: APProcessorOptions) { + const { activity, byActor } = options + return retryTransactionWrapper(processDislike, activity, byActor) +} + +// --------------------------------------------------------------------------- + +export { + processDislikeActivity +} + +// --------------------------------------------------------------------------- + +async function processDislike (activity: ActivityDislike, byActor: MActorSignature) { + const dislikeObject = activity.object + const byAccount = byActor.Account + + if (!byAccount) throw new Error('Cannot create dislike with the non account actor ' + byActor.url) + + const { video: onlyVideo } = await getOrCreateAPVideo({ videoObject: dislikeObject, fetchType: 'only-video' }) + + // We don't care about dislikes of remote videos + if (!onlyVideo.isOwned()) return + + return sequelizeTypescript.transaction(async t => { + const video = await VideoModel.loadFull(onlyVideo.id, t) + + const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) + if (existingRate && existingRate.type === 'dislike') return + + await video.increment('dislikes', { transaction: t }) + video.dislikes++ + + if (existingRate && existingRate.type === 'like') { + await video.decrement('likes', { transaction: t }) + video.likes-- + } + + const rate = existingRate || new AccountVideoRateModel() + rate.type = 'dislike' + rate.videoId = video.id + rate.accountId = byAccount.id + rate.url = activity.id + + await rate.save({ transaction: t }) + + await federateVideoIfNeeded(video, false, t) + }) +} diff --git a/server/server/lib/activitypub/process/process-flag.ts b/server/server/lib/activitypub/process/process-flag.ts new file mode 100644 index 000000000..348eaebc2 --- /dev/null +++ b/server/server/lib/activitypub/process/process-flag.ts @@ -0,0 +1,103 @@ +import { createAccountAbuse, createVideoAbuse, createVideoCommentAbuse } from '@server/lib/moderation.js' +import { AccountModel } from '@server/models/account/account.js' +import { VideoCommentModel } from '@server/models/video/video-comment.js' +import { VideoModel } from '@server/models/video/video.js' +import { abusePredefinedReasonsMap } from '@peertube/peertube-core-utils' +import { AbuseState, ActivityFlag } from '@peertube/peertube-models' +import { retryTransactionWrapper } from '../../../helpers/database-utils.js' +import { logger } from '../../../helpers/logger.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { getAPId } from '../../../lib/activitypub/activity.js' +import { APProcessorOptions } from '../../../types/activitypub-processor.model.js' +import { MAccountDefault, MActorSignature, MCommentOwnerVideo } from '../../../types/models/index.js' + +async function processFlagActivity (options: APProcessorOptions) { + const { activity, byActor } = options + + return retryTransactionWrapper(processCreateAbuse, activity, byActor) +} + +// --------------------------------------------------------------------------- + +export { + processFlagActivity +} + +// --------------------------------------------------------------------------- + +async function processCreateAbuse (flag: ActivityFlag, byActor: MActorSignature) { + const account = byActor.Account + if (!account) throw new Error('Cannot create abuse with the non account actor ' + byActor.url) + + const reporterAccount = await AccountModel.load(account.id) + + const objects = Array.isArray(flag.object) ? flag.object : [ flag.object ] + + const tags = Array.isArray(flag.tag) ? flag.tag : [] + const predefinedReasons = tags.map(tag => abusePredefinedReasonsMap[tag.name]) + .filter(v => !isNaN(v)) + + const startAt = flag.startAt + const endAt = flag.endAt + + for (const object of objects) { + try { + const uri = getAPId(object) + + logger.debug('Reporting remote abuse for object %s.', uri) + + await sequelizeTypescript.transaction(async t => { + const video = await VideoModel.loadByUrlAndPopulateAccount(uri, t) + let videoComment: MCommentOwnerVideo + let flaggedAccount: MAccountDefault + + if (!video) videoComment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(uri, t) + if (!videoComment) flaggedAccount = await AccountModel.loadByUrl(uri, t) + + if (!video && !videoComment && !flaggedAccount) { + logger.warn('Cannot flag unknown entity %s.', object) + return + } + + const baseAbuse = { + reporterAccountId: reporterAccount.id, + reason: flag.content, + state: AbuseState.PENDING, + predefinedReasons + } + + if (video) { + return createVideoAbuse({ + baseAbuse, + startAt, + endAt, + reporterAccount, + transaction: t, + videoInstance: video, + skipNotification: false + }) + } + + if (videoComment) { + return createVideoCommentAbuse({ + baseAbuse, + reporterAccount, + transaction: t, + commentInstance: videoComment, + skipNotification: false + }) + } + + return await createAccountAbuse({ + baseAbuse, + reporterAccount, + transaction: t, + accountInstance: flaggedAccount, + skipNotification: false + }) + }) + } catch (err) { + logger.debug('Cannot process report of %s', getAPId(object), { err }) + } + } +} diff --git a/server/server/lib/activitypub/process/process-follow.ts b/server/server/lib/activitypub/process/process-follow.ts new file mode 100644 index 000000000..172308048 --- /dev/null +++ b/server/server/lib/activitypub/process/process-follow.ts @@ -0,0 +1,156 @@ +import { Transaction } from 'sequelize' +import { ActivityFollow } from '@peertube/peertube-models' +import { isBlockedByServerOrAccount } from '@server/lib/blocklist.js' +import { AccountModel } from '@server/models/account/account.js' +import { getServerActor } from '@server/models/application/application.js' +import { retryTransactionWrapper } from '../../../helpers/database-utils.js' +import { logger } from '../../../helpers/logger.js' +import { CONFIG } from '../../../initializers/config.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { getAPId } from '../../../lib/activitypub/activity.js' +import { ActorFollowModel } from '../../../models/actor/actor-follow.js' +import { ActorModel } from '../../../models/actor/actor.js' +import { APProcessorOptions } from '../../../types/activitypub-processor.model.js' +import { MActorFollow, MActorFull, MActorId, MActorSignature } from '../../../types/models/index.js' +import { Notifier } from '../../notifier/index.js' +import { autoFollowBackIfNeeded } from '../follow.js' +import { sendAccept, sendReject } from '../send/index.js' + +async function processFollowActivity (options: APProcessorOptions) { + const { activity, byActor } = options + + const activityId = activity.id + const objectId = getAPId(activity.object) + + return retryTransactionWrapper(processFollow, byActor, activityId, objectId) +} + +// --------------------------------------------------------------------------- + +export { + processFollowActivity +} + +// --------------------------------------------------------------------------- + +async function processFollow (byActor: MActorSignature, activityId: string, targetActorURL: string) { + const { actorFollow, created, targetActor } = await sequelizeTypescript.transaction(async t => { + const targetActor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(targetActorURL, t) + + if (!targetActor) throw new Error('Unknown actor') + if (targetActor.isOwned() === false) throw new Error('This is not a local actor.') + + if (await rejectIfInstanceFollowDisabled(byActor, activityId, targetActor)) return { actorFollow: undefined } + if (await rejectIfMuted(byActor, activityId, targetActor)) return { actorFollow: undefined } + + const [ actorFollow, created ] = await ActorFollowModel.findOrCreateCustom({ + byActor, + targetActor, + activityId, + state: await isFollowingInstance(targetActor) && CONFIG.FOLLOWERS.INSTANCE.MANUAL_APPROVAL + ? 'pending' + : 'accepted', + transaction: t + }) + + if (rejectIfAlreadyRejected(actorFollow, byActor, activityId, targetActor)) return { actorFollow: undefined } + + await acceptIfNeeded(actorFollow, targetActor, t) + + await fixFollowURLIfNeeded(actorFollow, activityId, t) + + actorFollow.ActorFollower = byActor + actorFollow.ActorFollowing = targetActor + + // Target sends to actor he accepted the follow request + if (actorFollow.state === 'accepted') { + sendAccept(actorFollow) + + await autoFollowBackIfNeeded(actorFollow, t) + } + + return { actorFollow, created, targetActor } + }) + + // Rejected + if (!actorFollow) return + + if (created) { + const follower = await ActorModel.loadFull(byActor.id) + const actorFollowFull = Object.assign(actorFollow, { ActorFollowing: targetActor, ActorFollower: follower }) + + if (await isFollowingInstance(targetActor)) { + Notifier.Instance.notifyOfNewInstanceFollow(actorFollowFull) + } else { + Notifier.Instance.notifyOfNewUserFollow(actorFollowFull) + } + } + + logger.info('Actor %s is followed by actor %s.', targetActorURL, byActor.url) +} + +async function rejectIfInstanceFollowDisabled (byActor: MActorSignature, activityId: string, targetActor: MActorFull) { + if (await isFollowingInstance(targetActor) && CONFIG.FOLLOWERS.INSTANCE.ENABLED === false) { + logger.info('Rejecting %s because instance followers are disabled.', targetActor.url) + + sendReject(activityId, byActor, targetActor) + + return true + } + + return false +} + +async function rejectIfMuted (byActor: MActorSignature, activityId: string, targetActor: MActorFull) { + const followerAccount = await AccountModel.load(byActor.Account.id) + const followingAccountId = targetActor.Account + + if (followerAccount && await isBlockedByServerOrAccount(followerAccount, followingAccountId)) { + logger.info('Rejecting %s because follower is muted.', byActor.url) + + sendReject(activityId, byActor, targetActor) + + return true + } + + return false +} + +function rejectIfAlreadyRejected (actorFollow: MActorFollow, byActor: MActorSignature, activityId: string, targetActor: MActorFull) { + // Already rejected + if (actorFollow.state === 'rejected') { + logger.info('Rejecting %s because follow is already rejected.', byActor.url) + + sendReject(activityId, byActor, targetActor) + + return true + } + + return false +} + +async function acceptIfNeeded (actorFollow: MActorFollow, targetActor: MActorFull, transaction: Transaction) { + // Set the follow as accepted if the remote actor follows a channel or account + // Or if the instance automatically accepts followers + if (actorFollow.state === 'accepted') return + if (!await isFollowingInstance(targetActor)) return + if (CONFIG.FOLLOWERS.INSTANCE.MANUAL_APPROVAL === true && await isFollowingInstance(targetActor)) return + + actorFollow.state = 'accepted' + + await actorFollow.save({ transaction }) +} + +async function fixFollowURLIfNeeded (actorFollow: MActorFollow, activityId: string, transaction: Transaction) { + // Before PeerTube V3 we did not save the follow ID. Try to fix these old follows + if (!actorFollow.url) { + actorFollow.url = activityId + await actorFollow.save({ transaction }) + } +} + +async function isFollowingInstance (targetActor: MActorId) { + const serverActor = await getServerActor() + + return targetActor.id === serverActor.id +} diff --git a/server/server/lib/activitypub/process/process-like.ts b/server/server/lib/activitypub/process/process-like.ts new file mode 100644 index 000000000..03637f9a7 --- /dev/null +++ b/server/server/lib/activitypub/process/process-like.ts @@ -0,0 +1,60 @@ +import { ActivityLike } from '@peertube/peertube-models' +import { VideoModel } from '@server/models/video/video.js' +import { retryTransactionWrapper } from '../../../helpers/database-utils.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { getAPId } from '../../../lib/activitypub/activity.js' +import { AccountVideoRateModel } from '../../../models/account/account-video-rate.js' +import { APProcessorOptions } from '../../../types/activitypub-processor.model.js' +import { MActorSignature } from '../../../types/models/index.js' +import { federateVideoIfNeeded, getOrCreateAPVideo } from '../videos/index.js' + +async function processLikeActivity (options: APProcessorOptions) { + const { activity, byActor } = options + + return retryTransactionWrapper(processLikeVideo, byActor, activity) +} + +// --------------------------------------------------------------------------- + +export { + processLikeActivity +} + +// --------------------------------------------------------------------------- + +async function processLikeVideo (byActor: MActorSignature, activity: ActivityLike) { + const videoUrl = getAPId(activity.object) + + const byAccount = byActor.Account + if (!byAccount) throw new Error('Cannot create like with the non account actor ' + byActor.url) + + const { video: onlyVideo } = await getOrCreateAPVideo({ videoObject: videoUrl, fetchType: 'only-video' }) + + // We don't care about likes of remote videos + if (!onlyVideo.isOwned()) return + + return sequelizeTypescript.transaction(async t => { + const video = await VideoModel.loadFull(onlyVideo.id, t) + + const existingRate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byAccount.id, video.id, activity.id, t) + if (existingRate && existingRate.type === 'like') return + + if (existingRate && existingRate.type === 'dislike') { + await video.decrement('dislikes', { transaction: t }) + video.dislikes-- + } + + await video.increment('likes', { transaction: t }) + video.likes++ + + const rate = existingRate || new AccountVideoRateModel() + rate.type = 'like' + rate.videoId = video.id + rate.accountId = byAccount.id + rate.url = activity.id + + await rate.save({ transaction: t }) + + await federateVideoIfNeeded(video, false, t) + }) +} diff --git a/server/server/lib/activitypub/process/process-reject.ts b/server/server/lib/activitypub/process/process-reject.ts new file mode 100644 index 000000000..76dc14489 --- /dev/null +++ b/server/server/lib/activitypub/process/process-reject.ts @@ -0,0 +1,33 @@ +import { ActivityReject } from '@peertube/peertube-models' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { ActorFollowModel } from '../../../models/actor/actor-follow.js' +import { APProcessorOptions } from '../../../types/activitypub-processor.model.js' +import { MActor } from '../../../types/models/index.js' + +async function processRejectActivity (options: APProcessorOptions) { + const { byActor: targetActor, inboxActor } = options + if (inboxActor === undefined) throw new Error('Need to reject on explicit inbox.') + + return processReject(inboxActor, targetActor) +} + +// --------------------------------------------------------------------------- + +export { + processRejectActivity +} + +// --------------------------------------------------------------------------- + +async function processReject (follower: MActor, targetActor: MActor) { + return sequelizeTypescript.transaction(async t => { + const actorFollow = await ActorFollowModel.loadByActorAndTarget(follower.id, targetActor.id, t) + + if (!actorFollow) throw new Error(`'Unknown actor follow ${follower.id} -> ${targetActor.id}.`) + + actorFollow.state = 'rejected' + await actorFollow.save({ transaction: t }) + + return undefined + }) +} diff --git a/server/server/lib/activitypub/process/process-undo.ts b/server/server/lib/activitypub/process/process-undo.ts new file mode 100644 index 000000000..20a167f7b --- /dev/null +++ b/server/server/lib/activitypub/process/process-undo.ts @@ -0,0 +1,183 @@ +import { + ActivityAnnounce, + ActivityCreate, + ActivityDislike, + ActivityFollow, + ActivityLike, + ActivityUndo, + ActivityUndoObject, + CacheFileObject +} from '@peertube/peertube-models' +import { VideoModel } from '@server/models/video/video.js' +import { retryTransactionWrapper } from '../../../helpers/database-utils.js' +import { logger } from '../../../helpers/logger.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { AccountVideoRateModel } from '../../../models/account/account-video-rate.js' +import { ActorFollowModel } from '../../../models/actor/actor-follow.js' +import { ActorModel } from '../../../models/actor/actor.js' +import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy.js' +import { VideoShareModel } from '../../../models/video/video-share.js' +import { APProcessorOptions } from '../../../types/activitypub-processor.model.js' +import { MActorSignature } from '../../../types/models/index.js' +import { fetchAPObjectIfNeeded } from '../activity.js' +import { forwardVideoRelatedActivity } from '../send/shared/send-utils.js' +import { federateVideoIfNeeded, getOrCreateAPVideo } from '../videos/index.js' + +async function processUndoActivity (options: APProcessorOptions>) { + const { activity, byActor } = options + const activityToUndo = activity.object + + if (activityToUndo.type === 'Like') { + return retryTransactionWrapper(processUndoLike, byActor, activity) + } + + if (activityToUndo.type === 'Create') { + const objectToUndo = await fetchAPObjectIfNeeded(activityToUndo.object) + + if (objectToUndo.type === 'CacheFile') { + return retryTransactionWrapper(processUndoCacheFile, byActor, activity, objectToUndo) + } + } + + if (activityToUndo.type === 'Dislike') { + return retryTransactionWrapper(processUndoDislike, byActor, activity) + } + + if (activityToUndo.type === 'Follow') { + return retryTransactionWrapper(processUndoFollow, byActor, activityToUndo) + } + + if (activityToUndo.type === 'Announce') { + return retryTransactionWrapper(processUndoAnnounce, byActor, activityToUndo) + } + + logger.warn('Unknown activity object type %s -> %s when undo activity.', activityToUndo.type, { activity: activity.id }) + + return undefined +} + +// --------------------------------------------------------------------------- + +export { + processUndoActivity +} + +// --------------------------------------------------------------------------- + +async function processUndoLike (byActor: MActorSignature, activity: ActivityUndo) { + const likeActivity = activity.object + + const { video: onlyVideo } = await getOrCreateAPVideo({ videoObject: likeActivity.object }) + // We don't care about likes of remote videos + if (!onlyVideo.isOwned()) return + + return sequelizeTypescript.transaction(async t => { + if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) + + const video = await VideoModel.loadFull(onlyVideo.id, t) + const rate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byActor.Account.id, video.id, likeActivity.id, t) + if (!rate || rate.type !== 'like') { + logger.warn('Unknown like by account %d for video %d.', byActor.Account.id, video.id) + return + } + + await rate.destroy({ transaction: t }) + await video.decrement('likes', { transaction: t }) + + video.likes-- + await federateVideoIfNeeded(video, false, t) + }) +} + +async function processUndoDislike (byActor: MActorSignature, activity: ActivityUndo) { + const dislikeActivity = activity.object + + const { video: onlyVideo } = await getOrCreateAPVideo({ videoObject: dislikeActivity.object }) + // We don't care about likes of remote videos + if (!onlyVideo.isOwned()) return + + return sequelizeTypescript.transaction(async t => { + if (!byActor.Account) throw new Error('Unknown account ' + byActor.url) + + const video = await VideoModel.loadFull(onlyVideo.id, t) + const rate = await AccountVideoRateModel.loadByAccountAndVideoOrUrl(byActor.Account.id, video.id, dislikeActivity.id, t) + if (!rate || rate.type !== 'dislike') { + logger.warn(`Unknown dislike by account %d for video %d.`, byActor.Account.id, video.id) + return + } + + await rate.destroy({ transaction: t }) + await video.decrement('dislikes', { transaction: t }) + video.dislikes-- + + await federateVideoIfNeeded(video, false, t) + }) +} + +// --------------------------------------------------------------------------- + +async function processUndoCacheFile ( + byActor: MActorSignature, + activity: ActivityUndo>, + cacheFileObject: CacheFileObject +) { + const { video } = await getOrCreateAPVideo({ videoObject: cacheFileObject.object }) + + return sequelizeTypescript.transaction(async t => { + const cacheFile = await VideoRedundancyModel.loadByUrl(cacheFileObject.id, t) + if (!cacheFile) { + logger.debug('Cannot undo unknown video cache %s.', cacheFileObject.id) + return + } + + if (cacheFile.actorId !== byActor.id) throw new Error('Cannot delete redundancy ' + cacheFile.url + ' of another actor.') + + await cacheFile.destroy({ transaction: t }) + + if (video.isOwned()) { + // Don't resend the activity to the sender + const exceptions = [ byActor ] + + await forwardVideoRelatedActivity(activity, t, exceptions, video) + } + }) +} + +function processUndoAnnounce (byActor: MActorSignature, announceActivity: ActivityAnnounce) { + return sequelizeTypescript.transaction(async t => { + const share = await VideoShareModel.loadByUrl(announceActivity.id, t) + if (!share) { + logger.warn('Unknown video share %d', announceActivity.id) + return + } + + if (share.actorId !== byActor.id) throw new Error(`${share.url} is not shared by ${byActor.url}.`) + + await share.destroy({ transaction: t }) + + if (share.Video.isOwned()) { + // Don't resend the activity to the sender + const exceptions = [ byActor ] + + await forwardVideoRelatedActivity(announceActivity, t, exceptions, share.Video) + } + }) +} + +// --------------------------------------------------------------------------- + +function processUndoFollow (follower: MActorSignature, followActivity: ActivityFollow) { + return sequelizeTypescript.transaction(async t => { + const following = await ActorModel.loadByUrlAndPopulateAccountAndChannel(followActivity.object, t) + const actorFollow = await ActorFollowModel.loadByActorAndTarget(follower.id, following.id, t) + + if (!actorFollow) { + logger.warn('Unknown actor follow %d -> %d.', follower.id, following.id) + return + } + + await actorFollow.destroy({ transaction: t }) + + return undefined + }) +} diff --git a/server/server/lib/activitypub/process/process-update.ts b/server/server/lib/activitypub/process/process-update.ts new file mode 100644 index 000000000..d92b347ab --- /dev/null +++ b/server/server/lib/activitypub/process/process-update.ts @@ -0,0 +1,124 @@ +import { + ActivityPubActor, + ActivityUpdate, + ActivityUpdateObject, + CacheFileObject, + PlaylistObject, + VideoObject +} from '@peertube/peertube-models' +import { isRedundancyAccepted } from '@server/lib/redundancy.js' +import { isCacheFileObjectValid } from '../../../helpers/custom-validators/activitypub/cache-file.js' +import { sanitizeAndCheckVideoTorrentObject } from '../../../helpers/custom-validators/activitypub/videos.js' +import { retryTransactionWrapper } from '../../../helpers/database-utils.js' +import { logger } from '../../../helpers/logger.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { ActorModel } from '../../../models/actor/actor.js' +import { APProcessorOptions } from '../../../types/activitypub-processor.model.js' +import { MActorFull, MActorSignature } from '../../../types/models/index.js' +import { fetchAPObjectIfNeeded } from '../activity.js' +import { APActorUpdater } from '../actors/updater.js' +import { createOrUpdateCacheFile } from '../cache-file.js' +import { createOrUpdateVideoPlaylist } from '../playlists/index.js' +import { forwardVideoRelatedActivity } from '../send/shared/send-utils.js' +import { APVideoUpdater, getOrCreateAPVideo } from '../videos/index.js' + +async function processUpdateActivity (options: APProcessorOptions>) { + const { activity, byActor } = options + + const object = await fetchAPObjectIfNeeded(activity.object) + const objectType = object.type + + if (objectType === 'Video') { + return retryTransactionWrapper(processUpdateVideo, activity) + } + + if (objectType === 'Person' || objectType === 'Application' || objectType === 'Group') { + // We need more attributes + const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url) + return retryTransactionWrapper(processUpdateActor, byActorFull, object) + } + + if (objectType === 'CacheFile') { + // We need more attributes + const byActorFull = await ActorModel.loadByUrlAndPopulateAccountAndChannel(byActor.url) + return retryTransactionWrapper(processUpdateCacheFile, byActorFull, activity, object) + } + + if (objectType === 'Playlist') { + return retryTransactionWrapper(processUpdatePlaylist, byActor, activity, object) + } + + return undefined +} + +// --------------------------------------------------------------------------- + +export { + processUpdateActivity +} + +// --------------------------------------------------------------------------- + +async function processUpdateVideo (activity: ActivityUpdate) { + const videoObject = activity.object as VideoObject + + if (sanitizeAndCheckVideoTorrentObject(videoObject) === false) { + logger.debug('Video sent by update is not valid.', { videoObject }) + return undefined + } + + const { video, created } = await getOrCreateAPVideo({ + videoObject: videoObject.id, + allowRefresh: false, + fetchType: 'all' + }) + // We did not have this video, it has been created so no need to update + if (created) return + + const updater = new APVideoUpdater(videoObject, video) + return updater.update(activity.to) +} + +async function processUpdateCacheFile ( + byActor: MActorSignature, + activity: ActivityUpdate, + cacheFileObject: CacheFileObject +) { + if (await isRedundancyAccepted(activity, byActor) !== true) return + + if (!isCacheFileObjectValid(cacheFileObject)) { + logger.debug('Cache file object sent by update is not valid.', { cacheFileObject }) + return undefined + } + + const { video } = await getOrCreateAPVideo({ videoObject: cacheFileObject.object }) + + await sequelizeTypescript.transaction(async t => { + await createOrUpdateCacheFile(cacheFileObject, video, byActor, t) + }) + + if (video.isOwned()) { + // Don't resend the activity to the sender + const exceptions = [ byActor ] + + await forwardVideoRelatedActivity(activity, undefined, exceptions, video) + } +} + +async function processUpdateActor (actor: MActorFull, actorObject: ActivityPubActor) { + logger.debug('Updating remote account "%s".', actorObject.url) + + const updater = new APActorUpdater(actorObject, actor) + return updater.update() +} + +async function processUpdatePlaylist ( + byActor: MActorSignature, + activity: ActivityUpdate, + playlistObject: PlaylistObject +) { + const byAccount = byActor.Account + if (!byAccount) throw new Error('Cannot update video playlist with the non account actor ' + byActor.url) + + await createOrUpdateVideoPlaylist(playlistObject, activity.to) +} diff --git a/server/server/lib/activitypub/process/process-view.ts b/server/server/lib/activitypub/process/process-view.ts new file mode 100644 index 000000000..c96948e85 --- /dev/null +++ b/server/server/lib/activitypub/process/process-view.ts @@ -0,0 +1,42 @@ +import { VideoViewsManager } from '@server/lib/views/video-views-manager.js' +import { ActivityView } from '@peertube/peertube-models' +import { APProcessorOptions } from '../../../types/activitypub-processor.model.js' +import { MActorSignature } from '../../../types/models/index.js' +import { forwardVideoRelatedActivity } from '../send/shared/send-utils.js' +import { getOrCreateAPVideo } from '../videos/index.js' + +async function processViewActivity (options: APProcessorOptions) { + const { activity, byActor } = options + + return processCreateView(activity, byActor) +} + +// --------------------------------------------------------------------------- + +export { + processViewActivity +} + +// --------------------------------------------------------------------------- + +async function processCreateView (activity: ActivityView, byActor: MActorSignature) { + const videoObject = activity.object + + const { video } = await getOrCreateAPVideo({ + videoObject, + fetchType: 'only-video', + allowRefresh: false + }) + + const viewerExpires = activity.expires + ? new Date(activity.expires) + : undefined + + await VideoViewsManager.Instance.processRemoteView({ video, viewerId: activity.id, viewerExpires }) + + if (video.isOwned()) { + // Forward the view but don't resend the activity to the sender + const exceptions = [ byActor ] + await forwardVideoRelatedActivity(activity, undefined, exceptions, video) + } +} diff --git a/server/server/lib/activitypub/process/process.ts b/server/server/lib/activitypub/process/process.ts new file mode 100644 index 000000000..abc1ebf6f --- /dev/null +++ b/server/server/lib/activitypub/process/process.ts @@ -0,0 +1,92 @@ +import { StatsManager } from '@server/lib/stat-manager.js' +import { Activity, ActivityType } from '@peertube/peertube-models' +import { logger } from '../../../helpers/logger.js' +import { APProcessorOptions } from '../../../types/activitypub-processor.model.js' +import { MActorDefault, MActorSignature } from '../../../types/models/index.js' +import { getAPId } from '../activity.js' +import { getOrCreateAPActor } from '../actors/index.js' +import { checkUrlsSameHost } from '../url.js' +import { processAcceptActivity } from './process-accept.js' +import { processAnnounceActivity } from './process-announce.js' +import { processCreateActivity } from './process-create.js' +import { processDeleteActivity } from './process-delete.js' +import { processDislikeActivity } from './process-dislike.js' +import { processFlagActivity } from './process-flag.js' +import { processFollowActivity } from './process-follow.js' +import { processLikeActivity } from './process-like.js' +import { processRejectActivity } from './process-reject.js' +import { processUndoActivity } from './process-undo.js' +import { processUpdateActivity } from './process-update.js' +import { processViewActivity } from './process-view.js' + +const processActivity: { [ P in ActivityType ]: (options: APProcessorOptions) => Promise } = { + Create: processCreateActivity, + Update: processUpdateActivity, + Delete: processDeleteActivity, + Follow: processFollowActivity, + Accept: processAcceptActivity, + Reject: processRejectActivity, + Announce: processAnnounceActivity, + Undo: processUndoActivity, + Like: processLikeActivity, + Dislike: processDislikeActivity, + Flag: processFlagActivity, + View: processViewActivity +} + +async function processActivities ( + activities: Activity[], + options: { + signatureActor?: MActorSignature + inboxActor?: MActorDefault + outboxUrl?: string + fromFetch?: boolean + } = {} +) { + const { outboxUrl, signatureActor, inboxActor, fromFetch = false } = options + + const actorsCache: { [ url: string ]: MActorSignature } = {} + + for (const activity of activities) { + if (!signatureActor && [ 'Create', 'Announce', 'Like' ].includes(activity.type) === false) { + logger.error('Cannot process activity %s (type: %s) without the actor signature.', activity.id, activity.type) + continue + } + + const actorUrl = getAPId(activity.actor) + + // When we fetch remote data, we don't have signature + if (signatureActor && actorUrl !== signatureActor.url) { + logger.warn('Signature mismatch between %s and %s, skipping.', actorUrl, signatureActor.url) + continue + } + + if (outboxUrl && checkUrlsSameHost(outboxUrl, actorUrl) !== true) { + logger.warn('Host mismatch between outbox URL %s and actor URL %s, skipping.', outboxUrl, actorUrl) + continue + } + + const byActor = signatureActor || actorsCache[actorUrl] || await getOrCreateAPActor(actorUrl) + actorsCache[actorUrl] = byActor + + const activityProcessor = processActivity[activity.type] + if (activityProcessor === undefined) { + logger.warn('Unknown activity type %s.', activity.type, { activityId: activity.id }) + continue + } + + try { + await activityProcessor({ activity, byActor, inboxActor, fromFetch }) + + StatsManager.Instance.addInboxProcessedSuccess(activity.type) + } catch (err) { + logger.warn('Cannot process activity %s.', activity.type, { err }) + + StatsManager.Instance.addInboxProcessedError(activity.type) + } + } +} + +export { + processActivities +} diff --git a/server/server/lib/activitypub/send/http.ts b/server/server/lib/activitypub/send/http.ts new file mode 100644 index 000000000..8d15b970d --- /dev/null +++ b/server/server/lib/activitypub/send/http.ts @@ -0,0 +1,50 @@ +import { ContextType } from '@peertube/peertube-models' +import { signAndContextify } from '@server/helpers/activity-pub-utils.js' +import { HTTP_SIGNATURE } from '@server/initializers/constants.js' +import { ActorModel } from '@server/models/actor/actor.js' +import { getServerActor } from '@server/models/application/application.js' +import { MActor } from '@server/types/models/index.js' +import { getContextFilter } from '../context.js' + +type Payload = { body: T, contextType: ContextType, signatureActorId?: number } + +export async function computeBody ( + payload: Payload +): Promise { + let body = payload.body + + if (payload.signatureActorId) { + const actorSignature = await ActorModel.load(payload.signatureActorId) + if (!actorSignature) throw new Error('Unknown signature actor id.') + + body = await signAndContextify(actorSignature, payload.body, payload.contextType, getContextFilter()) + } + + return body +} + +export async function buildSignedRequestOptions (options: { + signatureActorId?: number + hasPayload: boolean +}) { + let actor: MActor | null + + if (options.signatureActorId) { + actor = await ActorModel.load(options.signatureActorId) + if (!actor) throw new Error('Unknown signature actor id.') + } else { + // We need to sign the request, so use the server + actor = await getServerActor() + } + + const keyId = actor.url + return { + algorithm: HTTP_SIGNATURE.ALGORITHM, + authorizationHeaderName: HTTP_SIGNATURE.HEADER_NAME, + keyId, + key: actor.privateKey, + headers: options.hasPayload + ? HTTP_SIGNATURE.HEADERS_TO_SIGN_WITH_PAYLOAD + : HTTP_SIGNATURE.HEADERS_TO_SIGN_WITHOUT_PAYLOAD + } +} diff --git a/server/server/lib/activitypub/send/index.ts b/server/server/lib/activitypub/send/index.ts new file mode 100644 index 000000000..8711cb17b --- /dev/null +++ b/server/server/lib/activitypub/send/index.ts @@ -0,0 +1,10 @@ +export * from './http.js' +export * from './send-accept.js' +export * from './send-announce.js' +export * from './send-create.js' +export * from './send-delete.js' +export * from './send-follow.js' +export * from './send-like.js' +export * from './send-reject.js' +export * from './send-undo.js' +export * from './send-update.js' diff --git a/server/server/lib/activitypub/send/send-accept.ts b/server/server/lib/activitypub/send/send-accept.ts new file mode 100644 index 000000000..8486564b8 --- /dev/null +++ b/server/server/lib/activitypub/send/send-accept.ts @@ -0,0 +1,47 @@ +import { ActivityAccept, ActivityFollow } from '@peertube/peertube-models' +import { logger } from '../../../helpers/logger.js' +import { MActor, MActorFollowActors } from '../../../types/models/index.js' +import { getLocalActorFollowAcceptActivityPubUrl } from '../url.js' +import { buildFollowActivity } from './send-follow.js' +import { unicastTo } from './shared/send-utils.js' + +function sendAccept (actorFollow: MActorFollowActors) { + const follower = actorFollow.ActorFollower + const me = actorFollow.ActorFollowing + + if (!follower.serverId) { // This should never happen + logger.warn('Do not sending accept to local follower.') + return + } + + logger.info('Creating job to accept follower %s.', follower.url) + + const followData = buildFollowActivity(actorFollow.url, follower, me) + + const url = getLocalActorFollowAcceptActivityPubUrl(actorFollow) + const data = buildAcceptActivity(url, me, followData) + + return unicastTo({ + data, + byActor: me, + toActorUrl: follower.inboxUrl, + contextType: 'Accept' + }) +} + +// --------------------------------------------------------------------------- + +export { + sendAccept +} + +// --------------------------------------------------------------------------- + +function buildAcceptActivity (url: string, byActor: MActor, followActivityData: ActivityFollow): ActivityAccept { + return { + type: 'Accept', + id: url, + actor: byActor.url, + object: followActivityData + } +} diff --git a/server/server/lib/activitypub/send/send-announce.ts b/server/server/lib/activitypub/send/send-announce.ts new file mode 100644 index 000000000..ab7da2c86 --- /dev/null +++ b/server/server/lib/activitypub/send/send-announce.ts @@ -0,0 +1,58 @@ +import { Transaction } from 'sequelize' +import { ActivityAnnounce, ActivityAudience } from '@peertube/peertube-models' +import { logger } from '../../../helpers/logger.js' +import { MActorLight, MVideo } from '../../../types/models/index.js' +import { MVideoShare } from '../../../types/models/video/index.js' +import { audiencify, getAudience } from '../audience.js' +import { getActorsInvolvedInVideo, getAudienceFromFollowersOf } from './shared/index.js' +import { broadcastToFollowers } from './shared/send-utils.js' + +async function buildAnnounceWithVideoAudience ( + byActor: MActorLight, + videoShare: MVideoShare, + video: MVideo, + t: Transaction +) { + const announcedObject = video.url + + const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, t) + const audience = getAudienceFromFollowersOf(actorsInvolvedInVideo) + + const activity = buildAnnounceActivity(videoShare.url, byActor, announcedObject, audience) + + return { activity, actorsInvolvedInVideo } +} + +async function sendVideoAnnounce (byActor: MActorLight, videoShare: MVideoShare, video: MVideo, transaction: Transaction) { + const { activity, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, transaction) + + logger.info('Creating job to send announce %s.', videoShare.url) + + return broadcastToFollowers({ + data: activity, + byActor, + toFollowersOf: actorsInvolvedInVideo, + transaction, + actorsException: [ byActor ], + contextType: 'Announce' + }) +} + +function buildAnnounceActivity (url: string, byActor: MActorLight, object: string, audience?: ActivityAudience): ActivityAnnounce { + if (!audience) audience = getAudience(byActor) + + return audiencify({ + type: 'Announce' as 'Announce', + id: url, + actor: byActor.url, + object + }, audience) +} + +// --------------------------------------------------------------------------- + +export { + sendVideoAnnounce, + buildAnnounceActivity, + buildAnnounceWithVideoAudience +} diff --git a/server/server/lib/activitypub/send/send-create.ts b/server/server/lib/activitypub/send/send-create.ts new file mode 100644 index 000000000..7f9721be3 --- /dev/null +++ b/server/server/lib/activitypub/send/send-create.ts @@ -0,0 +1,226 @@ +import { Transaction } from 'sequelize' +import { getServerActor } from '@server/models/application/application.js' +import { + ActivityAudience, + ActivityCreate, + ActivityCreateObject, + ContextType, + VideoCommentObject, + VideoPlaylistPrivacy, + VideoPrivacy +} from '@peertube/peertube-models' +import { logger, loggerTagsFactory } from '../../../helpers/logger.js' +import { VideoCommentModel } from '../../../models/video/video-comment.js' +import { + MActorLight, + MCommentOwnerVideo, + MLocalVideoViewerWithWatchSections, + MVideoAccountLight, + MVideoAP, + MVideoPlaylistFull, + MVideoRedundancyFileVideo, + MVideoRedundancyStreamingPlaylistVideo +} from '../../../types/models/index.js' +import { audiencify, getAudience } from '../audience.js' +import { + broadcastToActors, + broadcastToFollowers, + getActorsInvolvedInVideo, + getAudienceFromFollowersOf, + getVideoCommentAudience, + sendVideoActivityToOrigin, + sendVideoRelatedActivity, + unicastTo +} from './shared/index.js' + +const lTags = loggerTagsFactory('ap', 'create') + +async function sendCreateVideo (video: MVideoAP, transaction: Transaction) { + if (!video.hasPrivacyForFederation()) return undefined + + logger.info('Creating job to send video creation of %s.', video.url, lTags(video.uuid)) + + const byActor = video.VideoChannel.Account.Actor + const videoObject = await video.toActivityPubObject() + + const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC) + const createActivity = buildCreateActivity(video.url, byActor, videoObject, audience) + + return broadcastToFollowers({ + data: createActivity, + byActor, + toFollowersOf: [ byActor ], + transaction, + contextType: 'Video' + }) +} + +async function sendCreateCacheFile ( + byActor: MActorLight, + video: MVideoAccountLight, + fileRedundancy: MVideoRedundancyStreamingPlaylistVideo | MVideoRedundancyFileVideo +) { + logger.info('Creating job to send file cache of %s.', fileRedundancy.url, lTags(video.uuid)) + + return sendVideoRelatedCreateActivity({ + byActor, + video, + url: fileRedundancy.url, + object: fileRedundancy.toActivityPubObject(), + contextType: 'CacheFile' + }) +} + +async function sendCreateWatchAction (stats: MLocalVideoViewerWithWatchSections, transaction: Transaction) { + logger.info('Creating job to send create watch action %s.', stats.url, lTags(stats.uuid)) + + const byActor = await getServerActor() + + const activityBuilder = (audience: ActivityAudience) => { + return buildCreateActivity(stats.url, byActor, stats.toActivityPubObject(), audience) + } + + return sendVideoActivityToOrigin(activityBuilder, { byActor, video: stats.Video, transaction, contextType: 'WatchAction' }) +} + +async function sendCreateVideoPlaylist (playlist: MVideoPlaylistFull, transaction: Transaction) { + if (playlist.privacy === VideoPlaylistPrivacy.PRIVATE) return undefined + + logger.info('Creating job to send create video playlist of %s.', playlist.url, lTags(playlist.uuid)) + + const byActor = playlist.OwnerAccount.Actor + const audience = getAudience(byActor, playlist.privacy === VideoPlaylistPrivacy.PUBLIC) + + const object = await playlist.toActivityPubObject(null, transaction) + const createActivity = buildCreateActivity(playlist.url, byActor, object, audience) + + const serverActor = await getServerActor() + const toFollowersOf = [ byActor, serverActor ] + + if (playlist.VideoChannel) toFollowersOf.push(playlist.VideoChannel.Actor) + + return broadcastToFollowers({ + data: createActivity, + byActor, + toFollowersOf, + transaction, + contextType: 'Playlist' + }) +} + +async function sendCreateVideoComment (comment: MCommentOwnerVideo, transaction: Transaction) { + logger.info('Creating job to send comment %s.', comment.url) + + const isOrigin = comment.Video.isOwned() + + const byActor = comment.Account.Actor + const threadParentComments = await VideoCommentModel.listThreadParentComments(comment, transaction) + const commentObject = comment.toActivityPubObject(threadParentComments) as VideoCommentObject + + const actorsInvolvedInComment = await getActorsInvolvedInVideo(comment.Video, transaction) + // Add the actor that commented too + actorsInvolvedInComment.push(byActor) + + const parentsCommentActors = threadParentComments.filter(c => !c.isDeleted()) + .map(c => c.Account.Actor) + + let audience: ActivityAudience + if (isOrigin) { + audience = getVideoCommentAudience(comment, threadParentComments, actorsInvolvedInComment, isOrigin) + } else { + audience = getAudienceFromFollowersOf(actorsInvolvedInComment.concat(parentsCommentActors)) + } + + const createActivity = buildCreateActivity(comment.url, byActor, commentObject, audience) + + // This was a reply, send it to the parent actors + const actorsException = [ byActor ] + await broadcastToActors({ + data: createActivity, + byActor, + toActors: parentsCommentActors, + transaction, + actorsException, + contextType: 'Comment' + }) + + // Broadcast to our followers + await broadcastToFollowers({ + data: createActivity, + byActor, + toFollowersOf: [ byActor ], + transaction, + contextType: 'Comment' + }) + + // Send to actors involved in the comment + if (isOrigin) { + return broadcastToFollowers({ + data: createActivity, + byActor, + toFollowersOf: actorsInvolvedInComment, + transaction, + actorsException, + contextType: 'Comment' + }) + } + + // Send to origin + return transaction.afterCommit(() => { + return unicastTo({ + data: createActivity, + byActor, + toActorUrl: comment.Video.VideoChannel.Account.Actor.getSharedInbox(), + contextType: 'Comment' + }) + }) +} + +function buildCreateActivity ( + url: string, + byActor: MActorLight, + object: T, + audience?: ActivityAudience +): ActivityCreate { + if (!audience) audience = getAudience(byActor) + + return audiencify( + { + type: 'Create' as 'Create', + id: url + '/activity', + actor: byActor.url, + object: typeof object === 'string' + ? object + : audiencify(object, audience) + }, + audience + ) +} + +// --------------------------------------------------------------------------- + +export { + sendCreateVideo, + buildCreateActivity, + sendCreateVideoComment, + sendCreateVideoPlaylist, + sendCreateCacheFile, + sendCreateWatchAction +} + +// --------------------------------------------------------------------------- + +async function sendVideoRelatedCreateActivity (options: { + byActor: MActorLight + video: MVideoAccountLight + url: string + object: any + contextType: ContextType + transaction?: Transaction +}) { + const activityBuilder = (audience: ActivityAudience) => { + return buildCreateActivity(options.url, options.byActor, options.object, audience) + } + + return sendVideoRelatedActivity(activityBuilder, options) +} diff --git a/server/server/lib/activitypub/send/send-delete.ts b/server/server/lib/activitypub/send/send-delete.ts new file mode 100644 index 000000000..2f83da15d --- /dev/null +++ b/server/server/lib/activitypub/send/send-delete.ts @@ -0,0 +1,158 @@ +import { Transaction } from 'sequelize' +import { getServerActor } from '@server/models/application/application.js' +import { ActivityAudience, ActivityDelete } from '@peertube/peertube-models' +import { logger } from '../../../helpers/logger.js' +import { ActorModel } from '../../../models/actor/actor.js' +import { VideoCommentModel } from '../../../models/video/video-comment.js' +import { VideoShareModel } from '../../../models/video/video-share.js' +import { MActorUrl } from '../../../types/models/index.js' +import { MCommentOwnerVideo, MVideoAccountLight, MVideoPlaylistFullSummary } from '../../../types/models/video/index.js' +import { audiencify } from '../audience.js' +import { getDeleteActivityPubUrl } from '../url.js' +import { getActorsInvolvedInVideo, getVideoCommentAudience } from './shared/index.js' +import { broadcastToActors, broadcastToFollowers, sendVideoRelatedActivity, unicastTo } from './shared/send-utils.js' + +async function sendDeleteVideo (video: MVideoAccountLight, transaction: Transaction) { + logger.info('Creating job to broadcast delete of video %s.', video.url) + + const byActor = video.VideoChannel.Account.Actor + + const activityBuilder = (audience: ActivityAudience) => { + const url = getDeleteActivityPubUrl(video.url) + + return buildDeleteActivity(url, video.url, byActor, audience) + } + + return sendVideoRelatedActivity(activityBuilder, { byActor, video, contextType: 'Delete', transaction }) +} + +async function sendDeleteActor (byActor: ActorModel, transaction: Transaction) { + logger.info('Creating job to broadcast delete of actor %s.', byActor.url) + + const url = getDeleteActivityPubUrl(byActor.url) + const activity = buildDeleteActivity(url, byActor.url, byActor) + + const actorsInvolved = await VideoShareModel.loadActorsWhoSharedVideosOf(byActor.id, transaction) + + // In case the actor did not have any videos + const serverActor = await getServerActor() + actorsInvolved.push(serverActor) + + actorsInvolved.push(byActor) + + return broadcastToFollowers({ + data: activity, + byActor, + toFollowersOf: actorsInvolved, + contextType: 'Delete', + transaction + }) +} + +async function sendDeleteVideoComment (videoComment: MCommentOwnerVideo, transaction: Transaction) { + logger.info('Creating job to send delete of comment %s.', videoComment.url) + + const isVideoOrigin = videoComment.Video.isOwned() + + const url = getDeleteActivityPubUrl(videoComment.url) + const byActor = videoComment.isOwned() + ? videoComment.Account.Actor + : videoComment.Video.VideoChannel.Account.Actor + + const threadParentComments = await VideoCommentModel.listThreadParentComments(videoComment, transaction) + const threadParentCommentsFiltered = threadParentComments.filter(c => !c.isDeleted()) + + const actorsInvolvedInComment = await getActorsInvolvedInVideo(videoComment.Video, transaction) + actorsInvolvedInComment.push(byActor) // Add the actor that commented the video + + const audience = getVideoCommentAudience(videoComment, threadParentCommentsFiltered, actorsInvolvedInComment, isVideoOrigin) + const activity = buildDeleteActivity(url, videoComment.url, byActor, audience) + + // This was a reply, send it to the parent actors + const actorsException = [ byActor ] + await broadcastToActors({ + data: activity, + byActor, + toActors: threadParentCommentsFiltered.map(c => c.Account.Actor), + transaction, + contextType: 'Delete', + actorsException + }) + + // Broadcast to our followers + await broadcastToFollowers({ + data: activity, + byActor, + toFollowersOf: [ byActor ], + contextType: 'Delete', + transaction + }) + + // Send to actors involved in the comment + if (isVideoOrigin) { + return broadcastToFollowers({ + data: activity, + byActor, + toFollowersOf: actorsInvolvedInComment, + transaction, + contextType: 'Delete', + actorsException + }) + } + + // Send to origin + return transaction.afterCommit(() => { + return unicastTo({ + data: activity, + byActor, + toActorUrl: videoComment.Video.VideoChannel.Account.Actor.getSharedInbox(), + contextType: 'Delete' + }) + }) +} + +async function sendDeleteVideoPlaylist (videoPlaylist: MVideoPlaylistFullSummary, transaction: Transaction) { + logger.info('Creating job to send delete of playlist %s.', videoPlaylist.url) + + const byActor = videoPlaylist.OwnerAccount.Actor + + const url = getDeleteActivityPubUrl(videoPlaylist.url) + const activity = buildDeleteActivity(url, videoPlaylist.url, byActor) + + const serverActor = await getServerActor() + const toFollowersOf = [ byActor, serverActor ] + + if (videoPlaylist.VideoChannel) toFollowersOf.push(videoPlaylist.VideoChannel.Actor) + + return broadcastToFollowers({ + data: activity, + byActor, + toFollowersOf, + contextType: 'Delete', + transaction + }) +} + +// --------------------------------------------------------------------------- + +export { + sendDeleteVideo, + sendDeleteActor, + sendDeleteVideoComment, + sendDeleteVideoPlaylist +} + +// --------------------------------------------------------------------------- + +function buildDeleteActivity (url: string, object: string, byActor: MActorUrl, audience?: ActivityAudience): ActivityDelete { + const activity = { + type: 'Delete' as 'Delete', + id: url, + actor: byActor.url, + object + } + + if (audience) return audiencify(activity, audience) + + return activity +} diff --git a/server/server/lib/activitypub/send/send-dislike.ts b/server/server/lib/activitypub/send/send-dislike.ts new file mode 100644 index 000000000..0c4dcca5f --- /dev/null +++ b/server/server/lib/activitypub/send/send-dislike.ts @@ -0,0 +1,40 @@ +import { Transaction } from 'sequelize' +import { ActivityAudience, ActivityDislike } from '@peertube/peertube-models' +import { logger } from '../../../helpers/logger.js' +import { MActor, MActorAudience, MVideoAccountLight, MVideoUrl } from '../../../types/models/index.js' +import { audiencify, getAudience } from '../audience.js' +import { getVideoDislikeActivityPubUrlByLocalActor } from '../url.js' +import { sendVideoActivityToOrigin } from './shared/send-utils.js' + +function sendDislike (byActor: MActor, video: MVideoAccountLight, transaction: Transaction) { + logger.info('Creating job to dislike %s.', video.url) + + const activityBuilder = (audience: ActivityAudience) => { + const url = getVideoDislikeActivityPubUrlByLocalActor(byActor, video) + + return buildDislikeActivity(url, byActor, video, audience) + } + + return sendVideoActivityToOrigin(activityBuilder, { byActor, video, transaction, contextType: 'Rate' }) +} + +function buildDislikeActivity (url: string, byActor: MActorAudience, video: MVideoUrl, audience?: ActivityAudience): ActivityDislike { + if (!audience) audience = getAudience(byActor) + + return audiencify( + { + id: url, + type: 'Dislike' as 'Dislike', + actor: byActor.url, + object: video.url + }, + audience + ) +} + +// --------------------------------------------------------------------------- + +export { + sendDislike, + buildDislikeActivity +} diff --git a/server/server/lib/activitypub/send/send-flag.ts b/server/server/lib/activitypub/send/send-flag.ts new file mode 100644 index 000000000..bc9ba91e0 --- /dev/null +++ b/server/server/lib/activitypub/send/send-flag.ts @@ -0,0 +1,42 @@ +import { Transaction } from 'sequelize' +import { ActivityAudience, ActivityFlag } from '@peertube/peertube-models' +import { logger } from '../../../helpers/logger.js' +import { MAbuseAP, MAccountLight, MActor } from '../../../types/models/index.js' +import { audiencify, getAudience } from '../audience.js' +import { getLocalAbuseActivityPubUrl } from '../url.js' +import { unicastTo } from './shared/send-utils.js' + +function sendAbuse (byActor: MActor, abuse: MAbuseAP, flaggedAccount: MAccountLight, t: Transaction) { + if (!flaggedAccount.Actor.serverId) return // Local user + + const url = getLocalAbuseActivityPubUrl(abuse) + + logger.info('Creating job to send abuse %s.', url) + + // Custom audience, we only send the abuse to the origin instance + const audience = { to: [ flaggedAccount.Actor.url ], cc: [] } + const flagActivity = buildFlagActivity(url, byActor, abuse, audience) + + return t.afterCommit(() => { + return unicastTo({ + data: flagActivity, + byActor, + toActorUrl: flaggedAccount.Actor.getSharedInbox(), + contextType: 'Flag' + }) + }) +} + +function buildFlagActivity (url: string, byActor: MActor, abuse: MAbuseAP, audience: ActivityAudience): ActivityFlag { + if (!audience) audience = getAudience(byActor) + + const activity = { id: url, actor: byActor.url, ...abuse.toActivityPubObject() } + + return audiencify(activity, audience) +} + +// --------------------------------------------------------------------------- + +export { + sendAbuse +} diff --git a/server/server/lib/activitypub/send/send-follow.ts b/server/server/lib/activitypub/send/send-follow.ts new file mode 100644 index 000000000..26ea030ce --- /dev/null +++ b/server/server/lib/activitypub/send/send-follow.ts @@ -0,0 +1,37 @@ +import { Transaction } from 'sequelize' +import { ActivityFollow } from '@peertube/peertube-models' +import { logger } from '../../../helpers/logger.js' +import { MActor, MActorFollowActors } from '../../../types/models/index.js' +import { unicastTo } from './shared/send-utils.js' + +function sendFollow (actorFollow: MActorFollowActors, t: Transaction) { + const me = actorFollow.ActorFollower + const following = actorFollow.ActorFollowing + + // Same server as ours + if (!following.serverId) return + + logger.info('Creating job to send follow request to %s.', following.url) + + const data = buildFollowActivity(actorFollow.url, me, following) + + return t.afterCommit(() => { + return unicastTo({ data, byActor: me, toActorUrl: following.inboxUrl, contextType: 'Follow' }) + }) +} + +function buildFollowActivity (url: string, byActor: MActor, targetActor: MActor): ActivityFollow { + return { + type: 'Follow', + id: url, + actor: byActor.url, + object: targetActor.url + } +} + +// --------------------------------------------------------------------------- + +export { + sendFollow, + buildFollowActivity +} diff --git a/server/server/lib/activitypub/send/send-like.ts b/server/server/lib/activitypub/send/send-like.ts new file mode 100644 index 000000000..701c9325d --- /dev/null +++ b/server/server/lib/activitypub/send/send-like.ts @@ -0,0 +1,40 @@ +import { Transaction } from 'sequelize' +import { ActivityAudience, ActivityLike } from '@peertube/peertube-models' +import { logger } from '../../../helpers/logger.js' +import { MActor, MActorAudience, MVideoAccountLight, MVideoUrl } from '../../../types/models/index.js' +import { audiencify, getAudience } from '../audience.js' +import { getVideoLikeActivityPubUrlByLocalActor } from '../url.js' +import { sendVideoActivityToOrigin } from './shared/send-utils.js' + +function sendLike (byActor: MActor, video: MVideoAccountLight, transaction: Transaction) { + logger.info('Creating job to like %s.', video.url) + + const activityBuilder = (audience: ActivityAudience) => { + const url = getVideoLikeActivityPubUrlByLocalActor(byActor, video) + + return buildLikeActivity(url, byActor, video, audience) + } + + return sendVideoActivityToOrigin(activityBuilder, { byActor, video, transaction, contextType: 'Rate' }) +} + +function buildLikeActivity (url: string, byActor: MActorAudience, video: MVideoUrl, audience?: ActivityAudience): ActivityLike { + if (!audience) audience = getAudience(byActor) + + return audiencify( + { + id: url, + type: 'Like' as 'Like', + actor: byActor.url, + object: video.url + }, + audience + ) +} + +// --------------------------------------------------------------------------- + +export { + sendLike, + buildLikeActivity +} diff --git a/server/server/lib/activitypub/send/send-reject.ts b/server/server/lib/activitypub/send/send-reject.ts new file mode 100644 index 000000000..4076d4d73 --- /dev/null +++ b/server/server/lib/activitypub/send/send-reject.ts @@ -0,0 +1,39 @@ +import { ActivityFollow, ActivityReject } from '@peertube/peertube-models' +import { logger } from '../../../helpers/logger.js' +import { MActor } from '../../../types/models/index.js' +import { getLocalActorFollowRejectActivityPubUrl } from '../url.js' +import { buildFollowActivity } from './send-follow.js' +import { unicastTo } from './shared/send-utils.js' + +function sendReject (followUrl: string, follower: MActor, following: MActor) { + if (!follower.serverId) { // This should never happen + logger.warn('Do not sending reject to local follower.') + return + } + + logger.info('Creating job to reject follower %s.', follower.url) + + const followData = buildFollowActivity(followUrl, follower, following) + + const url = getLocalActorFollowRejectActivityPubUrl() + const data = buildRejectActivity(url, following, followData) + + return unicastTo({ data, byActor: following, toActorUrl: follower.inboxUrl, contextType: 'Reject' }) +} + +// --------------------------------------------------------------------------- + +export { + sendReject +} + +// --------------------------------------------------------------------------- + +function buildRejectActivity (url: string, byActor: MActor, followActivityData: ActivityFollow): ActivityReject { + return { + type: 'Reject', + id: url, + actor: byActor.url, + object: followActivityData + } +} diff --git a/server/server/lib/activitypub/send/send-undo.ts b/server/server/lib/activitypub/send/send-undo.ts new file mode 100644 index 000000000..6db165734 --- /dev/null +++ b/server/server/lib/activitypub/send/send-undo.ts @@ -0,0 +1,172 @@ +import { Transaction } from 'sequelize' +import { ActivityAudience, ActivityDislike, ActivityLike, ActivityUndo, ActivityUndoObject, ContextType } from '@peertube/peertube-models' +import { logger } from '../../../helpers/logger.js' +import { VideoModel } from '../../../models/video/video.js' +import { + MActor, + MActorAudience, + MActorFollowActors, + MActorLight, + MVideo, + MVideoAccountLight, + MVideoRedundancyVideo, + MVideoShare +} from '../../../types/models/index.js' +import { audiencify, getAudience } from '../audience.js' +import { getUndoActivityPubUrl, getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from '../url.js' +import { buildAnnounceWithVideoAudience } from './send-announce.js' +import { buildCreateActivity } from './send-create.js' +import { buildDislikeActivity } from './send-dislike.js' +import { buildFollowActivity } from './send-follow.js' +import { buildLikeActivity } from './send-like.js' +import { broadcastToFollowers, sendVideoActivityToOrigin, sendVideoRelatedActivity, unicastTo } from './shared/send-utils.js' + +function sendUndoFollow (actorFollow: MActorFollowActors, t: Transaction) { + const me = actorFollow.ActorFollower + const following = actorFollow.ActorFollowing + + // Same server as ours + if (!following.serverId) return + + logger.info('Creating job to send an unfollow request to %s.', following.url) + + const undoUrl = getUndoActivityPubUrl(actorFollow.url) + + const followActivity = buildFollowActivity(actorFollow.url, me, following) + const undoActivity = undoActivityData(undoUrl, me, followActivity) + + t.afterCommit(() => { + return unicastTo({ + data: undoActivity, + byActor: me, + toActorUrl: following.inboxUrl, + contextType: 'Follow' + }) + }) +} + +// --------------------------------------------------------------------------- + +async function sendUndoAnnounce (byActor: MActorLight, videoShare: MVideoShare, video: MVideo, transaction: Transaction) { + logger.info('Creating job to undo announce %s.', videoShare.url) + + const undoUrl = getUndoActivityPubUrl(videoShare.url) + + const { activity: announce, actorsInvolvedInVideo } = await buildAnnounceWithVideoAudience(byActor, videoShare, video, transaction) + const undoActivity = undoActivityData(undoUrl, byActor, announce) + + return broadcastToFollowers({ + data: undoActivity, + byActor, + toFollowersOf: actorsInvolvedInVideo, + transaction, + actorsException: [ byActor ], + contextType: 'Announce' + }) +} + +async function sendUndoCacheFile (byActor: MActor, redundancyModel: MVideoRedundancyVideo, transaction: Transaction) { + logger.info('Creating job to undo cache file %s.', redundancyModel.url) + + const associatedVideo = redundancyModel.getVideo() + if (!associatedVideo) { + logger.warn('Cannot send undo activity for redundancy %s: no video files associated.', redundancyModel.url) + return + } + + const video = await VideoModel.loadFull(associatedVideo.id) + const createActivity = buildCreateActivity(redundancyModel.url, byActor, redundancyModel.toActivityPubObject()) + + return sendUndoVideoRelatedActivity({ + byActor, + video, + url: redundancyModel.url, + activity: createActivity, + contextType: 'CacheFile', + transaction + }) +} + +// --------------------------------------------------------------------------- + +async function sendUndoLike (byActor: MActor, video: MVideoAccountLight, t: Transaction) { + logger.info('Creating job to undo a like of video %s.', video.url) + + const likeUrl = getVideoLikeActivityPubUrlByLocalActor(byActor, video) + const likeActivity = buildLikeActivity(likeUrl, byActor, video) + + return sendUndoVideoRateToOriginActivity({ byActor, video, url: likeUrl, activity: likeActivity, transaction: t }) +} + +async function sendUndoDislike (byActor: MActor, video: MVideoAccountLight, t: Transaction) { + logger.info('Creating job to undo a dislike of video %s.', video.url) + + const dislikeUrl = getVideoDislikeActivityPubUrlByLocalActor(byActor, video) + const dislikeActivity = buildDislikeActivity(dislikeUrl, byActor, video) + + return sendUndoVideoRateToOriginActivity({ byActor, video, url: dislikeUrl, activity: dislikeActivity, transaction: t }) +} + +// --------------------------------------------------------------------------- + +export { + sendUndoFollow, + sendUndoLike, + sendUndoDislike, + sendUndoAnnounce, + sendUndoCacheFile +} + +// --------------------------------------------------------------------------- + +function undoActivityData ( + url: string, + byActor: MActorAudience, + object: T, + audience?: ActivityAudience +): ActivityUndo { + if (!audience) audience = getAudience(byActor) + + return audiencify( + { + type: 'Undo' as 'Undo', + id: url, + actor: byActor.url, + object + }, + audience + ) +} + +async function sendUndoVideoRelatedActivity (options: { + byActor: MActor + video: MVideoAccountLight + url: string + activity: ActivityUndoObject + contextType: ContextType + transaction: Transaction +}) { + const activityBuilder = (audience: ActivityAudience) => { + const undoUrl = getUndoActivityPubUrl(options.url) + + return undoActivityData(undoUrl, options.byActor, options.activity, audience) + } + + return sendVideoRelatedActivity(activityBuilder, options) +} + +async function sendUndoVideoRateToOriginActivity (options: { + byActor: MActor + video: MVideoAccountLight + url: string + activity: ActivityLike | ActivityDislike + transaction: Transaction +}) { + const activityBuilder = (audience: ActivityAudience) => { + const undoUrl = getUndoActivityPubUrl(options.url) + + return undoActivityData(undoUrl, options.byActor, options.activity, audience) + } + + return sendVideoActivityToOrigin(activityBuilder, { ...options, contextType: 'Rate' }) +} diff --git a/server/server/lib/activitypub/send/send-update.ts b/server/server/lib/activitypub/send/send-update.ts new file mode 100644 index 000000000..f6b714c16 --- /dev/null +++ b/server/server/lib/activitypub/send/send-update.ts @@ -0,0 +1,157 @@ +import { Transaction } from 'sequelize' +import { getServerActor } from '@server/models/application/application.js' +import { ActivityAudience, ActivityUpdate, ActivityUpdateObject, VideoPlaylistPrivacy, VideoPrivacy } from '@peertube/peertube-models' +import { logger } from '../../../helpers/logger.js' +import { AccountModel } from '../../../models/account/account.js' +import { VideoModel } from '../../../models/video/video.js' +import { VideoShareModel } from '../../../models/video/video-share.js' +import { + MAccountDefault, + MActor, + MActorLight, + MChannelDefault, + MVideoAPLight, + MVideoPlaylistFull, + MVideoRedundancyVideo +} from '../../../types/models/index.js' +import { audiencify, getAudience } from '../audience.js' +import { getUpdateActivityPubUrl } from '../url.js' +import { getActorsInvolvedInVideo } from './shared/index.js' +import { broadcastToFollowers, sendVideoRelatedActivity } from './shared/send-utils.js' + +async function sendUpdateVideo (videoArg: MVideoAPLight, transaction: Transaction, overriddenByActor?: MActor) { + if (!videoArg.hasPrivacyForFederation()) return undefined + + const video = await videoArg.lightAPToFullAP(transaction) + + logger.info('Creating job to update video %s.', video.url) + + const byActor = overriddenByActor || video.VideoChannel.Account.Actor + + const url = getUpdateActivityPubUrl(video.url, video.updatedAt.toISOString()) + + const videoObject = await video.toActivityPubObject() + const audience = getAudience(byActor, video.privacy === VideoPrivacy.PUBLIC) + + const updateActivity = buildUpdateActivity(url, byActor, videoObject, audience) + + const actorsInvolved = await getActorsInvolvedInVideo(video, transaction) + if (overriddenByActor) actorsInvolved.push(overriddenByActor) + + return broadcastToFollowers({ + data: updateActivity, + byActor, + toFollowersOf: actorsInvolved, + contextType: 'Video', + transaction + }) +} + +async function sendUpdateActor (accountOrChannel: MChannelDefault | MAccountDefault, transaction: Transaction) { + const byActor = accountOrChannel.Actor + + logger.info('Creating job to update actor %s.', byActor.url) + + const url = getUpdateActivityPubUrl(byActor.url, byActor.updatedAt.toISOString()) + const accountOrChannelObject = await (accountOrChannel as any).toActivityPubObject() // FIXME: typescript bug? + const audience = getAudience(byActor) + const updateActivity = buildUpdateActivity(url, byActor, accountOrChannelObject, audience) + + let actorsInvolved: MActor[] + if (accountOrChannel instanceof AccountModel) { + // Actors that shared my videos are involved too + actorsInvolved = await VideoShareModel.loadActorsWhoSharedVideosOf(byActor.id, transaction) + } else { + // Actors that shared videos of my channel are involved too + actorsInvolved = await VideoShareModel.loadActorsByVideoChannel(accountOrChannel.id, transaction) + } + + actorsInvolved.push(byActor) + + return broadcastToFollowers({ + data: updateActivity, + byActor, + toFollowersOf: actorsInvolved, + transaction, + contextType: 'Actor' + }) +} + +async function sendUpdateCacheFile (byActor: MActorLight, redundancyModel: MVideoRedundancyVideo) { + logger.info('Creating job to update cache file %s.', redundancyModel.url) + + const associatedVideo = redundancyModel.getVideo() + if (!associatedVideo) { + logger.warn('Cannot send update activity for redundancy %s: no video files associated.', redundancyModel.url) + return + } + + const video = await VideoModel.loadFull(associatedVideo.id) + + const activityBuilder = (audience: ActivityAudience) => { + const redundancyObject = redundancyModel.toActivityPubObject() + const url = getUpdateActivityPubUrl(redundancyModel.url, redundancyModel.updatedAt.toISOString()) + + return buildUpdateActivity(url, byActor, redundancyObject, audience) + } + + return sendVideoRelatedActivity(activityBuilder, { byActor, video, contextType: 'CacheFile' }) +} + +async function sendUpdateVideoPlaylist (videoPlaylist: MVideoPlaylistFull, transaction: Transaction) { + if (videoPlaylist.privacy === VideoPlaylistPrivacy.PRIVATE) return undefined + + const byActor = videoPlaylist.OwnerAccount.Actor + + logger.info('Creating job to update video playlist %s.', videoPlaylist.url) + + const url = getUpdateActivityPubUrl(videoPlaylist.url, videoPlaylist.updatedAt.toISOString()) + + const object = await videoPlaylist.toActivityPubObject(null, transaction) + const audience = getAudience(byActor, videoPlaylist.privacy === VideoPlaylistPrivacy.PUBLIC) + + const updateActivity = buildUpdateActivity(url, byActor, object, audience) + + const serverActor = await getServerActor() + const toFollowersOf = [ byActor, serverActor ] + + if (videoPlaylist.VideoChannel) toFollowersOf.push(videoPlaylist.VideoChannel.Actor) + + return broadcastToFollowers({ + data: updateActivity, + byActor, + toFollowersOf, + transaction, + contextType: 'Playlist' + }) +} + +// --------------------------------------------------------------------------- + +export { + sendUpdateActor, + sendUpdateVideo, + sendUpdateCacheFile, + sendUpdateVideoPlaylist +} + +// --------------------------------------------------------------------------- + +function buildUpdateActivity ( + url: string, + byActor: MActorLight, + object: ActivityUpdateObject, + audience?: ActivityAudience +): ActivityUpdate { + if (!audience) audience = getAudience(byActor) + + return audiencify( + { + type: 'Update' as 'Update', + id: url, + actor: byActor.url, + object: audiencify(object, audience) + }, + audience + ) +} diff --git a/server/server/lib/activitypub/send/send-view.ts b/server/server/lib/activitypub/send/send-view.ts new file mode 100644 index 000000000..10cd9508f --- /dev/null +++ b/server/server/lib/activitypub/send/send-view.ts @@ -0,0 +1,62 @@ +import { Transaction } from 'sequelize' +import { VideoViewsManager } from '@server/lib/views/video-views-manager.js' +import { MActorAudience, MActorLight, MVideoImmutable, MVideoUrl } from '@server/types/models/index.js' +import { ActivityAudience, ActivityView } from '@peertube/peertube-models' +import { logger } from '../../../helpers/logger.js' +import { audiencify, getAudience } from '../audience.js' +import { getLocalVideoViewActivityPubUrl } from '../url.js' +import { sendVideoRelatedActivity } from './shared/send-utils.js' + +type ViewType = 'view' | 'viewer' + +async function sendView (options: { + byActor: MActorLight + type: ViewType + video: MVideoImmutable + viewerIdentifier: string + transaction?: Transaction +}) { + const { byActor, type, video, viewerIdentifier, transaction } = options + + logger.info('Creating job to send %s of %s.', type, video.url) + + const activityBuilder = (audience: ActivityAudience) => { + const url = getLocalVideoViewActivityPubUrl(byActor, video, viewerIdentifier) + + return buildViewActivity({ url, byActor, video, audience, type }) + } + + return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction, contextType: 'View', parallelizable: true }) +} + +// --------------------------------------------------------------------------- + +export { + sendView +} + +// --------------------------------------------------------------------------- + +function buildViewActivity (options: { + url: string + byActor: MActorAudience + video: MVideoUrl + type: ViewType + audience?: ActivityAudience +}): ActivityView { + const { url, byActor, type, video, audience = getAudience(byActor) } = options + + return audiencify( + { + id: url, + type: 'View' as 'View', + actor: byActor.url, + object: video.url, + + expires: type === 'viewer' + ? new Date(VideoViewsManager.Instance.buildViewerExpireTime()).toISOString() + : undefined + }, + audience + ) +} diff --git a/server/server/lib/activitypub/send/shared/audience-utils.ts b/server/server/lib/activitypub/send/shared/audience-utils.ts new file mode 100644 index 000000000..ac88c4567 --- /dev/null +++ b/server/server/lib/activitypub/send/shared/audience-utils.ts @@ -0,0 +1,74 @@ +import { Transaction } from 'sequelize' +import { ACTIVITY_PUB } from '@server/initializers/constants.js' +import { ActorModel } from '@server/models/actor/actor.js' +import { VideoModel } from '@server/models/video/video.js' +import { VideoShareModel } from '@server/models/video/video-share.js' +import { MActorFollowersUrl, MActorUrl, MCommentOwner, MCommentOwnerVideo, MVideoId } from '@server/types/models/index.js' +import { ActivityAudience } from '@peertube/peertube-models' + +function getOriginVideoAudience (accountActor: MActorUrl, actorsInvolvedInVideo: MActorFollowersUrl[] = []): ActivityAudience { + return { + to: [ accountActor.url ], + cc: actorsInvolvedInVideo.map(a => a.followersUrl) + } +} + +function getVideoCommentAudience ( + videoComment: MCommentOwnerVideo, + threadParentComments: MCommentOwner[], + actorsInvolvedInVideo: MActorFollowersUrl[], + isOrigin = false +): ActivityAudience { + const to = [ ACTIVITY_PUB.PUBLIC ] + const cc: string[] = [] + + // Owner of the video we comment + if (isOrigin === false) { + cc.push(videoComment.Video.VideoChannel.Account.Actor.url) + } + + // Followers of the poster + cc.push(videoComment.Account.Actor.followersUrl) + + // Send to actors we reply to + for (const parentComment of threadParentComments) { + if (parentComment.isDeleted()) continue + + cc.push(parentComment.Account.Actor.url) + } + + return { + to, + cc: cc.concat(actorsInvolvedInVideo.map(a => a.followersUrl)) + } +} + +function getAudienceFromFollowersOf (actorsInvolvedInObject: MActorFollowersUrl[]): ActivityAudience { + return { + to: [ ACTIVITY_PUB.PUBLIC ].concat(actorsInvolvedInObject.map(a => a.followersUrl)), + cc: [] + } +} + +async function getActorsInvolvedInVideo (video: MVideoId, t: Transaction) { + const actors = await VideoShareModel.listActorIdsAndFollowerUrlsByShare(video.id, t) + + const videoAll = video as VideoModel + + const videoActor = videoAll.VideoChannel?.Account + ? videoAll.VideoChannel.Account.Actor + : await ActorModel.loadAccountActorFollowerUrlByVideoId(video.id, t) + + actors.push(videoActor) + + return actors +} + +// --------------------------------------------------------------------------- + +export { + getOriginVideoAudience, + getActorsInvolvedInVideo, + getAudienceFromFollowersOf, + getVideoCommentAudience +} diff --git a/server/server/lib/activitypub/send/shared/index.ts b/server/server/lib/activitypub/send/shared/index.ts new file mode 100644 index 000000000..25beeafbf --- /dev/null +++ b/server/server/lib/activitypub/send/shared/index.ts @@ -0,0 +1,2 @@ +export * from './audience-utils.js' +export * from './send-utils.js' diff --git a/server/server/lib/activitypub/send/shared/send-utils.ts b/server/server/lib/activitypub/send/shared/send-utils.ts new file mode 100644 index 000000000..29e22ef58 --- /dev/null +++ b/server/server/lib/activitypub/send/shared/send-utils.ts @@ -0,0 +1,298 @@ +import { Transaction } from 'sequelize' +import { Activity, ActivityAudience, ActivitypubHttpBroadcastPayload, ContextType } from '@peertube/peertube-models' +import { ActorFollowHealthCache } from '@server/lib/actor-follow-health-cache.js' +import { getServerActor } from '@server/models/application/application.js' +import { afterCommitIfTransaction } from '../../../../helpers/database-utils.js' +import { logger } from '../../../../helpers/logger.js' +import { ActorFollowModel } from '../../../../models/actor/actor-follow.js' +import { ActorModel } from '../../../../models/actor/actor.js' +import { + MActor, + MActorId, + MActorLight, + MActorWithInboxes, + MVideoAccountLight, + MVideoId, + MVideoImmutable +} from '../../../../types/models/index.js' +import { JobQueue } from '../../../job-queue/index.js' +import { getActorsInvolvedInVideo, getAudienceFromFollowersOf, getOriginVideoAudience } from './audience-utils.js' + +async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAudience) => Activity, options: { + byActor: MActorLight + video: MVideoImmutable | MVideoAccountLight + contextType: ContextType + parallelizable?: boolean + transaction?: Transaction +}) { + const { byActor, video, transaction, contextType, parallelizable } = options + + // Send to origin + if (video.isOwned() === false) { + return sendVideoActivityToOrigin(activityBuilder, options) + } + + const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, transaction) + + // Send to followers + const audience = getAudienceFromFollowersOf(actorsInvolvedInVideo) + const activity = activityBuilder(audience) + + const actorsException = [ byActor ] + + return broadcastToFollowers({ + data: activity, + byActor, + toFollowersOf: actorsInvolvedInVideo, + transaction, + actorsException, + parallelizable, + contextType + }) +} + +async function sendVideoActivityToOrigin (activityBuilder: (audience: ActivityAudience) => Activity, options: { + byActor: MActorLight + video: MVideoImmutable | MVideoAccountLight + contextType: ContextType + + actorsInvolvedInVideo?: MActorLight[] + transaction?: Transaction +}) { + const { byActor, video, actorsInvolvedInVideo, transaction, contextType } = options + + if (video.isOwned()) throw new Error('Cannot send activity to owned video origin ' + video.url) + + let accountActor: MActorLight = (video as MVideoAccountLight).VideoChannel?.Account?.Actor + if (!accountActor) accountActor = await ActorModel.loadAccountActorByVideoId(video.id, transaction) + + const audience = getOriginVideoAudience(accountActor, actorsInvolvedInVideo) + const activity = activityBuilder(audience) + + return afterCommitIfTransaction(transaction, () => { + return unicastTo({ + data: activity, + byActor, + toActorUrl: accountActor.getSharedInbox(), + contextType + }) + }) +} + +// --------------------------------------------------------------------------- + +async function forwardVideoRelatedActivity ( + activity: Activity, + t: Transaction, + followersException: MActorWithInboxes[], + video: MVideoId +) { + // Mastodon does not add our announces in audience, so we forward to them manually + const additionalActors = await getActorsInvolvedInVideo(video, t) + const additionalFollowerUrls = additionalActors.map(a => a.followersUrl) + + return forwardActivity(activity, t, followersException, additionalFollowerUrls) +} + +async function forwardActivity ( + activity: Activity, + t: Transaction, + followersException: MActorWithInboxes[] = [], + additionalFollowerUrls: string[] = [] +) { + logger.info('Forwarding activity %s.', activity.id) + + const to = activity.to || [] + const cc = activity.cc || [] + + const followersUrls = additionalFollowerUrls + for (const dest of to.concat(cc)) { + if (dest.endsWith('/followers')) { + followersUrls.push(dest) + } + } + + const toActorFollowers = await ActorModel.listByFollowersUrls(followersUrls, t) + const uris = await computeFollowerUris(toActorFollowers, followersException, t) + + if (uris.length === 0) { + logger.info('0 followers for %s, no forwarding.', toActorFollowers.map(a => a.id).join(', ')) + return undefined + } + + logger.debug('Creating forwarding job.', { uris }) + + const payload: ActivitypubHttpBroadcastPayload = { + uris, + body: activity, + contextType: null + } + return afterCommitIfTransaction(t, () => JobQueue.Instance.createJobAsync({ type: 'activitypub-http-broadcast', payload })) +} + +// --------------------------------------------------------------------------- + +async function broadcastToFollowers (options: { + data: any + byActor: MActorId + toFollowersOf: MActorId[] + transaction: Transaction + contextType: ContextType + + parallelizable?: boolean + actorsException?: MActorWithInboxes[] +}) { + const { data, byActor, toFollowersOf, transaction, contextType, actorsException = [], parallelizable } = options + + const uris = await computeFollowerUris(toFollowersOf, actorsException, transaction) + + return afterCommitIfTransaction(transaction, () => { + return broadcastTo({ + uris, + data, + byActor, + parallelizable, + contextType + }) + }) +} + +async function broadcastToActors (options: { + data: any + byActor: MActorId + toActors: MActor[] + transaction: Transaction + contextType: ContextType + actorsException?: MActorWithInboxes[] +}) { + const { data, byActor, toActors, transaction, contextType, actorsException = [] } = options + + const uris = await computeUris(toActors, actorsException) + + return afterCommitIfTransaction(transaction, () => { + return broadcastTo({ + uris, + data, + byActor, + contextType + }) + }) +} + +function broadcastTo (options: { + uris: string[] + data: any + byActor: MActorId + contextType: ContextType + parallelizable?: boolean // default to false +}) { + const { uris, data, byActor, contextType, parallelizable } = options + + if (uris.length === 0) return undefined + + const broadcastUris: string[] = [] + const unicastUris: string[] = [] + + // Bad URIs could be slow to respond, prefer to process them in a dedicated queue + for (const uri of uris) { + if (ActorFollowHealthCache.Instance.isBadInbox(uri)) { + unicastUris.push(uri) + } else { + broadcastUris.push(uri) + } + } + + logger.debug('Creating broadcast job.', { broadcastUris, unicastUris }) + + if (broadcastUris.length !== 0) { + const payload = { + uris: broadcastUris, + signatureActorId: byActor.id, + body: data, + contextType + } + + JobQueue.Instance.createJobAsync({ + type: parallelizable + ? 'activitypub-http-broadcast-parallel' + : 'activitypub-http-broadcast', + + payload + }) + } + + for (const unicastUri of unicastUris) { + const payload = { + uri: unicastUri, + signatureActorId: byActor.id, + body: data, + contextType + } + + JobQueue.Instance.createJobAsync({ type: 'activitypub-http-unicast', payload }) + } +} + +function unicastTo (options: { + data: any + byActor: MActorId + toActorUrl: string + contextType: ContextType +}) { + const { data, byActor, toActorUrl, contextType } = options + + logger.debug('Creating unicast job.', { uri: toActorUrl }) + + const payload = { + uri: toActorUrl, + signatureActorId: byActor.id, + body: data, + contextType + } + + JobQueue.Instance.createJobAsync({ type: 'activitypub-http-unicast', payload }) +} + +// --------------------------------------------------------------------------- + +export { + broadcastToFollowers, + unicastTo, + forwardActivity, + broadcastToActors, + sendVideoActivityToOrigin, + forwardVideoRelatedActivity, + sendVideoRelatedActivity +} + +// --------------------------------------------------------------------------- + +async function computeFollowerUris (toFollowersOf: MActorId[], actorsException: MActorWithInboxes[], t: Transaction) { + const toActorFollowerIds = toFollowersOf.map(a => a.id) + + const result = await ActorFollowModel.listAcceptedFollowerSharedInboxUrls(toActorFollowerIds, t) + const sharedInboxesException = await buildSharedInboxesException(actorsException) + + return result.data.filter(sharedInbox => sharedInboxesException.includes(sharedInbox) === false) +} + +async function computeUris (toActors: MActor[], actorsException: MActorWithInboxes[] = []) { + const serverActor = await getServerActor() + const targetUrls = toActors + .filter(a => a.id !== serverActor.id) // Don't send to ourselves + .map(a => a.getSharedInbox()) + + const toActorSharedInboxesSet = new Set(targetUrls) + + const sharedInboxesException = await buildSharedInboxesException(actorsException) + return Array.from(toActorSharedInboxesSet) + .filter(sharedInbox => sharedInboxesException.includes(sharedInbox) === false) +} + +async function buildSharedInboxesException (actorsException: MActorWithInboxes[]) { + const serverActor = await getServerActor() + + return actorsException + .map(f => f.getSharedInbox()) + .concat([ serverActor.sharedInboxUrl ]) +} diff --git a/server/server/lib/activitypub/share.ts b/server/server/lib/activitypub/share.ts new file mode 100644 index 000000000..b1e49861e --- /dev/null +++ b/server/server/lib/activitypub/share.ts @@ -0,0 +1,120 @@ +import Bluebird from 'bluebird' +import { Transaction } from 'sequelize' +import { getServerActor } from '@server/models/application/application.js' +import { logger, loggerTagsFactory } from '../../helpers/logger.js' +import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants.js' +import { VideoShareModel } from '../../models/video/video-share.js' +import { MChannelActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../types/models/video/index.js' +import { fetchAP, getAPId } from './activity.js' +import { getOrCreateAPActor } from './actors/index.js' +import { sendUndoAnnounce, sendVideoAnnounce } from './send/index.js' +import { checkUrlsSameHost, getLocalVideoAnnounceActivityPubUrl } from './url.js' + +const lTags = loggerTagsFactory('share') + +async function shareVideoByServerAndChannel (video: MVideoAccountLight, t: Transaction) { + if (!video.hasPrivacyForFederation()) return undefined + + return Promise.all([ + shareByServer(video, t), + shareByVideoChannel(video, t) + ]) +} + +async function changeVideoChannelShare ( + video: MVideoAccountLight, + oldVideoChannel: MChannelActorLight, + t: Transaction +) { + logger.info( + 'Updating video channel of video %s: %s -> %s.', video.uuid, oldVideoChannel.name, video.VideoChannel.name, + lTags(video.uuid) + ) + + await undoShareByVideoChannel(video, oldVideoChannel, t) + + await shareByVideoChannel(video, t) +} + +async function addVideoShares (shareUrls: string[], video: MVideoId) { + await Bluebird.map(shareUrls, async shareUrl => { + try { + await addVideoShare(shareUrl, video) + } catch (err) { + logger.warn('Cannot add share %s.', shareUrl, { err }) + } + }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) +} + +export { + changeVideoChannelShare, + addVideoShares, + shareVideoByServerAndChannel +} + +// --------------------------------------------------------------------------- + +async function addVideoShare (shareUrl: string, video: MVideoId) { + const { body } = await fetchAP(shareUrl) + if (!body?.actor) throw new Error('Body or body actor is invalid') + + const actorUrl = getAPId(body.actor) + if (checkUrlsSameHost(shareUrl, actorUrl) !== true) { + throw new Error(`Actor url ${actorUrl} has not the same host than the share url ${shareUrl}`) + } + + const actor = await getOrCreateAPActor(actorUrl) + + const entry = { + actorId: actor.id, + videoId: video.id, + url: shareUrl + } + + await VideoShareModel.upsert(entry) +} + +async function shareByServer (video: MVideo, t: Transaction) { + const serverActor = await getServerActor() + + const serverShareUrl = getLocalVideoAnnounceActivityPubUrl(serverActor, video) + const [ serverShare ] = await VideoShareModel.findOrCreate({ + defaults: { + actorId: serverActor.id, + videoId: video.id, + url: serverShareUrl + }, + where: { + url: serverShareUrl + }, + transaction: t + }) + + return sendVideoAnnounce(serverActor, serverShare, video, t) +} + +async function shareByVideoChannel (video: MVideoAccountLight, t: Transaction) { + const videoChannelShareUrl = getLocalVideoAnnounceActivityPubUrl(video.VideoChannel.Actor, video) + const [ videoChannelShare ] = await VideoShareModel.findOrCreate({ + defaults: { + actorId: video.VideoChannel.actorId, + videoId: video.id, + url: videoChannelShareUrl + }, + where: { + url: videoChannelShareUrl + }, + transaction: t + }) + + return sendVideoAnnounce(video.VideoChannel.Actor, videoChannelShare, video, t) +} + +async function undoShareByVideoChannel (video: MVideo, oldVideoChannel: MChannelActorLight, t: Transaction) { + // Load old share + const oldShare = await VideoShareModel.load(oldVideoChannel.actorId, video.id, t) + if (!oldShare) return new Error('Cannot find old video channel share ' + oldVideoChannel.actorId + ' for video ' + video.id) + + await sendUndoAnnounce(oldVideoChannel.Actor, oldShare, video, t) + await oldShare.destroy({ transaction: t }) +} diff --git a/server/server/lib/activitypub/url.ts b/server/server/lib/activitypub/url.ts new file mode 100644 index 000000000..73f6f4849 --- /dev/null +++ b/server/server/lib/activitypub/url.ts @@ -0,0 +1,177 @@ +import { REMOTE_SCHEME, WEBSERVER } from '../../initializers/constants.js' +import { + MAbuseFull, + MAbuseId, + MActor, + MActorFollow, + MActorId, + MActorUrl, + MCommentId, + MLocalVideoViewer, + MVideoId, + MVideoPlaylistElement, + MVideoUrl, + MVideoUUID, + MVideoWithHost +} from '../../types/models/index.js' +import { MVideoFileVideoUUID } from '../../types/models/video/video-file.js' +import { MVideoPlaylist, MVideoPlaylistUUID } from '../../types/models/video/video-playlist.js' +import { MStreamingPlaylist } from '../../types/models/video/video-streaming-playlist.js' + +function getLocalVideoActivityPubUrl (video: MVideoUUID) { + return WEBSERVER.URL + '/videos/watch/' + video.uuid +} + +function getLocalVideoPlaylistActivityPubUrl (videoPlaylist: MVideoPlaylist) { + return WEBSERVER.URL + '/video-playlists/' + videoPlaylist.uuid +} + +function getLocalVideoPlaylistElementActivityPubUrl (videoPlaylist: MVideoPlaylistUUID, videoPlaylistElement: MVideoPlaylistElement) { + return WEBSERVER.URL + '/video-playlists/' + videoPlaylist.uuid + '/videos/' + videoPlaylistElement.id +} + +function getLocalVideoCacheFileActivityPubUrl (videoFile: MVideoFileVideoUUID) { + const suffixFPS = videoFile.fps && videoFile.fps !== -1 ? '-' + videoFile.fps : '' + + return `${WEBSERVER.URL}/redundancy/videos/${videoFile.Video.uuid}/${videoFile.resolution}${suffixFPS}` +} + +function getLocalVideoCacheStreamingPlaylistActivityPubUrl (video: MVideoUUID, playlist: MStreamingPlaylist) { + return `${WEBSERVER.URL}/redundancy/streaming-playlists/${playlist.getStringType()}/${video.uuid}` +} + +function getLocalVideoCommentActivityPubUrl (video: MVideoUUID, videoComment: MCommentId) { + return WEBSERVER.URL + '/videos/watch/' + video.uuid + '/comments/' + videoComment.id +} + +function getLocalVideoChannelActivityPubUrl (videoChannelName: string) { + return WEBSERVER.URL + '/video-channels/' + videoChannelName +} + +function getLocalAccountActivityPubUrl (accountName: string) { + return WEBSERVER.URL + '/accounts/' + accountName +} + +function getLocalAbuseActivityPubUrl (abuse: MAbuseId) { + return WEBSERVER.URL + '/admin/abuses/' + abuse.id +} + +function getLocalVideoViewActivityPubUrl (byActor: MActorUrl, video: MVideoId, viewerIdentifier: string) { + return byActor.url + '/views/videos/' + video.id + '/' + viewerIdentifier +} + +function getLocalVideoViewerActivityPubUrl (stats: MLocalVideoViewer) { + return WEBSERVER.URL + '/videos/local-viewer/' + stats.uuid +} + +function getVideoLikeActivityPubUrlByLocalActor (byActor: MActorUrl, video: MVideoId) { + return byActor.url + '/likes/' + video.id +} + +function getVideoDislikeActivityPubUrlByLocalActor (byActor: MActorUrl, video: MVideoId) { + return byActor.url + '/dislikes/' + video.id +} + +function getLocalVideoSharesActivityPubUrl (video: MVideoUrl) { + return video.url + '/announces' +} + +function getLocalVideoCommentsActivityPubUrl (video: MVideoUrl) { + return video.url + '/comments' +} + +function getLocalVideoLikesActivityPubUrl (video: MVideoUrl) { + return video.url + '/likes' +} + +function getLocalVideoDislikesActivityPubUrl (video: MVideoUrl) { + return video.url + '/dislikes' +} + +function getLocalActorFollowActivityPubUrl (follower: MActor, following: MActorId) { + return follower.url + '/follows/' + following.id +} + +function getLocalActorFollowAcceptActivityPubUrl (actorFollow: MActorFollow) { + return WEBSERVER.URL + '/accepts/follows/' + actorFollow.id +} + +function getLocalActorFollowRejectActivityPubUrl () { + return WEBSERVER.URL + '/rejects/follows/' + new Date().toISOString() +} + +function getLocalVideoAnnounceActivityPubUrl (byActor: MActorId, video: MVideoUrl) { + return video.url + '/announces/' + byActor.id +} + +function getDeleteActivityPubUrl (originalUrl: string) { + return originalUrl + '/delete' +} + +function getUpdateActivityPubUrl (originalUrl: string, updatedAt: string) { + return originalUrl + '/updates/' + updatedAt +} + +function getUndoActivityPubUrl (originalUrl: string) { + return originalUrl + '/undo' +} + +// --------------------------------------------------------------------------- + +function getAbuseTargetUrl (abuse: MAbuseFull) { + return abuse.VideoAbuse?.Video?.url || + abuse.VideoCommentAbuse?.VideoComment?.url || + abuse.FlaggedAccount.Actor.url +} + +// --------------------------------------------------------------------------- + +function buildRemoteVideoBaseUrl (video: MVideoWithHost, path: string, scheme?: string) { + if (!scheme) scheme = REMOTE_SCHEME.HTTP + + const host = video.VideoChannel.Actor.Server.host + + return scheme + '://' + host + path +} + +// --------------------------------------------------------------------------- + +function checkUrlsSameHost (url1: string, url2: string) { + const idHost = new URL(url1).host + const actorHost = new URL(url2).host + + return idHost && actorHost && idHost.toLowerCase() === actorHost.toLowerCase() +} + +// --------------------------------------------------------------------------- + +export { + getLocalVideoActivityPubUrl, + getLocalVideoPlaylistActivityPubUrl, + getLocalVideoPlaylistElementActivityPubUrl, + getLocalVideoCacheFileActivityPubUrl, + getLocalVideoCacheStreamingPlaylistActivityPubUrl, + getLocalVideoCommentActivityPubUrl, + getLocalVideoChannelActivityPubUrl, + getLocalAccountActivityPubUrl, + getLocalAbuseActivityPubUrl, + getLocalActorFollowActivityPubUrl, + getLocalActorFollowAcceptActivityPubUrl, + getLocalVideoAnnounceActivityPubUrl, + getUpdateActivityPubUrl, + getUndoActivityPubUrl, + getVideoLikeActivityPubUrlByLocalActor, + getLocalVideoViewActivityPubUrl, + getVideoDislikeActivityPubUrlByLocalActor, + getLocalActorFollowRejectActivityPubUrl, + getDeleteActivityPubUrl, + getLocalVideoSharesActivityPubUrl, + getLocalVideoCommentsActivityPubUrl, + getLocalVideoLikesActivityPubUrl, + getLocalVideoDislikesActivityPubUrl, + getLocalVideoViewerActivityPubUrl, + + getAbuseTargetUrl, + checkUrlsSameHost, + buildRemoteVideoBaseUrl +} diff --git a/server/server/lib/activitypub/video-comments.ts b/server/server/lib/activitypub/video-comments.ts new file mode 100644 index 000000000..9da89ef0c --- /dev/null +++ b/server/server/lib/activitypub/video-comments.ts @@ -0,0 +1,204 @@ +import Bluebird from 'bluebird' +import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments.js' +import { logger } from '../../helpers/logger.js' +import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants.js' +import { VideoCommentModel } from '../../models/video/video-comment.js' +import { MComment, MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video/index.js' +import { isRemoteVideoCommentAccepted } from '../moderation.js' +import { Hooks } from '../plugins/hooks.js' +import { fetchAP } from './activity.js' +import { getOrCreateAPActor } from './actors/index.js' +import { checkUrlsSameHost } from './url.js' +import { getOrCreateAPVideo } from './videos/index.js' + +type ResolveThreadParams = { + url: string + comments?: MCommentOwner[] + isVideo?: boolean + commentCreated?: boolean +} +type ResolveThreadResult = Promise<{ video: MVideoAccountLightBlacklistAllFiles, comment: MCommentOwnerVideo, commentCreated: boolean }> + +async function addVideoComments (commentUrls: string[]) { + return Bluebird.map(commentUrls, async commentUrl => { + try { + await resolveThread({ url: commentUrl, isVideo: false }) + } catch (err) { + logger.warn('Cannot resolve thread %s.', commentUrl, { err }) + } + }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) +} + +async function resolveThread (params: ResolveThreadParams): ResolveThreadResult { + const { url, isVideo } = params + + if (params.commentCreated === undefined) params.commentCreated = false + if (params.comments === undefined) params.comments = [] + + // If it is not a video, or if we don't know if it's a video, try to get the thread from DB + if (isVideo === false || isVideo === undefined) { + const result = await resolveCommentFromDB(params) + if (result) return result + } + + try { + // If it is a video, or if we don't know if it's a video + if (isVideo === true || isVideo === undefined) { + // Keep await so we catch the exception + return await tryToResolveThreadFromVideo(params) + } + } catch (err) { + logger.debug('Cannot resolve thread from video %s, maybe because it was not a video', url, { err }) + } + + return resolveRemoteParentComment(params) +} + +export { + addVideoComments, + resolveThread +} + +// --------------------------------------------------------------------------- + +async function resolveCommentFromDB (params: ResolveThreadParams) { + const { url, comments, commentCreated } = params + + const commentFromDatabase = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideoUrlAndAccount(url) + if (!commentFromDatabase) return undefined + + let parentComments = comments.concat([ commentFromDatabase ]) + + // Speed up things and resolve directly the thread + if (commentFromDatabase.InReplyToVideoComment) { + const data = await VideoCommentModel.listThreadParentComments(commentFromDatabase, undefined, 'DESC') + + parentComments = parentComments.concat(data) + } + + return resolveThread({ + url: commentFromDatabase.Video.url, + comments: parentComments, + isVideo: true, + commentCreated + }) +} + +async function tryToResolveThreadFromVideo (params: ResolveThreadParams) { + const { url, comments, commentCreated } = params + + // Maybe it's a reply to a video? + // If yes, it's done: we resolved all the thread + const syncParam = { rates: true, shares: true, comments: false, refreshVideo: false } + const { video } = await getOrCreateAPVideo({ videoObject: url, syncParam }) + + if (video.isOwned() && !video.hasPrivacyForFederation()) { + throw new Error('Cannot resolve thread of video with privacy that is not compatible with federation') + } + + let resultComment: MCommentOwnerVideo + if (comments.length !== 0) { + const firstReply = comments[comments.length - 1] as MCommentOwnerVideo + firstReply.inReplyToCommentId = null + firstReply.originCommentId = null + firstReply.videoId = video.id + firstReply.changed('updatedAt', true) + firstReply.Video = video + + if (await isRemoteCommentAccepted(firstReply) !== true) { + return undefined + } + + comments[comments.length - 1] = await firstReply.save() + + for (let i = comments.length - 2; i >= 0; i--) { + const comment = comments[i] as MCommentOwnerVideo + comment.originCommentId = firstReply.id + comment.inReplyToCommentId = comments[i + 1].id + comment.videoId = video.id + comment.changed('updatedAt', true) + comment.Video = video + + if (await isRemoteCommentAccepted(comment) !== true) { + return undefined + } + + comments[i] = await comment.save() + } + + resultComment = comments[0] as MCommentOwnerVideo + } + + return { video, comment: resultComment, commentCreated } +} + +async function resolveRemoteParentComment (params: ResolveThreadParams) { + const { url, comments } = params + + if (comments.length > ACTIVITY_PUB.MAX_RECURSION_COMMENTS) { + throw new Error('Recursion limit reached when resolving a thread') + } + + const { body } = await fetchAP(url) + + if (sanitizeAndCheckVideoCommentObject(body) === false) { + throw new Error(`Remote video comment JSON ${url} is not valid:` + JSON.stringify(body)) + } + + const actorUrl = body.attributedTo + if (!actorUrl && body.type !== 'Tombstone') throw new Error('Miss attributed to in comment') + + if (actorUrl && checkUrlsSameHost(url, actorUrl) !== true) { + throw new Error(`Actor url ${actorUrl} has not the same host than the comment url ${url}`) + } + + if (checkUrlsSameHost(body.id, url) !== true) { + throw new Error(`Comment url ${url} host is different from the AP object id ${body.id}`) + } + + const actor = actorUrl + ? await getOrCreateAPActor(actorUrl, 'all') + : null + + const comment = new VideoCommentModel({ + url: body.id, + text: body.content ? body.content : '', + videoId: null, + accountId: actor ? actor.Account.id : null, + inReplyToCommentId: null, + originCommentId: null, + createdAt: new Date(body.published), + updatedAt: new Date(body.updated), + deletedAt: body.deleted ? new Date(body.deleted) : null + }) as MCommentOwner + comment.Account = actor ? actor.Account : null + + return resolveThread({ + url: body.inReplyTo, + comments: comments.concat([ comment ]), + commentCreated: true + }) +} + +async function isRemoteCommentAccepted (comment: MComment) { + // Already created + if (comment.id) return true + + const acceptParameters = { + comment + } + + const acceptedResult = await Hooks.wrapFun( + isRemoteVideoCommentAccepted, + acceptParameters, + 'filter:activity-pub.remote-video-comment.create.accept.result' + ) + + if (!acceptedResult || acceptedResult.accepted !== true) { + logger.info('Refused to create a remote comment.', { acceptedResult, acceptParameters }) + + return false + } + + return true +} diff --git a/server/server/lib/activitypub/video-rates.ts b/server/server/lib/activitypub/video-rates.ts new file mode 100644 index 000000000..1c0d5ec14 --- /dev/null +++ b/server/server/lib/activitypub/video-rates.ts @@ -0,0 +1,59 @@ +import { Transaction } from 'sequelize' +import { VideoRateType } from '@peertube/peertube-models' +import { MAccountActor, MActorUrl, MVideoAccountLight, MVideoFullLight, MVideoId } from '../../types/models/index.js' +import { sendLike, sendUndoDislike, sendUndoLike } from './send/index.js' +import { sendDislike } from './send/send-dislike.js' +import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from './url.js' +import { federateVideoIfNeeded } from './videos/index.js' + +async function sendVideoRateChange ( + account: MAccountActor, + video: MVideoFullLight, + likes: number, + dislikes: number, + t: Transaction +) { + if (video.isOwned()) return federateVideoIfNeeded(video, false, t) + + return sendVideoRateChangeToOrigin(account, video, likes, dislikes, t) +} + +function getLocalRateUrl (rateType: VideoRateType, actor: MActorUrl, video: MVideoId) { + return rateType === 'like' + ? getVideoLikeActivityPubUrlByLocalActor(actor, video) + : getVideoDislikeActivityPubUrlByLocalActor(actor, video) +} + +// --------------------------------------------------------------------------- + +export { + getLocalRateUrl, + sendVideoRateChange +} + +// --------------------------------------------------------------------------- + +async function sendVideoRateChangeToOrigin ( + account: MAccountActor, + video: MVideoAccountLight, + likes: number, + dislikes: number, + t: Transaction +) { + // Local video, we don't need to send like + if (video.isOwned()) return + + const actor = account.Actor + + // Keep the order: first we undo and then we create + + // Undo Like + if (likes < 0) await sendUndoLike(actor, video, t) + // Undo Dislike + if (dislikes < 0) await sendUndoDislike(actor, video, t) + + // Like + if (likes > 0) await sendLike(actor, video, t) + // Dislike + if (dislikes > 0) await sendDislike(actor, video, t) +} diff --git a/server/server/lib/activitypub/videos/federate.ts b/server/server/lib/activitypub/videos/federate.ts new file mode 100644 index 000000000..76b9030f1 --- /dev/null +++ b/server/server/lib/activitypub/videos/federate.ts @@ -0,0 +1,29 @@ +import { Transaction } from 'sequelize' +import { MVideoAP, MVideoAPLight } from '@server/types/models/index.js' +import { sendCreateVideo, sendUpdateVideo } from '../send/index.js' +import { shareVideoByServerAndChannel } from '../share.js' + +async function federateVideoIfNeeded (videoArg: MVideoAPLight, isNewVideo: boolean, transaction?: Transaction) { + const video = videoArg as MVideoAP + + if ( + // Check this is not a blacklisted video, or unfederated blacklisted video + (video.isBlacklisted() === false || (isNewVideo === false && video.VideoBlacklist.unfederated === false)) && + // Check the video is public/unlisted and published + video.hasPrivacyForFederation() && video.hasStateForFederation() + ) { + const video = await videoArg.lightAPToFullAP(transaction) + + if (isNewVideo) { + // Now we'll add the video's meta data to our followers + await sendCreateVideo(video, transaction) + await shareVideoByServerAndChannel(video, transaction) + } else { + await sendUpdateVideo(video, transaction) + } + } +} + +export { + federateVideoIfNeeded +} diff --git a/server/server/lib/activitypub/videos/get.ts b/server/server/lib/activitypub/videos/get.ts new file mode 100644 index 000000000..6d70a490f --- /dev/null +++ b/server/server/lib/activitypub/videos/get.ts @@ -0,0 +1,116 @@ +import { retryTransactionWrapper } from '@server/helpers/database-utils.js' +import { logger } from '@server/helpers/logger.js' +import { JobQueue } from '@server/lib/job-queue/index.js' +import { loadVideoByUrl, VideoLoadByUrlType } from '@server/lib/model-loaders/index.js' +import { MVideoAccountLightBlacklistAllFiles, MVideoImmutable, MVideoThumbnail } from '@server/types/models/index.js' +import { APObjectId } from '@peertube/peertube-models' +import { getAPId } from '../activity.js' +import { refreshVideoIfNeeded } from './refresh.js' +import { APVideoCreator, fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared/index.js' + +type GetVideoResult = Promise<{ + video: T + created: boolean + autoBlacklisted?: boolean +}> + +type GetVideoParamAll = { + videoObject: APObjectId + syncParam?: SyncParam + fetchType?: 'all' + allowRefresh?: boolean +} + +type GetVideoParamImmutable = { + videoObject: APObjectId + syncParam?: SyncParam + fetchType: 'only-immutable-attributes' + allowRefresh: false +} + +type GetVideoParamOther = { + videoObject: APObjectId + syncParam?: SyncParam + fetchType?: 'all' | 'only-video' + allowRefresh?: boolean +} + +function getOrCreateAPVideo (options: GetVideoParamAll): GetVideoResult +function getOrCreateAPVideo (options: GetVideoParamImmutable): GetVideoResult +function getOrCreateAPVideo (options: GetVideoParamOther): GetVideoResult + +async function getOrCreateAPVideo ( + options: GetVideoParamAll | GetVideoParamImmutable | GetVideoParamOther +): GetVideoResult { + // Default params + const syncParam = options.syncParam || { rates: true, shares: true, comments: true, refreshVideo: false } + const fetchType = options.fetchType || 'all' + const allowRefresh = options.allowRefresh !== false + + // Get video url + const videoUrl = getAPId(options.videoObject) + let videoFromDatabase = await loadVideoByUrl(videoUrl, fetchType) + + if (videoFromDatabase) { + if (allowRefresh === true) { + // Typings ensure allowRefresh === false in only-immutable-attributes fetch type + videoFromDatabase = await scheduleRefresh(videoFromDatabase as MVideoThumbnail, fetchType, syncParam) + } + + return { video: videoFromDatabase, created: false } + } + + const { videoObject } = await fetchRemoteVideo(videoUrl) + if (!videoObject) throw new Error('Cannot fetch remote video with url: ' + videoUrl) + + // videoUrl is just an alias/rediraction, so process object id instead + if (videoObject.id !== videoUrl) return getOrCreateAPVideo({ ...options, fetchType: 'all', videoObject }) + + try { + const creator = new APVideoCreator(videoObject) + const { autoBlacklisted, videoCreated } = await retryTransactionWrapper(creator.create.bind(creator)) + + await syncVideoExternalAttributes(videoCreated, videoObject, syncParam) + + return { video: videoCreated, created: true, autoBlacklisted } + } catch (err) { + // Maybe a concurrent getOrCreateAPVideo call created this video + if (err.name === 'SequelizeUniqueConstraintError') { + const alreadyCreatedVideo = await loadVideoByUrl(videoUrl, fetchType) + if (alreadyCreatedVideo) return { video: alreadyCreatedVideo, created: false } + + logger.error('Cannot create video %s because of SequelizeUniqueConstraintError error, but cannot find it in database.', videoUrl) + } + + throw err + } +} + +// --------------------------------------------------------------------------- + +export { + getOrCreateAPVideo +} + +// --------------------------------------------------------------------------- + +async function scheduleRefresh (video: MVideoThumbnail, fetchType: VideoLoadByUrlType, syncParam: SyncParam) { + if (!video.isOutdated()) return video + + const refreshOptions = { + video, + fetchedType: fetchType, + syncParam + } + + if (syncParam.refreshVideo === true) { + return refreshVideoIfNeeded(refreshOptions) + } + + await JobQueue.Instance.createJob({ + type: 'activitypub-refresher', + payload: { type: 'video', url: video.url } + }) + + return video +} diff --git a/server/server/lib/activitypub/videos/index.ts b/server/server/lib/activitypub/videos/index.ts new file mode 100644 index 000000000..05af1431c --- /dev/null +++ b/server/server/lib/activitypub/videos/index.ts @@ -0,0 +1,4 @@ +export * from './federate.js' +export * from './get.js' +export * from './refresh.js' +export * from './updater.js' diff --git a/server/server/lib/activitypub/videos/refresh.ts b/server/server/lib/activitypub/videos/refresh.ts new file mode 100644 index 000000000..961bdf71c --- /dev/null +++ b/server/server/lib/activitypub/videos/refresh.ts @@ -0,0 +1,68 @@ +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { PeerTubeRequestError } from '@server/helpers/requests.js' +import { VideoLoadByUrlType } from '@server/lib/model-loaders/index.js' +import { VideoModel } from '@server/models/video/video.js' +import { MVideoAccountLightBlacklistAllFiles, MVideoThumbnail } from '@server/types/models/index.js' +import { HttpStatusCode } from '@peertube/peertube-models' +import { ActorFollowHealthCache } from '../../actor-follow-health-cache.js' +import { fetchRemoteVideo, SyncParam, syncVideoExternalAttributes } from './shared/index.js' +import { APVideoUpdater } from './updater.js' + +async function refreshVideoIfNeeded (options: { + video: MVideoThumbnail + fetchedType: VideoLoadByUrlType + syncParam: SyncParam +}): Promise { + if (!options.video.isOutdated()) return options.video + + // We need more attributes if the argument video was fetched with not enough joints + const video = options.fetchedType === 'all' + ? options.video as MVideoAccountLightBlacklistAllFiles + : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) + + const lTags = loggerTagsFactory('ap', 'video', 'refresh', video.uuid, video.url) + + logger.info('Refreshing video %s.', video.url, lTags()) + + try { + const { videoObject } = await fetchRemoteVideo(video.url) + + if (videoObject === undefined) { + logger.warn('Cannot refresh remote video %s: invalid body.', video.url, lTags()) + + await video.setAsRefreshed() + return video + } + + const videoUpdater = new APVideoUpdater(videoObject, video) + await videoUpdater.update() + + await syncVideoExternalAttributes(video, videoObject, options.syncParam) + + ActorFollowHealthCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId) + + return video + } catch (err) { + if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { + logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url, lTags()) + + // Video does not exist anymore + await video.destroy() + return undefined + } + + logger.warn('Cannot refresh video %s.', options.video.url, { err, ...lTags() }) + + ActorFollowHealthCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId) + + // Don't refresh in loop + await video.setAsRefreshed() + return video + } +} + +// --------------------------------------------------------------------------- + +export { + refreshVideoIfNeeded +} diff --git a/server/server/lib/activitypub/videos/shared/abstract-builder.ts b/server/server/lib/activitypub/videos/shared/abstract-builder.ts new file mode 100644 index 000000000..4397e578f --- /dev/null +++ b/server/server/lib/activitypub/videos/shared/abstract-builder.ts @@ -0,0 +1,190 @@ +import { CreationAttributes, Transaction } from 'sequelize' +import { ActivityTagObject, ThumbnailType, VideoObject, VideoStreamingPlaylistType_Type } from '@peertube/peertube-models' +import { deleteAllModels, filterNonExistingModels } from '@server/helpers/database-utils.js' +import { logger, LoggerTagsFn } from '@server/helpers/logger.js' +import { updateRemoteVideoThumbnail } from '@server/lib/thumbnail.js' +import { setVideoTags } from '@server/lib/video.js' +import { StoryboardModel } from '@server/models/video/storyboard.js' +import { VideoCaptionModel } from '@server/models/video/video-caption.js' +import { VideoFileModel } from '@server/models/video/video-file.js' +import { VideoLiveModel } from '@server/models/video/video-live.js' +import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js' +import { + MStreamingPlaylistFiles, + MStreamingPlaylistFilesVideo, + MVideoCaption, + MVideoFile, + MVideoFullLight, + MVideoThumbnail +} from '@server/types/models/index.js' +import { findOwner, getOrCreateAPActor } from '../../actors/index.js' +import { + getCaptionAttributesFromObject, + getFileAttributesFromUrl, + getLiveAttributesFromObject, + getPreviewFromIcons, + getStoryboardAttributeFromObject, + getStreamingPlaylistAttributesFromObject, + getTagsFromObject, + getThumbnailFromIcons +} from './object-to-model-attributes.js' +import { getTrackerUrls, setVideoTrackers } from './trackers.js' + +export abstract class APVideoAbstractBuilder { + protected abstract videoObject: VideoObject + protected abstract lTags: LoggerTagsFn + + protected async getOrCreateVideoChannelFromVideoObject () { + const channel = await findOwner(this.videoObject.id, this.videoObject.attributedTo, 'Group') + if (!channel) throw new Error('Cannot find associated video channel to video ' + this.videoObject.url) + + return getOrCreateAPActor(channel.id, 'all') + } + + protected async setThumbnail (video: MVideoThumbnail, t?: Transaction) { + const miniatureIcon = getThumbnailFromIcons(this.videoObject) + if (!miniatureIcon) { + logger.warn('Cannot find thumbnail in video object', { object: this.videoObject }) + return undefined + } + + const miniatureModel = updateRemoteVideoThumbnail({ + fileUrl: miniatureIcon.url, + video, + type: ThumbnailType.MINIATURE, + size: miniatureIcon, + onDisk: false // Lazy download remote thumbnails + }) + + await video.addAndSaveThumbnail(miniatureModel, t) + } + + protected async setPreview (video: MVideoFullLight, t?: Transaction) { + const previewIcon = getPreviewFromIcons(this.videoObject) + if (!previewIcon) return + + const previewModel = updateRemoteVideoThumbnail({ + fileUrl: previewIcon.url, + video, + type: ThumbnailType.PREVIEW, + size: previewIcon, + onDisk: false // Lazy download remote previews + }) + + await video.addAndSaveThumbnail(previewModel, t) + } + + protected async setTags (video: MVideoFullLight, t: Transaction) { + const tags = getTagsFromObject(this.videoObject) + await setVideoTags({ video, tags, transaction: t }) + } + + protected async setTrackers (video: MVideoFullLight, t: Transaction) { + const trackers = getTrackerUrls(this.videoObject, video) + await setVideoTrackers({ video, trackers, transaction: t }) + } + + protected async insertOrReplaceCaptions (video: MVideoFullLight, t: Transaction) { + const existingCaptions = await VideoCaptionModel.listVideoCaptions(video.id, t) + + let captionsToCreate = getCaptionAttributesFromObject(video, this.videoObject) + .map(a => new VideoCaptionModel(a) as MVideoCaption) + + for (const existingCaption of existingCaptions) { + // Only keep captions that do not already exist + const filtered = captionsToCreate.filter(c => !c.isEqual(existingCaption)) + + // This caption already exists, we don't need to destroy and create it + if (filtered.length !== captionsToCreate.length) { + captionsToCreate = filtered + continue + } + + // Destroy this caption that does not exist anymore + await existingCaption.destroy({ transaction: t }) + } + + for (const captionToCreate of captionsToCreate) { + await captionToCreate.save({ transaction: t }) + } + } + + protected async insertOrReplaceStoryboard (video: MVideoFullLight, t: Transaction) { + const existingStoryboard = await StoryboardModel.loadByVideo(video.id, t) + if (existingStoryboard) await existingStoryboard.destroy({ transaction: t }) + + const storyboardAttributes = getStoryboardAttributeFromObject(video, this.videoObject) + if (!storyboardAttributes) return + + return StoryboardModel.create(storyboardAttributes, { transaction: t }) + } + + protected async insertOrReplaceLive (video: MVideoFullLight, transaction: Transaction) { + const attributes = getLiveAttributesFromObject(video, this.videoObject) + const [ videoLive ] = await VideoLiveModel.upsert(attributes, { transaction, returning: true }) + + video.VideoLive = videoLive + } + + protected async setWebVideoFiles (video: MVideoFullLight, t: Transaction) { + const videoFileAttributes = getFileAttributesFromUrl(video, this.videoObject.url) + const newVideoFiles = videoFileAttributes.map(a => new VideoFileModel(a)) + + // Remove video files that do not exist anymore + await deleteAllModels(filterNonExistingModels(video.VideoFiles || [], newVideoFiles), t) + + // Update or add other one + const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'video', t)) + video.VideoFiles = await Promise.all(upsertTasks) + } + + protected async setStreamingPlaylists (video: MVideoFullLight, t: Transaction) { + const streamingPlaylistAttributes = getStreamingPlaylistAttributesFromObject(video, this.videoObject) + const newStreamingPlaylists = streamingPlaylistAttributes.map(a => new VideoStreamingPlaylistModel(a)) + + // Remove video playlists that do not exist anymore + await deleteAllModels(filterNonExistingModels(video.VideoStreamingPlaylists || [], newStreamingPlaylists), t) + + const oldPlaylists = video.VideoStreamingPlaylists + video.VideoStreamingPlaylists = [] + + for (const playlistAttributes of streamingPlaylistAttributes) { + const streamingPlaylistModel = await this.insertOrReplaceStreamingPlaylist(playlistAttributes, t) + streamingPlaylistModel.Video = video + + await this.setStreamingPlaylistFiles(oldPlaylists, streamingPlaylistModel, playlistAttributes.tagAPObject, t) + + video.VideoStreamingPlaylists.push(streamingPlaylistModel) + } + } + + private async insertOrReplaceStreamingPlaylist (attributes: CreationAttributes, t: Transaction) { + const [ streamingPlaylist ] = await VideoStreamingPlaylistModel.upsert(attributes, { returning: true, transaction: t }) + + return streamingPlaylist as MStreamingPlaylistFilesVideo + } + + private getStreamingPlaylistFiles (oldPlaylists: MStreamingPlaylistFiles[], type: VideoStreamingPlaylistType_Type) { + const playlist = oldPlaylists.find(s => s.type === type) + if (!playlist) return [] + + return playlist.VideoFiles + } + + private async setStreamingPlaylistFiles ( + oldPlaylists: MStreamingPlaylistFiles[], + playlistModel: MStreamingPlaylistFilesVideo, + tagObjects: ActivityTagObject[], + t: Transaction + ) { + const oldStreamingPlaylistFiles = this.getStreamingPlaylistFiles(oldPlaylists || [], playlistModel.type) + + const newVideoFiles: MVideoFile[] = getFileAttributesFromUrl(playlistModel, tagObjects).map(a => new VideoFileModel(a)) + + await deleteAllModels(filterNonExistingModels(oldStreamingPlaylistFiles, newVideoFiles), t) + + // Update or add other one + const upsertTasks = newVideoFiles.map(f => VideoFileModel.customUpsert(f, 'streaming-playlist', t)) + playlistModel.VideoFiles = await Promise.all(upsertTasks) + } +} diff --git a/server/server/lib/activitypub/videos/shared/creator.ts b/server/server/lib/activitypub/videos/shared/creator.ts new file mode 100644 index 000000000..5a3a46282 --- /dev/null +++ b/server/server/lib/activitypub/videos/shared/creator.ts @@ -0,0 +1,65 @@ + +import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger.js' +import { sequelizeTypescript } from '@server/initializers/database.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js' +import { VideoModel } from '@server/models/video/video.js' +import { MVideoFullLight, MVideoThumbnail } from '@server/types/models/index.js' +import { VideoObject } from '@peertube/peertube-models' +import { APVideoAbstractBuilder } from './abstract-builder.js' +import { getVideoAttributesFromObject } from './object-to-model-attributes.js' + +export class APVideoCreator extends APVideoAbstractBuilder { + protected lTags: LoggerTagsFn + + constructor (protected readonly videoObject: VideoObject) { + super() + + this.lTags = loggerTagsFactory('ap', 'video', 'create', this.videoObject.uuid, this.videoObject.id) + } + + async create () { + logger.debug('Adding remote video %s.', this.videoObject.id, this.lTags()) + + const channelActor = await this.getOrCreateVideoChannelFromVideoObject() + const channel = channelActor.VideoChannel + + const videoData = getVideoAttributesFromObject(channel, this.videoObject, this.videoObject.to) + const video = VideoModel.build({ ...videoData, likes: 0, dislikes: 0 }) as MVideoThumbnail + + const { autoBlacklisted, videoCreated } = await sequelizeTypescript.transaction(async t => { + const videoCreated = await video.save({ transaction: t }) as MVideoFullLight + videoCreated.VideoChannel = channel + + await this.setThumbnail(videoCreated, t) + await this.setPreview(videoCreated, t) + await this.setWebVideoFiles(videoCreated, t) + await this.setStreamingPlaylists(videoCreated, t) + await this.setTags(videoCreated, t) + await this.setTrackers(videoCreated, t) + await this.insertOrReplaceCaptions(videoCreated, t) + await this.insertOrReplaceLive(videoCreated, t) + await this.insertOrReplaceStoryboard(videoCreated, t) + + // We added a video in this channel, set it as updated + await channel.setAsUpdated(t) + + const autoBlacklisted = await autoBlacklistVideoIfNeeded({ + video: videoCreated, + user: undefined, + isRemote: true, + isNew: true, + isNewFile: true, + transaction: t + }) + + logger.info('Remote video with uuid %s inserted.', this.videoObject.uuid, this.lTags()) + + Hooks.runAction('action:activity-pub.remote-video.created', { video: videoCreated, videoAPObject: this.videoObject }) + + return { autoBlacklisted, videoCreated } + }) + + return { autoBlacklisted, videoCreated } + } +} diff --git a/server/server/lib/activitypub/videos/shared/index.ts b/server/server/lib/activitypub/videos/shared/index.ts new file mode 100644 index 000000000..6792af7da --- /dev/null +++ b/server/server/lib/activitypub/videos/shared/index.ts @@ -0,0 +1,6 @@ +export * from './abstract-builder.js' +export * from './creator.js' +export * from './object-to-model-attributes.js' +export * from './trackers.js' +export * from './url-to-object.js' +export * from './video-sync-attributes.js' diff --git a/server/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/server/lib/activitypub/videos/shared/object-to-model-attributes.ts new file mode 100644 index 000000000..c0a945331 --- /dev/null +++ b/server/server/lib/activitypub/videos/shared/object-to-model-attributes.ts @@ -0,0 +1,286 @@ +import { arrayify } from '@peertube/peertube-core-utils' +import { + ActivityHashTagObject, + ActivityMagnetUrlObject, + ActivityPlaylistSegmentHashesObject, + ActivityPlaylistUrlObject, + ActivityTagObject, + ActivityUrlObject, + ActivityVideoUrlObject, + VideoObject, + VideoPrivacy, + VideoStreamingPlaylistType +} from '@peertube/peertube-models' +import { isAPVideoFileUrlMetadataObject } from '@server/helpers/custom-validators/activitypub/videos.js' +import { isArray } from '@server/helpers/custom-validators/misc.js' +import { isVideoFileInfoHashValid } from '@server/helpers/custom-validators/videos.js' +import { generateImageFilename } from '@server/helpers/image-utils.js' +import { logger } from '@server/helpers/logger.js' +import { getExtFromMimetype } from '@server/helpers/video.js' +import { ACTIVITY_PUB, MIMETYPES, P2P_MEDIA_LOADER_PEER_VERSION, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '@server/initializers/constants.js' +import { generateTorrentFileName } from '@server/lib/paths.js' +import { VideoCaptionModel } from '@server/models/video/video-caption.js' +import { VideoFileModel } from '@server/models/video/video-file.js' +import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js' +import { FilteredModelAttributes } from '@server/types/index.js' +import { MChannelId, MStreamingPlaylistVideo, MVideo, MVideoId, isStreamingPlaylist } from '@server/types/models/index.js' +import maxBy from 'lodash-es/maxBy.js' +import minBy from 'lodash-es/minBy.js' +import { decode as magnetUriDecode } from 'magnet-uri' +import { basename, extname } from 'path' +import { getDurationFromActivityStream } from '../../activity.js' + +function getThumbnailFromIcons (videoObject: VideoObject) { + let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth) + // Fallback if there are not valid icons + if (validIcons.length === 0) validIcons = videoObject.icon + + return minBy(validIcons, 'width') +} + +function getPreviewFromIcons (videoObject: VideoObject) { + const validIcons = videoObject.icon.filter(i => i.width > PREVIEWS_SIZE.minWidth) + + return maxBy(validIcons, 'width') +} + +function getTagsFromObject (videoObject: VideoObject) { + return videoObject.tag + .filter(isAPHashTagObject) + .map(t => t.name) +} + +function getFileAttributesFromUrl ( + videoOrPlaylist: MVideo | MStreamingPlaylistVideo, + urls: (ActivityTagObject | ActivityUrlObject)[] +) { + const fileUrls = urls.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] + + if (fileUrls.length === 0) return [] + + const attributes: FilteredModelAttributes[] = [] + for (const fileUrl of fileUrls) { + // Fetch associated magnet uri + const magnet = urls.filter(isAPMagnetUrlObject) + .find(u => u.height === fileUrl.height) + + if (!magnet) throw new Error('Cannot find associated magnet uri for file ' + fileUrl.href) + + const parsed = magnetUriDecode(magnet.href) + if (!parsed || isVideoFileInfoHashValid(parsed.infoHash) === false) { + throw new Error('Cannot parse magnet URI ' + magnet.href) + } + + const torrentUrl = Array.isArray(parsed.xs) + ? parsed.xs[0] + : parsed.xs + + // Fetch associated metadata url, if any + const metadata = urls.filter(isAPVideoFileUrlMetadataObject) + .find(u => { + return u.height === fileUrl.height && + u.fps === fileUrl.fps && + u.rel.includes(fileUrl.mediaType) + }) + + const extname = getExtFromMimetype(MIMETYPES.VIDEO.MIMETYPE_EXT, fileUrl.mediaType) + const resolution = fileUrl.height + const videoId = isStreamingPlaylist(videoOrPlaylist) ? null : videoOrPlaylist.id + const videoStreamingPlaylistId = isStreamingPlaylist(videoOrPlaylist) ? videoOrPlaylist.id : null + + const attribute = { + extname, + infoHash: parsed.infoHash, + resolution, + size: fileUrl.size, + fps: fileUrl.fps || -1, + metadataUrl: metadata?.href, + + // Use the name of the remote file because we don't proxify video file requests + filename: basename(fileUrl.href), + fileUrl: fileUrl.href, + + torrentUrl, + // Use our own torrent name since we proxify torrent requests + torrentFilename: generateTorrentFileName(videoOrPlaylist, resolution), + + // This is a video file owned by a video or by a streaming playlist + videoId, + videoStreamingPlaylistId + } + + attributes.push(attribute) + } + + return attributes +} + +function getStreamingPlaylistAttributesFromObject (video: MVideoId, videoObject: VideoObject) { + const playlistUrls = videoObject.url.filter(u => isAPStreamingPlaylistUrlObject(u)) as ActivityPlaylistUrlObject[] + if (playlistUrls.length === 0) return [] + + const attributes: (FilteredModelAttributes & { tagAPObject?: ActivityTagObject[] })[] = [] + for (const playlistUrlObject of playlistUrls) { + const segmentsSha256UrlObject = playlistUrlObject.tag.find(isAPPlaylistSegmentHashesUrlObject) + + const files: unknown[] = playlistUrlObject.tag.filter(u => isAPVideoUrlObject(u)) as ActivityVideoUrlObject[] + + if (!segmentsSha256UrlObject) { + logger.warn('No segment sha256 URL found in AP playlist object.', { playlistUrl: playlistUrlObject }) + continue + } + + const attribute = { + type: VideoStreamingPlaylistType.HLS, + + playlistFilename: basename(playlistUrlObject.href), + playlistUrl: playlistUrlObject.href, + + segmentsSha256Filename: basename(segmentsSha256UrlObject.href), + segmentsSha256Url: segmentsSha256UrlObject.href, + + p2pMediaLoaderInfohashes: VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(playlistUrlObject.href, files), + p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, + videoId: video.id, + + tagAPObject: playlistUrlObject.tag + } + + attributes.push(attribute) + } + + return attributes +} + +function getLiveAttributesFromObject (video: MVideoId, videoObject: VideoObject) { + return { + saveReplay: videoObject.liveSaveReplay, + permanentLive: videoObject.permanentLive, + latencyMode: videoObject.latencyMode, + videoId: video.id + } +} + +function getCaptionAttributesFromObject (video: MVideoId, videoObject: VideoObject) { + return videoObject.subtitleLanguage.map(c => ({ + videoId: video.id, + filename: VideoCaptionModel.generateCaptionName(c.identifier), + language: c.identifier, + fileUrl: c.url + })) +} + +function getStoryboardAttributeFromObject (video: MVideoId, videoObject: VideoObject) { + if (!isArray(videoObject.preview)) return undefined + + const storyboard = videoObject.preview.find(p => p.rel.includes('storyboard')) + if (!storyboard) return undefined + + const url = arrayify(storyboard.url).find(u => u.mediaType === 'image/jpeg') + + return { + filename: generateImageFilename(extname(url.href)), + totalHeight: url.height, + totalWidth: url.width, + spriteHeight: url.tileHeight, + spriteWidth: url.tileWidth, + spriteDuration: getDurationFromActivityStream(url.tileDuration), + fileUrl: url.href, + videoId: video.id + } +} + +function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: VideoObject, to: string[] = []) { + const privacy = to.includes(ACTIVITY_PUB.PUBLIC) + ? VideoPrivacy.PUBLIC + : VideoPrivacy.UNLISTED + + const language = videoObject.language?.identifier + + const category = videoObject.category + ? parseInt(videoObject.category.identifier, 10) + : undefined + + const licence = videoObject.licence + ? parseInt(videoObject.licence.identifier, 10) + : undefined + + const description = videoObject.content || null + const support = videoObject.support || null + + return { + name: videoObject.name, + uuid: videoObject.uuid, + url: videoObject.id, + category, + licence, + language, + description, + support, + nsfw: videoObject.sensitive, + commentsEnabled: videoObject.commentsEnabled, + downloadEnabled: videoObject.downloadEnabled, + waitTranscoding: videoObject.waitTranscoding, + isLive: videoObject.isLiveBroadcast, + state: videoObject.state, + channelId: videoChannel.id, + duration: getDurationFromActivityStream(videoObject.duration), + createdAt: new Date(videoObject.published), + publishedAt: new Date(videoObject.published), + + originallyPublishedAt: videoObject.originallyPublishedAt + ? new Date(videoObject.originallyPublishedAt) + : null, + + inputFileUpdatedAt: videoObject.uploadDate + ? new Date(videoObject.uploadDate) + : null, + + updatedAt: new Date(videoObject.updated), + views: videoObject.views, + remote: true, + privacy + } +} + +// --------------------------------------------------------------------------- + +export { + getThumbnailFromIcons, + getPreviewFromIcons, + + getTagsFromObject, + + getFileAttributesFromUrl, + getStreamingPlaylistAttributesFromObject, + + getLiveAttributesFromObject, + getCaptionAttributesFromObject, + getStoryboardAttributeFromObject, + + getVideoAttributesFromObject +} + +// --------------------------------------------------------------------------- + +function isAPVideoUrlObject (url: any): url is ActivityVideoUrlObject { + const urlMediaType = url.mediaType + + return MIMETYPES.VIDEO.MIMETYPE_EXT[urlMediaType] && urlMediaType.startsWith('video/') +} + +function isAPStreamingPlaylistUrlObject (url: any): url is ActivityPlaylistUrlObject { + return url && url.mediaType === 'application/x-mpegURL' +} + +function isAPPlaylistSegmentHashesUrlObject (tag: any): tag is ActivityPlaylistSegmentHashesObject { + return tag && tag.name === 'sha256' && tag.type === 'Link' && tag.mediaType === 'application/json' +} + +function isAPMagnetUrlObject (url: any): url is ActivityMagnetUrlObject { + return url && url.mediaType === 'application/x-bittorrent;x-scheme-handler/magnet' +} + +function isAPHashTagObject (url: any): url is ActivityHashTagObject { + return url && url.type === 'Hashtag' +} diff --git a/server/server/lib/activitypub/videos/shared/trackers.ts b/server/server/lib/activitypub/videos/shared/trackers.ts new file mode 100644 index 000000000..a7ce6f4ad --- /dev/null +++ b/server/server/lib/activitypub/videos/shared/trackers.ts @@ -0,0 +1,43 @@ +import { Transaction } from 'sequelize' +import { isAPVideoTrackerUrlObject } from '@server/helpers/custom-validators/activitypub/videos.js' +import { isArray } from '@server/helpers/custom-validators/misc.js' +import { REMOTE_SCHEME } from '@server/initializers/constants.js' +import { TrackerModel } from '@server/models/server/tracker.js' +import { MVideo, MVideoWithHost } from '@server/types/models/index.js' +import { ActivityTrackerUrlObject, VideoObject } from '@peertube/peertube-models' +import { buildRemoteVideoBaseUrl } from '../../url.js' + +function getTrackerUrls (object: VideoObject, video: MVideoWithHost) { + let wsFound = false + + const trackers = object.url.filter(u => isAPVideoTrackerUrlObject(u)) + .map((u: ActivityTrackerUrlObject) => { + if (isArray(u.rel) && u.rel.includes('websocket')) wsFound = true + + return u.href + }) + + if (wsFound) return trackers + + return [ + buildRemoteVideoBaseUrl(video, '/tracker/socket', REMOTE_SCHEME.WS), + buildRemoteVideoBaseUrl(video, '/tracker/announce') + ] +} + +async function setVideoTrackers (options: { + video: MVideo + trackers: string[] + transaction: Transaction +}) { + const { video, trackers, transaction } = options + + const trackerInstances = await TrackerModel.findOrCreateTrackers(trackers, transaction) + + await video.$set('Trackers', trackerInstances, { transaction }) +} + +export { + getTrackerUrls, + setVideoTrackers +} diff --git a/server/server/lib/activitypub/videos/shared/url-to-object.ts b/server/server/lib/activitypub/videos/shared/url-to-object.ts new file mode 100644 index 000000000..f27f24971 --- /dev/null +++ b/server/server/lib/activitypub/videos/shared/url-to-object.ts @@ -0,0 +1,25 @@ +import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos.js' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { VideoObject } from '@peertube/peertube-models' +import { fetchAP } from '../../activity.js' +import { checkUrlsSameHost } from '../../url.js' + +const lTags = loggerTagsFactory('ap', 'video') + +async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { + logger.info('Fetching remote video %s.', videoUrl, lTags(videoUrl)) + + const { statusCode, body } = await fetchAP(videoUrl) + + if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { + logger.debug('Remote video JSON is not valid.', { body, ...lTags(videoUrl) }) + + return { statusCode, videoObject: undefined } + } + + return { statusCode, videoObject: body } +} + +export { + fetchRemoteVideo +} diff --git a/server/server/lib/activitypub/videos/shared/video-sync-attributes.ts b/server/server/lib/activitypub/videos/shared/video-sync-attributes.ts new file mode 100644 index 000000000..a07f519fc --- /dev/null +++ b/server/server/lib/activitypub/videos/shared/video-sync-attributes.ts @@ -0,0 +1,101 @@ +import { ActivityPubOrderedCollection, ActivitypubHttpFetcherPayload, VideoObject } from '@peertube/peertube-models' +import { runInReadCommittedTransaction } from '@server/helpers/database-utils.js' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { JobQueue } from '@server/lib/job-queue/index.js' +import { VideoCommentModel } from '@server/models/video/video-comment.js' +import { VideoShareModel } from '@server/models/video/video-share.js' +import { VideoModel } from '@server/models/video/video.js' +import { MVideo } from '@server/types/models/index.js' +import { fetchAP } from '../../activity.js' +import { crawlCollectionPage } from '../../crawl.js' +import { addVideoShares } from '../../share.js' +import { addVideoComments } from '../../video-comments.js' + +const lTags = loggerTagsFactory('ap', 'video') + +export type SyncParam = { + rates: boolean + shares: boolean + comments: boolean + refreshVideo?: boolean +} + +export async function syncVideoExternalAttributes ( + video: MVideo, + fetchedVideo: VideoObject, + syncParam: Pick +) { + logger.info('Adding likes/dislikes/shares/comments of video %s.', video.uuid) + + const ratePromise = updateVideoRates(video, fetchedVideo) + if (syncParam.rates) await ratePromise + + await syncShares(video, fetchedVideo, syncParam.shares) + + await syncComments(video, fetchedVideo, syncParam.comments) +} + +export async function updateVideoRates (video: MVideo, fetchedVideo: VideoObject) { + const [ likes, dislikes ] = await Promise.all([ + getRatesCount('like', video, fetchedVideo), + getRatesCount('dislike', video, fetchedVideo) + ]) + + return runInReadCommittedTransaction(async t => { + await VideoModel.updateRatesOf(video.id, 'like', likes, t) + await VideoModel.updateRatesOf(video.id, 'dislike', dislikes, t) + }) +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +async function getRatesCount (type: 'like' | 'dislike', video: MVideo, fetchedVideo: VideoObject) { + const uri = type === 'like' + ? fetchedVideo.likes + : fetchedVideo.dislikes + + logger.info('Sync %s of video %s', type, video.url) + + const { body } = await fetchAP>(uri) + + if (isNaN(body.totalItems)) { + logger.error('Cannot sync %s of video %s, totalItems is not a number', type, video.url, { body }) + return + } + + return body.totalItems +} + +function syncShares (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) { + const uri = fetchedVideo.shares + + if (!isSync) { + return createJob({ uri, videoId: video.id, type: 'video-shares' }) + } + + const handler = items => addVideoShares(items, video) + const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate) + + return crawlCollectionPage(uri, handler, cleaner) + .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err, rootUrl: uri, ...lTags(video.uuid, video.url) })) +} + +function syncComments (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) { + const uri = fetchedVideo.comments + + if (!isSync) { + return createJob({ uri, videoId: video.id, type: 'video-comments' }) + } + + const handler = items => addVideoComments(items) + const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate) + + return crawlCollectionPage(uri, handler, cleaner) + .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err, rootUrl: uri, ...lTags(video.uuid, video.url) })) +} + +function createJob (payload: ActivitypubHttpFetcherPayload) { + return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) +} diff --git a/server/server/lib/activitypub/videos/updater.ts b/server/server/lib/activitypub/videos/updater.ts new file mode 100644 index 000000000..37bf7411a --- /dev/null +++ b/server/server/lib/activitypub/videos/updater.ts @@ -0,0 +1,186 @@ +import { Transaction } from 'sequelize' +import { VideoObject, VideoPrivacy } from '@peertube/peertube-models' +import { resetSequelizeInstance, runInReadCommittedTransaction } from '@server/helpers/database-utils.js' +import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger.js' +import { Notifier } from '@server/lib/notifier/index.js' +import { PeerTubeSocket } from '@server/lib/peertube-socket.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js' +import { VideoLiveModel } from '@server/models/video/video-live.js' +import { + MActor, + MChannelAccountLight, + MChannelId, + MVideoAccountLightBlacklistAllFiles, + MVideoFullLight +} from '@server/types/models/index.js' +import { APVideoAbstractBuilder, getVideoAttributesFromObject, updateVideoRates } from './shared/index.js' + +export class APVideoUpdater extends APVideoAbstractBuilder { + private readonly wasPrivateVideo: boolean + private readonly wasUnlistedVideo: boolean + + private readonly oldVideoChannel: MChannelAccountLight + + protected lTags: LoggerTagsFn + + constructor ( + protected readonly videoObject: VideoObject, + private readonly video: MVideoAccountLightBlacklistAllFiles + ) { + super() + + this.wasPrivateVideo = this.video.privacy === VideoPrivacy.PRIVATE + this.wasUnlistedVideo = this.video.privacy === VideoPrivacy.UNLISTED + + this.oldVideoChannel = this.video.VideoChannel + + this.lTags = loggerTagsFactory('ap', 'video', 'update', video.uuid, video.url) + } + + async update (overrideTo?: string[]) { + logger.debug( + 'Updating remote video "%s".', this.videoObject.uuid, + { videoObject: this.videoObject, ...this.lTags() } + ) + + const oldInputFileUpdatedAt = this.video.inputFileUpdatedAt + + try { + const channelActor = await this.getOrCreateVideoChannelFromVideoObject() + + const thumbnailModel = await this.setThumbnail(this.video) + + this.checkChannelUpdateOrThrow(channelActor) + + const videoUpdated = await this.updateVideo(channelActor.VideoChannel, undefined, overrideTo) + + if (thumbnailModel) await videoUpdated.addAndSaveThumbnail(thumbnailModel) + + await runInReadCommittedTransaction(async t => { + await this.setWebVideoFiles(videoUpdated, t) + await this.setStreamingPlaylists(videoUpdated, t) + }) + + await Promise.all([ + runInReadCommittedTransaction(t => this.setTags(videoUpdated, t)), + runInReadCommittedTransaction(t => this.setTrackers(videoUpdated, t)), + runInReadCommittedTransaction(t => this.setStoryboard(videoUpdated, t)), + runInReadCommittedTransaction(t => { + return Promise.all([ + this.setPreview(videoUpdated, t), + this.setThumbnail(videoUpdated, t) + ]) + }), + this.setOrDeleteLive(videoUpdated) + ]) + + await runInReadCommittedTransaction(t => this.setCaptions(videoUpdated, t)) + + await autoBlacklistVideoIfNeeded({ + video: videoUpdated, + user: undefined, + isRemote: true, + isNew: false, + isNewFile: oldInputFileUpdatedAt !== videoUpdated.inputFileUpdatedAt, + transaction: undefined + }) + + await updateVideoRates(videoUpdated, this.videoObject) + + // Notify our users? + if (this.wasPrivateVideo || this.wasUnlistedVideo) { + Notifier.Instance.notifyOnNewVideoIfNeeded(videoUpdated) + } + + if (videoUpdated.isLive) { + PeerTubeSocket.Instance.sendVideoLiveNewState(videoUpdated) + } + + Hooks.runAction('action:activity-pub.remote-video.updated', { video: videoUpdated, videoAPObject: this.videoObject }) + + logger.info('Remote video with uuid %s updated', this.videoObject.uuid, this.lTags()) + + return videoUpdated + } catch (err) { + await this.catchUpdateError(err) + } + } + + // Check we can update the channel: we trust the remote server + private checkChannelUpdateOrThrow (newChannelActor: MActor) { + if (!this.oldVideoChannel.Actor.serverId || !newChannelActor.serverId) { + throw new Error('Cannot check old channel/new channel validity because `serverId` is null') + } + + if (this.oldVideoChannel.Actor.serverId !== newChannelActor.serverId) { + throw new Error(`New channel ${newChannelActor.url} is not on the same server than new channel ${this.oldVideoChannel.Actor.url}`) + } + } + + private updateVideo (channel: MChannelId, transaction?: Transaction, overrideTo?: string[]) { + const to = overrideTo || this.videoObject.to + const videoData = getVideoAttributesFromObject(channel, this.videoObject, to) + this.video.name = videoData.name + this.video.uuid = videoData.uuid + this.video.url = videoData.url + this.video.category = videoData.category + this.video.licence = videoData.licence + this.video.language = videoData.language + this.video.description = videoData.description + this.video.support = videoData.support + this.video.nsfw = videoData.nsfw + this.video.commentsEnabled = videoData.commentsEnabled + this.video.downloadEnabled = videoData.downloadEnabled + this.video.waitTranscoding = videoData.waitTranscoding + this.video.state = videoData.state + this.video.duration = videoData.duration + this.video.createdAt = videoData.createdAt + this.video.publishedAt = videoData.publishedAt + this.video.originallyPublishedAt = videoData.originallyPublishedAt + this.video.inputFileUpdatedAt = videoData.inputFileUpdatedAt + this.video.privacy = videoData.privacy + this.video.channelId = videoData.channelId + this.video.views = videoData.views + this.video.isLive = videoData.isLive + + // Ensures we update the updatedAt attribute, even if main attributes did not change + this.video.changed('updatedAt', true) + + return this.video.save({ transaction }) as Promise + } + + private async setCaptions (videoUpdated: MVideoFullLight, t: Transaction) { + await this.insertOrReplaceCaptions(videoUpdated, t) + } + + private async setStoryboard (videoUpdated: MVideoFullLight, t: Transaction) { + await this.insertOrReplaceStoryboard(videoUpdated, t) + } + + private async setOrDeleteLive (videoUpdated: MVideoFullLight, transaction?: Transaction) { + if (!this.video.isLive) return + + if (this.video.isLive) return this.insertOrReplaceLive(videoUpdated, transaction) + + // Delete existing live if it exists + await VideoLiveModel.destroy({ + where: { + videoId: this.video.id + }, + transaction + }) + + videoUpdated.VideoLive = null + } + + private async catchUpdateError (err: Error) { + if (this.video !== undefined) { + await resetSequelizeInstance(this.video) + } + + // This is just a debug because we will retry the insert + logger.debug('Cannot update the remote video.', { err, ...this.lTags() }) + throw err + } +} diff --git a/server/server/lib/actor-follow-health-cache.ts b/server/server/lib/actor-follow-health-cache.ts new file mode 100644 index 000000000..ec5b03d65 --- /dev/null +++ b/server/server/lib/actor-follow-health-cache.ts @@ -0,0 +1,86 @@ +import { ACTOR_FOLLOW_SCORE } from '../initializers/constants.js' +import { logger } from '../helpers/logger.js' + +// Cache follows scores, instead of writing them too often in database +// Keep data in memory, we don't really need Redis here as we don't really care to loose some scores +class ActorFollowHealthCache { + + private static instance: ActorFollowHealthCache + + private pendingFollowsScore: { [ url: string ]: number } = {} + + private pendingBadServer = new Set() + private pendingGoodServer = new Set() + + private readonly badInboxes = new Set() + + private constructor () {} + + static get Instance () { + return this.instance || (this.instance = new this()) + } + + updateActorFollowsHealth (goodInboxes: string[], badInboxes: string[]) { + this.badInboxes.clear() + + if (goodInboxes.length === 0 && badInboxes.length === 0) return + + logger.info( + 'Updating %d good actor follows and %d bad actor follows scores in cache.', + goodInboxes.length, badInboxes.length, { badInboxes } + ) + + for (const goodInbox of goodInboxes) { + if (this.pendingFollowsScore[goodInbox] === undefined) this.pendingFollowsScore[goodInbox] = 0 + + this.pendingFollowsScore[goodInbox] += ACTOR_FOLLOW_SCORE.BONUS + } + + for (const badInbox of badInboxes) { + if (this.pendingFollowsScore[badInbox] === undefined) this.pendingFollowsScore[badInbox] = 0 + + this.pendingFollowsScore[badInbox] += ACTOR_FOLLOW_SCORE.PENALTY + this.badInboxes.add(badInbox) + } + } + + isBadInbox (inboxUrl: string) { + return this.badInboxes.has(inboxUrl) + } + + addBadServerId (serverId: number) { + this.pendingBadServer.add(serverId) + } + + getBadFollowingServerIds () { + return Array.from(this.pendingBadServer) + } + + clearBadFollowingServerIds () { + this.pendingBadServer = new Set() + } + + addGoodServerId (serverId: number) { + this.pendingGoodServer.add(serverId) + } + + getGoodFollowingServerIds () { + return Array.from(this.pendingGoodServer) + } + + clearGoodFollowingServerIds () { + this.pendingGoodServer = new Set() + } + + getPendingFollowsScore () { + return this.pendingFollowsScore + } + + clearPendingFollowsScore () { + this.pendingFollowsScore = {} + } +} + +export { + ActorFollowHealthCache +} diff --git a/server/server/lib/actor-image.ts b/server/server/lib/actor-image.ts new file mode 100644 index 000000000..5b19a740d --- /dev/null +++ b/server/server/lib/actor-image.ts @@ -0,0 +1,14 @@ +import maxBy from 'lodash-es/maxBy.js' + +function getBiggestActorImage (images: T[]) { + const image = maxBy(images, 'width') + + // If width is null, maxBy won't return a value + if (!image) return images[0] + + return image +} + +export { + getBiggestActorImage +} diff --git a/server/server/lib/auth/external-auth.ts b/server/server/lib/auth/external-auth.ts new file mode 100644 index 000000000..e1879aa01 --- /dev/null +++ b/server/server/lib/auth/external-auth.ts @@ -0,0 +1,230 @@ +import { + isUserAdminFlagsValid, + isUserDisplayNameValid, + isUserRoleValid, + isUserUsernameValid, + isUserVideoQuotaDailyValid, + isUserVideoQuotaValid +} from '@server/helpers/custom-validators/users.js' +import { logger } from '@server/helpers/logger.js' +import { generateRandomString } from '@server/helpers/utils.js' +import { PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME } from '@server/initializers/constants.js' +import { PluginManager } from '@server/lib/plugins/plugin-manager.js' +import { OAuthTokenModel } from '@server/models/oauth/oauth-token.js' +import { MUser } from '@server/types/models/index.js' +import { + RegisterServerAuthenticatedResult, + RegisterServerAuthPassOptions, + RegisterServerExternalAuthenticatedResult +} from '@server/types/plugins/register-server-auth.model.js' +import { UserAdminFlag, UserRole } from '@peertube/peertube-models' +import { BypassLogin } from './oauth-model.js' + +export type ExternalUser = + Pick & + { displayName: string } + +// Token is the key, expiration date is the value +const authBypassTokens = new Map() + +async function onExternalUserAuthenticated (options: { + npmName: string + authName: string + authResult: RegisterServerExternalAuthenticatedResult +}) { + const { npmName, authName, authResult } = options + + if (!authResult.req || !authResult.res) { + logger.error('Cannot authenticate external user for auth %s of plugin %s: no req or res are provided.', authName, npmName) + return + } + + const { res } = authResult + + if (!isAuthResultValid(npmName, authName, authResult)) { + res.redirect('/login?externalAuthError=true') + return + } + + logger.info('Generating auth bypass token for %s in auth %s of plugin %s.', authResult.username, authName, npmName) + + const bypassToken = await generateRandomString(32) + + const expires = new Date() + expires.setTime(expires.getTime() + PLUGIN_EXTERNAL_AUTH_TOKEN_LIFETIME) + + const user = buildUserResult(authResult) + authBypassTokens.set(bypassToken, { + expires, + user, + npmName, + authName, + userUpdater: authResult.userUpdater + }) + + // Cleanup expired tokens + const now = new Date() + for (const [ key, value ] of authBypassTokens) { + if (value.expires.getTime() < now.getTime()) { + authBypassTokens.delete(key) + } + } + + res.redirect(`/login?externalAuthToken=${bypassToken}&username=${user.username}`) +} + +async function getAuthNameFromRefreshGrant (refreshToken?: string) { + if (!refreshToken) return undefined + + const tokenModel = await OAuthTokenModel.loadByRefreshToken(refreshToken) + + return tokenModel?.authName +} + +async function getBypassFromPasswordGrant (username: string, password: string): Promise { + const plugins = PluginManager.Instance.getIdAndPassAuths() + const pluginAuths: { npmName?: string, registerAuthOptions: RegisterServerAuthPassOptions }[] = [] + + for (const plugin of plugins) { + const auths = plugin.idAndPassAuths + + for (const auth of auths) { + pluginAuths.push({ + npmName: plugin.npmName, + registerAuthOptions: auth + }) + } + } + + pluginAuths.sort((a, b) => { + const aWeight = a.registerAuthOptions.getWeight() + const bWeight = b.registerAuthOptions.getWeight() + + // DESC weight order + if (aWeight === bWeight) return 0 + if (aWeight < bWeight) return 1 + return -1 + }) + + const loginOptions = { + id: username, + password + } + + for (const pluginAuth of pluginAuths) { + const authOptions = pluginAuth.registerAuthOptions + const authName = authOptions.authName + const npmName = pluginAuth.npmName + + logger.debug( + 'Using auth method %s of plugin %s to login %s with weight %d.', + authName, npmName, loginOptions.id, authOptions.getWeight() + ) + + try { + const loginResult = await authOptions.login(loginOptions) + + if (!loginResult) continue + if (!isAuthResultValid(pluginAuth.npmName, authOptions.authName, loginResult)) continue + + logger.info( + 'Login success with auth method %s of plugin %s for %s.', + authName, npmName, loginOptions.id + ) + + return { + bypass: true, + pluginName: pluginAuth.npmName, + authName: authOptions.authName, + user: buildUserResult(loginResult), + userUpdater: loginResult.userUpdater + } + } catch (err) { + logger.error('Error in auth method %s of plugin %s', authOptions.authName, pluginAuth.npmName, { err }) + } + } + + return undefined +} + +function getBypassFromExternalAuth (username: string, externalAuthToken: string): BypassLogin { + const obj = authBypassTokens.get(externalAuthToken) + if (!obj) throw new Error('Cannot authenticate user with unknown bypass token') + + const { expires, user, authName, npmName } = obj + + const now = new Date() + if (now.getTime() > expires.getTime()) { + throw new Error('Cannot authenticate user with an expired external auth token') + } + + if (user.username !== username) { + throw new Error(`Cannot authenticate user ${user.username} with invalid username ${username}`) + } + + logger.info( + 'Auth success with external auth method %s of plugin %s for %s.', + authName, npmName, user.email + ) + + return { + bypass: true, + pluginName: npmName, + authName, + userUpdater: obj.userUpdater, + user + } +} + +function isAuthResultValid (npmName: string, authName: string, result: RegisterServerAuthenticatedResult) { + const returnError = (field: string) => { + logger.error('Auth method %s of plugin %s did not provide a valid %s.', authName, npmName, field, { [field]: result[field] }) + return false + } + + if (!isUserUsernameValid(result.username)) return returnError('username') + if (!result.email) return returnError('email') + + // Following fields are optional + if (result.role && !isUserRoleValid(result.role)) return returnError('role') + if (result.displayName && !isUserDisplayNameValid(result.displayName)) return returnError('displayName') + if (result.adminFlags && !isUserAdminFlagsValid(result.adminFlags)) return returnError('adminFlags') + if (result.videoQuota && !isUserVideoQuotaValid(result.videoQuota + '')) return returnError('videoQuota') + if (result.videoQuotaDaily && !isUserVideoQuotaDailyValid(result.videoQuotaDaily + '')) return returnError('videoQuotaDaily') + + if (result.userUpdater && typeof result.userUpdater !== 'function') { + logger.error('Auth method %s of plugin %s did not provide a valid user updater function.', authName, npmName) + return false + } + + return true +} + +function buildUserResult (pluginResult: RegisterServerAuthenticatedResult) { + return { + username: pluginResult.username, + email: pluginResult.email, + role: pluginResult.role ?? UserRole.USER, + displayName: pluginResult.displayName || pluginResult.username, + + adminFlags: pluginResult.adminFlags ?? UserAdminFlag.NONE, + + videoQuota: pluginResult.videoQuota, + videoQuotaDaily: pluginResult.videoQuotaDaily + } +} + +// --------------------------------------------------------------------------- + +export { + onExternalUserAuthenticated, + getBypassFromExternalAuth, + getAuthNameFromRefreshGrant, + getBypassFromPasswordGrant +} diff --git a/server/server/lib/auth/oauth-model.ts b/server/server/lib/auth/oauth-model.ts new file mode 100644 index 000000000..26a9f8996 --- /dev/null +++ b/server/server/lib/auth/oauth-model.ts @@ -0,0 +1,294 @@ +import express from 'express' +import { AccessDeniedError } from '@node-oauth/oauth2-server' +import { PluginManager } from '@server/lib/plugins/plugin-manager.js' +import { AccountModel } from '@server/models/account/account.js' +import { AuthenticatedResultUpdaterFieldName, RegisterServerAuthenticatedResult } from '@server/types/index.js' +import { MOAuthClient } from '@server/types/models/index.js' +import { MOAuthTokenUser } from '@server/types/models/oauth/oauth-token.js' +import { MUser, MUserDefault } from '@server/types/models/user/user.js' +import { pick } from '@peertube/peertube-core-utils' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { logger } from '../../helpers/logger.js' +import { CONFIG } from '../../initializers/config.js' +import { OAuthClientModel } from '../../models/oauth/oauth-client.js' +import { OAuthTokenModel } from '../../models/oauth/oauth-token.js' +import { UserModel } from '../../models/user/user.js' +import { findAvailableLocalActorName } from '../local-actor.js' +import { buildUser, createUserAccountAndChannelAndPlaylist } from '../user.js' +import { ExternalUser } from './external-auth.js' +import { TokensCache } from './tokens-cache.js' + +type TokenInfo = { + accessToken: string + refreshToken: string + accessTokenExpiresAt: Date + refreshTokenExpiresAt: Date +} + +export type BypassLogin = { + bypass: boolean + pluginName: string + authName?: string + user: ExternalUser + userUpdater: RegisterServerAuthenticatedResult['userUpdater'] +} + +async function getAccessToken (bearerToken: string) { + logger.debug('Getting access token.') + + if (!bearerToken) return undefined + + let tokenModel: MOAuthTokenUser + + if (TokensCache.Instance.hasToken(bearerToken)) { + tokenModel = TokensCache.Instance.getByToken(bearerToken) + } else { + tokenModel = await OAuthTokenModel.getByTokenAndPopulateUser(bearerToken) + + if (tokenModel) TokensCache.Instance.setToken(tokenModel) + } + + if (!tokenModel) return undefined + + if (tokenModel.User.pluginAuth) { + const valid = await PluginManager.Instance.isTokenValid(tokenModel, 'access') + + if (valid !== true) return undefined + } + + return tokenModel +} + +function getClient (clientId: string, clientSecret: string) { + logger.debug('Getting Client (clientId: ' + clientId + ', clientSecret: ' + clientSecret + ').') + + return OAuthClientModel.getByIdAndSecret(clientId, clientSecret) +} + +async function getRefreshToken (refreshToken: string) { + logger.debug('Getting RefreshToken (refreshToken: ' + refreshToken + ').') + + const tokenInfo = await OAuthTokenModel.getByRefreshTokenAndPopulateClient(refreshToken) + if (!tokenInfo) return undefined + + const tokenModel = tokenInfo.token + + if (tokenModel.User.pluginAuth) { + const valid = await PluginManager.Instance.isTokenValid(tokenModel, 'refresh') + + if (valid !== true) return undefined + } + + return tokenInfo +} + +async function getUser (usernameOrEmail?: string, password?: string, bypassLogin?: BypassLogin) { + // Special treatment coming from a plugin + if (bypassLogin && bypassLogin.bypass === true) { + logger.info('Bypassing oauth login by plugin %s.', bypassLogin.pluginName) + + let user = await UserModel.loadByEmail(bypassLogin.user.email) + + if (!user) user = await createUserFromExternal(bypassLogin.pluginName, bypassLogin.user) + else user = await updateUserFromExternal(user, bypassLogin.user, bypassLogin.userUpdater) + + // Cannot create a user + if (!user) throw new AccessDeniedError('Cannot create such user: an actor with that name already exists.') + + // If the user does not belongs to a plugin, it was created before its installation + // Then we just go through a regular login process + if (user.pluginAuth !== null) { + // This user does not belong to this plugin, skip it + if (user.pluginAuth !== bypassLogin.pluginName) { + logger.info( + 'Cannot bypass oauth login by plugin %s because %s has another plugin auth method (%s).', + bypassLogin.pluginName, bypassLogin.user.email, user.pluginAuth + ) + + return null + } + + checkUserValidityOrThrow(user) + + return user + } + } + + logger.debug('Getting User (username/email: ' + usernameOrEmail + ', password: ******).') + + const user = await UserModel.loadByUsernameOrEmail(usernameOrEmail) + + // If we don't find the user, or if the user belongs to a plugin + if (!user || user.pluginAuth !== null || !password) return null + + const passwordMatch = await user.isPasswordMatch(password) + if (passwordMatch !== true) return null + + checkUserValidityOrThrow(user) + + if (CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION && user.emailVerified === false) { + throw new AccessDeniedError('User email is not verified.') + } + + return user +} + +async function revokeToken ( + tokenInfo: { refreshToken: string }, + options: { + req?: express.Request + explicitLogout?: boolean + } = {} +): Promise<{ success: boolean, redirectUrl?: string }> { + const { req, explicitLogout } = options + + const token = await OAuthTokenModel.getByRefreshTokenAndPopulateUser(tokenInfo.refreshToken) + + if (token) { + let redirectUrl: string + + if (explicitLogout === true && token.User.pluginAuth && token.authName) { + redirectUrl = await PluginManager.Instance.onLogout(token.User.pluginAuth, token.authName, token.User, req) + } + + TokensCache.Instance.clearCacheByToken(token.accessToken) + + token.destroy() + .catch(err => logger.error('Cannot destroy token when revoking token.', { err })) + + return { success: true, redirectUrl } + } + + return { success: false } +} + +async function saveToken ( + token: TokenInfo, + client: MOAuthClient, + user: MUser, + options: { + refreshTokenAuthName?: string + bypassLogin?: BypassLogin + } = {} +) { + const { refreshTokenAuthName, bypassLogin } = options + let authName: string = null + + if (bypassLogin?.bypass === true) { + authName = bypassLogin.authName + } else if (refreshTokenAuthName) { + authName = refreshTokenAuthName + } + + logger.debug('Saving token ' + token.accessToken + ' for client ' + client.id + ' and user ' + user.id + '.') + + const tokenToCreate = { + accessToken: token.accessToken, + accessTokenExpiresAt: token.accessTokenExpiresAt, + refreshToken: token.refreshToken, + refreshTokenExpiresAt: token.refreshTokenExpiresAt, + authName, + oAuthClientId: client.id, + userId: user.id + } + + const tokenCreated = await OAuthTokenModel.create(tokenToCreate) + + user.lastLoginDate = new Date() + await user.save() + + return { + accessToken: tokenCreated.accessToken, + accessTokenExpiresAt: tokenCreated.accessTokenExpiresAt, + refreshToken: tokenCreated.refreshToken, + refreshTokenExpiresAt: tokenCreated.refreshTokenExpiresAt, + client, + user, + accessTokenExpiresIn: buildExpiresIn(tokenCreated.accessTokenExpiresAt), + refreshTokenExpiresIn: buildExpiresIn(tokenCreated.refreshTokenExpiresAt) + } +} + +export { + getAccessToken, + getClient, + getRefreshToken, + getUser, + revokeToken, + saveToken +} + +// --------------------------------------------------------------------------- + +async function createUserFromExternal (pluginAuth: string, userOptions: ExternalUser) { + const username = await findAvailableLocalActorName(userOptions.username) + + const userToCreate = buildUser({ + ...pick(userOptions, [ 'email', 'role', 'adminFlags', 'videoQuota', 'videoQuotaDaily' ]), + + username, + emailVerified: null, + password: null, + pluginAuth + }) + + const { user } = await createUserAccountAndChannelAndPlaylist({ + userToCreate, + userDisplayName: userOptions.displayName + }) + + return user +} + +async function updateUserFromExternal ( + user: MUserDefault, + userOptions: ExternalUser, + userUpdater: RegisterServerAuthenticatedResult['userUpdater'] +) { + if (!userUpdater) return user + + { + type UserAttributeKeys = keyof AttributesOnly + const mappingKeys: { [ id in UserAttributeKeys ]?: AuthenticatedResultUpdaterFieldName } = { + role: 'role', + adminFlags: 'adminFlags', + videoQuota: 'videoQuota', + videoQuotaDaily: 'videoQuotaDaily' + } + + for (const modelKey of Object.keys(mappingKeys)) { + const pluginOptionKey = mappingKeys[modelKey] + + const newValue = userUpdater({ fieldName: pluginOptionKey, currentValue: user[modelKey], newValue: userOptions[pluginOptionKey] }) + user.set(modelKey, newValue) + } + } + + { + type AccountAttributeKeys = keyof Partial> + const mappingKeys: { [ id in AccountAttributeKeys ]?: AuthenticatedResultUpdaterFieldName } = { + name: 'displayName' + } + + for (const modelKey of Object.keys(mappingKeys)) { + const optionKey = mappingKeys[modelKey] + + const newValue = userUpdater({ fieldName: optionKey, currentValue: user.Account[modelKey], newValue: userOptions[optionKey] }) + user.Account.set(modelKey, newValue) + } + } + + logger.debug('Updated user %s with plugin userUpdated function.', user.email, { user, userOptions }) + + user.Account = await user.Account.save() + + return user.save() +} + +function checkUserValidityOrThrow (user: MUser) { + if (user.blocked) throw new AccessDeniedError('User is blocked.') +} + +function buildExpiresIn (expiresAt: Date) { + return Math.floor((expiresAt.getTime() - new Date().getTime()) / 1000) +} diff --git a/server/server/lib/auth/oauth.ts b/server/server/lib/auth/oauth.ts new file mode 100644 index 000000000..d91bfc37a --- /dev/null +++ b/server/server/lib/auth/oauth.ts @@ -0,0 +1,230 @@ +import express from 'express' +import OAuth2Server, { + InvalidClientError, + InvalidGrantError, + InvalidRequestError, + Request, + Response, + UnauthorizedClientError, + UnsupportedGrantTypeError +} from '@node-oauth/oauth2-server' +import { randomBytesPromise } from '@server/helpers/core-utils.js' +import { isOTPValid } from '@server/helpers/otp.js' +import { CONFIG } from '@server/initializers/config.js' +import { UserRegistrationModel } from '@server/models/user/user-registration.js' +import { MOAuthClient } from '@server/types/models/index.js' +import { sha1 } from '@peertube/peertube-node-utils' +import { HttpStatusCode, ServerErrorCode, UserRegistrationState } from '@peertube/peertube-models' +import { OTP } from '../../initializers/constants.js' +import { BypassLogin, getAccessToken, getClient, getRefreshToken, getUser, revokeToken, saveToken } from './oauth-model.js' + +class MissingTwoFactorError extends Error { + code = HttpStatusCode.UNAUTHORIZED_401 + name = ServerErrorCode.MISSING_TWO_FACTOR +} + +class InvalidTwoFactorError extends Error { + code = HttpStatusCode.BAD_REQUEST_400 + name = ServerErrorCode.INVALID_TWO_FACTOR +} + +class RegistrationWaitingForApproval extends Error { + code = HttpStatusCode.BAD_REQUEST_400 + name = ServerErrorCode.ACCOUNT_WAITING_FOR_APPROVAL +} + +class RegistrationApprovalRejected extends Error { + code = HttpStatusCode.BAD_REQUEST_400 + name = ServerErrorCode.ACCOUNT_APPROVAL_REJECTED +} + +/** + * + * Reimplement some functions of OAuth2Server to inject external auth methods + * + */ +const oAuthServer = new OAuth2Server({ + // Wants seconds + accessTokenLifetime: CONFIG.OAUTH2.TOKEN_LIFETIME.ACCESS_TOKEN / 1000, + refreshTokenLifetime: CONFIG.OAUTH2.TOKEN_LIFETIME.REFRESH_TOKEN / 1000, + + // See https://github.com/oauthjs/node-oauth2-server/wiki/Model-specification for the model specifications + model: { + getAccessToken, + getClient, + getRefreshToken, + getUser, + revokeToken, + saveToken + } as any // FIXME: typings +}) + +// --------------------------------------------------------------------------- + +async function handleOAuthToken (req: express.Request, options: { refreshTokenAuthName?: string, bypassLogin?: BypassLogin }) { + const request = new Request(req) + const { refreshTokenAuthName, bypassLogin } = options + + if (request.method !== 'POST') { + throw new InvalidRequestError('Invalid request: method must be POST') + } + + if (!request.is([ 'application/x-www-form-urlencoded' ])) { + throw new InvalidRequestError('Invalid request: content must be application/x-www-form-urlencoded') + } + + const clientId = request.body.client_id + const clientSecret = request.body.client_secret + + if (!clientId || !clientSecret) { + throw new InvalidClientError('Invalid client: cannot retrieve client credentials') + } + + const client = await getClient(clientId, clientSecret) + if (!client) { + throw new InvalidClientError('Invalid client: client is invalid') + } + + const grantType = request.body.grant_type + if (!grantType) { + throw new InvalidRequestError('Missing parameter: `grant_type`') + } + + if (![ 'password', 'refresh_token' ].includes(grantType)) { + throw new UnsupportedGrantTypeError('Unsupported grant type: `grant_type` is invalid') + } + + if (!client.grants.includes(grantType)) { + throw new UnauthorizedClientError('Unauthorized client: `grant_type` is invalid') + } + + if (grantType === 'password') { + return handlePasswordGrant({ + request, + client, + bypassLogin + }) + } + + return handleRefreshGrant({ + request, + client, + refreshTokenAuthName + }) +} + +function handleOAuthAuthenticate ( + req: express.Request, + res: express.Response +) { + return oAuthServer.authenticate(new Request(req), new Response(res)) +} + +export { + MissingTwoFactorError, + InvalidTwoFactorError, + + handleOAuthToken, + handleOAuthAuthenticate +} + +// --------------------------------------------------------------------------- + +async function handlePasswordGrant (options: { + request: Request + client: MOAuthClient + bypassLogin?: BypassLogin +}) { + const { request, client, bypassLogin } = options + + if (!request.body.username) { + throw new InvalidRequestError('Missing parameter: `username`') + } + + if (!bypassLogin && !request.body.password) { + throw new InvalidRequestError('Missing parameter: `password`') + } + + const user = await getUser(request.body.username, request.body.password, bypassLogin) + if (!user) { + const registration = await UserRegistrationModel.loadByEmailOrUsername(request.body.username) + + if (registration?.state === UserRegistrationState.REJECTED) { + throw new RegistrationApprovalRejected('Registration approval for this account has been rejected') + } else if (registration?.state === UserRegistrationState.PENDING) { + throw new RegistrationWaitingForApproval('Registration for this account is awaiting approval') + } + + throw new InvalidGrantError('Invalid grant: user credentials are invalid') + } + + if (user.otpSecret) { + if (!request.headers[OTP.HEADER_NAME]) { + throw new MissingTwoFactorError('Missing two factor header') + } + + if (await isOTPValid({ encryptedSecret: user.otpSecret, token: request.headers[OTP.HEADER_NAME] }) !== true) { + throw new InvalidTwoFactorError('Invalid two factor header') + } + } + + const token = await buildToken() + + return saveToken(token, client, user, { bypassLogin }) +} + +async function handleRefreshGrant (options: { + request: Request + client: MOAuthClient + refreshTokenAuthName: string +}) { + const { request, client, refreshTokenAuthName } = options + + if (!request.body.refresh_token) { + throw new InvalidRequestError('Missing parameter: `refresh_token`') + } + + const refreshToken = await getRefreshToken(request.body.refresh_token) + + if (!refreshToken) { + throw new InvalidGrantError('Invalid grant: refresh token is invalid') + } + + if (refreshToken.client.id !== client.id) { + throw new InvalidGrantError('Invalid grant: refresh token is invalid') + } + + if (refreshToken.refreshTokenExpiresAt && refreshToken.refreshTokenExpiresAt < new Date()) { + throw new InvalidGrantError('Invalid grant: refresh token has expired') + } + + await revokeToken({ refreshToken: refreshToken.refreshToken }) + + const token = await buildToken() + + return saveToken(token, client, refreshToken.user, { refreshTokenAuthName }) +} + +function generateRandomToken () { + return randomBytesPromise(256) + .then(buffer => sha1(buffer)) +} + +function getTokenExpiresAt (type: 'access' | 'refresh') { + const lifetime = type === 'access' + ? CONFIG.OAUTH2.TOKEN_LIFETIME.ACCESS_TOKEN + : CONFIG.OAUTH2.TOKEN_LIFETIME.REFRESH_TOKEN + + return new Date(Date.now() + lifetime) +} + +async function buildToken () { + const [ accessToken, refreshToken ] = await Promise.all([ generateRandomToken(), generateRandomToken() ]) + + return { + accessToken, + refreshToken, + accessTokenExpiresAt: getTokenExpiresAt('access'), + refreshTokenExpiresAt: getTokenExpiresAt('refresh') + } +} diff --git a/server/server/lib/auth/tokens-cache.ts b/server/server/lib/auth/tokens-cache.ts new file mode 100644 index 000000000..45d62bcaf --- /dev/null +++ b/server/server/lib/auth/tokens-cache.ts @@ -0,0 +1,52 @@ +import { LRUCache } from 'lru-cache' +import { MOAuthTokenUser } from '@server/types/models/index.js' +import { LRU_CACHE } from '../../initializers/constants.js' + +export class TokensCache { + + private static instance: TokensCache + + private readonly accessTokenCache = new LRUCache({ max: LRU_CACHE.USER_TOKENS.MAX_SIZE }) + private readonly userHavingToken = new LRUCache({ max: LRU_CACHE.USER_TOKENS.MAX_SIZE }) + + private constructor () { } + + static get Instance () { + return this.instance || (this.instance = new this()) + } + + hasToken (token: string) { + return this.accessTokenCache.has(token) + } + + getByToken (token: string) { + return this.accessTokenCache.get(token) + } + + setToken (token: MOAuthTokenUser) { + this.accessTokenCache.set(token.accessToken, token) + this.userHavingToken.set(token.userId, token.accessToken) + } + + deleteUserToken (userId: number) { + this.clearCacheByUserId(userId) + } + + clearCacheByUserId (userId: number) { + const token = this.userHavingToken.get(userId) + + if (token !== undefined) { + this.accessTokenCache.delete(token) + this.userHavingToken.delete(userId) + } + } + + clearCacheByToken (token: string) { + const tokenModel = this.accessTokenCache.get(token) + + if (tokenModel !== undefined) { + this.userHavingToken.delete(tokenModel.userId) + this.accessTokenCache.delete(token) + } + } +} diff --git a/server/server/lib/blocklist.ts b/server/server/lib/blocklist.ts new file mode 100644 index 000000000..7491d92d7 --- /dev/null +++ b/server/server/lib/blocklist.ts @@ -0,0 +1,62 @@ +import { sequelizeTypescript } from '@server/initializers/database.js' +import { getServerActor } from '@server/models/application/application.js' +import { MAccountBlocklist, MAccountId, MAccountHost, MServerBlocklist } from '@server/types/models/index.js' +import { AccountBlocklistModel } from '../models/account/account-blocklist.js' +import { ServerBlocklistModel } from '../models/server/server-blocklist.js' + +function addAccountInBlocklist (byAccountId: number, targetAccountId: number) { + return sequelizeTypescript.transaction(async t => { + return AccountBlocklistModel.upsert({ + accountId: byAccountId, + targetAccountId + }, { transaction: t }) + }) +} + +function addServerInBlocklist (byAccountId: number, targetServerId: number) { + return sequelizeTypescript.transaction(async t => { + return ServerBlocklistModel.upsert({ + accountId: byAccountId, + targetServerId + }, { transaction: t }) + }) +} + +function removeAccountFromBlocklist (accountBlock: MAccountBlocklist) { + return sequelizeTypescript.transaction(async t => { + return accountBlock.destroy({ transaction: t }) + }) +} + +function removeServerFromBlocklist (serverBlock: MServerBlocklist) { + return sequelizeTypescript.transaction(async t => { + return serverBlock.destroy({ transaction: t }) + }) +} + +async function isBlockedByServerOrAccount (targetAccount: MAccountHost, userAccount?: MAccountId) { + const serverAccountId = (await getServerActor()).Account.id + const sourceAccounts = [ serverAccountId ] + + if (userAccount) sourceAccounts.push(userAccount.id) + + const accountMutedHash = await AccountBlocklistModel.isAccountMutedByAccounts(sourceAccounts, targetAccount.id) + if (accountMutedHash[serverAccountId] || (userAccount && accountMutedHash[userAccount.id])) { + return true + } + + const instanceMutedHash = await ServerBlocklistModel.isServerMutedByAccounts(sourceAccounts, targetAccount.Actor.serverId) + if (instanceMutedHash[serverAccountId] || (userAccount && instanceMutedHash[userAccount.id])) { + return true + } + + return false +} + +export { + addAccountInBlocklist, + addServerInBlocklist, + removeAccountFromBlocklist, + removeServerFromBlocklist, + isBlockedByServerOrAccount +} diff --git a/server/server/lib/client-html.ts b/server/server/lib/client-html.ts new file mode 100644 index 000000000..f00b05fab --- /dev/null +++ b/server/server/lib/client-html.ts @@ -0,0 +1,619 @@ +import { buildFileLocale, escapeHTML, getDefaultLocale, is18nLocale, POSSIBLE_LOCALES } from '@peertube/peertube-core-utils' +import { HTMLServerConfig, HttpStatusCode, VideoPlaylistPrivacy, VideoPrivacy } from '@peertube/peertube-models' +import { isTestOrDevInstance, root, sha256 } from '@peertube/peertube-node-utils' +import { toCompleteUUID } from '@server/helpers/custom-validators/misc.js' +import { mdToOneLinePlainText } from '@server/helpers/markdown.js' +import { ActorImageModel } from '@server/models/actor/actor-image.js' +import express from 'express' +import { pathExists } from 'fs-extra/esm' +import { readFile } from 'fs/promises' +import truncate from 'lodash-es/truncate.js' +import { join } from 'path' +import validator from 'validator' +import { logger } from '../helpers/logger.js' +import { CONFIG } from '../initializers/config.js' +import { + ACCEPT_HEADERS, + CUSTOM_HTML_TAG_COMMENTS, + EMBED_SIZE, + FILES_CONTENT_HASH, + PLUGIN_GLOBAL_CSS_PATH, + WEBSERVER +} from '../initializers/constants.js' +import { AccountModel } from '../models/account/account.js' +import { VideoChannelModel } from '../models/video/video-channel.js' +import { VideoPlaylistModel } from '../models/video/video-playlist.js' +import { VideoModel } from '../models/video/video.js' +import { MAccountHost, MChannelHost, MVideo, MVideoPlaylist } from '../types/models/index.js' +import { getActivityStreamDuration } from './activitypub/activity.js' +import { getBiggestActorImage } from './actor-image.js' +import { Hooks } from './plugins/hooks.js' +import { ServerConfigManager } from './server-config-manager.js' +import { isVideoInPrivateDirectory } from './video-privacy.js' + +type Tags = { + ogType: string + twitterCard: 'player' | 'summary' | 'summary_large_image' + schemaType: string + + list?: { + numberOfItems: number + } + + escapedSiteName: string + escapedTitle: string + escapedTruncatedDescription: string + + url: string + originUrl: string + + disallowIndexation?: boolean + + embed?: { + url: string + createdAt: string + duration?: string + views?: number + } + + image: { + url: string + width?: number + height?: number + } +} + +type HookContext = { + video?: MVideo + playlist?: MVideoPlaylist +} + +class ClientHtml { + + private static htmlCache: { [path: string]: string } = {} + + static invalidCache () { + logger.info('Cleaning HTML cache.') + + ClientHtml.htmlCache = {} + } + + static async getDefaultHTMLPage (req: express.Request, res: express.Response, paramLang?: string) { + const html = paramLang + ? await ClientHtml.getIndexHTML(req, res, paramLang) + : await ClientHtml.getIndexHTML(req, res) + + let customHtml = ClientHtml.addTitleTag(html) + customHtml = ClientHtml.addDescriptionTag(customHtml) + + return customHtml + } + + static async getWatchHTMLPage (videoIdArg: string, req: express.Request, res: express.Response) { + const videoId = toCompleteUUID(videoIdArg) + + // Let Angular application handle errors + if (!validator.default.isInt(videoId) && !validator.default.isUUID(videoId, 4)) { + res.status(HttpStatusCode.NOT_FOUND_404) + return ClientHtml.getIndexHTML(req, res) + } + + const [ html, video ] = await Promise.all([ + ClientHtml.getIndexHTML(req, res), + VideoModel.loadWithBlacklist(videoId) + ]) + + // Let Angular application handle errors + if (!video || isVideoInPrivateDirectory(video.privacy) || video.VideoBlacklist) { + res.status(HttpStatusCode.NOT_FOUND_404) + return html + } + const escapedTruncatedDescription = buildEscapedTruncatedDescription(video.description) + + let customHtml = ClientHtml.addTitleTag(html, video.name) + customHtml = ClientHtml.addDescriptionTag(customHtml, escapedTruncatedDescription) + + const url = WEBSERVER.URL + video.getWatchStaticPath() + const originUrl = video.url + const title = video.name + const siteName = CONFIG.INSTANCE.NAME + + const image = { + url: WEBSERVER.URL + video.getPreviewStaticPath() + } + + const embed = { + url: WEBSERVER.URL + video.getEmbedStaticPath(), + createdAt: video.createdAt.toISOString(), + duration: getActivityStreamDuration(video.duration), + views: video.views + } + + const ogType = 'video' + const twitterCard = CONFIG.SERVICES.TWITTER.WHITELISTED ? 'player' : 'summary_large_image' + const schemaType = 'VideoObject' + + customHtml = await ClientHtml.addTags(customHtml, { + url, + originUrl, + escapedSiteName: escapeHTML(siteName), + escapedTitle: escapeHTML(title), + escapedTruncatedDescription, + disallowIndexation: video.privacy !== VideoPrivacy.PUBLIC, + image, + embed, + ogType, + twitterCard, + schemaType + }, { video }) + + return customHtml + } + + static async getWatchPlaylistHTMLPage (videoPlaylistIdArg: string, req: express.Request, res: express.Response) { + const videoPlaylistId = toCompleteUUID(videoPlaylistIdArg) + + // Let Angular application handle errors + if (!validator.default.isInt(videoPlaylistId) && !validator.default.isUUID(videoPlaylistId, 4)) { + res.status(HttpStatusCode.NOT_FOUND_404) + return ClientHtml.getIndexHTML(req, res) + } + + const [ html, videoPlaylist ] = await Promise.all([ + ClientHtml.getIndexHTML(req, res), + VideoPlaylistModel.loadWithAccountAndChannel(videoPlaylistId, null) + ]) + + // Let Angular application handle errors + if (!videoPlaylist || videoPlaylist.privacy === VideoPlaylistPrivacy.PRIVATE) { + res.status(HttpStatusCode.NOT_FOUND_404) + return html + } + + const escapedTruncatedDescription = buildEscapedTruncatedDescription(videoPlaylist.description) + + let customHtml = ClientHtml.addTitleTag(html, videoPlaylist.name) + customHtml = ClientHtml.addDescriptionTag(customHtml, escapedTruncatedDescription) + + const url = WEBSERVER.URL + videoPlaylist.getWatchStaticPath() + const originUrl = videoPlaylist.url + const title = videoPlaylist.name + const siteName = CONFIG.INSTANCE.NAME + + const image = { + url: videoPlaylist.getThumbnailUrl() + } + + const embed = { + url: WEBSERVER.URL + videoPlaylist.getEmbedStaticPath(), + createdAt: videoPlaylist.createdAt.toISOString() + } + + const list = { + numberOfItems: videoPlaylist.get('videosLength') as number + } + + const ogType = 'video' + const twitterCard = CONFIG.SERVICES.TWITTER.WHITELISTED ? 'player' : 'summary' + const schemaType = 'ItemList' + + customHtml = await ClientHtml.addTags(customHtml, { + url, + originUrl, + escapedSiteName: escapeHTML(siteName), + escapedTitle: escapeHTML(title), + escapedTruncatedDescription, + disallowIndexation: videoPlaylist.privacy !== VideoPlaylistPrivacy.PUBLIC, + embed, + image, + list, + ogType, + twitterCard, + schemaType + }, { playlist: videoPlaylist }) + + return customHtml + } + + static async getAccountHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) { + const accountModelPromise = AccountModel.loadByNameWithHost(nameWithHost) + return this.getAccountOrChannelHTMLPage(() => accountModelPromise, req, res) + } + + static async getVideoChannelHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) { + const videoChannelModelPromise = VideoChannelModel.loadByNameWithHostAndPopulateAccount(nameWithHost) + return this.getAccountOrChannelHTMLPage(() => videoChannelModelPromise, req, res) + } + + static async getActorHTMLPage (nameWithHost: string, req: express.Request, res: express.Response) { + const [ account, channel ] = await Promise.all([ + AccountModel.loadByNameWithHost(nameWithHost), + VideoChannelModel.loadByNameWithHostAndPopulateAccount(nameWithHost) + ]) + + return this.getAccountOrChannelHTMLPage(() => Promise.resolve(account || channel), req, res) + } + + static async getEmbedHTML () { + const path = ClientHtml.getEmbedPath() + + // Disable HTML cache in dev mode because webpack can regenerate JS files + if (!isTestOrDevInstance() && ClientHtml.htmlCache[path]) { + return ClientHtml.htmlCache[path] + } + + const buffer = await readFile(path) + const serverConfig = await ServerConfigManager.Instance.getHTMLServerConfig() + + let html = buffer.toString() + html = await ClientHtml.addAsyncPluginCSS(html) + html = ClientHtml.addCustomCSS(html) + html = ClientHtml.addTitleTag(html) + html = ClientHtml.addDescriptionTag(html) + html = ClientHtml.addServerConfig(html, serverConfig) + + ClientHtml.htmlCache[path] = html + + return html + } + + private static async getAccountOrChannelHTMLPage ( + loader: () => Promise, + req: express.Request, + res: express.Response + ) { + const [ html, entity ] = await Promise.all([ + ClientHtml.getIndexHTML(req, res), + loader() + ]) + + // Let Angular application handle errors + if (!entity) { + res.status(HttpStatusCode.NOT_FOUND_404) + return ClientHtml.getIndexHTML(req, res) + } + + const escapedTruncatedDescription = buildEscapedTruncatedDescription(entity.description) + + let customHtml = ClientHtml.addTitleTag(html, entity.getDisplayName()) + customHtml = ClientHtml.addDescriptionTag(customHtml, escapedTruncatedDescription) + + const url = entity.getClientUrl() + const originUrl = entity.Actor.url + const siteName = CONFIG.INSTANCE.NAME + const title = entity.getDisplayName() + + const avatar = getBiggestActorImage(entity.Actor.Avatars) + const image = { + url: ActorImageModel.getImageUrl(avatar), + width: avatar?.width, + height: avatar?.height + } + + const ogType = 'website' + const twitterCard = 'summary' + const schemaType = 'ProfilePage' + + customHtml = await ClientHtml.addTags(customHtml, { + url, + originUrl, + escapedTitle: escapeHTML(title), + escapedSiteName: escapeHTML(siteName), + escapedTruncatedDescription, + image, + ogType, + twitterCard, + schemaType, + disallowIndexation: !entity.Actor.isOwned() + }, {}) + + return customHtml + } + + private static async getIndexHTML (req: express.Request, res: express.Response, paramLang?: string) { + const path = ClientHtml.getIndexPath(req, res, paramLang) + if (ClientHtml.htmlCache[path]) return ClientHtml.htmlCache[path] + + const buffer = await readFile(path) + const serverConfig = await ServerConfigManager.Instance.getHTMLServerConfig() + + let html = buffer.toString() + + html = ClientHtml.addManifestContentHash(html) + html = ClientHtml.addFaviconContentHash(html) + html = ClientHtml.addLogoContentHash(html) + html = ClientHtml.addCustomCSS(html) + html = ClientHtml.addServerConfig(html, serverConfig) + html = await ClientHtml.addAsyncPluginCSS(html) + + ClientHtml.htmlCache[path] = html + + return html + } + + private static getIndexPath (req: express.Request, res: express.Response, paramLang: string) { + let lang: string + + // Check param lang validity + if (paramLang && is18nLocale(paramLang)) { + lang = paramLang + + // Save locale in cookies + res.cookie('clientLanguage', lang, { + secure: WEBSERVER.SCHEME === 'https', + sameSite: 'none', + maxAge: 1000 * 3600 * 24 * 90 // 3 months + }) + + } else if (req.cookies.clientLanguage && is18nLocale(req.cookies.clientLanguage)) { + lang = req.cookies.clientLanguage + } else { + lang = req.acceptsLanguages(POSSIBLE_LOCALES) || getDefaultLocale() + } + + logger.debug( + 'Serving %s HTML language', buildFileLocale(lang), + { cookie: req.cookies?.clientLanguage, paramLang, acceptLanguage: req.headers['accept-language'] } + ) + + return join(root(), 'client', 'dist', buildFileLocale(lang), 'index.html') + } + + private static getEmbedPath () { + return join(root(), 'client', 'dist', 'standalone', 'videos', 'embed.html') + } + + private static addManifestContentHash (htmlStringPage: string) { + return htmlStringPage.replace('[manifestContentHash]', FILES_CONTENT_HASH.MANIFEST) + } + + private static addFaviconContentHash (htmlStringPage: string) { + return htmlStringPage.replace('[faviconContentHash]', FILES_CONTENT_HASH.FAVICON) + } + + private static addLogoContentHash (htmlStringPage: string) { + return htmlStringPage.replace('[logoContentHash]', FILES_CONTENT_HASH.LOGO) + } + + private static addTitleTag (htmlStringPage: string, title?: string) { + let text = title || CONFIG.INSTANCE.NAME + if (title) text += ` - ${CONFIG.INSTANCE.NAME}` + + const titleTag = `${escapeHTML(text)}` + + return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.TITLE, titleTag) + } + + private static addDescriptionTag (htmlStringPage: string, escapedTruncatedDescription?: string) { + const content = escapedTruncatedDescription || escapeHTML(CONFIG.INSTANCE.SHORT_DESCRIPTION) + const descriptionTag = `` + + return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.DESCRIPTION, descriptionTag) + } + + private static addCustomCSS (htmlStringPage: string) { + const styleTag = `` + + return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.CUSTOM_CSS, styleTag) + } + + private static addServerConfig (htmlStringPage: string, serverConfig: HTMLServerConfig) { + // Stringify the JSON object, and then stringify the string object so we can inject it into the HTML + const serverConfigString = JSON.stringify(JSON.stringify(serverConfig)) + const configScriptTag = `` + + return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.SERVER_CONFIG, configScriptTag) + } + + private static async addAsyncPluginCSS (htmlStringPage: string) { + if (!pathExists(PLUGIN_GLOBAL_CSS_PATH)) { + logger.info('Plugin Global CSS file is not available (generation may still be in progress), ignoring it.') + return htmlStringPage + } + + let globalCSSContent: Buffer + + try { + globalCSSContent = await readFile(PLUGIN_GLOBAL_CSS_PATH) + } catch (err) { + logger.error('Error retrieving the Plugin Global CSS file, ignoring it.', { err }) + return htmlStringPage + } + + if (globalCSSContent.byteLength === 0) return htmlStringPage + + const fileHash = sha256(globalCSSContent) + const linkTag = `` + + return htmlStringPage.replace('', linkTag + '') + } + + private static generateOpenGraphMetaTags (tags: Tags) { + const metaTags = { + 'og:type': tags.ogType, + 'og:site_name': tags.escapedSiteName, + 'og:title': tags.escapedTitle, + 'og:image': tags.image.url + } + + if (tags.image.width && tags.image.height) { + metaTags['og:image:width'] = tags.image.width + metaTags['og:image:height'] = tags.image.height + } + + metaTags['og:url'] = tags.url + metaTags['og:description'] = tags.escapedTruncatedDescription + + if (tags.embed) { + metaTags['og:video:url'] = tags.embed.url + metaTags['og:video:secure_url'] = tags.embed.url + metaTags['og:video:type'] = 'text/html' + metaTags['og:video:width'] = EMBED_SIZE.width + metaTags['og:video:height'] = EMBED_SIZE.height + } + + return metaTags + } + + private static generateStandardMetaTags (tags: Tags) { + return { + name: tags.escapedTitle, + description: tags.escapedTruncatedDescription, + image: tags.image.url + } + } + + private static generateTwitterCardMetaTags (tags: Tags) { + const metaTags = { + 'twitter:card': tags.twitterCard, + 'twitter:site': CONFIG.SERVICES.TWITTER.USERNAME, + 'twitter:title': tags.escapedTitle, + 'twitter:description': tags.escapedTruncatedDescription, + 'twitter:image': tags.image.url + } + + if (tags.image.width && tags.image.height) { + metaTags['twitter:image:width'] = tags.image.width + metaTags['twitter:image:height'] = tags.image.height + } + + if (tags.twitterCard === 'player') { + metaTags['twitter:player'] = tags.embed.url + metaTags['twitter:player:width'] = EMBED_SIZE.width + metaTags['twitter:player:height'] = EMBED_SIZE.height + } + + return metaTags + } + + private static async generateSchemaTags (tags: Tags, context: HookContext) { + const schema = { + '@context': 'http://schema.org', + '@type': tags.schemaType, + 'name': tags.escapedTitle, + 'description': tags.escapedTruncatedDescription, + 'image': tags.image.url, + 'url': tags.url + } + + if (tags.list) { + schema['numberOfItems'] = tags.list.numberOfItems + schema['thumbnailUrl'] = tags.image.url + } + + if (tags.embed) { + schema['embedUrl'] = tags.embed.url + schema['uploadDate'] = tags.embed.createdAt + + if (tags.embed.duration) schema['duration'] = tags.embed.duration + + schema['thumbnailUrl'] = tags.image.url + schema['contentUrl'] = tags.url + } + + return Hooks.wrapObject(schema, 'filter:html.client.json-ld.result', context) + } + + private static async addTags (htmlStringPage: string, tagsValues: Tags, context: HookContext) { + const openGraphMetaTags = this.generateOpenGraphMetaTags(tagsValues) + const standardMetaTags = this.generateStandardMetaTags(tagsValues) + const twitterCardMetaTags = this.generateTwitterCardMetaTags(tagsValues) + const schemaTags = await this.generateSchemaTags(tagsValues, context) + + const { url, escapedTitle, embed, originUrl, disallowIndexation } = tagsValues + + const oembedLinkTags: { type: string, href: string, escapedTitle: string }[] = [] + + if (embed) { + oembedLinkTags.push({ + type: 'application/json+oembed', + href: WEBSERVER.URL + '/services/oembed?url=' + encodeURIComponent(url), + escapedTitle + }) + } + + let tagsStr = '' + + // Opengraph + Object.keys(openGraphMetaTags).forEach(tagName => { + const tagValue = openGraphMetaTags[tagName] + + tagsStr += `` + }) + + // Standard + Object.keys(standardMetaTags).forEach(tagName => { + const tagValue = standardMetaTags[tagName] + + tagsStr += `` + }) + + // Twitter card + Object.keys(twitterCardMetaTags).forEach(tagName => { + const tagValue = twitterCardMetaTags[tagName] + + tagsStr += `` + }) + + // OEmbed + for (const oembedLinkTag of oembedLinkTags) { + tagsStr += `` + } + + // Schema.org + if (schemaTags) { + tagsStr += `` + } + + // SEO, use origin URL + tagsStr += `` + + if (disallowIndexation) { + tagsStr += `` + } + + return htmlStringPage.replace(CUSTOM_HTML_TAG_COMMENTS.META_TAGS, tagsStr) + } +} + +function sendHTML (html: string, res: express.Response, localizedHTML: boolean = false) { + res.set('Content-Type', 'text/html; charset=UTF-8') + + if (localizedHTML) { + res.set('Vary', 'Accept-Language') + } + + return res.send(html) +} + +async function serveIndexHTML (req: express.Request, res: express.Response) { + if (req.accepts(ACCEPT_HEADERS) === 'html' || !req.headers.accept) { + try { + await generateHTMLPage(req, res, req.params.language) + return + } catch (err) { + logger.error('Cannot generate HTML page.', { err }) + return res.status(HttpStatusCode.INTERNAL_SERVER_ERROR_500).end() + } + } + + return res.status(HttpStatusCode.NOT_ACCEPTABLE_406).end() +} + +// --------------------------------------------------------------------------- + +export { + ClientHtml, + sendHTML, + serveIndexHTML +} + +async function generateHTMLPage (req: express.Request, res: express.Response, paramLang?: string) { + const html = await ClientHtml.getDefaultHTMLPage(req, res, paramLang) + + return sendHTML(html, res, true) +} + +function buildEscapedTruncatedDescription (description: string) { + return truncate(mdToOneLinePlainText(description), { length: 200 }) +} diff --git a/server/server/lib/emailer.ts b/server/server/lib/emailer.ts new file mode 100644 index 000000000..eea722d9d --- /dev/null +++ b/server/server/lib/emailer.ts @@ -0,0 +1,283 @@ +import { arrayify } from '@peertube/peertube-core-utils' +import { EmailPayload, SendEmailDefaultOptions, UserRegistrationState } from '@peertube/peertube-models' +import { isTestOrDevInstance, root } from '@peertube/peertube-node-utils' +import { readFileSync } from 'fs' +import merge from 'lodash-es/merge.js' +import { Transporter, createTransport } from 'nodemailer' +import { join } from 'path' +import { bunyanLogger, logger } from '../helpers/logger.js' +import { CONFIG, isEmailEnabled } from '../initializers/config.js' +import { WEBSERVER } from '../initializers/constants.js' +import { MRegistration, MUser } from '../types/models/index.js' +import { JobQueue } from './job-queue/index.js' + +class Emailer { + + private static instance: Emailer + private initialized = false + private transporter: Transporter + + private constructor () { + } + + init () { + // Already initialized + if (this.initialized === true) return + this.initialized = true + + if (!isEmailEnabled()) { + if (!isTestOrDevInstance()) { + logger.error('Cannot use SMTP server because of lack of configuration. PeerTube will not be able to send mails!') + } + + return + } + + if (CONFIG.SMTP.TRANSPORT === 'smtp') this.initSMTPTransport() + else if (CONFIG.SMTP.TRANSPORT === 'sendmail') this.initSendmailTransport() + } + + async checkConnection () { + if (!this.transporter || CONFIG.SMTP.TRANSPORT !== 'smtp') return + + logger.info('Testing SMTP server...') + + try { + const success = await this.transporter.verify() + if (success !== true) this.warnOnConnectionFailure() + + logger.info('Successfully connected to SMTP server.') + } catch (err) { + this.warnOnConnectionFailure(err) + } + } + + addPasswordResetEmailJob (username: string, to: string, resetPasswordUrl: string) { + const emailPayload: EmailPayload = { + template: 'password-reset', + to: [ to ], + subject: 'Reset your account password', + locals: { + username, + resetPasswordUrl, + + hideNotificationPreferencesLink: true + } + } + + return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) + } + + addPasswordCreateEmailJob (username: string, to: string, createPasswordUrl: string) { + const emailPayload: EmailPayload = { + template: 'password-create', + to: [ to ], + subject: 'Create your account password', + locals: { + username, + createPasswordUrl, + + hideNotificationPreferencesLink: true + } + } + + return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) + } + + addVerifyEmailJob (options: { + username: string + isRegistrationRequest: boolean + to: string + verifyEmailUrl: string + }) { + const { username, isRegistrationRequest, to, verifyEmailUrl } = options + + const emailPayload: EmailPayload = { + template: 'verify-email', + to: [ to ], + subject: `Verify your email on ${CONFIG.INSTANCE.NAME}`, + locals: { + username, + verifyEmailUrl, + isRegistrationRequest, + + hideNotificationPreferencesLink: true + } + } + + return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) + } + + addUserBlockJob (user: MUser, blocked: boolean, reason?: string) { + const reasonString = reason ? ` for the following reason: ${reason}` : '' + const blockedWord = blocked ? 'blocked' : 'unblocked' + + const to = user.email + const emailPayload: EmailPayload = { + to: [ to ], + subject: 'Account ' + blockedWord, + text: `Your account ${user.username} on ${CONFIG.INSTANCE.NAME} has been ${blockedWord}${reasonString}.` + } + + return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) + } + + addContactFormJob (fromEmail: string, fromName: string, subject: string, body: string) { + const emailPayload: EmailPayload = { + template: 'contact-form', + to: [ CONFIG.ADMIN.EMAIL ], + replyTo: `"${fromName}" <${fromEmail}>`, + subject: `(contact form) ${subject}`, + locals: { + fromName, + fromEmail, + body, + + // There are not notification preferences for the contact form + hideNotificationPreferencesLink: true + } + } + + return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) + } + + addUserRegistrationRequestProcessedJob (registration: MRegistration) { + let template: string + let subject: string + if (registration.state === UserRegistrationState.ACCEPTED) { + template = 'user-registration-request-accepted' + subject = `Your registration request for ${registration.username} has been accepted` + } else { + template = 'user-registration-request-rejected' + subject = `Your registration request for ${registration.username} has been rejected` + } + + const to = registration.email + const emailPayload: EmailPayload = { + to: [ to ], + template, + subject, + locals: { + username: registration.username, + moderationResponse: registration.moderationResponse, + loginLink: WEBSERVER.URL + '/login' + } + } + + return JobQueue.Instance.createJobAsync({ type: 'email', payload: emailPayload }) + } + + async sendMail (options: EmailPayload) { + if (!isEmailEnabled()) { + logger.info('Cannot send mail because SMTP is not configured.') + return + } + + const fromDisplayName = options.from + ? options.from + : CONFIG.INSTANCE.NAME + + const EmailTemplates = (await import('email-templates')).default + + const email = new EmailTemplates({ + send: true, + htmlToText: { + selectors: [ + { selector: 'img', format: 'skip' }, + { selector: 'a', options: { hideLinkHrefIfSameAsText: true } } + ] + }, + message: { + from: `"${fromDisplayName}" <${CONFIG.SMTP.FROM_ADDRESS}>` + }, + transport: this.transporter, + views: { + root: join(root(), 'dist', 'server', 'lib', 'emails') + }, + subjectPrefix: CONFIG.EMAIL.SUBJECT.PREFIX + }) + + const toEmails = arrayify(options.to) + + for (const to of toEmails) { + const baseOptions: SendEmailDefaultOptions = { + template: 'common', + message: { + to, + from: options.from, + subject: options.subject, + replyTo: options.replyTo + }, + locals: { // default variables available in all templates + WEBSERVER, + EMAIL: CONFIG.EMAIL, + instanceName: CONFIG.INSTANCE.NAME, + text: options.text, + subject: options.subject + } + } + + // overridden/new variables given for a specific template in the payload + const sendOptions = merge(baseOptions, options) + + await email.send(sendOptions) + .then(res => logger.debug('Sent email.', { res })) + .catch(err => logger.error('Error in email sender.', { err })) + } + } + + private warnOnConnectionFailure (err?: Error) { + logger.error('Failed to connect to SMTP %s:%d.', CONFIG.SMTP.HOSTNAME, CONFIG.SMTP.PORT, { err }) + } + + private initSMTPTransport () { + logger.info('Using %s:%s as SMTP server.', CONFIG.SMTP.HOSTNAME, CONFIG.SMTP.PORT) + + let tls + if (CONFIG.SMTP.CA_FILE) { + tls = { + ca: [ readFileSync(CONFIG.SMTP.CA_FILE) ] + } + } + + let auth + if (CONFIG.SMTP.USERNAME && CONFIG.SMTP.PASSWORD) { + auth = { + user: CONFIG.SMTP.USERNAME, + pass: CONFIG.SMTP.PASSWORD + } + } + + this.transporter = createTransport({ + host: CONFIG.SMTP.HOSTNAME, + port: CONFIG.SMTP.PORT, + secure: CONFIG.SMTP.TLS, + debug: CONFIG.LOG.LEVEL === 'debug', + logger: bunyanLogger as any, + ignoreTLS: CONFIG.SMTP.DISABLE_STARTTLS, + tls, + auth + }) + } + + private initSendmailTransport () { + logger.info('Using sendmail to send emails') + + this.transporter = createTransport({ + sendmail: true, + newline: 'unix', + path: CONFIG.SMTP.SENDMAIL, + logger: bunyanLogger + }) + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} + +// --------------------------------------------------------------------------- + +export { + Emailer +} diff --git a/server/lib/emails/abuse-new-message/html.pug b/server/server/lib/emails/abuse-new-message/html.pug similarity index 100% rename from server/lib/emails/abuse-new-message/html.pug rename to server/server/lib/emails/abuse-new-message/html.pug diff --git a/server/lib/emails/abuse-state-change/html.pug b/server/server/lib/emails/abuse-state-change/html.pug similarity index 100% rename from server/lib/emails/abuse-state-change/html.pug rename to server/server/lib/emails/abuse-state-change/html.pug diff --git a/server/lib/emails/account-abuse-new/html.pug b/server/server/lib/emails/account-abuse-new/html.pug similarity index 100% rename from server/lib/emails/account-abuse-new/html.pug rename to server/server/lib/emails/account-abuse-new/html.pug diff --git a/server/lib/emails/common/base.pug b/server/server/lib/emails/common/base.pug similarity index 100% rename from server/lib/emails/common/base.pug rename to server/server/lib/emails/common/base.pug diff --git a/server/lib/emails/common/greetings.pug b/server/server/lib/emails/common/greetings.pug similarity index 100% rename from server/lib/emails/common/greetings.pug rename to server/server/lib/emails/common/greetings.pug diff --git a/server/lib/emails/common/html.pug b/server/server/lib/emails/common/html.pug similarity index 100% rename from server/lib/emails/common/html.pug rename to server/server/lib/emails/common/html.pug diff --git a/server/lib/emails/common/mixins.pug b/server/server/lib/emails/common/mixins.pug similarity index 100% rename from server/lib/emails/common/mixins.pug rename to server/server/lib/emails/common/mixins.pug diff --git a/server/lib/emails/contact-form/html.pug b/server/server/lib/emails/contact-form/html.pug similarity index 100% rename from server/lib/emails/contact-form/html.pug rename to server/server/lib/emails/contact-form/html.pug diff --git a/server/lib/emails/follower-on-channel/html.pug b/server/server/lib/emails/follower-on-channel/html.pug similarity index 100% rename from server/lib/emails/follower-on-channel/html.pug rename to server/server/lib/emails/follower-on-channel/html.pug diff --git a/server/lib/emails/password-create/html.pug b/server/server/lib/emails/password-create/html.pug similarity index 100% rename from server/lib/emails/password-create/html.pug rename to server/server/lib/emails/password-create/html.pug diff --git a/server/lib/emails/password-reset/html.pug b/server/server/lib/emails/password-reset/html.pug similarity index 100% rename from server/lib/emails/password-reset/html.pug rename to server/server/lib/emails/password-reset/html.pug diff --git a/server/lib/emails/peertube-version-new/html.pug b/server/server/lib/emails/peertube-version-new/html.pug similarity index 100% rename from server/lib/emails/peertube-version-new/html.pug rename to server/server/lib/emails/peertube-version-new/html.pug diff --git a/server/lib/emails/plugin-version-new/html.pug b/server/server/lib/emails/plugin-version-new/html.pug similarity index 100% rename from server/lib/emails/plugin-version-new/html.pug rename to server/server/lib/emails/plugin-version-new/html.pug diff --git a/server/lib/emails/user-registered/html.pug b/server/server/lib/emails/user-registered/html.pug similarity index 100% rename from server/lib/emails/user-registered/html.pug rename to server/server/lib/emails/user-registered/html.pug diff --git a/server/lib/emails/user-registration-request-accepted/html.pug b/server/server/lib/emails/user-registration-request-accepted/html.pug similarity index 100% rename from server/lib/emails/user-registration-request-accepted/html.pug rename to server/server/lib/emails/user-registration-request-accepted/html.pug diff --git a/server/lib/emails/user-registration-request-rejected/html.pug b/server/server/lib/emails/user-registration-request-rejected/html.pug similarity index 100% rename from server/lib/emails/user-registration-request-rejected/html.pug rename to server/server/lib/emails/user-registration-request-rejected/html.pug diff --git a/server/lib/emails/user-registration-request/html.pug b/server/server/lib/emails/user-registration-request/html.pug similarity index 100% rename from server/lib/emails/user-registration-request/html.pug rename to server/server/lib/emails/user-registration-request/html.pug diff --git a/server/lib/emails/verify-email/html.pug b/server/server/lib/emails/verify-email/html.pug similarity index 100% rename from server/lib/emails/verify-email/html.pug rename to server/server/lib/emails/verify-email/html.pug diff --git a/server/lib/emails/video-abuse-new/html.pug b/server/server/lib/emails/video-abuse-new/html.pug similarity index 100% rename from server/lib/emails/video-abuse-new/html.pug rename to server/server/lib/emails/video-abuse-new/html.pug diff --git a/server/lib/emails/video-auto-blacklist-new/html.pug b/server/server/lib/emails/video-auto-blacklist-new/html.pug similarity index 100% rename from server/lib/emails/video-auto-blacklist-new/html.pug rename to server/server/lib/emails/video-auto-blacklist-new/html.pug diff --git a/server/lib/emails/video-comment-abuse-new/html.pug b/server/server/lib/emails/video-comment-abuse-new/html.pug similarity index 100% rename from server/lib/emails/video-comment-abuse-new/html.pug rename to server/server/lib/emails/video-comment-abuse-new/html.pug diff --git a/server/lib/emails/video-comment-mention/html.pug b/server/server/lib/emails/video-comment-mention/html.pug similarity index 100% rename from server/lib/emails/video-comment-mention/html.pug rename to server/server/lib/emails/video-comment-mention/html.pug diff --git a/server/lib/emails/video-comment-new/html.pug b/server/server/lib/emails/video-comment-new/html.pug similarity index 100% rename from server/lib/emails/video-comment-new/html.pug rename to server/server/lib/emails/video-comment-new/html.pug diff --git a/server/server/lib/files-cache/avatar-permanent-file-cache.ts b/server/server/lib/files-cache/avatar-permanent-file-cache.ts new file mode 100644 index 000000000..7103e0c7f --- /dev/null +++ b/server/server/lib/files-cache/avatar-permanent-file-cache.ts @@ -0,0 +1,27 @@ +import { CONFIG } from '@server/initializers/config.js' +import { ACTOR_IMAGES_SIZE } from '@server/initializers/constants.js' +import { ActorImageModel } from '@server/models/actor/actor-image.js' +import { MActorImage } from '@server/types/models/index.js' +import { AbstractPermanentFileCache } from './shared/index.js' + +export class AvatarPermanentFileCache extends AbstractPermanentFileCache { + + constructor () { + super(CONFIG.STORAGE.ACTOR_IMAGES_DIR) + } + + protected loadModel (filename: string) { + return ActorImageModel.loadByName(filename) + } + + protected getImageSize (image: MActorImage): { width: number, height: number } { + if (image.width && image.height) { + return { + height: image.height, + width: image.width + } + } + + return ACTOR_IMAGES_SIZE[image.type][0] + } +} diff --git a/server/server/lib/files-cache/index.ts b/server/server/lib/files-cache/index.ts new file mode 100644 index 000000000..219baed52 --- /dev/null +++ b/server/server/lib/files-cache/index.ts @@ -0,0 +1,6 @@ +export * from './avatar-permanent-file-cache.js' +export * from './video-miniature-permanent-file-cache.js' +export * from './video-captions-simple-file-cache.js' +export * from './video-previews-simple-file-cache.js' +export * from './video-storyboards-simple-file-cache.js' +export * from './video-torrents-simple-file-cache.js' diff --git a/server/server/lib/files-cache/shared/abstract-permanent-file-cache.ts b/server/server/lib/files-cache/shared/abstract-permanent-file-cache.ts new file mode 100644 index 000000000..f9e9878a8 --- /dev/null +++ b/server/server/lib/files-cache/shared/abstract-permanent-file-cache.ts @@ -0,0 +1,132 @@ +import express from 'express' +import { LRUCache } from 'lru-cache' +import { Model } from 'sequelize' +import { logger } from '@server/helpers/logger.js' +import { CachePromise } from '@server/helpers/promise-cache.js' +import { LRU_CACHE, STATIC_MAX_AGE } from '@server/initializers/constants.js' +import { downloadImageFromWorker } from '@server/lib/worker/parent-process.js' +import { HttpStatusCode } from '@peertube/peertube-models' + +type ImageModel = { + fileUrl: string + filename: string + onDisk: boolean + + isOwned (): boolean + getPath (): string + + save (): Promise +} + +export abstract class AbstractPermanentFileCache { + // Unsafe because it can return paths that do not exist anymore + private readonly filenameToPathUnsafeCache = new LRUCache({ + max: LRU_CACHE.FILENAME_TO_PATH_PERMANENT_FILE_CACHE.MAX_SIZE + }) + + protected abstract getImageSize (image: M): { width: number, height: number } + protected abstract loadModel (filename: string): Promise + + constructor (private readonly directory: string) { + + } + + async lazyServe (options: { + filename: string + res: express.Response + next: express.NextFunction + }) { + const { filename, res, next } = options + + if (this.filenameToPathUnsafeCache.has(filename)) { + return res.sendFile(this.filenameToPathUnsafeCache.get(filename), { maxAge: STATIC_MAX_AGE.SERVER }) + } + + const image = await this.lazyLoadIfNeeded(filename) + if (!image) return res.status(HttpStatusCode.NOT_FOUND_404).end() + + const path = image.getPath() + this.filenameToPathUnsafeCache.set(filename, path) + + return res.sendFile(path, { maxAge: STATIC_MAX_AGE.LAZY_SERVER }, (err: any) => { + if (!err) return + + this.onServeError({ err, image, next, filename }) + }) + } + + @CachePromise({ + keyBuilder: filename => filename + }) + private async lazyLoadIfNeeded (filename: string) { + const image = await this.loadModel(filename) + if (!image) return undefined + + if (image.onDisk === false) { + if (!image.fileUrl) return undefined + + try { + await this.downloadRemoteFile(image) + } catch (err) { + logger.warn('Cannot process remote image %s.', image.fileUrl, { err }) + + return undefined + } + } + + return image + } + + async downloadRemoteFile (image: M) { + logger.info('Download remote image %s lazily.', image.fileUrl) + + const destination = await this.downloadImage({ + filename: image.filename, + fileUrl: image.fileUrl, + size: this.getImageSize(image) + }) + + image.onDisk = true + image.save() + .catch(err => logger.error('Cannot save new image disk state.', { err })) + + return destination + } + + private onServeError (options: { + err: any + image: M + filename: string + next: express.NextFunction + }) { + const { err, image, filename, next } = options + + // It seems this actor image is not on the disk anymore + if (err.status === HttpStatusCode.NOT_FOUND_404 && !image.isOwned()) { + logger.error('Cannot lazy serve image %s.', filename, { err }) + + this.filenameToPathUnsafeCache.delete(filename) + + image.onDisk = false + image.save() + .catch(err => logger.error('Cannot save new image disk state.', { err })) + } + + return next(err) + } + + private downloadImage (options: { + fileUrl: string + filename: string + size: { width: number, height: number } + }) { + const downloaderOptions = { + url: options.fileUrl, + destDir: this.directory, + destName: options.filename, + size: options.size + } + + return downloadImageFromWorker(downloaderOptions) + } +} diff --git a/server/server/lib/files-cache/shared/abstract-simple-file-cache.ts b/server/server/lib/files-cache/shared/abstract-simple-file-cache.ts new file mode 100644 index 000000000..1dd45d221 --- /dev/null +++ b/server/server/lib/files-cache/shared/abstract-simple-file-cache.ts @@ -0,0 +1,30 @@ +import { remove } from 'fs-extra/esm' +import { logger } from '../../../helpers/logger.js' +import memoizee from 'memoizee' + +type GetFilePathResult = { isOwned: boolean, path: string, downloadName?: string } | undefined + +export abstract class AbstractSimpleFileCache { + + getFilePath: (params: T) => Promise + + abstract getFilePathImpl (params: T): Promise + + // Load and save the remote file, then return the local path from filesystem + protected abstract loadRemoteFile (key: string): Promise + + init (max: number, maxAge: number) { + this.getFilePath = memoizee(this.getFilePathImpl, { + maxAge, + max, + promise: true, + dispose: (result?: GetFilePathResult) => { + if (result && result.isOwned !== true) { + remove(result.path) + .then(() => logger.debug('%s removed from %s', result.path, this.constructor.name)) + .catch(err => logger.error('Cannot remove %s from cache %s.', result.path, this.constructor.name, { err })) + } + } + }) + } +} diff --git a/server/server/lib/files-cache/shared/index.ts b/server/server/lib/files-cache/shared/index.ts new file mode 100644 index 000000000..4dc4d8c4d --- /dev/null +++ b/server/server/lib/files-cache/shared/index.ts @@ -0,0 +1,2 @@ +export * from './abstract-permanent-file-cache.js' +export * from './abstract-simple-file-cache.js' diff --git a/server/server/lib/files-cache/video-captions-simple-file-cache.ts b/server/server/lib/files-cache/video-captions-simple-file-cache.ts new file mode 100644 index 000000000..26853706e --- /dev/null +++ b/server/server/lib/files-cache/video-captions-simple-file-cache.ts @@ -0,0 +1,61 @@ +import { join } from 'path' +import { logger } from '@server/helpers/logger.js' +import { doRequestAndSaveToFile } from '@server/helpers/requests.js' +import { CONFIG } from '../../initializers/config.js' +import { FILES_CACHE } from '../../initializers/constants.js' +import { VideoModel } from '../../models/video/video.js' +import { VideoCaptionModel } from '../../models/video/video-caption.js' +import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache.js' + +class VideoCaptionsSimpleFileCache extends AbstractSimpleFileCache { + + private static instance: VideoCaptionsSimpleFileCache + + private constructor () { + super() + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } + + async getFilePathImpl (filename: string) { + const videoCaption = await VideoCaptionModel.loadWithVideoByFilename(filename) + if (!videoCaption) return undefined + + if (videoCaption.isOwned()) { + return { isOwned: true, path: join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.filename) } + } + + return this.loadRemoteFile(filename) + } + + // Key is the caption filename + protected async loadRemoteFile (key: string) { + const videoCaption = await VideoCaptionModel.loadWithVideoByFilename(key) + if (!videoCaption) return undefined + + if (videoCaption.isOwned()) throw new Error('Cannot load remote caption of owned video.') + + // Used to fetch the path + const video = await VideoModel.loadFull(videoCaption.videoId) + if (!video) return undefined + + const remoteUrl = videoCaption.getFileUrl(video) + const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.filename) + + try { + await doRequestAndSaveToFile(remoteUrl, destPath) + + return { isOwned: false, path: destPath } + } catch (err) { + logger.info('Cannot fetch remote caption file %s.', remoteUrl, { err }) + + return undefined + } + } +} + +export { + VideoCaptionsSimpleFileCache +} diff --git a/server/server/lib/files-cache/video-miniature-permanent-file-cache.ts b/server/server/lib/files-cache/video-miniature-permanent-file-cache.ts new file mode 100644 index 000000000..d3366ba43 --- /dev/null +++ b/server/server/lib/files-cache/video-miniature-permanent-file-cache.ts @@ -0,0 +1,28 @@ +import { CONFIG } from '@server/initializers/config.js' +import { THUMBNAILS_SIZE } from '@server/initializers/constants.js' +import { ThumbnailModel } from '@server/models/video/thumbnail.js' +import { MThumbnail } from '@server/types/models/index.js' +import { ThumbnailType } from '@peertube/peertube-models' +import { AbstractPermanentFileCache } from './shared/index.js' + +export class VideoMiniaturePermanentFileCache extends AbstractPermanentFileCache { + + constructor () { + super(CONFIG.STORAGE.THUMBNAILS_DIR) + } + + protected loadModel (filename: string) { + return ThumbnailModel.loadByFilename(filename, ThumbnailType.MINIATURE) + } + + protected getImageSize (image: MThumbnail): { width: number, height: number } { + if (image.width && image.height) { + return { + height: image.height, + width: image.width + } + } + + return THUMBNAILS_SIZE + } +} diff --git a/server/server/lib/files-cache/video-previews-simple-file-cache.ts b/server/server/lib/files-cache/video-previews-simple-file-cache.ts new file mode 100644 index 000000000..4dd0fd148 --- /dev/null +++ b/server/server/lib/files-cache/video-previews-simple-file-cache.ts @@ -0,0 +1,58 @@ +import { join } from 'path' +import { FILES_CACHE } from '../../initializers/constants.js' +import { VideoModel } from '../../models/video/video.js' +import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache.js' +import { doRequestAndSaveToFile } from '@server/helpers/requests.js' +import { ThumbnailModel } from '@server/models/video/thumbnail.js' +import { ThumbnailType } from '@peertube/peertube-models' +import { logger } from '@server/helpers/logger.js' + +class VideoPreviewsSimpleFileCache extends AbstractSimpleFileCache { + + private static instance: VideoPreviewsSimpleFileCache + + private constructor () { + super() + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } + + async getFilePathImpl (filename: string) { + const thumbnail = await ThumbnailModel.loadWithVideoByFilename(filename, ThumbnailType.PREVIEW) + if (!thumbnail) return undefined + + if (thumbnail.Video.isOwned()) return { isOwned: true, path: thumbnail.getPath() } + + return this.loadRemoteFile(thumbnail.Video.uuid) + } + + // Key is the video UUID + protected async loadRemoteFile (key: string) { + const video = await VideoModel.loadFull(key) + if (!video) return undefined + + if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.') + + const preview = video.getPreview() + const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, preview.filename) + const remoteUrl = preview.getOriginFileUrl(video) + + try { + await doRequestAndSaveToFile(remoteUrl, destPath) + + logger.debug('Fetched remote preview %s to %s.', remoteUrl, destPath) + + return { isOwned: false, path: destPath } + } catch (err) { + logger.info('Cannot fetch remote preview file %s.', remoteUrl, { err }) + + return undefined + } + } +} + +export { + VideoPreviewsSimpleFileCache +} diff --git a/server/server/lib/files-cache/video-storyboards-simple-file-cache.ts b/server/server/lib/files-cache/video-storyboards-simple-file-cache.ts new file mode 100644 index 000000000..7991b8d59 --- /dev/null +++ b/server/server/lib/files-cache/video-storyboards-simple-file-cache.ts @@ -0,0 +1,53 @@ +import { join } from 'path' +import { logger } from '@server/helpers/logger.js' +import { doRequestAndSaveToFile } from '@server/helpers/requests.js' +import { StoryboardModel } from '@server/models/video/storyboard.js' +import { FILES_CACHE } from '../../initializers/constants.js' +import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache.js' + +class VideoStoryboardsSimpleFileCache extends AbstractSimpleFileCache { + + private static instance: VideoStoryboardsSimpleFileCache + + private constructor () { + super() + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } + + async getFilePathImpl (filename: string) { + const storyboard = await StoryboardModel.loadWithVideoByFilename(filename) + if (!storyboard) return undefined + + if (storyboard.Video.isOwned()) return { isOwned: true, path: storyboard.getPath() } + + return this.loadRemoteFile(storyboard.filename) + } + + // Key is the storyboard filename + protected async loadRemoteFile (key: string) { + const storyboard = await StoryboardModel.loadWithVideoByFilename(key) + if (!storyboard) return undefined + + const destPath = join(FILES_CACHE.STORYBOARDS.DIRECTORY, storyboard.filename) + const remoteUrl = storyboard.getOriginFileUrl(storyboard.Video) + + try { + await doRequestAndSaveToFile(remoteUrl, destPath) + + logger.debug('Fetched remote storyboard %s to %s.', remoteUrl, destPath) + + return { isOwned: false, path: destPath } + } catch (err) { + logger.info('Cannot fetch remote storyboard file %s.', remoteUrl, { err }) + + return undefined + } + } +} + +export { + VideoStoryboardsSimpleFileCache +} diff --git a/server/server/lib/files-cache/video-torrents-simple-file-cache.ts b/server/server/lib/files-cache/video-torrents-simple-file-cache.ts new file mode 100644 index 000000000..5a1ef3467 --- /dev/null +++ b/server/server/lib/files-cache/video-torrents-simple-file-cache.ts @@ -0,0 +1,70 @@ +import { join } from 'path' +import { logger } from '@server/helpers/logger.js' +import { doRequestAndSaveToFile } from '@server/helpers/requests.js' +import { VideoFileModel } from '@server/models/video/video-file.js' +import { MVideo, MVideoFile } from '@server/types/models/index.js' +import { CONFIG } from '../../initializers/config.js' +import { FILES_CACHE } from '../../initializers/constants.js' +import { VideoModel } from '../../models/video/video.js' +import { AbstractSimpleFileCache } from './shared/abstract-simple-file-cache.js' + +class VideoTorrentsSimpleFileCache extends AbstractSimpleFileCache { + + private static instance: VideoTorrentsSimpleFileCache + + private constructor () { + super() + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } + + async getFilePathImpl (filename: string) { + const file = await VideoFileModel.loadWithVideoOrPlaylistByTorrentFilename(filename) + if (!file) return undefined + + if (file.getVideo().isOwned()) { + const downloadName = this.buildDownloadName(file.getVideo(), file) + + return { isOwned: true, path: join(CONFIG.STORAGE.TORRENTS_DIR, file.torrentFilename), downloadName } + } + + return this.loadRemoteFile(filename) + } + + // Key is the torrent filename + protected async loadRemoteFile (key: string) { + const file = await VideoFileModel.loadWithVideoOrPlaylistByTorrentFilename(key) + if (!file) return undefined + + if (file.getVideo().isOwned()) throw new Error('Cannot load remote file of owned video.') + + // Used to fetch the path + const video = await VideoModel.loadFull(file.getVideo().id) + if (!video) return undefined + + const remoteUrl = file.getRemoteTorrentUrl(video) + const destPath = join(FILES_CACHE.TORRENTS.DIRECTORY, file.torrentFilename) + + try { + await doRequestAndSaveToFile(remoteUrl, destPath) + + const downloadName = this.buildDownloadName(video, file) + + return { isOwned: false, path: destPath, downloadName } + } catch (err) { + logger.info('Cannot fetch remote torrent file %s.', remoteUrl, { err }) + + return undefined + } + } + + private buildDownloadName (video: MVideo, file: MVideoFile) { + return `${video.name}-${file.resolution}p.torrent` + } +} + +export { + VideoTorrentsSimpleFileCache +} diff --git a/server/server/lib/hls.ts b/server/server/lib/hls.ts new file mode 100644 index 000000000..9a53593fb --- /dev/null +++ b/server/server/lib/hls.ts @@ -0,0 +1,286 @@ +import { uniqify, uuidRegex } from '@peertube/peertube-core-utils' +import { getVideoStreamDimensionsInfo } from '@peertube/peertube-ffmpeg' +import { VideoStorage } from '@peertube/peertube-models' +import { sha256 } from '@peertube/peertube-node-utils' +import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models/index.js' +import { ensureDir, move, outputJSON, remove } from 'fs-extra/esm' +import { open, readFile, stat, writeFile } from 'fs/promises' +import flatten from 'lodash-es/flatten.js' +import PQueue from 'p-queue' +import { basename, dirname, join } from 'path' +import { getAudioStreamCodec, getVideoStreamCodec } from '../helpers/ffmpeg/index.js' +import { logger, loggerTagsFactory } from '../helpers/logger.js' +import { doRequest, doRequestAndSaveToFile } from '../helpers/requests.js' +import { generateRandomString } from '../helpers/utils.js' +import { CONFIG } from '../initializers/config.js' +import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers/constants.js' +import { sequelizeTypescript } from '../initializers/database.js' +import { VideoFileModel } from '../models/video/video-file.js' +import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist.js' +import { storeHLSFileFromFilename } from './object-storage/index.js' +import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths.js' +import { VideoPathManager } from './video-path-manager.js' + +const lTags = loggerTagsFactory('hls') + +async function updateStreamingPlaylistsInfohashesIfNeeded () { + const playlistsToUpdate = await VideoStreamingPlaylistModel.listByIncorrectPeerVersion() + + // Use separate SQL queries, because we could have many videos to update + for (const playlist of playlistsToUpdate) { + await sequelizeTypescript.transaction(async t => { + const videoFiles = await VideoFileModel.listByStreamingPlaylist(playlist.id, t) + + playlist.assignP2PMediaLoaderInfoHashes(playlist.Video, videoFiles) + playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION + + await playlist.save({ transaction: t }) + }) + } +} + +async function updatePlaylistAfterFileChange (video: MVideo, playlist: MStreamingPlaylist) { + try { + let playlistWithFiles = await updateMasterHLSPlaylist(video, playlist) + playlistWithFiles = await updateSha256VODSegments(video, playlist) + + // Refresh playlist, operations can take some time + playlistWithFiles = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlist.id) + playlistWithFiles.assignP2PMediaLoaderInfoHashes(video, playlistWithFiles.VideoFiles) + await playlistWithFiles.save() + + video.setHLSPlaylist(playlistWithFiles) + } catch (err) { + logger.warn('Cannot update playlist after file change. Maybe due to concurrent transcoding', { err }) + } +} + +// --------------------------------------------------------------------------- + +// Avoid concurrency issues when updating streaming playlist files +const playlistFilesQueue = new PQueue({ concurrency: 1 }) + +function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingPlaylist): Promise { + return playlistFilesQueue.add(async () => { + const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id) + + const masterPlaylists: string[] = [ '#EXTM3U', '#EXT-X-VERSION:3' ] + + for (const file of playlist.VideoFiles) { + const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) + + await VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(playlist), async videoFilePath => { + const size = await getVideoStreamDimensionsInfo(videoFilePath) + + const bandwidth = 'BANDWIDTH=' + video.getBandwidthBits(file) + const resolution = `RESOLUTION=${size?.width || 0}x${size?.height || 0}` + + let line = `#EXT-X-STREAM-INF:${bandwidth},${resolution}` + if (file.fps) line += ',FRAME-RATE=' + file.fps + + const codecs = await Promise.all([ + getVideoStreamCodec(videoFilePath), + getAudioStreamCodec(videoFilePath) + ]) + + line += `,CODECS="${codecs.filter(c => !!c).join(',')}"` + + masterPlaylists.push(line) + masterPlaylists.push(playlistFilename) + }) + } + + if (playlist.playlistFilename) { + await video.removeStreamingPlaylistFile(playlist, playlist.playlistFilename) + } + playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive) + + const masterPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.playlistFilename) + await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') + + logger.info('Updating %s master playlist file of video %s', masterPlaylistPath, video.uuid, lTags(video.uuid)) + + if (playlist.storage === VideoStorage.OBJECT_STORAGE) { + playlist.playlistUrl = await storeHLSFileFromFilename(playlist, playlist.playlistFilename) + await remove(masterPlaylistPath) + } + + return playlist.save() + }, { throwOnTimeout: true }) +} + +// --------------------------------------------------------------------------- + +function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingPlaylist): Promise { + return playlistFilesQueue.add(async () => { + const json: { [filename: string]: { [range: string]: string } } = {} + + const playlist = await VideoStreamingPlaylistModel.loadWithVideoAndFiles(playlistArg.id) + + // For all the resolutions available for this video + for (const file of playlist.VideoFiles) { + const rangeHashes: { [range: string]: string } = {} + const fileWithPlaylist = file.withVideoOrPlaylist(playlist) + + await VideoPathManager.Instance.makeAvailableVideoFile(fileWithPlaylist, videoPath => { + + return VideoPathManager.Instance.makeAvailableResolutionPlaylistFile(fileWithPlaylist, async resolutionPlaylistPath => { + const playlistContent = await readFile(resolutionPlaylistPath) + const ranges = getRangesFromPlaylist(playlistContent.toString()) + + const fd = await open(videoPath, 'r') + for (const range of ranges) { + const buf = Buffer.alloc(range.length) + await fd.read(buf, 0, range.length, range.offset) + + rangeHashes[`${range.offset}-${range.offset + range.length - 1}`] = sha256(buf) + } + await fd.close() + + const videoFilename = file.filename + json[videoFilename] = rangeHashes + }) + }) + } + + if (playlist.segmentsSha256Filename) { + await video.removeStreamingPlaylistFile(playlist, playlist.segmentsSha256Filename) + } + playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive) + + const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) + await outputJSON(outputPath, json) + + if (playlist.storage === VideoStorage.OBJECT_STORAGE) { + playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlist, playlist.segmentsSha256Filename) + await remove(outputPath) + } + + return playlist.save() + }, { throwOnTimeout: true }) +} + +// --------------------------------------------------------------------------- + +async function buildSha256Segment (segmentPath: string) { + const buf = await readFile(segmentPath) + return sha256(buf) +} + +function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, timeout: number, bodyKBLimit: number) { + let timer + let remainingBodyKBLimit = bodyKBLimit + + logger.info('Importing HLS playlist %s', playlistUrl) + + return new Promise(async (res, rej) => { + const tmpDirectory = join(CONFIG.STORAGE.TMP_DIR, await generateRandomString(10)) + + await ensureDir(tmpDirectory) + + timer = setTimeout(() => { + deleteTmpDirectory(tmpDirectory) + + return rej(new Error('HLS download timeout.')) + }, timeout) + + try { + // Fetch master playlist + const subPlaylistUrls = await fetchUniqUrls(playlistUrl) + + const subRequests = subPlaylistUrls.map(u => fetchUniqUrls(u)) + const fileUrls = uniqify(flatten(await Promise.all(subRequests))) + + logger.debug('Will download %d HLS files.', fileUrls.length, { fileUrls }) + + for (const fileUrl of fileUrls) { + const destPath = join(tmpDirectory, basename(fileUrl)) + + await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit: remainingBodyKBLimit, timeout: REQUEST_TIMEOUTS.REDUNDANCY }) + + const { size } = await stat(destPath) + remainingBodyKBLimit -= (size / 1000) + + logger.debug('Downloaded HLS playlist file %s with %d kB remained limit.', fileUrl, Math.floor(remainingBodyKBLimit)) + } + + clearTimeout(timer) + + await move(tmpDirectory, destinationDir, { overwrite: true }) + + return res() + } catch (err) { + deleteTmpDirectory(tmpDirectory) + + return rej(err) + } + }) + + function deleteTmpDirectory (directory: string) { + remove(directory) + .catch(err => logger.error('Cannot delete path on HLS download error.', { err })) + } + + async function fetchUniqUrls (playlistUrl: string) { + const { body } = await doRequest(playlistUrl) + + if (!body) return [] + + const urls = body.split('\n') + .filter(line => line.endsWith('.m3u8') || line.endsWith('.mp4')) + .map(url => { + if (url.startsWith('http://') || url.startsWith('https://')) return url + + return `${dirname(playlistUrl)}/${url}` + }) + + return uniqify(urls) + } +} + +// --------------------------------------------------------------------------- + +async function renameVideoFileInPlaylist (playlistPath: string, newVideoFilename: string) { + const content = await readFile(playlistPath, 'utf8') + + const newContent = content.replace(new RegExp(`${uuidRegex}-\\d+-fragmented.mp4`, 'g'), newVideoFilename) + + await writeFile(playlistPath, newContent, 'utf8') +} + +// --------------------------------------------------------------------------- + +function injectQueryToPlaylistUrls (content: string, queryString: string) { + return content.replace(/\.(m3u8|ts|mp4)/gm, '.$1?' + queryString) +} + +// --------------------------------------------------------------------------- + +export { + updateMasterHLSPlaylist, + updateSha256VODSegments, + buildSha256Segment, + downloadPlaylistSegments, + updateStreamingPlaylistsInfohashesIfNeeded, + updatePlaylistAfterFileChange, + injectQueryToPlaylistUrls, + renameVideoFileInPlaylist +} + +// --------------------------------------------------------------------------- + +function getRangesFromPlaylist (playlistContent: string) { + const ranges: { offset: number, length: number }[] = [] + const lines = playlistContent.split('\n') + const regex = /^#EXT-X-BYTERANGE:(\d+)@(\d+)$/ + + for (const line of lines) { + const captured = regex.exec(line) + + if (captured) { + ranges.push({ length: parseInt(captured[1], 10), offset: parseInt(captured[2], 10) }) + } + } + + return ranges +} diff --git a/server/server/lib/internal-event-emitter.ts b/server/server/lib/internal-event-emitter.ts new file mode 100644 index 000000000..54f192982 --- /dev/null +++ b/server/server/lib/internal-event-emitter.ts @@ -0,0 +1,35 @@ +import { MChannel, MVideo } from '@server/types/models/index.js' +import { EventEmitter } from 'events' + +export interface PeerTubeInternalEvents { + 'video-created': (options: { video: MVideo }) => void + 'video-updated': (options: { video: MVideo }) => void + 'video-deleted': (options: { video: MVideo }) => void + + 'channel-created': (options: { channel: MChannel }) => void + 'channel-updated': (options: { channel: MChannel }) => void + 'channel-deleted': (options: { channel: MChannel }) => void +} + +declare interface InternalEventEmitter { + on( + event: U, listener: PeerTubeInternalEvents[U] + ): this + + emit( + event: U, ...args: Parameters + ): boolean +} + +class InternalEventEmitter extends EventEmitter { + + private static instance: InternalEventEmitter + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} + +export { + InternalEventEmitter +} diff --git a/server/server/lib/job-queue/handlers/activitypub-cleaner.ts b/server/server/lib/job-queue/handlers/activitypub-cleaner.ts new file mode 100644 index 000000000..7c8c68fdf --- /dev/null +++ b/server/server/lib/job-queue/handlers/activitypub-cleaner.ts @@ -0,0 +1,202 @@ +import Bluebird from 'bluebird' +import { Job } from 'bullmq' +import { + isAnnounceActivityValid, + isDislikeActivityValid, + isLikeActivityValid +} from '@server/helpers/custom-validators/activitypub/activity.js' +import { sanitizeAndCheckVideoCommentObject } from '@server/helpers/custom-validators/activitypub/video-comments.js' +import { PeerTubeRequestError } from '@server/helpers/requests.js' +import { AP_CLEANER } from '@server/initializers/constants.js' +import { fetchAP } from '@server/lib/activitypub/activity.js' +import { checkUrlsSameHost } from '@server/lib/activitypub/url.js' +import { Redis } from '@server/lib/redis.js' +import { VideoCommentModel } from '@server/models/video/video-comment.js' +import { VideoShareModel } from '@server/models/video/video-share.js' +import { VideoModel } from '@server/models/video/video.js' +import { HttpStatusCode } from '@peertube/peertube-models' +import { logger, loggerTagsFactory } from '../../../helpers/logger.js' +import { AccountVideoRateModel } from '../../../models/account/account-video-rate.js' + +const lTags = loggerTagsFactory('ap-cleaner') + +// Job to clean remote interactions off local videos + +async function processActivityPubCleaner (_job: Job) { + logger.info('Processing ActivityPub cleaner.', lTags()) + + { + const rateUrls = await AccountVideoRateModel.listRemoteRateUrlsOfLocalVideos() + const { bodyValidator, deleter, updater } = rateOptionsFactory() + + await Bluebird.map(rateUrls, async rateUrl => { + // TODO: remove when https://github.com/mastodon/mastodon/issues/13571 is fixed + if (rateUrl.includes('#')) return + + const result = await updateObjectIfNeeded({ url: rateUrl, bodyValidator, updater, deleter }) + + if (result?.status === 'deleted') { + const { videoId, type } = result.data + + await VideoModel.syncLocalRates(videoId, type, undefined) + } + }, { concurrency: AP_CLEANER.CONCURRENCY }) + } + + { + const shareUrls = await VideoShareModel.listRemoteShareUrlsOfLocalVideos() + const { bodyValidator, deleter, updater } = shareOptionsFactory() + + await Bluebird.map(shareUrls, async shareUrl => { + await updateObjectIfNeeded({ url: shareUrl, bodyValidator, updater, deleter }) + }, { concurrency: AP_CLEANER.CONCURRENCY }) + } + + { + const commentUrls = await VideoCommentModel.listRemoteCommentUrlsOfLocalVideos() + const { bodyValidator, deleter, updater } = commentOptionsFactory() + + await Bluebird.map(commentUrls, async commentUrl => { + await updateObjectIfNeeded({ url: commentUrl, bodyValidator, updater, deleter }) + }, { concurrency: AP_CLEANER.CONCURRENCY }) + } +} + +// --------------------------------------------------------------------------- + +export { + processActivityPubCleaner +} + +// --------------------------------------------------------------------------- + +async function updateObjectIfNeeded (options: { + url: string + bodyValidator: (body: any) => boolean + updater: (url: string, newUrl: string) => Promise + deleter: (url: string) => Promise } +): Promise<{ data: T, status: 'deleted' | 'updated' } | null> { + const { url, bodyValidator, updater, deleter } = options + + const on404OrTombstone = async () => { + logger.info('Removing remote AP object %s.', url, lTags(url)) + const data = await deleter(url) + + return { status: 'deleted' as 'deleted', data } + } + + try { + const { body } = await fetchAP(url) + + // If not same id, check same host and update + if (!body?.id || !bodyValidator(body)) throw new Error(`Body or body id of ${url} is invalid`) + + if (body.type === 'Tombstone') { + return on404OrTombstone() + } + + const newUrl = body.id + if (newUrl !== url) { + if (checkUrlsSameHost(newUrl, url) !== true) { + throw new Error(`New url ${newUrl} has not the same host than old url ${url}`) + } + + logger.info('Updating remote AP object %s.', url, lTags(url)) + const data = await updater(url, newUrl) + + return { status: 'updated', data } + } + + return null + } catch (err) { + // Does not exist anymore, remove entry + if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { + return on404OrTombstone() + } + + logger.debug('Remote AP object %s is unavailable.', url, lTags(url)) + + const unavailability = await Redis.Instance.addAPUnavailability(url) + if (unavailability >= AP_CLEANER.UNAVAILABLE_TRESHOLD) { + logger.info('Removing unavailable AP resource %s.', url, lTags(url)) + return on404OrTombstone() + } + + return null + } +} + +function rateOptionsFactory () { + return { + bodyValidator: (body: any) => isLikeActivityValid(body) || isDislikeActivityValid(body), + + updater: async (url: string, newUrl: string) => { + const rate = await AccountVideoRateModel.loadByUrl(url, undefined) + rate.url = newUrl + + const videoId = rate.videoId + const type = rate.type + + await rate.save() + + return { videoId, type } + }, + + deleter: async (url) => { + const rate = await AccountVideoRateModel.loadByUrl(url, undefined) + + const videoId = rate.videoId + const type = rate.type + + await rate.destroy() + + return { videoId, type } + } + } +} + +function shareOptionsFactory () { + return { + bodyValidator: (body: any) => isAnnounceActivityValid(body), + + updater: async (url: string, newUrl: string) => { + const share = await VideoShareModel.loadByUrl(url, undefined) + share.url = newUrl + + await share.save() + + return undefined + }, + + deleter: async (url) => { + const share = await VideoShareModel.loadByUrl(url, undefined) + + await share.destroy() + + return undefined + } + } +} + +function commentOptionsFactory () { + return { + bodyValidator: (body: any) => sanitizeAndCheckVideoCommentObject(body), + + updater: async (url: string, newUrl: string) => { + const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(url) + comment.url = newUrl + + await comment.save() + + return undefined + }, + + deleter: async (url) => { + const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(url) + + await comment.destroy() + + return undefined + } + } +} diff --git a/server/server/lib/job-queue/handlers/activitypub-follow.ts b/server/server/lib/job-queue/handlers/activitypub-follow.ts new file mode 100644 index 000000000..a5b3a4fd4 --- /dev/null +++ b/server/server/lib/job-queue/handlers/activitypub-follow.ts @@ -0,0 +1,82 @@ +import { Job } from 'bullmq' +import { getLocalActorFollowActivityPubUrl } from '@server/lib/activitypub/url.js' +import { ActivitypubFollowPayload } from '@peertube/peertube-models' +import { sanitizeHost } from '../../../helpers/core-utils.js' +import { retryTransactionWrapper } from '../../../helpers/database-utils.js' +import { logger } from '../../../helpers/logger.js' +import { REMOTE_SCHEME, WEBSERVER } from '../../../initializers/constants.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { ActorModel } from '../../../models/actor/actor.js' +import { ActorFollowModel } from '../../../models/actor/actor-follow.js' +import { MActor, MActorFull } from '../../../types/models/index.js' +import { getOrCreateAPActor, loadActorUrlOrGetFromWebfinger } from '../../activitypub/actors/index.js' +import { sendFollow } from '../../activitypub/send/index.js' +import { Notifier } from '../../notifier/index.js' + +async function processActivityPubFollow (job: Job) { + const payload = job.data as ActivitypubFollowPayload + const host = payload.host + + logger.info('Processing ActivityPub follow in job %s.', job.id) + + let targetActor: MActorFull + if (!host || host === WEBSERVER.HOST) { + targetActor = await ActorModel.loadLocalByName(payload.name) + } else { + const sanitizedHost = sanitizeHost(host, REMOTE_SCHEME.HTTP) + const actorUrl = await loadActorUrlOrGetFromWebfinger(payload.name + '@' + sanitizedHost) + targetActor = await getOrCreateAPActor(actorUrl, 'all') + } + + if (payload.assertIsChannel && !targetActor.VideoChannel) { + logger.warn('Do not follow %s@%s because it is not a channel.', payload.name, host) + return + } + + const fromActor = await ActorModel.load(payload.followerActorId) + + return retryTransactionWrapper(follow, fromActor, targetActor, payload.isAutoFollow) +} +// --------------------------------------------------------------------------- + +export { + processActivityPubFollow +} + +// --------------------------------------------------------------------------- + +async function follow (fromActor: MActor, targetActor: MActorFull, isAutoFollow = false) { + if (fromActor.id === targetActor.id) { + throw new Error('Follower is the same as target actor.') + } + + // Same server, direct accept + const state = !fromActor.serverId && !targetActor.serverId ? 'accepted' : 'pending' + + const actorFollow = await sequelizeTypescript.transaction(async t => { + const [ actorFollow ] = await ActorFollowModel.findOrCreateCustom({ + byActor: fromActor, + state, + targetActor, + activityId: getLocalActorFollowActivityPubUrl(fromActor, targetActor), + transaction: t + }) + + // Send a notification to remote server if our follow is not already accepted + if (actorFollow.state !== 'accepted') sendFollow(actorFollow, t) + + return actorFollow + }) + + const followerFull = await ActorModel.loadFull(fromActor.id) + + const actorFollowFull = Object.assign(actorFollow, { + ActorFollowing: targetActor, + ActorFollower: followerFull + }) + + if (actorFollow.state === 'accepted') Notifier.Instance.notifyOfNewUserFollow(actorFollowFull) + if (isAutoFollow === true) Notifier.Instance.notifyOfAutoInstanceFollowing(actorFollowFull) + + return actorFollow +} diff --git a/server/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/server/lib/job-queue/handlers/activitypub-http-broadcast.ts new file mode 100644 index 000000000..f7307c97d --- /dev/null +++ b/server/server/lib/job-queue/handlers/activitypub-http-broadcast.ts @@ -0,0 +1,50 @@ +import { Job } from 'bullmq' +import { ActivitypubHttpBroadcastPayload } from '@peertube/peertube-models' +import { buildGlobalHTTPHeaders } from '@server/helpers/activity-pub-utils.js' +import { buildSignedRequestOptions, computeBody } from '@server/lib/activitypub/send/index.js' +import { ActorFollowHealthCache } from '@server/lib/actor-follow-health-cache.js' +import { parallelHTTPBroadcastFromWorker, sequentialHTTPBroadcastFromWorker } from '@server/lib/worker/parent-process.js' +import { logger } from '../../../helpers/logger.js' + +// Prefer using a worker thread for HTTP requests because on high load we may have to sign many requests, which can be CPU intensive + +async function processActivityPubHttpSequentialBroadcast (job: Job) { + logger.info('Processing ActivityPub broadcast in job %s.', job.id) + + const requestOptions = await buildRequestOptions(job.data) + + const { badUrls, goodUrls } = await sequentialHTTPBroadcastFromWorker({ uris: job.data.uris, requestOptions }) + + return ActorFollowHealthCache.Instance.updateActorFollowsHealth(goodUrls, badUrls) +} + +async function processActivityPubParallelHttpBroadcast (job: Job) { + logger.info('Processing ActivityPub parallel broadcast in job %s.', job.id) + + const requestOptions = await buildRequestOptions(job.data) + + const { badUrls, goodUrls } = await parallelHTTPBroadcastFromWorker({ uris: job.data.uris, requestOptions }) + + return ActorFollowHealthCache.Instance.updateActorFollowsHealth(goodUrls, badUrls) +} + +// --------------------------------------------------------------------------- + +export { + processActivityPubHttpSequentialBroadcast, + processActivityPubParallelHttpBroadcast +} + +// --------------------------------------------------------------------------- + +async function buildRequestOptions (payload: ActivitypubHttpBroadcastPayload) { + const body = await computeBody(payload) + const httpSignatureOptions = await buildSignedRequestOptions({ signatureActorId: payload.signatureActorId, hasPayload: true }) + + return { + method: 'POST' as 'POST', + json: body, + httpSignature: httpSignatureOptions, + headers: buildGlobalHTTPHeaders(body) + } +} diff --git a/server/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/server/lib/job-queue/handlers/activitypub-http-fetcher.ts new file mode 100644 index 000000000..649d23650 --- /dev/null +++ b/server/server/lib/job-queue/handlers/activitypub-http-fetcher.ts @@ -0,0 +1,41 @@ +import { Job } from 'bullmq' +import { ActivitypubHttpFetcherPayload, FetchType } from '@peertube/peertube-models' +import { logger } from '../../../helpers/logger.js' +import { VideoModel } from '../../../models/video/video.js' +import { VideoCommentModel } from '../../../models/video/video-comment.js' +import { VideoShareModel } from '../../../models/video/video-share.js' +import { MVideoFullLight } from '../../../types/models/index.js' +import { crawlCollectionPage } from '../../activitypub/crawl.js' +import { createAccountPlaylists } from '../../activitypub/playlists/index.js' +import { processActivities } from '../../activitypub/process/index.js' +import { addVideoShares } from '../../activitypub/share.js' +import { addVideoComments } from '../../activitypub/video-comments.js' + +async function processActivityPubHttpFetcher (job: Job) { + logger.info('Processing ActivityPub fetcher in job %s.', job.id) + + const payload = job.data as ActivitypubHttpFetcherPayload + + let video: MVideoFullLight + if (payload.videoId) video = await VideoModel.loadFull(payload.videoId) + + const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise } = { + 'activity': items => processActivities(items, { outboxUrl: payload.uri, fromFetch: true }), + 'video-shares': items => addVideoShares(items, video), + 'video-comments': items => addVideoComments(items), + 'account-playlists': items => createAccountPlaylists(items) + } + + const cleanerType: { [ id in FetchType ]?: (crawlStartDate: Date) => Promise } = { + 'video-shares': crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate), + 'video-comments': crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate) + } + + return crawlCollectionPage(payload.uri, fetcherType[payload.type], cleanerType[payload.type]) +} + +// --------------------------------------------------------------------------- + +export { + processActivityPubHttpFetcher +} diff --git a/server/server/lib/job-queue/handlers/activitypub-http-unicast.ts b/server/server/lib/job-queue/handlers/activitypub-http-unicast.ts new file mode 100644 index 000000000..9b2f542f0 --- /dev/null +++ b/server/server/lib/job-queue/handlers/activitypub-http-unicast.ts @@ -0,0 +1,39 @@ +import { Job } from 'bullmq' +import { ActivitypubHttpUnicastPayload } from '@peertube/peertube-models' +import { buildGlobalHTTPHeaders } from '@server/helpers/activity-pub-utils.js' +import { buildSignedRequestOptions, computeBody } from '@server/lib/activitypub/send/index.js' +import { logger } from '../../../helpers/logger.js' +import { doRequest } from '../../../helpers/requests.js' +import { ActorFollowHealthCache } from '../../actor-follow-health-cache.js' + +async function processActivityPubHttpUnicast (job: Job) { + logger.info('Processing ActivityPub unicast in job %s.', job.id) + + const payload = job.data as ActivitypubHttpUnicastPayload + const uri = payload.uri + + const body = await computeBody(payload) + const httpSignatureOptions = await buildSignedRequestOptions({ signatureActorId: payload.signatureActorId, hasPayload: true }) + + const options = { + method: 'POST' as 'POST', + json: body, + httpSignature: httpSignatureOptions, + headers: buildGlobalHTTPHeaders(body) + } + + try { + await doRequest(uri, options) + ActorFollowHealthCache.Instance.updateActorFollowsHealth([ uri ], []) + } catch (err) { + ActorFollowHealthCache.Instance.updateActorFollowsHealth([], [ uri ]) + + throw err + } +} + +// --------------------------------------------------------------------------- + +export { + processActivityPubHttpUnicast +} diff --git a/server/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/server/lib/job-queue/handlers/activitypub-refresher.ts new file mode 100644 index 000000000..ae238396e --- /dev/null +++ b/server/server/lib/job-queue/handlers/activitypub-refresher.ts @@ -0,0 +1,60 @@ +import { Job } from 'bullmq' +import { refreshVideoPlaylistIfNeeded } from '@server/lib/activitypub/playlists/index.js' +import { refreshVideoIfNeeded } from '@server/lib/activitypub/videos/index.js' +import { loadVideoByUrl } from '@server/lib/model-loaders/index.js' +import { RefreshPayload } from '@peertube/peertube-models' +import { logger } from '../../../helpers/logger.js' +import { ActorModel } from '../../../models/actor/actor.js' +import { VideoPlaylistModel } from '../../../models/video/video-playlist.js' +import { refreshActorIfNeeded } from '../../activitypub/actors/index.js' + +async function refreshAPObject (job: Job) { + const payload = job.data as RefreshPayload + + logger.info('Processing AP refresher in job %s for %s.', job.id, payload.url) + + if (payload.type === 'video') return refreshVideo(payload.url) + if (payload.type === 'video-playlist') return refreshVideoPlaylist(payload.url) + if (payload.type === 'actor') return refreshActor(payload.url) +} + +// --------------------------------------------------------------------------- + +export { + refreshAPObject +} + +// --------------------------------------------------------------------------- + +async function refreshVideo (videoUrl: string) { + const fetchType = 'all' as 'all' + const syncParam = { rates: true, shares: true, comments: true } + + const videoFromDatabase = await loadVideoByUrl(videoUrl, fetchType) + if (videoFromDatabase) { + const refreshOptions = { + video: videoFromDatabase, + fetchedType: fetchType, + syncParam + } + + await refreshVideoIfNeeded(refreshOptions) + } +} + +async function refreshActor (actorUrl: string) { + const fetchType = 'all' as 'all' + const actor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorUrl) + + if (actor) { + await refreshActorIfNeeded({ actor, fetchedType: fetchType }) + } +} + +async function refreshVideoPlaylist (playlistUrl: string) { + const playlist = await VideoPlaylistModel.loadByUrlAndPopulateAccount(playlistUrl) + + if (playlist) { + await refreshVideoPlaylistIfNeeded(playlist) + } +} diff --git a/server/server/lib/job-queue/handlers/actor-keys.ts b/server/server/lib/job-queue/handlers/actor-keys.ts new file mode 100644 index 000000000..40dfc6d49 --- /dev/null +++ b/server/server/lib/job-queue/handlers/actor-keys.ts @@ -0,0 +1,20 @@ +import { Job } from 'bullmq' +import { generateAndSaveActorKeys } from '@server/lib/activitypub/actors/index.js' +import { ActorModel } from '@server/models/actor/actor.js' +import { ActorKeysPayload } from '@peertube/peertube-models' +import { logger } from '../../../helpers/logger.js' + +async function processActorKeys (job: Job) { + const payload = job.data as ActorKeysPayload + logger.info('Processing actor keys in job %s.', job.id) + + const actor = await ActorModel.load(payload.actorId) + + await generateAndSaveActorKeys(actor) +} + +// --------------------------------------------------------------------------- + +export { + processActorKeys +} diff --git a/server/server/lib/job-queue/handlers/after-video-channel-import.ts b/server/server/lib/job-queue/handlers/after-video-channel-import.ts new file mode 100644 index 000000000..91d889dcb --- /dev/null +++ b/server/server/lib/job-queue/handlers/after-video-channel-import.ts @@ -0,0 +1,37 @@ +import { Job } from 'bullmq' +import { logger } from '@server/helpers/logger.js' +import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync.js' +import { AfterVideoChannelImportPayload, VideoChannelSyncState, VideoImportPreventExceptionResult } from '@peertube/peertube-models' + +export async function processAfterVideoChannelImport (job: Job) { + const payload = job.data as AfterVideoChannelImportPayload + if (!payload.channelSyncId) return + + logger.info('Processing after video channel import in job %s.', job.id) + + const sync = await VideoChannelSyncModel.loadWithChannel(payload.channelSyncId) + if (!sync) { + logger.error('Unknown sync id %d.', payload.channelSyncId) + return + } + + const childrenValues = await job.getChildrenValues() + + let errors = 0 + let successes = 0 + + for (const value of Object.values(childrenValues)) { + if (value.resultType === 'success') successes++ + else if (value.resultType === 'error') errors++ + } + + if (errors > 0) { + sync.state = VideoChannelSyncState.FAILED + logger.error(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" with failures.`, { errors, successes }) + } else { + sync.state = VideoChannelSyncState.SYNCED + logger.info(`Finished synchronizing "${sync.VideoChannel.Actor.preferredUsername}" successfully.`, { successes }) + } + + await sync.save() +} diff --git a/server/server/lib/job-queue/handlers/email.ts b/server/server/lib/job-queue/handlers/email.ts new file mode 100644 index 000000000..26b084e03 --- /dev/null +++ b/server/server/lib/job-queue/handlers/email.ts @@ -0,0 +1,17 @@ +import { Job } from 'bullmq' +import { EmailPayload } from '@peertube/peertube-models' +import { logger } from '../../../helpers/logger.js' +import { Emailer } from '../../emailer.js' + +async function processEmail (job: Job) { + const payload = job.data as EmailPayload + logger.info('Processing email in job %s.', job.id) + + return Emailer.Instance.sendMail(payload) +} + +// --------------------------------------------------------------------------- + +export { + processEmail +} diff --git a/server/server/lib/job-queue/handlers/federate-video.ts b/server/server/lib/job-queue/handlers/federate-video.ts new file mode 100644 index 000000000..270ce6cc5 --- /dev/null +++ b/server/server/lib/job-queue/handlers/federate-video.ts @@ -0,0 +1,28 @@ +import { Job } from 'bullmq' +import { retryTransactionWrapper } from '@server/helpers/database-utils.js' +import { sequelizeTypescript } from '@server/initializers/database.js' +import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js' +import { VideoModel } from '@server/models/video/video.js' +import { FederateVideoPayload } from '@peertube/peertube-models' +import { logger } from '../../../helpers/logger.js' + +function processFederateVideo (job: Job) { + const payload = job.data as FederateVideoPayload + + logger.info('Processing video federation in job %s.', job.id) + + return retryTransactionWrapper(() => { + return sequelizeTypescript.transaction(async t => { + const video = await VideoModel.loadFull(payload.videoUUID, t) + if (!video) return + + return federateVideoIfNeeded(video, payload.isNewVideo, t) + }) + }) +} + +// --------------------------------------------------------------------------- + +export { + processFederateVideo +} diff --git a/server/server/lib/job-queue/handlers/generate-storyboard.ts b/server/server/lib/job-queue/handlers/generate-storyboard.ts new file mode 100644 index 000000000..6f4d00dc5 --- /dev/null +++ b/server/server/lib/job-queue/handlers/generate-storyboard.ts @@ -0,0 +1,163 @@ +import { Job } from 'bullmq' +import { join } from 'path' +import { retryTransactionWrapper } from '@server/helpers/database-utils.js' +import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js' +import { generateImageFilename, getImageSize } from '@server/helpers/image-utils.js' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { deleteFileAndCatch } from '@server/helpers/utils.js' +import { CONFIG } from '@server/initializers/config.js' +import { STORYBOARD } from '@server/initializers/constants.js' +import { sequelizeTypescript } from '@server/initializers/database.js' +import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js' +import { VideoPathManager } from '@server/lib/video-path-manager.js' +import { StoryboardModel } from '@server/models/video/storyboard.js' +import { VideoModel } from '@server/models/video/video.js' +import { MVideo } from '@server/types/models/index.js' +import { FFmpegImage, isAudioFile } from '@peertube/peertube-ffmpeg' +import { GenerateStoryboardPayload } from '@peertube/peertube-models' + +const lTagsBase = loggerTagsFactory('storyboard') + +async function processGenerateStoryboard (job: Job): Promise { + const payload = job.data as GenerateStoryboardPayload + const lTags = lTagsBase(payload.videoUUID) + + logger.info('Processing generate storyboard of %s in job %s.', payload.videoUUID, job.id, lTags) + + const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID) + + try { + const video = await VideoModel.loadFull(payload.videoUUID) + if (!video) { + logger.info('Video %s does not exist anymore, skipping storyboard generation.', payload.videoUUID, lTags) + return + } + + const inputFile = video.getMaxQualityFile() + + await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => { + const isAudio = await isAudioFile(videoPath) + + if (isAudio) { + logger.info('Do not generate a storyboard of %s since the video does not have a video stream', payload.videoUUID, lTags) + return + } + + const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail')) + + const filename = generateImageFilename() + const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename) + + const totalSprites = buildTotalSprites(video) + if (totalSprites === 0) { + logger.info('Do not generate a storyboard of %s because the video is not long enough', payload.videoUUID, lTags) + return + } + + const spriteDuration = Math.round(video.duration / totalSprites) + + const spritesCount = findGridSize({ + toFind: totalSprites, + maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT + }) + + logger.debug( + 'Generating storyboard from video of %s to %s', video.uuid, destination, + { ...lTags, spritesCount, spriteDuration, videoDuration: video.duration } + ) + + await ffmpeg.generateStoryboardFromVideo({ + destination, + path: videoPath, + sprites: { + size: STORYBOARD.SPRITE_SIZE, + count: spritesCount, + duration: spriteDuration + } + }) + + const imageSize = await getImageSize(destination) + + await retryTransactionWrapper(() => { + return sequelizeTypescript.transaction(async transaction => { + const videoStillExists = await VideoModel.load(video.id, transaction) + if (!videoStillExists) { + logger.info('Video %s does not exist anymore, skipping storyboard generation.', payload.videoUUID, lTags) + deleteFileAndCatch(destination) + return + } + + const existing = await StoryboardModel.loadByVideo(video.id, transaction) + if (existing) await existing.destroy({ transaction }) + + await StoryboardModel.create({ + filename, + totalHeight: imageSize.height, + totalWidth: imageSize.width, + spriteHeight: STORYBOARD.SPRITE_SIZE.height, + spriteWidth: STORYBOARD.SPRITE_SIZE.width, + spriteDuration, + videoId: video.id + }, { transaction }) + + logger.info('Storyboard generation %s ended for video %s.', destination, video.uuid, lTags) + + if (payload.federate) { + await federateVideoIfNeeded(video, false, transaction) + } + }) + }) + }) + } finally { + inputFileMutexReleaser() + } +} + +// --------------------------------------------------------------------------- + +export { + processGenerateStoryboard +} + +function buildTotalSprites (video: MVideo) { + const maxSprites = STORYBOARD.SPRITE_SIZE.height * STORYBOARD.SPRITE_SIZE.width + const totalSprites = Math.min(Math.ceil(video.duration), maxSprites) + + // We can generate a single line + if (totalSprites <= STORYBOARD.SPRITES_MAX_EDGE_COUNT) return totalSprites + + return findGridFit(totalSprites, STORYBOARD.SPRITES_MAX_EDGE_COUNT) +} + +function findGridSize (options: { + toFind: number + maxEdgeCount: number +}) { + const { toFind, maxEdgeCount } = options + + for (let i = 1; i <= maxEdgeCount; i++) { + for (let j = i; j <= maxEdgeCount; j++) { + if (toFind === i * j) return { width: j, height: i } + } + } + + throw new Error(`Could not find grid size (to find: ${toFind}, max edge count: ${maxEdgeCount}`) +} + +function findGridFit (value: number, maxMultiplier: number) { + for (let i = value; i--; i > 0) { + if (!isPrimeWithin(i, maxMultiplier)) return i + } + + throw new Error('Could not find prime number below ' + value) +} + +function isPrimeWithin (value: number, maxMultiplier: number) { + if (value < 2) return false + + for (let i = 2, end = Math.min(Math.sqrt(value), maxMultiplier); i <= end; i++) { + if (value % i === 0 && value / i <= maxMultiplier) return false + } + + return true +} diff --git a/server/server/lib/job-queue/handlers/manage-video-torrent.ts b/server/server/lib/job-queue/handlers/manage-video-torrent.ts new file mode 100644 index 000000000..24bfc5acb --- /dev/null +++ b/server/server/lib/job-queue/handlers/manage-video-torrent.ts @@ -0,0 +1,110 @@ +import { Job } from 'bullmq' +import { extractVideo } from '@server/helpers/video.js' +import { createTorrentAndSetInfoHash, updateTorrentMetadata } from '@server/helpers/webtorrent.js' +import { VideoPathManager } from '@server/lib/video-path-manager.js' +import { VideoModel } from '@server/models/video/video.js' +import { VideoFileModel } from '@server/models/video/video-file.js' +import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js' +import { ManageVideoTorrentPayload } from '@peertube/peertube-models' +import { logger } from '../../../helpers/logger.js' + +async function processManageVideoTorrent (job: Job) { + const payload = job.data as ManageVideoTorrentPayload + logger.info('Processing torrent in job %s.', job.id) + + if (payload.action === 'create') return doCreateAction(payload) + if (payload.action === 'update-metadata') return doUpdateMetadataAction(payload) +} + +// --------------------------------------------------------------------------- + +export { + processManageVideoTorrent +} + +// --------------------------------------------------------------------------- + +async function doCreateAction (payload: ManageVideoTorrentPayload & { action: 'create' }) { + const [ video, file ] = await Promise.all([ + loadVideoOrLog(payload.videoId), + loadFileOrLog(payload.videoFileId) + ]) + + if (!video || !file) return + + const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) + + try { + await video.reload() + await file.reload() + + await createTorrentAndSetInfoHash(video, file) + + // Refresh videoFile because the createTorrentAndSetInfoHash could be long + const refreshedFile = await VideoFileModel.loadWithVideo(file.id) + // File does not exist anymore, remove the generated torrent + if (!refreshedFile) return file.removeTorrent() + + refreshedFile.infoHash = file.infoHash + refreshedFile.torrentFilename = file.torrentFilename + + await refreshedFile.save() + } finally { + fileMutexReleaser() + } +} + +async function doUpdateMetadataAction (payload: ManageVideoTorrentPayload & { action: 'update-metadata' }) { + const [ video, streamingPlaylist, file ] = await Promise.all([ + loadVideoOrLog(payload.videoId), + loadStreamingPlaylistOrLog(payload.streamingPlaylistId), + loadFileOrLog(payload.videoFileId) + ]) + + if ((!video && !streamingPlaylist) || !file) return + + const extractedVideo = extractVideo(video || streamingPlaylist) + const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(extractedVideo.uuid) + + try { + await updateTorrentMetadata(video || streamingPlaylist, file) + + await file.save() + } finally { + fileMutexReleaser() + } +} + +async function loadVideoOrLog (videoId: number) { + if (!videoId) return undefined + + const video = await VideoModel.load(videoId) + if (!video) { + logger.debug('Do not process torrent for video %d: does not exist anymore.', videoId) + } + + return video +} + +async function loadStreamingPlaylistOrLog (streamingPlaylistId: number) { + if (!streamingPlaylistId) return undefined + + const streamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(streamingPlaylistId) + if (!streamingPlaylist) { + logger.debug('Do not process torrent for streaming playlist %d: does not exist anymore.', streamingPlaylistId) + } + + return streamingPlaylist +} + +async function loadFileOrLog (videoFileId: number) { + if (!videoFileId) return undefined + + const file = await VideoFileModel.load(videoFileId) + + if (!file) { + logger.debug('Do not process torrent for file %d: does not exist anymore.', videoFileId) + } + + return file +} diff --git a/server/server/lib/job-queue/handlers/move-to-object-storage.ts b/server/server/lib/job-queue/handlers/move-to-object-storage.ts new file mode 100644 index 000000000..be3021247 --- /dev/null +++ b/server/server/lib/job-queue/handlers/move-to-object-storage.ts @@ -0,0 +1,159 @@ +import { Job } from 'bullmq' +import { remove } from 'fs-extra/esm' +import { join } from 'path' +import { MoveObjectStoragePayload, VideoStateType, VideoStorage } from '@peertube/peertube-models' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { updateTorrentMetadata } from '@server/helpers/webtorrent.js' +import { P2P_MEDIA_LOADER_PEER_VERSION } from '@server/initializers/constants.js' +import { storeHLSFileFromFilename, storeWebVideoFile } from '@server/lib/object-storage/index.js' +import { getHLSDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js' +import { VideoPathManager } from '@server/lib/video-path-manager.js' +import { moveToFailedMoveToObjectStorageState, moveToNextState } from '@server/lib/video-state.js' +import { VideoJobInfoModel } from '@server/models/video/video-job-info.js' +import { VideoModel } from '@server/models/video/video.js' +import { MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoWithAllFiles } from '@server/types/models/index.js' + +const lTagsBase = loggerTagsFactory('move-object-storage') + +export async function processMoveToObjectStorage (job: Job) { + const payload = job.data as MoveObjectStoragePayload + logger.info('Moving video %s in job %s.', payload.videoUUID, job.id) + + const fileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID) + + const video = await VideoModel.loadWithFiles(payload.videoUUID) + // No video, maybe deleted? + if (!video) { + logger.info('Can\'t process job %d, video does not exist.', job.id, lTagsBase(payload.videoUUID)) + fileMutexReleaser() + return undefined + } + + const lTags = lTagsBase(video.uuid, video.url) + + try { + if (video.VideoFiles) { + logger.debug('Moving %d web video files for video %s.', video.VideoFiles.length, video.uuid, lTags) + + await moveWebVideoFiles(video) + } + + if (video.VideoStreamingPlaylists) { + logger.debug('Moving HLS playlist of %s.', video.uuid) + + await moveHLSFiles(video) + } + + const pendingMove = await VideoJobInfoModel.decrease(video.uuid, 'pendingMove') + if (pendingMove === 0) { + logger.info('Running cleanup after moving files to object storage (video %s in job %s)', video.uuid, job.id, lTags) + + await doAfterLastJob({ video, previousVideoState: payload.previousVideoState, isNewVideo: payload.isNewVideo }) + } + } catch (err) { + await onMoveToObjectStorageFailure(job, err) + + throw err + } finally { + fileMutexReleaser() + } + + return payload.videoUUID +} + +export async function onMoveToObjectStorageFailure (job: Job, err: any) { + const payload = job.data as MoveObjectStoragePayload + + const video = await VideoModel.loadWithFiles(payload.videoUUID) + if (!video) return + + logger.error('Cannot move video %s to object storage.', video.url, { err, ...lTagsBase(video.uuid, video.url) }) + + await moveToFailedMoveToObjectStorageState(video) + await VideoJobInfoModel.abortAllTasks(video.uuid, 'pendingMove') +} + +// --------------------------------------------------------------------------- + +async function moveWebVideoFiles (video: MVideoWithAllFiles) { + for (const file of video.VideoFiles) { + if (file.storage !== VideoStorage.FILE_SYSTEM) continue + + const fileUrl = await storeWebVideoFile(video, file) + + const oldPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, file) + await onFileMoved({ videoOrPlaylist: video, file, fileUrl, oldPath }) + } +} + +async function moveHLSFiles (video: MVideoWithAllFiles) { + for (const playlist of video.VideoStreamingPlaylists) { + const playlistWithVideo = playlist.withVideo(video) + + for (const file of playlist.VideoFiles) { + if (file.storage !== VideoStorage.FILE_SYSTEM) continue + + // Resolution playlist + const playlistFilename = getHlsResolutionPlaylistFilename(file.filename) + await storeHLSFileFromFilename(playlistWithVideo, playlistFilename) + + // Resolution fragmented file + const fileUrl = await storeHLSFileFromFilename(playlistWithVideo, file.filename) + + const oldPath = join(getHLSDirectory(video), file.filename) + + await onFileMoved({ videoOrPlaylist: Object.assign(playlist, { Video: video }), file, fileUrl, oldPath }) + } + } +} + +async function doAfterLastJob (options: { + video: MVideoWithAllFiles + previousVideoState: VideoStateType + isNewVideo: boolean +}) { + const { video, previousVideoState, isNewVideo } = options + + for (const playlist of video.VideoStreamingPlaylists) { + if (playlist.storage === VideoStorage.OBJECT_STORAGE) continue + + const playlistWithVideo = playlist.withVideo(video) + + // Master playlist + playlist.playlistUrl = await storeHLSFileFromFilename(playlistWithVideo, playlist.playlistFilename) + // Sha256 segments file + playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlistWithVideo, playlist.segmentsSha256Filename) + + playlist.storage = VideoStorage.OBJECT_STORAGE + + playlist.assignP2PMediaLoaderInfoHashes(video, playlist.VideoFiles) + playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION + + await playlist.save() + } + + // Remove empty hls video directory + if (video.VideoStreamingPlaylists) { + await remove(getHLSDirectory(video)) + } + + await moveToNextState({ video, previousVideoState, isNewVideo }) +} + +async function onFileMoved (options: { + videoOrPlaylist: MVideo | MStreamingPlaylistVideo + file: MVideoFile + fileUrl: string + oldPath: string +}) { + const { videoOrPlaylist, file, fileUrl, oldPath } = options + + file.fileUrl = fileUrl + file.storage = VideoStorage.OBJECT_STORAGE + + await updateTorrentMetadata(videoOrPlaylist, file) + await file.save() + + logger.debug('Removing %s because it\'s now on object storage', oldPath) + await remove(oldPath) +} diff --git a/server/server/lib/job-queue/handlers/notify.ts b/server/server/lib/job-queue/handlers/notify.ts new file mode 100644 index 000000000..a09f43bf8 --- /dev/null +++ b/server/server/lib/job-queue/handlers/notify.ts @@ -0,0 +1,27 @@ +import { Job } from 'bullmq' +import { Notifier } from '@server/lib/notifier/index.js' +import { VideoModel } from '@server/models/video/video.js' +import { NotifyPayload } from '@peertube/peertube-models' +import { logger } from '../../../helpers/logger.js' + +async function processNotify (job: Job) { + const payload = job.data as NotifyPayload + logger.info('Processing %s notification in job %s.', payload.action, job.id) + + if (payload.action === 'new-video') return doNotifyNewVideo(payload) +} + +// --------------------------------------------------------------------------- + +export { + processNotify +} + +// --------------------------------------------------------------------------- + +async function doNotifyNewVideo (payload: NotifyPayload & { action: 'new-video' }) { + const refreshedVideo = await VideoModel.loadFull(payload.videoUUID) + if (!refreshedVideo) return + + Notifier.Instance.notifyOnNewVideoIfNeeded(refreshedVideo) +} diff --git a/server/server/lib/job-queue/handlers/transcoding-job-builder.ts b/server/server/lib/job-queue/handlers/transcoding-job-builder.ts new file mode 100644 index 000000000..9e29b86da --- /dev/null +++ b/server/server/lib/job-queue/handlers/transcoding-job-builder.ts @@ -0,0 +1,48 @@ +import { Job } from 'bullmq' +import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job.js' +import { UserModel } from '@server/models/user/user.js' +import { VideoJobInfoModel } from '@server/models/video/video-job-info.js' +import { VideoModel } from '@server/models/video/video.js' +import { pick } from '@peertube/peertube-core-utils' +import { TranscodingJobBuilderPayload } from '@peertube/peertube-models' +import { logger } from '../../../helpers/logger.js' +import { JobQueue } from '../job-queue.js' + +async function processTranscodingJobBuilder (job: Job) { + const payload = job.data as TranscodingJobBuilderPayload + + logger.info('Processing transcoding job builder in job %s.', job.id) + + if (payload.optimizeJob) { + const video = await VideoModel.loadFull(payload.videoUUID) + const user = await UserModel.loadByVideoId(video.id) + const videoFile = video.getMaxQualityFile() + + await createOptimizeOrMergeAudioJobs({ + ...pick(payload.optimizeJob, [ 'isNewVideo' ]), + + video, + videoFile, + user, + videoFileAlreadyLocked: false + }) + } + + for (const job of (payload.jobs || [])) { + await JobQueue.Instance.createJob(job) + + await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode') + } + + for (const sequentialJobs of (payload.sequentialJobs || [])) { + await JobQueue.Instance.createSequentialJobFlow(...sequentialJobs) + + await VideoJobInfoModel.increaseOrCreate(payload.videoUUID, 'pendingTranscode', sequentialJobs.filter(s => !!s).length) + } +} + +// --------------------------------------------------------------------------- + +export { + processTranscodingJobBuilder +} diff --git a/server/server/lib/job-queue/handlers/video-channel-import.ts b/server/server/lib/job-queue/handlers/video-channel-import.ts new file mode 100644 index 000000000..b5f87e11f --- /dev/null +++ b/server/server/lib/job-queue/handlers/video-channel-import.ts @@ -0,0 +1,43 @@ +import { Job } from 'bullmq' +import { logger } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' +import { synchronizeChannel } from '@server/lib/sync-channel.js' +import { VideoChannelModel } from '@server/models/video/video-channel.js' +import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync.js' +import { MChannelSync } from '@server/types/models/index.js' +import { VideoChannelImportPayload } from '@peertube/peertube-models' + +export async function processVideoChannelImport (job: Job) { + const payload = job.data as VideoChannelImportPayload + + logger.info('Processing video channel import in job %s.', job.id) + + // Channel import requires only http upload to be allowed + if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) { + throw new Error('Cannot import channel as the HTTP upload is disabled') + } + + if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) { + throw new Error('Cannot import channel as the synchronization is disabled') + } + + let channelSync: MChannelSync + if (payload.partOfChannelSyncId) { + channelSync = await VideoChannelSyncModel.loadWithChannel(payload.partOfChannelSyncId) + + if (!channelSync) { + throw new Error('Unlnown channel sync specified in videos channel import') + } + } + + const videoChannel = await VideoChannelModel.loadAndPopulateAccount(payload.videoChannelId) + + logger.info(`Starting importing videos from external channel "${payload.externalChannelUrl}" to "${videoChannel.name}" `) + + await synchronizeChannel({ + channel: videoChannel, + externalChannelUrl: payload.externalChannelUrl, + channelSync, + videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.FULL_SYNC_VIDEOS_LIMIT + }) +} diff --git a/server/server/lib/job-queue/handlers/video-file-import.ts b/server/server/lib/job-queue/handlers/video-file-import.ts new file mode 100644 index 000000000..899b5dac2 --- /dev/null +++ b/server/server/lib/job-queue/handlers/video-file-import.ts @@ -0,0 +1,84 @@ +import { Job } from 'bullmq' +import { copy } from 'fs-extra/esm' +import { stat } from 'fs/promises' +import { VideoFileImportPayload, VideoStorage } from '@peertube/peertube-models' +import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent.js' +import { CONFIG } from '@server/initializers/config.js' +import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js' +import { generateWebVideoFilename } from '@server/lib/paths.js' +import { VideoPathManager } from '@server/lib/video-path-manager.js' +import { buildMoveToObjectStorageJob } from '@server/lib/video.js' +import { VideoFileModel } from '@server/models/video/video-file.js' +import { VideoModel } from '@server/models/video/video.js' +import { MVideoFullLight } from '@server/types/models/index.js' +import { getLowercaseExtension } from '@peertube/peertube-node-utils' +import { getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@peertube/peertube-ffmpeg' +import { logger } from '../../../helpers/logger.js' +import { JobQueue } from '../job-queue.js' + +async function processVideoFileImport (job: Job) { + const payload = job.data as VideoFileImportPayload + logger.info('Processing video file import in job %s.', job.id) + + const video = await VideoModel.loadFull(payload.videoUUID) + // No video, maybe deleted? + if (!video) { + logger.info('Do not process job %d, video does not exist.', job.id) + return undefined + } + + await updateVideoFile(video, payload.filePath) + + if (CONFIG.OBJECT_STORAGE.ENABLED) { + await JobQueue.Instance.createJob(await buildMoveToObjectStorageJob({ video, previousVideoState: video.state })) + } else { + await federateVideoIfNeeded(video, false) + } + + return video +} + +// --------------------------------------------------------------------------- + +export { + processVideoFileImport +} + +// --------------------------------------------------------------------------- + +async function updateVideoFile (video: MVideoFullLight, inputFilePath: string) { + const { resolution } = await getVideoStreamDimensionsInfo(inputFilePath) + const { size } = await stat(inputFilePath) + const fps = await getVideoStreamFPS(inputFilePath) + + const fileExt = getLowercaseExtension(inputFilePath) + + const currentVideoFile = video.VideoFiles.find(videoFile => videoFile.resolution === resolution) + + if (currentVideoFile) { + // Remove old file and old torrent + await video.removeWebVideoFile(currentVideoFile) + // Remove the old video file from the array + video.VideoFiles = video.VideoFiles.filter(f => f !== currentVideoFile) + + await currentVideoFile.destroy() + } + + const newVideoFile = new VideoFileModel({ + resolution, + extname: fileExt, + filename: generateWebVideoFilename(resolution, fileExt), + storage: VideoStorage.FILE_SYSTEM, + size, + fps, + videoId: video.id + }) + + const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newVideoFile) + await copy(inputFilePath, outputPath) + + video.VideoFiles.push(newVideoFile) + await createTorrentAndSetInfoHash(video, newVideoFile) + + await newVideoFile.save() +} diff --git a/server/server/lib/job-queue/handlers/video-import.ts b/server/server/lib/job-queue/handlers/video-import.ts new file mode 100644 index 000000000..7d5435a3b --- /dev/null +++ b/server/server/lib/job-queue/handlers/video-import.ts @@ -0,0 +1,356 @@ +import { Job } from 'bullmq' +import { move, remove } from 'fs-extra/esm' +import { stat } from 'fs/promises' +import { + ThumbnailType, + ThumbnailType_Type, + VideoImportPayload, + VideoImportPreventExceptionResult, + VideoImportState, + VideoImportTorrentPayload, + VideoImportTorrentPayloadType, + VideoImportYoutubeDLPayload, + VideoImportYoutubeDLPayloadType, + VideoResolution, + VideoState +} from '@peertube/peertube-models' +import { retryTransactionWrapper } from '@server/helpers/database-utils.js' +import { YoutubeDLWrapper } from '@server/helpers/youtube-dl/index.js' +import { CONFIG } from '@server/initializers/config.js' +import { isPostImportVideoAccepted } from '@server/lib/moderation.js' +import { generateWebVideoFilename } from '@server/lib/paths.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { ServerConfigManager } from '@server/lib/server-config-manager.js' +import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job.js' +import { isAbleToUploadVideo } from '@server/lib/user.js' +import { VideoPathManager } from '@server/lib/video-path-manager.js' +import { buildNextVideoState } from '@server/lib/video-state.js' +import { buildMoveToObjectStorageJob } from '@server/lib/video.js' +import { ThumbnailModel } from '@server/models/video/thumbnail.js' +import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models/index.js' +import { MVideoImport, MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import.js' +import { getLowercaseExtension } from '@peertube/peertube-node-utils' +import { + ffprobePromise, + getVideoStreamDimensionsInfo, + getVideoStreamDuration, + getVideoStreamFPS, + isAudioFile +} from '@peertube/peertube-ffmpeg' +import { logger } from '../../../helpers/logger.js' +import { getSecureTorrentName } from '../../../helpers/utils.js' +import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent.js' +import { JOB_TTL } from '../../../initializers/constants.js' +import { sequelizeTypescript } from '../../../initializers/database.js' +import { VideoFileModel } from '../../../models/video/video-file.js' +import { VideoImportModel } from '../../../models/video/video-import.js' +import { VideoModel } from '../../../models/video/video.js' +import { federateVideoIfNeeded } from '../../activitypub/videos/index.js' +import { Notifier } from '../../notifier/index.js' +import { generateLocalVideoMiniature } from '../../thumbnail.js' +import { JobQueue } from '../job-queue.js' + +async function processVideoImport (job: Job): Promise { + const payload = job.data as VideoImportPayload + + const videoImport = await getVideoImportOrDie(payload) + if (videoImport.state === VideoImportState.CANCELLED) { + logger.info('Do not process import since it has been cancelled', { payload }) + return { resultType: 'success' } + } + + videoImport.state = VideoImportState.PROCESSING + await videoImport.save() + + try { + if (payload.type === 'youtube-dl') await processYoutubeDLImport(job, videoImport, payload) + if (payload.type === 'magnet-uri' || payload.type === 'torrent-file') await processTorrentImport(job, videoImport, payload) + + return { resultType: 'success' } + } catch (err) { + if (!payload.preventException) throw err + + logger.warn('Catch error in video import to send value to parent job.', { payload, err }) + return { resultType: 'error' } + } +} + +// --------------------------------------------------------------------------- + +export { + processVideoImport +} + +// --------------------------------------------------------------------------- + +async function processTorrentImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportTorrentPayload) { + logger.info('Processing torrent video import in job %s.', job.id) + + const options = { type: payload.type, videoImportId: payload.videoImportId } + + const target = { + torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined, + uri: videoImport.magnetUri + } + return processFile(() => downloadWebTorrentVideo(target, JOB_TTL['video-import']), videoImport, options) +} + +async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportYoutubeDLPayload) { + logger.info('Processing youtubeDL video import in job %s.', job.id) + + const options = { type: payload.type, videoImportId: videoImport.id } + + const youtubeDL = new YoutubeDLWrapper( + videoImport.targetUrl, + ServerConfigManager.Instance.getEnabledResolutions('vod'), + CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION + ) + + return processFile( + () => youtubeDL.downloadVideo(payload.fileExt, JOB_TTL['video-import']), + videoImport, + options + ) +} + +async function getVideoImportOrDie (payload: VideoImportPayload) { + const videoImport = await VideoImportModel.loadAndPopulateVideo(payload.videoImportId) + if (!videoImport?.Video) { + throw new Error(`Cannot import video ${payload.videoImportId}: the video import or video linked to this import does not exist anymore.`) + } + + return videoImport +} + +type ProcessFileOptions = { + type: VideoImportYoutubeDLPayloadType | VideoImportTorrentPayloadType + videoImportId: number +} +async function processFile (downloader: () => Promise, videoImport: MVideoImportDefault, options: ProcessFileOptions) { + let tempVideoPath: string + let videoFile: VideoFileModel + + try { + // Download video from youtubeDL + tempVideoPath = await downloader() + + // Get information about this video + const stats = await stat(tempVideoPath) + const isAble = await isAbleToUploadVideo(videoImport.User.id, stats.size) + if (isAble === false) { + throw new Error('The user video quota is exceeded with this video to import.') + } + + const probe = await ffprobePromise(tempVideoPath) + + const { resolution } = await isAudioFile(tempVideoPath, probe) + ? { resolution: VideoResolution.H_NOVIDEO } + : await getVideoStreamDimensionsInfo(tempVideoPath, probe) + + const fps = await getVideoStreamFPS(tempVideoPath, probe) + const duration = await getVideoStreamDuration(tempVideoPath, probe) + + // Prepare video file object for creation in database + const fileExt = getLowercaseExtension(tempVideoPath) + const videoFileData = { + extname: fileExt, + resolution, + size: stats.size, + filename: generateWebVideoFilename(resolution, fileExt), + fps, + videoId: videoImport.videoId + } + videoFile = new VideoFileModel(videoFileData) + + const hookName = options.type === 'youtube-dl' + ? 'filter:api.video.post-import-url.accept.result' + : 'filter:api.video.post-import-torrent.accept.result' + + // Check we accept this video + const acceptParameters = { + videoImport, + video: videoImport.Video, + videoFilePath: tempVideoPath, + videoFile, + user: videoImport.User + } + const acceptedResult = await Hooks.wrapFun(isPostImportVideoAccepted, acceptParameters, hookName) + + if (acceptedResult.accepted !== true) { + logger.info('Refused imported video.', { acceptedResult, acceptParameters }) + + videoImport.state = VideoImportState.REJECTED + await videoImport.save() + + throw new Error(acceptedResult.errorMessage) + } + + // Video is accepted, resuming preparation + const videoFileLockReleaser = await VideoPathManager.Instance.lockFiles(videoImport.Video.uuid) + + try { + const videoImportWithFiles = await refreshVideoImportFromDB(videoImport, videoFile) + + // Move file + const videoDestFile = VideoPathManager.Instance.getFSVideoFileOutputPath(videoImportWithFiles.Video, videoFile) + await move(tempVideoPath, videoDestFile) + + tempVideoPath = null // This path is not used anymore + + let { + miniatureModel: thumbnailModel, + miniatureJSONSave: thumbnailSave + } = await generateMiniature(videoImportWithFiles, videoFile, ThumbnailType.MINIATURE) + + let { + miniatureModel: previewModel, + miniatureJSONSave: previewSave + } = await generateMiniature(videoImportWithFiles, videoFile, ThumbnailType.PREVIEW) + + // Create torrent + await createTorrentAndSetInfoHash(videoImportWithFiles.Video, videoFile) + + const videoFileSave = videoFile.toJSON() + + const { videoImportUpdated, video } = await retryTransactionWrapper(() => { + return sequelizeTypescript.transaction(async t => { + // Refresh video + const video = await VideoModel.load(videoImportWithFiles.videoId, t) + if (!video) throw new Error('Video linked to import ' + videoImportWithFiles.videoId + ' does not exist anymore.') + + await videoFile.save({ transaction: t }) + + // Update video DB object + video.duration = duration + video.state = buildNextVideoState(video.state) + await video.save({ transaction: t }) + + if (thumbnailModel) await video.addAndSaveThumbnail(thumbnailModel, t) + if (previewModel) await video.addAndSaveThumbnail(previewModel, t) + + // Now we can federate the video (reload from database, we need more attributes) + const videoForFederation = await VideoModel.loadFull(video.uuid, t) + await federateVideoIfNeeded(videoForFederation, true, t) + + // Update video import object + videoImportWithFiles.state = VideoImportState.SUCCESS + const videoImportUpdated = await videoImportWithFiles.save({ transaction: t }) as MVideoImport + + logger.info('Video %s imported.', video.uuid) + + return { videoImportUpdated, video: videoForFederation } + }).catch(err => { + // Reset fields + if (thumbnailModel) thumbnailModel = new ThumbnailModel(thumbnailSave) + if (previewModel) previewModel = new ThumbnailModel(previewSave) + + videoFile = new VideoFileModel(videoFileSave) + + throw err + }) + }) + + await afterImportSuccess({ videoImport: videoImportUpdated, video, videoFile, user: videoImport.User, videoFileAlreadyLocked: true }) + } finally { + videoFileLockReleaser() + } + } catch (err) { + await onImportError(err, tempVideoPath, videoImport) + + throw err + } +} + +async function refreshVideoImportFromDB (videoImport: MVideoImportDefault, videoFile: MVideoFile): Promise { + // Refresh video, privacy may have changed + const video = await videoImport.Video.reload() + const videoWithFiles = Object.assign(video, { VideoFiles: [ videoFile ], VideoStreamingPlaylists: [] }) + + return Object.assign(videoImport, { Video: videoWithFiles }) +} + +async function generateMiniature ( + videoImportWithFiles: MVideoImportDefaultFiles, + videoFile: MVideoFile, + thumbnailType: ThumbnailType_Type +) { + // Generate miniature if the import did not created it + const needsMiniature = thumbnailType === ThumbnailType.MINIATURE + ? !videoImportWithFiles.Video.getMiniature() + : !videoImportWithFiles.Video.getPreview() + + if (!needsMiniature) { + return { + miniatureModel: null, + miniatureJSONSave: null + } + } + + const miniatureModel = await generateLocalVideoMiniature({ + video: videoImportWithFiles.Video, + videoFile, + type: thumbnailType + }) + const miniatureJSONSave = miniatureModel.toJSON() + + return { + miniatureModel, + miniatureJSONSave + } +} + +async function afterImportSuccess (options: { + videoImport: MVideoImport + video: MVideoFullLight + videoFile: MVideoFile + user: MUserId + videoFileAlreadyLocked: boolean +}) { + const { video, videoFile, videoImport, user, videoFileAlreadyLocked } = options + + Notifier.Instance.notifyOnFinishedVideoImport({ videoImport: Object.assign(videoImport, { Video: video }), success: true }) + + if (video.isBlacklisted()) { + const videoBlacklist = Object.assign(video.VideoBlacklist, { Video: video }) + + Notifier.Instance.notifyOnVideoAutoBlacklist(videoBlacklist) + } else { + Notifier.Instance.notifyOnNewVideoIfNeeded(video) + } + + // Generate the storyboard in the job queue, and don't forget to federate an update after + await JobQueue.Instance.createJob({ + type: 'generate-video-storyboard' as 'generate-video-storyboard', + payload: { + videoUUID: video.uuid, + federate: true + } + }) + + if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { + await JobQueue.Instance.createJob( + await buildMoveToObjectStorageJob({ video, previousVideoState: VideoState.TO_IMPORT }) + ) + return + } + + if (video.state === VideoState.TO_TRANSCODE) { // Create transcoding jobs? + await createOptimizeOrMergeAudioJobs({ video, videoFile, isNewVideo: true, user, videoFileAlreadyLocked }) + } +} + +async function onImportError (err: Error, tempVideoPath: string, videoImport: MVideoImportVideo) { + try { + if (tempVideoPath) await remove(tempVideoPath) + } catch (errUnlink) { + logger.warn('Cannot cleanup files after a video import error.', { err: errUnlink }) + } + + videoImport.error = err.message + if (videoImport.state !== VideoImportState.REJECTED) { + videoImport.state = VideoImportState.FAILED + } + await videoImport.save() + + Notifier.Instance.notifyOnFinishedVideoImport({ videoImport, success: false }) +} diff --git a/server/server/lib/job-queue/handlers/video-live-ending.ts b/server/server/lib/job-queue/handlers/video-live-ending.ts new file mode 100644 index 000000000..0b4a4fd8b --- /dev/null +++ b/server/server/lib/job-queue/handlers/video-live-ending.ts @@ -0,0 +1,280 @@ +import { Job } from 'bullmq' +import { remove } from 'fs-extra/esm' +import { readdir } from 'fs/promises' +import { join } from 'path' +import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@peertube/peertube-models' +import { peertubeTruncate } from '@server/helpers/core-utils.js' +import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js' +import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url.js' +import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js' +import { cleanupAndDestroyPermanentLive, cleanupTMPLiveFiles, cleanupUnsavedNormalLive } from '@server/lib/live/index.js' +import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getLiveReplayBaseDirectory } from '@server/lib/paths.js' +import { generateLocalVideoMiniature, regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail.js' +import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/hls-transcoding.js' +import { VideoPathManager } from '@server/lib/video-path-manager.js' +import { moveToNextState } from '@server/lib/video-state.js' +import { VideoBlacklistModel } from '@server/models/video/video-blacklist.js' +import { VideoFileModel } from '@server/models/video/video-file.js' +import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting.js' +import { VideoLiveSessionModel } from '@server/models/video/video-live-session.js' +import { VideoLiveModel } from '@server/models/video/video-live.js' +import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js' +import { VideoModel } from '@server/models/video/video.js' +import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@server/types/models/index.js' +import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@peertube/peertube-ffmpeg' +import { logger, loggerTagsFactory } from '../../../helpers/logger.js' +import { JobQueue } from '../job-queue.js' + +const lTags = loggerTagsFactory('live', 'job') + +async function processVideoLiveEnding (job: Job) { + const payload = job.data as VideoLiveEndingPayload + + logger.info('Processing video live ending for %s.', payload.videoId, { payload, ...lTags() }) + + function logError () { + logger.warn('Video live %d does not exist anymore. Cannot process live ending.', payload.videoId, lTags()) + } + + const video = await VideoModel.load(payload.videoId) + const live = await VideoLiveModel.loadByVideoId(payload.videoId) + const liveSession = await VideoLiveSessionModel.load(payload.liveSessionId) + + if (!video || !live || !liveSession) { + logError() + return + } + + const permanentLive = live.permanentLive + + liveSession.endingProcessed = true + await liveSession.save() + + if (liveSession.saveReplay !== true) { + return cleanupLiveAndFederate({ permanentLive, video, streamingPlaylistId: payload.streamingPlaylistId }) + } + + if (permanentLive) { + await saveReplayToExternalVideo({ + liveVideo: video, + liveSession, + publishedAt: payload.publishedAt, + replayDirectory: payload.replayDirectory + }) + + return cleanupLiveAndFederate({ permanentLive, video, streamingPlaylistId: payload.streamingPlaylistId }) + } + + return replaceLiveByReplay({ + video, + liveSession, + live, + permanentLive, + replayDirectory: payload.replayDirectory + }) +} + +// --------------------------------------------------------------------------- + +export { + processVideoLiveEnding +} + +// --------------------------------------------------------------------------- + +async function saveReplayToExternalVideo (options: { + liveVideo: MVideo + liveSession: MVideoLiveSession + publishedAt: string + replayDirectory: string +}) { + const { liveVideo, liveSession, publishedAt, replayDirectory } = options + + const replaySettings = await VideoLiveReplaySettingModel.load(liveSession.replaySettingId) + + const videoNameSuffix = ` - ${new Date(publishedAt).toLocaleString()}` + const truncatedVideoName = peertubeTruncate(liveVideo.name, { + length: CONSTRAINTS_FIELDS.VIDEOS.NAME.max - videoNameSuffix.length + }) + + const replayVideo = new VideoModel({ + name: truncatedVideoName + videoNameSuffix, + isLive: false, + state: VideoState.TO_TRANSCODE, + duration: 0, + + remote: liveVideo.remote, + category: liveVideo.category, + licence: liveVideo.licence, + language: liveVideo.language, + commentsEnabled: liveVideo.commentsEnabled, + downloadEnabled: liveVideo.downloadEnabled, + waitTranscoding: true, + nsfw: liveVideo.nsfw, + description: liveVideo.description, + support: liveVideo.support, + privacy: replaySettings.privacy, + channelId: liveVideo.channelId + }) as MVideoWithAllFiles + + replayVideo.Thumbnails = [] + replayVideo.VideoFiles = [] + replayVideo.VideoStreamingPlaylists = [] + + replayVideo.url = getLocalVideoActivityPubUrl(replayVideo) + + await replayVideo.save() + + liveSession.replayVideoId = replayVideo.id + await liveSession.save() + + // If live is blacklisted, also blacklist the replay + const blacklist = await VideoBlacklistModel.loadByVideoId(liveVideo.id) + if (blacklist) { + await VideoBlacklistModel.create({ + videoId: replayVideo.id, + unfederated: blacklist.unfederated, + reason: blacklist.reason, + type: blacklist.type + }) + } + + await assignReplayFilesToVideo({ video: replayVideo, replayDirectory }) + + await remove(replayDirectory) + + for (const type of [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ]) { + const image = await generateLocalVideoMiniature({ video: replayVideo, videoFile: replayVideo.getMaxQualityFile(), type }) + await replayVideo.addAndSaveThumbnail(image) + } + + await moveToNextState({ video: replayVideo, isNewVideo: true }) + + await createStoryboardJob(replayVideo) +} + +async function replaceLiveByReplay (options: { + video: MVideo + liveSession: MVideoLiveSession + live: MVideoLive + permanentLive: boolean + replayDirectory: string +}) { + const { video, liveSession, live, permanentLive, replayDirectory } = options + + const replaySettings = await VideoLiveReplaySettingModel.load(liveSession.replaySettingId) + const videoWithFiles = await VideoModel.loadFull(video.id) + const hlsPlaylist = videoWithFiles.getHLSPlaylist() + + await cleanupTMPLiveFiles(videoWithFiles, hlsPlaylist) + + await live.destroy() + + videoWithFiles.isLive = false + videoWithFiles.privacy = replaySettings.privacy + videoWithFiles.waitTranscoding = true + videoWithFiles.state = VideoState.TO_TRANSCODE + + await videoWithFiles.save() + + liveSession.replayVideoId = videoWithFiles.id + await liveSession.save() + + await VideoFileModel.removeHLSFilesOfVideoId(hlsPlaylist.id) + + // Reset playlist + hlsPlaylist.VideoFiles = [] + hlsPlaylist.playlistFilename = generateHLSMasterPlaylistFilename() + hlsPlaylist.segmentsSha256Filename = generateHlsSha256SegmentsFilename() + await hlsPlaylist.save() + + await assignReplayFilesToVideo({ video: videoWithFiles, replayDirectory }) + + // Should not happen in this function, but we keep the code if in the future we can replace the permanent live by a replay + if (permanentLive) { // Remove session replay + await remove(replayDirectory) + } else { // We won't stream again in this live, we can delete the base replay directory + await remove(getLiveReplayBaseDirectory(videoWithFiles)) + } + + // Regenerate the thumbnail & preview? + await regenerateMiniaturesIfNeeded(videoWithFiles) + + // We consider this is a new video + await moveToNextState({ video: videoWithFiles, isNewVideo: true }) + + await createStoryboardJob(videoWithFiles) +} + +async function assignReplayFilesToVideo (options: { + video: MVideo + replayDirectory: string +}) { + const { video, replayDirectory } = options + + const concatenatedTsFiles = await readdir(replayDirectory) + + for (const concatenatedTsFile of concatenatedTsFiles) { + const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) + await video.reload() + + const concatenatedTsFilePath = join(replayDirectory, concatenatedTsFile) + + const probe = await ffprobePromise(concatenatedTsFilePath) + const { audioStream } = await getAudioStream(concatenatedTsFilePath, probe) + const { resolution } = await getVideoStreamDimensionsInfo(concatenatedTsFilePath, probe) + const fps = await getVideoStreamFPS(concatenatedTsFilePath, probe) + + try { + await generateHlsPlaylistResolutionFromTS({ + video, + inputFileMutexReleaser, + concatenatedTsFilePath, + resolution, + fps, + isAAC: audioStream?.codec_name === 'aac' + }) + } catch (err) { + logger.error('Cannot generate HLS playlist resolution from TS files.', { err }) + } + + inputFileMutexReleaser() + } + + return video +} + +async function cleanupLiveAndFederate (options: { + video: MVideo + permanentLive: boolean + streamingPlaylistId: number +}) { + const { permanentLive, video, streamingPlaylistId } = options + + const streamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(streamingPlaylistId) + + if (streamingPlaylist) { + if (permanentLive) { + await cleanupAndDestroyPermanentLive(video, streamingPlaylist) + } else { + await cleanupUnsavedNormalLive(video, streamingPlaylist) + } + } + + try { + const fullVideo = await VideoModel.loadFull(video.id) + return federateVideoIfNeeded(fullVideo, false, undefined) + } catch (err) { + logger.warn('Cannot federate live after cleanup', { videoId: video.id, err }) + } +} + +function createStoryboardJob (video: MVideo) { + return JobQueue.Instance.createJob({ + type: 'generate-video-storyboard' as 'generate-video-storyboard', + payload: { + videoUUID: video.uuid, + federate: true + } + }) +} diff --git a/server/server/lib/job-queue/handlers/video-redundancy.ts b/server/server/lib/job-queue/handlers/video-redundancy.ts new file mode 100644 index 000000000..b4c96219c --- /dev/null +++ b/server/server/lib/job-queue/handlers/video-redundancy.ts @@ -0,0 +1,17 @@ +import { Job } from 'bullmq' +import { VideosRedundancyScheduler } from '@server/lib/schedulers/videos-redundancy-scheduler.js' +import { VideoRedundancyPayload } from '@peertube/peertube-models' +import { logger } from '../../../helpers/logger.js' + +async function processVideoRedundancy (job: Job) { + const payload = job.data as VideoRedundancyPayload + logger.info('Processing video redundancy in job %s.', job.id) + + return VideosRedundancyScheduler.Instance.createManualRedundancy(payload.videoId) +} + +// --------------------------------------------------------------------------- + +export { + processVideoRedundancy +} diff --git a/server/server/lib/job-queue/handlers/video-studio-edition.ts b/server/server/lib/job-queue/handlers/video-studio-edition.ts new file mode 100644 index 000000000..3ddf0fa82 --- /dev/null +++ b/server/server/lib/job-queue/handlers/video-studio-edition.ts @@ -0,0 +1,180 @@ +import { Job } from 'bullmq' +import { remove } from 'fs-extra/esm' +import { join } from 'path' +import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js' +import { CONFIG } from '@server/initializers/config.js' +import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles.js' +import { isAbleToUploadVideo } from '@server/lib/user.js' +import { VideoPathManager } from '@server/lib/video-path-manager.js' +import { approximateIntroOutroAdditionalSize, onVideoStudioEnded, safeCleanupStudioTMPFiles } from '@server/lib/video-studio.js' +import { UserModel } from '@server/models/user/user.js' +import { VideoModel } from '@server/models/video/video.js' +import { MVideo, MVideoFullLight } from '@server/types/models/index.js' +import { pick } from '@peertube/peertube-core-utils' +import { buildUUID } from '@peertube/peertube-node-utils' +import { FFmpegEdition } from '@peertube/peertube-ffmpeg' +import { + VideoStudioEditionPayload, + VideoStudioTask, + VideoStudioTaskCutPayload, + VideoStudioTaskIntroPayload, + VideoStudioTaskOutroPayload, + VideoStudioTaskPayload, + VideoStudioTaskWatermarkPayload +} from '@peertube/peertube-models' +import { logger, loggerTagsFactory } from '../../../helpers/logger.js' + +const lTagsBase = loggerTagsFactory('video-studio') + +async function processVideoStudioEdition (job: Job) { + const payload = job.data as VideoStudioEditionPayload + const lTags = lTagsBase(payload.videoUUID) + + logger.info('Process video studio edition of %s in job %s.', payload.videoUUID, job.id, lTags) + + try { + const video = await VideoModel.loadFull(payload.videoUUID) + + // No video, maybe deleted? + if (!video) { + logger.info('Can\'t process job %d, video does not exist.', job.id, lTags) + + await safeCleanupStudioTMPFiles(payload.tasks) + return undefined + } + + await checkUserQuotaOrThrow(video, payload) + + const inputFile = video.getMaxQualityFile() + + const editionResultPath = await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async originalFilePath => { + let tmpInputFilePath: string + let outputPath: string + + for (const task of payload.tasks) { + const outputFilename = buildUUID() + inputFile.extname + outputPath = join(CONFIG.STORAGE.TMP_DIR, outputFilename) + + await processTask({ + inputPath: tmpInputFilePath ?? originalFilePath, + video, + outputPath, + task, + lTags + }) + + if (tmpInputFilePath) await remove(tmpInputFilePath) + + // For the next iteration + tmpInputFilePath = outputPath + } + + return outputPath + }) + + logger.info('Video edition ended for video %s.', video.uuid, lTags) + + await onVideoStudioEnded({ video, editionResultPath, tasks: payload.tasks }) + } catch (err) { + await safeCleanupStudioTMPFiles(payload.tasks) + + throw err + } +} + +// --------------------------------------------------------------------------- + +export { + processVideoStudioEdition +} + +// --------------------------------------------------------------------------- + +type TaskProcessorOptions = { + inputPath: string + outputPath: string + video: MVideo + task: T + lTags: { tags: string[] } +} + +const taskProcessors: { [id in VideoStudioTask['name']]: (options: TaskProcessorOptions) => Promise } = { + 'add-intro': processAddIntroOutro, + 'add-outro': processAddIntroOutro, + 'cut': processCut, + 'add-watermark': processAddWatermark +} + +async function processTask (options: TaskProcessorOptions) { + const { video, task, lTags } = options + + logger.info('Processing %s task for video %s.', task.name, video.uuid, { task, ...lTags }) + + const processor = taskProcessors[options.task.name] + if (!process) throw new Error('Unknown task ' + task.name) + + return processor(options) +} + +function processAddIntroOutro (options: TaskProcessorOptions) { + const { task, lTags } = options + + logger.debug('Will add intro/outro to the video.', { options, ...lTags }) + + return buildFFmpegEdition().addIntroOutro({ + ...pick(options, [ 'inputPath', 'outputPath' ]), + + introOutroPath: task.options.file, + type: task.name === 'add-intro' + ? 'intro' + : 'outro' + }) +} + +function processCut (options: TaskProcessorOptions) { + const { task, lTags } = options + + logger.debug('Will cut the video.', { options, ...lTags }) + + return buildFFmpegEdition().cutVideo({ + ...pick(options, [ 'inputPath', 'outputPath' ]), + + start: task.options.start, + end: task.options.end + }) +} + +function processAddWatermark (options: TaskProcessorOptions) { + const { task, lTags } = options + + logger.debug('Will add watermark to the video.', { options, ...lTags }) + + return buildFFmpegEdition().addWatermark({ + ...pick(options, [ 'inputPath', 'outputPath' ]), + + watermarkPath: task.options.file, + + videoFilters: { + watermarkSizeRatio: task.options.watermarkSizeRatio, + horitonzalMarginRatio: task.options.horitonzalMarginRatio, + verticalMarginRatio: task.options.verticalMarginRatio + } + }) +} + +// --------------------------------------------------------------------------- + +async function checkUserQuotaOrThrow (video: MVideoFullLight, payload: VideoStudioEditionPayload) { + const user = await UserModel.loadByVideoId(video.id) + + const filePathFinder = (i: number) => (payload.tasks[i] as VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload).options.file + + const additionalBytes = await approximateIntroOutroAdditionalSize(video, payload.tasks, filePathFinder) + if (await isAbleToUploadVideo(user.id, additionalBytes) === false) { + throw new Error('Quota exceeded for this user to edit the video') + } +} + +function buildFFmpegEdition () { + return new FFmpegEdition(getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders())) +} diff --git a/server/server/lib/job-queue/handlers/video-transcoding.ts b/server/server/lib/job-queue/handlers/video-transcoding.ts new file mode 100644 index 000000000..f46cf9a80 --- /dev/null +++ b/server/server/lib/job-queue/handlers/video-transcoding.ts @@ -0,0 +1,150 @@ +import { Job } from 'bullmq' +import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding.js' +import { generateHlsPlaylistResolution } from '@server/lib/transcoding/hls-transcoding.js' +import { mergeAudioVideofile, optimizeOriginalVideofile, transcodeNewWebVideoResolution } from '@server/lib/transcoding/web-transcoding.js' +import { removeAllWebVideoFiles } from '@server/lib/video-file.js' +import { VideoPathManager } from '@server/lib/video-path-manager.js' +import { moveToFailedTranscodingState } from '@server/lib/video-state.js' +import { UserModel } from '@server/models/user/user.js' +import { VideoJobInfoModel } from '@server/models/video/video-job-info.js' +import { MUser, MUserId, MVideoFullLight } from '@server/types/models/index.js' +import { + HLSTranscodingPayload, + MergeAudioTranscodingPayload, + NewWebVideoResolutionTranscodingPayload, + OptimizeTranscodingPayload, + VideoTranscodingPayload +} from '@peertube/peertube-models' +import { logger, loggerTagsFactory } from '../../../helpers/logger.js' +import { VideoModel } from '../../../models/video/video.js' + +type HandlerFunction = (job: Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise + +const handlers: { [ id in VideoTranscodingPayload['type'] ]: HandlerFunction } = { + 'new-resolution-to-hls': handleHLSJob, + 'new-resolution-to-web-video': handleNewWebVideoResolutionJob, + 'merge-audio-to-web-video': handleWebVideoMergeAudioJob, + 'optimize-to-web-video': handleWebVideoOptimizeJob +} + +const lTags = loggerTagsFactory('transcoding') + +async function processVideoTranscoding (job: Job) { + const payload = job.data as VideoTranscodingPayload + logger.info('Processing transcoding job %s.', job.id, lTags(payload.videoUUID)) + + const video = await VideoModel.loadFull(payload.videoUUID) + // No video, maybe deleted? + if (!video) { + logger.info('Do not process job %d, video does not exist.', job.id, lTags(payload.videoUUID)) + return undefined + } + + const user = await UserModel.loadByChannelActorId(video.VideoChannel.actorId) + + const handler = handlers[payload.type] + + if (!handler) { + await moveToFailedTranscodingState(video) + await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') + + throw new Error('Cannot find transcoding handler for ' + payload.type) + } + + try { + await handler(job, payload, video, user) + } catch (error) { + await moveToFailedTranscodingState(video) + + await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') + + throw error + } + + return video +} + +// --------------------------------------------------------------------------- + +export { + processVideoTranscoding +} + +// --------------------------------------------------------------------------- +// Job handlers +// --------------------------------------------------------------------------- + +async function handleWebVideoMergeAudioJob (job: Job, payload: MergeAudioTranscodingPayload, video: MVideoFullLight, user: MUserId) { + logger.info('Handling merge audio transcoding job for %s.', video.uuid, lTags(video.uuid), { payload }) + + await mergeAudioVideofile({ video, resolution: payload.resolution, fps: payload.fps, job }) + + logger.info('Merge audio transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload }) + + await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: !payload.hasChildren, video }) +} + +async function handleWebVideoOptimizeJob (job: Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) { + logger.info('Handling optimize transcoding job for %s.', video.uuid, lTags(video.uuid), { payload }) + + await optimizeOriginalVideofile({ video, inputVideoFile: video.getMaxQualityFile(), quickTranscode: payload.quickTranscode, job }) + + logger.info('Optimize transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload }) + + await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: !payload.hasChildren, video }) +} + +// --------------------------------------------------------------------------- + +async function handleNewWebVideoResolutionJob (job: Job, payload: NewWebVideoResolutionTranscodingPayload, video: MVideoFullLight) { + logger.info('Handling Web Video transcoding job for %s.', video.uuid, lTags(video.uuid), { payload }) + + await transcodeNewWebVideoResolution({ video, resolution: payload.resolution, fps: payload.fps, job }) + + logger.info('Web Video transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload }) + + await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video }) +} + +// --------------------------------------------------------------------------- + +async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, videoArg: MVideoFullLight) { + logger.info('Handling HLS transcoding job for %s.', videoArg.uuid, lTags(videoArg.uuid), { payload }) + + const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid) + let video: MVideoFullLight + + try { + video = await VideoModel.loadFull(videoArg.uuid) + + const videoFileInput = payload.copyCodecs + ? video.getWebVideoFile(payload.resolution) + : video.getMaxQualityFile() + + const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist() + + await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => { + return generateHlsPlaylistResolution({ + video, + videoInputPath, + inputFileMutexReleaser, + resolution: payload.resolution, + fps: payload.fps, + copyCodecs: payload.copyCodecs, + job + }) + }) + } finally { + inputFileMutexReleaser() + } + + logger.info('HLS transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload }) + + if (payload.deleteWebVideoFiles === true) { + logger.info('Removing Web Video files of %s now we have a HLS version of it.', video.uuid, lTags(video.uuid)) + + await removeAllWebVideoFiles(video) + } + + await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video }) +} diff --git a/server/server/lib/job-queue/handlers/video-views-stats.ts b/server/server/lib/job-queue/handlers/video-views-stats.ts new file mode 100644 index 000000000..479b47ed0 --- /dev/null +++ b/server/server/lib/job-queue/handlers/video-views-stats.ts @@ -0,0 +1,57 @@ +import { isTestOrDevInstance } from '@peertube/peertube-node-utils' +import { VideoViewModel } from '@server/models/view/video-view.js' +import { logger } from '../../../helpers/logger.js' +import { VideoModel } from '../../../models/video/video.js' +import { Redis } from '../../redis.js' + +async function processVideosViewsStats () { + const lastHour = new Date() + + // In test mode, we run this function multiple times per hour, so we don't want the values of the previous hour + if (!isTestOrDevInstance()) lastHour.setHours(lastHour.getHours() - 1) + + const hour = lastHour.getHours() + const startDate = lastHour.setMinutes(0, 0, 0) + const endDate = lastHour.setMinutes(59, 59, 999) + + const videoIds = await Redis.Instance.listVideosViewedForStats(hour) + if (videoIds.length === 0) return + + logger.info('Processing videos views stats in job for hour %d.', hour) + + for (const videoId of videoIds) { + try { + const views = await Redis.Instance.getVideoViewsStats(videoId, hour) + await Redis.Instance.deleteVideoViewsStats(videoId, hour) + + if (views) { + logger.debug('Adding %d views to video %d stats in hour %d.', views, videoId, hour) + + try { + const video = await VideoModel.load(videoId) + if (!video) { + logger.debug('Video %d does not exist anymore, skipping videos view stats.', videoId) + continue + } + + await VideoViewModel.create({ + startDate: new Date(startDate), + endDate: new Date(endDate), + views, + videoId + }) + } catch (err) { + logger.error('Cannot create video views stats for video %d in hour %d.', videoId, hour, { err }) + } + } + } catch (err) { + logger.error('Cannot update video views stats of video %d in hour %d.', videoId, hour, { err }) + } + } +} + +// --------------------------------------------------------------------------- + +export { + processVideosViewsStats +} diff --git a/server/server/lib/job-queue/index.ts b/server/server/lib/job-queue/index.ts new file mode 100644 index 000000000..098d4e7a3 --- /dev/null +++ b/server/server/lib/job-queue/index.ts @@ -0,0 +1 @@ +export * from './job-queue.js' diff --git a/server/server/lib/job-queue/job-queue.ts b/server/server/lib/job-queue/job-queue.ts new file mode 100644 index 000000000..390da209e --- /dev/null +++ b/server/server/lib/job-queue/job-queue.ts @@ -0,0 +1,540 @@ +import { + FlowJob, + FlowProducer, + Job, + JobsOptions, + Queue, + QueueEvents, + QueueEventsOptions, + QueueOptions, + Worker, + WorkerOptions +} from 'bullmq' +import { pick, timeoutPromise } from '@peertube/peertube-core-utils' +import { + ActivitypubFollowPayload, + ActivitypubHttpBroadcastPayload, + ActivitypubHttpFetcherPayload, + ActivitypubHttpUnicastPayload, + ActorKeysPayload, + AfterVideoChannelImportPayload, + DeleteResumableUploadMetaFilePayload, + EmailPayload, + FederateVideoPayload, + GenerateStoryboardPayload, + JobState, + JobType, + ManageVideoTorrentPayload, + MoveObjectStoragePayload, + NotifyPayload, + RefreshPayload, + TranscodingJobBuilderPayload, + VideoChannelImportPayload, + VideoFileImportPayload, + VideoImportPayload, + VideoLiveEndingPayload, + VideoRedundancyPayload, + VideoStudioEditionPayload, + VideoTranscodingPayload +} from '@peertube/peertube-models' +import { parseDurationToMs } from '@server/helpers/core-utils.js' +import { jobStates } from '@server/helpers/custom-validators/jobs.js' +import { CONFIG } from '@server/initializers/config.js' +import { processVideoRedundancy } from '@server/lib/job-queue/handlers/video-redundancy.js' +import { logger } from '../../helpers/logger.js' +import { JOB_ATTEMPTS, JOB_CONCURRENCY, JOB_REMOVAL_OPTIONS, JOB_TTL, REPEAT_JOBS, WEBSERVER } from '../../initializers/constants.js' +import { Hooks } from '../plugins/hooks.js' +import { Redis } from '../redis.js' +import { processActivityPubCleaner } from './handlers/activitypub-cleaner.js' +import { processActivityPubFollow } from './handlers/activitypub-follow.js' +import { + processActivityPubHttpSequentialBroadcast, + processActivityPubParallelHttpBroadcast +} from './handlers/activitypub-http-broadcast.js' +import { processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher.js' +import { processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast.js' +import { refreshAPObject } from './handlers/activitypub-refresher.js' +import { processActorKeys } from './handlers/actor-keys.js' +import { processAfterVideoChannelImport } from './handlers/after-video-channel-import.js' +import { processEmail } from './handlers/email.js' +import { processFederateVideo } from './handlers/federate-video.js' +import { processGenerateStoryboard } from './handlers/generate-storyboard.js' +import { processManageVideoTorrent } from './handlers/manage-video-torrent.js' +import { onMoveToObjectStorageFailure, processMoveToObjectStorage } from './handlers/move-to-object-storage.js' +import { processNotify } from './handlers/notify.js' +import { processTranscodingJobBuilder } from './handlers/transcoding-job-builder.js' +import { processVideoChannelImport } from './handlers/video-channel-import.js' +import { processVideoFileImport } from './handlers/video-file-import.js' +import { processVideoImport } from './handlers/video-import.js' +import { processVideoLiveEnding } from './handlers/video-live-ending.js' +import { processVideoStudioEdition } from './handlers/video-studio-edition.js' +import { processVideoTranscoding } from './handlers/video-transcoding.js' +import { processVideosViewsStats } from './handlers/video-views-stats.js' + +export type CreateJobArgument = + { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | + { type: 'activitypub-http-broadcast-parallel', payload: ActivitypubHttpBroadcastPayload } | + { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } | + { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | + { type: 'activitypub-cleaner', payload: {} } | + { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | + { type: 'video-file-import', payload: VideoFileImportPayload } | + { type: 'video-transcoding', payload: VideoTranscodingPayload } | + { type: 'email', payload: EmailPayload } | + { type: 'transcoding-job-builder', payload: TranscodingJobBuilderPayload } | + { type: 'video-import', payload: VideoImportPayload } | + { type: 'activitypub-refresher', payload: RefreshPayload } | + { type: 'videos-views-stats', payload: {} } | + { type: 'video-live-ending', payload: VideoLiveEndingPayload } | + { type: 'actor-keys', payload: ActorKeysPayload } | + { type: 'video-redundancy', payload: VideoRedundancyPayload } | + { type: 'delete-resumable-upload-meta-file', payload: DeleteResumableUploadMetaFilePayload } | + { type: 'video-studio-edition', payload: VideoStudioEditionPayload } | + { type: 'manage-video-torrent', payload: ManageVideoTorrentPayload } | + { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | + { type: 'video-channel-import', payload: VideoChannelImportPayload } | + { type: 'after-video-channel-import', payload: AfterVideoChannelImportPayload } | + { type: 'notify', payload: NotifyPayload } | + { type: 'move-to-object-storage', payload: MoveObjectStoragePayload } | + { type: 'federate-video', payload: FederateVideoPayload } | + { type: 'generate-video-storyboard', payload: GenerateStoryboardPayload } + +export type CreateJobOptions = { + delay?: number + priority?: number + failParentOnFailure?: boolean +} + +const handlers: { [id in JobType]: (job: Job) => Promise } = { + 'activitypub-cleaner': processActivityPubCleaner, + 'activitypub-follow': processActivityPubFollow, + 'activitypub-http-broadcast-parallel': processActivityPubParallelHttpBroadcast, + 'activitypub-http-broadcast': processActivityPubHttpSequentialBroadcast, + 'activitypub-http-fetcher': processActivityPubHttpFetcher, + 'activitypub-http-unicast': processActivityPubHttpUnicast, + 'activitypub-refresher': refreshAPObject, + 'actor-keys': processActorKeys, + 'after-video-channel-import': processAfterVideoChannelImport, + 'email': processEmail, + 'federate-video': processFederateVideo, + 'transcoding-job-builder': processTranscodingJobBuilder, + 'manage-video-torrent': processManageVideoTorrent, + 'move-to-object-storage': processMoveToObjectStorage, + 'notify': processNotify, + 'video-channel-import': processVideoChannelImport, + 'video-file-import': processVideoFileImport, + 'video-import': processVideoImport, + 'video-live-ending': processVideoLiveEnding, + 'video-redundancy': processVideoRedundancy, + 'video-studio-edition': processVideoStudioEdition, + 'video-transcoding': processVideoTranscoding, + 'videos-views-stats': processVideosViewsStats, + 'generate-video-storyboard': processGenerateStoryboard +} + +const errorHandlers: { [id in JobType]?: (job: Job, err: any) => Promise } = { + 'move-to-object-storage': onMoveToObjectStorageFailure +} + +const jobTypes: JobType[] = [ + 'activitypub-cleaner', + 'activitypub-follow', + 'activitypub-http-broadcast-parallel', + 'activitypub-http-broadcast', + 'activitypub-http-fetcher', + 'activitypub-http-unicast', + 'activitypub-refresher', + 'actor-keys', + 'after-video-channel-import', + 'email', + 'federate-video', + 'generate-video-storyboard', + 'manage-video-torrent', + 'move-to-object-storage', + 'notify', + 'transcoding-job-builder', + 'video-channel-import', + 'video-file-import', + 'video-import', + 'video-live-ending', + 'video-redundancy', + 'video-studio-edition', + 'video-transcoding', + 'videos-views-stats' +] + +const silentFailure = new Set([ 'activitypub-http-unicast' ]) + +class JobQueue { + + private static instance: JobQueue + + private workers: { [id in JobType]?: Worker } = {} + private queues: { [id in JobType]?: Queue } = {} + private queueEvents: { [id in JobType]?: QueueEvents } = {} + + private flowProducer: FlowProducer + + private initialized = false + private jobRedisPrefix: string + + private constructor () { + } + + init () { + // Already initialized + if (this.initialized === true) return + this.initialized = true + + this.jobRedisPrefix = 'bull-' + WEBSERVER.HOST + + for (const handlerName of Object.keys(handlers)) { + this.buildWorker(handlerName) + this.buildQueue(handlerName) + this.buildQueueEvent(handlerName) + } + + this.flowProducer = new FlowProducer({ + connection: Redis.getRedisClientOptions('FlowProducer'), + prefix: this.jobRedisPrefix + }) + this.flowProducer.on('error', err => { logger.error('Error in flow producer', { err }) }) + + this.addRepeatableJobs() + } + + private buildWorker (handlerName: JobType) { + const workerOptions: WorkerOptions = { + autorun: false, + concurrency: this.getJobConcurrency(handlerName), + prefix: this.jobRedisPrefix, + connection: Redis.getRedisClientOptions('Worker'), + maxStalledCount: 10 + } + + const handler = function (job: Job) { + const timeout = JOB_TTL[handlerName] + const p = handlers[handlerName](job) + + if (!timeout) return p + + return timeoutPromise(p, timeout) + } + + const processor = async (jobArg: Job) => { + const job = await Hooks.wrapObject(jobArg, 'filter:job-queue.process.params', { type: handlerName }) + + return Hooks.wrapPromiseFun(handler, job, 'filter:job-queue.process.result') + } + + const worker = new Worker(handlerName, processor, workerOptions) + + worker.on('failed', (job, err) => { + const logLevel = silentFailure.has(handlerName) + ? 'debug' + : 'error' + + logger.log(logLevel, 'Cannot execute job %s in queue %s.', job.id, handlerName, { payload: job.data, err }) + + if (errorHandlers[job.name]) { + errorHandlers[job.name](job, err) + .catch(err => logger.error('Cannot run error handler for job failure %d in queue %s.', job.id, handlerName, { err })) + } + }) + + worker.on('error', err => { logger.error('Error in job worker %s.', handlerName, { err }) }) + + this.workers[handlerName] = worker + } + + private buildQueue (handlerName: JobType) { + const queueOptions: QueueOptions = { + connection: Redis.getRedisClientOptions('Queue'), + prefix: this.jobRedisPrefix + } + + const queue = new Queue(handlerName, queueOptions) + queue.on('error', err => { logger.error('Error in job queue %s.', handlerName, { err }) }) + + this.queues[handlerName] = queue + } + + private buildQueueEvent (handlerName: JobType) { + const queueEventsOptions: QueueEventsOptions = { + autorun: false, + connection: Redis.getRedisClientOptions('QueueEvent'), + prefix: this.jobRedisPrefix + } + + const queueEvents = new QueueEvents(handlerName, queueEventsOptions) + queueEvents.on('error', err => { logger.error('Error in job queue events %s.', handlerName, { err }) }) + + this.queueEvents[handlerName] = queueEvents + } + + // --------------------------------------------------------------------------- + + async terminate () { + const promises = Object.keys(this.workers) + .map(handlerName => { + const worker: Worker = this.workers[handlerName] + const queue: Queue = this.queues[handlerName] + const queueEvent: QueueEvents = this.queueEvents[handlerName] + + return Promise.all([ + worker.close(false), + queue.close(), + queueEvent.close() + ]) + }) + + return Promise.all(promises) + } + + start () { + const promises = Object.keys(this.workers) + .map(handlerName => { + const worker: Worker = this.workers[handlerName] + const queueEvent: QueueEvents = this.queueEvents[handlerName] + + return Promise.all([ + worker.run(), + queueEvent.run() + ]) + }) + + return Promise.all(promises) + } + + async pause () { + for (const handlerName of Object.keys(this.workers)) { + const worker: Worker = this.workers[handlerName] + + await worker.pause() + } + } + + resume () { + for (const handlerName of Object.keys(this.workers)) { + const worker: Worker = this.workers[handlerName] + + worker.resume() + } + } + + // --------------------------------------------------------------------------- + + createJobAsync (options: CreateJobArgument & CreateJobOptions): void { + this.createJob(options) + .catch(err => logger.error('Cannot create job.', { err, options })) + } + + createJob (options: CreateJobArgument & CreateJobOptions) { + const queue: Queue = this.queues[options.type] + if (queue === undefined) { + logger.error('Unknown queue %s: cannot create job.', options.type) + return + } + + const jobOptions = this.buildJobOptions(options.type as JobType, pick(options, [ 'priority', 'delay' ])) + + return queue.add('job', options.payload, jobOptions) + } + + createSequentialJobFlow (...jobs: ((CreateJobArgument & CreateJobOptions) | undefined)[]) { + let lastJob: FlowJob + + for (const job of jobs) { + if (!job) continue + + lastJob = { + ...this.buildJobFlowOption(job), + + children: lastJob + ? [ lastJob ] + : [] + } + } + + return this.flowProducer.add(lastJob) + } + + createJobWithChildren (parent: CreateJobArgument & CreateJobOptions, children: (CreateJobArgument & CreateJobOptions)[]) { + return this.flowProducer.add({ + ...this.buildJobFlowOption(parent), + + children: children.map(c => this.buildJobFlowOption(c)) + }) + } + + private buildJobFlowOption (job: CreateJobArgument & CreateJobOptions): FlowJob { + return { + name: 'job', + data: job.payload, + queueName: job.type, + opts: { + failParentOnFailure: true, + + ...this.buildJobOptions(job.type as JobType, pick(job, [ 'priority', 'delay', 'failParentOnFailure' ])) + } + } + } + + private buildJobOptions (type: JobType, options: CreateJobOptions = {}): JobsOptions { + return { + backoff: { delay: 60 * 1000, type: 'exponential' }, + attempts: JOB_ATTEMPTS[type], + priority: options.priority, + delay: options.delay, + + ...this.buildJobRemovalOptions(type) + } + } + + // --------------------------------------------------------------------------- + + async listForApi (options: { + state?: JobState + start: number + count: number + asc?: boolean + jobType: JobType + }): Promise { + const { state, start, count, asc, jobType } = options + + const states = this.buildStateFilter(state) + const filteredJobTypes = this.buildTypeFilter(jobType) + + let results: Job[] = [] + + for (const jobType of filteredJobTypes) { + const queue: Queue = this.queues[jobType] + + if (queue === undefined) { + logger.error('Unknown queue %s to list jobs.', jobType) + continue + } + + const jobs = await queue.getJobs(states, 0, start + count, asc) + results = results.concat(jobs) + } + + results.sort((j1: any, j2: any) => { + if (j1.timestamp < j2.timestamp) return -1 + else if (j1.timestamp === j2.timestamp) return 0 + + return 1 + }) + + if (asc === false) results.reverse() + + return results.slice(start, start + count) + } + + async count (state: JobState, jobType?: JobType): Promise { + const states = state ? [ state ] : jobStates + const filteredJobTypes = this.buildTypeFilter(jobType) + + let total = 0 + + for (const type of filteredJobTypes) { + const queue = this.queues[type] + if (queue === undefined) { + logger.error('Unknown queue %s to count jobs.', type) + continue + } + + const counts = await queue.getJobCounts() + + for (const s of states) { + total += counts[s] + } + } + + return total + } + + private buildStateFilter (state?: JobState) { + if (!state) return jobStates + + const states = [ state ] + + // Include parent if filtering on waiting + if (state === 'waiting') states.push('waiting-children') + + return states + } + + private buildTypeFilter (jobType?: JobType) { + if (!jobType) return jobTypes + + return jobTypes.filter(t => t === jobType) + } + + async getStats () { + const promises = jobTypes.map(async t => ({ jobType: t, counts: await this.queues[t].getJobCounts() })) + + return Promise.all(promises) + } + + // --------------------------------------------------------------------------- + + async removeOldJobs () { + for (const key of Object.keys(this.queues)) { + const queue: Queue = this.queues[key] + await queue.clean(parseDurationToMs('7 days'), 1000, 'completed') + await queue.clean(parseDurationToMs('7 days'), 1000, 'failed') + } + } + + private addRepeatableJobs () { + this.queues['videos-views-stats'].add('job', {}, { + repeat: REPEAT_JOBS['videos-views-stats'], + + ...this.buildJobRemovalOptions('videos-views-stats') + }).catch(err => logger.error('Cannot add repeatable job.', { err })) + + if (CONFIG.FEDERATION.VIDEOS.CLEANUP_REMOTE_INTERACTIONS) { + this.queues['activitypub-cleaner'].add('job', {}, { + repeat: REPEAT_JOBS['activitypub-cleaner'], + + ...this.buildJobRemovalOptions('activitypub-cleaner') + }).catch(err => logger.error('Cannot add repeatable job.', { err })) + } + } + + private getJobConcurrency (jobType: JobType) { + if (jobType === 'video-transcoding') return CONFIG.TRANSCODING.CONCURRENCY + if (jobType === 'video-import') return CONFIG.IMPORT.VIDEOS.CONCURRENCY + + return JOB_CONCURRENCY[jobType] + } + + private buildJobRemovalOptions (queueName: string) { + return { + removeOnComplete: { + // Wants seconds + age: (JOB_REMOVAL_OPTIONS.SUCCESS[queueName] || JOB_REMOVAL_OPTIONS.SUCCESS.DEFAULT) / 1000, + + count: JOB_REMOVAL_OPTIONS.COUNT + }, + removeOnFail: { + // Wants seconds + age: (JOB_REMOVAL_OPTIONS.FAILURE[queueName] || JOB_REMOVAL_OPTIONS.FAILURE.DEFAULT) / 1000, + + count: JOB_REMOVAL_OPTIONS.COUNT / 1000 + } + } + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} + +// --------------------------------------------------------------------------- + +export { + jobTypes, + JobQueue +} diff --git a/server/server/lib/live/index.ts b/server/server/lib/live/index.ts new file mode 100644 index 000000000..d770aeb4b --- /dev/null +++ b/server/server/lib/live/index.ts @@ -0,0 +1,4 @@ +export * from './live-manager.js' +export * from './live-quota-store.js' +export * from './live-segment-sha-store.js' +export * from './live-utils.js' diff --git a/server/server/lib/live/live-manager.ts b/server/server/lib/live/live-manager.ts new file mode 100644 index 000000000..d003379c9 --- /dev/null +++ b/server/server/lib/live/live-manager.ts @@ -0,0 +1,557 @@ +import { readdir, readFile } from 'fs/promises' +import { createServer, Server } from 'net' +import context from 'node-media-server/src/node_core_ctx.js' +import nodeMediaServerLogger from 'node-media-server/src/node_core_logger.js' +import NodeRtmpSession from 'node-media-server/src/node_rtmp_session.js' +import { join } from 'path' +import { createServer as createServerTLS, Server as ServerTLS } from 'tls' +import { pick, wait } from '@peertube/peertube-core-utils' +import { LiveVideoError, LiveVideoErrorType, VideoState } from '@peertube/peertube-models' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { CONFIG, registerConfigChangedHandler } from '@server/initializers/config.js' +import { VIDEO_LIVE, WEBSERVER } from '@server/initializers/constants.js' +import { sequelizeTypescript } from '@server/initializers/database.js' +import { RunnerJobModel } from '@server/models/runner/runner-job.js' +import { UserModel } from '@server/models/user/user.js' +import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting.js' +import { VideoLiveSessionModel } from '@server/models/video/video-live-session.js' +import { VideoLiveModel } from '@server/models/video/video-live.js' +import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js' +import { VideoModel } from '@server/models/video/video.js' +import { MVideo, MVideoLiveSession, MVideoLiveVideo, MVideoLiveVideoWithSetting } from '@server/types/models/index.js' +import { + ffprobePromise, + getVideoStreamBitrate, + getVideoStreamDimensionsInfo, + getVideoStreamFPS, + hasAudioStream +} from '@peertube/peertube-ffmpeg' +import { federateVideoIfNeeded } from '../activitypub/videos/index.js' +import { JobQueue } from '../job-queue/index.js' +import { getLiveReplayBaseDirectory } from '../paths.js' +import { PeerTubeSocket } from '../peertube-socket.js' +import { Hooks } from '../plugins/hooks.js' +import { computeResolutionsToTranscode } from '../transcoding/transcoding-resolutions.js' +import { LiveQuotaStore } from './live-quota-store.js' +import { cleanupAndDestroyPermanentLive, getLiveSegmentTime } from './live-utils.js' +import { MuxingSession } from './shared/index.js' + +// Disable node media server logs +nodeMediaServerLogger.setLogType(0) + +const config = { + rtmp: { + port: CONFIG.LIVE.RTMP.PORT, + chunk_size: VIDEO_LIVE.RTMP.CHUNK_SIZE, + gop_cache: VIDEO_LIVE.RTMP.GOP_CACHE, + ping: VIDEO_LIVE.RTMP.PING, + ping_timeout: VIDEO_LIVE.RTMP.PING_TIMEOUT + } +} + +const lTags = loggerTagsFactory('live') + +class LiveManager { + + private static instance: LiveManager + + private readonly muxingSessions = new Map() + private readonly videoSessions = new Map() + + private rtmpServer: Server + private rtmpsServer: ServerTLS + + private running = false + + private constructor () { + } + + init () { + const events = this.getContext().nodeEvent + events.on('postPublish', (sessionId: string, streamPath: string) => { + logger.debug('RTMP received stream', { id: sessionId, streamPath, ...lTags(sessionId) }) + + const splittedPath = streamPath.split('/') + if (splittedPath.length !== 3 || splittedPath[1] !== VIDEO_LIVE.RTMP.BASE_PATH) { + logger.warn('Live path is incorrect.', { streamPath, ...lTags(sessionId) }) + return this.abortSession(sessionId) + } + + const session = this.getContext().sessions.get(sessionId) + const inputLocalUrl = session.inputOriginLocalUrl + streamPath + const inputPublicUrl = session.inputOriginPublicUrl + streamPath + + this.handleSession({ sessionId, inputPublicUrl, inputLocalUrl, streamKey: splittedPath[2] }) + .catch(err => logger.error('Cannot handle sessions.', { err, ...lTags(sessionId) })) + }) + + events.on('donePublish', sessionId => { + logger.info('Live session ended.', { sessionId, ...lTags(sessionId) }) + + // Force session aborting, so we kill ffmpeg even if it still has data to process (slow CPU) + setTimeout(() => this.abortSession(sessionId), 2000) + }) + + registerConfigChangedHandler(() => { + if (!this.running && CONFIG.LIVE.ENABLED === true) { + this.run().catch(err => logger.error('Cannot run live server.', { err })) + return + } + + if (this.running && CONFIG.LIVE.ENABLED === false) { + this.stop() + } + }) + + // Cleanup broken lives, that were terminated by a server restart for example + this.handleBrokenLives() + .catch(err => logger.error('Cannot handle broken lives.', { err, ...lTags() })) + } + + async run () { + this.running = true + + if (CONFIG.LIVE.RTMP.ENABLED) { + logger.info('Running RTMP server on port %d', CONFIG.LIVE.RTMP.PORT, lTags()) + + this.rtmpServer = createServer(socket => { + const session = new NodeRtmpSession(config, socket) + + session.inputOriginLocalUrl = 'rtmp://127.0.0.1:' + CONFIG.LIVE.RTMP.PORT + session.inputOriginPublicUrl = WEBSERVER.RTMP_URL + session.run() + }) + + this.rtmpServer.on('error', err => { + logger.error('Cannot run RTMP server.', { err, ...lTags() }) + }) + + this.rtmpServer.listen(CONFIG.LIVE.RTMP.PORT, CONFIG.LIVE.RTMP.HOSTNAME) + } + + if (CONFIG.LIVE.RTMPS.ENABLED) { + logger.info('Running RTMPS server on port %d', CONFIG.LIVE.RTMPS.PORT, lTags()) + + const [ key, cert ] = await Promise.all([ + readFile(CONFIG.LIVE.RTMPS.KEY_FILE), + readFile(CONFIG.LIVE.RTMPS.CERT_FILE) + ]) + const serverOptions = { key, cert } + + this.rtmpsServer = createServerTLS(serverOptions, socket => { + const session = new NodeRtmpSession(config, socket) + + session.inputOriginLocalUrl = 'rtmps://127.0.0.1:' + CONFIG.LIVE.RTMPS.PORT + session.inputOriginPublicUrl = WEBSERVER.RTMPS_URL + session.run() + }) + + this.rtmpsServer.on('error', err => { + logger.error('Cannot run RTMPS server.', { err, ...lTags() }) + }) + + this.rtmpsServer.listen(CONFIG.LIVE.RTMPS.PORT, CONFIG.LIVE.RTMPS.HOSTNAME) + } + } + + stop () { + this.running = false + + if (this.rtmpServer) { + logger.info('Stopping RTMP server.', lTags()) + + this.rtmpServer.close() + this.rtmpServer = undefined + } + + if (this.rtmpsServer) { + logger.info('Stopping RTMPS server.', lTags()) + + this.rtmpsServer.close() + this.rtmpsServer = undefined + } + + // Sessions is an object + this.getContext().sessions.forEach((session: any) => { + if (session instanceof NodeRtmpSession) { + session.stop() + } + }) + } + + isRunning () { + return !!this.rtmpServer + } + + hasSession (sessionId: string) { + return this.getContext().sessions.has(sessionId) + } + + stopSessionOf (videoUUID: string, error: LiveVideoErrorType | null) { + const sessionId = this.videoSessions.get(videoUUID) + if (!sessionId) { + logger.debug('No live session to stop for video %s', videoUUID, lTags(sessionId, videoUUID)) + return + } + + logger.info('Stopping live session of video %s', videoUUID, { error, ...lTags(sessionId, videoUUID) }) + + this.saveEndingSession(videoUUID, error) + .catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId, videoUUID) })) + + this.videoSessions.delete(videoUUID) + this.abortSession(sessionId) + } + + private getContext () { + return context + } + + private abortSession (sessionId: string) { + const session = this.getContext().sessions.get(sessionId) + if (session) { + session.stop() + this.getContext().sessions.delete(sessionId) + } + + const muxingSession = this.muxingSessions.get(sessionId) + if (muxingSession) { + // Muxing session will fire and event so we correctly cleanup the session + muxingSession.abort() + + this.muxingSessions.delete(sessionId) + } + } + + private async handleSession (options: { + sessionId: string + inputLocalUrl: string + inputPublicUrl: string + streamKey: string + }) { + const { inputLocalUrl, inputPublicUrl, sessionId, streamKey } = options + + const videoLive = await VideoLiveModel.loadByStreamKey(streamKey) + if (!videoLive) { + logger.warn('Unknown live video with stream key %s.', streamKey, lTags(sessionId)) + return this.abortSession(sessionId) + } + + const video = videoLive.Video + if (video.isBlacklisted()) { + logger.warn('Video is blacklisted. Refusing stream %s.', streamKey, lTags(sessionId, video.uuid)) + return this.abortSession(sessionId) + } + + if (this.videoSessions.has(video.uuid)) { + logger.warn('Video %s has already a live session. Refusing stream %s.', video.uuid, streamKey, lTags(sessionId, video.uuid)) + return this.abortSession(sessionId) + } + + // Cleanup old potential live (could happen with a permanent live) + const oldStreamingPlaylist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id) + if (oldStreamingPlaylist) { + if (!videoLive.permanentLive) throw new Error('Found previous session in a non permanent live: ' + video.uuid) + + await cleanupAndDestroyPermanentLive(video, oldStreamingPlaylist) + } + + this.videoSessions.set(video.uuid, sessionId) + + const now = Date.now() + const probe = await ffprobePromise(inputLocalUrl) + + const [ { resolution, ratio }, fps, bitrate, hasAudio ] = await Promise.all([ + getVideoStreamDimensionsInfo(inputLocalUrl, probe), + getVideoStreamFPS(inputLocalUrl, probe), + getVideoStreamBitrate(inputLocalUrl, probe), + hasAudioStream(inputLocalUrl, probe) + ]) + + logger.info( + '%s probing took %d ms (bitrate: %d, fps: %d, resolution: %d)', + inputLocalUrl, Date.now() - now, bitrate, fps, resolution, lTags(sessionId, video.uuid) + ) + + const allResolutions = await Hooks.wrapObject( + this.buildAllResolutionsToTranscode(resolution, hasAudio), + 'filter:transcoding.auto.resolutions-to-transcode.result', + { video } + ) + + logger.info( + 'Handling live video of original resolution %d.', resolution, + { allResolutions, ...lTags(sessionId, video.uuid) } + ) + + return this.runMuxingSession({ + sessionId, + videoLive, + + inputLocalUrl, + inputPublicUrl, + fps, + bitrate, + ratio, + allResolutions, + hasAudio + }) + } + + private async runMuxingSession (options: { + sessionId: string + videoLive: MVideoLiveVideoWithSetting + + inputLocalUrl: string + inputPublicUrl: string + + fps: number + bitrate: number + ratio: number + allResolutions: number[] + hasAudio: boolean + }) { + const { sessionId, videoLive } = options + const videoUUID = videoLive.Video.uuid + const localLTags = lTags(sessionId, videoUUID) + + const liveSession = await this.saveStartingSession(videoLive) + + const user = await UserModel.loadByLiveId(videoLive.id) + LiveQuotaStore.Instance.addNewLive(user.id, sessionId) + + const muxingSession = new MuxingSession({ + context: this.getContext(), + sessionId, + videoLive, + user, + + ...pick(options, [ 'inputLocalUrl', 'inputPublicUrl', 'bitrate', 'ratio', 'fps', 'allResolutions', 'hasAudio' ]) + }) + + muxingSession.on('live-ready', () => this.publishAndFederateLive(videoLive, localLTags)) + + muxingSession.on('bad-socket-health', ({ videoUUID }) => { + logger.error( + 'Too much data in client socket stream (ffmpeg is too slow to transcode the video).' + + ' Stopping session of video %s.', videoUUID, + localLTags + ) + + this.stopSessionOf(videoUUID, LiveVideoError.BAD_SOCKET_HEALTH) + }) + + muxingSession.on('duration-exceeded', ({ videoUUID }) => { + logger.info('Stopping session of %s: max duration exceeded.', videoUUID, localLTags) + + this.stopSessionOf(videoUUID, LiveVideoError.DURATION_EXCEEDED) + }) + + muxingSession.on('quota-exceeded', ({ videoUUID }) => { + logger.info('Stopping session of %s: user quota exceeded.', videoUUID, localLTags) + + this.stopSessionOf(videoUUID, LiveVideoError.QUOTA_EXCEEDED) + }) + + muxingSession.on('transcoding-error', ({ videoUUID }) => { + this.stopSessionOf(videoUUID, LiveVideoError.FFMPEG_ERROR) + }) + + muxingSession.on('transcoding-end', ({ videoUUID }) => { + this.onMuxingFFmpegEnd(videoUUID, sessionId) + }) + + muxingSession.on('after-cleanup', ({ videoUUID }) => { + this.muxingSessions.delete(sessionId) + + LiveQuotaStore.Instance.removeLive(user.id, sessionId) + + muxingSession.destroy() + + return this.onAfterMuxingCleanup({ videoUUID, liveSession }) + .catch(err => logger.error('Error in end transmuxing.', { err, ...localLTags })) + }) + + this.muxingSessions.set(sessionId, muxingSession) + + muxingSession.runMuxing() + .catch(err => { + logger.error('Cannot run muxing.', { err, ...localLTags }) + this.abortSession(sessionId) + }) + } + + private async publishAndFederateLive (live: MVideoLiveVideo, localLTags: { tags: string[] }) { + const videoId = live.videoId + + try { + const video = await VideoModel.loadFull(videoId) + + logger.info('Will publish and federate live %s.', video.url, localLTags) + + video.state = VideoState.PUBLISHED + video.publishedAt = new Date() + await video.save() + + live.Video = video + + await wait(getLiveSegmentTime(live.latencyMode) * 1000 * VIDEO_LIVE.EDGE_LIVE_DELAY_SEGMENTS_NOTIFICATION) + + try { + await federateVideoIfNeeded(video, false) + } catch (err) { + logger.error('Cannot federate live video %s.', video.url, { err, ...localLTags }) + } + + PeerTubeSocket.Instance.sendVideoLiveNewState(video) + + Hooks.runAction('action:live.video.state.updated', { video }) + } catch (err) { + logger.error('Cannot save/federate live video %d.', videoId, { err, ...localLTags }) + } + } + + private onMuxingFFmpegEnd (videoUUID: string, sessionId: string) { + // Session already cleaned up + if (!this.videoSessions.has(videoUUID)) return + + this.videoSessions.delete(videoUUID) + + this.saveEndingSession(videoUUID, null) + .catch(err => logger.error('Cannot save ending session.', { err, ...lTags(sessionId) })) + } + + private async onAfterMuxingCleanup (options: { + videoUUID: string + liveSession?: MVideoLiveSession + cleanupNow?: boolean // Default false + }) { + const { videoUUID, liveSession: liveSessionArg, cleanupNow = false } = options + + logger.debug('Live of video %s has been cleaned up. Moving to its next state.', videoUUID, lTags(videoUUID)) + + try { + const fullVideo = await VideoModel.loadFull(videoUUID) + if (!fullVideo) return + + const live = await VideoLiveModel.loadByVideoId(fullVideo.id) + + const liveSession = liveSessionArg ?? await VideoLiveSessionModel.findLatestSessionOf(fullVideo.id) + + // On server restart during a live + if (!liveSession.endDate) { + liveSession.endDate = new Date() + await liveSession.save() + } + + JobQueue.Instance.createJobAsync({ + type: 'video-live-ending', + payload: { + videoId: fullVideo.id, + + replayDirectory: live.saveReplay + ? await this.findReplayDirectory(fullVideo) + : undefined, + + liveSessionId: liveSession.id, + streamingPlaylistId: fullVideo.getHLSPlaylist()?.id, + + publishedAt: fullVideo.publishedAt.toISOString() + }, + + delay: cleanupNow + ? 0 + : VIDEO_LIVE.CLEANUP_DELAY + }) + + fullVideo.state = live.permanentLive + ? VideoState.WAITING_FOR_LIVE + : VideoState.LIVE_ENDED + + await fullVideo.save() + + PeerTubeSocket.Instance.sendVideoLiveNewState(fullVideo) + + await federateVideoIfNeeded(fullVideo, false) + + Hooks.runAction('action:live.video.state.updated', { video: fullVideo }) + } catch (err) { + logger.error('Cannot save/federate new video state of live streaming of video %s.', videoUUID, { err, ...lTags(videoUUID) }) + } + } + + private async handleBrokenLives () { + await RunnerJobModel.cancelAllJobs({ type: 'live-rtmp-hls-transcoding' }) + + const videoUUIDs = await VideoModel.listPublishedLiveUUIDs() + + for (const uuid of videoUUIDs) { + await this.onAfterMuxingCleanup({ videoUUID: uuid, cleanupNow: true }) + } + } + + private async findReplayDirectory (video: MVideo) { + const directory = getLiveReplayBaseDirectory(video) + const files = await readdir(directory) + + if (files.length === 0) return undefined + + return join(directory, files.sort().reverse()[0]) + } + + private buildAllResolutionsToTranscode (originResolution: number, hasAudio: boolean) { + const includeInput = CONFIG.LIVE.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION + + const resolutionsEnabled = CONFIG.LIVE.TRANSCODING.ENABLED + ? computeResolutionsToTranscode({ input: originResolution, type: 'live', includeInput, strictLower: false, hasAudio }) + : [] + + if (resolutionsEnabled.length === 0) { + return [ originResolution ] + } + + return resolutionsEnabled + } + + private async saveStartingSession (videoLive: MVideoLiveVideoWithSetting) { + const replaySettings = videoLive.saveReplay + ? new VideoLiveReplaySettingModel({ + privacy: videoLive.ReplaySetting.privacy + }) + : null + + return sequelizeTypescript.transaction(async t => { + if (videoLive.saveReplay) { + await replaySettings.save({ transaction: t }) + } + + return VideoLiveSessionModel.create({ + startDate: new Date(), + liveVideoId: videoLive.videoId, + saveReplay: videoLive.saveReplay, + replaySettingId: videoLive.saveReplay ? replaySettings.id : null, + endingProcessed: false + }, { transaction: t }) + }) + } + + private async saveEndingSession (videoUUID: string, error: LiveVideoErrorType | null) { + const liveSession = await VideoLiveSessionModel.findCurrentSessionOf(videoUUID) + if (!liveSession) return + + liveSession.endDate = new Date() + liveSession.error = error + + return liveSession.save() + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} + +// --------------------------------------------------------------------------- + +export { + LiveManager +} diff --git a/server/lib/live/live-quota-store.ts b/server/server/lib/live/live-quota-store.ts similarity index 100% rename from server/lib/live/live-quota-store.ts rename to server/server/lib/live/live-quota-store.ts diff --git a/server/server/lib/live/live-segment-sha-store.ts b/server/server/lib/live/live-segment-sha-store.ts new file mode 100644 index 000000000..fbe1b5e29 --- /dev/null +++ b/server/server/lib/live/live-segment-sha-store.ts @@ -0,0 +1,96 @@ +import { writeJson } from 'fs-extra/esm' +import { rename } from 'fs/promises' +import PQueue from 'p-queue' +import { basename } from 'path' +import { mapToJSON } from '@server/helpers/core-utils.js' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { MStreamingPlaylistVideo } from '@server/types/models/index.js' +import { buildSha256Segment } from '../hls.js' +import { storeHLSFileFromPath } from '../object-storage/index.js' + +const lTags = loggerTagsFactory('live') + +class LiveSegmentShaStore { + + private readonly segmentsSha256 = new Map() + + private readonly videoUUID: string + + private readonly sha256Path: string + private readonly sha256PathTMP: string + + private readonly streamingPlaylist: MStreamingPlaylistVideo + private readonly sendToObjectStorage: boolean + private readonly writeQueue = new PQueue({ concurrency: 1 }) + + constructor (options: { + videoUUID: string + sha256Path: string + streamingPlaylist: MStreamingPlaylistVideo + sendToObjectStorage: boolean + }) { + this.videoUUID = options.videoUUID + + this.sha256Path = options.sha256Path + this.sha256PathTMP = options.sha256Path + '.tmp' + + this.streamingPlaylist = options.streamingPlaylist + this.sendToObjectStorage = options.sendToObjectStorage + } + + async addSegmentSha (segmentPath: string) { + logger.debug('Adding live sha segment %s.', segmentPath, lTags(this.videoUUID)) + + const shaResult = await buildSha256Segment(segmentPath) + + const segmentName = basename(segmentPath) + this.segmentsSha256.set(segmentName, shaResult) + + try { + await this.writeToDisk() + } catch (err) { + logger.error('Cannot write sha segments to disk.', { err }) + } + } + + async removeSegmentSha (segmentPath: string) { + const segmentName = basename(segmentPath) + + logger.debug('Removing live sha segment %s.', segmentPath, lTags(this.videoUUID)) + + if (!this.segmentsSha256.has(segmentName)) { + logger.warn( + 'Unknown segment in live segment hash store for video %s and segment %s.', + this.videoUUID, segmentPath, lTags(this.videoUUID) + ) + return + } + + this.segmentsSha256.delete(segmentName) + + await this.writeToDisk() + } + + private writeToDisk () { + return this.writeQueue.add(async () => { + logger.debug(`Writing segment sha JSON ${this.sha256Path} of ${this.videoUUID} on disk.`, lTags(this.videoUUID)) + + // Atomic write: use rename instead of move that is not atomic + await writeJson(this.sha256PathTMP, mapToJSON(this.segmentsSha256)) + await rename(this.sha256PathTMP, this.sha256Path) + + if (this.sendToObjectStorage) { + const url = await storeHLSFileFromPath(this.streamingPlaylist, this.sha256Path) + + if (this.streamingPlaylist.segmentsSha256Url !== url) { + this.streamingPlaylist.segmentsSha256Url = url + await this.streamingPlaylist.save() + } + } + }) + } +} + +export { + LiveSegmentShaStore +} diff --git a/server/server/lib/live/live-utils.ts b/server/server/lib/live/live-utils.ts new file mode 100644 index 000000000..55b7984bf --- /dev/null +++ b/server/server/lib/live/live-utils.ts @@ -0,0 +1,100 @@ +import { pathExists, remove } from 'fs-extra/esm' +import { readdir } from 'fs/promises' +import { basename, join } from 'path' +import { LiveVideoLatencyMode, LiveVideoLatencyModeType, VideoStorage } from '@peertube/peertube-models' +import { logger } from '@server/helpers/logger.js' +import { VIDEO_LIVE } from '@server/initializers/constants.js' +import { MStreamingPlaylist, MStreamingPlaylistVideo, MVideo } from '@server/types/models/index.js' +import { listHLSFileKeysOf, removeHLSFileObjectStorageByFullKey, removeHLSObjectStorage } from '../object-storage/index.js' +import { getLiveDirectory } from '../paths.js' + +function buildConcatenatedName (segmentOrPlaylistPath: string) { + const num = basename(segmentOrPlaylistPath).match(/^(\d+)(-|\.)/) + + return 'concat-' + num[1] + '.ts' +} + +async function cleanupAndDestroyPermanentLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { + await cleanupTMPLiveFiles(video, streamingPlaylist) + + await streamingPlaylist.destroy() +} + +async function cleanupUnsavedNormalLive (video: MVideo, streamingPlaylist: MStreamingPlaylist) { + const hlsDirectory = getLiveDirectory(video) + + // We uploaded files to object storage too, remove them + if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { + await removeHLSObjectStorage(streamingPlaylist.withVideo(video)) + } + + await remove(hlsDirectory) + + await streamingPlaylist.destroy() +} + +async function cleanupTMPLiveFiles (video: MVideo, streamingPlaylist: MStreamingPlaylist) { + await cleanupTMPLiveFilesFromObjectStorage(streamingPlaylist.withVideo(video)) + + await cleanupTMPLiveFilesFromFilesystem(video) +} + +function getLiveSegmentTime (latencyMode: LiveVideoLatencyModeType) { + if (latencyMode === LiveVideoLatencyMode.SMALL_LATENCY) { + return VIDEO_LIVE.SEGMENT_TIME_SECONDS.SMALL_LATENCY + } + + return VIDEO_LIVE.SEGMENT_TIME_SECONDS.DEFAULT_LATENCY +} + +export { + cleanupAndDestroyPermanentLive, + cleanupUnsavedNormalLive, + cleanupTMPLiveFiles, + getLiveSegmentTime, + buildConcatenatedName +} + +// --------------------------------------------------------------------------- + +function isTMPLiveFile (name: string) { + return name.endsWith('.ts') || + name.endsWith('.m3u8') || + name.endsWith('.json') || + name.endsWith('.mpd') || + name.endsWith('.m4s') || + name.endsWith('.tmp') +} + +async function cleanupTMPLiveFilesFromFilesystem (video: MVideo) { + const hlsDirectory = getLiveDirectory(video) + + if (!await pathExists(hlsDirectory)) return + + logger.info('Cleanup TMP live files from filesystem of %s.', hlsDirectory) + + const files = await readdir(hlsDirectory) + + for (const filename of files) { + if (isTMPLiveFile(filename)) { + const p = join(hlsDirectory, filename) + + remove(p) + .catch(err => logger.error('Cannot remove %s.', p, { err })) + } + } +} + +async function cleanupTMPLiveFilesFromObjectStorage (streamingPlaylist: MStreamingPlaylistVideo) { + if (streamingPlaylist.storage !== VideoStorage.OBJECT_STORAGE) return + + logger.info('Cleanup TMP live files from object storage for %s.', streamingPlaylist.Video.uuid) + + const keys = await listHLSFileKeysOf(streamingPlaylist) + + for (const key of keys) { + if (isTMPLiveFile(key)) { + await removeHLSFileObjectStorageByFullKey(key) + } + } +} diff --git a/server/server/lib/live/shared/index.ts b/server/server/lib/live/shared/index.ts new file mode 100644 index 000000000..218eab284 --- /dev/null +++ b/server/server/lib/live/shared/index.ts @@ -0,0 +1 @@ +export * from './muxing-session.js' diff --git a/server/server/lib/live/shared/muxing-session.ts b/server/server/lib/live/shared/muxing-session.ts new file mode 100644 index 000000000..1a71a04d7 --- /dev/null +++ b/server/server/lib/live/shared/muxing-session.ts @@ -0,0 +1,519 @@ +import Bluebird from 'bluebird' +import { FSWatcher, watch } from 'chokidar' +import { EventEmitter } from 'events' +import { ensureDir } from 'fs-extra/esm' +import { appendFile, readFile, stat } from 'fs/promises' +import memoizee from 'memoizee' +import PQueue from 'p-queue' +import { basename, join } from 'path' +import { computeOutputFPS } from '@server/helpers/ffmpeg/index.js' +import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' +import { MEMOIZE_TTL, P2P_MEDIA_LOADER_PEER_VERSION, VIDEO_LIVE } from '@server/initializers/constants.js' +import { removeHLSFileObjectStorageByPath, storeHLSFileFromContent, storeHLSFileFromPath } from '@server/lib/object-storage/index.js' +import { VideoFileModel } from '@server/models/video/video-file.js' +import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js' +import { MStreamingPlaylistVideo, MUserId, MVideoLiveVideo } from '@server/types/models/index.js' +import { VideoStorage, VideoStreamingPlaylistType } from '@peertube/peertube-models' +import { + generateHLSMasterPlaylistFilename, + generateHlsSha256SegmentsFilename, + getLiveDirectory, + getLiveReplayBaseDirectory +} from '../../paths.js' +import { isAbleToUploadVideo } from '../../user.js' +import { LiveQuotaStore } from '../live-quota-store.js' +import { LiveSegmentShaStore } from '../live-segment-sha-store.js' +import { buildConcatenatedName, getLiveSegmentTime } from '../live-utils.js' +import { AbstractTranscodingWrapper, FFmpegTranscodingWrapper, RemoteTranscodingWrapper } from './transcoding-wrapper/index.js' + +interface MuxingSessionEvents { + 'live-ready': (options: { videoUUID: string }) => void + + 'bad-socket-health': (options: { videoUUID: string }) => void + 'duration-exceeded': (options: { videoUUID: string }) => void + 'quota-exceeded': (options: { videoUUID: string }) => void + + 'transcoding-end': (options: { videoUUID: string }) => void + 'transcoding-error': (options: { videoUUID: string }) => void + + 'after-cleanup': (options: { videoUUID: string }) => void +} + +declare interface MuxingSession { + on( + event: U, listener: MuxingSessionEvents[U] + ): this + + emit( + event: U, ...args: Parameters + ): boolean +} + +class MuxingSession extends EventEmitter { + + private transcodingWrapper: AbstractTranscodingWrapper + + private readonly context: any + private readonly user: MUserId + private readonly sessionId: string + private readonly videoLive: MVideoLiveVideo + + private readonly inputLocalUrl: string + private readonly inputPublicUrl: string + + private readonly fps: number + private readonly allResolutions: number[] + + private readonly bitrate: number + private readonly ratio: number + + private readonly hasAudio: boolean + + private readonly videoUUID: string + private readonly saveReplay: boolean + + private readonly outDirectory: string + private readonly replayDirectory: string + + private readonly lTags: LoggerTagsFn + + // Path -> Queue + private readonly objectStorageSendQueues = new Map() + + private segmentsToProcessPerPlaylist: { [playlistId: string]: string[] } = {} + + private streamingPlaylist: MStreamingPlaylistVideo + private liveSegmentShaStore: LiveSegmentShaStore + + private filesWatcher: FSWatcher + + private masterPlaylistCreated = false + private liveReady = false + + private aborted = false + + private readonly isAbleToUploadVideoWithCache = memoizee((userId: number) => { + return isAbleToUploadVideo(userId, 1000) + }, { maxAge: MEMOIZE_TTL.LIVE_ABLE_TO_UPLOAD }) + + private readonly hasClientSocketInBadHealthWithCache = memoizee((sessionId: string) => { + return this.hasClientSocketInBadHealth(sessionId) + }, { maxAge: MEMOIZE_TTL.LIVE_CHECK_SOCKET_HEALTH }) + + constructor (options: { + context: any + user: MUserId + sessionId: string + videoLive: MVideoLiveVideo + + inputLocalUrl: string + inputPublicUrl: string + + fps: number + bitrate: number + ratio: number + allResolutions: number[] + hasAudio: boolean + }) { + super() + + this.context = options.context + this.user = options.user + this.sessionId = options.sessionId + this.videoLive = options.videoLive + + this.inputLocalUrl = options.inputLocalUrl + this.inputPublicUrl = options.inputPublicUrl + + this.fps = options.fps + + this.bitrate = options.bitrate + this.ratio = options.ratio + + this.hasAudio = options.hasAudio + + this.allResolutions = options.allResolutions + + this.videoUUID = this.videoLive.Video.uuid + + this.saveReplay = this.videoLive.saveReplay + + this.outDirectory = getLiveDirectory(this.videoLive.Video) + this.replayDirectory = join(getLiveReplayBaseDirectory(this.videoLive.Video), new Date().toISOString()) + + this.lTags = loggerTagsFactory('live', this.sessionId, this.videoUUID) + } + + async runMuxing () { + this.streamingPlaylist = await this.createLivePlaylist() + + this.createLiveShaStore() + this.createFiles() + + await this.prepareDirectories() + + this.transcodingWrapper = this.buildTranscodingWrapper() + + this.transcodingWrapper.on('end', () => this.onTranscodedEnded()) + this.transcodingWrapper.on('error', () => this.onTranscodingError()) + + await this.transcodingWrapper.run() + + this.filesWatcher = watch(this.outDirectory, { depth: 0 }) + + this.watchMasterFile() + this.watchTSFiles() + } + + abort () { + if (!this.transcodingWrapper) return + + this.aborted = true + this.transcodingWrapper.abort() + } + + destroy () { + this.removeAllListeners() + this.isAbleToUploadVideoWithCache.clear() + this.hasClientSocketInBadHealthWithCache.clear() + } + + private watchMasterFile () { + this.filesWatcher.on('add', async path => { + if (path !== join(this.outDirectory, this.streamingPlaylist.playlistFilename)) return + if (this.masterPlaylistCreated === true) return + + try { + if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { + const masterContent = await readFile(path, 'utf-8') + logger.debug('Uploading live master playlist on object storage for %s', this.videoUUID, { masterContent, ...this.lTags() }) + + const url = await storeHLSFileFromContent(this.streamingPlaylist, this.streamingPlaylist.playlistFilename, masterContent) + + this.streamingPlaylist.playlistUrl = url + } + + this.streamingPlaylist.assignP2PMediaLoaderInfoHashes(this.videoLive.Video, this.allResolutions) + + await this.streamingPlaylist.save() + } catch (err) { + logger.error('Cannot update streaming playlist.', { err, ...this.lTags() }) + } + + this.masterPlaylistCreated = true + + logger.info('Master playlist file for %s has been created', this.videoUUID, this.lTags()) + }) + } + + private watchTSFiles () { + const startStreamDateTime = new Date().getTime() + + const addHandler = async (segmentPath: string) => { + if (segmentPath.endsWith('.ts') !== true) return + + logger.debug('Live add handler of TS file %s.', segmentPath, this.lTags()) + + const playlistId = this.getPlaylistIdFromTS(segmentPath) + + const segmentsToProcess = this.segmentsToProcessPerPlaylist[playlistId] || [] + this.processSegments(segmentsToProcess) + + this.segmentsToProcessPerPlaylist[playlistId] = [ segmentPath ] + + if (this.hasClientSocketInBadHealthWithCache(this.sessionId)) { + this.emit('bad-socket-health', { videoUUID: this.videoUUID }) + return + } + + // Duration constraint check + if (this.isDurationConstraintValid(startStreamDateTime) !== true) { + this.emit('duration-exceeded', { videoUUID: this.videoUUID }) + return + } + + // Check user quota if the user enabled replay saving + if (await this.isQuotaExceeded(segmentPath) === true) { + this.emit('quota-exceeded', { videoUUID: this.videoUUID }) + } + } + + const deleteHandler = async (segmentPath: string) => { + if (segmentPath.endsWith('.ts') !== true) return + + logger.debug('Live delete handler of TS file %s.', segmentPath, this.lTags()) + + try { + await this.liveSegmentShaStore.removeSegmentSha(segmentPath) + } catch (err) { + logger.warn('Cannot remove segment sha %s from sha store', segmentPath, { err, ...this.lTags() }) + } + + if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { + try { + await removeHLSFileObjectStorageByPath(this.streamingPlaylist, segmentPath) + } catch (err) { + logger.error('Cannot remove segment %s from object storage', segmentPath, { err, ...this.lTags() }) + } + } + } + + this.filesWatcher.on('add', p => addHandler(p)) + this.filesWatcher.on('unlink', p => deleteHandler(p)) + } + + private async isQuotaExceeded (segmentPath: string) { + if (this.saveReplay !== true) return false + if (this.aborted) return false + + try { + const segmentStat = await stat(segmentPath) + + LiveQuotaStore.Instance.addQuotaTo(this.user.id, this.sessionId, segmentStat.size) + + const canUpload = await this.isAbleToUploadVideoWithCache(this.user.id) + + return canUpload !== true + } catch (err) { + logger.error('Cannot stat %s or check quota of %d.', segmentPath, this.user.id, { err, ...this.lTags() }) + } + } + + private createFiles () { + for (let i = 0; i < this.allResolutions.length; i++) { + const resolution = this.allResolutions[i] + + const file = new VideoFileModel({ + resolution, + size: -1, + extname: '.ts', + infoHash: null, + fps: this.fps, + storage: this.streamingPlaylist.storage, + videoStreamingPlaylistId: this.streamingPlaylist.id + }) + + VideoFileModel.customUpsert(file, 'streaming-playlist', null) + .catch(err => logger.error('Cannot create file for live streaming.', { err, ...this.lTags() })) + } + } + + private async prepareDirectories () { + await ensureDir(this.outDirectory) + + if (this.videoLive.saveReplay === true) { + await ensureDir(this.replayDirectory) + } + } + + private isDurationConstraintValid (streamingStartTime: number) { + const maxDuration = CONFIG.LIVE.MAX_DURATION + // No limit + if (maxDuration < 0) return true + + const now = new Date().getTime() + const max = streamingStartTime + maxDuration + + return now <= max + } + + private processSegments (segmentPaths: string[]) { + Bluebird.mapSeries(segmentPaths, previousSegment => this.processSegment(previousSegment)) + .catch(err => { + if (this.aborted) return + + logger.error('Cannot process segments', { err, ...this.lTags() }) + }) + } + + private async processSegment (segmentPath: string) { + // Add sha hash of previous segments, because ffmpeg should have finished generating them + await this.liveSegmentShaStore.addSegmentSha(segmentPath) + + if (this.saveReplay) { + await this.addSegmentToReplay(segmentPath) + } + + if (this.streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { + try { + await storeHLSFileFromPath(this.streamingPlaylist, segmentPath) + + await this.processM3U8ToObjectStorage(segmentPath) + } catch (err) { + logger.error('Cannot store TS segment %s in object storage', segmentPath, { err, ...this.lTags() }) + } + } + + // Master playlist and segment JSON file are created, live is ready + if (this.masterPlaylistCreated && !this.liveReady) { + this.liveReady = true + + this.emit('live-ready', { videoUUID: this.videoUUID }) + } + } + + private async processM3U8ToObjectStorage (segmentPath: string) { + const m3u8Path = join(this.outDirectory, this.getPlaylistNameFromTS(segmentPath)) + + logger.debug('Process M3U8 file %s.', m3u8Path, this.lTags()) + + const segmentName = basename(segmentPath) + + const playlistContent = await readFile(m3u8Path, 'utf-8') + // Remove new chunk references, that will be processed later + const filteredPlaylistContent = playlistContent.substring(0, playlistContent.lastIndexOf(segmentName) + segmentName.length) + '\n' + + try { + if (!this.objectStorageSendQueues.has(m3u8Path)) { + this.objectStorageSendQueues.set(m3u8Path, new PQueue({ concurrency: 1 })) + } + + const queue = this.objectStorageSendQueues.get(m3u8Path) + await queue.add(() => storeHLSFileFromContent(this.streamingPlaylist, m3u8Path, filteredPlaylistContent)) + } catch (err) { + logger.error('Cannot store in object storage m3u8 file %s', m3u8Path, { err, ...this.lTags() }) + } + } + + private onTranscodingError () { + this.emit('transcoding-error', ({ videoUUID: this.videoUUID })) + } + + private onTranscodedEnded () { + this.emit('transcoding-end', ({ videoUUID: this.videoUUID })) + + logger.info('RTMP transmuxing for video %s ended. Scheduling cleanup', this.inputLocalUrl, this.lTags()) + + setTimeout(() => { + // Wait latest segments generation, and close watchers + + const promise = this.filesWatcher?.close() || Promise.resolve() + promise + .then(() => { + // Process remaining segments hash + for (const key of Object.keys(this.segmentsToProcessPerPlaylist)) { + this.processSegments(this.segmentsToProcessPerPlaylist[key]) + } + }) + .catch(err => { + logger.error( + 'Cannot close watchers of %s or process remaining hash segments.', this.outDirectory, + { err, ...this.lTags() } + ) + }) + + this.emit('after-cleanup', { videoUUID: this.videoUUID }) + }, 1000) + } + + private hasClientSocketInBadHealth (sessionId: string) { + const rtmpSession = this.context.sessions.get(sessionId) + + if (!rtmpSession) { + logger.warn('Cannot get session %s to check players socket health.', sessionId, this.lTags()) + return + } + + for (const playerSessionId of rtmpSession.players) { + const playerSession = this.context.sessions.get(playerSessionId) + + if (!playerSession) { + logger.error('Cannot get player session %s to check socket health.', playerSession, this.lTags()) + continue + } + + if (playerSession.socket.writableLength > VIDEO_LIVE.MAX_SOCKET_WAITING_DATA) { + return true + } + } + + return false + } + + private async addSegmentToReplay (segmentPath: string) { + const segmentName = basename(segmentPath) + const dest = join(this.replayDirectory, buildConcatenatedName(segmentName)) + + try { + const data = await readFile(segmentPath) + + await appendFile(dest, data) + } catch (err) { + logger.error('Cannot copy segment %s to replay directory.', segmentPath, { err, ...this.lTags() }) + } + } + + private async createLivePlaylist (): Promise { + const playlist = await VideoStreamingPlaylistModel.loadOrGenerate(this.videoLive.Video) + + playlist.playlistFilename = generateHLSMasterPlaylistFilename(true) + playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(true) + + playlist.p2pMediaLoaderPeerVersion = P2P_MEDIA_LOADER_PEER_VERSION + playlist.type = VideoStreamingPlaylistType.HLS + + playlist.storage = CONFIG.OBJECT_STORAGE.ENABLED + ? VideoStorage.OBJECT_STORAGE + : VideoStorage.FILE_SYSTEM + + return playlist.save() + } + + private createLiveShaStore () { + this.liveSegmentShaStore = new LiveSegmentShaStore({ + videoUUID: this.videoLive.Video.uuid, + sha256Path: join(this.outDirectory, this.streamingPlaylist.segmentsSha256Filename), + streamingPlaylist: this.streamingPlaylist, + sendToObjectStorage: CONFIG.OBJECT_STORAGE.ENABLED + }) + } + + private buildTranscodingWrapper () { + const options = { + streamingPlaylist: this.streamingPlaylist, + videoLive: this.videoLive, + + lTags: this.lTags, + + sessionId: this.sessionId, + inputLocalUrl: this.inputLocalUrl, + inputPublicUrl: this.inputPublicUrl, + + toTranscode: this.allResolutions.map(resolution => ({ + resolution, + fps: computeOutputFPS({ inputFPS: this.fps, resolution }) + })), + + fps: this.fps, + bitrate: this.bitrate, + ratio: this.ratio, + hasAudio: this.hasAudio, + + segmentListSize: VIDEO_LIVE.SEGMENTS_LIST_SIZE, + segmentDuration: getLiveSegmentTime(this.videoLive.latencyMode), + + outDirectory: this.outDirectory + } + + return CONFIG.LIVE.TRANSCODING.ENABLED && CONFIG.LIVE.TRANSCODING.REMOTE_RUNNERS.ENABLED + ? new RemoteTranscodingWrapper(options) + : new FFmpegTranscodingWrapper(options) + } + + private getPlaylistIdFromTS (segmentPath: string) { + const playlistIdMatcher = /^([\d+])-/ + + return basename(segmentPath).match(playlistIdMatcher)[1] + } + + private getPlaylistNameFromTS (segmentPath: string) { + return `${this.getPlaylistIdFromTS(segmentPath)}.m3u8` + } +} + +// --------------------------------------------------------------------------- + +export { + MuxingSession +} diff --git a/server/server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts b/server/server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts new file mode 100644 index 000000000..8cae0c45b --- /dev/null +++ b/server/server/lib/live/shared/transcoding-wrapper/abstract-transcoding-wrapper.ts @@ -0,0 +1,111 @@ +import { LiveVideoErrorType } from '@peertube/peertube-models' +import { LoggerTagsFn } from '@server/helpers/logger.js' +import { MStreamingPlaylistVideo, MVideoLiveVideo } from '@server/types/models/index.js' +import EventEmitter from 'events' + +interface TranscodingWrapperEvents { + 'end': () => void + + 'error': (options: { err: Error }) => void +} + +declare interface AbstractTranscodingWrapper { + on( + event: U, listener: TranscodingWrapperEvents[U] + ): this + + emit( + event: U, ...args: Parameters + ): boolean +} + +interface AbstractTranscodingWrapperOptions { + streamingPlaylist: MStreamingPlaylistVideo + videoLive: MVideoLiveVideo + + lTags: LoggerTagsFn + + sessionId: string + inputLocalUrl: string + inputPublicUrl: string + + fps: number + toTranscode: { + resolution: number + fps: number + }[] + + bitrate: number + ratio: number + hasAudio: boolean + + segmentListSize: number + segmentDuration: number + + outDirectory: string +} + +abstract class AbstractTranscodingWrapper extends EventEmitter { + protected readonly videoLive: MVideoLiveVideo + + protected readonly toTranscode: { + resolution: number + fps: number + }[] + + protected readonly sessionId: string + protected readonly inputLocalUrl: string + protected readonly inputPublicUrl: string + + protected readonly fps: number + protected readonly bitrate: number + protected readonly ratio: number + protected readonly hasAudio: boolean + + protected readonly segmentListSize: number + protected readonly segmentDuration: number + + protected readonly videoUUID: string + + protected readonly outDirectory: string + + protected readonly lTags: LoggerTagsFn + + protected readonly streamingPlaylist: MStreamingPlaylistVideo + + constructor (options: AbstractTranscodingWrapperOptions) { + super() + + this.lTags = options.lTags + + this.videoLive = options.videoLive + this.videoUUID = options.videoLive.Video.uuid + this.streamingPlaylist = options.streamingPlaylist + + this.sessionId = options.sessionId + this.inputLocalUrl = options.inputLocalUrl + this.inputPublicUrl = options.inputPublicUrl + + this.fps = options.fps + this.toTranscode = options.toTranscode + + this.bitrate = options.bitrate + this.ratio = options.ratio + this.hasAudio = options.hasAudio + + this.segmentListSize = options.segmentListSize + this.segmentDuration = options.segmentDuration + + this.outDirectory = options.outDirectory + } + + abstract run (): Promise + + abstract abort (error?: LiveVideoErrorType): void +} + +export { + type AbstractTranscodingWrapperOptions, + + AbstractTranscodingWrapper +} diff --git a/server/server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts b/server/server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts new file mode 100644 index 000000000..464686470 --- /dev/null +++ b/server/server/lib/live/shared/transcoding-wrapper/ffmpeg-transcoding-wrapper.ts @@ -0,0 +1,107 @@ +import { FfmpegCommand } from 'fluent-ffmpeg' +import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js' +import { logger } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' +import { VIDEO_LIVE } from '@server/initializers/constants.js' +import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles.js' +import { FFmpegLive } from '@peertube/peertube-ffmpeg' +import { getLiveSegmentTime } from '../../live-utils.js' +import { AbstractTranscodingWrapper } from './abstract-transcoding-wrapper.js' + +export class FFmpegTranscodingWrapper extends AbstractTranscodingWrapper { + private ffmpegCommand: FfmpegCommand + + private aborted = false + private errored = false + private ended = false + + async run () { + this.ffmpegCommand = CONFIG.LIVE.TRANSCODING.ENABLED + ? await this.buildFFmpegLive().getLiveTranscodingCommand({ + inputUrl: this.inputLocalUrl, + + outPath: this.outDirectory, + masterPlaylistName: this.streamingPlaylist.playlistFilename, + + segmentListSize: this.segmentListSize, + segmentDuration: this.segmentDuration, + + toTranscode: this.toTranscode, + + bitrate: this.bitrate, + ratio: this.ratio, + + hasAudio: this.hasAudio + }) + : this.buildFFmpegLive().getLiveMuxingCommand({ + inputUrl: this.inputLocalUrl, + outPath: this.outDirectory, + + masterPlaylistName: this.streamingPlaylist.playlistFilename, + + segmentListSize: VIDEO_LIVE.SEGMENTS_LIST_SIZE, + segmentDuration: getLiveSegmentTime(this.videoLive.latencyMode) + }) + + logger.info('Running local live muxing/transcoding for %s.', this.videoUUID, this.lTags()) + + let ffmpegShellCommand: string + this.ffmpegCommand.on('start', cmdline => { + ffmpegShellCommand = cmdline + + logger.debug('Running ffmpeg command for live', { ffmpegShellCommand, ...this.lTags() }) + }) + + this.ffmpegCommand.on('error', (err, stdout, stderr) => { + this.onFFmpegError({ err, stdout, stderr, ffmpegShellCommand }) + }) + + this.ffmpegCommand.on('end', () => { + this.onFFmpegEnded() + }) + + this.ffmpegCommand.run() + } + + abort () { + if (this.ended || this.errored || this.aborted) return + + logger.debug('Killing ffmpeg after live abort of ' + this.videoUUID, this.lTags()) + + this.ffmpegCommand.kill('SIGINT') + + this.aborted = true + this.emit('end') + } + + private onFFmpegError (options: { + err: any + stdout: string + stderr: string + ffmpegShellCommand: string + }) { + const { err, stdout, stderr, ffmpegShellCommand } = options + + // Don't care that we killed the ffmpeg process + if (err?.message?.includes('Exiting normally')) return + if (this.ended || this.errored || this.aborted) return + + logger.error('FFmpeg transcoding error.', { err, stdout, stderr, ffmpegShellCommand, ...this.lTags() }) + + this.errored = true + this.emit('error', { err }) + } + + private onFFmpegEnded () { + if (this.ended || this.errored || this.aborted) return + + logger.debug('Live ffmpeg transcoding ended for ' + this.videoUUID, this.lTags()) + + this.ended = true + this.emit('end') + } + + private buildFFmpegLive () { + return new FFmpegLive(getFFmpegCommandWrapperOptions('live', VideoTranscodingProfilesManager.Instance.getAvailableEncoders())) + } +} diff --git a/server/server/lib/live/shared/transcoding-wrapper/index.ts b/server/server/lib/live/shared/transcoding-wrapper/index.ts new file mode 100644 index 000000000..3ab9f9417 --- /dev/null +++ b/server/server/lib/live/shared/transcoding-wrapper/index.ts @@ -0,0 +1,3 @@ +export * from './abstract-transcoding-wrapper.js' +export * from './ffmpeg-transcoding-wrapper.js' +export * from './remote-transcoding-wrapper.js' diff --git a/server/server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts b/server/server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts new file mode 100644 index 000000000..4e375a749 --- /dev/null +++ b/server/server/lib/live/shared/transcoding-wrapper/remote-transcoding-wrapper.ts @@ -0,0 +1,21 @@ +import { LiveRTMPHLSTranscodingJobHandler } from '@server/lib/runners/index.js' +import { AbstractTranscodingWrapper } from './abstract-transcoding-wrapper.js' + +export class RemoteTranscodingWrapper extends AbstractTranscodingWrapper { + async run () { + await new LiveRTMPHLSTranscodingJobHandler().create({ + rtmpUrl: this.inputPublicUrl, + sessionId: this.sessionId, + toTranscode: this.toTranscode, + video: this.videoLive.Video, + outputDirectory: this.outDirectory, + playlist: this.streamingPlaylist, + segmentListSize: this.segmentListSize, + segmentDuration: this.segmentDuration + }) + } + + abort () { + this.emit('end') + } +} diff --git a/server/server/lib/local-actor.ts b/server/server/lib/local-actor.ts new file mode 100644 index 000000000..5ee9df875 --- /dev/null +++ b/server/server/lib/local-actor.ts @@ -0,0 +1,101 @@ +import { remove } from 'fs-extra/esm' +import { join } from 'path' +import { Transaction } from 'sequelize' +import { ActivityPubActorType, ActorImageType, ActorImageType_Type } from '@peertube/peertube-models' +import { ActorModel } from '@server/models/actor/actor.js' +import { buildUUID, getLowercaseExtension } from '@peertube/peertube-node-utils' +import { retryTransactionWrapper } from '../helpers/database-utils.js' +import { CONFIG } from '../initializers/config.js' +import { ACTOR_IMAGES_SIZE, WEBSERVER } from '../initializers/constants.js' +import { sequelizeTypescript } from '../initializers/database.js' +import { MAccountDefault, MActor, MChannelDefault } from '../types/models/index.js' +import { deleteActorImages, updateActorImages } from './activitypub/actors/index.js' +import { sendUpdateActor } from './activitypub/send/index.js' +import { processImageFromWorker } from './worker/parent-process.js' + +export function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string) { + return new ActorModel({ + type, + url, + preferredUsername, + publicKey: null, + privateKey: null, + followersCount: 0, + followingCount: 0, + inboxUrl: url + '/inbox', + outboxUrl: url + '/outbox', + sharedInboxUrl: WEBSERVER.URL + '/inbox', + followersUrl: url + '/followers', + followingUrl: url + '/following' + }) as MActor +} + +export async function updateLocalActorImageFiles ( + accountOrChannel: MAccountDefault | MChannelDefault, + imagePhysicalFile: Express.Multer.File, + type: ActorImageType_Type +) { + const processImageSize = async (imageSize: { width: number, height: number }) => { + const extension = getLowercaseExtension(imagePhysicalFile.filename) + + const imageName = buildUUID() + extension + const destination = join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, imageName) + await processImageFromWorker({ path: imagePhysicalFile.path, destination, newSize: imageSize, keepOriginal: true }) + + return { + imageName, + imageSize + } + } + + const processedImages = await Promise.all(ACTOR_IMAGES_SIZE[type].map(processImageSize)) + await remove(imagePhysicalFile.path) + + return retryTransactionWrapper(() => sequelizeTypescript.transaction(async t => { + const actorImagesInfo = processedImages.map(({ imageName, imageSize }) => ({ + name: imageName, + fileUrl: null, + height: imageSize.height, + width: imageSize.width, + onDisk: true + })) + + const updatedActor = await updateActorImages(accountOrChannel.Actor, type, actorImagesInfo, t) + await updatedActor.save({ transaction: t }) + + await sendUpdateActor(accountOrChannel, t) + + return type === ActorImageType.AVATAR + ? updatedActor.Avatars + : updatedActor.Banners + })) +} + +export async function deleteLocalActorImageFile (accountOrChannel: MAccountDefault | MChannelDefault, type: ActorImageType_Type) { + return retryTransactionWrapper(() => { + return sequelizeTypescript.transaction(async t => { + const updatedActor = await deleteActorImages(accountOrChannel.Actor, type, t) + await updatedActor.save({ transaction: t }) + + await sendUpdateActor(accountOrChannel, t) + + return updatedActor.Avatars + }) + }) +} + +// --------------------------------------------------------------------------- + +export async function findAvailableLocalActorName (baseActorName: string, transaction?: Transaction) { + let actor = await ActorModel.loadLocalByName(baseActorName, transaction) + if (!actor) return baseActorName + + for (let i = 1; i < 30; i++) { + const name = `${baseActorName}-${i}` + + actor = await ActorModel.loadLocalByName(name, transaction) + if (!actor) return name + } + + throw new Error('Cannot find available actor local name (too much iterations).') +} diff --git a/server/server/lib/model-loaders/actor.ts b/server/server/lib/model-loaders/actor.ts new file mode 100644 index 000000000..3c1179391 --- /dev/null +++ b/server/server/lib/model-loaders/actor.ts @@ -0,0 +1,16 @@ +import { ActorModel } from '../../models/actor/actor.js' +import { MActorAccountChannelId, MActorFull } from '../../types/models/index.js' + +type ActorLoadByUrlType = 'all' | 'association-ids' + +function loadActorByUrl (url: string, fetchType: ActorLoadByUrlType): Promise { + if (fetchType === 'all') return ActorModel.loadByUrlAndPopulateAccountAndChannel(url) + + if (fetchType === 'association-ids') return ActorModel.loadByUrl(url) +} + +export { + type ActorLoadByUrlType, + + loadActorByUrl +} diff --git a/server/server/lib/model-loaders/index.ts b/server/server/lib/model-loaders/index.ts new file mode 100644 index 000000000..7aa426a56 --- /dev/null +++ b/server/server/lib/model-loaders/index.ts @@ -0,0 +1,2 @@ +export * from './actor.js' +export * from './video.js' diff --git a/server/server/lib/model-loaders/video.ts b/server/server/lib/model-loaders/video.ts new file mode 100644 index 000000000..5ea2b67aa --- /dev/null +++ b/server/server/lib/model-loaders/video.ts @@ -0,0 +1,66 @@ +import { VideoModel } from '@server/models/video/video.js' +import { + MVideoAccountLightBlacklistAllFiles, + MVideoFormattableDetails, + MVideoFullLight, + MVideoId, + MVideoImmutable, + MVideoThumbnail +} from '@server/types/models/index.js' + +type VideoLoadType = 'for-api' | 'all' | 'only-video' | 'id' | 'none' | 'only-immutable-attributes' + +function loadVideo (id: number | string, fetchType: 'for-api', userId?: number): Promise +function loadVideo (id: number | string, fetchType: 'all', userId?: number): Promise +function loadVideo (id: number | string, fetchType: 'only-immutable-attributes'): Promise +function loadVideo (id: number | string, fetchType: 'only-video', userId?: number): Promise +function loadVideo (id: number | string, fetchType: 'id' | 'none', userId?: number): Promise +function loadVideo ( + id: number | string, + fetchType: VideoLoadType, + userId?: number +): Promise +function loadVideo ( + id: number | string, + fetchType: VideoLoadType, + userId?: number +): Promise { + + if (fetchType === 'for-api') return VideoModel.loadForGetAPI({ id, userId }) + + if (fetchType === 'all') return VideoModel.loadFull(id, undefined, userId) + + if (fetchType === 'only-immutable-attributes') return VideoModel.loadImmutableAttributes(id) + + if (fetchType === 'only-video') return VideoModel.load(id) + + if (fetchType === 'id' || fetchType === 'none') return VideoModel.loadOnlyId(id) +} + +type VideoLoadByUrlType = 'all' | 'only-video' | 'only-immutable-attributes' + +function loadVideoByUrl (url: string, fetchType: 'all'): Promise +function loadVideoByUrl (url: string, fetchType: 'only-immutable-attributes'): Promise +function loadVideoByUrl (url: string, fetchType: 'only-video'): Promise +function loadVideoByUrl ( + url: string, + fetchType: VideoLoadByUrlType +): Promise +function loadVideoByUrl ( + url: string, + fetchType: VideoLoadByUrlType +): Promise { + if (fetchType === 'all') return VideoModel.loadByUrlAndPopulateAccount(url) + + if (fetchType === 'only-immutable-attributes') return VideoModel.loadByUrlImmutableAttributes(url) + + if (fetchType === 'only-video') return VideoModel.loadByUrl(url) +} + +export { + type VideoLoadType, + type VideoLoadByUrlType, + + loadVideo, + loadVideoByUrl +} diff --git a/server/server/lib/moderation.ts b/server/server/lib/moderation.ts new file mode 100644 index 000000000..d02a12a30 --- /dev/null +++ b/server/server/lib/moderation.ts @@ -0,0 +1,257 @@ +import express, { VideoUploadFile } from 'express' +import { PathLike } from 'fs-extra/esm' +import { Transaction } from 'sequelize' +import { AbuseAuditView, auditLoggerFactory } from '@server/helpers/audit-logger.js' +import { afterCommitIfTransaction } from '@server/helpers/database-utils.js' +import { logger } from '@server/helpers/logger.js' +import { AbuseModel } from '@server/models/abuse/abuse.js' +import { VideoAbuseModel } from '@server/models/abuse/video-abuse.js' +import { VideoCommentAbuseModel } from '@server/models/abuse/video-comment-abuse.js' +import { VideoFileModel } from '@server/models/video/video-file.js' +import { FilteredModelAttributes } from '@server/types/index.js' +import { + MAbuseFull, + MAccountDefault, + MAccountLight, + MComment, + MCommentAbuseAccountVideo, + MCommentOwnerVideo, + MUser, + MVideoAbuseVideoFull, + MVideoAccountLightBlacklistAllFiles +} from '@server/types/models/index.js' +import { LiveVideoCreate, VideoCommentCreate, VideoCreate, VideoImportCreate } from '@peertube/peertube-models' +import { UserModel } from '../models/user/user.js' +import { VideoCommentModel } from '../models/video/video-comment.js' +import { VideoModel } from '../models/video/video.js' +import { sendAbuse } from './activitypub/send/send-flag.js' +import { Notifier } from './notifier/index.js' + +export type AcceptResult = { + accepted: boolean + errorMessage?: string +} + +// --------------------------------------------------------------------------- + +// Stub function that can be filtered by plugins +function isLocalVideoFileAccepted (object: { + videoBody: VideoCreate + videoFile: VideoUploadFile + user: UserModel +}): AcceptResult { + return { accepted: true } +} + +// --------------------------------------------------------------------------- + +// Stub function that can be filtered by plugins +function isLocalLiveVideoAccepted (object: { + liveVideoBody: LiveVideoCreate + user: UserModel +}): AcceptResult { + return { accepted: true } +} + +// --------------------------------------------------------------------------- + +// Stub function that can be filtered by plugins +function isLocalVideoThreadAccepted (_object: { + req: express.Request + commentBody: VideoCommentCreate + video: VideoModel + user: UserModel +}): AcceptResult { + return { accepted: true } +} + +// Stub function that can be filtered by plugins +function isLocalVideoCommentReplyAccepted (_object: { + req: express.Request + commentBody: VideoCommentCreate + parentComment: VideoCommentModel + video: VideoModel + user: UserModel +}): AcceptResult { + return { accepted: true } +} + +// --------------------------------------------------------------------------- + +// Stub function that can be filtered by plugins +function isRemoteVideoCommentAccepted (_object: { + comment: MComment +}): AcceptResult { + return { accepted: true } +} + +// --------------------------------------------------------------------------- + +// Stub function that can be filtered by plugins +function isPreImportVideoAccepted (object: { + videoImportBody: VideoImportCreate + user: MUser +}): AcceptResult { + return { accepted: true } +} + +// Stub function that can be filtered by plugins +function isPostImportVideoAccepted (object: { + videoFilePath: PathLike + videoFile: VideoFileModel + user: MUser +}): AcceptResult { + return { accepted: true } +} + +// --------------------------------------------------------------------------- + +async function createVideoAbuse (options: { + baseAbuse: FilteredModelAttributes + videoInstance: MVideoAccountLightBlacklistAllFiles + startAt: number + endAt: number + transaction: Transaction + reporterAccount: MAccountDefault + skipNotification: boolean +}) { + const { baseAbuse, videoInstance, startAt, endAt, transaction, reporterAccount, skipNotification } = options + + const associateFun = async (abuseInstance: MAbuseFull) => { + const videoAbuseInstance: MVideoAbuseVideoFull = await VideoAbuseModel.create({ + abuseId: abuseInstance.id, + videoId: videoInstance.id, + startAt, + endAt + }, { transaction }) + + videoAbuseInstance.Video = videoInstance + abuseInstance.VideoAbuse = videoAbuseInstance + + return { isOwned: videoInstance.isOwned() } + } + + return createAbuse({ + base: baseAbuse, + reporterAccount, + flaggedAccount: videoInstance.VideoChannel.Account, + transaction, + skipNotification, + associateFun + }) +} + +function createVideoCommentAbuse (options: { + baseAbuse: FilteredModelAttributes + commentInstance: MCommentOwnerVideo + transaction: Transaction + reporterAccount: MAccountDefault + skipNotification: boolean +}) { + const { baseAbuse, commentInstance, transaction, reporterAccount, skipNotification } = options + + const associateFun = async (abuseInstance: MAbuseFull) => { + const commentAbuseInstance: MCommentAbuseAccountVideo = await VideoCommentAbuseModel.create({ + abuseId: abuseInstance.id, + videoCommentId: commentInstance.id + }, { transaction }) + + commentAbuseInstance.VideoComment = commentInstance + abuseInstance.VideoCommentAbuse = commentAbuseInstance + + return { isOwned: commentInstance.isOwned() } + } + + return createAbuse({ + base: baseAbuse, + reporterAccount, + flaggedAccount: commentInstance.Account, + transaction, + skipNotification, + associateFun + }) +} + +function createAccountAbuse (options: { + baseAbuse: FilteredModelAttributes + accountInstance: MAccountDefault + transaction: Transaction + reporterAccount: MAccountDefault + skipNotification: boolean +}) { + const { baseAbuse, accountInstance, transaction, reporterAccount, skipNotification } = options + + const associateFun = () => { + return Promise.resolve({ isOwned: accountInstance.isOwned() }) + } + + return createAbuse({ + base: baseAbuse, + reporterAccount, + flaggedAccount: accountInstance, + transaction, + skipNotification, + associateFun + }) +} + +// --------------------------------------------------------------------------- + +export { + isLocalLiveVideoAccepted, + + isLocalVideoFileAccepted, + isLocalVideoThreadAccepted, + isRemoteVideoCommentAccepted, + isLocalVideoCommentReplyAccepted, + isPreImportVideoAccepted, + isPostImportVideoAccepted, + + createAbuse, + createVideoAbuse, + createVideoCommentAbuse, + createAccountAbuse +} + +// --------------------------------------------------------------------------- + +async function createAbuse (options: { + base: FilteredModelAttributes + reporterAccount: MAccountDefault + flaggedAccount: MAccountLight + associateFun: (abuseInstance: MAbuseFull) => Promise<{ isOwned: boolean }> + skipNotification: boolean + transaction: Transaction +}) { + const { base, reporterAccount, flaggedAccount, associateFun, transaction, skipNotification } = options + const auditLogger = auditLoggerFactory('abuse') + + const abuseAttributes = Object.assign({}, base, { flaggedAccountId: flaggedAccount.id }) + const abuseInstance: MAbuseFull = await AbuseModel.create(abuseAttributes, { transaction }) + + abuseInstance.ReporterAccount = reporterAccount + abuseInstance.FlaggedAccount = flaggedAccount + + const { isOwned } = await associateFun(abuseInstance) + + if (isOwned === false) { + sendAbuse(reporterAccount.Actor, abuseInstance, abuseInstance.FlaggedAccount, transaction) + } + + const abuseJSON = abuseInstance.toFormattedAdminJSON() + auditLogger.create(reporterAccount.Actor.getIdentifier(), new AbuseAuditView(abuseJSON)) + + if (!skipNotification) { + afterCommitIfTransaction(transaction, () => { + Notifier.Instance.notifyOnNewAbuse({ + abuse: abuseJSON, + abuseInstance, + reporter: reporterAccount.Actor.getIdentifier() + }) + }) + } + + logger.info('Abuse report %d created.', abuseInstance.id) + + return abuseJSON +} diff --git a/server/server/lib/notifier/index.ts b/server/server/lib/notifier/index.ts new file mode 100644 index 000000000..8476ba7cc --- /dev/null +++ b/server/server/lib/notifier/index.ts @@ -0,0 +1 @@ +export * from './notifier.js' diff --git a/server/server/lib/notifier/notifier.ts b/server/server/lib/notifier/notifier.ts new file mode 100644 index 000000000..5cb227bba --- /dev/null +++ b/server/server/lib/notifier/notifier.ts @@ -0,0 +1,292 @@ +import { UserNotificationSettingValue, UserNotificationSettingValueType } from '@peertube/peertube-models' +import { MRegistration, MUser, MUserDefault } from '@server/types/models/user/index.js' +import { MVideoBlacklistLightVideo, MVideoBlacklistVideo } from '@server/types/models/video/video-blacklist.js' +import { logger } from '../../helpers/logger.js' +import { CONFIG } from '../../initializers/config.js' +import { + MAbuseFull, + MAbuseMessage, + MActorFollowFull, + MApplication, + MCommentOwnerVideo, + MPlugin, + MVideoAccountLight, + MVideoFullLight +} from '../../types/models/index.js' +import { JobQueue } from '../job-queue/index.js' +import { PeerTubeSocket } from '../peertube-socket.js' +import { Hooks } from '../plugins/hooks.js' +import { + AbstractNotification, + AbuseStateChangeForReporter, + AutoFollowForInstance, + CommentMention, + DirectRegistrationForModerators, + FollowForInstance, + FollowForUser, + ImportFinishedForOwner, + ImportFinishedForOwnerPayload, + NewAbuseForModerators, + NewAbuseMessageForModerators, + NewAbuseMessageForReporter, + NewAbusePayload, + NewAutoBlacklistForModerators, + NewBlacklistForOwner, + NewCommentForVideoOwner, + NewPeerTubeVersionForAdmins, + NewPluginVersionForAdmins, + NewVideoForSubscribers, + OwnedPublicationAfterAutoUnblacklist, + OwnedPublicationAfterScheduleUpdate, + OwnedPublicationAfterTranscoding, + RegistrationRequestForModerators, + StudioEditionFinishedForOwner, + UnblacklistForOwner +} from './shared/index.js' + +class Notifier { + + private readonly notificationModels = { + newVideo: [ NewVideoForSubscribers ], + publicationAfterTranscoding: [ OwnedPublicationAfterTranscoding ], + publicationAfterScheduleUpdate: [ OwnedPublicationAfterScheduleUpdate ], + publicationAfterAutoUnblacklist: [ OwnedPublicationAfterAutoUnblacklist ], + newComment: [ CommentMention, NewCommentForVideoOwner ], + newAbuse: [ NewAbuseForModerators ], + newBlacklist: [ NewBlacklistForOwner ], + unblacklist: [ UnblacklistForOwner ], + importFinished: [ ImportFinishedForOwner ], + directRegistration: [ DirectRegistrationForModerators ], + registrationRequest: [ RegistrationRequestForModerators ], + userFollow: [ FollowForUser ], + instanceFollow: [ FollowForInstance ], + autoInstanceFollow: [ AutoFollowForInstance ], + newAutoBlacklist: [ NewAutoBlacklistForModerators ], + abuseStateChange: [ AbuseStateChangeForReporter ], + newAbuseMessage: [ NewAbuseMessageForReporter, NewAbuseMessageForModerators ], + newPeertubeVersion: [ NewPeerTubeVersionForAdmins ], + newPluginVersion: [ NewPluginVersionForAdmins ], + videoStudioEditionFinished: [ StudioEditionFinishedForOwner ] + } + + private static instance: Notifier + + private constructor () { + } + + notifyOnNewVideoIfNeeded (video: MVideoAccountLight): void { + const models = this.notificationModels.newVideo + + this.sendNotifications(models, video) + .catch(err => logger.error('Cannot notify subscribers of new video %s.', video.url, { err })) + } + + notifyOnVideoPublishedAfterTranscoding (video: MVideoFullLight): void { + const models = this.notificationModels.publicationAfterTranscoding + + this.sendNotifications(models, video) + .catch(err => logger.error('Cannot notify owner that its video %s has been published after transcoding.', video.url, { err })) + } + + notifyOnVideoPublishedAfterScheduledUpdate (video: MVideoFullLight): void { + const models = this.notificationModels.publicationAfterScheduleUpdate + + this.sendNotifications(models, video) + .catch(err => logger.error('Cannot notify owner that its video %s has been published after scheduled update.', video.url, { err })) + } + + notifyOnVideoPublishedAfterRemovedFromAutoBlacklist (video: MVideoFullLight): void { + const models = this.notificationModels.publicationAfterAutoUnblacklist + + this.sendNotifications(models, video) + .catch(err => { + logger.error('Cannot notify owner that its video %s has been published after removed from auto-blacklist.', video.url, { err }) + }) + } + + notifyOnNewComment (comment: MCommentOwnerVideo): void { + const models = this.notificationModels.newComment + + this.sendNotifications(models, comment) + .catch(err => logger.error('Cannot notify of new comment.', comment.url, { err })) + } + + notifyOnNewAbuse (payload: NewAbusePayload): void { + const models = this.notificationModels.newAbuse + + this.sendNotifications(models, payload) + .catch(err => logger.error('Cannot notify of new abuse %d.', payload.abuseInstance.id, { err })) + } + + notifyOnVideoAutoBlacklist (videoBlacklist: MVideoBlacklistLightVideo): void { + const models = this.notificationModels.newAutoBlacklist + + this.sendNotifications(models, videoBlacklist) + .catch(err => logger.error('Cannot notify of auto-blacklist of video %s.', videoBlacklist.Video.url, { err })) + } + + notifyOnVideoBlacklist (videoBlacklist: MVideoBlacklistVideo): void { + const models = this.notificationModels.newBlacklist + + this.sendNotifications(models, videoBlacklist) + .catch(err => logger.error('Cannot notify video owner of new video blacklist of %s.', videoBlacklist.Video.url, { err })) + } + + notifyOnVideoUnblacklist (video: MVideoFullLight): void { + const models = this.notificationModels.unblacklist + + this.sendNotifications(models, video) + .catch(err => logger.error('Cannot notify video owner of unblacklist of %s.', video.url, { err })) + } + + notifyOnFinishedVideoImport (payload: ImportFinishedForOwnerPayload): void { + const models = this.notificationModels.importFinished + + this.sendNotifications(models, payload) + .catch(err => { + logger.error('Cannot notify owner that its video import %s is finished.', payload.videoImport.getTargetIdentifier(), { err }) + }) + } + + notifyOnNewDirectRegistration (user: MUserDefault): void { + const models = this.notificationModels.directRegistration + + this.sendNotifications(models, user) + .catch(err => logger.error('Cannot notify moderators of new user registration (%s).', user.username, { err })) + } + + notifyOnNewRegistrationRequest (registration: MRegistration): void { + const models = this.notificationModels.registrationRequest + + this.sendNotifications(models, registration) + .catch(err => logger.error('Cannot notify moderators of new registration request (%s).', registration.username, { err })) + } + + notifyOfNewUserFollow (actorFollow: MActorFollowFull): void { + const models = this.notificationModels.userFollow + + this.sendNotifications(models, actorFollow) + .catch(err => { + logger.error( + 'Cannot notify owner of channel %s of a new follow by %s.', + actorFollow.ActorFollowing.VideoChannel.getDisplayName(), + actorFollow.ActorFollower.Account.getDisplayName(), + { err } + ) + }) + } + + notifyOfNewInstanceFollow (actorFollow: MActorFollowFull): void { + const models = this.notificationModels.instanceFollow + + this.sendNotifications(models, actorFollow) + .catch(err => logger.error('Cannot notify administrators of new follower %s.', actorFollow.ActorFollower.url, { err })) + } + + notifyOfAutoInstanceFollowing (actorFollow: MActorFollowFull): void { + const models = this.notificationModels.autoInstanceFollow + + this.sendNotifications(models, actorFollow) + .catch(err => logger.error('Cannot notify administrators of auto instance following %s.', actorFollow.ActorFollowing.url, { err })) + } + + notifyOnAbuseStateChange (abuse: MAbuseFull): void { + const models = this.notificationModels.abuseStateChange + + this.sendNotifications(models, abuse) + .catch(err => logger.error('Cannot notify of abuse %d state change.', abuse.id, { err })) + } + + notifyOnAbuseMessage (abuse: MAbuseFull, message: MAbuseMessage): void { + const models = this.notificationModels.newAbuseMessage + + this.sendNotifications(models, { abuse, message }) + .catch(err => logger.error('Cannot notify on new abuse %d message.', abuse.id, { err })) + } + + notifyOfNewPeerTubeVersion (application: MApplication, latestVersion: string) { + const models = this.notificationModels.newPeertubeVersion + + this.sendNotifications(models, { application, latestVersion }) + .catch(err => logger.error('Cannot notify on new PeerTubeb version %s.', latestVersion, { err })) + } + + notifyOfNewPluginVersion (plugin: MPlugin) { + const models = this.notificationModels.newPluginVersion + + this.sendNotifications(models, plugin) + .catch(err => logger.error('Cannot notify on new plugin version %s.', plugin.name, { err })) + } + + notifyOfFinishedVideoStudioEdition (video: MVideoFullLight) { + const models = this.notificationModels.videoStudioEditionFinished + + this.sendNotifications(models, video) + .catch(err => logger.error('Cannot notify on finished studio edition %s.', video.url, { err })) + } + + private async notify (object: AbstractNotification) { + await object.prepare() + + const users = object.getTargetUsers() + + if (users.length === 0) return + if (await object.isDisabled()) return + + object.log() + + const toEmails: string[] = [] + + for (const user of users) { + const setting = object.getSetting(user) + + const webNotificationEnabled = this.isWebNotificationEnabled(setting) + const emailNotificationEnabled = this.isEmailEnabled(user, setting) + const notification = object.createNotification(user) + + if (webNotificationEnabled) { + await notification.save() + + PeerTubeSocket.Instance.sendNotification(user.id, notification) + } + + if (emailNotificationEnabled) { + toEmails.push(user.email) + } + + Hooks.runAction('action:notifier.notification.created', { webNotificationEnabled, emailNotificationEnabled, user, notification }) + } + + for (const to of toEmails) { + const payload = await object.createEmail(to) + JobQueue.Instance.createJobAsync({ type: 'email', payload }) + } + } + + private isEmailEnabled (user: MUser, value: UserNotificationSettingValueType) { + if (CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION === true && user.emailVerified === false) return false + + return value & UserNotificationSettingValue.EMAIL + } + + private isWebNotificationEnabled (value: UserNotificationSettingValueType) { + return value & UserNotificationSettingValue.WEB + } + + private async sendNotifications (models: (new (payload: T) => AbstractNotification)[], payload: T) { + for (const model of models) { + // eslint-disable-next-line new-cap + await this.notify(new model(payload)) + } + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} + +// --------------------------------------------------------------------------- + +export { + Notifier +} diff --git a/server/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts b/server/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts new file mode 100644 index 000000000..f9a940299 --- /dev/null +++ b/server/server/lib/notifier/shared/abuse/abstract-new-abuse-message.ts @@ -0,0 +1,73 @@ +import { UserNotificationType } from '@peertube/peertube-models' +import { WEBSERVER } from '@server/initializers/constants.js' +import { AccountModel } from '@server/models/account/account.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { + MAbuseFull, + MAbuseMessage, + MAccountDefault, + MUserWithNotificationSetting, + UserNotificationModelForApi +} from '@server/types/models/index.js' +import { AbstractNotification } from '../common/abstract-notification.js' + +type NewAbuseMessagePayload = { + abuse: MAbuseFull + message: MAbuseMessage +} + +export abstract class AbstractNewAbuseMessage extends AbstractNotification { + protected messageAccount: MAccountDefault + + async loadMessageAccount () { + this.messageAccount = await AccountModel.load(this.message.accountId) + } + + getSetting (user: MUserWithNotificationSetting) { + return user.NotificationSetting.abuseNewMessage + } + + createNotification (user: MUserWithNotificationSetting) { + const notification = UserNotificationModel.build({ + type: UserNotificationType.ABUSE_NEW_MESSAGE, + userId: user.id, + abuseId: this.abuse.id + }) + notification.Abuse = this.abuse + + return notification + } + + protected createEmailFor (to: string, target: 'moderator' | 'reporter') { + const text = 'New message on report #' + this.abuse.id + const abuseUrl = target === 'moderator' + ? WEBSERVER.URL + '/admin/moderation/abuses/list?search=%23' + this.abuse.id + : WEBSERVER.URL + '/my-account/abuses?search=%23' + this.abuse.id + + const action = { + text: 'View report #' + this.abuse.id, + url: abuseUrl + } + + return { + template: 'abuse-new-message', + to, + subject: text, + locals: { + abuseId: this.abuse.id, + abuseUrl: action.url, + messageAccountName: this.messageAccount.getDisplayName(), + messageText: this.message.message, + action + } + } + } + + protected get abuse () { + return this.payload.abuse + } + + protected get message () { + return this.payload.message + } +} diff --git a/server/server/lib/notifier/shared/abuse/abuse-state-change-for-reporter.ts b/server/server/lib/notifier/shared/abuse/abuse-state-change-for-reporter.ts new file mode 100644 index 000000000..5e18e9fc9 --- /dev/null +++ b/server/server/lib/notifier/shared/abuse/abuse-state-change-for-reporter.ts @@ -0,0 +1,74 @@ +import { logger } from '@server/helpers/logger.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { getAbuseTargetUrl } from '@server/lib/activitypub/url.js' +import { UserModel } from '@server/models/user/user.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { MAbuseFull, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models/index.js' +import { AbuseState, UserNotificationType } from '@peertube/peertube-models' +import { AbstractNotification } from '../common/abstract-notification.js' + +export class AbuseStateChangeForReporter extends AbstractNotification { + + private user: MUserDefault + + async prepare () { + const reporter = this.abuse.ReporterAccount + if (reporter.isOwned() !== true) return + + this.user = await UserModel.loadByAccountActorId(this.abuse.ReporterAccount.actorId) + } + + log () { + logger.info('Notifying reporter of abuse % of state change.', getAbuseTargetUrl(this.abuse)) + } + + getSetting (user: MUserWithNotificationSetting) { + return user.NotificationSetting.abuseStateChange + } + + getTargetUsers () { + if (!this.user) return [] + + return [ this.user ] + } + + createNotification (user: MUserWithNotificationSetting) { + const notification = UserNotificationModel.build({ + type: UserNotificationType.ABUSE_STATE_CHANGE, + userId: user.id, + abuseId: this.abuse.id + }) + notification.Abuse = this.abuse + + return notification + } + + createEmail (to: string) { + const text = this.abuse.state === AbuseState.ACCEPTED + ? 'Report #' + this.abuse.id + ' has been accepted' + : 'Report #' + this.abuse.id + ' has been rejected' + + const abuseUrl = WEBSERVER.URL + '/my-account/abuses?search=%23' + this.abuse.id + + const action = { + text: 'View report #' + this.abuse.id, + url: abuseUrl + } + + return { + template: 'abuse-state-change', + to, + subject: text, + locals: { + action, + abuseId: this.abuse.id, + abuseUrl, + isAccepted: this.abuse.state === AbuseState.ACCEPTED + } + } + } + + private get abuse () { + return this.payload + } +} diff --git a/server/server/lib/notifier/shared/abuse/index.ts b/server/server/lib/notifier/shared/abuse/index.ts new file mode 100644 index 000000000..4f4b30503 --- /dev/null +++ b/server/server/lib/notifier/shared/abuse/index.ts @@ -0,0 +1,4 @@ +export * from './abuse-state-change-for-reporter.js' +export * from './new-abuse-for-moderators.js' +export * from './new-abuse-message-for-reporter.js' +export * from './new-abuse-message-for-moderators.js' diff --git a/server/server/lib/notifier/shared/abuse/new-abuse-for-moderators.ts b/server/server/lib/notifier/shared/abuse/new-abuse-for-moderators.ts new file mode 100644 index 000000000..2bd93133d --- /dev/null +++ b/server/server/lib/notifier/shared/abuse/new-abuse-for-moderators.ts @@ -0,0 +1,119 @@ +import { logger } from '@server/helpers/logger.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { getAbuseTargetUrl } from '@server/lib/activitypub/url.js' +import { UserModel } from '@server/models/user/user.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { MAbuseFull, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models/index.js' +import { UserAbuse, UserNotificationType, UserRight } from '@peertube/peertube-models' +import { AbstractNotification } from '../common/abstract-notification.js' + +export type NewAbusePayload = { abuse: UserAbuse, abuseInstance: MAbuseFull, reporter: string } + +export class NewAbuseForModerators extends AbstractNotification { + private moderators: MUserDefault[] + + async prepare () { + this.moderators = await UserModel.listWithRight(UserRight.MANAGE_ABUSES) + } + + log () { + logger.info('Notifying %s user/moderators of new abuse %s.', this.moderators.length, getAbuseTargetUrl(this.payload.abuseInstance)) + } + + getSetting (user: MUserWithNotificationSetting) { + return user.NotificationSetting.abuseAsModerator + } + + getTargetUsers () { + return this.moderators + } + + createNotification (user: MUserWithNotificationSetting) { + const notification = UserNotificationModel.build({ + type: UserNotificationType.NEW_ABUSE_FOR_MODERATORS, + userId: user.id, + abuseId: this.payload.abuseInstance.id + }) + notification.Abuse = this.payload.abuseInstance + + return notification + } + + createEmail (to: string) { + const abuseInstance = this.payload.abuseInstance + + if (abuseInstance.VideoAbuse) return this.createVideoAbuseEmail(to) + if (abuseInstance.VideoCommentAbuse) return this.createCommentAbuseEmail(to) + + return this.createAccountAbuseEmail(to) + } + + private createVideoAbuseEmail (to: string) { + const video = this.payload.abuseInstance.VideoAbuse.Video + const videoUrl = WEBSERVER.URL + video.getWatchStaticPath() + + return { + template: 'video-abuse-new', + to, + subject: `New video abuse report from ${this.payload.reporter}`, + locals: { + videoUrl, + isLocal: video.remote === false, + videoCreatedAt: new Date(video.createdAt).toLocaleString(), + videoPublishedAt: new Date(video.publishedAt).toLocaleString(), + videoName: video.name, + reason: this.payload.abuse.reason, + videoChannel: this.payload.abuse.video.channel, + reporter: this.payload.reporter, + action: this.buildEmailAction() + } + } + } + + private createCommentAbuseEmail (to: string) { + const comment = this.payload.abuseInstance.VideoCommentAbuse.VideoComment + const commentUrl = WEBSERVER.URL + comment.Video.getWatchStaticPath() + ';threadId=' + comment.getThreadId() + + return { + template: 'video-comment-abuse-new', + to, + subject: `New comment abuse report from ${this.payload.reporter}`, + locals: { + commentUrl, + videoName: comment.Video.name, + isLocal: comment.isOwned(), + commentCreatedAt: new Date(comment.createdAt).toLocaleString(), + reason: this.payload.abuse.reason, + flaggedAccount: this.payload.abuseInstance.FlaggedAccount.getDisplayName(), + reporter: this.payload.reporter, + action: this.buildEmailAction() + } + } + } + + private createAccountAbuseEmail (to: string) { + const account = this.payload.abuseInstance.FlaggedAccount + const accountUrl = account.getClientUrl() + + return { + template: 'account-abuse-new', + to, + subject: `New account abuse report from ${this.payload.reporter}`, + locals: { + accountUrl, + accountDisplayName: account.getDisplayName(), + isLocal: account.isOwned(), + reason: this.payload.abuse.reason, + reporter: this.payload.reporter, + action: this.buildEmailAction() + } + } + } + + private buildEmailAction () { + return { + text: 'View report #' + this.payload.abuseInstance.id, + url: WEBSERVER.URL + '/admin/moderation/abuses/list?search=%23' + this.payload.abuseInstance.id + } + } +} diff --git a/server/server/lib/notifier/shared/abuse/new-abuse-message-for-moderators.ts b/server/server/lib/notifier/shared/abuse/new-abuse-message-for-moderators.ts new file mode 100644 index 000000000..d3f590e46 --- /dev/null +++ b/server/server/lib/notifier/shared/abuse/new-abuse-message-for-moderators.ts @@ -0,0 +1,32 @@ +import { logger } from '@server/helpers/logger.js' +import { getAbuseTargetUrl } from '@server/lib/activitypub/url.js' +import { UserModel } from '@server/models/user/user.js' +import { MUserDefault } from '@server/types/models/index.js' +import { UserRight } from '@peertube/peertube-models' +import { AbstractNewAbuseMessage } from './abstract-new-abuse-message.js' + +export class NewAbuseMessageForModerators extends AbstractNewAbuseMessage { + private moderators: MUserDefault[] + + async prepare () { + this.moderators = await UserModel.listWithRight(UserRight.MANAGE_ABUSES) + + // Don't notify my own message + this.moderators = this.moderators.filter(m => m.Account.id !== this.message.accountId) + if (this.moderators.length === 0) return + + await this.loadMessageAccount() + } + + log () { + logger.info('Notifying moderators of new abuse message on %s.', getAbuseTargetUrl(this.abuse)) + } + + getTargetUsers () { + return this.moderators + } + + createEmail (to: string) { + return this.createEmailFor(to, 'moderator') + } +} diff --git a/server/server/lib/notifier/shared/abuse/new-abuse-message-for-reporter.ts b/server/server/lib/notifier/shared/abuse/new-abuse-message-for-reporter.ts new file mode 100644 index 000000000..d0ec709fe --- /dev/null +++ b/server/server/lib/notifier/shared/abuse/new-abuse-message-for-reporter.ts @@ -0,0 +1,36 @@ +import { logger } from '@server/helpers/logger.js' +import { getAbuseTargetUrl } from '@server/lib/activitypub/url.js' +import { UserModel } from '@server/models/user/user.js' +import { MUserDefault } from '@server/types/models/index.js' +import { AbstractNewAbuseMessage } from './abstract-new-abuse-message.js' + +export class NewAbuseMessageForReporter extends AbstractNewAbuseMessage { + private reporter: MUserDefault + + async prepare () { + // Only notify our users + if (this.abuse.ReporterAccount.isOwned() !== true) return + + await this.loadMessageAccount() + + const reporter = await UserModel.loadByAccountActorId(this.abuse.ReporterAccount.actorId) + // Don't notify my own message + if (reporter.Account.id === this.message.accountId) return + + this.reporter = reporter + } + + log () { + logger.info('Notifying reporter of new abuse message on %s.', getAbuseTargetUrl(this.abuse)) + } + + getTargetUsers () { + if (!this.reporter) return [] + + return [ this.reporter ] + } + + createEmail (to: string) { + return this.createEmailFor(to, 'reporter') + } +} diff --git a/server/server/lib/notifier/shared/blacklist/index.ts b/server/server/lib/notifier/shared/blacklist/index.ts new file mode 100644 index 000000000..9af153b5a --- /dev/null +++ b/server/server/lib/notifier/shared/blacklist/index.ts @@ -0,0 +1,3 @@ +export * from './new-auto-blacklist-for-moderators.js' +export * from './new-blacklist-for-owner.js' +export * from './unblacklist-for-owner.js' diff --git a/server/server/lib/notifier/shared/blacklist/new-auto-blacklist-for-moderators.ts b/server/server/lib/notifier/shared/blacklist/new-auto-blacklist-for-moderators.ts new file mode 100644 index 000000000..3389bcd67 --- /dev/null +++ b/server/server/lib/notifier/shared/blacklist/new-auto-blacklist-for-moderators.ts @@ -0,0 +1,65 @@ +import { UserNotificationType, UserRight } from '@peertube/peertube-models' +import { logger } from '@server/helpers/logger.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { UserModel } from '@server/models/user/user.js' +import { VideoChannelModel } from '@server/models/video/video-channel.js' +import { + MUserDefault, + MUserWithNotificationSetting, + MVideoBlacklistLightVideo, + UserNotificationModelForApi +} from '@server/types/models/index.js' +import { AbstractNotification } from '../common/abstract-notification.js' + +export class NewAutoBlacklistForModerators extends AbstractNotification { + private moderators: MUserDefault[] + + async prepare () { + this.moderators = await UserModel.listWithRight(UserRight.MANAGE_VIDEO_BLACKLIST) + } + + log () { + logger.info('Notifying %s moderators of video auto-blacklist %s.', this.moderators.length, this.payload.Video.url) + } + + getSetting (user: MUserWithNotificationSetting) { + return user.NotificationSetting.videoAutoBlacklistAsModerator + } + + getTargetUsers () { + return this.moderators + } + + createNotification (user: MUserWithNotificationSetting) { + const notification = UserNotificationModel.build({ + type: UserNotificationType.VIDEO_AUTO_BLACKLIST_FOR_MODERATORS, + userId: user.id, + videoBlacklistId: this.payload.id + }) + notification.VideoBlacklist = this.payload + + return notification + } + + async createEmail (to: string) { + const videoAutoBlacklistUrl = WEBSERVER.URL + '/admin/moderation/video-auto-blacklist/list' + const videoUrl = WEBSERVER.URL + this.payload.Video.getWatchStaticPath() + const channel = await VideoChannelModel.loadAndPopulateAccount(this.payload.Video.channelId) + + return { + template: 'video-auto-blacklist-new', + to, + subject: 'A new video is pending moderation', + locals: { + channel: channel.toFormattedSummaryJSON(), + videoUrl, + videoName: this.payload.Video.name, + action: { + text: 'Review autoblacklist', + url: videoAutoBlacklistUrl + } + } + } + } +} diff --git a/server/server/lib/notifier/shared/blacklist/new-blacklist-for-owner.ts b/server/server/lib/notifier/shared/blacklist/new-blacklist-for-owner.ts new file mode 100644 index 000000000..2ce8b1327 --- /dev/null +++ b/server/server/lib/notifier/shared/blacklist/new-blacklist-for-owner.ts @@ -0,0 +1,63 @@ +import { UserNotificationType } from '@peertube/peertube-models' +import { logger } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { UserModel } from '@server/models/user/user.js' +import { + MUserDefault, + MUserWithNotificationSetting, + MVideoBlacklistVideo, + UserNotificationModelForApi +} from '@server/types/models/index.js' +import { AbstractNotification } from '../common/abstract-notification.js' + +export class NewBlacklistForOwner extends AbstractNotification { + private user: MUserDefault + + async prepare () { + this.user = await UserModel.loadByVideoId(this.payload.videoId) + } + + log () { + logger.info('Notifying user %s that its video %s has been blacklisted.', this.user.username, this.payload.Video.url) + } + + getSetting (user: MUserWithNotificationSetting) { + return user.NotificationSetting.blacklistOnMyVideo + } + + getTargetUsers () { + if (!this.user) return [] + + return [ this.user ] + } + + createNotification (user: MUserWithNotificationSetting) { + const notification = UserNotificationModel.build({ + type: UserNotificationType.BLACKLIST_ON_MY_VIDEO, + userId: user.id, + videoBlacklistId: this.payload.id + }) + notification.VideoBlacklist = this.payload + + return notification + } + + createEmail (to: string) { + const videoName = this.payload.Video.name + const videoUrl = WEBSERVER.URL + this.payload.Video.getWatchStaticPath() + + const reasonString = this.payload.reason ? ` for the following reason: ${this.payload.reason}` : '' + const blockedString = `Your video ${videoName} (${videoUrl} on ${CONFIG.INSTANCE.NAME} has been blacklisted${reasonString}.` + + return { + to, + subject: `Video ${videoName} blacklisted`, + text: blockedString, + locals: { + title: 'Your video was blacklisted' + } + } + } +} diff --git a/server/server/lib/notifier/shared/blacklist/unblacklist-for-owner.ts b/server/server/lib/notifier/shared/blacklist/unblacklist-for-owner.ts new file mode 100644 index 000000000..66a6d1263 --- /dev/null +++ b/server/server/lib/notifier/shared/blacklist/unblacklist-for-owner.ts @@ -0,0 +1,55 @@ +import { logger } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { UserModel } from '@server/models/user/user.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { MUserDefault, MUserWithNotificationSetting, MVideoFullLight, UserNotificationModelForApi } from '@server/types/models/index.js' +import { UserNotificationType } from '@peertube/peertube-models' +import { AbstractNotification } from '../common/abstract-notification.js' + +export class UnblacklistForOwner extends AbstractNotification { + private user: MUserDefault + + async prepare () { + this.user = await UserModel.loadByVideoId(this.payload.id) + } + + log () { + logger.info('Notifying user %s that its video %s has been unblacklisted.', this.user.username, this.payload.url) + } + + getSetting (user: MUserWithNotificationSetting) { + return user.NotificationSetting.blacklistOnMyVideo + } + + getTargetUsers () { + if (!this.user) return [] + + return [ this.user ] + } + + createNotification (user: MUserWithNotificationSetting) { + const notification = UserNotificationModel.build({ + type: UserNotificationType.UNBLACKLIST_ON_MY_VIDEO, + userId: user.id, + videoId: this.payload.id + }) + notification.Video = this.payload + + return notification + } + + createEmail (to: string) { + const video = this.payload + const videoUrl = WEBSERVER.URL + video.getWatchStaticPath() + + return { + to, + subject: `Video ${video.name} unblacklisted`, + text: `Your video "${video.name}" (${videoUrl}) on ${CONFIG.INSTANCE.NAME} has been unblacklisted.`, + locals: { + title: 'Your video was unblacklisted' + } + } + } +} diff --git a/server/server/lib/notifier/shared/comment/comment-mention.ts b/server/server/lib/notifier/shared/comment/comment-mention.ts new file mode 100644 index 000000000..e6bba1eee --- /dev/null +++ b/server/server/lib/notifier/shared/comment/comment-mention.ts @@ -0,0 +1,111 @@ +import { logger } from '@server/helpers/logger.js' +import { toSafeHtml } from '@server/helpers/markdown.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { AccountBlocklistModel } from '@server/models/account/account-blocklist.js' +import { getServerActor } from '@server/models/application/application.js' +import { ServerBlocklistModel } from '@server/models/server/server-blocklist.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { UserModel } from '@server/models/user/user.js' +import { + MCommentOwnerVideo, + MUserDefault, + MUserNotifSettingAccount, + MUserWithNotificationSetting, + UserNotificationModelForApi +} from '@server/types/models/index.js' +import { UserNotificationSettingValue, UserNotificationType } from '@peertube/peertube-models' +import { AbstractNotification } from '../common/index.js' + +export class CommentMention extends AbstractNotification { + private users: MUserDefault[] + + private serverAccountId: number + + private accountMutedHash: { [ id: number ]: boolean } + private instanceMutedHash: { [ id: number ]: boolean } + + async prepare () { + const extractedUsernames = this.payload.extractMentions() + logger.debug( + 'Extracted %d username from comment %s.', extractedUsernames.length, this.payload.url, + { usernames: extractedUsernames, text: this.payload.text } + ) + + this.users = await UserModel.listByUsernames(extractedUsernames) + + if (this.payload.Video.isOwned()) { + const userException = await UserModel.loadByVideoId(this.payload.videoId) + this.users = this.users.filter(u => u.id !== userException.id) + } + + // Don't notify if I mentioned myself + this.users = this.users.filter(u => u.Account.id !== this.payload.accountId) + + if (this.users.length === 0) return + + this.serverAccountId = (await getServerActor()).Account.id + + const sourceAccounts = this.users.map(u => u.Account.id).concat([ this.serverAccountId ]) + + this.accountMutedHash = await AccountBlocklistModel.isAccountMutedByAccounts(sourceAccounts, this.payload.accountId) + this.instanceMutedHash = await ServerBlocklistModel.isServerMutedByAccounts(sourceAccounts, this.payload.Account.Actor.serverId) + } + + log () { + logger.info('Notifying %d users of new comment %s.', this.users.length, this.payload.url) + } + + getSetting (user: MUserNotifSettingAccount) { + const accountId = user.Account.id + if ( + this.accountMutedHash[accountId] === true || this.instanceMutedHash[accountId] === true || + this.accountMutedHash[this.serverAccountId] === true || this.instanceMutedHash[this.serverAccountId] === true + ) { + return UserNotificationSettingValue.NONE + } + + return user.NotificationSetting.commentMention + } + + getTargetUsers () { + return this.users + } + + createNotification (user: MUserWithNotificationSetting) { + const notification = UserNotificationModel.build({ + type: UserNotificationType.COMMENT_MENTION, + userId: user.id, + commentId: this.payload.id + }) + notification.VideoComment = this.payload + + return notification + } + + createEmail (to: string) { + const comment = this.payload + + const accountName = comment.Account.getDisplayName() + const video = comment.Video + const videoUrl = WEBSERVER.URL + comment.Video.getWatchStaticPath() + const commentUrl = WEBSERVER.URL + comment.getCommentStaticPath() + const commentHtml = toSafeHtml(comment.text) + + return { + template: 'video-comment-mention', + to, + subject: 'Mention on video ' + video.name, + locals: { + comment, + commentHtml, + video, + videoUrl, + accountName, + action: { + text: 'View comment', + url: commentUrl + } + } + } + } +} diff --git a/server/server/lib/notifier/shared/comment/index.ts b/server/server/lib/notifier/shared/comment/index.ts new file mode 100644 index 000000000..cbb93ebd3 --- /dev/null +++ b/server/server/lib/notifier/shared/comment/index.ts @@ -0,0 +1,2 @@ +export * from './comment-mention.js' +export * from './new-comment-for-video-owner.js' diff --git a/server/server/lib/notifier/shared/comment/new-comment-for-video-owner.ts b/server/server/lib/notifier/shared/comment/new-comment-for-video-owner.ts new file mode 100644 index 000000000..eddb162f3 --- /dev/null +++ b/server/server/lib/notifier/shared/comment/new-comment-for-video-owner.ts @@ -0,0 +1,76 @@ +import { logger } from '@server/helpers/logger.js' +import { toSafeHtml } from '@server/helpers/markdown.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { isBlockedByServerOrAccount } from '@server/lib/blocklist.js' +import { UserModel } from '@server/models/user/user.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { MCommentOwnerVideo, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models/index.js' +import { UserNotificationType } from '@peertube/peertube-models' +import { AbstractNotification } from '../common/abstract-notification.js' + +export class NewCommentForVideoOwner extends AbstractNotification { + private user: MUserDefault + + async prepare () { + this.user = await UserModel.loadByVideoId(this.payload.videoId) + } + + log () { + logger.info('Notifying owner of a video %s of new comment %s.', this.user.username, this.payload.url) + } + + isDisabled () { + if (this.payload.Video.isOwned() === false) return true + + // Not our user or user comments its own video + if (!this.user || this.payload.Account.userId === this.user.id) return true + + return isBlockedByServerOrAccount(this.payload.Account, this.user.Account) + } + + getSetting (user: MUserWithNotificationSetting) { + return user.NotificationSetting.newCommentOnMyVideo + } + + getTargetUsers () { + if (!this.user) return [] + + return [ this.user ] + } + + createNotification (user: MUserWithNotificationSetting) { + const notification = UserNotificationModel.build({ + type: UserNotificationType.NEW_COMMENT_ON_MY_VIDEO, + userId: user.id, + commentId: this.payload.id + }) + notification.VideoComment = this.payload + + return notification + } + + createEmail (to: string) { + const video = this.payload.Video + const videoUrl = WEBSERVER.URL + this.payload.Video.getWatchStaticPath() + const commentUrl = WEBSERVER.URL + this.payload.getCommentStaticPath() + const commentHtml = toSafeHtml(this.payload.text) + + return { + template: 'video-comment-new', + to, + subject: 'New comment on your video ' + video.name, + locals: { + accountName: this.payload.Account.getDisplayName(), + accountUrl: this.payload.Account.Actor.url, + comment: this.payload, + commentHtml, + video, + videoUrl, + action: { + text: 'View comment', + url: commentUrl + } + } + } + } +} diff --git a/server/server/lib/notifier/shared/common/abstract-notification.ts b/server/server/lib/notifier/shared/common/abstract-notification.ts new file mode 100644 index 000000000..c8b385c6d --- /dev/null +++ b/server/server/lib/notifier/shared/common/abstract-notification.ts @@ -0,0 +1,23 @@ +import { EmailPayload, UserNotificationSettingValueType } from '@peertube/peertube-models' +import { MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models/index.js' + +export abstract class AbstractNotification { + + constructor (protected readonly payload: T) { + + } + + abstract prepare (): Promise + abstract log (): void + + abstract getSetting (user: U): UserNotificationSettingValueType + abstract getTargetUsers (): U[] + + abstract createNotification (user: U): UserNotificationModelForApi + abstract createEmail (to: string): EmailPayload | Promise + + isDisabled (): boolean | Promise { + return false + } + +} diff --git a/server/server/lib/notifier/shared/common/index.ts b/server/server/lib/notifier/shared/common/index.ts new file mode 100644 index 000000000..322a35120 --- /dev/null +++ b/server/server/lib/notifier/shared/common/index.ts @@ -0,0 +1 @@ +export * from './abstract-notification.js' diff --git a/server/server/lib/notifier/shared/follow/auto-follow-for-instance.ts b/server/server/lib/notifier/shared/follow/auto-follow-for-instance.ts new file mode 100644 index 000000000..027bf8eef --- /dev/null +++ b/server/server/lib/notifier/shared/follow/auto-follow-for-instance.ts @@ -0,0 +1,51 @@ +import { logger } from '@server/helpers/logger.js' +import { UserModel } from '@server/models/user/user.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { MActorFollowFull, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models/index.js' +import { UserNotificationType, UserRight } from '@peertube/peertube-models' +import { AbstractNotification } from '../common/abstract-notification.js' + +export class AutoFollowForInstance extends AbstractNotification { + private admins: MUserDefault[] + + async prepare () { + this.admins = await UserModel.listWithRight(UserRight.MANAGE_SERVER_FOLLOW) + } + + log () { + logger.info('Notifying %d administrators of auto instance following: %s.', this.admins.length, this.actorFollow.ActorFollowing.url) + } + + getSetting (user: MUserWithNotificationSetting) { + return user.NotificationSetting.autoInstanceFollowing + } + + getTargetUsers () { + return this.admins + } + + createNotification (user: MUserWithNotificationSetting) { + const notification = UserNotificationModel.build({ + type: UserNotificationType.AUTO_INSTANCE_FOLLOWING, + userId: user.id, + actorFollowId: this.actorFollow.id + }) + notification.ActorFollow = this.actorFollow + + return notification + } + + createEmail (to: string) { + const instanceUrl = this.actorFollow.ActorFollowing.url + + return { + to, + subject: 'Auto instance following', + text: `Your instance automatically followed a new instance: ${instanceUrl}.` + } + } + + private get actorFollow () { + return this.payload + } +} diff --git a/server/server/lib/notifier/shared/follow/follow-for-instance.ts b/server/server/lib/notifier/shared/follow/follow-for-instance.ts new file mode 100644 index 000000000..76fa0332a --- /dev/null +++ b/server/server/lib/notifier/shared/follow/follow-for-instance.ts @@ -0,0 +1,68 @@ +import { logger } from '@server/helpers/logger.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { isBlockedByServerOrAccount } from '@server/lib/blocklist.js' +import { UserModel } from '@server/models/user/user.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { MActorFollowFull, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models/index.js' +import { UserNotificationType, UserRight } from '@peertube/peertube-models' +import { AbstractNotification } from '../common/abstract-notification.js' + +export class FollowForInstance extends AbstractNotification { + private admins: MUserDefault[] + + async prepare () { + this.admins = await UserModel.listWithRight(UserRight.MANAGE_SERVER_FOLLOW) + } + + isDisabled () { + const follower = Object.assign(this.actorFollow.ActorFollower.Account, { Actor: this.actorFollow.ActorFollower }) + + return isBlockedByServerOrAccount(follower) + } + + log () { + logger.info('Notifying %d administrators of new instance follower: %s.', this.admins.length, this.actorFollow.ActorFollower.url) + } + + getSetting (user: MUserWithNotificationSetting) { + return user.NotificationSetting.newInstanceFollower + } + + getTargetUsers () { + return this.admins + } + + createNotification (user: MUserWithNotificationSetting) { + const notification = UserNotificationModel.build({ + type: UserNotificationType.NEW_INSTANCE_FOLLOWER, + userId: user.id, + actorFollowId: this.actorFollow.id + }) + notification.ActorFollow = this.actorFollow + + return notification + } + + createEmail (to: string) { + const awaitingApproval = this.actorFollow.state === 'pending' + ? ' awaiting manual approval.' + : '' + + return { + to, + subject: 'New instance follower', + text: `Your instance has a new follower: ${this.actorFollow.ActorFollower.url}${awaitingApproval}.`, + locals: { + title: 'New instance follower', + action: { + text: 'Review followers', + url: WEBSERVER.URL + '/admin/follows/followers-list' + } + } + } + } + + private get actorFollow () { + return this.payload + } +} diff --git a/server/server/lib/notifier/shared/follow/follow-for-user.ts b/server/server/lib/notifier/shared/follow/follow-for-user.ts new file mode 100644 index 000000000..bf915dfeb --- /dev/null +++ b/server/server/lib/notifier/shared/follow/follow-for-user.ts @@ -0,0 +1,82 @@ +import { logger } from '@server/helpers/logger.js' +import { isBlockedByServerOrAccount } from '@server/lib/blocklist.js' +import { UserModel } from '@server/models/user/user.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { MActorFollowFull, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models/index.js' +import { UserNotificationType } from '@peertube/peertube-models' +import { AbstractNotification } from '../common/abstract-notification.js' + +export class FollowForUser extends AbstractNotification { + private followType: 'account' | 'channel' + private user: MUserDefault + + async prepare () { + // Account follows one of our account? + this.followType = 'channel' + this.user = await UserModel.loadByChannelActorId(this.actorFollow.ActorFollowing.id) + + // Account follows one of our channel? + if (!this.user) { + this.user = await UserModel.loadByAccountActorId(this.actorFollow.ActorFollowing.id) + this.followType = 'account' + } + } + + async isDisabled () { + if (this.payload.ActorFollowing.isOwned() === false) return true + + const followerAccount = this.actorFollow.ActorFollower.Account + const followerAccountWithActor = Object.assign(followerAccount, { Actor: this.actorFollow.ActorFollower }) + + return isBlockedByServerOrAccount(followerAccountWithActor, this.user.Account) + } + + log () { + logger.info('Notifying user %s of new follower: %s.', this.user.username, this.actorFollow.ActorFollower.Account.getDisplayName()) + } + + getSetting (user: MUserWithNotificationSetting) { + return user.NotificationSetting.newFollow + } + + getTargetUsers () { + if (!this.user) return [] + + return [ this.user ] + } + + createNotification (user: MUserWithNotificationSetting) { + const notification = UserNotificationModel.build({ + type: UserNotificationType.NEW_FOLLOW, + userId: user.id, + actorFollowId: this.actorFollow.id + }) + notification.ActorFollow = this.actorFollow + + return notification + } + + createEmail (to: string) { + const following = this.actorFollow.ActorFollowing + const follower = this.actorFollow.ActorFollower + + const followingName = (following.VideoChannel || following.Account).getDisplayName() + + return { + template: 'follower-on-channel', + to, + subject: `New follower on your channel ${followingName}`, + locals: { + followerName: follower.Account.getDisplayName(), + followerUrl: follower.url, + followingName, + followingUrl: following.url, + followType: this.followType + } + } + } + + private get actorFollow () { + return this.payload + } +} diff --git a/server/server/lib/notifier/shared/follow/index.ts b/server/server/lib/notifier/shared/follow/index.ts new file mode 100644 index 000000000..5fe27e4c9 --- /dev/null +++ b/server/server/lib/notifier/shared/follow/index.ts @@ -0,0 +1,3 @@ +export * from './auto-follow-for-instance.js' +export * from './follow-for-instance.js' +export * from './follow-for-user.js' diff --git a/server/server/lib/notifier/shared/index.ts b/server/server/lib/notifier/shared/index.ts new file mode 100644 index 000000000..f4f165c40 --- /dev/null +++ b/server/server/lib/notifier/shared/index.ts @@ -0,0 +1,7 @@ +export * from './abuse/index.js' +export * from './blacklist/index.js' +export * from './comment/index.js' +export * from './common/index.js' +export * from './follow/index.js' +export * from './instance/index.js' +export * from './video-publication/index.js' diff --git a/server/server/lib/notifier/shared/instance/direct-registration-for-moderators.ts b/server/server/lib/notifier/shared/instance/direct-registration-for-moderators.ts new file mode 100644 index 000000000..42078ba24 --- /dev/null +++ b/server/server/lib/notifier/shared/instance/direct-registration-for-moderators.ts @@ -0,0 +1,49 @@ +import { logger } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' +import { UserModel } from '@server/models/user/user.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models/index.js' +import { UserNotificationType, UserRight } from '@peertube/peertube-models' +import { AbstractNotification } from '../common/abstract-notification.js' + +export class DirectRegistrationForModerators extends AbstractNotification { + private moderators: MUserDefault[] + + async prepare () { + this.moderators = await UserModel.listWithRight(UserRight.MANAGE_USERS) + } + + log () { + logger.info('Notifying %s moderators of new user registration of %s.', this.moderators.length, this.payload.username) + } + + getSetting (user: MUserWithNotificationSetting) { + return user.NotificationSetting.newUserRegistration + } + + getTargetUsers () { + return this.moderators + } + + createNotification (user: MUserWithNotificationSetting) { + const notification = UserNotificationModel.build({ + type: UserNotificationType.NEW_USER_REGISTRATION, + userId: user.id, + accountId: this.payload.Account.id + }) + notification.Account = this.payload.Account + + return notification + } + + createEmail (to: string) { + return { + template: 'user-registered', + to, + subject: `A new user registered on ${CONFIG.INSTANCE.NAME}: ${this.payload.username}`, + locals: { + user: this.payload + } + } + } +} diff --git a/server/server/lib/notifier/shared/instance/index.ts b/server/server/lib/notifier/shared/instance/index.ts new file mode 100644 index 000000000..c922fcbe9 --- /dev/null +++ b/server/server/lib/notifier/shared/instance/index.ts @@ -0,0 +1,4 @@ +export * from './new-peertube-version-for-admins.js' +export * from './new-plugin-version-for-admins.js' +export * from './direct-registration-for-moderators.js' +export * from './registration-request-for-moderators.js' diff --git a/server/server/lib/notifier/shared/instance/new-peertube-version-for-admins.ts b/server/server/lib/notifier/shared/instance/new-peertube-version-for-admins.ts new file mode 100644 index 000000000..ff9b248e5 --- /dev/null +++ b/server/server/lib/notifier/shared/instance/new-peertube-version-for-admins.ts @@ -0,0 +1,54 @@ +import { logger } from '@server/helpers/logger.js' +import { UserModel } from '@server/models/user/user.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { MApplication, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models/index.js' +import { UserNotificationType, UserRight } from '@peertube/peertube-models' +import { AbstractNotification } from '../common/abstract-notification.js' + +export type NewPeerTubeVersionForAdminsPayload = { + application: MApplication + latestVersion: string +} + +export class NewPeerTubeVersionForAdmins extends AbstractNotification { + private admins: MUserDefault[] + + async prepare () { + // Use the debug right to know who is an administrator + this.admins = await UserModel.listWithRight(UserRight.MANAGE_DEBUG) + } + + log () { + logger.info('Notifying %s admins of new PeerTube version %s.', this.admins.length, this.payload.latestVersion) + } + + getSetting (user: MUserWithNotificationSetting) { + return user.NotificationSetting.newPeerTubeVersion + } + + getTargetUsers () { + return this.admins + } + + createNotification (user: MUserWithNotificationSetting) { + const notification = UserNotificationModel.build({ + type: UserNotificationType.NEW_PEERTUBE_VERSION, + userId: user.id, + applicationId: this.payload.application.id + }) + notification.Application = this.payload.application + + return notification + } + + createEmail (to: string) { + return { + to, + template: 'peertube-version-new', + subject: `A new PeerTube version is available: ${this.payload.latestVersion}`, + locals: { + latestVersion: this.payload.latestVersion + } + } + } +} diff --git a/server/server/lib/notifier/shared/instance/new-plugin-version-for-admins.ts b/server/server/lib/notifier/shared/instance/new-plugin-version-for-admins.ts new file mode 100644 index 000000000..4a9de9a0b --- /dev/null +++ b/server/server/lib/notifier/shared/instance/new-plugin-version-for-admins.ts @@ -0,0 +1,58 @@ +import { logger } from '@server/helpers/logger.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { UserModel } from '@server/models/user/user.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { MPlugin, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models/index.js' +import { UserNotificationType, UserRight } from '@peertube/peertube-models' +import { AbstractNotification } from '../common/abstract-notification.js' + +export class NewPluginVersionForAdmins extends AbstractNotification { + private admins: MUserDefault[] + + async prepare () { + // Use the debug right to know who is an administrator + this.admins = await UserModel.listWithRight(UserRight.MANAGE_DEBUG) + } + + log () { + logger.info('Notifying %s admins of new PeerTube version %s.', this.admins.length, this.payload.latestVersion) + } + + getSetting (user: MUserWithNotificationSetting) { + return user.NotificationSetting.newPluginVersion + } + + getTargetUsers () { + return this.admins + } + + createNotification (user: MUserWithNotificationSetting) { + const notification = UserNotificationModel.build({ + type: UserNotificationType.NEW_PLUGIN_VERSION, + userId: user.id, + pluginId: this.plugin.id + }) + notification.Plugin = this.plugin + + return notification + } + + createEmail (to: string) { + const pluginUrl = WEBSERVER.URL + '/admin/plugins/list-installed?pluginType=' + this.plugin.type + + return { + to, + template: 'plugin-version-new', + subject: `A new plugin/theme version is available: ${this.plugin.name}@${this.plugin.latestVersion}`, + locals: { + pluginName: this.plugin.name, + latestVersion: this.plugin.latestVersion, + pluginUrl + } + } + } + + private get plugin () { + return this.payload + } +} diff --git a/server/server/lib/notifier/shared/instance/registration-request-for-moderators.ts b/server/server/lib/notifier/shared/instance/registration-request-for-moderators.ts new file mode 100644 index 000000000..29dda28be --- /dev/null +++ b/server/server/lib/notifier/shared/instance/registration-request-for-moderators.ts @@ -0,0 +1,48 @@ +import { logger } from '@server/helpers/logger.js' +import { UserModel } from '@server/models/user/user.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { MRegistration, MUserDefault, MUserWithNotificationSetting, UserNotificationModelForApi } from '@server/types/models/index.js' +import { UserNotificationType, UserRight } from '@peertube/peertube-models' +import { AbstractNotification } from '../common/abstract-notification.js' + +export class RegistrationRequestForModerators extends AbstractNotification { + private moderators: MUserDefault[] + + async prepare () { + this.moderators = await UserModel.listWithRight(UserRight.MANAGE_REGISTRATIONS) + } + + log () { + logger.info('Notifying %s moderators of new user registration request of %s.', this.moderators.length, this.payload.username) + } + + getSetting (user: MUserWithNotificationSetting) { + return user.NotificationSetting.newUserRegistration + } + + getTargetUsers () { + return this.moderators + } + + createNotification (user: MUserWithNotificationSetting) { + const notification = UserNotificationModel.build({ + type: UserNotificationType.NEW_USER_REGISTRATION_REQUEST, + userId: user.id, + userRegistrationId: this.payload.id + }) + notification.UserRegistration = this.payload + + return notification + } + + createEmail (to: string) { + return { + template: 'user-registration-request', + to, + subject: `A new user wants to register: ${this.payload.username}`, + locals: { + registration: this.payload + } + } + } +} diff --git a/server/server/lib/notifier/shared/video-publication/abstract-owned-video-publication.ts b/server/server/lib/notifier/shared/video-publication/abstract-owned-video-publication.ts new file mode 100644 index 000000000..c0bb33620 --- /dev/null +++ b/server/server/lib/notifier/shared/video-publication/abstract-owned-video-publication.ts @@ -0,0 +1,57 @@ +import { logger } from '@server/helpers/logger.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { UserModel } from '@server/models/user/user.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { MUserDefault, MUserWithNotificationSetting, MVideoFullLight, UserNotificationModelForApi } from '@server/types/models/index.js' +import { UserNotificationType } from '@peertube/peertube-models' +import { AbstractNotification } from '../common/abstract-notification.js' + +export abstract class AbstractOwnedVideoPublication extends AbstractNotification { + protected user: MUserDefault + + async prepare () { + this.user = await UserModel.loadByVideoId(this.payload.id) + } + + log () { + logger.info('Notifying user %s of the publication of its video %s.', this.user.username, this.payload.url) + } + + getSetting (user: MUserWithNotificationSetting) { + return user.NotificationSetting.myVideoPublished + } + + getTargetUsers () { + if (!this.user) return [] + + return [ this.user ] + } + + createNotification (user: MUserWithNotificationSetting) { + const notification = UserNotificationModel.build({ + type: UserNotificationType.MY_VIDEO_PUBLISHED, + userId: user.id, + videoId: this.payload.id + }) + notification.Video = this.payload + + return notification + } + + createEmail (to: string) { + const videoUrl = WEBSERVER.URL + this.payload.getWatchStaticPath() + + return { + to, + subject: `Your video ${this.payload.name} has been published`, + text: `Your video "${this.payload.name}" has been published.`, + locals: { + title: 'Your video is live', + action: { + text: 'View video', + url: videoUrl + } + } + } + } +} diff --git a/server/server/lib/notifier/shared/video-publication/import-finished-for-owner.ts b/server/server/lib/notifier/shared/video-publication/import-finished-for-owner.ts new file mode 100644 index 000000000..38f12d8cf --- /dev/null +++ b/server/server/lib/notifier/shared/video-publication/import-finished-for-owner.ts @@ -0,0 +1,97 @@ +import { logger } from '@server/helpers/logger.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { UserModel } from '@server/models/user/user.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { MUserDefault, MUserWithNotificationSetting, MVideoImportVideo, UserNotificationModelForApi } from '@server/types/models/index.js' +import { UserNotificationType } from '@peertube/peertube-models' +import { AbstractNotification } from '../common/abstract-notification.js' + +export type ImportFinishedForOwnerPayload = { + videoImport: MVideoImportVideo + success: boolean +} + +export class ImportFinishedForOwner extends AbstractNotification { + private user: MUserDefault + + async prepare () { + this.user = await UserModel.loadByVideoImportId(this.videoImport.id) + } + + log () { + logger.info('Notifying user %s its video import %s is finished.', this.user.username, this.videoImport.getTargetIdentifier()) + } + + getSetting (user: MUserWithNotificationSetting) { + return user.NotificationSetting.myVideoImportFinished + } + + getTargetUsers () { + if (!this.user) return [] + + return [ this.user ] + } + + createNotification (user: MUserWithNotificationSetting) { + const notification = UserNotificationModel.build({ + type: this.payload.success + ? UserNotificationType.MY_VIDEO_IMPORT_SUCCESS + : UserNotificationType.MY_VIDEO_IMPORT_ERROR, + + userId: user.id, + videoImportId: this.videoImport.id + }) + notification.VideoImport = this.videoImport + + return notification + } + + createEmail (to: string) { + if (this.payload.success) return this.createSuccessEmail(to) + + return this.createFailEmail(to) + } + + private createSuccessEmail (to: string) { + const videoUrl = WEBSERVER.URL + this.videoImport.Video.getWatchStaticPath() + + return { + to, + subject: `Your video import ${this.videoImport.getTargetIdentifier()} is complete`, + text: `Your video "${this.videoImport.getTargetIdentifier()}" just finished importing.`, + locals: { + title: 'Import complete', + action: { + text: 'View video', + url: videoUrl + } + } + } + } + + private createFailEmail (to: string) { + const importUrl = WEBSERVER.URL + '/my-library/video-imports' + + const text = + `Your video import "${this.videoImport.getTargetIdentifier()}" encountered an error.` + + '\n\n' + + `See your videos import dashboard for more information: ${importUrl}.` + + return { + to, + subject: `Your video import "${this.videoImport.getTargetIdentifier()}" encountered an error`, + text, + locals: { + title: 'Import failed', + action: { + text: 'Review imports', + url: importUrl + } + } + } + } + + private get videoImport () { + return this.payload.videoImport + } +} diff --git a/server/server/lib/notifier/shared/video-publication/index.ts b/server/server/lib/notifier/shared/video-publication/index.ts new file mode 100644 index 000000000..fc36a9899 --- /dev/null +++ b/server/server/lib/notifier/shared/video-publication/index.ts @@ -0,0 +1,6 @@ +export * from './new-video-for-subscribers.js' +export * from './import-finished-for-owner.js' +export * from './owned-publication-after-auto-unblacklist.js' +export * from './owned-publication-after-schedule-update.js' +export * from './owned-publication-after-transcoding.js' +export * from './studio-edition-finished-for-owner.js' diff --git a/server/server/lib/notifier/shared/video-publication/new-video-for-subscribers.ts b/server/server/lib/notifier/shared/video-publication/new-video-for-subscribers.ts new file mode 100644 index 000000000..df2d89fc9 --- /dev/null +++ b/server/server/lib/notifier/shared/video-publication/new-video-for-subscribers.ts @@ -0,0 +1,61 @@ +import { logger } from '@server/helpers/logger.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { UserModel } from '@server/models/user/user.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { MUserWithNotificationSetting, MVideoAccountLight, UserNotificationModelForApi } from '@server/types/models/index.js' +import { UserNotificationType, VideoPrivacy, VideoState } from '@peertube/peertube-models' +import { AbstractNotification } from '../common/abstract-notification.js' + +export class NewVideoForSubscribers extends AbstractNotification { + private users: MUserWithNotificationSetting[] + + async prepare () { + // List all followers that are users + this.users = await UserModel.listUserSubscribersOf(this.payload.VideoChannel.actorId) + } + + log () { + logger.info('Notifying %d users of new video %s.', this.users.length, this.payload.url) + } + + isDisabled () { + return this.payload.privacy !== VideoPrivacy.PUBLIC || this.payload.state !== VideoState.PUBLISHED || this.payload.isBlacklisted() + } + + getSetting (user: MUserWithNotificationSetting) { + return user.NotificationSetting.newVideoFromSubscription + } + + getTargetUsers () { + return this.users + } + + createNotification (user: MUserWithNotificationSetting) { + const notification = UserNotificationModel.build({ + type: UserNotificationType.NEW_VIDEO_FROM_SUBSCRIPTION, + userId: user.id, + videoId: this.payload.id + }) + notification.Video = this.payload + + return notification + } + + createEmail (to: string) { + const channelName = this.payload.VideoChannel.getDisplayName() + const videoUrl = WEBSERVER.URL + this.payload.getWatchStaticPath() + + return { + to, + subject: channelName + ' just published a new video', + text: `Your subscription ${channelName} just published a new video: "${this.payload.name}".`, + locals: { + title: 'New content ', + action: { + text: 'View video', + url: videoUrl + } + } + } + } +} diff --git a/server/server/lib/notifier/shared/video-publication/owned-publication-after-auto-unblacklist.ts b/server/server/lib/notifier/shared/video-publication/owned-publication-after-auto-unblacklist.ts new file mode 100644 index 000000000..3dbb6e05f --- /dev/null +++ b/server/server/lib/notifier/shared/video-publication/owned-publication-after-auto-unblacklist.ts @@ -0,0 +1,11 @@ + +import { VideoState } from '@peertube/peertube-models' +import { AbstractOwnedVideoPublication } from './abstract-owned-video-publication.js' + +export class OwnedPublicationAfterAutoUnblacklist extends AbstractOwnedVideoPublication { + + isDisabled () { + // Don't notify if video is still waiting for transcoding or scheduled update + return !!this.payload.ScheduleVideoUpdate || (this.payload.waitTranscoding && this.payload.state !== VideoState.PUBLISHED) + } +} diff --git a/server/server/lib/notifier/shared/video-publication/owned-publication-after-schedule-update.ts b/server/server/lib/notifier/shared/video-publication/owned-publication-after-schedule-update.ts new file mode 100644 index 000000000..d8f6e848a --- /dev/null +++ b/server/server/lib/notifier/shared/video-publication/owned-publication-after-schedule-update.ts @@ -0,0 +1,10 @@ +import { VideoState } from '@peertube/peertube-models' +import { AbstractOwnedVideoPublication } from './abstract-owned-video-publication.js' + +export class OwnedPublicationAfterScheduleUpdate extends AbstractOwnedVideoPublication { + + isDisabled () { + // Don't notify if video is still blacklisted or waiting for transcoding + return !!this.payload.VideoBlacklist || (this.payload.waitTranscoding && this.payload.state !== VideoState.PUBLISHED) + } +} diff --git a/server/server/lib/notifier/shared/video-publication/owned-publication-after-transcoding.ts b/server/server/lib/notifier/shared/video-publication/owned-publication-after-transcoding.ts new file mode 100644 index 000000000..56b4aaaa9 --- /dev/null +++ b/server/server/lib/notifier/shared/video-publication/owned-publication-after-transcoding.ts @@ -0,0 +1,9 @@ +import { AbstractOwnedVideoPublication } from './abstract-owned-video-publication.js' + +export class OwnedPublicationAfterTranscoding extends AbstractOwnedVideoPublication { + + isDisabled () { + // Don't notify if didn't wait for transcoding or video is still blacklisted/waiting for scheduled update + return !this.payload.waitTranscoding || !!this.payload.VideoBlacklist || !!this.payload.ScheduleVideoUpdate + } +} diff --git a/server/server/lib/notifier/shared/video-publication/studio-edition-finished-for-owner.ts b/server/server/lib/notifier/shared/video-publication/studio-edition-finished-for-owner.ts new file mode 100644 index 000000000..a4d72567c --- /dev/null +++ b/server/server/lib/notifier/shared/video-publication/studio-edition-finished-for-owner.ts @@ -0,0 +1,57 @@ +import { logger } from '@server/helpers/logger.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { UserModel } from '@server/models/user/user.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { MUserDefault, MUserWithNotificationSetting, MVideoFullLight, UserNotificationModelForApi } from '@server/types/models/index.js' +import { UserNotificationType } from '@peertube/peertube-models' +import { AbstractNotification } from '../common/abstract-notification.js' + +export class StudioEditionFinishedForOwner extends AbstractNotification { + private user: MUserDefault + + async prepare () { + this.user = await UserModel.loadByVideoId(this.payload.id) + } + + log () { + logger.info('Notifying user %s its video studio edition %s is finished.', this.user.username, this.payload.url) + } + + getSetting (user: MUserWithNotificationSetting) { + return user.NotificationSetting.myVideoStudioEditionFinished + } + + getTargetUsers () { + if (!this.user) return [] + + return [ this.user ] + } + + createNotification (user: MUserWithNotificationSetting) { + const notification = UserNotificationModel.build({ + type: UserNotificationType.MY_VIDEO_STUDIO_EDITION_FINISHED, + userId: user.id, + videoId: this.payload.id + }) + notification.Video = this.payload + + return notification + } + + createEmail (to: string) { + const videoUrl = WEBSERVER.URL + this.payload.getWatchStaticPath() + + return { + to, + subject: `Edition of your video ${this.payload.name} has finished`, + text: `Edition of your video ${this.payload.name} has finished.`, + locals: { + title: 'Video edition has finished', + action: { + text: 'View video', + url: videoUrl + } + } + } + } +} diff --git a/server/server/lib/object-storage/index.ts b/server/server/lib/object-storage/index.ts new file mode 100644 index 000000000..238a652a6 --- /dev/null +++ b/server/server/lib/object-storage/index.ts @@ -0,0 +1,5 @@ +export * from './keys.js' +export * from './proxy.js' +export * from './pre-signed-urls.js' +export * from './urls.js' +export * from './videos.js' diff --git a/server/server/lib/object-storage/keys.ts b/server/server/lib/object-storage/keys.ts new file mode 100644 index 000000000..9e52a7f19 --- /dev/null +++ b/server/server/lib/object-storage/keys.ts @@ -0,0 +1,20 @@ +import { join } from 'path' +import { MStreamingPlaylistVideo } from '@server/types/models/index.js' + +function generateHLSObjectStorageKey (playlist: MStreamingPlaylistVideo, filename: string) { + return join(generateHLSObjectBaseStorageKey(playlist), filename) +} + +function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylistVideo) { + return join(playlist.getStringType(), playlist.Video.uuid) +} + +function generateWebVideoObjectStorageKey (filename: string) { + return filename +} + +export { + generateHLSObjectStorageKey, + generateHLSObjectBaseStorageKey, + generateWebVideoObjectStorageKey +} diff --git a/server/server/lib/object-storage/pre-signed-urls.ts b/server/server/lib/object-storage/pre-signed-urls.ts new file mode 100644 index 000000000..bbb19a57c --- /dev/null +++ b/server/server/lib/object-storage/pre-signed-urls.ts @@ -0,0 +1,50 @@ +import { CONFIG } from '@server/initializers/config.js' +import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models/index.js' +import { generateHLSObjectStorageKey, generateWebVideoObjectStorageKey } from './keys.js' +import { buildKey, getClient } from './shared/index.js' +import { getHLSPublicFileUrl, getWebVideoPublicFileUrl } from './urls.js' + +export async function generateWebVideoPresignedUrl (options: { + file: MVideoFile + downloadFilename: string +}) { + const { file, downloadFilename } = options + + const key = generateWebVideoObjectStorageKey(file.filename) + + const { GetObjectCommand } = await import('@aws-sdk/client-s3') + const { getSignedUrl } = await import('@aws-sdk/s3-request-presigner') + + const command = new GetObjectCommand({ + Bucket: CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BUCKET_NAME, + Key: buildKey(key, CONFIG.OBJECT_STORAGE.WEB_VIDEOS), + ResponseContentDisposition: `attachment; filename=${downloadFilename}` + }) + + const url = await getSignedUrl(await getClient(), command, { expiresIn: 3600 * 24 }) + + return getWebVideoPublicFileUrl(url) +} + +export async function generateHLSFilePresignedUrl (options: { + streamingPlaylist: MStreamingPlaylistVideo + file: MVideoFile + downloadFilename: string +}) { + const { streamingPlaylist, file, downloadFilename } = options + + const key = generateHLSObjectStorageKey(streamingPlaylist, file.filename) + + const { GetObjectCommand } = await import('@aws-sdk/client-s3') + const { getSignedUrl } = await import('@aws-sdk/s3-request-presigner') + + const command = new GetObjectCommand({ + Bucket: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME, + Key: buildKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS), + ResponseContentDisposition: `attachment; filename=${downloadFilename}` + }) + + const url = await getSignedUrl(await getClient(), command, { expiresIn: 3600 * 24 }) + + return getHLSPublicFileUrl(url) +} diff --git a/server/server/lib/object-storage/proxy.ts b/server/server/lib/object-storage/proxy.ts new file mode 100644 index 000000000..bef899fed --- /dev/null +++ b/server/server/lib/object-storage/proxy.ts @@ -0,0 +1,98 @@ +import express from 'express' +import { PassThrough, pipeline } from 'stream' +import { HttpStatusCode } from '@peertube/peertube-models' +import { buildReinjectVideoFileTokenQuery } from '@server/controllers/shared/m3u8-playlist.js' +import { logger } from '@server/helpers/logger.js' +import { StreamReplacer } from '@server/helpers/stream-replacer.js' +import { MStreamingPlaylist, MVideo } from '@server/types/models/index.js' +import { injectQueryToPlaylistUrls } from '../hls.js' +import { getHLSFileReadStream, getWebVideoFileReadStream } from './videos.js' + +import type { GetObjectCommandOutput } from '@aws-sdk/client-s3' + +export async function proxifyWebVideoFile (options: { + req: express.Request + res: express.Response + filename: string +}) { + const { req, res, filename } = options + + logger.debug('Proxifying Web Video file %s from object storage.', filename) + + try { + const { response: s3Response, stream } = await getWebVideoFileReadStream({ + filename, + rangeHeader: req.header('range') + }) + + setS3Headers(res, s3Response) + + return stream.pipe(res) + } catch (err) { + return handleObjectStorageFailure(res, err) + } +} + +export async function proxifyHLS (options: { + req: express.Request + res: express.Response + playlist: MStreamingPlaylist + video: MVideo + filename: string + reinjectVideoFileToken: boolean +}) { + const { req, res, playlist, video, filename, reinjectVideoFileToken } = options + + logger.debug('Proxifying HLS file %s from object storage.', filename) + + try { + const { response: s3Response, stream } = await getHLSFileReadStream({ + playlist: playlist.withVideo(video), + filename, + rangeHeader: req.header('range') + }) + + setS3Headers(res, s3Response) + + const streamReplacer = reinjectVideoFileToken + ? new StreamReplacer(line => injectQueryToPlaylistUrls(line, buildReinjectVideoFileTokenQuery(req, filename.endsWith('master.m3u8')))) + : new PassThrough() + + return pipeline( + stream, + streamReplacer, + res, + err => { + if (!err) return + + handleObjectStorageFailure(res, err) + } + ) + } catch (err) { + return handleObjectStorageFailure(res, err) + } +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +function handleObjectStorageFailure (res: express.Response, err: Error) { + if (err.name === 'NoSuchKey') { + logger.debug('Could not find key in object storage to proxify private HLS video file.', { err }) + return res.sendStatus(HttpStatusCode.NOT_FOUND_404) + } + + return res.fail({ + status: HttpStatusCode.INTERNAL_SERVER_ERROR_500, + message: err.message, + type: err.name + }) +} + +function setS3Headers (res: express.Response, s3Response: GetObjectCommandOutput) { + if (s3Response.$metadata.httpStatusCode === HttpStatusCode.PARTIAL_CONTENT_206) { + res.setHeader('Content-Range', s3Response.ContentRange) + res.status(HttpStatusCode.PARTIAL_CONTENT_206) + } +} diff --git a/server/server/lib/object-storage/shared/client.ts b/server/server/lib/object-storage/shared/client.ts new file mode 100644 index 000000000..085bdbc4c --- /dev/null +++ b/server/server/lib/object-storage/shared/client.ts @@ -0,0 +1,78 @@ +import type { S3Client } from '@aws-sdk/client-s3' +import { logger } from '@server/helpers/logger.js' +import { isProxyEnabled } from '@server/helpers/proxy.js' +import { getAgent } from '@server/helpers/requests.js' +import { CONFIG } from '@server/initializers/config.js' +import { lTags } from './logger.js' + +async function getProxyRequestHandler () { + if (!isProxyEnabled()) return null + + const { agent } = getAgent() + + const { NodeHttpHandler } = await import('@aws-sdk/node-http-handler') + + return new NodeHttpHandler({ + httpAgent: agent.http, + httpsAgent: agent.https + }) +} + +let endpointParsed: URL +function getEndpointParsed () { + if (endpointParsed) return endpointParsed + + endpointParsed = new URL(getEndpoint()) + + return endpointParsed +} + +let s3ClientPromise: Promise +function getClient () { + if (s3ClientPromise) return s3ClientPromise + + s3ClientPromise = (async () => { + const OBJECT_STORAGE = CONFIG.OBJECT_STORAGE + + const { S3Client } = await import('@aws-sdk/client-s3') + + const s3Client = new S3Client({ + endpoint: getEndpoint(), + region: OBJECT_STORAGE.REGION, + credentials: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID + ? { + accessKeyId: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID, + secretAccessKey: OBJECT_STORAGE.CREDENTIALS.SECRET_ACCESS_KEY + } + : undefined, + requestHandler: await getProxyRequestHandler() + }) + + logger.info('Initialized S3 client %s with region %s.', getEndpoint(), OBJECT_STORAGE.REGION, lTags()) + + return s3Client + })() + + return s3ClientPromise +} + +// --------------------------------------------------------------------------- + +export { + getEndpointParsed, + getClient +} + +// --------------------------------------------------------------------------- + +let endpoint: string +function getEndpoint () { + if (endpoint) return endpoint + + const endpointConfig = CONFIG.OBJECT_STORAGE.ENDPOINT + endpoint = endpointConfig.startsWith('http://') || endpointConfig.startsWith('https://') + ? CONFIG.OBJECT_STORAGE.ENDPOINT + : 'https://' + CONFIG.OBJECT_STORAGE.ENDPOINT + + return endpoint +} diff --git a/server/server/lib/object-storage/shared/index.ts b/server/server/lib/object-storage/shared/index.ts new file mode 100644 index 000000000..d6b76a3d5 --- /dev/null +++ b/server/server/lib/object-storage/shared/index.ts @@ -0,0 +1,3 @@ +export * from './client.js' +export * from './logger.js' +export * from './object-storage-helpers.js' diff --git a/server/server/lib/object-storage/shared/logger.ts b/server/server/lib/object-storage/shared/logger.ts new file mode 100644 index 000000000..8b8e19f68 --- /dev/null +++ b/server/server/lib/object-storage/shared/logger.ts @@ -0,0 +1,7 @@ +import { loggerTagsFactory } from '@server/helpers/logger.js' + +const lTags = loggerTagsFactory('object-storage') + +export { + lTags +} diff --git a/server/server/lib/object-storage/shared/object-storage-helpers.ts b/server/server/lib/object-storage/shared/object-storage-helpers.ts new file mode 100644 index 000000000..ce96696e5 --- /dev/null +++ b/server/server/lib/object-storage/shared/object-storage-helpers.ts @@ -0,0 +1,345 @@ +import { pipelinePromise } from '@server/helpers/core-utils.js' +import { isArray } from '@server/helpers/custom-validators/misc.js' +import { logger } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' +import Bluebird from 'bluebird' +import { createReadStream, createWriteStream, ReadStream } from 'fs' +import { ensureDir } from 'fs-extra/esm' +import { dirname } from 'path' +import { Readable } from 'stream' +import { getInternalUrl } from '../urls.js' +import { getClient } from './client.js' +import { lTags } from './logger.js' + +import type { _Object, CompleteMultipartUploadCommandOutput, PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3' + +type BucketInfo = { + BUCKET_NAME: string + PREFIX?: string +} + +async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) { + const s3Client = await getClient() + + const { ListObjectsV2Command } = await import('@aws-sdk/client-s3') + + const commandPrefix = bucketInfo.PREFIX + prefix + const listCommand = new ListObjectsV2Command({ + Bucket: bucketInfo.BUCKET_NAME, + Prefix: commandPrefix + }) + + const listedObjects = await s3Client.send(listCommand) + + if (isArray(listedObjects.Contents) !== true) return [] + + return listedObjects.Contents.map(c => c.Key) +} + +// --------------------------------------------------------------------------- + +async function storeObject (options: { + inputPath: string + objectStorageKey: string + bucketInfo: BucketInfo + isPrivate: boolean +}): Promise { + const { inputPath, objectStorageKey, bucketInfo, isPrivate } = options + + logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) + + const fileStream = createReadStream(inputPath) + + return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo, isPrivate }) +} + +async function storeContent (options: { + content: string + inputPath: string + objectStorageKey: string + bucketInfo: BucketInfo + isPrivate: boolean +}): Promise { + const { content, objectStorageKey, bucketInfo, inputPath, isPrivate } = options + + logger.debug('Uploading %s content to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) + + return uploadToStorage({ objectStorageKey, content, bucketInfo, isPrivate }) +} + +// --------------------------------------------------------------------------- + +async function updateObjectACL (options: { + objectStorageKey: string + bucketInfo: BucketInfo + isPrivate: boolean +}) { + const { objectStorageKey, bucketInfo, isPrivate } = options + + const acl = getACL(isPrivate) + if (!acl) return + + const key = buildKey(objectStorageKey, bucketInfo) + + logger.debug('Updating ACL file %s in bucket %s', key, bucketInfo.BUCKET_NAME, lTags()) + + const { PutObjectAclCommand } = await import('@aws-sdk/client-s3') + + const command = new PutObjectAclCommand({ + Bucket: bucketInfo.BUCKET_NAME, + Key: key, + ACL: acl + }) + + const client = await getClient() + await client.send(command) +} + +async function updatePrefixACL (options: { + prefix: string + bucketInfo: BucketInfo + isPrivate: boolean +}) { + const { prefix, bucketInfo, isPrivate } = options + + const acl = getACL(isPrivate) + if (!acl) return + + const { PutObjectAclCommand } = await import('@aws-sdk/client-s3') + + logger.debug('Updating ACL of files in prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags()) + + return applyOnPrefix({ + prefix, + bucketInfo, + commandBuilder: obj => { + logger.debug('Updating ACL of %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags()) + + return new PutObjectAclCommand({ + Bucket: bucketInfo.BUCKET_NAME, + Key: obj.Key, + ACL: acl + }) + } + }) +} + +// --------------------------------------------------------------------------- + +function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) { + const key = buildKey(objectStorageKey, bucketInfo) + + return removeObjectByFullKey(key, bucketInfo) +} + +async function removeObjectByFullKey (fullKey: string, bucketInfo: BucketInfo) { + logger.debug('Removing file %s in bucket %s', fullKey, bucketInfo.BUCKET_NAME, lTags()) + + const { DeleteObjectCommand } = await import('@aws-sdk/client-s3') + + const command = new DeleteObjectCommand({ + Bucket: bucketInfo.BUCKET_NAME, + Key: fullKey + }) + + const client = await getClient() + + return client.send(command) +} + +async function removePrefix (prefix: string, bucketInfo: BucketInfo) { + logger.debug('Removing prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags()) + + const { DeleteObjectCommand } = await import('@aws-sdk/client-s3') + + return applyOnPrefix({ + prefix, + bucketInfo, + commandBuilder: obj => { + logger.debug('Removing %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags()) + + return new DeleteObjectCommand({ + Bucket: bucketInfo.BUCKET_NAME, + Key: obj.Key + }) + } + }) +} + +// --------------------------------------------------------------------------- + +async function makeAvailable (options: { + key: string + destination: string + bucketInfo: BucketInfo +}) { + const { key, destination, bucketInfo } = options + + await ensureDir(dirname(options.destination)) + + const { GetObjectCommand } = await import('@aws-sdk/client-s3') + + const command = new GetObjectCommand({ + Bucket: bucketInfo.BUCKET_NAME, + Key: buildKey(key, bucketInfo) + }) + + const client = await getClient() + const response = await client.send(command) + + const file = createWriteStream(destination) + await pipelinePromise(response.Body as Readable, file) + + file.close() +} + +function buildKey (key: string, bucketInfo: BucketInfo) { + return bucketInfo.PREFIX + key +} + +// --------------------------------------------------------------------------- + +async function createObjectReadStream (options: { + key: string + bucketInfo: BucketInfo + rangeHeader: string +}) { + const { key, bucketInfo, rangeHeader } = options + + const { GetObjectCommand } = await import('@aws-sdk/client-s3') + + const command = new GetObjectCommand({ + Bucket: bucketInfo.BUCKET_NAME, + Key: buildKey(key, bucketInfo), + Range: rangeHeader + }) + + const client = await getClient() + const response = await client.send(command) + + return { + response, + stream: response.Body as Readable + } +} + +// --------------------------------------------------------------------------- + +export { + type BucketInfo, + + buildKey, + + storeObject, + storeContent, + + removeObject, + removeObjectByFullKey, + removePrefix, + + makeAvailable, + + updateObjectACL, + updatePrefixACL, + + listKeysOfPrefix, + createObjectReadStream +} + +// --------------------------------------------------------------------------- + +async function uploadToStorage (options: { + content: ReadStream | string + objectStorageKey: string + bucketInfo: BucketInfo + isPrivate: boolean +}) { + const { content, objectStorageKey, bucketInfo, isPrivate } = options + + const input: PutObjectCommandInput = { + Body: content, + Bucket: bucketInfo.BUCKET_NAME, + Key: buildKey(objectStorageKey, bucketInfo) + } + + const acl = getACL(isPrivate) + if (acl) input.ACL = acl + + const { Upload } = await import('@aws-sdk/lib-storage') + + const parallelUploads3 = new Upload({ + client: await getClient(), + queueSize: 4, + partSize: CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART, + + // `leavePartsOnError` must be set to `true` to avoid silently dropping failed parts + // More detailed explanation: + // https://github.com/aws/aws-sdk-js-v3/blob/v3.164.0/lib/lib-storage/src/Upload.ts#L274 + // https://github.com/aws/aws-sdk-js-v3/issues/2311#issuecomment-939413928 + leavePartsOnError: true, + params: input + }) + + const response = (await parallelUploads3.done()) as CompleteMultipartUploadCommandOutput + // Check is needed even if the HTTP status code is 200 OK + // For more information, see https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html + if (!response.Bucket) { + const message = `Error uploading ${objectStorageKey} to bucket ${bucketInfo.BUCKET_NAME}` + logger.error(message, { response, ...lTags() }) + throw new Error(message) + } + + logger.debug( + 'Completed %s%s in bucket %s', + bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, { ...lTags(), reseponseMetadata: response.$metadata } + ) + + return getInternalUrl(bucketInfo, objectStorageKey) +} + +async function applyOnPrefix (options: { + prefix: string + bucketInfo: BucketInfo + commandBuilder: (obj: _Object) => Parameters[0] + + continuationToken?: string +}) { + const { prefix, bucketInfo, commandBuilder, continuationToken } = options + + const s3Client = await getClient() + + const { ListObjectsV2Command } = await import('@aws-sdk/client-s3') + + const commandPrefix = buildKey(prefix, bucketInfo) + const listCommand = new ListObjectsV2Command({ + Bucket: bucketInfo.BUCKET_NAME, + Prefix: commandPrefix, + ContinuationToken: continuationToken + }) + + const listedObjects = await s3Client.send(listCommand) + + if (isArray(listedObjects.Contents) !== true) { + const message = `Cannot apply function on ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.` + + logger.error(message, { response: listedObjects, ...lTags() }) + throw new Error(message) + } + + await Bluebird.map(listedObjects.Contents, object => { + const command = commandBuilder(object) + + return s3Client.send(command) + }, { concurrency: 10 }) + + // Repeat if not all objects could be listed at once (limit of 1000?) + if (listedObjects.IsTruncated) { + await applyOnPrefix({ ...options, continuationToken: listedObjects.ContinuationToken }) + } +} + +function getACL (isPrivate: boolean) { + return isPrivate + ? CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PRIVATE + : CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PUBLIC +} diff --git a/server/server/lib/object-storage/urls.ts b/server/server/lib/object-storage/urls.ts new file mode 100644 index 000000000..0bd469ea8 --- /dev/null +++ b/server/server/lib/object-storage/urls.ts @@ -0,0 +1,63 @@ +import { CONFIG } from '@server/initializers/config.js' +import { OBJECT_STORAGE_PROXY_PATHS, WEBSERVER } from '@server/initializers/constants.js' +import { MVideoUUID } from '@server/types/models/index.js' +import { BucketInfo, buildKey, getEndpointParsed } from './shared/index.js' + +function getInternalUrl (config: BucketInfo, keyWithoutPrefix: string) { + return getBaseUrl(config) + buildKey(keyWithoutPrefix, config) +} + +// --------------------------------------------------------------------------- + +function getWebVideoPublicFileUrl (fileUrl: string) { + const baseUrl = CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BASE_URL + if (!baseUrl) return fileUrl + + return replaceByBaseUrl(fileUrl, baseUrl) +} + +function getHLSPublicFileUrl (fileUrl: string) { + const baseUrl = CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BASE_URL + if (!baseUrl) return fileUrl + + return replaceByBaseUrl(fileUrl, baseUrl) +} + +// --------------------------------------------------------------------------- + +function getHLSPrivateFileUrl (video: MVideoUUID, filename: string) { + return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + video.uuid + `/${filename}` +} + +function getWebVideoPrivateFileUrl (filename: string) { + return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEB_VIDEOS + filename +} + +// --------------------------------------------------------------------------- + +export { + getInternalUrl, + + getWebVideoPublicFileUrl, + getHLSPublicFileUrl, + + getHLSPrivateFileUrl, + getWebVideoPrivateFileUrl, + + replaceByBaseUrl +} + +// --------------------------------------------------------------------------- + +function getBaseUrl (bucketInfo: BucketInfo, baseUrl?: string) { + if (baseUrl) return baseUrl + + return `${getEndpointParsed().protocol}//${bucketInfo.BUCKET_NAME}.${getEndpointParsed().host}/` +} + +const regex = new RegExp('https?://[^/]+') +function replaceByBaseUrl (fileUrl: string, baseUrl: string) { + if (!fileUrl) return fileUrl + + return fileUrl.replace(regex, baseUrl) +} diff --git a/server/server/lib/object-storage/videos.ts b/server/server/lib/object-storage/videos.ts new file mode 100644 index 000000000..46d32b566 --- /dev/null +++ b/server/server/lib/object-storage/videos.ts @@ -0,0 +1,197 @@ +import { basename, join } from 'path' +import { logger } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' +import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/models/index.js' +import { getHLSDirectory } from '../paths.js' +import { VideoPathManager } from '../video-path-manager.js' +import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebVideoObjectStorageKey } from './keys.js' +import { + createObjectReadStream, + listKeysOfPrefix, + lTags, + makeAvailable, + removeObject, + removeObjectByFullKey, + removePrefix, + storeContent, + storeObject, + updateObjectACL, + updatePrefixACL +} from './shared/index.js' + +function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) { + return listKeysOfPrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) +} + +// --------------------------------------------------------------------------- + +function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename: string) { + return storeObject({ + inputPath: join(getHLSDirectory(playlist.Video), filename), + objectStorageKey: generateHLSObjectStorageKey(playlist, filename), + bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, + isPrivate: playlist.Video.hasPrivateStaticPath() + }) +} + +function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string) { + return storeObject({ + inputPath: path, + objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)), + bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, + isPrivate: playlist.Video.hasPrivateStaticPath() + }) +} + +function storeHLSFileFromContent (playlist: MStreamingPlaylistVideo, path: string, content: string) { + return storeContent({ + content, + inputPath: path, + objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)), + bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, + isPrivate: playlist.Video.hasPrivateStaticPath() + }) +} + +// --------------------------------------------------------------------------- + +function storeWebVideoFile (video: MVideo, file: MVideoFile) { + return storeObject({ + inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file), + objectStorageKey: generateWebVideoObjectStorageKey(file.filename), + bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS, + isPrivate: video.hasPrivateStaticPath() + }) +} + +// --------------------------------------------------------------------------- + +async function updateWebVideoFileACL (video: MVideo, file: MVideoFile) { + await updateObjectACL({ + objectStorageKey: generateWebVideoObjectStorageKey(file.filename), + bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS, + isPrivate: video.hasPrivateStaticPath() + }) +} + +async function updateHLSFilesACL (playlist: MStreamingPlaylistVideo) { + await updatePrefixACL({ + prefix: generateHLSObjectBaseStorageKey(playlist), + bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, + isPrivate: playlist.Video.hasPrivateStaticPath() + }) +} + +// --------------------------------------------------------------------------- + +function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) { + return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) +} + +function removeHLSFileObjectStorageByFilename (playlist: MStreamingPlaylistVideo, filename: string) { + return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) +} + +function removeHLSFileObjectStorageByPath (playlist: MStreamingPlaylistVideo, path: string) { + return removeObject(generateHLSObjectStorageKey(playlist, basename(path)), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) +} + +function removeHLSFileObjectStorageByFullKey (key: string) { + return removeObjectByFullKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) +} + +// --------------------------------------------------------------------------- + +function removeWebVideoObjectStorage (videoFile: MVideoFile) { + return removeObject(generateWebVideoObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.WEB_VIDEOS) +} + +// --------------------------------------------------------------------------- + +async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) { + const key = generateHLSObjectStorageKey(playlist, filename) + + logger.info('Fetching HLS file %s from object storage to %s.', key, destination, lTags()) + + await makeAvailable({ + key, + destination, + bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS + }) + + return destination +} + +async function makeWebVideoFileAvailable (filename: string, destination: string) { + const key = generateWebVideoObjectStorageKey(filename) + + logger.info('Fetching Web Video file %s from object storage to %s.', key, destination, lTags()) + + await makeAvailable({ + key, + destination, + bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS + }) + + return destination +} + +// --------------------------------------------------------------------------- + +function getWebVideoFileReadStream (options: { + filename: string + rangeHeader: string +}) { + const { filename, rangeHeader } = options + + const key = generateWebVideoObjectStorageKey(filename) + + return createObjectReadStream({ + key, + bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS, + rangeHeader + }) +} + +function getHLSFileReadStream (options: { + playlist: MStreamingPlaylistVideo + filename: string + rangeHeader: string +}) { + const { playlist, filename, rangeHeader } = options + + const key = generateHLSObjectStorageKey(playlist, filename) + + return createObjectReadStream({ + key, + bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, + rangeHeader + }) +} + +// --------------------------------------------------------------------------- + +export { + listHLSFileKeysOf, + + storeWebVideoFile, + storeHLSFileFromFilename, + storeHLSFileFromPath, + storeHLSFileFromContent, + + updateWebVideoFileACL, + updateHLSFilesACL, + + removeHLSObjectStorage, + removeHLSFileObjectStorageByFilename, + removeHLSFileObjectStorageByPath, + removeHLSFileObjectStorageByFullKey, + + removeWebVideoObjectStorage, + + makeWebVideoFileAvailable, + makeHLSFileAvailable, + + getWebVideoFileReadStream, + getHLSFileReadStream +} diff --git a/server/lib/opentelemetry/metric-helpers/bittorrent-tracker-observers-builder.ts b/server/server/lib/opentelemetry/metric-helpers/bittorrent-tracker-observers-builder.ts similarity index 100% rename from server/lib/opentelemetry/metric-helpers/bittorrent-tracker-observers-builder.ts rename to server/server/lib/opentelemetry/metric-helpers/bittorrent-tracker-observers-builder.ts diff --git a/server/server/lib/opentelemetry/metric-helpers/index.ts b/server/server/lib/opentelemetry/metric-helpers/index.ts new file mode 100644 index 000000000..ec9c35408 --- /dev/null +++ b/server/server/lib/opentelemetry/metric-helpers/index.ts @@ -0,0 +1,7 @@ +export * from './bittorrent-tracker-observers-builder.js' +export * from './lives-observers-builder.js' +export * from './job-queue-observers-builder.js' +export * from './nodejs-observers-builder.js' +export * from './playback-metrics.js' +export * from './stats-observers-builder.js' +export * from './viewers-observers-builder.js' diff --git a/server/server/lib/opentelemetry/metric-helpers/job-queue-observers-builder.ts b/server/server/lib/opentelemetry/metric-helpers/job-queue-observers-builder.ts new file mode 100644 index 000000000..c0bcb517e --- /dev/null +++ b/server/server/lib/opentelemetry/metric-helpers/job-queue-observers-builder.ts @@ -0,0 +1,24 @@ +import { Meter } from '@opentelemetry/api' +import { JobQueue } from '@server/lib/job-queue/index.js' + +export class JobQueueObserversBuilder { + + constructor (private readonly meter: Meter) { + + } + + buildObservers () { + this.meter.createObservableGauge('peertube_job_queue_total', { + description: 'Total jobs in the PeerTube job queue' + }).addCallback(async observableResult => { + const stats = await JobQueue.Instance.getStats() + + for (const { jobType, counts } of stats) { + for (const state of Object.keys(counts)) { + observableResult.observe(counts[state], { jobType, state }) + } + } + }) + } + +} diff --git a/server/server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts b/server/server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts new file mode 100644 index 000000000..45676ed2b --- /dev/null +++ b/server/server/lib/opentelemetry/metric-helpers/lives-observers-builder.ts @@ -0,0 +1,21 @@ +import { Meter } from '@opentelemetry/api' +import { VideoModel } from '@server/models/video/video.js' + +export class LivesObserversBuilder { + + constructor (private readonly meter: Meter) { + + } + + buildObservers () { + this.meter.createObservableGauge('peertube_running_lives_total', { + description: 'Total running lives on the instance' + }).addCallback(async observableResult => { + const local = await VideoModel.countLives({ remote: false, mode: 'published' }) + const remote = await VideoModel.countLives({ remote: true, mode: 'published' }) + + observableResult.observe(local, { liveOrigin: 'local' }) + observableResult.observe(remote, { liveOrigin: 'remote' }) + }) + } +} diff --git a/server/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts b/server/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts new file mode 100644 index 000000000..40df12b8d --- /dev/null +++ b/server/server/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts @@ -0,0 +1,202 @@ +import { readdir } from 'fs/promises' +import { constants, NodeGCPerformanceDetail, PerformanceObserver } from 'perf_hooks' +import * as process from 'process' +import { Meter, ObservableResult } from '@opentelemetry/api' +import { ExplicitBucketHistogramAggregation } from '@opentelemetry/sdk-metrics' +import { View } from '@opentelemetry/sdk-metrics/build/src/view/View.js' +import { logger } from '@server/helpers/logger.js' + +// Thanks to https://github.com/siimon/prom-client +// We took their logic and adapter it for opentelemetry +// Try to keep consistency with their metric name/description so it's easier to process (grafana dashboard template etc) + +export class NodeJSObserversBuilder { + + constructor (private readonly meter: Meter) { + } + + static getViews () { + return [ + new View({ + aggregation: new ExplicitBucketHistogramAggregation([ 0.001, 0.01, 0.1, 1, 2, 5 ]), + instrumentName: 'nodejs_gc_duration_seconds' + }) + ] + } + + buildObservers () { + this.buildCPUObserver() + this.buildMemoryObserver() + + this.buildHandlesObserver() + this.buildFileDescriptorsObserver() + + this.buildGCObserver() + this.buildEventLoopLagObserver() + + this.buildLibUVActiveRequestsObserver() + this.buildActiveResourcesObserver() + } + + private buildCPUObserver () { + const cpuTotal = this.meter.createObservableCounter('process_cpu_seconds_total', { + description: 'Total user and system CPU time spent in seconds.' + }) + const cpuUser = this.meter.createObservableCounter('process_cpu_user_seconds_total', { + description: 'Total user CPU time spent in seconds.' + }) + const cpuSystem = this.meter.createObservableCounter('process_cpu_system_seconds_total', { + description: 'Total system CPU time spent in seconds.' + }) + + let lastCpuUsage = process.cpuUsage() + + this.meter.addBatchObservableCallback(observableResult => { + const cpuUsage = process.cpuUsage() + + const userUsageMicros = cpuUsage.user - lastCpuUsage.user + const systemUsageMicros = cpuUsage.system - lastCpuUsage.system + + lastCpuUsage = cpuUsage + + observableResult.observe(cpuTotal, (userUsageMicros + systemUsageMicros) / 1e6) + observableResult.observe(cpuUser, userUsageMicros / 1e6) + observableResult.observe(cpuSystem, systemUsageMicros / 1e6) + + }, [ cpuTotal, cpuUser, cpuSystem ]) + } + + private buildMemoryObserver () { + this.meter.createObservableGauge('nodejs_memory_usage_bytes', { + description: 'Memory' + }).addCallback(observableResult => { + const current = process.memoryUsage() + + observableResult.observe(current.heapTotal, { memoryType: 'heapTotal' }) + observableResult.observe(current.heapUsed, { memoryType: 'heapUsed' }) + observableResult.observe(current.arrayBuffers, { memoryType: 'arrayBuffers' }) + observableResult.observe(current.external, { memoryType: 'external' }) + observableResult.observe(current.rss, { memoryType: 'rss' }) + }) + } + + private buildHandlesObserver () { + if (typeof (process as any)._getActiveHandles !== 'function') return + + this.meter.createObservableGauge('nodejs_active_handles_total', { + description: 'Total number of active handles.' + }).addCallback(observableResult => { + const handles = (process as any)._getActiveHandles() + + observableResult.observe(handles.length) + }) + } + + private buildGCObserver () { + const kinds = { + [constants.NODE_PERFORMANCE_GC_MAJOR]: 'major', + [constants.NODE_PERFORMANCE_GC_MINOR]: 'minor', + [constants.NODE_PERFORMANCE_GC_INCREMENTAL]: 'incremental', + [constants.NODE_PERFORMANCE_GC_WEAKCB]: 'weakcb' + } + + const histogram = this.meter.createHistogram('nodejs_gc_duration_seconds', { + description: 'Garbage collection duration by kind, one of major, minor, incremental or weakcb' + }) + + const obs = new PerformanceObserver(list => { + const entry = list.getEntries()[0] + + // Node < 16 uses entry.kind + // Node >= 16 uses entry.detail.kind + // See: https://nodejs.org/docs/latest-v16.x/api/deprecations.html#deprecations_dep0152_extension_performanceentry_properties + const kind = entry.detail + ? kinds[(entry.detail as NodeGCPerformanceDetail).kind] + : kinds[(entry as any).kind] + + // Convert duration from milliseconds to seconds + histogram.record(entry.duration / 1000, { + kind + }) + }) + + obs.observe({ entryTypes: [ 'gc' ] }) + } + + private buildEventLoopLagObserver () { + const reportEventloopLag = (start: [ number, number ], observableResult: ObservableResult, res: () => void) => { + const delta = process.hrtime(start) + const nanosec = delta[0] * 1e9 + delta[1] + const seconds = nanosec / 1e9 + + observableResult.observe(seconds) + + res() + } + + this.meter.createObservableGauge('nodejs_eventloop_lag_seconds', { + description: 'Lag of event loop in seconds.' + }).addCallback(observableResult => { + return new Promise(res => { + const start = process.hrtime() + + setImmediate(reportEventloopLag, start, observableResult, res) + }) + }) + } + + private buildFileDescriptorsObserver () { + this.meter.createObservableGauge('process_open_fds', { + description: 'Number of open file descriptors.' + }).addCallback(async observableResult => { + try { + const fds = await readdir('/proc/self/fd') + observableResult.observe(fds.length - 1) + } catch (err) { + logger.debug('Cannot list file descriptors of current process for OpenTelemetry.', { err }) + } + }) + } + + private buildLibUVActiveRequestsObserver () { + if (typeof (process as any)._getActiveRequests !== 'function') return + + this.meter.createObservableGauge('nodejs_active_requests_total', { + description: 'Total number of active libuv requests.' + }).addCallback(observableResult => { + const requests = (process as any)._getActiveRequests() + + observableResult.observe(requests.length) + }) + } + + private buildActiveResourcesObserver () { + if (typeof (process as any).getActiveResourcesInfo !== 'function') return + + const grouped = this.meter.createObservableCounter('nodejs_active_resources', { + description: 'Number of active resources that are currently keeping the event loop alive, grouped by async resource type.' + }) + const total = this.meter.createObservableCounter('nodejs_active_resources_total', { + description: 'Total number of active resources.' + }) + + this.meter.addBatchObservableCallback(observableResult => { + const resources = (process as any).getActiveResourcesInfo() + + const data = {} + + for (let i = 0; i < resources.length; i++) { + const resource = resources[i] + + if (data[resource] === undefined) data[resource] = 0 + data[resource] += 1 + } + + for (const type of Object.keys(data)) { + observableResult.observe(grouped, data[type], { type }) + } + + observableResult.observe(total, resources.length) + }, [ grouped, total ]) + } +} diff --git a/server/server/lib/opentelemetry/metric-helpers/playback-metrics.ts b/server/server/lib/opentelemetry/metric-helpers/playback-metrics.ts new file mode 100644 index 000000000..ec139f331 --- /dev/null +++ b/server/server/lib/opentelemetry/metric-helpers/playback-metrics.ts @@ -0,0 +1,85 @@ +import { Counter, Meter } from '@opentelemetry/api' +import { MVideoImmutable } from '@server/types/models/index.js' +import { PlaybackMetricCreate } from '@peertube/peertube-models' + +export class PlaybackMetrics { + private errorsCounter: Counter + private resolutionChangesCounter: Counter + + private downloadedBytesP2PCounter: Counter + private uploadedBytesP2PCounter: Counter + + private downloadedBytesHTTPCounter: Counter + + private peersP2PPeersGaugeBuffer: { + value: number + attributes: any + }[] = [] + + constructor (private readonly meter: Meter) { + + } + + buildCounters () { + this.errorsCounter = this.meter.createCounter('peertube_playback_errors_count', { + description: 'Errors collected from PeerTube player.' + }) + + this.resolutionChangesCounter = this.meter.createCounter('peertube_playback_resolution_changes_count', { + description: 'Resolution changes collected from PeerTube player.' + }) + + this.downloadedBytesHTTPCounter = this.meter.createCounter('peertube_playback_http_downloaded_bytes', { + description: 'Downloaded bytes with HTTP by PeerTube player.' + }) + this.downloadedBytesP2PCounter = this.meter.createCounter('peertube_playback_p2p_downloaded_bytes', { + description: 'Downloaded bytes with P2P by PeerTube player.' + }) + + this.uploadedBytesP2PCounter = this.meter.createCounter('peertube_playback_p2p_uploaded_bytes', { + description: 'Uploaded bytes with P2P by PeerTube player.' + }) + + this.meter.createObservableGauge('peertube_playback_p2p_peers', { + description: 'Total P2P peers connected to the PeerTube player.' + }).addCallback(observableResult => { + for (const gauge of this.peersP2PPeersGaugeBuffer) { + observableResult.observe(gauge.value, gauge.attributes) + } + + this.peersP2PPeersGaugeBuffer = [] + }) + } + + observe (video: MVideoImmutable, metrics: PlaybackMetricCreate) { + const attributes = { + videoOrigin: video.remote + ? 'remote' + : 'local', + + playerMode: metrics.playerMode, + + resolution: metrics.resolution + '', + fps: metrics.fps + '', + + p2pEnabled: metrics.p2pEnabled, + + videoUUID: video.uuid + } + + this.errorsCounter.add(metrics.errors, attributes) + this.resolutionChangesCounter.add(metrics.resolutionChanges, attributes) + + this.downloadedBytesHTTPCounter.add(metrics.downloadedBytesHTTP, attributes) + this.downloadedBytesP2PCounter.add(metrics.downloadedBytesP2P, attributes) + + this.uploadedBytesP2PCounter.add(metrics.uploadedBytesP2P, attributes) + + if (metrics.p2pPeers) { + this.peersP2PPeersGaugeBuffer.push({ + value: metrics.p2pPeers, + attributes + }) + } + } +} diff --git a/server/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts b/server/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts new file mode 100644 index 000000000..05e6431a4 --- /dev/null +++ b/server/server/lib/opentelemetry/metric-helpers/stats-observers-builder.ts @@ -0,0 +1,186 @@ +import memoizee from 'memoizee' +import { Meter } from '@opentelemetry/api' +import { MEMOIZE_TTL } from '@server/initializers/constants.js' +import { buildAvailableActivities } from '@server/lib/activitypub/activity.js' +import { StatsManager } from '@server/lib/stat-manager.js' + +export class StatsObserversBuilder { + + private readonly getInstanceStats = memoizee(() => { + return StatsManager.Instance.getStats() + }, { maxAge: MEMOIZE_TTL.GET_STATS_FOR_OPEN_TELEMETRY_METRICS }) + + constructor (private readonly meter: Meter) { + + } + + buildObservers () { + this.buildUserStatsObserver() + this.buildVideoStatsObserver() + this.buildCommentStatsObserver() + this.buildPlaylistStatsObserver() + this.buildChannelStatsObserver() + this.buildInstanceFollowsStatsObserver() + this.buildRedundancyStatsObserver() + this.buildActivityPubStatsObserver() + } + + private buildUserStatsObserver () { + this.meter.createObservableGauge('peertube_users_total', { + description: 'Total users on the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalUsers) + }) + + this.meter.createObservableGauge('peertube_active_users_total', { + description: 'Total active users on the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalDailyActiveUsers, { activeInterval: 'daily' }) + observableResult.observe(stats.totalWeeklyActiveUsers, { activeInterval: 'weekly' }) + observableResult.observe(stats.totalMonthlyActiveUsers, { activeInterval: 'monthly' }) + }) + } + + private buildChannelStatsObserver () { + this.meter.createObservableGauge('peertube_channels_total', { + description: 'Total channels on the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalLocalVideoChannels, { channelOrigin: 'local' }) + }) + + this.meter.createObservableGauge('peertube_active_channels_total', { + description: 'Total active channels on the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalLocalDailyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'daily' }) + observableResult.observe(stats.totalLocalWeeklyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'weekly' }) + observableResult.observe(stats.totalLocalMonthlyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'monthly' }) + }) + } + + private buildVideoStatsObserver () { + this.meter.createObservableGauge('peertube_videos_total', { + description: 'Total videos on the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalLocalVideos, { videoOrigin: 'local' }) + observableResult.observe(stats.totalVideos - stats.totalLocalVideos, { videoOrigin: 'remote' }) + }) + + this.meter.createObservableGauge('peertube_video_views_total', { + description: 'Total video views made on the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalLocalVideoViews, { viewOrigin: 'local' }) + }) + + this.meter.createObservableGauge('peertube_video_bytes_total', { + description: 'Total bytes of videos' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalLocalVideoFilesSize, { videoOrigin: 'local' }) + }) + } + + private buildCommentStatsObserver () { + this.meter.createObservableGauge('peertube_comments_total', { + description: 'Total comments on the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalLocalVideoComments, { accountOrigin: 'local' }) + }) + } + + private buildPlaylistStatsObserver () { + this.meter.createObservableGauge('peertube_playlists_total', { + description: 'Total playlists on the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalLocalPlaylists, { playlistOrigin: 'local' }) + }) + } + + private buildInstanceFollowsStatsObserver () { + this.meter.createObservableGauge('peertube_instance_followers_total', { + description: 'Total followers of the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalInstanceFollowers) + }) + + this.meter.createObservableGauge('peertube_instance_following_total', { + description: 'Total following of the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalInstanceFollowing) + }) + } + + private buildRedundancyStatsObserver () { + this.meter.createObservableGauge('peertube_redundancy_used_bytes_total', { + description: 'Total redundancy used of the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + for (const r of stats.videosRedundancy) { + observableResult.observe(r.totalUsed, { strategy: r.strategy }) + } + }) + + this.meter.createObservableGauge('peertube_redundancy_available_bytes_total', { + description: 'Total redundancy available of the instance' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + for (const r of stats.videosRedundancy) { + observableResult.observe(r.totalSize, { strategy: r.strategy }) + } + }) + } + + private buildActivityPubStatsObserver () { + const availableActivities = buildAvailableActivities() + + this.meter.createObservableGauge('peertube_ap_inbox_success_total', { + description: 'Total inbox messages processed with success' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + for (const type of availableActivities) { + observableResult.observe(stats[`totalActivityPub${type}MessagesSuccesses`], { activityType: type }) + } + }) + + this.meter.createObservableGauge('peertube_ap_inbox_error_total', { + description: 'Total inbox messages processed with error' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + for (const type of availableActivities) { + observableResult.observe(stats[`totalActivityPub${type}MessagesErrors`], { activityType: type }) + } + }) + + this.meter.createObservableGauge('peertube_ap_inbox_waiting_total', { + description: 'Total inbox messages waiting for being processed' + }).addCallback(async observableResult => { + const stats = await this.getInstanceStats() + + observableResult.observe(stats.totalActivityPubMessagesWaiting) + }) + } +} diff --git a/server/server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts b/server/server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts new file mode 100644 index 000000000..a1e1c7496 --- /dev/null +++ b/server/server/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts @@ -0,0 +1,24 @@ +import { Meter } from '@opentelemetry/api' +import { VideoScope, ViewerScope } from '@server/lib/views/shared/index.js' +import { VideoViewsManager } from '@server/lib/views/video-views-manager.js' + +export class ViewersObserversBuilder { + + constructor (private readonly meter: Meter) { + + } + + buildObservers () { + this.meter.createObservableGauge('peertube_viewers_total', { + description: 'Total viewers on the instance' + }).addCallback(observableResult => { + for (const viewerScope of [ 'local', 'remote' ] as ViewerScope[]) { + for (const videoScope of [ 'local', 'remote' ] as VideoScope[]) { + const result = VideoViewsManager.Instance.getTotalViewers({ viewerScope, videoScope }) + + observableResult.observe(result, { viewerOrigin: viewerScope, videoOrigin: videoScope }) + } + } + }) + } +} diff --git a/server/server/lib/opentelemetry/metrics.ts b/server/server/lib/opentelemetry/metrics.ts new file mode 100644 index 000000000..7276182ef --- /dev/null +++ b/server/server/lib/opentelemetry/metrics.ts @@ -0,0 +1,123 @@ +import { Application, Request, Response } from 'express' +import { Meter, metrics } from '@opentelemetry/api' +import { PrometheusExporter } from '@opentelemetry/exporter-prometheus' +import { MeterProvider } from '@opentelemetry/sdk-metrics' +import { logger } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' +import { MVideoImmutable } from '@server/types/models/index.js' +import { PlaybackMetricCreate } from '@peertube/peertube-models' +import { + BittorrentTrackerObserversBuilder, + JobQueueObserversBuilder, + LivesObserversBuilder, + NodeJSObserversBuilder, + PlaybackMetrics, + StatsObserversBuilder, + ViewersObserversBuilder +} from './metric-helpers/index.js' + +class OpenTelemetryMetrics { + + private static instance: OpenTelemetryMetrics + + private meter: Meter + + private onRequestDuration: (req: Request, res: Response) => void + + private playbackMetrics: PlaybackMetrics + + private constructor () {} + + init (app: Application) { + if (CONFIG.OPEN_TELEMETRY.METRICS.ENABLED !== true) return + + app.use((req, res, next) => { + res.once('finish', () => { + if (!this.onRequestDuration) return + + this.onRequestDuration(req as Request, res as Response) + }) + + next() + }) + } + + registerMetrics (options: { trackerServer: any }) { + if (CONFIG.OPEN_TELEMETRY.METRICS.ENABLED !== true) return + + logger.info('Registering Open Telemetry metrics') + + const provider = new MeterProvider({ + views: [ + ...NodeJSObserversBuilder.getViews() + ] + }) + + provider.addMetricReader(new PrometheusExporter({ + host: CONFIG.OPEN_TELEMETRY.METRICS.PROMETHEUS_EXPORTER.HOSTNAME, + port: CONFIG.OPEN_TELEMETRY.METRICS.PROMETHEUS_EXPORTER.PORT + })) + + metrics.setGlobalMeterProvider(provider) + + this.meter = metrics.getMeter('default') + + if (CONFIG.OPEN_TELEMETRY.METRICS.HTTP_REQUEST_DURATION.ENABLED === true) { + this.buildRequestObserver() + } + + this.playbackMetrics = new PlaybackMetrics(this.meter) + this.playbackMetrics.buildCounters() + + const nodeJSObserversBuilder = new NodeJSObserversBuilder(this.meter) + nodeJSObserversBuilder.buildObservers() + + const jobQueueObserversBuilder = new JobQueueObserversBuilder(this.meter) + jobQueueObserversBuilder.buildObservers() + + const statsObserversBuilder = new StatsObserversBuilder(this.meter) + statsObserversBuilder.buildObservers() + + const livesObserversBuilder = new LivesObserversBuilder(this.meter) + livesObserversBuilder.buildObservers() + + const viewersObserversBuilder = new ViewersObserversBuilder(this.meter) + viewersObserversBuilder.buildObservers() + + const bittorrentTrackerObserversBuilder = new BittorrentTrackerObserversBuilder(this.meter, options.trackerServer) + bittorrentTrackerObserversBuilder.buildObservers() + } + + observePlaybackMetric (video: MVideoImmutable, metrics: PlaybackMetricCreate) { + this.playbackMetrics.observe(video, metrics) + } + + private buildRequestObserver () { + const requestDuration = this.meter.createHistogram('http_request_duration_ms', { + unit: 'milliseconds', + description: 'Duration of HTTP requests in ms' + }) + + this.onRequestDuration = (req: Request, res: Response) => { + const duration = Date.now() - res.locals.requestStart + + requestDuration.record(duration, { + path: this.buildRequestPath(req.originalUrl), + method: req.method, + statusCode: res.statusCode + '' + }) + } + } + + private buildRequestPath (path: string) { + return path.split('?')[0] + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} + +export { + OpenTelemetryMetrics +} diff --git a/server/server/lib/opentelemetry/tracing.ts b/server/server/lib/opentelemetry/tracing.ts new file mode 100644 index 000000000..b306fa40c --- /dev/null +++ b/server/server/lib/opentelemetry/tracing.ts @@ -0,0 +1,140 @@ +import type { Span, Tracer } from '@opentelemetry/api' +import { logger } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' + +let tracer: Tracer | TrackerMock + +async function registerOpentelemetryTracing () { + if (CONFIG.OPEN_TELEMETRY.TRACING.ENABLED !== true) { + tracer = new TrackerMock() + + return + } + + const { diag, DiagLogLevel, trace } = await import('@opentelemetry/api') + tracer = trace.getTracer('peertube') + + const [ + { JaegerExporter }, + { registerInstrumentations }, + DnsInstrumentation, + ExpressInstrumentation, + { FsInstrumentation }, + { HttpInstrumentation }, + IORedisInstrumentation, + PgInstrumentation, + { SequelizeInstrumentation }, + Resource, + BatchSpanProcessor, + NodeTracerProvider, + SemanticResourceAttributes + ] = await Promise.all([ + import('@opentelemetry/exporter-jaeger'), + import('@opentelemetry/instrumentation'), + import('@opentelemetry/instrumentation-dns'), + import('@opentelemetry/instrumentation-express'), + import('@opentelemetry/instrumentation-fs'), + import('@opentelemetry/instrumentation-http'), + import('@opentelemetry/instrumentation-ioredis'), + import('@opentelemetry/instrumentation-pg'), + import('opentelemetry-instrumentation-sequelize'), + import('@opentelemetry/resources'), + import('@opentelemetry/sdk-trace-base'), + import('@opentelemetry/sdk-trace-node'), + import('@opentelemetry/semantic-conventions') + ]) + + logger.info('Registering Open Telemetry tracing') + + const customLogger = (level: string) => { + return (message: string, ...args: unknown[]) => { + let fullMessage = message + + for (const arg of args) { + if (typeof arg === 'string') fullMessage += arg + else break + } + + logger[level](fullMessage) + } + } + + diag.setLogger({ + error: customLogger('error'), + warn: customLogger('warn'), + info: customLogger('info'), + debug: customLogger('debug'), + verbose: customLogger('verbose') + }, DiagLogLevel.INFO) + + const tracerProvider = new NodeTracerProvider.default.NodeTracerProvider({ + resource: new Resource.default.Resource({ + [SemanticResourceAttributes.default.SemanticResourceAttributes.SERVICE_NAME]: 'peertube' + }) + }) + + registerInstrumentations({ + tracerProvider, + instrumentations: [ + new PgInstrumentation.default.PgInstrumentation({ + enhancedDatabaseReporting: true + }), + new DnsInstrumentation.default.DnsInstrumentation(), + new HttpInstrumentation(), + new ExpressInstrumentation.default.ExpressInstrumentation(), + new IORedisInstrumentation.default.IORedisInstrumentation({ + dbStatementSerializer: function (cmdName, cmdArgs) { + return [ cmdName, ...cmdArgs ].join(' ') + } + }), + new FsInstrumentation(), + new SequelizeInstrumentation() + ] + }) + + tracerProvider.addSpanProcessor( + new BatchSpanProcessor.default.BatchSpanProcessor( + new JaegerExporter({ endpoint: CONFIG.OPEN_TELEMETRY.TRACING.JAEGER_EXPORTER.ENDPOINT }) + ) + ) + + tracerProvider.register() +} + +async function wrapWithSpanAndContext (spanName: string, cb: () => Promise) { + const { context, trace } = await import('@opentelemetry/api') + + if (CONFIG.OPEN_TELEMETRY.TRACING.ENABLED !== true) { + return cb() + } + + const span = tracer.startSpan(spanName) + const activeContext = trace.setSpan(context.active(), span as Span) + + const result = await context.with(activeContext, () => cb()) + span.end() + + return result +} + +export { + registerOpentelemetryTracing, + tracer, + wrapWithSpanAndContext +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +class TrackerMock { + startSpan () { + return new SpanMock() + } +} + +class SpanMock { + end () { + + } +} diff --git a/server/server/lib/paths.ts b/server/server/lib/paths.ts new file mode 100644 index 000000000..208e78efa --- /dev/null +++ b/server/server/lib/paths.ts @@ -0,0 +1,92 @@ +import { join } from 'path' +import { CONFIG } from '@server/initializers/config.js' +import { DIRECTORIES, VIDEO_LIVE } from '@server/initializers/constants.js' +import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoFile, MVideoUUID } from '@server/types/models/index.js' +import { removeFragmentedMP4Ext } from '@peertube/peertube-core-utils' +import { buildUUID } from '@peertube/peertube-node-utils' +import { isVideoInPrivateDirectory } from './video-privacy.js' + +// ################## Video file name ################## + +function generateWebVideoFilename (resolution: number, extname: string) { + return buildUUID() + '-' + resolution + extname +} + +function generateHLSVideoFilename (resolution: number) { + return `${buildUUID()}-${resolution}-fragmented.mp4` +} + +// ################## Streaming playlist ################## + +function getLiveDirectory (video: MVideo) { + return getHLSDirectory(video) +} + +function getLiveReplayBaseDirectory (video: MVideo) { + return join(getLiveDirectory(video), VIDEO_LIVE.REPLAY_DIRECTORY) +} + +function getHLSDirectory (video: MVideo) { + if (isVideoInPrivateDirectory(video.privacy)) { + return join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, video.uuid) + } + + return join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, video.uuid) +} + +function getHLSRedundancyDirectory (video: MVideoUUID) { + return join(DIRECTORIES.HLS_REDUNDANCY, video.uuid) +} + +function getHlsResolutionPlaylistFilename (videoFilename: string) { + // Video file name already contain resolution + return removeFragmentedMP4Ext(videoFilename) + '.m3u8' +} + +function generateHLSMasterPlaylistFilename (isLive = false) { + if (isLive) return 'master.m3u8' + + return buildUUID() + '-master.m3u8' +} + +function generateHlsSha256SegmentsFilename (isLive = false) { + if (isLive) return 'segments-sha256.json' + + return buildUUID() + '-segments-sha256.json' +} + +// ################## Torrents ################## + +function generateTorrentFileName (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, resolution: number) { + const extension = '.torrent' + const uuid = buildUUID() + + if (isStreamingPlaylist(videoOrPlaylist)) { + return `${uuid}-${resolution}-${videoOrPlaylist.getStringType()}${extension}` + } + + return uuid + '-' + resolution + extension +} + +function getFSTorrentFilePath (videoFile: MVideoFile) { + return join(CONFIG.STORAGE.TORRENTS_DIR, videoFile.torrentFilename) +} + +// --------------------------------------------------------------------------- + +export { + generateHLSVideoFilename, + generateWebVideoFilename, + + generateTorrentFileName, + getFSTorrentFilePath, + + getHLSDirectory, + getLiveDirectory, + getLiveReplayBaseDirectory, + getHLSRedundancyDirectory, + + generateHLSMasterPlaylistFilename, + generateHlsSha256SegmentsFilename, + getHlsResolutionPlaylistFilename +} diff --git a/server/server/lib/peertube-socket.ts b/server/server/lib/peertube-socket.ts new file mode 100644 index 000000000..4bb82189e --- /dev/null +++ b/server/server/lib/peertube-socket.ts @@ -0,0 +1,129 @@ +import { Server as HTTPServer } from 'http' +import { Namespace, Server as SocketServer, Socket } from 'socket.io' +import { isIdValid } from '@server/helpers/custom-validators/misc.js' +import { Debounce } from '@server/helpers/debounce.js' +import { MVideo, MVideoImmutable } from '@server/types/models/index.js' +import { MRunner } from '@server/types/models/runners/index.js' +import { UserNotificationModelForApi } from '@server/types/models/user/index.js' +import { LiveVideoEventPayload, LiveVideoEventType } from '@peertube/peertube-models' +import { logger } from '../helpers/logger.js' +import { authenticateRunnerSocket, authenticateSocket } from '../middlewares/index.js' + +class PeerTubeSocket { + + private static instance: PeerTubeSocket + + private userNotificationSockets: { [ userId: number ]: Socket[] } = {} + private liveVideosNamespace: Namespace + private readonly runnerSockets = new Set() + + private constructor () {} + + init (server: HTTPServer) { + const io = new SocketServer(server) + + io.of('/user-notifications') + .use(authenticateSocket) + .on('connection', socket => { + const userId = socket.handshake.auth.user.id + + logger.debug('User %d connected to the notification system.', userId) + + if (!this.userNotificationSockets[userId]) this.userNotificationSockets[userId] = [] + + this.userNotificationSockets[userId].push(socket) + + socket.on('disconnect', () => { + logger.debug('User %d disconnected from SocketIO notifications.', userId) + + this.userNotificationSockets[userId] = this.userNotificationSockets[userId].filter(s => s !== socket) + }) + }) + + this.liveVideosNamespace = io.of('/live-videos') + .on('connection', socket => { + socket.on('subscribe', ({ videoId }) => { + if (!isIdValid(videoId)) return + + /* eslint-disable @typescript-eslint/no-floating-promises */ + socket.join(videoId) + }) + + socket.on('unsubscribe', ({ videoId }) => { + if (!isIdValid(videoId)) return + + /* eslint-disable @typescript-eslint/no-floating-promises */ + socket.leave(videoId) + }) + }) + + io.of('/runners') + .use(authenticateRunnerSocket) + .on('connection', socket => { + const runner: MRunner = socket.handshake.auth.runner + + logger.debug(`New runner "${runner.name}" connected to the notification system.`) + + this.runnerSockets.add(socket) + + socket.on('disconnect', () => { + logger.debug(`Runner "${runner.name}" disconnected from the notification system.`) + + this.runnerSockets.delete(socket) + }) + }) + } + + sendNotification (userId: number, notification: UserNotificationModelForApi) { + const sockets = this.userNotificationSockets[userId] + if (!sockets) return + + logger.debug('Sending user notification to user %d.', userId) + + const notificationMessage = notification.toFormattedJSON() + for (const socket of sockets) { + socket.emit('new-notification', notificationMessage) + } + } + + sendVideoLiveNewState (video: MVideo) { + const data: LiveVideoEventPayload = { state: video.state } + const type: LiveVideoEventType = 'state-change' + + logger.debug('Sending video live new state notification of %s.', video.url, { state: video.state }) + + this.liveVideosNamespace + .in(video.id) + .emit(type, data) + } + + sendVideoViewsUpdate (video: MVideoImmutable, numViewers: number) { + const data: LiveVideoEventPayload = { viewers: numViewers } + const type: LiveVideoEventType = 'views-change' + + logger.debug('Sending video live views update notification of %s.', video.url, { viewers: numViewers }) + + this.liveVideosNamespace + .in(video.id) + .emit(type, data) + } + + @Debounce({ timeoutMS: 1000 }) + sendAvailableJobsPingToRunners () { + logger.debug(`Sending available-jobs notification to ${this.runnerSockets.size} runner sockets`) + + for (const runners of this.runnerSockets) { + runners.emit('available-jobs') + } + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} + +// --------------------------------------------------------------------------- + +export { + PeerTubeSocket +} diff --git a/server/server/lib/plugins/hooks.ts b/server/server/lib/plugins/hooks.ts new file mode 100644 index 000000000..70aaf4786 --- /dev/null +++ b/server/server/lib/plugins/hooks.ts @@ -0,0 +1,35 @@ +import Bluebird from 'bluebird' +import { ServerActionHookName, ServerFilterHookName } from '@peertube/peertube-models' +import { logger } from '../../helpers/logger.js' +import { PluginManager } from './plugin-manager.js' + +type PromiseFunction = (params: U) => Promise | Bluebird +type RawFunction = (params: U) => T + +// Helpers to run hooks +const Hooks = { + wrapObject: (result: T, hookName: U, context?: any) => { + return PluginManager.Instance.runHook(hookName, result, context) + }, + + wrapPromiseFun: async (fun: PromiseFunction, params: U, hookName: V) => { + const result = await fun(params) + + return PluginManager.Instance.runHook(hookName, result, params) + }, + + wrapFun: async (fun: RawFunction, params: U, hookName: V) => { + const result = fun(params) + + return PluginManager.Instance.runHook(hookName, result, params) + }, + + runAction: (hookName: U, params?: T) => { + PluginManager.Instance.runHook(hookName, undefined, params) + .catch(err => logger.error('Fatal hook error.', { err })) + } +} + +export { + Hooks +} diff --git a/server/server/lib/plugins/plugin-helpers-builder.ts b/server/server/lib/plugins/plugin-helpers-builder.ts new file mode 100644 index 000000000..4c1bd2bf7 --- /dev/null +++ b/server/server/lib/plugins/plugin-helpers-builder.ts @@ -0,0 +1,262 @@ +import express from 'express' +import { Server } from 'http' +import { join } from 'path' +import { buildLogger } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { sequelizeTypescript } from '@server/initializers/database.js' +import { AccountModel } from '@server/models/account/account.js' +import { AccountBlocklistModel } from '@server/models/account/account-blocklist.js' +import { getServerActor } from '@server/models/application/application.js' +import { ServerModel } from '@server/models/server/server.js' +import { ServerBlocklistModel } from '@server/models/server/server-blocklist.js' +import { UserModel } from '@server/models/user/user.js' +import { VideoModel } from '@server/models/video/video.js' +import { VideoBlacklistModel } from '@server/models/video/video-blacklist.js' +import { MPlugin, MVideo, UserNotificationModelForApi } from '@server/types/models/index.js' +import { PeerTubeHelpers } from '@server/types/plugins/index.js' +import { ffprobePromise } from '@peertube/peertube-ffmpeg' +import { VideoBlacklistCreate, VideoStorage } from '@peertube/peertube-models' +import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../blocklist.js' +import { PeerTubeSocket } from '../peertube-socket.js' +import { ServerConfigManager } from '../server-config-manager.js' +import { blacklistVideo, unblacklistVideo } from '../video-blacklist.js' +import { VideoPathManager } from '../video-path-manager.js' + +function buildPluginHelpers (httpServer: Server, pluginModel: MPlugin, npmName: string): PeerTubeHelpers { + const logger = buildPluginLogger(npmName) + + const database = buildDatabaseHelpers() + const videos = buildVideosHelpers() + + const config = buildConfigHelpers() + + const server = buildServerHelpers(httpServer) + + const moderation = buildModerationHelpers() + + const plugin = buildPluginRelatedHelpers(pluginModel, npmName) + + const socket = buildSocketHelpers() + + const user = buildUserHelpers() + + return { + logger, + database, + videos, + config, + moderation, + plugin, + server, + socket, + user + } +} + +export { + buildPluginHelpers +} + +// --------------------------------------------------------------------------- + +function buildPluginLogger (npmName: string) { + return buildLogger(npmName) +} + +function buildDatabaseHelpers () { + return { + query: sequelizeTypescript.query.bind(sequelizeTypescript) + } +} + +function buildServerHelpers (httpServer: Server) { + return { + getHTTPServer: () => httpServer, + + getServerActor: () => getServerActor() + } +} + +function buildVideosHelpers () { + return { + loadByUrl: (url: string) => { + return VideoModel.loadByUrl(url) + }, + + loadByIdOrUUID: (id: number | string) => { + return VideoModel.load(id) + }, + + removeVideo: (id: number) => { + return sequelizeTypescript.transaction(async t => { + const video = await VideoModel.loadFull(id, t) + + await video.destroy({ transaction: t }) + }) + }, + + ffprobe: (path: string) => { + return ffprobePromise(path) + }, + + getFiles: async (id: number | string) => { + const video = await VideoModel.loadFull(id) + if (!video) return undefined + + const webVideoFiles = (video.VideoFiles || []).map(f => ({ + path: f.storage === VideoStorage.FILE_SYSTEM + ? VideoPathManager.Instance.getFSVideoFileOutputPath(video, f) + : null, + url: f.getFileUrl(video), + + resolution: f.resolution, + size: f.size, + fps: f.fps + })) + + const hls = video.getHLSPlaylist() + + const hlsVideoFiles = hls + ? (video.getHLSPlaylist().VideoFiles || []).map(f => { + return { + path: f.storage === VideoStorage.FILE_SYSTEM + ? VideoPathManager.Instance.getFSVideoFileOutputPath(hls, f) + : null, + url: f.getFileUrl(video), + resolution: f.resolution, + size: f.size, + fps: f.fps + } + }) + : [] + + const thumbnails = video.Thumbnails.map(t => ({ + type: t.type, + url: t.getOriginFileUrl(video), + path: t.getPath() + })) + + return { + webtorrent: { // TODO: remove in v7 + videoFiles: webVideoFiles + }, + + webVideo: { + videoFiles: webVideoFiles + }, + + hls: { + videoFiles: hlsVideoFiles + }, + + thumbnails + } + } + } +} + +function buildModerationHelpers () { + return { + blockServer: async (options: { byAccountId: number, hostToBlock: string }) => { + const serverToBlock = await ServerModel.loadOrCreateByHost(options.hostToBlock) + + await addServerInBlocklist(options.byAccountId, serverToBlock.id) + }, + + unblockServer: async (options: { byAccountId: number, hostToUnblock: string }) => { + const serverBlock = await ServerBlocklistModel.loadByAccountAndHost(options.byAccountId, options.hostToUnblock) + if (!serverBlock) return + + await removeServerFromBlocklist(serverBlock) + }, + + blockAccount: async (options: { byAccountId: number, handleToBlock: string }) => { + const accountToBlock = await AccountModel.loadByNameWithHost(options.handleToBlock) + if (!accountToBlock) return + + await addAccountInBlocklist(options.byAccountId, accountToBlock.id) + }, + + unblockAccount: async (options: { byAccountId: number, handleToUnblock: string }) => { + const targetAccount = await AccountModel.loadByNameWithHost(options.handleToUnblock) + if (!targetAccount) return + + const accountBlock = await AccountBlocklistModel.loadByAccountAndTarget(options.byAccountId, targetAccount.id) + if (!accountBlock) return + + await removeAccountFromBlocklist(accountBlock) + }, + + blacklistVideo: async (options: { videoIdOrUUID: number | string, createOptions: VideoBlacklistCreate }) => { + const video = await VideoModel.loadFull(options.videoIdOrUUID) + if (!video) return + + await blacklistVideo(video, options.createOptions) + }, + + unblacklistVideo: async (options: { videoIdOrUUID: number | string }) => { + const video = await VideoModel.loadFull(options.videoIdOrUUID) + if (!video) return + + const videoBlacklist = await VideoBlacklistModel.loadByVideoId(video.id) + if (!videoBlacklist) return + + await unblacklistVideo(videoBlacklist, video) + } + } +} + +function buildConfigHelpers () { + return { + getWebserverUrl () { + return WEBSERVER.URL + }, + + getServerListeningConfig () { + return { hostname: CONFIG.LISTEN.HOSTNAME, port: CONFIG.LISTEN.PORT } + }, + + getServerConfig () { + return ServerConfigManager.Instance.getServerConfig() + } + } +} + +function buildPluginRelatedHelpers (plugin: MPlugin, npmName: string) { + return { + getBaseStaticRoute: () => `/plugins/${plugin.name}/${plugin.version}/static/`, + + getBaseRouterRoute: () => `/plugins/${plugin.name}/${plugin.version}/router/`, + + getBaseWebSocketRoute: () => `/plugins/${plugin.name}/${plugin.version}/ws/`, + + getDataDirectoryPath: () => join(CONFIG.STORAGE.PLUGINS_DIR, 'data', npmName) + } +} + +function buildSocketHelpers () { + return { + sendNotification: (userId: number, notification: UserNotificationModelForApi) => { + PeerTubeSocket.Instance.sendNotification(userId, notification) + }, + sendVideoLiveNewState: (video: MVideo) => { + PeerTubeSocket.Instance.sendVideoLiveNewState(video) + } + } +} + +function buildUserHelpers () { + return { + loadById: (id: number) => { + return UserModel.loadByIdFull(id) + }, + + getAuthUser: (res: express.Response) => { + const user = res.locals.oauth?.token?.User || res.locals.videoFileToken?.user + if (!user) return undefined + + return UserModel.loadByIdFull(user.id) + } + } +} diff --git a/server/server/lib/plugins/plugin-index.ts b/server/server/lib/plugins/plugin-index.ts new file mode 100644 index 000000000..0f436f6b8 --- /dev/null +++ b/server/server/lib/plugins/plugin-index.ts @@ -0,0 +1,85 @@ +import { sanitizeUrl } from '@server/helpers/core-utils.js' +import { logger } from '@server/helpers/logger.js' +import { doJSONRequest } from '@server/helpers/requests.js' +import { CONFIG } from '@server/initializers/config.js' +import { PEERTUBE_VERSION } from '@server/initializers/constants.js' +import { PluginModel } from '@server/models/server/plugin.js' +import { + PeerTubePluginIndex, + PeertubePluginIndexList, + PeertubePluginLatestVersionRequest, + PeertubePluginLatestVersionResponse, + ResultList +} from '@peertube/peertube-models' +import { PluginManager } from './plugin-manager.js' + +async function listAvailablePluginsFromIndex (options: PeertubePluginIndexList) { + const { start = 0, count = 20, search, sort = 'npmName', pluginType } = options + + const searchParams: PeertubePluginIndexList & Record = { + start, + count, + sort, + pluginType, + search, + currentPeerTubeEngine: options.currentPeerTubeEngine || PEERTUBE_VERSION + } + + const uri = CONFIG.PLUGINS.INDEX.URL + '/api/v1/plugins' + + try { + const { body } = await doJSONRequest(uri, { searchParams }) + + logger.debug('Got result from PeerTube index.', { body }) + + addInstanceInformation(body) + + return body as ResultList + } catch (err) { + logger.error('Cannot list available plugins from index %s.', uri, { err }) + return undefined + } +} + +function addInstanceInformation (result: ResultList) { + for (const d of result.data) { + d.installed = PluginManager.Instance.isRegistered(d.npmName) + d.name = PluginModel.normalizePluginName(d.npmName) + } + + return result +} + +async function getLatestPluginsVersion (npmNames: string[]): Promise { + const bodyRequest: PeertubePluginLatestVersionRequest = { + npmNames, + currentPeerTubeEngine: PEERTUBE_VERSION + } + + const uri = sanitizeUrl(CONFIG.PLUGINS.INDEX.URL) + '/api/v1/plugins/latest-version' + + const options = { + json: bodyRequest, + method: 'POST' as 'POST' + } + const { body } = await doJSONRequest(uri, options) + + return body +} + +async function getLatestPluginVersion (npmName: string) { + const results = await getLatestPluginsVersion([ npmName ]) + + if (Array.isArray(results) === false || results.length !== 1) { + logger.warn('Cannot get latest supported plugin version of %s.', npmName) + return undefined + } + + return results[0].latestVersion +} + +export { + listAvailablePluginsFromIndex, + getLatestPluginVersion, + getLatestPluginsVersion +} diff --git a/server/server/lib/plugins/plugin-manager.ts b/server/server/lib/plugins/plugin-manager.ts new file mode 100644 index 000000000..c4b4fae43 --- /dev/null +++ b/server/server/lib/plugins/plugin-manager.ts @@ -0,0 +1,674 @@ +import express from 'express' +import { createReadStream, createWriteStream } from 'fs' +import { ensureDir, outputFile, readJSON } from 'fs-extra/esm' +import { Server } from 'http' +import { createRequire } from 'module' +import { basename, join } from 'path' +import { getCompleteLocale, getHookType, internalRunHook } from '@peertube/peertube-core-utils' +import { + ClientScriptJSON, + PluginPackageJSON, + PluginTranslation, + PluginTranslationPathsJSON, + PluginType, + PluginType_Type, + RegisterServerHookOptions, + ServerHook, + ServerHookName +} from '@peertube/peertube-models' +import { decachePlugin } from '@server/helpers/decache.js' +import { ApplicationModel } from '@server/models/application/application.js' +import { MOAuthTokenUser, MUser } from '@server/types/models/index.js' +import { isLibraryCodeValid, isPackageJSONValid } from '../../helpers/custom-validators/plugins.js' +import { logger } from '../../helpers/logger.js' +import { CONFIG } from '../../initializers/config.js' +import { PLUGIN_GLOBAL_CSS_PATH } from '../../initializers/constants.js' +import { PluginModel } from '../../models/server/plugin.js' +import { + PluginLibrary, + RegisterServerAuthExternalOptions, + RegisterServerAuthPassOptions, + RegisterServerOptions +} from '../../types/plugins/index.js' +import { ClientHtml } from '../client-html.js' +import { RegisterHelpers } from './register-helpers.js' +import { installNpmPlugin, installNpmPluginFromDisk, rebuildNativePlugins, removeNpmPlugin } from './yarn.js' + +const require = createRequire(import.meta.url) + +export interface RegisteredPlugin { + npmName: string + name: string + version: string + description: string + peertubeEngine: string + + type: PluginType_Type + + path: string + + staticDirs: { [name: string]: string } + clientScripts: { [name: string]: ClientScriptJSON } + + css: string[] + + // Only if this is a plugin + registerHelpers?: RegisterHelpers + unregister?: Function +} + +export interface HookInformationValue { + npmName: string + pluginName: string + handler: Function + priority: number +} + +type PluginLocalesTranslations = { + [locale: string]: PluginTranslation +} + +export class PluginManager implements ServerHook { + + private static instance: PluginManager + + private registeredPlugins: { [name: string]: RegisteredPlugin } = {} + + private hooks: { [name: string]: HookInformationValue[] } = {} + private translations: PluginLocalesTranslations = {} + + private server: Server + + private constructor () { + } + + init (server: Server) { + this.server = server + } + + registerWebSocketRouter () { + this.server.on('upgrade', (request, socket, head) => { + // Check if it's a plugin websocket connection + // No need to destroy the stream when we abort the request + // Other handlers in PeerTube will catch this upgrade event too (socket.io, tracker etc) + + const url = request.url + + const matched = url.match(`/plugins/([^/]+)/([^/]+/)?ws(/.*)`) + if (!matched) return + + const npmName = PluginModel.buildNpmName(matched[1], PluginType.PLUGIN) + const subRoute = matched[3] + + const result = this.getRegisteredPluginOrTheme(npmName) + if (!result) return + + const routes = result.registerHelpers.getWebSocketRoutes() + + const wss = routes.find(r => r.route.startsWith(subRoute)) + if (!wss) return + + try { + wss.handler(request, socket, head) + } catch (err) { + logger.error('Exception in plugin handler ' + npmName, { err }) + } + }) + } + + // ###################### Getters ###################### + + isRegistered (npmName: string) { + return !!this.getRegisteredPluginOrTheme(npmName) + } + + getRegisteredPluginOrTheme (npmName: string) { + return this.registeredPlugins[npmName] + } + + getRegisteredPluginByShortName (name: string) { + const npmName = PluginModel.buildNpmName(name, PluginType.PLUGIN) + const registered = this.getRegisteredPluginOrTheme(npmName) + + if (!registered || registered.type !== PluginType.PLUGIN) return undefined + + return registered + } + + getRegisteredThemeByShortName (name: string) { + const npmName = PluginModel.buildNpmName(name, PluginType.THEME) + const registered = this.getRegisteredPluginOrTheme(npmName) + + if (!registered || registered.type !== PluginType.THEME) return undefined + + return registered + } + + getRegisteredPlugins () { + return this.getRegisteredPluginsOrThemes(PluginType.PLUGIN) + } + + getRegisteredThemes () { + return this.getRegisteredPluginsOrThemes(PluginType.THEME) + } + + getIdAndPassAuths () { + return this.getRegisteredPlugins() + .map(p => ({ + npmName: p.npmName, + name: p.name, + version: p.version, + idAndPassAuths: p.registerHelpers.getIdAndPassAuths() + })) + .filter(v => v.idAndPassAuths.length !== 0) + } + + getExternalAuths () { + return this.getRegisteredPlugins() + .map(p => ({ + npmName: p.npmName, + name: p.name, + version: p.version, + externalAuths: p.registerHelpers.getExternalAuths() + })) + .filter(v => v.externalAuths.length !== 0) + } + + getRegisteredSettings (npmName: string) { + const result = this.getRegisteredPluginOrTheme(npmName) + if (!result || result.type !== PluginType.PLUGIN) return [] + + return result.registerHelpers.getSettings() + } + + getRouter (npmName: string) { + const result = this.getRegisteredPluginOrTheme(npmName) + if (!result || result.type !== PluginType.PLUGIN) return null + + return result.registerHelpers.getRouter() + } + + getTranslations (locale: string) { + return this.translations[locale] || {} + } + + async isTokenValid (token: MOAuthTokenUser, type: 'access' | 'refresh') { + const auth = this.getAuth(token.User.pluginAuth, token.authName) + if (!auth) return true + + if (auth.hookTokenValidity) { + try { + const { valid } = await auth.hookTokenValidity({ token, type }) + + if (valid === false) { + logger.info('Rejecting %s token validity from auth %s of plugin %s', type, token.authName, token.User.pluginAuth) + } + + return valid + } catch (err) { + logger.warn('Cannot run check token validity from auth %s of plugin %s.', token.authName, token.User.pluginAuth, { err }) + return true + } + } + + return true + } + + // ###################### External events ###################### + + async onLogout (npmName: string, authName: string, user: MUser, req: express.Request) { + const auth = this.getAuth(npmName, authName) + + if (auth?.onLogout) { + logger.info('Running onLogout function from auth %s of plugin %s', authName, npmName) + + try { + // Force await, in case or onLogout returns a promise + const result = await auth.onLogout(user, req) + + return typeof result === 'string' + ? result + : undefined + } catch (err) { + logger.warn('Cannot run onLogout function from auth %s of plugin %s.', authName, npmName, { err }) + } + } + + return undefined + } + + async onSettingsChanged (name: string, settings: any) { + const registered = this.getRegisteredPluginByShortName(name) + if (!registered) { + logger.error('Cannot find plugin %s to call on settings changed.', name) + } + + for (const cb of registered.registerHelpers.getOnSettingsChangedCallbacks()) { + try { + await cb(settings) + } catch (err) { + logger.error('Cannot run on settings changed callback for %s.', registered.npmName, { err }) + } + } + } + + // ###################### Hooks ###################### + + async runHook (hookName: ServerHookName, result?: T, params?: any): Promise { + if (!this.hooks[hookName]) return Promise.resolve(result) + + const hookType = getHookType(hookName) + + for (const hook of this.hooks[hookName]) { + logger.debug('Running hook %s of plugin %s.', hookName, hook.npmName) + + result = await internalRunHook({ + handler: hook.handler, + hookType, + result, + params, + onError: err => { logger.error('Cannot run hook %s of plugin %s.', hookName, hook.pluginName, { err }) } + }) + } + + return result + } + + // ###################### Registration ###################### + + async registerPluginsAndThemes () { + await this.resetCSSGlobalFile() + + const plugins = await PluginModel.listEnabledPluginsAndThemes() + + for (const plugin of plugins) { + try { + await this.registerPluginOrTheme(plugin) + } catch (err) { + // Try to unregister the plugin + try { + await this.unregister(PluginModel.buildNpmName(plugin.name, plugin.type)) + } catch { + // we don't care if we cannot unregister it + } + + logger.error('Cannot register plugin %s, skipping.', plugin.name, { err }) + } + } + + this.sortHooksByPriority() + } + + // Don't need the plugin type since themes cannot register server code + async unregister (npmName: string) { + logger.info('Unregister plugin %s.', npmName) + + const plugin = this.getRegisteredPluginOrTheme(npmName) + + if (!plugin) { + throw new Error(`Unknown plugin ${npmName} to unregister`) + } + + delete this.registeredPlugins[plugin.npmName] + + this.deleteTranslations(plugin.npmName) + + if (plugin.type === PluginType.PLUGIN) { + await plugin.unregister() + + // Remove hooks of this plugin + for (const key of Object.keys(this.hooks)) { + this.hooks[key] = this.hooks[key].filter(h => h.npmName !== npmName) + } + + const store = plugin.registerHelpers + store.reinitVideoConstants(plugin.npmName) + store.reinitTranscodingProfilesAndEncoders(plugin.npmName) + + logger.info('Regenerating registered plugin CSS to global file.') + await this.regeneratePluginGlobalCSS() + } + + ClientHtml.invalidCache() + } + + // ###################### Installation ###################### + + async install (options: { + toInstall: string + version?: string + fromDisk?: boolean // default false + register?: boolean // default true + }) { + const { toInstall, version, fromDisk = false, register = true } = options + + let plugin: PluginModel + let npmName: string + + logger.info('Installing plugin %s.', toInstall) + + try { + fromDisk + ? await installNpmPluginFromDisk(toInstall) + : await installNpmPlugin(toInstall, version) + + npmName = fromDisk ? basename(toInstall) : toInstall + const pluginType = PluginModel.getTypeFromNpmName(npmName) + const pluginName = PluginModel.normalizePluginName(npmName) + + const packageJSON = await this.getPackageJSON(pluginName, pluginType) + + this.sanitizeAndCheckPackageJSONOrThrow(packageJSON, pluginType); + + [ plugin ] = await PluginModel.upsert({ + name: pluginName, + description: packageJSON.description, + homepage: packageJSON.homepage, + type: pluginType, + version: packageJSON.version, + enabled: true, + uninstalled: false, + peertubeEngine: packageJSON.engine.peertube + }, { returning: true }) + + logger.info('Successful installation of plugin %s.', toInstall) + + if (register) { + await this.registerPluginOrTheme(plugin) + } + } catch (rootErr) { + logger.error('Cannot install plugin %s, removing it...', toInstall, { err: rootErr }) + + if (npmName) { + try { + await this.uninstall({ npmName }) + } catch (err) { + logger.error('Cannot uninstall plugin %s after failed installation.', toInstall, { err }) + + try { + await removeNpmPlugin(npmName) + } catch (err) { + logger.error('Cannot remove plugin %s after failed installation.', toInstall, { err }) + } + } + } + + throw rootErr + } + + return plugin + } + + async update (toUpdate: string, fromDisk = false) { + const npmName = fromDisk ? basename(toUpdate) : toUpdate + + logger.info('Updating plugin %s.', npmName) + + // Use the latest version from DB, to not upgrade to a version that does not support our PeerTube version + let version: string + if (!fromDisk) { + const plugin = await PluginModel.loadByNpmName(toUpdate) + version = plugin.latestVersion + } + + // Unregister old hooks + await this.unregister(npmName) + + return this.install({ toInstall: toUpdate, version, fromDisk }) + } + + async uninstall (options: { + npmName: string + unregister?: boolean // default true + }) { + const { npmName, unregister = true } = options + + logger.info('Uninstalling plugin %s.', npmName) + + if (unregister) { + try { + await this.unregister(npmName) + } catch (err) { + logger.warn('Cannot unregister plugin %s.', npmName, { err }) + } + } + + const plugin = await PluginModel.loadByNpmName(npmName) + if (!plugin || plugin.uninstalled === true) { + logger.error('Cannot uninstall plugin %s: it does not exist or is already uninstalled.', npmName) + return + } + + plugin.enabled = false + plugin.uninstalled = true + + await plugin.save() + + await removeNpmPlugin(npmName) + + logger.info('Plugin %s uninstalled.', npmName) + } + + async rebuildNativePluginsIfNeeded () { + if (!await ApplicationModel.nodeABIChanged()) return + + return rebuildNativePlugins() + } + + // ###################### Private register ###################### + + private async registerPluginOrTheme (plugin: PluginModel) { + const npmName = PluginModel.buildNpmName(plugin.name, plugin.type) + + logger.info('Registering plugin or theme %s.', npmName) + + const packageJSON = await this.getPackageJSON(plugin.name, plugin.type) + const pluginPath = this.getPluginPath(plugin.name, plugin.type) + + this.sanitizeAndCheckPackageJSONOrThrow(packageJSON, plugin.type) + + let library: PluginLibrary + let registerHelpers: RegisterHelpers + if (plugin.type === PluginType.PLUGIN) { + const result = await this.registerPlugin(plugin, pluginPath, packageJSON) + library = result.library + registerHelpers = result.registerStore + } + + const clientScripts: { [id: string]: ClientScriptJSON } = {} + for (const c of packageJSON.clientScripts) { + clientScripts[c.script] = c + } + + this.registeredPlugins[npmName] = { + npmName, + name: plugin.name, + type: plugin.type, + version: plugin.version, + description: plugin.description, + peertubeEngine: plugin.peertubeEngine, + path: pluginPath, + staticDirs: packageJSON.staticDirs, + clientScripts, + css: packageJSON.css, + registerHelpers: registerHelpers || undefined, + unregister: library ? library.unregister : undefined + } + + await this.addTranslations(plugin, npmName, packageJSON.translations) + + ClientHtml.invalidCache() + } + + private async registerPlugin (plugin: PluginModel, pluginPath: string, packageJSON: PluginPackageJSON) { + const npmName = PluginModel.buildNpmName(plugin.name, plugin.type) + + // Delete cache if needed + const modulePath = join(pluginPath, packageJSON.library) + decachePlugin(require, modulePath) + const library: PluginLibrary = require(modulePath) + + if (!isLibraryCodeValid(library)) { + throw new Error('Library code is not valid (miss register or unregister function)') + } + + const { registerOptions, registerStore } = this.getRegisterHelpers(npmName, plugin) + + await ensureDir(registerOptions.peertubeHelpers.plugin.getDataDirectoryPath()) + + await library.register(registerOptions) + + logger.info('Add plugin %s CSS to global file.', npmName) + + await this.addCSSToGlobalFile(pluginPath, packageJSON.css) + + return { library, registerStore } + } + + // ###################### Translations ###################### + + private async addTranslations (plugin: PluginModel, npmName: string, translationPaths: PluginTranslationPathsJSON) { + for (const locale of Object.keys(translationPaths)) { + const path = translationPaths[locale] + const json = await readJSON(join(this.getPluginPath(plugin.name, plugin.type), path)) + + const completeLocale = getCompleteLocale(locale) + + if (!this.translations[completeLocale]) this.translations[completeLocale] = {} + this.translations[completeLocale][npmName] = json + + logger.info('Added locale %s of plugin %s.', completeLocale, npmName) + } + } + + private deleteTranslations (npmName: string) { + for (const locale of Object.keys(this.translations)) { + delete this.translations[locale][npmName] + + logger.info('Deleted locale %s of plugin %s.', locale, npmName) + } + } + + // ###################### CSS ###################### + + private resetCSSGlobalFile () { + return outputFile(PLUGIN_GLOBAL_CSS_PATH, '') + } + + private async addCSSToGlobalFile (pluginPath: string, cssRelativePaths: string[]) { + for (const cssPath of cssRelativePaths) { + await this.concatFiles(join(pluginPath, cssPath), PLUGIN_GLOBAL_CSS_PATH) + } + } + + private concatFiles (input: string, output: string) { + return new Promise((res, rej) => { + const inputStream = createReadStream(input) + const outputStream = createWriteStream(output, { flags: 'a' }) + + inputStream.pipe(outputStream) + + inputStream.on('end', () => res()) + inputStream.on('error', err => rej(err)) + }) + } + + private async regeneratePluginGlobalCSS () { + await this.resetCSSGlobalFile() + + for (const plugin of this.getRegisteredPlugins()) { + await this.addCSSToGlobalFile(plugin.path, plugin.css) + } + } + + // ###################### Utils ###################### + + private sortHooksByPriority () { + for (const hookName of Object.keys(this.hooks)) { + this.hooks[hookName].sort((a, b) => { + return b.priority - a.priority + }) + } + } + + private getPackageJSON (pluginName: string, pluginType: PluginType_Type) { + const pluginPath = join(this.getPluginPath(pluginName, pluginType), 'package.json') + + return readJSON(pluginPath) as Promise + } + + private getPluginPath (pluginName: string, pluginType: PluginType_Type) { + const npmName = PluginModel.buildNpmName(pluginName, pluginType) + + return join(CONFIG.STORAGE.PLUGINS_DIR, 'node_modules', npmName) + } + + private getAuth (npmName: string, authName: string) { + const plugin = this.getRegisteredPluginOrTheme(npmName) + if (!plugin || plugin.type !== PluginType.PLUGIN) return null + + let auths: (RegisterServerAuthPassOptions | RegisterServerAuthExternalOptions)[] = plugin.registerHelpers.getIdAndPassAuths() + auths = auths.concat(plugin.registerHelpers.getExternalAuths()) + + return auths.find(a => a.authName === authName) + } + + // ###################### Private getters ###################### + + private getRegisteredPluginsOrThemes (type: PluginType_Type) { + const plugins: RegisteredPlugin[] = [] + + for (const npmName of Object.keys(this.registeredPlugins)) { + const plugin = this.registeredPlugins[npmName] + if (plugin.type !== type) continue + + plugins.push(plugin) + } + + return plugins + } + + // ###################### Generate register helpers ###################### + + private getRegisterHelpers ( + npmName: string, + plugin: PluginModel + ): { registerStore: RegisterHelpers, registerOptions: RegisterServerOptions } { + const onHookAdded = (options: RegisterServerHookOptions) => { + if (!this.hooks[options.target]) this.hooks[options.target] = [] + + this.hooks[options.target].push({ + npmName, + pluginName: plugin.name, + handler: options.handler, + priority: options.priority || 0 + }) + } + + const registerHelpers = new RegisterHelpers(npmName, plugin, this.server, onHookAdded.bind(this)) + + return { + registerStore: registerHelpers, + registerOptions: registerHelpers.buildRegisterHelpers() + } + } + + private sanitizeAndCheckPackageJSONOrThrow (packageJSON: PluginPackageJSON, pluginType: PluginType_Type) { + if (!packageJSON.staticDirs) packageJSON.staticDirs = {} + if (!packageJSON.css) packageJSON.css = [] + if (!packageJSON.clientScripts) packageJSON.clientScripts = [] + if (!packageJSON.translations) packageJSON.translations = {} + + const { result: packageJSONValid, badFields } = isPackageJSONValid(packageJSON, pluginType) + if (!packageJSONValid) { + const formattedFields = badFields.map(f => `"${f}"`) + .join(', ') + + throw new Error(`PackageJSON is invalid (invalid fields: ${formattedFields}).`) + } + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/server/lib/plugins/register-helpers.ts b/server/server/lib/plugins/register-helpers.ts new file mode 100644 index 000000000..7c44bff88 --- /dev/null +++ b/server/server/lib/plugins/register-helpers.ts @@ -0,0 +1,340 @@ +import express from 'express' +import { Server } from 'http' +import { + EncoderOptionsBuilder, + PluginSettingsManager, + PluginStorageManager, + RegisterServerHookOptions, + RegisterServerSettingOptions, + serverHookObject, + SettingsChangeCallback, + VideoPlaylistPrivacyType, + VideoPrivacyType +} from '@peertube/peertube-models' +import { logger } from '@server/helpers/logger.js' +import { onExternalUserAuthenticated } from '@server/lib/auth/external-auth.js' +import { VideoConstantManagerFactory } from '@server/lib/plugins/video-constant-manager-factory.js' +import { PluginModel } from '@server/models/server/plugin.js' +import { + RegisterServerAuthExternalOptions, + RegisterServerAuthExternalResult, + RegisterServerAuthPassOptions, + RegisterServerExternalAuthenticatedResult, + RegisterServerOptions, + RegisterServerWebSocketRouteOptions +} from '@server/types/plugins/index.js' +import { VideoTranscodingProfilesManager } from '../transcoding/default-transcoding-profiles.js' +import { buildPluginHelpers } from './plugin-helpers-builder.js' + +export class RegisterHelpers { + private readonly transcodingProfiles: { + [ npmName: string ]: { + type: 'vod' | 'live' + encoder: string + profile: string + }[] + } = {} + + private readonly transcodingEncoders: { + [ npmName: string ]: { + type: 'vod' | 'live' + streamType: 'audio' | 'video' + encoder: string + priority: number + }[] + } = {} + + private readonly settings: RegisterServerSettingOptions[] = [] + + private idAndPassAuths: RegisterServerAuthPassOptions[] = [] + private externalAuths: RegisterServerAuthExternalOptions[] = [] + + private readonly onSettingsChangeCallbacks: SettingsChangeCallback[] = [] + + private readonly webSocketRoutes: RegisterServerWebSocketRouteOptions[] = [] + + private readonly router: express.Router + private readonly videoConstantManagerFactory: VideoConstantManagerFactory + + constructor ( + private readonly npmName: string, + private readonly plugin: PluginModel, + private readonly server: Server, + private readonly onHookAdded: (options: RegisterServerHookOptions) => void + ) { + this.router = express.Router() + this.videoConstantManagerFactory = new VideoConstantManagerFactory(this.npmName) + } + + buildRegisterHelpers (): RegisterServerOptions { + const registerHook = this.buildRegisterHook() + const registerSetting = this.buildRegisterSetting() + + const getRouter = this.buildGetRouter() + const registerWebSocketRoute = this.buildRegisterWebSocketRoute() + + const settingsManager = this.buildSettingsManager() + const storageManager = this.buildStorageManager() + + const videoLanguageManager = this.videoConstantManagerFactory.createVideoConstantManager('language') + + const videoLicenceManager = this.videoConstantManagerFactory.createVideoConstantManager('licence') + const videoCategoryManager = this.videoConstantManagerFactory.createVideoConstantManager('category') + + const videoPrivacyManager = this.videoConstantManagerFactory.createVideoConstantManager('privacy') + const playlistPrivacyManager = this.videoConstantManagerFactory.createVideoConstantManager('playlistPrivacy') + + const transcodingManager = this.buildTranscodingManager() + + const registerIdAndPassAuth = this.buildRegisterIdAndPassAuth() + const registerExternalAuth = this.buildRegisterExternalAuth() + const unregisterIdAndPassAuth = this.buildUnregisterIdAndPassAuth() + const unregisterExternalAuth = this.buildUnregisterExternalAuth() + + const peertubeHelpers = buildPluginHelpers(this.server, this.plugin, this.npmName) + + return { + registerHook, + registerSetting, + + getRouter, + registerWebSocketRoute, + + settingsManager, + storageManager, + + videoLanguageManager: { + ...videoLanguageManager, + /** @deprecated use `addConstant` instead **/ + addLanguage: videoLanguageManager.addConstant, + /** @deprecated use `deleteConstant` instead **/ + deleteLanguage: videoLanguageManager.deleteConstant + }, + videoCategoryManager: { + ...videoCategoryManager, + /** @deprecated use `addConstant` instead **/ + addCategory: videoCategoryManager.addConstant, + /** @deprecated use `deleteConstant` instead **/ + deleteCategory: videoCategoryManager.deleteConstant + }, + videoLicenceManager: { + ...videoLicenceManager, + /** @deprecated use `addConstant` instead **/ + addLicence: videoLicenceManager.addConstant, + /** @deprecated use `deleteConstant` instead **/ + deleteLicence: videoLicenceManager.deleteConstant + }, + + videoPrivacyManager: { + ...videoPrivacyManager, + /** @deprecated use `deleteConstant` instead **/ + deletePrivacy: videoPrivacyManager.deleteConstant + }, + playlistPrivacyManager: { + ...playlistPrivacyManager, + /** @deprecated use `deleteConstant` instead **/ + deletePlaylistPrivacy: playlistPrivacyManager.deleteConstant + }, + + transcodingManager, + + registerIdAndPassAuth, + registerExternalAuth, + unregisterIdAndPassAuth, + unregisterExternalAuth, + + peertubeHelpers + } + } + + reinitVideoConstants (npmName: string) { + this.videoConstantManagerFactory.resetVideoConstants(npmName) + } + + reinitTranscodingProfilesAndEncoders (npmName: string) { + const profiles = this.transcodingProfiles[npmName] + if (Array.isArray(profiles)) { + for (const profile of profiles) { + VideoTranscodingProfilesManager.Instance.removeProfile(profile) + } + } + + const encoders = this.transcodingEncoders[npmName] + if (Array.isArray(encoders)) { + for (const o of encoders) { + VideoTranscodingProfilesManager.Instance.removeEncoderPriority(o.type, o.streamType, o.encoder, o.priority) + } + } + } + + getSettings () { + return this.settings + } + + getRouter () { + return this.router + } + + getIdAndPassAuths () { + return this.idAndPassAuths + } + + getExternalAuths () { + return this.externalAuths + } + + getOnSettingsChangedCallbacks () { + return this.onSettingsChangeCallbacks + } + + getWebSocketRoutes () { + return this.webSocketRoutes + } + + private buildGetRouter () { + return () => this.router + } + + private buildRegisterWebSocketRoute () { + return (options: RegisterServerWebSocketRouteOptions) => { + this.webSocketRoutes.push(options) + } + } + + private buildRegisterSetting () { + return (options: RegisterServerSettingOptions) => { + this.settings.push(options) + } + } + + private buildRegisterHook () { + return (options: RegisterServerHookOptions) => { + if (serverHookObject[options.target] !== true) { + logger.warn('Unknown hook %s of plugin %s. Skipping.', options.target, this.npmName) + return + } + + return this.onHookAdded(options) + } + } + + private buildRegisterIdAndPassAuth () { + return (options: RegisterServerAuthPassOptions) => { + if (!options.authName || typeof options.getWeight !== 'function' || typeof options.login !== 'function') { + logger.error('Cannot register auth plugin %s: authName, getWeight or login are not valid.', this.npmName, { options }) + return + } + + this.idAndPassAuths.push(options) + } + } + + private buildRegisterExternalAuth () { + const self = this + + return (options: RegisterServerAuthExternalOptions) => { + if (!options.authName || typeof options.authDisplayName !== 'function' || typeof options.onAuthRequest !== 'function') { + logger.error('Cannot register auth plugin %s: authName, authDisplayName or onAuthRequest are not valid.', this.npmName, { options }) + return + } + + this.externalAuths.push(options) + + return { + userAuthenticated (result: RegisterServerExternalAuthenticatedResult): void { + onExternalUserAuthenticated({ + npmName: self.npmName, + authName: options.authName, + authResult: result + }).catch(err => { + logger.error('Cannot execute onExternalUserAuthenticated.', { npmName: self.npmName, authName: options.authName, err }) + }) + } + } as RegisterServerAuthExternalResult + } + } + + private buildUnregisterExternalAuth () { + return (authName: string) => { + this.externalAuths = this.externalAuths.filter(a => a.authName !== authName) + } + } + + private buildUnregisterIdAndPassAuth () { + return (authName: string) => { + this.idAndPassAuths = this.idAndPassAuths.filter(a => a.authName !== authName) + } + } + + private buildSettingsManager (): PluginSettingsManager { + return { + getSetting: (name: string) => PluginModel.getSetting(this.plugin.name, this.plugin.type, name, this.settings), + + getSettings: (names: string[]) => PluginModel.getSettings(this.plugin.name, this.plugin.type, names, this.settings), + + setSetting: (name: string, value: string) => PluginModel.setSetting(this.plugin.name, this.plugin.type, name, value), + + onSettingsChange: (cb: SettingsChangeCallback) => this.onSettingsChangeCallbacks.push(cb) + } + } + + private buildStorageManager (): PluginStorageManager { + return { + getData: (key: string) => PluginModel.getData(this.plugin.name, this.plugin.type, key), + + storeData: (key: string, data: any) => PluginModel.storeData(this.plugin.name, this.plugin.type, key, data) + } + } + + private buildTranscodingManager () { + const self = this + + function addProfile (type: 'live' | 'vod', encoder: string, profile: string, builder: EncoderOptionsBuilder) { + if (profile === 'default') { + logger.error('A plugin cannot add a default live transcoding profile') + return false + } + + VideoTranscodingProfilesManager.Instance.addProfile({ + type, + encoder, + profile, + builder + }) + + if (!self.transcodingProfiles[self.npmName]) self.transcodingProfiles[self.npmName] = [] + self.transcodingProfiles[self.npmName].push({ type, encoder, profile }) + + return true + } + + function addEncoderPriority (type: 'live' | 'vod', streamType: 'audio' | 'video', encoder: string, priority: number) { + VideoTranscodingProfilesManager.Instance.addEncoderPriority(type, streamType, encoder, priority) + + if (!self.transcodingEncoders[self.npmName]) self.transcodingEncoders[self.npmName] = [] + self.transcodingEncoders[self.npmName].push({ type, streamType, encoder, priority }) + } + + return { + addLiveProfile (encoder: string, profile: string, builder: EncoderOptionsBuilder) { + return addProfile('live', encoder, profile, builder) + }, + + addVODProfile (encoder: string, profile: string, builder: EncoderOptionsBuilder) { + return addProfile('vod', encoder, profile, builder) + }, + + addLiveEncoderPriority (streamType: 'audio' | 'video', encoder: string, priority: number) { + return addEncoderPriority('live', streamType, encoder, priority) + }, + + addVODEncoderPriority (streamType: 'audio' | 'video', encoder: string, priority: number) { + return addEncoderPriority('vod', streamType, encoder, priority) + }, + + removeAllProfilesAndEncoderPriorities () { + return self.reinitTranscodingProfilesAndEncoders(self.npmName) + } + } + } +} diff --git a/server/server/lib/plugins/theme-utils.ts b/server/server/lib/plugins/theme-utils.ts new file mode 100644 index 000000000..4a64a579a --- /dev/null +++ b/server/server/lib/plugins/theme-utils.ts @@ -0,0 +1,24 @@ +import { DEFAULT_THEME_NAME, DEFAULT_USER_THEME_NAME } from '../../initializers/constants.js' +import { PluginManager } from './plugin-manager.js' +import { CONFIG } from '../../initializers/config.js' + +function getThemeOrDefault (name: string, defaultTheme: string) { + if (isThemeRegistered(name)) return name + + // Fallback to admin default theme + if (name !== CONFIG.THEME.DEFAULT) return getThemeOrDefault(CONFIG.THEME.DEFAULT, DEFAULT_THEME_NAME) + + return defaultTheme +} + +function isThemeRegistered (name: string) { + if (name === DEFAULT_THEME_NAME || name === DEFAULT_USER_THEME_NAME) return true + + return !!PluginManager.Instance.getRegisteredThemes() + .find(r => r.name === name) +} + +export { + getThemeOrDefault, + isThemeRegistered +} diff --git a/server/server/lib/plugins/video-constant-manager-factory.ts b/server/server/lib/plugins/video-constant-manager-factory.ts new file mode 100644 index 000000000..04c30cf69 --- /dev/null +++ b/server/server/lib/plugins/video-constant-manager-factory.ts @@ -0,0 +1,139 @@ +import { ConstantManager } from '@peertube/peertube-models' +import { logger } from '@server/helpers/logger.js' +import { + VIDEO_CATEGORIES, + VIDEO_LANGUAGES, + VIDEO_LICENCES, + VIDEO_PLAYLIST_PRIVACIES, + VIDEO_PRIVACIES +} from '@server/initializers/constants.js' + +type AlterableVideoConstant = 'language' | 'licence' | 'category' | 'privacy' | 'playlistPrivacy' +type VideoConstant = Record + +type UpdatedVideoConstant = { + [name in AlterableVideoConstant]: { + [ npmName: string]: { + added: VideoConstant[] + deleted: VideoConstant[] + } + } +} + +const constantsHash: { [key in AlterableVideoConstant]: VideoConstant } = { + language: VIDEO_LANGUAGES, + licence: VIDEO_LICENCES, + category: VIDEO_CATEGORIES, + privacy: VIDEO_PRIVACIES, + playlistPrivacy: VIDEO_PLAYLIST_PRIVACIES +} + +export class VideoConstantManagerFactory { + private readonly updatedVideoConstants: UpdatedVideoConstant = { + playlistPrivacy: { }, + privacy: { }, + language: { }, + licence: { }, + category: { } + } + + constructor ( + private readonly npmName: string + ) {} + + public resetVideoConstants (npmName: string) { + const types: AlterableVideoConstant[] = [ 'language', 'licence', 'category', 'privacy', 'playlistPrivacy' ] + for (const type of types) { + this.resetConstants({ npmName, type }) + } + } + + private resetConstants (parameters: { npmName: string, type: AlterableVideoConstant }) { + const { npmName, type } = parameters + const updatedConstants = this.updatedVideoConstants[type][npmName] + + if (!updatedConstants) return + + for (const added of updatedConstants.added) { + delete constantsHash[type][added.key] + } + + for (const deleted of updatedConstants.deleted) { + constantsHash[type][deleted.key] = deleted.label + } + + delete this.updatedVideoConstants[type][npmName] + } + + public createVideoConstantManager(type: AlterableVideoConstant): ConstantManager { + const { npmName } = this + return { + addConstant: (key: K, label: string) => this.addConstant({ npmName, type, key, label }), + deleteConstant: (key: K) => this.deleteConstant({ npmName, type, key }), + getConstantValue: (key: K) => constantsHash[type][key], + getConstants: () => constantsHash[type] as Record, + resetConstants: () => this.resetConstants({ npmName, type }) + } + } + + private addConstant (parameters: { + npmName: string + type: AlterableVideoConstant + key: T + label: string + }) { + const { npmName, type, key, label } = parameters + const obj = constantsHash[type] + + if (obj[key]) { + logger.warn('Cannot add %s %s by plugin %s: key already exists.', type, npmName, key) + return false + } + + if (!this.updatedVideoConstants[type][npmName]) { + this.updatedVideoConstants[type][npmName] = { + added: [], + deleted: [] + } + } + + this.updatedVideoConstants[type][npmName].added.push({ key, label } as VideoConstant) + obj[key] = label + + return true + } + + private deleteConstant (parameters: { + npmName: string + type: AlterableVideoConstant + key: T + }) { + const { npmName, type, key } = parameters + const obj = constantsHash[type] + + if (!obj[key]) { + logger.warn('Cannot delete %s by plugin %s: key %s does not exist.', type, npmName, key) + return false + } + + if (!this.updatedVideoConstants[type][npmName]) { + this.updatedVideoConstants[type][npmName] = { + added: [], + deleted: [] + } + } + + const updatedConstants = this.updatedVideoConstants[type][npmName] + + const alreadyAdded = updatedConstants.added.find(a => a.key === key) + if (alreadyAdded) { + updatedConstants.added.filter(a => a.key !== key) + } else if (obj[key]) { + updatedConstants.deleted.push({ key, label: obj[key] } as VideoConstant) + } + + delete obj[key] + + return true + } +} diff --git a/server/server/lib/plugins/yarn.ts b/server/server/lib/plugins/yarn.ts new file mode 100644 index 000000000..470880f44 --- /dev/null +++ b/server/server/lib/plugins/yarn.ts @@ -0,0 +1,73 @@ +import { outputJSON, pathExists } from 'fs-extra/esm' +import { join } from 'path' +import { execShell } from '../../helpers/core-utils.js' +import { isNpmPluginNameValid, isPluginStableOrUnstableVersionValid } from '../../helpers/custom-validators/plugins.js' +import { logger } from '../../helpers/logger.js' +import { CONFIG } from '../../initializers/config.js' +import { getLatestPluginVersion } from './plugin-index.js' + +async function installNpmPlugin (npmName: string, versionArg?: string) { + // Security check + checkNpmPluginNameOrThrow(npmName) + if (versionArg) checkPluginVersionOrThrow(versionArg) + + const version = versionArg || await getLatestPluginVersion(npmName) + + let toInstall = npmName + if (version) toInstall += `@${version}` + + const { stdout } = await execYarn('add ' + toInstall) + + logger.debug('Added a yarn package.', { yarnStdout: stdout }) +} + +async function installNpmPluginFromDisk (path: string) { + await execYarn('add file:' + path) +} + +async function removeNpmPlugin (name: string) { + checkNpmPluginNameOrThrow(name) + + await execYarn('remove ' + name) +} + +async function rebuildNativePlugins () { + await execYarn('install --pure-lockfile') +} + +// ############################################################################ + +export { + installNpmPlugin, + installNpmPluginFromDisk, + rebuildNativePlugins, + removeNpmPlugin +} + +// ############################################################################ + +async function execYarn (command: string) { + try { + const pluginDirectory = CONFIG.STORAGE.PLUGINS_DIR + const pluginPackageJSON = join(pluginDirectory, 'package.json') + + // Create empty package.json file if needed + if (!await pathExists(pluginPackageJSON)) { + await outputJSON(pluginPackageJSON, {}) + } + + return execShell(`yarn ${command}`, { cwd: pluginDirectory }) + } catch (result) { + logger.error('Cannot exec yarn.', { command, err: result.err, stderr: result.stderr }) + + throw result.err + } +} + +function checkNpmPluginNameOrThrow (name: string) { + if (!isNpmPluginNameValid(name)) throw new Error('Invalid NPM plugin name to install') +} + +function checkPluginVersionOrThrow (name: string) { + if (!isPluginStableOrUnstableVersionValid(name)) throw new Error('Invalid NPM plugin version to install') +} diff --git a/server/server/lib/redis.ts b/server/server/lib/redis.ts new file mode 100644 index 000000000..7c87bf2e3 --- /dev/null +++ b/server/server/lib/redis.ts @@ -0,0 +1,465 @@ +import { Redis as IoRedis, RedisOptions } from 'ioredis' +import { exists } from '@server/helpers/custom-validators/misc.js' +import { sha256 } from '@peertube/peertube-node-utils' +import { logger } from '../helpers/logger.js' +import { generateRandomString } from '../helpers/utils.js' +import { CONFIG } from '../initializers/config.js' +import { + AP_CLEANER, + CONTACT_FORM_LIFETIME, + EMAIL_VERIFY_LIFETIME, + RESUMABLE_UPLOAD_SESSION_LIFETIME, + TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME, + USER_PASSWORD_CREATE_LIFETIME, + USER_PASSWORD_RESET_LIFETIME, + VIEW_LIFETIME, + WEBSERVER +} from '../initializers/constants.js' + +class Redis { + + private static instance: Redis + private initialized = false + private connected = false + private client: IoRedis + private prefix: string + + private constructor () { + } + + init () { + // Already initialized + if (this.initialized === true) return + this.initialized = true + + const redisMode = CONFIG.REDIS.SENTINEL.ENABLED ? 'sentinel' : 'standalone' + logger.info('Connecting to redis ' + redisMode + '...') + + this.client = new IoRedis(Redis.getRedisClientOptions('', { enableAutoPipelining: true })) + this.client.on('error', err => logger.error('Redis failed to connect', { err })) + this.client.on('connect', () => { + logger.info('Connected to redis.') + + this.connected = true + }) + this.client.on('reconnecting', (ms) => { + logger.error(`Reconnecting to redis in ${ms}.`) + }) + this.client.on('close', () => { + logger.error('Connection to redis has closed.') + this.connected = false + }) + + this.client.on('end', () => { + logger.error('Connection to redis has closed and no more reconnects will be done.') + }) + + this.prefix = 'redis-' + WEBSERVER.HOST + '-' + } + + static getRedisClientOptions (name?: string, options: RedisOptions = {}): RedisOptions { + const connectionName = [ 'PeerTube', name ].join('') + const connectTimeout = 20000 // Could be slow since node use sync call to compile PeerTube + + if (CONFIG.REDIS.SENTINEL.ENABLED) { + return { + connectionName, + connectTimeout, + enableTLSForSentinelMode: CONFIG.REDIS.SENTINEL.ENABLE_TLS, + sentinelPassword: CONFIG.REDIS.AUTH, + sentinels: CONFIG.REDIS.SENTINEL.SENTINELS, + name: CONFIG.REDIS.SENTINEL.MASTER_NAME, + ...options + } + } + + return { + connectionName, + connectTimeout, + password: CONFIG.REDIS.AUTH, + db: CONFIG.REDIS.DB, + host: CONFIG.REDIS.HOSTNAME, + port: CONFIG.REDIS.PORT, + path: CONFIG.REDIS.SOCKET, + showFriendlyErrorStack: true, + ...options + } + } + + getClient () { + return this.client + } + + getPrefix () { + return this.prefix + } + + isConnected () { + return this.connected + } + + /* ************ Forgot password ************ */ + + async setResetPasswordVerificationString (userId: number) { + const generatedString = await generateRandomString(32) + + await this.setValue(this.generateResetPasswordKey(userId), generatedString, USER_PASSWORD_RESET_LIFETIME) + + return generatedString + } + + async setCreatePasswordVerificationString (userId: number) { + const generatedString = await generateRandomString(32) + + await this.setValue(this.generateResetPasswordKey(userId), generatedString, USER_PASSWORD_CREATE_LIFETIME) + + return generatedString + } + + async removePasswordVerificationString (userId: number) { + return this.removeValue(this.generateResetPasswordKey(userId)) + } + + async getResetPasswordVerificationString (userId: number) { + return this.getValue(this.generateResetPasswordKey(userId)) + } + + /* ************ Two factor auth request ************ */ + + async setTwoFactorRequest (userId: number, otpSecret: string) { + const requestToken = await generateRandomString(32) + + await this.setValue(this.generateTwoFactorRequestKey(userId, requestToken), otpSecret, TWO_FACTOR_AUTH_REQUEST_TOKEN_LIFETIME) + + return requestToken + } + + async getTwoFactorRequestToken (userId: number, requestToken: string) { + return this.getValue(this.generateTwoFactorRequestKey(userId, requestToken)) + } + + /* ************ Email verification ************ */ + + async setUserVerifyEmailVerificationString (userId: number) { + const generatedString = await generateRandomString(32) + + await this.setValue(this.generateUserVerifyEmailKey(userId), generatedString, EMAIL_VERIFY_LIFETIME) + + return generatedString + } + + async getUserVerifyEmailLink (userId: number) { + return this.getValue(this.generateUserVerifyEmailKey(userId)) + } + + async setRegistrationVerifyEmailVerificationString (registrationId: number) { + const generatedString = await generateRandomString(32) + + await this.setValue(this.generateRegistrationVerifyEmailKey(registrationId), generatedString, EMAIL_VERIFY_LIFETIME) + + return generatedString + } + + async getRegistrationVerifyEmailLink (registrationId: number) { + return this.getValue(this.generateRegistrationVerifyEmailKey(registrationId)) + } + + /* ************ Contact form per IP ************ */ + + async setContactFormIp (ip: string) { + return this.setValue(this.generateContactFormKey(ip), '1', CONTACT_FORM_LIFETIME) + } + + async doesContactFormIpExist (ip: string) { + return this.exists(this.generateContactFormKey(ip)) + } + + /* ************ Views per IP ************ */ + + setIPVideoView (ip: string, videoUUID: string) { + return this.setValue(this.generateIPViewKey(ip, videoUUID), '1', VIEW_LIFETIME.VIEW) + } + + async doesVideoIPViewExist (ip: string, videoUUID: string) { + return this.exists(this.generateIPViewKey(ip, videoUUID)) + } + + /* ************ Video views stats ************ */ + + addVideoViewStats (videoId: number) { + const { videoKey, setKey } = this.generateVideoViewStatsKeys({ videoId }) + + return Promise.all([ + this.addToSet(setKey, videoId.toString()), + this.increment(videoKey) + ]) + } + + async getVideoViewsStats (videoId: number, hour: number) { + const { videoKey } = this.generateVideoViewStatsKeys({ videoId, hour }) + + const valueString = await this.getValue(videoKey) + const valueInt = parseInt(valueString, 10) + + if (isNaN(valueInt)) { + logger.error('Cannot get videos views stats of video %d in hour %d: views number is NaN (%s).', videoId, hour, valueString) + return undefined + } + + return valueInt + } + + async listVideosViewedForStats (hour: number) { + const { setKey } = this.generateVideoViewStatsKeys({ hour }) + + const stringIds = await this.getSet(setKey) + return stringIds.map(s => parseInt(s, 10)) + } + + deleteVideoViewsStats (videoId: number, hour: number) { + const { setKey, videoKey } = this.generateVideoViewStatsKeys({ videoId, hour }) + + return Promise.all([ + this.deleteFromSet(setKey, videoId.toString()), + this.deleteKey(videoKey) + ]) + } + + /* ************ Local video views buffer ************ */ + + addLocalVideoView (videoId: number) { + const { videoKey, setKey } = this.generateLocalVideoViewsKeys(videoId) + + return Promise.all([ + this.addToSet(setKey, videoId.toString()), + this.increment(videoKey) + ]) + } + + async getLocalVideoViews (videoId: number) { + const { videoKey } = this.generateLocalVideoViewsKeys(videoId) + + const valueString = await this.getValue(videoKey) + const valueInt = parseInt(valueString, 10) + + if (isNaN(valueInt)) { + logger.error('Cannot get videos views of video %d: views number is NaN (%s).', videoId, valueString) + return undefined + } + + return valueInt + } + + async listLocalVideosViewed () { + const { setKey } = this.generateLocalVideoViewsKeys() + + const stringIds = await this.getSet(setKey) + return stringIds.map(s => parseInt(s, 10)) + } + + deleteLocalVideoViews (videoId: number) { + const { setKey, videoKey } = this.generateLocalVideoViewsKeys(videoId) + + return Promise.all([ + this.deleteFromSet(setKey, videoId.toString()), + this.deleteKey(videoKey) + ]) + } + + /* ************ Video viewers stats ************ */ + + getLocalVideoViewer (options: { + key?: string + // Or + ip?: string + videoId?: number + }) { + if (options.key) return this.getObject(options.key) + + const { viewerKey } = this.generateLocalVideoViewerKeys(options.ip, options.videoId) + + return this.getObject(viewerKey) + } + + setLocalVideoViewer (ip: string, videoId: number, object: any) { + const { setKey, viewerKey } = this.generateLocalVideoViewerKeys(ip, videoId) + + return Promise.all([ + this.addToSet(setKey, viewerKey), + this.setObject(viewerKey, object) + ]) + } + + listLocalVideoViewerKeys () { + const { setKey } = this.generateLocalVideoViewerKeys() + + return this.getSet(setKey) + } + + deleteLocalVideoViewersKeys (key: string) { + const { setKey } = this.generateLocalVideoViewerKeys() + + return Promise.all([ + this.deleteFromSet(setKey, key), + this.deleteKey(key) + ]) + } + + /* ************ Resumable uploads final responses ************ */ + + setUploadSession (uploadId: string, response?: { video: { id: number, shortUUID: string, uuid: string } }) { + return this.setValue( + 'resumable-upload-' + uploadId, + response + ? JSON.stringify(response) + : '', + RESUMABLE_UPLOAD_SESSION_LIFETIME + ) + } + + doesUploadSessionExist (uploadId: string) { + return this.exists('resumable-upload-' + uploadId) + } + + async getUploadSession (uploadId: string) { + const value = await this.getValue('resumable-upload-' + uploadId) + + return value + ? JSON.parse(value) as { video: { id: number, shortUUID: string, uuid: string } } + : undefined + } + + deleteUploadSession (uploadId: string) { + return this.deleteKey('resumable-upload-' + uploadId) + } + + /* ************ AP resource unavailability ************ */ + + async addAPUnavailability (url: string) { + const key = this.generateAPUnavailabilityKey(url) + + const value = await this.increment(key) + await this.setExpiration(key, AP_CLEANER.PERIOD * 2) + + return value + } + + /* ************ Keys generation ************ */ + + private generateLocalVideoViewsKeys (videoId: number): { setKey: string, videoKey: string } + private generateLocalVideoViewsKeys (): { setKey: string } + private generateLocalVideoViewsKeys (videoId?: number) { + return { setKey: `local-video-views-buffer`, videoKey: `local-video-views-buffer-${videoId}` } + } + + private generateLocalVideoViewerKeys (ip: string, videoId: number): { setKey: string, viewerKey: string } + private generateLocalVideoViewerKeys (): { setKey: string } + private generateLocalVideoViewerKeys (ip?: string, videoId?: number) { + return { setKey: `local-video-viewer-stats-keys`, viewerKey: `local-video-viewer-stats-${ip}-${videoId}` } + } + + private generateVideoViewStatsKeys (options: { videoId?: number, hour?: number }) { + const hour = exists(options.hour) + ? options.hour + : new Date().getHours() + + return { setKey: `videos-view-h${hour}`, videoKey: `video-view-${options.videoId}-h${hour}` } + } + + private generateResetPasswordKey (userId: number) { + return 'reset-password-' + userId + } + + private generateTwoFactorRequestKey (userId: number, token: string) { + return 'two-factor-request-' + userId + '-' + token + } + + private generateUserVerifyEmailKey (userId: number) { + return 'verify-email-user-' + userId + } + + private generateRegistrationVerifyEmailKey (registrationId: number) { + return 'verify-email-registration-' + registrationId + } + + private generateIPViewKey (ip: string, videoUUID: string) { + return `views-${videoUUID}-${ip}` + } + + private generateContactFormKey (ip: string) { + return 'contact-form-' + ip + } + + private generateAPUnavailabilityKey (url: string) { + return 'ap-unavailability-' + sha256(url) + } + + /* ************ Redis helpers ************ */ + + private getValue (key: string) { + return this.client.get(this.prefix + key) + } + + private getSet (key: string) { + return this.client.smembers(this.prefix + key) + } + + private addToSet (key: string, value: string) { + return this.client.sadd(this.prefix + key, value) + } + + private deleteFromSet (key: string, value: string) { + return this.client.srem(this.prefix + key, value) + } + + private deleteKey (key: string) { + return this.client.del(this.prefix + key) + } + + private async getObject (key: string) { + const value = await this.getValue(key) + if (!value) return null + + return JSON.parse(value) + } + + private setObject (key: string, value: { [ id: string ]: number | string }, expirationMilliseconds?: number) { + return this.setValue(key, JSON.stringify(value), expirationMilliseconds) + } + + private async setValue (key: string, value: string, expirationMilliseconds?: number) { + const result = expirationMilliseconds !== undefined + ? await this.client.set(this.prefix + key, value, 'PX', expirationMilliseconds) + : await this.client.set(this.prefix + key, value) + + if (result !== 'OK') throw new Error('Redis set result is not OK.') + } + + private removeValue (key: string) { + return this.client.del(this.prefix + key) + } + + private increment (key: string) { + return this.client.incr(this.prefix + key) + } + + private async exists (key: string) { + const result = await this.client.exists(this.prefix + key) + + return result !== 0 + } + + private setExpiration (key: string, ms: number) { + return this.client.expire(this.prefix + key, ms / 1000) + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} + +// --------------------------------------------------------------------------- + +export { + Redis +} diff --git a/server/server/lib/redundancy.ts b/server/server/lib/redundancy.ts new file mode 100644 index 000000000..b470373ce --- /dev/null +++ b/server/server/lib/redundancy.ts @@ -0,0 +1,59 @@ +import { Transaction } from 'sequelize' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' +import { ActorFollowModel } from '@server/models/actor/actor-follow.js' +import { getServerActor } from '@server/models/application/application.js' +import { MActorSignature, MVideoRedundancyVideo } from '@server/types/models/index.js' +import { Activity } from '@peertube/peertube-models' +import { VideoRedundancyModel } from '../models/redundancy/video-redundancy.js' +import { sendUndoCacheFile } from './activitypub/send/index.js' + +const lTags = loggerTagsFactory('redundancy') + +async function removeVideoRedundancy (videoRedundancy: MVideoRedundancyVideo, t?: Transaction) { + const serverActor = await getServerActor() + + // Local cache, send undo to remote instances + if (videoRedundancy.actorId === serverActor.id) await sendUndoCacheFile(serverActor, videoRedundancy, t) + + await videoRedundancy.destroy({ transaction: t }) +} + +async function removeRedundanciesOfServer (serverId: number) { + const redundancies = await VideoRedundancyModel.listLocalOfServer(serverId) + + for (const redundancy of redundancies) { + await removeVideoRedundancy(redundancy) + } +} + +async function isRedundancyAccepted (activity: Activity, byActor: MActorSignature) { + const configAcceptFrom = CONFIG.REMOTE_REDUNDANCY.VIDEOS.ACCEPT_FROM + if (configAcceptFrom === 'nobody') { + logger.info('Do not accept remote redundancy %s due instance accept policy.', activity.id, lTags()) + return false + } + + if (configAcceptFrom === 'followings') { + const serverActor = await getServerActor() + const allowed = await ActorFollowModel.isFollowedBy(byActor.id, serverActor.id) + + if (allowed !== true) { + logger.info( + 'Do not accept remote redundancy %s because actor %s is not followed by our instance.', + activity.id, byActor.url, lTags() + ) + return false + } + } + + return true +} + +// --------------------------------------------------------------------------- + +export { + isRedundancyAccepted, + removeRedundanciesOfServer, + removeVideoRedundancy +} diff --git a/server/server/lib/runners/index.ts b/server/server/lib/runners/index.ts new file mode 100644 index 000000000..43eb7351e --- /dev/null +++ b/server/server/lib/runners/index.ts @@ -0,0 +1,3 @@ +export * from './job-handlers/index.js' +export * from './runner.js' +export * from './runner-urls.js' diff --git a/server/server/lib/runners/job-handlers/abstract-job-handler.ts b/server/server/lib/runners/job-handlers/abstract-job-handler.ts new file mode 100644 index 000000000..b7e9a0a9d --- /dev/null +++ b/server/server/lib/runners/job-handlers/abstract-job-handler.ts @@ -0,0 +1,270 @@ +import { pick } from '@peertube/peertube-core-utils' +import { + RunnerJobLiveRTMPHLSTranscodingPayload, + RunnerJobLiveRTMPHLSTranscodingPrivatePayload, + RunnerJobState, + RunnerJobStateType, + RunnerJobStudioTranscodingPayload, + RunnerJobSuccessPayload, + RunnerJobType, + RunnerJobUpdatePayload, + RunnerJobVODAudioMergeTranscodingPayload, + RunnerJobVODAudioMergeTranscodingPrivatePayload, + RunnerJobVODHLSTranscodingPayload, + RunnerJobVODHLSTranscodingPrivatePayload, + RunnerJobVODWebVideoTranscodingPayload, + RunnerJobVODWebVideoTranscodingPrivatePayload, + RunnerJobVideoStudioTranscodingPrivatePayload +} from '@peertube/peertube-models' +import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { RUNNER_JOBS } from '@server/initializers/constants.js' +import { sequelizeTypescript } from '@server/initializers/database.js' +import { PeerTubeSocket } from '@server/lib/peertube-socket.js' +import { RunnerJobModel } from '@server/models/runner/runner-job.js' +import { setAsUpdated } from '@server/models/shared/index.js' +import { MRunnerJob } from '@server/types/models/runners/index.js' +import throttle from 'lodash-es/throttle.js' + +type CreateRunnerJobArg = + { + type: Extract + payload: RunnerJobVODWebVideoTranscodingPayload + privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload + } | + { + type: Extract + payload: RunnerJobVODHLSTranscodingPayload + privatePayload: RunnerJobVODHLSTranscodingPrivatePayload + } | + { + type: Extract + payload: RunnerJobVODAudioMergeTranscodingPayload + privatePayload: RunnerJobVODAudioMergeTranscodingPrivatePayload + } | + { + type: Extract + payload: RunnerJobLiveRTMPHLSTranscodingPayload + privatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload + } | + { + type: Extract + payload: RunnerJobStudioTranscodingPayload + privatePayload: RunnerJobVideoStudioTranscodingPrivatePayload + } + +export abstract class AbstractJobHandler { + + protected readonly lTags = loggerTagsFactory('runner') + + static setJobAsUpdatedThrottled = throttle(setAsUpdated, 2000) + + // --------------------------------------------------------------------------- + + abstract create (options: C): Promise + + protected async createRunnerJob (options: CreateRunnerJobArg & { + jobUUID: string + priority: number + dependsOnRunnerJob?: MRunnerJob + }): Promise { + const { priority, dependsOnRunnerJob } = options + + logger.debug('Creating runner job', { options, ...this.lTags(options.type) }) + + const runnerJob = new RunnerJobModel({ + ...pick(options, [ 'type', 'payload', 'privatePayload' ]), + + uuid: options.jobUUID, + + state: dependsOnRunnerJob + ? RunnerJobState.WAITING_FOR_PARENT_JOB + : RunnerJobState.PENDING, + + dependsOnRunnerJobId: dependsOnRunnerJob?.id, + + priority + }) + + const job = await sequelizeTypescript.transaction(async transaction => { + return runnerJob.save({ transaction }) + }) + + if (runnerJob.state === RunnerJobState.PENDING) { + PeerTubeSocket.Instance.sendAvailableJobsPingToRunners() + } + + return job + } + + // --------------------------------------------------------------------------- + + protected abstract specificUpdate (options: { + runnerJob: MRunnerJob + updatePayload?: U + }): Promise | void + + async update (options: { + runnerJob: MRunnerJob + progress?: number + updatePayload?: U + }) { + const { runnerJob, progress } = options + + await this.specificUpdate(options) + + if (progress) runnerJob.progress = progress + + if (!runnerJob.changed()) { + try { + await AbstractJobHandler.setJobAsUpdatedThrottled({ sequelize: sequelizeTypescript, table: 'runnerJob', id: runnerJob.id }) + } catch (err) { + logger.warn('Cannot set remote job as updated', { err, ...this.lTags(runnerJob.id, runnerJob.type) }) + } + + return + } + + await saveInTransactionWithRetries(runnerJob) + } + + // --------------------------------------------------------------------------- + + async complete (options: { + runnerJob: MRunnerJob + resultPayload: S + }) { + const { runnerJob } = options + + runnerJob.state = RunnerJobState.COMPLETING + await saveInTransactionWithRetries(runnerJob) + + try { + await this.specificComplete(options) + + runnerJob.state = RunnerJobState.COMPLETED + } catch (err) { + logger.error('Cannot complete runner job', { err, ...this.lTags(runnerJob.id, runnerJob.type) }) + + runnerJob.state = RunnerJobState.ERRORED + runnerJob.error = err.message + } + + runnerJob.progress = null + runnerJob.finishedAt = new Date() + + await saveInTransactionWithRetries(runnerJob) + + const [ affectedCount ] = await RunnerJobModel.updateDependantJobsOf(runnerJob) + + if (affectedCount !== 0) PeerTubeSocket.Instance.sendAvailableJobsPingToRunners() + } + + protected abstract specificComplete (options: { + runnerJob: MRunnerJob + resultPayload: S + }): Promise | void + + // --------------------------------------------------------------------------- + + async cancel (options: { + runnerJob: MRunnerJob + fromParent?: boolean + }) { + const { runnerJob, fromParent } = options + + await this.specificCancel(options) + + const cancelState = fromParent + ? RunnerJobState.PARENT_CANCELLED + : RunnerJobState.CANCELLED + + runnerJob.setToErrorOrCancel(cancelState) + + await saveInTransactionWithRetries(runnerJob) + + const children = await RunnerJobModel.listChildrenOf(runnerJob) + for (const child of children) { + logger.info(`Cancelling child job ${child.uuid} of ${runnerJob.uuid} because of parent cancel`, this.lTags(child.uuid)) + + await this.cancel({ runnerJob: child, fromParent: true }) + } + } + + protected abstract specificCancel (options: { + runnerJob: MRunnerJob + }): Promise | void + + // --------------------------------------------------------------------------- + + protected abstract isAbortSupported (): boolean + + async abort (options: { + runnerJob: MRunnerJob + }) { + const { runnerJob } = options + + if (this.isAbortSupported() !== true) { + return this.error({ runnerJob, message: 'Job has been aborted but it is not supported by this job type' }) + } + + await this.specificAbort(options) + + runnerJob.resetToPending() + + await saveInTransactionWithRetries(runnerJob) + } + + protected setAbortState (runnerJob: MRunnerJob) { + runnerJob.resetToPending() + } + + protected abstract specificAbort (options: { + runnerJob: MRunnerJob + }): Promise | void + + // --------------------------------------------------------------------------- + + async error (options: { + runnerJob: MRunnerJob + message: string + fromParent?: boolean + }) { + const { runnerJob, message, fromParent } = options + + const errorState = fromParent + ? RunnerJobState.PARENT_ERRORED + : RunnerJobState.ERRORED + + const nextState = errorState === RunnerJobState.ERRORED && this.isAbortSupported() && runnerJob.failures < RUNNER_JOBS.MAX_FAILURES + ? RunnerJobState.PENDING + : errorState + + await this.specificError({ ...options, nextState }) + + if (nextState === errorState) { + runnerJob.setToErrorOrCancel(nextState) + runnerJob.error = message + } else { + runnerJob.resetToPending() + } + + await saveInTransactionWithRetries(runnerJob) + + if (runnerJob.state === errorState) { + const children = await RunnerJobModel.listChildrenOf(runnerJob) + + for (const child of children) { + logger.info(`Erroring child job ${child.uuid} of ${runnerJob.uuid} because of parent error`, this.lTags(child.uuid)) + + await this.error({ runnerJob: child, message: 'Parent error', fromParent: true }) + } + } + } + + protected abstract specificError (options: { + runnerJob: MRunnerJob + message: string + nextState: RunnerJobStateType + }): Promise | void +} diff --git a/server/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts b/server/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts new file mode 100644 index 000000000..392ff9aec --- /dev/null +++ b/server/server/lib/runners/job-handlers/abstract-vod-transcoding-job-handler.ts @@ -0,0 +1,72 @@ + +import { + RunnerJobState, + RunnerJobStateType, + RunnerJobSuccessPayload, + RunnerJobUpdatePayload, + RunnerJobVODPrivatePayload +} from '@peertube/peertube-models' +import { retryTransactionWrapper } from '@server/helpers/database-utils.js' +import { logger } from '@server/helpers/logger.js' +import { moveToFailedTranscodingState, moveToNextState } from '@server/lib/video-state.js' +import { VideoJobInfoModel } from '@server/models/video/video-job-info.js' +import { MRunnerJob } from '@server/types/models/runners/index.js' +import { AbstractJobHandler } from './abstract-job-handler.js' +import { loadTranscodingRunnerVideo } from './shared/index.js' + +// eslint-disable-next-line max-len +export abstract class AbstractVODTranscodingJobHandler extends AbstractJobHandler { + + protected isAbortSupported () { + return true + } + + protected specificUpdate (_options: { + runnerJob: MRunnerJob + }) { + // empty + } + + protected specificAbort (_options: { + runnerJob: MRunnerJob + }) { + // empty + } + + protected async specificError (options: { + runnerJob: MRunnerJob + nextState: RunnerJobStateType + }) { + if (options.nextState !== RunnerJobState.ERRORED) return + + const video = await loadTranscodingRunnerVideo(options.runnerJob, this.lTags) + if (!video) return + + await moveToFailedTranscodingState(video) + + await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') + } + + protected async specificCancel (options: { + runnerJob: MRunnerJob + }) { + const { runnerJob } = options + + const video = await loadTranscodingRunnerVideo(options.runnerJob, this.lTags) + if (!video) return + + const pending = await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') + + logger.debug(`Pending transcode decreased to ${pending} after cancel`, this.lTags(video.uuid)) + + if (pending === 0) { + logger.info( + `All transcoding jobs of ${video.uuid} have been processed or canceled, moving it to its next state`, + this.lTags(video.uuid) + ) + + const privatePayload = runnerJob.privatePayload as RunnerJobVODPrivatePayload + await retryTransactionWrapper(moveToNextState, { video, isNewVideo: privatePayload.isNewVideo }) + } + } +} diff --git a/server/server/lib/runners/job-handlers/index.ts b/server/server/lib/runners/job-handlers/index.ts new file mode 100644 index 000000000..c3205fae3 --- /dev/null +++ b/server/server/lib/runners/job-handlers/index.ts @@ -0,0 +1,7 @@ +export * from './abstract-job-handler.js' +export * from './live-rtmp-hls-transcoding-job-handler.js' +export * from './runner-job-handlers.js' +export * from './video-studio-transcoding-job-handler.js' +export * from './vod-audio-merge-transcoding-job-handler.js' +export * from './vod-hls-transcoding-job-handler.js' +export * from './vod-web-video-transcoding-job-handler.js' diff --git a/server/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts b/server/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts new file mode 100644 index 000000000..695c64c8e --- /dev/null +++ b/server/server/lib/runners/job-handlers/live-rtmp-hls-transcoding-job-handler.ts @@ -0,0 +1,173 @@ +import { + LiveRTMPHLSTranscodingSuccess, + LiveRTMPHLSTranscodingUpdatePayload, + LiveVideoError, + RunnerJobLiveRTMPHLSTranscodingPayload, + RunnerJobLiveRTMPHLSTranscodingPrivatePayload, + RunnerJobStateType +} from '@peertube/peertube-models' +import { buildUUID } from '@peertube/peertube-node-utils' +import { logger } from '@server/helpers/logger.js' +import { JOB_PRIORITY } from '@server/initializers/constants.js' +import { LiveManager } from '@server/lib/live/index.js' +import { MStreamingPlaylist, MVideo } from '@server/types/models/index.js' +import { MRunnerJob } from '@server/types/models/runners/index.js' +import { move, remove } from 'fs-extra/esm' +import { join } from 'path' +import { AbstractJobHandler } from './abstract-job-handler.js' + +type CreateOptions = { + video: MVideo + playlist: MStreamingPlaylist + + sessionId: string + rtmpUrl: string + + toTranscode: { + resolution: number + fps: number + }[] + + segmentListSize: number + segmentDuration: number + + outputDirectory: string +} + +// eslint-disable-next-line max-len +export class LiveRTMPHLSTranscodingJobHandler extends AbstractJobHandler { + + async create (options: CreateOptions) { + const { video, rtmpUrl, toTranscode, playlist, segmentDuration, segmentListSize, outputDirectory, sessionId } = options + + const jobUUID = buildUUID() + const payload: RunnerJobLiveRTMPHLSTranscodingPayload = { + input: { + rtmpUrl + }, + output: { + toTranscode, + segmentListSize, + segmentDuration + } + } + + const privatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload = { + videoUUID: video.uuid, + masterPlaylistName: playlist.playlistFilename, + sessionId, + outputDirectory + } + + const job = await this.createRunnerJob({ + type: 'live-rtmp-hls-transcoding', + jobUUID, + payload, + privatePayload, + priority: JOB_PRIORITY.TRANSCODING + }) + + return job + } + + // --------------------------------------------------------------------------- + + protected async specificUpdate (options: { + runnerJob: MRunnerJob + updatePayload: LiveRTMPHLSTranscodingUpdatePayload + }) { + const { runnerJob, updatePayload } = options + + const privatePayload = runnerJob.privatePayload as RunnerJobLiveRTMPHLSTranscodingPrivatePayload + const outputDirectory = privatePayload.outputDirectory + const videoUUID = privatePayload.videoUUID + + // Always process the chunk first before moving m3u8 that references this chunk + if (updatePayload.type === 'add-chunk') { + await move( + updatePayload.videoChunkFile as string, + join(outputDirectory, updatePayload.videoChunkFilename), + { overwrite: true } + ) + } else if (updatePayload.type === 'remove-chunk') { + await remove(join(outputDirectory, updatePayload.videoChunkFilename)) + } + + if (updatePayload.resolutionPlaylistFile && updatePayload.resolutionPlaylistFilename) { + await move( + updatePayload.resolutionPlaylistFile as string, + join(outputDirectory, updatePayload.resolutionPlaylistFilename), + { overwrite: true } + ) + } + + if (updatePayload.masterPlaylistFile) { + await move(updatePayload.masterPlaylistFile as string, join(outputDirectory, privatePayload.masterPlaylistName), { overwrite: true }) + } + + logger.debug( + 'Runner live RTMP to HLS job %s for %s updated.', + runnerJob.uuid, videoUUID, { updatePayload, ...this.lTags(videoUUID, runnerJob.uuid) } + ) + } + + // --------------------------------------------------------------------------- + + protected specificComplete (options: { + runnerJob: MRunnerJob + }) { + return this.stopLive({ + runnerJob: options.runnerJob, + type: 'ended' + }) + } + + // --------------------------------------------------------------------------- + + protected isAbortSupported () { + return false + } + + protected specificAbort () { + throw new Error('Not implemented') + } + + protected specificError (options: { + runnerJob: MRunnerJob + nextState: RunnerJobStateType + }) { + return this.stopLive({ + runnerJob: options.runnerJob, + type: 'errored' + }) + } + + protected specificCancel (options: { + runnerJob: MRunnerJob + }) { + return this.stopLive({ + runnerJob: options.runnerJob, + type: 'cancelled' + }) + } + + private stopLive (options: { + runnerJob: MRunnerJob + type: 'ended' | 'errored' | 'cancelled' + }) { + const { runnerJob, type } = options + + const privatePayload = runnerJob.privatePayload as RunnerJobLiveRTMPHLSTranscodingPrivatePayload + const videoUUID = privatePayload.videoUUID + + const errorType = { + ended: null, + errored: LiveVideoError.RUNNER_JOB_ERROR, + cancelled: LiveVideoError.RUNNER_JOB_CANCEL + } + + LiveManager.Instance.stopSessionOf(privatePayload.videoUUID, errorType[type]) + + logger.info('Runner live RTMP to HLS job %s for video %s %s.', runnerJob.uuid, videoUUID, type, this.lTags(runnerJob.uuid, videoUUID)) + } +} diff --git a/server/server/lib/runners/job-handlers/runner-job-handlers.ts b/server/server/lib/runners/job-handlers/runner-job-handlers.ts new file mode 100644 index 000000000..f8a672e64 --- /dev/null +++ b/server/server/lib/runners/job-handlers/runner-job-handlers.ts @@ -0,0 +1,20 @@ +import { MRunnerJob } from '@server/types/models/runners/index.js' +import { RunnerJobSuccessPayload, RunnerJobType, RunnerJobUpdatePayload } from '@peertube/peertube-models' +import { AbstractJobHandler } from './abstract-job-handler.js' +import { LiveRTMPHLSTranscodingJobHandler } from './live-rtmp-hls-transcoding-job-handler.js' +import { VideoStudioTranscodingJobHandler } from './video-studio-transcoding-job-handler.js' +import { VODAudioMergeTranscodingJobHandler } from './vod-audio-merge-transcoding-job-handler.js' +import { VODHLSTranscodingJobHandler } from './vod-hls-transcoding-job-handler.js' +import { VODWebVideoTranscodingJobHandler } from './vod-web-video-transcoding-job-handler.js' + +const processors: Record AbstractJobHandler> = { + 'vod-web-video-transcoding': VODWebVideoTranscodingJobHandler, + 'vod-hls-transcoding': VODHLSTranscodingJobHandler, + 'vod-audio-merge-transcoding': VODAudioMergeTranscodingJobHandler, + 'live-rtmp-hls-transcoding': LiveRTMPHLSTranscodingJobHandler, + 'video-studio-transcoding': VideoStudioTranscodingJobHandler +} + +export function getRunnerJobHandlerClass (job: MRunnerJob) { + return processors[job.type] +} diff --git a/server/server/lib/runners/job-handlers/shared/index.ts b/server/server/lib/runners/job-handlers/shared/index.ts new file mode 100644 index 000000000..bec61e3a0 --- /dev/null +++ b/server/server/lib/runners/job-handlers/shared/index.ts @@ -0,0 +1 @@ +export * from './vod-helpers.js' diff --git a/server/server/lib/runners/job-handlers/shared/vod-helpers.ts b/server/server/lib/runners/job-handlers/shared/vod-helpers.ts new file mode 100644 index 000000000..f7c1b73c9 --- /dev/null +++ b/server/server/lib/runners/job-handlers/shared/vod-helpers.ts @@ -0,0 +1,44 @@ +import { move } from 'fs-extra/esm' +import { dirname, join } from 'path' +import { logger, LoggerTagsFn } from '@server/helpers/logger.js' +import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding.js' +import { onWebVideoFileTranscoding } from '@server/lib/transcoding/web-transcoding.js' +import { buildNewFile } from '@server/lib/video-file.js' +import { VideoModel } from '@server/models/video/video.js' +import { MVideoFullLight } from '@server/types/models/index.js' +import { MRunnerJob } from '@server/types/models/runners/index.js' +import { RunnerJobVODAudioMergeTranscodingPrivatePayload, RunnerJobVODWebVideoTranscodingPrivatePayload } from '@peertube/peertube-models' + +export async function onVODWebVideoOrAudioMergeTranscodingJob (options: { + video: MVideoFullLight + videoFilePath: string + privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload | RunnerJobVODAudioMergeTranscodingPrivatePayload +}) { + const { video, videoFilePath, privatePayload } = options + + const videoFile = await buildNewFile({ path: videoFilePath, mode: 'web-video' }) + videoFile.videoId = video.id + + const newVideoFilePath = join(dirname(videoFilePath), videoFile.filename) + await move(videoFilePath, newVideoFilePath) + + await onWebVideoFileTranscoding({ + video, + videoFile, + videoOutputPath: newVideoFilePath + }) + + await onTranscodingEnded({ isNewVideo: privatePayload.isNewVideo, moveVideoToNextState: true, video }) +} + +export async function loadTranscodingRunnerVideo (runnerJob: MRunnerJob, lTags: LoggerTagsFn) { + const videoUUID = runnerJob.privatePayload.videoUUID + + const video = await VideoModel.loadFull(videoUUID) + if (!video) { + logger.info('Video %s does not exist anymore after transcoding runner job.', videoUUID, lTags(videoUUID)) + return undefined + } + + return video +} diff --git a/server/server/lib/runners/job-handlers/video-studio-transcoding-job-handler.ts b/server/server/lib/runners/job-handlers/video-studio-transcoding-job-handler.ts new file mode 100644 index 000000000..952e3a3c6 --- /dev/null +++ b/server/server/lib/runners/job-handlers/video-studio-transcoding-job-handler.ts @@ -0,0 +1,158 @@ + +import { + RunnerJobState, + RunnerJobStateType, + RunnerJobStudioTranscodingPayload, + RunnerJobUpdatePayload, + RunnerJobVideoStudioTranscodingPrivatePayload, + VideoState, + VideoStudioTaskPayload, + VideoStudioTranscodingSuccess, + isVideoStudioTaskIntro, + isVideoStudioTaskOutro, + isVideoStudioTaskWatermark +} from '@peertube/peertube-models' +import { buildUUID } from '@peertube/peertube-node-utils' +import { logger } from '@server/helpers/logger.js' +import { onVideoStudioEnded, safeCleanupStudioTMPFiles } from '@server/lib/video-studio.js' +import { MVideo } from '@server/types/models/index.js' +import { MRunnerJob } from '@server/types/models/runners/index.js' +import { basename } from 'path' +import { generateRunnerEditionTranscodingVideoInputFileUrl, generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls.js' +import { AbstractJobHandler } from './abstract-job-handler.js' +import { loadTranscodingRunnerVideo } from './shared/index.js' + +type CreateOptions = { + video: MVideo + tasks: VideoStudioTaskPayload[] + priority: number +} + +// eslint-disable-next-line max-len +export class VideoStudioTranscodingJobHandler extends AbstractJobHandler { + + async create (options: CreateOptions) { + const { video, priority, tasks } = options + + const jobUUID = buildUUID() + const payload: RunnerJobStudioTranscodingPayload = { + input: { + videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid) + }, + tasks: tasks.map(t => { + if (isVideoStudioTaskIntro(t) || isVideoStudioTaskOutro(t)) { + return { + ...t, + + options: { + ...t.options, + + file: generateRunnerEditionTranscodingVideoInputFileUrl(jobUUID, video.uuid, basename(t.options.file)) + } + } + } + + if (isVideoStudioTaskWatermark(t)) { + return { + ...t, + + options: { + ...t.options, + + file: generateRunnerEditionTranscodingVideoInputFileUrl(jobUUID, video.uuid, basename(t.options.file)) + } + } + } + + return t + }) + } + + const privatePayload: RunnerJobVideoStudioTranscodingPrivatePayload = { + videoUUID: video.uuid, + originalTasks: tasks + } + + const job = await this.createRunnerJob({ + type: 'video-studio-transcoding', + jobUUID, + payload, + privatePayload, + priority + }) + + return job + } + + // --------------------------------------------------------------------------- + + protected isAbortSupported () { + return true + } + + protected specificUpdate (_options: { + runnerJob: MRunnerJob + }) { + // empty + } + + protected specificAbort (_options: { + runnerJob: MRunnerJob + }) { + // empty + } + + protected async specificComplete (options: { + runnerJob: MRunnerJob + resultPayload: VideoStudioTranscodingSuccess + }) { + const { runnerJob, resultPayload } = options + const privatePayload = runnerJob.privatePayload as RunnerJobVideoStudioTranscodingPrivatePayload + + const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) + if (!video) { + await safeCleanupStudioTMPFiles(privatePayload.originalTasks) + + } + + const videoFilePath = resultPayload.videoFile as string + + await onVideoStudioEnded({ video, editionResultPath: videoFilePath, tasks: privatePayload.originalTasks }) + + logger.info( + 'Runner video edition transcoding job %s for %s ended.', + runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid) + ) + } + + protected specificError (options: { + runnerJob: MRunnerJob + nextState: RunnerJobStateType + }) { + if (options.nextState === RunnerJobState.ERRORED) { + return this.specificErrorOrCancel(options) + } + + return Promise.resolve() + } + + protected specificCancel (options: { + runnerJob: MRunnerJob + }) { + return this.specificErrorOrCancel(options) + } + + private async specificErrorOrCancel (options: { + runnerJob: MRunnerJob + }) { + const { runnerJob } = options + + const payload = runnerJob.privatePayload as RunnerJobVideoStudioTranscodingPrivatePayload + await safeCleanupStudioTMPFiles(payload.originalTasks) + + const video = await loadTranscodingRunnerVideo(options.runnerJob, this.lTags) + if (!video) return + + return video.setNewState(VideoState.PUBLISHED, false, undefined) + } +} diff --git a/server/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts b/server/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts new file mode 100644 index 000000000..e3339f569 --- /dev/null +++ b/server/server/lib/runners/job-handlers/vod-audio-merge-transcoding-job-handler.ts @@ -0,0 +1,97 @@ +import { logger } from '@server/helpers/logger.js' +import { VideoJobInfoModel } from '@server/models/video/video-job-info.js' +import { MVideo } from '@server/types/models/index.js' +import { MRunnerJob } from '@server/types/models/runners/index.js' +import { pick } from '@peertube/peertube-core-utils' +import { buildUUID } from '@peertube/peertube-node-utils' +import { getVideoStreamDuration } from '@peertube/peertube-ffmpeg' +import { + RunnerJobUpdatePayload, + RunnerJobVODAudioMergeTranscodingPayload, + RunnerJobVODWebVideoTranscodingPrivatePayload, + VODAudioMergeTranscodingSuccess +} from '@peertube/peertube-models' +import { generateRunnerTranscodingVideoInputFileUrl, generateRunnerTranscodingVideoPreviewFileUrl } from '../runner-urls.js' +import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler.js' +import { loadTranscodingRunnerVideo, onVODWebVideoOrAudioMergeTranscodingJob } from './shared/index.js' + +type CreateOptions = { + video: MVideo + isNewVideo: boolean + resolution: number + fps: number + priority: number + dependsOnRunnerJob?: MRunnerJob +} + +// eslint-disable-next-line max-len +export class VODAudioMergeTranscodingJobHandler extends AbstractVODTranscodingJobHandler { + + async create (options: CreateOptions) { + const { video, resolution, fps, priority, dependsOnRunnerJob } = options + + const jobUUID = buildUUID() + const payload: RunnerJobVODAudioMergeTranscodingPayload = { + input: { + audioFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid), + previewFileUrl: generateRunnerTranscodingVideoPreviewFileUrl(jobUUID, video.uuid) + }, + output: { + resolution, + fps + } + } + + const privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload = { + ...pick(options, [ 'isNewVideo' ]), + + videoUUID: video.uuid + } + + const job = await this.createRunnerJob({ + type: 'vod-audio-merge-transcoding', + jobUUID, + payload, + privatePayload, + priority, + dependsOnRunnerJob + }) + + await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode') + + return job + } + + // --------------------------------------------------------------------------- + + protected async specificComplete (options: { + runnerJob: MRunnerJob + resultPayload: VODAudioMergeTranscodingSuccess + }) { + const { runnerJob, resultPayload } = options + const privatePayload = runnerJob.privatePayload as RunnerJobVODWebVideoTranscodingPrivatePayload + + const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) + if (!video) return + + const videoFilePath = resultPayload.videoFile as string + + // ffmpeg generated a new video file, so update the video duration + // See https://trac.ffmpeg.org/ticket/5456 + video.duration = await getVideoStreamDuration(videoFilePath) + await video.save() + + // We can remove the old audio file + const oldAudioFile = video.VideoFiles[0] + await video.removeWebVideoFile(oldAudioFile) + await oldAudioFile.destroy() + video.VideoFiles = [] + + await onVODWebVideoOrAudioMergeTranscodingJob({ video, videoFilePath, privatePayload }) + + logger.info( + 'Runner VOD audio merge transcoding job %s for %s ended.', + runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid) + ) + } +} diff --git a/server/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts b/server/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts new file mode 100644 index 000000000..e0b90313f --- /dev/null +++ b/server/server/lib/runners/job-handlers/vod-hls-transcoding-job-handler.ts @@ -0,0 +1,114 @@ +import { move } from 'fs-extra/esm' +import { dirname, join } from 'path' +import { logger } from '@server/helpers/logger.js' +import { renameVideoFileInPlaylist } from '@server/lib/hls.js' +import { getHlsResolutionPlaylistFilename } from '@server/lib/paths.js' +import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding.js' +import { onHLSVideoFileTranscoding } from '@server/lib/transcoding/hls-transcoding.js' +import { buildNewFile, removeAllWebVideoFiles } from '@server/lib/video-file.js' +import { VideoJobInfoModel } from '@server/models/video/video-job-info.js' +import { MVideo } from '@server/types/models/index.js' +import { MRunnerJob } from '@server/types/models/runners/index.js' +import { pick } from '@peertube/peertube-core-utils' +import { buildUUID } from '@peertube/peertube-node-utils' +import { + RunnerJobUpdatePayload, + RunnerJobVODHLSTranscodingPayload, + RunnerJobVODHLSTranscodingPrivatePayload, + VODHLSTranscodingSuccess +} from '@peertube/peertube-models' +import { generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls.js' +import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler.js' +import { loadTranscodingRunnerVideo } from './shared/index.js' + +type CreateOptions = { + video: MVideo + isNewVideo: boolean + deleteWebVideoFiles: boolean + resolution: number + fps: number + priority: number + dependsOnRunnerJob?: MRunnerJob +} + +// eslint-disable-next-line max-len +export class VODHLSTranscodingJobHandler extends AbstractVODTranscodingJobHandler { + + async create (options: CreateOptions) { + const { video, resolution, fps, dependsOnRunnerJob, priority } = options + + const jobUUID = buildUUID() + + const payload: RunnerJobVODHLSTranscodingPayload = { + input: { + videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid) + }, + output: { + resolution, + fps + } + } + + const privatePayload: RunnerJobVODHLSTranscodingPrivatePayload = { + ...pick(options, [ 'isNewVideo', 'deleteWebVideoFiles' ]), + + videoUUID: video.uuid + } + + const job = await this.createRunnerJob({ + type: 'vod-hls-transcoding', + jobUUID, + payload, + privatePayload, + priority, + dependsOnRunnerJob + }) + + await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode') + + return job + } + + // --------------------------------------------------------------------------- + + protected async specificComplete (options: { + runnerJob: MRunnerJob + resultPayload: VODHLSTranscodingSuccess + }) { + const { runnerJob, resultPayload } = options + const privatePayload = runnerJob.privatePayload as RunnerJobVODHLSTranscodingPrivatePayload + + const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) + if (!video) return + + const videoFilePath = resultPayload.videoFile as string + const resolutionPlaylistFilePath = resultPayload.resolutionPlaylistFile as string + + const videoFile = await buildNewFile({ path: videoFilePath, mode: 'hls' }) + const newVideoFilePath = join(dirname(videoFilePath), videoFile.filename) + await move(videoFilePath, newVideoFilePath) + + const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFile.filename) + const newResolutionPlaylistFilePath = join(dirname(resolutionPlaylistFilePath), resolutionPlaylistFilename) + await move(resolutionPlaylistFilePath, newResolutionPlaylistFilePath) + + await renameVideoFileInPlaylist(newResolutionPlaylistFilePath, videoFile.filename) + + await onHLSVideoFileTranscoding({ + video, + videoFile, + m3u8OutputPath: newResolutionPlaylistFilePath, + videoOutputPath: newVideoFilePath + }) + + await onTranscodingEnded({ isNewVideo: privatePayload.isNewVideo, moveVideoToNextState: true, video }) + + if (privatePayload.deleteWebVideoFiles === true) { + logger.info('Removing web video files of %s now we have a HLS version of it.', video.uuid, this.lTags(video.uuid)) + + await removeAllWebVideoFiles(video) + } + + logger.info('Runner VOD HLS job %s for %s ended.', runnerJob.uuid, video.uuid, this.lTags(runnerJob.uuid, video.uuid)) + } +} diff --git a/server/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts b/server/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts new file mode 100644 index 000000000..a137a5d48 --- /dev/null +++ b/server/server/lib/runners/job-handlers/vod-web-video-transcoding-job-handler.ts @@ -0,0 +1,84 @@ +import { logger } from '@server/helpers/logger.js' +import { VideoJobInfoModel } from '@server/models/video/video-job-info.js' +import { MVideo } from '@server/types/models/index.js' +import { MRunnerJob } from '@server/types/models/runners/index.js' +import { pick } from '@peertube/peertube-core-utils' +import { buildUUID } from '@peertube/peertube-node-utils' +import { + RunnerJobUpdatePayload, + RunnerJobVODWebVideoTranscodingPayload, + RunnerJobVODWebVideoTranscodingPrivatePayload, + VODWebVideoTranscodingSuccess +} from '@peertube/peertube-models' +import { generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls.js' +import { AbstractVODTranscodingJobHandler } from './abstract-vod-transcoding-job-handler.js' +import { loadTranscodingRunnerVideo, onVODWebVideoOrAudioMergeTranscodingJob } from './shared/index.js' + +type CreateOptions = { + video: MVideo + isNewVideo: boolean + resolution: number + fps: number + priority: number + dependsOnRunnerJob?: MRunnerJob +} + +// eslint-disable-next-line max-len +export class VODWebVideoTranscodingJobHandler extends AbstractVODTranscodingJobHandler { + + async create (options: CreateOptions) { + const { video, resolution, fps, priority, dependsOnRunnerJob } = options + + const jobUUID = buildUUID() + const payload: RunnerJobVODWebVideoTranscodingPayload = { + input: { + videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, video.uuid) + }, + output: { + resolution, + fps + } + } + + const privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload = { + ...pick(options, [ 'isNewVideo' ]), + + videoUUID: video.uuid + } + + const job = await this.createRunnerJob({ + type: 'vod-web-video-transcoding', + jobUUID, + payload, + privatePayload, + dependsOnRunnerJob, + priority + }) + + await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode') + + return job + } + + // --------------------------------------------------------------------------- + + protected async specificComplete (options: { + runnerJob: MRunnerJob + resultPayload: VODWebVideoTranscodingSuccess + }) { + const { runnerJob, resultPayload } = options + const privatePayload = runnerJob.privatePayload as RunnerJobVODWebVideoTranscodingPrivatePayload + + const video = await loadTranscodingRunnerVideo(runnerJob, this.lTags) + if (!video) return + + const videoFilePath = resultPayload.videoFile as string + + await onVODWebVideoOrAudioMergeTranscodingJob({ video, videoFilePath, privatePayload }) + + logger.info( + 'Runner VOD web video transcoding job %s for %s ended.', + runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid) + ) + } +} diff --git a/server/server/lib/runners/runner-urls.ts b/server/server/lib/runners/runner-urls.ts new file mode 100644 index 000000000..0c0a9966f --- /dev/null +++ b/server/server/lib/runners/runner-urls.ts @@ -0,0 +1,13 @@ +import { WEBSERVER } from '@server/initializers/constants.js' + +export function generateRunnerTranscodingVideoInputFileUrl (jobUUID: string, videoUUID: string) { + return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/max-quality' +} + +export function generateRunnerTranscodingVideoPreviewFileUrl (jobUUID: string, videoUUID: string) { + return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/previews/max-quality' +} + +export function generateRunnerEditionTranscodingVideoInputFileUrl (jobUUID: string, videoUUID: string, filename: string) { + return WEBSERVER.URL + '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID + '/studio/task-files/' + filename +} diff --git a/server/server/lib/runners/runner.ts b/server/server/lib/runners/runner.ts new file mode 100644 index 000000000..8152e6f5f --- /dev/null +++ b/server/server/lib/runners/runner.ts @@ -0,0 +1,49 @@ +import { RunnerJobState, RunnerJobStateType } from '@peertube/peertube-models' +import { retryTransactionWrapper } from '@server/helpers/database-utils.js' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { RUNNER_JOBS } from '@server/initializers/constants.js' +import { sequelizeTypescript } from '@server/initializers/database.js' +import { MRunner, MRunnerJob } from '@server/types/models/runners/index.js' +import express from 'express' + +const lTags = loggerTagsFactory('runner') + +const updatingRunner = new Set() + +function updateLastRunnerContact (req: express.Request, runner: MRunner) { + const now = new Date() + + // Don't update last runner contact too often + if (now.getTime() - runner.lastContact.getTime() < RUNNER_JOBS.LAST_CONTACT_UPDATE_INTERVAL) return + if (updatingRunner.has(runner.id)) return + + updatingRunner.add(runner.id) + + runner.lastContact = now + runner.ip = req.ip + + logger.debug('Updating last runner contact for %s', runner.name, lTags(runner.name)) + + retryTransactionWrapper(() => { + return sequelizeTypescript.transaction(async transaction => { + return runner.save({ transaction }) + }) + }) + .catch(err => logger.error('Cannot update last runner contact for %s', runner.name, { err, ...lTags(runner.name) })) + .finally(() => updatingRunner.delete(runner.id)) +} + +function runnerJobCanBeCancelled (runnerJob: MRunnerJob) { + const allowedStates = new Set([ + RunnerJobState.PENDING, + RunnerJobState.PROCESSING, + RunnerJobState.WAITING_FOR_PARENT_JOB + ]) + + return allowedStates.has(runnerJob.state) +} + +export { + updateLastRunnerContact, + runnerJobCanBeCancelled +} diff --git a/server/server/lib/schedulers/abstract-scheduler.ts b/server/server/lib/schedulers/abstract-scheduler.ts new file mode 100644 index 000000000..8b6eef3f4 --- /dev/null +++ b/server/server/lib/schedulers/abstract-scheduler.ts @@ -0,0 +1,35 @@ +import Bluebird from 'bluebird' +import { logger } from '../../helpers/logger.js' + +export abstract class AbstractScheduler { + + protected abstract schedulerIntervalMs: number + + private interval: NodeJS.Timer + private isRunning = false + + enable () { + if (!this.schedulerIntervalMs) throw new Error('Interval is not correctly set.') + + this.interval = setInterval(() => this.execute(), this.schedulerIntervalMs) + } + + disable () { + clearInterval(this.interval) + } + + async execute () { + if (this.isRunning === true) return + this.isRunning = true + + try { + await this.internalExecute() + } catch (err) { + logger.error('Cannot execute %s scheduler.', this.constructor.name, { err }) + } finally { + this.isRunning = false + } + } + + protected abstract internalExecute (): Promise | Bluebird +} diff --git a/server/server/lib/schedulers/actor-follow-scheduler.ts b/server/server/lib/schedulers/actor-follow-scheduler.ts new file mode 100644 index 000000000..86b83f60b --- /dev/null +++ b/server/server/lib/schedulers/actor-follow-scheduler.ts @@ -0,0 +1,54 @@ +import { isTestOrDevInstance } from '@peertube/peertube-node-utils' +import { logger } from '../../helpers/logger.js' +import { ACTOR_FOLLOW_SCORE, SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js' +import { ActorFollowModel } from '../../models/actor/actor-follow.js' +import { ActorFollowHealthCache } from '../actor-follow-health-cache.js' +import { AbstractScheduler } from './abstract-scheduler.js' + +export class ActorFollowScheduler extends AbstractScheduler { + + private static instance: AbstractScheduler + + protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.ACTOR_FOLLOW_SCORES + + private constructor () { + super() + } + + protected async internalExecute () { + await this.processPendingScores() + + await this.removeBadActorFollows() + } + + private async processPendingScores () { + const pendingScores = ActorFollowHealthCache.Instance.getPendingFollowsScore() + const badServerIds = ActorFollowHealthCache.Instance.getBadFollowingServerIds() + const goodServerIds = ActorFollowHealthCache.Instance.getGoodFollowingServerIds() + + ActorFollowHealthCache.Instance.clearPendingFollowsScore() + ActorFollowHealthCache.Instance.clearBadFollowingServerIds() + ActorFollowHealthCache.Instance.clearGoodFollowingServerIds() + + for (const inbox of Object.keys(pendingScores)) { + await ActorFollowModel.updateScore(inbox, pendingScores[inbox]) + } + + await ActorFollowModel.updateScoreByFollowingServers(badServerIds, ACTOR_FOLLOW_SCORE.PENALTY) + await ActorFollowModel.updateScoreByFollowingServers(goodServerIds, ACTOR_FOLLOW_SCORE.BONUS) + } + + private async removeBadActorFollows () { + if (!isTestOrDevInstance()) logger.info('Removing bad actor follows (scheduler).') + + try { + await ActorFollowModel.removeBadActorFollows() + } catch (err) { + logger.error('Error in bad actor follows scheduler.', { err }) + } + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/server/lib/schedulers/auto-follow-index-instances.ts b/server/server/lib/schedulers/auto-follow-index-instances.ts new file mode 100644 index 000000000..47a01ec0c --- /dev/null +++ b/server/server/lib/schedulers/auto-follow-index-instances.ts @@ -0,0 +1,75 @@ +import { doJSONRequest } from '@server/helpers/requests.js' +import { JobQueue } from '@server/lib/job-queue/index.js' +import { ActorFollowModel } from '@server/models/actor/actor-follow.js' +import { getServerActor } from '@server/models/application/application.js' +import chunk from 'lodash-es/chunk.js' +import { logger } from '../../helpers/logger.js' +import { CONFIG } from '../../initializers/config.js' +import { SCHEDULER_INTERVALS_MS, SERVER_ACTOR_NAME } from '../../initializers/constants.js' +import { AbstractScheduler } from './abstract-scheduler.js' + +export class AutoFollowIndexInstances extends AbstractScheduler { + + private static instance: AbstractScheduler + + protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.AUTO_FOLLOW_INDEX_INSTANCES + + private lastCheck: Date + + private constructor () { + super() + } + + protected async internalExecute () { + return this.autoFollow() + } + + private async autoFollow () { + if (CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.ENABLED === false) return + + const indexUrl = CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.INDEX_URL + + logger.info('Auto follow instances of index %s.', indexUrl) + + try { + const serverActor = await getServerActor() + + const searchParams = { count: 1000 } + if (this.lastCheck) Object.assign(searchParams, { since: this.lastCheck.toISOString() }) + + this.lastCheck = new Date() + + const { body } = await doJSONRequest(indexUrl, { searchParams }) + if (!body.data || Array.isArray(body.data) === false) { + logger.error('Cannot auto follow instances of index %s. Please check the auto follow URL.', indexUrl, { body }) + return + } + + const hosts: string[] = body.data.map(o => o.host) + const chunks = chunk(hosts, 20) + + for (const chunk of chunks) { + const unfollowedHosts = await ActorFollowModel.keepUnfollowedInstance(chunk) + + for (const unfollowedHost of unfollowedHosts) { + const payload = { + host: unfollowedHost, + name: SERVER_ACTOR_NAME, + followerActorId: serverActor.id, + isAutoFollow: true + } + + JobQueue.Instance.createJobAsync({ type: 'activitypub-follow', payload }) + } + } + + } catch (err) { + logger.error('Cannot auto follow hosts of index %s.', indexUrl, { err }) + } + + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/server/lib/schedulers/geo-ip-update-scheduler.ts b/server/server/lib/schedulers/geo-ip-update-scheduler.ts new file mode 100644 index 000000000..b59aa71e5 --- /dev/null +++ b/server/server/lib/schedulers/geo-ip-update-scheduler.ts @@ -0,0 +1,22 @@ +import { GeoIP } from '@server/helpers/geo-ip.js' +import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js' +import { AbstractScheduler } from './abstract-scheduler.js' + +export class GeoIPUpdateScheduler extends AbstractScheduler { + + private static instance: AbstractScheduler + + protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.GEO_IP_UPDATE + + private constructor () { + super() + } + + protected internalExecute () { + return GeoIP.Instance.updateDatabase() + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/server/lib/schedulers/peertube-version-check-scheduler.ts b/server/server/lib/schedulers/peertube-version-check-scheduler.ts new file mode 100644 index 000000000..c706aba63 --- /dev/null +++ b/server/server/lib/schedulers/peertube-version-check-scheduler.ts @@ -0,0 +1,55 @@ + +import { doJSONRequest } from '@server/helpers/requests.js' +import { ApplicationModel } from '@server/models/application/application.js' +import { compareSemVer } from '@peertube/peertube-core-utils' +import { JoinPeerTubeVersions } from '@peertube/peertube-models' +import { logger } from '../../helpers/logger.js' +import { CONFIG } from '../../initializers/config.js' +import { PEERTUBE_VERSION, SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js' +import { Notifier } from '../notifier/index.js' +import { AbstractScheduler } from './abstract-scheduler.js' + +export class PeerTubeVersionCheckScheduler extends AbstractScheduler { + + private static instance: AbstractScheduler + + protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHECK_PEERTUBE_VERSION + + private constructor () { + super() + } + + protected async internalExecute () { + return this.checkLatestVersion() + } + + private async checkLatestVersion () { + if (CONFIG.PEERTUBE.CHECK_LATEST_VERSION.ENABLED === false) return + + logger.info('Checking latest PeerTube version.') + + const { body } = await doJSONRequest(CONFIG.PEERTUBE.CHECK_LATEST_VERSION.URL) + + if (!body?.peertube?.latestVersion) { + logger.warn('Cannot check latest PeerTube version: body is invalid.', { body }) + return + } + + const latestVersion = body.peertube.latestVersion + const application = await ApplicationModel.load() + + // Already checked this version + if (application.latestPeerTubeVersion === latestVersion) return + + if (compareSemVer(PEERTUBE_VERSION, latestVersion) < 0) { + application.latestPeerTubeVersion = latestVersion + await application.save() + + Notifier.Instance.notifyOfNewPeerTubeVersion(application, latestVersion) + } + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/server/lib/schedulers/plugins-check-scheduler.ts b/server/server/lib/schedulers/plugins-check-scheduler.ts new file mode 100644 index 000000000..627289663 --- /dev/null +++ b/server/server/lib/schedulers/plugins-check-scheduler.ts @@ -0,0 +1,74 @@ +import { compareSemVer } from '@peertube/peertube-core-utils' +import chunk from 'lodash-es/chunk.js' +import { logger } from '../../helpers/logger.js' +import { CONFIG } from '../../initializers/config.js' +import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js' +import { PluginModel } from '../../models/server/plugin.js' +import { Notifier } from '../notifier/index.js' +import { getLatestPluginsVersion } from '../plugins/plugin-index.js' +import { AbstractScheduler } from './abstract-scheduler.js' + +export class PluginsCheckScheduler extends AbstractScheduler { + + private static instance: AbstractScheduler + + protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHECK_PLUGINS + + private constructor () { + super() + } + + protected async internalExecute () { + return this.checkLatestPluginsVersion() + } + + private async checkLatestPluginsVersion () { + if (CONFIG.PLUGINS.INDEX.ENABLED === false) return + + logger.info('Checking latest plugins version.') + + const plugins = await PluginModel.listInstalled() + + // Process 10 plugins in 1 HTTP request + const chunks = chunk(plugins, 10) + for (const chunk of chunks) { + // Find plugins according to their npm name + const pluginIndex: { [npmName: string]: PluginModel } = {} + for (const plugin of chunk) { + pluginIndex[PluginModel.buildNpmName(plugin.name, plugin.type)] = plugin + } + + const npmNames = Object.keys(pluginIndex) + + try { + const results = await getLatestPluginsVersion(npmNames) + + for (const result of results) { + const plugin = pluginIndex[result.npmName] + if (!result.latestVersion) continue + + if ( + !plugin.latestVersion || + (plugin.latestVersion !== result.latestVersion && compareSemVer(plugin.latestVersion, result.latestVersion) < 0) + ) { + plugin.latestVersion = result.latestVersion + await plugin.save() + + // Notify if there is an higher plugin version available + if (compareSemVer(plugin.version, result.latestVersion) < 0) { + Notifier.Instance.notifyOfNewPluginVersion(plugin) + } + + logger.info('Plugin %s has a new latest version %s.', result.npmName, plugin.latestVersion) + } + } + } catch (err) { + logger.error('Cannot get latest plugins version.', { npmNames, err }) + } + } + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts b/server/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts new file mode 100644 index 000000000..6d5e70b08 --- /dev/null +++ b/server/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts @@ -0,0 +1,40 @@ + +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants.js' +import { uploadx } from '../uploadx.js' +import { AbstractScheduler } from './abstract-scheduler.js' + +const lTags = loggerTagsFactory('scheduler', 'resumable-upload', 'cleaner') + +export class RemoveDanglingResumableUploadsScheduler extends AbstractScheduler { + + private static instance: AbstractScheduler + private lastExecutionTimeMs: number + + protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.REMOVE_DANGLING_RESUMABLE_UPLOADS + + private constructor () { + super() + + this.lastExecutionTimeMs = new Date().getTime() + } + + protected async internalExecute () { + logger.debug('Removing dangling resumable uploads', lTags()) + + const now = new Date().getTime() + + try { + // Remove files that were not updated since the last execution + await uploadx.storage.purge(now - this.lastExecutionTimeMs) + } catch (error) { + logger.error('Failed to handle file during resumable video upload folder cleanup', { error, ...lTags() }) + } finally { + this.lastExecutionTimeMs = now + } + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/server/lib/schedulers/remove-old-history-scheduler.ts b/server/server/lib/schedulers/remove-old-history-scheduler.ts new file mode 100644 index 000000000..3a557ae97 --- /dev/null +++ b/server/server/lib/schedulers/remove-old-history-scheduler.ts @@ -0,0 +1,31 @@ +import { logger } from '../../helpers/logger.js' +import { AbstractScheduler } from './abstract-scheduler.js' +import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js' +import { UserVideoHistoryModel } from '../../models/user/user-video-history.js' +import { CONFIG } from '../../initializers/config.js' + +export class RemoveOldHistoryScheduler extends AbstractScheduler { + + private static instance: AbstractScheduler + + protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.REMOVE_OLD_HISTORY + + private constructor () { + super() + } + + protected internalExecute () { + if (CONFIG.HISTORY.VIDEOS.MAX_AGE === -1) return + + logger.info('Removing old videos history.') + + const now = new Date() + const beforeDate = new Date(now.getTime() - CONFIG.HISTORY.VIDEOS.MAX_AGE).toISOString() + + return UserVideoHistoryModel.removeOldHistory(beforeDate) + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/server/lib/schedulers/remove-old-views-scheduler.ts b/server/server/lib/schedulers/remove-old-views-scheduler.ts new file mode 100644 index 000000000..7bd28b17b --- /dev/null +++ b/server/server/lib/schedulers/remove-old-views-scheduler.ts @@ -0,0 +1,31 @@ +import { VideoViewModel } from '@server/models/view/video-view.js' +import { logger } from '../../helpers/logger.js' +import { CONFIG } from '../../initializers/config.js' +import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js' +import { AbstractScheduler } from './abstract-scheduler.js' + +export class RemoveOldViewsScheduler extends AbstractScheduler { + + private static instance: AbstractScheduler + + protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.REMOVE_OLD_VIEWS + + private constructor () { + super() + } + + protected internalExecute () { + if (CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE === -1) return + + logger.info('Removing old videos views.') + + const now = new Date() + const beforeDate = new Date(now.getTime() - CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE).toISOString() + + return VideoViewModel.removeOldRemoteViewsHistory(beforeDate) + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/server/lib/schedulers/runner-job-watch-dog-scheduler.ts b/server/server/lib/schedulers/runner-job-watch-dog-scheduler.ts new file mode 100644 index 000000000..941e79c3e --- /dev/null +++ b/server/server/lib/schedulers/runner-job-watch-dog-scheduler.ts @@ -0,0 +1,42 @@ +import { CONFIG } from '@server/initializers/config.js' +import { RunnerJobModel } from '@server/models/runner/runner-job.js' +import { logger, loggerTagsFactory } from '../../helpers/logger.js' +import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js' +import { getRunnerJobHandlerClass } from '../runners/index.js' +import { AbstractScheduler } from './abstract-scheduler.js' + +const lTags = loggerTagsFactory('runner') + +export class RunnerJobWatchDogScheduler extends AbstractScheduler { + + private static instance: AbstractScheduler + + protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.RUNNER_JOB_WATCH_DOG + + private constructor () { + super() + } + + protected async internalExecute () { + const vodStalledJobs = await RunnerJobModel.listStalledJobs({ + staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.VOD, + types: [ 'vod-audio-merge-transcoding', 'vod-hls-transcoding', 'vod-web-video-transcoding' ] + }) + + const liveStalledJobs = await RunnerJobModel.listStalledJobs({ + staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.LIVE, + types: [ 'live-rtmp-hls-transcoding' ] + }) + + for (const stalled of [ ...vodStalledJobs, ...liveStalledJobs ]) { + logger.info('Abort stalled runner job %s (%s)', stalled.uuid, stalled.type, lTags(stalled.uuid, stalled.type)) + + const Handler = getRunnerJobHandlerClass(stalled) + await new Handler().abort({ runnerJob: stalled }) + } + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/server/lib/schedulers/update-videos-scheduler.ts b/server/server/lib/schedulers/update-videos-scheduler.ts new file mode 100644 index 000000000..2d4892013 --- /dev/null +++ b/server/server/lib/schedulers/update-videos-scheduler.ts @@ -0,0 +1,89 @@ +import { VideoPrivacy, VideoPrivacyType, VideoState } from '@peertube/peertube-models' +import { VideoModel } from '@server/models/video/video.js' +import { MScheduleVideoUpdate } from '@server/types/models/index.js' +import { logger } from '../../helpers/logger.js' +import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js' +import { sequelizeTypescript } from '../../initializers/database.js' +import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-update.js' +import { Notifier } from '../notifier/index.js' +import { VideoPathManager } from '../video-path-manager.js' +import { setVideoPrivacy } from '../video-privacy.js' +import { addVideoJobsAfterUpdate } from '../video.js' +import { AbstractScheduler } from './abstract-scheduler.js' + +export class UpdateVideosScheduler extends AbstractScheduler { + + private static instance: AbstractScheduler + + protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.UPDATE_VIDEOS + + private constructor () { + super() + } + + protected async internalExecute () { + return this.updateVideos() + } + + private async updateVideos () { + if (!await ScheduleVideoUpdateModel.areVideosToUpdate()) return undefined + + const schedules = await ScheduleVideoUpdateModel.listVideosToUpdate() + + for (const schedule of schedules) { + const videoOnly = await VideoModel.load(schedule.videoId) + const mutexReleaser = await VideoPathManager.Instance.lockFiles(videoOnly.uuid) + + try { + const { video, published } = await this.updateAVideo(schedule) + + if (published) Notifier.Instance.notifyOnVideoPublishedAfterScheduledUpdate(video) + } catch (err) { + logger.error('Cannot update video', { err }) + } + + mutexReleaser() + } + } + + private async updateAVideo (schedule: MScheduleVideoUpdate) { + let oldPrivacy: VideoPrivacyType + let isNewVideo: boolean + let published = false + + const video = await sequelizeTypescript.transaction(async t => { + const video = await VideoModel.loadFull(schedule.videoId, t) + if (video.state === VideoState.TO_TRANSCODE) return null + + logger.info('Executing scheduled video update on %s.', video.uuid) + + if (schedule.privacy) { + isNewVideo = video.isNewVideo(schedule.privacy) + oldPrivacy = video.privacy + + setVideoPrivacy(video, schedule.privacy) + await video.save({ transaction: t }) + + if (oldPrivacy === VideoPrivacy.PRIVATE) { + published = true + } + } + + await schedule.destroy({ transaction: t }) + + return video + }) + + if (!video) { + return { video, published: false } + } + + await addVideoJobsAfterUpdate({ video, oldPrivacy, isNewVideo, nameChanged: false }) + + return { video, published } + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/server/lib/schedulers/video-channel-sync-latest-scheduler.ts b/server/server/lib/schedulers/video-channel-sync-latest-scheduler.ts new file mode 100644 index 000000000..2a3ad18b1 --- /dev/null +++ b/server/server/lib/schedulers/video-channel-sync-latest-scheduler.ts @@ -0,0 +1,50 @@ +import { logger } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' +import { VideoChannelModel } from '@server/models/video/video-channel.js' +import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync.js' +import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js' +import { synchronizeChannel } from '../sync-channel.js' +import { AbstractScheduler } from './abstract-scheduler.js' + +export class VideoChannelSyncLatestScheduler extends AbstractScheduler { + private static instance: AbstractScheduler + protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.CHANNEL_SYNC_CHECK_INTERVAL + + private constructor () { + super() + } + + protected async internalExecute () { + if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) { + logger.debug('Discard channels synchronization as the feature is disabled') + return + } + + logger.info('Checking channels to synchronize') + + const channelSyncs = await VideoChannelSyncModel.listSyncs() + + for (const sync of channelSyncs) { + const channel = await VideoChannelModel.loadAndPopulateAccount(sync.videoChannelId) + + logger.info( + 'Creating video import jobs for "%s" sync with external channel "%s"', + channel.Actor.preferredUsername, sync.externalChannelUrl + ) + + const onlyAfter = sync.lastSyncAt || sync.createdAt + + await synchronizeChannel({ + channel, + externalChannelUrl: sync.externalChannelUrl, + videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION, + channelSync: sync, + onlyAfter + }) + } + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/server/lib/schedulers/video-views-buffer-scheduler.ts b/server/server/lib/schedulers/video-views-buffer-scheduler.ts new file mode 100644 index 000000000..fd8a61324 --- /dev/null +++ b/server/server/lib/schedulers/video-views-buffer-scheduler.ts @@ -0,0 +1,52 @@ +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { VideoModel } from '@server/models/video/video.js' +import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js' +import { federateVideoIfNeeded } from '../activitypub/videos/index.js' +import { Redis } from '../redis.js' +import { AbstractScheduler } from './abstract-scheduler.js' + +const lTags = loggerTagsFactory('views') + +export class VideoViewsBufferScheduler extends AbstractScheduler { + + private static instance: AbstractScheduler + + protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.VIDEO_VIEWS_BUFFER_UPDATE + + private constructor () { + super() + } + + protected async internalExecute () { + const videoIds = await Redis.Instance.listLocalVideosViewed() + if (videoIds.length === 0) return + + for (const videoId of videoIds) { + try { + const views = await Redis.Instance.getLocalVideoViews(videoId) + await Redis.Instance.deleteLocalVideoViews(videoId) + + const video = await VideoModel.loadFull(videoId) + if (!video) { + logger.debug('Video %d does not exist anymore, skipping videos view addition.', videoId, lTags()) + continue + } + + logger.info('Processing local video %s views buffer.', video.uuid, lTags(video.uuid)) + + // If this is a remote video, the origin instance will send us an update + await VideoModel.incrementViews(videoId, views) + + // Send video update + video.views += views + await federateVideoIfNeeded(video, false) + } catch (err) { + logger.error('Cannot process local video views buffer of video %d.', videoId, { err, ...lTags() }) + } + } + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/server/lib/schedulers/videos-redundancy-scheduler.ts new file mode 100644 index 000000000..60d8686b2 --- /dev/null +++ b/server/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -0,0 +1,375 @@ +import { move } from 'fs-extra/esm' +import { join } from 'path' +import { getServerActor } from '@server/models/application/application.js' +import { VideoModel } from '@server/models/video/video.js' +import { + MStreamingPlaylistFiles, + MVideoAccountLight, + MVideoFile, + MVideoFileVideo, + MVideoRedundancyFileVideo, + MVideoRedundancyStreamingPlaylistVideo, + MVideoRedundancyVideo, + MVideoWithAllFiles +} from '@server/types/models/index.js' +import { VideosRedundancyStrategy } from '@peertube/peertube-models' +import { logger, loggerTagsFactory } from '../../helpers/logger.js' +import { downloadWebTorrentVideo } from '../../helpers/webtorrent.js' +import { CONFIG } from '../../initializers/config.js' +import { DIRECTORIES, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers/constants.js' +import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy.js' +import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send/index.js' +import { getLocalVideoCacheFileActivityPubUrl, getLocalVideoCacheStreamingPlaylistActivityPubUrl } from '../activitypub/url.js' +import { getOrCreateAPVideo } from '../activitypub/videos/index.js' +import { downloadPlaylistSegments } from '../hls.js' +import { removeVideoRedundancy } from '../redundancy.js' +import { generateHLSRedundancyUrl, generateWebVideoRedundancyUrl } from '../video-urls.js' +import { AbstractScheduler } from './abstract-scheduler.js' + +const lTags = loggerTagsFactory('redundancy') + +type CandidateToDuplicate = { + redundancy: VideosRedundancyStrategy + video: MVideoWithAllFiles + files: MVideoFile[] + streamingPlaylists: MStreamingPlaylistFiles[] +} + +function isMVideoRedundancyFileVideo ( + o: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo +): o is MVideoRedundancyFileVideo { + return !!(o as MVideoRedundancyFileVideo).VideoFile +} + +export class VideosRedundancyScheduler extends AbstractScheduler { + + private static instance: VideosRedundancyScheduler + + protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL + + private constructor () { + super() + } + + async createManualRedundancy (videoId: number) { + const videoToDuplicate = await VideoModel.loadWithFiles(videoId) + + if (!videoToDuplicate) { + logger.warn('Video to manually duplicate %d does not exist anymore.', videoId, lTags()) + return + } + + return this.createVideoRedundancies({ + video: videoToDuplicate, + redundancy: null, + files: videoToDuplicate.VideoFiles, + streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists + }) + } + + protected async internalExecute () { + for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) { + logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy, lTags()) + + try { + const videoToDuplicate = await this.findVideoToDuplicate(redundancyConfig) + if (!videoToDuplicate) continue + + const candidateToDuplicate = { + video: videoToDuplicate, + redundancy: redundancyConfig, + files: videoToDuplicate.VideoFiles, + streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists + } + + await this.purgeCacheIfNeeded(candidateToDuplicate) + + if (await this.isTooHeavy(candidateToDuplicate)) { + logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url, lTags(videoToDuplicate.uuid)) + continue + } + + logger.info( + 'Will duplicate video %s in redundancy scheduler "%s".', + videoToDuplicate.url, redundancyConfig.strategy, lTags(videoToDuplicate.uuid) + ) + + await this.createVideoRedundancies(candidateToDuplicate) + } catch (err) { + logger.error('Cannot run videos redundancy %s.', redundancyConfig.strategy, { err, ...lTags() }) + } + } + + await this.extendsLocalExpiration() + + await this.purgeRemoteExpired() + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } + + private async extendsLocalExpiration () { + const expired = await VideoRedundancyModel.listLocalExpired() + + for (const redundancyModel of expired) { + try { + const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) + + // If the admin disabled the redundancy, remove this redundancy instead of extending it + if (!redundancyConfig) { + logger.info( + 'Destroying redundancy %s because the redundancy %s does not exist anymore.', + redundancyModel.url, redundancyModel.strategy + ) + + await removeVideoRedundancy(redundancyModel) + continue + } + + const { totalUsed } = await VideoRedundancyModel.getStats(redundancyConfig.strategy) + + // If the admin decreased the cache size, remove this redundancy instead of extending it + if (totalUsed > redundancyConfig.size) { + logger.info('Destroying redundancy %s because the cache size %s is too heavy.', redundancyModel.url, redundancyModel.strategy) + + await removeVideoRedundancy(redundancyModel) + continue + } + + await this.extendsRedundancy(redundancyModel) + } catch (err) { + logger.error( + 'Cannot extend or remove expiration of %s video from our redundancy system.', + this.buildEntryLogId(redundancyModel), { err, ...lTags(redundancyModel.getVideoUUID()) } + ) + } + } + } + + private async extendsRedundancy (redundancyModel: MVideoRedundancyVideo) { + const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy) + // Redundancy strategy disabled, remove our redundancy instead of extending expiration + if (!redundancy) { + await removeVideoRedundancy(redundancyModel) + return + } + + await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime) + } + + private async purgeRemoteExpired () { + const expired = await VideoRedundancyModel.listRemoteExpired() + + for (const redundancyModel of expired) { + try { + await removeVideoRedundancy(redundancyModel) + } catch (err) { + logger.error( + 'Cannot remove redundancy %s from our redundancy system.', + this.buildEntryLogId(redundancyModel), lTags(redundancyModel.getVideoUUID()) + ) + } + } + } + + private findVideoToDuplicate (cache: VideosRedundancyStrategy) { + if (cache.strategy === 'most-views') { + return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) + } + + if (cache.strategy === 'trending') { + return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR) + } + + if (cache.strategy === 'recently-added') { + const minViews = cache.minViews + return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews) + } + } + + private async createVideoRedundancies (data: CandidateToDuplicate) { + const video = await this.loadAndRefreshVideo(data.video.url) + + if (!video) { + logger.info('Video %s we want to duplicate does not existing anymore, skipping.', data.video.url, lTags(data.video.uuid)) + + return + } + + for (const file of data.files) { + const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id) + if (existingRedundancy) { + await this.extendsRedundancy(existingRedundancy) + + continue + } + + await this.createVideoFileRedundancy(data.redundancy, video, file) + } + + for (const streamingPlaylist of data.streamingPlaylists) { + const existingRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(streamingPlaylist.id) + if (existingRedundancy) { + await this.extendsRedundancy(existingRedundancy) + + continue + } + + await this.createStreamingPlaylistRedundancy(data.redundancy, video, streamingPlaylist) + } + } + + private async createVideoFileRedundancy (redundancy: VideosRedundancyStrategy | null, video: MVideoAccountLight, fileArg: MVideoFile) { + let strategy = 'manual' + let expiresOn: Date = null + + if (redundancy) { + strategy = redundancy.strategy + expiresOn = this.buildNewExpiration(redundancy.minLifetime) + } + + const file = fileArg as MVideoFileVideo + file.Video = video + + const serverActor = await getServerActor() + + logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy, lTags(video.uuid)) + + const tmpPath = await downloadWebTorrentVideo({ uri: file.torrentUrl }, VIDEO_IMPORT_TIMEOUT) + + const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, file.filename) + await move(tmpPath, destPath, { overwrite: true }) + + const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({ + expiresOn, + url: getLocalVideoCacheFileActivityPubUrl(file), + fileUrl: generateWebVideoRedundancyUrl(file), + strategy, + videoFileId: file.id, + actorId: serverActor.id + }) + + createdModel.VideoFile = file + + await sendCreateCacheFile(serverActor, video, createdModel) + + logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url, lTags(video.uuid)) + } + + private async createStreamingPlaylistRedundancy ( + redundancy: VideosRedundancyStrategy, + video: MVideoAccountLight, + playlistArg: MStreamingPlaylistFiles + ) { + let strategy = 'manual' + let expiresOn: Date = null + + if (redundancy) { + strategy = redundancy.strategy + expiresOn = this.buildNewExpiration(redundancy.minLifetime) + } + + const playlist = Object.assign(playlistArg, { Video: video }) + const serverActor = await getServerActor() + + logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy, lTags(video.uuid)) + + const destDirectory = join(DIRECTORIES.HLS_REDUNDANCY, video.uuid) + const masterPlaylistUrl = playlist.getMasterPlaylistUrl(video) + + const maxSizeKB = this.getTotalFileSizes([], [ playlist ]) / 1000 + const toleranceKB = maxSizeKB + ((5 * maxSizeKB) / 100) // 5% more tolerance + await downloadPlaylistSegments(masterPlaylistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT, toleranceKB) + + const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({ + expiresOn, + url: getLocalVideoCacheStreamingPlaylistActivityPubUrl(video, playlist), + fileUrl: generateHLSRedundancyUrl(video, playlistArg), + strategy, + videoStreamingPlaylistId: playlist.id, + actorId: serverActor.id + }) + + createdModel.VideoStreamingPlaylist = playlist + + await sendCreateCacheFile(serverActor, video, createdModel) + + logger.info('Duplicated playlist %s -> %s.', masterPlaylistUrl, createdModel.url, lTags(video.uuid)) + } + + private async extendsExpirationOf (redundancy: MVideoRedundancyVideo, expiresAfterMs: number) { + logger.info('Extending expiration of %s.', redundancy.url, lTags(redundancy.getVideoUUID())) + + const serverActor = await getServerActor() + + redundancy.expiresOn = this.buildNewExpiration(expiresAfterMs) + await redundancy.save() + + await sendUpdateCacheFile(serverActor, redundancy) + } + + private async purgeCacheIfNeeded (candidateToDuplicate: CandidateToDuplicate) { + while (await this.isTooHeavy(candidateToDuplicate)) { + const redundancy = candidateToDuplicate.redundancy + const toDelete = await VideoRedundancyModel.loadOldestLocalExpired(redundancy.strategy, redundancy.minLifetime) + if (!toDelete) return + + const videoId = toDelete.VideoFile + ? toDelete.VideoFile.videoId + : toDelete.VideoStreamingPlaylist.videoId + + const redundancies = await VideoRedundancyModel.listLocalByVideoId(videoId) + + for (const redundancy of redundancies) { + await removeVideoRedundancy(redundancy) + } + } + } + + private async isTooHeavy (candidateToDuplicate: CandidateToDuplicate) { + const maxSize = candidateToDuplicate.redundancy.size + + const { totalUsed: alreadyUsed } = await VideoRedundancyModel.getStats(candidateToDuplicate.redundancy.strategy) + + const videoSize = this.getTotalFileSizes(candidateToDuplicate.files, candidateToDuplicate.streamingPlaylists) + const willUse = alreadyUsed + videoSize + + logger.debug('Checking candidate size.', { maxSize, alreadyUsed, videoSize, willUse, ...lTags(candidateToDuplicate.video.uuid) }) + + return willUse > maxSize + } + + private buildNewExpiration (expiresAfterMs: number) { + return new Date(Date.now() + expiresAfterMs) + } + + private buildEntryLogId (object: MVideoRedundancyFileVideo | MVideoRedundancyStreamingPlaylistVideo) { + if (isMVideoRedundancyFileVideo(object)) return `${object.VideoFile.Video.url}-${object.VideoFile.resolution}` + + return `${object.VideoStreamingPlaylist.getMasterPlaylistUrl(object.VideoStreamingPlaylist.Video)}` + } + + private getTotalFileSizes (files: MVideoFile[], playlists: MStreamingPlaylistFiles[]): number { + const fileReducer = (previous: number, current: MVideoFile) => previous + current.size + + let allFiles = files + for (const p of playlists) { + allFiles = allFiles.concat(p.VideoFiles) + } + + return allFiles.reduce(fileReducer, 0) + } + + private async loadAndRefreshVideo (videoUrl: string) { + // We need more attributes and check if the video still exists + const getVideoOptions = { + videoObject: videoUrl, + syncParam: { rates: false, shares: false, comments: false, refreshVideo: true }, + fetchType: 'all' as 'all' + } + const { video } = await getOrCreateAPVideo(getVideoOptions) + + return video + } +} diff --git a/server/server/lib/schedulers/youtube-dl-update-scheduler.ts b/server/server/lib/schedulers/youtube-dl-update-scheduler.ts new file mode 100644 index 000000000..9e3cbf7b2 --- /dev/null +++ b/server/server/lib/schedulers/youtube-dl-update-scheduler.ts @@ -0,0 +1,22 @@ +import { YoutubeDLCLI } from '@server/helpers/youtube-dl/index.js' +import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants.js' +import { AbstractScheduler } from './abstract-scheduler.js' + +export class YoutubeDlUpdateScheduler extends AbstractScheduler { + + private static instance: AbstractScheduler + + protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.YOUTUBE_DL_UPDATE + + private constructor () { + super() + } + + protected internalExecute () { + return YoutubeDLCLI.updateYoutubeDLBinary() + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/server/lib/search.ts b/server/server/lib/search.ts new file mode 100644 index 000000000..567eced7c --- /dev/null +++ b/server/server/lib/search.ts @@ -0,0 +1,49 @@ +import express from 'express' +import { CONFIG } from '@server/initializers/config.js' +import { AccountBlocklistModel } from '@server/models/account/account-blocklist.js' +import { getServerActor } from '@server/models/application/application.js' +import { ServerBlocklistModel } from '@server/models/server/server-blocklist.js' +import { SearchTargetQuery } from '@peertube/peertube-models' + +function isSearchIndexSearch (query: SearchTargetQuery) { + if (query.searchTarget === 'search-index') return true + + const searchIndexConfig = CONFIG.SEARCH.SEARCH_INDEX + + if (searchIndexConfig.ENABLED !== true) return false + + if (searchIndexConfig.IS_DEFAULT_SEARCH && !query.searchTarget) return true + + return false +} + +async function buildMutedForSearchIndex (res: express.Response) { + const serverActor = await getServerActor() + const accountIds = [ serverActor.Account.id ] + + if (res.locals.oauth) { + accountIds.push(res.locals.oauth.token.User.Account.id) + } + + const [ blockedHosts, blockedAccounts ] = await Promise.all([ + ServerBlocklistModel.listHostsBlockedBy(accountIds), + AccountBlocklistModel.listHandlesBlockedBy(accountIds) + ]) + + return { + blockedHosts, + blockedAccounts + } +} + +function isURISearch (search: string) { + if (!search) return false + + return search.startsWith('http://') || search.startsWith('https://') +} + +export { + isSearchIndexSearch, + buildMutedForSearchIndex, + isURISearch +} diff --git a/server/server/lib/server-config-manager.ts b/server/server/lib/server-config-manager.ts new file mode 100644 index 000000000..8b3b957fe --- /dev/null +++ b/server/server/lib/server-config-manager.ts @@ -0,0 +1,390 @@ +import { + HTMLServerConfig, + RegisteredExternalAuthConfig, + RegisteredIdAndPassAuthConfig, + ServerConfig, + VideoResolutionType +} from '@peertube/peertube-models' +import { getServerCommit } from '@server/helpers/version.js' +import { CONFIG, isEmailEnabled } from '@server/initializers/config.js' +import { CONSTRAINTS_FIELDS, DEFAULT_THEME_NAME, PEERTUBE_VERSION } from '@server/initializers/constants.js' +import { isSignupAllowed, isSignupAllowedForCurrentIP } from '@server/lib/signup.js' +import { ActorCustomPageModel } from '@server/models/account/actor-custom-page.js' +import { PluginModel } from '@server/models/server/plugin.js' +import { Hooks } from './plugins/hooks.js' +import { PluginManager } from './plugins/plugin-manager.js' +import { getThemeOrDefault } from './plugins/theme-utils.js' +import { VideoTranscodingProfilesManager } from './transcoding/default-transcoding-profiles.js' + +/** + * + * Used to send the server config to clients (using REST/API or plugins API) + * We need a singleton class to manage config state depending on external events (to build menu entries etc) + * + */ + +class ServerConfigManager { + + private static instance: ServerConfigManager + + private serverCommit: string + + private homepageEnabled = false + + private constructor () {} + + async init () { + const instanceHomepage = await ActorCustomPageModel.loadInstanceHomepage() + + this.updateHomepageState(instanceHomepage?.content) + } + + updateHomepageState (content: string) { + this.homepageEnabled = !!content + } + + async getHTMLServerConfig (): Promise { + if (this.serverCommit === undefined) this.serverCommit = await getServerCommit() + + const defaultTheme = getThemeOrDefault(CONFIG.THEME.DEFAULT, DEFAULT_THEME_NAME) + + return { + client: { + videos: { + miniature: { + displayAuthorAvatar: CONFIG.CLIENT.VIDEOS.MINIATURE.DISPLAY_AUTHOR_AVATAR, + preferAuthorDisplayName: CONFIG.CLIENT.VIDEOS.MINIATURE.PREFER_AUTHOR_DISPLAY_NAME + }, + resumableUpload: { + maxChunkSize: CONFIG.CLIENT.VIDEOS.RESUMABLE_UPLOAD.MAX_CHUNK_SIZE + } + }, + menu: { + login: { + redirectOnSingleExternalAuth: CONFIG.CLIENT.MENU.LOGIN.REDIRECT_ON_SINGLE_EXTERNAL_AUTH + } + } + }, + + defaults: { + publish: { + downloadEnabled: CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED, + commentsEnabled: CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED, + privacy: CONFIG.DEFAULTS.PUBLISH.PRIVACY, + licence: CONFIG.DEFAULTS.PUBLISH.LICENCE + }, + p2p: { + webapp: { + enabled: CONFIG.DEFAULTS.P2P.WEBAPP.ENABLED + }, + embed: { + enabled: CONFIG.DEFAULTS.P2P.EMBED.ENABLED + } + } + }, + + webadmin: { + configuration: { + edition: { + allowed: CONFIG.WEBADMIN.CONFIGURATION.EDITION.ALLOWED + } + } + }, + + instance: { + name: CONFIG.INSTANCE.NAME, + shortDescription: CONFIG.INSTANCE.SHORT_DESCRIPTION, + isNSFW: CONFIG.INSTANCE.IS_NSFW, + defaultNSFWPolicy: CONFIG.INSTANCE.DEFAULT_NSFW_POLICY, + defaultClientRoute: CONFIG.INSTANCE.DEFAULT_CLIENT_ROUTE, + customizations: { + javascript: CONFIG.INSTANCE.CUSTOMIZATIONS.JAVASCRIPT, + css: CONFIG.INSTANCE.CUSTOMIZATIONS.CSS + } + }, + search: { + remoteUri: { + users: CONFIG.SEARCH.REMOTE_URI.USERS, + anonymous: CONFIG.SEARCH.REMOTE_URI.ANONYMOUS + }, + searchIndex: { + enabled: CONFIG.SEARCH.SEARCH_INDEX.ENABLED, + url: CONFIG.SEARCH.SEARCH_INDEX.URL, + disableLocalSearch: CONFIG.SEARCH.SEARCH_INDEX.DISABLE_LOCAL_SEARCH, + isDefaultSearch: CONFIG.SEARCH.SEARCH_INDEX.IS_DEFAULT_SEARCH + } + }, + plugin: { + registered: this.getRegisteredPlugins(), + registeredExternalAuths: this.getExternalAuthsPlugins(), + registeredIdAndPassAuths: this.getIdAndPassAuthPlugins() + }, + theme: { + registered: this.getRegisteredThemes(), + default: defaultTheme + }, + email: { + enabled: isEmailEnabled() + }, + contactForm: { + enabled: CONFIG.CONTACT_FORM.ENABLED + }, + serverVersion: PEERTUBE_VERSION, + serverCommit: this.serverCommit, + transcoding: { + remoteRunners: { + enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED + }, + hls: { + enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.HLS.ENABLED + }, + web_videos: { + enabled: CONFIG.TRANSCODING.ENABLED && CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED + }, + enabledResolutions: this.getEnabledResolutions('vod'), + profile: CONFIG.TRANSCODING.PROFILE, + availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('vod') + }, + live: { + enabled: CONFIG.LIVE.ENABLED, + + allowReplay: CONFIG.LIVE.ALLOW_REPLAY, + latencySetting: { + enabled: CONFIG.LIVE.LATENCY_SETTING.ENABLED + }, + + maxDuration: CONFIG.LIVE.MAX_DURATION, + maxInstanceLives: CONFIG.LIVE.MAX_INSTANCE_LIVES, + maxUserLives: CONFIG.LIVE.MAX_USER_LIVES, + + transcoding: { + enabled: CONFIG.LIVE.TRANSCODING.ENABLED, + remoteRunners: { + enabled: CONFIG.LIVE.TRANSCODING.ENABLED && CONFIG.LIVE.TRANSCODING.REMOTE_RUNNERS.ENABLED + }, + enabledResolutions: this.getEnabledResolutions('live'), + profile: CONFIG.LIVE.TRANSCODING.PROFILE, + availableProfiles: VideoTranscodingProfilesManager.Instance.getAvailableProfiles('live') + }, + + rtmp: { + port: CONFIG.LIVE.RTMP.PORT + } + }, + videoStudio: { + enabled: CONFIG.VIDEO_STUDIO.ENABLED, + remoteRunners: { + enabled: CONFIG.VIDEO_STUDIO.REMOTE_RUNNERS.ENABLED + } + }, + videoFile: { + update: { + enabled: CONFIG.VIDEO_FILE.UPDATE.ENABLED + } + }, + import: { + videos: { + http: { + enabled: CONFIG.IMPORT.VIDEOS.HTTP.ENABLED + }, + torrent: { + enabled: CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED + } + }, + videoChannelSynchronization: { + enabled: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED + } + }, + autoBlacklist: { + videos: { + ofUsers: { + enabled: CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED + } + } + }, + avatar: { + file: { + size: { + max: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max + }, + extensions: CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME + } + }, + banner: { + file: { + size: { + max: CONSTRAINTS_FIELDS.ACTORS.IMAGE.FILE_SIZE.max + }, + extensions: CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME + } + }, + video: { + image: { + extensions: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.EXTNAME, + size: { + max: CONSTRAINTS_FIELDS.VIDEOS.IMAGE.FILE_SIZE.max + } + }, + file: { + extensions: CONSTRAINTS_FIELDS.VIDEOS.EXTNAME + } + }, + videoCaption: { + file: { + size: { + max: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max + }, + extensions: CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.EXTNAME + } + }, + user: { + videoQuota: CONFIG.USER.VIDEO_QUOTA, + videoQuotaDaily: CONFIG.USER.VIDEO_QUOTA_DAILY + }, + videoChannels: { + maxPerUser: CONFIG.VIDEO_CHANNELS.MAX_PER_USER + }, + trending: { + videos: { + intervalDays: CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS, + algorithms: { + enabled: CONFIG.TRENDING.VIDEOS.ALGORITHMS.ENABLED, + default: CONFIG.TRENDING.VIDEOS.ALGORITHMS.DEFAULT + } + } + }, + tracker: { + enabled: CONFIG.TRACKER.ENABLED + }, + + followings: { + instance: { + autoFollowIndex: { + indexUrl: CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.INDEX_URL + } + } + }, + + broadcastMessage: { + enabled: CONFIG.BROADCAST_MESSAGE.ENABLED, + message: CONFIG.BROADCAST_MESSAGE.MESSAGE, + level: CONFIG.BROADCAST_MESSAGE.LEVEL, + dismissable: CONFIG.BROADCAST_MESSAGE.DISMISSABLE + }, + + homepage: { + enabled: this.homepageEnabled + } + } + } + + async getServerConfig (ip?: string): Promise { + const { allowed } = await Hooks.wrapPromiseFun( + isSignupAllowed, + + { + ip, + signupMode: CONFIG.SIGNUP.REQUIRES_APPROVAL + ? 'request-registration' + : 'direct-registration' + }, + + CONFIG.SIGNUP.REQUIRES_APPROVAL + ? 'filter:api.user.request-signup.allowed.result' + : 'filter:api.user.signup.allowed.result' + ) + + const allowedForCurrentIP = isSignupAllowedForCurrentIP(ip) + + const signup = { + allowed, + allowedForCurrentIP, + minimumAge: CONFIG.SIGNUP.MINIMUM_AGE, + requiresApproval: CONFIG.SIGNUP.REQUIRES_APPROVAL, + requiresEmailVerification: CONFIG.SIGNUP.REQUIRES_EMAIL_VERIFICATION + } + + const htmlConfig = await this.getHTMLServerConfig() + + return { ...htmlConfig, signup } + } + + getRegisteredThemes () { + return PluginManager.Instance.getRegisteredThemes() + .map(t => ({ + npmName: PluginModel.buildNpmName(t.name, t.type), + name: t.name, + version: t.version, + description: t.description, + css: t.css, + clientScripts: t.clientScripts + })) + } + + getRegisteredPlugins () { + return PluginManager.Instance.getRegisteredPlugins() + .map(p => ({ + npmName: PluginModel.buildNpmName(p.name, p.type), + name: p.name, + version: p.version, + description: p.description, + clientScripts: p.clientScripts + })) + } + + getEnabledResolutions (type: 'vod' | 'live') { + const transcoding = type === 'vod' + ? CONFIG.TRANSCODING + : CONFIG.LIVE.TRANSCODING + + return Object.keys(transcoding.RESOLUTIONS) + .filter(key => transcoding.ENABLED && transcoding.RESOLUTIONS[key] === true) + .map(r => parseInt(r, 10) as VideoResolutionType) + } + + private getIdAndPassAuthPlugins () { + const result: RegisteredIdAndPassAuthConfig[] = [] + + for (const p of PluginManager.Instance.getIdAndPassAuths()) { + for (const auth of p.idAndPassAuths) { + result.push({ + npmName: p.npmName, + name: p.name, + version: p.version, + authName: auth.authName, + weight: auth.getWeight() + }) + } + } + + return result + } + + private getExternalAuthsPlugins () { + const result: RegisteredExternalAuthConfig[] = [] + + for (const p of PluginManager.Instance.getExternalAuths()) { + for (const auth of p.externalAuths) { + result.push({ + npmName: p.npmName, + name: p.name, + version: p.version, + authName: auth.authName, + authDisplayName: auth.authDisplayName() + }) + } + } + + return result + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} + +// --------------------------------------------------------------------------- + +export { + ServerConfigManager +} diff --git a/server/server/lib/signup.ts b/server/server/lib/signup.ts new file mode 100644 index 000000000..5ad5be9c3 --- /dev/null +++ b/server/server/lib/signup.ts @@ -0,0 +1,74 @@ +import ipaddr from 'ipaddr.js' +import isCidr from 'is-cidr' +import { CONFIG } from '../initializers/config.js' +import { UserModel } from '../models/user/user.js' + +export type SignupMode = 'direct-registration' | 'request-registration' + +async function isSignupAllowed (options: { + signupMode: SignupMode + + ip: string // For plugins + body?: any +}): Promise<{ allowed: boolean, errorMessage?: string }> { + const { signupMode } = options + + if (CONFIG.SIGNUP.ENABLED === false) { + return { allowed: false, errorMessage: 'User registration is not allowed' } + } + + if (signupMode === 'direct-registration' && CONFIG.SIGNUP.REQUIRES_APPROVAL === true) { + return { allowed: false, errorMessage: 'User registration requires approval' } + } + + // No limit and signup is enabled + if (CONFIG.SIGNUP.LIMIT === -1) { + return { allowed: true } + } + + const totalUsers = await UserModel.countTotal() + + return { allowed: totalUsers < CONFIG.SIGNUP.LIMIT, errorMessage: 'User limit is reached on this instance' } +} + +function isSignupAllowedForCurrentIP (ip: string) { + if (!ip) return false + + const addr = ipaddr.parse(ip) + const excludeList = [ 'blacklist' ] + let matched = '' + + // if there is a valid, non-empty whitelist, we exclude all unknown addresses too + if (CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr(cidr)).length > 0) { + excludeList.push('unknown') + } + + if (addr.kind() === 'ipv4') { + const addrV4 = ipaddr.IPv4.parse(ip) + const rangeList = { + whitelist: CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr.v4(cidr)) + .map(cidr => ipaddr.IPv4.parseCIDR(cidr)), + blacklist: CONFIG.SIGNUP.FILTERS.CIDR.BLACKLIST.filter(cidr => isCidr.v4(cidr)) + .map(cidr => ipaddr.IPv4.parseCIDR(cidr)) + } + matched = ipaddr.subnetMatch(addrV4, rangeList, 'unknown') + } else if (addr.kind() === 'ipv6') { + const addrV6 = ipaddr.IPv6.parse(ip) + const rangeList = { + whitelist: CONFIG.SIGNUP.FILTERS.CIDR.WHITELIST.filter(cidr => isCidr.v6(cidr)) + .map(cidr => ipaddr.IPv6.parseCIDR(cidr)), + blacklist: CONFIG.SIGNUP.FILTERS.CIDR.BLACKLIST.filter(cidr => isCidr.v6(cidr)) + .map(cidr => ipaddr.IPv6.parseCIDR(cidr)) + } + matched = ipaddr.subnetMatch(addrV6, rangeList, 'unknown') + } + + return !excludeList.includes(matched) +} + +// --------------------------------------------------------------------------- + +export { + isSignupAllowed, + isSignupAllowedForCurrentIP +} diff --git a/server/server/lib/stat-manager.ts b/server/server/lib/stat-manager.ts new file mode 100644 index 000000000..f27266571 --- /dev/null +++ b/server/server/lib/stat-manager.ts @@ -0,0 +1,182 @@ +import Bluebird from 'bluebird' +import { CONFIG } from '@server/initializers/config.js' +import { ActorFollowModel } from '@server/models/actor/actor-follow.js' +import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy.js' +import { UserModel } from '@server/models/user/user.js' +import { VideoModel } from '@server/models/video/video.js' +import { VideoChannelModel } from '@server/models/video/video-channel.js' +import { VideoCommentModel } from '@server/models/video/video-comment.js' +import { VideoFileModel } from '@server/models/video/video-file.js' +import { VideoPlaylistModel } from '@server/models/video/video-playlist.js' +import { ActivityType, ServerStats, VideoRedundancyStrategyWithManual } from '@peertube/peertube-models' + +class StatsManager { + + private static instance: StatsManager + + private readonly instanceStartDate = new Date() + + private readonly inboxMessages = { + processed: 0, + errors: 0, + successes: 0, + waiting: 0, + errorsPerType: this.buildAPPerType(), + successesPerType: this.buildAPPerType() + } + + private constructor () {} + + updateInboxWaiting (inboxMessagesWaiting: number) { + this.inboxMessages.waiting = inboxMessagesWaiting + } + + addInboxProcessedSuccess (type: ActivityType) { + this.inboxMessages.processed++ + this.inboxMessages.successes++ + this.inboxMessages.successesPerType[type]++ + } + + addInboxProcessedError (type: ActivityType) { + this.inboxMessages.processed++ + this.inboxMessages.errors++ + this.inboxMessages.errorsPerType[type]++ + } + + async getStats () { + const { totalLocalVideos, totalLocalVideoViews, totalVideos } = await VideoModel.getStats() + const { totalLocalVideoComments, totalVideoComments } = await VideoCommentModel.getStats() + const { totalUsers, totalDailyActiveUsers, totalWeeklyActiveUsers, totalMonthlyActiveUsers } = await UserModel.getStats() + const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats() + const { totalLocalVideoFilesSize } = await VideoFileModel.getStats() + const { + totalLocalVideoChannels, + totalLocalDailyActiveVideoChannels, + totalLocalWeeklyActiveVideoChannels, + totalLocalMonthlyActiveVideoChannels + } = await VideoChannelModel.getStats() + const { totalLocalPlaylists } = await VideoPlaylistModel.getStats() + + const videosRedundancyStats = await this.buildRedundancyStats() + + const data: ServerStats = { + totalUsers, + totalDailyActiveUsers, + totalWeeklyActiveUsers, + totalMonthlyActiveUsers, + + totalLocalVideos, + totalLocalVideoViews, + totalLocalVideoComments, + totalLocalVideoFilesSize, + + totalVideos, + totalVideoComments, + + totalLocalVideoChannels, + totalLocalDailyActiveVideoChannels, + totalLocalWeeklyActiveVideoChannels, + totalLocalMonthlyActiveVideoChannels, + + totalLocalPlaylists, + + totalInstanceFollowers, + totalInstanceFollowing, + + videosRedundancy: videosRedundancyStats, + + ...this.buildAPStats() + } + + return data + } + + private buildActivityPubMessagesProcessedPerSecond () { + const now = new Date() + const startedSeconds = (now.getTime() - this.instanceStartDate.getTime()) / 1000 + + return this.inboxMessages.processed / startedSeconds + } + + private buildRedundancyStats () { + const strategies = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES + .map(r => ({ + strategy: r.strategy as VideoRedundancyStrategyWithManual, + size: r.size + })) + + strategies.push({ strategy: 'manual', size: null }) + + return Bluebird.mapSeries(strategies, r => { + return VideoRedundancyModel.getStats(r.strategy) + .then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size })) + }) + } + + private buildAPPerType () { + return { + Create: 0, + Update: 0, + Delete: 0, + Follow: 0, + Accept: 0, + Reject: 0, + Announce: 0, + Undo: 0, + Like: 0, + Dislike: 0, + Flag: 0, + View: 0 + } + } + + private buildAPStats () { + return { + totalActivityPubMessagesProcessed: this.inboxMessages.processed, + + totalActivityPubMessagesSuccesses: this.inboxMessages.successes, + + // Dirty, but simpler and with type checking + totalActivityPubCreateMessagesSuccesses: this.inboxMessages.successesPerType.Create, + totalActivityPubUpdateMessagesSuccesses: this.inboxMessages.successesPerType.Update, + totalActivityPubDeleteMessagesSuccesses: this.inboxMessages.successesPerType.Delete, + totalActivityPubFollowMessagesSuccesses: this.inboxMessages.successesPerType.Follow, + totalActivityPubAcceptMessagesSuccesses: this.inboxMessages.successesPerType.Accept, + totalActivityPubRejectMessagesSuccesses: this.inboxMessages.successesPerType.Reject, + totalActivityPubAnnounceMessagesSuccesses: this.inboxMessages.successesPerType.Announce, + totalActivityPubUndoMessagesSuccesses: this.inboxMessages.successesPerType.Undo, + totalActivityPubLikeMessagesSuccesses: this.inboxMessages.successesPerType.Like, + totalActivityPubDislikeMessagesSuccesses: this.inboxMessages.successesPerType.Dislike, + totalActivityPubFlagMessagesSuccesses: this.inboxMessages.successesPerType.Flag, + totalActivityPubViewMessagesSuccesses: this.inboxMessages.successesPerType.View, + + totalActivityPubCreateMessagesErrors: this.inboxMessages.errorsPerType.Create, + totalActivityPubUpdateMessagesErrors: this.inboxMessages.errorsPerType.Update, + totalActivityPubDeleteMessagesErrors: this.inboxMessages.errorsPerType.Delete, + totalActivityPubFollowMessagesErrors: this.inboxMessages.errorsPerType.Follow, + totalActivityPubAcceptMessagesErrors: this.inboxMessages.errorsPerType.Accept, + totalActivityPubRejectMessagesErrors: this.inboxMessages.errorsPerType.Reject, + totalActivityPubAnnounceMessagesErrors: this.inboxMessages.errorsPerType.Announce, + totalActivityPubUndoMessagesErrors: this.inboxMessages.errorsPerType.Undo, + totalActivityPubLikeMessagesErrors: this.inboxMessages.errorsPerType.Like, + totalActivityPubDislikeMessagesErrors: this.inboxMessages.errorsPerType.Dislike, + totalActivityPubFlagMessagesErrors: this.inboxMessages.errorsPerType.Flag, + totalActivityPubViewMessagesErrors: this.inboxMessages.errorsPerType.View, + + totalActivityPubMessagesErrors: this.inboxMessages.errors, + + activityPubMessagesProcessedPerSecond: this.buildActivityPubMessagesProcessedPerSecond(), + totalActivityPubMessagesWaiting: this.inboxMessages.waiting + } + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} + +// --------------------------------------------------------------------------- + +export { + StatsManager +} diff --git a/server/server/lib/sync-channel.ts b/server/server/lib/sync-channel.ts new file mode 100644 index 000000000..b0cbf43ba --- /dev/null +++ b/server/server/lib/sync-channel.ts @@ -0,0 +1,111 @@ +import { logger } from '@server/helpers/logger.js' +import { YoutubeDLWrapper } from '@server/helpers/youtube-dl/index.js' +import { CONFIG } from '@server/initializers/config.js' +import { buildYoutubeDLImport } from '@server/lib/video-pre-import.js' +import { UserModel } from '@server/models/user/user.js' +import { VideoImportModel } from '@server/models/video/video-import.js' +import { MChannel, MChannelAccountDefault, MChannelSync } from '@server/types/models/index.js' +import { VideoChannelSyncState, VideoPrivacy } from '@peertube/peertube-models' +import { CreateJobArgument, JobQueue } from './job-queue/index.js' +import { ServerConfigManager } from './server-config-manager.js' + +export async function synchronizeChannel (options: { + channel: MChannelAccountDefault + externalChannelUrl: string + videosCountLimit: number + channelSync?: MChannelSync + onlyAfter?: Date +}) { + const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options + + if (channelSync) { + channelSync.state = VideoChannelSyncState.PROCESSING + channelSync.lastSyncAt = new Date() + await channelSync.save() + } + + try { + const user = await UserModel.loadByChannelActorId(channel.actorId) + const youtubeDL = new YoutubeDLWrapper( + externalChannelUrl, + ServerConfigManager.Instance.getEnabledResolutions('vod'), + CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION + ) + + const targetUrls = await youtubeDL.getInfoForListImport({ latestVideosCount: videosCountLimit }) + + logger.info( + 'Fetched %d candidate URLs for sync channel %s.', + targetUrls.length, channel.Actor.preferredUsername, { targetUrls } + ) + + if (targetUrls.length === 0) { + if (channelSync) { + channelSync.state = VideoChannelSyncState.SYNCED + await channelSync.save() + } + + return + } + + const children: CreateJobArgument[] = [] + + for (const targetUrl of targetUrls) { + if (await skipImport(channel, targetUrl, onlyAfter)) continue + + const { job } = await buildYoutubeDLImport({ + user, + channel, + targetUrl, + channelSync, + importDataOverride: { + privacy: VideoPrivacy.PUBLIC + } + }) + + children.push(job) + } + + // Will update the channel sync status + const parent: CreateJobArgument = { + type: 'after-video-channel-import', + payload: { + channelSyncId: channelSync?.id + } + } + + await JobQueue.Instance.createJobWithChildren(parent, children) + } catch (err) { + logger.error(`Failed to import ${externalChannelUrl} in channel ${channel.name}`, { err }) + channelSync.state = VideoChannelSyncState.FAILED + await channelSync.save() + } +} + +// --------------------------------------------------------------------------- + +async function skipImport (channel: MChannel, targetUrl: string, onlyAfter?: Date) { + if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) { + logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', targetUrl, channel.name) + return true + } + + if (onlyAfter) { + const youtubeDL = new YoutubeDLWrapper( + targetUrl, + ServerConfigManager.Instance.getEnabledResolutions('vod'), + CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION + ) + + const videoInfo = await youtubeDL.getInfoForDownload() + + const onlyAfterWithoutTime = new Date(onlyAfter) + onlyAfterWithoutTime.setHours(0, 0, 0, 0) + + if (videoInfo.originallyPublishedAtWithoutTime.getTime() < onlyAfterWithoutTime.getTime()) { + return true + } + } + + return false +} diff --git a/server/server/lib/thumbnail.ts b/server/server/lib/thumbnail.ts new file mode 100644 index 000000000..e5424973b --- /dev/null +++ b/server/server/lib/thumbnail.ts @@ -0,0 +1,327 @@ +import { join } from 'path' +import { ThumbnailType, ThumbnailType_Type } from '@peertube/peertube-models' +import { generateImageFilename, generateImageFromVideoFile } from '../helpers/image-utils.js' +import { CONFIG } from '../initializers/config.js' +import { ASSETS_PATH, PREVIEWS_SIZE, THUMBNAILS_SIZE } from '../initializers/constants.js' +import { ThumbnailModel } from '../models/video/thumbnail.js' +import { MVideoFile, MVideoThumbnail, MVideoUUID, MVideoWithAllFiles } from '../types/models/index.js' +import { MThumbnail } from '../types/models/video/thumbnail.js' +import { MVideoPlaylistThumbnail } from '../types/models/video/video-playlist.js' +import { VideoPathManager } from './video-path-manager.js' +import { downloadImageFromWorker, processImageFromWorker } from './worker/parent-process.js' + +type ImageSize = { height?: number, width?: number } + +function updateLocalPlaylistMiniatureFromExisting (options: { + inputPath: string + playlist: MVideoPlaylistThumbnail + automaticallyGenerated: boolean + keepOriginal?: boolean // default to false + size?: ImageSize +}) { + const { inputPath, playlist, automaticallyGenerated, keepOriginal = false, size } = options + const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size) + const type = ThumbnailType.MINIATURE + + const thumbnailCreator = () => { + return processImageFromWorker({ path: inputPath, destination: outputPath, newSize: { width, height }, keepOriginal }) + } + + return updateThumbnailFromFunction({ + thumbnailCreator, + filename, + height, + width, + type, + automaticallyGenerated, + onDisk: true, + existingThumbnail + }) +} + +function updateRemotePlaylistMiniatureFromUrl (options: { + downloadUrl: string + playlist: MVideoPlaylistThumbnail + size?: ImageSize +}) { + const { downloadUrl, playlist, size } = options + const { filename, basePath, height, width, existingThumbnail } = buildMetadataFromPlaylist(playlist, size) + const type = ThumbnailType.MINIATURE + + // Only save the file URL if it is a remote playlist + const fileUrl = playlist.isOwned() + ? null + : downloadUrl + + const thumbnailCreator = () => { + return downloadImageFromWorker({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } }) + } + + return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl, onDisk: true }) +} + +function updateLocalVideoMiniatureFromExisting (options: { + inputPath: string + video: MVideoThumbnail + type: ThumbnailType_Type + automaticallyGenerated: boolean + size?: ImageSize + keepOriginal?: boolean // default to false +}) { + const { inputPath, video, type, automaticallyGenerated, size, keepOriginal = false } = options + + const { filename, outputPath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) + + const thumbnailCreator = () => { + return processImageFromWorker({ path: inputPath, destination: outputPath, newSize: { width, height }, keepOriginal }) + } + + return updateThumbnailFromFunction({ + thumbnailCreator, + filename, + height, + width, + type, + automaticallyGenerated, + existingThumbnail, + onDisk: true + }) +} + +function generateLocalVideoMiniature (options: { + video: MVideoThumbnail + videoFile: MVideoFile + type: ThumbnailType_Type +}) { + const { video, videoFile, type } = options + + return VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), input => { + const { filename, basePath, height, width, existingThumbnail, outputPath } = buildMetadataFromVideo(video, type) + + const thumbnailCreator = videoFile.isAudio() + ? () => processImageFromWorker({ + path: ASSETS_PATH.DEFAULT_AUDIO_BACKGROUND, + destination: outputPath, + newSize: { width, height }, + keepOriginal: true + }) + : () => generateImageFromVideoFile({ + fromPath: input, + folder: basePath, + imageName: filename, + size: { height, width } + }) + + return updateThumbnailFromFunction({ + thumbnailCreator, + filename, + height, + width, + type, + automaticallyGenerated: true, + onDisk: true, + existingThumbnail + }) + }) +} + +// --------------------------------------------------------------------------- + +function updateLocalVideoMiniatureFromUrl (options: { + downloadUrl: string + video: MVideoThumbnail + type: ThumbnailType_Type + size?: ImageSize +}) { + const { downloadUrl, video, type, size } = options + const { filename: updatedFilename, basePath, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) + + // Only save the file URL if it is a remote video + const fileUrl = video.isOwned() + ? null + : downloadUrl + + const thumbnailUrlChanged = hasThumbnailUrlChanged(existingThumbnail, downloadUrl, video) + + // Do not change the thumbnail filename if the file did not change + const filename = thumbnailUrlChanged + ? updatedFilename + : existingThumbnail.filename + + const thumbnailCreator = () => { + if (thumbnailUrlChanged) { + return downloadImageFromWorker({ url: downloadUrl, destDir: basePath, destName: filename, size: { width, height } }) + } + + return Promise.resolve() + } + + return updateThumbnailFromFunction({ thumbnailCreator, filename, height, width, type, existingThumbnail, fileUrl, onDisk: true }) +} + +function updateRemoteVideoThumbnail (options: { + fileUrl: string + video: MVideoThumbnail + type: ThumbnailType_Type + size: ImageSize + onDisk: boolean +}) { + const { fileUrl, video, type, size, onDisk } = options + const { filename: generatedFilename, height, width, existingThumbnail } = buildMetadataFromVideo(video, type, size) + + const thumbnail = existingThumbnail || new ThumbnailModel() + + // Do not change the thumbnail filename if the file did not change + if (hasThumbnailUrlChanged(existingThumbnail, fileUrl, video)) { + thumbnail.filename = generatedFilename + } + + thumbnail.height = height + thumbnail.width = width + thumbnail.type = type + thumbnail.fileUrl = fileUrl + thumbnail.onDisk = onDisk + + return thumbnail +} + +// --------------------------------------------------------------------------- + +async function regenerateMiniaturesIfNeeded (video: MVideoWithAllFiles) { + if (video.getMiniature().automaticallyGenerated === true) { + const miniature = await generateLocalVideoMiniature({ + video, + videoFile: video.getMaxQualityFile(), + type: ThumbnailType.MINIATURE + }) + await video.addAndSaveThumbnail(miniature) + } + + if (video.getPreview().automaticallyGenerated === true) { + const preview = await generateLocalVideoMiniature({ + video, + videoFile: video.getMaxQualityFile(), + type: ThumbnailType.PREVIEW + }) + await video.addAndSaveThumbnail(preview) + } +} + +// --------------------------------------------------------------------------- + +export { + generateLocalVideoMiniature, + regenerateMiniaturesIfNeeded, + updateLocalVideoMiniatureFromUrl, + updateLocalVideoMiniatureFromExisting, + updateRemoteVideoThumbnail, + updateRemotePlaylistMiniatureFromUrl, + updateLocalPlaylistMiniatureFromExisting +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +function hasThumbnailUrlChanged (existingThumbnail: MThumbnail, downloadUrl: string, video: MVideoUUID) { + const existingUrl = existingThumbnail + ? existingThumbnail.fileUrl + : null + + // If the thumbnail URL did not change and has a unique filename (introduced in 3.1), avoid thumbnail processing + return !existingUrl || existingUrl !== downloadUrl || downloadUrl.endsWith(`${video.uuid}.jpg`) +} + +function buildMetadataFromPlaylist (playlist: MVideoPlaylistThumbnail, size: ImageSize) { + const filename = playlist.generateThumbnailName() + const basePath = CONFIG.STORAGE.THUMBNAILS_DIR + + return { + filename, + basePath, + existingThumbnail: playlist.Thumbnail, + outputPath: join(basePath, filename), + height: size ? size.height : THUMBNAILS_SIZE.height, + width: size ? size.width : THUMBNAILS_SIZE.width + } +} + +function buildMetadataFromVideo (video: MVideoThumbnail, type: ThumbnailType_Type, size?: ImageSize) { + const existingThumbnail = Array.isArray(video.Thumbnails) + ? video.Thumbnails.find(t => t.type === type) + : undefined + + if (type === ThumbnailType.MINIATURE) { + const filename = generateImageFilename() + const basePath = CONFIG.STORAGE.THUMBNAILS_DIR + + return { + filename, + basePath, + existingThumbnail, + outputPath: join(basePath, filename), + height: size ? size.height : THUMBNAILS_SIZE.height, + width: size ? size.width : THUMBNAILS_SIZE.width + } + } + + if (type === ThumbnailType.PREVIEW) { + const filename = generateImageFilename() + const basePath = CONFIG.STORAGE.PREVIEWS_DIR + + return { + filename, + basePath, + existingThumbnail, + outputPath: join(basePath, filename), + height: size ? size.height : PREVIEWS_SIZE.height, + width: size ? size.width : PREVIEWS_SIZE.width + } + } + + return undefined +} + +async function updateThumbnailFromFunction (parameters: { + thumbnailCreator: () => Promise + filename: string + height: number + width: number + type: ThumbnailType_Type + onDisk: boolean + automaticallyGenerated?: boolean + fileUrl?: string + existingThumbnail?: MThumbnail +}) { + const { + thumbnailCreator, + filename, + width, + height, + type, + existingThumbnail, + onDisk, + automaticallyGenerated = null, + fileUrl = null + } = parameters + + const oldFilename = existingThumbnail && existingThumbnail.filename !== filename + ? existingThumbnail.filename + : undefined + + const thumbnail: MThumbnail = existingThumbnail || new ThumbnailModel() + + thumbnail.filename = filename + thumbnail.height = height + thumbnail.width = width + thumbnail.type = type + thumbnail.fileUrl = fileUrl + thumbnail.automaticallyGenerated = automaticallyGenerated + thumbnail.onDisk = onDisk + + if (oldFilename) thumbnail.previousThumbnailFilename = oldFilename + + await thumbnailCreator() + + return thumbnail +} diff --git a/server/server/lib/timeserie.ts b/server/server/lib/timeserie.ts new file mode 100644 index 000000000..9723266ff --- /dev/null +++ b/server/server/lib/timeserie.ts @@ -0,0 +1,61 @@ +import { logger } from '@server/helpers/logger.js' + +function buildGroupByAndBoundaries (startDateString: string, endDateString: string) { + const startDate = new Date(startDateString) + const endDate = new Date(endDateString) + + const groupInterval = buildGroupInterval(startDate, endDate) + + logger.debug('Found "%s" group interval.', groupInterval, { startDate, endDate }) + + // Remove parts of the date we don't need + if (groupInterval.endsWith(' month') || groupInterval.endsWith(' months')) { + startDate.setDate(1) + startDate.setHours(0, 0, 0, 0) + } else if (groupInterval.endsWith(' day') || groupInterval.endsWith(' days')) { + startDate.setHours(0, 0, 0, 0) + } else if (groupInterval.endsWith(' hour') || groupInterval.endsWith(' hours')) { + startDate.setMinutes(0, 0, 0) + } else { + startDate.setSeconds(0, 0) + } + + return { + groupInterval, + startDate, + endDate + } +} + +// --------------------------------------------------------------------------- + +export { + buildGroupByAndBoundaries +} + +// --------------------------------------------------------------------------- + +function buildGroupInterval (startDate: Date, endDate: Date): string { + const aYear = 31536000 + const aMonth = 2678400 + const aDay = 86400 + const anHour = 3600 + const aMinute = 60 + + const diffSeconds = (endDate.getTime() - startDate.getTime()) / 1000 + + if (diffSeconds >= 6 * aYear) return '6 months' + if (diffSeconds >= 2 * aYear) return '1 month' + if (diffSeconds >= 6 * aMonth) return '7 days' + if (diffSeconds >= 2 * aMonth) return '2 days' + + if (diffSeconds >= 15 * aDay) return '1 day' + if (diffSeconds >= 8 * aDay) return '12 hours' + if (diffSeconds >= 4 * aDay) return '6 hours' + + if (diffSeconds >= 15 * anHour) return '1 hour' + + if (diffSeconds >= 180 * aMinute) return '10 minutes' + + return '1 minute' +} diff --git a/server/server/lib/transcoding/create-transcoding-job.ts b/server/server/lib/transcoding/create-transcoding-job.ts new file mode 100644 index 000000000..aea18b632 --- /dev/null +++ b/server/server/lib/transcoding/create-transcoding-job.ts @@ -0,0 +1,37 @@ +import { CONFIG } from '@server/initializers/config.js' +import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models/index.js' +import { TranscodingJobQueueBuilder, TranscodingRunnerJobBuilder } from './shared/index.js' + +export function createOptimizeOrMergeAudioJobs (options: { + video: MVideoFullLight + videoFile: MVideoFile + isNewVideo: boolean + user: MUserId + videoFileAlreadyLocked: boolean +}) { + return getJobBuilder().createOptimizeOrMergeAudioJobs(options) +} + +// --------------------------------------------------------------------------- + +export function createTranscodingJobs (options: { + transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7 + video: MVideoFullLight + resolutions: number[] + isNewVideo: boolean + user: MUserId +}) { + return getJobBuilder().createTranscodingJobs(options) +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +function getJobBuilder () { + if (CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED === true) { + return new TranscodingRunnerJobBuilder() + } + + return new TranscodingJobQueueBuilder() +} diff --git a/server/server/lib/transcoding/default-transcoding-profiles.ts b/server/server/lib/transcoding/default-transcoding-profiles.ts new file mode 100644 index 000000000..90407d103 --- /dev/null +++ b/server/server/lib/transcoding/default-transcoding-profiles.ts @@ -0,0 +1,143 @@ + +import { logger } from '@server/helpers/logger.js' +import { FFmpegCommandWrapper, getDefaultAvailableEncoders } from '@peertube/peertube-ffmpeg' +import { AvailableEncoders, EncoderOptionsBuilder } from '@peertube/peertube-models' + +// --------------------------------------------------------------------------- +// Profile manager to get and change default profiles +// --------------------------------------------------------------------------- + +class VideoTranscodingProfilesManager { + private static instance: VideoTranscodingProfilesManager + + // 1 === less priority + private readonly encodersPriorities = { + vod: this.buildDefaultEncodersPriorities(), + live: this.buildDefaultEncodersPriorities() + } + + private readonly availableEncoders = getDefaultAvailableEncoders() + + private availableProfiles = { + vod: [] as string[], + live: [] as string[] + } + + private constructor () { + this.buildAvailableProfiles() + } + + getAvailableEncoders (): AvailableEncoders { + return { + available: this.availableEncoders, + encodersToTry: { + vod: { + video: this.getEncodersByPriority('vod', 'video'), + audio: this.getEncodersByPriority('vod', 'audio') + }, + live: { + video: this.getEncodersByPriority('live', 'video'), + audio: this.getEncodersByPriority('live', 'audio') + } + } + } + } + + getAvailableProfiles (type: 'vod' | 'live') { + return this.availableProfiles[type] + } + + addProfile (options: { + type: 'vod' | 'live' + encoder: string + profile: string + builder: EncoderOptionsBuilder + }) { + const { type, encoder, profile, builder } = options + + const encoders = this.availableEncoders[type] + + if (!encoders[encoder]) encoders[encoder] = {} + encoders[encoder][profile] = builder + + this.buildAvailableProfiles() + } + + removeProfile (options: { + type: 'vod' | 'live' + encoder: string + profile: string + }) { + const { type, encoder, profile } = options + + delete this.availableEncoders[type][encoder][profile] + this.buildAvailableProfiles() + } + + addEncoderPriority (type: 'vod' | 'live', streamType: 'audio' | 'video', encoder: string, priority: number) { + this.encodersPriorities[type][streamType].push({ name: encoder, priority }) + + FFmpegCommandWrapper.resetSupportedEncoders() + } + + removeEncoderPriority (type: 'vod' | 'live', streamType: 'audio' | 'video', encoder: string, priority: number) { + this.encodersPriorities[type][streamType] = this.encodersPriorities[type][streamType] + .filter(o => o.name !== encoder && o.priority !== priority) + + FFmpegCommandWrapper.resetSupportedEncoders() + } + + private getEncodersByPriority (type: 'vod' | 'live', streamType: 'audio' | 'video') { + return this.encodersPriorities[type][streamType] + .sort((e1, e2) => { + if (e1.priority > e2.priority) return -1 + else if (e1.priority === e2.priority) return 0 + + return 1 + }) + .map(e => e.name) + } + + private buildAvailableProfiles () { + for (const type of [ 'vod', 'live' ]) { + const result = new Set() + + const encoders = this.availableEncoders[type] + + for (const encoderName of Object.keys(encoders)) { + for (const profile of Object.keys(encoders[encoderName])) { + result.add(profile) + } + } + + this.availableProfiles[type] = Array.from(result) + } + + logger.debug('Available transcoding profiles built.', { availableProfiles: this.availableProfiles }) + } + + private buildDefaultEncodersPriorities () { + return { + video: [ + { name: 'libx264', priority: 100 } + ], + + // Try the first one, if not available try the second one etc + audio: [ + // we favor VBR, if a good AAC encoder is available + { name: 'libfdk_aac', priority: 200 }, + { name: 'aac', priority: 100 } + ] + } + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} + +// --------------------------------------------------------------------------- + +export { + VideoTranscodingProfilesManager +} diff --git a/server/server/lib/transcoding/ended-transcoding.ts b/server/server/lib/transcoding/ended-transcoding.ts new file mode 100644 index 000000000..468a3aa83 --- /dev/null +++ b/server/server/lib/transcoding/ended-transcoding.ts @@ -0,0 +1,18 @@ +import { retryTransactionWrapper } from '@server/helpers/database-utils.js' +import { VideoJobInfoModel } from '@server/models/video/video-job-info.js' +import { MVideo } from '@server/types/models/index.js' +import { moveToNextState } from '../video-state.js' + +export async function onTranscodingEnded (options: { + video: MVideo + isNewVideo: boolean + moveVideoToNextState: boolean +}) { + const { video, isNewVideo, moveVideoToNextState } = options + + await VideoJobInfoModel.decrease(video.uuid, 'pendingTranscode') + + if (moveVideoToNextState) { + await retryTransactionWrapper(moveToNextState, { video, isNewVideo }) + } +} diff --git a/server/server/lib/transcoding/hls-transcoding.ts b/server/server/lib/transcoding/hls-transcoding.ts new file mode 100644 index 000000000..5f07f112a --- /dev/null +++ b/server/server/lib/transcoding/hls-transcoding.ts @@ -0,0 +1,180 @@ +import { MutexInterface } from 'async-mutex' +import { Job } from 'bullmq' +import { ensureDir, move } from 'fs-extra/esm' +import { stat } from 'fs/promises' +import { basename, extname as extnameUtil, join } from 'path' +import { pick } from '@peertube/peertube-core-utils' +import { retryTransactionWrapper } from '@server/helpers/database-utils.js' +import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent.js' +import { sequelizeTypescript } from '@server/initializers/database.js' +import { MVideo, MVideoFile } from '@server/types/models/index.js' +import { getVideoStreamDuration, getVideoStreamFPS } from '@peertube/peertube-ffmpeg' +import { CONFIG } from '../../initializers/config.js' +import { VideoFileModel } from '../../models/video/video-file.js' +import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist.js' +import { updatePlaylistAfterFileChange } from '../hls.js' +import { generateHLSVideoFilename, getHlsResolutionPlaylistFilename } from '../paths.js' +import { buildFileMetadata } from '../video-file.js' +import { VideoPathManager } from '../video-path-manager.js' +import { buildFFmpegVOD } from './shared/index.js' + +// Concat TS segments from a live video to a fragmented mp4 HLS playlist +export async function generateHlsPlaylistResolutionFromTS (options: { + video: MVideo + concatenatedTsFilePath: string + resolution: number + fps: number + isAAC: boolean + inputFileMutexReleaser: MutexInterface.Releaser +}) { + return generateHlsPlaylistCommon({ + type: 'hls-from-ts' as 'hls-from-ts', + inputPath: options.concatenatedTsFilePath, + + ...pick(options, [ 'video', 'resolution', 'fps', 'inputFileMutexReleaser', 'isAAC' ]) + }) +} + +// Generate an HLS playlist from an input file, and update the master playlist +export function generateHlsPlaylistResolution (options: { + video: MVideo + videoInputPath: string + resolution: number + fps: number + copyCodecs: boolean + inputFileMutexReleaser: MutexInterface.Releaser + job?: Job +}) { + return generateHlsPlaylistCommon({ + type: 'hls' as 'hls', + inputPath: options.videoInputPath, + + ...pick(options, [ 'video', 'resolution', 'fps', 'copyCodecs', 'inputFileMutexReleaser', 'job' ]) + }) +} + +export async function onHLSVideoFileTranscoding (options: { + video: MVideo + videoFile: MVideoFile + videoOutputPath: string + m3u8OutputPath: string +}) { + const { video, videoFile, videoOutputPath, m3u8OutputPath } = options + + // Create or update the playlist + const playlist = await retryTransactionWrapper(() => { + return sequelizeTypescript.transaction(async transaction => { + return VideoStreamingPlaylistModel.loadOrGenerate(video, transaction) + }) + }) + videoFile.videoStreamingPlaylistId = playlist.id + + const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) + + try { + await video.reload() + + const videoFilePath = VideoPathManager.Instance.getFSVideoFileOutputPath(playlist, videoFile) + await ensureDir(VideoPathManager.Instance.getFSHLSOutputPath(video)) + + // Move playlist file + const resolutionPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, basename(m3u8OutputPath)) + await move(m3u8OutputPath, resolutionPlaylistPath, { overwrite: true }) + // Move video file + await move(videoOutputPath, videoFilePath, { overwrite: true }) + + // Update video duration if it was not set (in case of a live for example) + if (!video.duration) { + video.duration = await getVideoStreamDuration(videoFilePath) + await video.save() + } + + const stats = await stat(videoFilePath) + + videoFile.size = stats.size + videoFile.fps = await getVideoStreamFPS(videoFilePath) + videoFile.metadata = await buildFileMetadata(videoFilePath) + + await createTorrentAndSetInfoHash(playlist, videoFile) + + const oldFile = await VideoFileModel.loadHLSFile({ + playlistId: playlist.id, + fps: videoFile.fps, + resolution: videoFile.resolution + }) + + if (oldFile) { + await video.removeStreamingPlaylistVideoFile(playlist, oldFile) + await oldFile.destroy() + } + + const savedVideoFile = await VideoFileModel.customUpsert(videoFile, 'streaming-playlist', undefined) + + await updatePlaylistAfterFileChange(video, playlist) + + return { resolutionPlaylistPath, videoFile: savedVideoFile } + } finally { + mutexReleaser() + } +} + +// --------------------------------------------------------------------------- + +async function generateHlsPlaylistCommon (options: { + type: 'hls' | 'hls-from-ts' + video: MVideo + inputPath: string + + resolution: number + fps: number + + inputFileMutexReleaser: MutexInterface.Releaser + + copyCodecs?: boolean + isAAC?: boolean + + job?: Job +}) { + const { type, video, inputPath, resolution, fps, copyCodecs, isAAC, job, inputFileMutexReleaser } = options + const transcodeDirectory = CONFIG.STORAGE.TMP_DIR + + const videoTranscodedBasePath = join(transcodeDirectory, type) + await ensureDir(videoTranscodedBasePath) + + const videoFilename = generateHLSVideoFilename(resolution) + const videoOutputPath = join(videoTranscodedBasePath, videoFilename) + + const resolutionPlaylistFilename = getHlsResolutionPlaylistFilename(videoFilename) + const m3u8OutputPath = join(videoTranscodedBasePath, resolutionPlaylistFilename) + + const transcodeOptions = { + type, + + inputPath, + outputPath: m3u8OutputPath, + + resolution, + fps, + copyCodecs, + + isAAC, + + inputFileMutexReleaser, + + hlsPlaylist: { + videoFilename + } + } + + await buildFFmpegVOD(job).transcode(transcodeOptions) + + const newVideoFile = new VideoFileModel({ + resolution, + extname: extnameUtil(videoFilename), + size: 0, + filename: videoFilename, + fps: -1 + }) + + await onHLSVideoFileTranscoding({ video, videoFile: newVideoFile, videoOutputPath, m3u8OutputPath }) +} diff --git a/server/server/lib/transcoding/shared/ffmpeg-builder.ts b/server/server/lib/transcoding/shared/ffmpeg-builder.ts new file mode 100644 index 000000000..b28ffdb1f --- /dev/null +++ b/server/server/lib/transcoding/shared/ffmpeg-builder.ts @@ -0,0 +1,18 @@ +import { Job } from 'bullmq' +import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js' +import { logger } from '@server/helpers/logger.js' +import { FFmpegVOD } from '@peertube/peertube-ffmpeg' +import { VideoTranscodingProfilesManager } from '../default-transcoding-profiles.js' + +export function buildFFmpegVOD (job?: Job) { + return new FFmpegVOD({ + ...getFFmpegCommandWrapperOptions('vod', VideoTranscodingProfilesManager.Instance.getAvailableEncoders()), + + updateJobProgress: progress => { + if (!job) return + + job.updateProgress(progress) + .catch(err => logger.error('Cannot update ffmpeg job progress', { err })) + } + }) +} diff --git a/server/server/lib/transcoding/shared/index.ts b/server/server/lib/transcoding/shared/index.ts new file mode 100644 index 000000000..835ee6632 --- /dev/null +++ b/server/server/lib/transcoding/shared/index.ts @@ -0,0 +1,2 @@ +export * from './job-builders/index.js' +export * from './ffmpeg-builder.js' diff --git a/server/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts b/server/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts new file mode 100644 index 000000000..96c8f990b --- /dev/null +++ b/server/server/lib/transcoding/shared/job-builders/abstract-job-builder.ts @@ -0,0 +1,21 @@ + +import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models/index.js' + +export abstract class AbstractJobBuilder { + + abstract createOptimizeOrMergeAudioJobs (options: { + video: MVideoFullLight + videoFile: MVideoFile + isNewVideo: boolean + user: MUserId + videoFileAlreadyLocked: boolean + }): Promise + + abstract createTranscodingJobs (options: { + transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7 + video: MVideoFullLight + resolutions: number[] + isNewVideo: boolean + user: MUserId | null + }): Promise +} diff --git a/server/server/lib/transcoding/shared/job-builders/index.ts b/server/server/lib/transcoding/shared/job-builders/index.ts new file mode 100644 index 000000000..099c6646e --- /dev/null +++ b/server/server/lib/transcoding/shared/job-builders/index.ts @@ -0,0 +1,2 @@ +export * from './transcoding-job-queue-builder.js' +export * from './transcoding-runner-job-builder.js' diff --git a/server/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts b/server/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts new file mode 100644 index 000000000..0f221472f --- /dev/null +++ b/server/server/lib/transcoding/shared/job-builders/transcoding-job-queue-builder.ts @@ -0,0 +1,322 @@ +import Bluebird from 'bluebird' +import { computeOutputFPS } from '@server/helpers/ffmpeg/index.js' +import { logger } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' +import { DEFAULT_AUDIO_RESOLUTION, VIDEO_TRANSCODING_FPS } from '@server/initializers/constants.js' +import { CreateJobArgument, JobQueue } from '@server/lib/job-queue/index.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { VideoPathManager } from '@server/lib/video-path-manager.js' +import { VideoJobInfoModel } from '@server/models/video/video-job-info.js' +import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models/index.js' +import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@peertube/peertube-ffmpeg' +import { + HLSTranscodingPayload, + MergeAudioTranscodingPayload, + NewWebVideoResolutionTranscodingPayload, + OptimizeTranscodingPayload, + VideoTranscodingPayload +} from '@peertube/peertube-models' +import { getTranscodingJobPriority } from '../../transcoding-priority.js' +import { canDoQuickTranscode } from '../../transcoding-quick-transcode.js' +import { buildOriginalFileResolution, computeResolutionsToTranscode } from '../../transcoding-resolutions.js' +import { AbstractJobBuilder } from './abstract-job-builder.js' + +export class TranscodingJobQueueBuilder extends AbstractJobBuilder { + + async createOptimizeOrMergeAudioJobs (options: { + video: MVideoFullLight + videoFile: MVideoFile + isNewVideo: boolean + user: MUserId + videoFileAlreadyLocked: boolean + }) { + const { video, videoFile, isNewVideo, user, videoFileAlreadyLocked } = options + + let mergeOrOptimizePayload: MergeAudioTranscodingPayload | OptimizeTranscodingPayload + let nextTranscodingSequentialJobPayloads: (NewWebVideoResolutionTranscodingPayload | HLSTranscodingPayload)[][] = [] + + const mutexReleaser = videoFileAlreadyLocked + ? () => {} + : await VideoPathManager.Instance.lockFiles(video.uuid) + + try { + await video.reload() + await videoFile.reload() + + await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), async videoFilePath => { + const probe = await ffprobePromise(videoFilePath) + + const { resolution } = await getVideoStreamDimensionsInfo(videoFilePath, probe) + const hasAudio = await hasAudioStream(videoFilePath, probe) + const quickTranscode = await canDoQuickTranscode(videoFilePath, probe) + const inputFPS = videoFile.isAudio() + ? VIDEO_TRANSCODING_FPS.AUDIO_MERGE // The first transcoding job will transcode to this FPS value + : await getVideoStreamFPS(videoFilePath, probe) + + const maxResolution = await isAudioFile(videoFilePath, probe) + ? DEFAULT_AUDIO_RESOLUTION + : buildOriginalFileResolution(resolution) + + if (CONFIG.TRANSCODING.HLS.ENABLED === true) { + nextTranscodingSequentialJobPayloads.push([ + this.buildHLSJobPayload({ + deleteWebVideoFiles: CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED === false, + + // We had some issues with a web video quick transcoded while producing a HLS version of it + copyCodecs: !quickTranscode, + + resolution: maxResolution, + fps: computeOutputFPS({ inputFPS, resolution: maxResolution }), + videoUUID: video.uuid, + isNewVideo + }) + ]) + } + + const lowerResolutionJobPayloads = await this.buildLowerResolutionJobPayloads({ + video, + inputVideoResolution: maxResolution, + inputVideoFPS: inputFPS, + hasAudio, + isNewVideo + }) + + nextTranscodingSequentialJobPayloads = [ ...nextTranscodingSequentialJobPayloads, ...lowerResolutionJobPayloads ] + + const hasChildren = nextTranscodingSequentialJobPayloads.length !== 0 + mergeOrOptimizePayload = videoFile.isAudio() + ? this.buildMergeAudioPayload({ videoUUID: video.uuid, isNewVideo, hasChildren }) + : this.buildOptimizePayload({ videoUUID: video.uuid, isNewVideo, quickTranscode, hasChildren }) + }) + } finally { + mutexReleaser() + } + + const nextTranscodingSequentialJobs = await Bluebird.mapSeries(nextTranscodingSequentialJobPayloads, payloads => { + return Bluebird.mapSeries(payloads, payload => { + return this.buildTranscodingJob({ payload, user }) + }) + }) + + const transcodingJobBuilderJob: CreateJobArgument = { + type: 'transcoding-job-builder', + payload: { + videoUUID: video.uuid, + sequentialJobs: nextTranscodingSequentialJobs + } + } + + const mergeOrOptimizeJob = await this.buildTranscodingJob({ payload: mergeOrOptimizePayload, user }) + + await JobQueue.Instance.createSequentialJobFlow(...[ mergeOrOptimizeJob, transcodingJobBuilderJob ]) + + await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingTranscode') + } + + // --------------------------------------------------------------------------- + + async createTranscodingJobs (options: { + transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7 + video: MVideoFullLight + resolutions: number[] + isNewVideo: boolean + user: MUserId | null + }) { + const { video, transcodingType, resolutions, isNewVideo } = options + + const maxResolution = Math.max(...resolutions) + const childrenResolutions = resolutions.filter(r => r !== maxResolution) + + logger.info('Manually creating transcoding jobs for %s.', transcodingType, { childrenResolutions, maxResolution }) + + const { fps: inputFPS } = await video.probeMaxQualityFile() + + const children = childrenResolutions.map(resolution => { + const fps = computeOutputFPS({ inputFPS, resolution }) + + if (transcodingType === 'hls') { + return this.buildHLSJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo }) + } + + if (transcodingType === 'webtorrent' || transcodingType === 'web-video') { + return this.buildWebVideoJobPayload({ videoUUID: video.uuid, resolution, fps, isNewVideo }) + } + + throw new Error('Unknown transcoding type') + }) + + const fps = computeOutputFPS({ inputFPS, resolution: maxResolution }) + + const parent = transcodingType === 'hls' + ? this.buildHLSJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo }) + : this.buildWebVideoJobPayload({ videoUUID: video.uuid, resolution: maxResolution, fps, isNewVideo }) + + // Process the last resolution after the other ones to prevent concurrency issue + // Because low resolutions use the biggest one as ffmpeg input + await this.createTranscodingJobsWithChildren({ videoUUID: video.uuid, parent, children, user: null }) + } + + // --------------------------------------------------------------------------- + + private async createTranscodingJobsWithChildren (options: { + videoUUID: string + parent: (HLSTranscodingPayload | NewWebVideoResolutionTranscodingPayload) + children: (HLSTranscodingPayload | NewWebVideoResolutionTranscodingPayload)[] + user: MUserId | null + }) { + const { videoUUID, parent, children, user } = options + + const parentJob = await this.buildTranscodingJob({ payload: parent, user }) + const childrenJobs = await Bluebird.mapSeries(children, c => this.buildTranscodingJob({ payload: c, user })) + + await JobQueue.Instance.createJobWithChildren(parentJob, childrenJobs) + + await VideoJobInfoModel.increaseOrCreate(videoUUID, 'pendingTranscode', 1 + children.length) + } + + private async buildTranscodingJob (options: { + payload: VideoTranscodingPayload + user: MUserId | null // null means we don't want priority + }) { + const { user, payload } = options + + return { + type: 'video-transcoding' as 'video-transcoding', + priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: undefined }), + payload + } + } + + private async buildLowerResolutionJobPayloads (options: { + video: MVideoWithFileThumbnail + inputVideoResolution: number + inputVideoFPS: number + hasAudio: boolean + isNewVideo: boolean + }) { + const { video, inputVideoResolution, inputVideoFPS, isNewVideo, hasAudio } = options + + // Create transcoding jobs if there are enabled resolutions + const resolutionsEnabled = await Hooks.wrapObject( + computeResolutionsToTranscode({ input: inputVideoResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }), + 'filter:transcoding.auto.resolutions-to-transcode.result', + options + ) + + const sequentialPayloads: (NewWebVideoResolutionTranscodingPayload | HLSTranscodingPayload)[][] = [] + + for (const resolution of resolutionsEnabled) { + const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution }) + + if (CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED) { + const payloads: (NewWebVideoResolutionTranscodingPayload | HLSTranscodingPayload)[] = [ + this.buildWebVideoJobPayload({ + videoUUID: video.uuid, + resolution, + fps, + isNewVideo + }) + ] + + // Create a subsequent job to create HLS resolution that will just copy web video codecs + if (CONFIG.TRANSCODING.HLS.ENABLED) { + payloads.push( + this.buildHLSJobPayload({ + videoUUID: video.uuid, + resolution, + fps, + isNewVideo, + copyCodecs: true + }) + ) + } + + sequentialPayloads.push(payloads) + } else if (CONFIG.TRANSCODING.HLS.ENABLED) { + sequentialPayloads.push([ + this.buildHLSJobPayload({ + videoUUID: video.uuid, + resolution, + fps, + copyCodecs: false, + isNewVideo + }) + ]) + } + } + + return sequentialPayloads + } + + private buildHLSJobPayload (options: { + videoUUID: string + resolution: number + fps: number + isNewVideo: boolean + deleteWebVideoFiles?: boolean // default false + copyCodecs?: boolean // default false + }): HLSTranscodingPayload { + const { videoUUID, resolution, fps, isNewVideo, deleteWebVideoFiles = false, copyCodecs = false } = options + + return { + type: 'new-resolution-to-hls', + videoUUID, + resolution, + fps, + copyCodecs, + isNewVideo, + deleteWebVideoFiles + } + } + + private buildWebVideoJobPayload (options: { + videoUUID: string + resolution: number + fps: number + isNewVideo: boolean + }): NewWebVideoResolutionTranscodingPayload { + const { videoUUID, resolution, fps, isNewVideo } = options + + return { + type: 'new-resolution-to-web-video', + videoUUID, + isNewVideo, + resolution, + fps + } + } + + private buildMergeAudioPayload (options: { + videoUUID: string + isNewVideo: boolean + hasChildren: boolean + }): MergeAudioTranscodingPayload { + const { videoUUID, isNewVideo, hasChildren } = options + + return { + type: 'merge-audio-to-web-video', + resolution: DEFAULT_AUDIO_RESOLUTION, + fps: VIDEO_TRANSCODING_FPS.AUDIO_MERGE, + videoUUID, + isNewVideo, + hasChildren + } + } + + private buildOptimizePayload (options: { + videoUUID: string + quickTranscode: boolean + isNewVideo: boolean + hasChildren: boolean + }): OptimizeTranscodingPayload { + const { videoUUID, quickTranscode, isNewVideo, hasChildren } = options + + return { + type: 'optimize-to-web-video', + videoUUID, + isNewVideo, + hasChildren, + quickTranscode + } + } +} diff --git a/server/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts b/server/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts new file mode 100644 index 000000000..68d84a286 --- /dev/null +++ b/server/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts @@ -0,0 +1,200 @@ +import { computeOutputFPS } from '@server/helpers/ffmpeg/index.js' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' +import { DEFAULT_AUDIO_RESOLUTION, VIDEO_TRANSCODING_FPS } from '@server/initializers/constants.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { + VODAudioMergeTranscodingJobHandler, + VODHLSTranscodingJobHandler, + VODWebVideoTranscodingJobHandler +} from '@server/lib/runners/index.js' +import { VideoPathManager } from '@server/lib/video-path-manager.js' +import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models/index.js' +import { MRunnerJob } from '@server/types/models/runners/index.js' +import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@peertube/peertube-ffmpeg' +import { getTranscodingJobPriority } from '../../transcoding-priority.js' +import { computeResolutionsToTranscode } from '../../transcoding-resolutions.js' +import { AbstractJobBuilder } from './abstract-job-builder.js' + +/** + * + * Class to build transcoding job in the local job queue + * + */ + +const lTags = loggerTagsFactory('transcoding') + +export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { + + async createOptimizeOrMergeAudioJobs (options: { + video: MVideoFullLight + videoFile: MVideoFile + isNewVideo: boolean + user: MUserId + videoFileAlreadyLocked: boolean + }) { + const { video, videoFile, isNewVideo, user, videoFileAlreadyLocked } = options + + const mutexReleaser = videoFileAlreadyLocked + ? () => {} + : await VideoPathManager.Instance.lockFiles(video.uuid) + + try { + await video.reload() + await videoFile.reload() + + await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), async videoFilePath => { + const probe = await ffprobePromise(videoFilePath) + + const { resolution } = await getVideoStreamDimensionsInfo(videoFilePath, probe) + const hasAudio = await hasAudioStream(videoFilePath, probe) + const inputFPS = videoFile.isAudio() + ? VIDEO_TRANSCODING_FPS.AUDIO_MERGE // The first transcoding job will transcode to this FPS value + : await getVideoStreamFPS(videoFilePath, probe) + + const maxResolution = await isAudioFile(videoFilePath, probe) + ? DEFAULT_AUDIO_RESOLUTION + : resolution + + const fps = computeOutputFPS({ inputFPS, resolution: maxResolution }) + const priority = await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) + + const mainRunnerJob = videoFile.isAudio() + ? await new VODAudioMergeTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority }) + : await new VODWebVideoTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority }) + + if (CONFIG.TRANSCODING.HLS.ENABLED === true) { + await new VODHLSTranscodingJobHandler().create({ + video, + deleteWebVideoFiles: CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED === false, + resolution: maxResolution, + fps, + isNewVideo, + dependsOnRunnerJob: mainRunnerJob, + priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) + }) + } + + await this.buildLowerResolutionJobPayloads({ + video, + inputVideoResolution: maxResolution, + inputVideoFPS: inputFPS, + hasAudio, + isNewVideo, + mainRunnerJob, + user + }) + }) + } finally { + mutexReleaser() + } + } + + // --------------------------------------------------------------------------- + + async createTranscodingJobs (options: { + transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7 + video: MVideoFullLight + resolutions: number[] + isNewVideo: boolean + user: MUserId | null + }) { + const { video, transcodingType, resolutions, isNewVideo, user } = options + + const maxResolution = Math.max(...resolutions) + const { fps: inputFPS } = await video.probeMaxQualityFile() + const maxFPS = computeOutputFPS({ inputFPS, resolution: maxResolution }) + const priority = await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) + + const childrenResolutions = resolutions.filter(r => r !== maxResolution) + + logger.info('Manually creating transcoding jobs for %s.', transcodingType, { childrenResolutions, maxResolution }) + + // Process the last resolution before the other ones to prevent concurrency issue + // Because low resolutions use the biggest one as ffmpeg input + const mainJob = transcodingType === 'hls' + // eslint-disable-next-line max-len + ? await new VODHLSTranscodingJobHandler().create({ video, resolution: maxResolution, fps: maxFPS, isNewVideo, deleteWebVideoFiles: false, priority }) + : await new VODWebVideoTranscodingJobHandler().create({ video, resolution: maxResolution, fps: maxFPS, isNewVideo, priority }) + + for (const resolution of childrenResolutions) { + const dependsOnRunnerJob = mainJob + const fps = computeOutputFPS({ inputFPS, resolution }) + + if (transcodingType === 'hls') { + await new VODHLSTranscodingJobHandler().create({ + video, + resolution, + fps, + isNewVideo, + deleteWebVideoFiles: false, + dependsOnRunnerJob, + priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) + }) + continue + } + + if (transcodingType === 'webtorrent' || transcodingType === 'web-video') { + await new VODWebVideoTranscodingJobHandler().create({ + video, + resolution, + fps, + isNewVideo, + dependsOnRunnerJob, + priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) + }) + continue + } + + throw new Error('Unknown transcoding type') + } + } + + private async buildLowerResolutionJobPayloads (options: { + mainRunnerJob: MRunnerJob + video: MVideoWithFileThumbnail + inputVideoResolution: number + inputVideoFPS: number + hasAudio: boolean + isNewVideo: boolean + user: MUserId + }) { + const { video, inputVideoResolution, inputVideoFPS, isNewVideo, hasAudio, mainRunnerJob, user } = options + + // Create transcoding jobs if there are enabled resolutions + const resolutionsEnabled = await Hooks.wrapObject( + computeResolutionsToTranscode({ input: inputVideoResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }), + 'filter:transcoding.auto.resolutions-to-transcode.result', + options + ) + + logger.debug('Lower resolutions build for %s.', video.uuid, { resolutionsEnabled, ...lTags(video.uuid) }) + + for (const resolution of resolutionsEnabled) { + const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution }) + + if (CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED) { + await new VODWebVideoTranscodingJobHandler().create({ + video, + resolution, + fps, + isNewVideo, + dependsOnRunnerJob: mainRunnerJob, + priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) + }) + } + + if (CONFIG.TRANSCODING.HLS.ENABLED) { + await new VODHLSTranscodingJobHandler().create({ + video, + resolution, + fps, + isNewVideo, + deleteWebVideoFiles: false, + dependsOnRunnerJob: mainRunnerJob, + priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) + }) + } + } + } +} diff --git a/server/server/lib/transcoding/transcoding-priority.ts b/server/server/lib/transcoding/transcoding-priority.ts new file mode 100644 index 000000000..42061d0f9 --- /dev/null +++ b/server/server/lib/transcoding/transcoding-priority.ts @@ -0,0 +1,24 @@ +import { JOB_PRIORITY } from '@server/initializers/constants.js' +import { VideoModel } from '@server/models/video/video.js' +import { MUserId } from '@server/types/models/index.js' + +export async function getTranscodingJobPriority (options: { + user: MUserId + fallback: number + type: 'vod' | 'studio' +}) { + const { user, fallback, type } = options + + if (!user) return fallback + + const now = new Date() + const lastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7) + + const videoUploadedByUser = await VideoModel.countVideosUploadedByUserSince(user.id, lastWeek) + + const base = type === 'vod' + ? JOB_PRIORITY.TRANSCODING + : JOB_PRIORITY.VIDEO_STUDIO + + return base + videoUploadedByUser +} diff --git a/server/server/lib/transcoding/transcoding-quick-transcode.ts b/server/server/lib/transcoding/transcoding-quick-transcode.ts new file mode 100644 index 000000000..1d054ece3 --- /dev/null +++ b/server/server/lib/transcoding/transcoding-quick-transcode.ts @@ -0,0 +1,12 @@ +import { FfprobeData } from 'fluent-ffmpeg' +import { CONFIG } from '@server/initializers/config.js' +import { canDoQuickAudioTranscode, canDoQuickVideoTranscode, ffprobePromise } from '@peertube/peertube-ffmpeg' + +export async function canDoQuickTranscode (path: string, existingProbe?: FfprobeData): Promise { + if (CONFIG.TRANSCODING.PROFILE !== 'default') return false + + const probe = existingProbe || await ffprobePromise(path) + + return await canDoQuickVideoTranscode(path, probe) && + await canDoQuickAudioTranscode(path, probe) +} diff --git a/server/server/lib/transcoding/transcoding-resolutions.ts b/server/server/lib/transcoding/transcoding-resolutions.ts new file mode 100644 index 000000000..b1c55edf9 --- /dev/null +++ b/server/server/lib/transcoding/transcoding-resolutions.ts @@ -0,0 +1,73 @@ +import { toEven } from '@peertube/peertube-core-utils' +import { VideoResolution, VideoResolutionType } from '@peertube/peertube-models' +import { CONFIG } from '@server/initializers/config.js' + +export function buildOriginalFileResolution (inputResolution: number) { + if (CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION === true) { + return toEven(inputResolution) + } + + const resolutions = computeResolutionsToTranscode({ + input: inputResolution, + type: 'vod', + includeInput: false, + strictLower: false, + // We don't really care about the audio resolution in this context + hasAudio: true + }) + + if (resolutions.length === 0) { + return toEven(inputResolution) + } + + return Math.max(...resolutions) +} + +export function computeResolutionsToTranscode (options: { + input: number + type: 'vod' | 'live' + includeInput: boolean + strictLower: boolean + hasAudio: boolean +}) { + const { input, type, includeInput, strictLower, hasAudio } = options + + const configResolutions = type === 'vod' + ? CONFIG.TRANSCODING.RESOLUTIONS + : CONFIG.LIVE.TRANSCODING.RESOLUTIONS + + const resolutionsEnabled = new Set() + + // Put in the order we want to proceed jobs + const availableResolutions: VideoResolutionType[] = [ + VideoResolution.H_NOVIDEO, + VideoResolution.H_480P, + VideoResolution.H_360P, + VideoResolution.H_720P, + VideoResolution.H_240P, + VideoResolution.H_144P, + VideoResolution.H_1080P, + VideoResolution.H_1440P, + VideoResolution.H_4K + ] + + for (const resolution of availableResolutions) { + // Resolution not enabled + if (configResolutions[resolution + 'p'] !== true) continue + // Too big resolution for input file + if (input < resolution) continue + // We only want lower resolutions than input file + if (strictLower && input === resolution) continue + // Audio resolutio but no audio in the video + if (resolution === VideoResolution.H_NOVIDEO && !hasAudio) continue + + resolutionsEnabled.add(resolution) + } + + if (includeInput) { + // Always use an even resolution to avoid issues with ffmpeg + resolutionsEnabled.add(toEven(input)) + } + + return Array.from(resolutionsEnabled) +} diff --git a/server/server/lib/transcoding/web-transcoding.ts b/server/server/lib/transcoding/web-transcoding.ts new file mode 100644 index 000000000..e8247e958 --- /dev/null +++ b/server/server/lib/transcoding/web-transcoding.ts @@ -0,0 +1,264 @@ +import { Job } from 'bullmq' +import { move, remove } from 'fs-extra/esm' +import { copyFile, stat } from 'fs/promises' +import { basename, join } from 'path' +import { VideoStorage } from '@peertube/peertube-models' +import { computeOutputFPS } from '@server/helpers/ffmpeg/index.js' +import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent.js' +import { VideoModel } from '@server/models/video/video.js' +import { MVideoFile, MVideoFullLight } from '@server/types/models/index.js' +import { ffprobePromise, getVideoStreamDuration, getVideoStreamFPS, TranscodeVODOptionsType } from '@peertube/peertube-ffmpeg' +import { CONFIG } from '../../initializers/config.js' +import { VideoFileModel } from '../../models/video/video-file.js' +import { JobQueue } from '../job-queue/index.js' +import { generateWebVideoFilename } from '../paths.js' +import { buildFileMetadata } from '../video-file.js' +import { VideoPathManager } from '../video-path-manager.js' +import { buildFFmpegVOD } from './shared/index.js' +import { buildOriginalFileResolution } from './transcoding-resolutions.js' + +// Optimize the original video file and replace it. The resolution is not changed. +export async function optimizeOriginalVideofile (options: { + video: MVideoFullLight + inputVideoFile: MVideoFile + quickTranscode: boolean + job: Job +}) { + const { video, inputVideoFile, quickTranscode, job } = options + + const transcodeDirectory = CONFIG.STORAGE.TMP_DIR + const newExtname = '.mp4' + + // Will be released by our transcodeVOD function once ffmpeg is ran + const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) + + try { + await video.reload() + await inputVideoFile.reload() + + const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video) + + const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async videoInputPath => { + const videoOutputPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) + + const transcodeType: TranscodeVODOptionsType = quickTranscode + ? 'quick-transcode' + : 'video' + + const resolution = buildOriginalFileResolution(inputVideoFile.resolution) + const fps = computeOutputFPS({ inputFPS: inputVideoFile.fps, resolution }) + + // Could be very long! + await buildFFmpegVOD(job).transcode({ + type: transcodeType, + + inputPath: videoInputPath, + outputPath: videoOutputPath, + + inputFileMutexReleaser, + + resolution, + fps + }) + + // Important to do this before getVideoFilename() to take in account the new filename + inputVideoFile.resolution = resolution + inputVideoFile.extname = newExtname + inputVideoFile.filename = generateWebVideoFilename(resolution, newExtname) + inputVideoFile.storage = VideoStorage.FILE_SYSTEM + + const { videoFile } = await onWebVideoFileTranscoding({ + video, + videoFile: inputVideoFile, + videoOutputPath + }) + + await remove(videoInputPath) + + return { transcodeType, videoFile } + }) + + return result + } finally { + inputFileMutexReleaser() + } +} + +// Transcode the original video file to a lower resolution compatible with web browsers +export async function transcodeNewWebVideoResolution (options: { + video: MVideoFullLight + resolution: number + fps: number + job: Job +}) { + const { video: videoArg, resolution, fps, job } = options + + const transcodeDirectory = CONFIG.STORAGE.TMP_DIR + const newExtname = '.mp4' + + const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid) + + try { + const video = await VideoModel.loadFull(videoArg.uuid) + const file = video.getMaxQualityFile().withVideoOrPlaylist(video) + + const result = await VideoPathManager.Instance.makeAvailableVideoFile(file, async videoInputPath => { + const newVideoFile = new VideoFileModel({ + resolution, + extname: newExtname, + filename: generateWebVideoFilename(resolution, newExtname), + size: 0, + videoId: video.id + }) + + const videoOutputPath = join(transcodeDirectory, newVideoFile.filename) + + const transcodeOptions = { + type: 'video' as 'video', + + inputPath: videoInputPath, + outputPath: videoOutputPath, + + inputFileMutexReleaser, + + resolution, + fps + } + + await buildFFmpegVOD(job).transcode(transcodeOptions) + + return onWebVideoFileTranscoding({ video, videoFile: newVideoFile, videoOutputPath }) + }) + + return result + } finally { + inputFileMutexReleaser() + } +} + +// Merge an image with an audio file to create a video +export async function mergeAudioVideofile (options: { + video: MVideoFullLight + resolution: number + fps: number + job: Job +}) { + const { video: videoArg, resolution, fps, job } = options + + const transcodeDirectory = CONFIG.STORAGE.TMP_DIR + const newExtname = '.mp4' + + const inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(videoArg.uuid) + + try { + const video = await VideoModel.loadFull(videoArg.uuid) + const inputVideoFile = video.getMinQualityFile() + + const fileWithVideoOrPlaylist = inputVideoFile.withVideoOrPlaylist(video) + + const result = await VideoPathManager.Instance.makeAvailableVideoFile(fileWithVideoOrPlaylist, async audioInputPath => { + const videoOutputPath = join(transcodeDirectory, video.id + '-transcoded' + newExtname) + + // If the user updates the video preview during transcoding + const previewPath = video.getPreview().getPath() + const tmpPreviewPath = join(CONFIG.STORAGE.TMP_DIR, basename(previewPath)) + await copyFile(previewPath, tmpPreviewPath) + + const transcodeOptions = { + type: 'merge-audio' as 'merge-audio', + + inputPath: tmpPreviewPath, + outputPath: videoOutputPath, + + inputFileMutexReleaser, + + audioPath: audioInputPath, + resolution, + fps + } + + try { + await buildFFmpegVOD(job).transcode(transcodeOptions) + + await remove(audioInputPath) + await remove(tmpPreviewPath) + } catch (err) { + await remove(tmpPreviewPath) + throw err + } + + // Important to do this before getVideoFilename() to take in account the new file extension + inputVideoFile.extname = newExtname + inputVideoFile.resolution = resolution + inputVideoFile.filename = generateWebVideoFilename(inputVideoFile.resolution, newExtname) + + // ffmpeg generated a new video file, so update the video duration + // See https://trac.ffmpeg.org/ticket/5456 + video.duration = await getVideoStreamDuration(videoOutputPath) + await video.save() + + return onWebVideoFileTranscoding({ + video, + videoFile: inputVideoFile, + videoOutputPath, + wasAudioFile: true + }) + }) + + return result + } finally { + inputFileMutexReleaser() + } +} + +export async function onWebVideoFileTranscoding (options: { + video: MVideoFullLight + videoFile: MVideoFile + videoOutputPath: string + wasAudioFile?: boolean // default false +}) { + const { video, videoFile, videoOutputPath, wasAudioFile } = options + + const mutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) + + try { + await video.reload() + + const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile) + + const stats = await stat(videoOutputPath) + + const probe = await ffprobePromise(videoOutputPath) + const fps = await getVideoStreamFPS(videoOutputPath, probe) + const metadata = await buildFileMetadata(videoOutputPath, probe) + + await move(videoOutputPath, outputPath, { overwrite: true }) + + videoFile.size = stats.size + videoFile.fps = fps + videoFile.metadata = metadata + + await createTorrentAndSetInfoHash(video, videoFile) + + const oldFile = await VideoFileModel.loadWebVideoFile({ videoId: video.id, fps: videoFile.fps, resolution: videoFile.resolution }) + if (oldFile) await video.removeWebVideoFile(oldFile) + + await VideoFileModel.customUpsert(videoFile, 'video', undefined) + video.VideoFiles = await video.$get('VideoFiles') + + if (wasAudioFile) { + await JobQueue.Instance.createJob({ + type: 'generate-video-storyboard' as 'generate-video-storyboard', + payload: { + videoUUID: video.uuid, + // No need to federate, we process these jobs sequentially + federate: false + } + }) + } + + return { video, videoFile } + } finally { + mutexReleaser() + } +} diff --git a/server/server/lib/uploadx.ts b/server/server/lib/uploadx.ts new file mode 100644 index 000000000..81c93f77c --- /dev/null +++ b/server/server/lib/uploadx.ts @@ -0,0 +1,37 @@ +import express from 'express' +import { buildLogger } from '@server/helpers/logger.js' +import { getResumableUploadPath } from '@server/helpers/upload.js' +import { CONFIG } from '@server/initializers/config.js' +import { LogLevel, Uploadx } from '@uploadx/core' +import { extname } from 'path' + +const logger = buildLogger('uploadx') + +const uploadx = new Uploadx({ + directory: getResumableUploadPath(), + + expiration: { maxAge: undefined, rolling: true }, + + // Could be big with thumbnails/previews + maxMetadataSize: '10MB', + + logger: { + logLevel: CONFIG.LOG.LEVEL as LogLevel, + debug: logger.debug.bind(logger), + info: logger.info.bind(logger), + warn: logger.warn.bind(logger), + error: logger.error.bind(logger) + }, + + userIdentifier: (_, res: express.Response) => { + if (!res.locals.oauth) return undefined + + return res.locals.oauth.token.user.id + '' + }, + + filename: file => `${file.userId}-${file.id}${extname(file.metadata.filename)}` +}) + +export { + uploadx +} diff --git a/server/server/lib/user.ts b/server/server/lib/user.ts new file mode 100644 index 000000000..d391f384b --- /dev/null +++ b/server/server/lib/user.ts @@ -0,0 +1,308 @@ +import { Transaction } from 'sequelize' +import { + ActivityPubActorType, + UserAdminFlag, + UserAdminFlagType, + UserNotificationSetting, + UserNotificationSettingValue, + UserRole, + UserRoleType +} from '@peertube/peertube-models' +import { logger } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' +import { UserModel } from '@server/models/user/user.js' +import { MActorDefault } from '@server/types/models/actor/index.js' +import { SERVER_ACTOR_NAME, WEBSERVER } from '../initializers/constants.js' +import { sequelizeTypescript } from '../initializers/database.js' +import { AccountModel } from '../models/account/account.js' +import { UserNotificationSettingModel } from '../models/user/user-notification-setting.js' +import { MAccountDefault, MChannelActor } from '../types/models/index.js' +import { MRegistration, MUser, MUserDefault, MUserId } from '../types/models/user/index.js' +import { generateAndSaveActorKeys } from './activitypub/actors/index.js' +import { getLocalAccountActivityPubUrl } from './activitypub/url.js' +import { Emailer } from './emailer.js' +import { LiveQuotaStore } from './live/live-quota-store.js' +import { buildActorInstance, findAvailableLocalActorName } from './local-actor.js' +import { Redis } from './redis.js' +import { createLocalVideoChannel } from './video-channel.js' +import { createWatchLaterPlaylist } from './video-playlist.js' + +type ChannelNames = { name: string, displayName: string } + +function buildUser (options: { + username: string + password: string + email: string + + role?: UserRoleType // Default to UserRole.User + adminFlags?: UserAdminFlagType // Default to UserAdminFlag.NONE + + emailVerified: boolean | null + + videoQuota?: number // Default to CONFIG.USER.VIDEO_QUOTA + videoQuotaDaily?: number // Default to CONFIG.USER.VIDEO_QUOTA_DAILY + + pluginAuth?: string +}): MUser { + const { + username, + password, + email, + role = UserRole.USER, + emailVerified, + videoQuota = CONFIG.USER.VIDEO_QUOTA, + videoQuotaDaily = CONFIG.USER.VIDEO_QUOTA_DAILY, + adminFlags = UserAdminFlag.NONE, + pluginAuth + } = options + + return new UserModel({ + username, + password, + email, + + nsfwPolicy: CONFIG.INSTANCE.DEFAULT_NSFW_POLICY, + p2pEnabled: CONFIG.DEFAULTS.P2P.WEBAPP.ENABLED, + videosHistoryEnabled: CONFIG.USER.HISTORY.VIDEOS.ENABLED, + + autoPlayVideo: true, + + role, + emailVerified, + adminFlags, + + videoQuota, + videoQuotaDaily, + + pluginAuth + }) +} + +// --------------------------------------------------------------------------- + +async function createUserAccountAndChannelAndPlaylist (parameters: { + userToCreate: MUser + userDisplayName?: string + channelNames?: ChannelNames + validateUser?: boolean +}): Promise<{ user: MUserDefault, account: MAccountDefault, videoChannel: MChannelActor }> { + const { userToCreate, userDisplayName, channelNames, validateUser = true } = parameters + + const { user, account, videoChannel } = await sequelizeTypescript.transaction(async t => { + const userOptions = { + transaction: t, + validate: validateUser + } + + const userCreated: MUserDefault = await userToCreate.save(userOptions) + userCreated.NotificationSetting = await createDefaultUserNotificationSettings(userCreated, t) + + const accountCreated = await createLocalAccountWithoutKeys({ + name: userCreated.username, + displayName: userDisplayName, + userId: userCreated.id, + applicationId: null, + t + }) + userCreated.Account = accountCreated + + const channelAttributes = await buildChannelAttributes({ user: userCreated, transaction: t, channelNames }) + const videoChannel = await createLocalVideoChannel(channelAttributes, accountCreated, t) + + const videoPlaylist = await createWatchLaterPlaylist(accountCreated, t) + + return { user: userCreated, account: accountCreated, videoChannel, videoPlaylist } + }) + + const [ accountActorWithKeys, channelActorWithKeys ] = await Promise.all([ + generateAndSaveActorKeys(account.Actor), + generateAndSaveActorKeys(videoChannel.Actor) + ]) + + account.Actor = accountActorWithKeys + videoChannel.Actor = channelActorWithKeys + + return { user, account, videoChannel } +} + +async function createLocalAccountWithoutKeys (parameters: { + name: string + displayName?: string + userId: number | null + applicationId: number | null + t: Transaction | undefined + type?: ActivityPubActorType +}) { + const { name, displayName, userId, applicationId, t, type = 'Person' } = parameters + const url = getLocalAccountActivityPubUrl(name) + + const actorInstance = buildActorInstance(type, url, name) + const actorInstanceCreated: MActorDefault = await actorInstance.save({ transaction: t }) + + const accountInstance = new AccountModel({ + name: displayName || name, + userId, + applicationId, + actorId: actorInstanceCreated.id + }) + + const accountInstanceCreated: MAccountDefault = await accountInstance.save({ transaction: t }) + accountInstanceCreated.Actor = actorInstanceCreated + + return accountInstanceCreated +} + +async function createApplicationActor (applicationId: number) { + const accountCreated = await createLocalAccountWithoutKeys({ + name: SERVER_ACTOR_NAME, + userId: null, + applicationId, + t: undefined, + type: 'Application' + }) + + accountCreated.Actor = await generateAndSaveActorKeys(accountCreated.Actor) + + return accountCreated +} + +// --------------------------------------------------------------------------- + +async function sendVerifyUserEmail (user: MUser, isPendingEmail = false) { + const verificationString = await Redis.Instance.setUserVerifyEmailVerificationString(user.id) + let verifyEmailUrl = `${WEBSERVER.URL}/verify-account/email?userId=${user.id}&verificationString=${verificationString}` + + if (isPendingEmail) verifyEmailUrl += '&isPendingEmail=true' + + const to = isPendingEmail + ? user.pendingEmail + : user.email + + const username = user.username + + Emailer.Instance.addVerifyEmailJob({ username, to, verifyEmailUrl, isRegistrationRequest: false }) +} + +async function sendVerifyRegistrationEmail (registration: MRegistration) { + const verificationString = await Redis.Instance.setRegistrationVerifyEmailVerificationString(registration.id) + const verifyEmailUrl = `${WEBSERVER.URL}/verify-account/email?registrationId=${registration.id}&verificationString=${verificationString}` + + const to = registration.email + const username = registration.username + + Emailer.Instance.addVerifyEmailJob({ username, to, verifyEmailUrl, isRegistrationRequest: true }) +} + +// --------------------------------------------------------------------------- + +async function getOriginalVideoFileTotalFromUser (user: MUserId) { + // Don't use sequelize because we need to use a sub query + const query = UserModel.generateUserQuotaBaseSQL({ + withSelect: true, + whereUserId: '$userId', + daily: false + }) + + const base = await UserModel.getTotalRawQuery(query, user.id) + + return base + LiveQuotaStore.Instance.getLiveQuotaOf(user.id) +} + +// Returns cumulative size of all video files uploaded in the last 24 hours. +async function getOriginalVideoFileTotalDailyFromUser (user: MUserId) { + // Don't use sequelize because we need to use a sub query + const query = UserModel.generateUserQuotaBaseSQL({ + withSelect: true, + whereUserId: '$userId', + daily: true + }) + + const base = await UserModel.getTotalRawQuery(query, user.id) + + return base + LiveQuotaStore.Instance.getLiveQuotaOf(user.id) +} + +async function isAbleToUploadVideo (userId: number, newVideoSize: number) { + const user = await UserModel.loadById(userId) + + if (user.videoQuota === -1 && user.videoQuotaDaily === -1) return Promise.resolve(true) + + const [ totalBytes, totalBytesDaily ] = await Promise.all([ + getOriginalVideoFileTotalFromUser(user), + getOriginalVideoFileTotalDailyFromUser(user) + ]) + + const uploadedTotal = newVideoSize + totalBytes + const uploadedDaily = newVideoSize + totalBytesDaily + + logger.debug( + 'Check user %d quota to upload another video.', userId, + { totalBytes, totalBytesDaily, videoQuota: user.videoQuota, videoQuotaDaily: user.videoQuotaDaily, newVideoSize } + ) + + if (user.videoQuotaDaily === -1) return uploadedTotal < user.videoQuota + if (user.videoQuota === -1) return uploadedDaily < user.videoQuotaDaily + + return uploadedTotal < user.videoQuota && uploadedDaily < user.videoQuotaDaily +} + +// --------------------------------------------------------------------------- + +export { + getOriginalVideoFileTotalFromUser, + getOriginalVideoFileTotalDailyFromUser, + createApplicationActor, + createUserAccountAndChannelAndPlaylist, + createLocalAccountWithoutKeys, + + sendVerifyUserEmail, + sendVerifyRegistrationEmail, + + isAbleToUploadVideo, + buildUser +} + +// --------------------------------------------------------------------------- + +function createDefaultUserNotificationSettings (user: MUserId, t: Transaction | undefined) { + const values: UserNotificationSetting & { userId: number } = { + userId: user.id, + newVideoFromSubscription: UserNotificationSettingValue.WEB, + newCommentOnMyVideo: UserNotificationSettingValue.WEB, + myVideoImportFinished: UserNotificationSettingValue.WEB, + myVideoPublished: UserNotificationSettingValue.WEB, + abuseAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + videoAutoBlacklistAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + blacklistOnMyVideo: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + newUserRegistration: UserNotificationSettingValue.WEB, + commentMention: UserNotificationSettingValue.WEB, + newFollow: UserNotificationSettingValue.WEB, + newInstanceFollower: UserNotificationSettingValue.WEB, + abuseNewMessage: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + abuseStateChange: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + autoInstanceFollowing: UserNotificationSettingValue.WEB, + newPeerTubeVersion: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, + newPluginVersion: UserNotificationSettingValue.WEB, + myVideoStudioEditionFinished: UserNotificationSettingValue.WEB + } + + return UserNotificationSettingModel.create(values, { transaction: t }) +} + +async function buildChannelAttributes (options: { + user: MUser + transaction?: Transaction + channelNames?: ChannelNames +}) { + const { user, transaction, channelNames } = options + + if (channelNames) return channelNames + + const channelName = await findAvailableLocalActorName(user.username + '_channel', transaction) + const videoChannelDisplayName = `Main ${user.username} channel` + + return { + name: channelName, + displayName: videoChannelDisplayName + } +} diff --git a/server/server/lib/video-blacklist.ts b/server/server/lib/video-blacklist.ts new file mode 100644 index 000000000..3fd0a59ed --- /dev/null +++ b/server/server/lib/video-blacklist.ts @@ -0,0 +1,144 @@ +import { Transaction } from 'sequelize' +import { LiveVideoError, UserAdminFlag, UserRight, VideoBlacklistCreate, VideoBlacklistType } from '@peertube/peertube-models' +import { afterCommitIfTransaction } from '@server/helpers/database-utils.js' +import { sequelizeTypescript } from '@server/initializers/database.js' +import { + MUser, + MVideoAccountLight, + MVideoBlacklist, + MVideoBlacklistVideo, + MVideoFullLight, + MVideoWithBlacklistLight +} from '@server/types/models/index.js' +import { logger, loggerTagsFactory } from '../helpers/logger.js' +import { CONFIG } from '../initializers/config.js' +import { VideoBlacklistModel } from '../models/video/video-blacklist.js' +import { sendDeleteVideo } from './activitypub/send/index.js' +import { federateVideoIfNeeded } from './activitypub/videos/index.js' +import { LiveManager } from './live/live-manager.js' +import { Notifier } from './notifier/index.js' +import { Hooks } from './plugins/hooks.js' + +const lTags = loggerTagsFactory('blacklist') + +async function autoBlacklistVideoIfNeeded (parameters: { + video: MVideoWithBlacklistLight + user?: MUser + isRemote: boolean + isNew: boolean + isNewFile: boolean + notify?: boolean + transaction?: Transaction +}) { + const { video, user, isRemote, isNew, isNewFile, notify = true, transaction } = parameters + const doAutoBlacklist = await Hooks.wrapFun( + autoBlacklistNeeded, + { video, user, isRemote, isNew, isNewFile }, + 'filter:video.auto-blacklist.result' + ) + + if (!doAutoBlacklist) return false + + const videoBlacklistToCreate = { + videoId: video.id, + unfederated: true, + reason: 'Auto-blacklisted. Moderator review required.', + type: VideoBlacklistType.AUTO_BEFORE_PUBLISHED + } + const [ videoBlacklist ] = await VideoBlacklistModel.findOrCreate({ + where: { + videoId: video.id + }, + defaults: videoBlacklistToCreate, + transaction + }) + video.VideoBlacklist = videoBlacklist + + videoBlacklist.Video = video + + if (notify) { + afterCommitIfTransaction(transaction, () => { + Notifier.Instance.notifyOnVideoAutoBlacklist(videoBlacklist) + }) + } + + logger.info('Video %s auto-blacklisted.', video.uuid, lTags(video.uuid)) + + return true +} + +async function blacklistVideo (videoInstance: MVideoAccountLight, options: VideoBlacklistCreate) { + const blacklist: MVideoBlacklistVideo = await VideoBlacklistModel.create({ + videoId: videoInstance.id, + unfederated: options.unfederate === true, + reason: options.reason, + type: VideoBlacklistType.MANUAL + }) + blacklist.Video = videoInstance + + if (options.unfederate === true) { + await sendDeleteVideo(videoInstance, undefined) + } + + if (videoInstance.isLive) { + LiveManager.Instance.stopSessionOf(videoInstance.uuid, LiveVideoError.BLACKLISTED) + } + + Notifier.Instance.notifyOnVideoBlacklist(blacklist) +} + +async function unblacklistVideo (videoBlacklist: MVideoBlacklist, video: MVideoFullLight) { + const videoBlacklistType = await sequelizeTypescript.transaction(async t => { + const unfederated = videoBlacklist.unfederated + const videoBlacklistType = videoBlacklist.type + + await videoBlacklist.destroy({ transaction: t }) + video.VideoBlacklist = undefined + + // Re federate the video + if (unfederated === true) { + await federateVideoIfNeeded(video, true, t) + } + + return videoBlacklistType + }) + + Notifier.Instance.notifyOnVideoUnblacklist(video) + + if (videoBlacklistType === VideoBlacklistType.AUTO_BEFORE_PUBLISHED) { + Notifier.Instance.notifyOnVideoPublishedAfterRemovedFromAutoBlacklist(video) + + // Delete on object so new video notifications will send + delete video.VideoBlacklist + Notifier.Instance.notifyOnNewVideoIfNeeded(video) + } +} + +// --------------------------------------------------------------------------- + +export { + autoBlacklistVideoIfNeeded, + blacklistVideo, + unblacklistVideo +} + +// --------------------------------------------------------------------------- + +function autoBlacklistNeeded (parameters: { + video: MVideoWithBlacklistLight + isRemote: boolean + isNew: boolean + isNewFile: boolean + user?: MUser +}) { + const { user, video, isRemote, isNew, isNewFile } = parameters + + // Already blacklisted + if (video.VideoBlacklist) return false + if (!CONFIG.AUTO_BLACKLIST.VIDEOS.OF_USERS.ENABLED || !user) return false + if (isRemote || (isNew === false && isNewFile === false)) return false + + if (user.hasRight(UserRight.MANAGE_VIDEO_BLACKLIST) || user.hasAdminFlag(UserAdminFlag.BYPASS_VIDEO_AUTO_BLACKLIST)) return false + + return true +} diff --git a/server/server/lib/video-channel.ts b/server/server/lib/video-channel.ts new file mode 100644 index 000000000..ea24b5f1d --- /dev/null +++ b/server/server/lib/video-channel.ts @@ -0,0 +1,50 @@ +import * as Sequelize from 'sequelize' +import { VideoChannelCreate } from '@peertube/peertube-models' +import { VideoChannelModel } from '../models/video/video-channel.js' +import { VideoModel } from '../models/video/video.js' +import { MAccountId, MChannelId } from '../types/models/index.js' +import { getLocalVideoChannelActivityPubUrl } from './activitypub/url.js' +import { federateVideoIfNeeded } from './activitypub/videos/index.js' +import { buildActorInstance } from './local-actor.js' + +async function createLocalVideoChannel (videoChannelInfo: VideoChannelCreate, account: MAccountId, t: Sequelize.Transaction) { + const url = getLocalVideoChannelActivityPubUrl(videoChannelInfo.name) + const actorInstance = buildActorInstance('Group', url, videoChannelInfo.name) + + const actorInstanceCreated = await actorInstance.save({ transaction: t }) + + const videoChannelData = { + name: videoChannelInfo.displayName, + description: videoChannelInfo.description, + support: videoChannelInfo.support, + accountId: account.id, + actorId: actorInstanceCreated.id + } + + const videoChannel = new VideoChannelModel(videoChannelData) + + const options = { transaction: t } + const videoChannelCreated = await videoChannel.save(options) + + videoChannelCreated.Actor = actorInstanceCreated + + // No need to send this empty video channel to followers + return videoChannelCreated +} + +async function federateAllVideosOfChannel (videoChannel: MChannelId) { + const videoIds = await VideoModel.getAllIdsFromChannel(videoChannel) + + for (const videoId of videoIds) { + const video = await VideoModel.loadFull(videoId) + + await federateVideoIfNeeded(video, false) + } +} + +// --------------------------------------------------------------------------- + +export { + createLocalVideoChannel, + federateAllVideosOfChannel +} diff --git a/server/server/lib/video-comment.ts b/server/server/lib/video-comment.ts new file mode 100644 index 000000000..f89215671 --- /dev/null +++ b/server/server/lib/video-comment.ts @@ -0,0 +1,115 @@ +import { ResultList, VideoCommentThreadTree } from '@peertube/peertube-models' +import { logger } from '@server/helpers/logger.js' +import { sequelizeTypescript } from '@server/initializers/database.js' +import express from 'express' +import cloneDeep from 'lodash-es/cloneDeep.js' +import * as Sequelize from 'sequelize' +import { VideoCommentModel } from '../models/video/video-comment.js' +import { + MAccountDefault, + MComment, + MCommentFormattable, + MCommentOwnerVideo, + MCommentOwnerVideoReply, + MVideoFullLight +} from '../types/models/index.js' +import { sendCreateVideoComment, sendDeleteVideoComment } from './activitypub/send/index.js' +import { getLocalVideoCommentActivityPubUrl } from './activitypub/url.js' +import { Hooks } from './plugins/hooks.js' + +async function removeComment (commentArg: MComment, req: express.Request, res: express.Response) { + let videoCommentInstanceBefore: MCommentOwnerVideo + + await sequelizeTypescript.transaction(async t => { + const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(commentArg.url, t) + + videoCommentInstanceBefore = cloneDeep(comment) + + if (comment.isOwned() || comment.Video.isOwned()) { + await sendDeleteVideoComment(comment, t) + } + + comment.markAsDeleted() + + await comment.save({ transaction: t }) + + logger.info('Video comment %d deleted.', comment.id) + }) + + Hooks.runAction('action:api.video-comment.deleted', { comment: videoCommentInstanceBefore, req, res }) +} + +async function createVideoComment (obj: { + text: string + inReplyToComment: MComment | null + video: MVideoFullLight + account: MAccountDefault +}, t: Sequelize.Transaction) { + let originCommentId: number | null = null + let inReplyToCommentId: number | null = null + + if (obj.inReplyToComment && obj.inReplyToComment !== null) { + originCommentId = obj.inReplyToComment.originCommentId || obj.inReplyToComment.id + inReplyToCommentId = obj.inReplyToComment.id + } + + const comment = await VideoCommentModel.create({ + text: obj.text, + originCommentId, + inReplyToCommentId, + videoId: obj.video.id, + accountId: obj.account.id, + url: new Date().toISOString() + }, { transaction: t, validate: false }) + + comment.url = getLocalVideoCommentActivityPubUrl(obj.video, comment) + + const savedComment: MCommentOwnerVideoReply = await comment.save({ transaction: t }) + savedComment.InReplyToVideoComment = obj.inReplyToComment + savedComment.Video = obj.video + savedComment.Account = obj.account + + await sendCreateVideoComment(savedComment, t) + + return savedComment +} + +function buildFormattedCommentTree (resultList: ResultList): VideoCommentThreadTree { + // Comments are sorted by id ASC + const comments = resultList.data + + const comment = comments.shift() + const thread: VideoCommentThreadTree = { + comment: comment.toFormattedJSON(), + children: [] + } + const idx = { + [comment.id]: thread + } + + while (comments.length !== 0) { + const childComment = comments.shift() + + const childCommentThread: VideoCommentThreadTree = { + comment: childComment.toFormattedJSON(), + children: [] + } + + const parentCommentThread = idx[childComment.inReplyToCommentId] + // Maybe the parent comment was blocked by the admin/user + if (!parentCommentThread) continue + + parentCommentThread.children.push(childCommentThread) + idx[childComment.id] = childCommentThread + } + + return thread +} + +// --------------------------------------------------------------------------- + +export { + removeComment, + createVideoComment, + buildFormattedCommentTree +} diff --git a/server/server/lib/video-file.ts b/server/server/lib/video-file.ts new file mode 100644 index 000000000..9d8a6e8fc --- /dev/null +++ b/server/server/lib/video-file.ts @@ -0,0 +1,144 @@ +import { FfprobeData } from 'fluent-ffmpeg' +import { VideoFileMetadata, VideoResolution } from '@peertube/peertube-models' +import { logger } from '@server/helpers/logger.js' +import { VideoFileModel } from '@server/models/video/video-file.js' +import { MVideoWithAllFiles } from '@server/types/models/index.js' +import { getFileSize, getLowercaseExtension } from '@peertube/peertube-node-utils' +import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, isAudioFile } from '@peertube/peertube-ffmpeg' +import { lTags } from './object-storage/shared/index.js' +import { generateHLSVideoFilename, generateWebVideoFilename } from './paths.js' +import { VideoPathManager } from './video-path-manager.js' + +async function buildNewFile (options: { + path: string + mode: 'web-video' | 'hls' +}) { + const { path, mode } = options + + const probe = await ffprobePromise(path) + const size = await getFileSize(path) + + const videoFile = new VideoFileModel({ + extname: getLowercaseExtension(path), + size, + metadata: await buildFileMetadata(path, probe) + }) + + if (await isAudioFile(path, probe)) { + videoFile.resolution = VideoResolution.H_NOVIDEO + } else { + videoFile.fps = await getVideoStreamFPS(path, probe) + videoFile.resolution = (await getVideoStreamDimensionsInfo(path, probe)).resolution + } + + videoFile.filename = mode === 'web-video' + ? generateWebVideoFilename(videoFile.resolution, videoFile.extname) + : generateHLSVideoFilename(videoFile.resolution) + + return videoFile +} + +// --------------------------------------------------------------------------- + +async function removeHLSPlaylist (video: MVideoWithAllFiles) { + const hls = video.getHLSPlaylist() + if (!hls) return + + const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) + + try { + await video.removeStreamingPlaylistFiles(hls) + await hls.destroy() + + video.VideoStreamingPlaylists = video.VideoStreamingPlaylists.filter(p => p.id !== hls.id) + } finally { + videoFileMutexReleaser() + } +} + +async function removeHLSFile (video: MVideoWithAllFiles, fileToDeleteId: number) { + logger.info('Deleting HLS file %d of %s.', fileToDeleteId, video.url, lTags(video.uuid)) + + const hls = video.getHLSPlaylist() + const files = hls.VideoFiles + + if (files.length === 1) { + await removeHLSPlaylist(video) + return undefined + } + + const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) + + try { + const toDelete = files.find(f => f.id === fileToDeleteId) + await video.removeStreamingPlaylistVideoFile(video.getHLSPlaylist(), toDelete) + await toDelete.destroy() + + hls.VideoFiles = hls.VideoFiles.filter(f => f.id !== toDelete.id) + } finally { + videoFileMutexReleaser() + } + + return hls +} + +// --------------------------------------------------------------------------- + +async function removeAllWebVideoFiles (video: MVideoWithAllFiles) { + const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) + + try { + for (const file of video.VideoFiles) { + await video.removeWebVideoFile(file) + await file.destroy() + } + + video.VideoFiles = [] + } finally { + videoFileMutexReleaser() + } + + return video +} + +async function removeWebVideoFile (video: MVideoWithAllFiles, fileToDeleteId: number) { + const files = video.VideoFiles + + if (files.length === 1) { + return removeAllWebVideoFiles(video) + } + + const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(video.uuid) + try { + const toDelete = files.find(f => f.id === fileToDeleteId) + await video.removeWebVideoFile(toDelete) + await toDelete.destroy() + + video.VideoFiles = files.filter(f => f.id !== toDelete.id) + } finally { + videoFileMutexReleaser() + } + + return video +} + +// --------------------------------------------------------------------------- + +async function buildFileMetadata (path: string, existingProbe?: FfprobeData) { + const metadata = existingProbe || await ffprobePromise(path) + + return new VideoFileMetadata(metadata) +} + +// --------------------------------------------------------------------------- + +export { + buildNewFile, + + removeHLSPlaylist, + removeHLSFile, + removeAllWebVideoFiles, + removeWebVideoFile, + + buildFileMetadata +} diff --git a/server/server/lib/video-path-manager.ts b/server/server/lib/video-path-manager.ts new file mode 100644 index 000000000..0a8604a58 --- /dev/null +++ b/server/server/lib/video-path-manager.ts @@ -0,0 +1,180 @@ +import { Mutex } from 'async-mutex' +import { remove } from 'fs-extra/esm' +import { extname, join } from 'path' +import { VideoStorage } from '@peertube/peertube-models' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { extractVideo } from '@server/helpers/video.js' +import { CONFIG } from '@server/initializers/config.js' +import { DIRECTORIES } from '@server/initializers/constants.js' +import { + MStreamingPlaylistVideo, + MVideo, + MVideoFile, + MVideoFileStreamingPlaylistVideo, + MVideoFileVideo +} from '@server/types/models/index.js' +import { buildUUID } from '@peertube/peertube-node-utils' +import { makeHLSFileAvailable, makeWebVideoFileAvailable } from './object-storage/index.js' +import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from './paths.js' +import { isVideoInPrivateDirectory } from './video-privacy.js' + +type MakeAvailableCB = (path: string) => Promise | T + +const lTags = loggerTagsFactory('video-path-manager') + +class VideoPathManager { + + private static instance: VideoPathManager + + // Key is a video UUID + private readonly videoFileMutexStore = new Map() + + private constructor () {} + + getFSHLSOutputPath (video: MVideo, filename?: string) { + const base = getHLSDirectory(video) + if (!filename) return base + + return join(base, filename) + } + + getFSRedundancyVideoFilePath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { + if (videoFile.isHLS()) { + const video = extractVideo(videoOrPlaylist) + + return join(getHLSRedundancyDirectory(video), videoFile.filename) + } + + return join(CONFIG.STORAGE.REDUNDANCY_DIR, videoFile.filename) + } + + getFSVideoFileOutputPath (videoOrPlaylist: MVideo | MStreamingPlaylistVideo, videoFile: MVideoFile) { + const video = extractVideo(videoOrPlaylist) + + if (videoFile.isHLS()) { + return join(getHLSDirectory(video), videoFile.filename) + } + + if (isVideoInPrivateDirectory(video.privacy)) { + return join(DIRECTORIES.VIDEOS.PRIVATE, videoFile.filename) + } + + return join(DIRECTORIES.VIDEOS.PUBLIC, videoFile.filename) + } + + async makeAvailableVideoFile (videoFile: MVideoFileVideo | MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB) { + if (videoFile.storage === VideoStorage.FILE_SYSTEM) { + return this.makeAvailableFactory( + () => this.getFSVideoFileOutputPath(videoFile.getVideoOrStreamingPlaylist(), videoFile), + false, + cb + ) + } + + const destination = this.buildTMPDestination(videoFile.filename) + + if (videoFile.isHLS()) { + const playlist = (videoFile as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist + + return this.makeAvailableFactory( + () => makeHLSFileAvailable(playlist, videoFile.filename, destination), + true, + cb + ) + } + + return this.makeAvailableFactory( + () => makeWebVideoFileAvailable(videoFile.filename, destination), + true, + cb + ) + } + + async makeAvailableResolutionPlaylistFile (videoFile: MVideoFileStreamingPlaylistVideo, cb: MakeAvailableCB) { + const filename = getHlsResolutionPlaylistFilename(videoFile.filename) + + if (videoFile.storage === VideoStorage.FILE_SYSTEM) { + return this.makeAvailableFactory( + () => join(getHLSDirectory(videoFile.getVideo()), filename), + false, + cb + ) + } + + const playlist = videoFile.VideoStreamingPlaylist + return this.makeAvailableFactory( + () => makeHLSFileAvailable(playlist, filename, this.buildTMPDestination(filename)), + true, + cb + ) + } + + async makeAvailablePlaylistFile (playlist: MStreamingPlaylistVideo, filename: string, cb: MakeAvailableCB) { + if (playlist.storage === VideoStorage.FILE_SYSTEM) { + return this.makeAvailableFactory( + () => join(getHLSDirectory(playlist.Video), filename), + false, + cb + ) + } + + return this.makeAvailableFactory( + () => makeHLSFileAvailable(playlist, filename, this.buildTMPDestination(filename)), + true, + cb + ) + } + + async lockFiles (videoUUID: string) { + if (!this.videoFileMutexStore.has(videoUUID)) { + this.videoFileMutexStore.set(videoUUID, new Mutex()) + } + + const mutex = this.videoFileMutexStore.get(videoUUID) + const releaser = await mutex.acquire() + + logger.debug('Locked files of %s.', videoUUID, lTags(videoUUID)) + + return releaser + } + + unlockFiles (videoUUID: string) { + const mutex = this.videoFileMutexStore.get(videoUUID) + + mutex.release() + + logger.debug('Released lockfiles of %s.', videoUUID, lTags(videoUUID)) + } + + private async makeAvailableFactory (method: () => Promise | string, clean: boolean, cb: MakeAvailableCB) { + let result: T + + const destination = await method() + + try { + result = await cb(destination) + } catch (err) { + if (destination && clean) await remove(destination) + throw err + } + + if (clean) await remove(destination) + + return result + } + + private buildTMPDestination (filename: string) { + return join(CONFIG.STORAGE.TMP_DIR, buildUUID() + extname(filename)) + + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} + +// --------------------------------------------------------------------------- + +export { + VideoPathManager +} diff --git a/server/server/lib/video-playlist.ts b/server/server/lib/video-playlist.ts new file mode 100644 index 000000000..5c97f9c9a --- /dev/null +++ b/server/server/lib/video-playlist.ts @@ -0,0 +1,29 @@ +import * as Sequelize from 'sequelize' +import { VideoPlaylistPrivacy, VideoPlaylistType } from '@peertube/peertube-models' +import { VideoPlaylistModel } from '../models/video/video-playlist.js' +import { MAccount } from '../types/models/index.js' +import { MVideoPlaylistOwner } from '../types/models/video/video-playlist.js' +import { getLocalVideoPlaylistActivityPubUrl } from './activitypub/url.js' + +async function createWatchLaterPlaylist (account: MAccount, t: Sequelize.Transaction) { + const videoPlaylist: MVideoPlaylistOwner = new VideoPlaylistModel({ + name: 'Watch later', + privacy: VideoPlaylistPrivacy.PRIVATE, + type: VideoPlaylistType.WATCH_LATER, + ownerAccountId: account.id + }) + + videoPlaylist.url = getLocalVideoPlaylistActivityPubUrl(videoPlaylist) // We use the UUID, so set the URL after building the object + + await videoPlaylist.save({ transaction: t }) + + videoPlaylist.OwnerAccount = account + + return videoPlaylist +} + +// --------------------------------------------------------------------------- + +export { + createWatchLaterPlaylist +} diff --git a/server/server/lib/video-pre-import.ts b/server/server/lib/video-pre-import.ts new file mode 100644 index 000000000..0298e121e --- /dev/null +++ b/server/server/lib/video-pre-import.ts @@ -0,0 +1,331 @@ +import { remove } from 'fs-extra/esm' +import { + ThumbnailType, + ThumbnailType_Type, + VideoImportCreate, + VideoImportPayload, + VideoImportState, + VideoPrivacy, + VideoState +} from '@peertube/peertube-models' +import { moveAndProcessCaptionFile } from '@server/helpers/captions-utils.js' +import { isVTTFileValid } from '@server/helpers/custom-validators/video-captions.js' +import { isVideoFileExtnameValid } from '@server/helpers/custom-validators/videos.js' +import { isResolvingToUnicastOnly } from '@server/helpers/dns.js' +import { logger } from '@server/helpers/logger.js' +import { YoutubeDLInfo, YoutubeDLWrapper } from '@server/helpers/youtube-dl/index.js' +import { CONFIG } from '@server/initializers/config.js' +import { sequelizeTypescript } from '@server/initializers/database.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { ServerConfigManager } from '@server/lib/server-config-manager.js' +import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js' +import { setVideoTags } from '@server/lib/video.js' +import { VideoCaptionModel } from '@server/models/video/video-caption.js' +import { VideoImportModel } from '@server/models/video/video-import.js' +import { VideoPasswordModel } from '@server/models/video/video-password.js' +import { VideoModel } from '@server/models/video/video.js' +import { FilteredModelAttributes } from '@server/types/index.js' +import { + MChannelAccountDefault, + MChannelSync, + MThumbnail, + MUser, + MVideoAccountDefault, + MVideoCaption, + MVideoImportFormattable, + MVideoTag, + MVideoThumbnail, + MVideoWithBlacklistLight +} from '@server/types/models/index.js' +import { getLocalVideoActivityPubUrl } from './activitypub/url.js' +import { updateLocalVideoMiniatureFromExisting, updateLocalVideoMiniatureFromUrl } from './thumbnail.js' + +class YoutubeDlImportError extends Error { + code: YoutubeDlImportError.CODE + cause?: Error // Property to remove once ES2022 is used + constructor ({ message, code }) { + super(message) + this.code = code + } + + static fromError (err: Error, code: YoutubeDlImportError.CODE, message?: string) { + const ytDlErr = new this({ message: message ?? err.message, code }) + ytDlErr.cause = err + ytDlErr.stack = err.stack // Useless once ES2022 is used + return ytDlErr + } +} + +namespace YoutubeDlImportError { + export enum CODE { + FETCH_ERROR, + NOT_ONLY_UNICAST_URL + } +} + +// --------------------------------------------------------------------------- + +async function insertFromImportIntoDB (parameters: { + video: MVideoThumbnail + thumbnailModel: MThumbnail + previewModel: MThumbnail + videoChannel: MChannelAccountDefault + tags: string[] + videoImportAttributes: FilteredModelAttributes + user: MUser + videoPasswords?: string[] +}): Promise { + const { video, thumbnailModel, previewModel, videoChannel, tags, videoImportAttributes, user, videoPasswords } = parameters + + const videoImport = await sequelizeTypescript.transaction(async t => { + const sequelizeOptions = { transaction: t } + + // Save video object in database + const videoCreated = await video.save(sequelizeOptions) as (MVideoAccountDefault & MVideoWithBlacklistLight & MVideoTag) + videoCreated.VideoChannel = videoChannel + + if (thumbnailModel) await videoCreated.addAndSaveThumbnail(thumbnailModel, t) + if (previewModel) await videoCreated.addAndSaveThumbnail(previewModel, t) + + if (videoCreated.privacy === VideoPrivacy.PASSWORD_PROTECTED) { + await VideoPasswordModel.addPasswords(videoPasswords, video.id, t) + } + + await autoBlacklistVideoIfNeeded({ + video: videoCreated, + user, + notify: false, + isRemote: false, + isNew: true, + isNewFile: true, + transaction: t + }) + + await setVideoTags({ video: videoCreated, tags, transaction: t }) + + // Create video import object in database + const videoImport = await VideoImportModel.create( + Object.assign({ videoId: videoCreated.id }, videoImportAttributes), + sequelizeOptions + ) as MVideoImportFormattable + videoImport.Video = videoCreated + + return videoImport + }) + + return videoImport +} + +async function buildVideoFromImport ({ channelId, importData, importDataOverride, importType }: { + channelId: number + importData: YoutubeDLInfo + importDataOverride?: Partial + importType: 'url' | 'torrent' +}): Promise { + let videoData = { + name: importDataOverride?.name || importData.name || 'Unknown name', + remote: false, + category: importDataOverride?.category || importData.category, + licence: importDataOverride?.licence ?? importData.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE, + language: importDataOverride?.language || importData.language, + commentsEnabled: importDataOverride?.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED, + downloadEnabled: importDataOverride?.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED, + waitTranscoding: importDataOverride?.waitTranscoding ?? true, + state: VideoState.TO_IMPORT, + nsfw: importDataOverride?.nsfw || importData.nsfw || false, + description: importDataOverride?.description || importData.description, + support: importDataOverride?.support || null, + privacy: importDataOverride?.privacy || VideoPrivacy.PRIVATE, + duration: 0, // duration will be set by the import job + channelId, + originallyPublishedAt: importDataOverride?.originallyPublishedAt + ? new Date(importDataOverride?.originallyPublishedAt) + : importData.originallyPublishedAtWithoutTime + } + + videoData = await Hooks.wrapObject( + videoData, + importType === 'url' + ? 'filter:api.video.import-url.video-attribute.result' + : 'filter:api.video.import-torrent.video-attribute.result' + ) + + const video = new VideoModel(videoData) + video.url = getLocalVideoActivityPubUrl(video) + + return video +} + +async function buildYoutubeDLImport (options: { + targetUrl: string + channel: MChannelAccountDefault + user: MUser + channelSync?: MChannelSync + importDataOverride?: Partial + thumbnailFilePath?: string + previewFilePath?: string +}) { + const { targetUrl, channel, channelSync, importDataOverride, thumbnailFilePath, previewFilePath, user } = options + + const youtubeDL = new YoutubeDLWrapper( + targetUrl, + ServerConfigManager.Instance.getEnabledResolutions('vod'), + CONFIG.TRANSCODING.ALWAYS_TRANSCODE_ORIGINAL_RESOLUTION + ) + + // Get video infos + let youtubeDLInfo: YoutubeDLInfo + try { + youtubeDLInfo = await youtubeDL.getInfoForDownload() + } catch (err) { + throw YoutubeDlImportError.fromError( + err, YoutubeDlImportError.CODE.FETCH_ERROR, `Cannot fetch information from import for URL ${targetUrl}` + ) + } + + if (!await hasUnicastURLsOnly(youtubeDLInfo)) { + throw new YoutubeDlImportError({ + message: 'Cannot use non unicast IP as targetUrl.', + code: YoutubeDlImportError.CODE.NOT_ONLY_UNICAST_URL + }) + } + + const video = await buildVideoFromImport({ + channelId: channel.id, + importData: youtubeDLInfo, + importDataOverride, + importType: 'url' + }) + + const thumbnailModel = await forgeThumbnail({ + inputPath: thumbnailFilePath, + downloadUrl: youtubeDLInfo.thumbnailUrl, + video, + type: ThumbnailType.MINIATURE + }) + + const previewModel = await forgeThumbnail({ + inputPath: previewFilePath, + downloadUrl: youtubeDLInfo.thumbnailUrl, + video, + type: ThumbnailType.PREVIEW + }) + + const videoImport = await insertFromImportIntoDB({ + video, + thumbnailModel, + previewModel, + videoChannel: channel, + tags: importDataOverride?.tags || youtubeDLInfo.tags, + user, + videoImportAttributes: { + targetUrl, + state: VideoImportState.PENDING, + userId: user.id, + videoChannelSyncId: channelSync?.id + }, + videoPasswords: importDataOverride.videoPasswords + }) + + // Get video subtitles + await processYoutubeSubtitles(youtubeDL, targetUrl, video.id) + + let fileExt = `.${youtubeDLInfo.ext}` + if (!isVideoFileExtnameValid(fileExt)) fileExt = '.mp4' + + const payload: VideoImportPayload = { + type: 'youtube-dl' as 'youtube-dl', + videoImportId: videoImport.id, + fileExt, + // If part of a sync process, there is a parent job that will aggregate children results + preventException: !!channelSync + } + + return { + videoImport, + job: { type: 'video-import' as 'video-import', payload } + } +} + +// --------------------------------------------------------------------------- + +export { + buildYoutubeDLImport, + YoutubeDlImportError, + insertFromImportIntoDB, + buildVideoFromImport +} + +// --------------------------------------------------------------------------- + +async function forgeThumbnail ({ inputPath, video, downloadUrl, type }: { + inputPath?: string + downloadUrl?: string + video: MVideoThumbnail + type: ThumbnailType_Type +}): Promise { + if (inputPath) { + return updateLocalVideoMiniatureFromExisting({ + inputPath, + video, + type, + automaticallyGenerated: false + }) + } + + if (downloadUrl) { + try { + return await updateLocalVideoMiniatureFromUrl({ downloadUrl, video, type }) + } catch (err) { + logger.warn('Cannot process thumbnail %s from youtube-dl.', downloadUrl, { err }) + } + } + + return null +} + +async function processYoutubeSubtitles (youtubeDL: YoutubeDLWrapper, targetUrl: string, videoId: number) { + try { + const subtitles = await youtubeDL.getSubtitles() + + logger.info('Found %s subtitles candidates from youtube-dl import %s.', subtitles.length, targetUrl) + + for (const subtitle of subtitles) { + if (!await isVTTFileValid(subtitle.path)) { + logger.info('%s is not a valid youtube-dl subtitle, skipping', subtitle.path) + await remove(subtitle.path) + continue + } + + const videoCaption = new VideoCaptionModel({ + videoId, + language: subtitle.language, + filename: VideoCaptionModel.generateCaptionName(subtitle.language) + }) as MVideoCaption + + // Move physical file + await moveAndProcessCaptionFile(subtitle, videoCaption) + + await sequelizeTypescript.transaction(async t => { + await VideoCaptionModel.insertOrReplaceLanguage(videoCaption, t) + }) + + logger.info('Added %s youtube-dl subtitle', subtitle.path) + } + } catch (err) { + logger.warn('Cannot get video subtitles.', { err }) + } +} + +async function hasUnicastURLsOnly (youtubeDLInfo: YoutubeDLInfo) { + const hosts = youtubeDLInfo.urls.map(u => new URL(u).hostname) + const uniqHosts = new Set(hosts) + + for (const h of uniqHosts) { + if (await isResolvingToUnicastOnly(h) !== true) { + return false + } + } + + return true +} diff --git a/server/server/lib/video-privacy.ts b/server/server/lib/video-privacy.ts new file mode 100644 index 000000000..df8ac974b --- /dev/null +++ b/server/server/lib/video-privacy.ts @@ -0,0 +1,133 @@ +import { move } from 'fs-extra/esm' +import { join } from 'path' +import { VideoPrivacy, VideoPrivacyType, VideoStorage } from '@peertube/peertube-models' +import { logger } from '@server/helpers/logger.js' +import { DIRECTORIES } from '@server/initializers/constants.js' +import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models/index.js' +import { updateHLSFilesACL, updateWebVideoFileACL } from './object-storage/index.js' + +const validPrivacySet = new Set([ + VideoPrivacy.PRIVATE, + VideoPrivacy.INTERNAL, + VideoPrivacy.PASSWORD_PROTECTED +]) + +function setVideoPrivacy (video: MVideo, newPrivacy: VideoPrivacyType) { + if (video.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) { + video.publishedAt = new Date() + } + + video.privacy = newPrivacy +} + +function isVideoInPrivateDirectory (privacy) { + return validPrivacySet.has(privacy) +} + +function isVideoInPublicDirectory (privacy: VideoPrivacyType) { + return !isVideoInPrivateDirectory(privacy) +} + +async function moveFilesIfPrivacyChanged (video: MVideoFullLight, oldPrivacy: VideoPrivacyType) { + // Now public, previously private + if (isVideoInPublicDirectory(video.privacy) && isVideoInPrivateDirectory(oldPrivacy)) { + await moveFiles({ type: 'private-to-public', video }) + + return true + } + + // Now private, previously public + if (isVideoInPrivateDirectory(video.privacy) && isVideoInPublicDirectory(oldPrivacy)) { + await moveFiles({ type: 'public-to-private', video }) + + return true + } + + return false +} + +export { + setVideoPrivacy, + + isVideoInPrivateDirectory, + isVideoInPublicDirectory, + + moveFilesIfPrivacyChanged +} + +// --------------------------------------------------------------------------- + +type MoveType = 'private-to-public' | 'public-to-private' + +async function moveFiles (options: { + type: MoveType + video: MVideoFullLight +}) { + const { type, video } = options + + for (const file of video.VideoFiles) { + if (file.storage === VideoStorage.FILE_SYSTEM) { + await moveWebVideoFileOnFS(type, video, file) + } else { + await updateWebVideoFileACL(video, file) + } + } + + const hls = video.getHLSPlaylist() + + if (hls) { + if (hls.storage === VideoStorage.FILE_SYSTEM) { + await moveHLSFilesOnFS(type, video) + } else { + await updateHLSFilesACL(hls) + } + } +} + +async function moveWebVideoFileOnFS (type: MoveType, video: MVideo, file: MVideoFile) { + const directories = getWebVideoDirectories(type) + + const source = join(directories.old, file.filename) + const destination = join(directories.new, file.filename) + + try { + logger.info('Moving web video files of %s after privacy change (%s -> %s).', video.uuid, source, destination) + + await move(source, destination) + } catch (err) { + logger.error('Cannot move web video file %s to %s after privacy change', source, destination, { err }) + } +} + +function getWebVideoDirectories (moveType: MoveType) { + if (moveType === 'private-to-public') { + return { old: DIRECTORIES.VIDEOS.PRIVATE, new: DIRECTORIES.VIDEOS.PUBLIC } + } + + return { old: DIRECTORIES.VIDEOS.PUBLIC, new: DIRECTORIES.VIDEOS.PRIVATE } +} + +// --------------------------------------------------------------------------- + +async function moveHLSFilesOnFS (type: MoveType, video: MVideo) { + const directories = getHLSDirectories(type) + + const source = join(directories.old, video.uuid) + const destination = join(directories.new, video.uuid) + + try { + logger.info('Moving HLS files of %s after privacy change (%s -> %s).', video.uuid, source, destination) + + await move(source, destination) + } catch (err) { + logger.error('Cannot move HLS file %s to %s after privacy change', source, destination, { err }) + } +} + +function getHLSDirectories (moveType: MoveType) { + if (moveType === 'private-to-public') { + return { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC } + } + + return { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE } +} diff --git a/server/server/lib/video-state.ts b/server/server/lib/video-state.ts new file mode 100644 index 000000000..3b17877af --- /dev/null +++ b/server/server/lib/video-state.ts @@ -0,0 +1,154 @@ +import { Transaction } from 'sequelize' +import { VideoState, VideoStateType } from '@peertube/peertube-models' +import { retryTransactionWrapper } from '@server/helpers/database-utils.js' +import { logger } from '@server/helpers/logger.js' +import { CONFIG } from '@server/initializers/config.js' +import { sequelizeTypescript } from '@server/initializers/database.js' +import { VideoJobInfoModel } from '@server/models/video/video-job-info.js' +import { VideoModel } from '@server/models/video/video.js' +import { MVideo, MVideoFullLight, MVideoUUID } from '@server/types/models/index.js' +import { federateVideoIfNeeded } from './activitypub/videos/index.js' +import { JobQueue } from './job-queue/index.js' +import { Notifier } from './notifier/index.js' +import { buildMoveToObjectStorageJob } from './video.js' + +function buildNextVideoState (currentState?: VideoStateType) { + if (currentState === VideoState.PUBLISHED) { + throw new Error('Video is already in its final state') + } + + if ( + currentState !== VideoState.TO_EDIT && + currentState !== VideoState.TO_TRANSCODE && + currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE && + CONFIG.TRANSCODING.ENABLED + ) { + return VideoState.TO_TRANSCODE + } + + if ( + currentState !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE && + CONFIG.OBJECT_STORAGE.ENABLED + ) { + return VideoState.TO_MOVE_TO_EXTERNAL_STORAGE + } + + return VideoState.PUBLISHED +} + +function moveToNextState (options: { + video: MVideoUUID + previousVideoState?: VideoStateType + isNewVideo?: boolean // Default true +}) { + const { video, previousVideoState, isNewVideo = true } = options + + return retryTransactionWrapper(() => { + return sequelizeTypescript.transaction(async t => { + // Maybe the video changed in database, refresh it + const videoDatabase = await VideoModel.loadFull(video.uuid, t) + // Video does not exist anymore + if (!videoDatabase) return undefined + + // Already in its final state + if (videoDatabase.state === VideoState.PUBLISHED) { + return federateVideoIfNeeded(videoDatabase, false, t) + } + + const newState = buildNextVideoState(videoDatabase.state) + + if (newState === VideoState.PUBLISHED) { + return moveToPublishedState({ video: videoDatabase, previousVideoState, isNewVideo, transaction: t }) + } + + if (newState === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { + return moveToExternalStorageState({ video: videoDatabase, isNewVideo, transaction: t }) + } + }) + }) +} + +async function moveToExternalStorageState (options: { + video: MVideoFullLight + isNewVideo: boolean + transaction: Transaction +}) { + const { video, isNewVideo, transaction } = options + + const videoJobInfo = await VideoJobInfoModel.load(video.id, transaction) + const pendingTranscode = videoJobInfo?.pendingTranscode || 0 + + // We want to wait all transcoding jobs before moving the video on an external storage + if (pendingTranscode !== 0) return false + + const previousVideoState = video.state + + if (video.state !== VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) { + await video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE, isNewVideo, transaction) + } + + logger.info('Creating external storage move job for video %s.', video.uuid, { tags: [ video.uuid ] }) + + try { + await JobQueue.Instance.createJob(await buildMoveToObjectStorageJob({ video, previousVideoState, isNewVideo })) + + return true + } catch (err) { + logger.error('Cannot add move to object storage job', { err }) + + return false + } +} + +function moveToFailedTranscodingState (video: MVideo) { + if (video.state === VideoState.TRANSCODING_FAILED) return + + return video.setNewState(VideoState.TRANSCODING_FAILED, false, undefined) +} + +function moveToFailedMoveToObjectStorageState (video: MVideo) { + if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED) return + + return video.setNewState(VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED, false, undefined) +} + +// --------------------------------------------------------------------------- + +export { + buildNextVideoState, + moveToExternalStorageState, + moveToFailedTranscodingState, + moveToFailedMoveToObjectStorageState, + moveToNextState +} + +// --------------------------------------------------------------------------- + +async function moveToPublishedState (options: { + video: MVideoFullLight + isNewVideo: boolean + transaction: Transaction + previousVideoState?: VideoStateType +}) { + const { video, isNewVideo, transaction, previousVideoState } = options + const previousState = previousVideoState ?? video.state + + logger.info('Publishing video %s.', video.uuid, { isNewVideo, previousState, tags: [ video.uuid ] }) + + await video.setNewState(VideoState.PUBLISHED, isNewVideo, transaction) + + await federateVideoIfNeeded(video, isNewVideo, transaction) + + if (previousState === VideoState.TO_EDIT) { + Notifier.Instance.notifyOfFinishedVideoStudioEdition(video) + return + } + + if (isNewVideo) { + Notifier.Instance.notifyOnNewVideoIfNeeded(video) + + if (previousState === VideoState.TO_TRANSCODE) { + Notifier.Instance.notifyOnVideoPublishedAfterTranscoding(video) + } + } +} diff --git a/server/server/lib/video-studio.ts b/server/server/lib/video-studio.ts new file mode 100644 index 000000000..79b07cf4a --- /dev/null +++ b/server/server/lib/video-studio.ts @@ -0,0 +1,130 @@ +import { move, remove } from 'fs-extra/esm' +import { join } from 'path' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { createTorrentAndSetInfoHashFromPath } from '@server/helpers/webtorrent.js' +import { CONFIG } from '@server/initializers/config.js' +import { UserModel } from '@server/models/user/user.js' +import { MUser, MVideo, MVideoFile, MVideoFullLight, MVideoWithAllFiles } from '@server/types/models/index.js' +import { getVideoStreamDuration } from '@peertube/peertube-ffmpeg' +import { VideoStudioEditionPayload, VideoStudioTask, VideoStudioTaskPayload } from '@peertube/peertube-models' +import { federateVideoIfNeeded } from './activitypub/videos/index.js' +import { JobQueue } from './job-queue/index.js' +import { VideoStudioTranscodingJobHandler } from './runners/index.js' +import { createOptimizeOrMergeAudioJobs } from './transcoding/create-transcoding-job.js' +import { getTranscodingJobPriority } from './transcoding/transcoding-priority.js' +import { buildNewFile, removeHLSPlaylist, removeWebVideoFile } from './video-file.js' +import { VideoPathManager } from './video-path-manager.js' + +const lTags = loggerTagsFactory('video-studio') + +export function buildTaskFileFieldname (indice: number, fieldName = 'file') { + return `tasks[${indice}][options][${fieldName}]` +} + +export function getTaskFileFromReq (files: Express.Multer.File[], indice: number, fieldName = 'file') { + return files.find(f => f.fieldname === buildTaskFileFieldname(indice, fieldName)) +} + +export function getStudioTaskFilePath (filename: string) { + return join(CONFIG.STORAGE.TMP_PERSISTENT_DIR, filename) +} + +export async function safeCleanupStudioTMPFiles (tasks: VideoStudioTaskPayload[]) { + logger.info('Removing studio task files', { tasks, ...lTags() }) + + for (const task of tasks) { + try { + if (task.name === 'add-intro' || task.name === 'add-outro') { + await remove(task.options.file) + } else if (task.name === 'add-watermark') { + await remove(task.options.file) + } + } catch (err) { + logger.error('Cannot remove studio file', { err }) + } + } +} + +// --------------------------------------------------------------------------- + +export async function approximateIntroOutroAdditionalSize ( + video: MVideoFullLight, + tasks: VideoStudioTask[], + fileFinder: (i: number) => string +) { + let additionalDuration = 0 + + for (let i = 0; i < tasks.length; i++) { + const task = tasks[i] + + if (task.name !== 'add-intro' && task.name !== 'add-outro') continue + + const filePath = fileFinder(i) + additionalDuration += await getVideoStreamDuration(filePath) + } + + return (video.getMaxQualityFile().size / video.duration) * additionalDuration +} + +// --------------------------------------------------------------------------- + +export async function createVideoStudioJob (options: { + video: MVideo + user: MUser + payload: VideoStudioEditionPayload +}) { + const { video, user, payload } = options + + const priority = await getTranscodingJobPriority({ user, type: 'studio', fallback: 0 }) + + if (CONFIG.VIDEO_STUDIO.REMOTE_RUNNERS.ENABLED) { + await new VideoStudioTranscodingJobHandler().create({ video, tasks: payload.tasks, priority }) + return + } + + await JobQueue.Instance.createJob({ type: 'video-studio-edition', payload, priority }) +} + +export async function onVideoStudioEnded (options: { + editionResultPath: string + tasks: VideoStudioTaskPayload[] + video: MVideoFullLight +}) { + const { video, tasks, editionResultPath } = options + + const newFile = await buildNewFile({ path: editionResultPath, mode: 'web-video' }) + newFile.videoId = video.id + + const outputPath = VideoPathManager.Instance.getFSVideoFileOutputPath(video, newFile) + await move(editionResultPath, outputPath) + + await safeCleanupStudioTMPFiles(tasks) + + await createTorrentAndSetInfoHashFromPath(video, newFile, outputPath) + await removeAllFiles(video, newFile) + + await newFile.save() + + video.duration = await getVideoStreamDuration(outputPath) + await video.save() + + await federateVideoIfNeeded(video, false, undefined) + + const user = await UserModel.loadByVideoId(video.id) + + await createOptimizeOrMergeAudioJobs({ video, videoFile: newFile, isNewVideo: false, user, videoFileAlreadyLocked: false }) +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +async function removeAllFiles (video: MVideoWithAllFiles, webVideoFileException: MVideoFile) { + await removeHLSPlaylist(video) + + for (const file of video.VideoFiles) { + if (file.id === webVideoFileException.id) continue + + await removeWebVideoFile(video, file.id) + } +} diff --git a/server/server/lib/video-tokens-manager.ts b/server/server/lib/video-tokens-manager.ts new file mode 100644 index 000000000..57128721a --- /dev/null +++ b/server/server/lib/video-tokens-manager.ts @@ -0,0 +1,78 @@ +import { LRUCache } from 'lru-cache' +import { LRU_CACHE } from '@server/initializers/constants.js' +import { MUserAccountUrl } from '@server/types/models/index.js' +import { pick } from '@peertube/peertube-core-utils' +import { buildUUID } from '@peertube/peertube-node-utils' + +// --------------------------------------------------------------------------- +// Create temporary tokens that can be used as URL query parameters to access video static files +// --------------------------------------------------------------------------- + +class VideoTokensManager { + + private static instance: VideoTokensManager + + private readonly lruCache = new LRUCache({ + max: LRU_CACHE.VIDEO_TOKENS.MAX_SIZE, + ttl: LRU_CACHE.VIDEO_TOKENS.TTL + }) + + private constructor () {} + + createForAuthUser (options: { + user: MUserAccountUrl + videoUUID: string + }) { + const { token, expires } = this.generateVideoToken() + + this.lruCache.set(token, pick(options, [ 'user', 'videoUUID' ])) + + return { token, expires } + } + + createForPasswordProtectedVideo (options: { + videoUUID: string + }) { + const { token, expires } = this.generateVideoToken() + + this.lruCache.set(token, pick(options, [ 'videoUUID' ])) + + return { token, expires } + } + + hasToken (options: { + token: string + videoUUID: string + }) { + const value = this.lruCache.get(options.token) + if (!value) return false + + return value.videoUUID === options.videoUUID + } + + getUserFromToken (options: { + token: string + }) { + const value = this.lruCache.get(options.token) + if (!value) return undefined + + return value.user + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } + + private generateVideoToken () { + const token = buildUUID() + const expires = new Date(new Date().getTime() + LRU_CACHE.VIDEO_TOKENS.TTL) + + return { token, expires } + } +} + +// --------------------------------------------------------------------------- + +export { + VideoTokensManager +} diff --git a/server/server/lib/video-urls.ts b/server/server/lib/video-urls.ts new file mode 100644 index 000000000..bf399847e --- /dev/null +++ b/server/server/lib/video-urls.ts @@ -0,0 +1,31 @@ + +import { STATIC_PATHS, WEBSERVER } from '@server/initializers/constants.js' +import { MStreamingPlaylist, MVideo, MVideoFile, MVideoUUID } from '@server/types/models/index.js' + +// ################## Redundancy ################## + +function generateHLSRedundancyUrl (video: MVideo, playlist: MStreamingPlaylist) { + // Base URL used by our HLS player + return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + playlist.getStringType() + '/' + video.uuid +} + +function generateWebVideoRedundancyUrl (file: MVideoFile) { + return WEBSERVER.URL + STATIC_PATHS.REDUNDANCY + file.filename +} + +// ################## Meta data ################## + +function getLocalVideoFileMetadataUrl (video: MVideoUUID, videoFile: MVideoFile) { + const path = '/api/v1/videos/' + + return WEBSERVER.URL + path + video.uuid + '/metadata/' + videoFile.id +} + +// --------------------------------------------------------------------------- + +export { + getLocalVideoFileMetadataUrl, + + generateWebVideoRedundancyUrl, + generateHLSRedundancyUrl +} diff --git a/server/server/lib/video.ts b/server/server/lib/video.ts new file mode 100644 index 000000000..46346c3ed --- /dev/null +++ b/server/server/lib/video.ts @@ -0,0 +1,201 @@ +import { UploadFiles } from 'express' +import memoizee from 'memoizee' +import { Transaction } from 'sequelize' +import { + ManageVideoTorrentPayload, + ThumbnailType, + ThumbnailType_Type, + VideoCreate, + VideoPrivacy, + VideoPrivacyType, + VideoStateType +} from '@peertube/peertube-models' +import { CONFIG } from '@server/initializers/config.js' +import { MEMOIZE_LENGTH, MEMOIZE_TTL } from '@server/initializers/constants.js' +import { TagModel } from '@server/models/video/tag.js' +import { VideoJobInfoModel } from '@server/models/video/video-job-info.js' +import { VideoModel } from '@server/models/video/video.js' +import { FilteredModelAttributes } from '@server/types/index.js' +import { MThumbnail, MVideoFullLight, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models/index.js' +import { CreateJobArgument, JobQueue } from './job-queue/job-queue.js' +import { updateLocalVideoMiniatureFromExisting } from './thumbnail.js' +import { moveFilesIfPrivacyChanged } from './video-privacy.js' + +function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes { + return { + name: videoInfo.name, + remote: false, + category: videoInfo.category, + licence: videoInfo.licence ?? CONFIG.DEFAULTS.PUBLISH.LICENCE, + language: videoInfo.language, + commentsEnabled: videoInfo.commentsEnabled ?? CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED, + downloadEnabled: videoInfo.downloadEnabled ?? CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED, + waitTranscoding: videoInfo.waitTranscoding || false, + nsfw: videoInfo.nsfw || false, + description: videoInfo.description, + support: videoInfo.support, + privacy: videoInfo.privacy || VideoPrivacy.PRIVATE, + channelId, + originallyPublishedAt: videoInfo.originallyPublishedAt + ? new Date(videoInfo.originallyPublishedAt) + : null + } +} + +async function buildVideoThumbnailsFromReq (options: { + video: MVideoThumbnail + files: UploadFiles + fallback: (type: ThumbnailType_Type) => Promise + automaticallyGenerated?: boolean +}) { + const { video, files, fallback, automaticallyGenerated } = options + + const promises = [ + { + type: ThumbnailType.MINIATURE, + fieldName: 'thumbnailfile' + }, + { + type: ThumbnailType.PREVIEW, + fieldName: 'previewfile' + } + ].map(p => { + const fields = files?.[p.fieldName] + + if (fields) { + return updateLocalVideoMiniatureFromExisting({ + inputPath: fields[0].path, + video, + type: p.type, + automaticallyGenerated: automaticallyGenerated || false + }) + } + + return fallback(p.type) + }) + + return Promise.all(promises) +} + +// --------------------------------------------------------------------------- + +async function setVideoTags (options: { + video: MVideoTag + tags: string[] + transaction?: Transaction +}) { + const { video, tags, transaction } = options + + const internalTags = tags || [] + const tagInstances = await TagModel.findOrCreateTags(internalTags, transaction) + + await video.$set('Tags', tagInstances, { transaction }) + video.Tags = tagInstances +} + +// --------------------------------------------------------------------------- + +async function buildMoveToObjectStorageJob (options: { + video: MVideoUUID + previousVideoState: VideoStateType + isNewVideo?: boolean // Default true +}) { + const { video, previousVideoState, isNewVideo = true } = options + + await VideoJobInfoModel.increaseOrCreate(video.uuid, 'pendingMove') + + return { + type: 'move-to-object-storage' as 'move-to-object-storage', + payload: { + videoUUID: video.uuid, + isNewVideo, + previousVideoState + } + } +} + +// --------------------------------------------------------------------------- + +async function getVideoDuration (videoId: number | string) { + const video = await VideoModel.load(videoId) + + const duration = video.isLive + ? undefined + : video.duration + + return { duration, isLive: video.isLive } +} + +const getCachedVideoDuration = memoizee(getVideoDuration, { + promise: true, + max: MEMOIZE_LENGTH.VIDEO_DURATION, + maxAge: MEMOIZE_TTL.VIDEO_DURATION +}) + +// --------------------------------------------------------------------------- + +async function addVideoJobsAfterUpdate (options: { + video: MVideoFullLight + isNewVideo: boolean + + nameChanged: boolean + oldPrivacy: VideoPrivacyType +}) { + const { video, nameChanged, oldPrivacy, isNewVideo } = options + const jobs: CreateJobArgument[] = [] + + const filePathChanged = await moveFilesIfPrivacyChanged(video, oldPrivacy) + + if (!video.isLive && (nameChanged || filePathChanged)) { + for (const file of (video.VideoFiles || [])) { + const payload: ManageVideoTorrentPayload = { action: 'update-metadata', videoId: video.id, videoFileId: file.id } + + jobs.push({ type: 'manage-video-torrent', payload }) + } + + const hls = video.getHLSPlaylist() + + for (const file of (hls?.VideoFiles || [])) { + const payload: ManageVideoTorrentPayload = { action: 'update-metadata', streamingPlaylistId: hls.id, videoFileId: file.id } + + jobs.push({ type: 'manage-video-torrent', payload }) + } + } + + jobs.push({ + type: 'federate-video', + payload: { + videoUUID: video.uuid, + isNewVideo + } + }) + + const wasConfidentialVideo = new Set([ + VideoPrivacy.PRIVATE, + VideoPrivacy.UNLISTED, + VideoPrivacy.INTERNAL + ]).has(oldPrivacy) + + if (wasConfidentialVideo) { + jobs.push({ + type: 'notify', + payload: { + action: 'new-video', + videoUUID: video.uuid + } + }) + } + + return JobQueue.Instance.createSequentialJobFlow(...jobs) +} + +// --------------------------------------------------------------------------- + +export { + buildLocalVideoFromReq, + buildVideoThumbnailsFromReq, + setVideoTags, + buildMoveToObjectStorageJob, + addVideoJobsAfterUpdate, + getCachedVideoDuration +} diff --git a/server/server/lib/views/shared/index.ts b/server/server/lib/views/shared/index.ts new file mode 100644 index 000000000..e311fb22f --- /dev/null +++ b/server/server/lib/views/shared/index.ts @@ -0,0 +1,3 @@ +export * from './video-viewer-counters.js' +export * from './video-viewer-stats.js' +export * from './video-views.js' diff --git a/server/server/lib/views/shared/video-viewer-counters.ts b/server/server/lib/views/shared/video-viewer-counters.ts new file mode 100644 index 000000000..605e45645 --- /dev/null +++ b/server/server/lib/views/shared/video-viewer-counters.ts @@ -0,0 +1,197 @@ +import { buildUUID, isTestOrDevInstance, sha256 } from '@peertube/peertube-node-utils' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { VIEW_LIFETIME } from '@server/initializers/constants.js' +import { sendView } from '@server/lib/activitypub/send/send-view.js' +import { PeerTubeSocket } from '@server/lib/peertube-socket.js' +import { getServerActor } from '@server/models/application/application.js' +import { VideoModel } from '@server/models/video/video.js' +import { MVideo, MVideoImmutable } from '@server/types/models/index.js' + +const lTags = loggerTagsFactory('views') + +export type ViewerScope = 'local' | 'remote' +export type VideoScope = 'local' | 'remote' + +type Viewer = { + expires: number + id: string + viewerScope: ViewerScope + videoScope: VideoScope + lastFederation?: number +} + +export class VideoViewerCounters { + + // expires is new Date().getTime() + private readonly viewersPerVideo = new Map() + private readonly idToViewer = new Map() + + private readonly salt = buildUUID() + + private processingViewerCounters = false + + constructor () { + setInterval(() => this.cleanViewerCounters(), VIEW_LIFETIME.VIEWER_COUNTER) + } + + // --------------------------------------------------------------------------- + + async addLocalViewer (options: { + video: MVideoImmutable + ip: string + }) { + const { video, ip } = options + + logger.debug('Adding local viewer to video viewers counter %s.', video.uuid, { ...lTags(video.uuid) }) + + const viewerId = this.generateViewerId(ip, video.uuid) + const viewer = this.idToViewer.get(viewerId) + + if (viewer) { + viewer.expires = this.buildViewerExpireTime() + await this.federateViewerIfNeeded(video, viewer) + + return false + } + + const newViewer = await this.addViewerToVideo({ viewerId, video, viewerScope: 'local' }) + await this.federateViewerIfNeeded(video, newViewer) + + return true + } + + async addRemoteViewer (options: { + video: MVideo + viewerId: string + viewerExpires: Date + }) { + const { video, viewerExpires, viewerId } = options + + logger.debug('Adding remote viewer to video %s.', video.uuid, { ...lTags(video.uuid) }) + + await this.addViewerToVideo({ video, viewerExpires, viewerId, viewerScope: 'remote' }) + + return true + } + + // --------------------------------------------------------------------------- + + getTotalViewers (options: { + viewerScope: ViewerScope + videoScope: VideoScope + }) { + let total = 0 + + for (const viewers of this.viewersPerVideo.values()) { + total += viewers.filter(v => v.viewerScope === options.viewerScope && v.videoScope === options.videoScope).length + } + + return total + } + + getViewers (video: MVideo) { + const viewers = this.viewersPerVideo.get(video.id) + if (!viewers) return 0 + + return viewers.length + } + + buildViewerExpireTime () { + return new Date().getTime() + VIEW_LIFETIME.VIEWER_COUNTER + } + + // --------------------------------------------------------------------------- + + private async addViewerToVideo (options: { + video: MVideoImmutable + viewerId: string + viewerScope: ViewerScope + viewerExpires?: Date + }) { + const { video, viewerExpires, viewerId, viewerScope } = options + + let watchers = this.viewersPerVideo.get(video.id) + + if (!watchers) { + watchers = [] + this.viewersPerVideo.set(video.id, watchers) + } + + const expires = viewerExpires + ? viewerExpires.getTime() + : this.buildViewerExpireTime() + + const videoScope: VideoScope = video.remote + ? 'remote' + : 'local' + + const viewer = { id: viewerId, expires, videoScope, viewerScope } + watchers.push(viewer) + + this.idToViewer.set(viewerId, viewer) + + await this.notifyClients(video.id, watchers.length) + + return viewer + } + + private async cleanViewerCounters () { + if (this.processingViewerCounters) return + this.processingViewerCounters = true + + if (!isTestOrDevInstance()) logger.info('Cleaning video viewers.', lTags()) + + try { + for (const videoId of this.viewersPerVideo.keys()) { + const notBefore = new Date().getTime() + + const viewers = this.viewersPerVideo.get(videoId) + + // Only keep not expired viewers + const newViewers: Viewer[] = [] + + // Filter new viewers + for (const viewer of viewers) { + if (viewer.expires > notBefore) { + newViewers.push(viewer) + } else { + this.idToViewer.delete(viewer.id) + } + } + + if (newViewers.length === 0) this.viewersPerVideo.delete(videoId) + else this.viewersPerVideo.set(videoId, newViewers) + + await this.notifyClients(videoId, newViewers.length) + } + } catch (err) { + logger.error('Error in video clean viewers scheduler.', { err, ...lTags() }) + } + + this.processingViewerCounters = false + } + + private async notifyClients (videoId: string | number, viewersLength: number) { + const video = await VideoModel.loadImmutableAttributes(videoId) + if (!video) return + + PeerTubeSocket.Instance.sendVideoViewsUpdate(video, viewersLength) + + logger.debug('Video viewers update for %s is %d.', video.url, viewersLength, lTags()) + } + + private generateViewerId (ip: string, videoUUID: string) { + return sha256(this.salt + '-' + ip + '-' + videoUUID) + } + + private async federateViewerIfNeeded (video: MVideoImmutable, viewer: Viewer) { + // Federate the viewer if it's been a "long" time we did not + const now = new Date().getTime() + const federationLimit = now - (VIEW_LIFETIME.VIEWER_COUNTER * 0.75) + + if (viewer.lastFederation && viewer.lastFederation > federationLimit) return + + await sendView({ byActor: await getServerActor(), video, type: 'viewer', viewerIdentifier: viewer.id }) + viewer.lastFederation = now + } +} diff --git a/server/server/lib/views/shared/video-viewer-stats.ts b/server/server/lib/views/shared/video-viewer-stats.ts new file mode 100644 index 000000000..35ef5a7ea --- /dev/null +++ b/server/server/lib/views/shared/video-viewer-stats.ts @@ -0,0 +1,196 @@ +import { Transaction } from 'sequelize' +import { VideoViewEvent } from '@peertube/peertube-models' +import { isTestOrDevInstance } from '@peertube/peertube-node-utils' +import { GeoIP } from '@server/helpers/geo-ip.js' +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { MAX_LOCAL_VIEWER_WATCH_SECTIONS, VIEW_LIFETIME } from '@server/initializers/constants.js' +import { sequelizeTypescript } from '@server/initializers/database.js' +import { sendCreateWatchAction } from '@server/lib/activitypub/send/index.js' +import { getLocalVideoViewerActivityPubUrl } from '@server/lib/activitypub/url.js' +import { Redis } from '@server/lib/redis.js' +import { VideoModel } from '@server/models/video/video.js' +import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section.js' +import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer.js' +import { MVideo, MVideoImmutable } from '@server/types/models/index.js' + +const lTags = loggerTagsFactory('views') + +type LocalViewerStats = { + firstUpdated: number // Date.getTime() + lastUpdated: number // Date.getTime() + + watchSections: { + start: number + end: number + }[] + + watchTime: number + + country: string + + videoId: number +} + +export class VideoViewerStats { + private processingViewersStats = false + + constructor () { + setInterval(() => this.processViewerStats(), VIEW_LIFETIME.VIEWER_STATS) + } + + // --------------------------------------------------------------------------- + + async addLocalViewer (options: { + video: MVideoImmutable + currentTime: number + ip: string + viewEvent?: VideoViewEvent + }) { + const { video, ip, viewEvent, currentTime } = options + + logger.debug('Adding local viewer to video stats %s.', video.uuid, { currentTime, viewEvent, ...lTags(video.uuid) }) + + return this.updateLocalViewerStats({ video, viewEvent, currentTime, ip }) + } + + // --------------------------------------------------------------------------- + + async getWatchTime (videoId: number, ip: string) { + const stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ ip, videoId }) + + return stats?.watchTime || 0 + } + + // --------------------------------------------------------------------------- + + private async updateLocalViewerStats (options: { + video: MVideoImmutable + ip: string + currentTime: number + viewEvent?: VideoViewEvent + }) { + const { video, ip, viewEvent, currentTime } = options + const nowMs = new Date().getTime() + + let stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ ip, videoId: video.id }) + + if (stats && stats.watchSections.length >= MAX_LOCAL_VIEWER_WATCH_SECTIONS) { + logger.warn('Too much watch section to store for a viewer, skipping this one', { currentTime, viewEvent, ...lTags(video.uuid) }) + return + } + + if (!stats) { + const country = await GeoIP.Instance.safeCountryISOLookup(ip) + + stats = { + firstUpdated: nowMs, + lastUpdated: nowMs, + + watchSections: [], + + watchTime: 0, + + country, + videoId: video.id + } + } + + stats.lastUpdated = nowMs + + if (viewEvent === 'seek' || stats.watchSections.length === 0) { + stats.watchSections.push({ + start: currentTime, + end: currentTime + }) + } else { + const lastSection = stats.watchSections[stats.watchSections.length - 1] + + if (lastSection.start > currentTime) { + logger.debug('Invalid end watch section %d. Last start record was at %d. Starting a new section.', currentTime, lastSection.start) + + stats.watchSections.push({ + start: currentTime, + end: currentTime + }) + } else { + lastSection.end = currentTime + } + } + + stats.watchTime = this.buildWatchTimeFromSections(stats.watchSections) + + logger.debug('Set local video viewer stats for video %s.', video.uuid, { stats, ...lTags(video.uuid) }) + + await Redis.Instance.setLocalVideoViewer(ip, video.id, stats) + } + + async processViewerStats () { + if (this.processingViewersStats) return + this.processingViewersStats = true + + if (!isTestOrDevInstance()) logger.info('Processing viewer statistics.', lTags()) + + const now = new Date().getTime() + + try { + const allKeys = await Redis.Instance.listLocalVideoViewerKeys() + + for (const key of allKeys) { + const stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ key }) + + // Process expired stats + if (stats.lastUpdated > now - VIEW_LIFETIME.VIEWER_STATS) { + continue + } + + try { + await sequelizeTypescript.transaction(async t => { + const video = await VideoModel.load(stats.videoId, t) + if (!video) return + + const statsModel = await this.saveViewerStats(video, stats, t) + + if (video.remote) { + await sendCreateWatchAction(statsModel, t) + } + }) + + await Redis.Instance.deleteLocalVideoViewersKeys(key) + } catch (err) { + logger.error('Cannot process viewer stats for Redis key %s.', key, { err, ...lTags() }) + } + } + } catch (err) { + logger.error('Error in video save viewers stats scheduler.', { err, ...lTags() }) + } + + this.processingViewersStats = false + } + + private async saveViewerStats (video: MVideo, stats: LocalViewerStats, transaction: Transaction) { + const statsModel = new LocalVideoViewerModel({ + startDate: new Date(stats.firstUpdated), + endDate: new Date(stats.lastUpdated), + watchTime: stats.watchTime, + country: stats.country, + videoId: video.id + }) + + statsModel.url = getLocalVideoViewerActivityPubUrl(statsModel) + statsModel.Video = video as VideoModel + + await statsModel.save({ transaction }) + + statsModel.WatchSections = await LocalVideoViewerWatchSectionModel.bulkCreateSections({ + localVideoViewerId: statsModel.id, + watchSections: stats.watchSections, + transaction + }) + + return statsModel + } + + private buildWatchTimeFromSections (sections: { start: number, end: number }[]) { + return sections.reduce((p, current) => p + (current.end - current.start), 0) + } +} diff --git a/server/server/lib/views/shared/video-views.ts b/server/server/lib/views/shared/video-views.ts new file mode 100644 index 000000000..ee56e30bc --- /dev/null +++ b/server/server/lib/views/shared/video-views.ts @@ -0,0 +1,70 @@ +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { sendView } from '@server/lib/activitypub/send/send-view.js' +import { getCachedVideoDuration } from '@server/lib/video.js' +import { getServerActor } from '@server/models/application/application.js' +import { MVideo, MVideoImmutable } from '@server/types/models/index.js' +import { buildUUID } from '@peertube/peertube-node-utils' +import { Redis } from '../../redis.js' + +const lTags = loggerTagsFactory('views') + +export class VideoViews { + + async addLocalView (options: { + video: MVideoImmutable + ip: string + watchTime: number + }) { + const { video, ip, watchTime } = options + + logger.debug('Adding local view to video %s.', video.uuid, { watchTime, ...lTags(video.uuid) }) + + if (!await this.hasEnoughWatchTime(video, watchTime)) return false + + const viewExists = await Redis.Instance.doesVideoIPViewExist(ip, video.uuid) + if (viewExists) return false + + await Redis.Instance.setIPVideoView(ip, video.uuid) + + await this.addView(video) + + await sendView({ byActor: await getServerActor(), video, type: 'view', viewerIdentifier: buildUUID() }) + + return true + } + + async addRemoteView (options: { + video: MVideo + }) { + const { video } = options + + logger.debug('Adding remote view to video %s.', video.uuid, { ...lTags(video.uuid) }) + + await this.addView(video) + + return true + } + + // --------------------------------------------------------------------------- + + private async addView (video: MVideoImmutable) { + const promises: Promise[] = [] + + if (video.isOwned()) { + promises.push(Redis.Instance.addLocalVideoView(video.id)) + } + + promises.push(Redis.Instance.addVideoViewStats(video.id)) + + await Promise.all(promises) + } + + private async hasEnoughWatchTime (video: MVideoImmutable, watchTime: number) { + const { duration, isLive } = await getCachedVideoDuration(video.id) + + if (isLive || duration >= 30) return watchTime >= 30 + + // Check more than 50% of the video is watched + return duration / watchTime < 2 + } +} diff --git a/server/server/lib/views/video-views-manager.ts b/server/server/lib/views/video-views-manager.ts new file mode 100644 index 000000000..9fc76d55e --- /dev/null +++ b/server/server/lib/views/video-views-manager.ts @@ -0,0 +1,100 @@ +import { logger, loggerTagsFactory } from '@server/helpers/logger.js' +import { MVideo, MVideoImmutable } from '@server/types/models/index.js' +import { VideoViewEvent } from '@peertube/peertube-models' +import { VideoScope, VideoViewerCounters, VideoViewerStats, VideoViews, ViewerScope } from './shared/index.js' + +/** + * If processing a local view: + * - We update viewer information (segments watched, watch time etc) + * - We add +1 to video viewers counter if this is a new viewer + * - We add +1 to video views counter if this is a new view and if the user watched enough seconds + * - We send AP message to notify about this viewer and this view + * - We update last video time for the user if authenticated + * + * If processing a remote view: + * - We add +1 to video viewers counter + * - We add +1 to video views counter + * + * A viewer is a someone that watched one or multiple sections of a video + * A viewer that watched only a few seconds of a video may not increment the video views counter + * Viewers statistics are sent to origin instance using the `WatchAction` ActivityPub object + * + */ + +const lTags = loggerTagsFactory('views') + +export class VideoViewsManager { + + private static instance: VideoViewsManager + + private videoViewerStats: VideoViewerStats + private videoViewerCounters: VideoViewerCounters + private videoViews: VideoViews + + private constructor () { + } + + init () { + this.videoViewerStats = new VideoViewerStats() + this.videoViewerCounters = new VideoViewerCounters() + this.videoViews = new VideoViews() + } + + async processLocalView (options: { + video: MVideoImmutable + currentTime: number + ip: string | null + viewEvent?: VideoViewEvent + }) { + const { video, ip, viewEvent, currentTime } = options + + logger.debug('Processing local view for %s and ip %s.', video.url, ip, lTags()) + + await this.videoViewerStats.addLocalViewer({ video, ip, viewEvent, currentTime }) + + const successViewer = await this.videoViewerCounters.addLocalViewer({ video, ip }) + + // Do it after added local viewer to fetch updated information + const watchTime = await this.videoViewerStats.getWatchTime(video.id, ip) + + const successView = await this.videoViews.addLocalView({ video, watchTime, ip }) + + return { successView, successViewer } + } + + async processRemoteView (options: { + video: MVideo + viewerId: string | null + viewerExpires?: Date + }) { + const { video, viewerId, viewerExpires } = options + + logger.debug('Processing remote view for %s.', video.url, { viewerExpires, viewerId, ...lTags() }) + + if (viewerExpires) await this.videoViewerCounters.addRemoteViewer({ video, viewerId, viewerExpires }) + else await this.videoViews.addRemoteView({ video }) + } + + getViewers (video: MVideo) { + return this.videoViewerCounters.getViewers(video) + } + + getTotalViewers (options: { + viewerScope: ViewerScope + videoScope: VideoScope + }) { + return this.videoViewerCounters.getTotalViewers(options) + } + + buildViewerExpireTime () { + return this.videoViewerCounters.buildViewerExpireTime() + } + + processViewerStats () { + return this.videoViewerStats.processViewerStats() + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/server/lib/worker/parent-process.ts b/server/server/lib/worker/parent-process.ts new file mode 100644 index 000000000..2b56b4526 --- /dev/null +++ b/server/server/lib/worker/parent-process.ts @@ -0,0 +1,77 @@ +import { join } from 'path' +import Piscina from 'piscina' +import { JOB_CONCURRENCY, WORKER_THREADS } from '@server/initializers/constants.js' +import httpBroadcast from './workers/http-broadcast.js' +import downloadImage from './workers/image-downloader.js' +import processImage from './workers/image-processor.js' + +let downloadImageWorker: Piscina + +function downloadImageFromWorker (options: Parameters[0]): Promise> { + if (!downloadImageWorker) { + downloadImageWorker = new Piscina({ + filename: new URL(join('workers', 'image-downloader.js'), import.meta.url).href, + concurrentTasksPerWorker: WORKER_THREADS.DOWNLOAD_IMAGE.CONCURRENCY, + maxThreads: WORKER_THREADS.DOWNLOAD_IMAGE.MAX_THREADS + }) + } + + return downloadImageWorker.run(options) +} + +// --------------------------------------------------------------------------- + +let processImageWorker: Piscina + +function processImageFromWorker (options: Parameters[0]): Promise> { + if (!processImageWorker) { + processImageWorker = new Piscina({ + filename: new URL(join('workers', 'image-processor.js'), import.meta.url).href, + concurrentTasksPerWorker: WORKER_THREADS.PROCESS_IMAGE.CONCURRENCY, + maxThreads: WORKER_THREADS.PROCESS_IMAGE.MAX_THREADS + }) + } + + return processImageWorker.run(options) +} + +// --------------------------------------------------------------------------- + +let parallelHTTPBroadcastWorker: Piscina + +function parallelHTTPBroadcastFromWorker (options: Parameters[0]): Promise> { + if (!parallelHTTPBroadcastWorker) { + parallelHTTPBroadcastWorker = new Piscina({ + filename: new URL(join('workers', 'http-broadcast.js'), import.meta.url).href, + // Keep it sync with job concurrency so the worker will accept all the requests sent by the parallelized jobs + concurrentTasksPerWorker: JOB_CONCURRENCY['activitypub-http-broadcast-parallel'], + maxThreads: 1 + }) + } + + return parallelHTTPBroadcastWorker.run(options) +} + +// --------------------------------------------------------------------------- + +let sequentialHTTPBroadcastWorker: Piscina + +function sequentialHTTPBroadcastFromWorker (options: Parameters[0]): Promise> { + if (!sequentialHTTPBroadcastWorker) { + sequentialHTTPBroadcastWorker = new Piscina({ + filename: new URL(join('workers', 'http-broadcast.js'), import.meta.url).href, + // Keep it sync with job concurrency so the worker will accept all the requests sent by the parallelized jobs + concurrentTasksPerWorker: JOB_CONCURRENCY['activitypub-http-broadcast'], + maxThreads: 1 + }) + } + + return sequentialHTTPBroadcastWorker.run(options) +} + +export { + downloadImageFromWorker, + processImageFromWorker, + parallelHTTPBroadcastFromWorker, + sequentialHTTPBroadcastFromWorker +} diff --git a/server/server/lib/worker/workers/http-broadcast.ts b/server/server/lib/worker/workers/http-broadcast.ts new file mode 100644 index 000000000..29895a00a --- /dev/null +++ b/server/server/lib/worker/workers/http-broadcast.ts @@ -0,0 +1,28 @@ +import Bluebird from 'bluebird' +import { logger } from '@server/helpers/logger.js' +import { doRequest, PeerTubeRequestOptions } from '@server/helpers/requests.js' +import { BROADCAST_CONCURRENCY } from '@server/initializers/constants.js' + +async function httpBroadcast (payload: { + uris: string[] + requestOptions: PeerTubeRequestOptions +}) { + const { uris, requestOptions } = payload + + const badUrls: string[] = [] + const goodUrls: string[] = [] + + await Bluebird.map(uris, async uri => { + try { + await doRequest(uri, requestOptions) + goodUrls.push(uri) + } catch (err) { + logger.debug('HTTP broadcast to %s failed.', uri, { err }) + badUrls.push(uri) + } + }, { concurrency: BROADCAST_CONCURRENCY }) + + return { goodUrls, badUrls } +} + +export default httpBroadcast diff --git a/server/server/lib/worker/workers/image-downloader.ts b/server/server/lib/worker/workers/image-downloader.ts new file mode 100644 index 000000000..c75acf0cd --- /dev/null +++ b/server/server/lib/worker/workers/image-downloader.ts @@ -0,0 +1,31 @@ +import { remove } from 'fs-extra/esm' +import { join } from 'path' +import { processImage } from '@server/helpers/image-utils.js' +import { doRequestAndSaveToFile } from '@server/helpers/requests.js' +import { CONFIG } from '@server/initializers/config.js' + +async function downloadImage (options: { + url: string + destDir: string + destName: string + size: { width: number, height: number } +}) { + const { url, destDir, destName, size } = options + + const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName) + await doRequestAndSaveToFile(url, tmpPath) + + const destPath = join(destDir, destName) + + try { + await processImage({ path: tmpPath, destination: destPath, newSize: size }) + } catch (err) { + await remove(tmpPath) + + throw err + } + + return destPath +} + +export default downloadImage diff --git a/server/server/lib/worker/workers/image-processor.ts b/server/server/lib/worker/workers/image-processor.ts new file mode 100644 index 000000000..6aa8dcc29 --- /dev/null +++ b/server/server/lib/worker/workers/image-processor.ts @@ -0,0 +1,3 @@ +import { processImage } from '@server/helpers/image-utils.js' + +export default processImage diff --git a/server/server/middlewares/activitypub.ts b/server/server/middlewares/activitypub.ts new file mode 100644 index 000000000..0d333061e --- /dev/null +++ b/server/server/middlewares/activitypub.ts @@ -0,0 +1,156 @@ +import { NextFunction, Request, Response } from 'express' +import { isActorDeleteActivityValid } from '@server/helpers/custom-validators/activitypub/actor.js' +import { getAPId } from '@server/lib/activitypub/activity.js' +import { wrapWithSpanAndContext } from '@server/lib/opentelemetry/tracing.js' +import { ActivityDelete, ActivityPubSignature, HttpStatusCode } from '@peertube/peertube-models' +import { logger } from '../helpers/logger.js' +import { isHTTPSignatureVerified, isJsonLDSignatureVerified, parseHTTPSignature } from '../helpers/peertube-crypto.js' +import { ACCEPT_HEADERS, ACTIVITY_PUB, HTTP_SIGNATURE } from '../initializers/constants.js' +import { getOrCreateAPActor, loadActorUrlOrGetFromWebfinger } from '../lib/activitypub/actors/index.js' + +async function checkSignature (req: Request, res: Response, next: NextFunction) { + try { + const httpSignatureChecked = await checkHttpSignature(req, res) + if (httpSignatureChecked !== true) return + + const actor = res.locals.signature.actor + + // Forwarded activity + const bodyActor = req.body.actor + const bodyActorId = getAPId(bodyActor) + if (bodyActorId && bodyActorId !== actor.url) { + const jsonLDSignatureChecked = await checkJsonLDSignature(req, res) + if (jsonLDSignatureChecked !== true) return + } + + return next() + } catch (err) { + const activity: ActivityDelete = req.body + if (isActorDeleteActivityValid(activity) && activity.object === activity.actor) { + logger.debug('Handling signature error on actor delete activity', { err }) + return res.status(HttpStatusCode.NO_CONTENT_204).end() + } + + logger.warn('Error in ActivityPub signature checker.', { err }) + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'ActivityPub signature could not be checked' + }) + } +} + +function executeIfActivityPub (req: Request, res: Response, next: NextFunction) { + const accepted = req.accepts(ACCEPT_HEADERS) + if (accepted === false || ACTIVITY_PUB.POTENTIAL_ACCEPT_HEADERS.includes(accepted) === false) { + // Bypass this route + return next('route') + } + + logger.debug('ActivityPub request for %s.', req.url) + + return next() +} + +// --------------------------------------------------------------------------- + +export { + checkSignature, + executeIfActivityPub, + checkHttpSignature +} + +// --------------------------------------------------------------------------- + +async function checkHttpSignature (req: Request, res: Response) { + return wrapWithSpanAndContext('peertube.activitypub.checkHTTPSignature', async () => { + // FIXME: compatibility with http-signature < v1.3 + const sig = req.headers[HTTP_SIGNATURE.HEADER_NAME] as string + if (sig && sig.startsWith('Signature ') === true) req.headers[HTTP_SIGNATURE.HEADER_NAME] = sig.replace(/^Signature /, '') + + let parsed: any + + try { + parsed = parseHTTPSignature(req, HTTP_SIGNATURE.CLOCK_SKEW_SECONDS) + } catch (err) { + logger.warn('Invalid signature because of exception in signature parser', { reqBody: req.body, err }) + + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: err.message + }) + return false + } + + const keyId = parsed.keyId + if (!keyId) { + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Invalid key ID', + data: { + keyId + } + }) + return false + } + + logger.debug('Checking HTTP signature of actor %s...', keyId) + + let [ actorUrl ] = keyId.split('#') + if (actorUrl.startsWith('acct:')) { + actorUrl = await loadActorUrlOrGetFromWebfinger(actorUrl.replace(/^acct:/, '')) + } + + const actor = await getOrCreateAPActor(actorUrl) + + const verified = isHTTPSignatureVerified(parsed, actor) + if (verified !== true) { + logger.warn('Signature from %s is invalid', actorUrl, { parsed }) + + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Invalid signature', + data: { + actorUrl + } + }) + return false + } + + res.locals.signature = { actor } + return true + }) +} + +async function checkJsonLDSignature (req: Request, res: Response) { + return wrapWithSpanAndContext('peertube.activitypub.JSONLDSignature', async () => { + const signatureObject: ActivityPubSignature = req.body.signature + + if (!signatureObject?.creator) { + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Object and creator signature do not match' + }) + return false + } + + const [ creator ] = signatureObject.creator.split('#') + + logger.debug('Checking JsonLD signature of actor %s...', creator) + + const actor = await getOrCreateAPActor(creator) + const verified = await isJsonLDSignatureVerified(actor, req.body) + + if (verified !== true) { + logger.warn('Signature not verified.', req.body) + + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Signature could not be verified' + }) + return false + } + + res.locals.signature = { actor } + return true + }) +} diff --git a/server/server/middlewares/async.ts b/server/server/middlewares/async.ts new file mode 100644 index 000000000..b6997e31b --- /dev/null +++ b/server/server/middlewares/async.ts @@ -0,0 +1,44 @@ +import Bluebird from 'bluebird' +import { NextFunction, Request, RequestHandler, Response } from 'express' +import { ValidationChain } from 'express-validator' +import { ExpressPromiseHandler } from '@server/types/express-handler.js' +import { retryTransactionWrapper } from '../helpers/database-utils.js' + +// Syntactic sugar to avoid try/catch in express controllers/middlewares + +export type RequestPromiseHandler = ValidationChain | ExpressPromiseHandler + +function asyncMiddleware (fun: RequestPromiseHandler | RequestPromiseHandler[]) { + return (req: Request, res: Response, next: NextFunction) => { + if (Array.isArray(fun) === true) { + return Bluebird.each(fun as RequestPromiseHandler[], f => { + return new Promise((resolve, reject) => { + return asyncMiddleware(f)(req, res, err => { + if (err) return reject(err) + + return resolve() + }) + }) + }).then(() => next()) + .catch(err => next(err)) + } + + return Promise.resolve((fun as RequestHandler)(req, res, next)) + .catch(err => next(err)) + } +} + +function asyncRetryTransactionMiddleware (fun: (req: Request, res: Response, next: NextFunction) => Promise) { + return (req: Request, res: Response, next: NextFunction) => { + return Promise.resolve( + retryTransactionWrapper(fun, req, res, next) + ).catch(err => next(err)) + } +} + +// --------------------------------------------------------------------------- + +export { + asyncMiddleware, + asyncRetryTransactionMiddleware +} diff --git a/server/server/middlewares/auth.ts b/server/server/middlewares/auth.ts new file mode 100644 index 000000000..6eddd80bd --- /dev/null +++ b/server/server/middlewares/auth.ts @@ -0,0 +1,112 @@ +import express from 'express' +import { Socket } from 'socket.io' +import { HttpStatusCode, HttpStatusCodeType, ServerErrorCodeType } from '@peertube/peertube-models' +import { getAccessToken } from '@server/lib/auth/oauth-model.js' +import { RunnerModel } from '@server/models/runner/runner.js' +import { logger } from '../helpers/logger.js' +import { handleOAuthAuthenticate } from '../lib/auth/oauth.js' + +function authenticate (req: express.Request, res: express.Response, next: express.NextFunction) { + handleOAuthAuthenticate(req, res) + .then((token: any) => { + res.locals.oauth = { token } + res.locals.authenticated = true + + return next() + }) + .catch(err => { + logger.info('Cannot authenticate.', { err }) + + return res.fail({ + status: err.status, + message: 'Token is invalid', + type: err.name + }) + }) +} + +function authenticateSocket (socket: Socket, next: (err?: any) => void) { + const accessToken = socket.handshake.query['accessToken'] + + logger.debug('Checking access token in runner.') + + if (!accessToken) return next(new Error('No access token provided')) + if (typeof accessToken !== 'string') return next(new Error('Access token is invalid')) + + getAccessToken(accessToken) + .then(tokenDB => { + const now = new Date() + + if (!tokenDB || tokenDB.accessTokenExpiresAt < now || tokenDB.refreshTokenExpiresAt < now) { + return next(new Error('Invalid access token.')) + } + + socket.handshake.auth.user = tokenDB.User + + return next() + }) + .catch(err => logger.error('Cannot get access token.', { err })) +} + +function authenticatePromise (options: { + req: express.Request + res: express.Response + errorMessage?: string + errorStatus?: HttpStatusCodeType + errorType?: ServerErrorCodeType +}) { + const { req, res, errorMessage = 'Not authenticated', errorStatus = HttpStatusCode.UNAUTHORIZED_401, errorType } = options + return new Promise(resolve => { + // Already authenticated? (or tried to) + if (res.locals.oauth?.token.User) return resolve() + + if (res.locals.authenticated === false) { + return res.fail({ + status: errorStatus, + type: errorType, + message: errorMessage + }) + } + + authenticate(req, res, () => resolve()) + }) +} + +function optionalAuthenticate (req: express.Request, res: express.Response, next: express.NextFunction) { + if (req.header('authorization')) return authenticate(req, res, next) + + res.locals.authenticated = false + + return next() +} + +// --------------------------------------------------------------------------- + +function authenticateRunnerSocket (socket: Socket, next: (err?: any) => void) { + const runnerToken = socket.handshake.auth['runnerToken'] + + logger.debug('Checking runner token in socket.') + + if (!runnerToken) return next(new Error('No runner token provided')) + if (typeof runnerToken !== 'string') return next(new Error('Runner token is invalid')) + + RunnerModel.loadByToken(runnerToken) + .then(runner => { + if (!runner) return next(new Error('Invalid runner token.')) + + socket.handshake.auth.runner = runner + + return next() + }) + .catch(err => logger.error('Cannot get runner token.', { err })) +} + +// --------------------------------------------------------------------------- + +export { + authenticate, + authenticateSocket, + authenticatePromise, + optionalAuthenticate, + authenticateRunnerSocket +} diff --git a/server/server/middlewares/cache/cache.ts b/server/server/middlewares/cache/cache.ts new file mode 100644 index 000000000..6cf37e322 --- /dev/null +++ b/server/server/middlewares/cache/cache.ts @@ -0,0 +1,38 @@ +import { HttpStatusCode } from '@peertube/peertube-models' +import { ApiCache, APICacheOptions } from './shared/index.js' + +const defaultOptions: APICacheOptions = { + excludeStatus: [ + HttpStatusCode.FORBIDDEN_403, + HttpStatusCode.NOT_FOUND_404 + ] +} + +function cacheRoute (duration: string) { + const instance = new ApiCache(defaultOptions) + + return instance.buildMiddleware(duration) +} + +function cacheRouteFactory (options: APICacheOptions) { + const instance = new ApiCache({ ...defaultOptions, ...options }) + + return { instance, middleware: instance.buildMiddleware.bind(instance) } +} + +// --------------------------------------------------------------------------- + +function buildPodcastGroupsCache (options: { + channelId: number +}) { + return 'podcast-feed-' + options.channelId +} + +// --------------------------------------------------------------------------- + +export { + cacheRoute, + cacheRouteFactory, + + buildPodcastGroupsCache +} diff --git a/server/server/middlewares/cache/index.ts b/server/server/middlewares/cache/index.ts new file mode 100644 index 000000000..96c489a16 --- /dev/null +++ b/server/server/middlewares/cache/index.ts @@ -0,0 +1 @@ +export * from './cache.js' diff --git a/server/server/middlewares/cache/shared/api-cache.ts b/server/server/middlewares/cache/shared/api-cache.ts new file mode 100644 index 000000000..514fc2f99 --- /dev/null +++ b/server/server/middlewares/cache/shared/api-cache.ts @@ -0,0 +1,315 @@ +// Thanks: https://github.com/kwhitley/apicache +// We duplicated the library because it is unmaintened and prevent us to upgrade to recent NodeJS versions + +import express from 'express' +import { OutgoingHttpHeaders } from 'http' +import { HttpStatusCodeType } from '@peertube/peertube-models' +import { isTestInstance } from '@peertube/peertube-node-utils' +import { parseDurationToMs } from '@server/helpers/core-utils.js' +import { logger } from '@server/helpers/logger.js' +import { Redis } from '@server/lib/redis.js' +import { asyncMiddleware } from '@server/middlewares/index.js' + +export interface APICacheOptions { + headerBlacklist?: string[] + excludeStatus?: HttpStatusCodeType[] +} + +interface CacheObject { + status: number + headers: OutgoingHttpHeaders + data: any + encoding: BufferEncoding + timestamp: number +} + +export class ApiCache { + + private readonly options: APICacheOptions + private readonly timers: { [ id: string ]: NodeJS.Timeout } = {} + + private readonly index = { + groups: [] as string[], + all: [] as string[] + } + + // Cache keys per group + private groups: { [groupIndex: string]: string[] } = {} + + private readonly seed: number + + constructor (options: APICacheOptions) { + this.seed = new Date().getTime() + + this.options = { + headerBlacklist: [], + excludeStatus: [], + + ...options + } + } + + buildMiddleware (strDuration: string) { + const duration = parseDurationToMs(strDuration) + + return asyncMiddleware( + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + const key = this.getCacheKey(req) + const redis = Redis.Instance.getClient() + + if (!Redis.Instance.isConnected()) return this.makeResponseCacheable(res, next, key, duration) + + try { + const obj = await redis.hgetall(key) + if (obj?.response) { + return this.sendCachedResponse(req, res, JSON.parse(obj.response), duration) + } + + return this.makeResponseCacheable(res, next, key, duration) + } catch (err) { + return this.makeResponseCacheable(res, next, key, duration) + } + } + ) + } + + clearGroupSafe (group: string) { + const run = async () => { + const cacheKeys = this.groups[group] + if (!cacheKeys) return + + for (const key of cacheKeys) { + try { + await this.clear(key) + } catch (err) { + logger.error('Cannot clear ' + key, { err }) + } + } + + delete this.groups[group] + } + + void run() + } + + private getCacheKey (req: express.Request) { + return Redis.Instance.getPrefix() + 'api-cache-' + this.seed + '-' + req.originalUrl + } + + private shouldCacheResponse (response: express.Response) { + if (!response) return false + if (this.options.excludeStatus.includes(response.statusCode as HttpStatusCodeType)) return false + + return true + } + + private addIndexEntries (key: string, res: express.Response) { + this.index.all.unshift(key) + + const groups = res.locals.apicacheGroups || [] + + for (const group of groups) { + if (!this.groups[group]) this.groups[group] = [] + + this.groups[group].push(key) + } + } + + private filterBlacklistedHeaders (headers: OutgoingHttpHeaders) { + return Object.keys(headers) + .filter(key => !this.options.headerBlacklist.includes(key)) + .reduce((acc, header) => { + acc[header] = headers[header] + + return acc + }, {}) + } + + private createCacheObject (status: number, headers: OutgoingHttpHeaders, data: any, encoding: BufferEncoding) { + return { + status, + headers: this.filterBlacklistedHeaders(headers), + data, + encoding, + + // Seconds since epoch, used to properly decrement max-age headers in cached responses. + timestamp: new Date().getTime() / 1000 + } as CacheObject + } + + private async cacheResponse (key: string, value: object, duration: number) { + const redis = Redis.Instance.getClient() + + if (Redis.Instance.isConnected()) { + await Promise.all([ + redis.hset(key, 'response', JSON.stringify(value)), + redis.hset(key, 'duration', duration + ''), + redis.expire(key, duration / 1000) + ]) + } + + // add automatic cache clearing from duration, includes max limit on setTimeout + this.timers[key] = setTimeout(() => { + this.clear(key) + .catch(err => logger.error('Cannot clear Redis key %s.', key, { err })) + }, Math.min(duration, 2147483647)) + } + + private accumulateContent (res: express.Response, content: any) { + if (!content) return + + if (typeof content === 'string') { + res.locals.apicache.content = (res.locals.apicache.content || '') + content + return + } + + if (Buffer.isBuffer(content)) { + let oldContent = res.locals.apicache.content + + if (typeof oldContent === 'string') { + oldContent = Buffer.from(oldContent) + } + + if (!oldContent) { + oldContent = Buffer.alloc(0) + } + + res.locals.apicache.content = Buffer.concat( + [ oldContent, content ], + oldContent.length + content.length + ) + + return + } + + res.locals.apicache.content = content + } + + private makeResponseCacheable (res: express.Response, next: express.NextFunction, key: string, duration: number) { + const self = this + + res.locals.apicache = { + write: res.write, + writeHead: res.writeHead, + end: res.end, + cacheable: true, + content: undefined, + headers: undefined + } + + // Patch express + res.writeHead = function () { + if (self.shouldCacheResponse(res)) { + res.setHeader('cache-control', 'max-age=' + (duration / 1000).toFixed(0)) + } else { + res.setHeader('cache-control', 'no-cache, no-store, must-revalidate') + } + + res.locals.apicache.headers = Object.assign({}, res.getHeaders()) + return res.locals.apicache.writeHead.apply(this, arguments as any) + } + + res.write = function (chunk: any) { + self.accumulateContent(res, chunk) + return res.locals.apicache.write.apply(this, arguments as any) + } + + res.end = function (content: any, encoding: BufferEncoding) { + if (self.shouldCacheResponse(res)) { + self.accumulateContent(res, content) + + if (res.locals.apicache.cacheable && res.locals.apicache.content) { + self.addIndexEntries(key, res) + + const headers = res.locals.apicache.headers || res.getHeaders() + const cacheObject = self.createCacheObject( + res.statusCode, + headers, + res.locals.apicache.content, + encoding + ) + self.cacheResponse(key, cacheObject, duration) + .catch(err => logger.error('Cannot cache response', { err })) + } + } + + res.locals.apicache.end.apply(this, arguments as any) + } as any + + next() + } + + private sendCachedResponse (request: express.Request, response: express.Response, cacheObject: CacheObject, duration: number) { + const headers = response.getHeaders() + + if (isTestInstance()) { + Object.assign(headers, { + 'x-api-cache-cached': 'true' + }) + } + + Object.assign(headers, this.filterBlacklistedHeaders(cacheObject.headers || {}), { + // Set properly decremented max-age header + // This ensures that max-age is in sync with the cache expiration + 'cache-control': + 'max-age=' + + Math.max( + 0, + (duration / 1000 - (new Date().getTime() / 1000 - cacheObject.timestamp)) + ).toFixed(0) + }) + + // unstringify buffers + let data = cacheObject.data + if (data && data.type === 'Buffer') { + data = typeof data.data === 'number' + ? Buffer.alloc(data.data) + : Buffer.from(data.data) + } + + // Test Etag against If-None-Match for 304 + const cachedEtag = cacheObject.headers.etag + const requestEtag = request.headers['if-none-match'] + + if (requestEtag && cachedEtag === requestEtag) { + response.writeHead(304, headers) + return response.end() + } + + response.writeHead(cacheObject.status || 200, headers) + + return response.end(data, cacheObject.encoding) + } + + private async clear (target: string) { + const redis = Redis.Instance.getClient() + + if (target) { + clearTimeout(this.timers[target]) + delete this.timers[target] + + try { + await redis.del(target) + } catch (err) { + logger.error('Cannot delete %s in redis cache.', target, { err }) + } + + this.index.all = this.index.all.filter(key => key !== target) + } else { + for (const key of this.index.all) { + clearTimeout(this.timers[key]) + delete this.timers[key] + + try { + await redis.del(key) + } catch (err) { + logger.error('Cannot delete %s in redis cache.', key, { err }) + } + } + + this.index.all = [] + } + + return this.index + } +} diff --git a/server/server/middlewares/cache/shared/index.ts b/server/server/middlewares/cache/shared/index.ts new file mode 100644 index 000000000..0e0c44f93 --- /dev/null +++ b/server/server/middlewares/cache/shared/index.ts @@ -0,0 +1 @@ +export * from './api-cache.js' diff --git a/server/server/middlewares/csp.ts b/server/server/middlewares/csp.ts new file mode 100644 index 000000000..cc218c670 --- /dev/null +++ b/server/server/middlewares/csp.ts @@ -0,0 +1,40 @@ +import { contentSecurityPolicy } from 'helmet' +import { CONFIG } from '../initializers/config.js' + +const baseDirectives = Object.assign({}, + { + defaultSrc: [ '\'none\'' ], // by default, not specifying default-src = '*' + connectSrc: [ '*', 'data:' ], + mediaSrc: [ '\'self\'', 'https:', 'blob:' ], + fontSrc: [ '\'self\'', 'data:' ], + imgSrc: [ '\'self\'', 'data:', 'blob:' ], + scriptSrc: [ '\'self\' \'unsafe-inline\' \'unsafe-eval\'', 'blob:' ], + styleSrc: [ '\'self\' \'unsafe-inline\'' ], + objectSrc: [ '\'none\'' ], // only define to allow plugins, else let defaultSrc 'none' block it + formAction: [ '\'self\'' ], + frameAncestors: [ '\'none\'' ], + baseUri: [ '\'self\'' ], + manifestSrc: [ '\'self\'' ], + frameSrc: [ '\'self\'' ], // instead of deprecated child-src / self because of test-embed + workerSrc: [ '\'self\'', 'blob:' ] // instead of deprecated child-src + }, + CONFIG.CSP.REPORT_URI ? { reportUri: CONFIG.CSP.REPORT_URI } : {}, + CONFIG.WEBSERVER.SCHEME === 'https' ? { upgradeInsecureRequests: [] } : {} +) + +const baseCSP = contentSecurityPolicy({ + directives: baseDirectives, + reportOnly: CONFIG.CSP.REPORT_ONLY +}) + +const embedCSP = contentSecurityPolicy({ + directives: Object.assign({}, baseDirectives, { frameAncestors: [ '*' ] }), + reportOnly: CONFIG.CSP.REPORT_ONLY +}) + +// --------------------------------------------------------------------------- + +export { + baseCSP, + embedCSP +} diff --git a/server/middlewares/dnt.ts b/server/server/middlewares/dnt.ts similarity index 100% rename from server/middlewares/dnt.ts rename to server/server/middlewares/dnt.ts diff --git a/server/middlewares/doc.ts b/server/server/middlewares/doc.ts similarity index 100% rename from server/middlewares/doc.ts rename to server/server/middlewares/doc.ts diff --git a/server/server/middlewares/error.ts b/server/server/middlewares/error.ts new file mode 100644 index 000000000..706009795 --- /dev/null +++ b/server/server/middlewares/error.ts @@ -0,0 +1,63 @@ +import express from 'express' +import { ProblemDocument, ProblemDocumentExtension } from 'http-problem-details' +import { logger } from '@server/helpers/logger.js' +import { HttpStatusCode } from '@peertube/peertube-models' + +function apiFailMiddleware (req: express.Request, res: express.Response, next: express.NextFunction) { + res.fail = options => { + const { status = HttpStatusCode.BAD_REQUEST_400, message, title, type, data, instance, tags } = options + + const extension = new ProblemDocumentExtension({ + ...data, + + docs: res.locals.docUrl, + code: type, + + // For <= 3.2 compatibility + error: message + }) + + res.status(status) + + if (!res.headersSent) { + res.setHeader('Content-Type', 'application/problem+json') + } + + const json = new ProblemDocument({ + status, + title, + instance, + + detail: message, + + type: type + ? `https://docs.joinpeertube.org/api-rest-reference.html#section/Errors/${type}` + : undefined + }, extension) + + logger.debug('Bad HTTP request.', { json, tags }) + + res.json(json) + } + + if (next) next() +} + +function handleStaticError (err: any, req: express.Request, res: express.Response, next: express.NextFunction) { + const message = err.message || '' + + if (message.includes('ENOENT')) { + return res.fail({ + status: err.status || HttpStatusCode.INTERNAL_SERVER_ERROR_500, + message: err.message, + type: err.name + }) + } + + return next(err) +} + +export { + apiFailMiddleware, + handleStaticError +} diff --git a/server/server/middlewares/index.ts b/server/server/middlewares/index.ts new file mode 100644 index 000000000..96ea240d1 --- /dev/null +++ b/server/server/middlewares/index.ts @@ -0,0 +1,15 @@ +export * from './validators/index.js' +export * from './cache/index.js' +export * from './activitypub.js' +export * from './async.js' +export * from './auth.js' +export * from './pagination.js' +export * from './rate-limiter.js' +export * from './robots.js' +export * from './servers.js' +export * from './sort.js' +export * from './user-right.js' +export * from './dnt.js' +export * from './error.js' +export * from './doc.js' +export * from './csp.js' diff --git a/server/server/middlewares/pagination.ts b/server/server/middlewares/pagination.ts new file mode 100644 index 000000000..acee2f54d --- /dev/null +++ b/server/server/middlewares/pagination.ts @@ -0,0 +1,19 @@ +import express from 'express' +import { forceNumber } from '@peertube/peertube-core-utils' +import { PAGINATION } from '../initializers/constants.js' + +function setDefaultPagination (req: express.Request, res: express.Response, next: express.NextFunction) { + if (!req.query.start) req.query.start = 0 + else req.query.start = forceNumber(req.query.start) + + if (!req.query.count) req.query.count = PAGINATION.GLOBAL.COUNT.DEFAULT + else req.query.count = forceNumber(req.query.count) + + return next() +} + +// --------------------------------------------------------------------------- + +export { + setDefaultPagination +} diff --git a/server/server/middlewares/rate-limiter.ts b/server/server/middlewares/rate-limiter.ts new file mode 100644 index 000000000..3a707709e --- /dev/null +++ b/server/server/middlewares/rate-limiter.ts @@ -0,0 +1,59 @@ +import express from 'express' +import RateLimit, { Options as RateLimitHandlerOptions } from 'express-rate-limit' +import { UserRole, UserRoleType } from '@peertube/peertube-models' +import { CONFIG } from '@server/initializers/config.js' +import { RunnerModel } from '@server/models/runner/runner.js' +import { optionalAuthenticate } from './auth.js' + +const whitelistRoles = new Set([ UserRole.ADMINISTRATOR, UserRole.MODERATOR ]) + +export function buildRateLimiter (options: { + windowMs: number + max: number + skipFailedRequests?: boolean +}) { + return RateLimit({ + windowMs: options.windowMs, + max: options.max, + skipFailedRequests: options.skipFailedRequests, + + handler: (req, res, next, options) => { + // Bypass rate limit for registered runners + if (req.body?.runnerToken) { + return RunnerModel.loadByToken(req.body.runnerToken) + .then(runner => { + if (runner) return next() + + return sendRateLimited(res, options) + }) + } + + // Bypass rate limit for admins/moderators + return optionalAuthenticate(req, res, () => { + if (res.locals.authenticated === true && whitelistRoles.has(res.locals.oauth.token.User.role)) { + return next() + } + + return sendRateLimited(res, options) + }) + } + }) +} + +export const apiRateLimiter = buildRateLimiter({ + windowMs: CONFIG.RATES_LIMIT.API.WINDOW_MS, + max: CONFIG.RATES_LIMIT.API.MAX +}) + +export const activityPubRateLimiter = buildRateLimiter({ + windowMs: CONFIG.RATES_LIMIT.ACTIVITY_PUB.WINDOW_MS, + max: CONFIG.RATES_LIMIT.ACTIVITY_PUB.MAX +}) + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +function sendRateLimited (res: express.Response, options: RateLimitHandlerOptions) { + return res.status(options.statusCode).send(options.message) +} diff --git a/server/middlewares/robots.ts b/server/server/middlewares/robots.ts similarity index 100% rename from server/middlewares/robots.ts rename to server/server/middlewares/robots.ts diff --git a/server/server/middlewares/servers.ts b/server/server/middlewares/servers.ts new file mode 100644 index 000000000..7971f4d24 --- /dev/null +++ b/server/server/middlewares/servers.ts @@ -0,0 +1,29 @@ +import express from 'express' +import { HttpStatusCode } from '@peertube/peertube-models' +import { getHostWithPort } from '../helpers/express-utils.js' + +function setBodyHostsPort (req: express.Request, res: express.Response, next: express.NextFunction) { + if (!req.body.hosts) return next() + + for (let i = 0; i < req.body.hosts.length; i++) { + const hostWithPort = getHostWithPort(req.body.hosts[i]) + + // Problem with the url parsing? + if (hostWithPort === null) { + return res.fail({ + status: HttpStatusCode.INTERNAL_SERVER_ERROR_500, + message: 'Could not parse hosts' + }) + } + + req.body.hosts[i] = hostWithPort + } + + return next() +} + +// --------------------------------------------------------------------------- + +export { + setBodyHostsPort +} diff --git a/server/middlewares/sort.ts b/server/server/middlewares/sort.ts similarity index 100% rename from server/middlewares/sort.ts rename to server/server/middlewares/sort.ts diff --git a/server/server/middlewares/user-right.ts b/server/server/middlewares/user-right.ts new file mode 100644 index 000000000..4240fab2a --- /dev/null +++ b/server/server/middlewares/user-right.ts @@ -0,0 +1,26 @@ +import express from 'express' +import { HttpStatusCode, UserRightType } from '@peertube/peertube-models' +import { logger } from '../helpers/logger.js' + +function ensureUserHasRight (userRight: UserRightType) { + return function (req: express.Request, res: express.Response, next: express.NextFunction) { + const user = res.locals.oauth.token.user + if (user.hasRight(userRight) === false) { + const message = `User ${user.username} does not have right ${userRight} to access to ${req.path}.` + logger.info(message) + + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message + }) + } + + return next() + } +} + +// --------------------------------------------------------------------------- + +export { + ensureUserHasRight +} diff --git a/server/server/middlewares/validators/abuse.ts b/server/server/middlewares/validators/abuse.ts new file mode 100644 index 000000000..48fe06f4c --- /dev/null +++ b/server/server/middlewares/validators/abuse.ts @@ -0,0 +1,254 @@ +import express from 'express' +import { body, param, query } from 'express-validator' +import { forceNumber } from '@peertube/peertube-core-utils' +import { AbuseCreate, HttpStatusCode, UserRight } from '@peertube/peertube-models' +import { + areAbusePredefinedReasonsValid, + isAbuseFilterValid, + isAbuseMessageValid, + isAbuseModerationCommentValid, + isAbusePredefinedReasonValid, + isAbuseReasonValid, + isAbuseStateValid, + isAbuseTimestampCoherent, + isAbuseTimestampValid, + isAbuseVideoIsValid +} from '@server/helpers/custom-validators/abuses.js' +import { exists, isIdOrUUIDValid, isIdValid, toCompleteUUID, toIntOrNull } from '@server/helpers/custom-validators/misc.js' +import { logger } from '@server/helpers/logger.js' +import { AbuseMessageModel } from '@server/models/abuse/abuse-message.js' +import { areValidationErrors, doesAbuseExist, doesAccountIdExist, doesCommentIdExist, doesVideoExist } from './shared/index.js' + +const abuseReportValidator = [ + body('account.id') + .optional() + .custom(isIdValid), + + body('video.id') + .optional() + .customSanitizer(toCompleteUUID) + .custom(isIdOrUUIDValid), + body('video.startAt') + .optional() + .customSanitizer(toIntOrNull) + .custom(isAbuseTimestampValid), + body('video.endAt') + .optional() + .customSanitizer(toIntOrNull) + .custom(isAbuseTimestampValid) + .bail() + .custom(isAbuseTimestampCoherent) + .withMessage('Should have a startAt timestamp beginning before endAt'), + + body('comment.id') + .optional() + .custom(isIdValid), + + body('reason') + .custom(isAbuseReasonValid), + + body('predefinedReasons') + .optional() + .custom(areAbusePredefinedReasonsValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const body: AbuseCreate = req.body + + if (body.video?.id && !await doesVideoExist(body.video.id, res)) return + if (body.account?.id && !await doesAccountIdExist(body.account.id, res)) return + if (body.comment?.id && !await doesCommentIdExist(body.comment.id, res)) return + + if (!body.video?.id && !body.account?.id && !body.comment?.id) { + res.fail({ message: 'video id or account id or comment id is required.' }) + return + } + + return next() + } +] + +const abuseGetValidator = [ + param('id') + .custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesAbuseExist(req.params.id, res)) return + + return next() + } +] + +const abuseUpdateValidator = [ + param('id') + .custom(isIdValid), + + body('state') + .optional() + .custom(isAbuseStateValid), + body('moderationComment') + .optional() + .custom(isAbuseModerationCommentValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesAbuseExist(req.params.id, res)) return + + return next() + } +] + +const abuseListForAdminsValidator = [ + query('id') + .optional() + .custom(isIdValid), + query('filter') + .optional() + .custom(isAbuseFilterValid), + query('predefinedReason') + .optional() + .custom(isAbusePredefinedReasonValid), + query('search') + .optional() + .custom(exists), + query('state') + .optional() + .custom(isAbuseStateValid), + query('videoIs') + .optional() + .custom(isAbuseVideoIsValid), + query('searchReporter') + .optional() + .custom(exists), + query('searchReportee') + .optional() + .custom(exists), + query('searchVideo') + .optional() + .custom(exists), + query('searchVideoChannel') + .optional() + .custom(exists), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const abuseListForUserValidator = [ + query('id') + .optional() + .custom(isIdValid), + + query('search') + .optional() + .custom(exists), + + query('state') + .optional() + .custom(isAbuseStateValid), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const getAbuseValidator = [ + param('id') + .custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesAbuseExist(req.params.id, res)) return + + const user = res.locals.oauth.token.user + const abuse = res.locals.abuse + + if (user.hasRight(UserRight.MANAGE_ABUSES) !== true && abuse.reporterAccountId !== user.Account.id) { + const message = `User ${user.username} does not have right to get abuse ${abuse.id}` + logger.warn(message) + + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message + }) + } + + return next() + } +] + +const checkAbuseValidForMessagesValidator = [ + (req: express.Request, res: express.Response, next: express.NextFunction) => { + const abuse = res.locals.abuse + if (abuse.ReporterAccount.isOwned() === false) { + return res.fail({ message: 'This abuse was created by a user of your instance.' }) + } + + return next() + } +] + +const addAbuseMessageValidator = [ + body('message') + .custom(isAbuseMessageValid), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const deleteAbuseMessageValidator = [ + param('messageId') + .custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const user = res.locals.oauth.token.user + const abuse = res.locals.abuse + + const messageId = forceNumber(req.params.messageId) + const abuseMessage = await AbuseMessageModel.loadByIdAndAbuseId(messageId, abuse.id) + + if (!abuseMessage) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Abuse message not found' + }) + } + + if (user.hasRight(UserRight.MANAGE_ABUSES) !== true && abuseMessage.accountId !== user.Account.id) { + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Cannot delete this abuse message' + }) + } + + res.locals.abuseMessage = abuseMessage + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + abuseListForAdminsValidator, + abuseReportValidator, + abuseGetValidator, + addAbuseMessageValidator, + checkAbuseValidForMessagesValidator, + abuseUpdateValidator, + deleteAbuseMessageValidator, + abuseListForUserValidator, + getAbuseValidator +} diff --git a/server/server/middlewares/validators/account.ts b/server/server/middlewares/validators/account.ts new file mode 100644 index 000000000..a8793752e --- /dev/null +++ b/server/server/middlewares/validators/account.ts @@ -0,0 +1,35 @@ +import express from 'express' +import { param } from 'express-validator' +import { isAccountNameValid } from '../../helpers/custom-validators/accounts.js' +import { areValidationErrors, doesAccountNameWithHostExist, doesLocalAccountNameExist } from './shared/index.js' + +const localAccountValidator = [ + param('name') + .custom(isAccountNameValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesLocalAccountNameExist(req.params.name, res)) return + + return next() + } +] + +const accountNameWithHostGetValidator = [ + param('accountName') + .exists(), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesAccountNameWithHostExist(req.params.accountName, res)) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + localAccountValidator, + accountNameWithHostGetValidator +} diff --git a/server/server/middlewares/validators/activitypub/activity.ts b/server/server/middlewares/validators/activitypub/activity.ts new file mode 100644 index 000000000..9d6bf396b --- /dev/null +++ b/server/server/middlewares/validators/activitypub/activity.ts @@ -0,0 +1,29 @@ +import express from 'express' +import { HttpStatusCode } from '@peertube/peertube-models' +import { getServerActor } from '@server/models/application/application.js' +import { isRootActivityValid } from '../../../helpers/custom-validators/activitypub/activity.js' +import { logger } from '../../../helpers/logger.js' + +async function activityPubValidator (req: express.Request, res: express.Response, next: express.NextFunction) { + logger.debug('Checking activity pub parameters') + + if (!isRootActivityValid(req.body)) { + logger.warn('Incorrect activity parameters.', { activity: req.body }) + return res.fail({ message: 'Incorrect activity' }) + } + + const serverActor = await getServerActor() + const remoteActor = res.locals.signature.actor + if (serverActor.id === remoteActor.id || remoteActor.serverId === null) { + logger.error('Receiving request in INBOX by ourselves!', req.body) + return res.status(HttpStatusCode.CONFLICT_409).end() + } + + return next() +} + +// --------------------------------------------------------------------------- + +export { + activityPubValidator +} diff --git a/server/server/middlewares/validators/activitypub/index.ts b/server/server/middlewares/validators/activitypub/index.ts new file mode 100644 index 000000000..ef02625ba --- /dev/null +++ b/server/server/middlewares/validators/activitypub/index.ts @@ -0,0 +1,3 @@ +export * from './activity.js' +export * from './signature.js' +export * from './pagination.js' diff --git a/server/server/middlewares/validators/activitypub/pagination.ts b/server/server/middlewares/validators/activitypub/pagination.ts new file mode 100644 index 000000000..66cbfb870 --- /dev/null +++ b/server/server/middlewares/validators/activitypub/pagination.ts @@ -0,0 +1,25 @@ +import express from 'express' +import { query } from 'express-validator' +import { PAGINATION } from '@server/initializers/constants.js' +import { areValidationErrors } from '../shared/index.js' + +const apPaginationValidator = [ + query('page') + .optional() + .isInt({ min: 1 }), + query('size') + .optional() + .isInt({ min: 0, max: PAGINATION.OUTBOX.COUNT.MAX }).withMessage(`Should have a valid page size (max: ${PAGINATION.OUTBOX.COUNT.MAX})`), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + apPaginationValidator +} diff --git a/server/server/middlewares/validators/activitypub/signature.ts b/server/server/middlewares/validators/activitypub/signature.ts new file mode 100644 index 000000000..a318c9463 --- /dev/null +++ b/server/server/middlewares/validators/activitypub/signature.ts @@ -0,0 +1,39 @@ +import express from 'express' +import { body } from 'express-validator' +import { + isSignatureCreatorValid, + isSignatureTypeValid, + isSignatureValueValid +} from '../../../helpers/custom-validators/activitypub/signature.js' +import { isDateValid } from '../../../helpers/custom-validators/misc.js' +import { logger } from '../../../helpers/logger.js' +import { areValidationErrors } from '../shared/index.js' + +const signatureValidator = [ + body('signature.type') + .optional() + .custom(isSignatureTypeValid), + body('signature.created') + .optional() + .custom(isDateValid).withMessage('Should have a signature created date that conforms to ISO 8601'), + body('signature.creator') + .optional() + .custom(isSignatureCreatorValid), + body('signature.signatureValue') + .optional() + .custom(isSignatureValueValid), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + logger.debug('Checking Linked Data Signature parameter', { parameters: { signature: req.body.signature } }) + + if (areValidationErrors(req, res, { omitLog: true })) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + signatureValidator +} diff --git a/server/server/middlewares/validators/actor-image.ts b/server/server/middlewares/validators/actor-image.ts new file mode 100644 index 000000000..807d73c52 --- /dev/null +++ b/server/server/middlewares/validators/actor-image.ts @@ -0,0 +1,27 @@ +import express from 'express' +import { body } from 'express-validator' +import { isActorImageFile } from '@server/helpers/custom-validators/actor-images.js' +import { cleanUpReqFiles } from '../../helpers/express-utils.js' +import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js' +import { areValidationErrors } from './shared/index.js' + +const updateActorImageValidatorFactory = (fieldname: string) => ([ + body(fieldname).custom((value, { req }) => isActorImageFile(req.files, fieldname)).withMessage( + 'This file is not supported or too large. Please, make sure it is of the following type : ' + + CONSTRAINTS_FIELDS.ACTORS.IMAGE.EXTNAME.join(', ') + ), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return cleanUpReqFiles(req) + + return next() + } +]) + +const updateAvatarValidator = updateActorImageValidatorFactory('avatarfile') +const updateBannerValidator = updateActorImageValidatorFactory('bannerfile') + +export { + updateAvatarValidator, + updateBannerValidator +} diff --git a/server/server/middlewares/validators/blocklist.ts b/server/server/middlewares/validators/blocklist.ts new file mode 100644 index 000000000..19f1bff1d --- /dev/null +++ b/server/server/middlewares/validators/blocklist.ts @@ -0,0 +1,179 @@ +import express from 'express' +import { body, param, query } from 'express-validator' +import { areValidActorHandles } from '@server/helpers/custom-validators/activitypub/actor.js' +import { getServerActor } from '@server/models/application/application.js' +import { arrayify } from '@peertube/peertube-core-utils' +import { HttpStatusCode } from '@peertube/peertube-models' +import { isEachUniqueHostValid, isHostValid } from '../../helpers/custom-validators/servers.js' +import { WEBSERVER } from '../../initializers/constants.js' +import { AccountBlocklistModel } from '../../models/account/account-blocklist.js' +import { ServerBlocklistModel } from '../../models/server/server-blocklist.js' +import { ServerModel } from '../../models/server/server.js' +import { areValidationErrors, doesAccountNameWithHostExist } from './shared/index.js' + +const blockAccountValidator = [ + body('accountName') + .exists(), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesAccountNameWithHostExist(req.body.accountName, res)) return + + const user = res.locals.oauth.token.User + const accountToBlock = res.locals.account + + if (user.Account.id === accountToBlock.id) { + res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'You cannot block yourself.' + }) + return + } + + return next() + } +] + +const unblockAccountByAccountValidator = [ + param('accountName') + .exists(), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesAccountNameWithHostExist(req.params.accountName, res)) return + + const user = res.locals.oauth.token.User + const targetAccount = res.locals.account + if (!await doesUnblockAccountExist(user.Account.id, targetAccount.id, res)) return + + return next() + } +] + +const unblockAccountByServerValidator = [ + param('accountName') + .exists(), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesAccountNameWithHostExist(req.params.accountName, res)) return + + const serverActor = await getServerActor() + const targetAccount = res.locals.account + if (!await doesUnblockAccountExist(serverActor.Account.id, targetAccount.id, res)) return + + return next() + } +] + +const blockServerValidator = [ + body('host') + .custom(isHostValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const host: string = req.body.host + + if (host === WEBSERVER.HOST) { + return res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'You cannot block your own server.' + }) + } + + const server = await ServerModel.loadOrCreateByHost(host) + + res.locals.server = server + + return next() + } +] + +const unblockServerByAccountValidator = [ + param('host') + .custom(isHostValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const user = res.locals.oauth.token.User + if (!await doesUnblockServerExist(user.Account.id, req.params.host, res)) return + + return next() + } +] + +const unblockServerByServerValidator = [ + param('host') + .custom(isHostValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const serverActor = await getServerActor() + if (!await doesUnblockServerExist(serverActor.Account.id, req.params.host, res)) return + + return next() + } +] + +const blocklistStatusValidator = [ + query('hosts') + .optional() + .customSanitizer(arrayify) + .custom(isEachUniqueHostValid).withMessage('Should have a valid hosts array'), + + query('accounts') + .optional() + .customSanitizer(arrayify) + .custom(areValidActorHandles).withMessage('Should have a valid accounts array'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + blockServerValidator, + blockAccountValidator, + unblockAccountByAccountValidator, + unblockServerByAccountValidator, + unblockAccountByServerValidator, + unblockServerByServerValidator, + blocklistStatusValidator +} + +// --------------------------------------------------------------------------- + +async function doesUnblockAccountExist (accountId: number, targetAccountId: number, res: express.Response) { + const accountBlock = await AccountBlocklistModel.loadByAccountAndTarget(accountId, targetAccountId) + if (!accountBlock) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Account block entry not found.' + }) + return false + } + + res.locals.accountBlock = accountBlock + return true +} + +async function doesUnblockServerExist (accountId: number, host: string, res: express.Response) { + const serverBlock = await ServerBlocklistModel.loadByAccountAndHost(accountId, host) + if (!serverBlock) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Server block entry not found.' + }) + return false + } + + res.locals.serverBlock = serverBlock + return true +} diff --git a/server/server/middlewares/validators/bulk.ts b/server/server/middlewares/validators/bulk.ts new file mode 100644 index 000000000..3c25757c4 --- /dev/null +++ b/server/server/middlewares/validators/bulk.ts @@ -0,0 +1,37 @@ +import express from 'express' +import { body } from 'express-validator' +import { BulkRemoveCommentsOfBody, HttpStatusCode, UserRight } from '@peertube/peertube-models' +import { isBulkRemoveCommentsOfScopeValid } from '@server/helpers/custom-validators/bulk.js' +import { areValidationErrors, doesAccountNameWithHostExist } from './shared/index.js' + +const bulkRemoveCommentsOfValidator = [ + body('accountName') + .exists(), + body('scope') + .custom(isBulkRemoveCommentsOfScopeValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesAccountNameWithHostExist(req.body.accountName, res)) return + + const user = res.locals.oauth.token.User + const body = req.body as BulkRemoveCommentsOfBody + + if (body.scope === 'instance' && user.hasRight(UserRight.REMOVE_ANY_VIDEO_COMMENT) !== true) { + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'User cannot remove any comments of this instance.' + }) + } + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + bulkRemoveCommentsOfValidator +} + +// --------------------------------------------------------------------------- diff --git a/server/server/middlewares/validators/config.ts b/server/server/middlewares/validators/config.ts new file mode 100644 index 000000000..e495bb959 --- /dev/null +++ b/server/server/middlewares/validators/config.ts @@ -0,0 +1,193 @@ +import express from 'express' +import { body } from 'express-validator' +import { CustomConfig, HttpStatusCode } from '@peertube/peertube-models' +import { isIntOrNull } from '@server/helpers/custom-validators/misc.js' +import { CONFIG, isEmailEnabled } from '@server/initializers/config.js' +import { isThemeNameValid } from '../../helpers/custom-validators/plugins.js' +import { isUserNSFWPolicyValid, isUserVideoQuotaDailyValid, isUserVideoQuotaValid } from '../../helpers/custom-validators/users.js' +import { isThemeRegistered } from '../../lib/plugins/theme-utils.js' +import { areValidationErrors } from './shared/index.js' + +const customConfigUpdateValidator = [ + body('instance.name').exists(), + body('instance.shortDescription').exists(), + body('instance.description').exists(), + body('instance.terms').exists(), + body('instance.defaultNSFWPolicy').custom(isUserNSFWPolicyValid), + body('instance.defaultClientRoute').exists(), + body('instance.customizations.css').exists(), + body('instance.customizations.javascript').exists(), + + body('services.twitter.username').exists(), + body('services.twitter.whitelisted').isBoolean(), + + body('cache.previews.size').isInt(), + body('cache.captions.size').isInt(), + body('cache.torrents.size').isInt(), + body('cache.storyboards.size').isInt(), + + body('signup.enabled').isBoolean(), + body('signup.limit').isInt(), + body('signup.requiresEmailVerification').isBoolean(), + body('signup.requiresApproval').isBoolean(), + body('signup.minimumAge').isInt(), + + body('admin.email').isEmail(), + body('contactForm.enabled').isBoolean(), + + body('user.history.videos.enabled').isBoolean(), + body('user.videoQuota').custom(isUserVideoQuotaValid), + body('user.videoQuotaDaily').custom(isUserVideoQuotaDailyValid), + + body('videoChannels.maxPerUser').isInt(), + + body('transcoding.enabled').isBoolean(), + body('transcoding.allowAdditionalExtensions').isBoolean(), + body('transcoding.threads').isInt(), + body('transcoding.concurrency').isInt({ min: 1 }), + body('transcoding.resolutions.0p').isBoolean(), + body('transcoding.resolutions.144p').isBoolean(), + body('transcoding.resolutions.240p').isBoolean(), + body('transcoding.resolutions.360p').isBoolean(), + body('transcoding.resolutions.480p').isBoolean(), + body('transcoding.resolutions.720p').isBoolean(), + body('transcoding.resolutions.1080p').isBoolean(), + body('transcoding.resolutions.1440p').isBoolean(), + body('transcoding.resolutions.2160p').isBoolean(), + body('transcoding.remoteRunners.enabled').isBoolean(), + + body('transcoding.alwaysTranscodeOriginalResolution').isBoolean(), + + body('transcoding.webVideos.enabled').isBoolean(), + body('transcoding.hls.enabled').isBoolean(), + + body('videoStudio.enabled').isBoolean(), + body('videoStudio.remoteRunners.enabled').isBoolean(), + + body('videoFile.update.enabled').isBoolean(), + + body('import.videos.concurrency').isInt({ min: 0 }), + body('import.videos.http.enabled').isBoolean(), + body('import.videos.torrent.enabled').isBoolean(), + + body('import.videoChannelSynchronization.enabled').isBoolean(), + + body('trending.videos.algorithms.default').exists(), + body('trending.videos.algorithms.enabled').exists(), + + body('followers.instance.enabled').isBoolean(), + body('followers.instance.manualApproval').isBoolean(), + + body('theme.default').custom(v => isThemeNameValid(v) && isThemeRegistered(v)), + + body('broadcastMessage.enabled').isBoolean(), + body('broadcastMessage.message').exists(), + body('broadcastMessage.level').exists(), + body('broadcastMessage.dismissable').isBoolean(), + + body('live.enabled').isBoolean(), + body('live.allowReplay').isBoolean(), + body('live.maxDuration').isInt(), + body('live.maxInstanceLives').custom(isIntOrNull), + body('live.maxUserLives').custom(isIntOrNull), + body('live.transcoding.enabled').isBoolean(), + body('live.transcoding.threads').isInt(), + body('live.transcoding.resolutions.144p').isBoolean(), + body('live.transcoding.resolutions.240p').isBoolean(), + body('live.transcoding.resolutions.360p').isBoolean(), + body('live.transcoding.resolutions.480p').isBoolean(), + body('live.transcoding.resolutions.720p').isBoolean(), + body('live.transcoding.resolutions.1080p').isBoolean(), + body('live.transcoding.resolutions.1440p').isBoolean(), + body('live.transcoding.resolutions.2160p').isBoolean(), + body('live.transcoding.alwaysTranscodeOriginalResolution').isBoolean(), + body('live.transcoding.remoteRunners.enabled').isBoolean(), + + body('search.remoteUri.users').isBoolean(), + body('search.remoteUri.anonymous').isBoolean(), + body('search.searchIndex.enabled').isBoolean(), + body('search.searchIndex.url').exists(), + body('search.searchIndex.disableLocalSearch').isBoolean(), + body('search.searchIndex.isDefaultSearch').isBoolean(), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!checkInvalidConfigIfEmailDisabled(req.body, res)) return + if (!checkInvalidTranscodingConfig(req.body, res)) return + if (!checkInvalidSynchronizationConfig(req.body, res)) return + if (!checkInvalidLiveConfig(req.body, res)) return + if (!checkInvalidVideoStudioConfig(req.body, res)) return + + return next() + } +] + +function ensureConfigIsEditable (req: express.Request, res: express.Response, next: express.NextFunction) { + if (!CONFIG.WEBADMIN.CONFIGURATION.EDITION.ALLOWED) { + return res.fail({ + status: HttpStatusCode.METHOD_NOT_ALLOWED_405, + message: 'Server configuration is static and cannot be edited' + }) + } + + return next() +} + +// --------------------------------------------------------------------------- + +export { + customConfigUpdateValidator, + ensureConfigIsEditable +} + +function checkInvalidConfigIfEmailDisabled (customConfig: CustomConfig, res: express.Response) { + if (isEmailEnabled()) return true + + if (customConfig.signup.requiresEmailVerification === true) { + res.fail({ message: 'SMTP is not configured but you require signup email verification.' }) + return false + } + + return true +} + +function checkInvalidTranscodingConfig (customConfig: CustomConfig, res: express.Response) { + if (customConfig.transcoding.enabled === false) return true + + if (customConfig.transcoding.webVideos.enabled === false && customConfig.transcoding.hls.enabled === false) { + res.fail({ message: 'You need to enable at least web_videos transcoding or hls transcoding' }) + return false + } + + return true +} + +function checkInvalidSynchronizationConfig (customConfig: CustomConfig, res: express.Response) { + if (customConfig.import.videoChannelSynchronization.enabled && !customConfig.import.videos.http.enabled) { + res.fail({ message: 'You need to enable HTTP video import in order to enable channel synchronization' }) + return false + } + return true +} + +function checkInvalidLiveConfig (customConfig: CustomConfig, res: express.Response) { + if (customConfig.live.enabled === false) return true + + if (customConfig.live.allowReplay === true && customConfig.transcoding.enabled === false) { + res.fail({ message: 'You cannot allow live replay if transcoding is not enabled' }) + return false + } + + return true +} + +function checkInvalidVideoStudioConfig (customConfig: CustomConfig, res: express.Response) { + if (customConfig.videoStudio.enabled === false) return true + + if (customConfig.videoStudio.enabled === true && customConfig.transcoding.enabled === false) { + res.fail({ message: 'You cannot enable video studio if transcoding is not enabled' }) + return false + } + + return true +} diff --git a/server/middlewares/validators/express.ts b/server/server/middlewares/validators/express.ts similarity index 100% rename from server/middlewares/validators/express.ts rename to server/server/middlewares/validators/express.ts diff --git a/server/server/middlewares/validators/feeds.ts b/server/server/middlewares/validators/feeds.ts new file mode 100644 index 000000000..ec99b6920 --- /dev/null +++ b/server/server/middlewares/validators/feeds.ts @@ -0,0 +1,178 @@ +import express from 'express' +import { param, query } from 'express-validator' +import { HttpStatusCode } from '@peertube/peertube-models' +import { isValidRSSFeed } from '../../helpers/custom-validators/feeds.js' +import { exists, isIdOrUUIDValid, isIdValid, toCompleteUUID } from '../../helpers/custom-validators/misc.js' +import { buildPodcastGroupsCache } from '../cache/index.js' +import { + areValidationErrors, + checkCanSeeVideo, + doesAccountIdExist, + doesAccountNameWithHostExist, + doesUserFeedTokenCorrespond, + doesVideoChannelIdExist, + doesVideoChannelNameWithHostExist, + doesVideoExist +} from './shared/index.js' + +const feedsFormatValidator = [ + param('format') + .optional() + .custom(isValidRSSFeed).withMessage('Should have a valid format (rss, atom, json)'), + query('format') + .optional() + .custom(isValidRSSFeed).withMessage('Should have a valid format (rss, atom, json)'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +function setFeedFormatContentType (req: express.Request, res: express.Response, next: express.NextFunction) { + const format = req.query.format || req.params.format || 'rss' + + let acceptableContentTypes: string[] + if (format === 'atom' || format === 'atom1') { + acceptableContentTypes = [ 'application/atom+xml', 'application/xml', 'text/xml' ] + } else if (format === 'json' || format === 'json1') { + acceptableContentTypes = [ 'application/json' ] + } else if (format === 'rss' || format === 'rss2') { + acceptableContentTypes = [ 'application/rss+xml', 'application/xml', 'text/xml' ] + } else { + acceptableContentTypes = [ 'application/xml', 'text/xml' ] + } + + return feedContentTypeResponse(req, res, next, acceptableContentTypes) +} + +function setFeedPodcastContentType (req: express.Request, res: express.Response, next: express.NextFunction) { + const acceptableContentTypes = [ 'application/rss+xml', 'application/xml', 'text/xml' ] + + return feedContentTypeResponse(req, res, next, acceptableContentTypes) +} + +function feedContentTypeResponse ( + req: express.Request, + res: express.Response, + next: express.NextFunction, + acceptableContentTypes: string[] +) { + if (req.accepts(acceptableContentTypes)) { + res.set('Content-Type', req.accepts(acceptableContentTypes) as string) + } else { + return res.fail({ + status: HttpStatusCode.NOT_ACCEPTABLE_406, + message: `You should accept at least one of the following content-types: ${acceptableContentTypes.join(', ')}` + }) + } + + return next() +} + +// --------------------------------------------------------------------------- + +const feedsAccountOrChannelFiltersValidator = [ + query('accountId') + .optional() + .custom(isIdValid), + + query('accountName') + .optional(), + + query('videoChannelId') + .optional() + .custom(isIdValid), + + query('videoChannelName') + .optional(), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (req.query.accountId && !await doesAccountIdExist(req.query.accountId, res)) return + if (req.query.videoChannelId && !await doesVideoChannelIdExist(req.query.videoChannelId, res)) return + if (req.query.accountName && !await doesAccountNameWithHostExist(req.query.accountName, res)) return + if (req.query.videoChannelName && !await doesVideoChannelNameWithHostExist(req.query.videoChannelName, res)) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +const videoFeedsPodcastValidator = [ + query('videoChannelId') + .custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoChannelIdExist(req.query.videoChannelId, res)) return + + return next() + } +] + +const videoFeedsPodcastSetCacheKey = [ + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (req.query.videoChannelId) { + res.locals.apicacheGroups = [ buildPodcastGroupsCache({ channelId: req.query.videoChannelId }) ] + } + + return next() + } +] +// --------------------------------------------------------------------------- + +const videoSubscriptionFeedsValidator = [ + query('accountId') + .custom(isIdValid), + + query('token') + .custom(exists), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (!await doesAccountIdExist(req.query.accountId, res)) return + if (!await doesUserFeedTokenCorrespond(res.locals.account.userId, req.query.token, res)) return + + return next() + } +] + +const videoCommentsFeedsValidator = [ + query('videoId') + .optional() + .customSanitizer(toCompleteUUID) + .custom(isIdOrUUIDValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (req.query.videoId && (req.query.videoChannelId || req.query.videoChannelName)) { + return res.fail({ message: 'videoId cannot be mixed with a channel filter' }) + } + + if (req.query.videoId) { + if (!await doesVideoExist(req.query.videoId, res)) return + if (!await checkCanSeeVideo({ req, res, paramId: req.query.videoId, video: res.locals.videoAll })) return + } + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + feedsFormatValidator, + setFeedFormatContentType, + setFeedPodcastContentType, + feedsAccountOrChannelFiltersValidator, + videoFeedsPodcastValidator, + videoSubscriptionFeedsValidator, + videoFeedsPodcastSetCacheKey, + videoCommentsFeedsValidator +} diff --git a/server/server/middlewares/validators/follows.ts b/server/server/middlewares/validators/follows.ts new file mode 100644 index 000000000..a124dcf27 --- /dev/null +++ b/server/server/middlewares/validators/follows.ts @@ -0,0 +1,157 @@ +import express from 'express' +import { body, param, query } from 'express-validator' +import { HttpStatusCode, ServerFollowCreate } from '@peertube/peertube-models' +import { isProdInstance } from '@peertube/peertube-node-utils' +import { isEachUniqueHandleValid, isFollowStateValid, isRemoteHandleValid } from '@server/helpers/custom-validators/follows.js' +import { loadActorUrlOrGetFromWebfinger } from '@server/lib/activitypub/actors/index.js' +import { getRemoteNameAndHost } from '@server/lib/activitypub/follow.js' +import { getServerActor } from '@server/models/application/application.js' +import { MActorFollowActorsDefault } from '@server/types/models/index.js' +import { isActorTypeValid, isValidActorHandle } from '../../helpers/custom-validators/activitypub/actor.js' +import { isEachUniqueHostValid, isHostValid } from '../../helpers/custom-validators/servers.js' +import { logger } from '../../helpers/logger.js' +import { WEBSERVER } from '../../initializers/constants.js' +import { ActorFollowModel } from '../../models/actor/actor-follow.js' +import { ActorModel } from '../../models/actor/actor.js' +import { areValidationErrors } from './shared/index.js' + +const listFollowsValidator = [ + query('state') + .optional() + .custom(isFollowStateValid), + query('actorType') + .optional() + .custom(isActorTypeValid), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const followValidator = [ + body('hosts') + .toArray() + .custom(isEachUniqueHostValid).withMessage('Should have an array of unique hosts'), + + body('handles') + .toArray() + .custom(isEachUniqueHandleValid).withMessage('Should have an array of handles'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + // Force https if the administrator wants to follow remote actors + if (isProdInstance() && WEBSERVER.SCHEME === 'http') { + return res + .status(HttpStatusCode.INTERNAL_SERVER_ERROR_500) + .json({ + error: 'Cannot follow on a non HTTPS web server.' + }) + } + + if (areValidationErrors(req, res)) return + + const body: ServerFollowCreate = req.body + if (body.hosts.length === 0 && body.handles.length === 0) { + + return res + .status(HttpStatusCode.BAD_REQUEST_400) + .json({ + error: 'You must provide at least one handle or one host.' + }) + } + + return next() + } +] + +const removeFollowingValidator = [ + param('hostOrHandle') + .custom(value => isHostValid(value) || isRemoteHandleValid(value)), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const serverActor = await getServerActor() + + const { name, host } = getRemoteNameAndHost(req.params.hostOrHandle) + const follow = await ActorFollowModel.loadByActorAndTargetNameAndHostForAPI({ + actorId: serverActor.id, + targetName: name, + targetHost: host + }) + + if (!follow) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: `Follow ${req.params.hostOrHandle} not found.` + }) + } + + res.locals.follow = follow + return next() + } +] + +const getFollowerValidator = [ + param('nameWithHost') + .custom(isValidActorHandle), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + let follow: MActorFollowActorsDefault + try { + const actorUrl = await loadActorUrlOrGetFromWebfinger(req.params.nameWithHost) + const actor = await ActorModel.loadByUrl(actorUrl) + + const serverActor = await getServerActor() + follow = await ActorFollowModel.loadByActorAndTarget(actor.id, serverActor.id) + } catch (err) { + logger.warn('Cannot get actor from handle.', { handle: req.params.nameWithHost, err }) + } + + if (!follow) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: `Follower ${req.params.nameWithHost} not found.` + }) + } + + res.locals.follow = follow + return next() + } +] + +const acceptFollowerValidator = [ + (req: express.Request, res: express.Response, next: express.NextFunction) => { + const follow = res.locals.follow + if (follow.state !== 'pending' && follow.state !== 'rejected') { + return res.fail({ message: 'Follow is not in pending/rejected state.' }) + } + + return next() + } +] + +const rejectFollowerValidator = [ + (req: express.Request, res: express.Response, next: express.NextFunction) => { + const follow = res.locals.follow + if (follow.state !== 'pending' && follow.state !== 'accepted') { + return res.fail({ message: 'Follow is not in pending/accepted state.' }) + } + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + followValidator, + removeFollowingValidator, + getFollowerValidator, + acceptFollowerValidator, + rejectFollowerValidator, + listFollowsValidator +} diff --git a/server/server/middlewares/validators/index.ts b/server/server/middlewares/validators/index.ts new file mode 100644 index 000000000..5c6beb40e --- /dev/null +++ b/server/server/middlewares/validators/index.ts @@ -0,0 +1,32 @@ +export * from './abuse.js' +export * from './account.js' +export * from './activitypub/index.js' +export * from './actor-image.js' +export * from './blocklist.js' +export * from './bulk.js' +export * from './config.js' +export * from './express.js' +export * from './feeds.js' +export * from './follows.js' +export * from './jobs.js' +export * from './logs.js' +export * from './metrics.js' +export * from './object-storage-proxy.js' +export * from './oembed.js' +export * from './pagination.js' +export * from './plugins.js' +export * from './redundancy.js' +export * from './search.js' +export * from './server.js' +export * from './sort.js' +export * from './static.js' +export * from './themes.js' +export * from './user-email-verification.js' +export * from './user-history.js' +export * from './user-notifications.js' +export * from './user-registrations.js' +export * from './user-subscriptions.js' +export * from './users.js' +export * from './videos/index.js' +export * from './webfinger.js' +export * from './runners/index.js' diff --git a/server/server/middlewares/validators/jobs.ts b/server/server/middlewares/validators/jobs.ts new file mode 100644 index 000000000..070cef32e --- /dev/null +++ b/server/server/middlewares/validators/jobs.ts @@ -0,0 +1,29 @@ +import express from 'express' +import { param, query } from 'express-validator' +import { isValidJobState, isValidJobType } from '../../helpers/custom-validators/jobs.js' +import { loggerTagsFactory } from '../../helpers/logger.js' +import { areValidationErrors } from './shared/index.js' + +const lTags = loggerTagsFactory('validators', 'jobs') + +const listJobsValidator = [ + param('state') + .optional() + .custom(isValidJobState), + + query('jobType') + .optional() + .custom(isValidJobType), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, lTags())) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + listJobsValidator +} diff --git a/server/server/middlewares/validators/logs.ts b/server/server/middlewares/validators/logs.ts new file mode 100644 index 000000000..e93d8a618 --- /dev/null +++ b/server/server/middlewares/validators/logs.ts @@ -0,0 +1,93 @@ +import express from 'express' +import { body, query } from 'express-validator' +import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js' +import { isStringArray } from '@server/helpers/custom-validators/search.js' +import { CONFIG } from '@server/initializers/config.js' +import { arrayify } from '@peertube/peertube-core-utils' +import { HttpStatusCode } from '@peertube/peertube-models' +import { + isValidClientLogLevel, + isValidClientLogMessage, + isValidClientLogMeta, + isValidClientLogStackTrace, + isValidClientLogUserAgent, + isValidLogLevel +} from '../../helpers/custom-validators/logs.js' +import { isDateValid } from '../../helpers/custom-validators/misc.js' +import { areValidationErrors } from './shared/index.js' + +const createClientLogValidator = [ + body('message') + .custom(isValidClientLogMessage), + + body('url') + .custom(isUrlValid), + + body('level') + .custom(isValidClientLogLevel), + + body('stackTrace') + .optional() + .custom(isValidClientLogStackTrace), + + body('meta') + .optional() + .custom(isValidClientLogMeta), + + body('userAgent') + .optional() + .custom(isValidClientLogUserAgent), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (CONFIG.LOG.ACCEPT_CLIENT_LOG !== true) { + return res.sendStatus(HttpStatusCode.FORBIDDEN_403) + } + + if (areValidationErrors(req, res)) return + + return next() + } +] + +const getLogsValidator = [ + query('startDate') + .custom(isDateValid).withMessage('Should have a start date that conforms to ISO 8601'), + query('level') + .optional() + .custom(isValidLogLevel), + query('tagsOneOf') + .optional() + .customSanitizer(arrayify) + .custom(isStringArray).withMessage('Should have a valid one of tags array'), + query('endDate') + .optional() + .custom(isDateValid).withMessage('Should have an end date that conforms to ISO 8601'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const getAuditLogsValidator = [ + query('startDate') + .custom(isDateValid).withMessage('Should have a start date that conforms to ISO 8601'), + query('endDate') + .optional() + .custom(isDateValid).withMessage('Should have a end date that conforms to ISO 8601'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + getLogsValidator, + getAuditLogsValidator, + createClientLogValidator +} diff --git a/server/server/middlewares/validators/metrics.ts b/server/server/middlewares/validators/metrics.ts new file mode 100644 index 000000000..6e1e11952 --- /dev/null +++ b/server/server/middlewares/validators/metrics.ts @@ -0,0 +1,60 @@ +import express from 'express' +import { body } from 'express-validator' +import { isValidPlayerMode } from '@server/helpers/custom-validators/metrics.js' +import { isIdOrUUIDValid, toCompleteUUID } from '@server/helpers/custom-validators/misc.js' +import { CONFIG } from '@server/initializers/config.js' +import { HttpStatusCode, PlaybackMetricCreate } from '@peertube/peertube-models' +import { areValidationErrors, doesVideoExist } from './shared/index.js' + +const addPlaybackMetricValidator = [ + body('resolution') + .isInt({ min: 0 }), + body('fps') + .optional() + .isInt({ min: 0 }), + + body('p2pPeers') + .optional() + .isInt({ min: 0 }), + + body('p2pEnabled') + .isBoolean(), + + body('playerMode') + .custom(isValidPlayerMode), + + body('resolutionChanges') + .isInt({ min: 0 }), + + body('errors') + .isInt({ min: 0 }), + + body('downloadedBytesP2P') + .isInt({ min: 0 }), + body('downloadedBytesHTTP') + .isInt({ min: 0 }), + + body('uploadedBytesP2P') + .isInt({ min: 0 }), + + body('videoId') + .customSanitizer(toCompleteUUID) + .custom(isIdOrUUIDValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (!CONFIG.OPEN_TELEMETRY.METRICS.ENABLED) return res.sendStatus(HttpStatusCode.NO_CONTENT_204) + + const body: PlaybackMetricCreate = req.body + + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(body.videoId, res, 'only-immutable-attributes')) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + addPlaybackMetricValidator +} diff --git a/server/server/middlewares/validators/object-storage-proxy.ts b/server/server/middlewares/validators/object-storage-proxy.ts new file mode 100644 index 000000000..755e15a2f --- /dev/null +++ b/server/server/middlewares/validators/object-storage-proxy.ts @@ -0,0 +1,20 @@ +import express from 'express' +import { CONFIG } from '@server/initializers/config.js' +import { HttpStatusCode } from '@peertube/peertube-models' + +const ensurePrivateObjectStorageProxyIsEnabled = [ + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES !== true) { + return res.fail({ + message: 'Private object storage proxy is not enabled', + status: HttpStatusCode.BAD_REQUEST_400 + }) + } + + return next() + } +] + +export { + ensurePrivateObjectStorageProxyIsEnabled +} diff --git a/server/server/middlewares/validators/oembed.ts b/server/server/middlewares/validators/oembed.ts new file mode 100644 index 000000000..32217f693 --- /dev/null +++ b/server/server/middlewares/validators/oembed.ts @@ -0,0 +1,157 @@ +import express from 'express' +import { query } from 'express-validator' +import { join } from 'path' +import { HttpStatusCode, VideoPlaylistPrivacy, VideoPrivacy } from '@peertube/peertube-models' +import { isTestOrDevInstance } from '@peertube/peertube-node-utils' +import { loadVideo } from '@server/lib/model-loaders/index.js' +import { VideoPlaylistModel } from '@server/models/video/video-playlist.js' +import { isIdOrUUIDValid, isUUIDValid, toCompleteUUID } from '../../helpers/custom-validators/misc.js' +import { WEBSERVER } from '../../initializers/constants.js' +import { areValidationErrors } from './shared/index.js' + +const playlistPaths = [ + join('videos', 'watch', 'playlist'), + join('w', 'p') +] + +const videoPaths = [ + join('videos', 'watch'), + 'w' +] + +function buildUrls (paths: string[]) { + return paths.map(p => WEBSERVER.SCHEME + '://' + join(WEBSERVER.HOST, p) + '/') +} + +const startPlaylistURLs = buildUrls(playlistPaths) +const startVideoURLs = buildUrls(videoPaths) + +const isURLOptions = { + require_host: true, + require_tld: true +} + +// We validate 'localhost', so we don't have the top level domain +if (isTestOrDevInstance()) { + isURLOptions.require_tld = false +} + +const oembedValidator = [ + query('url') + .isURL(isURLOptions), + query('maxwidth') + .optional() + .isInt(), + query('maxheight') + .optional() + .isInt(), + query('format') + .optional() + .isIn([ 'xml', 'json' ]), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (req.query.format !== undefined && req.query.format !== 'json') { + return res.fail({ + status: HttpStatusCode.NOT_IMPLEMENTED_501, + message: 'Requested format is not implemented on server.', + data: { + format: req.query.format + } + }) + } + + const url = req.query.url as string + + let urlPath: string + + try { + urlPath = new URL(url).pathname + } catch (err) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: err.message, + data: { + url + } + }) + } + + const isPlaylist = startPlaylistURLs.some(u => url.startsWith(u)) + const isVideo = isPlaylist ? false : startVideoURLs.some(u => url.startsWith(u)) + + const startIsOk = isVideo || isPlaylist + + const parts = urlPath.split('/') + + if (startIsOk === false || parts.length === 0) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Invalid url.', + data: { + url + } + }) + } + + const elementId = toCompleteUUID(parts.pop()) + if (isIdOrUUIDValid(elementId) === false) { + return res.fail({ message: 'Invalid video or playlist id.' }) + } + + if (isVideo) { + const video = await loadVideo(elementId, 'all') + + if (!video) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video not found' + }) + } + + if ( + video.privacy === VideoPrivacy.PUBLIC || + (video.privacy === VideoPrivacy.UNLISTED && isUUIDValid(elementId) === true) + ) { + res.locals.videoAll = video + return next() + } + + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Video is not publicly available' + }) + } + + // Is playlist + + const videoPlaylist = await VideoPlaylistModel.loadWithAccountAndChannelSummary(elementId, undefined) + if (!videoPlaylist) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video playlist not found' + }) + } + + if ( + videoPlaylist.privacy === VideoPlaylistPrivacy.PUBLIC || + (videoPlaylist.privacy === VideoPlaylistPrivacy.UNLISTED && isUUIDValid(elementId)) + ) { + res.locals.videoPlaylistSummary = videoPlaylist + return next() + } + + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Playlist is not public' + }) + } + +] + +// --------------------------------------------------------------------------- + +export { + oembedValidator +} diff --git a/server/server/middlewares/validators/pagination.ts b/server/server/middlewares/validators/pagination.ts new file mode 100644 index 000000000..de5729b2a --- /dev/null +++ b/server/server/middlewares/validators/pagination.ts @@ -0,0 +1,30 @@ +import express from 'express' +import { query } from 'express-validator' +import { PAGINATION } from '@server/initializers/constants.js' +import { areValidationErrors } from './shared/index.js' + +const paginationValidator = paginationValidatorBuilder() + +function paginationValidatorBuilder (tags: string[] = []) { + return [ + query('start') + .optional() + .isInt({ min: 0 }), + query('count') + .optional() + .isInt({ min: 0, max: PAGINATION.GLOBAL.COUNT.MAX }).withMessage(`Should have a number count (max: ${PAGINATION.GLOBAL.COUNT.MAX})`), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { tags })) return + + return next() + } + ] +} + +// --------------------------------------------------------------------------- + +export { + paginationValidator, + paginationValidatorBuilder +} diff --git a/server/server/middlewares/validators/plugins.ts b/server/server/middlewares/validators/plugins.ts new file mode 100644 index 000000000..0d2b7605a --- /dev/null +++ b/server/server/middlewares/validators/plugins.ts @@ -0,0 +1,216 @@ +import express from 'express' +import { body, param, query, ValidationChain } from 'express-validator' +import { HttpStatusCode, InstallOrUpdatePlugin, PluginType_Type } from '@peertube/peertube-models' +import { exists, isBooleanValid, isSafePath, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc.js' +import { + isNpmPluginNameValid, + isPluginNameValid, + isPluginStableOrUnstableVersionValid, + isPluginTypeValid +} from '../../helpers/custom-validators/plugins.js' +import { CONFIG } from '../../initializers/config.js' +import { PluginManager } from '../../lib/plugins/plugin-manager.js' +import { PluginModel } from '../../models/server/plugin.js' +import { areValidationErrors } from './shared/index.js' + +const getPluginValidator = (pluginType: PluginType_Type, withVersion = true) => { + const validators: (ValidationChain | express.Handler)[] = [ + param('pluginName') + .custom(isPluginNameValid) + ] + + if (withVersion) { + validators.push( + param('pluginVersion') + .custom(isPluginStableOrUnstableVersionValid) + ) + } + + return validators.concat([ + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const npmName = PluginModel.buildNpmName(req.params.pluginName, pluginType) + const plugin = PluginManager.Instance.getRegisteredPluginOrTheme(npmName) + + if (!plugin) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'No plugin found named ' + npmName + }) + } + if (withVersion && plugin.version !== req.params.pluginVersion) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'No plugin found named ' + npmName + ' with version ' + req.params.pluginVersion + }) + } + + res.locals.registeredPlugin = plugin + + return next() + } + ]) +} + +const getExternalAuthValidator = [ + param('authName') + .custom(exists), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const plugin = res.locals.registeredPlugin + if (!plugin.registerHelpers) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'No registered helpers were found for this plugin' + }) + } + + const externalAuth = plugin.registerHelpers.getExternalAuths().find(a => a.authName === req.params.authName) + if (!externalAuth) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'No external auths were found for this plugin' + }) + } + + res.locals.externalAuth = externalAuth + + return next() + } +] + +const pluginStaticDirectoryValidator = [ + param('staticEndpoint') + .custom(isSafePath), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const listPluginsValidator = [ + query('pluginType') + .optional() + .customSanitizer(toIntOrNull) + .custom(isPluginTypeValid), + query('uninstalled') + .optional() + .customSanitizer(toBooleanOrNull) + .custom(isBooleanValid), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const installOrUpdatePluginValidator = [ + body('npmName') + .optional() + .custom(isNpmPluginNameValid), + body('pluginVersion') + .optional() + .custom(isPluginStableOrUnstableVersionValid), + body('path') + .optional() + .custom(isSafePath), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const body: InstallOrUpdatePlugin = req.body + if (!body.path && !body.npmName) { + return res.fail({ message: 'Should have either a npmName or a path' }) + } + if (body.pluginVersion && !body.npmName) { + return res.fail({ message: 'Should have a npmName when specifying a pluginVersion' }) + } + + return next() + } +] + +const uninstallPluginValidator = [ + body('npmName') + .custom(isNpmPluginNameValid), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const existingPluginValidator = [ + param('npmName') + .custom(isNpmPluginNameValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const plugin = await PluginModel.loadByNpmName(req.params.npmName) + if (!plugin) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Plugin not found' + }) + } + + res.locals.plugin = plugin + return next() + } +] + +const updatePluginSettingsValidator = [ + body('settings') + .exists(), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const listAvailablePluginsValidator = [ + query('search') + .optional() + .exists(), + query('pluginType') + .optional() + .customSanitizer(toIntOrNull) + .custom(isPluginTypeValid), + query('currentPeerTubeEngine') + .optional() + .custom(isPluginStableOrUnstableVersionValid), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (CONFIG.PLUGINS.INDEX.ENABLED === false) { + return res.fail({ message: 'Plugin index is not enabled' }) + } + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + pluginStaticDirectoryValidator, + getPluginValidator, + updatePluginSettingsValidator, + uninstallPluginValidator, + listAvailablePluginsValidator, + existingPluginValidator, + installOrUpdatePluginValidator, + listPluginsValidator, + getExternalAuthValidator +} diff --git a/server/server/middlewares/validators/redundancy.ts b/server/server/middlewares/validators/redundancy.ts new file mode 100644 index 000000000..d46a645f5 --- /dev/null +++ b/server/server/middlewares/validators/redundancy.ts @@ -0,0 +1,198 @@ +import express from 'express' +import { body, param, query } from 'express-validator' +import { forceNumber } from '@peertube/peertube-core-utils' +import { HttpStatusCode } from '@peertube/peertube-models' +import { isVideoRedundancyTarget } from '@server/helpers/custom-validators/video-redundancies.js' +import { + exists, + isBooleanValid, + isIdOrUUIDValid, + isIdValid, + toBooleanOrNull, + toCompleteUUID, + toIntOrNull +} from '../../helpers/custom-validators/misc.js' +import { isHostValid } from '../../helpers/custom-validators/servers.js' +import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy.js' +import { ServerModel } from '../../models/server/server.js' +import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from './shared/index.js' + +const videoFileRedundancyGetValidator = [ + isValidVideoIdParam('videoId'), + + param('resolution') + .customSanitizer(toIntOrNull) + .custom(exists), + param('fps') + .optional() + .customSanitizer(toIntOrNull) + .custom(exists), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.videoId, res)) return + + const video = res.locals.videoAll + + const paramResolution = req.params.resolution as unknown as number // We casted to int above + const paramFPS = req.params.fps as unknown as number // We casted to int above + + const videoFile = video.VideoFiles.find(f => { + return f.resolution === paramResolution && (!req.params.fps || paramFPS) + }) + + if (!videoFile) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video file not found.' + }) + } + res.locals.videoFile = videoFile + + const videoRedundancy = await VideoRedundancyModel.loadLocalByFileId(videoFile.id) + if (!videoRedundancy) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video redundancy not found.' + }) + } + res.locals.videoRedundancy = videoRedundancy + + return next() + } +] + +const videoPlaylistRedundancyGetValidator = [ + isValidVideoIdParam('videoId'), + + param('streamingPlaylistType') + .customSanitizer(toIntOrNull) + .custom(exists), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.videoId, res)) return + + const video = res.locals.videoAll + + const paramPlaylistType = req.params.streamingPlaylistType as unknown as number // We casted to int above + const videoStreamingPlaylist = video.VideoStreamingPlaylists.find(p => p.type === paramPlaylistType) + + if (!videoStreamingPlaylist) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video playlist not found.' + }) + } + res.locals.videoStreamingPlaylist = videoStreamingPlaylist + + const videoRedundancy = await VideoRedundancyModel.loadLocalByStreamingPlaylistId(videoStreamingPlaylist.id) + if (!videoRedundancy) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video redundancy not found.' + }) + } + res.locals.videoRedundancy = videoRedundancy + + return next() + } +] + +const updateServerRedundancyValidator = [ + param('host') + .custom(isHostValid), + + body('redundancyAllowed') + .customSanitizer(toBooleanOrNull) + .custom(isBooleanValid).withMessage('Should have a valid redundancyAllowed boolean'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const server = await ServerModel.loadByHost(req.params.host) + + if (!server) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: `Server ${req.params.host} not found.` + }) + } + + res.locals.server = server + return next() + } +] + +const listVideoRedundanciesValidator = [ + query('target') + .custom(isVideoRedundancyTarget), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const addVideoRedundancyValidator = [ + body('videoId') + .customSanitizer(toCompleteUUID) + .custom(isIdOrUUIDValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (!await doesVideoExist(req.body.videoId, res, 'only-video')) return + + if (res.locals.onlyVideo.remote === false) { + return res.fail({ message: 'Cannot create a redundancy on a local video' }) + } + + if (res.locals.onlyVideo.isLive) { + return res.fail({ message: 'Cannot create a redundancy of a live video' }) + } + + const alreadyExists = await VideoRedundancyModel.isLocalByVideoUUIDExists(res.locals.onlyVideo.uuid) + if (alreadyExists) { + return res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'This video is already duplicated by your instance.' + }) + } + + return next() + } +] + +const removeVideoRedundancyValidator = [ + param('redundancyId') + .custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const redundancy = await VideoRedundancyModel.loadByIdWithVideo(forceNumber(req.params.redundancyId)) + if (!redundancy) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video redundancy not found' + }) + } + + res.locals.videoRedundancy = redundancy + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + videoFileRedundancyGetValidator, + videoPlaylistRedundancyGetValidator, + updateServerRedundancyValidator, + listVideoRedundanciesValidator, + addVideoRedundancyValidator, + removeVideoRedundancyValidator +} diff --git a/server/server/middlewares/validators/runners/index.ts b/server/server/middlewares/validators/runners/index.ts new file mode 100644 index 000000000..f8cfedec3 --- /dev/null +++ b/server/server/middlewares/validators/runners/index.ts @@ -0,0 +1,3 @@ +export * from './jobs.js' +export * from './registration-token.js' +export * from './runners.js' diff --git a/server/server/middlewares/validators/runners/job-files.ts b/server/server/middlewares/validators/runners/job-files.ts new file mode 100644 index 000000000..8bfd2cce7 --- /dev/null +++ b/server/server/middlewares/validators/runners/job-files.ts @@ -0,0 +1,60 @@ +import express from 'express' +import { param } from 'express-validator' +import { basename } from 'path' +import { isSafeFilename } from '@server/helpers/custom-validators/misc.js' +import { hasVideoStudioTaskFile, HttpStatusCode, RunnerJobStudioTranscodingPayload } from '@peertube/peertube-models' +import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared/index.js' + +const tags = [ 'runner' ] + +export const runnerJobGetVideoTranscodingFileValidator = [ + isValidVideoIdParam('videoId'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (!await doesVideoExist(req.params.videoId, res, 'all')) return + + const runnerJob = res.locals.runnerJob + + if (runnerJob.privatePayload.videoUUID !== res.locals.videoAll.uuid) { + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Job is not associated to this video', + tags: [ ...tags, res.locals.videoAll.uuid ] + }) + } + + return next() + } +] + +export const runnerJobGetVideoStudioTaskFileValidator = [ + param('filename').custom(v => isSafeFilename(v)), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const filename = req.params.filename + + const payload = res.locals.runnerJob.payload as RunnerJobStudioTranscodingPayload + + const found = Array.isArray(payload?.tasks) && payload.tasks.some(t => { + if (hasVideoStudioTaskFile(t)) { + return basename(t.options.file) === filename + } + + return false + }) + + if (!found) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'File is not associated to this edition task', + tags: [ ...tags, res.locals.videoAll.uuid ] + }) + } + + return next() + } +] diff --git a/server/server/middlewares/validators/runners/jobs.ts b/server/server/middlewares/validators/runners/jobs.ts new file mode 100644 index 000000000..9110fce39 --- /dev/null +++ b/server/server/middlewares/validators/runners/jobs.ts @@ -0,0 +1,217 @@ +import { arrayify } from '@peertube/peertube-core-utils' +import { + HttpStatusCode, + RunnerJobLiveRTMPHLSTranscodingPrivatePayload, + RunnerJobState, + RunnerJobStateType, + RunnerJobSuccessBody, + RunnerJobUpdateBody, + ServerErrorCode +} from '@peertube/peertube-models' +import { exists, isUUIDValid } from '@server/helpers/custom-validators/misc.js' +import { + isRunnerJobAbortReasonValid, + isRunnerJobArrayOfStateValid, + isRunnerJobErrorMessageValid, + isRunnerJobProgressValid, + isRunnerJobSuccessPayloadValid, + isRunnerJobTokenValid, + isRunnerJobUpdatePayloadValid +} from '@server/helpers/custom-validators/runners/jobs.js' +import { isRunnerTokenValid } from '@server/helpers/custom-validators/runners/runners.js' +import { cleanUpReqFiles } from '@server/helpers/express-utils.js' +import { LiveManager } from '@server/lib/live/index.js' +import { runnerJobCanBeCancelled } from '@server/lib/runners/index.js' +import { RunnerJobModel } from '@server/models/runner/runner-job.js' +import express from 'express' +import { body, param, query } from 'express-validator' +import { areValidationErrors } from '../shared/index.js' + +const tags = [ 'runner' ] + +export const acceptRunnerJobValidator = [ + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (res.locals.runnerJob.state !== RunnerJobState.PENDING) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'This runner job is not in pending state', + tags + }) + } + + return next() + } +] + +export const abortRunnerJobValidator = [ + body('reason').custom(isRunnerJobAbortReasonValid), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { tags })) return + + return next() + } +] + +export const updateRunnerJobValidator = [ + body('progress').optional().custom(isRunnerJobProgressValid), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { tags })) return cleanUpReqFiles(req) + + const body = req.body as RunnerJobUpdateBody + const job = res.locals.runnerJob + + if (isRunnerJobUpdatePayloadValid(body.payload, job.type, req.files) !== true) { + cleanUpReqFiles(req) + + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Payload is invalid', + tags + }) + } + + if (res.locals.runnerJob.type === 'live-rtmp-hls-transcoding') { + const privatePayload = job.privatePayload as RunnerJobLiveRTMPHLSTranscodingPrivatePayload + + if (!LiveManager.Instance.hasSession(privatePayload.sessionId)) { + cleanUpReqFiles(req) + + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + type: ServerErrorCode.RUNNER_JOB_NOT_IN_PROCESSING_STATE, + message: 'Session of this live ended', + tags + }) + } + } + + return next() + } +] + +export const errorRunnerJobValidator = [ + body('message').custom(isRunnerJobErrorMessageValid), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { tags })) return + + return next() + } +] + +export const successRunnerJobValidator = [ + (req: express.Request, res: express.Response, next: express.NextFunction) => { + const body = req.body as RunnerJobSuccessBody + + if (isRunnerJobSuccessPayloadValid(body.payload, res.locals.runnerJob.type, req.files) !== true) { + cleanUpReqFiles(req) + + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Payload is invalid', + tags + }) + } + + return next() + } +] + +export const cancelRunnerJobValidator = [ + (req: express.Request, res: express.Response, next: express.NextFunction) => { + const runnerJob = res.locals.runnerJob + + if (runnerJobCanBeCancelled(runnerJob) !== true) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Cannot cancel this job that is not in "pending", "processing" or "waiting for parent job" state', + tags + }) + } + + return next() + } +] + +export const listRunnerJobsValidator = [ + query('search') + .optional() + .custom(exists), + + query('stateOneOf') + .optional() + .customSanitizer(arrayify) + .custom(isRunnerJobArrayOfStateValid), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + return next() + } +] + +export const runnerJobGetValidator = [ + param('jobUUID').custom(isUUIDValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { tags })) return + + const runnerJob = await RunnerJobModel.loadWithRunner(req.params.jobUUID) + + if (!runnerJob) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Unknown runner job', + tags + }) + } + + res.locals.runnerJob = runnerJob + + return next() + } +] + +export function jobOfRunnerGetValidatorFactory (allowedStates: RunnerJobStateType[]) { + return [ + param('jobUUID').custom(isUUIDValid), + + body('runnerToken').custom(isRunnerTokenValid), + body('jobToken').custom(isRunnerJobTokenValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { tags })) return cleanUpReqFiles(req) + + const runnerJob = await RunnerJobModel.loadByRunnerAndJobTokensWithRunner({ + uuid: req.params.jobUUID, + runnerToken: req.body.runnerToken, + jobToken: req.body.jobToken + }) + + if (!runnerJob) { + cleanUpReqFiles(req) + + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Unknown runner job', + tags + }) + } + + if (!allowedStates.includes(runnerJob.state)) { + cleanUpReqFiles(req) + + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + type: ServerErrorCode.RUNNER_JOB_NOT_IN_PROCESSING_STATE, + message: 'Job is not in "processing" state', + tags + }) + } + + res.locals.runnerJob = runnerJob + + return next() + } + ] +} diff --git a/server/server/middlewares/validators/runners/registration-token.ts b/server/server/middlewares/validators/runners/registration-token.ts new file mode 100644 index 000000000..046b2c988 --- /dev/null +++ b/server/server/middlewares/validators/runners/registration-token.ts @@ -0,0 +1,37 @@ +import express from 'express' +import { param } from 'express-validator' +import { isIdValid } from '@server/helpers/custom-validators/misc.js' +import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token.js' +import { forceNumber } from '@peertube/peertube-core-utils' +import { HttpStatusCode } from '@peertube/peertube-models' +import { areValidationErrors } from '../shared/utils.js' + +const tags = [ 'runner' ] + +const deleteRegistrationTokenValidator = [ + param('id').custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { tags })) return + + const registrationToken = await RunnerRegistrationTokenModel.load(forceNumber(req.params.id)) + + if (!registrationToken) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Registration token not found', + tags + }) + } + + res.locals.runnerRegistrationToken = registrationToken + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + deleteRegistrationTokenValidator +} diff --git a/server/server/middlewares/validators/runners/runners.ts b/server/server/middlewares/validators/runners/runners.ts new file mode 100644 index 000000000..80525c609 --- /dev/null +++ b/server/server/middlewares/validators/runners/runners.ts @@ -0,0 +1,104 @@ +import express from 'express' +import { body, param } from 'express-validator' +import { isIdValid } from '@server/helpers/custom-validators/misc.js' +import { + isRunnerDescriptionValid, + isRunnerNameValid, + isRunnerRegistrationTokenValid, + isRunnerTokenValid +} from '@server/helpers/custom-validators/runners/runners.js' +import { RunnerModel } from '@server/models/runner/runner.js' +import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token.js' +import { forceNumber } from '@peertube/peertube-core-utils' +import { HttpStatusCode, RegisterRunnerBody, ServerErrorCode } from '@peertube/peertube-models' +import { areValidationErrors } from '../shared/utils.js' + +const tags = [ 'runner' ] + +const registerRunnerValidator = [ + body('registrationToken').custom(isRunnerRegistrationTokenValid), + body('name').custom(isRunnerNameValid), + body('description').optional().custom(isRunnerDescriptionValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { tags })) return + + const body: RegisterRunnerBody = req.body + + const runnerRegistrationToken = await RunnerRegistrationTokenModel.loadByRegistrationToken(body.registrationToken) + + if (!runnerRegistrationToken) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Registration token is invalid', + tags + }) + } + + const existing = await RunnerModel.loadByName(body.name) + if (existing) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'This runner name already exists on this instance', + tags + }) + } + + res.locals.runnerRegistrationToken = runnerRegistrationToken + + return next() + } +] + +const deleteRunnerValidator = [ + param('runnerId').custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { tags })) return + + const runner = await RunnerModel.load(forceNumber(req.params.runnerId)) + + if (!runner) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Runner not found', + tags + }) + } + + res.locals.runner = runner + + return next() + } +] + +const getRunnerFromTokenValidator = [ + body('runnerToken').custom(isRunnerTokenValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { tags })) return + + const runner = await RunnerModel.loadByToken(req.body.runnerToken) + + if (!runner) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Unknown runner token', + type: ServerErrorCode.UNKNOWN_RUNNER_TOKEN, + tags + }) + } + + res.locals.runner = runner + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + registerRunnerValidator, + deleteRunnerValidator, + getRunnerFromTokenValidator +} diff --git a/server/server/middlewares/validators/search.ts b/server/server/middlewares/validators/search.ts new file mode 100644 index 000000000..9bfcfd79e --- /dev/null +++ b/server/server/middlewares/validators/search.ts @@ -0,0 +1,112 @@ +import express from 'express' +import { query } from 'express-validator' +import { isSearchTargetValid } from '@server/helpers/custom-validators/search.js' +import { isHostValid } from '@server/helpers/custom-validators/servers.js' +import { areUUIDsValid, isDateValid, isNotEmptyStringArray, toCompleteUUIDs } from '../../helpers/custom-validators/misc.js' +import { areValidationErrors } from './shared/index.js' + +const videosSearchValidator = [ + query('search') + .optional() + .not().isEmpty(), + + query('host') + .optional() + .custom(isHostValid), + + query('startDate') + .optional() + .custom(isDateValid).withMessage('Should have a start date that conforms to ISO 8601'), + query('endDate') + .optional() + .custom(isDateValid).withMessage('Should have a end date that conforms to ISO 8601'), + + query('originallyPublishedStartDate') + .optional() + .custom(isDateValid).withMessage('Should have a published start date that conforms to ISO 8601'), + query('originallyPublishedEndDate') + .optional() + .custom(isDateValid).withMessage('Should have a published end date that conforms to ISO 8601'), + + query('durationMin') + .optional() + .isInt(), + query('durationMax') + .optional() + .isInt(), + + query('uuids') + .optional() + .toArray() + .customSanitizer(toCompleteUUIDs) + .custom(areUUIDsValid).withMessage('Should have valid array of uuid'), + + query('searchTarget') + .optional() + .custom(isSearchTargetValid), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const videoChannelsListSearchValidator = [ + query('search') + .optional() + .not().isEmpty(), + + query('host') + .optional() + .custom(isHostValid), + + query('searchTarget') + .optional() + .custom(isSearchTargetValid), + + query('handles') + .optional() + .toArray() + .custom(isNotEmptyStringArray).withMessage('Should have valid array of handles'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const videoPlaylistsListSearchValidator = [ + query('search') + .optional() + .not().isEmpty(), + + query('host') + .optional() + .custom(isHostValid), + + query('searchTarget') + .optional() + .custom(isSearchTargetValid), + + query('uuids') + .optional() + .toArray() + .customSanitizer(toCompleteUUIDs) + .custom(areUUIDsValid).withMessage('Should have valid array of uuid'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + videosSearchValidator, + videoChannelsListSearchValidator, + videoPlaylistsListSearchValidator +} diff --git a/server/server/middlewares/validators/server.ts b/server/server/middlewares/validators/server.ts new file mode 100644 index 000000000..b8d7a8aa4 --- /dev/null +++ b/server/server/middlewares/validators/server.ts @@ -0,0 +1,75 @@ +import express from 'express' +import { body } from 'express-validator' +import { HttpStatusCode } from '@peertube/peertube-models' +import { isHostValid, isValidContactBody } from '../../helpers/custom-validators/servers.js' +import { isUserDisplayNameValid } from '../../helpers/custom-validators/users.js' +import { logger } from '../../helpers/logger.js' +import { CONFIG, isEmailEnabled } from '../../initializers/config.js' +import { Redis } from '../../lib/redis.js' +import { ServerModel } from '../../models/server/server.js' +import { areValidationErrors } from './shared/index.js' + +const serverGetValidator = [ + body('host').custom(isHostValid).withMessage('Should have a valid host'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const server = await ServerModel.loadByHost(req.body.host) + if (!server) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Server host not found.' + }) + } + + res.locals.server = server + + return next() + } +] + +const contactAdministratorValidator = [ + body('fromName') + .custom(isUserDisplayNameValid), + body('fromEmail') + .isEmail(), + body('body') + .custom(isValidContactBody), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (CONFIG.CONTACT_FORM.ENABLED === false) { + return res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'Contact form is not enabled on this instance.' + }) + } + + if (isEmailEnabled() === false) { + return res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'SMTP is not configured on this instance.' + }) + } + + if (await Redis.Instance.doesContactFormIpExist(req.ip)) { + logger.info('Refusing a contact form by %s: already sent one recently.', req.ip) + + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'You already sent a contact form recently.' + }) + } + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + serverGetValidator, + contactAdministratorValidator +} diff --git a/server/server/middlewares/validators/shared/abuses.ts b/server/server/middlewares/validators/shared/abuses.ts new file mode 100644 index 000000000..dfe18fb4f --- /dev/null +++ b/server/server/middlewares/validators/shared/abuses.ts @@ -0,0 +1,26 @@ +import { Response } from 'express' +import { AbuseModel } from '@server/models/abuse/abuse.js' +import { HttpStatusCode } from '@peertube/peertube-models' +import { forceNumber } from '@peertube/peertube-core-utils' + +async function doesAbuseExist (abuseId: number | string, res: Response) { + const abuse = await AbuseModel.loadByIdWithReporter(forceNumber(abuseId)) + + if (!abuse) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Abuse not found' + }) + + return false + } + + res.locals.abuse = abuse + return true +} + +// --------------------------------------------------------------------------- + +export { + doesAbuseExist +} diff --git a/server/server/middlewares/validators/shared/accounts.ts b/server/server/middlewares/validators/shared/accounts.ts new file mode 100644 index 000000000..27c56a09e --- /dev/null +++ b/server/server/middlewares/validators/shared/accounts.ts @@ -0,0 +1,66 @@ +import { Response } from 'express' +import { AccountModel } from '@server/models/account/account.js' +import { UserModel } from '@server/models/user/user.js' +import { MAccountDefault } from '@server/types/models/index.js' +import { forceNumber } from '@peertube/peertube-core-utils' +import { HttpStatusCode } from '@peertube/peertube-models' + +function doesAccountIdExist (id: number | string, res: Response, sendNotFound = true) { + const promise = AccountModel.load(forceNumber(id)) + + return doesAccountExist(promise, res, sendNotFound) +} + +function doesLocalAccountNameExist (name: string, res: Response, sendNotFound = true) { + const promise = AccountModel.loadLocalByName(name) + + return doesAccountExist(promise, res, sendNotFound) +} + +function doesAccountNameWithHostExist (nameWithDomain: string, res: Response, sendNotFound = true) { + const promise = AccountModel.loadByNameWithHost(nameWithDomain) + + return doesAccountExist(promise, res, sendNotFound) +} + +async function doesAccountExist (p: Promise, res: Response, sendNotFound: boolean) { + const account = await p + + if (!account) { + if (sendNotFound === true) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Account not found' + }) + } + return false + } + + res.locals.account = account + return true +} + +async function doesUserFeedTokenCorrespond (id: number, token: string, res: Response) { + const user = await UserModel.loadByIdWithChannels(forceNumber(id)) + + if (token !== user.feedToken) { + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'User and token mismatch' + }) + return false + } + + res.locals.user = user + return true +} + +// --------------------------------------------------------------------------- + +export { + doesAccountIdExist, + doesLocalAccountNameExist, + doesAccountNameWithHostExist, + doesAccountExist, + doesUserFeedTokenCorrespond +} diff --git a/server/server/middlewares/validators/shared/index.ts b/server/server/middlewares/validators/shared/index.ts new file mode 100644 index 000000000..d60ac49ca --- /dev/null +++ b/server/server/middlewares/validators/shared/index.ts @@ -0,0 +1,14 @@ +export * from './abuses.js' +export * from './accounts.js' +export * from './users.js' +export * from './utils.js' +export * from './video-blacklists.js' +export * from './video-captions.js' +export * from './video-channels.js' +export * from './video-channel-syncs.js' +export * from './video-comments.js' +export * from './video-imports.js' +export * from './video-ownerships.js' +export * from './video-playlists.js' +export * from './video-passwords.js' +export * from './videos.js' diff --git a/server/server/middlewares/validators/shared/user-registrations.ts b/server/server/middlewares/validators/shared/user-registrations.ts new file mode 100644 index 000000000..ede9d6b91 --- /dev/null +++ b/server/server/middlewares/validators/shared/user-registrations.ts @@ -0,0 +1,60 @@ +import express from 'express' +import { UserRegistrationModel } from '@server/models/user/user-registration.js' +import { MRegistration } from '@server/types/models/index.js' +import { forceNumber, pick } from '@peertube/peertube-core-utils' +import { HttpStatusCode } from '@peertube/peertube-models' + +function checkRegistrationIdExist (idArg: number | string, res: express.Response) { + const id = forceNumber(idArg) + return checkRegistrationExist(() => UserRegistrationModel.load(id), res) +} + +function checkRegistrationEmailExist (email: string, res: express.Response, abortResponse = true) { + return checkRegistrationExist(() => UserRegistrationModel.loadByEmail(email), res, abortResponse) +} + +async function checkRegistrationHandlesDoNotAlreadyExist (options: { + username: string + channelHandle: string + email: string + res: express.Response +}) { + const { res } = options + + const registration = await UserRegistrationModel.loadByEmailOrHandle(pick(options, [ 'username', 'email', 'channelHandle' ])) + + if (registration) { + res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'Registration with this username, channel name or email already exists.' + }) + return false + } + + return true +} + +async function checkRegistrationExist (finder: () => Promise, res: express.Response, abortResponse = true) { + const registration = await finder() + + if (!registration) { + if (abortResponse === true) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'User not found' + }) + } + + return false + } + + res.locals.userRegistration = registration + return true +} + +export { + checkRegistrationIdExist, + checkRegistrationEmailExist, + checkRegistrationHandlesDoNotAlreadyExist, + checkRegistrationExist +} diff --git a/server/server/middlewares/validators/shared/users.ts b/server/server/middlewares/validators/shared/users.ts new file mode 100644 index 000000000..025f2a335 --- /dev/null +++ b/server/server/middlewares/validators/shared/users.ts @@ -0,0 +1,63 @@ +import express from 'express' +import { ActorModel } from '@server/models/actor/actor.js' +import { UserModel } from '@server/models/user/user.js' +import { MUserDefault } from '@server/types/models/index.js' +import { forceNumber } from '@peertube/peertube-core-utils' +import { HttpStatusCode } from '@peertube/peertube-models' + +function checkUserIdExist (idArg: number | string, res: express.Response, withStats = false) { + const id = forceNumber(idArg) + return checkUserExist(() => UserModel.loadByIdWithChannels(id, withStats), res) +} + +function checkUserEmailExist (email: string, res: express.Response, abortResponse = true) { + return checkUserExist(() => UserModel.loadByEmail(email), res, abortResponse) +} + +async function checkUserNameOrEmailDoNotAlreadyExist (username: string, email: string, res: express.Response) { + const user = await UserModel.loadByUsernameOrEmail(username, email) + + if (user) { + res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'User with this username or email already exists.' + }) + return false + } + + const actor = await ActorModel.loadLocalByName(username) + if (actor) { + res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'Another actor (account/channel) with this name on this instance already exists or has already existed.' + }) + return false + } + + return true +} + +async function checkUserExist (finder: () => Promise, res: express.Response, abortResponse = true) { + const user = await finder() + + if (!user) { + if (abortResponse === true) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'User not found' + }) + } + + return false + } + + res.locals.user = user + return true +} + +export { + checkUserIdExist, + checkUserEmailExist, + checkUserNameOrEmailDoNotAlreadyExist, + checkUserExist +} diff --git a/server/server/middlewares/validators/shared/utils.ts b/server/server/middlewares/validators/shared/utils.ts new file mode 100644 index 000000000..c36525a6d --- /dev/null +++ b/server/server/middlewares/validators/shared/utils.ts @@ -0,0 +1,69 @@ +import express from 'express' +import { param, validationResult } from 'express-validator' +import { isIdOrUUIDValid, toCompleteUUID } from '@server/helpers/custom-validators/misc.js' +import { logger } from '../../../helpers/logger.js' + +function areValidationErrors ( + req: express.Request, + res: express.Response, + options: { + omitLog?: boolean + omitBodyLog?: boolean + tags?: string[] + } = {}) { + const { omitLog = false, omitBodyLog = false, tags = [] } = options + + if (!omitLog) { + logger.debug( + 'Checking %s - %s parameters', + req.method, req.originalUrl, + { + body: omitBodyLog + ? 'omitted' + : req.body, + params: req.params, + query: req.query, + files: req.files, + tags + } + ) + } + + const errors = validationResult(req) + + if (!errors.isEmpty()) { + logger.warn('Incorrect request parameters', { path: req.originalUrl, err: errors.mapped() }) + + res.fail({ + message: 'Incorrect request parameters: ' + Object.keys(errors.mapped()).join(', '), + instance: req.originalUrl, + data: { + 'invalid-params': errors.mapped() + } + }) + + return true + } + + return false +} + +function isValidVideoIdParam (paramName: string) { + return param(paramName) + .customSanitizer(toCompleteUUID) + .custom(isIdOrUUIDValid).withMessage('Should have a valid video id (id, short UUID or UUID)') +} + +function isValidPlaylistIdParam (paramName: string) { + return param(paramName) + .customSanitizer(toCompleteUUID) + .custom(isIdOrUUIDValid).withMessage('Should have a valid playlist id (id, short UUID or UUID)') +} + +// --------------------------------------------------------------------------- + +export { + areValidationErrors, + isValidVideoIdParam, + isValidPlaylistIdParam +} diff --git a/server/server/middlewares/validators/shared/video-blacklists.ts b/server/server/middlewares/validators/shared/video-blacklists.ts new file mode 100644 index 000000000..fa81305de --- /dev/null +++ b/server/server/middlewares/validators/shared/video-blacklists.ts @@ -0,0 +1,24 @@ +import { Response } from 'express' +import { VideoBlacklistModel } from '@server/models/video/video-blacklist.js' +import { HttpStatusCode } from '@peertube/peertube-models' + +async function doesVideoBlacklistExist (videoId: number, res: Response) { + const videoBlacklist = await VideoBlacklistModel.loadByVideoId(videoId) + + if (videoBlacklist === null) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Blacklisted video not found' + }) + return false + } + + res.locals.videoBlacklist = videoBlacklist + return true +} + +// --------------------------------------------------------------------------- + +export { + doesVideoBlacklistExist +} diff --git a/server/server/middlewares/validators/shared/video-captions.ts b/server/server/middlewares/validators/shared/video-captions.ts new file mode 100644 index 000000000..bf2c27a7f --- /dev/null +++ b/server/server/middlewares/validators/shared/video-captions.ts @@ -0,0 +1,25 @@ +import { Response } from 'express' +import { VideoCaptionModel } from '@server/models/video/video-caption.js' +import { MVideoId } from '@server/types/models/index.js' +import { HttpStatusCode } from '@peertube/peertube-models' + +async function doesVideoCaptionExist (video: MVideoId, language: string, res: Response) { + const videoCaption = await VideoCaptionModel.loadByVideoIdAndLanguage(video.id, language) + + if (!videoCaption) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video caption not found' + }) + return false + } + + res.locals.videoCaption = videoCaption + return true +} + +// --------------------------------------------------------------------------- + +export { + doesVideoCaptionExist +} diff --git a/server/server/middlewares/validators/shared/video-channel-syncs.ts b/server/server/middlewares/validators/shared/video-channel-syncs.ts new file mode 100644 index 000000000..5d6872833 --- /dev/null +++ b/server/server/middlewares/validators/shared/video-channel-syncs.ts @@ -0,0 +1,24 @@ +import express from 'express' +import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync.js' +import { HttpStatusCode } from '@peertube/peertube-models' + +async function doesVideoChannelSyncIdExist (id: number, res: express.Response) { + const sync = await VideoChannelSyncModel.loadWithChannel(+id) + + if (!sync) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video channel sync not found' + }) + return false + } + + res.locals.videoChannelSync = sync + return true +} + +// --------------------------------------------------------------------------- + +export { + doesVideoChannelSyncIdExist +} diff --git a/server/server/middlewares/validators/shared/video-channels.ts b/server/server/middlewares/validators/shared/video-channels.ts new file mode 100644 index 000000000..eee379c19 --- /dev/null +++ b/server/server/middlewares/validators/shared/video-channels.ts @@ -0,0 +1,36 @@ +import express from 'express' +import { VideoChannelModel } from '@server/models/video/video-channel.js' +import { MChannelBannerAccountDefault } from '@server/types/models/index.js' +import { HttpStatusCode } from '@peertube/peertube-models' + +async function doesVideoChannelIdExist (id: number, res: express.Response) { + const videoChannel = await VideoChannelModel.loadAndPopulateAccount(+id) + + return processVideoChannelExist(videoChannel, res) +} + +async function doesVideoChannelNameWithHostExist (nameWithDomain: string, res: express.Response) { + const videoChannel = await VideoChannelModel.loadByNameWithHostAndPopulateAccount(nameWithDomain) + + return processVideoChannelExist(videoChannel, res) +} + +// --------------------------------------------------------------------------- + +export { + doesVideoChannelIdExist, + doesVideoChannelNameWithHostExist +} + +function processVideoChannelExist (videoChannel: MChannelBannerAccountDefault, res: express.Response) { + if (!videoChannel) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video channel not found' + }) + return false + } + + res.locals.videoChannel = videoChannel + return true +} diff --git a/server/server/middlewares/validators/shared/video-comments.ts b/server/server/middlewares/validators/shared/video-comments.ts new file mode 100644 index 000000000..e11696dbf --- /dev/null +++ b/server/server/middlewares/validators/shared/video-comments.ts @@ -0,0 +1,80 @@ +import express from 'express' +import { VideoCommentModel } from '@server/models/video/video-comment.js' +import { MVideoId } from '@server/types/models/index.js' +import { forceNumber } from '@peertube/peertube-core-utils' +import { HttpStatusCode, ServerErrorCode } from '@peertube/peertube-models' + +async function doesVideoCommentThreadExist (idArg: number | string, video: MVideoId, res: express.Response) { + const id = forceNumber(idArg) + const videoComment = await VideoCommentModel.loadById(id) + + if (!videoComment) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video comment thread not found' + }) + return false + } + + if (videoComment.videoId !== video.id) { + res.fail({ + type: ServerErrorCode.COMMENT_NOT_ASSOCIATED_TO_VIDEO, + message: 'Video comment is not associated to this video.' + }) + return false + } + + if (videoComment.inReplyToCommentId !== null) { + res.fail({ message: 'Video comment is not a thread.' }) + return false + } + + res.locals.videoCommentThread = videoComment + return true +} + +async function doesVideoCommentExist (idArg: number | string, video: MVideoId, res: express.Response) { + const id = forceNumber(idArg) + const videoComment = await VideoCommentModel.loadByIdAndPopulateVideoAndAccountAndReply(id) + + if (!videoComment) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video comment thread not found' + }) + return false + } + + if (videoComment.videoId !== video.id) { + res.fail({ + type: ServerErrorCode.COMMENT_NOT_ASSOCIATED_TO_VIDEO, + message: 'Video comment is not associated to this video.' + }) + return false + } + + res.locals.videoCommentFull = videoComment + return true +} + +async function doesCommentIdExist (idArg: number | string, res: express.Response) { + const id = forceNumber(idArg) + const videoComment = await VideoCommentModel.loadByIdAndPopulateVideoAndAccountAndReply(id) + + if (!videoComment) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video comment thread not found' + }) + return false + } + + res.locals.videoCommentFull = videoComment + return true +} + +export { + doesVideoCommentThreadExist, + doesVideoCommentExist, + doesCommentIdExist +} diff --git a/server/server/middlewares/validators/shared/video-imports.ts b/server/server/middlewares/validators/shared/video-imports.ts new file mode 100644 index 000000000..9b7d8e7f0 --- /dev/null +++ b/server/server/middlewares/validators/shared/video-imports.ts @@ -0,0 +1,22 @@ +import express from 'express' +import { VideoImportModel } from '@server/models/video/video-import.js' +import { HttpStatusCode } from '@peertube/peertube-models' + +async function doesVideoImportExist (id: number, res: express.Response) { + const videoImport = await VideoImportModel.loadAndPopulateVideo(id) + + if (!videoImport) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video import not found' + }) + return false + } + + res.locals.videoImport = videoImport + return true +} + +export { + doesVideoImportExist +} diff --git a/server/server/middlewares/validators/shared/video-ownerships.ts b/server/server/middlewares/validators/shared/video-ownerships.ts new file mode 100644 index 000000000..9f3d7b2c2 --- /dev/null +++ b/server/server/middlewares/validators/shared/video-ownerships.ts @@ -0,0 +1,25 @@ +import express from 'express' +import { VideoChangeOwnershipModel } from '@server/models/video/video-change-ownership.js' +import { forceNumber } from '@peertube/peertube-core-utils' +import { HttpStatusCode } from '@peertube/peertube-models' + +async function doesChangeVideoOwnershipExist (idArg: number | string, res: express.Response) { + const id = forceNumber(idArg) + const videoChangeOwnership = await VideoChangeOwnershipModel.load(id) + + if (!videoChangeOwnership) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video change ownership not found' + }) + return false + } + + res.locals.videoChangeOwnership = videoChangeOwnership + + return true +} + +export { + doesChangeVideoOwnershipExist +} diff --git a/server/server/middlewares/validators/shared/video-passwords.ts b/server/server/middlewares/validators/shared/video-passwords.ts new file mode 100644 index 000000000..dc82be572 --- /dev/null +++ b/server/server/middlewares/validators/shared/video-passwords.ts @@ -0,0 +1,80 @@ +import express from 'express' +import { HttpStatusCode, UserRight, VideoPrivacy } from '@peertube/peertube-models' +import { forceNumber } from '@peertube/peertube-core-utils' +import { VideoPasswordModel } from '@server/models/video/video-password.js' +import { header } from 'express-validator' +import { getVideoWithAttributes } from '@server/helpers/video.js' + +function isValidVideoPasswordHeader () { + return header('x-peertube-video-password') + .optional() + .isString() +} + +function checkVideoIsPasswordProtected (res: express.Response) { + const video = getVideoWithAttributes(res) + if (video.privacy !== VideoPrivacy.PASSWORD_PROTECTED) { + res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Video is not password protected' + }) + return false + } + + return true +} + +async function doesVideoPasswordExist (idArg: number | string, res: express.Response) { + const video = getVideoWithAttributes(res) + const id = forceNumber(idArg) + const videoPassword = await VideoPasswordModel.loadByIdAndVideo({ id, videoId: video.id }) + + if (!videoPassword) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video password not found' + }) + return false + } + + res.locals.videoPassword = videoPassword + + return true +} + +async function isVideoPasswordDeletable (res: express.Response) { + const user = res.locals.oauth.token.User + const userAccount = user.Account + const video = res.locals.videoAll + + // Check if the user who did the request is able to delete the video passwords + if ( + user.hasRight(UserRight.UPDATE_ANY_VIDEO) === false && // Not a moderator + video.VideoChannel.accountId !== userAccount.id // Not the video owner + ) { + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Cannot remove passwords of another user\'s video' + }) + return false + } + + const passwordCount = await VideoPasswordModel.countByVideoId(video.id) + + if (passwordCount <= 1) { + res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Cannot delete the last password of the protected video' + }) + return false + } + + return true +} + +export { + isValidVideoPasswordHeader, + checkVideoIsPasswordProtected as isVideoPasswordProtected, + doesVideoPasswordExist, + isVideoPasswordDeletable +} diff --git a/server/server/middlewares/validators/shared/video-playlists.ts b/server/server/middlewares/validators/shared/video-playlists.ts new file mode 100644 index 000000000..caa0bfaf4 --- /dev/null +++ b/server/server/middlewares/validators/shared/video-playlists.ts @@ -0,0 +1,39 @@ +import express from 'express' +import { VideoPlaylistModel } from '@server/models/video/video-playlist.js' +import { MVideoPlaylist } from '@server/types/models/index.js' +import { HttpStatusCode } from '@peertube/peertube-models' + +export type VideoPlaylistFetchType = 'summary' | 'all' +async function doesVideoPlaylistExist (id: number | string, res: express.Response, fetchType: VideoPlaylistFetchType = 'summary') { + if (fetchType === 'summary') { + const videoPlaylist = await VideoPlaylistModel.loadWithAccountAndChannelSummary(id, undefined) + res.locals.videoPlaylistSummary = videoPlaylist + + return handleVideoPlaylist(videoPlaylist, res) + } + + const videoPlaylist = await VideoPlaylistModel.loadWithAccountAndChannel(id, undefined) + res.locals.videoPlaylistFull = videoPlaylist + + return handleVideoPlaylist(videoPlaylist, res) +} + +// --------------------------------------------------------------------------- + +export { + doesVideoPlaylistExist +} + +// --------------------------------------------------------------------------- + +function handleVideoPlaylist (videoPlaylist: MVideoPlaylist, res: express.Response) { + if (!videoPlaylist) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video playlist not found' + }) + return false + } + + return true +} diff --git a/server/server/middlewares/validators/shared/videos.ts b/server/server/middlewares/validators/shared/videos.ts new file mode 100644 index 000000000..0e7dfebcb --- /dev/null +++ b/server/server/middlewares/validators/shared/videos.ts @@ -0,0 +1,311 @@ +import { Request, Response } from 'express' +import { HttpStatusCode, ServerErrorCode, UserRight, UserRightType, VideoPrivacy } from '@peertube/peertube-models' +import { exists } from '@server/helpers/custom-validators/misc.js' +import { loadVideo, VideoLoadType } from '@server/lib/model-loaders/index.js' +import { isAbleToUploadVideo } from '@server/lib/user.js' +import { VideoTokensManager } from '@server/lib/video-tokens-manager.js' +import { authenticatePromise } from '@server/middlewares/auth.js' +import { VideoChannelModel } from '@server/models/video/video-channel.js' +import { VideoFileModel } from '@server/models/video/video-file.js' +import { VideoPasswordModel } from '@server/models/video/video-password.js' +import { VideoModel } from '@server/models/video/video.js' +import { + MUser, + MUserAccountId, + MUserId, + MVideo, + MVideoAccountLight, + MVideoFormattableDetails, + MVideoFullLight, + MVideoId, + MVideoImmutable, + MVideoThumbnail, + MVideoWithRights +} from '@server/types/models/index.js' + +async function doesVideoExist (id: number | string, res: Response, fetchType: VideoLoadType = 'all') { + const userId = res.locals.oauth ? res.locals.oauth.token.User.id : undefined + + const video = await loadVideo(id, fetchType, userId) + + if (!video) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video not found' + }) + + return false + } + + switch (fetchType) { + case 'for-api': + res.locals.videoAPI = video as MVideoFormattableDetails + break + + case 'all': + res.locals.videoAll = video as MVideoFullLight + break + + case 'only-immutable-attributes': + res.locals.onlyImmutableVideo = video as MVideoImmutable + break + + case 'id': + res.locals.videoId = video as MVideoId + break + + case 'only-video': + res.locals.onlyVideo = video as MVideoThumbnail + break + } + + return true +} + +// --------------------------------------------------------------------------- + +async function doesVideoFileOfVideoExist (id: number, videoIdOrUUID: number | string, res: Response) { + if (!await VideoFileModel.doesVideoExistForVideoFile(id, videoIdOrUUID)) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'VideoFile matching Video not found' + }) + return false + } + + return true +} + +// --------------------------------------------------------------------------- + +async function doesVideoChannelOfAccountExist (channelId: number, user: MUserAccountId, res: Response) { + const videoChannel = await VideoChannelModel.loadAndPopulateAccount(channelId) + + if (videoChannel === null) { + res.fail({ message: 'Unknown video "video channel" for this instance.' }) + return false + } + + // Don't check account id if the user can update any video + if (user.hasRight(UserRight.UPDATE_ANY_VIDEO) === true) { + res.locals.videoChannel = videoChannel + return true + } + + if (videoChannel.Account.id !== user.Account.id) { + res.fail({ + message: 'Unknown video "video channel" for this account.' + }) + return false + } + + res.locals.videoChannel = videoChannel + return true +} + +// --------------------------------------------------------------------------- + +async function checkCanSeeVideo (options: { + req: Request + res: Response + paramId: string + video: MVideo +}) { + const { req, res, video, paramId } = options + + if (video.requiresUserAuth({ urlParamId: paramId, checkBlacklist: true })) { + return checkCanSeeUserAuthVideo({ req, res, video }) + } + + if (video.privacy === VideoPrivacy.PASSWORD_PROTECTED) { + return checkCanSeePasswordProtectedVideo({ req, res, video }) + } + + if (video.privacy === VideoPrivacy.UNLISTED || video.privacy === VideoPrivacy.PUBLIC) { + return true + } + + throw new Error('Unknown video privacy when checking video right ' + video.url) +} + +async function checkCanSeeUserAuthVideo (options: { + req: Request + res: Response + video: MVideoId | MVideoWithRights +}) { + const { req, res, video } = options + + const fail = () => { + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Cannot fetch information of private/internal/blocked video' + }) + + return false + } + + await authenticatePromise({ req, res }) + + const user = res.locals.oauth?.token.User + if (!user) return fail() + + const videoWithRights = await getVideoWithRights(video as MVideoWithRights) + + const privacy = videoWithRights.privacy + + if (privacy === VideoPrivacy.INTERNAL) { + // We know we have a user + return true + } + + if (videoWithRights.isBlacklisted()) { + if (canUserAccessVideo(user, videoWithRights, UserRight.MANAGE_VIDEO_BLACKLIST)) return true + + return fail() + } + + if (privacy === VideoPrivacy.PRIVATE || privacy === VideoPrivacy.UNLISTED) { + if (canUserAccessVideo(user, videoWithRights, UserRight.SEE_ALL_VIDEOS)) return true + + return fail() + } + + // Should not happen + return fail() +} + +async function checkCanSeePasswordProtectedVideo (options: { + req: Request + res: Response + video: MVideo +}) { + const { req, res, video } = options + + const videoWithRights = await getVideoWithRights(video as MVideoWithRights) + + const videoPassword = req.header('x-peertube-video-password') + + if (!exists(videoPassword)) { + const errorMessage = 'Please provide a password to access this password protected video' + const errorType = ServerErrorCode.VIDEO_REQUIRES_PASSWORD + + if (req.header('authorization')) { + await authenticatePromise({ req, res, errorMessage, errorStatus: HttpStatusCode.FORBIDDEN_403, errorType }) + const user = res.locals.oauth?.token.User + + if (canUserAccessVideo(user, videoWithRights, UserRight.SEE_ALL_VIDEOS)) return true + } + + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + type: errorType, + message: errorMessage + }) + return false + } + + if (await VideoPasswordModel.isACorrectPassword({ videoId: video.id, password: videoPassword })) return true + + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + type: ServerErrorCode.INCORRECT_VIDEO_PASSWORD, + message: 'Incorrect video password. Access to the video is denied.' + }) + + return false +} + +function canUserAccessVideo (user: MUser, video: MVideoWithRights | MVideoAccountLight, right: UserRightType) { + const isOwnedByUser = video.VideoChannel.Account.userId === user.id + + return isOwnedByUser || user.hasRight(right) +} + +async function getVideoWithRights (video: MVideoWithRights): Promise { + return video.VideoChannel?.Account?.userId + ? video + : VideoModel.loadFull(video.id) +} + +// --------------------------------------------------------------------------- + +async function checkCanAccessVideoStaticFiles (options: { + video: MVideo + req: Request + res: Response + paramId: string +}) { + const { video, req, res } = options + + if (res.locals.oauth?.token.User || exists(req.header('x-peertube-video-password'))) { + return checkCanSeeVideo(options) + } + + const videoFileToken = req.query.videoFileToken + if (videoFileToken && VideoTokensManager.Instance.hasToken({ token: videoFileToken, videoUUID: video.uuid })) { + const user = VideoTokensManager.Instance.getUserFromToken({ token: videoFileToken }) + + res.locals.videoFileToken = { user } + return true + } + + if (!video.hasPrivateStaticPath()) return true + + res.sendStatus(HttpStatusCode.FORBIDDEN_403) + return false +} + +// --------------------------------------------------------------------------- + +function checkUserCanManageVideo (user: MUser, video: MVideoAccountLight, right: UserRightType, res: Response, onlyOwned = true) { + // Retrieve the user who did the request + if (onlyOwned && video.isOwned() === false) { + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Cannot manage a video of another server.' + }) + return false + } + + // Check if the user can delete the video + // The user can delete it if he has the right + // Or if s/he is the video's account + const account = video.VideoChannel.Account + if (user.hasRight(right) === false && account.userId !== user.id) { + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Cannot manage a video of another user.' + }) + return false + } + + return true +} + +// --------------------------------------------------------------------------- + +async function checkUserQuota (user: MUserId, videoFileSize: number, res: Response) { + if (await isAbleToUploadVideo(user.id, videoFileSize) === false) { + res.fail({ + status: HttpStatusCode.PAYLOAD_TOO_LARGE_413, + message: 'The user video quota is exceeded with this video.', + type: ServerErrorCode.QUOTA_REACHED + }) + return false + } + + return true +} + +// --------------------------------------------------------------------------- + +export { + doesVideoChannelOfAccountExist, + doesVideoExist, + doesVideoFileOfVideoExist, + + checkCanAccessVideoStaticFiles, + checkUserCanManageVideo, + checkCanSeeVideo, + checkUserQuota +} diff --git a/server/server/middlewares/validators/sort.ts b/server/server/middlewares/validators/sort.ts new file mode 100644 index 000000000..3f3b2eb4e --- /dev/null +++ b/server/server/middlewares/validators/sort.ts @@ -0,0 +1,66 @@ +import express from 'express' +import { query } from 'express-validator' +import { SORTABLE_COLUMNS } from '../../initializers/constants.js' +import { areValidationErrors } from './shared/index.js' + +export const adminUsersSortValidator = checkSortFactory(SORTABLE_COLUMNS.ADMIN_USERS) +export const accountsSortValidator = checkSortFactory(SORTABLE_COLUMNS.ACCOUNTS) +export const jobsSortValidator = checkSortFactory(SORTABLE_COLUMNS.JOBS, [ 'jobs' ]) +export const abusesSortValidator = checkSortFactory(SORTABLE_COLUMNS.ABUSES) +export const videosSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEOS) +export const videoImportsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_IMPORTS) +export const videosSearchSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEOS_SEARCH) +export const videoChannelsSearchSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_CHANNELS_SEARCH) +export const videoPlaylistsSearchSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_PLAYLISTS_SEARCH) +export const videoCommentsValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_COMMENTS) +export const videoCommentThreadsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_COMMENT_THREADS) +export const videoRatesSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_RATES) +export const blacklistSortValidator = checkSortFactory(SORTABLE_COLUMNS.BLACKLISTS) +export const videoChannelsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_CHANNELS) +export const instanceFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.INSTANCE_FOLLOWERS) +export const instanceFollowingSortValidator = checkSortFactory(SORTABLE_COLUMNS.INSTANCE_FOLLOWING) +export const userSubscriptionsSortValidator = checkSortFactory(SORTABLE_COLUMNS.USER_SUBSCRIPTIONS) +export const accountsBlocklistSortValidator = checkSortFactory(SORTABLE_COLUMNS.ACCOUNTS_BLOCKLIST) +export const serversBlocklistSortValidator = checkSortFactory(SORTABLE_COLUMNS.SERVERS_BLOCKLIST) +export const userNotificationsSortValidator = checkSortFactory(SORTABLE_COLUMNS.USER_NOTIFICATIONS) +export const videoPlaylistsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_PLAYLISTS) +export const pluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.PLUGINS) +export const availablePluginsSortValidator = checkSortFactory(SORTABLE_COLUMNS.AVAILABLE_PLUGINS) +export const videoRedundanciesSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_REDUNDANCIES) +export const videoChannelSyncsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_CHANNEL_SYNCS) +export const videoPasswordsSortValidator = checkSortFactory(SORTABLE_COLUMNS.VIDEO_PASSWORDS) + +export const accountsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.ACCOUNT_FOLLOWERS) +export const videoChannelsFollowersSortValidator = checkSortFactory(SORTABLE_COLUMNS.CHANNEL_FOLLOWERS) + +export const userRegistrationsSortValidator = checkSortFactory(SORTABLE_COLUMNS.USER_REGISTRATIONS) + +export const runnersSortValidator = checkSortFactory(SORTABLE_COLUMNS.RUNNERS) +export const runnerRegistrationTokensSortValidator = checkSortFactory(SORTABLE_COLUMNS.RUNNER_REGISTRATION_TOKENS) +export const runnerJobsSortValidator = checkSortFactory(SORTABLE_COLUMNS.RUNNER_JOBS) + +// --------------------------------------------------------------------------- + +function checkSortFactory (columns: string[], tags: string[] = []) { + return checkSort(createSortableColumns(columns), tags) +} + +function checkSort (sortableColumns: string[], tags: string[] = []) { + return [ + query('sort') + .optional() + .isIn(sortableColumns), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { tags })) return + + return next() + } + ] +} + +function createSortableColumns (sortableColumns: string[]) { + const sortableColumnDesc = sortableColumns.map(sortableColumn => '-' + sortableColumn) + + return sortableColumns.concat(sortableColumnDesc) +} diff --git a/server/server/middlewares/validators/static.ts b/server/server/middlewares/validators/static.ts new file mode 100644 index 000000000..afd1441ab --- /dev/null +++ b/server/server/middlewares/validators/static.ts @@ -0,0 +1,184 @@ +import express from 'express' +import { query } from 'express-validator' +import { LRUCache } from 'lru-cache' +import { basename, dirname } from 'path' +import { exists, isSafePeerTubeFilenameWithoutExtension, isUUIDValid, toBooleanOrNull } from '@server/helpers/custom-validators/misc.js' +import { logger } from '@server/helpers/logger.js' +import { LRU_CACHE } from '@server/initializers/constants.js' +import { VideoModel } from '@server/models/video/video.js' +import { VideoFileModel } from '@server/models/video/video-file.js' +import { MStreamingPlaylist, MVideoFile, MVideoThumbnail } from '@server/types/models/index.js' +import { HttpStatusCode } from '@peertube/peertube-models' +import { areValidationErrors, checkCanAccessVideoStaticFiles, isValidVideoPasswordHeader } from './shared/index.js' + +type LRUValue = { + allowed: boolean + video?: MVideoThumbnail + file?: MVideoFile + playlist?: MStreamingPlaylist } + +const staticFileTokenBypass = new LRUCache({ + max: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.MAX_SIZE, + ttl: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.TTL +}) + +const ensureCanAccessVideoPrivateWebVideoFiles = [ + query('videoFileToken').optional().custom(exists), + + isValidVideoPasswordHeader(), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const token = extractTokenOrDie(req, res) + if (!token) return + + const cacheKey = token + '-' + req.originalUrl + + if (staticFileTokenBypass.has(cacheKey)) { + const { allowed, file, video } = staticFileTokenBypass.get(cacheKey) + + if (allowed === true) { + res.locals.onlyVideo = video + res.locals.videoFile = file + + return next() + } + + return res.sendStatus(HttpStatusCode.FORBIDDEN_403) + } + + const result = await isWebVideoAllowed(req, res) + + staticFileTokenBypass.set(cacheKey, result) + + if (result.allowed !== true) return + + res.locals.onlyVideo = result.video + res.locals.videoFile = result.file + + return next() + } +] + +const ensureCanAccessPrivateVideoHLSFiles = [ + query('videoFileToken') + .optional() + .custom(exists), + + query('reinjectVideoFileToken') + .optional() + .customSanitizer(toBooleanOrNull) + .isBoolean().withMessage('Should be a valid reinjectVideoFileToken boolean'), + + query('playlistName') + .optional() + .customSanitizer(isSafePeerTubeFilenameWithoutExtension), + + isValidVideoPasswordHeader(), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const videoUUID = basename(dirname(req.originalUrl)) + + if (!isUUIDValid(videoUUID)) { + logger.debug('Path does not contain valid video UUID to serve static file %s', req.originalUrl) + + return res.sendStatus(HttpStatusCode.FORBIDDEN_403) + } + + const token = extractTokenOrDie(req, res) + if (!token) return + + const cacheKey = token + '-' + videoUUID + + if (staticFileTokenBypass.has(cacheKey)) { + const { allowed, file, playlist, video } = staticFileTokenBypass.get(cacheKey) + + if (allowed === true) { + res.locals.onlyVideo = video + res.locals.videoFile = file + res.locals.videoStreamingPlaylist = playlist + + return next() + } + + return res.sendStatus(HttpStatusCode.FORBIDDEN_403) + } + + const result = await isHLSAllowed(req, res, videoUUID) + + staticFileTokenBypass.set(cacheKey, result) + + if (result.allowed !== true) return + + res.locals.onlyVideo = result.video + res.locals.videoFile = result.file + res.locals.videoStreamingPlaylist = result.playlist + + return next() + } +] + +export { + ensureCanAccessVideoPrivateWebVideoFiles, + ensureCanAccessPrivateVideoHLSFiles +} + +// --------------------------------------------------------------------------- + +async function isWebVideoAllowed (req: express.Request, res: express.Response) { + const filename = basename(req.path) + + const file = await VideoFileModel.loadWithVideoByFilename(filename) + if (!file) { + logger.debug('Unknown static file %s to serve', req.originalUrl, { filename }) + + res.sendStatus(HttpStatusCode.FORBIDDEN_403) + return { allowed: false } + } + + const video = await VideoModel.load(file.getVideo().id) + + return { + file, + video, + allowed: await checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid }) + } +} + +async function isHLSAllowed (req: express.Request, res: express.Response, videoUUID: string) { + const filename = basename(req.path) + + const video = await VideoModel.loadWithFiles(videoUUID) + + if (!video) { + logger.debug('Unknown static file %s to serve', req.originalUrl, { videoUUID }) + + res.sendStatus(HttpStatusCode.FORBIDDEN_403) + return { allowed: false } + } + + const file = await VideoFileModel.loadByFilename(filename) + + return { + file, + video, + playlist: video.getHLSPlaylist(), + allowed: await checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid }) + } +} + +function extractTokenOrDie (req: express.Request, res: express.Response) { + const token = req.header('x-peertube-video-password') || req.query.videoFileToken || res.locals.oauth?.token.accessToken + + if (!token) { + return res.fail({ + message: 'Video password header, video file token query parameter and bearer token are all missing', // + status: HttpStatusCode.FORBIDDEN_403 + }) + } + + return token +} diff --git a/server/server/middlewares/validators/themes.ts b/server/server/middlewares/validators/themes.ts new file mode 100644 index 000000000..ec7ce3b4f --- /dev/null +++ b/server/server/middlewares/validators/themes.ts @@ -0,0 +1,46 @@ +import express from 'express' +import { param } from 'express-validator' +import { HttpStatusCode } from '@peertube/peertube-models' +import { isSafePath } from '../../helpers/custom-validators/misc.js' +import { isPluginNameValid, isPluginStableOrUnstableVersionValid } from '../../helpers/custom-validators/plugins.js' +import { PluginManager } from '../../lib/plugins/plugin-manager.js' +import { areValidationErrors } from './shared/index.js' + +const serveThemeCSSValidator = [ + param('themeName') + .custom(isPluginNameValid), + param('themeVersion') + .custom(isPluginStableOrUnstableVersionValid), + param('staticEndpoint') + .custom(isSafePath), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const theme = PluginManager.Instance.getRegisteredThemeByShortName(req.params.themeName) + + if (!theme || theme.version !== req.params.themeVersion) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'No theme named ' + req.params.themeName + ' was found with version ' + req.params.themeVersion + }) + } + + if (theme.css.includes(req.params.staticEndpoint) === false) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'No static endpoint was found for this theme' + }) + } + + res.locals.registeredPlugin = theme + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + serveThemeCSSValidator +} diff --git a/server/server/middlewares/validators/two-factor.ts b/server/server/middlewares/validators/two-factor.ts new file mode 100644 index 000000000..950f72528 --- /dev/null +++ b/server/server/middlewares/validators/two-factor.ts @@ -0,0 +1,81 @@ +import express from 'express' +import { body, param } from 'express-validator' +import { HttpStatusCode, UserRight } from '@peertube/peertube-models' +import { exists, isIdValid } from '../../helpers/custom-validators/misc.js' +import { areValidationErrors, checkUserIdExist } from './shared/index.js' + +const requestOrConfirmTwoFactorValidator = [ + param('id').custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (!await checkCanEnableOrDisableTwoFactor(req.params.id, res)) return + + if (res.locals.user.otpSecret) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: `Two factor is already enabled.` + }) + } + + return next() + } +] + +const confirmTwoFactorValidator = [ + body('requestToken').custom(exists), + body('otpToken').custom(exists), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const disableTwoFactorValidator = [ + param('id').custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (!await checkCanEnableOrDisableTwoFactor(req.params.id, res)) return + + if (!res.locals.user.otpSecret) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: `Two factor is already disabled.` + }) + } + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + requestOrConfirmTwoFactorValidator, + confirmTwoFactorValidator, + disableTwoFactorValidator +} + +// --------------------------------------------------------------------------- + +async function checkCanEnableOrDisableTwoFactor (userId: number | string, res: express.Response) { + const authUser = res.locals.oauth.token.user + + if (!await checkUserIdExist(userId, res)) return + + if (res.locals.user.id !== authUser.id && authUser.hasRight(UserRight.MANAGE_USERS) !== true) { + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: `User ${authUser.username} does not have right to change two factor setting of this user.` + }) + + return false + } + + return true +} diff --git a/server/server/middlewares/validators/user-email-verification.ts b/server/server/middlewares/validators/user-email-verification.ts new file mode 100644 index 000000000..50151f642 --- /dev/null +++ b/server/server/middlewares/validators/user-email-verification.ts @@ -0,0 +1,94 @@ +import express from 'express' +import { body, param } from 'express-validator' +import { toBooleanOrNull } from '@server/helpers/custom-validators/misc.js' +import { HttpStatusCode } from '@peertube/peertube-models' +import { logger } from '../../helpers/logger.js' +import { Redis } from '../../lib/redis.js' +import { areValidationErrors, checkUserEmailExist, checkUserIdExist } from './shared/index.js' +import { checkRegistrationEmailExist, checkRegistrationIdExist } from './shared/user-registrations.js' + +const usersAskSendVerifyEmailValidator = [ + body('email').isEmail().not().isEmpty().withMessage('Should have a valid email'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const [ userExists, registrationExists ] = await Promise.all([ + checkUserEmailExist(req.body.email, res, false), + checkRegistrationEmailExist(req.body.email, res, false) + ]) + + if (!userExists && !registrationExists) { + logger.debug('User or registration with email %s does not exist (asking verify email).', req.body.email) + // Do not leak our emails + return res.status(HttpStatusCode.NO_CONTENT_204).end() + } + + if (res.locals.user?.pluginAuth) { + return res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'Cannot ask verification email of a user that uses a plugin authentication.' + }) + } + + return next() + } +] + +const usersVerifyEmailValidator = [ + param('id') + .isInt().not().isEmpty().withMessage('Should have a valid id'), + + body('verificationString') + .not().isEmpty().withMessage('Should have a valid verification string'), + body('isPendingEmail') + .optional() + .customSanitizer(toBooleanOrNull), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await checkUserIdExist(req.params.id, res)) return + + const user = res.locals.user + const redisVerificationString = await Redis.Instance.getUserVerifyEmailLink(user.id) + + if (redisVerificationString !== req.body.verificationString) { + return res.fail({ status: HttpStatusCode.FORBIDDEN_403, message: 'Invalid verification string.' }) + } + + return next() + } +] + +// --------------------------------------------------------------------------- + +const registrationVerifyEmailValidator = [ + param('registrationId') + .isInt().not().isEmpty().withMessage('Should have a valid registrationId'), + + body('verificationString') + .not().isEmpty().withMessage('Should have a valid verification string'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await checkRegistrationIdExist(req.params.registrationId, res)) return + + const registration = res.locals.userRegistration + const redisVerificationString = await Redis.Instance.getRegistrationVerifyEmailLink(registration.id) + + if (redisVerificationString !== req.body.verificationString) { + return res.fail({ status: HttpStatusCode.FORBIDDEN_403, message: 'Invalid verification string.' }) + } + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + usersAskSendVerifyEmailValidator, + usersVerifyEmailValidator, + + registrationVerifyEmailValidator +} diff --git a/server/server/middlewares/validators/user-history.ts b/server/server/middlewares/validators/user-history.ts new file mode 100644 index 000000000..8415dab2a --- /dev/null +++ b/server/server/middlewares/validators/user-history.ts @@ -0,0 +1,47 @@ +import express from 'express' +import { body, param, query } from 'express-validator' +import { exists, isDateValid, isIdValid } from '../../helpers/custom-validators/misc.js' +import { areValidationErrors } from './shared/index.js' + +const userHistoryListValidator = [ + query('search') + .optional() + .custom(exists), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const userHistoryRemoveAllValidator = [ + body('beforeDate') + .optional() + .custom(isDateValid).withMessage('Should have a before date that conforms to ISO 8601'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const userHistoryRemoveElementValidator = [ + param('videoId') + .custom(isIdValid), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + userHistoryListValidator, + userHistoryRemoveElementValidator, + userHistoryRemoveAllValidator +} diff --git a/server/server/middlewares/validators/user-notifications.ts b/server/server/middlewares/validators/user-notifications.ts new file mode 100644 index 000000000..16bbc6693 --- /dev/null +++ b/server/server/middlewares/validators/user-notifications.ts @@ -0,0 +1,71 @@ +import express from 'express' +import { body, query } from 'express-validator' +import { isNotEmptyIntArray, toBooleanOrNull } from '../../helpers/custom-validators/misc.js' +import { isUserNotificationSettingValid } from '../../helpers/custom-validators/user-notifications.js' +import { areValidationErrors } from './shared/index.js' + +const listUserNotificationsValidator = [ + query('unread') + .optional() + .customSanitizer(toBooleanOrNull) + .isBoolean().withMessage('Should have a valid unread boolean'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const updateNotificationSettingsValidator = [ + body('newVideoFromSubscription') + .custom(isUserNotificationSettingValid), + body('newCommentOnMyVideo') + .custom(isUserNotificationSettingValid), + body('abuseAsModerator') + .custom(isUserNotificationSettingValid), + body('videoAutoBlacklistAsModerator') + .custom(isUserNotificationSettingValid), + body('blacklistOnMyVideo') + .custom(isUserNotificationSettingValid), + body('myVideoImportFinished') + .custom(isUserNotificationSettingValid), + body('myVideoPublished') + .custom(isUserNotificationSettingValid), + body('commentMention') + .custom(isUserNotificationSettingValid), + body('newFollow') + .custom(isUserNotificationSettingValid), + body('newUserRegistration') + .custom(isUserNotificationSettingValid), + body('newInstanceFollower') + .custom(isUserNotificationSettingValid), + body('autoInstanceFollowing') + .custom(isUserNotificationSettingValid), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const markAsReadUserNotificationsValidator = [ + body('ids') + .optional() + .custom(isNotEmptyIntArray).withMessage('Should have a valid array of notification ids'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + listUserNotificationsValidator, + updateNotificationSettingsValidator, + markAsReadUserNotificationsValidator +} diff --git a/server/server/middlewares/validators/user-registrations.ts b/server/server/middlewares/validators/user-registrations.ts new file mode 100644 index 000000000..392fe52a6 --- /dev/null +++ b/server/server/middlewares/validators/user-registrations.ts @@ -0,0 +1,208 @@ +import express from 'express' +import { body, param, query, ValidationChain } from 'express-validator' +import { exists, isBooleanValid, isIdValid, toBooleanOrNull } from '@server/helpers/custom-validators/misc.js' +import { isRegistrationModerationResponseValid, isRegistrationReasonValid } from '@server/helpers/custom-validators/user-registration.js' +import { CONFIG } from '@server/initializers/config.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { HttpStatusCode, UserRegister, UserRegistrationRequest, UserRegistrationState } from '@peertube/peertube-models' +import { isUserDisplayNameValid, isUserPasswordValid, isUserUsernameValid } from '../../helpers/custom-validators/users.js' +import { isVideoChannelDisplayNameValid, isVideoChannelUsernameValid } from '../../helpers/custom-validators/video-channels.js' +import { isSignupAllowed, isSignupAllowedForCurrentIP, SignupMode } from '../../lib/signup.js' +import { ActorModel } from '../../models/actor/actor.js' +import { areValidationErrors, checkUserNameOrEmailDoNotAlreadyExist } from './shared/index.js' +import { checkRegistrationHandlesDoNotAlreadyExist, checkRegistrationIdExist } from './shared/user-registrations.js' + +const usersDirectRegistrationValidator = usersCommonRegistrationValidatorFactory() + +const usersRequestRegistrationValidator = [ + ...usersCommonRegistrationValidatorFactory([ + body('registrationReason') + .custom(isRegistrationReasonValid) + ]), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + const body: UserRegistrationRequest = req.body + + if (CONFIG.SIGNUP.REQUIRES_APPROVAL !== true) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Signup approval is not enabled on this instance' + }) + } + + const options = { username: body.username, email: body.email, channelHandle: body.channel?.name, res } + if (!await checkRegistrationHandlesDoNotAlreadyExist(options)) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +function ensureUserRegistrationAllowedFactory (signupMode: SignupMode) { + return async (req: express.Request, res: express.Response, next: express.NextFunction) => { + const allowedParams = { + body: req.body, + ip: req.ip, + signupMode + } + + const allowedResult = await Hooks.wrapPromiseFun( + isSignupAllowed, + allowedParams, + + signupMode === 'direct-registration' + ? 'filter:api.user.signup.allowed.result' + : 'filter:api.user.request-signup.allowed.result' + ) + + if (allowedResult.allowed === false) { + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: allowedResult.errorMessage || 'User registration is not allowed' + }) + } + + return next() + } +} + +const ensureUserRegistrationAllowedForIP = [ + (req: express.Request, res: express.Response, next: express.NextFunction) => { + const allowed = isSignupAllowedForCurrentIP(req.ip) + + if (allowed === false) { + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'You are not on a network authorized for registration.' + }) + } + + return next() + } +] + +// --------------------------------------------------------------------------- + +const acceptOrRejectRegistrationValidator = [ + param('registrationId') + .custom(isIdValid), + + body('moderationResponse') + .custom(isRegistrationModerationResponseValid), + + body('preventEmailDelivery') + .optional() + .customSanitizer(toBooleanOrNull) + .custom(isBooleanValid).withMessage('Should have preventEmailDelivery boolean'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await checkRegistrationIdExist(req.params.registrationId, res)) return + + if (res.locals.userRegistration.state !== UserRegistrationState.PENDING) { + return res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'This registration is already accepted or rejected.' + }) + } + + return next() + } +] + +// --------------------------------------------------------------------------- + +const getRegistrationValidator = [ + param('registrationId') + .custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await checkRegistrationIdExist(req.params.registrationId, res)) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +const listRegistrationsValidator = [ + query('search') + .optional() + .custom(exists), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + usersDirectRegistrationValidator, + usersRequestRegistrationValidator, + + ensureUserRegistrationAllowedFactory, + ensureUserRegistrationAllowedForIP, + + getRegistrationValidator, + listRegistrationsValidator, + + acceptOrRejectRegistrationValidator +} + +// --------------------------------------------------------------------------- + +function usersCommonRegistrationValidatorFactory (additionalValidationChain: ValidationChain[] = []) { + return [ + body('username') + .custom(isUserUsernameValid), + body('password') + .custom(isUserPasswordValid), + body('email') + .isEmail(), + body('displayName') + .optional() + .custom(isUserDisplayNameValid), + + body('channel.name') + .optional() + .custom(isVideoChannelUsernameValid), + body('channel.displayName') + .optional() + .custom(isVideoChannelDisplayNameValid), + + ...additionalValidationChain, + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { omitBodyLog: true })) return + + const body: UserRegister | UserRegistrationRequest = req.body + + if (!await checkUserNameOrEmailDoNotAlreadyExist(body.username, body.email, res)) return + + if (body.channel) { + if (!body.channel.name || !body.channel.displayName) { + return res.fail({ message: 'Channel is optional but if you specify it, channel.name and channel.displayName are required.' }) + } + + if (body.channel.name === body.username) { + return res.fail({ message: 'Channel name cannot be the same as user username.' }) + } + + const existing = await ActorModel.loadLocalByName(body.channel.name) + if (existing) { + return res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: `Channel with name ${body.channel.name} already exists.` + }) + } + } + + return next() + } + ] +} diff --git a/server/server/middlewares/validators/user-subscriptions.ts b/server/server/middlewares/validators/user-subscriptions.ts new file mode 100644 index 000000000..78bea840f --- /dev/null +++ b/server/server/middlewares/validators/user-subscriptions.ts @@ -0,0 +1,110 @@ +import express from 'express' +import { body, param, query } from 'express-validator' +import { arrayify } from '@peertube/peertube-core-utils' +import { FollowState, HttpStatusCode } from '@peertube/peertube-models' +import { areValidActorHandles, isValidActorHandle } from '../../helpers/custom-validators/activitypub/actor.js' +import { WEBSERVER } from '../../initializers/constants.js' +import { ActorFollowModel } from '../../models/actor/actor-follow.js' +import { areValidationErrors } from './shared/index.js' + +const userSubscriptionListValidator = [ + query('search') + .optional() + .not().isEmpty(), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const userSubscriptionAddValidator = [ + body('uri') + .custom(isValidActorHandle).withMessage('Should have a valid URI to follow (username@domain)'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const areSubscriptionsExistValidator = [ + query('uris') + .customSanitizer(arrayify) + .custom(areValidActorHandles).withMessage('Should have a valid array of URIs'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const userSubscriptionGetValidator = [ + param('uri') + .custom(isValidActorHandle), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesSubscriptionExist({ uri: req.params.uri, res, state: 'accepted' })) return + + return next() + } +] + +const userSubscriptionDeleteValidator = [ + param('uri') + .custom(isValidActorHandle), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesSubscriptionExist({ uri: req.params.uri, res })) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + areSubscriptionsExistValidator, + userSubscriptionListValidator, + userSubscriptionAddValidator, + userSubscriptionGetValidator, + userSubscriptionDeleteValidator +} + +// --------------------------------------------------------------------------- + +async function doesSubscriptionExist (options: { + uri: string + res: express.Response + state?: FollowState +}) { + const { uri, res, state } = options + + let [ name, host ] = uri.split('@') + if (host === WEBSERVER.HOST) host = null + + const user = res.locals.oauth.token.User + const subscription = await ActorFollowModel.loadByActorAndTargetNameAndHostForAPI({ + actorId: user.Account.Actor.id, + targetName: name, + targetHost: host, + state + }) + + if (!subscription?.ActorFollowing.VideoChannel) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: `Subscription ${uri} not found.` + }) + return false + } + + res.locals.subscription = subscription + + return true +} diff --git a/server/server/middlewares/validators/users.ts b/server/server/middlewares/validators/users.ts new file mode 100644 index 000000000..010ae496d --- /dev/null +++ b/server/server/middlewares/validators/users.ts @@ -0,0 +1,489 @@ +import express from 'express' +import { body, param, query } from 'express-validator' +import { forceNumber } from '@peertube/peertube-core-utils' +import { HttpStatusCode, UserRight, UserRole } from '@peertube/peertube-models' +import { exists, isBooleanValid, isIdValid, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc.js' +import { isThemeNameValid } from '../../helpers/custom-validators/plugins.js' +import { + isUserAdminFlagsValid, + isUserAutoPlayNextVideoValid, + isUserAutoPlayVideoValid, + isUserBlockedReasonValid, + isUserDescriptionValid, + isUserDisplayNameValid, + isUserEmailPublicValid, + isUserNoModal, + isUserNSFWPolicyValid, + isUserP2PEnabledValid, + isUserPasswordValid, + isUserPasswordValidOrEmpty, + isUserRoleValid, + isUserUsernameValid, + isUserVideoLanguages, + isUserVideoQuotaDailyValid, + isUserVideoQuotaValid, + isUserVideosHistoryEnabledValid +} from '../../helpers/custom-validators/users.js' +import { isVideoChannelUsernameValid } from '../../helpers/custom-validators/video-channels.js' +import { logger } from '../../helpers/logger.js' +import { isThemeRegistered } from '../../lib/plugins/theme-utils.js' +import { Redis } from '../../lib/redis.js' +import { ActorModel } from '../../models/actor/actor.js' +import { + areValidationErrors, + checkUserEmailExist, + checkUserIdExist, + checkUserNameOrEmailDoNotAlreadyExist, + doesVideoChannelIdExist, + doesVideoExist, + isValidVideoIdParam +} from './shared/index.js' + +const usersListValidator = [ + query('blocked') + .optional() + .customSanitizer(toBooleanOrNull) + .isBoolean().withMessage('Should be a valid blocked boolean'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const usersAddValidator = [ + body('username') + .custom(isUserUsernameValid) + .withMessage('Should have a valid username (lowercase alphanumeric characters)'), + body('password') + .custom(isUserPasswordValidOrEmpty), + body('email') + .isEmail(), + + body('channelName') + .optional() + .custom(isVideoChannelUsernameValid), + + body('videoQuota') + .optional() + .custom(isUserVideoQuotaValid), + + body('videoQuotaDaily') + .optional() + .custom(isUserVideoQuotaDailyValid), + + body('role') + .customSanitizer(toIntOrNull) + .custom(isUserRoleValid), + + body('adminFlags') + .optional() + .custom(isUserAdminFlagsValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { omitBodyLog: true })) return + if (!await checkUserNameOrEmailDoNotAlreadyExist(req.body.username, req.body.email, res)) return + + const authUser = res.locals.oauth.token.User + if (authUser.role !== UserRole.ADMINISTRATOR && req.body.role !== UserRole.USER) { + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'You can only create users (and not administrators or moderators)' + }) + } + + if (req.body.channelName) { + if (req.body.channelName === req.body.username) { + return res.fail({ message: 'Channel name cannot be the same as user username.' }) + } + + const existing = await ActorModel.loadLocalByName(req.body.channelName) + if (existing) { + return res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: `Channel with name ${req.body.channelName} already exists.` + }) + } + } + + return next() + } +] + +const usersRemoveValidator = [ + param('id') + .custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await checkUserIdExist(req.params.id, res)) return + + const user = res.locals.user + if (user.username === 'root') { + return res.fail({ message: 'Cannot remove the root user' }) + } + + return next() + } +] + +const usersBlockingValidator = [ + param('id') + .custom(isIdValid), + body('reason') + .optional() + .custom(isUserBlockedReasonValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await checkUserIdExist(req.params.id, res)) return + + const user = res.locals.user + if (user.username === 'root') { + return res.fail({ message: 'Cannot block the root user' }) + } + + return next() + } +] + +const deleteMeValidator = [ + (req: express.Request, res: express.Response, next: express.NextFunction) => { + const user = res.locals.oauth.token.User + if (user.username === 'root') { + return res.fail({ message: 'You cannot delete your root account.' }) + } + + return next() + } +] + +const usersUpdateValidator = [ + param('id').custom(isIdValid), + + body('password') + .optional() + .custom(isUserPasswordValid), + body('email') + .optional() + .isEmail(), + body('emailVerified') + .optional() + .isBoolean(), + body('videoQuota') + .optional() + .custom(isUserVideoQuotaValid), + body('videoQuotaDaily') + .optional() + .custom(isUserVideoQuotaDailyValid), + body('pluginAuth') + .optional() + .exists(), + body('role') + .optional() + .customSanitizer(toIntOrNull) + .custom(isUserRoleValid), + body('adminFlags') + .optional() + .custom(isUserAdminFlagsValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { omitBodyLog: true })) return + if (!await checkUserIdExist(req.params.id, res)) return + + const user = res.locals.user + if (user.username === 'root' && req.body.role !== undefined && user.role !== req.body.role) { + return res.fail({ message: 'Cannot change root role.' }) + } + + return next() + } +] + +const usersUpdateMeValidator = [ + body('displayName') + .optional() + .custom(isUserDisplayNameValid), + body('description') + .optional() + .custom(isUserDescriptionValid), + body('currentPassword') + .optional() + .custom(isUserPasswordValid), + body('password') + .optional() + .custom(isUserPasswordValid), + body('emailPublic') + .optional() + .custom(isUserEmailPublicValid), + body('email') + .optional() + .isEmail(), + body('nsfwPolicy') + .optional() + .custom(isUserNSFWPolicyValid), + body('autoPlayVideo') + .optional() + .custom(isUserAutoPlayVideoValid), + body('p2pEnabled') + .optional() + .custom(isUserP2PEnabledValid).withMessage('Should have a valid p2p enabled boolean'), + body('videoLanguages') + .optional() + .custom(isUserVideoLanguages), + body('videosHistoryEnabled') + .optional() + .custom(isUserVideosHistoryEnabledValid).withMessage('Should have a valid videos history enabled boolean'), + body('theme') + .optional() + .custom(v => isThemeNameValid(v) && isThemeRegistered(v)), + + body('noInstanceConfigWarningModal') + .optional() + .custom(v => isUserNoModal(v)).withMessage('Should have a valid noInstanceConfigWarningModal boolean'), + body('noWelcomeModal') + .optional() + .custom(v => isUserNoModal(v)).withMessage('Should have a valid noWelcomeModal boolean'), + body('noAccountSetupWarningModal') + .optional() + .custom(v => isUserNoModal(v)).withMessage('Should have a valid noAccountSetupWarningModal boolean'), + + body('autoPlayNextVideo') + .optional() + .custom(v => isUserAutoPlayNextVideoValid(v)).withMessage('Should have a valid autoPlayNextVideo boolean'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + const user = res.locals.oauth.token.User + + if (req.body.password || req.body.email) { + if (user.pluginAuth !== null) { + return res.fail({ message: 'You cannot update your email or password that is associated with an external auth system.' }) + } + + if (!req.body.currentPassword) { + return res.fail({ message: 'currentPassword parameter is missing.' }) + } + + if (await user.isPasswordMatch(req.body.currentPassword) !== true) { + return res.fail({ + status: HttpStatusCode.UNAUTHORIZED_401, + message: 'currentPassword is invalid.' + }) + } + } + + if (areValidationErrors(req, res, { omitBodyLog: true })) return + + return next() + } +] + +const usersGetValidator = [ + param('id') + .custom(isIdValid), + query('withStats') + .optional() + .isBoolean().withMessage('Should have a valid withStats boolean'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await checkUserIdExist(req.params.id, res, req.query.withStats)) return + + return next() + } +] + +const usersVideoRatingValidator = [ + isValidVideoIdParam('videoId'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.videoId, res, 'id')) return + + return next() + } +] + +const usersVideosValidator = [ + query('isLive') + .optional() + .customSanitizer(toBooleanOrNull) + .custom(isBooleanValid).withMessage('Should have a valid isLive boolean'), + + query('channelId') + .optional() + .customSanitizer(toIntOrNull) + .custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (req.query.channelId && !await doesVideoChannelIdExist(req.query.channelId, res)) return + + return next() + } +] + +const usersAskResetPasswordValidator = [ + body('email') + .isEmail(), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const exists = await checkUserEmailExist(req.body.email, res, false) + if (!exists) { + logger.debug('User with email %s does not exist (asking reset password).', req.body.email) + // Do not leak our emails + return res.status(HttpStatusCode.NO_CONTENT_204).end() + } + + if (res.locals.user.pluginAuth) { + return res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'Cannot recover password of a user that uses a plugin authentication.' + }) + } + + return next() + } +] + +const usersResetPasswordValidator = [ + param('id') + .custom(isIdValid), + body('verificationString') + .not().isEmpty(), + body('password') + .custom(isUserPasswordValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await checkUserIdExist(req.params.id, res)) return + + const user = res.locals.user + const redisVerificationString = await Redis.Instance.getResetPasswordVerificationString(user.id) + + if (redisVerificationString !== req.body.verificationString) { + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Invalid verification string.' + }) + } + + return next() + } +] + +const usersCheckCurrentPasswordFactory = (targetUserIdGetter: (req: express.Request) => number | string) => { + return [ + body('currentPassword').optional().custom(exists), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const user = res.locals.oauth.token.User + const isAdminOrModerator = user.role === UserRole.ADMINISTRATOR || user.role === UserRole.MODERATOR + const targetUserId = forceNumber(targetUserIdGetter(req)) + + // Admin/moderator action on another user, skip the password check + if (isAdminOrModerator && targetUserId !== user.id) { + return next() + } + + if (!req.body.currentPassword) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'currentPassword is missing' + }) + } + + if (await user.isPasswordMatch(req.body.currentPassword) !== true) { + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'currentPassword is invalid.' + }) + } + + return next() + } + ] +} + +const userAutocompleteValidator = [ + param('search') + .isString() + .not().isEmpty() +] + +const ensureAuthUserOwnsAccountValidator = [ + (req: express.Request, res: express.Response, next: express.NextFunction) => { + const user = res.locals.oauth.token.User + + if (res.locals.account.id !== user.Account.id) { + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Only owner of this account can access this resource.' + }) + } + + return next() + } +] + +const ensureCanManageChannelOrAccount = [ + (req: express.Request, res: express.Response, next: express.NextFunction) => { + const user = res.locals.oauth.token.user + const account = res.locals.videoChannel?.Account ?? res.locals.account + const isUserOwner = account.userId === user.id + + if (!isUserOwner && user.hasRight(UserRight.MANAGE_ANY_VIDEO_CHANNEL) === false) { + const message = `User ${user.username} does not have right this channel or account.` + + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message + }) + } + + return next() + } +] + +const ensureCanModerateUser = [ + (req: express.Request, res: express.Response, next: express.NextFunction) => { + const authUser = res.locals.oauth.token.User + const onUser = res.locals.user + + if (authUser.role === UserRole.ADMINISTRATOR) return next() + if (authUser.role === UserRole.MODERATOR && onUser.role === UserRole.USER) return next() + + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'A moderator can only manage users.' + }) + } +] + +// --------------------------------------------------------------------------- + +export { + usersListValidator, + usersAddValidator, + deleteMeValidator, + usersBlockingValidator, + usersRemoveValidator, + usersUpdateValidator, + usersUpdateMeValidator, + usersVideoRatingValidator, + usersCheckCurrentPasswordFactory, + usersGetValidator, + usersVideosValidator, + usersAskResetPasswordValidator, + usersResetPasswordValidator, + userAutocompleteValidator, + ensureAuthUserOwnsAccountValidator, + ensureCanModerateUser, + ensureCanManageChannelOrAccount +} diff --git a/server/server/middlewares/validators/videos/index.ts b/server/server/middlewares/validators/videos/index.ts new file mode 100644 index 000000000..05c6659ae --- /dev/null +++ b/server/server/middlewares/validators/videos/index.ts @@ -0,0 +1,19 @@ +export * from './video-blacklist.js' +export * from './video-captions.js' +export * from './video-channel-sync.js' +export * from './video-channels.js' +export * from './video-comments.js' +export * from './video-files.js' +export * from './video-imports.js' +export * from './video-live.js' +export * from './video-ownership-changes.js' +export * from './video-passwords.js' +export * from './video-rates.js' +export * from './video-shares.js' +export * from './video-source.js' +export * from './video-stats.js' +export * from './video-studio.js' +export * from './video-token.js' +export * from './video-transcoding.js' +export * from './video-view.js' +export * from './videos.js' diff --git a/server/server/middlewares/validators/videos/shared/index.ts b/server/server/middlewares/validators/videos/shared/index.ts new file mode 100644 index 000000000..cb1a25a74 --- /dev/null +++ b/server/server/middlewares/validators/videos/shared/index.ts @@ -0,0 +1,2 @@ +export * from './upload.js' +export * from './video-validators.js' diff --git a/server/server/middlewares/validators/videos/shared/upload.ts b/server/server/middlewares/validators/videos/shared/upload.ts new file mode 100644 index 000000000..6798acc43 --- /dev/null +++ b/server/server/middlewares/validators/videos/shared/upload.ts @@ -0,0 +1,39 @@ +import express from 'express' +import { logger } from '@server/helpers/logger.js' +import { getVideoStreamDuration } from '@peertube/peertube-ffmpeg' +import { HttpStatusCode } from '@peertube/peertube-models' + +export async function addDurationToVideoFileIfNeeded (options: { + res: express.Response + videoFile: { path: string, duration?: number } + middlewareName: string +}) { + const { res, middlewareName, videoFile } = options + + try { + if (!videoFile.duration) await addDurationToVideo(videoFile) + } catch (err) { + logger.error('Invalid input file in ' + middlewareName, { err }) + + res.fail({ + status: HttpStatusCode.UNPROCESSABLE_ENTITY_422, + message: 'Video file unreadable.' + }) + return false + } + + return true +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +async function addDurationToVideo (videoFile: { path: string, duration?: number }) { + const duration = await getVideoStreamDuration(videoFile.path) + + // FFmpeg may not be able to guess video duration + // For example with m2v files: https://trac.ffmpeg.org/ticket/9726#comment:2 + if (isNaN(duration)) videoFile.duration = 0 + else videoFile.duration = duration +} diff --git a/server/server/middlewares/validators/videos/shared/video-validators.ts b/server/server/middlewares/validators/videos/shared/video-validators.ts new file mode 100644 index 000000000..27d86a35e --- /dev/null +++ b/server/server/middlewares/validators/videos/shared/video-validators.ts @@ -0,0 +1,105 @@ +import express from 'express' +import { HttpStatusCode, ServerErrorCode, ServerFilterHookName, VideoState, VideoStateType } from '@peertube/peertube-models' +import { isVideoFileMimeTypeValid, isVideoFileSizeValid } from '@server/helpers/custom-validators/videos.js' +import { logger } from '@server/helpers/logger.js' +import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js' +import { isLocalVideoFileAccepted } from '@server/lib/moderation.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { MUserAccountId, MVideo } from '@server/types/models/index.js' +import { checkUserQuota } from '../../shared/index.js' + +export async function commonVideoFileChecks (options: { + res: express.Response + user: MUserAccountId + videoFileSize: number + files: express.UploadFilesForCheck +}): Promise { + const { res, user, videoFileSize, files } = options + + if (!isVideoFileMimeTypeValid(files)) { + res.fail({ + status: HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415, + message: 'This file is not supported. Please, make sure it is of the following type: ' + + CONSTRAINTS_FIELDS.VIDEOS.EXTNAME.join(', ') + }) + return false + } + + if (!isVideoFileSizeValid(videoFileSize.toString())) { + res.fail({ + status: HttpStatusCode.PAYLOAD_TOO_LARGE_413, + message: 'This file is too large. It exceeds the maximum file size authorized.', + type: ServerErrorCode.MAX_FILE_SIZE_REACHED + }) + return false + } + + if (await checkUserQuota(user, videoFileSize, res) === false) return false + + return true +} + +export async function isVideoFileAccepted (options: { + req: express.Request + res: express.Response + videoFile: express.VideoUploadFile + hook: Extract +}) { + const { req, res, videoFile, hook } = options + + // Check we accept this video + const acceptParameters = { + videoBody: req.body, + videoFile, + user: res.locals.oauth.token.User + } + const acceptedResult = await Hooks.wrapFun(isLocalVideoFileAccepted, acceptParameters, hook) + + if (!acceptedResult || acceptedResult.accepted !== true) { + logger.info('Refused local video file.', { acceptedResult, acceptParameters }) + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: acceptedResult.errorMessage || 'Refused local video file' + }) + return false + } + + return true +} + +export function checkVideoFileCanBeEdited (video: MVideo, res: express.Response) { + if (video.isLive) { + res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Cannot edit a live video' + }) + + return false + } + + if (video.state === VideoState.TO_TRANSCODE || video.state === VideoState.TO_EDIT) { + res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'Cannot edit video that is already waiting for transcoding/edition' + }) + + return false + } + + const validStates = new Set([ + VideoState.PUBLISHED, + VideoState.TO_MOVE_TO_EXTERNAL_STORAGE_FAILED, + VideoState.TRANSCODING_FAILED + ]) + + if (!validStates.has(video.state)) { + res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Video state is not compatible with edition' + }) + + return false + } + + return true +} diff --git a/server/server/middlewares/validators/videos/video-blacklist.ts b/server/server/middlewares/validators/videos/video-blacklist.ts new file mode 100644 index 000000000..8790a02a6 --- /dev/null +++ b/server/server/middlewares/validators/videos/video-blacklist.ts @@ -0,0 +1,87 @@ +import express from 'express' +import { body, query } from 'express-validator' +import { HttpStatusCode } from '@peertube/peertube-models' +import { isBooleanValid, toBooleanOrNull, toIntOrNull } from '../../../helpers/custom-validators/misc.js' +import { isVideoBlacklistReasonValid, isVideoBlacklistTypeValid } from '../../../helpers/custom-validators/video-blacklist.js' +import { areValidationErrors, doesVideoBlacklistExist, doesVideoExist, isValidVideoIdParam } from '../shared/index.js' + +const videosBlacklistRemoveValidator = [ + isValidVideoIdParam('videoId'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.videoId, res)) return + if (!await doesVideoBlacklistExist(res.locals.videoAll.id, res)) return + + return next() + } +] + +const videosBlacklistAddValidator = [ + isValidVideoIdParam('videoId'), + + body('unfederate') + .optional() + .customSanitizer(toBooleanOrNull) + .custom(isBooleanValid).withMessage('Should have a valid unfederate boolean'), + body('reason') + .optional() + .custom(isVideoBlacklistReasonValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.videoId, res)) return + + const video = res.locals.videoAll + if (req.body.unfederate === true && video.remote === true) { + return res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'You cannot unfederate a remote video.' + }) + } + + return next() + } +] + +const videosBlacklistUpdateValidator = [ + isValidVideoIdParam('videoId'), + + body('reason') + .optional() + .custom(isVideoBlacklistReasonValid).withMessage('Should have a valid reason'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.videoId, res)) return + if (!await doesVideoBlacklistExist(res.locals.videoAll.id, res)) return + + return next() + } +] + +const videosBlacklistFiltersValidator = [ + query('type') + .optional() + .customSanitizer(toIntOrNull) + .custom(isVideoBlacklistTypeValid).withMessage('Should have a valid video blacklist type attribute'), + query('search') + .optional() + .not() + .isEmpty().withMessage('Should have a valid search'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + videosBlacklistAddValidator, + videosBlacklistRemoveValidator, + videosBlacklistUpdateValidator, + videosBlacklistFiltersValidator +} diff --git a/server/server/middlewares/validators/videos/video-captions.ts b/server/server/middlewares/validators/videos/video-captions.ts new file mode 100644 index 000000000..d89b48f5f --- /dev/null +++ b/server/server/middlewares/validators/videos/video-captions.ts @@ -0,0 +1,83 @@ +import express from 'express' +import { body, param } from 'express-validator' +import { UserRight } from '@peertube/peertube-models' +import { isVideoCaptionFile, isVideoCaptionLanguageValid } from '../../../helpers/custom-validators/video-captions.js' +import { cleanUpReqFiles } from '../../../helpers/express-utils.js' +import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../../initializers/constants.js' +import { + areValidationErrors, + checkCanSeeVideo, + checkUserCanManageVideo, + doesVideoCaptionExist, + doesVideoExist, + isValidVideoIdParam, + isValidVideoPasswordHeader +} from '../shared/index.js' + +const addVideoCaptionValidator = [ + isValidVideoIdParam('videoId'), + + param('captionLanguage') + .custom(isVideoCaptionLanguageValid).not().isEmpty(), + + body('captionfile') + .custom((_, { req }) => isVideoCaptionFile(req.files, 'captionfile')) + .withMessage( + 'This caption file is not supported or too large. ' + + `Please, make sure it is under ${CONSTRAINTS_FIELDS.VIDEO_CAPTIONS.CAPTION_FILE.FILE_SIZE.max} bytes ` + + 'and one of the following mimetypes: ' + + Object.keys(MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT).map(key => `${key} (${MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT[key]})`).join(', ') + ), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return cleanUpReqFiles(req) + if (!await doesVideoExist(req.params.videoId, res)) return cleanUpReqFiles(req) + + // Check if the user who did the request is able to update the video + const user = res.locals.oauth.token.User + if (!checkUserCanManageVideo(user, res.locals.videoAll, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req) + + return next() + } +] + +const deleteVideoCaptionValidator = [ + isValidVideoIdParam('videoId'), + + param('captionLanguage') + .custom(isVideoCaptionLanguageValid).not().isEmpty().withMessage('Should have a valid caption language'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.videoId, res)) return + if (!await doesVideoCaptionExist(res.locals.videoAll, req.params.captionLanguage, res)) return + + // Check if the user who did the request is able to update the video + const user = res.locals.oauth.token.User + if (!checkUserCanManageVideo(user, res.locals.videoAll, UserRight.UPDATE_ANY_VIDEO, res)) return + + return next() + } +] + +const listVideoCaptionsValidator = [ + isValidVideoIdParam('videoId'), + + isValidVideoPasswordHeader(), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.videoId, res, 'only-video')) return + + const video = res.locals.onlyVideo + if (!await checkCanSeeVideo({ req, res, video, paramId: req.params.videoId })) return + + return next() + } +] + +export { + addVideoCaptionValidator, + listVideoCaptionsValidator, + deleteVideoCaptionValidator +} diff --git a/server/server/middlewares/validators/videos/video-channel-sync.ts b/server/server/middlewares/validators/videos/video-channel-sync.ts new file mode 100644 index 000000000..1daf40625 --- /dev/null +++ b/server/server/middlewares/validators/videos/video-channel-sync.ts @@ -0,0 +1,56 @@ +import * as express from 'express' +import { body, param } from 'express-validator' +import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js' +import { CONFIG } from '@server/initializers/config.js' +import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync.js' +import { HttpStatusCode, VideoChannelSyncCreate } from '@peertube/peertube-models' +import { areValidationErrors, doesVideoChannelIdExist } from '../shared/index.js' +import { doesVideoChannelSyncIdExist } from '../shared/video-channel-syncs.js' + +export const ensureSyncIsEnabled = (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (!CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.ENABLED) { + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Synchronization is impossible as video channel synchronization is not enabled on the server' + }) + } + + return next() +} + +export const videoChannelSyncValidator = [ + body('externalChannelUrl') + .custom(isUrlValid), + + body('videoChannelId') + .isInt(), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const body: VideoChannelSyncCreate = req.body + if (!await doesVideoChannelIdExist(body.videoChannelId, res)) return + + const count = await VideoChannelSyncModel.countByAccount(res.locals.videoChannel.accountId) + if (count >= CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER) { + return res.fail({ + message: `You cannot create more than ${CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.MAX_PER_USER} channel synchronizations` + }) + } + + return next() + } +] + +export const ensureSyncExists = [ + param('id').exists().isInt().withMessage('Should have an sync id'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (!await doesVideoChannelSyncIdExist(+req.params.id, res)) return + if (!await doesVideoChannelIdExist(res.locals.videoChannelSync.videoChannelId, res)) return + + return next() + } +] diff --git a/server/server/middlewares/validators/videos/video-channels.ts b/server/server/middlewares/validators/videos/video-channels.ts new file mode 100644 index 000000000..f67bc2574 --- /dev/null +++ b/server/server/middlewares/validators/videos/video-channels.ts @@ -0,0 +1,193 @@ +import express from 'express' +import { body, param, query } from 'express-validator' +import { HttpStatusCode, VideosImportInChannelCreate } from '@peertube/peertube-models' +import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js' +import { CONFIG } from '@server/initializers/config.js' +import { MChannelAccountDefault } from '@server/types/models/index.js' +import { isBooleanValid, isIdValid, toBooleanOrNull } from '../../../helpers/custom-validators/misc.js' +import { + isVideoChannelDescriptionValid, + isVideoChannelDisplayNameValid, + isVideoChannelSupportValid, + isVideoChannelUsernameValid +} from '../../../helpers/custom-validators/video-channels.js' +import { ActorModel } from '../../../models/actor/actor.js' +import { VideoChannelModel } from '../../../models/video/video-channel.js' +import { areValidationErrors, checkUserQuota, doesVideoChannelNameWithHostExist } from '../shared/index.js' +import { doesVideoChannelSyncIdExist } from '../shared/video-channel-syncs.js' + +export const videoChannelsAddValidator = [ + body('name') + .custom(isVideoChannelUsernameValid), + body('displayName') + .custom(isVideoChannelDisplayNameValid), + body('description') + .optional() + .custom(isVideoChannelDescriptionValid), + body('support') + .optional() + .custom(isVideoChannelSupportValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const actor = await ActorModel.loadLocalByName(req.body.name) + if (actor) { + res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'Another actor (account/channel) with this name on this instance already exists or has already existed.' + }) + return false + } + + const count = await VideoChannelModel.countByAccount(res.locals.oauth.token.User.Account.id) + if (count >= CONFIG.VIDEO_CHANNELS.MAX_PER_USER) { + res.fail({ message: `You cannot create more than ${CONFIG.VIDEO_CHANNELS.MAX_PER_USER} channels` }) + return false + } + + return next() + } +] + +export const videoChannelsUpdateValidator = [ + param('nameWithHost') + .exists(), + + body('displayName') + .optional() + .custom(isVideoChannelDisplayNameValid), + body('description') + .optional() + .custom(isVideoChannelDescriptionValid), + body('support') + .optional() + .custom(isVideoChannelSupportValid), + body('bulkVideosSupportUpdate') + .optional() + .custom(isBooleanValid).withMessage('Should have a valid bulkVideosSupportUpdate boolean field'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +export const videoChannelsRemoveValidator = [ + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (!await checkVideoChannelIsNotTheLastOne(res.locals.videoChannel, res)) return + + return next() + } +] + +export const videoChannelsNameWithHostValidator = [ + param('nameWithHost') + .exists(), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (!await doesVideoChannelNameWithHostExist(req.params.nameWithHost, res)) return + + return next() + } +] + +export const ensureIsLocalChannel = [ + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (res.locals.videoChannel.Actor.isOwned() === false) { + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'This channel is not owned.' + }) + } + + return next() + } +] + +export const ensureChannelOwnerCanUpload = [ + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + const channel = res.locals.videoChannel + const user = { id: channel.Account.userId } + + if (!await checkUserQuota(user, 1, res)) return + + next() + } +] + +export const videoChannelStatsValidator = [ + query('withStats') + .optional() + .customSanitizer(toBooleanOrNull) + .custom(isBooleanValid).withMessage('Should have a valid stats flag boolean'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + return next() + } +] + +export const videoChannelsListValidator = [ + query('search') + .optional() + .not().isEmpty(), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +export const videoChannelImportVideosValidator = [ + body('externalChannelUrl') + .custom(isUrlValid), + + body('videoChannelSyncId') + .optional() + .custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const body: VideosImportInChannelCreate = req.body + + if (!CONFIG.IMPORT.VIDEOS.HTTP.ENABLED) { + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Channel import is impossible as video upload via HTTP is not enabled on the server' + }) + } + + if (body.videoChannelSyncId && !await doesVideoChannelSyncIdExist(body.videoChannelSyncId, res)) return + + if (res.locals.videoChannelSync && res.locals.videoChannelSync.videoChannelId !== res.locals.videoChannel.id) { + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'This channel sync is not owned by this channel' + }) + } + + return next() + } +] + +// --------------------------------------------------------------------------- + +async function checkVideoChannelIsNotTheLastOne (videoChannel: MChannelAccountDefault, res: express.Response) { + const count = await VideoChannelModel.countByAccount(videoChannel.Account.id) + + if (count <= 1) { + res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'Cannot remove the last channel of this user' + }) + return false + } + + return true +} diff --git a/server/server/middlewares/validators/videos/video-comments.ts b/server/server/middlewares/validators/videos/video-comments.ts new file mode 100644 index 000000000..284d87e85 --- /dev/null +++ b/server/server/middlewares/validators/videos/video-comments.ts @@ -0,0 +1,249 @@ +import express from 'express' +import { body, param, query } from 'express-validator' +import { MUserAccountUrl } from '@server/types/models/index.js' +import { HttpStatusCode, UserRight } from '@peertube/peertube-models' +import { exists, isBooleanValid, isIdValid, toBooleanOrNull } from '../../../helpers/custom-validators/misc.js' +import { isValidVideoCommentText } from '../../../helpers/custom-validators/video-comments.js' +import { logger } from '../../../helpers/logger.js' +import { AcceptResult, isLocalVideoCommentReplyAccepted, isLocalVideoThreadAccepted } from '../../../lib/moderation.js' +import { Hooks } from '../../../lib/plugins/hooks.js' +import { MCommentOwnerVideoReply, MVideo, MVideoFullLight } from '../../../types/models/video/index.js' +import { + areValidationErrors, + checkCanSeeVideo, + doesVideoCommentExist, + doesVideoCommentThreadExist, + doesVideoExist, + isValidVideoIdParam, + isValidVideoPasswordHeader +} from '../shared/index.js' + +const listVideoCommentsValidator = [ + query('isLocal') + .optional() + .customSanitizer(toBooleanOrNull) + .custom(isBooleanValid) + .withMessage('Should have a valid isLocal boolean'), + + query('onLocalVideo') + .optional() + .customSanitizer(toBooleanOrNull) + .custom(isBooleanValid) + .withMessage('Should have a valid onLocalVideo boolean'), + + query('search') + .optional() + .custom(exists), + + query('searchAccount') + .optional() + .custom(exists), + + query('searchVideo') + .optional() + .custom(exists), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const listVideoCommentThreadsValidator = [ + isValidVideoIdParam('videoId'), + isValidVideoPasswordHeader(), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.videoId, res, 'only-video')) return + + if (!await checkCanSeeVideo({ req, res, paramId: req.params.videoId, video: res.locals.onlyVideo })) return + + return next() + } +] + +const listVideoThreadCommentsValidator = [ + isValidVideoIdParam('videoId'), + + param('threadId') + .custom(isIdValid), + isValidVideoPasswordHeader(), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.videoId, res, 'only-video')) return + if (!await doesVideoCommentThreadExist(req.params.threadId, res.locals.onlyVideo, res)) return + + if (!await checkCanSeeVideo({ req, res, paramId: req.params.videoId, video: res.locals.onlyVideo })) return + + return next() + } +] + +const addVideoCommentThreadValidator = [ + isValidVideoIdParam('videoId'), + + body('text') + .custom(isValidVideoCommentText), + isValidVideoPasswordHeader(), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.videoId, res)) return + + if (!await checkCanSeeVideo({ req, res, paramId: req.params.videoId, video: res.locals.videoAll })) return + + if (!isVideoCommentsEnabled(res.locals.videoAll, res)) return + if (!await isVideoCommentAccepted(req, res, res.locals.videoAll, false)) return + + return next() + } +] + +const addVideoCommentReplyValidator = [ + isValidVideoIdParam('videoId'), + + param('commentId').custom(isIdValid), + isValidVideoPasswordHeader(), + + body('text').custom(isValidVideoCommentText), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.videoId, res)) return + + if (!await checkCanSeeVideo({ req, res, paramId: req.params.videoId, video: res.locals.videoAll })) return + + if (!isVideoCommentsEnabled(res.locals.videoAll, res)) return + if (!await doesVideoCommentExist(req.params.commentId, res.locals.videoAll, res)) return + if (!await isVideoCommentAccepted(req, res, res.locals.videoAll, true)) return + + return next() + } +] + +const videoCommentGetValidator = [ + isValidVideoIdParam('videoId'), + + param('commentId') + .custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.videoId, res, 'id')) return + if (!await doesVideoCommentExist(req.params.commentId, res.locals.videoId, res)) return + + return next() + } +] + +const removeVideoCommentValidator = [ + isValidVideoIdParam('videoId'), + + param('commentId') + .custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.videoId, res)) return + if (!await doesVideoCommentExist(req.params.commentId, res.locals.videoAll, res)) return + + // Check if the user who did the request is able to delete the video + if (!checkUserCanDeleteVideoComment(res.locals.oauth.token.User, res.locals.videoCommentFull, res)) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + listVideoCommentThreadsValidator, + listVideoThreadCommentsValidator, + addVideoCommentThreadValidator, + listVideoCommentsValidator, + addVideoCommentReplyValidator, + videoCommentGetValidator, + removeVideoCommentValidator +} + +// --------------------------------------------------------------------------- + +function isVideoCommentsEnabled (video: MVideo, res: express.Response) { + if (video.commentsEnabled !== true) { + res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'Video comments are disabled for this video.' + }) + return false + } + + return true +} + +function checkUserCanDeleteVideoComment (user: MUserAccountUrl, videoComment: MCommentOwnerVideoReply, res: express.Response) { + if (videoComment.isDeleted()) { + res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'This comment is already deleted' + }) + return false + } + + const userAccount = user.Account + + if ( + user.hasRight(UserRight.REMOVE_ANY_VIDEO_COMMENT) === false && // Not a moderator + videoComment.accountId !== userAccount.id && // Not the comment owner + videoComment.Video.VideoChannel.accountId !== userAccount.id // Not the video owner + ) { + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Cannot remove video comment of another user' + }) + return false + } + + return true +} + +async function isVideoCommentAccepted (req: express.Request, res: express.Response, video: MVideoFullLight, isReply: boolean) { + const acceptParameters = { + video, + commentBody: req.body, + user: res.locals.oauth.token.User, + req + } + + let acceptedResult: AcceptResult + + if (isReply) { + const acceptReplyParameters = Object.assign(acceptParameters, { parentComment: res.locals.videoCommentFull }) + + acceptedResult = await Hooks.wrapFun( + isLocalVideoCommentReplyAccepted, + acceptReplyParameters, + 'filter:api.video-comment-reply.create.accept.result' + ) + } else { + acceptedResult = await Hooks.wrapFun( + isLocalVideoThreadAccepted, + acceptParameters, + 'filter:api.video-thread.create.accept.result' + ) + } + + if (!acceptedResult || acceptedResult.accepted !== true) { + logger.info('Refused local comment.', { acceptedResult, acceptParameters }) + + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: acceptedResult?.errorMessage || 'Comment has been rejected.' + }) + return false + } + + return true +} diff --git a/server/server/middlewares/validators/videos/video-files.ts b/server/server/middlewares/validators/videos/video-files.ts new file mode 100644 index 000000000..0a14fa134 --- /dev/null +++ b/server/server/middlewares/validators/videos/video-files.ts @@ -0,0 +1,163 @@ +import express from 'express' +import { param } from 'express-validator' +import { isIdValid } from '@server/helpers/custom-validators/misc.js' +import { MVideo } from '@server/types/models/index.js' +import { HttpStatusCode } from '@peertube/peertube-models' +import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared/index.js' + +const videoFilesDeleteWebVideoValidator = [ + isValidVideoIdParam('id'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.id, res)) return + + const video = res.locals.videoAll + + if (!checkLocalVideo(video, res)) return + + if (!video.hasWebVideoFiles()) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'This video does not have Web Video files' + }) + } + + if (!video.getHLSPlaylist()) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Cannot delete Web Video files since this video does not have HLS playlist' + }) + } + + return next() + } +] + +const videoFilesDeleteWebVideoFileValidator = [ + isValidVideoIdParam('id'), + + param('videoFileId') + .custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.id, res)) return + + const video = res.locals.videoAll + + if (!checkLocalVideo(video, res)) return + + const files = video.VideoFiles + if (!files.find(f => f.id === +req.params.videoFileId)) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'This video does not have this Web Video file id' + }) + } + + if (files.length === 1 && !video.getHLSPlaylist()) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Cannot delete Web Video files since this video does not have HLS playlist' + }) + } + + return next() + } +] + +// --------------------------------------------------------------------------- + +const videoFilesDeleteHLSValidator = [ + isValidVideoIdParam('id'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.id, res)) return + + const video = res.locals.videoAll + + if (!checkLocalVideo(video, res)) return + + if (!video.getHLSPlaylist()) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'This video does not have HLS files' + }) + } + + if (!video.hasWebVideoFiles()) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Cannot delete HLS playlist since this video does not have Web Video files' + }) + } + + return next() + } +] + +const videoFilesDeleteHLSFileValidator = [ + isValidVideoIdParam('id'), + + param('videoFileId') + .custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.id, res)) return + + const video = res.locals.videoAll + + if (!checkLocalVideo(video, res)) return + + if (!video.getHLSPlaylist()) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'This video does not have HLS files' + }) + } + + const hlsFiles = video.getHLSPlaylist().VideoFiles + if (!hlsFiles.find(f => f.id === +req.params.videoFileId)) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'This HLS playlist does not have this file id' + }) + } + + // Last file to delete + if (hlsFiles.length === 1 && !video.hasWebVideoFiles()) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Cannot delete last HLS playlist file since this video does not have Web Video files' + }) + } + + return next() + } +] + +export { + videoFilesDeleteWebVideoValidator, + videoFilesDeleteWebVideoFileValidator, + + videoFilesDeleteHLSValidator, + videoFilesDeleteHLSFileValidator +} + +// --------------------------------------------------------------------------- + +function checkLocalVideo (video: MVideo, res: express.Response) { + if (video.remote) { + res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Cannot delete files of remote video' + }) + + return false + } + + return true +} diff --git a/server/server/middlewares/validators/videos/video-imports.ts b/server/server/middlewares/validators/videos/video-imports.ts new file mode 100644 index 000000000..c9fceadae --- /dev/null +++ b/server/server/middlewares/validators/videos/video-imports.ts @@ -0,0 +1,204 @@ +import express from 'express' +import { body, param, query } from 'express-validator' +import { forceNumber } from '@peertube/peertube-core-utils' +import { HttpStatusCode, UserRight, VideoImportCreate, VideoImportState } from '@peertube/peertube-models' +import { isResolvingToUnicastOnly } from '@server/helpers/dns.js' +import { isPreImportVideoAccepted } from '@server/lib/moderation.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { MUserAccountId, MVideoImport } from '@server/types/models/index.js' +import { isIdValid, toIntOrNull } from '../../../helpers/custom-validators/misc.js' +import { isVideoImportTargetUrlValid, isVideoImportTorrentFile } from '../../../helpers/custom-validators/video-imports.js' +import { isValidPasswordProtectedPrivacy, isVideoMagnetUriValid, isVideoNameValid } from '../../../helpers/custom-validators/videos.js' +import { cleanUpReqFiles } from '../../../helpers/express-utils.js' +import { logger } from '../../../helpers/logger.js' +import { CONFIG } from '../../../initializers/config.js' +import { CONSTRAINTS_FIELDS } from '../../../initializers/constants.js' +import { areValidationErrors, doesVideoChannelOfAccountExist, doesVideoImportExist } from '../shared/index.js' +import { getCommonVideoEditAttributes } from './videos.js' + +const videoImportAddValidator = getCommonVideoEditAttributes().concat([ + body('channelId') + .customSanitizer(toIntOrNull) + .custom(isIdValid), + body('targetUrl') + .optional() + .custom(isVideoImportTargetUrlValid), + body('magnetUri') + .optional() + .custom(isVideoMagnetUriValid), + body('torrentfile') + .custom((value, { req }) => isVideoImportTorrentFile(req.files)) + .withMessage( + 'This torrent file is not supported or too large. Please, make sure it is of the following type: ' + + CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_FILE.EXTNAME.join(', ') + ), + body('name') + .optional() + .custom(isVideoNameValid).withMessage( + `Should have a video name between ${CONSTRAINTS_FIELDS.VIDEOS.NAME.min} and ${CONSTRAINTS_FIELDS.VIDEOS.NAME.max} characters long` + ), + body('videoPasswords') + .optional() + .isArray() + .withMessage('Video passwords should be an array.'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + const user = res.locals.oauth.token.User + const torrentFile = req.files?.['torrentfile'] ? req.files['torrentfile'][0] : undefined + + if (areValidationErrors(req, res)) return cleanUpReqFiles(req) + + if (!isValidPasswordProtectedPrivacy(req, res)) return cleanUpReqFiles(req) + + if (CONFIG.IMPORT.VIDEOS.HTTP.ENABLED !== true && req.body.targetUrl) { + cleanUpReqFiles(req) + + return res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'HTTP import is not enabled on this instance.' + }) + } + + if (CONFIG.IMPORT.VIDEOS.TORRENT.ENABLED !== true && (req.body.magnetUri || torrentFile)) { + cleanUpReqFiles(req) + + return res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'Torrent/magnet URI import is not enabled on this instance.' + }) + } + + if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) + + // Check we have at least 1 required param + if (!req.body.targetUrl && !req.body.magnetUri && !torrentFile) { + cleanUpReqFiles(req) + + return res.fail({ message: 'Should have a magnetUri or a targetUrl or a torrent file.' }) + } + + if (req.body.targetUrl) { + const hostname = new URL(req.body.targetUrl).hostname + + if (await isResolvingToUnicastOnly(hostname) !== true) { + cleanUpReqFiles(req) + + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Cannot use non unicast IP as targetUrl.' + }) + } + } + + if (!await isImportAccepted(req, res)) return cleanUpReqFiles(req) + + return next() + } +]) + +const getMyVideoImportsValidator = [ + query('videoChannelSyncId') + .optional() + .custom(isIdValid), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const videoImportDeleteValidator = [ + param('id') + .custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (!await doesVideoImportExist(parseInt(req.params.id), res)) return + if (!checkUserCanManageImport(res.locals.oauth.token.user, res.locals.videoImport, res)) return + + if (res.locals.videoImport.state === VideoImportState.PENDING) { + return res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'Cannot delete a pending video import. Cancel it or wait for the end of the import first.' + }) + } + + return next() + } +] + +const videoImportCancelValidator = [ + param('id') + .custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (!await doesVideoImportExist(forceNumber(req.params.id), res)) return + if (!checkUserCanManageImport(res.locals.oauth.token.user, res.locals.videoImport, res)) return + + if (res.locals.videoImport.state !== VideoImportState.PENDING) { + return res.fail({ + status: HttpStatusCode.CONFLICT_409, + message: 'Cannot cancel a non pending video import.' + }) + } + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + videoImportAddValidator, + videoImportCancelValidator, + videoImportDeleteValidator, + getMyVideoImportsValidator +} + +// --------------------------------------------------------------------------- + +async function isImportAccepted (req: express.Request, res: express.Response) { + const body: VideoImportCreate = req.body + const hookName = body.targetUrl + ? 'filter:api.video.pre-import-url.accept.result' + : 'filter:api.video.pre-import-torrent.accept.result' + + // Check we accept this video + const acceptParameters = { + videoImportBody: body, + user: res.locals.oauth.token.User + } + const acceptedResult = await Hooks.wrapFun( + isPreImportVideoAccepted, + acceptParameters, + hookName + ) + + if (!acceptedResult || acceptedResult.accepted !== true) { + logger.info('Refused to import video.', { acceptedResult, acceptParameters }) + + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: acceptedResult.errorMessage || 'Refused to import video' + }) + return false + } + + return true +} + +function checkUserCanManageImport (user: MUserAccountId, videoImport: MVideoImport, res: express.Response) { + if (user.hasRight(UserRight.MANAGE_VIDEO_IMPORTS) === false && videoImport.userId !== user.id) { + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Cannot manage video import of another user' + }) + return false + } + + return true +} diff --git a/server/server/middlewares/validators/videos/video-live.ts b/server/server/middlewares/validators/videos/video-live.ts new file mode 100644 index 000000000..097da079d --- /dev/null +++ b/server/server/middlewares/validators/videos/video-live.ts @@ -0,0 +1,342 @@ +import express from 'express' +import { body } from 'express-validator' +import { isLiveLatencyModeValid } from '@server/helpers/custom-validators/video-lives.js' +import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js' +import { isLocalLiveVideoAccepted } from '@server/lib/moderation.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { VideoModel } from '@server/models/video/video.js' +import { VideoLiveModel } from '@server/models/video/video-live.js' +import { VideoLiveSessionModel } from '@server/models/video/video-live-session.js' +import { + HttpStatusCode, + LiveVideoCreate, + LiveVideoLatencyMode, + LiveVideoUpdate, + ServerErrorCode, + UserRight, + VideoState +} from '@peertube/peertube-models' +import { exists, isBooleanValid, isIdValid, toBooleanOrNull, toIntOrNull } from '../../../helpers/custom-validators/misc.js' +import { isValidPasswordProtectedPrivacy, isVideoNameValid, isVideoReplayPrivacyValid } from '../../../helpers/custom-validators/videos.js' +import { cleanUpReqFiles } from '../../../helpers/express-utils.js' +import { logger } from '../../../helpers/logger.js' +import { CONFIG } from '../../../initializers/config.js' +import { + areValidationErrors, + checkUserCanManageVideo, + doesVideoChannelOfAccountExist, + doesVideoExist, + isValidVideoIdParam +} from '../shared/index.js' +import { getCommonVideoEditAttributes } from './videos.js' + +const videoLiveGetValidator = [ + isValidVideoIdParam('videoId'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.videoId, res, 'all')) return + + const videoLive = await VideoLiveModel.loadByVideoId(res.locals.videoAll.id) + if (!videoLive) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Live video not found' + }) + } + + res.locals.videoLive = videoLive + + return next() + } +] + +const videoLiveAddValidator = getCommonVideoEditAttributes().concat([ + body('channelId') + .customSanitizer(toIntOrNull) + .custom(isIdValid), + + body('name') + .custom(isVideoNameValid).withMessage( + `Should have a video name between ${CONSTRAINTS_FIELDS.VIDEOS.NAME.min} and ${CONSTRAINTS_FIELDS.VIDEOS.NAME.max} characters long` + ), + + body('saveReplay') + .optional() + .customSanitizer(toBooleanOrNull) + .custom(isBooleanValid).withMessage('Should have a valid saveReplay boolean'), + + body('replaySettings.privacy') + .optional() + .customSanitizer(toIntOrNull) + .custom(isVideoReplayPrivacyValid), + + body('permanentLive') + .optional() + .customSanitizer(toBooleanOrNull) + .custom(isBooleanValid).withMessage('Should have a valid permanentLive boolean'), + + body('latencyMode') + .optional() + .customSanitizer(toIntOrNull) + .custom(isLiveLatencyModeValid), + + body('videoPasswords') + .optional() + .isArray() + .withMessage('Video passwords should be an array.'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return cleanUpReqFiles(req) + + if (!isValidPasswordProtectedPrivacy(req, res)) return cleanUpReqFiles(req) + + if (CONFIG.LIVE.ENABLED !== true) { + cleanUpReqFiles(req) + + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Live is not enabled on this instance', + type: ServerErrorCode.LIVE_NOT_ENABLED + }) + } + + const body: LiveVideoCreate = req.body + + if (hasValidSaveReplay(body) !== true) { + cleanUpReqFiles(req) + + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Saving live replay is not enabled on this instance', + type: ServerErrorCode.LIVE_NOT_ALLOWING_REPLAY + }) + } + + if (hasValidLatencyMode(body) !== true) { + cleanUpReqFiles(req) + + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Custom latency mode is not allowed by this instance' + }) + } + + if (body.saveReplay && !body.replaySettings?.privacy) { + cleanUpReqFiles(req) + + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Live replay is enabled but privacy replay setting is missing' + }) + } + + const user = res.locals.oauth.token.User + if (!await doesVideoChannelOfAccountExist(body.channelId, user, res)) return cleanUpReqFiles(req) + + if (CONFIG.LIVE.MAX_INSTANCE_LIVES !== -1) { + const totalInstanceLives = await VideoModel.countLives({ remote: false, mode: 'not-ended' }) + + if (totalInstanceLives >= CONFIG.LIVE.MAX_INSTANCE_LIVES) { + cleanUpReqFiles(req) + + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Cannot create this live because the max instance lives limit is reached.', + type: ServerErrorCode.MAX_INSTANCE_LIVES_LIMIT_REACHED + }) + } + } + + if (CONFIG.LIVE.MAX_USER_LIVES !== -1) { + const totalUserLives = await VideoModel.countLivesOfAccount(user.Account.id) + + if (totalUserLives >= CONFIG.LIVE.MAX_USER_LIVES) { + cleanUpReqFiles(req) + + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Cannot create this live because the max user lives limit is reached.', + type: ServerErrorCode.MAX_USER_LIVES_LIMIT_REACHED + }) + } + } + + if (!await isLiveVideoAccepted(req, res)) return cleanUpReqFiles(req) + + return next() + } +]) + +const videoLiveUpdateValidator = [ + body('saveReplay') + .optional() + .customSanitizer(toBooleanOrNull) + .custom(isBooleanValid).withMessage('Should have a valid saveReplay boolean'), + + body('replaySettings.privacy') + .optional() + .customSanitizer(toIntOrNull) + .custom(isVideoReplayPrivacyValid), + + body('latencyMode') + .optional() + .customSanitizer(toIntOrNull) + .custom(isLiveLatencyModeValid), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const body: LiveVideoUpdate = req.body + + if (hasValidSaveReplay(body) !== true) { + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Saving live replay is not allowed by this instance' + }) + } + + if (hasValidLatencyMode(body) !== true) { + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Custom latency mode is not allowed by this instance' + }) + } + + if (!checkLiveSettingsReplayConsistency({ res, body })) return + + if (res.locals.videoAll.state !== VideoState.WAITING_FOR_LIVE) { + return res.fail({ message: 'Cannot update a live that has already started' }) + } + + // Check the user can manage the live + const user = res.locals.oauth.token.User + if (!checkUserCanManageVideo(user, res.locals.videoAll, UserRight.GET_ANY_LIVE, res)) return + + return next() + } +] + +const videoLiveListSessionsValidator = [ + (req: express.Request, res: express.Response, next: express.NextFunction) => { + // Check the user can manage the live + const user = res.locals.oauth.token.User + if (!checkUserCanManageVideo(user, res.locals.videoAll, UserRight.GET_ANY_LIVE, res)) return + + return next() + } +] + +const videoLiveFindReplaySessionValidator = [ + isValidVideoIdParam('videoId'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.videoId, res, 'id')) return + + const session = await VideoLiveSessionModel.findSessionOfReplay(res.locals.videoId.id) + if (!session) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'No live replay found' + }) + } + + res.locals.videoLiveSession = session + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + videoLiveAddValidator, + videoLiveUpdateValidator, + videoLiveListSessionsValidator, + videoLiveFindReplaySessionValidator, + videoLiveGetValidator +} + +// --------------------------------------------------------------------------- + +async function isLiveVideoAccepted (req: express.Request, res: express.Response) { + // Check we accept this video + const acceptParameters = { + liveVideoBody: req.body, + user: res.locals.oauth.token.User + } + const acceptedResult = await Hooks.wrapFun( + isLocalLiveVideoAccepted, + acceptParameters, + 'filter:api.live-video.create.accept.result' + ) + + if (!acceptedResult || acceptedResult.accepted !== true) { + logger.info('Refused local live video.', { acceptedResult, acceptParameters }) + + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: acceptedResult.errorMessage || 'Refused local live video' + }) + return false + } + + return true +} + +function hasValidSaveReplay (body: LiveVideoUpdate | LiveVideoCreate) { + if (CONFIG.LIVE.ALLOW_REPLAY !== true && body.saveReplay === true) return false + + return true +} + +function hasValidLatencyMode (body: LiveVideoUpdate | LiveVideoCreate) { + if ( + CONFIG.LIVE.LATENCY_SETTING.ENABLED !== true && + exists(body.latencyMode) && + body.latencyMode !== LiveVideoLatencyMode.DEFAULT + ) return false + + return true +} + +function checkLiveSettingsReplayConsistency (options: { + res: express.Response + body: LiveVideoUpdate +}) { + const { res, body } = options + + // We now save replays of this live, so replay settings are mandatory + if (res.locals.videoLive.saveReplay !== true && body.saveReplay === true) { + + if (!exists(body.replaySettings)) { + res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Replay settings are missing now the live replay is saved' + }) + return false + } + + if (!exists(body.replaySettings.privacy)) { + res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Privacy replay setting is missing now the live replay is saved' + }) + return false + } + } + + // Save replay was and is not enabled, so send an error the user if it specified replay settings + if ((!exists(body.saveReplay) && res.locals.videoLive.saveReplay === false) || body.saveReplay === false) { + if (exists(body.replaySettings)) { + res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Cannot save replay settings since live replay is not enabled' + }) + return false + } + } + + return true +} diff --git a/server/server/middlewares/validators/videos/video-ownership-changes.ts b/server/server/middlewares/validators/videos/video-ownership-changes.ts new file mode 100644 index 000000000..0cdb077b3 --- /dev/null +++ b/server/server/middlewares/validators/videos/video-ownership-changes.ts @@ -0,0 +1,107 @@ +import express from 'express' +import { param } from 'express-validator' +import { isIdValid } from '@server/helpers/custom-validators/misc.js' +import { checkUserCanTerminateOwnershipChange } from '@server/helpers/custom-validators/video-ownership.js' +import { AccountModel } from '@server/models/account/account.js' +import { MVideoWithAllFiles } from '@server/types/models/index.js' +import { HttpStatusCode, UserRight, VideoChangeOwnershipAccept, VideoChangeOwnershipStatus, VideoState } from '@peertube/peertube-models' +import { + areValidationErrors, + checkUserCanManageVideo, + checkUserQuota, + doesChangeVideoOwnershipExist, + doesVideoChannelOfAccountExist, + doesVideoExist, + isValidVideoIdParam +} from '../shared/index.js' + +const videosChangeOwnershipValidator = [ + isValidVideoIdParam('videoId'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.videoId, res)) return + + // Check if the user who did the request is able to change the ownership of the video + if (!checkUserCanManageVideo(res.locals.oauth.token.User, res.locals.videoAll, UserRight.CHANGE_VIDEO_OWNERSHIP, res)) return + + const nextOwner = await AccountModel.loadLocalByName(req.body.username) + if (!nextOwner) { + res.fail({ message: 'Changing video ownership to a remote account is not supported yet' }) + return + } + + res.locals.nextOwner = nextOwner + return next() + } +] + +const videosTerminateChangeOwnershipValidator = [ + param('id') + .custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesChangeVideoOwnershipExist(req.params.id, res)) return + + // Check if the user who did the request is able to change the ownership of the video + if (!checkUserCanTerminateOwnershipChange(res.locals.oauth.token.User, res.locals.videoChangeOwnership, res)) return + + const videoChangeOwnership = res.locals.videoChangeOwnership + + if (videoChangeOwnership.status !== VideoChangeOwnershipStatus.WAITING) { + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Ownership already accepted or refused' + }) + return + } + + return next() + } +] + +const videosAcceptChangeOwnershipValidator = [ + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + const body = req.body as VideoChangeOwnershipAccept + if (!await doesVideoChannelOfAccountExist(body.channelId, res.locals.oauth.token.User, res)) return + + const videoChangeOwnership = res.locals.videoChangeOwnership + + const video = videoChangeOwnership.Video + + if (!await checkCanAccept(video, res)) return + + return next() + } +] + +export { + videosChangeOwnershipValidator, + videosTerminateChangeOwnershipValidator, + videosAcceptChangeOwnershipValidator +} + +// --------------------------------------------------------------------------- + +async function checkCanAccept (video: MVideoWithAllFiles, res: express.Response): Promise { + if (video.isLive) { + + if (video.state !== VideoState.WAITING_FOR_LIVE) { + res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'You can accept an ownership change of a published live.' + }) + + return false + } + + return true + } + + const user = res.locals.oauth.token.User + + if (!await checkUserQuota(user, video.getMaxQualityFile().size, res)) return false + + return true +} diff --git a/server/server/middlewares/validators/videos/video-passwords.ts b/server/server/middlewares/validators/videos/video-passwords.ts new file mode 100644 index 000000000..58ca224f4 --- /dev/null +++ b/server/server/middlewares/validators/videos/video-passwords.ts @@ -0,0 +1,77 @@ +import express from 'express' +import { + areValidationErrors, + doesVideoExist, + isVideoPasswordProtected, + isValidVideoIdParam, + doesVideoPasswordExist, + isVideoPasswordDeletable, + checkUserCanManageVideo +} from '../shared/index.js' +import { body, param } from 'express-validator' +import { isIdValid } from '@server/helpers/custom-validators/misc.js' +import { isValidPasswordProtectedPrivacy } from '@server/helpers/custom-validators/videos.js' +import { UserRight } from '@peertube/peertube-models' + +const listVideoPasswordValidator = [ + isValidVideoIdParam('videoId'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (!await doesVideoExist(req.params.videoId, res)) return + if (!isVideoPasswordProtected(res)) return + + // Check if the user who did the request is able to access video password list + const user = res.locals.oauth.token.User + if (!checkUserCanManageVideo(user, res.locals.videoAll, UserRight.SEE_ALL_VIDEOS, res)) return + + return next() + } +] + +const updateVideoPasswordListValidator = [ + body('passwords') + .optional() + .isArray() + .withMessage('Video passwords should be an array.'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (!await doesVideoExist(req.params.videoId, res)) return + if (!isValidPasswordProtectedPrivacy(req, res)) return + + // Check if the user who did the request is able to update video passwords + const user = res.locals.oauth.token.User + if (!checkUserCanManageVideo(user, res.locals.videoAll, UserRight.UPDATE_ANY_VIDEO, res)) return + + return next() + } +] + +const removeVideoPasswordValidator = [ + isValidVideoIdParam('videoId'), + + param('passwordId') + .custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (!await doesVideoExist(req.params.videoId, res)) return + if (!isVideoPasswordProtected(res)) return + if (!await doesVideoPasswordExist(req.params.passwordId, res)) return + if (!await isVideoPasswordDeletable(res)) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + listVideoPasswordValidator, + updateVideoPasswordListValidator, + removeVideoPasswordValidator +} diff --git a/server/server/middlewares/validators/videos/video-playlists.ts b/server/server/middlewares/validators/videos/video-playlists.ts new file mode 100644 index 000000000..1e4998101 --- /dev/null +++ b/server/server/middlewares/validators/videos/video-playlists.ts @@ -0,0 +1,425 @@ +import express from 'express' +import { body, param, query, ValidationChain } from 'express-validator' +import { forceNumber } from '@peertube/peertube-core-utils' +import { + HttpStatusCode, + UserRight, + UserRightType, + VideoPlaylistCreate, + VideoPlaylistPrivacy, + VideoPlaylistType, + VideoPlaylistUpdate +} from '@peertube/peertube-models' +import { ExpressPromiseHandler } from '@server/types/express-handler.js' +import { MUserAccountId } from '@server/types/models/index.js' +import { + isArrayOf, + isIdOrUUIDValid, + isIdValid, + isUUIDValid, + toCompleteUUID, + toIntArray, + toIntOrNull, + toValueOrNull +} from '../../../helpers/custom-validators/misc.js' +import { + isVideoPlaylistDescriptionValid, + isVideoPlaylistNameValid, + isVideoPlaylistPrivacyValid, + isVideoPlaylistTimestampValid, + isVideoPlaylistTypeValid +} from '../../../helpers/custom-validators/video-playlists.js' +import { isVideoImageValid } from '../../../helpers/custom-validators/videos.js' +import { cleanUpReqFiles } from '../../../helpers/express-utils.js' +import { CONSTRAINTS_FIELDS } from '../../../initializers/constants.js' +import { VideoPlaylistElementModel } from '../../../models/video/video-playlist-element.js' +import { MVideoPlaylist } from '../../../types/models/video/video-playlist.js' +import { authenticatePromise } from '../../auth.js' +import { + areValidationErrors, + doesVideoChannelIdExist, + doesVideoExist, + doesVideoPlaylistExist, + isValidPlaylistIdParam, + VideoPlaylistFetchType +} from '../shared/index.js' + +const videoPlaylistsAddValidator = getCommonPlaylistEditAttributes().concat([ + body('displayName') + .custom(isVideoPlaylistNameValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return cleanUpReqFiles(req) + + const body: VideoPlaylistCreate = req.body + if (body.videoChannelId && !await doesVideoChannelIdExist(body.videoChannelId, res)) return cleanUpReqFiles(req) + + if ( + !body.videoChannelId && + (body.privacy === VideoPlaylistPrivacy.PUBLIC || body.privacy === VideoPlaylistPrivacy.UNLISTED) + ) { + cleanUpReqFiles(req) + + return res.fail({ message: 'Cannot set "public" or "unlisted" a playlist that is not assigned to a channel.' }) + } + + return next() + } +]) + +const videoPlaylistsUpdateValidator = getCommonPlaylistEditAttributes().concat([ + isValidPlaylistIdParam('playlistId'), + + body('displayName') + .optional() + .custom(isVideoPlaylistNameValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return cleanUpReqFiles(req) + + if (!await doesVideoPlaylistExist(req.params.playlistId, res, 'all')) return cleanUpReqFiles(req) + + const videoPlaylist = getPlaylist(res) + + if (!checkUserCanManageVideoPlaylist(res.locals.oauth.token.User, videoPlaylist, UserRight.REMOVE_ANY_VIDEO_PLAYLIST, res)) { + return cleanUpReqFiles(req) + } + + const body: VideoPlaylistUpdate = req.body + + const newPrivacy = body.privacy || videoPlaylist.privacy + if (newPrivacy === VideoPlaylistPrivacy.PUBLIC && + ( + (!videoPlaylist.videoChannelId && !body.videoChannelId) || + body.videoChannelId === null + ) + ) { + cleanUpReqFiles(req) + + return res.fail({ message: 'Cannot set "public" a playlist that is not assigned to a channel.' }) + } + + if (videoPlaylist.type === VideoPlaylistType.WATCH_LATER) { + cleanUpReqFiles(req) + + return res.fail({ message: 'Cannot update a watch later playlist.' }) + } + + if (body.videoChannelId && !await doesVideoChannelIdExist(body.videoChannelId, res)) return cleanUpReqFiles(req) + + return next() + } +]) + +const videoPlaylistsDeleteValidator = [ + isValidPlaylistIdParam('playlistId'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (!await doesVideoPlaylistExist(req.params.playlistId, res)) return + + const videoPlaylist = getPlaylist(res) + if (videoPlaylist.type === VideoPlaylistType.WATCH_LATER) { + return res.fail({ message: 'Cannot delete a watch later playlist.' }) + } + + if (!checkUserCanManageVideoPlaylist(res.locals.oauth.token.User, videoPlaylist, UserRight.REMOVE_ANY_VIDEO_PLAYLIST, res)) { + return + } + + return next() + } +] + +const videoPlaylistsGetValidator = (fetchType: VideoPlaylistFetchType) => { + return [ + isValidPlaylistIdParam('playlistId'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (!await doesVideoPlaylistExist(req.params.playlistId, res, fetchType)) return + + const videoPlaylist = res.locals.videoPlaylistFull || res.locals.videoPlaylistSummary + + // Playlist is unlisted, check we used the uuid to fetch it + if (videoPlaylist.privacy === VideoPlaylistPrivacy.UNLISTED) { + if (isUUIDValid(req.params.playlistId)) return next() + + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Playlist not found' + }) + } + + if (videoPlaylist.privacy === VideoPlaylistPrivacy.PRIVATE) { + await authenticatePromise({ req, res }) + + const user = res.locals.oauth ? res.locals.oauth.token.User : null + + if ( + !user || + (videoPlaylist.OwnerAccount.id !== user.Account.id && !user.hasRight(UserRight.UPDATE_ANY_VIDEO_PLAYLIST)) + ) { + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Cannot get this private video playlist.' + }) + } + + return next() + } + + return next() + } + ] +} + +const videoPlaylistsSearchValidator = [ + query('search') + .optional() + .not().isEmpty(), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const videoPlaylistsAddVideoValidator = [ + isValidPlaylistIdParam('playlistId'), + + body('videoId') + .customSanitizer(toCompleteUUID) + .custom(isIdOrUUIDValid).withMessage('Should have a valid video id/uuid/short uuid'), + body('startTimestamp') + .optional() + .custom(isVideoPlaylistTimestampValid), + body('stopTimestamp') + .optional() + .custom(isVideoPlaylistTimestampValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (!await doesVideoPlaylistExist(req.params.playlistId, res, 'all')) return + if (!await doesVideoExist(req.body.videoId, res, 'only-video')) return + + const videoPlaylist = getPlaylist(res) + + if (!checkUserCanManageVideoPlaylist(res.locals.oauth.token.User, videoPlaylist, UserRight.UPDATE_ANY_VIDEO_PLAYLIST, res)) { + return + } + + return next() + } +] + +const videoPlaylistsUpdateOrRemoveVideoValidator = [ + isValidPlaylistIdParam('playlistId'), + param('playlistElementId') + .customSanitizer(toCompleteUUID) + .custom(isIdValid).withMessage('Should have an element id/uuid/short uuid'), + body('startTimestamp') + .optional() + .custom(isVideoPlaylistTimestampValid), + body('stopTimestamp') + .optional() + .custom(isVideoPlaylistTimestampValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (!await doesVideoPlaylistExist(req.params.playlistId, res, 'all')) return + + const videoPlaylist = getPlaylist(res) + + const videoPlaylistElement = await VideoPlaylistElementModel.loadById(req.params.playlistElementId) + if (!videoPlaylistElement) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video playlist element not found' + }) + return + } + res.locals.videoPlaylistElement = videoPlaylistElement + + if (!checkUserCanManageVideoPlaylist(res.locals.oauth.token.User, videoPlaylist, UserRight.UPDATE_ANY_VIDEO_PLAYLIST, res)) return + + return next() + } +] + +const videoPlaylistElementAPGetValidator = [ + isValidPlaylistIdParam('playlistId'), + param('playlistElementId') + .custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const playlistElementId = forceNumber(req.params.playlistElementId) + const playlistId = req.params.playlistId + + const videoPlaylistElement = await VideoPlaylistElementModel.loadByPlaylistAndElementIdForAP(playlistId, playlistElementId) + if (!videoPlaylistElement) { + res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video playlist element not found' + }) + return + } + + if (videoPlaylistElement.VideoPlaylist.privacy === VideoPlaylistPrivacy.PRIVATE) { + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Cannot get this private video playlist.' + }) + } + + res.locals.videoPlaylistElementAP = videoPlaylistElement + + return next() + } +] + +const videoPlaylistsReorderVideosValidator = [ + isValidPlaylistIdParam('playlistId'), + + body('startPosition') + .isInt({ min: 1 }), + body('insertAfterPosition') + .isInt({ min: 0 }), + body('reorderLength') + .optional() + .isInt({ min: 1 }), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + if (!await doesVideoPlaylistExist(req.params.playlistId, res, 'all')) return + + const videoPlaylist = getPlaylist(res) + if (!checkUserCanManageVideoPlaylist(res.locals.oauth.token.User, videoPlaylist, UserRight.UPDATE_ANY_VIDEO_PLAYLIST, res)) return + + const nextPosition = await VideoPlaylistElementModel.getNextPositionOf(videoPlaylist.id) + const startPosition: number = req.body.startPosition + const insertAfterPosition: number = req.body.insertAfterPosition + const reorderLength: number = req.body.reorderLength + + if (startPosition >= nextPosition || insertAfterPosition >= nextPosition) { + res.fail({ message: `Start position or insert after position exceed the playlist limits (max: ${nextPosition - 1})` }) + return + } + + if (reorderLength && reorderLength + startPosition > nextPosition) { + res.fail({ message: `Reorder length with this start position exceeds the playlist limits (max: ${nextPosition - startPosition})` }) + return + } + + return next() + } +] + +const commonVideoPlaylistFiltersValidator = [ + query('playlistType') + .optional() + .custom(isVideoPlaylistTypeValid), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +const doVideosInPlaylistExistValidator = [ + query('videoIds') + .customSanitizer(toIntArray) + .custom(v => isArrayOf(v, isIdValid)).withMessage('Should have a valid video ids array'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + videoPlaylistsAddValidator, + videoPlaylistsUpdateValidator, + videoPlaylistsDeleteValidator, + videoPlaylistsGetValidator, + videoPlaylistsSearchValidator, + + videoPlaylistsAddVideoValidator, + videoPlaylistsUpdateOrRemoveVideoValidator, + videoPlaylistsReorderVideosValidator, + + videoPlaylistElementAPGetValidator, + + commonVideoPlaylistFiltersValidator, + + doVideosInPlaylistExistValidator +} + +// --------------------------------------------------------------------------- + +function getCommonPlaylistEditAttributes () { + return [ + body('thumbnailfile') + .custom((value, { req }) => isVideoImageValid(req.files, 'thumbnailfile')) + .withMessage( + 'This thumbnail file is not supported or too large. Please, make sure it is of the following type: ' + + CONSTRAINTS_FIELDS.VIDEO_PLAYLISTS.IMAGE.EXTNAME.join(', ') + ), + + body('description') + .optional() + .customSanitizer(toValueOrNull) + .custom(isVideoPlaylistDescriptionValid), + body('privacy') + .optional() + .customSanitizer(toIntOrNull) + .custom(isVideoPlaylistPrivacyValid), + body('videoChannelId') + .optional() + .customSanitizer(toIntOrNull) + ] as (ValidationChain | ExpressPromiseHandler)[] +} + +function checkUserCanManageVideoPlaylist ( + user: MUserAccountId, + videoPlaylist: MVideoPlaylist, + right: UserRightType, + res: express.Response +) { + if (videoPlaylist.isOwned() === false) { + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Cannot manage video playlist of another server.' + }) + return false + } + + // Check if the user can manage the video playlist + // The user can delete it if s/he is an admin + // Or if s/he is the video playlist's owner + if (user.hasRight(right) === false && videoPlaylist.ownerAccountId !== user.Account.id) { + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Cannot manage video playlist of another user' + }) + return false + } + + return true +} + +function getPlaylist (res: express.Response) { + return res.locals.videoPlaylistFull || res.locals.videoPlaylistSummary +} diff --git a/server/server/middlewares/validators/videos/video-rates.ts b/server/server/middlewares/validators/videos/video-rates.ts new file mode 100644 index 000000000..f7b784fe3 --- /dev/null +++ b/server/server/middlewares/validators/videos/video-rates.ts @@ -0,0 +1,71 @@ +import express from 'express' +import { body, param, query } from 'express-validator' +import { HttpStatusCode, VideoRateType } from '@peertube/peertube-models' +import { isAccountNameValid } from '../../../helpers/custom-validators/accounts.js' +import { isIdValid } from '../../../helpers/custom-validators/misc.js' +import { isRatingValid } from '../../../helpers/custom-validators/video-rates.js' +import { isVideoRatingTypeValid } from '../../../helpers/custom-validators/videos.js' +import { AccountVideoRateModel } from '../../../models/account/account-video-rate.js' +import { areValidationErrors, checkCanSeeVideo, doesVideoExist, isValidVideoIdParam, isValidVideoPasswordHeader } from '../shared/index.js' + +const videoUpdateRateValidator = [ + isValidVideoIdParam('id'), + + body('rating') + .custom(isVideoRatingTypeValid), + isValidVideoPasswordHeader(), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.id, res)) return + + if (!await checkCanSeeVideo({ req, res, paramId: req.params.id, video: res.locals.videoAll })) return + + return next() + } +] + +const getAccountVideoRateValidatorFactory = function (rateType: VideoRateType) { + return [ + param('name') + .custom(isAccountNameValid), + param('videoId') + .custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const rate = await AccountVideoRateModel.loadLocalAndPopulateVideo(rateType, req.params.name, +req.params.videoId) + if (!rate) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video rate not found' + }) + } + + res.locals.accountVideoRate = rate + + return next() + } + ] +} + +const videoRatingValidator = [ + query('rating') + .optional() + .custom(isRatingValid).withMessage('Value must be one of "like" or "dislike"'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + videoUpdateRateValidator, + getAccountVideoRateValidatorFactory, + videoRatingValidator +} diff --git a/server/server/middlewares/validators/videos/video-shares.ts b/server/server/middlewares/validators/videos/video-shares.ts new file mode 100644 index 000000000..1ecf0fd04 --- /dev/null +++ b/server/server/middlewares/validators/videos/video-shares.ts @@ -0,0 +1,35 @@ +import express from 'express' +import { param } from 'express-validator' +import { HttpStatusCode } from '@peertube/peertube-models' +import { isIdValid } from '../../../helpers/custom-validators/misc.js' +import { VideoShareModel } from '../../../models/video/video-share.js' +import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared/index.js' + +const videosShareValidator = [ + isValidVideoIdParam('id'), + + param('actorId') + .custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.id, res)) return + + const video = res.locals.videoAll + + const share = await VideoShareModel.load(req.params.actorId, video.id) + if (!share) { + return res.status(HttpStatusCode.NOT_FOUND_404) + .end() + } + + res.locals.videoShare = share + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + videosShareValidator +} diff --git a/server/server/middlewares/validators/videos/video-source.ts b/server/server/middlewares/validators/videos/video-source.ts new file mode 100644 index 000000000..3d4f77e21 --- /dev/null +++ b/server/server/middlewares/validators/videos/video-source.ts @@ -0,0 +1,130 @@ +import express from 'express' +import { body, header } from 'express-validator' +import { getResumableUploadPath } from '@server/helpers/upload.js' +import { getVideoWithAttributes } from '@server/helpers/video.js' +import { CONFIG } from '@server/initializers/config.js' +import { uploadx } from '@server/lib/uploadx.js' +import { VideoSourceModel } from '@server/models/video/video-source.js' +import { MVideoFullLight } from '@server/types/models/index.js' +import { HttpStatusCode, UserRight } from '@peertube/peertube-models' +import { Metadata as UploadXMetadata } from '@uploadx/core' +import { logger } from '../../../helpers/logger.js' +import { areValidationErrors, checkUserCanManageVideo, doesVideoExist, isValidVideoIdParam } from '../shared/index.js' +import { addDurationToVideoFileIfNeeded, checkVideoFileCanBeEdited, commonVideoFileChecks, isVideoFileAccepted } from './shared/index.js' + +export const videoSourceGetLatestValidator = [ + isValidVideoIdParam('id'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.id, res, 'all')) return + + const video = getVideoWithAttributes(res) as MVideoFullLight + + const user = res.locals.oauth.token.User + if (!checkUserCanManageVideo(user, video, UserRight.UPDATE_ANY_VIDEO, res)) return + + res.locals.videoSource = await VideoSourceModel.loadLatest(video.id) + + if (!res.locals.videoSource) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Video source not found' + }) + } + + return next() + } +] + +export const replaceVideoSourceResumableValidator = [ + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + const body: express.CustomUploadXFile = req.body + const file = { ...body, duration: undefined, path: getResumableUploadPath(body.name), filename: body.metadata.filename } + const cleanup = () => uploadx.storage.delete(file).catch(err => logger.error('Cannot delete the file %s', file.name, { err })) + + if (!await checkCanUpdateVideoFile({ req, res })) { + return cleanup() + } + + if (!await addDurationToVideoFileIfNeeded({ videoFile: file, res, middlewareName: 'updateVideoFileResumableValidator' })) { + return cleanup() + } + + if (!await isVideoFileAccepted({ req, res, videoFile: file, hook: 'filter:api.video.update-file.accept.result' })) { + return cleanup() + } + + res.locals.updateVideoFileResumable = { ...file, originalname: file.filename } + + return next() + } +] + +export const replaceVideoSourceResumableInitValidator = [ + body('filename') + .exists(), + + header('x-upload-content-length') + .isNumeric() + .exists() + .withMessage('Should specify the file length'), + header('x-upload-content-type') + .isString() + .exists() + .withMessage('Should specify the file mimetype'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + const user = res.locals.oauth.token.User + + logger.debug('Checking updateVideoFileResumableInitValidator parameters and headers', { + parameters: req.body, + headers: req.headers + }) + + if (areValidationErrors(req, res, { omitLog: true })) return + + if (!await checkCanUpdateVideoFile({ req, res })) return + + const videoFileMetadata = { + mimetype: req.headers['x-upload-content-type'] as string, + size: +req.headers['x-upload-content-length'], + originalname: req.body.filename + } + + const files = { videofile: [ videoFileMetadata ] } + if (await commonVideoFileChecks({ res, user, videoFileSize: videoFileMetadata.size, files }) === false) return + + return next() + } +] + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +async function checkCanUpdateVideoFile (options: { + req: express.Request + res: express.Response +}) { + const { req, res } = options + + if (!CONFIG.VIDEO_FILE.UPDATE.ENABLED) { + res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Updating the file of an existing video is not allowed on this instance' + }) + return false + } + + if (!await doesVideoExist(req.params.id, res)) return false + + const user = res.locals.oauth.token.User + const video = res.locals.videoAll + + if (!checkUserCanManageVideo(user, video, UserRight.UPDATE_ANY_VIDEO, res)) return false + + if (!checkVideoFileCanBeEdited(video, res)) return false + + return true +} diff --git a/server/server/middlewares/validators/videos/video-stats.ts b/server/server/middlewares/validators/videos/video-stats.ts new file mode 100644 index 000000000..c490c8f82 --- /dev/null +++ b/server/server/middlewares/validators/videos/video-stats.ts @@ -0,0 +1,108 @@ +import express from 'express' +import { param, query } from 'express-validator' +import { isDateValid } from '@server/helpers/custom-validators/misc.js' +import { isValidStatTimeserieMetric } from '@server/helpers/custom-validators/video-stats.js' +import { STATS_TIMESERIE } from '@server/initializers/constants.js' +import { HttpStatusCode, UserRight, VideoStatsTimeserieQuery } from '@peertube/peertube-models' +import { areValidationErrors, checkUserCanManageVideo, doesVideoExist, isValidVideoIdParam } from '../shared/index.js' + +const videoOverallStatsValidator = [ + isValidVideoIdParam('videoId'), + + query('startDate') + .optional() + .custom(isDateValid), + + query('endDate') + .optional() + .custom(isDateValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await commonStatsCheck(req, res)) return + + return next() + } +] + +const videoRetentionStatsValidator = [ + isValidVideoIdParam('videoId'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await commonStatsCheck(req, res)) return + + if (res.locals.videoAll.isLive) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Cannot get retention stats of live video' + }) + } + + return next() + } +] + +const videoTimeserieStatsValidator = [ + isValidVideoIdParam('videoId'), + + param('metric') + .custom(isValidStatTimeserieMetric), + + query('startDate') + .optional() + .custom(isDateValid), + + query('endDate') + .optional() + .custom(isDateValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await commonStatsCheck(req, res)) return + + const query: VideoStatsTimeserieQuery = req.query + if ( + (query.startDate && !query.endDate) || + (!query.startDate && query.endDate) + ) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Both start date and end date should be defined if one of them is specified' + }) + } + + if (query.startDate && getIntervalByDays(query.startDate, query.endDate) > STATS_TIMESERIE.MAX_DAYS) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Star date and end date interval is too big' + }) + } + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + videoOverallStatsValidator, + videoTimeserieStatsValidator, + videoRetentionStatsValidator +} + +// --------------------------------------------------------------------------- + +async function commonStatsCheck (req: express.Request, res: express.Response) { + if (!await doesVideoExist(req.params.videoId, res, 'all')) return false + if (!checkUserCanManageVideo(res.locals.oauth.token.User, res.locals.videoAll, UserRight.SEE_ALL_VIDEOS, res)) return false + + return true +} + +function getIntervalByDays (startDateString: string, endDateString: string) { + const startDate = new Date(startDateString) + const endDate = new Date(endDateString) + + return (endDate.getTime() - startDate.getTime()) / 1000 / 86400 +} diff --git a/server/server/middlewares/validators/videos/video-studio.ts b/server/server/middlewares/validators/videos/video-studio.ts new file mode 100644 index 000000000..6f6f2dc54 --- /dev/null +++ b/server/server/middlewares/validators/videos/video-studio.ts @@ -0,0 +1,105 @@ +import express from 'express' +import { body, param } from 'express-validator' +import { isIdOrUUIDValid } from '@server/helpers/custom-validators/misc.js' +import { + isStudioCutTaskValid, + isStudioTaskAddIntroOutroValid, + isStudioTaskAddWatermarkValid, + isValidStudioTasksArray +} from '@server/helpers/custom-validators/video-studio.js' +import { cleanUpReqFiles } from '@server/helpers/express-utils.js' +import { CONFIG } from '@server/initializers/config.js' +import { approximateIntroOutroAdditionalSize, getTaskFileFromReq } from '@server/lib/video-studio.js' +import { isAudioFile } from '@peertube/peertube-ffmpeg' +import { HttpStatusCode, UserRight, VideoStudioCreateEdition, VideoStudioTask } from '@peertube/peertube-models' +import { areValidationErrors, checkUserCanManageVideo, checkUserQuota, doesVideoExist } from '../shared/index.js' +import { checkVideoFileCanBeEdited } from './shared/index.js' + +const videoStudioAddEditionValidator = [ + param('videoId') + .custom(isIdOrUUIDValid).withMessage('Should have a valid video id/uuid/short uuid'), + + body('tasks') + .custom(isValidStudioTasksArray).withMessage('Should have a valid array of tasks'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (CONFIG.VIDEO_STUDIO.ENABLED !== true) { + res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Video studio is disabled on this instance' + }) + + return cleanUpReqFiles(req) + } + + if (areValidationErrors(req, res)) return cleanUpReqFiles(req) + + const body: VideoStudioCreateEdition = req.body + const files = req.files as Express.Multer.File[] + + for (let i = 0; i < body.tasks.length; i++) { + const task = body.tasks[i] + + if (!checkTask(req, task, i)) { + res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: `Task ${task.name} is invalid` + }) + + return cleanUpReqFiles(req) + } + + if (task.name === 'add-intro' || task.name === 'add-outro') { + const filePath = getTaskFileFromReq(files, i).path + + // Our concat filter needs a video stream + if (await isAudioFile(filePath)) { + res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: `Task ${task.name} is invalid: file does not contain a video stream` + }) + + return cleanUpReqFiles(req) + } + } + } + + if (!await doesVideoExist(req.params.videoId, res)) return cleanUpReqFiles(req) + + const video = res.locals.videoAll + if (!checkVideoFileCanBeEdited(video, res)) return cleanUpReqFiles(req) + + const user = res.locals.oauth.token.User + if (!checkUserCanManageVideo(user, video, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req) + + // Try to make an approximation of bytes added by the intro/outro + const additionalBytes = await approximateIntroOutroAdditionalSize(video, body.tasks, i => getTaskFileFromReq(files, i).path) + if (await checkUserQuota(user, additionalBytes, res) === false) return cleanUpReqFiles(req) + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + videoStudioAddEditionValidator +} + +// --------------------------------------------------------------------------- + +const taskCheckers: { + [id in VideoStudioTask['name']]: (task: VideoStudioTask, indice?: number, files?: Express.Multer.File[]) => boolean +} = { + 'cut': isStudioCutTaskValid, + 'add-intro': isStudioTaskAddIntroOutroValid, + 'add-outro': isStudioTaskAddIntroOutroValid, + 'add-watermark': isStudioTaskAddWatermarkValid +} + +function checkTask (req: express.Request, task: VideoStudioTask, indice?: number) { + const checker = taskCheckers[task.name] + if (!checker) return false + + return checker(task, indice, req.files as Express.Multer.File[]) +} diff --git a/server/server/middlewares/validators/videos/video-token.ts b/server/server/middlewares/validators/videos/video-token.ts new file mode 100644 index 000000000..748913755 --- /dev/null +++ b/server/server/middlewares/validators/videos/video-token.ts @@ -0,0 +1,23 @@ +import express from 'express' +import { HttpStatusCode, VideoPrivacy } from '@peertube/peertube-models' +import { exists } from '@server/helpers/custom-validators/misc.js' + +const videoFileTokenValidator = [ + (req: express.Request, res: express.Response, next: express.NextFunction) => { + const video = res.locals.onlyVideo + if (video.privacy !== VideoPrivacy.PASSWORD_PROTECTED && !exists(res.locals.oauth.token.User)) { + return res.fail({ + status: HttpStatusCode.UNAUTHORIZED_401, + message: 'Not authenticated' + }) + } + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + videoFileTokenValidator +} diff --git a/server/server/middlewares/validators/videos/video-transcoding.ts b/server/server/middlewares/validators/videos/video-transcoding.ts new file mode 100644 index 000000000..28f057979 --- /dev/null +++ b/server/server/middlewares/validators/videos/video-transcoding.ts @@ -0,0 +1,61 @@ +import express from 'express' +import { body } from 'express-validator' +import { isBooleanValid, toBooleanOrNull } from '@server/helpers/custom-validators/misc.js' +import { isValidCreateTranscodingType } from '@server/helpers/custom-validators/video-transcoding.js' +import { CONFIG } from '@server/initializers/config.js' +import { VideoJobInfoModel } from '@server/models/video/video-job-info.js' +import { HttpStatusCode, ServerErrorCode, VideoTranscodingCreate } from '@peertube/peertube-models' +import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared/index.js' + +const createTranscodingValidator = [ + isValidVideoIdParam('videoId'), + + body('transcodingType') + .custom(isValidCreateTranscodingType), + + body('forceTranscoding') + .optional() + .customSanitizer(toBooleanOrNull) + .custom(isBooleanValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.videoId, res, 'all')) return + + const video = res.locals.videoAll + + if (video.remote) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Cannot run transcoding job on a remote video' + }) + } + + if (CONFIG.TRANSCODING.ENABLED !== true) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Cannot run transcoding job because transcoding is disabled on this instance' + }) + } + + const body = req.body as VideoTranscodingCreate + if (body.forceTranscoding === true) return next() + + const info = await VideoJobInfoModel.load(video.id) + if (info && info.pendingTranscode > 0) { + return res.fail({ + status: HttpStatusCode.CONFLICT_409, + type: ServerErrorCode.VIDEO_ALREADY_BEING_TRANSCODED, + message: 'This video is already being transcoded' + }) + } + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + createTranscodingValidator +} diff --git a/server/server/middlewares/validators/videos/video-view.ts b/server/server/middlewares/validators/videos/video-view.ts new file mode 100644 index 000000000..14ee1da46 --- /dev/null +++ b/server/server/middlewares/validators/videos/video-view.ts @@ -0,0 +1,61 @@ +import express from 'express' +import { body, param } from 'express-validator' +import { HttpStatusCode } from '@peertube/peertube-models' +import { isVideoTimeValid } from '@server/helpers/custom-validators/video-view.js' +import { getCachedVideoDuration } from '@server/lib/video.js' +import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer.js' +import { isIdValid, isIntOrNull, toIntOrNull } from '../../../helpers/custom-validators/misc.js' +import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared/index.js' + +const getVideoLocalViewerValidator = [ + param('localViewerId') + .custom(isIdValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const localViewer = await LocalVideoViewerModel.loadFullById(+req.params.localViewerId) + if (!localViewer) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Local viewer not found' + }) + } + + res.locals.localViewerFull = localViewer + + return next() + } +] + +const videoViewValidator = [ + isValidVideoIdParam('videoId'), + + body('currentTime') + .customSanitizer(toIntOrNull) + .custom(isIntOrNull), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.videoId, res, 'only-immutable-attributes')) return + + const video = res.locals.onlyImmutableVideo + const { duration } = await getCachedVideoDuration(video.id) + + if (!isVideoTimeValid(req.body.currentTime, duration)) { + return res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Current time is invalid' + }) + } + + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + videoViewValidator, + getVideoLocalViewerValidator +} diff --git a/server/server/middlewares/validators/videos/videos.ts b/server/server/middlewares/validators/videos/videos.ts new file mode 100644 index 000000000..a1f6e72c8 --- /dev/null +++ b/server/server/middlewares/validators/videos/videos.ts @@ -0,0 +1,575 @@ +import express from 'express' +import { body, header, param, query, ValidationChain } from 'express-validator' +import { arrayify } from '@peertube/peertube-core-utils' +import { HttpStatusCode, ServerErrorCode, UserRight, VideoState } from '@peertube/peertube-models' +import { isTestInstance } from '@peertube/peertube-node-utils' +import { getResumableUploadPath } from '@server/helpers/upload.js' +import { Redis } from '@server/lib/redis.js' +import { uploadx } from '@server/lib/uploadx.js' +import { getServerActor } from '@server/models/application/application.js' +import { ExpressPromiseHandler } from '@server/types/express-handler.js' +import { MUserAccountId, MVideoFullLight } from '@server/types/models/index.js' +import { + exists, + isBooleanValid, + isDateValid, + isFileValid, + isIdValid, + toBooleanOrNull, + toIntOrNull, + toValueOrNull +} from '../../../helpers/custom-validators/misc.js' +import { isBooleanBothQueryValid, isNumberArray, isStringArray } from '../../../helpers/custom-validators/search.js' +import { + areVideoTagsValid, + isScheduleVideoUpdatePrivacyValid, + isValidPasswordProtectedPrivacy, + isVideoCategoryValid, + isVideoDescriptionValid, + isVideoImageValid, + isVideoIncludeValid, + isVideoLanguageValid, + isVideoLicenceValid, + isVideoNameValid, + isVideoOriginallyPublishedAtValid, + isVideoPrivacyValid, + isVideoSupportValid +} from '../../../helpers/custom-validators/videos.js' +import { cleanUpReqFiles } from '../../../helpers/express-utils.js' +import { logger } from '../../../helpers/logger.js' +import { getVideoWithAttributes } from '../../../helpers/video.js' +import { CONFIG } from '../../../initializers/config.js' +import { CONSTRAINTS_FIELDS, OVERVIEWS } from '../../../initializers/constants.js' +import { VideoModel } from '../../../models/video/video.js' +import { + areValidationErrors, + checkCanAccessVideoStaticFiles, + checkCanSeeVideo, + checkUserCanManageVideo, + doesVideoChannelOfAccountExist, + doesVideoExist, + doesVideoFileOfVideoExist, + isValidVideoIdParam, + isValidVideoPasswordHeader +} from '../shared/index.js' +import { addDurationToVideoFileIfNeeded, commonVideoFileChecks, isVideoFileAccepted } from './shared/index.js' + +const videosAddLegacyValidator = getCommonVideoEditAttributes().concat([ + body('videofile') + .custom((_, { req }) => isFileValid({ files: req.files, field: 'videofile', mimeTypeRegex: null, maxSize: null })) + .withMessage('Should have a file'), + body('name') + .trim() + .custom(isVideoNameValid).withMessage( + `Should have a video name between ${CONSTRAINTS_FIELDS.VIDEOS.NAME.min} and ${CONSTRAINTS_FIELDS.VIDEOS.NAME.max} characters long` + ), + body('channelId') + .customSanitizer(toIntOrNull) + .custom(isIdValid), + body('videoPasswords') + .optional() + .isArray() + .withMessage('Video passwords should be an array.'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return cleanUpReqFiles(req) + + const videoFile: express.VideoUploadFile = req.files['videofile'][0] + const user = res.locals.oauth.token.User + + if ( + !await commonVideoChecksPass({ req, res, user, videoFileSize: videoFile.size, files: req.files }) || + !isValidPasswordProtectedPrivacy(req, res) || + !await addDurationToVideoFileIfNeeded({ videoFile, res, middlewareName: 'videosAddvideosAddLegacyValidatorResumableValidator' }) || + !await isVideoFileAccepted({ req, res, videoFile, hook: 'filter:api.video.upload.accept.result' }) + ) { + return cleanUpReqFiles(req) + } + + return next() + } +]) + +/** + * Gets called after the last PUT request + */ +const videosAddResumableValidator = [ + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + const user = res.locals.oauth.token.User + const body: express.CustomUploadXFile = req.body + const file = { ...body, duration: undefined, path: getResumableUploadPath(body.name), filename: body.metadata.filename } + const cleanup = () => uploadx.storage.delete(file).catch(err => logger.error('Cannot delete the file %s', file.name, { err })) + + const uploadId = req.query.upload_id + const sessionExists = await Redis.Instance.doesUploadSessionExist(uploadId) + + if (sessionExists) { + const sessionResponse = await Redis.Instance.getUploadSession(uploadId) + + if (!sessionResponse) { + res.setHeader('Retry-After', 300) // ask to retry after 5 min, knowing the upload_id is kept for up to 15 min after completion + + return res.fail({ + status: HttpStatusCode.SERVICE_UNAVAILABLE_503, + message: 'The upload is already being processed' + }) + } + + const videoStillExists = await VideoModel.load(sessionResponse.video.id) + + if (videoStillExists) { + if (isTestInstance()) { + res.setHeader('x-resumable-upload-cached', 'true') + } + + return res.json(sessionResponse) + } + } + + await Redis.Instance.setUploadSession(uploadId) + + if (!await doesVideoChannelOfAccountExist(file.metadata.channelId, user, res)) return cleanup() + if (!await addDurationToVideoFileIfNeeded({ videoFile: file, res, middlewareName: 'videosAddResumableValidator' })) return cleanup() + if (!await isVideoFileAccepted({ req, res, videoFile: file, hook: 'filter:api.video.upload.accept.result' })) return cleanup() + + res.locals.uploadVideoFileResumable = { ...file, originalname: file.filename } + + return next() + } +] + +/** + * File is created in POST initialisation, and its body is saved as a 'metadata' field is saved by uploadx for later use. + * see https://github.com/kukhariev/node-uploadx/blob/dc9fb4a8ac5a6f481902588e93062f591ec6ef03/packages/core/src/handlers/uploadx.ts + * + * Uploadx doesn't use next() until the upload completes, so this middleware has to be placed before uploadx + * see https://github.com/kukhariev/node-uploadx/blob/dc9fb4a8ac5a6f481902588e93062f591ec6ef03/packages/core/src/handlers/base-handler.ts + * + */ +const videosAddResumableInitValidator = getCommonVideoEditAttributes().concat([ + body('filename') + .exists(), + body('name') + .trim() + .custom(isVideoNameValid).withMessage( + `Should have a video name between ${CONSTRAINTS_FIELDS.VIDEOS.NAME.min} and ${CONSTRAINTS_FIELDS.VIDEOS.NAME.max} characters long` + ), + body('channelId') + .customSanitizer(toIntOrNull) + .custom(isIdValid), + body('videoPasswords') + .optional() + .isArray() + .withMessage('Video passwords should be an array.'), + + header('x-upload-content-length') + .isNumeric() + .exists() + .withMessage('Should specify the file length'), + header('x-upload-content-type') + .isString() + .exists() + .withMessage('Should specify the file mimetype'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + const videoFileMetadata = { + mimetype: req.headers['x-upload-content-type'] as string, + size: +req.headers['x-upload-content-length'], + originalname: req.body.filename + } + + const user = res.locals.oauth.token.User + const cleanup = () => cleanUpReqFiles(req) + + logger.debug('Checking videosAddResumableInitValidator parameters and headers', { + parameters: req.body, + headers: req.headers, + files: req.files + }) + + if (areValidationErrors(req, res, { omitLog: true })) return cleanup() + + const files = { videofile: [ videoFileMetadata ] } + if (!await commonVideoChecksPass({ req, res, user, videoFileSize: videoFileMetadata.size, files })) return cleanup() + + if (!isValidPasswordProtectedPrivacy(req, res)) return cleanup() + + // Multer required unsetting the Content-Type, now we can set it for node-uploadx + req.headers['content-type'] = 'application/json; charset=utf-8' + + // Place thumbnail/previewfile in metadata so that uploadx saves it in .META + if (req.files?.['previewfile']) req.body.previewfile = req.files['previewfile'] + if (req.files?.['thumbnailfile']) req.body.thumbnailfile = req.files['thumbnailfile'] + + return next() + } +]) + +const videosUpdateValidator = getCommonVideoEditAttributes().concat([ + isValidVideoIdParam('id'), + + body('name') + .optional() + .trim() + .custom(isVideoNameValid).withMessage( + `Should have a video name between ${CONSTRAINTS_FIELDS.VIDEOS.NAME.min} and ${CONSTRAINTS_FIELDS.VIDEOS.NAME.max} characters long` + ), + body('channelId') + .optional() + .customSanitizer(toIntOrNull) + .custom(isIdValid), + body('videoPasswords') + .optional() + .isArray() + .withMessage('Video passwords should be an array.'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return cleanUpReqFiles(req) + if (areErrorsInScheduleUpdate(req, res)) return cleanUpReqFiles(req) + if (!await doesVideoExist(req.params.id, res)) return cleanUpReqFiles(req) + + if (!isValidPasswordProtectedPrivacy(req, res)) return cleanUpReqFiles(req) + + const video = getVideoWithAttributes(res) + if (video.isLive && video.privacy !== req.body.privacy && video.state !== VideoState.WAITING_FOR_LIVE) { + return res.fail({ message: 'Cannot update privacy of a live that has already started' }) + } + + // Check if the user who did the request is able to update the video + const user = res.locals.oauth.token.User + if (!checkUserCanManageVideo(user, res.locals.videoAll, UserRight.UPDATE_ANY_VIDEO, res)) return cleanUpReqFiles(req) + + if (req.body.channelId && !await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) + + return next() + } +]) + +async function checkVideoFollowConstraints (req: express.Request, res: express.Response, next: express.NextFunction) { + const video = getVideoWithAttributes(res) + + // Anybody can watch local videos + if (video.isOwned() === true) return next() + + // Logged user + if (res.locals.oauth) { + // Users can search or watch remote videos + if (CONFIG.SEARCH.REMOTE_URI.USERS === true) return next() + } + + // Anybody can search or watch remote videos + if (CONFIG.SEARCH.REMOTE_URI.ANONYMOUS === true) return next() + + // Check our instance follows an actor that shared this video + const serverActor = await getServerActor() + if (await VideoModel.checkVideoHasInstanceFollow(video.id, serverActor.id) === true) return next() + + return res.fail({ + status: HttpStatusCode.FORBIDDEN_403, + message: 'Cannot get this video regarding follow constraints', + type: ServerErrorCode.DOES_NOT_RESPECT_FOLLOW_CONSTRAINTS, + data: { + originUrl: video.url + } + }) +} + +const videosCustomGetValidator = (fetchType: 'for-api' | 'all' | 'only-video' | 'only-immutable-attributes') => { + return [ + isValidVideoIdParam('id'), + + isValidVideoPasswordHeader(), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.id, res, fetchType)) return + + // Controllers does not need to check video rights + if (fetchType === 'only-immutable-attributes') return next() + + const video = getVideoWithAttributes(res) as MVideoFullLight + + if (!await checkCanSeeVideo({ req, res, video, paramId: req.params.id })) return + + return next() + } + ] +} + +const videosGetValidator = videosCustomGetValidator('all') + +const videoFileMetadataGetValidator = getCommonVideoEditAttributes().concat([ + isValidVideoIdParam('id'), + + param('videoFileId') + .custom(isIdValid).not().isEmpty().withMessage('Should have a valid videoFileId'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoFileOfVideoExist(+req.params.videoFileId, req.params.id, res)) return + + return next() + } +]) + +const videosDownloadValidator = [ + isValidVideoIdParam('id'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.id, res, 'all')) return + + const video = getVideoWithAttributes(res) + + if (!await checkCanAccessVideoStaticFiles({ req, res, video, paramId: req.params.id })) return + + return next() + } +] + +const videosRemoveValidator = [ + isValidVideoIdParam('id'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + if (!await doesVideoExist(req.params.id, res)) return + + // Check if the user who did the request is able to delete the video + if (!checkUserCanManageVideo(res.locals.oauth.token.User, res.locals.videoAll, UserRight.REMOVE_ANY_VIDEO, res)) return + + return next() + } +] + +const videosOverviewValidator = [ + query('page') + .optional() + .isInt({ min: 1, max: OVERVIEWS.VIDEOS.SAMPLES_COUNT }), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + return next() + } +] + +function getCommonVideoEditAttributes () { + return [ + body('thumbnailfile') + .custom((value, { req }) => isVideoImageValid(req.files, 'thumbnailfile')).withMessage( + 'This thumbnail file is not supported or too large. Please, make sure it is of the following type: ' + + CONSTRAINTS_FIELDS.VIDEOS.IMAGE.EXTNAME.join(', ') + ), + body('previewfile') + .custom((value, { req }) => isVideoImageValid(req.files, 'previewfile')).withMessage( + 'This preview file is not supported or too large. Please, make sure it is of the following type: ' + + CONSTRAINTS_FIELDS.VIDEOS.IMAGE.EXTNAME.join(', ') + ), + + body('category') + .optional() + .customSanitizer(toIntOrNull) + .custom(isVideoCategoryValid), + body('licence') + .optional() + .customSanitizer(toIntOrNull) + .custom(isVideoLicenceValid), + body('language') + .optional() + .customSanitizer(toValueOrNull) + .custom(isVideoLanguageValid), + body('nsfw') + .optional() + .customSanitizer(toBooleanOrNull) + .custom(isBooleanValid).withMessage('Should have a valid nsfw boolean'), + body('waitTranscoding') + .optional() + .customSanitizer(toBooleanOrNull) + .custom(isBooleanValid).withMessage('Should have a valid waitTranscoding boolean'), + body('privacy') + .optional() + .customSanitizer(toIntOrNull) + .custom(isVideoPrivacyValid), + body('description') + .optional() + .customSanitizer(toValueOrNull) + .custom(isVideoDescriptionValid), + body('support') + .optional() + .customSanitizer(toValueOrNull) + .custom(isVideoSupportValid), + body('tags') + .optional() + .customSanitizer(toValueOrNull) + .custom(areVideoTagsValid) + .withMessage( + `Should have an array of up to ${CONSTRAINTS_FIELDS.VIDEOS.TAGS.max} tags between ` + + `${CONSTRAINTS_FIELDS.VIDEOS.TAG.min} and ${CONSTRAINTS_FIELDS.VIDEOS.TAG.max} characters each` + ), + body('commentsEnabled') + .optional() + .customSanitizer(toBooleanOrNull) + .custom(isBooleanValid).withMessage('Should have commentsEnabled boolean'), + body('downloadEnabled') + .optional() + .customSanitizer(toBooleanOrNull) + .custom(isBooleanValid).withMessage('Should have downloadEnabled boolean'), + body('originallyPublishedAt') + .optional() + .customSanitizer(toValueOrNull) + .custom(isVideoOriginallyPublishedAtValid), + body('scheduleUpdate') + .optional() + .customSanitizer(toValueOrNull), + body('scheduleUpdate.updateAt') + .optional() + .custom(isDateValid).withMessage('Should have a schedule update date that conforms to ISO 8601'), + body('scheduleUpdate.privacy') + .optional() + .customSanitizer(toIntOrNull) + .custom(isScheduleVideoUpdatePrivacyValid) + ] as (ValidationChain | ExpressPromiseHandler)[] +} + +const commonVideosFiltersValidator = [ + query('categoryOneOf') + .optional() + .customSanitizer(arrayify) + .custom(isNumberArray).withMessage('Should have a valid categoryOneOf array'), + query('licenceOneOf') + .optional() + .customSanitizer(arrayify) + .custom(isNumberArray).withMessage('Should have a valid licenceOneOf array'), + query('languageOneOf') + .optional() + .customSanitizer(arrayify) + .custom(isStringArray).withMessage('Should have a valid languageOneOf array'), + query('privacyOneOf') + .optional() + .customSanitizer(arrayify) + .custom(isNumberArray).withMessage('Should have a valid privacyOneOf array'), + query('tagsOneOf') + .optional() + .customSanitizer(arrayify) + .custom(isStringArray).withMessage('Should have a valid tagsOneOf array'), + query('tagsAllOf') + .optional() + .customSanitizer(arrayify) + .custom(isStringArray).withMessage('Should have a valid tagsAllOf array'), + query('nsfw') + .optional() + .custom(isBooleanBothQueryValid), + query('isLive') + .optional() + .customSanitizer(toBooleanOrNull) + .custom(isBooleanValid).withMessage('Should have a valid isLive boolean'), + query('include') + .optional() + .custom(isVideoIncludeValid), + query('isLocal') + .optional() + .customSanitizer(toBooleanOrNull) + .custom(isBooleanValid).withMessage('Should have a valid isLocal boolean'), + query('hasHLSFiles') + .optional() + .customSanitizer(toBooleanOrNull) + .custom(isBooleanValid).withMessage('Should have a valid hasHLSFiles boolean'), + query('hasWebtorrentFiles') // TODO: remove in v7 + .optional() + .customSanitizer(toBooleanOrNull) + .custom(isBooleanValid).withMessage('Should have a valid hasWebtorrentFiles boolean'), + query('hasWebVideoFiles') + .optional() + .customSanitizer(toBooleanOrNull) + .custom(isBooleanValid).withMessage('Should have a valid hasWebVideoFiles boolean'), + query('skipCount') + .optional() + .customSanitizer(toBooleanOrNull) + .custom(isBooleanValid).withMessage('Should have a valid skipCount boolean'), + query('search') + .optional() + .custom(exists), + query('excludeAlreadyWatched') + .optional() + .customSanitizer(toBooleanOrNull) + .isBoolean().withMessage('Should be a valid excludeAlreadyWatched boolean'), + + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + const user = res.locals.oauth?.token.User + + if ((!user || user.hasRight(UserRight.SEE_ALL_VIDEOS) !== true)) { + if (req.query.include || req.query.privacyOneOf) { + return res.fail({ + status: HttpStatusCode.UNAUTHORIZED_401, + message: 'You are not allowed to see all videos.' + }) + } + } + + if (!user && exists(req.query.excludeAlreadyWatched)) { + res.fail({ + status: HttpStatusCode.BAD_REQUEST_400, + message: 'Cannot use excludeAlreadyWatched parameter when auth token is not provided' + }) + return false + } + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + videosAddLegacyValidator, + videosAddResumableValidator, + videosAddResumableInitValidator, + + videosUpdateValidator, + videosGetValidator, + videoFileMetadataGetValidator, + videosDownloadValidator, + checkVideoFollowConstraints, + videosCustomGetValidator, + videosRemoveValidator, + + getCommonVideoEditAttributes, + + commonVideosFiltersValidator, + + videosOverviewValidator +} + +// --------------------------------------------------------------------------- + +function areErrorsInScheduleUpdate (req: express.Request, res: express.Response) { + if (req.body.scheduleUpdate) { + if (!req.body.scheduleUpdate.updateAt) { + logger.warn('Invalid parameters: scheduleUpdate.updateAt is mandatory.') + + res.fail({ message: 'Schedule update at is mandatory.' }) + return true + } + } + + return false +} + +async function commonVideoChecksPass (options: { + req: express.Request + res: express.Response + user: MUserAccountId + videoFileSize: number + files: express.UploadFilesForCheck +}): Promise { + const { req, res, user } = options + + if (areErrorsInScheduleUpdate(req, res)) return false + + if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return false + + if (!await commonVideoFileChecks(options)) return false + + return true +} diff --git a/server/server/middlewares/validators/webfinger.ts b/server/server/middlewares/validators/webfinger.ts new file mode 100644 index 000000000..63855bbf3 --- /dev/null +++ b/server/server/middlewares/validators/webfinger.ts @@ -0,0 +1,37 @@ +import express from 'express' +import { query } from 'express-validator' +import { HttpStatusCode } from '@peertube/peertube-models' +import { isWebfingerLocalResourceValid } from '../../helpers/custom-validators/webfinger.js' +import { getHostWithPort } from '../../helpers/express-utils.js' +import { ActorModel } from '../../models/actor/actor.js' +import { areValidationErrors } from './shared/index.js' + +const webfingerValidator = [ + query('resource') + .custom(isWebfingerLocalResourceValid), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res)) return + + // Remove 'acct:' from the beginning of the string + const nameWithHost = getHostWithPort(req.query.resource.substr(5)) + const [ name ] = nameWithHost.split('@') + + const actor = await ActorModel.loadLocalUrlByName(name) + if (!actor) { + return res.fail({ + status: HttpStatusCode.NOT_FOUND_404, + message: 'Actor not found' + }) + } + + res.locals.actorUrl = actor + return next() + } +] + +// --------------------------------------------------------------------------- + +export { + webfingerValidator +} diff --git a/server/server/models/abuse/abuse-message.ts b/server/server/models/abuse/abuse-message.ts new file mode 100644 index 000000000..9ba080e77 --- /dev/null +++ b/server/server/models/abuse/abuse-message.ts @@ -0,0 +1,114 @@ +import { FindOptions } from 'sequelize' +import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { isAbuseMessageValid } from '@server/helpers/custom-validators/abuses.js' +import { MAbuseMessage, MAbuseMessageFormattable } from '@server/types/models/index.js' +import { AbuseMessage } from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { AccountModel, ScopeNames as AccountScopeNames } from '../account/account.js' +import { getSort, throwIfNotValid } from '../shared/index.js' +import { AbuseModel } from './abuse.js' + +@Table({ + tableName: 'abuseMessage', + indexes: [ + { + fields: [ 'abuseId' ] + }, + { + fields: [ 'accountId' ] + } + ] +}) +export class AbuseMessageModel extends Model>> { + + @AllowNull(false) + @Is('AbuseMessage', value => throwIfNotValid(value, isAbuseMessageValid, 'message')) + @Column(DataType.TEXT) + message: string + + @AllowNull(false) + @Column + byModerator: boolean + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => AccountModel) + @Column + accountId: number + + @BelongsTo(() => AccountModel, { + foreignKey: { + name: 'accountId', + allowNull: true + }, + onDelete: 'set null' + }) + Account: Awaited + + @ForeignKey(() => AbuseModel) + @Column + abuseId: number + + @BelongsTo(() => AbuseModel, { + foreignKey: { + name: 'abuseId', + allowNull: false + }, + onDelete: 'cascade' + }) + Abuse: Awaited + + static listForApi (abuseId: number) { + const getQuery = (forCount: boolean) => { + const query: FindOptions = { + where: { abuseId }, + order: getSort('createdAt') + } + + if (forCount !== true) { + query.include = [ + { + model: AccountModel.scope(AccountScopeNames.SUMMARY), + required: false + } + ] + } + + return query + } + + return Promise.all([ + AbuseMessageModel.count(getQuery(true)), + AbuseMessageModel.findAll(getQuery(false)) + ]).then(([ total, data ]) => ({ total, data })) + } + + static loadByIdAndAbuseId (messageId: number, abuseId: number): Promise { + return AbuseMessageModel.findOne({ + where: { + id: messageId, + abuseId + } + }) + } + + toFormattedJSON (this: MAbuseMessageFormattable): AbuseMessage { + const account = this.Account + ? this.Account.toFormattedSummaryJSON() + : null + + return { + id: this.id, + createdAt: this.createdAt, + + byModerator: this.byModerator, + message: this.message, + + account + } + } +} diff --git a/server/server/models/abuse/abuse.ts b/server/server/models/abuse/abuse.ts new file mode 100644 index 000000000..8a8e292e4 --- /dev/null +++ b/server/server/models/abuse/abuse.ts @@ -0,0 +1,631 @@ +import { abusePredefinedReasonsMap } from '@peertube/peertube-core-utils' +import { + AbuseFilter, + AbuseObject, + AbusePredefinedReasonsString, + AbusePredefinedReasonsType, + AbuseVideoIs, + AdminAbuse, + AdminVideoAbuse, + AdminVideoCommentAbuse, + UserAbuse, + UserVideoAbuse, + type AbuseStateType +} from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { isAbuseModerationCommentValid, isAbuseReasonValid, isAbuseStateValid } from '@server/helpers/custom-validators/abuses.js' +import invert from 'lodash-es/invert.js' +import { Op, QueryTypes, literal } from 'sequelize' +import { + AllowNull, + BelongsTo, + Column, + CreatedAt, + DataType, + Default, + ForeignKey, + HasOne, + Is, + Model, + Scopes, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { ABUSE_STATES, CONSTRAINTS_FIELDS } from '../../initializers/constants.js' +import { + MAbuseAP, + MAbuseAdminFormattable, + MAbuseFull, + MAbuseReporter, + MAbuseUserFormattable, + MUserAccountId +} from '../../types/models/index.js' +import { AccountModel, ScopeNames as AccountScopeNames, SummaryOptions as AccountSummaryOptions } from '../account/account.js' +import { getSort, throwIfNotValid } from '../shared/index.js' +import { ThumbnailModel } from '../video/thumbnail.js' +import { VideoBlacklistModel } from '../video/video-blacklist.js' +import { SummaryOptions as ChannelSummaryOptions, VideoChannelModel, ScopeNames as VideoChannelScopeNames } from '../video/video-channel.js' +import { ScopeNames as CommentScopeNames, VideoCommentModel } from '../video/video-comment.js' +import { VideoModel, ScopeNames as VideoScopeNames } from '../video/video.js' +import { BuildAbusesQueryOptions, buildAbuseListQuery } from './sql/abuse-query-builder.js' +import { VideoAbuseModel } from './video-abuse.js' +import { VideoCommentAbuseModel } from './video-comment-abuse.js' + +export enum ScopeNames { + FOR_API = 'FOR_API' +} + +@Scopes(() => ({ + [ScopeNames.FOR_API]: () => { + return { + attributes: { + include: [ + [ + literal( + '(' + + 'SELECT count(*) ' + + 'FROM "abuseMessage" ' + + 'WHERE "abuseId" = "AbuseModel"."id"' + + ')' + ), + 'countMessages' + ], + [ + // we don't care about this count for deleted videos, so there are not included + literal( + '(' + + 'SELECT count(*) ' + + 'FROM "videoAbuse" ' + + 'WHERE "videoId" = "VideoAbuse"."videoId" AND "videoId" IS NOT NULL' + + ')' + ), + 'countReportsForVideo' + ], + [ + // we don't care about this count for deleted videos, so there are not included + literal( + '(' + + 'SELECT t.nth ' + + 'FROM ( ' + + 'SELECT id, ' + + 'row_number() OVER (PARTITION BY "videoId" ORDER BY "createdAt") AS nth ' + + 'FROM "videoAbuse" ' + + ') t ' + + 'WHERE t.id = "VideoAbuse".id AND t.id IS NOT NULL' + + ')' + ), + 'nthReportForVideo' + ], + [ + literal( + '(' + + 'SELECT count("abuse"."id") ' + + 'FROM "abuse" ' + + 'WHERE "abuse"."reporterAccountId" = "AbuseModel"."reporterAccountId"' + + ')' + ), + 'countReportsForReporter' + ], + [ + literal( + '(' + + 'SELECT count("abuse"."id") ' + + 'FROM "abuse" ' + + 'WHERE "abuse"."flaggedAccountId" = "AbuseModel"."flaggedAccountId"' + + ')' + ), + 'countReportsForReportee' + ] + ] + }, + include: [ + { + model: AccountModel.scope({ + method: [ + AccountScopeNames.SUMMARY, + { actorRequired: false } as AccountSummaryOptions + ] + }), + as: 'ReporterAccount' + }, + { + model: AccountModel.scope({ + method: [ + AccountScopeNames.SUMMARY, + { actorRequired: false } as AccountSummaryOptions + ] + }), + as: 'FlaggedAccount' + }, + { + model: VideoCommentAbuseModel.unscoped(), + include: [ + { + model: VideoCommentModel.unscoped(), + include: [ + { + model: VideoModel.unscoped(), + attributes: [ 'name', 'id', 'uuid' ] + } + ] + } + ] + }, + { + model: VideoAbuseModel.unscoped(), + include: [ + { + attributes: [ 'id', 'uuid', 'name', 'nsfw' ], + model: VideoModel.unscoped(), + include: [ + { + attributes: [ 'filename', 'fileUrl', 'type' ], + model: ThumbnailModel + }, + { + model: VideoChannelModel.scope({ + method: [ + VideoChannelScopeNames.SUMMARY, + { withAccount: false, actorRequired: false } as ChannelSummaryOptions + ] + }), + required: false + }, + { + attributes: [ 'id', 'reason', 'unfederated' ], + required: false, + model: VideoBlacklistModel + } + ] + } + ] + } + ] + } + } +})) +@Table({ + tableName: 'abuse', + indexes: [ + { + fields: [ 'reporterAccountId' ] + }, + { + fields: [ 'flaggedAccountId' ] + } + ] +}) +export class AbuseModel extends Model>> { + + @AllowNull(false) + @Default(null) + @Is('AbuseReason', value => throwIfNotValid(value, isAbuseReasonValid, 'reason')) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.ABUSES.REASON.max)) + reason: string + + @AllowNull(false) + @Default(null) + @Is('AbuseState', value => throwIfNotValid(value, isAbuseStateValid, 'state')) + @Column + state: AbuseStateType + + @AllowNull(true) + @Default(null) + @Is('AbuseModerationComment', value => throwIfNotValid(value, isAbuseModerationCommentValid, 'moderationComment', true)) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.ABUSES.MODERATION_COMMENT.max)) + moderationComment: string + + @AllowNull(true) + @Default(null) + @Column(DataType.ARRAY(DataType.INTEGER)) + predefinedReasons: AbusePredefinedReasonsType[] + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => AccountModel) + @Column + reporterAccountId: number + + @BelongsTo(() => AccountModel, { + foreignKey: { + name: 'reporterAccountId', + allowNull: true + }, + as: 'ReporterAccount', + onDelete: 'set null' + }) + ReporterAccount: Awaited + + @ForeignKey(() => AccountModel) + @Column + flaggedAccountId: number + + @BelongsTo(() => AccountModel, { + foreignKey: { + name: 'flaggedAccountId', + allowNull: true + }, + as: 'FlaggedAccount', + onDelete: 'set null' + }) + FlaggedAccount: Awaited + + @HasOne(() => VideoCommentAbuseModel, { + foreignKey: { + name: 'abuseId', + allowNull: false + }, + onDelete: 'cascade' + }) + VideoCommentAbuse: Awaited + + @HasOne(() => VideoAbuseModel, { + foreignKey: { + name: 'abuseId', + allowNull: false + }, + onDelete: 'cascade' + }) + VideoAbuse: Awaited + + static loadByIdWithReporter (id: number): Promise { + const query = { + where: { + id + }, + include: [ + { + model: AccountModel, + as: 'ReporterAccount' + } + ] + } + + return AbuseModel.findOne(query) + } + + static loadFull (id: number): Promise { + const query = { + where: { + id + }, + include: [ + { + model: AccountModel.scope(AccountScopeNames.SUMMARY), + required: false, + as: 'ReporterAccount' + }, + { + model: AccountModel.scope(AccountScopeNames.SUMMARY), + as: 'FlaggedAccount' + }, + { + model: VideoAbuseModel, + required: false, + include: [ + { + model: VideoModel.scope([ VideoScopeNames.WITH_ACCOUNT_DETAILS ]) + } + ] + }, + { + model: VideoCommentAbuseModel, + required: false, + include: [ + { + model: VideoCommentModel.scope([ + CommentScopeNames.WITH_ACCOUNT + ]), + include: [ + { + model: VideoModel + } + ] + } + ] + } + ] + } + + return AbuseModel.findOne(query) + } + + static async listForAdminApi (parameters: { + start: number + count: number + sort: string + + filter?: AbuseFilter + + serverAccountId: number + user?: MUserAccountId + + id?: number + predefinedReason?: AbusePredefinedReasonsString + state?: AbuseStateType + videoIs?: AbuseVideoIs + + search?: string + searchReporter?: string + searchReportee?: string + searchVideo?: string + searchVideoChannel?: string + }) { + const { + start, + count, + sort, + search, + user, + serverAccountId, + state, + videoIs, + predefinedReason, + searchReportee, + searchVideo, + filter, + searchVideoChannel, + searchReporter, + id + } = parameters + + const userAccountId = user ? user.Account.id : undefined + const predefinedReasonId = predefinedReason ? abusePredefinedReasonsMap[predefinedReason] : undefined + + const queryOptions: BuildAbusesQueryOptions = { + start, + count, + sort, + id, + filter, + predefinedReasonId, + search, + state, + videoIs, + searchReportee, + searchVideo, + searchVideoChannel, + searchReporter, + serverAccountId, + userAccountId + } + + const [ total, data ] = await Promise.all([ + AbuseModel.internalCountForApi(queryOptions), + AbuseModel.internalListForApi(queryOptions) + ]) + + return { total, data } + } + + static async listForUserApi (parameters: { + user: MUserAccountId + + start: number + count: number + sort: string + + id?: number + search?: string + state?: AbuseStateType + }) { + const { + start, + count, + sort, + search, + user, + state, + id + } = parameters + + const queryOptions: BuildAbusesQueryOptions = { + start, + count, + sort, + id, + search, + state, + reporterAccountId: user.Account.id + } + + const [ total, data ] = await Promise.all([ + AbuseModel.internalCountForApi(queryOptions), + AbuseModel.internalListForApi(queryOptions) + ]) + + return { total, data } + } + + buildBaseVideoCommentAbuse (this: MAbuseUserFormattable) { + // Associated video comment could have been destroyed if the video has been deleted + if (!this.VideoCommentAbuse?.VideoComment) return null + + const entity = this.VideoCommentAbuse.VideoComment + + return { + id: entity.id, + threadId: entity.getThreadId(), + + text: entity.text ?? '', + + deleted: entity.isDeleted(), + + video: { + id: entity.Video.id, + name: entity.Video.name, + uuid: entity.Video.uuid + } + } + } + + buildBaseVideoAbuse (this: MAbuseUserFormattable): UserVideoAbuse { + if (!this.VideoAbuse) return null + + const abuseModel = this.VideoAbuse + const entity = abuseModel.Video || abuseModel.deletedVideo + + return { + id: entity.id, + uuid: entity.uuid, + name: entity.name, + nsfw: entity.nsfw, + + startAt: abuseModel.startAt, + endAt: abuseModel.endAt, + + deleted: !abuseModel.Video, + blacklisted: abuseModel.Video?.isBlacklisted() || false, + thumbnailPath: abuseModel.Video?.getMiniatureStaticPath(), + + channel: abuseModel.Video?.VideoChannel.toFormattedJSON() || abuseModel.deletedVideo?.channel + } + } + + buildBaseAbuse (this: MAbuseUserFormattable, countMessages: number): UserAbuse { + const predefinedReasons = AbuseModel.getPredefinedReasonsStrings(this.predefinedReasons) + + return { + id: this.id, + reason: this.reason, + predefinedReasons, + + flaggedAccount: this.FlaggedAccount + ? this.FlaggedAccount.toFormattedJSON() + : null, + + state: { + id: this.state, + label: AbuseModel.getStateLabel(this.state) + }, + + countMessages, + + createdAt: this.createdAt, + updatedAt: this.updatedAt + } + } + + toFormattedAdminJSON (this: MAbuseAdminFormattable): AdminAbuse { + const countReportsForVideo = this.get('countReportsForVideo') as number + const nthReportForVideo = this.get('nthReportForVideo') as number + + const countReportsForReporter = this.get('countReportsForReporter') as number + const countReportsForReportee = this.get('countReportsForReportee') as number + + const countMessages = this.get('countMessages') as number + + const baseVideo = this.buildBaseVideoAbuse() + const video: AdminVideoAbuse = baseVideo + ? Object.assign(baseVideo, { + countReports: countReportsForVideo, + nthReport: nthReportForVideo + }) + : null + + const comment: AdminVideoCommentAbuse = this.buildBaseVideoCommentAbuse() + + const abuse = this.buildBaseAbuse(countMessages || 0) + + return Object.assign(abuse, { + video, + comment, + + moderationComment: this.moderationComment, + + reporterAccount: this.ReporterAccount + ? this.ReporterAccount.toFormattedJSON() + : null, + + countReportsForReporter: (countReportsForReporter || 0), + countReportsForReportee: (countReportsForReportee || 0) + }) + } + + toFormattedUserJSON (this: MAbuseUserFormattable): UserAbuse { + const countMessages = this.get('countMessages') as number + + const video = this.buildBaseVideoAbuse() + const comment = this.buildBaseVideoCommentAbuse() + const abuse = this.buildBaseAbuse(countMessages || 0) + + return Object.assign(abuse, { + video, + comment + }) + } + + toActivityPubObject (this: MAbuseAP): AbuseObject { + const predefinedReasons = AbuseModel.getPredefinedReasonsStrings(this.predefinedReasons) + + const object = this.VideoAbuse?.Video?.url || this.VideoCommentAbuse?.VideoComment?.url || this.FlaggedAccount.Actor.url + + const startAt = this.VideoAbuse?.startAt + const endAt = this.VideoAbuse?.endAt + + return { + type: 'Flag' as 'Flag', + content: this.reason, + mediaType: 'text/markdown', + object, + tag: predefinedReasons.map(r => ({ + type: 'Hashtag' as 'Hashtag', + name: r + })), + startAt, + endAt + } + } + + private static async internalCountForApi (parameters: BuildAbusesQueryOptions) { + const { query, replacements } = buildAbuseListQuery(parameters, 'count') + const options = { + type: QueryTypes.SELECT as QueryTypes.SELECT, + replacements + } + + const [ { total } ] = await AbuseModel.sequelize.query<{ total: string }>(query, options) + if (total === null) return 0 + + return parseInt(total, 10) + } + + private static async internalListForApi (parameters: BuildAbusesQueryOptions) { + const { query, replacements } = buildAbuseListQuery(parameters, 'id') + const options = { + type: QueryTypes.SELECT as QueryTypes.SELECT, + replacements + } + + const rows = await AbuseModel.sequelize.query<{ id: string }>(query, options) + const ids = rows.map(r => r.id) + + if (ids.length === 0) return [] + + return AbuseModel.scope(ScopeNames.FOR_API) + .findAll({ + order: getSort(parameters.sort), + where: { + id: { + [Op.in]: ids + } + } + }) + } + + private static getStateLabel (id: number) { + return ABUSE_STATES[id] || 'Unknown' + } + + private static getPredefinedReasonsStrings (predefinedReasons: AbusePredefinedReasonsType[]): AbusePredefinedReasonsString[] { + const invertedPredefinedReasons = invert(abusePredefinedReasonsMap) + + return (predefinedReasons || []) + .map(r => invertedPredefinedReasons[r] as AbusePredefinedReasonsString) + .filter(v => !!v) + } +} diff --git a/server/server/models/abuse/sql/abuse-query-builder.ts b/server/server/models/abuse/sql/abuse-query-builder.ts new file mode 100644 index 000000000..aeed91676 --- /dev/null +++ b/server/server/models/abuse/sql/abuse-query-builder.ts @@ -0,0 +1,167 @@ + +import { forceNumber } from '@peertube/peertube-core-utils' +import { AbuseFilter, AbuseStateType, AbuseVideoIs } from '@peertube/peertube-models' +import { exists } from '@server/helpers/custom-validators/misc.js' +import { buildBlockedAccountSQL, buildSortDirectionAndField } from '../../shared/index.js' + +export type BuildAbusesQueryOptions = { + start: number + count: number + sort: string + + // search + search?: string + searchReporter?: string + searchReportee?: string + + // video related + searchVideo?: string + searchVideoChannel?: string + videoIs?: AbuseVideoIs + + // filters + id?: number + predefinedReasonId?: number + filter?: AbuseFilter + + state?: AbuseStateType + + // accountIds + serverAccountId?: number + userAccountId?: number + + reporterAccountId?: number +} + +function buildAbuseListQuery (options: BuildAbusesQueryOptions, type: 'count' | 'id') { + const whereAnd: string[] = [] + const replacements: any = {} + + const joins = [ + 'LEFT JOIN "videoAbuse" ON "videoAbuse"."abuseId" = "abuse"."id"', + 'LEFT JOIN "video" ON "videoAbuse"."videoId" = "video"."id"', + 'LEFT JOIN "videoBlacklist" ON "videoBlacklist"."videoId" = "video"."id"', + 'LEFT JOIN "videoChannel" ON "video"."channelId" = "videoChannel"."id"', + 'LEFT JOIN "account" "reporterAccount" ON "reporterAccount"."id" = "abuse"."reporterAccountId"', + 'LEFT JOIN "account" "flaggedAccount" ON "flaggedAccount"."id" = "abuse"."flaggedAccountId"', + 'LEFT JOIN "commentAbuse" ON "commentAbuse"."abuseId" = "abuse"."id"', + 'LEFT JOIN "videoComment" ON "commentAbuse"."videoCommentId" = "videoComment"."id"' + ] + + if (options.serverAccountId || options.userAccountId) { + whereAnd.push( + '"abuse"."reporterAccountId" IS NULL OR ' + + '"abuse"."reporterAccountId" NOT IN (' + buildBlockedAccountSQL([ options.serverAccountId, options.userAccountId ]) + ')' + ) + } + + if (options.reporterAccountId) { + whereAnd.push('"abuse"."reporterAccountId" = :reporterAccountId') + replacements.reporterAccountId = options.reporterAccountId + } + + if (options.search) { + const searchWhereOr = [ + '"video"."name" ILIKE :search', + '"videoChannel"."name" ILIKE :search', + `"videoAbuse"."deletedVideo"->>'name' ILIKE :search`, + `"videoAbuse"."deletedVideo"->'channel'->>'displayName' ILIKE :search`, + '"reporterAccount"."name" ILIKE :search', + '"flaggedAccount"."name" ILIKE :search' + ] + + replacements.search = `%${options.search}%` + whereAnd.push('(' + searchWhereOr.join(' OR ') + ')') + } + + if (options.searchVideo) { + whereAnd.push('"video"."name" ILIKE :searchVideo') + replacements.searchVideo = `%${options.searchVideo}%` + } + + if (options.searchVideoChannel) { + whereAnd.push('"videoChannel"."name" ILIKE :searchVideoChannel') + replacements.searchVideoChannel = `%${options.searchVideoChannel}%` + } + + if (options.id) { + whereAnd.push('"abuse"."id" = :id') + replacements.id = options.id + } + + if (options.state) { + whereAnd.push('"abuse"."state" = :state') + replacements.state = options.state + } + + if (options.videoIs === 'deleted') { + whereAnd.push('"videoAbuse"."deletedVideo" IS NOT NULL') + } else if (options.videoIs === 'blacklisted') { + whereAnd.push('"videoBlacklist"."id" IS NOT NULL') + } + + if (options.predefinedReasonId) { + whereAnd.push(':predefinedReasonId = ANY("abuse"."predefinedReasons")') + replacements.predefinedReasonId = options.predefinedReasonId + } + + if (options.filter === 'video') { + whereAnd.push('"videoAbuse"."id" IS NOT NULL') + } else if (options.filter === 'comment') { + whereAnd.push('"commentAbuse"."id" IS NOT NULL') + } else if (options.filter === 'account') { + whereAnd.push('"videoAbuse"."id" IS NULL AND "commentAbuse"."id" IS NULL') + } + + if (options.searchReporter) { + whereAnd.push('"reporterAccount"."name" ILIKE :searchReporter') + replacements.searchReporter = `%${options.searchReporter}%` + } + + if (options.searchReportee) { + whereAnd.push('"flaggedAccount"."name" ILIKE :searchReportee') + replacements.searchReportee = `%${options.searchReportee}%` + } + + const prefix = type === 'count' + ? 'SELECT COUNT("abuse"."id") AS "total"' + : 'SELECT "abuse"."id" ' + + let suffix = '' + if (type !== 'count') { + + if (options.sort) { + const order = buildAbuseOrder(options.sort) + suffix += `${order} ` + } + + if (exists(options.count)) { + const count = forceNumber(options.count) + suffix += `LIMIT ${count} ` + } + + if (exists(options.start)) { + const start = forceNumber(options.start) + suffix += `OFFSET ${start} ` + } + } + + const where = whereAnd.length !== 0 + ? `WHERE ${whereAnd.join(' AND ')}` + : '' + + return { + query: `${prefix} FROM "abuse" ${joins.join(' ')} ${where} ${suffix}`, + replacements + } +} + +function buildAbuseOrder (value: string) { + const { direction, field } = buildSortDirectionAndField(value) + + return `ORDER BY "abuse"."${field}" ${direction}` +} + +export { + buildAbuseListQuery +} diff --git a/server/server/models/abuse/video-abuse.ts b/server/server/models/abuse/video-abuse.ts new file mode 100644 index 000000000..1f2f85f1c --- /dev/null +++ b/server/server/models/abuse/video-abuse.ts @@ -0,0 +1,64 @@ +import { type VideoDetails } from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { VideoModel } from '../video/video.js' +import { AbuseModel } from './abuse.js' + +@Table({ + tableName: 'videoAbuse', + indexes: [ + { + fields: [ 'abuseId' ] + }, + { + fields: [ 'videoId' ] + } + ] +}) +export class VideoAbuseModel extends Model>> { + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(true) + @Default(null) + @Column + startAt: number + + @AllowNull(true) + @Default(null) + @Column + endAt: number + + @AllowNull(true) + @Default(null) + @Column(DataType.JSONB) + deletedVideo: VideoDetails + + @ForeignKey(() => AbuseModel) + @Column + abuseId: number + + @BelongsTo(() => AbuseModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + Abuse: Awaited + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'set null' + }) + Video: Awaited +} diff --git a/server/server/models/abuse/video-comment-abuse.ts b/server/server/models/abuse/video-comment-abuse.ts new file mode 100644 index 000000000..6c5078aa8 --- /dev/null +++ b/server/server/models/abuse/video-comment-abuse.ts @@ -0,0 +1,48 @@ +import { BelongsTo, Column, CreatedAt, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { VideoCommentModel } from '../video/video-comment.js' +import { AbuseModel } from './abuse.js' + +@Table({ + tableName: 'commentAbuse', + indexes: [ + { + fields: [ 'abuseId' ] + }, + { + fields: [ 'videoCommentId' ] + } + ] +}) +export class VideoCommentAbuseModel extends Model>> { + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => AbuseModel) + @Column + abuseId: number + + @BelongsTo(() => AbuseModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + Abuse: Awaited + + @ForeignKey(() => VideoCommentModel) + @Column + videoCommentId: number + + @BelongsTo(() => VideoCommentModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'set null' + }) + VideoComment: Awaited +} diff --git a/server/server/models/account/account-blocklist.ts b/server/server/models/account/account-blocklist.ts new file mode 100644 index 000000000..fa7fa8021 --- /dev/null +++ b/server/server/models/account/account-blocklist.ts @@ -0,0 +1,236 @@ +import { FindOptions, Op, QueryTypes } from 'sequelize' +import { BelongsTo, Column, CreatedAt, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { AccountBlock } from '@peertube/peertube-models' +import { handlesToNameAndHost } from '@server/helpers/actors.js' +import { MAccountBlocklist, MAccountBlocklistFormattable } from '@server/types/models/index.js' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { ActorModel } from '../actor/actor.js' +import { ServerModel } from '../server/server.js' +import { createSafeIn, getSort, searchAttribute } from '../shared/index.js' +import { AccountModel } from './account.js' + +@Table({ + tableName: 'accountBlocklist', + indexes: [ + { + fields: [ 'accountId', 'targetAccountId' ], + unique: true + }, + { + fields: [ 'targetAccountId' ] + } + ] +}) +export class AccountBlocklistModel extends Model>> { + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => AccountModel) + @Column + accountId: number + + @BelongsTo(() => AccountModel, { + foreignKey: { + name: 'accountId', + allowNull: false + }, + as: 'ByAccount', + onDelete: 'CASCADE' + }) + ByAccount: Awaited + + @ForeignKey(() => AccountModel) + @Column + targetAccountId: number + + @BelongsTo(() => AccountModel, { + foreignKey: { + name: 'targetAccountId', + allowNull: false + }, + as: 'BlockedAccount', + onDelete: 'CASCADE' + }) + BlockedAccount: Awaited + + static isAccountMutedByAccounts (accountIds: number[], targetAccountId: number) { + const query = { + attributes: [ 'accountId', 'id' ], + where: { + accountId: { + [Op.in]: accountIds + }, + targetAccountId + }, + raw: true + } + + return AccountBlocklistModel.unscoped() + .findAll(query) + .then(rows => { + const result: { [accountId: number]: boolean } = {} + + for (const accountId of accountIds) { + result[accountId] = !!rows.find(r => r.accountId === accountId) + } + + return result + }) + } + + static loadByAccountAndTarget (accountId: number, targetAccountId: number): Promise { + const query = { + where: { + accountId, + targetAccountId + } + } + + return AccountBlocklistModel.findOne(query) + } + + static listForApi (parameters: { + start: number + count: number + sort: string + search?: string + accountId: number + }) { + const { start, count, sort, search, accountId } = parameters + + const getQuery = (forCount: boolean) => { + const query: FindOptions = { + offset: start, + limit: count, + order: getSort(sort), + where: { accountId } + } + + if (search) { + Object.assign(query.where, { + [Op.or]: [ + searchAttribute(search, '$BlockedAccount.name$'), + searchAttribute(search, '$BlockedAccount.Actor.url$') + ] + }) + } + + if (forCount !== true) { + query.include = [ + { + model: AccountModel, + required: true, + as: 'ByAccount' + }, + { + model: AccountModel, + required: true, + as: 'BlockedAccount' + } + ] + } else if (search) { // We need some joins when counting with search + query.include = [ + { + model: AccountModel.unscoped(), + required: true, + as: 'BlockedAccount', + include: [ + { + model: ActorModel.unscoped(), + required: true + } + ] + } + ] + } + + return query + } + + return Promise.all([ + AccountBlocklistModel.count(getQuery(true)), + AccountBlocklistModel.findAll(getQuery(false)) + ]).then(([ total, data ]) => ({ total, data })) + } + + static listHandlesBlockedBy (accountIds: number[]): Promise { + const query = { + attributes: [ 'id' ], + where: { + accountId: { + [Op.in]: accountIds + } + }, + include: [ + { + attributes: [ 'id' ], + model: AccountModel.unscoped(), + required: true, + as: 'BlockedAccount', + include: [ + { + attributes: [ 'preferredUsername' ], + model: ActorModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'host' ], + model: ServerModel.unscoped(), + required: true + } + ] + } + ] + } + ] + } + + return AccountBlocklistModel.findAll(query) + .then(entries => entries.map(e => `${e.BlockedAccount.Actor.preferredUsername}@${e.BlockedAccount.Actor.Server.host}`)) + } + + static getBlockStatus (byAccountIds: number[], handles: string[]): Promise<{ name: string, host: string, accountId: number }[]> { + const sanitizedHandles = handlesToNameAndHost(handles) + + const localHandles = sanitizedHandles.filter(h => !h.host) + .map(h => h.name) + + const remoteHandles = sanitizedHandles.filter(h => !!h.host) + .map(h => ([ h.name, h.host ])) + + const handlesWhere: string[] = [] + + if (localHandles.length !== 0) { + handlesWhere.push(`("actor"."preferredUsername" IN (:localHandles) AND "server"."id" IS NULL)`) + } + + if (remoteHandles.length !== 0) { + handlesWhere.push(`(("actor"."preferredUsername", "server"."host") IN (:remoteHandles))`) + } + + const rawQuery = `SELECT "accountBlocklist"."accountId", "actor"."preferredUsername" AS "name", "server"."host" ` + + `FROM "accountBlocklist" ` + + `INNER JOIN "account" ON "account"."id" = "accountBlocklist"."targetAccountId" ` + + `INNER JOIN "actor" ON "actor"."id" = "account"."actorId" ` + + `LEFT JOIN "server" ON "server"."id" = "actor"."serverId" ` + + `WHERE "accountBlocklist"."accountId" IN (${createSafeIn(AccountBlocklistModel.sequelize, byAccountIds)}) ` + + `AND (${handlesWhere.join(' OR ')})` + + return AccountBlocklistModel.sequelize.query(rawQuery, { + type: QueryTypes.SELECT as QueryTypes.SELECT, + replacements: { byAccountIds, localHandles, remoteHandles } + }) + } + + toFormattedJSON (this: MAccountBlocklistFormattable): AccountBlock { + return { + byAccount: this.ByAccount.toFormattedJSON(), + blockedAccount: this.BlockedAccount.toFormattedJSON(), + createdAt: this.createdAt + } + } +} diff --git a/server/server/models/account/account-video-rate.ts b/server/server/models/account/account-video-rate.ts new file mode 100644 index 000000000..0d099ff9f --- /dev/null +++ b/server/server/models/account/account-video-rate.ts @@ -0,0 +1,259 @@ +import { AccountVideoRate, type VideoRateType } from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { + MAccountVideoRate, + MAccountVideoRateAccountUrl, + MAccountVideoRateAccountVideo, + MAccountVideoRateFormattable +} from '@server/types/models/index.js' +import { FindOptions, Op, QueryTypes, Transaction } from 'sequelize' +import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc.js' +import { CONSTRAINTS_FIELDS, VIDEO_RATE_TYPES } from '../../initializers/constants.js' +import { ActorModel } from '../actor/actor.js' +import { getSort, throwIfNotValid } from '../shared/index.js' +import { SummaryOptions, VideoChannelModel, ScopeNames as VideoChannelScopeNames } from '../video/video-channel.js' +import { VideoModel } from '../video/video.js' +import { AccountModel } from './account.js' + +/* + Account rates per video. +*/ +@Table({ + tableName: 'accountVideoRate', + indexes: [ + { + fields: [ 'videoId', 'accountId' ], + unique: true + }, + { + fields: [ 'videoId' ] + }, + { + fields: [ 'accountId' ] + }, + { + fields: [ 'videoId', 'type' ] + }, + { + fields: [ 'url' ], + unique: true + } + ] +}) +export class AccountVideoRateModel extends Model>> { + + @AllowNull(false) + @Column(DataType.ENUM(...Object.values(VIDEO_RATE_TYPES))) + type: VideoRateType + + @AllowNull(false) + @Is('AccountVideoRateUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url')) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_RATES.URL.max)) + url: string + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + Video: Awaited + + @ForeignKey(() => AccountModel) + @Column + accountId: number + + @BelongsTo(() => AccountModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + Account: Awaited + + static load (accountId: number, videoId: number, transaction?: Transaction): Promise { + const options: FindOptions = { + where: { + accountId, + videoId + } + } + if (transaction) options.transaction = transaction + + return AccountVideoRateModel.findOne(options) + } + + static loadByAccountAndVideoOrUrl (accountId: number, videoId: number, url: string, t?: Transaction): Promise { + const options: FindOptions = { + where: { + [Op.or]: [ + { + accountId, + videoId + }, + { + url + } + ] + } + } + if (t) options.transaction = t + + return AccountVideoRateModel.findOne(options) + } + + static listByAccountForApi (options: { + start: number + count: number + sort: string + type?: string + accountId: number + }) { + const getQuery = (forCount: boolean) => { + const query: FindOptions = { + offset: options.start, + limit: options.count, + order: getSort(options.sort), + where: { + accountId: options.accountId + } + } + + if (options.type) query.where['type'] = options.type + + if (forCount !== true) { + query.include = [ + { + model: VideoModel, + required: true, + include: [ + { + model: VideoChannelModel.scope({ method: [ VideoChannelScopeNames.SUMMARY, { withAccount: true } as SummaryOptions ] }), + required: true + } + ] + } + ] + } + + return query + } + + return Promise.all([ + AccountVideoRateModel.count(getQuery(true)), + AccountVideoRateModel.findAll(getQuery(false)) + ]).then(([ total, data ]) => ({ total, data })) + } + + static listRemoteRateUrlsOfLocalVideos () { + const query = `SELECT "accountVideoRate".url FROM "accountVideoRate" ` + + `INNER JOIN account ON account.id = "accountVideoRate"."accountId" ` + + `INNER JOIN actor ON actor.id = account."actorId" AND actor."serverId" IS NOT NULL ` + + `INNER JOIN video ON video.id = "accountVideoRate"."videoId" AND video.remote IS FALSE` + + return AccountVideoRateModel.sequelize.query<{ url: string }>(query, { + type: QueryTypes.SELECT, + raw: true + }).then(rows => rows.map(r => r.url)) + } + + static loadLocalAndPopulateVideo ( + rateType: VideoRateType, + accountName: string, + videoId: number, + t?: Transaction + ): Promise { + const options: FindOptions = { + where: { + videoId, + type: rateType + }, + include: [ + { + model: AccountModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'id', 'url', 'followersUrl', 'preferredUsername' ], + model: ActorModel.unscoped(), + required: true, + where: { + [Op.and]: [ + ActorModel.wherePreferredUsername(accountName), + { serverId: null } + ] + } + } + ] + }, + { + model: VideoModel.unscoped(), + required: true + } + ] + } + if (t) options.transaction = t + + return AccountVideoRateModel.findOne(options) + } + + static loadByUrl (url: string, transaction: Transaction) { + const options: FindOptions = { + where: { + url + } + } + if (transaction) options.transaction = transaction + + return AccountVideoRateModel.findOne(options) + } + + static listAndCountAccountUrlsByVideoId (rateType: VideoRateType, videoId: number, start: number, count: number, t?: Transaction) { + const query = { + offset: start, + limit: count, + where: { + videoId, + type: rateType + }, + transaction: t, + include: [ + { + attributes: [ 'actorId' ], + model: AccountModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'url' ], + model: ActorModel.unscoped(), + required: true + } + ] + } + ] + } + + return Promise.all([ + AccountVideoRateModel.count(query), + AccountVideoRateModel.findAll(query) + ]).then(([ total, data ]) => ({ total, data })) + } + + toFormattedJSON (this: MAccountVideoRateFormattable): AccountVideoRate { + return { + video: this.Video.toFormattedJSON(), + rating: this.type + } + } +} diff --git a/server/server/models/account/account.ts b/server/server/models/account/account.ts new file mode 100644 index 000000000..7c2660c40 --- /dev/null +++ b/server/server/models/account/account.ts @@ -0,0 +1,468 @@ +import { FindOptions, Includeable, IncludeOptions, Op, Transaction, WhereOptions } from 'sequelize' +import { + AllowNull, + BeforeDestroy, + BelongsTo, + Column, + CreatedAt, + DataType, + Default, + DefaultScope, + ForeignKey, + HasMany, + Is, + Model, + Scopes, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { Account, AccountSummary } from '@peertube/peertube-models' +import { ModelCache } from '@server/models/shared/model-cache.js' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { isAccountDescriptionValid } from '../../helpers/custom-validators/accounts.js' +import { CONSTRAINTS_FIELDS, SERVER_ACTOR_NAME, WEBSERVER } from '../../initializers/constants.js' +import { sendDeleteActor } from '../../lib/activitypub/send/send-delete.js' +import { + MAccount, + MAccountActor, + MAccountAP, + MAccountDefault, + MAccountFormattable, + MAccountHost, + MAccountSummaryFormattable, + MChannelHost +} from '../../types/models/index.js' +import { ActorFollowModel } from '../actor/actor-follow.js' +import { ActorImageModel } from '../actor/actor-image.js' +import { ActorModel } from '../actor/actor.js' +import { ApplicationModel } from '../application/application.js' +import { ServerBlocklistModel } from '../server/server-blocklist.js' +import { ServerModel } from '../server/server.js' +import { buildSQLAttributes, getSort, throwIfNotValid } from '../shared/index.js' +import { UserModel } from '../user/user.js' +import { VideoChannelModel } from '../video/video-channel.js' +import { VideoCommentModel } from '../video/video-comment.js' +import { VideoPlaylistModel } from '../video/video-playlist.js' +import { VideoModel } from '../video/video.js' +import { AccountBlocklistModel } from './account-blocklist.js' + +export enum ScopeNames { + SUMMARY = 'SUMMARY' +} + +export type SummaryOptions = { + actorRequired?: boolean // Default: true + whereActor?: WhereOptions + whereServer?: WhereOptions + withAccountBlockerIds?: number[] + forCount?: boolean +} + +@DefaultScope(() => ({ + include: [ + { + model: ActorModel, // Default scope includes avatar and server + required: true + } + ] +})) +@Scopes(() => ({ + [ScopeNames.SUMMARY]: (options: SummaryOptions = {}) => { + const serverInclude: IncludeOptions = { + attributes: [ 'host' ], + model: ServerModel.unscoped(), + required: !!options.whereServer, + where: options.whereServer + } + + const actorInclude: Includeable = { + attributes: [ 'id', 'preferredUsername', 'url', 'serverId' ], + model: ActorModel.unscoped(), + required: options.actorRequired ?? true, + where: options.whereActor, + include: [ serverInclude ] + } + + if (options.forCount !== true) { + actorInclude.include.push({ + model: ActorImageModel, + as: 'Avatars', + required: false + }) + } + + const queryInclude: Includeable[] = [ + actorInclude + ] + + const query: FindOptions = { + attributes: [ 'id', 'name', 'actorId' ] + } + + if (options.withAccountBlockerIds) { + queryInclude.push({ + attributes: [ 'id' ], + model: AccountBlocklistModel.unscoped(), + as: 'BlockedBy', + required: false, + where: { + accountId: { + [Op.in]: options.withAccountBlockerIds + } + } + }) + + serverInclude.include = [ + { + attributes: [ 'id' ], + model: ServerBlocklistModel.unscoped(), + required: false, + where: { + accountId: { + [Op.in]: options.withAccountBlockerIds + } + } + } + ] + } + + query.include = queryInclude + + return query + } +})) +@Table({ + tableName: 'account', + indexes: [ + { + fields: [ 'actorId' ], + unique: true + }, + { + fields: [ 'applicationId' ] + }, + { + fields: [ 'userId' ] + } + ] +}) +export class AccountModel extends Model>> { + + @AllowNull(false) + @Column + name: string + + @AllowNull(true) + @Default(null) + @Is('AccountDescription', value => throwIfNotValid(value, isAccountDescriptionValid, 'description', true)) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.USERS.DESCRIPTION.max)) + description: string + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => ActorModel) + @Column + actorId: number + + @BelongsTo(() => ActorModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + Actor: Awaited + + @ForeignKey(() => UserModel) + @Column + userId: number + + @BelongsTo(() => UserModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'cascade' + }) + User: Awaited + + @ForeignKey(() => ApplicationModel) + @Column + applicationId: number + + @BelongsTo(() => ApplicationModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'cascade' + }) + Application: Awaited + + @HasMany(() => VideoChannelModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade', + hooks: true + }) + VideoChannels: Awaited[] + + @HasMany(() => VideoPlaylistModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade', + hooks: true + }) + VideoPlaylists: Awaited[] + + @HasMany(() => VideoCommentModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'cascade', + hooks: true + }) + VideoComments: Awaited[] + + @HasMany(() => AccountBlocklistModel, { + foreignKey: { + name: 'targetAccountId', + allowNull: false + }, + as: 'BlockedBy', + onDelete: 'CASCADE' + }) + BlockedBy: Awaited[] + + @BeforeDestroy + static async sendDeleteIfOwned (instance: AccountModel, options) { + if (!instance.Actor) { + instance.Actor = await instance.$get('Actor', { transaction: options.transaction }) + } + + await ActorFollowModel.removeFollowsOf(instance.Actor.id, options.transaction) + + if (instance.isOwned()) { + return sendDeleteActor(instance.Actor, options.transaction) + } + + return undefined + } + + // --------------------------------------------------------------------------- + + static getSQLAttributes (tableName: string, aliasPrefix = '') { + return buildSQLAttributes({ + model: this, + tableName, + aliasPrefix + }) + } + + // --------------------------------------------------------------------------- + + static load (id: number, transaction?: Transaction): Promise { + return AccountModel.findByPk(id, { transaction }) + } + + static loadByNameWithHost (nameWithHost: string): Promise { + const [ accountName, host ] = nameWithHost.split('@') + + if (!host || host === WEBSERVER.HOST) return AccountModel.loadLocalByName(accountName) + + return AccountModel.loadByNameAndHost(accountName, host) + } + + static loadLocalByName (name: string): Promise { + const fun = () => { + const query = { + where: { + [Op.or]: [ + { + userId: { + [Op.ne]: null + } + }, + { + applicationId: { + [Op.ne]: null + } + } + ] + }, + include: [ + { + model: ActorModel, + required: true, + where: ActorModel.wherePreferredUsername(name) + } + ] + } + + return AccountModel.findOne(query) + } + + return ModelCache.Instance.doCache({ + cacheType: 'local-account-name', + key: name, + fun, + // The server actor never change, so we can easily cache it + whitelist: () => name === SERVER_ACTOR_NAME + }) + } + + static loadByNameAndHost (name: string, host: string): Promise { + const query = { + include: [ + { + model: ActorModel, + required: true, + where: ActorModel.wherePreferredUsername(name), + include: [ + { + model: ServerModel, + required: true, + where: { + host + } + } + ] + } + ] + } + + return AccountModel.findOne(query) + } + + static loadByUrl (url: string, transaction?: Transaction): Promise { + const query = { + include: [ + { + model: ActorModel, + required: true, + where: { + url + } + } + ], + transaction + } + + return AccountModel.findOne(query) + } + + static listForApi (start: number, count: number, sort: string) { + const query = { + offset: start, + limit: count, + order: getSort(sort) + } + + return Promise.all([ + AccountModel.count(), + AccountModel.findAll(query) + ]).then(([ total, data ]) => ({ total, data })) + } + + static loadAccountIdFromVideo (videoId: number): Promise { + const query = { + include: [ + { + attributes: [ 'id', 'accountId' ], + model: VideoChannelModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'id', 'channelId' ], + model: VideoModel.unscoped(), + where: { + id: videoId + } + } + ] + } + ] + } + + return AccountModel.findOne(query) + } + + static listLocalsForSitemap (sort: string): Promise { + const query = { + attributes: [ ], + offset: 0, + order: getSort(sort), + include: [ + { + attributes: [ 'preferredUsername', 'serverId' ], + model: ActorModel.unscoped(), + where: { + serverId: null + } + } + ] + } + + return AccountModel + .unscoped() + .findAll(query) + } + + toFormattedJSON (this: MAccountFormattable): Account { + return { + ...this.Actor.toFormattedJSON(), + + id: this.id, + displayName: this.getDisplayName(), + description: this.description, + updatedAt: this.updatedAt, + userId: this.userId ?? undefined + } + } + + toFormattedSummaryJSON (this: MAccountSummaryFormattable): AccountSummary { + const actor = this.Actor.toFormattedSummaryJSON() + + return { + id: this.id, + displayName: this.getDisplayName(), + + name: actor.name, + url: actor.url, + host: actor.host, + avatars: actor.avatars + } + } + + async toActivityPubObject (this: MAccountAP) { + const obj = await this.Actor.toActivityPubObject(this.name) + + return Object.assign(obj, { + summary: this.description + }) + } + + isOwned () { + return this.Actor.isOwned() + } + + isOutdated () { + return this.Actor.isOutdated() + } + + getDisplayName () { + return this.name + } + + // Avoid error when running this method on MAccount... | MChannel... + getClientUrl (this: MAccountHost | MChannelHost) { + return WEBSERVER.URL + '/a/' + this.Actor.getIdentifier() + } + + isBlocked () { + return this.BlockedBy && this.BlockedBy.length !== 0 + } +} diff --git a/server/server/models/account/actor-custom-page.ts b/server/server/models/account/actor-custom-page.ts new file mode 100644 index 000000000..8a9b09706 --- /dev/null +++ b/server/server/models/account/actor-custom-page.ts @@ -0,0 +1,69 @@ +import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { CustomPage } from '@peertube/peertube-models' +import { ActorModel } from '../actor/actor.js' +import { getServerActor } from '../application/application.js' + +@Table({ + tableName: 'actorCustomPage', + indexes: [ + { + fields: [ 'actorId', 'type' ], + unique: true + } + ] +}) +export class ActorCustomPageModel extends Model { + + @AllowNull(true) + @Column(DataType.TEXT) + content: string + + @AllowNull(false) + @Column + type: 'homepage' + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => ActorModel) + @Column + actorId: number + + @BelongsTo(() => ActorModel, { + foreignKey: { + name: 'actorId', + allowNull: false + }, + onDelete: 'cascade' + }) + Actor: Awaited + + static async updateInstanceHomepage (content: string) { + const serverActor = await getServerActor() + + return ActorCustomPageModel.upsert({ + content, + actorId: serverActor.id, + type: 'homepage' + }) + } + + static async loadInstanceHomepage () { + const serverActor = await getServerActor() + + return ActorCustomPageModel.findOne({ + where: { + actorId: serverActor.id + } + }) + } + + toFormattedJSON (): CustomPage { + return { + content: this.content + } + } +} diff --git a/server/server/models/actor/actor-follow.ts b/server/server/models/actor/actor-follow.ts new file mode 100644 index 000000000..268edb5b4 --- /dev/null +++ b/server/server/models/actor/actor-follow.ts @@ -0,0 +1,661 @@ +import { ActorFollow, type FollowState } from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js' +import { afterCommitIfTransaction } from '@server/helpers/database-utils.js' +import { getServerActor } from '@server/models/application/application.js' +import { + MActor, + MActorFollowActors, + MActorFollowActorsDefault, + MActorFollowActorsDefaultSubscription, + MActorFollowFollowingHost, + MActorFollowFormattable, + MActorFollowSubscriptions +} from '@server/types/models/index.js' +import difference from 'lodash-es/difference.js' +import { Attributes, FindOptions, IncludeOptions, Includeable, Op, QueryTypes, Transaction, WhereAttributeHash } from 'sequelize' +import { + AfterCreate, + AfterDestroy, + AfterUpdate, + AllowNull, + BelongsTo, + Column, + CreatedAt, + DataType, + Default, + ForeignKey, + Is, + IsInt, + Max, + Model, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { logger } from '../../helpers/logger.js' +import { ACTOR_FOLLOW_SCORE, CONSTRAINTS_FIELDS, FOLLOW_STATES, SERVER_ACTOR_NAME, SORTABLE_COLUMNS } from '../../initializers/constants.js' +import { AccountModel } from '../account/account.js' +import { ServerModel } from '../server/server.js' +import { buildSQLAttributes, createSafeIn, getSort, searchAttribute, throwIfNotValid } from '../shared/index.js' +import { doesExist } from '../shared/query.js' +import { VideoChannelModel } from '../video/video-channel.js' +import { ActorModel, unusedActorAttributesForAPI } from './actor.js' +import { InstanceListFollowersQueryBuilder, ListFollowersOptions } from './sql/instance-list-followers-query-builder.js' +import { InstanceListFollowingQueryBuilder, ListFollowingOptions } from './sql/instance-list-following-query-builder.js' + +@Table({ + tableName: 'actorFollow', + indexes: [ + { + fields: [ 'actorId' ] + }, + { + fields: [ 'targetActorId' ] + }, + { + fields: [ 'actorId', 'targetActorId' ], + unique: true + }, + { + fields: [ 'score' ] + }, + { + fields: [ 'url' ], + unique: true + } + ] +}) +export class ActorFollowModel extends Model>> { + + @AllowNull(false) + @Column(DataType.ENUM(...Object.values(FOLLOW_STATES))) + state: FollowState + + @AllowNull(false) + @Default(ACTOR_FOLLOW_SCORE.BASE) + @IsInt + @Max(ACTOR_FOLLOW_SCORE.MAX) + @Column + score: number + + // Allow null because we added this column in PeerTube v3, and don't want to generate fake URLs of remote follows + @AllowNull(true) + @Is('ActorFollowUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url')) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.COMMONS.URL.max)) + url: string + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => ActorModel) + @Column + actorId: number + + @BelongsTo(() => ActorModel, { + foreignKey: { + name: 'actorId', + allowNull: false + }, + as: 'ActorFollower', + onDelete: 'CASCADE' + }) + ActorFollower: Awaited + + @ForeignKey(() => ActorModel) + @Column + targetActorId: number + + @BelongsTo(() => ActorModel, { + foreignKey: { + name: 'targetActorId', + allowNull: false + }, + as: 'ActorFollowing', + onDelete: 'CASCADE' + }) + ActorFollowing: Awaited + + @AfterCreate + @AfterUpdate + static incrementFollowerAndFollowingCount (instance: ActorFollowModel, options: any) { + return afterCommitIfTransaction(options.transaction, () => { + return Promise.all([ + ActorModel.rebuildFollowsCount(instance.actorId, 'following'), + ActorModel.rebuildFollowsCount(instance.targetActorId, 'followers') + ]) + }) + } + + @AfterDestroy + static decrementFollowerAndFollowingCount (instance: ActorFollowModel, options: any) { + return afterCommitIfTransaction(options.transaction, () => { + return Promise.all([ + ActorModel.rebuildFollowsCount(instance.actorId, 'following'), + ActorModel.rebuildFollowsCount(instance.targetActorId, 'followers') + ]) + }) + } + + // --------------------------------------------------------------------------- + + static getSQLAttributes (tableName: string, aliasPrefix = '') { + return buildSQLAttributes({ + model: this, + tableName, + aliasPrefix + }) + } + + // --------------------------------------------------------------------------- + + /* + * @deprecated Use `findOrCreateCustom` instead + */ + static findOrCreate (): any { + throw new Error('Must not be called') + } + + // findOrCreate has issues with actor follow hooks + static async findOrCreateCustom (options: { + byActor: MActor + targetActor: MActor + activityId: string + state: FollowState + transaction: Transaction + }): Promise<[ MActorFollowActors, boolean ]> { + const { byActor, targetActor, activityId, state, transaction } = options + + let created = false + let actorFollow: MActorFollowActors = await ActorFollowModel.loadByActorAndTarget(byActor.id, targetActor.id, transaction) + + if (!actorFollow) { + created = true + + actorFollow = await ActorFollowModel.create({ + actorId: byActor.id, + targetActorId: targetActor.id, + url: activityId, + + state + }, { transaction }) + + actorFollow.ActorFollowing = targetActor + actorFollow.ActorFollower = byActor + } + + return [ actorFollow, created ] + } + + static removeFollowsOf (actorId: number, t?: Transaction) { + const query = { + where: { + [Op.or]: [ + { + actorId + }, + { + targetActorId: actorId + } + ] + }, + transaction: t + } + + return ActorFollowModel.destroy(query) + } + + // Remove actor follows with a score of 0 (too many requests where they were unreachable) + static async removeBadActorFollows () { + const actorFollows = await ActorFollowModel.listBadActorFollows() + + const actorFollowsRemovePromises = actorFollows.map(actorFollow => actorFollow.destroy()) + await Promise.all(actorFollowsRemovePromises) + + const numberOfActorFollowsRemoved = actorFollows.length + + if (numberOfActorFollowsRemoved) logger.info('Removed bad %d actor follows.', numberOfActorFollowsRemoved) + } + + static isFollowedBy (actorId: number, followerActorId: number) { + const query = `SELECT 1 FROM "actorFollow" ` + + `WHERE "actorId" = $followerActorId AND "targetActorId" = $actorId AND "state" = 'accepted' ` + + `LIMIT 1` + + return doesExist(this.sequelize, query, { actorId, followerActorId }) + } + + static loadByActorAndTarget (actorId: number, targetActorId: number, t?: Transaction): Promise { + const query = { + where: { + actorId, + targetActorId + }, + include: [ + { + model: ActorModel, + required: true, + as: 'ActorFollower' + }, + { + model: ActorModel, + required: true, + as: 'ActorFollowing' + } + ], + transaction: t + } + + return ActorFollowModel.findOne(query) + } + + static loadByActorAndTargetNameAndHostForAPI (options: { + actorId: number + targetName: string + targetHost: string + state?: FollowState + transaction?: Transaction + }): Promise { + const { actorId, targetHost, targetName, state, transaction } = options + + const actorFollowingPartInclude: IncludeOptions = { + model: ActorModel, + required: true, + as: 'ActorFollowing', + where: ActorModel.wherePreferredUsername(targetName), + include: [ + { + model: VideoChannelModel.unscoped(), + required: false + } + ] + } + + if (targetHost === null) { + actorFollowingPartInclude.where['serverId'] = null + } else { + actorFollowingPartInclude.include.push({ + model: ServerModel, + required: true, + where: { + host: targetHost + } + }) + } + + const where: WhereAttributeHash> = { actorId } + if (state) where.state = state + + const query: FindOptions> = { + where, + include: [ + actorFollowingPartInclude, + { + model: ActorModel, + required: true, + as: 'ActorFollower' + } + ], + transaction + } + + return ActorFollowModel.findOne(query) + } + + static listSubscriptionsOf (actorId: number, targets: { name: string, host?: string }[]): Promise { + const whereTab = targets + .map(t => { + if (t.host) { + return { + [Op.and]: [ + ActorModel.wherePreferredUsername(t.name), + { $host$: t.host } + ] + } + } + + return { + [Op.and]: [ + ActorModel.wherePreferredUsername(t.name), + { $serverId$: null } + ] + } + }) + + const query = { + attributes: [ 'id' ], + where: { + [Op.and]: [ + { + [Op.or]: whereTab + }, + { + state: 'accepted', + actorId + } + ] + }, + include: [ + { + attributes: [ 'preferredUsername' ], + model: ActorModel.unscoped(), + required: true, + as: 'ActorFollowing', + include: [ + { + attributes: [ 'host' ], + model: ServerModel.unscoped(), + required: false + } + ] + } + ] + } + + return ActorFollowModel.findAll(query) + } + + static listInstanceFollowingForApi (options: ListFollowingOptions) { + return Promise.all([ + new InstanceListFollowingQueryBuilder(this.sequelize, options).countFollowing(), + new InstanceListFollowingQueryBuilder(this.sequelize, options).listFollowing() + ]).then(([ total, data ]) => ({ total, data })) + } + + static listFollowersForApi (options: ListFollowersOptions) { + return Promise.all([ + new InstanceListFollowersQueryBuilder(this.sequelize, options).countFollowers(), + new InstanceListFollowersQueryBuilder(this.sequelize, options).listFollowers() + ]).then(([ total, data ]) => ({ total, data })) + } + + static listSubscriptionsForApi (options: { + actorId: number + start: number + count: number + sort: string + search?: string + }) { + const { actorId, start, count, sort } = options + const where = { + state: 'accepted', + actorId + } + + if (options.search) { + Object.assign(where, { + [Op.or]: [ + searchAttribute(options.search, '$ActorFollowing.preferredUsername$'), + searchAttribute(options.search, '$ActorFollowing.VideoChannel.name$') + ] + }) + } + + const getQuery = (forCount: boolean) => { + let channelInclude: Includeable[] = [] + + if (forCount !== true) { + channelInclude = [ + { + attributes: { + exclude: unusedActorAttributesForAPI + }, + model: ActorModel, + required: true + }, + { + model: AccountModel.unscoped(), + required: true, + include: [ + { + attributes: { + exclude: unusedActorAttributesForAPI + }, + model: ActorModel, + required: true + } + ] + } + ] + } + + return { + attributes: forCount === true + ? [] + : SORTABLE_COLUMNS.USER_SUBSCRIPTIONS, + distinct: true, + offset: start, + limit: count, + order: getSort(sort), + where, + include: [ + { + attributes: [ 'id' ], + model: ActorModel.unscoped(), + as: 'ActorFollowing', + required: true, + include: [ + { + model: VideoChannelModel.unscoped(), + required: true, + include: channelInclude + } + ] + } + ] + } + } + + return Promise.all([ + ActorFollowModel.count(getQuery(true)), + ActorFollowModel.findAll(getQuery(false)) + ]).then(([ total, rows ]) => ({ + total, + data: rows.map(r => r.ActorFollowing.VideoChannel) + })) + } + + static async keepUnfollowedInstance (hosts: string[]) { + const followerId = (await getServerActor()).id + + const query = { + attributes: [ 'id' ], + where: { + actorId: followerId + }, + include: [ + { + attributes: [ 'id' ], + model: ActorModel.unscoped(), + required: true, + as: 'ActorFollowing', + where: { + preferredUsername: SERVER_ACTOR_NAME + }, + include: [ + { + attributes: [ 'host' ], + model: ServerModel.unscoped(), + required: true, + where: { + host: { + [Op.in]: hosts + } + } + } + ] + } + ] + } + + const res = await ActorFollowModel.findAll(query) + const followedHosts = res.map(row => row.ActorFollowing.Server.host) + + return difference(hosts, followedHosts) + } + + static listAcceptedFollowerUrlsForAP (actorIds: number[], t: Transaction, start?: number, count?: number) { + return ActorFollowModel.createListAcceptedFollowForApiQuery('followers', actorIds, t, start, count) + } + + static listAcceptedFollowerSharedInboxUrls (actorIds: number[], t: Transaction) { + return ActorFollowModel.createListAcceptedFollowForApiQuery( + 'followers', + actorIds, + t, + undefined, + undefined, + 'sharedInboxUrl', + true + ) + } + + static listAcceptedFollowingUrlsForApi (actorIds: number[], t: Transaction, start?: number, count?: number) { + return ActorFollowModel.createListAcceptedFollowForApiQuery('following', actorIds, t, start, count) + } + + static async getStats () { + const serverActor = await getServerActor() + + const totalInstanceFollowing = await ActorFollowModel.count({ + where: { + actorId: serverActor.id, + state: 'accepted' + } + }) + + const totalInstanceFollowers = await ActorFollowModel.count({ + where: { + targetActorId: serverActor.id, + state: 'accepted' + } + }) + + return { + totalInstanceFollowing, + totalInstanceFollowers + } + } + + static updateScore (inboxUrl: string, value: number, t?: Transaction) { + const query = `UPDATE "actorFollow" SET "score" = LEAST("score" + ${value}, ${ACTOR_FOLLOW_SCORE.MAX}) ` + + 'WHERE id IN (' + + 'SELECT "actorFollow"."id" FROM "actorFollow" ' + + 'INNER JOIN "actor" ON "actor"."id" = "actorFollow"."actorId" ' + + `WHERE "actor"."inboxUrl" = '${inboxUrl}' OR "actor"."sharedInboxUrl" = '${inboxUrl}'` + + ')' + + const options = { + type: QueryTypes.BULKUPDATE, + transaction: t + } + + return ActorFollowModel.sequelize.query(query, options) + } + + static async updateScoreByFollowingServers (serverIds: number[], value: number, t?: Transaction) { + if (serverIds.length === 0) return + + const me = await getServerActor() + const serverIdsString = createSafeIn(ActorFollowModel.sequelize, serverIds) + + const query = `UPDATE "actorFollow" SET "score" = LEAST("score" + ${value}, ${ACTOR_FOLLOW_SCORE.MAX}) ` + + 'WHERE id IN (' + + 'SELECT "actorFollow"."id" FROM "actorFollow" ' + + 'INNER JOIN "actor" ON "actor"."id" = "actorFollow"."targetActorId" ' + + `WHERE "actorFollow"."actorId" = ${me.Account.actorId} ` + // I'm the follower + `AND "actor"."serverId" IN (${serverIdsString})` + // Criteria on followings + ')' + + const options = { + type: QueryTypes.BULKUPDATE, + transaction: t + } + + return ActorFollowModel.sequelize.query(query, options) + } + + private static async createListAcceptedFollowForApiQuery ( + type: 'followers' | 'following', + actorIds: number[], + t: Transaction, + start?: number, + count?: number, + columnUrl = 'url', + distinct = false + ) { + let firstJoin: string + let secondJoin: string + + if (type === 'followers') { + firstJoin = 'targetActorId' + secondJoin = 'actorId' + } else { + firstJoin = 'actorId' + secondJoin = 'targetActorId' + } + + const selections: string[] = [] + if (distinct === true) selections.push(`DISTINCT("Follows"."${columnUrl}") AS "selectionUrl"`) + else selections.push(`"Follows"."${columnUrl}" AS "selectionUrl"`) + + selections.push('COUNT(*) AS "total"') + + const tasks: Promise[] = [] + + for (const selection of selections) { + let query = 'SELECT ' + selection + ' FROM "actor" ' + + 'INNER JOIN "actorFollow" ON "actorFollow"."' + firstJoin + '" = "actor"."id" ' + + 'INNER JOIN "actor" AS "Follows" ON "actorFollow"."' + secondJoin + '" = "Follows"."id" ' + + `WHERE "actor"."id" = ANY ($actorIds) AND "actorFollow"."state" = 'accepted' AND "Follows"."${columnUrl}" IS NOT NULL ` + + if (count !== undefined) query += 'LIMIT ' + count + if (start !== undefined) query += ' OFFSET ' + start + + const options = { + bind: { actorIds }, + type: QueryTypes.SELECT, + transaction: t + } + tasks.push(ActorFollowModel.sequelize.query(query, options)) + } + + const [ followers, [ dataTotal ] ] = await Promise.all(tasks) + const urls: string[] = followers.map(f => f.selectionUrl) + + return { + data: urls, + total: dataTotal ? parseInt(dataTotal.total, 10) : 0 + } + } + + private static listBadActorFollows () { + const query = { + where: { + score: { + [Op.lte]: 0 + } + }, + logging: false + } + + return ActorFollowModel.findAll(query) + } + + toFormattedJSON (this: MActorFollowFormattable): ActorFollow { + const follower = this.ActorFollower.toFormattedJSON() + const following = this.ActorFollowing.toFormattedJSON() + + return { + id: this.id, + follower, + following, + score: this.score, + state: this.state, + createdAt: this.createdAt, + updatedAt: this.updatedAt + } + } +} diff --git a/server/server/models/actor/actor-image.ts b/server/server/models/actor/actor-image.ts new file mode 100644 index 000000000..727e89f04 --- /dev/null +++ b/server/server/models/actor/actor-image.ts @@ -0,0 +1,170 @@ +import { ActivityIconObject, ActorImage, ActorImageType, type ActorImageType_Type } from '@peertube/peertube-models' +import { getLowercaseExtension } from '@peertube/peertube-node-utils' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { MActorImage, MActorImageFormattable } from '@server/types/models/index.js' +import { remove } from 'fs-extra/esm' +import { join } from 'path' +import { + AfterDestroy, + AllowNull, + BelongsTo, + Column, + CreatedAt, + Default, + ForeignKey, + Is, + Model, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc.js' +import { logger } from '../../helpers/logger.js' +import { CONFIG } from '../../initializers/config.js' +import { LAZY_STATIC_PATHS, MIMETYPES, WEBSERVER } from '../../initializers/constants.js' +import { buildSQLAttributes, throwIfNotValid } from '../shared/index.js' +import { ActorModel } from './actor.js' + +@Table({ + tableName: 'actorImage', + indexes: [ + { + fields: [ 'filename' ], + unique: true + }, + { + fields: [ 'actorId', 'type', 'width' ], + unique: true + } + ] +}) +export class ActorImageModel extends Model>> { + + @AllowNull(false) + @Column + filename: string + + @AllowNull(true) + @Default(null) + @Column + height: number + + @AllowNull(true) + @Default(null) + @Column + width: number + + @AllowNull(true) + @Is('ActorImageFileUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'fileUrl', true)) + @Column + fileUrl: string + + @AllowNull(false) + @Column + onDisk: boolean + + @AllowNull(false) + @Column + type: ActorImageType_Type + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => ActorModel) + @Column + actorId: number + + @BelongsTo(() => ActorModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + Actor: Awaited // Remove awaited: https://github.com/sequelize/sequelize-typescript/issues/825 + + @AfterDestroy + static removeFilesAndSendDelete (instance: ActorImageModel) { + logger.info('Removing actor image file %s.', instance.filename) + + // Don't block the transaction + instance.removeImage() + .catch(err => logger.error('Cannot remove actor image file %s.', instance.filename, { err })) + } + + // --------------------------------------------------------------------------- + + static getSQLAttributes (tableName: string, aliasPrefix = '') { + return buildSQLAttributes({ + model: this, + tableName, + aliasPrefix + }) + } + + // --------------------------------------------------------------------------- + + static loadByName (filename: string) { + const query = { + where: { + filename + } + } + + return ActorImageModel.findOne(query) + } + + static getImageUrl (image: MActorImage) { + if (!image) return undefined + + return WEBSERVER.URL + image.getStaticPath() + } + + toFormattedJSON (this: MActorImageFormattable): ActorImage { + return { + width: this.width, + path: this.getStaticPath(), + createdAt: this.createdAt, + updatedAt: this.updatedAt + } + } + + toActivityPubObject (): ActivityIconObject { + const extension = getLowercaseExtension(this.filename) + + return { + type: 'Image', + mediaType: MIMETYPES.IMAGE.EXT_MIMETYPE[extension], + height: this.height, + width: this.width, + url: ActorImageModel.getImageUrl(this) + } + } + + getStaticPath () { + switch (this.type) { + case ActorImageType.AVATAR: + return join(LAZY_STATIC_PATHS.AVATARS, this.filename) + + case ActorImageType.BANNER: + return join(LAZY_STATIC_PATHS.BANNERS, this.filename) + + default: + throw new Error('Unknown actor image type: ' + this.type) + } + } + + getPath () { + return join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, this.filename) + } + + removeImage () { + const imagePath = join(CONFIG.STORAGE.ACTOR_IMAGES_DIR, this.filename) + return remove(imagePath) + } + + isOwned () { + return !this.fileUrl + } +} diff --git a/server/server/models/actor/actor.ts b/server/server/models/actor/actor.ts new file mode 100644 index 000000000..adb009968 --- /dev/null +++ b/server/server/models/actor/actor.ts @@ -0,0 +1,690 @@ +import { forceNumber } from '@peertube/peertube-core-utils' +import { ActivityIconObject, ActorImageType, ActorImageType_Type, type ActivityPubActorType } from '@peertube/peertube-models' +import { getLowercaseExtension } from '@peertube/peertube-node-utils' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { activityPubContextify } from '@server/helpers/activity-pub-utils.js' +import { getContextFilter } from '@server/lib/activitypub/context.js' +import { getBiggestActorImage } from '@server/lib/actor-image.js' +import { ModelCache } from '@server/models/shared/model-cache.js' +import { col, fn, literal, Op, QueryTypes, Transaction, where } from 'sequelize' +import { + AllowNull, + BelongsTo, + Column, + CreatedAt, + DataType, + DefaultScope, + ForeignKey, + HasMany, + HasOne, + Is, + Model, + Scopes, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { Where } from 'sequelize/types/utils' +import { + isActorFollowersCountValid, + isActorFollowingCountValid, + isActorPreferredUsernameValid, + isActorPrivateKeyValid, + isActorPublicKeyValid +} from '../../helpers/custom-validators/activitypub/actor.js' +import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc.js' +import { + ACTIVITY_PUB, + ACTIVITY_PUB_ACTOR_TYPES, + CONSTRAINTS_FIELDS, + MIMETYPES, + SERVER_ACTOR_NAME, + WEBSERVER +} from '../../initializers/constants.js' +import { + MActor, + MActorAccountChannelId, + MActorAPAccount, + MActorAPChannel, + MActorFollowersUrl, + MActorFormattable, + MActorFull, + MActorHost, + MActorHostOnly, + MActorId, + MActorSummaryFormattable, + MActorUrl, + MActorWithInboxes +} from '../../types/models/index.js' +import { AccountModel } from '../account/account.js' +import { getServerActor } from '../application/application.js' +import { ServerModel } from '../server/server.js' +import { buildSQLAttributes, isOutdated, throwIfNotValid } from '../shared/index.js' +import { VideoChannelModel } from '../video/video-channel.js' +import { VideoModel } from '../video/video.js' +import { ActorFollowModel } from './actor-follow.js' +import { ActorImageModel } from './actor-image.js' + +enum ScopeNames { + FULL = 'FULL' +} + +export const unusedActorAttributesForAPI: (keyof AttributesOnly)[] = [ + 'publicKey', + 'privateKey', + 'inboxUrl', + 'outboxUrl', + 'sharedInboxUrl', + 'followersUrl', + 'followingUrl' +] + +@DefaultScope(() => ({ + include: [ + { + model: ServerModel, + required: false + }, + { + model: ActorImageModel, + as: 'Avatars', + required: false + } + ] +})) +@Scopes(() => ({ + [ScopeNames.FULL]: { + include: [ + { + model: AccountModel.unscoped(), + required: false + }, + { + model: VideoChannelModel.unscoped(), + required: false, + include: [ + { + model: AccountModel, + required: true + } + ] + }, + { + model: ServerModel, + required: false + }, + { + model: ActorImageModel, + as: 'Avatars', + required: false + }, + { + model: ActorImageModel, + as: 'Banners', + required: false + } + ] + } +})) +@Table({ + tableName: 'actor', + indexes: [ + { + fields: [ 'url' ], + unique: true + }, + { + fields: [ fn('lower', col('preferredUsername')), 'serverId' ], + name: 'actor_preferred_username_lower_server_id', + unique: true, + where: { + serverId: { + [Op.ne]: null + } + } + }, + { + fields: [ fn('lower', col('preferredUsername')) ], + name: 'actor_preferred_username_lower', + unique: true, + where: { + serverId: null + } + }, + { + fields: [ 'inboxUrl', 'sharedInboxUrl' ] + }, + { + fields: [ 'sharedInboxUrl' ] + }, + { + fields: [ 'serverId' ] + }, + { + fields: [ 'followersUrl' ] + } + ] +}) +export class ActorModel extends Model>> { + + @AllowNull(false) + @Column(DataType.ENUM(...Object.values(ACTIVITY_PUB_ACTOR_TYPES))) + type: ActivityPubActorType + + @AllowNull(false) + @Is('ActorPreferredUsername', value => throwIfNotValid(value, isActorPreferredUsernameValid, 'actor preferred username')) + @Column + preferredUsername: string + + @AllowNull(false) + @Is('ActorUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url')) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.ACTORS.URL.max)) + url: string + + @AllowNull(true) + @Is('ActorPublicKey', value => throwIfNotValid(value, isActorPublicKeyValid, 'public key', true)) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.ACTORS.PUBLIC_KEY.max)) + publicKey: string + + @AllowNull(true) + @Is('ActorPublicKey', value => throwIfNotValid(value, isActorPrivateKeyValid, 'private key', true)) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.ACTORS.PRIVATE_KEY.max)) + privateKey: string + + @AllowNull(false) + @Is('ActorFollowersCount', value => throwIfNotValid(value, isActorFollowersCountValid, 'followers count')) + @Column + followersCount: number + + @AllowNull(false) + @Is('ActorFollowersCount', value => throwIfNotValid(value, isActorFollowingCountValid, 'following count')) + @Column + followingCount: number + + @AllowNull(false) + @Is('ActorInboxUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'inbox url')) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.ACTORS.URL.max)) + inboxUrl: string + + @AllowNull(true) + @Is('ActorOutboxUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'outbox url', true)) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.ACTORS.URL.max)) + outboxUrl: string + + @AllowNull(true) + @Is('ActorSharedInboxUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'shared inbox url', true)) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.ACTORS.URL.max)) + sharedInboxUrl: string + + @AllowNull(true) + @Is('ActorFollowersUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'followers url', true)) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.ACTORS.URL.max)) + followersUrl: string + + @AllowNull(true) + @Is('ActorFollowingUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'following url', true)) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.ACTORS.URL.max)) + followingUrl: string + + @AllowNull(true) + @Column + remoteCreatedAt: Date + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @HasMany(() => ActorImageModel, { + as: 'Avatars', + onDelete: 'cascade', + hooks: true, + foreignKey: { + allowNull: false + }, + scope: { + type: ActorImageType.AVATAR + } + }) + Avatars: Awaited[] + + @HasMany(() => ActorImageModel, { + as: 'Banners', + onDelete: 'cascade', + hooks: true, + foreignKey: { + allowNull: false + }, + scope: { + type: ActorImageType.BANNER + } + }) + Banners: Awaited[] + + @HasMany(() => ActorFollowModel, { + foreignKey: { + name: 'actorId', + allowNull: false + }, + as: 'ActorFollowings', + onDelete: 'cascade' + }) + ActorFollowing: Awaited[] + + @HasMany(() => ActorFollowModel, { + foreignKey: { + name: 'targetActorId', + allowNull: false + }, + as: 'ActorFollowers', + onDelete: 'cascade' + }) + ActorFollowers: Awaited[] + + @ForeignKey(() => ServerModel) + @Column + serverId: number + + @BelongsTo(() => ServerModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'cascade' + }) + Server: Awaited + + @HasOne(() => AccountModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'cascade', + hooks: true + }) + Account: Awaited + + @HasOne(() => VideoChannelModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'cascade', + hooks: true + }) + VideoChannel: Awaited + + // --------------------------------------------------------------------------- + + static getSQLAttributes (tableName: string, aliasPrefix = '') { + return buildSQLAttributes({ + model: this, + tableName, + aliasPrefix + }) + } + + static getSQLAPIAttributes (tableName: string, aliasPrefix = '') { + return buildSQLAttributes({ + model: this, + tableName, + aliasPrefix, + excludeAttributes: unusedActorAttributesForAPI + }) + } + + // --------------------------------------------------------------------------- + + // FIXME: have to specify the result type to not break peertube typings generation + static wherePreferredUsername (preferredUsername: string, colName = 'preferredUsername'): Where { + return where(fn('lower', col(colName)), preferredUsername.toLowerCase()) + } + + // --------------------------------------------------------------------------- + + static async load (id: number): Promise { + const actorServer = await getServerActor() + if (id === actorServer.id) return actorServer + + return ActorModel.unscoped().findByPk(id) + } + + static loadFull (id: number): Promise { + return ActorModel.scope(ScopeNames.FULL).findByPk(id) + } + + static loadAccountActorFollowerUrlByVideoId (videoId: number, transaction: Transaction) { + const query = `SELECT "actor"."id" AS "id", "actor"."followersUrl" AS "followersUrl" ` + + `FROM "actor" ` + + `INNER JOIN "account" ON "actor"."id" = "account"."actorId" ` + + `INNER JOIN "videoChannel" ON "videoChannel"."accountId" = "account"."id" ` + + `INNER JOIN "video" ON "video"."channelId" = "videoChannel"."id" AND "video"."id" = :videoId` + + const options = { + type: QueryTypes.SELECT as QueryTypes.SELECT, + replacements: { videoId }, + plain: true as true, + transaction + } + + return ActorModel.sequelize.query(query, options) + } + + static listByFollowersUrls (followersUrls: string[], transaction?: Transaction): Promise { + const query = { + where: { + followersUrl: { + [Op.in]: followersUrls + } + }, + transaction + } + + return ActorModel.scope(ScopeNames.FULL).findAll(query) + } + + static loadLocalByName (preferredUsername: string, transaction?: Transaction): Promise { + const fun = () => { + const query = { + where: { + [Op.and]: [ + this.wherePreferredUsername(preferredUsername, '"ActorModel"."preferredUsername"'), + { + serverId: null + } + ] + }, + transaction + } + + return ActorModel.scope(ScopeNames.FULL).findOne(query) + } + + return ModelCache.Instance.doCache({ + cacheType: 'local-actor-name', + key: preferredUsername, + // The server actor never change, so we can easily cache it + whitelist: () => preferredUsername === SERVER_ACTOR_NAME, + fun + }) + } + + static loadLocalUrlByName (preferredUsername: string, transaction?: Transaction): Promise { + const fun = () => { + const query = { + attributes: [ 'url' ], + where: { + [Op.and]: [ + this.wherePreferredUsername(preferredUsername), + { + serverId: null + } + ] + }, + transaction + } + + return ActorModel.unscoped().findOne(query) + } + + return ModelCache.Instance.doCache({ + cacheType: 'local-actor-url', + key: preferredUsername, + // The server actor never change, so we can easily cache it + whitelist: () => preferredUsername === SERVER_ACTOR_NAME, + fun + }) + } + + static loadByNameAndHost (preferredUsername: string, host: string): Promise { + const query = { + where: this.wherePreferredUsername(preferredUsername, '"ActorModel"."preferredUsername"'), + include: [ + { + model: ServerModel, + required: true, + where: { + host + } + } + ] + } + + return ActorModel.scope(ScopeNames.FULL).findOne(query) + } + + static loadByUrl (url: string, transaction?: Transaction): Promise { + const query = { + where: { + url + }, + transaction, + include: [ + { + attributes: [ 'id' ], + model: AccountModel.unscoped(), + required: false + }, + { + attributes: [ 'id' ], + model: VideoChannelModel.unscoped(), + required: false + } + ] + } + + return ActorModel.unscoped().findOne(query) + } + + static loadByUrlAndPopulateAccountAndChannel (url: string, transaction?: Transaction): Promise { + const query = { + where: { + url + }, + transaction + } + + return ActorModel.scope(ScopeNames.FULL).findOne(query) + } + + static rebuildFollowsCount (ofId: number, type: 'followers' | 'following', transaction?: Transaction) { + const sanitizedOfId = forceNumber(ofId) + const where = { id: sanitizedOfId } + + let columnToUpdate: string + let columnOfCount: string + + if (type === 'followers') { + columnToUpdate = 'followersCount' + columnOfCount = 'targetActorId' + } else { + columnToUpdate = 'followingCount' + columnOfCount = 'actorId' + } + + return ActorModel.update({ + [columnToUpdate]: literal(`(SELECT COUNT(*) FROM "actorFollow" WHERE "${columnOfCount}" = ${sanitizedOfId} AND "state" = 'accepted')`) + }, { where, transaction }) + } + + static loadAccountActorByVideoId (videoId: number, transaction: Transaction): Promise { + const query = { + include: [ + { + attributes: [ 'id' ], + model: AccountModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'id', 'accountId' ], + model: VideoChannelModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'id', 'channelId' ], + model: VideoModel.unscoped(), + where: { + id: videoId + } + } + ] + } + ] + } + ], + transaction + } + + return ActorModel.unscoped().findOne(query) + } + + getSharedInbox (this: MActorWithInboxes) { + return this.sharedInboxUrl || this.inboxUrl + } + + toFormattedSummaryJSON (this: MActorSummaryFormattable) { + return { + url: this.url, + name: this.preferredUsername, + host: this.getHost(), + avatars: (this.Avatars || []).map(a => a.toFormattedJSON()) + } + } + + toFormattedJSON (this: MActorFormattable) { + return { + ...this.toFormattedSummaryJSON(), + + id: this.id, + hostRedundancyAllowed: this.getRedundancyAllowed(), + followingCount: this.followingCount, + followersCount: this.followersCount, + createdAt: this.getCreatedAt(), + + banners: (this.Banners || []).map(b => b.toFormattedJSON()) + } + } + + toActivityPubObject (this: MActorAPChannel | MActorAPAccount, name: string) { + let icon: ActivityIconObject[] + let image: ActivityIconObject + + if (this.hasImage(ActorImageType.AVATAR)) { + icon = this.Avatars.map(a => a.toActivityPubObject()) + } + + if (this.hasImage(ActorImageType.BANNER)) { + const banner = getBiggestActorImage((this as MActorAPChannel).Banners) + const extension = getLowercaseExtension(banner.filename) + + image = { + type: 'Image', + mediaType: MIMETYPES.IMAGE.EXT_MIMETYPE[extension], + height: banner.height, + width: banner.width, + url: ActorImageModel.getImageUrl(banner) + } + } + + const json = { + type: this.type, + id: this.url, + following: this.getFollowingUrl(), + followers: this.getFollowersUrl(), + playlists: this.getPlaylistsUrl(), + inbox: this.inboxUrl, + outbox: this.outboxUrl, + preferredUsername: this.preferredUsername, + url: this.url, + name, + endpoints: { + sharedInbox: this.sharedInboxUrl + }, + publicKey: { + id: this.getPublicKeyUrl(), + owner: this.url, + publicKeyPem: this.publicKey + }, + published: this.getCreatedAt().toISOString(), + + icon, + + image + } + + return activityPubContextify(json, 'Actor', getContextFilter()) + } + + getFollowerSharedInboxUrls (t: Transaction) { + const query = { + attributes: [ 'sharedInboxUrl' ], + include: [ + { + attribute: [], + model: ActorFollowModel.unscoped(), + required: true, + as: 'ActorFollowing', + where: { + state: 'accepted', + targetActorId: this.id + } + } + ], + transaction: t + } + + return ActorModel.findAll(query) + .then(accounts => accounts.map(a => a.sharedInboxUrl)) + } + + getFollowingUrl () { + return this.url + '/following' + } + + getFollowersUrl () { + return this.url + '/followers' + } + + getPlaylistsUrl () { + return this.url + '/playlists' + } + + getPublicKeyUrl () { + return this.url + '#main-key' + } + + isOwned () { + return this.serverId === null + } + + getWebfingerUrl (this: MActorHost) { + return 'acct:' + this.preferredUsername + '@' + this.getHost() + } + + getIdentifier (this: MActorHost) { + return this.Server ? `${this.preferredUsername}@${this.Server.host}` : this.preferredUsername + } + + getHost (this: MActorHostOnly) { + return this.Server ? this.Server.host : WEBSERVER.HOST + } + + getRedundancyAllowed () { + return this.Server ? this.Server.redundancyAllowed : false + } + + hasImage (type: ActorImageType_Type) { + const images = type === ActorImageType.AVATAR + ? this.Avatars + : this.Banners + + return Array.isArray(images) && images.length !== 0 + } + + isOutdated () { + if (this.isOwned()) return false + + return isOutdated(this, ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL) + } + + getCreatedAt (this: MActorAPChannel | MActorAPAccount | MActorFormattable) { + return this.remoteCreatedAt || this.createdAt + } +} diff --git a/server/server/models/actor/sql/instance-list-followers-query-builder.ts b/server/server/models/actor/sql/instance-list-followers-query-builder.ts new file mode 100644 index 000000000..ce91bca0d --- /dev/null +++ b/server/server/models/actor/sql/instance-list-followers-query-builder.ts @@ -0,0 +1,69 @@ +import { Sequelize } from 'sequelize' +import { ModelBuilder } from '@server/models/shared/index.js' +import { MActorFollowActorsDefault } from '@server/types/models/index.js' +import { ActivityPubActorType, FollowState } from '@peertube/peertube-models' +import { parseRowCountResult } from '../../shared/index.js' +import { InstanceListFollowsQueryBuilder } from './shared/instance-list-follows-query-builder.js' + +export interface ListFollowersOptions { + actorIds: number[] + start: number + count: number + sort: string + state?: FollowState + actorType?: ActivityPubActorType + search?: string +} + +export class InstanceListFollowersQueryBuilder extends InstanceListFollowsQueryBuilder { + + constructor ( + protected readonly sequelize: Sequelize, + protected readonly options: ListFollowersOptions + ) { + super(sequelize, options) + } + + async listFollowers () { + this.buildListQuery() + + const results = await this.runQuery({ nest: true }) + const modelBuilder = new ModelBuilder(this.sequelize) + + return modelBuilder.createModels(results, 'ActorFollow') + } + + async countFollowers () { + this.buildCountQuery() + + const result = await this.runQuery() + + return parseRowCountResult(result) + } + + protected getWhere () { + let where = 'WHERE "ActorFollowing"."id" IN (:actorIds) ' + this.replacements.actorIds = this.options.actorIds + + if (this.options.state) { + where += 'AND "ActorFollowModel"."state" = :state ' + this.replacements.state = this.options.state + } + + if (this.options.search) { + const escapedLikeSearch = this.sequelize.escape('%' + this.options.search + '%') + + where += `AND (` + + `"ActorFollower->Server"."host" ILIKE ${escapedLikeSearch} ` + + `OR "ActorFollower"."preferredUsername" ILIKE ${escapedLikeSearch} ` + + `)` + } + + if (this.options.actorType) { + where += `AND "ActorFollower"."type" = :actorType ` + this.replacements.actorType = this.options.actorType + } + + return where + } +} diff --git a/server/server/models/actor/sql/instance-list-following-query-builder.ts b/server/server/models/actor/sql/instance-list-following-query-builder.ts new file mode 100644 index 000000000..dab70db43 --- /dev/null +++ b/server/server/models/actor/sql/instance-list-following-query-builder.ts @@ -0,0 +1,69 @@ +import { Sequelize } from 'sequelize' +import { ModelBuilder } from '@server/models/shared/index.js' +import { MActorFollowActorsDefault } from '@server/types/models/index.js' +import { ActivityPubActorType, FollowState } from '@peertube/peertube-models' +import { parseRowCountResult } from '../../shared/index.js' +import { InstanceListFollowsQueryBuilder } from './shared/instance-list-follows-query-builder.js' + +export interface ListFollowingOptions { + followerId: number + start: number + count: number + sort: string + state?: FollowState + actorType?: ActivityPubActorType + search?: string +} + +export class InstanceListFollowingQueryBuilder extends InstanceListFollowsQueryBuilder { + + constructor ( + protected readonly sequelize: Sequelize, + protected readonly options: ListFollowingOptions + ) { + super(sequelize, options) + } + + async listFollowing () { + this.buildListQuery() + + const results = await this.runQuery({ nest: true }) + const modelBuilder = new ModelBuilder(this.sequelize) + + return modelBuilder.createModels(results, 'ActorFollow') + } + + async countFollowing () { + this.buildCountQuery() + + const result = await this.runQuery() + + return parseRowCountResult(result) + } + + protected getWhere () { + let where = 'WHERE "ActorFollowModel"."actorId" = :followerId ' + this.replacements.followerId = this.options.followerId + + if (this.options.state) { + where += 'AND "ActorFollowModel"."state" = :state ' + this.replacements.state = this.options.state + } + + if (this.options.search) { + const escapedLikeSearch = this.sequelize.escape('%' + this.options.search + '%') + + where += `AND (` + + `"ActorFollowing->Server"."host" ILIKE ${escapedLikeSearch} ` + + `OR "ActorFollowing"."preferredUsername" ILIKE ${escapedLikeSearch} ` + + `)` + } + + if (this.options.actorType) { + where += `AND "ActorFollowing"."type" = :actorType ` + this.replacements.actorType = this.options.actorType + } + + return where + } +} diff --git a/server/server/models/actor/sql/shared/actor-follow-table-attributes.ts b/server/server/models/actor/sql/shared/actor-follow-table-attributes.ts new file mode 100644 index 000000000..409c2f0e2 --- /dev/null +++ b/server/server/models/actor/sql/shared/actor-follow-table-attributes.ts @@ -0,0 +1,28 @@ +import { Memoize } from '@server/helpers/memoize.js' +import { ServerModel } from '@server/models/server/server.js' +import { ActorModel } from '../../actor.js' +import { ActorFollowModel } from '../../actor-follow.js' +import { ActorImageModel } from '../../actor-image.js' + +export class ActorFollowTableAttributes { + + @Memoize() + getFollowAttributes () { + return ActorFollowModel.getSQLAttributes('ActorFollowModel').join(', ') + } + + @Memoize() + getActorAttributes (actorTableName: string) { + return ActorModel.getSQLAttributes(actorTableName, `${actorTableName}.`).join(', ') + } + + @Memoize() + getServerAttributes (actorTableName: string) { + return ServerModel.getSQLAttributes(`${actorTableName}->Server`, `${actorTableName}.Server.`).join(', ') + } + + @Memoize() + getAvatarAttributes (actorTableName: string) { + return ActorImageModel.getSQLAttributes(`${actorTableName}->Avatars`, `${actorTableName}.Avatars.`).join(', ') + } +} diff --git a/server/server/models/actor/sql/shared/instance-list-follows-query-builder.ts b/server/server/models/actor/sql/shared/instance-list-follows-query-builder.ts new file mode 100644 index 000000000..e0569e2f8 --- /dev/null +++ b/server/server/models/actor/sql/shared/instance-list-follows-query-builder.ts @@ -0,0 +1,97 @@ +import { Sequelize } from 'sequelize' +import { AbstractRunQuery } from '@server/models/shared/index.js' +import { ActorImageType } from '@peertube/peertube-models' +import { getInstanceFollowsSort } from '../../../shared/index.js' +import { ActorFollowTableAttributes } from './actor-follow-table-attributes.js' + +type BaseOptions = { + sort: string + count: number + start: number +} + +export abstract class InstanceListFollowsQueryBuilder extends AbstractRunQuery { + protected readonly tableAttributes = new ActorFollowTableAttributes() + + protected innerQuery: string + + constructor ( + protected readonly sequelize: Sequelize, + protected readonly options: T + ) { + super(sequelize) + } + + protected abstract getWhere (): string + + protected getJoins () { + return 'INNER JOIN "actor" "ActorFollower" ON "ActorFollower"."id" = "ActorFollowModel"."actorId" ' + + 'INNER JOIN "actor" "ActorFollowing" ON "ActorFollowing"."id" = "ActorFollowModel"."targetActorId" ' + } + + protected getServerJoin (actorName: string) { + return `LEFT JOIN "server" "${actorName}->Server" ON "${actorName}"."serverId" = "${actorName}->Server"."id" ` + } + + protected getAvatarsJoin (actorName: string) { + return `LEFT JOIN "actorImage" "${actorName}->Avatars" ON "${actorName}.id" = "${actorName}->Avatars"."actorId" ` + + `AND "${actorName}->Avatars"."type" = ${ActorImageType.AVATAR}` + } + + private buildInnerQuery () { + this.innerQuery = `${this.getInnerSelect()} ` + + `FROM "actorFollow" AS "ActorFollowModel" ` + + `${this.getJoins()} ` + + `${this.getServerJoin('ActorFollowing')} ` + + `${this.getServerJoin('ActorFollower')} ` + + `${this.getWhere()} ` + + `${this.getOrder()} ` + + `LIMIT :limit OFFSET :offset ` + + this.replacements.limit = this.options.count + this.replacements.offset = this.options.start + } + + protected buildListQuery () { + this.buildInnerQuery() + + this.query = `${this.getSelect()} ` + + `FROM (${this.innerQuery}) AS "ActorFollowModel" ` + + `${this.getAvatarsJoin('ActorFollower')} ` + + `${this.getAvatarsJoin('ActorFollowing')} ` + + `${this.getOrder()}` + } + + protected buildCountQuery () { + this.query = `SELECT COUNT(*) AS "total" ` + + `FROM "actorFollow" AS "ActorFollowModel" ` + + `${this.getJoins()} ` + + `${this.getServerJoin('ActorFollowing')} ` + + `${this.getServerJoin('ActorFollower')} ` + + `${this.getWhere()}` + } + + private getInnerSelect () { + return this.buildSelect([ + this.tableAttributes.getFollowAttributes(), + this.tableAttributes.getActorAttributes('ActorFollower'), + this.tableAttributes.getActorAttributes('ActorFollowing'), + this.tableAttributes.getServerAttributes('ActorFollower'), + this.tableAttributes.getServerAttributes('ActorFollowing') + ]) + } + + private getSelect () { + return this.buildSelect([ + '"ActorFollowModel".*', + this.tableAttributes.getAvatarAttributes('ActorFollower'), + this.tableAttributes.getAvatarAttributes('ActorFollowing') + ]) + } + + private getOrder () { + const orders = getInstanceFollowsSort(this.options.sort) + + return 'ORDER BY ' + orders.map(o => `"${o[0]}" ${o[1]}`).join(', ') + } +} diff --git a/server/server/models/application/application.ts b/server/server/models/application/application.ts new file mode 100644 index 000000000..1b1ff15b1 --- /dev/null +++ b/server/server/models/application/application.ts @@ -0,0 +1,79 @@ +import memoizee from 'memoizee' +import { AllowNull, Column, Default, DefaultScope, HasOne, IsInt, Model, Table } from 'sequelize-typescript' +import { getNodeABIVersion } from '@server/helpers/version.js' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { AccountModel } from '../account/account.js' + +export const getServerActor = memoizee(async function () { + const application = await ApplicationModel.load() + if (!application) throw Error('Could not load Application from database.') + + const actor = application.Account.Actor + actor.Account = application.Account + + return actor +}, { promise: true }) + +@DefaultScope(() => ({ + include: [ + { + model: AccountModel, + required: true + } + ] +})) +@Table({ + tableName: 'application', + timestamps: false +}) +export class ApplicationModel extends Model>> { + + @AllowNull(false) + @Default(0) + @IsInt + @Column + migrationVersion: number + + @AllowNull(true) + @Column + latestPeerTubeVersion: string + + @AllowNull(false) + @Column + nodeVersion: string + + @AllowNull(false) + @Column + nodeABIVersion: number + + @HasOne(() => AccountModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'cascade' + }) + Account: Awaited + + static countTotal () { + return ApplicationModel.count() + } + + static load () { + return ApplicationModel.findOne() + } + + static async nodeABIChanged () { + const application = await this.load() + + return application.nodeABIVersion !== getNodeABIVersion() + } + + static async updateNodeVersions () { + const application = await this.load() + + application.nodeABIVersion = getNodeABIVersion() + application.nodeVersion = process.version + + await application.save() + } +} diff --git a/server/server/models/oauth/oauth-client.ts b/server/server/models/oauth/oauth-client.ts new file mode 100644 index 000000000..3d17f2431 --- /dev/null +++ b/server/server/models/oauth/oauth-client.ts @@ -0,0 +1,63 @@ +import { AllowNull, Column, CreatedAt, DataType, HasMany, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { OAuthTokenModel } from './oauth-token.js' + +@Table({ + tableName: 'oAuthClient', + indexes: [ + { + fields: [ 'clientId' ], + unique: true + }, + { + fields: [ 'clientId', 'clientSecret' ], + unique: true + } + ] +}) +export class OAuthClientModel extends Model>> { + + @AllowNull(false) + @Column + clientId: string + + @AllowNull(false) + @Column + clientSecret: string + + @Column(DataType.ARRAY(DataType.STRING)) + grants: string[] + + @Column(DataType.ARRAY(DataType.STRING)) + redirectUris: string[] + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @HasMany(() => OAuthTokenModel, { + onDelete: 'cascade' + }) + OAuthTokens: Awaited[] + + static countTotal () { + return OAuthClientModel.count() + } + + static loadFirstClient () { + return OAuthClientModel.findOne() + } + + static getByIdAndSecret (clientId: string, clientSecret: string) { + const query = { + where: { + clientId, + clientSecret + } + } + + return OAuthClientModel.findOne(query) + } +} diff --git a/server/server/models/oauth/oauth-token.ts b/server/server/models/oauth/oauth-token.ts new file mode 100644 index 000000000..7101e199a --- /dev/null +++ b/server/server/models/oauth/oauth-token.ts @@ -0,0 +1,222 @@ +import { Transaction } from 'sequelize' +import { + AfterDestroy, + AfterUpdate, + AllowNull, + BelongsTo, + Column, + CreatedAt, + ForeignKey, + Model, + Scopes, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { TokensCache } from '@server/lib/auth/tokens-cache.js' +import { MUserAccountId } from '@server/types/models/index.js' +import { MOAuthTokenUser } from '@server/types/models/oauth/oauth-token.js' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { logger } from '../../helpers/logger.js' +import { AccountModel } from '../account/account.js' +import { ActorModel } from '../actor/actor.js' +import { UserModel } from '../user/user.js' +import { OAuthClientModel } from './oauth-client.js' + +export type OAuthTokenInfo = { + refreshToken: string + refreshTokenExpiresAt: Date + client: { + id: number + grants: string[] + } + user: MUserAccountId + token: MOAuthTokenUser +} + +enum ScopeNames { + WITH_USER = 'WITH_USER' +} + +@Scopes(() => ({ + [ScopeNames.WITH_USER]: { + include: [ + { + model: UserModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'id' ], + model: AccountModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'id', 'url' ], + model: ActorModel.unscoped(), + required: true + } + ] + } + ] + } + ] + } +})) +@Table({ + tableName: 'oAuthToken', + indexes: [ + { + fields: [ 'refreshToken' ], + unique: true + }, + { + fields: [ 'accessToken' ], + unique: true + }, + { + fields: [ 'userId' ] + }, + { + fields: [ 'oAuthClientId' ] + } + ] +}) +export class OAuthTokenModel extends Model>> { + + @AllowNull(false) + @Column + accessToken: string + + @AllowNull(false) + @Column + accessTokenExpiresAt: Date + + @AllowNull(false) + @Column + refreshToken: string + + @AllowNull(false) + @Column + refreshTokenExpiresAt: Date + + @Column + authName: string + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => UserModel) + @Column + userId: number + + @BelongsTo(() => UserModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + User: Awaited + + @ForeignKey(() => OAuthClientModel) + @Column + oAuthClientId: number + + @BelongsTo(() => OAuthClientModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + OAuthClients: Awaited[] + + @AfterUpdate + @AfterDestroy + static removeTokenCache (token: OAuthTokenModel) { + return TokensCache.Instance.clearCacheByToken(token.accessToken) + } + + static loadByRefreshToken (refreshToken: string) { + const query = { + where: { refreshToken } + } + + return OAuthTokenModel.findOne(query) + } + + static getByRefreshTokenAndPopulateClient (refreshToken: string) { + const query = { + where: { + refreshToken + }, + include: [ OAuthClientModel ] + } + + return OAuthTokenModel.scope(ScopeNames.WITH_USER) + .findOne(query) + .then(token => { + if (!token) return null + + return { + refreshToken: token.refreshToken, + refreshTokenExpiresAt: token.refreshTokenExpiresAt, + client: { + id: token.oAuthClientId, + grants: [] + }, + user: token.User, + token + } as OAuthTokenInfo + }) + .catch(err => { + logger.error('getRefreshToken error.', { err }) + throw err + }) + } + + static getByTokenAndPopulateUser (bearerToken: string): Promise { + const query = { + where: { + accessToken: bearerToken + } + } + + return OAuthTokenModel.scope(ScopeNames.WITH_USER) + .findOne(query) + .then(token => { + if (!token) return null + + return Object.assign(token, { user: token.User }) + }) + } + + static getByRefreshTokenAndPopulateUser (refreshToken: string): Promise { + const query = { + where: { + refreshToken + } + } + + return OAuthTokenModel.scope(ScopeNames.WITH_USER) + .findOne(query) + .then(token => { + if (!token) return undefined + + return Object.assign(token, { user: token.User }) + }) + } + + static deleteUserToken (userId: number, t?: Transaction) { + TokensCache.Instance.deleteUserToken(userId) + + const query = { + where: { + userId + }, + transaction: t + } + + return OAuthTokenModel.destroy(query) + } +} diff --git a/server/server/models/redundancy/video-redundancy.ts b/server/server/models/redundancy/video-redundancy.ts new file mode 100644 index 000000000..26089594d --- /dev/null +++ b/server/server/models/redundancy/video-redundancy.ts @@ -0,0 +1,793 @@ +import sample from 'lodash-es/sample.js' +import { literal, Op, QueryTypes, Transaction, WhereOptions } from 'sequelize' +import { + AllowNull, + BeforeDestroy, + BelongsTo, + Column, + CreatedAt, + DataType, + ForeignKey, + Is, + Model, + Scopes, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { + CacheFileObject, + FileRedundancyInformation, + StreamingPlaylistRedundancyInformation, + VideoPrivacy, + VideoRedundanciesTarget, + VideoRedundancy, + VideoRedundancyStrategy, + VideoRedundancyStrategyWithManual +} from '@peertube/peertube-models' +import { isTestInstance } from '@peertube/peertube-node-utils' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { getServerActor } from '@server/models/application/application.js' +import { MActor, MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models/index.js' +import { isActivityPubUrlValid, isUrlValid } from '../../helpers/custom-validators/activitypub/misc.js' +import { logger } from '../../helpers/logger.js' +import { CONFIG } from '../../initializers/config.js' +import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../initializers/constants.js' +import { ActorModel } from '../actor/actor.js' +import { ServerModel } from '../server/server.js' +import { getSort, getVideoSort, parseAggregateResult, throwIfNotValid } from '../shared/index.js' +import { ScheduleVideoUpdateModel } from '../video/schedule-video-update.js' +import { VideoChannelModel } from '../video/video-channel.js' +import { VideoFileModel } from '../video/video-file.js' +import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist.js' +import { VideoModel } from '../video/video.js' + +export enum ScopeNames { + WITH_VIDEO = 'WITH_VIDEO' +} + +@Scopes(() => ({ + [ScopeNames.WITH_VIDEO]: { + include: [ + { + model: VideoFileModel, + required: false, + include: [ + { + model: VideoModel, + required: true + } + ] + }, + { + model: VideoStreamingPlaylistModel, + required: false, + include: [ + { + model: VideoModel, + required: true + } + ] + } + ] + } +})) + +@Table({ + tableName: 'videoRedundancy', + indexes: [ + { + fields: [ 'videoFileId' ] + }, + { + fields: [ 'actorId' ] + }, + { + fields: [ 'expiresOn' ] + }, + { + fields: [ 'url' ], + unique: true + } + ] +}) +export class VideoRedundancyModel extends Model>> { + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(true) + @Column + expiresOn: Date + + @AllowNull(false) + @Is('VideoRedundancyFileUrl', value => throwIfNotValid(value, isUrlValid, 'fileUrl')) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max)) + fileUrl: string + + @AllowNull(false) + @Is('VideoRedundancyUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url')) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS_REDUNDANCY.URL.max)) + url: string + + @AllowNull(true) + @Column + strategy: string // Only used by us + + @ForeignKey(() => VideoFileModel) + @Column + videoFileId: number + + @BelongsTo(() => VideoFileModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'cascade' + }) + VideoFile: Awaited + + @ForeignKey(() => VideoStreamingPlaylistModel) + @Column + videoStreamingPlaylistId: number + + @BelongsTo(() => VideoStreamingPlaylistModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'cascade' + }) + VideoStreamingPlaylist: Awaited + + @ForeignKey(() => ActorModel) + @Column + actorId: number + + @BelongsTo(() => ActorModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + Actor: Awaited + + @BeforeDestroy + static async removeFile (instance: VideoRedundancyModel) { + if (!instance.isOwned()) return + + if (instance.videoFileId) { + const videoFile = await VideoFileModel.loadWithVideo(instance.videoFileId) + + const logIdentifier = `${videoFile.Video.uuid}-${videoFile.resolution}` + logger.info('Removing duplicated video file %s.', logIdentifier) + + videoFile.Video.removeWebVideoFile(videoFile, true) + .catch(err => logger.error('Cannot delete %s files.', logIdentifier, { err })) + } + + if (instance.videoStreamingPlaylistId) { + const videoStreamingPlaylist = await VideoStreamingPlaylistModel.loadWithVideo(instance.videoStreamingPlaylistId) + + const videoUUID = videoStreamingPlaylist.Video.uuid + logger.info('Removing duplicated video streaming playlist %s.', videoUUID) + + videoStreamingPlaylist.Video.removeStreamingPlaylistFiles(videoStreamingPlaylist, true) + .catch(err => logger.error('Cannot delete video streaming playlist files of %s.', videoUUID, { err })) + } + + return undefined + } + + static async loadLocalByFileId (videoFileId: number): Promise { + const actor = await getServerActor() + + const query = { + where: { + actorId: actor.id, + videoFileId + } + } + + return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) + } + + static async listLocalByVideoId (videoId: number): Promise { + const actor = await getServerActor() + + const queryStreamingPlaylist = { + where: { + actorId: actor.id + }, + include: [ + { + model: VideoStreamingPlaylistModel.unscoped(), + required: true, + include: [ + { + model: VideoModel.unscoped(), + required: true, + where: { + id: videoId + } + } + ] + } + ] + } + + const queryFiles = { + where: { + actorId: actor.id + }, + include: [ + { + model: VideoFileModel, + required: true, + include: [ + { + model: VideoModel, + required: true, + where: { + id: videoId + } + } + ] + } + ] + } + + return Promise.all([ + VideoRedundancyModel.findAll(queryStreamingPlaylist), + VideoRedundancyModel.findAll(queryFiles) + ]).then(([ r1, r2 ]) => r1.concat(r2)) + } + + static async loadLocalByStreamingPlaylistId (videoStreamingPlaylistId: number): Promise { + const actor = await getServerActor() + + const query = { + where: { + actorId: actor.id, + videoStreamingPlaylistId + } + } + + return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) + } + + static loadByIdWithVideo (id: number, transaction?: Transaction): Promise { + const query = { + where: { id }, + transaction + } + + return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query) + } + + static loadByUrl (url: string, transaction?: Transaction): Promise { + const query = { + where: { + url + }, + transaction + } + + return VideoRedundancyModel.findOne(query) + } + + static async isLocalByVideoUUIDExists (uuid: string) { + const actor = await getServerActor() + + const query = { + raw: true, + attributes: [ 'id' ], + where: { + actorId: actor.id + }, + include: [ + { + attributes: [], + model: VideoFileModel, + required: true, + include: [ + { + attributes: [], + model: VideoModel, + required: true, + where: { + uuid + } + } + ] + } + ] + } + + return VideoRedundancyModel.findOne(query) + .then(r => !!r) + } + + static async getVideoSample (p: Promise) { + const rows = await p + if (rows.length === 0) return undefined + + const ids = rows.map(r => r.id) + const id = sample(ids) + + return VideoModel.loadWithFiles(id, undefined, !isTestInstance()) + } + + static async findMostViewToDuplicate (randomizedFactor: number) { + const peertubeActor = await getServerActor() + + // On VideoModel! + const query = { + attributes: [ 'id', 'views' ], + limit: randomizedFactor, + order: getVideoSort('-views'), + where: { + privacy: VideoPrivacy.PUBLIC, + isLive: false, + ...this.buildVideoIdsForDuplication(peertubeActor) + }, + include: [ + VideoRedundancyModel.buildServerRedundancyInclude() + ] + } + + return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) + } + + static async findTrendingToDuplicate (randomizedFactor: number) { + const peertubeActor = await getServerActor() + + // On VideoModel! + const query = { + attributes: [ 'id', 'views' ], + subQuery: false, + group: 'VideoModel.id', + limit: randomizedFactor, + order: getVideoSort('-trending'), + where: { + privacy: VideoPrivacy.PUBLIC, + isLive: false, + ...this.buildVideoIdsForDuplication(peertubeActor) + }, + include: [ + VideoRedundancyModel.buildServerRedundancyInclude(), + + VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS) + ] + } + + return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) + } + + static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) { + const peertubeActor = await getServerActor() + + // On VideoModel! + const query = { + attributes: [ 'id', 'publishedAt' ], + limit: randomizedFactor, + order: getVideoSort('-publishedAt'), + where: { + privacy: VideoPrivacy.PUBLIC, + isLive: false, + views: { + [Op.gte]: minViews + }, + ...this.buildVideoIdsForDuplication(peertubeActor) + }, + include: [ + VideoRedundancyModel.buildServerRedundancyInclude(), + + // Required by publishedAt sort + { + model: ScheduleVideoUpdateModel.unscoped(), + required: false + } + ] + } + + return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query)) + } + + static async loadOldestLocalExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number): Promise { + const expiredDate = new Date() + expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs) + + const actor = await getServerActor() + + const query = { + where: { + actorId: actor.id, + strategy, + createdAt: { + [Op.lt]: expiredDate + } + } + } + + return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query) + } + + static async listLocalExpired (): Promise { + const actor = await getServerActor() + + const query = { + where: { + actorId: actor.id, + expiresOn: { + [Op.lt]: new Date() + } + } + } + + return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query) + } + + static async listRemoteExpired () { + const actor = await getServerActor() + + const query = { + where: { + actorId: { + [Op.ne]: actor.id + }, + expiresOn: { + [Op.lt]: new Date(), + [Op.ne]: null + } + } + } + + return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query) + } + + static async listLocalOfServer (serverId: number) { + const actor = await getServerActor() + const buildVideoInclude = () => ({ + model: VideoModel, + required: true, + include: [ + { + attributes: [], + model: VideoChannelModel.unscoped(), + required: true, + include: [ + { + attributes: [], + model: ActorModel.unscoped(), + required: true, + where: { + serverId + } + } + ] + } + ] + }) + + const query = { + where: { + [Op.and]: [ + { + actorId: actor.id + }, + { + [Op.or]: [ + { + '$VideoStreamingPlaylist.id$': { + [Op.ne]: null + } + }, + { + '$VideoFile.id$': { + [Op.ne]: null + } + } + ] + } + ] + }, + include: [ + { + model: VideoFileModel.unscoped(), + required: false, + include: [ buildVideoInclude() ] + }, + { + model: VideoStreamingPlaylistModel.unscoped(), + required: false, + include: [ buildVideoInclude() ] + } + ] + } + + return VideoRedundancyModel.findAll(query) + } + + static listForApi (options: { + start: number + count: number + sort: string + target: VideoRedundanciesTarget + strategy?: string + }) { + const { start, count, sort, target, strategy } = options + const redundancyWhere: WhereOptions = {} + const videosWhere: WhereOptions = {} + let redundancySqlSuffix = '' + + if (target === 'my-videos') { + Object.assign(videosWhere, { remote: false }) + } else if (target === 'remote-videos') { + Object.assign(videosWhere, { remote: true }) + Object.assign(redundancyWhere, { strategy: { [Op.ne]: null } }) + redundancySqlSuffix = ' AND "videoRedundancy"."strategy" IS NOT NULL' + } + + if (strategy) { + Object.assign(redundancyWhere, { strategy }) + } + + const videoFilterWhere = { + [Op.and]: [ + { + [Op.or]: [ + { + id: { + [Op.in]: literal( + '(' + + 'SELECT "videoId" FROM "videoFile" ' + + 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoFileId" = "videoFile".id' + + redundancySqlSuffix + + ')' + ) + } + }, + { + id: { + [Op.in]: literal( + '(' + + 'select "videoId" FROM "videoStreamingPlaylist" ' + + 'INNER JOIN "videoRedundancy" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id' + + redundancySqlSuffix + + ')' + ) + } + } + ] + }, + + videosWhere + ] + } + + // /!\ On video model /!\ + const findOptions = { + offset: start, + limit: count, + order: getSort(sort), + include: [ + { + required: false, + model: VideoFileModel, + include: [ + { + model: VideoRedundancyModel.unscoped(), + required: false, + where: redundancyWhere + } + ] + }, + { + required: false, + model: VideoStreamingPlaylistModel.unscoped(), + include: [ + { + model: VideoRedundancyModel.unscoped(), + required: false, + where: redundancyWhere + }, + { + model: VideoFileModel, + required: false + } + ] + } + ], + where: videoFilterWhere + } + + // /!\ On video model /!\ + const countOptions = { + where: videoFilterWhere + } + + return Promise.all([ + VideoModel.findAll(findOptions), + + VideoModel.count(countOptions) + ]).then(([ data, total ]) => ({ total, data })) + } + + static async getStats (strategy: VideoRedundancyStrategyWithManual) { + const actor = await getServerActor() + + const sql = `WITH "tmp" AS ` + + `(` + + `SELECT "videoFile"."size" AS "videoFileSize", "videoStreamingFile"."size" AS "videoStreamingFileSize", ` + + `"videoFile"."videoId" AS "videoFileVideoId", "videoStreamingPlaylist"."videoId" AS "videoStreamingVideoId"` + + `FROM "videoRedundancy" AS "videoRedundancy" ` + + `LEFT JOIN "videoFile" AS "videoFile" ON "videoRedundancy"."videoFileId" = "videoFile"."id" ` + + `LEFT JOIN "videoStreamingPlaylist" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ` + + `LEFT JOIN "videoFile" AS "videoStreamingFile" ` + + `ON "videoStreamingPlaylist"."id" = "videoStreamingFile"."videoStreamingPlaylistId" ` + + `WHERE "videoRedundancy"."strategy" = :strategy AND "videoRedundancy"."actorId" = :actorId` + + `), ` + + `"videoIds" AS (` + + `SELECT "videoFileVideoId" AS "videoId" FROM "tmp" ` + + `UNION SELECT "videoStreamingVideoId" AS "videoId" FROM "tmp" ` + + `) ` + + `SELECT ` + + `COALESCE(SUM("videoFileSize"), '0') + COALESCE(SUM("videoStreamingFileSize"), '0') AS "totalUsed", ` + + `(SELECT COUNT("videoIds"."videoId") FROM "videoIds") AS "totalVideos", ` + + `COUNT(*) AS "totalVideoFiles" ` + + `FROM "tmp"` + + return VideoRedundancyModel.sequelize.query(sql, { + replacements: { strategy, actorId: actor.id }, + type: QueryTypes.SELECT + }).then(([ row ]) => ({ + totalUsed: parseAggregateResult(row.totalUsed), + totalVideos: row.totalVideos, + totalVideoFiles: row.totalVideoFiles + })) + } + + static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy { + const filesRedundancies: FileRedundancyInformation[] = [] + const streamingPlaylistsRedundancies: StreamingPlaylistRedundancyInformation[] = [] + + for (const file of video.VideoFiles) { + for (const redundancy of file.RedundancyVideos) { + filesRedundancies.push({ + id: redundancy.id, + fileUrl: redundancy.fileUrl, + strategy: redundancy.strategy, + createdAt: redundancy.createdAt, + updatedAt: redundancy.updatedAt, + expiresOn: redundancy.expiresOn, + size: file.size + }) + } + } + + for (const playlist of video.VideoStreamingPlaylists) { + const size = playlist.VideoFiles.reduce((a, b) => a + b.size, 0) + + for (const redundancy of playlist.RedundancyVideos) { + streamingPlaylistsRedundancies.push({ + id: redundancy.id, + fileUrl: redundancy.fileUrl, + strategy: redundancy.strategy, + createdAt: redundancy.createdAt, + updatedAt: redundancy.updatedAt, + expiresOn: redundancy.expiresOn, + size + }) + } + } + + return { + id: video.id, + name: video.name, + url: video.url, + uuid: video.uuid, + + redundancies: { + files: filesRedundancies, + streamingPlaylists: streamingPlaylistsRedundancies + } + } + } + + getVideo () { + if (this.VideoFile?.Video) return this.VideoFile.Video + + if (this.VideoStreamingPlaylist?.Video) return this.VideoStreamingPlaylist.Video + + return undefined + } + + getVideoUUID () { + const video = this.getVideo() + if (!video) return undefined + + return video.uuid + } + + isOwned () { + return !!this.strategy + } + + toActivityPubObject (this: MVideoRedundancyAP): CacheFileObject { + if (this.VideoStreamingPlaylist) { + return { + id: this.url, + type: 'CacheFile' as 'CacheFile', + object: this.VideoStreamingPlaylist.Video.url, + expires: this.expiresOn ? this.expiresOn.toISOString() : null, + url: { + type: 'Link', + mediaType: 'application/x-mpegURL', + href: this.fileUrl + } + } + } + + return { + id: this.url, + type: 'CacheFile' as 'CacheFile', + object: this.VideoFile.Video.url, + expires: this.expiresOn ? this.expiresOn.toISOString() : null, + url: { + type: 'Link', + mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[this.VideoFile.extname] as any, + href: this.fileUrl, + height: this.VideoFile.resolution, + size: this.VideoFile.size, + fps: this.VideoFile.fps + } + } + } + + // Don't include video files we already duplicated + private static buildVideoIdsForDuplication (peertubeActor: MActor) { + const notIn = literal( + '(' + + `SELECT "videoFile"."videoId" AS "videoId" FROM "videoRedundancy" ` + + `INNER JOIN "videoFile" ON "videoFile"."id" = "videoRedundancy"."videoFileId" ` + + `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` + + `UNION ` + + `SELECT "videoStreamingPlaylist"."videoId" AS "videoId" FROM "videoRedundancy" ` + + `INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoRedundancy"."videoStreamingPlaylistId" ` + + `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` + + ')' + ) + + return { + id: { + [Op.notIn]: notIn + } + } + } + + private static buildServerRedundancyInclude () { + return { + attributes: [], + model: VideoChannelModel.unscoped(), + required: true, + include: [ + { + attributes: [], + model: ActorModel.unscoped(), + required: true, + include: [ + { + attributes: [], + model: ServerModel.unscoped(), + required: true, + where: { + redundancyAllowed: true + } + } + ] + } + ] + } + } +} diff --git a/server/server/models/runner/runner-job.ts b/server/server/models/runner/runner-job.ts new file mode 100644 index 000000000..1ebcdae1d --- /dev/null +++ b/server/server/models/runner/runner-job.ts @@ -0,0 +1,366 @@ +import { + RunnerJob, + RunnerJobAdmin, + RunnerJobState, + type RunnerJobPayload, + type RunnerJobPrivatePayload, + type RunnerJobStateType, + type RunnerJobType +} from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { isArray, isUUIDValid } from '@server/helpers/custom-validators/misc.js' +import { CONSTRAINTS_FIELDS, RUNNER_JOB_STATES } from '@server/initializers/constants.js' +import { MRunnerJob, MRunnerJobRunner, MRunnerJobRunnerParent } from '@server/types/models/runners/index.js' +import { Op, Transaction } from 'sequelize' +import { + AllowNull, + BelongsTo, + Column, + CreatedAt, + DataType, + Default, + ForeignKey, + IsUUID, + Model, + Scopes, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { getSort, searchAttribute } from '../shared/index.js' +import { RunnerModel } from './runner.js' + +enum ScopeNames { + WITH_RUNNER = 'WITH_RUNNER', + WITH_PARENT = 'WITH_PARENT' +} + +@Scopes(() => ({ + [ScopeNames.WITH_RUNNER]: { + include: [ + { + model: RunnerModel.unscoped(), + required: false + } + ] + }, + [ScopeNames.WITH_PARENT]: { + include: [ + { + model: RunnerJobModel.unscoped(), + required: false + } + ] + } +})) +@Table({ + tableName: 'runnerJob', + indexes: [ + { + fields: [ 'uuid' ], + unique: true + }, + { + fields: [ 'processingJobToken' ], + unique: true + }, + { + fields: [ 'runnerId' ] + } + ] +}) +export class RunnerJobModel extends Model>> { + + @AllowNull(false) + @IsUUID(4) + @Column(DataType.UUID) + uuid: string + + @AllowNull(false) + @Column + type: RunnerJobType + + @AllowNull(false) + @Column(DataType.JSONB) + payload: RunnerJobPayload + + @AllowNull(false) + @Column(DataType.JSONB) + privatePayload: RunnerJobPrivatePayload + + @AllowNull(false) + @Column + state: RunnerJobStateType + + @AllowNull(false) + @Default(0) + @Column + failures: number + + @AllowNull(true) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.RUNNER_JOBS.ERROR_MESSAGE.max)) + error: string + + // Less has priority + @AllowNull(false) + @Column + priority: number + + // Used to fetch the appropriate job when the runner wants to post the result + @AllowNull(true) + @Column + processingJobToken: string + + @AllowNull(true) + @Column + progress: number + + @AllowNull(true) + @Column + startedAt: Date + + @AllowNull(true) + @Column + finishedAt: Date + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => RunnerJobModel) + @Column + dependsOnRunnerJobId: number + + @BelongsTo(() => RunnerJobModel, { + foreignKey: { + name: 'dependsOnRunnerJobId', + allowNull: true + }, + onDelete: 'cascade' + }) + DependsOnRunnerJob: Awaited + + @ForeignKey(() => RunnerModel) + @Column + runnerId: number + + @BelongsTo(() => RunnerModel, { + foreignKey: { + name: 'runnerId', + allowNull: true + }, + onDelete: 'SET NULL' + }) + Runner: Awaited + + // --------------------------------------------------------------------------- + + static loadWithRunner (uuid: string) { + const query = { + where: { uuid } + } + + return RunnerJobModel.scope(ScopeNames.WITH_RUNNER).findOne(query) + } + + static loadByRunnerAndJobTokensWithRunner (options: { + uuid: string + runnerToken: string + jobToken: string + }) { + const { uuid, runnerToken, jobToken } = options + + const query = { + where: { + uuid, + processingJobToken: jobToken + }, + include: { + model: RunnerModel.unscoped(), + required: true, + where: { + runnerToken + } + } + } + + return RunnerJobModel.findOne(query) + } + + static listAvailableJobs () { + const query = { + limit: 10, + order: getSort('priority'), + where: { + state: RunnerJobState.PENDING + } + } + + return RunnerJobModel.findAll(query) + } + + static listStalledJobs (options: { + staleTimeMS: number + types: RunnerJobType[] + }) { + const before = new Date(Date.now() - options.staleTimeMS) + + return RunnerJobModel.findAll({ + where: { + type: { + [Op.in]: options.types + }, + state: RunnerJobState.PROCESSING, + updatedAt: { + [Op.lt]: before + } + } + }) + } + + static listChildrenOf (job: MRunnerJob, transaction?: Transaction) { + const query = { + where: { + dependsOnRunnerJobId: job.id + }, + transaction + } + + return RunnerJobModel.findAll(query) + } + + static listForApi (options: { + start: number + count: number + sort: string + search?: string + stateOneOf?: RunnerJobStateType[] + }) { + const { start, count, sort, search, stateOneOf } = options + + const query = { + offset: start, + limit: count, + order: getSort(sort), + where: [] + } + + if (search) { + if (isUUIDValid(search)) { + query.where.push({ uuid: search }) + } else { + query.where.push({ + [Op.or]: [ + searchAttribute(search, 'type'), + searchAttribute(search, '$Runner.name$') + ] + }) + } + } + + if (isArray(stateOneOf) && stateOneOf.length !== 0) { + query.where.push({ + state: { + [Op.in]: stateOneOf + } + }) + } + + return Promise.all([ + RunnerJobModel.scope([ ScopeNames.WITH_RUNNER ]).count(query), + RunnerJobModel.scope([ ScopeNames.WITH_RUNNER, ScopeNames.WITH_PARENT ]).findAll(query) + ]).then(([ total, data ]) => ({ total, data })) + } + + static updateDependantJobsOf (runnerJob: MRunnerJob) { + const where = { + dependsOnRunnerJobId: runnerJob.id + } + + return RunnerJobModel.update({ state: RunnerJobState.PENDING }, { where }) + } + + static cancelAllJobs (options: { type: RunnerJobType }) { + const where = { + type: options.type + } + + return RunnerJobModel.update({ state: RunnerJobState.CANCELLED }, { where }) + } + + // --------------------------------------------------------------------------- + + resetToPending () { + this.state = RunnerJobState.PENDING + this.processingJobToken = null + this.progress = null + this.startedAt = null + this.runnerId = null + } + + setToErrorOrCancel ( + // eslint-disable-next-line max-len + state: typeof RunnerJobState.PARENT_ERRORED | typeof RunnerJobState.ERRORED | typeof RunnerJobState.CANCELLED | typeof RunnerJobState.PARENT_CANCELLED + ) { + this.state = state + this.processingJobToken = null + this.finishedAt = new Date() + } + + toFormattedJSON (this: MRunnerJobRunnerParent): RunnerJob { + const runner = this.Runner + ? { + id: this.Runner.id, + name: this.Runner.name, + description: this.Runner.description + } + : null + + const parent = this.DependsOnRunnerJob + ? { + id: this.DependsOnRunnerJob.id, + uuid: this.DependsOnRunnerJob.uuid, + type: this.DependsOnRunnerJob.type, + state: { + id: this.DependsOnRunnerJob.state, + label: RUNNER_JOB_STATES[this.DependsOnRunnerJob.state] + } + } + : undefined + + return { + uuid: this.uuid, + type: this.type, + + state: { + id: this.state, + label: RUNNER_JOB_STATES[this.state] + }, + + progress: this.progress, + priority: this.priority, + failures: this.failures, + error: this.error, + + payload: this.payload, + + startedAt: this.startedAt?.toISOString(), + finishedAt: this.finishedAt?.toISOString(), + + createdAt: this.createdAt.toISOString(), + updatedAt: this.updatedAt.toISOString(), + + parent, + runner + } + } + + toFormattedAdminJSON (this: MRunnerJobRunnerParent): RunnerJobAdmin { + return { + ...this.toFormattedJSON(), + + privatePayload: this.privatePayload + } + } +} diff --git a/server/server/models/runner/runner-registration-token.ts b/server/server/models/runner/runner-registration-token.ts new file mode 100644 index 000000000..1bde519ca --- /dev/null +++ b/server/server/models/runner/runner-registration-token.ts @@ -0,0 +1,103 @@ +import { FindOptions, literal } from 'sequelize' +import { AllowNull, Column, CreatedAt, HasMany, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { MRunnerRegistrationToken } from '@server/types/models/runners/index.js' +import { RunnerRegistrationToken } from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { getSort } from '../shared/index.js' +import { RunnerModel } from './runner.js' + +/** + * + * Tokens used by PeerTube runners to register themselves to the PeerTube instance + * + */ + +@Table({ + tableName: 'runnerRegistrationToken', + indexes: [ + { + fields: [ 'registrationToken' ], + unique: true + } + ] +}) +export class RunnerRegistrationTokenModel extends Model>> { + + @AllowNull(false) + @Column + registrationToken: string + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @HasMany(() => RunnerModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'cascade' + }) + Runners: Awaited[] + + static load (id: number) { + return RunnerRegistrationTokenModel.findByPk(id) + } + + static loadByRegistrationToken (registrationToken: string) { + const query = { + where: { registrationToken } + } + + return RunnerRegistrationTokenModel.findOne(query) + } + + static countTotal () { + return RunnerRegistrationTokenModel.unscoped().count() + } + + static listForApi (options: { + start: number + count: number + sort: string + }) { + const { start, count, sort } = options + + const query: FindOptions = { + attributes: { + include: [ + [ + literal('(SELECT COUNT(*) FROM "runner" WHERE "runner"."runnerRegistrationTokenId" = "RunnerRegistrationTokenModel"."id")'), + 'registeredRunnersCount' + ] + ] + }, + offset: start, + limit: count, + order: getSort(sort) + } + + return Promise.all([ + RunnerRegistrationTokenModel.count(query), + RunnerRegistrationTokenModel.findAll(query) + ]).then(([ total, data ]) => ({ total, data })) + } + + // --------------------------------------------------------------------------- + + toFormattedJSON (this: MRunnerRegistrationToken): RunnerRegistrationToken { + const registeredRunnersCount = this.get('registeredRunnersCount') as number + + return { + id: this.id, + + registrationToken: this.registrationToken, + + createdAt: this.createdAt, + updatedAt: this.updatedAt, + + registeredRunnersCount + } + } +} diff --git a/server/server/models/runner/runner.ts b/server/server/models/runner/runner.ts new file mode 100644 index 000000000..5c968b1c2 --- /dev/null +++ b/server/server/models/runner/runner.ts @@ -0,0 +1,124 @@ +import { FindOptions } from 'sequelize' +import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { MRunner } from '@server/types/models/runners/index.js' +import { Runner } from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { getSort } from '../shared/index.js' +import { RunnerRegistrationTokenModel } from './runner-registration-token.js' +import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js' + +@Table({ + tableName: 'runner', + indexes: [ + { + fields: [ 'runnerToken' ], + unique: true + }, + { + fields: [ 'runnerRegistrationTokenId' ] + }, + { + fields: [ 'name' ], + unique: true + } + ] +}) +export class RunnerModel extends Model>> { + + // Used to identify the appropriate runner when it uses the runner REST API + @AllowNull(false) + @Column + runnerToken: string + + @AllowNull(false) + @Column + name: string + + @AllowNull(true) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.RUNNERS.DESCRIPTION.max)) + description: string + + @AllowNull(false) + @Column + lastContact: Date + + @AllowNull(false) + @Column + ip: string + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => RunnerRegistrationTokenModel) + @Column + runnerRegistrationTokenId: number + + @BelongsTo(() => RunnerRegistrationTokenModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + RunnerRegistrationToken: Awaited + + // --------------------------------------------------------------------------- + + static load (id: number) { + return RunnerModel.findByPk(id) + } + + static loadByToken (runnerToken: string) { + const query = { + where: { runnerToken } + } + + return RunnerModel.findOne(query) + } + + static loadByName (name: string) { + const query = { + where: { name } + } + + return RunnerModel.findOne(query) + } + + static listForApi (options: { + start: number + count: number + sort: string + }) { + const { start, count, sort } = options + + const query: FindOptions = { + offset: start, + limit: count, + order: getSort(sort) + } + + return Promise.all([ + RunnerModel.count(query), + RunnerModel.findAll(query) + ]).then(([ total, data ]) => ({ total, data })) + } + + // --------------------------------------------------------------------------- + + toFormattedJSON (this: MRunner): Runner { + return { + id: this.id, + + name: this.name, + description: this.description, + + ip: this.ip, + lastContact: this.lastContact, + + createdAt: this.createdAt, + updatedAt: this.updatedAt + } + } +} diff --git a/server/server/models/server/plugin.ts b/server/server/models/server/plugin.ts new file mode 100644 index 000000000..ff8350b6a --- /dev/null +++ b/server/server/models/server/plugin.ts @@ -0,0 +1,317 @@ +import { + PeerTubePlugin, + PluginType, + RegisterServerSettingOptions, + SettingEntries, + SettingValue, + type PluginType_Type +} from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { MPlugin, MPluginFormattable } from '@server/types/models/index.js' +import { FindAndCountOptions, json, QueryTypes } from 'sequelize' +import { AllowNull, Column, CreatedAt, DataType, DefaultScope, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { + isPluginDescriptionValid, + isPluginHomepage, + isPluginNameValid, + isPluginStableOrUnstableVersionValid, + isPluginStableVersionValid, + isPluginTypeValid +} from '../../helpers/custom-validators/plugins.js' +import { getSort, throwIfNotValid } from '../shared/index.js' + +@DefaultScope(() => ({ + attributes: { + exclude: [ 'storage' ] + } +})) + +@Table({ + tableName: 'plugin', + indexes: [ + { + fields: [ 'name', 'type' ], + unique: true + } + ] +}) +export class PluginModel extends Model>> { + + @AllowNull(false) + @Is('PluginName', value => throwIfNotValid(value, isPluginNameValid, 'name')) + @Column + name: string + + @AllowNull(false) + @Is('PluginType', value => throwIfNotValid(value, isPluginTypeValid, 'type')) + @Column + type: PluginType_Type + + @AllowNull(false) + @Is('PluginVersion', value => throwIfNotValid(value, isPluginStableOrUnstableVersionValid, 'version')) + @Column + version: string + + @AllowNull(true) + @Is('PluginLatestVersion', value => throwIfNotValid(value, isPluginStableVersionValid, 'version')) + @Column + latestVersion: string + + @AllowNull(false) + @Column + enabled: boolean + + @AllowNull(false) + @Column + uninstalled: boolean + + @AllowNull(false) + @Column + peertubeEngine: string + + @AllowNull(true) + @Is('PluginDescription', value => throwIfNotValid(value, isPluginDescriptionValid, 'description')) + @Column + description: string + + @AllowNull(false) + @Is('PluginHomepage', value => throwIfNotValid(value, isPluginHomepage, 'homepage')) + @Column + homepage: string + + @AllowNull(true) + @Column(DataType.JSONB) + settings: any + + @AllowNull(true) + @Column(DataType.JSONB) + storage: any + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + static listEnabledPluginsAndThemes (): Promise { + const query = { + where: { + enabled: true, + uninstalled: false + } + } + + return PluginModel.findAll(query) + } + + static loadByNpmName (npmName: string): Promise { + const name = this.normalizePluginName(npmName) + const type = this.getTypeFromNpmName(npmName) + + const query = { + where: { + name, + type + } + } + + return PluginModel.findOne(query) + } + + static getSetting ( + pluginName: string, + pluginType: PluginType_Type, + settingName: string, + registeredSettings: RegisterServerSettingOptions[] + ) { + const query = { + attributes: [ 'settings' ], + where: { + name: pluginName, + type: pluginType + } + } + + return PluginModel.findOne(query) + .then(p => { + if (!p?.settings || p.settings === undefined) { + const registered = registeredSettings.find(s => s.name === settingName) + if (!registered || registered.default === undefined) return undefined + + return registered.default + } + + return p.settings[settingName] + }) + } + + static getSettings ( + pluginName: string, + pluginType: PluginType_Type, + settingNames: string[], + registeredSettings: RegisterServerSettingOptions[] + ) { + const query = { + attributes: [ 'settings' ], + where: { + name: pluginName, + type: pluginType + } + } + + return PluginModel.findOne(query) + .then(p => { + const result: SettingEntries = {} + + for (const name of settingNames) { + if (!p?.settings || p.settings[name] === undefined) { + const registered = registeredSettings.find(s => s.name === name) + + if (registered?.default !== undefined) { + result[name] = registered.default + } + } else { + result[name] = p.settings[name] + } + } + + return result + }) + } + + static setSetting (pluginName: string, pluginType: PluginType_Type, settingName: string, settingValue: SettingValue) { + const query = { + where: { + name: pluginName, + type: pluginType + } + } + + const toSave = { + [`settings.${settingName}`]: settingValue + } + + return PluginModel.update(toSave, query) + .then(() => undefined) + } + + static getData (pluginName: string, pluginType: PluginType_Type, key: string) { + const query = { + raw: true, + attributes: [ [ json('storage.' + key), 'value' ] as any ], // FIXME: typings + where: { + name: pluginName, + type: pluginType + } + } + + return PluginModel.findOne(query) + .then((c: any) => { + if (!c) return undefined + const value = c.value + + try { + return JSON.parse(value) + } catch { + return value + } + }) + } + + static storeData (pluginName: string, pluginType: PluginType_Type, key: string, data: any) { + const query = 'UPDATE "plugin" SET "storage" = jsonb_set(coalesce("storage", \'{}\'), :key, :data::jsonb) ' + + 'WHERE "name" = :pluginName AND "type" = :pluginType' + + const jsonPath = '{' + key + '}' + + const options = { + replacements: { pluginName, pluginType, key: jsonPath, data: JSON.stringify(data) }, + type: QueryTypes.UPDATE + } + + return PluginModel.sequelize.query(query, options) + .then(() => undefined) + } + + static listForApi (options: { + pluginType?: PluginType_Type + uninstalled?: boolean + start: number + count: number + sort: string + }) { + const { uninstalled = false } = options + const query: FindAndCountOptions = { + offset: options.start, + limit: options.count, + order: getSort(options.sort), + where: { + uninstalled + } + } + + if (options.pluginType) query.where['type'] = options.pluginType + + return Promise.all([ + PluginModel.count(query), + PluginModel.findAll(query) + ]).then(([ total, data ]) => ({ total, data })) + } + + static listInstalled (): Promise { + const query = { + where: { + uninstalled: false + } + } + + return PluginModel.findAll(query) + } + + static normalizePluginName (npmName: string) { + return npmName.replace(/^peertube-((theme)|(plugin))-/, '') + } + + static getTypeFromNpmName (npmName: string) { + return npmName.startsWith('peertube-plugin-') + ? PluginType.PLUGIN + : PluginType.THEME + } + + static buildNpmName (name: string, type: PluginType_Type) { + if (type === PluginType.THEME) return 'peertube-theme-' + name + + return 'peertube-plugin-' + name + } + + getPublicSettings (registeredSettings: RegisterServerSettingOptions[]) { + const result: SettingEntries = {} + const settings = this.settings || {} + + for (const r of registeredSettings) { + if (r.private !== false) continue + + result[r.name] = settings[r.name] ?? r.default ?? null + } + + return result + } + + toFormattedJSON (this: MPluginFormattable): PeerTubePlugin { + return { + name: this.name, + type: this.type, + version: this.version, + latestVersion: this.latestVersion, + enabled: this.enabled, + uninstalled: this.uninstalled, + peertubeEngine: this.peertubeEngine, + description: this.description, + homepage: this.homepage, + settings: this.settings, + createdAt: this.createdAt, + updatedAt: this.updatedAt + } + } + +} diff --git a/server/server/models/server/server-blocklist.ts b/server/server/models/server/server-blocklist.ts new file mode 100644 index 000000000..7817d122e --- /dev/null +++ b/server/server/models/server/server-blocklist.ts @@ -0,0 +1,190 @@ +import { Op, QueryTypes } from 'sequelize' +import { BelongsTo, Column, CreatedAt, ForeignKey, Model, Scopes, Table, UpdatedAt } from 'sequelize-typescript' +import { MServerBlocklist, MServerBlocklistAccountServer, MServerBlocklistFormattable } from '@server/types/models/index.js' +import { ServerBlock } from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { AccountModel } from '../account/account.js' +import { createSafeIn, getSort, searchAttribute } from '../shared/index.js' +import { ServerModel } from './server.js' + +enum ScopeNames { + WITH_ACCOUNT = 'WITH_ACCOUNT', + WITH_SERVER = 'WITH_SERVER' +} + +@Scopes(() => ({ + [ScopeNames.WITH_ACCOUNT]: { + include: [ + { + model: AccountModel, + required: true + } + ] + }, + [ScopeNames.WITH_SERVER]: { + include: [ + { + model: ServerModel, + required: true + } + ] + } +})) + +@Table({ + tableName: 'serverBlocklist', + indexes: [ + { + fields: [ 'accountId', 'targetServerId' ], + unique: true + }, + { + fields: [ 'targetServerId' ] + } + ] +}) +export class ServerBlocklistModel extends Model>> { + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => AccountModel) + @Column + accountId: number + + @BelongsTo(() => AccountModel, { + foreignKey: { + name: 'accountId', + allowNull: false + }, + onDelete: 'CASCADE' + }) + ByAccount: Awaited + + @ForeignKey(() => ServerModel) + @Column + targetServerId: number + + @BelongsTo(() => ServerModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + BlockedServer: Awaited + + static isServerMutedByAccounts (accountIds: number[], targetServerId: number) { + const query = { + attributes: [ 'accountId', 'id' ], + where: { + accountId: { + [Op.in]: accountIds + }, + targetServerId + }, + raw: true + } + + return ServerBlocklistModel.unscoped() + .findAll(query) + .then(rows => { + const result: { [accountId: number]: boolean } = {} + + for (const accountId of accountIds) { + result[accountId] = !!rows.find(r => r.accountId === accountId) + } + + return result + }) + } + + static loadByAccountAndHost (accountId: number, host: string): Promise { + const query = { + where: { + accountId + }, + include: [ + { + model: ServerModel, + where: { + host + }, + required: true + } + ] + } + + return ServerBlocklistModel.findOne(query) + } + + static listHostsBlockedBy (accountIds: number[]): Promise { + const query = { + attributes: [ ], + where: { + accountId: { + [Op.in]: accountIds + } + }, + include: [ + { + attributes: [ 'host' ], + model: ServerModel.unscoped(), + required: true + } + ] + } + + return ServerBlocklistModel.findAll(query) + .then(entries => entries.map(e => e.BlockedServer.host)) + } + + static getBlockStatus (byAccountIds: number[], hosts: string[]): Promise<{ host: string, accountId: number }[]> { + const rawQuery = `SELECT "server"."host", "serverBlocklist"."accountId" ` + + `FROM "serverBlocklist" ` + + `INNER JOIN "server" ON "server"."id" = "serverBlocklist"."targetServerId" ` + + `WHERE "server"."host" IN (:hosts) ` + + `AND "serverBlocklist"."accountId" IN (${createSafeIn(ServerBlocklistModel.sequelize, byAccountIds)})` + + return ServerBlocklistModel.sequelize.query(rawQuery, { + type: QueryTypes.SELECT as QueryTypes.SELECT, + replacements: { hosts } + }) + } + + static listForApi (parameters: { + start: number + count: number + sort: string + search?: string + accountId: number + }) { + const { start, count, sort, search, accountId } = parameters + + const query = { + offset: start, + limit: count, + order: getSort(sort), + where: { + accountId, + + ...searchAttribute(search, '$BlockedServer.host$') + } + } + + return Promise.all([ + ServerBlocklistModel.scope(ScopeNames.WITH_SERVER).count(query), + ServerBlocklistModel.scope([ ScopeNames.WITH_ACCOUNT, ScopeNames.WITH_SERVER ]).findAll(query) + ]).then(([ total, data ]) => ({ total, data })) + } + + toFormattedJSON (this: MServerBlocklistFormattable): ServerBlock { + return { + byAccount: this.ByAccount.toFormattedJSON(), + blockedServer: this.BlockedServer.toFormattedJSON(), + createdAt: this.createdAt + } + } +} diff --git a/server/server/models/server/server.ts b/server/server/models/server/server.ts new file mode 100644 index 000000000..a50cc25d8 --- /dev/null +++ b/server/server/models/server/server.ts @@ -0,0 +1,104 @@ +import { Transaction } from 'sequelize' +import { AllowNull, Column, CreatedAt, Default, HasMany, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { MServer, MServerFormattable } from '@server/types/models/server/index.js' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { isHostValid } from '../../helpers/custom-validators/servers.js' +import { ActorModel } from '../actor/actor.js' +import { buildSQLAttributes, throwIfNotValid } from '../shared/index.js' +import { ServerBlocklistModel } from './server-blocklist.js' + +@Table({ + tableName: 'server', + indexes: [ + { + fields: [ 'host' ], + unique: true + } + ] +}) +export class ServerModel extends Model>> { + + @AllowNull(false) + @Is('Host', value => throwIfNotValid(value, isHostValid, 'valid host')) + @Column + host: string + + @AllowNull(false) + @Default(false) + @Column + redundancyAllowed: boolean + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @HasMany(() => ActorModel, { + foreignKey: { + name: 'serverId', + allowNull: true + }, + onDelete: 'CASCADE', + hooks: true + }) + Actors: Awaited[] + + @HasMany(() => ServerBlocklistModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + BlockedBy: Awaited[] + + // --------------------------------------------------------------------------- + + static getSQLAttributes (tableName: string, aliasPrefix = '') { + return buildSQLAttributes({ + model: this, + tableName, + aliasPrefix + }) + } + + // --------------------------------------------------------------------------- + + static load (id: number, transaction?: Transaction): Promise { + const query = { + where: { + id + }, + transaction + } + + return ServerModel.findOne(query) + } + + static loadByHost (host: string): Promise { + const query = { + where: { + host + } + } + + return ServerModel.findOne(query) + } + + static async loadOrCreateByHost (host: string) { + let server = await ServerModel.loadByHost(host) + if (!server) server = await ServerModel.create({ host }) + + return server + } + + isBlocked () { + return this.BlockedBy && this.BlockedBy.length !== 0 + } + + toFormattedJSON (this: MServerFormattable) { + return { + host: this.host + } + } +} diff --git a/server/server/models/server/tracker.ts b/server/server/models/server/tracker.ts new file mode 100644 index 000000000..46955768d --- /dev/null +++ b/server/server/models/server/tracker.ts @@ -0,0 +1,74 @@ +import { AllowNull, BelongsToMany, Column, CreatedAt, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { Transaction } from 'sequelize' +import { MTracker } from '@server/types/models/server/tracker.js' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { VideoModel } from '../video/video.js' +import { VideoTrackerModel } from './video-tracker.js' + +@Table({ + tableName: 'tracker', + indexes: [ + { + fields: [ 'url' ], + unique: true + } + ] +}) +export class TrackerModel extends Model>> { + + @AllowNull(false) + @Column + url: string + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @BelongsToMany(() => VideoModel, { + foreignKey: 'trackerId', + through: () => VideoTrackerModel, + onDelete: 'CASCADE' + }) + Videos: Awaited[] + + static listUrlsByVideoId (videoId: number) { + const query = { + include: [ + { + attributes: [ 'id' ], + model: VideoModel.unscoped(), + required: true, + where: { id: videoId } + } + ] + } + + return TrackerModel.findAll(query) + .then(rows => rows.map(rows => rows.url)) + } + + static findOrCreateTrackers (trackers: string[], transaction: Transaction): Promise { + if (trackers === null) return Promise.resolve([]) + + const tasks: Promise[] = [] + trackers.forEach(tracker => { + const query = { + where: { + url: tracker + }, + defaults: { + url: tracker + }, + transaction + } + + const promise = TrackerModel.findOrCreate(query) + .then(([ trackerInstance ]) => trackerInstance) + tasks.push(promise) + }) + + return Promise.all(tasks) + } +} diff --git a/server/server/models/server/video-tracker.ts b/server/server/models/server/video-tracker.ts new file mode 100644 index 000000000..a925dc5c1 --- /dev/null +++ b/server/server/models/server/video-tracker.ts @@ -0,0 +1,31 @@ +import { Column, CreatedAt, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { VideoModel } from '../video/video.js' +import { TrackerModel } from './tracker.js' + +@Table({ + tableName: 'videoTracker', + indexes: [ + { + fields: [ 'videoId' ] + }, + { + fields: [ 'trackerId' ] + } + ] +}) +export class VideoTrackerModel extends Model>> { + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @ForeignKey(() => TrackerModel) + @Column + trackerId: number +} diff --git a/server/models/shared/abstract-run-query.ts b/server/server/models/shared/abstract-run-query.ts similarity index 100% rename from server/models/shared/abstract-run-query.ts rename to server/server/models/shared/abstract-run-query.ts diff --git a/server/server/models/shared/index.ts b/server/server/models/shared/index.ts new file mode 100644 index 000000000..53cf65d5f --- /dev/null +++ b/server/server/models/shared/index.ts @@ -0,0 +1,8 @@ +export * from './abstract-run-query.js' +export * from './model-builder.js' +export * from './model-cache.js' +export * from './query.js' +export * from './sequelize-helpers.js' +export * from './sort.js' +export * from './sql.js' +export * from './update.js' diff --git a/server/server/models/shared/model-builder.ts b/server/server/models/shared/model-builder.ts new file mode 100644 index 000000000..c19ce2d56 --- /dev/null +++ b/server/server/models/shared/model-builder.ts @@ -0,0 +1,118 @@ +import isPlainObject from 'lodash-es/isPlainObject.js' +import { Model as SequelizeModel, ModelStatic, Sequelize } from 'sequelize' +import { logger } from '@server/helpers/logger.js' + +/** + * + * Build Sequelize models from sequelize raw query (that must use { nest: true } options) + * + * In order to sequelize to correctly build the JSON this class will ingest, + * the columns selected in the raw query should be in the following form: + * * All tables must be Pascal Cased (for example "VideoChannel") + * * Root table must end with `Model` (for example "VideoCommentModel") + * * Joined tables must contain the origin table name + '->JoinedTable'. For example: + * * "Actor" is joined to "Account": "Actor" table must be renamed "Account->Actor" + * * "Account->Actor" is joined to "Server": "Server" table must be renamed to "Account->Actor->Server" + * * Selected columns must be renamed to contain the JSON path: + * * "videoComment"."id": "VideoCommentModel"."id" + * * "Account"."Actor"."Server"."id": "Account.Actor.Server.id" + * * All tables must contain the row id + */ + +export class ModelBuilder { + private readonly modelRegistry = new Map() + + constructor (private readonly sequelize: Sequelize) { + + } + + createModels (jsonArray: any[], baseModelName: string): T[] { + const result: T[] = [] + + for (const json of jsonArray) { + const { created, model } = this.createModel(json, baseModelName, json.id + '.' + baseModelName) + + if (created) result.push(model) + } + + return result + } + + private createModel (json: any, modelName: string, keyPath: string) { + if (!json.id) return { created: false, model: null } + + const { created, model } = this.createOrFindModel(json, modelName, keyPath) + + for (const key of Object.keys(json)) { + const value = json[key] + if (!value) continue + + // Child model + if (isPlainObject(value)) { + const { created, model: subModel } = this.createModel(value, key, keyPath + '.' + json.id + '.' + key) + if (!created || !subModel) continue + + const Model = this.findModelBuilder(modelName) + const association = Model.associations[key] + + if (!association) { + logger.error('Cannot find association %s of model %s', key, modelName, { associations: Object.keys(Model.associations) }) + continue + } + + if (association.isMultiAssociation) { + if (!Array.isArray(model[key])) model[key] = [] + + model[key].push(subModel) + } else { + model[key] = subModel + } + } + } + + return { created, model } + } + + private createOrFindModel (json: any, modelName: string, keyPath: string) { + const registryKey = this.getModelRegistryKey(json, keyPath) + if (this.modelRegistry.has(registryKey)) { + return { + created: false, + model: this.modelRegistry.get(registryKey) + } + } + + const Model = this.findModelBuilder(modelName) + + if (!Model) { + logger.error( + 'Cannot build model %s that does not exist', this.buildSequelizeModelName(modelName), + { existing: this.sequelize.modelManager.all.map(m => m.name) } + ) + return { created: false, model: null } + } + + const model = Model.build(json, { raw: true, isNewRecord: false }) + + this.modelRegistry.set(registryKey, model) + + return { created: true, model } + } + + private findModelBuilder (modelName: string) { + return this.sequelize.modelManager.getModel(this.buildSequelizeModelName(modelName)) as ModelStatic + } + + private buildSequelizeModelName (modelName: string) { + if (modelName === 'Avatars') return 'ActorImageModel' + if (modelName === 'ActorFollowing') return 'ActorModel' + if (modelName === 'ActorFollower') return 'ActorModel' + if (modelName === 'FlaggedAccount') return 'AccountModel' + + return modelName + 'Model' + } + + private getModelRegistryKey (json: any, keyPath: string) { + return keyPath + json.id + } +} diff --git a/server/server/models/shared/model-cache.ts b/server/server/models/shared/model-cache.ts new file mode 100644 index 000000000..273d8e608 --- /dev/null +++ b/server/server/models/shared/model-cache.ts @@ -0,0 +1,90 @@ +import { Model } from 'sequelize-typescript' +import { logger } from '@server/helpers/logger.js' + +type ModelCacheType = + 'local-account-name' + | 'local-actor-name' + | 'local-actor-url' + | 'load-video-immutable-id' + | 'load-video-immutable-url' + +type DeleteKey = + 'video' + +class ModelCache { + + private static instance: ModelCache + + private readonly localCache: { [id in ModelCacheType]: Map } = { + 'local-account-name': new Map(), + 'local-actor-name': new Map(), + 'local-actor-url': new Map(), + 'load-video-immutable-id': new Map(), + 'load-video-immutable-url': new Map() + } + + private readonly deleteIds: { + [deleteKey in DeleteKey]: Map + } = { + video: new Map() + } + + private constructor () { + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } + + doCache (options: { + cacheType: ModelCacheType + key: string + fun: () => Promise + whitelist?: () => boolean + deleteKey?: DeleteKey + }) { + const { cacheType, key, fun, whitelist, deleteKey } = options + + if (whitelist && whitelist() !== true) return fun() + + const cache = this.localCache[cacheType] + + if (cache.has(key)) { + logger.debug('Model cache hit for %s -> %s.', cacheType, key) + return Promise.resolve(cache.get(key)) + } + + return fun().then(m => { + if (!m) return m + + if (!whitelist || whitelist()) cache.set(key, m) + + if (deleteKey) { + const map = this.deleteIds[deleteKey] + if (!map.has(m.id)) map.set(m.id, []) + + const a = map.get(m.id) + a.push({ cacheType, key }) + } + + return m + }) + } + + invalidateCache (deleteKey: DeleteKey, modelId: number) { + const map = this.deleteIds[deleteKey] + + if (!map.has(modelId)) return + + for (const toDelete of map.get(modelId)) { + logger.debug('Removing %s -> %d of model cache %s -> %s.', deleteKey, modelId, toDelete.cacheType, toDelete.key) + this.localCache[toDelete.cacheType].delete(toDelete.key) + } + + map.delete(modelId) + } +} + +export { + ModelCache +} diff --git a/server/server/models/shared/query.ts b/server/server/models/shared/query.ts new file mode 100644 index 000000000..0158454e8 --- /dev/null +++ b/server/server/models/shared/query.ts @@ -0,0 +1,84 @@ +import { BindOrReplacements, Op, QueryTypes, Sequelize } from 'sequelize' +import { Fn } from 'sequelize/types/utils' +import validator from 'validator' +import { forceNumber } from '@peertube/peertube-core-utils' + +function doesExist (sequelize: Sequelize, query: string, bind?: BindOrReplacements) { + const options = { + type: QueryTypes.SELECT as QueryTypes.SELECT, + bind, + raw: true + } + + return sequelize.query(query, options) + .then(results => results.length === 1) +} + +// FIXME: have to specify the result type to not break peertube typings generation +function createSimilarityAttribute (col: string, value: string): Fn { + return Sequelize.fn( + 'similarity', + + searchTrigramNormalizeCol(col), + + searchTrigramNormalizeValue(value) + ) +} + +function buildWhereIdOrUUID (id: number | string) { + return validator.default.isInt('' + id) ? { id } : { uuid: id } +} + +function parseAggregateResult (result: any) { + if (!result) return 0 + + const total = forceNumber(result) + if (isNaN(total)) return 0 + + return total +} + +function parseRowCountResult (result: any) { + if (result.length !== 0) return result[0].total + + return 0 +} + +function createSafeIn (sequelize: Sequelize, toEscape: (string | number)[], additionalUnescaped: string[] = []) { + return toEscape.map(t => { + return t === null + ? null + : sequelize.escape('' + t) + }).concat(additionalUnescaped).join(', ') +} + +function searchAttribute (sourceField?: string, targetField?: string) { + if (!sourceField) return {} + + return { + [targetField]: { + // FIXME: ts error + [Op.iLike as any]: `%${sourceField}%` + } + } +} + +export { + doesExist, + createSimilarityAttribute, + buildWhereIdOrUUID, + parseAggregateResult, + parseRowCountResult, + createSafeIn, + searchAttribute +} + +// --------------------------------------------------------------------------- + +function searchTrigramNormalizeValue (value: string) { + return Sequelize.fn('lower', Sequelize.fn('immutable_unaccent', value)) +} + +function searchTrigramNormalizeCol (col: string) { + return Sequelize.fn('lower', Sequelize.fn('immutable_unaccent', Sequelize.col(col))) +} diff --git a/server/models/shared/sequelize-helpers.ts b/server/server/models/shared/sequelize-helpers.ts similarity index 100% rename from server/models/shared/sequelize-helpers.ts rename to server/server/models/shared/sequelize-helpers.ts diff --git a/server/models/shared/sort.ts b/server/server/models/shared/sort.ts similarity index 100% rename from server/models/shared/sort.ts rename to server/server/models/shared/sort.ts diff --git a/server/server/models/shared/sql.ts b/server/server/models/shared/sql.ts new file mode 100644 index 000000000..3f22dd4e3 --- /dev/null +++ b/server/server/models/shared/sql.ts @@ -0,0 +1,71 @@ +import { literal, Model, ModelStatic } from 'sequelize' +import { Literal } from 'sequelize/types/utils' +import { forceNumber } from '@peertube/peertube-core-utils' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' + +// FIXME: have to specify the result type to not break peertube typings generation +function buildLocalAccountIdsIn (): Literal { + return literal( + '(SELECT "account"."id" FROM "account" INNER JOIN "actor" ON "actor"."id" = "account"."actorId" AND "actor"."serverId" IS NULL)' + ) +} + +// FIXME: have to specify the result type to not break peertube typings generation +function buildLocalActorIdsIn (): Literal { + return literal( + '(SELECT "actor"."id" FROM "actor" WHERE "actor"."serverId" IS NULL)' + ) +} + +function buildBlockedAccountSQL (blockerIds: number[]) { + const blockerIdsString = blockerIds.join(', ') + + return 'SELECT "targetAccountId" AS "id" FROM "accountBlocklist" WHERE "accountId" IN (' + blockerIdsString + ')' + + ' UNION ' + + 'SELECT "account"."id" AS "id" FROM account INNER JOIN "actor" ON account."actorId" = actor.id ' + + 'INNER JOIN "serverBlocklist" ON "actor"."serverId" = "serverBlocklist"."targetServerId" ' + + 'WHERE "serverBlocklist"."accountId" IN (' + blockerIdsString + ')' +} + +function buildServerIdsFollowedBy (actorId: any) { + const actorIdNumber = forceNumber(actorId) + + return '(' + + 'SELECT "actor"."serverId" FROM "actorFollow" ' + + 'INNER JOIN "actor" ON actor.id = "actorFollow"."targetActorId" ' + + 'WHERE "actorFollow"."actorId" = ' + actorIdNumber + + ')' +} + +function buildSQLAttributes (options: { + model: ModelStatic + tableName: string + + excludeAttributes?: Exclude, symbol>[] + aliasPrefix?: string +}) { + const { model, tableName, aliasPrefix, excludeAttributes } = options + + const attributes = Object.keys(model.getAttributes()) as Exclude, symbol>[] + + return attributes + .filter(a => { + if (!excludeAttributes) return true + if (excludeAttributes.includes(a)) return false + + return true + }) + .map(a => { + return `"${tableName}"."${a}" AS "${aliasPrefix || ''}${a}"` + }) +} + +// --------------------------------------------------------------------------- + +export { + buildSQLAttributes, + buildBlockedAccountSQL, + buildServerIdsFollowedBy, + buildLocalAccountIdsIn, + buildLocalActorIdsIn +} diff --git a/server/models/shared/update.ts b/server/server/models/shared/update.ts similarity index 100% rename from server/models/shared/update.ts rename to server/server/models/shared/update.ts diff --git a/server/server/models/user/sql/user-notitication-list-query-builder.ts b/server/server/models/user/sql/user-notitication-list-query-builder.ts new file mode 100644 index 000000000..ba3a2d70f --- /dev/null +++ b/server/server/models/user/sql/user-notitication-list-query-builder.ts @@ -0,0 +1,273 @@ +import { Sequelize } from 'sequelize' +import { AbstractRunQuery, ModelBuilder } from '@server/models/shared/index.js' +import { UserNotificationModelForApi } from '@server/types/models/index.js' +import { ActorImageType } from '@peertube/peertube-models' +import { getSort } from '../../shared/index.js' + +export interface ListNotificationsOptions { + userId: number + unread?: boolean + sort: string + offset: number + limit: number +} + +export class UserNotificationListQueryBuilder extends AbstractRunQuery { + private innerQuery: string + + constructor ( + protected readonly sequelize: Sequelize, + private readonly options: ListNotificationsOptions + ) { + super(sequelize) + } + + async listNotifications () { + this.buildQuery() + + const results = await this.runQuery({ nest: true }) + const modelBuilder = new ModelBuilder(this.sequelize) + + return modelBuilder.createModels(results, 'UserNotification') + } + + private buildInnerQuery () { + this.innerQuery = `SELECT * FROM "userNotification" AS "UserNotificationModel" ` + + `${this.getWhere()} ` + + `${this.getOrder()} ` + + `LIMIT :limit OFFSET :offset ` + + this.replacements.limit = this.options.limit + this.replacements.offset = this.options.offset + } + + private buildQuery () { + this.buildInnerQuery() + + this.query = ` + ${this.getSelect()} + FROM (${this.innerQuery}) "UserNotificationModel" + ${this.getJoins()} + ${this.getOrder()}` + } + + private getWhere () { + let base = '"UserNotificationModel"."userId" = :userId ' + this.replacements.userId = this.options.userId + + if (this.options.unread === true) { + base += 'AND "UserNotificationModel"."read" IS FALSE ' + } else if (this.options.unread === false) { + base += 'AND "UserNotificationModel"."read" IS TRUE ' + } + + return `WHERE ${base}` + } + + private getOrder () { + const orders = getSort(this.options.sort) + + return 'ORDER BY ' + orders.map(o => `"UserNotificationModel"."${o[0]}" ${o[1]}`).join(', ') + } + + private getSelect () { + return `SELECT + "UserNotificationModel"."id", + "UserNotificationModel"."type", + "UserNotificationModel"."read", + "UserNotificationModel"."createdAt", + "UserNotificationModel"."updatedAt", + "Video"."id" AS "Video.id", + "Video"."uuid" AS "Video.uuid", + "Video"."name" AS "Video.name", + "Video->VideoChannel"."id" AS "Video.VideoChannel.id", + "Video->VideoChannel"."name" AS "Video.VideoChannel.name", + "Video->VideoChannel->Actor"."id" AS "Video.VideoChannel.Actor.id", + "Video->VideoChannel->Actor"."preferredUsername" AS "Video.VideoChannel.Actor.preferredUsername", + "Video->VideoChannel->Actor->Avatars"."id" AS "Video.VideoChannel.Actor.Avatars.id", + "Video->VideoChannel->Actor->Avatars"."width" AS "Video.VideoChannel.Actor.Avatars.width", + "Video->VideoChannel->Actor->Avatars"."type" AS "Video.VideoChannel.Actor.Avatars.type", + "Video->VideoChannel->Actor->Avatars"."filename" AS "Video.VideoChannel.Actor.Avatars.filename", + "Video->VideoChannel->Actor->Server"."id" AS "Video.VideoChannel.Actor.Server.id", + "Video->VideoChannel->Actor->Server"."host" AS "Video.VideoChannel.Actor.Server.host", + "VideoComment"."id" AS "VideoComment.id", + "VideoComment"."originCommentId" AS "VideoComment.originCommentId", + "VideoComment->Account"."id" AS "VideoComment.Account.id", + "VideoComment->Account"."name" AS "VideoComment.Account.name", + "VideoComment->Account->Actor"."id" AS "VideoComment.Account.Actor.id", + "VideoComment->Account->Actor"."preferredUsername" AS "VideoComment.Account.Actor.preferredUsername", + "VideoComment->Account->Actor->Avatars"."id" AS "VideoComment.Account.Actor.Avatars.id", + "VideoComment->Account->Actor->Avatars"."width" AS "VideoComment.Account.Actor.Avatars.width", + "VideoComment->Account->Actor->Avatars"."type" AS "VideoComment.Account.Actor.Avatars.type", + "VideoComment->Account->Actor->Avatars"."filename" AS "VideoComment.Account.Actor.Avatars.filename", + "VideoComment->Account->Actor->Server"."id" AS "VideoComment.Account.Actor.Server.id", + "VideoComment->Account->Actor->Server"."host" AS "VideoComment.Account.Actor.Server.host", + "VideoComment->Video"."id" AS "VideoComment.Video.id", + "VideoComment->Video"."uuid" AS "VideoComment.Video.uuid", + "VideoComment->Video"."name" AS "VideoComment.Video.name", + "Abuse"."id" AS "Abuse.id", + "Abuse"."state" AS "Abuse.state", + "Abuse->VideoAbuse"."id" AS "Abuse.VideoAbuse.id", + "Abuse->VideoAbuse->Video"."id" AS "Abuse.VideoAbuse.Video.id", + "Abuse->VideoAbuse->Video"."uuid" AS "Abuse.VideoAbuse.Video.uuid", + "Abuse->VideoAbuse->Video"."name" AS "Abuse.VideoAbuse.Video.name", + "Abuse->VideoCommentAbuse"."id" AS "Abuse.VideoCommentAbuse.id", + "Abuse->VideoCommentAbuse->VideoComment"."id" AS "Abuse.VideoCommentAbuse.VideoComment.id", + "Abuse->VideoCommentAbuse->VideoComment"."originCommentId" AS "Abuse.VideoCommentAbuse.VideoComment.originCommentId", + "Abuse->VideoCommentAbuse->VideoComment->Video"."id" AS "Abuse.VideoCommentAbuse.VideoComment.Video.id", + "Abuse->VideoCommentAbuse->VideoComment->Video"."name" AS "Abuse.VideoCommentAbuse.VideoComment.Video.name", + "Abuse->VideoCommentAbuse->VideoComment->Video"."uuid" AS "Abuse.VideoCommentAbuse.VideoComment.Video.uuid", + "Abuse->FlaggedAccount"."id" AS "Abuse.FlaggedAccount.id", + "Abuse->FlaggedAccount"."name" AS "Abuse.FlaggedAccount.name", + "Abuse->FlaggedAccount"."description" AS "Abuse.FlaggedAccount.description", + "Abuse->FlaggedAccount"."actorId" AS "Abuse.FlaggedAccount.actorId", + "Abuse->FlaggedAccount"."userId" AS "Abuse.FlaggedAccount.userId", + "Abuse->FlaggedAccount"."applicationId" AS "Abuse.FlaggedAccount.applicationId", + "Abuse->FlaggedAccount"."createdAt" AS "Abuse.FlaggedAccount.createdAt", + "Abuse->FlaggedAccount"."updatedAt" AS "Abuse.FlaggedAccount.updatedAt", + "Abuse->FlaggedAccount->Actor"."id" AS "Abuse.FlaggedAccount.Actor.id", + "Abuse->FlaggedAccount->Actor"."preferredUsername" AS "Abuse.FlaggedAccount.Actor.preferredUsername", + "Abuse->FlaggedAccount->Actor->Avatars"."id" AS "Abuse.FlaggedAccount.Actor.Avatars.id", + "Abuse->FlaggedAccount->Actor->Avatars"."width" AS "Abuse.FlaggedAccount.Actor.Avatars.width", + "Abuse->FlaggedAccount->Actor->Avatars"."type" AS "Abuse.FlaggedAccount.Actor.Avatars.type", + "Abuse->FlaggedAccount->Actor->Avatars"."filename" AS "Abuse.FlaggedAccount.Actor.Avatars.filename", + "Abuse->FlaggedAccount->Actor->Server"."id" AS "Abuse.FlaggedAccount.Actor.Server.id", + "Abuse->FlaggedAccount->Actor->Server"."host" AS "Abuse.FlaggedAccount.Actor.Server.host", + "VideoBlacklist"."id" AS "VideoBlacklist.id", + "VideoBlacklist->Video"."id" AS "VideoBlacklist.Video.id", + "VideoBlacklist->Video"."uuid" AS "VideoBlacklist.Video.uuid", + "VideoBlacklist->Video"."name" AS "VideoBlacklist.Video.name", + "VideoImport"."id" AS "VideoImport.id", + "VideoImport"."magnetUri" AS "VideoImport.magnetUri", + "VideoImport"."targetUrl" AS "VideoImport.targetUrl", + "VideoImport"."torrentName" AS "VideoImport.torrentName", + "VideoImport->Video"."id" AS "VideoImport.Video.id", + "VideoImport->Video"."uuid" AS "VideoImport.Video.uuid", + "VideoImport->Video"."name" AS "VideoImport.Video.name", + "Plugin"."id" AS "Plugin.id", + "Plugin"."name" AS "Plugin.name", + "Plugin"."type" AS "Plugin.type", + "Plugin"."latestVersion" AS "Plugin.latestVersion", + "Application"."id" AS "Application.id", + "Application"."latestPeerTubeVersion" AS "Application.latestPeerTubeVersion", + "ActorFollow"."id" AS "ActorFollow.id", + "ActorFollow"."state" AS "ActorFollow.state", + "ActorFollow->ActorFollower"."id" AS "ActorFollow.ActorFollower.id", + "ActorFollow->ActorFollower"."preferredUsername" AS "ActorFollow.ActorFollower.preferredUsername", + "ActorFollow->ActorFollower->Account"."id" AS "ActorFollow.ActorFollower.Account.id", + "ActorFollow->ActorFollower->Account"."name" AS "ActorFollow.ActorFollower.Account.name", + "ActorFollow->ActorFollower->Avatars"."id" AS "ActorFollow.ActorFollower.Avatars.id", + "ActorFollow->ActorFollower->Avatars"."width" AS "ActorFollow.ActorFollower.Avatars.width", + "ActorFollow->ActorFollower->Avatars"."type" AS "ActorFollow.ActorFollower.Avatars.type", + "ActorFollow->ActorFollower->Avatars"."filename" AS "ActorFollow.ActorFollower.Avatars.filename", + "ActorFollow->ActorFollower->Server"."id" AS "ActorFollow.ActorFollower.Server.id", + "ActorFollow->ActorFollower->Server"."host" AS "ActorFollow.ActorFollower.Server.host", + "ActorFollow->ActorFollowing"."id" AS "ActorFollow.ActorFollowing.id", + "ActorFollow->ActorFollowing"."preferredUsername" AS "ActorFollow.ActorFollowing.preferredUsername", + "ActorFollow->ActorFollowing"."type" AS "ActorFollow.ActorFollowing.type", + "ActorFollow->ActorFollowing->VideoChannel"."id" AS "ActorFollow.ActorFollowing.VideoChannel.id", + "ActorFollow->ActorFollowing->VideoChannel"."name" AS "ActorFollow.ActorFollowing.VideoChannel.name", + "ActorFollow->ActorFollowing->Account"."id" AS "ActorFollow.ActorFollowing.Account.id", + "ActorFollow->ActorFollowing->Account"."name" AS "ActorFollow.ActorFollowing.Account.name", + "ActorFollow->ActorFollowing->Server"."id" AS "ActorFollow.ActorFollowing.Server.id", + "ActorFollow->ActorFollowing->Server"."host" AS "ActorFollow.ActorFollowing.Server.host", + "Account"."id" AS "Account.id", + "Account"."name" AS "Account.name", + "Account->Actor"."id" AS "Account.Actor.id", + "Account->Actor"."preferredUsername" AS "Account.Actor.preferredUsername", + "Account->Actor->Avatars"."id" AS "Account.Actor.Avatars.id", + "Account->Actor->Avatars"."width" AS "Account.Actor.Avatars.width", + "Account->Actor->Avatars"."type" AS "Account.Actor.Avatars.type", + "Account->Actor->Avatars"."filename" AS "Account.Actor.Avatars.filename", + "Account->Actor->Server"."id" AS "Account.Actor.Server.id", + "Account->Actor->Server"."host" AS "Account.Actor.Server.host", + "UserRegistration"."id" AS "UserRegistration.id", + "UserRegistration"."username" AS "UserRegistration.username"` + } + + private getJoins () { + return ` + LEFT JOIN ( + "video" AS "Video" + INNER JOIN "videoChannel" AS "Video->VideoChannel" ON "Video"."channelId" = "Video->VideoChannel"."id" + INNER JOIN "actor" AS "Video->VideoChannel->Actor" ON "Video->VideoChannel"."actorId" = "Video->VideoChannel->Actor"."id" + LEFT JOIN "actorImage" AS "Video->VideoChannel->Actor->Avatars" + ON "Video->VideoChannel->Actor"."id" = "Video->VideoChannel->Actor->Avatars"."actorId" + AND "Video->VideoChannel->Actor->Avatars"."type" = ${ActorImageType.AVATAR} + LEFT JOIN "server" AS "Video->VideoChannel->Actor->Server" + ON "Video->VideoChannel->Actor"."serverId" = "Video->VideoChannel->Actor->Server"."id" + ) ON "UserNotificationModel"."videoId" = "Video"."id" + + LEFT JOIN ( + "videoComment" AS "VideoComment" + INNER JOIN "account" AS "VideoComment->Account" ON "VideoComment"."accountId" = "VideoComment->Account"."id" + INNER JOIN "actor" AS "VideoComment->Account->Actor" ON "VideoComment->Account"."actorId" = "VideoComment->Account->Actor"."id" + LEFT JOIN "actorImage" AS "VideoComment->Account->Actor->Avatars" + ON "VideoComment->Account->Actor"."id" = "VideoComment->Account->Actor->Avatars"."actorId" + AND "VideoComment->Account->Actor->Avatars"."type" = ${ActorImageType.AVATAR} + LEFT JOIN "server" AS "VideoComment->Account->Actor->Server" + ON "VideoComment->Account->Actor"."serverId" = "VideoComment->Account->Actor->Server"."id" + INNER JOIN "video" AS "VideoComment->Video" ON "VideoComment"."videoId" = "VideoComment->Video"."id" + ) ON "UserNotificationModel"."commentId" = "VideoComment"."id" + + LEFT JOIN "abuse" AS "Abuse" ON "UserNotificationModel"."abuseId" = "Abuse"."id" + LEFT JOIN "videoAbuse" AS "Abuse->VideoAbuse" ON "Abuse"."id" = "Abuse->VideoAbuse"."abuseId" + LEFT JOIN "video" AS "Abuse->VideoAbuse->Video" ON "Abuse->VideoAbuse"."videoId" = "Abuse->VideoAbuse->Video"."id" + LEFT JOIN "commentAbuse" AS "Abuse->VideoCommentAbuse" ON "Abuse"."id" = "Abuse->VideoCommentAbuse"."abuseId" + LEFT JOIN "videoComment" AS "Abuse->VideoCommentAbuse->VideoComment" + ON "Abuse->VideoCommentAbuse"."videoCommentId" = "Abuse->VideoCommentAbuse->VideoComment"."id" + LEFT JOIN "video" AS "Abuse->VideoCommentAbuse->VideoComment->Video" + ON "Abuse->VideoCommentAbuse->VideoComment"."videoId" = "Abuse->VideoCommentAbuse->VideoComment->Video"."id" + LEFT JOIN ( + "account" AS "Abuse->FlaggedAccount" + INNER JOIN "actor" AS "Abuse->FlaggedAccount->Actor" ON "Abuse->FlaggedAccount"."actorId" = "Abuse->FlaggedAccount->Actor"."id" + LEFT JOIN "actorImage" AS "Abuse->FlaggedAccount->Actor->Avatars" + ON "Abuse->FlaggedAccount->Actor"."id" = "Abuse->FlaggedAccount->Actor->Avatars"."actorId" + AND "Abuse->FlaggedAccount->Actor->Avatars"."type" = ${ActorImageType.AVATAR} + LEFT JOIN "server" AS "Abuse->FlaggedAccount->Actor->Server" + ON "Abuse->FlaggedAccount->Actor"."serverId" = "Abuse->FlaggedAccount->Actor->Server"."id" + ) ON "Abuse"."flaggedAccountId" = "Abuse->FlaggedAccount"."id" + + LEFT JOIN ( + "videoBlacklist" AS "VideoBlacklist" + INNER JOIN "video" AS "VideoBlacklist->Video" ON "VideoBlacklist"."videoId" = "VideoBlacklist->Video"."id" + ) ON "UserNotificationModel"."videoBlacklistId" = "VideoBlacklist"."id" + + LEFT JOIN "videoImport" AS "VideoImport" ON "UserNotificationModel"."videoImportId" = "VideoImport"."id" + LEFT JOIN "video" AS "VideoImport->Video" ON "VideoImport"."videoId" = "VideoImport->Video"."id" + + LEFT JOIN "plugin" AS "Plugin" ON "UserNotificationModel"."pluginId" = "Plugin"."id" + + LEFT JOIN "application" AS "Application" ON "UserNotificationModel"."applicationId" = "Application"."id" + + LEFT JOIN ( + "actorFollow" AS "ActorFollow" + INNER JOIN "actor" AS "ActorFollow->ActorFollower" ON "ActorFollow"."actorId" = "ActorFollow->ActorFollower"."id" + INNER JOIN "account" AS "ActorFollow->ActorFollower->Account" + ON "ActorFollow->ActorFollower"."id" = "ActorFollow->ActorFollower->Account"."actorId" + LEFT JOIN "actorImage" AS "ActorFollow->ActorFollower->Avatars" + ON "ActorFollow->ActorFollower"."id" = "ActorFollow->ActorFollower->Avatars"."actorId" + AND "ActorFollow->ActorFollower->Avatars"."type" = ${ActorImageType.AVATAR} + LEFT JOIN "server" AS "ActorFollow->ActorFollower->Server" + ON "ActorFollow->ActorFollower"."serverId" = "ActorFollow->ActorFollower->Server"."id" + INNER JOIN "actor" AS "ActorFollow->ActorFollowing" ON "ActorFollow"."targetActorId" = "ActorFollow->ActorFollowing"."id" + LEFT JOIN "videoChannel" AS "ActorFollow->ActorFollowing->VideoChannel" + ON "ActorFollow->ActorFollowing"."id" = "ActorFollow->ActorFollowing->VideoChannel"."actorId" + LEFT JOIN "account" AS "ActorFollow->ActorFollowing->Account" + ON "ActorFollow->ActorFollowing"."id" = "ActorFollow->ActorFollowing->Account"."actorId" + LEFT JOIN "server" AS "ActorFollow->ActorFollowing->Server" + ON "ActorFollow->ActorFollowing"."serverId" = "ActorFollow->ActorFollowing->Server"."id" + ) ON "UserNotificationModel"."actorFollowId" = "ActorFollow"."id" + + LEFT JOIN ( + "account" AS "Account" + INNER JOIN "actor" AS "Account->Actor" ON "Account"."actorId" = "Account->Actor"."id" + LEFT JOIN "actorImage" AS "Account->Actor->Avatars" + ON "Account->Actor"."id" = "Account->Actor->Avatars"."actorId" + AND "Account->Actor->Avatars"."type" = ${ActorImageType.AVATAR} + LEFT JOIN "server" AS "Account->Actor->Server" ON "Account->Actor"."serverId" = "Account->Actor->Server"."id" + ) ON "UserNotificationModel"."accountId" = "Account"."id" + + LEFT JOIN "userRegistration" as "UserRegistration" ON "UserNotificationModel"."userRegistrationId" = "UserRegistration"."id"` + } +} diff --git a/server/server/models/user/user-notification-setting.ts b/server/server/models/user/user-notification-setting.ts new file mode 100644 index 000000000..8b59fbe70 --- /dev/null +++ b/server/server/models/user/user-notification-setting.ts @@ -0,0 +1,232 @@ +import { type UserNotificationSetting, type UserNotificationSettingValueType } from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { TokensCache } from '@server/lib/auth/tokens-cache.js' +import { MNotificationSettingFormattable } from '@server/types/models/index.js' +import { + AfterDestroy, + AfterUpdate, + AllowNull, + BelongsTo, + Column, + CreatedAt, + Default, + ForeignKey, + Is, + Model, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { isUserNotificationSettingValid } from '../../helpers/custom-validators/user-notifications.js' +import { throwIfNotValid } from '../shared/index.js' +import { UserModel } from './user.js' + +@Table({ + tableName: 'userNotificationSetting', + indexes: [ + { + fields: [ 'userId' ], + unique: true + } + ] +}) +export class UserNotificationSettingModel extends Model>> { + + @AllowNull(false) + @Default(null) + @Is( + 'UserNotificationSettingNewVideoFromSubscription', + value => throwIfNotValid(value, isUserNotificationSettingValid, 'newVideoFromSubscription') + ) + @Column + newVideoFromSubscription: UserNotificationSettingValueType + + @AllowNull(false) + @Default(null) + @Is( + 'UserNotificationSettingNewCommentOnMyVideo', + value => throwIfNotValid(value, isUserNotificationSettingValid, 'newCommentOnMyVideo') + ) + @Column + newCommentOnMyVideo: UserNotificationSettingValueType + + @AllowNull(false) + @Default(null) + @Is( + 'UserNotificationSettingAbuseAsModerator', + value => throwIfNotValid(value, isUserNotificationSettingValid, 'abuseAsModerator') + ) + @Column + abuseAsModerator: UserNotificationSettingValueType + + @AllowNull(false) + @Default(null) + @Is( + 'UserNotificationSettingVideoAutoBlacklistAsModerator', + value => throwIfNotValid(value, isUserNotificationSettingValid, 'videoAutoBlacklistAsModerator') + ) + @Column + videoAutoBlacklistAsModerator: UserNotificationSettingValueType + + @AllowNull(false) + @Default(null) + @Is( + 'UserNotificationSettingBlacklistOnMyVideo', + value => throwIfNotValid(value, isUserNotificationSettingValid, 'blacklistOnMyVideo') + ) + @Column + blacklistOnMyVideo: UserNotificationSettingValueType + + @AllowNull(false) + @Default(null) + @Is( + 'UserNotificationSettingMyVideoPublished', + value => throwIfNotValid(value, isUserNotificationSettingValid, 'myVideoPublished') + ) + @Column + myVideoPublished: UserNotificationSettingValueType + + @AllowNull(false) + @Default(null) + @Is( + 'UserNotificationSettingMyVideoImportFinished', + value => throwIfNotValid(value, isUserNotificationSettingValid, 'myVideoImportFinished') + ) + @Column + myVideoImportFinished: UserNotificationSettingValueType + + @AllowNull(false) + @Default(null) + @Is( + 'UserNotificationSettingNewUserRegistration', + value => throwIfNotValid(value, isUserNotificationSettingValid, 'newUserRegistration') + ) + @Column + newUserRegistration: UserNotificationSettingValueType + + @AllowNull(false) + @Default(null) + @Is( + 'UserNotificationSettingNewInstanceFollower', + value => throwIfNotValid(value, isUserNotificationSettingValid, 'newInstanceFollower') + ) + @Column + newInstanceFollower: UserNotificationSettingValueType + + @AllowNull(false) + @Default(null) + @Is( + 'UserNotificationSettingNewInstanceFollower', + value => throwIfNotValid(value, isUserNotificationSettingValid, 'autoInstanceFollowing') + ) + @Column + autoInstanceFollowing: UserNotificationSettingValueType + + @AllowNull(false) + @Default(null) + @Is( + 'UserNotificationSettingNewFollow', + value => throwIfNotValid(value, isUserNotificationSettingValid, 'newFollow') + ) + @Column + newFollow: UserNotificationSettingValueType + + @AllowNull(false) + @Default(null) + @Is( + 'UserNotificationSettingCommentMention', + value => throwIfNotValid(value, isUserNotificationSettingValid, 'commentMention') + ) + @Column + commentMention: UserNotificationSettingValueType + + @AllowNull(false) + @Default(null) + @Is( + 'UserNotificationSettingAbuseStateChange', + value => throwIfNotValid(value, isUserNotificationSettingValid, 'abuseStateChange') + ) + @Column + abuseStateChange: UserNotificationSettingValueType + + @AllowNull(false) + @Default(null) + @Is( + 'UserNotificationSettingAbuseNewMessage', + value => throwIfNotValid(value, isUserNotificationSettingValid, 'abuseNewMessage') + ) + @Column + abuseNewMessage: UserNotificationSettingValueType + + @AllowNull(false) + @Default(null) + @Is( + 'UserNotificationSettingNewPeerTubeVersion', + value => throwIfNotValid(value, isUserNotificationSettingValid, 'newPeerTubeVersion') + ) + @Column + newPeerTubeVersion: UserNotificationSettingValueType + + @AllowNull(false) + @Default(null) + @Is( + 'UserNotificationSettingNewPeerPluginVersion', + value => throwIfNotValid(value, isUserNotificationSettingValid, 'newPluginVersion') + ) + @Column + newPluginVersion: UserNotificationSettingValueType + + @AllowNull(false) + @Default(null) + @Is( + 'UserNotificationSettingMyVideoStudioEditionFinished', + value => throwIfNotValid(value, isUserNotificationSettingValid, 'myVideoStudioEditionFinished') + ) + @Column + myVideoStudioEditionFinished: UserNotificationSettingValueType + + @ForeignKey(() => UserModel) + @Column + userId: number + + @BelongsTo(() => UserModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + User: Awaited + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AfterUpdate + @AfterDestroy + static removeTokenCache (instance: UserNotificationSettingModel) { + return TokensCache.Instance.clearCacheByUserId(instance.userId) + } + + toFormattedJSON (this: MNotificationSettingFormattable): UserNotificationSetting { + return { + newCommentOnMyVideo: this.newCommentOnMyVideo, + newVideoFromSubscription: this.newVideoFromSubscription, + abuseAsModerator: this.abuseAsModerator, + videoAutoBlacklistAsModerator: this.videoAutoBlacklistAsModerator, + blacklistOnMyVideo: this.blacklistOnMyVideo, + myVideoPublished: this.myVideoPublished, + myVideoImportFinished: this.myVideoImportFinished, + newUserRegistration: this.newUserRegistration, + commentMention: this.commentMention, + newFollow: this.newFollow, + newInstanceFollower: this.newInstanceFollower, + autoInstanceFollowing: this.autoInstanceFollowing, + abuseNewMessage: this.abuseNewMessage, + abuseStateChange: this.abuseStateChange, + newPeerTubeVersion: this.newPeerTubeVersion, + myVideoStudioEditionFinished: this.myVideoStudioEditionFinished, + newPluginVersion: this.newPluginVersion + } + } +} diff --git a/server/server/models/user/user-notification.ts b/server/server/models/user/user-notification.ts new file mode 100644 index 000000000..a96cb666c --- /dev/null +++ b/server/server/models/user/user-notification.ts @@ -0,0 +1,534 @@ +import { forceNumber } from '@peertube/peertube-core-utils' +import { UserNotification, type UserNotificationType_Type } from '@peertube/peertube-models' +import { uuidToShort } from '@peertube/peertube-node-utils' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { getBiggestActorImage } from '@server/lib/actor-image.js' +import { UserNotificationIncludes, UserNotificationModelForApi } from '@server/types/models/user/index.js' +import { ModelIndexesOptions, Op, WhereOptions } from 'sequelize' +import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { isBooleanValid } from '../../helpers/custom-validators/misc.js' +import { isUserNotificationTypeValid } from '../../helpers/custom-validators/user-notifications.js' +import { AbuseModel } from '../abuse/abuse.js' +import { AccountModel } from '../account/account.js' +import { ActorFollowModel } from '../actor/actor-follow.js' +import { ApplicationModel } from '../application/application.js' +import { PluginModel } from '../server/plugin.js' +import { throwIfNotValid } from '../shared/index.js' +import { VideoBlacklistModel } from '../video/video-blacklist.js' +import { VideoCommentModel } from '../video/video-comment.js' +import { VideoImportModel } from '../video/video-import.js' +import { VideoModel } from '../video/video.js' +import { UserNotificationListQueryBuilder } from './sql/user-notitication-list-query-builder.js' +import { UserRegistrationModel } from './user-registration.js' +import { UserModel } from './user.js' + +@Table({ + tableName: 'userNotification', + indexes: [ + { + fields: [ 'userId' ] + }, + { + fields: [ 'videoId' ], + where: { + videoId: { + [Op.ne]: null + } + } + }, + { + fields: [ 'commentId' ], + where: { + commentId: { + [Op.ne]: null + } + } + }, + { + fields: [ 'abuseId' ], + where: { + abuseId: { + [Op.ne]: null + } + } + }, + { + fields: [ 'videoBlacklistId' ], + where: { + videoBlacklistId: { + [Op.ne]: null + } + } + }, + { + fields: [ 'videoImportId' ], + where: { + videoImportId: { + [Op.ne]: null + } + } + }, + { + fields: [ 'accountId' ], + where: { + accountId: { + [Op.ne]: null + } + } + }, + { + fields: [ 'actorFollowId' ], + where: { + actorFollowId: { + [Op.ne]: null + } + } + }, + { + fields: [ 'pluginId' ], + where: { + pluginId: { + [Op.ne]: null + } + } + }, + { + fields: [ 'applicationId' ], + where: { + applicationId: { + [Op.ne]: null + } + } + }, + { + fields: [ 'userRegistrationId' ], + where: { + userRegistrationId: { + [Op.ne]: null + } + } + } + ] as (ModelIndexesOptions & { where?: WhereOptions })[] +}) +export class UserNotificationModel extends Model>> { + + @AllowNull(false) + @Default(null) + @Is('UserNotificationType', value => throwIfNotValid(value, isUserNotificationTypeValid, 'type')) + @Column + type: UserNotificationType_Type + + @AllowNull(false) + @Default(false) + @Is('UserNotificationRead', value => throwIfNotValid(value, isBooleanValid, 'read')) + @Column + read: boolean + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => UserModel) + @Column + userId: number + + @BelongsTo(() => UserModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + User: Awaited + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'cascade' + }) + Video: Awaited + + @ForeignKey(() => VideoCommentModel) + @Column + commentId: number + + @BelongsTo(() => VideoCommentModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'cascade' + }) + VideoComment: Awaited + + @ForeignKey(() => AbuseModel) + @Column + abuseId: number + + @BelongsTo(() => AbuseModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'cascade' + }) + Abuse: Awaited + + @ForeignKey(() => VideoBlacklistModel) + @Column + videoBlacklistId: number + + @BelongsTo(() => VideoBlacklistModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'cascade' + }) + VideoBlacklist: Awaited + + @ForeignKey(() => VideoImportModel) + @Column + videoImportId: number + + @BelongsTo(() => VideoImportModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'cascade' + }) + VideoImport: Awaited + + @ForeignKey(() => AccountModel) + @Column + accountId: number + + @BelongsTo(() => AccountModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'cascade' + }) + Account: Awaited + + @ForeignKey(() => ActorFollowModel) + @Column + actorFollowId: number + + @BelongsTo(() => ActorFollowModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'cascade' + }) + ActorFollow: Awaited + + @ForeignKey(() => PluginModel) + @Column + pluginId: number + + @BelongsTo(() => PluginModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'cascade' + }) + Plugin: Awaited + + @ForeignKey(() => ApplicationModel) + @Column + applicationId: number + + @BelongsTo(() => ApplicationModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'cascade' + }) + Application: Awaited + + @ForeignKey(() => UserRegistrationModel) + @Column + userRegistrationId: number + + @BelongsTo(() => UserRegistrationModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'cascade' + }) + UserRegistration: Awaited + + static listForApi (userId: number, start: number, count: number, sort: string, unread?: boolean) { + const where = { userId } + + const query = { + userId, + unread, + offset: start, + limit: count, + sort, + where + } + + if (unread !== undefined) query.where['read'] = !unread + + return Promise.all([ + UserNotificationModel.count({ where }) + .then(count => count || 0), + + count === 0 + ? [] as UserNotificationModelForApi[] + : new UserNotificationListQueryBuilder(this.sequelize, query).listNotifications() + ]).then(([ total, data ]) => ({ total, data })) + } + + static markAsRead (userId: number, notificationIds: number[]) { + const query = { + where: { + userId, + id: { + [Op.in]: notificationIds + } + } + } + + return UserNotificationModel.update({ read: true }, query) + } + + static markAllAsRead (userId: number) { + const query = { where: { userId } } + + return UserNotificationModel.update({ read: true }, query) + } + + static removeNotificationsOf (options: { id: number, type: 'account' | 'server', forUserId?: number }) { + const id = forceNumber(options.id) + + function buildAccountWhereQuery (base: string) { + const whereSuffix = options.forUserId + ? ` AND "userNotification"."userId" = ${options.forUserId}` + : '' + + if (options.type === 'account') { + return base + + ` WHERE "account"."id" = ${id} ${whereSuffix}` + } + + return base + + ` WHERE "actor"."serverId" = ${id} ${whereSuffix}` + } + + const queries = [ + buildAccountWhereQuery( + `SELECT "userNotification"."id" FROM "userNotification" ` + + `INNER JOIN "account" ON "userNotification"."accountId" = "account"."id" ` + + `INNER JOIN actor ON "actor"."id" = "account"."actorId" ` + ), + + // Remove notifications from muted accounts that followed ours + buildAccountWhereQuery( + `SELECT "userNotification"."id" FROM "userNotification" ` + + `INNER JOIN "actorFollow" ON "actorFollow".id = "userNotification"."actorFollowId" ` + + `INNER JOIN actor ON actor.id = "actorFollow"."actorId" ` + + `INNER JOIN account ON account."actorId" = actor.id ` + ), + + // Remove notifications from muted accounts that commented something + buildAccountWhereQuery( + `SELECT "userNotification"."id" FROM "userNotification" ` + + `INNER JOIN "actorFollow" ON "actorFollow".id = "userNotification"."actorFollowId" ` + + `INNER JOIN actor ON actor.id = "actorFollow"."actorId" ` + + `INNER JOIN account ON account."actorId" = actor.id ` + ), + + buildAccountWhereQuery( + `SELECT "userNotification"."id" FROM "userNotification" ` + + `INNER JOIN "videoComment" ON "videoComment".id = "userNotification"."commentId" ` + + `INNER JOIN account ON account.id = "videoComment"."accountId" ` + + `INNER JOIN actor ON "actor"."id" = "account"."actorId" ` + ) + ] + + const query = `DELETE FROM "userNotification" WHERE id IN (${queries.join(' UNION ')})` + + return UserNotificationModel.sequelize.query(query) + } + + toFormattedJSON (this: UserNotificationModelForApi): UserNotification { + const video = this.Video + ? { + ...this.formatVideo(this.Video), + + channel: this.formatActor(this.Video.VideoChannel) + } + : undefined + + const videoImport = this.VideoImport + ? { + id: this.VideoImport.id, + video: this.VideoImport.Video + ? this.formatVideo(this.VideoImport.Video) + : undefined, + torrentName: this.VideoImport.torrentName, + magnetUri: this.VideoImport.magnetUri, + targetUrl: this.VideoImport.targetUrl + } + : undefined + + const comment = this.VideoComment + ? { + id: this.VideoComment.id, + threadId: this.VideoComment.getThreadId(), + account: this.formatActor(this.VideoComment.Account), + video: this.formatVideo(this.VideoComment.Video) + } + : undefined + + const abuse = this.Abuse ? this.formatAbuse(this.Abuse) : undefined + + const videoBlacklist = this.VideoBlacklist + ? { + id: this.VideoBlacklist.id, + video: this.formatVideo(this.VideoBlacklist.Video) + } + : undefined + + const account = this.Account ? this.formatActor(this.Account) : undefined + + const actorFollowingType = { + Application: 'instance' as 'instance', + Group: 'channel' as 'channel', + Person: 'account' as 'account' + } + const actorFollow = this.ActorFollow + ? { + id: this.ActorFollow.id, + state: this.ActorFollow.state, + follower: { + id: this.ActorFollow.ActorFollower.Account.id, + displayName: this.ActorFollow.ActorFollower.Account.getDisplayName(), + name: this.ActorFollow.ActorFollower.preferredUsername, + host: this.ActorFollow.ActorFollower.getHost(), + + ...this.formatAvatars(this.ActorFollow.ActorFollower.Avatars) + }, + following: { + type: actorFollowingType[this.ActorFollow.ActorFollowing.type], + displayName: (this.ActorFollow.ActorFollowing.VideoChannel || this.ActorFollow.ActorFollowing.Account).getDisplayName(), + name: this.ActorFollow.ActorFollowing.preferredUsername, + host: this.ActorFollow.ActorFollowing.getHost() + } + } + : undefined + + const plugin = this.Plugin + ? { + name: this.Plugin.name, + type: this.Plugin.type, + latestVersion: this.Plugin.latestVersion + } + : undefined + + const peertube = this.Application + ? { latestVersion: this.Application.latestPeerTubeVersion } + : undefined + + const registration = this.UserRegistration + ? { id: this.UserRegistration.id, username: this.UserRegistration.username } + : undefined + + return { + id: this.id, + type: this.type, + read: this.read, + video, + videoImport, + comment, + abuse, + videoBlacklist, + account, + actorFollow, + plugin, + peertube, + registration, + createdAt: this.createdAt.toISOString(), + updatedAt: this.updatedAt.toISOString() + } + } + + formatVideo (video: UserNotificationIncludes.VideoInclude) { + return { + id: video.id, + uuid: video.uuid, + shortUUID: uuidToShort(video.uuid), + name: video.name + } + } + + formatAbuse (abuse: UserNotificationIncludes.AbuseInclude) { + const commentAbuse = abuse.VideoCommentAbuse?.VideoComment + ? { + threadId: abuse.VideoCommentAbuse.VideoComment.getThreadId(), + + video: abuse.VideoCommentAbuse.VideoComment.Video + ? { + id: abuse.VideoCommentAbuse.VideoComment.Video.id, + name: abuse.VideoCommentAbuse.VideoComment.Video.name, + shortUUID: uuidToShort(abuse.VideoCommentAbuse.VideoComment.Video.uuid), + uuid: abuse.VideoCommentAbuse.VideoComment.Video.uuid + } + : undefined + } + : undefined + + const videoAbuse = abuse.VideoAbuse?.Video + ? this.formatVideo(abuse.VideoAbuse.Video) + : undefined + + const accountAbuse = (!commentAbuse && !videoAbuse && abuse.FlaggedAccount) + ? this.formatActor(abuse.FlaggedAccount) + : undefined + + return { + id: abuse.id, + state: abuse.state, + video: videoAbuse, + comment: commentAbuse, + account: accountAbuse + } + } + + formatActor ( + accountOrChannel: UserNotificationIncludes.AccountIncludeActor | UserNotificationIncludes.VideoChannelIncludeActor + ) { + return { + id: accountOrChannel.id, + displayName: accountOrChannel.getDisplayName(), + name: accountOrChannel.Actor.preferredUsername, + host: accountOrChannel.Actor.getHost(), + + ...this.formatAvatars(accountOrChannel.Actor.Avatars) + } + } + + formatAvatars (avatars: UserNotificationIncludes.ActorImageInclude[]) { + if (!avatars || avatars.length === 0) return { avatar: undefined, avatars: [] } + + return { + avatar: this.formatAvatar(getBiggestActorImage(avatars)), + + avatars: avatars.map(a => this.formatAvatar(a)) + } + } + + formatAvatar (a: UserNotificationIncludes.ActorImageInclude) { + return { + path: a.getStaticPath(), + width: a.width + } + } +} diff --git a/server/server/models/user/user-registration.ts b/server/server/models/user/user-registration.ts new file mode 100644 index 000000000..c4bf50b1d --- /dev/null +++ b/server/server/models/user/user-registration.ts @@ -0,0 +1,259 @@ +import { UserRegistration, type UserRegistrationStateType } from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { + isRegistrationModerationResponseValid, + isRegistrationReasonValid, + isRegistrationStateValid +} from '@server/helpers/custom-validators/user-registration.js' +import { isVideoChannelDisplayNameValid } from '@server/helpers/custom-validators/video-channels.js' +import { cryptPassword } from '@server/helpers/peertube-crypto.js' +import { USER_REGISTRATION_STATES } from '@server/initializers/constants.js' +import { MRegistration, MRegistrationFormattable } from '@server/types/models/index.js' +import { FindOptions, Op, WhereOptions } from 'sequelize' +import { + AllowNull, + BeforeCreate, + BelongsTo, + Column, + CreatedAt, + DataType, + ForeignKey, + Is, + IsEmail, + Model, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { isUserDisplayNameValid, isUserEmailVerifiedValid, isUserPasswordValid } from '../../helpers/custom-validators/users.js' +import { getSort, throwIfNotValid } from '../shared/index.js' +import { UserModel } from './user.js' + +@Table({ + tableName: 'userRegistration', + indexes: [ + { + fields: [ 'username' ], + unique: true + }, + { + fields: [ 'email' ], + unique: true + }, + { + fields: [ 'channelHandle' ], + unique: true + }, + { + fields: [ 'userId' ], + unique: true + } + ] +}) +export class UserRegistrationModel extends Model>> { + + @AllowNull(false) + @Is('RegistrationState', value => throwIfNotValid(value, isRegistrationStateValid, 'state')) + @Column + state: UserRegistrationStateType + + @AllowNull(false) + @Is('RegistrationReason', value => throwIfNotValid(value, isRegistrationReasonValid, 'registration reason')) + @Column(DataType.TEXT) + registrationReason: string + + @AllowNull(true) + @Is('RegistrationModerationResponse', value => throwIfNotValid(value, isRegistrationModerationResponseValid, 'moderation response', true)) + @Column(DataType.TEXT) + moderationResponse: string + + @AllowNull(true) + @Is('RegistrationPassword', value => throwIfNotValid(value, isUserPasswordValid, 'registration password', true)) + @Column + password: string + + @AllowNull(false) + @Column + username: string + + @AllowNull(false) + @IsEmail + @Column(DataType.STRING(400)) + email: string + + @AllowNull(true) + @Is('RegistrationEmailVerified', value => throwIfNotValid(value, isUserEmailVerifiedValid, 'email verified boolean', true)) + @Column + emailVerified: boolean + + @AllowNull(true) + @Is('RegistrationAccountDisplayName', value => throwIfNotValid(value, isUserDisplayNameValid, 'account display name', true)) + @Column + accountDisplayName: string + + @AllowNull(true) + @Is('ChannelHandle', value => throwIfNotValid(value, isVideoChannelDisplayNameValid, 'channel handle', true)) + @Column + channelHandle: string + + @AllowNull(true) + @Is('ChannelDisplayName', value => throwIfNotValid(value, isVideoChannelDisplayNameValid, 'channel display name', true)) + @Column + channelDisplayName: string + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => UserModel) + @Column + userId: number + + @BelongsTo(() => UserModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'SET NULL' + }) + User: Awaited + + @BeforeCreate + static async cryptPasswordIfNeeded (instance: UserRegistrationModel) { + instance.password = await cryptPassword(instance.password) + } + + static load (id: number): Promise { + return UserRegistrationModel.findByPk(id) + } + + static loadByEmail (email: string): Promise { + const query = { + where: { email } + } + + return UserRegistrationModel.findOne(query) + } + + static loadByEmailOrUsername (emailOrUsername: string): Promise { + const query = { + where: { + [Op.or]: [ + { email: emailOrUsername }, + { username: emailOrUsername } + ] + } + } + + return UserRegistrationModel.findOne(query) + } + + static loadByEmailOrHandle (options: { + email: string + username: string + channelHandle?: string + }): Promise { + const { email, username, channelHandle } = options + + let or: WhereOptions = [ + { email }, + { channelHandle: username }, + { username } + ] + + if (channelHandle) { + or = or.concat([ + { username: channelHandle }, + { channelHandle } + ]) + } + + const query = { + where: { + [Op.or]: or + } + } + + return UserRegistrationModel.findOne(query) + } + + // --------------------------------------------------------------------------- + + static listForApi (options: { + start: number + count: number + sort: string + search?: string + }) { + const { start, count, sort, search } = options + + const where: WhereOptions = {} + + if (search) { + Object.assign(where, { + [Op.or]: [ + { + email: { + [Op.iLike]: '%' + search + '%' + } + }, + { + username: { + [Op.iLike]: '%' + search + '%' + } + } + ] + }) + } + + const query: FindOptions = { + offset: start, + limit: count, + order: getSort(sort), + where, + include: [ + { + model: UserModel.unscoped(), + required: false + } + ] + } + + return Promise.all([ + UserRegistrationModel.count(query), + UserRegistrationModel.findAll(query) + ]).then(([ total, data ]) => ({ total, data })) + } + + // --------------------------------------------------------------------------- + + toFormattedJSON (this: MRegistrationFormattable): UserRegistration { + return { + id: this.id, + + state: { + id: this.state, + label: USER_REGISTRATION_STATES[this.state] + }, + + registrationReason: this.registrationReason, + moderationResponse: this.moderationResponse, + + username: this.username, + email: this.email, + emailVerified: this.emailVerified, + + accountDisplayName: this.accountDisplayName, + + channelHandle: this.channelHandle, + channelDisplayName: this.channelDisplayName, + + createdAt: this.createdAt, + updatedAt: this.updatedAt, + + user: this.User + ? { id: this.User.id } + : null + } + } +} diff --git a/server/server/models/user/user-video-history.ts b/server/server/models/user/user-video-history.ts new file mode 100644 index 000000000..4764f58eb --- /dev/null +++ b/server/server/models/user/user-video-history.ts @@ -0,0 +1,113 @@ +import { DestroyOptions, Op, Transaction } from 'sequelize' +import { AllowNull, BelongsTo, Column, CreatedAt, ForeignKey, IsInt, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { ResultList } from '@peertube/peertube-models' +import { MUserAccountId, MUserId } from '@server/types/models/index.js' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { VideoModel } from '../video/video.js' +import { UserModel } from './user.js' + +@Table({ + tableName: 'userVideoHistory', + indexes: [ + { + fields: [ 'userId', 'videoId' ], + unique: true + }, + { + fields: [ 'userId' ] + }, + { + fields: [ 'videoId' ] + } + ] +}) +export class UserVideoHistoryModel extends Model>> { + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(false) + @IsInt + @Column + currentTime: number + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + Video: Awaited + + @ForeignKey(() => UserModel) + @Column + userId: number + + @BelongsTo(() => UserModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + User: Awaited + + // FIXME: have to specify the result type to not break peertube typings generation + static listForApi (user: MUserAccountId, start: number, count: number, search?: string): Promise> { + return VideoModel.listForApi({ + start, + count, + search, + sort: '-"userVideoHistory"."updatedAt"', + nsfw: null, // All + displayOnlyForFollower: null, + user, + historyOfUser: user + }) + } + + static removeUserHistoryElement (user: MUserId, videoId: number) { + const query: DestroyOptions = { + where: { + userId: user.id, + videoId + } + } + + return UserVideoHistoryModel.destroy(query) + } + + static removeUserHistoryBefore (user: MUserId, beforeDate: string, t: Transaction) { + const query: DestroyOptions = { + where: { + userId: user.id + }, + transaction: t + } + + if (beforeDate) { + query.where['updatedAt'] = { + [Op.lt]: beforeDate + } + } + + return UserVideoHistoryModel.destroy(query) + } + + static removeOldHistory (beforeDate: string) { + const query: DestroyOptions = { + where: { + updatedAt: { + [Op.lt]: beforeDate + } + } + } + + return UserVideoHistoryModel.destroy(query) + } +} diff --git a/server/server/models/user/user.ts b/server/server/models/user/user.ts new file mode 100644 index 000000000..3c4495e3e --- /dev/null +++ b/server/server/models/user/user.ts @@ -0,0 +1,989 @@ +import { forceNumber, hasUserRight, USER_ROLE_LABELS } from '@peertube/peertube-core-utils' +import { + AbuseState, + MyUser, + User, + UserAdminFlag, + UserRightType, + VideoPlaylistType, + type NSFWPolicyType, + type UserAdminFlagType, + type UserRoleType +} from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { TokensCache } from '@server/lib/auth/tokens-cache.js' +import { LiveQuotaStore } from '@server/lib/live/index.js' +import { + MMyUserFormattable, + MUser, + MUserDefault, + MUserFormattable, + MUserNotifSettingChannelDefault, + MUserWithNotificationSetting +} from '@server/types/models/index.js' +import { col, FindOptions, fn, literal, Op, QueryTypes, where, WhereOptions } from 'sequelize' +import { + AfterDestroy, + AfterUpdate, + AllowNull, + BeforeCreate, + BeforeUpdate, + Column, + CreatedAt, + DataType, + Default, + DefaultScope, + HasMany, + HasOne, + Is, + IsEmail, + IsUUID, + Model, + Scopes, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { isThemeNameValid } from '../../helpers/custom-validators/plugins.js' +import { + isUserAdminFlagsValid, + isUserAutoPlayNextVideoPlaylistValid, + isUserAutoPlayNextVideoValid, + isUserAutoPlayVideoValid, + isUserBlockedReasonValid, + isUserBlockedValid, + isUserEmailVerifiedValid, + isUserNoModal, + isUserNSFWPolicyValid, + isUserP2PEnabledValid, + isUserPasswordValid, + isUserRoleValid, + isUserVideoLanguages, + isUserVideoQuotaDailyValid, + isUserVideoQuotaValid, + isUserVideosHistoryEnabledValid +} from '../../helpers/custom-validators/users.js' +import { comparePassword, cryptPassword } from '../../helpers/peertube-crypto.js' +import { DEFAULT_USER_THEME_NAME, NSFW_POLICY_TYPES } from '../../initializers/constants.js' +import { getThemeOrDefault } from '../../lib/plugins/theme-utils.js' +import { AccountModel } from '../account/account.js' +import { ActorFollowModel } from '../actor/actor-follow.js' +import { ActorImageModel } from '../actor/actor-image.js' +import { ActorModel } from '../actor/actor.js' +import { OAuthTokenModel } from '../oauth/oauth-token.js' +import { getAdminUsersSort, throwIfNotValid } from '../shared/index.js' +import { VideoChannelModel } from '../video/video-channel.js' +import { VideoImportModel } from '../video/video-import.js' +import { VideoLiveModel } from '../video/video-live.js' +import { VideoPlaylistModel } from '../video/video-playlist.js' +import { VideoModel } from '../video/video.js' +import { UserNotificationSettingModel } from './user-notification-setting.js' + +enum ScopeNames { + FOR_ME_API = 'FOR_ME_API', + WITH_VIDEOCHANNELS = 'WITH_VIDEOCHANNELS', + WITH_QUOTA = 'WITH_QUOTA', + WITH_STATS = 'WITH_STATS' +} + +@DefaultScope(() => ({ + include: [ + { + model: AccountModel, + required: true + }, + { + model: UserNotificationSettingModel, + required: true + } + ] +})) +@Scopes(() => ({ + [ScopeNames.FOR_ME_API]: { + include: [ + { + model: AccountModel, + include: [ + { + model: VideoChannelModel.unscoped(), + include: [ + { + model: ActorModel, + required: true, + include: [ + { + model: ActorImageModel, + as: 'Banners', + required: false + } + ] + } + ] + }, + { + attributes: [ 'id', 'name', 'type' ], + model: VideoPlaylistModel.unscoped(), + required: true, + where: { + type: { + [Op.ne]: VideoPlaylistType.REGULAR + } + } + } + ] + }, + { + model: UserNotificationSettingModel, + required: true + } + ] + }, + [ScopeNames.WITH_VIDEOCHANNELS]: { + include: [ + { + model: AccountModel, + include: [ + { + model: VideoChannelModel + }, + { + attributes: [ 'id', 'name', 'type' ], + model: VideoPlaylistModel.unscoped(), + required: true, + where: { + type: { + [Op.ne]: VideoPlaylistType.REGULAR + } + } + } + ] + } + ] + }, + [ScopeNames.WITH_QUOTA]: { + attributes: { + include: [ + [ + literal( + '(' + + UserModel.generateUserQuotaBaseSQL({ + withSelect: false, + whereUserId: '"UserModel"."id"', + daily: false + }) + + ')' + ), + 'videoQuotaUsed' + ], + [ + literal( + '(' + + UserModel.generateUserQuotaBaseSQL({ + withSelect: false, + whereUserId: '"UserModel"."id"', + daily: true + }) + + ')' + ), + 'videoQuotaUsedDaily' + ] + ] + } + }, + [ScopeNames.WITH_STATS]: { + attributes: { + include: [ + [ + literal( + '(' + + 'SELECT COUNT("video"."id") ' + + 'FROM "video" ' + + 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' + + 'INNER JOIN "account" ON "account"."id" = "videoChannel"."accountId" ' + + 'WHERE "account"."userId" = "UserModel"."id"' + + ')' + ), + 'videosCount' + ], + [ + literal( + '(' + + `SELECT concat_ws(':', "abuses", "acceptedAbuses") ` + + 'FROM (' + + 'SELECT COUNT("abuse"."id") AS "abuses", ' + + `COUNT("abuse"."id") FILTER (WHERE "abuse"."state" = ${AbuseState.ACCEPTED}) AS "acceptedAbuses" ` + + 'FROM "abuse" ' + + 'INNER JOIN "account" ON "account"."id" = "abuse"."flaggedAccountId" ' + + 'WHERE "account"."userId" = "UserModel"."id"' + + ') t' + + ')' + ), + 'abusesCount' + ], + [ + literal( + '(' + + 'SELECT COUNT("abuse"."id") ' + + 'FROM "abuse" ' + + 'INNER JOIN "account" ON "account"."id" = "abuse"."reporterAccountId" ' + + 'WHERE "account"."userId" = "UserModel"."id"' + + ')' + ), + 'abusesCreatedCount' + ], + [ + literal( + '(' + + 'SELECT COUNT("videoComment"."id") ' + + 'FROM "videoComment" ' + + 'INNER JOIN "account" ON "account"."id" = "videoComment"."accountId" ' + + 'WHERE "account"."userId" = "UserModel"."id"' + + ')' + ), + 'videoCommentsCount' + ] + ] + } + } +})) +@Table({ + tableName: 'user', + indexes: [ + { + fields: [ 'username' ], + unique: true + }, + { + fields: [ 'email' ], + unique: true + } + ] +}) +export class UserModel extends Model>> { + + @AllowNull(true) + @Is('UserPassword', value => throwIfNotValid(value, isUserPasswordValid, 'user password', true)) + @Column + password: string + + @AllowNull(false) + @Column + username: string + + @AllowNull(false) + @IsEmail + @Column(DataType.STRING(400)) + email: string + + @AllowNull(true) + @IsEmail + @Column(DataType.STRING(400)) + pendingEmail: string + + @AllowNull(true) + @Default(null) + @Is('UserEmailVerified', value => throwIfNotValid(value, isUserEmailVerifiedValid, 'email verified boolean', true)) + @Column + emailVerified: boolean + + @AllowNull(false) + @Is('UserNSFWPolicy', value => throwIfNotValid(value, isUserNSFWPolicyValid, 'NSFW policy')) + @Column(DataType.ENUM(...Object.values(NSFW_POLICY_TYPES))) + nsfwPolicy: NSFWPolicyType + + @AllowNull(false) + @Is('p2pEnabled', value => throwIfNotValid(value, isUserP2PEnabledValid, 'P2P enabled')) + @Column + p2pEnabled: boolean + + @AllowNull(false) + @Default(true) + @Is('UserVideosHistoryEnabled', value => throwIfNotValid(value, isUserVideosHistoryEnabledValid, 'Videos history enabled')) + @Column + videosHistoryEnabled: boolean + + @AllowNull(false) + @Default(true) + @Is('UserAutoPlayVideo', value => throwIfNotValid(value, isUserAutoPlayVideoValid, 'auto play video boolean')) + @Column + autoPlayVideo: boolean + + @AllowNull(false) + @Default(false) + @Is('UserAutoPlayNextVideo', value => throwIfNotValid(value, isUserAutoPlayNextVideoValid, 'auto play next video boolean')) + @Column + autoPlayNextVideo: boolean + + @AllowNull(false) + @Default(true) + @Is( + 'UserAutoPlayNextVideoPlaylist', + value => throwIfNotValid(value, isUserAutoPlayNextVideoPlaylistValid, 'auto play next video for playlists boolean') + ) + @Column + autoPlayNextVideoPlaylist: boolean + + @AllowNull(true) + @Default(null) + @Is('UserVideoLanguages', value => throwIfNotValid(value, isUserVideoLanguages, 'video languages')) + @Column(DataType.ARRAY(DataType.STRING)) + videoLanguages: string[] + + @AllowNull(false) + @Default(UserAdminFlag.NONE) + @Is('UserAdminFlags', value => throwIfNotValid(value, isUserAdminFlagsValid, 'user admin flags')) + @Column + adminFlags?: UserAdminFlagType + + @AllowNull(false) + @Default(false) + @Is('UserBlocked', value => throwIfNotValid(value, isUserBlockedValid, 'blocked boolean')) + @Column + blocked: boolean + + @AllowNull(true) + @Default(null) + @Is('UserBlockedReason', value => throwIfNotValid(value, isUserBlockedReasonValid, 'blocked reason', true)) + @Column + blockedReason: string + + @AllowNull(false) + @Is('UserRole', value => throwIfNotValid(value, isUserRoleValid, 'role')) + @Column + role: UserRoleType + + @AllowNull(false) + @Is('UserVideoQuota', value => throwIfNotValid(value, isUserVideoQuotaValid, 'video quota')) + @Column(DataType.BIGINT) + videoQuota: number + + @AllowNull(false) + @Is('UserVideoQuotaDaily', value => throwIfNotValid(value, isUserVideoQuotaDailyValid, 'video quota daily')) + @Column(DataType.BIGINT) + videoQuotaDaily: number + + @AllowNull(false) + @Default(DEFAULT_USER_THEME_NAME) + @Is('UserTheme', value => throwIfNotValid(value, isThemeNameValid, 'theme')) + @Column + theme: string + + @AllowNull(false) + @Default(false) + @Is( + 'UserNoInstanceConfigWarningModal', + value => throwIfNotValid(value, isUserNoModal, 'no instance config warning modal') + ) + @Column + noInstanceConfigWarningModal: boolean + + @AllowNull(false) + @Default(false) + @Is( + 'UserNoWelcomeModal', + value => throwIfNotValid(value, isUserNoModal, 'no welcome modal') + ) + @Column + noWelcomeModal: boolean + + @AllowNull(false) + @Default(false) + @Is( + 'UserNoAccountSetupWarningModal', + value => throwIfNotValid(value, isUserNoModal, 'no account setup warning modal') + ) + @Column + noAccountSetupWarningModal: boolean + + @AllowNull(true) + @Default(null) + @Column + pluginAuth: string + + @AllowNull(false) + @Default(DataType.UUIDV4) + @IsUUID(4) + @Column(DataType.UUID) + feedToken: string + + @AllowNull(true) + @Default(null) + @Column + lastLoginDate: Date + + @AllowNull(false) + @Default(false) + @Column + emailPublic: boolean + + @AllowNull(true) + @Default(null) + @Column + otpSecret: string + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @HasOne(() => AccountModel, { + foreignKey: 'userId', + onDelete: 'cascade', + hooks: true + }) + Account: Awaited + + @HasOne(() => UserNotificationSettingModel, { + foreignKey: 'userId', + onDelete: 'cascade', + hooks: true + }) + NotificationSetting: Awaited + + @HasMany(() => VideoImportModel, { + foreignKey: 'userId', + onDelete: 'cascade' + }) + VideoImports: Awaited[] + + @HasMany(() => OAuthTokenModel, { + foreignKey: 'userId', + onDelete: 'cascade' + }) + OAuthTokens: Awaited[] + + // Used if we already set an encrypted password in user model + skipPasswordEncryption = false + + @BeforeCreate + @BeforeUpdate + static async cryptPasswordIfNeeded (instance: UserModel) { + if (instance.skipPasswordEncryption) return + if (!instance.changed('password')) return + if (!instance.password) return + + instance.password = await cryptPassword(instance.password) + } + + @AfterUpdate + @AfterDestroy + static removeTokenCache (instance: UserModel) { + return TokensCache.Instance.clearCacheByUserId(instance.id) + } + + static countTotal () { + return UserModel.unscoped().count() + } + + static listForAdminApi (parameters: { + start: number + count: number + sort: string + search?: string + blocked?: boolean + }) { + const { start, count, sort, search, blocked } = parameters + const where: WhereOptions = {} + + if (search) { + Object.assign(where, { + [Op.or]: [ + { + email: { + [Op.iLike]: '%' + search + '%' + } + }, + { + username: { + [Op.iLike]: '%' + search + '%' + } + } + ] + }) + } + + if (blocked !== undefined) { + Object.assign(where, { blocked }) + } + + const query: FindOptions = { + offset: start, + limit: count, + order: getAdminUsersSort(sort), + where + } + + return Promise.all([ + UserModel.unscoped().count(query), + UserModel.scope([ 'defaultScope', ScopeNames.WITH_QUOTA ]).findAll(query) + ]).then(([ total, data ]) => ({ total, data })) + } + + static listWithRight (right: UserRightType): Promise { + const roles = Object.keys(USER_ROLE_LABELS) + .map(k => parseInt(k, 10) as UserRoleType) + .filter(role => hasUserRight(role, right)) + + const query = { + where: { + role: { + [Op.in]: roles + } + } + } + + return UserModel.findAll(query) + } + + static listUserSubscribersOf (actorId: number): Promise { + const query = { + include: [ + { + model: UserNotificationSettingModel.unscoped(), + required: true + }, + { + attributes: [ 'userId' ], + model: AccountModel.unscoped(), + required: true, + include: [ + { + attributes: [], + model: ActorModel.unscoped(), + required: true, + where: { + serverId: null + }, + include: [ + { + attributes: [], + as: 'ActorFollowings', + model: ActorFollowModel.unscoped(), + required: true, + where: { + state: 'accepted', + targetActorId: actorId + } + } + ] + } + ] + } + ] + } + + return UserModel.unscoped().findAll(query) + } + + static listByUsernames (usernames: string[]): Promise { + const query = { + where: { + username: usernames + } + } + + return UserModel.findAll(query) + } + + static loadById (id: number): Promise { + return UserModel.unscoped().findByPk(id) + } + + static loadByIdFull (id: number): Promise { + return UserModel.findByPk(id) + } + + static loadByIdWithChannels (id: number, withStats = false): Promise { + const scopes = [ + ScopeNames.WITH_VIDEOCHANNELS + ] + + if (withStats) { + scopes.push(ScopeNames.WITH_QUOTA) + scopes.push(ScopeNames.WITH_STATS) + } + + return UserModel.scope(scopes).findByPk(id) + } + + static loadByUsername (username: string): Promise { + const query = { + where: { + username + } + } + + return UserModel.findOne(query) + } + + static loadForMeAPI (id: number): Promise { + const query = { + where: { + id + } + } + + return UserModel.scope(ScopeNames.FOR_ME_API).findOne(query) + } + + static loadByEmail (email: string): Promise { + const query = { + where: { + email + } + } + + return UserModel.findOne(query) + } + + static loadByUsernameOrEmail (username: string, email?: string): Promise { + if (!email) email = username + + const query = { + where: { + [Op.or]: [ + where(fn('lower', col('username')), fn('lower', username) as any), + + { email } + ] + } + } + + return UserModel.findOne(query) + } + + static loadByVideoId (videoId: number): Promise { + const query = { + include: [ + { + required: true, + attributes: [ 'id' ], + model: AccountModel.unscoped(), + include: [ + { + required: true, + attributes: [ 'id' ], + model: VideoChannelModel.unscoped(), + include: [ + { + required: true, + attributes: [ 'id' ], + model: VideoModel.unscoped(), + where: { + id: videoId + } + } + ] + } + ] + } + ] + } + + return UserModel.findOne(query) + } + + static loadByVideoImportId (videoImportId: number): Promise { + const query = { + include: [ + { + required: true, + attributes: [ 'id' ], + model: VideoImportModel.unscoped(), + where: { + id: videoImportId + } + } + ] + } + + return UserModel.findOne(query) + } + + static loadByChannelActorId (videoChannelActorId: number): Promise { + const query = { + include: [ + { + required: true, + attributes: [ 'id' ], + model: AccountModel.unscoped(), + include: [ + { + required: true, + attributes: [ 'id' ], + model: VideoChannelModel.unscoped(), + where: { + actorId: videoChannelActorId + } + } + ] + } + ] + } + + return UserModel.findOne(query) + } + + static loadByAccountActorId (accountActorId: number): Promise { + const query = { + include: [ + { + required: true, + attributes: [ 'id' ], + model: AccountModel.unscoped(), + where: { + actorId: accountActorId + } + } + ] + } + + return UserModel.findOne(query) + } + + static loadByLiveId (liveId: number): Promise { + const query = { + include: [ + { + attributes: [ 'id' ], + model: AccountModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'id' ], + model: VideoChannelModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'id' ], + model: VideoModel.unscoped(), + required: true, + include: [ + { + attributes: [], + model: VideoLiveModel.unscoped(), + required: true, + where: { + id: liveId + } + } + ] + } + ] + } + ] + } + ] + } + + return UserModel.unscoped().findOne(query) + } + + static generateUserQuotaBaseSQL (options: { + whereUserId: '$userId' | '"UserModel"."id"' + withSelect: boolean + daily: boolean + }) { + const andWhere = options.daily === true + ? 'AND "video"."createdAt" > now() - interval \'24 hours\'' + : '' + + const videoChannelJoin = 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' + + 'INNER JOIN "account" ON "videoChannel"."accountId" = "account"."id" ' + + `WHERE "account"."userId" = ${options.whereUserId} ${andWhere}` + + const webVideoFiles = 'SELECT "videoFile"."size" AS "size", "video"."id" AS "videoId" FROM "videoFile" ' + + 'INNER JOIN "video" ON "videoFile"."videoId" = "video"."id" AND "video"."isLive" IS FALSE ' + + videoChannelJoin + + const hlsFiles = 'SELECT "videoFile"."size" AS "size", "video"."id" AS "videoId" FROM "videoFile" ' + + 'INNER JOIN "videoStreamingPlaylist" ON "videoFile"."videoStreamingPlaylistId" = "videoStreamingPlaylist".id ' + + 'INNER JOIN "video" ON "videoStreamingPlaylist"."videoId" = "video"."id" AND "video"."isLive" IS FALSE ' + + videoChannelJoin + + return 'SELECT COALESCE(SUM("size"), 0) AS "total" ' + + 'FROM (' + + `SELECT MAX("t1"."size") AS "size" FROM (${webVideoFiles} UNION ${hlsFiles}) t1 ` + + 'GROUP BY "t1"."videoId"' + + ') t2' + } + + static getTotalRawQuery (query: string, userId: number) { + const options = { + bind: { userId }, + type: QueryTypes.SELECT as QueryTypes.SELECT + } + + return UserModel.sequelize.query<{ total: string }>(query, options) + .then(([ { total } ]) => { + if (total === null) return 0 + + return parseInt(total, 10) + }) + } + + static async getStats () { + function getActiveUsers (days: number) { + const query = { + where: { + [Op.and]: [ + literal(`"lastLoginDate" > NOW() - INTERVAL '${days}d'`) + ] + } + } + + return UserModel.unscoped().count(query) + } + + const totalUsers = await UserModel.unscoped().count() + const totalDailyActiveUsers = await getActiveUsers(1) + const totalWeeklyActiveUsers = await getActiveUsers(7) + const totalMonthlyActiveUsers = await getActiveUsers(30) + const totalHalfYearActiveUsers = await getActiveUsers(180) + + return { + totalUsers, + totalDailyActiveUsers, + totalWeeklyActiveUsers, + totalMonthlyActiveUsers, + totalHalfYearActiveUsers + } + } + + static autoComplete (search: string) { + const query = { + where: { + username: { + [Op.like]: `%${search}%` + } + }, + limit: 10 + } + + return UserModel.findAll(query) + .then(u => u.map(u => u.username)) + } + + hasRight (right: UserRightType) { + return hasUserRight(this.role, right) + } + + hasAdminFlag (flag: UserAdminFlagType) { + return this.adminFlags & flag + } + + isPasswordMatch (password: string) { + return comparePassword(password, this.password) + } + + toFormattedJSON (this: MUserFormattable, parameters: { withAdminFlags?: boolean } = {}): User { + const videoQuotaUsed = this.get('videoQuotaUsed') + const videoQuotaUsedDaily = this.get('videoQuotaUsedDaily') + const videosCount = this.get('videosCount') + const [ abusesCount, abusesAcceptedCount ] = (this.get('abusesCount') as string || ':').split(':') + const abusesCreatedCount = this.get('abusesCreatedCount') + const videoCommentsCount = this.get('videoCommentsCount') + + const json: User = { + id: this.id, + username: this.username, + email: this.email, + theme: getThemeOrDefault(this.theme, DEFAULT_USER_THEME_NAME), + + pendingEmail: this.pendingEmail, + emailPublic: this.emailPublic, + emailVerified: this.emailVerified, + + nsfwPolicy: this.nsfwPolicy, + + p2pEnabled: this.p2pEnabled, + + videosHistoryEnabled: this.videosHistoryEnabled, + autoPlayVideo: this.autoPlayVideo, + autoPlayNextVideo: this.autoPlayNextVideo, + autoPlayNextVideoPlaylist: this.autoPlayNextVideoPlaylist, + videoLanguages: this.videoLanguages, + + role: { + id: this.role, + label: USER_ROLE_LABELS[this.role] + }, + + videoQuota: this.videoQuota, + videoQuotaDaily: this.videoQuotaDaily, + + videoQuotaUsed: videoQuotaUsed !== undefined + ? forceNumber(videoQuotaUsed) + LiveQuotaStore.Instance.getLiveQuotaOf(this.id) + : undefined, + + videoQuotaUsedDaily: videoQuotaUsedDaily !== undefined + ? forceNumber(videoQuotaUsedDaily) + LiveQuotaStore.Instance.getLiveQuotaOf(this.id) + : undefined, + + videosCount: videosCount !== undefined + ? forceNumber(videosCount) + : undefined, + abusesCount: abusesCount + ? forceNumber(abusesCount) + : undefined, + abusesAcceptedCount: abusesAcceptedCount + ? forceNumber(abusesAcceptedCount) + : undefined, + abusesCreatedCount: abusesCreatedCount !== undefined + ? forceNumber(abusesCreatedCount) + : undefined, + videoCommentsCount: videoCommentsCount !== undefined + ? forceNumber(videoCommentsCount) + : undefined, + + noInstanceConfigWarningModal: this.noInstanceConfigWarningModal, + noWelcomeModal: this.noWelcomeModal, + noAccountSetupWarningModal: this.noAccountSetupWarningModal, + + blocked: this.blocked, + blockedReason: this.blockedReason, + + account: this.Account.toFormattedJSON(), + + notificationSettings: this.NotificationSetting + ? this.NotificationSetting.toFormattedJSON() + : undefined, + + videoChannels: [], + + createdAt: this.createdAt, + + pluginAuth: this.pluginAuth, + + lastLoginDate: this.lastLoginDate, + + twoFactorEnabled: !!this.otpSecret + } + + if (parameters.withAdminFlags) { + Object.assign(json, { adminFlags: this.adminFlags }) + } + + if (Array.isArray(this.Account.VideoChannels) === true) { + json.videoChannels = this.Account.VideoChannels + .map(c => c.toFormattedJSON()) + .sort((v1, v2) => { + if (v1.createdAt < v2.createdAt) return -1 + if (v1.createdAt === v2.createdAt) return 0 + + return 1 + }) + } + + return json + } + + toMeFormattedJSON (this: MMyUserFormattable): MyUser { + const formatted = this.toFormattedJSON({ withAdminFlags: true }) + + const specialPlaylists = this.Account.VideoPlaylists + .map(p => ({ id: p.id, name: p.name, type: p.type })) + + return Object.assign(formatted, { specialPlaylists }) + } +} diff --git a/server/server/models/video/formatter/index.ts b/server/server/models/video/formatter/index.ts new file mode 100644 index 000000000..856595ad5 --- /dev/null +++ b/server/server/models/video/formatter/index.ts @@ -0,0 +1,2 @@ +export * from './video-activity-pub-format.js' +export * from './video-api-format.js' diff --git a/server/server/models/video/formatter/shared/index.ts b/server/server/models/video/formatter/shared/index.ts new file mode 100644 index 000000000..66fe5ff5b --- /dev/null +++ b/server/server/models/video/formatter/shared/index.ts @@ -0,0 +1 @@ +export * from './video-format-utils.js' diff --git a/server/server/models/video/formatter/shared/video-format-utils.ts b/server/server/models/video/formatter/shared/video-format-utils.ts new file mode 100644 index 000000000..8a8d9f4b2 --- /dev/null +++ b/server/server/models/video/formatter/shared/video-format-utils.ts @@ -0,0 +1,7 @@ +import { MVideoFile } from '@server/types/models/index.js' + +export function sortByResolutionDesc (fileA: MVideoFile, fileB: MVideoFile) { + if (fileA.resolution < fileB.resolution) return 1 + if (fileA.resolution === fileB.resolution) return 0 + return -1 +} diff --git a/server/server/models/video/formatter/video-activity-pub-format.ts b/server/server/models/video/formatter/video-activity-pub-format.ts new file mode 100644 index 000000000..759e6dbbc --- /dev/null +++ b/server/server/models/video/formatter/video-activity-pub-format.ts @@ -0,0 +1,296 @@ +import { isArray } from '@server/helpers/custom-validators/misc.js' +import { generateMagnetUri } from '@server/helpers/webtorrent.js' +import { getActivityStreamDuration } from '@server/lib/activitypub/activity.js' +import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls.js' +import { + ActivityIconObject, + ActivityPlaylistUrlObject, + ActivityPubStoryboard, + ActivityTagObject, + ActivityTrackerUrlObject, + ActivityUrlObject, + VideoObject +} from '@peertube/peertube-models' +import { MIMETYPES, WEBSERVER } from '../../../initializers/constants.js' +import { + getLocalVideoCommentsActivityPubUrl, + getLocalVideoDislikesActivityPubUrl, + getLocalVideoLikesActivityPubUrl, + getLocalVideoSharesActivityPubUrl +} from '../../../lib/activitypub/url.js' +import { MStreamingPlaylistFiles, MUserId, MVideo, MVideoAP, MVideoFile } from '../../../types/models/index.js' +import { VideoCaptionModel } from '../video-caption.js' +import { sortByResolutionDesc } from './shared/index.js' +import { getCategoryLabel, getLanguageLabel, getLicenceLabel } from './video-api-format.js' + +export function videoModelToActivityPubObject (video: MVideoAP): VideoObject { + const language = video.language + ? { identifier: video.language, name: getLanguageLabel(video.language) } + : undefined + + const category = video.category + ? { identifier: video.category + '', name: getCategoryLabel(video.category) } + : undefined + + const licence = video.licence + ? { identifier: video.licence + '', name: getLicenceLabel(video.licence) } + : undefined + + const url: ActivityUrlObject[] = [ + // HTML url should be the first element in the array so Mastodon correctly displays the embed + { + type: 'Link', + mediaType: 'text/html', + href: WEBSERVER.URL + '/videos/watch/' + video.uuid + } as ActivityUrlObject, + + ...buildVideoFileUrls({ video, files: video.VideoFiles }), + + ...buildStreamingPlaylistUrls(video), + + ...buildTrackerUrls(video) + ] + + return { + type: 'Video' as 'Video', + id: video.url, + name: video.name, + duration: getActivityStreamDuration(video.duration), + uuid: video.uuid, + category, + licence, + language, + views: video.views, + sensitive: video.nsfw, + waitTranscoding: video.waitTranscoding, + + state: video.state, + commentsEnabled: video.commentsEnabled, + downloadEnabled: video.downloadEnabled, + published: video.publishedAt.toISOString(), + + originallyPublishedAt: video.originallyPublishedAt + ? video.originallyPublishedAt.toISOString() + : null, + + updated: video.updatedAt.toISOString(), + + uploadDate: video.inputFileUpdatedAt?.toISOString(), + + tag: buildTags(video), + + mediaType: 'text/markdown', + content: video.description, + support: video.support, + + subtitleLanguage: buildSubtitleLanguage(video), + + icon: buildIcon(video), + + preview: buildPreviewAPAttribute(video), + + url, + + likes: getLocalVideoLikesActivityPubUrl(video), + dislikes: getLocalVideoDislikesActivityPubUrl(video), + shares: getLocalVideoSharesActivityPubUrl(video), + comments: getLocalVideoCommentsActivityPubUrl(video), + + attributedTo: [ + { + type: 'Person', + id: video.VideoChannel.Account.Actor.url + }, + { + type: 'Group', + id: video.VideoChannel.Actor.url + } + ], + + ...buildLiveAPAttributes(video) + } +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +function buildLiveAPAttributes (video: MVideoAP) { + if (!video.isLive) { + return { + isLiveBroadcast: false, + liveSaveReplay: null, + permanentLive: null, + latencyMode: null + } + } + + return { + isLiveBroadcast: true, + liveSaveReplay: video.VideoLive.saveReplay, + permanentLive: video.VideoLive.permanentLive, + latencyMode: video.VideoLive.latencyMode + } +} + +function buildPreviewAPAttribute (video: MVideoAP): ActivityPubStoryboard[] { + if (!video.Storyboard) return undefined + + const storyboard = video.Storyboard + + return [ + { + type: 'Image', + rel: [ 'storyboard' ], + url: [ + { + mediaType: 'image/jpeg', + + href: storyboard.getOriginFileUrl(video), + + width: storyboard.totalWidth, + height: storyboard.totalHeight, + + tileWidth: storyboard.spriteWidth, + tileHeight: storyboard.spriteHeight, + tileDuration: getActivityStreamDuration(storyboard.spriteDuration) + } + ] + } + ] +} + +function buildVideoFileUrls (options: { + video: MVideo + files: MVideoFile[] + user?: MUserId +}): ActivityUrlObject[] { + const { video, files } = options + + if (!isArray(files)) return [] + + const urls: ActivityUrlObject[] = [] + + const trackerUrls = video.getTrackerUrls() + const sortedFiles = files + .filter(f => !f.isLive()) + .sort(sortByResolutionDesc) + + for (const file of sortedFiles) { + urls.push({ + type: 'Link', + mediaType: MIMETYPES.VIDEO.EXT_MIMETYPE[file.extname] as any, + href: file.getFileUrl(video), + height: file.resolution, + size: file.size, + fps: file.fps + }) + + urls.push({ + type: 'Link', + rel: [ 'metadata', MIMETYPES.VIDEO.EXT_MIMETYPE[file.extname] ], + mediaType: 'application/json' as 'application/json', + href: getLocalVideoFileMetadataUrl(video, file), + height: file.resolution, + fps: file.fps + }) + + if (file.hasTorrent()) { + urls.push({ + type: 'Link', + mediaType: 'application/x-bittorrent' as 'application/x-bittorrent', + href: file.getTorrentUrl(), + height: file.resolution + }) + + urls.push({ + type: 'Link', + mediaType: 'application/x-bittorrent;x-scheme-handler/magnet' as 'application/x-bittorrent;x-scheme-handler/magnet', + href: generateMagnetUri(video, file, trackerUrls), + height: file.resolution + }) + } + } + + return urls +} + +// --------------------------------------------------------------------------- + +function buildStreamingPlaylistUrls (video: MVideoAP): ActivityPlaylistUrlObject[] { + if (!isArray(video.VideoStreamingPlaylists)) return [] + + return video.VideoStreamingPlaylists + .map(playlist => ({ + type: 'Link', + mediaType: 'application/x-mpegURL' as 'application/x-mpegURL', + href: playlist.getMasterPlaylistUrl(video), + tag: buildStreamingPlaylistTags(video, playlist) + })) +} + +function buildStreamingPlaylistTags (video: MVideoAP, playlist: MStreamingPlaylistFiles) { + return [ + ...playlist.p2pMediaLoaderInfohashes.map(i => ({ type: 'Infohash' as 'Infohash', name: i })), + + { + type: 'Link', + name: 'sha256', + mediaType: 'application/json' as 'application/json', + href: playlist.getSha256SegmentsUrl(video) + }, + + ...buildVideoFileUrls({ video, files: playlist.VideoFiles }) + ] as ActivityTagObject[] +} + +// --------------------------------------------------------------------------- + +function buildTrackerUrls (video: MVideoAP): ActivityTrackerUrlObject[] { + return video.getTrackerUrls() + .map(trackerUrl => { + const rel2 = trackerUrl.startsWith('http') + ? 'http' + : 'websocket' + + return { + type: 'Link', + name: `tracker-${rel2}`, + rel: [ 'tracker', rel2 ], + href: trackerUrl + } + }) +} + +// --------------------------------------------------------------------------- + +function buildTags (video: MVideoAP) { + if (!isArray(video.Tags)) return [] + + return video.Tags.map(t => ({ + type: 'Hashtag' as 'Hashtag', + name: t.name + })) +} + +function buildIcon (video: MVideoAP): ActivityIconObject[] { + return [ video.getMiniature(), video.getPreview() ] + .map(i => ({ + type: 'Image', + url: i.getOriginFileUrl(video), + mediaType: 'image/jpeg', + width: i.width, + height: i.height + })) +} + +function buildSubtitleLanguage (video: MVideoAP) { + if (!isArray(video.VideoCaptions)) return [] + + return video.VideoCaptions + .map(caption => ({ + identifier: caption.language, + name: VideoCaptionModel.getLanguageLabel(caption.language), + url: caption.getFileUrl(video) + })) +} diff --git a/server/server/models/video/formatter/video-api-format.ts b/server/server/models/video/formatter/video-api-format.ts new file mode 100644 index 000000000..958832485 --- /dev/null +++ b/server/server/models/video/formatter/video-api-format.ts @@ -0,0 +1,305 @@ +import { generateMagnetUri } from '@server/helpers/webtorrent.js' +import { tracer } from '@server/lib/opentelemetry/tracing.js' +import { getLocalVideoFileMetadataUrl } from '@server/lib/video-urls.js' +import { VideoViewsManager } from '@server/lib/views/video-views-manager.js' +import { uuidToShort } from '@peertube/peertube-node-utils' +import { + Video, + VideoAdditionalAttributes, + VideoDetails, + VideoFile, + VideoInclude, + VideosCommonQueryAfterSanitize, + VideoStreamingPlaylist +} from '@peertube/peertube-models' +import { isArray } from '../../../helpers/custom-validators/misc.js' +import { VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES, VIDEO_STATES } from '../../../initializers/constants.js' +import { MServer, MStreamingPlaylistRedundanciesOpt, MVideoFormattable, MVideoFormattableDetails } from '../../../types/models/index.js' +import { MVideoFileRedundanciesOpt } from '../../../types/models/video/video-file.js' +import { sortByResolutionDesc } from './shared/index.js' + +export type VideoFormattingJSONOptions = { + completeDescription?: boolean + + additionalAttributes?: { + state?: boolean + waitTranscoding?: boolean + scheduledUpdate?: boolean + blacklistInfo?: boolean + files?: boolean + blockedOwner?: boolean + } +} + +export function guessAdditionalAttributesFromQuery (query: VideosCommonQueryAfterSanitize): VideoFormattingJSONOptions { + if (!query?.include) return {} + + return { + additionalAttributes: { + state: !!(query.include & VideoInclude.NOT_PUBLISHED_STATE), + waitTranscoding: !!(query.include & VideoInclude.NOT_PUBLISHED_STATE), + scheduledUpdate: !!(query.include & VideoInclude.NOT_PUBLISHED_STATE), + blacklistInfo: !!(query.include & VideoInclude.BLACKLISTED), + files: !!(query.include & VideoInclude.FILES), + blockedOwner: !!(query.include & VideoInclude.BLOCKED_OWNER) + } + } +} + +// --------------------------------------------------------------------------- + +export function videoModelToFormattedJSON (video: MVideoFormattable, options: VideoFormattingJSONOptions = {}): Video { + const span = tracer.startSpan('peertube.VideoModel.toFormattedJSON') + + const userHistory = isArray(video.UserVideoHistories) + ? video.UserVideoHistories[0] + : undefined + + const videoObject: Video = { + id: video.id, + uuid: video.uuid, + shortUUID: uuidToShort(video.uuid), + + url: video.url, + + name: video.name, + category: { + id: video.category, + label: getCategoryLabel(video.category) + }, + licence: { + id: video.licence, + label: getLicenceLabel(video.licence) + }, + language: { + id: video.language, + label: getLanguageLabel(video.language) + }, + privacy: { + id: video.privacy, + label: getPrivacyLabel(video.privacy) + }, + nsfw: video.nsfw, + + truncatedDescription: video.getTruncatedDescription(), + description: options && options.completeDescription === true + ? video.description + : video.getTruncatedDescription(), + + isLocal: video.isOwned(), + duration: video.duration, + + views: video.views, + viewers: VideoViewsManager.Instance.getViewers(video), + + likes: video.likes, + dislikes: video.dislikes, + thumbnailPath: video.getMiniatureStaticPath(), + previewPath: video.getPreviewStaticPath(), + embedPath: video.getEmbedStaticPath(), + createdAt: video.createdAt, + updatedAt: video.updatedAt, + publishedAt: video.publishedAt, + originallyPublishedAt: video.originallyPublishedAt, + + isLive: video.isLive, + + account: video.VideoChannel.Account.toFormattedSummaryJSON(), + channel: video.VideoChannel.toFormattedSummaryJSON(), + + userHistory: userHistory + ? { currentTime: userHistory.currentTime } + : undefined, + + // Can be added by external plugins + pluginData: (video as any).pluginData, + + ...buildAdditionalAttributes(video, options) + } + + span.end() + + return videoObject +} + +export function videoModelToFormattedDetailsJSON (video: MVideoFormattableDetails): VideoDetails { + const span = tracer.startSpan('peertube.VideoModel.toFormattedDetailsJSON') + + const videoJSON = video.toFormattedJSON({ + completeDescription: true, + additionalAttributes: { + scheduledUpdate: true, + blacklistInfo: true, + files: true + } + }) as Video & Required> + + const tags = video.Tags + ? video.Tags.map(t => t.name) + : [] + + const detailsJSON = { + ...videoJSON, + + support: video.support, + descriptionPath: video.getDescriptionAPIPath(), + channel: video.VideoChannel.toFormattedJSON(), + account: video.VideoChannel.Account.toFormattedJSON(), + tags, + commentsEnabled: video.commentsEnabled, + downloadEnabled: video.downloadEnabled, + waitTranscoding: video.waitTranscoding, + inputFileUpdatedAt: video.inputFileUpdatedAt, + state: { + id: video.state, + label: getStateLabel(video.state) + }, + + trackerUrls: video.getTrackerUrls() + } + + span.end() + + return detailsJSON +} + +export function streamingPlaylistsModelToFormattedJSON ( + video: MVideoFormattable, + playlists: MStreamingPlaylistRedundanciesOpt[] +): VideoStreamingPlaylist[] { + if (isArray(playlists) === false) return [] + + return playlists + .map(playlist => ({ + id: playlist.id, + type: playlist.type, + + playlistUrl: playlist.getMasterPlaylistUrl(video), + segmentsSha256Url: playlist.getSha256SegmentsUrl(video), + + redundancies: isArray(playlist.RedundancyVideos) + ? playlist.RedundancyVideos.map(r => ({ baseUrl: r.fileUrl })) + : [], + + files: videoFilesModelToFormattedJSON(video, playlist.VideoFiles) + })) +} + +export function videoFilesModelToFormattedJSON ( + video: MVideoFormattable, + videoFiles: MVideoFileRedundanciesOpt[], + options: { + includeMagnet?: boolean // default true + } = {} +): VideoFile[] { + const { includeMagnet = true } = options + + if (isArray(videoFiles) === false) return [] + + const trackerUrls = includeMagnet + ? video.getTrackerUrls() + : [] + + return videoFiles + .filter(f => !f.isLive()) + .sort(sortByResolutionDesc) + .map(videoFile => { + return { + id: videoFile.id, + + resolution: { + id: videoFile.resolution, + label: videoFile.resolution === 0 + ? 'Audio' + : `${videoFile.resolution}p` + }, + + magnetUri: includeMagnet && videoFile.hasTorrent() + ? generateMagnetUri(video, videoFile, trackerUrls) + : undefined, + + size: videoFile.size, + fps: videoFile.fps, + + torrentUrl: videoFile.getTorrentUrl(), + torrentDownloadUrl: videoFile.getTorrentDownloadUrl(), + + fileUrl: videoFile.getFileUrl(video), + fileDownloadUrl: videoFile.getFileDownloadUrl(video), + + metadataUrl: videoFile.metadataUrl ?? getLocalVideoFileMetadataUrl(video, videoFile) + } + }) +} + +// --------------------------------------------------------------------------- + +export function getCategoryLabel (id: number) { + return VIDEO_CATEGORIES[id] || 'Unknown' +} + +export function getLicenceLabel (id: number) { + return VIDEO_LICENCES[id] || 'Unknown' +} + +export function getLanguageLabel (id: string) { + return VIDEO_LANGUAGES[id] || 'Unknown' +} + +export function getPrivacyLabel (id: number) { + return VIDEO_PRIVACIES[id] || 'Unknown' +} + +export function getStateLabel (id: number) { + return VIDEO_STATES[id] || 'Unknown' +} + +// --------------------------------------------------------------------------- +// Private +// --------------------------------------------------------------------------- + +function buildAdditionalAttributes (video: MVideoFormattable, options: VideoFormattingJSONOptions) { + const add = options.additionalAttributes + + const result: Partial = {} + + if (add?.state === true) { + result.state = { + id: video.state, + label: getStateLabel(video.state) + } + } + + if (add?.waitTranscoding === true) { + result.waitTranscoding = video.waitTranscoding + } + + if (add?.scheduledUpdate === true && video.ScheduleVideoUpdate) { + result.scheduledUpdate = { + updateAt: video.ScheduleVideoUpdate.updateAt, + privacy: video.ScheduleVideoUpdate.privacy || undefined + } + } + + if (add?.blacklistInfo === true) { + result.blacklisted = !!video.VideoBlacklist + result.blacklistedReason = + video.VideoBlacklist + ? video.VideoBlacklist.reason + : null + } + + if (add?.blockedOwner === true) { + result.blockedOwner = video.VideoChannel.Account.isBlocked() + + const server = video.VideoChannel.Account.Actor.Server as MServer + result.blockedServer = !!(server?.isBlocked()) + } + + if (add?.files === true) { + result.streamingPlaylists = streamingPlaylistsModelToFormattedJSON(video, video.VideoStreamingPlaylists) + result.files = videoFilesModelToFormattedJSON(video, video.VideoFiles) + } + + return result +} diff --git a/server/server/models/video/schedule-video-update.ts b/server/server/models/video/schedule-video-update.ts new file mode 100644 index 000000000..229610d87 --- /dev/null +++ b/server/server/models/video/schedule-video-update.ts @@ -0,0 +1,95 @@ +import { Op, Transaction } from 'sequelize' +import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { VideoPrivacy } from '@peertube/peertube-models' +import { MScheduleVideoUpdate, MScheduleVideoUpdateFormattable } from '@server/types/models/index.js' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { VideoModel } from './video.js' + +@Table({ + tableName: 'scheduleVideoUpdate', + indexes: [ + { + fields: [ 'videoId' ], + unique: true + }, + { + fields: [ 'updateAt' ] + } + ] +}) +export class ScheduleVideoUpdateModel extends Model>> { + + @AllowNull(false) + @Default(null) + @Column + updateAt: Date + + @AllowNull(true) + @Default(null) + @Column(DataType.INTEGER) + privacy: typeof VideoPrivacy.PUBLIC | typeof VideoPrivacy.UNLISTED | typeof VideoPrivacy.INTERNAL + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + Video: Awaited + + static areVideosToUpdate () { + const query = { + logging: false, + attributes: [ 'id' ], + where: { + updateAt: { + [Op.lte]: new Date() + } + } + } + + return ScheduleVideoUpdateModel.findOne(query) + .then(res => !!res) + } + + static listVideosToUpdate (transaction?: Transaction) { + const query = { + where: { + updateAt: { + [Op.lte]: new Date() + } + }, + transaction + } + + return ScheduleVideoUpdateModel.findAll(query) + } + + static deleteByVideoId (videoId: number, t: Transaction) { + const query = { + where: { + videoId + }, + transaction: t + } + + return ScheduleVideoUpdateModel.destroy(query) + } + + toFormattedJSON (this: MScheduleVideoUpdateFormattable) { + return { + updateAt: this.updateAt, + privacy: this.privacy || undefined + } + } +} diff --git a/server/server/models/video/sql/comment/video-comment-list-query-builder.ts b/server/server/models/video/sql/comment/video-comment-list-query-builder.ts new file mode 100644 index 000000000..0a9388fda --- /dev/null +++ b/server/server/models/video/sql/comment/video-comment-list-query-builder.ts @@ -0,0 +1,400 @@ +import { Model, Sequelize, Transaction } from 'sequelize' +import { AbstractRunQuery, ModelBuilder } from '@server/models/shared/index.js' +import { ActorImageType, VideoPrivacy } from '@peertube/peertube-models' +import { createSafeIn, getSort, parseRowCountResult } from '../../../shared/index.js' +import { VideoCommentTableAttributes } from './video-comment-table-attributes.js' + +export interface ListVideoCommentsOptions { + selectType: 'api' | 'feed' | 'comment-only' + + start?: number + count?: number + sort?: string + + videoId?: number + threadId?: number + accountId?: number + videoChannelId?: number + + blockerAccountIds?: number[] + + isThread?: boolean + notDeleted?: boolean + isLocal?: boolean + onLocalVideo?: boolean + onPublicVideo?: boolean + videoAccountOwnerId?: boolean + + search?: string + searchAccount?: string + searchVideo?: string + + includeReplyCounters?: boolean + + transaction?: Transaction +} + +export class VideoCommentListQueryBuilder extends AbstractRunQuery { + private readonly tableAttributes = new VideoCommentTableAttributes() + + private innerQuery: string + + private select = '' + private joins = '' + + private innerSelect = '' + private innerJoins = '' + private innerLateralJoins = '' + private innerWhere = '' + + private readonly built = { + cte: false, + accountJoin: false, + videoJoin: false, + videoChannelJoin: false, + avatarJoin: false + } + + constructor ( + protected readonly sequelize: Sequelize, + private readonly options: ListVideoCommentsOptions + ) { + super(sequelize) + + if (this.options.includeReplyCounters && !this.options.videoId) { + throw new Error('Cannot include reply counters without videoId') + } + } + + async listComments () { + this.buildListQuery() + + const results = await this.runQuery({ nest: true, transaction: this.options.transaction }) + const modelBuilder = new ModelBuilder(this.sequelize) + + return modelBuilder.createModels(results, 'VideoComment') + } + + async countComments () { + this.buildCountQuery() + + const result = await this.runQuery({ transaction: this.options.transaction }) + + return parseRowCountResult(result) + } + + // --------------------------------------------------------------------------- + + private buildListQuery () { + this.buildInnerListQuery() + this.buildListSelect() + + this.query = `${this.select} ` + + `FROM (${this.innerQuery}) AS "VideoCommentModel" ` + + `${this.joins} ` + + `${this.getOrder()}` + } + + private buildInnerListQuery () { + this.buildWhere() + this.buildInnerListSelect() + + this.innerQuery = `${this.innerSelect} ` + + `FROM "videoComment" AS "VideoCommentModel" ` + + `${this.innerJoins} ` + + `${this.innerLateralJoins} ` + + `${this.innerWhere} ` + + `${this.getOrder()} ` + + `${this.getInnerLimit()}` + } + + // --------------------------------------------------------------------------- + + private buildCountQuery () { + this.buildWhere() + + this.query = `SELECT COUNT(*) AS "total" ` + + `FROM "videoComment" AS "VideoCommentModel" ` + + `${this.innerJoins} ` + + `${this.innerWhere}` + } + + // --------------------------------------------------------------------------- + + private buildWhere () { + let where: string[] = [] + + if (this.options.videoId) { + this.replacements.videoId = this.options.videoId + + where.push('"VideoCommentModel"."videoId" = :videoId') + } + + if (this.options.threadId) { + this.replacements.threadId = this.options.threadId + + where.push('("VideoCommentModel"."id" = :threadId OR "VideoCommentModel"."originCommentId" = :threadId)') + } + + if (this.options.accountId) { + this.replacements.accountId = this.options.accountId + + where.push('"VideoCommentModel"."accountId" = :accountId') + } + + if (this.options.videoChannelId) { + this.buildVideoChannelJoin() + + this.replacements.videoChannelId = this.options.videoChannelId + + where.push('"Account->VideoChannel"."id" = :videoChannelId') + } + + if (this.options.blockerAccountIds) { + this.buildVideoChannelJoin() + + where = where.concat(this.getBlockWhere('VideoCommentModel', 'Video->VideoChannel')) + } + + if (this.options.isThread === true) { + where.push('"VideoCommentModel"."inReplyToCommentId" IS NULL') + } + + if (this.options.notDeleted === true) { + where.push('"VideoCommentModel"."deletedAt" IS NULL') + } + + if (this.options.isLocal === true) { + this.buildAccountJoin() + + where.push('"Account->Actor"."serverId" IS NULL') + } else if (this.options.isLocal === false) { + this.buildAccountJoin() + + where.push('"Account->Actor"."serverId" IS NOT NULL') + } + + if (this.options.onLocalVideo === true) { + this.buildVideoJoin() + + where.push('"Video"."remote" IS FALSE') + } else if (this.options.onLocalVideo === false) { + this.buildVideoJoin() + + where.push('"Video"."remote" IS TRUE') + } + + if (this.options.onPublicVideo === true) { + this.buildVideoJoin() + + where.push(`"Video"."privacy" = ${VideoPrivacy.PUBLIC}`) + } + + if (this.options.videoAccountOwnerId) { + this.buildVideoChannelJoin() + + this.replacements.videoAccountOwnerId = this.options.videoAccountOwnerId + + where.push(`"Video->VideoChannel"."accountId" = :videoAccountOwnerId`) + } + + if (this.options.search) { + this.buildVideoJoin() + this.buildAccountJoin() + + const escapedLikeSearch = this.sequelize.escape('%' + this.options.search + '%') + + where.push( + `(` + + `"VideoCommentModel"."text" ILIKE ${escapedLikeSearch} OR ` + + `"Account->Actor"."preferredUsername" ILIKE ${escapedLikeSearch} OR ` + + `"Account"."name" ILIKE ${escapedLikeSearch} OR ` + + `"Video"."name" ILIKE ${escapedLikeSearch} ` + + `)` + ) + } + + if (this.options.searchAccount) { + this.buildAccountJoin() + + const escapedLikeSearch = this.sequelize.escape('%' + this.options.searchAccount + '%') + + where.push( + `(` + + `"Account->Actor"."preferredUsername" ILIKE ${escapedLikeSearch} OR ` + + `"Account"."name" ILIKE ${escapedLikeSearch} ` + + `)` + ) + } + + if (this.options.searchVideo) { + this.buildVideoJoin() + + const escapedLikeSearch = this.sequelize.escape('%' + this.options.searchVideo + '%') + + where.push(`"Video"."name" ILIKE ${escapedLikeSearch}`) + } + + if (where.length !== 0) { + this.innerWhere = `WHERE ${where.join(' AND ')}` + } + } + + private buildAccountJoin () { + if (this.built.accountJoin) return + + this.innerJoins += ' LEFT JOIN "account" "Account" ON "Account"."id" = "VideoCommentModel"."accountId" ' + + 'LEFT JOIN "actor" "Account->Actor" ON "Account->Actor"."id" = "Account"."actorId" ' + + 'LEFT JOIN "server" "Account->Actor->Server" ON "Account->Actor"."serverId" = "Account->Actor->Server"."id" ' + + this.built.accountJoin = true + } + + private buildVideoJoin () { + if (this.built.videoJoin) return + + this.innerJoins += ' LEFT JOIN "video" "Video" ON "Video"."id" = "VideoCommentModel"."videoId" ' + + this.built.videoJoin = true + } + + private buildVideoChannelJoin () { + if (this.built.videoChannelJoin) return + + this.buildVideoJoin() + + this.innerJoins += ' LEFT JOIN "videoChannel" "Video->VideoChannel" ON "Video"."channelId" = "Video->VideoChannel"."id" ' + + this.built.videoChannelJoin = true + } + + private buildAvatarsJoin () { + if (this.options.selectType !== 'api' && this.options.selectType !== 'feed') return '' + if (this.built.avatarJoin) return + + this.joins += `LEFT JOIN "actorImage" "Account->Actor->Avatars" ` + + `ON "VideoCommentModel"."Account.Actor.id" = "Account->Actor->Avatars"."actorId" ` + + `AND "Account->Actor->Avatars"."type" = ${ActorImageType.AVATAR}` + + this.built.avatarJoin = true + } + + // --------------------------------------------------------------------------- + + private buildListSelect () { + const toSelect = [ '"VideoCommentModel".*' ] + + if (this.options.selectType === 'api' || this.options.selectType === 'feed') { + this.buildAvatarsJoin() + + toSelect.push(this.tableAttributes.getAvatarAttributes()) + } + + this.select = this.buildSelect(toSelect) + } + + private buildInnerListSelect () { + let toSelect = [ this.tableAttributes.getVideoCommentAttributes() ] + + if (this.options.selectType === 'api' || this.options.selectType === 'feed') { + this.buildAccountJoin() + this.buildVideoJoin() + + toSelect = toSelect.concat([ + this.tableAttributes.getVideoAttributes(), + this.tableAttributes.getAccountAttributes(), + this.tableAttributes.getActorAttributes(), + this.tableAttributes.getServerAttributes() + ]) + } + + if (this.options.includeReplyCounters === true) { + this.buildTotalRepliesSelect() + this.buildAuthorTotalRepliesSelect() + + toSelect.push('"totalRepliesFromVideoAuthor"."count" AS "totalRepliesFromVideoAuthor"') + toSelect.push('"totalReplies"."count" AS "totalReplies"') + } + + this.innerSelect = this.buildSelect(toSelect) + } + + // --------------------------------------------------------------------------- + + private getBlockWhere (commentTableName: string, channelTableName: string) { + const where: string[] = [] + + const blockerIdsString = createSafeIn( + this.sequelize, + this.options.blockerAccountIds, + [ `"${channelTableName}"."accountId"` ] + ) + + where.push( + `NOT EXISTS (` + + `SELECT 1 FROM "accountBlocklist" ` + + `WHERE "targetAccountId" = "${commentTableName}"."accountId" ` + + `AND "accountId" IN (${blockerIdsString})` + + `)` + ) + + where.push( + `NOT EXISTS (` + + `SELECT 1 FROM "account" ` + + `INNER JOIN "actor" ON account."actorId" = actor.id ` + + `INNER JOIN "serverBlocklist" ON "actor"."serverId" = "serverBlocklist"."targetServerId" ` + + `WHERE "account"."id" = "${commentTableName}"."accountId" ` + + `AND "serverBlocklist"."accountId" IN (${blockerIdsString})` + + `)` + ) + + return where + } + + // --------------------------------------------------------------------------- + + private buildTotalRepliesSelect () { + const blockWhereString = this.getBlockWhere('replies', 'videoChannel').join(' AND ') + + // Help the planner by providing videoId that should filter out many comments + this.replacements.videoId = this.options.videoId + + this.innerLateralJoins += `LEFT JOIN LATERAL (` + + `SELECT COUNT("replies"."id") AS "count" FROM "videoComment" AS "replies" ` + + `INNER JOIN "video" ON "video"."id" = "replies"."videoId" AND "replies"."videoId" = :videoId ` + + `LEFT JOIN "videoChannel" ON "video"."channelId" = "videoChannel"."id" ` + + `WHERE "replies"."originCommentId" = "VideoCommentModel"."id" ` + + `AND "deletedAt" IS NULL ` + + `AND ${blockWhereString} ` + + `) "totalReplies" ON TRUE ` + } + + private buildAuthorTotalRepliesSelect () { + // Help the planner by providing videoId that should filter out many comments + this.replacements.videoId = this.options.videoId + + this.innerLateralJoins += `LEFT JOIN LATERAL (` + + `SELECT COUNT("replies"."id") AS "count" FROM "videoComment" AS "replies" ` + + `INNER JOIN "video" ON "video"."id" = "replies"."videoId" AND "replies"."videoId" = :videoId ` + + `INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ` + + `WHERE "replies"."originCommentId" = "VideoCommentModel"."id" AND "replies"."accountId" = "videoChannel"."accountId"` + + `) "totalRepliesFromVideoAuthor" ON TRUE ` + } + + private getOrder () { + if (!this.options.sort) return '' + + const orders = getSort(this.options.sort) + + return 'ORDER BY ' + orders.map(o => `"${o[0]}" ${o[1]}`).join(', ') + } + + private getInnerLimit () { + if (!this.options.count) return '' + + this.replacements.limit = this.options.count + this.replacements.offset = this.options.start || 0 + + return `LIMIT :limit OFFSET :offset ` + } +} diff --git a/server/server/models/video/sql/comment/video-comment-table-attributes.ts b/server/server/models/video/sql/comment/video-comment-table-attributes.ts new file mode 100644 index 000000000..c7a8a9768 --- /dev/null +++ b/server/server/models/video/sql/comment/video-comment-table-attributes.ts @@ -0,0 +1,43 @@ +import { Memoize } from '@server/helpers/memoize.js' +import { AccountModel } from '@server/models/account/account.js' +import { ActorModel } from '@server/models/actor/actor.js' +import { ActorImageModel } from '@server/models/actor/actor-image.js' +import { ServerModel } from '@server/models/server/server.js' +import { VideoCommentModel } from '../../video-comment.js' + +export class VideoCommentTableAttributes { + + @Memoize() + getVideoCommentAttributes () { + return VideoCommentModel.getSQLAttributes('VideoCommentModel').join(', ') + } + + @Memoize() + getAccountAttributes () { + return AccountModel.getSQLAttributes('Account', 'Account.').join(', ') + } + + @Memoize() + getVideoAttributes () { + return [ + `"Video"."id" AS "Video.id"`, + `"Video"."uuid" AS "Video.uuid"`, + `"Video"."name" AS "Video.name"` + ].join(', ') + } + + @Memoize() + getActorAttributes () { + return ActorModel.getSQLAPIAttributes('Account->Actor', `Account.Actor.`).join(', ') + } + + @Memoize() + getServerAttributes () { + return ServerModel.getSQLAttributes('Account->Actor->Server', `Account.Actor.Server.`).join(', ') + } + + @Memoize() + getAvatarAttributes () { + return ActorImageModel.getSQLAttributes('Account->Actor->Avatars', 'Account.Actor.Avatars.').join(', ') + } +} diff --git a/server/server/models/video/sql/video/index.ts b/server/server/models/video/sql/video/index.ts new file mode 100644 index 000000000..003f59861 --- /dev/null +++ b/server/server/models/video/sql/video/index.ts @@ -0,0 +1,3 @@ +export * from './video-model-get-query-builder.js' +export * from './videos-id-list-query-builder.js' +export * from './videos-model-list-query-builder.js' diff --git a/server/server/models/video/sql/video/shared/abstract-video-query-builder.ts b/server/server/models/video/sql/video/shared/abstract-video-query-builder.ts new file mode 100644 index 000000000..1175c75eb --- /dev/null +++ b/server/server/models/video/sql/video/shared/abstract-video-query-builder.ts @@ -0,0 +1,340 @@ +import { Sequelize } from 'sequelize' +import validator from 'validator' +import { MUserAccountId } from '@server/types/models/index.js' +import { ActorImageType } from '@peertube/peertube-models' +import { AbstractRunQuery } from '../../../../shared/abstract-run-query.js' +import { createSafeIn } from '../../../../shared/index.js' +import { VideoTableAttributes } from './video-table-attributes.js' + +/** + * + * Abstract builder to create SQL query and fetch video models + * + */ + +export class AbstractVideoQueryBuilder extends AbstractRunQuery { + protected attributes: { [key: string]: string } = {} + + protected joins = '' + protected where: string + + protected tables: VideoTableAttributes + + constructor ( + protected readonly sequelize: Sequelize, + protected readonly mode: 'list' | 'get' + ) { + super(sequelize) + + this.tables = new VideoTableAttributes(this.mode) + } + + protected buildSelect () { + return 'SELECT ' + Object.keys(this.attributes).map(key => { + const value = this.attributes[key] + if (value) return `${key} AS ${value}` + + return key + }).join(', ') + } + + protected includeChannels () { + this.addJoin('INNER JOIN "videoChannel" AS "VideoChannel" ON "video"."channelId" = "VideoChannel"."id"') + this.addJoin('INNER JOIN "actor" AS "VideoChannel->Actor" ON "VideoChannel"."actorId" = "VideoChannel->Actor"."id"') + + this.addJoin( + 'LEFT OUTER JOIN "server" AS "VideoChannel->Actor->Server" ON "VideoChannel->Actor"."serverId" = "VideoChannel->Actor->Server"."id"' + ) + + this.addJoin( + 'LEFT OUTER JOIN "actorImage" AS "VideoChannel->Actor->Avatars" ' + + 'ON "VideoChannel->Actor"."id" = "VideoChannel->Actor->Avatars"."actorId" ' + + `AND "VideoChannel->Actor->Avatars"."type" = ${ActorImageType.AVATAR}` + ) + + this.attributes = { + ...this.attributes, + + ...this.buildAttributesObject('VideoChannel', this.tables.getChannelAttributes()), + ...this.buildActorInclude('VideoChannel->Actor'), + ...this.buildAvatarInclude('VideoChannel->Actor->Avatars'), + ...this.buildServerInclude('VideoChannel->Actor->Server') + } + } + + protected includeAccounts () { + this.addJoin('INNER JOIN "account" AS "VideoChannel->Account" ON "VideoChannel"."accountId" = "VideoChannel->Account"."id"') + this.addJoin( + 'INNER JOIN "actor" AS "VideoChannel->Account->Actor" ON "VideoChannel->Account"."actorId" = "VideoChannel->Account->Actor"."id"' + ) + + this.addJoin( + 'LEFT OUTER JOIN "server" AS "VideoChannel->Account->Actor->Server" ' + + 'ON "VideoChannel->Account->Actor"."serverId" = "VideoChannel->Account->Actor->Server"."id"' + ) + + this.addJoin( + 'LEFT OUTER JOIN "actorImage" AS "VideoChannel->Account->Actor->Avatars" ' + + 'ON "VideoChannel->Account"."actorId"= "VideoChannel->Account->Actor->Avatars"."actorId" ' + + `AND "VideoChannel->Account->Actor->Avatars"."type" = ${ActorImageType.AVATAR}` + ) + + this.attributes = { + ...this.attributes, + + ...this.buildAttributesObject('VideoChannel->Account', this.tables.getAccountAttributes()), + ...this.buildActorInclude('VideoChannel->Account->Actor'), + ...this.buildAvatarInclude('VideoChannel->Account->Actor->Avatars'), + ...this.buildServerInclude('VideoChannel->Account->Actor->Server') + } + } + + protected includeOwnerUser () { + this.addJoin('INNER JOIN "videoChannel" AS "VideoChannel" ON "video"."channelId" = "VideoChannel"."id"') + this.addJoin('INNER JOIN "account" AS "VideoChannel->Account" ON "VideoChannel"."accountId" = "VideoChannel->Account"."id"') + + this.attributes = { + ...this.attributes, + + ...this.buildAttributesObject('VideoChannel', this.tables.getChannelAttributes()), + ...this.buildAttributesObject('VideoChannel->Account', this.tables.getUserAccountAttributes()) + } + } + + protected includeThumbnails () { + this.addJoin('LEFT OUTER JOIN "thumbnail" AS "Thumbnails" ON "video"."id" = "Thumbnails"."videoId"') + + this.attributes = { + ...this.attributes, + + ...this.buildAttributesObject('Thumbnails', this.tables.getThumbnailAttributes()) + } + } + + protected includeWebVideoFiles () { + this.addJoin('LEFT JOIN "videoFile" AS "VideoFiles" ON "VideoFiles"."videoId" = "video"."id"') + + this.attributes = { + ...this.attributes, + + ...this.buildAttributesObject('VideoFiles', this.tables.getFileAttributes()) + } + } + + protected includeStreamingPlaylistFiles () { + this.addJoin( + 'LEFT JOIN "videoStreamingPlaylist" AS "VideoStreamingPlaylists" ON "VideoStreamingPlaylists"."videoId" = "video"."id"' + ) + + this.addJoin( + 'LEFT JOIN "videoFile" AS "VideoStreamingPlaylists->VideoFiles" ' + + 'ON "VideoStreamingPlaylists->VideoFiles"."videoStreamingPlaylistId" = "VideoStreamingPlaylists"."id"' + ) + + this.attributes = { + ...this.attributes, + + ...this.buildAttributesObject('VideoStreamingPlaylists', this.tables.getStreamingPlaylistAttributes()), + ...this.buildAttributesObject('VideoStreamingPlaylists->VideoFiles', this.tables.getFileAttributes()) + } + } + + protected includeUserHistory (userId: number) { + this.addJoin( + 'LEFT OUTER JOIN "userVideoHistory" ' + + 'ON "video"."id" = "userVideoHistory"."videoId" AND "userVideoHistory"."userId" = :userVideoHistoryId' + ) + + this.replacements.userVideoHistoryId = userId + + this.attributes = { + ...this.attributes, + + ...this.buildAttributesObject('userVideoHistory', this.tables.getUserHistoryAttributes()) + } + } + + protected includePlaylist (playlistId: number) { + this.addJoin( + 'INNER JOIN "videoPlaylistElement" as "VideoPlaylistElement" ON "videoPlaylistElement"."videoId" = "video"."id" ' + + 'AND "VideoPlaylistElement"."videoPlaylistId" = :videoPlaylistId' + ) + + this.replacements.videoPlaylistId = playlistId + + this.attributes = { + ...this.attributes, + + ...this.buildAttributesObject('VideoPlaylistElement', this.tables.getPlaylistAttributes()) + } + } + + protected includeTags () { + this.addJoin( + 'LEFT OUTER JOIN (' + + '"videoTag" AS "Tags->VideoTagModel" INNER JOIN "tag" AS "Tags" ON "Tags"."id" = "Tags->VideoTagModel"."tagId"' + + ') ' + + 'ON "video"."id" = "Tags->VideoTagModel"."videoId"' + ) + + this.attributes = { + ...this.attributes, + + ...this.buildAttributesObject('Tags', this.tables.getTagAttributes()), + ...this.buildAttributesObject('Tags->VideoTagModel', this.tables.getVideoTagAttributes()) + } + } + + protected includeBlacklisted () { + this.addJoin( + 'LEFT OUTER JOIN "videoBlacklist" AS "VideoBlacklist" ON "video"."id" = "VideoBlacklist"."videoId"' + ) + + this.attributes = { + ...this.attributes, + + ...this.buildAttributesObject('VideoBlacklist', this.tables.getBlacklistedAttributes()) + } + } + + protected includeBlockedOwnerAndServer (serverAccountId: number, user?: MUserAccountId) { + const blockerIds = [ serverAccountId ] + if (user) blockerIds.push(user.Account.id) + + const inClause = createSafeIn(this.sequelize, blockerIds) + + this.addJoin( + 'LEFT JOIN "accountBlocklist" AS "VideoChannel->Account->AccountBlocklist" ' + + 'ON "VideoChannel->Account"."id" = "VideoChannel->Account->AccountBlocklist"."targetAccountId" ' + + 'AND "VideoChannel->Account->AccountBlocklist"."accountId" IN (' + inClause + ')' + ) + + this.addJoin( + 'LEFT JOIN "serverBlocklist" AS "VideoChannel->Account->Actor->Server->ServerBlocklist" ' + + 'ON "VideoChannel->Account->Actor->Server->ServerBlocklist"."targetServerId" = "VideoChannel->Account->Actor"."serverId" ' + + 'AND "VideoChannel->Account->Actor->Server->ServerBlocklist"."accountId" IN (' + inClause + ') ' + ) + + this.attributes = { + ...this.attributes, + + ...this.buildAttributesObject('VideoChannel->Account->AccountBlocklist', this.tables.getBlocklistAttributes()), + ...this.buildAttributesObject('VideoChannel->Account->Actor->Server->ServerBlocklist', this.tables.getBlocklistAttributes()) + } + } + + protected includeScheduleUpdate () { + this.addJoin( + 'LEFT OUTER JOIN "scheduleVideoUpdate" AS "ScheduleVideoUpdate" ON "video"."id" = "ScheduleVideoUpdate"."videoId"' + ) + + this.attributes = { + ...this.attributes, + + ...this.buildAttributesObject('ScheduleVideoUpdate', this.tables.getScheduleUpdateAttributes()) + } + } + + protected includeLive () { + this.addJoin( + 'LEFT OUTER JOIN "videoLive" AS "VideoLive" ON "video"."id" = "VideoLive"."videoId"' + ) + + this.attributes = { + ...this.attributes, + + ...this.buildAttributesObject('VideoLive', this.tables.getLiveAttributes()) + } + } + + protected includeTrackers () { + this.addJoin( + 'LEFT OUTER JOIN (' + + '"videoTracker" AS "Trackers->VideoTrackerModel" ' + + 'INNER JOIN "tracker" AS "Trackers" ON "Trackers"."id" = "Trackers->VideoTrackerModel"."trackerId"' + + ') ON "video"."id" = "Trackers->VideoTrackerModel"."videoId"' + ) + + this.attributes = { + ...this.attributes, + + ...this.buildAttributesObject('Trackers', this.tables.getTrackerAttributes()), + ...this.buildAttributesObject('Trackers->VideoTrackerModel', this.tables.getVideoTrackerAttributes()) + } + } + + protected includeWebVideoRedundancies () { + this.addJoin( + 'LEFT OUTER JOIN "videoRedundancy" AS "VideoFiles->RedundancyVideos" ON ' + + '"VideoFiles"."id" = "VideoFiles->RedundancyVideos"."videoFileId"' + ) + + this.attributes = { + ...this.attributes, + + ...this.buildAttributesObject('VideoFiles->RedundancyVideos', this.tables.getRedundancyAttributes()) + } + } + + protected includeStreamingPlaylistRedundancies () { + this.addJoin( + 'LEFT OUTER JOIN "videoRedundancy" AS "VideoStreamingPlaylists->RedundancyVideos" ' + + 'ON "VideoStreamingPlaylists"."id" = "VideoStreamingPlaylists->RedundancyVideos"."videoStreamingPlaylistId"' + ) + + this.attributes = { + ...this.attributes, + + ...this.buildAttributesObject('VideoStreamingPlaylists->RedundancyVideos', this.tables.getRedundancyAttributes()) + } + } + + protected buildActorInclude (prefixKey: string) { + return this.buildAttributesObject(prefixKey, this.tables.getActorAttributes()) + } + + protected buildAvatarInclude (prefixKey: string) { + return this.buildAttributesObject(prefixKey, this.tables.getAvatarAttributes()) + } + + protected buildServerInclude (prefixKey: string) { + return this.buildAttributesObject(prefixKey, this.tables.getServerAttributes()) + } + + protected buildAttributesObject (prefixKey: string, attributeKeys: string[]) { + const result: { [id: string]: string } = {} + + const prefixValue = prefixKey.replace(/->/g, '.') + + for (const attribute of attributeKeys) { + result[`"${prefixKey}"."${attribute}"`] = `"${prefixValue}.${attribute}"` + } + + return result + } + + protected whereId (options: { ids?: number[], id?: string | number, url?: string }) { + if (options.ids) { + this.where = `WHERE "video"."id" IN (${createSafeIn(this.sequelize, options.ids)})` + return + } + + if (options.url) { + this.where = 'WHERE "video"."url" = :videoUrl' + this.replacements.videoUrl = options.url + return + } + + if (validator.default.isInt('' + options.id)) { + this.where = 'WHERE "video".id = :videoId' + } else { + this.where = 'WHERE uuid = :videoId' + } + + this.replacements.videoId = options.id + } + + protected addJoin (join: string) { + this.joins += join + ' ' + } +} diff --git a/server/server/models/video/sql/video/shared/video-file-query-builder.ts b/server/server/models/video/sql/video/shared/video-file-query-builder.ts new file mode 100644 index 000000000..b7d3e06d9 --- /dev/null +++ b/server/server/models/video/sql/video/shared/video-file-query-builder.ts @@ -0,0 +1,75 @@ +import { Sequelize, Transaction } from 'sequelize' +import { AbstractVideoQueryBuilder } from './abstract-video-query-builder.js' + +export type FileQueryOptions = { + id?: string | number + url?: string + + includeRedundancy: boolean + + transaction?: Transaction + + logging?: boolean +} + +/** + * + * Fetch files (web videos and streaming playlist) according to a video + * + */ + +export class VideoFileQueryBuilder extends AbstractVideoQueryBuilder { + protected attributes: { [key: string]: string } + + constructor (protected readonly sequelize: Sequelize) { + super(sequelize, 'get') + } + + queryWebVideos (options: FileQueryOptions) { + this.buildWebVideoFilesQuery(options) + + return this.runQuery(options) + } + + queryStreamingPlaylistVideos (options: FileQueryOptions) { + this.buildVideoStreamingPlaylistFilesQuery(options) + + return this.runQuery(options) + } + + private buildWebVideoFilesQuery (options: FileQueryOptions) { + this.attributes = { + '"video"."id"': '' + } + + this.includeWebVideoFiles() + + if (options.includeRedundancy) { + this.includeWebVideoRedundancies() + } + + this.whereId(options) + + this.query = this.buildQuery() + } + + private buildVideoStreamingPlaylistFilesQuery (options: FileQueryOptions) { + this.attributes = { + '"video"."id"': '' + } + + this.includeStreamingPlaylistFiles() + + if (options.includeRedundancy) { + this.includeStreamingPlaylistRedundancies() + } + + this.whereId(options) + + this.query = this.buildQuery() + } + + private buildQuery () { + return `${this.buildSelect()} FROM "video" ${this.joins} ${this.where}` + } +} diff --git a/server/server/models/video/sql/video/shared/video-model-builder.ts b/server/server/models/video/sql/video/shared/video-model-builder.ts new file mode 100644 index 000000000..d5746b41a --- /dev/null +++ b/server/server/models/video/sql/video/shared/video-model-builder.ts @@ -0,0 +1,407 @@ +import { VideoInclude, VideoIncludeType } from '@peertube/peertube-models' +import { AccountBlocklistModel } from '@server/models/account/account-blocklist.js' +import { AccountModel } from '@server/models/account/account.js' +import { ActorImageModel } from '@server/models/actor/actor-image.js' +import { ActorModel } from '@server/models/actor/actor.js' +import { VideoRedundancyModel } from '@server/models/redundancy/video-redundancy.js' +import { ServerBlocklistModel } from '@server/models/server/server-blocklist.js' +import { ServerModel } from '@server/models/server/server.js' +import { TrackerModel } from '@server/models/server/tracker.js' +import { UserVideoHistoryModel } from '@server/models/user/user-video-history.js' +import { ScheduleVideoUpdateModel } from '../../../schedule-video-update.js' +import { TagModel } from '../../../tag.js' +import { ThumbnailModel } from '../../../thumbnail.js' +import { VideoBlacklistModel } from '../../../video-blacklist.js' +import { VideoChannelModel } from '../../../video-channel.js' +import { VideoFileModel } from '../../../video-file.js' +import { VideoLiveModel } from '../../../video-live.js' +import { VideoStreamingPlaylistModel } from '../../../video-streaming-playlist.js' +import { VideoModel } from '../../../video.js' +import { VideoTableAttributes } from './video-table-attributes.js' + +type SQLRow = { [id: string]: string | number } + +/** + * + * Build video models from SQL rows + * + */ + +export class VideoModelBuilder { + private videosMemo: { [ id: number ]: VideoModel } + private videoStreamingPlaylistMemo: { [ id: number ]: VideoStreamingPlaylistModel } + private videoFileMemo: { [ id: number ]: VideoFileModel } + + private thumbnailsDone: Set + private actorImagesDone: Set + private historyDone: Set + private blacklistDone: Set + private accountBlocklistDone: Set + private serverBlocklistDone: Set + private liveDone: Set + private redundancyDone: Set + private scheduleVideoUpdateDone: Set + + private trackersDone: Set + private tagsDone: Set + + private videos: VideoModel[] + + private readonly buildOpts = { raw: true, isNewRecord: false } + + constructor ( + private readonly mode: 'get' | 'list', + private readonly tables: VideoTableAttributes + ) { + + } + + buildVideosFromRows (options: { + rows: SQLRow[] + include?: VideoIncludeType + rowsWebVideoFiles?: SQLRow[] + rowsStreamingPlaylist?: SQLRow[] + }) { + const { rows, rowsWebVideoFiles, rowsStreamingPlaylist, include } = options + + this.reinit() + + for (const row of rows) { + this.buildVideoAndAccount(row) + + const videoModel = this.videosMemo[row.id as number] + + this.setUserHistory(row, videoModel) + this.addThumbnail(row, videoModel) + + const channelActor = videoModel.VideoChannel?.Actor + if (channelActor) { + this.addActorAvatar(row, 'VideoChannel.Actor', channelActor) + } + + const accountActor = videoModel.VideoChannel?.Account?.Actor + if (accountActor) { + this.addActorAvatar(row, 'VideoChannel.Account.Actor', accountActor) + } + + if (!rowsWebVideoFiles) { + this.addWebVideoFile(row, videoModel) + } + + if (!rowsStreamingPlaylist) { + this.addStreamingPlaylist(row, videoModel) + this.addStreamingPlaylistFile(row) + } + + if (this.mode === 'get') { + this.addTag(row, videoModel) + this.addTracker(row, videoModel) + this.setBlacklisted(row, videoModel) + this.setScheduleVideoUpdate(row, videoModel) + this.setLive(row, videoModel) + } else { + if (include & VideoInclude.BLACKLISTED) { + this.setBlacklisted(row, videoModel) + } + + if (include & VideoInclude.BLOCKED_OWNER) { + this.setBlockedOwner(row, videoModel) + this.setBlockedServer(row, videoModel) + } + } + } + + this.grabSeparateWebVideoFiles(rowsWebVideoFiles) + this.grabSeparateStreamingPlaylistFiles(rowsStreamingPlaylist) + + return this.videos + } + + private reinit () { + this.videosMemo = {} + this.videoStreamingPlaylistMemo = {} + this.videoFileMemo = {} + + this.thumbnailsDone = new Set() + this.actorImagesDone = new Set() + this.historyDone = new Set() + this.blacklistDone = new Set() + this.liveDone = new Set() + this.redundancyDone = new Set() + this.scheduleVideoUpdateDone = new Set() + + this.accountBlocklistDone = new Set() + this.serverBlocklistDone = new Set() + + this.trackersDone = new Set() + this.tagsDone = new Set() + + this.videos = [] + } + + private grabSeparateWebVideoFiles (rowsWebVideoFiles?: SQLRow[]) { + if (!rowsWebVideoFiles) return + + for (const row of rowsWebVideoFiles) { + const id = row['VideoFiles.id'] + if (!id) continue + + const videoModel = this.videosMemo[row.id] + this.addWebVideoFile(row, videoModel) + this.addRedundancy(row, 'VideoFiles', this.videoFileMemo[id]) + } + } + + private grabSeparateStreamingPlaylistFiles (rowsStreamingPlaylist?: SQLRow[]) { + if (!rowsStreamingPlaylist) return + + for (const row of rowsStreamingPlaylist) { + const id = row['VideoStreamingPlaylists.id'] + if (!id) continue + + const videoModel = this.videosMemo[row.id] + + this.addStreamingPlaylist(row, videoModel) + this.addStreamingPlaylistFile(row) + this.addRedundancy(row, 'VideoStreamingPlaylists', this.videoStreamingPlaylistMemo[id]) + } + } + + private buildVideoAndAccount (row: SQLRow) { + if (this.videosMemo[row.id]) return + + const videoModel = new VideoModel(this.grab(row, this.tables.getVideoAttributes(), ''), this.buildOpts) + + videoModel.UserVideoHistories = [] + videoModel.Thumbnails = [] + videoModel.VideoFiles = [] + videoModel.VideoStreamingPlaylists = [] + videoModel.Tags = [] + videoModel.Trackers = [] + + this.buildAccount(row, videoModel) + + this.videosMemo[row.id] = videoModel + + // Keep rows order + this.videos.push(videoModel) + } + + private buildAccount (row: SQLRow, videoModel: VideoModel) { + const id = row['VideoChannel.Account.id'] + if (!id) return + + const channelModel = new VideoChannelModel(this.grab(row, this.tables.getChannelAttributes(), 'VideoChannel'), this.buildOpts) + channelModel.Actor = this.buildActor(row, 'VideoChannel') + + const accountModel = new AccountModel(this.grab(row, this.tables.getAccountAttributes(), 'VideoChannel.Account'), this.buildOpts) + accountModel.Actor = this.buildActor(row, 'VideoChannel.Account') + + accountModel.BlockedBy = [] + + channelModel.Account = accountModel + + videoModel.VideoChannel = channelModel + } + + private buildActor (row: SQLRow, prefix: string) { + const actorPrefix = `${prefix}.Actor` + const serverPrefix = `${actorPrefix}.Server` + + const serverModel = row[`${serverPrefix}.id`] !== null + ? new ServerModel(this.grab(row, this.tables.getServerAttributes(), serverPrefix), this.buildOpts) + : null + + if (serverModel) serverModel.BlockedBy = [] + + const actorModel = new ActorModel(this.grab(row, this.tables.getActorAttributes(), actorPrefix), this.buildOpts) + actorModel.Server = serverModel + actorModel.Avatars = [] + + return actorModel + } + + private setUserHistory (row: SQLRow, videoModel: VideoModel) { + const id = row['userVideoHistory.id'] + if (!id || this.historyDone.has(id)) return + + const attributes = this.grab(row, this.tables.getUserHistoryAttributes(), 'userVideoHistory') + const historyModel = new UserVideoHistoryModel(attributes, this.buildOpts) + videoModel.UserVideoHistories.push(historyModel) + + this.historyDone.add(id) + } + + private addActorAvatar (row: SQLRow, actorPrefix: string, actor: ActorModel) { + const avatarPrefix = `${actorPrefix}.Avatars` + const id = row[`${avatarPrefix}.id`] + const key = `${row.id}${id}` + + if (!id || this.actorImagesDone.has(key)) return + + const attributes = this.grab(row, this.tables.getAvatarAttributes(), avatarPrefix) + const avatarModel = new ActorImageModel(attributes, this.buildOpts) + actor.Avatars.push(avatarModel) + + this.actorImagesDone.add(key) + } + + private addThumbnail (row: SQLRow, videoModel: VideoModel) { + const id = row['Thumbnails.id'] + if (!id || this.thumbnailsDone.has(id)) return + + const attributes = this.grab(row, this.tables.getThumbnailAttributes(), 'Thumbnails') + const thumbnailModel = new ThumbnailModel(attributes, this.buildOpts) + videoModel.Thumbnails.push(thumbnailModel) + + this.thumbnailsDone.add(id) + } + + private addWebVideoFile (row: SQLRow, videoModel: VideoModel) { + const id = row['VideoFiles.id'] + if (!id || this.videoFileMemo[id]) return + + const attributes = this.grab(row, this.tables.getFileAttributes(), 'VideoFiles') + const videoFileModel = new VideoFileModel(attributes, this.buildOpts) + videoModel.VideoFiles.push(videoFileModel) + + this.videoFileMemo[id] = videoFileModel + } + + private addStreamingPlaylist (row: SQLRow, videoModel: VideoModel) { + const id = row['VideoStreamingPlaylists.id'] + if (!id || this.videoStreamingPlaylistMemo[id]) return + + const attributes = this.grab(row, this.tables.getStreamingPlaylistAttributes(), 'VideoStreamingPlaylists') + const streamingPlaylist = new VideoStreamingPlaylistModel(attributes, this.buildOpts) + streamingPlaylist.VideoFiles = [] + + videoModel.VideoStreamingPlaylists.push(streamingPlaylist) + + this.videoStreamingPlaylistMemo[id] = streamingPlaylist + } + + private addStreamingPlaylistFile (row: SQLRow) { + const id = row['VideoStreamingPlaylists.VideoFiles.id'] + if (!id || this.videoFileMemo[id]) return + + const streamingPlaylist = this.videoStreamingPlaylistMemo[row['VideoStreamingPlaylists.id']] + + const attributes = this.grab(row, this.tables.getFileAttributes(), 'VideoStreamingPlaylists.VideoFiles') + const videoFileModel = new VideoFileModel(attributes, this.buildOpts) + streamingPlaylist.VideoFiles.push(videoFileModel) + + this.videoFileMemo[id] = videoFileModel + } + + private addRedundancy (row: SQLRow, prefix: string, to: VideoFileModel | VideoStreamingPlaylistModel) { + if (!to.RedundancyVideos) to.RedundancyVideos = [] + + const redundancyPrefix = `${prefix}.RedundancyVideos` + const id = row[`${redundancyPrefix}.id`] + + if (!id || this.redundancyDone.has(id)) return + + const attributes = this.grab(row, this.tables.getRedundancyAttributes(), redundancyPrefix) + const redundancyModel = new VideoRedundancyModel(attributes, this.buildOpts) + to.RedundancyVideos.push(redundancyModel) + + this.redundancyDone.add(id) + } + + private addTag (row: SQLRow, videoModel: VideoModel) { + if (!row['Tags.name']) return + + const key = `${row['Tags.VideoTagModel.videoId']}-${row['Tags.VideoTagModel.tagId']}` + if (this.tagsDone.has(key)) return + + const attributes = this.grab(row, this.tables.getTagAttributes(), 'Tags') + const tagModel = new TagModel(attributes, this.buildOpts) + videoModel.Tags.push(tagModel) + + this.tagsDone.add(key) + } + + private addTracker (row: SQLRow, videoModel: VideoModel) { + if (!row['Trackers.id']) return + + const key = `${row['Trackers.VideoTrackerModel.videoId']}-${row['Trackers.VideoTrackerModel.trackerId']}` + if (this.trackersDone.has(key)) return + + const attributes = this.grab(row, this.tables.getTrackerAttributes(), 'Trackers') + const trackerModel = new TrackerModel(attributes, this.buildOpts) + videoModel.Trackers.push(trackerModel) + + this.trackersDone.add(key) + } + + private setBlacklisted (row: SQLRow, videoModel: VideoModel) { + const id = row['VideoBlacklist.id'] + if (!id || this.blacklistDone.has(id)) return + + const attributes = this.grab(row, this.tables.getBlacklistedAttributes(), 'VideoBlacklist') + videoModel.VideoBlacklist = new VideoBlacklistModel(attributes, this.buildOpts) + + this.blacklistDone.add(id) + } + + private setBlockedOwner (row: SQLRow, videoModel: VideoModel) { + const id = row['VideoChannel.Account.AccountBlocklist.id'] + if (!id) return + + const key = `${videoModel.id}-${id}` + if (this.accountBlocklistDone.has(key)) return + + const attributes = this.grab(row, this.tables.getBlocklistAttributes(), 'VideoChannel.Account.AccountBlocklist') + videoModel.VideoChannel.Account.BlockedBy.push(new AccountBlocklistModel(attributes, this.buildOpts)) + + this.accountBlocklistDone.add(key) + } + + private setBlockedServer (row: SQLRow, videoModel: VideoModel) { + const id = row['VideoChannel.Account.Actor.Server.ServerBlocklist.id'] + if (!id || this.serverBlocklistDone.has(id)) return + + const key = `${videoModel.id}-${id}` + if (this.serverBlocklistDone.has(key)) return + + const attributes = this.grab(row, this.tables.getBlocklistAttributes(), 'VideoChannel.Account.Actor.Server.ServerBlocklist') + videoModel.VideoChannel.Account.Actor.Server.BlockedBy.push(new ServerBlocklistModel(attributes, this.buildOpts)) + + this.serverBlocklistDone.add(key) + } + + private setScheduleVideoUpdate (row: SQLRow, videoModel: VideoModel) { + const id = row['ScheduleVideoUpdate.id'] + if (!id || this.scheduleVideoUpdateDone.has(id)) return + + const attributes = this.grab(row, this.tables.getScheduleUpdateAttributes(), 'ScheduleVideoUpdate') + videoModel.ScheduleVideoUpdate = new ScheduleVideoUpdateModel(attributes, this.buildOpts) + + this.scheduleVideoUpdateDone.add(id) + } + + private setLive (row: SQLRow, videoModel: VideoModel) { + const id = row['VideoLive.id'] + if (!id || this.liveDone.has(id)) return + + const attributes = this.grab(row, this.tables.getLiveAttributes(), 'VideoLive') + videoModel.VideoLive = new VideoLiveModel(attributes, this.buildOpts) + + this.liveDone.add(id) + } + + private grab (row: SQLRow, attributes: string[], prefix: string) { + const result: { [ id: string ]: string | number } = {} + + for (const a of attributes) { + const key = prefix + ? prefix + '.' + a + : a + + result[a] = row[key] + } + + return result + } +} diff --git a/server/models/video/sql/video/shared/video-table-attributes.ts b/server/server/models/video/sql/video/shared/video-table-attributes.ts similarity index 100% rename from server/models/video/sql/video/shared/video-table-attributes.ts rename to server/server/models/video/sql/video/shared/video-table-attributes.ts diff --git a/server/server/models/video/sql/video/video-model-get-query-builder.ts b/server/server/models/video/sql/video/video-model-get-query-builder.ts new file mode 100644 index 000000000..1d55e3e93 --- /dev/null +++ b/server/server/models/video/sql/video/video-model-get-query-builder.ts @@ -0,0 +1,189 @@ +import { Sequelize, Transaction } from 'sequelize' +import { pick } from '@peertube/peertube-core-utils' +import { AbstractVideoQueryBuilder } from './shared/abstract-video-query-builder.js' +import { VideoFileQueryBuilder } from './shared/video-file-query-builder.js' +import { VideoModelBuilder } from './shared/video-model-builder.js' +import { VideoTableAttributes } from './shared/video-table-attributes.js' + +/** + * + * Build a GET SQL query, fetch rows and create the video model + * + */ + +export type GetType = + 'api' | + 'full' | + 'account-blacklist-files' | + 'all-files' | + 'thumbnails' | + 'thumbnails-blacklist' | + 'id' | + 'blacklist-rights' + +export type BuildVideoGetQueryOptions = { + id?: number | string + url?: string + + type: GetType + + userId?: number + transaction?: Transaction + + logging?: boolean +} + +export class VideoModelGetQueryBuilder { + videoQueryBuilder: VideosModelGetQuerySubBuilder + webVideoFilesQueryBuilder: VideoFileQueryBuilder + streamingPlaylistFilesQueryBuilder: VideoFileQueryBuilder + + private readonly videoModelBuilder: VideoModelBuilder + + private static readonly videoFilesInclude = new Set([ 'api', 'full', 'account-blacklist-files', 'all-files' ]) + + constructor (protected readonly sequelize: Sequelize) { + this.videoQueryBuilder = new VideosModelGetQuerySubBuilder(sequelize) + this.webVideoFilesQueryBuilder = new VideoFileQueryBuilder(sequelize) + this.streamingPlaylistFilesQueryBuilder = new VideoFileQueryBuilder(sequelize) + + this.videoModelBuilder = new VideoModelBuilder('get', new VideoTableAttributes('get')) + } + + async queryVideo (options: BuildVideoGetQueryOptions) { + const fileQueryOptions = { + ...pick(options, [ 'id', 'url', 'transaction', 'logging' ]), + + includeRedundancy: this.shouldIncludeRedundancies(options) + } + + const [ videoRows, webVideoFilesRows, streamingPlaylistFilesRows ] = await Promise.all([ + this.videoQueryBuilder.queryVideos(options), + + VideoModelGetQueryBuilder.videoFilesInclude.has(options.type) + ? this.webVideoFilesQueryBuilder.queryWebVideos(fileQueryOptions) + : Promise.resolve(undefined), + + VideoModelGetQueryBuilder.videoFilesInclude.has(options.type) + ? this.streamingPlaylistFilesQueryBuilder.queryStreamingPlaylistVideos(fileQueryOptions) + : Promise.resolve(undefined) + ]) + + const videos = this.videoModelBuilder.buildVideosFromRows({ + rows: videoRows, + rowsWebVideoFiles: webVideoFilesRows, + rowsStreamingPlaylist: streamingPlaylistFilesRows + }) + + if (videos.length > 1) { + throw new Error('Video results is more than 1') + } + + if (videos.length === 0) return null + + return videos[0] + } + + private shouldIncludeRedundancies (options: BuildVideoGetQueryOptions) { + return options.type === 'api' + } +} + +export class VideosModelGetQuerySubBuilder extends AbstractVideoQueryBuilder { + protected attributes: { [key: string]: string } + + protected webVideoFilesQuery: string + protected streamingPlaylistFilesQuery: string + + private static readonly trackersInclude = new Set([ 'api' ]) + private static readonly liveInclude = new Set([ 'api', 'full' ]) + private static readonly scheduleUpdateInclude = new Set([ 'api', 'full' ]) + private static readonly tagsInclude = new Set([ 'api', 'full' ]) + private static readonly userHistoryInclude = new Set([ 'api', 'full' ]) + private static readonly accountInclude = new Set([ 'api', 'full', 'account-blacklist-files' ]) + private static readonly ownerUserInclude = new Set([ 'blacklist-rights' ]) + + private static readonly blacklistedInclude = new Set([ + 'api', + 'full', + 'account-blacklist-files', + 'thumbnails-blacklist', + 'blacklist-rights' + ]) + + private static readonly thumbnailsInclude = new Set([ + 'api', + 'full', + 'account-blacklist-files', + 'all-files', + 'thumbnails', + 'thumbnails-blacklist' + ]) + + constructor (protected readonly sequelize: Sequelize) { + super(sequelize, 'get') + } + + queryVideos (options: BuildVideoGetQueryOptions) { + this.buildMainGetQuery(options) + + return this.runQuery(options) + } + + private buildMainGetQuery (options: BuildVideoGetQueryOptions) { + this.attributes = { + '"video".*': '' + } + + if (VideosModelGetQuerySubBuilder.thumbnailsInclude.has(options.type)) { + this.includeThumbnails() + } + + if (VideosModelGetQuerySubBuilder.blacklistedInclude.has(options.type)) { + this.includeBlacklisted() + } + + if (VideosModelGetQuerySubBuilder.accountInclude.has(options.type)) { + this.includeChannels() + this.includeAccounts() + } + + if (VideosModelGetQuerySubBuilder.tagsInclude.has(options.type)) { + this.includeTags() + } + + if (VideosModelGetQuerySubBuilder.scheduleUpdateInclude.has(options.type)) { + this.includeScheduleUpdate() + } + + if (VideosModelGetQuerySubBuilder.liveInclude.has(options.type)) { + this.includeLive() + } + + if (options.userId && VideosModelGetQuerySubBuilder.userHistoryInclude.has(options.type)) { + this.includeUserHistory(options.userId) + } + + if (VideosModelGetQuerySubBuilder.ownerUserInclude.has(options.type)) { + this.includeOwnerUser() + } + + if (VideosModelGetQuerySubBuilder.trackersInclude.has(options.type)) { + this.includeTrackers() + } + + this.whereId(options) + + this.query = this.buildQuery(options) + } + + private buildQuery (options: BuildVideoGetQueryOptions) { + const order = VideosModelGetQuerySubBuilder.tagsInclude.has(options.type) + ? 'ORDER BY "Tags"."name" ASC' + : '' + + const from = `SELECT * FROM "video" ${this.where} LIMIT 1` + + return `${this.buildSelect()} FROM (${from}) AS "video" ${this.joins} ${order}` + } +} diff --git a/server/server/models/video/sql/video/videos-id-list-query-builder.ts b/server/server/models/video/sql/video/videos-id-list-query-builder.ts new file mode 100644 index 000000000..090eaddbc --- /dev/null +++ b/server/server/models/video/sql/video/videos-id-list-query-builder.ts @@ -0,0 +1,728 @@ +import { Sequelize, Transaction } from 'sequelize' +import validator from 'validator' +import { forceNumber } from '@peertube/peertube-core-utils' +import { VideoInclude, VideoIncludeType, VideoPrivacy, VideoPrivacyType, VideoState } from '@peertube/peertube-models' +import { exists } from '@server/helpers/custom-validators/misc.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { buildSortDirectionAndField } from '@server/models/shared/index.js' +import { MUserAccountId, MUserId } from '@server/types/models/index.js' +import { AbstractRunQuery } from '../../../shared/abstract-run-query.js' +import { createSafeIn, parseRowCountResult } from '../../../shared/index.js' + +/** + * + * Build videos list SQL query to fetch rows + * + */ + +export type DisplayOnlyForFollowerOptions = { + actorId: number + orLocalVideos: boolean +} + +export type BuildVideosListQueryOptions = { + attributes?: string[] + + serverAccountIdForBlock: number + + displayOnlyForFollower: DisplayOnlyForFollowerOptions + + count: number + start: number + sort: string + + nsfw?: boolean + host?: string + isLive?: boolean + isLocal?: boolean + include?: VideoIncludeType + + categoryOneOf?: number[] + licenceOneOf?: number[] + languageOneOf?: string[] + tagsOneOf?: string[] + tagsAllOf?: string[] + privacyOneOf?: VideoPrivacyType[] + + uuids?: string[] + + hasFiles?: boolean + hasHLSFiles?: boolean + + hasWebVideoFiles?: boolean + hasWebtorrentFiles?: boolean // TODO: Remove in v7 + + accountId?: number + videoChannelId?: number + + videoPlaylistId?: number + + trendingAlgorithm?: string // best, hot, or any other algorithm implemented + trendingDays?: number + + user?: MUserAccountId + historyOfUser?: MUserId + + startDate?: string // ISO 8601 + endDate?: string // ISO 8601 + originallyPublishedStartDate?: string + originallyPublishedEndDate?: string + + durationMin?: number // seconds + durationMax?: number // seconds + + search?: string + + isCount?: boolean + + group?: string + having?: string + + transaction?: Transaction + logging?: boolean + + excludeAlreadyWatched?: boolean +} + +export class VideosIdListQueryBuilder extends AbstractRunQuery { + protected replacements: any = {} + + private attributes: string[] + private joins: string[] = [] + + private readonly and: string[] = [] + + private readonly cte: string[] = [] + + private group = '' + private having = '' + + private sort = '' + private limit = '' + private offset = '' + + constructor (protected readonly sequelize: Sequelize) { + super(sequelize) + } + + queryVideoIds (options: BuildVideosListQueryOptions) { + this.buildIdsListQuery(options) + + return this.runQuery() + } + + countVideoIds (countOptions: BuildVideosListQueryOptions): Promise { + this.buildIdsListQuery(countOptions) + + return this.runQuery().then(rows => parseRowCountResult(rows)) + } + + getQuery (options: BuildVideosListQueryOptions) { + this.buildIdsListQuery(options) + + return { query: this.query, sort: this.sort, replacements: this.replacements } + } + + private buildIdsListQuery (options: BuildVideosListQueryOptions) { + this.attributes = options.attributes || [ '"video"."id"' ] + + if (options.group) this.group = options.group + if (options.having) this.having = options.having + + this.joins = this.joins.concat([ + 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId"', + 'INNER JOIN "account" ON "account"."id" = "videoChannel"."accountId"', + 'INNER JOIN "actor" "accountActor" ON "account"."actorId" = "accountActor"."id"' + ]) + + if (!(options.include & VideoInclude.BLACKLISTED)) { + this.whereNotBlacklisted() + } + + if (options.serverAccountIdForBlock && !(options.include & VideoInclude.BLOCKED_OWNER)) { + this.whereNotBlocked(options.serverAccountIdForBlock, options.user) + } + + // Only list published videos + if (!(options.include & VideoInclude.NOT_PUBLISHED_STATE)) { + this.whereStateAvailable() + } + + if (options.videoPlaylistId) { + this.joinPlaylist(options.videoPlaylistId) + } + + if (exists(options.isLocal)) { + this.whereLocal(options.isLocal) + } + + if (options.host) { + this.whereHost(options.host) + } + + if (options.accountId) { + this.whereAccountId(options.accountId) + } + + if (options.videoChannelId) { + this.whereChannelId(options.videoChannelId) + } + + if (options.displayOnlyForFollower) { + this.whereFollowerActorId(options.displayOnlyForFollower) + } + + if (options.hasFiles === true) { + this.whereFileExists() + } + + if (exists(options.hasWebtorrentFiles)) { + this.whereWebVideoFileExists(options.hasWebtorrentFiles) + } else if (exists(options.hasWebVideoFiles)) { + this.whereWebVideoFileExists(options.hasWebVideoFiles) + } + + if (exists(options.hasHLSFiles)) { + this.whereHLSFileExists(options.hasHLSFiles) + } + + if (options.tagsOneOf) { + this.whereTagsOneOf(options.tagsOneOf) + } + + if (options.tagsAllOf) { + this.whereTagsAllOf(options.tagsAllOf) + } + + if (options.privacyOneOf) { + this.wherePrivacyOneOf(options.privacyOneOf) + } else { + // Only list videos with the appropriate privacy + this.wherePrivacyAvailable(options.user) + } + + if (options.uuids) { + this.whereUUIDs(options.uuids) + } + + if (options.nsfw === true) { + this.whereNSFW() + } else if (options.nsfw === false) { + this.whereSFW() + } + + if (options.isLive === true) { + this.whereLive() + } else if (options.isLive === false) { + this.whereVOD() + } + + if (options.categoryOneOf) { + this.whereCategoryOneOf(options.categoryOneOf) + } + + if (options.licenceOneOf) { + this.whereLicenceOneOf(options.licenceOneOf) + } + + if (options.languageOneOf) { + this.whereLanguageOneOf(options.languageOneOf) + } + + // We don't exclude results in this so if we do a count we don't need to add this complex clause + if (options.isCount !== true) { + if (options.trendingDays) { + this.groupForTrending(options.trendingDays) + } else if ([ 'best', 'hot' ].includes(options.trendingAlgorithm)) { + this.groupForHotOrBest(options.trendingAlgorithm, options.user) + } + } + + if (options.historyOfUser) { + this.joinHistory(options.historyOfUser.id) + } + + if (options.startDate) { + this.whereStartDate(options.startDate) + } + + if (options.endDate) { + this.whereEndDate(options.endDate) + } + + if (options.originallyPublishedStartDate) { + this.whereOriginallyPublishedStartDate(options.originallyPublishedStartDate) + } + + if (options.originallyPublishedEndDate) { + this.whereOriginallyPublishedEndDate(options.originallyPublishedEndDate) + } + + if (options.durationMin) { + this.whereDurationMin(options.durationMin) + } + + if (options.durationMax) { + this.whereDurationMax(options.durationMax) + } + + if (options.excludeAlreadyWatched) { + if (exists(options.user.id)) { + this.whereExcludeAlreadyWatched(options.user.id) + } else { + throw new Error('Cannot use excludeAlreadyWatched parameter when auth token is not provided') + } + } + + this.whereSearch(options.search) + + if (options.isCount === true) { + this.setCountAttribute() + } else { + if (exists(options.sort)) { + this.setSort(options.sort) + } + + if (exists(options.count)) { + this.setLimit(options.count) + } + + if (exists(options.start)) { + this.setOffset(options.start) + } + } + + const cteString = this.cte.length !== 0 + ? `WITH ${this.cte.join(', ')} ` + : '' + + this.query = cteString + + 'SELECT ' + this.attributes.join(', ') + ' ' + + 'FROM "video" ' + this.joins.join(' ') + ' ' + + 'WHERE ' + this.and.join(' AND ') + ' ' + + this.group + ' ' + + this.having + ' ' + + this.sort + ' ' + + this.limit + ' ' + + this.offset + } + + private setCountAttribute () { + this.attributes = [ 'COUNT(*) as "total"' ] + } + + private joinHistory (userId: number) { + this.joins.push('INNER JOIN "userVideoHistory" ON "video"."id" = "userVideoHistory"."videoId"') + + this.and.push('"userVideoHistory"."userId" = :historyOfUser') + + this.replacements.historyOfUser = userId + } + + private joinPlaylist (playlistId: number) { + this.joins.push( + 'INNER JOIN "videoPlaylistElement" "video"."id" = "videoPlaylistElement"."videoId" ' + + 'AND "videoPlaylistElement"."videoPlaylistId" = :videoPlaylistId' + ) + + this.replacements.videoPlaylistId = playlistId + } + + private whereStateAvailable () { + this.and.push( + `("video"."state" = ${VideoState.PUBLISHED} OR ` + + `("video"."state" = ${VideoState.TO_TRANSCODE} AND "video"."waitTranscoding" IS false))` + ) + } + + private wherePrivacyAvailable (user?: MUserAccountId) { + if (user) { + this.and.push( + `("video"."privacy" = ${VideoPrivacy.PUBLIC} OR "video"."privacy" = ${VideoPrivacy.INTERNAL})` + ) + } else { // Or only public videos + this.and.push( + `"video"."privacy" = ${VideoPrivacy.PUBLIC}` + ) + } + } + + private whereLocal (isLocal: boolean) { + const isRemote = isLocal ? 'FALSE' : 'TRUE' + + this.and.push('"video"."remote" IS ' + isRemote) + } + + private whereHost (host: string) { + // Local instance + if (host === WEBSERVER.HOST) { + this.and.push('"accountActor"."serverId" IS NULL') + return + } + + this.joins.push('INNER JOIN "server" ON "server"."id" = "accountActor"."serverId"') + + this.and.push('"server"."host" = :host') + this.replacements.host = host + } + + private whereAccountId (accountId: number) { + this.and.push('"account"."id" = :accountId') + this.replacements.accountId = accountId + } + + private whereChannelId (channelId: number) { + this.and.push('"videoChannel"."id" = :videoChannelId') + this.replacements.videoChannelId = channelId + } + + private whereFollowerActorId (options: { actorId: number, orLocalVideos: boolean }) { + let query = + '(' + + ' EXISTS (' + // Videos shared by actors we follow + ' SELECT 1 FROM "videoShare" ' + + ' INNER JOIN "actorFollow" "actorFollowShare" ON "actorFollowShare"."targetActorId" = "videoShare"."actorId" ' + + ' AND "actorFollowShare"."actorId" = :followerActorId AND "actorFollowShare"."state" = \'accepted\' ' + + ' WHERE "videoShare"."videoId" = "video"."id"' + + ' )' + + ' OR' + + ' EXISTS (' + // Videos published by channels or accounts we follow + ' SELECT 1 from "actorFollow" ' + + ' WHERE ("actorFollow"."targetActorId" = "account"."actorId" OR "actorFollow"."targetActorId" = "videoChannel"."actorId") ' + + ' AND "actorFollow"."actorId" = :followerActorId ' + + ' AND "actorFollow"."state" = \'accepted\'' + + ' )' + + if (options.orLocalVideos) { + query += ' OR "video"."remote" IS FALSE' + } + + query += ')' + + this.and.push(query) + this.replacements.followerActorId = options.actorId + } + + private whereFileExists () { + this.and.push(`(${this.buildWebVideoFileExistsQuery(true)} OR ${this.buildHLSFileExistsQuery(true)})`) + } + + private whereWebVideoFileExists (exists: boolean) { + this.and.push(this.buildWebVideoFileExistsQuery(exists)) + } + + private whereHLSFileExists (exists: boolean) { + this.and.push(this.buildHLSFileExistsQuery(exists)) + } + + private buildWebVideoFileExistsQuery (exists: boolean) { + const prefix = exists ? '' : 'NOT ' + + return prefix + 'EXISTS (SELECT 1 FROM "videoFile" WHERE "videoFile"."videoId" = "video"."id")' + } + + private buildHLSFileExistsQuery (exists: boolean) { + const prefix = exists ? '' : 'NOT ' + + return prefix + 'EXISTS (' + + ' SELECT 1 FROM "videoStreamingPlaylist" ' + + ' INNER JOIN "videoFile" ON "videoFile"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ' + + ' WHERE "videoStreamingPlaylist"."videoId" = "video"."id"' + + ')' + } + + private whereTagsOneOf (tagsOneOf: string[]) { + const tagsOneOfLower = tagsOneOf.map(t => t.toLowerCase()) + + this.and.push( + 'EXISTS (' + + ' SELECT 1 FROM "videoTag" ' + + ' INNER JOIN "tag" ON "tag"."id" = "videoTag"."tagId" ' + + ' WHERE lower("tag"."name") IN (' + createSafeIn(this.sequelize, tagsOneOfLower) + ') ' + + ' AND "video"."id" = "videoTag"."videoId"' + + ')' + ) + } + + private whereTagsAllOf (tagsAllOf: string[]) { + const tagsAllOfLower = tagsAllOf.map(t => t.toLowerCase()) + + this.and.push( + 'EXISTS (' + + ' SELECT 1 FROM "videoTag" ' + + ' INNER JOIN "tag" ON "tag"."id" = "videoTag"."tagId" ' + + ' WHERE lower("tag"."name") IN (' + createSafeIn(this.sequelize, tagsAllOfLower) + ') ' + + ' AND "video"."id" = "videoTag"."videoId" ' + + ' GROUP BY "videoTag"."videoId" HAVING COUNT(*) = ' + tagsAllOfLower.length + + ')' + ) + } + + private wherePrivacyOneOf (privacyOneOf: VideoPrivacyType[]) { + this.and.push('"video"."privacy" IN (:privacyOneOf)') + this.replacements.privacyOneOf = privacyOneOf + } + + private whereUUIDs (uuids: string[]) { + this.and.push('"video"."uuid" IN (' + createSafeIn(this.sequelize, uuids) + ')') + } + + private whereCategoryOneOf (categoryOneOf: number[]) { + this.and.push('"video"."category" IN (:categoryOneOf)') + this.replacements.categoryOneOf = categoryOneOf + } + + private whereLicenceOneOf (licenceOneOf: number[]) { + this.and.push('"video"."licence" IN (:licenceOneOf)') + this.replacements.licenceOneOf = licenceOneOf + } + + private whereLanguageOneOf (languageOneOf: string[]) { + const languages = languageOneOf.filter(l => l && l !== '_unknown') + const languagesQueryParts: string[] = [] + + if (languages.length !== 0) { + languagesQueryParts.push('"video"."language" IN (:languageOneOf)') + this.replacements.languageOneOf = languages + + languagesQueryParts.push( + 'EXISTS (' + + ' SELECT 1 FROM "videoCaption" WHERE "videoCaption"."language" ' + + ' IN (' + createSafeIn(this.sequelize, languages) + ') AND ' + + ' "videoCaption"."videoId" = "video"."id"' + + ')' + ) + } + + if (languageOneOf.includes('_unknown')) { + languagesQueryParts.push('"video"."language" IS NULL') + } + + if (languagesQueryParts.length !== 0) { + this.and.push('(' + languagesQueryParts.join(' OR ') + ')') + } + } + + private whereNSFW () { + this.and.push('"video"."nsfw" IS TRUE') + } + + private whereSFW () { + this.and.push('"video"."nsfw" IS FALSE') + } + + private whereLive () { + this.and.push('"video"."isLive" IS TRUE') + } + + private whereVOD () { + this.and.push('"video"."isLive" IS FALSE') + } + + private whereNotBlocked (serverAccountId: number, user?: MUserAccountId) { + const blockerIds = [ serverAccountId ] + if (user) blockerIds.push(user.Account.id) + + const inClause = createSafeIn(this.sequelize, blockerIds) + + this.and.push( + 'NOT EXISTS (' + + ' SELECT 1 FROM "accountBlocklist" ' + + ' WHERE "accountBlocklist"."accountId" IN (' + inClause + ') ' + + ' AND "accountBlocklist"."targetAccountId" = "account"."id" ' + + ')' + + 'AND NOT EXISTS (' + + ' SELECT 1 FROM "serverBlocklist" WHERE "serverBlocklist"."accountId" IN (' + inClause + ') ' + + ' AND "serverBlocklist"."targetServerId" = "accountActor"."serverId"' + + ')' + ) + } + + private whereSearch (search?: string) { + if (!search) { + this.attributes.push('0 as similarity') + return + } + + const escapedSearch = this.sequelize.escape(search) + const escapedLikeSearch = this.sequelize.escape('%' + search + '%') + + this.cte.push( + '"trigramSearch" AS (' + + ' SELECT "video"."id", ' + + ` similarity(lower(immutable_unaccent("video"."name")), lower(immutable_unaccent(${escapedSearch}))) as similarity ` + + ' FROM "video" ' + + ' WHERE lower(immutable_unaccent("video"."name")) % lower(immutable_unaccent(' + escapedSearch + ')) OR ' + + ' lower(immutable_unaccent("video"."name")) LIKE lower(immutable_unaccent(' + escapedLikeSearch + '))' + + ')' + ) + + this.joins.push('LEFT JOIN "trigramSearch" ON "video"."id" = "trigramSearch"."id"') + + let base = '(' + + ' "trigramSearch"."id" IS NOT NULL OR ' + + ' EXISTS (' + + ' SELECT 1 FROM "videoTag" ' + + ' INNER JOIN "tag" ON "tag"."id" = "videoTag"."tagId" ' + + ` WHERE lower("tag"."name") = lower(${escapedSearch}) ` + + ' AND "video"."id" = "videoTag"."videoId"' + + ' )' + + if (validator.default.isUUID(search)) { + base += ` OR "video"."uuid" = ${escapedSearch}` + } + + base += ')' + + this.and.push(base) + this.attributes.push(`COALESCE("trigramSearch"."similarity", 0) as similarity`) + } + + private whereNotBlacklisted () { + this.and.push('"video"."id" NOT IN (SELECT "videoBlacklist"."videoId" FROM "videoBlacklist")') + } + + private whereStartDate (startDate: string) { + this.and.push('"video"."publishedAt" >= :startDate') + this.replacements.startDate = startDate + } + + private whereEndDate (endDate: string) { + this.and.push('"video"."publishedAt" <= :endDate') + this.replacements.endDate = endDate + } + + private whereOriginallyPublishedStartDate (startDate: string) { + this.and.push('"video"."originallyPublishedAt" >= :originallyPublishedStartDate') + this.replacements.originallyPublishedStartDate = startDate + } + + private whereOriginallyPublishedEndDate (endDate: string) { + this.and.push('"video"."originallyPublishedAt" <= :originallyPublishedEndDate') + this.replacements.originallyPublishedEndDate = endDate + } + + private whereDurationMin (durationMin: number) { + this.and.push('"video"."duration" >= :durationMin') + this.replacements.durationMin = durationMin + } + + private whereDurationMax (durationMax: number) { + this.and.push('"video"."duration" <= :durationMax') + this.replacements.durationMax = durationMax + } + + private whereExcludeAlreadyWatched (userId: number) { + this.and.push( + 'NOT EXISTS (' + + ' SELECT 1' + + ' FROM "userVideoHistory"' + + ' WHERE "video"."id" = "userVideoHistory"."videoId"' + + ' AND "userVideoHistory"."userId" = :excludeAlreadyWatchedUserId' + + ')' + ) + this.replacements.excludeAlreadyWatchedUserId = userId + } + + private groupForTrending (trendingDays: number) { + const viewsGteDate = new Date(new Date().getTime() - (24 * 3600 * 1000) * trendingDays) + + this.joins.push('LEFT JOIN "videoView" ON "video"."id" = "videoView"."videoId" AND "videoView"."startDate" >= :viewsGteDate') + this.replacements.viewsGteDate = viewsGteDate + + this.attributes.push('COALESCE(SUM("videoView"."views"), 0) AS "score"') + + this.group = 'GROUP BY "video"."id"' + } + + private groupForHotOrBest (trendingAlgorithm: string, user?: MUserAccountId) { + /** + * "Hotness" is a measure based on absolute view/comment/like/dislike numbers, + * with fixed weights only applied to their log values. + * + * This algorithm gives little chance for an old video to have a good score, + * for which recent spikes in interactions could be a sign of "hotness" and + * justify a better score. However there are multiple ways to achieve that + * goal, which is left for later. Yes, this is a TODO :) + * + * notes: + * - weights and base score are in number of half-days. + * - all comments are counted, regardless of being written by the video author or not + * see https://github.com/reddit-archive/reddit/blob/master/r2/r2/lib/db/_sorts.pyx#L47-L58 + * - we have less interactions than on reddit, so multiply weights by an arbitrary factor + */ + const weights = { + like: 3 * 50, + dislike: -3 * 50, + view: Math.floor((1 / 3) * 50), + comment: 2 * 50, // a comment takes more time than a like to do, but can be done multiple times + history: -2 * 50 + } + + this.joins.push('LEFT JOIN "videoComment" ON "video"."id" = "videoComment"."videoId"') + + let attribute = + `LOG(GREATEST(1, "video"."likes" - 1)) * ${weights.like} ` + // likes (+) + `+ LOG(GREATEST(1, "video"."dislikes" - 1)) * ${weights.dislike} ` + // dislikes (-) + `+ LOG("video"."views" + 1) * ${weights.view} ` + // views (+) + `+ LOG(GREATEST(1, COUNT(DISTINCT "videoComment"."id"))) * ${weights.comment} ` + // comments (+) + '+ (SELECT (EXTRACT(epoch FROM "video"."publishedAt") - 1446156582) / 47000) ' // base score (in number of half-days) + + if (trendingAlgorithm === 'best' && user) { + this.joins.push( + 'LEFT JOIN "userVideoHistory" ON "video"."id" = "userVideoHistory"."videoId" AND "userVideoHistory"."userId" = :bestUser' + ) + this.replacements.bestUser = user.id + + attribute += `+ POWER(COUNT(DISTINCT "userVideoHistory"."id"), 2.0) * ${weights.history} ` + } + + attribute += 'AS "score"' + this.attributes.push(attribute) + + this.group = 'GROUP BY "video"."id"' + } + + private setSort (sort: string) { + if (sort === '-originallyPublishedAt' || sort === 'originallyPublishedAt') { + this.attributes.push('COALESCE("video"."originallyPublishedAt", "video"."publishedAt") AS "publishedAtForOrder"') + } + + this.sort = this.buildOrder(sort) + } + + private buildOrder (value: string) { + const { direction, field } = buildSortDirectionAndField(value) + if (field.match(/^[a-zA-Z."]+$/) === null) throw new Error('Invalid sort column ' + field) + + if (field.toLowerCase() === 'random') return 'ORDER BY RANDOM()' + + if ([ 'trending', 'hot', 'best' ].includes(field.toLowerCase())) { // Sort by aggregation + return `ORDER BY "score" ${direction}, "video"."views" ${direction}` + } + + let firstSort: string + + if (field.toLowerCase() === 'match') { // Search + firstSort = '"similarity"' + } else if (field === 'originallyPublishedAt') { + firstSort = '"publishedAtForOrder"' + } else if (field.includes('.')) { + firstSort = field + } else { + firstSort = `"video"."${field}"` + } + + return `ORDER BY ${firstSort} ${direction}, "video"."id" ASC` + } + + private setLimit (countArg: number) { + const count = forceNumber(countArg) + this.limit = `LIMIT ${count}` + } + + private setOffset (startArg: number) { + const start = forceNumber(startArg) + this.offset = `OFFSET ${start}` + } +} diff --git a/server/server/models/video/sql/video/videos-model-list-query-builder.ts b/server/server/models/video/sql/video/videos-model-list-query-builder.ts new file mode 100644 index 000000000..9eb26085d --- /dev/null +++ b/server/server/models/video/sql/video/videos-model-list-query-builder.ts @@ -0,0 +1,103 @@ +import { Sequelize } from 'sequelize' +import { pick } from '@peertube/peertube-core-utils' +import { VideoInclude } from '@peertube/peertube-models' +import { AbstractVideoQueryBuilder } from './shared/abstract-video-query-builder.js' +import { VideoFileQueryBuilder } from './shared/video-file-query-builder.js' +import { VideoModelBuilder } from './shared/video-model-builder.js' +import { BuildVideosListQueryOptions, VideosIdListQueryBuilder } from './videos-id-list-query-builder.js' + +/** + * + * Build videos list SQL query and create video models + * + */ + +export class VideosModelListQueryBuilder extends AbstractVideoQueryBuilder { + protected attributes: { [key: string]: string } + + private innerQuery: string + private innerSort: string + + webVideoFilesQueryBuilder: VideoFileQueryBuilder + streamingPlaylistFilesQueryBuilder: VideoFileQueryBuilder + + private readonly videoModelBuilder: VideoModelBuilder + + constructor (protected readonly sequelize: Sequelize) { + super(sequelize, 'list') + + this.videoModelBuilder = new VideoModelBuilder(this.mode, this.tables) + this.webVideoFilesQueryBuilder = new VideoFileQueryBuilder(sequelize) + this.streamingPlaylistFilesQueryBuilder = new VideoFileQueryBuilder(sequelize) + } + + async queryVideos (options: BuildVideosListQueryOptions) { + this.buildInnerQuery(options) + this.buildMainQuery(options) + + const rows = await this.runQuery() + + if (options.include & VideoInclude.FILES) { + const videoIds = Array.from(new Set(rows.map(r => r.id))) + + if (videoIds.length !== 0) { + const fileQueryOptions = { + ...pick(options, [ 'transaction', 'logging' ]), + + ids: videoIds, + includeRedundancy: false + } + + const [ rowsWebVideoFiles, rowsStreamingPlaylist ] = await Promise.all([ + this.webVideoFilesQueryBuilder.queryWebVideos(fileQueryOptions), + this.streamingPlaylistFilesQueryBuilder.queryStreamingPlaylistVideos(fileQueryOptions) + ]) + + return this.videoModelBuilder.buildVideosFromRows({ rows, include: options.include, rowsStreamingPlaylist, rowsWebVideoFiles }) + } + } + + return this.videoModelBuilder.buildVideosFromRows({ rows, include: options.include }) + } + + private buildInnerQuery (options: BuildVideosListQueryOptions) { + const idsQueryBuilder = new VideosIdListQueryBuilder(this.sequelize) + const { query, sort, replacements } = idsQueryBuilder.getQuery(options) + + this.replacements = replacements + this.innerQuery = query + this.innerSort = sort + } + + private buildMainQuery (options: BuildVideosListQueryOptions) { + this.attributes = { + '"video".*': '' + } + + this.addJoin('INNER JOIN "video" ON "tmp"."id" = "video"."id"') + + this.includeChannels() + this.includeAccounts() + this.includeThumbnails() + + if (options.user) { + this.includeUserHistory(options.user.id) + } + + if (options.videoPlaylistId) { + this.includePlaylist(options.videoPlaylistId) + } + + if (options.include & VideoInclude.BLACKLISTED) { + this.includeBlacklisted() + } + + if (options.include & VideoInclude.BLOCKED_OWNER) { + this.includeBlockedOwnerAndServer(options.serverAccountIdForBlock, options.user) + } + + const select = this.buildSelect() + + this.query = `${select} FROM (${this.innerQuery}) AS "tmp" ${this.joins} ${this.innerSort}` + } +} diff --git a/server/server/models/video/storyboard.ts b/server/server/models/video/storyboard.ts new file mode 100644 index 000000000..1d0a8e429 --- /dev/null +++ b/server/server/models/video/storyboard.ts @@ -0,0 +1,169 @@ +import { remove } from 'fs-extra/esm' +import { join } from 'path' +import { AfterDestroy, AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { CONFIG } from '@server/initializers/config.js' +import { MStoryboard, MStoryboardVideo, MVideo } from '@server/types/models/index.js' +import { Storyboard } from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { logger } from '../../helpers/logger.js' +import { CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, WEBSERVER } from '../../initializers/constants.js' +import { VideoModel } from './video.js' +import { Transaction } from 'sequelize' + +@Table({ + tableName: 'storyboard', + indexes: [ + { + fields: [ 'videoId' ], + unique: true + }, + { + fields: [ 'filename' ], + unique: true + } + ] +}) +export class StoryboardModel extends Model>> { + + @AllowNull(false) + @Column + filename: string + + @AllowNull(false) + @Column + totalHeight: number + + @AllowNull(false) + @Column + totalWidth: number + + @AllowNull(false) + @Column + spriteHeight: number + + @AllowNull(false) + @Column + spriteWidth: number + + @AllowNull(false) + @Column + spriteDuration: number + + @AllowNull(true) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.COMMONS.URL.max)) + fileUrl: string + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + Video: Awaited + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AfterDestroy + static removeInstanceFile (instance: StoryboardModel) { + logger.info('Removing storyboard file %s.', instance.filename) + + // Don't block the transaction + instance.removeFile() + .catch(err => logger.error('Cannot remove storyboard file %s.', instance.filename, { err })) + } + + static loadByVideo (videoId: number, transaction?: Transaction): Promise { + const query = { + where: { + videoId + }, + transaction + } + + return StoryboardModel.findOne(query) + } + + static loadByFilename (filename: string): Promise { + const query = { + where: { + filename + } + } + + return StoryboardModel.findOne(query) + } + + static loadWithVideoByFilename (filename: string): Promise { + const query = { + where: { + filename + }, + include: [ + { + model: VideoModel.unscoped(), + required: true + } + ] + } + + return StoryboardModel.findOne(query) + } + + // --------------------------------------------------------------------------- + + static async listStoryboardsOf (video: MVideo): Promise { + const query = { + where: { + videoId: video.id + } + } + + const storyboards = await StoryboardModel.findAll(query) + + return storyboards.map(s => Object.assign(s, { Video: video })) + } + + // --------------------------------------------------------------------------- + + getOriginFileUrl (video: MVideo) { + if (video.isOwned()) { + return WEBSERVER.URL + this.getLocalStaticPath() + } + + return this.fileUrl + } + + getLocalStaticPath () { + return LAZY_STATIC_PATHS.STORYBOARDS + this.filename + } + + getPath () { + return join(CONFIG.STORAGE.STORYBOARDS_DIR, this.filename) + } + + removeFile () { + return remove(this.getPath()) + } + + toFormattedJSON (this: MStoryboardVideo): Storyboard { + return { + storyboardPath: this.getLocalStaticPath(), + + totalHeight: this.totalHeight, + totalWidth: this.totalWidth, + + spriteWidth: this.spriteWidth, + spriteHeight: this.spriteHeight, + + spriteDuration: this.spriteDuration + } + } +} diff --git a/server/server/models/video/tag.ts b/server/server/models/video/tag.ts new file mode 100644 index 000000000..dee954795 --- /dev/null +++ b/server/server/models/video/tag.ts @@ -0,0 +1,86 @@ +import { col, fn, QueryTypes, Transaction } from 'sequelize' +import { AllowNull, BelongsToMany, Column, CreatedAt, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { VideoPrivacy, VideoState } from '@peertube/peertube-models' +import { MTag } from '@server/types/models/index.js' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { isVideoTagValid } from '../../helpers/custom-validators/videos.js' +import { throwIfNotValid } from '../shared/index.js' +import { VideoTagModel } from './video-tag.js' +import { VideoModel } from './video.js' + +@Table({ + tableName: 'tag', + timestamps: false, + indexes: [ + { + fields: [ 'name' ], + unique: true + }, + { + name: 'tag_lower_name', + fields: [ fn('lower', col('name')) ] + } + ] +}) +export class TagModel extends Model>> { + + @AllowNull(false) + @Is('VideoTag', value => throwIfNotValid(value, isVideoTagValid, 'tag')) + @Column + name: string + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @BelongsToMany(() => VideoModel, { + foreignKey: 'tagId', + through: () => VideoTagModel, + onDelete: 'CASCADE' + }) + Videos: Awaited[] + + static findOrCreateTags (tags: string[], transaction: Transaction): Promise { + if (tags === null) return Promise.resolve([]) + + const uniqueTags = new Set(tags) + + const tasks = Array.from(uniqueTags).map(tag => { + const query = { + where: { + name: tag + }, + defaults: { + name: tag + }, + transaction + } + + return TagModel.findOrCreate(query) + .then(([ tagInstance ]) => tagInstance) + }) + + return Promise.all(tasks) + } + + // threshold corresponds to how many video the field should have to be returned + static getRandomSamples (threshold: number, count: number): Promise { + const query = 'SELECT tag.name FROM tag ' + + 'INNER JOIN "videoTag" ON "videoTag"."tagId" = tag.id ' + + 'INNER JOIN video ON video.id = "videoTag"."videoId" ' + + 'WHERE video.privacy = $videoPrivacy AND video.state = $videoState ' + + 'GROUP BY tag.name HAVING COUNT(tag.name) >= $threshold ' + + 'ORDER BY random() ' + + 'LIMIT $count' + + const options = { + bind: { threshold, count, videoPrivacy: VideoPrivacy.PUBLIC, videoState: VideoState.PUBLISHED }, + type: QueryTypes.SELECT as QueryTypes.SELECT + } + + return TagModel.sequelize.query<{ name: string }>(query, options) + .then(data => data.map(d => d.name)) + } +} diff --git a/server/server/models/video/thumbnail.ts b/server/server/models/video/thumbnail.ts new file mode 100644 index 000000000..4791b8b75 --- /dev/null +++ b/server/server/models/video/thumbnail.ts @@ -0,0 +1,208 @@ +import { ThumbnailType, type ThumbnailType_Type } from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { afterCommitIfTransaction } from '@server/helpers/database-utils.js' +import { MThumbnail, MThumbnailVideo, MVideo } from '@server/types/models/index.js' +import { remove } from 'fs-extra/esm' +import { join } from 'path' +import { + AfterDestroy, + AllowNull, + BeforeCreate, + BeforeUpdate, + BelongsTo, + Column, + CreatedAt, + DataType, + Default, + ForeignKey, + Model, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { logger } from '../../helpers/logger.js' +import { CONFIG } from '../../initializers/config.js' +import { CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, WEBSERVER } from '../../initializers/constants.js' +import { VideoPlaylistModel } from './video-playlist.js' +import { VideoModel } from './video.js' + +@Table({ + tableName: 'thumbnail', + indexes: [ + { + fields: [ 'videoId' ] + }, + { + fields: [ 'videoPlaylistId' ], + unique: true + }, + { + fields: [ 'filename', 'type' ], + unique: true + } + ] +}) +export class ThumbnailModel extends Model>> { + + @AllowNull(false) + @Column + filename: string + + @AllowNull(true) + @Default(null) + @Column + height: number + + @AllowNull(true) + @Default(null) + @Column + width: number + + @AllowNull(false) + @Column + type: ThumbnailType_Type + + @AllowNull(true) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.COMMONS.URL.max)) + fileUrl: string + + @AllowNull(true) + @Column + automaticallyGenerated: boolean + + @AllowNull(false) + @Column + onDisk: boolean + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'CASCADE' + }) + Video: Awaited + + @ForeignKey(() => VideoPlaylistModel) + @Column + videoPlaylistId: number + + @BelongsTo(() => VideoPlaylistModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'CASCADE' + }) + VideoPlaylist: Awaited + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + // If this thumbnail replaced existing one, track the old name + previousThumbnailFilename: string + + private static readonly types: { [ id in ThumbnailType_Type ]: { label: string, directory: string, staticPath: string } } = { + [ThumbnailType.MINIATURE]: { + label: 'miniature', + directory: CONFIG.STORAGE.THUMBNAILS_DIR, + staticPath: LAZY_STATIC_PATHS.THUMBNAILS + }, + [ThumbnailType.PREVIEW]: { + label: 'preview', + directory: CONFIG.STORAGE.PREVIEWS_DIR, + staticPath: LAZY_STATIC_PATHS.PREVIEWS + } + } + + @BeforeCreate + @BeforeUpdate + static removeOldFile (instance: ThumbnailModel, options) { + return afterCommitIfTransaction(options.transaction, () => instance.removePreviousFilenameIfNeeded()) + } + + @AfterDestroy + static removeFiles (instance: ThumbnailModel) { + logger.info('Removing %s file %s.', ThumbnailModel.types[instance.type].label, instance.filename) + + // Don't block the transaction + instance.removeThumbnail() + .catch(err => logger.error('Cannot remove thumbnail file %s.', instance.filename, { err })) + } + + static loadByFilename (filename: string, thumbnailType: ThumbnailType_Type): Promise { + const query = { + where: { + filename, + type: thumbnailType + } + } + + return ThumbnailModel.findOne(query) + } + + static loadWithVideoByFilename (filename: string, thumbnailType: ThumbnailType_Type): Promise { + const query = { + where: { + filename, + type: thumbnailType + }, + include: [ + { + model: VideoModel.unscoped(), + required: true + } + ] + } + + return ThumbnailModel.findOne(query) + } + + static buildPath (type: ThumbnailType_Type, filename: string) { + const directory = ThumbnailModel.types[type].directory + + return join(directory, filename) + } + + getOriginFileUrl (video: MVideo) { + const staticPath = ThumbnailModel.types[this.type].staticPath + this.filename + + if (video.isOwned()) return WEBSERVER.URL + staticPath + + return this.fileUrl + } + + getLocalStaticPath () { + return ThumbnailModel.types[this.type].staticPath + this.filename + } + + getPath () { + return ThumbnailModel.buildPath(this.type, this.filename) + } + + getPreviousPath () { + return ThumbnailModel.buildPath(this.type, this.previousThumbnailFilename) + } + + removeThumbnail () { + return remove(this.getPath()) + } + + removePreviousFilenameIfNeeded () { + if (!this.previousThumbnailFilename) return + + const previousPath = this.getPreviousPath() + remove(previousPath) + .catch(err => logger.error('Cannot remove previous thumbnail file %s.', previousPath, { err })) + + this.previousThumbnailFilename = undefined + } + + isOwned () { + return !this.fileUrl + } +} diff --git a/server/server/models/video/video-blacklist.ts b/server/server/models/video/video-blacklist.ts new file mode 100644 index 000000000..0f6930034 --- /dev/null +++ b/server/server/models/video/video-blacklist.ts @@ -0,0 +1,134 @@ +import { VideoBlacklist, type VideoBlacklistType_Type } from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { MVideoBlacklist, MVideoBlacklistFormattable } from '@server/types/models/index.js' +import { FindOptions } from 'sequelize' +import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { isVideoBlacklistReasonValid, isVideoBlacklistTypeValid } from '../../helpers/custom-validators/video-blacklist.js' +import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js' +import { getBlacklistSort, searchAttribute, throwIfNotValid } from '../shared/index.js' +import { ThumbnailModel } from './thumbnail.js' +import { SummaryOptions, VideoChannelModel, ScopeNames as VideoChannelScopeNames } from './video-channel.js' +import { VideoModel } from './video.js' + +@Table({ + tableName: 'videoBlacklist', + indexes: [ + { + fields: [ 'videoId' ], + unique: true + } + ] +}) +export class VideoBlacklistModel extends Model>> { + + @AllowNull(true) + @Is('VideoBlacklistReason', value => throwIfNotValid(value, isVideoBlacklistReasonValid, 'reason', true)) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_BLACKLIST.REASON.max)) + reason: string + + @AllowNull(false) + @Column + unfederated: boolean + + @AllowNull(false) + @Default(null) + @Is('VideoBlacklistType', value => throwIfNotValid(value, isVideoBlacklistTypeValid, 'type')) + @Column + type: VideoBlacklistType_Type + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + Video: Awaited + + static listForApi (parameters: { + start: number + count: number + sort: string + search?: string + type?: VideoBlacklistType_Type + }) { + const { start, count, sort, search, type } = parameters + + function buildBaseQuery (): FindOptions { + return { + offset: start, + limit: count, + order: getBlacklistSort(sort) + } + } + + const countQuery = buildBaseQuery() + + const findQuery = buildBaseQuery() + findQuery.include = [ + { + model: VideoModel, + required: true, + where: searchAttribute(search, 'name'), + include: [ + { + model: VideoChannelModel.scope({ method: [ VideoChannelScopeNames.SUMMARY, { withAccount: true } as SummaryOptions ] }), + required: true + }, + { + model: ThumbnailModel, + attributes: [ 'type', 'filename' ], + required: false + } + ] + } + ] + + if (type) { + countQuery.where = { type } + findQuery.where = { type } + } + + return Promise.all([ + VideoBlacklistModel.count(countQuery), + VideoBlacklistModel.findAll(findQuery) + ]).then(([ count, rows ]) => { + return { + data: rows, + total: count + } + }) + } + + static loadByVideoId (id: number): Promise { + const query = { + where: { + videoId: id + } + } + + return VideoBlacklistModel.findOne(query) + } + + toFormattedJSON (this: MVideoBlacklistFormattable): VideoBlacklist { + return { + id: this.id, + createdAt: this.createdAt, + updatedAt: this.updatedAt, + reason: this.reason, + unfederated: this.unfederated, + type: this.type, + + video: this.Video.toFormattedJSON() + } + } +} diff --git a/server/server/models/video/video-caption.ts b/server/server/models/video/video-caption.ts new file mode 100644 index 000000000..312f23a30 --- /dev/null +++ b/server/server/models/video/video-caption.ts @@ -0,0 +1,253 @@ +import { remove } from 'fs-extra/esm' +import { join } from 'path' +import { Op, OrderItem, Transaction } from 'sequelize' +import { + AllowNull, + BeforeDestroy, + BelongsTo, + Column, + CreatedAt, + DataType, + ForeignKey, + Is, + Model, + Scopes, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { VideoCaption } from '@peertube/peertube-models' +import { + MVideo, + MVideoCaption, + MVideoCaptionFormattable, + MVideoCaptionLanguageUrl, + MVideoCaptionVideo +} from '@server/types/models/index.js' +import { buildUUID } from '@peertube/peertube-node-utils' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { isVideoCaptionLanguageValid } from '../../helpers/custom-validators/video-captions.js' +import { logger } from '../../helpers/logger.js' +import { CONFIG } from '../../initializers/config.js' +import { CONSTRAINTS_FIELDS, LAZY_STATIC_PATHS, VIDEO_LANGUAGES, WEBSERVER } from '../../initializers/constants.js' +import { buildWhereIdOrUUID, throwIfNotValid } from '../shared/index.js' +import { VideoModel } from './video.js' + +export enum ScopeNames { + WITH_VIDEO_UUID_AND_REMOTE = 'WITH_VIDEO_UUID_AND_REMOTE' +} + +@Scopes(() => ({ + [ScopeNames.WITH_VIDEO_UUID_AND_REMOTE]: { + include: [ + { + attributes: [ 'id', 'uuid', 'remote' ], + model: VideoModel.unscoped(), + required: true + } + ] + } +})) + +@Table({ + tableName: 'videoCaption', + indexes: [ + { + fields: [ 'filename' ], + unique: true + }, + { + fields: [ 'videoId' ] + }, + { + fields: [ 'videoId', 'language' ], + unique: true + } + ] +}) +export class VideoCaptionModel extends Model>> { + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(false) + @Is('VideoCaptionLanguage', value => throwIfNotValid(value, isVideoCaptionLanguageValid, 'language')) + @Column + language: string + + @AllowNull(false) + @Column + filename: string + + @AllowNull(true) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.COMMONS.URL.max)) + fileUrl: string + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + Video: Awaited + + @BeforeDestroy + static async removeFiles (instance: VideoCaptionModel, options) { + if (!instance.Video) { + instance.Video = await instance.$get('Video', { transaction: options.transaction }) + } + + if (instance.isOwned()) { + logger.info('Removing caption %s.', instance.filename) + + try { + await instance.removeCaptionFile() + } catch (err) { + logger.error('Cannot remove caption file %s.', instance.filename) + } + } + + return undefined + } + + static loadByVideoIdAndLanguage (videoId: string | number, language: string, transaction?: Transaction): Promise { + const videoInclude = { + model: VideoModel.unscoped(), + attributes: [ 'id', 'remote', 'uuid' ], + where: buildWhereIdOrUUID(videoId) + } + + const query = { + where: { + language + }, + include: [ + videoInclude + ], + transaction + } + + return VideoCaptionModel.findOne(query) + } + + static loadWithVideoByFilename (filename: string): Promise { + const query = { + where: { + filename + }, + include: [ + { + model: VideoModel.unscoped(), + attributes: [ 'id', 'remote', 'uuid' ] + } + ] + } + + return VideoCaptionModel.findOne(query) + } + + static async insertOrReplaceLanguage (caption: MVideoCaption, transaction: Transaction) { + const existing = await VideoCaptionModel.loadByVideoIdAndLanguage(caption.videoId, caption.language, transaction) + + // Delete existing file + if (existing) await existing.destroy({ transaction }) + + return caption.save({ transaction }) + } + + static listVideoCaptions (videoId: number, transaction?: Transaction): Promise { + const query = { + order: [ [ 'language', 'ASC' ] ] as OrderItem[], + where: { + videoId + }, + transaction + } + + return VideoCaptionModel.scope(ScopeNames.WITH_VIDEO_UUID_AND_REMOTE).findAll(query) + } + + static async listCaptionsOfMultipleVideos (videoIds: number[], transaction?: Transaction) { + const query = { + order: [ [ 'language', 'ASC' ] ] as OrderItem[], + where: { + videoId: { + [Op.in]: videoIds + } + }, + transaction + } + + const captions = await VideoCaptionModel.scope(ScopeNames.WITH_VIDEO_UUID_AND_REMOTE).findAll(query) + const result: { [ id: number ]: MVideoCaptionVideo[] } = {} + + for (const id of videoIds) { + result[id] = [] + } + + for (const caption of captions) { + result[caption.videoId].push(caption) + } + + return result + } + + static getLanguageLabel (language: string) { + return VIDEO_LANGUAGES[language] || 'Unknown' + } + + static deleteAllCaptionsOfRemoteVideo (videoId: number, transaction: Transaction) { + const query = { + where: { + videoId + }, + transaction + } + + return VideoCaptionModel.destroy(query) + } + + static generateCaptionName (language: string) { + return `${buildUUID()}-${language}.vtt` + } + + isOwned () { + return this.Video.remote === false + } + + toFormattedJSON (this: MVideoCaptionFormattable): VideoCaption { + return { + language: { + id: this.language, + label: VideoCaptionModel.getLanguageLabel(this.language) + }, + captionPath: this.getCaptionStaticPath(), + updatedAt: this.updatedAt.toISOString() + } + } + + getCaptionStaticPath (this: MVideoCaptionLanguageUrl) { + return join(LAZY_STATIC_PATHS.VIDEO_CAPTIONS, this.filename) + } + + removeCaptionFile (this: MVideoCaption) { + return remove(CONFIG.STORAGE.CAPTIONS_DIR + this.filename) + } + + getFileUrl (this: MVideoCaptionLanguageUrl, video: MVideo) { + if (video.isOwned()) return WEBSERVER.URL + this.getCaptionStaticPath() + + return this.fileUrl + } + + isEqual (this: MVideoCaption, other: MVideoCaption) { + if (this.fileUrl) return this.fileUrl === other.fileUrl + + return this.filename === other.filename + } +} diff --git a/server/server/models/video/video-change-ownership.ts b/server/server/models/video/video-change-ownership.ts new file mode 100644 index 000000000..152f85a22 --- /dev/null +++ b/server/server/models/video/video-change-ownership.ts @@ -0,0 +1,137 @@ +import { VideoChangeOwnership, type VideoChangeOwnershipStatusType } from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { MVideoChangeOwnershipFormattable, MVideoChangeOwnershipFull } from '@server/types/models/video/video-change-ownership.js' +import { AllowNull, BelongsTo, Column, CreatedAt, ForeignKey, Model, Scopes, Table, UpdatedAt } from 'sequelize-typescript' +import { AccountModel } from '../account/account.js' +import { getSort } from '../shared/index.js' +import { VideoModel, ScopeNames as VideoScopeNames } from './video.js' + +enum ScopeNames { + WITH_ACCOUNTS = 'WITH_ACCOUNTS', + WITH_VIDEO = 'WITH_VIDEO' +} + +@Table({ + tableName: 'videoChangeOwnership', + indexes: [ + { + fields: [ 'videoId' ] + }, + { + fields: [ 'initiatorAccountId' ] + }, + { + fields: [ 'nextOwnerAccountId' ] + } + ] +}) +@Scopes(() => ({ + [ScopeNames.WITH_ACCOUNTS]: { + include: [ + { + model: AccountModel, + as: 'Initiator', + required: true + }, + { + model: AccountModel, + as: 'NextOwner', + required: true + } + ] + }, + [ScopeNames.WITH_VIDEO]: { + include: [ + { + model: VideoModel.scope([ + VideoScopeNames.WITH_THUMBNAILS, + VideoScopeNames.WITH_WEB_VIDEO_FILES, + VideoScopeNames.WITH_STREAMING_PLAYLISTS, + VideoScopeNames.WITH_ACCOUNT_DETAILS + ]), + required: true + } + ] + } +})) +export class VideoChangeOwnershipModel extends Model>> { + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(false) + @Column + status: VideoChangeOwnershipStatusType + + @ForeignKey(() => AccountModel) + @Column + initiatorAccountId: number + + @BelongsTo(() => AccountModel, { + foreignKey: { + name: 'initiatorAccountId', + allowNull: false + }, + onDelete: 'cascade' + }) + Initiator: Awaited + + @ForeignKey(() => AccountModel) + @Column + nextOwnerAccountId: number + + @BelongsTo(() => AccountModel, { + foreignKey: { + name: 'nextOwnerAccountId', + allowNull: false + }, + onDelete: 'cascade' + }) + NextOwner: Awaited + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + Video: Awaited + + static listForApi (nextOwnerId: number, start: number, count: number, sort: string) { + const query = { + offset: start, + limit: count, + order: getSort(sort), + where: { + nextOwnerAccountId: nextOwnerId + } + } + + return Promise.all([ + VideoChangeOwnershipModel.scope(ScopeNames.WITH_ACCOUNTS).count(query), + VideoChangeOwnershipModel.scope([ ScopeNames.WITH_ACCOUNTS, ScopeNames.WITH_VIDEO ]).findAll(query) + ]).then(([ count, rows ]) => ({ total: count, data: rows })) + } + + static load (id: number): Promise { + return VideoChangeOwnershipModel.scope([ ScopeNames.WITH_ACCOUNTS, ScopeNames.WITH_VIDEO ]) + .findByPk(id) + } + + toFormattedJSON (this: MVideoChangeOwnershipFormattable): VideoChangeOwnership { + return { + id: this.id, + status: this.status, + initiatorAccount: this.Initiator.toFormattedJSON(), + nextOwnerAccount: this.NextOwner.toFormattedJSON(), + video: this.Video.toFormattedJSON(), + createdAt: this.createdAt + } + } +} diff --git a/server/server/models/video/video-channel-sync.ts b/server/server/models/video/video-channel-sync.ts new file mode 100644 index 000000000..da2a99593 --- /dev/null +++ b/server/server/models/video/video-channel-sync.ts @@ -0,0 +1,176 @@ +import { VideoChannelSync, VideoChannelSyncState, type VideoChannelSyncStateType } from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js' +import { isVideoChannelSyncStateValid } from '@server/helpers/custom-validators/video-channel-syncs.js' +import { CONSTRAINTS_FIELDS, VIDEO_CHANNEL_SYNC_STATE } from '@server/initializers/constants.js' +import { MChannelSync, MChannelSyncChannel, MChannelSyncFormattable } from '@server/types/models/index.js' +import { Op } from 'sequelize' +import { + AllowNull, + BelongsTo, + Column, + CreatedAt, + DataType, + Default, + DefaultScope, + ForeignKey, + Is, + Model, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { AccountModel } from '../account/account.js' +import { getChannelSyncSort, throwIfNotValid } from '../shared/index.js' +import { UserModel } from '../user/user.js' +import { VideoChannelModel } from './video-channel.js' + +@DefaultScope(() => ({ + include: [ + { + model: VideoChannelModel, // Default scope includes avatar and server + required: true + } + ] +})) +@Table({ + tableName: 'videoChannelSync', + indexes: [ + { + fields: [ 'videoChannelId' ] + } + ] +}) +export class VideoChannelSyncModel extends Model>> { + + @AllowNull(false) + @Default(null) + @Is('VideoChannelExternalChannelUrl', value => throwIfNotValid(value, isUrlValid, 'externalChannelUrl', true)) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_CHANNEL_SYNCS.EXTERNAL_CHANNEL_URL.max)) + externalChannelUrl: string + + @AllowNull(false) + @Default(VideoChannelSyncState.WAITING_FIRST_RUN) + @Is('VideoChannelSyncState', value => throwIfNotValid(value, isVideoChannelSyncStateValid, 'state')) + @Column + state: VideoChannelSyncStateType + + @AllowNull(true) + @Column(DataType.DATE) + lastSyncAt: Date + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => VideoChannelModel) + @Column + videoChannelId: number + + @BelongsTo(() => VideoChannelModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + VideoChannel: Awaited + + static listByAccountForAPI (options: { + accountId: number + start: number + count: number + sort: string + }) { + const getQuery = (forCount: boolean) => { + const videoChannelModel = forCount + ? VideoChannelModel.unscoped() + : VideoChannelModel + + return { + offset: options.start, + limit: options.count, + order: getChannelSyncSort(options.sort), + include: [ + { + model: videoChannelModel, + required: true, + where: { + accountId: options.accountId + } + } + ] + } + } + + return Promise.all([ + VideoChannelSyncModel.unscoped().count(getQuery(true)), + VideoChannelSyncModel.unscoped().findAll(getQuery(false)) + ]).then(([ total, data ]) => ({ total, data })) + } + + static countByAccount (accountId: number) { + const query = { + include: [ + { + model: VideoChannelModel.unscoped(), + required: true, + where: { + accountId + } + } + ] + } + + return VideoChannelSyncModel.unscoped().count(query) + } + + static loadWithChannel (id: number): Promise { + return VideoChannelSyncModel.findByPk(id) + } + + static async listSyncs (): Promise { + const query = { + include: [ + { + model: VideoChannelModel.unscoped(), + required: true, + include: [ + { + model: AccountModel.unscoped(), + required: true, + include: [ { + attributes: [], + model: UserModel.unscoped(), + required: true, + where: { + videoQuota: { + [Op.ne]: 0 + }, + videoQuotaDaily: { + [Op.ne]: 0 + } + } + } ] + } + ] + } + ] + } + return VideoChannelSyncModel.unscoped().findAll(query) + } + + toFormattedJSON (this: MChannelSyncFormattable): VideoChannelSync { + return { + id: this.id, + state: { + id: this.state, + label: VIDEO_CHANNEL_SYNC_STATE[this.state] + }, + externalChannelUrl: this.externalChannelUrl, + createdAt: this.createdAt.toISOString(), + channel: this.VideoChannel.toFormattedSummaryJSON(), + lastSyncAt: this.lastSyncAt?.toISOString() + } + } +} diff --git a/server/server/models/video/video-channel.ts b/server/server/models/video/video-channel.ts new file mode 100644 index 000000000..5a13fee24 --- /dev/null +++ b/server/server/models/video/video-channel.ts @@ -0,0 +1,859 @@ +import { forceNumber, pick } from '@peertube/peertube-core-utils' +import { ActivityPubActor, VideoChannel, VideoChannelSummary } from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { CONFIG } from '@server/initializers/config.js' +import { InternalEventEmitter } from '@server/lib/internal-event-emitter.js' +import { MAccountHost } from '@server/types/models/index.js' +import { FindOptions, Includeable, literal, Op, QueryTypes, ScopeOptions, Transaction, WhereOptions } from 'sequelize' +import { + AfterCreate, + AfterDestroy, + AfterUpdate, + AllowNull, + BeforeDestroy, + BelongsTo, + Column, + CreatedAt, + DataType, + Default, + DefaultScope, + ForeignKey, + HasMany, + Is, + Model, + Scopes, + Sequelize, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { + isVideoChannelDescriptionValid, + isVideoChannelDisplayNameValid, + isVideoChannelSupportValid +} from '../../helpers/custom-validators/video-channels.js' +import { CONSTRAINTS_FIELDS, WEBSERVER } from '../../initializers/constants.js' +import { sendDeleteActor } from '../../lib/activitypub/send/index.js' +import { + MChannelActor, + MChannelAP, + MChannelBannerAccountDefault, + MChannelFormattable, + MChannelHost, + MChannelSummaryFormattable, + type MChannel +} from '../../types/models/video/index.js' +import { AccountModel, ScopeNames as AccountModelScopeNames, SummaryOptions as AccountSummaryOptions } from '../account/account.js' +import { ActorFollowModel } from '../actor/actor-follow.js' +import { ActorImageModel } from '../actor/actor-image.js' +import { ActorModel, unusedActorAttributesForAPI } from '../actor/actor.js' +import { ServerModel } from '../server/server.js' +import { + buildServerIdsFollowedBy, + buildTrigramSearchIndex, + createSimilarityAttribute, + getSort, + setAsUpdated, + throwIfNotValid +} from '../shared/index.js' +import { VideoPlaylistModel } from './video-playlist.js' +import { VideoModel } from './video.js' + +export enum ScopeNames { + FOR_API = 'FOR_API', + SUMMARY = 'SUMMARY', + WITH_ACCOUNT = 'WITH_ACCOUNT', + WITH_ACTOR = 'WITH_ACTOR', + WITH_ACTOR_BANNER = 'WITH_ACTOR_BANNER', + WITH_VIDEOS = 'WITH_VIDEOS', + WITH_STATS = 'WITH_STATS' +} + +type AvailableForListOptions = { + actorId: number + search?: string + host?: string + handles?: string[] + forCount?: boolean +} + +type AvailableWithStatsOptions = { + daysPrior: number +} + +export type SummaryOptions = { + actorRequired?: boolean // Default: true + withAccount?: boolean // Default: false + withAccountBlockerIds?: number[] +} + +@DefaultScope(() => ({ + include: [ + { + model: ActorModel, + required: true + } + ] +})) +@Scopes(() => ({ + [ScopeNames.FOR_API]: (options: AvailableForListOptions) => { + // Only list local channels OR channels that are on an instance followed by actorId + const inQueryInstanceFollow = buildServerIdsFollowedBy(options.actorId) + + const whereActorAnd: WhereOptions[] = [ + { + [Op.or]: [ + { + serverId: null + }, + { + serverId: { + [Op.in]: Sequelize.literal(inQueryInstanceFollow) + } + } + ] + } + ] + + let serverRequired = false + let whereServer: WhereOptions + + if (options.host && options.host !== WEBSERVER.HOST) { + serverRequired = true + whereServer = { host: options.host } + } + + if (options.host === WEBSERVER.HOST) { + whereActorAnd.push({ + serverId: null + }) + } + + if (Array.isArray(options.handles) && options.handles.length !== 0) { + const or: string[] = [] + + for (const handle of options.handles || []) { + const [ preferredUsername, host ] = handle.split('@') + + const sanitizedPreferredUsername = VideoChannelModel.sequelize.escape(preferredUsername.toLowerCase()) + const sanitizedHost = VideoChannelModel.sequelize.escape(host) + + if (!host || host === WEBSERVER.HOST) { + or.push(`(LOWER("preferredUsername") = ${sanitizedPreferredUsername} AND "serverId" IS NULL)`) + } else { + or.push( + `(` + + `LOWER("preferredUsername") = ${sanitizedPreferredUsername} ` + + `AND "host" = ${sanitizedHost}` + + `)` + ) + } + } + + whereActorAnd.push({ + id: { + [Op.in]: literal(`(SELECT "actor".id FROM actor LEFT JOIN server on server.id = actor."serverId" WHERE ${or.join(' OR ')})`) + } + }) + } + + const channelActorInclude: Includeable[] = [] + const accountActorInclude: Includeable[] = [] + + if (options.forCount !== true) { + accountActorInclude.push({ + model: ServerModel, + required: false + }) + + accountActorInclude.push({ + model: ActorImageModel, + as: 'Avatars', + required: false + }) + + channelActorInclude.push({ + model: ActorImageModel, + as: 'Avatars', + required: false + }) + + channelActorInclude.push({ + model: ActorImageModel, + as: 'Banners', + required: false + }) + } + + if (options.forCount !== true || serverRequired) { + channelActorInclude.push({ + model: ServerModel, + duplicating: false, + required: serverRequired, + where: whereServer + }) + } + + return { + include: [ + { + attributes: { + exclude: unusedActorAttributesForAPI + }, + model: ActorModel.unscoped(), + where: { + [Op.and]: whereActorAnd + }, + include: channelActorInclude + }, + { + model: AccountModel.unscoped(), + required: true, + include: [ + { + attributes: { + exclude: unusedActorAttributesForAPI + }, + model: ActorModel.unscoped(), + required: true, + include: accountActorInclude + } + ] + } + ] + } + }, + [ScopeNames.SUMMARY]: (options: SummaryOptions = {}) => { + const include: Includeable[] = [ + { + attributes: [ 'id', 'preferredUsername', 'url', 'serverId' ], + model: ActorModel.unscoped(), + required: options.actorRequired ?? true, + include: [ + { + attributes: [ 'host' ], + model: ServerModel.unscoped(), + required: false + }, + { + model: ActorImageModel, + as: 'Avatars', + required: false + } + ] + } + ] + + const base: FindOptions = { + attributes: [ 'id', 'name', 'description', 'actorId' ] + } + + if (options.withAccount === true) { + include.push({ + model: AccountModel.scope({ + method: [ AccountModelScopeNames.SUMMARY, { withAccountBlockerIds: options.withAccountBlockerIds } as AccountSummaryOptions ] + }), + required: true + }) + } + + base.include = include + + return base + }, + [ScopeNames.WITH_ACCOUNT]: { + include: [ + { + model: AccountModel, + required: true + } + ] + }, + [ScopeNames.WITH_ACTOR]: { + include: [ + ActorModel + ] + }, + [ScopeNames.WITH_ACTOR_BANNER]: { + include: [ + { + model: ActorModel, + include: [ + { + model: ActorImageModel, + required: false, + as: 'Banners' + } + ] + } + ] + }, + [ScopeNames.WITH_VIDEOS]: { + include: [ + VideoModel + ] + }, + [ScopeNames.WITH_STATS]: (options: AvailableWithStatsOptions = { daysPrior: 30 }) => { + const daysPrior = forceNumber(options.daysPrior) + + return { + attributes: { + include: [ + [ + literal('(SELECT COUNT(*) FROM "video" WHERE "channelId" = "VideoChannelModel"."id")'), + 'videosCount' + ], + [ + literal( + '(' + + `SELECT string_agg(concat_ws('|', t.day, t.views), ',') ` + + 'FROM ( ' + + 'WITH ' + + 'days AS ( ' + + `SELECT generate_series(date_trunc('day', now()) - '${daysPrior} day'::interval, ` + + `date_trunc('day', now()), '1 day'::interval) AS day ` + + ') ' + + 'SELECT days.day AS day, COALESCE(SUM("videoView".views), 0) AS views ' + + 'FROM days ' + + 'LEFT JOIN (' + + '"videoView" INNER JOIN "video" ON "videoView"."videoId" = "video"."id" ' + + 'AND "video"."channelId" = "VideoChannelModel"."id"' + + `) ON date_trunc('day', "videoView"."startDate") = date_trunc('day', days.day) ` + + 'GROUP BY day ' + + 'ORDER BY day ' + + ') t' + + ')' + ), + 'viewsPerDay' + ], + [ + literal( + '(' + + 'SELECT COALESCE(SUM("video".views), 0) AS totalViews ' + + 'FROM "video" ' + + 'WHERE "video"."channelId" = "VideoChannelModel"."id"' + + ')' + ), + 'totalViews' + ] + ] + } + } + } +})) +@Table({ + tableName: 'videoChannel', + indexes: [ + buildTrigramSearchIndex('video_channel_name_trigram', 'name'), + + { + fields: [ 'accountId' ] + }, + { + fields: [ 'actorId' ] + } + ] +}) +export class VideoChannelModel extends Model>> { + + @AllowNull(false) + @Is('VideoChannelName', value => throwIfNotValid(value, isVideoChannelDisplayNameValid, 'name')) + @Column + name: string + + @AllowNull(true) + @Default(null) + @Is('VideoChannelDescription', value => throwIfNotValid(value, isVideoChannelDescriptionValid, 'description', true)) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_CHANNELS.DESCRIPTION.max)) + description: string + + @AllowNull(true) + @Default(null) + @Is('VideoChannelSupport', value => throwIfNotValid(value, isVideoChannelSupportValid, 'support', true)) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_CHANNELS.SUPPORT.max)) + support: string + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => ActorModel) + @Column + actorId: number + + @BelongsTo(() => ActorModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + Actor: Awaited + + @ForeignKey(() => AccountModel) + @Column + accountId: number + + @BelongsTo(() => AccountModel, { + foreignKey: { + allowNull: false + } + }) + Account: Awaited + + @HasMany(() => VideoModel, { + foreignKey: { + name: 'channelId', + allowNull: false + }, + onDelete: 'CASCADE', + hooks: true + }) + Videos: Awaited[] + + @HasMany(() => VideoPlaylistModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'CASCADE', + hooks: true + }) + VideoPlaylists: Awaited[] + + @AfterCreate + static notifyCreate (channel: MChannel) { + InternalEventEmitter.Instance.emit('channel-created', { channel }) + } + + @AfterUpdate + static notifyUpdate (channel: MChannel) { + InternalEventEmitter.Instance.emit('channel-updated', { channel }) + } + + @AfterDestroy + static notifyDestroy (channel: MChannel) { + InternalEventEmitter.Instance.emit('channel-deleted', { channel }) + } + + @BeforeDestroy + static async sendDeleteIfOwned (instance: VideoChannelModel, options) { + if (!instance.Actor) { + instance.Actor = await instance.$get('Actor', { transaction: options.transaction }) + } + + await ActorFollowModel.removeFollowsOf(instance.Actor.id, options.transaction) + + if (instance.Actor.isOwned()) { + return sendDeleteActor(instance.Actor, options.transaction) + } + + return undefined + } + + static countByAccount (accountId: number) { + const query = { + where: { + accountId + } + } + + return VideoChannelModel.unscoped().count(query) + } + + static async getStats () { + + function getLocalVideoChannelStats (days?: number) { + const options = { + type: QueryTypes.SELECT as QueryTypes.SELECT, + raw: true + } + + const videoJoin = days + ? `INNER JOIN "video" AS "Videos" ON "VideoChannelModel"."id" = "Videos"."channelId" ` + + `AND ("Videos"."publishedAt" > Now() - interval '${days}d')` + : '' + + const query = ` + SELECT COUNT(DISTINCT("VideoChannelModel"."id")) AS "count" + FROM "videoChannel" AS "VideoChannelModel" + ${videoJoin} + INNER JOIN "account" AS "Account" ON "VideoChannelModel"."accountId" = "Account"."id" + INNER JOIN "actor" AS "Account->Actor" ON "Account"."actorId" = "Account->Actor"."id" + AND "Account->Actor"."serverId" IS NULL` + + return VideoChannelModel.sequelize.query<{ count: string }>(query, options) + .then(r => parseInt(r[0].count, 10)) + } + + const totalLocalVideoChannels = await getLocalVideoChannelStats() + const totalLocalDailyActiveVideoChannels = await getLocalVideoChannelStats(1) + const totalLocalWeeklyActiveVideoChannels = await getLocalVideoChannelStats(7) + const totalLocalMonthlyActiveVideoChannels = await getLocalVideoChannelStats(30) + const totalLocalHalfYearActiveVideoChannels = await getLocalVideoChannelStats(180) + + return { + totalLocalVideoChannels, + totalLocalDailyActiveVideoChannels, + totalLocalWeeklyActiveVideoChannels, + totalLocalMonthlyActiveVideoChannels, + totalLocalHalfYearActiveVideoChannels + } + } + + static listLocalsForSitemap (sort: string): Promise { + const query = { + attributes: [ ], + offset: 0, + order: getSort(sort), + include: [ + { + attributes: [ 'preferredUsername', 'serverId' ], + model: ActorModel.unscoped(), + where: { + serverId: null + } + } + ] + } + + return VideoChannelModel + .unscoped() + .findAll(query) + } + + static listForApi (parameters: Pick & { + start: number + count: number + sort: string + }) { + const { actorId } = parameters + + const query = { + offset: parameters.start, + limit: parameters.count, + order: getSort(parameters.sort) + } + + const getScope = (forCount: boolean) => { + return { method: [ ScopeNames.FOR_API, { actorId, forCount } as AvailableForListOptions ] } + } + + return Promise.all([ + VideoChannelModel.scope(getScope(true)).count(), + VideoChannelModel.scope(getScope(false)).findAll(query) + ]).then(([ total, data ]) => ({ total, data })) + } + + static searchForApi (options: Pick & { + start: number + count: number + sort: string + }) { + let attributesInclude: any[] = [ literal('0 as similarity') ] + let where: WhereOptions + + if (options.search) { + const escapedSearch = VideoChannelModel.sequelize.escape(options.search) + const escapedLikeSearch = VideoChannelModel.sequelize.escape('%' + options.search + '%') + attributesInclude = [ createSimilarityAttribute('VideoChannelModel.name', options.search) ] + + where = { + [Op.or]: [ + Sequelize.literal( + 'lower(immutable_unaccent("VideoChannelModel"."name")) % lower(immutable_unaccent(' + escapedSearch + '))' + ), + Sequelize.literal( + 'lower(immutable_unaccent("VideoChannelModel"."name")) LIKE lower(immutable_unaccent(' + escapedLikeSearch + '))' + ) + ] + } + } + + const query = { + attributes: { + include: attributesInclude + }, + offset: options.start, + limit: options.count, + order: getSort(options.sort), + where + } + + const getScope = (forCount: boolean) => { + return { + method: [ + ScopeNames.FOR_API, { + ...pick(options, [ 'actorId', 'host', 'handles' ]), + + forCount + } as AvailableForListOptions + ] + } + } + + return Promise.all([ + VideoChannelModel.scope(getScope(true)).count(query), + VideoChannelModel.scope(getScope(false)).findAll(query) + ]).then(([ total, data ]) => ({ total, data })) + } + + static listByAccountForAPI (options: { + accountId: number + start: number + count: number + sort: string + withStats?: boolean + search?: string + }) { + const escapedSearch = VideoModel.sequelize.escape(options.search) + const escapedLikeSearch = VideoModel.sequelize.escape('%' + options.search + '%') + const where = options.search + ? { + [Op.or]: [ + Sequelize.literal( + 'lower(immutable_unaccent("VideoChannelModel"."name")) % lower(immutable_unaccent(' + escapedSearch + '))' + ), + Sequelize.literal( + 'lower(immutable_unaccent("VideoChannelModel"."name")) LIKE lower(immutable_unaccent(' + escapedLikeSearch + '))' + ) + ] + } + : null + + const getQuery = (forCount: boolean) => { + const accountModel = forCount + ? AccountModel.unscoped() + : AccountModel + + return { + offset: options.start, + limit: options.count, + order: getSort(options.sort), + include: [ + { + model: accountModel, + where: { + id: options.accountId + }, + required: true + } + ], + where + } + } + + const findScopes: string | ScopeOptions | (string | ScopeOptions)[] = [ ScopeNames.WITH_ACTOR_BANNER ] + + if (options.withStats === true) { + findScopes.push({ + method: [ ScopeNames.WITH_STATS, { daysPrior: 30 } as AvailableWithStatsOptions ] + }) + } + + return Promise.all([ + VideoChannelModel.unscoped().count(getQuery(true)), + VideoChannelModel.scope(findScopes).findAll(getQuery(false)) + ]).then(([ total, data ]) => ({ total, data })) + } + + static listAllByAccount (accountId: number): Promise { + const query = { + limit: CONFIG.VIDEO_CHANNELS.MAX_PER_USER, + include: [ + { + attributes: [], + model: AccountModel.unscoped(), + where: { + id: accountId + }, + required: true + } + ] + } + + return VideoChannelModel.findAll(query) + } + + static loadAndPopulateAccount (id: number, transaction?: Transaction): Promise { + return VideoChannelModel.unscoped() + .scope([ ScopeNames.WITH_ACTOR_BANNER, ScopeNames.WITH_ACCOUNT ]) + .findByPk(id, { transaction }) + } + + static loadByUrlAndPopulateAccount (url: string): Promise { + const query = { + include: [ + { + model: ActorModel, + required: true, + where: { + url + }, + include: [ + { + model: ActorImageModel, + required: false, + as: 'Banners' + } + ] + } + ] + } + + return VideoChannelModel + .scope([ ScopeNames.WITH_ACCOUNT ]) + .findOne(query) + } + + static loadByNameWithHostAndPopulateAccount (nameWithHost: string) { + const [ name, host ] = nameWithHost.split('@') + + if (!host || host === WEBSERVER.HOST) return VideoChannelModel.loadLocalByNameAndPopulateAccount(name) + + return VideoChannelModel.loadByNameAndHostAndPopulateAccount(name, host) + } + + static loadLocalByNameAndPopulateAccount (name: string): Promise { + const query = { + include: [ + { + model: ActorModel, + required: true, + where: { + [Op.and]: [ + ActorModel.wherePreferredUsername(name, 'Actor.preferredUsername'), + { serverId: null } + ] + }, + include: [ + { + model: ActorImageModel, + required: false, + as: 'Banners' + } + ] + } + ] + } + + return VideoChannelModel.unscoped() + .scope([ ScopeNames.WITH_ACCOUNT ]) + .findOne(query) + } + + static loadByNameAndHostAndPopulateAccount (name: string, host: string): Promise { + const query = { + include: [ + { + model: ActorModel, + required: true, + where: ActorModel.wherePreferredUsername(name, 'Actor.preferredUsername'), + include: [ + { + model: ServerModel, + required: true, + where: { host } + }, + { + model: ActorImageModel, + required: false, + as: 'Banners' + } + ] + } + ] + } + + return VideoChannelModel.unscoped() + .scope([ ScopeNames.WITH_ACCOUNT ]) + .findOne(query) + } + + toFormattedSummaryJSON (this: MChannelSummaryFormattable): VideoChannelSummary { + const actor = this.Actor.toFormattedSummaryJSON() + + return { + id: this.id, + name: actor.name, + displayName: this.getDisplayName(), + url: actor.url, + host: actor.host, + avatars: actor.avatars + } + } + + toFormattedJSON (this: MChannelFormattable): VideoChannel { + const viewsPerDayString = this.get('viewsPerDay') as string + const videosCount = this.get('videosCount') as number + + let viewsPerDay: { date: Date, views: number }[] + + if (viewsPerDayString) { + viewsPerDay = viewsPerDayString.split(',') + .map(v => { + const [ dateString, amount ] = v.split('|') + + return { + date: new Date(dateString), + views: +amount + } + }) + } + + const totalViews = this.get('totalViews') as number + + const actor = this.Actor.toFormattedJSON() + const videoChannel = { + id: this.id, + displayName: this.getDisplayName(), + description: this.description, + support: this.support, + isLocal: this.Actor.isOwned(), + updatedAt: this.updatedAt, + + ownerAccount: undefined, + + videosCount, + viewsPerDay, + totalViews, + + avatars: actor.avatars + } + + if (this.Account) videoChannel.ownerAccount = this.Account.toFormattedJSON() + + return Object.assign(actor, videoChannel) + } + + async toActivityPubObject (this: MChannelAP): Promise { + const obj = await this.Actor.toActivityPubObject(this.name) + + return Object.assign(obj, { + summary: this.description, + support: this.support, + attributedTo: [ + { + type: 'Person' as 'Person', + id: this.Account.Actor.url + } + ] + }) + } + + // Avoid error when running this method on MAccount... | MChannel... + getClientUrl (this: MAccountHost | MChannelHost) { + return WEBSERVER.URL + '/c/' + this.Actor.getIdentifier() + } + + getDisplayName () { + return this.name + } + + isOutdated () { + return this.Actor.isOutdated() + } + + setAsUpdated (transaction?: Transaction) { + return setAsUpdated({ sequelize: this.sequelize, table: 'videoChannel', id: this.id, transaction }) + } +} diff --git a/server/server/models/video/video-comment.ts b/server/server/models/video/video-comment.ts new file mode 100644 index 000000000..001ef85bb --- /dev/null +++ b/server/server/models/video/video-comment.ts @@ -0,0 +1,646 @@ +import { FindOptions, Op, Order, QueryTypes, Sequelize, Transaction } from 'sequelize' +import { + AllowNull, + BelongsTo, + Column, + CreatedAt, + DataType, + ForeignKey, + HasMany, + Is, + Model, + Scopes, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { pick } from '@peertube/peertube-core-utils' +import { ActivityTagObject, ActivityTombstoneObject, VideoComment, VideoCommentAdmin, VideoCommentObject } from '@peertube/peertube-models' +import { extractMentions } from '@server/helpers/mentions.js' +import { getServerActor } from '@server/models/application/application.js' +import { MAccount, MAccountId, MUserAccountId } from '@server/types/models/index.js' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc.js' +import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js' +import { + MComment, + MCommentAdminFormattable, + MCommentAP, + MCommentFormattable, + MCommentId, + MCommentOwner, + MCommentOwnerReplyVideoLight, + MCommentOwnerVideo, + MCommentOwnerVideoFeed, + MCommentOwnerVideoReply, + MVideoImmutable +} from '../../types/models/video/index.js' +import { VideoCommentAbuseModel } from '../abuse/video-comment-abuse.js' +import { AccountModel } from '../account/account.js' +import { ActorModel } from '../actor/actor.js' +import { buildLocalAccountIdsIn, buildSQLAttributes, throwIfNotValid } from '../shared/index.js' +import { ListVideoCommentsOptions, VideoCommentListQueryBuilder } from './sql/comment/video-comment-list-query-builder.js' +import { VideoChannelModel } from './video-channel.js' +import { VideoModel } from './video.js' + +export enum ScopeNames { + WITH_ACCOUNT = 'WITH_ACCOUNT', + WITH_IN_REPLY_TO = 'WITH_IN_REPLY_TO', + WITH_VIDEO = 'WITH_VIDEO' +} + +@Scopes(() => ({ + [ScopeNames.WITH_ACCOUNT]: { + include: [ + { + model: AccountModel + } + ] + }, + [ScopeNames.WITH_IN_REPLY_TO]: { + include: [ + { + model: VideoCommentModel, + as: 'InReplyToVideoComment' + } + ] + }, + [ScopeNames.WITH_VIDEO]: { + include: [ + { + model: VideoModel, + required: true, + include: [ + { + model: VideoChannelModel, + required: true, + include: [ + { + model: AccountModel, + required: true + } + ] + } + ] + } + ] + } +})) +@Table({ + tableName: 'videoComment', + indexes: [ + { + fields: [ 'videoId' ] + }, + { + fields: [ 'videoId', 'originCommentId' ] + }, + { + fields: [ 'url' ], + unique: true + }, + { + fields: [ 'accountId' ] + }, + { + fields: [ + { name: 'createdAt', order: 'DESC' } + ] + } + ] +}) +export class VideoCommentModel extends Model>> { + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(true) + @Column(DataType.DATE) + deletedAt: Date + + @AllowNull(false) + @Is('VideoCommentUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url')) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max)) + url: string + + @AllowNull(false) + @Column(DataType.TEXT) + text: string + + @ForeignKey(() => VideoCommentModel) + @Column + originCommentId: number + + @BelongsTo(() => VideoCommentModel, { + foreignKey: { + name: 'originCommentId', + allowNull: true + }, + as: 'OriginVideoComment', + onDelete: 'CASCADE' + }) + OriginVideoComment: Awaited + + @ForeignKey(() => VideoCommentModel) + @Column + inReplyToCommentId: number + + @BelongsTo(() => VideoCommentModel, { + foreignKey: { + name: 'inReplyToCommentId', + allowNull: true + }, + as: 'InReplyToVideoComment', + onDelete: 'CASCADE' + }) + InReplyToVideoComment: Awaited | null + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + Video: Awaited + + @ForeignKey(() => AccountModel) + @Column + accountId: number + + @BelongsTo(() => AccountModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'CASCADE' + }) + Account: Awaited + + @HasMany(() => VideoCommentAbuseModel, { + foreignKey: { + name: 'videoCommentId', + allowNull: true + }, + onDelete: 'set null' + }) + CommentAbuses: Awaited[] + + // --------------------------------------------------------------------------- + + static getSQLAttributes (tableName: string, aliasPrefix = '') { + return buildSQLAttributes({ + model: this, + tableName, + aliasPrefix + }) + } + + // --------------------------------------------------------------------------- + + static loadById (id: number, t?: Transaction): Promise { + const query: FindOptions = { + where: { + id + } + } + + if (t !== undefined) query.transaction = t + + return VideoCommentModel.findOne(query) + } + + static loadByIdAndPopulateVideoAndAccountAndReply (id: number, t?: Transaction): Promise { + const query: FindOptions = { + where: { + id + } + } + + if (t !== undefined) query.transaction = t + + return VideoCommentModel + .scope([ ScopeNames.WITH_VIDEO, ScopeNames.WITH_ACCOUNT, ScopeNames.WITH_IN_REPLY_TO ]) + .findOne(query) + } + + static loadByUrlAndPopulateAccountAndVideo (url: string, t?: Transaction): Promise { + const query: FindOptions = { + where: { + url + } + } + + if (t !== undefined) query.transaction = t + + return VideoCommentModel.scope([ ScopeNames.WITH_ACCOUNT, ScopeNames.WITH_VIDEO ]).findOne(query) + } + + static loadByUrlAndPopulateReplyAndVideoUrlAndAccount (url: string, t?: Transaction): Promise { + const query: FindOptions = { + where: { + url + }, + include: [ + { + attributes: [ 'id', 'url' ], + model: VideoModel.unscoped() + } + ] + } + + if (t !== undefined) query.transaction = t + + return VideoCommentModel.scope([ ScopeNames.WITH_IN_REPLY_TO, ScopeNames.WITH_ACCOUNT ]).findOne(query) + } + + static listCommentsForApi (parameters: { + start: number + count: number + sort: string + + onLocalVideo?: boolean + isLocal?: boolean + search?: string + searchAccount?: string + searchVideo?: string + }) { + const queryOptions: ListVideoCommentsOptions = { + ...pick(parameters, [ 'start', 'count', 'sort', 'isLocal', 'search', 'searchVideo', 'searchAccount', 'onLocalVideo' ]), + + selectType: 'api', + notDeleted: true + } + + return Promise.all([ + new VideoCommentListQueryBuilder(VideoCommentModel.sequelize, queryOptions).listComments(), + new VideoCommentListQueryBuilder(VideoCommentModel.sequelize, queryOptions).countComments() + ]).then(([ rows, count ]) => { + return { total: count, data: rows } + }) + } + + static async listThreadsForApi (parameters: { + videoId: number + isVideoOwned: boolean + start: number + count: number + sort: string + user?: MUserAccountId + }) { + const { videoId, user } = parameters + + const blockerAccountIds = await VideoCommentModel.buildBlockerAccountIds({ user }) + + const commonOptions: ListVideoCommentsOptions = { + selectType: 'api', + videoId, + blockerAccountIds + } + + const listOptions: ListVideoCommentsOptions = { + ...commonOptions, + ...pick(parameters, [ 'sort', 'start', 'count' ]), + + isThread: true, + includeReplyCounters: true + } + + const countOptions: ListVideoCommentsOptions = { + ...commonOptions, + + isThread: true + } + + const notDeletedCountOptions: ListVideoCommentsOptions = { + ...commonOptions, + + notDeleted: true + } + + return Promise.all([ + new VideoCommentListQueryBuilder(VideoCommentModel.sequelize, listOptions).listComments(), + new VideoCommentListQueryBuilder(VideoCommentModel.sequelize, countOptions).countComments(), + new VideoCommentListQueryBuilder(VideoCommentModel.sequelize, notDeletedCountOptions).countComments() + ]).then(([ rows, count, totalNotDeletedComments ]) => { + return { total: count, data: rows, totalNotDeletedComments } + }) + } + + static async listThreadCommentsForApi (parameters: { + videoId: number + threadId: number + user?: MUserAccountId + }) { + const { user } = parameters + + const blockerAccountIds = await VideoCommentModel.buildBlockerAccountIds({ user }) + + const queryOptions: ListVideoCommentsOptions = { + ...pick(parameters, [ 'videoId', 'threadId' ]), + + selectType: 'api', + sort: 'createdAt', + + blockerAccountIds, + includeReplyCounters: true + } + + return Promise.all([ + new VideoCommentListQueryBuilder(VideoCommentModel.sequelize, queryOptions).listComments(), + new VideoCommentListQueryBuilder(VideoCommentModel.sequelize, queryOptions).countComments() + ]).then(([ rows, count ]) => { + return { total: count, data: rows } + }) + } + + static listThreadParentComments (comment: MCommentId, t: Transaction, order: 'ASC' | 'DESC' = 'ASC'): Promise { + const query = { + order: [ [ 'createdAt', order ] ] as Order, + where: { + id: { + [Op.in]: Sequelize.literal('(' + + 'WITH RECURSIVE children (id, "inReplyToCommentId") AS ( ' + + `SELECT id, "inReplyToCommentId" FROM "videoComment" WHERE id = ${comment.id} ` + + 'UNION ' + + 'SELECT "parent"."id", "parent"."inReplyToCommentId" FROM "videoComment" "parent" ' + + 'INNER JOIN "children" ON "children"."inReplyToCommentId" = "parent"."id"' + + ') ' + + 'SELECT id FROM children' + + ')'), + [Op.ne]: comment.id + } + }, + transaction: t + } + + return VideoCommentModel + .scope([ ScopeNames.WITH_ACCOUNT ]) + .findAll(query) + } + + static async listAndCountByVideoForAP (parameters: { + video: MVideoImmutable + start: number + count: number + }) { + const { video } = parameters + + const blockerAccountIds = await VideoCommentModel.buildBlockerAccountIds({ user: null }) + + const queryOptions: ListVideoCommentsOptions = { + ...pick(parameters, [ 'start', 'count' ]), + + selectType: 'comment-only', + videoId: video.id, + sort: 'createdAt', + + blockerAccountIds + } + + return Promise.all([ + new VideoCommentListQueryBuilder(VideoCommentModel.sequelize, queryOptions).listComments(), + new VideoCommentListQueryBuilder(VideoCommentModel.sequelize, queryOptions).countComments() + ]).then(([ rows, count ]) => { + return { total: count, data: rows } + }) + } + + static async listForFeed (parameters: { + start: number + count: number + videoId?: number + accountId?: number + videoChannelId?: number + }) { + const blockerAccountIds = await VideoCommentModel.buildBlockerAccountIds({ user: null }) + + const queryOptions: ListVideoCommentsOptions = { + ...pick(parameters, [ 'start', 'count', 'accountId', 'videoId', 'videoChannelId' ]), + + selectType: 'feed', + + sort: '-createdAt', + onPublicVideo: true, + notDeleted: true, + + blockerAccountIds + } + + return new VideoCommentListQueryBuilder(VideoCommentModel.sequelize, queryOptions).listComments() + } + + static listForBulkDelete (ofAccount: MAccount, filter: { onVideosOfAccount?: MAccountId } = {}) { + const queryOptions: ListVideoCommentsOptions = { + selectType: 'comment-only', + + accountId: ofAccount.id, + videoAccountOwnerId: filter.onVideosOfAccount?.id, + + notDeleted: true, + count: 5000 + } + + return new VideoCommentListQueryBuilder(VideoCommentModel.sequelize, queryOptions).listComments() + } + + static async getStats () { + const totalLocalVideoComments = await VideoCommentModel.count({ + include: [ + { + model: AccountModel.unscoped(), + required: true, + include: [ + { + model: ActorModel.unscoped(), + required: true, + where: { + serverId: null + } + } + ] + } + ] + }) + const totalVideoComments = await VideoCommentModel.count() + + return { + totalLocalVideoComments, + totalVideoComments + } + } + + static listRemoteCommentUrlsOfLocalVideos () { + const query = `SELECT "videoComment".url FROM "videoComment" ` + + `INNER JOIN account ON account.id = "videoComment"."accountId" ` + + `INNER JOIN actor ON actor.id = "account"."actorId" AND actor."serverId" IS NOT NULL ` + + `INNER JOIN video ON video.id = "videoComment"."videoId" AND video.remote IS FALSE` + + return VideoCommentModel.sequelize.query<{ url: string }>(query, { + type: QueryTypes.SELECT, + raw: true + }).then(rows => rows.map(r => r.url)) + } + + static cleanOldCommentsOf (videoId: number, beforeUpdatedAt: Date) { + const query = { + where: { + updatedAt: { + [Op.lt]: beforeUpdatedAt + }, + videoId, + accountId: { + [Op.notIn]: buildLocalAccountIdsIn() + }, + // Do not delete Tombstones + deletedAt: null + } + } + + return VideoCommentModel.destroy(query) + } + + getCommentStaticPath () { + return this.Video.getWatchStaticPath() + ';threadId=' + this.getThreadId() + } + + getThreadId (): number { + return this.originCommentId || this.id + } + + isOwned () { + if (!this.Account) return false + + return this.Account.isOwned() + } + + markAsDeleted () { + this.text = '' + this.deletedAt = new Date() + this.accountId = null + } + + isDeleted () { + return this.deletedAt !== null + } + + extractMentions () { + return extractMentions(this.text, this.isOwned()) + } + + toFormattedJSON (this: MCommentFormattable) { + return { + id: this.id, + url: this.url, + text: this.text, + + threadId: this.getThreadId(), + inReplyToCommentId: this.inReplyToCommentId || null, + videoId: this.videoId, + + createdAt: this.createdAt, + updatedAt: this.updatedAt, + deletedAt: this.deletedAt, + + isDeleted: this.isDeleted(), + + totalRepliesFromVideoAuthor: this.get('totalRepliesFromVideoAuthor') || 0, + totalReplies: this.get('totalReplies') || 0, + + account: this.Account + ? this.Account.toFormattedJSON() + : null + } as VideoComment + } + + toFormattedAdminJSON (this: MCommentAdminFormattable) { + return { + id: this.id, + url: this.url, + text: this.text, + + threadId: this.getThreadId(), + inReplyToCommentId: this.inReplyToCommentId || null, + videoId: this.videoId, + + createdAt: this.createdAt, + updatedAt: this.updatedAt, + + video: { + id: this.Video.id, + uuid: this.Video.uuid, + name: this.Video.name + }, + + account: this.Account + ? this.Account.toFormattedJSON() + : null + } as VideoCommentAdmin + } + + toActivityPubObject (this: MCommentAP, threadParentComments: MCommentOwner[]): VideoCommentObject | ActivityTombstoneObject { + let inReplyTo: string + // New thread, so in AS we reply to the video + if (this.inReplyToCommentId === null) { + inReplyTo = this.Video.url + } else { + inReplyTo = this.InReplyToVideoComment.url + } + + if (this.isDeleted()) { + return { + id: this.url, + type: 'Tombstone', + formerType: 'Note', + inReplyTo, + published: this.createdAt.toISOString(), + updated: this.updatedAt.toISOString(), + deleted: this.deletedAt.toISOString() + } + } + + const tag: ActivityTagObject[] = [] + for (const parentComment of threadParentComments) { + if (!parentComment.Account) continue + + const actor = parentComment.Account.Actor + + tag.push({ + type: 'Mention', + href: actor.url, + name: `@${actor.preferredUsername}@${actor.getHost()}` + }) + } + + return { + type: 'Note' as 'Note', + id: this.url, + + content: this.text, + mediaType: 'text/markdown', + + inReplyTo, + updated: this.updatedAt.toISOString(), + published: this.createdAt.toISOString(), + url: this.url, + attributedTo: this.Account.Actor.url, + tag + } + } + + private static async buildBlockerAccountIds (options: { + user: MUserAccountId + }): Promise { + const { user } = options + + const serverActor = await getServerActor() + const blockerAccountIds = [ serverActor.Account.id ] + + if (user) blockerAccountIds.push(user.Account.id) + + return blockerAccountIds + } +} diff --git a/server/server/models/video/video-file.ts b/server/server/models/video/video-file.ts new file mode 100644 index 000000000..735bda2d5 --- /dev/null +++ b/server/server/models/video/video-file.ts @@ -0,0 +1,635 @@ +import { VideoResolution, VideoStorage, type VideoStorageType } from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { logger } from '@server/helpers/logger.js' +import { extractVideo } from '@server/helpers/video.js' +import { CONFIG } from '@server/initializers/config.js' +import { buildRemoteVideoBaseUrl } from '@server/lib/activitypub/url.js' +import { + getHLSPrivateFileUrl, + getHLSPublicFileUrl, + getWebVideoPrivateFileUrl, + getWebVideoPublicFileUrl +} from '@server/lib/object-storage/index.js' +import { getFSTorrentFilePath } from '@server/lib/paths.js' +import { isVideoInPrivateDirectory } from '@server/lib/video-privacy.js' +import { MStreamingPlaylistVideo, MVideo, MVideoWithHost, isStreamingPlaylist } from '@server/types/models/index.js' +import { remove } from 'fs-extra/esm' +import memoizee from 'memoizee' +import { join } from 'path' +import { FindOptions, Op, Transaction, WhereOptions } from 'sequelize' +import { + AllowNull, + BelongsTo, + Column, + CreatedAt, + DataType, + Default, + DefaultScope, + ForeignKey, + HasMany, + Is, + Model, + Scopes, + Table, + UpdatedAt +} from 'sequelize-typescript' +import validator from 'validator' +import { + isVideoFPSResolutionValid, + isVideoFileExtnameValid, + isVideoFileInfoHashValid, + isVideoFileResolutionValid, + isVideoFileSizeValid +} from '../../helpers/custom-validators/videos.js' +import { + LAZY_STATIC_PATHS, + MEMOIZE_LENGTH, + MEMOIZE_TTL, + STATIC_DOWNLOAD_PATHS, + STATIC_PATHS, + WEBSERVER +} from '../../initializers/constants.js' +import { MVideoFile, MVideoFileStreamingPlaylistVideo, MVideoFileVideo } from '../../types/models/video/video-file.js' +import { VideoRedundancyModel } from '../redundancy/video-redundancy.js' +import { doesExist, parseAggregateResult, throwIfNotValid } from '../shared/index.js' +import { VideoStreamingPlaylistModel } from './video-streaming-playlist.js' +import { VideoModel } from './video.js' + +export enum ScopeNames { + WITH_VIDEO = 'WITH_VIDEO', + WITH_METADATA = 'WITH_METADATA', + WITH_VIDEO_OR_PLAYLIST = 'WITH_VIDEO_OR_PLAYLIST' +} + +@DefaultScope(() => ({ + attributes: { + exclude: [ 'metadata' ] + } +})) +@Scopes(() => ({ + [ScopeNames.WITH_VIDEO]: { + include: [ + { + model: VideoModel.unscoped(), + required: true + } + ] + }, + [ScopeNames.WITH_VIDEO_OR_PLAYLIST]: (options: { whereVideo?: WhereOptions } = {}) => { + return { + include: [ + { + model: VideoModel.unscoped(), + required: false, + where: options.whereVideo + }, + { + model: VideoStreamingPlaylistModel.unscoped(), + required: false, + include: [ + { + model: VideoModel.unscoped(), + required: true, + where: options.whereVideo + } + ] + } + ] + } + }, + [ScopeNames.WITH_METADATA]: { + attributes: { + include: [ 'metadata' ] + } + } +})) +@Table({ + tableName: 'videoFile', + indexes: [ + { + fields: [ 'videoId' ], + where: { + videoId: { + [Op.ne]: null + } + } + }, + { + fields: [ 'videoStreamingPlaylistId' ], + where: { + videoStreamingPlaylistId: { + [Op.ne]: null + } + } + }, + + { + fields: [ 'infoHash' ] + }, + + { + fields: [ 'torrentFilename' ], + unique: true + }, + + { + fields: [ 'filename' ], + unique: true + }, + + { + fields: [ 'videoId', 'resolution', 'fps' ], + unique: true, + where: { + videoId: { + [Op.ne]: null + } + } + }, + { + fields: [ 'videoStreamingPlaylistId', 'resolution', 'fps' ], + unique: true, + where: { + videoStreamingPlaylistId: { + [Op.ne]: null + } + } + } + ] +}) +export class VideoFileModel extends Model>> { + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(false) + @Is('VideoFileResolution', value => throwIfNotValid(value, isVideoFileResolutionValid, 'resolution')) + @Column + resolution: number + + @AllowNull(false) + @Is('VideoFileSize', value => throwIfNotValid(value, isVideoFileSizeValid, 'size')) + @Column(DataType.BIGINT) + size: number + + @AllowNull(false) + @Is('VideoFileExtname', value => throwIfNotValid(value, isVideoFileExtnameValid, 'extname')) + @Column + extname: string + + @AllowNull(true) + @Is('VideoFileInfohash', value => throwIfNotValid(value, isVideoFileInfoHashValid, 'info hash', true)) + @Column + infoHash: string + + @AllowNull(false) + @Default(-1) + @Is('VideoFileFPS', value => throwIfNotValid(value, isVideoFPSResolutionValid, 'fps')) + @Column + fps: number + + @AllowNull(true) + @Column(DataType.JSONB) + metadata: any + + @AllowNull(true) + @Column + metadataUrl: string + + // Could be null for remote files + @AllowNull(true) + @Column + fileUrl: string + + // Could be null for live files + @AllowNull(true) + @Column + filename: string + + // Could be null for remote files + @AllowNull(true) + @Column + torrentUrl: string + + // Could be null for live files + @AllowNull(true) + @Column + torrentFilename: string + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @AllowNull(false) + @Default(VideoStorage.FILE_SYSTEM) + @Column + storage: VideoStorageType + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'CASCADE' + }) + Video: Awaited + + @ForeignKey(() => VideoStreamingPlaylistModel) + @Column + videoStreamingPlaylistId: number + + @BelongsTo(() => VideoStreamingPlaylistModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'CASCADE' + }) + VideoStreamingPlaylist: Awaited + + @HasMany(() => VideoRedundancyModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'CASCADE', + hooks: true + }) + RedundancyVideos: Awaited[] + + static doesInfohashExistCached = memoizee(VideoFileModel.doesInfohashExist, { + promise: true, + max: MEMOIZE_LENGTH.INFO_HASH_EXISTS, + maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS + }) + + static doesInfohashExist (infoHash: string) { + const query = 'SELECT 1 FROM "videoFile" WHERE "infoHash" = $infoHash LIMIT 1' + + return doesExist(this.sequelize, query, { infoHash }) + } + + static async doesVideoExistForVideoFile (id: number, videoIdOrUUID: number | string) { + const videoFile = await VideoFileModel.loadWithVideoOrPlaylist(id, videoIdOrUUID) + + return !!videoFile + } + + static async doesOwnedTorrentFileExist (filename: string) { + const query = 'SELECT 1 FROM "videoFile" ' + + 'LEFT JOIN "video" "webvideo" ON "webvideo"."id" = "videoFile"."videoId" AND "webvideo"."remote" IS FALSE ' + + 'LEFT JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoFile"."videoStreamingPlaylistId" ' + + 'LEFT JOIN "video" "hlsVideo" ON "hlsVideo"."id" = "videoStreamingPlaylist"."videoId" AND "hlsVideo"."remote" IS FALSE ' + + 'WHERE "torrentFilename" = $filename AND ("hlsVideo"."id" IS NOT NULL OR "webvideo"."id" IS NOT NULL) LIMIT 1' + + return doesExist(this.sequelize, query, { filename }) + } + + static async doesOwnedWebVideoFileExist (filename: string) { + const query = 'SELECT 1 FROM "videoFile" INNER JOIN "video" ON "video"."id" = "videoFile"."videoId" AND "video"."remote" IS FALSE ' + + `WHERE "filename" = $filename AND "storage" = ${VideoStorage.FILE_SYSTEM} LIMIT 1` + + return doesExist(this.sequelize, query, { filename }) + } + + static loadByFilename (filename: string) { + const query = { + where: { + filename + } + } + + return VideoFileModel.findOne(query) + } + + static loadWithVideoByFilename (filename: string): Promise { + const query = { + where: { + filename + } + } + + return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query) + } + + static loadWithVideoOrPlaylistByTorrentFilename (filename: string) { + const query = { + where: { + torrentFilename: filename + } + } + + return VideoFileModel.scope(ScopeNames.WITH_VIDEO_OR_PLAYLIST).findOne(query) + } + + static load (id: number): Promise { + return VideoFileModel.findByPk(id) + } + + static loadWithMetadata (id: number) { + return VideoFileModel.scope(ScopeNames.WITH_METADATA).findByPk(id) + } + + static loadWithVideo (id: number) { + return VideoFileModel.scope(ScopeNames.WITH_VIDEO).findByPk(id) + } + + static loadWithVideoOrPlaylist (id: number, videoIdOrUUID: number | string) { + const whereVideo = validator.default.isUUID(videoIdOrUUID + '') + ? { uuid: videoIdOrUUID } + : { id: videoIdOrUUID } + + const options = { + where: { + id + } + } + + return VideoFileModel.scope({ method: [ ScopeNames.WITH_VIDEO_OR_PLAYLIST, whereVideo ] }) + .findOne(options) + .then(file => { + // We used `required: false` so check we have at least a video or a streaming playlist + if (!file.Video && !file.VideoStreamingPlaylist) return null + + return file + }) + } + + static listByStreamingPlaylist (streamingPlaylistId: number, transaction: Transaction) { + const query = { + include: [ + { + model: VideoModel.unscoped(), + required: true, + include: [ + { + model: VideoStreamingPlaylistModel.unscoped(), + required: true, + where: { + id: streamingPlaylistId + } + } + ] + } + ], + transaction + } + + return VideoFileModel.findAll(query) + } + + static getStats () { + const webVideoFilesQuery: FindOptions = { + include: [ + { + attributes: [], + required: true, + model: VideoModel.unscoped(), + where: { + remote: false + } + } + ] + } + + const hlsFilesQuery: FindOptions = { + include: [ + { + attributes: [], + required: true, + model: VideoStreamingPlaylistModel.unscoped(), + include: [ + { + attributes: [], + model: VideoModel.unscoped(), + required: true, + where: { + remote: false + } + } + ] + } + ] + } + + return Promise.all([ + VideoFileModel.aggregate('size', 'SUM', webVideoFilesQuery), + VideoFileModel.aggregate('size', 'SUM', hlsFilesQuery) + ]).then(([ webVideoResult, hlsResult ]) => ({ + totalLocalVideoFilesSize: parseAggregateResult(webVideoResult) + parseAggregateResult(hlsResult) + })) + } + + // Redefine upsert because sequelize does not use an appropriate where clause in the update query with 2 unique indexes + static async customUpsert ( + videoFile: MVideoFile, + mode: 'streaming-playlist' | 'video', + transaction: Transaction + ) { + const baseFind = { + fps: videoFile.fps, + resolution: videoFile.resolution, + transaction + } + + const element = mode === 'streaming-playlist' + ? await VideoFileModel.loadHLSFile({ ...baseFind, playlistId: videoFile.videoStreamingPlaylistId }) + : await VideoFileModel.loadWebVideoFile({ ...baseFind, videoId: videoFile.videoId }) + + if (!element) return videoFile.save({ transaction }) + + for (const k of Object.keys(videoFile.toJSON())) { + element.set(k, videoFile[k]) + } + + return element.save({ transaction }) + } + + static async loadWebVideoFile (options: { + videoId: number + fps: number + resolution: number + transaction?: Transaction + }) { + const where = { + fps: options.fps, + resolution: options.resolution, + videoId: options.videoId + } + + return VideoFileModel.findOne({ where, transaction: options.transaction }) + } + + static async loadHLSFile (options: { + playlistId: number + fps: number + resolution: number + transaction?: Transaction + }) { + const where = { + fps: options.fps, + resolution: options.resolution, + videoStreamingPlaylistId: options.playlistId + } + + return VideoFileModel.findOne({ where, transaction: options.transaction }) + } + + static removeHLSFilesOfVideoId (videoStreamingPlaylistId: number) { + const options = { + where: { videoStreamingPlaylistId } + } + + return VideoFileModel.destroy(options) + } + + hasTorrent () { + return this.infoHash && this.torrentFilename + } + + getVideoOrStreamingPlaylist (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo | MStreamingPlaylistVideo { + if (this.videoId || (this as MVideoFileVideo).Video) return (this as MVideoFileVideo).Video + + return (this as MVideoFileStreamingPlaylistVideo).VideoStreamingPlaylist + } + + getVideo (this: MVideoFileVideo | MVideoFileStreamingPlaylistVideo): MVideo { + return extractVideo(this.getVideoOrStreamingPlaylist()) + } + + isAudio () { + return this.resolution === VideoResolution.H_NOVIDEO + } + + isLive () { + return this.size === -1 + } + + isHLS () { + return !!this.videoStreamingPlaylistId + } + + // --------------------------------------------------------------------------- + + getObjectStorageUrl (video: MVideo) { + if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) { + return this.getPrivateObjectStorageUrl(video) + } + + return this.getPublicObjectStorageUrl() + } + + private getPrivateObjectStorageUrl (video: MVideo) { + if (this.isHLS()) { + return getHLSPrivateFileUrl(video, this.filename) + } + + return getWebVideoPrivateFileUrl(this.filename) + } + + private getPublicObjectStorageUrl () { + if (this.isHLS()) { + return getHLSPublicFileUrl(this.fileUrl) + } + + return getWebVideoPublicFileUrl(this.fileUrl) + } + + // --------------------------------------------------------------------------- + + getFileUrl (video: MVideo) { + if (video.isOwned()) { + if (this.storage === VideoStorage.OBJECT_STORAGE) { + return this.getObjectStorageUrl(video) + } + + return WEBSERVER.URL + this.getFileStaticPath(video) + } + + return this.fileUrl + } + + // --------------------------------------------------------------------------- + + getFileStaticPath (video: MVideo) { + if (this.isHLS()) return this.getHLSFileStaticPath(video) + + return this.getWebVideoFileStaticPath(video) + } + + private getWebVideoFileStaticPath (video: MVideo) { + if (isVideoInPrivateDirectory(video.privacy)) { + return join(STATIC_PATHS.PRIVATE_WEB_VIDEOS, this.filename) + } + + return join(STATIC_PATHS.WEB_VIDEOS, this.filename) + } + + private getHLSFileStaticPath (video: MVideo) { + if (isVideoInPrivateDirectory(video.privacy)) { + return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.filename) + } + + return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename) + } + + // --------------------------------------------------------------------------- + + getFileDownloadUrl (video: MVideoWithHost) { + const path = this.isHLS() + ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`) + : join(STATIC_DOWNLOAD_PATHS.VIDEOS, `${video.uuid}-${this.resolution}${this.extname}`) + + if (video.isOwned()) return WEBSERVER.URL + path + + // FIXME: don't guess remote URL + return buildRemoteVideoBaseUrl(video, path) + } + + getRemoteTorrentUrl (video: MVideo) { + if (video.isOwned()) throw new Error(`Video ${video.url} is not a remote video`) + + return this.torrentUrl + } + + // We proxify torrent requests so use a local URL + getTorrentUrl () { + if (!this.torrentFilename) return null + + return WEBSERVER.URL + this.getTorrentStaticPath() + } + + getTorrentStaticPath () { + if (!this.torrentFilename) return null + + return join(LAZY_STATIC_PATHS.TORRENTS, this.torrentFilename) + } + + getTorrentDownloadUrl () { + if (!this.torrentFilename) return null + + return WEBSERVER.URL + join(STATIC_DOWNLOAD_PATHS.TORRENTS, this.torrentFilename) + } + + removeTorrent () { + if (!this.torrentFilename) return null + + const torrentPath = getFSTorrentFilePath(this) + return remove(torrentPath) + .catch(err => logger.warn('Cannot delete torrent %s.', torrentPath, { err })) + } + + hasSameUniqueKeysThan (other: MVideoFile) { + return this.fps === other.fps && + this.resolution === other.resolution && + ( + (this.videoId !== null && this.videoId === other.videoId) || + (this.videoStreamingPlaylistId !== null && this.videoStreamingPlaylistId === other.videoStreamingPlaylistId) + ) + } + + withVideoOrPlaylist (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) { + if (isStreamingPlaylist(videoOrPlaylist)) return Object.assign(this, { VideoStreamingPlaylist: videoOrPlaylist }) + + return Object.assign(this, { Video: videoOrPlaylist }) + } +} diff --git a/server/server/models/video/video-import.ts b/server/server/models/video/video-import.ts new file mode 100644 index 000000000..4de0cb104 --- /dev/null +++ b/server/server/models/video/video-import.ts @@ -0,0 +1,267 @@ +import { VideoImport, VideoImportState, type VideoImportStateType } from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { afterCommitIfTransaction } from '@server/helpers/database-utils.js' +import { MVideoImportDefault, MVideoImportFormattable } from '@server/types/models/video/video-import.js' +import { IncludeOptions, Op, WhereOptions } from 'sequelize' +import { + AfterUpdate, + AllowNull, + BelongsTo, + Column, + CreatedAt, + DataType, + Default, + DefaultScope, + ForeignKey, + Is, + Model, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { isVideoImportStateValid, isVideoImportTargetUrlValid } from '../../helpers/custom-validators/video-imports.js' +import { isVideoMagnetUriValid } from '../../helpers/custom-validators/videos.js' +import { CONSTRAINTS_FIELDS, VIDEO_IMPORT_STATES } from '../../initializers/constants.js' +import { getSort, searchAttribute, throwIfNotValid } from '../shared/index.js' +import { UserModel } from '../user/user.js' +import { VideoChannelSyncModel } from './video-channel-sync.js' +import { VideoModel, ScopeNames as VideoModelScopeNames } from './video.js' + +const defaultVideoScope = () => { + return VideoModel.scope([ + VideoModelScopeNames.WITH_ACCOUNT_DETAILS, + VideoModelScopeNames.WITH_TAGS, + VideoModelScopeNames.WITH_THUMBNAILS + ]) +} + +@DefaultScope(() => ({ + include: [ + { + model: UserModel.unscoped(), + required: true + }, + { + model: defaultVideoScope(), + required: false + }, + { + model: VideoChannelSyncModel.unscoped(), + required: false + } + ] +})) + +@Table({ + tableName: 'videoImport', + indexes: [ + { + fields: [ 'videoId' ], + unique: true + }, + { + fields: [ 'userId' ] + } + ] +}) +export class VideoImportModel extends Model>> { + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(true) + @Default(null) + @Is('VideoImportTargetUrl', value => throwIfNotValid(value, isVideoImportTargetUrlValid, 'targetUrl', true)) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max)) + targetUrl: string + + @AllowNull(true) + @Default(null) + @Is('VideoImportMagnetUri', value => throwIfNotValid(value, isVideoMagnetUriValid, 'magnetUri', true)) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.URL.max)) // Use the same constraints than URLs + magnetUri: string + + @AllowNull(true) + @Default(null) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_IMPORTS.TORRENT_NAME.max)) + torrentName: string + + @AllowNull(false) + @Default(null) + @Is('VideoImportState', value => throwIfNotValid(value, isVideoImportStateValid, 'state')) + @Column + state: VideoImportStateType + + @AllowNull(true) + @Default(null) + @Column(DataType.TEXT) + error: string + + @ForeignKey(() => UserModel) + @Column + userId: number + + @BelongsTo(() => UserModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + User: Awaited + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'set null' + }) + Video: Awaited + + @ForeignKey(() => VideoChannelSyncModel) + @Column + videoChannelSyncId: number + + @BelongsTo(() => VideoChannelSyncModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'set null' + }) + VideoChannelSync: Awaited + + @AfterUpdate + static deleteVideoIfFailed (instance: VideoImportModel, options) { + if (instance.state === VideoImportState.FAILED) { + return afterCommitIfTransaction(options.transaction, () => instance.Video.destroy()) + } + + return undefined + } + + static loadAndPopulateVideo (id: number): Promise { + return VideoImportModel.findByPk(id) + } + + static listUserVideoImportsForApi (options: { + userId: number + start: number + count: number + sort: string + + search?: string + targetUrl?: string + videoChannelSyncId?: number + }) { + const { userId, start, count, sort, targetUrl, videoChannelSyncId, search } = options + + const where: WhereOptions = { userId } + const include: IncludeOptions[] = [ + { + attributes: [ 'id' ], + model: UserModel.unscoped(), // FIXME: Without this, sequelize try to COUNT(DISTINCT(*)) which is an invalid SQL query + required: true + }, + { + model: VideoChannelSyncModel.unscoped(), + required: false + } + ] + + if (targetUrl) where['targetUrl'] = targetUrl + if (videoChannelSyncId) where['videoChannelSyncId'] = videoChannelSyncId + + if (search) { + include.push({ + model: defaultVideoScope(), + required: true, + where: searchAttribute(search, 'name') + }) + } else { + include.push({ + model: defaultVideoScope(), + required: false + }) + } + + const query = { + distinct: true, + include, + offset: start, + limit: count, + order: getSort(sort), + where + } + + return Promise.all([ + VideoImportModel.unscoped().count(query), + VideoImportModel.findAll(query) + ]).then(([ total, data ]) => ({ total, data })) + } + + static async urlAlreadyImported (channelId: number, targetUrl: string): Promise { + const element = await VideoImportModel.unscoped().findOne({ + where: { + targetUrl, + state: { + [Op.in]: [ VideoImportState.PENDING, VideoImportState.PROCESSING, VideoImportState.SUCCESS ] + } + }, + include: [ + { + model: VideoModel, + required: true, + where: { + channelId + } + } + ] + }) + + return !!element + } + + getTargetIdentifier () { + return this.targetUrl || this.magnetUri || this.torrentName + } + + toFormattedJSON (this: MVideoImportFormattable): VideoImport { + const videoFormatOptions = { + completeDescription: true, + additionalAttributes: { state: true, waitTranscoding: true, scheduledUpdate: true } + } + const video = this.Video + ? Object.assign(this.Video.toFormattedJSON(videoFormatOptions), { tags: this.Video.Tags.map(t => t.name) }) + : undefined + + const videoChannelSync = this.VideoChannelSync + ? { id: this.VideoChannelSync.id, externalChannelUrl: this.VideoChannelSync.externalChannelUrl } + : undefined + + return { + id: this.id, + + targetUrl: this.targetUrl, + magnetUri: this.magnetUri, + torrentName: this.torrentName, + + state: { + id: this.state, + label: VideoImportModel.getStateLabel(this.state) + }, + error: this.error, + updatedAt: this.updatedAt.toISOString(), + createdAt: this.createdAt.toISOString(), + video, + videoChannelSync + } + } + + private static getStateLabel (id: number) { + return VIDEO_IMPORT_STATES[id] || 'Unknown' + } +} diff --git a/server/server/models/video/video-job-info.ts b/server/server/models/video/video-job-info.ts new file mode 100644 index 000000000..4fd956528 --- /dev/null +++ b/server/server/models/video/video-job-info.ts @@ -0,0 +1,121 @@ +import { Op, QueryTypes, Transaction } from 'sequelize' +import { AllowNull, BelongsTo, Column, CreatedAt, Default, ForeignKey, IsInt, Model, Table, Unique, UpdatedAt } from 'sequelize-typescript' +import { forceNumber } from '@peertube/peertube-core-utils' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { VideoModel } from './video.js' + +export type VideoJobInfoColumnType = 'pendingMove' | 'pendingTranscode' + +@Table({ + tableName: 'videoJobInfo', + indexes: [ + { + fields: [ 'videoId' ], + where: { + videoId: { + [Op.ne]: null + } + } + } + ] +}) + +export class VideoJobInfoModel extends Model>> { + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(false) + @Default(0) + @IsInt + @Column + pendingMove: number + + @AllowNull(false) + @Default(0) + @IsInt + @Column + pendingTranscode: number + + @ForeignKey(() => VideoModel) + @Unique + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + Video: Awaited + + static load (videoId: number, transaction?: Transaction) { + const where = { + videoId + } + + return VideoJobInfoModel.findOne({ where, transaction }) + } + + static async increaseOrCreate (videoUUID: string, column: VideoJobInfoColumnType, amountArg = 1): Promise { + const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } } + const amount = forceNumber(amountArg) + + const [ result ] = await VideoJobInfoModel.sequelize.query<{ pendingMove: number }>(` + INSERT INTO "videoJobInfo" ("videoId", "${column}", "createdAt", "updatedAt") + SELECT + "video"."id" AS "videoId", ${amount}, NOW(), NOW() + FROM + "video" + WHERE + "video"."uuid" = $videoUUID + ON CONFLICT ("videoId") DO UPDATE + SET + "${column}" = "videoJobInfo"."${column}" + ${amount}, + "updatedAt" = NOW() + RETURNING + "${column}" + `, options) + + return result[column] + } + + static async decrease (videoUUID: string, column: VideoJobInfoColumnType): Promise { + const options = { type: QueryTypes.SELECT as QueryTypes.SELECT, bind: { videoUUID } } + + const result = await VideoJobInfoModel.sequelize.query(` + UPDATE + "videoJobInfo" + SET + "${column}" = "videoJobInfo"."${column}" - 1, + "updatedAt" = NOW() + FROM "video" + WHERE + "video"."id" = "videoJobInfo"."videoId" AND "video"."uuid" = $videoUUID + RETURNING + "${column}"; + `, options) + + if (result.length === 0) return undefined + + return result[0][column] + } + + static async abortAllTasks (videoUUID: string, column: VideoJobInfoColumnType): Promise { + const options = { type: QueryTypes.UPDATE as QueryTypes.UPDATE, bind: { videoUUID } } + + await VideoJobInfoModel.sequelize.query(` + UPDATE + "videoJobInfo" + SET + "${column}" = 0, + "updatedAt" = NOW() + FROM "video" + WHERE + "video"."id" = "videoJobInfo"."videoId" AND "video"."uuid" = $videoUUID + `, options) + } +} diff --git a/server/server/models/video/video-live-replay-setting.ts b/server/server/models/video/video-live-replay-setting.ts new file mode 100644 index 000000000..174d0c1f5 --- /dev/null +++ b/server/server/models/video/video-live-replay-setting.ts @@ -0,0 +1,42 @@ +import { type VideoPrivacyType } from '@peertube/peertube-models' +import { isVideoPrivacyValid } from '@server/helpers/custom-validators/videos.js' +import { MLiveReplaySetting } from '@server/types/models/video/video-live-replay-setting.js' +import { Transaction } from 'sequelize' +import { AllowNull, Column, CreatedAt, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { throwIfNotValid } from '../shared/sequelize-helpers.js' + +@Table({ + tableName: 'videoLiveReplaySetting' +}) +export class VideoLiveReplaySettingModel extends Model { + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(false) + @Is('VideoPrivacy', value => throwIfNotValid(value, isVideoPrivacyValid, 'privacy')) + @Column + privacy: VideoPrivacyType + + static load (id: number, transaction?: Transaction): Promise { + return VideoLiveReplaySettingModel.findOne({ + where: { id }, + transaction + }) + } + + static removeSettings (id: number) { + return VideoLiveReplaySettingModel.destroy({ + where: { id } + }) + } + + toFormattedJSON () { + return { + privacy: this.privacy + } + } +} diff --git a/server/server/models/video/video-live-session.ts b/server/server/models/video/video-live-session.ts new file mode 100644 index 000000000..33bd40d89 --- /dev/null +++ b/server/server/models/video/video-live-session.ts @@ -0,0 +1,217 @@ +import { LiveVideoSession, type LiveVideoErrorType } from '@peertube/peertube-models' +import { uuidToShort } from '@peertube/peertube-node-utils' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { MVideoLiveSession, MVideoLiveSessionReplay } from '@server/types/models/index.js' +import { FindOptions } from 'sequelize' +import { + AllowNull, + BeforeDestroy, + BelongsTo, + Column, + CreatedAt, + DataType, + ForeignKey, + Model, + Scopes, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { VideoLiveReplaySettingModel } from './video-live-replay-setting.js' +import { VideoModel } from './video.js' + +export enum ScopeNames { + WITH_REPLAY = 'WITH_REPLAY' +} + +@Scopes(() => ({ + [ScopeNames.WITH_REPLAY]: { + include: [ + { + model: VideoModel.unscoped(), + as: 'ReplayVideo', + required: false + }, + { + model: VideoLiveReplaySettingModel, + required: false + } + ] + } +})) +@Table({ + tableName: 'videoLiveSession', + indexes: [ + { + fields: [ 'replayVideoId' ], + unique: true + }, + { + fields: [ 'liveVideoId' ] + }, + { + fields: [ 'replaySettingId' ], + unique: true + } + ] +}) +export class VideoLiveSessionModel extends Model>> { + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(false) + @Column(DataType.DATE) + startDate: Date + + @AllowNull(true) + @Column(DataType.DATE) + endDate: Date + + @AllowNull(true) + @Column + error: LiveVideoErrorType + + @AllowNull(false) + @Column + saveReplay: boolean + + @AllowNull(false) + @Column + endingProcessed: boolean + + @ForeignKey(() => VideoModel) + @Column + replayVideoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: true, + name: 'replayVideoId' + }, + as: 'ReplayVideo', + onDelete: 'set null' + }) + ReplayVideo: Awaited + + @ForeignKey(() => VideoModel) + @Column + liveVideoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: true, + name: 'liveVideoId' + }, + as: 'LiveVideo', + onDelete: 'set null' + }) + LiveVideo: Awaited + + @ForeignKey(() => VideoLiveReplaySettingModel) + @Column + replaySettingId: number + + @BelongsTo(() => VideoLiveReplaySettingModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'set null' + }) + ReplaySetting: Awaited + + @BeforeDestroy + static deleteReplaySetting (instance: VideoLiveSessionModel) { + return VideoLiveReplaySettingModel.destroy({ + where: { + id: instance.replaySettingId + } + }) + } + + static load (id: number): Promise { + return VideoLiveSessionModel.findOne({ + where: { id } + }) + } + + static findSessionOfReplay (replayVideoId: number) { + const query = { + where: { + replayVideoId + } + } + + return VideoLiveSessionModel.scope(ScopeNames.WITH_REPLAY).findOne(query) + } + + static findCurrentSessionOf (videoUUID: string) { + return VideoLiveSessionModel.findOne({ + where: { + endDate: null + }, + include: [ + { + model: VideoModel.unscoped(), + as: 'LiveVideo', + required: true, + where: { + uuid: videoUUID + } + } + ], + order: [ [ 'startDate', 'DESC' ] ] + }) + } + + static findLatestSessionOf (videoId: number) { + return VideoLiveSessionModel.findOne({ + where: { + liveVideoId: videoId + }, + order: [ [ 'startDate', 'DESC' ] ] + }) + } + + static listSessionsOfLiveForAPI (options: { videoId: number }) { + const { videoId } = options + + const query: FindOptions> = { + where: { + liveVideoId: videoId + }, + order: [ [ 'startDate', 'ASC' ] ] + } + + return VideoLiveSessionModel.scope(ScopeNames.WITH_REPLAY).findAll(query) + } + + toFormattedJSON (this: MVideoLiveSessionReplay): LiveVideoSession { + const replayVideo = this.ReplayVideo + ? { + id: this.ReplayVideo.id, + uuid: this.ReplayVideo.uuid, + shortUUID: uuidToShort(this.ReplayVideo.uuid) + } + : undefined + + const replaySettings = this.replaySettingId + ? this.ReplaySetting.toFormattedJSON() + : undefined + + return { + id: this.id, + startDate: this.startDate.toISOString(), + endDate: this.endDate + ? this.endDate.toISOString() + : null, + endingProcessed: this.endingProcessed, + saveReplay: this.saveReplay, + replaySettings, + replayVideo, + error: this.error + } + } +} diff --git a/server/server/models/video/video-live.ts b/server/server/models/video/video-live.ts new file mode 100644 index 000000000..fefaa053a --- /dev/null +++ b/server/server/models/video/video-live.ts @@ -0,0 +1,184 @@ +import { LiveVideo, VideoState, type LiveVideoLatencyModeType } from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { CONFIG } from '@server/initializers/config.js' +import { WEBSERVER } from '@server/initializers/constants.js' +import { MVideoLive, MVideoLiveVideoWithSetting } from '@server/types/models/index.js' +import { Transaction } from 'sequelize' +import { + AllowNull, + BeforeDestroy, + BelongsTo, + Column, + CreatedAt, + DataType, + DefaultScope, + ForeignKey, + Model, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { VideoBlacklistModel } from './video-blacklist.js' +import { VideoLiveReplaySettingModel } from './video-live-replay-setting.js' +import { VideoModel } from './video.js' + +@DefaultScope(() => ({ + include: [ + { + model: VideoModel, + required: true, + include: [ + { + model: VideoBlacklistModel, + required: false + } + ] + }, + { + model: VideoLiveReplaySettingModel, + required: false + } + ] +})) +@Table({ + tableName: 'videoLive', + indexes: [ + { + fields: [ 'videoId' ], + unique: true + }, + { + fields: [ 'replaySettingId' ], + unique: true + } + ] +}) +export class VideoLiveModel extends Model>> { + + @AllowNull(true) + @Column(DataType.STRING) + streamKey: string + + @AllowNull(false) + @Column + saveReplay: boolean + + @AllowNull(false) + @Column + permanentLive: boolean + + @AllowNull(false) + @Column + latencyMode: LiveVideoLatencyModeType + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + Video: Awaited + + @ForeignKey(() => VideoLiveReplaySettingModel) + @Column + replaySettingId: number + + @BelongsTo(() => VideoLiveReplaySettingModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'set null' + }) + ReplaySetting: Awaited + + @BeforeDestroy + static deleteReplaySetting (instance: VideoLiveModel, options: { transaction: Transaction }) { + return VideoLiveReplaySettingModel.destroy({ + where: { + id: instance.replaySettingId + }, + transaction: options.transaction + }) + } + + static loadByStreamKey (streamKey: string) { + const query = { + where: { + streamKey + }, + include: [ + { + model: VideoModel.unscoped(), + required: true, + where: { + state: VideoState.WAITING_FOR_LIVE + }, + include: [ + { + model: VideoBlacklistModel.unscoped(), + required: false + } + ] + }, + { + model: VideoLiveReplaySettingModel.unscoped(), + required: false + } + ] + } + + return VideoLiveModel.findOne(query) + } + + static loadByVideoId (videoId: number) { + const query = { + where: { + videoId + } + } + + return VideoLiveModel.findOne(query) + } + + toFormattedJSON (canSeePrivateInformation: boolean): LiveVideo { + let privateInformation: Pick | {} = {} + + // If we don't have a stream key, it means this is a remote live so we don't specify the rtmp URL + // We also display these private information only to the live owne/moderators + if (this.streamKey && canSeePrivateInformation === true) { + privateInformation = { + streamKey: this.streamKey, + + rtmpUrl: CONFIG.LIVE.RTMP.ENABLED + ? WEBSERVER.RTMP_BASE_LIVE_URL + : null, + + rtmpsUrl: CONFIG.LIVE.RTMPS.ENABLED + ? WEBSERVER.RTMPS_BASE_LIVE_URL + : null + } + } + + const replaySettings = this.replaySettingId + ? this.ReplaySetting.toFormattedJSON() + : undefined + + return { + ...privateInformation, + + permanentLive: this.permanentLive, + saveReplay: this.saveReplay, + replaySettings, + latencyMode: this.latencyMode + } + } +} diff --git a/server/server/models/video/video-password.ts b/server/server/models/video/video-password.ts new file mode 100644 index 000000000..f841f320e --- /dev/null +++ b/server/server/models/video/video-password.ts @@ -0,0 +1,137 @@ +import { AllowNull, BelongsTo, Column, CreatedAt, DefaultScope, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { VideoModel } from './video.js' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { ResultList, VideoPassword } from '@peertube/peertube-models' +import { getSort, throwIfNotValid } from '../shared/index.js' +import { FindOptions, Transaction } from 'sequelize' +import { MVideoPassword } from '@server/types/models/index.js' +import { isPasswordValid } from '@server/helpers/custom-validators/videos.js' +import { pick } from '@peertube/peertube-core-utils' + +@DefaultScope(() => ({ + include: [ + { + model: VideoModel.unscoped(), + required: true + } + ] +})) +@Table({ + tableName: 'videoPassword', + indexes: [ + { + fields: [ 'videoId', 'password' ], + unique: true + } + ] +}) +export class VideoPasswordModel extends Model>> { + + @AllowNull(false) + @Is('VideoPassword', value => throwIfNotValid(value, isPasswordValid, 'videoPassword')) + @Column + password: string + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + Video: Awaited + + static async countByVideoId (videoId: number, t?: Transaction) { + const query: FindOptions = { + where: { + videoId + }, + transaction: t + } + + return VideoPasswordModel.count(query) + } + + static async loadByIdAndVideo (options: { id: number, videoId: number, t?: Transaction }): Promise { + const { id, videoId, t } = options + const query: FindOptions = { + where: { + id, + videoId + }, + transaction: t + } + + return VideoPasswordModel.findOne(query) + } + + static async listPasswords (options: { + start: number + count: number + sort: string + videoId: number + }): Promise> { + const { start, count, sort, videoId } = options + + const { count: total, rows: data } = await VideoPasswordModel.findAndCountAll({ + where: { videoId }, + order: getSort(sort), + offset: start, + limit: count + }) + + return { total, data } + } + + static async addPasswords (passwords: string[], videoId: number, transaction?: Transaction): Promise { + for (const password of passwords) { + await VideoPasswordModel.create({ + password, + videoId + }, { transaction }) + } + } + + static async deleteAllPasswords (videoId: number, transaction?: Transaction) { + await VideoPasswordModel.destroy({ + where: { videoId }, + transaction + }) + } + + static async deletePassword (passwordId: number, transaction?: Transaction) { + await VideoPasswordModel.destroy({ + where: { id: passwordId }, + transaction + }) + } + + static async isACorrectPassword (options: { + videoId: number + password: string + }) { + const query = { + where: pick(options, [ 'videoId', 'password' ]) + } + return VideoPasswordModel.findOne(query) + } + + toFormattedJSON (): VideoPassword { + return { + id: this.id, + password: this.password, + videoId: this.videoId, + createdAt: this.createdAt, + updatedAt: this.updatedAt + } + } +} diff --git a/server/server/models/video/video-playlist-element.ts b/server/server/models/video/video-playlist-element.ts new file mode 100644 index 000000000..62b0e3434 --- /dev/null +++ b/server/server/models/video/video-playlist-element.ts @@ -0,0 +1,375 @@ +import { AggregateOptions, Op, ScopeOptions, Sequelize, Transaction } from 'sequelize' +import { + AllowNull, + BelongsTo, + Column, + CreatedAt, + DataType, + Default, + ForeignKey, + Is, + IsInt, + Min, + Model, + Table, + UpdatedAt +} from 'sequelize-typescript' +import validator from 'validator' +import { forceNumber } from '@peertube/peertube-core-utils' +import { + PlaylistElementObject, + VideoPlaylistElement, + VideoPlaylistElementType, + VideoPrivacy, + VideoPrivacyType +} from '@peertube/peertube-models' +import { MUserAccountId } from '@server/types/models/index.js' +import { + MVideoPlaylistElement, + MVideoPlaylistElementAP, + MVideoPlaylistElementFormattable, + MVideoPlaylistElementVideoUrlPlaylistPrivacy, + MVideoPlaylistVideoThumbnail +} from '@server/types/models/video/video-playlist-element.js' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc.js' +import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js' +import { AccountModel } from '../account/account.js' +import { getSort, throwIfNotValid } from '../shared/index.js' +import { VideoPlaylistModel } from './video-playlist.js' +import { ForAPIOptions, ScopeNames as VideoScopeNames, VideoModel } from './video.js' + +@Table({ + tableName: 'videoPlaylistElement', + indexes: [ + { + fields: [ 'videoPlaylistId' ] + }, + { + fields: [ 'videoId' ] + }, + { + fields: [ 'url' ], + unique: true + } + ] +}) +export class VideoPlaylistElementModel extends Model>> { + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(true) + @Is('VideoPlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url', true)) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_PLAYLISTS.URL.max)) + url: string + + @AllowNull(false) + @Default(1) + @IsInt + @Min(1) + @Column + position: number + + @AllowNull(true) + @IsInt + @Min(0) + @Column + startTimestamp: number + + @AllowNull(true) + @IsInt + @Min(0) + @Column + stopTimestamp: number + + @ForeignKey(() => VideoPlaylistModel) + @Column + videoPlaylistId: number + + @BelongsTo(() => VideoPlaylistModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + VideoPlaylist: Awaited + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'set null' + }) + Video: Awaited + + static deleteAllOf (videoPlaylistId: number, transaction?: Transaction) { + const query = { + where: { + videoPlaylistId + }, + transaction + } + + return VideoPlaylistElementModel.destroy(query) + } + + static listForApi (options: { + start: number + count: number + videoPlaylistId: number + serverAccount: AccountModel + user?: MUserAccountId + }) { + const accountIds = [ options.serverAccount.id ] + const videoScope: (ScopeOptions | string)[] = [ + VideoScopeNames.WITH_BLACKLISTED + ] + + if (options.user) { + accountIds.push(options.user.Account.id) + videoScope.push({ method: [ VideoScopeNames.WITH_USER_HISTORY, options.user.id ] }) + } + + const forApiOptions: ForAPIOptions = { withAccountBlockerIds: accountIds } + videoScope.push({ + method: [ + VideoScopeNames.FOR_API, forApiOptions + ] + }) + + const findQuery = { + offset: options.start, + limit: options.count, + order: getSort('position'), + where: { + videoPlaylistId: options.videoPlaylistId + }, + include: [ + { + model: VideoModel.scope(videoScope), + required: false + } + ] + } + + const countQuery = { + where: { + videoPlaylistId: options.videoPlaylistId + } + } + + return Promise.all([ + VideoPlaylistElementModel.count(countQuery), + VideoPlaylistElementModel.findAll(findQuery) + ]).then(([ total, data ]) => ({ total, data })) + } + + static loadByPlaylistAndVideo (videoPlaylistId: number, videoId: number): Promise { + const query = { + where: { + videoPlaylistId, + videoId + } + } + + return VideoPlaylistElementModel.findOne(query) + } + + static loadById (playlistElementId: number | string): Promise { + return VideoPlaylistElementModel.findByPk(playlistElementId) + } + + static loadByPlaylistAndElementIdForAP ( + playlistId: number | string, + playlistElementId: number + ): Promise { + const playlistWhere = validator.default.isUUID('' + playlistId) + ? { uuid: playlistId } + : { id: playlistId } + + const query = { + include: [ + { + attributes: [ 'privacy' ], + model: VideoPlaylistModel.unscoped(), + where: playlistWhere + }, + { + attributes: [ 'url' ], + model: VideoModel.unscoped() + } + ], + where: { + id: playlistElementId + } + } + + return VideoPlaylistElementModel.findOne(query) + } + + static listUrlsOfForAP (videoPlaylistId: number, start: number, count: number, t?: Transaction) { + const getQuery = (forCount: boolean) => { + return { + attributes: forCount + ? [] + : [ 'url' ], + offset: start, + limit: count, + order: getSort('position'), + where: { + videoPlaylistId + }, + transaction: t + } + } + + return Promise.all([ + VideoPlaylistElementModel.count(getQuery(true)), + VideoPlaylistElementModel.findAll(getQuery(false)) + ]).then(([ total, rows ]) => ({ + total, + data: rows.map(e => e.url) + })) + } + + static loadFirstElementWithVideoThumbnail (videoPlaylistId: number): Promise { + const query = { + order: getSort('position'), + where: { + videoPlaylistId + }, + include: [ + { + model: VideoModel.scope(VideoScopeNames.WITH_THUMBNAILS), + required: true + } + ] + } + + return VideoPlaylistElementModel + .findOne(query) + } + + static getNextPositionOf (videoPlaylistId: number, transaction?: Transaction) { + const query: AggregateOptions = { + where: { + videoPlaylistId + }, + transaction + } + + return VideoPlaylistElementModel.max('position', query) + .then(position => position ? position + 1 : 1) + } + + static reassignPositionOf (options: { + videoPlaylistId: number + firstPosition: number + endPosition: number + newPosition: number + transaction?: Transaction + }) { + const { videoPlaylistId, firstPosition, endPosition, newPosition, transaction } = options + + const query = { + where: { + videoPlaylistId, + position: { + [Op.gte]: firstPosition, + [Op.lte]: endPosition + } + }, + transaction, + validate: false // We use a literal to update the position + } + + const positionQuery = Sequelize.literal(`${forceNumber(newPosition)} + "position" - ${forceNumber(firstPosition)}`) + return VideoPlaylistElementModel.update({ position: positionQuery }, query) + } + + static increasePositionOf ( + videoPlaylistId: number, + fromPosition: number, + by = 1, + transaction?: Transaction + ) { + const query = { + where: { + videoPlaylistId, + position: { + [Op.gte]: fromPosition + } + }, + transaction + } + + return VideoPlaylistElementModel.increment({ position: by }, query) + } + + toFormattedJSON ( + this: MVideoPlaylistElementFormattable, + options: { accountId?: number } = {} + ): VideoPlaylistElement { + return { + id: this.id, + position: this.position, + startTimestamp: this.startTimestamp, + stopTimestamp: this.stopTimestamp, + + type: this.getType(options.accountId), + + video: this.getVideoElement(options.accountId) + } + } + + getType (this: MVideoPlaylistElementFormattable, accountId?: number) { + const video = this.Video + + if (!video) return VideoPlaylistElementType.DELETED + + // Owned video, don't filter it + if (accountId && video.VideoChannel.Account.id === accountId) return VideoPlaylistElementType.REGULAR + + // Internal video? + if (video.privacy === VideoPrivacy.INTERNAL && accountId) return VideoPlaylistElementType.REGULAR + + // Private, internal and password protected videos cannot be read without appropriate access (ownership, internal) + const protectedPrivacy = new Set([ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL, VideoPrivacy.PASSWORD_PROTECTED ]) + if (protectedPrivacy.has(video.privacy)) { + return VideoPlaylistElementType.PRIVATE + } + + if (video.isBlacklisted() || video.isBlocked()) return VideoPlaylistElementType.UNAVAILABLE + + return VideoPlaylistElementType.REGULAR + } + + getVideoElement (this: MVideoPlaylistElementFormattable, accountId?: number) { + if (!this.Video) return null + if (this.getType(accountId) !== VideoPlaylistElementType.REGULAR) return null + + return this.Video.toFormattedJSON() + } + + toActivityPubObject (this: MVideoPlaylistElementAP): PlaylistElementObject { + const base: PlaylistElementObject = { + id: this.url, + type: 'PlaylistElement', + + url: this.Video?.url || null, + position: this.position + } + + if (this.startTimestamp) base.startTimestamp = this.startTimestamp + if (this.stopTimestamp) base.stopTimestamp = this.stopTimestamp + + return base + } +} diff --git a/server/server/models/video/video-playlist.ts b/server/server/models/video/video-playlist.ts new file mode 100644 index 000000000..716e78c4c --- /dev/null +++ b/server/server/models/video/video-playlist.ts @@ -0,0 +1,733 @@ +import { buildPlaylistEmbedPath, buildPlaylistWatchPath, pick } from '@peertube/peertube-core-utils' +import { + ActivityIconObject, + PlaylistObject, + VideoPlaylist, + VideoPlaylistPrivacy, + VideoPlaylistType, + type VideoPlaylistPrivacyType, + type VideoPlaylistType_Type +} from '@peertube/peertube-models' +import { buildUUID, uuidToShort } from '@peertube/peertube-node-utils' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { activityPubCollectionPagination } from '@server/lib/activitypub/collection.js' +import { MAccountId, MChannelId } from '@server/types/models/index.js' +import { join } from 'path' +import { FindOptions, Includeable, literal, Op, ScopeOptions, Sequelize, Transaction, WhereOptions } from 'sequelize' +import { + AllowNull, + BelongsTo, + Column, + CreatedAt, + DataType, + Default, + ForeignKey, + HasMany, + HasOne, + Is, + IsUUID, + Model, + Scopes, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc.js' +import { + isVideoPlaylistDescriptionValid, + isVideoPlaylistNameValid, + isVideoPlaylistPrivacyValid +} from '../../helpers/custom-validators/video-playlists.js' +import { + ACTIVITY_PUB, + CONSTRAINTS_FIELDS, + LAZY_STATIC_PATHS, + THUMBNAILS_SIZE, + VIDEO_PLAYLIST_PRIVACIES, + VIDEO_PLAYLIST_TYPES, + WEBSERVER +} from '../../initializers/constants.js' +import { MThumbnail } from '../../types/models/video/thumbnail.js' +import { + MVideoPlaylistAccountThumbnail, + MVideoPlaylistAP, + MVideoPlaylistFormattable, + MVideoPlaylistFull, + MVideoPlaylistFullSummary, + MVideoPlaylistSummaryWithElements +} from '../../types/models/video/video-playlist.js' +import { AccountModel, ScopeNames as AccountScopeNames, SummaryOptions } from '../account/account.js' +import { ActorModel } from '../actor/actor.js' +import { + buildServerIdsFollowedBy, + buildTrigramSearchIndex, + buildWhereIdOrUUID, + createSimilarityAttribute, + getPlaylistSort, + isOutdated, + setAsUpdated, + throwIfNotValid +} from '../shared/index.js' +import { ThumbnailModel } from './thumbnail.js' +import { VideoChannelModel, ScopeNames as VideoChannelScopeNames } from './video-channel.js' +import { VideoPlaylistElementModel } from './video-playlist-element.js' + +enum ScopeNames { + AVAILABLE_FOR_LIST = 'AVAILABLE_FOR_LIST', + WITH_VIDEOS_LENGTH = 'WITH_VIDEOS_LENGTH', + WITH_ACCOUNT_AND_CHANNEL_SUMMARY = 'WITH_ACCOUNT_AND_CHANNEL_SUMMARY', + WITH_ACCOUNT = 'WITH_ACCOUNT', + WITH_THUMBNAIL = 'WITH_THUMBNAIL', + WITH_ACCOUNT_AND_CHANNEL = 'WITH_ACCOUNT_AND_CHANNEL' +} + +type AvailableForListOptions = { + followerActorId?: number + type?: VideoPlaylistType_Type + accountId?: number + videoChannelId?: number + listMyPlaylists?: boolean + search?: string + host?: string + uuids?: string[] + withVideos?: boolean + forCount?: boolean +} + +function getVideoLengthSelect () { + return 'SELECT COUNT("id") FROM "videoPlaylistElement" WHERE "videoPlaylistId" = "VideoPlaylistModel"."id"' +} + +@Scopes(() => ({ + [ScopeNames.WITH_THUMBNAIL]: { + include: [ + { + model: ThumbnailModel, + required: false + } + ] + }, + [ScopeNames.WITH_VIDEOS_LENGTH]: { + attributes: { + include: [ + [ + literal(`(${getVideoLengthSelect()})`), + 'videosLength' + ] + ] + } + } as FindOptions, + [ScopeNames.WITH_ACCOUNT]: { + include: [ + { + model: AccountModel, + required: true + } + ] + }, + [ScopeNames.WITH_ACCOUNT_AND_CHANNEL_SUMMARY]: { + include: [ + { + model: AccountModel.scope(AccountScopeNames.SUMMARY), + required: true + }, + { + model: VideoChannelModel.scope(VideoChannelScopeNames.SUMMARY), + required: false + } + ] + }, + [ScopeNames.WITH_ACCOUNT_AND_CHANNEL]: { + include: [ + { + model: AccountModel, + required: true + }, + { + model: VideoChannelModel, + required: false + } + ] + }, + [ScopeNames.AVAILABLE_FOR_LIST]: (options: AvailableForListOptions) => { + const whereAnd: WhereOptions[] = [] + + const whereServer = options.host && options.host !== WEBSERVER.HOST + ? { host: options.host } + : undefined + + let whereActor: WhereOptions = {} + + if (options.host === WEBSERVER.HOST) { + whereActor = { + [Op.and]: [ { serverId: null } ] + } + } + + if (options.listMyPlaylists !== true) { + whereAnd.push({ + privacy: VideoPlaylistPrivacy.PUBLIC + }) + + // Only list local playlists + const whereActorOr: WhereOptions[] = [ + { + serverId: null + } + ] + + // … OR playlists that are on an instance followed by actorId + if (options.followerActorId) { + const inQueryInstanceFollow = buildServerIdsFollowedBy(options.followerActorId) + + whereActorOr.push({ + serverId: { + [Op.in]: literal(inQueryInstanceFollow) + } + }) + } + + Object.assign(whereActor, { [Op.or]: whereActorOr }) + } + + if (options.accountId) { + whereAnd.push({ + ownerAccountId: options.accountId + }) + } + + if (options.videoChannelId) { + whereAnd.push({ + videoChannelId: options.videoChannelId + }) + } + + if (options.type) { + whereAnd.push({ + type: options.type + }) + } + + if (options.uuids) { + whereAnd.push({ + uuid: { + [Op.in]: options.uuids + } + }) + } + + if (options.withVideos === true) { + whereAnd.push( + literal(`(${getVideoLengthSelect()}) != 0`) + ) + } + + let attributesInclude: any[] = [ literal('0 as similarity') ] + + if (options.search) { + const escapedSearch = VideoPlaylistModel.sequelize.escape(options.search) + const escapedLikeSearch = VideoPlaylistModel.sequelize.escape('%' + options.search + '%') + attributesInclude = [ createSimilarityAttribute('VideoPlaylistModel.name', options.search) ] + + whereAnd.push({ + [Op.or]: [ + Sequelize.literal( + 'lower(immutable_unaccent("VideoPlaylistModel"."name")) % lower(immutable_unaccent(' + escapedSearch + '))' + ), + Sequelize.literal( + 'lower(immutable_unaccent("VideoPlaylistModel"."name")) LIKE lower(immutable_unaccent(' + escapedLikeSearch + '))' + ) + ] + }) + } + + const where = { + [Op.and]: whereAnd + } + + const include: Includeable[] = [ + { + model: AccountModel.scope({ + method: [ AccountScopeNames.SUMMARY, { whereActor, whereServer, forCount: options.forCount } as SummaryOptions ] + }), + required: true + } + ] + + if (options.forCount !== true) { + include.push({ + model: VideoChannelModel.scope(VideoChannelScopeNames.SUMMARY), + required: false + }) + } + + return { + attributes: { + include: attributesInclude + }, + where, + include + } as FindOptions + } +})) + +@Table({ + tableName: 'videoPlaylist', + indexes: [ + buildTrigramSearchIndex('video_playlist_name_trigram', 'name'), + + { + fields: [ 'ownerAccountId' ] + }, + { + fields: [ 'videoChannelId' ] + }, + { + fields: [ 'url' ], + unique: true + } + ] +}) +export class VideoPlaylistModel extends Model>> { + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(false) + @Is('VideoPlaylistName', value => throwIfNotValid(value, isVideoPlaylistNameValid, 'name')) + @Column + name: string + + @AllowNull(true) + @Is('VideoPlaylistDescription', value => throwIfNotValid(value, isVideoPlaylistDescriptionValid, 'description', true)) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_PLAYLISTS.DESCRIPTION.max)) + description: string + + @AllowNull(false) + @Is('VideoPlaylistPrivacy', value => throwIfNotValid(value, isVideoPlaylistPrivacyValid, 'privacy')) + @Column + privacy: VideoPlaylistPrivacyType + + @AllowNull(false) + @Is('VideoPlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url')) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_PLAYLISTS.URL.max)) + url: string + + @AllowNull(false) + @Default(DataType.UUIDV4) + @IsUUID(4) + @Column(DataType.UUID) + uuid: string + + @AllowNull(false) + @Default(VideoPlaylistType.REGULAR) + @Column + type: VideoPlaylistType_Type + + @ForeignKey(() => AccountModel) + @Column + ownerAccountId: number + + @BelongsTo(() => AccountModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + OwnerAccount: Awaited + + @ForeignKey(() => VideoChannelModel) + @Column + videoChannelId: number + + @BelongsTo(() => VideoChannelModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'CASCADE' + }) + VideoChannel: Awaited + + @HasMany(() => VideoPlaylistElementModel, { + foreignKey: { + name: 'videoPlaylistId', + allowNull: false + }, + onDelete: 'CASCADE' + }) + VideoPlaylistElements: Awaited[] + + @HasOne(() => ThumbnailModel, { + foreignKey: { + name: 'videoPlaylistId', + allowNull: true + }, + onDelete: 'CASCADE', + hooks: true + }) + Thumbnail: Awaited + + static listForApi (options: AvailableForListOptions & { + start: number + count: number + sort: string + }) { + const query = { + offset: options.start, + limit: options.count, + order: getPlaylistSort(options.sort) + } + + const commonAvailableForListOptions = pick(options, [ + 'type', + 'followerActorId', + 'accountId', + 'videoChannelId', + 'listMyPlaylists', + 'search', + 'host', + 'uuids' + ]) + + const scopesFind: (string | ScopeOptions)[] = [ + { + method: [ + ScopeNames.AVAILABLE_FOR_LIST, + { + ...commonAvailableForListOptions, + + withVideos: options.withVideos || false + } as AvailableForListOptions + ] + }, + ScopeNames.WITH_VIDEOS_LENGTH, + ScopeNames.WITH_THUMBNAIL + ] + + const scopesCount: (string | ScopeOptions)[] = [ + { + method: [ + ScopeNames.AVAILABLE_FOR_LIST, + + { + ...commonAvailableForListOptions, + + withVideos: options.withVideos || false, + forCount: true + } as AvailableForListOptions + ] + }, + ScopeNames.WITH_VIDEOS_LENGTH + ] + + return Promise.all([ + VideoPlaylistModel.scope(scopesCount).count(), + VideoPlaylistModel.scope(scopesFind).findAll(query) + ]).then(([ count, rows ]) => ({ total: count, data: rows })) + } + + static searchForApi (options: Pick & { + start: number + count: number + sort: string + }) { + return VideoPlaylistModel.listForApi({ + ...options, + + type: VideoPlaylistType.REGULAR, + listMyPlaylists: false, + withVideos: true + }) + } + + static listPublicUrlsOfForAP (options: { account?: MAccountId, channel?: MChannelId }, start: number, count: number) { + const where = { + privacy: VideoPlaylistPrivacy.PUBLIC + } + + if (options.account) { + Object.assign(where, { ownerAccountId: options.account.id }) + } + + if (options.channel) { + Object.assign(where, { videoChannelId: options.channel.id }) + } + + const getQuery = (forCount: boolean) => { + return { + attributes: forCount === true + ? [] + : [ 'url' ], + offset: start, + limit: count, + where + } + } + + return Promise.all([ + VideoPlaylistModel.count(getQuery(true)), + VideoPlaylistModel.findAll(getQuery(false)) + ]).then(([ total, rows ]) => ({ + total, + data: rows.map(p => p.url) + })) + } + + static listPlaylistSummariesOf (accountId: number, videoIds: number[]): Promise { + const query = { + attributes: [ 'id', 'name', 'uuid' ], + where: { + ownerAccountId: accountId + }, + include: [ + { + attributes: [ 'id', 'videoId', 'startTimestamp', 'stopTimestamp' ], + model: VideoPlaylistElementModel.unscoped(), + where: { + videoId: { + [Op.in]: videoIds + } + }, + required: true + } + ] + } + + return VideoPlaylistModel.findAll(query) + } + + static doesPlaylistExist (url: string) { + const query = { + attributes: [ 'id' ], + where: { + url + } + } + + return VideoPlaylistModel + .findOne(query) + .then(e => !!e) + } + + static loadWithAccountAndChannelSummary (id: number | string, transaction: Transaction): Promise { + const where = buildWhereIdOrUUID(id) + + const query = { + where, + transaction + } + + return VideoPlaylistModel + .scope([ ScopeNames.WITH_ACCOUNT_AND_CHANNEL_SUMMARY, ScopeNames.WITH_VIDEOS_LENGTH, ScopeNames.WITH_THUMBNAIL ]) + .findOne(query) + } + + static loadWithAccountAndChannel (id: number | string, transaction: Transaction): Promise { + const where = buildWhereIdOrUUID(id) + + const query = { + where, + transaction + } + + return VideoPlaylistModel + .scope([ ScopeNames.WITH_ACCOUNT_AND_CHANNEL, ScopeNames.WITH_VIDEOS_LENGTH, ScopeNames.WITH_THUMBNAIL ]) + .findOne(query) + } + + static loadByUrlAndPopulateAccount (url: string): Promise { + const query = { + where: { + url + } + } + + return VideoPlaylistModel.scope([ ScopeNames.WITH_ACCOUNT, ScopeNames.WITH_THUMBNAIL ]).findOne(query) + } + + static loadByUrlWithAccountAndChannelSummary (url: string): Promise { + const query = { + where: { + url + } + } + + return VideoPlaylistModel + .scope([ ScopeNames.WITH_ACCOUNT_AND_CHANNEL_SUMMARY, ScopeNames.WITH_VIDEOS_LENGTH, ScopeNames.WITH_THUMBNAIL ]) + .findOne(query) + } + + static getPrivacyLabel (privacy: VideoPlaylistPrivacyType) { + return VIDEO_PLAYLIST_PRIVACIES[privacy] || 'Unknown' + } + + static getTypeLabel (type: VideoPlaylistType_Type) { + return VIDEO_PLAYLIST_TYPES[type] || 'Unknown' + } + + static resetPlaylistsOfChannel (videoChannelId: number, transaction: Transaction) { + const query = { + where: { + videoChannelId + }, + transaction + } + + return VideoPlaylistModel.update({ privacy: VideoPlaylistPrivacy.PRIVATE, videoChannelId: null }, query) + } + + async setAndSaveThumbnail (thumbnail: MThumbnail, t: Transaction) { + thumbnail.videoPlaylistId = this.id + + this.Thumbnail = await thumbnail.save({ transaction: t }) + } + + hasThumbnail () { + return !!this.Thumbnail + } + + hasGeneratedThumbnail () { + return this.hasThumbnail() && this.Thumbnail.automaticallyGenerated === true + } + + generateThumbnailName () { + const extension = '.jpg' + + return 'playlist-' + buildUUID() + extension + } + + getThumbnailUrl () { + if (!this.hasThumbnail()) return null + + return WEBSERVER.URL + LAZY_STATIC_PATHS.THUMBNAILS + this.Thumbnail.filename + } + + getThumbnailStaticPath () { + if (!this.hasThumbnail()) return null + + return join(LAZY_STATIC_PATHS.THUMBNAILS, this.Thumbnail.filename) + } + + getWatchStaticPath () { + return buildPlaylistWatchPath({ shortUUID: uuidToShort(this.uuid) }) + } + + getEmbedStaticPath () { + return buildPlaylistEmbedPath(this) + } + + static async getStats () { + const totalLocalPlaylists = await VideoPlaylistModel.count({ + include: [ + { + model: AccountModel.unscoped(), + required: true, + include: [ + { + model: ActorModel.unscoped(), + required: true, + where: { + serverId: null + } + } + ] + } + ], + where: { + privacy: VideoPlaylistPrivacy.PUBLIC + } + }) + + return { + totalLocalPlaylists + } + } + + setAsRefreshed () { + return setAsUpdated({ sequelize: this.sequelize, table: 'videoPlaylist', id: this.id }) + } + + setVideosLength (videosLength: number) { + this.set('videosLength' as any, videosLength, { raw: true }) + } + + isOwned () { + return this.OwnerAccount.isOwned() + } + + isOutdated () { + if (this.isOwned()) return false + + return isOutdated(this, ACTIVITY_PUB.VIDEO_PLAYLIST_REFRESH_INTERVAL) + } + + toFormattedJSON (this: MVideoPlaylistFormattable): VideoPlaylist { + return { + id: this.id, + uuid: this.uuid, + shortUUID: uuidToShort(this.uuid), + + isLocal: this.isOwned(), + + url: this.url, + + displayName: this.name, + description: this.description, + privacy: { + id: this.privacy, + label: VideoPlaylistModel.getPrivacyLabel(this.privacy) + }, + + thumbnailPath: this.getThumbnailStaticPath(), + embedPath: this.getEmbedStaticPath(), + + type: { + id: this.type, + label: VideoPlaylistModel.getTypeLabel(this.type) + }, + + videosLength: this.get('videosLength') as number, + + createdAt: this.createdAt, + updatedAt: this.updatedAt, + + ownerAccount: this.OwnerAccount.toFormattedSummaryJSON(), + videoChannel: this.VideoChannel + ? this.VideoChannel.toFormattedSummaryJSON() + : null + } + } + + toActivityPubObject (this: MVideoPlaylistAP, page: number, t: Transaction): Promise { + const handler = (start: number, count: number) => { + return VideoPlaylistElementModel.listUrlsOfForAP(this.id, start, count, t) + } + + let icon: ActivityIconObject + if (this.hasThumbnail()) { + icon = { + type: 'Image' as 'Image', + url: this.getThumbnailUrl(), + mediaType: 'image/jpeg' as 'image/jpeg', + width: THUMBNAILS_SIZE.width, + height: THUMBNAILS_SIZE.height + } + } + + return activityPubCollectionPagination(this.url, handler, page) + .then(o => { + return Object.assign(o, { + type: 'Playlist' as 'Playlist', + name: this.name, + content: this.description, + mediaType: 'text/markdown' as 'text/markdown', + uuid: this.uuid, + published: this.createdAt.toISOString(), + updated: this.updatedAt.toISOString(), + attributedTo: this.VideoChannel ? [ this.VideoChannel.Actor.url ] : [], + icon + }) + }) + } +} diff --git a/server/server/models/video/video-share.ts b/server/server/models/video/video-share.ts new file mode 100644 index 000000000..a3f571fed --- /dev/null +++ b/server/server/models/video/video-share.ts @@ -0,0 +1,216 @@ +import { literal, Op, QueryTypes, Transaction } from 'sequelize' +import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Is, Model, Scopes, Table, UpdatedAt } from 'sequelize-typescript' +import { forceNumber } from '@peertube/peertube-core-utils' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc.js' +import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js' +import { MActorDefault, MActorFollowersUrl, MActorId } from '../../types/models/index.js' +import { MVideoShareActor, MVideoShareFull } from '../../types/models/video/index.js' +import { ActorModel } from '../actor/actor.js' +import { buildLocalActorIdsIn, throwIfNotValid } from '../shared/index.js' +import { VideoModel } from './video.js' + +enum ScopeNames { + FULL = 'FULL', + WITH_ACTOR = 'WITH_ACTOR' +} + +@Scopes(() => ({ + [ScopeNames.FULL]: { + include: [ + { + model: ActorModel, + required: true + }, + { + model: VideoModel, + required: true + } + ] + }, + [ScopeNames.WITH_ACTOR]: { + include: [ + { + model: ActorModel, + required: true + } + ] + } +})) +@Table({ + tableName: 'videoShare', + indexes: [ + { + fields: [ 'actorId' ] + }, + { + fields: [ 'videoId' ] + }, + { + fields: [ 'url' ], + unique: true + } + ] +}) +export class VideoShareModel extends Model>> { + + @AllowNull(false) + @Is('VideoShareUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url')) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEO_SHARE.URL.max)) + url: string + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => ActorModel) + @Column + actorId: number + + @BelongsTo(() => ActorModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + Actor: Awaited + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + Video: Awaited + + static load (actorId: number | string, videoId: number | string, t?: Transaction): Promise { + return VideoShareModel.scope(ScopeNames.WITH_ACTOR).findOne({ + where: { + actorId, + videoId + }, + transaction: t + }) + } + + static loadByUrl (url: string, t: Transaction): Promise { + return VideoShareModel.scope(ScopeNames.FULL).findOne({ + where: { + url + }, + transaction: t + }) + } + + static listActorIdsAndFollowerUrlsByShare (videoId: number, t: Transaction) { + const query = `SELECT "actor"."id" AS "id", "actor"."followersUrl" AS "followersUrl" ` + + `FROM "videoShare" ` + + `INNER JOIN "actor" ON "actor"."id" = "videoShare"."actorId" ` + + `WHERE "videoShare"."videoId" = :videoId` + + const options = { + type: QueryTypes.SELECT as QueryTypes.SELECT, + replacements: { videoId }, + transaction: t + } + + return VideoShareModel.sequelize.query(query, options) + } + + static loadActorsWhoSharedVideosOf (actorOwnerId: number, t: Transaction): Promise { + const safeOwnerId = forceNumber(actorOwnerId) + + // /!\ On actor model + const query = { + where: { + [Op.and]: [ + literal( + `EXISTS (` + + ` SELECT 1 FROM "videoShare" ` + + ` INNER JOIN "video" ON "videoShare"."videoId" = "video"."id" ` + + ` INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ` + + ` INNER JOIN "account" ON "account"."id" = "videoChannel"."accountId" ` + + ` WHERE "videoShare"."actorId" = "ActorModel"."id" AND "account"."actorId" = ${safeOwnerId} ` + + ` LIMIT 1` + + `)` + ) + ] + }, + transaction: t + } + + return ActorModel.findAll(query) + } + + static loadActorsByVideoChannel (videoChannelId: number, t: Transaction): Promise { + const safeChannelId = forceNumber(videoChannelId) + + // /!\ On actor model + const query = { + where: { + [Op.and]: [ + literal( + `EXISTS (` + + ` SELECT 1 FROM "videoShare" ` + + ` INNER JOIN "video" ON "videoShare"."videoId" = "video"."id" ` + + ` WHERE "videoShare"."actorId" = "ActorModel"."id" AND "video"."channelId" = ${safeChannelId} ` + + ` LIMIT 1` + + `)` + ) + ] + }, + transaction: t + } + + return ActorModel.findAll(query) + } + + static listAndCountByVideoId (videoId: number, start: number, count: number, t?: Transaction) { + const query = { + offset: start, + limit: count, + where: { + videoId + }, + transaction: t + } + + return Promise.all([ + VideoShareModel.count(query), + VideoShareModel.findAll(query) + ]).then(([ total, data ]) => ({ total, data })) + } + + static listRemoteShareUrlsOfLocalVideos () { + const query = `SELECT "videoShare".url FROM "videoShare" ` + + `INNER JOIN actor ON actor.id = "videoShare"."actorId" AND actor."serverId" IS NOT NULL ` + + `INNER JOIN video ON video.id = "videoShare"."videoId" AND video.remote IS FALSE` + + return VideoShareModel.sequelize.query<{ url: string }>(query, { + type: QueryTypes.SELECT, + raw: true + }).then(rows => rows.map(r => r.url)) + } + + static cleanOldSharesOf (videoId: number, beforeUpdatedAt: Date) { + const query = { + where: { + updatedAt: { + [Op.lt]: beforeUpdatedAt + }, + videoId, + actorId: { + [Op.notIn]: buildLocalActorIdsIn() + } + } + } + + return VideoShareModel.destroy(query) + } +} diff --git a/server/server/models/video/video-source.ts b/server/server/models/video/video-source.ts new file mode 100644 index 000000000..32743e796 --- /dev/null +++ b/server/server/models/video/video-source.ts @@ -0,0 +1,56 @@ +import { Transaction } from 'sequelize' +import { AllowNull, BelongsTo, Column, CreatedAt, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { VideoSource } from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { getSort } from '../shared/index.js' +import { VideoModel } from './video.js' + +@Table({ + tableName: 'videoSource', + indexes: [ + { + fields: [ 'videoId' ] + }, + { + fields: [ { name: 'createdAt', order: 'DESC' } ] + } + ] +}) +export class VideoSourceModel extends Model>> { + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(false) + @Column + filename: string + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + Video: Awaited + + static loadLatest (videoId: number, transaction?: Transaction) { + return VideoSourceModel.findOne({ + where: { videoId }, + order: getSort('-createdAt'), + transaction + }) + } + + toFormattedJSON (): VideoSource { + return { + filename: this.filename, + createdAt: this.createdAt.toISOString() + } + } +} diff --git a/server/server/models/video/video-streaming-playlist.ts b/server/server/models/video/video-streaming-playlist.ts new file mode 100644 index 000000000..0c4043c44 --- /dev/null +++ b/server/server/models/video/video-streaming-playlist.ts @@ -0,0 +1,332 @@ +import { + VideoStorage, + VideoStreamingPlaylistType, + type VideoStorageType, + type VideoStreamingPlaylistType_Type +} from '@peertube/peertube-models' +import { sha1 } from '@peertube/peertube-node-utils' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { CONFIG } from '@server/initializers/config.js' +import { getHLSPrivateFileUrl, getHLSPublicFileUrl } from '@server/lib/object-storage/index.js' +import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '@server/lib/paths.js' +import { isVideoInPrivateDirectory } from '@server/lib/video-privacy.js' +import { VideoFileModel } from '@server/models/video/video-file.js' +import { MStreamingPlaylist, MStreamingPlaylistFilesVideo, MVideo } from '@server/types/models/index.js' +import memoizee from 'memoizee' +import { join } from 'path' +import { Op, Transaction } from 'sequelize' +import { + AllowNull, + BelongsTo, + Column, + CreatedAt, + DataType, + Default, + ForeignKey, + HasMany, + Is, + Model, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc.js' +import { isArrayOf } from '../../helpers/custom-validators/misc.js' +import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos.js' +import { + CONSTRAINTS_FIELDS, + MEMOIZE_LENGTH, + MEMOIZE_TTL, + P2P_MEDIA_LOADER_PEER_VERSION, + STATIC_PATHS, + WEBSERVER +} from '../../initializers/constants.js' +import { VideoRedundancyModel } from '../redundancy/video-redundancy.js' +import { doesExist, throwIfNotValid } from '../shared/index.js' +import { VideoModel } from './video.js' + +@Table({ + tableName: 'videoStreamingPlaylist', + indexes: [ + { + fields: [ 'videoId' ] + }, + { + fields: [ 'videoId', 'type' ], + unique: true + }, + { + fields: [ 'p2pMediaLoaderInfohashes' ], + using: 'gin' + } + ] +}) +export class VideoStreamingPlaylistModel extends Model>> { + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(false) + @Column + type: VideoStreamingPlaylistType_Type + + @AllowNull(false) + @Column + playlistFilename: string + + @AllowNull(true) + @Is('PlaylistUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'playlist url', true)) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max)) + playlistUrl: string + + @AllowNull(false) + @Is('VideoStreamingPlaylistInfoHashes', value => throwIfNotValid(value, v => isArrayOf(v, isVideoFileInfoHashValid), 'info hashes')) + @Column(DataType.ARRAY(DataType.STRING)) + p2pMediaLoaderInfohashes: string[] + + @AllowNull(false) + @Column + p2pMediaLoaderPeerVersion: number + + @AllowNull(false) + @Column + segmentsSha256Filename: string + + @AllowNull(true) + @Is('VideoStreamingSegmentsSha256Url', value => throwIfNotValid(value, isActivityPubUrlValid, 'segments sha256 url', true)) + @Column + segmentsSha256Url: string + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @AllowNull(false) + @Default(VideoStorage.FILE_SYSTEM) + @Column + storage: VideoStorageType + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + Video: Awaited + + @HasMany(() => VideoFileModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'CASCADE' + }) + VideoFiles: Awaited[] + + @HasMany(() => VideoRedundancyModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE', + hooks: true + }) + RedundancyVideos: Awaited[] + + static doesInfohashExistCached = memoizee(VideoStreamingPlaylistModel.doesInfohashExist, { + promise: true, + max: MEMOIZE_LENGTH.INFO_HASH_EXISTS, + maxAge: MEMOIZE_TTL.INFO_HASH_EXISTS + }) + + static doesInfohashExist (infoHash: string) { + const query = 'SELECT 1 FROM "videoStreamingPlaylist" WHERE $infoHash = ANY("p2pMediaLoaderInfohashes") LIMIT 1' + + return doesExist(this.sequelize, query, { infoHash }) + } + + static buildP2PMediaLoaderInfoHashes (playlistUrl: string, files: unknown[]) { + const hashes: string[] = [] + + // https://github.com/Novage/p2p-media-loader/blob/master/p2p-media-loader-core/lib/p2p-media-manager.ts#L115 + for (let i = 0; i < files.length; i++) { + hashes.push(sha1(`${P2P_MEDIA_LOADER_PEER_VERSION}${playlistUrl}+V${i}`)) + } + + return hashes + } + + static listByIncorrectPeerVersion () { + const query = { + where: { + p2pMediaLoaderPeerVersion: { + [Op.ne]: P2P_MEDIA_LOADER_PEER_VERSION + } + }, + include: [ + { + model: VideoModel.unscoped(), + required: true + } + ] + } + + return VideoStreamingPlaylistModel.findAll(query) + } + + static loadWithVideoAndFiles (id: number) { + const options = { + include: [ + { + model: VideoModel.unscoped(), + required: true + }, + { + model: VideoFileModel.unscoped() + } + ] + } + + return VideoStreamingPlaylistModel.findByPk(id, options) + } + + static loadWithVideo (id: number) { + const options = { + include: [ + { + model: VideoModel.unscoped(), + required: true + } + ] + } + + return VideoStreamingPlaylistModel.findByPk(id, options) + } + + static loadHLSPlaylistByVideo (videoId: number, transaction?: Transaction): Promise { + const options = { + where: { + type: VideoStreamingPlaylistType.HLS, + videoId + }, + transaction + } + + return VideoStreamingPlaylistModel.findOne(options) + } + + static async loadOrGenerate (video: MVideo, transaction?: Transaction) { + let playlist = await VideoStreamingPlaylistModel.loadHLSPlaylistByVideo(video.id, transaction) + + if (!playlist) { + playlist = new VideoStreamingPlaylistModel({ + p2pMediaLoaderPeerVersion: P2P_MEDIA_LOADER_PEER_VERSION, + type: VideoStreamingPlaylistType.HLS, + storage: VideoStorage.FILE_SYSTEM, + p2pMediaLoaderInfohashes: [], + playlistFilename: generateHLSMasterPlaylistFilename(video.isLive), + segmentsSha256Filename: generateHlsSha256SegmentsFilename(video.isLive), + videoId: video.id + }) + + await playlist.save({ transaction }) + } + + return Object.assign(playlist, { Video: video }) + } + + static doesOwnedHLSPlaylistExist (videoUUID: string) { + const query = `SELECT 1 FROM "videoStreamingPlaylist" ` + + `INNER JOIN "video" ON "video"."id" = "videoStreamingPlaylist"."videoId" ` + + `AND "video"."remote" IS FALSE AND "video"."uuid" = $videoUUID ` + + `AND "storage" = ${VideoStorage.FILE_SYSTEM} LIMIT 1` + + return doesExist(this.sequelize, query, { videoUUID }) + } + + assignP2PMediaLoaderInfoHashes (video: MVideo, files: unknown[]) { + const masterPlaylistUrl = this.getMasterPlaylistUrl(video) + + this.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(masterPlaylistUrl, files) + } + + // --------------------------------------------------------------------------- + + getMasterPlaylistUrl (video: MVideo) { + if (video.isOwned()) { + if (this.storage === VideoStorage.OBJECT_STORAGE) { + return this.getMasterPlaylistObjectStorageUrl(video) + } + + return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video) + } + + return this.playlistUrl + } + + private getMasterPlaylistObjectStorageUrl (video: MVideo) { + if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) { + return getHLSPrivateFileUrl(video, this.playlistFilename) + } + + return getHLSPublicFileUrl(this.playlistUrl) + } + + // --------------------------------------------------------------------------- + + getSha256SegmentsUrl (video: MVideo) { + if (video.isOwned()) { + if (this.storage === VideoStorage.OBJECT_STORAGE) { + return this.getSha256SegmentsObjectStorageUrl(video) + } + + return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video) + } + + return this.segmentsSha256Url + } + + private getSha256SegmentsObjectStorageUrl (video: MVideo) { + if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) { + return getHLSPrivateFileUrl(video, this.segmentsSha256Filename) + } + + return getHLSPublicFileUrl(this.segmentsSha256Url) + } + + // --------------------------------------------------------------------------- + + getStringType () { + if (this.type === VideoStreamingPlaylistType.HLS) return 'hls' + + return 'unknown' + } + + getTrackerUrls (baseUrlHttp: string, baseUrlWs: string) { + return [ baseUrlWs + '/tracker/socket', baseUrlHttp + '/tracker/announce' ] + } + + hasSameUniqueKeysThan (other: MStreamingPlaylist) { + return this.type === other.type && + this.videoId === other.videoId + } + + withVideo (video: MVideo) { + return Object.assign(this, { Video: video }) + } + + private getMasterPlaylistStaticPath (video: MVideo) { + if (isVideoInPrivateDirectory(video.privacy)) { + return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.playlistFilename) + } + + return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.playlistFilename) + } + + private getSha256SegmentsStaticPath (video: MVideo) { + if (isVideoInPrivateDirectory(video.privacy)) { + return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.segmentsSha256Filename) + } + + return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.segmentsSha256Filename) + } +} diff --git a/server/server/models/video/video-tag.ts b/server/server/models/video/video-tag.ts new file mode 100644 index 000000000..5c36a8de5 --- /dev/null +++ b/server/server/models/video/video-tag.ts @@ -0,0 +1,31 @@ +import { Column, CreatedAt, ForeignKey, Model, Table, UpdatedAt } from 'sequelize-typescript' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { TagModel } from './tag.js' +import { VideoModel } from './video.js' + +@Table({ + tableName: 'videoTag', + indexes: [ + { + fields: [ 'videoId' ] + }, + { + fields: [ 'tagId' ] + } + ] +}) +export class VideoTagModel extends Model>> { + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @ForeignKey(() => TagModel) + @Column + tagId: number +} diff --git a/server/server/models/video/video.ts b/server/server/models/video/video.ts new file mode 100644 index 000000000..a6383b7c7 --- /dev/null +++ b/server/server/models/video/video.ts @@ -0,0 +1,2055 @@ +import { buildVideoEmbedPath, buildVideoWatchPath, pick } from '@peertube/peertube-core-utils' +import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream } from '@peertube/peertube-ffmpeg' +import { + ResultList, + ThumbnailType, + UserRight, + Video, + VideoDetails, + VideoFile, + VideoInclude, + VideoIncludeType, + VideoObject, + VideoPrivacy, + VideoRateType, + VideoState, + VideoStorage, + VideoStreamingPlaylistType, + type VideoPrivacyType, + type VideoStateType +} from '@peertube/peertube-models' +import { uuidToShort } from '@peertube/peertube-node-utils' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { getPrivaciesForFederation, isPrivacyForFederation, isStateForFederation } from '@server/helpers/video.js' +import { InternalEventEmitter } from '@server/lib/internal-event-emitter.js' +import { LiveManager } from '@server/lib/live/live-manager.js' +import { + removeHLSFileObjectStorageByFilename, + removeHLSObjectStorage, + removeWebVideoObjectStorage +} from '@server/lib/object-storage/index.js' +import { tracer } from '@server/lib/opentelemetry/tracing.js' +import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths.js' +import { Hooks } from '@server/lib/plugins/hooks.js' +import { VideoPathManager } from '@server/lib/video-path-manager.js' +import { isVideoInPrivateDirectory } from '@server/lib/video-privacy.js' +import { getServerActor } from '@server/models/application/application.js' +import { ModelCache } from '@server/models/shared/model-cache.js' +import Bluebird from 'bluebird' +import { remove } from 'fs-extra/esm' +import maxBy from 'lodash-es/maxBy.js' +import minBy from 'lodash-es/minBy.js' +import { FindOptions, IncludeOptions, Includeable, Op, QueryTypes, ScopeOptions, Sequelize, Transaction, WhereOptions } from 'sequelize' +import { + AfterCreate, + AfterDestroy, + AfterUpdate, + AllowNull, + BeforeDestroy, + BelongsTo, + BelongsToMany, + Column, + CreatedAt, + DataType, + Default, + ForeignKey, + HasMany, + HasOne, + Is, + IsInt, + IsUUID, + Min, + Model, + Scopes, + Table, + UpdatedAt +} from 'sequelize-typescript' +import { peertubeTruncate } from '../../helpers/core-utils.js' +import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc.js' +import { exists, isArray, isBooleanValid, isUUIDValid } from '../../helpers/custom-validators/misc.js' +import { + isVideoDescriptionValid, + isVideoDurationValid, + isVideoNameValid, + isVideoPrivacyValid, + isVideoStateValid, + isVideoSupportValid +} from '../../helpers/custom-validators/videos.js' +import { logger } from '../../helpers/logger.js' +import { CONFIG } from '../../initializers/config.js' +import { ACTIVITY_PUB, API_VERSION, CONSTRAINTS_FIELDS, WEBSERVER } from '../../initializers/constants.js' +import { sendDeleteVideo } from '../../lib/activitypub/send/index.js' +import { + MChannel, + MChannelAccountDefault, + MChannelId, + MStoryboard, + MStreamingPlaylist, + MStreamingPlaylistFilesVideo, + MUserAccountId, + MUserId, + MVideoAP, + MVideoAPLight, + MVideoAccountLightBlacklistAllFiles, + MVideoCaptionLanguageUrl, + MVideoDetails, + MVideoFileVideo, + MVideoForUser, + MVideoFormattable, + MVideoFormattableDetails, + MVideoFullLight, + MVideoId, + MVideoImmutable, + MVideoThumbnail, + MVideoThumbnailBlacklist, + MVideoWithAllFiles, + MVideoWithFile, + type MVideo, + type MVideoAccountLight +} from '../../types/models/index.js' +import { MThumbnail } from '../../types/models/video/thumbnail.js' +import { MVideoFile, MVideoFileStreamingPlaylistVideo } from '../../types/models/video/video-file.js' +import { VideoAbuseModel } from '../abuse/video-abuse.js' +import { AccountVideoRateModel } from '../account/account-video-rate.js' +import { AccountModel } from '../account/account.js' +import { ActorImageModel } from '../actor/actor-image.js' +import { ActorModel } from '../actor/actor.js' +import { VideoRedundancyModel } from '../redundancy/video-redundancy.js' +import { ServerModel } from '../server/server.js' +import { TrackerModel } from '../server/tracker.js' +import { VideoTrackerModel } from '../server/video-tracker.js' +import { buildTrigramSearchIndex, buildWhereIdOrUUID, getVideoSort, isOutdated, setAsUpdated, throwIfNotValid } from '../shared/index.js' +import { UserVideoHistoryModel } from '../user/user-video-history.js' +import { UserModel } from '../user/user.js' +import { VideoViewModel } from '../view/video-view.js' +import { videoModelToActivityPubObject } from './formatter/video-activity-pub-format.js' +import { + VideoFormattingJSONOptions, + videoFilesModelToFormattedJSON, + videoModelToFormattedDetailsJSON, + videoModelToFormattedJSON +} from './formatter/video-api-format.js' +import { ScheduleVideoUpdateModel } from './schedule-video-update.js' +import { + BuildVideosListQueryOptions, + DisplayOnlyForFollowerOptions, + VideoModelGetQueryBuilder, + VideosIdListQueryBuilder, + VideosModelListQueryBuilder +} from './sql/video/index.js' +import { StoryboardModel } from './storyboard.js' +import { TagModel } from './tag.js' +import { ThumbnailModel } from './thumbnail.js' +import { VideoBlacklistModel } from './video-blacklist.js' +import { VideoCaptionModel } from './video-caption.js' +import { SummaryOptions, VideoChannelModel, ScopeNames as VideoChannelScopeNames } from './video-channel.js' +import { VideoCommentModel } from './video-comment.js' +import { VideoFileModel } from './video-file.js' +import { VideoImportModel } from './video-import.js' +import { VideoJobInfoModel } from './video-job-info.js' +import { VideoLiveModel } from './video-live.js' +import { VideoPasswordModel } from './video-password.js' +import { VideoPlaylistElementModel } from './video-playlist-element.js' +import { VideoShareModel } from './video-share.js' +import { VideoSourceModel } from './video-source.js' +import { VideoStreamingPlaylistModel } from './video-streaming-playlist.js' +import { VideoTagModel } from './video-tag.js' + +export enum ScopeNames { + FOR_API = 'FOR_API', + WITH_ACCOUNT_DETAILS = 'WITH_ACCOUNT_DETAILS', + WITH_TAGS = 'WITH_TAGS', + WITH_WEB_VIDEO_FILES = 'WITH_WEB_VIDEO_FILES', + WITH_SCHEDULED_UPDATE = 'WITH_SCHEDULED_UPDATE', + WITH_BLACKLISTED = 'WITH_BLACKLISTED', + WITH_STREAMING_PLAYLISTS = 'WITH_STREAMING_PLAYLISTS', + WITH_IMMUTABLE_ATTRIBUTES = 'WITH_IMMUTABLE_ATTRIBUTES', + WITH_USER_HISTORY = 'WITH_USER_HISTORY', + WITH_THUMBNAILS = 'WITH_THUMBNAILS' +} + +export type ForAPIOptions = { + ids?: number[] + + videoPlaylistId?: number + + withAccountBlockerIds?: number[] +} + +@Scopes(() => ({ + [ScopeNames.WITH_IMMUTABLE_ATTRIBUTES]: { + attributes: [ 'id', 'url', 'uuid', 'remote' ] + }, + [ScopeNames.FOR_API]: (options: ForAPIOptions) => { + const include: Includeable[] = [ + { + model: VideoChannelModel.scope({ + method: [ + VideoChannelScopeNames.SUMMARY, { + withAccount: true, + withAccountBlockerIds: options.withAccountBlockerIds + } as SummaryOptions + ] + }), + required: true + }, + { + attributes: [ 'type', 'filename' ], + model: ThumbnailModel, + required: false + } + ] + + const query: FindOptions = {} + + if (options.ids) { + query.where = { + id: { + [Op.in]: options.ids + } + } + } + + if (options.videoPlaylistId) { + include.push({ + model: VideoPlaylistElementModel.unscoped(), + required: true, + where: { + videoPlaylistId: options.videoPlaylistId + } + }) + } + + query.include = include + + return query + }, + [ScopeNames.WITH_THUMBNAILS]: { + include: [ + { + model: ThumbnailModel, + required: false + } + ] + }, + [ScopeNames.WITH_ACCOUNT_DETAILS]: { + include: [ + { + model: VideoChannelModel.unscoped(), + required: true, + include: [ + { + attributes: { + exclude: [ 'privateKey', 'publicKey' ] + }, + model: ActorModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'host' ], + model: ServerModel.unscoped(), + required: false + }, + { + model: ActorImageModel, + as: 'Avatars', + required: false + } + ] + }, + { + model: AccountModel.unscoped(), + required: true, + include: [ + { + model: ActorModel.unscoped(), + attributes: { + exclude: [ 'privateKey', 'publicKey' ] + }, + required: true, + include: [ + { + attributes: [ 'host' ], + model: ServerModel.unscoped(), + required: false + }, + { + model: ActorImageModel, + as: 'Avatars', + required: false + } + ] + } + ] + } + ] + } + ] + }, + [ScopeNames.WITH_TAGS]: { + include: [ TagModel ] + }, + [ScopeNames.WITH_BLACKLISTED]: { + include: [ + { + attributes: [ 'id', 'reason', 'unfederated' ], + model: VideoBlacklistModel, + required: false + } + ] + }, + [ScopeNames.WITH_WEB_VIDEO_FILES]: (withRedundancies = false) => { + let subInclude: any[] = [] + + if (withRedundancies === true) { + subInclude = [ + { + attributes: [ 'fileUrl' ], + model: VideoRedundancyModel.unscoped(), + required: false + } + ] + } + + return { + include: [ + { + model: VideoFileModel, + separate: true, + required: false, + include: subInclude + } + ] + } + }, + [ScopeNames.WITH_STREAMING_PLAYLISTS]: (withRedundancies = false) => { + const subInclude: IncludeOptions[] = [ + { + model: VideoFileModel, + required: false + } + ] + + if (withRedundancies === true) { + subInclude.push({ + attributes: [ 'fileUrl' ], + model: VideoRedundancyModel.unscoped(), + required: false + }) + } + + return { + include: [ + { + model: VideoStreamingPlaylistModel.unscoped(), + required: false, + separate: true, + include: subInclude + } + ] + } + }, + [ScopeNames.WITH_SCHEDULED_UPDATE]: { + include: [ + { + model: ScheduleVideoUpdateModel.unscoped(), + required: false + } + ] + }, + [ScopeNames.WITH_USER_HISTORY]: (userId: number) => { + return { + include: [ + { + attributes: [ 'currentTime' ], + model: UserVideoHistoryModel.unscoped(), + required: false, + where: { + userId + } + } + ] + } + } +})) +@Table({ + tableName: 'video', + indexes: [ + buildTrigramSearchIndex('video_name_trigram', 'name'), + + { fields: [ 'createdAt' ] }, + { + fields: [ + { name: 'publishedAt', order: 'DESC' }, + { name: 'id', order: 'ASC' } + ] + }, + { fields: [ 'duration' ] }, + { + fields: [ + { name: 'views', order: 'DESC' }, + { name: 'id', order: 'ASC' } + ] + }, + { fields: [ 'channelId' ] }, + { + fields: [ 'originallyPublishedAt' ], + where: { + originallyPublishedAt: { + [Op.ne]: null + } + } + }, + { + fields: [ 'category' ], // We don't care videos with an unknown category + where: { + category: { + [Op.ne]: null + } + } + }, + { + fields: [ 'licence' ], // We don't care videos with an unknown licence + where: { + licence: { + [Op.ne]: null + } + } + }, + { + fields: [ 'language' ], // We don't care videos with an unknown language + where: { + language: { + [Op.ne]: null + } + } + }, + { + fields: [ 'nsfw' ], // Most of the videos are not NSFW + where: { + nsfw: true + } + }, + { + fields: [ 'remote' ], // Only index local videos + where: { + remote: false + } + }, + { + fields: [ 'uuid' ], + unique: true + }, + { + fields: [ 'url' ], + unique: true + } + ] +}) +export class VideoModel extends Model>> { + + @AllowNull(false) + @Default(DataType.UUIDV4) + @IsUUID(4) + @Column(DataType.UUID) + uuid: string + + @AllowNull(false) + @Is('VideoName', value => throwIfNotValid(value, isVideoNameValid, 'name')) + @Column + name: string + + @AllowNull(true) + @Default(null) + @Column + category: number + + @AllowNull(true) + @Default(null) + @Column + licence: number + + @AllowNull(true) + @Default(null) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.LANGUAGE.max)) + language: string + + @AllowNull(false) + @Is('VideoPrivacy', value => throwIfNotValid(value, isVideoPrivacyValid, 'privacy')) + @Column(DataType.INTEGER) + privacy: VideoPrivacyType + + @AllowNull(false) + @Is('VideoNSFW', value => throwIfNotValid(value, isBooleanValid, 'NSFW boolean')) + @Column + nsfw: boolean + + @AllowNull(true) + @Default(null) + @Is('VideoDescription', value => throwIfNotValid(value, isVideoDescriptionValid, 'description', true)) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.DESCRIPTION.max)) + description: string + + @AllowNull(true) + @Default(null) + @Is('VideoSupport', value => throwIfNotValid(value, isVideoSupportValid, 'support', true)) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.SUPPORT.max)) + support: string + + @AllowNull(false) + @Is('VideoDuration', value => throwIfNotValid(value, isVideoDurationValid, 'duration')) + @Column + duration: number + + @AllowNull(false) + @Default(0) + @IsInt + @Min(0) + @Column + views: number + + @AllowNull(false) + @Default(0) + @IsInt + @Min(0) + @Column + likes: number + + @AllowNull(false) + @Default(0) + @IsInt + @Min(0) + @Column + dislikes: number + + @AllowNull(false) + @Column + remote: boolean + + @AllowNull(false) + @Default(false) + @Column + isLive: boolean + + @AllowNull(false) + @Is('VideoUrl', value => throwIfNotValid(value, isActivityPubUrlValid, 'url')) + @Column(DataType.STRING(CONSTRAINTS_FIELDS.VIDEOS.URL.max)) + url: string + + @AllowNull(false) + @Column + commentsEnabled: boolean + + @AllowNull(false) + @Column + downloadEnabled: boolean + + @AllowNull(false) + @Column + waitTranscoding: boolean + + @AllowNull(false) + @Default(null) + @Is('VideoState', value => throwIfNotValid(value, isVideoStateValid, 'state')) + @Column + state: VideoStateType + + // We already have the information in videoSource table for local videos, but we prefer to normalize it for performance + // And also to store the info from remote instances + @AllowNull(true) + @Column + inputFileUpdatedAt: Date + + @CreatedAt + createdAt: Date + + @UpdatedAt + updatedAt: Date + + @AllowNull(false) + @Default(DataType.NOW) + @Column + publishedAt: Date + + @AllowNull(true) + @Default(null) + @Column + originallyPublishedAt: Date + + @ForeignKey(() => VideoChannelModel) + @Column + channelId: number + + @BelongsTo(() => VideoChannelModel, { + foreignKey: { + allowNull: true + }, + onDelete: 'cascade' + }) + VideoChannel: Awaited + + @BelongsToMany(() => TagModel, { + foreignKey: 'videoId', + through: () => VideoTagModel, + onDelete: 'CASCADE' + }) + Tags: Awaited[] + + @BelongsToMany(() => TrackerModel, { + foreignKey: 'videoId', + through: () => VideoTrackerModel, + onDelete: 'CASCADE' + }) + Trackers: Awaited[] + + @HasMany(() => ThumbnailModel, { + foreignKey: { + name: 'videoId', + allowNull: true + }, + hooks: true, + onDelete: 'cascade' + }) + Thumbnails: Awaited[] + + @HasMany(() => VideoPlaylistElementModel, { + foreignKey: { + name: 'videoId', + allowNull: true + }, + onDelete: 'set null' + }) + VideoPlaylistElements: Awaited[] + + @HasOne(() => VideoSourceModel, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'CASCADE' + }) + VideoSource: Awaited + + @HasMany(() => VideoAbuseModel, { + foreignKey: { + name: 'videoId', + allowNull: true + }, + onDelete: 'set null' + }) + VideoAbuses: Awaited[] + + @HasMany(() => VideoFileModel, { + foreignKey: { + name: 'videoId', + allowNull: true + }, + hooks: true, + onDelete: 'cascade' + }) + VideoFiles: Awaited[] + + @HasMany(() => VideoStreamingPlaylistModel, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + hooks: true, + onDelete: 'cascade' + }) + VideoStreamingPlaylists: Awaited[] + + @HasMany(() => VideoShareModel, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade' + }) + VideoShares: Awaited[] + + @HasMany(() => AccountVideoRateModel, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade' + }) + AccountVideoRates: Awaited[] + + @HasMany(() => VideoCommentModel, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade', + hooks: true + }) + VideoComments: Awaited[] + + @HasMany(() => VideoViewModel, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade' + }) + VideoViews: Awaited[] + + @HasMany(() => UserVideoHistoryModel, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade' + }) + UserVideoHistories: Awaited[] + + @HasOne(() => ScheduleVideoUpdateModel, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade' + }) + ScheduleVideoUpdate: Awaited + + @HasOne(() => VideoBlacklistModel, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade' + }) + VideoBlacklist: Awaited + + @HasOne(() => VideoLiveModel, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + hooks: true, + onDelete: 'cascade' + }) + VideoLive: Awaited + + @HasOne(() => VideoImportModel, { + foreignKey: { + name: 'videoId', + allowNull: true + }, + onDelete: 'set null' + }) + VideoImport: Awaited + + @HasMany(() => VideoCaptionModel, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade', + hooks: true, + ['separate' as any]: true + }) + VideoCaptions: Awaited[] + + @HasMany(() => VideoPasswordModel, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade' + }) + VideoPasswords: Awaited[] + + @HasOne(() => VideoJobInfoModel, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade' + }) + VideoJobInfo: Awaited + + @HasOne(() => StoryboardModel, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade', + hooks: true + }) + Storyboard: Awaited + + @AfterCreate + static notifyCreate (video: MVideo) { + InternalEventEmitter.Instance.emit('video-created', { video }) + } + + @AfterUpdate + static notifyUpdate (video: MVideo) { + InternalEventEmitter.Instance.emit('video-updated', { video }) + } + + @AfterDestroy + static notifyDestroy (video: MVideo) { + InternalEventEmitter.Instance.emit('video-deleted', { video }) + } + + @BeforeDestroy + static async sendDelete (instance: MVideoAccountLight, options: { transaction: Transaction }) { + if (!instance.isOwned()) return undefined + + // Lazy load channels + if (!instance.VideoChannel) { + instance.VideoChannel = await instance.$get('VideoChannel', { + include: [ + ActorModel, + AccountModel + ], + transaction: options.transaction + }) as MChannelAccountDefault + } + + return sendDeleteVideo(instance, options.transaction) + } + + @BeforeDestroy + static async removeFiles (instance: VideoModel, options) { + const tasks: Promise[] = [] + + logger.info('Removing files of video %s.', instance.url) + + if (instance.isOwned()) { + if (!Array.isArray(instance.VideoFiles)) { + instance.VideoFiles = await instance.$get('VideoFiles', { transaction: options.transaction }) + } + + // Remove physical files and torrents + instance.VideoFiles.forEach(file => { + tasks.push(instance.removeWebVideoFile(file)) + }) + + // Remove playlists file + if (!Array.isArray(instance.VideoStreamingPlaylists)) { + instance.VideoStreamingPlaylists = await instance.$get('VideoStreamingPlaylists', { transaction: options.transaction }) + } + + for (const p of instance.VideoStreamingPlaylists) { + tasks.push(instance.removeStreamingPlaylistFiles(p)) + } + } + + // Do not wait video deletion because we could be in a transaction + Promise.all(tasks) + .then(() => logger.info('Removed files of video %s.', instance.url)) + .catch(err => logger.error('Some errors when removing files of video %s in before destroy hook.', instance.uuid, { err })) + + return undefined + } + + @BeforeDestroy + static stopLiveIfNeeded (instance: VideoModel) { + if (!instance.isLive) return + + logger.info('Stopping live of video %s after video deletion.', instance.uuid) + + LiveManager.Instance.stopSessionOf(instance.uuid, null) + } + + @BeforeDestroy + static invalidateCache (instance: VideoModel) { + ModelCache.Instance.invalidateCache('video', instance.id) + } + + @BeforeDestroy + static async saveEssentialDataToAbuses (instance: VideoModel, options) { + const tasks: Promise[] = [] + + if (!Array.isArray(instance.VideoAbuses)) { + instance.VideoAbuses = await instance.$get('VideoAbuses', { transaction: options.transaction }) + + if (instance.VideoAbuses.length === 0) return undefined + } + + logger.info('Saving video abuses details of video %s.', instance.url) + + if (!instance.Trackers) instance.Trackers = await instance.$get('Trackers', { transaction: options.transaction }) + const details = instance.toFormattedDetailsJSON() + + for (const abuse of instance.VideoAbuses) { + abuse.deletedVideo = details + tasks.push(abuse.save({ transaction: options.transaction })) + } + + await Promise.all(tasks) + } + + static listLocalIds (): Promise { + const query = { + attributes: [ 'id' ], + raw: true, + where: { + remote: false + } + } + + return VideoModel.findAll(query) + .then(rows => rows.map(r => r.id)) + } + + static listAllAndSharedByActorForOutbox (actorId: number, start: number, count: number) { + function getRawQuery (select: string) { + const queryVideo = 'SELECT ' + select + ' FROM "video" AS "Video" ' + + 'INNER JOIN "videoChannel" AS "VideoChannel" ON "VideoChannel"."id" = "Video"."channelId" ' + + 'INNER JOIN "account" AS "Account" ON "Account"."id" = "VideoChannel"."accountId" ' + + 'WHERE "Account"."actorId" = ' + actorId + const queryVideoShare = 'SELECT ' + select + ' FROM "videoShare" AS "VideoShare" ' + + 'INNER JOIN "video" AS "Video" ON "Video"."id" = "VideoShare"."videoId" ' + + 'WHERE "VideoShare"."actorId" = ' + actorId + + return `(${queryVideo}) UNION (${queryVideoShare})` + } + + const rawQuery = getRawQuery('"Video"."id"') + const rawCountQuery = getRawQuery('COUNT("Video"."id") as "total"') + + const query = { + distinct: true, + offset: start, + limit: count, + order: getVideoSort('-createdAt', [ 'Tags', 'name', 'ASC' ]), + where: { + id: { + [Op.in]: Sequelize.literal('(' + rawQuery + ')') + }, + [Op.or]: getPrivaciesForFederation() + }, + include: [ + { + attributes: [ 'filename', 'language', 'fileUrl' ], + model: VideoCaptionModel.unscoped(), + required: false + }, + { + model: StoryboardModel.unscoped(), + required: false + }, + { + attributes: [ 'id', 'url' ], + model: VideoShareModel.unscoped(), + required: false, + // We only want videos shared by this actor + where: { + [Op.and]: [ + { + id: { + [Op.not]: null + } + }, + { + actorId + } + ] + }, + include: [ + { + attributes: [ 'id', 'url' ], + model: ActorModel.unscoped() + } + ] + }, + { + model: VideoChannelModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'name' ], + model: AccountModel.unscoped(), + required: true, + include: [ + { + attributes: [ 'id', 'url', 'followersUrl' ], + model: ActorModel.unscoped(), + required: true + } + ] + }, + { + attributes: [ 'id', 'url', 'followersUrl' ], + model: ActorModel.unscoped(), + required: true + } + ] + }, + { + model: VideoStreamingPlaylistModel.unscoped(), + required: false, + include: [ + { + model: VideoFileModel, + required: false + } + ] + }, + VideoLiveModel.unscoped(), + VideoFileModel, + TagModel + ] + } + + return Bluebird.all([ + VideoModel.scope(ScopeNames.WITH_THUMBNAILS).findAll(query), + VideoModel.sequelize.query<{ total: string }>(rawCountQuery, { type: QueryTypes.SELECT }) + ]).then(([ rows, totals ]) => { + // totals: totalVideos + totalVideoShares + let totalVideos = 0 + let totalVideoShares = 0 + if (totals[0]) totalVideos = parseInt(totals[0].total, 10) + if (totals[1]) totalVideoShares = parseInt(totals[1].total, 10) + + const total = totalVideos + totalVideoShares + return { + data: rows, + total + } + }) + } + + static async listPublishedLiveUUIDs () { + const options = { + attributes: [ 'uuid' ], + where: { + isLive: true, + remote: false, + state: VideoState.PUBLISHED + } + } + + const result = await VideoModel.findAll(options) + + return result.map(v => v.uuid) + } + + static listUserVideosForApi (options: { + accountId: number + start: number + count: number + sort: string + + channelId?: number + isLive?: boolean + search?: string + }) { + const { accountId, channelId, start, count, sort, search, isLive } = options + + function buildBaseQuery (forCount: boolean): FindOptions { + const where: WhereOptions = {} + + if (search) { + where.name = { + [Op.iLike]: '%' + search + '%' + } + } + + if (exists(isLive)) { + where.isLive = isLive + } + + const channelWhere = channelId + ? { id: channelId } + : {} + + const baseQuery = { + offset: start, + limit: count, + where, + order: getVideoSort(sort), + include: [ + { + model: forCount + ? VideoChannelModel.unscoped() + : VideoChannelModel, + required: true, + where: channelWhere, + include: [ + { + model: forCount + ? AccountModel.unscoped() + : AccountModel, + where: { + id: accountId + }, + required: true + } + ] + } + ] + } + + return baseQuery + } + + const countQuery = buildBaseQuery(true) + const findQuery = buildBaseQuery(false) + + const findScopes: (string | ScopeOptions)[] = [ + ScopeNames.WITH_SCHEDULED_UPDATE, + ScopeNames.WITH_BLACKLISTED, + ScopeNames.WITH_THUMBNAILS + ] + + return Promise.all([ + VideoModel.count(countQuery), + VideoModel.scope(findScopes).findAll(findQuery) + ]).then(([ count, rows ]) => { + return { + data: rows, + total: count + } + }) + } + + static async listForApi (options: { + start: number + count: number + sort: string + + nsfw: boolean + isLive?: boolean + isLocal?: boolean + include?: VideoIncludeType + + hasFiles?: boolean // default false + + hasWebtorrentFiles?: boolean // TODO: remove in v7 + hasWebVideoFiles?: boolean + + hasHLSFiles?: boolean + + categoryOneOf?: number[] + licenceOneOf?: number[] + languageOneOf?: string[] + tagsOneOf?: string[] + tagsAllOf?: string[] + privacyOneOf?: VideoPrivacyType[] + + accountId?: number + videoChannelId?: number + + displayOnlyForFollower: DisplayOnlyForFollowerOptions | null + + videoPlaylistId?: number + + trendingDays?: number + + user?: MUserAccountId + historyOfUser?: MUserId + + countVideos?: boolean + + search?: string + + excludeAlreadyWatched?: boolean + }) { + VideoModel.throwIfPrivateIncludeWithoutUser(options.include, options.user) + VideoModel.throwIfPrivacyOneOfWithoutUser(options.privacyOneOf, options.user) + + const trendingDays = options.sort.endsWith('trending') + ? CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS + : undefined + + let trendingAlgorithm: string + if (options.sort.endsWith('hot')) trendingAlgorithm = 'hot' + if (options.sort.endsWith('best')) trendingAlgorithm = 'best' + + const serverActor = await getServerActor() + + const queryOptions = { + ...pick(options, [ + 'start', + 'count', + 'sort', + 'nsfw', + 'isLive', + 'categoryOneOf', + 'licenceOneOf', + 'languageOneOf', + 'tagsOneOf', + 'tagsAllOf', + 'privacyOneOf', + 'isLocal', + 'include', + 'displayOnlyForFollower', + 'hasFiles', + 'accountId', + 'videoChannelId', + 'videoPlaylistId', + 'user', + 'historyOfUser', + 'hasHLSFiles', + 'hasWebtorrentFiles', + 'hasWebVideoFiles', + 'search', + 'excludeAlreadyWatched' + ]), + + serverAccountIdForBlock: serverActor.Account.id, + trendingDays, + trendingAlgorithm + } + + return VideoModel.getAvailableForApi(queryOptions, options.countVideos) + } + + static async searchAndPopulateAccountAndServer (options: { + start: number + count: number + sort: string + + nsfw?: boolean + isLive?: boolean + isLocal?: boolean + include?: VideoIncludeType + + categoryOneOf?: number[] + licenceOneOf?: number[] + languageOneOf?: string[] + tagsOneOf?: string[] + tagsAllOf?: string[] + privacyOneOf?: VideoPrivacyType[] + + displayOnlyForFollower: DisplayOnlyForFollowerOptions | null + + user?: MUserAccountId + + hasWebtorrentFiles?: boolean // TODO: remove in v7 + hasWebVideoFiles?: boolean + + hasHLSFiles?: boolean + + search?: string + + host?: string + startDate?: string // ISO 8601 + endDate?: string // ISO 8601 + originallyPublishedStartDate?: string + originallyPublishedEndDate?: string + + durationMin?: number // seconds + durationMax?: number // seconds + uuids?: string[] + + excludeAlreadyWatched?: boolean + }) { + VideoModel.throwIfPrivateIncludeWithoutUser(options.include, options.user) + VideoModel.throwIfPrivacyOneOfWithoutUser(options.privacyOneOf, options.user) + + const serverActor = await getServerActor() + + const queryOptions = { + ...pick(options, [ + 'include', + 'nsfw', + 'isLive', + 'categoryOneOf', + 'licenceOneOf', + 'languageOneOf', + 'tagsOneOf', + 'tagsAllOf', + 'privacyOneOf', + 'user', + 'isLocal', + 'host', + 'start', + 'count', + 'sort', + 'startDate', + 'endDate', + 'originallyPublishedStartDate', + 'originallyPublishedEndDate', + 'durationMin', + 'durationMax', + 'hasHLSFiles', + 'hasWebtorrentFiles', + 'hasWebVideoFiles', + 'uuids', + 'search', + 'displayOnlyForFollower', + 'excludeAlreadyWatched' + ]), + serverAccountIdForBlock: serverActor.Account.id + } + + return VideoModel.getAvailableForApi(queryOptions) + } + + static countLives (options: { + remote: boolean + mode: 'published' | 'not-ended' + }) { + const query = { + where: { + remote: options.remote, + isLive: true, + state: options.mode === 'not-ended' + ? { [Op.ne]: VideoState.LIVE_ENDED } + : { [Op.eq]: VideoState.PUBLISHED } + } + } + + return VideoModel.count(query) + } + + static countVideosUploadedByUserSince (userId: number, since: Date) { + const options = { + include: [ + { + model: VideoChannelModel.unscoped(), + required: true, + include: [ + { + model: AccountModel.unscoped(), + required: true, + include: [ + { + model: UserModel.unscoped(), + required: true, + where: { + id: userId + } + } + ] + } + ] + } + ], + where: { + createdAt: { + [Op.gte]: since + } + } + } + + return VideoModel.unscoped().count(options) + } + + static countLivesOfAccount (accountId: number) { + const options = { + where: { + remote: false, + isLive: true, + state: { + [Op.ne]: VideoState.LIVE_ENDED + } + }, + include: [ + { + required: true, + model: VideoChannelModel.unscoped(), + where: { + accountId + } + } + ] + } + + return VideoModel.count(options) + } + + static load (id: number | string, transaction?: Transaction): Promise { + const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize) + + return queryBuilder.queryVideo({ id, transaction, type: 'thumbnails' }) + } + + static loadWithBlacklist (id: number | string, transaction?: Transaction): Promise { + const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize) + + return queryBuilder.queryVideo({ id, transaction, type: 'thumbnails-blacklist' }) + } + + static loadImmutableAttributes (id: number | string, t?: Transaction): Promise { + const fun = () => { + const query = { + where: buildWhereIdOrUUID(id), + transaction: t + } + + return VideoModel.scope(ScopeNames.WITH_IMMUTABLE_ATTRIBUTES).findOne(query) + } + + return ModelCache.Instance.doCache({ + cacheType: 'load-video-immutable-id', + key: '' + id, + deleteKey: 'video', + fun + }) + } + + static loadByUrlImmutableAttributes (url: string, transaction?: Transaction): Promise { + const fun = () => { + const query: FindOptions = { + where: { + url + }, + transaction + } + + return VideoModel.scope(ScopeNames.WITH_IMMUTABLE_ATTRIBUTES).findOne(query) + } + + return ModelCache.Instance.doCache({ + cacheType: 'load-video-immutable-url', + key: url, + deleteKey: 'video', + fun + }) + } + + static loadOnlyId (id: number | string, transaction?: Transaction): Promise { + const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize) + + return queryBuilder.queryVideo({ id, transaction, type: 'id' }) + } + + static loadWithFiles (id: number | string, transaction?: Transaction, logging?: boolean): Promise { + const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize) + + return queryBuilder.queryVideo({ id, transaction, type: 'all-files', logging }) + } + + static loadByUrl (url: string, transaction?: Transaction): Promise { + const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize) + + return queryBuilder.queryVideo({ url, transaction, type: 'thumbnails' }) + } + + static loadByUrlAndPopulateAccount (url: string, transaction?: Transaction): Promise { + const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize) + + return queryBuilder.queryVideo({ url, transaction, type: 'account-blacklist-files' }) + } + + static loadFull (id: number | string, t?: Transaction, userId?: number): Promise { + const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize) + + return queryBuilder.queryVideo({ id, transaction: t, type: 'full', userId }) + } + + static loadForGetAPI (parameters: { + id: number | string + transaction?: Transaction + userId?: number + }): Promise { + const { id, transaction, userId } = parameters + const queryBuilder = new VideoModelGetQueryBuilder(VideoModel.sequelize) + + return queryBuilder.queryVideo({ id, transaction, type: 'api', userId }) + } + + static async getStats () { + const serverActor = await getServerActor() + + let totalLocalVideoViews = await VideoModel.sum('views', { + where: { + remote: false + } + }) + + // Sequelize could return null... + if (!totalLocalVideoViews) totalLocalVideoViews = 0 + + const baseOptions = { + start: 0, + count: 0, + sort: '-publishedAt', + nsfw: null, + displayOnlyForFollower: { + actorId: serverActor.id, + orLocalVideos: true + } + } + + const { total: totalLocalVideos } = await VideoModel.listForApi({ + ...baseOptions, + + isLocal: true + }) + + const { total: totalVideos } = await VideoModel.listForApi(baseOptions) + + return { + totalLocalVideos, + totalLocalVideoViews, + totalVideos + } + } + + static incrementViews (id: number, views: number) { + return VideoModel.increment('views', { + by: views, + where: { + id + } + }) + } + + static updateRatesOf (videoId: number, type: VideoRateType, count: number, t: Transaction) { + const field = type === 'like' + ? 'likes' + : 'dislikes' + + const rawQuery = `UPDATE "video" SET "${field}" = :count WHERE "video"."id" = :videoId` + + return AccountVideoRateModel.sequelize.query(rawQuery, { + transaction: t, + replacements: { videoId, rateType: type, count }, + type: QueryTypes.UPDATE + }) + } + + static syncLocalRates (videoId: number, type: VideoRateType, t: Transaction) { + const field = type === 'like' + ? 'likes' + : 'dislikes' + + const rawQuery = `UPDATE "video" SET "${field}" = ` + + '(' + + 'SELECT COUNT(id) FROM "accountVideoRate" WHERE "accountVideoRate"."videoId" = "video"."id" AND type = :rateType' + + ') ' + + 'WHERE "video"."id" = :videoId' + + return AccountVideoRateModel.sequelize.query(rawQuery, { + transaction: t, + replacements: { videoId, rateType: type }, + type: QueryTypes.UPDATE + }) + } + + static checkVideoHasInstanceFollow (videoId: number, followerActorId: number) { + // Instances only share videos + const query = 'SELECT 1 FROM "videoShare" ' + + 'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "videoShare"."actorId" ' + + 'WHERE "actorFollow"."actorId" = $followerActorId AND "actorFollow"."state" = \'accepted\' AND "videoShare"."videoId" = $videoId ' + + 'UNION ' + + 'SELECT 1 FROM "video" ' + + 'INNER JOIN "videoChannel" ON "videoChannel"."id" = "video"."channelId" ' + + 'INNER JOIN "account" ON "account"."id" = "videoChannel"."accountId" ' + + 'INNER JOIN "actorFollow" ON "actorFollow"."targetActorId" = "account"."actorId" ' + + 'WHERE "actorFollow"."actorId" = $followerActorId AND "actorFollow"."state" = \'accepted\' AND "video"."id" = $videoId ' + + 'LIMIT 1' + + const options = { + type: QueryTypes.SELECT as QueryTypes.SELECT, + bind: { followerActorId, videoId }, + raw: true + } + + return VideoModel.sequelize.query(query, options) + .then(results => results.length === 1) + } + + static bulkUpdateSupportField (ofChannel: MChannel, t: Transaction) { + const options = { + where: { + channelId: ofChannel.id + }, + transaction: t + } + + return VideoModel.update({ support: ofChannel.support }, options) + } + + static getAllIdsFromChannel (videoChannel: MChannelId): Promise { + const query = { + attributes: [ 'id' ], + where: { + channelId: videoChannel.id + } + } + + return VideoModel.findAll(query) + .then(videos => videos.map(v => v.id)) + } + + // threshold corresponds to how many video the field should have to be returned + static async getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) { + const serverActor = await getServerActor() + + const queryOptions: BuildVideosListQueryOptions = { + attributes: [ `"${field}"` ], + group: `GROUP BY "${field}"`, + having: `HAVING COUNT("${field}") >= ${threshold}`, + start: 0, + sort: 'random', + count, + serverAccountIdForBlock: serverActor.Account.id, + displayOnlyForFollower: { + actorId: serverActor.id, + orLocalVideos: true + } + } + + const queryBuilder = new VideosIdListQueryBuilder(VideoModel.sequelize) + + return queryBuilder.queryVideoIds(queryOptions) + .then(rows => rows.map(r => r[field])) + } + + static buildTrendingQuery (trendingDays: number) { + return { + attributes: [], + subQuery: false, + model: VideoViewModel, + required: false, + where: { + startDate: { + // FIXME: ts error + [Op.gte as any]: new Date(new Date().getTime() - (24 * 3600 * 1000) * trendingDays) + } + } + } + } + + private static async getAvailableForApi ( + options: BuildVideosListQueryOptions, + countVideos = true + ): Promise> { + const span = tracer.startSpan('peertube.VideoModel.getAvailableForApi') + + function getCount () { + if (countVideos !== true) return Promise.resolve(undefined) + + const countOptions = Object.assign({}, options, { isCount: true }) + const queryBuilder = new VideosIdListQueryBuilder(VideoModel.sequelize) + + return queryBuilder.countVideoIds(countOptions) + } + + function getModels () { + if (options.count === 0) return Promise.resolve([]) + + const queryBuilder = new VideosModelListQueryBuilder(VideoModel.sequelize) + + return queryBuilder.queryVideos(options) + } + + const [ count, rows ] = await Promise.all([ getCount(), getModels() ]) + + span.end() + + return { + data: rows, + total: count + } + } + + private static throwIfPrivateIncludeWithoutUser (include: VideoIncludeType, user: MUserAccountId) { + if (VideoModel.isPrivateInclude(include) && !user?.hasRight(UserRight.SEE_ALL_VIDEOS)) { + throw new Error('Try to include protected videos but user cannot see all videos') + } + } + + private static throwIfPrivacyOneOfWithoutUser (privacyOneOf: VideoPrivacyType[], user: MUserAccountId) { + if (privacyOneOf && !user?.hasRight(UserRight.SEE_ALL_VIDEOS)) { + throw new Error('Try to choose video privacies but user cannot see all videos') + } + } + + private static isPrivateInclude (include: VideoIncludeType) { + return include & VideoInclude.BLACKLISTED || + include & VideoInclude.BLOCKED_OWNER || + include & VideoInclude.NOT_PUBLISHED_STATE + } + + isBlacklisted () { + return !!this.VideoBlacklist + } + + isBlocked () { + return this.VideoChannel.Account.Actor.Server?.isBlocked() || this.VideoChannel.Account.isBlocked() + } + + getQualityFileBy (this: T, fun: (files: MVideoFile[], it: (file: MVideoFile) => number) => MVideoFile) { + const files = this.getAllFiles() + const file = fun(files, file => file.resolution) + if (!file) return undefined + + if (file.videoId) { + return Object.assign(file, { Video: this }) + } + + if (file.videoStreamingPlaylistId) { + const streamingPlaylistWithVideo = Object.assign(this.VideoStreamingPlaylists[0], { Video: this }) + + return Object.assign(file, { VideoStreamingPlaylist: streamingPlaylistWithVideo }) + } + + throw new Error('File is not associated to a video of a playlist') + } + + getMaxQualityFile (this: T): MVideoFileVideo | MVideoFileStreamingPlaylistVideo { + return this.getQualityFileBy(maxBy) + } + + getMinQualityFile (this: T): MVideoFileVideo | MVideoFileStreamingPlaylistVideo { + return this.getQualityFileBy(minBy) + } + + getWebVideoFile (this: T, resolution: number): MVideoFileVideo { + if (Array.isArray(this.VideoFiles) === false) return undefined + + const file = this.VideoFiles.find(f => f.resolution === resolution) + if (!file) return undefined + + return Object.assign(file, { Video: this }) + } + + hasWebVideoFiles () { + return Array.isArray(this.VideoFiles) === true && this.VideoFiles.length !== 0 + } + + async addAndSaveThumbnail (thumbnail: MThumbnail, transaction?: Transaction) { + thumbnail.videoId = this.id + + const savedThumbnail = await thumbnail.save({ transaction }) + + if (Array.isArray(this.Thumbnails) === false) this.Thumbnails = [] + + this.Thumbnails = this.Thumbnails.filter(t => t.id !== savedThumbnail.id) + this.Thumbnails.push(savedThumbnail) + } + + getMiniature () { + if (Array.isArray(this.Thumbnails) === false) return undefined + + return this.Thumbnails.find(t => t.type === ThumbnailType.MINIATURE) + } + + hasPreview () { + return !!this.getPreview() + } + + getPreview () { + if (Array.isArray(this.Thumbnails) === false) return undefined + + return this.Thumbnails.find(t => t.type === ThumbnailType.PREVIEW) + } + + isOwned () { + return this.remote === false + } + + getWatchStaticPath () { + return buildVideoWatchPath({ shortUUID: uuidToShort(this.uuid) }) + } + + getEmbedStaticPath () { + return buildVideoEmbedPath(this) + } + + getMiniatureStaticPath () { + const thumbnail = this.getMiniature() + if (!thumbnail) return null + + return thumbnail.getLocalStaticPath() + } + + getPreviewStaticPath () { + const preview = this.getPreview() + if (!preview) return null + + return preview.getLocalStaticPath() + } + + toFormattedJSON (this: MVideoFormattable, options?: VideoFormattingJSONOptions): Video { + return videoModelToFormattedJSON(this, options) + } + + toFormattedDetailsJSON (this: MVideoFormattableDetails): VideoDetails { + return videoModelToFormattedDetailsJSON(this) + } + + getFormattedWebVideoFilesJSON (includeMagnet = true): VideoFile[] { + return videoFilesModelToFormattedJSON(this, this.VideoFiles, { includeMagnet }) + } + + getFormattedHLSVideoFilesJSON (includeMagnet = true): VideoFile[] { + let acc: VideoFile[] = [] + + for (const p of this.VideoStreamingPlaylists) { + acc = acc.concat(videoFilesModelToFormattedJSON(this, p.VideoFiles, { includeMagnet })) + } + + return acc + } + + getFormattedAllVideoFilesJSON (includeMagnet = true): VideoFile[] { + let files: VideoFile[] = [] + + if (Array.isArray(this.VideoFiles)) { + files = files.concat(this.getFormattedWebVideoFilesJSON(includeMagnet)) + } + + if (Array.isArray(this.VideoStreamingPlaylists)) { + files = files.concat(this.getFormattedHLSVideoFilesJSON(includeMagnet)) + } + + return files + } + + toActivityPubObject (this: MVideoAP): Promise { + return Hooks.wrapObject( + videoModelToActivityPubObject(this), + 'filter:activity-pub.video.json-ld.build.result', + { video: this } + ) + } + + async lightAPToFullAP (this: MVideoAPLight, transaction: Transaction): Promise { + const videoAP = this as MVideoAP + + const getCaptions = () => { + if (isArray(videoAP.VideoCaptions)) return videoAP.VideoCaptions + + return this.$get('VideoCaptions', { + attributes: [ 'filename', 'language', 'fileUrl' ], + transaction + }) as Promise + } + + const getStoryboard = () => { + if (videoAP.Storyboard) return videoAP.Storyboard + + return this.$get('Storyboard', { transaction }) as Promise + } + + const [ captions, storyboard ] = await Promise.all([ getCaptions(), getStoryboard() ]) + + return Object.assign(this, { + VideoCaptions: captions, + Storyboard: storyboard + }) + } + + getTruncatedDescription () { + if (!this.description) return null + + const maxLength = CONSTRAINTS_FIELDS.VIDEOS.TRUNCATED_DESCRIPTION.max + return peertubeTruncate(this.description, { length: maxLength }) + } + + getAllFiles () { + let files: MVideoFile[] = [] + + if (Array.isArray(this.VideoFiles)) { + files = files.concat(this.VideoFiles) + } + + if (Array.isArray(this.VideoStreamingPlaylists)) { + for (const p of this.VideoStreamingPlaylists) { + if (Array.isArray(p.VideoFiles)) { + files = files.concat(p.VideoFiles) + } + } + } + + return files + } + + probeMaxQualityFile () { + const file = this.getMaxQualityFile() + const videoOrPlaylist = file.getVideoOrStreamingPlaylist() + + return VideoPathManager.Instance.makeAvailableVideoFile(file.withVideoOrPlaylist(videoOrPlaylist), async originalFilePath => { + const probe = await ffprobePromise(originalFilePath) + + const { audioStream } = await getAudioStream(originalFilePath, probe) + const hasAudio = await hasAudioStream(originalFilePath, probe) + const fps = await getVideoStreamFPS(originalFilePath, probe) + + return { + audioStream, + hasAudio, + fps, + + ...await getVideoStreamDimensionsInfo(originalFilePath, probe) + } + }) + } + + getDescriptionAPIPath () { + return `/api/${API_VERSION}/videos/${this.uuid}/description` + } + + getHLSPlaylist (): MStreamingPlaylistFilesVideo { + if (!this.VideoStreamingPlaylists) return undefined + + const playlist = this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) + if (!playlist) return undefined + + return playlist.withVideo(this) + } + + setHLSPlaylist (playlist: MStreamingPlaylist) { + const toAdd = [ playlist ] as [ VideoStreamingPlaylistModel ] + + if (Array.isArray(this.VideoStreamingPlaylists) === false || this.VideoStreamingPlaylists.length === 0) { + this.VideoStreamingPlaylists = toAdd + return + } + + this.VideoStreamingPlaylists = this.VideoStreamingPlaylists + .filter(s => s.type !== VideoStreamingPlaylistType.HLS) + .concat(toAdd) + } + + removeWebVideoFile (videoFile: MVideoFile, isRedundancy = false) { + const filePath = isRedundancy + ? VideoPathManager.Instance.getFSRedundancyVideoFilePath(this, videoFile) + : VideoPathManager.Instance.getFSVideoFileOutputPath(this, videoFile) + + const promises: Promise[] = [ remove(filePath) ] + if (!isRedundancy) promises.push(videoFile.removeTorrent()) + + if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { + promises.push(removeWebVideoObjectStorage(videoFile)) + } + + return Promise.all(promises) + } + + async removeStreamingPlaylistFiles (streamingPlaylist: MStreamingPlaylist, isRedundancy = false) { + const directoryPath = isRedundancy + ? getHLSRedundancyDirectory(this) + : getHLSDirectory(this) + + await remove(directoryPath) + + if (isRedundancy !== true) { + const streamingPlaylistWithFiles = streamingPlaylist as MStreamingPlaylistFilesVideo + streamingPlaylistWithFiles.Video = this + + if (!Array.isArray(streamingPlaylistWithFiles.VideoFiles)) { + streamingPlaylistWithFiles.VideoFiles = await streamingPlaylistWithFiles.$get('VideoFiles') + } + + // Remove physical files and torrents + await Promise.all( + streamingPlaylistWithFiles.VideoFiles.map(file => file.removeTorrent()) + ) + + if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { + await removeHLSObjectStorage(streamingPlaylist.withVideo(this)) + } + } + } + + async removeStreamingPlaylistVideoFile (streamingPlaylist: MStreamingPlaylist, videoFile: MVideoFile) { + const filePath = VideoPathManager.Instance.getFSHLSOutputPath(this, videoFile.filename) + await videoFile.removeTorrent() + await remove(filePath) + + const resolutionFilename = getHlsResolutionPlaylistFilename(videoFile.filename) + await remove(VideoPathManager.Instance.getFSHLSOutputPath(this, resolutionFilename)) + + if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { + await removeHLSFileObjectStorageByFilename(streamingPlaylist.withVideo(this), videoFile.filename) + await removeHLSFileObjectStorageByFilename(streamingPlaylist.withVideo(this), resolutionFilename) + } + } + + async removeStreamingPlaylistFile (streamingPlaylist: MStreamingPlaylist, filename: string) { + const filePath = VideoPathManager.Instance.getFSHLSOutputPath(this, filename) + await remove(filePath) + + if (streamingPlaylist.storage === VideoStorage.OBJECT_STORAGE) { + await removeHLSFileObjectStorageByFilename(streamingPlaylist.withVideo(this), filename) + } + } + + isOutdated () { + if (this.isOwned()) return false + + return isOutdated(this, ACTIVITY_PUB.VIDEO_REFRESH_INTERVAL) + } + + hasPrivacyForFederation () { + return isPrivacyForFederation(this.privacy) + } + + hasStateForFederation () { + return isStateForFederation(this.state) + } + + isNewVideo (newPrivacy: VideoPrivacyType) { + return this.hasPrivacyForFederation() === false && isPrivacyForFederation(newPrivacy) === true + } + + setAsRefreshed (transaction?: Transaction) { + return setAsUpdated({ sequelize: this.sequelize, table: 'video', id: this.id, transaction }) + } + + // --------------------------------------------------------------------------- + + requiresUserAuth (options: { + urlParamId: string + checkBlacklist: boolean + }) { + const { urlParamId, checkBlacklist } = options + + if (this.privacy === VideoPrivacy.PRIVATE || this.privacy === VideoPrivacy.INTERNAL) { + return true + } + + if (this.privacy === VideoPrivacy.UNLISTED) { + if (urlParamId && !isUUIDValid(urlParamId)) return true + + return false + } + + if (checkBlacklist && this.VideoBlacklist) return true + + if (this.privacy === VideoPrivacy.PUBLIC || this.privacy === VideoPrivacy.PASSWORD_PROTECTED) { + return false + } + + throw new Error(`Unknown video privacy ${this.privacy} to know if the video requires auth`) + } + + hasPrivateStaticPath () { + return isVideoInPrivateDirectory(this.privacy) + } + + // --------------------------------------------------------------------------- + + async setNewState (newState: VideoStateType, isNewVideo: boolean, transaction: Transaction) { + if (this.state === newState) throw new Error('Cannot use same state ' + newState) + + this.state = newState + + if (this.state === VideoState.PUBLISHED && isNewVideo) { + this.publishedAt = new Date() + } + + await this.save({ transaction }) + } + + getBandwidthBits (this: MVideo, videoFile: MVideoFile) { + if (!this.duration) return videoFile.size + + return Math.ceil((videoFile.size * 8) / this.duration) + } + + getTrackerUrls () { + if (this.isOwned()) { + return [ + WEBSERVER.URL + '/tracker/announce', + WEBSERVER.WS + '://' + WEBSERVER.HOSTNAME + ':' + WEBSERVER.PORT + '/tracker/socket' + ] + } + + return this.Trackers.map(t => t.url) + } +} diff --git a/server/server/models/view/local-video-viewer-watch-section.ts b/server/server/models/view/local-video-viewer-watch-section.ts new file mode 100644 index 000000000..b04dcf4bd --- /dev/null +++ b/server/server/models/view/local-video-viewer-watch-section.ts @@ -0,0 +1,63 @@ +import { Transaction } from 'sequelize' +import { AllowNull, BelongsTo, Column, CreatedAt, ForeignKey, Model, Table } from 'sequelize-typescript' +import { MLocalVideoViewerWatchSection } from '@server/types/models/index.js' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { LocalVideoViewerModel } from './local-video-viewer.js' + +@Table({ + tableName: 'localVideoViewerWatchSection', + updatedAt: false, + indexes: [ + { + fields: [ 'localVideoViewerId' ] + } + ] +}) +export class LocalVideoViewerWatchSectionModel extends Model>> { + @CreatedAt + createdAt: Date + + @AllowNull(false) + @Column + watchStart: number + + @AllowNull(false) + @Column + watchEnd: number + + @ForeignKey(() => LocalVideoViewerModel) + @Column + localVideoViewerId: number + + @BelongsTo(() => LocalVideoViewerModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + LocalVideoViewer: Awaited + + static async bulkCreateSections (options: { + localVideoViewerId: number + watchSections: { + start: number + end: number + }[] + transaction?: Transaction + }) { + const { localVideoViewerId, watchSections, transaction } = options + const models: MLocalVideoViewerWatchSection[] = [] + + for (const section of watchSections) { + const model = await this.create({ + watchStart: section.start, + watchEnd: section.end, + localVideoViewerId + }, { transaction }) + + models.push(model) + } + + return models + } +} diff --git a/server/server/models/view/local-video-viewer.ts b/server/server/models/view/local-video-viewer.ts new file mode 100644 index 000000000..b3d1ada79 --- /dev/null +++ b/server/server/models/view/local-video-viewer.ts @@ -0,0 +1,374 @@ +import { QueryTypes } from 'sequelize' +import { AllowNull, BelongsTo, Column, CreatedAt, DataType, Default, ForeignKey, HasMany, IsUUID, Model, Table } from 'sequelize-typescript' +import { getActivityStreamDuration } from '@server/lib/activitypub/activity.js' +import { buildGroupByAndBoundaries } from '@server/lib/timeserie.js' +import { MLocalVideoViewer, MLocalVideoViewerWithWatchSections, MVideo } from '@server/types/models/index.js' +import { + VideoStatsOverall, + VideoStatsRetention, + VideoStatsTimeserie, + VideoStatsTimeserieMetric, + WatchActionObject +} from '@peertube/peertube-models' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { VideoModel } from '../video/video.js' +import { LocalVideoViewerWatchSectionModel } from './local-video-viewer-watch-section.js' + +/** + * + * Aggregate viewers of local videos only to display statistics to video owners + * A viewer is a user that watched one or multiple sections of a specific video inside a time window + * + */ + +@Table({ + tableName: 'localVideoViewer', + updatedAt: false, + indexes: [ + { + fields: [ 'videoId' ] + }, + { + fields: [ 'url' ], + unique: true + } + ] +}) +export class LocalVideoViewerModel extends Model>> { + @CreatedAt + createdAt: Date + + @AllowNull(false) + @Column(DataType.DATE) + startDate: Date + + @AllowNull(false) + @Column(DataType.DATE) + endDate: Date + + @AllowNull(false) + @Column + watchTime: number + + @AllowNull(true) + @Column + country: string + + @AllowNull(false) + @Default(DataType.UUIDV4) + @IsUUID(4) + @Column(DataType.UUID) + uuid: string + + @AllowNull(false) + @Column + url: string + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + Video: Awaited + + @HasMany(() => LocalVideoViewerWatchSectionModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'cascade' + }) + WatchSections: Awaited[] + + static loadByUrl (url: string): Promise { + return this.findOne({ + where: { + url + } + }) + } + + static loadFullById (id: number): Promise { + return this.findOne({ + include: [ + { + model: VideoModel.unscoped(), + required: true + }, + { + model: LocalVideoViewerWatchSectionModel.unscoped(), + required: true + } + ], + where: { + id + } + }) + } + + static async getOverallStats (options: { + video: MVideo + startDate?: string + endDate?: string + }): Promise { + const { video, startDate, endDate } = options + + const queryOptions = { + type: QueryTypes.SELECT as QueryTypes.SELECT, + replacements: { videoId: video.id } as any + } + + if (startDate) queryOptions.replacements.startDate = startDate + if (endDate) queryOptions.replacements.endDate = endDate + + const buildTotalViewersPromise = () => { + let totalViewersDateWhere = '' + + if (startDate) totalViewersDateWhere += ' AND "localVideoViewer"."endDate" >= :startDate' + if (endDate) totalViewersDateWhere += ' AND "localVideoViewer"."startDate" <= :endDate' + + const totalViewersQuery = `SELECT ` + + `COUNT("localVideoViewer"."id") AS "totalViewers" ` + + `FROM "localVideoViewer" ` + + `WHERE "videoId" = :videoId ${totalViewersDateWhere}` + + return LocalVideoViewerModel.sequelize.query(totalViewersQuery, queryOptions) + } + + const buildWatchTimePromise = () => { + let watchTimeDateWhere = '' + + // We know this where is not exact + // But we prefer to take into account only watch section that started and ended **in** the interval + if (startDate) watchTimeDateWhere += ' AND "localVideoViewer"."startDate" >= :startDate' + if (endDate) watchTimeDateWhere += ' AND "localVideoViewer"."endDate" <= :endDate' + + const watchTimeQuery = `SELECT ` + + `SUM("localVideoViewer"."watchTime") AS "totalWatchTime", ` + + `AVG("localVideoViewer"."watchTime") AS "averageWatchTime" ` + + `FROM "localVideoViewer" ` + + `WHERE "videoId" = :videoId ${watchTimeDateWhere}` + + return LocalVideoViewerModel.sequelize.query(watchTimeQuery, queryOptions) + } + + const buildWatchPeakPromise = () => { + let watchPeakDateWhereStart = '' + let watchPeakDateWhereEnd = '' + + if (startDate) { + watchPeakDateWhereStart += ' AND "localVideoViewer"."startDate" >= :startDate' + watchPeakDateWhereEnd += ' AND "localVideoViewer"."endDate" >= :startDate' + } + + if (endDate) { + watchPeakDateWhereStart += ' AND "localVideoViewer"."startDate" <= :endDate' + watchPeakDateWhereEnd += ' AND "localVideoViewer"."endDate" <= :endDate' + } + + // Add viewers that were already here, before our start date + const beforeWatchersQuery = startDate + // eslint-disable-next-line max-len + ? `SELECT COUNT(*) AS "total" FROM "localVideoViewer" WHERE "localVideoViewer"."startDate" < :startDate AND "localVideoViewer"."endDate" >= :startDate` + : `SELECT 0 AS "total"` + + const watchPeakQuery = `WITH + "beforeWatchers" AS (${beforeWatchersQuery}), + "watchPeakValues" AS ( + SELECT "startDate" AS "dateBreakpoint", 1 AS "inc" + FROM "localVideoViewer" + WHERE "videoId" = :videoId ${watchPeakDateWhereStart} + UNION ALL + SELECT "endDate" AS "dateBreakpoint", -1 AS "inc" + FROM "localVideoViewer" + WHERE "videoId" = :videoId ${watchPeakDateWhereEnd} + ) + SELECT "dateBreakpoint", "concurrent" + FROM ( + SELECT "dateBreakpoint", SUM(SUM("inc")) OVER (ORDER BY "dateBreakpoint") + (SELECT "total" FROM "beforeWatchers") AS "concurrent" + FROM "watchPeakValues" + GROUP BY "dateBreakpoint" + ) tmp + ORDER BY "concurrent" DESC + FETCH FIRST 1 ROW ONLY` + + return LocalVideoViewerModel.sequelize.query(watchPeakQuery, queryOptions) + } + + const buildCountriesPromise = () => { + let countryDateWhere = '' + + if (startDate) countryDateWhere += ' AND "localVideoViewer"."endDate" >= :startDate' + if (endDate) countryDateWhere += ' AND "localVideoViewer"."startDate" <= :endDate' + + const countriesQuery = `SELECT country, COUNT(country) as viewers ` + + `FROM "localVideoViewer" ` + + `WHERE "videoId" = :videoId AND country IS NOT NULL ${countryDateWhere} ` + + `GROUP BY country ` + + `ORDER BY viewers DESC` + + return LocalVideoViewerModel.sequelize.query(countriesQuery, queryOptions) + } + + const [ rowsTotalViewers, rowsWatchTime, rowsWatchPeak, rowsCountries ] = await Promise.all([ + buildTotalViewersPromise(), + buildWatchTimePromise(), + buildWatchPeakPromise(), + buildCountriesPromise() + ]) + + const viewersPeak = rowsWatchPeak.length !== 0 + ? parseInt(rowsWatchPeak[0].concurrent) || 0 + : 0 + + return { + totalWatchTime: rowsWatchTime.length !== 0 + ? Math.round(rowsWatchTime[0].totalWatchTime) || 0 + : 0, + averageWatchTime: rowsWatchTime.length !== 0 + ? Math.round(rowsWatchTime[0].averageWatchTime) || 0 + : 0, + + totalViewers: rowsTotalViewers.length !== 0 + ? Math.round(rowsTotalViewers[0].totalViewers) || 0 + : 0, + + viewersPeak, + viewersPeakDate: rowsWatchPeak.length !== 0 && viewersPeak !== 0 + ? rowsWatchPeak[0].dateBreakpoint || null + : null, + + countries: rowsCountries.map(r => ({ + isoCode: r.country, + viewers: r.viewers + })) + } + } + + static async getRetentionStats (video: MVideo): Promise { + const step = Math.max(Math.round(video.duration / 100), 1) + + const query = `WITH "total" AS (SELECT COUNT(*) AS viewers FROM "localVideoViewer" WHERE "videoId" = :videoId) ` + + `SELECT serie AS "second", ` + + `(COUNT("localVideoViewer".id)::float / (SELECT GREATEST("total"."viewers", 1) FROM "total")) AS "retention" ` + + `FROM generate_series(0, ${video.duration}, ${step}) serie ` + + `LEFT JOIN "localVideoViewer" ON "localVideoViewer"."videoId" = :videoId ` + + `AND EXISTS (` + + `SELECT 1 FROM "localVideoViewerWatchSection" ` + + `WHERE "localVideoViewer"."id" = "localVideoViewerWatchSection"."localVideoViewerId" ` + + `AND serie >= "localVideoViewerWatchSection"."watchStart" ` + + `AND serie <= "localVideoViewerWatchSection"."watchEnd"` + + `)` + + `GROUP BY serie ` + + `ORDER BY serie ASC` + + const queryOptions = { + type: QueryTypes.SELECT as QueryTypes.SELECT, + replacements: { videoId: video.id } + } + + const rows = await LocalVideoViewerModel.sequelize.query(query, queryOptions) + + return { + data: rows.map(r => ({ + second: r.second, + retentionPercent: parseFloat(r.retention) * 100 + })) + } + } + + static async getTimeserieStats (options: { + video: MVideo + metric: VideoStatsTimeserieMetric + startDate: string + endDate: string + }): Promise { + const { video, metric } = options + + const { groupInterval, startDate, endDate } = buildGroupByAndBoundaries(options.startDate, options.endDate) + + const selectMetrics: { [ id in VideoStatsTimeserieMetric ]: string } = { + viewers: 'COUNT("localVideoViewer"."id")', + aggregateWatchTime: 'SUM("localVideoViewer"."watchTime")' + } + + const intervalWhere: { [ id in VideoStatsTimeserieMetric ]: string } = { + // Viewer is still in the interval. Overlap algorithm + viewers: '"localVideoViewer"."startDate" <= "intervals"."endDate" ' + + 'AND "localVideoViewer"."endDate" >= "intervals"."startDate"', + + // We do an aggregation, so only sum things once. Arbitrary we use the end date for that purpose + aggregateWatchTime: '"localVideoViewer"."endDate" >= "intervals"."startDate" ' + + 'AND "localVideoViewer"."endDate" <= "intervals"."endDate"' + } + + const query = `WITH "intervals" AS ( + SELECT + "time" AS "startDate", "time" + :groupInterval::interval as "endDate" + FROM + generate_series(:startDate::timestamptz, :endDate::timestamptz, :groupInterval::interval) serie("time") + ) + SELECT "intervals"."startDate" as "date", COALESCE(${selectMetrics[metric]}, 0) AS value + FROM + intervals + LEFT JOIN "localVideoViewer" ON "localVideoViewer"."videoId" = :videoId + AND ${intervalWhere[metric]} + GROUP BY + "intervals"."startDate" + ORDER BY + "intervals"."startDate"` + + const queryOptions = { + type: QueryTypes.SELECT as QueryTypes.SELECT, + replacements: { + startDate, + endDate, + groupInterval, + videoId: video.id + } + } + + const rows = await LocalVideoViewerModel.sequelize.query(query, queryOptions) + + return { + groupInterval, + data: rows.map(r => ({ + date: r.date, + value: parseInt(r.value) + })) + } + } + + toActivityPubObject (this: MLocalVideoViewerWithWatchSections): WatchActionObject { + const location = this.country + ? { + location: { + addressCountry: this.country + } + } + : {} + + return { + id: this.url, + type: 'WatchAction', + duration: getActivityStreamDuration(this.watchTime), + startTime: this.startDate.toISOString(), + endTime: this.endDate.toISOString(), + + object: this.Video.url, + uuid: this.uuid, + actionStatus: 'CompletedActionStatus', + + watchSections: this.WatchSections.map(w => ({ + startTimestamp: w.watchStart, + endTimestamp: w.watchEnd + })), + + ...location + } + } +} diff --git a/server/server/models/view/video-view.ts b/server/server/models/view/video-view.ts new file mode 100644 index 000000000..a7aaccf26 --- /dev/null +++ b/server/server/models/view/video-view.ts @@ -0,0 +1,67 @@ +import { literal, Op } from 'sequelize' +import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Model, Table } from 'sequelize-typescript' +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { VideoModel } from '../video/video.js' + +/** + * + * Aggregate views of all videos federated with our instance + * Mainly used by the trending/hot algorithms + * + */ + +@Table({ + tableName: 'videoView', + updatedAt: false, + indexes: [ + { + fields: [ 'videoId' ] + }, + { + fields: [ 'startDate' ] + } + ] +}) +export class VideoViewModel extends Model>> { + @CreatedAt + createdAt: Date + + @AllowNull(false) + @Column(DataType.DATE) + startDate: Date + + @AllowNull(false) + @Column(DataType.DATE) + endDate: Date + + @AllowNull(false) + @Column + views: number + + @ForeignKey(() => VideoModel) + @Column + videoId: number + + @BelongsTo(() => VideoModel, { + foreignKey: { + allowNull: false + }, + onDelete: 'CASCADE' + }) + Video: Awaited + + static removeOldRemoteViewsHistory (beforeDate: string) { + const query = { + where: { + startDate: { + [Op.lt]: beforeDate + }, + videoId: { + [Op.in]: literal('(SELECT "id" FROM "video" WHERE "remote" IS TRUE)') + } + } + } + + return VideoViewModel.destroy(query) + } +} diff --git a/server/static/dnt-policy/dnt-policy-1.0.txt b/server/server/static/dnt-policy/dnt-policy-1.0.txt similarity index 100% rename from server/static/dnt-policy/dnt-policy-1.0.txt rename to server/server/static/dnt-policy/dnt-policy-1.0.txt diff --git a/server/server/types/activitypub-processor.model.ts b/server/server/types/activitypub-processor.model.ts new file mode 100644 index 000000000..0cb429770 --- /dev/null +++ b/server/server/types/activitypub-processor.model.ts @@ -0,0 +1,9 @@ +import { Activity } from '@peertube/peertube-models' +import { MActorDefault, MActorSignature } from './models/index.js' + +export type APProcessorOptions = { + activity: T + byActor: MActorSignature + inboxActor?: MActorDefault + fromFetch?: boolean +} diff --git a/server/types/express-handler.ts b/server/server/types/express-handler.ts similarity index 100% rename from server/types/express-handler.ts rename to server/server/types/express-handler.ts diff --git a/server/server/types/express.d.ts b/server/server/types/express.d.ts new file mode 100644 index 000000000..3ead23b40 --- /dev/null +++ b/server/server/types/express.d.ts @@ -0,0 +1,222 @@ +import { OutgoingHttpHeaders } from 'http' +import { Writable } from 'stream' +import { HttpMethodType, PeerTubeProblemDocumentData, ServerErrorCode, VideoCreate } from '@peertube/peertube-models' +import { RegisterServerAuthExternalOptions } from '@server/types/index.js' +import { + MAbuseMessage, + MAbuseReporter, + MAccountBlocklist, + MActorFollowActorsDefault, + MActorUrl, + MChannelBannerAccountDefault, + MChannelSyncChannel, + MRegistration, + MStreamingPlaylist, + MUserAccountUrl, + MVideoChangeOwnershipFull, + MVideoFile, + MVideoFormattableDetails, + MVideoId, + MVideoImmutable, + MVideoLiveFormattable, + MVideoPassword, + MVideoPlaylistFull, + MVideoPlaylistFullSummary +} from '@server/types/models/index.js' +import { MOAuthTokenUser } from '@server/types/models/oauth/oauth-token.js' +import { MPlugin, MServer, MServerBlocklist } from '@server/types/models/server.js' +import { MVideoImportDefault } from '@server/types/models/video/video-import.js' +import { MVideoPlaylistElement, MVideoPlaylistElementVideoUrlPlaylistPrivacy } from '@server/types/models/video/video-playlist-element.js' +import { MAccountVideoRateAccountVideo } from '@server/types/models/video/video-rate.js' +import { File as UploadXFile, Metadata } from '@uploadx/core' +import { RegisteredPlugin } from '../../lib/plugins/plugin-manager.js' +import { + MAccountDefault, + MActorAccountChannelId, + MActorFollowActorsDefaultSubscription, + MActorFull, + MComment, + MCommentOwnerVideoReply, + MUserDefault, + MVideoBlacklist, + MVideoCaptionVideo, + MVideoFullLight, + MVideoRedundancyVideo, + MVideoShareActor, + MVideoThumbnail +} from './models/index.js' +import { MRunner, MRunnerJobRunner, MRunnerRegistrationToken } from './models/runners/index.js' +import { MVideoSource } from './models/video/video-source.js' + +declare module 'express' { + export interface Request { + query: any + method: HttpMethodType + } + + // Upload using multer or uploadx middleware + export type MulterOrUploadXFile = UploadXFile | Express.Multer.File + + export type UploadFiles = { + [fieldname: string]: MulterOrUploadXFile[] + } | MulterOrUploadXFile[] + + // Partial object used by some functions to check the file mimetype/extension + export type UploadFileForCheck = { + originalname: string + mimetype: string + size: number + } + + export type UploadFilesForCheck = { + [fieldname: string]: UploadFileForCheck[] + } | UploadFileForCheck[] + + // Upload file with a duration added by our middleware + export type VideoUploadFile = Pick & { + duration: number + } + + // Extends Metadata property of UploadX object + export type UploadXFileMetadata = Metadata & VideoCreate & { + previewfile: Express.Multer.File[] + thumbnailfile: Express.Multer.File[] + } + + // Our custom UploadXFile object using our custom metadata + export type CustomUploadXFile = UploadXFile & { metadata: T } + + export type EnhancedUploadXFile = CustomUploadXFile & { + duration: number + path: string + filename: string + originalname: string + } + + export type UploadNewVideoUploadXFile = EnhancedUploadXFile & CustomUploadXFile + + // Extends Response with added functions and potential variables passed by middlewares + interface Response { + fail: (options: { + message: string + + title?: string + status?: number + type?: ServerErrorCode | string + instance?: string + + data?: PeerTubeProblemDocumentData + + tags?: string[] + }) => void + + locals: { + requestStart: number + + apicacheGroups: string[] + + apicache: { + content: string | Buffer + write: Writable['write'] + writeHead: Response['writeHead'] + end: Response['end'] + cacheable: boolean + headers: OutgoingHttpHeaders + } + + docUrl?: string + + videoAPI?: MVideoFormattableDetails + videoAll?: MVideoFullLight + onlyImmutableVideo?: MVideoImmutable + onlyVideo?: MVideoThumbnail + videoId?: MVideoId + + videoLive?: MVideoLiveFormattable + videoLiveSession?: MVideoLiveSession + + videoShare?: MVideoShareActor + + videoSource?: MVideoSource + + videoFile?: MVideoFile + + uploadVideoFileResumable?: UploadNewVideoUploadXFile + updateVideoFileResumable?: EnhancedUploadXFile + + videoImport?: MVideoImportDefault + + videoBlacklist?: MVideoBlacklist + + videoCaption?: MVideoCaptionVideo + + abuse?: MAbuseReporter + abuseMessage?: MAbuseMessage + + videoStreamingPlaylist?: MStreamingPlaylist + + videoChannel?: MChannelBannerAccountDefault + videoChannelSync?: MChannelSyncChannel + + videoPlaylistFull?: MVideoPlaylistFull + videoPlaylistSummary?: MVideoPlaylistFullSummary + + videoPlaylistElement?: MVideoPlaylistElement + videoPlaylistElementAP?: MVideoPlaylistElementVideoUrlPlaylistPrivacy + + accountVideoRate?: MAccountVideoRateAccountVideo + + videoCommentFull?: MCommentOwnerVideoReply + videoCommentThread?: MComment + + videoPassword?: MVideoPassword + + follow?: MActorFollowActorsDefault + subscription?: MActorFollowActorsDefaultSubscription + + nextOwner?: MAccountDefault + videoChangeOwnership?: MVideoChangeOwnershipFull + + account?: MAccountDefault + + actorUrl?: MActorUrl + actorFull?: MActorFull + + user?: MUserDefault + userRegistration?: MRegistration + + server?: MServer + + videoRedundancy?: MVideoRedundancyVideo + + accountBlock?: MAccountBlocklist + serverBlock?: MServerBlocklist + + oauth?: { + token: MOAuthTokenUser + } + + signature?: { + actor: MActorAccountChannelId + } + + videoFileToken?: { + user: MUserAccountUrl + } + + authenticated?: boolean + + registeredPlugin?: RegisteredPlugin + + externalAuth?: RegisterServerAuthExternalOptions + + plugin?: MPlugin + + localViewerFull?: MLocalVideoViewerWithWatchSections + + runner?: MRunner + runnerRegistrationToken?: MRunnerRegistrationToken + runnerJob?: MRunnerJobRunner + } + } +} diff --git a/server/server/types/index.ts b/server/server/types/index.ts new file mode 100644 index 000000000..4afef90a8 --- /dev/null +++ b/server/server/types/index.ts @@ -0,0 +1,3 @@ +export * from './plugins/index.js' +export * from './activitypub-processor.model.js' +export * from './sequelize.js' diff --git a/server/types/lib.d.ts b/server/server/types/lib.d.ts similarity index 100% rename from server/types/lib.d.ts rename to server/server/types/lib.d.ts diff --git a/server/server/types/models/abuse/abuse-message.ts b/server/server/types/models/abuse/abuse-message.ts new file mode 100644 index 000000000..85ea028d2 --- /dev/null +++ b/server/server/types/models/abuse/abuse-message.ts @@ -0,0 +1,20 @@ +import { AbuseMessageModel } from '@server/models/abuse/abuse-message.js' +import { PickWith } from '@peertube/peertube-typescript-utils' +import { AbuseModel } from '../../../models/abuse/abuse.js' +import { MAccountFormattable } from '../account/index.js' + +type Use = PickWith + +// ############################################################################ + +export type MAbuseMessage = Omit + +export type MAbuseMessageId = Pick + +// ############################################################################ + +// Format for API + +export type MAbuseMessageFormattable = + MAbuseMessage & + Use<'Account', MAccountFormattable> diff --git a/server/server/types/models/abuse/abuse.ts b/server/server/types/models/abuse/abuse.ts new file mode 100644 index 000000000..bf6680470 --- /dev/null +++ b/server/server/types/models/abuse/abuse.ts @@ -0,0 +1,114 @@ +import { VideoAbuseModel } from '@server/models/abuse/video-abuse.js' +import { VideoCommentAbuseModel } from '@server/models/abuse/video-comment-abuse.js' +import { VideoCommentModel } from '@server/models/video/video-comment.js' +import { PickWith } from '@peertube/peertube-typescript-utils' +import { AbuseModel } from '../../../models/abuse/abuse.js' +import { MAccountDefault, MAccountFormattable, MAccountLight, MAccountUrl } from '../account/index.js' +import { MComment, MCommentOwner, MCommentUrl, MCommentVideo, MVideoUrl } from '../video/index.js' +import { MVideo, MVideoAccountLightBlacklistAllFiles } from '../video/video.js' + +type Use = PickWith +type UseVideoAbuse = PickWith +type UseCommentAbuse = PickWith + +// ############################################################################ + +export type MAbuse = Omit + +export type MVideoAbuse = Omit + +export type MCommentAbuse = Omit + +export type MAbuseReporter = + MAbuse & + Use<'ReporterAccount', MAccountDefault> + +// ############################################################################ + +export type MVideoAbuseVideo = + MVideoAbuse & + UseVideoAbuse<'Video', MVideo> + +export type MVideoAbuseVideoUrl = + MVideoAbuse & + UseVideoAbuse<'Video', MVideoUrl> + +export type MVideoAbuseVideoFull = + MVideoAbuse & + UseVideoAbuse<'Video', Omit> + +export type MVideoAbuseFormattable = + MVideoAbuse & + UseVideoAbuse<'Video', Pick> + +// ############################################################################ + +export type MCommentAbuseAccount = + MCommentAbuse & + UseCommentAbuse<'VideoComment', MCommentOwner> + +export type MCommentAbuseAccountVideo = + MCommentAbuse & + UseCommentAbuse<'VideoComment', MCommentOwner & PickWith> + +export type MCommentAbuseUrl = + MCommentAbuse & + UseCommentAbuse<'VideoComment', MCommentUrl> + +export type MCommentAbuseFormattable = + MCommentAbuse & + UseCommentAbuse<'VideoComment', MComment & PickWith>> + +// ############################################################################ + +export type MAbuseId = Pick + +export type MAbuseVideo = + MAbuse & + Pick & + Use<'VideoAbuse', MVideoAbuseVideo> + +export type MAbuseUrl = + MAbuse & + Use<'VideoAbuse', MVideoAbuseVideoUrl> & + Use<'VideoCommentAbuse', MCommentAbuseUrl> + +export type MAbuseAccountVideo = + MAbuse & + Pick & + Use<'VideoAbuse', MVideoAbuseVideoFull> & + Use<'ReporterAccount', MAccountDefault> + +export type MAbuseFull = + MAbuse & + Pick & + Use<'ReporterAccount', MAccountLight> & + Use<'FlaggedAccount', MAccountLight> & + Use<'VideoAbuse', MVideoAbuseVideoFull> & + Use<'VideoCommentAbuse', MCommentAbuseAccountVideo> + +// ############################################################################ + +// Format for API or AP object + +export type MAbuseAdminFormattable = + MAbuse & + Use<'ReporterAccount', MAccountFormattable> & + Use<'FlaggedAccount', MAccountFormattable> & + Use<'VideoAbuse', MVideoAbuseFormattable> & + Use<'VideoCommentAbuse', MCommentAbuseFormattable> + +export type MAbuseUserFormattable = + MAbuse & + Use<'FlaggedAccount', MAccountFormattable> & + Use<'VideoAbuse', MVideoAbuseFormattable> & + Use<'VideoCommentAbuse', MCommentAbuseFormattable> + +export type MAbuseAP = + MAbuse & + Pick & + Use<'ReporterAccount', MAccountUrl> & + Use<'FlaggedAccount', MAccountUrl> & + Use<'VideoAbuse', MVideoAbuseVideo> & + Use<'VideoCommentAbuse', MCommentAbuseAccount> diff --git a/server/server/types/models/abuse/index.ts b/server/server/types/models/abuse/index.ts new file mode 100644 index 000000000..fe6b95af3 --- /dev/null +++ b/server/server/types/models/abuse/index.ts @@ -0,0 +1,2 @@ +export * from './abuse.js' +export * from './abuse-message.js' diff --git a/server/server/types/models/account/account-blocklist.ts b/server/server/types/models/account/account-blocklist.ts new file mode 100644 index 000000000..d761b2de4 --- /dev/null +++ b/server/server/types/models/account/account-blocklist.ts @@ -0,0 +1,27 @@ +import { AccountBlocklistModel } from '../../../models/account/account-blocklist.js' +import { PickWith } from '@peertube/peertube-typescript-utils' +import { MAccountDefault, MAccountFormattable } from './account.js' + +type Use = PickWith + +// ############################################################################ + +export type MAccountBlocklist = Omit + +// ############################################################################ + +export type MAccountBlocklistId = Pick + +export type MAccountBlocklistAccounts = + MAccountBlocklist & + Use<'ByAccount', MAccountDefault> & + Use<'BlockedAccount', MAccountDefault> + +// ############################################################################ + +// Format for API or AP object + +export type MAccountBlocklistFormattable = + Pick & + Use<'ByAccount', MAccountFormattable> & + Use<'BlockedAccount', MAccountFormattable> diff --git a/server/server/types/models/account/account.ts b/server/server/types/models/account/account.ts new file mode 100644 index 000000000..4a5e80725 --- /dev/null +++ b/server/server/types/models/account/account.ts @@ -0,0 +1,108 @@ +import { FunctionProperties, PickWith } from '@peertube/peertube-typescript-utils' +import { AccountModel } from '../../../models/account/account.js' +import { + MActor, + MActorAPAccount, + MActorAPI, + MActorAudience, + MActorDefault, + MActorDefaultLight, + MActorFormattable, + MActorHost, + MActorId, + MActorSummary, + MActorSummaryFormattable, + MActorUrl +} from '../actor/index.js' +import { MChannelDefault } from '../video/video-channels.js' +import { MAccountBlocklistId } from './account-blocklist.js' + +type Use = PickWith + +// ############################################################################ + +export type MAccount = + Omit + +// ############################################################################ + +// Only some attributes +export type MAccountId = Pick +export type MAccountUserId = Pick + +// Only some Actor attributes +export type MAccountUrl = Use<'Actor', MActorUrl> +export type MAccountAudience = Use<'Actor', MActorAudience> + +export type MAccountIdActor = + MAccountId & + Use<'Actor', MActor> + +export type MAccountIdActorId = + MAccountId & + Use<'Actor', MActorId> + +// ############################################################################ + +// Default scope +export type MAccountDefault = + MAccount & + Use<'Actor', MActorDefault> + +// Default with default association scopes +export type MAccountDefaultChannelDefault = + MAccount & + Use<'Actor', MActorDefault> & + Use<'VideoChannels', MChannelDefault[]> + +// We don't need some actors attributes +export type MAccountLight = + MAccount & + Use<'Actor', MActorDefaultLight> + +// ############################################################################ + +// Full actor +export type MAccountActor = + MAccount & + Use<'Actor', MActor> + +export type MAccountHost = + MAccount & + Use<'Actor', MActorHost> + +// ############################################################################ + +// For API + +export type MAccountSummary = + FunctionProperties & + Pick & + Use<'Actor', MActorSummary> + +export type MAccountSummaryBlocks = + MAccountSummary & + Use<'BlockedBy', MAccountBlocklistId[]> + +export type MAccountAPI = + MAccount & + Use<'Actor', MActorAPI> + +// ############################################################################ + +// Format for API or AP object + +export type MAccountSummaryFormattable = + FunctionProperties & + Pick & + Use<'Actor', MActorSummaryFormattable> + +export type MAccountFormattable = + FunctionProperties & + Pick & + Use<'Actor', MActorFormattable> + +export type MAccountAP = + Pick & + Use<'Actor', MActorAPAccount> diff --git a/server/server/types/models/account/actor-custom-page.ts b/server/server/types/models/account/actor-custom-page.ts new file mode 100644 index 000000000..de2cfd41e --- /dev/null +++ b/server/server/types/models/account/actor-custom-page.ts @@ -0,0 +1,3 @@ +import { ActorCustomPageModel } from '../../../models/account/actor-custom-page.js' + +export type MActorCustomPage = Omit diff --git a/server/server/types/models/account/index.ts b/server/server/types/models/account/index.ts new file mode 100644 index 000000000..e00026a6f --- /dev/null +++ b/server/server/types/models/account/index.ts @@ -0,0 +1,3 @@ +export * from './account.js' +export * from './actor-custom-page.js' +export * from './account-blocklist.js' diff --git a/server/server/types/models/actor/actor-follow.ts b/server/server/types/models/actor/actor-follow.ts new file mode 100644 index 000000000..a130751f9 --- /dev/null +++ b/server/server/types/models/actor/actor-follow.ts @@ -0,0 +1,65 @@ +import { PickWith } from '@peertube/peertube-typescript-utils' +import { ActorFollowModel } from '../../../models/actor/actor-follow.js' +import { + MActor, + MActorChannelAccountActor, + MActorDefault, + MActorDefaultAccountChannel, + MActorDefaultChannelId, + MActorFormattable, + MActorHostOnly, + MActorUsername +} from './actor.js' + +type Use = PickWith + +// ############################################################################ + +export type MActorFollow = Omit + +// ############################################################################ + +export type MActorFollowFollowingHost = + MActorFollow & + Use<'ActorFollowing', MActorUsername & MActorHostOnly> + +// ############################################################################ + +// With actors or actors default + +export type MActorFollowActors = + MActorFollow & + Use<'ActorFollower', MActor> & + Use<'ActorFollowing', MActor> + +export type MActorFollowActorsDefault = + MActorFollow & + Use<'ActorFollower', MActorDefault> & + Use<'ActorFollowing', MActorDefault> + +export type MActorFollowFull = + MActorFollow & + Use<'ActorFollower', MActorDefaultAccountChannel> & + Use<'ActorFollowing', MActorDefaultAccountChannel> + +// ############################################################################ + +// For subscriptions + +export type MActorFollowActorsDefaultSubscription = + MActorFollow & + Use<'ActorFollower', MActorDefault> & + Use<'ActorFollowing', MActorDefaultChannelId> + +export type MActorFollowSubscriptions = + MActorFollow & + Use<'ActorFollowing', MActorChannelAccountActor> + +// ############################################################################ + +// Format for API or AP object + +export type MActorFollowFormattable = + Pick & + Use<'ActorFollower', MActorFormattable> & + Use<'ActorFollowing', MActorFormattable> diff --git a/server/server/types/models/actor/actor-image.ts b/server/server/types/models/actor/actor-image.ts new file mode 100644 index 000000000..de8a62216 --- /dev/null +++ b/server/server/types/models/actor/actor-image.ts @@ -0,0 +1,12 @@ +import { FunctionProperties } from '@peertube/peertube-typescript-utils' +import { ActorImageModel } from '../../../models/actor/actor-image.js' + +export type MActorImage = ActorImageModel + +// ############################################################################ + +// Format for API or AP object + +export type MActorImageFormattable = + FunctionProperties & + Pick diff --git a/server/server/types/models/actor/actor.ts b/server/server/types/models/actor/actor.ts new file mode 100644 index 000000000..0d594545f --- /dev/null +++ b/server/server/types/models/actor/actor.ts @@ -0,0 +1,170 @@ +import { FunctionProperties, PickWith, PickWithOpt } from '@peertube/peertube-typescript-utils' +import { ActorModel } from '../../../models/actor/actor.js' +import { MAccount, MAccountDefault, MAccountId, MAccountIdActor } from '../account/index.js' +import { MServer, MServerHost, MServerHostBlocks, MServerRedundancyAllowed } from '../server/index.js' +import { MChannel, MChannelAccountActor, MChannelAccountDefault, MChannelId, MChannelIdActor } from '../video/index.js' +import { MActorImage, MActorImageFormattable } from './actor-image.js' + +type Use = PickWith +type UseOpt = PickWithOpt + +// ############################################################################ + +export type MActor = Omit + +// ############################################################################ + +export type MActorUrl = Pick +export type MActorId = Pick +export type MActorUsername = Pick + +export type MActorFollowersUrl = Pick +export type MActorAudience = MActorUrl & MActorFollowersUrl +export type MActorWithInboxes = Pick +export type MActorSignature = MActorAccountChannelId + +export type MActorLight = Omit + +// ############################################################################ + +// Some association attributes + +export type MActorHostOnly = Use<'Server', MServerHost> +export type MActorHost = + MActorLight & + Use<'Server', MServerHost> + +export type MActorRedundancyAllowedOpt = PickWithOpt + +export type MActorDefaultLight = + MActorLight & + Use<'Server', MServerHost> & + Use<'Avatars', MActorImage[]> + +export type MActorAccountId = + MActor & + Use<'Account', MAccountId> +export type MActorAccountIdActor = + MActor & + Use<'Account', MAccountIdActor> + +export type MActorChannelId = + MActor & + Use<'VideoChannel', MChannelId> +export type MActorChannelIdActor = + MActor & + Use<'VideoChannel', MChannelIdActor> + +export type MActorAccountChannelId = MActorAccountId & MActorChannelId +export type MActorAccountChannelIdActor = MActorAccountIdActor & MActorChannelIdActor + +// ############################################################################ + +// Include raw account/channel/server + +export type MActorAccount = + MActor & + Use<'Account', MAccount> + +export type MActorChannel = + MActor & + Use<'VideoChannel', MChannel> + +export type MActorDefaultAccountChannel = MActorDefault & MActorAccount & MActorChannel + +export type MActorServerLight = + MActorLight & + Use<'Server', MServer> + +// ############################################################################ + +// Complex actor associations + +export type MActorImages = + MActor & + Use<'Avatars', MActorImage[]> & + UseOpt<'Banners', MActorImage[]> + +export type MActorDefault = + MActor & + Use<'Server', MServer> & + Use<'Avatars', MActorImage[]> + +export type MActorDefaultChannelId = + MActorDefault & + Use<'VideoChannel', MChannelId> + +export type MActorDefaultBanner = + MActor & + Use<'Server', MServer> & + Use<'Avatars', MActorImage[]> & + Use<'Banners', MActorImage[]> + +// Actor with channel that is associated to an account and its actor +// Actor -> VideoChannel -> Account -> Actor +export type MActorChannelAccountActor = + MActor & + Use<'VideoChannel', MChannelAccountActor> + +export type MActorFull = + MActor & + Use<'Server', MServer> & + Use<'Avatars', MActorImage[]> & + Use<'Banners', MActorImage[]> & + Use<'Account', MAccount> & + Use<'VideoChannel', MChannelAccountActor> + +// Same than ActorFull, but the account and the channel have their actor +export type MActorFullActor = + MActor & + Use<'Server', MServer> & + Use<'Avatars', MActorImage[]> & + Use<'Banners', MActorImage[]> & + Use<'Account', MAccountDefault> & + Use<'VideoChannel', MChannelAccountDefault> + +// ############################################################################ + +// API + +export type MActorSummary = + FunctionProperties & + Pick & + Use<'Server', MServerHost> & + Use<'Avatars', MActorImage[]> + +export type MActorSummaryBlocks = + MActorSummary & + Use<'Server', MServerHostBlocks> + +export type MActorAPI = + Omit + +// ############################################################################ + +// Format for API or AP object + +export type MActorSummaryFormattable = + FunctionProperties & + Pick & + Use<'Server', MServerHost> & + Use<'Avatars', MActorImageFormattable[]> + +export type MActorFormattable = + MActorSummaryFormattable & + Pick & + Use<'Server', MServerHost & Partial>> & + UseOpt<'Banners', MActorImageFormattable[]> & + UseOpt<'Avatars', MActorImageFormattable[]> + +type MActorAPBase = + MActor & + Use<'Avatars', MActorImage[]> + +export type MActorAPAccount = + MActorAPBase + +export type MActorAPChannel = + MActorAPBase & + Use<'Banners', MActorImage[]> diff --git a/server/server/types/models/actor/index.ts b/server/server/types/models/actor/index.ts new file mode 100644 index 000000000..f7724f54a --- /dev/null +++ b/server/server/types/models/actor/index.ts @@ -0,0 +1,3 @@ +export * from './actor-follow.js' +export * from './actor-image.js' +export * from './actor.js' diff --git a/server/server/types/models/application/application.ts b/server/server/types/models/application/application.ts new file mode 100644 index 000000000..bffd6d718 --- /dev/null +++ b/server/server/types/models/application/application.ts @@ -0,0 +1,5 @@ +import { ApplicationModel } from '@server/models/application/application.js' + +// ############################################################################ + +export type MApplication = Omit diff --git a/server/server/types/models/application/index.ts b/server/server/types/models/application/index.ts new file mode 100644 index 000000000..fbbab9760 --- /dev/null +++ b/server/server/types/models/application/index.ts @@ -0,0 +1 @@ +export * from './application.js' diff --git a/server/server/types/models/index.ts b/server/server/types/models/index.ts new file mode 100644 index 000000000..8c90db53c --- /dev/null +++ b/server/server/types/models/index.ts @@ -0,0 +1,8 @@ +export * from './abuse/index.js' +export * from './account/index.js' +export * from './actor/index.js' +export * from './application/index.js' +export * from './oauth/index.js' +export * from './server/index.js' +export * from './user/index.js' +export * from './video/index.js' diff --git a/server/server/types/models/oauth/index.ts b/server/server/types/models/oauth/index.ts new file mode 100644 index 000000000..c9bbaa53d --- /dev/null +++ b/server/server/types/models/oauth/index.ts @@ -0,0 +1,2 @@ +export * from './oauth-client.js' +export * from './oauth-token.js' diff --git a/server/server/types/models/oauth/oauth-client.ts b/server/server/types/models/oauth/oauth-client.ts new file mode 100644 index 000000000..589a69d16 --- /dev/null +++ b/server/server/types/models/oauth/oauth-client.ts @@ -0,0 +1,3 @@ +import { OAuthClientModel } from '@server/models/oauth/oauth-client.js' + +export type MOAuthClient = Omit diff --git a/server/server/types/models/oauth/oauth-token.ts b/server/server/types/models/oauth/oauth-token.ts new file mode 100644 index 000000000..cf4412938 --- /dev/null +++ b/server/server/types/models/oauth/oauth-token.ts @@ -0,0 +1,14 @@ +import { OAuthTokenModel } from '@server/models/oauth/oauth-token.js' +import { PickWith } from '@peertube/peertube-typescript-utils' +import { MUserAccountUrl } from '../user/user.js' + +type Use = PickWith + +// ############################################################################ + +export type MOAuthToken = Omit + +export type MOAuthTokenUser = + MOAuthToken & + Use<'User', MUserAccountUrl> & + { user?: MUserAccountUrl } diff --git a/server/server/types/models/runners/index.ts b/server/server/types/models/runners/index.ts new file mode 100644 index 000000000..94f33a47a --- /dev/null +++ b/server/server/types/models/runners/index.ts @@ -0,0 +1,3 @@ +export * from './runner.js' +export * from './runner-job.js' +export * from './runner-registration-token.js' diff --git a/server/server/types/models/runners/runner-job.ts b/server/server/types/models/runners/runner-job.ts new file mode 100644 index 000000000..4fa024f57 --- /dev/null +++ b/server/server/types/models/runners/runner-job.ts @@ -0,0 +1,20 @@ +import { RunnerJobModel } from '@server/models/runner/runner-job.js' +import { PickWith } from '@peertube/peertube-typescript-utils' +import { MRunner } from './runner.js' + +type Use = PickWith + +// ############################################################################ + +export type MRunnerJob = Omit + +// ############################################################################ + +export type MRunnerJobRunner = + MRunnerJob & + Use<'Runner', MRunner> + +export type MRunnerJobRunnerParent = + MRunnerJob & + Use<'Runner', MRunner> & + Use<'DependsOnRunnerJob', MRunnerJob> diff --git a/server/server/types/models/runners/runner-registration-token.ts b/server/server/types/models/runners/runner-registration-token.ts new file mode 100644 index 000000000..5dd18a338 --- /dev/null +++ b/server/server/types/models/runners/runner-registration-token.ts @@ -0,0 +1,5 @@ +import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token.js' + +// ############################################################################ + +export type MRunnerRegistrationToken = Omit diff --git a/server/server/types/models/runners/runner.ts b/server/server/types/models/runners/runner.ts new file mode 100644 index 000000000..bde83145e --- /dev/null +++ b/server/server/types/models/runners/runner.ts @@ -0,0 +1,5 @@ +import { RunnerModel } from '@server/models/runner/runner.js' + +// ############################################################################ + +export type MRunner = Omit diff --git a/server/server/types/models/server/index.ts b/server/server/types/models/server/index.ts new file mode 100644 index 000000000..ed5482b70 --- /dev/null +++ b/server/server/types/models/server/index.ts @@ -0,0 +1,3 @@ +export * from './plugin.js' +export * from './server.js' +export * from './server-blocklist.js' diff --git a/server/server/types/models/server/plugin.ts b/server/server/types/models/server/plugin.ts new file mode 100644 index 000000000..0d5a8fb02 --- /dev/null +++ b/server/server/types/models/server/plugin.ts @@ -0,0 +1,11 @@ +import { PluginModel } from '@server/models/server/plugin.js' + +export type MPlugin = PluginModel + +// ############################################################################ + +// Format for API or AP object + +export type MPluginFormattable = + Pick diff --git a/server/server/types/models/server/server-blocklist.ts b/server/server/types/models/server/server-blocklist.ts new file mode 100644 index 000000000..e944e7328 --- /dev/null +++ b/server/server/types/models/server/server-blocklist.ts @@ -0,0 +1,26 @@ +import { ServerBlocklistModel } from '@server/models/server/server-blocklist.js' +import { PickWith } from '@peertube/peertube-typescript-utils' +import { MAccountDefault, MAccountFormattable } from '../account/account.js' +import { MServer, MServerFormattable } from './server.js' + +type Use = PickWith + +// ############################################################################ + +export type MServerBlocklist = Omit + +// ############################################################################ + +export type MServerBlocklistAccountServer = + MServerBlocklist & + Use<'ByAccount', MAccountDefault> & + Use<'BlockedServer', MServer> + +// ############################################################################ + +// Format for API or AP object + +export type MServerBlocklistFormattable = + Pick & + Use<'ByAccount', MAccountFormattable> & + Use<'BlockedServer', MServerFormattable> diff --git a/server/server/types/models/server/server.ts b/server/server/types/models/server/server.ts new file mode 100644 index 000000000..fe3d362b5 --- /dev/null +++ b/server/server/types/models/server/server.ts @@ -0,0 +1,26 @@ +import { FunctionProperties, PickWith } from '@peertube/peertube-typescript-utils' +import { ServerModel } from '../../../models/server/server.js' +import { MAccountBlocklistId } from '../account/index.js' + +type Use = PickWith + +// ############################################################################ + +export type MServer = Omit + +// ############################################################################ + +export type MServerHost = Pick +export type MServerRedundancyAllowed = Pick + +export type MServerHostBlocks = + MServerHost & + Use<'BlockedBy', MAccountBlocklistId[]> + +// ############################################################################ + +// Format for API or AP object + +export type MServerFormattable = + FunctionProperties & + Pick diff --git a/server/server/types/models/server/tracker.ts b/server/server/types/models/server/tracker.ts new file mode 100644 index 000000000..adcd3f435 --- /dev/null +++ b/server/server/types/models/server/tracker.ts @@ -0,0 +1,7 @@ +import { TrackerModel } from '../../../models/server/tracker.js' + +export type MTracker = Omit + +// ############################################################################ + +export type MTrackerUrl = Pick diff --git a/server/server/types/models/user/index.ts b/server/server/types/models/user/index.ts new file mode 100644 index 000000000..53db3944d --- /dev/null +++ b/server/server/types/models/user/index.ts @@ -0,0 +1,5 @@ +export * from './user.js' +export * from './user-notification.js' +export * from './user-notification-setting.js' +export * from './user-registration.js' +export * from './user-video-history.js' diff --git a/server/server/types/models/user/user-notification-setting.ts b/server/server/types/models/user/user-notification-setting.ts new file mode 100644 index 000000000..61f93b6fa --- /dev/null +++ b/server/server/types/models/user/user-notification-setting.ts @@ -0,0 +1,9 @@ +import { UserNotificationSettingModel } from '@server/models/user/user-notification-setting.js' + +export type MNotificationSetting = Omit + +// ############################################################################ + +// Format for API or AP object + +export type MNotificationSettingFormattable = MNotificationSetting diff --git a/server/server/types/models/user/user-notification.ts b/server/server/types/models/user/user-notification.ts new file mode 100644 index 000000000..5a0b57e7a --- /dev/null +++ b/server/server/types/models/user/user-notification.ts @@ -0,0 +1,122 @@ +import { VideoAbuseModel } from '@server/models/abuse/video-abuse.js' +import { VideoCommentAbuseModel } from '@server/models/abuse/video-comment-abuse.js' +import { ApplicationModel } from '@server/models/application/application.js' +import { PluginModel } from '@server/models/server/plugin.js' +import { UserNotificationModel } from '@server/models/user/user-notification.js' +import { UserRegistrationModel } from '@server/models/user/user-registration.js' +import { PickWith, PickWithOpt } from '@peertube/peertube-typescript-utils' +import { AbuseModel } from '../../../models/abuse/abuse.js' +import { AccountModel } from '../../../models/account/account.js' +import { ActorModel } from '../../../models/actor/actor.js' +import { ActorFollowModel } from '../../../models/actor/actor-follow.js' +import { ActorImageModel } from '../../../models/actor/actor-image.js' +import { ServerModel } from '../../../models/server/server.js' +import { VideoModel } from '../../../models/video/video.js' +import { VideoBlacklistModel } from '../../../models/video/video-blacklist.js' +import { VideoChannelModel } from '../../../models/video/video-channel.js' +import { VideoCommentModel } from '../../../models/video/video-comment.js' +import { VideoImportModel } from '../../../models/video/video-import.js' + +type Use = PickWith + +// ############################################################################ + +export module UserNotificationIncludes { + export type ActorImageInclude = Pick + + export type VideoInclude = Pick + export type VideoIncludeChannel = + VideoInclude & + PickWith + + export type ActorInclude = + Pick & + PickWith & + PickWith> + + export type VideoChannelInclude = Pick + export type VideoChannelIncludeActor = + VideoChannelInclude & + PickWith + + export type AccountInclude = Pick + export type AccountIncludeActor = + AccountInclude & + PickWith + + export type VideoCommentInclude = + Pick & + PickWith & + PickWith + + export type VideoAbuseInclude = + Pick & + PickWith + + export type VideoCommentAbuseInclude = + Pick & + PickWith & + PickWith>> + + export type AbuseInclude = + Pick & + PickWith & + PickWith & + PickWith + + export type VideoBlacklistInclude = + Pick & + PickWith + + export type VideoImportInclude = + Pick & + PickWith + + export type ActorFollower = + Pick & + PickWith & + PickWith> & + PickWithOpt + + export type ActorFollowing = + Pick & + PickWith & + PickWith & + PickWith> + + export type ActorFollowInclude = + Pick & + PickWith & + PickWith + + export type PluginInclude = + Pick + + export type ApplicationInclude = + Pick + + export type UserRegistrationInclude = + Pick +} + +// ############################################################################ + +export type MUserNotification = + Omit + +// ############################################################################ + +export type UserNotificationModelForApi = + MUserNotification & + Use<'Video', UserNotificationIncludes.VideoIncludeChannel> & + Use<'VideoComment', UserNotificationIncludes.VideoCommentInclude> & + Use<'Abuse', UserNotificationIncludes.AbuseInclude> & + Use<'VideoBlacklist', UserNotificationIncludes.VideoBlacklistInclude> & + Use<'VideoImport', UserNotificationIncludes.VideoImportInclude> & + Use<'ActorFollow', UserNotificationIncludes.ActorFollowInclude> & + Use<'Plugin', UserNotificationIncludes.PluginInclude> & + Use<'Application', UserNotificationIncludes.ApplicationInclude> & + Use<'Account', UserNotificationIncludes.AccountIncludeActor> & + Use<'UserRegistration', UserNotificationIncludes.UserRegistrationInclude> diff --git a/server/server/types/models/user/user-registration.ts b/server/server/types/models/user/user-registration.ts new file mode 100644 index 000000000..bcbe52fbd --- /dev/null +++ b/server/server/types/models/user/user-registration.ts @@ -0,0 +1,15 @@ +import { UserRegistrationModel } from '@server/models/user/user-registration.js' +import { PickWith } from '@peertube/peertube-typescript-utils' +import { MUserId } from './user.js' + +type Use = PickWith + +// ############################################################################ + +export type MRegistration = Omit + +// ############################################################################ + +export type MRegistrationFormattable = + MRegistration & + Use<'User', MUserId> diff --git a/server/server/types/models/user/user-video-history.ts b/server/server/types/models/user/user-video-history.ts new file mode 100644 index 000000000..d61742070 --- /dev/null +++ b/server/server/types/models/user/user-video-history.ts @@ -0,0 +1,5 @@ +import { UserVideoHistoryModel } from '../../../models/user/user-video-history.js' + +export type MUserVideoHistory = Omit + +export type MUserVideoHistoryTime = Pick diff --git a/server/server/types/models/user/user.ts b/server/server/types/models/user/user.ts new file mode 100644 index 000000000..4a655c792 --- /dev/null +++ b/server/server/types/models/user/user.ts @@ -0,0 +1,89 @@ +import { AccountModel } from '@server/models/account/account.js' +import { UserModel } from '@server/models/user/user.js' +import { MVideoPlaylist } from '@server/types/models/index.js' +import { PickWith, PickWithOpt } from '@peertube/peertube-typescript-utils' +import { + MAccount, + MAccountDefault, + MAccountDefaultChannelDefault, + MAccountFormattable, + MAccountId, + MAccountIdActorId, + MAccountUrl +} from '../account/index.js' +import { MChannelFormattable } from '../video/video-channels.js' +import { MNotificationSetting, MNotificationSettingFormattable } from './user-notification-setting.js' + +type Use = PickWith + +// ############################################################################ + +export type MUser = Omit + +// ############################################################################ + +export type MUserQuotaUsed = MUser & { videoQuotaUsed?: number, videoQuotaUsedDaily?: number } +export type MUserId = Pick + +// ############################################################################ + +// With account + +export type MUserAccountId = + MUser & + Use<'Account', MAccountId> + +export type MUserAccountUrl = + MUser & + Use<'Account', MAccountUrl & MAccountIdActorId> + +export type MUserAccount = + MUser & + Use<'Account', MAccount> + +export type MUserAccountDefault = + MUser & + Use<'Account', MAccountDefault> + +// With channel + +export type MUserNotifSettingChannelDefault = + MUser & + Use<'NotificationSetting', MNotificationSetting> & + Use<'Account', MAccountDefaultChannelDefault> + +// With notification settings + +export type MUserWithNotificationSetting = + MUser & + Use<'NotificationSetting', MNotificationSetting> + +export type MUserNotifSettingAccount = + MUser & + Use<'NotificationSetting', MNotificationSetting> & + Use<'Account', MAccount> + +// Default scope + +export type MUserDefault = + MUser & + Use<'NotificationSetting', MNotificationSetting> & + Use<'Account', MAccountDefault> + +// ############################################################################ + +// Format for API or AP object + +type MAccountWithChannels = MAccountFormattable & PickWithOpt +type MAccountWithChannelsAndSpecialPlaylists = + MAccountWithChannels & + PickWithOpt + +export type MUserFormattable = + MUserQuotaUsed & + Use<'Account', MAccountWithChannels> & + PickWithOpt + +export type MMyUserFormattable = + MUserFormattable & + Use<'Account', MAccountWithChannelsAndSpecialPlaylists> diff --git a/server/server/types/models/video/index.ts b/server/server/types/models/video/index.ts new file mode 100644 index 000000000..f88198b67 --- /dev/null +++ b/server/server/types/models/video/index.ts @@ -0,0 +1,26 @@ +export * from './local-video-viewer-watch-section.js' +export * from './local-video-viewer-watch-section.js' +export * from './local-video-viewer.js' +export * from './storyboard.js' +export * from './schedule-video-update.js' +export * from './tag.js' +export * from './thumbnail.js' +export * from './video.js' +export * from './video-blacklist.js' +export * from './video-caption.js' +export * from './video-change-ownership.js' +export * from './video-channel-sync.js' +export * from './video-channels.js' +export * from './video-comment.js' +export * from './video-file.js' +export * from './video-import.js' +export * from './video-live-replay-setting.js' +export * from './video-live-session.js' +export * from './video-live.js' +export * from './video-password.js' +export * from './video-playlist.js' +export * from './video-playlist-element.js' +export * from './video-rate.js' +export * from './video-redundancy.js' +export * from './video-share.js' +export * from './video-streaming-playlist.js' diff --git a/server/server/types/models/video/local-video-viewer-watch-section.ts b/server/server/types/models/video/local-video-viewer-watch-section.ts new file mode 100644 index 000000000..c621dd1cc --- /dev/null +++ b/server/server/types/models/video/local-video-viewer-watch-section.ts @@ -0,0 +1,5 @@ +import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section.js' + +// ############################################################################ + +export type MLocalVideoViewerWatchSection = Omit diff --git a/server/server/types/models/video/local-video-viewer.ts b/server/server/types/models/video/local-video-viewer.ts new file mode 100644 index 000000000..90535dac0 --- /dev/null +++ b/server/server/types/models/video/local-video-viewer.ts @@ -0,0 +1,19 @@ +import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer.js' +import { PickWith } from '@peertube/peertube-typescript-utils' +import { MLocalVideoViewerWatchSection } from './local-video-viewer-watch-section.js' +import { MVideo } from './video.js' + +type Use = PickWith + +// ############################################################################ + +export type MLocalVideoViewer = Omit + +export type MLocalVideoViewerVideo = + MLocalVideoViewer & + Use<'Video', MVideo> + +export type MLocalVideoViewerWithWatchSections = + MLocalVideoViewer & + Use<'Video', MVideo> & + Use<'WatchSections', MLocalVideoViewerWatchSection[]> diff --git a/server/server/types/models/video/schedule-video-update.ts b/server/server/types/models/video/schedule-video-update.ts new file mode 100644 index 000000000..7bf998b49 --- /dev/null +++ b/server/server/types/models/video/schedule-video-update.ts @@ -0,0 +1,11 @@ +import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update.js' + +// ############################################################################ + +export type MScheduleVideoUpdate = Omit + +// ############################################################################ + +// Format for API or AP object + +export type MScheduleVideoUpdateFormattable = Pick diff --git a/server/server/types/models/video/storyboard.ts b/server/server/types/models/video/storyboard.ts new file mode 100644 index 000000000..d8a0e39ea --- /dev/null +++ b/server/server/types/models/video/storyboard.ts @@ -0,0 +1,15 @@ +import { StoryboardModel } from '@server/models/video/storyboard.js' +import { PickWith } from '@peertube/peertube-typescript-utils' +import { MVideo } from './video.js' + +type Use = PickWith + +// ############################################################################ + +export type MStoryboard = Omit + +// ############################################################################ + +export type MStoryboardVideo = + MStoryboard & + Use<'Video', MVideo> diff --git a/server/server/types/models/video/tag.ts b/server/server/types/models/video/tag.ts new file mode 100644 index 000000000..23fdf89e0 --- /dev/null +++ b/server/server/types/models/video/tag.ts @@ -0,0 +1,3 @@ +import { TagModel } from '../../../models/video/tag.js' + +export type MTag = Omit diff --git a/server/server/types/models/video/thumbnail.ts b/server/server/types/models/video/thumbnail.ts new file mode 100644 index 000000000..b952c06a0 --- /dev/null +++ b/server/server/types/models/video/thumbnail.ts @@ -0,0 +1,15 @@ +import { PickWith } from '@peertube/peertube-typescript-utils' +import { ThumbnailModel } from '../../../models/video/thumbnail.js' +import { MVideo } from './video.js' + +type Use = PickWith + +// ############################################################################ + +export type MThumbnail = Omit + +// ############################################################################ + +export type MThumbnailVideo = + MThumbnail & + Use<'Video', MVideo> diff --git a/server/server/types/models/video/video-blacklist.ts b/server/server/types/models/video/video-blacklist.ts new file mode 100644 index 000000000..b202e9261 --- /dev/null +++ b/server/server/types/models/video/video-blacklist.ts @@ -0,0 +1,30 @@ +import { PickWith } from '@peertube/peertube-typescript-utils' +import { VideoBlacklistModel } from '../../../models/video/video-blacklist.js' +import { MVideo, MVideoFormattable } from './video.js' + +type Use = PickWith + +// ############################################################################ + +export type MVideoBlacklist = Omit + +export type MVideoBlacklistLight = Pick +export type MVideoBlacklistUnfederated = Pick + +// ############################################################################ + +export type MVideoBlacklistLightVideo = + MVideoBlacklistLight & + Use<'Video', MVideo> + +export type MVideoBlacklistVideo = + MVideoBlacklist & + Use<'Video', MVideo> + +// ############################################################################ + +// Format for API or AP object + +export type MVideoBlacklistFormattable = + MVideoBlacklist & + Use<'Video', MVideoFormattable> diff --git a/server/server/types/models/video/video-caption.ts b/server/server/types/models/video/video-caption.ts new file mode 100644 index 000000000..8e8393d92 --- /dev/null +++ b/server/server/types/models/video/video-caption.ts @@ -0,0 +1,27 @@ +import { PickWith } from '@peertube/peertube-typescript-utils' +import { VideoCaptionModel } from '../../../models/video/video-caption.js' +import { MVideo, MVideoUUID } from './video.js' + +type Use = PickWith + +// ############################################################################ + +export type MVideoCaption = Omit + +// ############################################################################ + +export type MVideoCaptionLanguage = Pick +export type MVideoCaptionLanguageUrl = Pick + +export type MVideoCaptionVideo = + MVideoCaption & + Use<'Video', Pick> + +// ############################################################################ + +// Format for API or AP object + +export type MVideoCaptionFormattable = + MVideoCaption & + Pick & + Use<'Video', MVideoUUID> diff --git a/server/server/types/models/video/video-change-ownership.ts b/server/server/types/models/video/video-change-ownership.ts new file mode 100644 index 000000000..70753e30b --- /dev/null +++ b/server/server/types/models/video/video-change-ownership.ts @@ -0,0 +1,26 @@ +import { VideoChangeOwnershipModel } from '@server/models/video/video-change-ownership.js' +import { PickWith } from '@peertube/peertube-typescript-utils' +import { MAccountDefault, MAccountFormattable } from '../account/account.js' +import { MVideoFormattable, MVideoWithAllFiles } from './video.js' + +type Use = PickWith + +// ############################################################################ + +export type MVideoChangeOwnership = Omit + +export type MVideoChangeOwnershipFull = + MVideoChangeOwnership & + Use<'Initiator', MAccountDefault> & + Use<'NextOwner', MAccountDefault> & + Use<'Video', MVideoWithAllFiles> + +// ############################################################################ + +// Format for API or AP object + +export type MVideoChangeOwnershipFormattable = + Pick & + Use<'Initiator', MAccountFormattable> & + Use<'NextOwner', MAccountFormattable> & + Use<'Video', MVideoFormattable> diff --git a/server/server/types/models/video/video-channel-sync.ts b/server/server/types/models/video/video-channel-sync.ts new file mode 100644 index 000000000..2b3a3930f --- /dev/null +++ b/server/server/types/models/video/video-channel-sync.ts @@ -0,0 +1,17 @@ +import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync.js' +import { FunctionProperties, PickWith } from '@peertube/peertube-typescript-utils' +import { MChannelAccountDefault, MChannelFormattable } from './video-channels.js' + +type Use = PickWith + +export type MChannelSync = Omit + +export type MChannelSyncChannel = + MChannelSync & + Use<'VideoChannel', MChannelAccountDefault> & + FunctionProperties + +export type MChannelSyncFormattable = + FunctionProperties & + Use<'VideoChannel', MChannelFormattable> & + MChannelSync diff --git a/server/server/types/models/video/video-channels.ts b/server/server/types/models/video/video-channels.ts new file mode 100644 index 000000000..e8cb9cb26 --- /dev/null +++ b/server/server/types/models/video/video-channels.ts @@ -0,0 +1,153 @@ +import { FunctionProperties, PickWith, PickWithOpt } from '@peertube/peertube-typescript-utils' +import { VideoChannelModel } from '../../../models/video/video-channel.js' +import { + MAccountActor, + MAccountAPI, + MAccountDefault, + MAccountFormattable, + MAccountLight, + MAccountSummaryBlocks, + MAccountSummaryFormattable, + MAccountUrl, + MAccountUserId +} from '../account/index.js' +import { + MActor, + MActorAccountChannelId, + MActorAPChannel, + MActorAPI, + MActorDefault, + MActorDefaultBanner, + MActorDefaultLight, + MActorFormattable, + MActorHost, + MActorHostOnly, + MActorLight, + MActorSummary, + MActorSummaryFormattable, + MActorUrl +} from '../actor/index.js' +import { MVideo } from './video.js' + +type Use = PickWith + +// ############################################################################ + +export type MChannel = Omit + +// ############################################################################ + +export type MChannelId = Pick + +// ############################################################################ + +export type MChannelIdActor = + MChannelId & + Use<'Actor', MActorAccountChannelId> + +export type MChannelUserId = + Pick & + Use<'Account', MAccountUserId> + +export type MChannelActor = + MChannel & + Use<'Actor', MActor> + +export type MChannelUrl = Use<'Actor', MActorUrl> + +// Default scope +export type MChannelDefault = + MChannel & + Use<'Actor', MActorDefault> + +export type MChannelBannerDefault = + MChannel & + Use<'Actor', MActorDefaultBanner> + +// ############################################################################ + +// Not all association attributes + +export type MChannelActorLight = + MChannel & + Use<'Actor', MActorLight> + +export type MChannelAccountLight = + MChannel & + Use<'Actor', MActorDefaultLight> & + Use<'Account', MAccountLight> + +export type MChannelHost = + MChannel & + Use<'Actor', MActorHost> + +export type MChannelHostOnly = + MChannelId & + Use<'Actor', MActorHostOnly> + +// ############################################################################ + +// Account associations + +export type MChannelAccountActor = + MChannel & + Use<'Account', MAccountActor> + +export type MChannelBannerAccountDefault = + MChannel & + Use<'Actor', MActorDefaultBanner> & + Use<'Account', MAccountDefault> + +export type MChannelAccountDefault = + MChannel & + Use<'Actor', MActorDefault> & + Use<'Account', MAccountDefault> + +// ############################################################################ + +// Videos associations +export type MChannelVideos = + MChannel & + Use<'Videos', MVideo[]> + +// ############################################################################ + +// For API + +export type MChannelSummary = + FunctionProperties & + Pick & + Use<'Actor', MActorSummary> + +export type MChannelSummaryAccount = + MChannelSummary & + Use<'Account', MAccountSummaryBlocks> + +export type MChannelAPI = + MChannel & + Use<'Actor', MActorAPI> & + Use<'Account', MAccountAPI> + +// ############################################################################ + +// Format for API or AP object + +export type MChannelSummaryFormattable = + FunctionProperties & + Pick & + Use<'Actor', MActorSummaryFormattable> + +export type MChannelAccountSummaryFormattable = + MChannelSummaryFormattable & + Use<'Account', MAccountSummaryFormattable> + +export type MChannelFormattable = + FunctionProperties & + Pick & + Use<'Actor', MActorFormattable> & + PickWithOpt + +export type MChannelAP = + Pick & + Use<'Actor', MActorAPChannel> & + Use<'Account', MAccountUrl> diff --git a/server/server/types/models/video/video-comment.ts b/server/server/types/models/video/video-comment.ts new file mode 100644 index 000000000..d4fd34f7c --- /dev/null +++ b/server/server/types/models/video/video-comment.ts @@ -0,0 +1,71 @@ +import { PickWith, PickWithOpt } from '@peertube/peertube-typescript-utils' +import { VideoCommentModel } from '../../../models/video/video-comment.js' +import { MAccountDefault, MAccountFormattable, MAccountUrl } from '../account/index.js' +import { MVideo, MVideoAccountLight, MVideoFeed, MVideoIdUrl, MVideoUrl } from './video.js' + +type Use = PickWith + +// ############################################################################ + +export type MComment = Omit +export type MCommentTotalReplies = MComment & { totalReplies?: number } +export type MCommentId = Pick +export type MCommentUrl = Pick + +// ############################################################################ + +export type MCommentOwner = + MComment & + Use<'Account', MAccountDefault> + +export type MCommentVideo = + MComment & + Use<'Video', MVideoAccountLight> + +export type MCommentReply = + MComment & + Use<'InReplyToVideoComment', MComment> + +export type MCommentOwnerVideo = + MComment & + Use<'Account', MAccountDefault> & + Use<'Video', MVideoAccountLight> + +export type MCommentOwnerVideoReply = + MComment & + Use<'Account', MAccountDefault> & + Use<'Video', MVideoAccountLight> & + Use<'InReplyToVideoComment', MComment> + +export type MCommentOwnerReplyVideoLight = + MComment & + Use<'Account', MAccountDefault> & + Use<'InReplyToVideoComment', MComment> & + Use<'Video', MVideoIdUrl> + +export type MCommentOwnerVideoFeed = + MCommentOwner & + Use<'Video', MVideoFeed> + +// ############################################################################ + +export type MCommentAPI = MComment & { totalReplies: number } + +// ############################################################################ + +// Format for API or AP object + +export type MCommentFormattable = + MCommentTotalReplies & + Use<'Account', MAccountFormattable> + +export type MCommentAdminFormattable = + MComment & + Use<'Account', MAccountFormattable> & + Use<'Video', MVideo> + +export type MCommentAP = + MComment & + Use<'Account', MAccountUrl> & + PickWithOpt & + PickWithOpt diff --git a/server/server/types/models/video/video-file.ts b/server/server/types/models/video/video-file.ts new file mode 100644 index 000000000..8431b6f5a --- /dev/null +++ b/server/server/types/models/video/video-file.ts @@ -0,0 +1,43 @@ +import { PickWith, PickWithOpt } from '@peertube/peertube-typescript-utils' +import { VideoFileModel } from '../../../models/video/video-file.js' +import { MVideo, MVideoUUID } from './video.js' +import { MVideoRedundancy, MVideoRedundancyFileUrl } from './video-redundancy.js' +import { MStreamingPlaylist, MStreamingPlaylistVideo } from './video-streaming-playlist.js' + +type Use = PickWith + +// ############################################################################ + +export type MVideoFile = Omit + +export type MVideoFileVideo = + MVideoFile & + Use<'Video', MVideo> + +export type MVideoFileStreamingPlaylist = + MVideoFile & + Use<'VideoStreamingPlaylist', MStreamingPlaylist> + +export type MVideoFileStreamingPlaylistVideo = + MVideoFile & + Use<'VideoStreamingPlaylist', MStreamingPlaylistVideo> + +export type MVideoFileVideoUUID = + MVideoFile & + Use<'Video', MVideoUUID> + +export type MVideoFileRedundanciesAll = + MVideoFile & + PickWithOpt + +export type MVideoFileRedundanciesOpt = + MVideoFile & + PickWithOpt + +export function isStreamingPlaylistFile (file: any): file is MVideoFileStreamingPlaylist { + return !!file.videoStreamingPlaylistId +} + +export function isWebVideoFile (file: any): file is MVideoFileVideo { + return !!file.videoId +} diff --git a/server/server/types/models/video/video-import.ts b/server/server/types/models/video/video-import.ts new file mode 100644 index 000000000..4920dfac3 --- /dev/null +++ b/server/server/types/models/video/video-import.ts @@ -0,0 +1,36 @@ +import { VideoImportModel } from '@server/models/video/video-import.js' +import { PickWith, PickWithOpt } from '@peertube/peertube-typescript-utils' +import { MUser } from '../user/user.js' +import { MVideo, MVideoAccountLight, MVideoFormattable, MVideoTag, MVideoThumbnail, MVideoWithFile } from './video.js' + +type Use = PickWith + +// ############################################################################ + +export type MVideoImport = Omit + +export type MVideoImportVideo = + MVideoImport & + Use<'Video', MVideo> + +// ############################################################################ + +type VideoAssociation = MVideoTag & MVideoAccountLight & MVideoThumbnail + +export type MVideoImportDefault = + MVideoImport & + Use<'User', MUser> & + Use<'Video', VideoAssociation> + +export type MVideoImportDefaultFiles = + MVideoImport & + Use<'User', MUser> & + Use<'Video', VideoAssociation & MVideoWithFile> + +// ############################################################################ + +// Format for API or AP object + +export type MVideoImportFormattable = + MVideoImport & + PickWithOpt diff --git a/server/server/types/models/video/video-live-replay-setting.ts b/server/server/types/models/video/video-live-replay-setting.ts new file mode 100644 index 000000000..a7e1be8e4 --- /dev/null +++ b/server/server/types/models/video/video-live-replay-setting.ts @@ -0,0 +1,3 @@ +import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting.js' + +export type MLiveReplaySetting = Omit diff --git a/server/server/types/models/video/video-live-session.ts b/server/server/types/models/video/video-live-session.ts new file mode 100644 index 000000000..994726244 --- /dev/null +++ b/server/server/types/models/video/video-live-session.ts @@ -0,0 +1,17 @@ +import { VideoLiveSessionModel } from '@server/models/video/video-live-session.js' +import { PickWith } from '@peertube/peertube-typescript-utils' +import { MVideo } from './video.js' +import { MLiveReplaySetting } from './video-live-replay-setting.js' + +type Use = PickWith + +// ############################################################################ + +export type MVideoLiveSession = Omit + +// ############################################################################ + +export type MVideoLiveSessionReplay = + MVideoLiveSession & + Use<'ReplayVideo', MVideo> & + Use<'ReplaySetting', MLiveReplaySetting> diff --git a/server/server/types/models/video/video-live.ts b/server/server/types/models/video/video-live.ts new file mode 100644 index 000000000..6a3ca8f49 --- /dev/null +++ b/server/server/types/models/video/video-live.ts @@ -0,0 +1,22 @@ +import { VideoLiveModel } from '@server/models/video/video-live.js' +import { PickWith } from '@peertube/peertube-typescript-utils' +import { MVideo } from './video.js' +import { MLiveReplaySetting } from './video-live-replay-setting.js' + +type Use = PickWith + +// ############################################################################ + +export type MVideoLive = Omit + +// ############################################################################ + +export type MVideoLiveVideo = + MVideoLive & + Use<'Video', MVideo> + +// ############################################################################ + +export type MVideoLiveVideoWithSetting = + MVideoLiveVideo & + Use<'ReplaySetting', MLiveReplaySetting> diff --git a/server/server/types/models/video/video-password.ts b/server/server/types/models/video/video-password.ts new file mode 100644 index 000000000..6e0e1a5e7 --- /dev/null +++ b/server/server/types/models/video/video-password.ts @@ -0,0 +1,3 @@ +import { VideoPasswordModel } from '@server/models/video/video-password.js' + +export type MVideoPassword = Omit diff --git a/server/server/types/models/video/video-playlist-element.ts b/server/server/types/models/video/video-playlist-element.ts new file mode 100644 index 000000000..bf8ff6daf --- /dev/null +++ b/server/server/types/models/video/video-playlist-element.ts @@ -0,0 +1,39 @@ +import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element.js' +import { PickWith } from '@peertube/peertube-typescript-utils' +import { MVideoFormattable, MVideoThumbnail, MVideoUrl } from './video.js' +import { MVideoPlaylistPrivacy } from './video-playlist.js' + +type Use = PickWith + +// ############################################################################ + +export type MVideoPlaylistElement = Omit + +// ############################################################################ + +export type MVideoPlaylistElementId = Pick + +export type MVideoPlaylistElementLight = Pick + +// ############################################################################ + +export type MVideoPlaylistVideoThumbnail = + MVideoPlaylistElement & + Use<'Video', MVideoThumbnail> + +export type MVideoPlaylistElementVideoUrlPlaylistPrivacy = + MVideoPlaylistElement & + Use<'Video', MVideoUrl> & + Use<'VideoPlaylist', MVideoPlaylistPrivacy> + +// ############################################################################ + +// Format for API or AP object + +export type MVideoPlaylistElementFormattable = + MVideoPlaylistElement & + Use<'Video', MVideoFormattable> + +export type MVideoPlaylistElementAP = + MVideoPlaylistElement & + Use<'Video', MVideoUrl> diff --git a/server/server/types/models/video/video-playlist.ts b/server/server/types/models/video/video-playlist.ts new file mode 100644 index 000000000..3d99bf4e5 --- /dev/null +++ b/server/server/types/models/video/video-playlist.ts @@ -0,0 +1,104 @@ +import { MVideoPlaylistElementLight } from '@server/types/models/video/video-playlist-element.js' +import { PickWith } from '@peertube/peertube-typescript-utils' +import { VideoPlaylistModel } from '../../../models/video/video-playlist.js' +import { MAccount, MAccountDefault, MAccountSummary, MAccountSummaryFormattable } from '../account/index.js' +import { MThumbnail } from './thumbnail.js' +import { MChannelDefault, MChannelSummary, MChannelSummaryFormattable, MChannelUrl } from './video-channels.js' + +type Use = PickWith + +// ############################################################################ + +export type MVideoPlaylist = Omit + +// ############################################################################ + +export type MVideoPlaylistId = Pick +export type MVideoPlaylistSummary = + Pick & + Pick & + Pick +export type MVideoPlaylistPrivacy = Pick +export type MVideoPlaylistUUID = Pick +export type MVideoPlaylistVideosLength = MVideoPlaylist & { videosLength?: number } + +// ############################################################################ + +// With elements + +export type MVideoPlaylistSummaryWithElements = + MVideoPlaylistSummary & + Use<'VideoPlaylistElements', MVideoPlaylistElementLight[]> + +// ############################################################################ + +// With account + +export type MVideoPlaylistOwner = + MVideoPlaylist & + Use<'OwnerAccount', MAccount> + +export type MVideoPlaylistOwnerDefault = + MVideoPlaylist & + Use<'OwnerAccount', MAccountDefault> + +// ############################################################################ + +// With thumbnail + +export type MVideoPlaylistThumbnail = + MVideoPlaylist & + Use<'Thumbnail', MThumbnail> + +export type MVideoPlaylistAccountThumbnail = + MVideoPlaylist & + Use<'OwnerAccount', MAccountDefault> & + Use<'Thumbnail', MThumbnail> + +// ############################################################################ + +// With channel + +export type MVideoPlaylistAccountChannelDefault = + MVideoPlaylist & + Use<'OwnerAccount', MAccountDefault> & + Use<'VideoChannel', MChannelDefault> + +// ############################################################################ + +// With all associations + +export type MVideoPlaylistFull = + MVideoPlaylistVideosLength & + Use<'OwnerAccount', MAccountDefault> & + Use<'VideoChannel', MChannelDefault> & + Use<'Thumbnail', MThumbnail> + +// ############################################################################ + +// For API + +export type MVideoPlaylistAccountChannelSummary = + MVideoPlaylist & + Use<'OwnerAccount', MAccountSummary> & + Use<'VideoChannel', MChannelSummary> + +export type MVideoPlaylistFullSummary = + MVideoPlaylistVideosLength & + Use<'Thumbnail', MThumbnail> & + Use<'OwnerAccount', MAccountSummary> & + Use<'VideoChannel', MChannelSummary> + +// ############################################################################ + +// Format for API or AP object + +export type MVideoPlaylistFormattable = + MVideoPlaylistVideosLength & + Use<'OwnerAccount', MAccountSummaryFormattable> & + Use<'VideoChannel', MChannelSummaryFormattable> + +export type MVideoPlaylistAP = + MVideoPlaylist & + Use<'Thumbnail', MThumbnail> & + Use<'VideoChannel', MChannelUrl> diff --git a/server/server/types/models/video/video-rate.ts b/server/server/types/models/video/video-rate.ts new file mode 100644 index 000000000..873b289e6 --- /dev/null +++ b/server/server/types/models/video/video-rate.ts @@ -0,0 +1,27 @@ +import { AccountVideoRateModel } from '@server/models/account/account-video-rate.js' +import { PickWith } from '@peertube/peertube-typescript-utils' +import { MAccountAudience, MAccountUrl } from '../account/account.js' +import { MVideo, MVideoFormattable } from './video.js' + +type Use = PickWith + +// ############################################################################ + +export type MAccountVideoRate = Omit + +export type MAccountVideoRateAccountUrl = + MAccountVideoRate & + Use<'Account', MAccountUrl> + +export type MAccountVideoRateAccountVideo = + MAccountVideoRate & + Use<'Account', MAccountAudience> & + Use<'Video', MVideo> + +// ############################################################################ + +// Format for API or AP object + +export type MAccountVideoRateFormattable = + Pick & + Use<'Video', MVideoFormattable> diff --git a/server/server/types/models/video/video-redundancy.ts b/server/server/types/models/video/video-redundancy.ts new file mode 100644 index 000000000..55014cdf6 --- /dev/null +++ b/server/server/types/models/video/video-redundancy.ts @@ -0,0 +1,43 @@ +import { VideoFileModel } from '@server/models/video/video-file.js' +import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js' +import { PickWith, PickWithOpt } from '@peertube/peertube-typescript-utils' +import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy.js' +import { MVideoUrl } from './video.js' +import { MVideoFile, MVideoFileVideo } from './video-file.js' +import { MStreamingPlaylistVideo } from './video-streaming-playlist.js' + +type Use = PickWith + +// ############################################################################ + +export type MVideoRedundancy = Omit + +export type MVideoRedundancyFileUrl = Pick + +// ############################################################################ + +export type MVideoRedundancyFile = + MVideoRedundancy & + Use<'VideoFile', MVideoFile> + +export type MVideoRedundancyFileVideo = + MVideoRedundancy & + Use<'VideoFile', MVideoFileVideo> + +export type MVideoRedundancyStreamingPlaylistVideo = + MVideoRedundancy & + Use<'VideoStreamingPlaylist', MStreamingPlaylistVideo> + +export type MVideoRedundancyVideo = + MVideoRedundancy & + Use<'VideoFile', MVideoFileVideo> & + Use<'VideoStreamingPlaylist', MStreamingPlaylistVideo> + +// ############################################################################ + +// Format for API or AP object + +export type MVideoRedundancyAP = + MVideoRedundancy & + PickWithOpt> & + PickWithOpt> diff --git a/server/server/types/models/video/video-share.ts b/server/server/types/models/video/video-share.ts new file mode 100644 index 000000000..52e98d8ec --- /dev/null +++ b/server/server/types/models/video/video-share.ts @@ -0,0 +1,19 @@ +import { PickWith } from '@peertube/peertube-typescript-utils' +import { VideoShareModel } from '../../../models/video/video-share.js' +import { MActorDefault } from '../actor/index.js' +import { MVideo } from './video.js' + +type Use = PickWith + +// ############################################################################ + +export type MVideoShare = Omit + +export type MVideoShareActor = + MVideoShare & + Use<'Actor', MActorDefault> + +export type MVideoShareFull = + MVideoShare & + Use<'Actor', MActorDefault> & + Use<'Video', MVideo> diff --git a/server/server/types/models/video/video-source.ts b/server/server/types/models/video/video-source.ts new file mode 100644 index 000000000..e29feec14 --- /dev/null +++ b/server/server/types/models/video/video-source.ts @@ -0,0 +1,3 @@ +import { VideoSourceModel } from '@server/models/video/video-source.js' + +export type MVideoSource = Omit diff --git a/server/server/types/models/video/video-streaming-playlist.ts b/server/server/types/models/video/video-streaming-playlist.ts new file mode 100644 index 000000000..a41646969 --- /dev/null +++ b/server/server/types/models/video/video-streaming-playlist.ts @@ -0,0 +1,43 @@ +import { PickWith, PickWithOpt } from '@peertube/peertube-typescript-utils' +import { VideoStreamingPlaylistModel } from '../../../models/video/video-streaming-playlist.js' +import { MVideo } from './video.js' +import { MVideoFile } from './video-file.js' +import { MVideoRedundancy, MVideoRedundancyFileUrl } from './video-redundancy.js' + +type Use = PickWith + +// ############################################################################ + +export type MStreamingPlaylist = Omit + +export type MStreamingPlaylistFiles = + MStreamingPlaylist & + Use<'VideoFiles', MVideoFile[]> + +export type MStreamingPlaylistVideo = + MStreamingPlaylist & + Use<'Video', MVideo> + +export type MStreamingPlaylistFilesVideo = + MStreamingPlaylist & + Use<'VideoFiles', MVideoFile[]> & + Use<'Video', MVideo> + +export type MStreamingPlaylistRedundanciesAll = + MStreamingPlaylist & + Use<'VideoFiles', MVideoFile[]> & + Use<'RedundancyVideos', MVideoRedundancy[]> + +export type MStreamingPlaylistRedundancies = + MStreamingPlaylist & + Use<'VideoFiles', MVideoFile[]> & + Use<'RedundancyVideos', MVideoRedundancyFileUrl[]> + +export type MStreamingPlaylistRedundanciesOpt = + MStreamingPlaylist & + Use<'VideoFiles', MVideoFile[]> & + PickWithOpt + +export function isStreamingPlaylist (value: MVideo | MStreamingPlaylistVideo): value is MStreamingPlaylistVideo { + return !!(value as MStreamingPlaylist).videoId +} diff --git a/server/server/types/models/video/video.ts b/server/server/types/models/video/video.ts new file mode 100644 index 000000000..b7f8652be --- /dev/null +++ b/server/server/types/models/video/video.ts @@ -0,0 +1,225 @@ +import { PickWith, PickWithOpt } from '@peertube/peertube-typescript-utils' +import { VideoModel } from '../../../models/video/video.js' +import { MTrackerUrl } from '../server/tracker.js' +import { MUserVideoHistoryTime } from '../user/user-video-history.js' +import { MScheduleVideoUpdate } from './schedule-video-update.js' +import { MStoryboard } from './storyboard.js' +import { MTag } from './tag.js' +import { MThumbnail } from './thumbnail.js' +import { MVideoBlacklist, MVideoBlacklistLight, MVideoBlacklistUnfederated } from './video-blacklist.js' +import { MVideoCaptionLanguage, MVideoCaptionLanguageUrl } from './video-caption.js' +import { + MChannelAccountDefault, + MChannelAccountLight, + MChannelAccountSummaryFormattable, + MChannelActor, + MChannelFormattable, + MChannelHostOnly, + MChannelUserId +} from './video-channels.js' +import { MVideoFile, MVideoFileRedundanciesAll, MVideoFileRedundanciesOpt } from './video-file.js' +import { MVideoLive } from './video-live.js' +import { + MStreamingPlaylistFiles, + MStreamingPlaylistRedundancies, + MStreamingPlaylistRedundanciesAll, + MStreamingPlaylistRedundanciesOpt +} from './video-streaming-playlist.js' + +type Use = PickWith + +// ############################################################################ + +export type MVideo = + Omit + +// ############################################################################ + +export type MVideoId = Pick +export type MVideoUrl = Pick +export type MVideoUUID = Pick + +export type MVideoImmutable = Pick +export type MVideoIdUrl = MVideoId & MVideoUrl +export type MVideoFeed = Pick + +// ############################################################################ + +// Video raw associations: schedules, video files, tags, thumbnails, captions, streaming playlists, passwords + +// "With" to not confuse with the VideoFile model +export type MVideoWithFile = + MVideo & + Use<'VideoFiles', MVideoFile[]> & + Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]> + +export type MVideoThumbnail = + MVideo & + Use<'Thumbnails', MThumbnail[]> + +export type MVideoIdThumbnail = + MVideoId & + Use<'Thumbnails', MThumbnail[]> + +export type MVideoWithFileThumbnail = + MVideo & + Use<'VideoFiles', MVideoFile[]> & + Use<'Thumbnails', MThumbnail[]> + +export type MVideoThumbnailBlacklist = + MVideo & + Use<'Thumbnails', MThumbnail[]> & + Use<'VideoBlacklist', MVideoBlacklistLight> + +export type MVideoTag = + MVideo & + Use<'Tags', MTag[]> + +export type MVideoWithSchedule = + MVideo & + PickWithOpt + +export type MVideoWithCaptions = + MVideo & + Use<'VideoCaptions', MVideoCaptionLanguage[]> + +export type MVideoWithStreamingPlaylist = + MVideo & + Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]> + +// ############################################################################ + +// Associations with not all their attributes + +export type MVideoUserHistory = + MVideo & + Use<'UserVideoHistories', MUserVideoHistoryTime[]> + +export type MVideoWithBlacklistLight = + MVideo & + Use<'VideoBlacklist', MVideoBlacklistLight> + +export type MVideoAccountLight = + MVideo & + Use<'VideoChannel', MChannelAccountLight> + +export type MVideoWithRights = + MVideo & + Use<'VideoBlacklist', MVideoBlacklistLight> & + Use<'VideoChannel', MChannelUserId> + +// ############################################################################ + +// All files with some additional associations + +export type MVideoWithAllFiles = + MVideo & + Use<'VideoFiles', MVideoFile[]> & + Use<'Thumbnails', MThumbnail[]> & + Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]> + +export type MVideoAccountLightBlacklistAllFiles = + MVideo & + Use<'VideoFiles', MVideoFile[]> & + Use<'Thumbnails', MThumbnail[]> & + Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]> & + Use<'VideoChannel', MChannelAccountLight> & + Use<'VideoBlacklist', MVideoBlacklistLight> + +// ############################################################################ + +// With account + +export type MVideoAccountDefault = + MVideo & + Use<'VideoChannel', MChannelAccountDefault> + +export type MVideoThumbnailAccountDefault = + MVideo & + Use<'Thumbnails', MThumbnail[]> & + Use<'VideoChannel', MChannelAccountDefault> + +export type MVideoWithChannelActor = + MVideo & + Use<'VideoChannel', MChannelActor> + +export type MVideoWithHost = + MVideo & + Use<'VideoChannel', MChannelHostOnly> + +export type MVideoFullLight = + MVideo & + Use<'Thumbnails', MThumbnail[]> & + Use<'VideoBlacklist', MVideoBlacklistLight> & + Use<'Tags', MTag[]> & + Use<'VideoChannel', MChannelAccountLight> & + Use<'UserVideoHistories', MUserVideoHistoryTime[]> & + Use<'VideoFiles', MVideoFile[]> & + Use<'ScheduleVideoUpdate', MScheduleVideoUpdate> & + Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]> & + Use<'VideoLive', MVideoLive> + +// ############################################################################ + +// API + +export type MVideoAP = + MVideo & + Use<'Tags', MTag[]> & + Use<'VideoChannel', MChannelAccountLight> & + Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]> & + Use<'VideoCaptions', MVideoCaptionLanguageUrl[]> & + Use<'VideoBlacklist', MVideoBlacklistUnfederated> & + Use<'VideoFiles', MVideoFileRedundanciesOpt[]> & + Use<'Thumbnails', MThumbnail[]> & + Use<'VideoLive', MVideoLive> & + Use<'Storyboard', MStoryboard> + +export type MVideoAPLight = Omit + +export type MVideoDetails = + MVideo & + Use<'VideoBlacklist', MVideoBlacklistLight> & + Use<'Tags', MTag[]> & + Use<'VideoChannel', MChannelAccountLight> & + Use<'ScheduleVideoUpdate', MScheduleVideoUpdate> & + Use<'Thumbnails', MThumbnail[]> & + Use<'UserVideoHistories', MUserVideoHistoryTime[]> & + Use<'VideoStreamingPlaylists', MStreamingPlaylistRedundancies[]> & + Use<'VideoFiles', MVideoFileRedundanciesOpt[]> & + Use<'Trackers', MTrackerUrl[]> + +export type MVideoForUser = + MVideo & + Use<'VideoChannel', MChannelAccountDefault> & + Use<'ScheduleVideoUpdate', MScheduleVideoUpdate> & + Use<'VideoBlacklist', MVideoBlacklistLight> & + Use<'Thumbnails', MThumbnail[]> + +export type MVideoForRedundancyAPI = + MVideo & + Use<'VideoFiles', MVideoFileRedundanciesAll[]> & + Use<'VideoStreamingPlaylists', MStreamingPlaylistRedundanciesAll[]> + +// ############################################################################ + +// Format for API or AP object + +export type MVideoFormattable = + MVideo & + PickWithOpt & + Use<'VideoChannel', MChannelAccountSummaryFormattable> & + PickWithOpt> & + PickWithOpt> & + PickWithOpt & + PickWithOpt + +export type MVideoFormattableDetails = + MVideoFormattable & + Use<'VideoChannel', MChannelFormattable> & + Use<'Tags', MTag[]> & + Use<'VideoStreamingPlaylists', MStreamingPlaylistRedundanciesOpt[]> & + Use<'VideoFiles', MVideoFileRedundanciesOpt[]> & + PickWithOpt diff --git a/server/server/types/plugins/index.ts b/server/server/types/plugins/index.ts new file mode 100644 index 000000000..f50ac6fe9 --- /dev/null +++ b/server/server/types/plugins/index.ts @@ -0,0 +1,4 @@ +export * from './plugin-library.model.js' +export * from './register-server-auth.model.js' +export * from './register-server-option.model.js' +export * from './register-server-websocket-route.model.js' diff --git a/server/server/types/plugins/plugin-library.model.ts b/server/server/types/plugins/plugin-library.model.ts new file mode 100644 index 000000000..e8992ce85 --- /dev/null +++ b/server/server/types/plugins/plugin-library.model.ts @@ -0,0 +1,7 @@ +import { RegisterServerOptions } from './register-server-option.model.js' + +export interface PluginLibrary { + register: (options: RegisterServerOptions) => Promise + + unregister: () => Promise +} diff --git a/server/server/types/plugins/register-server-auth.model.ts b/server/server/types/plugins/register-server-auth.model.ts new file mode 100644 index 000000000..377682af6 --- /dev/null +++ b/server/server/types/plugins/register-server-auth.model.ts @@ -0,0 +1,72 @@ +import express from 'express' +import { UserAdminFlagType, UserRoleType } from '@peertube/peertube-models' +import { MOAuthToken, MUser } from '../models/index.js' + +export type RegisterServerAuthOptions = RegisterServerAuthPassOptions | RegisterServerAuthExternalOptions + +export type AuthenticatedResultUpdaterFieldName = 'displayName' | 'role' | 'adminFlags' | 'videoQuota' | 'videoQuotaDaily' + +export interface RegisterServerAuthenticatedResult { + // Update the user profile if it already exists + // Default behaviour is no update + // Introduced in PeerTube >= 5.1 + userUpdater?: (options: { + fieldName: AuthenticatedResultUpdaterFieldName + currentValue: T + newValue: T + }) => T + + username: string + email: string + role?: UserRoleType + displayName?: string + + // PeerTube >= 5.1 + adminFlags?: UserAdminFlagType + + // PeerTube >= 5.1 + videoQuota?: number + // PeerTube >= 5.1 + videoQuotaDaily?: number +} + +export interface RegisterServerExternalAuthenticatedResult extends RegisterServerAuthenticatedResult { + req: express.Request + res: express.Response +} + +interface RegisterServerAuthBase { + // Authentication name (a plugin can register multiple auth strategies) + authName: string + + // Called by PeerTube when a user from your plugin logged out + // Returns a redirectUrl sent to the client or nothing + onLogout?(user: MUser, req: express.Request): Promise + + // Your plugin can hook PeerTube access/refresh token validity + // So you can control for your plugin the user session lifetime + hookTokenValidity?(options: { token: MOAuthToken, type: 'access' | 'refresh' }): Promise<{ valid: boolean }> +} + +export interface RegisterServerAuthPassOptions extends RegisterServerAuthBase { + // Weight of this authentication so PeerTube tries the auth methods in DESC weight order + getWeight(): number + + // Used by PeerTube to login a user + // Returns null if the login failed, or { username, email } on success + login(body: { + id: string + password: string + }): Promise +} + +export interface RegisterServerAuthExternalOptions extends RegisterServerAuthBase { + // Will be displayed in a block next to the login form + authDisplayName: () => string + + onAuthRequest: (req: express.Request, res: express.Response) => void +} + +export interface RegisterServerAuthExternalResult { + userAuthenticated (options: RegisterServerExternalAuthenticatedResult): void +} diff --git a/server/server/types/plugins/register-server-option.model.ts b/server/server/types/plugins/register-server-option.model.ts new file mode 100644 index 000000000..e31aca1b8 --- /dev/null +++ b/server/server/types/plugins/register-server-option.model.ts @@ -0,0 +1,171 @@ +import { Response, Router } from 'express' +import { Server } from 'http' +import { Logger } from 'winston' +import { + PluginPlaylistPrivacyManager, + PluginSettingsManager, + PluginStorageManager, + PluginTranscodingManager, + PluginVideoCategoryManager, + PluginVideoLanguageManager, + PluginVideoLicenceManager, + PluginVideoPrivacyManager, + RegisterServerHookOptions, + RegisterServerSettingOptions, + ServerConfig, + ThumbnailType_Type, + VideoBlacklistCreate +} from '@peertube/peertube-models' +import { ActorModel } from '@server/models/actor/actor.js' +import { MUserDefault, MVideo, MVideoThumbnail, UserNotificationModelForApi } from '../models/index.js' +import { + RegisterServerAuthExternalOptions, + RegisterServerAuthExternalResult, + RegisterServerAuthPassOptions +} from './register-server-auth.model.js' +import { RegisterServerWebSocketRouteOptions } from './register-server-websocket-route.model.js' + +export type PeerTubeHelpers = { + logger: Logger + + database: { + query: Function + } + + videos: { + loadByUrl: (url: string) => Promise + loadByIdOrUUID: (id: number | string) => Promise + + removeVideo: (videoId: number) => Promise + + ffprobe: (path: string) => Promise + + getFiles: (id: number | string) => Promise<{ + webtorrent: { // TODO: remove in v7 + videoFiles: { + path: string // Could be null if using remote storage + url: string + resolution: number + size: number + fps: number + }[] + } + + webVideo: { + videoFiles: { + path: string // Could be null if using remote storage + url: string + resolution: number + size: number + fps: number + }[] + } + + hls: { + videoFiles: { + path: string // Could be null if using remote storage + url: string + resolution: number + size: number + fps: number + }[] + } + + thumbnails: { + type: ThumbnailType_Type + path: string + }[] + }> + } + + config: { + getWebserverUrl: () => string + + // PeerTube >= 5.1 + getServerListeningConfig: () => { hostname: string, port: number } + + getServerConfig: () => Promise + } + + moderation: { + blockServer: (options: { byAccountId: number, hostToBlock: string }) => Promise + unblockServer: (options: { byAccountId: number, hostToUnblock: string }) => Promise + blockAccount: (options: { byAccountId: number, handleToBlock: string }) => Promise + unblockAccount: (options: { byAccountId: number, handleToUnblock: string }) => Promise + + blacklistVideo: (options: { videoIdOrUUID: number | string, createOptions: VideoBlacklistCreate }) => Promise + unblacklistVideo: (options: { videoIdOrUUID: number | string }) => Promise + } + + server: { + // PeerTube >= 5.0 + getHTTPServer: () => Server + + getServerActor: () => Promise + } + + socket: { + sendNotification: (userId: number, notification: UserNotificationModelForApi) => void + sendVideoLiveNewState: (video: MVideo) => void + } + + plugin: { + // PeerTube >= 3.2 + getBaseStaticRoute: () => string + + // PeerTube >= 3.2 + getBaseRouterRoute: () => string + // PeerTube >= 5.0 + getBaseWebSocketRoute: () => string + + // PeerTube >= 3.2 + getDataDirectoryPath: () => string + } + + user: { + // PeerTube >= 3.2 + getAuthUser: (response: Response) => Promise + + // PeerTube >= 4.3 + loadById: (id: number) => Promise + } +} + +export type RegisterServerOptions = { + registerHook: (options: RegisterServerHookOptions) => void + + registerSetting: (options: RegisterServerSettingOptions) => void + + settingsManager: PluginSettingsManager + + storageManager: PluginStorageManager + + videoCategoryManager: PluginVideoCategoryManager + videoLanguageManager: PluginVideoLanguageManager + videoLicenceManager: PluginVideoLicenceManager + + videoPrivacyManager: PluginVideoPrivacyManager + playlistPrivacyManager: PluginPlaylistPrivacyManager + + transcodingManager: PluginTranscodingManager + + registerIdAndPassAuth: (options: RegisterServerAuthPassOptions) => void + registerExternalAuth: (options: RegisterServerAuthExternalOptions) => RegisterServerAuthExternalResult + unregisterIdAndPassAuth: (authName: string) => void + unregisterExternalAuth: (authName: string) => void + + // Get plugin router to create custom routes + // Base routes of this router are + // * /plugins/:pluginName/:pluginVersion/router/... + // * /plugins/:pluginName/router/... + getRouter(): Router + + // PeerTube >= 5.0 + // Register WebSocket route + // Base routes of the WebSocket router are + // * /plugins/:pluginName/:pluginVersion/ws/... + // * /plugins/:pluginName/ws/... + registerWebSocketRoute: (options: RegisterServerWebSocketRouteOptions) => void + + peertubeHelpers: PeerTubeHelpers +} diff --git a/server/types/plugins/register-server-websocket-route.model.ts b/server/server/types/plugins/register-server-websocket-route.model.ts similarity index 100% rename from server/types/plugins/register-server-websocket-route.model.ts rename to server/server/types/plugins/register-server-websocket-route.model.ts diff --git a/server/server/types/sequelize.ts b/server/server/types/sequelize.ts new file mode 100644 index 000000000..cf0d040ae --- /dev/null +++ b/server/server/types/sequelize.ts @@ -0,0 +1,19 @@ +import { AttributesOnly } from '@peertube/peertube-typescript-utils' +import { Model } from 'sequelize' + +// Thanks to sequelize-typescript: https://github.com/RobinBuschmann/sequelize-typescript + +export type Diff = + ({ [P in T]: P } & { [P in U]: never } & { [ x: string ]: never })[T] + +export type Omit = { [P in Diff]: T[P] } + +export type RecursivePartial = { [P in keyof T]?: RecursivePartial } + +export type FilteredModelAttributes> = Partial> & { + id?: number | any + createdAt?: Date | any + updatedAt?: Date | any + deletedAt?: Date | any + version?: number | any +} diff --git a/server/tests/api/activitypub/cleaner.ts b/server/tests/api/activitypub/cleaner.ts deleted file mode 100644 index d67175e20..000000000 --- a/server/tests/api/activitypub/cleaner.ts +++ /dev/null @@ -1,342 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { SQLCommand } from '@server/tests/shared' -import { wait } from '@shared/core-utils' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test AP cleaner', function () { - let servers: PeerTubeServer[] = [] - const sqlCommands: SQLCommand[] = [] - - let videoUUID1: string - let videoUUID2: string - let videoUUID3: string - - let videoUUIDs: string[] - - before(async function () { - this.timeout(120000) - - const config = { - federation: { - videos: { cleanup_remote_interactions: true } - } - } - servers = await createMultipleServers(3, config) - - // Get the access tokens - await setAccessTokensToServers(servers) - - await Promise.all([ - doubleFollow(servers[0], servers[1]), - doubleFollow(servers[1], servers[2]), - doubleFollow(servers[0], servers[2]) - ]) - - // Update 1 local share, check 6 shares - - // Create 1 comment per video - // Update 1 remote URL and 1 local URL on - - videoUUID1 = (await servers[0].videos.quickUpload({ name: 'server 1' })).uuid - videoUUID2 = (await servers[1].videos.quickUpload({ name: 'server 2' })).uuid - videoUUID3 = (await servers[2].videos.quickUpload({ name: 'server 3' })).uuid - - videoUUIDs = [ videoUUID1, videoUUID2, videoUUID3 ] - - await waitJobs(servers) - - for (const server of servers) { - for (const uuid of videoUUIDs) { - await server.videos.rate({ id: uuid, rating: 'like' }) - await server.comments.createThread({ videoId: uuid, text: 'comment' }) - } - - sqlCommands.push(new SQLCommand(server)) - } - - await waitJobs(servers) - }) - - it('Should have the correct likes', async function () { - for (const server of servers) { - for (const uuid of videoUUIDs) { - const video = await server.videos.get({ id: uuid }) - - expect(video.likes).to.equal(3) - expect(video.dislikes).to.equal(0) - } - } - }) - - it('Should destroy server 3 internal likes and correctly clean them', async function () { - this.timeout(20000) - - await sqlCommands[2].deleteAll('accountVideoRate') - for (const uuid of videoUUIDs) { - await sqlCommands[2].setVideoField(uuid, 'likes', '0') - } - - await wait(5000) - await waitJobs(servers) - - // Updated rates of my video - { - const video = await servers[0].videos.get({ id: videoUUID1 }) - expect(video.likes).to.equal(2) - expect(video.dislikes).to.equal(0) - } - - // Did not update rates of a remote video - { - const video = await servers[0].videos.get({ id: videoUUID2 }) - expect(video.likes).to.equal(3) - expect(video.dislikes).to.equal(0) - } - }) - - it('Should update rates to dislikes', async function () { - this.timeout(20000) - - for (const server of servers) { - for (const uuid of videoUUIDs) { - await server.videos.rate({ id: uuid, rating: 'dislike' }) - } - } - - await waitJobs(servers) - - for (const server of servers) { - for (const uuid of videoUUIDs) { - const video = await server.videos.get({ id: uuid }) - expect(video.likes).to.equal(0) - expect(video.dislikes).to.equal(3) - } - } - }) - - it('Should destroy server 3 internal dislikes and correctly clean them', async function () { - this.timeout(20000) - - await sqlCommands[2].deleteAll('accountVideoRate') - - for (const uuid of videoUUIDs) { - await sqlCommands[2].setVideoField(uuid, 'dislikes', '0') - } - - await wait(5000) - await waitJobs(servers) - - // Updated rates of my video - { - const video = await servers[0].videos.get({ id: videoUUID1 }) - expect(video.likes).to.equal(0) - expect(video.dislikes).to.equal(2) - } - - // Did not update rates of a remote video - { - const video = await servers[0].videos.get({ id: videoUUID2 }) - expect(video.likes).to.equal(0) - expect(video.dislikes).to.equal(3) - } - }) - - it('Should destroy server 3 internal shares and correctly clean them', async function () { - this.timeout(20000) - - const preCount = await sqlCommands[0].getVideoShareCount() - expect(preCount).to.equal(6) - - await sqlCommands[2].deleteAll('videoShare') - await wait(5000) - await waitJobs(servers) - - // Still 6 because we don't have remote shares on local videos - const postCount = await sqlCommands[0].getVideoShareCount() - expect(postCount).to.equal(6) - }) - - it('Should destroy server 3 internal comments and correctly clean them', async function () { - this.timeout(20000) - - { - const { total } = await servers[0].comments.listThreads({ videoId: videoUUID1 }) - expect(total).to.equal(3) - } - - await sqlCommands[2].deleteAll('videoComment') - - await wait(5000) - await waitJobs(servers) - - { - const { total } = await servers[0].comments.listThreads({ videoId: videoUUID1 }) - expect(total).to.equal(2) - } - }) - - it('Should correctly update rate URLs', async function () { - this.timeout(30000) - - async function check (like: string, ofServerUrl: string, urlSuffix: string, remote: 'true' | 'false') { - const query = `SELECT "videoId", "accountVideoRate".url FROM "accountVideoRate" ` + - `INNER JOIN video ON "accountVideoRate"."videoId" = video.id AND remote IS ${remote} WHERE "accountVideoRate"."url" LIKE '${like}'` - const res = await sqlCommands[0].selectQuery<{ url: string }>(query) - - for (const rate of res) { - const matcher = new RegExp(`^${ofServerUrl}/accounts/root/dislikes/\\d+${urlSuffix}$`) - expect(rate.url).to.match(matcher) - } - } - - async function checkLocal () { - const startsWith = 'http://' + servers[0].host + '%' - // On local videos - await check(startsWith, servers[0].url, '', 'false') - // On remote videos - await check(startsWith, servers[0].url, '', 'true') - } - - async function checkRemote (suffix: string) { - const startsWith = 'http://' + servers[1].host + '%' - // On local videos - await check(startsWith, servers[1].url, suffix, 'false') - // On remote videos, we should not update URLs so no suffix - await check(startsWith, servers[1].url, '', 'true') - } - - await checkLocal() - await checkRemote('') - - { - const query = `UPDATE "accountVideoRate" SET url = url || 'stan'` - await sqlCommands[1].updateQuery(query) - - await wait(5000) - await waitJobs(servers) - } - - await checkLocal() - await checkRemote('stan') - }) - - it('Should correctly update comment URLs', async function () { - this.timeout(30000) - - async function check (like: string, ofServerUrl: string, urlSuffix: string, remote: 'true' | 'false') { - const query = `SELECT "videoId", "videoComment".url, uuid as "videoUUID" FROM "videoComment" ` + - `INNER JOIN video ON "videoComment"."videoId" = video.id AND remote IS ${remote} WHERE "videoComment"."url" LIKE '${like}'` - - const res = await sqlCommands[0].selectQuery<{ url: string, videoUUID: string }>(query) - - for (const comment of res) { - const matcher = new RegExp(`${ofServerUrl}/videos/watch/${comment.videoUUID}/comments/\\d+${urlSuffix}`) - expect(comment.url).to.match(matcher) - } - } - - async function checkLocal () { - const startsWith = 'http://' + servers[0].host + '%' - // On local videos - await check(startsWith, servers[0].url, '', 'false') - // On remote videos - await check(startsWith, servers[0].url, '', 'true') - } - - async function checkRemote (suffix: string) { - const startsWith = 'http://' + servers[1].host + '%' - // On local videos - await check(startsWith, servers[1].url, suffix, 'false') - // On remote videos, we should not update URLs so no suffix - await check(startsWith, servers[1].url, '', 'true') - } - - { - const query = `UPDATE "videoComment" SET url = url || 'kyle'` - await sqlCommands[1].updateQuery(query) - - await wait(5000) - await waitJobs(servers) - } - - await checkLocal() - await checkRemote('kyle') - }) - - it('Should remove unavailable remote resources', async function () { - this.timeout(240000) - - async function expectNotDeleted () { - { - const video = await servers[0].videos.get({ id: uuid }) - - expect(video.likes).to.equal(3) - expect(video.dislikes).to.equal(0) - } - - { - const { total } = await servers[0].comments.listThreads({ videoId: uuid }) - expect(total).to.equal(3) - } - } - - async function expectDeleted () { - { - const video = await servers[0].videos.get({ id: uuid }) - - expect(video.likes).to.equal(2) - expect(video.dislikes).to.equal(0) - } - - { - const { total } = await servers[0].comments.listThreads({ videoId: uuid }) - expect(total).to.equal(2) - } - } - - const uuid = (await servers[0].videos.quickUpload({ name: 'server 1 video 2' })).uuid - - await waitJobs(servers) - - for (const server of servers) { - await server.videos.rate({ id: uuid, rating: 'like' }) - await server.comments.createThread({ videoId: uuid, text: 'comment' }) - } - - await waitJobs(servers) - - await expectNotDeleted() - - await servers[1].kill() - - await wait(5000) - await expectNotDeleted() - - let continueWhile = true - - do { - try { - await expectDeleted() - continueWhile = false - } catch { - } - } while (continueWhile) - }) - - after(async function () { - for (const sql of sqlCommands) { - await sql.cleanup() - } - - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/activitypub/client.ts b/server/tests/api/activitypub/client.ts deleted file mode 100644 index 572a358a0..000000000 --- a/server/tests/api/activitypub/client.ts +++ /dev/null @@ -1,136 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { processViewersStats } from '@server/tests/shared' -import { HttpStatusCode, VideoPlaylistPrivacy, WatchActionObject } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - makeActivityPubGetRequest, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel -} from '@shared/server-commands' - -describe('Test activitypub', function () { - let servers: PeerTubeServer[] = [] - let video: { id: number, uuid: string, shortUUID: string } - let playlist: { id: number, uuid: string, shortUUID: string } - - async function testAccount (path: string) { - const res = await makeActivityPubGetRequest(servers[0].url, path) - const object = res.body - - expect(object.type).to.equal('Person') - expect(object.id).to.equal(servers[0].url + '/accounts/root') - expect(object.name).to.equal('root') - expect(object.preferredUsername).to.equal('root') - } - - async function testChannel (path: string) { - const res = await makeActivityPubGetRequest(servers[0].url, path) - const object = res.body - - expect(object.type).to.equal('Group') - expect(object.id).to.equal(servers[0].url + '/video-channels/root_channel') - expect(object.name).to.equal('Main root channel') - expect(object.preferredUsername).to.equal('root_channel') - } - - async function testVideo (path: string) { - const res = await makeActivityPubGetRequest(servers[0].url, path) - const object = res.body - - expect(object.type).to.equal('Video') - expect(object.id).to.equal(servers[0].url + '/videos/watch/' + video.uuid) - expect(object.name).to.equal('video') - } - - async function testPlaylist (path: string) { - const res = await makeActivityPubGetRequest(servers[0].url, path) - const object = res.body - - expect(object.type).to.equal('Playlist') - expect(object.id).to.equal(servers[0].url + '/video-playlists/' + playlist.uuid) - expect(object.name).to.equal('playlist') - } - - before(async function () { - this.timeout(30000) - - servers = await createMultipleServers(2) - - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - - { - video = await servers[0].videos.quickUpload({ name: 'video' }) - } - - { - const attributes = { displayName: 'playlist', privacy: VideoPlaylistPrivacy.PUBLIC, videoChannelId: servers[0].store.channel.id } - playlist = await servers[0].playlists.create({ attributes }) - } - - await doubleFollow(servers[0], servers[1]) - }) - - it('Should return the account object', async function () { - await testAccount('/accounts/root') - await testAccount('/a/root') - }) - - it('Should return the channel object', async function () { - await testChannel('/video-channels/root_channel') - await testChannel('/c/root_channel') - }) - - it('Should return the video object', async function () { - await testVideo('/videos/watch/' + video.id) - await testVideo('/videos/watch/' + video.uuid) - await testVideo('/videos/watch/' + video.shortUUID) - await testVideo('/w/' + video.id) - await testVideo('/w/' + video.uuid) - await testVideo('/w/' + video.shortUUID) - }) - - it('Should return the playlist object', async function () { - await testPlaylist('/video-playlists/' + playlist.id) - await testPlaylist('/video-playlists/' + playlist.uuid) - await testPlaylist('/video-playlists/' + playlist.shortUUID) - await testPlaylist('/w/p/' + playlist.id) - await testPlaylist('/w/p/' + playlist.uuid) - await testPlaylist('/w/p/' + playlist.shortUUID) - await testPlaylist('/videos/watch/playlist/' + playlist.id) - await testPlaylist('/videos/watch/playlist/' + playlist.uuid) - await testPlaylist('/videos/watch/playlist/' + playlist.shortUUID) - }) - - it('Should redirect to the origin video object', async function () { - const res = await makeActivityPubGetRequest(servers[1].url, '/videos/watch/' + video.uuid, HttpStatusCode.FOUND_302) - - expect(res.header.location).to.equal(servers[0].url + '/videos/watch/' + video.uuid) - }) - - it('Should return the watch action', async function () { - this.timeout(50000) - - await servers[0].views.simulateViewer({ id: video.uuid, currentTimes: [ 0, 2 ] }) - await processViewersStats(servers) - - const res = await makeActivityPubGetRequest(servers[0].url, '/videos/local-viewer/1', HttpStatusCode.OK_200) - - const object: WatchActionObject = res.body - expect(object.type).to.equal('WatchAction') - expect(object.duration).to.equal('PT2S') - expect(object.actionStatus).to.equal('CompletedActionStatus') - expect(object.watchSections).to.have.lengthOf(1) - expect(object.watchSections[0].startTimestamp).to.equal(0) - expect(object.watchSections[0].endTimestamp).to.equal(2) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/activitypub/fetch.ts b/server/tests/api/activitypub/fetch.ts deleted file mode 100644 index 3899a6a49..000000000 --- a/server/tests/api/activitypub/fetch.ts +++ /dev/null @@ -1,82 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { SQLCommand } from '@server/tests/shared' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test ActivityPub fetcher', function () { - let servers: PeerTubeServer[] - let sqlCommandServer1: SQLCommand - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(60000) - - servers = await createMultipleServers(3) - - // Get the access tokens - await setAccessTokensToServers(servers) - - const user = { username: 'user1', password: 'password' } - for (const server of servers) { - await server.users.create({ username: user.username, password: user.password }) - } - - const userAccessToken = await servers[0].login.getAccessToken(user) - - await servers[0].videos.upload({ attributes: { name: 'video root' } }) - const { uuid } = await servers[0].videos.upload({ attributes: { name: 'bad video root' } }) - await servers[0].videos.upload({ token: userAccessToken, attributes: { name: 'video user' } }) - - sqlCommandServer1 = new SQLCommand(servers[0]) - - { - const to = servers[0].url + '/accounts/user1' - const value = servers[1].url + '/accounts/user1' - await sqlCommandServer1.setActorField(to, 'url', value) - } - - { - const value = servers[2].url + '/videos/watch/' + uuid - await sqlCommandServer1.setVideoField(uuid, 'url', value) - } - }) - - it('Should add only the video with a valid actor URL', async function () { - this.timeout(60000) - - await doubleFollow(servers[0], servers[1]) - await waitJobs(servers) - - { - const { total, data } = await servers[0].videos.list({ sort: 'createdAt' }) - - expect(total).to.equal(3) - expect(data[0].name).to.equal('video root') - expect(data[1].name).to.equal('bad video root') - expect(data[2].name).to.equal('video user') - } - - { - const { total, data } = await servers[1].videos.list({ sort: 'createdAt' }) - - expect(total).to.equal(1) - expect(data[0].name).to.equal('video root') - } - }) - - after(async function () { - this.timeout(20000) - - await sqlCommandServer1.cleanup() - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/activitypub/helpers.ts b/server/tests/api/activitypub/helpers.ts deleted file mode 100644 index bad86ef47..000000000 --- a/server/tests/api/activitypub/helpers.ts +++ /dev/null @@ -1,167 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { cloneDeep } from 'lodash' -import { signAndContextify } from '@server/lib/activitypub/send' -import { buildRequestStub } from '@server/tests/shared' -import { buildAbsoluteFixturePath } from '@shared/core-utils' -import { isHTTPSignatureVerified, isJsonLDSignatureVerified, parseHTTPSignature } from '../../../helpers/peertube-crypto' - -describe('Test activity pub helpers', function () { - - describe('When checking the Linked Signature', function () { - - it('Should fail with an invalid Mastodon signature', async function () { - const body = require(buildAbsoluteFixturePath('./ap-json/mastodon/create-bad-signature.json')) - const publicKey = require(buildAbsoluteFixturePath('./ap-json/mastodon/public-key.json')).publicKey - const fromActor = { publicKey, url: 'http://localhost:9002/accounts/peertube' } - - const result = await isJsonLDSignatureVerified(fromActor as any, body) - - expect(result).to.be.false - }) - - it('Should fail with an invalid public key', async function () { - const body = require(buildAbsoluteFixturePath('./ap-json/mastodon/create.json')) - const publicKey = require(buildAbsoluteFixturePath('./ap-json/mastodon/bad-public-key.json')).publicKey - const fromActor = { publicKey, url: 'http://localhost:9002/accounts/peertube' } - - const result = await isJsonLDSignatureVerified(fromActor as any, body) - - expect(result).to.be.false - }) - - it('Should succeed with a valid Mastodon signature', async function () { - const body = require(buildAbsoluteFixturePath('./ap-json/mastodon/create.json')) - const publicKey = require(buildAbsoluteFixturePath('./ap-json/mastodon/public-key.json')).publicKey - const fromActor = { publicKey, url: 'http://localhost:9002/accounts/peertube' } - - const result = await isJsonLDSignatureVerified(fromActor as any, body) - - expect(result).to.be.true - }) - - it('Should fail with an invalid PeerTube signature', async function () { - const keys = require(buildAbsoluteFixturePath('./ap-json/peertube/invalid-keys.json')) - const body = require(buildAbsoluteFixturePath('./ap-json/peertube/announce-without-context.json')) - - const actorSignature = { url: 'http://localhost:9002/accounts/peertube', privateKey: keys.privateKey } - const signedBody = await signAndContextify(actorSignature as any, body, 'Announce') - - const fromActor = { publicKey: keys.publicKey, url: 'http://localhost:9002/accounts/peertube' } - const result = await isJsonLDSignatureVerified(fromActor as any, signedBody) - - expect(result).to.be.false - }) - - it('Should succeed with a valid PeerTube signature', async function () { - const keys = require(buildAbsoluteFixturePath('./ap-json/peertube/keys.json')) - const body = require(buildAbsoluteFixturePath('./ap-json/peertube/announce-without-context.json')) - - const actorSignature = { url: 'http://localhost:9002/accounts/peertube', privateKey: keys.privateKey } - const signedBody = await signAndContextify(actorSignature as any, body, 'Announce') - - const fromActor = { publicKey: keys.publicKey, url: 'http://localhost:9002/accounts/peertube' } - const result = await isJsonLDSignatureVerified(fromActor as any, signedBody) - - expect(result).to.be.true - }) - }) - - describe('When checking HTTP signature', function () { - it('Should fail with an invalid http signature', async function () { - const req = buildRequestStub() - req.method = 'POST' - req.url = '/accounts/ronan/inbox' - - const mastodonObject = cloneDeep(require(buildAbsoluteFixturePath('./ap-json/mastodon/bad-http-signature.json'))) - req.body = mastodonObject.body - req.headers = mastodonObject.headers - - const parsed = parseHTTPSignature(req, 3600 * 1000 * 365 * 10) - const publicKey = require(buildAbsoluteFixturePath('./ap-json/mastodon/public-key.json')).publicKey - - const actor = { publicKey } - const verified = isHTTPSignatureVerified(parsed, actor as any) - - expect(verified).to.be.false - }) - - it('Should fail with an invalid public key', async function () { - const req = buildRequestStub() - req.method = 'POST' - req.url = '/accounts/ronan/inbox' - - const mastodonObject = cloneDeep(require(buildAbsoluteFixturePath('./ap-json/mastodon/http-signature.json'))) - req.body = mastodonObject.body - req.headers = mastodonObject.headers - - const parsed = parseHTTPSignature(req, 3600 * 1000 * 365 * 10) - const publicKey = require(buildAbsoluteFixturePath('./ap-json/mastodon/bad-public-key.json')).publicKey - - const actor = { publicKey } - const verified = isHTTPSignatureVerified(parsed, actor as any) - - expect(verified).to.be.false - }) - - it('Should fail because of clock skew', async function () { - const req = buildRequestStub() - req.method = 'POST' - req.url = '/accounts/ronan/inbox' - - const mastodonObject = cloneDeep(require(buildAbsoluteFixturePath('./ap-json/mastodon/http-signature.json'))) - req.body = mastodonObject.body - req.headers = mastodonObject.headers - - let errored = false - try { - parseHTTPSignature(req) - } catch { - errored = true - } - - expect(errored).to.be.true - }) - - it('Should with a scheme', async function () { - const req = buildRequestStub() - req.method = 'POST' - req.url = '/accounts/ronan/inbox' - - const mastodonObject = cloneDeep(require(buildAbsoluteFixturePath('./ap-json/mastodon/http-signature.json'))) - req.body = mastodonObject.body - req.headers = mastodonObject.headers - req.headers = 'Signature ' + mastodonObject.headers - - let errored = false - try { - parseHTTPSignature(req, 3600 * 1000 * 365 * 10) - } catch { - errored = true - } - - expect(errored).to.be.true - }) - - it('Should succeed with a valid signature', async function () { - const req = buildRequestStub() - req.method = 'POST' - req.url = '/accounts/ronan/inbox' - - const mastodonObject = cloneDeep(require(buildAbsoluteFixturePath('./ap-json/mastodon/http-signature.json'))) - req.body = mastodonObject.body - req.headers = mastodonObject.headers - - const parsed = parseHTTPSignature(req, 3600 * 1000 * 365 * 10) - const publicKey = require(buildAbsoluteFixturePath('./ap-json/mastodon/public-key.json')).publicKey - - const actor = { publicKey } - const verified = isHTTPSignatureVerified(parsed, actor as any) - - expect(verified).to.be.true - }) - - }) - -}) diff --git a/server/tests/api/activitypub/index.ts b/server/tests/api/activitypub/index.ts deleted file mode 100644 index 324b444e4..000000000 --- a/server/tests/api/activitypub/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -import './cleaner' -import './client' -import './fetch' -import './refresher' -import './helpers' -import './security' diff --git a/server/tests/api/activitypub/refresher.ts b/server/tests/api/activitypub/refresher.ts deleted file mode 100644 index 4ea7929ec..000000000 --- a/server/tests/api/activitypub/refresher.ts +++ /dev/null @@ -1,157 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { SQLCommand } from '@server/tests/shared' -import { wait } from '@shared/core-utils' -import { HttpStatusCode, VideoPlaylistPrivacy } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - killallServers, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - waitJobs -} from '@shared/server-commands' - -describe('Test AP refresher', function () { - let servers: PeerTubeServer[] = [] - let sqlCommandServer2: SQLCommand - let videoUUID1: string - let videoUUID2: string - let videoUUID3: string - let playlistUUID1: string - let playlistUUID2: string - - before(async function () { - this.timeout(60000) - - servers = await createMultipleServers(2) - - // Get the access tokens - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - - for (const server of servers) { - await server.config.disableTranscoding() - } - - { - videoUUID1 = (await servers[1].videos.quickUpload({ name: 'video1' })).uuid - videoUUID2 = (await servers[1].videos.quickUpload({ name: 'video2' })).uuid - videoUUID3 = (await servers[1].videos.quickUpload({ name: 'video3' })).uuid - } - - { - const token1 = await servers[1].users.generateUserAndToken('user1') - await servers[1].videos.upload({ token: token1, attributes: { name: 'video4' } }) - - const token2 = await servers[1].users.generateUserAndToken('user2') - await servers[1].videos.upload({ token: token2, attributes: { name: 'video5' } }) - } - - { - const attributes = { displayName: 'playlist1', privacy: VideoPlaylistPrivacy.PUBLIC, videoChannelId: servers[1].store.channel.id } - const created = await servers[1].playlists.create({ attributes }) - playlistUUID1 = created.uuid - } - - { - const attributes = { displayName: 'playlist2', privacy: VideoPlaylistPrivacy.PUBLIC, videoChannelId: servers[1].store.channel.id } - const created = await servers[1].playlists.create({ attributes }) - playlistUUID2 = created.uuid - } - - await doubleFollow(servers[0], servers[1]) - - sqlCommandServer2 = new SQLCommand(servers[1]) - }) - - describe('Videos refresher', function () { - - it('Should remove a deleted remote video', async function () { - this.timeout(60000) - - await wait(10000) - - // Change UUID so the remote server returns a 404 - await sqlCommandServer2.setVideoField(videoUUID1, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174f') - - await servers[0].videos.get({ id: videoUUID1 }) - await servers[0].videos.get({ id: videoUUID2 }) - - await waitJobs(servers) - - await servers[0].videos.get({ id: videoUUID1, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - await servers[0].videos.get({ id: videoUUID2 }) - }) - - it('Should not update a remote video if the remote instance is down', async function () { - this.timeout(70000) - - await killallServers([ servers[1] ]) - - await sqlCommandServer2.setVideoField(videoUUID3, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b174e') - - // Video will need a refresh - await wait(10000) - - await servers[0].videos.get({ id: videoUUID3 }) - // The refresh should fail - await waitJobs([ servers[0] ]) - - await servers[1].run() - - await servers[0].videos.get({ id: videoUUID3 }) - }) - }) - - describe('Actors refresher', function () { - - it('Should remove a deleted actor', async function () { - this.timeout(60000) - - const command = servers[0].accounts - - await wait(10000) - - // Change actor name so the remote server returns a 404 - const to = servers[1].url + '/accounts/user2' - await sqlCommandServer2.setActorField(to, 'preferredUsername', 'toto') - - await command.get({ accountName: 'user1@' + servers[1].host }) - await command.get({ accountName: 'user2@' + servers[1].host }) - - await waitJobs(servers) - - await command.get({ accountName: 'user1@' + servers[1].host, expectedStatus: HttpStatusCode.OK_200 }) - await command.get({ accountName: 'user2@' + servers[1].host, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - }) - - describe('Playlist refresher', function () { - - it('Should remove a deleted playlist', async function () { - this.timeout(60000) - - await wait(10000) - - // Change UUID so the remote server returns a 404 - await sqlCommandServer2.setPlaylistField(playlistUUID2, 'uuid', '304afe4f-39f9-4d49-8ed7-ac57b86b178e') - - await servers[0].playlists.get({ playlistId: playlistUUID1 }) - await servers[0].playlists.get({ playlistId: playlistUUID2 }) - - await waitJobs(servers) - - await servers[0].playlists.get({ playlistId: playlistUUID1, expectedStatus: HttpStatusCode.OK_200 }) - await servers[0].playlists.get({ playlistId: playlistUUID2, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - }) - - after(async function () { - await sqlCommandServer2.cleanup() - - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/activitypub/security.ts b/server/tests/api/activitypub/security.ts deleted file mode 100644 index 8e87361a9..000000000 --- a/server/tests/api/activitypub/security.ts +++ /dev/null @@ -1,321 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { buildDigest } from '@server/helpers/peertube-crypto' -import { ACTIVITY_PUB, HTTP_SIGNATURE } from '@server/initializers/constants' -import { activityPubContextify } from '@server/lib/activitypub/context' -import { buildGlobalHeaders, signAndContextify } from '@server/lib/activitypub/send' -import { makePOSTAPRequest, SQLCommand } from '@server/tests/shared' -import { buildAbsoluteFixturePath, wait } from '@shared/core-utils' -import { HttpStatusCode } from '@shared/models' -import { cleanupTests, createMultipleServers, killallServers, PeerTubeServer } from '@shared/server-commands' - -function setKeysOfServer (onServer: SQLCommand, ofServerUrl: string, publicKey: string, privateKey: string) { - const url = ofServerUrl + '/accounts/peertube' - - return Promise.all([ - onServer.setActorField(url, 'publicKey', publicKey), - onServer.setActorField(url, 'privateKey', privateKey) - ]) -} - -function setUpdatedAtOfServer (onServer: SQLCommand, ofServerUrl: string, updatedAt: string) { - const url = ofServerUrl + '/accounts/peertube' - - return Promise.all([ - onServer.setActorField(url, 'createdAt', updatedAt), - onServer.setActorField(url, 'updatedAt', updatedAt) - ]) -} - -function getAnnounceWithoutContext (server: PeerTubeServer) { - const json = require(buildAbsoluteFixturePath('./ap-json/peertube/announce-without-context.json')) - const result: typeof json = {} - - for (const key of Object.keys(json)) { - if (Array.isArray(json[key])) { - result[key] = json[key].map(v => v.replace(':9002', `:${server.port}`)) - } else { - result[key] = json[key].replace(':9002', `:${server.port}`) - } - } - - return result -} - -async function makeFollowRequest (to: { url: string }, by: { url: string, privateKey }) { - const follow = { - type: 'Follow', - id: by.url + '/' + new Date().getTime(), - actor: by.url, - object: to.url - } - - const body = await activityPubContextify(follow, 'Follow') - - const httpSignature = { - algorithm: HTTP_SIGNATURE.ALGORITHM, - authorizationHeaderName: HTTP_SIGNATURE.HEADER_NAME, - keyId: by.url, - key: by.privateKey, - headers: HTTP_SIGNATURE.HEADERS_TO_SIGN_WITH_PAYLOAD - } - const headers = { - 'digest': buildDigest(body), - 'content-type': 'application/activity+json', - 'accept': ACTIVITY_PUB.ACCEPT_HEADER - } - - return makePOSTAPRequest(to.url + '/inbox', body, httpSignature, headers) -} - -describe('Test ActivityPub security', function () { - let servers: PeerTubeServer[] - let sqlCommands: SQLCommand[] = [] - - let url: string - - const keys = require(buildAbsoluteFixturePath('./ap-json/peertube/keys.json')) - const invalidKeys = require(buildAbsoluteFixturePath('./ap-json/peertube/invalid-keys.json')) - const baseHttpSignature = () => ({ - algorithm: HTTP_SIGNATURE.ALGORITHM, - authorizationHeaderName: HTTP_SIGNATURE.HEADER_NAME, - keyId: 'acct:peertube@' + servers[1].host, - key: keys.privateKey, - headers: HTTP_SIGNATURE.HEADERS_TO_SIGN_WITH_PAYLOAD - }) - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(60000) - - servers = await createMultipleServers(3) - - sqlCommands = servers.map(s => new SQLCommand(s)) - - url = servers[0].url + '/inbox' - - await setKeysOfServer(sqlCommands[0], servers[1].url, keys.publicKey, null) - await setKeysOfServer(sqlCommands[1], servers[1].url, keys.publicKey, keys.privateKey) - - const to = { url: servers[0].url + '/accounts/peertube' } - const by = { url: servers[1].url + '/accounts/peertube', privateKey: keys.privateKey } - await makeFollowRequest(to, by) - }) - - describe('When checking HTTP signature', function () { - - it('Should fail with an invalid digest', async function () { - const body = await activityPubContextify(getAnnounceWithoutContext(servers[1]), 'Announce') - const headers = { - Digest: buildDigest({ hello: 'coucou' }) - } - - try { - await makePOSTAPRequest(url, body, baseHttpSignature(), headers) - expect(true, 'Did not throw').to.be.false - } catch (err) { - expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) - } - }) - - it('Should fail with an invalid date', async function () { - const body = await activityPubContextify(getAnnounceWithoutContext(servers[1]), 'Announce') - const headers = buildGlobalHeaders(body) - headers['date'] = 'Wed, 21 Oct 2015 07:28:00 GMT' - - try { - await makePOSTAPRequest(url, body, baseHttpSignature(), headers) - expect(true, 'Did not throw').to.be.false - } catch (err) { - expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) - } - }) - - it('Should fail with bad keys', async function () { - await setKeysOfServer(sqlCommands[0], servers[1].url, invalidKeys.publicKey, invalidKeys.privateKey) - await setKeysOfServer(sqlCommands[1], servers[1].url, invalidKeys.publicKey, invalidKeys.privateKey) - - const body = await activityPubContextify(getAnnounceWithoutContext(servers[1]), 'Announce') - const headers = buildGlobalHeaders(body) - - try { - await makePOSTAPRequest(url, body, baseHttpSignature(), headers) - expect(true, 'Did not throw').to.be.false - } catch (err) { - expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) - } - }) - - it('Should reject requests without appropriate signed headers', async function () { - await setKeysOfServer(sqlCommands[0], servers[1].url, keys.publicKey, keys.privateKey) - await setKeysOfServer(sqlCommands[1], servers[1].url, keys.publicKey, keys.privateKey) - - const body = await activityPubContextify(getAnnounceWithoutContext(servers[1]), 'Announce') - const headers = buildGlobalHeaders(body) - - const signatureOptions = baseHttpSignature() - const badHeadersMatrix = [ - [ '(request-target)', 'date', 'digest' ], - [ 'host', 'date', 'digest' ], - [ '(request-target)', 'host', 'digest' ] - ] - - for (const badHeaders of badHeadersMatrix) { - signatureOptions.headers = badHeaders - - try { - await makePOSTAPRequest(url, body, signatureOptions, headers) - expect(true, 'Did not throw').to.be.false - } catch (err) { - expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) - } - } - }) - - it('Should succeed with a valid HTTP signature draft 11 (without date but with (created))', async function () { - const body = await activityPubContextify(getAnnounceWithoutContext(servers[1]), 'Announce') - const headers = buildGlobalHeaders(body) - - const signatureOptions = baseHttpSignature() - signatureOptions.headers = [ '(request-target)', '(created)', 'host', 'digest' ] - - const { statusCode } = await makePOSTAPRequest(url, body, signatureOptions, headers) - expect(statusCode).to.equal(HttpStatusCode.NO_CONTENT_204) - }) - - it('Should succeed with a valid HTTP signature', async function () { - const body = await activityPubContextify(getAnnounceWithoutContext(servers[1]), 'Announce') - const headers = buildGlobalHeaders(body) - - const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers) - expect(statusCode).to.equal(HttpStatusCode.NO_CONTENT_204) - }) - - it('Should refresh the actor keys', async function () { - this.timeout(20000) - - // Update keys of server 2 to invalid keys - // Server 1 should refresh the actor and fail - await setKeysOfServer(sqlCommands[1], servers[1].url, invalidKeys.publicKey, invalidKeys.privateKey) - await setUpdatedAtOfServer(sqlCommands[0], servers[1].url, '2015-07-17 22:00:00+00') - - // Invalid peertube actor cache - await killallServers([ servers[1] ]) - await servers[1].run() - - const body = await activityPubContextify(getAnnounceWithoutContext(servers[1]), 'Announce') - const headers = buildGlobalHeaders(body) - - try { - await makePOSTAPRequest(url, body, baseHttpSignature(), headers) - expect(true, 'Did not throw').to.be.false - } catch (err) { - console.error(err) - expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) - } - }) - }) - - describe('When checking Linked Data Signature', function () { - before(async function () { - await setKeysOfServer(sqlCommands[0], servers[1].url, keys.publicKey, keys.privateKey) - await setKeysOfServer(sqlCommands[1], servers[1].url, keys.publicKey, keys.privateKey) - await setKeysOfServer(sqlCommands[2], servers[2].url, keys.publicKey, keys.privateKey) - - const to = { url: servers[0].url + '/accounts/peertube' } - const by = { url: servers[2].url + '/accounts/peertube', privateKey: keys.privateKey } - await makeFollowRequest(to, by) - }) - - it('Should fail with bad keys', async function () { - await setKeysOfServer(sqlCommands[0], servers[2].url, invalidKeys.publicKey, invalidKeys.privateKey) - await setKeysOfServer(sqlCommands[2], servers[2].url, invalidKeys.publicKey, invalidKeys.privateKey) - - const body = getAnnounceWithoutContext(servers[1]) - body.actor = servers[2].url + '/accounts/peertube' - - const signer: any = { privateKey: invalidKeys.privateKey, url: servers[2].url + '/accounts/peertube' } - const signedBody = await signAndContextify(signer, body, 'Announce') - - const headers = buildGlobalHeaders(signedBody) - - try { - await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) - expect(true, 'Did not throw').to.be.false - } catch (err) { - expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) - } - }) - - it('Should fail with an altered body', async function () { - await setKeysOfServer(sqlCommands[0], servers[2].url, keys.publicKey, keys.privateKey) - await setKeysOfServer(sqlCommands[0], servers[2].url, keys.publicKey, keys.privateKey) - - const body = getAnnounceWithoutContext(servers[1]) - body.actor = servers[2].url + '/accounts/peertube' - - const signer: any = { privateKey: keys.privateKey, url: servers[2].url + '/accounts/peertube' } - const signedBody = await signAndContextify(signer, body, 'Announce') - - signedBody.actor = servers[2].url + '/account/peertube' - - const headers = buildGlobalHeaders(signedBody) - - try { - await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) - expect(true, 'Did not throw').to.be.false - } catch (err) { - expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) - } - }) - - it('Should succeed with a valid signature', async function () { - const body = getAnnounceWithoutContext(servers[1]) - body.actor = servers[2].url + '/accounts/peertube' - - const signer: any = { privateKey: keys.privateKey, url: servers[2].url + '/accounts/peertube' } - const signedBody = await signAndContextify(signer, body, 'Announce') - - const headers = buildGlobalHeaders(signedBody) - - const { statusCode } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) - expect(statusCode).to.equal(HttpStatusCode.NO_CONTENT_204) - }) - - it('Should refresh the actor keys', async function () { - this.timeout(20000) - - // Wait refresh invalidation - await wait(10000) - - // Update keys of server 3 to invalid keys - // Server 1 should refresh the actor and fail - await setKeysOfServer(sqlCommands[2], servers[2].url, invalidKeys.publicKey, invalidKeys.privateKey) - - const body = getAnnounceWithoutContext(servers[1]) - body.actor = servers[2].url + '/accounts/peertube' - - const signer: any = { privateKey: keys.privateKey, url: servers[2].url + '/accounts/peertube' } - const signedBody = await signAndContextify(signer, body, 'Announce') - - const headers = buildGlobalHeaders(signedBody) - - try { - await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) - expect(true, 'Did not throw').to.be.false - } catch (err) { - expect(err.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) - } - }) - }) - - after(async function () { - for (const sql of sqlCommands) { - await sql.cleanup() - } - - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/check-params/abuses.ts b/server/tests/api/check-params/abuses.ts deleted file mode 100644 index 331d3f8f7..000000000 --- a/server/tests/api/check-params/abuses.ts +++ /dev/null @@ -1,438 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared' -import { AbuseCreate, AbuseState, HttpStatusCode } from '@shared/models' -import { - AbusesCommand, - cleanupTests, - createSingleServer, - doubleFollow, - makeGetRequest, - makePostBodyRequest, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test abuses API validators', function () { - const basePath = '/api/v1/abuses/' - - let server: PeerTubeServer - - let userToken = '' - let userToken2 = '' - let abuseId: number - let messageId: number - - let command: AbusesCommand - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - - userToken = await server.users.generateUserAndToken('user_1') - userToken2 = await server.users.generateUserAndToken('user_2') - - server.store.videoCreated = await server.videos.upload() - - command = server.abuses - }) - - describe('When listing abuses for admins', function () { - const path = basePath - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a non authenticated user', async function () { - await makeGetRequest({ - url: server.url, - path, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with a non admin user', async function () { - await makeGetRequest({ - url: server.url, - path, - token: userToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with a bad id filter', async function () { - await makeGetRequest({ url: server.url, path, token: server.accessToken, query: { id: 'toto' } }) - }) - - it('Should fail with a bad filter', async function () { - await makeGetRequest({ url: server.url, path, token: server.accessToken, query: { filter: 'toto' } }) - await makeGetRequest({ url: server.url, path, token: server.accessToken, query: { filter: 'videos' } }) - }) - - it('Should fail with bad predefined reason', async function () { - await makeGetRequest({ url: server.url, path, token: server.accessToken, query: { predefinedReason: 'violentOrRepulsives' } }) - }) - - it('Should fail with a bad state filter', async function () { - await makeGetRequest({ url: server.url, path, token: server.accessToken, query: { state: 'toto' } }) - await makeGetRequest({ url: server.url, path, token: server.accessToken, query: { state: 0 } }) - }) - - it('Should fail with a bad videoIs filter', async function () { - await makeGetRequest({ url: server.url, path, token: server.accessToken, query: { videoIs: 'toto' } }) - }) - - it('Should succeed with the correct params', async function () { - const query = { - id: 13, - predefinedReason: 'violentOrRepulsive', - filter: 'comment', - state: 2, - videoIs: 'deleted' - } - - await makeGetRequest({ url: server.url, path, token: server.accessToken, query, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - describe('When listing abuses for users', function () { - const path = '/api/v1/users/me/abuses' - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, userToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, userToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, userToken) - }) - - it('Should fail with a non authenticated user', async function () { - await makeGetRequest({ - url: server.url, - path, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with a bad id filter', async function () { - await makeGetRequest({ url: server.url, path, token: userToken, query: { id: 'toto' } }) - }) - - it('Should fail with a bad state filter', async function () { - await makeGetRequest({ url: server.url, path, token: userToken, query: { state: 'toto' } }) - await makeGetRequest({ url: server.url, path, token: userToken, query: { state: 0 } }) - }) - - it('Should succeed with the correct params', async function () { - const query = { - id: 13, - state: 2 - } - - await makeGetRequest({ url: server.url, path, token: userToken, query, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - describe('When reporting an abuse', function () { - const path = basePath - - it('Should fail with nothing', async function () { - const fields = {} - await makePostBodyRequest({ url: server.url, path, token: userToken, fields }) - }) - - it('Should fail with a wrong video', async function () { - const fields = { video: { id: 'blabla' }, reason: 'my super reason' } - await makePostBodyRequest({ url: server.url, path, token: userToken, fields }) - }) - - it('Should fail with an unknown video', async function () { - const fields = { video: { id: 42 }, reason: 'my super reason' } - await makePostBodyRequest({ - url: server.url, - path, - token: userToken, - fields, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail with a wrong comment', async function () { - const fields = { comment: { id: 'blabla' }, reason: 'my super reason' } - await makePostBodyRequest({ url: server.url, path, token: userToken, fields }) - }) - - it('Should fail with an unknown comment', async function () { - const fields = { comment: { id: 42 }, reason: 'my super reason' } - await makePostBodyRequest({ - url: server.url, - path, - token: userToken, - fields, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail with a wrong account', async function () { - const fields = { account: { id: 'blabla' }, reason: 'my super reason' } - await makePostBodyRequest({ url: server.url, path, token: userToken, fields }) - }) - - it('Should fail with an unknown account', async function () { - const fields = { account: { id: 42 }, reason: 'my super reason' } - await makePostBodyRequest({ - url: server.url, - path, - token: userToken, - fields, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail with not account, comment or video', async function () { - const fields = { reason: 'my super reason' } - await makePostBodyRequest({ - url: server.url, - path, - token: userToken, - fields, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with a non authenticated user', async function () { - const fields = { video: { id: server.store.videoCreated.id }, reason: 'my super reason' } - - await makePostBodyRequest({ url: server.url, path, token: 'hello', fields, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with a reason too short', async function () { - const fields = { video: { id: server.store.videoCreated.id }, reason: 'h' } - - await makePostBodyRequest({ url: server.url, path, token: userToken, fields }) - }) - - it('Should fail with a too big reason', async function () { - const fields = { video: { id: server.store.videoCreated.id }, reason: 'super'.repeat(605) } - - await makePostBodyRequest({ url: server.url, path, token: userToken, fields }) - }) - - it('Should succeed with the correct parameters (basic)', async function () { - const fields: AbuseCreate = { video: { id: server.store.videoCreated.shortUUID }, reason: 'my super reason' } - - const res = await makePostBodyRequest({ - url: server.url, - path, - token: userToken, - fields, - expectedStatus: HttpStatusCode.OK_200 - }) - abuseId = res.body.abuse.id - }) - - it('Should fail with a wrong predefined reason', async function () { - const fields = { video: server.store.videoCreated, reason: 'my super reason', predefinedReasons: [ 'wrongPredefinedReason' ] } - - await makePostBodyRequest({ url: server.url, path, token: userToken, fields }) - }) - - it('Should fail with negative timestamps', async function () { - const fields = { video: { id: server.store.videoCreated.id, startAt: -1 }, reason: 'my super reason' } - - await makePostBodyRequest({ url: server.url, path, token: userToken, fields }) - }) - - it('Should fail mith misordered startAt/endAt', async function () { - const fields = { video: { id: server.store.videoCreated.id, startAt: 5, endAt: 1 }, reason: 'my super reason' } - - await makePostBodyRequest({ url: server.url, path, token: userToken, fields }) - }) - - it('Should succeed with the correct parameters (advanced)', async function () { - const fields: AbuseCreate = { - video: { - id: server.store.videoCreated.id, - startAt: 1, - endAt: 5 - }, - reason: 'my super reason', - predefinedReasons: [ 'serverRules' ] - } - - await makePostBodyRequest({ url: server.url, path, token: userToken, fields, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - describe('When updating an abuse', function () { - - it('Should fail with a non authenticated user', async function () { - await command.update({ token: 'blabla', abuseId, body: {}, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with a non admin user', async function () { - await command.update({ token: userToken, abuseId, body: {}, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should fail with a bad abuse id', async function () { - await command.update({ abuseId: 45, body: {}, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should fail with a bad state', async function () { - const body = { state: 5 } - await command.update({ abuseId, body, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with a bad moderation comment', async function () { - const body = { moderationComment: 'b'.repeat(3001) } - await command.update({ abuseId, body, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should succeed with the correct params', async function () { - const body = { state: AbuseState.ACCEPTED } - await command.update({ abuseId, body }) - }) - }) - - describe('When creating an abuse message', function () { - const message = 'my super message' - - it('Should fail with an invalid abuse id', async function () { - await command.addMessage({ token: userToken2, abuseId: 888, message, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should fail with a non authenticated user', async function () { - await command.addMessage({ token: 'fake_token', abuseId, message, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with an invalid logged in user', async function () { - await command.addMessage({ token: userToken2, abuseId, message, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should fail with an invalid message', async function () { - await command.addMessage({ token: userToken, abuseId, message: 'a'.repeat(5000), expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should succeed with the correct params', async function () { - const res = await command.addMessage({ token: userToken, abuseId, message }) - messageId = res.body.abuseMessage.id - }) - }) - - describe('When listing abuse messages', function () { - - it('Should fail with an invalid abuse id', async function () { - await command.listMessages({ token: userToken, abuseId: 888, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should fail with a non authenticated user', async function () { - await command.listMessages({ token: 'fake_token', abuseId, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with an invalid logged in user', async function () { - await command.listMessages({ token: userToken2, abuseId, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should succeed with the correct params', async function () { - await command.listMessages({ token: userToken, abuseId }) - }) - }) - - describe('When deleting an abuse message', function () { - it('Should fail with an invalid abuse id', async function () { - await command.deleteMessage({ token: userToken, abuseId: 888, messageId, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should fail with an invalid message id', async function () { - await command.deleteMessage({ token: userToken, abuseId, messageId: 888, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should fail with a non authenticated user', async function () { - await command.deleteMessage({ token: 'fake_token', abuseId, messageId, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with an invalid logged in user', async function () { - await command.deleteMessage({ token: userToken2, abuseId, messageId, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should succeed with the correct params', async function () { - await command.deleteMessage({ token: userToken, abuseId, messageId }) - }) - }) - - describe('When deleting a video abuse', function () { - - it('Should fail with a non authenticated user', async function () { - await command.delete({ token: 'blabla', abuseId, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with a non admin user', async function () { - await command.delete({ token: userToken, abuseId, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should fail with a bad abuse id', async function () { - await command.delete({ abuseId: 45, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should succeed with the correct params', async function () { - await command.delete({ abuseId }) - }) - }) - - describe('When trying to manage messages of a remote abuse', function () { - let remoteAbuseId: number - let anotherServer: PeerTubeServer - - before(async function () { - this.timeout(50000) - - anotherServer = await createSingleServer(2) - await setAccessTokensToServers([ anotherServer ]) - - await doubleFollow(anotherServer, server) - - const server2VideoId = await anotherServer.videos.getId({ uuid: server.store.videoCreated.uuid }) - await anotherServer.abuses.report({ reason: 'remote server', videoId: server2VideoId }) - - await waitJobs([ server, anotherServer ]) - - const body = await command.getAdminList({ sort: '-createdAt' }) - remoteAbuseId = body.data[0].id - }) - - it('Should fail when listing abuse messages of a remote abuse', async function () { - await command.listMessages({ abuseId: remoteAbuseId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail when creating abuse message of a remote abuse', async function () { - await command.addMessage({ abuseId: remoteAbuseId, message: 'message', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - after(async function () { - await cleanupTests([ anotherServer ]) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/accounts.ts b/server/tests/api/check-params/accounts.ts deleted file mode 100644 index afc0049ff..000000000 --- a/server/tests/api/check-params/accounts.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared' -import { HttpStatusCode } from '@shared/models' -import { cleanupTests, createSingleServer, PeerTubeServer } from '@shared/server-commands' - -describe('Test accounts API validators', function () { - const path = '/api/v1/accounts/' - let server: PeerTubeServer - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - }) - - describe('When listing accounts', function () { - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, server.accessToken) - }) - }) - - describe('When getting an account', function () { - - it('Should return 404 with a non existing name', async function () { - await server.accounts.get({ accountName: 'arfaze', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/blocklist.ts b/server/tests/api/check-params/blocklist.ts deleted file mode 100644 index 169b591a3..000000000 --- a/server/tests/api/check-params/blocklist.ts +++ /dev/null @@ -1,556 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared' -import { HttpStatusCode } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - makeDeleteRequest, - makeGetRequest, - makePostBodyRequest, - PeerTubeServer, - setAccessTokensToServers -} from '@shared/server-commands' - -describe('Test blocklist API validators', function () { - let servers: PeerTubeServer[] - let server: PeerTubeServer - let userAccessToken: string - - before(async function () { - this.timeout(60000) - - servers = await createMultipleServers(2) - await setAccessTokensToServers(servers) - - server = servers[0] - - const user = { username: 'user1', password: 'password' } - await server.users.create({ username: user.username, password: user.password }) - - userAccessToken = await server.login.getAccessToken(user) - - await doubleFollow(servers[0], servers[1]) - }) - - // --------------------------------------------------------------- - - describe('When managing user blocklist', function () { - - describe('When managing user accounts blocklist', function () { - const path = '/api/v1/users/me/blocklist/accounts' - - describe('When listing blocked accounts', function () { - it('Should fail with an unauthenticated user', async function () { - await makeGetRequest({ - url: server.url, - path, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, server.accessToken) - }) - }) - - describe('When blocking an account', function () { - it('Should fail with an unauthenticated user', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: { accountName: 'user1' }, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with an unknown account', async function () { - await makePostBodyRequest({ - url: server.url, - token: server.accessToken, - path, - fields: { accountName: 'user2' }, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail to block ourselves', async function () { - await makePostBodyRequest({ - url: server.url, - token: server.accessToken, - path, - fields: { accountName: 'root' }, - expectedStatus: HttpStatusCode.CONFLICT_409 - }) - }) - - it('Should succeed with the correct params', async function () { - await makePostBodyRequest({ - url: server.url, - token: server.accessToken, - path, - fields: { accountName: 'user1' }, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - describe('When unblocking an account', function () { - it('Should fail with an unauthenticated user', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/user1', - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with an unknown account block', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/user2', - token: server.accessToken, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should succeed with the correct params', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/user1', - token: server.accessToken, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - }) - - describe('When managing user servers blocklist', function () { - const path = '/api/v1/users/me/blocklist/servers' - - describe('When listing blocked servers', function () { - it('Should fail with an unauthenticated user', async function () { - await makeGetRequest({ - url: server.url, - path, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, server.accessToken) - }) - }) - - describe('When blocking a server', function () { - it('Should fail with an unauthenticated user', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: { host: '127.0.0.1:9002' }, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should succeed with an unknown server', async function () { - await makePostBodyRequest({ - url: server.url, - token: server.accessToken, - path, - fields: { host: '127.0.0.1:9003' }, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - - it('Should fail with our own server', async function () { - await makePostBodyRequest({ - url: server.url, - token: server.accessToken, - path, - fields: { host: server.host }, - expectedStatus: HttpStatusCode.CONFLICT_409 - }) - }) - - it('Should succeed with the correct params', async function () { - await makePostBodyRequest({ - url: server.url, - token: server.accessToken, - path, - fields: { host: servers[1].host }, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - describe('When unblocking a server', function () { - it('Should fail with an unauthenticated user', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/' + servers[1].host, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with an unknown server block', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/127.0.0.1:9004', - token: server.accessToken, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should succeed with the correct params', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/' + servers[1].host, - token: server.accessToken, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - }) - }) - - describe('When managing server blocklist', function () { - - describe('When managing server accounts blocklist', function () { - const path = '/api/v1/server/blocklist/accounts' - - describe('When listing blocked accounts', function () { - it('Should fail with an unauthenticated user', async function () { - await makeGetRequest({ - url: server.url, - path, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with a user without the appropriate rights', async function () { - await makeGetRequest({ - url: server.url, - token: userAccessToken, - path, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, server.accessToken) - }) - }) - - describe('When blocking an account', function () { - it('Should fail with an unauthenticated user', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: { accountName: 'user1' }, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with a user without the appropriate rights', async function () { - await makePostBodyRequest({ - url: server.url, - token: userAccessToken, - path, - fields: { accountName: 'user1' }, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with an unknown account', async function () { - await makePostBodyRequest({ - url: server.url, - token: server.accessToken, - path, - fields: { accountName: 'user2' }, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail to block ourselves', async function () { - await makePostBodyRequest({ - url: server.url, - token: server.accessToken, - path, - fields: { accountName: 'root' }, - expectedStatus: HttpStatusCode.CONFLICT_409 - }) - }) - - it('Should succeed with the correct params', async function () { - await makePostBodyRequest({ - url: server.url, - token: server.accessToken, - path, - fields: { accountName: 'user1' }, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - describe('When unblocking an account', function () { - it('Should fail with an unauthenticated user', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/user1', - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with a user without the appropriate rights', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/user1', - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with an unknown account block', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/user2', - token: server.accessToken, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should succeed with the correct params', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/user1', - token: server.accessToken, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - }) - - describe('When managing server servers blocklist', function () { - const path = '/api/v1/server/blocklist/servers' - - describe('When listing blocked servers', function () { - it('Should fail with an unauthenticated user', async function () { - await makeGetRequest({ - url: server.url, - path, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with a user without the appropriate rights', async function () { - await makeGetRequest({ - url: server.url, - token: userAccessToken, - path, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, server.accessToken) - }) - }) - - describe('When blocking a server', function () { - it('Should fail with an unauthenticated user', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: { host: servers[1].host }, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with a user without the appropriate rights', async function () { - await makePostBodyRequest({ - url: server.url, - token: userAccessToken, - path, - fields: { host: servers[1].host }, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should succeed with an unknown server', async function () { - await makePostBodyRequest({ - url: server.url, - token: server.accessToken, - path, - fields: { host: '127.0.0.1:9003' }, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - - it('Should fail with our own server', async function () { - await makePostBodyRequest({ - url: server.url, - token: server.accessToken, - path, - fields: { host: server.host }, - expectedStatus: HttpStatusCode.CONFLICT_409 - }) - }) - - it('Should succeed with the correct params', async function () { - await makePostBodyRequest({ - url: server.url, - token: server.accessToken, - path, - fields: { host: servers[1].host }, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - describe('When unblocking a server', function () { - it('Should fail with an unauthenticated user', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/' + servers[1].host, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with a user without the appropriate rights', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/' + servers[1].host, - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with an unknown server block', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/127.0.0.1:9004', - token: server.accessToken, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should succeed with the correct params', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/' + servers[1].host, - token: server.accessToken, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - }) - }) - - describe('When getting blocklist status', function () { - const path = '/api/v1/blocklist/status' - - it('Should fail with a bad token', async function () { - await makeGetRequest({ - url: server.url, - path, - token: 'false', - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with a bad accounts field', async function () { - await makeGetRequest({ - url: server.url, - path, - query: { - accounts: 1 - }, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - - await makeGetRequest({ - url: server.url, - path, - query: { - accounts: [ 1 ] - }, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with a bad hosts field', async function () { - await makeGetRequest({ - url: server.url, - path, - query: { - hosts: 1 - }, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - - await makeGetRequest({ - url: server.url, - path, - query: { - hosts: [ 1 ] - }, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should succeed with the correct parameters', async function () { - await makeGetRequest({ - url: server.url, - path, - query: {}, - expectedStatus: HttpStatusCode.OK_200 - }) - - await makeGetRequest({ - url: server.url, - path, - query: { - hosts: [ 'example.com' ], - accounts: [ 'john@example.com' ] - }, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/check-params/bulk.ts b/server/tests/api/check-params/bulk.ts deleted file mode 100644 index f03264b4f..000000000 --- a/server/tests/api/check-params/bulk.ts +++ /dev/null @@ -1,80 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { cleanupTests, createSingleServer, makePostBodyRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' -import { HttpStatusCode } from '@shared/models' - -describe('Test bulk API validators', function () { - let server: PeerTubeServer - let userAccessToken: string - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(120000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - - const user = { username: 'user1', password: 'password' } - await server.users.create({ username: user.username, password: user.password }) - - userAccessToken = await server.login.getAccessToken(user) - }) - - describe('When removing comments of', function () { - const path = '/api/v1/bulk/remove-comments-of' - - it('Should fail with an unauthenticated user', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: { accountName: 'user1', scope: 'my-videos' }, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with an unknown account', async function () { - await makePostBodyRequest({ - url: server.url, - token: server.accessToken, - path, - fields: { accountName: 'user2', scope: 'my-videos' }, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail with an invalid scope', async function () { - await makePostBodyRequest({ - url: server.url, - token: server.accessToken, - path, - fields: { accountName: 'user1', scope: 'my-videoss' }, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail to delete comments of the instance without the appropriate rights', async function () { - await makePostBodyRequest({ - url: server.url, - token: userAccessToken, - path, - fields: { accountName: 'user1', scope: 'instance' }, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should succeed with the correct params', async function () { - await makePostBodyRequest({ - url: server.url, - token: server.accessToken, - path, - fields: { accountName: 'user1', scope: 'instance' }, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/channel-import-videos.ts b/server/tests/api/check-params/channel-import-videos.ts deleted file mode 100644 index 2de13b629..000000000 --- a/server/tests/api/check-params/channel-import-videos.ts +++ /dev/null @@ -1,209 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { FIXTURE_URLS } from '@server/tests/shared' -import { areHttpImportTestsDisabled } from '@shared/core-utils' -import { HttpStatusCode } from '@shared/models' -import { - ChannelsCommand, - cleanupTests, - createSingleServer, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel -} from '@shared/server-commands' - -describe('Test videos import in a channel API validator', function () { - let server: PeerTubeServer - const userInfo = { - accessToken: '', - channelName: 'fake_channel', - channelId: -1, - id: -1, - videoQuota: -1, - videoQuotaDaily: -1, - channelSyncId: -1 - } - let command: ChannelsCommand - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(120000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - await server.config.enableImports() - await server.config.enableChannelSync() - - const userCreds = { - username: 'fake', - password: 'fake_password' - } - - { - const user = await server.users.create({ username: userCreds.username, password: userCreds.password }) - userInfo.id = user.id - userInfo.accessToken = await server.login.getAccessToken(userCreds) - - const info = await server.users.getMyInfo({ token: userInfo.accessToken }) - userInfo.channelId = info.videoChannels[0].id - } - - { - const { videoChannelSync } = await server.channelSyncs.create({ - token: userInfo.accessToken, - attributes: { - externalChannelUrl: FIXTURE_URLS.youtubeChannel, - videoChannelId: userInfo.channelId - } - }) - userInfo.channelSyncId = videoChannelSync.id - } - - command = server.channels - }) - - it('Should fail when HTTP upload is disabled', async function () { - await server.config.disableChannelSync() - await server.config.disableImports() - - await command.importVideos({ - channelName: server.store.channel.name, - externalChannelUrl: FIXTURE_URLS.youtubeChannel, - token: server.accessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - - await server.config.enableImports() - }) - - it('Should fail when externalChannelUrl is not provided', async function () { - await command.importVideos({ - channelName: server.store.channel.name, - externalChannelUrl: null, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail when externalChannelUrl is malformed', async function () { - await command.importVideos({ - channelName: server.store.channel.name, - externalChannelUrl: 'not-a-url', - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with a bad sync id', async function () { - await command.importVideos({ - channelName: server.store.channel.name, - externalChannelUrl: FIXTURE_URLS.youtubeChannel, - videoChannelSyncId: 'toto' as any, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with a unknown sync id', async function () { - await command.importVideos({ - channelName: server.store.channel.name, - externalChannelUrl: FIXTURE_URLS.youtubeChannel, - videoChannelSyncId: 42, - token: server.accessToken, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail with a sync id of another channel', async function () { - await command.importVideos({ - channelName: server.store.channel.name, - externalChannelUrl: FIXTURE_URLS.youtubeChannel, - videoChannelSyncId: userInfo.channelSyncId, - token: server.accessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with no authentication', async function () { - await command.importVideos({ - channelName: server.store.channel.name, - externalChannelUrl: FIXTURE_URLS.youtubeChannel, - token: null, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail when sync is not owned by the user', async function () { - await command.importVideos({ - channelName: server.store.channel.name, - externalChannelUrl: FIXTURE_URLS.youtubeChannel, - token: userInfo.accessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail when the user has no quota', async function () { - await server.users.update({ - userId: userInfo.id, - videoQuota: 0 - }) - - await command.importVideos({ - channelName: 'fake_channel', - externalChannelUrl: FIXTURE_URLS.youtubeChannel, - token: userInfo.accessToken, - expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413 - }) - - await server.users.update({ - userId: userInfo.id, - videoQuota: userInfo.videoQuota - }) - }) - - it('Should fail when the user has no daily quota', async function () { - await server.users.update({ - userId: userInfo.id, - videoQuotaDaily: 0 - }) - - await command.importVideos({ - channelName: 'fake_channel', - externalChannelUrl: FIXTURE_URLS.youtubeChannel, - token: userInfo.accessToken, - expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413 - }) - - await server.users.update({ - userId: userInfo.id, - videoQuotaDaily: userInfo.videoQuotaDaily - }) - }) - - it('Should succeed when sync is run by its owner', async function () { - if (!areHttpImportTestsDisabled()) return - - await command.importVideos({ - channelName: 'fake_channel', - externalChannelUrl: FIXTURE_URLS.youtubeChannel, - token: userInfo.accessToken - }) - }) - - it('Should succeed when sync is run with root and for another user\'s channel', async function () { - if (!areHttpImportTestsDisabled()) return - - await command.importVideos({ - channelName: 'fake_channel', - externalChannelUrl: FIXTURE_URLS.youtubeChannel - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/config.ts b/server/tests/api/check-params/config.ts deleted file mode 100644 index 2f523d4ce..000000000 --- a/server/tests/api/check-params/config.ts +++ /dev/null @@ -1,428 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { merge } from 'lodash' -import { omit } from '@shared/core-utils' -import { CustomConfig, HttpStatusCode } from '@shared/models' -import { - cleanupTests, - createSingleServer, - makeDeleteRequest, - makeGetRequest, - makePutBodyRequest, - PeerTubeServer, - setAccessTokensToServers -} from '@shared/server-commands' - -describe('Test config API validators', function () { - const path = '/api/v1/config/custom' - let server: PeerTubeServer - let userAccessToken: string - const updateParams: CustomConfig = { - instance: { - name: 'PeerTube updated', - shortDescription: 'my short description', - description: 'my super description', - terms: 'my super terms', - codeOfConduct: 'my super coc', - - creationReason: 'my super reason', - moderationInformation: 'my super moderation information', - administrator: 'Kuja', - maintenanceLifetime: 'forever', - businessModel: 'my super business model', - hardwareInformation: '2vCore 3GB RAM', - - languages: [ 'en', 'es' ], - categories: [ 1, 2 ], - - isNSFW: true, - defaultNSFWPolicy: 'blur', - - defaultClientRoute: '/videos/recently-added', - - customizations: { - javascript: 'alert("coucou")', - css: 'body { background-color: red; }' - } - }, - theme: { - default: 'default' - }, - services: { - twitter: { - username: '@MySuperUsername', - whitelisted: true - } - }, - client: { - videos: { - miniature: { - preferAuthorDisplayName: false - } - }, - menu: { - login: { - redirectOnSingleExternalAuth: false - } - } - }, - cache: { - previews: { - size: 2 - }, - captions: { - size: 3 - }, - torrents: { - size: 4 - }, - storyboards: { - size: 5 - } - }, - signup: { - enabled: false, - limit: 5, - requiresApproval: false, - requiresEmailVerification: false, - minimumAge: 16 - }, - admin: { - email: 'superadmin1@example.com' - }, - contactForm: { - enabled: false - }, - user: { - history: { - videos: { - enabled: true - } - }, - videoQuota: 5242881, - videoQuotaDaily: 318742 - }, - videoChannels: { - maxPerUser: 20 - }, - transcoding: { - enabled: true, - remoteRunners: { - enabled: true - }, - allowAdditionalExtensions: true, - allowAudioFiles: true, - concurrency: 1, - threads: 1, - profile: 'vod_profile', - resolutions: { - '0p': false, - '144p': false, - '240p': false, - '360p': true, - '480p': true, - '720p': false, - '1080p': false, - '1440p': false, - '2160p': false - }, - alwaysTranscodeOriginalResolution: false, - webVideos: { - enabled: true - }, - hls: { - enabled: false - } - }, - live: { - enabled: true, - - allowReplay: false, - latencySetting: { - enabled: false - }, - maxDuration: 30, - maxInstanceLives: -1, - maxUserLives: 50, - - transcoding: { - enabled: true, - remoteRunners: { - enabled: true - }, - threads: 4, - profile: 'live_profile', - resolutions: { - '144p': true, - '240p': true, - '360p': true, - '480p': true, - '720p': true, - '1080p': true, - '1440p': true, - '2160p': true - }, - alwaysTranscodeOriginalResolution: false - } - }, - videoStudio: { - enabled: true, - remoteRunners: { - enabled: true - } - }, - videoFile: { - update: { - enabled: true - } - }, - import: { - videos: { - concurrency: 1, - http: { - enabled: false - }, - torrent: { - enabled: false - } - }, - videoChannelSynchronization: { - enabled: false, - maxPerUser: 10 - } - }, - trending: { - videos: { - algorithms: { - enabled: [ 'hot', 'most-viewed', 'most-liked' ], - default: 'most-viewed' - } - } - }, - autoBlacklist: { - videos: { - ofUsers: { - enabled: false - } - } - }, - followers: { - instance: { - enabled: false, - manualApproval: true - } - }, - followings: { - instance: { - autoFollowBack: { - enabled: true - }, - autoFollowIndex: { - enabled: true, - indexUrl: 'https://index.example.com' - } - } - }, - broadcastMessage: { - enabled: true, - dismissable: true, - message: 'super message', - level: 'warning' - }, - search: { - remoteUri: { - users: true, - anonymous: true - }, - searchIndex: { - enabled: true, - url: 'https://search.joinpeertube.org', - disableLocalSearch: true, - isDefaultSearch: true - } - } - } - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - - const user = { - username: 'user1', - password: 'password' - } - await server.users.create({ username: user.username, password: user.password }) - userAccessToken = await server.login.getAccessToken(user) - }) - - describe('When getting the configuration', function () { - it('Should fail without token', async function () { - await makeGetRequest({ - url: server.url, - path, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail if the user is not an administrator', async function () { - await makeGetRequest({ - url: server.url, - path, - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - }) - - describe('When updating the configuration', function () { - it('Should fail without token', async function () { - await makePutBodyRequest({ - url: server.url, - path, - fields: updateParams, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail if the user is not an administrator', async function () { - await makePutBodyRequest({ - url: server.url, - path, - fields: updateParams, - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail if it misses a key', async function () { - const newUpdateParams = { ...updateParams, admin: omit(updateParams.admin, [ 'email' ]) } - - await makePutBodyRequest({ - url: server.url, - path, - fields: newUpdateParams, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with a bad default NSFW policy', async function () { - const newUpdateParams = { - ...updateParams, - - instance: { - defaultNSFWPolicy: 'hello' - } - } - - await makePutBodyRequest({ - url: server.url, - path, - fields: newUpdateParams, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail if email disabled and signup requires email verification', async function () { - // opposite scenario - success when enable enabled - covered via tests/api/users/user-verification.ts - const newUpdateParams = { - ...updateParams, - - signup: { - enabled: true, - limit: 5, - requiresApproval: true, - requiresEmailVerification: true - } - } - - await makePutBodyRequest({ - url: server.url, - path, - fields: newUpdateParams, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with a disabled web videos & hls transcoding', async function () { - const newUpdateParams = { - ...updateParams, - - transcoding: { - hls: { - enabled: false - }, - web_videos: { - enabled: false - } - } - } - - await makePutBodyRequest({ - url: server.url, - path, - fields: newUpdateParams, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with a disabled http upload & enabled sync', async function () { - const newUpdateParams: CustomConfig = merge({}, updateParams, { - import: { - videos: { - http: { enabled: false } - }, - videoChannelSynchronization: { enabled: true } - } - }) - - await makePutBodyRequest({ - url: server.url, - path, - fields: newUpdateParams, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should succeed with the correct parameters', async function () { - await makePutBodyRequest({ - url: server.url, - path, - fields: updateParams, - token: server.accessToken, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - - describe('When deleting the configuration', function () { - it('Should fail without token', async function () { - await makeDeleteRequest({ - url: server.url, - path, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail if the user is not an administrator', async function () { - await makeDeleteRequest({ - url: server.url, - path, - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/contact-form.ts b/server/tests/api/check-params/contact-form.ts deleted file mode 100644 index f0f8819b9..000000000 --- a/server/tests/api/check-params/contact-form.ts +++ /dev/null @@ -1,86 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { MockSmtpServer } from '@server/tests/shared' -import { HttpStatusCode } from '@shared/models' -import { - cleanupTests, - ConfigCommand, - ContactFormCommand, - createSingleServer, - killallServers, - PeerTubeServer -} from '@shared/server-commands' - -describe('Test contact form API validators', function () { - let server: PeerTubeServer - const emails: object[] = [] - const defaultBody = { - fromName: 'super name', - fromEmail: 'toto@example.com', - subject: 'my subject', - body: 'Hello, how are you?' - } - let emailPort: number - let command: ContactFormCommand - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(60000) - - emailPort = await MockSmtpServer.Instance.collectEmails(emails) - - // Email is disabled - server = await createSingleServer(1) - command = server.contactForm - }) - - it('Should not accept a contact form if emails are disabled', async function () { - await command.send({ ...defaultBody, expectedStatus: HttpStatusCode.CONFLICT_409 }) - }) - - it('Should not accept a contact form if it is disabled in the configuration', async function () { - this.timeout(25000) - - await killallServers([ server ]) - - // Contact form is disabled - await server.run({ ...ConfigCommand.getEmailOverrideConfig(emailPort), contact_form: { enabled: false } }) - await command.send({ ...defaultBody, expectedStatus: HttpStatusCode.CONFLICT_409 }) - }) - - it('Should not accept a contact form if from email is invalid', async function () { - this.timeout(25000) - - await killallServers([ server ]) - - // Email & contact form enabled - await server.run(ConfigCommand.getEmailOverrideConfig(emailPort)) - - await command.send({ ...defaultBody, fromEmail: 'badEmail', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await command.send({ ...defaultBody, fromEmail: 'badEmail@', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await command.send({ ...defaultBody, fromEmail: undefined, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should not accept a contact form if from name is invalid', async function () { - await command.send({ ...defaultBody, fromName: 'name'.repeat(100), expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await command.send({ ...defaultBody, fromName: '', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await command.send({ ...defaultBody, fromName: undefined, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should not accept a contact form if body is invalid', async function () { - await command.send({ ...defaultBody, body: 'body'.repeat(5000), expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await command.send({ ...defaultBody, body: 'a', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await command.send({ ...defaultBody, body: undefined, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should accept a contact form with the correct parameters', async function () { - await command.send(defaultBody) - }) - - after(async function () { - MockSmtpServer.Instance.kill() - - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/custom-pages.ts b/server/tests/api/check-params/custom-pages.ts deleted file mode 100644 index 63e3da3d5..000000000 --- a/server/tests/api/check-params/custom-pages.ts +++ /dev/null @@ -1,79 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { HttpStatusCode } from '@shared/models' -import { - cleanupTests, - createSingleServer, - makeGetRequest, - makePutBodyRequest, - PeerTubeServer, - setAccessTokensToServers -} from '@shared/server-commands' - -describe('Test custom pages validators', function () { - const path = '/api/v1/custom-pages/homepage/instance' - - let server: PeerTubeServer - let userAccessToken: string - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(120000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - - const user = { username: 'user1', password: 'password' } - await server.users.create({ username: user.username, password: user.password }) - - userAccessToken = await server.login.getAccessToken(user) - }) - - describe('When updating instance homepage', function () { - - it('Should fail with an unauthenticated user', async function () { - await makePutBodyRequest({ - url: server.url, - path, - fields: { content: 'super content' }, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with a non admin user', async function () { - await makePutBodyRequest({ - url: server.url, - path, - token: userAccessToken, - fields: { content: 'super content' }, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should succeed with the correct params', async function () { - await makePutBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields: { content: 'super content' }, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - describe('When getting instance homapage', function () { - - it('Should succeed with the correct params', async function () { - await makeGetRequest({ - url: server.url, - path, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/debug.ts b/server/tests/api/check-params/debug.ts deleted file mode 100644 index d7b68f163..000000000 --- a/server/tests/api/check-params/debug.ts +++ /dev/null @@ -1,61 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { cleanupTests, createSingleServer, makeGetRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' -import { HttpStatusCode } from '@shared/models' - -describe('Test debug API validators', function () { - const path = '/api/v1/server/debug' - let server: PeerTubeServer - let userAccessToken = '' - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(120000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - - const user = { - username: 'user1', - password: 'my super password' - } - await server.users.create({ username: user.username, password: user.password }) - userAccessToken = await server.login.getAccessToken(user) - }) - - describe('When getting debug endpoint', function () { - - it('Should fail with a non authenticated user', async function () { - await makeGetRequest({ - url: server.url, - path, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with a non admin user', async function () { - await makeGetRequest({ - url: server.url, - path, - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should succeed with the correct params', async function () { - await makeGetRequest({ - url: server.url, - path, - token: server.accessToken, - query: { startDate: new Date().toISOString() }, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/follows.ts b/server/tests/api/check-params/follows.ts deleted file mode 100644 index 3c911dcee..000000000 --- a/server/tests/api/check-params/follows.ts +++ /dev/null @@ -1,369 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared' -import { HttpStatusCode } from '@shared/models' -import { - cleanupTests, - createSingleServer, - makeDeleteRequest, - makeGetRequest, - makePostBodyRequest, - PeerTubeServer, - setAccessTokensToServers -} from '@shared/server-commands' - -describe('Test server follows API validators', function () { - let server: PeerTubeServer - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - }) - - describe('When managing following', function () { - let userAccessToken = null - - before(async function () { - userAccessToken = await server.users.generateUserAndToken('user1') - }) - - describe('When adding follows', function () { - const path = '/api/v1/server/following' - - it('Should fail with nothing', async function () { - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail if hosts is not composed by hosts', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: { hosts: [ '127.0.0.1:9002', '127.0.0.1:coucou' ] }, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail if hosts is composed with http schemes', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: { hosts: [ '127.0.0.1:9002', 'http://127.0.0.1:9003' ] }, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail if hosts are not unique', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: { urls: [ '127.0.0.1:9002', '127.0.0.1:9002' ] }, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail if handles is not composed by handles', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: { handles: [ 'hello@example.com', '127.0.0.1:9001' ] }, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail if handles are not unique', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: { urls: [ 'hello@example.com', 'hello@example.com' ] }, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with an invalid token', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: { hosts: [ '127.0.0.1:9002' ] }, - token: 'fake_token', - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail if the user is not an administrator', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: { hosts: [ '127.0.0.1:9002' ] }, - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - }) - - describe('When listing followings', function () { - const path = '/api/v1/server/following' - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path) - }) - - it('Should fail with an incorrect state', async function () { - await makeGetRequest({ - url: server.url, - path, - query: { - state: 'blabla' - } - }) - }) - - it('Should fail with an incorrect actor type', async function () { - await makeGetRequest({ - url: server.url, - path, - query: { - actorType: 'blabla' - } - }) - }) - - it('Should fail succeed with the correct params', async function () { - await makeGetRequest({ - url: server.url, - path, - expectedStatus: HttpStatusCode.OK_200, - query: { - state: 'accepted', - actorType: 'Application' - } - }) - }) - }) - - describe('When listing followers', function () { - const path = '/api/v1/server/followers' - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path) - }) - - it('Should fail with an incorrect actor type', async function () { - await makeGetRequest({ - url: server.url, - path, - query: { - actorType: 'blabla' - } - }) - }) - - it('Should fail with an incorrect state', async function () { - await makeGetRequest({ - url: server.url, - path, - query: { - state: 'blabla', - actorType: 'Application' - } - }) - }) - - it('Should fail succeed with the correct params', async function () { - await makeGetRequest({ - url: server.url, - path, - expectedStatus: HttpStatusCode.OK_200, - query: { - state: 'accepted' - } - }) - }) - }) - - describe('When removing a follower', function () { - const path = '/api/v1/server/followers' - - it('Should fail with an invalid token', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/toto@127.0.0.1:9002', - token: 'fake_token', - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail if the user is not an administrator', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/toto@127.0.0.1:9002', - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with an invalid follower', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/toto', - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with an unknown follower', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/toto@127.0.0.1:9003', - token: server.accessToken, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - }) - - describe('When accepting a follower', function () { - const path = '/api/v1/server/followers' - - it('Should fail with an invalid token', async function () { - await makePostBodyRequest({ - url: server.url, - path: path + '/toto@127.0.0.1:9002/accept', - token: 'fake_token', - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail if the user is not an administrator', async function () { - await makePostBodyRequest({ - url: server.url, - path: path + '/toto@127.0.0.1:9002/accept', - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with an invalid follower', async function () { - await makePostBodyRequest({ - url: server.url, - path: path + '/toto/accept', - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with an unknown follower', async function () { - await makePostBodyRequest({ - url: server.url, - path: path + '/toto@127.0.0.1:9003/accept', - token: server.accessToken, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - }) - - describe('When rejecting a follower', function () { - const path = '/api/v1/server/followers' - - it('Should fail with an invalid token', async function () { - await makePostBodyRequest({ - url: server.url, - path: path + '/toto@127.0.0.1:9002/reject', - token: 'fake_token', - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail if the user is not an administrator', async function () { - await makePostBodyRequest({ - url: server.url, - path: path + '/toto@127.0.0.1:9002/reject', - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with an invalid follower', async function () { - await makePostBodyRequest({ - url: server.url, - path: path + '/toto/reject', - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with an unknown follower', async function () { - await makePostBodyRequest({ - url: server.url, - path: path + '/toto@127.0.0.1:9003/reject', - token: server.accessToken, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - }) - - describe('When removing following', function () { - const path = '/api/v1/server/following' - - it('Should fail with an invalid token', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/127.0.0.1:9002', - token: 'fake_token', - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail if the user is not an administrator', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/127.0.0.1:9002', - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail if we do not follow this server', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/example.com', - token: server.accessToken, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts deleted file mode 100644 index c2a7ccd78..000000000 --- a/server/tests/api/check-params/index.ts +++ /dev/null @@ -1,45 +0,0 @@ -import './abuses' -import './accounts' -import './blocklist' -import './bulk' -import './channel-import-videos' -import './config' -import './contact-form' -import './custom-pages' -import './debug' -import './follows' -import './jobs' -import './live' -import './logs' -import './metrics' -import './my-user' -import './plugins' -import './redundancy' -import './registrations' -import './runners' -import './search' -import './services' -import './transcoding' -import './two-factor' -import './upload-quota' -import './user-notifications' -import './user-subscriptions' -import './users-admin' -import './users-emails' -import './video-blacklist' -import './video-captions' -import './video-channel-syncs' -import './video-channels' -import './video-comments' -import './video-files' -import './video-imports' -import './video-playlists' -import './video-storyboards' -import './video-source' -import './video-studio' -import './video-token' -import './videos-common-filters' -import './videos-history' -import './videos-overviews' -import './videos' -import './views' diff --git a/server/tests/api/check-params/jobs.ts b/server/tests/api/check-params/jobs.ts deleted file mode 100644 index 873da3955..000000000 --- a/server/tests/api/check-params/jobs.ts +++ /dev/null @@ -1,125 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared' -import { HttpStatusCode } from '@shared/models' -import { - cleanupTests, - createSingleServer, - makeGetRequest, - makePostBodyRequest, - PeerTubeServer, - setAccessTokensToServers -} from '@shared/server-commands' - -describe('Test jobs API validators', function () { - const path = '/api/v1/jobs/failed' - let server: PeerTubeServer - let userAccessToken = '' - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(120000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - - const user = { - username: 'user1', - password: 'my super password' - } - await server.users.create({ username: user.username, password: user.password }) - userAccessToken = await server.login.getAccessToken(user) - }) - - describe('When listing jobs', function () { - - it('Should fail with a bad state', async function () { - await makeGetRequest({ - url: server.url, - token: server.accessToken, - path: path + 'ade' - }) - }) - - it('Should fail with an incorrect job type', async function () { - await makeGetRequest({ - url: server.url, - token: server.accessToken, - path, - query: { - jobType: 'toto' - } - }) - }) - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a non authenticated user', async function () { - await makeGetRequest({ - url: server.url, - path, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with a non admin user', async function () { - await makeGetRequest({ - url: server.url, - path, - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - }) - - describe('When pausing/resuming the job queue', async function () { - const commands = [ 'pause', 'resume' ] - - it('Should fail with a non authenticated user', async function () { - for (const command of commands) { - await makePostBodyRequest({ - url: server.url, - path: '/api/v1/jobs/' + command, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - } - }) - - it('Should fail with a non admin user', async function () { - for (const command of commands) { - await makePostBodyRequest({ - url: server.url, - path: '/api/v1/jobs/' + command, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - } - }) - - it('Should succeed with the correct params', async function () { - for (const command of commands) { - await makePostBodyRequest({ - url: server.url, - path: '/api/v1/jobs/' + command, - token: server.accessToken, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/live.ts b/server/tests/api/check-params/live.ts deleted file mode 100644 index 5021db516..000000000 --- a/server/tests/api/check-params/live.ts +++ /dev/null @@ -1,589 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { buildAbsoluteFixturePath, omit } from '@shared/core-utils' -import { HttpStatusCode, LiveVideoLatencyMode, VideoCreateResult, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createSingleServer, - LiveCommand, - makePostBodyRequest, - makeUploadRequest, - PeerTubeServer, - sendRTMPStream, - setAccessTokensToServers, - stopFfmpeg -} from '@shared/server-commands' - -describe('Test video lives API validator', function () { - const path = '/api/v1/videos/live' - let server: PeerTubeServer - let userAccessToken = '' - let channelId: number - let video: VideoCreateResult - let videoIdNotLive: number - let command: LiveCommand - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - - await server.config.updateCustomSubConfig({ - newConfig: { - live: { - enabled: true, - latencySetting: { - enabled: false - }, - maxInstanceLives: 20, - maxUserLives: 20, - allowReplay: true - } - } - }) - - const username = 'user1' - const password = 'my super password' - await server.users.create({ username, password }) - userAccessToken = await server.login.getAccessToken({ username, password }) - - { - const { videoChannels } = await server.users.getMyInfo() - channelId = videoChannels[0].id - } - - { - videoIdNotLive = (await server.videos.quickUpload({ name: 'not live' })).id - } - - command = server.live - }) - - describe('When creating a live', function () { - let baseCorrectParams - - before(function () { - baseCorrectParams = { - name: 'my super name', - category: 5, - licence: 1, - language: 'pt', - nsfw: false, - commentsEnabled: true, - downloadEnabled: true, - waitTranscoding: true, - description: 'my super description', - support: 'my super support text', - tags: [ 'tag1', 'tag2' ], - privacy: VideoPrivacy.PUBLIC, - channelId, - saveReplay: false, - replaySettings: undefined, - permanentLive: false, - latencyMode: LiveVideoLatencyMode.DEFAULT - } - }) - - it('Should fail with nothing', async function () { - const fields = {} - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a long name', async function () { - const fields = { ...baseCorrectParams, name: 'super'.repeat(65) } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a bad category', async function () { - const fields = { ...baseCorrectParams, category: 125 } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a bad licence', async function () { - const fields = { ...baseCorrectParams, licence: 125 } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a bad language', async function () { - const fields = { ...baseCorrectParams, language: 'a'.repeat(15) } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a long description', async function () { - const fields = { ...baseCorrectParams, description: 'super'.repeat(2500) } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a long support text', async function () { - const fields = { ...baseCorrectParams, support: 'super'.repeat(201) } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail without a channel', async function () { - const fields = omit(baseCorrectParams, [ 'channelId' ]) - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a bad channel', async function () { - const fields = { ...baseCorrectParams, channelId: 545454 } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a bad privacy for replay settings', async function () { - const fields = { ...baseCorrectParams, saveReplay: true, replaySettings: { privacy: 999 } } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with another user channel', async function () { - const user = { - username: 'fake', - password: 'fake_password' - } - await server.users.create({ username: user.username, password: user.password }) - - const accessTokenUser = await server.login.getAccessToken(user) - const { videoChannels } = await server.users.getMyInfo({ token: accessTokenUser }) - const customChannelId = videoChannels[0].id - - const fields = { ...baseCorrectParams, channelId: customChannelId } - - await makePostBodyRequest({ url: server.url, path, token: userAccessToken, fields }) - }) - - it('Should fail with too many tags', async function () { - const fields = { ...baseCorrectParams, tags: [ 'tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6' ] } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a tag length too low', async function () { - const fields = { ...baseCorrectParams, tags: [ 'tag1', 't' ] } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a tag length too big', async function () { - const fields = { ...baseCorrectParams, tags: [ 'tag1', 'my_super_tag_too_long_long_long_long_long_long' ] } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with an incorrect thumbnail file', async function () { - const fields = baseCorrectParams - const attaches = { - thumbnailfile: buildAbsoluteFixturePath('video_short.mp4') - } - - await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches }) - }) - - it('Should fail with a big thumbnail file', async function () { - const fields = baseCorrectParams - const attaches = { - thumbnailfile: buildAbsoluteFixturePath('custom-preview-big.png') - } - - await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches }) - }) - - it('Should fail with an incorrect preview file', async function () { - const fields = baseCorrectParams - const attaches = { - previewfile: buildAbsoluteFixturePath('video_short.mp4') - } - - await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches }) - }) - - it('Should fail with a big preview file', async function () { - const fields = baseCorrectParams - const attaches = { - previewfile: buildAbsoluteFixturePath('custom-preview-big.png') - } - - await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches }) - }) - - it('Should fail with bad latency setting', async function () { - const fields = { ...baseCorrectParams, latencyMode: 42 } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail to set latency if the server does not allow it', async function () { - const fields = { ...baseCorrectParams, latencyMode: LiveVideoLatencyMode.HIGH_LATENCY } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should succeed with the correct parameters', async function () { - this.timeout(30000) - - const res = await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields: baseCorrectParams, - expectedStatus: HttpStatusCode.OK_200 - }) - - video = res.body.video - }) - - it('Should forbid if live is disabled', async function () { - await server.config.updateCustomSubConfig({ - newConfig: { - live: { - enabled: false - } - } - }) - - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields: baseCorrectParams, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should forbid to save replay if not enabled by the admin', async function () { - const fields = { ...baseCorrectParams, saveReplay: true, replaySettings: { privacy: VideoPrivacy.PUBLIC } } - - await server.config.updateCustomSubConfig({ - newConfig: { - live: { - enabled: true, - allowReplay: false - } - } - }) - - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should allow to save replay if enabled by the admin', async function () { - const fields = { ...baseCorrectParams, saveReplay: true, replaySettings: { privacy: VideoPrivacy.PUBLIC } } - - await server.config.updateCustomSubConfig({ - newConfig: { - live: { - enabled: true, - allowReplay: true - } - } - }) - - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - - it('Should not allow live if max instance lives is reached', async function () { - await server.config.updateCustomSubConfig({ - newConfig: { - live: { - enabled: true, - maxInstanceLives: 1 - } - } - }) - - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields: baseCorrectParams, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should not allow live if max user lives is reached', async function () { - await server.config.updateCustomSubConfig({ - newConfig: { - live: { - enabled: true, - maxInstanceLives: 20, - maxUserLives: 1 - } - } - }) - - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields: baseCorrectParams, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - }) - - describe('When getting live information', function () { - - it('Should fail with a bad access token', async function () { - await command.get({ token: 'toto', videoId: video.id, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should not display private information without access token', async function () { - const live = await command.get({ token: '', videoId: video.id }) - - expect(live.rtmpUrl).to.not.exist - expect(live.streamKey).to.not.exist - expect(live.latencyMode).to.exist - }) - - it('Should not display private information with token of another user', async function () { - const live = await command.get({ token: userAccessToken, videoId: video.id }) - - expect(live.rtmpUrl).to.not.exist - expect(live.streamKey).to.not.exist - expect(live.latencyMode).to.exist - }) - - it('Should display private information with appropriate token', async function () { - const live = await command.get({ videoId: video.id }) - - expect(live.rtmpUrl).to.exist - expect(live.streamKey).to.exist - expect(live.latencyMode).to.exist - }) - - it('Should fail with a bad video id', async function () { - await command.get({ videoId: 'toto', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with an unknown video id', async function () { - await command.get({ videoId: 454555, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should fail with a non live video', async function () { - await command.get({ videoId: videoIdNotLive, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should succeed with the correct params', async function () { - await command.get({ videoId: video.id }) - await command.get({ videoId: video.uuid }) - await command.get({ videoId: video.shortUUID }) - }) - }) - - describe('When getting live sessions', function () { - - it('Should fail with a bad access token', async function () { - await command.listSessions({ token: 'toto', videoId: video.id, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail without token', async function () { - await command.listSessions({ token: null, videoId: video.id, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with the token of another user', async function () { - await command.listSessions({ token: userAccessToken, videoId: video.id, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should fail with a bad video id', async function () { - await command.listSessions({ videoId: 'toto', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with an unknown video id', async function () { - await command.listSessions({ videoId: 454555, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should fail with a non live video', async function () { - await command.listSessions({ videoId: videoIdNotLive, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should succeed with the correct params', async function () { - await command.listSessions({ videoId: video.id }) - }) - }) - - describe('When getting live session of a replay', function () { - - it('Should fail with a bad video id', async function () { - await command.getReplaySession({ videoId: 'toto', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with an unknown video id', async function () { - await command.getReplaySession({ videoId: 454555, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should fail with a non replay video', async function () { - await command.getReplaySession({ videoId: videoIdNotLive, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - }) - - describe('When updating live information', async function () { - - it('Should fail without access token', async function () { - await command.update({ token: '', videoId: video.id, fields: {}, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with a bad access token', async function () { - await command.update({ token: 'toto', videoId: video.id, fields: {}, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with access token of another user', async function () { - await command.update({ token: userAccessToken, videoId: video.id, fields: {}, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should fail with a bad video id', async function () { - await command.update({ videoId: 'toto', fields: {}, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with an unknown video id', async function () { - await command.update({ videoId: 454555, fields: {}, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should fail with a non live video', async function () { - await command.update({ videoId: videoIdNotLive, fields: {}, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should fail with bad latency setting', async function () { - const fields = { latencyMode: 42 } - - await command.update({ videoId: video.id, fields, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with a bad privacy for replay settings', async function () { - const fields = { saveReplay: true, replaySettings: { privacy: 999 } } - - await command.update({ videoId: video.id, fields, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with save replay enabled but without replay settings', async function () { - await server.config.updateCustomSubConfig({ - newConfig: { - live: { - enabled: true, - allowReplay: true - } - } - }) - - const fields = { saveReplay: true } - - await command.update({ videoId: video.id, fields, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with save replay disabled and replay settings', async function () { - const fields = { saveReplay: false, replaySettings: { privacy: VideoPrivacy.INTERNAL } } - - await command.update({ videoId: video.id, fields, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with only replay settings when save replay is disabled', async function () { - const fields = { replaySettings: { privacy: VideoPrivacy.INTERNAL } } - - await command.update({ videoId: video.id, fields, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail to set latency if the server does not allow it', async function () { - const fields = { latencyMode: LiveVideoLatencyMode.HIGH_LATENCY } - - await command.update({ videoId: video.id, fields, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should succeed with the correct params', async function () { - await command.update({ videoId: video.id, fields: { saveReplay: false } }) - await command.update({ videoId: video.uuid, fields: { saveReplay: false } }) - await command.update({ videoId: video.shortUUID, fields: { saveReplay: false } }) - - await command.update({ videoId: video.id, fields: { saveReplay: true, replaySettings: { privacy: VideoPrivacy.PUBLIC } } }) - - }) - - it('Should fail to update replay status if replay is not allowed on the instance', async function () { - await server.config.updateCustomSubConfig({ - newConfig: { - live: { - enabled: true, - allowReplay: false - } - } - }) - - await command.update({ videoId: video.id, fields: { saveReplay: true }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should fail to update a live if it has already started', async function () { - this.timeout(40000) - - const live = await command.get({ videoId: video.id }) - - const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) - - await command.waitUntilPublished({ videoId: video.id }) - await command.update({ videoId: video.id, fields: {}, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - - await stopFfmpeg(ffmpegCommand) - }) - - it('Should fail to change live privacy if it has already started', async function () { - this.timeout(40000) - - const live = await command.get({ videoId: video.id }) - - const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) - - await command.waitUntilPublished({ videoId: video.id }) - - await server.videos.update({ - id: video.id, - attributes: { privacy: VideoPrivacy.PUBLIC } // Same privacy, it's fine - }) - - await server.videos.update({ - id: video.id, - attributes: { privacy: VideoPrivacy.UNLISTED }, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - - await stopFfmpeg(ffmpegCommand) - }) - - it('Should fail to stream twice in the save live', async function () { - this.timeout(40000) - - const live = await command.get({ videoId: video.id }) - - const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) - - await command.waitUntilPublished({ videoId: video.id }) - - await command.runAndTestStreamError({ videoId: video.id, shouldHaveError: true }) - - await stopFfmpeg(ffmpegCommand) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/logs.ts b/server/tests/api/check-params/logs.ts deleted file mode 100644 index 2496cee31..000000000 --- a/server/tests/api/check-params/logs.ts +++ /dev/null @@ -1,157 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { HttpStatusCode } from '@shared/models' -import { cleanupTests, createSingleServer, makeGetRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' - -describe('Test logs API validators', function () { - const path = '/api/v1/server/logs' - let server: PeerTubeServer - let userAccessToken = '' - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(120000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - - const user = { - username: 'user1', - password: 'my super password' - } - await server.users.create({ username: user.username, password: user.password }) - userAccessToken = await server.login.getAccessToken(user) - }) - - describe('When getting logs', function () { - - it('Should fail with a non authenticated user', async function () { - await makeGetRequest({ - url: server.url, - path, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with a non admin user', async function () { - await makeGetRequest({ - url: server.url, - path, - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with a missing startDate query', async function () { - await makeGetRequest({ - url: server.url, - path, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with a bad startDate query', async function () { - await makeGetRequest({ - url: server.url, - path, - token: server.accessToken, - query: { startDate: 'toto' }, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with a bad endDate query', async function () { - await makeGetRequest({ - url: server.url, - path, - token: server.accessToken, - query: { startDate: new Date().toISOString(), endDate: 'toto' }, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with a bad level parameter', async function () { - await makeGetRequest({ - url: server.url, - path, - token: server.accessToken, - query: { startDate: new Date().toISOString(), level: 'toto' }, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should succeed with the correct params', async function () { - await makeGetRequest({ - url: server.url, - path, - token: server.accessToken, - query: { startDate: new Date().toISOString() }, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - - describe('When creating client logs', function () { - const base = { - level: 'warn' as 'warn', - message: 'my super message', - url: 'https://example.com/toto' - } - const expectedStatus = HttpStatusCode.BAD_REQUEST_400 - - it('Should fail with an invalid level', async function () { - await server.logs.createLogClient({ payload: { ...base, level: '' as any }, expectedStatus }) - await server.logs.createLogClient({ payload: { ...base, level: undefined }, expectedStatus }) - await server.logs.createLogClient({ payload: { ...base, level: 'toto' as any }, expectedStatus }) - }) - - it('Should fail with an invalid message', async function () { - await server.logs.createLogClient({ payload: { ...base, message: undefined }, expectedStatus }) - await server.logs.createLogClient({ payload: { ...base, message: '' }, expectedStatus }) - await server.logs.createLogClient({ payload: { ...base, message: 'm'.repeat(2500) }, expectedStatus }) - }) - - it('Should fail with an invalid url', async function () { - await server.logs.createLogClient({ payload: { ...base, url: undefined }, expectedStatus }) - await server.logs.createLogClient({ payload: { ...base, url: 'toto' }, expectedStatus }) - }) - - it('Should fail with an invalid stackTrace', async function () { - await server.logs.createLogClient({ payload: { ...base, stackTrace: 's'.repeat(20000) }, expectedStatus }) - }) - - it('Should fail with an invalid userAgent', async function () { - await server.logs.createLogClient({ payload: { ...base, userAgent: 's'.repeat(500) }, expectedStatus }) - }) - - it('Should fail with an invalid meta', async function () { - await server.logs.createLogClient({ payload: { ...base, meta: 's'.repeat(10000) }, expectedStatus }) - }) - - it('Should succeed with the correct params', async function () { - await server.logs.createLogClient({ payload: { ...base, stackTrace: 'stackTrace', meta: '{toto}', userAgent: 'userAgent' } }) - }) - - it('Should rate limit log creation', async function () { - let fail = false - - for (let i = 0; i < 10; i++) { - try { - await server.logs.createLogClient({ token: null, payload: base }) - } catch { - fail = true - } - } - - expect(fail).to.be.true - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/metrics.ts b/server/tests/api/check-params/metrics.ts deleted file mode 100644 index 302bef4f5..000000000 --- a/server/tests/api/check-params/metrics.ts +++ /dev/null @@ -1,208 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { omit } from '@shared/core-utils' -import { HttpStatusCode, PlaybackMetricCreate, VideoResolution } from '@shared/models' -import { cleanupTests, createSingleServer, makePostBodyRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' - -describe('Test metrics API validators', function () { - let server: PeerTubeServer - let videoUUID: string - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(120000) - - server = await createSingleServer(1, { - open_telemetry: { - metrics: { - enabled: true - } - } - }) - - await setAccessTokensToServers([ server ]) - - const { uuid } = await server.videos.quickUpload({ name: 'video' }) - videoUUID = uuid - }) - - describe('When adding playback metrics', function () { - const path = '/api/v1/metrics/playback' - let baseParams: PlaybackMetricCreate - - before(function () { - baseParams = { - playerMode: 'p2p-media-loader', - resolution: VideoResolution.H_1080P, - fps: 30, - resolutionChanges: 1, - errors: 2, - p2pEnabled: true, - downloadedBytesP2P: 0, - downloadedBytesHTTP: 0, - uploadedBytesP2P: 0, - videoId: videoUUID - } - }) - - it('Should fail with an invalid resolution', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: { ...baseParams, resolution: 'toto' } - }) - }) - - it('Should fail with an invalid fps', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: { ...baseParams, fps: 'toto' } - }) - }) - - it('Should fail with a missing/invalid player mode', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: omit(baseParams, [ 'playerMode' ]) - }) - - await makePostBodyRequest({ - url: server.url, - path, - fields: { ...baseParams, playerMode: 'toto' } - }) - }) - - it('Should fail with an missing/invalid resolution changes', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: omit(baseParams, [ 'resolutionChanges' ]) - }) - - await makePostBodyRequest({ - url: server.url, - path, - fields: { ...baseParams, resolutionChanges: 'toto' } - }) - }) - - it('Should fail with an missing/invalid errors', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: omit(baseParams, [ 'errors' ]) - }) - - await makePostBodyRequest({ - url: server.url, - path, - fields: { ...baseParams, errors: 'toto' } - }) - }) - - it('Should fail with an missing/invalid downloadedBytesP2P', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: omit(baseParams, [ 'downloadedBytesP2P' ]) - }) - - await makePostBodyRequest({ - url: server.url, - path, - fields: { ...baseParams, downloadedBytesP2P: 'toto' } - }) - }) - - it('Should fail with an missing/invalid downloadedBytesHTTP', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: omit(baseParams, [ 'downloadedBytesHTTP' ]) - }) - - await makePostBodyRequest({ - url: server.url, - path, - fields: { ...baseParams, downloadedBytesHTTP: 'toto' } - }) - }) - - it('Should fail with an missing/invalid uploadedBytesP2P', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: omit(baseParams, [ 'uploadedBytesP2P' ]) - }) - - await makePostBodyRequest({ - url: server.url, - path, - fields: { ...baseParams, uploadedBytesP2P: 'toto' } - }) - }) - - it('Should fail with a missing/invalid p2pEnabled', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: omit(baseParams, [ 'p2pEnabled' ]) - }) - - await makePostBodyRequest({ - url: server.url, - path, - fields: { ...baseParams, p2pEnabled: 'toto' } - }) - }) - - it('Should fail with an invalid totalPeers', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: { ...baseParams, p2pPeers: 'toto' } - }) - }) - - it('Should fail with a bad video id', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: { ...baseParams, videoId: 'toto' } - }) - }) - - it('Should fail with an unknown video', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: { ...baseParams, videoId: 42 }, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should succeed with the correct params', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: baseParams, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - - await makePostBodyRequest({ - url: server.url, - path, - fields: { ...baseParams, p2pEnabled: false, totalPeers: 32 }, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/my-user.ts b/server/tests/api/check-params/my-user.ts deleted file mode 100644 index 18f32d46b..000000000 --- a/server/tests/api/check-params/my-user.ts +++ /dev/null @@ -1,491 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination, MockSmtpServer } from '@server/tests/shared' -import { buildAbsoluteFixturePath } from '@shared/core-utils' -import { HttpStatusCode, UserRole, VideoCreateResult } from '@shared/models' -import { - cleanupTests, - createSingleServer, - makeGetRequest, - makePutBodyRequest, - makeUploadRequest, - PeerTubeServer, - setAccessTokensToServers, - UsersCommand -} from '@shared/server-commands' - -describe('Test my user API validators', function () { - const path = '/api/v1/users/' - let userId: number - let rootId: number - let moderatorId: number - let video: VideoCreateResult - let server: PeerTubeServer - let userToken = '' - let moderatorToken = '' - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(30000) - - { - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - } - - { - const result = await server.users.generate('user1') - userToken = result.token - userId = result.userId - } - - { - const result = await server.users.generate('moderator1', UserRole.MODERATOR) - moderatorToken = result.token - } - - { - const result = await server.users.generate('moderator2', UserRole.MODERATOR) - moderatorId = result.userId - } - - { - video = await server.videos.upload() - } - }) - - describe('When updating my account', function () { - - it('Should fail with an invalid email attribute', async function () { - const fields = { - email: 'blabla' - } - - await makePutBodyRequest({ url: server.url, path: path + 'me', token: server.accessToken, fields }) - }) - - it('Should fail with a too small password', async function () { - const fields = { - currentPassword: 'password', - password: 'bla' - } - - await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) - }) - - it('Should fail with a too long password', async function () { - const fields = { - currentPassword: 'password', - password: 'super'.repeat(61) - } - - await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) - }) - - it('Should fail without the current password', async function () { - const fields = { - currentPassword: 'password', - password: 'super'.repeat(61) - } - - await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) - }) - - it('Should fail with an invalid current password', async function () { - const fields = { - currentPassword: 'my super password fail', - password: 'super'.repeat(61) - } - - await makePutBodyRequest({ - url: server.url, - path: path + 'me', - token: userToken, - fields, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with an invalid NSFW policy attribute', async function () { - const fields = { - nsfwPolicy: 'hello' - } - - await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) - }) - - it('Should fail with an invalid autoPlayVideo attribute', async function () { - const fields = { - autoPlayVideo: -1 - } - - await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) - }) - - it('Should fail with an invalid autoPlayNextVideo attribute', async function () { - const fields = { - autoPlayNextVideo: -1 - } - - await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) - }) - - it('Should fail with an invalid videosHistoryEnabled attribute', async function () { - const fields = { - videosHistoryEnabled: -1 - } - - await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) - }) - - it('Should fail with an non authenticated user', async function () { - const fields = { - currentPassword: 'password', - password: 'my super password' - } - - await makePutBodyRequest({ - url: server.url, - path: path + 'me', - token: 'super token', - fields, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with a too long description', async function () { - const fields = { - description: 'super'.repeat(201) - } - - await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) - }) - - it('Should fail with an invalid videoLanguages attribute', async function () { - { - const fields = { - videoLanguages: 'toto' - } - - await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) - } - - { - const languages = [] - for (let i = 0; i < 1000; i++) { - languages.push('fr') - } - - const fields = { - videoLanguages: languages - } - - await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) - } - }) - - it('Should fail with an invalid theme', async function () { - const fields = { theme: 'invalid' } - await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) - }) - - it('Should fail with an unknown theme', async function () { - const fields = { theme: 'peertube-theme-unknown' } - await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) - }) - - it('Should fail with invalid no modal attributes', async function () { - const keys = [ - 'noInstanceConfigWarningModal', - 'noAccountSetupWarningModal', - 'noWelcomeModal' - ] - - for (const key of keys) { - const fields = { - [key]: -1 - } - - await makePutBodyRequest({ url: server.url, path: path + 'me', token: userToken, fields }) - } - }) - - it('Should succeed to change password with the correct params', async function () { - const fields = { - currentPassword: 'password', - password: 'my super password', - nsfwPolicy: 'blur', - autoPlayVideo: false, - email: 'super_email@example.com', - theme: 'default', - noInstanceConfigWarningModal: true, - noWelcomeModal: true, - noAccountSetupWarningModal: true - } - - await makePutBodyRequest({ - url: server.url, - path: path + 'me', - token: userToken, - fields, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - - it('Should succeed without password change with the correct params', async function () { - const fields = { - nsfwPolicy: 'blur', - autoPlayVideo: false - } - - await makePutBodyRequest({ - url: server.url, - path: path + 'me', - token: userToken, - fields, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - describe('When updating my avatar', function () { - it('Should fail without an incorrect input file', async function () { - const fields = {} - const attaches = { - avatarfile: buildAbsoluteFixturePath('video_short.mp4') - } - await makeUploadRequest({ url: server.url, path: path + '/me/avatar/pick', token: server.accessToken, fields, attaches }) - }) - - it('Should fail with a big file', async function () { - const fields = {} - const attaches = { - avatarfile: buildAbsoluteFixturePath('avatar-big.png') - } - await makeUploadRequest({ url: server.url, path: path + '/me/avatar/pick', token: server.accessToken, fields, attaches }) - }) - - it('Should fail with an unauthenticated user', async function () { - const fields = {} - const attaches = { - avatarfile: buildAbsoluteFixturePath('avatar.png') - } - await makeUploadRequest({ - url: server.url, - path: path + '/me/avatar/pick', - fields, - attaches, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should succeed with the correct params', async function () { - const fields = {} - const attaches = { - avatarfile: buildAbsoluteFixturePath('avatar.png') - } - await makeUploadRequest({ - url: server.url, - path: path + '/me/avatar/pick', - token: server.accessToken, - fields, - attaches, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - - describe('When managing my scoped tokens', function () { - - it('Should fail to get my scoped tokens with an non authenticated user', async function () { - await server.users.getMyScopedTokens({ token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail to get my scoped tokens with a bad token', async function () { - await server.users.getMyScopedTokens({ token: 'bad', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - - }) - - it('Should succeed to get my scoped tokens', async function () { - await server.users.getMyScopedTokens() - }) - - it('Should fail to renew my scoped tokens with an non authenticated user', async function () { - await server.users.renewMyScopedTokens({ token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail to renew my scoped tokens with a bad token', async function () { - await server.users.renewMyScopedTokens({ token: 'bad', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should succeed to renew my scoped tokens', async function () { - await server.users.renewMyScopedTokens() - }) - }) - - describe('When getting my information', function () { - it('Should fail with a non authenticated user', async function () { - await server.users.getMyInfo({ token: 'fake_token', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should success with the correct parameters', async function () { - await server.users.getMyInfo({ token: userToken }) - }) - }) - - describe('When getting my video rating', function () { - let command: UsersCommand - - before(function () { - command = server.users - }) - - it('Should fail with a non authenticated user', async function () { - await command.getMyRating({ token: 'fake_token', videoId: video.id, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with an incorrect video uuid', async function () { - await command.getMyRating({ videoId: 'blabla', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with an unknown video', async function () { - await command.getMyRating({ videoId: '4da6fde3-88f7-4d16-b119-108df5630b06', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should succeed with the correct parameters', async function () { - await command.getMyRating({ videoId: video.id }) - await command.getMyRating({ videoId: video.uuid }) - await command.getMyRating({ videoId: video.shortUUID }) - }) - }) - - describe('When retrieving my global ratings', function () { - const path = '/api/v1/accounts/user1/ratings' - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, userToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, userToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, userToken) - }) - - it('Should fail with a unauthenticated user', async function () { - await makeGetRequest({ url: server.url, path, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with a another user', async function () { - await makeGetRequest({ url: server.url, path, token: server.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should fail with a bad type', async function () { - await makeGetRequest({ - url: server.url, - path, - token: userToken, - query: { rating: 'toto ' }, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should succeed with the correct params', async function () { - await makeGetRequest({ url: server.url, path, token: userToken, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - describe('When getting my global followers', function () { - const path = '/api/v1/accounts/user1/followers' - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, userToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, userToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, userToken) - }) - - it('Should fail with a unauthenticated user', async function () { - await makeGetRequest({ url: server.url, path, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with a another user', async function () { - await makeGetRequest({ url: server.url, path, token: server.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should succeed with the correct params', async function () { - await makeGetRequest({ url: server.url, path, token: userToken, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - describe('When blocking/unblocking/removing user', function () { - - it('Should fail with an incorrect id', async function () { - const options = { userId: 'blabla' as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 } - - await server.users.remove(options) - await server.users.banUser({ userId: 'blabla' as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await server.users.unbanUser({ userId: 'blabla' as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with the root user', async function () { - const options = { userId: rootId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 } - - await server.users.remove(options) - await server.users.banUser(options) - await server.users.unbanUser(options) - }) - - it('Should return 404 with a non existing id', async function () { - const options = { userId: 4545454, expectedStatus: HttpStatusCode.NOT_FOUND_404 } - - await server.users.remove(options) - await server.users.banUser(options) - await server.users.unbanUser(options) - }) - - it('Should fail with a non admin user', async function () { - const options = { userId, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 } - - await server.users.remove(options) - await server.users.banUser(options) - await server.users.unbanUser(options) - }) - - it('Should fail on a moderator with a moderator', async function () { - const options = { userId: moderatorId, token: moderatorToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 } - - await server.users.remove(options) - await server.users.banUser(options) - await server.users.unbanUser(options) - }) - - it('Should succeed on a user with a moderator', async function () { - const options = { userId, token: moderatorToken } - - await server.users.banUser(options) - await server.users.unbanUser(options) - }) - }) - - describe('When deleting our account', function () { - - it('Should fail with with the root account', async function () { - await server.users.deleteMe({ expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - }) - - after(async function () { - MockSmtpServer.Instance.kill() - - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/plugins.ts b/server/tests/api/check-params/plugins.ts deleted file mode 100644 index e08cd7ab8..000000000 --- a/server/tests/api/check-params/plugins.ts +++ /dev/null @@ -1,490 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared' -import { HttpStatusCode, PeerTubePlugin, PluginType } from '@shared/models' -import { - cleanupTests, - createSingleServer, - makeGetRequest, - makePostBodyRequest, - makePutBodyRequest, - PeerTubeServer, - setAccessTokensToServers -} from '@shared/server-commands' - -describe('Test server plugins API validators', function () { - let server: PeerTubeServer - let userAccessToken = null - - const npmPlugin = 'peertube-plugin-hello-world' - const pluginName = 'hello-world' - let npmVersion: string - - const themePlugin = 'peertube-theme-background-red' - const themeName = 'background-red' - let themeVersion: string - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(60000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - - const user = { - username: 'user1', - password: 'password' - } - - await server.users.create({ username: user.username, password: user.password }) - userAccessToken = await server.login.getAccessToken(user) - - { - const res = await server.plugins.install({ npmName: npmPlugin }) - const plugin = res.body as PeerTubePlugin - npmVersion = plugin.version - } - - { - const res = await server.plugins.install({ npmName: themePlugin }) - const plugin = res.body as PeerTubePlugin - themeVersion = plugin.version - } - }) - - describe('With static plugin routes', function () { - it('Should fail with an unknown plugin name/plugin version', async function () { - const paths = [ - '/plugins/' + pluginName + '/0.0.1/auth/fake-auth', - '/plugins/' + pluginName + '/0.0.1/static/images/chocobo.png', - '/plugins/' + pluginName + '/0.0.1/client-scripts/client/common-client-plugin.js', - '/themes/' + themeName + '/0.0.1/static/images/chocobo.png', - '/themes/' + themeName + '/0.0.1/client-scripts/client/video-watch-client-plugin.js', - '/themes/' + themeName + '/0.0.1/css/assets/style1.css' - ] - - for (const p of paths) { - await makeGetRequest({ url: server.url, path: p, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - } - }) - - it('Should fail when requesting a plugin in the theme path', async function () { - await makeGetRequest({ - url: server.url, - path: '/themes/' + pluginName + '/' + npmVersion + '/static/images/chocobo.png', - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail with invalid versions', async function () { - const paths = [ - '/plugins/' + pluginName + '/0.0.1.1/auth/fake-auth', - '/plugins/' + pluginName + '/0.0.1.1/static/images/chocobo.png', - '/plugins/' + pluginName + '/0.1/client-scripts/client/common-client-plugin.js', - '/themes/' + themeName + '/1/static/images/chocobo.png', - '/themes/' + themeName + '/0.0.1000a/client-scripts/client/video-watch-client-plugin.js', - '/themes/' + themeName + '/0.a.1/css/assets/style1.css' - ] - - for (const p of paths) { - await makeGetRequest({ url: server.url, path: p, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - } - }) - - it('Should fail with invalid paths', async function () { - const paths = [ - '/plugins/' + pluginName + '/' + npmVersion + '/static/images/../chocobo.png', - '/plugins/' + pluginName + '/' + npmVersion + '/client-scripts/../client/common-client-plugin.js', - '/themes/' + themeName + '/' + themeVersion + '/static/../images/chocobo.png', - '/themes/' + themeName + '/' + themeVersion + '/client-scripts/client/video-watch-client-plugin.js/..', - '/themes/' + themeName + '/' + themeVersion + '/css/../assets/style1.css' - ] - - for (const p of paths) { - await makeGetRequest({ url: server.url, path: p, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - } - }) - - it('Should fail with an unknown auth name', async function () { - const path = '/plugins/' + pluginName + '/' + npmVersion + '/auth/bad-auth' - - await makeGetRequest({ url: server.url, path, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should fail with an unknown static file', async function () { - const paths = [ - '/plugins/' + pluginName + '/' + npmVersion + '/static/fake/chocobo.png', - '/plugins/' + pluginName + '/' + npmVersion + '/client-scripts/client/fake.js', - '/themes/' + themeName + '/' + themeVersion + '/static/fake/chocobo.png', - '/themes/' + themeName + '/' + themeVersion + '/client-scripts/client/fake.js' - ] - - for (const p of paths) { - await makeGetRequest({ url: server.url, path: p, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - } - }) - - it('Should fail with an unknown CSS file', async function () { - await makeGetRequest({ - url: server.url, - path: '/themes/' + themeName + '/' + themeVersion + '/css/assets/fake.css', - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should succeed with the correct parameters', async function () { - const paths = [ - '/plugins/' + pluginName + '/' + npmVersion + '/static/images/chocobo.png', - '/plugins/' + pluginName + '/' + npmVersion + '/client-scripts/client/common-client-plugin.js', - '/themes/' + themeName + '/' + themeVersion + '/static/images/chocobo.png', - '/themes/' + themeName + '/' + themeVersion + '/client-scripts/client/video-watch-client-plugin.js', - '/themes/' + themeName + '/' + themeVersion + '/css/assets/style1.css' - ] - - for (const p of paths) { - await makeGetRequest({ url: server.url, path: p, expectedStatus: HttpStatusCode.OK_200 }) - } - - const authPath = '/plugins/' + pluginName + '/' + npmVersion + '/auth/fake-auth' - await makeGetRequest({ url: server.url, path: authPath, expectedStatus: HttpStatusCode.FOUND_302 }) - }) - }) - - describe('When listing available plugins/themes', function () { - const path = '/api/v1/plugins/available' - const baseQuery = { - search: 'super search', - pluginType: PluginType.PLUGIN, - currentPeerTubeEngine: '1.2.3' - } - - it('Should fail with an invalid token', async function () { - await makeGetRequest({ - url: server.url, - path, - token: 'fake_token', - query: baseQuery, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail if the user is not an administrator', async function () { - await makeGetRequest({ - url: server.url, - path, - token: userAccessToken, - query: baseQuery, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an invalid plugin type', async function () { - const query = { ...baseQuery, pluginType: 5 } - - await makeGetRequest({ - url: server.url, - path, - token: server.accessToken, - query - }) - }) - - it('Should fail with an invalid current peertube engine', async function () { - const query = { ...baseQuery, currentPeerTubeEngine: '1.0' } - - await makeGetRequest({ - url: server.url, - path, - token: server.accessToken, - query - }) - }) - - it('Should success with the correct parameters', async function () { - await makeGetRequest({ - url: server.url, - path, - token: server.accessToken, - query: baseQuery, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - - describe('When listing local plugins/themes', function () { - const path = '/api/v1/plugins' - const baseQuery = { - pluginType: PluginType.THEME - } - - it('Should fail with an invalid token', async function () { - await makeGetRequest({ - url: server.url, - path, - token: 'fake_token', - query: baseQuery, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail if the user is not an administrator', async function () { - await makeGetRequest({ - url: server.url, - path, - token: userAccessToken, - query: baseQuery, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an invalid plugin type', async function () { - const query = { ...baseQuery, pluginType: 5 } - - await makeGetRequest({ - url: server.url, - path, - token: server.accessToken, - query - }) - }) - - it('Should success with the correct parameters', async function () { - await makeGetRequest({ - url: server.url, - path, - token: server.accessToken, - query: baseQuery, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - - describe('When getting a plugin or the registered settings or public settings', function () { - const path = '/api/v1/plugins/' - - it('Should fail with an invalid token', async function () { - for (const suffix of [ npmPlugin, `${npmPlugin}/registered-settings` ]) { - await makeGetRequest({ - url: server.url, - path: path + suffix, - token: 'fake_token', - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - } - }) - - it('Should fail if the user is not an administrator', async function () { - for (const suffix of [ npmPlugin, `${npmPlugin}/registered-settings` ]) { - await makeGetRequest({ - url: server.url, - path: path + suffix, - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - } - }) - - it('Should fail with an invalid npm name', async function () { - for (const suffix of [ 'toto', 'toto/registered-settings', 'toto/public-settings' ]) { - await makeGetRequest({ - url: server.url, - path: path + suffix, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - } - - for (const suffix of [ 'peertube-plugin-TOTO', 'peertube-plugin-TOTO/registered-settings', 'peertube-plugin-TOTO/public-settings' ]) { - await makeGetRequest({ - url: server.url, - path: path + suffix, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - } - }) - - it('Should fail with an unknown plugin', async function () { - for (const suffix of [ 'peertube-plugin-toto', 'peertube-plugin-toto/registered-settings', 'peertube-plugin-toto/public-settings' ]) { - await makeGetRequest({ - url: server.url, - path: path + suffix, - token: server.accessToken, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - } - }) - - it('Should succeed with the correct parameters', async function () { - for (const suffix of [ npmPlugin, `${npmPlugin}/registered-settings`, `${npmPlugin}/public-settings` ]) { - await makeGetRequest({ - url: server.url, - path: path + suffix, - token: server.accessToken, - expectedStatus: HttpStatusCode.OK_200 - }) - } - }) - }) - - describe('When updating plugin settings', function () { - const path = '/api/v1/plugins/' - const settings = { setting1: 'value1' } - - it('Should fail with an invalid token', async function () { - await makePutBodyRequest({ - url: server.url, - path: path + npmPlugin + '/settings', - fields: { settings }, - token: 'fake_token', - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail if the user is not an administrator', async function () { - await makePutBodyRequest({ - url: server.url, - path: path + npmPlugin + '/settings', - fields: { settings }, - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with an invalid npm name', async function () { - await makePutBodyRequest({ - url: server.url, - path: path + 'toto/settings', - fields: { settings }, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - - await makePutBodyRequest({ - url: server.url, - path: path + 'peertube-plugin-TOTO/settings', - fields: { settings }, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with an unknown plugin', async function () { - await makePutBodyRequest({ - url: server.url, - path: path + 'peertube-plugin-toto/settings', - fields: { settings }, - token: server.accessToken, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should succeed with the correct parameters', async function () { - await makePutBodyRequest({ - url: server.url, - path: path + npmPlugin + '/settings', - fields: { settings }, - token: server.accessToken, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - describe('When installing/updating/uninstalling a plugin', function () { - const path = '/api/v1/plugins/' - - it('Should fail with an invalid token', async function () { - for (const suffix of [ 'install', 'update', 'uninstall' ]) { - await makePostBodyRequest({ - url: server.url, - path: path + suffix, - fields: { npmName: npmPlugin }, - token: 'fake_token', - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - } - }) - - it('Should fail if the user is not an administrator', async function () { - for (const suffix of [ 'install', 'update', 'uninstall' ]) { - await makePostBodyRequest({ - url: server.url, - path: path + suffix, - fields: { npmName: npmPlugin }, - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - } - }) - - it('Should fail with an invalid npm name', async function () { - for (const suffix of [ 'install', 'update', 'uninstall' ]) { - await makePostBodyRequest({ - url: server.url, - path: path + suffix, - fields: { npmName: 'toto' }, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - } - - for (const suffix of [ 'install', 'update', 'uninstall' ]) { - await makePostBodyRequest({ - url: server.url, - path: path + suffix, - fields: { npmName: 'peertube-plugin-TOTO' }, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - } - }) - - it('Should succeed with the correct parameters', async function () { - const it = [ - { suffix: 'install', status: HttpStatusCode.OK_200 }, - { suffix: 'update', status: HttpStatusCode.OK_200 }, - { suffix: 'uninstall', status: HttpStatusCode.NO_CONTENT_204 } - ] - - for (const obj of it) { - await makePostBodyRequest({ - url: server.url, - path: path + obj.suffix, - fields: { npmName: npmPlugin }, - token: server.accessToken, - expectedStatus: obj.status - }) - } - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/redundancy.ts b/server/tests/api/check-params/redundancy.ts deleted file mode 100644 index 73dfd489d..000000000 --- a/server/tests/api/check-params/redundancy.ts +++ /dev/null @@ -1,240 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared' -import { HttpStatusCode, VideoCreateResult } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - makeDeleteRequest, - makeGetRequest, - makePostBodyRequest, - makePutBodyRequest, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test server redundancy API validators', function () { - let servers: PeerTubeServer[] - let userAccessToken = null - let videoIdLocal: number - let videoRemote: VideoCreateResult - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(160000) - - servers = await createMultipleServers(2) - - await setAccessTokensToServers(servers) - await doubleFollow(servers[0], servers[1]) - - const user = { - username: 'user1', - password: 'password' - } - - await servers[0].users.create({ username: user.username, password: user.password }) - userAccessToken = await servers[0].login.getAccessToken(user) - - videoIdLocal = (await servers[0].videos.quickUpload({ name: 'video' })).id - - const remoteUUID = (await servers[1].videos.quickUpload({ name: 'video' })).uuid - - await waitJobs(servers) - - videoRemote = await servers[0].videos.get({ id: remoteUUID }) - }) - - describe('When listing redundancies', function () { - const path = '/api/v1/server/redundancy/videos' - - let url: string - let token: string - - before(function () { - url = servers[0].url - token = servers[0].accessToken - }) - - it('Should fail with an invalid token', async function () { - await makeGetRequest({ url, path, token: 'fake_token', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail if the user is not an administrator', async function () { - await makeGetRequest({ url, path, token: userAccessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(url, path, servers[0].accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(url, path, servers[0].accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(url, path, servers[0].accessToken) - }) - - it('Should fail with a bad target', async function () { - await makeGetRequest({ url, path, token, query: { target: 'bad target' } }) - }) - - it('Should fail without target', async function () { - await makeGetRequest({ url, path, token }) - }) - - it('Should succeed with the correct params', async function () { - await makeGetRequest({ url, path, token, query: { target: 'my-videos' }, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - describe('When manually adding a redundancy', function () { - const path = '/api/v1/server/redundancy/videos' - - let url: string - let token: string - - before(function () { - url = servers[0].url - token = servers[0].accessToken - }) - - it('Should fail with an invalid token', async function () { - await makePostBodyRequest({ url, path, token: 'fake_token', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail if the user is not an administrator', async function () { - await makePostBodyRequest({ url, path, token: userAccessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should fail without a video id', async function () { - await makePostBodyRequest({ url, path, token }) - }) - - it('Should fail with an incorrect video id', async function () { - await makePostBodyRequest({ url, path, token, fields: { videoId: 'peertube' } }) - }) - - it('Should fail with a not found video id', async function () { - await makePostBodyRequest({ url, path, token, fields: { videoId: 6565 }, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should fail with a local a video id', async function () { - await makePostBodyRequest({ url, path, token, fields: { videoId: videoIdLocal } }) - }) - - it('Should succeed with the correct params', async function () { - await makePostBodyRequest({ - url, - path, - token, - fields: { videoId: videoRemote.shortUUID }, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - - it('Should fail if the video is already duplicated', async function () { - this.timeout(30000) - - await waitJobs(servers) - - await makePostBodyRequest({ - url, - path, - token, - fields: { videoId: videoRemote.uuid }, - expectedStatus: HttpStatusCode.CONFLICT_409 - }) - }) - }) - - describe('When manually removing a redundancy', function () { - const path = '/api/v1/server/redundancy/videos/' - - let url: string - let token: string - - before(function () { - url = servers[0].url - token = servers[0].accessToken - }) - - it('Should fail with an invalid token', async function () { - await makeDeleteRequest({ url, path: path + '1', token: 'fake_token', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail if the user is not an administrator', async function () { - await makeDeleteRequest({ url, path: path + '1', token: userAccessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should fail with an incorrect video id', async function () { - await makeDeleteRequest({ url, path: path + 'toto', token }) - }) - - it('Should fail with a not found video redundancy', async function () { - await makeDeleteRequest({ url, path: path + '454545', token, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - }) - - describe('When updating server redundancy', function () { - const path = '/api/v1/server/redundancy' - - it('Should fail with an invalid token', async function () { - await makePutBodyRequest({ - url: servers[0].url, - path: path + '/' + servers[1].host, - fields: { redundancyAllowed: true }, - token: 'fake_token', - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail if the user is not an administrator', async function () { - await makePutBodyRequest({ - url: servers[0].url, - path: path + '/' + servers[1].host, - fields: { redundancyAllowed: true }, - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail if we do not follow this server', async function () { - await makePutBodyRequest({ - url: servers[0].url, - path: path + '/example.com', - fields: { redundancyAllowed: true }, - token: servers[0].accessToken, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail without de redundancyAllowed param', async function () { - await makePutBodyRequest({ - url: servers[0].url, - path: path + '/' + servers[1].host, - fields: { blabla: true }, - token: servers[0].accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should succeed with the correct parameters', async function () { - await makePutBodyRequest({ - url: servers[0].url, - path: path + '/' + servers[1].host, - fields: { redundancyAllowed: true }, - token: servers[0].accessToken, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/check-params/registrations.ts b/server/tests/api/check-params/registrations.ts deleted file mode 100644 index 8cbecdd07..000000000 --- a/server/tests/api/check-params/registrations.ts +++ /dev/null @@ -1,446 +0,0 @@ -import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared' -import { omit } from '@shared/core-utils' -import { HttpStatusCode, UserRole } from '@shared/models' -import { - cleanupTests, - createSingleServer, - makePostBodyRequest, - PeerTubeServer, - setAccessTokensToServers, - setDefaultAccountAvatar, - setDefaultChannelAvatar -} from '@shared/server-commands' - -describe('Test registrations API validators', function () { - let server: PeerTubeServer - let userToken: string - let moderatorToken: string - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - await setDefaultAccountAvatar([ server ]) - await setDefaultChannelAvatar([ server ]) - - await server.config.enableSignup(false); - - ({ token: moderatorToken } = await server.users.generate('moderator', UserRole.MODERATOR)); - ({ token: userToken } = await server.users.generate('user', UserRole.USER)) - }) - - describe('Register', function () { - const registrationPath = '/api/v1/users/register' - const registrationRequestPath = '/api/v1/users/registrations/request' - - const baseCorrectParams = { - username: 'user3', - displayName: 'super user', - email: 'test3@example.com', - password: 'my super password', - registrationReason: 'my super registration reason' - } - - describe('When registering a new user or requesting user registration', function () { - - async function check (fields: any, expectedStatus = HttpStatusCode.BAD_REQUEST_400) { - await server.config.enableSignup(false) - await makePostBodyRequest({ url: server.url, path: registrationPath, fields, expectedStatus }) - - await server.config.enableSignup(true) - await makePostBodyRequest({ url: server.url, path: registrationRequestPath, fields, expectedStatus }) - } - - it('Should fail with a too small username', async function () { - const fields = { ...baseCorrectParams, username: '' } - - await check(fields) - }) - - it('Should fail with a too long username', async function () { - const fields = { ...baseCorrectParams, username: 'super'.repeat(50) } - - await check(fields) - }) - - it('Should fail with an incorrect username', async function () { - const fields = { ...baseCorrectParams, username: 'my username' } - - await check(fields) - }) - - it('Should fail with a missing email', async function () { - const fields = omit(baseCorrectParams, [ 'email' ]) - - await check(fields) - }) - - it('Should fail with an invalid email', async function () { - const fields = { ...baseCorrectParams, email: 'test_example.com' } - - await check(fields) - }) - - it('Should fail with a too small password', async function () { - const fields = { ...baseCorrectParams, password: 'bla' } - - await check(fields) - }) - - it('Should fail with a too long password', async function () { - const fields = { ...baseCorrectParams, password: 'super'.repeat(61) } - - await check(fields) - }) - - it('Should fail if we register a user with the same username', async function () { - const fields = { ...baseCorrectParams, username: 'root' } - - await check(fields, HttpStatusCode.CONFLICT_409) - }) - - it('Should fail with a "peertube" username', async function () { - const fields = { ...baseCorrectParams, username: 'peertube' } - - await check(fields, HttpStatusCode.CONFLICT_409) - }) - - it('Should fail if we register a user with the same email', async function () { - const fields = { ...baseCorrectParams, email: 'admin' + server.internalServerNumber + '@example.com' } - - await check(fields, HttpStatusCode.CONFLICT_409) - }) - - it('Should fail with a bad display name', async function () { - const fields = { ...baseCorrectParams, displayName: 'a'.repeat(150) } - - await check(fields) - }) - - it('Should fail with a bad channel name', async function () { - const fields = { ...baseCorrectParams, channel: { name: '[]azf', displayName: 'toto' } } - - await check(fields) - }) - - it('Should fail with a bad channel display name', async function () { - const fields = { ...baseCorrectParams, channel: { name: 'toto', displayName: '' } } - - await check(fields) - }) - - it('Should fail with a channel name that is the same as username', async function () { - const source = { username: 'super_user', channel: { name: 'super_user', displayName: 'display name' } } - const fields = { ...baseCorrectParams, ...source } - - await check(fields) - }) - - it('Should fail with an existing channel', async function () { - const attributes = { name: 'existing_channel', displayName: 'hello', description: 'super description' } - await server.channels.create({ attributes }) - - const fields = { ...baseCorrectParams, channel: { name: 'existing_channel', displayName: 'toto' } } - - await check(fields, HttpStatusCode.CONFLICT_409) - }) - - it('Should fail on a server with registration disabled', async function () { - this.timeout(60000) - - await server.config.updateExistingSubConfig({ - newConfig: { - signup: { - enabled: false - } - } - }) - - await server.registrations.register({ username: 'user4', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await server.registrations.requestRegistration({ - username: 'user4', - registrationReason: 'reason', - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail if the user limit is reached', async function () { - this.timeout(60000) - - const { total } = await server.users.list() - - await server.config.enableSignup(false, total) - await server.registrations.register({ username: 'user42', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - - await server.config.enableSignup(true, total) - await server.registrations.requestRegistration({ - username: 'user42', - registrationReason: 'reason', - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should succeed if the user limit is not reached', async function () { - this.timeout(60000) - - const { total } = await server.users.list() - - await server.config.enableSignup(false, total + 1) - await server.registrations.register({ username: 'user43', expectedStatus: HttpStatusCode.NO_CONTENT_204 }) - - await server.config.enableSignup(true, total + 2) - await server.registrations.requestRegistration({ - username: 'user44', - registrationReason: 'reason', - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - - describe('On direct registration', function () { - - it('Should succeed with the correct params', async function () { - await server.config.enableSignup(false) - - const fields = { - username: 'user_direct_1', - displayName: 'super user direct 1', - email: 'user_direct_1@example.com', - password: 'my super password', - channel: { name: 'super_user_direct_1_channel', displayName: 'super user direct 1 channel' } - } - - await makePostBodyRequest({ url: server.url, path: registrationPath, fields, expectedStatus: HttpStatusCode.NO_CONTENT_204 }) - }) - - it('Should fail if the instance requires approval', async function () { - this.timeout(60000) - - await server.config.enableSignup(true) - await server.registrations.register({ username: 'user42', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - }) - - describe('On registration request', function () { - - before(async function () { - this.timeout(60000) - - await server.config.enableSignup(true) - }) - - it('Should fail with an invalid registration reason', async function () { - for (const registrationReason of [ '', 't', 't'.repeat(5000) ]) { - await server.registrations.requestRegistration({ - username: 'user_request_1', - registrationReason, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - } - }) - - it('Should succeed with the correct params', async function () { - await server.registrations.requestRegistration({ - username: 'user_request_2', - registrationReason: 'tt', - channel: { - displayName: 'my user request 2 channel', - name: 'user_request_2_channel' - } - }) - }) - - it('Should fail if the username is already awaiting registration approval', async function () { - await server.registrations.requestRegistration({ - username: 'user_request_2', - registrationReason: 'tt', - channel: { - displayName: 'my user request 42 channel', - name: 'user_request_42_channel' - }, - expectedStatus: HttpStatusCode.CONFLICT_409 - }) - }) - - it('Should fail if the email is already awaiting registration approval', async function () { - await server.registrations.requestRegistration({ - username: 'user42', - email: 'user_request_2@example.com', - registrationReason: 'tt', - channel: { - displayName: 'my user request 42 channel', - name: 'user_request_42_channel' - }, - expectedStatus: HttpStatusCode.CONFLICT_409 - }) - }) - - it('Should fail if the channel is already awaiting registration approval', async function () { - await server.registrations.requestRegistration({ - username: 'user42', - registrationReason: 'tt', - channel: { - displayName: 'my user request 2 channel', - name: 'user_request_2_channel' - }, - expectedStatus: HttpStatusCode.CONFLICT_409 - }) - }) - - it('Should fail if the instance does not require approval', async function () { - this.timeout(60000) - - await server.config.enableSignup(false) - - await server.registrations.requestRegistration({ - username: 'user42', - registrationReason: 'toto', - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - }) - }) - - describe('Registrations accept/reject', function () { - let id1: number - let id2: number - - before(async function () { - this.timeout(60000) - - await server.config.enableSignup(true); - - ({ id: id1 } = await server.registrations.requestRegistration({ username: 'request_2', registrationReason: 'toto' })); - ({ id: id2 } = await server.registrations.requestRegistration({ username: 'request_3', registrationReason: 'toto' })) - }) - - it('Should fail to accept/reject registration without token', async function () { - const options = { id: id1, moderationResponse: 'tt', token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 } - await server.registrations.accept(options) - await server.registrations.reject(options) - }) - - it('Should fail to accept/reject registration with a non moderator user', async function () { - const options = { id: id1, moderationResponse: 'tt', token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 } - await server.registrations.accept(options) - await server.registrations.reject(options) - }) - - it('Should fail to accept/reject registration with a bad registration id', async function () { - { - const options = { id: 't' as any, moderationResponse: 'tt', token: moderatorToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 } - await server.registrations.accept(options) - await server.registrations.reject(options) - } - - { - const options = { id: 42, moderationResponse: 'tt', token: moderatorToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 } - await server.registrations.accept(options) - await server.registrations.reject(options) - } - }) - - it('Should fail to accept/reject registration with a bad moderation resposne', async function () { - for (const moderationResponse of [ '', 't', 't'.repeat(5000) ]) { - const options = { id: id1, moderationResponse, token: moderatorToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 } - await server.registrations.accept(options) - await server.registrations.reject(options) - } - }) - - it('Should succeed to accept a registration', async function () { - await server.registrations.accept({ id: id1, moderationResponse: 'tt', token: moderatorToken }) - }) - - it('Should succeed to reject a registration', async function () { - await server.registrations.reject({ id: id2, moderationResponse: 'tt', token: moderatorToken }) - }) - - it('Should fail to accept/reject a registration that was already accepted/rejected', async function () { - for (const id of [ id1, id2 ]) { - const options = { id, moderationResponse: 'tt', token: moderatorToken, expectedStatus: HttpStatusCode.CONFLICT_409 } - await server.registrations.accept(options) - await server.registrations.reject(options) - } - }) - }) - - describe('Registrations deletion', function () { - let id1: number - let id2: number - let id3: number - - before(async function () { - ({ id: id1 } = await server.registrations.requestRegistration({ username: 'request_4', registrationReason: 'toto' })); - ({ id: id2 } = await server.registrations.requestRegistration({ username: 'request_5', registrationReason: 'toto' })); - ({ id: id3 } = await server.registrations.requestRegistration({ username: 'request_6', registrationReason: 'toto' })) - - await server.registrations.accept({ id: id2, moderationResponse: 'tt' }) - await server.registrations.reject({ id: id3, moderationResponse: 'tt' }) - }) - - it('Should fail to delete registration without token', async function () { - await server.registrations.delete({ id: id1, token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail to delete registration with a non moderator user', async function () { - await server.registrations.delete({ id: id1, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should fail to delete registration with a bad registration id', async function () { - await server.registrations.delete({ id: 't' as any, token: moderatorToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await server.registrations.delete({ id: 42, token: moderatorToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should succeed with the correct params', async function () { - await server.registrations.delete({ id: id1, token: moderatorToken }) - await server.registrations.delete({ id: id2, token: moderatorToken }) - await server.registrations.delete({ id: id3, token: moderatorToken }) - }) - }) - - describe('Listing registrations', function () { - const path = '/api/v1/users/registrations' - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a non authenticated user', async function () { - await server.registrations.list({ - token: null, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with a non admin user', async function () { - await server.registrations.list({ - token: userToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should succeed with the correct params', async function () { - await server.registrations.list({ - token: moderatorToken, - search: 'toto' - }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/runners.ts b/server/tests/api/check-params/runners.ts deleted file mode 100644 index 0e5012da5..000000000 --- a/server/tests/api/check-params/runners.ts +++ /dev/null @@ -1,910 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { basename } from 'path' -import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared' -import { - HttpStatusCode, - isVideoStudioTaskIntro, - RunnerJob, - RunnerJobState, - RunnerJobStudioTranscodingPayload, - RunnerJobSuccessPayload, - RunnerJobUpdatePayload, - VideoPrivacy, - VideoStudioTaskIntro -} from '@shared/models' -import { - cleanupTests, - createSingleServer, - makePostBodyRequest, - PeerTubeServer, - sendRTMPStream, - setAccessTokensToServers, - setDefaultVideoChannel, - stopFfmpeg, - VideoStudioCommand, - waitJobs -} from '@shared/server-commands' - -const badUUID = '910ec12a-d9e6-458b-a274-0abb655f9464' - -describe('Test managing runners', function () { - let server: PeerTubeServer - - let userToken: string - - let registrationTokenId: number - let registrationToken: string - - let runnerToken: string - let runnerToken2: string - - let completedJobToken: string - let completedJobUUID: string - - let cancelledJobToken: string - let cancelledJobUUID: string - - before(async function () { - this.timeout(120000) - - const config = { - rates_limit: { - api: { - max: 5000 - } - } - } - - server = await createSingleServer(1, config) - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - userToken = await server.users.generateUserAndToken('user1') - - const { data } = await server.runnerRegistrationTokens.list() - registrationToken = data[0].registrationToken - registrationTokenId = data[0].id - - await server.config.enableTranscoding({ hls: true, webVideo: true }) - await server.config.enableStudio() - await server.config.enableRemoteTranscoding() - await server.config.enableRemoteStudio() - - runnerToken = await server.runners.autoRegisterRunner() - runnerToken2 = await server.runners.autoRegisterRunner() - - { - await server.videos.quickUpload({ name: 'video 1' }) - await server.videos.quickUpload({ name: 'video 2' }) - - await waitJobs([ server ]) - - { - const job = await server.runnerJobs.autoProcessWebVideoJob(runnerToken) - completedJobToken = job.jobToken - completedJobUUID = job.uuid - } - - { - const { job } = await server.runnerJobs.autoAccept({ runnerToken }) - cancelledJobToken = job.jobToken - cancelledJobUUID = job.uuid - await server.runnerJobs.cancelByAdmin({ jobUUID: cancelledJobUUID }) - } - } - }) - - describe('Managing runner registration tokens', function () { - - describe('Common', function () { - - it('Should fail to generate, list or delete runner registration token without oauth token', async function () { - const expectedStatus = HttpStatusCode.UNAUTHORIZED_401 - - await server.runnerRegistrationTokens.generate({ token: null, expectedStatus }) - await server.runnerRegistrationTokens.list({ token: null, expectedStatus }) - await server.runnerRegistrationTokens.delete({ token: null, id: registrationTokenId, expectedStatus }) - }) - - it('Should fail to generate, list or delete runner registration token without admin rights', async function () { - const expectedStatus = HttpStatusCode.FORBIDDEN_403 - - await server.runnerRegistrationTokens.generate({ token: userToken, expectedStatus }) - await server.runnerRegistrationTokens.list({ token: userToken, expectedStatus }) - await server.runnerRegistrationTokens.delete({ token: userToken, id: registrationTokenId, expectedStatus }) - }) - }) - - describe('Delete', function () { - - it('Should fail to delete with a bad id', async function () { - await server.runnerRegistrationTokens.delete({ id: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - }) - - describe('List', function () { - const path = '/api/v1/runners/registration-tokens' - - it('Should fail to list with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, server.accessToken) - }) - - it('Should fail to list with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, server.accessToken) - }) - - it('Should fail to list with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, server.accessToken) - }) - - it('Should succeed to list with the correct params', async function () { - await server.runnerRegistrationTokens.list({ start: 0, count: 5, sort: '-createdAt' }) - }) - }) - }) - - describe('Managing runners', function () { - let toDeleteId: number - - describe('Register', function () { - const name = 'runner name' - - it('Should fail with a bad registration token', async function () { - const expectedStatus = HttpStatusCode.BAD_REQUEST_400 - - await server.runners.register({ name, registrationToken: 'a'.repeat(4000), expectedStatus }) - await server.runners.register({ name, registrationToken: null, expectedStatus }) - }) - - it('Should fail with an unknown registration token', async function () { - await server.runners.register({ name, registrationToken: 'aaa', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should fail with a bad name', async function () { - const expectedStatus = HttpStatusCode.BAD_REQUEST_400 - - await server.runners.register({ name: '', registrationToken, expectedStatus }) - await server.runners.register({ name: 'a'.repeat(200), registrationToken, expectedStatus }) - }) - - it('Should fail with an invalid description', async function () { - const expectedStatus = HttpStatusCode.BAD_REQUEST_400 - - await server.runners.register({ name, description: '', registrationToken, expectedStatus }) - await server.runners.register({ name, description: 'a'.repeat(5000), registrationToken, expectedStatus }) - }) - - it('Should succeed with the correct params', async function () { - const { id } = await server.runners.register({ name, description: 'super description', registrationToken }) - - toDeleteId = id - }) - - it('Should fail with the same runner name', async function () { - await server.runners.register({ - name, - description: 'super description', - registrationToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - }) - - describe('Delete', function () { - - it('Should fail without oauth token', async function () { - await server.runners.delete({ token: null, id: toDeleteId, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail without admin rights', async function () { - await server.runners.delete({ token: userToken, id: toDeleteId, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should fail with a bad id', async function () { - await server.runners.delete({ id: 'hi' as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with an unknown id', async function () { - await server.runners.delete({ id: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should succeed with the correct params', async function () { - await server.runners.delete({ id: toDeleteId }) - }) - }) - - describe('List', function () { - const path = '/api/v1/runners' - - it('Should fail without oauth token', async function () { - await server.runners.list({ token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail without admin rights', async function () { - await server.runners.list({ token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should fail to list with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, server.accessToken) - }) - - it('Should fail to list with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, server.accessToken) - }) - - it('Should fail to list with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an invalid state', async function () { - await server.runners.list({ start: 0, count: 5, sort: '-createdAt' }) - }) - - it('Should succeed to list with the correct params', async function () { - await server.runners.list({ start: 0, count: 5, sort: '-createdAt' }) - }) - }) - - }) - - describe('Runner jobs by admin', function () { - - describe('Cancel', function () { - let jobUUID: string - - before(async function () { - this.timeout(60000) - - await server.videos.quickUpload({ name: 'video' }) - await waitJobs([ server ]) - - const { availableJobs } = await server.runnerJobs.request({ runnerToken }) - jobUUID = availableJobs[0].uuid - }) - - it('Should fail without oauth token', async function () { - await server.runnerJobs.cancelByAdmin({ token: null, jobUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail without admin rights', async function () { - await server.runnerJobs.cancelByAdmin({ token: userToken, jobUUID, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should fail with a bad job uuid', async function () { - await server.runnerJobs.cancelByAdmin({ jobUUID: 'hello', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with an unknown job uuid', async function () { - const jobUUID = badUUID - await server.runnerJobs.cancelByAdmin({ jobUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should fail with an already cancelled job', async function () { - await server.runnerJobs.cancelByAdmin({ jobUUID: cancelledJobUUID, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should succeed with the correct params', async function () { - await server.runnerJobs.cancelByAdmin({ jobUUID }) - }) - }) - - describe('List', function () { - const path = '/api/v1/runners/jobs' - - it('Should fail without oauth token', async function () { - await server.runnerJobs.list({ token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail without admin rights', async function () { - await server.runnerJobs.list({ token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should fail to list with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, server.accessToken) - }) - - it('Should fail to list with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, server.accessToken) - }) - - it('Should fail to list with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an invalid state', async function () { - await server.runnerJobs.list({ start: 0, count: 5, sort: '-createdAt', stateOneOf: 42 as any }) - await server.runnerJobs.list({ start: 0, count: 5, sort: '-createdAt', stateOneOf: [ 42 ] as any }) - }) - - it('Should succeed with the correct params', async function () { - await server.runnerJobs.list({ start: 0, count: 5, sort: '-createdAt', stateOneOf: [ RunnerJobState.COMPLETED ] }) - }) - }) - - describe('Delete', function () { - let jobUUID: string - - before(async function () { - this.timeout(60000) - - await server.videos.quickUpload({ name: 'video' }) - await waitJobs([ server ]) - - const { availableJobs } = await server.runnerJobs.request({ runnerToken }) - jobUUID = availableJobs[0].uuid - }) - - it('Should fail without oauth token', async function () { - await server.runnerJobs.deleteByAdmin({ token: null, jobUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail without admin rights', async function () { - await server.runnerJobs.deleteByAdmin({ token: userToken, jobUUID, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should fail with a bad job uuid', async function () { - await server.runnerJobs.deleteByAdmin({ jobUUID: 'hello', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with an unknown job uuid', async function () { - const jobUUID = badUUID - await server.runnerJobs.deleteByAdmin({ jobUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should succeed with the correct params', async function () { - await server.runnerJobs.deleteByAdmin({ jobUUID }) - }) - }) - - }) - - describe('Runner jobs by runners', function () { - let jobUUID: string - let jobToken: string - let videoUUID: string - - let jobUUID2: string - let jobToken2: string - - let videoUUID2: string - - let pendingUUID: string - - let videoStudioUUID: string - let studioFile: string - - let liveAcceptedJob: RunnerJob & { jobToken: string } - let studioAcceptedJob: RunnerJob & { jobToken: string } - - async function fetchVideoInputFiles (options: { - jobUUID: string - videoUUID: string - runnerToken: string - jobToken: string - expectedStatus: HttpStatusCode - }) { - const { jobUUID, expectedStatus, videoUUID, runnerToken, jobToken } = options - - const basePath = '/api/v1/runners/jobs/' + jobUUID + '/files/videos/' + videoUUID - const paths = [ `${basePath}/max-quality`, `${basePath}/previews/max-quality` ] - - for (const path of paths) { - await makePostBodyRequest({ url: server.url, path, fields: { runnerToken, jobToken }, expectedStatus }) - } - } - - async function fetchStudioFiles (options: { - jobUUID: string - videoUUID: string - runnerToken: string - jobToken: string - studioFile?: string - expectedStatus: HttpStatusCode - }) { - const { jobUUID, expectedStatus, videoUUID, runnerToken, jobToken, studioFile } = options - - const path = `/api/v1/runners/jobs/${jobUUID}/files/videos/${videoUUID}/studio/task-files/${studioFile}` - - await makePostBodyRequest({ url: server.url, path, fields: { runnerToken, jobToken }, expectedStatus }) - } - - before(async function () { - this.timeout(120000) - - { - await server.runnerJobs.cancelAllJobs({ state: RunnerJobState.PENDING }) - } - - { - const { uuid } = await server.videos.quickUpload({ name: 'video' }) - videoUUID = uuid - - await waitJobs([ server ]) - - const { job } = await server.runnerJobs.autoAccept({ runnerToken }) - jobUUID = job.uuid - jobToken = job.jobToken - } - - { - const { uuid } = await server.videos.quickUpload({ name: 'video' }) - videoUUID2 = uuid - - await waitJobs([ server ]) - - const { job } = await server.runnerJobs.autoAccept({ runnerToken: runnerToken2 }) - jobUUID2 = job.uuid - jobToken2 = job.jobToken - } - - { - await server.videos.quickUpload({ name: 'video' }) - await waitJobs([ server ]) - - const { availableJobs } = await server.runnerJobs.request({ runnerToken }) - pendingUUID = availableJobs[0].uuid - } - - { - await server.config.disableTranscoding() - - const { uuid } = await server.videos.quickUpload({ name: 'video studio' }) - videoStudioUUID = uuid - - await server.config.enableTranscoding({ hls: true, webVideo: true }) - await server.config.enableStudio() - - await server.videoStudio.createEditionTasks({ - videoId: videoStudioUUID, - tasks: VideoStudioCommand.getComplexTask() - }) - - const { job } = await server.runnerJobs.autoAccept({ runnerToken, type: 'video-studio-transcoding' }) - studioAcceptedJob = job - - const tasks = (job.payload as RunnerJobStudioTranscodingPayload).tasks - const fileUrl = (tasks.find(t => isVideoStudioTaskIntro(t)) as VideoStudioTaskIntro).options.file as string - studioFile = basename(fileUrl) - } - - { - await server.config.enableLive({ - allowReplay: false, - resolutions: 'max', - transcoding: true - }) - - const { live } = await server.live.quickCreate({ permanentLive: true, saveReplay: false, privacy: VideoPrivacy.PUBLIC }) - - const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) - await waitJobs([ server ]) - - await server.runnerJobs.requestLiveJob(runnerToken) - - const { job } = await server.runnerJobs.autoAccept({ runnerToken, type: 'live-rtmp-hls-transcoding' }) - liveAcceptedJob = job - - await stopFfmpeg(ffmpegCommand) - } - }) - - describe('Common runner tokens validations', function () { - - async function testEndpoints (options: { - jobUUID: string - runnerToken: string - jobToken: string - expectedStatus: HttpStatusCode - }) { - await server.runnerJobs.abort({ ...options, reason: 'reason' }) - await server.runnerJobs.update({ ...options }) - await server.runnerJobs.error({ ...options, message: 'message' }) - await server.runnerJobs.success({ ...options, payload: { videoFile: 'video_short.mp4' } }) - } - - it('Should fail with an invalid job uuid', async function () { - const options = { jobUUID: 'a', runnerToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 } - - await testEndpoints({ ...options, jobToken }) - await fetchVideoInputFiles({ ...options, videoUUID, jobToken }) - await fetchStudioFiles({ ...options, videoUUID, jobToken: studioAcceptedJob.jobToken, studioFile }) - }) - - it('Should fail with an unknown job uuid', async function () { - const options = { jobUUID: badUUID, runnerToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 } - - await testEndpoints({ ...options, jobToken }) - await fetchVideoInputFiles({ ...options, videoUUID, jobToken }) - await fetchStudioFiles({ ...options, jobToken: studioAcceptedJob.jobToken, videoUUID, studioFile }) - }) - - it('Should fail with an invalid runner token', async function () { - const options = { runnerToken: '', expectedStatus: HttpStatusCode.BAD_REQUEST_400 } - - await testEndpoints({ ...options, jobUUID, jobToken }) - await fetchVideoInputFiles({ ...options, jobUUID, videoUUID, jobToken }) - await fetchStudioFiles({ - ...options, - jobToken: studioAcceptedJob.jobToken, - jobUUID: studioAcceptedJob.uuid, - videoUUID: videoStudioUUID, - studioFile - }) - }) - - it('Should fail with an unknown runner token', async function () { - const options = { runnerToken: badUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 } - - await testEndpoints({ ...options, jobUUID, jobToken }) - await fetchVideoInputFiles({ ...options, jobUUID, videoUUID, jobToken }) - await fetchStudioFiles({ - ...options, - jobToken: studioAcceptedJob.jobToken, - jobUUID: studioAcceptedJob.uuid, - videoUUID: videoStudioUUID, - studioFile - }) - }) - - it('Should fail with an invalid job token job uuid', async function () { - const options = { runnerToken, jobToken: '', expectedStatus: HttpStatusCode.BAD_REQUEST_400 } - - await testEndpoints({ ...options, jobUUID }) - await fetchVideoInputFiles({ ...options, jobUUID, videoUUID }) - await fetchStudioFiles({ ...options, jobUUID: studioAcceptedJob.uuid, videoUUID: videoStudioUUID, studioFile }) - }) - - it('Should fail with an unknown job token job uuid', async function () { - const options = { runnerToken, jobToken: badUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 } - - await testEndpoints({ ...options, jobUUID }) - await fetchVideoInputFiles({ ...options, jobUUID, videoUUID }) - await fetchStudioFiles({ ...options, jobUUID: studioAcceptedJob.uuid, videoUUID: videoStudioUUID, studioFile }) - }) - - it('Should fail with a runner token not associated to this job', async function () { - const options = { runnerToken: runnerToken2, expectedStatus: HttpStatusCode.NOT_FOUND_404 } - - await testEndpoints({ ...options, jobUUID, jobToken }) - await fetchVideoInputFiles({ ...options, jobUUID, videoUUID, jobToken }) - await fetchStudioFiles({ - ...options, - jobToken: studioAcceptedJob.jobToken, - jobUUID: studioAcceptedJob.uuid, - videoUUID: videoStudioUUID, - studioFile - }) - }) - - it('Should fail with a job uuid not associated to the job token', async function () { - { - const options = { jobUUID: jobUUID2, runnerToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 } - - await testEndpoints({ ...options, jobToken }) - await fetchVideoInputFiles({ ...options, jobToken, videoUUID }) - await fetchStudioFiles({ ...options, jobToken: studioAcceptedJob.jobToken, videoUUID: videoStudioUUID, studioFile }) - } - - { - const options = { runnerToken, jobToken: jobToken2, expectedStatus: HttpStatusCode.NOT_FOUND_404 } - - await testEndpoints({ ...options, jobUUID }) - await fetchVideoInputFiles({ ...options, jobUUID, videoUUID }) - await fetchStudioFiles({ ...options, jobUUID: studioAcceptedJob.uuid, videoUUID: videoStudioUUID, studioFile }) - } - }) - }) - - describe('Unregister', function () { - - it('Should fail without a runner token', async function () { - await server.runners.unregister({ runnerToken: null, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with a bad a runner token', async function () { - await server.runners.unregister({ runnerToken: '', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with an unknown runner token', async function () { - await server.runners.unregister({ runnerToken: badUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - }) - - describe('Request', function () { - - it('Should fail without a runner token', async function () { - await server.runnerJobs.request({ runnerToken: null, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with a bad a runner token', async function () { - await server.runnerJobs.request({ runnerToken: '', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with an unknown runner token', async function () { - await server.runnerJobs.request({ runnerToken: badUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - }) - - describe('Accept', function () { - - it('Should fail with a bad a job uuid', async function () { - await server.runnerJobs.accept({ jobUUID: '', runnerToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with an unknown job uuid', async function () { - await server.runnerJobs.accept({ jobUUID: badUUID, runnerToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should fail with a job not in pending state', async function () { - await server.runnerJobs.accept({ jobUUID: completedJobUUID, runnerToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await server.runnerJobs.accept({ jobUUID: cancelledJobUUID, runnerToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail without a runner token', async function () { - await server.runnerJobs.accept({ jobUUID: pendingUUID, runnerToken: null, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with a bad a runner token', async function () { - await server.runnerJobs.accept({ jobUUID: pendingUUID, runnerToken: '', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with an unknown runner token', async function () { - await server.runnerJobs.accept({ jobUUID: pendingUUID, runnerToken: badUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - }) - - describe('Abort', function () { - - it('Should fail without a reason', async function () { - await server.runnerJobs.abort({ jobUUID, jobToken, runnerToken, reason: null, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with a bad reason', async function () { - const reason = 'reason'.repeat(5000) - await server.runnerJobs.abort({ jobUUID, jobToken, runnerToken, reason, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with a job not in processing state', async function () { - await server.runnerJobs.abort({ - jobUUID: completedJobUUID, - jobToken: completedJobToken, - runnerToken, - reason: 'reason', - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - }) - - describe('Update', function () { - - describe('Common', function () { - - it('Should fail with an invalid progress', async function () { - await server.runnerJobs.update({ jobUUID, jobToken, runnerToken, progress: 101, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with a job not in processing state', async function () { - await server.runnerJobs.update({ - jobUUID: cancelledJobUUID, - jobToken: cancelledJobToken, - runnerToken, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - }) - - describe('Live RTMP to HLS', function () { - const base: RunnerJobUpdatePayload = { - masterPlaylistFile: 'live/master.m3u8', - resolutionPlaylistFilename: '0.m3u8', - resolutionPlaylistFile: 'live/1.m3u8', - type: 'add-chunk', - videoChunkFile: 'live/1-000069.ts', - videoChunkFilename: '1-000068.ts' - } - - function testUpdate (payload: RunnerJobUpdatePayload) { - return server.runnerJobs.update({ - jobUUID: liveAcceptedJob.uuid, - jobToken: liveAcceptedJob.jobToken, - payload, - runnerToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - } - - it('Should fail with an invalid resolutionPlaylistFilename', async function () { - await testUpdate({ ...base, resolutionPlaylistFilename: undefined }) - await testUpdate({ ...base, resolutionPlaylistFilename: 'coucou/hello' }) - await testUpdate({ ...base, resolutionPlaylistFilename: 'hello' }) - }) - - it('Should fail with an invalid videoChunkFilename', async function () { - await testUpdate({ ...base, resolutionPlaylistFilename: undefined }) - await testUpdate({ ...base, resolutionPlaylistFilename: 'coucou/hello' }) - await testUpdate({ ...base, resolutionPlaylistFilename: 'hello' }) - }) - - it('Should fail with an invalid type', async function () { - await testUpdate({ ...base, type: undefined }) - await testUpdate({ ...base, type: 'toto' as any }) - }) - }) - }) - - describe('Error', function () { - - it('Should fail with a missing error message', async function () { - await server.runnerJobs.error({ jobUUID, jobToken, runnerToken, message: null, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with an invalid error messgae', async function () { - const message = 'a'.repeat(6000) - await server.runnerJobs.error({ jobUUID, jobToken, runnerToken, message, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with a job not in processing state', async function () { - await server.runnerJobs.error({ - jobUUID: completedJobUUID, - jobToken: completedJobToken, - message: 'my message', - runnerToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - }) - - describe('Success', function () { - let vodJobUUID: string - let vodJobToken: string - - describe('Common', function () { - - it('Should fail with a job not in processing state', async function () { - await server.runnerJobs.success({ - jobUUID: completedJobUUID, - jobToken: completedJobToken, - payload: { videoFile: 'video_short.mp4' }, - runnerToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - }) - - describe('VOD', function () { - - it('Should fail with an invalid vod web video payload', async function () { - const { job } = await server.runnerJobs.autoAccept({ runnerToken, type: 'vod-web-video-transcoding' }) - - await server.runnerJobs.success({ - jobUUID: job.uuid, - jobToken: job.jobToken, - payload: { hello: 'video_short.mp4' } as any, - runnerToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - - vodJobUUID = job.uuid - vodJobToken = job.jobToken - }) - - it('Should fail with an invalid vod hls payload', async function () { - // To create HLS jobs - const payload: RunnerJobSuccessPayload = { videoFile: 'video_short.mp4' } - await server.runnerJobs.success({ runnerToken, jobUUID: vodJobUUID, jobToken: vodJobToken, payload }) - - await waitJobs([ server ]) - - const { job } = await server.runnerJobs.autoAccept({ runnerToken, type: 'vod-hls-transcoding' }) - - await server.runnerJobs.success({ - jobUUID: job.uuid, - jobToken: job.jobToken, - payload: { videoFile: 'video_short.mp4' } as any, - runnerToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with an invalid vod audio merge payload', async function () { - const attributes = { name: 'audio_with_preview', previewfile: 'custom-preview.jpg', fixture: 'sample.ogg' } - await server.videos.upload({ attributes, mode: 'legacy' }) - - await waitJobs([ server ]) - - const { job } = await server.runnerJobs.autoAccept({ runnerToken, type: 'vod-audio-merge-transcoding' }) - - await server.runnerJobs.success({ - jobUUID: job.uuid, - jobToken: job.jobToken, - payload: { hello: 'video_short.mp4' } as any, - runnerToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - }) - - describe('Video studio', function () { - - it('Should fail with an invalid video studio transcoding payload', async function () { - await server.runnerJobs.success({ - jobUUID: studioAcceptedJob.uuid, - jobToken: studioAcceptedJob.jobToken, - payload: { hello: 'video_short.mp4' } as any, - runnerToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - }) - }) - - describe('Job files', function () { - - describe('Check video param for common job file routes', function () { - - async function fetchFiles (options: { - videoUUID?: string - expectedStatus: HttpStatusCode - }) { - await fetchVideoInputFiles({ videoUUID, ...options, jobToken, jobUUID, runnerToken }) - - await fetchStudioFiles({ - videoUUID: videoStudioUUID, - - ...options, - - jobToken: studioAcceptedJob.jobToken, - jobUUID: studioAcceptedJob.uuid, - runnerToken, - studioFile - }) - } - - it('Should fail with an invalid video id', async function () { - await fetchFiles({ - videoUUID: 'a', - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with an unknown video id', async function () { - const videoUUID = '910ec12a-d9e6-458b-a274-0abb655f9464' - - await fetchFiles({ - videoUUID, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail with a video id not associated to this job', async function () { - await fetchFiles({ - videoUUID: videoUUID2, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should succeed with the correct params', async function () { - await fetchFiles({ expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - describe('Video studio tasks file routes', function () { - - it('Should fail with an invalid studio filename', async function () { - await fetchStudioFiles({ - videoUUID: videoStudioUUID, - jobUUID: studioAcceptedJob.uuid, - runnerToken, - jobToken: studioAcceptedJob.jobToken, - studioFile: 'toto', - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/search.ts b/server/tests/api/check-params/search.ts deleted file mode 100644 index b04d30b7f..000000000 --- a/server/tests/api/check-params/search.ts +++ /dev/null @@ -1,272 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared' -import { HttpStatusCode } from '@shared/models' -import { cleanupTests, createSingleServer, makeGetRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' - -function updateSearchIndex (server: PeerTubeServer, enabled: boolean, disableLocalSearch = false) { - return server.config.updateCustomSubConfig({ - newConfig: { - search: { - searchIndex: { - enabled, - disableLocalSearch - } - } - } - }) -} - -describe('Test videos API validator', function () { - let server: PeerTubeServer - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - }) - - describe('When searching videos', function () { - const path = '/api/v1/search/videos/' - - const query = { - search: 'coucou' - } - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, null, query) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, null, query) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, null, query) - }) - - it('Should succeed with the correct parameters', async function () { - await makeGetRequest({ url: server.url, path, query, expectedStatus: HttpStatusCode.OK_200 }) - }) - - it('Should fail with an invalid category', async function () { - const customQuery1 = { ...query, categoryOneOf: [ 'aa', 'b' ] } - await makeGetRequest({ url: server.url, path, query: customQuery1, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - - const customQuery2 = { ...query, categoryOneOf: 'a' } - await makeGetRequest({ url: server.url, path, query: customQuery2, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should succeed with a valid category', async function () { - const customQuery1 = { ...query, categoryOneOf: [ 1, 7 ] } - await makeGetRequest({ url: server.url, path, query: customQuery1, expectedStatus: HttpStatusCode.OK_200 }) - - const customQuery2 = { ...query, categoryOneOf: 1 } - await makeGetRequest({ url: server.url, path, query: customQuery2, expectedStatus: HttpStatusCode.OK_200 }) - }) - - it('Should fail with an invalid licence', async function () { - const customQuery1 = { ...query, licenceOneOf: [ 'aa', 'b' ] } - await makeGetRequest({ url: server.url, path, query: customQuery1, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - - const customQuery2 = { ...query, licenceOneOf: 'a' } - await makeGetRequest({ url: server.url, path, query: customQuery2, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should succeed with a valid licence', async function () { - const customQuery1 = { ...query, licenceOneOf: [ 1, 2 ] } - await makeGetRequest({ url: server.url, path, query: customQuery1, expectedStatus: HttpStatusCode.OK_200 }) - - const customQuery2 = { ...query, licenceOneOf: 1 } - await makeGetRequest({ url: server.url, path, query: customQuery2, expectedStatus: HttpStatusCode.OK_200 }) - }) - - it('Should succeed with a valid language', async function () { - const customQuery1 = { ...query, languageOneOf: [ 'fr', 'en' ] } - await makeGetRequest({ url: server.url, path, query: customQuery1, expectedStatus: HttpStatusCode.OK_200 }) - - const customQuery2 = { ...query, languageOneOf: 'fr' } - await makeGetRequest({ url: server.url, path, query: customQuery2, expectedStatus: HttpStatusCode.OK_200 }) - }) - - it('Should succeed with valid tags', async function () { - const customQuery1 = { ...query, tagsOneOf: [ 'tag1', 'tag2' ] } - await makeGetRequest({ url: server.url, path, query: customQuery1, expectedStatus: HttpStatusCode.OK_200 }) - - const customQuery2 = { ...query, tagsOneOf: 'tag1' } - await makeGetRequest({ url: server.url, path, query: customQuery2, expectedStatus: HttpStatusCode.OK_200 }) - - const customQuery3 = { ...query, tagsAllOf: [ 'tag1', 'tag2' ] } - await makeGetRequest({ url: server.url, path, query: customQuery3, expectedStatus: HttpStatusCode.OK_200 }) - - const customQuery4 = { ...query, tagsAllOf: 'tag1' } - await makeGetRequest({ url: server.url, path, query: customQuery4, expectedStatus: HttpStatusCode.OK_200 }) - }) - - it('Should fail with invalid durations', async function () { - const customQuery1 = { ...query, durationMin: 'hello' } - await makeGetRequest({ url: server.url, path, query: customQuery1, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - - const customQuery2 = { ...query, durationMax: 'hello' } - await makeGetRequest({ url: server.url, path, query: customQuery2, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with invalid dates', async function () { - const customQuery1 = { ...query, startDate: 'hello' } - await makeGetRequest({ url: server.url, path, query: customQuery1, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - - const customQuery2 = { ...query, endDate: 'hello' } - await makeGetRequest({ url: server.url, path, query: customQuery2, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - - const customQuery3 = { ...query, originallyPublishedStartDate: 'hello' } - await makeGetRequest({ url: server.url, path, query: customQuery3, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - - const customQuery4 = { ...query, originallyPublishedEndDate: 'hello' } - await makeGetRequest({ url: server.url, path, query: customQuery4, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with an invalid host', async function () { - const customQuery = { ...query, host: '6565' } - await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should succeed with a host', async function () { - const customQuery = { ...query, host: 'example.com' } - await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.OK_200 }) - }) - - it('Should fail with invalid uuids', async function () { - const customQuery = { ...query, uuids: [ '6565', 'dfd70b83-639f-4980-94af-304a56ab4b35' ] } - await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should succeed with valid uuids', async function () { - const customQuery = { ...query, uuids: [ 'dfd70b83-639f-4980-94af-304a56ab4b35' ] } - await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - describe('When searching video playlists', function () { - const path = '/api/v1/search/video-playlists/' - - const query = { - search: 'coucou', - host: 'example.com' - } - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, null, query) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, null, query) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, null, query) - }) - - it('Should fail with an invalid host', async function () { - await makeGetRequest({ url: server.url, path, query: { ...query, host: '6565' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with invalid uuids', async function () { - const customQuery = { ...query, uuids: [ '6565', 'dfd70b83-639f-4980-94af-304a56ab4b35' ] } - await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should succeed with the correct parameters', async function () { - await makeGetRequest({ url: server.url, path, query, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - describe('When searching video channels', function () { - const path = '/api/v1/search/video-channels/' - - const query = { - search: 'coucou', - host: 'example.com' - } - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, null, query) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, null, query) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, null, query) - }) - - it('Should fail with an invalid host', async function () { - await makeGetRequest({ url: server.url, path, query: { ...query, host: '6565' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with invalid handles', async function () { - await makeGetRequest({ url: server.url, path, query: { ...query, handles: [ '' ] }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should succeed with the correct parameters', async function () { - await makeGetRequest({ url: server.url, path, query, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - describe('Search target', function () { - - it('Should fail/succeed depending on the search target', async function () { - const query = { search: 'coucou' } - const paths = [ - '/api/v1/search/video-playlists/', - '/api/v1/search/video-channels/', - '/api/v1/search/videos/' - ] - - for (const path of paths) { - { - const customQuery = { ...query, searchTarget: 'hello' } - await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - } - - { - const customQuery = { ...query, searchTarget: undefined } - await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.OK_200 }) - } - - { - const customQuery = { ...query, searchTarget: 'local' } - await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.OK_200 }) - } - - { - const customQuery = { ...query, searchTarget: 'search-index' } - await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - } - - await updateSearchIndex(server, true, true) - - { - const customQuery = { ...query, searchTarget: 'search-index' } - await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.OK_200 }) - } - - await updateSearchIndex(server, true, false) - - { - const customQuery = { ...query, searchTarget: 'local' } - await makeGetRequest({ url: server.url, path, query: customQuery, expectedStatus: HttpStatusCode.OK_200 }) - } - - await updateSearchIndex(server, false, false) - } - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/services.ts b/server/tests/api/check-params/services.ts deleted file mode 100644 index d45868f36..000000000 --- a/server/tests/api/check-params/services.ts +++ /dev/null @@ -1,195 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { HttpStatusCode, VideoCreateResult, VideoPlaylistCreateResult, VideoPlaylistPrivacy, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createSingleServer, - makeGetRequest, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel -} from '@shared/server-commands' - -describe('Test services API validators', function () { - let server: PeerTubeServer - let playlistUUID: string - - let privateVideo: VideoCreateResult - let unlistedVideo: VideoCreateResult - - let privatePlaylist: VideoPlaylistCreateResult - let unlistedPlaylist: VideoPlaylistCreateResult - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(60000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - server.store.videoCreated = await server.videos.upload({ attributes: { name: 'my super name' } }) - - privateVideo = await server.videos.quickUpload({ name: 'private', privacy: VideoPrivacy.PRIVATE }) - unlistedVideo = await server.videos.quickUpload({ name: 'unlisted', privacy: VideoPrivacy.UNLISTED }) - - { - const created = await server.playlists.create({ - attributes: { - displayName: 'super playlist', - privacy: VideoPlaylistPrivacy.PUBLIC, - videoChannelId: server.store.channel.id - } - }) - - playlistUUID = created.uuid - - privatePlaylist = await server.playlists.create({ - attributes: { - displayName: 'private', - privacy: VideoPlaylistPrivacy.PRIVATE, - videoChannelId: server.store.channel.id - } - }) - - unlistedPlaylist = await server.playlists.create({ - attributes: { - displayName: 'unlisted', - privacy: VideoPlaylistPrivacy.UNLISTED, - videoChannelId: server.store.channel.id - } - }) - } - }) - - describe('Test oEmbed API validators', function () { - - it('Should fail with an invalid url', async function () { - const embedUrl = 'hello.com' - await checkParamEmbed(server, embedUrl) - }) - - it('Should fail with an invalid host', async function () { - const embedUrl = 'http://hello.com/videos/watch/' + server.store.videoCreated.uuid - await checkParamEmbed(server, embedUrl) - }) - - it('Should fail with an invalid element id', async function () { - const embedUrl = `${server.url}/videos/watch/blabla` - await checkParamEmbed(server, embedUrl) - }) - - it('Should fail with an unknown element', async function () { - const embedUrl = `${server.url}/videos/watch/88fc0165-d1f0-4a35-a51a-3b47f668689c` - await checkParamEmbed(server, embedUrl, HttpStatusCode.NOT_FOUND_404) - }) - - it('Should fail with an invalid path', async function () { - const embedUrl = `${server.url}/videos/watchs/${server.store.videoCreated.uuid}` - - await checkParamEmbed(server, embedUrl) - }) - - it('Should fail with an invalid max height', async function () { - const embedUrl = `${server.url}/videos/watch/${server.store.videoCreated.uuid}` - - await checkParamEmbed(server, embedUrl, HttpStatusCode.BAD_REQUEST_400, { maxheight: 'hello' }) - }) - - it('Should fail with an invalid max width', async function () { - const embedUrl = `${server.url}/videos/watch/${server.store.videoCreated.uuid}` - - await checkParamEmbed(server, embedUrl, HttpStatusCode.BAD_REQUEST_400, { maxwidth: 'hello' }) - }) - - it('Should fail with an invalid format', async function () { - const embedUrl = `${server.url}/videos/watch/${server.store.videoCreated.uuid}` - - await checkParamEmbed(server, embedUrl, HttpStatusCode.BAD_REQUEST_400, { format: 'blabla' }) - }) - - it('Should fail with a non supported format', async function () { - const embedUrl = `${server.url}/videos/watch/${server.store.videoCreated.uuid}` - - await checkParamEmbed(server, embedUrl, HttpStatusCode.NOT_IMPLEMENTED_501, { format: 'xml' }) - }) - - it('Should fail with a private video', async function () { - const embedUrl = `${server.url}/videos/watch/${privateVideo.uuid}` - - await checkParamEmbed(server, embedUrl, HttpStatusCode.FORBIDDEN_403) - }) - - it('Should fail with an unlisted video with the int id', async function () { - const embedUrl = `${server.url}/videos/watch/${unlistedVideo.id}` - - await checkParamEmbed(server, embedUrl, HttpStatusCode.FORBIDDEN_403) - }) - - it('Should succeed with an unlisted video using the uuid id', async function () { - for (const uuid of [ unlistedVideo.uuid, unlistedVideo.shortUUID ]) { - const embedUrl = `${server.url}/videos/watch/${uuid}` - - await checkParamEmbed(server, embedUrl, HttpStatusCode.OK_200) - } - }) - - it('Should fail with a private playlist', async function () { - const embedUrl = `${server.url}/videos/watch/playlist/${privatePlaylist.uuid}` - - await checkParamEmbed(server, embedUrl, HttpStatusCode.FORBIDDEN_403) - }) - - it('Should fail with an unlisted playlist using the int id', async function () { - const embedUrl = `${server.url}/videos/watch/playlist/${unlistedPlaylist.id}` - - await checkParamEmbed(server, embedUrl, HttpStatusCode.FORBIDDEN_403) - }) - - it('Should succeed with an unlisted playlist using the uuid id', async function () { - for (const uuid of [ unlistedPlaylist.uuid, unlistedPlaylist.shortUUID ]) { - const embedUrl = `${server.url}/videos/watch/playlist/${uuid}` - - await checkParamEmbed(server, embedUrl, HttpStatusCode.OK_200) - } - }) - - it('Should succeed with the correct params with a video', async function () { - const embedUrl = `${server.url}/videos/watch/${server.store.videoCreated.uuid}` - const query = { - format: 'json', - maxheight: 400, - maxwidth: 400 - } - - await checkParamEmbed(server, embedUrl, HttpStatusCode.OK_200, query) - }) - - it('Should succeed with the correct params with a playlist', async function () { - const embedUrl = `${server.url}/videos/watch/playlist/${playlistUUID}` - const query = { - format: 'json', - maxheight: 400, - maxwidth: 400 - } - - await checkParamEmbed(server, embedUrl, HttpStatusCode.OK_200, query) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) - -function checkParamEmbed (server: PeerTubeServer, embedUrl: string, expectedStatus = HttpStatusCode.BAD_REQUEST_400, query = {}) { - const path = '/services/oembed' - - return makeGetRequest({ - url: server.url, - path, - query: Object.assign(query, { url: embedUrl }), - expectedStatus - }) -} diff --git a/server/tests/api/check-params/transcoding.ts b/server/tests/api/check-params/transcoding.ts deleted file mode 100644 index d5899e11b..000000000 --- a/server/tests/api/check-params/transcoding.ts +++ /dev/null @@ -1,112 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { HttpStatusCode, UserRole } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test transcoding API validators', function () { - let servers: PeerTubeServer[] - - let userToken: string - let moderatorToken: string - - let remoteId: string - let validId: string - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(2) - await setAccessTokensToServers(servers) - - await doubleFollow(servers[0], servers[1]) - - userToken = await servers[0].users.generateUserAndToken('user', UserRole.USER) - moderatorToken = await servers[0].users.generateUserAndToken('moderator', UserRole.MODERATOR) - - { - const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' }) - remoteId = uuid - } - - { - const { uuid } = await servers[0].videos.quickUpload({ name: 'both 1' }) - validId = uuid - } - - await waitJobs(servers) - - await servers[0].config.enableTranscoding() - }) - - it('Should not run transcoding of a unknown video', async function () { - await servers[0].videos.runTranscoding({ videoId: 404, transcodingType: 'hls', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - await servers[0].videos.runTranscoding({ videoId: 404, transcodingType: 'web-video', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should not run transcoding of a remote video', async function () { - const expectedStatus = HttpStatusCode.BAD_REQUEST_400 - - await servers[0].videos.runTranscoding({ videoId: remoteId, transcodingType: 'hls', expectedStatus }) - await servers[0].videos.runTranscoding({ videoId: remoteId, transcodingType: 'web-video', expectedStatus }) - }) - - it('Should not run transcoding by a non admin user', async function () { - const expectedStatus = HttpStatusCode.FORBIDDEN_403 - - await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'hls', token: userToken, expectedStatus }) - await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'web-video', token: moderatorToken, expectedStatus }) - }) - - it('Should not run transcoding without transcoding type', async function () { - await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: undefined, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should not run transcoding with an incorrect transcoding type', async function () { - const expectedStatus = HttpStatusCode.BAD_REQUEST_400 - - await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'toto' as any, expectedStatus }) - }) - - it('Should not run transcoding if the instance disabled it', async function () { - const expectedStatus = HttpStatusCode.BAD_REQUEST_400 - - await servers[0].config.disableTranscoding() - - await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'hls', expectedStatus }) - await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'web-video', expectedStatus }) - }) - - it('Should run transcoding', async function () { - this.timeout(120_000) - - await servers[0].config.enableTranscoding() - - await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'hls' }) - await waitJobs(servers) - - await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'web-video', forceTranscoding: true }) - await waitJobs(servers) - }) - - it('Should not run transcoding on a video that is already being transcoded if forceTranscoding is not set', async function () { - await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'web-video' }) - - const expectedStatus = HttpStatusCode.CONFLICT_409 - await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'web-video', expectedStatus }) - - await servers[0].videos.runTranscoding({ videoId: validId, transcodingType: 'web-video', forceTranscoding: true }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/check-params/two-factor.ts b/server/tests/api/check-params/two-factor.ts deleted file mode 100644 index f8365f1b5..000000000 --- a/server/tests/api/check-params/two-factor.ts +++ /dev/null @@ -1,288 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { HttpStatusCode } from '@shared/models' -import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers, TwoFactorCommand } from '@shared/server-commands' - -describe('Test two factor API validators', function () { - let server: PeerTubeServer - - let rootId: number - let rootPassword: string - let rootRequestToken: string - let rootOTPToken: string - - let userId: number - let userToken = '' - let userPassword: string - let userRequestToken: string - let userOTPToken: string - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(30000) - - { - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - } - - { - const result = await server.users.generate('user1') - userToken = result.token - userId = result.userId - userPassword = result.password - } - - { - const { id } = await server.users.getMyInfo() - rootId = id - rootPassword = server.store.user.password - } - }) - - describe('When requesting two factor', function () { - - it('Should fail with an unknown user id', async function () { - await server.twoFactor.request({ userId: 42, currentPassword: rootPassword, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should fail with an invalid user id', async function () { - await server.twoFactor.request({ - userId: 'invalid' as any, - currentPassword: rootPassword, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail to request another user two factor without the appropriate rights', async function () { - await server.twoFactor.request({ - userId: rootId, - token: userToken, - currentPassword: userPassword, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should succeed to request another user two factor with the appropriate rights', async function () { - await server.twoFactor.request({ userId, currentPassword: rootPassword }) - }) - - it('Should fail to request two factor without a password', async function () { - await server.twoFactor.request({ - userId, - token: userToken, - currentPassword: undefined, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail to request two factor with an incorrect password', async function () { - await server.twoFactor.request({ - userId, - token: userToken, - currentPassword: rootPassword, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should succeed to request two factor without a password when targeting a remote user with an admin account', async function () { - await server.twoFactor.request({ userId }) - }) - - it('Should fail to request two factor without a password when targeting myself with an admin account', async function () { - await server.twoFactor.request({ userId: rootId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await server.twoFactor.request({ userId: rootId, currentPassword: 'bad', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should succeed to request my two factor auth', async function () { - { - const { otpRequest } = await server.twoFactor.request({ userId, token: userToken, currentPassword: userPassword }) - userRequestToken = otpRequest.requestToken - userOTPToken = TwoFactorCommand.buildOTP({ secret: otpRequest.secret }).generate() - } - - { - const { otpRequest } = await server.twoFactor.request({ userId: rootId, currentPassword: rootPassword }) - rootRequestToken = otpRequest.requestToken - rootOTPToken = TwoFactorCommand.buildOTP({ secret: otpRequest.secret }).generate() - } - }) - }) - - describe('When confirming two factor request', function () { - - it('Should fail with an unknown user id', async function () { - await server.twoFactor.confirmRequest({ - userId: 42, - requestToken: rootRequestToken, - otpToken: rootOTPToken, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail with an invalid user id', async function () { - await server.twoFactor.confirmRequest({ - userId: 'invalid' as any, - requestToken: rootRequestToken, - otpToken: rootOTPToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail to confirm another user two factor request without the appropriate rights', async function () { - await server.twoFactor.confirmRequest({ - userId: rootId, - token: userToken, - requestToken: rootRequestToken, - otpToken: rootOTPToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail without request token', async function () { - await server.twoFactor.confirmRequest({ - userId, - requestToken: undefined, - otpToken: userOTPToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with an invalid request token', async function () { - await server.twoFactor.confirmRequest({ - userId, - requestToken: 'toto', - otpToken: userOTPToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with request token of another user', async function () { - await server.twoFactor.confirmRequest({ - userId, - requestToken: rootRequestToken, - otpToken: userOTPToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail without an otp token', async function () { - await server.twoFactor.confirmRequest({ - userId, - requestToken: userRequestToken, - otpToken: undefined, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with a bad otp token', async function () { - await server.twoFactor.confirmRequest({ - userId, - requestToken: userRequestToken, - otpToken: '123456', - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should succeed to confirm another user two factor request with the appropriate rights', async function () { - await server.twoFactor.confirmRequest({ - userId, - requestToken: userRequestToken, - otpToken: userOTPToken - }) - - // Reinit - await server.twoFactor.disable({ userId, currentPassword: rootPassword }) - }) - - it('Should succeed to confirm my two factor request', async function () { - await server.twoFactor.confirmRequest({ - userId, - token: userToken, - requestToken: userRequestToken, - otpToken: userOTPToken - }) - }) - - it('Should fail to confirm again two factor request', async function () { - await server.twoFactor.confirmRequest({ - userId, - token: userToken, - requestToken: userRequestToken, - otpToken: userOTPToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - }) - - describe('When disabling two factor', function () { - - it('Should fail with an unknown user id', async function () { - await server.twoFactor.disable({ - userId: 42, - currentPassword: rootPassword, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail with an invalid user id', async function () { - await server.twoFactor.disable({ - userId: 'invalid' as any, - currentPassword: rootPassword, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail to disable another user two factor without the appropriate rights', async function () { - await server.twoFactor.disable({ - userId: rootId, - token: userToken, - currentPassword: userPassword, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail to disable two factor with an incorrect password', async function () { - await server.twoFactor.disable({ - userId, - token: userToken, - currentPassword: rootPassword, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should succeed to disable two factor without a password when targeting a remote user with an admin account', async function () { - await server.twoFactor.disable({ userId }) - await server.twoFactor.requestAndConfirm({ userId }) - }) - - it('Should fail to disable two factor without a password when targeting myself with an admin account', async function () { - await server.twoFactor.disable({ userId: rootId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await server.twoFactor.disable({ userId: rootId, currentPassword: 'bad', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should succeed to disable another user two factor with the appropriate rights', async function () { - await server.twoFactor.disable({ userId, currentPassword: rootPassword }) - - await server.twoFactor.requestAndConfirm({ userId }) - }) - - it('Should succeed to update my two factor auth', async function () { - await server.twoFactor.disable({ userId, token: userToken, currentPassword: userPassword }) - }) - - it('Should fail to disable again two factor', async function () { - await server.twoFactor.disable({ - userId, - token: userToken, - currentPassword: userPassword, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/upload-quota.ts b/server/tests/api/check-params/upload-quota.ts deleted file mode 100644 index 06698c056..000000000 --- a/server/tests/api/check-params/upload-quota.ts +++ /dev/null @@ -1,134 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { FIXTURE_URLS } from '@server/tests/shared' -import { randomInt } from '@shared/core-utils' -import { HttpStatusCode, VideoImportState, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createSingleServer, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - VideosCommand, - waitJobs -} from '@shared/server-commands' - -describe('Test upload quota', function () { - let server: PeerTubeServer - let rootId: number - let command: VideosCommand - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - const user = await server.users.getMyInfo() - rootId = user.id - - await server.users.update({ userId: rootId, videoQuota: 42 }) - - command = server.videos - }) - - describe('When having a video quota', function () { - - it('Should fail with a registered user having too many videos with legacy upload', async function () { - this.timeout(120000) - - const user = { username: 'registered' + randomInt(1, 1500), password: 'password' } - await server.registrations.register(user) - const userToken = await server.login.getAccessToken(user) - - const attributes = { fixture: 'video_short2.webm' } - for (let i = 0; i < 5; i++) { - await command.upload({ token: userToken, attributes }) - } - - await command.upload({ token: userToken, attributes, expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413, mode: 'legacy' }) - }) - - it('Should fail with a registered user having too many videos with resumable upload', async function () { - this.timeout(120000) - - const user = { username: 'registered' + randomInt(1, 1500), password: 'password' } - await server.registrations.register(user) - const userToken = await server.login.getAccessToken(user) - - const attributes = { fixture: 'video_short2.webm' } - for (let i = 0; i < 5; i++) { - await command.upload({ token: userToken, attributes }) - } - - await command.upload({ token: userToken, attributes, expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413, mode: 'resumable' }) - }) - - it('Should fail to import with HTTP/Torrent/magnet', async function () { - this.timeout(120_000) - - const baseAttributes = { - channelId: server.store.channel.id, - privacy: VideoPrivacy.PUBLIC - } - await server.imports.importVideo({ attributes: { ...baseAttributes, targetUrl: FIXTURE_URLS.goodVideo } }) - await server.imports.importVideo({ attributes: { ...baseAttributes, magnetUri: FIXTURE_URLS.magnet } }) - await server.imports.importVideo({ attributes: { ...baseAttributes, torrentfile: 'video-720p.torrent' as any } }) - - await waitJobs([ server ]) - - const { total, data: videoImports } = await server.imports.getMyVideoImports() - expect(total).to.equal(3) - - expect(videoImports).to.have.lengthOf(3) - - for (const videoImport of videoImports) { - expect(videoImport.state.id).to.equal(VideoImportState.FAILED) - expect(videoImport.error).not.to.be.undefined - expect(videoImport.error).to.contain('user video quota is exceeded') - } - }) - }) - - describe('When having a daily video quota', function () { - - it('Should fail with a user having too many videos daily', async function () { - await server.users.update({ userId: rootId, videoQuotaDaily: 42 }) - - await command.upload({ expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413, mode: 'legacy' }) - await command.upload({ expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413, mode: 'resumable' }) - }) - }) - - describe('When having an absolute and daily video quota', function () { - it('Should fail if exceeding total quota', async function () { - await server.users.update({ - userId: rootId, - videoQuota: 42, - videoQuotaDaily: 1024 * 1024 * 1024 - }) - - await command.upload({ expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413, mode: 'legacy' }) - await command.upload({ expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413, mode: 'resumable' }) - }) - - it('Should fail if exceeding daily quota', async function () { - await server.users.update({ - userId: rootId, - videoQuota: 1024 * 1024 * 1024, - videoQuotaDaily: 42 - }) - - await command.upload({ expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413, mode: 'legacy' }) - await command.upload({ expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413, mode: 'resumable' }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/user-notifications.ts b/server/tests/api/check-params/user-notifications.ts deleted file mode 100644 index 6a588e446..000000000 --- a/server/tests/api/check-params/user-notifications.ts +++ /dev/null @@ -1,290 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { io } from 'socket.io-client' -import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared' -import { wait } from '@shared/core-utils' -import { HttpStatusCode, UserNotificationSetting, UserNotificationSettingValue } from '@shared/models' -import { - cleanupTests, - createSingleServer, - makeGetRequest, - makePostBodyRequest, - makePutBodyRequest, - PeerTubeServer, - setAccessTokensToServers -} from '@shared/server-commands' - -describe('Test user notifications API validators', function () { - let server: PeerTubeServer - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - }) - - describe('When listing my notifications', function () { - const path = '/api/v1/users/me/notifications' - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an incorrect unread parameter', async function () { - await makeGetRequest({ - url: server.url, - path, - query: { - unread: 'toto' - }, - token: server.accessToken, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - - it('Should fail with a non authenticated user', async function () { - await makeGetRequest({ - url: server.url, - path, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should succeed with the correct parameters', async function () { - await makeGetRequest({ - url: server.url, - path, - token: server.accessToken, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - - describe('When marking as read my notifications', function () { - const path = '/api/v1/users/me/notifications/read' - - it('Should fail with wrong ids parameters', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: { - ids: [ 'hello' ] - }, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - - await makePostBodyRequest({ - url: server.url, - path, - fields: { - ids: [ ] - }, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - - await makePostBodyRequest({ - url: server.url, - path, - fields: { - ids: 5 - }, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with a non authenticated user', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: { - ids: [ 5 ] - }, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should succeed with the correct parameters', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: { - ids: [ 5 ] - }, - token: server.accessToken, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - describe('When marking as read my notifications', function () { - const path = '/api/v1/users/me/notifications/read-all' - - it('Should fail with a non authenticated user', async function () { - await makePostBodyRequest({ - url: server.url, - path, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should succeed with the correct parameters', async function () { - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - describe('When updating my notification settings', function () { - const path = '/api/v1/users/me/notification-settings' - const correctFields: UserNotificationSetting = { - newVideoFromSubscription: UserNotificationSettingValue.WEB, - newCommentOnMyVideo: UserNotificationSettingValue.WEB, - abuseAsModerator: UserNotificationSettingValue.WEB, - videoAutoBlacklistAsModerator: UserNotificationSettingValue.WEB, - blacklistOnMyVideo: UserNotificationSettingValue.WEB, - myVideoImportFinished: UserNotificationSettingValue.WEB, - myVideoPublished: UserNotificationSettingValue.WEB, - commentMention: UserNotificationSettingValue.WEB, - newFollow: UserNotificationSettingValue.WEB, - newUserRegistration: UserNotificationSettingValue.WEB, - newInstanceFollower: UserNotificationSettingValue.WEB, - autoInstanceFollowing: UserNotificationSettingValue.WEB, - abuseNewMessage: UserNotificationSettingValue.WEB, - abuseStateChange: UserNotificationSettingValue.WEB, - newPeerTubeVersion: UserNotificationSettingValue.WEB, - myVideoStudioEditionFinished: UserNotificationSettingValue.WEB, - newPluginVersion: UserNotificationSettingValue.WEB - } - - it('Should fail with missing fields', async function () { - await makePutBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields: { newVideoFromSubscription: UserNotificationSettingValue.WEB }, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with incorrect field values', async function () { - { - const fields = { ...correctFields, newCommentOnMyVideo: 15 } - - await makePutBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - } - - { - const fields = { ...correctFields, newCommentOnMyVideo: 'toto' } - - await makePutBodyRequest({ - url: server.url, - path, - fields, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - } - }) - - it('Should fail with a non authenticated user', async function () { - await makePutBodyRequest({ - url: server.url, - path, - fields: correctFields, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should succeed with the correct parameters', async function () { - await makePutBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields: correctFields, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - describe('When connecting to my notification socket', function () { - - it('Should fail with no token', function (next) { - const socket = io(`${server.url}/user-notifications`, { reconnection: false }) - - socket.once('connect_error', function () { - socket.disconnect() - next() - }) - - socket.on('connect', () => { - socket.disconnect() - next(new Error('Connected with a missing token.')) - }) - }) - - it('Should fail with an invalid token', function (next) { - const socket = io(`${server.url}/user-notifications`, { - query: { accessToken: 'bad_access_token' }, - reconnection: false - }) - - socket.once('connect_error', function () { - socket.disconnect() - next() - }) - - socket.on('connect', () => { - socket.disconnect() - next(new Error('Connected with an invalid token.')) - }) - }) - - it('Should success with the correct token', function (next) { - const socket = io(`${server.url}/user-notifications`, { - query: { accessToken: server.accessToken }, - reconnection: false - }) - - function errorListener (err) { - next(new Error('Error in connection: ' + err)) - } - - socket.on('connect_error', errorListener) - - socket.once('connect', async () => { - socket.disconnect() - - await wait(500) - next() - }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/user-subscriptions.ts b/server/tests/api/check-params/user-subscriptions.ts deleted file mode 100644 index c4922c7a2..000000000 --- a/server/tests/api/check-params/user-subscriptions.ts +++ /dev/null @@ -1,298 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { - cleanupTests, - createSingleServer, - makeDeleteRequest, - makeGetRequest, - makePostBodyRequest, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' -import { HttpStatusCode } from '@shared/models' -import { checkBadStartPagination, checkBadCountPagination, checkBadSortPagination } from '@server/tests/shared' - -describe('Test user subscriptions API validators', function () { - const path = '/api/v1/users/me/subscriptions' - let server: PeerTubeServer - let userAccessToken = '' - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - - const user = { - username: 'user1', - password: 'my super password' - } - await server.users.create({ username: user.username, password: user.password }) - userAccessToken = await server.login.getAccessToken(user) - }) - - describe('When listing my subscriptions', function () { - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a non authenticated user', async function () { - await makeGetRequest({ - url: server.url, - path, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should succeed with the correct parameters', async function () { - await makeGetRequest({ - url: server.url, - path, - token: userAccessToken, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - - describe('When listing my subscriptions videos', function () { - const path = '/api/v1/users/me/subscriptions/videos' - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a non authenticated user', async function () { - await makeGetRequest({ - url: server.url, - path, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should succeed with the correct parameters', async function () { - await makeGetRequest({ - url: server.url, - path, - token: userAccessToken, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - - describe('When adding a subscription', function () { - it('Should fail with a non authenticated user', async function () { - await makePostBodyRequest({ - url: server.url, - path, - fields: { uri: 'user1_channel@' + server.host }, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with bad URIs', async function () { - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields: { uri: 'root' }, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields: { uri: 'root@' }, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields: { uri: 'root@hello@' }, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should succeed with the correct parameters', async function () { - this.timeout(20000) - - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields: { uri: 'user1_channel@' + server.host }, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - - await waitJobs([ server ]) - }) - }) - - describe('When getting a subscription', function () { - it('Should fail with a non authenticated user', async function () { - await makeGetRequest({ - url: server.url, - path: path + '/user1_channel@' + server.host, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with bad URIs', async function () { - await makeGetRequest({ - url: server.url, - path: path + '/root', - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - - await makeGetRequest({ - url: server.url, - path: path + '/root@', - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - - await makeGetRequest({ - url: server.url, - path: path + '/root@hello@', - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with an unknown subscription', async function () { - await makeGetRequest({ - url: server.url, - path: path + '/root1@' + server.host, - token: server.accessToken, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should succeed with the correct parameters', async function () { - await makeGetRequest({ - url: server.url, - path: path + '/user1_channel@' + server.host, - token: server.accessToken, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - - describe('When checking if subscriptions exist', function () { - const existPath = path + '/exist' - - it('Should fail with a non authenticated user', async function () { - await makeGetRequest({ - url: server.url, - path: existPath, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with bad URIs', async function () { - await makeGetRequest({ - url: server.url, - path: existPath, - query: { uris: 'toto' }, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - - await makeGetRequest({ - url: server.url, - path: existPath, - query: { 'uris[]': 1 }, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should succeed with the correct parameters', async function () { - await makeGetRequest({ - url: server.url, - path: existPath, - query: { 'uris[]': 'coucou@' + server.host }, - token: server.accessToken, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - - describe('When removing a subscription', function () { - it('Should fail with a non authenticated user', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/user1_channel@' + server.host, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with bad URIs', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/root', - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - - await makeDeleteRequest({ - url: server.url, - path: path + '/root@', - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - - await makeDeleteRequest({ - url: server.url, - path: path + '/root@hello@', - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with an unknown subscription', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/root1@' + server.host, - token: server.accessToken, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should succeed with the correct parameters', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/user1_channel@' + server.host, - token: server.accessToken, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/users-admin.ts b/server/tests/api/check-params/users-admin.ts deleted file mode 100644 index 819da0bb2..000000000 --- a/server/tests/api/check-params/users-admin.ts +++ /dev/null @@ -1,456 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination, MockSmtpServer } from '@server/tests/shared' -import { omit } from '@shared/core-utils' -import { HttpStatusCode, UserAdminFlag, UserRole } from '@shared/models' -import { - cleanupTests, - ConfigCommand, - createSingleServer, - killallServers, - makeGetRequest, - makePostBodyRequest, - makePutBodyRequest, - PeerTubeServer, - setAccessTokensToServers -} from '@shared/server-commands' - -describe('Test users admin API validators', function () { - const path = '/api/v1/users/' - let userId: number - let rootId: number - let moderatorId: number - let server: PeerTubeServer - let userToken = '' - let moderatorToken = '' - let emailPort: number - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(30000) - - const emails: object[] = [] - emailPort = await MockSmtpServer.Instance.collectEmails(emails) - - { - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - } - - { - const result = await server.users.generate('user1') - userToken = result.token - userId = result.userId - } - - { - const result = await server.users.generate('moderator1', UserRole.MODERATOR) - moderatorToken = result.token - } - - { - const result = await server.users.generate('moderator2', UserRole.MODERATOR) - moderatorId = result.userId - } - }) - - describe('When listing users', function () { - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a non authenticated user', async function () { - await makeGetRequest({ - url: server.url, - path, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with a non admin user', async function () { - await makeGetRequest({ - url: server.url, - path, - token: userToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - }) - - describe('When adding a new user', function () { - const baseCorrectParams = { - username: 'user2', - email: 'test@example.com', - password: 'my super password', - videoQuota: -1, - videoQuotaDaily: -1, - role: UserRole.USER, - adminFlags: UserAdminFlag.BYPASS_VIDEO_AUTO_BLACKLIST - } - - it('Should fail with a too small username', async function () { - const fields = { ...baseCorrectParams, username: '' } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a too long username', async function () { - const fields = { ...baseCorrectParams, username: 'super'.repeat(50) } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a not lowercase username', async function () { - const fields = { ...baseCorrectParams, username: 'Toto' } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with an incorrect username', async function () { - const fields = { ...baseCorrectParams, username: 'my username' } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a missing email', async function () { - const fields = omit(baseCorrectParams, [ 'email' ]) - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with an invalid email', async function () { - const fields = { ...baseCorrectParams, email: 'test_example.com' } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a too small password', async function () { - const fields = { ...baseCorrectParams, password: 'bla' } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a too long password', async function () { - const fields = { ...baseCorrectParams, password: 'super'.repeat(61) } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with empty password and no smtp configured', async function () { - const fields = { ...baseCorrectParams, password: '' } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should succeed with no password on a server with smtp enabled', async function () { - this.timeout(20000) - - await killallServers([ server ]) - - await server.run(ConfigCommand.getEmailOverrideConfig(emailPort)) - - const fields = { - ...baseCorrectParams, - - password: '', - username: 'create_password', - email: 'create_password@example.com' - } - - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - - it('Should fail with invalid admin flags', async function () { - const fields = { ...baseCorrectParams, adminFlags: 'toto' } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with an non authenticated user', async function () { - await makePostBodyRequest({ - url: server.url, - path, - token: 'super token', - fields: baseCorrectParams, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail if we add a user with the same username', async function () { - const fields = { ...baseCorrectParams, username: 'user1' } - - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields, - expectedStatus: HttpStatusCode.CONFLICT_409 - }) - }) - - it('Should fail if we add a user with the same email', async function () { - const fields = { ...baseCorrectParams, email: 'user1@example.com' } - - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields, - expectedStatus: HttpStatusCode.CONFLICT_409 - }) - }) - - it('Should fail with an invalid videoQuota', async function () { - const fields = { ...baseCorrectParams, videoQuota: -5 } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with an invalid videoQuotaDaily', async function () { - const fields = { ...baseCorrectParams, videoQuotaDaily: -7 } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail without a user role', async function () { - const fields = omit(baseCorrectParams, [ 'role' ]) - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with an invalid user role', async function () { - const fields = { ...baseCorrectParams, role: 88989 } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a "peertube" username', async function () { - const fields = { ...baseCorrectParams, username: 'peertube' } - - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields, - expectedStatus: HttpStatusCode.CONFLICT_409 - }) - }) - - it('Should fail to create a moderator or an admin with a moderator', async function () { - for (const role of [ UserRole.MODERATOR, UserRole.ADMINISTRATOR ]) { - const fields = { ...baseCorrectParams, role } - - await makePostBodyRequest({ - url: server.url, - path, - token: moderatorToken, - fields, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - } - }) - - it('Should succeed to create a user with a moderator', async function () { - const fields = { ...baseCorrectParams, username: 'a4656', email: 'a4656@example.com', role: UserRole.USER } - - await makePostBodyRequest({ - url: server.url, - path, - token: moderatorToken, - fields, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - - it('Should succeed with the correct params', async function () { - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields: baseCorrectParams, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - - it('Should fail with a non admin user', async function () { - const user = { username: 'user1' } - userToken = await server.login.getAccessToken(user) - - const fields = { - username: 'user3', - email: 'test@example.com', - password: 'my super password', - videoQuota: 42000000 - } - await makePostBodyRequest({ url: server.url, path, token: userToken, fields, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - }) - - describe('When getting a user', function () { - - it('Should fail with an non authenticated user', async function () { - await makeGetRequest({ - url: server.url, - path: path + userId, - token: 'super token', - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with a non admin user', async function () { - await makeGetRequest({ url: server.url, path, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should succeed with the correct params', async function () { - await makeGetRequest({ url: server.url, path: path + userId, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - describe('When updating a user', function () { - - it('Should fail with an invalid email attribute', async function () { - const fields = { - email: 'blabla' - } - - await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields }) - }) - - it('Should fail with an invalid emailVerified attribute', async function () { - const fields = { - emailVerified: 'yes' - } - - await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields }) - }) - - it('Should fail with an invalid videoQuota attribute', async function () { - const fields = { - videoQuota: -90 - } - - await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields }) - }) - - it('Should fail with an invalid user role attribute', async function () { - const fields = { - role: 54878 - } - - await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields }) - }) - - it('Should fail with a too small password', async function () { - const fields = { - currentPassword: 'password', - password: 'bla' - } - - await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields }) - }) - - it('Should fail with a too long password', async function () { - const fields = { - currentPassword: 'password', - password: 'super'.repeat(61) - } - - await makePutBodyRequest({ url: server.url, path: path + userId, token: server.accessToken, fields }) - }) - - it('Should fail with an non authenticated user', async function () { - const fields = { - videoQuota: 42 - } - - await makePutBodyRequest({ - url: server.url, - path: path + userId, - token: 'super token', - fields, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail when updating root role', async function () { - const fields = { - role: UserRole.MODERATOR - } - - await makePutBodyRequest({ url: server.url, path: path + rootId, token: server.accessToken, fields }) - }) - - it('Should fail with invalid admin flags', async function () { - const fields = { adminFlags: 'toto' } - - await makePutBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail to update an admin with a moderator', async function () { - const fields = { - videoQuota: 42 - } - - await makePutBodyRequest({ - url: server.url, - path: path + moderatorId, - token: moderatorToken, - fields, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should succeed to update a user with a moderator', async function () { - const fields = { - videoQuota: 42 - } - - await makePutBodyRequest({ - url: server.url, - path: path + userId, - token: moderatorToken, - fields, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - - it('Should succeed with the correct params', async function () { - const fields = { - email: 'email@example.com', - emailVerified: true, - videoQuota: 42, - role: UserRole.USER - } - - await makePutBodyRequest({ - url: server.url, - path: path + userId, - token: server.accessToken, - fields, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - after(async function () { - MockSmtpServer.Instance.kill() - - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/users-emails.ts b/server/tests/api/check-params/users-emails.ts deleted file mode 100644 index 6ebcc8ffe..000000000 --- a/server/tests/api/check-params/users-emails.ts +++ /dev/null @@ -1,116 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { HttpStatusCode, UserRole } from '@shared/models' -import { cleanupTests, createSingleServer, makePostBodyRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' - -describe('Test users API validators', function () { - let server: PeerTubeServer - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1, { - rates_limit: { - ask_send_email: { - max: 10 - } - } - }) - - await setAccessTokensToServers([ server ]) - await server.config.enableSignup(true) - - await server.users.generate('moderator2', UserRole.MODERATOR) - - await server.registrations.requestRegistration({ - username: 'request1', - registrationReason: 'tt' - }) - }) - - describe('When asking a password reset', function () { - const path = '/api/v1/users/ask-reset-password' - - it('Should fail with a missing email', async function () { - const fields = {} - - await makePostBodyRequest({ url: server.url, path, fields }) - }) - - it('Should fail with an invalid email', async function () { - const fields = { email: 'hello' } - - await makePostBodyRequest({ url: server.url, path, fields }) - }) - - it('Should success with the correct params', async function () { - const fields = { email: 'admin@example.com' } - - await makePostBodyRequest({ - url: server.url, - path, - fields, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - describe('When asking for an account verification email', function () { - const path = '/api/v1/users/ask-send-verify-email' - - it('Should fail with a missing email', async function () { - const fields = {} - - await makePostBodyRequest({ url: server.url, path, fields }) - }) - - it('Should fail with an invalid email', async function () { - const fields = { email: 'hello' } - - await makePostBodyRequest({ url: server.url, path, fields }) - }) - - it('Should succeed with the correct params', async function () { - const fields = { email: 'admin@example.com' } - - await makePostBodyRequest({ - url: server.url, - path, - fields, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - describe('When asking for a registration verification email', function () { - const path = '/api/v1/users/registrations/ask-send-verify-email' - - it('Should fail with a missing email', async function () { - const fields = {} - - await makePostBodyRequest({ url: server.url, path, fields }) - }) - - it('Should fail with an invalid email', async function () { - const fields = { email: 'hello' } - - await makePostBodyRequest({ url: server.url, path, fields }) - }) - - it('Should succeed with the correct params', async function () { - const fields = { email: 'request1@example.com' } - - await makePostBodyRequest({ - url: server.url, - path, - fields, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/video-blacklist.ts b/server/tests/api/check-params/video-blacklist.ts deleted file mode 100644 index 8e9f61596..000000000 --- a/server/tests/api/check-params/video-blacklist.ts +++ /dev/null @@ -1,292 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared' -import { HttpStatusCode, VideoBlacklistType } from '@shared/models' -import { - BlacklistCommand, - cleanupTests, - createMultipleServers, - doubleFollow, - makePostBodyRequest, - makePutBodyRequest, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test video blacklist API validators', function () { - let servers: PeerTubeServer[] - let notBlacklistedVideoId: string - let remoteVideoUUID: string - let userAccessToken1 = '' - let userAccessToken2 = '' - let command: BlacklistCommand - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(2) - - await setAccessTokensToServers(servers) - await doubleFollow(servers[0], servers[1]) - - { - const username = 'user1' - const password = 'my super password' - await servers[0].users.create({ username, password }) - userAccessToken1 = await servers[0].login.getAccessToken({ username, password }) - } - - { - const username = 'user2' - const password = 'my super password' - await servers[0].users.create({ username, password }) - userAccessToken2 = await servers[0].login.getAccessToken({ username, password }) - } - - { - servers[0].store.videoCreated = await servers[0].videos.upload({ token: userAccessToken1 }) - } - - { - const { uuid } = await servers[0].videos.upload() - notBlacklistedVideoId = uuid - } - - { - const { uuid } = await servers[1].videos.upload() - remoteVideoUUID = uuid - } - - await waitJobs(servers) - - command = servers[0].blacklist - }) - - describe('When adding a video in blacklist', function () { - const basePath = '/api/v1/videos/' - - it('Should fail with nothing', async function () { - const path = basePath + servers[0].store.videoCreated + '/blacklist' - const fields = {} - await makePostBodyRequest({ url: servers[0].url, path, token: servers[0].accessToken, fields }) - }) - - it('Should fail with a wrong video', async function () { - const wrongPath = '/api/v1/videos/blabla/blacklist' - const fields = {} - await makePostBodyRequest({ url: servers[0].url, path: wrongPath, token: servers[0].accessToken, fields }) - }) - - it('Should fail with a non authenticated user', async function () { - const path = basePath + servers[0].store.videoCreated + '/blacklist' - const fields = {} - await makePostBodyRequest({ url: servers[0].url, path, token: 'hello', fields, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with a non admin user', async function () { - const path = basePath + servers[0].store.videoCreated + '/blacklist' - const fields = {} - await makePostBodyRequest({ - url: servers[0].url, - path, - token: userAccessToken2, - fields, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with an invalid reason', async function () { - const path = basePath + servers[0].store.videoCreated.uuid + '/blacklist' - const fields = { reason: 'a'.repeat(305) } - - await makePostBodyRequest({ url: servers[0].url, path, token: servers[0].accessToken, fields }) - }) - - it('Should fail to unfederate a remote video', async function () { - const path = basePath + remoteVideoUUID + '/blacklist' - const fields = { unfederate: true } - - await makePostBodyRequest({ - url: servers[0].url, - path, - token: servers[0].accessToken, - fields, - expectedStatus: HttpStatusCode.CONFLICT_409 - }) - }) - - it('Should succeed with the correct params', async function () { - const path = basePath + servers[0].store.videoCreated.uuid + '/blacklist' - const fields = {} - - await makePostBodyRequest({ - url: servers[0].url, - path, - token: servers[0].accessToken, - fields, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - describe('When updating a video in blacklist', function () { - const basePath = '/api/v1/videos/' - - it('Should fail with a wrong video', async function () { - const wrongPath = '/api/v1/videos/blabla/blacklist' - const fields = {} - await makePutBodyRequest({ url: servers[0].url, path: wrongPath, token: servers[0].accessToken, fields }) - }) - - it('Should fail with a video not blacklisted', async function () { - const path = '/api/v1/videos/' + notBlacklistedVideoId + '/blacklist' - const fields = {} - await makePutBodyRequest({ - url: servers[0].url, - path, - token: servers[0].accessToken, - fields, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail with a non authenticated user', async function () { - const path = basePath + servers[0].store.videoCreated + '/blacklist' - const fields = {} - await makePutBodyRequest({ url: servers[0].url, path, token: 'hello', fields, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with a non admin user', async function () { - const path = basePath + servers[0].store.videoCreated + '/blacklist' - const fields = {} - await makePutBodyRequest({ - url: servers[0].url, - path, - token: userAccessToken2, - fields, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with an invalid reason', async function () { - const path = basePath + servers[0].store.videoCreated.uuid + '/blacklist' - const fields = { reason: 'a'.repeat(305) } - - await makePutBodyRequest({ url: servers[0].url, path, token: servers[0].accessToken, fields }) - }) - - it('Should succeed with the correct params', async function () { - const path = basePath + servers[0].store.videoCreated.shortUUID + '/blacklist' - const fields = { reason: 'hello' } - - await makePutBodyRequest({ - url: servers[0].url, - path, - token: servers[0].accessToken, - fields, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - describe('When getting blacklisted video', function () { - - it('Should fail with a non authenticated user', async function () { - await servers[0].videos.get({ id: servers[0].store.videoCreated.uuid, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with another user', async function () { - await servers[0].videos.getWithToken({ - token: userAccessToken2, - id: servers[0].store.videoCreated.uuid, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should succeed with the owner authenticated user', async function () { - const video = await servers[0].videos.getWithToken({ token: userAccessToken1, id: servers[0].store.videoCreated.uuid }) - expect(video.blacklisted).to.be.true - }) - - it('Should succeed with an admin', async function () { - const video = servers[0].store.videoCreated - - for (const id of [ video.id, video.uuid, video.shortUUID ]) { - const video = await servers[0].videos.getWithToken({ id, expectedStatus: HttpStatusCode.OK_200 }) - expect(video.blacklisted).to.be.true - } - }) - }) - - describe('When removing a video in blacklist', function () { - - it('Should fail with a non authenticated user', async function () { - await command.remove({ - token: 'fake token', - videoId: servers[0].store.videoCreated.uuid, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with a non admin user', async function () { - await command.remove({ - token: userAccessToken2, - videoId: servers[0].store.videoCreated.uuid, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with an incorrect id', async function () { - await command.remove({ videoId: 'hello', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with a not blacklisted video', async function () { - // The video was not added to the blacklist so it should fail - await command.remove({ videoId: notBlacklistedVideoId, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should succeed with the correct params', async function () { - await command.remove({ videoId: servers[0].store.videoCreated.uuid, expectedStatus: HttpStatusCode.NO_CONTENT_204 }) - }) - }) - - describe('When listing videos in blacklist', function () { - const basePath = '/api/v1/videos/blacklist/' - - it('Should fail with a non authenticated user', async function () { - await servers[0].blacklist.list({ token: 'fake token', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with a non admin user', async function () { - await servers[0].blacklist.list({ token: userAccessToken2, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(servers[0].url, basePath, servers[0].accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(servers[0].url, basePath, servers[0].accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(servers[0].url, basePath, servers[0].accessToken) - }) - - it('Should fail with an invalid type', async function () { - await servers[0].blacklist.list({ type: 0 as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should succeed with the correct parameters', async function () { - await servers[0].blacklist.list({ type: VideoBlacklistType.MANUAL }) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/check-params/video-captions.ts b/server/tests/api/check-params/video-captions.ts deleted file mode 100644 index 532dab1c4..000000000 --- a/server/tests/api/check-params/video-captions.ts +++ /dev/null @@ -1,307 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { buildAbsoluteFixturePath } from '@shared/core-utils' -import { HttpStatusCode, VideoCreateResult, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createSingleServer, - makeDeleteRequest, - makeGetRequest, - makeUploadRequest, - PeerTubeServer, - setAccessTokensToServers -} from '@shared/server-commands' - -describe('Test video captions API validator', function () { - const path = '/api/v1/videos/' - - let server: PeerTubeServer - let userAccessToken: string - let video: VideoCreateResult - let privateVideo: VideoCreateResult - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(120000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - - video = await server.videos.upload() - privateVideo = await server.videos.upload({ attributes: { privacy: VideoPrivacy.PRIVATE } }) - - { - const user = { - username: 'user1', - password: 'my super password' - } - await server.users.create({ username: user.username, password: user.password }) - userAccessToken = await server.login.getAccessToken(user) - } - }) - - describe('When adding video caption', function () { - const fields = { } - const attaches = { - captionfile: buildAbsoluteFixturePath('subtitle-good1.vtt') - } - - it('Should fail without a valid uuid', async function () { - await makeUploadRequest({ - method: 'PUT', - url: server.url, - path: path + '4da6fde3-88f7-4d16-b119-108df563d0b06/captions/fr', - token: server.accessToken, - fields, - attaches - }) - }) - - it('Should fail with an unknown id', async function () { - await makeUploadRequest({ - method: 'PUT', - url: server.url, - path: path + '4da6fde3-88f7-4d16-b119-108df5630b06/captions/fr', - token: server.accessToken, - fields, - attaches, - expectedStatus: 404 - }) - }) - - it('Should fail with a missing language in path', async function () { - const captionPath = path + video.uuid + '/captions' - await makeUploadRequest({ - method: 'PUT', - url: server.url, - path: captionPath, - token: server.accessToken, - fields, - attaches - }) - }) - - it('Should fail with an unknown language', async function () { - const captionPath = path + video.uuid + '/captions/15' - await makeUploadRequest({ - method: 'PUT', - url: server.url, - path: captionPath, - token: server.accessToken, - fields, - attaches - }) - }) - - it('Should fail without access token', async function () { - const captionPath = path + video.uuid + '/captions/fr' - await makeUploadRequest({ - method: 'PUT', - url: server.url, - path: captionPath, - fields, - attaches, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with a bad access token', async function () { - const captionPath = path + video.uuid + '/captions/fr' - await makeUploadRequest({ - method: 'PUT', - url: server.url, - path: captionPath, - token: 'blabla', - fields, - attaches, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - // We accept any file now - // it('Should fail with an invalid captionfile extension', async function () { - // const attaches = { - // 'captionfile': buildAbsoluteFixturePath('subtitle-bad.txt') - // } - // - // const captionPath = path + video.uuid + '/captions/fr' - // await makeUploadRequest({ - // method: 'PUT', - // url: server.url, - // path: captionPath, - // token: server.accessToken, - // fields, - // attaches, - // expectedStatus: HttpStatusCode.BAD_REQUEST_400 - // }) - // }) - - // We don't check the extension yet - // it('Should fail with an invalid captionfile extension and octet-stream mime type', async function () { - // await createVideoCaption({ - // url: server.url, - // accessToken: server.accessToken, - // language: 'zh', - // videoId: video.uuid, - // fixture: 'subtitle-bad.txt', - // mimeType: 'application/octet-stream', - // expectedStatus: HttpStatusCode.BAD_REQUEST_400 - // }) - // }) - - it('Should succeed with a valid captionfile extension and octet-stream mime type', async function () { - await server.captions.add({ - language: 'zh', - videoId: video.uuid, - fixture: 'subtitle-good.srt', - mimeType: 'application/octet-stream' - }) - }) - - // We don't check the file validity yet - // it('Should fail with an invalid captionfile srt', async function () { - // const attaches = { - // 'captionfile': buildAbsoluteFixturePath('subtitle-bad.srt') - // } - // - // const captionPath = path + video.uuid + '/captions/fr' - // await makeUploadRequest({ - // method: 'PUT', - // url: server.url, - // path: captionPath, - // token: server.accessToken, - // fields, - // attaches, - // expectedStatus: HttpStatusCode.INTERNAL_SERVER_ERROR_500 - // }) - // }) - - it('Should success with the correct parameters', async function () { - const captionPath = path + video.uuid + '/captions/fr' - await makeUploadRequest({ - method: 'PUT', - url: server.url, - path: captionPath, - token: server.accessToken, - fields, - attaches, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - describe('When listing video captions', function () { - it('Should fail without a valid uuid', async function () { - await makeGetRequest({ url: server.url, path: path + '4da6fde3-88f7-4d16-b119-108df563d0b06/captions' }) - }) - - it('Should fail with an unknown id', async function () { - await makeGetRequest({ - url: server.url, - path: path + '4da6fde3-88f7-4d16-b119-108df5630b06/captions', - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail with a private video without token', async function () { - await makeGetRequest({ - url: server.url, - path: path + privateVideo.shortUUID + '/captions', - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with another user token', async function () { - await makeGetRequest({ - url: server.url, - token: userAccessToken, - path: path + privateVideo.shortUUID + '/captions', - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should success with the correct parameters', async function () { - await makeGetRequest({ url: server.url, path: path + video.shortUUID + '/captions', expectedStatus: HttpStatusCode.OK_200 }) - - await makeGetRequest({ - url: server.url, - path: path + privateVideo.shortUUID + '/captions', - token: server.accessToken, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - - describe('When deleting video caption', function () { - it('Should fail without a valid uuid', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '4da6fde3-88f7-4d16-b119-108df563d0b06/captions/fr', - token: server.accessToken - }) - }) - - it('Should fail with an unknown id', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '4da6fde3-88f7-4d16-b119-108df5630b06/captions/fr', - token: server.accessToken, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail with an invalid language', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '4da6fde3-88f7-4d16-b119-108df5630b06/captions/16', - token: server.accessToken - }) - }) - - it('Should fail with a missing language', async function () { - const captionPath = path + video.shortUUID + '/captions' - await makeDeleteRequest({ url: server.url, path: captionPath, token: server.accessToken }) - }) - - it('Should fail with an unknown language', async function () { - const captionPath = path + video.shortUUID + '/captions/15' - await makeDeleteRequest({ url: server.url, path: captionPath, token: server.accessToken }) - }) - - it('Should fail without access token', async function () { - const captionPath = path + video.shortUUID + '/captions/fr' - await makeDeleteRequest({ url: server.url, path: captionPath, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with a bad access token', async function () { - const captionPath = path + video.shortUUID + '/captions/fr' - await makeDeleteRequest({ url: server.url, path: captionPath, token: 'coucou', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with another user', async function () { - const captionPath = path + video.shortUUID + '/captions/fr' - await makeDeleteRequest({ - url: server.url, - path: captionPath, - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should success with the correct parameters', async function () { - const captionPath = path + video.shortUUID + '/captions/fr' - await makeDeleteRequest({ - url: server.url, - path: captionPath, - token: server.accessToken, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/video-channel-syncs.ts b/server/tests/api/check-params/video-channel-syncs.ts deleted file mode 100644 index bcd8984df..000000000 --- a/server/tests/api/check-params/video-channel-syncs.ts +++ /dev/null @@ -1,318 +0,0 @@ -import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination, FIXTURE_URLS } from '@server/tests/shared' -import { HttpStatusCode, VideoChannelSyncCreate } from '@shared/models' -import { - ChannelSyncsCommand, - createSingleServer, - makePostBodyRequest, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel -} from '@shared/server-commands' - -describe('Test video channel sync API validator', () => { - const path = '/api/v1/video-channel-syncs' - let server: PeerTubeServer - let command: ChannelSyncsCommand - let rootChannelId: number - let rootChannelSyncId: number - const userInfo = { - accessToken: '', - username: 'user1', - id: -1, - channelId: -1, - syncId: -1 - } - - async function withChannelSyncDisabled (callback: () => Promise): Promise { - try { - await server.config.disableChannelSync() - await callback() - } finally { - await server.config.enableChannelSync() - } - } - - async function withMaxSyncsPerUser (maxSync: number, callback: () => Promise): Promise { - const origConfig = await server.config.getCustomConfig() - - await server.config.updateExistingSubConfig({ - newConfig: { - import: { - videoChannelSynchronization: { - maxPerUser: maxSync - } - } - } - }) - - try { - await callback() - } finally { - await server.config.updateCustomConfig({ newCustomConfig: origConfig }) - } - } - - before(async function () { - this.timeout(30_000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - command = server.channelSyncs - - rootChannelId = server.store.channel.id - - { - userInfo.accessToken = await server.users.generateUserAndToken(userInfo.username) - - const { videoChannels, id: userId } = await server.users.getMyInfo({ token: userInfo.accessToken }) - userInfo.id = userId - userInfo.channelId = videoChannels[0].id - } - - await server.config.enableChannelSync() - }) - - describe('When creating a sync', function () { - let baseCorrectParams: VideoChannelSyncCreate - - before(function () { - baseCorrectParams = { - externalChannelUrl: FIXTURE_URLS.youtubeChannel, - videoChannelId: rootChannelId - } - }) - - it('Should fail when sync is disabled', async function () { - await withChannelSyncDisabled(async () => { - await command.create({ - token: server.accessToken, - attributes: baseCorrectParams, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - }) - - it('Should fail with nothing', async function () { - const fields = {} - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with no authentication', async function () { - await command.create({ - token: null, - attributes: baseCorrectParams, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail without a target url', async function () { - const attributes: VideoChannelSyncCreate = { - ...baseCorrectParams, - externalChannelUrl: null - } - await command.create({ - token: server.accessToken, - attributes, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail without a channelId', async function () { - const attributes: VideoChannelSyncCreate = { - ...baseCorrectParams, - videoChannelId: null - } - await command.create({ - token: server.accessToken, - attributes, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with a channelId refering nothing', async function () { - const attributes: VideoChannelSyncCreate = { - ...baseCorrectParams, - videoChannelId: 42 - } - await command.create({ - token: server.accessToken, - attributes, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail to create a sync when the user does not own the channel', async function () { - await command.create({ - token: userInfo.accessToken, - attributes: baseCorrectParams, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should succeed to create a sync with root and for another user\'s channel', async function () { - const { videoChannelSync } = await command.create({ - token: server.accessToken, - attributes: { - ...baseCorrectParams, - videoChannelId: userInfo.channelId - }, - expectedStatus: HttpStatusCode.OK_200 - }) - userInfo.syncId = videoChannelSync.id - }) - - it('Should succeed with the correct parameters', async function () { - const { videoChannelSync } = await command.create({ - token: server.accessToken, - attributes: baseCorrectParams, - expectedStatus: HttpStatusCode.OK_200 - }) - rootChannelSyncId = videoChannelSync.id - }) - - it('Should fail when the user exceeds allowed number of synchronizations', async function () { - await withMaxSyncsPerUser(1, async () => { - await command.create({ - token: server.accessToken, - attributes: { - ...baseCorrectParams, - videoChannelId: userInfo.channelId - }, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - }) - }) - - describe('When listing my channel syncs', function () { - const myPath = '/api/v1/accounts/root/video-channel-syncs' - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, myPath, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, myPath, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, myPath, server.accessToken) - }) - - it('Should succeed with the correct parameters', async function () { - await command.listByAccount({ - accountName: 'root', - token: server.accessToken, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - - it('Should fail with no authentication', async function () { - await command.listByAccount({ - accountName: 'root', - token: null, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail when a simple user lists another user\'s synchronizations', async function () { - await command.listByAccount({ - accountName: 'root', - token: userInfo.accessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should succeed when root lists another user\'s synchronizations', async function () { - await command.listByAccount({ - accountName: userInfo.username, - token: server.accessToken, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - - it('Should succeed even with synchronization disabled', async function () { - await withChannelSyncDisabled(async function () { - await command.listByAccount({ - accountName: 'root', - token: server.accessToken, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - }) - - describe('When triggering deletion', function () { - it('should fail with no authentication', async function () { - await command.delete({ - channelSyncId: userInfo.syncId, - token: null, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail when channelSyncId does not refer to any sync', async function () { - await command.delete({ - channelSyncId: 42, - token: server.accessToken, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail when sync is not owned by the user', async function () { - await command.delete({ - channelSyncId: rootChannelSyncId, - token: userInfo.accessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should succeed when root delete a sync they do not own', async function () { - await command.delete({ - channelSyncId: userInfo.syncId, - token: server.accessToken, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - - it('should succeed when user delete a sync they own', async function () { - const { videoChannelSync } = await command.create({ - attributes: { - externalChannelUrl: FIXTURE_URLS.youtubeChannel, - videoChannelId: userInfo.channelId - }, - token: server.accessToken, - expectedStatus: HttpStatusCode.OK_200 - }) - - await command.delete({ - channelSyncId: videoChannelSync.id, - token: server.accessToken, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - - it('Should succeed even when synchronization is disabled', async function () { - await withChannelSyncDisabled(async function () { - await command.delete({ - channelSyncId: rootChannelSyncId, - token: server.accessToken, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - }) - - after(async function () { - await server?.kill() - }) -}) diff --git a/server/tests/api/check-params/video-channels.ts b/server/tests/api/check-params/video-channels.ts deleted file mode 100644 index 1782474fd..000000000 --- a/server/tests/api/check-params/video-channels.ts +++ /dev/null @@ -1,378 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared' -import { buildAbsoluteFixturePath, omit } from '@shared/core-utils' -import { HttpStatusCode, VideoChannelUpdate } from '@shared/models' -import { - ChannelsCommand, - cleanupTests, - createSingleServer, - makeGetRequest, - makePostBodyRequest, - makePutBodyRequest, - makeUploadRequest, - PeerTubeServer, - setAccessTokensToServers -} from '@shared/server-commands' - -describe('Test video channels API validator', function () { - const videoChannelPath = '/api/v1/video-channels' - let server: PeerTubeServer - const userInfo = { - accessToken: '', - channelName: 'fake_channel', - id: -1, - videoQuota: -1, - videoQuotaDaily: -1 - } - let command: ChannelsCommand - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - - const userCreds = { - username: 'fake', - password: 'fake_password' - } - - { - const user = await server.users.create({ username: userCreds.username, password: userCreds.password }) - userInfo.id = user.id - userInfo.accessToken = await server.login.getAccessToken(userCreds) - } - - command = server.channels - }) - - describe('When listing a video channels', function () { - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, videoChannelPath, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, videoChannelPath, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, videoChannelPath, server.accessToken) - }) - }) - - describe('When listing account video channels', function () { - const accountChannelPath = '/api/v1/accounts/fake/video-channels' - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, accountChannelPath, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, accountChannelPath, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, accountChannelPath, server.accessToken) - }) - - it('Should fail with a unknown account', async function () { - await server.channels.listByAccount({ accountName: 'unknown', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should succeed with the correct parameters', async function () { - await makeGetRequest({ - url: server.url, - path: accountChannelPath, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - - describe('When adding a video channel', function () { - const baseCorrectParams = { - name: 'super_channel', - displayName: 'hello', - description: 'super description', - support: 'super support text' - } - - it('Should fail with a non authenticated user', async function () { - await makePostBodyRequest({ - url: server.url, - path: videoChannelPath, - token: 'none', - fields: baseCorrectParams, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with nothing', async function () { - const fields = {} - await makePostBodyRequest({ url: server.url, path: videoChannelPath, token: server.accessToken, fields }) - }) - - it('Should fail without a name', async function () { - const fields = omit(baseCorrectParams, [ 'name' ]) - await makePostBodyRequest({ url: server.url, path: videoChannelPath, token: server.accessToken, fields }) - }) - - it('Should fail with a bad name', async function () { - const fields = { ...baseCorrectParams, name: 'super name' } - await makePostBodyRequest({ url: server.url, path: videoChannelPath, token: server.accessToken, fields }) - }) - - it('Should fail without a name', async function () { - const fields = omit(baseCorrectParams, [ 'displayName' ]) - await makePostBodyRequest({ url: server.url, path: videoChannelPath, token: server.accessToken, fields }) - }) - - it('Should fail with a long name', async function () { - const fields = { ...baseCorrectParams, displayName: 'super'.repeat(25) } - await makePostBodyRequest({ url: server.url, path: videoChannelPath, token: server.accessToken, fields }) - }) - - it('Should fail with a long description', async function () { - const fields = { ...baseCorrectParams, description: 'super'.repeat(201) } - await makePostBodyRequest({ url: server.url, path: videoChannelPath, token: server.accessToken, fields }) - }) - - it('Should fail with a long support text', async function () { - const fields = { ...baseCorrectParams, support: 'super'.repeat(201) } - await makePostBodyRequest({ url: server.url, path: videoChannelPath, token: server.accessToken, fields }) - }) - - it('Should succeed with the correct parameters', async function () { - await makePostBodyRequest({ - url: server.url, - path: videoChannelPath, - token: server.accessToken, - fields: baseCorrectParams, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - - it('Should fail when adding a channel with the same username', async function () { - await makePostBodyRequest({ - url: server.url, - path: videoChannelPath, - token: server.accessToken, - fields: baseCorrectParams, - expectedStatus: HttpStatusCode.CONFLICT_409 - }) - }) - }) - - describe('When updating a video channel', function () { - const baseCorrectParams: VideoChannelUpdate = { - displayName: 'hello', - description: 'super description', - support: 'toto', - bulkVideosSupportUpdate: false - } - let path: string - - before(async function () { - path = videoChannelPath + '/super_channel' - }) - - it('Should fail with a non authenticated user', async function () { - await makePutBodyRequest({ - url: server.url, - path, - token: 'hi', - fields: baseCorrectParams, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with another authenticated user', async function () { - await makePutBodyRequest({ - url: server.url, - path, - token: userInfo.accessToken, - fields: baseCorrectParams, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with a long name', async function () { - const fields = { ...baseCorrectParams, displayName: 'super'.repeat(25) } - await makePutBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a long description', async function () { - const fields = { ...baseCorrectParams, description: 'super'.repeat(201) } - await makePutBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a long support text', async function () { - const fields = { ...baseCorrectParams, support: 'super'.repeat(201) } - await makePutBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a bad bulkVideosSupportUpdate field', async function () { - const fields = { ...baseCorrectParams, bulkVideosSupportUpdate: 'super' } - await makePutBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should succeed with the correct parameters', async function () { - await makePutBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields: baseCorrectParams, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - describe('When updating video channel avatars/banners', function () { - const types = [ 'avatar', 'banner' ] - let path: string - - before(async function () { - path = videoChannelPath + '/super_channel' - }) - - it('Should fail with an incorrect input file', async function () { - for (const type of types) { - const fields = {} - const attaches = { - [type + 'file']: buildAbsoluteFixturePath('video_short.mp4') - } - - await makeUploadRequest({ url: server.url, path: `${path}/${type}/pick`, token: server.accessToken, fields, attaches }) - } - }) - - it('Should fail with a big file', async function () { - for (const type of types) { - const fields = {} - const attaches = { - [type + 'file']: buildAbsoluteFixturePath('avatar-big.png') - } - await makeUploadRequest({ url: server.url, path: `${path}/${type}/pick`, token: server.accessToken, fields, attaches }) - } - }) - - it('Should fail with an unauthenticated user', async function () { - for (const type of types) { - const fields = {} - const attaches = { - [type + 'file']: buildAbsoluteFixturePath('avatar.png') - } - await makeUploadRequest({ - url: server.url, - path: `${path}/${type}/pick`, - fields, - attaches, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - } - }) - - it('Should succeed with the correct params', async function () { - for (const type of types) { - const fields = {} - const attaches = { - [type + 'file']: buildAbsoluteFixturePath('avatar.png') - } - await makeUploadRequest({ - url: server.url, - path: `${path}/${type}/pick`, - token: server.accessToken, - fields, - attaches, - expectedStatus: HttpStatusCode.OK_200 - }) - } - }) - }) - - describe('When getting a video channel', function () { - it('Should return the list of the video channels with nothing', async function () { - const res = await makeGetRequest({ - url: server.url, - path: videoChannelPath, - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(res.body.data).to.be.an('array') - }) - - it('Should return 404 with an incorrect video channel', async function () { - await makeGetRequest({ - url: server.url, - path: videoChannelPath + '/super_channel2', - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should succeed with the correct parameters', async function () { - await makeGetRequest({ - url: server.url, - path: videoChannelPath + '/super_channel', - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - - describe('When getting channel followers', function () { - const path = '/api/v1/video-channels/super_channel/followers' - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a unauthenticated user', async function () { - await makeGetRequest({ url: server.url, path, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with a another user', async function () { - await makeGetRequest({ url: server.url, path, token: userInfo.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should succeed with the correct params', async function () { - await makeGetRequest({ url: server.url, path, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - describe('When deleting a video channel', function () { - it('Should fail with a non authenticated user', async function () { - await command.delete({ token: 'coucou', channelName: 'super_channel', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with another authenticated user', async function () { - await command.delete({ token: userInfo.accessToken, channelName: 'super_channel', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should fail with an unknown video channel id', async function () { - await command.delete({ channelName: 'super_channel2', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should succeed with the correct parameters', async function () { - await command.delete({ channelName: 'super_channel' }) - }) - - it('Should fail to delete the last user video channel', async function () { - await command.delete({ channelName: 'root_channel', expectedStatus: HttpStatusCode.CONFLICT_409 }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/video-comments.ts b/server/tests/api/check-params/video-comments.ts deleted file mode 100644 index 9f497c0cf..000000000 --- a/server/tests/api/check-params/video-comments.ts +++ /dev/null @@ -1,484 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared' -import { HttpStatusCode, VideoCreateResult, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createSingleServer, - makeDeleteRequest, - makeGetRequest, - makePostBodyRequest, - PeerTubeServer, - setAccessTokensToServers -} from '@shared/server-commands' - -describe('Test video comments API validator', function () { - let pathThread: string - let pathComment: string - - let server: PeerTubeServer - - let video: VideoCreateResult - - let userAccessToken: string - let userAccessToken2: string - - let commentId: number - let privateCommentId: number - let privateVideo: VideoCreateResult - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(120000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - - { - video = await server.videos.upload({ attributes: {} }) - pathThread = '/api/v1/videos/' + video.uuid + '/comment-threads' - } - - { - privateVideo = await server.videos.upload({ attributes: { privacy: VideoPrivacy.PRIVATE } }) - } - - { - const created = await server.comments.createThread({ videoId: video.uuid, text: 'coucou' }) - commentId = created.id - pathComment = '/api/v1/videos/' + video.uuid + '/comments/' + commentId - } - - { - const created = await server.comments.createThread({ videoId: privateVideo.uuid, text: 'coucou' }) - privateCommentId = created.id - } - - { - const user = { username: 'user1', password: 'my super password' } - await server.users.create({ username: user.username, password: user.password }) - userAccessToken = await server.login.getAccessToken(user) - } - - { - const user = { username: 'user2', password: 'my super password' } - await server.users.create({ username: user.username, password: user.password }) - userAccessToken2 = await server.login.getAccessToken(user) - } - }) - - describe('When listing video comment threads', function () { - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, pathThread, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, pathThread, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, pathThread, server.accessToken) - }) - - it('Should fail with an incorrect video', async function () { - await makeGetRequest({ - url: server.url, - path: '/api/v1/videos/ba708d62-e3d7-45d9-9d73-41b9097cc02d/comment-threads', - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail with a private video without token', async function () { - await makeGetRequest({ - url: server.url, - path: '/api/v1/videos/' + privateVideo.shortUUID + '/comment-threads', - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with another user token', async function () { - await makeGetRequest({ - url: server.url, - token: userAccessToken, - path: '/api/v1/videos/' + privateVideo.shortUUID + '/comment-threads', - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should succeed with the correct params', async function () { - await makeGetRequest({ - url: server.url, - token: server.accessToken, - path: '/api/v1/videos/' + privateVideo.shortUUID + '/comment-threads', - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - - describe('When listing comments of a thread', function () { - it('Should fail with an incorrect video', async function () { - await makeGetRequest({ - url: server.url, - path: '/api/v1/videos/ba708d62-e3d7-45d9-9d73-41b9097cc02d/comment-threads/' + commentId, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail with an incorrect thread id', async function () { - await makeGetRequest({ - url: server.url, - path: '/api/v1/videos/' + video.shortUUID + '/comment-threads/156', - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail with a private video without token', async function () { - await makeGetRequest({ - url: server.url, - path: '/api/v1/videos/' + privateVideo.shortUUID + '/comment-threads/' + privateCommentId, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with another user token', async function () { - await makeGetRequest({ - url: server.url, - token: userAccessToken, - path: '/api/v1/videos/' + privateVideo.shortUUID + '/comment-threads/' + privateCommentId, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should success with the correct params', async function () { - await makeGetRequest({ - url: server.url, - token: server.accessToken, - path: '/api/v1/videos/' + privateVideo.shortUUID + '/comment-threads/' + privateCommentId, - expectedStatus: HttpStatusCode.OK_200 - }) - - await makeGetRequest({ - url: server.url, - path: '/api/v1/videos/' + video.shortUUID + '/comment-threads/' + commentId, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - - describe('When adding a video thread', function () { - - it('Should fail with a non authenticated user', async function () { - const fields = { - text: 'text' - } - await makePostBodyRequest({ - url: server.url, - path: pathThread, - token: 'none', - fields, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with nothing', async function () { - const fields = {} - await makePostBodyRequest({ url: server.url, path: pathThread, token: server.accessToken, fields }) - }) - - it('Should fail with a short comment', async function () { - const fields = { - text: '' - } - await makePostBodyRequest({ url: server.url, path: pathThread, token: server.accessToken, fields }) - }) - - it('Should fail with a long comment', async function () { - const fields = { - text: 'h'.repeat(10001) - } - await makePostBodyRequest({ url: server.url, path: pathThread, token: server.accessToken, fields }) - }) - - it('Should fail with an incorrect video', async function () { - const path = '/api/v1/videos/ba708d62-e3d7-45d9-9d73-41b9097cc02d/comment-threads' - const fields = { text: 'super comment' } - - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail with a private video of another user', async function () { - const fields = { text: 'super comment' } - - await makePostBodyRequest({ - url: server.url, - path: '/api/v1/videos/' + privateVideo.shortUUID + '/comment-threads', - token: userAccessToken, - fields, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should succeed with the correct parameters', async function () { - const fields = { text: 'super comment' } - - await makePostBodyRequest({ - url: server.url, - path: pathThread, - token: server.accessToken, - fields, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - - describe('When adding a comment to a thread', function () { - - it('Should fail with a non authenticated user', async function () { - const fields = { - text: 'text' - } - await makePostBodyRequest({ - url: server.url, - path: pathComment, - token: 'none', - fields, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with nothing', async function () { - const fields = {} - await makePostBodyRequest({ url: server.url, path: pathComment, token: server.accessToken, fields }) - }) - - it('Should fail with a short comment', async function () { - const fields = { - text: '' - } - await makePostBodyRequest({ url: server.url, path: pathComment, token: server.accessToken, fields }) - }) - - it('Should fail with a long comment', async function () { - const fields = { - text: 'h'.repeat(10001) - } - await makePostBodyRequest({ url: server.url, path: pathComment, token: server.accessToken, fields }) - }) - - it('Should fail with an incorrect video', async function () { - const path = '/api/v1/videos/ba708d62-e3d7-45d9-9d73-41b9097cc02d/comments/' + commentId - const fields = { - text: 'super comment' - } - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail with a private video of another user', async function () { - const fields = { text: 'super comment' } - - await makePostBodyRequest({ - url: server.url, - path: '/api/v1/videos/' + privateVideo.uuid + '/comments/' + privateCommentId, - token: userAccessToken, - fields, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with an incorrect comment', async function () { - const path = '/api/v1/videos/' + video.uuid + '/comments/124' - const fields = { - text: 'super comment' - } - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should succeed with the correct parameters', async function () { - const fields = { - text: 'super comment' - } - await makePostBodyRequest({ - url: server.url, - path: pathComment, - token: server.accessToken, - fields, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - - describe('When removing video comments', function () { - it('Should fail with a non authenticated user', async function () { - await makeDeleteRequest({ url: server.url, path: pathComment, token: 'none', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with another user', async function () { - await makeDeleteRequest({ - url: server.url, - path: pathComment, - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with an incorrect video', async function () { - const path = '/api/v1/videos/ba708d62-e3d7-45d9-9d73-41b9097cc02d/comments/' + commentId - await makeDeleteRequest({ url: server.url, path, token: server.accessToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should fail with an incorrect comment', async function () { - const path = '/api/v1/videos/' + video.uuid + '/comments/124' - await makeDeleteRequest({ url: server.url, path, token: server.accessToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should succeed with the same user', async function () { - let commentToDelete: number - - { - const created = await server.comments.createThread({ videoId: video.uuid, token: userAccessToken, text: 'hello' }) - commentToDelete = created.id - } - - const path = '/api/v1/videos/' + video.uuid + '/comments/' + commentToDelete - - await makeDeleteRequest({ url: server.url, path, token: userAccessToken2, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeDeleteRequest({ url: server.url, path, token: userAccessToken, expectedStatus: HttpStatusCode.NO_CONTENT_204 }) - }) - - it('Should succeed with the owner of the video', async function () { - let commentToDelete: number - let anotherVideoUUID: string - - { - const { uuid } = await server.videos.upload({ token: userAccessToken, attributes: { name: 'video' } }) - anotherVideoUUID = uuid - } - - { - const created = await server.comments.createThread({ videoId: anotherVideoUUID, text: 'hello' }) - commentToDelete = created.id - } - - const path = '/api/v1/videos/' + anotherVideoUUID + '/comments/' + commentToDelete - - await makeDeleteRequest({ url: server.url, path, token: userAccessToken2, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeDeleteRequest({ url: server.url, path, token: userAccessToken, expectedStatus: HttpStatusCode.NO_CONTENT_204 }) - }) - - it('Should succeed with the correct parameters', async function () { - await makeDeleteRequest({ - url: server.url, - path: pathComment, - token: server.accessToken, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - describe('When a video has comments disabled', function () { - before(async function () { - video = await server.videos.upload({ attributes: { commentsEnabled: false } }) - pathThread = '/api/v1/videos/' + video.uuid + '/comment-threads' - }) - - it('Should return an empty thread list', async function () { - const res = await makeGetRequest({ - url: server.url, - path: pathThread, - expectedStatus: HttpStatusCode.OK_200 - }) - expect(res.body.total).to.equal(0) - expect(res.body.data).to.have.lengthOf(0) - }) - - it('Should return an thread comments list') - - it('Should return conflict on thread add', async function () { - const fields = { - text: 'super comment' - } - await makePostBodyRequest({ - url: server.url, - path: pathThread, - token: server.accessToken, - fields, - expectedStatus: HttpStatusCode.CONFLICT_409 - }) - }) - - it('Should return conflict on comment thread add') - }) - - describe('When listing admin comments threads', function () { - const path = '/api/v1/videos/comments' - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a non authenticated user', async function () { - await makeGetRequest({ - url: server.url, - path, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with a non admin user', async function () { - await makeGetRequest({ - url: server.url, - path, - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should succeed with the correct params', async function () { - await makeGetRequest({ - url: server.url, - path, - token: server.accessToken, - query: { - isLocal: false, - search: 'toto', - searchAccount: 'toto', - searchVideo: 'toto' - }, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/video-files.ts b/server/tests/api/check-params/video-files.ts deleted file mode 100644 index 01d6a912b..000000000 --- a/server/tests/api/check-params/video-files.ts +++ /dev/null @@ -1,195 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { getAllFiles } from '@shared/core-utils' -import { HttpStatusCode, UserRole, VideoDetails, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - makeRawRequest, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test videos files', function () { - let servers: PeerTubeServer[] - - let userToken: string - let moderatorToken: string - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(300_000) - - servers = await createMultipleServers(2) - await setAccessTokensToServers(servers) - - await doubleFollow(servers[0], servers[1]) - - userToken = await servers[0].users.generateUserAndToken('user', UserRole.USER) - moderatorToken = await servers[0].users.generateUserAndToken('moderator', UserRole.MODERATOR) - }) - - describe('Getting metadata', function () { - let video: VideoDetails - - before(async function () { - const { uuid } = await servers[0].videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) - video = await servers[0].videos.getWithToken({ id: uuid }) - }) - - it('Should not get metadata of private video without token', async function () { - for (const file of getAllFiles(video)) { - await makeRawRequest({ url: file.metadataUrl, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - } - }) - - it('Should not get metadata of private video without the appropriate token', async function () { - for (const file of getAllFiles(video)) { - await makeRawRequest({ url: file.metadataUrl, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - } - }) - - it('Should get metadata of private video with the appropriate token', async function () { - for (const file of getAllFiles(video)) { - await makeRawRequest({ url: file.metadataUrl, token: servers[0].accessToken, expectedStatus: HttpStatusCode.OK_200 }) - } - }) - }) - - describe('Deleting files', function () { - let webVideoId: string - let hlsId: string - let remoteId: string - - let validId1: string - let validId2: string - - let hlsFileId: number - let webVideoFileId: number - - let remoteHLSFileId: number - let remoteWebVideoFileId: number - - before(async function () { - this.timeout(300_000) - - { - const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' }) - await waitJobs(servers) - - const video = await servers[1].videos.get({ id: uuid }) - remoteId = video.uuid - remoteHLSFileId = video.streamingPlaylists[0].files[0].id - remoteWebVideoFileId = video.files[0].id - } - - { - await servers[0].config.enableTranscoding({ hls: true, webVideo: true }) - - { - const { uuid } = await servers[0].videos.quickUpload({ name: 'both 1' }) - await waitJobs(servers) - - const video = await servers[0].videos.get({ id: uuid }) - validId1 = video.uuid - hlsFileId = video.streamingPlaylists[0].files[0].id - webVideoFileId = video.files[0].id - } - - { - const { uuid } = await servers[0].videos.quickUpload({ name: 'both 2' }) - validId2 = uuid - } - } - - await waitJobs(servers) - - { - await servers[0].config.enableTranscoding({ hls: true, webVideo: false }) - const { uuid } = await servers[0].videos.quickUpload({ name: 'hls' }) - hlsId = uuid - } - - await waitJobs(servers) - - { - await servers[0].config.enableTranscoding({ webVideo: true, hls: false }) - const { uuid } = await servers[0].videos.quickUpload({ name: 'web-video' }) - webVideoId = uuid - } - - await waitJobs(servers) - }) - - it('Should not delete files of a unknown video', async function () { - const expectedStatus = HttpStatusCode.NOT_FOUND_404 - - await servers[0].videos.removeHLSPlaylist({ videoId: 404, expectedStatus }) - await servers[0].videos.removeAllWebVideoFiles({ videoId: 404, expectedStatus }) - - await servers[0].videos.removeHLSFile({ videoId: 404, fileId: hlsFileId, expectedStatus }) - await servers[0].videos.removeWebVideoFile({ videoId: 404, fileId: webVideoFileId, expectedStatus }) - }) - - it('Should not delete unknown files', async function () { - const expectedStatus = HttpStatusCode.NOT_FOUND_404 - - await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: webVideoFileId, expectedStatus }) - await servers[0].videos.removeWebVideoFile({ videoId: validId1, fileId: hlsFileId, expectedStatus }) - }) - - it('Should not delete files of a remote video', async function () { - const expectedStatus = HttpStatusCode.BAD_REQUEST_400 - - await servers[0].videos.removeHLSPlaylist({ videoId: remoteId, expectedStatus }) - await servers[0].videos.removeAllWebVideoFiles({ videoId: remoteId, expectedStatus }) - - await servers[0].videos.removeHLSFile({ videoId: remoteId, fileId: remoteHLSFileId, expectedStatus }) - await servers[0].videos.removeWebVideoFile({ videoId: remoteId, fileId: remoteWebVideoFileId, expectedStatus }) - }) - - it('Should not delete files by a non admin user', async function () { - const expectedStatus = HttpStatusCode.FORBIDDEN_403 - - await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: userToken, expectedStatus }) - await servers[0].videos.removeHLSPlaylist({ videoId: validId1, token: moderatorToken, expectedStatus }) - - await servers[0].videos.removeAllWebVideoFiles({ videoId: validId1, token: userToken, expectedStatus }) - await servers[0].videos.removeAllWebVideoFiles({ videoId: validId1, token: moderatorToken, expectedStatus }) - - await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: userToken, expectedStatus }) - await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId, token: moderatorToken, expectedStatus }) - - await servers[0].videos.removeWebVideoFile({ videoId: validId1, fileId: webVideoFileId, token: userToken, expectedStatus }) - await servers[0].videos.removeWebVideoFile({ videoId: validId1, fileId: webVideoFileId, token: moderatorToken, expectedStatus }) - }) - - it('Should not delete files if the files are not available', async function () { - await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await servers[0].videos.removeAllWebVideoFiles({ videoId: webVideoId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - - await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - await servers[0].videos.removeWebVideoFile({ videoId: webVideoId, fileId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should not delete files if no both versions are available', async function () { - await servers[0].videos.removeHLSPlaylist({ videoId: hlsId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await servers[0].videos.removeAllWebVideoFiles({ videoId: webVideoId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should delete files if both versions are available', async function () { - await servers[0].videos.removeHLSFile({ videoId: validId1, fileId: hlsFileId }) - await servers[0].videos.removeWebVideoFile({ videoId: validId1, fileId: webVideoFileId }) - - await servers[0].videos.removeHLSPlaylist({ videoId: validId1 }) - await servers[0].videos.removeAllWebVideoFiles({ videoId: validId2 }) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/check-params/video-imports.ts b/server/tests/api/check-params/video-imports.ts deleted file mode 100644 index 8c6f43c12..000000000 --- a/server/tests/api/check-params/video-imports.ts +++ /dev/null @@ -1,431 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination, FIXTURE_URLS } from '@server/tests/shared' -import { buildAbsoluteFixturePath, omit } from '@shared/core-utils' -import { HttpStatusCode, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createSingleServer, - makeGetRequest, - makePostBodyRequest, - makeUploadRequest, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - waitJobs -} from '@shared/server-commands' - -describe('Test video imports API validator', function () { - const path = '/api/v1/videos/imports' - let server: PeerTubeServer - let userAccessToken = '' - let channelId: number - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - const username = 'user1' - const password = 'my super password' - await server.users.create({ username, password }) - userAccessToken = await server.login.getAccessToken({ username, password }) - - { - const { videoChannels } = await server.users.getMyInfo() - channelId = videoChannels[0].id - } - }) - - describe('When listing my video imports', function () { - const myPath = '/api/v1/users/me/videos/imports' - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, myPath, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, myPath, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, myPath, server.accessToken) - }) - - it('Should fail with a bad videoChannelSyncId param', async function () { - await makeGetRequest({ - url: server.url, - path: myPath, - query: { videoChannelSyncId: 'toto' }, - token: server.accessToken - }) - }) - - it('Should success with the correct parameters', async function () { - await makeGetRequest({ url: server.url, path: myPath, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken }) - }) - }) - - describe('When adding a video import', function () { - let baseCorrectParams - - before(function () { - baseCorrectParams = { - targetUrl: FIXTURE_URLS.goodVideo, - name: 'my super name', - category: 5, - licence: 1, - language: 'pt', - nsfw: false, - commentsEnabled: true, - downloadEnabled: true, - waitTranscoding: true, - description: 'my super description', - support: 'my super support text', - tags: [ 'tag1', 'tag2' ], - privacy: VideoPrivacy.PUBLIC, - channelId - } - }) - - it('Should fail with nothing', async function () { - const fields = {} - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail without a target url', async function () { - const fields = omit(baseCorrectParams, [ 'targetUrl' ]) - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with a bad target url', async function () { - const fields = { ...baseCorrectParams, targetUrl: 'htt://hello' } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with localhost', async function () { - const fields = { ...baseCorrectParams, targetUrl: 'http://localhost:8000' } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a private IP target urls', async function () { - const targetUrls = [ - 'http://127.0.0.1:8000', - 'http://127.0.0.1', - 'http://127.0.0.1/hello', - 'https://192.168.1.42', - 'http://192.168.1.42', - 'http://127.0.0.1.cpy.re' - ] - - for (const targetUrl of targetUrls) { - const fields = { ...baseCorrectParams, targetUrl } - - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - } - }) - - it('Should fail with a long name', async function () { - const fields = { ...baseCorrectParams, name: 'super'.repeat(65) } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a bad category', async function () { - const fields = { ...baseCorrectParams, category: 125 } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a bad licence', async function () { - const fields = { ...baseCorrectParams, licence: 125 } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a bad language', async function () { - const fields = { ...baseCorrectParams, language: 'a'.repeat(15) } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a long description', async function () { - const fields = { ...baseCorrectParams, description: 'super'.repeat(2500) } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a long support text', async function () { - const fields = { ...baseCorrectParams, support: 'super'.repeat(201) } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail without a channel', async function () { - const fields = omit(baseCorrectParams, [ 'channelId' ]) - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a bad channel', async function () { - const fields = { ...baseCorrectParams, channelId: 545454 } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with another user channel', async function () { - const user = { - username: 'fake', - password: 'fake_password' - } - await server.users.create({ username: user.username, password: user.password }) - - const accessTokenUser = await server.login.getAccessToken(user) - const { videoChannels } = await server.users.getMyInfo({ token: accessTokenUser }) - const customChannelId = videoChannels[0].id - - const fields = { ...baseCorrectParams, channelId: customChannelId } - - await makePostBodyRequest({ url: server.url, path, token: userAccessToken, fields }) - }) - - it('Should fail with too many tags', async function () { - const fields = { ...baseCorrectParams, tags: [ 'tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6' ] } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a tag length too low', async function () { - const fields = { ...baseCorrectParams, tags: [ 'tag1', 't' ] } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with a tag length too big', async function () { - const fields = { ...baseCorrectParams, tags: [ 'tag1', 'my_super_tag_too_long_long_long_long_long_long' ] } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail with an incorrect thumbnail file', async function () { - const fields = baseCorrectParams - const attaches = { - thumbnailfile: buildAbsoluteFixturePath('video_short.mp4') - } - - await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches }) - }) - - it('Should fail with a big thumbnail file', async function () { - const fields = baseCorrectParams - const attaches = { - thumbnailfile: buildAbsoluteFixturePath('custom-preview-big.png') - } - - await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches }) - }) - - it('Should fail with an incorrect preview file', async function () { - const fields = baseCorrectParams - const attaches = { - previewfile: buildAbsoluteFixturePath('video_short.mp4') - } - - await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches }) - }) - - it('Should fail with a big preview file', async function () { - const fields = baseCorrectParams - const attaches = { - previewfile: buildAbsoluteFixturePath('custom-preview-big.png') - } - - await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches }) - }) - - it('Should fail with an invalid torrent file', async function () { - const fields = omit(baseCorrectParams, [ 'targetUrl' ]) - const attaches = { - torrentfile: buildAbsoluteFixturePath('avatar-big.png') - } - - await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches }) - }) - - it('Should fail with an invalid magnet URI', async function () { - let fields = omit(baseCorrectParams, [ 'targetUrl' ]) - fields = { ...fields, magnetUri: 'blabla' } - - await makePostBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should succeed with the correct parameters', async function () { - this.timeout(120000) - - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields: baseCorrectParams, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - - it('Should forbid to import http videos', async function () { - await server.config.updateCustomSubConfig({ - newConfig: { - import: { - videos: { - http: { - enabled: false - }, - torrent: { - enabled: true - } - } - } - } - }) - - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields: baseCorrectParams, - expectedStatus: HttpStatusCode.CONFLICT_409 - }) - }) - - it('Should forbid to import torrent videos', async function () { - await server.config.updateCustomSubConfig({ - newConfig: { - import: { - videos: { - http: { - enabled: true - }, - torrent: { - enabled: false - } - } - } - } - }) - - let fields = omit(baseCorrectParams, [ 'targetUrl' ]) - fields = { ...fields, magnetUri: FIXTURE_URLS.magnet } - - await makePostBodyRequest({ - url: server.url, - path, - token: server.accessToken, - fields, - expectedStatus: HttpStatusCode.CONFLICT_409 - }) - - fields = omit(fields, [ 'magnetUri' ]) - const attaches = { - torrentfile: buildAbsoluteFixturePath('video-720p.torrent') - } - - await makeUploadRequest({ - url: server.url, - path, - token: server.accessToken, - fields, - attaches, - expectedStatus: HttpStatusCode.CONFLICT_409 - }) - }) - }) - - describe('Deleting/cancelling a video import', function () { - let importId: number - - async function importVideo () { - const attributes = { channelId: server.store.channel.id, targetUrl: FIXTURE_URLS.goodVideo } - const res = await server.imports.importVideo({ attributes }) - - return res.id - } - - before(async function () { - importId = await importVideo() - }) - - it('Should fail with an invalid import id', async function () { - await server.imports.cancel({ importId: 'artyom' as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await server.imports.delete({ importId: 'artyom' as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with an unknown import id', async function () { - await server.imports.cancel({ importId: 42, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - await server.imports.delete({ importId: 42, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should fail without token', async function () { - await server.imports.cancel({ importId, token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - await server.imports.delete({ importId, token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with another user token', async function () { - await server.imports.cancel({ importId, token: userAccessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await server.imports.delete({ importId, token: userAccessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should fail to cancel non pending import', async function () { - this.timeout(60000) - - await waitJobs([ server ]) - - await server.imports.cancel({ importId, expectedStatus: HttpStatusCode.CONFLICT_409 }) - }) - - it('Should succeed to delete an import', async function () { - await server.imports.delete({ importId }) - }) - - it('Should fail to delete a pending import', async function () { - await server.jobs.pauseJobQueue() - - importId = await importVideo() - - await server.imports.delete({ importId, expectedStatus: HttpStatusCode.CONFLICT_409 }) - }) - - it('Should succeed to cancel an import', async function () { - importId = await importVideo() - - await server.imports.cancel({ importId }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/video-passwords.ts b/server/tests/api/check-params/video-passwords.ts deleted file mode 100644 index 50b0bacb3..000000000 --- a/server/tests/api/check-params/video-passwords.ts +++ /dev/null @@ -1,609 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { - FIXTURE_URLS, - checkBadCountPagination, - checkBadSortPagination, - checkBadStartPagination, - checkUploadVideoParam -} from '@server/tests/shared' -import { root } from '@shared/core-utils' -import { - HttpStatusCode, - PeerTubeProblemDocument, - ServerErrorCode, - VideoCreateResult, - VideoPrivacy -} from '@shared/models' -import { - cleanupTests, - createSingleServer, - makePostBodyRequest, - PeerTubeServer, - setAccessTokensToServers -} from '@shared/server-commands' -import { expect } from 'chai' -import { join } from 'path' - -describe('Test video passwords validator', function () { - let path: string - let server: PeerTubeServer - let userAccessToken = '' - let video: VideoCreateResult - let channelId: number - let publicVideo: VideoCreateResult - let commentId: number - // --------------------------------------------------------------- - - before(async function () { - this.timeout(50000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - - await server.config.updateCustomSubConfig({ - newConfig: { - live: { - enabled: true, - latencySetting: { - enabled: false - }, - allowReplay: false - }, - import: { - videos: { - http:{ - enabled: true - } - } - } - } - }) - - userAccessToken = await server.users.generateUserAndToken('user1') - - { - const body = await server.users.getMyInfo() - channelId = body.videoChannels[0].id - } - - { - video = await server.videos.quickUpload({ - name: 'password protected video', - privacy: VideoPrivacy.PASSWORD_PROTECTED, - videoPasswords: [ 'password1', 'password2' ] - }) - } - path = '/api/v1/videos/' - }) - - async function checkVideoPasswordOptions (options: { - server: PeerTubeServer - token: string - videoPasswords: string[] - expectedStatus: HttpStatusCode - mode: 'uploadLegacy' | 'uploadResumable' | 'import' | 'updateVideo' | 'updatePasswords' | 'live' - }) { - const { server, token, videoPasswords, expectedStatus = HttpStatusCode.OK_200, mode } = options - const attaches = { - fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.webm') - } - const baseCorrectParams = { - name: 'my super name', - category: 5, - licence: 1, - language: 'pt', - nsfw: false, - commentsEnabled: true, - downloadEnabled: true, - waitTranscoding: true, - description: 'my super description', - support: 'my super support text', - tags: [ 'tag1', 'tag2' ], - privacy: VideoPrivacy.PASSWORD_PROTECTED, - channelId, - originallyPublishedAt: new Date().toISOString() - } - if (mode === 'uploadLegacy') { - const fields = { ...baseCorrectParams, videoPasswords } - return checkUploadVideoParam({ server, token, attributes: { ...fields, ...attaches }, expectedStatus, mode: 'legacy' }) - } - - if (mode === 'uploadResumable') { - const fields = { ...baseCorrectParams, videoPasswords } - return checkUploadVideoParam({ server, token, attributes: { ...fields, ...attaches }, expectedStatus, mode: 'resumable' }) - } - - if (mode === 'import') { - const attributes = { ...baseCorrectParams, targetUrl: FIXTURE_URLS.goodVideo, videoPasswords } - return server.imports.importVideo({ attributes, expectedStatus }) - } - - if (mode === 'updateVideo') { - const attributes = { ...baseCorrectParams, videoPasswords } - return server.videos.update({ token, expectedStatus, id: video.id, attributes }) - } - - if (mode === 'updatePasswords') { - return server.videoPasswords.updateAll({ token, expectedStatus, videoId: video.id, passwords: videoPasswords }) - } - - if (mode === 'live') { - const fields = { ...baseCorrectParams, videoPasswords } - - return server.live.create({ fields, expectedStatus }) - } - } - - function validateVideoPasswordList (mode: 'uploadLegacy' | 'uploadResumable' | 'import' | 'updateVideo' | 'updatePasswords' | 'live') { - - it('Should fail with a password protected privacy without providing a password', async function () { - await checkVideoPasswordOptions({ - server, - token: server.accessToken, - videoPasswords: undefined, - expectedStatus: HttpStatusCode.BAD_REQUEST_400, - mode - }) - }) - - it('Should fail with a password protected privacy and an empty password list', async function () { - const videoPasswords = [] - - await checkVideoPasswordOptions({ - server, - token: server.accessToken, - videoPasswords, - expectedStatus: HttpStatusCode.BAD_REQUEST_400, - mode - }) - }) - - it('Should fail with a password protected privacy and a too short password', async function () { - const videoPasswords = [ 'p' ] - - await checkVideoPasswordOptions({ - server, - token: server.accessToken, - videoPasswords, - expectedStatus: HttpStatusCode.BAD_REQUEST_400, - mode - }) - }) - - it('Should fail with a password protected privacy and a too long password', async function () { - const videoPasswords = [ 'Very very very very very very very very very very very very very very very very very very long password' ] - - await checkVideoPasswordOptions({ - server, - token: server.accessToken, - videoPasswords, - expectedStatus: HttpStatusCode.BAD_REQUEST_400, - mode - }) - }) - - it('Should fail with a password protected privacy and an empty password', async function () { - const videoPasswords = [ '' ] - - await checkVideoPasswordOptions({ - server, - token: server.accessToken, - videoPasswords, - expectedStatus: HttpStatusCode.BAD_REQUEST_400, - mode - }) - }) - - it('Should fail with a password protected privacy and duplicated passwords', async function () { - const videoPasswords = [ 'password', 'password' ] - - await checkVideoPasswordOptions({ - server, - token: server.accessToken, - videoPasswords, - expectedStatus: HttpStatusCode.BAD_REQUEST_400, - mode - }) - }) - - if (mode === 'updatePasswords') { - it('Should fail for an unauthenticated user', async function () { - const videoPasswords = [ 'password' ] - await checkVideoPasswordOptions({ - server, - token: null, - videoPasswords, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401, - mode - }) - }) - - it('Should fail for an unauthorized user', async function () { - const videoPasswords = [ 'password' ] - await checkVideoPasswordOptions({ - server, - token: userAccessToken, - videoPasswords, - expectedStatus: HttpStatusCode.FORBIDDEN_403, - mode - }) - }) - } - - it('Should succeed with a password protected privacy and correct passwords', async function () { - const videoPasswords = [ 'password1', 'password2' ] - const expectedStatus = mode === 'updatePasswords' || mode === 'updateVideo' - ? HttpStatusCode.NO_CONTENT_204 - : HttpStatusCode.OK_200 - - await checkVideoPasswordOptions({ server, token: server.accessToken, videoPasswords, expectedStatus, mode }) - }) - } - - describe('When adding or updating a video', function () { - describe('Resumable upload', function () { - validateVideoPasswordList('uploadResumable') - }) - - describe('Legacy upload', function () { - validateVideoPasswordList('uploadLegacy') - }) - - describe('When importing a video', function () { - validateVideoPasswordList('import') - }) - - describe('When updating a video', function () { - validateVideoPasswordList('updateVideo') - }) - - describe('When updating the password list of a video', function () { - validateVideoPasswordList('updatePasswords') - }) - - describe('When creating a live', function () { - validateVideoPasswordList('live') - }) - }) - - async function checkVideoAccessOptions (options: { - server: PeerTubeServer - token?: string - videoPassword?: string - expectedStatus: HttpStatusCode - mode: 'get' | 'getWithPassword' | 'getWithToken' | 'listCaptions' | 'createThread' | 'listThreads' | 'replyThread' | 'rate' | 'token' - }) { - const { server, token = null, videoPassword, expectedStatus, mode } = options - - if (mode === 'get') { - return server.videos.get({ id: video.id, expectedStatus }) - } - - if (mode === 'getWithToken') { - return server.videos.getWithToken({ - id: video.id, - token, - expectedStatus - }) - } - - if (mode === 'getWithPassword') { - return server.videos.getWithPassword({ - id: video.id, - token, - expectedStatus, - password: videoPassword - }) - } - - if (mode === 'rate') { - return server.videos.rate({ - id: video.id, - token, - expectedStatus, - rating: 'like', - videoPassword - }) - } - - if (mode === 'createThread') { - const fields = { text: 'super comment' } - const headers = videoPassword !== undefined && videoPassword !== null - ? { 'x-peertube-video-password': videoPassword } - : undefined - const body = await makePostBodyRequest({ - url: server.url, - path: path + video.uuid + '/comment-threads', - token, - fields, - headers, - expectedStatus - }) - return JSON.parse(body.text) - } - - if (mode === 'replyThread') { - const fields = { text: 'super reply' } - const headers = videoPassword !== undefined && videoPassword !== null - ? { 'x-peertube-video-password': videoPassword } - : undefined - return makePostBodyRequest({ - url: server.url, - path: path + video.uuid + '/comments/' + commentId, - token, - fields, - headers, - expectedStatus - }) - } - if (mode === 'listThreads') { - return server.comments.listThreads({ - videoId: video.id, - token, - expectedStatus, - videoPassword - }) - } - - if (mode === 'listCaptions') { - return server.captions.list({ - videoId: video.id, - token, - expectedStatus, - videoPassword - }) - } - - if (mode === 'token') { - return server.videoToken.create({ - videoId: video.id, - token, - expectedStatus, - videoPassword - }) - } - } - - function checkVideoError (error: any, mode: 'providePassword' | 'incorrectPassword') { - const serverCode = mode === 'providePassword' - ? ServerErrorCode.VIDEO_REQUIRES_PASSWORD - : ServerErrorCode.INCORRECT_VIDEO_PASSWORD - - const message = mode === 'providePassword' - ? 'Please provide a password to access this password protected video' - : 'Incorrect video password. Access to the video is denied.' - - if (!error.code) { - error = JSON.parse(error.text) - } - - expect(error.code).to.equal(serverCode) - expect(error.detail).to.equal(message) - expect(error.error).to.equal(message) - - expect(error.status).to.equal(HttpStatusCode.FORBIDDEN_403) - } - - function validateVideoAccess (mode: 'get' | 'listCaptions' | 'createThread' | 'listThreads' | 'replyThread' | 'rate' | 'token') { - const requiresUserAuth = [ 'createThread', 'replyThread', 'rate' ].includes(mode) - let tokens: string[] - if (!requiresUserAuth) { - it('Should fail without providing a password for an unlogged user', async function () { - const body = await checkVideoAccessOptions({ server, expectedStatus: HttpStatusCode.FORBIDDEN_403, mode }) - const error = body as unknown as PeerTubeProblemDocument - - checkVideoError(error, 'providePassword') - }) - } - - it('Should fail without providing a password for an unauthorised user', async function () { - const tmp = mode === 'get' ? 'getWithToken' : mode - - const body = await checkVideoAccessOptions({ - server, - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403, - mode: tmp - }) - - const error = body as unknown as PeerTubeProblemDocument - - checkVideoError(error, 'providePassword') - }) - - it('Should fail if a wrong password is entered', async function () { - const tmp = mode === 'get' ? 'getWithPassword' : mode - tokens = [ userAccessToken, server.accessToken ] - - if (!requiresUserAuth) tokens.push(null) - - for (const token of tokens) { - const body = await checkVideoAccessOptions({ - server, - token, - videoPassword: 'toto', - expectedStatus: HttpStatusCode.FORBIDDEN_403, - mode: tmp - }) - const error = body as unknown as PeerTubeProblemDocument - - checkVideoError(error, 'incorrectPassword') - } - }) - - it('Should fail if an empty password is entered', async function () { - const tmp = mode === 'get' ? 'getWithPassword' : mode - - for (const token of tokens) { - const body = await checkVideoAccessOptions({ - server, - token, - videoPassword: '', - expectedStatus: HttpStatusCode.FORBIDDEN_403, - mode: tmp - }) - const error = body as unknown as PeerTubeProblemDocument - - checkVideoError(error, 'incorrectPassword') - } - }) - - it('Should fail if an inccorect password containing the correct password is entered', async function () { - const tmp = mode === 'get' ? 'getWithPassword' : mode - - for (const token of tokens) { - const body = await checkVideoAccessOptions({ - server, - token, - videoPassword: 'password11', - expectedStatus: HttpStatusCode.FORBIDDEN_403, - mode: tmp - }) - const error = body as unknown as PeerTubeProblemDocument - - checkVideoError(error, 'incorrectPassword') - } - }) - - it('Should succeed without providing a password for an authorised user', async function () { - const tmp = mode === 'get' ? 'getWithToken' : mode - const expectedStatus = mode === 'rate' ? HttpStatusCode.NO_CONTENT_204 : HttpStatusCode.OK_200 - - const body = await checkVideoAccessOptions({ server, token: server.accessToken, expectedStatus, mode: tmp }) - - if (mode === 'createThread') commentId = body.comment.id - }) - - it('Should succeed using correct passwords', async function () { - const tmp = mode === 'get' ? 'getWithPassword' : mode - const expectedStatus = mode === 'rate' ? HttpStatusCode.NO_CONTENT_204 : HttpStatusCode.OK_200 - - for (const token of tokens) { - await checkVideoAccessOptions({ server, videoPassword: 'password1', token, expectedStatus, mode: tmp }) - await checkVideoAccessOptions({ server, videoPassword: 'password2', token, expectedStatus, mode: tmp }) - } - }) - } - - describe('When accessing password protected video', function () { - - describe('For getting a password protected video', function () { - validateVideoAccess('get') - }) - - describe('For rating a video', function () { - validateVideoAccess('rate') - }) - - describe('For creating a thread', function () { - validateVideoAccess('createThread') - }) - - describe('For replying to a thread', function () { - validateVideoAccess('replyThread') - }) - - describe('For listing threads', function () { - validateVideoAccess('listThreads') - }) - - describe('For getting captions', function () { - validateVideoAccess('listCaptions') - }) - - describe('For creating video file token', function () { - validateVideoAccess('token') - }) - }) - - describe('When listing passwords', function () { - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path + video.uuid + '/passwords', server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path + video.uuid + '/passwords', server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path + video.uuid + '/passwords', server.accessToken) - }) - - it('Should fail for unauthenticated user', async function () { - await server.videoPasswords.list({ - token: null, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401, - videoId: video.id - }) - }) - - it('Should fail for unauthorized user', async function () { - await server.videoPasswords.list({ - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403, - videoId: video.id - }) - }) - - it('Should succeed with the correct parameters', async function () { - await server.videoPasswords.list({ - token: server.accessToken, - expectedStatus: HttpStatusCode.OK_200, - videoId: video.id - }) - }) - }) - - describe('When deleting a password', async function () { - const passwords = (await server.videoPasswords.list({ videoId: video.id })).data - - it('Should fail with wrong password id', async function () { - await server.videoPasswords.remove({ id: -1, videoId: video.id, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should fail for unauthenticated user', async function () { - await server.videoPasswords.remove({ - id: passwords[0].id, - token: null, - videoId: video.id, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail for unauthorized user', async function () { - await server.videoPasswords.remove({ - id: passwords[0].id, - token: userAccessToken, - videoId: video.id, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail for non password protected video', async function () { - publicVideo = await server.videos.quickUpload({ name: 'public video' }) - await server.videoPasswords.remove({ id: passwords[0].id, videoId: publicVideo.id, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail for password not linked to correct video', async function () { - const video2 = await server.videos.quickUpload({ - name: 'password protected video', - privacy: VideoPrivacy.PASSWORD_PROTECTED, - videoPasswords: [ 'password1', 'password2' ] - }) - await server.videoPasswords.remove({ id: passwords[0].id, videoId: video2.id, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should succeed with correct parameter', async function () { - await server.videoPasswords.remove({ id: passwords[0].id, videoId: video.id, expectedStatus: HttpStatusCode.NO_CONTENT_204 }) - }) - - it('Should fail for last password of a video', async function () { - await server.videoPasswords.remove({ id: passwords[1].id, videoId: video.id, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/video-playlists.ts b/server/tests/api/check-params/video-playlists.ts deleted file mode 100644 index 8c3233e0b..000000000 --- a/server/tests/api/check-params/video-playlists.ts +++ /dev/null @@ -1,695 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '@server/tests/shared' -import { - HttpStatusCode, - VideoPlaylistCreate, - VideoPlaylistCreateResult, - VideoPlaylistElementCreate, - VideoPlaylistElementUpdate, - VideoPlaylistPrivacy, - VideoPlaylistReorder, - VideoPlaylistType -} from '@shared/models' -import { - cleanupTests, - createSingleServer, - makeGetRequest, - PeerTubeServer, - PlaylistsCommand, - setAccessTokensToServers, - setDefaultVideoChannel -} from '@shared/server-commands' - -describe('Test video playlists API validator', function () { - let server: PeerTubeServer - let userAccessToken: string - - let playlist: VideoPlaylistCreateResult - let privatePlaylistUUID: string - - let watchLaterPlaylistId: number - let videoId: number - let elementId: number - - let command: PlaylistsCommand - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - userAccessToken = await server.users.generateUserAndToken('user1') - videoId = (await server.videos.quickUpload({ name: 'video 1' })).id - - command = server.playlists - - { - const { data } = await command.listByAccount({ - token: server.accessToken, - handle: 'root', - start: 0, - count: 5, - playlistType: VideoPlaylistType.WATCH_LATER - }) - watchLaterPlaylistId = data[0].id - } - - { - playlist = await command.create({ - attributes: { - displayName: 'super playlist', - privacy: VideoPlaylistPrivacy.PUBLIC, - videoChannelId: server.store.channel.id - } - }) - } - - { - const created = await command.create({ - attributes: { - displayName: 'private', - privacy: VideoPlaylistPrivacy.PRIVATE - } - }) - privatePlaylistUUID = created.uuid - } - }) - - describe('When listing playlists', function () { - const globalPath = '/api/v1/video-playlists' - const accountPath = '/api/v1/accounts/root/video-playlists' - const videoChannelPath = '/api/v1/video-channels/root_channel/video-playlists' - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, globalPath, server.accessToken) - await checkBadStartPagination(server.url, accountPath, server.accessToken) - await checkBadStartPagination(server.url, videoChannelPath, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, globalPath, server.accessToken) - await checkBadCountPagination(server.url, accountPath, server.accessToken) - await checkBadCountPagination(server.url, videoChannelPath, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, globalPath, server.accessToken) - await checkBadSortPagination(server.url, accountPath, server.accessToken) - await checkBadSortPagination(server.url, videoChannelPath, server.accessToken) - }) - - it('Should fail with a bad playlist type', async function () { - await makeGetRequest({ url: server.url, path: globalPath, query: { playlistType: 3 } }) - await makeGetRequest({ url: server.url, path: accountPath, query: { playlistType: 3 } }) - await makeGetRequest({ url: server.url, path: videoChannelPath, query: { playlistType: 3 } }) - }) - - it('Should fail with a bad account parameter', async function () { - const accountPath = '/api/v1/accounts/root2/video-playlists' - - await makeGetRequest({ - url: server.url, - path: accountPath, - expectedStatus: HttpStatusCode.NOT_FOUND_404, - token: server.accessToken - }) - }) - - it('Should fail with a bad video channel parameter', async function () { - const accountPath = '/api/v1/video-channels/bad_channel/video-playlists' - - await makeGetRequest({ - url: server.url, - path: accountPath, - expectedStatus: HttpStatusCode.NOT_FOUND_404, - token: server.accessToken - }) - }) - - it('Should success with the correct parameters', async function () { - await makeGetRequest({ url: server.url, path: globalPath, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken }) - await makeGetRequest({ url: server.url, path: accountPath, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken }) - await makeGetRequest({ - url: server.url, - path: videoChannelPath, - expectedStatus: HttpStatusCode.OK_200, - token: server.accessToken - }) - }) - }) - - describe('When listing videos of a playlist', function () { - const path = '/api/v1/video-playlists/' - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path + playlist.shortUUID + '/videos', server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path + playlist.shortUUID + '/videos', server.accessToken) - }) - - it('Should success with the correct parameters', async function () { - await makeGetRequest({ url: server.url, path: path + playlist.shortUUID + '/videos', expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - describe('When getting a video playlist', function () { - it('Should fail with a bad id or uuid', async function () { - await command.get({ playlistId: 'toto', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with an unknown playlist', async function () { - await command.get({ playlistId: 42, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should fail to get an unlisted playlist with the number id', async function () { - const playlist = await command.create({ - attributes: { - displayName: 'super playlist', - videoChannelId: server.store.channel.id, - privacy: VideoPlaylistPrivacy.UNLISTED - } - }) - - await command.get({ playlistId: playlist.id, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - await command.get({ playlistId: playlist.uuid, expectedStatus: HttpStatusCode.OK_200 }) - }) - - it('Should succeed with the correct params', async function () { - await command.get({ playlistId: playlist.uuid, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - describe('When creating/updating a video playlist', function () { - const getBase = ( - attributes?: Partial, - wrapper?: Partial[0]> - ) => { - return { - attributes: { - displayName: 'display name', - privacy: VideoPlaylistPrivacy.UNLISTED, - thumbnailfile: 'custom-thumbnail.jpg', - videoChannelId: server.store.channel.id, - - ...attributes - }, - - expectedStatus: HttpStatusCode.BAD_REQUEST_400, - - ...wrapper - } - } - const getUpdate = (params: any, playlistId: number | string) => { - return { ...params, playlistId } - } - - it('Should fail with an unauthenticated user', async function () { - const params = getBase({}, { token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - - await command.create(params) - await command.update(getUpdate(params, playlist.shortUUID)) - }) - - it('Should fail without displayName', async function () { - const params = getBase({ displayName: undefined }) - - await command.create(params) - }) - - it('Should fail with an incorrect display name', async function () { - const params = getBase({ displayName: 's'.repeat(300) }) - - await command.create(params) - await command.update(getUpdate(params, playlist.shortUUID)) - }) - - it('Should fail with an incorrect description', async function () { - const params = getBase({ description: 't' }) - - await command.create(params) - await command.update(getUpdate(params, playlist.shortUUID)) - }) - - it('Should fail with an incorrect privacy', async function () { - const params = getBase({ privacy: 45 as any }) - - await command.create(params) - await command.update(getUpdate(params, playlist.shortUUID)) - }) - - it('Should fail with an unknown video channel id', async function () { - const params = getBase({ videoChannelId: 42 }, { expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - - await command.create(params) - await command.update(getUpdate(params, playlist.shortUUID)) - }) - - it('Should fail with an incorrect thumbnail file', async function () { - const params = getBase({ thumbnailfile: 'video_short.mp4' }) - - await command.create(params) - await command.update(getUpdate(params, playlist.shortUUID)) - }) - - it('Should fail with a thumbnail file too big', async function () { - const params = getBase({ thumbnailfile: 'custom-preview-big.png' }) - - await command.create(params) - await command.update(getUpdate(params, playlist.shortUUID)) - }) - - it('Should fail to set "public" a playlist not assigned to a channel', async function () { - const params = getBase({ privacy: VideoPlaylistPrivacy.PUBLIC, videoChannelId: undefined }) - const params2 = getBase({ privacy: VideoPlaylistPrivacy.PUBLIC, videoChannelId: 'null' as any }) - const params3 = getBase({ privacy: undefined, videoChannelId: 'null' as any }) - - await command.create(params) - await command.create(params2) - await command.update(getUpdate(params, privatePlaylistUUID)) - await command.update(getUpdate(params2, playlist.shortUUID)) - await command.update(getUpdate(params3, playlist.shortUUID)) - }) - - it('Should fail with an unknown playlist to update', async function () { - await command.update(getUpdate( - getBase({}, { expectedStatus: HttpStatusCode.NOT_FOUND_404 }), - 42 - )) - }) - - it('Should fail to update a playlist of another user', async function () { - await command.update(getUpdate( - getBase({}, { token: userAccessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }), - playlist.shortUUID - )) - }) - - it('Should fail to update the watch later playlist', async function () { - await command.update(getUpdate( - getBase({}, { expectedStatus: HttpStatusCode.BAD_REQUEST_400 }), - watchLaterPlaylistId - )) - }) - - it('Should succeed with the correct params', async function () { - { - const params = getBase({}, { expectedStatus: HttpStatusCode.OK_200 }) - await command.create(params) - } - - { - const params = getBase({}, { expectedStatus: HttpStatusCode.NO_CONTENT_204 }) - await command.update(getUpdate(params, playlist.shortUUID)) - } - }) - }) - - describe('When adding an element in a playlist', function () { - const getBase = ( - attributes?: Partial, - wrapper?: Partial[0]> - ) => { - return { - attributes: { - videoId, - startTimestamp: 2, - stopTimestamp: 3, - - ...attributes - }, - - expectedStatus: HttpStatusCode.BAD_REQUEST_400, - playlistId: playlist.id, - - ...wrapper - } - } - - it('Should fail with an unauthenticated user', async function () { - const params = getBase({}, { token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - await command.addElement(params) - }) - - it('Should fail with the playlist of another user', async function () { - const params = getBase({}, { token: userAccessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await command.addElement(params) - }) - - it('Should fail with an unknown or incorrect playlist id', async function () { - { - const params = getBase({}, { playlistId: 'toto' }) - await command.addElement(params) - } - - { - const params = getBase({}, { playlistId: 42, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - await command.addElement(params) - } - }) - - it('Should fail with an unknown or incorrect video id', async function () { - const params = getBase({ videoId: 42 }, { expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - await command.addElement(params) - }) - - it('Should fail with a bad start/stop timestamp', async function () { - { - const params = getBase({ startTimestamp: -42 }) - await command.addElement(params) - } - - { - const params = getBase({ stopTimestamp: 'toto' as any }) - await command.addElement(params) - } - }) - - it('Succeed with the correct params', async function () { - const params = getBase({}, { expectedStatus: HttpStatusCode.OK_200 }) - const created = await command.addElement(params) - elementId = created.id - }) - }) - - describe('When updating an element in a playlist', function () { - const getBase = ( - attributes?: Partial, - wrapper?: Partial[0]> - ) => { - return { - attributes: { - startTimestamp: 1, - stopTimestamp: 2, - - ...attributes - }, - - elementId, - playlistId: playlist.id, - expectedStatus: HttpStatusCode.BAD_REQUEST_400, - - ...wrapper - } - } - - it('Should fail with an unauthenticated user', async function () { - const params = getBase({}, { token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - await command.updateElement(params) - }) - - it('Should fail with the playlist of another user', async function () { - const params = getBase({}, { token: userAccessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await command.updateElement(params) - }) - - it('Should fail with an unknown or incorrect playlist id', async function () { - { - const params = getBase({}, { playlistId: 'toto' }) - await command.updateElement(params) - } - - { - const params = getBase({}, { playlistId: 42, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - await command.updateElement(params) - } - }) - - it('Should fail with an unknown or incorrect playlistElement id', async function () { - { - const params = getBase({}, { elementId: 'toto' }) - await command.updateElement(params) - } - - { - const params = getBase({}, { elementId: 42, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - await command.updateElement(params) - } - }) - - it('Should fail with a bad start/stop timestamp', async function () { - { - const params = getBase({ startTimestamp: 'toto' as any }) - await command.updateElement(params) - } - - { - const params = getBase({ stopTimestamp: -42 }) - await command.updateElement(params) - } - }) - - it('Should fail with an unknown element', async function () { - const params = getBase({}, { elementId: 888, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - await command.updateElement(params) - }) - - it('Succeed with the correct params', async function () { - const params = getBase({}, { expectedStatus: HttpStatusCode.NO_CONTENT_204 }) - await command.updateElement(params) - }) - }) - - describe('When reordering elements of a playlist', function () { - let videoId3: number - let videoId4: number - - const getBase = ( - attributes?: Partial, - wrapper?: Partial[0]> - ) => { - return { - attributes: { - startPosition: 1, - insertAfterPosition: 2, - reorderLength: 3, - - ...attributes - }, - - playlistId: playlist.shortUUID, - expectedStatus: HttpStatusCode.BAD_REQUEST_400, - - ...wrapper - } - } - - before(async function () { - videoId3 = (await server.videos.quickUpload({ name: 'video 3' })).id - videoId4 = (await server.videos.quickUpload({ name: 'video 4' })).id - - for (const id of [ videoId3, videoId4 ]) { - await command.addElement({ playlistId: playlist.shortUUID, attributes: { videoId: id } }) - } - }) - - it('Should fail with an unauthenticated user', async function () { - const params = getBase({}, { token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - await command.reorderElements(params) - }) - - it('Should fail with the playlist of another user', async function () { - const params = getBase({}, { token: userAccessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await command.reorderElements(params) - }) - - it('Should fail with an invalid playlist', async function () { - { - const params = getBase({}, { playlistId: 'toto' }) - await command.reorderElements(params) - } - - { - const params = getBase({}, { playlistId: 42, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - await command.reorderElements(params) - } - }) - - it('Should fail with an invalid start position', async function () { - { - const params = getBase({ startPosition: -1 }) - await command.reorderElements(params) - } - - { - const params = getBase({ startPosition: 'toto' as any }) - await command.reorderElements(params) - } - - { - const params = getBase({ startPosition: 42 }) - await command.reorderElements(params) - } - }) - - it('Should fail with an invalid insert after position', async function () { - { - const params = getBase({ insertAfterPosition: 'toto' as any }) - await command.reorderElements(params) - } - - { - const params = getBase({ insertAfterPosition: -2 }) - await command.reorderElements(params) - } - - { - const params = getBase({ insertAfterPosition: 42 }) - await command.reorderElements(params) - } - }) - - it('Should fail with an invalid reorder length', async function () { - { - const params = getBase({ reorderLength: 'toto' as any }) - await command.reorderElements(params) - } - - { - const params = getBase({ reorderLength: -2 }) - await command.reorderElements(params) - } - - { - const params = getBase({ reorderLength: 42 }) - await command.reorderElements(params) - } - }) - - it('Succeed with the correct params', async function () { - const params = getBase({}, { expectedStatus: HttpStatusCode.NO_CONTENT_204 }) - await command.reorderElements(params) - }) - }) - - describe('When checking exists in playlist endpoint', function () { - const path = '/api/v1/users/me/video-playlists/videos-exist' - - it('Should fail with an unauthenticated user', async function () { - await makeGetRequest({ - url: server.url, - path, - query: { videoIds: [ 1, 2 ] }, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with invalid video ids', async function () { - await makeGetRequest({ - url: server.url, - token: server.accessToken, - path, - query: { videoIds: 'toto' } - }) - - await makeGetRequest({ - url: server.url, - token: server.accessToken, - path, - query: { videoIds: [ 'toto' ] } - }) - - await makeGetRequest({ - url: server.url, - token: server.accessToken, - path, - query: { videoIds: [ 1, 'toto' ] } - }) - }) - - it('Should succeed with the correct params', async function () { - await makeGetRequest({ - url: server.url, - token: server.accessToken, - path, - query: { videoIds: [ 1, 2 ] }, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - }) - - describe('When deleting an element in a playlist', function () { - const getBase = (wrapper: Partial[0]>) => { - return { - elementId, - playlistId: playlist.uuid, - expectedStatus: HttpStatusCode.BAD_REQUEST_400, - - ...wrapper - } - } - - it('Should fail with an unauthenticated user', async function () { - const params = getBase({ token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - await command.removeElement(params) - }) - - it('Should fail with the playlist of another user', async function () { - const params = getBase({ token: userAccessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await command.removeElement(params) - }) - - it('Should fail with an unknown or incorrect playlist id', async function () { - { - const params = getBase({ playlistId: 'toto' }) - await command.removeElement(params) - } - - { - const params = getBase({ playlistId: 42, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - await command.removeElement(params) - } - }) - - it('Should fail with an unknown or incorrect video id', async function () { - { - const params = getBase({ elementId: 'toto' as any }) - await command.removeElement(params) - } - - { - const params = getBase({ elementId: 42, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - await command.removeElement(params) - } - }) - - it('Should fail with an unknown element', async function () { - const params = getBase({ elementId: 888, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - await command.removeElement(params) - }) - - it('Succeed with the correct params', async function () { - const params = getBase({ expectedStatus: HttpStatusCode.NO_CONTENT_204 }) - await command.removeElement(params) - }) - }) - - describe('When deleting a playlist', function () { - it('Should fail with an unknown playlist', async function () { - await command.delete({ playlistId: 42, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should fail with a playlist of another user', async function () { - await command.delete({ token: userAccessToken, playlistId: playlist.uuid, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should fail with the watch later playlist', async function () { - await command.delete({ playlistId: watchLaterPlaylistId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should succeed with the correct params', async function () { - await command.delete({ playlistId: playlist.uuid }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/video-source.ts b/server/tests/api/check-params/video-source.ts deleted file mode 100644 index 767590d5e..000000000 --- a/server/tests/api/check-params/video-source.ts +++ /dev/null @@ -1,154 +0,0 @@ -import { HttpStatusCode } from '@shared/models' -import { - cleanupTests, - createSingleServer, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - waitJobs -} from '@shared/server-commands' - -describe('Test video sources API validator', function () { - let server: PeerTubeServer = null - let uuid: string - let userToken: string - - before(async function () { - this.timeout(120000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - userToken = await server.users.generateUserAndToken('user1') - }) - - describe('When getting latest source', function () { - - before(async function () { - const created = await server.videos.quickUpload({ name: 'video' }) - uuid = created.uuid - }) - - it('Should fail without a valid uuid', async function () { - await server.videos.getSource({ id: '4da6fde3-88f7-4d16-b119-108df563d0b0', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should receive 404 when passing a non existing video id', async function () { - await server.videos.getSource({ id: '4da6fde3-88f7-4d16-b119-108df5630b06', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should not get the source as unauthenticated', async function () { - await server.videos.getSource({ id: uuid, expectedStatus: HttpStatusCode.UNAUTHORIZED_401, token: null }) - }) - - it('Should not get the source with another user', async function () { - await server.videos.getSource({ id: uuid, expectedStatus: HttpStatusCode.FORBIDDEN_403, token: userToken }) - }) - - it('Should succeed with the correct parameters get the source as another user', async function () { - await server.videos.getSource({ id: uuid }) - }) - }) - - describe('When updating source video file', function () { - let userAccessToken: string - let userId: number - - let videoId: string - let userVideoId: string - - before(async function () { - const res = await server.users.generate('user2') - userAccessToken = res.token - userId = res.userId - - const { uuid } = await server.videos.quickUpload({ name: 'video' }) - videoId = uuid - - await waitJobs([ server ]) - }) - - it('Should fail if not enabled on the instance', async function () { - await server.config.disableFileUpdate() - - await server.videos.replaceSourceFile({ videoId, fixture: 'video_short.mp4', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should fail on an unknown video', async function () { - await server.config.enableFileUpdate() - - await server.videos.replaceSourceFile({ videoId: 404, fixture: 'video_short.mp4', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should fail with an invalid video', async function () { - await server.config.enableLive({ allowReplay: false }) - - const { video } = await server.live.quickCreate({ saveReplay: false, permanentLive: true }) - await server.videos.replaceSourceFile({ - videoId: video.uuid, - fixture: 'video_short.mp4', - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail without token', async function () { - await server.videos.replaceSourceFile({ - token: null, - videoId, - fixture: 'video_short.mp4', - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with another user', async function () { - await server.videos.replaceSourceFile({ - token: userAccessToken, - videoId, - fixture: 'video_short.mp4', - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with an incorrect input file', async function () { - await server.videos.replaceSourceFile({ - fixture: 'video_short_fake.webm', - videoId, - completedExpectedStatus: HttpStatusCode.UNPROCESSABLE_ENTITY_422 - }) - - await server.videos.replaceSourceFile({ - fixture: 'video_short.mkv', - videoId, - expectedStatus: HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415 - }) - }) - - it('Should fail if quota is exceeded', async function () { - this.timeout(60000) - - const { uuid } = await server.videos.quickUpload({ name: 'user video' }) - userVideoId = uuid - await waitJobs([ server ]) - - await server.users.update({ userId, videoQuota: 1 }) - await server.videos.replaceSourceFile({ - token: userAccessToken, - videoId: uuid, - fixture: 'video_short.mp4', - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should succeed with the correct params', async function () { - this.timeout(60000) - - await server.users.update({ userId, videoQuota: 1000 * 1000 * 1000 }) - await server.videos.replaceSourceFile({ videoId: userVideoId, fixture: 'video_short.mp4' }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/video-storyboards.ts b/server/tests/api/check-params/video-storyboards.ts deleted file mode 100644 index c038e7370..000000000 --- a/server/tests/api/check-params/video-storyboards.ts +++ /dev/null @@ -1,45 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { HttpStatusCode, VideoPrivacy } from '@shared/models' -import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' - -describe('Test video storyboards API validator', function () { - let server: PeerTubeServer - - let publicVideo: { uuid: string } - let privateVideo: { uuid: string } - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(120000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - - publicVideo = await server.videos.quickUpload({ name: 'public' }) - privateVideo = await server.videos.quickUpload({ name: 'private', privacy: VideoPrivacy.PRIVATE }) - }) - - it('Should fail without a valid uuid', async function () { - await server.storyboard.list({ id: '4da6fde3-88f7-4d16-b119-108df563d0b0', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should receive 404 when passing a non existing video id', async function () { - await server.storyboard.list({ id: '4da6fde3-88f7-4d16-b119-108df5630b06', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should not get the private storyboard without the appropriate token', async function () { - await server.storyboard.list({ id: privateVideo.uuid, expectedStatus: HttpStatusCode.UNAUTHORIZED_401, token: null }) - await server.storyboard.list({ id: publicVideo.uuid, expectedStatus: HttpStatusCode.OK_200, token: null }) - }) - - it('Should succeed with the correct parameters', async function () { - await server.storyboard.list({ id: privateVideo.uuid }) - await server.storyboard.list({ id: publicVideo.uuid }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/video-studio.ts b/server/tests/api/check-params/video-studio.ts deleted file mode 100644 index 4ac0d93ed..000000000 --- a/server/tests/api/check-params/video-studio.ts +++ /dev/null @@ -1,388 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { HttpStatusCode, VideoStudioTask } from '@shared/models' -import { - cleanupTests, - createSingleServer, - PeerTubeServer, - setAccessTokensToServers, - VideoStudioCommand, - waitJobs -} from '@shared/server-commands' - -describe('Test video studio API validator', function () { - let server: PeerTubeServer - let command: VideoStudioCommand - let userAccessToken: string - let videoUUID: string - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(120_000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - userAccessToken = await server.users.generateUserAndToken('user1') - - await server.config.enableMinimumTranscoding() - - const { uuid } = await server.videos.quickUpload({ name: 'video' }) - videoUUID = uuid - - command = server.videoStudio - - await waitJobs([ server ]) - }) - - describe('Task creation', function () { - - describe('Config settings', function () { - - it('Should fail if studio is disabled', async function () { - await server.config.updateExistingSubConfig({ - newConfig: { - videoStudio: { - enabled: false - } - } - }) - - await command.createEditionTasks({ - videoId: videoUUID, - tasks: VideoStudioCommand.getComplexTask(), - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail to enable studio if transcoding is disabled', async function () { - await server.config.updateExistingSubConfig({ - newConfig: { - videoStudio: { - enabled: true - }, - transcoding: { - enabled: false - } - }, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should succeed to enable video studio', async function () { - await server.config.updateExistingSubConfig({ - newConfig: { - videoStudio: { - enabled: true - }, - transcoding: { - enabled: true - } - } - }) - }) - }) - - describe('Common tasks', function () { - - it('Should fail without token', async function () { - await command.createEditionTasks({ - token: null, - videoId: videoUUID, - tasks: VideoStudioCommand.getComplexTask(), - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with another user token', async function () { - await command.createEditionTasks({ - token: userAccessToken, - videoId: videoUUID, - tasks: VideoStudioCommand.getComplexTask(), - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with an invalid video', async function () { - await command.createEditionTasks({ - videoId: 'tintin', - tasks: VideoStudioCommand.getComplexTask(), - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with an unknown video', async function () { - await command.createEditionTasks({ - videoId: 42, - tasks: VideoStudioCommand.getComplexTask(), - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail with an already in transcoding state video', async function () { - this.timeout(60000) - - const { uuid } = await server.videos.quickUpload({ name: 'transcoded video' }) - await waitJobs([ server ]) - - await server.jobs.pauseJobQueue() - await server.videos.runTranscoding({ videoId: uuid, transcodingType: 'hls' }) - - await command.createEditionTasks({ - videoId: uuid, - tasks: VideoStudioCommand.getComplexTask(), - expectedStatus: HttpStatusCode.CONFLICT_409 - }) - - await server.jobs.resumeJobQueue() - }) - - it('Should fail with a bad complex task', async function () { - await command.createEditionTasks({ - videoId: videoUUID, - tasks: [ - { - name: 'cut', - options: { - start: 1, - end: 2 - } - }, - { - name: 'hadock', - options: { - start: 1, - end: 2 - } - } - ] as any, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail without task', async function () { - await command.createEditionTasks({ - videoId: videoUUID, - tasks: [], - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with too many tasks', async function () { - const tasks: VideoStudioTask[] = [] - - for (let i = 0; i < 110; i++) { - tasks.push({ - name: 'cut', - options: { - start: 1 - } - }) - } - - await command.createEditionTasks({ - videoId: videoUUID, - tasks, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should succeed with correct parameters', async function () { - await server.jobs.pauseJobQueue() - - await command.createEditionTasks({ - videoId: videoUUID, - tasks: VideoStudioCommand.getComplexTask(), - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - - it('Should fail with a video that is already waiting for edition', async function () { - this.timeout(120000) - - await command.createEditionTasks({ - videoId: videoUUID, - tasks: VideoStudioCommand.getComplexTask(), - expectedStatus: HttpStatusCode.CONFLICT_409 - }) - - await server.jobs.resumeJobQueue() - - await waitJobs([ server ]) - }) - }) - - describe('Cut task', function () { - - async function cut (start: number, end: number, expectedStatus = HttpStatusCode.BAD_REQUEST_400) { - await command.createEditionTasks({ - videoId: videoUUID, - tasks: [ - { - name: 'cut', - options: { - start, - end - } - } - ], - expectedStatus - }) - } - - it('Should fail with bad start/end', async function () { - const invalid = [ - 'tintin', - -1, - undefined - ] - - for (const value of invalid) { - await cut(value as any, undefined) - await cut(undefined, value as any) - } - }) - - it('Should fail with the same start/end', async function () { - await cut(2, 2) - }) - - it('Should fail with inconsistents start/end', async function () { - await cut(2, 1) - }) - - it('Should fail without start and end', async function () { - await cut(undefined, undefined) - }) - - it('Should succeed with the correct params', async function () { - this.timeout(120000) - - await cut(0, 2, HttpStatusCode.NO_CONTENT_204) - - await waitJobs([ server ]) - }) - }) - - describe('Watermark task', function () { - - async function addWatermark (file: string, expectedStatus = HttpStatusCode.BAD_REQUEST_400) { - await command.createEditionTasks({ - videoId: videoUUID, - tasks: [ - { - name: 'add-watermark', - options: { - file - } - } - ], - expectedStatus - }) - } - - it('Should fail without waterkmark', async function () { - await addWatermark(undefined) - }) - - it('Should fail with an invalid watermark', async function () { - await addWatermark('video_short.mp4') - }) - - it('Should succeed with the correct params', async function () { - this.timeout(120000) - - await addWatermark('custom-thumbnail.jpg', HttpStatusCode.NO_CONTENT_204) - - await waitJobs([ server ]) - }) - }) - - describe('Intro/Outro task', function () { - - async function addIntroOutro (type: 'add-intro' | 'add-outro', file: string, expectedStatus = HttpStatusCode.BAD_REQUEST_400) { - await command.createEditionTasks({ - videoId: videoUUID, - tasks: [ - { - name: type, - options: { - file - } - } - ], - expectedStatus - }) - } - - it('Should fail without file', async function () { - await addIntroOutro('add-intro', undefined) - await addIntroOutro('add-outro', undefined) - }) - - it('Should fail with an invalid file', async function () { - await addIntroOutro('add-intro', 'custom-thumbnail.jpg') - await addIntroOutro('add-outro', 'custom-thumbnail.jpg') - }) - - it('Should fail with a file that does not contain video stream', async function () { - await addIntroOutro('add-intro', 'sample.ogg') - await addIntroOutro('add-outro', 'sample.ogg') - - }) - - it('Should succeed with the correct params', async function () { - this.timeout(120000) - - await addIntroOutro('add-intro', 'video_very_short_240p.mp4', HttpStatusCode.NO_CONTENT_204) - await waitJobs([ server ]) - - await addIntroOutro('add-outro', 'video_very_short_240p.mp4', HttpStatusCode.NO_CONTENT_204) - await waitJobs([ server ]) - }) - - it('Should check total quota when creating the task', async function () { - this.timeout(120000) - - const user = await server.users.create({ username: 'user_quota_1' }) - const token = await server.login.getAccessToken('user_quota_1') - const { uuid } = await server.videos.quickUpload({ token, name: 'video_quota_1', fixture: 'video_short.mp4' }) - - const addIntroOutroByUser = (type: 'add-intro' | 'add-outro', expectedStatus: HttpStatusCode) => { - return command.createEditionTasks({ - token, - videoId: uuid, - tasks: [ - { - name: type, - options: { - file: 'video_short.mp4' - } - } - ], - expectedStatus - }) - } - - await waitJobs([ server ]) - - const { videoQuotaUsed } = await server.users.getMyQuotaUsed({ token }) - await server.users.update({ userId: user.id, videoQuota: Math.round(videoQuotaUsed * 2.5) }) - - // Still valid - await addIntroOutroByUser('add-intro', HttpStatusCode.NO_CONTENT_204) - - await waitJobs([ server ]) - - // Too much quota - await addIntroOutroByUser('add-intro', HttpStatusCode.PAYLOAD_TOO_LARGE_413) - await addIntroOutroByUser('add-outro', HttpStatusCode.PAYLOAD_TOO_LARGE_413) - }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/video-token.ts b/server/tests/api/check-params/video-token.ts deleted file mode 100644 index 7cb3e84a2..000000000 --- a/server/tests/api/check-params/video-token.ts +++ /dev/null @@ -1,70 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { HttpStatusCode, VideoPrivacy } from '@shared/models' -import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' - -describe('Test video tokens', function () { - let server: PeerTubeServer - let privateVideoId: string - let passwordProtectedVideoId: string - let userToken: string - - const videoPassword = 'password' - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(300_000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - { - const { uuid } = await server.videos.quickUpload({ name: 'private video', privacy: VideoPrivacy.PRIVATE }) - privateVideoId = uuid - } - { - const { uuid } = await server.videos.quickUpload({ - name: 'password protected video', - privacy: VideoPrivacy.PASSWORD_PROTECTED, - videoPasswords: [ videoPassword ] - }) - passwordProtectedVideoId = uuid - } - userToken = await server.users.generateUserAndToken('user1') - }) - - it('Should not generate tokens on private video for unauthenticated user', async function () { - await server.videoToken.create({ videoId: privateVideoId, token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should not generate tokens of unknown video', async function () { - await server.videoToken.create({ videoId: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should not generate tokens with incorrect password', async function () { - await server.videoToken.create({ - videoId: passwordProtectedVideoId, - token: null, - expectedStatus: HttpStatusCode.FORBIDDEN_403, - videoPassword: 'incorrectPassword' - }) - }) - - it('Should not generate tokens of a non owned video', async function () { - await server.videoToken.create({ videoId: privateVideoId, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should generate token', async function () { - await server.videoToken.create({ videoId: privateVideoId }) - }) - - it('Should generate token on password protected video', async function () { - await server.videoToken.create({ videoId: passwordProtectedVideoId, videoPassword, token: null }) - await server.videoToken.create({ videoId: passwordProtectedVideoId, videoPassword, token: userToken }) - await server.videoToken.create({ videoId: passwordProtectedVideoId, videoPassword }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/videos-common-filters.ts b/server/tests/api/check-params/videos-common-filters.ts deleted file mode 100644 index 603f7f777..000000000 --- a/server/tests/api/check-params/videos-common-filters.ts +++ /dev/null @@ -1,163 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { HttpStatusCode, UserRole, VideoInclude, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createSingleServer, - makeGetRequest, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel -} from '@shared/server-commands' - -describe('Test video filters validators', function () { - let server: PeerTubeServer - let userAccessToken: string - let moderatorAccessToken: string - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - const user = { username: 'user1', password: 'my super password' } - await server.users.create({ username: user.username, password: user.password }) - userAccessToken = await server.login.getAccessToken(user) - - const moderator = { username: 'moderator', password: 'my super password' } - await server.users.create({ username: moderator.username, password: moderator.password, role: UserRole.MODERATOR }) - - moderatorAccessToken = await server.login.getAccessToken(moderator) - }) - - describe('When setting video filters', function () { - - const validIncludes = [ - VideoInclude.NONE, - VideoInclude.BLOCKED_OWNER, - VideoInclude.NOT_PUBLISHED_STATE | VideoInclude.BLACKLISTED - ] - - async function testEndpoints (options: { - token?: string - isLocal?: boolean - include?: VideoInclude - privacyOneOf?: VideoPrivacy[] - expectedStatus: HttpStatusCode - excludeAlreadyWatched?: boolean - unauthenticatedUser?: boolean - }) { - const paths = [ - '/api/v1/video-channels/root_channel/videos', - '/api/v1/accounts/root/videos', - '/api/v1/videos', - '/api/v1/search/videos' - ] - - for (const path of paths) { - const token = options.unauthenticatedUser - ? undefined - : options.token || server.accessToken - - await makeGetRequest({ - url: server.url, - path, - token, - query: { - isLocal: options.isLocal, - privacyOneOf: options.privacyOneOf, - include: options.include, - excludeAlreadyWatched: options.excludeAlreadyWatched - }, - expectedStatus: options.expectedStatus - }) - } - } - - it('Should fail with a bad privacyOneOf', async function () { - await testEndpoints({ privacyOneOf: [ 'toto' ] as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should succeed with a good privacyOneOf', async function () { - await testEndpoints({ privacyOneOf: [ VideoPrivacy.INTERNAL ], expectedStatus: HttpStatusCode.OK_200 }) - }) - - it('Should fail to use privacyOneOf with a simple user', async function () { - await testEndpoints({ - privacyOneOf: [ VideoPrivacy.INTERNAL ], - token: userAccessToken, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with a bad include', async function () { - await testEndpoints({ include: 'toto' as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should succeed with a good include', async function () { - for (const include of validIncludes) { - await testEndpoints({ include, expectedStatus: HttpStatusCode.OK_200 }) - } - }) - - it('Should fail to include more videos with a simple user', async function () { - for (const include of validIncludes) { - await testEndpoints({ token: userAccessToken, include, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - } - }) - - it('Should succeed to list all local/all with a moderator', async function () { - for (const include of validIncludes) { - await testEndpoints({ token: moderatorAccessToken, include, expectedStatus: HttpStatusCode.OK_200 }) - } - }) - - it('Should succeed to list all local/all with an admin', async function () { - for (const include of validIncludes) { - await testEndpoints({ token: server.accessToken, include, expectedStatus: HttpStatusCode.OK_200 }) - } - }) - - // Because we cannot authenticate the user on the RSS endpoint - it('Should fail on the feeds endpoint with the all filter', async function () { - for (const include of [ VideoInclude.NOT_PUBLISHED_STATE ]) { - await makeGetRequest({ - url: server.url, - path: '/feeds/videos.json', - expectedStatus: HttpStatusCode.UNAUTHORIZED_401, - query: { - include - } - }) - } - }) - - it('Should succeed on the feeds endpoint with the local filter', async function () { - await makeGetRequest({ - url: server.url, - path: '/feeds/videos.json', - expectedStatus: HttpStatusCode.OK_200, - query: { - isLocal: true - } - }) - }) - - it('Should fail when trying to exclude already watched videos for an unlogged user', async function () { - await testEndpoints({ excludeAlreadyWatched: true, unauthenticatedUser: true, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should succeed when trying to exclude already watched videos for a logged user', async function () { - await testEndpoints({ token: userAccessToken, excludeAlreadyWatched: true, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/videos-history.ts b/server/tests/api/check-params/videos-history.ts deleted file mode 100644 index d96fe7ca9..000000000 --- a/server/tests/api/check-params/videos-history.ts +++ /dev/null @@ -1,145 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { checkBadCountPagination, checkBadStartPagination } from '@server/tests/shared' -import { HttpStatusCode } from '@shared/models' -import { - cleanupTests, - createSingleServer, - makeDeleteRequest, - makeGetRequest, - makePostBodyRequest, - makePutBodyRequest, - PeerTubeServer, - setAccessTokensToServers -} from '@shared/server-commands' - -describe('Test videos history API validator', function () { - const myHistoryPath = '/api/v1/users/me/history/videos' - const myHistoryRemove = myHistoryPath + '/remove' - let viewPath: string - let server: PeerTubeServer - let videoId: number - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - - const { id, uuid } = await server.videos.upload() - viewPath = '/api/v1/videos/' + uuid + '/views' - videoId = id - }) - - describe('When notifying a user is watching a video', function () { - - it('Should fail with a bad token', async function () { - const fields = { currentTime: 5 } - await makePutBodyRequest({ url: server.url, path: viewPath, fields, token: 'bad', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should succeed with the correct parameters', async function () { - const fields = { currentTime: 5 } - - await makePutBodyRequest({ - url: server.url, - path: viewPath, - fields, - token: server.accessToken, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - describe('When listing user videos history', function () { - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, myHistoryPath, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, myHistoryPath, server.accessToken) - }) - - it('Should fail with an unauthenticated user', async function () { - await makeGetRequest({ url: server.url, path: myHistoryPath, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should succeed with the correct params', async function () { - await makeGetRequest({ url: server.url, token: server.accessToken, path: myHistoryPath, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - describe('When removing a specific user video history element', function () { - let path: string - - before(function () { - path = myHistoryPath + '/' + videoId - }) - - it('Should fail with an unauthenticated user', async function () { - await makeDeleteRequest({ url: server.url, path, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with a bad videoId parameter', async function () { - await makeDeleteRequest({ - url: server.url, - token: server.accessToken, - path: myHistoryRemove + '/hi', - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should succeed with the correct parameters', async function () { - await makeDeleteRequest({ - url: server.url, - token: server.accessToken, - path, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - describe('When removing all user videos history', function () { - it('Should fail with an unauthenticated user', async function () { - await makePostBodyRequest({ url: server.url, path: myHistoryPath + '/remove', expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with a bad beforeDate parameter', async function () { - const body = { beforeDate: '15' } - await makePostBodyRequest({ - url: server.url, - token: server.accessToken, - path: myHistoryRemove, - fields: body, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should succeed with a valid beforeDate param', async function () { - const body = { beforeDate: new Date().toISOString() } - await makePostBodyRequest({ - url: server.url, - token: server.accessToken, - path: myHistoryRemove, - fields: body, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - - it('Should succeed without body', async function () { - await makePostBodyRequest({ - url: server.url, - token: server.accessToken, - path: myHistoryRemove, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/videos-overviews.ts b/server/tests/api/check-params/videos-overviews.ts deleted file mode 100644 index ae7de24dd..000000000 --- a/server/tests/api/check-params/videos-overviews.ts +++ /dev/null @@ -1,31 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { cleanupTests, createSingleServer, PeerTubeServer } from '@shared/server-commands' - -describe('Test videos overview API validator', function () { - let server: PeerTubeServer - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - }) - - describe('When getting videos overview', function () { - - it('Should fail with a bad pagination', async function () { - await server.overviews.getVideos({ page: 0, expectedStatus: 400 }) - await server.overviews.getVideos({ page: 100, expectedStatus: 400 }) - }) - - it('Should succeed with a good pagination', async function () { - await server.overviews.getVideos({ page: 1 }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/videos.ts b/server/tests/api/check-params/videos.ts deleted file mode 100644 index f00698fe3..000000000 --- a/server/tests/api/check-params/videos.ts +++ /dev/null @@ -1,881 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { join } from 'path' -import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination, checkUploadVideoParam } from '@server/tests/shared' -import { omit, randomInt, root } from '@shared/core-utils' -import { HttpStatusCode, PeerTubeProblemDocument, VideoCreateResult, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createSingleServer, - makeDeleteRequest, - makeGetRequest, - makePutBodyRequest, - makeUploadRequest, - PeerTubeServer, - setAccessTokensToServers -} from '@shared/server-commands' - -describe('Test videos API validator', function () { - const path = '/api/v1/videos/' - let server: PeerTubeServer - let userAccessToken = '' - let accountName: string - let channelId: number - let channelName: string - let video: VideoCreateResult - let privateVideo: VideoCreateResult - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - - userAccessToken = await server.users.generateUserAndToken('user1') - - { - const body = await server.users.getMyInfo() - channelId = body.videoChannels[0].id - channelName = body.videoChannels[0].name - accountName = body.account.name + '@' + body.account.host - } - - { - privateVideo = await server.videos.quickUpload({ name: 'private video', privacy: VideoPrivacy.PRIVATE }) - } - }) - - describe('When listing videos', function () { - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path) - }) - - it('Should fail with a bad skipVideos query', async function () { - await makeGetRequest({ url: server.url, path, expectedStatus: HttpStatusCode.OK_200, query: { skipCount: 'toto' } }) - }) - - it('Should success with the correct parameters', async function () { - await makeGetRequest({ url: server.url, path, expectedStatus: HttpStatusCode.OK_200, query: { skipCount: false } }) - }) - }) - - describe('When searching a video', function () { - - it('Should fail with nothing', async function () { - await makeGetRequest({ - url: server.url, - path: join(path, 'search'), - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, join(path, 'search', 'test')) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, join(path, 'search', 'test')) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, join(path, 'search', 'test')) - }) - - it('Should success with the correct parameters', async function () { - await makeGetRequest({ url: server.url, path, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - describe('When listing my videos', function () { - const path = '/api/v1/users/me/videos' - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an invalid channel', async function () { - await makeGetRequest({ url: server.url, token: server.accessToken, path, query: { channelId: 'toto' } }) - }) - - it('Should fail with an unknown channel', async function () { - await makeGetRequest({ - url: server.url, - token: server.accessToken, - path, - query: { channelId: 89898 }, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should success with the correct parameters', async function () { - await makeGetRequest({ url: server.url, token: server.accessToken, path, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - describe('When listing account videos', function () { - let path: string - - before(async function () { - path = '/api/v1/accounts/' + accountName + '/videos' - }) - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, server.accessToken) - }) - - it('Should success with the correct parameters', async function () { - await makeGetRequest({ url: server.url, path, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - describe('When listing video channel videos', function () { - let path: string - - before(async function () { - path = '/api/v1/video-channels/' + channelName + '/videos' - }) - - it('Should fail with a bad start pagination', async function () { - await checkBadStartPagination(server.url, path, server.accessToken) - }) - - it('Should fail with a bad count pagination', async function () { - await checkBadCountPagination(server.url, path, server.accessToken) - }) - - it('Should fail with an incorrect sort', async function () { - await checkBadSortPagination(server.url, path, server.accessToken) - }) - - it('Should success with the correct parameters', async function () { - await makeGetRequest({ url: server.url, path, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - describe('When adding a video', function () { - let baseCorrectParams - const baseCorrectAttaches = { - fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.webm') - } - - before(function () { - // Put in before to have channelId - baseCorrectParams = { - name: 'my super name', - category: 5, - licence: 1, - language: 'pt', - nsfw: false, - commentsEnabled: true, - downloadEnabled: true, - waitTranscoding: true, - description: 'my super description', - support: 'my super support text', - tags: [ 'tag1', 'tag2' ], - privacy: VideoPrivacy.PUBLIC, - channelId, - originallyPublishedAt: new Date().toISOString() - } - }) - - function runSuite (mode: 'legacy' | 'resumable') { - - const baseOptions = () => { - return { - server, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400, - mode - } - } - - it('Should fail with nothing', async function () { - const fields = {} - const attaches = {} - await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) - }) - - it('Should fail without name', async function () { - const fields = omit(baseCorrectParams, [ 'name' ]) - const attaches = baseCorrectAttaches - - await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) - }) - - it('Should fail with a long name', async function () { - const fields = { ...baseCorrectParams, name: 'super'.repeat(65) } - const attaches = baseCorrectAttaches - - await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) - }) - - it('Should fail with a bad category', async function () { - const fields = { ...baseCorrectParams, category: 125 } - const attaches = baseCorrectAttaches - - await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) - }) - - it('Should fail with a bad licence', async function () { - const fields = { ...baseCorrectParams, licence: 125 } - const attaches = baseCorrectAttaches - - await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) - }) - - it('Should fail with a bad language', async function () { - const fields = { ...baseCorrectParams, language: 'a'.repeat(15) } - const attaches = baseCorrectAttaches - - await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) - }) - - it('Should fail with a long description', async function () { - const fields = { ...baseCorrectParams, description: 'super'.repeat(2500) } - const attaches = baseCorrectAttaches - - await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) - }) - - it('Should fail with a long support text', async function () { - const fields = { ...baseCorrectParams, support: 'super'.repeat(201) } - const attaches = baseCorrectAttaches - - await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) - }) - - it('Should fail without a channel', async function () { - const fields = omit(baseCorrectParams, [ 'channelId' ]) - const attaches = baseCorrectAttaches - - await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) - }) - - it('Should fail with a bad channel', async function () { - const fields = { ...baseCorrectParams, channelId: 545454 } - const attaches = baseCorrectAttaches - - await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) - }) - - it('Should fail with another user channel', async function () { - const user = { - username: 'fake' + randomInt(0, 1500), - password: 'fake_password' - } - await server.users.create({ username: user.username, password: user.password }) - - const accessTokenUser = await server.login.getAccessToken(user) - const { videoChannels } = await server.users.getMyInfo({ token: accessTokenUser }) - const customChannelId = videoChannels[0].id - - const fields = { ...baseCorrectParams, channelId: customChannelId } - const attaches = baseCorrectAttaches - - await checkUploadVideoParam({ - ...baseOptions(), - token: userAccessToken, - attributes: { ...fields, ...attaches } - }) - }) - - it('Should fail with too many tags', async function () { - const fields = { ...baseCorrectParams, tags: [ 'tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6' ] } - const attaches = baseCorrectAttaches - - await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) - }) - - it('Should fail with a tag length too low', async function () { - const fields = { ...baseCorrectParams, tags: [ 'tag1', 't' ] } - const attaches = baseCorrectAttaches - - await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) - }) - - it('Should fail with a tag length too big', async function () { - const fields = { ...baseCorrectParams, tags: [ 'tag1', 'my_super_tag_too_long_long_long_long_long_long' ] } - const attaches = baseCorrectAttaches - - await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) - }) - - it('Should fail with a bad schedule update (miss updateAt)', async function () { - const fields = { ...baseCorrectParams, scheduleUpdate: { privacy: VideoPrivacy.PUBLIC } } - const attaches = baseCorrectAttaches - - await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) - }) - - it('Should fail with a bad schedule update (wrong updateAt)', async function () { - const fields = { - ...baseCorrectParams, - - scheduleUpdate: { - privacy: VideoPrivacy.PUBLIC, - updateAt: 'toto' - } - } - const attaches = baseCorrectAttaches - - await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) - }) - - it('Should fail with a bad originally published at attribute', async function () { - const fields = { ...baseCorrectParams, originallyPublishedAt: 'toto' } - const attaches = baseCorrectAttaches - - await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) - }) - - it('Should fail without an input file', async function () { - const fields = baseCorrectParams - const attaches = {} - await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) - }) - - it('Should fail with an incorrect input file', async function () { - const fields = baseCorrectParams - let attaches = { fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short_fake.webm') } - - await checkUploadVideoParam({ - ...baseOptions(), - attributes: { ...fields, ...attaches }, - // 200 for the init request, 422 when the file has finished being uploaded - expectedStatus: undefined, - completedExpectedStatus: HttpStatusCode.UNPROCESSABLE_ENTITY_422 - }) - - attaches = { fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mkv') } - await checkUploadVideoParam({ - ...baseOptions(), - attributes: { ...fields, ...attaches }, - expectedStatus: HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415 - }) - }) - - it('Should fail with an incorrect thumbnail file', async function () { - const fields = baseCorrectParams - const attaches = { - thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4'), - fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - } - - await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) - }) - - it('Should fail with a big thumbnail file', async function () { - const fields = baseCorrectParams - const attaches = { - thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'custom-preview-big.png'), - fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - } - - await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) - }) - - it('Should fail with an incorrect preview file', async function () { - const fields = baseCorrectParams - const attaches = { - previewfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4'), - fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - } - - await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) - }) - - it('Should fail with a big preview file', async function () { - const fields = baseCorrectParams - const attaches = { - previewfile: join(root(), 'server', 'tests', 'fixtures', 'custom-preview-big.png'), - fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - } - - await checkUploadVideoParam({ ...baseOptions(), attributes: { ...fields, ...attaches } }) - }) - - it('Should report the appropriate error', async function () { - const fields = { ...baseCorrectParams, language: 'a'.repeat(15) } - const attaches = baseCorrectAttaches - - const attributes = { ...fields, ...attaches } - const body = await checkUploadVideoParam({ ...baseOptions(), attributes }) - - const error = body as unknown as PeerTubeProblemDocument - - if (mode === 'legacy') { - expect(error.docs).to.equal('https://docs.joinpeertube.org/api-rest-reference.html#operation/uploadLegacy') - } else { - expect(error.docs).to.equal('https://docs.joinpeertube.org/api-rest-reference.html#operation/uploadResumableInit') - } - - expect(error.type).to.equal('about:blank') - expect(error.title).to.equal('Bad Request') - - expect(error.detail).to.equal('Incorrect request parameters: language') - expect(error.error).to.equal('Incorrect request parameters: language') - - expect(error.status).to.equal(HttpStatusCode.BAD_REQUEST_400) - expect(error['invalid-params'].language).to.exist - }) - - it('Should succeed with the correct parameters', async function () { - this.timeout(30000) - - const fields = baseCorrectParams - - { - const attaches = baseCorrectAttaches - await checkUploadVideoParam({ - ...baseOptions(), - attributes: { ...fields, ...attaches }, - expectedStatus: HttpStatusCode.OK_200 - }) - } - - { - const attaches = { - ...baseCorrectAttaches, - - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - } - - await checkUploadVideoParam({ - ...baseOptions(), - attributes: { ...fields, ...attaches }, - expectedStatus: HttpStatusCode.OK_200 - }) - } - - { - const attaches = { - ...baseCorrectAttaches, - - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.ogv') - } - - await checkUploadVideoParam({ - ...baseOptions(), - attributes: { ...fields, ...attaches }, - expectedStatus: HttpStatusCode.OK_200 - }) - } - }) - } - - describe('Resumable upload', function () { - runSuite('resumable') - }) - - describe('Legacy upload', function () { - runSuite('legacy') - }) - }) - - describe('When updating a video', function () { - const baseCorrectParams = { - name: 'my super name', - category: 5, - licence: 2, - language: 'pt', - nsfw: false, - commentsEnabled: false, - downloadEnabled: false, - description: 'my super description', - privacy: VideoPrivacy.PUBLIC, - tags: [ 'tag1', 'tag2' ] - } - - before(async function () { - const { data } = await server.videos.list() - video = data[0] - }) - - it('Should fail with nothing', async function () { - const fields = {} - await makePutBodyRequest({ url: server.url, path, token: server.accessToken, fields }) - }) - - it('Should fail without a valid uuid', async function () { - const fields = baseCorrectParams - await makePutBodyRequest({ url: server.url, path: path + 'blabla', token: server.accessToken, fields }) - }) - - it('Should fail with an unknown id', async function () { - const fields = baseCorrectParams - - await makePutBodyRequest({ - url: server.url, - path: path + '4da6fde3-88f7-4d16-b119-108df5630b06', - token: server.accessToken, - fields, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail with a long name', async function () { - const fields = { ...baseCorrectParams, name: 'super'.repeat(65) } - - await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) - }) - - it('Should fail with a bad category', async function () { - const fields = { ...baseCorrectParams, category: 125 } - - await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) - }) - - it('Should fail with a bad licence', async function () { - const fields = { ...baseCorrectParams, licence: 125 } - - await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) - }) - - it('Should fail with a bad language', async function () { - const fields = { ...baseCorrectParams, language: 'a'.repeat(15) } - - await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) - }) - - it('Should fail with a long description', async function () { - const fields = { ...baseCorrectParams, description: 'super'.repeat(2500) } - - await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) - }) - - it('Should fail with a long support text', async function () { - const fields = { ...baseCorrectParams, support: 'super'.repeat(201) } - - await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) - }) - - it('Should fail with a bad channel', async function () { - const fields = { ...baseCorrectParams, channelId: 545454 } - - await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) - }) - - it('Should fail with too many tags', async function () { - const fields = { ...baseCorrectParams, tags: [ 'tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6' ] } - - await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) - }) - - it('Should fail with a tag length too low', async function () { - const fields = { ...baseCorrectParams, tags: [ 'tag1', 't' ] } - - await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) - }) - - it('Should fail with a tag length too big', async function () { - const fields = { ...baseCorrectParams, tags: [ 'tag1', 'my_super_tag_too_long_long_long_long_long_long' ] } - - await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) - }) - - it('Should fail with a bad schedule update (miss updateAt)', async function () { - const fields = { ...baseCorrectParams, scheduleUpdate: { privacy: VideoPrivacy.PUBLIC } } - - await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) - }) - - it('Should fail with a bad schedule update (wrong updateAt)', async function () { - const fields = { ...baseCorrectParams, scheduleUpdate: { updateAt: 'toto', privacy: VideoPrivacy.PUBLIC } } - - await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) - }) - - it('Should fail with a bad originally published at param', async function () { - const fields = { ...baseCorrectParams, originallyPublishedAt: 'toto' } - - await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) - }) - - it('Should fail with an incorrect thumbnail file', async function () { - const fields = baseCorrectParams - const attaches = { - thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - } - - await makeUploadRequest({ - url: server.url, - method: 'PUT', - path: path + video.shortUUID, - token: server.accessToken, - fields, - attaches - }) - }) - - it('Should fail with a big thumbnail file', async function () { - const fields = baseCorrectParams - const attaches = { - thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'custom-preview-big.png') - } - - await makeUploadRequest({ - url: server.url, - method: 'PUT', - path: path + video.shortUUID, - token: server.accessToken, - fields, - attaches - }) - }) - - it('Should fail with an incorrect preview file', async function () { - const fields = baseCorrectParams - const attaches = { - previewfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - } - - await makeUploadRequest({ - url: server.url, - method: 'PUT', - path: path + video.shortUUID, - token: server.accessToken, - fields, - attaches - }) - }) - - it('Should fail with a big preview file', async function () { - const fields = baseCorrectParams - const attaches = { - previewfile: join(root(), 'server', 'tests', 'fixtures', 'custom-preview-big.png') - } - - await makeUploadRequest({ - url: server.url, - method: 'PUT', - path: path + video.shortUUID, - token: server.accessToken, - fields, - attaches - }) - }) - - it('Should fail with a video of another user without the appropriate right', async function () { - const fields = baseCorrectParams - - await makePutBodyRequest({ - url: server.url, - path: path + video.shortUUID, - token: userAccessToken, - fields, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with a video of another server') - - it('Shoud report the appropriate error', async function () { - const fields = { ...baseCorrectParams, licence: 125 } - - const res = await makePutBodyRequest({ url: server.url, path: path + video.shortUUID, token: server.accessToken, fields }) - const error = res.body as PeerTubeProblemDocument - - expect(error.docs).to.equal('https://docs.joinpeertube.org/api-rest-reference.html#operation/putVideo') - - expect(error.type).to.equal('about:blank') - expect(error.title).to.equal('Bad Request') - - expect(error.detail).to.equal('Incorrect request parameters: licence') - expect(error.error).to.equal('Incorrect request parameters: licence') - - expect(error.status).to.equal(HttpStatusCode.BAD_REQUEST_400) - expect(error['invalid-params'].licence).to.exist - }) - - it('Should succeed with the correct parameters', async function () { - const fields = baseCorrectParams - - await makePutBodyRequest({ - url: server.url, - path: path + video.shortUUID, - token: server.accessToken, - fields, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - describe('When getting a video', function () { - it('Should return the list of the videos with nothing', async function () { - const res = await makeGetRequest({ - url: server.url, - path, - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(res.body.data).to.be.an('array') - expect(res.body.data.length).to.equal(6) - }) - - it('Should fail without a correct uuid', async function () { - await server.videos.get({ id: 'coucou', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should return 404 with an incorrect video', async function () { - await server.videos.get({ id: '4da6fde3-88f7-4d16-b119-108df5630b06', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Shoud report the appropriate error', async function () { - const body = await server.videos.get({ id: 'hi', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - const error = body as unknown as PeerTubeProblemDocument - - expect(error.docs).to.equal('https://docs.joinpeertube.org/api-rest-reference.html#operation/getVideo') - - expect(error.type).to.equal('about:blank') - expect(error.title).to.equal('Bad Request') - - expect(error.detail).to.equal('Incorrect request parameters: id') - expect(error.error).to.equal('Incorrect request parameters: id') - - expect(error.status).to.equal(HttpStatusCode.BAD_REQUEST_400) - expect(error['invalid-params'].id).to.exist - }) - - it('Should succeed with the correct parameters', async function () { - await server.videos.get({ id: video.shortUUID }) - }) - }) - - describe('When rating a video', function () { - let videoId: number - - before(async function () { - const { data } = await server.videos.list() - videoId = data[0].id - }) - - it('Should fail without a valid uuid', async function () { - const fields = { - rating: 'like' - } - await makePutBodyRequest({ url: server.url, path: path + 'blabla/rate', token: server.accessToken, fields }) - }) - - it('Should fail with an unknown id', async function () { - const fields = { - rating: 'like' - } - await makePutBodyRequest({ - url: server.url, - path: path + '4da6fde3-88f7-4d16-b119-108df5630b06/rate', - token: server.accessToken, - fields, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should fail with a wrong rating', async function () { - const fields = { - rating: 'likes' - } - await makePutBodyRequest({ url: server.url, path: path + videoId + '/rate', token: server.accessToken, fields }) - }) - - it('Should fail with a private video of another user', async function () { - const fields = { - rating: 'like' - } - await makePutBodyRequest({ - url: server.url, - path: path + privateVideo.uuid + '/rate', - token: userAccessToken, - fields, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should succeed with the correct parameters', async function () { - const fields = { - rating: 'like' - } - await makePutBodyRequest({ - url: server.url, - path: path + videoId + '/rate', - token: server.accessToken, - fields, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - }) - }) - - describe('When removing a video', function () { - it('Should have 404 with nothing', async function () { - await makeDeleteRequest({ - url: server.url, - path, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail without a correct uuid', async function () { - await server.videos.remove({ id: 'hello', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with a video which does not exist', async function () { - await server.videos.remove({ id: '4da6fde3-88f7-4d16-b119-108df5630b06', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should fail with a video of another user without the appropriate right', async function () { - await server.videos.remove({ token: userAccessToken, id: video.uuid, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should fail with a video of another server') - - it('Shoud report the appropriate error', async function () { - const body = await server.videos.remove({ id: 'hello', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - const error = body as PeerTubeProblemDocument - - expect(error.docs).to.equal('https://docs.joinpeertube.org/api-rest-reference.html#operation/delVideo') - - expect(error.type).to.equal('about:blank') - expect(error.title).to.equal('Bad Request') - - expect(error.detail).to.equal('Incorrect request parameters: id') - expect(error.error).to.equal('Incorrect request parameters: id') - - expect(error.status).to.equal(HttpStatusCode.BAD_REQUEST_400) - expect(error['invalid-params'].id).to.exist - }) - - it('Should succeed with the correct parameters', async function () { - await server.videos.remove({ id: video.uuid }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/check-params/views.ts b/server/tests/api/check-params/views.ts deleted file mode 100644 index 11416ccb8..000000000 --- a/server/tests/api/check-params/views.ts +++ /dev/null @@ -1,227 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { HttpStatusCode, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel -} from '@shared/server-commands' - -describe('Test videos views', function () { - let servers: PeerTubeServer[] - let liveVideoId: string - let videoId: string - let remoteVideoId: string - let userAccessToken: string - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(2) - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - - await servers[0].config.enableLive({ allowReplay: false, transcoding: false }); - - ({ uuid: videoId } = await servers[0].videos.quickUpload({ name: 'video' })); - ({ uuid: remoteVideoId } = await servers[1].videos.quickUpload({ name: 'video' })); - ({ uuid: liveVideoId } = await servers[0].live.create({ - fields: { - name: 'live', - privacy: VideoPrivacy.PUBLIC, - channelId: servers[0].store.channel.id - } - })) - - userAccessToken = await servers[0].users.generateUserAndToken('user') - - await doubleFollow(servers[0], servers[1]) - }) - - describe('When viewing a video', async function () { - - it('Should fail without current time', async function () { - await servers[0].views.view({ id: videoId, currentTime: undefined, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with an invalid current time', async function () { - await servers[0].views.view({ id: videoId, currentTime: -1, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await servers[0].views.view({ id: videoId, currentTime: 10, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should succeed with correct parameters', async function () { - await servers[0].views.view({ id: videoId, currentTime: 1 }) - }) - }) - - describe('When getting overall stats', function () { - - it('Should fail with a remote video', async function () { - await servers[0].videoStats.getOverallStats({ videoId: remoteVideoId, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should fail without token', async function () { - await servers[0].videoStats.getOverallStats({ videoId, token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should fail with another token', async function () { - await servers[0].videoStats.getOverallStats({ - videoId, - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with an invalid start date', async function () { - await servers[0].videoStats.getOverallStats({ - videoId, - startDate: 'fake' as any, - endDate: new Date().toISOString(), - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with an invalid end date', async function () { - await servers[0].videoStats.getOverallStats({ - videoId, - startDate: new Date().toISOString(), - endDate: 'fake' as any, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should succeed with the correct parameters', async function () { - await servers[0].videoStats.getOverallStats({ - videoId, - startDate: new Date().toISOString(), - endDate: new Date().toISOString() - }) - }) - }) - - describe('When getting timeserie stats', function () { - - it('Should fail with a remote video', async function () { - await servers[0].videoStats.getTimeserieStats({ - videoId: remoteVideoId, - metric: 'viewers', - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail without token', async function () { - await servers[0].videoStats.getTimeserieStats({ - videoId, - token: null, - metric: 'viewers', - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with another token', async function () { - await servers[0].videoStats.getTimeserieStats({ - videoId, - token: userAccessToken, - metric: 'viewers', - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail with an invalid metric', async function () { - await servers[0].videoStats.getTimeserieStats({ videoId, metric: 'hello' as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should fail with an invalid start date', async function () { - await servers[0].videoStats.getTimeserieStats({ - videoId, - metric: 'viewers', - startDate: 'fake' as any, - endDate: new Date(), - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with an invalid end date', async function () { - await servers[0].videoStats.getTimeserieStats({ - videoId, - metric: 'viewers', - startDate: new Date(), - endDate: 'fake' as any, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail if start date is specified but not end date', async function () { - await servers[0].videoStats.getTimeserieStats({ - videoId, - metric: 'viewers', - startDate: new Date(), - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail if end date is specified but not start date', async function () { - await servers[0].videoStats.getTimeserieStats({ - videoId, - metric: 'viewers', - endDate: new Date(), - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should fail with a too big interval', async function () { - await servers[0].videoStats.getTimeserieStats({ - videoId, - metric: 'viewers', - startDate: new Date('2000-04-07T08:31:57.126Z'), - endDate: new Date(), - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should succeed with the correct parameters', async function () { - await servers[0].videoStats.getTimeserieStats({ videoId, metric: 'viewers' }) - }) - }) - - describe('When getting retention stats', function () { - - it('Should fail with a remote video', async function () { - await servers[0].videoStats.getRetentionStats({ - videoId: remoteVideoId, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail without token', async function () { - await servers[0].videoStats.getRetentionStats({ - videoId, - token: null, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - }) - - it('Should fail with another token', async function () { - await servers[0].videoStats.getRetentionStats({ - videoId, - token: userAccessToken, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should fail on live video', async function () { - await servers[0].videoStats.getRetentionStats({ videoId: liveVideoId, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should succeed with the correct parameters', async function () { - await servers[0].videoStats.getRetentionStats({ videoId }) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/index.ts b/server/tests/api/index.ts deleted file mode 100644 index ef0c83294..000000000 --- a/server/tests/api/index.ts +++ /dev/null @@ -1,13 +0,0 @@ -// Order of the tests we want to execute -import './activitypub' -import './check-params' -import './moderation' -import './object-storage' -import './notifications' -import './redundancy' -import './runners' -import './search' -import './server' -import './transcoding' -import './users' -import './videos' diff --git a/server/tests/api/live/index.ts b/server/tests/api/live/index.ts deleted file mode 100644 index c88943f65..000000000 --- a/server/tests/api/live/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -import './live-constraints' -import './live-fast-restream' -import './live-socket-messages' -import './live-permanent' -import './live-rtmps' -import './live-save-replay' -import './live' diff --git a/server/tests/api/live/live-constraints.ts b/server/tests/api/live/live-constraints.ts deleted file mode 100644 index 697d808d5..000000000 --- a/server/tests/api/live/live-constraints.ts +++ /dev/null @@ -1,237 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { wait } from '@shared/core-utils' -import { LiveVideoError, UserVideoQuota, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - ConfigCommand, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - stopFfmpeg, - waitJobs, - waitUntilLiveReplacedByReplayOnAllServers, - waitUntilLiveWaitingOnAllServers -} from '@shared/server-commands' -import { checkLiveCleanup } from '../../shared' - -describe('Test live constraints', function () { - let servers: PeerTubeServer[] = [] - let userId: number - let userAccessToken: string - let userChannelId: number - - async function createLiveWrapper (options: { replay: boolean, permanent: boolean }) { - const { replay, permanent } = options - - const liveAttributes = { - name: 'user live', - channelId: userChannelId, - privacy: VideoPrivacy.PUBLIC, - saveReplay: replay, - replaySettings: options.replay ? { privacy: VideoPrivacy.PUBLIC } : undefined, - permanentLive: permanent - } - - const { uuid } = await servers[0].live.create({ token: userAccessToken, fields: liveAttributes }) - return uuid - } - - async function checkSaveReplay (videoId: string, resolutions = [ 720 ]) { - for (const server of servers) { - const video = await server.videos.get({ id: videoId }) - expect(video.isLive).to.be.false - expect(video.duration).to.be.greaterThan(0) - } - - await checkLiveCleanup({ server: servers[0], permanent: false, videoUUID: videoId, savedResolutions: resolutions }) - } - - function updateQuota (options: { total: number, daily: number }) { - return servers[0].users.update({ - userId, - videoQuota: options.total, - videoQuotaDaily: options.daily - }) - } - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(2) - - // Get the access tokens - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - - await servers[0].config.updateCustomSubConfig({ - newConfig: { - live: { - enabled: true, - allowReplay: true, - transcoding: { - enabled: false - } - } - } - }) - - { - const res = await servers[0].users.generate('user1') - userId = res.userId - userChannelId = res.userChannelId - userAccessToken = res.token - - await updateQuota({ total: 1, daily: -1 }) - } - - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - }) - - it('Should not have size limit if save replay is disabled', async function () { - this.timeout(60000) - - const userVideoLiveoId = await createLiveWrapper({ replay: false, permanent: false }) - await servers[0].live.runAndTestStreamError({ token: userAccessToken, videoId: userVideoLiveoId, shouldHaveError: false }) - }) - - it('Should have size limit depending on user global quota if save replay is enabled on non permanent live', async function () { - this.timeout(60000) - - // Wait for user quota memoize cache invalidation - await wait(5000) - - const userVideoLiveoId = await createLiveWrapper({ replay: true, permanent: false }) - await servers[0].live.runAndTestStreamError({ token: userAccessToken, videoId: userVideoLiveoId, shouldHaveError: true }) - - await waitUntilLiveReplacedByReplayOnAllServers(servers, userVideoLiveoId) - await waitJobs(servers) - - await checkSaveReplay(userVideoLiveoId) - - const session = await servers[0].live.getReplaySession({ videoId: userVideoLiveoId }) - expect(session.error).to.equal(LiveVideoError.QUOTA_EXCEEDED) - }) - - it('Should have size limit depending on user global quota if save replay is enabled on a permanent live', async function () { - this.timeout(60000) - - // Wait for user quota memoize cache invalidation - await wait(5000) - - const userVideoLiveoId = await createLiveWrapper({ replay: true, permanent: true }) - await servers[0].live.runAndTestStreamError({ token: userAccessToken, videoId: userVideoLiveoId, shouldHaveError: true }) - - await waitJobs(servers) - await waitUntilLiveWaitingOnAllServers(servers, userVideoLiveoId) - - const session = await servers[0].live.findLatestSession({ videoId: userVideoLiveoId }) - expect(session.error).to.equal(LiveVideoError.QUOTA_EXCEEDED) - }) - - it('Should have size limit depending on user daily quota if save replay is enabled', async function () { - this.timeout(60000) - - // Wait for user quota memoize cache invalidation - await wait(5000) - - await updateQuota({ total: -1, daily: 1 }) - - const userVideoLiveoId = await createLiveWrapper({ replay: true, permanent: false }) - await servers[0].live.runAndTestStreamError({ token: userAccessToken, videoId: userVideoLiveoId, shouldHaveError: true }) - - await waitUntilLiveReplacedByReplayOnAllServers(servers, userVideoLiveoId) - await waitJobs(servers) - - await checkSaveReplay(userVideoLiveoId) - - const session = await servers[0].live.getReplaySession({ videoId: userVideoLiveoId }) - expect(session.error).to.equal(LiveVideoError.QUOTA_EXCEEDED) - }) - - it('Should succeed without quota limit', async function () { - this.timeout(60000) - - // Wait for user quota memoize cache invalidation - await wait(5000) - - await updateQuota({ total: 10 * 1000 * 1000, daily: -1 }) - - const userVideoLiveoId = await createLiveWrapper({ replay: true, permanent: false }) - await servers[0].live.runAndTestStreamError({ token: userAccessToken, videoId: userVideoLiveoId, shouldHaveError: false }) - }) - - it('Should have the same quota in admin and as a user', async function () { - this.timeout(120000) - - const userVideoLiveoId = await createLiveWrapper({ replay: true, permanent: false }) - const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ token: userAccessToken, videoId: userVideoLiveoId }) - - await servers[0].live.waitUntilPublished({ videoId: userVideoLiveoId }) - // Wait previous live cleanups - await wait(3000) - - const baseQuota = await servers[0].users.getMyQuotaUsed({ token: userAccessToken }) - - let quotaUser: UserVideoQuota - - do { - await wait(500) - - quotaUser = await servers[0].users.getMyQuotaUsed({ token: userAccessToken }) - } while (quotaUser.videoQuotaUsed <= baseQuota.videoQuotaUsed) - - const { data } = await servers[0].users.list() - const quotaAdmin = data.find(u => u.username === 'user1') - - expect(quotaUser.videoQuotaUsed).to.be.above(baseQuota.videoQuotaUsed) - expect(quotaUser.videoQuotaUsedDaily).to.be.above(baseQuota.videoQuotaUsedDaily) - - expect(quotaAdmin.videoQuotaUsed).to.be.above(baseQuota.videoQuotaUsed) - expect(quotaAdmin.videoQuotaUsedDaily).to.be.above(baseQuota.videoQuotaUsedDaily) - - expect(quotaUser.videoQuotaUsed).to.be.above(10) - expect(quotaUser.videoQuotaUsedDaily).to.be.above(10) - expect(quotaAdmin.videoQuotaUsed).to.be.above(10) - expect(quotaAdmin.videoQuotaUsedDaily).to.be.above(10) - - await stopFfmpeg(ffmpegCommand) - }) - - it('Should have max duration limit', async function () { - this.timeout(240000) - - await servers[0].config.updateCustomSubConfig({ - newConfig: { - live: { - enabled: true, - allowReplay: true, - maxDuration: 15, - transcoding: { - enabled: true, - resolutions: ConfigCommand.getCustomConfigResolutions(true) - } - } - } - }) - - const userVideoLiveoId = await createLiveWrapper({ replay: true, permanent: false }) - await servers[0].live.runAndTestStreamError({ token: userAccessToken, videoId: userVideoLiveoId, shouldHaveError: true }) - - await waitUntilLiveReplacedByReplayOnAllServers(servers, userVideoLiveoId) - await waitJobs(servers) - - await checkSaveReplay(userVideoLiveoId, [ 720, 480, 360, 240, 144 ]) - - const session = await servers[0].live.getReplaySession({ videoId: userVideoLiveoId }) - expect(session.error).to.equal(LiveVideoError.DURATION_EXCEEDED) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/live/live-fast-restream.ts b/server/tests/api/live/live-fast-restream.ts deleted file mode 100644 index 1b7fddd8b..000000000 --- a/server/tests/api/live/live-fast-restream.ts +++ /dev/null @@ -1,153 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { wait } from '@shared/core-utils' -import { LiveVideoCreate, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createSingleServer, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - stopFfmpeg, - waitJobs -} from '@shared/server-commands' - -describe('Fast restream in live', function () { - let server: PeerTubeServer - - async function createLiveWrapper (options: { permanent: boolean, replay: boolean }) { - const attributes: LiveVideoCreate = { - channelId: server.store.channel.id, - privacy: VideoPrivacy.PUBLIC, - name: 'my super live', - saveReplay: options.replay, - replaySettings: options.replay ? { privacy: VideoPrivacy.PUBLIC } : undefined, - permanentLive: options.permanent - } - - const { uuid } = await server.live.create({ fields: attributes }) - return uuid - } - - async function fastRestreamWrapper ({ replay }: { replay: boolean }) { - const liveVideoUUID = await createLiveWrapper({ permanent: true, replay }) - await waitJobs([ server ]) - - const rtmpOptions = { - videoId: liveVideoUUID, - copyCodecs: true, - fixtureName: 'video_short.mp4' - } - - // Streaming session #1 - let ffmpegCommand = await server.live.sendRTMPStreamInVideo(rtmpOptions) - await server.live.waitUntilPublished({ videoId: liveVideoUUID }) - - const video = await server.videos.get({ id: liveVideoUUID }) - const session1PlaylistId = video.streamingPlaylists[0].id - - await stopFfmpeg(ffmpegCommand) - await server.live.waitUntilWaiting({ videoId: liveVideoUUID }) - - // Streaming session #2 - ffmpegCommand = await server.live.sendRTMPStreamInVideo(rtmpOptions) - - let hasNewPlaylist = false - do { - const video = await server.videos.get({ id: liveVideoUUID }) - hasNewPlaylist = video.streamingPlaylists.length === 1 && video.streamingPlaylists[0].id !== session1PlaylistId - - await wait(100) - } while (!hasNewPlaylist) - - await server.live.waitUntilSegmentGeneration({ - server, - videoUUID: liveVideoUUID, - segment: 1, - playlistNumber: 0 - }) - - return { ffmpegCommand, liveVideoUUID } - } - - async function ensureLastLiveWorks (liveId: string) { - // Equivalent to PEERTUBE_TEST_CONSTANTS_VIDEO_LIVE_CLEANUP_DELAY - for (let i = 0; i < 100; i++) { - const video = await server.videos.get({ id: liveId }) - expect(video.streamingPlaylists).to.have.lengthOf(1) - - try { - await server.live.getSegmentFile({ videoUUID: liveId, segment: 0, playlistNumber: 0 }) - await server.streamingPlaylists.get({ url: video.streamingPlaylists[0].playlistUrl }) - await server.streamingPlaylists.getSegmentSha256({ url: video.streamingPlaylists[0].segmentsSha256Url }) - } catch (err) { - // FIXME: try to debug error in CI "Unexpected end of JSON input" - console.error(err) - throw err - } - - await wait(100) - } - } - - async function runTest (replay: boolean) { - const { ffmpegCommand, liveVideoUUID } = await fastRestreamWrapper({ replay }) - - // TODO: remove, we try to debug a test timeout failure here - console.log('Ensuring last live works') - - await ensureLastLiveWorks(liveVideoUUID) - - await stopFfmpeg(ffmpegCommand) - await server.live.waitUntilWaiting({ videoId: liveVideoUUID }) - - // Wait for replays - await waitJobs([ server ]) - - const { total, data: sessions } = await server.live.listSessions({ videoId: liveVideoUUID }) - - expect(total).to.equal(2) - expect(sessions).to.have.lengthOf(2) - - for (const session of sessions) { - expect(session.error).to.be.null - - if (replay) { - expect(session.replayVideo).to.exist - - await server.videos.get({ id: session.replayVideo.uuid }) - } else { - expect(session.replayVideo).to.not.exist - } - } - } - - before(async function () { - this.timeout(120000) - - const env = { PEERTUBE_TEST_CONSTANTS_VIDEO_LIVE_CLEANUP_DELAY: '10000' } - server = await createSingleServer(1, {}, { env }) - - // Get the access tokens - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - await server.config.enableMinimumTranscoding({ webVideo: false, hls: true }) - await server.config.enableLive({ allowReplay: true, transcoding: true, resolutions: 'min' }) - }) - - it('Should correctly fast restream in a permanent live with and without save replay', async function () { - this.timeout(480000) - - // A test can take a long time, so prefer to run them in parallel - await Promise.all([ - runTest(true), - runTest(false) - ]) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/live/live-permanent.ts b/server/tests/api/live/live-permanent.ts deleted file mode 100644 index 4203b1bfc..000000000 --- a/server/tests/api/live/live-permanent.ts +++ /dev/null @@ -1,204 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { checkLiveCleanup } from '@server/tests/shared' -import { wait } from '@shared/core-utils' -import { LiveVideoCreate, VideoPrivacy, VideoState } from '@shared/models' -import { - cleanupTests, - ConfigCommand, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - stopFfmpeg, - waitJobs -} from '@shared/server-commands' - -describe('Permanent live', function () { - let servers: PeerTubeServer[] = [] - let videoUUID: string - - async function createLiveWrapper (permanentLive: boolean) { - const attributes: LiveVideoCreate = { - channelId: servers[0].store.channel.id, - privacy: VideoPrivacy.PUBLIC, - name: 'my super live', - saveReplay: false, - permanentLive - } - - const { uuid } = await servers[0].live.create({ fields: attributes }) - return uuid - } - - async function checkVideoState (videoId: string, state: VideoState) { - for (const server of servers) { - const video = await server.videos.get({ id: videoId }) - expect(video.state.id).to.equal(state) - } - } - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(2) - - // Get the access tokens - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - - await servers[0].config.updateCustomSubConfig({ - newConfig: { - live: { - enabled: true, - allowReplay: true, - maxDuration: -1, - transcoding: { - enabled: true, - resolutions: ConfigCommand.getCustomConfigResolutions(true) - } - } - } - }) - }) - - it('Should create a non permanent live and update it to be a permanent live', async function () { - this.timeout(20000) - - const videoUUID = await createLiveWrapper(false) - - { - const live = await servers[0].live.get({ videoId: videoUUID }) - expect(live.permanentLive).to.be.false - } - - await servers[0].live.update({ videoId: videoUUID, fields: { permanentLive: true } }) - - { - const live = await servers[0].live.get({ videoId: videoUUID }) - expect(live.permanentLive).to.be.true - } - }) - - it('Should create a permanent live', async function () { - this.timeout(20000) - - videoUUID = await createLiveWrapper(true) - - const live = await servers[0].live.get({ videoId: videoUUID }) - expect(live.permanentLive).to.be.true - - await waitJobs(servers) - }) - - it('Should stream into this permanent live', async function () { - this.timeout(240_000) - - const beforePublication = new Date() - const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID }) - - for (const server of servers) { - await server.live.waitUntilPublished({ videoId: videoUUID }) - } - - await checkVideoState(videoUUID, VideoState.PUBLISHED) - - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - expect(new Date(video.publishedAt)).greaterThan(beforePublication) - } - - await stopFfmpeg(ffmpegCommand) - await servers[0].live.waitUntilWaiting({ videoId: videoUUID }) - - await waitJobs(servers) - }) - - it('Should have cleaned up this live', async function () { - this.timeout(40000) - - await wait(5000) - await waitJobs(servers) - - for (const server of servers) { - const videoDetails = await server.videos.get({ id: videoUUID }) - - expect(videoDetails.streamingPlaylists).to.have.lengthOf(0) - } - - await checkLiveCleanup({ server: servers[0], permanent: true, videoUUID }) - }) - - it('Should have set this live to waiting for live state', async function () { - this.timeout(20000) - - await checkVideoState(videoUUID, VideoState.WAITING_FOR_LIVE) - }) - - it('Should be able to stream again in the permanent live', async function () { - this.timeout(60000) - - await servers[0].config.updateCustomSubConfig({ - newConfig: { - live: { - enabled: true, - allowReplay: true, - maxDuration: -1, - transcoding: { - enabled: true, - resolutions: ConfigCommand.getCustomConfigResolutions(false) - } - } - } - }) - - const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID }) - - for (const server of servers) { - await server.live.waitUntilPublished({ videoId: videoUUID }) - } - - await checkVideoState(videoUUID, VideoState.PUBLISHED) - - const count = await servers[0].live.countPlaylists({ videoUUID }) - // master playlist and 720p playlist - expect(count).to.equal(2) - - await stopFfmpeg(ffmpegCommand) - }) - - it('Should have appropriate sessions', async function () { - this.timeout(60000) - - await servers[0].live.waitUntilWaiting({ videoId: videoUUID }) - - const { data, total } = await servers[0].live.listSessions({ videoId: videoUUID }) - expect(total).to.equal(2) - expect(data).to.have.lengthOf(2) - - for (const session of data) { - expect(session.startDate).to.exist - expect(session.endDate).to.exist - - expect(session.error).to.not.exist - } - }) - - it('Should remove the live and have cleaned up the directory', async function () { - this.timeout(60000) - - await servers[0].videos.remove({ id: videoUUID }) - await waitJobs(servers) - - await checkLiveCleanup({ server: servers[0], permanent: true, videoUUID }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/live/live-rtmps.ts b/server/tests/api/live/live-rtmps.ts deleted file mode 100644 index dcaee90cf..000000000 --- a/server/tests/api/live/live-rtmps.ts +++ /dev/null @@ -1,143 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { buildAbsoluteFixturePath } from '@shared/core-utils' -import { VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createSingleServer, - PeerTubeServer, - sendRTMPStream, - setAccessTokensToServers, - setDefaultVideoChannel, - stopFfmpeg, - testFfmpegStreamError, - waitUntilLivePublishedOnAllServers -} from '@shared/server-commands' - -describe('Test live RTMPS', function () { - let server: PeerTubeServer - let rtmpUrl: string - let rtmpsUrl: string - - async function createLiveWrapper () { - const liveAttributes = { - name: 'live', - channelId: server.store.channel.id, - privacy: VideoPrivacy.PUBLIC, - saveReplay: false - } - - const { uuid } = await server.live.create({ fields: liveAttributes }) - - const live = await server.live.get({ videoId: uuid }) - const video = await server.videos.get({ id: uuid }) - - return Object.assign(video, live) - } - - before(async function () { - this.timeout(120000) - - server = await createSingleServer(1) - - // Get the access tokens - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - await server.config.updateCustomSubConfig({ - newConfig: { - live: { - enabled: true, - allowReplay: true, - transcoding: { - enabled: false - } - } - } - }) - - rtmpUrl = 'rtmp://' + server.hostname + ':' + server.rtmpPort + '/live' - rtmpsUrl = 'rtmps://' + server.hostname + ':' + server.rtmpsPort + '/live' - }) - - it('Should enable RTMPS endpoint only', async function () { - this.timeout(240000) - - await server.kill() - await server.run({ - live: { - rtmp: { - enabled: false - }, - rtmps: { - enabled: true, - port: server.rtmpsPort, - key_file: buildAbsoluteFixturePath('rtmps.key'), - cert_file: buildAbsoluteFixturePath('rtmps.cert') - } - } - }) - - { - const liveVideo = await createLiveWrapper() - - expect(liveVideo.rtmpUrl).to.not.exist - expect(liveVideo.rtmpsUrl).to.equal(rtmpsUrl) - - const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl, streamKey: liveVideo.streamKey }) - await testFfmpegStreamError(command, true) - } - - { - const liveVideo = await createLiveWrapper() - - const command = sendRTMPStream({ rtmpBaseUrl: rtmpsUrl, streamKey: liveVideo.streamKey }) - await waitUntilLivePublishedOnAllServers([ server ], liveVideo.uuid) - await stopFfmpeg(command) - } - }) - - it('Should enable both RTMP and RTMPS', async function () { - this.timeout(240000) - - await server.kill() - await server.run({ - live: { - rtmp: { - enabled: true, - port: server.rtmpPort - }, - rtmps: { - enabled: true, - port: server.rtmpsPort, - key_file: buildAbsoluteFixturePath('rtmps.key'), - cert_file: buildAbsoluteFixturePath('rtmps.cert') - } - } - }) - - { - const liveVideo = await createLiveWrapper() - - expect(liveVideo.rtmpUrl).to.equal(rtmpUrl) - expect(liveVideo.rtmpsUrl).to.equal(rtmpsUrl) - - const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl, streamKey: liveVideo.streamKey }) - await waitUntilLivePublishedOnAllServers([ server ], liveVideo.uuid) - await stopFfmpeg(command) - } - - { - const liveVideo = await createLiveWrapper() - - const command = sendRTMPStream({ rtmpBaseUrl: rtmpsUrl, streamKey: liveVideo.streamKey }) - await waitUntilLivePublishedOnAllServers([ server ], liveVideo.uuid) - await stopFfmpeg(command) - } - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/live/live-save-replay.ts b/server/tests/api/live/live-save-replay.ts deleted file mode 100644 index d554cf208..000000000 --- a/server/tests/api/live/live-save-replay.ts +++ /dev/null @@ -1,570 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { FfmpegCommand } from 'fluent-ffmpeg' -import { checkLiveCleanup } from '@server/tests/shared' -import { wait } from '@shared/core-utils' -import { HttpStatusCode, LiveVideoCreate, LiveVideoError, VideoPrivacy, VideoState } from '@shared/models' -import { - cleanupTests, - ConfigCommand, - createMultipleServers, - doubleFollow, - findExternalSavedVideo, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - stopFfmpeg, - testFfmpegStreamError, - waitJobs, - waitUntilLivePublishedOnAllServers, - waitUntilLiveReplacedByReplayOnAllServers, - waitUntilLiveWaitingOnAllServers -} from '@shared/server-commands' - -describe('Save replay setting', function () { - let servers: PeerTubeServer[] = [] - let liveVideoUUID: string - let ffmpegCommand: FfmpegCommand - - async function createLiveWrapper (options: { permanent: boolean, replay: boolean, replaySettings?: { privacy: VideoPrivacy } }) { - if (liveVideoUUID) { - try { - await servers[0].videos.remove({ id: liveVideoUUID }) - await waitJobs(servers) - } catch {} - } - - const attributes: LiveVideoCreate = { - channelId: servers[0].store.channel.id, - privacy: VideoPrivacy.PUBLIC, - name: 'live'.repeat(30), - saveReplay: options.replay, - replaySettings: options.replaySettings, - permanentLive: options.permanent - } - - const { uuid } = await servers[0].live.create({ fields: attributes }) - return uuid - } - - async function publishLive (options: { permanent: boolean, replay: boolean, replaySettings?: { privacy: VideoPrivacy } }) { - liveVideoUUID = await createLiveWrapper(options) - - const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) - await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) - - const liveDetails = await servers[0].videos.get({ id: liveVideoUUID }) - - await waitJobs(servers) - await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) - - return { ffmpegCommand, liveDetails } - } - - async function publishLiveAndDelete (options: { permanent: boolean, replay: boolean, replaySettings?: { privacy: VideoPrivacy } }) { - const { ffmpegCommand, liveDetails } = await publishLive(options) - - await Promise.all([ - servers[0].videos.remove({ id: liveVideoUUID }), - testFfmpegStreamError(ffmpegCommand, true) - ]) - - await waitJobs(servers) - await wait(5000) - await waitJobs(servers) - - return { liveDetails } - } - - async function publishLiveAndBlacklist (options: { permanent: boolean, replay: boolean, replaySettings?: { privacy: VideoPrivacy } }) { - const { ffmpegCommand, liveDetails } = await publishLive(options) - - await Promise.all([ - servers[0].blacklist.add({ videoId: liveVideoUUID, reason: 'bad live', unfederate: true }), - testFfmpegStreamError(ffmpegCommand, true) - ]) - - await waitJobs(servers) - await wait(5000) - await waitJobs(servers) - - return { liveDetails } - } - - async function checkVideosExist (videoId: string, existsInList: boolean, expectedStatus?: number) { - for (const server of servers) { - const length = existsInList ? 1 : 0 - - const { data, total } = await server.videos.list() - expect(data).to.have.lengthOf(length) - expect(total).to.equal(length) - - if (expectedStatus) { - await server.videos.get({ id: videoId, expectedStatus }) - } - } - } - - async function checkVideoState (videoId: string, state: VideoState) { - for (const server of servers) { - const video = await server.videos.get({ id: videoId }) - expect(video.state.id).to.equal(state) - } - } - - async function checkVideoPrivacy (videoId: string, privacy: VideoPrivacy) { - for (const server of servers) { - const video = await server.videos.get({ id: videoId }) - expect(video.privacy.id).to.equal(privacy) - } - } - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(2) - - // Get the access tokens - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - - await servers[0].config.updateCustomSubConfig({ - newConfig: { - live: { - enabled: true, - allowReplay: true, - maxDuration: -1, - transcoding: { - enabled: false, - resolutions: ConfigCommand.getCustomConfigResolutions(true) - } - } - } - }) - }) - - describe('With save replay disabled', function () { - let sessionStartDateMin: Date - let sessionStartDateMax: Date - let sessionEndDateMin: Date - - it('Should correctly create and federate the "waiting for stream" live', async function () { - this.timeout(40000) - - liveVideoUUID = await createLiveWrapper({ permanent: false, replay: false }) - - await waitJobs(servers) - - await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200) - await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE) - }) - - it('Should correctly have updated the live and federated it when streaming in the live', async function () { - this.timeout(120000) - - ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) - - sessionStartDateMin = new Date() - await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) - sessionStartDateMax = new Date() - - await waitJobs(servers) - - await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) - await checkVideoState(liveVideoUUID, VideoState.PUBLISHED) - }) - - it('Should correctly delete the video files after the stream ended', async function () { - this.timeout(120000) - - sessionEndDateMin = new Date() - await stopFfmpeg(ffmpegCommand) - - for (const server of servers) { - await server.live.waitUntilEnded({ videoId: liveVideoUUID }) - } - await waitJobs(servers) - - // Live still exist, but cannot be played anymore - await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200) - await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED) - - // No resolutions saved since we did not save replay - await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false }) - }) - - it('Should have appropriate ended session', async function () { - const { data, total } = await servers[0].live.listSessions({ videoId: liveVideoUUID }) - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - - const session = data[0] - - const startDate = new Date(session.startDate) - expect(startDate).to.be.above(sessionStartDateMin) - expect(startDate).to.be.below(sessionStartDateMax) - - expect(session.endDate).to.exist - expect(new Date(session.endDate)).to.be.above(sessionEndDateMin) - - expect(session.saveReplay).to.be.false - expect(session.error).to.not.exist - expect(session.replayVideo).to.not.exist - }) - - it('Should correctly terminate the stream on blacklist and delete the live', async function () { - this.timeout(120000) - - await publishLiveAndBlacklist({ permanent: false, replay: false }) - - await checkVideosExist(liveVideoUUID, false) - - await servers[0].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - await servers[1].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - - await wait(5000) - await waitJobs(servers) - await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false }) - }) - - it('Should have blacklisted session error', async function () { - const session = await servers[0].live.findLatestSession({ videoId: liveVideoUUID }) - expect(session.startDate).to.exist - expect(session.endDate).to.exist - - expect(session.error).to.equal(LiveVideoError.BLACKLISTED) - expect(session.replayVideo).to.not.exist - }) - - it('Should correctly terminate the stream on delete and delete the video', async function () { - this.timeout(120000) - - await publishLiveAndDelete({ permanent: false, replay: false }) - - await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) - await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false }) - }) - }) - - describe('With save replay enabled on non permanent live', function () { - - it('Should correctly create and federate the "waiting for stream" live', async function () { - this.timeout(120000) - - liveVideoUUID = await createLiveWrapper({ permanent: false, replay: true, replaySettings: { privacy: VideoPrivacy.UNLISTED } }) - - await waitJobs(servers) - - await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200) - await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE) - await checkVideoPrivacy(liveVideoUUID, VideoPrivacy.PUBLIC) - }) - - it('Should correctly have updated the live and federated it when streaming in the live', async function () { - this.timeout(120000) - - ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) - await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) - - await waitJobs(servers) - - await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) - await checkVideoState(liveVideoUUID, VideoState.PUBLISHED) - await checkVideoPrivacy(liveVideoUUID, VideoPrivacy.PUBLIC) - }) - - it('Should correctly have saved the live and federated it after the streaming', async function () { - this.timeout(120000) - - const session = await servers[0].live.findLatestSession({ videoId: liveVideoUUID }) - expect(session.endDate).to.not.exist - expect(session.endingProcessed).to.be.false - expect(session.saveReplay).to.be.true - expect(session.replaySettings).to.exist - expect(session.replaySettings.privacy).to.equal(VideoPrivacy.UNLISTED) - - await stopFfmpeg(ffmpegCommand) - - await waitUntilLiveReplacedByReplayOnAllServers(servers, liveVideoUUID) - await waitJobs(servers) - - // Live has been transcoded - await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200) - await checkVideoState(liveVideoUUID, VideoState.PUBLISHED) - await checkVideoPrivacy(liveVideoUUID, VideoPrivacy.UNLISTED) - }) - - it('Should find the replay live session', async function () { - const session = await servers[0].live.getReplaySession({ videoId: liveVideoUUID }) - - expect(session).to.exist - - expect(session.startDate).to.exist - expect(session.endDate).to.exist - - expect(session.error).to.not.exist - expect(session.saveReplay).to.be.true - expect(session.endingProcessed).to.be.true - expect(session.replaySettings).to.exist - expect(session.replaySettings.privacy).to.equal(VideoPrivacy.UNLISTED) - - expect(session.replayVideo).to.exist - expect(session.replayVideo.id).to.exist - expect(session.replayVideo.shortUUID).to.exist - expect(session.replayVideo.uuid).to.equal(liveVideoUUID) - }) - - it('Should update the saved live and correctly federate the updated attributes', async function () { - this.timeout(120000) - - await servers[0].videos.update({ id: liveVideoUUID, attributes: { name: 'video updated', privacy: VideoPrivacy.PUBLIC } }) - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: liveVideoUUID }) - expect(video.name).to.equal('video updated') - expect(video.isLive).to.be.false - expect(video.privacy.id).to.equal(VideoPrivacy.PUBLIC) - } - }) - - it('Should have cleaned up the live files', async function () { - await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false, savedResolutions: [ 720 ] }) - }) - - it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () { - this.timeout(120000) - - await publishLiveAndBlacklist({ permanent: false, replay: true, replaySettings: { privacy: VideoPrivacy.PUBLIC } }) - - await checkVideosExist(liveVideoUUID, false) - - await servers[0].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - await servers[1].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - - await wait(5000) - await waitJobs(servers) - await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false, savedResolutions: [ 720 ] }) - }) - - it('Should correctly terminate the stream on delete and delete the video', async function () { - this.timeout(120000) - - await publishLiveAndDelete({ permanent: false, replay: true, replaySettings: { privacy: VideoPrivacy.PUBLIC } }) - - await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) - await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false }) - }) - }) - - describe('With save replay enabled on permanent live', function () { - let lastReplayUUID: string - - describe('With a first live and its replay', function () { - - it('Should correctly create and federate the "waiting for stream" live', async function () { - this.timeout(120000) - - liveVideoUUID = await createLiveWrapper({ permanent: true, replay: true, replaySettings: { privacy: VideoPrivacy.UNLISTED } }) - - await waitJobs(servers) - - await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200) - await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE) - await checkVideoPrivacy(liveVideoUUID, VideoPrivacy.PUBLIC) - }) - - it('Should correctly have updated the live and federated it when streaming in the live', async function () { - this.timeout(120000) - - ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) - await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) - - await waitJobs(servers) - - await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) - await checkVideoState(liveVideoUUID, VideoState.PUBLISHED) - await checkVideoPrivacy(liveVideoUUID, VideoPrivacy.PUBLIC) - }) - - it('Should correctly have saved the live and federated it after the streaming', async function () { - this.timeout(120000) - - const liveDetails = await servers[0].videos.get({ id: liveVideoUUID }) - - await stopFfmpeg(ffmpegCommand) - - await waitUntilLiveWaitingOnAllServers(servers, liveVideoUUID) - await waitJobs(servers) - - const video = await findExternalSavedVideo(servers[0], liveDetails) - expect(video).to.exist - - for (const server of servers) { - await server.videos.get({ id: video.uuid }) - } - - lastReplayUUID = video.uuid - }) - - it('Should have appropriate ended session and replay live session', async function () { - const { data, total } = await servers[0].live.listSessions({ videoId: liveVideoUUID }) - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - - const sessionFromLive = data[0] - const sessionFromReplay = await servers[0].live.getReplaySession({ videoId: lastReplayUUID }) - - for (const session of [ sessionFromLive, sessionFromReplay ]) { - expect(session.startDate).to.exist - expect(session.endDate).to.exist - - expect(session.replaySettings).to.exist - expect(session.replaySettings.privacy).to.equal(VideoPrivacy.UNLISTED) - - expect(session.error).to.not.exist - - expect(session.replayVideo).to.exist - expect(session.replayVideo.id).to.exist - expect(session.replayVideo.shortUUID).to.exist - expect(session.replayVideo.uuid).to.equal(lastReplayUUID) - } - }) - - it('Should have the first live replay with correct settings', async function () { - await checkVideosExist(lastReplayUUID, false, HttpStatusCode.OK_200) - await checkVideoState(lastReplayUUID, VideoState.PUBLISHED) - await checkVideoPrivacy(lastReplayUUID, VideoPrivacy.UNLISTED) - }) - }) - - describe('With a second live and its replay', function () { - - it('Should update the replay settings', async function () { - await servers[0].live.update({ videoId: liveVideoUUID, fields: { replaySettings: { privacy: VideoPrivacy.PUBLIC } } }) - await waitJobs(servers) - - const live = await servers[0].live.get({ videoId: liveVideoUUID }) - - expect(live.saveReplay).to.be.true - expect(live.replaySettings).to.exist - expect(live.replaySettings.privacy).to.equal(VideoPrivacy.PUBLIC) - - }) - - it('Should correctly have updated the live and federated it when streaming in the live', async function () { - this.timeout(120000) - - ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) - await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) - - await waitJobs(servers) - - await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) - await checkVideoState(liveVideoUUID, VideoState.PUBLISHED) - await checkVideoPrivacy(liveVideoUUID, VideoPrivacy.PUBLIC) - }) - - it('Should correctly have saved the live and federated it after the streaming', async function () { - this.timeout(120000) - - const liveDetails = await servers[0].videos.get({ id: liveVideoUUID }) - - await stopFfmpeg(ffmpegCommand) - - await waitUntilLiveWaitingOnAllServers(servers, liveVideoUUID) - await waitJobs(servers) - - const video = await findExternalSavedVideo(servers[0], liveDetails) - expect(video).to.exist - - for (const server of servers) { - await server.videos.get({ id: video.uuid }) - } - - lastReplayUUID = video.uuid - }) - - it('Should have appropriate ended session and replay live session', async function () { - const { data, total } = await servers[0].live.listSessions({ videoId: liveVideoUUID }) - expect(total).to.equal(2) - expect(data).to.have.lengthOf(2) - - const sessionFromLive = data[1] - const sessionFromReplay = await servers[0].live.getReplaySession({ videoId: lastReplayUUID }) - - for (const session of [ sessionFromLive, sessionFromReplay ]) { - expect(session.startDate).to.exist - expect(session.endDate).to.exist - - expect(session.replaySettings).to.exist - expect(session.replaySettings.privacy).to.equal(VideoPrivacy.PUBLIC) - - expect(session.error).to.not.exist - - expect(session.replayVideo).to.exist - expect(session.replayVideo.id).to.exist - expect(session.replayVideo.shortUUID).to.exist - expect(session.replayVideo.uuid).to.equal(lastReplayUUID) - } - }) - - it('Should have the first live replay with correct settings', async function () { - await checkVideosExist(lastReplayUUID, true, HttpStatusCode.OK_200) - await checkVideoState(lastReplayUUID, VideoState.PUBLISHED) - await checkVideoPrivacy(lastReplayUUID, VideoPrivacy.PUBLIC) - }) - - it('Should have cleaned up the live files', async function () { - await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false }) - }) - - it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () { - this.timeout(120000) - - await servers[0].videos.remove({ id: lastReplayUUID }) - const { liveDetails } = await publishLiveAndBlacklist({ - permanent: true, - replay: true, - replaySettings: { privacy: VideoPrivacy.PUBLIC } - }) - - const replay = await findExternalSavedVideo(servers[0], liveDetails) - expect(replay).to.exist - - for (const videoId of [ liveVideoUUID, replay.uuid ]) { - await checkVideosExist(videoId, false) - - await servers[0].videos.get({ id: videoId, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - await servers[1].videos.get({ id: videoId, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - } - - await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false }) - }) - - it('Should correctly terminate the stream on delete and not save the video', async function () { - this.timeout(120000) - - const { liveDetails } = await publishLiveAndDelete({ - permanent: true, - replay: true, - replaySettings: { privacy: VideoPrivacy.PUBLIC } - }) - - const replay = await findExternalSavedVideo(servers[0], liveDetails) - expect(replay).to.not.exist - - await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404) - await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false }) - }) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/live/live-socket-messages.ts b/server/tests/api/live/live-socket-messages.ts deleted file mode 100644 index 0cccd1594..000000000 --- a/server/tests/api/live/live-socket-messages.ts +++ /dev/null @@ -1,186 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { wait } from '@shared/core-utils' -import { LiveVideoEventPayload, VideoPrivacy, VideoState } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - stopFfmpeg, - waitJobs, - waitUntilLivePublishedOnAllServers -} from '@shared/server-commands' - -describe('Test live socket messages', function () { - let servers: PeerTubeServer[] = [] - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(2) - - // Get the access tokens - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - - await servers[0].config.updateCustomSubConfig({ - newConfig: { - live: { - enabled: true, - allowReplay: true, - transcoding: { - enabled: false - } - } - } - }) - - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - }) - - describe('Live socket messages', function () { - - async function createLiveWrapper () { - const liveAttributes = { - name: 'live video', - channelId: servers[0].store.channel.id, - privacy: VideoPrivacy.PUBLIC - } - - const { uuid } = await servers[0].live.create({ fields: liveAttributes }) - return uuid - } - - it('Should correctly send a message when the live starts and ends', async function () { - this.timeout(60000) - - const localStateChanges: VideoState[] = [] - const remoteStateChanges: VideoState[] = [] - - const liveVideoUUID = await createLiveWrapper() - await waitJobs(servers) - - { - const videoId = await servers[0].videos.getId({ uuid: liveVideoUUID }) - - const localSocket = servers[0].socketIO.getLiveNotificationSocket() - localSocket.on('state-change', data => localStateChanges.push(data.state)) - localSocket.emit('subscribe', { videoId }) - } - - { - const videoId = await servers[1].videos.getId({ uuid: liveVideoUUID }) - - const remoteSocket = servers[1].socketIO.getLiveNotificationSocket() - remoteSocket.on('state-change', data => remoteStateChanges.push(data.state)) - remoteSocket.emit('subscribe', { videoId }) - } - - const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) - - await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) - await waitJobs(servers) - - for (const stateChanges of [ localStateChanges, remoteStateChanges ]) { - expect(stateChanges).to.have.length.at.least(1) - expect(stateChanges[stateChanges.length - 1]).to.equal(VideoState.PUBLISHED) - } - - await stopFfmpeg(ffmpegCommand) - - for (const server of servers) { - await server.live.waitUntilEnded({ videoId: liveVideoUUID }) - } - await waitJobs(servers) - - for (const stateChanges of [ localStateChanges, remoteStateChanges ]) { - expect(stateChanges).to.have.length.at.least(2) - expect(stateChanges[stateChanges.length - 1]).to.equal(VideoState.LIVE_ENDED) - } - }) - - it('Should correctly send views change notification', async function () { - this.timeout(60000) - - let localLastVideoViews = 0 - let remoteLastVideoViews = 0 - - const liveVideoUUID = await createLiveWrapper() - await waitJobs(servers) - - { - const videoId = await servers[0].videos.getId({ uuid: liveVideoUUID }) - - const localSocket = servers[0].socketIO.getLiveNotificationSocket() - localSocket.on('views-change', (data: LiveVideoEventPayload) => { localLastVideoViews = data.viewers }) - localSocket.emit('subscribe', { videoId }) - } - - { - const videoId = await servers[1].videos.getId({ uuid: liveVideoUUID }) - - const remoteSocket = servers[1].socketIO.getLiveNotificationSocket() - remoteSocket.on('views-change', (data: LiveVideoEventPayload) => { remoteLastVideoViews = data.viewers }) - remoteSocket.emit('subscribe', { videoId }) - } - - const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) - - await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) - await waitJobs(servers) - - expect(localLastVideoViews).to.equal(0) - expect(remoteLastVideoViews).to.equal(0) - - await servers[0].views.simulateView({ id: liveVideoUUID }) - await servers[1].views.simulateView({ id: liveVideoUUID }) - - await waitJobs(servers) - - expect(localLastVideoViews).to.equal(2) - expect(remoteLastVideoViews).to.equal(2) - - await stopFfmpeg(ffmpegCommand) - }) - - it('Should not receive a notification after unsubscribe', async function () { - this.timeout(120000) - - const stateChanges: VideoState[] = [] - - const liveVideoUUID = await createLiveWrapper() - await waitJobs(servers) - - const videoId = await servers[0].videos.getId({ uuid: liveVideoUUID }) - - const socket = servers[0].socketIO.getLiveNotificationSocket() - socket.on('state-change', data => stateChanges.push(data.state)) - socket.emit('subscribe', { videoId }) - - const command = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) - - await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID) - await waitJobs(servers) - - // Notifier waits before sending a notification - await wait(10000) - - expect(stateChanges).to.have.lengthOf(1) - socket.emit('unsubscribe', { videoId }) - - await stopFfmpeg(command) - await waitJobs(servers) - - expect(stateChanges).to.have.lengthOf(1) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/live/live.ts b/server/tests/api/live/live.ts deleted file mode 100644 index 2b302a8a2..000000000 --- a/server/tests/api/live/live.ts +++ /dev/null @@ -1,764 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { basename, join } from 'path' -import { SQLCommand, testImageGeneratedByFFmpeg, testLiveVideoResolutions } from '@server/tests/shared' -import { getAllFiles, wait } from '@shared/core-utils' -import { ffprobePromise, getVideoStream } from '@shared/ffmpeg' -import { - HttpStatusCode, - LiveVideo, - LiveVideoCreate, - LiveVideoLatencyMode, - VideoDetails, - VideoPrivacy, - VideoState, - VideoStreamingPlaylistType -} from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - killallServers, - LiveCommand, - makeGetRequest, - makeRawRequest, - PeerTubeServer, - sendRTMPStream, - setAccessTokensToServers, - setDefaultVideoChannel, - stopFfmpeg, - testFfmpegStreamError, - waitJobs, - waitUntilLivePublishedOnAllServers -} from '@shared/server-commands' - -describe('Test live', function () { - let servers: PeerTubeServer[] = [] - let commands: LiveCommand[] - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(2) - - // Get the access tokens - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - - await servers[0].config.updateCustomSubConfig({ - newConfig: { - live: { - enabled: true, - allowReplay: true, - latencySetting: { - enabled: true - }, - transcoding: { - enabled: false - } - } - } - }) - - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - - commands = servers.map(s => s.live) - }) - - describe('Live creation, update and delete', function () { - let liveVideoUUID: string - - it('Should create a live with the appropriate parameters', async function () { - this.timeout(20000) - - const attributes: LiveVideoCreate = { - category: 1, - licence: 2, - language: 'fr', - description: 'super live description', - support: 'support field', - channelId: servers[0].store.channel.id, - nsfw: false, - waitTranscoding: false, - name: 'my super live', - tags: [ 'tag1', 'tag2' ], - commentsEnabled: false, - downloadEnabled: false, - saveReplay: true, - replaySettings: { privacy: VideoPrivacy.PUBLIC }, - latencyMode: LiveVideoLatencyMode.SMALL_LATENCY, - privacy: VideoPrivacy.PUBLIC, - previewfile: 'video_short1-preview.webm.jpg', - thumbnailfile: 'video_short1.webm.jpg' - } - - const live = await commands[0].create({ fields: attributes }) - liveVideoUUID = live.uuid - - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: liveVideoUUID }) - - expect(video.category.id).to.equal(1) - expect(video.licence.id).to.equal(2) - expect(video.language.id).to.equal('fr') - expect(video.description).to.equal('super live description') - expect(video.support).to.equal('support field') - - expect(video.channel.name).to.equal(servers[0].store.channel.name) - expect(video.channel.host).to.equal(servers[0].store.channel.host) - - expect(video.isLive).to.be.true - - expect(video.nsfw).to.be.false - expect(video.waitTranscoding).to.be.false - expect(video.name).to.equal('my super live') - expect(video.tags).to.deep.equal([ 'tag1', 'tag2' ]) - expect(video.commentsEnabled).to.be.false - expect(video.downloadEnabled).to.be.false - expect(video.privacy.id).to.equal(VideoPrivacy.PUBLIC) - - await testImageGeneratedByFFmpeg(server.url, 'video_short1-preview.webm', video.previewPath) - await testImageGeneratedByFFmpeg(server.url, 'video_short1.webm', video.thumbnailPath) - - const live = await server.live.get({ videoId: liveVideoUUID }) - - if (server.url === servers[0].url) { - expect(live.rtmpUrl).to.equal('rtmp://' + server.hostname + ':' + servers[0].rtmpPort + '/live') - expect(live.streamKey).to.not.be.empty - - expect(live.replaySettings).to.exist - expect(live.replaySettings.privacy).to.equal(VideoPrivacy.PUBLIC) - } else { - expect(live.rtmpUrl).to.not.exist - expect(live.streamKey).to.not.exist - } - - expect(live.saveReplay).to.be.true - expect(live.latencyMode).to.equal(LiveVideoLatencyMode.SMALL_LATENCY) - } - }) - - it('Should have a default preview and thumbnail', async function () { - this.timeout(20000) - - const attributes: LiveVideoCreate = { - name: 'default live thumbnail', - channelId: servers[0].store.channel.id, - privacy: VideoPrivacy.UNLISTED, - nsfw: true - } - - const live = await commands[0].create({ fields: attributes }) - const videoId = live.uuid - - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: videoId }) - expect(video.privacy.id).to.equal(VideoPrivacy.UNLISTED) - expect(video.nsfw).to.be.true - - await makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) - await makeGetRequest({ url: server.url, path: video.previewPath, expectedStatus: HttpStatusCode.OK_200 }) - } - }) - - it('Should not have the live listed since nobody streams into', async function () { - for (const server of servers) { - const { total, data } = await server.videos.list() - - expect(total).to.equal(0) - expect(data).to.have.lengthOf(0) - } - }) - - it('Should not be able to update a live of another server', async function () { - await commands[1].update({ videoId: liveVideoUUID, fields: { saveReplay: false }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should update the live', async function () { - await commands[0].update({ videoId: liveVideoUUID, fields: { saveReplay: false, latencyMode: LiveVideoLatencyMode.DEFAULT } }) - await waitJobs(servers) - }) - - it('Have the live updated', async function () { - for (const server of servers) { - const live = await server.live.get({ videoId: liveVideoUUID }) - - if (server.url === servers[0].url) { - expect(live.rtmpUrl).to.equal('rtmp://' + server.hostname + ':' + servers[0].rtmpPort + '/live') - expect(live.streamKey).to.not.be.empty - } else { - expect(live.rtmpUrl).to.not.exist - expect(live.streamKey).to.not.exist - } - - expect(live.saveReplay).to.be.false - expect(live.replaySettings).to.not.exist - expect(live.latencyMode).to.equal(LiveVideoLatencyMode.DEFAULT) - } - }) - - it('Delete the live', async function () { - await servers[0].videos.remove({ id: liveVideoUUID }) - await waitJobs(servers) - }) - - it('Should have the live deleted', async function () { - for (const server of servers) { - await server.videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - await server.live.get({ videoId: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - } - }) - }) - - describe('Live filters', function () { - let ffmpegCommand: any - let liveVideoId: string - let vodVideoId: string - - before(async function () { - this.timeout(240000) - - vodVideoId = (await servers[0].videos.quickUpload({ name: 'vod video' })).uuid - - const liveOptions = { name: 'live', privacy: VideoPrivacy.PUBLIC, channelId: servers[0].store.channel.id } - const live = await commands[0].create({ fields: liveOptions }) - liveVideoId = live.uuid - - ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoId }) - await waitUntilLivePublishedOnAllServers(servers, liveVideoId) - await waitJobs(servers) - }) - - it('Should only display lives', async function () { - const { data, total } = await servers[0].videos.list({ isLive: true }) - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - expect(data[0].name).to.equal('live') - }) - - it('Should not display lives', async function () { - const { data, total } = await servers[0].videos.list({ isLive: false }) - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - expect(data[0].name).to.equal('vod video') - }) - - it('Should display my lives', async function () { - this.timeout(60000) - - await stopFfmpeg(ffmpegCommand) - await waitJobs(servers) - - const { data } = await servers[0].videos.listMyVideos({ isLive: true }) - - const result = data.every(v => v.isLive) - expect(result).to.be.true - }) - - it('Should not display my lives', async function () { - const { data } = await servers[0].videos.listMyVideos({ isLive: false }) - - const result = data.every(v => !v.isLive) - expect(result).to.be.true - }) - - after(async function () { - await servers[0].videos.remove({ id: vodVideoId }) - await servers[0].videos.remove({ id: liveVideoId }) - }) - }) - - describe('Stream checks', function () { - let liveVideo: LiveVideo & VideoDetails - let rtmpUrl: string - - before(function () { - rtmpUrl = 'rtmp://' + servers[0].hostname + ':' + servers[0].rtmpPort + '' - }) - - async function createLiveWrapper () { - const liveAttributes = { - name: 'user live', - channelId: servers[0].store.channel.id, - privacy: VideoPrivacy.PUBLIC, - saveReplay: false - } - - const { uuid } = await commands[0].create({ fields: liveAttributes }) - - const live = await commands[0].get({ videoId: uuid }) - const video = await servers[0].videos.get({ id: uuid }) - - return Object.assign(video, live) - } - - it('Should not allow a stream without the appropriate path', async function () { - this.timeout(60000) - - liveVideo = await createLiveWrapper() - - const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/bad-live', streamKey: liveVideo.streamKey }) - await testFfmpegStreamError(command, true) - }) - - it('Should not allow a stream without the appropriate stream key', async function () { - this.timeout(60000) - - const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: 'bad-stream-key' }) - await testFfmpegStreamError(command, true) - }) - - it('Should succeed with the correct params', async function () { - this.timeout(60000) - - const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: liveVideo.streamKey }) - await testFfmpegStreamError(command, false) - }) - - it('Should list this live now someone stream into it', async function () { - for (const server of servers) { - const { total, data } = await server.videos.list() - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - - const video = data[0] - expect(video.name).to.equal('user live') - expect(video.isLive).to.be.true - } - }) - - it('Should not allow a stream on a live that was blacklisted', async function () { - this.timeout(60000) - - liveVideo = await createLiveWrapper() - - await servers[0].blacklist.add({ videoId: liveVideo.uuid }) - - const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: liveVideo.streamKey }) - await testFfmpegStreamError(command, true) - }) - - it('Should not allow a stream on a live that was deleted', async function () { - this.timeout(60000) - - liveVideo = await createLiveWrapper() - - await servers[0].videos.remove({ id: liveVideo.uuid }) - - const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: liveVideo.streamKey }) - await testFfmpegStreamError(command, true) - }) - }) - - describe('Live transcoding', function () { - let liveVideoId: string - let sqlCommandServer1: SQLCommand - - async function createLiveWrapper (saveReplay: boolean) { - const liveAttributes = { - name: 'live video', - channelId: servers[0].store.channel.id, - privacy: VideoPrivacy.PUBLIC, - saveReplay, - replaySettings: saveReplay - ? { privacy: VideoPrivacy.PUBLIC } - : undefined - } - - const { uuid } = await commands[0].create({ fields: liveAttributes }) - return uuid - } - - function updateConf (resolutions: number[]) { - return servers[0].config.updateCustomSubConfig({ - newConfig: { - live: { - enabled: true, - allowReplay: true, - maxDuration: -1, - transcoding: { - enabled: true, - resolutions: { - '144p': resolutions.includes(144), - '240p': resolutions.includes(240), - '360p': resolutions.includes(360), - '480p': resolutions.includes(480), - '720p': resolutions.includes(720), - '1080p': resolutions.includes(1080), - '2160p': resolutions.includes(2160) - } - } - } - } - }) - } - - before(async function () { - await updateConf([]) - - sqlCommandServer1 = new SQLCommand(servers[0]) - }) - - it('Should enable transcoding without additional resolutions', async function () { - this.timeout(120000) - - liveVideoId = await createLiveWrapper(false) - - const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId }) - await waitUntilLivePublishedOnAllServers(servers, liveVideoId) - await waitJobs(servers) - - await testLiveVideoResolutions({ - originServer: servers[0], - sqlCommand: sqlCommandServer1, - servers, - liveVideoId, - resolutions: [ 720 ], - transcoded: true - }) - - await stopFfmpeg(ffmpegCommand) - }) - - it('Should transcode audio only RTMP stream', async function () { - this.timeout(120000) - - liveVideoId = await createLiveWrapper(false) - - const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_short_no_audio.mp4' }) - await waitUntilLivePublishedOnAllServers(servers, liveVideoId) - await waitJobs(servers) - - await stopFfmpeg(ffmpegCommand) - }) - - it('Should enable transcoding with some resolutions', async function () { - this.timeout(240000) - - const resolutions = [ 240, 480 ] - await updateConf(resolutions) - liveVideoId = await createLiveWrapper(false) - - const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId }) - await waitUntilLivePublishedOnAllServers(servers, liveVideoId) - await waitJobs(servers) - - await testLiveVideoResolutions({ - originServer: servers[0], - sqlCommand: sqlCommandServer1, - servers, - liveVideoId, - resolutions: resolutions.concat([ 720 ]), - transcoded: true - }) - - await stopFfmpeg(ffmpegCommand) - }) - - it('Should correctly set the appropriate bitrate depending on the input', async function () { - this.timeout(120000) - - liveVideoId = await createLiveWrapper(false) - - const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ - videoId: liveVideoId, - fixtureName: 'video_short.mp4', - copyCodecs: true - }) - await waitUntilLivePublishedOnAllServers(servers, liveVideoId) - await waitJobs(servers) - - const video = await servers[0].videos.get({ id: liveVideoId }) - - const masterPlaylist = video.streamingPlaylists[0].playlistUrl - const probe = await ffprobePromise(masterPlaylist) - - const bitrates = probe.streams.map(s => parseInt(s.tags.variant_bitrate)) - for (const bitrate of bitrates) { - expect(bitrate).to.exist - expect(isNaN(bitrate)).to.be.false - expect(bitrate).to.be.below(61_000_000) // video_short.mp4 bitrate - } - - await stopFfmpeg(ffmpegCommand) - }) - - it('Should enable transcoding with some resolutions and correctly save them', async function () { - this.timeout(500_000) - - const resolutions = [ 240, 360, 720 ] - - await updateConf(resolutions) - liveVideoId = await createLiveWrapper(true) - - const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_short2.webm' }) - await waitUntilLivePublishedOnAllServers(servers, liveVideoId) - await waitJobs(servers) - - await testLiveVideoResolutions({ - originServer: servers[0], - sqlCommand: sqlCommandServer1, - servers, - liveVideoId, - resolutions, - transcoded: true - }) - - await stopFfmpeg(ffmpegCommand) - await commands[0].waitUntilEnded({ videoId: liveVideoId }) - - await waitJobs(servers) - - await waitUntilLivePublishedOnAllServers(servers, liveVideoId) - - const maxBitrateLimits = { - 720: 6500 * 1000, // 60FPS - 360: 1250 * 1000, - 240: 700 * 1000 - } - - const minBitrateLimits = { - 720: 4800 * 1000, - 360: 1000 * 1000, - 240: 550 * 1000 - } - - for (const server of servers) { - const video = await server.videos.get({ id: liveVideoId }) - - expect(video.state.id).to.equal(VideoState.PUBLISHED) - expect(video.duration).to.be.greaterThan(1) - expect(video.files).to.have.lengthOf(0) - - const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS) - await makeRawRequest({ url: hlsPlaylist.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ url: hlsPlaylist.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) - - // We should have generated random filenames - expect(basename(hlsPlaylist.playlistUrl)).to.not.equal('master.m3u8') - expect(basename(hlsPlaylist.segmentsSha256Url)).to.not.equal('segments-sha256.json') - - expect(hlsPlaylist.files).to.have.lengthOf(resolutions.length) - - for (const resolution of resolutions) { - const file = hlsPlaylist.files.find(f => f.resolution.id === resolution) - - expect(file).to.exist - expect(file.size).to.be.greaterThan(1) - - if (resolution >= 720) { - expect(file.fps).to.be.approximately(60, 10) - } else { - expect(file.fps).to.be.approximately(30, 3) - } - - const filename = basename(file.fileUrl) - expect(filename).to.not.contain(video.uuid) - - const segmentPath = servers[0].servers.buildDirectory(join('streaming-playlists', 'hls', video.uuid, filename)) - - const probe = await ffprobePromise(segmentPath) - const videoStream = await getVideoStream(segmentPath, probe) - - expect(probe.format.bit_rate).to.be.below(maxBitrateLimits[videoStream.height]) - expect(probe.format.bit_rate).to.be.at.least(minBitrateLimits[videoStream.height]) - - await makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) - } - } - }) - - it('Should not generate an upper resolution than original file', async function () { - this.timeout(500_000) - - const resolutions = [ 240, 480 ] - await updateConf(resolutions) - - await servers[0].config.updateExistingSubConfig({ - newConfig: { - live: { - transcoding: { - alwaysTranscodeOriginalResolution: false - } - } - } - }) - - liveVideoId = await createLiveWrapper(true) - - const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_short2.webm' }) - await waitUntilLivePublishedOnAllServers(servers, liveVideoId) - await waitJobs(servers) - - await testLiveVideoResolutions({ - originServer: servers[0], - sqlCommand: sqlCommandServer1, - servers, - liveVideoId, - resolutions, - transcoded: true - }) - - await stopFfmpeg(ffmpegCommand) - await commands[0].waitUntilEnded({ videoId: liveVideoId }) - - await waitJobs(servers) - - await waitUntilLivePublishedOnAllServers(servers, liveVideoId) - - const video = await servers[0].videos.get({ id: liveVideoId }) - const hlsFiles = video.streamingPlaylists[0].files - - expect(video.files).to.have.lengthOf(0) - expect(hlsFiles).to.have.lengthOf(resolutions.length) - - // eslint-disable-next-line @typescript-eslint/require-array-sort-compare - expect(getAllFiles(video).map(f => f.resolution.id).sort()).to.deep.equal(resolutions) - }) - - it('Should only keep the original resolution if all resolutions are disabled', async function () { - this.timeout(600_000) - - await updateConf([]) - liveVideoId = await createLiveWrapper(true) - - const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_short2.webm' }) - await waitUntilLivePublishedOnAllServers(servers, liveVideoId) - await waitJobs(servers) - - await testLiveVideoResolutions({ - originServer: servers[0], - sqlCommand: sqlCommandServer1, - servers, - liveVideoId, - resolutions: [ 720 ], - transcoded: true - }) - - await stopFfmpeg(ffmpegCommand) - await commands[0].waitUntilEnded({ videoId: liveVideoId }) - - await waitJobs(servers) - - await waitUntilLivePublishedOnAllServers(servers, liveVideoId) - - const video = await servers[0].videos.get({ id: liveVideoId }) - const hlsFiles = video.streamingPlaylists[0].files - - expect(video.files).to.have.lengthOf(0) - expect(hlsFiles).to.have.lengthOf(1) - - expect(hlsFiles[0].resolution.id).to.equal(720) - }) - - after(async function () { - await sqlCommandServer1.cleanup() - }) - }) - - describe('After a server restart', function () { - let liveVideoId: string - let liveVideoReplayId: string - let permanentLiveVideoReplayId: string - - let permanentLiveReplayName: string - - let beforeServerRestart: Date - - async function createLiveWrapper (options: { saveReplay: boolean, permanent: boolean }) { - const liveAttributes: LiveVideoCreate = { - name: 'live video', - channelId: servers[0].store.channel.id, - privacy: VideoPrivacy.PUBLIC, - saveReplay: options.saveReplay, - replaySettings: options.saveReplay - ? { privacy: VideoPrivacy.PUBLIC } - : undefined, - permanentLive: options.permanent - } - - const { uuid } = await commands[0].create({ fields: liveAttributes }) - return uuid - } - - before(async function () { - this.timeout(600_000) - - liveVideoId = await createLiveWrapper({ saveReplay: false, permanent: false }) - liveVideoReplayId = await createLiveWrapper({ saveReplay: true, permanent: false }) - permanentLiveVideoReplayId = await createLiveWrapper({ saveReplay: true, permanent: true }) - - await Promise.all([ - commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId }), - commands[0].sendRTMPStreamInVideo({ videoId: permanentLiveVideoReplayId }), - commands[0].sendRTMPStreamInVideo({ videoId: liveVideoReplayId }) - ]) - - await Promise.all([ - commands[0].waitUntilPublished({ videoId: liveVideoId }), - commands[0].waitUntilPublished({ videoId: permanentLiveVideoReplayId }), - commands[0].waitUntilPublished({ videoId: liveVideoReplayId }) - ]) - - for (const videoUUID of [ liveVideoId, liveVideoReplayId, permanentLiveVideoReplayId ]) { - await commands[0].waitUntilSegmentGeneration({ - server: servers[0], - videoUUID, - playlistNumber: 0, - segment: 2 - }) - } - - { - const video = await servers[0].videos.get({ id: permanentLiveVideoReplayId }) - permanentLiveReplayName = video.name + ' - ' + new Date(video.publishedAt).toLocaleString() - } - - await killallServers([ servers[0] ]) - - beforeServerRestart = new Date() - await servers[0].run() - - await wait(5000) - await waitJobs(servers) - }) - - it('Should cleanup lives', async function () { - this.timeout(60000) - - await commands[0].waitUntilEnded({ videoId: liveVideoId }) - await commands[0].waitUntilWaiting({ videoId: permanentLiveVideoReplayId }) - }) - - it('Should save a non permanent live replay', async function () { - this.timeout(240000) - - await commands[0].waitUntilPublished({ videoId: liveVideoReplayId }) - - const session = await commands[0].getReplaySession({ videoId: liveVideoReplayId }) - expect(session.endDate).to.exist - expect(new Date(session.endDate)).to.be.above(beforeServerRestart) - }) - - it('Should have saved a permanent live replay', async function () { - this.timeout(120000) - - const { data } = await servers[0].videos.listMyVideos({ sort: '-publishedAt' }) - expect(data.find(v => v.name === permanentLiveReplayName)).to.exist - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/moderation/abuses.ts b/server/tests/api/moderation/abuses.ts deleted file mode 100644 index 9fc296ea8..000000000 --- a/server/tests/api/moderation/abuses.ts +++ /dev/null @@ -1,887 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { AbuseMessage, AbusePredefinedReasonsString, AbuseState, AdminAbuse, UserAbuse } from '@shared/models' -import { - AbusesCommand, - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - setDefaultAccountAvatar, - setDefaultChannelAvatar, - waitJobs -} from '@shared/server-commands' - -describe('Test abuses', function () { - let servers: PeerTubeServer[] = [] - let abuseServer1: AdminAbuse - let abuseServer2: AdminAbuse - let commands: AbusesCommand[] - - before(async function () { - this.timeout(50000) - - // Run servers - servers = await createMultipleServers(2) - - await setAccessTokensToServers(servers) - await setDefaultChannelAvatar(servers) - await setDefaultAccountAvatar(servers) - - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - - commands = servers.map(s => s.abuses) - }) - - describe('Video abuses', function () { - - before(async function () { - this.timeout(50000) - - // Upload some videos on each servers - { - const attributes = { - name: 'my super name for server 1', - description: 'my super description for server 1' - } - await servers[0].videos.upload({ attributes }) - } - - { - const attributes = { - name: 'my super name for server 2', - description: 'my super description for server 2' - } - await servers[1].videos.upload({ attributes }) - } - - // Wait videos propagation, server 2 has transcoding enabled - await waitJobs(servers) - - const { data } = await servers[0].videos.list() - expect(data.length).to.equal(2) - - servers[0].store.videoCreated = data.find(video => video.name === 'my super name for server 1') - servers[1].store.videoCreated = data.find(video => video.name === 'my super name for server 2') - }) - - it('Should not have abuses', async function () { - const body = await commands[0].getAdminList() - - expect(body.total).to.equal(0) - expect(body.data).to.be.an('array') - expect(body.data.length).to.equal(0) - }) - - it('Should report abuse on a local video', async function () { - this.timeout(15000) - - const reason = 'my super bad reason' - await commands[0].report({ videoId: servers[0].store.videoCreated.id, reason }) - - // We wait requests propagation, even if the server 1 is not supposed to make a request to server 2 - await waitJobs(servers) - }) - - it('Should have 1 video abuses on server 1 and 0 on server 2', async function () { - { - const body = await commands[0].getAdminList() - - expect(body.total).to.equal(1) - expect(body.data).to.be.an('array') - expect(body.data.length).to.equal(1) - - const abuse = body.data[0] - expect(abuse.reason).to.equal('my super bad reason') - - expect(abuse.reporterAccount.name).to.equal('root') - expect(abuse.reporterAccount.host).to.equal(servers[0].host) - - expect(abuse.video.id).to.equal(servers[0].store.videoCreated.id) - expect(abuse.video.channel).to.exist - - expect(abuse.comment).to.be.null - - expect(abuse.flaggedAccount.name).to.equal('root') - expect(abuse.flaggedAccount.host).to.equal(servers[0].host) - - expect(abuse.video.countReports).to.equal(1) - expect(abuse.video.nthReport).to.equal(1) - - expect(abuse.countReportsForReporter).to.equal(1) - expect(abuse.countReportsForReportee).to.equal(1) - } - - { - const body = await commands[1].getAdminList() - expect(body.total).to.equal(0) - expect(body.data).to.be.an('array') - expect(body.data.length).to.equal(0) - } - }) - - it('Should report abuse on a remote video', async function () { - const reason = 'my super bad reason 2' - const videoId = await servers[0].videos.getId({ uuid: servers[1].store.videoCreated.uuid }) - await commands[0].report({ videoId, reason }) - - // We wait requests propagation - await waitJobs(servers) - }) - - it('Should have 2 video abuses on server 1 and 1 on server 2', async function () { - { - const body = await commands[0].getAdminList() - - expect(body.total).to.equal(2) - expect(body.data.length).to.equal(2) - - const abuse1 = body.data[0] - expect(abuse1.reason).to.equal('my super bad reason') - expect(abuse1.reporterAccount.name).to.equal('root') - expect(abuse1.reporterAccount.host).to.equal(servers[0].host) - - expect(abuse1.video.id).to.equal(servers[0].store.videoCreated.id) - expect(abuse1.video.countReports).to.equal(1) - expect(abuse1.video.nthReport).to.equal(1) - - expect(abuse1.comment).to.be.null - - expect(abuse1.flaggedAccount.name).to.equal('root') - expect(abuse1.flaggedAccount.host).to.equal(servers[0].host) - - expect(abuse1.state.id).to.equal(AbuseState.PENDING) - expect(abuse1.state.label).to.equal('Pending') - expect(abuse1.moderationComment).to.be.null - - const abuse2 = body.data[1] - expect(abuse2.reason).to.equal('my super bad reason 2') - - expect(abuse2.reporterAccount.name).to.equal('root') - expect(abuse2.reporterAccount.host).to.equal(servers[0].host) - - expect(abuse2.video.uuid).to.equal(servers[1].store.videoCreated.uuid) - - expect(abuse2.comment).to.be.null - - expect(abuse2.flaggedAccount.name).to.equal('root') - expect(abuse2.flaggedAccount.host).to.equal(servers[1].host) - - expect(abuse2.state.id).to.equal(AbuseState.PENDING) - expect(abuse2.state.label).to.equal('Pending') - expect(abuse2.moderationComment).to.be.null - } - - { - const body = await commands[1].getAdminList() - expect(body.total).to.equal(1) - expect(body.data.length).to.equal(1) - - abuseServer2 = body.data[0] - expect(abuseServer2.reason).to.equal('my super bad reason 2') - expect(abuseServer2.reporterAccount.name).to.equal('root') - expect(abuseServer2.reporterAccount.host).to.equal(servers[0].host) - - expect(abuseServer2.flaggedAccount.name).to.equal('root') - expect(abuseServer2.flaggedAccount.host).to.equal(servers[1].host) - - expect(abuseServer2.state.id).to.equal(AbuseState.PENDING) - expect(abuseServer2.state.label).to.equal('Pending') - expect(abuseServer2.moderationComment).to.be.null - } - }) - - it('Should hide video abuses from blocked accounts', async function () { - { - const videoId = await servers[1].videos.getId({ uuid: servers[0].store.videoCreated.uuid }) - await commands[1].report({ videoId, reason: 'will mute this' }) - await waitJobs(servers) - - const body = await commands[0].getAdminList() - expect(body.total).to.equal(3) - } - - const accountToBlock = 'root@' + servers[1].host - - { - await servers[0].blocklist.addToServerBlocklist({ account: accountToBlock }) - - const body = await commands[0].getAdminList() - expect(body.total).to.equal(2) - - const abuse = body.data.find(a => a.reason === 'will mute this') - expect(abuse).to.be.undefined - } - - { - await servers[0].blocklist.removeFromServerBlocklist({ account: accountToBlock }) - - const body = await commands[0].getAdminList() - expect(body.total).to.equal(3) - } - }) - - it('Should hide video abuses from blocked servers', async function () { - const serverToBlock = servers[1].host - - { - await servers[0].blocklist.addToServerBlocklist({ server: serverToBlock }) - - const body = await commands[0].getAdminList() - expect(body.total).to.equal(2) - - const abuse = body.data.find(a => a.reason === 'will mute this') - expect(abuse).to.be.undefined - } - - { - await servers[0].blocklist.removeFromServerBlocklist({ server: serverToBlock }) - - const body = await commands[0].getAdminList() - expect(body.total).to.equal(3) - } - }) - - it('Should keep the video abuse when deleting the video', async function () { - await servers[1].videos.remove({ id: abuseServer2.video.uuid }) - - await waitJobs(servers) - - const body = await commands[1].getAdminList() - expect(body.total).to.equal(2, 'wrong number of videos returned') - expect(body.data).to.have.lengthOf(2, 'wrong number of videos returned') - - const abuse = body.data[0] - expect(abuse.id).to.equal(abuseServer2.id, 'wrong origin server id for first video') - expect(abuse.video.id).to.equal(abuseServer2.video.id, 'wrong video id') - expect(abuse.video.channel).to.exist - expect(abuse.video.deleted).to.be.true - }) - - it('Should include counts of reports from reporter and reportee', async function () { - // register a second user to have two reporters/reportees - const user = { username: 'user2', password: 'password' } - await servers[0].users.create({ ...user }) - const userAccessToken = await servers[0].login.getAccessToken(user) - - // upload a third video via this user - const attributes = { - name: 'my second super name for server 1', - description: 'my second super description for server 1' - } - const { id } = await servers[0].videos.upload({ token: userAccessToken, attributes }) - const video3Id = id - - // resume with the test - const reason3 = 'my super bad reason 3' - await commands[0].report({ videoId: video3Id, reason: reason3 }) - - const reason4 = 'my super bad reason 4' - await commands[0].report({ token: userAccessToken, videoId: servers[0].store.videoCreated.id, reason: reason4 }) - - { - const body = await commands[0].getAdminList() - const abuses = body.data - - const abuseVideo3 = body.data.find(a => a.video.id === video3Id) - expect(abuseVideo3).to.not.be.undefined - expect(abuseVideo3.video.countReports).to.equal(1, 'wrong reports count for video 3') - expect(abuseVideo3.video.nthReport).to.equal(1, 'wrong report position in report list for video 3') - expect(abuseVideo3.countReportsForReportee).to.equal(1, 'wrong reports count for reporter on video 3 abuse') - expect(abuseVideo3.countReportsForReporter).to.equal(3, 'wrong reports count for reportee on video 3 abuse') - - const abuseServer1 = abuses.find(a => a.video.id === servers[0].store.videoCreated.id) - expect(abuseServer1.countReportsForReportee).to.equal(3, 'wrong reports count for reporter on video 1 abuse') - } - }) - - it('Should list predefined reasons as well as timestamps for the reported video', async function () { - const reason5 = 'my super bad reason 5' - const predefinedReasons5: AbusePredefinedReasonsString[] = [ 'violentOrRepulsive', 'captions' ] - const createRes = await commands[0].report({ - videoId: servers[0].store.videoCreated.id, - reason: reason5, - predefinedReasons: predefinedReasons5, - startAt: 1, - endAt: 5 - }) - - const body = await commands[0].getAdminList() - - { - const abuse = body.data.find(a => a.id === createRes.abuse.id) - expect(abuse.reason).to.equals(reason5) - expect(abuse.predefinedReasons).to.deep.equals(predefinedReasons5, 'predefined reasons do not match the one reported') - expect(abuse.video.startAt).to.equal(1, "starting timestamp doesn't match the one reported") - expect(abuse.video.endAt).to.equal(5, "ending timestamp doesn't match the one reported") - } - }) - - it('Should delete the video abuse', async function () { - await commands[1].delete({ abuseId: abuseServer2.id }) - - await waitJobs(servers) - - { - const body = await commands[1].getAdminList() - expect(body.total).to.equal(1) - expect(body.data.length).to.equal(1) - expect(body.data[0].id).to.not.equal(abuseServer2.id) - } - - { - const body = await commands[0].getAdminList() - expect(body.total).to.equal(6) - } - }) - - it('Should list and filter video abuses', async function () { - async function list (query: Parameters[0]) { - const body = await commands[0].getAdminList(query) - - return body.data - } - - expect(await list({ id: 56 })).to.have.lengthOf(0) - expect(await list({ id: 1 })).to.have.lengthOf(1) - - expect(await list({ search: 'my super name for server 1' })).to.have.lengthOf(4) - expect(await list({ search: 'aaaaaaaaaaaaaaaaaaaaaaaaaa' })).to.have.lengthOf(0) - - expect(await list({ searchVideo: 'my second super name for server 1' })).to.have.lengthOf(1) - - expect(await list({ searchVideoChannel: 'root' })).to.have.lengthOf(4) - expect(await list({ searchVideoChannel: 'aaaa' })).to.have.lengthOf(0) - - expect(await list({ searchReporter: 'user2' })).to.have.lengthOf(1) - expect(await list({ searchReporter: 'root' })).to.have.lengthOf(5) - - expect(await list({ searchReportee: 'root' })).to.have.lengthOf(5) - expect(await list({ searchReportee: 'aaaa' })).to.have.lengthOf(0) - - expect(await list({ videoIs: 'deleted' })).to.have.lengthOf(1) - expect(await list({ videoIs: 'blacklisted' })).to.have.lengthOf(0) - - expect(await list({ state: AbuseState.ACCEPTED })).to.have.lengthOf(0) - expect(await list({ state: AbuseState.PENDING })).to.have.lengthOf(6) - - expect(await list({ predefinedReason: 'violentOrRepulsive' })).to.have.lengthOf(1) - expect(await list({ predefinedReason: 'serverRules' })).to.have.lengthOf(0) - }) - }) - - describe('Comment abuses', function () { - - async function getComment (server: PeerTubeServer, videoIdArg: number | string) { - const videoId = typeof videoIdArg === 'string' - ? await server.videos.getId({ uuid: videoIdArg }) - : videoIdArg - - const { data } = await server.comments.listThreads({ videoId }) - - return data[0] - } - - before(async function () { - this.timeout(50000) - - servers[0].store.videoCreated = await servers[0].videos.quickUpload({ name: 'server 1' }) - servers[1].store.videoCreated = await servers[1].videos.quickUpload({ name: 'server 2' }) - - await servers[0].comments.createThread({ videoId: servers[0].store.videoCreated.id, text: 'comment server 1' }) - await servers[1].comments.createThread({ videoId: servers[1].store.videoCreated.id, text: 'comment server 2' }) - - await waitJobs(servers) - }) - - it('Should report abuse on a comment', async function () { - this.timeout(15000) - - const comment = await getComment(servers[0], servers[0].store.videoCreated.id) - - const reason = 'it is a bad comment' - await commands[0].report({ commentId: comment.id, reason }) - - await waitJobs(servers) - }) - - it('Should have 1 comment abuse on server 1 and 0 on server 2', async function () { - { - const comment = await getComment(servers[0], servers[0].store.videoCreated.id) - const body = await commands[0].getAdminList({ filter: 'comment' }) - - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - - const abuse = body.data[0] - expect(abuse.reason).to.equal('it is a bad comment') - - expect(abuse.reporterAccount.name).to.equal('root') - expect(abuse.reporterAccount.host).to.equal(servers[0].host) - - expect(abuse.video).to.be.null - - expect(abuse.comment.deleted).to.be.false - expect(abuse.comment.id).to.equal(comment.id) - expect(abuse.comment.text).to.equal(comment.text) - expect(abuse.comment.video.name).to.equal('server 1') - expect(abuse.comment.video.id).to.equal(servers[0].store.videoCreated.id) - expect(abuse.comment.video.uuid).to.equal(servers[0].store.videoCreated.uuid) - - expect(abuse.countReportsForReporter).to.equal(5) - expect(abuse.countReportsForReportee).to.equal(5) - } - - { - const body = await commands[1].getAdminList({ filter: 'comment' }) - expect(body.total).to.equal(0) - expect(body.data.length).to.equal(0) - } - }) - - it('Should report abuse on a remote comment', async function () { - const comment = await getComment(servers[0], servers[1].store.videoCreated.uuid) - - const reason = 'it is a really bad comment' - await commands[0].report({ commentId: comment.id, reason }) - - await waitJobs(servers) - }) - - it('Should have 2 comment abuses on server 1 and 1 on server 2', async function () { - const commentServer2 = await getComment(servers[0], servers[1].store.videoCreated.shortUUID) - - { - const body = await commands[0].getAdminList({ filter: 'comment' }) - expect(body.total).to.equal(2) - expect(body.data.length).to.equal(2) - - const abuse = body.data[0] - expect(abuse.reason).to.equal('it is a bad comment') - expect(abuse.countReportsForReporter).to.equal(6) - expect(abuse.countReportsForReportee).to.equal(5) - - const abuse2 = body.data[1] - - expect(abuse2.reason).to.equal('it is a really bad comment') - - expect(abuse2.reporterAccount.name).to.equal('root') - expect(abuse2.reporterAccount.host).to.equal(servers[0].host) - - expect(abuse2.video).to.be.null - - expect(abuse2.comment.deleted).to.be.false - expect(abuse2.comment.id).to.equal(commentServer2.id) - expect(abuse2.comment.text).to.equal(commentServer2.text) - expect(abuse2.comment.video.name).to.equal('server 2') - expect(abuse2.comment.video.uuid).to.equal(servers[1].store.videoCreated.uuid) - - expect(abuse2.state.id).to.equal(AbuseState.PENDING) - expect(abuse2.state.label).to.equal('Pending') - - expect(abuse2.moderationComment).to.be.null - - expect(abuse2.countReportsForReporter).to.equal(6) - expect(abuse2.countReportsForReportee).to.equal(2) - } - - { - const body = await commands[1].getAdminList({ filter: 'comment' }) - expect(body.total).to.equal(1) - expect(body.data.length).to.equal(1) - - abuseServer2 = body.data[0] - expect(abuseServer2.reason).to.equal('it is a really bad comment') - expect(abuseServer2.reporterAccount.name).to.equal('root') - expect(abuseServer2.reporterAccount.host).to.equal(servers[0].host) - - expect(abuseServer2.state.id).to.equal(AbuseState.PENDING) - expect(abuseServer2.state.label).to.equal('Pending') - - expect(abuseServer2.moderationComment).to.be.null - - expect(abuseServer2.countReportsForReporter).to.equal(1) - expect(abuseServer2.countReportsForReportee).to.equal(1) - } - }) - - it('Should keep the comment abuse when deleting the comment', async function () { - const commentServer2 = await getComment(servers[0], servers[1].store.videoCreated.uuid) - - await servers[0].comments.delete({ videoId: servers[1].store.videoCreated.uuid, commentId: commentServer2.id }) - - await waitJobs(servers) - - const body = await commands[0].getAdminList({ filter: 'comment' }) - expect(body.total).to.equal(2) - expect(body.data).to.have.lengthOf(2) - - const abuse = body.data.find(a => a.comment?.id === commentServer2.id) - expect(abuse).to.not.be.undefined - - expect(abuse.comment.text).to.be.empty - expect(abuse.comment.video.name).to.equal('server 2') - expect(abuse.comment.deleted).to.be.true - }) - - it('Should delete the comment abuse', async function () { - await commands[1].delete({ abuseId: abuseServer2.id }) - - await waitJobs(servers) - - { - const body = await commands[1].getAdminList({ filter: 'comment' }) - expect(body.total).to.equal(0) - expect(body.data.length).to.equal(0) - } - - { - const body = await commands[0].getAdminList({ filter: 'comment' }) - expect(body.total).to.equal(2) - } - }) - - it('Should list and filter video abuses', async function () { - { - const body = await commands[0].getAdminList({ filter: 'comment', searchReportee: 'foo' }) - expect(body.total).to.equal(0) - } - - { - const body = await commands[0].getAdminList({ filter: 'comment', searchReportee: 'ot' }) - expect(body.total).to.equal(2) - } - - { - const body = await commands[0].getAdminList({ filter: 'comment', start: 1, count: 1, sort: 'createdAt' }) - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].comment.text).to.be.empty - } - - { - const body = await commands[0].getAdminList({ filter: 'comment', start: 1, count: 1, sort: '-createdAt' }) - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].comment.text).to.equal('comment server 1') - } - }) - }) - - describe('Account abuses', function () { - - function getAccountFromServer (server: PeerTubeServer, targetName: string, targetServer: PeerTubeServer) { - return server.accounts.get({ accountName: targetName + '@' + targetServer.host }) - } - - before(async function () { - this.timeout(50000) - - await servers[0].users.create({ username: 'user_1', password: 'donald' }) - - const token = await servers[1].users.generateUserAndToken('user_2') - await servers[1].videos.upload({ token, attributes: { name: 'super video' } }) - - await waitJobs(servers) - }) - - it('Should report abuse on an account', async function () { - this.timeout(15000) - - const account = await getAccountFromServer(servers[0], 'user_1', servers[0]) - - const reason = 'it is a bad account' - await commands[0].report({ accountId: account.id, reason }) - - await waitJobs(servers) - }) - - it('Should have 1 account abuse on server 1 and 0 on server 2', async function () { - { - const body = await commands[0].getAdminList({ filter: 'account' }) - - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - - const abuse = body.data[0] - expect(abuse.reason).to.equal('it is a bad account') - - expect(abuse.reporterAccount.name).to.equal('root') - expect(abuse.reporterAccount.host).to.equal(servers[0].host) - - expect(abuse.video).to.be.null - expect(abuse.comment).to.be.null - - expect(abuse.flaggedAccount.name).to.equal('user_1') - expect(abuse.flaggedAccount.host).to.equal(servers[0].host) - } - - { - const body = await commands[1].getAdminList({ filter: 'comment' }) - expect(body.total).to.equal(0) - expect(body.data.length).to.equal(0) - } - }) - - it('Should report abuse on a remote account', async function () { - const account = await getAccountFromServer(servers[0], 'user_2', servers[1]) - - const reason = 'it is a really bad account' - await commands[0].report({ accountId: account.id, reason }) - - await waitJobs(servers) - }) - - it('Should have 2 comment abuses on server 1 and 1 on server 2', async function () { - { - const body = await commands[0].getAdminList({ filter: 'account' }) - expect(body.total).to.equal(2) - expect(body.data.length).to.equal(2) - - const abuse: AdminAbuse = body.data[0] - expect(abuse.reason).to.equal('it is a bad account') - - const abuse2: AdminAbuse = body.data[1] - expect(abuse2.reason).to.equal('it is a really bad account') - - expect(abuse2.reporterAccount.name).to.equal('root') - expect(abuse2.reporterAccount.host).to.equal(servers[0].host) - - expect(abuse2.video).to.be.null - expect(abuse2.comment).to.be.null - - expect(abuse2.state.id).to.equal(AbuseState.PENDING) - expect(abuse2.state.label).to.equal('Pending') - - expect(abuse2.moderationComment).to.be.null - } - - { - const body = await commands[1].getAdminList({ filter: 'account' }) - expect(body.total).to.equal(1) - expect(body.data.length).to.equal(1) - - abuseServer2 = body.data[0] - - expect(abuseServer2.reason).to.equal('it is a really bad account') - - expect(abuseServer2.reporterAccount.name).to.equal('root') - expect(abuseServer2.reporterAccount.host).to.equal(servers[0].host) - - expect(abuseServer2.state.id).to.equal(AbuseState.PENDING) - expect(abuseServer2.state.label).to.equal('Pending') - - expect(abuseServer2.moderationComment).to.be.null - } - }) - - it('Should keep the account abuse when deleting the account', async function () { - const account = await getAccountFromServer(servers[1], 'user_2', servers[1]) - await servers[1].users.remove({ userId: account.userId }) - - await waitJobs(servers) - - const body = await commands[0].getAdminList({ filter: 'account' }) - expect(body.total).to.equal(2) - expect(body.data).to.have.lengthOf(2) - - const abuse = body.data.find(a => a.reason === 'it is a really bad account') - expect(abuse).to.not.be.undefined - }) - - it('Should delete the account abuse', async function () { - await commands[1].delete({ abuseId: abuseServer2.id }) - - await waitJobs(servers) - - { - const body = await commands[1].getAdminList({ filter: 'account' }) - expect(body.total).to.equal(0) - expect(body.data.length).to.equal(0) - } - - { - const body = await commands[0].getAdminList({ filter: 'account' }) - expect(body.total).to.equal(2) - - abuseServer1 = body.data[0] - } - }) - }) - - describe('Common actions on abuses', function () { - - it('Should update the state of an abuse', async function () { - await commands[0].update({ abuseId: abuseServer1.id, body: { state: AbuseState.REJECTED } }) - - const body = await commands[0].getAdminList({ id: abuseServer1.id }) - expect(body.data[0].state.id).to.equal(AbuseState.REJECTED) - }) - - it('Should add a moderation comment', async function () { - await commands[0].update({ abuseId: abuseServer1.id, body: { state: AbuseState.ACCEPTED, moderationComment: 'Valid' } }) - - const body = await commands[0].getAdminList({ id: abuseServer1.id }) - expect(body.data[0].state.id).to.equal(AbuseState.ACCEPTED) - expect(body.data[0].moderationComment).to.equal('Valid') - }) - }) - - describe('My abuses', async function () { - let abuseId1: number - let userAccessToken: string - - before(async function () { - userAccessToken = await servers[0].users.generateUserAndToken('user_42') - - await commands[0].report({ token: userAccessToken, videoId: servers[0].store.videoCreated.id, reason: 'user reason 1' }) - - const videoId = await servers[0].videos.getId({ uuid: servers[1].store.videoCreated.uuid }) - await commands[0].report({ token: userAccessToken, videoId, reason: 'user reason 2' }) - }) - - it('Should correctly list my abuses', async function () { - { - const body = await commands[0].getUserList({ token: userAccessToken, start: 0, count: 5, sort: 'createdAt' }) - expect(body.total).to.equal(2) - - const abuses = body.data - expect(abuses[0].reason).to.equal('user reason 1') - expect(abuses[1].reason).to.equal('user reason 2') - - abuseId1 = abuses[0].id - } - - { - const body = await commands[0].getUserList({ token: userAccessToken, start: 1, count: 1, sort: 'createdAt' }) - expect(body.total).to.equal(2) - - const abuses: UserAbuse[] = body.data - expect(abuses[0].reason).to.equal('user reason 2') - } - - { - const body = await commands[0].getUserList({ token: userAccessToken, start: 1, count: 1, sort: '-createdAt' }) - expect(body.total).to.equal(2) - - const abuses: UserAbuse[] = body.data - expect(abuses[0].reason).to.equal('user reason 1') - } - }) - - it('Should correctly filter my abuses by id', async function () { - const body = await commands[0].getUserList({ token: userAccessToken, id: abuseId1 }) - expect(body.total).to.equal(1) - - const abuses: UserAbuse[] = body.data - expect(abuses[0].reason).to.equal('user reason 1') - }) - - it('Should correctly filter my abuses by search', async function () { - const body = await commands[0].getUserList({ token: userAccessToken, search: 'server 2' }) - expect(body.total).to.equal(1) - - const abuses: UserAbuse[] = body.data - expect(abuses[0].reason).to.equal('user reason 2') - }) - - it('Should correctly filter my abuses by state', async function () { - await commands[0].update({ abuseId: abuseId1, body: { state: AbuseState.REJECTED } }) - - const body = await commands[0].getUserList({ token: userAccessToken, state: AbuseState.REJECTED }) - expect(body.total).to.equal(1) - - const abuses: UserAbuse[] = body.data - expect(abuses[0].reason).to.equal('user reason 1') - }) - }) - - describe('Abuse messages', async function () { - let abuseId: number - let userToken: string - let abuseMessageUserId: number - let abuseMessageModerationId: number - - before(async function () { - userToken = await servers[0].users.generateUserAndToken('user_43') - - const body = await commands[0].report({ token: userToken, videoId: servers[0].store.videoCreated.id, reason: 'user 43 reason 1' }) - abuseId = body.abuse.id - }) - - it('Should create some messages on the abuse', async function () { - await commands[0].addMessage({ token: userToken, abuseId, message: 'message 1' }) - await commands[0].addMessage({ abuseId, message: 'message 2' }) - await commands[0].addMessage({ abuseId, message: 'message 3' }) - await commands[0].addMessage({ token: userToken, abuseId, message: 'message 4' }) - }) - - it('Should have the correct messages count when listing abuses', async function () { - const results = await Promise.all([ - commands[0].getAdminList({ start: 0, count: 50 }), - commands[0].getUserList({ token: userToken, start: 0, count: 50 }) - ]) - - for (const body of results) { - const abuses = body.data - const abuse = abuses.find(a => a.id === abuseId) - expect(abuse.countMessages).to.equal(4) - } - }) - - it('Should correctly list messages of this abuse', async function () { - const results = await Promise.all([ - commands[0].listMessages({ abuseId }), - commands[0].listMessages({ token: userToken, abuseId }) - ]) - - for (const body of results) { - expect(body.total).to.equal(4) - - const abuseMessages: AbuseMessage[] = body.data - - expect(abuseMessages[0].message).to.equal('message 1') - expect(abuseMessages[0].byModerator).to.be.false - expect(abuseMessages[0].account.name).to.equal('user_43') - - abuseMessageUserId = abuseMessages[0].id - - expect(abuseMessages[1].message).to.equal('message 2') - expect(abuseMessages[1].byModerator).to.be.true - expect(abuseMessages[1].account.name).to.equal('root') - - expect(abuseMessages[2].message).to.equal('message 3') - expect(abuseMessages[2].byModerator).to.be.true - expect(abuseMessages[2].account.name).to.equal('root') - abuseMessageModerationId = abuseMessages[2].id - - expect(abuseMessages[3].message).to.equal('message 4') - expect(abuseMessages[3].byModerator).to.be.false - expect(abuseMessages[3].account.name).to.equal('user_43') - } - }) - - it('Should delete messages', async function () { - await commands[0].deleteMessage({ abuseId, messageId: abuseMessageModerationId }) - await commands[0].deleteMessage({ token: userToken, abuseId, messageId: abuseMessageUserId }) - - const results = await Promise.all([ - commands[0].listMessages({ abuseId }), - commands[0].listMessages({ token: userToken, abuseId }) - ]) - - for (const body of results) { - expect(body.total).to.equal(2) - - const abuseMessages: AbuseMessage[] = body.data - expect(abuseMessages[0].message).to.equal('message 2') - expect(abuseMessages[1].message).to.equal('message 4') - } - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/moderation/blocklist-notification.ts b/server/tests/api/moderation/blocklist-notification.ts deleted file mode 100644 index 9c2863a58..000000000 --- a/server/tests/api/moderation/blocklist-notification.ts +++ /dev/null @@ -1,231 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { UserNotificationType } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -async function checkNotifications (server: PeerTubeServer, token: string, expected: UserNotificationType[]) { - const { data } = await server.notifications.list({ token, start: 0, count: 10, unread: true }) - expect(data).to.have.lengthOf(expected.length) - - for (const type of expected) { - expect(data.find(n => n.type === type)).to.exist - } -} - -describe('Test blocklist notifications', function () { - let servers: PeerTubeServer[] - let videoUUID: string - - let userToken1: string - let userToken2: string - let remoteUserToken: string - - async function resetState () { - try { - await servers[1].subscriptions.remove({ token: remoteUserToken, uri: 'user1_channel@' + servers[0].host }) - await servers[1].subscriptions.remove({ token: remoteUserToken, uri: 'user2_channel@' + servers[0].host }) - } catch {} - - await waitJobs(servers) - - await servers[0].notifications.markAsReadAll({ token: userToken1 }) - await servers[0].notifications.markAsReadAll({ token: userToken2 }) - - { - const { uuid } = await servers[0].videos.upload({ token: userToken1, attributes: { name: 'video' } }) - videoUUID = uuid - - await waitJobs(servers) - } - - { - await servers[1].comments.createThread({ - token: remoteUserToken, - videoId: videoUUID, - text: '@user2@' + servers[0].host + ' hello' - }) - } - - { - - await servers[1].subscriptions.add({ token: remoteUserToken, targetUri: 'user1_channel@' + servers[0].host }) - await servers[1].subscriptions.add({ token: remoteUserToken, targetUri: 'user2_channel@' + servers[0].host }) - } - - await waitJobs(servers) - } - - before(async function () { - this.timeout(60000) - - servers = await createMultipleServers(2) - await setAccessTokensToServers(servers) - - { - const user = { username: 'user1', password: 'password' } - await servers[0].users.create({ - username: user.username, - password: user.password, - videoQuota: -1, - videoQuotaDaily: -1 - }) - - userToken1 = await servers[0].login.getAccessToken(user) - await servers[0].videos.upload({ token: userToken1, attributes: { name: 'video user 1' } }) - } - - { - const user = { username: 'user2', password: 'password' } - await servers[0].users.create({ username: user.username, password: user.password }) - - userToken2 = await servers[0].login.getAccessToken(user) - } - - { - const user = { username: 'user3', password: 'password' } - await servers[1].users.create({ username: user.username, password: user.password }) - - remoteUserToken = await servers[1].login.getAccessToken(user) - } - - await doubleFollow(servers[0], servers[1]) - }) - - describe('User blocks another user', function () { - - before(async function () { - this.timeout(30000) - - await resetState() - }) - - it('Should have appropriate notifications', async function () { - const notifs = [ UserNotificationType.NEW_COMMENT_ON_MY_VIDEO, UserNotificationType.NEW_FOLLOW ] - await checkNotifications(servers[0], userToken1, notifs) - }) - - it('Should block an account', async function () { - await servers[0].blocklist.addToMyBlocklist({ token: userToken1, account: 'user3@' + servers[1].host }) - await waitJobs(servers) - }) - - it('Should not have notifications from this account', async function () { - await checkNotifications(servers[0], userToken1, []) - }) - - it('Should have notifications of this account on user 2', async function () { - const notifs = [ UserNotificationType.COMMENT_MENTION, UserNotificationType.NEW_FOLLOW ] - - await checkNotifications(servers[0], userToken2, notifs) - - await servers[0].blocklist.removeFromMyBlocklist({ token: userToken1, account: 'user3@' + servers[1].host }) - }) - }) - - describe('User blocks another server', function () { - - before(async function () { - this.timeout(30000) - - await resetState() - }) - - it('Should have appropriate notifications', async function () { - const notifs = [ UserNotificationType.NEW_COMMENT_ON_MY_VIDEO, UserNotificationType.NEW_FOLLOW ] - await checkNotifications(servers[0], userToken1, notifs) - }) - - it('Should block an account', async function () { - await servers[0].blocklist.addToMyBlocklist({ token: userToken1, server: servers[1].host }) - await waitJobs(servers) - }) - - it('Should not have notifications from this account', async function () { - await checkNotifications(servers[0], userToken1, []) - }) - - it('Should have notifications of this account on user 2', async function () { - const notifs = [ UserNotificationType.COMMENT_MENTION, UserNotificationType.NEW_FOLLOW ] - - await checkNotifications(servers[0], userToken2, notifs) - - await servers[0].blocklist.removeFromMyBlocklist({ token: userToken1, server: servers[1].host }) - }) - }) - - describe('Server blocks a user', function () { - - before(async function () { - this.timeout(30000) - - await resetState() - }) - - it('Should have appropriate notifications', async function () { - { - const notifs = [ UserNotificationType.NEW_COMMENT_ON_MY_VIDEO, UserNotificationType.NEW_FOLLOW ] - await checkNotifications(servers[0], userToken1, notifs) - } - - { - const notifs = [ UserNotificationType.COMMENT_MENTION, UserNotificationType.NEW_FOLLOW ] - await checkNotifications(servers[0], userToken2, notifs) - } - }) - - it('Should block an account', async function () { - await servers[0].blocklist.addToServerBlocklist({ account: 'user3@' + servers[1].host }) - await waitJobs(servers) - }) - - it('Should not have notifications from this account', async function () { - await checkNotifications(servers[0], userToken1, []) - await checkNotifications(servers[0], userToken2, []) - - await servers[0].blocklist.removeFromServerBlocklist({ account: 'user3@' + servers[1].host }) - }) - }) - - describe('Server blocks a server', function () { - - before(async function () { - this.timeout(30000) - - await resetState() - }) - - it('Should have appropriate notifications', async function () { - { - const notifs = [ UserNotificationType.NEW_COMMENT_ON_MY_VIDEO, UserNotificationType.NEW_FOLLOW ] - await checkNotifications(servers[0], userToken1, notifs) - } - - { - const notifs = [ UserNotificationType.COMMENT_MENTION, UserNotificationType.NEW_FOLLOW ] - await checkNotifications(servers[0], userToken2, notifs) - } - }) - - it('Should block an account', async function () { - await servers[0].blocklist.addToServerBlocklist({ server: servers[1].host }) - await waitJobs(servers) - }) - - it('Should not have notifications from this account', async function () { - await checkNotifications(servers[0], userToken1, []) - await checkNotifications(servers[0], userToken2, []) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/moderation/blocklist.ts b/server/tests/api/moderation/blocklist.ts deleted file mode 100644 index b90d8c16c..000000000 --- a/server/tests/api/moderation/blocklist.ts +++ /dev/null @@ -1,902 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { UserNotificationType } from '@shared/models' -import { - BlocklistCommand, - cleanupTests, - CommentsCommand, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - setDefaultAccountAvatar, - waitJobs -} from '@shared/server-commands' - -async function checkAllVideos (server: PeerTubeServer, token: string) { - { - const { data } = await server.videos.listWithToken({ token }) - expect(data).to.have.lengthOf(5) - } - - { - const { data } = await server.videos.list() - expect(data).to.have.lengthOf(5) - } -} - -async function checkAllComments (server: PeerTubeServer, token: string, videoUUID: string) { - const { data } = await server.comments.listThreads({ videoId: videoUUID, start: 0, count: 25, sort: '-createdAt', token }) - - const threads = data.filter(t => t.isDeleted === false) - expect(threads).to.have.lengthOf(2) - - for (const thread of threads) { - const tree = await server.comments.getThread({ videoId: videoUUID, threadId: thread.id, token }) - expect(tree.children).to.have.lengthOf(1) - } -} - -async function checkCommentNotification ( - mainServer: PeerTubeServer, - comment: { server: PeerTubeServer, token: string, videoUUID: string, text: string }, - check: 'presence' | 'absence' -) { - const command = comment.server.comments - - const { threadId, createdAt } = await command.createThread({ token: comment.token, videoId: comment.videoUUID, text: comment.text }) - - await waitJobs([ mainServer, comment.server ]) - - const { data } = await mainServer.notifications.list({ start: 0, count: 30 }) - const commentNotifications = data.filter(n => n.comment && n.comment.video.uuid === comment.videoUUID && n.createdAt >= createdAt) - - if (check === 'presence') expect(commentNotifications).to.have.lengthOf(1) - else expect(commentNotifications).to.have.lengthOf(0) - - await command.delete({ token: comment.token, videoId: comment.videoUUID, commentId: threadId }) - - await waitJobs([ mainServer, comment.server ]) -} - -describe('Test blocklist', function () { - let servers: PeerTubeServer[] - let videoUUID1: string - let videoUUID2: string - let videoUUID3: string - let userToken1: string - let userModeratorToken: string - let userToken2: string - - let command: BlocklistCommand - let commentsCommand: CommentsCommand[] - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(3) - await setAccessTokensToServers(servers) - await setDefaultAccountAvatar(servers) - - command = servers[0].blocklist - commentsCommand = servers.map(s => s.comments) - - { - const user = { username: 'user1', password: 'password' } - await servers[0].users.create({ username: user.username, password: user.password }) - - userToken1 = await servers[0].login.getAccessToken(user) - await servers[0].videos.upload({ token: userToken1, attributes: { name: 'video user 1' } }) - } - - { - const user = { username: 'moderator', password: 'password' } - await servers[0].users.create({ username: user.username, password: user.password }) - - userModeratorToken = await servers[0].login.getAccessToken(user) - } - - { - const user = { username: 'user2', password: 'password' } - await servers[1].users.create({ username: user.username, password: user.password }) - - userToken2 = await servers[1].login.getAccessToken(user) - await servers[1].videos.upload({ token: userToken2, attributes: { name: 'video user 2' } }) - } - - { - const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video server 1' } }) - videoUUID1 = uuid - } - - { - const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video server 2' } }) - videoUUID2 = uuid - } - - { - const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video 2 server 1' } }) - videoUUID3 = uuid - } - - await doubleFollow(servers[0], servers[1]) - await doubleFollow(servers[0], servers[2]) - - { - const created = await commentsCommand[0].createThread({ videoId: videoUUID1, text: 'comment root 1' }) - const reply = await commentsCommand[0].addReply({ - token: userToken1, - videoId: videoUUID1, - toCommentId: created.id, - text: 'comment user 1' - }) - await commentsCommand[0].addReply({ videoId: videoUUID1, toCommentId: reply.id, text: 'comment root 1' }) - } - - { - const created = await commentsCommand[0].createThread({ token: userToken1, videoId: videoUUID1, text: 'comment user 1' }) - await commentsCommand[0].addReply({ videoId: videoUUID1, toCommentId: created.id, text: 'comment root 1' }) - } - - await waitJobs(servers) - }) - - describe('User blocklist', function () { - - describe('When managing account blocklist', function () { - it('Should list all videos', function () { - return checkAllVideos(servers[0], servers[0].accessToken) - }) - - it('Should list the comments', function () { - return checkAllComments(servers[0], servers[0].accessToken, videoUUID1) - }) - - it('Should block a remote account', async function () { - await command.addToMyBlocklist({ account: 'user2@' + servers[1].host }) - }) - - it('Should hide its videos', async function () { - const { data } = await servers[0].videos.listWithToken() - - expect(data).to.have.lengthOf(4) - - const v = data.find(v => v.name === 'video user 2') - expect(v).to.be.undefined - }) - - it('Should block a local account', async function () { - await command.addToMyBlocklist({ account: 'user1' }) - }) - - it('Should hide its videos', async function () { - const { data } = await servers[0].videos.listWithToken() - - expect(data).to.have.lengthOf(3) - - const v = data.find(v => v.name === 'video user 1') - expect(v).to.be.undefined - }) - - it('Should hide its comments', async function () { - const { data } = await commentsCommand[0].listThreads({ - token: servers[0].accessToken, - videoId: videoUUID1, - start: 0, - count: 25, - sort: '-createdAt' - }) - - expect(data).to.have.lengthOf(1) - expect(data[0].totalReplies).to.equal(1) - - const t = data.find(t => t.text === 'comment user 1') - expect(t).to.be.undefined - - for (const thread of data) { - const tree = await commentsCommand[0].getThread({ - videoId: videoUUID1, - threadId: thread.id, - token: servers[0].accessToken - }) - expect(tree.children).to.have.lengthOf(0) - } - }) - - it('Should not have notifications from blocked accounts', async function () { - this.timeout(20000) - - { - const comment = { server: servers[0], token: userToken1, videoUUID: videoUUID1, text: 'hidden comment' } - await checkCommentNotification(servers[0], comment, 'absence') - } - - { - const comment = { - server: servers[0], - token: userToken1, - videoUUID: videoUUID2, - text: 'hello @root@' + servers[0].host - } - await checkCommentNotification(servers[0], comment, 'absence') - } - }) - - it('Should list all the videos with another user', async function () { - return checkAllVideos(servers[0], userToken1) - }) - - it('Should list blocked accounts', async function () { - { - const body = await command.listMyAccountBlocklist({ start: 0, count: 1, sort: 'createdAt' }) - expect(body.total).to.equal(2) - - const block = body.data[0] - expect(block.byAccount.displayName).to.equal('root') - expect(block.byAccount.name).to.equal('root') - expect(block.blockedAccount.displayName).to.equal('user2') - expect(block.blockedAccount.name).to.equal('user2') - expect(block.blockedAccount.host).to.equal('' + servers[1].host) - } - - { - const body = await command.listMyAccountBlocklist({ start: 1, count: 2, sort: 'createdAt' }) - expect(body.total).to.equal(2) - - const block = body.data[0] - expect(block.byAccount.displayName).to.equal('root') - expect(block.byAccount.name).to.equal('root') - expect(block.blockedAccount.displayName).to.equal('user1') - expect(block.blockedAccount.name).to.equal('user1') - expect(block.blockedAccount.host).to.equal('' + servers[0].host) - } - }) - - it('Should search blocked accounts', async function () { - const body = await command.listMyAccountBlocklist({ start: 0, count: 10, search: 'user2' }) - expect(body.total).to.equal(1) - - expect(body.data[0].blockedAccount.name).to.equal('user2') - }) - - it('Should get blocked status', async function () { - const remoteHandle = 'user2@' + servers[1].host - const localHandle = 'user1@' + servers[0].host - const unknownHandle = 'user5@' + servers[0].host - - { - const status = await command.getStatus({ accounts: [ remoteHandle ] }) - expect(Object.keys(status.accounts)).to.have.lengthOf(1) - expect(status.accounts[remoteHandle].blockedByUser).to.be.false - expect(status.accounts[remoteHandle].blockedByServer).to.be.false - - expect(Object.keys(status.hosts)).to.have.lengthOf(0) - } - - { - const status = await command.getStatus({ token: servers[0].accessToken, accounts: [ remoteHandle ] }) - expect(Object.keys(status.accounts)).to.have.lengthOf(1) - expect(status.accounts[remoteHandle].blockedByUser).to.be.true - expect(status.accounts[remoteHandle].blockedByServer).to.be.false - - expect(Object.keys(status.hosts)).to.have.lengthOf(0) - } - - { - const status = await command.getStatus({ token: servers[0].accessToken, accounts: [ localHandle, remoteHandle, unknownHandle ] }) - expect(Object.keys(status.accounts)).to.have.lengthOf(3) - - for (const handle of [ localHandle, remoteHandle ]) { - expect(status.accounts[handle].blockedByUser).to.be.true - expect(status.accounts[handle].blockedByServer).to.be.false - } - - expect(status.accounts[unknownHandle].blockedByUser).to.be.false - expect(status.accounts[unknownHandle].blockedByServer).to.be.false - - expect(Object.keys(status.hosts)).to.have.lengthOf(0) - } - }) - - it('Should not allow a remote blocked user to comment my videos', async function () { - this.timeout(60000) - - { - await commentsCommand[1].createThread({ token: userToken2, videoId: videoUUID3, text: 'comment user 2' }) - await waitJobs(servers) - - await commentsCommand[0].createThread({ token: servers[0].accessToken, videoId: videoUUID3, text: 'uploader' }) - await waitJobs(servers) - - const commentId = await commentsCommand[1].findCommentId({ videoId: videoUUID3, text: 'uploader' }) - const message = 'reply by user 2' - const reply = await commentsCommand[1].addReply({ token: userToken2, videoId: videoUUID3, toCommentId: commentId, text: message }) - await commentsCommand[1].addReply({ videoId: videoUUID3, toCommentId: reply.id, text: 'another reply' }) - - await waitJobs(servers) - } - - // Server 2 has all the comments - { - const { data } = await commentsCommand[1].listThreads({ videoId: videoUUID3, count: 25, sort: '-createdAt' }) - - expect(data).to.have.lengthOf(2) - expect(data[0].text).to.equal('uploader') - expect(data[1].text).to.equal('comment user 2') - - const tree = await commentsCommand[1].getThread({ videoId: videoUUID3, threadId: data[0].id }) - expect(tree.children).to.have.lengthOf(1) - expect(tree.children[0].comment.text).to.equal('reply by user 2') - expect(tree.children[0].children).to.have.lengthOf(1) - expect(tree.children[0].children[0].comment.text).to.equal('another reply') - } - - // Server 1 and 3 should only have uploader comments - for (const server of [ servers[0], servers[2] ]) { - const { data } = await server.comments.listThreads({ videoId: videoUUID3, count: 25, sort: '-createdAt' }) - - expect(data).to.have.lengthOf(1) - expect(data[0].text).to.equal('uploader') - - const tree = await server.comments.getThread({ videoId: videoUUID3, threadId: data[0].id }) - - if (server.serverNumber === 1) expect(tree.children).to.have.lengthOf(0) - else expect(tree.children).to.have.lengthOf(1) - } - }) - - it('Should unblock the remote account', async function () { - await command.removeFromMyBlocklist({ account: 'user2@' + servers[1].host }) - }) - - it('Should display its videos', async function () { - const { data } = await servers[0].videos.listWithToken() - expect(data).to.have.lengthOf(4) - - const v = data.find(v => v.name === 'video user 2') - expect(v).not.to.be.undefined - }) - - it('Should display its comments on my video', async function () { - for (const server of servers) { - const { data } = await server.comments.listThreads({ videoId: videoUUID3, count: 25, sort: '-createdAt' }) - - // Server 3 should not have 2 comment threads, because server 1 did not forward the server 2 comment - if (server.serverNumber === 3) { - expect(data).to.have.lengthOf(1) - continue - } - - expect(data).to.have.lengthOf(2) - expect(data[0].text).to.equal('uploader') - expect(data[1].text).to.equal('comment user 2') - - const tree = await server.comments.getThread({ videoId: videoUUID3, threadId: data[0].id }) - expect(tree.children).to.have.lengthOf(1) - expect(tree.children[0].comment.text).to.equal('reply by user 2') - expect(tree.children[0].children).to.have.lengthOf(1) - expect(tree.children[0].children[0].comment.text).to.equal('another reply') - } - }) - - it('Should unblock the local account', async function () { - await command.removeFromMyBlocklist({ account: 'user1' }) - }) - - it('Should display its comments', function () { - return checkAllComments(servers[0], servers[0].accessToken, videoUUID1) - }) - - it('Should have a notification from a non blocked account', async function () { - this.timeout(20000) - - { - const comment = { server: servers[1], token: userToken2, videoUUID: videoUUID1, text: 'displayed comment' } - await checkCommentNotification(servers[0], comment, 'presence') - } - - { - const comment = { - server: servers[0], - token: userToken1, - videoUUID: videoUUID2, - text: 'hello @root@' + servers[0].host - } - await checkCommentNotification(servers[0], comment, 'presence') - } - }) - }) - - describe('When managing server blocklist', function () { - - it('Should list all videos', function () { - return checkAllVideos(servers[0], servers[0].accessToken) - }) - - it('Should list the comments', function () { - return checkAllComments(servers[0], servers[0].accessToken, videoUUID1) - }) - - it('Should block a remote server', async function () { - await command.addToMyBlocklist({ server: '' + servers[1].host }) - }) - - it('Should hide its videos', async function () { - const { data } = await servers[0].videos.listWithToken() - - expect(data).to.have.lengthOf(3) - - const v1 = data.find(v => v.name === 'video user 2') - const v2 = data.find(v => v.name === 'video server 2') - - expect(v1).to.be.undefined - expect(v2).to.be.undefined - }) - - it('Should list all the videos with another user', async function () { - return checkAllVideos(servers[0], userToken1) - }) - - it('Should hide its comments', async function () { - const { id } = await commentsCommand[1].createThread({ token: userToken2, videoId: videoUUID1, text: 'hidden comment 2' }) - - await waitJobs(servers) - - await checkAllComments(servers[0], servers[0].accessToken, videoUUID1) - - await commentsCommand[1].delete({ token: userToken2, videoId: videoUUID1, commentId: id }) - }) - - it('Should not have notifications from blocked server', async function () { - this.timeout(20000) - - { - const comment = { server: servers[1], token: userToken2, videoUUID: videoUUID1, text: 'hidden comment' } - await checkCommentNotification(servers[0], comment, 'absence') - } - - { - const comment = { - server: servers[1], - token: userToken2, - videoUUID: videoUUID1, - text: 'hello @root@' + servers[0].host - } - await checkCommentNotification(servers[0], comment, 'absence') - } - }) - - it('Should list blocked servers', async function () { - const body = await command.listMyServerBlocklist({ start: 0, count: 1, sort: 'createdAt' }) - expect(body.total).to.equal(1) - - const block = body.data[0] - expect(block.byAccount.displayName).to.equal('root') - expect(block.byAccount.name).to.equal('root') - expect(block.blockedServer.host).to.equal('' + servers[1].host) - }) - - it('Should search blocked servers', async function () { - const body = await command.listMyServerBlocklist({ start: 0, count: 10, search: servers[1].host }) - expect(body.total).to.equal(1) - - expect(body.data[0].blockedServer.host).to.equal(servers[1].host) - }) - - it('Should get blocklist status', async function () { - const blockedServer = servers[1].host - const notBlockedServer = 'example.com' - - { - const status = await command.getStatus({ hosts: [ blockedServer, notBlockedServer ] }) - expect(Object.keys(status.accounts)).to.have.lengthOf(0) - - expect(Object.keys(status.hosts)).to.have.lengthOf(2) - expect(status.hosts[blockedServer].blockedByUser).to.be.false - expect(status.hosts[blockedServer].blockedByServer).to.be.false - - expect(status.hosts[notBlockedServer].blockedByUser).to.be.false - expect(status.hosts[notBlockedServer].blockedByServer).to.be.false - } - - { - const status = await command.getStatus({ token: servers[0].accessToken, hosts: [ blockedServer, notBlockedServer ] }) - expect(Object.keys(status.accounts)).to.have.lengthOf(0) - - expect(Object.keys(status.hosts)).to.have.lengthOf(2) - expect(status.hosts[blockedServer].blockedByUser).to.be.true - expect(status.hosts[blockedServer].blockedByServer).to.be.false - - expect(status.hosts[notBlockedServer].blockedByUser).to.be.false - expect(status.hosts[notBlockedServer].blockedByServer).to.be.false - } - }) - - it('Should unblock the remote server', async function () { - await command.removeFromMyBlocklist({ server: '' + servers[1].host }) - }) - - it('Should display its videos', function () { - return checkAllVideos(servers[0], servers[0].accessToken) - }) - - it('Should display its comments', function () { - return checkAllComments(servers[0], servers[0].accessToken, videoUUID1) - }) - - it('Should have notification from unblocked server', async function () { - this.timeout(20000) - - { - const comment = { server: servers[1], token: userToken2, videoUUID: videoUUID1, text: 'displayed comment' } - await checkCommentNotification(servers[0], comment, 'presence') - } - - { - const comment = { - server: servers[1], - token: userToken2, - videoUUID: videoUUID1, - text: 'hello @root@' + servers[0].host - } - await checkCommentNotification(servers[0], comment, 'presence') - } - }) - }) - }) - - describe('Server blocklist', function () { - - describe('When managing account blocklist', function () { - it('Should list all videos', async function () { - for (const token of [ userModeratorToken, servers[0].accessToken ]) { - await checkAllVideos(servers[0], token) - } - }) - - it('Should list the comments', async function () { - for (const token of [ userModeratorToken, servers[0].accessToken ]) { - await checkAllComments(servers[0], token, videoUUID1) - } - }) - - it('Should block a remote account', async function () { - await command.addToServerBlocklist({ account: 'user2@' + servers[1].host }) - }) - - it('Should hide its videos', async function () { - for (const token of [ userModeratorToken, servers[0].accessToken ]) { - const { data } = await servers[0].videos.listWithToken({ token }) - - expect(data).to.have.lengthOf(4) - - const v = data.find(v => v.name === 'video user 2') - expect(v).to.be.undefined - } - }) - - it('Should block a local account', async function () { - await command.addToServerBlocklist({ account: 'user1' }) - }) - - it('Should hide its videos', async function () { - for (const token of [ userModeratorToken, servers[0].accessToken ]) { - const { data } = await servers[0].videos.listWithToken({ token }) - - expect(data).to.have.lengthOf(3) - - const v = data.find(v => v.name === 'video user 1') - expect(v).to.be.undefined - } - }) - - it('Should hide its comments', async function () { - for (const token of [ userModeratorToken, servers[0].accessToken ]) { - const { data } = await commentsCommand[0].listThreads({ videoId: videoUUID1, count: 20, sort: '-createdAt', token }) - const threads = data.filter(t => t.isDeleted === false) - - expect(threads).to.have.lengthOf(1) - expect(threads[0].totalReplies).to.equal(1) - - const t = threads.find(t => t.text === 'comment user 1') - expect(t).to.be.undefined - - for (const thread of threads) { - const tree = await commentsCommand[0].getThread({ videoId: videoUUID1, threadId: thread.id, token }) - expect(tree.children).to.have.lengthOf(0) - } - } - }) - - it('Should not have notification from blocked accounts by instance', async function () { - this.timeout(20000) - - { - const comment = { server: servers[0], token: userToken1, videoUUID: videoUUID1, text: 'hidden comment' } - await checkCommentNotification(servers[0], comment, 'absence') - } - - { - const comment = { - server: servers[1], - token: userToken2, - videoUUID: videoUUID1, - text: 'hello @root@' + servers[0].host - } - await checkCommentNotification(servers[0], comment, 'absence') - } - }) - - it('Should list blocked accounts', async function () { - { - const body = await command.listServerAccountBlocklist({ start: 0, count: 1, sort: 'createdAt' }) - expect(body.total).to.equal(2) - - const block = body.data[0] - expect(block.byAccount.displayName).to.equal('peertube') - expect(block.byAccount.name).to.equal('peertube') - expect(block.blockedAccount.displayName).to.equal('user2') - expect(block.blockedAccount.name).to.equal('user2') - expect(block.blockedAccount.host).to.equal('' + servers[1].host) - } - - { - const body = await command.listServerAccountBlocklist({ start: 1, count: 2, sort: 'createdAt' }) - expect(body.total).to.equal(2) - - const block = body.data[0] - expect(block.byAccount.displayName).to.equal('peertube') - expect(block.byAccount.name).to.equal('peertube') - expect(block.blockedAccount.displayName).to.equal('user1') - expect(block.blockedAccount.name).to.equal('user1') - expect(block.blockedAccount.host).to.equal('' + servers[0].host) - } - }) - - it('Should search blocked accounts', async function () { - const body = await command.listServerAccountBlocklist({ start: 0, count: 10, search: 'user2' }) - expect(body.total).to.equal(1) - - expect(body.data[0].blockedAccount.name).to.equal('user2') - }) - - it('Should get blocked status', async function () { - const remoteHandle = 'user2@' + servers[1].host - const localHandle = 'user1@' + servers[0].host - const unknownHandle = 'user5@' + servers[0].host - - for (const token of [ undefined, servers[0].accessToken ]) { - const status = await command.getStatus({ token, accounts: [ localHandle, remoteHandle, unknownHandle ] }) - expect(Object.keys(status.accounts)).to.have.lengthOf(3) - - for (const handle of [ localHandle, remoteHandle ]) { - expect(status.accounts[handle].blockedByUser).to.be.false - expect(status.accounts[handle].blockedByServer).to.be.true - } - - expect(status.accounts[unknownHandle].blockedByUser).to.be.false - expect(status.accounts[unknownHandle].blockedByServer).to.be.false - - expect(Object.keys(status.hosts)).to.have.lengthOf(0) - } - }) - - it('Should unblock the remote account', async function () { - await command.removeFromServerBlocklist({ account: 'user2@' + servers[1].host }) - }) - - it('Should display its videos', async function () { - for (const token of [ userModeratorToken, servers[0].accessToken ]) { - const { data } = await servers[0].videos.listWithToken({ token }) - expect(data).to.have.lengthOf(4) - - const v = data.find(v => v.name === 'video user 2') - expect(v).not.to.be.undefined - } - }) - - it('Should unblock the local account', async function () { - await command.removeFromServerBlocklist({ account: 'user1' }) - }) - - it('Should display its comments', async function () { - for (const token of [ userModeratorToken, servers[0].accessToken ]) { - await checkAllComments(servers[0], token, videoUUID1) - } - }) - - it('Should have notifications from unblocked accounts', async function () { - this.timeout(20000) - - { - const comment = { server: servers[0], token: userToken1, videoUUID: videoUUID1, text: 'displayed comment' } - await checkCommentNotification(servers[0], comment, 'presence') - } - - { - const comment = { - server: servers[1], - token: userToken2, - videoUUID: videoUUID1, - text: 'hello @root@' + servers[0].host - } - await checkCommentNotification(servers[0], comment, 'presence') - } - }) - }) - - describe('When managing server blocklist', function () { - - it('Should list all videos', async function () { - for (const token of [ userModeratorToken, servers[0].accessToken ]) { - await checkAllVideos(servers[0], token) - } - }) - - it('Should list the comments', async function () { - for (const token of [ userModeratorToken, servers[0].accessToken ]) { - await checkAllComments(servers[0], token, videoUUID1) - } - }) - - it('Should block a remote server', async function () { - await command.addToServerBlocklist({ server: '' + servers[1].host }) - }) - - it('Should hide its videos', async function () { - for (const token of [ userModeratorToken, servers[0].accessToken ]) { - const requests = [ - servers[0].videos.list(), - servers[0].videos.listWithToken({ token }) - ] - - for (const req of requests) { - const { data } = await req - expect(data).to.have.lengthOf(3) - - const v1 = data.find(v => v.name === 'video user 2') - const v2 = data.find(v => v.name === 'video server 2') - - expect(v1).to.be.undefined - expect(v2).to.be.undefined - } - } - }) - - it('Should hide its comments', async function () { - const { id } = await commentsCommand[1].createThread({ token: userToken2, videoId: videoUUID1, text: 'hidden comment 2' }) - - await waitJobs(servers) - - await checkAllComments(servers[0], servers[0].accessToken, videoUUID1) - - await commentsCommand[1].delete({ token: userToken2, videoId: videoUUID1, commentId: id }) - }) - - it('Should not have notification from blocked instances by instance', async function () { - this.timeout(50000) - - { - const comment = { server: servers[1], token: userToken2, videoUUID: videoUUID1, text: 'hidden comment' } - await checkCommentNotification(servers[0], comment, 'absence') - } - - { - const comment = { - server: servers[1], - token: userToken2, - videoUUID: videoUUID1, - text: 'hello @root@' + servers[0].host - } - await checkCommentNotification(servers[0], comment, 'absence') - } - - { - const now = new Date() - await servers[1].follows.unfollow({ target: servers[0] }) - await waitJobs(servers) - await servers[1].follows.follow({ hosts: [ servers[0].host ] }) - - await waitJobs(servers) - - const { data } = await servers[0].notifications.list({ start: 0, count: 30 }) - const commentNotifications = data.filter(n => { - return n.type === UserNotificationType.NEW_INSTANCE_FOLLOWER && n.createdAt >= now.toISOString() - }) - - expect(commentNotifications).to.have.lengthOf(0) - } - }) - - it('Should list blocked servers', async function () { - const body = await command.listServerServerBlocklist({ start: 0, count: 1, sort: 'createdAt' }) - expect(body.total).to.equal(1) - - const block = body.data[0] - expect(block.byAccount.displayName).to.equal('peertube') - expect(block.byAccount.name).to.equal('peertube') - expect(block.blockedServer.host).to.equal('' + servers[1].host) - }) - - it('Should search blocked servers', async function () { - const body = await command.listServerServerBlocklist({ start: 0, count: 10, search: servers[1].host }) - expect(body.total).to.equal(1) - - expect(body.data[0].blockedServer.host).to.equal(servers[1].host) - }) - - it('Should get blocklist status', async function () { - const blockedServer = servers[1].host - const notBlockedServer = 'example.com' - - for (const token of [ undefined, servers[0].accessToken ]) { - const status = await command.getStatus({ token, hosts: [ blockedServer, notBlockedServer ] }) - expect(Object.keys(status.accounts)).to.have.lengthOf(0) - - expect(Object.keys(status.hosts)).to.have.lengthOf(2) - expect(status.hosts[blockedServer].blockedByUser).to.be.false - expect(status.hosts[blockedServer].blockedByServer).to.be.true - - expect(status.hosts[notBlockedServer].blockedByUser).to.be.false - expect(status.hosts[notBlockedServer].blockedByServer).to.be.false - } - }) - - it('Should unblock the remote server', async function () { - await command.removeFromServerBlocklist({ server: '' + servers[1].host }) - }) - - it('Should list all videos', async function () { - for (const token of [ userModeratorToken, servers[0].accessToken ]) { - await checkAllVideos(servers[0], token) - } - }) - - it('Should list the comments', async function () { - for (const token of [ userModeratorToken, servers[0].accessToken ]) { - await checkAllComments(servers[0], token, videoUUID1) - } - }) - - it('Should have notification from unblocked instances', async function () { - this.timeout(50000) - - { - const comment = { server: servers[1], token: userToken2, videoUUID: videoUUID1, text: 'displayed comment' } - await checkCommentNotification(servers[0], comment, 'presence') - } - - { - const comment = { - server: servers[1], - token: userToken2, - videoUUID: videoUUID1, - text: 'hello @root@' + servers[0].host - } - await checkCommentNotification(servers[0], comment, 'presence') - } - - { - const now = new Date() - await servers[1].follows.unfollow({ target: servers[0] }) - await waitJobs(servers) - await servers[1].follows.follow({ hosts: [ servers[0].host ] }) - - await waitJobs(servers) - - const { data } = await servers[0].notifications.list({ start: 0, count: 30 }) - const commentNotifications = data.filter(n => { - return n.type === UserNotificationType.NEW_INSTANCE_FOLLOWER && n.createdAt >= now.toISOString() - }) - - expect(commentNotifications).to.have.lengthOf(1) - } - }) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/moderation/index.ts b/server/tests/api/moderation/index.ts deleted file mode 100644 index 874be03d5..000000000 --- a/server/tests/api/moderation/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './abuses' -export * from './blocklist-notification' -export * from './blocklist' -export * from './video-blacklist' diff --git a/server/tests/api/moderation/video-blacklist.ts b/server/tests/api/moderation/video-blacklist.ts deleted file mode 100644 index ef087a93b..000000000 --- a/server/tests/api/moderation/video-blacklist.ts +++ /dev/null @@ -1,414 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { FIXTURE_URLS } from '@server/tests/shared' -import { sortObjectComparator } from '@shared/core-utils' -import { UserAdminFlag, UserRole, VideoBlacklist, VideoBlacklistType } from '@shared/models' -import { - BlacklistCommand, - cleanupTests, - createMultipleServers, - doubleFollow, - killallServers, - PeerTubeServer, - setAccessTokensToServers, - setDefaultChannelAvatar, - waitJobs -} from '@shared/server-commands' - -describe('Test video blacklist', function () { - let servers: PeerTubeServer[] = [] - let videoId: number - let command: BlacklistCommand - - async function blacklistVideosOnServer (server: PeerTubeServer) { - const { data } = await server.videos.list() - - for (const video of data) { - await server.blacklist.add({ videoId: video.id, reason: 'super reason' }) - } - } - - before(async function () { - this.timeout(120000) - - // Run servers - servers = await createMultipleServers(2) - - // Get the access tokens - await setAccessTokensToServers(servers) - - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - await setDefaultChannelAvatar(servers[0]) - - // Upload 2 videos on server 2 - await servers[1].videos.upload({ attributes: { name: 'My 1st video', description: 'A video on server 2' } }) - await servers[1].videos.upload({ attributes: { name: 'My 2nd video', description: 'A video on server 2' } }) - - // Wait videos propagation, server 2 has transcoding enabled - await waitJobs(servers) - - command = servers[0].blacklist - - // Blacklist the two videos on server 1 - await blacklistVideosOnServer(servers[0]) - }) - - describe('When listing/searching videos', function () { - - it('Should not have the video blacklisted in videos list/search on server 1', async function () { - { - const { total, data } = await servers[0].videos.list() - - expect(total).to.equal(0) - expect(data).to.be.an('array') - expect(data.length).to.equal(0) - } - - { - const body = await servers[0].search.searchVideos({ search: 'video' }) - - expect(body.total).to.equal(0) - expect(body.data).to.be.an('array') - expect(body.data.length).to.equal(0) - } - }) - - it('Should have the blacklisted video in videos list/search on server 2', async function () { - { - const { total, data } = await servers[1].videos.list() - - expect(total).to.equal(2) - expect(data).to.be.an('array') - expect(data.length).to.equal(2) - } - - { - const body = await servers[1].search.searchVideos({ search: 'video' }) - - expect(body.total).to.equal(2) - expect(body.data).to.be.an('array') - expect(body.data.length).to.equal(2) - } - }) - }) - - describe('When listing manually blacklisted videos', function () { - it('Should display all the blacklisted videos', async function () { - const body = await command.list() - expect(body.total).to.equal(2) - - const blacklistedVideos = body.data - expect(blacklistedVideos).to.be.an('array') - expect(blacklistedVideos.length).to.equal(2) - - for (const blacklistedVideo of blacklistedVideos) { - expect(blacklistedVideo.reason).to.equal('super reason') - videoId = blacklistedVideo.video.id - } - }) - - it('Should display all the blacklisted videos when applying manual type filter', async function () { - const body = await command.list({ type: VideoBlacklistType.MANUAL }) - expect(body.total).to.equal(2) - - const blacklistedVideos = body.data - expect(blacklistedVideos).to.be.an('array') - expect(blacklistedVideos.length).to.equal(2) - }) - - it('Should display nothing when applying automatic type filter', async function () { - const body = await command.list({ type: VideoBlacklistType.AUTO_BEFORE_PUBLISHED }) - expect(body.total).to.equal(0) - - const blacklistedVideos = body.data - expect(blacklistedVideos).to.be.an('array') - expect(blacklistedVideos.length).to.equal(0) - }) - - it('Should get the correct sort when sorting by descending id', async function () { - const body = await command.list({ sort: '-id' }) - expect(body.total).to.equal(2) - - const blacklistedVideos = body.data - expect(blacklistedVideos).to.be.an('array') - expect(blacklistedVideos.length).to.equal(2) - - const result = [ ...body.data ].sort(sortObjectComparator('id', 'desc')) - expect(blacklistedVideos).to.deep.equal(result) - }) - - it('Should get the correct sort when sorting by descending video name', async function () { - const body = await command.list({ sort: '-name' }) - expect(body.total).to.equal(2) - - const blacklistedVideos = body.data - expect(blacklistedVideos).to.be.an('array') - expect(blacklistedVideos.length).to.equal(2) - - const result = [ ...body.data ].sort(sortObjectComparator('name', 'desc')) - expect(blacklistedVideos).to.deep.equal(result) - }) - - it('Should get the correct sort when sorting by ascending creation date', async function () { - const body = await command.list({ sort: 'createdAt' }) - expect(body.total).to.equal(2) - - const blacklistedVideos = body.data - expect(blacklistedVideos).to.be.an('array') - expect(blacklistedVideos.length).to.equal(2) - - const result = [ ...body.data ].sort(sortObjectComparator('createdAt', 'asc')) - expect(blacklistedVideos).to.deep.equal(result) - }) - }) - - describe('When updating blacklisted videos', function () { - it('Should change the reason', async function () { - await command.update({ videoId, reason: 'my super reason updated' }) - - const body = await command.list({ sort: '-name' }) - const video = body.data.find(b => b.video.id === videoId) - - expect(video.reason).to.equal('my super reason updated') - }) - }) - - describe('When listing my videos', function () { - it('Should display blacklisted videos', async function () { - await blacklistVideosOnServer(servers[1]) - - const { total, data } = await servers[1].videos.listMyVideos() - - expect(total).to.equal(2) - expect(data).to.have.lengthOf(2) - - for (const video of data) { - expect(video.blacklisted).to.be.true - expect(video.blacklistedReason).to.equal('super reason') - } - }) - }) - - describe('When removing a blacklisted video', function () { - let videoToRemove: VideoBlacklist - let blacklist = [] - - it('Should not have any video in videos list on server 1', async function () { - const { total, data } = await servers[0].videos.list() - expect(total).to.equal(0) - expect(data).to.be.an('array') - expect(data.length).to.equal(0) - }) - - it('Should remove a video from the blacklist on server 1', async function () { - // Get one video in the blacklist - const body = await command.list({ sort: '-name' }) - videoToRemove = body.data[0] - blacklist = body.data.slice(1) - - // Remove it - await command.remove({ videoId: videoToRemove.video.id }) - }) - - it('Should have the ex-blacklisted video in videos list on server 1', async function () { - const { total, data } = await servers[0].videos.list() - expect(total).to.equal(1) - - expect(data).to.be.an('array') - expect(data.length).to.equal(1) - - expect(data[0].name).to.equal(videoToRemove.video.name) - expect(data[0].id).to.equal(videoToRemove.video.id) - }) - - it('Should not have the ex-blacklisted video in videos blacklist list on server 1', async function () { - const body = await command.list({ sort: '-name' }) - expect(body.total).to.equal(1) - - const videos = body.data - expect(videos).to.be.an('array') - expect(videos.length).to.equal(1) - expect(videos).to.deep.equal(blacklist) - }) - }) - - describe('When blacklisting local videos', function () { - let video3UUID: string - let video4UUID: string - - before(async function () { - { - const { uuid } = await servers[0].videos.upload({ attributes: { name: 'Video 3' } }) - video3UUID = uuid - } - { - const { uuid } = await servers[0].videos.upload({ attributes: { name: 'Video 4' } }) - video4UUID = uuid - } - - await waitJobs(servers) - }) - - it('Should blacklist video 3 and keep it federated', async function () { - await command.add({ videoId: video3UUID, reason: 'super reason', unfederate: false }) - - await waitJobs(servers) - - { - const { data } = await servers[0].videos.list() - expect(data.find(v => v.uuid === video3UUID)).to.be.undefined - } - - { - const { data } = await servers[1].videos.list() - expect(data.find(v => v.uuid === video3UUID)).to.not.be.undefined - } - }) - - it('Should unfederate the video', async function () { - await command.add({ videoId: video4UUID, reason: 'super reason', unfederate: true }) - - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.videos.list() - expect(data.find(v => v.uuid === video4UUID)).to.be.undefined - } - }) - - it('Should have the video unfederated even after an Update AP message', async function () { - await servers[0].videos.update({ id: video4UUID, attributes: { description: 'super description' } }) - - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.videos.list() - expect(data.find(v => v.uuid === video4UUID)).to.be.undefined - } - }) - - it('Should have the correct video blacklist unfederate attribute', async function () { - const body = await command.list({ sort: 'createdAt' }) - - const blacklistedVideos = body.data - const video3Blacklisted = blacklistedVideos.find(b => b.video.uuid === video3UUID) - const video4Blacklisted = blacklistedVideos.find(b => b.video.uuid === video4UUID) - - expect(video3Blacklisted.unfederated).to.be.false - expect(video4Blacklisted.unfederated).to.be.true - }) - - it('Should remove the video from blacklist and refederate the video', async function () { - await command.remove({ videoId: video4UUID }) - - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.videos.list() - expect(data.find(v => v.uuid === video4UUID)).to.not.be.undefined - } - }) - - }) - - describe('When auto blacklist videos', function () { - let userWithoutFlag: string - let userWithFlag: string - let channelOfUserWithoutFlag: number - - before(async function () { - this.timeout(20000) - - await killallServers([ servers[0] ]) - - const config = { - auto_blacklist: { - videos: { - of_users: { - enabled: true - } - } - } - } - await servers[0].run(config) - - { - const user = { username: 'user_without_flag', password: 'password' } - await servers[0].users.create({ - username: user.username, - adminFlags: UserAdminFlag.NONE, - password: user.password, - role: UserRole.USER - }) - - userWithoutFlag = await servers[0].login.getAccessToken(user) - - const { videoChannels } = await servers[0].users.getMyInfo({ token: userWithoutFlag }) - channelOfUserWithoutFlag = videoChannels[0].id - } - - { - const user = { username: 'user_with_flag', password: 'password' } - await servers[0].users.create({ - username: user.username, - adminFlags: UserAdminFlag.BYPASS_VIDEO_AUTO_BLACKLIST, - password: user.password, - role: UserRole.USER - }) - - userWithFlag = await servers[0].login.getAccessToken(user) - } - - await waitJobs(servers) - }) - - it('Should auto blacklist a video on upload', async function () { - await servers[0].videos.upload({ token: userWithoutFlag, attributes: { name: 'blacklisted' } }) - - const body = await command.list({ type: VideoBlacklistType.AUTO_BEFORE_PUBLISHED }) - expect(body.total).to.equal(1) - expect(body.data[0].video.name).to.equal('blacklisted') - }) - - it('Should auto blacklist a video on URL import', async function () { - this.timeout(15000) - - const attributes = { - targetUrl: FIXTURE_URLS.goodVideo, - name: 'URL import', - channelId: channelOfUserWithoutFlag - } - await servers[0].imports.importVideo({ token: userWithoutFlag, attributes }) - - const body = await command.list({ sort: 'createdAt', type: VideoBlacklistType.AUTO_BEFORE_PUBLISHED }) - expect(body.total).to.equal(2) - expect(body.data[1].video.name).to.equal('URL import') - }) - - it('Should auto blacklist a video on torrent import', async function () { - const attributes = { - magnetUri: FIXTURE_URLS.magnet, - name: 'Torrent import', - channelId: channelOfUserWithoutFlag - } - await servers[0].imports.importVideo({ token: userWithoutFlag, attributes }) - - const body = await command.list({ sort: 'createdAt', type: VideoBlacklistType.AUTO_BEFORE_PUBLISHED }) - expect(body.total).to.equal(3) - expect(body.data[2].video.name).to.equal('Torrent import') - }) - - it('Should not auto blacklist a video on upload if the user has the bypass blacklist flag', async function () { - await servers[0].videos.upload({ token: userWithFlag, attributes: { name: 'not blacklisted' } }) - - const body = await command.list({ type: VideoBlacklistType.AUTO_BEFORE_PUBLISHED }) - expect(body.total).to.equal(3) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/notifications/admin-notifications.ts b/server/tests/api/notifications/admin-notifications.ts deleted file mode 100644 index 4824542c9..000000000 --- a/server/tests/api/notifications/admin-notifications.ts +++ /dev/null @@ -1,159 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { - CheckerBaseParams, - checkNewPeerTubeVersion, - checkNewPluginVersion, - MockJoinPeerTubeVersions, - MockSmtpServer, - prepareNotificationsTest, - SQLCommand -} from '@server/tests/shared' -import { wait } from '@shared/core-utils' -import { PluginType, UserNotification, UserNotificationType } from '@shared/models' -import { cleanupTests, PeerTubeServer } from '@shared/server-commands' - -describe('Test admin notifications', function () { - let server: PeerTubeServer - let sqlCommand: SQLCommand - let userNotifications: UserNotification[] = [] - let adminNotifications: UserNotification[] = [] - let emails: object[] = [] - let baseParams: CheckerBaseParams - let joinPeerTubeServer: MockJoinPeerTubeVersions - - before(async function () { - this.timeout(120000) - - joinPeerTubeServer = new MockJoinPeerTubeVersions() - const port = await joinPeerTubeServer.initialize() - - const config = { - peertube: { - check_latest_version: { - enabled: true, - url: `http://127.0.0.1:${port}/versions.json` - } - }, - plugins: { - index: { - enabled: true, - check_latest_versions_interval: '3 seconds' - } - } - } - - const res = await prepareNotificationsTest(1, config) - emails = res.emails - server = res.servers[0] - - userNotifications = res.userNotifications - adminNotifications = res.adminNotifications - - baseParams = { - server, - emails, - socketNotifications: adminNotifications, - token: server.accessToken - } - - await server.plugins.install({ npmName: 'peertube-plugin-hello-world' }) - await server.plugins.install({ npmName: 'peertube-theme-background-red' }) - - sqlCommand = new SQLCommand(server) - }) - - describe('Latest PeerTube version notification', function () { - - it('Should not send a notification to admins if there is no new version', async function () { - this.timeout(30000) - - joinPeerTubeServer.setLatestVersion('1.4.2') - - await wait(3000) - await checkNewPeerTubeVersion({ ...baseParams, latestVersion: '1.4.2', checkType: 'absence' }) - }) - - it('Should send a notification to admins on new version', async function () { - this.timeout(30000) - - joinPeerTubeServer.setLatestVersion('15.4.2') - - await wait(3000) - await checkNewPeerTubeVersion({ ...baseParams, latestVersion: '15.4.2', checkType: 'presence' }) - }) - - it('Should not send the same notification to admins', async function () { - this.timeout(30000) - - await wait(3000) - expect(adminNotifications.filter(n => n.type === UserNotificationType.NEW_PEERTUBE_VERSION)).to.have.lengthOf(1) - }) - - it('Should not have sent a notification to users', async function () { - this.timeout(30000) - - expect(userNotifications.filter(n => n.type === UserNotificationType.NEW_PEERTUBE_VERSION)).to.have.lengthOf(0) - }) - - it('Should send a new notification after a new release', async function () { - this.timeout(30000) - - joinPeerTubeServer.setLatestVersion('15.4.3') - - await wait(3000) - await checkNewPeerTubeVersion({ ...baseParams, latestVersion: '15.4.3', checkType: 'presence' }) - expect(adminNotifications.filter(n => n.type === UserNotificationType.NEW_PEERTUBE_VERSION)).to.have.lengthOf(2) - }) - }) - - describe('Latest plugin version notification', function () { - - it('Should not send a notification to admins if there is no new plugin version', async function () { - this.timeout(30000) - - await wait(6000) - await checkNewPluginVersion({ ...baseParams, pluginType: PluginType.PLUGIN, pluginName: 'hello-world', checkType: 'absence' }) - }) - - it('Should send a notification to admins on new plugin version', async function () { - this.timeout(30000) - - await sqlCommand.setPluginVersion('hello-world', '0.0.1') - await sqlCommand.setPluginLatestVersion('hello-world', '0.0.1') - await wait(6000) - - await checkNewPluginVersion({ ...baseParams, pluginType: PluginType.PLUGIN, pluginName: 'hello-world', checkType: 'presence' }) - }) - - it('Should not send the same notification to admins', async function () { - this.timeout(30000) - - await wait(6000) - - expect(adminNotifications.filter(n => n.type === UserNotificationType.NEW_PLUGIN_VERSION)).to.have.lengthOf(1) - }) - - it('Should not have sent a notification to users', async function () { - expect(userNotifications.filter(n => n.type === UserNotificationType.NEW_PLUGIN_VERSION)).to.have.lengthOf(0) - }) - - it('Should send a new notification after a new plugin release', async function () { - this.timeout(30000) - - await sqlCommand.setPluginVersion('hello-world', '0.0.1') - await sqlCommand.setPluginLatestVersion('hello-world', '0.0.1') - await wait(6000) - - expect(adminNotifications.filter(n => n.type === UserNotificationType.NEW_PEERTUBE_VERSION)).to.have.lengthOf(2) - }) - }) - - after(async function () { - MockSmtpServer.Instance.kill() - - await sqlCommand.cleanup() - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/notifications/comments-notifications.ts b/server/tests/api/notifications/comments-notifications.ts deleted file mode 100644 index 0a4bfc5e4..000000000 --- a/server/tests/api/notifications/comments-notifications.ts +++ /dev/null @@ -1,305 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { - checkCommentMention, - CheckerBaseParams, - checkNewCommentOnMyVideo, - MockSmtpServer, - prepareNotificationsTest -} from '@server/tests/shared' -import { UserNotification } from '@shared/models' -import { cleanupTests, PeerTubeServer, waitJobs } from '@shared/server-commands' - -describe('Test comments notifications', function () { - let servers: PeerTubeServer[] = [] - let userToken: string - let userNotifications: UserNotification[] = [] - let emails: object[] = [] - - const commentText = '**hello** world,

what do you think about peertube?

' - const expectedHtml = 'hello world' + - ',

what do you think about peertube?' - - before(async function () { - this.timeout(120000) - - const res = await prepareNotificationsTest(2) - emails = res.emails - userToken = res.userAccessToken - servers = res.servers - userNotifications = res.userNotifications - }) - - describe('Comment on my video notifications', function () { - let baseParams: CheckerBaseParams - - before(() => { - baseParams = { - server: servers[0], - emails, - socketNotifications: userNotifications, - token: userToken - } - }) - - it('Should not send a new comment notification after a comment on another video', async function () { - this.timeout(30000) - - const { uuid, shortUUID } = await servers[0].videos.upload({ attributes: { name: 'super video' } }) - - const created = await servers[0].comments.createThread({ videoId: uuid, text: 'comment' }) - const commentId = created.id - - await waitJobs(servers) - await checkNewCommentOnMyVideo({ ...baseParams, shortUUID, threadId: commentId, commentId, checkType: 'absence' }) - }) - - it('Should not send a new comment notification if I comment my own video', async function () { - this.timeout(30000) - - const { uuid, shortUUID } = await servers[0].videos.upload({ token: userToken, attributes: { name: 'super video' } }) - - const created = await servers[0].comments.createThread({ token: userToken, videoId: uuid, text: 'comment' }) - const commentId = created.id - - await waitJobs(servers) - await checkNewCommentOnMyVideo({ ...baseParams, shortUUID, threadId: commentId, commentId, checkType: 'absence' }) - }) - - it('Should not send a new comment notification if the account is muted', async function () { - this.timeout(30000) - - await servers[0].blocklist.addToMyBlocklist({ token: userToken, account: 'root' }) - - const { uuid, shortUUID } = await servers[0].videos.upload({ token: userToken, attributes: { name: 'super video' } }) - - const created = await servers[0].comments.createThread({ videoId: uuid, text: 'comment' }) - const commentId = created.id - - await waitJobs(servers) - await checkNewCommentOnMyVideo({ ...baseParams, shortUUID, threadId: commentId, commentId, checkType: 'absence' }) - - await servers[0].blocklist.removeFromMyBlocklist({ token: userToken, account: 'root' }) - }) - - it('Should send a new comment notification after a local comment on my video', async function () { - this.timeout(30000) - - const { uuid, shortUUID } = await servers[0].videos.upload({ token: userToken, attributes: { name: 'super video' } }) - - const created = await servers[0].comments.createThread({ videoId: uuid, text: 'comment' }) - const commentId = created.id - - await waitJobs(servers) - await checkNewCommentOnMyVideo({ ...baseParams, shortUUID, threadId: commentId, commentId, checkType: 'presence' }) - }) - - it('Should send a new comment notification after a remote comment on my video', async function () { - this.timeout(30000) - - const { uuid, shortUUID } = await servers[0].videos.upload({ token: userToken, attributes: { name: 'super video' } }) - - await waitJobs(servers) - - await servers[1].comments.createThread({ videoId: uuid, text: 'comment' }) - - await waitJobs(servers) - - const { data } = await servers[0].comments.listThreads({ videoId: uuid }) - expect(data).to.have.lengthOf(1) - - const commentId = data[0].id - await checkNewCommentOnMyVideo({ ...baseParams, shortUUID, threadId: commentId, commentId, checkType: 'presence' }) - }) - - it('Should send a new comment notification after a local reply on my video', async function () { - this.timeout(30000) - - const { uuid, shortUUID } = await servers[0].videos.upload({ token: userToken, attributes: { name: 'super video' } }) - - const { id: threadId } = await servers[0].comments.createThread({ videoId: uuid, text: 'comment' }) - - const { id: commentId } = await servers[0].comments.addReply({ videoId: uuid, toCommentId: threadId, text: 'reply' }) - - await waitJobs(servers) - await checkNewCommentOnMyVideo({ ...baseParams, shortUUID, threadId, commentId, checkType: 'presence' }) - }) - - it('Should send a new comment notification after a remote reply on my video', async function () { - this.timeout(30000) - - const { uuid, shortUUID } = await servers[0].videos.upload({ token: userToken, attributes: { name: 'super video' } }) - await waitJobs(servers) - - { - const created = await servers[1].comments.createThread({ videoId: uuid, text: 'comment' }) - const threadId = created.id - await servers[1].comments.addReply({ videoId: uuid, toCommentId: threadId, text: 'reply' }) - } - - await waitJobs(servers) - - const { data } = await servers[0].comments.listThreads({ videoId: uuid }) - expect(data).to.have.lengthOf(1) - - const threadId = data[0].id - const tree = await servers[0].comments.getThread({ videoId: uuid, threadId }) - - expect(tree.children).to.have.lengthOf(1) - const commentId = tree.children[0].comment.id - - await checkNewCommentOnMyVideo({ ...baseParams, shortUUID, threadId, commentId, checkType: 'presence' }) - }) - - it('Should convert markdown in comment to html', async function () { - this.timeout(30000) - - const { uuid } = await servers[0].videos.upload({ token: userToken, attributes: { name: 'cool video' } }) - - await servers[0].comments.createThread({ videoId: uuid, text: commentText }) - - await waitJobs(servers) - - const latestEmail = emails[emails.length - 1] - expect(latestEmail['html']).to.contain(expectedHtml) - }) - }) - - describe('Mention notifications', function () { - let baseParams: CheckerBaseParams - const byAccountDisplayName = 'super root name' - - before(async function () { - baseParams = { - server: servers[0], - emails, - socketNotifications: userNotifications, - token: userToken - } - - await servers[0].users.updateMe({ displayName: 'super root name' }) - await servers[1].users.updateMe({ displayName: 'super root 2 name' }) - }) - - it('Should not send a new mention comment notification if I mention the video owner', async function () { - this.timeout(30000) - - const { uuid, shortUUID } = await servers[0].videos.upload({ token: userToken, attributes: { name: 'super video' } }) - - const { id: commentId } = await servers[0].comments.createThread({ videoId: uuid, text: '@user_1 hello' }) - - await waitJobs(servers) - await checkCommentMention({ ...baseParams, shortUUID, threadId: commentId, commentId, byAccountDisplayName, checkType: 'absence' }) - }) - - it('Should not send a new mention comment notification if I mention myself', async function () { - this.timeout(30000) - - const { uuid, shortUUID } = await servers[0].videos.upload({ attributes: { name: 'super video' } }) - - const { id: commentId } = await servers[0].comments.createThread({ token: userToken, videoId: uuid, text: '@user_1 hello' }) - - await waitJobs(servers) - await checkCommentMention({ ...baseParams, shortUUID, threadId: commentId, commentId, byAccountDisplayName, checkType: 'absence' }) - }) - - it('Should not send a new mention notification if the account is muted', async function () { - this.timeout(30000) - - await servers[0].blocklist.addToMyBlocklist({ token: userToken, account: 'root' }) - - const { uuid, shortUUID } = await servers[0].videos.upload({ attributes: { name: 'super video' } }) - - const { id: commentId } = await servers[0].comments.createThread({ videoId: uuid, text: '@user_1 hello' }) - - await waitJobs(servers) - await checkCommentMention({ ...baseParams, shortUUID, threadId: commentId, commentId, byAccountDisplayName, checkType: 'absence' }) - - await servers[0].blocklist.removeFromMyBlocklist({ token: userToken, account: 'root' }) - }) - - it('Should not send a new mention notification if the remote account mention a local account', async function () { - this.timeout(30000) - - const { uuid, shortUUID } = await servers[0].videos.upload({ attributes: { name: 'super video' } }) - - await waitJobs(servers) - const { id: threadId } = await servers[1].comments.createThread({ videoId: uuid, text: '@user_1 hello' }) - - await waitJobs(servers) - - const byAccountDisplayName = 'super root 2 name' - await checkCommentMention({ ...baseParams, shortUUID, threadId, commentId: threadId, byAccountDisplayName, checkType: 'absence' }) - }) - - it('Should send a new mention notification after local comments', async function () { - this.timeout(30000) - - const { uuid, shortUUID } = await servers[0].videos.upload({ attributes: { name: 'super video' } }) - - const { id: threadId } = await servers[0].comments.createThread({ videoId: uuid, text: '@user_1 hellotext: 1' }) - - await waitJobs(servers) - await checkCommentMention({ ...baseParams, shortUUID, threadId, commentId: threadId, byAccountDisplayName, checkType: 'presence' }) - - const { id: commentId } = await servers[0].comments.addReply({ videoId: uuid, toCommentId: threadId, text: 'hello 2 @user_1' }) - - await waitJobs(servers) - await checkCommentMention({ ...baseParams, shortUUID, commentId, threadId, byAccountDisplayName, checkType: 'presence' }) - }) - - it('Should send a new mention notification after remote comments', async function () { - this.timeout(30000) - - const { uuid, shortUUID } = await servers[0].videos.upload({ attributes: { name: 'super video' } }) - - await waitJobs(servers) - - const text1 = `hello @user_1@${servers[0].host} 1` - const { id: server2ThreadId } = await servers[1].comments.createThread({ videoId: uuid, text: text1 }) - - await waitJobs(servers) - - const { data } = await servers[0].comments.listThreads({ videoId: uuid }) - expect(data).to.have.lengthOf(1) - - const byAccountDisplayName = 'super root 2 name' - const threadId = data[0].id - await checkCommentMention({ ...baseParams, shortUUID, commentId: threadId, threadId, byAccountDisplayName, checkType: 'presence' }) - - const text2 = `@user_1@${servers[0].host} hello 2 @root@${servers[0].host}` - await servers[1].comments.addReply({ videoId: uuid, toCommentId: server2ThreadId, text: text2 }) - - await waitJobs(servers) - - const tree = await servers[0].comments.getThread({ videoId: uuid, threadId }) - - expect(tree.children).to.have.lengthOf(1) - const commentId = tree.children[0].comment.id - - await checkCommentMention({ ...baseParams, shortUUID, commentId, threadId, byAccountDisplayName, checkType: 'presence' }) - }) - - it('Should convert markdown in comment to html', async function () { - this.timeout(30000) - - const { uuid } = await servers[0].videos.upload({ attributes: { name: 'super video' } }) - - const { id: threadId } = await servers[0].comments.createThread({ videoId: uuid, text: '@user_1 hello 1' }) - - await servers[0].comments.addReply({ videoId: uuid, toCommentId: threadId, text: '@user_1 ' + commentText }) - - await waitJobs(servers) - - const latestEmail = emails[emails.length - 1] - expect(latestEmail['html']).to.contain(expectedHtml) - }) - }) - - after(async function () { - MockSmtpServer.Instance.kill() - - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/notifications/index.ts b/server/tests/api/notifications/index.ts deleted file mode 100644 index c0216b74f..000000000 --- a/server/tests/api/notifications/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -import './admin-notifications' -import './comments-notifications' -import './moderation-notifications' -import './notifications-api' -import './registrations-notifications' -import './user-notifications' diff --git a/server/tests/api/notifications/moderation-notifications.ts b/server/tests/api/notifications/moderation-notifications.ts deleted file mode 100644 index e7a5c47e9..000000000 --- a/server/tests/api/notifications/moderation-notifications.ts +++ /dev/null @@ -1,609 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { - checkAbuseStateChange, - checkAutoInstanceFollowing, - CheckerBaseParams, - checkNewAbuseMessage, - checkNewAccountAbuseForModerators, - checkNewBlacklistOnMyVideo, - checkNewCommentAbuseForModerators, - checkNewInstanceFollower, - checkNewVideoAbuseForModerators, - checkNewVideoFromSubscription, - checkVideoAutoBlacklistForModerators, - checkVideoIsPublished, - MockInstancesIndex, - MockSmtpServer, - prepareNotificationsTest -} from '@server/tests/shared' -import { wait } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { AbuseState, CustomConfig, UserNotification, UserRole, VideoPrivacy } from '@shared/models' -import { cleanupTests, PeerTubeServer, waitJobs } from '@shared/server-commands' - -describe('Test moderation notifications', function () { - let servers: PeerTubeServer[] = [] - let userToken1: string - let userToken2: string - - let userNotifications: UserNotification[] = [] - let adminNotifications: UserNotification[] = [] - let adminNotificationsServer2: UserNotification[] = [] - let emails: object[] = [] - - before(async function () { - this.timeout(120000) - - const res = await prepareNotificationsTest(3) - emails = res.emails - userToken1 = res.userAccessToken - servers = res.servers - userNotifications = res.userNotifications - adminNotifications = res.adminNotifications - adminNotificationsServer2 = res.adminNotificationsServer2 - - userToken2 = await servers[1].users.generateUserAndToken('user2', UserRole.USER) - }) - - describe('Abuse for moderators notification', function () { - let baseParams: CheckerBaseParams - - before(() => { - baseParams = { - server: servers[0], - emails, - socketNotifications: adminNotifications, - token: servers[0].accessToken - } - }) - - it('Should not send a notification to moderators on local abuse reported by an admin', async function () { - this.timeout(50000) - - const name = 'video for abuse ' + buildUUID() - const video = await servers[0].videos.upload({ token: userToken1, attributes: { name } }) - - await servers[0].abuses.report({ videoId: video.id, reason: 'super reason' }) - - await waitJobs(servers) - await checkNewVideoAbuseForModerators({ ...baseParams, shortUUID: video.shortUUID, videoName: name, checkType: 'absence' }) - }) - - it('Should send a notification to moderators on local video abuse', async function () { - this.timeout(50000) - - const name = 'video for abuse ' + buildUUID() - const video = await servers[0].videos.upload({ token: userToken1, attributes: { name } }) - - await servers[0].abuses.report({ token: userToken1, videoId: video.id, reason: 'super reason' }) - - await waitJobs(servers) - await checkNewVideoAbuseForModerators({ ...baseParams, shortUUID: video.shortUUID, videoName: name, checkType: 'presence' }) - }) - - it('Should send a notification to moderators on remote video abuse', async function () { - this.timeout(50000) - - const name = 'video for abuse ' + buildUUID() - const video = await servers[0].videos.upload({ token: userToken1, attributes: { name } }) - - await waitJobs(servers) - - const videoId = await servers[1].videos.getId({ uuid: video.uuid }) - await servers[1].abuses.report({ token: userToken2, videoId, reason: 'super reason' }) - - await waitJobs(servers) - await checkNewVideoAbuseForModerators({ ...baseParams, shortUUID: video.shortUUID, videoName: name, checkType: 'presence' }) - }) - - it('Should send a notification to moderators on local comment abuse', async function () { - this.timeout(50000) - - const name = 'video for abuse ' + buildUUID() - const video = await servers[0].videos.upload({ token: userToken1, attributes: { name } }) - const comment = await servers[0].comments.createThread({ - token: userToken1, - videoId: video.id, - text: 'comment abuse ' + buildUUID() - }) - - await waitJobs(servers) - - await servers[0].abuses.report({ token: userToken1, commentId: comment.id, reason: 'super reason' }) - - await waitJobs(servers) - await checkNewCommentAbuseForModerators({ ...baseParams, shortUUID: video.shortUUID, videoName: name, checkType: 'presence' }) - }) - - it('Should send a notification to moderators on remote comment abuse', async function () { - this.timeout(50000) - - const name = 'video for abuse ' + buildUUID() - const video = await servers[0].videos.upload({ token: userToken1, attributes: { name } }) - - await servers[0].comments.createThread({ - token: userToken1, - videoId: video.id, - text: 'comment abuse ' + buildUUID() - }) - - await waitJobs(servers) - - const { data } = await servers[1].comments.listThreads({ videoId: video.uuid }) - const commentId = data[0].id - await servers[1].abuses.report({ token: userToken2, commentId, reason: 'super reason' }) - - await waitJobs(servers) - await checkNewCommentAbuseForModerators({ ...baseParams, shortUUID: video.shortUUID, videoName: name, checkType: 'presence' }) - }) - - it('Should send a notification to moderators on local account abuse', async function () { - this.timeout(50000) - - const username = 'user' + new Date().getTime() - const { account } = await servers[0].users.create({ username, password: 'donald' }) - const accountId = account.id - - await servers[0].abuses.report({ token: userToken1, accountId, reason: 'super reason' }) - - await waitJobs(servers) - await checkNewAccountAbuseForModerators({ ...baseParams, displayName: username, checkType: 'presence' }) - }) - - it('Should send a notification to moderators on remote account abuse', async function () { - this.timeout(50000) - - const username = 'user' + new Date().getTime() - const tmpToken = await servers[0].users.generateUserAndToken(username) - await servers[0].videos.upload({ token: tmpToken, attributes: { name: 'super video' } }) - - await waitJobs(servers) - - const account = await servers[1].accounts.get({ accountName: username + '@' + servers[0].host }) - await servers[1].abuses.report({ token: userToken2, accountId: account.id, reason: 'super reason' }) - - await waitJobs(servers) - await checkNewAccountAbuseForModerators({ ...baseParams, displayName: username, checkType: 'presence' }) - }) - }) - - describe('Abuse state change notification', function () { - let baseParams: CheckerBaseParams - let abuseId: number - - before(async function () { - baseParams = { - server: servers[0], - emails, - socketNotifications: userNotifications, - token: userToken1 - } - - const name = 'abuse ' + buildUUID() - const video = await servers[0].videos.upload({ token: userToken1, attributes: { name } }) - - const body = await servers[0].abuses.report({ token: userToken1, videoId: video.id, reason: 'super reason' }) - abuseId = body.abuse.id - }) - - it('Should send a notification to reporter if the abuse has been accepted', async function () { - this.timeout(30000) - - await servers[0].abuses.update({ abuseId, body: { state: AbuseState.ACCEPTED } }) - await waitJobs(servers) - - await checkAbuseStateChange({ ...baseParams, abuseId, state: AbuseState.ACCEPTED, checkType: 'presence' }) - }) - - it('Should send a notification to reporter if the abuse has been rejected', async function () { - this.timeout(30000) - - await servers[0].abuses.update({ abuseId, body: { state: AbuseState.REJECTED } }) - await waitJobs(servers) - - await checkAbuseStateChange({ ...baseParams, abuseId, state: AbuseState.REJECTED, checkType: 'presence' }) - }) - }) - - describe('New abuse message notification', function () { - let baseParamsUser: CheckerBaseParams - let baseParamsAdmin: CheckerBaseParams - let abuseId: number - let abuseId2: number - - before(async function () { - baseParamsUser = { - server: servers[0], - emails, - socketNotifications: userNotifications, - token: userToken1 - } - - baseParamsAdmin = { - server: servers[0], - emails, - socketNotifications: adminNotifications, - token: servers[0].accessToken - } - - const name = 'abuse ' + buildUUID() - const video = await servers[0].videos.upload({ token: userToken1, attributes: { name } }) - - { - const body = await servers[0].abuses.report({ token: userToken1, videoId: video.id, reason: 'super reason' }) - abuseId = body.abuse.id - } - - { - const body = await servers[0].abuses.report({ token: userToken1, videoId: video.id, reason: 'super reason 2' }) - abuseId2 = body.abuse.id - } - }) - - it('Should send a notification to reporter on new message', async function () { - this.timeout(30000) - - const message = 'my super message to users' - await servers[0].abuses.addMessage({ abuseId, message }) - await waitJobs(servers) - - await checkNewAbuseMessage({ ...baseParamsUser, abuseId, message, toEmail: 'user_1@example.com', checkType: 'presence' }) - }) - - it('Should not send a notification to the admin if sent by the admin', async function () { - this.timeout(30000) - - const message = 'my super message that should not be sent to the admin' - await servers[0].abuses.addMessage({ abuseId, message }) - await waitJobs(servers) - - const toEmail = 'admin' + servers[0].internalServerNumber + '@example.com' - await checkNewAbuseMessage({ ...baseParamsAdmin, abuseId, message, toEmail, checkType: 'absence' }) - }) - - it('Should send a notification to moderators', async function () { - this.timeout(30000) - - const message = 'my super message to moderators' - await servers[0].abuses.addMessage({ token: userToken1, abuseId: abuseId2, message }) - await waitJobs(servers) - - const toEmail = 'admin' + servers[0].internalServerNumber + '@example.com' - await checkNewAbuseMessage({ ...baseParamsAdmin, abuseId: abuseId2, message, toEmail, checkType: 'presence' }) - }) - - it('Should not send a notification to reporter if sent by the reporter', async function () { - this.timeout(30000) - - const message = 'my super message that should not be sent to reporter' - await servers[0].abuses.addMessage({ token: userToken1, abuseId: abuseId2, message }) - await waitJobs(servers) - - const toEmail = 'user_1@example.com' - await checkNewAbuseMessage({ ...baseParamsUser, abuseId: abuseId2, message, toEmail, checkType: 'absence' }) - }) - }) - - describe('Video blacklist on my video', function () { - let baseParams: CheckerBaseParams - - before(() => { - baseParams = { - server: servers[0], - emails, - socketNotifications: userNotifications, - token: userToken1 - } - }) - - it('Should send a notification to video owner on blacklist', async function () { - this.timeout(30000) - - const name = 'video for abuse ' + buildUUID() - const { uuid, shortUUID } = await servers[0].videos.upload({ token: userToken1, attributes: { name } }) - - await servers[0].blacklist.add({ videoId: uuid }) - - await waitJobs(servers) - await checkNewBlacklistOnMyVideo({ ...baseParams, shortUUID, videoName: name, blacklistType: 'blacklist' }) - }) - - it('Should send a notification to video owner on unblacklist', async function () { - this.timeout(30000) - - const name = 'video for abuse ' + buildUUID() - const { uuid, shortUUID } = await servers[0].videos.upload({ token: userToken1, attributes: { name } }) - - await servers[0].blacklist.add({ videoId: uuid }) - - await waitJobs(servers) - await servers[0].blacklist.remove({ videoId: uuid }) - await waitJobs(servers) - - await wait(500) - await checkNewBlacklistOnMyVideo({ ...baseParams, shortUUID, videoName: name, blacklistType: 'unblacklist' }) - }) - }) - - describe('New instance follows', function () { - const instanceIndexServer = new MockInstancesIndex() - let config: any - let baseParams: CheckerBaseParams - - before(async function () { - baseParams = { - server: servers[0], - emails, - socketNotifications: adminNotifications, - token: servers[0].accessToken - } - - const port = await instanceIndexServer.initialize() - instanceIndexServer.addInstance(servers[1].host) - - config = { - followings: { - instance: { - autoFollowIndex: { - indexUrl: `http://127.0.0.1:${port}/api/v1/instances/hosts`, - enabled: true - } - } - } - } - }) - - it('Should send a notification only to admin when there is a new instance follower', async function () { - this.timeout(60000) - - await servers[2].follows.follow({ hosts: [ servers[0].url ] }) - - await waitJobs(servers) - - await checkNewInstanceFollower({ ...baseParams, followerHost: servers[2].host, checkType: 'presence' }) - - const userOverride = { socketNotifications: userNotifications, token: userToken1, check: { web: true, mail: false } } - await checkNewInstanceFollower({ ...baseParams, ...userOverride, followerHost: servers[2].host, checkType: 'absence' }) - }) - - it('Should send a notification on auto follow back', async function () { - this.timeout(40000) - - await servers[2].follows.unfollow({ target: servers[0] }) - await waitJobs(servers) - - const config = { - followings: { - instance: { - autoFollowBack: { enabled: true } - } - } - } - await servers[0].config.updateCustomSubConfig({ newConfig: config }) - - await servers[2].follows.follow({ hosts: [ servers[0].url ] }) - - await waitJobs(servers) - - const followerHost = servers[0].host - const followingHost = servers[2].host - await checkAutoInstanceFollowing({ ...baseParams, followerHost, followingHost, checkType: 'presence' }) - - const userOverride = { socketNotifications: userNotifications, token: userToken1, check: { web: true, mail: false } } - await checkAutoInstanceFollowing({ ...baseParams, ...userOverride, followerHost, followingHost, checkType: 'absence' }) - - config.followings.instance.autoFollowBack.enabled = false - await servers[0].config.updateCustomSubConfig({ newConfig: config }) - await servers[0].follows.unfollow({ target: servers[2] }) - await servers[2].follows.unfollow({ target: servers[0] }) - }) - - it('Should send a notification on auto instances index follow', async function () { - this.timeout(30000) - await servers[0].follows.unfollow({ target: servers[1] }) - - await servers[0].config.updateCustomSubConfig({ newConfig: config }) - - await wait(5000) - await waitJobs(servers) - - const followerHost = servers[0].host - const followingHost = servers[1].host - await checkAutoInstanceFollowing({ ...baseParams, followerHost, followingHost, checkType: 'presence' }) - - config.followings.instance.autoFollowIndex.enabled = false - await servers[0].config.updateCustomSubConfig({ newConfig: config }) - await servers[0].follows.unfollow({ target: servers[1] }) - }) - }) - - describe('Video-related notifications when video auto-blacklist is enabled', function () { - let userBaseParams: CheckerBaseParams - let adminBaseParamsServer1: CheckerBaseParams - let adminBaseParamsServer2: CheckerBaseParams - let uuid: string - let shortUUID: string - let videoName: string - let currentCustomConfig: CustomConfig - - before(async function () { - - adminBaseParamsServer1 = { - server: servers[0], - emails, - socketNotifications: adminNotifications, - token: servers[0].accessToken - } - - adminBaseParamsServer2 = { - server: servers[1], - emails, - socketNotifications: adminNotificationsServer2, - token: servers[1].accessToken - } - - userBaseParams = { - server: servers[0], - emails, - socketNotifications: userNotifications, - token: userToken1 - } - - currentCustomConfig = await servers[0].config.getCustomConfig() - - const autoBlacklistTestsCustomConfig = { - ...currentCustomConfig, - - autoBlacklist: { - videos: { - ofUsers: { - enabled: true - } - } - } - } - - // enable transcoding otherwise own publish notification after transcoding not expected - autoBlacklistTestsCustomConfig.transcoding.enabled = true - await servers[0].config.updateCustomConfig({ newCustomConfig: autoBlacklistTestsCustomConfig }) - - await servers[0].subscriptions.add({ targetUri: 'user_1_channel@' + servers[0].host }) - await servers[1].subscriptions.add({ targetUri: 'user_1_channel@' + servers[0].host }) - }) - - it('Should send notification to moderators on new video with auto-blacklist', async function () { - this.timeout(120000) - - videoName = 'video with auto-blacklist ' + buildUUID() - const video = await servers[0].videos.upload({ token: userToken1, attributes: { name: videoName } }) - shortUUID = video.shortUUID - uuid = video.uuid - - await waitJobs(servers) - await checkVideoAutoBlacklistForModerators({ ...adminBaseParamsServer1, shortUUID, videoName, checkType: 'presence' }) - }) - - it('Should not send video publish notification if auto-blacklisted', async function () { - this.timeout(120000) - - await checkVideoIsPublished({ ...userBaseParams, videoName, shortUUID, checkType: 'absence' }) - }) - - it('Should not send a local user subscription notification if auto-blacklisted', async function () { - this.timeout(120000) - - await checkNewVideoFromSubscription({ ...adminBaseParamsServer1, videoName, shortUUID, checkType: 'absence' }) - }) - - it('Should not send a remote user subscription notification if auto-blacklisted', async function () { - await checkNewVideoFromSubscription({ ...adminBaseParamsServer2, videoName, shortUUID, checkType: 'absence' }) - }) - - it('Should send video published and unblacklist after video unblacklisted', async function () { - this.timeout(120000) - - await servers[0].blacklist.remove({ videoId: uuid }) - - await waitJobs(servers) - - // FIXME: Can't test as two notifications sent to same user and util only checks last one - // One notification might be better anyways - // await checkNewBlacklistOnMyVideo(userBaseParams, videoUUID, videoName, 'unblacklist') - // await checkVideoIsPublished(userBaseParams, videoName, videoUUID, 'presence') - }) - - it('Should send a local user subscription notification after removed from blacklist', async function () { - this.timeout(120000) - - await checkNewVideoFromSubscription({ ...adminBaseParamsServer1, videoName, shortUUID, checkType: 'presence' }) - }) - - it('Should send a remote user subscription notification after removed from blacklist', async function () { - this.timeout(120000) - - await checkNewVideoFromSubscription({ ...adminBaseParamsServer2, videoName, shortUUID, checkType: 'presence' }) - }) - - it('Should send unblacklist but not published/subscription notes after unblacklisted if scheduled update pending', async function () { - this.timeout(120000) - - const updateAt = new Date(new Date().getTime() + 1000000) - - const name = 'video with auto-blacklist and future schedule ' + buildUUID() - - const attributes = { - name, - privacy: VideoPrivacy.PRIVATE, - scheduleUpdate: { - updateAt: updateAt.toISOString(), - privacy: VideoPrivacy.PUBLIC as VideoPrivacy.PUBLIC - } - } - - const { shortUUID, uuid } = await servers[0].videos.upload({ token: userToken1, attributes }) - - await servers[0].blacklist.remove({ videoId: uuid }) - - await waitJobs(servers) - await checkNewBlacklistOnMyVideo({ ...userBaseParams, shortUUID, videoName: name, blacklistType: 'unblacklist' }) - - // FIXME: Can't test absence as two notifications sent to same user and util only checks last one - // One notification might be better anyways - // await checkVideoIsPublished(userBaseParams, name, uuid, 'absence') - - await checkNewVideoFromSubscription({ ...adminBaseParamsServer1, videoName: name, shortUUID, checkType: 'absence' }) - await checkNewVideoFromSubscription({ ...adminBaseParamsServer2, videoName: name, shortUUID, checkType: 'absence' }) - }) - - it('Should not send publish/subscription notifications after scheduled update if video still auto-blacklisted', async function () { - this.timeout(120000) - - // In 2 seconds - const updateAt = new Date(new Date().getTime() + 2000) - - const name = 'video with schedule done and still auto-blacklisted ' + buildUUID() - - const attributes = { - name, - privacy: VideoPrivacy.PRIVATE, - scheduleUpdate: { - updateAt: updateAt.toISOString(), - privacy: VideoPrivacy.PUBLIC as VideoPrivacy.PUBLIC - } - } - - const { shortUUID } = await servers[0].videos.upload({ token: userToken1, attributes }) - - await wait(6000) - await checkVideoIsPublished({ ...userBaseParams, videoName: name, shortUUID, checkType: 'absence' }) - await checkNewVideoFromSubscription({ ...adminBaseParamsServer1, videoName: name, shortUUID, checkType: 'absence' }) - await checkNewVideoFromSubscription({ ...adminBaseParamsServer2, videoName: name, shortUUID, checkType: 'absence' }) - }) - - it('Should not send a notification to moderators on new video without auto-blacklist', async function () { - this.timeout(120000) - - const name = 'video without auto-blacklist ' + buildUUID() - - // admin with blacklist right will not be auto-blacklisted - const { shortUUID } = await servers[0].videos.upload({ attributes: { name } }) - - await waitJobs(servers) - await checkVideoAutoBlacklistForModerators({ ...adminBaseParamsServer1, shortUUID, videoName: name, checkType: 'absence' }) - }) - - after(async () => { - await servers[0].config.updateCustomConfig({ newCustomConfig: currentCustomConfig }) - - await servers[0].subscriptions.remove({ uri: 'user_1_channel@' + servers[0].host }) - await servers[1].subscriptions.remove({ uri: 'user_1_channel@' + servers[0].host }) - }) - }) - - after(async function () { - MockSmtpServer.Instance.kill() - - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/notifications/notifications-api.ts b/server/tests/api/notifications/notifications-api.ts deleted file mode 100644 index 1fc861160..000000000 --- a/server/tests/api/notifications/notifications-api.ts +++ /dev/null @@ -1,206 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { - CheckerBaseParams, - checkNewVideoFromSubscription, - getAllNotificationsSettings, - MockSmtpServer, - prepareNotificationsTest -} from '@server/tests/shared' -import { UserNotification, UserNotificationSettingValue } from '@shared/models' -import { cleanupTests, PeerTubeServer, waitJobs } from '@shared/server-commands' - -describe('Test notifications API', function () { - let server: PeerTubeServer - let userNotifications: UserNotification[] = [] - let userToken: string - let emails: object[] = [] - - before(async function () { - this.timeout(120000) - - const res = await prepareNotificationsTest(1) - emails = res.emails - userToken = res.userAccessToken - userNotifications = res.userNotifications - server = res.servers[0] - - await server.subscriptions.add({ token: userToken, targetUri: 'root_channel@' + server.host }) - - for (let i = 0; i < 10; i++) { - await server.videos.randomUpload({ wait: false }) - } - - await waitJobs([ server ]) - }) - - describe('Notification list & count', function () { - - it('Should correctly list notifications', async function () { - const { data, total } = await server.notifications.list({ token: userToken, start: 0, count: 2 }) - - expect(data).to.have.lengthOf(2) - expect(total).to.equal(10) - }) - }) - - describe('Mark as read', function () { - - it('Should mark as read some notifications', async function () { - const { data } = await server.notifications.list({ token: userToken, start: 2, count: 3 }) - const ids = data.map(n => n.id) - - await server.notifications.markAsRead({ token: userToken, ids }) - }) - - it('Should have the notifications marked as read', async function () { - const { data } = await server.notifications.list({ token: userToken, start: 0, count: 10 }) - - expect(data[0].read).to.be.false - expect(data[1].read).to.be.false - expect(data[2].read).to.be.true - expect(data[3].read).to.be.true - expect(data[4].read).to.be.true - expect(data[5].read).to.be.false - }) - - it('Should only list read notifications', async function () { - const { data } = await server.notifications.list({ token: userToken, start: 0, count: 10, unread: false }) - - for (const notification of data) { - expect(notification.read).to.be.true - } - }) - - it('Should only list unread notifications', async function () { - const { data } = await server.notifications.list({ token: userToken, start: 0, count: 10, unread: true }) - - for (const notification of data) { - expect(notification.read).to.be.false - } - }) - - it('Should mark as read all notifications', async function () { - await server.notifications.markAsReadAll({ token: userToken }) - - const body = await server.notifications.list({ token: userToken, start: 0, count: 10, unread: true }) - - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - }) - }) - - describe('Notification settings', function () { - let baseParams: CheckerBaseParams - - before(() => { - baseParams = { - server, - emails, - socketNotifications: userNotifications, - token: userToken - } - }) - - it('Should not have notifications', async function () { - this.timeout(20000) - - await server.notifications.updateMySettings({ - token: userToken, - settings: { ...getAllNotificationsSettings(), newVideoFromSubscription: UserNotificationSettingValue.NONE } - }) - - { - const info = await server.users.getMyInfo({ token: userToken }) - expect(info.notificationSettings.newVideoFromSubscription).to.equal(UserNotificationSettingValue.NONE) - } - - const { name, shortUUID } = await server.videos.randomUpload() - - const check = { web: true, mail: true } - await checkNewVideoFromSubscription({ ...baseParams, check, videoName: name, shortUUID, checkType: 'absence' }) - }) - - it('Should only have web notifications', async function () { - this.timeout(20000) - - await server.notifications.updateMySettings({ - token: userToken, - settings: { ...getAllNotificationsSettings(), newVideoFromSubscription: UserNotificationSettingValue.WEB } - }) - - { - const info = await server.users.getMyInfo({ token: userToken }) - expect(info.notificationSettings.newVideoFromSubscription).to.equal(UserNotificationSettingValue.WEB) - } - - const { name, shortUUID } = await server.videos.randomUpload() - - { - const check = { mail: true, web: false } - await checkNewVideoFromSubscription({ ...baseParams, check, videoName: name, shortUUID, checkType: 'absence' }) - } - - { - const check = { mail: false, web: true } - await checkNewVideoFromSubscription({ ...baseParams, check, videoName: name, shortUUID, checkType: 'presence' }) - } - }) - - it('Should only have mail notifications', async function () { - this.timeout(20000) - - await server.notifications.updateMySettings({ - token: userToken, - settings: { ...getAllNotificationsSettings(), newVideoFromSubscription: UserNotificationSettingValue.EMAIL } - }) - - { - const info = await server.users.getMyInfo({ token: userToken }) - expect(info.notificationSettings.newVideoFromSubscription).to.equal(UserNotificationSettingValue.EMAIL) - } - - const { name, shortUUID } = await server.videos.randomUpload() - - { - const check = { mail: false, web: true } - await checkNewVideoFromSubscription({ ...baseParams, check, videoName: name, shortUUID, checkType: 'absence' }) - } - - { - const check = { mail: true, web: false } - await checkNewVideoFromSubscription({ ...baseParams, check, videoName: name, shortUUID, checkType: 'presence' }) - } - }) - - it('Should have email and web notifications', async function () { - this.timeout(20000) - - await server.notifications.updateMySettings({ - token: userToken, - settings: { - ...getAllNotificationsSettings(), - newVideoFromSubscription: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL - } - }) - - { - const info = await server.users.getMyInfo({ token: userToken }) - expect(info.notificationSettings.newVideoFromSubscription).to.equal( - UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL - ) - } - - const { name, shortUUID } = await server.videos.randomUpload() - - await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) - }) - }) - - after(async function () { - MockSmtpServer.Instance.kill() - - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/notifications/registrations-notifications.ts b/server/tests/api/notifications/registrations-notifications.ts deleted file mode 100644 index d20fc8df3..000000000 --- a/server/tests/api/notifications/registrations-notifications.ts +++ /dev/null @@ -1,88 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { - CheckerBaseParams, - checkRegistrationRequest, - checkUserRegistered, - MockSmtpServer, - prepareNotificationsTest -} from '@server/tests/shared' -import { UserNotification } from '@shared/models' -import { cleanupTests, PeerTubeServer, waitJobs } from '@shared/server-commands' - -describe('Test registrations notifications', function () { - let server: PeerTubeServer - let userToken1: string - - let userNotifications: UserNotification[] = [] - let adminNotifications: UserNotification[] = [] - let emails: object[] = [] - - let baseParams: CheckerBaseParams - - before(async function () { - this.timeout(120000) - - const res = await prepareNotificationsTest(1) - - server = res.servers[0] - emails = res.emails - userToken1 = res.userAccessToken - adminNotifications = res.adminNotifications - userNotifications = res.userNotifications - - baseParams = { - server, - emails, - socketNotifications: adminNotifications, - token: server.accessToken - } - }) - - describe('New direct registration for moderators', function () { - - before(async function () { - await server.config.enableSignup(false) - }) - - it('Should send a notification only to moderators when a user registers on the instance', async function () { - this.timeout(50000) - - await server.registrations.register({ username: 'user_10' }) - - await waitJobs([ server ]) - - await checkUserRegistered({ ...baseParams, username: 'user_10', checkType: 'presence' }) - - const userOverride = { socketNotifications: userNotifications, token: userToken1, check: { web: true, mail: false } } - await checkUserRegistered({ ...baseParams, ...userOverride, username: 'user_10', checkType: 'absence' }) - }) - }) - - describe('New registration request for moderators', function () { - - before(async function () { - await server.config.enableSignup(true) - }) - - it('Should send a notification on new registration request', async function () { - this.timeout(50000) - - const registrationReason = 'my reason' - await server.registrations.requestRegistration({ username: 'user_11', registrationReason }) - - await waitJobs([ server ]) - - await checkRegistrationRequest({ ...baseParams, username: 'user_11', registrationReason, checkType: 'presence' }) - - const userOverride = { socketNotifications: userNotifications, token: userToken1, check: { web: true, mail: false } } - await checkRegistrationRequest({ ...baseParams, ...userOverride, username: 'user_11', registrationReason, checkType: 'absence' }) - }) - }) - - after(async function () { - MockSmtpServer.Instance.kill() - - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/notifications/user-notifications.ts b/server/tests/api/notifications/user-notifications.ts deleted file mode 100644 index 55da10265..000000000 --- a/server/tests/api/notifications/user-notifications.ts +++ /dev/null @@ -1,574 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { - CheckerBaseParams, - checkMyVideoImportIsFinished, - checkNewActorFollow, - checkNewVideoFromSubscription, - checkVideoIsPublished, - checkVideoStudioEditionIsFinished, - FIXTURE_URLS, - MockSmtpServer, - prepareNotificationsTest, - uploadRandomVideoOnServers -} from '@server/tests/shared' -import { wait } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { UserNotification, UserNotificationType, VideoPrivacy, VideoStudioTask } from '@shared/models' -import { cleanupTests, findExternalSavedVideo, PeerTubeServer, stopFfmpeg, waitJobs } from '@shared/server-commands' - -describe('Test user notifications', function () { - let servers: PeerTubeServer[] = [] - let userAccessToken: string - - let userNotifications: UserNotification[] = [] - let adminNotifications: UserNotification[] = [] - let adminNotificationsServer2: UserNotification[] = [] - let emails: object[] = [] - - let channelId: number - - before(async function () { - this.timeout(120000) - - const res = await prepareNotificationsTest(3) - emails = res.emails - userAccessToken = res.userAccessToken - servers = res.servers - userNotifications = res.userNotifications - adminNotifications = res.adminNotifications - adminNotificationsServer2 = res.adminNotificationsServer2 - channelId = res.channelId - }) - - describe('New video from my subscription notification', function () { - let baseParams: CheckerBaseParams - - before(() => { - baseParams = { - server: servers[0], - emails, - socketNotifications: userNotifications, - token: userAccessToken - } - }) - - it('Should not send notifications if the user does not follow the video publisher', async function () { - this.timeout(50000) - - await uploadRandomVideoOnServers(servers, 1) - - const notification = await servers[0].notifications.getLatest({ token: userAccessToken }) - expect(notification).to.be.undefined - - expect(emails).to.have.lengthOf(0) - expect(userNotifications).to.have.lengthOf(0) - }) - - it('Should send a new video notification if the user follows the local video publisher', async function () { - this.timeout(15000) - - await servers[0].subscriptions.add({ token: userAccessToken, targetUri: 'root_channel@' + servers[0].host }) - await waitJobs(servers) - - const { name, shortUUID } = await uploadRandomVideoOnServers(servers, 1) - await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) - }) - - it('Should send a new video notification from a remote account', async function () { - this.timeout(150000) // Server 2 has transcoding enabled - - await servers[0].subscriptions.add({ token: userAccessToken, targetUri: 'root_channel@' + servers[1].host }) - await waitJobs(servers) - - const { name, shortUUID } = await uploadRandomVideoOnServers(servers, 2) - await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) - }) - - it('Should send a new video notification on a scheduled publication', async function () { - this.timeout(50000) - - // In 2 seconds - const updateAt = new Date(new Date().getTime() + 2000) - - const data = { - privacy: VideoPrivacy.PRIVATE, - scheduleUpdate: { - updateAt: updateAt.toISOString(), - privacy: VideoPrivacy.PUBLIC as VideoPrivacy.PUBLIC - } - } - const { name, shortUUID } = await uploadRandomVideoOnServers(servers, 1, data) - - await wait(6000) - await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) - }) - - it('Should send a new video notification on a remote scheduled publication', async function () { - this.timeout(100000) - - // In 2 seconds - const updateAt = new Date(new Date().getTime() + 2000) - - const data = { - privacy: VideoPrivacy.PRIVATE, - scheduleUpdate: { - updateAt: updateAt.toISOString(), - privacy: VideoPrivacy.PUBLIC as VideoPrivacy.PUBLIC - } - } - const { name, shortUUID } = await uploadRandomVideoOnServers(servers, 2, data) - await waitJobs(servers) - - await wait(6000) - await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) - }) - - it('Should not send a notification before the video is published', async function () { - this.timeout(150000) - - const updateAt = new Date(new Date().getTime() + 1000000) - - const data = { - privacy: VideoPrivacy.PRIVATE, - scheduleUpdate: { - updateAt: updateAt.toISOString(), - privacy: VideoPrivacy.PUBLIC as VideoPrivacy.PUBLIC - } - } - const { name, shortUUID } = await uploadRandomVideoOnServers(servers, 1, data) - - await wait(6000) - await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'absence' }) - }) - - it('Should send a new video notification when a video becomes public', async function () { - this.timeout(50000) - - const data = { privacy: VideoPrivacy.PRIVATE } - const { name, uuid, shortUUID } = await uploadRandomVideoOnServers(servers, 1, data) - - await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'absence' }) - - await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } }) - - await waitJobs(servers) - await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) - }) - - it('Should send a new video notification when a remote video becomes public', async function () { - this.timeout(120000) - - const data = { privacy: VideoPrivacy.PRIVATE } - const { name, uuid, shortUUID } = await uploadRandomVideoOnServers(servers, 2, data) - - await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'absence' }) - - await servers[1].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } }) - - await waitJobs(servers) - await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) - }) - - it('Should not send a new video notification when a video becomes unlisted', async function () { - this.timeout(50000) - - const data = { privacy: VideoPrivacy.PRIVATE } - const { name, uuid, shortUUID } = await uploadRandomVideoOnServers(servers, 1, data) - - await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.UNLISTED } }) - - await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'absence' }) - }) - - it('Should not send a new video notification when a remote video becomes unlisted', async function () { - this.timeout(100000) - - const data = { privacy: VideoPrivacy.PRIVATE } - const { name, uuid, shortUUID } = await uploadRandomVideoOnServers(servers, 2, data) - - await servers[1].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.UNLISTED } }) - - await waitJobs(servers) - await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID, checkType: 'absence' }) - }) - - it('Should send a new video notification after a video import', async function () { - this.timeout(100000) - - const name = 'video import ' + buildUUID() - - const attributes = { - name, - channelId, - privacy: VideoPrivacy.PUBLIC, - targetUrl: FIXTURE_URLS.goodVideo - } - const { video } = await servers[0].imports.importVideo({ attributes }) - - await waitJobs(servers) - - await checkNewVideoFromSubscription({ ...baseParams, videoName: name, shortUUID: video.shortUUID, checkType: 'presence' }) - }) - }) - - describe('My video is published', function () { - let baseParams: CheckerBaseParams - - before(() => { - baseParams = { - server: servers[1], - emails, - socketNotifications: adminNotificationsServer2, - token: servers[1].accessToken - } - }) - - it('Should not send a notification if transcoding is not enabled', async function () { - this.timeout(50000) - - const { name, shortUUID } = await uploadRandomVideoOnServers(servers, 1) - await waitJobs(servers) - - await checkVideoIsPublished({ ...baseParams, videoName: name, shortUUID, checkType: 'absence' }) - }) - - it('Should not send a notification if the wait transcoding is false', async function () { - this.timeout(100_000) - - await uploadRandomVideoOnServers(servers, 2, { waitTranscoding: false }) - await waitJobs(servers) - - const notification = await servers[0].notifications.getLatest({ token: userAccessToken }) - if (notification) { - expect(notification.type).to.not.equal(UserNotificationType.MY_VIDEO_PUBLISHED) - } - }) - - it('Should send a notification even if the video is not transcoded in other resolutions', async function () { - this.timeout(100_000) - - const { name, shortUUID } = await uploadRandomVideoOnServers(servers, 2, { waitTranscoding: true, fixture: 'video_short_240p.mp4' }) - await waitJobs(servers) - - await checkVideoIsPublished({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) - }) - - it('Should send a notification with a transcoded video', async function () { - this.timeout(100_000) - - const { name, shortUUID } = await uploadRandomVideoOnServers(servers, 2, { waitTranscoding: true }) - await waitJobs(servers) - - await checkVideoIsPublished({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) - }) - - it('Should send a notification when an imported video is transcoded', async function () { - this.timeout(120000) - - const name = 'video import ' + buildUUID() - - const attributes = { - name, - channelId, - privacy: VideoPrivacy.PUBLIC, - targetUrl: FIXTURE_URLS.goodVideo, - waitTranscoding: true - } - const { video } = await servers[1].imports.importVideo({ attributes }) - - await waitJobs(servers) - await checkVideoIsPublished({ ...baseParams, videoName: name, shortUUID: video.shortUUID, checkType: 'presence' }) - }) - - it('Should send a notification when the scheduled update has been proceeded', async function () { - this.timeout(70000) - - // In 2 seconds - const updateAt = new Date(new Date().getTime() + 2000) - - const data = { - privacy: VideoPrivacy.PRIVATE, - scheduleUpdate: { - updateAt: updateAt.toISOString(), - privacy: VideoPrivacy.PUBLIC as VideoPrivacy.PUBLIC - } - } - const { name, shortUUID } = await uploadRandomVideoOnServers(servers, 2, data) - - await wait(6000) - await checkVideoIsPublished({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) - }) - - it('Should not send a notification before the video is published', async function () { - this.timeout(150000) - - const updateAt = new Date(new Date().getTime() + 1000000) - - const data = { - privacy: VideoPrivacy.PRIVATE, - scheduleUpdate: { - updateAt: updateAt.toISOString(), - privacy: VideoPrivacy.PUBLIC as VideoPrivacy.PUBLIC - } - } - const { name, shortUUID } = await uploadRandomVideoOnServers(servers, 2, data) - - await wait(6000) - await checkVideoIsPublished({ ...baseParams, videoName: name, shortUUID, checkType: 'absence' }) - }) - }) - - describe('My live replay is published', function () { - - let baseParams: CheckerBaseParams - - before(() => { - baseParams = { - server: servers[1], - emails, - socketNotifications: adminNotificationsServer2, - token: servers[1].accessToken - } - }) - - it('Should send a notification is a live replay of a non permanent live is published', async function () { - this.timeout(120000) - - const { shortUUID } = await servers[1].live.create({ - fields: { - name: 'non permanent live', - privacy: VideoPrivacy.PUBLIC, - channelId: servers[1].store.channel.id, - saveReplay: true, - replaySettings: { privacy: VideoPrivacy.PUBLIC }, - permanentLive: false - } - }) - - const ffmpegCommand = await servers[1].live.sendRTMPStreamInVideo({ videoId: shortUUID }) - - await waitJobs(servers) - await servers[1].live.waitUntilPublished({ videoId: shortUUID }) - - await stopFfmpeg(ffmpegCommand) - await servers[1].live.waitUntilReplacedByReplay({ videoId: shortUUID }) - - await waitJobs(servers) - await checkVideoIsPublished({ ...baseParams, videoName: 'non permanent live', shortUUID, checkType: 'presence' }) - }) - - it('Should send a notification is a live replay of a permanent live is published', async function () { - this.timeout(120000) - - const { shortUUID } = await servers[1].live.create({ - fields: { - name: 'permanent live', - privacy: VideoPrivacy.PUBLIC, - channelId: servers[1].store.channel.id, - saveReplay: true, - replaySettings: { privacy: VideoPrivacy.PUBLIC }, - permanentLive: true - } - }) - - const ffmpegCommand = await servers[1].live.sendRTMPStreamInVideo({ videoId: shortUUID }) - - await waitJobs(servers) - await servers[1].live.waitUntilPublished({ videoId: shortUUID }) - - const liveDetails = await servers[1].videos.get({ id: shortUUID }) - - await stopFfmpeg(ffmpegCommand) - - await servers[1].live.waitUntilWaiting({ videoId: shortUUID }) - await waitJobs(servers) - - const video = await findExternalSavedVideo(servers[1], liveDetails) - expect(video).to.exist - - await checkVideoIsPublished({ ...baseParams, videoName: video.name, shortUUID: video.shortUUID, checkType: 'presence' }) - }) - }) - - describe('Video studio', function () { - let baseParams: CheckerBaseParams - - before(() => { - baseParams = { - server: servers[1], - emails, - socketNotifications: adminNotificationsServer2, - token: servers[1].accessToken - } - }) - - it('Should send a notification after studio edition', async function () { - this.timeout(240000) - - const { name, shortUUID, id } = await uploadRandomVideoOnServers(servers, 2, { waitTranscoding: true }) - - await waitJobs(servers) - await checkVideoIsPublished({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) - - const tasks: VideoStudioTask[] = [ - { - name: 'cut', - options: { - start: 0, - end: 1 - } - } - ] - await servers[1].videoStudio.createEditionTasks({ videoId: id, tasks }) - await waitJobs(servers) - - await checkVideoStudioEditionIsFinished({ ...baseParams, videoName: name, shortUUID, checkType: 'presence' }) - }) - }) - - describe('My video is imported', function () { - let baseParams: CheckerBaseParams - - before(() => { - baseParams = { - server: servers[0], - emails, - socketNotifications: adminNotifications, - token: servers[0].accessToken - } - }) - - it('Should send a notification when the video import failed', async function () { - this.timeout(70000) - - const name = 'video import ' + buildUUID() - - const attributes = { - name, - channelId, - privacy: VideoPrivacy.PRIVATE, - targetUrl: FIXTURE_URLS.badVideo - } - const { video: { shortUUID } } = await servers[0].imports.importVideo({ attributes }) - - await waitJobs(servers) - - const url = FIXTURE_URLS.badVideo - await checkMyVideoImportIsFinished({ ...baseParams, videoName: name, shortUUID, url, success: false, checkType: 'presence' }) - }) - - it('Should send a notification when the video import succeeded', async function () { - this.timeout(70000) - - const name = 'video import ' + buildUUID() - - const attributes = { - name, - channelId, - privacy: VideoPrivacy.PRIVATE, - targetUrl: FIXTURE_URLS.goodVideo - } - const { video: { shortUUID } } = await servers[0].imports.importVideo({ attributes }) - - await waitJobs(servers) - - const url = FIXTURE_URLS.goodVideo - await checkMyVideoImportIsFinished({ ...baseParams, videoName: name, shortUUID, url, success: true, checkType: 'presence' }) - }) - }) - - describe('New actor follow', function () { - let baseParams: CheckerBaseParams - const myChannelName = 'super channel name' - const myUserName = 'super user name' - - before(async function () { - baseParams = { - server: servers[0], - emails, - socketNotifications: userNotifications, - token: userAccessToken - } - - await servers[0].users.updateMe({ displayName: 'super root name' }) - - await servers[0].users.updateMe({ - token: userAccessToken, - displayName: myUserName - }) - - await servers[1].users.updateMe({ displayName: 'super root 2 name' }) - - await servers[0].channels.update({ - token: userAccessToken, - channelName: 'user_1_channel', - attributes: { displayName: myChannelName } - }) - }) - - it('Should notify when a local channel is following one of our channel', async function () { - this.timeout(50000) - - await servers[0].subscriptions.add({ targetUri: 'user_1_channel@' + servers[0].host }) - await waitJobs(servers) - - await checkNewActorFollow({ - ...baseParams, - followType: 'channel', - followerName: 'root', - followerDisplayName: 'super root name', - followingDisplayName: myChannelName, - checkType: 'presence' - }) - - await servers[0].subscriptions.remove({ uri: 'user_1_channel@' + servers[0].host }) - }) - - it('Should notify when a remote channel is following one of our channel', async function () { - this.timeout(50000) - - await servers[1].subscriptions.add({ targetUri: 'user_1_channel@' + servers[0].host }) - await waitJobs(servers) - - await checkNewActorFollow({ - ...baseParams, - followType: 'channel', - followerName: 'root', - followerDisplayName: 'super root 2 name', - followingDisplayName: myChannelName, - checkType: 'presence' - }) - - await servers[1].subscriptions.remove({ uri: 'user_1_channel@' + servers[0].host }) - }) - - // PeerTube does not support account -> account follows - // it('Should notify when a local account is following one of our channel', async function () { - // this.timeout(50000) - // - // await addUserSubscription(servers[0].url, servers[0].accessToken, 'user_1@' + servers[0].host) - // - // await waitJobs(servers) - // - // await checkNewActorFollow(baseParams, 'account', 'root', 'super root name', myUserName, 'presence') - // }) - - // it('Should notify when a remote account is following one of our channel', async function () { - // this.timeout(50000) - // - // await addUserSubscription(servers[1].url, servers[1].accessToken, 'user_1@' + servers[0].host) - // - // await waitJobs(servers) - // - // await checkNewActorFollow(baseParams, 'account', 'root', 'super root 2 name', myUserName, 'presence') - // }) - }) - - after(async function () { - MockSmtpServer.Instance.kill() - - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/object-storage/index.ts b/server/tests/api/object-storage/index.ts deleted file mode 100644 index 1f4489fa3..000000000 --- a/server/tests/api/object-storage/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './live' -export * from './video-imports' -export * from './video-static-file-privacy' -export * from './videos' diff --git a/server/tests/api/object-storage/live.ts b/server/tests/api/object-storage/live.ts deleted file mode 100644 index 07ff4763b..000000000 --- a/server/tests/api/object-storage/live.ts +++ /dev/null @@ -1,311 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { expectStartWith, MockObjectStorageProxy, SQLCommand, testLiveVideoResolutions } from '@server/tests/shared' -import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' -import { HttpStatusCode, LiveVideoCreate, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - findExternalSavedVideo, - makeRawRequest, - ObjectStorageCommand, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - stopFfmpeg, - waitJobs, - waitUntilLivePublishedOnAllServers, - waitUntilLiveReplacedByReplayOnAllServers, - waitUntilLiveWaitingOnAllServers -} from '@shared/server-commands' - -async function createLive (server: PeerTubeServer, permanent: boolean) { - const attributes: LiveVideoCreate = { - channelId: server.store.channel.id, - privacy: VideoPrivacy.PUBLIC, - name: 'my super live', - saveReplay: true, - replaySettings: { privacy: VideoPrivacy.PUBLIC }, - permanentLive: permanent - } - - const { uuid } = await server.live.create({ fields: attributes }) - - return uuid -} - -async function checkFilesExist (options: { - servers: PeerTubeServer[] - videoUUID: string - numberOfFiles: number - objectStorage: ObjectStorageCommand -}) { - const { servers, videoUUID, numberOfFiles, objectStorage } = options - - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - - expect(video.files).to.have.lengthOf(0) - expect(video.streamingPlaylists).to.have.lengthOf(1) - - const files = video.streamingPlaylists[0].files - expect(files).to.have.lengthOf(numberOfFiles) - - for (const file of files) { - expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl()) - - await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) - } - } -} - -async function checkFilesCleanup (options: { - server: PeerTubeServer - videoUUID: string - resolutions: number[] - objectStorage: ObjectStorageCommand -}) { - const { server, videoUUID, resolutions, objectStorage } = options - - const resolutionFiles = resolutions.map((_value, i) => `${i}.m3u8`) - - for (const playlistName of [ 'master.m3u8' ].concat(resolutionFiles)) { - await server.live.getPlaylistFile({ - videoUUID, - playlistName, - expectedStatus: HttpStatusCode.NOT_FOUND_404, - objectStorage - }) - } - - await server.live.getSegmentFile({ - videoUUID, - playlistNumber: 0, - segment: 0, - objectStorage, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) -} - -describe('Object storage for lives', function () { - if (areMockObjectStorageTestsDisabled()) return - - let servers: PeerTubeServer[] - let sqlCommandServer1: SQLCommand - const objectStorage = new ObjectStorageCommand() - - before(async function () { - this.timeout(120000) - - await objectStorage.prepareDefaultMockBuckets() - servers = await createMultipleServers(2, objectStorage.getDefaultMockConfig()) - - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - await doubleFollow(servers[0], servers[1]) - - await servers[0].config.enableTranscoding() - - sqlCommandServer1 = new SQLCommand(servers[0]) - }) - - describe('Without live transcoding', function () { - let videoUUID: string - - before(async function () { - await servers[0].config.enableLive({ transcoding: false }) - - videoUUID = await createLive(servers[0], false) - }) - - it('Should create a live and publish it on object storage', async function () { - this.timeout(220000) - - const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID }) - await waitUntilLivePublishedOnAllServers(servers, videoUUID) - - await testLiveVideoResolutions({ - originServer: servers[0], - sqlCommand: sqlCommandServer1, - servers, - liveVideoId: videoUUID, - resolutions: [ 720 ], - transcoded: false, - objectStorage - }) - - await stopFfmpeg(ffmpegCommand) - }) - - it('Should have saved the replay on object storage', async function () { - this.timeout(220000) - - await waitUntilLiveReplacedByReplayOnAllServers(servers, videoUUID) - await waitJobs(servers) - - await checkFilesExist({ servers, videoUUID, numberOfFiles: 1, objectStorage }) - }) - - it('Should have cleaned up live files from object storage', async function () { - await checkFilesCleanup({ server: servers[0], videoUUID, resolutions: [ 720 ], objectStorage }) - }) - }) - - describe('With live transcoding', function () { - const resolutions = [ 720, 480, 360, 240, 144 ] - - before(async function () { - await servers[0].config.enableLive({ transcoding: true }) - }) - - describe('Normal replay', function () { - let videoUUIDNonPermanent: string - - before(async function () { - videoUUIDNonPermanent = await createLive(servers[0], false) - }) - - it('Should create a live and publish it on object storage', async function () { - this.timeout(240000) - - const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUIDNonPermanent }) - await waitUntilLivePublishedOnAllServers(servers, videoUUIDNonPermanent) - - await testLiveVideoResolutions({ - originServer: servers[0], - sqlCommand: sqlCommandServer1, - servers, - liveVideoId: videoUUIDNonPermanent, - resolutions, - transcoded: true, - objectStorage - }) - - await stopFfmpeg(ffmpegCommand) - }) - - it('Should have saved the replay on object storage', async function () { - this.timeout(220000) - - await waitUntilLiveReplacedByReplayOnAllServers(servers, videoUUIDNonPermanent) - await waitJobs(servers) - - await checkFilesExist({ servers, videoUUID: videoUUIDNonPermanent, numberOfFiles: 5, objectStorage }) - }) - - it('Should have cleaned up live files from object storage', async function () { - await checkFilesCleanup({ server: servers[0], videoUUID: videoUUIDNonPermanent, resolutions, objectStorage }) - }) - }) - - describe('Permanent replay', function () { - let videoUUIDPermanent: string - - before(async function () { - videoUUIDPermanent = await createLive(servers[0], true) - }) - - it('Should create a live and publish it on object storage', async function () { - this.timeout(240000) - - const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUIDPermanent }) - await waitUntilLivePublishedOnAllServers(servers, videoUUIDPermanent) - - await testLiveVideoResolutions({ - originServer: servers[0], - sqlCommand: sqlCommandServer1, - servers, - liveVideoId: videoUUIDPermanent, - resolutions, - transcoded: true, - objectStorage - }) - - await stopFfmpeg(ffmpegCommand) - }) - - it('Should have saved the replay on object storage', async function () { - this.timeout(220000) - - await waitUntilLiveWaitingOnAllServers(servers, videoUUIDPermanent) - await waitJobs(servers) - - const videoLiveDetails = await servers[0].videos.get({ id: videoUUIDPermanent }) - const replay = await findExternalSavedVideo(servers[0], videoLiveDetails) - - await checkFilesExist({ servers, videoUUID: replay.uuid, numberOfFiles: 5, objectStorage }) - }) - - it('Should have cleaned up live files from object storage', async function () { - await checkFilesCleanup({ server: servers[0], videoUUID: videoUUIDPermanent, resolutions, objectStorage }) - }) - }) - }) - - describe('With object storage base url', function () { - const mockObjectStorageProxy = new MockObjectStorageProxy() - let baseMockUrl: string - - before(async function () { - this.timeout(120000) - - const port = await mockObjectStorageProxy.initialize() - const bucketName = objectStorage.getMockStreamingPlaylistsBucketName() - baseMockUrl = `http://127.0.0.1:${port}/${bucketName}` - - await objectStorage.prepareDefaultMockBuckets() - - const config = { - object_storage: { - enabled: true, - endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(), - region: ObjectStorageCommand.getMockRegion(), - - credentials: ObjectStorageCommand.getMockCredentialsConfig(), - - streaming_playlists: { - bucket_name: bucketName, - prefix: '', - base_url: baseMockUrl - } - } - } - - await servers[0].kill() - await servers[0].run(config) - - await servers[0].config.enableLive({ transcoding: true, resolutions: 'min' }) - }) - - it('Should publish a live and replace the base url', async function () { - this.timeout(240000) - - const videoUUIDPermanent = await createLive(servers[0], true) - - const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUIDPermanent }) - await waitUntilLivePublishedOnAllServers(servers, videoUUIDPermanent) - - await testLiveVideoResolutions({ - originServer: servers[0], - sqlCommand: sqlCommandServer1, - servers, - liveVideoId: videoUUIDPermanent, - resolutions: [ 720 ], - transcoded: true, - objectStorage, - objectStorageBaseUrl: baseMockUrl - }) - - await stopFfmpeg(ffmpegCommand) - }) - }) - - after(async function () { - await sqlCommandServer1.cleanup() - await objectStorage.cleanupMock() - - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/object-storage/video-imports.ts b/server/tests/api/object-storage/video-imports.ts deleted file mode 100644 index 57150e5a6..000000000 --- a/server/tests/api/object-storage/video-imports.ts +++ /dev/null @@ -1,111 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { expectStartWith, FIXTURE_URLS } from '@server/tests/shared' -import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' -import { HttpStatusCode, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createSingleServer, - makeRawRequest, - ObjectStorageCommand, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - waitJobs -} from '@shared/server-commands' - -async function importVideo (server: PeerTubeServer) { - const attributes = { - name: 'import 2', - privacy: VideoPrivacy.PUBLIC, - channelId: server.store.channel.id, - targetUrl: FIXTURE_URLS.goodVideo720 - } - - const { video: { uuid } } = await server.imports.importVideo({ attributes }) - - return uuid -} - -describe('Object storage for video import', function () { - if (areMockObjectStorageTestsDisabled()) return - - let server: PeerTubeServer - const objectStorage = new ObjectStorageCommand() - - before(async function () { - this.timeout(120000) - - await objectStorage.prepareDefaultMockBuckets() - - server = await createSingleServer(1, objectStorage.getDefaultMockConfig()) - - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - await server.config.enableImports() - }) - - describe('Without transcoding', async function () { - - before(async function () { - await server.config.disableTranscoding() - }) - - it('Should import a video and have sent it to object storage', async function () { - this.timeout(120000) - - const uuid = await importVideo(server) - await waitJobs(server) - - const video = await server.videos.get({ id: uuid }) - - expect(video.files).to.have.lengthOf(1) - expect(video.streamingPlaylists).to.have.lengthOf(0) - - const fileUrl = video.files[0].fileUrl - expectStartWith(fileUrl, objectStorage.getMockWebVideosBaseUrl()) - - await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - describe('With transcoding', async function () { - - before(async function () { - await server.config.enableTranscoding() - }) - - it('Should import a video and have sent it to object storage', async function () { - this.timeout(120000) - - const uuid = await importVideo(server) - await waitJobs(server) - - const video = await server.videos.get({ id: uuid }) - - expect(video.files).to.have.lengthOf(5) - expect(video.streamingPlaylists).to.have.lengthOf(1) - expect(video.streamingPlaylists[0].files).to.have.lengthOf(5) - - for (const file of video.files) { - expectStartWith(file.fileUrl, objectStorage.getMockWebVideosBaseUrl()) - - await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) - } - - for (const file of video.streamingPlaylists[0].files) { - expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl()) - - await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) - } - }) - }) - - after(async function () { - await objectStorage.cleanupMock() - - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/object-storage/video-static-file-privacy.ts b/server/tests/api/object-storage/video-static-file-privacy.ts deleted file mode 100644 index 64ab542a5..000000000 --- a/server/tests/api/object-storage/video-static-file-privacy.ts +++ /dev/null @@ -1,570 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { basename } from 'path' -import { checkVideoFileTokenReinjection, expectStartWith, SQLCommand } from '@server/tests/shared' -import { areScalewayObjectStorageTestsDisabled, getAllFiles, getHLS } from '@shared/core-utils' -import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createSingleServer, - findExternalSavedVideo, - makeRawRequest, - ObjectStorageCommand, - PeerTubeServer, - sendRTMPStream, - setAccessTokensToServers, - setDefaultVideoChannel, - stopFfmpeg, - waitJobs -} from '@shared/server-commands' - -function extractFilenameFromUrl (url: string) { - const parts = basename(url).split(':') - - return parts[parts.length - 1] -} - -describe('Object storage for video static file privacy', function () { - // We need real world object storage to check ACL - if (areScalewayObjectStorageTestsDisabled()) return - - let server: PeerTubeServer - let sqlCommand: SQLCommand - let userToken: string - - // --------------------------------------------------------------------------- - - async function checkPrivateVODFiles (uuid: string) { - const video = await server.videos.getWithToken({ id: uuid }) - - for (const file of video.files) { - expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/web-videos/private/') - - await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - } - - for (const file of getAllFiles(video)) { - const internalFileUrl = await sqlCommand.getInternalFileUrl(file.id) - expectStartWith(internalFileUrl, ObjectStorageCommand.getScalewayBaseUrl()) - await makeRawRequest({ url: internalFileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - } - - const hls = getHLS(video) - - if (hls) { - for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) { - expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') - } - - await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - - for (const file of hls.files) { - expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') - - await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - } - } - } - - async function checkPublicVODFiles (uuid: string) { - const video = await server.videos.getWithToken({ id: uuid }) - - for (const file of getAllFiles(video)) { - expectStartWith(file.fileUrl, ObjectStorageCommand.getScalewayBaseUrl()) - - await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) - } - - const hls = getHLS(video) - - if (hls) { - expectStartWith(hls.playlistUrl, ObjectStorageCommand.getScalewayBaseUrl()) - expectStartWith(hls.segmentsSha256Url, ObjectStorageCommand.getScalewayBaseUrl()) - - await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) - } - } - - // --------------------------------------------------------------------------- - - before(async function () { - this.timeout(120000) - - server = await createSingleServer(1, ObjectStorageCommand.getDefaultScalewayConfig({ serverNumber: 1 })) - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - await server.config.enableMinimumTranscoding() - - userToken = await server.users.generateUserAndToken('user1') - - sqlCommand = new SQLCommand(server) - }) - - describe('VOD', function () { - let privateVideoUUID: string - let publicVideoUUID: string - let passwordProtectedVideoUUID: string - let userPrivateVideoUUID: string - - const correctPassword = 'my super password' - const correctPasswordHeader = { 'x-peertube-video-password': correctPassword } - const incorrectPasswordHeader = { 'x-peertube-video-password': correctPassword + 'toto' } - - // --------------------------------------------------------------------------- - - async function getSampleFileUrls (videoId: string) { - const video = await server.videos.getWithToken({ id: videoId }) - - return { - webVideoFile: video.files[0].fileUrl, - hlsFile: getHLS(video).files[0].fileUrl - } - } - - // --------------------------------------------------------------------------- - - it('Should upload a private video and have appropriate object storage ACL', async function () { - this.timeout(120000) - - { - const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) - privateVideoUUID = uuid - } - - { - const { uuid } = await server.videos.quickUpload({ name: 'user video', token: userToken, privacy: VideoPrivacy.PRIVATE }) - userPrivateVideoUUID = uuid - } - - await waitJobs([ server ]) - - await checkPrivateVODFiles(privateVideoUUID) - }) - - it('Should upload a password protected video and have appropriate object storage ACL', async function () { - this.timeout(120000) - - { - const { uuid } = await server.videos.quickUpload({ - name: 'video', - privacy: VideoPrivacy.PASSWORD_PROTECTED, - videoPasswords: [ correctPassword ] - }) - passwordProtectedVideoUUID = uuid - } - await waitJobs([ server ]) - - await checkPrivateVODFiles(passwordProtectedVideoUUID) - }) - - it('Should upload a public video and have appropriate object storage ACL', async function () { - this.timeout(120000) - - const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.UNLISTED }) - await waitJobs([ server ]) - - publicVideoUUID = uuid - - await checkPublicVODFiles(publicVideoUUID) - }) - - it('Should not get files without appropriate OAuth token', async function () { - this.timeout(60000) - - const { webVideoFile, hlsFile } = await getSampleFileUrls(privateVideoUUID) - - await makeRawRequest({ url: webVideoFile, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeRawRequest({ url: webVideoFile, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - - await makeRawRequest({ url: hlsFile, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeRawRequest({ url: hlsFile, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - }) - - it('Should not get files without appropriate password or appropriate OAuth token', async function () { - this.timeout(60000) - - const { webVideoFile, hlsFile } = await getSampleFileUrls(passwordProtectedVideoUUID) - - await makeRawRequest({ url: webVideoFile, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeRawRequest({ - url: webVideoFile, - token: null, - headers: incorrectPasswordHeader, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - await makeRawRequest({ url: webVideoFile, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ - url: webVideoFile, - token: null, - headers: correctPasswordHeader, - expectedStatus: HttpStatusCode.OK_200 - }) - - await makeRawRequest({ url: hlsFile, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeRawRequest({ - url: hlsFile, - token: null, - headers: incorrectPasswordHeader, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - await makeRawRequest({ url: hlsFile, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ - url: hlsFile, - token: null, - headers: correctPasswordHeader, - expectedStatus: HttpStatusCode.OK_200 - }) - }) - - it('Should not get HLS file of another video', async function () { - this.timeout(60000) - - const privateVideo = await server.videos.getWithToken({ id: privateVideoUUID }) - const hlsFilename = basename(getHLS(privateVideo).files[0].fileUrl) - - const badUrl = server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + userPrivateVideoUUID + '/' + hlsFilename - const goodUrl = server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + privateVideoUUID + '/' + hlsFilename - - await makeRawRequest({ url: badUrl, token: server.accessToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - await makeRawRequest({ url: goodUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - }) - - it('Should correctly check OAuth, video file token of private video', async function () { - this.timeout(60000) - - const badVideoFileToken = await server.videoToken.getVideoFileToken({ token: userToken, videoId: userPrivateVideoUUID }) - const goodVideoFileToken = await server.videoToken.getVideoFileToken({ videoId: privateVideoUUID }) - - const { webVideoFile, hlsFile } = await getSampleFileUrls(privateVideoUUID) - - for (const url of [ webVideoFile, hlsFile ]) { - await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - - await makeRawRequest({ url, query: { videoFileToken: badVideoFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeRawRequest({ url, query: { videoFileToken: goodVideoFileToken }, expectedStatus: HttpStatusCode.OK_200 }) - - } - }) - - it('Should correctly check OAuth, video file token or video password of password protected video', async function () { - this.timeout(60000) - - const badVideoFileToken = await server.videoToken.getVideoFileToken({ token: userToken, videoId: userPrivateVideoUUID }) - const goodVideoFileToken = await server.videoToken.getVideoFileToken({ - videoId: passwordProtectedVideoUUID, - videoPassword: correctPassword - }) - - const { webVideoFile, hlsFile } = await getSampleFileUrls(passwordProtectedVideoUUID) - - for (const url of [ hlsFile, webVideoFile ]) { - await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - - await makeRawRequest({ url, query: { videoFileToken: badVideoFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeRawRequest({ url, query: { videoFileToken: goodVideoFileToken }, expectedStatus: HttpStatusCode.OK_200 }) - - await makeRawRequest({ - url, - headers: incorrectPasswordHeader, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - await makeRawRequest({ url, headers: correctPasswordHeader, expectedStatus: HttpStatusCode.OK_200 }) - } - }) - - it('Should reinject video file token', async function () { - this.timeout(120000) - - const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: privateVideoUUID }) - - await checkVideoFileTokenReinjection({ - server, - videoUUID: privateVideoUUID, - videoFileToken, - resolutions: [ 240, 720 ], - isLive: false - }) - }) - - it('Should update public video to private', async function () { - this.timeout(60000) - - await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.INTERNAL } }) - - await checkPrivateVODFiles(publicVideoUUID) - }) - - it('Should update private video to public', async function () { - this.timeout(60000) - - await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.PUBLIC } }) - - await checkPublicVODFiles(publicVideoUUID) - }) - }) - - describe('Live', function () { - let normalLiveId: string - let normalLive: LiveVideo - - let permanentLiveId: string - let permanentLive: LiveVideo - - let passwordProtectedLiveId: string - let passwordProtectedLive: LiveVideo - - const correctPassword = 'my super password' - - let unrelatedFileToken: string - - // --------------------------------------------------------------------------- - - async function checkLiveFiles (live: LiveVideo, liveId: string, videoPassword?: string) { - const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) - await server.live.waitUntilPublished({ videoId: liveId }) - - const video = videoPassword - ? await server.videos.getWithPassword({ id: liveId, password: videoPassword }) - : await server.videos.getWithToken({ id: liveId }) - - const fileToken = videoPassword - ? await server.videoToken.getVideoFileToken({ token: null, videoId: video.uuid, videoPassword }) - : await server.videoToken.getVideoFileToken({ videoId: video.uuid }) - - const hls = video.streamingPlaylists[0] - - for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) { - expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') - - await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - - await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) - if (videoPassword) { - await makeRawRequest({ url, headers: { 'x-peertube-video-password': videoPassword }, expectedStatus: HttpStatusCode.OK_200 }) - } - await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - if (videoPassword) { - await makeRawRequest({ - url, - headers: { 'x-peertube-video-password': 'incorrectPassword' }, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - } - } - - await stopFfmpeg(ffmpegCommand) - } - - async function checkReplay (replay: VideoDetails) { - const fileToken = await server.videoToken.getVideoFileToken({ videoId: replay.uuid }) - - const hls = replay.streamingPlaylists[0] - expect(hls.files).to.not.have.lengthOf(0) - - for (const file of hls.files) { - await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ url: file.fileUrl, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) - - await makeRawRequest({ url: file.fileUrl, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeRawRequest({ - url: file.fileUrl, - query: { videoFileToken: unrelatedFileToken }, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - } - - for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) { - expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') - - await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) - - await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - } - } - - // --------------------------------------------------------------------------- - - before(async function () { - await server.config.enableMinimumTranscoding() - - const { uuid } = await server.videos.quickUpload({ name: 'another video' }) - unrelatedFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid }) - - await server.config.enableLive({ - allowReplay: true, - transcoding: true, - resolutions: 'min' - }) - - { - const { video, live } = await server.live.quickCreate({ - saveReplay: true, - permanentLive: false, - privacy: VideoPrivacy.PRIVATE - }) - normalLiveId = video.uuid - normalLive = live - } - - { - const { video, live } = await server.live.quickCreate({ - saveReplay: true, - permanentLive: true, - privacy: VideoPrivacy.PRIVATE - }) - permanentLiveId = video.uuid - permanentLive = live - } - - { - const { video, live } = await server.live.quickCreate({ - saveReplay: false, - permanentLive: false, - privacy: VideoPrivacy.PASSWORD_PROTECTED, - videoPasswords: [ correctPassword ] - }) - passwordProtectedLiveId = video.uuid - passwordProtectedLive = live - } - }) - - it('Should create a private normal live and have a private static path', async function () { - this.timeout(240000) - - await checkLiveFiles(normalLive, normalLiveId) - }) - - it('Should create a private permanent live and have a private static path', async function () { - this.timeout(240000) - - await checkLiveFiles(permanentLive, permanentLiveId) - }) - - it('Should create a password protected live and have a private static path', async function () { - this.timeout(240000) - - await checkLiveFiles(passwordProtectedLive, passwordProtectedLiveId, correctPassword) - }) - - it('Should reinject video file token in permanent live', async function () { - this.timeout(240000) - - const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: permanentLive.rtmpUrl, streamKey: permanentLive.streamKey }) - await server.live.waitUntilPublished({ videoId: permanentLiveId }) - - const video = await server.videos.getWithToken({ id: permanentLiveId }) - const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid }) - - await checkVideoFileTokenReinjection({ - server, - videoUUID: permanentLiveId, - videoFileToken, - resolutions: [ 720 ], - isLive: true - }) - - await stopFfmpeg(ffmpegCommand) - }) - - it('Should have created a replay of the normal live with a private static path', async function () { - this.timeout(240000) - - await server.live.waitUntilReplacedByReplay({ videoId: normalLiveId }) - - const replay = await server.videos.getWithToken({ id: normalLiveId }) - await checkReplay(replay) - }) - - it('Should have created a replay of the permanent live with a private static path', async function () { - this.timeout(240000) - - await server.live.waitUntilWaiting({ videoId: permanentLiveId }) - await waitJobs([ server ]) - - const live = await server.videos.getWithToken({ id: permanentLiveId }) - const replayFromList = await findExternalSavedVideo(server, live) - const replay = await server.videos.getWithToken({ id: replayFromList.id }) - - await checkReplay(replay) - }) - }) - - describe('With private files proxy disabled and public ACL for private files', function () { - let videoUUID: string - - before(async function () { - this.timeout(240000) - - await server.kill() - - const config = ObjectStorageCommand.getDefaultScalewayConfig({ - serverNumber: 1, - enablePrivateProxy: false, - privateACL: 'public-read' - }) - await server.run(config) - - const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) - videoUUID = uuid - - await waitJobs([ server ]) - }) - - it('Should display object storage path for a private video and be able to access them', async function () { - this.timeout(60000) - - await checkPublicVODFiles(videoUUID) - }) - - it('Should not be able to access object storage proxy', async function () { - const privateVideo = await server.videos.getWithToken({ id: videoUUID }) - const webVideoFilename = extractFilenameFromUrl(privateVideo.files[0].fileUrl) - const hlsFilename = extractFilenameFromUrl(getHLS(privateVideo).files[0].fileUrl) - - await makeRawRequest({ - url: server.url + '/object-storage-proxy/web-videos/private/' + webVideoFilename, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - - await makeRawRequest({ - url: server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + videoUUID + '/' + hlsFilename, - token: server.accessToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - }) - - after(async function () { - this.timeout(240000) - - const { data } = await server.videos.listAllForAdmin() - - for (const v of data) { - await server.videos.remove({ id: v.uuid }) - } - - for (const v of data) { - await server.servers.waitUntilLog('Removed files of video ' + v.url) - } - - await sqlCommand.cleanup() - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/object-storage/videos.ts b/server/tests/api/object-storage/videos.ts deleted file mode 100644 index dcc52ef06..000000000 --- a/server/tests/api/object-storage/videos.ts +++ /dev/null @@ -1,438 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import bytes from 'bytes' -import { expect } from 'chai' -import { stat } from 'fs-extra' -import { merge } from 'lodash' -import { - checkTmpIsEmpty, - checkWebTorrentWorks, - expectLogDoesNotContain, - expectStartWith, - generateHighBitrateVideo, - MockObjectStorageProxy, - SQLCommand -} from '@server/tests/shared' -import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' -import { sha1 } from '@shared/extra-utils' -import { HttpStatusCode, VideoDetails } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - createSingleServer, - doubleFollow, - killallServers, - makeRawRequest, - ObjectStorageCommand, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -async function checkFiles (options: { - server: PeerTubeServer - originServer: PeerTubeServer - originSQLCommand: SQLCommand - - video: VideoDetails - - baseMockUrl?: string - - playlistBucket: string - playlistPrefix?: string - - webVideoBucket: string - webVideoPrefix?: string -}) { - const { - server, - originServer, - originSQLCommand, - video, - playlistBucket, - webVideoBucket, - baseMockUrl, - playlistPrefix, - webVideoPrefix - } = options - - let allFiles = video.files - - for (const file of video.files) { - const baseUrl = baseMockUrl - ? `${baseMockUrl}/${webVideoBucket}/` - : `http://${webVideoBucket}.${ObjectStorageCommand.getMockEndpointHost()}/` - - const prefix = webVideoPrefix || '' - const start = baseUrl + prefix - - expectStartWith(file.fileUrl, start) - - const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 }) - const location = res.headers['location'] - expectStartWith(location, start) - - await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 }) - } - - const hls = video.streamingPlaylists[0] - - if (hls) { - allFiles = allFiles.concat(hls.files) - - const baseUrl = baseMockUrl - ? `${baseMockUrl}/${playlistBucket}/` - : `http://${playlistBucket}.${ObjectStorageCommand.getMockEndpointHost()}/` - - const prefix = playlistPrefix || '' - const start = baseUrl + prefix - - expectStartWith(hls.playlistUrl, start) - expectStartWith(hls.segmentsSha256Url, start) - - await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) - - const resSha = await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) - expect(JSON.stringify(resSha.body)).to.not.throw - - let i = 0 - for (const file of hls.files) { - expectStartWith(file.fileUrl, start) - - const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 }) - const location = res.headers['location'] - expectStartWith(location, start) - - await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 }) - - if (originServer.internalServerNumber === server.internalServerNumber) { - const infohash = sha1(`${2 + hls.playlistUrl}+V${i}`) - const dbInfohashes = await originSQLCommand.getPlaylistInfohash(hls.id) - - expect(dbInfohashes).to.include(infohash) - } - - i++ - } - } - - for (const file of allFiles) { - await checkWebTorrentWorks(file.magnetUri) - - const res = await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) - expect(res.body).to.have.length.above(100) - } - - return allFiles.map(f => f.fileUrl) -} - -function runTestSuite (options: { - fixture?: string - - maxUploadPart?: string - - playlistBucket: string - playlistPrefix?: string - - webVideoBucket: string - webVideoPrefix?: string - - useMockBaseUrl?: boolean -}) { - const mockObjectStorageProxy = new MockObjectStorageProxy() - const { fixture } = options - let baseMockUrl: string - - let servers: PeerTubeServer[] - let sqlCommands: SQLCommand[] = [] - const objectStorage = new ObjectStorageCommand() - - let keptUrls: string[] = [] - - const uuidsToDelete: string[] = [] - let deletedUrls: string[] = [] - - before(async function () { - this.timeout(240000) - - const port = await mockObjectStorageProxy.initialize() - baseMockUrl = options.useMockBaseUrl - ? `http://127.0.0.1:${port}` - : undefined - - await objectStorage.createMockBucket(options.playlistBucket) - await objectStorage.createMockBucket(options.webVideoBucket) - - const config = { - object_storage: { - enabled: true, - endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(), - region: ObjectStorageCommand.getMockRegion(), - - credentials: ObjectStorageCommand.getMockCredentialsConfig(), - - max_upload_part: options.maxUploadPart || '5MB', - - streaming_playlists: { - bucket_name: options.playlistBucket, - prefix: options.playlistPrefix, - base_url: baseMockUrl - ? `${baseMockUrl}/${options.playlistBucket}` - : undefined - }, - - web_videos: { - bucket_name: options.webVideoBucket, - prefix: options.webVideoPrefix, - base_url: baseMockUrl - ? `${baseMockUrl}/${options.webVideoBucket}` - : undefined - } - } - } - - servers = await createMultipleServers(2, config) - - await setAccessTokensToServers(servers) - await doubleFollow(servers[0], servers[1]) - - for (const server of servers) { - const { uuid } = await server.videos.quickUpload({ name: 'video to keep' }) - await waitJobs(servers) - - const files = await server.videos.listFiles({ id: uuid }) - keptUrls = keptUrls.concat(files.map(f => f.fileUrl)) - } - - sqlCommands = servers.map(s => new SQLCommand(s)) - }) - - it('Should upload a video and move it to the object storage without transcoding', async function () { - this.timeout(40000) - - const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1', fixture }) - uuidsToDelete.push(uuid) - - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: uuid }) - const files = await checkFiles({ ...options, server, originServer: servers[0], originSQLCommand: sqlCommands[0], video, baseMockUrl }) - - deletedUrls = deletedUrls.concat(files) - } - }) - - it('Should upload a video and move it to the object storage with transcoding', async function () { - this.timeout(120000) - - const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2', fixture }) - uuidsToDelete.push(uuid) - - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: uuid }) - const files = await checkFiles({ ...options, server, originServer: servers[0], originSQLCommand: sqlCommands[0], video, baseMockUrl }) - - deletedUrls = deletedUrls.concat(files) - } - }) - - it('Should fetch correctly all the files', async function () { - for (const url of deletedUrls.concat(keptUrls)) { - await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) - } - }) - - it('Should correctly delete the files', async function () { - await servers[0].videos.remove({ id: uuidsToDelete[0] }) - await servers[1].videos.remove({ id: uuidsToDelete[1] }) - - await waitJobs(servers) - - for (const url of deletedUrls) { - await makeRawRequest({ url, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - } - }) - - it('Should have kept other files', async function () { - for (const url of keptUrls) { - await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) - } - }) - - it('Should have an empty tmp directory', async function () { - for (const server of servers) { - await checkTmpIsEmpty(server) - } - }) - - it('Should not have downloaded files from object storage', async function () { - for (const server of servers) { - await expectLogDoesNotContain(server, 'from object storage') - } - }) - - after(async function () { - await mockObjectStorageProxy.terminate() - await objectStorage.cleanupMock() - - for (const sqlCommand of sqlCommands) { - await sqlCommand.cleanup() - } - - await cleanupTests(servers) - }) -} - -describe('Object storage for videos', function () { - if (areMockObjectStorageTestsDisabled()) return - - const objectStorage = new ObjectStorageCommand() - - describe('Test config', function () { - let server: PeerTubeServer - - const baseConfig = objectStorage.getDefaultMockConfig() - - const badCredentials = { - access_key_id: 'AKIAIOSFODNN7EXAMPLE', - secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' - } - - it('Should fail with same bucket names without prefix', function (done) { - const config = merge({}, baseConfig, { - object_storage: { - streaming_playlists: { - bucket_name: 'aaa' - }, - - web_videos: { - bucket_name: 'aaa' - } - } - }) - - createSingleServer(1, config) - .then(() => done(new Error('Did not throw'))) - .catch(() => done()) - }) - - it('Should fail with bad credentials', async function () { - this.timeout(60000) - - await objectStorage.prepareDefaultMockBuckets() - - const config = merge({}, baseConfig, { - object_storage: { - credentials: badCredentials - } - }) - - server = await createSingleServer(1, config) - await setAccessTokensToServers([ server ]) - - const { uuid } = await server.videos.quickUpload({ name: 'video' }) - - await waitJobs([ server ], { skipDelayed: true }) - const video = await server.videos.get({ id: uuid }) - - expectStartWith(video.files[0].fileUrl, server.url) - - await killallServers([ server ]) - }) - - it('Should succeed with credentials from env', async function () { - this.timeout(60000) - - await objectStorage.prepareDefaultMockBuckets() - - const config = merge({}, baseConfig, { - object_storage: { - credentials: { - access_key_id: '', - secret_access_key: '' - } - } - }) - - const goodCredentials = ObjectStorageCommand.getMockCredentialsConfig() - - server = await createSingleServer(1, config, { - env: { - AWS_ACCESS_KEY_ID: goodCredentials.access_key_id, - AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key - } - }) - - await setAccessTokensToServers([ server ]) - - const { uuid } = await server.videos.quickUpload({ name: 'video' }) - - await waitJobs([ server ], { skipDelayed: true }) - const video = await server.videos.get({ id: uuid }) - - expectStartWith(video.files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl()) - }) - - after(async function () { - await objectStorage.cleanupMock() - - await cleanupTests([ server ]) - }) - }) - - describe('Test simple object storage', function () { - runTestSuite({ - playlistBucket: objectStorage.getMockBucketName('streaming-playlists'), - webVideoBucket: objectStorage.getMockBucketName('web-videos') - }) - }) - - describe('Test object storage with prefix', function () { - runTestSuite({ - playlistBucket: objectStorage.getMockBucketName('mybucket'), - webVideoBucket: objectStorage.getMockBucketName('mybucket'), - - playlistPrefix: 'streaming-playlists_', - webVideoPrefix: 'webvideo_' - }) - }) - - describe('Test object storage with prefix and base URL', function () { - runTestSuite({ - playlistBucket: objectStorage.getMockBucketName('mybucket'), - webVideoBucket: objectStorage.getMockBucketName('mybucket'), - - playlistPrefix: 'streaming-playlists/', - webVideoPrefix: 'webvideo/', - - useMockBaseUrl: true - }) - }) - - describe('Test object storage with file bigger than upload part', function () { - let fixture: string - const maxUploadPart = '5MB' - - before(async function () { - this.timeout(120000) - - fixture = await generateHighBitrateVideo() - - const { size } = await stat(fixture) - - if (bytes.parse(maxUploadPart) > size) { - throw Error(`Fixture file is too small (${size}) to make sense for this test.`) - } - }) - - runTestSuite({ - maxUploadPart, - playlistBucket: objectStorage.getMockBucketName('streaming-playlists'), - webVideoBucket: objectStorage.getMockBucketName('web-videos'), - fixture - }) - }) -}) diff --git a/server/tests/api/redundancy/index.ts b/server/tests/api/redundancy/index.ts deleted file mode 100644 index 37dc3f88c..000000000 --- a/server/tests/api/redundancy/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -import './redundancy-constraints' -import './redundancy' -import './manage-redundancy' diff --git a/server/tests/api/redundancy/manage-redundancy.ts b/server/tests/api/redundancy/manage-redundancy.ts deleted file mode 100644 index 404b65a99..000000000 --- a/server/tests/api/redundancy/manage-redundancy.ts +++ /dev/null @@ -1,324 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - RedundancyCommand, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' -import { VideoPrivacy, VideoRedundanciesTarget } from '@shared/models' - -describe('Test manage videos redundancy', function () { - const targets: VideoRedundanciesTarget[] = [ 'my-videos', 'remote-videos' ] - - let servers: PeerTubeServer[] - let video1Server2UUID: string - let video2Server2UUID: string - let redundanciesToRemove: number[] = [] - - let commands: RedundancyCommand[] - - before(async function () { - this.timeout(120000) - - const config = { - transcoding: { - hls: { - enabled: true - } - }, - redundancy: { - videos: { - check_interval: '1 second', - strategies: [ - { - strategy: 'recently-added', - min_lifetime: '1 hour', - size: '10MB', - min_views: 0 - } - ] - } - } - } - servers = await createMultipleServers(3, config) - - // Get the access tokens - await setAccessTokensToServers(servers) - - commands = servers.map(s => s.redundancy) - - { - const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video 1 server 2' } }) - video1Server2UUID = uuid - } - - { - const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video 2 server 2' } }) - video2Server2UUID = uuid - } - - await waitJobs(servers) - - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - await doubleFollow(servers[0], servers[2]) - await commands[0].updateRedundancy({ host: servers[1].host, redundancyAllowed: true }) - - await waitJobs(servers) - }) - - it('Should not have redundancies on server 3', async function () { - for (const target of targets) { - const body = await commands[2].listVideos({ target }) - - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - } - }) - - it('Should correctly list followings by redundancy', async function () { - const body = await servers[0].follows.getFollowings({ sort: '-redundancyAllowed' }) - - expect(body.total).to.equal(2) - expect(body.data).to.have.lengthOf(2) - - expect(body.data[0].following.host).to.equal(servers[1].host) - expect(body.data[1].following.host).to.equal(servers[2].host) - }) - - it('Should not have "remote-videos" redundancies on server 2', async function () { - this.timeout(120000) - - await waitJobs(servers) - await servers[0].servers.waitUntilLog('Duplicated ', 10) - await waitJobs(servers) - - const body = await commands[1].listVideos({ target: 'remote-videos' }) - - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - }) - - it('Should have "my-videos" redundancies on server 2', async function () { - this.timeout(120000) - - const body = await commands[1].listVideos({ target: 'my-videos' }) - expect(body.total).to.equal(2) - - const videos = body.data - expect(videos).to.have.lengthOf(2) - - const videos1 = videos.find(v => v.uuid === video1Server2UUID) - const videos2 = videos.find(v => v.uuid === video2Server2UUID) - - expect(videos1.name).to.equal('video 1 server 2') - expect(videos2.name).to.equal('video 2 server 2') - - expect(videos1.redundancies.files).to.have.lengthOf(4) - expect(videos1.redundancies.streamingPlaylists).to.have.lengthOf(1) - - const redundancies = videos1.redundancies.files.concat(videos1.redundancies.streamingPlaylists) - - for (const r of redundancies) { - expect(r.strategy).to.be.null - expect(r.fileUrl).to.exist - expect(r.createdAt).to.exist - expect(r.updatedAt).to.exist - expect(r.expiresOn).to.exist - } - }) - - it('Should not have "my-videos" redundancies on server 1', async function () { - const body = await commands[0].listVideos({ target: 'my-videos' }) - - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - }) - - it('Should have "remote-videos" redundancies on server 1', async function () { - this.timeout(120000) - - const body = await commands[0].listVideos({ target: 'remote-videos' }) - expect(body.total).to.equal(2) - - const videos = body.data - expect(videos).to.have.lengthOf(2) - - const videos1 = videos.find(v => v.uuid === video1Server2UUID) - const videos2 = videos.find(v => v.uuid === video2Server2UUID) - - expect(videos1.name).to.equal('video 1 server 2') - expect(videos2.name).to.equal('video 2 server 2') - - expect(videos1.redundancies.files).to.have.lengthOf(4) - expect(videos1.redundancies.streamingPlaylists).to.have.lengthOf(1) - - const redundancies = videos1.redundancies.files.concat(videos1.redundancies.streamingPlaylists) - - for (const r of redundancies) { - expect(r.strategy).to.equal('recently-added') - expect(r.fileUrl).to.exist - expect(r.createdAt).to.exist - expect(r.updatedAt).to.exist - expect(r.expiresOn).to.exist - } - }) - - it('Should correctly paginate and sort results', async function () { - { - const body = await commands[0].listVideos({ - target: 'remote-videos', - sort: 'name', - start: 0, - count: 2 - }) - - const videos = body.data - expect(videos[0].name).to.equal('video 1 server 2') - expect(videos[1].name).to.equal('video 2 server 2') - } - - { - const body = await commands[0].listVideos({ - target: 'remote-videos', - sort: '-name', - start: 0, - count: 2 - }) - - const videos = body.data - expect(videos[0].name).to.equal('video 2 server 2') - expect(videos[1].name).to.equal('video 1 server 2') - } - - { - const body = await commands[0].listVideos({ - target: 'remote-videos', - sort: '-name', - start: 1, - count: 1 - }) - - expect(body.data[0].name).to.equal('video 1 server 2') - } - }) - - it('Should manually add a redundancy and list it', async function () { - this.timeout(120000) - - const uuid = (await servers[1].videos.quickUpload({ name: 'video 3 server 2', privacy: VideoPrivacy.UNLISTED })).uuid - await waitJobs(servers) - const videoId = await servers[0].videos.getId({ uuid }) - - await commands[0].addVideo({ videoId }) - - await waitJobs(servers) - await servers[0].servers.waitUntilLog('Duplicated ', 15) - await waitJobs(servers) - - { - const body = await commands[0].listVideos({ - target: 'remote-videos', - sort: '-name', - start: 0, - count: 5 - }) - - const video = body.data[0] - - expect(video.name).to.equal('video 3 server 2') - expect(video.redundancies.files).to.have.lengthOf(4) - expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1) - - const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists) - - for (const r of redundancies) { - redundanciesToRemove.push(r.id) - - expect(r.strategy).to.equal('manual') - expect(r.fileUrl).to.exist - expect(r.createdAt).to.exist - expect(r.updatedAt).to.exist - expect(r.expiresOn).to.be.null - } - } - - const body = await commands[1].listVideos({ - target: 'my-videos', - sort: '-name', - start: 0, - count: 5 - }) - - const video = body.data[0] - expect(video.name).to.equal('video 3 server 2') - expect(video.redundancies.files).to.have.lengthOf(4) - expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1) - - const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists) - - for (const r of redundancies) { - expect(r.strategy).to.be.null - expect(r.fileUrl).to.exist - expect(r.createdAt).to.exist - expect(r.updatedAt).to.exist - expect(r.expiresOn).to.be.null - } - }) - - it('Should manually remove a redundancy and remove it from the list', async function () { - this.timeout(120000) - - for (const redundancyId of redundanciesToRemove) { - await commands[0].removeVideo({ redundancyId }) - } - - { - const body = await commands[0].listVideos({ - target: 'remote-videos', - sort: '-name', - start: 0, - count: 5 - }) - - const videos = body.data - - expect(videos).to.have.lengthOf(2) - - const video = videos[0] - expect(video.name).to.equal('video 2 server 2') - expect(video.redundancies.files).to.have.lengthOf(4) - expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1) - - const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists) - - redundanciesToRemove = redundancies.map(r => r.id) - } - }) - - it('Should remove another (auto) redundancy', async function () { - for (const redundancyId of redundanciesToRemove) { - await commands[0].removeVideo({ redundancyId }) - } - - const body = await commands[0].listVideos({ - target: 'remote-videos', - sort: '-name', - start: 0, - count: 5 - }) - - const videos = body.data - expect(videos).to.have.lengthOf(1) - expect(videos[0].name).to.equal('video 1 server 2') - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/redundancy/redundancy-constraints.ts b/server/tests/api/redundancy/redundancy-constraints.ts deleted file mode 100644 index c86573168..000000000 --- a/server/tests/api/redundancy/redundancy-constraints.ts +++ /dev/null @@ -1,191 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createSingleServer, - killallServers, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test redundancy constraints', function () { - let remoteServer: PeerTubeServer - let localServer: PeerTubeServer - let servers: PeerTubeServer[] - - const remoteServerConfig = { - redundancy: { - videos: { - check_interval: '1 second', - strategies: [ - { - strategy: 'recently-added', - min_lifetime: '1 hour', - size: '100MB', - min_views: 0 - } - ] - } - } - } - - async function uploadWrapper (videoName: string) { - // Wait for transcoding - const { id } = await localServer.videos.upload({ attributes: { name: 'to transcode', privacy: VideoPrivacy.PRIVATE } }) - await waitJobs([ localServer ]) - - // Update video to schedule a federation - await localServer.videos.update({ id, attributes: { name: videoName, privacy: VideoPrivacy.PUBLIC } }) - } - - async function getTotalRedundanciesLocalServer () { - const body = await localServer.redundancy.listVideos({ target: 'my-videos' }) - - return body.total - } - - async function getTotalRedundanciesRemoteServer () { - const body = await remoteServer.redundancy.listVideos({ target: 'remote-videos' }) - - return body.total - } - - before(async function () { - this.timeout(120000) - - { - remoteServer = await createSingleServer(1, remoteServerConfig) - } - - { - const config = { - remote_redundancy: { - videos: { - accept_from: 'nobody' - } - } - } - localServer = await createSingleServer(2, config) - } - - servers = [ remoteServer, localServer ] - - // Get the access tokens - await setAccessTokensToServers(servers) - - await localServer.videos.upload({ attributes: { name: 'video 1 server 2' } }) - - await waitJobs(servers) - - // Server 1 and server 2 follow each other - await remoteServer.follows.follow({ hosts: [ localServer.url ] }) - await waitJobs(servers) - await remoteServer.redundancy.updateRedundancy({ host: localServer.host, redundancyAllowed: true }) - - await waitJobs(servers) - }) - - it('Should have redundancy on server 1 but not on server 2 with a nobody filter', async function () { - this.timeout(120000) - - await waitJobs(servers) - await remoteServer.servers.waitUntilLog('Duplicated ', 5) - await waitJobs(servers) - - { - const total = await getTotalRedundanciesRemoteServer() - expect(total).to.equal(1) - } - - { - const total = await getTotalRedundanciesLocalServer() - expect(total).to.equal(0) - } - }) - - it('Should have redundancy on server 1 and on server 2 with an anybody filter', async function () { - this.timeout(120000) - - const config = { - remote_redundancy: { - videos: { - accept_from: 'anybody' - } - } - } - await killallServers([ localServer ]) - await localServer.run(config) - - await uploadWrapper('video 2 server 2') - - await remoteServer.servers.waitUntilLog('Duplicated ', 10) - await waitJobs(servers) - - { - const total = await getTotalRedundanciesRemoteServer() - expect(total).to.equal(2) - } - - { - const total = await getTotalRedundanciesLocalServer() - expect(total).to.equal(1) - } - }) - - it('Should have redundancy on server 1 but not on server 2 with a followings filter', async function () { - this.timeout(120000) - - const config = { - remote_redundancy: { - videos: { - accept_from: 'followings' - } - } - } - await killallServers([ localServer ]) - await localServer.run(config) - - await uploadWrapper('video 3 server 2') - - await remoteServer.servers.waitUntilLog('Duplicated ', 15) - await waitJobs(servers) - - { - const total = await getTotalRedundanciesRemoteServer() - expect(total).to.equal(3) - } - - { - const total = await getTotalRedundanciesLocalServer() - expect(total).to.equal(1) - } - }) - - it('Should have redundancy on server 1 and on server 2 with followings filter now server 2 follows server 1', async function () { - this.timeout(120000) - - await localServer.follows.follow({ hosts: [ remoteServer.url ] }) - await waitJobs(servers) - - await uploadWrapper('video 4 server 2') - await remoteServer.servers.waitUntilLog('Duplicated ', 20) - await waitJobs(servers) - - { - const total = await getTotalRedundanciesRemoteServer() - expect(total).to.equal(4) - } - - { - const total = await getTotalRedundanciesLocalServer() - expect(total).to.equal(2) - } - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts deleted file mode 100644 index 0c5c27225..000000000 --- a/server/tests/api/redundancy/redundancy.ts +++ /dev/null @@ -1,742 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { readdir } from 'fs-extra' -import { decode as magnetUriDecode } from 'magnet-uri' -import { basename, join } from 'path' -import { checkSegmentHash, checkVideoFilesWereRemoved, saveVideoInServers } from '@server/tests/shared' -import { wait } from '@shared/core-utils' -import { - HttpStatusCode, - VideoDetails, - VideoFile, - VideoPrivacy, - VideoRedundancyStrategy, - VideoRedundancyStrategyWithManual -} from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - killallServers, - makeRawRequest, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -let servers: PeerTubeServer[] = [] -let video1Server2: VideoDetails - -async function checkMagnetWebseeds (file: VideoFile, baseWebseeds: string[], server: PeerTubeServer) { - const parsed = magnetUriDecode(file.magnetUri) - - for (const ws of baseWebseeds) { - const found = parsed.urlList.find(url => url === `${ws}${basename(file.fileUrl)}`) - expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined - } - - expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) - - for (const url of parsed.urlList) { - await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) - } -} - -async function createServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebVideo = true) { - const strategies: any[] = [] - - if (strategy !== null) { - strategies.push( - { - min_lifetime: '1 hour', - strategy, - size: '400KB', - - ...additionalParams - } - ) - } - - const config = { - transcoding: { - web_videos: { - enabled: withWebVideo - }, - hls: { - enabled: true - } - }, - redundancy: { - videos: { - check_interval: '5 seconds', - strategies - } - } - } - - servers = await createMultipleServers(3, config) - - // Get the access tokens - await setAccessTokensToServers(servers) - - { - const { id } = await servers[1].videos.upload({ attributes: { name: 'video 1 server 2' } }) - video1Server2 = await servers[1].videos.get({ id }) - - await servers[1].views.simulateView({ id }) - } - - await waitJobs(servers) - - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - // Server 1 and server 3 follow each other - await doubleFollow(servers[0], servers[2]) - // Server 2 and server 3 follow each other - await doubleFollow(servers[1], servers[2]) - - await waitJobs(servers) -} - -async function ensureSameFilenames (videoUUID: string) { - let webVideoFilenames: string[] - let hlsFilenames: string[] - - for (const server of servers) { - const video = await server.videos.getWithToken({ id: videoUUID }) - - // Ensure we use the same filenames that the origin - - const localWebVideoFilenames = video.files.map(f => basename(f.fileUrl)).sort() - const localHLSFilenames = video.streamingPlaylists[0].files.map(f => basename(f.fileUrl)).sort() - - if (webVideoFilenames) expect(webVideoFilenames).to.deep.equal(localWebVideoFilenames) - else webVideoFilenames = localWebVideoFilenames - - if (hlsFilenames) expect(hlsFilenames).to.deep.equal(localHLSFilenames) - else hlsFilenames = localHLSFilenames - } - - return { webVideoFilenames, hlsFilenames } -} - -async function check1WebSeed (videoUUID?: string) { - if (!videoUUID) videoUUID = video1Server2.uuid - - const webseeds = [ - `${servers[1].url}/static/web-videos/` - ] - - for (const server of servers) { - // With token to avoid issues with video follow constraints - const video = await server.videos.getWithToken({ id: videoUUID }) - - for (const f of video.files) { - await checkMagnetWebseeds(f, webseeds, server) - } - } - - await ensureSameFilenames(videoUUID) -} - -async function check2Webseeds (videoUUID?: string) { - if (!videoUUID) videoUUID = video1Server2.uuid - - const webseeds = [ - `${servers[0].url}/static/redundancy/`, - `${servers[1].url}/static/web-videos/` - ] - - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - - for (const file of video.files) { - await checkMagnetWebseeds(file, webseeds, server) - } - } - - const { webVideoFilenames } = await ensureSameFilenames(videoUUID) - - const directories = [ - servers[0].getDirectoryPath('redundancy'), - servers[1].getDirectoryPath('web-videos') - ] - - for (const directory of directories) { - const files = await readdir(directory) - expect(files).to.have.length.at.least(4) - - // Ensure we files exist on disk - expect(files.find(f => webVideoFilenames.includes(f))).to.exist - } -} - -async function check0PlaylistRedundancies (videoUUID?: string) { - if (!videoUUID) videoUUID = video1Server2.uuid - - for (const server of servers) { - // With token to avoid issues with video follow constraints - const video = await server.videos.getWithToken({ id: videoUUID }) - - expect(video.streamingPlaylists).to.be.an('array') - expect(video.streamingPlaylists).to.have.lengthOf(1) - expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0) - } - - await ensureSameFilenames(videoUUID) -} - -async function check1PlaylistRedundancies (videoUUID?: string) { - if (!videoUUID) videoUUID = video1Server2.uuid - - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - - expect(video.streamingPlaylists).to.have.lengthOf(1) - expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1) - - const redundancy = video.streamingPlaylists[0].redundancies[0] - - expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID) - } - - const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls/' + videoUUID - const baseUrlSegment = servers[0].url + '/static/redundancy/hls/' + videoUUID - - const video = await servers[0].videos.get({ id: videoUUID }) - const hlsPlaylist = video.streamingPlaylists[0] - - for (const resolution of [ 240, 360, 480, 720 ]) { - await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, resolution, hlsPlaylist }) - } - - const { hlsFilenames } = await ensureSameFilenames(videoUUID) - - const directories = [ - servers[0].getDirectoryPath('redundancy/hls'), - servers[1].getDirectoryPath('streaming-playlists/hls') - ] - - for (const directory of directories) { - const files = await readdir(join(directory, videoUUID)) - expect(files).to.have.length.at.least(4) - - // Ensure we files exist on disk - expect(files.find(f => hlsFilenames.includes(f))).to.exist - } -} - -async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) { - let totalSize: number = null - let statsLength = 1 - - if (strategy !== 'manual') { - totalSize = 409600 - statsLength = 2 - } - - const data = await servers[0].stats.get() - expect(data.videosRedundancy).to.have.lengthOf(statsLength) - - const stat = data.videosRedundancy[0] - expect(stat.strategy).to.equal(strategy) - expect(stat.totalSize).to.equal(totalSize) - - return stat -} - -async function checkStatsWith1Redundancy (strategy: VideoRedundancyStrategyWithManual, onlyHls = false) { - const stat = await checkStatsGlobal(strategy) - - expect(stat.totalUsed).to.be.at.least(1).and.below(409601) - expect(stat.totalVideoFiles).to.equal(onlyHls ? 4 : 8) - expect(stat.totalVideos).to.equal(1) -} - -async function checkStatsWithoutRedundancy (strategy: VideoRedundancyStrategyWithManual) { - const stat = await checkStatsGlobal(strategy) - - expect(stat.totalUsed).to.equal(0) - expect(stat.totalVideoFiles).to.equal(0) - expect(stat.totalVideos).to.equal(0) -} - -async function findServerFollows () { - const body = await servers[0].follows.getFollowings({ start: 0, count: 5, sort: '-createdAt' }) - const follows = body.data - const server2 = follows.find(f => f.following.host === `${servers[1].host}`) - const server3 = follows.find(f => f.following.host === `${servers[2].host}`) - - return { server2, server3 } -} - -async function enableRedundancyOnServer1 () { - await servers[0].redundancy.updateRedundancy({ host: servers[1].host, redundancyAllowed: true }) - - const { server2, server3 } = await findServerFollows() - - expect(server3).to.not.be.undefined - expect(server3.following.hostRedundancyAllowed).to.be.false - - expect(server2).to.not.be.undefined - expect(server2.following.hostRedundancyAllowed).to.be.true -} - -async function disableRedundancyOnServer1 () { - await servers[0].redundancy.updateRedundancy({ host: servers[1].host, redundancyAllowed: false }) - - const { server2, server3 } = await findServerFollows() - - expect(server3).to.not.be.undefined - expect(server3.following.hostRedundancyAllowed).to.be.false - - expect(server2).to.not.be.undefined - expect(server2.following.hostRedundancyAllowed).to.be.false -} - -describe('Test videos redundancy', function () { - - describe('With most-views strategy', function () { - const strategy = 'most-views' - - before(function () { - this.timeout(240000) - - return createServers(strategy) - }) - - it('Should have 1 webseed on the first video', async function () { - await check1WebSeed() - await check0PlaylistRedundancies() - await checkStatsWithoutRedundancy(strategy) - }) - - it('Should enable redundancy on server 1', function () { - return enableRedundancyOnServer1() - }) - - it('Should have 2 webseeds on the first video', async function () { - this.timeout(80000) - - await waitJobs(servers) - await servers[0].servers.waitUntilLog('Duplicated ', 5) - await waitJobs(servers) - - await check2Webseeds() - await check1PlaylistRedundancies() - await checkStatsWith1Redundancy(strategy) - }) - - it('Should undo redundancy on server 1 and remove duplicated videos', async function () { - this.timeout(80000) - - await disableRedundancyOnServer1() - - await waitJobs(servers) - await wait(5000) - - await check1WebSeed() - await check0PlaylistRedundancies() - - await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true }) - }) - - after(async function () { - return cleanupTests(servers) - }) - }) - - describe('With trending strategy', function () { - const strategy = 'trending' - - before(function () { - this.timeout(240000) - - return createServers(strategy) - }) - - it('Should have 1 webseed on the first video', async function () { - await check1WebSeed() - await check0PlaylistRedundancies() - await checkStatsWithoutRedundancy(strategy) - }) - - it('Should enable redundancy on server 1', function () { - return enableRedundancyOnServer1() - }) - - it('Should have 2 webseeds on the first video', async function () { - this.timeout(80000) - - await waitJobs(servers) - await servers[0].servers.waitUntilLog('Duplicated ', 5) - await waitJobs(servers) - - await check2Webseeds() - await check1PlaylistRedundancies() - await checkStatsWith1Redundancy(strategy) - }) - - it('Should unfollow server 3 and keep duplicated videos', async function () { - this.timeout(80000) - - await servers[0].follows.unfollow({ target: servers[2] }) - - await waitJobs(servers) - await wait(5000) - - await check2Webseeds() - await check1PlaylistRedundancies() - await checkStatsWith1Redundancy(strategy) - }) - - it('Should unfollow server 2 and remove duplicated videos', async function () { - this.timeout(80000) - - await servers[0].follows.unfollow({ target: servers[1] }) - - await waitJobs(servers) - await wait(5000) - - await check1WebSeed() - await check0PlaylistRedundancies() - - await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true }) - }) - - after(async function () { - await cleanupTests(servers) - }) - }) - - describe('With recently added strategy', function () { - const strategy = 'recently-added' - - before(function () { - this.timeout(240000) - - return createServers(strategy, { min_views: 3 }) - }) - - it('Should have 1 webseed on the first video', async function () { - await check1WebSeed() - await check0PlaylistRedundancies() - await checkStatsWithoutRedundancy(strategy) - }) - - it('Should enable redundancy on server 1', function () { - return enableRedundancyOnServer1() - }) - - it('Should still have 1 webseed on the first video', async function () { - this.timeout(80000) - - await waitJobs(servers) - await wait(15000) - await waitJobs(servers) - - await check1WebSeed() - await check0PlaylistRedundancies() - await checkStatsWithoutRedundancy(strategy) - }) - - it('Should view 2 times the first video to have > min_views config', async function () { - this.timeout(80000) - - await servers[0].views.simulateView({ id: video1Server2.uuid }) - await servers[2].views.simulateView({ id: video1Server2.uuid }) - - await wait(10000) - await waitJobs(servers) - }) - - it('Should have 2 webseeds on the first video', async function () { - this.timeout(80000) - - await waitJobs(servers) - await servers[0].servers.waitUntilLog('Duplicated ', 5) - await waitJobs(servers) - - await check2Webseeds() - await check1PlaylistRedundancies() - await checkStatsWith1Redundancy(strategy) - }) - - it('Should remove the video and the redundancy files', async function () { - this.timeout(20000) - - await saveVideoInServers(servers, video1Server2.uuid) - await servers[1].videos.remove({ id: video1Server2.uuid }) - - await waitJobs(servers) - - for (const server of servers) { - await checkVideoFilesWereRemoved({ server, video: server.store.videoDetails }) - } - }) - - after(async function () { - await cleanupTests(servers) - }) - }) - - describe('With only HLS files', function () { - const strategy = 'recently-added' - - before(async function () { - this.timeout(240000) - - await createServers(strategy, { min_views: 3 }, false) - }) - - it('Should have 0 playlist redundancy on the first video', async function () { - await check1WebSeed() - await check0PlaylistRedundancies() - }) - - it('Should enable redundancy on server 1', function () { - return enableRedundancyOnServer1() - }) - - it('Should still have 0 redundancy on the first video', async function () { - this.timeout(80000) - - await waitJobs(servers) - await wait(15000) - await waitJobs(servers) - - await check0PlaylistRedundancies() - await checkStatsWithoutRedundancy(strategy) - }) - - it('Should have 1 redundancy on the first video', async function () { - this.timeout(160000) - - await servers[0].views.simulateView({ id: video1Server2.uuid }) - await servers[2].views.simulateView({ id: video1Server2.uuid }) - - await wait(10000) - await waitJobs(servers) - - await waitJobs(servers) - await servers[0].servers.waitUntilLog('Duplicated ', 1) - await waitJobs(servers) - - await check1PlaylistRedundancies() - await checkStatsWith1Redundancy(strategy, true) - }) - - it('Should remove the video and the redundancy files', async function () { - this.timeout(20000) - - await saveVideoInServers(servers, video1Server2.uuid) - await servers[1].videos.remove({ id: video1Server2.uuid }) - - await waitJobs(servers) - - for (const server of servers) { - await checkVideoFilesWereRemoved({ server, video: server.store.videoDetails }) - } - }) - - after(async function () { - await cleanupTests(servers) - }) - }) - - describe('With manual strategy', function () { - before(function () { - this.timeout(240000) - - return createServers(null) - }) - - it('Should have 1 webseed on the first video', async function () { - await check1WebSeed() - await check0PlaylistRedundancies() - await checkStatsWithoutRedundancy('manual') - }) - - it('Should create a redundancy on first video', async function () { - await servers[0].redundancy.addVideo({ videoId: video1Server2.id }) - }) - - it('Should have 2 webseeds on the first video', async function () { - this.timeout(80000) - - await waitJobs(servers) - await servers[0].servers.waitUntilLog('Duplicated ', 5) - await waitJobs(servers) - - await check2Webseeds() - await check1PlaylistRedundancies() - await checkStatsWith1Redundancy('manual') - }) - - it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () { - this.timeout(80000) - - const body = await servers[0].redundancy.listVideos({ target: 'remote-videos' }) - - const videos = body.data - expect(videos).to.have.lengthOf(1) - - const video = videos[0] - - for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) { - await servers[0].redundancy.removeVideo({ redundancyId: r.id }) - } - - await waitJobs(servers) - await wait(5000) - - await check1WebSeed() - await check0PlaylistRedundancies() - - await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true }) - }) - - after(async function () { - await cleanupTests(servers) - }) - }) - - describe('Test expiration', function () { - const strategy = 'recently-added' - - async function checkContains (servers: PeerTubeServer[], str: string) { - for (const server of servers) { - const video = await server.videos.get({ id: video1Server2.uuid }) - - for (const f of video.files) { - expect(f.magnetUri).to.contain(str) - } - } - } - - async function checkNotContains (servers: PeerTubeServer[], str: string) { - for (const server of servers) { - const video = await server.videos.get({ id: video1Server2.uuid }) - - for (const f of video.files) { - expect(f.magnetUri).to.not.contain(str) - } - } - } - - before(async function () { - this.timeout(240000) - - await createServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) - - await enableRedundancyOnServer1() - }) - - it('Should still have 2 webseeds after 10 seconds', async function () { - this.timeout(80000) - - await wait(10000) - - try { - await checkContains(servers, 'http%3A%2F%2F' + servers[0].hostname + '%3A' + servers[0].port) - } catch { - // Maybe a server deleted a redundancy in the scheduler - await wait(2000) - - await checkContains(servers, 'http%3A%2F%2F' + servers[0].hostname + '%3A' + servers[0].port) - } - }) - - it('Should stop server 1 and expire video redundancy', async function () { - this.timeout(80000) - - await killallServers([ servers[0] ]) - - await wait(15000) - - await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2F' + servers[0].port + '%3A' + servers[0].port) - }) - - after(async function () { - await cleanupTests(servers) - }) - }) - - describe('Test file replacement', function () { - let video2Server2UUID: string - const strategy = 'recently-added' - - before(async function () { - this.timeout(240000) - - await createServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) - - await enableRedundancyOnServer1() - - await waitJobs(servers) - await servers[0].servers.waitUntilLog('Duplicated ', 5) - await waitJobs(servers) - - await check2Webseeds() - await check1PlaylistRedundancies() - await checkStatsWith1Redundancy(strategy) - - const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video 2 server 2', privacy: VideoPrivacy.PRIVATE } }) - video2Server2UUID = uuid - - // Wait transcoding before federation - await waitJobs(servers) - - await servers[1].videos.update({ id: video2Server2UUID, attributes: { privacy: VideoPrivacy.PUBLIC } }) - }) - - it('Should cache video 2 webseeds on the first video', async function () { - this.timeout(240000) - - await waitJobs(servers) - - let checked = false - - while (checked === false) { - await wait(1000) - - try { - await check1WebSeed() - await check0PlaylistRedundancies() - - await check2Webseeds(video2Server2UUID) - await check1PlaylistRedundancies(video2Server2UUID) - - checked = true - } catch { - checked = false - } - } - }) - - it('Should disable strategy and remove redundancies', async function () { - this.timeout(80000) - - await waitJobs(servers) - - await killallServers([ servers[0] ]) - await servers[0].run({ - redundancy: { - videos: { - check_interval: '1 second', - strategies: [] - } - } - }) - - await waitJobs(servers) - - await checkVideoFilesWereRemoved({ server: servers[0], video: video1Server2, onlyVideoFiles: true }) - }) - - after(async function () { - await cleanupTests(servers) - }) - }) -}) diff --git a/server/tests/api/runners/index.ts b/server/tests/api/runners/index.ts deleted file mode 100644 index 642a3a96d..000000000 --- a/server/tests/api/runners/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -export * from './runner-common' -export * from './runner-live-transcoding' -export * from './runner-socket' -export * from './runner-studio-transcoding' -export * from './runner-vod-transcoding' diff --git a/server/tests/api/runners/runner-common.ts b/server/tests/api/runners/runner-common.ts deleted file mode 100644 index 9b2eb8b27..000000000 --- a/server/tests/api/runners/runner-common.ts +++ /dev/null @@ -1,743 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { wait } from '@shared/core-utils' -import { - HttpStatusCode, - Runner, - RunnerJob, - RunnerJobAdmin, - RunnerJobState, - RunnerJobVODWebVideoTranscodingPayload, - RunnerRegistrationToken -} from '@shared/models' -import { - cleanupTests, - createSingleServer, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - waitJobs -} from '@shared/server-commands' - -describe('Test runner common actions', function () { - let server: PeerTubeServer - let registrationToken: string - let runnerToken: string - let jobMaxPriority: string - - before(async function () { - this.timeout(120_000) - - server = await createSingleServer(1, { - remote_runners: { - stalled_jobs: { - vod: '5 seconds' - } - } - }) - - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - await server.config.enableTranscoding({ hls: true, webVideo: true }) - await server.config.enableRemoteTranscoding() - }) - - describe('Managing runner registration tokens', function () { - let base: RunnerRegistrationToken[] - let registrationTokenToDelete: RunnerRegistrationToken - - it('Should have a default registration token', async function () { - const { total, data } = await server.runnerRegistrationTokens.list() - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - - const token = data[0] - expect(token.id).to.exist - expect(token.createdAt).to.exist - expect(token.updatedAt).to.exist - expect(token.registeredRunnersCount).to.equal(0) - expect(token.registrationToken).to.exist - }) - - it('Should create other registration tokens', async function () { - await server.runnerRegistrationTokens.generate() - await server.runnerRegistrationTokens.generate() - - const { total, data } = await server.runnerRegistrationTokens.list() - expect(total).to.equal(3) - expect(data).to.have.lengthOf(3) - }) - - it('Should list registration tokens', async function () { - { - const { total, data } = await server.runnerRegistrationTokens.list({ sort: 'createdAt' }) - expect(total).to.equal(3) - expect(data).to.have.lengthOf(3) - expect(new Date(data[0].createdAt)).to.be.below(new Date(data[1].createdAt)) - expect(new Date(data[1].createdAt)).to.be.below(new Date(data[2].createdAt)) - - base = data - - registrationTokenToDelete = data[0] - registrationToken = data[1].registrationToken - } - - { - const { total, data } = await server.runnerRegistrationTokens.list({ sort: '-createdAt', start: 2, count: 1 }) - expect(total).to.equal(3) - expect(data).to.have.lengthOf(1) - expect(data[0].registrationToken).to.equal(base[0].registrationToken) - } - }) - - it('Should have appropriate registeredRunnersCount for registration tokens', async function () { - await server.runners.register({ name: 'to delete 1', registrationToken: registrationTokenToDelete.registrationToken }) - await server.runners.register({ name: 'to delete 2', registrationToken: registrationTokenToDelete.registrationToken }) - - const { data } = await server.runnerRegistrationTokens.list() - - for (const d of data) { - if (d.registrationToken === registrationTokenToDelete.registrationToken) { - expect(d.registeredRunnersCount).to.equal(2) - } else { - expect(d.registeredRunnersCount).to.equal(0) - } - } - - const { data: runners } = await server.runners.list() - expect(runners).to.have.lengthOf(2) - }) - - it('Should delete a registration token', async function () { - await server.runnerRegistrationTokens.delete({ id: registrationTokenToDelete.id }) - - const { total, data } = await server.runnerRegistrationTokens.list({ sort: 'createdAt' }) - expect(total).to.equal(2) - expect(data).to.have.lengthOf(2) - - for (const d of data) { - expect(d.registeredRunnersCount).to.equal(0) - expect(d.registrationToken).to.not.equal(registrationTokenToDelete.registrationToken) - } - }) - - it('Should have removed runners of this registration token', async function () { - const { data: runners } = await server.runners.list() - expect(runners).to.have.lengthOf(0) - }) - }) - - describe('Managing runners', function () { - let toDelete: Runner - - it('Should not have runners available', async function () { - const { total, data } = await server.runners.list() - - expect(data).to.have.lengthOf(0) - expect(total).to.equal(0) - }) - - it('Should register runners', async function () { - const now = new Date() - - const result = await server.runners.register({ - name: 'runner 1', - description: 'my super runner 1', - registrationToken - }) - expect(result.runnerToken).to.exist - runnerToken = result.runnerToken - - await server.runners.register({ - name: 'runner 2', - registrationToken - }) - - const { total, data } = await server.runners.list({ sort: 'createdAt' }) - expect(total).to.equal(2) - expect(data).to.have.lengthOf(2) - - for (const d of data) { - expect(d.id).to.exist - expect(d.createdAt).to.exist - expect(d.updatedAt).to.exist - expect(new Date(d.createdAt)).to.be.above(now) - expect(new Date(d.updatedAt)).to.be.above(now) - expect(new Date(d.lastContact)).to.be.above(now) - expect(d.ip).to.exist - } - - expect(data[0].name).to.equal('runner 1') - expect(data[0].description).to.equal('my super runner 1') - - expect(data[1].name).to.equal('runner 2') - expect(data[1].description).to.be.null - - toDelete = data[1] - }) - - it('Should list runners', async function () { - const { total, data } = await server.runners.list({ sort: '-createdAt', start: 1, count: 1 }) - - expect(total).to.equal(2) - expect(data).to.have.lengthOf(1) - expect(data[0].name).to.equal('runner 1') - }) - - it('Should delete a runner', async function () { - await server.runners.delete({ id: toDelete.id }) - - const { total, data } = await server.runners.list() - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - expect(data[0].name).to.equal('runner 1') - }) - - it('Should unregister a runner', async function () { - const registered = await server.runners.autoRegisterRunner() - - { - const { total, data } = await server.runners.list() - expect(total).to.equal(2) - expect(data).to.have.lengthOf(2) - } - - await server.runners.unregister({ runnerToken: registered }) - - { - const { total, data } = await server.runners.list() - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - expect(data[0].name).to.equal('runner 1') - } - }) - }) - - describe('Managing runner jobs', function () { - let jobUUID: string - let jobToken: string - let lastRunnerContact: Date - let failedJob: RunnerJob - - async function checkMainJobState ( - mainJobState: RunnerJobState, - otherJobStates: RunnerJobState[] = [ RunnerJobState.PENDING, RunnerJobState.WAITING_FOR_PARENT_JOB ] - ) { - const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' }) - - for (const job of data) { - if (job.uuid === jobUUID) { - expect(job.state.id).to.equal(mainJobState) - } else { - expect(otherJobStates).to.include(job.state.id) - } - } - } - - function getMainJob () { - return server.runnerJobs.getJob({ uuid: jobUUID }) - } - - describe('List jobs', function () { - - it('Should not have jobs', async function () { - const { total, data } = await server.runnerJobs.list() - - expect(data).to.have.lengthOf(0) - expect(total).to.equal(0) - }) - - it('Should upload a video and have available jobs', async function () { - await server.videos.quickUpload({ name: 'to transcode' }) - await waitJobs([ server ]) - - const { total, data } = await server.runnerJobs.list() - - expect(data).to.have.lengthOf(10) - expect(total).to.equal(10) - - for (const job of data) { - expect(job.startedAt).to.not.exist - expect(job.finishedAt).to.not.exist - expect(job.payload).to.exist - expect(job.privatePayload).to.exist - } - - const hlsJobs = data.filter(d => d.type === 'vod-hls-transcoding') - const webVideoJobs = data.filter(d => d.type === 'vod-web-video-transcoding') - - expect(hlsJobs).to.have.lengthOf(5) - expect(webVideoJobs).to.have.lengthOf(5) - - const pendingJobs = data.filter(d => d.state.id === RunnerJobState.PENDING) - const waitingJobs = data.filter(d => d.state.id === RunnerJobState.WAITING_FOR_PARENT_JOB) - - expect(pendingJobs).to.have.lengthOf(1) - expect(waitingJobs).to.have.lengthOf(9) - }) - - it('Should upload another video and list/sort jobs', async function () { - await server.videos.quickUpload({ name: 'to transcode 2' }) - await waitJobs([ server ]) - - { - const { total, data } = await server.runnerJobs.list({ start: 0, count: 30 }) - - expect(data).to.have.lengthOf(20) - expect(total).to.equal(20) - - jobUUID = data[16].uuid - } - - { - const { total, data } = await server.runnerJobs.list({ start: 3, count: 1, sort: 'createdAt' }) - expect(total).to.equal(20) - - expect(data).to.have.lengthOf(1) - expect(data[0].uuid).to.equal(jobUUID) - } - - { - let previousPriority = Infinity - const { total, data } = await server.runnerJobs.list({ start: 0, count: 100, sort: '-priority' }) - expect(total).to.equal(20) - - for (const job of data) { - expect(job.priority).to.be.at.most(previousPriority) - previousPriority = job.priority - - if (job.state.id === RunnerJobState.PENDING) { - jobMaxPriority = job.uuid - } - } - } - }) - - it('Should search jobs', async function () { - { - const { total, data } = await server.runnerJobs.list({ search: jobUUID }) - - expect(data).to.have.lengthOf(1) - expect(total).to.equal(1) - - expect(data[0].uuid).to.equal(jobUUID) - } - - { - const { total, data } = await server.runnerJobs.list({ search: 'toto' }) - - expect(data).to.have.lengthOf(0) - expect(total).to.equal(0) - } - - { - const { total, data } = await server.runnerJobs.list({ search: 'hls' }) - - expect(data).to.not.have.lengthOf(0) - expect(total).to.not.equal(0) - - for (const job of data) { - expect(job.type).to.include('hls') - } - } - }) - - it('Should filter jobs', async function () { - { - const { total, data } = await server.runnerJobs.list({ stateOneOf: [ RunnerJobState.WAITING_FOR_PARENT_JOB ] }) - - expect(data).to.not.have.lengthOf(0) - expect(total).to.not.equal(0) - - for (const job of data) { - expect(job.state.label).to.equal('Waiting for parent job to finish') - } - } - - { - const { total, data } = await server.runnerJobs.list({ stateOneOf: [ RunnerJobState.COMPLETED ] }) - - expect(data).to.have.lengthOf(0) - expect(total).to.equal(0) - } - }) - }) - - describe('Accept/update/abort/process a job', function () { - - it('Should request available jobs', async function () { - lastRunnerContact = new Date() - - const { availableJobs } = await server.runnerJobs.request({ runnerToken }) - - // Only optimize jobs are available - expect(availableJobs).to.have.lengthOf(2) - - for (const job of availableJobs) { - expect(job.uuid).to.exist - expect(job.payload.input).to.exist - expect((job.payload as RunnerJobVODWebVideoTranscodingPayload).output).to.exist - - expect((job as RunnerJobAdmin).privatePayload).to.not.exist - } - - const hlsJobs = availableJobs.filter(d => d.type === 'vod-hls-transcoding') - const webVideoJobs = availableJobs.filter(d => d.type === 'vod-web-video-transcoding') - - expect(hlsJobs).to.have.lengthOf(0) - expect(webVideoJobs).to.have.lengthOf(2) - - jobUUID = webVideoJobs[0].uuid - }) - - it('Should have sorted available jobs by priority', async function () { - const { availableJobs } = await server.runnerJobs.request({ runnerToken }) - - expect(availableJobs[0].uuid).to.equal(jobMaxPriority) - }) - - it('Should have last runner contact updated', async function () { - await wait(1000) - - const { data } = await server.runners.list({ sort: 'createdAt' }) - expect(new Date(data[0].lastContact)).to.be.above(lastRunnerContact) - }) - - it('Should accept a job', async function () { - const startedAt = new Date() - - const { job } = await server.runnerJobs.accept({ runnerToken, jobUUID }) - jobToken = job.jobToken - - const checkProcessingJob = (job: RunnerJob & { jobToken?: string }, fromAccept: boolean) => { - expect(job.uuid).to.equal(jobUUID) - - expect(job.type).to.equal('vod-web-video-transcoding') - expect(job.state.label).to.equal('Processing') - expect(job.state.id).to.equal(RunnerJobState.PROCESSING) - - expect(job.runner).to.exist - expect(job.runner.name).to.equal('runner 1') - expect(job.runner.description).to.equal('my super runner 1') - - expect(job.progress).to.be.null - - expect(job.startedAt).to.exist - expect(new Date(job.startedAt)).to.be.above(startedAt) - - expect(job.finishedAt).to.not.exist - - expect(job.failures).to.equal(0) - - expect(job.payload).to.exist - - if (fromAccept) { - expect(job.jobToken).to.exist - expect((job as RunnerJobAdmin).privatePayload).to.not.exist - } else { - expect(job.jobToken).to.not.exist - expect((job as RunnerJobAdmin).privatePayload).to.exist - } - } - - checkProcessingJob(job, true) - - const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' }) - - const processingJob = data.find(j => j.uuid === jobUUID) - checkProcessingJob(processingJob, false) - - await checkMainJobState(RunnerJobState.PROCESSING) - }) - - it('Should update a job', async function () { - await server.runnerJobs.update({ runnerToken, jobUUID, jobToken, progress: 53 }) - - const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' }) - - for (const job of data) { - if (job.state.id === RunnerJobState.PROCESSING) { - expect(job.progress).to.equal(53) - } else { - expect(job.progress).to.be.null - } - } - }) - - it('Should abort a job', async function () { - await server.runnerJobs.abort({ runnerToken, jobUUID, jobToken, reason: 'for tests' }) - - await checkMainJobState(RunnerJobState.PENDING) - - const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' }) - for (const job of data) { - expect(job.progress).to.be.null - } - }) - - it('Should accept the same job again and post a success', async function () { - const { availableJobs } = await server.runnerJobs.request({ runnerToken }) - expect(availableJobs.find(j => j.uuid === jobUUID)).to.exist - - const { job } = await server.runnerJobs.accept({ runnerToken, jobUUID }) - jobToken = job.jobToken - - await checkMainJobState(RunnerJobState.PROCESSING) - - const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' }) - - for (const job of data) { - expect(job.progress).to.be.null - } - - const payload = { - videoFile: 'video_short.mp4' - } - - await server.runnerJobs.success({ runnerToken, jobUUID, jobToken, payload }) - }) - - it('Should not have available jobs anymore', async function () { - await checkMainJobState(RunnerJobState.COMPLETED) - - const job = await getMainJob() - expect(job.finishedAt).to.exist - - const { availableJobs } = await server.runnerJobs.request({ runnerToken }) - expect(availableJobs.find(j => j.uuid === jobUUID)).to.not.exist - }) - }) - - describe('Error job', function () { - - it('Should accept another job and post an error', async function () { - await server.runnerJobs.cancelAllJobs() - await server.videos.quickUpload({ name: 'video' }) - await waitJobs([ server ]) - - const { availableJobs } = await server.runnerJobs.request({ runnerToken }) - jobUUID = availableJobs[0].uuid - - const { job } = await server.runnerJobs.accept({ runnerToken, jobUUID }) - jobToken = job.jobToken - - await server.runnerJobs.error({ runnerToken, jobUUID, jobToken, message: 'Error' }) - }) - - it('Should have job failures increased', async function () { - const job = await getMainJob() - expect(job.state.id).to.equal(RunnerJobState.PENDING) - expect(job.failures).to.equal(1) - expect(job.error).to.be.null - expect(job.progress).to.be.null - expect(job.finishedAt).to.not.exist - }) - - it('Should error a job when job attempts is too big', async function () { - for (let i = 0; i < 4; i++) { - const { job } = await server.runnerJobs.accept({ runnerToken, jobUUID }) - jobToken = job.jobToken - - await server.runnerJobs.error({ runnerToken, jobUUID, jobToken, message: 'Error ' + i }) - } - - const job = await getMainJob() - expect(job.failures).to.equal(5) - expect(job.state.id).to.equal(RunnerJobState.ERRORED) - expect(job.state.label).to.equal('Errored') - expect(job.error).to.equal('Error 3') - expect(job.progress).to.be.null - expect(job.finishedAt).to.exist - - failedJob = job - }) - - it('Should have failed children jobs too', async function () { - const { data } = await server.runnerJobs.list({ count: 50, sort: '-updatedAt' }) - - const children = data.filter(j => j.parent?.uuid === failedJob.uuid) - expect(children).to.have.lengthOf(9) - - for (const child of children) { - expect(child.parent.uuid).to.equal(failedJob.uuid) - expect(child.parent.type).to.equal(failedJob.type) - expect(child.parent.state.id).to.equal(failedJob.state.id) - expect(child.parent.state.label).to.equal(failedJob.state.label) - - expect(child.state.id).to.equal(RunnerJobState.PARENT_ERRORED) - expect(child.state.label).to.equal('Parent job failed') - } - }) - }) - - describe('Cancel', function () { - - it('Should cancel a pending job', async function () { - await server.videos.quickUpload({ name: 'video' }) - await waitJobs([ server ]) - - { - const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' }) - - const pendingJob = data.find(j => j.state.id === RunnerJobState.PENDING) - jobUUID = pendingJob.uuid - - await server.runnerJobs.cancelByAdmin({ jobUUID }) - } - - { - const job = await getMainJob() - expect(job.state.id).to.equal(RunnerJobState.CANCELLED) - expect(job.state.label).to.equal('Cancelled') - } - - { - const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' }) - const children = data.filter(j => j.parent?.uuid === jobUUID) - expect(children).to.have.lengthOf(9) - - for (const child of children) { - expect(child.state.id).to.equal(RunnerJobState.PARENT_CANCELLED) - } - } - }) - - it('Should cancel an already accepted job and skip success/error', async function () { - await server.videos.quickUpload({ name: 'video' }) - await waitJobs([ server ]) - - const { availableJobs } = await server.runnerJobs.request({ runnerToken }) - jobUUID = availableJobs[0].uuid - - const { job } = await server.runnerJobs.accept({ runnerToken, jobUUID }) - jobToken = job.jobToken - - await server.runnerJobs.cancelByAdmin({ jobUUID }) - - await server.runnerJobs.abort({ runnerToken, jobUUID, jobToken, reason: 'aborted', expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - }) - - describe('Remove', function () { - - it('Should remove a pending job', async function () { - await server.videos.quickUpload({ name: 'video' }) - await waitJobs([ server ]) - - { - const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' }) - - const pendingJob = data.find(j => j.state.id === RunnerJobState.PENDING) - jobUUID = pendingJob.uuid - - await server.runnerJobs.deleteByAdmin({ jobUUID }) - } - - { - const { data } = await server.runnerJobs.list({ count: 10, sort: '-updatedAt' }) - - const parent = data.find(j => j.uuid === jobUUID) - expect(parent).to.not.exist - - const children = data.filter(j => j.parent?.uuid === jobUUID) - expect(children).to.have.lengthOf(0) - } - }) - }) - - describe('Stalled jobs', function () { - - it('Should abort stalled jobs', async function () { - this.timeout(60000) - - await server.videos.quickUpload({ name: 'video' }) - await server.videos.quickUpload({ name: 'video' }) - await waitJobs([ server ]) - - const { job: job1 } = await server.runnerJobs.autoAccept({ runnerToken }) - const { job: stalledJob } = await server.runnerJobs.autoAccept({ runnerToken }) - - for (let i = 0; i < 6; i++) { - await wait(2000) - - await server.runnerJobs.update({ runnerToken, jobToken: job1.jobToken, jobUUID: job1.uuid }) - } - - const refreshedJob1 = await server.runnerJobs.getJob({ uuid: job1.uuid }) - const refreshedStalledJob = await server.runnerJobs.getJob({ uuid: stalledJob.uuid }) - - expect(refreshedJob1.state.id).to.equal(RunnerJobState.PROCESSING) - expect(refreshedStalledJob.state.id).to.equal(RunnerJobState.PENDING) - }) - }) - - describe('Rate limit', function () { - - before(async function () { - this.timeout(60000) - - await server.kill() - - await server.run({ - rates_limit: { - api: { - max: 10 - } - } - }) - }) - - it('Should rate limit an unknown runner, but not a registered one', async function () { - this.timeout(60000) - - await server.videos.quickUpload({ name: 'video' }) - await waitJobs([ server ]) - - const { job } = await server.runnerJobs.autoAccept({ runnerToken }) - - for (let i = 0; i < 20; i++) { - try { - await server.runnerJobs.request({ runnerToken }) - await server.runnerJobs.update({ runnerToken, jobToken: job.jobToken, jobUUID: job.uuid }) - } catch {} - } - - // Invalid - { - await server.runnerJobs.request({ runnerToken: 'toto', expectedStatus: HttpStatusCode.TOO_MANY_REQUESTS_429 }) - await server.runnerJobs.update({ - runnerToken: 'toto', - jobToken: job.jobToken, - jobUUID: job.uuid, - expectedStatus: HttpStatusCode.TOO_MANY_REQUESTS_429 - }) - } - - // Not provided - { - await server.runnerJobs.request({ runnerToken: undefined, expectedStatus: HttpStatusCode.TOO_MANY_REQUESTS_429 }) - await server.runnerJobs.update({ - runnerToken: undefined, - jobToken: job.jobToken, - jobUUID: job.uuid, - expectedStatus: HttpStatusCode.TOO_MANY_REQUESTS_429 - }) - } - - // Registered - { - await server.runnerJobs.request({ runnerToken }) - await server.runnerJobs.update({ runnerToken, jobToken: job.jobToken, jobUUID: job.uuid }) - } - }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/runners/runner-live-transcoding.ts b/server/tests/api/runners/runner-live-transcoding.ts deleted file mode 100644 index b11d54039..000000000 --- a/server/tests/api/runners/runner-live-transcoding.ts +++ /dev/null @@ -1,330 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { FfmpegCommand } from 'fluent-ffmpeg' -import { readFile } from 'fs-extra' -import { buildAbsoluteFixturePath, wait } from '@shared/core-utils' -import { - HttpStatusCode, - LiveRTMPHLSTranscodingUpdatePayload, - LiveVideo, - LiveVideoError, - RunnerJob, - RunnerJobLiveRTMPHLSTranscodingPayload, - Video, - VideoPrivacy, - VideoState -} from '@shared/models' -import { - cleanupTests, - createSingleServer, - makeRawRequest, - PeerTubeServer, - sendRTMPStream, - setAccessTokensToServers, - setDefaultVideoChannel, - stopFfmpeg, - testFfmpegStreamError, - waitJobs -} from '@shared/server-commands' - -describe('Test runner live transcoding', function () { - let server: PeerTubeServer - let runnerToken: string - let baseUrl: string - - before(async function () { - this.timeout(120_000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - await server.config.enableRemoteTranscoding() - await server.config.enableTranscoding() - runnerToken = await server.runners.autoRegisterRunner() - - baseUrl = server.url + '/static/streaming-playlists/hls' - }) - - describe('Without transcoding enabled', function () { - - before(async function () { - await server.config.enableLive({ - allowReplay: false, - resolutions: 'min', - transcoding: false - }) - }) - - it('Should not have available jobs', async function () { - this.timeout(120000) - - const { live, video } = await server.live.quickCreate({ permanentLive: true, saveReplay: false, privacy: VideoPrivacy.PUBLIC }) - - const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) - await server.live.waitUntilPublished({ videoId: video.id }) - - await waitJobs([ server ]) - - const { availableJobs } = await server.runnerJobs.requestLive({ runnerToken }) - expect(availableJobs).to.have.lengthOf(0) - - await stopFfmpeg(ffmpegCommand) - }) - }) - - describe('With transcoding enabled on classic live', function () { - let live: LiveVideo - let video: Video - let ffmpegCommand: FfmpegCommand - let jobUUID: string - let acceptedJob: RunnerJob & { jobToken: string } - - async function testPlaylistFile (fixture: string, expected: string) { - const text = await server.streamingPlaylists.get({ url: `${baseUrl}/${video.uuid}/${fixture}` }) - expect(await readFile(buildAbsoluteFixturePath(expected), 'utf-8')).to.equal(text) - - } - - async function testTSFile (fixture: string, expected: string) { - const { body } = await makeRawRequest({ url: `${baseUrl}/${video.uuid}/${fixture}`, expectedStatus: HttpStatusCode.OK_200 }) - expect(await readFile(buildAbsoluteFixturePath(expected))).to.deep.equal(body) - } - - before(async function () { - await server.config.enableLive({ - allowReplay: true, - resolutions: 'max', - transcoding: true - }) - }) - - it('Should publish a a live and have available jobs', async function () { - this.timeout(120000) - - const data = await server.live.quickCreate({ permanentLive: false, saveReplay: false, privacy: VideoPrivacy.PUBLIC }) - live = data.live - video = data.video - - ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) - await waitJobs([ server ]) - - const job = await server.runnerJobs.requestLiveJob(runnerToken) - jobUUID = job.uuid - - expect(job.type).to.equal('live-rtmp-hls-transcoding') - expect(job.payload.input.rtmpUrl).to.exist - - expect(job.payload.output.toTranscode).to.have.lengthOf(5) - - for (const { resolution, fps } of job.payload.output.toTranscode) { - expect([ 720, 480, 360, 240, 144 ]).to.contain(resolution) - - expect(fps).to.be.above(25) - expect(fps).to.be.below(70) - } - }) - - it('Should update the live with a new chunk', async function () { - this.timeout(120000) - - const { job } = await server.runnerJobs.accept({ jobUUID, runnerToken }) - acceptedJob = job - - { - const payload: LiveRTMPHLSTranscodingUpdatePayload = { - masterPlaylistFile: 'live/master.m3u8', - resolutionPlaylistFile: 'live/0.m3u8', - resolutionPlaylistFilename: '0.m3u8', - type: 'add-chunk', - videoChunkFile: 'live/0-000067.ts', - videoChunkFilename: '0-000067.ts' - } - await server.runnerJobs.update({ jobUUID, runnerToken, jobToken: job.jobToken, payload, progress: 50 }) - - const updatedJob = await server.runnerJobs.getJob({ uuid: job.uuid }) - expect(updatedJob.progress).to.equal(50) - } - - { - const payload: LiveRTMPHLSTranscodingUpdatePayload = { - resolutionPlaylistFile: 'live/1.m3u8', - resolutionPlaylistFilename: '1.m3u8', - type: 'add-chunk', - videoChunkFile: 'live/1-000068.ts', - videoChunkFilename: '1-000068.ts' - } - await server.runnerJobs.update({ jobUUID, runnerToken, jobToken: job.jobToken, payload }) - } - - await wait(1000) - - await testPlaylistFile('master.m3u8', 'live/master.m3u8') - await testPlaylistFile('0.m3u8', 'live/0.m3u8') - await testPlaylistFile('1.m3u8', 'live/1.m3u8') - - await testTSFile('0-000067.ts', 'live/0-000067.ts') - await testTSFile('1-000068.ts', 'live/1-000068.ts') - }) - - it('Should replace existing m3u8 on update', async function () { - this.timeout(120000) - - const payload: LiveRTMPHLSTranscodingUpdatePayload = { - masterPlaylistFile: 'live/1.m3u8', - resolutionPlaylistFilename: '0.m3u8', - resolutionPlaylistFile: 'live/1.m3u8', - type: 'add-chunk', - videoChunkFile: 'live/1-000069.ts', - videoChunkFilename: '1-000068.ts' - } - await server.runnerJobs.update({ jobUUID, runnerToken, jobToken: acceptedJob.jobToken, payload }) - await wait(1000) - - await testPlaylistFile('master.m3u8', 'live/1.m3u8') - await testPlaylistFile('0.m3u8', 'live/1.m3u8') - await testTSFile('1-000068.ts', 'live/1-000069.ts') - }) - - it('Should update the live with removed chunks', async function () { - this.timeout(120000) - - const payload: LiveRTMPHLSTranscodingUpdatePayload = { - resolutionPlaylistFile: 'live/0.m3u8', - resolutionPlaylistFilename: '0.m3u8', - type: 'remove-chunk', - videoChunkFilename: '1-000068.ts' - } - await server.runnerJobs.update({ jobUUID, runnerToken, jobToken: acceptedJob.jobToken, payload }) - - await wait(1000) - - await server.streamingPlaylists.get({ url: `${baseUrl}/${video.uuid}/master.m3u8` }) - await server.streamingPlaylists.get({ url: `${baseUrl}/${video.uuid}/0.m3u8` }) - await server.streamingPlaylists.get({ url: `${baseUrl}/${video.uuid}/1.m3u8` }) - await makeRawRequest({ url: `${baseUrl}/${video.uuid}/0-000067.ts`, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ url: `${baseUrl}/${video.uuid}/1-000068.ts`, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should complete the live and save the replay', async function () { - this.timeout(120000) - - for (const segment of [ '0-000069.ts', '0-000070.ts' ]) { - const payload: LiveRTMPHLSTranscodingUpdatePayload = { - masterPlaylistFile: 'live/master.m3u8', - resolutionPlaylistFilename: '0.m3u8', - resolutionPlaylistFile: 'live/0.m3u8', - type: 'add-chunk', - videoChunkFile: 'live/' + segment, - videoChunkFilename: segment - } - await server.runnerJobs.update({ jobUUID, runnerToken, jobToken: acceptedJob.jobToken, payload }) - - await wait(1000) - } - - await waitJobs([ server ]) - - { - const { state } = await server.videos.get({ id: video.uuid }) - expect(state.id).to.equal(VideoState.PUBLISHED) - } - - await stopFfmpeg(ffmpegCommand) - - await server.runnerJobs.success({ jobUUID, runnerToken, jobToken: acceptedJob.jobToken, payload: {} }) - - await wait(1500) - await waitJobs([ server ]) - - { - const { state } = await server.videos.get({ id: video.uuid }) - expect(state.id).to.equal(VideoState.LIVE_ENDED) - - const session = await server.live.findLatestSession({ videoId: video.uuid }) - expect(session.error).to.be.null - } - }) - }) - - describe('With transcoding enabled on cancelled/aborted/errored live', function () { - let live: LiveVideo - let video: Video - let ffmpegCommand: FfmpegCommand - - async function prepare () { - ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) - await server.runnerJobs.requestLiveJob(runnerToken) - - const { job } = await server.runnerJobs.autoAccept({ runnerToken, type: 'live-rtmp-hls-transcoding' }) - - return job - } - - async function checkSessionError (error: LiveVideoError) { - await wait(1500) - await waitJobs([ server ]) - - const session = await server.live.findLatestSession({ videoId: video.uuid }) - expect(session.error).to.equal(error) - } - - before(async function () { - await server.config.enableLive({ - allowReplay: true, - resolutions: 'max', - transcoding: true - }) - - const data = await server.live.quickCreate({ permanentLive: true, saveReplay: false, privacy: VideoPrivacy.PUBLIC }) - live = data.live - video = data.video - }) - - it('Should abort a running live', async function () { - this.timeout(120000) - - const job = await prepare() - - await Promise.all([ - server.runnerJobs.abort({ jobUUID: job.uuid, runnerToken, jobToken: job.jobToken, reason: 'abort' }), - testFfmpegStreamError(ffmpegCommand, true) - ]) - - // Abort is not supported - await checkSessionError(LiveVideoError.RUNNER_JOB_ERROR) - }) - - it('Should cancel a running live', async function () { - this.timeout(120000) - - const job = await prepare() - - await Promise.all([ - server.runnerJobs.cancelByAdmin({ jobUUID: job.uuid }), - testFfmpegStreamError(ffmpegCommand, true) - ]) - - await checkSessionError(LiveVideoError.RUNNER_JOB_CANCEL) - }) - - it('Should error a running live', async function () { - this.timeout(120000) - - const job = await prepare() - - await Promise.all([ - server.runnerJobs.error({ jobUUID: job.uuid, runnerToken, jobToken: job.jobToken, message: 'error' }), - testFfmpegStreamError(ffmpegCommand, true) - ]) - - await checkSessionError(LiveVideoError.RUNNER_JOB_ERROR) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/runners/runner-socket.ts b/server/tests/api/runners/runner-socket.ts deleted file mode 100644 index 215164e48..000000000 --- a/server/tests/api/runners/runner-socket.ts +++ /dev/null @@ -1,120 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { wait } from '@shared/core-utils' -import { - cleanupTests, - createSingleServer, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - waitJobs -} from '@shared/server-commands' - -describe('Test runner socket', function () { - let server: PeerTubeServer - let runnerToken: string - - before(async function () { - this.timeout(120_000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - await server.config.enableTranscoding({ hls: true, webVideo: true }) - await server.config.enableRemoteTranscoding() - runnerToken = await server.runners.autoRegisterRunner() - }) - - it('Should throw an error without runner token', function (done) { - const localSocket = server.socketIO.getRunnersSocket({ runnerToken: null }) - localSocket.on('connect_error', err => { - expect(err.message).to.contain('No runner token provided') - done() - }) - }) - - it('Should throw an error with a bad runner token', function (done) { - const localSocket = server.socketIO.getRunnersSocket({ runnerToken: 'ergag' }) - localSocket.on('connect_error', err => { - expect(err.message).to.contain('Invalid runner token') - done() - }) - }) - - it('Should not send ping if there is no available jobs', async function () { - let pings = 0 - const localSocket = server.socketIO.getRunnersSocket({ runnerToken }) - localSocket.on('available-jobs', () => pings++) - - expect(pings).to.equal(0) - }) - - it('Should send a ping on available job', async function () { - let pings = 0 - const localSocket = server.socketIO.getRunnersSocket({ runnerToken }) - localSocket.on('available-jobs', () => pings++) - - await server.videos.quickUpload({ name: 'video1' }) - await waitJobs([ server ]) - - // eslint-disable-next-line no-unmodified-loop-condition - while (pings !== 1) { - await wait(500) - } - - await server.videos.quickUpload({ name: 'video2' }) - await waitJobs([ server ]) - - // eslint-disable-next-line no-unmodified-loop-condition - while ((pings as number) !== 2) { - await wait(500) - } - - await server.runnerJobs.cancelAllJobs() - }) - - it('Should send a ping when a child is ready', async function () { - let pings = 0 - const localSocket = server.socketIO.getRunnersSocket({ runnerToken }) - localSocket.on('available-jobs', () => pings++) - - await server.videos.quickUpload({ name: 'video3' }) - await waitJobs([ server ]) - - // eslint-disable-next-line no-unmodified-loop-condition - while (pings !== 1) { - await wait(500) - } - - await server.runnerJobs.autoProcessWebVideoJob(runnerToken) - await waitJobs([ server ]) - - // eslint-disable-next-line no-unmodified-loop-condition - while ((pings as number) !== 2) { - await wait(500) - } - }) - - it('Should not send a ping if the ended job does not have a child', async function () { - let pings = 0 - const localSocket = server.socketIO.getRunnersSocket({ runnerToken }) - localSocket.on('available-jobs', () => pings++) - - const { availableJobs } = await server.runnerJobs.request({ runnerToken }) - const job = availableJobs.find(j => j.type === 'vod-web-video-transcoding') - await server.runnerJobs.autoProcessWebVideoJob(runnerToken, job.uuid) - - // Wait for debounce - await wait(1000) - await waitJobs([ server ]) - - expect(pings).to.equal(0) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/runners/runner-studio-transcoding.ts b/server/tests/api/runners/runner-studio-transcoding.ts deleted file mode 100644 index f5cea6cea..000000000 --- a/server/tests/api/runners/runner-studio-transcoding.ts +++ /dev/null @@ -1,168 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { readFile } from 'fs-extra' -import { checkPersistentTmpIsEmpty, checkVideoDuration } from '@server/tests/shared' -import { buildAbsoluteFixturePath } from '@shared/core-utils' -import { - RunnerJobStudioTranscodingPayload, - VideoStudioTranscodingSuccess, - VideoState, - VideoStudioTask, - VideoStudioTaskIntro -} from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - VideoStudioCommand, - waitJobs -} from '@shared/server-commands' - -describe('Test runner video studio transcoding', function () { - let servers: PeerTubeServer[] = [] - let runnerToken: string - let videoUUID: string - let jobUUID: string - - async function renewStudio (tasks: VideoStudioTask[] = VideoStudioCommand.getComplexTask()) { - const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) - videoUUID = uuid - - await waitJobs(servers) - - await servers[0].videoStudio.createEditionTasks({ videoId: uuid, tasks }) - await waitJobs(servers) - - const { availableJobs } = await servers[0].runnerJobs.request({ runnerToken }) - expect(availableJobs).to.have.lengthOf(1) - - jobUUID = availableJobs[0].uuid - } - - before(async function () { - this.timeout(120_000) - - servers = await createMultipleServers(2) - - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - - await doubleFollow(servers[0], servers[1]) - - await servers[0].config.enableTranscoding({ hls: true, webVideo: true }) - await servers[0].config.enableStudio() - await servers[0].config.enableRemoteStudio() - - runnerToken = await servers[0].runners.autoRegisterRunner() - }) - - it('Should error a studio transcoding job', async function () { - this.timeout(60000) - - await renewStudio() - - for (let i = 0; i < 5; i++) { - const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) - const jobToken = job.jobToken - - await servers[0].runnerJobs.error({ runnerToken, jobUUID, jobToken, message: 'Error' }) - } - - const video = await servers[0].videos.get({ id: videoUUID }) - expect(video.state.id).to.equal(VideoState.PUBLISHED) - - await checkPersistentTmpIsEmpty(servers[0]) - }) - - it('Should cancel a transcoding job', async function () { - this.timeout(60000) - - await renewStudio() - - await servers[0].runnerJobs.cancelByAdmin({ jobUUID }) - - const video = await servers[0].videos.get({ id: videoUUID }) - expect(video.state.id).to.equal(VideoState.PUBLISHED) - - await checkPersistentTmpIsEmpty(servers[0]) - }) - - it('Should execute a remote studio job', async function () { - this.timeout(240_000) - - const tasks = [ - { - name: 'add-outro' as 'add-outro', - options: { - file: 'video_short.webm' - } - }, - { - name: 'add-watermark' as 'add-watermark', - options: { - file: 'custom-thumbnail.png' - } - }, - { - name: 'add-intro' as 'add-intro', - options: { - file: 'video_very_short_240p.mp4' - } - } - ] - - await renewStudio(tasks) - - for (const server of servers) { - await checkVideoDuration(server, videoUUID, 5) - } - - const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) - const jobToken = job.jobToken - - expect(job.type === 'video-studio-transcoding') - expect(job.payload.input.videoFileUrl).to.exist - - // Check video input file - { - await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) - } - - // Check task files - for (let i = 0; i < tasks.length; i++) { - const task = tasks[i] - const payloadTask = job.payload.tasks[i] - - expect(payloadTask.name).to.equal(task.name) - - const inputFile = await readFile(buildAbsoluteFixturePath(task.options.file)) - - const { body } = await servers[0].runnerJobs.getJobFile({ - url: (payloadTask as VideoStudioTaskIntro).options.file as string, - jobToken, - runnerToken - }) - - expect(body).to.deep.equal(inputFile) - } - - const payload: VideoStudioTranscodingSuccess = { videoFile: 'video_very_short_240p.mp4' } - await servers[0].runnerJobs.success({ runnerToken, jobUUID, jobToken, payload }) - - await waitJobs(servers) - - for (const server of servers) { - await checkVideoDuration(server, videoUUID, 2) - } - - await checkPersistentTmpIsEmpty(servers[0]) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/runners/runner-vod-transcoding.ts b/server/tests/api/runners/runner-vod-transcoding.ts deleted file mode 100644 index ee6be4ee9..000000000 --- a/server/tests/api/runners/runner-vod-transcoding.ts +++ /dev/null @@ -1,545 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { readFile } from 'fs-extra' -import { completeCheckHlsPlaylist } from '@server/tests/shared' -import { buildAbsoluteFixturePath } from '@shared/core-utils' -import { - HttpStatusCode, - RunnerJobSuccessPayload, - RunnerJobVODAudioMergeTranscodingPayload, - RunnerJobVODHLSTranscodingPayload, - RunnerJobVODPayload, - RunnerJobVODWebVideoTranscodingPayload, - VideoState, - VODAudioMergeTranscodingSuccess, - VODHLSTranscodingSuccess, - VODWebVideoTranscodingSuccess -} from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - makeGetRequest, - makeRawRequest, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - waitJobs -} from '@shared/server-commands' - -async function processAllJobs (server: PeerTubeServer, runnerToken: string) { - do { - const { availableJobs } = await server.runnerJobs.requestVOD({ runnerToken }) - if (availableJobs.length === 0) break - - const { job } = await server.runnerJobs.accept({ runnerToken, jobUUID: availableJobs[0].uuid }) - - const payload: RunnerJobSuccessPayload = { - videoFile: `video_short_${job.payload.output.resolution}p.mp4`, - resolutionPlaylistFile: `video_short_${job.payload.output.resolution}p.m3u8` - } - await server.runnerJobs.success({ runnerToken, jobUUID: job.uuid, jobToken: job.jobToken, payload }) - } while (true) - - await waitJobs([ server ]) -} - -describe('Test runner VOD transcoding', function () { - let servers: PeerTubeServer[] = [] - let runnerToken: string - - before(async function () { - this.timeout(120_000) - - servers = await createMultipleServers(2) - - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - - await doubleFollow(servers[0], servers[1]) - - await servers[0].config.enableRemoteTranscoding() - runnerToken = await servers[0].runners.autoRegisterRunner() - }) - - describe('Without transcoding', function () { - - before(async function () { - this.timeout(60000) - - await servers[0].config.disableTranscoding() - await servers[0].videos.quickUpload({ name: 'video' }) - - await waitJobs(servers) - }) - - it('Should not have available jobs', async function () { - const { availableJobs } = await servers[0].runnerJobs.requestVOD({ runnerToken }) - expect(availableJobs).to.have.lengthOf(0) - }) - }) - - describe('With classic transcoding enabled', function () { - - before(async function () { - this.timeout(60000) - - await servers[0].config.enableTranscoding({ hls: true, webVideo: true }) - }) - - it('Should error a transcoding job', async function () { - this.timeout(60000) - - await servers[0].runnerJobs.cancelAllJobs() - const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) - await waitJobs(servers) - - const { availableJobs } = await servers[0].runnerJobs.request({ runnerToken }) - const jobUUID = availableJobs[0].uuid - - for (let i = 0; i < 5; i++) { - const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) - const jobToken = job.jobToken - - await servers[0].runnerJobs.error({ runnerToken, jobUUID, jobToken, message: 'Error' }) - } - - const video = await servers[0].videos.get({ id: uuid }) - expect(video.state.id).to.equal(VideoState.TRANSCODING_FAILED) - }) - - it('Should cancel a transcoding job', async function () { - await servers[0].runnerJobs.cancelAllJobs() - const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) - await waitJobs(servers) - - const { availableJobs } = await servers[0].runnerJobs.request({ runnerToken }) - const jobUUID = availableJobs[0].uuid - - await servers[0].runnerJobs.cancelByAdmin({ jobUUID }) - - const video = await servers[0].videos.get({ id: uuid }) - expect(video.state.id).to.equal(VideoState.PUBLISHED) - }) - }) - - describe('Web video transcoding only', function () { - let videoUUID: string - let jobToken: string - let jobUUID: string - - before(async function () { - this.timeout(60000) - - await servers[0].runnerJobs.cancelAllJobs() - await servers[0].config.enableTranscoding({ hls: false, webVideo: true }) - - const { uuid } = await servers[0].videos.quickUpload({ name: 'web video', fixture: 'video_short.webm' }) - videoUUID = uuid - - await waitJobs(servers) - }) - - it('Should have jobs available for remote runners', async function () { - const { availableJobs } = await servers[0].runnerJobs.requestVOD({ runnerToken }) - expect(availableJobs).to.have.lengthOf(1) - - jobUUID = availableJobs[0].uuid - }) - - it('Should have a valid first transcoding job', async function () { - const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) - jobToken = job.jobToken - - expect(job.type === 'vod-web-video-transcoding') - expect(job.payload.input.videoFileUrl).to.exist - expect(job.payload.output.resolution).to.equal(720) - expect(job.payload.output.fps).to.equal(25) - - const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) - const inputFile = await readFile(buildAbsoluteFixturePath('video_short.webm')) - - expect(body).to.deep.equal(inputFile) - }) - - it('Should transcode the max video resolution and send it back to the server', async function () { - this.timeout(60000) - - const payload: VODWebVideoTranscodingSuccess = { - videoFile: 'video_short.mp4' - } - await servers[0].runnerJobs.success({ runnerToken, jobUUID, jobToken, payload }) - - await waitJobs(servers) - }) - - it('Should have the video updated', async function () { - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - expect(video.files).to.have.lengthOf(1) - expect(video.streamingPlaylists).to.have.lengthOf(0) - - const { body } = await makeRawRequest({ url: video.files[0].fileUrl, expectedStatus: HttpStatusCode.OK_200 }) - expect(body).to.deep.equal(await readFile(buildAbsoluteFixturePath('video_short.mp4'))) - } - }) - - it('Should have 4 lower resolution to transcode', async function () { - const { availableJobs } = await servers[0].runnerJobs.requestVOD({ runnerToken }) - expect(availableJobs).to.have.lengthOf(4) - - for (const resolution of [ 480, 360, 240, 144 ]) { - const job = availableJobs.find(j => j.payload.output.resolution === resolution) - expect(job).to.exist - expect(job.type).to.equal('vod-web-video-transcoding') - - if (resolution === 240) jobUUID = job.uuid - } - }) - - it('Should process one of these transcoding jobs', async function () { - const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) - jobToken = job.jobToken - - const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) - const inputFile = await readFile(buildAbsoluteFixturePath('video_short.mp4')) - - expect(body).to.deep.equal(inputFile) - - const payload: VODWebVideoTranscodingSuccess = { videoFile: `video_short_${job.payload.output.resolution}p.mp4` } - await servers[0].runnerJobs.success({ runnerToken, jobUUID, jobToken, payload }) - }) - - it('Should process all other jobs', async function () { - const { availableJobs } = await servers[0].runnerJobs.requestVOD({ runnerToken }) - expect(availableJobs).to.have.lengthOf(3) - - for (const resolution of [ 480, 360, 144 ]) { - const availableJob = availableJobs.find(j => j.payload.output.resolution === resolution) - expect(availableJob).to.exist - jobUUID = availableJob.uuid - - const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) - jobToken = job.jobToken - - const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) - const inputFile = await readFile(buildAbsoluteFixturePath('video_short.mp4')) - expect(body).to.deep.equal(inputFile) - - const payload: VODWebVideoTranscodingSuccess = { videoFile: `video_short_${resolution}p.mp4` } - await servers[0].runnerJobs.success({ runnerToken, jobUUID, jobToken, payload }) - } - - await waitJobs(servers) - }) - - it('Should have the video updated', async function () { - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - expect(video.files).to.have.lengthOf(5) - expect(video.streamingPlaylists).to.have.lengthOf(0) - - const { body } = await makeRawRequest({ url: video.files[0].fileUrl, expectedStatus: HttpStatusCode.OK_200 }) - expect(body).to.deep.equal(await readFile(buildAbsoluteFixturePath('video_short.mp4'))) - - for (const file of video.files) { - await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 }) - } - } - }) - - it('Should not have available jobs anymore', async function () { - const { availableJobs } = await servers[0].runnerJobs.requestVOD({ runnerToken }) - expect(availableJobs).to.have.lengthOf(0) - }) - }) - - describe('HLS transcoding only', function () { - let videoUUID: string - let jobToken: string - let jobUUID: string - - before(async function () { - this.timeout(60000) - - await servers[0].config.enableTranscoding({ hls: true, webVideo: false }) - - const { uuid } = await servers[0].videos.quickUpload({ name: 'hls video', fixture: 'video_short.webm' }) - videoUUID = uuid - - await waitJobs(servers) - }) - - it('Should run the optimize job', async function () { - this.timeout(60000) - - await servers[0].runnerJobs.autoProcessWebVideoJob(runnerToken) - }) - - it('Should have 5 HLS resolution to transcode', async function () { - const { availableJobs } = await servers[0].runnerJobs.requestVOD({ runnerToken }) - expect(availableJobs).to.have.lengthOf(5) - - for (const resolution of [ 720, 480, 360, 240, 144 ]) { - const job = availableJobs.find(j => j.payload.output.resolution === resolution) - expect(job).to.exist - expect(job.type).to.equal('vod-hls-transcoding') - - if (resolution === 480) jobUUID = job.uuid - } - }) - - it('Should process one of these transcoding jobs', async function () { - this.timeout(60000) - - const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) - jobToken = job.jobToken - - const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) - const inputFile = await readFile(buildAbsoluteFixturePath('video_short.mp4')) - - expect(body).to.deep.equal(inputFile) - - const payload: VODHLSTranscodingSuccess = { - videoFile: 'video_short_480p.mp4', - resolutionPlaylistFile: 'video_short_480p.m3u8' - } - await servers[0].runnerJobs.success({ runnerToken, jobUUID, jobToken, payload }) - - await waitJobs(servers) - }) - - it('Should have the video updated', async function () { - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - - expect(video.files).to.have.lengthOf(1) - expect(video.streamingPlaylists).to.have.lengthOf(1) - - const hls = video.streamingPlaylists[0] - expect(hls.files).to.have.lengthOf(1) - - await completeCheckHlsPlaylist({ videoUUID, hlsOnly: false, servers, resolutions: [ 480 ] }) - } - }) - - it('Should process all other jobs', async function () { - this.timeout(60000) - - const { availableJobs } = await servers[0].runnerJobs.requestVOD({ runnerToken }) - expect(availableJobs).to.have.lengthOf(4) - - let maxQualityFile = 'video_short.mp4' - - for (const resolution of [ 720, 360, 240, 144 ]) { - const availableJob = availableJobs.find(j => j.payload.output.resolution === resolution) - expect(availableJob).to.exist - jobUUID = availableJob.uuid - - const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) - jobToken = job.jobToken - - const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) - const inputFile = await readFile(buildAbsoluteFixturePath(maxQualityFile)) - expect(body).to.deep.equal(inputFile) - - const payload: VODHLSTranscodingSuccess = { - videoFile: `video_short_${resolution}p.mp4`, - resolutionPlaylistFile: `video_short_${resolution}p.m3u8` - } - await servers[0].runnerJobs.success({ runnerToken, jobUUID, jobToken, payload }) - - if (resolution === 720) { - maxQualityFile = 'video_short_720p.mp4' - } - } - - await waitJobs(servers) - }) - - it('Should have the video updated', async function () { - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - - expect(video.files).to.have.lengthOf(0) - expect(video.streamingPlaylists).to.have.lengthOf(1) - - const hls = video.streamingPlaylists[0] - expect(hls.files).to.have.lengthOf(5) - - await completeCheckHlsPlaylist({ videoUUID, hlsOnly: true, servers, resolutions: [ 720, 480, 360, 240, 144 ] }) - } - }) - - it('Should not have available jobs anymore', async function () { - const { availableJobs } = await servers[0].runnerJobs.requestVOD({ runnerToken }) - expect(availableJobs).to.have.lengthOf(0) - }) - }) - - describe('Web video and HLS transcoding', function () { - - before(async function () { - this.timeout(60000) - - await servers[0].config.enableTranscoding({ hls: true, webVideo: true }) - - await servers[0].videos.quickUpload({ name: 'web video and hls video', fixture: 'video_short.webm' }) - - await waitJobs(servers) - }) - - it('Should process the first optimize job', async function () { - this.timeout(60000) - - await servers[0].runnerJobs.autoProcessWebVideoJob(runnerToken) - }) - - it('Should have 9 jobs to process', async function () { - const { availableJobs } = await servers[0].runnerJobs.requestVOD({ runnerToken }) - - expect(availableJobs).to.have.lengthOf(9) - - const webVideoJobs = availableJobs.filter(j => j.type === 'vod-web-video-transcoding') - const hlsJobs = availableJobs.filter(j => j.type === 'vod-hls-transcoding') - - expect(webVideoJobs).to.have.lengthOf(4) - expect(hlsJobs).to.have.lengthOf(5) - }) - - it('Should process all available jobs', async function () { - await processAllJobs(servers[0], runnerToken) - }) - }) - - describe('Audio merge transcoding', function () { - let videoUUID: string - let jobToken: string - let jobUUID: string - - before(async function () { - this.timeout(60000) - - await servers[0].config.enableTranscoding({ hls: true, webVideo: true }) - - const attributes = { name: 'audio_with_preview', previewfile: 'custom-preview.jpg', fixture: 'sample.ogg' } - const { uuid } = await servers[0].videos.upload({ attributes, mode: 'legacy' }) - videoUUID = uuid - - await waitJobs(servers) - }) - - it('Should have an audio merge transcoding job', async function () { - const { availableJobs } = await servers[0].runnerJobs.requestVOD({ runnerToken }) - expect(availableJobs).to.have.lengthOf(1) - - expect(availableJobs[0].type).to.equal('vod-audio-merge-transcoding') - - jobUUID = availableJobs[0].uuid - }) - - it('Should have a valid remote audio merge transcoding job', async function () { - const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) - jobToken = job.jobToken - - expect(job.type === 'vod-audio-merge-transcoding') - expect(job.payload.input.audioFileUrl).to.exist - expect(job.payload.input.previewFileUrl).to.exist - expect(job.payload.output.resolution).to.equal(480) - - { - const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.audioFileUrl, jobToken, runnerToken }) - const inputFile = await readFile(buildAbsoluteFixturePath('sample.ogg')) - expect(body).to.deep.equal(inputFile) - } - - { - const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.previewFileUrl, jobToken, runnerToken }) - - const video = await servers[0].videos.get({ id: videoUUID }) - const { body: inputFile } = await makeGetRequest({ - url: servers[0].url, - path: video.previewPath, - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(body).to.deep.equal(inputFile) - } - }) - - it('Should merge the audio', async function () { - this.timeout(60000) - - const payload: VODAudioMergeTranscodingSuccess = { videoFile: 'video_short_480p.mp4' } - await servers[0].runnerJobs.success({ runnerToken, jobUUID, jobToken, payload }) - - await waitJobs(servers) - }) - - it('Should have the video updated', async function () { - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - expect(video.files).to.have.lengthOf(1) - expect(video.streamingPlaylists).to.have.lengthOf(0) - - const { body } = await makeRawRequest({ url: video.files[0].fileUrl, expectedStatus: HttpStatusCode.OK_200 }) - expect(body).to.deep.equal(await readFile(buildAbsoluteFixturePath('video_short_480p.mp4'))) - } - }) - - it('Should have 7 lower resolutions to transcode', async function () { - const { availableJobs } = await servers[0].runnerJobs.requestVOD({ runnerToken }) - expect(availableJobs).to.have.lengthOf(7) - - for (const resolution of [ 360, 240, 144 ]) { - const jobs = availableJobs.filter(j => j.payload.output.resolution === resolution) - expect(jobs).to.have.lengthOf(2) - } - - jobUUID = availableJobs.find(j => j.payload.output.resolution === 480).uuid - }) - - it('Should process one other job', async function () { - this.timeout(60000) - - const { job } = await servers[0].runnerJobs.accept({ runnerToken, jobUUID }) - jobToken = job.jobToken - - const { body } = await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken, runnerToken }) - const inputFile = await readFile(buildAbsoluteFixturePath('video_short_480p.mp4')) - expect(body).to.deep.equal(inputFile) - - const payload: VODHLSTranscodingSuccess = { - videoFile: `video_short_480p.mp4`, - resolutionPlaylistFile: `video_short_480p.m3u8` - } - await servers[0].runnerJobs.success({ runnerToken, jobUUID, jobToken, payload }) - - await waitJobs(servers) - }) - - it('Should have the video updated', async function () { - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - - expect(video.files).to.have.lengthOf(1) - expect(video.streamingPlaylists).to.have.lengthOf(1) - - const hls = video.streamingPlaylists[0] - expect(hls.files).to.have.lengthOf(1) - - await completeCheckHlsPlaylist({ videoUUID, hlsOnly: false, servers, resolutions: [ 480 ] }) - } - }) - - it('Should process all available jobs', async function () { - await processAllJobs(servers[0], runnerToken) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/search/index.ts b/server/tests/api/search/index.ts deleted file mode 100644 index a976d210d..000000000 --- a/server/tests/api/search/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -import './search-activitypub-video-playlists' -import './search-activitypub-video-channels' -import './search-activitypub-videos' -import './search-channels' -import './search-index' -import './search-playlists' -import './search-videos' diff --git a/server/tests/api/search/search-activitypub-video-channels.ts b/server/tests/api/search/search-activitypub-video-channels.ts deleted file mode 100644 index 003bd34d0..000000000 --- a/server/tests/api/search/search-activitypub-video-channels.ts +++ /dev/null @@ -1,255 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { wait } from '@shared/core-utils' -import { VideoChannel } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - PeerTubeServer, - SearchCommand, - setAccessTokensToServers, - setDefaultAccountAvatar, - setDefaultVideoChannel, - waitJobs -} from '@shared/server-commands' - -describe('Test ActivityPub video channels search', function () { - let servers: PeerTubeServer[] - let userServer2Token: string - let videoServer2UUID: string - let channelIdServer2: number - let command: SearchCommand - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(2) - - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - await setDefaultAccountAvatar(servers) - - { - await servers[0].users.create({ username: 'user1_server1', password: 'password' }) - const channel = { - name: 'channel1_server1', - displayName: 'Channel 1 server 1' - } - await servers[0].channels.create({ attributes: channel }) - } - - { - const user = { username: 'user1_server2', password: 'password' } - await servers[1].users.create({ username: user.username, password: user.password }) - userServer2Token = await servers[1].login.getAccessToken(user) - - const channel = { - name: 'channel1_server2', - displayName: 'Channel 1 server 2' - } - const created = await servers[1].channels.create({ token: userServer2Token, attributes: channel }) - channelIdServer2 = created.id - - const attributes = { name: 'video 1 server 2', channelId: channelIdServer2 } - const { uuid } = await servers[1].videos.upload({ token: userServer2Token, attributes }) - videoServer2UUID = uuid - } - - await waitJobs(servers) - - command = servers[0].search - }) - - it('Should not find a remote video channel', async function () { - this.timeout(15000) - - { - const search = servers[1].url + '/video-channels/channel1_server3' - const body = await command.searchChannels({ search, token: servers[0].accessToken }) - - expect(body.total).to.equal(0) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(0) - } - - { - // Without token - const search = servers[1].url + '/video-channels/channel1_server2' - const body = await command.searchChannels({ search }) - - expect(body.total).to.equal(0) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(0) - } - }) - - it('Should search a local video channel', async function () { - const searches = [ - servers[0].url + '/video-channels/channel1_server1', - 'channel1_server1@' + servers[0].host - ] - - for (const search of searches) { - const body = await command.searchChannels({ search }) - - expect(body.total).to.equal(1) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].name).to.equal('channel1_server1') - expect(body.data[0].displayName).to.equal('Channel 1 server 1') - } - }) - - it('Should search a local video channel with an alternative URL', async function () { - const search = servers[0].url + '/c/channel1_server1' - - for (const token of [ undefined, servers[0].accessToken ]) { - const body = await command.searchChannels({ search, token }) - - expect(body.total).to.equal(1) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].name).to.equal('channel1_server1') - expect(body.data[0].displayName).to.equal('Channel 1 server 1') - } - }) - - it('Should search a local video channel with a query in URL', async function () { - const searches = [ - servers[0].url + '/video-channels/channel1_server1', - servers[0].url + '/c/channel1_server1' - ] - - for (const search of searches) { - for (const token of [ undefined, servers[0].accessToken ]) { - const body = await command.searchChannels({ search: search + '?param=2', token }) - - expect(body.total).to.equal(1) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].name).to.equal('channel1_server1') - expect(body.data[0].displayName).to.equal('Channel 1 server 1') - } - } - }) - - it('Should search a remote video channel with URL or handle', async function () { - const searches = [ - servers[1].url + '/video-channels/channel1_server2', - servers[1].url + '/c/channel1_server2', - servers[1].url + '/c/channel1_server2/videos', - 'channel1_server2@' + servers[1].host - ] - - for (const search of searches) { - const body = await command.searchChannels({ search, token: servers[0].accessToken }) - - expect(body.total).to.equal(1) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].name).to.equal('channel1_server2') - expect(body.data[0].displayName).to.equal('Channel 1 server 2') - } - }) - - it('Should not list this remote video channel', async function () { - const body = await servers[0].channels.list() - expect(body.total).to.equal(3) - expect(body.data).to.have.lengthOf(3) - expect(body.data[0].name).to.equal('channel1_server1') - expect(body.data[1].name).to.equal('user1_server1_channel') - expect(body.data[2].name).to.equal('root_channel') - }) - - it('Should list video channel videos of server 2 without token', async function () { - this.timeout(30000) - - await waitJobs(servers) - - const { total, data } = await servers[0].videos.listByChannel({ - token: null, - handle: 'channel1_server2@' + servers[1].host - }) - expect(total).to.equal(0) - expect(data).to.have.lengthOf(0) - }) - - it('Should list video channel videos of server 2 with token', async function () { - const { total, data } = await servers[0].videos.listByChannel({ - handle: 'channel1_server2@' + servers[1].host - }) - - expect(total).to.equal(1) - expect(data[0].name).to.equal('video 1 server 2') - }) - - it('Should update video channel of server 2, and refresh it on server 1', async function () { - this.timeout(120000) - - await servers[1].channels.update({ - token: userServer2Token, - channelName: 'channel1_server2', - attributes: { displayName: 'channel updated' } - }) - await servers[1].users.updateMe({ token: userServer2Token, displayName: 'user updated' }) - - await waitJobs(servers) - // Expire video channel - await wait(10000) - - const search = servers[1].url + '/video-channels/channel1_server2' - const body = await command.searchChannels({ search, token: servers[0].accessToken }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - - const videoChannel: VideoChannel = body.data[0] - expect(videoChannel.displayName).to.equal('channel updated') - - // We don't return the owner account for now - // expect(videoChannel.ownerAccount.displayName).to.equal('user updated') - }) - - it('Should update and add a video on server 2, and update it on server 1 after a search', async function () { - this.timeout(120000) - - await servers[1].videos.update({ token: userServer2Token, id: videoServer2UUID, attributes: { name: 'video 1 updated' } }) - await servers[1].videos.upload({ token: userServer2Token, attributes: { name: 'video 2 server 2', channelId: channelIdServer2 } }) - - await waitJobs(servers) - - // Expire video channel - await wait(10000) - - const search = servers[1].url + '/video-channels/channel1_server2' - await command.searchChannels({ search, token: servers[0].accessToken }) - - await waitJobs(servers) - - const handle = 'channel1_server2@' + servers[1].host - const { total, data } = await servers[0].videos.listByChannel({ handle, sort: '-createdAt' }) - - expect(total).to.equal(2) - expect(data[0].name).to.equal('video 2 server 2') - expect(data[1].name).to.equal('video 1 updated') - }) - - it('Should delete video channel of server 2, and delete it on server 1', async function () { - this.timeout(120000) - - await servers[1].channels.delete({ token: userServer2Token, channelName: 'channel1_server2' }) - - await waitJobs(servers) - // Expire video - await wait(10000) - - const search = servers[1].url + '/video-channels/channel1_server2' - const body = await command.searchChannels({ search, token: servers[0].accessToken }) - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/search/search-activitypub-video-playlists.ts b/server/tests/api/search/search-activitypub-video-playlists.ts deleted file mode 100644 index 2bb5d869a..000000000 --- a/server/tests/api/search/search-activitypub-video-playlists.ts +++ /dev/null @@ -1,214 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { wait } from '@shared/core-utils' -import { VideoPlaylistPrivacy } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - PeerTubeServer, - SearchCommand, - setAccessTokensToServers, - setDefaultAccountAvatar, - setDefaultVideoChannel, - waitJobs -} from '@shared/server-commands' - -describe('Test ActivityPub playlists search', function () { - let servers: PeerTubeServer[] - let playlistServer1UUID: string - let playlistServer2UUID: string - let video2Server2: string - - let command: SearchCommand - - before(async function () { - this.timeout(240000) - - servers = await createMultipleServers(2) - - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - await setDefaultAccountAvatar(servers) - - { - const video1 = (await servers[0].videos.quickUpload({ name: 'video 1' })).uuid - const video2 = (await servers[0].videos.quickUpload({ name: 'video 2' })).uuid - - const attributes = { - displayName: 'playlist 1 on server 1', - privacy: VideoPlaylistPrivacy.PUBLIC, - videoChannelId: servers[0].store.channel.id - } - const created = await servers[0].playlists.create({ attributes }) - playlistServer1UUID = created.uuid - - for (const videoId of [ video1, video2 ]) { - await servers[0].playlists.addElement({ playlistId: playlistServer1UUID, attributes: { videoId } }) - } - } - - { - const videoId = (await servers[1].videos.quickUpload({ name: 'video 1' })).uuid - video2Server2 = (await servers[1].videos.quickUpload({ name: 'video 2' })).uuid - - const attributes = { - displayName: 'playlist 1 on server 2', - privacy: VideoPlaylistPrivacy.PUBLIC, - videoChannelId: servers[1].store.channel.id - } - const created = await servers[1].playlists.create({ attributes }) - playlistServer2UUID = created.uuid - - await servers[1].playlists.addElement({ playlistId: playlistServer2UUID, attributes: { videoId } }) - } - - await waitJobs(servers) - - command = servers[0].search - }) - - it('Should not find a remote playlist', async function () { - { - const search = servers[1].url + '/video-playlists/43' - const body = await command.searchPlaylists({ search, token: servers[0].accessToken }) - - expect(body.total).to.equal(0) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(0) - } - - { - // Without token - const search = servers[1].url + '/video-playlists/' + playlistServer2UUID - const body = await command.searchPlaylists({ search }) - - expect(body.total).to.equal(0) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(0) - } - }) - - it('Should search a local playlist', async function () { - const search = servers[0].url + '/video-playlists/' + playlistServer1UUID - const body = await command.searchPlaylists({ search }) - - expect(body.total).to.equal(1) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].displayName).to.equal('playlist 1 on server 1') - expect(body.data[0].videosLength).to.equal(2) - }) - - it('Should search a local playlist with an alternative URL', async function () { - const searches = [ - servers[0].url + '/videos/watch/playlist/' + playlistServer1UUID, - servers[0].url + '/w/p/' + playlistServer1UUID - ] - - for (const search of searches) { - for (const token of [ undefined, servers[0].accessToken ]) { - const body = await command.searchPlaylists({ search, token }) - - expect(body.total).to.equal(1) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].displayName).to.equal('playlist 1 on server 1') - expect(body.data[0].videosLength).to.equal(2) - } - } - }) - - it('Should search a local playlist with a query in URL', async function () { - const searches = [ - servers[0].url + '/videos/watch/playlist/' + playlistServer1UUID, - servers[0].url + '/w/p/' + playlistServer1UUID - ] - - for (const search of searches) { - for (const token of [ undefined, servers[0].accessToken ]) { - const body = await command.searchPlaylists({ search: search + '?param=1', token }) - - expect(body.total).to.equal(1) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].displayName).to.equal('playlist 1 on server 1') - expect(body.data[0].videosLength).to.equal(2) - } - } - }) - - it('Should search a remote playlist', async function () { - const searches = [ - servers[1].url + '/video-playlists/' + playlistServer2UUID, - servers[1].url + '/videos/watch/playlist/' + playlistServer2UUID, - servers[1].url + '/w/p/' + playlistServer2UUID - ] - - for (const search of searches) { - const body = await command.searchPlaylists({ search, token: servers[0].accessToken }) - - expect(body.total).to.equal(1) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].displayName).to.equal('playlist 1 on server 2') - expect(body.data[0].videosLength).to.equal(1) - } - }) - - it('Should not list this remote playlist', async function () { - const body = await servers[0].playlists.list({ start: 0, count: 10 }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].displayName).to.equal('playlist 1 on server 1') - }) - - it('Should update the playlist of server 2, and refresh it on server 1', async function () { - this.timeout(60000) - - await servers[1].playlists.addElement({ playlistId: playlistServer2UUID, attributes: { videoId: video2Server2 } }) - - await waitJobs(servers) - // Expire playlist - await wait(10000) - - // Will run refresh async - const search = servers[1].url + '/video-playlists/' + playlistServer2UUID - await command.searchPlaylists({ search, token: servers[0].accessToken }) - - // Wait refresh - await wait(5000) - - const body = await command.searchPlaylists({ search, token: servers[0].accessToken }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - - const playlist = body.data[0] - expect(playlist.videosLength).to.equal(2) - }) - - it('Should delete playlist of server 2, and delete it on server 1', async function () { - this.timeout(60000) - - await servers[1].playlists.delete({ playlistId: playlistServer2UUID }) - - await waitJobs(servers) - // Expiration - await wait(10000) - - // Will run refresh async - const search = servers[1].url + '/video-playlists/' + playlistServer2UUID - await command.searchPlaylists({ search, token: servers[0].accessToken }) - - // Wait refresh - await wait(5000) - - const body = await command.searchPlaylists({ search, token: servers[0].accessToken }) - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/search/search-activitypub-videos.ts b/server/tests/api/search/search-activitypub-videos.ts deleted file mode 100644 index 4c7118422..000000000 --- a/server/tests/api/search/search-activitypub-videos.ts +++ /dev/null @@ -1,196 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { wait } from '@shared/core-utils' -import { VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - PeerTubeServer, - SearchCommand, - setAccessTokensToServers, - setDefaultAccountAvatar, - setDefaultVideoChannel, - waitJobs -} from '@shared/server-commands' - -describe('Test ActivityPub videos search', function () { - let servers: PeerTubeServer[] - let videoServer1UUID: string - let videoServer2UUID: string - - let command: SearchCommand - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(2) - - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - await setDefaultAccountAvatar(servers) - - { - const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video 1 on server 1' } }) - videoServer1UUID = uuid - } - - { - const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video 1 on server 2' } }) - videoServer2UUID = uuid - } - - await waitJobs(servers) - - command = servers[0].search - }) - - it('Should not find a remote video', async function () { - { - const search = servers[1].url + '/videos/watch/43' - const body = await command.searchVideos({ search, token: servers[0].accessToken }) - - expect(body.total).to.equal(0) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(0) - } - - { - // Without token - const search = servers[1].url + '/videos/watch/' + videoServer2UUID - const body = await command.searchVideos({ search }) - - expect(body.total).to.equal(0) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(0) - } - }) - - it('Should search a local video', async function () { - const search = servers[0].url + '/videos/watch/' + videoServer1UUID - const body = await command.searchVideos({ search }) - - expect(body.total).to.equal(1) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].name).to.equal('video 1 on server 1') - }) - - it('Should search a local video with an alternative URL', async function () { - const search = servers[0].url + '/w/' + videoServer1UUID - const body1 = await command.searchVideos({ search }) - const body2 = await command.searchVideos({ search, token: servers[0].accessToken }) - - for (const body of [ body1, body2 ]) { - expect(body.total).to.equal(1) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].name).to.equal('video 1 on server 1') - } - }) - - it('Should search a local video with a query in URL', async function () { - const searches = [ - servers[0].url + '/w/' + videoServer1UUID, - servers[0].url + '/videos/watch/' + videoServer1UUID - ] - - for (const search of searches) { - for (const token of [ undefined, servers[0].accessToken ]) { - const body = await command.searchVideos({ search: search + '?startTime=4', token }) - - expect(body.total).to.equal(1) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].name).to.equal('video 1 on server 1') - } - } - }) - - it('Should search a remote video', async function () { - const searches = [ - servers[1].url + '/w/' + videoServer2UUID, - servers[1].url + '/videos/watch/' + videoServer2UUID - ] - - for (const search of searches) { - const body = await command.searchVideos({ search, token: servers[0].accessToken }) - - expect(body.total).to.equal(1) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].name).to.equal('video 1 on server 2') - } - }) - - it('Should not list this remote video', async function () { - const { total, data } = await servers[0].videos.list() - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - expect(data[0].name).to.equal('video 1 on server 1') - }) - - it('Should update video of server 2, and refresh it on server 1', async function () { - this.timeout(120000) - - const channelAttributes = { - name: 'super_channel', - displayName: 'super channel' - } - const created = await servers[1].channels.create({ attributes: channelAttributes }) - const videoChannelId = created.id - - const attributes = { - name: 'updated', - tag: [ 'tag1', 'tag2' ], - privacy: VideoPrivacy.UNLISTED, - channelId: videoChannelId - } - await servers[1].videos.update({ id: videoServer2UUID, attributes }) - - await waitJobs(servers) - // Expire video - await wait(10000) - - // Will run refresh async - const search = servers[1].url + '/videos/watch/' + videoServer2UUID - await command.searchVideos({ search, token: servers[0].accessToken }) - - // Wait refresh - await wait(5000) - - const body = await command.searchVideos({ search, token: servers[0].accessToken }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - - const video = body.data[0] - expect(video.name).to.equal('updated') - expect(video.channel.name).to.equal('super_channel') - expect(video.privacy.id).to.equal(VideoPrivacy.UNLISTED) - }) - - it('Should delete video of server 2, and delete it on server 1', async function () { - this.timeout(120000) - - await servers[1].videos.remove({ id: videoServer2UUID }) - - await waitJobs(servers) - // Expire video - await wait(10000) - - // Will run refresh async - const search = servers[1].url + '/videos/watch/' + videoServer2UUID - await command.searchVideos({ search, token: servers[0].accessToken }) - - // Wait refresh - await wait(5000) - - const body = await command.searchVideos({ search, token: servers[0].accessToken }) - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/search/search-channels.ts b/server/tests/api/search/search-channels.ts deleted file mode 100644 index c6b098a93..000000000 --- a/server/tests/api/search/search-channels.ts +++ /dev/null @@ -1,159 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { VideoChannel } from '@shared/models' -import { - cleanupTests, - createSingleServer, - doubleFollow, - PeerTubeServer, - SearchCommand, - setAccessTokensToServers, - setDefaultAccountAvatar, - setDefaultChannelAvatar -} from '@shared/server-commands' - -describe('Test channels search', function () { - let server: PeerTubeServer - let remoteServer: PeerTubeServer - let command: SearchCommand - - before(async function () { - this.timeout(120000) - - const servers = await Promise.all([ - createSingleServer(1), - createSingleServer(2) - ]) - server = servers[0] - remoteServer = servers[1] - - await setAccessTokensToServers([ server, remoteServer ]) - await setDefaultChannelAvatar(server) - await setDefaultAccountAvatar(server) - - await servers[1].config.disableTranscoding() - - { - await server.users.create({ username: 'user1' }) - const channel = { - name: 'squall_channel', - displayName: 'Squall channel' - } - await server.channels.create({ attributes: channel }) - } - - { - await remoteServer.users.create({ username: 'user1' }) - const channel = { - name: 'zell_channel', - displayName: 'Zell channel' - } - const { id } = await remoteServer.channels.create({ attributes: channel }) - - await remoteServer.videos.upload({ attributes: { channelId: id } }) - } - - await doubleFollow(server, remoteServer) - - command = server.search - }) - - it('Should make a simple search and not have results', async function () { - const body = await command.searchChannels({ search: 'abc' }) - - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - }) - - it('Should make a search and have results', async function () { - { - const search = { - search: 'Squall', - start: 0, - count: 1 - } - const body = await command.advancedChannelSearch({ search }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - - const channel: VideoChannel = body.data[0] - expect(channel.name).to.equal('squall_channel') - expect(channel.displayName).to.equal('Squall channel') - } - - { - const search = { - search: 'Squall', - start: 1, - count: 1 - } - - const body = await command.advancedChannelSearch({ search }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(0) - } - }) - - it('Should filter by host', async function () { - { - const search = { search: 'channel', host: remoteServer.host } - - const body = await command.advancedChannelSearch({ search }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].displayName).to.equal('Zell channel') - } - - { - const search = { search: 'Sq', host: server.host } - - const body = await command.advancedChannelSearch({ search }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].displayName).to.equal('Squall channel') - } - - { - const search = { search: 'Squall', host: 'example.com' } - - const body = await command.advancedChannelSearch({ search }) - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - } - }) - - it('Should filter by names', async function () { - { - const body = await command.advancedChannelSearch({ search: { handles: [ 'squall_channel', 'zell_channel' ] } }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].displayName).to.equal('Squall channel') - } - - { - const body = await command.advancedChannelSearch({ search: { handles: [ 'squall_channel@' + server.host ] } }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].displayName).to.equal('Squall channel') - } - - { - const body = await command.advancedChannelSearch({ search: { handles: [ 'chocobozzz_channel' ] } }) - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - } - - { - const body = await command.advancedChannelSearch({ search: { handles: [ 'squall_channel', 'zell_channel@' + remoteServer.host ] } }) - expect(body.total).to.equal(2) - expect(body.data).to.have.lengthOf(2) - expect(body.data[0].displayName).to.equal('Squall channel') - expect(body.data[1].displayName).to.equal('Zell channel') - } - }) - - after(async function () { - await cleanupTests([ server, remoteServer ]) - }) -}) diff --git a/server/tests/api/search/search-index.ts b/server/tests/api/search/search-index.ts deleted file mode 100644 index cbe628ccc..000000000 --- a/server/tests/api/search/search-index.ts +++ /dev/null @@ -1,432 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { - BooleanBothQuery, - VideoChannelsSearchQuery, - VideoPlaylistPrivacy, - VideoPlaylistsSearchQuery, - VideoPlaylistType, - VideosSearchQuery -} from '@shared/models' -import { cleanupTests, createSingleServer, PeerTubeServer, SearchCommand, setAccessTokensToServers } from '@shared/server-commands' - -describe('Test index search', function () { - const localVideoName = 'local video' + new Date().toISOString() - - let server: PeerTubeServer = null - let command: SearchCommand - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - - await server.videos.upload({ attributes: { name: localVideoName } }) - - command = server.search - }) - - describe('Default search', async function () { - - it('Should make a local videos search by default', async function () { - await server.config.updateCustomSubConfig({ - newConfig: { - search: { - searchIndex: { - enabled: true, - isDefaultSearch: false, - disableLocalSearch: false - } - } - } - }) - - const body = await command.searchVideos({ search: 'local video' }) - - expect(body.total).to.equal(1) - expect(body.data[0].name).to.equal(localVideoName) - }) - - it('Should make a local channels search by default', async function () { - const body = await command.searchChannels({ search: 'root' }) - - expect(body.total).to.equal(1) - expect(body.data[0].name).to.equal('root_channel') - expect(body.data[0].host).to.equal(server.host) - }) - - it('Should make an index videos search by default', async function () { - await server.config.updateCustomSubConfig({ - newConfig: { - search: { - searchIndex: { - enabled: true, - isDefaultSearch: true, - disableLocalSearch: false - } - } - } - }) - - const body = await command.searchVideos({ search: 'local video' }) - expect(body.total).to.be.greaterThan(2) - }) - - it('Should make an index channels search by default', async function () { - const body = await command.searchChannels({ search: 'root' }) - expect(body.total).to.be.greaterThan(2) - }) - }) - - describe('Videos search', async function () { - - async function check (search: VideosSearchQuery, exists = true) { - const body = await command.advancedVideoSearch({ search }) - - if (exists === false) { - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - return - } - - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - - const video = body.data[0] - - expect(video.name).to.equal('What is PeerTube?') - expect(video.category.label).to.equal('Science & Technology') - expect(video.licence.label).to.equal('Attribution - Share Alike') - expect(video.privacy.label).to.equal('Public') - expect(video.duration).to.equal(113) - expect(video.thumbnailUrl.startsWith('https://framatube.org/static/thumbnails')).to.be.true - - expect(video.account.host).to.equal('framatube.org') - expect(video.account.name).to.equal('framasoft') - expect(video.account.url).to.equal('https://framatube.org/accounts/framasoft') - expect(video.account.avatars.length).to.equal(2, 'Account should have one avatar image') - - expect(video.channel.host).to.equal('framatube.org') - expect(video.channel.name).to.equal('joinpeertube') - expect(video.channel.url).to.equal('https://framatube.org/video-channels/joinpeertube') - expect(video.channel.avatars.length).to.equal(2, 'Channel should have one avatar image') - } - - const baseSearch: VideosSearchQuery = { - search: 'what is peertube', - start: 0, - count: 2, - categoryOneOf: [ 15 ], - licenceOneOf: [ 2 ], - tagsAllOf: [ 'framasoft', 'peertube' ], - startDate: '2018-10-01T10:50:46.396Z', - endDate: '2018-10-01T10:55:46.396Z' - } - - it('Should make a simple search and not have results', async function () { - const body = await command.searchVideos({ search: 'djidane'.repeat(50) }) - - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - }) - - it('Should make a simple search and have results', async function () { - const body = await command.searchVideos({ search: 'What is PeerTube' }) - - expect(body.total).to.be.greaterThan(1) - }) - - it('Should make a simple search', async function () { - await check(baseSearch) - }) - - it('Should search by start date', async function () { - const search = { ...baseSearch, startDate: '2018-10-01T10:54:46.396Z' } - await check(search, false) - }) - - it('Should search by tags', async function () { - const search = { ...baseSearch, tagsAllOf: [ 'toto', 'framasoft' ] } - await check(search, false) - }) - - it('Should search by duration', async function () { - const search = { ...baseSearch, durationMin: 2000 } - await check(search, false) - }) - - it('Should search by nsfw attribute', async function () { - { - const search = { ...baseSearch, nsfw: 'true' as BooleanBothQuery } - await check(search, false) - } - - { - const search = { ...baseSearch, nsfw: 'false' as BooleanBothQuery } - await check(search, true) - } - - { - const search = { ...baseSearch, nsfw: 'both' as BooleanBothQuery } - await check(search, true) - } - }) - - it('Should search by host', async function () { - { - const search = { ...baseSearch, host: 'example.com' } - await check(search, false) - } - - { - const search = { ...baseSearch, host: 'framatube.org' } - await check(search, true) - } - }) - - it('Should search by uuids', async function () { - const goodUUID = '9c9de5e8-0a1e-484a-b099-e80766180a6d' - const goodShortUUID = 'kkGMgK9ZtnKfYAgnEtQxbv' - const badUUID = 'c29c5b77-4a04-493d-96a9-2e9267e308f0' - const badShortUUID = 'rP5RgUeX9XwTSrspCdkDej' - - { - const uuidsMatrix = [ - [ goodUUID ], - [ goodUUID, badShortUUID ], - [ badShortUUID, goodShortUUID ], - [ goodUUID, goodShortUUID ] - ] - - for (const uuids of uuidsMatrix) { - const search = { ...baseSearch, uuids } - await check(search, true) - } - } - - { - const uuidsMatrix = [ - [ badUUID ], - [ badShortUUID ] - ] - - for (const uuids of uuidsMatrix) { - const search = { ...baseSearch, uuids } - await check(search, false) - } - } - }) - - it('Should have a correct pagination', async function () { - const search = { - search: 'video', - start: 0, - count: 5 - } - - const body = await command.advancedVideoSearch({ search }) - - expect(body.total).to.be.greaterThan(5) - expect(body.data).to.have.lengthOf(5) - }) - - it('Should use the nsfw instance policy as default', async function () { - let nsfwUUID: string - - { - await server.config.updateCustomSubConfig({ - newConfig: { - instance: { defaultNSFWPolicy: 'display' } - } - }) - - const body = await command.searchVideos({ search: 'NSFW search index', sort: '-match' }) - expect(body.data).to.have.length.greaterThan(0) - - const video = body.data[0] - expect(video.nsfw).to.be.true - - nsfwUUID = video.uuid - } - - { - await server.config.updateCustomSubConfig({ - newConfig: { - instance: { defaultNSFWPolicy: 'do_not_list' } - } - }) - - const body = await command.searchVideos({ search: 'NSFW search index', sort: '-match' }) - - try { - expect(body.data).to.have.lengthOf(0) - } catch { - const video = body.data[0] - - expect(video.uuid).not.equal(nsfwUUID) - } - } - }) - }) - - describe('Channels search', async function () { - - async function check (search: VideoChannelsSearchQuery, exists = true) { - const body = await command.advancedChannelSearch({ search }) - - if (exists === false) { - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - return - } - - expect(body.total).to.be.greaterThan(0) - expect(body.data).to.have.length.greaterThan(0) - - const videoChannel = body.data[0] - expect(videoChannel.url).to.equal('https://framatube.org/video-channels/bf54d359-cfad-4935-9d45-9d6be93f63e8') - expect(videoChannel.host).to.equal('framatube.org') - expect(videoChannel.avatars.length).to.equal(2, 'Channel should have two avatar images') - expect(videoChannel.displayName).to.exist - - expect(videoChannel.ownerAccount.url).to.equal('https://framatube.org/accounts/framasoft') - expect(videoChannel.ownerAccount.name).to.equal('framasoft') - expect(videoChannel.ownerAccount.host).to.equal('framatube.org') - expect(videoChannel.ownerAccount.avatars.length).to.equal(2, 'Account should have two avatar images') - } - - it('Should make a simple search and not have results', async function () { - const body = await command.searchChannels({ search: 'a'.repeat(500) }) - - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - }) - - it('Should make a search and have results', async function () { - await check({ search: 'Framasoft', sort: 'createdAt' }, true) - }) - - it('Should make host search and have appropriate results', async function () { - await check({ search: 'Framasoft videos', host: 'example.com' }, false) - await check({ search: 'Framasoft videos', host: 'framatube.org' }, true) - }) - - it('Should make handles search and have appropriate results', async function () { - await check({ handles: [ 'bf54d359-cfad-4935-9d45-9d6be93f63e8@framatube.org' ] }, true) - await check({ handles: [ 'jeanine', 'bf54d359-cfad-4935-9d45-9d6be93f63e8@framatube.org' ] }, true) - await check({ handles: [ 'jeanine', 'chocobozzz_channel2@peertube2.cpy.re' ] }, false) - }) - - it('Should have a correct pagination', async function () { - const body = await command.advancedChannelSearch({ search: { search: 'root', start: 0, count: 2 } }) - - expect(body.total).to.be.greaterThan(2) - expect(body.data).to.have.lengthOf(2) - }) - }) - - describe('Playlists search', async function () { - - async function check (search: VideoPlaylistsSearchQuery, exists = true) { - const body = await command.advancedPlaylistSearch({ search }) - - if (exists === false) { - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - return - } - - expect(body.total).to.be.greaterThan(0) - expect(body.data).to.have.length.greaterThan(0) - - const videoPlaylist = body.data[0] - - expect(videoPlaylist.url).to.equal('https://peertube2.cpy.re/videos/watch/playlist/73804a40-da9a-40c2-b1eb-2c6d9eec8f0a') - expect(videoPlaylist.thumbnailUrl).to.exist - expect(videoPlaylist.embedUrl).to.equal('https://peertube2.cpy.re/video-playlists/embed/73804a40-da9a-40c2-b1eb-2c6d9eec8f0a') - - expect(videoPlaylist.type.id).to.equal(VideoPlaylistType.REGULAR) - expect(videoPlaylist.privacy.id).to.equal(VideoPlaylistPrivacy.PUBLIC) - expect(videoPlaylist.videosLength).to.exist - - expect(videoPlaylist.createdAt).to.exist - expect(videoPlaylist.updatedAt).to.exist - - expect(videoPlaylist.uuid).to.equal('73804a40-da9a-40c2-b1eb-2c6d9eec8f0a') - expect(videoPlaylist.displayName).to.exist - - expect(videoPlaylist.ownerAccount.url).to.equal('https://peertube2.cpy.re/accounts/chocobozzz') - expect(videoPlaylist.ownerAccount.name).to.equal('chocobozzz') - expect(videoPlaylist.ownerAccount.host).to.equal('peertube2.cpy.re') - expect(videoPlaylist.ownerAccount.avatars.length).to.equal(2, 'Account should have two avatar images') - - expect(videoPlaylist.videoChannel.url).to.equal('https://peertube2.cpy.re/video-channels/chocobozzz_channel') - expect(videoPlaylist.videoChannel.name).to.equal('chocobozzz_channel') - expect(videoPlaylist.videoChannel.host).to.equal('peertube2.cpy.re') - expect(videoPlaylist.videoChannel.avatars.length).to.equal(2, 'Channel should have two avatar images') - } - - it('Should make a simple search and not have results', async function () { - const body = await command.searchPlaylists({ search: 'a'.repeat(500) }) - - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - }) - - it('Should make a search and have results', async function () { - await check({ search: 'E2E playlist', sort: '-match' }, true) - }) - - it('Should make host search and have appropriate results', async function () { - await check({ search: 'E2E playlist', host: 'example.com' }, false) - await check({ search: 'E2E playlist', host: 'peertube2.cpy.re', sort: '-match' }, true) - }) - - it('Should make a search by uuids and have appropriate results', async function () { - const goodUUID = '73804a40-da9a-40c2-b1eb-2c6d9eec8f0a' - const goodShortUUID = 'fgei1ws1oa6FCaJ2qZPG29' - const badUUID = 'c29c5b77-4a04-493d-96a9-2e9267e308f0' - const badShortUUID = 'rP5RgUeX9XwTSrspCdkDej' - - { - const uuidsMatrix = [ - [ goodUUID ], - [ goodUUID, badShortUUID ], - [ badShortUUID, goodShortUUID ], - [ goodUUID, goodShortUUID ] - ] - - for (const uuids of uuidsMatrix) { - const search = { search: 'E2E playlist', sort: '-match', uuids } - await check(search, true) - } - } - - { - const uuidsMatrix = [ - [ badUUID ], - [ badShortUUID ] - ] - - for (const uuids of uuidsMatrix) { - const search = { search: 'E2E playlist', sort: '-match', uuids } - await check(search, false) - } - } - }) - - it('Should have a correct pagination', async function () { - const body = await command.advancedChannelSearch({ search: { search: 'root', start: 0, count: 2 } }) - - expect(body.total).to.be.greaterThan(2) - expect(body.data).to.have.lengthOf(2) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/search/search-playlists.ts b/server/tests/api/search/search-playlists.ts deleted file mode 100644 index a357674c2..000000000 --- a/server/tests/api/search/search-playlists.ts +++ /dev/null @@ -1,180 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { VideoPlaylistPrivacy } from '@shared/models' -import { - cleanupTests, - createSingleServer, - doubleFollow, - PeerTubeServer, - SearchCommand, - setAccessTokensToServers, - setDefaultAccountAvatar, - setDefaultChannelAvatar, - setDefaultVideoChannel -} from '@shared/server-commands' - -describe('Test playlists search', function () { - let server: PeerTubeServer - let remoteServer: PeerTubeServer - let command: SearchCommand - let playlistUUID: string - let playlistShortUUID: string - - before(async function () { - this.timeout(120000) - - const servers = await Promise.all([ - createSingleServer(1), - createSingleServer(2) - ]) - server = servers[0] - remoteServer = servers[1] - - await setAccessTokensToServers([ remoteServer, server ]) - await setDefaultVideoChannel([ remoteServer, server ]) - await setDefaultChannelAvatar([ remoteServer, server ]) - await setDefaultAccountAvatar([ remoteServer, server ]) - - await servers[1].config.disableTranscoding() - - { - const videoId = (await server.videos.upload()).uuid - - const attributes = { - displayName: 'Dr. Kenzo Tenma hospital videos', - privacy: VideoPlaylistPrivacy.PUBLIC, - videoChannelId: server.store.channel.id - } - const created = await server.playlists.create({ attributes }) - playlistUUID = created.uuid - playlistShortUUID = created.shortUUID - - await server.playlists.addElement({ playlistId: created.id, attributes: { videoId } }) - } - - { - const videoId = (await remoteServer.videos.upload()).uuid - - const attributes = { - displayName: 'Johan & Anna Libert music videos', - privacy: VideoPlaylistPrivacy.PUBLIC, - videoChannelId: remoteServer.store.channel.id - } - const created = await remoteServer.playlists.create({ attributes }) - - await remoteServer.playlists.addElement({ playlistId: created.id, attributes: { videoId } }) - } - - { - const attributes = { - displayName: 'Inspector Lunge playlist', - privacy: VideoPlaylistPrivacy.PUBLIC, - videoChannelId: server.store.channel.id - } - await server.playlists.create({ attributes }) - } - - await doubleFollow(server, remoteServer) - - command = server.search - }) - - it('Should make a simple search and not have results', async function () { - const body = await command.searchPlaylists({ search: 'abc' }) - - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - }) - - it('Should make a search and have results', async function () { - { - const search = { - search: 'tenma', - start: 0, - count: 1 - } - const body = await command.advancedPlaylistSearch({ search }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - - const playlist = body.data[0] - expect(playlist.displayName).to.equal('Dr. Kenzo Tenma hospital videos') - expect(playlist.url).to.equal(server.url + '/video-playlists/' + playlist.uuid) - } - - { - const search = { - search: 'Anna Livert music', - start: 0, - count: 1 - } - const body = await command.advancedPlaylistSearch({ search }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - - const playlist = body.data[0] - expect(playlist.displayName).to.equal('Johan & Anna Libert music videos') - } - }) - - it('Should filter by host', async function () { - { - const search = { search: 'tenma', host: server.host } - const body = await command.advancedPlaylistSearch({ search }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - - const playlist = body.data[0] - expect(playlist.displayName).to.equal('Dr. Kenzo Tenma hospital videos') - } - - { - const search = { search: 'Anna', host: 'example.com' } - const body = await command.advancedPlaylistSearch({ search }) - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - } - - { - const search = { search: 'video', host: remoteServer.host } - const body = await command.advancedPlaylistSearch({ search }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - - const playlist = body.data[0] - expect(playlist.displayName).to.equal('Johan & Anna Libert music videos') - } - }) - - it('Should filter by UUIDs', async function () { - for (const uuid of [ playlistUUID, playlistShortUUID ]) { - const body = await command.advancedPlaylistSearch({ search: { uuids: [ uuid ] } }) - - expect(body.total).to.equal(1) - expect(body.data[0].displayName).to.equal('Dr. Kenzo Tenma hospital videos') - } - - { - const body = await command.advancedPlaylistSearch({ search: { uuids: [ 'dfd70b83-639f-4980-94af-304a56ab4b35' ] } }) - - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - } - }) - - it('Should not display playlists without videos', async function () { - const search = { - search: 'Lunge', - start: 0, - count: 1 - } - const body = await command.advancedPlaylistSearch({ search }) - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - }) - - after(async function () { - await cleanupTests([ server, remoteServer ]) - }) -}) diff --git a/server/tests/api/search/search-videos.ts b/server/tests/api/search/search-videos.ts deleted file mode 100644 index f7f50147d..000000000 --- a/server/tests/api/search/search-videos.ts +++ /dev/null @@ -1,568 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { wait } from '@shared/core-utils' -import { VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createSingleServer, - doubleFollow, - PeerTubeServer, - SearchCommand, - setAccessTokensToServers, - setDefaultAccountAvatar, - setDefaultChannelAvatar, - setDefaultVideoChannel, - stopFfmpeg -} from '@shared/server-commands' - -describe('Test videos search', function () { - let server: PeerTubeServer - let remoteServer: PeerTubeServer - let startDate: string - let videoUUID: string - let videoShortUUID: string - - let command: SearchCommand - - before(async function () { - this.timeout(360000) - - const servers = await Promise.all([ - createSingleServer(1), - createSingleServer(2) - ]) - server = servers[0] - remoteServer = servers[1] - - await setAccessTokensToServers([ server, remoteServer ]) - await setDefaultVideoChannel([ server, remoteServer ]) - await setDefaultChannelAvatar(server) - await setDefaultAccountAvatar(servers) - - { - const attributes1 = { - name: '1111 2222 3333', - fixture: '60fps_720p_small.mp4', // 2 seconds - category: 1, - licence: 1, - nsfw: false, - language: 'fr' - } - await server.videos.upload({ attributes: attributes1 }) - - const attributes2 = { ...attributes1, name: attributes1.name + ' - 2', fixture: 'video_short.mp4' } - await server.videos.upload({ attributes: attributes2 }) - - { - const attributes3 = { ...attributes1, name: attributes1.name + ' - 3', language: undefined } - const { id, uuid, shortUUID } = await server.videos.upload({ attributes: attributes3 }) - videoUUID = uuid - videoShortUUID = shortUUID - - await server.captions.add({ - language: 'en', - videoId: id, - fixture: 'subtitle-good2.vtt', - mimeType: 'application/octet-stream' - }) - - await server.captions.add({ - language: 'aa', - videoId: id, - fixture: 'subtitle-good2.vtt', - mimeType: 'application/octet-stream' - }) - } - - const attributes4 = { ...attributes1, name: attributes1.name + ' - 4', language: 'pl', nsfw: true } - await server.videos.upload({ attributes: attributes4 }) - - await wait(1000) - - startDate = new Date().toISOString() - - const attributes5 = { ...attributes1, name: attributes1.name + ' - 5', licence: 2, language: undefined } - await server.videos.upload({ attributes: attributes5 }) - - const attributes6 = { ...attributes1, name: attributes1.name + ' - 6', tags: [ 't1', 't2' ] } - await server.videos.upload({ attributes: attributes6 }) - - const attributes7 = { ...attributes1, name: attributes1.name + ' - 7', originallyPublishedAt: '2019-02-12T09:58:08.286Z' } - await server.videos.upload({ attributes: attributes7 }) - - const attributes8 = { ...attributes1, name: attributes1.name + ' - 8', licence: 4 } - await server.videos.upload({ attributes: attributes8 }) - } - - { - const attributes = { - name: '3333 4444 5555', - fixture: 'video_short.mp4', - category: 2, - licence: 2, - language: 'en' - } - await server.videos.upload({ attributes }) - - await server.videos.upload({ attributes: { ...attributes, name: attributes.name + ' duplicate' } }) - } - - { - const attributes = { - name: '6666 7777 8888', - fixture: 'video_short.mp4', - category: 3, - licence: 3, - language: 'pl' - } - await server.videos.upload({ attributes }) - } - - { - const attributes1 = { - name: '9999', - tags: [ 'aaaa', 'bbbb', 'cccc' ], - category: 1 - } - await server.videos.upload({ attributes: attributes1 }) - await server.videos.upload({ attributes: { ...attributes1, category: 2 } }) - - await server.videos.upload({ attributes: { ...attributes1, tags: [ 'cccc', 'dddd' ] } }) - await server.videos.upload({ attributes: { ...attributes1, tags: [ 'eeee', 'ffff' ] } }) - } - - { - const attributes1 = { - name: 'aaaa 2', - category: 1 - } - await server.videos.upload({ attributes: attributes1 }) - await server.videos.upload({ attributes: { ...attributes1, category: 2 } }) - } - - { - await remoteServer.videos.upload({ attributes: { name: 'remote video 1' } }) - await remoteServer.videos.upload({ attributes: { name: 'remote video 2' } }) - } - - await doubleFollow(server, remoteServer) - - command = server.search - }) - - it('Should make a simple search and not have results', async function () { - const body = await command.searchVideos({ search: 'abc' }) - - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - }) - - it('Should make a simple search and have results', async function () { - const body = await command.searchVideos({ search: '4444 5555 duplicate' }) - - expect(body.total).to.equal(2) - - const videos = body.data - expect(videos).to.have.lengthOf(2) - - // bestmatch - expect(videos[0].name).to.equal('3333 4444 5555 duplicate') - expect(videos[1].name).to.equal('3333 4444 5555') - }) - - it('Should make a search on tags too, and have results', async function () { - const search = { - search: 'aaaa', - categoryOneOf: [ 1 ] - } - const body = await command.advancedVideoSearch({ search }) - - expect(body.total).to.equal(2) - - const videos = body.data - expect(videos).to.have.lengthOf(2) - - // bestmatch - expect(videos[0].name).to.equal('aaaa 2') - expect(videos[1].name).to.equal('9999') - }) - - it('Should filter on tags without a search', async function () { - const search = { - tagsAllOf: [ 'bbbb' ] - } - const body = await command.advancedVideoSearch({ search }) - - expect(body.total).to.equal(2) - - const videos = body.data - expect(videos).to.have.lengthOf(2) - - expect(videos[0].name).to.equal('9999') - expect(videos[1].name).to.equal('9999') - }) - - it('Should filter on category without a search', async function () { - const search = { - categoryOneOf: [ 3 ] - } - const body = await command.advancedVideoSearch({ search }) - - expect(body.total).to.equal(1) - - const videos = body.data - expect(videos).to.have.lengthOf(1) - - expect(videos[0].name).to.equal('6666 7777 8888') - }) - - it('Should search by tags (one of)', async function () { - const query = { - search: '9999', - categoryOneOf: [ 1 ], - tagsOneOf: [ 'aAaa', 'ffff' ] - } - - { - const body = await command.advancedVideoSearch({ search: query }) - expect(body.total).to.equal(2) - } - - { - const body = await command.advancedVideoSearch({ search: { ...query, tagsOneOf: [ 'blabla' ] } }) - expect(body.total).to.equal(0) - } - }) - - it('Should search by tags (all of)', async function () { - const query = { - search: '9999', - categoryOneOf: [ 1 ], - tagsAllOf: [ 'CCcc' ] - } - - { - const body = await command.advancedVideoSearch({ search: query }) - expect(body.total).to.equal(2) - } - - { - const body = await command.advancedVideoSearch({ search: { ...query, tagsAllOf: [ 'blAbla' ] } }) - expect(body.total).to.equal(0) - } - - { - const body = await command.advancedVideoSearch({ search: { ...query, tagsAllOf: [ 'bbbb', 'CCCC' ] } }) - expect(body.total).to.equal(1) - } - }) - - it('Should search by category', async function () { - const query = { - search: '6666', - categoryOneOf: [ 3 ] - } - - { - const body = await command.advancedVideoSearch({ search: query }) - expect(body.total).to.equal(1) - expect(body.data[0].name).to.equal('6666 7777 8888') - } - - { - const body = await command.advancedVideoSearch({ search: { ...query, categoryOneOf: [ 2 ] } }) - expect(body.total).to.equal(0) - } - }) - - it('Should search by licence', async function () { - const query = { - search: '4444 5555', - licenceOneOf: [ 2 ] - } - - { - const body = await command.advancedVideoSearch({ search: query }) - expect(body.total).to.equal(2) - expect(body.data[0].name).to.equal('3333 4444 5555') - expect(body.data[1].name).to.equal('3333 4444 5555 duplicate') - } - - { - const body = await command.advancedVideoSearch({ search: { ...query, licenceOneOf: [ 3 ] } }) - expect(body.total).to.equal(0) - } - }) - - it('Should search by languages', async function () { - const query = { - search: '1111 2222 3333', - languageOneOf: [ 'pl', 'en' ] - } - - { - const body = await command.advancedVideoSearch({ search: query }) - expect(body.total).to.equal(2) - expect(body.data[0].name).to.equal('1111 2222 3333 - 3') - expect(body.data[1].name).to.equal('1111 2222 3333 - 4') - } - - { - const body = await command.advancedVideoSearch({ search: { ...query, languageOneOf: [ 'pl', 'en', '_unknown' ] } }) - expect(body.total).to.equal(3) - expect(body.data[0].name).to.equal('1111 2222 3333 - 3') - expect(body.data[1].name).to.equal('1111 2222 3333 - 4') - expect(body.data[2].name).to.equal('1111 2222 3333 - 5') - } - - { - const body = await command.advancedVideoSearch({ search: { ...query, languageOneOf: [ 'eo' ] } }) - expect(body.total).to.equal(0) - } - }) - - it('Should search by start date', async function () { - const query = { - search: '1111 2222 3333', - startDate - } - - const body = await command.advancedVideoSearch({ search: query }) - expect(body.total).to.equal(4) - - const videos = body.data - expect(videos[0].name).to.equal('1111 2222 3333 - 5') - expect(videos[1].name).to.equal('1111 2222 3333 - 6') - expect(videos[2].name).to.equal('1111 2222 3333 - 7') - expect(videos[3].name).to.equal('1111 2222 3333 - 8') - }) - - it('Should make an advanced search', async function () { - const query = { - search: '1111 2222 3333', - languageOneOf: [ 'pl', 'fr' ], - durationMax: 4, - nsfw: 'false' as 'false', - licenceOneOf: [ 1, 4 ] - } - - const body = await command.advancedVideoSearch({ search: query }) - expect(body.total).to.equal(4) - - const videos = body.data - expect(videos[0].name).to.equal('1111 2222 3333') - expect(videos[1].name).to.equal('1111 2222 3333 - 6') - expect(videos[2].name).to.equal('1111 2222 3333 - 7') - expect(videos[3].name).to.equal('1111 2222 3333 - 8') - }) - - it('Should make an advanced search and sort results', async function () { - const query = { - search: '1111 2222 3333', - languageOneOf: [ 'pl', 'fr' ], - durationMax: 4, - nsfw: 'false' as 'false', - licenceOneOf: [ 1, 4 ], - sort: '-name' - } - - const body = await command.advancedVideoSearch({ search: query }) - expect(body.total).to.equal(4) - - const videos = body.data - expect(videos[0].name).to.equal('1111 2222 3333 - 8') - expect(videos[1].name).to.equal('1111 2222 3333 - 7') - expect(videos[2].name).to.equal('1111 2222 3333 - 6') - expect(videos[3].name).to.equal('1111 2222 3333') - }) - - it('Should make an advanced search and only show the first result', async function () { - const query = { - search: '1111 2222 3333', - languageOneOf: [ 'pl', 'fr' ], - durationMax: 4, - nsfw: 'false' as 'false', - licenceOneOf: [ 1, 4 ], - sort: '-name', - start: 0, - count: 1 - } - - const body = await command.advancedVideoSearch({ search: query }) - expect(body.total).to.equal(4) - - const videos = body.data - expect(videos[0].name).to.equal('1111 2222 3333 - 8') - }) - - it('Should make an advanced search and only show the last result', async function () { - const query = { - search: '1111 2222 3333', - languageOneOf: [ 'pl', 'fr' ], - durationMax: 4, - nsfw: 'false' as 'false', - licenceOneOf: [ 1, 4 ], - sort: '-name', - start: 3, - count: 1 - } - - const body = await command.advancedVideoSearch({ search: query }) - expect(body.total).to.equal(4) - - const videos = body.data - expect(videos[0].name).to.equal('1111 2222 3333') - }) - - it('Should search on originally published date', async function () { - const baseQuery = { - search: '1111 2222 3333', - languageOneOf: [ 'pl', 'fr' ], - durationMax: 4, - nsfw: 'false' as 'false', - licenceOneOf: [ 1, 4 ] - } - - { - const query = { ...baseQuery, originallyPublishedStartDate: '2019-02-11T09:58:08.286Z' } - const body = await command.advancedVideoSearch({ search: query }) - - expect(body.total).to.equal(1) - expect(body.data[0].name).to.equal('1111 2222 3333 - 7') - } - - { - const query = { ...baseQuery, originallyPublishedEndDate: '2019-03-11T09:58:08.286Z' } - const body = await command.advancedVideoSearch({ search: query }) - - expect(body.total).to.equal(1) - expect(body.data[0].name).to.equal('1111 2222 3333 - 7') - } - - { - const query = { ...baseQuery, originallyPublishedEndDate: '2019-01-11T09:58:08.286Z' } - const body = await command.advancedVideoSearch({ search: query }) - - expect(body.total).to.equal(0) - } - - { - const query = { ...baseQuery, originallyPublishedStartDate: '2019-03-11T09:58:08.286Z' } - const body = await command.advancedVideoSearch({ search: query }) - - expect(body.total).to.equal(0) - } - - { - const query = { - ...baseQuery, - originallyPublishedStartDate: '2019-01-11T09:58:08.286Z', - originallyPublishedEndDate: '2019-01-10T09:58:08.286Z' - } - const body = await command.advancedVideoSearch({ search: query }) - - expect(body.total).to.equal(0) - } - - { - const query = { - ...baseQuery, - originallyPublishedStartDate: '2019-01-11T09:58:08.286Z', - originallyPublishedEndDate: '2019-04-11T09:58:08.286Z' - } - const body = await command.advancedVideoSearch({ search: query }) - - expect(body.total).to.equal(1) - expect(body.data[0].name).to.equal('1111 2222 3333 - 7') - } - }) - - it('Should search by UUID', async function () { - const search = videoUUID - const body = await command.advancedVideoSearch({ search: { search } }) - - expect(body.total).to.equal(1) - expect(body.data[0].name).to.equal('1111 2222 3333 - 3') - }) - - it('Should filter by UUIDs', async function () { - for (const uuid of [ videoUUID, videoShortUUID ]) { - const body = await command.advancedVideoSearch({ search: { uuids: [ uuid ] } }) - - expect(body.total).to.equal(1) - expect(body.data[0].name).to.equal('1111 2222 3333 - 3') - } - - { - const body = await command.advancedVideoSearch({ search: { uuids: [ 'dfd70b83-639f-4980-94af-304a56ab4b35' ] } }) - - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - } - }) - - it('Should search by host', async function () { - { - const body = await command.advancedVideoSearch({ search: { search: '6666 7777 8888', host: server.host } }) - expect(body.total).to.equal(1) - expect(body.data[0].name).to.equal('6666 7777 8888') - } - - { - const body = await command.advancedVideoSearch({ search: { search: '1111', host: 'example.com' } }) - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - } - - { - const body = await command.advancedVideoSearch({ search: { search: 'remote', host: remoteServer.host } }) - expect(body.total).to.equal(2) - expect(body.data).to.have.lengthOf(2) - expect(body.data[0].name).to.equal('remote video 1') - expect(body.data[1].name).to.equal('remote video 2') - } - }) - - it('Should search by live', async function () { - this.timeout(120000) - - { - const newConfig = { - search: { - searchIndex: { enabled: false } - }, - live: { enabled: true } - } - await server.config.updateCustomSubConfig({ newConfig }) - } - - { - const body = await command.advancedVideoSearch({ search: { isLive: true } }) - - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - } - - { - const liveCommand = server.live - - const liveAttributes = { name: 'live', privacy: VideoPrivacy.PUBLIC, channelId: server.store.channel.id } - const live = await liveCommand.create({ fields: liveAttributes }) - - const ffmpegCommand = await liveCommand.sendRTMPStreamInVideo({ videoId: live.id }) - await liveCommand.waitUntilPublished({ videoId: live.id }) - - const body = await command.advancedVideoSearch({ search: { isLive: true } }) - - expect(body.total).to.equal(1) - expect(body.data[0].name).to.equal('live') - - await stopFfmpeg(ffmpegCommand) - } - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/server/auto-follows.ts b/server/tests/api/server/auto-follows.ts deleted file mode 100644 index 6ce1a3799..000000000 --- a/server/tests/api/server/auto-follows.ts +++ /dev/null @@ -1,189 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { MockInstancesIndex } from '@server/tests/shared' -import { wait } from '@shared/core-utils' -import { cleanupTests, createMultipleServers, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/server-commands' - -async function checkFollow (follower: PeerTubeServer, following: PeerTubeServer, exists: boolean) { - { - const body = await following.follows.getFollowers({ start: 0, count: 5, sort: '-createdAt' }) - const follow = body.data.find(f => f.follower.host === follower.host && f.state === 'accepted') - - if (exists === true) expect(follow, `Follower ${follower.url} should exist on ${following.url}`).to.exist - else expect(follow, `Follower ${follower.url} should not exist on ${following.url}`).to.be.undefined - } - - { - const body = await follower.follows.getFollowings({ start: 0, count: 5, sort: '-createdAt' }) - const follow = body.data.find(f => f.following.host === following.host && f.state === 'accepted') - - if (exists === true) expect(follow, `Following ${following.url} should exist on ${follower.url}`).to.exist - else expect(follow, `Following ${following.url} should not exist on ${follower.url}`).to.be.undefined - } -} - -async function server1Follows2 (servers: PeerTubeServer[]) { - await servers[0].follows.follow({ hosts: [ servers[1].host ] }) - - await waitJobs(servers) -} - -async function resetFollows (servers: PeerTubeServer[]) { - try { - await servers[0].follows.unfollow({ target: servers[1] }) - await servers[1].follows.unfollow({ target: servers[0] }) - } catch { /* empty */ - } - - await waitJobs(servers) - - await checkFollow(servers[0], servers[1], false) - await checkFollow(servers[1], servers[0], false) -} - -describe('Test auto follows', function () { - let servers: PeerTubeServer[] = [] - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(3) - - // Get the access tokens - await setAccessTokensToServers(servers) - }) - - describe('Auto follow back', function () { - - it('Should not auto follow back if the option is not enabled', async function () { - this.timeout(15000) - - await server1Follows2(servers) - - await checkFollow(servers[0], servers[1], true) - await checkFollow(servers[1], servers[0], false) - - await resetFollows(servers) - }) - - it('Should auto follow back on auto accept if the option is enabled', async function () { - this.timeout(15000) - - const config = { - followings: { - instance: { - autoFollowBack: { enabled: true } - } - } - } - await servers[1].config.updateCustomSubConfig({ newConfig: config }) - - await server1Follows2(servers) - - await checkFollow(servers[0], servers[1], true) - await checkFollow(servers[1], servers[0], true) - - await resetFollows(servers) - }) - - it('Should wait the acceptation before auto follow back', async function () { - this.timeout(30000) - - const config = { - followings: { - instance: { - autoFollowBack: { enabled: true } - } - }, - followers: { - instance: { - manualApproval: true - } - } - } - await servers[1].config.updateCustomSubConfig({ newConfig: config }) - - await server1Follows2(servers) - - await checkFollow(servers[0], servers[1], false) - await checkFollow(servers[1], servers[0], false) - - await servers[1].follows.acceptFollower({ follower: 'peertube@' + servers[0].host }) - await waitJobs(servers) - - await checkFollow(servers[0], servers[1], true) - await checkFollow(servers[1], servers[0], true) - - await resetFollows(servers) - - config.followings.instance.autoFollowBack.enabled = false - config.followers.instance.manualApproval = false - await servers[1].config.updateCustomSubConfig({ newConfig: config }) - }) - }) - - describe('Auto follow index', function () { - const instanceIndexServer = new MockInstancesIndex() - let port: number - - before(async function () { - port = await instanceIndexServer.initialize() - }) - - it('Should not auto follow index if the option is not enabled', async function () { - this.timeout(30000) - - await wait(5000) - await waitJobs(servers) - - await checkFollow(servers[0], servers[1], false) - await checkFollow(servers[1], servers[0], false) - }) - - it('Should auto follow the index', async function () { - this.timeout(30000) - - instanceIndexServer.addInstance(servers[1].host) - - const config = { - followings: { - instance: { - autoFollowIndex: { - indexUrl: `http://127.0.0.1:${port}/api/v1/instances/hosts`, - enabled: true - } - } - } - } - await servers[0].config.updateCustomSubConfig({ newConfig: config }) - - await wait(5000) - await waitJobs(servers) - - await checkFollow(servers[0], servers[1], true) - - await resetFollows(servers) - }) - - it('Should follow new added instances in the index but not old ones', async function () { - this.timeout(30000) - - instanceIndexServer.addInstance(servers[2].host) - - await wait(5000) - await waitJobs(servers) - - await checkFollow(servers[0], servers[1], false) - await checkFollow(servers[0], servers[2], true) - }) - - after(async function () { - await instanceIndexServer.terminate() - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/server/bulk.ts b/server/tests/api/server/bulk.ts deleted file mode 100644 index 66d791a0f..000000000 --- a/server/tests/api/server/bulk.ts +++ /dev/null @@ -1,185 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { - BulkCommand, - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test bulk actions', function () { - const commentsUser3: { videoId: number, commentId: number }[] = [] - - let servers: PeerTubeServer[] = [] - let user1Token: string - let user2Token: string - let user3Token: string - - let bulkCommand: BulkCommand - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(2) - - // Get the access tokens - await setAccessTokensToServers(servers) - - { - const user = { username: 'user1', password: 'password' } - await servers[0].users.create({ username: user.username, password: user.password }) - - user1Token = await servers[0].login.getAccessToken(user) - } - - { - const user = { username: 'user2', password: 'password' } - await servers[0].users.create({ username: user.username, password: user.password }) - - user2Token = await servers[0].login.getAccessToken(user) - } - - { - const user = { username: 'user3', password: 'password' } - await servers[1].users.create({ username: user.username, password: user.password }) - - user3Token = await servers[1].login.getAccessToken(user) - } - - await doubleFollow(servers[0], servers[1]) - - bulkCommand = new BulkCommand(servers[0]) - }) - - describe('Bulk remove comments', function () { - async function checkInstanceCommentsRemoved () { - { - const { data } = await servers[0].videos.list() - - // Server 1 should not have these comments anymore - for (const video of data) { - const { data } = await servers[0].comments.listThreads({ videoId: video.id }) - const comment = data.find(c => c.text === 'comment by user 3') - - expect(comment).to.not.exist - } - } - - { - const { data } = await servers[1].videos.list() - - // Server 1 should not have these comments on videos of server 1 - for (const video of data) { - const { data } = await servers[1].comments.listThreads({ videoId: video.id }) - const comment = data.find(c => c.text === 'comment by user 3') - - if (video.account.host === servers[0].host) { - expect(comment).to.not.exist - } else { - expect(comment).to.exist - } - } - } - } - - before(async function () { - this.timeout(240000) - - await servers[0].videos.upload({ attributes: { name: 'video 1 server 1' } }) - await servers[0].videos.upload({ attributes: { name: 'video 2 server 1' } }) - await servers[0].videos.upload({ token: user1Token, attributes: { name: 'video 3 server 1' } }) - - await servers[1].videos.upload({ attributes: { name: 'video 1 server 2' } }) - - await waitJobs(servers) - - { - const { data } = await servers[0].videos.list() - for (const video of data) { - await servers[0].comments.createThread({ videoId: video.id, text: 'comment by root server 1' }) - await servers[0].comments.createThread({ token: user1Token, videoId: video.id, text: 'comment by user 1' }) - await servers[0].comments.createThread({ token: user2Token, videoId: video.id, text: 'comment by user 2' }) - } - } - - { - const { data } = await servers[1].videos.list() - - for (const video of data) { - await servers[1].comments.createThread({ videoId: video.id, text: 'comment by root server 2' }) - - const comment = await servers[1].comments.createThread({ token: user3Token, videoId: video.id, text: 'comment by user 3' }) - commentsUser3.push({ videoId: video.id, commentId: comment.id }) - } - } - - await waitJobs(servers) - }) - - it('Should delete comments of an account on my videos', async function () { - this.timeout(60000) - - await bulkCommand.removeCommentsOf({ - token: user1Token, - attributes: { - accountName: 'user2', - scope: 'my-videos' - } - }) - - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.videos.list() - - for (const video of data) { - const { data } = await server.comments.listThreads({ videoId: video.id }) - const comment = data.find(c => c.text === 'comment by user 2') - - if (video.name === 'video 3 server 1') expect(comment).to.not.exist - else expect(comment).to.exist - } - } - }) - - it('Should delete comments of an account on the instance', async function () { - this.timeout(60000) - - await bulkCommand.removeCommentsOf({ - attributes: { - accountName: 'user3@' + servers[1].host, - scope: 'instance' - } - }) - - await waitJobs(servers) - - await checkInstanceCommentsRemoved() - }) - - it('Should not re create the comment on video update', async function () { - this.timeout(60000) - - for (const obj of commentsUser3) { - await servers[1].comments.addReply({ - token: user3Token, - videoId: obj.videoId, - toCommentId: obj.commentId, - text: 'comment by user 3 bis' - }) - } - - await waitJobs(servers) - - await checkInstanceCommentsRemoved() - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/server/config-defaults.ts b/server/tests/api/server/config-defaults.ts deleted file mode 100644 index 041032f2b..000000000 --- a/server/tests/api/server/config-defaults.ts +++ /dev/null @@ -1,288 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { FIXTURE_URLS } from '@server/tests/shared' -import { VideoDetails, VideoPrivacy } from '@shared/models' -import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers, setDefaultVideoChannel } from '@shared/server-commands' - -describe('Test config defaults', function () { - let server: PeerTubeServer - let channelId: number - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - channelId = server.store.channel.id - }) - - describe('Default publish values', function () { - - before(async function () { - const overrideConfig = { - defaults: { - publish: { - comments_enabled: false, - download_enabled: false, - privacy: VideoPrivacy.INTERNAL, - licence: 4 - } - } - } - - await server.kill() - await server.run(overrideConfig) - }) - - const attributes = { - name: 'video', - downloadEnabled: undefined, - commentsEnabled: undefined, - licence: undefined, - privacy: VideoPrivacy.PUBLIC // Privacy is mandatory for server - } - - function checkVideo (video: VideoDetails) { - expect(video.downloadEnabled).to.be.false - expect(video.commentsEnabled).to.be.false - expect(video.licence.id).to.equal(4) - } - - before(async function () { - await server.config.disableTranscoding() - await server.config.enableImports() - await server.config.enableLive({ allowReplay: false, transcoding: false }) - }) - - it('Should have the correct server configuration', async function () { - const config = await server.config.getConfig() - - expect(config.defaults.publish.commentsEnabled).to.be.false - expect(config.defaults.publish.downloadEnabled).to.be.false - expect(config.defaults.publish.licence).to.equal(4) - expect(config.defaults.publish.privacy).to.equal(VideoPrivacy.INTERNAL) - }) - - it('Should respect default values when uploading a video', async function () { - for (const mode of [ 'legacy' as 'legacy', 'resumable' as 'resumable' ]) { - const { id } = await server.videos.upload({ attributes, mode }) - - const video = await server.videos.get({ id }) - checkVideo(video) - } - }) - - it('Should respect default values when importing a video using URL', async function () { - const { video: { id } } = await server.imports.importVideo({ - attributes: { - ...attributes, - channelId, - targetUrl: FIXTURE_URLS.goodVideo - } - }) - - const video = await server.videos.get({ id }) - checkVideo(video) - }) - - it('Should respect default values when importing a video using magnet URI', async function () { - const { video: { id } } = await server.imports.importVideo({ - attributes: { - ...attributes, - channelId, - magnetUri: FIXTURE_URLS.magnet - } - }) - - const video = await server.videos.get({ id }) - checkVideo(video) - }) - - it('Should respect default values when creating a live', async function () { - const { id } = await server.live.create({ - fields: { - ...attributes, - channelId - } - }) - - const video = await server.videos.get({ id }) - checkVideo(video) - }) - }) - - describe('Default P2P values', function () { - - describe('Webapp default value', function () { - - before(async function () { - const overrideConfig = { - defaults: { - p2p: { - webapp: { - enabled: false - } - } - } - } - - await server.kill() - await server.run(overrideConfig) - }) - - it('Should have appropriate P2P config', async function () { - const config = await server.config.getConfig() - - expect(config.defaults.p2p.webapp.enabled).to.be.false - expect(config.defaults.p2p.embed.enabled).to.be.true - }) - - it('Should create a user with this default setting', async function () { - await server.users.create({ username: 'user_p2p_1' }) - const userToken = await server.login.getAccessToken('user_p2p_1') - - const { p2pEnabled } = await server.users.getMyInfo({ token: userToken }) - expect(p2pEnabled).to.be.false - }) - - it('Should register a user with this default setting', async function () { - await server.registrations.register({ username: 'user_p2p_2' }) - - const userToken = await server.login.getAccessToken('user_p2p_2') - - const { p2pEnabled } = await server.users.getMyInfo({ token: userToken }) - expect(p2pEnabled).to.be.false - }) - }) - - describe('Embed default value', function () { - - before(async function () { - const overrideConfig = { - defaults: { - p2p: { - embed: { - enabled: false - } - } - }, - signup: { - limit: 15 - } - } - - await server.kill() - await server.run(overrideConfig) - }) - - it('Should have appropriate P2P config', async function () { - const config = await server.config.getConfig() - - expect(config.defaults.p2p.webapp.enabled).to.be.true - expect(config.defaults.p2p.embed.enabled).to.be.false - }) - - it('Should create a user with this default setting', async function () { - await server.users.create({ username: 'user_p2p_3' }) - const userToken = await server.login.getAccessToken('user_p2p_3') - - const { p2pEnabled } = await server.users.getMyInfo({ token: userToken }) - expect(p2pEnabled).to.be.true - }) - - it('Should register a user with this default setting', async function () { - await server.registrations.register({ username: 'user_p2p_4' }) - - const userToken = await server.login.getAccessToken('user_p2p_4') - - const { p2pEnabled } = await server.users.getMyInfo({ token: userToken }) - expect(p2pEnabled).to.be.true - }) - }) - }) - - describe('Default user attributes', function () { - it('Should create a user and register a user with the default config', async function () { - await server.config.updateCustomSubConfig({ - newConfig: { - user: { - history: { - videos: { - enabled: true - } - }, - videoQuota : -1, - videoQuotaDaily: -1 - }, - signup: { - enabled: true, - requiresApproval: false - } - } - }) - - const config = await server.config.getConfig() - - expect(config.user.videoQuota).to.equal(-1) - expect(config.user.videoQuotaDaily).to.equal(-1) - - const user1Token = await server.users.generateUserAndToken('user1') - const user1 = await server.users.getMyInfo({ token: user1Token }) - - const user = { displayName: 'super user 2', username: 'user2', password: 'super password' } - const channel = { name: 'my_user_2_channel', displayName: 'my channel' } - await server.registrations.register({ ...user, channel }) - const user2Token = await server.login.getAccessToken(user) - const user2 = await server.users.getMyInfo({ token: user2Token }) - - for (const user of [ user1, user2 ]) { - expect(user.videosHistoryEnabled).to.be.true - expect(user.videoQuota).to.equal(-1) - expect(user.videoQuotaDaily).to.equal(-1) - } - }) - - it('Should update config and create a user and register a user with the new default config', async function () { - await server.config.updateCustomSubConfig({ - newConfig: { - user: { - history: { - videos: { - enabled: false - } - }, - videoQuota : 5242881, - videoQuotaDaily: 318742 - }, - signup: { - enabled: true, - requiresApproval: false - } - } - }) - - const user3Token = await server.users.generateUserAndToken('user3') - const user3 = await server.users.getMyInfo({ token: user3Token }) - - const user = { displayName: 'super user 4', username: 'user4', password: 'super password' } - const channel = { name: 'my_user_4_channel', displayName: 'my channel' } - await server.registrations.register({ ...user, channel }) - const user4Token = await server.login.getAccessToken(user) - const user4 = await server.users.getMyInfo({ token: user4Token }) - - for (const user of [ user3, user4 ]) { - expect(user.videosHistoryEnabled).to.be.false - expect(user.videoQuota).to.equal(5242881) - expect(user.videoQuotaDaily).to.equal(318742) - } - }) - - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/server/config.ts b/server/tests/api/server/config.ts deleted file mode 100644 index a614d92d2..000000000 --- a/server/tests/api/server/config.ts +++ /dev/null @@ -1,645 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { parallelTests } from '@shared/core-utils' -import { CustomConfig, HttpStatusCode } from '@shared/models' -import { - cleanupTests, - createSingleServer, - killallServers, - makeGetRequest, - PeerTubeServer, - setAccessTokensToServers -} from '@shared/server-commands' - -function checkInitialConfig (server: PeerTubeServer, data: CustomConfig) { - expect(data.instance.name).to.equal('PeerTube') - expect(data.instance.shortDescription).to.equal( - 'PeerTube, an ActivityPub-federated video streaming platform using P2P directly in your web browser.' - ) - expect(data.instance.description).to.equal('Welcome to this PeerTube instance!') - - expect(data.instance.terms).to.equal('No terms for now.') - expect(data.instance.creationReason).to.be.empty - expect(data.instance.codeOfConduct).to.be.empty - expect(data.instance.moderationInformation).to.be.empty - expect(data.instance.administrator).to.be.empty - expect(data.instance.maintenanceLifetime).to.be.empty - expect(data.instance.businessModel).to.be.empty - expect(data.instance.hardwareInformation).to.be.empty - - expect(data.instance.languages).to.have.lengthOf(0) - expect(data.instance.categories).to.have.lengthOf(0) - - expect(data.instance.defaultClientRoute).to.equal('/videos/trending') - expect(data.instance.isNSFW).to.be.false - expect(data.instance.defaultNSFWPolicy).to.equal('display') - expect(data.instance.customizations.css).to.be.empty - expect(data.instance.customizations.javascript).to.be.empty - - expect(data.services.twitter.username).to.equal('@Chocobozzz') - expect(data.services.twitter.whitelisted).to.be.false - - expect(data.client.videos.miniature.preferAuthorDisplayName).to.be.false - expect(data.client.menu.login.redirectOnSingleExternalAuth).to.be.false - - expect(data.cache.previews.size).to.equal(1) - expect(data.cache.captions.size).to.equal(1) - expect(data.cache.torrents.size).to.equal(1) - expect(data.cache.storyboards.size).to.equal(1) - - expect(data.signup.enabled).to.be.true - expect(data.signup.limit).to.equal(4) - expect(data.signup.minimumAge).to.equal(16) - expect(data.signup.requiresApproval).to.be.false - expect(data.signup.requiresEmailVerification).to.be.false - - expect(data.admin.email).to.equal('admin' + server.internalServerNumber + '@example.com') - expect(data.contactForm.enabled).to.be.true - - expect(data.user.history.videos.enabled).to.be.true - expect(data.user.videoQuota).to.equal(5242880) - expect(data.user.videoQuotaDaily).to.equal(-1) - - expect(data.videoChannels.maxPerUser).to.equal(20) - - expect(data.transcoding.enabled).to.be.false - expect(data.transcoding.remoteRunners.enabled).to.be.false - expect(data.transcoding.allowAdditionalExtensions).to.be.false - expect(data.transcoding.allowAudioFiles).to.be.false - expect(data.transcoding.threads).to.equal(2) - expect(data.transcoding.concurrency).to.equal(2) - expect(data.transcoding.profile).to.equal('default') - expect(data.transcoding.resolutions['144p']).to.be.false - expect(data.transcoding.resolutions['240p']).to.be.true - expect(data.transcoding.resolutions['360p']).to.be.true - expect(data.transcoding.resolutions['480p']).to.be.true - expect(data.transcoding.resolutions['720p']).to.be.true - expect(data.transcoding.resolutions['1080p']).to.be.true - expect(data.transcoding.resolutions['1440p']).to.be.true - expect(data.transcoding.resolutions['2160p']).to.be.true - expect(data.transcoding.alwaysTranscodeOriginalResolution).to.be.true - expect(data.transcoding.webVideos.enabled).to.be.true - expect(data.transcoding.hls.enabled).to.be.true - - expect(data.live.enabled).to.be.false - expect(data.live.allowReplay).to.be.false - expect(data.live.latencySetting.enabled).to.be.true - expect(data.live.maxDuration).to.equal(-1) - expect(data.live.maxInstanceLives).to.equal(20) - expect(data.live.maxUserLives).to.equal(3) - expect(data.live.transcoding.enabled).to.be.false - expect(data.live.transcoding.remoteRunners.enabled).to.be.false - expect(data.live.transcoding.threads).to.equal(2) - expect(data.live.transcoding.profile).to.equal('default') - expect(data.live.transcoding.resolutions['144p']).to.be.false - expect(data.live.transcoding.resolutions['240p']).to.be.false - expect(data.live.transcoding.resolutions['360p']).to.be.false - expect(data.live.transcoding.resolutions['480p']).to.be.false - expect(data.live.transcoding.resolutions['720p']).to.be.false - expect(data.live.transcoding.resolutions['1080p']).to.be.false - expect(data.live.transcoding.resolutions['1440p']).to.be.false - expect(data.live.transcoding.resolutions['2160p']).to.be.false - expect(data.live.transcoding.alwaysTranscodeOriginalResolution).to.be.true - - expect(data.videoStudio.enabled).to.be.false - expect(data.videoStudio.remoteRunners.enabled).to.be.false - - expect(data.videoFile.update.enabled).to.be.false - - expect(data.import.videos.concurrency).to.equal(2) - expect(data.import.videos.http.enabled).to.be.true - expect(data.import.videos.torrent.enabled).to.be.true - expect(data.autoBlacklist.videos.ofUsers.enabled).to.be.false - - expect(data.followers.instance.enabled).to.be.true - expect(data.followers.instance.manualApproval).to.be.false - - expect(data.followings.instance.autoFollowBack.enabled).to.be.false - expect(data.followings.instance.autoFollowIndex.enabled).to.be.false - expect(data.followings.instance.autoFollowIndex.indexUrl).to.equal('') - - expect(data.broadcastMessage.enabled).to.be.false - expect(data.broadcastMessage.level).to.equal('info') - expect(data.broadcastMessage.message).to.equal('') - expect(data.broadcastMessage.dismissable).to.be.false -} - -function checkUpdatedConfig (data: CustomConfig) { - expect(data.instance.name).to.equal('PeerTube updated') - expect(data.instance.shortDescription).to.equal('my short description') - expect(data.instance.description).to.equal('my super description') - - expect(data.instance.terms).to.equal('my super terms') - expect(data.instance.creationReason).to.equal('my super creation reason') - expect(data.instance.codeOfConduct).to.equal('my super coc') - expect(data.instance.moderationInformation).to.equal('my super moderation information') - expect(data.instance.administrator).to.equal('Kuja') - expect(data.instance.maintenanceLifetime).to.equal('forever') - expect(data.instance.businessModel).to.equal('my super business model') - expect(data.instance.hardwareInformation).to.equal('2vCore 3GB RAM') - - expect(data.instance.languages).to.deep.equal([ 'en', 'es' ]) - expect(data.instance.categories).to.deep.equal([ 1, 2 ]) - - expect(data.instance.defaultClientRoute).to.equal('/videos/recently-added') - expect(data.instance.isNSFW).to.be.true - expect(data.instance.defaultNSFWPolicy).to.equal('blur') - expect(data.instance.customizations.javascript).to.equal('alert("coucou")') - expect(data.instance.customizations.css).to.equal('body { background-color: red; }') - - expect(data.services.twitter.username).to.equal('@Kuja') - expect(data.services.twitter.whitelisted).to.be.true - - expect(data.client.videos.miniature.preferAuthorDisplayName).to.be.true - expect(data.client.menu.login.redirectOnSingleExternalAuth).to.be.true - - expect(data.cache.previews.size).to.equal(2) - expect(data.cache.captions.size).to.equal(3) - expect(data.cache.torrents.size).to.equal(4) - expect(data.cache.storyboards.size).to.equal(5) - - expect(data.signup.enabled).to.be.false - expect(data.signup.limit).to.equal(5) - expect(data.signup.requiresApproval).to.be.false - expect(data.signup.requiresEmailVerification).to.be.false - expect(data.signup.minimumAge).to.equal(10) - - // We override admin email in parallel tests, so skip this exception - if (parallelTests() === false) { - expect(data.admin.email).to.equal('superadmin1@example.com') - } - - expect(data.contactForm.enabled).to.be.false - - expect(data.user.history.videos.enabled).to.be.false - expect(data.user.videoQuota).to.equal(5242881) - expect(data.user.videoQuotaDaily).to.equal(318742) - - expect(data.videoChannels.maxPerUser).to.equal(24) - - expect(data.transcoding.enabled).to.be.true - expect(data.transcoding.remoteRunners.enabled).to.be.true - expect(data.transcoding.threads).to.equal(1) - expect(data.transcoding.concurrency).to.equal(3) - expect(data.transcoding.allowAdditionalExtensions).to.be.true - expect(data.transcoding.allowAudioFiles).to.be.true - expect(data.transcoding.profile).to.equal('vod_profile') - expect(data.transcoding.resolutions['144p']).to.be.false - expect(data.transcoding.resolutions['240p']).to.be.false - expect(data.transcoding.resolutions['360p']).to.be.true - expect(data.transcoding.resolutions['480p']).to.be.true - expect(data.transcoding.resolutions['720p']).to.be.false - expect(data.transcoding.resolutions['1080p']).to.be.false - expect(data.transcoding.resolutions['2160p']).to.be.false - expect(data.transcoding.alwaysTranscodeOriginalResolution).to.be.false - expect(data.transcoding.hls.enabled).to.be.false - expect(data.transcoding.webVideos.enabled).to.be.true - - expect(data.live.enabled).to.be.true - expect(data.live.allowReplay).to.be.true - expect(data.live.latencySetting.enabled).to.be.false - expect(data.live.maxDuration).to.equal(5000) - expect(data.live.maxInstanceLives).to.equal(-1) - expect(data.live.maxUserLives).to.equal(10) - expect(data.live.transcoding.enabled).to.be.true - expect(data.live.transcoding.remoteRunners.enabled).to.be.true - expect(data.live.transcoding.threads).to.equal(4) - expect(data.live.transcoding.profile).to.equal('live_profile') - expect(data.live.transcoding.resolutions['144p']).to.be.true - expect(data.live.transcoding.resolutions['240p']).to.be.true - expect(data.live.transcoding.resolutions['360p']).to.be.true - expect(data.live.transcoding.resolutions['480p']).to.be.true - expect(data.live.transcoding.resolutions['720p']).to.be.true - expect(data.live.transcoding.resolutions['1080p']).to.be.true - expect(data.live.transcoding.resolutions['2160p']).to.be.true - expect(data.live.transcoding.alwaysTranscodeOriginalResolution).to.be.false - - expect(data.videoStudio.enabled).to.be.true - expect(data.videoStudio.remoteRunners.enabled).to.be.true - - expect(data.videoFile.update.enabled).to.be.true - - expect(data.import.videos.concurrency).to.equal(4) - expect(data.import.videos.http.enabled).to.be.false - expect(data.import.videos.torrent.enabled).to.be.false - expect(data.autoBlacklist.videos.ofUsers.enabled).to.be.true - - expect(data.followers.instance.enabled).to.be.false - expect(data.followers.instance.manualApproval).to.be.true - - expect(data.followings.instance.autoFollowBack.enabled).to.be.true - expect(data.followings.instance.autoFollowIndex.enabled).to.be.true - expect(data.followings.instance.autoFollowIndex.indexUrl).to.equal('https://updated.example.com') - - expect(data.broadcastMessage.enabled).to.be.true - expect(data.broadcastMessage.level).to.equal('error') - expect(data.broadcastMessage.message).to.equal('super bad message') - expect(data.broadcastMessage.dismissable).to.be.true -} - -const newCustomConfig: CustomConfig = { - instance: { - name: 'PeerTube updated', - shortDescription: 'my short description', - description: 'my super description', - terms: 'my super terms', - codeOfConduct: 'my super coc', - - creationReason: 'my super creation reason', - moderationInformation: 'my super moderation information', - administrator: 'Kuja', - maintenanceLifetime: 'forever', - businessModel: 'my super business model', - hardwareInformation: '2vCore 3GB RAM', - - languages: [ 'en', 'es' ], - categories: [ 1, 2 ], - - isNSFW: true, - defaultNSFWPolicy: 'blur' as 'blur', - - defaultClientRoute: '/videos/recently-added', - - customizations: { - javascript: 'alert("coucou")', - css: 'body { background-color: red; }' - } - }, - theme: { - default: 'default' - }, - services: { - twitter: { - username: '@Kuja', - whitelisted: true - } - }, - client: { - videos: { - miniature: { - preferAuthorDisplayName: true - } - }, - menu: { - login: { - redirectOnSingleExternalAuth: true - } - } - }, - cache: { - previews: { - size: 2 - }, - captions: { - size: 3 - }, - torrents: { - size: 4 - }, - storyboards: { - size: 5 - } - }, - signup: { - enabled: false, - limit: 5, - requiresApproval: false, - requiresEmailVerification: false, - minimumAge: 10 - }, - admin: { - email: 'superadmin1@example.com' - }, - contactForm: { - enabled: false - }, - user: { - history: { - videos: { - enabled: false - } - }, - videoQuota: 5242881, - videoQuotaDaily: 318742 - }, - videoChannels: { - maxPerUser: 24 - }, - transcoding: { - enabled: true, - remoteRunners: { - enabled: true - }, - allowAdditionalExtensions: true, - allowAudioFiles: true, - threads: 1, - concurrency: 3, - profile: 'vod_profile', - resolutions: { - '0p': false, - '144p': false, - '240p': false, - '360p': true, - '480p': true, - '720p': false, - '1080p': false, - '1440p': false, - '2160p': false - }, - alwaysTranscodeOriginalResolution: false, - webVideos: { - enabled: true - }, - hls: { - enabled: false - } - }, - live: { - enabled: true, - allowReplay: true, - latencySetting: { - enabled: false - }, - maxDuration: 5000, - maxInstanceLives: -1, - maxUserLives: 10, - transcoding: { - enabled: true, - remoteRunners: { - enabled: true - }, - threads: 4, - profile: 'live_profile', - resolutions: { - '144p': true, - '240p': true, - '360p': true, - '480p': true, - '720p': true, - '1080p': true, - '1440p': true, - '2160p': true - }, - alwaysTranscodeOriginalResolution: false - } - }, - videoStudio: { - enabled: true, - remoteRunners: { - enabled: true - } - }, - videoFile: { - update: { - enabled: true - } - }, - import: { - videos: { - concurrency: 4, - http: { - enabled: false - }, - torrent: { - enabled: false - } - }, - videoChannelSynchronization: { - enabled: false, - maxPerUser: 10 - } - }, - trending: { - videos: { - algorithms: { - enabled: [ 'hot', 'most-viewed', 'most-liked' ], - default: 'hot' - } - } - }, - autoBlacklist: { - videos: { - ofUsers: { - enabled: true - } - } - }, - followers: { - instance: { - enabled: false, - manualApproval: true - } - }, - followings: { - instance: { - autoFollowBack: { - enabled: true - }, - autoFollowIndex: { - enabled: true, - indexUrl: 'https://updated.example.com' - } - } - }, - broadcastMessage: { - enabled: true, - level: 'error', - message: 'super bad message', - dismissable: true - }, - search: { - remoteUri: { - anonymous: true, - users: true - }, - searchIndex: { - enabled: true, - url: 'https://search.joinpeertube.org', - disableLocalSearch: true, - isDefaultSearch: true - } - } -} - -describe('Test static config', function () { - let server: PeerTubeServer = null - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1, { webadmin: { configuration: { edition: { allowed: false } } } }) - await setAccessTokensToServers([ server ]) - }) - - it('Should tell the client that edits are not allowed', async function () { - const data = await server.config.getConfig() - - expect(data.webadmin.configuration.edition.allowed).to.be.false - }) - - it('Should error when client tries to update', async function () { - await server.config.updateCustomConfig({ newCustomConfig, expectedStatus: 405 }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) - -describe('Test config', function () { - let server: PeerTubeServer = null - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - }) - - it('Should have a correct config on a server with registration enabled', async function () { - const data = await server.config.getConfig() - - expect(data.signup.allowed).to.be.true - }) - - it('Should have a correct config on a server with registration enabled and a users limit', async function () { - this.timeout(5000) - - await Promise.all([ - server.registrations.register({ username: 'user1' }), - server.registrations.register({ username: 'user2' }), - server.registrations.register({ username: 'user3' }) - ]) - - const data = await server.config.getConfig() - - expect(data.signup.allowed).to.be.false - }) - - it('Should have the correct video allowed extensions', async function () { - const data = await server.config.getConfig() - - expect(data.video.file.extensions).to.have.lengthOf(3) - expect(data.video.file.extensions).to.contain('.mp4') - expect(data.video.file.extensions).to.contain('.webm') - expect(data.video.file.extensions).to.contain('.ogv') - - await server.videos.upload({ attributes: { fixture: 'video_short.mkv' }, expectedStatus: HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415 }) - await server.videos.upload({ attributes: { fixture: 'sample.ogg' }, expectedStatus: HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415 }) - - expect(data.contactForm.enabled).to.be.true - }) - - it('Should get the customized configuration', async function () { - const data = await server.config.getCustomConfig() - - checkInitialConfig(server, data) - }) - - it('Should update the customized configuration', async function () { - await server.config.updateCustomConfig({ newCustomConfig }) - - const data = await server.config.getCustomConfig() - checkUpdatedConfig(data) - }) - - it('Should have the correct updated video allowed extensions', async function () { - this.timeout(30000) - - const data = await server.config.getConfig() - - expect(data.video.file.extensions).to.have.length.above(4) - expect(data.video.file.extensions).to.contain('.mp4') - expect(data.video.file.extensions).to.contain('.webm') - expect(data.video.file.extensions).to.contain('.ogv') - expect(data.video.file.extensions).to.contain('.flv') - expect(data.video.file.extensions).to.contain('.wmv') - expect(data.video.file.extensions).to.contain('.mkv') - expect(data.video.file.extensions).to.contain('.mp3') - expect(data.video.file.extensions).to.contain('.ogg') - expect(data.video.file.extensions).to.contain('.flac') - - await server.videos.upload({ attributes: { fixture: 'video_short.mkv' }, expectedStatus: HttpStatusCode.OK_200 }) - await server.videos.upload({ attributes: { fixture: 'sample.ogg' }, expectedStatus: HttpStatusCode.OK_200 }) - }) - - it('Should have the configuration updated after a restart', async function () { - this.timeout(30000) - - await killallServers([ server ]) - - await server.run() - - const data = await server.config.getCustomConfig() - - checkUpdatedConfig(data) - }) - - it('Should fetch the about information', async function () { - const data = await server.config.getAbout() - - expect(data.instance.name).to.equal('PeerTube updated') - expect(data.instance.shortDescription).to.equal('my short description') - expect(data.instance.description).to.equal('my super description') - expect(data.instance.terms).to.equal('my super terms') - expect(data.instance.codeOfConduct).to.equal('my super coc') - - expect(data.instance.creationReason).to.equal('my super creation reason') - expect(data.instance.moderationInformation).to.equal('my super moderation information') - expect(data.instance.administrator).to.equal('Kuja') - expect(data.instance.maintenanceLifetime).to.equal('forever') - expect(data.instance.businessModel).to.equal('my super business model') - expect(data.instance.hardwareInformation).to.equal('2vCore 3GB RAM') - - expect(data.instance.languages).to.deep.equal([ 'en', 'es' ]) - expect(data.instance.categories).to.deep.equal([ 1, 2 ]) - }) - - it('Should remove the custom configuration', async function () { - await server.config.deleteCustomConfig() - - const data = await server.config.getCustomConfig() - checkInitialConfig(server, data) - }) - - it('Should enable/disable security headers', async function () { - this.timeout(25000) - - { - const res = await makeGetRequest({ - url: server.url, - path: '/api/v1/config', - expectedStatus: 200 - }) - - expect(res.headers['x-frame-options']).to.exist - expect(res.headers['x-powered-by']).to.equal('PeerTube') - } - - await killallServers([ server ]) - - const config = { - security: { - frameguard: { enabled: false }, - powered_by_header: { enabled: false } - } - } - await server.run(config) - - { - const res = await makeGetRequest({ - url: server.url, - path: '/api/v1/config', - expectedStatus: 200 - }) - - expect(res.headers['x-frame-options']).to.not.exist - expect(res.headers['x-powered-by']).to.not.exist - } - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/server/contact-form.ts b/server/tests/api/server/contact-form.ts deleted file mode 100644 index 0256cc193..000000000 --- a/server/tests/api/server/contact-form.ts +++ /dev/null @@ -1,101 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { MockSmtpServer } from '@server/tests/shared' -import { wait } from '@shared/core-utils' -import { HttpStatusCode } from '@shared/models' -import { - cleanupTests, - ConfigCommand, - ContactFormCommand, - createSingleServer, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test contact form', function () { - let server: PeerTubeServer - const emails: object[] = [] - let command: ContactFormCommand - - before(async function () { - this.timeout(30000) - - const port = await MockSmtpServer.Instance.collectEmails(emails) - - server = await createSingleServer(1, ConfigCommand.getEmailOverrideConfig(port)) - await setAccessTokensToServers([ server ]) - - command = server.contactForm - }) - - it('Should send a contact form', async function () { - await command.send({ - fromEmail: 'toto@example.com', - body: 'my super message', - subject: 'my subject', - fromName: 'Super toto' - }) - - await waitJobs(server) - - expect(emails).to.have.lengthOf(1) - - const email = emails[0] - - expect(email['from'][0]['address']).equal('test-admin@127.0.0.1') - expect(email['replyTo'][0]['address']).equal('toto@example.com') - expect(email['to'][0]['address']).equal('admin' + server.internalServerNumber + '@example.com') - expect(email['subject']).contains('my subject') - expect(email['text']).contains('my super message') - }) - - it('Should not have duplicated email address in text message', async function () { - const text = emails[0]['text'] as string - - const matches = text.match(/toto@example.com/g) - expect(matches).to.have.lengthOf(1) - }) - - it('Should not be able to send another contact form because of the anti spam checker', async function () { - await wait(1000) - - await command.send({ - fromEmail: 'toto@example.com', - body: 'my super message', - subject: 'my subject', - fromName: 'Super toto' - }) - - await command.send({ - fromEmail: 'toto@example.com', - body: 'my super message', - fromName: 'Super toto', - subject: 'my subject', - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should be able to send another contact form after a while', async function () { - await wait(1000) - - await command.send({ - fromEmail: 'toto@example.com', - fromName: 'Super toto', - subject: 'my subject', - body: 'my super message' - }) - }) - - it('Should not have the manage preferences link in the email', async function () { - const email = emails[0] - expect(email['text']).to.not.contain('Manage your notification preferences') - }) - - after(async function () { - MockSmtpServer.Instance.kill() - - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/server/email.ts b/server/tests/api/server/email.ts deleted file mode 100644 index 9141cc697..000000000 --- a/server/tests/api/server/email.ts +++ /dev/null @@ -1,371 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { MockSmtpServer } from '@server/tests/shared' -import { HttpStatusCode } from '@shared/models' -import { - cleanupTests, - ConfigCommand, - createSingleServer, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test emails', function () { - let server: PeerTubeServer - let userId: number - let userId2: number - let userAccessToken: string - - let videoShortUUID: string - let videoId: number - - let videoUserUUID: string - - let verificationString: string - let verificationString2: string - - const emails: object[] = [] - const user = { - username: 'user_1', - password: 'super_password' - } - - before(async function () { - this.timeout(120000) - - const emailPort = await MockSmtpServer.Instance.collectEmails(emails) - server = await createSingleServer(1, ConfigCommand.getEmailOverrideConfig(emailPort)) - - await setAccessTokensToServers([ server ]) - await server.config.enableSignup(true) - - { - const created = await server.users.create({ username: user.username, password: user.password }) - userId = created.id - - userAccessToken = await server.login.getAccessToken(user) - } - - { - const attributes = { name: 'my super user video' } - const { uuid } = await server.videos.upload({ token: userAccessToken, attributes }) - videoUserUUID = uuid - } - - { - const attributes = { - name: 'my super name' - } - const { shortUUID, id } = await server.videos.upload({ attributes }) - videoShortUUID = shortUUID - videoId = id - } - }) - - describe('When resetting user password', function () { - - it('Should ask to reset the password', async function () { - await server.users.askResetPassword({ email: 'user_1@example.com' }) - - await waitJobs(server) - expect(emails).to.have.lengthOf(1) - - const email = emails[0] - - expect(email['from'][0]['name']).equal('PeerTube') - expect(email['from'][0]['address']).equal('test-admin@127.0.0.1') - expect(email['to'][0]['address']).equal('user_1@example.com') - expect(email['subject']).contains('password') - - const verificationStringMatches = /verificationString=([a-z0-9]+)/.exec(email['text']) - expect(verificationStringMatches).not.to.be.null - - verificationString = verificationStringMatches[1] - expect(verificationString).to.have.length.above(2) - - const userIdMatches = /userId=([0-9]+)/.exec(email['text']) - expect(userIdMatches).not.to.be.null - - userId = parseInt(userIdMatches[1], 10) - expect(verificationString).to.not.be.undefined - }) - - it('Should not reset the password with an invalid verification string', async function () { - await server.users.resetPassword({ - userId, - verificationString: verificationString + 'b', - password: 'super_password2', - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should reset the password', async function () { - await server.users.resetPassword({ userId, verificationString, password: 'super_password2' }) - }) - - it('Should not reset the password with the same verification string', async function () { - await server.users.resetPassword({ - userId, - verificationString, - password: 'super_password3', - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should login with this new password', async function () { - user.password = 'super_password2' - - await server.login.getAccessToken(user) - }) - }) - - describe('When creating a user without password', function () { - - it('Should send a create password email', async function () { - await server.users.create({ username: 'create_password', password: '' }) - - await waitJobs(server) - expect(emails).to.have.lengthOf(2) - - const email = emails[1] - - expect(email['from'][0]['name']).equal('PeerTube') - expect(email['from'][0]['address']).equal('test-admin@127.0.0.1') - expect(email['to'][0]['address']).equal('create_password@example.com') - expect(email['subject']).contains('account') - expect(email['subject']).contains('password') - - const verificationStringMatches = /verificationString=([a-z0-9]+)/.exec(email['text']) - expect(verificationStringMatches).not.to.be.null - - verificationString2 = verificationStringMatches[1] - expect(verificationString2).to.have.length.above(2) - - const userIdMatches = /userId=([0-9]+)/.exec(email['text']) - expect(userIdMatches).not.to.be.null - - userId2 = parseInt(userIdMatches[1], 10) - }) - - it('Should not reset the password with an invalid verification string', async function () { - await server.users.resetPassword({ - userId: userId2, - verificationString: verificationString2 + 'c', - password: 'newly_created_password', - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should reset the password', async function () { - await server.users.resetPassword({ - userId: userId2, - verificationString: verificationString2, - password: 'newly_created_password' - }) - }) - - it('Should login with this new password', async function () { - await server.login.getAccessToken({ - username: 'create_password', - password: 'newly_created_password' - }) - }) - }) - - describe('When creating an abuse', function () { - - it('Should send the notification email', async function () { - const reason = 'my super bad reason' - await server.abuses.report({ token: userAccessToken, videoId, reason }) - - await waitJobs(server) - expect(emails).to.have.lengthOf(3) - - const email = emails[2] - - expect(email['from'][0]['name']).equal('PeerTube') - expect(email['from'][0]['address']).equal('test-admin@127.0.0.1') - expect(email['to'][0]['address']).equal('admin' + server.internalServerNumber + '@example.com') - expect(email['subject']).contains('abuse') - expect(email['text']).contains(videoShortUUID) - }) - }) - - describe('When blocking/unblocking user', function () { - - it('Should send the notification email when blocking a user', async function () { - const reason = 'my super bad reason' - await server.users.banUser({ userId, reason }) - - await waitJobs(server) - expect(emails).to.have.lengthOf(4) - - const email = emails[3] - - expect(email['from'][0]['name']).equal('PeerTube') - expect(email['from'][0]['address']).equal('test-admin@127.0.0.1') - expect(email['to'][0]['address']).equal('user_1@example.com') - expect(email['subject']).contains(' blocked') - expect(email['text']).contains(' blocked') - expect(email['text']).contains('bad reason') - }) - - it('Should send the notification email when unblocking a user', async function () { - await server.users.unbanUser({ userId }) - - await waitJobs(server) - expect(emails).to.have.lengthOf(5) - - const email = emails[4] - - expect(email['from'][0]['name']).equal('PeerTube') - expect(email['from'][0]['address']).equal('test-admin@127.0.0.1') - expect(email['to'][0]['address']).equal('user_1@example.com') - expect(email['subject']).contains(' unblocked') - expect(email['text']).contains(' unblocked') - }) - }) - - describe('When blacklisting a video', function () { - it('Should send the notification email', async function () { - const reason = 'my super reason' - await server.blacklist.add({ videoId: videoUserUUID, reason }) - - await waitJobs(server) - expect(emails).to.have.lengthOf(6) - - const email = emails[5] - - expect(email['from'][0]['name']).equal('PeerTube') - expect(email['from'][0]['address']).equal('test-admin@127.0.0.1') - expect(email['to'][0]['address']).equal('user_1@example.com') - expect(email['subject']).contains(' blacklisted') - expect(email['text']).contains('my super user video') - expect(email['text']).contains('my super reason') - }) - - it('Should send the notification email', async function () { - await server.blacklist.remove({ videoId: videoUserUUID }) - - await waitJobs(server) - expect(emails).to.have.lengthOf(7) - - const email = emails[6] - - expect(email['from'][0]['name']).equal('PeerTube') - expect(email['from'][0]['address']).equal('test-admin@127.0.0.1') - expect(email['to'][0]['address']).equal('user_1@example.com') - expect(email['subject']).contains(' unblacklisted') - expect(email['text']).contains('my super user video') - }) - - it('Should have the manage preferences link in the email', async function () { - const email = emails[6] - expect(email['text']).to.contain('Manage your notification preferences') - }) - }) - - describe('When verifying a user email', function () { - - it('Should ask to send the verification email', async function () { - await server.users.askSendVerifyEmail({ email: 'user_1@example.com' }) - - await waitJobs(server) - expect(emails).to.have.lengthOf(8) - - const email = emails[7] - - expect(email['from'][0]['name']).equal('PeerTube') - expect(email['from'][0]['address']).equal('test-admin@127.0.0.1') - expect(email['to'][0]['address']).equal('user_1@example.com') - expect(email['subject']).contains('Verify') - - const verificationStringMatches = /verificationString=([a-z0-9]+)/.exec(email['text']) - expect(verificationStringMatches).not.to.be.null - - verificationString = verificationStringMatches[1] - expect(verificationString).to.not.be.undefined - expect(verificationString).to.have.length.above(2) - - const userIdMatches = /userId=([0-9]+)/.exec(email['text']) - expect(userIdMatches).not.to.be.null - - userId = parseInt(userIdMatches[1], 10) - }) - - it('Should not verify the email with an invalid verification string', async function () { - await server.users.verifyEmail({ - userId, - verificationString: verificationString + 'b', - isPendingEmail: false, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should verify the email', async function () { - await server.users.verifyEmail({ userId, verificationString }) - }) - }) - - describe('When verifying a registration email', function () { - let registrationId: number - let registrationIdEmail: number - - before(async function () { - const { id } = await server.registrations.requestRegistration({ - username: 'request_1', - email: 'request_1@example.com', - registrationReason: 'tt' - }) - registrationId = id - }) - - it('Should ask to send the verification email', async function () { - await server.registrations.askSendVerifyEmail({ email: 'request_1@example.com' }) - - await waitJobs(server) - expect(emails).to.have.lengthOf(9) - - const email = emails[8] - - expect(email['from'][0]['name']).equal('PeerTube') - expect(email['from'][0]['address']).equal('test-admin@127.0.0.1') - expect(email['to'][0]['address']).equal('request_1@example.com') - expect(email['subject']).contains('Verify') - - const verificationStringMatches = /verificationString=([a-z0-9]+)/.exec(email['text']) - expect(verificationStringMatches).not.to.be.null - - verificationString = verificationStringMatches[1] - expect(verificationString).to.not.be.undefined - expect(verificationString).to.have.length.above(2) - - const registrationIdMatches = /registrationId=([0-9]+)/.exec(email['text']) - expect(registrationIdMatches).not.to.be.null - - registrationIdEmail = parseInt(registrationIdMatches[1], 10) - - expect(registrationId).to.equal(registrationIdEmail) - }) - - it('Should not verify the email with an invalid verification string', async function () { - await server.registrations.verifyEmail({ - registrationId: registrationIdEmail, - verificationString: verificationString + 'b', - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should verify the email', async function () { - await server.registrations.verifyEmail({ registrationId: registrationIdEmail, verificationString }) - }) - }) - - after(async function () { - MockSmtpServer.Instance.kill() - - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/server/follow-constraints.ts b/server/tests/api/server/follow-constraints.ts deleted file mode 100644 index ff5332858..000000000 --- a/server/tests/api/server/follow-constraints.ts +++ /dev/null @@ -1,321 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { HttpStatusCode, PeerTubeProblemDocument, ServerErrorCode } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test follow constraints', function () { - let servers: PeerTubeServer[] = [] - let video1UUID: string - let video2UUID: string - let userToken: string - - before(async function () { - this.timeout(240000) - - servers = await createMultipleServers(2) - - // Get the access tokens - await setAccessTokensToServers(servers) - - { - const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video server 1' } }) - video1UUID = uuid - } - { - const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video server 2' } }) - video2UUID = uuid - } - - const user = { - username: 'user1', - password: 'super_password' - } - await servers[0].users.create({ username: user.username, password: user.password }) - userToken = await servers[0].login.getAccessToken(user) - - await doubleFollow(servers[0], servers[1]) - }) - - describe('With a followed instance', function () { - - describe('With an unlogged user', function () { - - it('Should get the local video', async function () { - await servers[0].videos.get({ id: video1UUID }) - }) - - it('Should get the remote video', async function () { - await servers[0].videos.get({ id: video2UUID }) - }) - - it('Should list local account videos', async function () { - const { total, data } = await servers[0].videos.listByAccount({ handle: 'root@' + servers[0].host }) - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - }) - - it('Should list remote account videos', async function () { - const { total, data } = await servers[0].videos.listByAccount({ handle: 'root@' + servers[1].host }) - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - }) - - it('Should list local channel videos', async function () { - const handle = 'root_channel@' + servers[0].host - const { total, data } = await servers[0].videos.listByChannel({ handle }) - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - }) - - it('Should list remote channel videos', async function () { - const handle = 'root_channel@' + servers[1].host - const { total, data } = await servers[0].videos.listByChannel({ handle }) - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - }) - }) - - describe('With a logged user', function () { - it('Should get the local video', async function () { - await servers[0].videos.getWithToken({ token: userToken, id: video1UUID }) - }) - - it('Should get the remote video', async function () { - await servers[0].videos.getWithToken({ token: userToken, id: video2UUID }) - }) - - it('Should list local account videos', async function () { - const { total, data } = await servers[0].videos.listByAccount({ token: userToken, handle: 'root@' + servers[0].host }) - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - }) - - it('Should list remote account videos', async function () { - const { total, data } = await servers[0].videos.listByAccount({ token: userToken, handle: 'root@' + servers[1].host }) - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - }) - - it('Should list local channel videos', async function () { - const handle = 'root_channel@' + servers[0].host - const { total, data } = await servers[0].videos.listByChannel({ token: userToken, handle }) - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - }) - - it('Should list remote channel videos', async function () { - const handle = 'root_channel@' + servers[1].host - const { total, data } = await servers[0].videos.listByChannel({ token: userToken, handle }) - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - }) - }) - }) - - describe('With a non followed instance', function () { - - before(async function () { - this.timeout(30000) - - await servers[0].follows.unfollow({ target: servers[1] }) - }) - - describe('With an unlogged user', function () { - - it('Should get the local video', async function () { - await servers[0].videos.get({ id: video1UUID }) - }) - - it('Should not get the remote video', async function () { - const body = await servers[0].videos.get({ id: video2UUID, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - const error = body as unknown as PeerTubeProblemDocument - - const doc = 'https://docs.joinpeertube.org/api-rest-reference.html#section/Errors/does_not_respect_follow_constraints' - expect(error.type).to.equal(doc) - expect(error.code).to.equal(ServerErrorCode.DOES_NOT_RESPECT_FOLLOW_CONSTRAINTS) - - expect(error.detail).to.equal('Cannot get this video regarding follow constraints') - expect(error.error).to.equal(error.detail) - - expect(error.status).to.equal(HttpStatusCode.FORBIDDEN_403) - - expect(error.originUrl).to.contains(servers[1].url) - }) - - it('Should list local account videos', async function () { - const { total, data } = await servers[0].videos.listByAccount({ - token: null, - handle: 'root@' + servers[0].host - }) - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - }) - - it('Should not list remote account videos', async function () { - const { total, data } = await servers[0].videos.listByAccount({ - token: null, - handle: 'root@' + servers[1].host - }) - - expect(total).to.equal(0) - expect(data).to.have.lengthOf(0) - }) - - it('Should list local channel videos', async function () { - const handle = 'root_channel@' + servers[0].host - const { total, data } = await servers[0].videos.listByChannel({ token: null, handle }) - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - }) - - it('Should not list remote channel videos', async function () { - const handle = 'root_channel@' + servers[1].host - const { total, data } = await servers[0].videos.listByChannel({ token: null, handle }) - - expect(total).to.equal(0) - expect(data).to.have.lengthOf(0) - }) - }) - - describe('With a logged user', function () { - - it('Should get the local video', async function () { - await servers[0].videos.getWithToken({ token: userToken, id: video1UUID }) - }) - - it('Should get the remote video', async function () { - await servers[0].videos.getWithToken({ token: userToken, id: video2UUID }) - }) - - it('Should list local account videos', async function () { - const { total, data } = await servers[0].videos.listByAccount({ token: userToken, handle: 'root@' + servers[0].host }) - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - }) - - it('Should list remote account videos', async function () { - const { total, data } = await servers[0].videos.listByAccount({ token: userToken, handle: 'root@' + servers[1].host }) - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - }) - - it('Should list local channel videos', async function () { - const handle = 'root_channel@' + servers[0].host - const { total, data } = await servers[0].videos.listByChannel({ token: userToken, handle }) - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - }) - - it('Should list remote channel videos', async function () { - const handle = 'root_channel@' + servers[1].host - const { total, data } = await servers[0].videos.listByChannel({ token: userToken, handle }) - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - }) - }) - }) - - describe('When following a remote account', function () { - - before(async function () { - this.timeout(60000) - - await servers[0].follows.follow({ handles: [ 'root@' + servers[1].host ] }) - await waitJobs(servers) - }) - - it('Should get the remote video with an unlogged user', async function () { - await servers[0].videos.get({ id: video2UUID }) - }) - - it('Should get the remote video with a logged in user', async function () { - await servers[0].videos.getWithToken({ token: userToken, id: video2UUID }) - }) - }) - - describe('When unfollowing a remote account', function () { - - before(async function () { - this.timeout(60000) - - await servers[0].follows.unfollow({ target: 'root@' + servers[1].host }) - await waitJobs(servers) - }) - - it('Should not get the remote video with an unlogged user', async function () { - const body = await servers[0].videos.get({ id: video2UUID, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - - const error = body as unknown as PeerTubeProblemDocument - expect(error.code).to.equal(ServerErrorCode.DOES_NOT_RESPECT_FOLLOW_CONSTRAINTS) - }) - - it('Should get the remote video with a logged in user', async function () { - await servers[0].videos.getWithToken({ token: userToken, id: video2UUID }) - }) - }) - - describe('When following a remote channel', function () { - - before(async function () { - this.timeout(60000) - - await servers[0].follows.follow({ handles: [ 'root_channel@' + servers[1].host ] }) - await waitJobs(servers) - }) - - it('Should get the remote video with an unlogged user', async function () { - await servers[0].videos.get({ id: video2UUID }) - }) - - it('Should get the remote video with a logged in user', async function () { - await servers[0].videos.getWithToken({ token: userToken, id: video2UUID }) - }) - }) - - describe('When unfollowing a remote channel', function () { - - before(async function () { - this.timeout(60000) - - await servers[0].follows.unfollow({ target: 'root_channel@' + servers[1].host }) - await waitJobs(servers) - }) - - it('Should not get the remote video with an unlogged user', async function () { - const body = await servers[0].videos.get({ id: video2UUID, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - - const error = body as unknown as PeerTubeProblemDocument - expect(error.code).to.equal(ServerErrorCode.DOES_NOT_RESPECT_FOLLOW_CONSTRAINTS) - }) - - it('Should get the remote video with a logged in user', async function () { - await servers[0].videos.getWithToken({ token: userToken, id: video2UUID }) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/server/follows-moderation.ts b/server/tests/api/server/follows-moderation.ts deleted file mode 100644 index d145dd9d2..000000000 --- a/server/tests/api/server/follows-moderation.ts +++ /dev/null @@ -1,364 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { expectStartWith } from '@server/tests/shared' -import { ActorFollow, FollowState } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - FollowsCommand, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -async function checkServer1And2HasFollowers (servers: PeerTubeServer[], state = 'accepted') { - const fns = [ - servers[0].follows.getFollowings.bind(servers[0].follows), - servers[1].follows.getFollowers.bind(servers[1].follows) - ] - - for (const fn of fns) { - const body = await fn({ start: 0, count: 5, sort: 'createdAt' }) - expect(body.total).to.equal(1) - - const follow = body.data[0] - expect(follow.state).to.equal(state) - expect(follow.follower.url).to.equal(servers[0].url + '/accounts/peertube') - expect(follow.following.url).to.equal(servers[1].url + '/accounts/peertube') - } -} - -async function checkFollows (options: { - follower: PeerTubeServer - followerState: FollowState | 'deleted' - - following: PeerTubeServer - followingState: FollowState | 'deleted' -}) { - const { follower, followerState, followingState, following } = options - - const followerUrl = follower.url + '/accounts/peertube' - const followingUrl = following.url + '/accounts/peertube' - const finder = (d: ActorFollow) => d.follower.url === followerUrl && d.following.url === followingUrl - - { - const { data } = await follower.follows.getFollowings() - const follow = data.find(finder) - - if (followerState === 'deleted') { - expect(follow).to.not.exist - } else { - expect(follow.state).to.equal(followerState) - expect(follow.follower.url).to.equal(followerUrl) - expect(follow.following.url).to.equal(followingUrl) - } - } - - { - const { data } = await following.follows.getFollowers() - const follow = data.find(finder) - - if (followingState === 'deleted') { - expect(follow).to.not.exist - } else { - expect(follow.state).to.equal(followingState) - expect(follow.follower.url).to.equal(followerUrl) - expect(follow.following.url).to.equal(followingUrl) - } - } -} - -async function checkNoFollowers (servers: PeerTubeServer[]) { - const fns = [ - servers[0].follows.getFollowings.bind(servers[0].follows), - servers[1].follows.getFollowers.bind(servers[1].follows) - ] - - for (const fn of fns) { - const body = await fn({ start: 0, count: 5, sort: 'createdAt', state: 'accepted' }) - expect(body.total).to.equal(0) - } -} - -describe('Test follows moderation', function () { - let servers: PeerTubeServer[] = [] - let commands: FollowsCommand[] - - before(async function () { - this.timeout(240000) - - servers = await createMultipleServers(3) - - // Get the access tokens - await setAccessTokensToServers(servers) - - commands = servers.map(s => s.follows) - }) - - describe('Default behaviour', function () { - - it('Should have server 1 following server 2', async function () { - this.timeout(30000) - - await commands[0].follow({ hosts: [ servers[1].url ] }) - - await waitJobs(servers) - }) - - it('Should have correct follows', async function () { - await checkServer1And2HasFollowers(servers) - }) - - it('Should remove follower on server 2', async function () { - await commands[1].removeFollower({ follower: servers[0] }) - - await waitJobs(servers) - }) - - it('Should not not have follows anymore', async function () { - await checkNoFollowers(servers) - }) - }) - - describe('Disabled/Enabled followers', function () { - - it('Should disable followers on server 2', async function () { - const subConfig = { - followers: { - instance: { - enabled: false, - manualApproval: false - } - } - } - - await servers[1].config.updateCustomSubConfig({ newConfig: subConfig }) - - await commands[0].follow({ hosts: [ servers[1].url ] }) - await waitJobs(servers) - - await checkNoFollowers(servers) - }) - - it('Should re enable followers on server 2', async function () { - const subConfig = { - followers: { - instance: { - enabled: true, - manualApproval: false - } - } - } - - await servers[1].config.updateCustomSubConfig({ newConfig: subConfig }) - - await commands[0].follow({ hosts: [ servers[1].url ] }) - await waitJobs(servers) - - await checkServer1And2HasFollowers(servers) - }) - }) - - describe('Manual approbation', function () { - - it('Should manually approve followers', async function () { - this.timeout(20000) - - await commands[0].unfollow({ target: servers[1] }) - await waitJobs(servers) - - const subConfig = { - followers: { - instance: { - enabled: true, - manualApproval: true - } - } - } - - await servers[1].config.updateCustomSubConfig({ newConfig: subConfig }) - await servers[2].config.updateCustomSubConfig({ newConfig: subConfig }) - - await commands[0].follow({ hosts: [ servers[1].url ] }) - await waitJobs(servers) - - await checkServer1And2HasFollowers(servers, 'pending') - }) - - it('Should accept a follower', async function () { - await commands[1].acceptFollower({ follower: 'peertube@' + servers[0].host }) - await waitJobs(servers) - - await checkServer1And2HasFollowers(servers) - }) - - it('Should reject another follower', async function () { - this.timeout(20000) - - await commands[0].follow({ hosts: [ servers[2].url ] }) - await waitJobs(servers) - - { - const body = await commands[0].getFollowings() - expect(body.total).to.equal(2) - } - - { - const body = await commands[1].getFollowers() - expect(body.total).to.equal(1) - } - - { - const body = await commands[2].getFollowers() - expect(body.total).to.equal(1) - } - - await commands[2].rejectFollower({ follower: 'peertube@' + servers[0].host }) - await waitJobs(servers) - - { // server 1 - { - const { data } = await commands[0].getFollowings({ state: 'accepted' }) - expect(data).to.have.lengthOf(1) - } - - { - const { data } = await commands[0].getFollowings({ state: 'rejected' }) - expect(data).to.have.lengthOf(1) - expectStartWith(data[0].following.url, servers[2].url) - } - } - - { // server 3 - { - const { data } = await commands[2].getFollowers({ state: 'accepted' }) - expect(data).to.have.lengthOf(0) - } - - { - const { data } = await commands[2].getFollowers({ state: 'rejected' }) - expect(data).to.have.lengthOf(1) - expectStartWith(data[0].follower.url, servers[0].url) - } - } - }) - - it('Should still auto accept channel followers', async function () { - await commands[0].follow({ handles: [ 'root_channel@' + servers[1].host ] }) - - await waitJobs(servers) - - const body = await commands[0].getFollowings() - const follow = body.data[0] - expect(follow.following.name).to.equal('root_channel') - expect(follow.state).to.equal('accepted') - }) - }) - - describe('Accept/reject state', function () { - - it('Should not change the follow on refollow with and without auto accept', async function () { - const run = async () => { - await commands[0].follow({ hosts: [ servers[2].url ] }) - await waitJobs(servers) - - await checkFollows({ - follower: servers[0], - followerState: 'rejected', - following: servers[2], - followingState: 'rejected' - }) - } - - await servers[2].config.updateExistingSubConfig({ newConfig: { followers: { instance: { manualApproval: false } } } }) - await run() - - await servers[2].config.updateExistingSubConfig({ newConfig: { followers: { instance: { manualApproval: true } } } }) - await run() - }) - - it('Should not change the rejected status on unfollow', async function () { - await commands[0].unfollow({ target: servers[2] }) - await waitJobs(servers) - - await checkFollows({ - follower: servers[0], - followerState: 'deleted', - following: servers[2], - followingState: 'rejected' - }) - }) - - it('Should delete the follower and add again the follower', async function () { - await commands[2].removeFollower({ follower: servers[0] }) - await waitJobs(servers) - - await commands[0].follow({ hosts: [ servers[2].url ] }) - await waitJobs(servers) - - await checkFollows({ - follower: servers[0], - followerState: 'pending', - following: servers[2], - followingState: 'pending' - }) - }) - - it('Should be able to reject a previously accepted follower', async function () { - await commands[1].rejectFollower({ follower: 'peertube@' + servers[0].host }) - await waitJobs(servers) - - await checkFollows({ - follower: servers[0], - followerState: 'rejected', - following: servers[1], - followingState: 'rejected' - }) - }) - - it('Should be able to re accept a previously rejected follower', async function () { - await commands[1].acceptFollower({ follower: 'peertube@' + servers[0].host }) - await waitJobs(servers) - - await checkFollows({ - follower: servers[0], - followerState: 'accepted', - following: servers[1], - followingState: 'accepted' - }) - }) - }) - - describe('Muted servers', function () { - - it('Should ignore follow requests of muted servers', async function () { - await servers[1].blocklist.addToServerBlocklist({ server: servers[0].host }) - - await commands[0].unfollow({ target: servers[1] }) - - await waitJobs(servers) - - await checkFollows({ - follower: servers[0], - followerState: 'deleted', - following: servers[1], - followingState: 'deleted' - }) - - await commands[0].follow({ hosts: [ servers[1].host ] }) - await waitJobs(servers) - - await checkFollows({ - follower: servers[0], - followerState: 'rejected', - following: servers[1], - followingState: 'deleted' - }) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/server/follows.ts b/server/tests/api/server/follows.ts deleted file mode 100644 index e3e4605ee..000000000 --- a/server/tests/api/server/follows.ts +++ /dev/null @@ -1,641 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { completeVideoCheck, dateIsValid, expectAccountFollows, expectChannelsFollows, testCaptionFile } from '@server/tests/shared' -import { Video, VideoPrivacy } from '@shared/models' -import { cleanupTests, createMultipleServers, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/server-commands' - -describe('Test follows', function () { - - describe('Complex follow', function () { - let servers: PeerTubeServer[] = [] - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(3) - - // Get the access tokens - await setAccessTokensToServers(servers) - }) - - describe('Data propagation after follow', function () { - - it('Should not have followers/followings', async function () { - for (const server of servers) { - const bodies = await Promise.all([ - server.follows.getFollowers({ start: 0, count: 5, sort: 'createdAt' }), - server.follows.getFollowings({ start: 0, count: 5, sort: 'createdAt' }) - ]) - - for (const body of bodies) { - expect(body.total).to.equal(0) - - const follows = body.data - expect(follows).to.be.an('array') - expect(follows).to.have.lengthOf(0) - } - } - }) - - it('Should have server 1 following root account of server 2 and server 3', async function () { - this.timeout(30000) - - await servers[0].follows.follow({ - hosts: [ servers[2].url ], - handles: [ 'root@' + servers[1].host ] - }) - - await waitJobs(servers) - }) - - it('Should have 2 followings on server 1', async function () { - const body = await servers[0].follows.getFollowings({ start: 0, count: 1, sort: 'createdAt' }) - expect(body.total).to.equal(2) - - let follows = body.data - expect(follows).to.be.an('array') - expect(follows).to.have.lengthOf(1) - - const body2 = await servers[0].follows.getFollowings({ start: 1, count: 1, sort: 'createdAt' }) - follows = follows.concat(body2.data) - - const server2Follow = follows.find(f => f.following.host === servers[1].host) - const server3Follow = follows.find(f => f.following.host === servers[2].host) - - expect(server2Follow).to.not.be.undefined - expect(server2Follow.following.name).to.equal('root') - expect(server2Follow.state).to.equal('accepted') - - expect(server3Follow).to.not.be.undefined - expect(server3Follow.following.name).to.equal('peertube') - expect(server3Follow.state).to.equal('accepted') - }) - - it('Should have 0 followings on server 2 and 3', async function () { - for (const server of [ servers[1], servers[2] ]) { - const body = await server.follows.getFollowings({ start: 0, count: 5, sort: 'createdAt' }) - expect(body.total).to.equal(0) - - const follows = body.data - expect(follows).to.be.an('array') - expect(follows).to.have.lengthOf(0) - } - }) - - it('Should have 1 followers on server 3', async function () { - const body = await servers[2].follows.getFollowers({ start: 0, count: 1, sort: 'createdAt' }) - expect(body.total).to.equal(1) - - const follows = body.data - expect(follows).to.be.an('array') - expect(follows).to.have.lengthOf(1) - expect(follows[0].follower.host).to.equal(servers[0].host) - }) - - it('Should have 0 followers on server 1 and 2', async function () { - for (const server of [ servers[0], servers[1] ]) { - const body = await server.follows.getFollowers({ start: 0, count: 5, sort: 'createdAt' }) - expect(body.total).to.equal(0) - - const follows = body.data - expect(follows).to.be.an('array') - expect(follows).to.have.lengthOf(0) - } - }) - - it('Should search/filter followings on server 1', async function () { - const sort = 'createdAt' - const start = 0 - const count = 1 - - { - const search = ':' + servers[1].port - - { - const body = await servers[0].follows.getFollowings({ start, count, sort, search }) - expect(body.total).to.equal(1) - - const follows = body.data - expect(follows).to.have.lengthOf(1) - expect(follows[0].following.host).to.equal(servers[1].host) - } - - { - const body = await servers[0].follows.getFollowings({ start, count, sort, search, state: 'accepted' }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - } - - { - const body = await servers[0].follows.getFollowings({ start, count, sort, search, state: 'accepted', actorType: 'Person' }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - } - - { - const body = await servers[0].follows.getFollowings({ - start, - count, - sort, - search, - state: 'accepted', - actorType: 'Application' - }) - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - } - - { - const body = await servers[0].follows.getFollowings({ start, count, sort, search, state: 'pending' }) - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - } - } - - { - const body = await servers[0].follows.getFollowings({ start, count, sort, search: 'root' }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - } - - { - const body = await servers[0].follows.getFollowings({ start, count, sort, search: 'bla' }) - expect(body.total).to.equal(0) - - expect(body.data).to.have.lengthOf(0) - } - }) - - it('Should search/filter followers on server 2', async function () { - const start = 0 - const count = 5 - const sort = 'createdAt' - - { - const search = servers[0].port + '' - - { - const body = await servers[2].follows.getFollowers({ start, count, sort, search }) - expect(body.total).to.equal(1) - - const follows = body.data - expect(follows).to.have.lengthOf(1) - expect(follows[0].following.host).to.equal(servers[2].host) - } - - { - const body = await servers[2].follows.getFollowers({ start, count, sort, search, state: 'accepted' }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - } - - { - const body = await servers[2].follows.getFollowers({ start, count, sort, search, state: 'accepted', actorType: 'Person' }) - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - } - - { - const body = await servers[2].follows.getFollowers({ - start, - count, - sort, - search, - state: 'accepted', - actorType: 'Application' - }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - } - - { - const body = await servers[2].follows.getFollowers({ start, count, sort, search, state: 'pending' }) - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - } - } - - { - const body = await servers[2].follows.getFollowers({ start, count, sort, search: 'bla' }) - expect(body.total).to.equal(0) - - const follows = body.data - expect(follows).to.have.lengthOf(0) - } - }) - - it('Should have the correct follows counts', async function () { - await expectAccountFollows({ server: servers[0], handle: 'peertube@' + servers[0].host, followers: 0, following: 2 }) - await expectAccountFollows({ server: servers[0], handle: 'root@' + servers[1].host, followers: 1, following: 0 }) - await expectAccountFollows({ server: servers[0], handle: 'peertube@' + servers[2].host, followers: 1, following: 0 }) - - // Server 2 and 3 does not know server 1 follow another server (there was not a refresh) - await expectAccountFollows({ server: servers[1], handle: 'peertube@' + servers[0].host, followers: 0, following: 1 }) - await expectAccountFollows({ server: servers[1], handle: 'root@' + servers[1].host, followers: 1, following: 0 }) - await expectAccountFollows({ server: servers[1], handle: 'peertube@' + servers[1].host, followers: 0, following: 0 }) - - await expectAccountFollows({ server: servers[2], handle: 'peertube@' + servers[0].host, followers: 0, following: 1 }) - await expectAccountFollows({ server: servers[2], handle: 'peertube@' + servers[2].host, followers: 1, following: 0 }) - }) - - it('Should unfollow server 3 on server 1', async function () { - this.timeout(15000) - - await servers[0].follows.unfollow({ target: servers[2] }) - - await waitJobs(servers) - }) - - it('Should not follow server 3 on server 1 anymore', async function () { - const body = await servers[0].follows.getFollowings({ start: 0, count: 2, sort: 'createdAt' }) - expect(body.total).to.equal(1) - - const follows = body.data - expect(follows).to.be.an('array') - expect(follows).to.have.lengthOf(1) - - expect(follows[0].following.host).to.equal(servers[1].host) - }) - - it('Should not have server 1 as follower on server 3 anymore', async function () { - const body = await servers[2].follows.getFollowers({ start: 0, count: 1, sort: 'createdAt' }) - expect(body.total).to.equal(0) - - const follows = body.data - expect(follows).to.be.an('array') - expect(follows).to.have.lengthOf(0) - }) - - it('Should have the correct follows counts after the unfollow', async function () { - await expectAccountFollows({ server: servers[0], handle: 'peertube@' + servers[0].host, followers: 0, following: 1 }) - await expectAccountFollows({ server: servers[0], handle: 'root@' + servers[1].host, followers: 1, following: 0 }) - await expectAccountFollows({ server: servers[0], handle: 'peertube@' + servers[2].host, followers: 0, following: 0 }) - - await expectAccountFollows({ server: servers[1], handle: 'peertube@' + servers[0].host, followers: 0, following: 1 }) - await expectAccountFollows({ server: servers[1], handle: 'root@' + servers[1].host, followers: 1, following: 0 }) - await expectAccountFollows({ server: servers[1], handle: 'peertube@' + servers[1].host, followers: 0, following: 0 }) - - await expectAccountFollows({ server: servers[2], handle: 'peertube@' + servers[0].host, followers: 0, following: 0 }) - await expectAccountFollows({ server: servers[2], handle: 'peertube@' + servers[2].host, followers: 0, following: 0 }) - }) - - it('Should upload a video on server 2 and 3 and propagate only the video of server 2', async function () { - this.timeout(160000) - - await servers[1].videos.upload({ attributes: { name: 'server2' } }) - await servers[2].videos.upload({ attributes: { name: 'server3' } }) - - await waitJobs(servers) - - { - const { total, data } = await servers[0].videos.list() - expect(total).to.equal(1) - expect(data[0].name).to.equal('server2') - } - - { - const { total, data } = await servers[1].videos.list() - expect(total).to.equal(1) - expect(data[0].name).to.equal('server2') - } - - { - const { total, data } = await servers[2].videos.list() - expect(total).to.equal(1) - expect(data[0].name).to.equal('server3') - } - }) - - it('Should remove account follow', async function () { - this.timeout(15000) - - await servers[0].follows.unfollow({ target: 'root@' + servers[1].host }) - - await waitJobs(servers) - }) - - it('Should have removed the account follow', async function () { - await expectAccountFollows({ server: servers[0], handle: 'root@' + servers[1].host, followers: 0, following: 0 }) - await expectAccountFollows({ server: servers[1], handle: 'root@' + servers[1].host, followers: 0, following: 0 }) - - { - const { total, data } = await servers[0].follows.getFollowings() - expect(total).to.equal(0) - expect(data).to.have.lengthOf(0) - } - - { - const { total, data } = await servers[0].videos.list() - expect(total).to.equal(0) - expect(data).to.have.lengthOf(0) - } - }) - - it('Should follow a channel', async function () { - this.timeout(15000) - - await servers[0].follows.follow({ - handles: [ 'root_channel@' + servers[1].host ] - }) - - await waitJobs(servers) - - await expectChannelsFollows({ server: servers[0], handle: 'root_channel@' + servers[1].host, followers: 1, following: 0 }) - await expectChannelsFollows({ server: servers[1], handle: 'root_channel@' + servers[1].host, followers: 1, following: 0 }) - - { - const { total, data } = await servers[0].follows.getFollowings() - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - } - - { - const { total, data } = await servers[0].videos.list() - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - } - }) - }) - - describe('Should propagate data on a new server follow', function () { - let video4: Video - - before(async function () { - this.timeout(240000) - - const video4Attributes = { - name: 'server3-4', - category: 2, - nsfw: true, - licence: 6, - tags: [ 'tag1', 'tag2', 'tag3' ] - } - - await servers[2].videos.upload({ attributes: { name: 'server3-2' } }) - await servers[2].videos.upload({ attributes: { name: 'server3-3' } }) - - const video4CreateResult = await servers[2].videos.upload({ attributes: video4Attributes }) - - await servers[2].videos.upload({ attributes: { name: 'server3-5' } }) - await servers[2].videos.upload({ attributes: { name: 'server3-6' } }) - - { - const userAccessToken = await servers[2].users.generateUserAndToken('captain') - - await servers[2].videos.rate({ id: video4CreateResult.id, rating: 'like' }) - await servers[2].videos.rate({ token: userAccessToken, id: video4CreateResult.id, rating: 'dislike' }) - } - - { - await servers[2].comments.createThread({ videoId: video4CreateResult.id, text: 'my super first comment' }) - - await servers[2].comments.addReplyToLastThread({ text: 'my super answer to thread 1' }) - await servers[2].comments.addReplyToLastReply({ text: 'my super answer to answer of thread 1' }) - await servers[2].comments.addReplyToLastThread({ text: 'my second answer to thread 1' }) - } - - { - const { id: threadId } = await servers[2].comments.createThread({ videoId: video4CreateResult.id, text: 'will be deleted' }) - await servers[2].comments.addReplyToLastThread({ text: 'answer to deleted' }) - - const { id: replyId } = await servers[2].comments.addReplyToLastThread({ text: 'will also be deleted' }) - - await servers[2].comments.addReplyToLastReply({ text: 'my second answer to deleted' }) - - await servers[2].comments.delete({ videoId: video4CreateResult.id, commentId: threadId }) - await servers[2].comments.delete({ videoId: video4CreateResult.id, commentId: replyId }) - } - - await servers[2].captions.add({ - language: 'ar', - videoId: video4CreateResult.id, - fixture: 'subtitle-good2.vtt' - }) - - await waitJobs(servers) - - // Server 1 follows server 3 - await servers[0].follows.follow({ hosts: [ servers[2].url ] }) - - await waitJobs(servers) - }) - - it('Should have the correct follows counts', async function () { - await expectAccountFollows({ server: servers[0], handle: 'peertube@' + servers[0].host, followers: 0, following: 2 }) - await expectAccountFollows({ server: servers[0], handle: 'root@' + servers[1].host, followers: 0, following: 0 }) - await expectChannelsFollows({ server: servers[0], handle: 'root_channel@' + servers[1].host, followers: 1, following: 0 }) - await expectAccountFollows({ server: servers[0], handle: 'peertube@' + servers[2].host, followers: 1, following: 0 }) - - await expectAccountFollows({ server: servers[1], handle: 'peertube@' + servers[0].host, followers: 0, following: 1 }) - await expectAccountFollows({ server: servers[1], handle: 'peertube@' + servers[1].host, followers: 0, following: 0 }) - await expectAccountFollows({ server: servers[1], handle: 'root@' + servers[1].host, followers: 0, following: 0 }) - await expectChannelsFollows({ server: servers[1], handle: 'root_channel@' + servers[1].host, followers: 1, following: 0 }) - - await expectAccountFollows({ server: servers[2], handle: 'peertube@' + servers[0].host, followers: 0, following: 1 }) - await expectAccountFollows({ server: servers[2], handle: 'peertube@' + servers[2].host, followers: 1, following: 0 }) - }) - - it('Should have propagated videos', async function () { - const { total, data } = await servers[0].videos.list() - expect(total).to.equal(7) - - const video2 = data.find(v => v.name === 'server3-2') - video4 = data.find(v => v.name === 'server3-4') - const video6 = data.find(v => v.name === 'server3-6') - - expect(video2).to.not.be.undefined - expect(video4).to.not.be.undefined - expect(video6).to.not.be.undefined - - const isLocal = false - const checkAttributes = { - name: 'server3-4', - category: 2, - licence: 6, - language: 'zh', - nsfw: true, - description: 'my super description', - support: 'my super support text', - account: { - name: 'root', - host: servers[2].host - }, - isLocal, - commentsEnabled: true, - downloadEnabled: true, - duration: 5, - tags: [ 'tag1', 'tag2', 'tag3' ], - privacy: VideoPrivacy.PUBLIC, - likes: 1, - dislikes: 1, - channel: { - displayName: 'Main root channel', - name: 'root_channel', - description: '', - isLocal - }, - fixture: 'video_short.webm', - files: [ - { - resolution: 720, - size: 218910 - } - ] - } - await completeVideoCheck({ - server: servers[0], - originServer: servers[2], - videoUUID: video4.uuid, - attributes: checkAttributes - }) - }) - - it('Should have propagated comments', async function () { - const { total, data } = await servers[0].comments.listThreads({ videoId: video4.id, sort: 'createdAt' }) - - expect(total).to.equal(2) - expect(data).to.be.an('array') - expect(data).to.have.lengthOf(2) - - { - const comment = data[0] - expect(comment.inReplyToCommentId).to.be.null - expect(comment.text).equal('my super first comment') - expect(comment.videoId).to.equal(video4.id) - expect(comment.id).to.equal(comment.threadId) - expect(comment.account.name).to.equal('root') - expect(comment.account.host).to.equal(servers[2].host) - expect(comment.totalReplies).to.equal(3) - expect(dateIsValid(comment.createdAt as string)).to.be.true - expect(dateIsValid(comment.updatedAt as string)).to.be.true - - const threadId = comment.threadId - - const tree = await servers[0].comments.getThread({ videoId: video4.id, threadId }) - expect(tree.comment.text).equal('my super first comment') - expect(tree.children).to.have.lengthOf(2) - - const firstChild = tree.children[0] - expect(firstChild.comment.text).to.equal('my super answer to thread 1') - expect(firstChild.children).to.have.lengthOf(1) - - const childOfFirstChild = firstChild.children[0] - expect(childOfFirstChild.comment.text).to.equal('my super answer to answer of thread 1') - expect(childOfFirstChild.children).to.have.lengthOf(0) - - const secondChild = tree.children[1] - expect(secondChild.comment.text).to.equal('my second answer to thread 1') - expect(secondChild.children).to.have.lengthOf(0) - } - - { - const deletedComment = data[1] - expect(deletedComment).to.not.be.undefined - expect(deletedComment.isDeleted).to.be.true - expect(deletedComment.deletedAt).to.not.be.null - expect(deletedComment.text).to.equal('') - expect(deletedComment.inReplyToCommentId).to.be.null - expect(deletedComment.account).to.be.null - expect(deletedComment.totalReplies).to.equal(2) - expect(dateIsValid(deletedComment.deletedAt as string)).to.be.true - - const tree = await servers[0].comments.getThread({ videoId: video4.id, threadId: deletedComment.threadId }) - const [ commentRoot, deletedChildRoot ] = tree.children - - expect(deletedChildRoot).to.not.be.undefined - expect(deletedChildRoot.comment.isDeleted).to.be.true - expect(deletedChildRoot.comment.deletedAt).to.not.be.null - expect(deletedChildRoot.comment.text).to.equal('') - expect(deletedChildRoot.comment.inReplyToCommentId).to.equal(deletedComment.id) - expect(deletedChildRoot.comment.account).to.be.null - expect(deletedChildRoot.children).to.have.lengthOf(1) - - const answerToDeletedChild = deletedChildRoot.children[0] - expect(answerToDeletedChild.comment).to.not.be.undefined - expect(answerToDeletedChild.comment.inReplyToCommentId).to.equal(deletedChildRoot.comment.id) - expect(answerToDeletedChild.comment.text).to.equal('my second answer to deleted') - expect(answerToDeletedChild.comment.account.name).to.equal('root') - - expect(commentRoot.comment).to.not.be.undefined - expect(commentRoot.comment.inReplyToCommentId).to.equal(deletedComment.id) - expect(commentRoot.comment.text).to.equal('answer to deleted') - expect(commentRoot.comment.account.name).to.equal('root') - } - }) - - it('Should have propagated captions', async function () { - const body = await servers[0].captions.list({ videoId: video4.id }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - - const caption1 = body.data[0] - expect(caption1.language.id).to.equal('ar') - expect(caption1.language.label).to.equal('Arabic') - expect(caption1.captionPath).to.match(new RegExp('^/lazy-static/video-captions/.+-ar.vtt$')) - await testCaptionFile(servers[0].url, caption1.captionPath, 'Subtitle good 2.') - }) - - it('Should unfollow server 3 on server 1 and does not list server 3 videos', async function () { - this.timeout(5000) - - await servers[0].follows.unfollow({ target: servers[2] }) - - await waitJobs(servers) - - const { total } = await servers[0].videos.list() - expect(total).to.equal(1) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) - }) - - describe('Simple data propagation propagate data on a new channel follow', function () { - let servers: PeerTubeServer[] = [] - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(3) - await setAccessTokensToServers(servers) - - await servers[0].videos.upload({ attributes: { name: 'video to add' } }) - - await waitJobs(servers) - - for (const server of [ servers[1], servers[2] ]) { - const video = await server.videos.find({ name: 'video to add' }) - expect(video).to.not.exist - } - }) - - it('Should have propagated video after new channel follow', async function () { - this.timeout(60000) - - await servers[1].follows.follow({ handles: [ 'root_channel@' + servers[0].host ] }) - - await waitJobs(servers) - - const video = await servers[1].videos.find({ name: 'video to add' }) - expect(video).to.exist - }) - - it('Should have propagated video after new account follow', async function () { - this.timeout(60000) - - await servers[2].follows.follow({ handles: [ 'root@' + servers[0].host ] }) - - await waitJobs(servers) - - const video = await servers[2].videos.find({ name: 'video to add' }) - expect(video).to.exist - }) - - after(async function () { - await cleanupTests(servers) - }) - }) -}) diff --git a/server/tests/api/server/handle-down.ts b/server/tests/api/server/handle-down.ts deleted file mode 100644 index 1d524aa93..000000000 --- a/server/tests/api/server/handle-down.ts +++ /dev/null @@ -1,338 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { completeVideoCheck, SQLCommand } from '@server/tests/shared' -import { wait } from '@shared/core-utils' -import { HttpStatusCode, JobState, VideoCreateResult, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - CommentsCommand, - createMultipleServers, - killallServers, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test handle downs', function () { - let servers: PeerTubeServer[] = [] - let sqlCommands: SQLCommand[] = [] - - let threadIdServer1: number - let threadIdServer2: number - let commentIdServer1: number - let commentIdServer2: number - let missedVideo1: VideoCreateResult - let missedVideo2: VideoCreateResult - let unlistedVideo: VideoCreateResult - - const videoIdsServer1: string[] = [] - - const videoAttributes = { - name: 'my super name for server 1', - category: 5, - licence: 4, - language: 'ja', - nsfw: true, - privacy: VideoPrivacy.PUBLIC, - description: 'my super description for server 1', - support: 'my super support text for server 1', - tags: [ 'tag1p1', 'tag2p1' ], - fixture: 'video_short1.webm' - } - - const unlistedVideoAttributes = { ...videoAttributes, privacy: VideoPrivacy.UNLISTED } - - let checkAttributes: any - let unlistedCheckAttributes: any - - let commentCommands: CommentsCommand[] - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(3) - commentCommands = servers.map(s => s.comments) - - checkAttributes = { - name: 'my super name for server 1', - category: 5, - licence: 4, - language: 'ja', - nsfw: true, - description: 'my super description for server 1', - support: 'my super support text for server 1', - account: { - name: 'root', - host: servers[0].host - }, - isLocal: false, - duration: 10, - tags: [ 'tag1p1', 'tag2p1' ], - privacy: VideoPrivacy.PUBLIC, - commentsEnabled: true, - downloadEnabled: true, - channel: { - name: 'root_channel', - displayName: 'Main root channel', - description: '', - isLocal: false - }, - fixture: 'video_short1.webm', - files: [ - { - resolution: 720, - size: 572456 - } - ] - } - unlistedCheckAttributes = { ...checkAttributes, privacy: VideoPrivacy.UNLISTED } - - // Get the access tokens - await setAccessTokensToServers(servers) - - sqlCommands = servers.map(s => new SQLCommand(s)) - }) - - it('Should remove followers that are often down', async function () { - this.timeout(240000) - - // Server 2 and 3 follow server 1 - await servers[1].follows.follow({ hosts: [ servers[0].url ] }) - await servers[2].follows.follow({ hosts: [ servers[0].url ] }) - - await waitJobs(servers) - - // Upload a video to server 1 - await servers[0].videos.upload({ attributes: videoAttributes }) - - await waitJobs(servers) - - // And check all servers have this video - for (const server of servers) { - const { data } = await server.videos.list() - expect(data).to.be.an('array') - expect(data).to.have.lengthOf(1) - } - - // Kill server 2 - await killallServers([ servers[1] ]) - - // Remove server 2 follower - for (let i = 0; i < 10; i++) { - await servers[0].videos.upload({ attributes: videoAttributes }) - } - - await waitJobs([ servers[0], servers[2] ]) - - // Kill server 3 - await killallServers([ servers[2] ]) - - missedVideo1 = await servers[0].videos.upload({ attributes: videoAttributes }) - - missedVideo2 = await servers[0].videos.upload({ attributes: videoAttributes }) - - // Unlisted video - unlistedVideo = await servers[0].videos.upload({ attributes: unlistedVideoAttributes }) - - // Add comments to video 2 - { - const text = 'thread 1' - let comment = await commentCommands[0].createThread({ videoId: missedVideo2.uuid, text }) - threadIdServer1 = comment.id - - comment = await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: comment.id, text: 'comment 1-1' }) - - const created = await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: comment.id, text: 'comment 1-2' }) - commentIdServer1 = created.id - } - - await waitJobs(servers[0]) - // Wait scheduler - await wait(11000) - - // Only server 3 is still a follower of server 1 - const body = await servers[0].follows.getFollowers({ start: 0, count: 2, sort: 'createdAt' }) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].follower.host).to.equal(servers[2].host) - }) - - it('Should not have pending/processing jobs anymore', async function () { - const states: JobState[] = [ 'waiting', 'active' ] - - for (const state of states) { - const body = await servers[0].jobs.list({ - state, - start: 0, - count: 50, - sort: '-createdAt' - }) - expect(body.data).to.have.length(0) - } - }) - - it('Should re-follow server 1', async function () { - this.timeout(70000) - - await servers[1].run() - await servers[2].run() - - await servers[1].follows.unfollow({ target: servers[0] }) - await waitJobs(servers) - - await servers[1].follows.follow({ hosts: [ servers[0].url ] }) - - await waitJobs(servers) - - const body = await servers[0].follows.getFollowers({ start: 0, count: 2, sort: 'createdAt' }) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(2) - }) - - it('Should send an update to server 3, and automatically fetch the video', async function () { - this.timeout(15000) - - { - const { data } = await servers[2].videos.list() - expect(data).to.be.an('array') - expect(data).to.have.lengthOf(11) - } - - await servers[0].videos.update({ id: missedVideo1.uuid }) - await servers[0].videos.update({ id: unlistedVideo.uuid }) - - await waitJobs(servers) - - { - const { data } = await servers[2].videos.list() - expect(data).to.be.an('array') - // 1 video is unlisted - expect(data).to.have.lengthOf(12) - } - - // Check unlisted video - const video = await servers[2].videos.get({ id: unlistedVideo.uuid }) - await completeVideoCheck({ server: servers[2], originServer: servers[0], videoUUID: video.uuid, attributes: unlistedCheckAttributes }) - }) - - it('Should send comments on a video to server 3, and automatically fetch the video', async function () { - this.timeout(25000) - - await commentCommands[0].addReply({ videoId: missedVideo2.uuid, toCommentId: commentIdServer1, text: 'comment 1-3' }) - - await waitJobs(servers) - - await servers[2].videos.get({ id: missedVideo2.uuid }) - - { - const { data } = await servers[2].comments.listThreads({ videoId: missedVideo2.uuid }) - expect(data).to.be.an('array') - expect(data).to.have.lengthOf(1) - - threadIdServer2 = data[0].id - - const tree = await servers[2].comments.getThread({ videoId: missedVideo2.uuid, threadId: threadIdServer2 }) - expect(tree.comment.text).equal('thread 1') - expect(tree.children).to.have.lengthOf(1) - - const firstChild = tree.children[0] - expect(firstChild.comment.text).to.equal('comment 1-1') - expect(firstChild.children).to.have.lengthOf(1) - - const childOfFirstChild = firstChild.children[0] - expect(childOfFirstChild.comment.text).to.equal('comment 1-2') - expect(childOfFirstChild.children).to.have.lengthOf(1) - - const childOfChildFirstChild = childOfFirstChild.children[0] - expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3') - expect(childOfChildFirstChild.children).to.have.lengthOf(0) - - commentIdServer2 = childOfChildFirstChild.comment.id - } - }) - - it('Should correctly reply to the comment', async function () { - this.timeout(15000) - - await servers[2].comments.addReply({ videoId: missedVideo2.uuid, toCommentId: commentIdServer2, text: 'comment 1-4' }) - - await waitJobs(servers) - - const tree = await commentCommands[0].getThread({ videoId: missedVideo2.uuid, threadId: threadIdServer1 }) - - expect(tree.comment.text).equal('thread 1') - expect(tree.children).to.have.lengthOf(1) - - const firstChild = tree.children[0] - expect(firstChild.comment.text).to.equal('comment 1-1') - expect(firstChild.children).to.have.lengthOf(1) - - const childOfFirstChild = firstChild.children[0] - expect(childOfFirstChild.comment.text).to.equal('comment 1-2') - expect(childOfFirstChild.children).to.have.lengthOf(1) - - const childOfChildFirstChild = childOfFirstChild.children[0] - expect(childOfChildFirstChild.comment.text).to.equal('comment 1-3') - expect(childOfChildFirstChild.children).to.have.lengthOf(1) - - const childOfChildOfChildOfFirstChild = childOfChildFirstChild.children[0] - expect(childOfChildOfChildOfFirstChild.comment.text).to.equal('comment 1-4') - expect(childOfChildOfChildOfFirstChild.children).to.have.lengthOf(0) - }) - - it('Should upload many videos on server 1', async function () { - this.timeout(240000) - - for (let i = 0; i < 10; i++) { - const uuid = (await servers[0].videos.quickUpload({ name: 'video ' + i })).uuid - videoIdsServer1.push(uuid) - } - - await waitJobs(servers) - - for (const id of videoIdsServer1) { - await servers[1].videos.get({ id }) - } - - await waitJobs(servers) - await sqlCommands[1].setActorFollowScores(20) - - // Wait video expiration - await wait(11000) - - // Refresh video -> score + 10 = 30 - await servers[1].videos.get({ id: videoIdsServer1[0] }) - - await waitJobs(servers) - }) - - it('Should remove followings that are down', async function () { - this.timeout(120000) - - await killallServers([ servers[0] ]) - - // Wait video expiration - await wait(11000) - - for (let i = 0; i < 5; i++) { - try { - await servers[1].videos.get({ id: videoIdsServer1[i] }) - await waitJobs([ servers[1] ]) - await wait(1500) - } catch {} - } - - for (const id of videoIdsServer1) { - await servers[1].videos.get({ id, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - } - }) - - after(async function () { - for (const sqlCommand of sqlCommands) { - await sqlCommand.cleanup() - } - - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/server/homepage.ts b/server/tests/api/server/homepage.ts deleted file mode 100644 index 9c45800f2..000000000 --- a/server/tests/api/server/homepage.ts +++ /dev/null @@ -1,81 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { HttpStatusCode } from '@shared/models' -import { - cleanupTests, - createSingleServer, - CustomPagesCommand, - killallServers, - PeerTubeServer, - setAccessTokensToServers, - setDefaultAccountAvatar, - setDefaultChannelAvatar -} from '../../../../shared/server-commands/index' - -async function getHomepageState (server: PeerTubeServer) { - const config = await server.config.getConfig() - - return config.homepage.enabled -} - -describe('Test instance homepage actions', function () { - let server: PeerTubeServer - let command: CustomPagesCommand - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - await setDefaultChannelAvatar(server) - await setDefaultAccountAvatar(server) - - command = server.customPage - }) - - it('Should not have a homepage', async function () { - const state = await getHomepageState(server) - expect(state).to.be.false - - await command.getInstanceHomepage({ expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should set a homepage', async function () { - await command.updateInstanceHomepage({ content: '' }) - - const page = await command.getInstanceHomepage() - expect(page.content).to.equal('') - - const state = await getHomepageState(server) - expect(state).to.be.true - }) - - it('Should have the same homepage after a restart', async function () { - this.timeout(30000) - - await killallServers([ server ]) - - await server.run() - - const page = await command.getInstanceHomepage() - expect(page.content).to.equal('') - - const state = await getHomepageState(server) - expect(state).to.be.true - }) - - it('Should empty the homepage', async function () { - await command.updateInstanceHomepage({ content: '' }) - - const page = await command.getInstanceHomepage() - expect(page.content).to.be.empty - - const state = await getHomepageState(server) - expect(state).to.be.false - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/server/index.ts b/server/tests/api/server/index.ts deleted file mode 100644 index 78522c246..000000000 --- a/server/tests/api/server/index.ts +++ /dev/null @@ -1,22 +0,0 @@ -import './auto-follows' -import './bulk' -import './config-defaults' -import './config' -import './contact-form' -import './email' -import './follow-constraints' -import './follows' -import './follows-moderation' -import './homepage' -import './handle-down' -import './jobs' -import './logs' -import './reverse-proxy' -import './services' -import './slow-follows' -import './stats' -import './tracker' -import './no-client' -import './open-telemetry' -import './plugins' -import './proxy' diff --git a/server/tests/api/server/jobs.ts b/server/tests/api/server/jobs.ts deleted file mode 100644 index d0e6df719..000000000 --- a/server/tests/api/server/jobs.ts +++ /dev/null @@ -1,128 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { dateIsValid } from '@server/tests/shared' -import { wait } from '@shared/core-utils' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test jobs', function () { - let servers: PeerTubeServer[] - - before(async function () { - this.timeout(240000) - - servers = await createMultipleServers(2) - - await setAccessTokensToServers(servers) - - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - }) - - it('Should create some jobs', async function () { - this.timeout(240000) - - await servers[1].videos.upload({ attributes: { name: 'video1' } }) - await servers[1].videos.upload({ attributes: { name: 'video2' } }) - - await waitJobs(servers) - }) - - it('Should list jobs', async function () { - const body = await servers[1].jobs.list({ state: 'completed' }) - expect(body.total).to.be.above(2) - expect(body.data).to.have.length.above(2) - }) - - it('Should list jobs with sort, pagination and job type', async function () { - { - const body = await servers[1].jobs.list({ - state: 'completed', - start: 1, - count: 2, - sort: 'createdAt' - }) - expect(body.total).to.be.above(2) - expect(body.data).to.have.lengthOf(2) - - let job = body.data[0] - // Skip repeat jobs - if (job.type === 'videos-views-stats') job = body.data[1] - - expect(job.state).to.equal('completed') - expect(dateIsValid(job.createdAt as string)).to.be.true - expect(dateIsValid(job.processedOn as string)).to.be.true - expect(dateIsValid(job.finishedOn as string)).to.be.true - } - - { - const body = await servers[1].jobs.list({ - state: 'completed', - start: 0, - count: 100, - sort: 'createdAt', - jobType: 'activitypub-http-broadcast' - }) - expect(body.total).to.be.above(2) - - for (const j of body.data) { - expect(j.type).to.equal('activitypub-http-broadcast') - } - } - }) - - it('Should list all jobs', async function () { - const body = await servers[1].jobs.list() - expect(body.total).to.be.above(2) - - const jobs = body.data - expect(jobs).to.have.length.above(2) - - expect(jobs.find(j => j.state === 'completed')).to.not.be.undefined - }) - - it('Should pause the job queue', async function () { - this.timeout(120000) - - const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video2' } }) - await waitJobs(servers) - - await servers[1].jobs.pauseJobQueue() - await servers[1].videos.runTranscoding({ videoId: uuid, transcodingType: 'hls' }) - - await wait(5000) - - { - const body = await servers[1].jobs.list({ state: 'waiting', jobType: 'video-transcoding' }) - // waiting includes waiting-children - expect(body.data).to.have.lengthOf(4) - } - - { - const body = await servers[1].jobs.list({ state: 'waiting-children', jobType: 'video-transcoding' }) - expect(body.data).to.have.lengthOf(1) - } - }) - - it('Should resume the job queue', async function () { - this.timeout(120000) - - await servers[1].jobs.resumeJobQueue() - - await waitJobs(servers) - - const body = await servers[1].jobs.list({ state: 'waiting', jobType: 'video-transcoding' }) - expect(body.data).to.have.lengthOf(0) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/server/logs.ts b/server/tests/api/server/logs.ts deleted file mode 100644 index 9cf04c501..000000000 --- a/server/tests/api/server/logs.ts +++ /dev/null @@ -1,265 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { HttpStatusCode } from '@shared/models' -import { - cleanupTests, - createSingleServer, - killallServers, - LogsCommand, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test logs', function () { - let server: PeerTubeServer - let logsCommand: LogsCommand - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - - logsCommand = server.logs - }) - - describe('With the standard log file', function () { - - it('Should get logs with a start date', async function () { - this.timeout(60000) - - await server.videos.upload({ attributes: { name: 'video 1' } }) - await waitJobs([ server ]) - - const now = new Date() - - await server.videos.upload({ attributes: { name: 'video 2' } }) - await waitJobs([ server ]) - - const body = await logsCommand.getLogs({ startDate: now }) - const logsString = JSON.stringify(body) - - expect(logsString.includes('Video with name video 1')).to.be.false - expect(logsString.includes('Video with name video 2')).to.be.true - }) - - it('Should get logs with an end date', async function () { - this.timeout(60000) - - await server.videos.upload({ attributes: { name: 'video 3' } }) - await waitJobs([ server ]) - - const now1 = new Date() - - await server.videos.upload({ attributes: { name: 'video 4' } }) - await waitJobs([ server ]) - - const now2 = new Date() - - await server.videos.upload({ attributes: { name: 'video 5' } }) - await waitJobs([ server ]) - - const body = await logsCommand.getLogs({ startDate: now1, endDate: now2 }) - const logsString = JSON.stringify(body) - - expect(logsString.includes('Video with name video 3')).to.be.false - expect(logsString.includes('Video with name video 4')).to.be.true - expect(logsString.includes('Video with name video 5')).to.be.false - }) - - it('Should filter by level', async function () { - this.timeout(60000) - - const now = new Date() - - await server.videos.upload({ attributes: { name: 'video 6' } }) - await waitJobs([ server ]) - - { - const body = await logsCommand.getLogs({ startDate: now, level: 'info' }) - const logsString = JSON.stringify(body) - - expect(logsString.includes('Video with name video 6')).to.be.true - } - - { - const body = await logsCommand.getLogs({ startDate: now, level: 'warn' }) - const logsString = JSON.stringify(body) - - expect(logsString.includes('Video with name video 6')).to.be.false - } - }) - - it('Should filter by tag', async function () { - const now = new Date() - - const { uuid } = await server.videos.upload({ attributes: { name: 'video 6' } }) - await waitJobs([ server ]) - - { - const body = await logsCommand.getLogs({ startDate: now, level: 'debug', tagsOneOf: [ 'toto' ] }) - expect(body).to.have.lengthOf(0) - } - - { - const body = await logsCommand.getLogs({ startDate: now, level: 'debug', tagsOneOf: [ uuid ] }) - expect(body).to.not.have.lengthOf(0) - - for (const line of body) { - expect(line.tags).to.contain(uuid) - } - } - }) - - it('Should log ping requests', async function () { - const now = new Date() - - await server.servers.ping() - - const body = await logsCommand.getLogs({ startDate: now, level: 'info' }) - const logsString = JSON.stringify(body) - - expect(logsString.includes('/api/v1/ping')).to.be.true - }) - - it('Should not log ping requests', async function () { - this.timeout(60000) - - await killallServers([ server ]) - - await server.run({ log: { log_ping_requests: false } }) - - const now = new Date() - - await server.servers.ping() - - const body = await logsCommand.getLogs({ startDate: now, level: 'info' }) - const logsString = JSON.stringify(body) - - expect(logsString.includes('/api/v1/ping')).to.be.false - }) - }) - - describe('With the audit log', function () { - - it('Should get logs with a start date', async function () { - this.timeout(60000) - - await server.videos.upload({ attributes: { name: 'video 7' } }) - await waitJobs([ server ]) - - const now = new Date() - - await server.videos.upload({ attributes: { name: 'video 8' } }) - await waitJobs([ server ]) - - const body = await logsCommand.getAuditLogs({ startDate: now }) - const logsString = JSON.stringify(body) - - expect(logsString.includes('video 7')).to.be.false - expect(logsString.includes('video 8')).to.be.true - - expect(body).to.have.lengthOf(1) - - const item = body[0] - - const message = JSON.parse(item.message) - expect(message.domain).to.equal('videos') - expect(message.action).to.equal('create') - }) - - it('Should get logs with an end date', async function () { - this.timeout(60000) - - await server.videos.upload({ attributes: { name: 'video 9' } }) - await waitJobs([ server ]) - - const now1 = new Date() - - await server.videos.upload({ attributes: { name: 'video 10' } }) - await waitJobs([ server ]) - - const now2 = new Date() - - await server.videos.upload({ attributes: { name: 'video 11' } }) - await waitJobs([ server ]) - - const body = await logsCommand.getAuditLogs({ startDate: now1, endDate: now2 }) - const logsString = JSON.stringify(body) - - expect(logsString.includes('video 9')).to.be.false - expect(logsString.includes('video 10')).to.be.true - expect(logsString.includes('video 11')).to.be.false - }) - }) - - describe('When creating log from the client', function () { - - it('Should create a warn client log', async function () { - const now = new Date() - - await server.logs.createLogClient({ - payload: { - level: 'warn', - url: 'http://example.com', - message: 'my super client message' - }, - token: null - }) - - const body = await logsCommand.getLogs({ startDate: now }) - const logsString = JSON.stringify(body) - - expect(logsString.includes('my super client message')).to.be.true - }) - - it('Should create an error authenticated client log', async function () { - const now = new Date() - - await server.logs.createLogClient({ - payload: { - url: 'https://example.com/page1', - level: 'error', - message: 'my super client message 2', - userAgent: 'super user agent', - meta: '{hello}', - stackTrace: 'super stack trace' - } - }) - - const body = await logsCommand.getLogs({ startDate: now }) - const logsString = JSON.stringify(body) - - expect(logsString.includes('my super client message 2')).to.be.true - expect(logsString.includes('super user agent')).to.be.true - expect(logsString.includes('super stack trace')).to.be.true - expect(logsString.includes('{hello}')).to.be.true - expect(logsString.includes('https://example.com/page1')).to.be.true - }) - - it('Should refuse to create client logs', async function () { - await server.kill() - - await server.run({ - log: { - accept_client_log: false - } - }) - - await server.logs.createLogClient({ - payload: { - level: 'warn', - url: 'http://example.com', - message: 'my super client message' - }, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/server/no-client.ts b/server/tests/api/server/no-client.ts deleted file mode 100644 index 193f6c987..000000000 --- a/server/tests/api/server/no-client.ts +++ /dev/null @@ -1,24 +0,0 @@ -import request from 'supertest' -import { HttpStatusCode } from '@shared/models' -import { cleanupTests, createSingleServer, PeerTubeServer } from '@shared/server-commands' - -describe('Start and stop server without web client routes', function () { - let server: PeerTubeServer - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1, {}, { peertubeArgs: [ '--no-client' ] }) - }) - - it('Should fail getting the client', function () { - const req = request(server.url) - .get('/') - - return req.expect(HttpStatusCode.NOT_FOUND_404) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/server/open-telemetry.ts b/server/tests/api/server/open-telemetry.ts deleted file mode 100644 index 508e9d649..000000000 --- a/server/tests/api/server/open-telemetry.ts +++ /dev/null @@ -1,186 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { expectLogContain, expectLogDoesNotContain, MockHTTP } from '@server/tests/shared' -import { HttpStatusCode, PlaybackMetricCreate, VideoPrivacy, VideoResolution } from '@shared/models' -import { cleanupTests, createSingleServer, makeRawRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' - -describe('Open Telemetry', function () { - let server: PeerTubeServer - - describe('Metrics', function () { - const metricsUrl = 'http://127.0.0.1:9092/metrics' - - it('Should not enable open telemetry metrics', async function () { - this.timeout(60000) - - server = await createSingleServer(1) - - let hasError = false - try { - await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - } catch (err) { - hasError = err.message.includes('ECONNREFUSED') - } - - expect(hasError).to.be.true - - await server.kill() - }) - - it('Should enable open telemetry metrics', async function () { - this.timeout(120000) - - await server.run({ - open_telemetry: { - metrics: { - enabled: true - } - } - }) - - // Simulate a HTTP request - await server.videos.list() - - const res = await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.OK_200 }) - expect(res.text).to.contain('peertube_job_queue_total{') - expect(res.text).to.contain('http_request_duration_ms_bucket{') - }) - - it('Should have playback metrics', async function () { - await setAccessTokensToServers([ server ]) - - const video = await server.videos.quickUpload({ name: 'video' }) - - await server.metrics.addPlaybackMetric({ - metrics: { - playerMode: 'p2p-media-loader', - resolution: VideoResolution.H_1080P, - fps: 30, - resolutionChanges: 1, - errors: 2, - downloadedBytesP2P: 0, - downloadedBytesHTTP: 0, - uploadedBytesP2P: 5, - p2pPeers: 1, - p2pEnabled: false, - videoId: video.uuid - } - }) - - const res = await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.OK_200 }) - - expect(res.text).to.contain('peertube_playback_http_downloaded_bytes_total{') - expect(res.text).to.contain('peertube_playback_p2p_peers{') - expect(res.text).to.contain('p2pEnabled="false"') - }) - - it('Should take the last playback metric', async function () { - await setAccessTokensToServers([ server ]) - - const video = await server.videos.quickUpload({ name: 'video' }) - - const metrics = { - playerMode: 'p2p-media-loader', - resolution: VideoResolution.H_1080P, - fps: 30, - resolutionChanges: 1, - errors: 2, - downloadedBytesP2P: 0, - downloadedBytesHTTP: 0, - uploadedBytesP2P: 5, - p2pPeers: 7, - p2pEnabled: false, - videoId: video.uuid - } as PlaybackMetricCreate - - await server.metrics.addPlaybackMetric({ metrics }) - - metrics.p2pPeers = 42 - await server.metrics.addPlaybackMetric({ metrics }) - - const res = await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.OK_200 }) - - // eslint-disable-next-line max-len - const label = `{videoOrigin="local",playerMode="p2p-media-loader",resolution="1080",fps="30",p2pEnabled="false",videoUUID="${video.uuid}"}` - expect(res.text).to.contain(`peertube_playback_p2p_peers${label} 42`) - expect(res.text).to.not.contain(`peertube_playback_p2p_peers${label} 7`) - }) - - it('Should disable http request duration metrics', async function () { - await server.kill() - - await server.run({ - open_telemetry: { - metrics: { - enabled: true, - http_request_duration: { - enabled: false - } - } - } - }) - - // Simulate a HTTP request - await server.videos.list() - - const res = await makeRawRequest({ url: metricsUrl, expectedStatus: HttpStatusCode.OK_200 }) - expect(res.text).to.not.contain('http_request_duration_ms_bucket{') - }) - - after(async function () { - await server.kill() - }) - }) - - describe('Tracing', function () { - let mockHTTP: MockHTTP - let mockPort: number - - before(async function () { - mockHTTP = new MockHTTP() - mockPort = await mockHTTP.initialize() - }) - - it('Should enable open telemetry tracing', async function () { - server = await createSingleServer(1) - - await expectLogDoesNotContain(server, 'Registering Open Telemetry tracing') - - await server.kill() - }) - - it('Should enable open telemetry metrics', async function () { - await server.run({ - open_telemetry: { - tracing: { - enabled: true, - jaeger_exporter: { - endpoint: 'http://127.0.0.1:' + mockPort - } - } - } - }) - - await expectLogContain(server, 'Registering Open Telemetry tracing') - }) - - it('Should upload a video and correctly works', async function () { - await setAccessTokensToServers([ server ]) - - const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PUBLIC }) - - const video = await server.videos.get({ id: uuid }) - - expect(video.name).to.equal('video') - }) - - after(async function () { - await mockHTTP.terminate() - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/server/plugins.ts b/server/tests/api/server/plugins.ts deleted file mode 100644 index a0e9db1d3..000000000 --- a/server/tests/api/server/plugins.ts +++ /dev/null @@ -1,409 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { pathExists, remove } from 'fs-extra' -import { join } from 'path' -import { SQLCommand, testHelloWorldRegisteredSettings } from '@server/tests/shared' -import { wait } from '@shared/core-utils' -import { HttpStatusCode, PluginType } from '@shared/models' -import { - cleanupTests, - createSingleServer, - killallServers, - makeGetRequest, - PeerTubeServer, - PluginsCommand, - setAccessTokensToServers -} from '@shared/server-commands' - -describe('Test plugins', function () { - let server: PeerTubeServer - let sqlCommand: SQLCommand - let command: PluginsCommand - - before(async function () { - this.timeout(30000) - - const configOverride = { - plugins: { - index: { check_latest_versions_interval: '5 seconds' } - } - } - server = await createSingleServer(1, configOverride) - await setAccessTokensToServers([ server ]) - - command = server.plugins - - sqlCommand = new SQLCommand(server) - }) - - it('Should list and search available plugins and themes', async function () { - this.timeout(30000) - - { - const body = await command.listAvailable({ - count: 1, - start: 0, - pluginType: PluginType.THEME, - search: 'background-red' - }) - - expect(body.total).to.be.at.least(1) - expect(body.data).to.have.lengthOf(1) - } - - { - const body1 = await command.listAvailable({ - count: 2, - start: 0, - sort: 'npmName' - }) - expect(body1.total).to.be.at.least(2) - - const data1 = body1.data - expect(data1).to.have.lengthOf(2) - - const body2 = await command.listAvailable({ - count: 2, - start: 0, - sort: '-npmName' - }) - expect(body2.total).to.be.at.least(2) - - const data2 = body2.data - expect(data2).to.have.lengthOf(2) - - expect(data1[0].npmName).to.not.equal(data2[0].npmName) - } - - { - const body = await command.listAvailable({ - count: 10, - start: 0, - pluginType: PluginType.THEME, - search: 'background-red', - currentPeerTubeEngine: '1.0.0' - }) - - const p = body.data.find(p => p.npmName === 'peertube-theme-background-red') - expect(p).to.be.undefined - } - }) - - it('Should install a plugin and a theme', async function () { - this.timeout(30000) - - await command.install({ npmName: 'peertube-plugin-hello-world' }) - await command.install({ npmName: 'peertube-theme-background-red' }) - }) - - it('Should have the plugin loaded in the configuration', async function () { - for (const config of [ await server.config.getConfig(), await server.config.getIndexHTMLConfig() ]) { - const theme = config.theme.registered.find(r => r.name === 'background-red') - expect(theme).to.not.be.undefined - expect(theme.npmName).to.equal('peertube-theme-background-red') - - const plugin = config.plugin.registered.find(r => r.name === 'hello-world') - expect(plugin).to.not.be.undefined - expect(plugin.npmName).to.equal('peertube-plugin-hello-world') - } - }) - - it('Should update the default theme in the configuration', async function () { - await server.config.updateCustomSubConfig({ - newConfig: { - theme: { default: 'background-red' } - } - }) - - for (const config of [ await server.config.getConfig(), await server.config.getIndexHTMLConfig() ]) { - expect(config.theme.default).to.equal('background-red') - } - }) - - it('Should update my default theme', async function () { - await server.users.updateMe({ theme: 'background-red' }) - - const user = await server.users.getMyInfo() - expect(user.theme).to.equal('background-red') - }) - - it('Should list plugins and themes', async function () { - { - const body = await command.list({ - count: 1, - start: 0, - pluginType: PluginType.THEME - }) - expect(body.total).to.be.at.least(1) - - const data = body.data - expect(data).to.have.lengthOf(1) - expect(data[0].name).to.equal('background-red') - } - - { - const { data } = await command.list({ - count: 2, - start: 0, - sort: 'name' - }) - - expect(data[0].name).to.equal('background-red') - expect(data[1].name).to.equal('hello-world') - } - - { - const body = await command.list({ - count: 2, - start: 1, - sort: 'name' - }) - - expect(body.data[0].name).to.equal('hello-world') - } - }) - - it('Should get registered settings', async function () { - await testHelloWorldRegisteredSettings(server) - }) - - it('Should get public settings', async function () { - const body = await command.getPublicSettings({ npmName: 'peertube-plugin-hello-world' }) - const publicSettings = body.publicSettings - - expect(Object.keys(publicSettings)).to.have.lengthOf(1) - expect(Object.keys(publicSettings)).to.deep.equal([ 'user-name' ]) - expect(publicSettings['user-name']).to.be.null - }) - - it('Should update the settings', async function () { - const settings = { - 'admin-name': 'Cid' - } - - await command.updateSettings({ - npmName: 'peertube-plugin-hello-world', - settings - }) - }) - - it('Should have watched settings changes', async function () { - await server.servers.waitUntilLog('Settings changed!') - }) - - it('Should get a plugin and a theme', async function () { - { - const plugin = await command.get({ npmName: 'peertube-plugin-hello-world' }) - - expect(plugin.type).to.equal(PluginType.PLUGIN) - expect(plugin.name).to.equal('hello-world') - expect(plugin.description).to.exist - expect(plugin.homepage).to.exist - expect(plugin.uninstalled).to.be.false - expect(plugin.enabled).to.be.true - expect(plugin.description).to.exist - expect(plugin.version).to.exist - expect(plugin.peertubeEngine).to.exist - expect(plugin.createdAt).to.exist - - expect(plugin.settings).to.not.be.undefined - expect(plugin.settings['admin-name']).to.equal('Cid') - } - - { - const plugin = await command.get({ npmName: 'peertube-theme-background-red' }) - - expect(plugin.type).to.equal(PluginType.THEME) - expect(plugin.name).to.equal('background-red') - expect(plugin.description).to.exist - expect(plugin.homepage).to.exist - expect(plugin.uninstalled).to.be.false - expect(plugin.enabled).to.be.true - expect(plugin.description).to.exist - expect(plugin.version).to.exist - expect(plugin.peertubeEngine).to.exist - expect(plugin.createdAt).to.exist - - expect(plugin.settings).to.be.null - } - }) - - it('Should update the plugin and the theme', async function () { - this.timeout(180000) - - // Wait the scheduler that get the latest plugins versions - await wait(6000) - - async function testUpdate (type: 'plugin' | 'theme', name: string) { - // Fake update our plugin version - await sqlCommand.setPluginVersion(name, '0.0.1') - - // Fake update package.json - const packageJSON = await command.getPackageJSON(`peertube-${type}-${name}`) - const oldVersion = packageJSON.version - - packageJSON.version = '0.0.1' - await command.updatePackageJSON(`peertube-${type}-${name}`, packageJSON) - - // Restart the server to take into account this change - await killallServers([ server ]) - await server.run() - - const checkConfig = async (version: string) => { - for (const config of [ await server.config.getConfig(), await server.config.getIndexHTMLConfig() ]) { - expect(config[type].registered.find(r => r.name === name).version).to.equal(version) - } - } - - const getPluginFromAPI = async () => { - const body = await command.list({ pluginType: type === 'plugin' ? PluginType.PLUGIN : PluginType.THEME }) - - return body.data.find(p => p.name === name) - } - - { - const plugin = await getPluginFromAPI() - expect(plugin.version).to.equal('0.0.1') - expect(plugin.latestVersion).to.exist - expect(plugin.latestVersion).to.not.equal('0.0.1') - - await checkConfig('0.0.1') - } - - { - await command.update({ npmName: `peertube-${type}-${name}` }) - - const plugin = await getPluginFromAPI() - expect(plugin.version).to.equal(oldVersion) - - const updatedPackageJSON = await command.getPackageJSON(`peertube-${type}-${name}`) - expect(updatedPackageJSON.version).to.equal(oldVersion) - - await checkConfig(oldVersion) - } - } - - await testUpdate('theme', 'background-red') - await testUpdate('plugin', 'hello-world') - }) - - it('Should uninstall the plugin', async function () { - await command.uninstall({ npmName: 'peertube-plugin-hello-world' }) - - const body = await command.list({ pluginType: PluginType.PLUGIN }) - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - }) - - it('Should list uninstalled plugins', async function () { - const body = await command.list({ pluginType: PluginType.PLUGIN, uninstalled: true }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - - const plugin = body.data[0] - expect(plugin.name).to.equal('hello-world') - expect(plugin.enabled).to.be.false - expect(plugin.uninstalled).to.be.true - }) - - it('Should uninstall the theme', async function () { - await command.uninstall({ npmName: 'peertube-theme-background-red' }) - }) - - it('Should have updated the configuration', async function () { - for (const config of [ await server.config.getConfig(), await server.config.getIndexHTMLConfig() ]) { - expect(config.theme.default).to.equal('default') - - const theme = config.theme.registered.find(r => r.name === 'background-red') - expect(theme).to.be.undefined - - const plugin = config.plugin.registered.find(r => r.name === 'hello-world') - expect(plugin).to.be.undefined - } - }) - - it('Should have updated the user theme', async function () { - const user = await server.users.getMyInfo() - expect(user.theme).to.equal('instance-default') - }) - - it('Should not install a broken plugin', async function () { - this.timeout(60000) - - async function check () { - const body = await command.list({ pluginType: PluginType.PLUGIN }) - const plugins = body.data - expect(plugins.find(p => p.name === 'test-broken')).to.not.exist - } - - await command.install({ - path: PluginsCommand.getPluginTestPath('-broken'), - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - - await check() - - await killallServers([ server ]) - await server.run() - - await check() - }) - - it('Should rebuild native modules on Node ABI change', async function () { - this.timeout(60000) - - const removeNativeModule = async () => { - await remove(join(baseNativeModule, 'build')) - await remove(join(baseNativeModule, 'prebuilds')) - } - - await command.install({ path: PluginsCommand.getPluginTestPath('-native') }) - - await makeGetRequest({ - url: server.url, - path: '/plugins/test-native/router', - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - - const query = `UPDATE "application" SET "nodeABIVersion" = 1` - await sqlCommand.updateQuery(query) - - const baseNativeModule = server.servers.buildDirectory(join('plugins', 'node_modules', 'a-native-example')) - - await removeNativeModule() - await server.kill() - await server.run() - - await wait(3000) - - expect(await pathExists(join(baseNativeModule, 'build'))).to.be.true - expect(await pathExists(join(baseNativeModule, 'prebuilds'))).to.be.true - - await makeGetRequest({ - url: server.url, - path: '/plugins/test-native/router', - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - - await removeNativeModule() - - await server.kill() - await server.run() - - expect(await pathExists(join(baseNativeModule, 'build'))).to.be.false - expect(await pathExists(join(baseNativeModule, 'prebuilds'))).to.be.false - - await makeGetRequest({ - url: server.url, - path: '/plugins/test-native/router', - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - after(async function () { - await sqlCommand.cleanup() - - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/server/proxy.ts b/server/tests/api/server/proxy.ts deleted file mode 100644 index 9337468d5..000000000 --- a/server/tests/api/server/proxy.ts +++ /dev/null @@ -1,171 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { expectNotStartWith, expectStartWith, FIXTURE_URLS, MockProxy } from '@server/tests/shared' -import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' -import { HttpStatusCode, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - ObjectStorageCommand, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - waitJobs -} from '@shared/server-commands' - -describe('Test proxy', function () { - let servers: PeerTubeServer[] = [] - let proxy: MockProxy - - const goodEnv = { HTTP_PROXY: '' } - const badEnv = { HTTP_PROXY: 'http://127.0.0.1:9000' } - - before(async function () { - this.timeout(120000) - - proxy = new MockProxy() - - const proxyPort = await proxy.initialize() - servers = await createMultipleServers(2) - - goodEnv.HTTP_PROXY = 'http://127.0.0.1:' + proxyPort - - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - await doubleFollow(servers[0], servers[1]) - }) - - describe('Federation', function () { - - it('Should succeed federation with the appropriate proxy config', async function () { - this.timeout(40000) - - await servers[0].kill() - await servers[0].run({}, { env: goodEnv }) - - await servers[0].videos.quickUpload({ name: 'video 1' }) - - await waitJobs(servers) - - for (const server of servers) { - const { total, data } = await server.videos.list() - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - } - }) - - it('Should fail federation with a wrong proxy config', async function () { - this.timeout(40000) - - await servers[0].kill() - await servers[0].run({}, { env: badEnv }) - - await servers[0].videos.quickUpload({ name: 'video 2' }) - - await waitJobs(servers) - - { - const { total, data } = await servers[0].videos.list() - expect(total).to.equal(2) - expect(data).to.have.lengthOf(2) - } - - { - const { total, data } = await servers[1].videos.list() - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - } - }) - }) - - describe('Videos import', async function () { - - function quickImport (expectedStatus: HttpStatusCode = HttpStatusCode.OK_200) { - return servers[0].imports.importVideo({ - attributes: { - name: 'video import', - channelId: servers[0].store.channel.id, - privacy: VideoPrivacy.PUBLIC, - targetUrl: FIXTURE_URLS.peertube_long - }, - expectedStatus - }) - } - - it('Should succeed import with the appropriate proxy config', async function () { - this.timeout(240000) - - await servers[0].kill() - await servers[0].run({}, { env: goodEnv }) - - await quickImport() - - await waitJobs(servers) - - const { total, data } = await servers[0].videos.list() - expect(total).to.equal(3) - expect(data).to.have.lengthOf(3) - }) - - it('Should fail import with a wrong proxy config', async function () { - this.timeout(120000) - - await servers[0].kill() - await servers[0].run({}, { env: badEnv }) - - await quickImport(HttpStatusCode.BAD_REQUEST_400) - }) - }) - - describe('Object storage', function () { - if (areMockObjectStorageTestsDisabled()) return - - const objectStorage = new ObjectStorageCommand() - - before(async function () { - this.timeout(30000) - - await objectStorage.prepareDefaultMockBuckets() - }) - - it('Should succeed to upload to object storage with the appropriate proxy config', async function () { - this.timeout(120000) - - await servers[0].kill() - await servers[0].run(objectStorage.getDefaultMockConfig(), { env: goodEnv }) - - const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) - await waitJobs(servers) - - const video = await servers[0].videos.get({ id: uuid }) - - expectStartWith(video.files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl()) - }) - - it('Should fail to upload to object storage with a wrong proxy config', async function () { - this.timeout(120000) - - await servers[0].kill() - await servers[0].run(objectStorage.getDefaultMockConfig(), { env: badEnv }) - - const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) - await waitJobs(servers, { skipDelayed: true }) - - const video = await servers[0].videos.get({ id: uuid }) - - expectNotStartWith(video.files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl()) - }) - - after(async function () { - await objectStorage.cleanupMock() - }) - }) - - after(async function () { - await proxy.terminate() - - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/server/reverse-proxy.ts b/server/tests/api/server/reverse-proxy.ts deleted file mode 100644 index 11c96c4b5..000000000 --- a/server/tests/api/server/reverse-proxy.ts +++ /dev/null @@ -1,156 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { wait } from '@shared/core-utils' -import { HttpStatusCode } from '@shared/models' -import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' - -describe('Test application behind a reverse proxy', function () { - let server: PeerTubeServer - let userAccessToken: string - let videoId: string - - before(async function () { - this.timeout(60000) - - const config = { - rates_limit: { - api: { - max: 50, - window: 5000 - }, - signup: { - max: 3, - window: 5000 - }, - login: { - max: 20 - } - }, - signup: { - limit: 20 - } - } - - server = await createSingleServer(1, config) - await setAccessTokensToServers([ server ]) - - userAccessToken = await server.users.generateUserAndToken('user') - - const { uuid } = await server.videos.upload() - videoId = uuid - }) - - it('Should view a video only once with the same IP by default', async function () { - this.timeout(40000) - - await server.views.simulateView({ id: videoId }) - await server.views.simulateView({ id: videoId }) - - // Wait the repeatable job - await wait(8000) - - const video = await server.videos.get({ id: videoId }) - expect(video.views).to.equal(1) - }) - - it('Should view a video 2 times with the X-Forwarded-For header set', async function () { - this.timeout(20000) - - await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.1,127.0.0.1' }) - await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.2,127.0.0.1' }) - - // Wait the repeatable job - await wait(8000) - - const video = await server.videos.get({ id: videoId }) - expect(video.views).to.equal(3) - }) - - it('Should view a video only once with the same client IP in the X-Forwarded-For header', async function () { - this.timeout(20000) - - await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.4,0.0.0.3,::ffff:127.0.0.1' }) - await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.5,0.0.0.3,127.0.0.1' }) - - // Wait the repeatable job - await wait(8000) - - const video = await server.videos.get({ id: videoId }) - expect(video.views).to.equal(4) - }) - - it('Should view a video two times with a different client IP in the X-Forwarded-For header', async function () { - this.timeout(20000) - - await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.8,0.0.0.6,127.0.0.1' }) - await server.views.simulateView({ id: videoId, xForwardedFor: '0.0.0.8,0.0.0.7,127.0.0.1' }) - - // Wait the repeatable job - await wait(8000) - - const video = await server.videos.get({ id: videoId }) - expect(video.views).to.equal(6) - }) - - it('Should rate limit logins', async function () { - const user = { username: 'root', password: 'fail' } - - for (let i = 0; i < 18; i++) { - await server.login.login({ user, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - } - - await server.login.login({ user, expectedStatus: HttpStatusCode.TOO_MANY_REQUESTS_429 }) - }) - - it('Should rate limit signup', async function () { - for (let i = 0; i < 10; i++) { - try { - await server.registrations.register({ username: 'test' + i }) - } catch { - // empty - } - } - - await server.registrations.register({ username: 'test42', expectedStatus: HttpStatusCode.TOO_MANY_REQUESTS_429 }) - }) - - it('Should not rate limit failed signup', async function () { - this.timeout(30000) - - await wait(7000) - - for (let i = 0; i < 3; i++) { - await server.registrations.register({ username: 'test' + i, expectedStatus: HttpStatusCode.CONFLICT_409 }) - } - - await server.registrations.register({ username: 'test43', expectedStatus: HttpStatusCode.NO_CONTENT_204 }) - - }) - - it('Should rate limit API calls', async function () { - this.timeout(30000) - - await wait(7000) - - for (let i = 0; i < 100; i++) { - try { - await server.videos.get({ id: videoId }) - } catch { - // don't care if it fails - } - } - - await server.videos.get({ id: videoId, expectedStatus: HttpStatusCode.TOO_MANY_REQUESTS_429 }) - }) - - it('Should rate limit API calls with a user but not with an admin', async function () { - await server.videos.get({ id: videoId, token: userAccessToken, expectedStatus: HttpStatusCode.TOO_MANY_REQUESTS_429 }) - - await server.videos.get({ id: videoId, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/server/services.ts b/server/tests/api/server/services.ts deleted file mode 100644 index a10e9baed..000000000 --- a/server/tests/api/server/services.ts +++ /dev/null @@ -1,137 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers, setDefaultVideoChannel } from '@shared/server-commands' -import { Video, VideoPlaylistPrivacy } from '@shared/models' - -describe('Test services', function () { - let server: PeerTubeServer = null - let playlistUUID: string - let playlistDisplayName: string - let video: Video - - const urlSuffixes = [ - { - input: '', - output: '' - }, - { - input: '?param=1', - output: '' - }, - { - input: '?muted=1&warningTitle=0&toto=1', - output: '?muted=1&warningTitle=0' - } - ] - - before(async function () { - this.timeout(120000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - { - const attributes = { name: 'my super name' } - await server.videos.upload({ attributes }) - - const { data } = await server.videos.list() - video = data[0] - } - - { - const created = await server.playlists.create({ - attributes: { - displayName: 'The Life and Times of Scrooge McDuck', - privacy: VideoPlaylistPrivacy.PUBLIC, - videoChannelId: server.store.channel.id - } - }) - - playlistUUID = created.uuid - playlistDisplayName = 'The Life and Times of Scrooge McDuck' - - await server.playlists.addElement({ - playlistId: created.id, - attributes: { - videoId: video.id - } - }) - } - }) - - it('Should have a valid oEmbed video response', async function () { - for (const basePath of [ '/videos/watch/', '/w/' ]) { - for (const suffix of urlSuffixes) { - const oembedUrl = server.url + basePath + video.uuid + suffix.input - - const res = await server.services.getOEmbed({ oembedUrl }) - const expectedHtml = '' - - const expectedThumbnailUrl = 'http://' + server.host + video.previewPath - - expect(res.body.html).to.equal(expectedHtml) - expect(res.body.title).to.equal(video.name) - expect(res.body.author_name).to.equal(server.store.channel.displayName) - expect(res.body.width).to.equal(560) - expect(res.body.height).to.equal(315) - expect(res.body.thumbnail_url).to.equal(expectedThumbnailUrl) - expect(res.body.thumbnail_width).to.equal(850) - expect(res.body.thumbnail_height).to.equal(480) - } - } - }) - - it('Should have a valid playlist oEmbed response', async function () { - for (const basePath of [ '/videos/watch/playlist/', '/w/p/' ]) { - for (const suffix of urlSuffixes) { - const oembedUrl = server.url + basePath + playlistUUID + suffix.input - - const res = await server.services.getOEmbed({ oembedUrl }) - const expectedHtml = '' - - expect(res.body.html).to.equal(expectedHtml) - expect(res.body.title).to.equal('The Life and Times of Scrooge McDuck') - expect(res.body.author_name).to.equal(server.store.channel.displayName) - expect(res.body.width).to.equal(560) - expect(res.body.height).to.equal(315) - expect(res.body.thumbnail_url).exist - expect(res.body.thumbnail_width).to.equal(280) - expect(res.body.thumbnail_height).to.equal(157) - } - } - }) - - it('Should have a valid oEmbed response with small max height query', async function () { - for (const basePath of [ '/videos/watch/', '/w/' ]) { - const oembedUrl = 'http://' + server.host + basePath + video.uuid - const format = 'json' - const maxHeight = 50 - const maxWidth = 50 - - const res = await server.services.getOEmbed({ oembedUrl, format, maxHeight, maxWidth }) - const expectedHtml = '' - - expect(res.body.html).to.equal(expectedHtml) - expect(res.body.title).to.equal(video.name) - expect(res.body.author_name).to.equal(server.store.channel.displayName) - expect(res.body.height).to.equal(50) - expect(res.body.width).to.equal(50) - expect(res.body).to.not.have.property('thumbnail_url') - expect(res.body).to.not.have.property('thumbnail_width') - expect(res.body).to.not.have.property('thumbnail_height') - } - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/server/slow-follows.ts b/server/tests/api/server/slow-follows.ts deleted file mode 100644 index a967fa724..000000000 --- a/server/tests/api/server/slow-follows.ts +++ /dev/null @@ -1,85 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { Job } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test slow follows', function () { - let servers: PeerTubeServer[] = [] - - let afterFollows: Date - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(3) - - // Get the access tokens - await setAccessTokensToServers(servers) - - await doubleFollow(servers[0], servers[1]) - await doubleFollow(servers[0], servers[2]) - - afterFollows = new Date() - - for (let i = 0; i < 5; i++) { - await servers[0].videos.quickUpload({ name: 'video ' + i }) - } - - await waitJobs(servers) - }) - - it('Should only have broadcast jobs', async function () { - const { data } = await servers[0].jobs.list({ jobType: 'activitypub-http-unicast', sort: '-createdAt' }) - - for (const job of data) { - expect(new Date(job.createdAt)).below(afterFollows) - } - }) - - it('Should process bad follower', async function () { - this.timeout(30000) - - await servers[1].kill() - - // Set server 2 as bad follower - await servers[0].videos.quickUpload({ name: 'video 6' }) - await waitJobs(servers[0]) - - afterFollows = new Date() - const filter = (job: Job) => new Date(job.createdAt) > afterFollows - - // Resend another broadcast job - await servers[0].videos.quickUpload({ name: 'video 7' }) - await waitJobs(servers[0]) - - const resBroadcast = await servers[0].jobs.list({ jobType: 'activitypub-http-broadcast', sort: '-createdAt' }) - const resUnicast = await servers[0].jobs.list({ jobType: 'activitypub-http-unicast', sort: '-createdAt' }) - - const broadcast = resBroadcast.data.filter(filter) - const unicast = resUnicast.data.filter(filter) - - expect(unicast).to.have.lengthOf(2) - expect(broadcast).to.have.lengthOf(2) - - for (const u of unicast) { - expect(u.data.uri).to.equal(servers[1].url + '/inbox') - } - - for (const b of broadcast) { - expect(b.data.uris).to.have.lengthOf(1) - expect(b.data.uris[0]).to.equal(servers[2].url + '/inbox') - } - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/server/stats.ts b/server/tests/api/server/stats.ts deleted file mode 100644 index a1bf189fa..000000000 --- a/server/tests/api/server/stats.ts +++ /dev/null @@ -1,279 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { wait } from '@shared/core-utils' -import { ActivityType, VideoPlaylistPrivacy } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - setDefaultAccountAvatar, - setDefaultChannelAvatar, - waitJobs -} from '@shared/server-commands' - -describe('Test stats (excluding redundancy)', function () { - let servers: PeerTubeServer[] = [] - let channelId - const user = { - username: 'user1', - password: 'super_password' - } - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(3) - - await setAccessTokensToServers(servers) - await setDefaultChannelAvatar(servers) - await setDefaultAccountAvatar(servers) - - await doubleFollow(servers[0], servers[1]) - - await servers[0].users.create({ username: user.username, password: user.password }) - - const { uuid } = await servers[0].videos.upload({ attributes: { fixture: 'video_short.webm' } }) - - await servers[0].comments.createThread({ videoId: uuid, text: 'comment' }) - - await servers[0].views.simulateView({ id: uuid }) - - // Wait the video views repeatable job - await wait(8000) - - await servers[2].follows.follow({ hosts: [ servers[0].url ] }) - await waitJobs(servers) - }) - - it('Should have the correct stats on instance 1', async function () { - const data = await servers[0].stats.get() - - expect(data.totalLocalVideoComments).to.equal(1) - expect(data.totalLocalVideos).to.equal(1) - expect(data.totalLocalVideoViews).to.equal(1) - expect(data.totalLocalVideoFilesSize).to.equal(218910) - expect(data.totalUsers).to.equal(2) - expect(data.totalVideoComments).to.equal(1) - expect(data.totalVideos).to.equal(1) - expect(data.totalInstanceFollowers).to.equal(2) - expect(data.totalInstanceFollowing).to.equal(1) - expect(data.totalLocalPlaylists).to.equal(0) - }) - - it('Should have the correct stats on instance 2', async function () { - const data = await servers[1].stats.get() - - expect(data.totalLocalVideoComments).to.equal(0) - expect(data.totalLocalVideos).to.equal(0) - expect(data.totalLocalVideoViews).to.equal(0) - expect(data.totalLocalVideoFilesSize).to.equal(0) - expect(data.totalUsers).to.equal(1) - expect(data.totalVideoComments).to.equal(1) - expect(data.totalVideos).to.equal(1) - expect(data.totalInstanceFollowers).to.equal(1) - expect(data.totalInstanceFollowing).to.equal(1) - expect(data.totalLocalPlaylists).to.equal(0) - }) - - it('Should have the correct stats on instance 3', async function () { - const data = await servers[2].stats.get() - - expect(data.totalLocalVideoComments).to.equal(0) - expect(data.totalLocalVideos).to.equal(0) - expect(data.totalLocalVideoViews).to.equal(0) - expect(data.totalUsers).to.equal(1) - expect(data.totalVideoComments).to.equal(1) - expect(data.totalVideos).to.equal(1) - expect(data.totalInstanceFollowing).to.equal(1) - expect(data.totalInstanceFollowers).to.equal(0) - expect(data.totalLocalPlaylists).to.equal(0) - }) - - it('Should have the correct total videos stats after an unfollow', async function () { - this.timeout(15000) - - await servers[2].follows.unfollow({ target: servers[0] }) - await waitJobs(servers) - - const data = await servers[2].stats.get() - - expect(data.totalVideos).to.equal(0) - }) - - it('Should have the correct active user stats', async function () { - const server = servers[0] - - { - const data = await server.stats.get() - - expect(data.totalDailyActiveUsers).to.equal(1) - expect(data.totalWeeklyActiveUsers).to.equal(1) - expect(data.totalMonthlyActiveUsers).to.equal(1) - } - - { - await server.login.getAccessToken(user) - - const data = await server.stats.get() - - expect(data.totalDailyActiveUsers).to.equal(2) - expect(data.totalWeeklyActiveUsers).to.equal(2) - expect(data.totalMonthlyActiveUsers).to.equal(2) - } - }) - - it('Should have the correct active channel stats', async function () { - const server = servers[0] - - { - const data = await server.stats.get() - - expect(data.totalLocalVideoChannels).to.equal(2) - expect(data.totalLocalDailyActiveVideoChannels).to.equal(1) - expect(data.totalLocalWeeklyActiveVideoChannels).to.equal(1) - expect(data.totalLocalMonthlyActiveVideoChannels).to.equal(1) - } - - { - const attributes = { - name: 'stats_channel', - displayName: 'My stats channel' - } - const created = await server.channels.create({ attributes }) - channelId = created.id - - const data = await server.stats.get() - - expect(data.totalLocalVideoChannels).to.equal(3) - expect(data.totalLocalDailyActiveVideoChannels).to.equal(1) - expect(data.totalLocalWeeklyActiveVideoChannels).to.equal(1) - expect(data.totalLocalMonthlyActiveVideoChannels).to.equal(1) - } - - { - await server.videos.upload({ attributes: { fixture: 'video_short.webm', channelId } }) - - const data = await server.stats.get() - - expect(data.totalLocalVideoChannels).to.equal(3) - expect(data.totalLocalDailyActiveVideoChannels).to.equal(2) - expect(data.totalLocalWeeklyActiveVideoChannels).to.equal(2) - expect(data.totalLocalMonthlyActiveVideoChannels).to.equal(2) - } - }) - - it('Should have the correct playlist stats', async function () { - const server = servers[0] - - { - const data = await server.stats.get() - expect(data.totalLocalPlaylists).to.equal(0) - } - - { - await server.playlists.create({ - attributes: { - displayName: 'playlist for count', - privacy: VideoPlaylistPrivacy.PUBLIC, - videoChannelId: channelId - } - }) - - const data = await server.stats.get() - expect(data.totalLocalPlaylists).to.equal(1) - } - }) - - it('Should correctly count video file sizes if transcoding is enabled', async function () { - this.timeout(120000) - - await servers[0].config.updateCustomSubConfig({ - newConfig: { - transcoding: { - enabled: true, - webVideos: { - enabled: true - }, - hls: { - enabled: true - }, - resolutions: { - '0p': false, - '144p': false, - '240p': false, - '360p': false, - '480p': false, - '720p': false, - '1080p': false, - '1440p': false, - '2160p': false - } - } - } - }) - - await servers[0].videos.upload({ attributes: { name: 'video', fixture: 'video_short.webm' } }) - - await waitJobs(servers) - - { - const data = await servers[1].stats.get() - expect(data.totalLocalVideoFilesSize).to.equal(0) - } - - { - const data = await servers[0].stats.get() - expect(data.totalLocalVideoFilesSize).to.be.greaterThan(500000) - expect(data.totalLocalVideoFilesSize).to.be.lessThan(600000) - } - }) - - it('Should have the correct AP stats', async function () { - this.timeout(120000) - - await servers[0].config.disableTranscoding() - - const first = await servers[1].stats.get() - - for (let i = 0; i < 10; i++) { - await servers[0].videos.upload({ attributes: { name: 'video' } }) - } - - await waitJobs(servers) - - await wait(6000) - - const second = await servers[1].stats.get() - expect(second.totalActivityPubMessagesProcessed).to.be.greaterThan(first.totalActivityPubMessagesProcessed) - - const apTypes: ActivityType[] = [ - 'Create', 'Update', 'Delete', 'Follow', 'Accept', 'Announce', 'Undo', 'Like', 'Reject', 'View', 'Dislike', 'Flag' - ] - - const processed = apTypes.reduce( - (previous, type) => previous + second['totalActivityPub' + type + 'MessagesSuccesses'], - 0 - ) - expect(second.totalActivityPubMessagesProcessed).to.equal(processed) - expect(second.totalActivityPubMessagesSuccesses).to.equal(processed) - - expect(second.totalActivityPubMessagesErrors).to.equal(0) - - for (const apType of apTypes) { - expect(second['totalActivityPub' + apType + 'MessagesErrors']).to.equal(0) - } - - await wait(6000) - - const third = await servers[1].stats.get() - expect(third.totalActivityPubMessagesWaiting).to.equal(0) - expect(third.activityPubMessagesProcessedPerSecond).to.be.lessThan(second.activityPubMessagesProcessedPerSecond) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/server/tracker.ts b/server/tests/api/server/tracker.ts deleted file mode 100644 index a0ce2ca35..000000000 --- a/server/tests/api/server/tracker.ts +++ /dev/null @@ -1,104 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await,@typescript-eslint/no-floating-promises */ - -import { decode as magnetUriDecode, encode as magnetUriEncode } from 'magnet-uri' -import WebTorrent from 'webtorrent' -import { cleanupTests, createSingleServer, killallServers, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' - -describe('Test tracker', function () { - let server: PeerTubeServer - let badMagnet: string - let goodMagnet: string - - before(async function () { - this.timeout(60000) - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - - { - const { uuid } = await server.videos.upload() - const video = await server.videos.get({ id: uuid }) - goodMagnet = video.files[0].magnetUri - - const parsed = magnetUriDecode(goodMagnet) - parsed.infoHash = '010597bb88b1968a5693a4fa8267c592ca65f2e9' - - badMagnet = magnetUriEncode(parsed) - } - }) - - it('Should succeed with the correct infohash', function (done) { - const webtorrent = new WebTorrent() - - const torrent = webtorrent.add(goodMagnet) - - torrent.on('error', done) - torrent.on('warning', warn => { - const message = typeof warn === 'string' ? warn : warn.message - if (message.includes('Unknown infoHash ')) return done(new Error('Error on infohash')) - }) - - torrent.on('done', done) - }) - - it('Should disable the tracker', function (done) { - this.timeout(20000) - - const errCb = () => done(new Error('Tracker is enabled')) - - killallServers([ server ]) - .then(() => server.run({ tracker: { enabled: false } })) - .then(() => { - const webtorrent = new WebTorrent() - - const torrent = webtorrent.add(goodMagnet) - - torrent.on('error', done) - torrent.on('warning', warn => { - const message = typeof warn === 'string' ? warn : warn.message - if (message.includes('disabled ')) { - torrent.off('done', errCb) - - return done() - } - }) - - torrent.on('done', errCb) - }) - }) - - it('Should return an error when adding an incorrect infohash', function (done) { - this.timeout(20000) - - killallServers([ server ]) - .then(() => server.run()) - .then(() => { - const webtorrent = new WebTorrent() - - const torrent = webtorrent.add(badMagnet) - - torrent.on('error', done) - torrent.on('warning', warn => { - const message = typeof warn === 'string' ? warn : warn.message - if (message.includes('Unknown infoHash ')) return done() - }) - - torrent.on('done', () => done(new Error('No error on infohash'))) - }) - }) - - it('Should block the IP after the failed infohash', function (done) { - const webtorrent = new WebTorrent() - - const torrent = webtorrent.add(goodMagnet) - - torrent.on('error', done) - torrent.on('warning', warn => { - const message = typeof warn === 'string' ? warn : warn.message - if (message.includes('Unsupported tracker protocol')) return done() - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/transcoding/audio-only.ts b/server/tests/api/transcoding/audio-only.ts deleted file mode 100644 index f4cc012ef..000000000 --- a/server/tests/api/transcoding/audio-only.ts +++ /dev/null @@ -1,104 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { getAudioStream, getVideoStreamDimensionsInfo } from '@shared/ffmpeg' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test audio only video transcoding', function () { - let servers: PeerTubeServer[] = [] - let videoUUID: string - let webVideoAudioFileUrl: string - let fragmentedAudioFileUrl: string - - before(async function () { - this.timeout(120000) - - const configOverride = { - transcoding: { - enabled: true, - resolutions: { - '0p': true, - '144p': false, - '240p': true, - '360p': false, - '480p': false, - '720p': false, - '1080p': false, - '1440p': false, - '2160p': false - }, - hls: { - enabled: true - }, - web_videos: { - enabled: true - } - } - } - servers = await createMultipleServers(2, configOverride) - - // Get the access tokens - await setAccessTokensToServers(servers) - - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - }) - - it('Should upload a video and transcode it', async function () { - this.timeout(120000) - - const { uuid } = await servers[0].videos.upload({ attributes: { name: 'audio only' } }) - videoUUID = uuid - - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - expect(video.streamingPlaylists).to.have.lengthOf(1) - - for (const files of [ video.files, video.streamingPlaylists[0].files ]) { - expect(files).to.have.lengthOf(3) - expect(files[0].resolution.id).to.equal(720) - expect(files[1].resolution.id).to.equal(240) - expect(files[2].resolution.id).to.equal(0) - } - - if (server.serverNumber === 1) { - webVideoAudioFileUrl = video.files[2].fileUrl - fragmentedAudioFileUrl = video.streamingPlaylists[0].files[2].fileUrl - } - } - }) - - it('0p transcoded video should not have video', async function () { - const paths = [ - servers[0].servers.buildWebVideoFilePath(webVideoAudioFileUrl), - servers[0].servers.buildFragmentedFilePath(videoUUID, fragmentedAudioFileUrl) - ] - - for (const path of paths) { - const { audioStream } = await getAudioStream(path) - expect(audioStream['codec_name']).to.be.equal('aac') - expect(audioStream['bit_rate']).to.be.at.most(384 * 8000) - - const size = await getVideoStreamDimensionsInfo(path) - - expect(size.height).to.equal(0) - expect(size.width).to.equal(0) - expect(size.isPortraitMode).to.be.false - expect(size.ratio).to.equal(0) - expect(size.resolution).to.equal(0) - } - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/transcoding/create-transcoding.ts b/server/tests/api/transcoding/create-transcoding.ts deleted file mode 100644 index 9a891043c..000000000 --- a/server/tests/api/transcoding/create-transcoding.ts +++ /dev/null @@ -1,266 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { checkResolutionsInMasterPlaylist, expectStartWith } from '@server/tests/shared' -import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' -import { HttpStatusCode, VideoDetails } from '@shared/models' -import { - cleanupTests, - ConfigCommand, - createMultipleServers, - doubleFollow, - expectNoFailedTranscodingJob, - makeRawRequest, - ObjectStorageCommand, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -async function checkFilesInObjectStorage (objectStorage: ObjectStorageCommand, video: VideoDetails) { - for (const file of video.files) { - expectStartWith(file.fileUrl, objectStorage.getMockWebVideosBaseUrl()) - await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) - } - - if (video.streamingPlaylists.length === 0) return - - const hlsPlaylist = video.streamingPlaylists[0] - for (const file of hlsPlaylist.files) { - expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl()) - await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) - } - - expectStartWith(hlsPlaylist.playlistUrl, objectStorage.getMockPlaylistBaseUrl()) - await makeRawRequest({ url: hlsPlaylist.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) - - expectStartWith(hlsPlaylist.segmentsSha256Url, objectStorage.getMockPlaylistBaseUrl()) - await makeRawRequest({ url: hlsPlaylist.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) -} - -function runTests (enableObjectStorage: boolean) { - let servers: PeerTubeServer[] = [] - let videoUUID: string - let publishedAt: string - - let shouldBeDeleted: string[] - const objectStorage = new ObjectStorageCommand() - - before(async function () { - this.timeout(120000) - - const config = enableObjectStorage - ? objectStorage.getDefaultMockConfig() - : {} - - // Run server 2 to have transcoding enabled - servers = await createMultipleServers(2, config) - await setAccessTokensToServers(servers) - - await servers[0].config.disableTranscoding() - - await doubleFollow(servers[0], servers[1]) - - if (enableObjectStorage) await objectStorage.prepareDefaultMockBuckets() - - const { shortUUID } = await servers[0].videos.quickUpload({ name: 'video' }) - videoUUID = shortUUID - - await waitJobs(servers) - - const video = await servers[0].videos.get({ id: videoUUID }) - publishedAt = video.publishedAt as string - - await servers[0].config.enableTranscoding() - }) - - it('Should generate HLS', async function () { - this.timeout(60000) - - await servers[0].videos.runTranscoding({ - videoId: videoUUID, - transcodingType: 'hls' - }) - - await waitJobs(servers) - await expectNoFailedTranscodingJob(servers[0]) - - for (const server of servers) { - const videoDetails = await server.videos.get({ id: videoUUID }) - - expect(videoDetails.files).to.have.lengthOf(1) - expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) - expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5) - - if (enableObjectStorage) await checkFilesInObjectStorage(objectStorage, videoDetails) - } - }) - - it('Should generate Web Video', async function () { - this.timeout(60000) - - await servers[0].videos.runTranscoding({ - videoId: videoUUID, - transcodingType: 'web-video' - }) - - await waitJobs(servers) - - for (const server of servers) { - const videoDetails = await server.videos.get({ id: videoUUID }) - - expect(videoDetails.files).to.have.lengthOf(5) - expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) - expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5) - - if (enableObjectStorage) await checkFilesInObjectStorage(objectStorage, videoDetails) - } - }) - - it('Should generate Web Video from HLS only video', async function () { - this.timeout(60000) - - await servers[0].videos.removeAllWebVideoFiles({ videoId: videoUUID }) - await waitJobs(servers) - - await servers[0].videos.runTranscoding({ videoId: videoUUID, transcodingType: 'web-video' }) - await waitJobs(servers) - - for (const server of servers) { - const videoDetails = await server.videos.get({ id: videoUUID }) - - expect(videoDetails.files).to.have.lengthOf(5) - expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) - expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5) - - if (enableObjectStorage) await checkFilesInObjectStorage(objectStorage, videoDetails) - } - }) - - it('Should only generate Web Video', async function () { - this.timeout(60000) - - await servers[0].videos.removeHLSPlaylist({ videoId: videoUUID }) - await waitJobs(servers) - - await servers[0].videos.runTranscoding({ videoId: videoUUID, transcodingType: 'web-video' }) - await waitJobs(servers) - - for (const server of servers) { - const videoDetails = await server.videos.get({ id: videoUUID }) - - expect(videoDetails.files).to.have.lengthOf(5) - expect(videoDetails.streamingPlaylists).to.have.lengthOf(0) - - if (enableObjectStorage) await checkFilesInObjectStorage(objectStorage, videoDetails) - } - }) - - it('Should correctly update HLS playlist on resolution change', async function () { - this.timeout(120000) - - await servers[0].config.updateExistingSubConfig({ - newConfig: { - transcoding: { - enabled: true, - resolutions: ConfigCommand.getCustomConfigResolutions(false), - - webVideos: { - enabled: true - }, - hls: { - enabled: true - } - } - } - }) - - const { uuid } = await servers[0].videos.quickUpload({ name: 'quick' }) - - await waitJobs(servers) - - for (const server of servers) { - const videoDetails = await server.videos.get({ id: uuid }) - - expect(videoDetails.files).to.have.lengthOf(1) - expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) - expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(1) - - if (enableObjectStorage) await checkFilesInObjectStorage(objectStorage, videoDetails) - - shouldBeDeleted = [ - videoDetails.streamingPlaylists[0].files[0].fileUrl, - videoDetails.streamingPlaylists[0].playlistUrl, - videoDetails.streamingPlaylists[0].segmentsSha256Url - ] - } - - await servers[0].config.updateExistingSubConfig({ - newConfig: { - transcoding: { - enabled: true, - resolutions: ConfigCommand.getCustomConfigResolutions(true), - - webVideos: { - enabled: true - }, - hls: { - enabled: true - } - } - } - }) - - await servers[0].videos.runTranscoding({ videoId: uuid, transcodingType: 'hls' }) - await waitJobs(servers) - - for (const server of servers) { - const videoDetails = await server.videos.get({ id: uuid }) - - expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) - expect(videoDetails.streamingPlaylists[0].files).to.have.lengthOf(5) - - if (enableObjectStorage) { - await checkFilesInObjectStorage(objectStorage, videoDetails) - - const hlsPlaylist = videoDetails.streamingPlaylists[0] - const resolutions = hlsPlaylist.files.map(f => f.resolution.id) - await checkResolutionsInMasterPlaylist({ server: servers[0], playlistUrl: hlsPlaylist.playlistUrl, resolutions }) - - const shaBody = await servers[0].streamingPlaylists.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url, withRetry: true }) - expect(Object.keys(shaBody)).to.have.lengthOf(5) - } - } - }) - - it('Should have correctly deleted previous files', async function () { - for (const fileUrl of shouldBeDeleted) { - await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - } - }) - - it('Should not have updated published at attributes', async function () { - const video = await servers[0].videos.get({ id: videoUUID }) - - expect(video.publishedAt).to.equal(publishedAt) - }) - - after(async function () { - if (objectStorage) await objectStorage.cleanupMock() - - await cleanupTests(servers) - }) -} - -describe('Test create transcoding jobs from API', function () { - - describe('On filesystem', function () { - runTests(false) - }) - - describe('On object storage', function () { - if (areMockObjectStorageTestsDisabled()) return - - runTests(true) - }) -}) diff --git a/server/tests/api/transcoding/hls.ts b/server/tests/api/transcoding/hls.ts deleted file mode 100644 index d67043c2a..000000000 --- a/server/tests/api/transcoding/hls.ts +++ /dev/null @@ -1,175 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { join } from 'path' -import { checkDirectoryIsEmpty, checkTmpIsEmpty, completeCheckHlsPlaylist } from '@server/tests/shared' -import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' -import { HttpStatusCode } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - ObjectStorageCommand, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' -import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants' - -describe('Test HLS videos', function () { - let servers: PeerTubeServer[] = [] - - function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) { - const videoUUIDs: string[] = [] - - it('Should upload a video and transcode it to HLS', async function () { - this.timeout(120000) - - const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video 1', fixture: 'video_short.webm' } }) - videoUUIDs.push(uuid) - - await waitJobs(servers) - - await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl }) - }) - - it('Should upload an audio file and transcode it to HLS', async function () { - this.timeout(120000) - - const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video audio', fixture: 'sample.ogg' } }) - videoUUIDs.push(uuid) - - await waitJobs(servers) - - await completeCheckHlsPlaylist({ - servers, - videoUUID: uuid, - hlsOnly, - resolutions: [ DEFAULT_AUDIO_RESOLUTION, 360, 240 ], - objectStorageBaseUrl - }) - }) - - it('Should update the video', async function () { - this.timeout(30000) - - await servers[0].videos.update({ id: videoUUIDs[0], attributes: { name: 'video 1 updated' } }) - - await waitJobs(servers) - - await completeCheckHlsPlaylist({ servers, videoUUID: videoUUIDs[0], hlsOnly, objectStorageBaseUrl }) - }) - - it('Should delete videos', async function () { - for (const uuid of videoUUIDs) { - await servers[0].videos.remove({ id: uuid }) - } - - await waitJobs(servers) - - for (const server of servers) { - for (const uuid of videoUUIDs) { - await server.videos.get({ id: uuid, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - } - } - }) - - it('Should have the playlists/segment deleted from the disk', async function () { - for (const server of servers) { - await checkDirectoryIsEmpty(server, 'web-videos', [ 'private' ]) - await checkDirectoryIsEmpty(server, join('web-videos', 'private')) - - await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls'), [ 'private' ]) - await checkDirectoryIsEmpty(server, join('streaming-playlists', 'hls', 'private')) - } - }) - - it('Should have an empty tmp directory', async function () { - for (const server of servers) { - await checkTmpIsEmpty(server) - } - }) - } - - before(async function () { - this.timeout(120000) - - const configOverride = { - transcoding: { - enabled: true, - allow_audio_files: true, - hls: { - enabled: true - } - } - } - servers = await createMultipleServers(2, configOverride) - - // Get the access tokens - await setAccessTokensToServers(servers) - - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - }) - - describe('With Web Video & HLS enabled', function () { - runTestSuite(false) - }) - - describe('With only HLS enabled', function () { - - before(async function () { - await servers[0].config.updateCustomSubConfig({ - newConfig: { - transcoding: { - enabled: true, - allowAudioFiles: true, - resolutions: { - '144p': false, - '240p': true, - '360p': true, - '480p': true, - '720p': true, - '1080p': true, - '1440p': true, - '2160p': true - }, - hls: { - enabled: true - }, - webVideos: { - enabled: false - } - } - } - }) - }) - - runTestSuite(true) - }) - - describe('With object storage enabled', function () { - if (areMockObjectStorageTestsDisabled()) return - - const objectStorage = new ObjectStorageCommand() - - before(async function () { - this.timeout(120000) - - const configOverride = objectStorage.getDefaultMockConfig() - await objectStorage.prepareDefaultMockBuckets() - - await servers[0].kill() - await servers[0].run(configOverride) - }) - - runTestSuite(true, objectStorage.getMockPlaylistBaseUrl()) - - after(async function () { - await objectStorage.cleanupMock() - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/transcoding/index.ts b/server/tests/api/transcoding/index.ts deleted file mode 100644 index 9866418d6..000000000 --- a/server/tests/api/transcoding/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from './audio-only' -export * from './create-transcoding' -export * from './hls' -export * from './transcoder' -export * from './update-while-transcoding' -export * from './video-studio' diff --git a/server/tests/api/transcoding/transcoder.ts b/server/tests/api/transcoding/transcoder.ts deleted file mode 100644 index 5386d236f..000000000 --- a/server/tests/api/transcoding/transcoder.ts +++ /dev/null @@ -1,800 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { canDoQuickTranscode } from '@server/lib/transcoding/transcoding-quick-transcode' -import { checkWebTorrentWorks, generateHighBitrateVideo, generateVideoWithFramerate } from '@server/tests/shared' -import { buildAbsoluteFixturePath, getAllFiles, getMaxTheoreticalBitrate, getMinTheoreticalBitrate, omit } from '@shared/core-utils' -import { - ffprobePromise, - getAudioStream, - getVideoStreamBitrate, - getVideoStreamDimensionsInfo, - getVideoStreamFPS, - hasAudioStream -} from '@shared/ffmpeg' -import { HttpStatusCode, VideoFileMetadata, VideoState } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - makeGetRequest, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -function updateConfigForTranscoding (server: PeerTubeServer) { - return server.config.updateCustomSubConfig({ - newConfig: { - transcoding: { - enabled: true, - allowAdditionalExtensions: true, - allowAudioFiles: true, - hls: { enabled: true }, - webVideos: { enabled: true }, - resolutions: { - '0p': false, - '144p': true, - '240p': true, - '360p': true, - '480p': true, - '720p': true, - '1080p': true, - '1440p': true, - '2160p': true - } - } - } - }) -} - -describe('Test video transcoding', function () { - let servers: PeerTubeServer[] = [] - let video4k: string - - before(async function () { - this.timeout(30_000) - - // Run servers - servers = await createMultipleServers(2) - - await setAccessTokensToServers(servers) - - await doubleFollow(servers[0], servers[1]) - - await updateConfigForTranscoding(servers[1]) - }) - - describe('Basic transcoding (or not)', function () { - - it('Should not transcode video on server 1', async function () { - this.timeout(60_000) - - const attributes = { - name: 'my super name for server 1', - description: 'my super description for server 1', - fixture: 'video_short.webm' - } - await servers[0].videos.upload({ attributes }) - - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.videos.list() - const video = data[0] - - const videoDetails = await server.videos.get({ id: video.id }) - expect(videoDetails.files).to.have.lengthOf(1) - - const magnetUri = videoDetails.files[0].magnetUri - expect(magnetUri).to.match(/\.webm/) - - await checkWebTorrentWorks(magnetUri, /\.webm$/) - } - }) - - it('Should transcode video on server 2', async function () { - this.timeout(120_000) - - const attributes = { - name: 'my super name for server 2', - description: 'my super description for server 2', - fixture: 'video_short.webm' - } - await servers[1].videos.upload({ attributes }) - - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.videos.list() - - const video = data.find(v => v.name === attributes.name) - const videoDetails = await server.videos.get({ id: video.id }) - - expect(videoDetails.files).to.have.lengthOf(5) - - const magnetUri = videoDetails.files[0].magnetUri - expect(magnetUri).to.match(/\.mp4/) - - await checkWebTorrentWorks(magnetUri, /\.mp4$/) - } - }) - - it('Should wait for transcoding before publishing the video', async function () { - this.timeout(160_000) - - { - // Upload the video, but wait transcoding - const attributes = { - name: 'waiting video', - fixture: 'video_short1.webm', - waitTranscoding: true - } - const { uuid } = await servers[1].videos.upload({ attributes }) - const videoId = uuid - - // Should be in transcode state - const body = await servers[1].videos.get({ id: videoId }) - expect(body.name).to.equal('waiting video') - expect(body.state.id).to.equal(VideoState.TO_TRANSCODE) - expect(body.state.label).to.equal('To transcode') - expect(body.waitTranscoding).to.be.true - - { - // Should have my video - const { data } = await servers[1].videos.listMyVideos() - const videoToFindInMine = data.find(v => v.name === attributes.name) - expect(videoToFindInMine).not.to.be.undefined - expect(videoToFindInMine.state.id).to.equal(VideoState.TO_TRANSCODE) - expect(videoToFindInMine.state.label).to.equal('To transcode') - expect(videoToFindInMine.waitTranscoding).to.be.true - } - - { - // Should not list this video - const { data } = await servers[1].videos.list() - const videoToFindInList = data.find(v => v.name === attributes.name) - expect(videoToFindInList).to.be.undefined - } - - // Server 1 should not have the video yet - await servers[0].videos.get({ id: videoId, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - } - - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.videos.list() - const videoToFind = data.find(v => v.name === 'waiting video') - expect(videoToFind).not.to.be.undefined - - const videoDetails = await server.videos.get({ id: videoToFind.id }) - - expect(videoDetails.state.id).to.equal(VideoState.PUBLISHED) - expect(videoDetails.state.label).to.equal('Published') - expect(videoDetails.waitTranscoding).to.be.true - } - }) - - it('Should accept and transcode additional extensions', async function () { - this.timeout(300_000) - - for (const fixture of [ 'video_short.mkv', 'video_short.avi' ]) { - const attributes = { - name: fixture, - fixture - } - - await servers[1].videos.upload({ attributes }) - - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.videos.list() - - const video = data.find(v => v.name === attributes.name) - const videoDetails = await server.videos.get({ id: video.id }) - expect(videoDetails.files).to.have.lengthOf(5) - - const magnetUri = videoDetails.files[0].magnetUri - expect(magnetUri).to.contain('.mp4') - } - } - }) - - it('Should transcode a 4k video', async function () { - this.timeout(200_000) - - const attributes = { - name: '4k video', - fixture: 'video_short_4k.mp4' - } - - const { uuid } = await servers[1].videos.upload({ attributes }) - video4k = uuid - - await waitJobs(servers) - - const resolutions = [ 144, 240, 360, 480, 720, 1080, 1440, 2160 ] - - for (const server of servers) { - const videoDetails = await server.videos.get({ id: video4k }) - expect(videoDetails.files).to.have.lengthOf(resolutions.length) - - for (const r of resolutions) { - expect(videoDetails.files.find(f => f.resolution.id === r)).to.not.be.undefined - expect(videoDetails.streamingPlaylists[0].files.find(f => f.resolution.id === r)).to.not.be.undefined - } - } - }) - }) - - describe('Audio transcoding', function () { - - it('Should transcode high bit rate mp3 to proper bit rate', async function () { - this.timeout(60_000) - - const attributes = { - name: 'mp3_256k', - fixture: 'video_short_mp3_256k.mp4' - } - await servers[1].videos.upload({ attributes }) - - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.videos.list() - - const video = data.find(v => v.name === attributes.name) - const videoDetails = await server.videos.get({ id: video.id }) - - expect(videoDetails.files).to.have.lengthOf(5) - - const file = videoDetails.files.find(f => f.resolution.id === 240) - const path = servers[1].servers.buildWebVideoFilePath(file.fileUrl) - const probe = await getAudioStream(path) - - if (probe.audioStream) { - expect(probe.audioStream['codec_name']).to.be.equal('aac') - expect(probe.audioStream['bit_rate']).to.be.at.most(384 * 8000) - } else { - this.fail('Could not retrieve the audio stream on ' + probe.absolutePath) - } - } - }) - - it('Should transcode video with no audio and have no audio itself', async function () { - this.timeout(60_000) - - const attributes = { - name: 'no_audio', - fixture: 'video_short_no_audio.mp4' - } - await servers[1].videos.upload({ attributes }) - - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.videos.list() - - const video = data.find(v => v.name === attributes.name) - const videoDetails = await server.videos.get({ id: video.id }) - - const file = videoDetails.files.find(f => f.resolution.id === 240) - const path = servers[1].servers.buildWebVideoFilePath(file.fileUrl) - - expect(await hasAudioStream(path)).to.be.false - } - }) - - it('Should leave the audio untouched, but properly transcode the video', async function () { - this.timeout(60_000) - - const attributes = { - name: 'untouched_audio', - fixture: 'video_short.mp4' - } - await servers[1].videos.upload({ attributes }) - - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.videos.list() - - const video = data.find(v => v.name === attributes.name) - const videoDetails = await server.videos.get({ id: video.id }) - - expect(videoDetails.files).to.have.lengthOf(5) - - const fixturePath = buildAbsoluteFixturePath(attributes.fixture) - const fixtureVideoProbe = await getAudioStream(fixturePath) - - const file = videoDetails.files.find(f => f.resolution.id === 240) - const path = servers[1].servers.buildWebVideoFilePath(file.fileUrl) - - const videoProbe = await getAudioStream(path) - - if (videoProbe.audioStream && fixtureVideoProbe.audioStream) { - const toOmit = [ 'max_bit_rate', 'duration', 'duration_ts', 'nb_frames', 'start_time', 'start_pts' ] - expect(omit(videoProbe.audioStream, toOmit)).to.be.deep.equal(omit(fixtureVideoProbe.audioStream, toOmit)) - } else { - this.fail('Could not retrieve the audio stream on ' + videoProbe.absolutePath) - } - } - }) - }) - - describe('Audio upload', function () { - - function runSuite (mode: 'legacy' | 'resumable') { - - before(async function () { - await servers[1].config.updateCustomSubConfig({ - newConfig: { - transcoding: { - hls: { enabled: true }, - webVideos: { enabled: true }, - resolutions: { - '0p': false, - '144p': false, - '240p': false, - '360p': false, - '480p': false, - '720p': false, - '1080p': false, - '1440p': false, - '2160p': false - } - } - } - }) - }) - - it('Should merge an audio file with the preview file', async function () { - this.timeout(60_000) - - const attributes = { name: 'audio_with_preview', previewfile: 'custom-preview.jpg', fixture: 'sample.ogg' } - await servers[1].videos.upload({ attributes, mode }) - - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.videos.list() - - const video = data.find(v => v.name === 'audio_with_preview') - const videoDetails = await server.videos.get({ id: video.id }) - - expect(videoDetails.files).to.have.lengthOf(1) - - await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) - await makeGetRequest({ url: server.url, path: videoDetails.previewPath, expectedStatus: HttpStatusCode.OK_200 }) - - const magnetUri = videoDetails.files[0].magnetUri - expect(magnetUri).to.contain('.mp4') - } - }) - - it('Should upload an audio file and choose a default background image', async function () { - this.timeout(60_000) - - const attributes = { name: 'audio_without_preview', fixture: 'sample.ogg' } - await servers[1].videos.upload({ attributes, mode }) - - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.videos.list() - - const video = data.find(v => v.name === 'audio_without_preview') - const videoDetails = await server.videos.get({ id: video.id }) - - expect(videoDetails.files).to.have.lengthOf(1) - - await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) - await makeGetRequest({ url: server.url, path: videoDetails.previewPath, expectedStatus: HttpStatusCode.OK_200 }) - - const magnetUri = videoDetails.files[0].magnetUri - expect(magnetUri).to.contain('.mp4') - } - }) - - it('Should upload an audio file and create an audio version only', async function () { - this.timeout(60_000) - - await servers[1].config.updateCustomSubConfig({ - newConfig: { - transcoding: { - hls: { enabled: true }, - webVideos: { enabled: true }, - resolutions: { - '0p': true, - '144p': false, - '240p': false, - '360p': false - } - } - } - }) - - const attributes = { name: 'audio_with_preview', previewfile: 'custom-preview.jpg', fixture: 'sample.ogg' } - const { id } = await servers[1].videos.upload({ attributes, mode }) - - await waitJobs(servers) - - for (const server of servers) { - const videoDetails = await server.videos.get({ id }) - - for (const files of [ videoDetails.files, videoDetails.streamingPlaylists[0].files ]) { - expect(files).to.have.lengthOf(2) - expect(files.find(f => f.resolution.id === 0)).to.not.be.undefined - } - } - - await updateConfigForTranscoding(servers[1]) - }) - } - - describe('Legacy upload', function () { - runSuite('legacy') - }) - - describe('Resumable upload', function () { - runSuite('resumable') - }) - }) - - describe('Framerate', function () { - - it('Should transcode a 60 FPS video', async function () { - this.timeout(60_000) - - const attributes = { - name: 'my super 30fps name for server 2', - description: 'my super 30fps description for server 2', - fixture: '60fps_720p_small.mp4' - } - await servers[1].videos.upload({ attributes }) - - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.videos.list() - - const video = data.find(v => v.name === attributes.name) - const videoDetails = await server.videos.get({ id: video.id }) - - expect(videoDetails.files).to.have.lengthOf(5) - expect(videoDetails.files[0].fps).to.be.above(58).and.below(62) - expect(videoDetails.files[1].fps).to.be.below(31) - expect(videoDetails.files[2].fps).to.be.below(31) - expect(videoDetails.files[3].fps).to.be.below(31) - expect(videoDetails.files[4].fps).to.be.below(31) - - for (const resolution of [ 144, 240, 360, 480 ]) { - const file = videoDetails.files.find(f => f.resolution.id === resolution) - const path = servers[1].servers.buildWebVideoFilePath(file.fileUrl) - const fps = await getVideoStreamFPS(path) - - expect(fps).to.be.below(31) - } - - const file = videoDetails.files.find(f => f.resolution.id === 720) - const path = servers[1].servers.buildWebVideoFilePath(file.fileUrl) - const fps = await getVideoStreamFPS(path) - - expect(fps).to.be.above(58).and.below(62) - } - }) - - it('Should downscale to the closest divisor standard framerate', async function () { - this.timeout(200_000) - - let tempFixturePath: string - - { - tempFixturePath = await generateVideoWithFramerate(59) - - const fps = await getVideoStreamFPS(tempFixturePath) - expect(fps).to.be.equal(59) - } - - const attributes = { - name: '59fps video', - description: '59fps video', - fixture: tempFixturePath - } - - await servers[1].videos.upload({ attributes }) - - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.videos.list() - - const { id } = data.find(v => v.name === attributes.name) - const video = await server.videos.get({ id }) - - { - const file = video.files.find(f => f.resolution.id === 240) - const path = servers[1].servers.buildWebVideoFilePath(file.fileUrl) - const fps = await getVideoStreamFPS(path) - expect(fps).to.be.equal(25) - } - - { - const file = video.files.find(f => f.resolution.id === 720) - const path = servers[1].servers.buildWebVideoFilePath(file.fileUrl) - const fps = await getVideoStreamFPS(path) - expect(fps).to.be.equal(59) - } - } - }) - }) - - describe('Bitrate control', function () { - - it('Should respect maximum bitrate values', async function () { - this.timeout(160_000) - - const tempFixturePath = await generateHighBitrateVideo() - - const attributes = { - name: 'high bitrate video', - description: 'high bitrate video', - fixture: tempFixturePath - } - - await servers[1].videos.upload({ attributes }) - - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.videos.list() - - const { id } = data.find(v => v.name === attributes.name) - const video = await server.videos.get({ id }) - - for (const resolution of [ 240, 360, 480, 720, 1080 ]) { - const file = video.files.find(f => f.resolution.id === resolution) - const path = servers[1].servers.buildWebVideoFilePath(file.fileUrl) - - const bitrate = await getVideoStreamBitrate(path) - const fps = await getVideoStreamFPS(path) - const dataResolution = await getVideoStreamDimensionsInfo(path) - - expect(resolution).to.equal(resolution) - - const maxBitrate = getMaxTheoreticalBitrate({ ...dataResolution, fps }) - expect(bitrate).to.be.below(maxBitrate) - } - } - }) - - it('Should not transcode to an higher bitrate than the original file but above our low limit', async function () { - this.timeout(160_000) - - const newConfig = { - transcoding: { - enabled: true, - resolutions: { - '144p': true, - '240p': true, - '360p': true, - '480p': true, - '720p': true, - '1080p': true, - '1440p': true, - '2160p': true - }, - webVideos: { enabled: true }, - hls: { enabled: true } - } - } - await servers[1].config.updateCustomSubConfig({ newConfig }) - - const attributes = { - name: 'low bitrate', - fixture: 'low-bitrate.mp4' - } - - const { id } = await servers[1].videos.upload({ attributes }) - - await waitJobs(servers) - - const video = await servers[1].videos.get({ id }) - - const resolutions = [ 240, 360, 480, 720, 1080 ] - for (const r of resolutions) { - const file = video.files.find(f => f.resolution.id === r) - - const path = servers[1].servers.buildWebVideoFilePath(file.fileUrl) - const bitrate = await getVideoStreamBitrate(path) - - const inputBitrate = 60_000 - const limit = getMinTheoreticalBitrate({ fps: 10, ratio: 1, resolution: r }) - let belowValue = Math.max(inputBitrate, limit) - belowValue += belowValue * 0.20 // Apply 20% margin because bitrate control is not very precise - - expect(bitrate, `${path} not below ${limit}`).to.be.below(belowValue) - } - }) - }) - - describe('FFprobe', function () { - - it('Should provide valid ffprobe data', async function () { - this.timeout(160_000) - - const videoUUID = (await servers[1].videos.quickUpload({ name: 'ffprobe data' })).uuid - await waitJobs(servers) - - { - const video = await servers[1].videos.get({ id: videoUUID }) - const file = video.files.find(f => f.resolution.id === 240) - const path = servers[1].servers.buildWebVideoFilePath(file.fileUrl) - - const probe = await ffprobePromise(path) - const metadata = new VideoFileMetadata(probe) - - // expected format properties - for (const p of [ - 'tags.encoder', - 'format_long_name', - 'size', - 'bit_rate' - ]) { - expect(metadata.format).to.have.nested.property(p) - } - - // expected stream properties - for (const p of [ - 'codec_long_name', - 'profile', - 'width', - 'height', - 'display_aspect_ratio', - 'avg_frame_rate', - 'pix_fmt' - ]) { - expect(metadata.streams[0]).to.have.nested.property(p) - } - - expect(metadata).to.not.have.nested.property('format.filename') - } - - for (const server of servers) { - const videoDetails = await server.videos.get({ id: videoUUID }) - - const videoFiles = getAllFiles(videoDetails) - expect(videoFiles).to.have.lengthOf(10) - - for (const file of videoFiles) { - expect(file.metadata).to.be.undefined - expect(file.metadataUrl).to.exist - expect(file.metadataUrl).to.contain(servers[1].url) - expect(file.metadataUrl).to.contain(videoUUID) - - const metadata = await server.videos.getFileMetadata({ url: file.metadataUrl }) - expect(metadata).to.have.nested.property('format.size') - } - } - }) - - it('Should correctly detect if quick transcode is possible', async function () { - this.timeout(10_000) - - expect(await canDoQuickTranscode(buildAbsoluteFixturePath('video_short.mp4'))).to.be.true - expect(await canDoQuickTranscode(buildAbsoluteFixturePath('video_short.webm'))).to.be.false - }) - }) - - describe('Transcoding job queue', function () { - - it('Should have the appropriate priorities for transcoding jobs', async function () { - const body = await servers[1].jobs.list({ - start: 0, - count: 100, - sort: 'createdAt', - jobType: 'video-transcoding' - }) - - const jobs = body.data - const transcodingJobs = jobs.filter(j => j.data.videoUUID === video4k) - - expect(transcodingJobs).to.have.lengthOf(16) - - const hlsJobs = transcodingJobs.filter(j => j.data.type === 'new-resolution-to-hls') - const webVideoJobs = transcodingJobs.filter(j => j.data.type === 'new-resolution-to-web-video') - const optimizeJobs = transcodingJobs.filter(j => j.data.type === 'optimize-to-web-video') - - expect(hlsJobs).to.have.lengthOf(8) - expect(webVideoJobs).to.have.lengthOf(7) - expect(optimizeJobs).to.have.lengthOf(1) - - for (const j of optimizeJobs.concat(hlsJobs.concat(webVideoJobs))) { - expect(j.priority).to.be.greaterThan(100) - expect(j.priority).to.be.lessThan(150) - } - }) - }) - - describe('Bounded transcoding', function () { - - it('Should not generate an upper resolution than original file', async function () { - this.timeout(120_000) - - await servers[0].config.updateExistingSubConfig({ - newConfig: { - transcoding: { - enabled: true, - hls: { enabled: true }, - webVideos: { enabled: true }, - resolutions: { - '0p': false, - '144p': false, - '240p': true, - '360p': false, - '480p': true, - '720p': false, - '1080p': false, - '1440p': false, - '2160p': false - }, - alwaysTranscodeOriginalResolution: false - } - } - }) - - const { uuid } = await servers[0].videos.quickUpload({ name: 'video', fixture: 'video_short.webm' }) - await waitJobs(servers) - - const video = await servers[0].videos.get({ id: uuid }) - const hlsFiles = video.streamingPlaylists[0].files - - expect(video.files).to.have.lengthOf(2) - expect(hlsFiles).to.have.lengthOf(2) - - // eslint-disable-next-line @typescript-eslint/require-array-sort-compare - const resolutions = getAllFiles(video).map(f => f.resolution.id).sort() - expect(resolutions).to.deep.equal([ 240, 240, 480, 480 ]) - }) - - it('Should only keep the original resolution if all resolutions are disabled', async function () { - this.timeout(120_000) - - await servers[0].config.updateExistingSubConfig({ - newConfig: { - transcoding: { - resolutions: { - '0p': false, - '144p': false, - '240p': false, - '360p': false, - '480p': false, - '720p': false, - '1080p': false, - '1440p': false, - '2160p': false - } - } - } - }) - - const { uuid } = await servers[0].videos.quickUpload({ name: 'video', fixture: 'video_short.webm' }) - await waitJobs(servers) - - const video = await servers[0].videos.get({ id: uuid }) - const hlsFiles = video.streamingPlaylists[0].files - - expect(video.files).to.have.lengthOf(1) - expect(hlsFiles).to.have.lengthOf(1) - - expect(video.files[0].resolution.id).to.equal(720) - expect(hlsFiles[0].resolution.id).to.equal(720) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/transcoding/update-while-transcoding.ts b/server/tests/api/transcoding/update-while-transcoding.ts deleted file mode 100644 index cfb4fa0cc..000000000 --- a/server/tests/api/transcoding/update-while-transcoding.ts +++ /dev/null @@ -1,160 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { completeCheckHlsPlaylist } from '@server/tests/shared' -import { areMockObjectStorageTestsDisabled, wait } from '@shared/core-utils' -import { VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - ObjectStorageCommand, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test update video privacy while transcoding', function () { - let servers: PeerTubeServer[] = [] - - const videoUUIDs: string[] = [] - - function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) { - - it('Should not have an error while quickly updating a private video to public after upload #1', async function () { - this.timeout(360_000) - - const attributes = { - name: 'quick update', - privacy: VideoPrivacy.PRIVATE - } - - const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: false }) - await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } }) - videoUUIDs.push(uuid) - - await waitJobs(servers) - - await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl }) - }) - - it('Should not have an error while quickly updating a private video to public after upload #2', async function () { - this.timeout(60000) - - { - const attributes = { - name: 'quick update 2', - privacy: VideoPrivacy.PRIVATE - } - - const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: true }) - await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } }) - videoUUIDs.push(uuid) - - await waitJobs(servers) - - await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl }) - } - }) - - it('Should not have an error while quickly updating a private video to public after upload #3', async function () { - this.timeout(60000) - - const attributes = { - name: 'quick update 3', - privacy: VideoPrivacy.PRIVATE - } - - const { uuid } = await servers[0].videos.upload({ attributes, waitTorrentGeneration: true }) - await wait(1000) - await servers[0].videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } }) - videoUUIDs.push(uuid) - - await waitJobs(servers) - - await completeCheckHlsPlaylist({ servers, videoUUID: uuid, hlsOnly, objectStorageBaseUrl }) - }) - } - - before(async function () { - this.timeout(120000) - - const configOverride = { - transcoding: { - enabled: true, - allow_audio_files: true, - hls: { - enabled: true - } - } - } - servers = await createMultipleServers(2, configOverride) - - // Get the access tokens - await setAccessTokensToServers(servers) - - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - }) - - describe('With Web Video & HLS enabled', function () { - runTestSuite(false) - }) - - describe('With only HLS enabled', function () { - - before(async function () { - await servers[0].config.updateCustomSubConfig({ - newConfig: { - transcoding: { - enabled: true, - allowAudioFiles: true, - resolutions: { - '144p': false, - '240p': true, - '360p': true, - '480p': true, - '720p': true, - '1080p': true, - '1440p': true, - '2160p': true - }, - hls: { - enabled: true - }, - webVideos: { - enabled: false - } - } - } - }) - }) - - runTestSuite(true) - }) - - describe('With object storage enabled', function () { - if (areMockObjectStorageTestsDisabled()) return - - const objectStorage = new ObjectStorageCommand() - - before(async function () { - this.timeout(120000) - - const configOverride = objectStorage.getDefaultMockConfig() - await objectStorage.prepareDefaultMockBuckets() - - await servers[0].kill() - await servers[0].run(configOverride) - }) - - runTestSuite(true, objectStorage.getMockPlaylistBaseUrl()) - - after(async function () { - await objectStorage.cleanupMock() - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/transcoding/video-studio.ts b/server/tests/api/transcoding/video-studio.ts deleted file mode 100644 index ba68f8e24..000000000 --- a/server/tests/api/transcoding/video-studio.ts +++ /dev/null @@ -1,377 +0,0 @@ -import { expect } from 'chai' -import { checkPersistentTmpIsEmpty, checkVideoDuration, expectStartWith } from '@server/tests/shared' -import { areMockObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils' -import { VideoStudioTask } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - ObjectStorageCommand, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - VideoStudioCommand, - waitJobs -} from '@shared/server-commands' - -describe('Test video studio', function () { - let servers: PeerTubeServer[] = [] - let videoUUID: string - - async function renewVideo (fixture = 'video_short.webm') { - const video = await servers[0].videos.quickUpload({ name: 'video', fixture }) - videoUUID = video.uuid - - await waitJobs(servers) - } - - async function createTasks (tasks: VideoStudioTask[]) { - await servers[0].videoStudio.createEditionTasks({ videoId: videoUUID, tasks }) - await waitJobs(servers) - } - - before(async function () { - this.timeout(120_000) - - servers = await createMultipleServers(2) - - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - - await doubleFollow(servers[0], servers[1]) - - await servers[0].config.enableMinimumTranscoding() - - await servers[0].config.enableStudio() - }) - - describe('Cutting', function () { - - it('Should cut the beginning of the video', async function () { - this.timeout(120_000) - - await renewVideo() - await waitJobs(servers) - - const beforeTasks = new Date() - - await createTasks([ - { - name: 'cut', - options: { - start: 2 - } - } - ]) - - for (const server of servers) { - await checkVideoDuration(server, videoUUID, 3) - - const video = await server.videos.get({ id: videoUUID }) - expect(new Date(video.publishedAt)).to.be.below(beforeTasks) - } - }) - - it('Should cut the end of the video', async function () { - this.timeout(120_000) - await renewVideo() - - await createTasks([ - { - name: 'cut', - options: { - end: 2 - } - } - ]) - - for (const server of servers) { - await checkVideoDuration(server, videoUUID, 2) - } - }) - - it('Should cut start/end of the video', async function () { - this.timeout(120_000) - await renewVideo('video_short1.webm') // 10 seconds video duration - - await createTasks([ - { - name: 'cut', - options: { - start: 2, - end: 6 - } - } - ]) - - for (const server of servers) { - await checkVideoDuration(server, videoUUID, 4) - } - }) - }) - - describe('Intro/Outro', function () { - - it('Should add an intro', async function () { - this.timeout(120_000) - await renewVideo() - - await createTasks([ - { - name: 'add-intro', - options: { - file: 'video_short.webm' - } - } - ]) - - for (const server of servers) { - await checkVideoDuration(server, videoUUID, 10) - } - }) - - it('Should add an outro', async function () { - this.timeout(120_000) - await renewVideo() - - await createTasks([ - { - name: 'add-outro', - options: { - file: 'video_very_short_240p.mp4' - } - } - ]) - - for (const server of servers) { - await checkVideoDuration(server, videoUUID, 7) - } - }) - - it('Should add an intro/outro', async function () { - this.timeout(120_000) - await renewVideo() - - await createTasks([ - { - name: 'add-intro', - options: { - file: 'video_very_short_240p.mp4' - } - }, - { - name: 'add-outro', - options: { - // Different frame rate - file: 'video_short2.webm' - } - } - ]) - - for (const server of servers) { - await checkVideoDuration(server, videoUUID, 12) - } - }) - - it('Should add an intro to a video without audio', async function () { - this.timeout(120_000) - await renewVideo('video_short_no_audio.mp4') - - await createTasks([ - { - name: 'add-intro', - options: { - file: 'video_very_short_240p.mp4' - } - } - ]) - - for (const server of servers) { - await checkVideoDuration(server, videoUUID, 7) - } - }) - - it('Should add an outro without audio to a video with audio', async function () { - this.timeout(120_000) - await renewVideo() - - await createTasks([ - { - name: 'add-outro', - options: { - file: 'video_short_no_audio.mp4' - } - } - ]) - - for (const server of servers) { - await checkVideoDuration(server, videoUUID, 10) - } - }) - - it('Should add an outro without audio to a video with audio', async function () { - this.timeout(120_000) - await renewVideo('video_short_no_audio.mp4') - - await createTasks([ - { - name: 'add-outro', - options: { - file: 'video_short_no_audio.mp4' - } - } - ]) - - for (const server of servers) { - await checkVideoDuration(server, videoUUID, 10) - } - }) - }) - - describe('Watermark', function () { - - it('Should add a watermark to the video', async function () { - this.timeout(120_000) - await renewVideo() - - const video = await servers[0].videos.get({ id: videoUUID }) - const oldFileUrls = getAllFiles(video).map(f => f.fileUrl) - - await createTasks([ - { - name: 'add-watermark', - options: { - file: 'custom-thumbnail.png' - } - } - ]) - - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - const fileUrls = getAllFiles(video).map(f => f.fileUrl) - - for (const oldUrl of oldFileUrls) { - expect(fileUrls).to.not.include(oldUrl) - } - } - }) - }) - - describe('Complex tasks', function () { - it('Should run a complex task', async function () { - this.timeout(240_000) - await renewVideo() - - await createTasks(VideoStudioCommand.getComplexTask()) - - for (const server of servers) { - await checkVideoDuration(server, videoUUID, 9) - } - }) - }) - - describe('HLS only studio edition', function () { - - before(async function () { - // Disable Web Videos - await servers[0].config.updateExistingSubConfig({ - newConfig: { - transcoding: { - webVideos: { - enabled: false - } - } - } - }) - }) - - it('Should run a complex task on HLS only video', async function () { - this.timeout(240_000) - await renewVideo() - - await createTasks(VideoStudioCommand.getComplexTask()) - - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - expect(video.files).to.have.lengthOf(0) - - await checkVideoDuration(server, videoUUID, 9) - } - }) - }) - - describe('Server restart', function () { - - it('Should still be able to run video edition after a server restart', async function () { - this.timeout(240_000) - - await renewVideo() - await servers[0].videoStudio.createEditionTasks({ videoId: videoUUID, tasks: VideoStudioCommand.getComplexTask() }) - - await servers[0].kill() - await servers[0].run() - - await waitJobs(servers) - - for (const server of servers) { - await checkVideoDuration(server, videoUUID, 9) - } - }) - - it('Should have an empty persistent tmp directory', async function () { - await checkPersistentTmpIsEmpty(servers[0]) - }) - }) - - describe('Object storage studio edition', function () { - if (areMockObjectStorageTestsDisabled()) return - - const objectStorage = new ObjectStorageCommand() - - before(async function () { - await objectStorage.prepareDefaultMockBuckets() - - await servers[0].kill() - await servers[0].run(objectStorage.getDefaultMockConfig()) - - await servers[0].config.enableMinimumTranscoding() - }) - - it('Should run a complex task on a video in object storage', async function () { - this.timeout(240_000) - await renewVideo() - - const video = await servers[0].videos.get({ id: videoUUID }) - const oldFileUrls = getAllFiles(video).map(f => f.fileUrl) - - await createTasks(VideoStudioCommand.getComplexTask()) - - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - const files = getAllFiles(video) - - for (const f of files) { - expect(oldFileUrls).to.not.include(f.fileUrl) - } - - for (const webVideoFile of video.files) { - expectStartWith(webVideoFile.fileUrl, objectStorage.getMockWebVideosBaseUrl()) - } - - for (const hlsFile of video.streamingPlaylists[0].files) { - expectStartWith(hlsFile.fileUrl, objectStorage.getMockPlaylistBaseUrl()) - } - - await checkVideoDuration(server, videoUUID, 9) - } - }) - - after(async function () { - await objectStorage.cleanupMock() - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/users/index.ts b/server/tests/api/users/index.ts deleted file mode 100644 index a4443a8ec..000000000 --- a/server/tests/api/users/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -import './oauth' -import './registrations`' -import './two-factor' -import './user-subscriptions' -import './user-videos' -import './users' -import './users-multiple-servers' -import './users-email-verification' diff --git a/server/tests/api/users/oauth.ts b/server/tests/api/users/oauth.ts deleted file mode 100644 index 153615875..000000000 --- a/server/tests/api/users/oauth.ts +++ /dev/null @@ -1,197 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { SQLCommand } from '@server/tests/shared' -import { wait } from '@shared/core-utils' -import { HttpStatusCode, OAuth2ErrorCode, PeerTubeProblemDocument } from '@shared/models' -import { cleanupTests, createSingleServer, killallServers, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' - -describe('Test oauth', function () { - let server: PeerTubeServer - let sqlCommand: SQLCommand - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1, { - rates_limit: { - login: { - max: 30 - } - } - }) - - await setAccessTokensToServers([ server ]) - - sqlCommand = new SQLCommand(server) - }) - - describe('OAuth client', function () { - - function expectInvalidClient (body: PeerTubeProblemDocument) { - expect(body.code).to.equal(OAuth2ErrorCode.INVALID_CLIENT) - expect(body.error).to.contain('client is invalid') - expect(body.type.startsWith('https://')).to.be.true - expect(body.type).to.contain(OAuth2ErrorCode.INVALID_CLIENT) - } - - it('Should create a new client') - - it('Should return the first client') - - it('Should remove the last client') - - it('Should not login with an invalid client id', async function () { - const client = { id: 'client', secret: server.store.client.secret } - const body = await server.login.login({ client, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - - expectInvalidClient(body) - }) - - it('Should not login with an invalid client secret', async function () { - const client = { id: server.store.client.id, secret: 'coucou' } - const body = await server.login.login({ client, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - - expectInvalidClient(body) - }) - }) - - describe('Login', function () { - - function expectInvalidCredentials (body: PeerTubeProblemDocument) { - expect(body.code).to.equal(OAuth2ErrorCode.INVALID_GRANT) - expect(body.error).to.contain('credentials are invalid') - expect(body.type.startsWith('https://')).to.be.true - expect(body.type).to.contain(OAuth2ErrorCode.INVALID_GRANT) - } - - it('Should not login with an invalid username', async function () { - const user = { username: 'captain crochet', password: server.store.user.password } - const body = await server.login.login({ user, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - - expectInvalidCredentials(body) - }) - - it('Should not login with an invalid password', async function () { - const user = { username: server.store.user.username, password: 'mew_three' } - const body = await server.login.login({ user, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - - expectInvalidCredentials(body) - }) - - it('Should be able to login', async function () { - await server.login.login({ expectedStatus: HttpStatusCode.OK_200 }) - }) - - it('Should be able to login with an insensitive username', async function () { - const user = { username: 'RoOt', password: server.store.user.password } - await server.login.login({ user, expectedStatus: HttpStatusCode.OK_200 }) - - const user2 = { username: 'rOoT', password: server.store.user.password } - await server.login.login({ user: user2, expectedStatus: HttpStatusCode.OK_200 }) - - const user3 = { username: 'ROOt', password: server.store.user.password } - await server.login.login({ user: user3, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - describe('Logout', function () { - - it('Should logout (revoke token)', async function () { - await server.login.logout({ token: server.accessToken }) - }) - - it('Should not be able to get the user information', async function () { - await server.users.getMyInfo({ expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should not be able to upload a video', async function () { - await server.videos.upload({ attributes: { name: 'video' }, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should be able to login again', async function () { - const body = await server.login.login() - server.accessToken = body.access_token - server.refreshToken = body.refresh_token - }) - - it('Should be able to get my user information again', async function () { - await server.users.getMyInfo() - }) - - it('Should have an expired access token', async function () { - this.timeout(60000) - - await sqlCommand.setTokenField(server.accessToken, 'accessTokenExpiresAt', new Date().toISOString()) - await sqlCommand.setTokenField(server.accessToken, 'refreshTokenExpiresAt', new Date().toISOString()) - - await killallServers([ server ]) - await server.run() - - await server.users.getMyInfo({ expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should not be able to refresh an access token with an expired refresh token', async function () { - await server.login.refreshToken({ refreshToken: server.refreshToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should refresh the token', async function () { - this.timeout(50000) - - const futureDate = new Date(new Date().getTime() + 1000 * 60).toISOString() - await sqlCommand.setTokenField(server.accessToken, 'refreshTokenExpiresAt', futureDate) - - await killallServers([ server ]) - await server.run() - - const res = await server.login.refreshToken({ refreshToken: server.refreshToken }) - server.accessToken = res.body.access_token - server.refreshToken = res.body.refresh_token - }) - - it('Should be able to get my user information again', async function () { - await server.users.getMyInfo() - }) - }) - - describe('Custom token lifetime', function () { - before(async function () { - this.timeout(120_000) - - await server.kill() - await server.run({ - oauth2: { - token_lifetime: { - access_token: '2 seconds', - refresh_token: '2 seconds' - } - } - }) - }) - - it('Should have a very short access token lifetime', async function () { - this.timeout(50000) - - const { access_token: accessToken } = await server.login.login() - await server.users.getMyInfo({ token: accessToken }) - - await wait(3000) - await server.users.getMyInfo({ token: accessToken, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should have a very short refresh token lifetime', async function () { - this.timeout(50000) - - const { refresh_token: refreshToken } = await server.login.login() - await server.login.refreshToken({ refreshToken }) - - await wait(3000) - await server.login.refreshToken({ refreshToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - }) - - after(async function () { - await sqlCommand.cleanup() - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/users/registrations.ts b/server/tests/api/users/registrations.ts deleted file mode 100644 index e6524f07d..000000000 --- a/server/tests/api/users/registrations.ts +++ /dev/null @@ -1,415 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { MockSmtpServer } from '@server/tests/shared' -import { UserRegistrationState, UserRole } from '@shared/models' -import { - cleanupTests, - ConfigCommand, - createSingleServer, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test registrations', function () { - let server: PeerTubeServer - - const emails: object[] = [] - let emailPort: number - - before(async function () { - this.timeout(30000) - - emailPort = await MockSmtpServer.Instance.collectEmails(emails) - - server = await createSingleServer(1, ConfigCommand.getEmailOverrideConfig(emailPort)) - - await setAccessTokensToServers([ server ]) - await server.config.enableSignup(false) - }) - - describe('Direct registrations of a new user', function () { - let user1Token: string - - it('Should register a new user', async function () { - const user = { displayName: 'super user 1', username: 'user_1', password: 'my super password' } - const channel = { name: 'my_user_1_channel', displayName: 'my channel rocks' } - - await server.registrations.register({ ...user, channel }) - }) - - it('Should be able to login with this registered user', async function () { - const user1 = { username: 'user_1', password: 'my super password' } - - user1Token = await server.login.getAccessToken(user1) - }) - - it('Should have the correct display name', async function () { - const user = await server.users.getMyInfo({ token: user1Token }) - expect(user.account.displayName).to.equal('super user 1') - }) - - it('Should have the correct video quota', async function () { - const user = await server.users.getMyInfo({ token: user1Token }) - expect(user.videoQuota).to.equal(5 * 1024 * 1024) - }) - - it('Should have created the channel', async function () { - const { displayName } = await server.channels.get({ channelName: 'my_user_1_channel' }) - - expect(displayName).to.equal('my channel rocks') - }) - - it('Should remove me', async function () { - { - const { data } = await server.users.list() - expect(data.find(u => u.username === 'user_1')).to.not.be.undefined - } - - await server.users.deleteMe({ token: user1Token }) - - { - const { data } = await server.users.list() - expect(data.find(u => u.username === 'user_1')).to.be.undefined - } - }) - }) - - describe('Registration requests', function () { - let id2: number - let id3: number - let id4: number - - let user2Token: string - let user3Token: string - - before(async function () { - this.timeout(60000) - - await server.config.enableSignup(true) - - { - const { id } = await server.registrations.requestRegistration({ - username: 'user4', - registrationReason: 'registration reason 4' - }) - - id4 = id - } - }) - - it('Should request a registration without a channel', async function () { - { - const { id } = await server.registrations.requestRegistration({ - username: 'user2', - displayName: 'my super user 2', - email: 'user2@example.com', - password: 'user2password', - registrationReason: 'registration reason 2' - }) - - id2 = id - } - }) - - it('Should request a registration with a channel', async function () { - const { id } = await server.registrations.requestRegistration({ - username: 'user3', - displayName: 'my super user 3', - channel: { - displayName: 'my user 3 channel', - name: 'super_user3_channel' - }, - email: 'user3@example.com', - password: 'user3password', - registrationReason: 'registration reason 3' - }) - - id3 = id - }) - - it('Should list these registration requests', async function () { - { - const { total, data } = await server.registrations.list({ sort: '-createdAt' }) - expect(total).to.equal(3) - expect(data).to.have.lengthOf(3) - - { - expect(data[0].id).to.equal(id3) - expect(data[0].username).to.equal('user3') - expect(data[0].accountDisplayName).to.equal('my super user 3') - - expect(data[0].channelDisplayName).to.equal('my user 3 channel') - expect(data[0].channelHandle).to.equal('super_user3_channel') - - expect(data[0].createdAt).to.exist - expect(data[0].updatedAt).to.exist - - expect(data[0].email).to.equal('user3@example.com') - expect(data[0].emailVerified).to.be.null - - expect(data[0].moderationResponse).to.be.null - expect(data[0].registrationReason).to.equal('registration reason 3') - expect(data[0].state.id).to.equal(UserRegistrationState.PENDING) - expect(data[0].state.label).to.equal('Pending') - expect(data[0].user).to.be.null - } - - { - expect(data[1].id).to.equal(id2) - expect(data[1].username).to.equal('user2') - expect(data[1].accountDisplayName).to.equal('my super user 2') - - expect(data[1].channelDisplayName).to.be.null - expect(data[1].channelHandle).to.be.null - - expect(data[1].createdAt).to.exist - expect(data[1].updatedAt).to.exist - - expect(data[1].email).to.equal('user2@example.com') - expect(data[1].emailVerified).to.be.null - - expect(data[1].moderationResponse).to.be.null - expect(data[1].registrationReason).to.equal('registration reason 2') - expect(data[1].state.id).to.equal(UserRegistrationState.PENDING) - expect(data[1].state.label).to.equal('Pending') - expect(data[1].user).to.be.null - } - - { - expect(data[2].username).to.equal('user4') - } - } - - { - const { total, data } = await server.registrations.list({ count: 1, start: 1, sort: 'createdAt' }) - - expect(total).to.equal(3) - expect(data).to.have.lengthOf(1) - expect(data[0].id).to.equal(id2) - } - - { - const { total, data } = await server.registrations.list({ search: 'user3' }) - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - expect(data[0].id).to.equal(id3) - } - }) - - it('Should reject a registration request', async function () { - await server.registrations.reject({ id: id4, moderationResponse: 'I do not want id 4 on this instance' }) - }) - - it('Should have sent an email to the user explanining the registration has been rejected', async function () { - this.timeout(50000) - - await waitJobs([ server ]) - - const email = emails.find(e => e['to'][0]['address'] === 'user4@example.com') - expect(email).to.exist - - expect(email['subject']).to.contain('been rejected') - expect(email['text']).to.contain('been rejected') - expect(email['text']).to.contain('I do not want id 4 on this instance') - }) - - it('Should accept registration requests', async function () { - await server.registrations.accept({ id: id2, moderationResponse: 'Welcome id 2' }) - await server.registrations.accept({ id: id3, moderationResponse: 'Welcome id 3' }) - }) - - it('Should have sent an email to the user explanining the registration has been accepted', async function () { - this.timeout(50000) - - await waitJobs([ server ]) - - { - const email = emails.find(e => e['to'][0]['address'] === 'user2@example.com') - expect(email).to.exist - - expect(email['subject']).to.contain('been accepted') - expect(email['text']).to.contain('been accepted') - expect(email['text']).to.contain('Welcome id 2') - } - - { - const email = emails.find(e => e['to'][0]['address'] === 'user3@example.com') - expect(email).to.exist - - expect(email['subject']).to.contain('been accepted') - expect(email['text']).to.contain('been accepted') - expect(email['text']).to.contain('Welcome id 3') - } - }) - - it('Should login with these users', async function () { - user2Token = await server.login.getAccessToken({ username: 'user2', password: 'user2password' }) - user3Token = await server.login.getAccessToken({ username: 'user3', password: 'user3password' }) - }) - - it('Should have created the appropriate attributes for user 2', async function () { - const me = await server.users.getMyInfo({ token: user2Token }) - - expect(me.username).to.equal('user2') - expect(me.account.displayName).to.equal('my super user 2') - expect(me.videoQuota).to.equal(5 * 1024 * 1024) - expect(me.videoChannels[0].name).to.equal('user2_channel') - expect(me.videoChannels[0].displayName).to.equal('Main user2 channel') - expect(me.role.id).to.equal(UserRole.USER) - expect(me.email).to.equal('user2@example.com') - }) - - it('Should have created the appropriate attributes for user 3', async function () { - const me = await server.users.getMyInfo({ token: user3Token }) - - expect(me.username).to.equal('user3') - expect(me.account.displayName).to.equal('my super user 3') - expect(me.videoQuota).to.equal(5 * 1024 * 1024) - expect(me.videoChannels[0].name).to.equal('super_user3_channel') - expect(me.videoChannels[0].displayName).to.equal('my user 3 channel') - expect(me.role.id).to.equal(UserRole.USER) - expect(me.email).to.equal('user3@example.com') - }) - - it('Should list these accepted/rejected registration requests', async function () { - const { data } = await server.registrations.list({ sort: 'createdAt' }) - const { data: users } = await server.users.list() - - { - expect(data[0].id).to.equal(id4) - expect(data[0].state.id).to.equal(UserRegistrationState.REJECTED) - expect(data[0].state.label).to.equal('Rejected') - - expect(data[0].moderationResponse).to.equal('I do not want id 4 on this instance') - expect(data[0].user).to.be.null - - expect(users.find(u => u.username === 'user4')).to.not.exist - } - - { - expect(data[1].id).to.equal(id2) - expect(data[1].state.id).to.equal(UserRegistrationState.ACCEPTED) - expect(data[1].state.label).to.equal('Accepted') - - expect(data[1].moderationResponse).to.equal('Welcome id 2') - expect(data[1].user).to.exist - - const user2 = users.find(u => u.username === 'user2') - expect(data[1].user.id).to.equal(user2.id) - } - - { - expect(data[2].id).to.equal(id3) - expect(data[2].state.id).to.equal(UserRegistrationState.ACCEPTED) - expect(data[2].state.label).to.equal('Accepted') - - expect(data[2].moderationResponse).to.equal('Welcome id 3') - expect(data[2].user).to.exist - - const user3 = users.find(u => u.username === 'user3') - expect(data[2].user.id).to.equal(user3.id) - } - }) - - it('Shoulde delete a registration', async function () { - await server.registrations.delete({ id: id2 }) - await server.registrations.delete({ id: id3 }) - - const { total, data } = await server.registrations.list() - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - expect(data[0].id).to.equal(id4) - - const { data: users } = await server.users.list() - - for (const username of [ 'user2', 'user3' ]) { - expect(users.find(u => u.username === username)).to.exist - } - }) - - it('Should be able to prevent email delivery on accept/reject', async function () { - this.timeout(50000) - - let id1: number - let id2: number - - { - const { id } = await server.registrations.requestRegistration({ - username: 'user7', - email: 'user7@example.com', - registrationReason: 'tt' - }) - id1 = id - } - { - const { id } = await server.registrations.requestRegistration({ - username: 'user8', - email: 'user8@example.com', - registrationReason: 'tt' - }) - id2 = id - } - - await server.registrations.accept({ id: id1, moderationResponse: 'tt', preventEmailDelivery: true }) - await server.registrations.reject({ id: id2, moderationResponse: 'tt', preventEmailDelivery: true }) - - await waitJobs([ server ]) - - const filtered = emails.filter(e => { - const address = e['to'][0]['address'] - return address === 'user7@example.com' || address === 'user8@example.com' - }) - - expect(filtered).to.have.lengthOf(0) - }) - - it('Should request a registration without a channel, that will conflict with an already existing channel', async function () { - let id1: number - let id2: number - - { - const { id } = await server.registrations.requestRegistration({ - registrationReason: 'tt', - username: 'user5', - password: 'user5password', - channel: { - displayName: 'channel 6', - name: 'user6_channel' - } - }) - - id1 = id - } - - { - const { id } = await server.registrations.requestRegistration({ - registrationReason: 'tt', - username: 'user6', - password: 'user6password' - }) - - id2 = id - } - - await server.registrations.accept({ id: id1, moderationResponse: 'tt' }) - await server.registrations.accept({ id: id2, moderationResponse: 'tt' }) - - const user5Token = await server.login.getAccessToken('user5', 'user5password') - const user6Token = await server.login.getAccessToken('user6', 'user6password') - - const user5 = await server.users.getMyInfo({ token: user5Token }) - const user6 = await server.users.getMyInfo({ token: user6Token }) - - expect(user5.videoChannels[0].name).to.equal('user6_channel') - expect(user6.videoChannels[0].name).to.equal('user6_channel-1') - }) - }) - - after(async function () { - MockSmtpServer.Instance.kill() - - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/users/two-factor.ts b/server/tests/api/users/two-factor.ts deleted file mode 100644 index 0dcab9e17..000000000 --- a/server/tests/api/users/two-factor.ts +++ /dev/null @@ -1,200 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { expectStartWith } from '@server/tests/shared' -import { HttpStatusCode } from '@shared/models' -import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers, TwoFactorCommand } from '@shared/server-commands' - -async function login (options: { - server: PeerTubeServer - username: string - password: string - otpToken?: string - expectedStatus?: HttpStatusCode -}) { - const { server, username, password, otpToken, expectedStatus } = options - - const user = { username, password } - const { res, body: { access_token: token } } = await server.login.loginAndGetResponse({ user, otpToken, expectedStatus }) - - return { res, token } -} - -describe('Test users', function () { - let server: PeerTubeServer - let otpSecret: string - let requestToken: string - - const userUsername = 'user1' - let userId: number - let userPassword: string - let userToken: string - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - const res = await server.users.generate(userUsername) - userId = res.userId - userPassword = res.password - userToken = res.token - }) - - it('Should not add the header on login if two factor is not enabled', async function () { - const { res, token } = await login({ server, username: userUsername, password: userPassword }) - - expect(res.header['x-peertube-otp']).to.not.exist - - await server.users.getMyInfo({ token }) - }) - - it('Should request two factor and get the secret and uri', async function () { - const { otpRequest } = await server.twoFactor.request({ userId, token: userToken, currentPassword: userPassword }) - - expect(otpRequest.requestToken).to.exist - - expect(otpRequest.secret).to.exist - expect(otpRequest.secret).to.have.lengthOf(32) - - expect(otpRequest.uri).to.exist - expectStartWith(otpRequest.uri, 'otpauth://') - expect(otpRequest.uri).to.include(otpRequest.secret) - - requestToken = otpRequest.requestToken - otpSecret = otpRequest.secret - }) - - it('Should not have two factor confirmed yet', async function () { - const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken }) - expect(twoFactorEnabled).to.be.false - }) - - it('Should confirm two factor', async function () { - await server.twoFactor.confirmRequest({ - userId, - token: userToken, - otpToken: TwoFactorCommand.buildOTP({ secret: otpSecret }).generate(), - requestToken - }) - }) - - it('Should not add the header on login if two factor is enabled and password is incorrect', async function () { - const { res, token } = await login({ server, username: userUsername, password: 'fake', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - - expect(res.header['x-peertube-otp']).to.not.exist - expect(token).to.not.exist - }) - - it('Should add the header on login if two factor is enabled and password is correct', async function () { - const { res, token } = await login({ - server, - username: userUsername, - password: userPassword, - expectedStatus: HttpStatusCode.UNAUTHORIZED_401 - }) - - expect(res.header['x-peertube-otp']).to.exist - expect(token).to.not.exist - - await server.users.getMyInfo({ token }) - }) - - it('Should not login with correct password and incorrect otp secret', async function () { - const otp = TwoFactorCommand.buildOTP({ secret: 'a'.repeat(32) }) - - const { res, token } = await login({ - server, - username: userUsername, - password: userPassword, - otpToken: otp.generate(), - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - - expect(res.header['x-peertube-otp']).to.not.exist - expect(token).to.not.exist - }) - - it('Should not login with correct password and incorrect otp code', async function () { - const { res, token } = await login({ - server, - username: userUsername, - password: userPassword, - otpToken: '123456', - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - - expect(res.header['x-peertube-otp']).to.not.exist - expect(token).to.not.exist - }) - - it('Should not login with incorrect password and correct otp code', async function () { - const otpToken = TwoFactorCommand.buildOTP({ secret: otpSecret }).generate() - - const { res, token } = await login({ - server, - username: userUsername, - password: 'fake', - otpToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - - expect(res.header['x-peertube-otp']).to.not.exist - expect(token).to.not.exist - }) - - it('Should correctly login with correct password and otp code', async function () { - const otpToken = TwoFactorCommand.buildOTP({ secret: otpSecret }).generate() - - const { res, token } = await login({ server, username: userUsername, password: userPassword, otpToken }) - - expect(res.header['x-peertube-otp']).to.not.exist - expect(token).to.exist - - await server.users.getMyInfo({ token }) - }) - - it('Should have two factor enabled when getting my info', async function () { - const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken }) - expect(twoFactorEnabled).to.be.true - }) - - it('Should disable two factor and be able to login without otp token', async function () { - await server.twoFactor.disable({ userId, token: userToken, currentPassword: userPassword }) - - const { res, token } = await login({ server, username: userUsername, password: userPassword }) - expect(res.header['x-peertube-otp']).to.not.exist - - await server.users.getMyInfo({ token }) - }) - - it('Should have two factor disabled when getting my info', async function () { - const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken }) - expect(twoFactorEnabled).to.be.false - }) - - it('Should enable two factor auth without password from an admin', async function () { - const { otpRequest } = await server.twoFactor.request({ userId }) - - await server.twoFactor.confirmRequest({ - userId, - otpToken: TwoFactorCommand.buildOTP({ secret: otpRequest.secret }).generate(), - requestToken: otpRequest.requestToken - }) - - const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken }) - expect(twoFactorEnabled).to.be.true - }) - - it('Should disable two factor auth without password from an admin', async function () { - await server.twoFactor.disable({ userId }) - - const { twoFactorEnabled } = await server.users.getMyInfo({ token: userToken }) - expect(twoFactorEnabled).to.be.false - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/users/user-subscriptions.ts b/server/tests/api/users/user-subscriptions.ts deleted file mode 100644 index ad2b82a4a..000000000 --- a/server/tests/api/users/user-subscriptions.ts +++ /dev/null @@ -1,614 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - setDefaultAccountAvatar, - setDefaultChannelAvatar, - SubscriptionsCommand, - waitJobs -} from '@shared/server-commands' - -describe('Test users subscriptions', function () { - let servers: PeerTubeServer[] = [] - const users: { accessToken: string }[] = [] - let video3UUID: string - - let command: SubscriptionsCommand - - before(async function () { - this.timeout(240000) - - servers = await createMultipleServers(3) - - // Get the access tokens - await setAccessTokensToServers(servers) - await setDefaultChannelAvatar(servers) - await setDefaultAccountAvatar(servers) - - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - - for (const server of servers) { - const user = { username: 'user' + server.serverNumber, password: 'password' } - await server.users.create({ username: user.username, password: user.password }) - - const accessToken = await server.login.getAccessToken(user) - users.push({ accessToken }) - - const videoName1 = 'video 1-' + server.serverNumber - await server.videos.upload({ token: accessToken, attributes: { name: videoName1 } }) - - const videoName2 = 'video 2-' + server.serverNumber - await server.videos.upload({ token: accessToken, attributes: { name: videoName2 } }) - } - - await waitJobs(servers) - - command = servers[0].subscriptions - }) - - describe('Destinction between server videos and user videos', function () { - it('Should display videos of server 2 on server 1', async function () { - const { total } = await servers[0].videos.list() - - expect(total).to.equal(4) - }) - - it('User of server 1 should follow user of server 3 and root of server 1', async function () { - this.timeout(60000) - - await command.add({ token: users[0].accessToken, targetUri: 'user3_channel@' + servers[2].host }) - await command.add({ token: users[0].accessToken, targetUri: 'root_channel@' + servers[0].host }) - - await waitJobs(servers) - - const attributes = { name: 'video server 3 added after follow' } - const { uuid } = await servers[2].videos.upload({ token: users[2].accessToken, attributes }) - video3UUID = uuid - - await waitJobs(servers) - }) - - it('Should not display videos of server 3 on server 1', async function () { - const { total, data } = await servers[0].videos.list() - expect(total).to.equal(4) - - for (const video of data) { - expect(video.name).to.not.contain('1-3') - expect(video.name).to.not.contain('2-3') - expect(video.name).to.not.contain('video server 3 added after follow') - } - }) - }) - - describe('Subscription endpoints', function () { - - it('Should list subscriptions', async function () { - { - const body = await command.list() - expect(body.total).to.equal(0) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(0) - } - - { - const body = await command.list({ token: users[0].accessToken, sort: 'createdAt' }) - expect(body.total).to.equal(2) - - const subscriptions = body.data - expect(subscriptions).to.be.an('array') - expect(subscriptions).to.have.lengthOf(2) - - expect(subscriptions[0].name).to.equal('user3_channel') - expect(subscriptions[1].name).to.equal('root_channel') - } - }) - - it('Should get subscription', async function () { - { - const videoChannel = await command.get({ token: users[0].accessToken, uri: 'user3_channel@' + servers[2].host }) - - expect(videoChannel.name).to.equal('user3_channel') - expect(videoChannel.host).to.equal(servers[2].host) - expect(videoChannel.displayName).to.equal('Main user3 channel') - expect(videoChannel.followingCount).to.equal(0) - expect(videoChannel.followersCount).to.equal(1) - } - - { - const videoChannel = await command.get({ token: users[0].accessToken, uri: 'root_channel@' + servers[0].host }) - - expect(videoChannel.name).to.equal('root_channel') - expect(videoChannel.host).to.equal(servers[0].host) - expect(videoChannel.displayName).to.equal('Main root channel') - expect(videoChannel.followingCount).to.equal(0) - expect(videoChannel.followersCount).to.equal(1) - } - }) - - it('Should return the existing subscriptions', async function () { - const uris = [ - 'user3_channel@' + servers[2].host, - 'root2_channel@' + servers[0].host, - 'root_channel@' + servers[0].host, - 'user3_channel@' + servers[0].host - ] - - const body = await command.exist({ token: users[0].accessToken, uris }) - - expect(body['user3_channel@' + servers[2].host]).to.be.true - expect(body['root2_channel@' + servers[0].host]).to.be.false - expect(body['root_channel@' + servers[0].host]).to.be.true - expect(body['user3_channel@' + servers[0].host]).to.be.false - }) - - it('Should search among subscriptions', async function () { - { - const body = await command.list({ token: users[0].accessToken, sort: '-createdAt', search: 'user3_channel' }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - } - - { - const body = await command.list({ token: users[0].accessToken, sort: '-createdAt', search: 'toto' }) - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - } - }) - }) - - describe('Subscription videos', function () { - - it('Should list subscription videos', async function () { - { - const body = await servers[0].videos.listMySubscriptionVideos() - expect(body.total).to.equal(0) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(0) - } - - { - const body = await servers[0].videos.listMySubscriptionVideos({ token: users[0].accessToken, sort: 'createdAt' }) - expect(body.total).to.equal(3) - - const videos = body.data - expect(videos).to.be.an('array') - expect(videos).to.have.lengthOf(3) - - expect(videos[0].name).to.equal('video 1-3') - expect(videos[1].name).to.equal('video 2-3') - expect(videos[2].name).to.equal('video server 3 added after follow') - } - - { - const body = await servers[0].videos.listMySubscriptionVideos({ token: users[0].accessToken, count: 1, start: 1 }) - expect(body.total).to.equal(3) - - const videos = body.data - expect(videos).to.be.an('array') - expect(videos).to.have.lengthOf(1) - - expect(videos[0].name).to.equal('video 2-3') - } - }) - - it('Should upload a video by root on server 1 and see it in the subscription videos', async function () { - this.timeout(60000) - - const videoName = 'video server 1 added after follow' - await servers[0].videos.upload({ attributes: { name: videoName } }) - - await waitJobs(servers) - - { - const body = await servers[0].videos.listMySubscriptionVideos() - expect(body.total).to.equal(0) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(0) - } - - { - const body = await servers[0].videos.listMySubscriptionVideos({ token: users[0].accessToken, sort: 'createdAt' }) - expect(body.total).to.equal(4) - - const videos = body.data - expect(videos).to.be.an('array') - expect(videos).to.have.lengthOf(4) - - expect(videos[0].name).to.equal('video 1-3') - expect(videos[1].name).to.equal('video 2-3') - expect(videos[2].name).to.equal('video server 3 added after follow') - expect(videos[3].name).to.equal('video server 1 added after follow') - } - - { - const { data, total } = await servers[0].videos.list() - expect(total).to.equal(5) - - for (const video of data) { - expect(video.name).to.not.contain('1-3') - expect(video.name).to.not.contain('2-3') - expect(video.name).to.not.contain('video server 3 added after follow') - } - } - }) - - it('Should have server 1 following server 3 and display server 3 videos', async function () { - this.timeout(60000) - - await servers[0].follows.follow({ hosts: [ servers[2].url ] }) - - await waitJobs(servers) - - const { data, total } = await servers[0].videos.list() - expect(total).to.equal(8) - - const names = [ '1-3', '2-3', 'video server 3 added after follow' ] - for (const name of names) { - const video = data.find(v => v.name.includes(name)) - expect(video).to.not.be.undefined - } - }) - - it('Should remove follow server 1 -> server 3 and hide server 3 videos', async function () { - this.timeout(60000) - - await servers[0].follows.unfollow({ target: servers[2] }) - - await waitJobs(servers) - - const { total, data } = await servers[0].videos.list() - expect(total).to.equal(5) - - for (const video of data) { - expect(video.name).to.not.contain('1-3') - expect(video.name).to.not.contain('2-3') - expect(video.name).to.not.contain('video server 3 added after follow') - } - }) - - it('Should still list subscription videos', async function () { - { - const body = await servers[0].videos.listMySubscriptionVideos() - expect(body.total).to.equal(0) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(0) - } - - { - const body = await servers[0].videos.listMySubscriptionVideos({ token: users[0].accessToken, sort: 'createdAt' }) - expect(body.total).to.equal(4) - - const videos = body.data - expect(videos).to.be.an('array') - expect(videos).to.have.lengthOf(4) - - expect(videos[0].name).to.equal('video 1-3') - expect(videos[1].name).to.equal('video 2-3') - expect(videos[2].name).to.equal('video server 3 added after follow') - expect(videos[3].name).to.equal('video server 1 added after follow') - } - }) - }) - - describe('Existing subscription video update', function () { - - it('Should update a video of server 3 and see the updated video on server 1', async function () { - this.timeout(30000) - - await servers[2].videos.update({ id: video3UUID, attributes: { name: 'video server 3 added after follow updated' } }) - - await waitJobs(servers) - - const body = await servers[0].videos.listMySubscriptionVideos({ token: users[0].accessToken, sort: 'createdAt' }) - expect(body.data[2].name).to.equal('video server 3 added after follow updated') - }) - }) - - describe('Subscription removal', function () { - - it('Should remove user of server 3 subscription', async function () { - this.timeout(30000) - - await command.remove({ token: users[0].accessToken, uri: 'user3_channel@' + servers[2].host }) - - await waitJobs(servers) - }) - - it('Should not display its videos anymore', async function () { - const body = await servers[0].videos.listMySubscriptionVideos({ token: users[0].accessToken, sort: 'createdAt' }) - expect(body.total).to.equal(1) - - const videos = body.data - expect(videos).to.be.an('array') - expect(videos).to.have.lengthOf(1) - - expect(videos[0].name).to.equal('video server 1 added after follow') - }) - - it('Should remove the root subscription and not display the videos anymore', async function () { - this.timeout(30000) - - await command.remove({ token: users[0].accessToken, uri: 'root_channel@' + servers[0].host }) - - await waitJobs(servers) - - { - const body = await command.list({ token: users[0].accessToken, sort: 'createdAt' }) - expect(body.total).to.equal(0) - - const videos = body.data - expect(videos).to.be.an('array') - expect(videos).to.have.lengthOf(0) - } - }) - - it('Should correctly display public videos on server 1', async function () { - const { total, data } = await servers[0].videos.list() - expect(total).to.equal(5) - - for (const video of data) { - expect(video.name).to.not.contain('1-3') - expect(video.name).to.not.contain('2-3') - expect(video.name).to.not.contain('video server 3 added after follow updated') - } - }) - }) - - describe('Re-follow', function () { - - it('Should follow user of server 3 again', async function () { - this.timeout(60000) - - await command.add({ token: users[0].accessToken, targetUri: 'user3_channel@' + servers[2].host }) - - await waitJobs(servers) - - { - const body = await servers[0].videos.listMySubscriptionVideos({ token: users[0].accessToken, sort: 'createdAt' }) - expect(body.total).to.equal(3) - - const videos = body.data - expect(videos).to.be.an('array') - expect(videos).to.have.lengthOf(3) - - expect(videos[0].name).to.equal('video 1-3') - expect(videos[1].name).to.equal('video 2-3') - expect(videos[2].name).to.equal('video server 3 added after follow updated') - } - - { - const { total, data } = await servers[0].videos.list() - expect(total).to.equal(5) - - for (const video of data) { - expect(video.name).to.not.contain('1-3') - expect(video.name).to.not.contain('2-3') - expect(video.name).to.not.contain('video server 3 added after follow updated') - } - } - }) - - it('Should follow user channels of server 3 by root of server 3', async function () { - this.timeout(60000) - - await servers[2].channels.create({ token: users[2].accessToken, attributes: { name: 'user3_channel2' } }) - - await servers[2].subscriptions.add({ token: servers[2].accessToken, targetUri: 'user3_channel@' + servers[2].host }) - await servers[2].subscriptions.add({ token: servers[2].accessToken, targetUri: 'user3_channel2@' + servers[2].host }) - - await waitJobs(servers) - }) - }) - - describe('Followers listing', function () { - - it('Should list user 3 followers', async function () { - { - const { total, data } = await servers[2].accounts.listFollowers({ - token: users[2].accessToken, - accountName: 'user3', - start: 0, - count: 5, - sort: 'createdAt' - }) - - expect(total).to.equal(3) - expect(data).to.have.lengthOf(3) - - expect(data[0].following.host).to.equal(servers[2].host) - expect(data[0].following.name).to.equal('user3_channel') - expect(data[0].follower.host).to.equal(servers[0].host) - expect(data[0].follower.name).to.equal('user1') - - expect(data[1].following.host).to.equal(servers[2].host) - expect(data[1].following.name).to.equal('user3_channel') - expect(data[1].follower.host).to.equal(servers[2].host) - expect(data[1].follower.name).to.equal('root') - - expect(data[2].following.host).to.equal(servers[2].host) - expect(data[2].following.name).to.equal('user3_channel2') - expect(data[2].follower.host).to.equal(servers[2].host) - expect(data[2].follower.name).to.equal('root') - } - - { - const { total, data } = await servers[2].accounts.listFollowers({ - token: users[2].accessToken, - accountName: 'user3', - start: 0, - count: 1, - sort: '-createdAt' - }) - - expect(total).to.equal(3) - expect(data).to.have.lengthOf(1) - - expect(data[0].following.host).to.equal(servers[2].host) - expect(data[0].following.name).to.equal('user3_channel2') - expect(data[0].follower.host).to.equal(servers[2].host) - expect(data[0].follower.name).to.equal('root') - } - - { - const { total, data } = await servers[2].accounts.listFollowers({ - token: users[2].accessToken, - accountName: 'user3', - start: 1, - count: 1, - sort: '-createdAt' - }) - - expect(total).to.equal(3) - expect(data).to.have.lengthOf(1) - - expect(data[0].following.host).to.equal(servers[2].host) - expect(data[0].following.name).to.equal('user3_channel') - expect(data[0].follower.host).to.equal(servers[2].host) - expect(data[0].follower.name).to.equal('root') - } - - { - const { total, data } = await servers[2].accounts.listFollowers({ - token: users[2].accessToken, - accountName: 'user3', - search: 'user1', - sort: '-createdAt' - }) - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - - expect(data[0].following.host).to.equal(servers[2].host) - expect(data[0].following.name).to.equal('user3_channel') - expect(data[0].follower.host).to.equal(servers[0].host) - expect(data[0].follower.name).to.equal('user1') - } - }) - - it('Should list user3_channel followers', async function () { - { - const { total, data } = await servers[2].channels.listFollowers({ - token: users[2].accessToken, - channelName: 'user3_channel', - start: 0, - count: 5, - sort: 'createdAt' - }) - - expect(total).to.equal(2) - expect(data).to.have.lengthOf(2) - - expect(data[0].following.host).to.equal(servers[2].host) - expect(data[0].following.name).to.equal('user3_channel') - expect(data[0].follower.host).to.equal(servers[0].host) - expect(data[0].follower.name).to.equal('user1') - - expect(data[1].following.host).to.equal(servers[2].host) - expect(data[1].following.name).to.equal('user3_channel') - expect(data[1].follower.host).to.equal(servers[2].host) - expect(data[1].follower.name).to.equal('root') - } - - { - const { total, data } = await servers[2].channels.listFollowers({ - token: users[2].accessToken, - channelName: 'user3_channel', - start: 0, - count: 1, - sort: '-createdAt' - }) - - expect(total).to.equal(2) - expect(data).to.have.lengthOf(1) - - expect(data[0].following.host).to.equal(servers[2].host) - expect(data[0].following.name).to.equal('user3_channel') - expect(data[0].follower.host).to.equal(servers[2].host) - expect(data[0].follower.name).to.equal('root') - } - - { - const { total, data } = await servers[2].channels.listFollowers({ - token: users[2].accessToken, - channelName: 'user3_channel', - start: 1, - count: 1, - sort: '-createdAt' - }) - - expect(total).to.equal(2) - expect(data).to.have.lengthOf(1) - - expect(data[0].following.host).to.equal(servers[2].host) - expect(data[0].following.name).to.equal('user3_channel') - expect(data[0].follower.host).to.equal(servers[0].host) - expect(data[0].follower.name).to.equal('user1') - } - - { - const { total, data } = await servers[2].channels.listFollowers({ - token: users[2].accessToken, - channelName: 'user3_channel', - search: 'user1', - sort: '-createdAt' - }) - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - - expect(data[0].following.host).to.equal(servers[2].host) - expect(data[0].following.name).to.equal('user3_channel') - expect(data[0].follower.host).to.equal(servers[0].host) - expect(data[0].follower.name).to.equal('user1') - } - }) - }) - - describe('Subscription videos privacy', function () { - - it('Should update video as internal and not see from remote server', async function () { - this.timeout(30000) - - await servers[2].videos.update({ id: video3UUID, attributes: { name: 'internal', privacy: VideoPrivacy.INTERNAL } }) - await waitJobs(servers) - - { - const { data } = await servers[0].videos.listMySubscriptionVideos({ token: users[0].accessToken }) - expect(data.find(v => v.name === 'internal')).to.not.exist - } - }) - - it('Should see internal from local user', async function () { - const { data } = await servers[2].videos.listMySubscriptionVideos({ token: servers[2].accessToken }) - expect(data.find(v => v.name === 'internal')).to.exist - }) - - it('Should update video as private and not see from anyone server', async function () { - this.timeout(30000) - - await servers[2].videos.update({ id: video3UUID, attributes: { name: 'private', privacy: VideoPrivacy.PRIVATE } }) - await waitJobs(servers) - - { - const { data } = await servers[0].videos.listMySubscriptionVideos({ token: users[0].accessToken }) - expect(data.find(v => v.name === 'private')).to.not.exist - } - - { - const { data } = await servers[2].videos.listMySubscriptionVideos({ token: servers[2].accessToken }) - expect(data.find(v => v.name === 'private')).to.not.exist - } - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/users/user-videos.ts b/server/tests/api/users/user-videos.ts deleted file mode 100644 index 77226e48e..000000000 --- a/server/tests/api/users/user-videos.ts +++ /dev/null @@ -1,219 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { HttpStatusCode } from '@shared/models' -import { - cleanupTests, - createSingleServer, - PeerTubeServer, - setAccessTokensToServers, - setDefaultAccountAvatar, - setDefaultChannelAvatar, - waitJobs -} from '@shared/server-commands' - -describe('Test user videos', function () { - let server: PeerTubeServer - let videoId: number - let videoId2: number - let token: string - let anotherUserToken: string - - before(async function () { - this.timeout(120000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - await setDefaultChannelAvatar([ server ]) - await setDefaultAccountAvatar([ server ]) - - await server.videos.quickUpload({ name: 'root video' }) - await server.videos.quickUpload({ name: 'root video 2' }) - - token = await server.users.generateUserAndToken('user') - anotherUserToken = await server.users.generateUserAndToken('user2') - }) - - describe('List my videos', function () { - - it('Should list my videos', async function () { - const { data, total } = await server.videos.listMyVideos() - - expect(total).to.equal(2) - expect(data).to.have.lengthOf(2) - }) - }) - - describe('Upload', function () { - - it('Should upload the video with the correct token', async function () { - await server.videos.upload({ token }) - const { data } = await server.videos.list() - const video = data[0] - - expect(video.account.name).to.equal('user') - videoId = video.id - }) - - it('Should upload the video again with the correct token', async function () { - const { id } = await server.videos.upload({ token }) - videoId2 = id - }) - }) - - describe('Ratings', function () { - - it('Should retrieve a video rating', async function () { - await server.videos.rate({ id: videoId, token, rating: 'like' }) - const rating = await server.users.getMyRating({ token, videoId }) - - expect(rating.videoId).to.equal(videoId) - expect(rating.rating).to.equal('like') - }) - - it('Should retrieve ratings list', async function () { - await server.videos.rate({ id: videoId, token, rating: 'like' }) - - const body = await server.accounts.listRatings({ accountName: 'user', token }) - - expect(body.total).to.equal(1) - expect(body.data[0].video.id).to.equal(videoId) - expect(body.data[0].rating).to.equal('like') - }) - - it('Should retrieve ratings list by rating type', async function () { - { - const body = await server.accounts.listRatings({ accountName: 'user', token, rating: 'like' }) - expect(body.data.length).to.equal(1) - } - - { - const body = await server.accounts.listRatings({ accountName: 'user', token, rating: 'dislike' }) - expect(body.data.length).to.equal(0) - } - }) - }) - - describe('Remove video', function () { - - it('Should not be able to remove the video with an incorrect token', async function () { - await server.videos.remove({ token: 'bad_token', id: videoId, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should not be able to remove the video with the token of another account', async function () { - await server.videos.remove({ token: anotherUserToken, id: videoId, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should be able to remove the video with the correct token', async function () { - await server.videos.remove({ token, id: videoId }) - await server.videos.remove({ token, id: videoId2 }) - }) - }) - - describe('My videos & quotas', function () { - - it('Should be able to upload a video with a user', async function () { - this.timeout(30000) - - const attributes = { - name: 'super user video', - fixture: 'video_short.webm' - } - await server.videos.upload({ token, attributes }) - - await server.channels.create({ token, attributes: { name: 'other_channel' } }) - }) - - it('Should have video quota updated', async function () { - const quota = await server.users.getMyQuotaUsed({ token }) - expect(quota.videoQuotaUsed).to.equal(218910) - expect(quota.videoQuotaUsedDaily).to.equal(218910) - - const { data } = await server.users.list() - const tmpUser = data.find(u => u.username === 'user') - expect(tmpUser.videoQuotaUsed).to.equal(218910) - expect(tmpUser.videoQuotaUsedDaily).to.equal(218910) - }) - - it('Should be able to list my videos', async function () { - const { total, data } = await server.videos.listMyVideos({ token }) - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - - const video = data[0] - expect(video.name).to.equal('super user video') - expect(video.thumbnailPath).to.not.be.null - expect(video.previewPath).to.not.be.null - }) - - it('Should be able to filter by channel in my videos', async function () { - const myInfo = await server.users.getMyInfo({ token }) - const mainChannel = myInfo.videoChannels.find(c => c.name !== 'other_channel') - const otherChannel = myInfo.videoChannels.find(c => c.name === 'other_channel') - - { - const { total, data } = await server.videos.listMyVideos({ token, channelId: mainChannel.id }) - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - - const video = data[0] - expect(video.name).to.equal('super user video') - expect(video.thumbnailPath).to.not.be.null - expect(video.previewPath).to.not.be.null - } - - { - const { total, data } = await server.videos.listMyVideos({ token, channelId: otherChannel.id }) - expect(total).to.equal(0) - expect(data).to.have.lengthOf(0) - } - }) - - it('Should be able to search in my videos', async function () { - { - const { total, data } = await server.videos.listMyVideos({ token, sort: '-createdAt', search: 'user video' }) - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - } - - { - const { total, data } = await server.videos.listMyVideos({ token, sort: '-createdAt', search: 'toto' }) - expect(total).to.equal(0) - expect(data).to.have.lengthOf(0) - } - }) - - it('Should disable web videos, enable HLS, and update my quota', async function () { - this.timeout(160000) - - { - const config = await server.config.getCustomConfig() - config.transcoding.webVideos.enabled = false - config.transcoding.hls.enabled = true - config.transcoding.enabled = true - await server.config.updateCustomSubConfig({ newConfig: config }) - } - - { - const attributes = { - name: 'super user video 2', - fixture: 'video_short.webm' - } - await server.videos.upload({ token, attributes }) - - await waitJobs([ server ]) - } - - { - const data = await server.users.getMyQuotaUsed({ token }) - expect(data.videoQuotaUsed).to.be.greaterThan(220000) - expect(data.videoQuotaUsedDaily).to.be.greaterThan(220000) - } - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/users/users-email-verification.ts b/server/tests/api/users/users-email-verification.ts deleted file mode 100644 index 909226311..000000000 --- a/server/tests/api/users/users-email-verification.ts +++ /dev/null @@ -1,165 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { MockSmtpServer } from '@server/tests/shared' -import { HttpStatusCode } from '@shared/models' -import { - cleanupTests, - ConfigCommand, - createSingleServer, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test users email verification', function () { - let server: PeerTubeServer - let userId: number - let userAccessToken: string - let verificationString: string - let expectedEmailsLength = 0 - const user1 = { - username: 'user_1', - password: 'super password' - } - const user2 = { - username: 'user_2', - password: 'super password' - } - const emails: object[] = [] - - before(async function () { - this.timeout(30000) - - const port = await MockSmtpServer.Instance.collectEmails(emails) - server = await createSingleServer(1, ConfigCommand.getEmailOverrideConfig(port)) - - await setAccessTokensToServers([ server ]) - }) - - it('Should register user and send verification email if verification required', async function () { - this.timeout(30000) - - await server.config.updateExistingSubConfig({ - newConfig: { - signup: { - enabled: true, - requiresApproval: false, - requiresEmailVerification: true, - limit: 10 - } - } - }) - - await server.registrations.register(user1) - - await waitJobs(server) - expectedEmailsLength++ - expect(emails).to.have.lengthOf(expectedEmailsLength) - - const email = emails[expectedEmailsLength - 1] - - const verificationStringMatches = /verificationString=([a-z0-9]+)/.exec(email['text']) - expect(verificationStringMatches).not.to.be.null - - verificationString = verificationStringMatches[1] - expect(verificationString).to.have.length.above(2) - - const userIdMatches = /userId=([0-9]+)/.exec(email['text']) - expect(userIdMatches).not.to.be.null - - userId = parseInt(userIdMatches[1], 10) - - const body = await server.users.get({ userId }) - expect(body.emailVerified).to.be.false - }) - - it('Should not allow login for user with unverified email', async function () { - const { detail } = await server.login.login({ user: user1, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - expect(detail).to.contain('User email is not verified.') - }) - - it('Should verify the user via email and allow login', async function () { - await server.users.verifyEmail({ userId, verificationString }) - - const body = await server.login.login({ user: user1 }) - userAccessToken = body.access_token - - const user = await server.users.get({ userId }) - expect(user.emailVerified).to.be.true - }) - - it('Should be able to change the user email', async function () { - let updateVerificationString: string - - { - await server.users.updateMe({ - token: userAccessToken, - email: 'updated@example.com', - currentPassword: user1.password - }) - - await waitJobs(server) - expectedEmailsLength++ - expect(emails).to.have.lengthOf(expectedEmailsLength) - - const email = emails[expectedEmailsLength - 1] - - const verificationStringMatches = /verificationString=([a-z0-9]+)/.exec(email['text']) - updateVerificationString = verificationStringMatches[1] - } - - { - const me = await server.users.getMyInfo({ token: userAccessToken }) - expect(me.email).to.equal('user_1@example.com') - expect(me.pendingEmail).to.equal('updated@example.com') - } - - { - await server.users.verifyEmail({ userId, verificationString: updateVerificationString, isPendingEmail: true }) - - const me = await server.users.getMyInfo({ token: userAccessToken }) - expect(me.email).to.equal('updated@example.com') - expect(me.pendingEmail).to.be.null - } - }) - - it('Should register user not requiring email verification if setting not enabled', async function () { - this.timeout(5000) - await server.config.updateExistingSubConfig({ - newConfig: { - signup: { - requiresEmailVerification: false - } - } - }) - - await server.registrations.register(user2) - - await waitJobs(server) - expect(emails).to.have.lengthOf(expectedEmailsLength) - - const accessToken = await server.login.getAccessToken(user2) - - const user = await server.users.getMyInfo({ token: accessToken }) - expect(user.emailVerified).to.be.null - }) - - it('Should allow login for user with unverified email when setting later enabled', async function () { - await server.config.updateCustomSubConfig({ - newConfig: { - signup: { - requiresEmailVerification: true - } - } - }) - - await server.login.getAccessToken(user2) - }) - - after(async function () { - MockSmtpServer.Instance.kill() - - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/users/users-multiple-servers.ts b/server/tests/api/users/users-multiple-servers.ts deleted file mode 100644 index 3823b74ef..000000000 --- a/server/tests/api/users/users-multiple-servers.ts +++ /dev/null @@ -1,216 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { - checkActorFilesWereRemoved, - checkTmpIsEmpty, - checkVideoFilesWereRemoved, - saveVideoInServers, - testImage -} from '@server/tests/shared' -import { MyUser } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - setDefaultChannelAvatar, - waitJobs -} from '@shared/server-commands' - -describe('Test users with multiple servers', function () { - let servers: PeerTubeServer[] = [] - - let user: MyUser - let userId: number - - let videoUUID: string - let userAccessToken: string - let userAvatarFilenames: string[] - - before(async function () { - this.timeout(120_000) - - servers = await createMultipleServers(3) - - // Get the access tokens - await setAccessTokensToServers(servers) - await setDefaultChannelAvatar(servers) - - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - // Server 1 and server 3 follow each other - await doubleFollow(servers[0], servers[2]) - // Server 2 and server 3 follow each other - await doubleFollow(servers[1], servers[2]) - - // The root user of server 1 is propagated to servers 2 and 3 - await servers[0].videos.upload() - - { - const username = 'user1' - const created = await servers[0].users.create({ username }) - userId = created.id - userAccessToken = await servers[0].login.getAccessToken(username) - } - - { - const { uuid } = await servers[0].videos.upload({ token: userAccessToken }) - videoUUID = uuid - - await waitJobs(servers) - - await saveVideoInServers(servers, videoUUID) - } - }) - - it('Should be able to update my display name', async function () { - await servers[0].users.updateMe({ displayName: 'my super display name' }) - - user = await servers[0].users.getMyInfo() - expect(user.account.displayName).to.equal('my super display name') - - await waitJobs(servers) - }) - - it('Should be able to update my description', async function () { - this.timeout(10_000) - - await servers[0].users.updateMe({ description: 'my super description updated' }) - - user = await servers[0].users.getMyInfo() - expect(user.account.displayName).to.equal('my super display name') - expect(user.account.description).to.equal('my super description updated') - - await waitJobs(servers) - }) - - it('Should be able to update my avatar', async function () { - this.timeout(10_000) - - const fixture = 'avatar2.png' - - await servers[0].users.updateMyAvatar({ fixture }) - - user = await servers[0].users.getMyInfo() - userAvatarFilenames = user.account.avatars.map(({ path }) => path) - - for (const avatar of user.account.avatars) { - await testImage(servers[0].url, `avatar2-resized-${avatar.width}x${avatar.width}`, avatar.path, '.png') - } - - await waitJobs(servers) - }) - - it('Should have updated my profile on other servers too', async function () { - let createdAt: string | Date - - for (const server of servers) { - const body = await server.accounts.list({ sort: '-createdAt' }) - - const resList = body.data.find(a => a.name === 'root' && a.host === servers[0].host) - expect(resList).not.to.be.undefined - - const account = await server.accounts.get({ accountName: resList.name + '@' + resList.host }) - - if (!createdAt) createdAt = account.createdAt - - expect(account.name).to.equal('root') - expect(account.host).to.equal(servers[0].host) - expect(account.displayName).to.equal('my super display name') - expect(account.description).to.equal('my super description updated') - expect(createdAt).to.equal(account.createdAt) - - if (server.serverNumber === 1) { - expect(account.userId).to.be.a('number') - } else { - expect(account.userId).to.be.undefined - } - - for (const avatar of account.avatars) { - await testImage(server.url, `avatar2-resized-${avatar.width}x${avatar.width}`, avatar.path, '.png') - } - } - }) - - it('Should list account videos', async function () { - for (const server of servers) { - const { total, data } = await server.videos.listByAccount({ handle: 'user1@' + servers[0].host }) - - expect(total).to.equal(1) - expect(data).to.be.an('array') - expect(data).to.have.lengthOf(1) - expect(data[0].uuid).to.equal(videoUUID) - } - }) - - it('Should search through account videos', async function () { - const created = await servers[0].videos.upload({ token: userAccessToken, attributes: { name: 'Kami no chikara' } }) - - await waitJobs(servers) - - for (const server of servers) { - const { total, data } = await server.videos.listByAccount({ handle: 'user1@' + servers[0].host, search: 'Kami' }) - - expect(total).to.equal(1) - expect(data).to.be.an('array') - expect(data).to.have.lengthOf(1) - expect(data[0].uuid).to.equal(created.uuid) - } - }) - - it('Should remove the user', async function () { - this.timeout(10_000) - - for (const server of servers) { - const body = await server.accounts.list({ sort: '-createdAt' }) - - const accountDeleted = body.data.find(a => a.name === 'user1' && a.host === servers[0].host) - expect(accountDeleted).not.to.be.undefined - - const { data } = await server.channels.list() - const videoChannelDeleted = data.find(a => a.displayName === 'Main user1 channel' && a.host === servers[0].host) - expect(videoChannelDeleted).not.to.be.undefined - } - - await servers[0].users.remove({ userId }) - - await waitJobs(servers) - - for (const server of servers) { - const body = await server.accounts.list({ sort: '-createdAt' }) - - const accountDeleted = body.data.find(a => a.name === 'user1' && a.host === servers[0].host) - expect(accountDeleted).to.be.undefined - - const { data } = await server.channels.list() - const videoChannelDeleted = data.find(a => a.name === 'Main user1 channel' && a.host === servers[0].host) - expect(videoChannelDeleted).to.be.undefined - } - }) - - it('Should not have actor files', async () => { - for (const server of servers) { - for (const userAvatarFilename of userAvatarFilenames) { - await checkActorFilesWereRemoved(userAvatarFilename, server) - } - } - }) - - it('Should not have video files', async () => { - for (const server of servers) { - await checkVideoFilesWereRemoved({ server, video: server.store.videoDetails }) - } - }) - - it('Should have an empty tmp directory', async function () { - for (const server of servers) { - await checkTmpIsEmpty(server) - } - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/users/users.ts b/server/tests/api/users/users.ts deleted file mode 100644 index 67ade1d0d..000000000 --- a/server/tests/api/users/users.ts +++ /dev/null @@ -1,529 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { testImageSize } from '@server/tests/shared' -import { AbuseState, HttpStatusCode, UserAdminFlag, UserRole, VideoPlaylistType } from '@shared/models' -import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' - -describe('Test users', function () { - let server: PeerTubeServer - let token: string - let userToken: string - let videoId: number - let userId: number - const user = { - username: 'user_1', - password: 'super password' - } - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1, { - rates_limit: { - login: { - max: 30 - } - } - }) - - await setAccessTokensToServers([ server ]) - - await server.plugins.install({ npmName: 'peertube-theme-background-red' }) - }) - - describe('Creating a user', function () { - - it('Should be able to create a new user', async function () { - await server.users.create({ ...user, videoQuota: 2 * 1024 * 1024, adminFlags: UserAdminFlag.BYPASS_VIDEO_AUTO_BLACKLIST }) - }) - - it('Should be able to login with this user', async function () { - userToken = await server.login.getAccessToken(user) - }) - - it('Should be able to get user information', async function () { - const userMe = await server.users.getMyInfo({ token: userToken }) - - const userGet = await server.users.get({ userId: userMe.id, withStats: true }) - - for (const user of [ userMe, userGet ]) { - expect(user.username).to.equal('user_1') - expect(user.email).to.equal('user_1@example.com') - expect(user.nsfwPolicy).to.equal('display') - expect(user.videoQuota).to.equal(2 * 1024 * 1024) - expect(user.role.label).to.equal('User') - expect(user.id).to.be.a('number') - expect(user.account.displayName).to.equal('user_1') - expect(user.account.description).to.be.null - } - - expect(userMe.adminFlags).to.equal(UserAdminFlag.BYPASS_VIDEO_AUTO_BLACKLIST) - expect(userGet.adminFlags).to.equal(UserAdminFlag.BYPASS_VIDEO_AUTO_BLACKLIST) - - expect(userMe.specialPlaylists).to.have.lengthOf(1) - expect(userMe.specialPlaylists[0].type).to.equal(VideoPlaylistType.WATCH_LATER) - - // Check stats are included with withStats - expect(userGet.videosCount).to.be.a('number') - expect(userGet.videosCount).to.equal(0) - expect(userGet.videoCommentsCount).to.be.a('number') - expect(userGet.videoCommentsCount).to.equal(0) - expect(userGet.abusesCount).to.be.a('number') - expect(userGet.abusesCount).to.equal(0) - expect(userGet.abusesAcceptedCount).to.be.a('number') - expect(userGet.abusesAcceptedCount).to.equal(0) - }) - }) - - describe('Users listing', function () { - - it('Should list all the users', async function () { - const { data, total } = await server.users.list() - - expect(total).to.equal(2) - expect(data).to.be.an('array') - expect(data.length).to.equal(2) - - const user = data[0] - expect(user.username).to.equal('user_1') - expect(user.email).to.equal('user_1@example.com') - expect(user.nsfwPolicy).to.equal('display') - - const rootUser = data[1] - expect(rootUser.username).to.equal('root') - expect(rootUser.email).to.equal('admin' + server.internalServerNumber + '@example.com') - expect(user.nsfwPolicy).to.equal('display') - - expect(rootUser.lastLoginDate).to.exist - expect(user.lastLoginDate).to.exist - - userId = user.id - }) - - it('Should list only the first user by username asc', async function () { - const { total, data } = await server.users.list({ start: 0, count: 1, sort: 'username' }) - - expect(total).to.equal(2) - expect(data.length).to.equal(1) - - const user = data[0] - expect(user.username).to.equal('root') - expect(user.email).to.equal('admin' + server.internalServerNumber + '@example.com') - expect(user.role.label).to.equal('Administrator') - expect(user.nsfwPolicy).to.equal('display') - }) - - it('Should list only the first user by username desc', async function () { - const { total, data } = await server.users.list({ start: 0, count: 1, sort: '-username' }) - - expect(total).to.equal(2) - expect(data.length).to.equal(1) - - const user = data[0] - expect(user.username).to.equal('user_1') - expect(user.email).to.equal('user_1@example.com') - expect(user.nsfwPolicy).to.equal('display') - }) - - it('Should list only the second user by createdAt desc', async function () { - const { data, total } = await server.users.list({ start: 0, count: 1, sort: '-createdAt' }) - expect(total).to.equal(2) - - expect(data.length).to.equal(1) - - const user = data[0] - expect(user.username).to.equal('user_1') - expect(user.email).to.equal('user_1@example.com') - expect(user.nsfwPolicy).to.equal('display') - }) - - it('Should list all the users by createdAt asc', async function () { - const { data, total } = await server.users.list({ start: 0, count: 2, sort: 'createdAt' }) - - expect(total).to.equal(2) - expect(data.length).to.equal(2) - - expect(data[0].username).to.equal('root') - expect(data[0].email).to.equal('admin' + server.internalServerNumber + '@example.com') - expect(data[0].nsfwPolicy).to.equal('display') - - expect(data[1].username).to.equal('user_1') - expect(data[1].email).to.equal('user_1@example.com') - expect(data[1].nsfwPolicy).to.equal('display') - }) - - it('Should search user by username', async function () { - const { data, total } = await server.users.list({ start: 0, count: 2, sort: 'createdAt', search: 'oot' }) - expect(total).to.equal(1) - expect(data.length).to.equal(1) - expect(data[0].username).to.equal('root') - }) - - it('Should search user by email', async function () { - { - const { total, data } = await server.users.list({ start: 0, count: 2, sort: 'createdAt', search: 'r_1@exam' }) - expect(total).to.equal(1) - expect(data.length).to.equal(1) - expect(data[0].username).to.equal('user_1') - expect(data[0].email).to.equal('user_1@example.com') - } - - { - const { total, data } = await server.users.list({ start: 0, count: 2, sort: 'createdAt', search: 'example' }) - expect(total).to.equal(2) - expect(data.length).to.equal(2) - expect(data[0].username).to.equal('root') - expect(data[1].username).to.equal('user_1') - } - }) - }) - - describe('Update my account', function () { - - it('Should update my password', async function () { - await server.users.updateMe({ - token: userToken, - currentPassword: 'super password', - password: 'new password' - }) - user.password = 'new password' - - await server.login.login({ user }) - }) - - it('Should be able to change the NSFW display attribute', async function () { - await server.users.updateMe({ - token: userToken, - nsfwPolicy: 'do_not_list' - }) - - const user = await server.users.getMyInfo({ token: userToken }) - expect(user.username).to.equal('user_1') - expect(user.email).to.equal('user_1@example.com') - expect(user.nsfwPolicy).to.equal('do_not_list') - expect(user.videoQuota).to.equal(2 * 1024 * 1024) - expect(user.id).to.be.a('number') - expect(user.account.displayName).to.equal('user_1') - expect(user.account.description).to.be.null - }) - - it('Should be able to change the autoPlayVideo attribute', async function () { - await server.users.updateMe({ - token: userToken, - autoPlayVideo: false - }) - - const user = await server.users.getMyInfo({ token: userToken }) - expect(user.autoPlayVideo).to.be.false - }) - - it('Should be able to change the autoPlayNextVideo attribute', async function () { - await server.users.updateMe({ - token: userToken, - autoPlayNextVideo: true - }) - - const user = await server.users.getMyInfo({ token: userToken }) - expect(user.autoPlayNextVideo).to.be.true - }) - - it('Should be able to change the p2p attribute', async function () { - await server.users.updateMe({ - token: userToken, - p2pEnabled: true - }) - - const user = await server.users.getMyInfo({ token: userToken }) - expect(user.p2pEnabled).to.be.true - }) - - it('Should be able to change the email attribute', async function () { - await server.users.updateMe({ - token: userToken, - currentPassword: 'new password', - email: 'updated@example.com' - }) - - const user = await server.users.getMyInfo({ token: userToken }) - expect(user.username).to.equal('user_1') - expect(user.email).to.equal('updated@example.com') - expect(user.nsfwPolicy).to.equal('do_not_list') - expect(user.videoQuota).to.equal(2 * 1024 * 1024) - expect(user.id).to.be.a('number') - expect(user.account.displayName).to.equal('user_1') - expect(user.account.description).to.be.null - }) - - it('Should be able to update my avatar with a gif', async function () { - const fixture = 'avatar.gif' - - await server.users.updateMyAvatar({ token: userToken, fixture }) - - const user = await server.users.getMyInfo({ token: userToken }) - for (const avatar of user.account.avatars) { - await testImageSize(server.url, `avatar-resized-${avatar.width}x${avatar.width}`, avatar.path, '.gif') - } - }) - - it('Should be able to update my avatar with a gif, and then a png', async function () { - for (const extension of [ '.png', '.gif' ]) { - const fixture = 'avatar' + extension - - await server.users.updateMyAvatar({ token: userToken, fixture }) - - const user = await server.users.getMyInfo({ token: userToken }) - for (const avatar of user.account.avatars) { - await testImageSize(server.url, `avatar-resized-${avatar.width}x${avatar.width}`, avatar.path, extension) - } - } - }) - - it('Should be able to update my display name', async function () { - await server.users.updateMe({ token: userToken, displayName: 'new display name' }) - - const user = await server.users.getMyInfo({ token: userToken }) - expect(user.username).to.equal('user_1') - expect(user.email).to.equal('updated@example.com') - expect(user.nsfwPolicy).to.equal('do_not_list') - expect(user.videoQuota).to.equal(2 * 1024 * 1024) - expect(user.id).to.be.a('number') - expect(user.account.displayName).to.equal('new display name') - expect(user.account.description).to.be.null - }) - - it('Should be able to update my description', async function () { - await server.users.updateMe({ token: userToken, description: 'my super description updated' }) - - const user = await server.users.getMyInfo({ token: userToken }) - expect(user.username).to.equal('user_1') - expect(user.email).to.equal('updated@example.com') - expect(user.nsfwPolicy).to.equal('do_not_list') - expect(user.videoQuota).to.equal(2 * 1024 * 1024) - expect(user.id).to.be.a('number') - expect(user.account.displayName).to.equal('new display name') - expect(user.account.description).to.equal('my super description updated') - expect(user.noWelcomeModal).to.be.false - expect(user.noInstanceConfigWarningModal).to.be.false - expect(user.noAccountSetupWarningModal).to.be.false - }) - - it('Should be able to update my theme', async function () { - for (const theme of [ 'background-red', 'default', 'instance-default' ]) { - await server.users.updateMe({ token: userToken, theme }) - - const user = await server.users.getMyInfo({ token: userToken }) - expect(user.theme).to.equal(theme) - } - }) - - it('Should be able to update my modal preferences', async function () { - await server.users.updateMe({ - token: userToken, - noInstanceConfigWarningModal: true, - noWelcomeModal: true, - noAccountSetupWarningModal: true - }) - - const user = await server.users.getMyInfo({ token: userToken }) - expect(user.noWelcomeModal).to.be.true - expect(user.noInstanceConfigWarningModal).to.be.true - expect(user.noAccountSetupWarningModal).to.be.true - }) - }) - - describe('Updating another user', function () { - - it('Should be able to update another user', async function () { - await server.users.update({ - userId, - token, - email: 'updated2@example.com', - emailVerified: true, - videoQuota: 42, - role: UserRole.MODERATOR, - adminFlags: UserAdminFlag.NONE, - pluginAuth: 'toto' - }) - - const user = await server.users.get({ token, userId }) - - expect(user.username).to.equal('user_1') - expect(user.email).to.equal('updated2@example.com') - expect(user.emailVerified).to.be.true - expect(user.nsfwPolicy).to.equal('do_not_list') - expect(user.videoQuota).to.equal(42) - expect(user.role.label).to.equal('Moderator') - expect(user.id).to.be.a('number') - expect(user.adminFlags).to.equal(UserAdminFlag.NONE) - expect(user.pluginAuth).to.equal('toto') - }) - - it('Should reset the auth plugin', async function () { - await server.users.update({ userId, token, pluginAuth: null }) - - const user = await server.users.get({ token, userId }) - expect(user.pluginAuth).to.be.null - }) - - it('Should have removed the user token', async function () { - await server.users.getMyQuotaUsed({ token: userToken, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - - userToken = await server.login.getAccessToken(user) - }) - - it('Should be able to update another user password', async function () { - await server.users.update({ userId, token, password: 'password updated' }) - - await server.users.getMyQuotaUsed({ token: userToken, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - - await server.login.login({ user, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - - user.password = 'password updated' - userToken = await server.login.getAccessToken(user) - }) - }) - - describe('Remove a user', function () { - - before(async function () { - await server.users.update({ - userId, - token, - videoQuota: 2 * 1024 * 1024 - }) - - await server.videos.quickUpload({ name: 'user video', token: userToken, fixture: 'video_short.webm' }) - await server.videos.quickUpload({ name: 'root video' }) - - const { total } = await server.videos.list() - expect(total).to.equal(2) - }) - - it('Should be able to remove this user', async function () { - await server.users.remove({ userId, token }) - }) - - it('Should not be able to login with this user', async function () { - await server.login.login({ user, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should not have videos of this user', async function () { - const { data, total } = await server.videos.list() - expect(total).to.equal(1) - - const video = data[0] - expect(video.account.name).to.equal('root') - }) - }) - - describe('User blocking', function () { - let user16Id: number - let user16AccessToken: string - - const user16 = { - username: 'user_16', - password: 'my super password' - } - - it('Should block a user', async function () { - const user = await server.users.create({ ...user16 }) - user16Id = user.id - - user16AccessToken = await server.login.getAccessToken(user16) - - await server.users.getMyInfo({ token: user16AccessToken, expectedStatus: HttpStatusCode.OK_200 }) - await server.users.banUser({ userId: user16Id }) - - await server.users.getMyInfo({ token: user16AccessToken, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - await server.login.login({ user: user16, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should search user by banned status', async function () { - { - const { data, total } = await server.users.list({ start: 0, count: 2, sort: 'createdAt', blocked: true }) - expect(total).to.equal(1) - expect(data.length).to.equal(1) - - expect(data[0].username).to.equal(user16.username) - } - - { - const { data, total } = await server.users.list({ start: 0, count: 2, sort: 'createdAt', blocked: false }) - expect(total).to.equal(1) - expect(data.length).to.equal(1) - - expect(data[0].username).to.not.equal(user16.username) - } - }) - - it('Should unblock a user', async function () { - await server.users.unbanUser({ userId: user16Id }) - user16AccessToken = await server.login.getAccessToken(user16) - await server.users.getMyInfo({ token: user16AccessToken, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - describe('User stats', function () { - let user17Id: number - let user17AccessToken: string - - it('Should report correct initial statistics about a user', async function () { - const user17 = { - username: 'user_17', - password: 'my super password' - } - const created = await server.users.create({ ...user17 }) - - user17Id = created.id - user17AccessToken = await server.login.getAccessToken(user17) - - const user = await server.users.get({ userId: user17Id, withStats: true }) - expect(user.videosCount).to.equal(0) - expect(user.videoCommentsCount).to.equal(0) - expect(user.abusesCount).to.equal(0) - expect(user.abusesCreatedCount).to.equal(0) - expect(user.abusesAcceptedCount).to.equal(0) - }) - - it('Should report correct videos count', async function () { - const attributes = { name: 'video to test user stats' } - await server.videos.upload({ token: user17AccessToken, attributes }) - - const { data } = await server.videos.list() - videoId = data.find(video => video.name === attributes.name).id - - const user = await server.users.get({ userId: user17Id, withStats: true }) - expect(user.videosCount).to.equal(1) - }) - - it('Should report correct video comments for user', async function () { - const text = 'super comment' - await server.comments.createThread({ token: user17AccessToken, videoId, text }) - - const user = await server.users.get({ userId: user17Id, withStats: true }) - expect(user.videoCommentsCount).to.equal(1) - }) - - it('Should report correct abuses counts', async function () { - const reason = 'my super bad reason' - await server.abuses.report({ token: user17AccessToken, videoId, reason }) - - const body1 = await server.abuses.getAdminList() - const abuseId = body1.data[0].id - - const user2 = await server.users.get({ userId: user17Id, withStats: true }) - expect(user2.abusesCount).to.equal(1) // number of incriminations - expect(user2.abusesCreatedCount).to.equal(1) // number of reports created - - await server.abuses.update({ abuseId, body: { state: AbuseState.ACCEPTED } }) - - const user3 = await server.users.get({ userId: user17Id, withStats: true }) - expect(user3.abusesAcceptedCount).to.equal(1) // number of reports created accepted - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/videos/channel-import-videos.ts b/server/tests/api/videos/channel-import-videos.ts deleted file mode 100644 index a66f88a0e..000000000 --- a/server/tests/api/videos/channel-import-videos.ts +++ /dev/null @@ -1,161 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { FIXTURE_URLS } from '@server/tests/shared' -import { areHttpImportTestsDisabled } from '@shared/core-utils' -import { - createSingleServer, - getServerImportConfig, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - waitJobs -} from '@shared/server-commands' - -describe('Test videos import in a channel', function () { - if (areHttpImportTestsDisabled()) return - - function runSuite (mode: 'youtube-dl' | 'yt-dlp') { - - describe('Import using ' + mode, function () { - let server: PeerTubeServer - - before(async function () { - this.timeout(120_000) - - server = await createSingleServer(1, getServerImportConfig(mode)) - - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - await server.config.enableChannelSync() - }) - - it('Should import a whole channel without specifying the sync id', async function () { - this.timeout(240_000) - - await server.channels.importVideos({ channelName: server.store.channel.name, externalChannelUrl: FIXTURE_URLS.youtubeChannel }) - await waitJobs(server) - - const videos = await server.videos.listByChannel({ handle: server.store.channel.name }) - expect(videos.total).to.equal(2) - }) - - it('These imports should not have a sync id', async function () { - const { total, data } = await server.imports.getMyVideoImports() - - expect(total).to.equal(2) - expect(data).to.have.lengthOf(2) - - for (const videoImport of data) { - expect(videoImport.videoChannelSync).to.not.exist - } - }) - - it('Should import a whole channel and specifying the sync id', async function () { - this.timeout(240_000) - - { - server.store.channel.name = 'channel2' - const { id } = await server.channels.create({ attributes: { name: server.store.channel.name } }) - server.store.channel.id = id - } - - { - const attributes = { - externalChannelUrl: FIXTURE_URLS.youtubeChannel, - videoChannelId: server.store.channel.id - } - - const { videoChannelSync } = await server.channelSyncs.create({ attributes }) - server.store.videoChannelSync = videoChannelSync - - await waitJobs(server) - } - - await server.channels.importVideos({ - channelName: server.store.channel.name, - externalChannelUrl: FIXTURE_URLS.youtubeChannel, - videoChannelSyncId: server.store.videoChannelSync.id - }) - - await waitJobs(server) - }) - - it('These imports should have a sync id', async function () { - const { total, data } = await server.imports.getMyVideoImports() - - expect(total).to.equal(4) - expect(data).to.have.lengthOf(4) - - const importsWithSyncId = data.filter(i => !!i.videoChannelSync) - expect(importsWithSyncId).to.have.lengthOf(2) - - for (const videoImport of importsWithSyncId) { - expect(videoImport.videoChannelSync).to.exist - expect(videoImport.videoChannelSync.id).to.equal(server.store.videoChannelSync.id) - } - }) - - it('Should be able to filter imports by this sync id', async function () { - const { total, data } = await server.imports.getMyVideoImports({ videoChannelSyncId: server.store.videoChannelSync.id }) - - expect(total).to.equal(2) - expect(data).to.have.lengthOf(2) - - for (const videoImport of data) { - expect(videoImport.videoChannelSync).to.exist - expect(videoImport.videoChannelSync.id).to.equal(server.store.videoChannelSync.id) - } - }) - - it('Should limit max amount of videos synced on full sync', async function () { - this.timeout(240_000) - - await server.kill() - await server.run({ - import: { - video_channel_synchronization: { - full_sync_videos_limit: 1 - } - } - }) - - const { id } = await server.channels.create({ attributes: { name: 'channel3' } }) - const channel3Id = id - - const { videoChannelSync } = await server.channelSyncs.create({ - attributes: { - externalChannelUrl: FIXTURE_URLS.youtubeChannel, - videoChannelId: channel3Id - } - }) - const syncId = videoChannelSync.id - - await waitJobs(server) - - await server.channels.importVideos({ - channelName: 'channel3', - externalChannelUrl: FIXTURE_URLS.youtubeChannel, - videoChannelSyncId: syncId - }) - - await waitJobs(server) - - const { total, data } = await server.videos.listByChannel({ handle: 'channel3' }) - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - }) - - after(async function () { - await server?.kill() - }) - }) - } - - runSuite('yt-dlp') - - // FIXME: With recent changes on youtube, youtube-dl doesn't fetch live replays which means the test suite fails - // runSuite('youtube-dl') -}) diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts deleted file mode 100644 index 01d0c5852..000000000 --- a/server/tests/api/videos/index.ts +++ /dev/null @@ -1,23 +0,0 @@ -import './multiple-servers' -import './resumable-upload' -import './single-server' -import './video-captions' -import './video-change-ownership' -import './video-channels' -import './channel-import-videos' -import './video-channel-syncs' -import './video-comments' -import './video-description' -import './video-files' -import './video-imports' -import './video-nsfw' -import './video-playlists' -import './video-playlist-thumbnails' -import './video-source' -import './video-privacy' -import './video-schedule-update' -import './videos-common-filters' -import './videos-history' -import './videos-overview' -import './video-static-file-privacy' -import './video-storyboard' diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts deleted file mode 100644 index e9aa0e3a1..000000000 --- a/server/tests/api/videos/multiple-servers.ts +++ /dev/null @@ -1,1099 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import request from 'supertest' -import { - checkTmpIsEmpty, - checkVideoFilesWereRemoved, - checkWebTorrentWorks, - completeVideoCheck, - dateIsValid, - saveVideoInServers, - testImageGeneratedByFFmpeg -} from '@server/tests/shared' -import { buildAbsoluteFixturePath, wait } from '@shared/core-utils' -import { HttpStatusCode, VideoCommentThreadTree, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - makeGetRequest, - PeerTubeServer, - setAccessTokensToServers, - setDefaultAccountAvatar, - setDefaultChannelAvatar, - waitJobs -} from '@shared/server-commands' - -describe('Test multiple servers', function () { - let servers: PeerTubeServer[] = [] - const toRemove = [] - let videoUUID = '' - let videoChannelId: number - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(3) - - // Get the access tokens - await setAccessTokensToServers(servers) - - { - const videoChannel = { - name: 'super_channel_name', - displayName: 'my channel', - description: 'super channel' - } - await servers[0].channels.create({ attributes: videoChannel }) - await setDefaultChannelAvatar(servers[0], videoChannel.name) - await setDefaultAccountAvatar(servers) - - const { data } = await servers[0].channels.list({ start: 0, count: 1 }) - videoChannelId = data[0].id - } - - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - // Server 1 and server 3 follow each other - await doubleFollow(servers[0], servers[2]) - // Server 2 and server 3 follow each other - await doubleFollow(servers[1], servers[2]) - }) - - it('Should not have videos for all servers', async function () { - for (const server of servers) { - const { data } = await server.videos.list() - expect(data).to.be.an('array') - expect(data.length).to.equal(0) - } - }) - - describe('Should upload the video and propagate on each server', function () { - - it('Should upload the video on server 1 and propagate on each server', async function () { - this.timeout(60000) - - const attributes = { - name: 'my super name for server 1', - category: 5, - licence: 4, - language: 'ja', - nsfw: true, - description: 'my super description for server 1', - support: 'my super support text for server 1', - originallyPublishedAt: '2019-02-10T13:38:14.449Z', - tags: [ 'tag1p1', 'tag2p1' ], - channelId: videoChannelId, - fixture: 'video_short1.webm' - } - await servers[0].videos.upload({ attributes }) - - await waitJobs(servers) - - // All servers should have this video - let publishedAt: string = null - for (const server of servers) { - const isLocal = server.port === servers[0].port - const checkAttributes = { - name: 'my super name for server 1', - category: 5, - licence: 4, - language: 'ja', - nsfw: true, - description: 'my super description for server 1', - support: 'my super support text for server 1', - originallyPublishedAt: '2019-02-10T13:38:14.449Z', - account: { - name: 'root', - host: servers[0].host - }, - isLocal, - publishedAt, - duration: 10, - tags: [ 'tag1p1', 'tag2p1' ], - privacy: VideoPrivacy.PUBLIC, - commentsEnabled: true, - downloadEnabled: true, - channel: { - displayName: 'my channel', - name: 'super_channel_name', - description: 'super channel', - isLocal - }, - fixture: 'video_short1.webm', - files: [ - { - resolution: 720, - size: 572456 - } - ] - } - - const { data } = await server.videos.list() - expect(data).to.be.an('array') - expect(data.length).to.equal(1) - const video = data[0] - - await completeVideoCheck({ server, originServer: servers[0], videoUUID: video.uuid, attributes: checkAttributes }) - publishedAt = video.publishedAt as string - - expect(video.channel.avatars).to.have.lengthOf(2) - expect(video.account.avatars).to.have.lengthOf(2) - - for (const image of [ ...video.channel.avatars, ...video.account.avatars ]) { - expect(image.createdAt).to.exist - expect(image.updatedAt).to.exist - expect(image.width).to.be.above(20).and.below(1000) - expect(image.path).to.exist - - await makeGetRequest({ - url: server.url, - path: image.path, - expectedStatus: HttpStatusCode.OK_200 - }) - } - } - }) - - it('Should upload the video on server 2 and propagate on each server', async function () { - this.timeout(240000) - - const user = { - username: 'user1', - password: 'super_password' - } - await servers[1].users.create({ username: user.username, password: user.password }) - const userAccessToken = await servers[1].login.getAccessToken(user) - - const attributes = { - name: 'my super name for server 2', - category: 4, - licence: 3, - language: 'de', - nsfw: true, - description: 'my super description for server 2', - support: 'my super support text for server 2', - tags: [ 'tag1p2', 'tag2p2', 'tag3p2' ], - fixture: 'video_short2.webm', - thumbnailfile: 'custom-thumbnail.jpg', - previewfile: 'custom-preview.jpg' - } - await servers[1].videos.upload({ token: userAccessToken, attributes, mode: 'resumable' }) - - // Transcoding - await waitJobs(servers) - - // All servers should have this video - for (const server of servers) { - const isLocal = server.url === servers[1].url - const checkAttributes = { - name: 'my super name for server 2', - category: 4, - licence: 3, - language: 'de', - nsfw: true, - description: 'my super description for server 2', - support: 'my super support text for server 2', - account: { - name: 'user1', - host: servers[1].host - }, - isLocal, - commentsEnabled: true, - downloadEnabled: true, - duration: 5, - tags: [ 'tag1p2', 'tag2p2', 'tag3p2' ], - privacy: VideoPrivacy.PUBLIC, - channel: { - displayName: 'Main user1 channel', - name: 'user1_channel', - description: 'super channel', - isLocal - }, - fixture: 'video_short2.webm', - files: [ - { - resolution: 240, - size: 270000 - }, - { - resolution: 360, - size: 359000 - }, - { - resolution: 480, - size: 465000 - }, - { - resolution: 720, - size: 750000 - } - ], - thumbnailfile: 'custom-thumbnail', - previewfile: 'custom-preview' - } - - const { data } = await server.videos.list() - expect(data).to.be.an('array') - expect(data.length).to.equal(2) - const video = data[1] - - await completeVideoCheck({ server, originServer: servers[1], videoUUID: video.uuid, attributes: checkAttributes }) - } - }) - - it('Should upload two videos on server 3 and propagate on each server', async function () { - this.timeout(45000) - - { - const attributes = { - name: 'my super name for server 3', - category: 6, - licence: 5, - language: 'de', - nsfw: true, - description: 'my super description for server 3', - support: 'my super support text for server 3', - tags: [ 'tag1p3' ], - fixture: 'video_short3.webm' - } - await servers[2].videos.upload({ attributes }) - } - - { - const attributes = { - name: 'my super name for server 3-2', - category: 7, - licence: 6, - language: 'ko', - nsfw: false, - description: 'my super description for server 3-2', - support: 'my super support text for server 3-2', - tags: [ 'tag2p3', 'tag3p3', 'tag4p3' ], - fixture: 'video_short.webm' - } - await servers[2].videos.upload({ attributes }) - } - - await waitJobs(servers) - - // All servers should have this video - for (const server of servers) { - const isLocal = server.url === servers[2].url - const { data } = await server.videos.list() - - expect(data).to.be.an('array') - expect(data.length).to.equal(4) - - // We not sure about the order of the two last uploads - let video1 = null - let video2 = null - if (data[2].name === 'my super name for server 3') { - video1 = data[2] - video2 = data[3] - } else { - video1 = data[3] - video2 = data[2] - } - - const checkAttributesVideo1 = { - name: 'my super name for server 3', - category: 6, - licence: 5, - language: 'de', - nsfw: true, - description: 'my super description for server 3', - support: 'my super support text for server 3', - account: { - name: 'root', - host: servers[2].host - }, - isLocal, - duration: 5, - commentsEnabled: true, - downloadEnabled: true, - tags: [ 'tag1p3' ], - privacy: VideoPrivacy.PUBLIC, - channel: { - displayName: 'Main root channel', - name: 'root_channel', - description: '', - isLocal - }, - fixture: 'video_short3.webm', - files: [ - { - resolution: 720, - size: 292677 - } - ] - } - await completeVideoCheck({ server, originServer: servers[2], videoUUID: video1.uuid, attributes: checkAttributesVideo1 }) - - const checkAttributesVideo2 = { - name: 'my super name for server 3-2', - category: 7, - licence: 6, - language: 'ko', - nsfw: false, - description: 'my super description for server 3-2', - support: 'my super support text for server 3-2', - account: { - name: 'root', - host: servers[2].host - }, - commentsEnabled: true, - downloadEnabled: true, - isLocal, - duration: 5, - tags: [ 'tag2p3', 'tag3p3', 'tag4p3' ], - privacy: VideoPrivacy.PUBLIC, - channel: { - displayName: 'Main root channel', - name: 'root_channel', - description: '', - isLocal - }, - fixture: 'video_short.webm', - files: [ - { - resolution: 720, - size: 218910 - } - ] - } - await completeVideoCheck({ server, originServer: servers[2], videoUUID: video2.uuid, attributes: checkAttributesVideo2 }) - } - }) - }) - - describe('It should list local videos', function () { - it('Should list only local videos on server 1', async function () { - const { data, total } = await servers[0].videos.list({ isLocal: true }) - - expect(total).to.equal(1) - expect(data).to.be.an('array') - expect(data.length).to.equal(1) - expect(data[0].name).to.equal('my super name for server 1') - }) - - it('Should list only local videos on server 2', async function () { - const { data, total } = await servers[1].videos.list({ isLocal: true }) - - expect(total).to.equal(1) - expect(data).to.be.an('array') - expect(data.length).to.equal(1) - expect(data[0].name).to.equal('my super name for server 2') - }) - - it('Should list only local videos on server 3', async function () { - const { data, total } = await servers[2].videos.list({ isLocal: true }) - - expect(total).to.equal(2) - expect(data).to.be.an('array') - expect(data.length).to.equal(2) - expect(data[0].name).to.equal('my super name for server 3') - expect(data[1].name).to.equal('my super name for server 3-2') - }) - }) - - describe('Should seed the uploaded video', function () { - - it('Should add the file 1 by asking server 3', async function () { - this.retries(2) - this.timeout(30000) - - const { data } = await servers[2].videos.list() - - const video = data[0] - toRemove.push(data[2]) - toRemove.push(data[3]) - - const videoDetails = await servers[2].videos.get({ id: video.id }) - - await checkWebTorrentWorks(videoDetails.files[0].magnetUri) - }) - - it('Should add the file 2 by asking server 1', async function () { - this.retries(2) - this.timeout(30000) - - const { data } = await servers[0].videos.list() - - const video = data[1] - const videoDetails = await servers[0].videos.get({ id: video.id }) - - await checkWebTorrentWorks(videoDetails.files[0].magnetUri) - }) - - it('Should add the file 3 by asking server 2', async function () { - this.retries(2) - this.timeout(30000) - - const { data } = await servers[1].videos.list() - - const video = data[2] - const videoDetails = await servers[1].videos.get({ id: video.id }) - - await checkWebTorrentWorks(videoDetails.files[0].magnetUri) - }) - - it('Should add the file 3-2 by asking server 1', async function () { - this.retries(2) - this.timeout(30000) - - const { data } = await servers[0].videos.list() - - const video = data[3] - const videoDetails = await servers[0].videos.get({ id: video.id }) - - await checkWebTorrentWorks(videoDetails.files[0].magnetUri) - }) - - it('Should add the file 2 in 360p by asking server 1', async function () { - this.retries(2) - this.timeout(30000) - - const { data } = await servers[0].videos.list() - - const video = data.find(v => v.name === 'my super name for server 2') - const videoDetails = await servers[0].videos.get({ id: video.id }) - - const file = videoDetails.files.find(f => f.resolution.id === 360) - expect(file).not.to.be.undefined - - await checkWebTorrentWorks(file.magnetUri) - }) - }) - - describe('Should update video views, likes and dislikes', function () { - let localVideosServer3 = [] - let remoteVideosServer1 = [] - let remoteVideosServer2 = [] - let remoteVideosServer3 = [] - - before(async function () { - { - const { data } = await servers[0].videos.list() - remoteVideosServer1 = data.filter(video => video.isLocal === false).map(video => video.uuid) - } - - { - const { data } = await servers[1].videos.list() - remoteVideosServer2 = data.filter(video => video.isLocal === false).map(video => video.uuid) - } - - { - const { data } = await servers[2].videos.list() - localVideosServer3 = data.filter(video => video.isLocal === true).map(video => video.uuid) - remoteVideosServer3 = data.filter(video => video.isLocal === false).map(video => video.uuid) - } - }) - - it('Should view multiple videos on owned servers', async function () { - this.timeout(30000) - - await servers[2].views.simulateView({ id: localVideosServer3[0] }) - await wait(1000) - - await servers[2].views.simulateView({ id: localVideosServer3[0] }) - await servers[2].views.simulateView({ id: localVideosServer3[1] }) - - await wait(1000) - - await servers[2].views.simulateView({ id: localVideosServer3[0] }) - await servers[2].views.simulateView({ id: localVideosServer3[0] }) - - await waitJobs(servers) - - for (const server of servers) { - await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } }) - } - - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.videos.list() - - const video0 = data.find(v => v.uuid === localVideosServer3[0]) - const video1 = data.find(v => v.uuid === localVideosServer3[1]) - - expect(video0.views).to.equal(3) - expect(video1.views).to.equal(1) - } - }) - - it('Should view multiple videos on each servers', async function () { - this.timeout(45000) - - const tasks: Promise[] = [] - tasks.push(servers[0].views.simulateView({ id: remoteVideosServer1[0] })) - tasks.push(servers[1].views.simulateView({ id: remoteVideosServer2[0] })) - tasks.push(servers[1].views.simulateView({ id: remoteVideosServer2[0] })) - tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[0] })) - tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[1] })) - tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[1] })) - tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[1] })) - tasks.push(servers[2].views.simulateView({ id: localVideosServer3[1] })) - tasks.push(servers[2].views.simulateView({ id: localVideosServer3[1] })) - tasks.push(servers[2].views.simulateView({ id: localVideosServer3[1] })) - - await Promise.all(tasks) - - await waitJobs(servers) - - for (const server of servers) { - await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } }) - } - - await waitJobs(servers) - - let baseVideos = null - - for (const server of servers) { - const { data } = await server.videos.list() - - // Initialize base videos for future comparisons - if (baseVideos === null) { - baseVideos = data - continue - } - - for (const baseVideo of baseVideos) { - const sameVideo = data.find(video => video.name === baseVideo.name) - expect(baseVideo.views).to.equal(sameVideo.views) - } - } - }) - - it('Should like and dislikes videos on different services', async function () { - this.timeout(50000) - - await servers[0].videos.rate({ id: remoteVideosServer1[0], rating: 'like' }) - await wait(500) - await servers[0].videos.rate({ id: remoteVideosServer1[0], rating: 'dislike' }) - await wait(500) - await servers[0].videos.rate({ id: remoteVideosServer1[0], rating: 'like' }) - await servers[2].videos.rate({ id: localVideosServer3[1], rating: 'like' }) - await wait(500) - await servers[2].videos.rate({ id: localVideosServer3[1], rating: 'dislike' }) - await servers[2].videos.rate({ id: remoteVideosServer3[1], rating: 'dislike' }) - await wait(500) - await servers[2].videos.rate({ id: remoteVideosServer3[0], rating: 'like' }) - - await waitJobs(servers) - await wait(5000) - await waitJobs(servers) - - let baseVideos = null - for (const server of servers) { - const { data } = await server.videos.list() - - // Initialize base videos for future comparisons - if (baseVideos === null) { - baseVideos = data - continue - } - - for (const baseVideo of baseVideos) { - const sameVideo = data.find(video => video.name === baseVideo.name) - expect(baseVideo.likes).to.equal(sameVideo.likes, `Likes of ${sameVideo.uuid} do not correspond`) - expect(baseVideo.dislikes).to.equal(sameVideo.dislikes, `Dislikes of ${sameVideo.uuid} do not correspond`) - } - } - }) - }) - - describe('Should manipulate these videos', function () { - let updatedAtMin: Date - - it('Should update video 3', async function () { - this.timeout(30000) - - const attributes = { - name: 'my super video updated', - category: 10, - licence: 7, - language: 'fr', - nsfw: true, - description: 'my super description updated', - support: 'my super support text updated', - tags: [ 'tag_up_1', 'tag_up_2' ], - thumbnailfile: 'custom-thumbnail.jpg', - originallyPublishedAt: '2019-02-11T13:38:14.449Z', - previewfile: 'custom-preview.jpg' - } - - updatedAtMin = new Date() - await servers[2].videos.update({ id: toRemove[0].id, attributes }) - - await waitJobs(servers) - }) - - it('Should have the video 3 updated on each server', async function () { - this.timeout(30000) - - for (const server of servers) { - const { data } = await server.videos.list() - - const videoUpdated = data.find(video => video.name === 'my super video updated') - expect(!!videoUpdated).to.be.true - - expect(new Date(videoUpdated.updatedAt)).to.be.greaterThan(updatedAtMin) - - const isLocal = server.url === servers[2].url - const checkAttributes = { - name: 'my super video updated', - category: 10, - licence: 7, - language: 'fr', - nsfw: true, - description: 'my super description updated', - support: 'my super support text updated', - originallyPublishedAt: '2019-02-11T13:38:14.449Z', - account: { - name: 'root', - host: servers[2].host - }, - isLocal, - duration: 5, - commentsEnabled: true, - downloadEnabled: true, - tags: [ 'tag_up_1', 'tag_up_2' ], - privacy: VideoPrivacy.PUBLIC, - channel: { - displayName: 'Main root channel', - name: 'root_channel', - description: '', - isLocal - }, - fixture: 'video_short3.webm', - files: [ - { - resolution: 720, - size: 292677 - } - ], - thumbnailfile: 'custom-thumbnail', - previewfile: 'custom-preview' - } - await completeVideoCheck({ server, originServer: servers[2], videoUUID: videoUpdated.uuid, attributes: checkAttributes }) - } - }) - - it('Should only update thumbnail and update updatedAt attribute', async function () { - this.timeout(30000) - - const attributes = { - thumbnailfile: 'custom-thumbnail.jpg' - } - - updatedAtMin = new Date() - await servers[2].videos.update({ id: toRemove[0].id, attributes }) - - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.videos.list() - - const videoUpdated = data.find(video => video.name === 'my super video updated') - expect(new Date(videoUpdated.updatedAt)).to.be.greaterThan(updatedAtMin) - } - }) - - it('Should remove the videos 3 and 3-2 by asking server 3 and correctly delete files', async function () { - this.timeout(30000) - - for (const id of [ toRemove[0].id, toRemove[1].id ]) { - await saveVideoInServers(servers, id) - - await servers[2].videos.remove({ id }) - - await waitJobs(servers) - - for (const server of servers) { - await checkVideoFilesWereRemoved({ server, video: server.store.videoDetails }) - } - } - }) - - it('Should have videos 1 and 3 on each server', async function () { - for (const server of servers) { - const { data } = await server.videos.list() - - expect(data).to.be.an('array') - expect(data.length).to.equal(2) - expect(data[0].name).not.to.equal(data[1].name) - expect(data[0].name).not.to.equal(toRemove[0].name) - expect(data[1].name).not.to.equal(toRemove[0].name) - expect(data[0].name).not.to.equal(toRemove[1].name) - expect(data[1].name).not.to.equal(toRemove[1].name) - - videoUUID = data.find(video => video.name === 'my super name for server 1').uuid - } - }) - - it('Should get the same video by UUID on each server', async function () { - let baseVideo = null - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - - if (baseVideo === null) { - baseVideo = video - continue - } - - expect(baseVideo.name).to.equal(video.name) - expect(baseVideo.uuid).to.equal(video.uuid) - expect(baseVideo.category.id).to.equal(video.category.id) - expect(baseVideo.language.id).to.equal(video.language.id) - expect(baseVideo.licence.id).to.equal(video.licence.id) - expect(baseVideo.nsfw).to.equal(video.nsfw) - expect(baseVideo.account.name).to.equal(video.account.name) - expect(baseVideo.account.displayName).to.equal(video.account.displayName) - expect(baseVideo.account.url).to.equal(video.account.url) - expect(baseVideo.account.host).to.equal(video.account.host) - expect(baseVideo.tags).to.deep.equal(video.tags) - } - }) - - it('Should get the preview from each server', async function () { - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - - await testImageGeneratedByFFmpeg(server.url, 'video_short1-preview.webm', video.previewPath) - } - }) - }) - - describe('Should comment these videos', function () { - let childOfFirstChild: VideoCommentThreadTree - - it('Should add comment (threads and replies)', async function () { - this.timeout(25000) - - { - const text = 'my super first comment' - await servers[0].comments.createThread({ videoId: videoUUID, text }) - } - - { - const text = 'my super second comment' - await servers[2].comments.createThread({ videoId: videoUUID, text }) - } - - await waitJobs(servers) - - { - const threadId = await servers[1].comments.findCommentId({ videoId: videoUUID, text: 'my super first comment' }) - - const text = 'my super answer to thread 1' - await servers[1].comments.addReply({ videoId: videoUUID, toCommentId: threadId, text }) - } - - await waitJobs(servers) - - { - const threadId = await servers[2].comments.findCommentId({ videoId: videoUUID, text: 'my super first comment' }) - - const body = await servers[2].comments.getThread({ videoId: videoUUID, threadId }) - const childCommentId = body.children[0].comment.id - - const text3 = 'my second answer to thread 1' - await servers[2].comments.addReply({ videoId: videoUUID, toCommentId: threadId, text: text3 }) - - const text2 = 'my super answer to answer of thread 1' - await servers[2].comments.addReply({ videoId: videoUUID, toCommentId: childCommentId, text: text2 }) - } - - await waitJobs(servers) - }) - - it('Should have these threads', async function () { - for (const server of servers) { - const body = await server.comments.listThreads({ videoId: videoUUID }) - - expect(body.total).to.equal(2) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(2) - - { - const comment = body.data.find(c => c.text === 'my super first comment') - expect(comment).to.not.be.undefined - expect(comment.inReplyToCommentId).to.be.null - expect(comment.account.name).to.equal('root') - expect(comment.account.host).to.equal(servers[0].host) - expect(comment.totalReplies).to.equal(3) - expect(dateIsValid(comment.createdAt as string)).to.be.true - expect(dateIsValid(comment.updatedAt as string)).to.be.true - } - - { - const comment = body.data.find(c => c.text === 'my super second comment') - expect(comment).to.not.be.undefined - expect(comment.inReplyToCommentId).to.be.null - expect(comment.account.name).to.equal('root') - expect(comment.account.host).to.equal(servers[2].host) - expect(comment.totalReplies).to.equal(0) - expect(dateIsValid(comment.createdAt as string)).to.be.true - expect(dateIsValid(comment.updatedAt as string)).to.be.true - } - } - }) - - it('Should have these comments', async function () { - for (const server of servers) { - const body = await server.comments.listThreads({ videoId: videoUUID }) - const threadId = body.data.find(c => c.text === 'my super first comment').id - - const tree = await server.comments.getThread({ videoId: videoUUID, threadId }) - - expect(tree.comment.text).equal('my super first comment') - expect(tree.comment.account.name).equal('root') - expect(tree.comment.account.host).equal(servers[0].host) - expect(tree.children).to.have.lengthOf(2) - - const firstChild = tree.children[0] - expect(firstChild.comment.text).to.equal('my super answer to thread 1') - expect(firstChild.comment.account.name).equal('root') - expect(firstChild.comment.account.host).equal(servers[1].host) - expect(firstChild.children).to.have.lengthOf(1) - - childOfFirstChild = firstChild.children[0] - expect(childOfFirstChild.comment.text).to.equal('my super answer to answer of thread 1') - expect(childOfFirstChild.comment.account.name).equal('root') - expect(childOfFirstChild.comment.account.host).equal(servers[2].host) - expect(childOfFirstChild.children).to.have.lengthOf(0) - - const secondChild = tree.children[1] - expect(secondChild.comment.text).to.equal('my second answer to thread 1') - expect(secondChild.comment.account.name).equal('root') - expect(secondChild.comment.account.host).equal(servers[2].host) - expect(secondChild.children).to.have.lengthOf(0) - } - }) - - it('Should delete a reply', async function () { - this.timeout(30000) - - await servers[2].comments.delete({ videoId: videoUUID, commentId: childOfFirstChild.comment.id }) - - await waitJobs(servers) - }) - - it('Should have this comment marked as deleted', async function () { - for (const server of servers) { - const { data } = await server.comments.listThreads({ videoId: videoUUID }) - const threadId = data.find(c => c.text === 'my super first comment').id - - const tree = await server.comments.getThread({ videoId: videoUUID, threadId }) - expect(tree.comment.text).equal('my super first comment') - - const firstChild = tree.children[0] - expect(firstChild.comment.text).to.equal('my super answer to thread 1') - expect(firstChild.children).to.have.lengthOf(1) - - const deletedComment = firstChild.children[0].comment - expect(deletedComment.isDeleted).to.be.true - expect(deletedComment.deletedAt).to.not.be.null - expect(deletedComment.account).to.be.null - expect(deletedComment.text).to.equal('') - - const secondChild = tree.children[1] - expect(secondChild.comment.text).to.equal('my second answer to thread 1') - } - }) - - it('Should delete the thread comments', async function () { - this.timeout(30000) - - const { data } = await servers[0].comments.listThreads({ videoId: videoUUID }) - const commentId = data.find(c => c.text === 'my super first comment').id - await servers[0].comments.delete({ videoId: videoUUID, commentId }) - - await waitJobs(servers) - }) - - it('Should have the threads marked as deleted on other servers too', async function () { - for (const server of servers) { - const body = await server.comments.listThreads({ videoId: videoUUID }) - - expect(body.total).to.equal(2) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(2) - - { - const comment = body.data[0] - expect(comment).to.not.be.undefined - expect(comment.inReplyToCommentId).to.be.null - expect(comment.account.name).to.equal('root') - expect(comment.account.host).to.equal(servers[2].host) - expect(comment.totalReplies).to.equal(0) - expect(dateIsValid(comment.createdAt as string)).to.be.true - expect(dateIsValid(comment.updatedAt as string)).to.be.true - } - - { - const deletedComment = body.data[1] - expect(deletedComment).to.not.be.undefined - expect(deletedComment.isDeleted).to.be.true - expect(deletedComment.deletedAt).to.not.be.null - expect(deletedComment.text).to.equal('') - expect(deletedComment.inReplyToCommentId).to.be.null - expect(deletedComment.account).to.be.null - expect(deletedComment.totalReplies).to.equal(2) - expect(dateIsValid(deletedComment.createdAt as string)).to.be.true - expect(dateIsValid(deletedComment.updatedAt as string)).to.be.true - expect(dateIsValid(deletedComment.deletedAt as string)).to.be.true - } - } - }) - - it('Should delete a remote thread by the origin server', async function () { - this.timeout(5000) - - const { data } = await servers[0].comments.listThreads({ videoId: videoUUID }) - const commentId = data.find(c => c.text === 'my super second comment').id - await servers[0].comments.delete({ videoId: videoUUID, commentId }) - - await waitJobs(servers) - }) - - it('Should have the threads marked as deleted on other servers too', async function () { - for (const server of servers) { - const body = await server.comments.listThreads({ videoId: videoUUID }) - - expect(body.total).to.equal(2) - expect(body.data).to.have.lengthOf(2) - - { - const comment = body.data[0] - expect(comment.text).to.equal('') - expect(comment.isDeleted).to.be.true - expect(comment.createdAt).to.not.be.null - expect(comment.deletedAt).to.not.be.null - expect(comment.account).to.be.null - expect(comment.totalReplies).to.equal(0) - } - - { - const comment = body.data[1] - expect(comment.text).to.equal('') - expect(comment.isDeleted).to.be.true - expect(comment.createdAt).to.not.be.null - expect(comment.deletedAt).to.not.be.null - expect(comment.account).to.be.null - expect(comment.totalReplies).to.equal(2) - } - } - }) - - it('Should disable comments and download', async function () { - this.timeout(20000) - - const attributes = { - commentsEnabled: false, - downloadEnabled: false - } - - await servers[0].videos.update({ id: videoUUID, attributes }) - - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - expect(video.commentsEnabled).to.be.false - expect(video.downloadEnabled).to.be.false - - const text = 'my super forbidden comment' - await server.comments.createThread({ videoId: videoUUID, text, expectedStatus: HttpStatusCode.CONFLICT_409 }) - } - }) - }) - - describe('With minimum parameters', function () { - it('Should upload and propagate the video', async function () { - this.timeout(120000) - - const path = '/api/v1/videos/upload' - - const req = request(servers[1].url) - .post(path) - .set('Accept', 'application/json') - .set('Authorization', 'Bearer ' + servers[1].accessToken) - .field('name', 'minimum parameters') - .field('privacy', '1') - .field('channelId', '1') - - await req.attach('videofile', buildAbsoluteFixturePath('video_short.webm')) - .expect(HttpStatusCode.OK_200) - - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.videos.list() - const video = data.find(v => v.name === 'minimum parameters') - - const isLocal = server.url === servers[1].url - const checkAttributes = { - name: 'minimum parameters', - category: null, - licence: null, - language: null, - nsfw: false, - description: null, - support: null, - account: { - name: 'root', - host: servers[1].host - }, - isLocal, - duration: 5, - commentsEnabled: true, - downloadEnabled: true, - tags: [], - privacy: VideoPrivacy.PUBLIC, - channel: { - displayName: 'Main root channel', - name: 'root_channel', - description: '', - isLocal - }, - fixture: 'video_short.webm', - files: [ - { - resolution: 720, - size: 61000 - }, - { - resolution: 480, - size: 40000 - }, - { - resolution: 360, - size: 32000 - }, - { - resolution: 240, - size: 23000 - } - ] - } - await completeVideoCheck({ server, originServer: servers[1], videoUUID: video.uuid, attributes: checkAttributes }) - } - }) - }) - - describe('TMP directory', function () { - it('Should have an empty tmp directory', async function () { - for (const server of servers) { - await checkTmpIsEmpty(server) - } - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/videos/resumable-upload.ts b/server/tests/api/videos/resumable-upload.ts deleted file mode 100644 index cac1201e9..000000000 --- a/server/tests/api/videos/resumable-upload.ts +++ /dev/null @@ -1,310 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { pathExists, readdir, stat } from 'fs-extra' -import { join } from 'path' -import { buildAbsoluteFixturePath } from '@shared/core-utils' -import { sha1 } from '@shared/extra-utils' -import { HttpStatusCode, VideoPrivacy } from '@shared/models' -import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers, setDefaultVideoChannel } from '@shared/server-commands' - -// Most classic resumable upload tests are done in other test suites - -describe('Test resumable upload', function () { - const path = '/api/v1/videos/upload-resumable' - const defaultFixture = 'video_short.mp4' - let server: PeerTubeServer - let rootId: number - let userAccessToken: string - let userChannelId: number - - async function buildSize (fixture: string, size?: number) { - if (size !== undefined) return size - - const baseFixture = buildAbsoluteFixturePath(fixture) - return (await stat(baseFixture)).size - } - - async function prepareUpload (options: { - channelId?: number - token?: string - size?: number - originalName?: string - lastModified?: number - } = {}) { - const { token, originalName, lastModified } = options - - const size = await buildSize(defaultFixture, options.size) - - const attributes = { - name: 'video', - channelId: options.channelId ?? server.store.channel.id, - privacy: VideoPrivacy.PUBLIC, - fixture: defaultFixture - } - - const mimetype = 'video/mp4' - - const res = await server.videos.prepareResumableUpload({ path, token, attributes, size, mimetype, originalName, lastModified }) - - return res.header['location'].split('?')[1] - } - - async function sendChunks (options: { - token?: string - pathUploadId: string - size?: number - expectedStatus?: HttpStatusCode - contentLength?: number - contentRange?: string - contentRangeBuilder?: (start: number, chunk: any) => string - digestBuilder?: (chunk: any) => string - }) { - const { token, pathUploadId, expectedStatus, contentLength, contentRangeBuilder, digestBuilder } = options - - const size = await buildSize(defaultFixture, options.size) - const absoluteFilePath = buildAbsoluteFixturePath(defaultFixture) - - return server.videos.sendResumableChunks({ - token, - path, - pathUploadId, - videoFilePath: absoluteFilePath, - size, - contentLength, - contentRangeBuilder, - digestBuilder, - expectedStatus - }) - } - - async function checkFileSize (uploadIdArg: string, expectedSize: number | null) { - const uploadId = uploadIdArg.replace(/^upload_id=/, '') - - const subPath = join('tmp', 'resumable-uploads', `${rootId}-${uploadId}.mp4`) - const filePath = server.servers.buildDirectory(subPath) - const exists = await pathExists(filePath) - - if (expectedSize === null) { - expect(exists).to.be.false - return - } - - expect(exists).to.be.true - - expect((await stat(filePath)).size).to.equal(expectedSize) - } - - async function countResumableUploads (wait?: number) { - const subPath = join('tmp', 'resumable-uploads') - const filePath = server.servers.buildDirectory(subPath) - await new Promise(resolve => setTimeout(resolve, wait)) - const files = await readdir(filePath) - return files.length - } - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - const body = await server.users.getMyInfo() - rootId = body.id - - { - userAccessToken = await server.users.generateUserAndToken('user1') - const { videoChannels } = await server.users.getMyInfo({ token: userAccessToken }) - userChannelId = videoChannels[0].id - } - - await server.users.update({ userId: rootId, videoQuota: 10_000_000 }) - }) - - describe('Directory cleaning', function () { - - it('Should correctly delete files after an upload', async function () { - const uploadId = await prepareUpload() - await sendChunks({ pathUploadId: uploadId }) - await server.videos.endResumableUpload({ path, pathUploadId: uploadId }) - - expect(await countResumableUploads()).to.equal(0) - }) - - it('Should correctly delete corrupt files', async function () { - const uploadId = await prepareUpload({ size: 8 * 1024 }) - await sendChunks({ pathUploadId: uploadId, size: 8 * 1024, expectedStatus: HttpStatusCode.UNPROCESSABLE_ENTITY_422 }) - - expect(await countResumableUploads(2000)).to.equal(0) - }) - - it('Should not delete files after an unfinished upload', async function () { - await prepareUpload() - - expect(await countResumableUploads()).to.equal(2) - }) - - it('Should not delete recent uploads', async function () { - await server.debug.sendCommand({ body: { command: 'remove-dandling-resumable-uploads' } }) - - expect(await countResumableUploads()).to.equal(2) - }) - - it('Should delete old uploads', async function () { - await server.debug.sendCommand({ body: { command: 'remove-dandling-resumable-uploads' } }) - - expect(await countResumableUploads()).to.equal(0) - }) - }) - - describe('Resumable upload and chunks', function () { - - it('Should accept the same amount of chunks', async function () { - const uploadId = await prepareUpload() - await sendChunks({ pathUploadId: uploadId }) - - await checkFileSize(uploadId, null) - }) - - it('Should not accept more chunks than expected', async function () { - const uploadId = await prepareUpload({ size: 100 }) - - await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.CONFLICT_409 }) - await checkFileSize(uploadId, 0) - }) - - it('Should not accept more chunks than expected with an invalid content length/content range', async function () { - const uploadId = await prepareUpload({ size: 1500 }) - - // Content length check can be different depending on the node version - try { - await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.CONFLICT_409, contentLength: 1000 }) - await checkFileSize(uploadId, 0) - } catch { - await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.BAD_REQUEST_400, contentLength: 1000 }) - await checkFileSize(uploadId, 0) - } - }) - - it('Should not accept more chunks than expected with an invalid content length', async function () { - const uploadId = await prepareUpload({ size: 500 }) - - const size = 1000 - - // Content length check seems to have changed in v16 - const expectedStatus = process.version.startsWith('v16') - ? HttpStatusCode.CONFLICT_409 - : HttpStatusCode.BAD_REQUEST_400 - - const contentRangeBuilder = (start: number) => `bytes ${start}-${start + size - 1}/${size}` - await sendChunks({ pathUploadId: uploadId, expectedStatus, contentRangeBuilder, contentLength: size }) - await checkFileSize(uploadId, 0) - }) - - it('Should be able to accept 2 PUT requests', async function () { - const uploadId = await prepareUpload() - - const result1 = await sendChunks({ pathUploadId: uploadId }) - const result2 = await sendChunks({ pathUploadId: uploadId }) - - expect(result1.body.video.uuid).to.exist - expect(result1.body.video.uuid).to.equal(result2.body.video.uuid) - - expect(result1.headers['x-resumable-upload-cached']).to.not.exist - expect(result2.headers['x-resumable-upload-cached']).to.equal('true') - - await checkFileSize(uploadId, null) - }) - - it('Should not have the same upload id with 2 different users', async function () { - const originalName = 'toto.mp4' - const lastModified = new Date().getTime() - - const uploadId1 = await prepareUpload({ originalName, lastModified, token: server.accessToken }) - const uploadId2 = await prepareUpload({ originalName, lastModified, channelId: userChannelId, token: userAccessToken }) - - expect(uploadId1).to.not.equal(uploadId2) - }) - - it('Should have the same upload id with the same user', async function () { - const originalName = 'toto.mp4' - const lastModified = new Date().getTime() - - const uploadId1 = await prepareUpload({ originalName, lastModified }) - const uploadId2 = await prepareUpload({ originalName, lastModified }) - - expect(uploadId1).to.equal(uploadId2) - }) - - it('Should not cache a request with 2 different users', async function () { - const originalName = 'toto.mp4' - const lastModified = new Date().getTime() - - const uploadId = await prepareUpload({ originalName, lastModified, token: server.accessToken }) - - await sendChunks({ pathUploadId: uploadId, token: server.accessToken }) - await sendChunks({ pathUploadId: uploadId, token: userAccessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should not cache a request after a delete', async function () { - const originalName = 'toto.mp4' - const lastModified = new Date().getTime() - const uploadId1 = await prepareUpload({ originalName, lastModified, token: server.accessToken }) - - await sendChunks({ pathUploadId: uploadId1 }) - await server.videos.endResumableUpload({ path, pathUploadId: uploadId1 }) - - const uploadId2 = await prepareUpload({ originalName, lastModified, token: server.accessToken }) - expect(uploadId1).to.equal(uploadId2) - - const result2 = await sendChunks({ pathUploadId: uploadId1 }) - expect(result2.headers['x-resumable-upload-cached']).to.not.exist - }) - - it('Should not cache after video deletion', async function () { - const originalName = 'toto.mp4' - const lastModified = new Date().getTime() - - const uploadId1 = await prepareUpload({ originalName, lastModified }) - const result1 = await sendChunks({ pathUploadId: uploadId1 }) - await server.videos.remove({ id: result1.body.video.uuid }) - - const uploadId2 = await prepareUpload({ originalName, lastModified }) - const result2 = await sendChunks({ pathUploadId: uploadId2 }) - expect(result1.body.video.uuid).to.not.equal(result2.body.video.uuid) - - expect(result2.headers['x-resumable-upload-cached']).to.not.exist - - await checkFileSize(uploadId1, null) - await checkFileSize(uploadId2, null) - }) - - it('Should refuse an invalid digest', async function () { - const uploadId = await prepareUpload({ token: server.accessToken }) - - await sendChunks({ - pathUploadId: uploadId, - token: server.accessToken, - digestBuilder: () => 'sha=' + 'a'.repeat(40), - expectedStatus: 460 as any - }) - }) - - it('Should accept an appropriate digest', async function () { - const uploadId = await prepareUpload({ token: server.accessToken }) - - await sendChunks({ - pathUploadId: uploadId, - token: server.accessToken, - digestBuilder: (chunk: Buffer) => { - return 'sha1=' + sha1(chunk, 'base64') - } - }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/videos/single-server.ts b/server/tests/api/videos/single-server.ts deleted file mode 100644 index 66414aa5b..000000000 --- a/server/tests/api/videos/single-server.ts +++ /dev/null @@ -1,460 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { checkVideoFilesWereRemoved, completeVideoCheck, testImageGeneratedByFFmpeg } from '@server/tests/shared' -import { wait } from '@shared/core-utils' -import { Video, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createSingleServer, - PeerTubeServer, - setAccessTokensToServers, - setDefaultAccountAvatar, - setDefaultChannelAvatar, - waitJobs -} from '@shared/server-commands' - -describe('Test a single server', function () { - - function runSuite (mode: 'legacy' | 'resumable') { - let server: PeerTubeServer = null - let videoId: number | string - let videoId2: string - let videoUUID = '' - let videosListBase: any[] = null - - const getCheckAttributes = () => ({ - name: 'my super name', - category: 2, - licence: 6, - language: 'zh', - nsfw: true, - description: 'my super description', - support: 'my super support text', - account: { - name: 'root', - host: server.host - }, - isLocal: true, - duration: 5, - tags: [ 'tag1', 'tag2', 'tag3' ], - privacy: VideoPrivacy.PUBLIC, - commentsEnabled: true, - downloadEnabled: true, - channel: { - displayName: 'Main root channel', - name: 'root_channel', - description: '', - isLocal: true - }, - fixture: 'video_short.webm', - files: [ - { - resolution: 720, - size: 218910 - } - ] - }) - - const updateCheckAttributes = () => ({ - name: 'my super video updated', - category: 4, - licence: 2, - language: 'ar', - nsfw: false, - description: 'my super description updated', - support: 'my super support text updated', - account: { - name: 'root', - host: server.host - }, - isLocal: true, - tags: [ 'tagup1', 'tagup2' ], - privacy: VideoPrivacy.PUBLIC, - duration: 5, - commentsEnabled: false, - downloadEnabled: false, - channel: { - name: 'root_channel', - displayName: 'Main root channel', - description: '', - isLocal: true - }, - fixture: 'video_short3.webm', - files: [ - { - resolution: 720, - size: 292677 - } - ] - }) - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - await setDefaultChannelAvatar(server) - await setDefaultAccountAvatar(server) - }) - - it('Should list video categories', async function () { - const categories = await server.videos.getCategories() - expect(Object.keys(categories)).to.have.length.above(10) - - expect(categories[11]).to.equal('News & Politics') - }) - - it('Should list video licences', async function () { - const licences = await server.videos.getLicences() - expect(Object.keys(licences)).to.have.length.above(5) - - expect(licences[3]).to.equal('Attribution - No Derivatives') - }) - - it('Should list video languages', async function () { - const languages = await server.videos.getLanguages() - expect(Object.keys(languages)).to.have.length.above(5) - - expect(languages['ru']).to.equal('Russian') - }) - - it('Should list video privacies', async function () { - const privacies = await server.videos.getPrivacies() - expect(Object.keys(privacies)).to.have.length.at.least(3) - - expect(privacies[3]).to.equal('Private') - }) - - it('Should not have videos', async function () { - const { data, total } = await server.videos.list() - - expect(total).to.equal(0) - expect(data).to.be.an('array') - expect(data.length).to.equal(0) - }) - - it('Should upload the video', async function () { - const attributes = { - name: 'my super name', - category: 2, - nsfw: true, - licence: 6, - tags: [ 'tag1', 'tag2', 'tag3' ] - } - const video = await server.videos.upload({ attributes, mode }) - expect(video).to.not.be.undefined - expect(video.id).to.equal(1) - expect(video.uuid).to.have.length.above(5) - - videoId = video.id - videoUUID = video.uuid - }) - - it('Should get and seed the uploaded video', async function () { - this.timeout(5000) - - const { data, total } = await server.videos.list() - - expect(total).to.equal(1) - expect(data).to.be.an('array') - expect(data.length).to.equal(1) - - const video = data[0] - await completeVideoCheck({ server, originServer: server, videoUUID: video.uuid, attributes: getCheckAttributes() }) - }) - - it('Should get the video by UUID', async function () { - this.timeout(5000) - - const video = await server.videos.get({ id: videoUUID }) - await completeVideoCheck({ server, originServer: server, videoUUID: video.uuid, attributes: getCheckAttributes() }) - }) - - it('Should have the views updated', async function () { - this.timeout(20000) - - await server.views.simulateView({ id: videoId }) - await server.views.simulateView({ id: videoId }) - await server.views.simulateView({ id: videoId }) - - await wait(1500) - - await server.views.simulateView({ id: videoId }) - await server.views.simulateView({ id: videoId }) - - await wait(1500) - - await server.views.simulateView({ id: videoId }) - await server.views.simulateView({ id: videoId }) - - await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } }) - - const video = await server.videos.get({ id: videoId }) - expect(video.views).to.equal(3) - }) - - it('Should remove the video', async function () { - const video = await server.videos.get({ id: videoId }) - await server.videos.remove({ id: videoId }) - - await checkVideoFilesWereRemoved({ video, server }) - }) - - it('Should not have videos', async function () { - const { total, data } = await server.videos.list() - - expect(total).to.equal(0) - expect(data).to.be.an('array') - expect(data).to.have.lengthOf(0) - }) - - it('Should upload 6 videos', async function () { - this.timeout(120000) - - const videos = new Set([ - 'video_short.mp4', 'video_short.ogv', 'video_short.webm', - 'video_short1.webm', 'video_short2.webm', 'video_short3.webm' - ]) - - for (const video of videos) { - const attributes = { - name: video + ' name', - description: video + ' description', - category: 2, - licence: 1, - language: 'en', - nsfw: true, - tags: [ 'tag1', 'tag2', 'tag3' ], - fixture: video - } - - await server.videos.upload({ attributes, mode }) - } - }) - - it('Should have the correct durations', async function () { - const { total, data } = await server.videos.list() - - expect(total).to.equal(6) - expect(data).to.be.an('array') - expect(data).to.have.lengthOf(6) - - const videosByName: { [ name: string ]: Video } = {} - data.forEach(v => { videosByName[v.name] = v }) - - expect(videosByName['video_short.mp4 name'].duration).to.equal(5) - expect(videosByName['video_short.ogv name'].duration).to.equal(5) - expect(videosByName['video_short.webm name'].duration).to.equal(5) - expect(videosByName['video_short1.webm name'].duration).to.equal(10) - expect(videosByName['video_short2.webm name'].duration).to.equal(5) - expect(videosByName['video_short3.webm name'].duration).to.equal(5) - }) - - it('Should have the correct thumbnails', async function () { - const { data } = await server.videos.list() - - // For the next test - videosListBase = data - - for (const video of data) { - const videoName = video.name.replace(' name', '') - await testImageGeneratedByFFmpeg(server.url, videoName, video.thumbnailPath) - } - }) - - it('Should list only the two first videos', async function () { - const { total, data } = await server.videos.list({ start: 0, count: 2, sort: 'name' }) - - expect(total).to.equal(6) - expect(data.length).to.equal(2) - expect(data[0].name).to.equal(videosListBase[0].name) - expect(data[1].name).to.equal(videosListBase[1].name) - }) - - it('Should list only the next three videos', async function () { - const { total, data } = await server.videos.list({ start: 2, count: 3, sort: 'name' }) - - expect(total).to.equal(6) - expect(data.length).to.equal(3) - expect(data[0].name).to.equal(videosListBase[2].name) - expect(data[1].name).to.equal(videosListBase[3].name) - expect(data[2].name).to.equal(videosListBase[4].name) - }) - - it('Should list the last video', async function () { - const { total, data } = await server.videos.list({ start: 5, count: 6, sort: 'name' }) - - expect(total).to.equal(6) - expect(data.length).to.equal(1) - expect(data[0].name).to.equal(videosListBase[5].name) - }) - - it('Should not have the total field', async function () { - const { total, data } = await server.videos.list({ start: 5, count: 6, sort: 'name', skipCount: true }) - - expect(total).to.not.exist - expect(data.length).to.equal(1) - expect(data[0].name).to.equal(videosListBase[5].name) - }) - - it('Should list and sort by name in descending order', async function () { - const { total, data } = await server.videos.list({ sort: '-name' }) - - expect(total).to.equal(6) - expect(data.length).to.equal(6) - expect(data[0].name).to.equal('video_short.webm name') - expect(data[1].name).to.equal('video_short.ogv name') - expect(data[2].name).to.equal('video_short.mp4 name') - expect(data[3].name).to.equal('video_short3.webm name') - expect(data[4].name).to.equal('video_short2.webm name') - expect(data[5].name).to.equal('video_short1.webm name') - - videoId = data[3].uuid - videoId2 = data[5].uuid - }) - - it('Should list and sort by trending in descending order', async function () { - const { total, data } = await server.videos.list({ start: 0, count: 2, sort: '-trending' }) - - expect(total).to.equal(6) - expect(data.length).to.equal(2) - }) - - it('Should list and sort by hotness in descending order', async function () { - const { total, data } = await server.videos.list({ start: 0, count: 2, sort: '-hot' }) - - expect(total).to.equal(6) - expect(data.length).to.equal(2) - }) - - it('Should list and sort by best in descending order', async function () { - const { total, data } = await server.videos.list({ start: 0, count: 2, sort: '-best' }) - - expect(total).to.equal(6) - expect(data.length).to.equal(2) - }) - - it('Should update a video', async function () { - const attributes = { - name: 'my super video updated', - category: 4, - licence: 2, - language: 'ar', - nsfw: false, - description: 'my super description updated', - commentsEnabled: false, - downloadEnabled: false, - tags: [ 'tagup1', 'tagup2' ] - } - await server.videos.update({ id: videoId, attributes }) - }) - - it('Should have the video updated', async function () { - this.timeout(60000) - - await waitJobs([ server ]) - - const video = await server.videos.get({ id: videoId }) - - await completeVideoCheck({ server, originServer: server, videoUUID: video.uuid, attributes: updateCheckAttributes() }) - }) - - it('Should update only the tags of a video', async function () { - const attributes = { - tags: [ 'supertag', 'tag1', 'tag2' ] - } - await server.videos.update({ id: videoId, attributes }) - - const video = await server.videos.get({ id: videoId }) - - await completeVideoCheck({ - server, - originServer: server, - videoUUID: video.uuid, - attributes: Object.assign(updateCheckAttributes(), attributes) - }) - }) - - it('Should update only the description of a video', async function () { - const attributes = { - description: 'hello everybody' - } - await server.videos.update({ id: videoId, attributes }) - - const video = await server.videos.get({ id: videoId }) - - await completeVideoCheck({ - server, - originServer: server, - videoUUID: video.uuid, - attributes: Object.assign(updateCheckAttributes(), { tags: [ 'supertag', 'tag1', 'tag2' ] }, attributes) - }) - }) - - it('Should like a video', async function () { - await server.videos.rate({ id: videoId, rating: 'like' }) - - const video = await server.videos.get({ id: videoId }) - - expect(video.likes).to.equal(1) - expect(video.dislikes).to.equal(0) - }) - - it('Should dislike the same video', async function () { - await server.videos.rate({ id: videoId, rating: 'dislike' }) - - const video = await server.videos.get({ id: videoId }) - - expect(video.likes).to.equal(0) - expect(video.dislikes).to.equal(1) - }) - - it('Should sort by originallyPublishedAt', async function () { - { - const now = new Date() - const attributes = { originallyPublishedAt: now.toISOString() } - await server.videos.update({ id: videoId, attributes }) - - const { data } = await server.videos.list({ sort: '-originallyPublishedAt' }) - const names = data.map(v => v.name) - - expect(names[0]).to.equal('my super video updated') - expect(names[1]).to.equal('video_short2.webm name') - expect(names[2]).to.equal('video_short1.webm name') - expect(names[3]).to.equal('video_short.webm name') - expect(names[4]).to.equal('video_short.ogv name') - expect(names[5]).to.equal('video_short.mp4 name') - } - - { - const now = new Date() - const attributes = { originallyPublishedAt: now.toISOString() } - await server.videos.update({ id: videoId2, attributes }) - - const { data } = await server.videos.list({ sort: '-originallyPublishedAt' }) - const names = data.map(v => v.name) - - expect(names[0]).to.equal('video_short1.webm name') - expect(names[1]).to.equal('my super video updated') - expect(names[2]).to.equal('video_short2.webm name') - expect(names[3]).to.equal('video_short.webm name') - expect(names[4]).to.equal('video_short.ogv name') - expect(names[5]).to.equal('video_short.mp4 name') - } - }) - - after(async function () { - await cleanupTests([ server ]) - }) - } - - describe('Legacy upload', function () { - runSuite('legacy') - }) - - describe('Resumable upload', function () { - runSuite('resumable') - }) -}) diff --git a/server/tests/api/videos/video-captions.ts b/server/tests/api/videos/video-captions.ts deleted file mode 100644 index 0630c9d3a..000000000 --- a/server/tests/api/videos/video-captions.ts +++ /dev/null @@ -1,188 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { checkVideoFilesWereRemoved, testCaptionFile } from '@server/tests/shared' -import { wait } from '@shared/core-utils' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test video captions', function () { - const uuidRegex = '[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}' - - let servers: PeerTubeServer[] - let videoUUID: string - - before(async function () { - this.timeout(60000) - - servers = await createMultipleServers(2) - - await setAccessTokensToServers(servers) - await doubleFollow(servers[0], servers[1]) - - await waitJobs(servers) - - const { uuid } = await servers[0].videos.upload({ attributes: { name: 'my video name' } }) - videoUUID = uuid - - await waitJobs(servers) - }) - - it('Should list the captions and return an empty list', async function () { - for (const server of servers) { - const body = await server.captions.list({ videoId: videoUUID }) - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - } - }) - - it('Should create two new captions', async function () { - this.timeout(30000) - - await servers[0].captions.add({ - language: 'ar', - videoId: videoUUID, - fixture: 'subtitle-good1.vtt' - }) - - await servers[0].captions.add({ - language: 'zh', - videoId: videoUUID, - fixture: 'subtitle-good2.vtt', - mimeType: 'application/octet-stream' - }) - - await waitJobs(servers) - }) - - it('Should list these uploaded captions', async function () { - for (const server of servers) { - const body = await server.captions.list({ videoId: videoUUID }) - expect(body.total).to.equal(2) - expect(body.data).to.have.lengthOf(2) - - const caption1 = body.data[0] - expect(caption1.language.id).to.equal('ar') - expect(caption1.language.label).to.equal('Arabic') - expect(caption1.captionPath).to.match(new RegExp('^/lazy-static/video-captions/' + uuidRegex + '-ar.vtt$')) - await testCaptionFile(server.url, caption1.captionPath, 'Subtitle good 1.') - - const caption2 = body.data[1] - expect(caption2.language.id).to.equal('zh') - expect(caption2.language.label).to.equal('Chinese') - expect(caption2.captionPath).to.match(new RegExp('^/lazy-static/video-captions/' + uuidRegex + '-zh.vtt$')) - await testCaptionFile(server.url, caption2.captionPath, 'Subtitle good 2.') - } - }) - - it('Should replace an existing caption', async function () { - this.timeout(30000) - - await servers[0].captions.add({ - language: 'ar', - videoId: videoUUID, - fixture: 'subtitle-good2.vtt' - }) - - await waitJobs(servers) - }) - - it('Should have this caption updated', async function () { - for (const server of servers) { - const body = await server.captions.list({ videoId: videoUUID }) - expect(body.total).to.equal(2) - expect(body.data).to.have.lengthOf(2) - - const caption1 = body.data[0] - expect(caption1.language.id).to.equal('ar') - expect(caption1.language.label).to.equal('Arabic') - expect(caption1.captionPath).to.match(new RegExp('^/lazy-static/video-captions/' + uuidRegex + '-ar.vtt$')) - await testCaptionFile(server.url, caption1.captionPath, 'Subtitle good 2.') - } - }) - - it('Should replace an existing caption with a srt file and convert it', async function () { - this.timeout(30000) - - await servers[0].captions.add({ - language: 'ar', - videoId: videoUUID, - fixture: 'subtitle-good.srt' - }) - - await waitJobs(servers) - - // Cache invalidation - await wait(3000) - }) - - it('Should have this caption updated and converted', async function () { - for (const server of servers) { - const body = await server.captions.list({ videoId: videoUUID }) - expect(body.total).to.equal(2) - expect(body.data).to.have.lengthOf(2) - - const caption1 = body.data[0] - expect(caption1.language.id).to.equal('ar') - expect(caption1.language.label).to.equal('Arabic') - expect(caption1.captionPath).to.match(new RegExp('^/lazy-static/video-captions/' + uuidRegex + '-ar.vtt$')) - - const expected = 'WEBVTT FILE\r\n' + - '\r\n' + - '1\r\n' + - '00:00:01.600 --> 00:00:04.200\r\n' + - 'English (US)\r\n' + - '\r\n' + - '2\r\n' + - '00:00:05.900 --> 00:00:07.999\r\n' + - 'This is a subtitle in American English\r\n' + - '\r\n' + - '3\r\n' + - '00:00:10.000 --> 00:00:14.000\r\n' + - 'Adding subtitles is very easy to do\r\n' - await testCaptionFile(server.url, caption1.captionPath, expected) - } - }) - - it('Should remove one caption', async function () { - this.timeout(30000) - - await servers[0].captions.delete({ videoId: videoUUID, language: 'ar' }) - - await waitJobs(servers) - }) - - it('Should only list the caption that was not deleted', async function () { - for (const server of servers) { - const body = await server.captions.list({ videoId: videoUUID }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - - const caption = body.data[0] - - expect(caption.language.id).to.equal('zh') - expect(caption.language.label).to.equal('Chinese') - expect(caption.captionPath).to.match(new RegExp('^/lazy-static/video-captions/' + uuidRegex + '-zh.vtt$')) - await testCaptionFile(server.url, caption.captionPath, 'Subtitle good 2.') - } - }) - - it('Should remove the video, and thus all video captions', async function () { - const video = await servers[0].videos.get({ id: videoUUID }) - const { data: captions } = await servers[0].captions.list({ videoId: videoUUID }) - - await servers[0].videos.remove({ id: videoUUID }) - - await checkVideoFilesWereRemoved({ server: servers[0], video, captions }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/videos/video-change-ownership.ts b/server/tests/api/videos/video-change-ownership.ts deleted file mode 100644 index 99d774c2b..000000000 --- a/server/tests/api/videos/video-change-ownership.ts +++ /dev/null @@ -1,314 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { - ChangeOwnershipCommand, - cleanupTests, - createMultipleServers, - createSingleServer, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - waitJobs -} from '@shared/server-commands' -import { HttpStatusCode, VideoPrivacy } from '@shared/models' - -describe('Test video change ownership - nominal', function () { - let servers: PeerTubeServer[] = [] - - const firstUser = 'first' - const secondUser = 'second' - - let firstUserToken = '' - let firstUserChannelId: number - - let secondUserToken = '' - let secondUserChannelId: number - - let lastRequestId: number - - let liveId: number - - let command: ChangeOwnershipCommand - - before(async function () { - this.timeout(50000) - - servers = await createMultipleServers(2) - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - - await servers[0].config.updateCustomSubConfig({ - newConfig: { - transcoding: { - enabled: false - }, - live: { - enabled: true - } - } - }) - - firstUserToken = await servers[0].users.generateUserAndToken(firstUser) - secondUserToken = await servers[0].users.generateUserAndToken(secondUser) - - { - const { videoChannels } = await servers[0].users.getMyInfo({ token: firstUserToken }) - firstUserChannelId = videoChannels[0].id - } - - { - const { videoChannels } = await servers[0].users.getMyInfo({ token: secondUserToken }) - secondUserChannelId = videoChannels[0].id - } - - { - const attributes = { - name: 'my super name', - description: 'my super description' - } - const { id } = await servers[0].videos.upload({ token: firstUserToken, attributes }) - - servers[0].store.videoCreated = await servers[0].videos.get({ id }) - } - - { - const attributes = { name: 'live', channelId: firstUserChannelId, privacy: VideoPrivacy.PUBLIC } - const video = await servers[0].live.create({ token: firstUserToken, fields: attributes }) - - liveId = video.id - } - - command = servers[0].changeOwnership - - await doubleFollow(servers[0], servers[1]) - }) - - it('Should not have video change ownership', async function () { - { - const body = await command.list({ token: firstUserToken }) - - expect(body.total).to.equal(0) - expect(body.data).to.be.an('array') - expect(body.data.length).to.equal(0) - } - - { - const body = await command.list({ token: secondUserToken }) - - expect(body.total).to.equal(0) - expect(body.data).to.be.an('array') - expect(body.data.length).to.equal(0) - } - }) - - it('Should send a request to change ownership of a video', async function () { - this.timeout(15000) - - await command.create({ token: firstUserToken, videoId: servers[0].store.videoCreated.id, username: secondUser }) - }) - - it('Should only return a request to change ownership for the second user', async function () { - { - const body = await command.list({ token: firstUserToken }) - - expect(body.total).to.equal(0) - expect(body.data).to.be.an('array') - expect(body.data.length).to.equal(0) - } - - { - const body = await command.list({ token: secondUserToken }) - - expect(body.total).to.equal(1) - expect(body.data).to.be.an('array') - expect(body.data.length).to.equal(1) - - lastRequestId = body.data[0].id - } - }) - - it('Should accept the same change ownership request without crashing', async function () { - await command.create({ token: firstUserToken, videoId: servers[0].store.videoCreated.id, username: secondUser }) - }) - - it('Should not create multiple change ownership requests while one is waiting', async function () { - const body = await command.list({ token: secondUserToken }) - - expect(body.total).to.equal(1) - expect(body.data).to.be.an('array') - expect(body.data.length).to.equal(1) - }) - - it('Should not be possible to refuse the change of ownership from first user', async function () { - await command.refuse({ token: firstUserToken, ownershipId: lastRequestId, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should be possible to refuse the change of ownership from second user', async function () { - await command.refuse({ token: secondUserToken, ownershipId: lastRequestId }) - }) - - it('Should send a new request to change ownership of a video', async function () { - this.timeout(15000) - - await command.create({ token: firstUserToken, videoId: servers[0].store.videoCreated.id, username: secondUser }) - }) - - it('Should return two requests to change ownership for the second user', async function () { - { - const body = await command.list({ token: firstUserToken }) - - expect(body.total).to.equal(0) - expect(body.data).to.be.an('array') - expect(body.data.length).to.equal(0) - } - - { - const body = await command.list({ token: secondUserToken }) - - expect(body.total).to.equal(2) - expect(body.data).to.be.an('array') - expect(body.data.length).to.equal(2) - - lastRequestId = body.data[0].id - } - }) - - it('Should not be possible to accept the change of ownership from first user', async function () { - await command.accept({ - token: firstUserToken, - ownershipId: lastRequestId, - channelId: secondUserChannelId, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should be possible to accept the change of ownership from second user', async function () { - await command.accept({ token: secondUserToken, ownershipId: lastRequestId, channelId: secondUserChannelId }) - - await waitJobs(servers) - }) - - it('Should have the channel of the video updated', async function () { - for (const server of servers) { - const video = await server.videos.get({ id: servers[0].store.videoCreated.uuid }) - - expect(video.name).to.equal('my super name') - expect(video.channel.displayName).to.equal('Main second channel') - expect(video.channel.name).to.equal('second_channel') - } - }) - - it('Should send a request to change ownership of a live', async function () { - this.timeout(15000) - - await command.create({ token: firstUserToken, videoId: liveId, username: secondUser }) - - const body = await command.list({ token: secondUserToken }) - - expect(body.total).to.equal(3) - expect(body.data.length).to.equal(3) - - lastRequestId = body.data[0].id - }) - - it('Should accept a live ownership change', async function () { - this.timeout(20000) - - await command.accept({ token: secondUserToken, ownershipId: lastRequestId, channelId: secondUserChannelId }) - - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: servers[0].store.videoCreated.uuid }) - - expect(video.name).to.equal('my super name') - expect(video.channel.displayName).to.equal('Main second channel') - expect(video.channel.name).to.equal('second_channel') - } - }) - - after(async function () { - await cleanupTests(servers) - }) -}) - -describe('Test video change ownership - quota too small', function () { - let server: PeerTubeServer - const firstUser = 'first' - const secondUser = 'second' - - let firstUserToken = '' - let secondUserToken = '' - let lastRequestId: number - - before(async function () { - this.timeout(50000) - - // Run one server - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - - await server.users.create({ username: secondUser, videoQuota: 10 }) - - firstUserToken = await server.users.generateUserAndToken(firstUser) - secondUserToken = await server.login.getAccessToken(secondUser) - - // Upload some videos on the server - const attributes = { - name: 'my super name', - description: 'my super description' - } - await server.videos.upload({ token: firstUserToken, attributes }) - - await waitJobs(server) - - const { data } = await server.videos.list() - expect(data.length).to.equal(1) - - server.store.videoCreated = data.find(video => video.name === 'my super name') - }) - - it('Should send a request to change ownership of a video', async function () { - this.timeout(15000) - - await server.changeOwnership.create({ token: firstUserToken, videoId: server.store.videoCreated.id, username: secondUser }) - }) - - it('Should only return a request to change ownership for the second user', async function () { - { - const body = await server.changeOwnership.list({ token: firstUserToken }) - - expect(body.total).to.equal(0) - expect(body.data).to.be.an('array') - expect(body.data.length).to.equal(0) - } - - { - const body = await server.changeOwnership.list({ token: secondUserToken }) - - expect(body.total).to.equal(1) - expect(body.data).to.be.an('array') - expect(body.data.length).to.equal(1) - - lastRequestId = body.data[0].id - } - }) - - it('Should not be possible to accept the change of ownership from second user because of exceeded quota', async function () { - const { videoChannels } = await server.users.getMyInfo({ token: secondUserToken }) - const channelId = videoChannels[0].id - - await server.changeOwnership.accept({ - token: secondUserToken, - ownershipId: lastRequestId, - channelId, - expectedStatus: HttpStatusCode.PAYLOAD_TOO_LARGE_413 - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/videos/video-channel-syncs.ts b/server/tests/api/videos/video-channel-syncs.ts deleted file mode 100644 index 7f688c7d6..000000000 --- a/server/tests/api/videos/video-channel-syncs.ts +++ /dev/null @@ -1,320 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { FIXTURE_URLS, SQLCommand } from '@server/tests/shared' -import { areHttpImportTestsDisabled } from '@shared/core-utils' -import { VideoChannelSyncState, VideoInclude, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - getServerImportConfig, - PeerTubeServer, - setAccessTokensToServers, - setDefaultAccountAvatar, - setDefaultChannelAvatar, - setDefaultVideoChannel, - waitJobs -} from '@shared/server-commands' - -describe('Test channel synchronizations', function () { - if (areHttpImportTestsDisabled()) return - - function runSuite (mode: 'youtube-dl' | 'yt-dlp') { - - describe('Sync using ' + mode, function () { - let servers: PeerTubeServer[] - let sqlCommands: SQLCommand[] = [] - - let startTestDate: Date - - let rootChannelSyncId: number - const userInfo = { - accessToken: '', - username: 'user1', - channelName: 'user1_channel', - channelId: -1, - syncId: -1 - } - - async function changeDateForSync (channelSyncId: number, newDate: string) { - await sqlCommands[0].updateQuery( - `UPDATE "videoChannelSync" ` + - `SET "createdAt"='${newDate}', "lastSyncAt"='${newDate}' ` + - `WHERE id=${channelSyncId}` - ) - } - - async function listAllVideosOfChannel (channelName: string) { - return servers[0].videos.listByChannel({ - handle: channelName, - include: VideoInclude.NOT_PUBLISHED_STATE - }) - } - - async function forceSyncAll (videoChannelSyncId: number, fromDate = '1970-01-01') { - await changeDateForSync(videoChannelSyncId, fromDate) - - await servers[0].debug.sendCommand({ - body: { - command: 'process-video-channel-sync-latest' - } - }) - - await waitJobs(servers) - } - - before(async function () { - this.timeout(240_000) - - startTestDate = new Date() - - servers = await createMultipleServers(2, getServerImportConfig(mode)) - - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - await setDefaultChannelAvatar(servers) - await setDefaultAccountAvatar(servers) - - await servers[0].config.enableChannelSync() - - { - userInfo.accessToken = await servers[0].users.generateUserAndToken(userInfo.username) - - const { videoChannels } = await servers[0].users.getMyInfo({ token: userInfo.accessToken }) - userInfo.channelId = videoChannels[0].id - } - - sqlCommands = servers.map(s => new SQLCommand(s)) - }) - - it('Should fetch the latest channel videos of a remote channel', async function () { - this.timeout(120_000) - - { - const { video } = await servers[0].imports.importVideo({ - attributes: { - channelId: servers[0].store.channel.id, - privacy: VideoPrivacy.PUBLIC, - targetUrl: FIXTURE_URLS.youtube - } - }) - - expect(video.name).to.equal('small video - youtube') - expect(video.waitTranscoding).to.be.true - - const { total } = await listAllVideosOfChannel('root_channel') - expect(total).to.equal(1) - } - - const { videoChannelSync } = await servers[0].channelSyncs.create({ - attributes: { - externalChannelUrl: FIXTURE_URLS.youtubeChannel, - videoChannelId: servers[0].store.channel.id - } - }) - rootChannelSyncId = videoChannelSync.id - - await forceSyncAll(rootChannelSyncId) - - { - const { total, data } = await listAllVideosOfChannel('root_channel') - expect(total).to.equal(2) - expect(data[0].name).to.equal('test') - expect(data[0].waitTranscoding).to.be.true - } - }) - - it('Should add another synchronization', async function () { - const externalChannelUrl = FIXTURE_URLS.youtubeChannel + '?foo=bar' - - const { videoChannelSync } = await servers[0].channelSyncs.create({ - attributes: { - externalChannelUrl, - videoChannelId: servers[0].store.channel.id - } - }) - - expect(videoChannelSync.externalChannelUrl).to.equal(externalChannelUrl) - expect(videoChannelSync.channel.id).to.equal(servers[0].store.channel.id) - expect(videoChannelSync.channel.name).to.equal('root_channel') - expect(videoChannelSync.state.id).to.equal(VideoChannelSyncState.WAITING_FIRST_RUN) - expect(new Date(videoChannelSync.createdAt)).to.be.above(startTestDate).and.to.be.at.most(new Date()) - }) - - it('Should add a synchronization for another user', async function () { - const { videoChannelSync } = await servers[0].channelSyncs.create({ - attributes: { - externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux', - videoChannelId: userInfo.channelId - }, - token: userInfo.accessToken - }) - userInfo.syncId = videoChannelSync.id - }) - - it('Should not import a channel if not asked', async function () { - await waitJobs(servers) - - const { data } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username }) - - expect(data[0].state).to.contain({ - id: VideoChannelSyncState.WAITING_FIRST_RUN, - label: 'Waiting first run' - }) - }) - - it('Should only fetch the videos newer than the creation date', async function () { - this.timeout(120_000) - - await forceSyncAll(userInfo.syncId, '2019-03-01') - - const { data, total } = await listAllVideosOfChannel(userInfo.channelName) - - expect(total).to.equal(1) - expect(data[0].name).to.equal('test') - }) - - it('Should list channel synchronizations', async function () { - // Root - { - const { total, data } = await servers[0].channelSyncs.listByAccount({ accountName: 'root' }) - expect(total).to.equal(2) - - expect(data[0]).to.deep.contain({ - externalChannelUrl: FIXTURE_URLS.youtubeChannel, - state: { - id: VideoChannelSyncState.SYNCED, - label: 'Synchronized' - } - }) - - expect(new Date(data[0].lastSyncAt)).to.be.greaterThan(startTestDate) - - expect(data[0].channel).to.contain({ id: servers[0].store.channel.id }) - expect(data[1]).to.contain({ externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?foo=bar' }) - } - - // User - { - const { total, data } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username }) - expect(total).to.equal(1) - expect(data[0]).to.deep.contain({ - externalChannelUrl: FIXTURE_URLS.youtubeChannel + '?baz=qux', - state: { - id: VideoChannelSyncState.SYNCED, - label: 'Synchronized' - } - }) - } - }) - - it('Should list imports of a channel synchronization', async function () { - const { total, data } = await servers[0].imports.getMyVideoImports({ videoChannelSyncId: rootChannelSyncId }) - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - expect(data[0].video.name).to.equal('test') - }) - - it('Should remove user\'s channel synchronizations', async function () { - await servers[0].channelSyncs.delete({ channelSyncId: userInfo.syncId }) - - const { total } = await servers[0].channelSyncs.listByAccount({ accountName: userInfo.username }) - expect(total).to.equal(0) - }) - - // FIXME: youtube-dl/yt-dlp doesn't work when speicifying a port after the hostname - // it('Should import a remote PeerTube channel', async function () { - // this.timeout(240_000) - - // await servers[1].videos.quickUpload({ name: 'remote 1' }) - // await waitJobs(servers) - - // const { videoChannelSync } = await servers[0].channelSyncs.create({ - // attributes: { - // externalChannelUrl: servers[1].url + '/c/root_channel', - // videoChannelId: userInfo.channelId - // }, - // token: userInfo.accessToken - // }) - // await servers[0].channels.importVideos({ - // channelName: userInfo.channelName, - // externalChannelUrl: servers[1].url + '/c/root_channel', - // videoChannelSyncId: videoChannelSync.id, - // token: userInfo.accessToken - // }) - - // await waitJobs(servers) - - // const { data, total } = await servers[0].videos.listByChannel({ - // handle: userInfo.channelName, - // include: VideoInclude.NOT_PUBLISHED_STATE - // }) - - // expect(total).to.equal(2) - // expect(data[0].name).to.equal('remote 1') - // }) - - // it('Should keep synced a remote PeerTube channel', async function () { - // this.timeout(240_000) - - // await servers[1].videos.quickUpload({ name: 'remote 2' }) - // await waitJobs(servers) - - // await servers[0].debug.sendCommand({ - // body: { - // command: 'process-video-channel-sync-latest' - // } - // }) - - // await waitJobs(servers) - - // const { data, total } = await servers[0].videos.listByChannel({ - // handle: userInfo.channelName, - // include: VideoInclude.NOT_PUBLISHED_STATE - // }) - // expect(total).to.equal(2) - // expect(data[0].name).to.equal('remote 2') - // }) - - it('Should fetch the latest videos of a youtube playlist', async function () { - this.timeout(120_000) - - const { id: channelId } = await servers[0].channels.create({ - attributes: { - name: 'channel2' - } - }) - - const { videoChannelSync: { id: videoChannelSyncId } } = await servers[0].channelSyncs.create({ - attributes: { - externalChannelUrl: FIXTURE_URLS.youtubePlaylist, - videoChannelId: channelId - } - }) - - await forceSyncAll(videoChannelSyncId) - - { - - const { total, data } = await listAllVideosOfChannel('channel2') - expect(total).to.equal(2) - expect(data[0].name).to.equal('test') - expect(data[1].name).to.equal('small video - youtube') - } - }) - - after(async function () { - for (const sqlCommand of sqlCommands) { - await sqlCommand.cleanup() - } - - await cleanupTests(servers) - }) - }) - } - - // FIXME: suite is broken with youtube-dl - // runSuite('youtube-dl') - runSuite('yt-dlp') -}) diff --git a/server/tests/api/videos/video-channels.ts b/server/tests/api/videos/video-channels.ts deleted file mode 100644 index f7cf84618..000000000 --- a/server/tests/api/videos/video-channels.ts +++ /dev/null @@ -1,555 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { basename } from 'path' -import { ACTOR_IMAGES_SIZE } from '@server/initializers/constants' -import { SQLCommand, testFileExistsOrNot, testImage } from '@server/tests/shared' -import { wait } from '@shared/core-utils' -import { ActorImageType, User, VideoChannel } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - setDefaultAccountAvatar, - setDefaultVideoChannel, - waitJobs -} from '@shared/server-commands' - -async function findChannel (server: PeerTubeServer, channelId: number) { - const body = await server.channels.list({ sort: '-name' }) - - return body.data.find(c => c.id === channelId) -} - -describe('Test video channels', function () { - let servers: PeerTubeServer[] - let sqlCommands: SQLCommand[] = [] - - let userInfo: User - let secondVideoChannelId: number - let totoChannel: number - let videoUUID: string - let accountName: string - let secondUserChannelName: string - - const avatarPaths: { [ port: number ]: string } = {} - const bannerPaths: { [ port: number ]: string } = {} - - before(async function () { - this.timeout(60000) - - servers = await createMultipleServers(2) - - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - await setDefaultAccountAvatar(servers) - - await doubleFollow(servers[0], servers[1]) - - sqlCommands = servers.map(s => new SQLCommand(s)) - }) - - it('Should have one video channel (created with root)', async () => { - const body = await servers[0].channels.list({ start: 0, count: 2 }) - - expect(body.total).to.equal(1) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(1) - }) - - it('Should create another video channel', async function () { - this.timeout(30000) - - { - const videoChannel = { - name: 'second_video_channel', - displayName: 'second video channel', - description: 'super video channel description', - support: 'super video channel support text' - } - const created = await servers[0].channels.create({ attributes: videoChannel }) - secondVideoChannelId = created.id - } - - // The channel is 1 is propagated to servers 2 - { - const attributes = { name: 'my video name', channelId: secondVideoChannelId, support: 'video support field' } - const { uuid } = await servers[0].videos.upload({ attributes }) - videoUUID = uuid - } - - await waitJobs(servers) - }) - - it('Should have two video channels when getting my information', async () => { - userInfo = await servers[0].users.getMyInfo() - - expect(userInfo.videoChannels).to.be.an('array') - expect(userInfo.videoChannels).to.have.lengthOf(2) - - const videoChannels = userInfo.videoChannels - expect(videoChannels[0].name).to.equal('root_channel') - expect(videoChannels[0].displayName).to.equal('Main root channel') - - expect(videoChannels[1].name).to.equal('second_video_channel') - expect(videoChannels[1].displayName).to.equal('second video channel') - expect(videoChannels[1].description).to.equal('super video channel description') - expect(videoChannels[1].support).to.equal('super video channel support text') - - accountName = userInfo.account.name + '@' + userInfo.account.host - }) - - it('Should have two video channels when getting account channels on server 1', async function () { - const body = await servers[0].channels.listByAccount({ accountName }) - expect(body.total).to.equal(2) - - const videoChannels = body.data - - expect(videoChannels).to.be.an('array') - expect(videoChannels).to.have.lengthOf(2) - - expect(videoChannels[0].name).to.equal('root_channel') - expect(videoChannels[0].displayName).to.equal('Main root channel') - - expect(videoChannels[1].name).to.equal('second_video_channel') - expect(videoChannels[1].displayName).to.equal('second video channel') - expect(videoChannels[1].description).to.equal('super video channel description') - expect(videoChannels[1].support).to.equal('super video channel support text') - }) - - it('Should paginate and sort account channels', async function () { - { - const body = await servers[0].channels.listByAccount({ - accountName, - start: 0, - count: 1, - sort: 'createdAt' - }) - - expect(body.total).to.equal(2) - expect(body.data).to.have.lengthOf(1) - - const videoChannel: VideoChannel = body.data[0] - expect(videoChannel.name).to.equal('root_channel') - } - - { - const body = await servers[0].channels.listByAccount({ - accountName, - start: 0, - count: 1, - sort: '-createdAt' - }) - - expect(body.total).to.equal(2) - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].name).to.equal('second_video_channel') - } - - { - const body = await servers[0].channels.listByAccount({ - accountName, - start: 1, - count: 1, - sort: '-createdAt' - }) - - expect(body.total).to.equal(2) - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].name).to.equal('root_channel') - } - }) - - it('Should have one video channel when getting account channels on server 2', async function () { - const body = await servers[1].channels.listByAccount({ accountName }) - - expect(body.total).to.equal(1) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(1) - - const videoChannel = body.data[0] - expect(videoChannel.name).to.equal('second_video_channel') - expect(videoChannel.displayName).to.equal('second video channel') - expect(videoChannel.description).to.equal('super video channel description') - expect(videoChannel.support).to.equal('super video channel support text') - }) - - it('Should list video channels', async function () { - const body = await servers[0].channels.list({ start: 1, count: 1, sort: '-name' }) - - expect(body.total).to.equal(2) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].name).to.equal('root_channel') - expect(body.data[0].displayName).to.equal('Main root channel') - }) - - it('Should update video channel', async function () { - this.timeout(15000) - - const videoChannelAttributes = { - displayName: 'video channel updated', - description: 'video channel description updated', - support: 'support updated' - } - - await servers[0].channels.update({ channelName: 'second_video_channel', attributes: videoChannelAttributes }) - - await waitJobs(servers) - }) - - it('Should have video channel updated', async function () { - for (const server of servers) { - const body = await server.channels.list({ start: 0, count: 1, sort: '-name' }) - - expect(body.total).to.equal(2) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(1) - - expect(body.data[0].name).to.equal('second_video_channel') - expect(body.data[0].displayName).to.equal('video channel updated') - expect(body.data[0].description).to.equal('video channel description updated') - expect(body.data[0].support).to.equal('support updated') - } - }) - - it('Should not have updated the video support field', async function () { - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - expect(video.support).to.equal('video support field') - } - }) - - it('Should update another accounts video channel', async function () { - this.timeout(15000) - - const result = await servers[0].users.generate('second_user') - secondUserChannelName = result.userChannelName - - await servers[0].videos.quickUpload({ name: 'video', token: result.token }) - - const videoChannelAttributes = { - displayName: 'video channel updated', - description: 'video channel description updated', - support: 'support updated' - } - - await servers[0].channels.update({ channelName: secondUserChannelName, attributes: videoChannelAttributes }) - - await waitJobs(servers) - }) - - it('Should have another accounts video channel updated', async function () { - for (const server of servers) { - const body = await server.channels.get({ channelName: `${secondUserChannelName}@${servers[0].host}` }) - - expect(body.displayName).to.equal('video channel updated') - expect(body.description).to.equal('video channel description updated') - expect(body.support).to.equal('support updated') - } - }) - - it('Should update the channel support field and update videos too', async function () { - this.timeout(35000) - - const videoChannelAttributes = { - support: 'video channel support text updated', - bulkVideosSupportUpdate: true - } - - await servers[0].channels.update({ channelName: 'second_video_channel', attributes: videoChannelAttributes }) - - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - expect(video.support).to.equal(videoChannelAttributes.support) - } - }) - - it('Should update video channel avatar', async function () { - this.timeout(15000) - - const fixture = 'avatar.png' - - await servers[0].channels.updateImage({ - channelName: 'second_video_channel', - fixture, - type: 'avatar' - }) - - await waitJobs(servers) - - for (let i = 0; i < servers.length; i++) { - const server = servers[i] - - const videoChannel = await findChannel(server, secondVideoChannelId) - const expectedSizes = ACTOR_IMAGES_SIZE[ActorImageType.AVATAR] - - expect(videoChannel.avatars.length).to.equal(expectedSizes.length, 'Expected avatars to be generated in all sizes') - - for (const avatar of videoChannel.avatars) { - avatarPaths[server.port] = avatar.path - await testImage(server.url, `avatar-resized-${avatar.width}x${avatar.width}`, avatarPaths[server.port], '.png') - await testFileExistsOrNot(server, 'avatars', basename(avatarPaths[server.port]), true) - - const row = await sqlCommands[i].getActorImage(basename(avatarPaths[server.port])) - - expect(expectedSizes.some(({ height, width }) => row.height === height && row.width === width)).to.equal(true) - } - } - }) - - it('Should update video channel banner', async function () { - this.timeout(15000) - - const fixture = 'banner.jpg' - - await servers[0].channels.updateImage({ - channelName: 'second_video_channel', - fixture, - type: 'banner' - }) - - await waitJobs(servers) - - for (let i = 0; i < servers.length; i++) { - const server = servers[i] - - const videoChannel = await server.channels.get({ channelName: 'second_video_channel@' + servers[0].host }) - - bannerPaths[server.port] = videoChannel.banners[0].path - await testImage(server.url, 'banner-resized', bannerPaths[server.port]) - await testFileExistsOrNot(server, 'avatars', basename(bannerPaths[server.port]), true) - - const row = await sqlCommands[i].getActorImage(basename(bannerPaths[server.port])) - expect(row.height).to.equal(ACTOR_IMAGES_SIZE[ActorImageType.BANNER][0].height) - expect(row.width).to.equal(ACTOR_IMAGES_SIZE[ActorImageType.BANNER][0].width) - } - }) - - it('Should still correctly list channels', async function () { - { - const body = await servers[0].channels.list({ start: 1, count: 1, sort: 'createdAt' }) - - expect(body.total).to.equal(3) - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].name).to.equal('second_video_channel') - } - - { - const body = await servers[0].channels.listByAccount({ accountName, start: 1, count: 1, sort: 'createdAt' }) - - expect(body.total).to.equal(2) - expect(body.data).to.have.lengthOf(1) - expect(body.data[0].name).to.equal('second_video_channel') - } - }) - - it('Should delete the video channel avatar', async function () { - this.timeout(15000) - await servers[0].channels.deleteImage({ channelName: 'second_video_channel', type: 'avatar' }) - - await waitJobs(servers) - - for (const server of servers) { - const videoChannel = await findChannel(server, secondVideoChannelId) - await testFileExistsOrNot(server, 'avatars', basename(avatarPaths[server.port]), false) - - expect(videoChannel.avatars).to.be.empty - } - }) - - it('Should delete the video channel banner', async function () { - this.timeout(15000) - - await servers[0].channels.deleteImage({ channelName: 'second_video_channel', type: 'banner' }) - - await waitJobs(servers) - - for (const server of servers) { - const videoChannel = await findChannel(server, secondVideoChannelId) - await testFileExistsOrNot(server, 'avatars', basename(bannerPaths[server.port]), false) - - expect(videoChannel.banners).to.be.empty - } - }) - - it('Should list the second video channel videos', async function () { - for (const server of servers) { - const channelURI = 'second_video_channel@' + servers[0].host - const { total, data } = await server.videos.listByChannel({ handle: channelURI }) - - expect(total).to.equal(1) - expect(data).to.be.an('array') - expect(data).to.have.lengthOf(1) - expect(data[0].name).to.equal('my video name') - } - }) - - it('Should change the video channel of a video', async function () { - await servers[0].videos.update({ id: videoUUID, attributes: { channelId: servers[0].store.channel.id } }) - - await waitJobs(servers) - }) - - it('Should list the first video channel videos', async function () { - for (const server of servers) { - { - const secondChannelURI = 'second_video_channel@' + servers[0].host - const { total } = await server.videos.listByChannel({ handle: secondChannelURI }) - expect(total).to.equal(0) - } - - { - const channelURI = 'root_channel@' + servers[0].host - const { total, data } = await server.videos.listByChannel({ handle: channelURI }) - expect(total).to.equal(1) - - expect(data).to.be.an('array') - expect(data).to.have.lengthOf(1) - expect(data[0].name).to.equal('my video name') - } - } - }) - - it('Should delete video channel', async function () { - await servers[0].channels.delete({ channelName: 'second_video_channel' }) - }) - - it('Should have video channel deleted', async function () { - const body = await servers[0].channels.list({ start: 0, count: 10, sort: 'createdAt' }) - - expect(body.total).to.equal(2) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(2) - expect(body.data[0].displayName).to.equal('Main root channel') - expect(body.data[1].displayName).to.equal('video channel updated') - }) - - it('Should create the main channel with a suffix if there is a conflict', async function () { - { - const videoChannel = { name: 'toto_channel', displayName: 'My toto channel' } - const created = await servers[0].channels.create({ attributes: videoChannel }) - totoChannel = created.id - } - - { - await servers[0].users.create({ username: 'toto', password: 'password' }) - const accessToken = await servers[0].login.getAccessToken({ username: 'toto', password: 'password' }) - - const { videoChannels } = await servers[0].users.getMyInfo({ token: accessToken }) - expect(videoChannels[0].name).to.equal('toto_channel-1') - } - }) - - it('Should report correct channel views per days', async function () { - { - const { data } = await servers[0].channels.listByAccount({ accountName, withStats: true }) - - for (const channel of data) { - expect(channel).to.haveOwnProperty('viewsPerDay') - expect(channel.viewsPerDay).to.have.length(30 + 1) // daysPrior + today - - for (const v of channel.viewsPerDay) { - expect(v.date).to.be.an('string') - expect(v.views).to.equal(0) - } - } - } - - { - // video has been posted on channel servers[0].store.videoChannel.id since last update - await servers[0].views.simulateView({ id: videoUUID, xForwardedFor: '0.0.0.1,127.0.0.1' }) - await servers[0].views.simulateView({ id: videoUUID, xForwardedFor: '0.0.0.2,127.0.0.1' }) - - // Wait the repeatable job - await wait(8000) - - const { data } = await servers[0].channels.listByAccount({ accountName, withStats: true }) - const channelWithView = data.find(channel => channel.id === servers[0].store.channel.id) - expect(channelWithView.viewsPerDay.slice(-1)[0].views).to.equal(2) - } - }) - - it('Should report correct total views count', async function () { - // check if there's the property - { - const { data } = await servers[0].channels.listByAccount({ accountName, withStats: true }) - - for (const channel of data) { - expect(channel).to.haveOwnProperty('totalViews') - expect(channel.totalViews).to.be.a('number') - } - } - - // Check if the totalViews count can be updated - { - const { data } = await servers[0].channels.listByAccount({ accountName, withStats: true }) - const channelWithView = data.find(channel => channel.id === servers[0].store.channel.id) - expect(channelWithView.totalViews).to.equal(2) - } - }) - - it('Should report correct videos count', async function () { - const { data } = await servers[0].channels.listByAccount({ accountName, withStats: true }) - - const totoChannel = data.find(c => c.name === 'toto_channel') - const rootChannel = data.find(c => c.name === 'root_channel') - - expect(rootChannel.videosCount).to.equal(1) - expect(totoChannel.videosCount).to.equal(0) - }) - - it('Should search among account video channels', async function () { - { - const body = await servers[0].channels.listByAccount({ accountName, search: 'root' }) - expect(body.total).to.equal(1) - - const channels = body.data - expect(channels).to.have.lengthOf(1) - } - - { - const body = await servers[0].channels.listByAccount({ accountName, search: 'does not exist' }) - expect(body.total).to.equal(0) - - const channels = body.data - expect(channels).to.have.lengthOf(0) - } - }) - - it('Should list channels by updatedAt desc if a video has been uploaded', async function () { - this.timeout(30000) - - await servers[0].videos.upload({ attributes: { channelId: totoChannel } }) - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.channels.listByAccount({ accountName, sort: '-updatedAt' }) - - expect(data[0].name).to.equal('toto_channel') - expect(data[1].name).to.equal('root_channel') - } - - await servers[0].videos.upload({ attributes: { channelId: servers[0].store.channel.id } }) - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.channels.listByAccount({ accountName, sort: '-updatedAt' }) - - expect(data[0].name).to.equal('root_channel') - expect(data[1].name).to.equal('toto_channel') - } - }) - - after(async function () { - for (const sqlCommand of sqlCommands) { - await sqlCommand.cleanup() - } - - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/videos/video-comments.ts b/server/tests/api/videos/video-comments.ts deleted file mode 100644 index b7d5624a6..000000000 --- a/server/tests/api/videos/video-comments.ts +++ /dev/null @@ -1,335 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { dateIsValid, testImage } from '@server/tests/shared' -import { - cleanupTests, - CommentsCommand, - createSingleServer, - PeerTubeServer, - setAccessTokensToServers, - setDefaultAccountAvatar, - setDefaultChannelAvatar -} from '@shared/server-commands' - -describe('Test video comments', function () { - let server: PeerTubeServer - let videoId: number - let videoUUID: string - let threadId: number - let replyToDeleteId: number - - let userAccessTokenServer1: string - - let command: CommentsCommand - - before(async function () { - this.timeout(120000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - - const { id, uuid } = await server.videos.upload() - videoUUID = uuid - videoId = id - - await setDefaultChannelAvatar(server) - await setDefaultAccountAvatar(server) - - userAccessTokenServer1 = await server.users.generateUserAndToken('user1') - await setDefaultChannelAvatar(server, 'user1_channel') - await setDefaultAccountAvatar(server, userAccessTokenServer1) - - command = server.comments - }) - - describe('User comments', function () { - - it('Should not have threads on this video', async function () { - const body = await command.listThreads({ videoId: videoUUID }) - - expect(body.total).to.equal(0) - expect(body.totalNotDeletedComments).to.equal(0) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(0) - }) - - it('Should create a thread in this video', async function () { - const text = 'my super first comment' - - const comment = await command.createThread({ videoId: videoUUID, text }) - - expect(comment.inReplyToCommentId).to.be.null - expect(comment.text).equal('my super first comment') - expect(comment.videoId).to.equal(videoId) - expect(comment.id).to.equal(comment.threadId) - expect(comment.account.name).to.equal('root') - expect(comment.account.host).to.equal(server.host) - expect(comment.account.url).to.equal(server.url + '/accounts/root') - expect(comment.totalReplies).to.equal(0) - expect(comment.totalRepliesFromVideoAuthor).to.equal(0) - expect(dateIsValid(comment.createdAt as string)).to.be.true - expect(dateIsValid(comment.updatedAt as string)).to.be.true - }) - - it('Should list threads of this video', async function () { - const body = await command.listThreads({ videoId: videoUUID }) - - expect(body.total).to.equal(1) - expect(body.totalNotDeletedComments).to.equal(1) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(1) - - const comment = body.data[0] - expect(comment.inReplyToCommentId).to.be.null - expect(comment.text).equal('my super first comment') - expect(comment.videoId).to.equal(videoId) - expect(comment.id).to.equal(comment.threadId) - expect(comment.account.name).to.equal('root') - expect(comment.account.host).to.equal(server.host) - - for (const avatar of comment.account.avatars) { - await testImage(server.url, `avatar-resized-${avatar.width}x${avatar.width}`, avatar.path, '.png') - } - - expect(comment.totalReplies).to.equal(0) - expect(comment.totalRepliesFromVideoAuthor).to.equal(0) - expect(dateIsValid(comment.createdAt as string)).to.be.true - expect(dateIsValid(comment.updatedAt as string)).to.be.true - - threadId = comment.threadId - }) - - it('Should get all the thread created', async function () { - const body = await command.getThread({ videoId: videoUUID, threadId }) - - const rootComment = body.comment - expect(rootComment.inReplyToCommentId).to.be.null - expect(rootComment.text).equal('my super first comment') - expect(rootComment.videoId).to.equal(videoId) - expect(dateIsValid(rootComment.createdAt as string)).to.be.true - expect(dateIsValid(rootComment.updatedAt as string)).to.be.true - }) - - it('Should create multiple replies in this thread', async function () { - const text1 = 'my super answer to thread 1' - const created = await command.addReply({ videoId, toCommentId: threadId, text: text1 }) - const childCommentId = created.id - - const text2 = 'my super answer to answer of thread 1' - await command.addReply({ videoId, toCommentId: childCommentId, text: text2 }) - - const text3 = 'my second answer to thread 1' - await command.addReply({ videoId, toCommentId: threadId, text: text3 }) - }) - - it('Should get correctly the replies', async function () { - const tree = await command.getThread({ videoId: videoUUID, threadId }) - - expect(tree.comment.text).equal('my super first comment') - expect(tree.children).to.have.lengthOf(2) - - const firstChild = tree.children[0] - expect(firstChild.comment.text).to.equal('my super answer to thread 1') - expect(firstChild.children).to.have.lengthOf(1) - - const childOfFirstChild = firstChild.children[0] - expect(childOfFirstChild.comment.text).to.equal('my super answer to answer of thread 1') - expect(childOfFirstChild.children).to.have.lengthOf(0) - - const secondChild = tree.children[1] - expect(secondChild.comment.text).to.equal('my second answer to thread 1') - expect(secondChild.children).to.have.lengthOf(0) - - replyToDeleteId = secondChild.comment.id - }) - - it('Should create other threads', async function () { - const text1 = 'super thread 2' - await command.createThread({ videoId: videoUUID, text: text1 }) - - const text2 = 'super thread 3' - await command.createThread({ videoId: videoUUID, text: text2 }) - }) - - it('Should list the threads', async function () { - const body = await command.listThreads({ videoId: videoUUID, sort: 'createdAt' }) - - expect(body.total).to.equal(3) - expect(body.totalNotDeletedComments).to.equal(6) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(3) - - expect(body.data[0].text).to.equal('my super first comment') - expect(body.data[0].totalReplies).to.equal(3) - expect(body.data[1].text).to.equal('super thread 2') - expect(body.data[1].totalReplies).to.equal(0) - expect(body.data[2].text).to.equal('super thread 3') - expect(body.data[2].totalReplies).to.equal(0) - }) - - it('Should list the and sort them by total replies', async function () { - const body = await command.listThreads({ videoId: videoUUID, sort: 'totalReplies' }) - - expect(body.data[2].text).to.equal('my super first comment') - expect(body.data[2].totalReplies).to.equal(3) - }) - - it('Should delete a reply', async function () { - await command.delete({ videoId, commentId: replyToDeleteId }) - - { - const body = await command.listThreads({ videoId: videoUUID, sort: 'createdAt' }) - - expect(body.total).to.equal(3) - expect(body.totalNotDeletedComments).to.equal(5) - } - - { - const tree = await command.getThread({ videoId: videoUUID, threadId }) - - expect(tree.comment.text).equal('my super first comment') - expect(tree.children).to.have.lengthOf(2) - - const firstChild = tree.children[0] - expect(firstChild.comment.text).to.equal('my super answer to thread 1') - expect(firstChild.children).to.have.lengthOf(1) - - const childOfFirstChild = firstChild.children[0] - expect(childOfFirstChild.comment.text).to.equal('my super answer to answer of thread 1') - expect(childOfFirstChild.children).to.have.lengthOf(0) - - const deletedChildOfFirstChild = tree.children[1] - expect(deletedChildOfFirstChild.comment.text).to.equal('') - expect(deletedChildOfFirstChild.comment.isDeleted).to.be.true - expect(deletedChildOfFirstChild.comment.deletedAt).to.not.be.null - expect(deletedChildOfFirstChild.comment.account).to.be.null - expect(deletedChildOfFirstChild.children).to.have.lengthOf(0) - } - }) - - it('Should delete a complete thread', async function () { - await command.delete({ videoId, commentId: threadId }) - - const body = await command.listThreads({ videoId: videoUUID, sort: 'createdAt' }) - expect(body.total).to.equal(3) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(3) - - expect(body.data[0].text).to.equal('') - expect(body.data[0].isDeleted).to.be.true - expect(body.data[0].deletedAt).to.not.be.null - expect(body.data[0].account).to.be.null - expect(body.data[0].totalReplies).to.equal(2) - expect(body.data[1].text).to.equal('super thread 2') - expect(body.data[1].totalReplies).to.equal(0) - expect(body.data[2].text).to.equal('super thread 3') - expect(body.data[2].totalReplies).to.equal(0) - }) - - it('Should count replies from the video author correctly', async function () { - await command.createThread({ videoId: videoUUID, text: 'my super first comment' }) - - const { data } = await command.listThreads({ videoId: videoUUID }) - const threadId2 = data[0].threadId - - const text2 = 'a first answer to thread 4 by a third party' - await command.addReply({ token: userAccessTokenServer1, videoId, toCommentId: threadId2, text: text2 }) - - const text3 = 'my second answer to thread 4' - await command.addReply({ videoId, toCommentId: threadId2, text: text3 }) - - const tree = await command.getThread({ videoId: videoUUID, threadId: threadId2 }) - expect(tree.comment.totalRepliesFromVideoAuthor).to.equal(1) - expect(tree.comment.totalReplies).to.equal(2) - }) - }) - - describe('All instance comments', function () { - - it('Should list instance comments as admin', async function () { - { - const { data, total } = await command.listForAdmin({ start: 0, count: 1 }) - - expect(total).to.equal(7) - expect(data).to.have.lengthOf(1) - expect(data[0].text).to.equal('my second answer to thread 4') - expect(data[0].account.name).to.equal('root') - expect(data[0].account.displayName).to.equal('root') - expect(data[0].account.avatars).to.have.lengthOf(2) - } - - { - const { data, total } = await command.listForAdmin({ start: 1, count: 2 }) - - expect(total).to.equal(7) - expect(data).to.have.lengthOf(2) - - expect(data[0].account.avatars).to.have.lengthOf(2) - expect(data[1].account.avatars).to.have.lengthOf(2) - } - }) - - it('Should filter instance comments by isLocal', async function () { - const { total, data } = await command.listForAdmin({ isLocal: false }) - - expect(data).to.have.lengthOf(0) - expect(total).to.equal(0) - }) - - it('Should filter instance comments by onLocalVideo', async function () { - { - const { total, data } = await command.listForAdmin({ onLocalVideo: false }) - - expect(data).to.have.lengthOf(0) - expect(total).to.equal(0) - } - - { - const { total, data } = await command.listForAdmin({ onLocalVideo: true }) - - expect(data).to.not.have.lengthOf(0) - expect(total).to.not.equal(0) - } - }) - - it('Should search instance comments by account', async function () { - const { total, data } = await command.listForAdmin({ searchAccount: 'user' }) - - expect(data).to.have.lengthOf(1) - expect(total).to.equal(1) - - expect(data[0].text).to.equal('a first answer to thread 4 by a third party') - }) - - it('Should search instance comments by video', async function () { - { - const { total, data } = await command.listForAdmin({ searchVideo: 'video' }) - - expect(data).to.have.lengthOf(7) - expect(total).to.equal(7) - } - - { - const { total, data } = await command.listForAdmin({ searchVideo: 'hello' }) - - expect(data).to.have.lengthOf(0) - expect(total).to.equal(0) - } - }) - - it('Should search instance comments', async function () { - const { total, data } = await command.listForAdmin({ search: 'super thread 3' }) - - expect(total).to.equal(1) - - expect(data).to.have.lengthOf(1) - expect(data[0].text).to.equal('super thread 3') - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/videos/video-description.ts b/server/tests/api/videos/video-description.ts deleted file mode 100644 index 1f3d4adbb..000000000 --- a/server/tests/api/videos/video-description.ts +++ /dev/null @@ -1,103 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test video description', function () { - let servers: PeerTubeServer[] = [] - let videoUUID = '' - let videoId: number - - const longDescription = 'my super description for server 1'.repeat(50) - - // 30 characters * 6 -> 240 characters - const truncatedDescription = 'my super description for server 1'.repeat(7) + 'my super descrip...' - - before(async function () { - this.timeout(40000) - - // Run servers - servers = await createMultipleServers(2) - - // Get the access tokens - await setAccessTokensToServers(servers) - - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - }) - - it('Should upload video with long description', async function () { - this.timeout(30000) - - const attributes = { - description: longDescription - } - await servers[0].videos.upload({ attributes }) - - await waitJobs(servers) - - const { data } = await servers[0].videos.list() - - videoId = data[0].id - videoUUID = data[0].uuid - }) - - it('Should have a truncated description on each server when listing videos', async function () { - for (const server of servers) { - const { data } = await server.videos.list() - const video = data.find(v => v.uuid === videoUUID) - - expect(video.description).to.equal(truncatedDescription) - expect(video.truncatedDescription).to.equal(truncatedDescription) - } - }) - - it('Should not have a truncated description on each server when getting videos', async function () { - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - - expect(video.description).to.equal(longDescription) - expect(video.truncatedDescription).to.equal(truncatedDescription) - } - }) - - it('Should fetch long description on each server', async function () { - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - - const { description } = await server.videos.getDescription({ descriptionPath: video.descriptionPath }) - expect(description).to.equal(longDescription) - } - }) - - it('Should update with a short description', async function () { - const attributes = { - description: 'short description' - } - await servers[0].videos.update({ id: videoId, attributes }) - - await waitJobs(servers) - }) - - it('Should have a small description on each server', async function () { - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - - expect(video.description).to.equal('short description') - - const { description } = await server.videos.getDescription({ descriptionPath: video.descriptionPath }) - expect(description).to.equal('short description') - } - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/videos/video-files.ts b/server/tests/api/videos/video-files.ts deleted file mode 100644 index 4f75cd106..000000000 --- a/server/tests/api/videos/video-files.ts +++ /dev/null @@ -1,202 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { HttpStatusCode } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - makeRawRequest, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test videos files', function () { - let servers: PeerTubeServer[] - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(150_000) - - servers = await createMultipleServers(2) - await setAccessTokensToServers(servers) - - await doubleFollow(servers[0], servers[1]) - - await servers[0].config.enableTranscoding({ hls: true, webVideo: true }) - }) - - describe('When deleting all files', function () { - let validId1: string - let validId2: string - - before(async function () { - this.timeout(360_000) - - { - const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' }) - validId1 = uuid - } - - { - const { uuid } = await servers[0].videos.quickUpload({ name: 'video 2' }) - validId2 = uuid - } - - await waitJobs(servers) - }) - - it('Should delete web video files', async function () { - this.timeout(30_000) - - await servers[0].videos.removeAllWebVideoFiles({ videoId: validId1 }) - - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: validId1 }) - - expect(video.files).to.have.lengthOf(0) - expect(video.streamingPlaylists).to.have.lengthOf(1) - } - }) - - it('Should delete HLS files', async function () { - this.timeout(30_000) - - await servers[0].videos.removeHLSPlaylist({ videoId: validId2 }) - - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: validId2 }) - - expect(video.files).to.have.length.above(0) - expect(video.streamingPlaylists).to.have.lengthOf(0) - } - }) - }) - - describe('When deleting a specific file', function () { - let webVideoId: string - let hlsId: string - - before(async function () { - this.timeout(120_000) - - { - const { uuid } = await servers[0].videos.quickUpload({ name: 'web-video' }) - webVideoId = uuid - } - - { - const { uuid } = await servers[0].videos.quickUpload({ name: 'hls' }) - hlsId = uuid - } - - await waitJobs(servers) - }) - - it('Shoulde delete a web video file', async function () { - this.timeout(30_000) - - const video = await servers[0].videos.get({ id: webVideoId }) - const files = video.files - - await servers[0].videos.removeWebVideoFile({ videoId: webVideoId, fileId: files[0].id }) - - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: webVideoId }) - - expect(video.files).to.have.lengthOf(files.length - 1) - expect(video.files.find(f => f.id === files[0].id)).to.not.exist - } - }) - - it('Should delete all web video files', async function () { - this.timeout(30_000) - - const video = await servers[0].videos.get({ id: webVideoId }) - const files = video.files - - for (const file of files) { - await servers[0].videos.removeWebVideoFile({ videoId: webVideoId, fileId: file.id }) - } - - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: webVideoId }) - - expect(video.files).to.have.lengthOf(0) - } - }) - - it('Should delete a hls file', async function () { - this.timeout(30_000) - - const video = await servers[0].videos.get({ id: hlsId }) - const files = video.streamingPlaylists[0].files - const toDelete = files[0] - - await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: toDelete.id }) - - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: hlsId }) - - expect(video.streamingPlaylists[0].files).to.have.lengthOf(files.length - 1) - expect(video.streamingPlaylists[0].files.find(f => f.id === toDelete.id)).to.not.exist - - const { text } = await makeRawRequest({ url: video.streamingPlaylists[0].playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) - - expect(text.includes(`-${toDelete.resolution.id}.m3u8`)).to.be.false - expect(text.includes(`-${video.streamingPlaylists[0].files[0].resolution.id}.m3u8`)).to.be.true - } - }) - - it('Should delete all hls files', async function () { - this.timeout(30_000) - - const video = await servers[0].videos.get({ id: hlsId }) - const files = video.streamingPlaylists[0].files - - for (const file of files) { - await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: file.id }) - } - - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: hlsId }) - - expect(video.streamingPlaylists).to.have.lengthOf(0) - } - }) - - it('Should not delete last file of a video', async function () { - this.timeout(60_000) - - const webVideoOnly = await servers[0].videos.get({ id: hlsId }) - const hlsOnly = await servers[0].videos.get({ id: webVideoId }) - - for (let i = 0; i < 4; i++) { - await servers[0].videos.removeWebVideoFile({ videoId: webVideoOnly.id, fileId: webVideoOnly.files[i].id }) - await servers[0].videos.removeHLSFile({ videoId: hlsOnly.id, fileId: hlsOnly.streamingPlaylists[0].files[i].id }) - } - - const expectedStatus = HttpStatusCode.BAD_REQUEST_400 - await servers[0].videos.removeWebVideoFile({ videoId: webVideoOnly.id, fileId: webVideoOnly.files[4].id, expectedStatus }) - await servers[0].videos.removeHLSFile({ videoId: hlsOnly.id, fileId: hlsOnly.streamingPlaylists[0].files[4].id, expectedStatus }) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/videos/video-imports.ts b/server/tests/api/videos/video-imports.ts deleted file mode 100644 index b78b4f344..000000000 --- a/server/tests/api/videos/video-imports.ts +++ /dev/null @@ -1,631 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { pathExists, readdir, remove } from 'fs-extra' -import { join } from 'path' -import { FIXTURE_URLS, testCaptionFile, testImageGeneratedByFFmpeg } from '@server/tests/shared' -import { areHttpImportTestsDisabled } from '@shared/core-utils' -import { CustomConfig, HttpStatusCode, Video, VideoImportState, VideoPrivacy, VideoResolution, VideoState } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - createSingleServer, - doubleFollow, - getServerImportConfig, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - waitJobs -} from '@shared/server-commands' -import { DeepPartial } from '@shared/typescript-utils' - -async function checkVideosServer1 (server: PeerTubeServer, idHttp: string, idMagnet: string, idTorrent: string) { - const videoHttp = await server.videos.get({ id: idHttp }) - - expect(videoHttp.name).to.equal('small video - youtube') - expect(videoHttp.category.label).to.equal('News & Politics') - expect(videoHttp.licence.label).to.equal('Attribution') - expect(videoHttp.language.label).to.equal('Unknown') - expect(videoHttp.nsfw).to.be.false - expect(videoHttp.description).to.equal('this is a super description') - expect(videoHttp.tags).to.deep.equal([ 'tag1', 'tag2' ]) - expect(videoHttp.files).to.have.lengthOf(1) - - const originallyPublishedAt = new Date(videoHttp.originallyPublishedAt) - expect(originallyPublishedAt.getDate()).to.equal(14) - expect(originallyPublishedAt.getMonth()).to.equal(0) - expect(originallyPublishedAt.getFullYear()).to.equal(2019) - - const videoMagnet = await server.videos.get({ id: idMagnet }) - const videoTorrent = await server.videos.get({ id: idTorrent }) - - for (const video of [ videoMagnet, videoTorrent ]) { - expect(video.category.label).to.equal('Unknown') - expect(video.licence.label).to.equal('Unknown') - expect(video.language.label).to.equal('Unknown') - expect(video.nsfw).to.be.false - expect(video.description).to.equal('this is a super torrent description') - expect(video.tags).to.deep.equal([ 'tag_torrent1', 'tag_torrent2' ]) - expect(video.files).to.have.lengthOf(1) - } - - expect(videoTorrent.name).to.contain('你好 世界 720p.mp4') - expect(videoMagnet.name).to.contain('super peertube2 video') - - const bodyCaptions = await server.captions.list({ videoId: idHttp }) - expect(bodyCaptions.total).to.equal(2) -} - -async function checkVideoServer2 (server: PeerTubeServer, id: number | string) { - const video = await server.videos.get({ id }) - - expect(video.name).to.equal('my super name') - expect(video.category.label).to.equal('Entertainment') - expect(video.licence.label).to.equal('Public Domain Dedication') - expect(video.language.label).to.equal('English') - expect(video.nsfw).to.be.false - expect(video.description).to.equal('my super description') - expect(video.tags).to.deep.equal([ 'supertag1', 'supertag2' ]) - - await testImageGeneratedByFFmpeg(server.url, 'custom-thumbnail', video.thumbnailPath) - - expect(video.files).to.have.lengthOf(1) - - const bodyCaptions = await server.captions.list({ videoId: id }) - expect(bodyCaptions.total).to.equal(2) -} - -describe('Test video imports', function () { - - if (areHttpImportTestsDisabled()) return - - function runSuite (mode: 'youtube-dl' | 'yt-dlp') { - - describe('Import ' + mode, function () { - let servers: PeerTubeServer[] = [] - - before(async function () { - this.timeout(60_000) - - servers = await createMultipleServers(2, getServerImportConfig(mode)) - - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - - for (const server of servers) { - await server.config.updateExistingSubConfig({ - newConfig: { - transcoding: { - alwaysTranscodeOriginalResolution: false - } - } - }) - } - - await doubleFollow(servers[0], servers[1]) - }) - - it('Should import videos on server 1', async function () { - this.timeout(60_000) - - const baseAttributes = { - channelId: servers[0].store.channel.id, - privacy: VideoPrivacy.PUBLIC - } - - { - const attributes = { ...baseAttributes, targetUrl: FIXTURE_URLS.youtube } - const { video } = await servers[0].imports.importVideo({ attributes }) - expect(video.name).to.equal('small video - youtube') - - { - expect(video.thumbnailPath).to.match(new RegExp(`^/lazy-static/thumbnails/.+.jpg$`)) - expect(video.previewPath).to.match(new RegExp(`^/lazy-static/previews/.+.jpg$`)) - - const suffix = mode === 'yt-dlp' - ? '_yt_dlp' - : '' - - await testImageGeneratedByFFmpeg(servers[0].url, 'video_import_thumbnail' + suffix, video.thumbnailPath) - await testImageGeneratedByFFmpeg(servers[0].url, 'video_import_preview' + suffix, video.previewPath) - } - - const bodyCaptions = await servers[0].captions.list({ videoId: video.id }) - const videoCaptions = bodyCaptions.data - expect(videoCaptions).to.have.lengthOf(2) - - { - const enCaption = videoCaptions.find(caption => caption.language.id === 'en') - expect(enCaption).to.exist - expect(enCaption.language.label).to.equal('English') - expect(enCaption.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/.+-en.vtt$`)) - - const regex = `WEBVTT[ \n]+Kind: captions[ \n]+` + - `(Language: en[ \n]+)?` + - `00:00:01.600 --> 00:00:04.200( position:\\d+% line:\\d+%)?[ \n]+English \\(US\\)[ \n]+` + - `00:00:05.900 --> 00:00:07.999( position:\\d+% line:\\d+%)?[ \n]+This is a subtitle in American English[ \n]+` + - `00:00:10.000 --> 00:00:14.000( position:\\d+% line:\\d+%)?[ \n]+Adding subtitles is very easy to do` - await testCaptionFile(servers[0].url, enCaption.captionPath, new RegExp(regex)) - } - - { - const frCaption = videoCaptions.find(caption => caption.language.id === 'fr') - expect(frCaption).to.exist - expect(frCaption.language.label).to.equal('French') - expect(frCaption.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/.+-fr.vtt`)) - - const regex = `WEBVTT[ \n]+Kind: captions[ \n]+` + - `(Language: fr[ \n]+)?` + - `00:00:01.600 --> 00:00:04.200( position:\\d+% line:\\d+%)?[ \n]+Français \\(FR\\)[ \n]+` + - `00:00:05.900 --> 00:00:07.999( position:\\d+% line:\\d+%)?[ \n]+C'est un sous-titre français[ \n]+` + - `00:00:10.000 --> 00:00:14.000( position:\\d+% line:\\d+%)?[ \n]+Ajouter un sous-titre est vraiment facile` - - await testCaptionFile(servers[0].url, frCaption.captionPath, new RegExp(regex)) - } - } - - { - const attributes = { - ...baseAttributes, - magnetUri: FIXTURE_URLS.magnet, - description: 'this is a super torrent description', - tags: [ 'tag_torrent1', 'tag_torrent2' ] - } - const { video } = await servers[0].imports.importVideo({ attributes }) - expect(video.name).to.equal('super peertube2 video') - } - - { - const attributes = { - ...baseAttributes, - torrentfile: 'video-720p.torrent' as any, - description: 'this is a super torrent description', - tags: [ 'tag_torrent1', 'tag_torrent2' ] - } - const { video } = await servers[0].imports.importVideo({ attributes }) - expect(video.name).to.equal('你好 世界 720p.mp4') - } - }) - - it('Should list the videos to import in my videos on server 1', async function () { - const { total, data } = await servers[0].videos.listMyVideos({ sort: 'createdAt' }) - - expect(total).to.equal(3) - - expect(data).to.have.lengthOf(3) - expect(data[0].name).to.equal('small video - youtube') - expect(data[1].name).to.equal('super peertube2 video') - expect(data[2].name).to.equal('你好 世界 720p.mp4') - }) - - it('Should list the videos to import in my imports on server 1', async function () { - const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ sort: '-createdAt' }) - expect(total).to.equal(3) - - expect(videoImports).to.have.lengthOf(3) - - expect(videoImports[2].targetUrl).to.equal(FIXTURE_URLS.youtube) - expect(videoImports[2].magnetUri).to.be.null - expect(videoImports[2].torrentName).to.be.null - expect(videoImports[2].video.name).to.equal('small video - youtube') - - expect(videoImports[1].targetUrl).to.be.null - expect(videoImports[1].magnetUri).to.equal(FIXTURE_URLS.magnet) - expect(videoImports[1].torrentName).to.be.null - expect(videoImports[1].video.name).to.equal('super peertube2 video') - - expect(videoImports[0].targetUrl).to.be.null - expect(videoImports[0].magnetUri).to.be.null - expect(videoImports[0].torrentName).to.equal('video-720p.torrent') - expect(videoImports[0].video.name).to.equal('你好 世界 720p.mp4') - }) - - it('Should filter my imports on target URL', async function () { - const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ targetUrl: FIXTURE_URLS.youtube }) - expect(total).to.equal(1) - expect(videoImports).to.have.lengthOf(1) - - expect(videoImports[0].targetUrl).to.equal(FIXTURE_URLS.youtube) - }) - - it('Should search in my imports', async function () { - const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ search: 'peertube2' }) - expect(total).to.equal(1) - expect(videoImports).to.have.lengthOf(1) - - expect(videoImports[0].magnetUri).to.equal(FIXTURE_URLS.magnet) - expect(videoImports[0].video.name).to.equal('super peertube2 video') - }) - - it('Should have the video listed on the two instances', async function () { - this.timeout(120_000) - - await waitJobs(servers) - - for (const server of servers) { - const { total, data } = await server.videos.list() - expect(total).to.equal(3) - expect(data).to.have.lengthOf(3) - - const [ videoHttp, videoMagnet, videoTorrent ] = data - await checkVideosServer1(server, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid) - } - }) - - it('Should import a video on server 2 with some fields', async function () { - this.timeout(60_000) - - const { video } = await servers[1].imports.importVideo({ - attributes: { - targetUrl: FIXTURE_URLS.youtube, - channelId: servers[1].store.channel.id, - privacy: VideoPrivacy.PUBLIC, - category: 10, - licence: 7, - language: 'en', - name: 'my super name', - description: 'my super description', - tags: [ 'supertag1', 'supertag2' ], - thumbnailfile: 'custom-thumbnail.jpg' - } - }) - expect(video.name).to.equal('my super name') - }) - - it('Should have the videos listed on the two instances', async function () { - this.timeout(120_000) - - await waitJobs(servers) - - for (const server of servers) { - const { total, data } = await server.videos.list() - expect(total).to.equal(4) - expect(data).to.have.lengthOf(4) - - await checkVideoServer2(server, data[0].uuid) - - const [ , videoHttp, videoMagnet, videoTorrent ] = data - await checkVideosServer1(server, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid) - } - }) - - it('Should import a video that will be transcoded', async function () { - this.timeout(240_000) - - const attributes = { - name: 'transcoded video', - magnetUri: FIXTURE_URLS.magnet, - channelId: servers[1].store.channel.id, - privacy: VideoPrivacy.PUBLIC - } - const { video } = await servers[1].imports.importVideo({ attributes }) - const videoUUID = video.uuid - - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - - expect(video.name).to.equal('transcoded video') - expect(video.files).to.have.lengthOf(4) - } - }) - - it('Should import no HDR version on a HDR video', async function () { - this.timeout(300_000) - - const config: DeepPartial = { - transcoding: { - enabled: true, - resolutions: { - '0p': false, - '144p': true, - '240p': true, - '360p': false, - '480p': false, - '720p': false, - '1080p': false, // the resulting resolution shouldn't be higher than this, and not vp9.2/av01 - '1440p': false, - '2160p': false - }, - webVideos: { enabled: true }, - hls: { enabled: false } - } - } - await servers[0].config.updateExistingSubConfig({ newConfig: config }) - - const attributes = { - name: 'hdr video', - targetUrl: FIXTURE_URLS.youtubeHDR, - channelId: servers[0].store.channel.id, - privacy: VideoPrivacy.PUBLIC - } - const { video: videoImported } = await servers[0].imports.importVideo({ attributes }) - const videoUUID = videoImported.uuid - - await waitJobs(servers) - - // test resolution - const video = await servers[0].videos.get({ id: videoUUID }) - expect(video.name).to.equal('hdr video') - const maxResolution = Math.max.apply(Math, video.files.map(function (o) { return o.resolution.id })) - expect(maxResolution, 'expected max resolution not met').to.equals(VideoResolution.H_240P) - }) - - it('Should not import resolution higher than enabled transcoding resolution', async function () { - this.timeout(300_000) - - const config: DeepPartial = { - transcoding: { - enabled: true, - resolutions: { - '0p': false, - '144p': true, - '240p': false, - '360p': false, - '480p': false, - '720p': false, - '1080p': false, - '1440p': false, - '2160p': false - }, - alwaysTranscodeOriginalResolution: false - } - } - await servers[0].config.updateExistingSubConfig({ newConfig: config }) - - const attributes = { - name: 'small resolution video', - targetUrl: FIXTURE_URLS.youtube, - channelId: servers[0].store.channel.id, - privacy: VideoPrivacy.PUBLIC - } - const { video: videoImported } = await servers[0].imports.importVideo({ attributes }) - const videoUUID = videoImported.uuid - - await waitJobs(servers) - - // test resolution - const video = await servers[0].videos.get({ id: videoUUID }) - expect(video.name).to.equal('small resolution video') - expect(video.files).to.have.lengthOf(1) - expect(video.files[0].resolution.id).to.equal(144) - }) - - it('Should import resolution higher than enabled transcoding resolution', async function () { - this.timeout(300_000) - - const config: DeepPartial = { - transcoding: { - alwaysTranscodeOriginalResolution: true - } - } - await servers[0].config.updateExistingSubConfig({ newConfig: config }) - - const attributes = { - name: 'bigger resolution video', - targetUrl: FIXTURE_URLS.youtube, - channelId: servers[0].store.channel.id, - privacy: VideoPrivacy.PUBLIC - } - const { video: videoImported } = await servers[0].imports.importVideo({ attributes }) - const videoUUID = videoImported.uuid - - await waitJobs(servers) - - // test resolution - const video = await servers[0].videos.get({ id: videoUUID }) - expect(video.name).to.equal('bigger resolution video') - - expect(video.files).to.have.lengthOf(2) - expect(video.files.find(f => f.resolution.id === 240)).to.exist - expect(video.files.find(f => f.resolution.id === 144)).to.exist - }) - - it('Should import a peertube video', async function () { - this.timeout(120_000) - - const toTest = [ FIXTURE_URLS.peertube_long ] - - // TODO: include peertube_short when https://github.com/ytdl-org/youtube-dl/pull/29475 is merged - if (mode === 'yt-dlp') { - toTest.push(FIXTURE_URLS.peertube_short) - } - - for (const targetUrl of toTest) { - await servers[0].config.disableTranscoding() - - const attributes = { - targetUrl, - channelId: servers[0].store.channel.id, - privacy: VideoPrivacy.PUBLIC - } - const { video } = await servers[0].imports.importVideo({ attributes }) - const videoUUID = video.uuid - - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: videoUUID }) - - expect(video.name).to.equal('E2E tests') - - const { data: captions } = await server.captions.list({ videoId: videoUUID }) - expect(captions).to.have.lengthOf(1) - expect(captions[0].language.id).to.equal('fr') - - const str = `WEBVTT FILE\r?\n\r?\n` + - `1\r?\n` + - `00:00:04.000 --> 00:00:09.000\r?\n` + - `January 1, 1994. The North American` - await testCaptionFile(server.url, captions[0].captionPath, new RegExp(str)) - } - } - }) - - after(async function () { - await cleanupTests(servers) - }) - }) - } - - // FIXME: youtube-dl seems broken - // runSuite('youtube-dl') - - runSuite('yt-dlp') - - describe('Delete/cancel an import', function () { - let server: PeerTubeServer - - let finishedImportId: number - let finishedVideo: Video - let pendingImportId: number - - async function importVideo (name: string) { - const attributes = { name, channelId: server.store.channel.id, targetUrl: FIXTURE_URLS.goodVideo } - const res = await server.imports.importVideo({ attributes }) - - return res.id - } - - before(async function () { - this.timeout(120_000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - finishedImportId = await importVideo('finished') - await waitJobs([ server ]) - - await server.jobs.pauseJobQueue() - pendingImportId = await importVideo('pending') - - const { data } = await server.imports.getMyVideoImports() - expect(data).to.have.lengthOf(2) - - finishedVideo = data.find(i => i.id === finishedImportId).video - }) - - it('Should delete a video import', async function () { - await server.imports.delete({ importId: finishedImportId }) - - const { data } = await server.imports.getMyVideoImports() - expect(data).to.have.lengthOf(1) - expect(data[0].id).to.equal(pendingImportId) - expect(data[0].state.id).to.equal(VideoImportState.PENDING) - }) - - it('Should not have deleted the associated video', async function () { - const video = await server.videos.get({ id: finishedVideo.id, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - expect(video.name).to.equal('finished') - expect(video.state.id).to.equal(VideoState.PUBLISHED) - }) - - it('Should cancel a video import', async function () { - await server.imports.cancel({ importId: pendingImportId }) - - const { data } = await server.imports.getMyVideoImports() - expect(data).to.have.lengthOf(1) - expect(data[0].id).to.equal(pendingImportId) - expect(data[0].state.id).to.equal(VideoImportState.CANCELLED) - }) - - it('Should not have processed the cancelled video import', async function () { - this.timeout(60_000) - - await server.jobs.resumeJobQueue() - - await waitJobs([ server ]) - - const { data } = await server.imports.getMyVideoImports() - expect(data).to.have.lengthOf(1) - expect(data[0].id).to.equal(pendingImportId) - expect(data[0].state.id).to.equal(VideoImportState.CANCELLED) - expect(data[0].video.state.id).to.equal(VideoState.TO_IMPORT) - }) - - it('Should delete the cancelled video import', async function () { - await server.imports.delete({ importId: pendingImportId }) - const { data } = await server.imports.getMyVideoImports() - expect(data).to.have.lengthOf(0) - }) - - after(async function () { - await cleanupTests([ server ]) - }) - }) - - describe('Auto update', function () { - let server: PeerTubeServer - - function quickPeerTubeImport () { - const attributes = { - targetUrl: FIXTURE_URLS.peertube_long, - channelId: server.store.channel.id, - privacy: VideoPrivacy.PUBLIC - } - - return server.imports.importVideo({ attributes }) - } - - async function testBinaryUpdate (releaseUrl: string, releaseName: string) { - await remove(join(server.servers.buildDirectory('bin'), releaseName)) - - await server.kill() - await server.run({ - import: { - videos: { - http: { - youtube_dl_release: { - url: releaseUrl, - name: releaseName - } - } - } - } - }) - - await quickPeerTubeImport() - - const base = server.servers.buildDirectory('bin') - const content = await readdir(base) - const binaryPath = join(base, releaseName) - - expect(await pathExists(binaryPath), `${binaryPath} does not exist in ${base} (${content.join(', ')})`).to.be.true - } - - before(async function () { - this.timeout(30_000) - - // Run servers - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - }) - - it('Should update youtube-dl from github URL', async function () { - this.timeout(120_000) - - await testBinaryUpdate('https://api.github.com/repos/ytdl-org/youtube-dl/releases', 'youtube-dl') - }) - - it('Should update youtube-dl from raw URL', async function () { - this.timeout(120_000) - - await testBinaryUpdate('https://yt-dl.org/downloads/latest/youtube-dl', 'youtube-dl') - }) - - it('Should update youtube-dl from youtube-dl fork', async function () { - this.timeout(120_000) - - await testBinaryUpdate('https://api.github.com/repos/yt-dlp/yt-dlp/releases', 'yt-dlp') - }) - - after(async function () { - await cleanupTests([ server ]) - }) - }) -}) diff --git a/server/tests/api/videos/video-nsfw.ts b/server/tests/api/videos/video-nsfw.ts deleted file mode 100644 index 65e9c8730..000000000 --- a/server/tests/api/videos/video-nsfw.ts +++ /dev/null @@ -1,227 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' -import { BooleanBothQuery, CustomConfig, ResultList, Video, VideosOverview } from '@shared/models' - -function createOverviewRes (overview: VideosOverview) { - const videos = overview.categories[0].videos - return { data: videos, total: videos.length } -} - -describe('Test video NSFW policy', function () { - let server: PeerTubeServer - let userAccessToken: string - let customConfig: CustomConfig - - async function getVideosFunctions (token?: string, query: { nsfw?: BooleanBothQuery } = {}) { - const user = await server.users.getMyInfo() - - const channelName = user.videoChannels[0].name - const accountName = user.account.name + '@' + user.account.host - - const hasQuery = Object.keys(query).length !== 0 - let promises: Promise>[] - - if (token) { - promises = [ - server.search.advancedVideoSearch({ token, search: { search: 'n', sort: '-publishedAt', ...query } }), - server.videos.listWithToken({ token, ...query }), - server.videos.listByAccount({ token, handle: accountName, ...query }), - server.videos.listByChannel({ token, handle: channelName, ...query }) - ] - - // Overviews do not support video filters - if (!hasQuery) { - const p = server.overviews.getVideos({ page: 1, token }) - .then(res => createOverviewRes(res)) - promises.push(p) - } - - return Promise.all(promises) - } - - promises = [ - server.search.searchVideos({ search: 'n', sort: '-publishedAt' }), - server.videos.list(), - server.videos.listByAccount({ token: null, handle: accountName }), - server.videos.listByChannel({ token: null, handle: channelName }) - ] - - // Overviews do not support video filters - if (!hasQuery) { - const p = server.overviews.getVideos({ page: 1 }) - .then(res => createOverviewRes(res)) - promises.push(p) - } - - return Promise.all(promises) - } - - before(async function () { - this.timeout(50000) - server = await createSingleServer(1) - - // Get the access tokens - await setAccessTokensToServers([ server ]) - - { - const attributes = { name: 'nsfw', nsfw: true, category: 1 } - await server.videos.upload({ attributes }) - } - - { - const attributes = { name: 'normal', nsfw: false, category: 1 } - await server.videos.upload({ attributes }) - } - - customConfig = await server.config.getCustomConfig() - }) - - describe('Instance default NSFW policy', function () { - - it('Should display NSFW videos with display default NSFW policy', async function () { - const serverConfig = await server.config.getConfig() - expect(serverConfig.instance.defaultNSFWPolicy).to.equal('display') - - for (const body of await getVideosFunctions()) { - expect(body.total).to.equal(2) - - const videos = body.data - expect(videos).to.have.lengthOf(2) - expect(videos[0].name).to.equal('normal') - expect(videos[1].name).to.equal('nsfw') - } - }) - - it('Should not display NSFW videos with do_not_list default NSFW policy', async function () { - customConfig.instance.defaultNSFWPolicy = 'do_not_list' - await server.config.updateCustomConfig({ newCustomConfig: customConfig }) - - const serverConfig = await server.config.getConfig() - expect(serverConfig.instance.defaultNSFWPolicy).to.equal('do_not_list') - - for (const body of await getVideosFunctions()) { - expect(body.total).to.equal(1) - - const videos = body.data - expect(videos).to.have.lengthOf(1) - expect(videos[0].name).to.equal('normal') - } - }) - - it('Should display NSFW videos with blur default NSFW policy', async function () { - customConfig.instance.defaultNSFWPolicy = 'blur' - await server.config.updateCustomConfig({ newCustomConfig: customConfig }) - - const serverConfig = await server.config.getConfig() - expect(serverConfig.instance.defaultNSFWPolicy).to.equal('blur') - - for (const body of await getVideosFunctions()) { - expect(body.total).to.equal(2) - - const videos = body.data - expect(videos).to.have.lengthOf(2) - expect(videos[0].name).to.equal('normal') - expect(videos[1].name).to.equal('nsfw') - } - }) - }) - - describe('User NSFW policy', function () { - - it('Should create a user having the default nsfw policy', async function () { - const username = 'user1' - const password = 'my super password' - await server.users.create({ username, password }) - - userAccessToken = await server.login.getAccessToken({ username, password }) - - const user = await server.users.getMyInfo({ token: userAccessToken }) - expect(user.nsfwPolicy).to.equal('blur') - }) - - it('Should display NSFW videos with blur user NSFW policy', async function () { - customConfig.instance.defaultNSFWPolicy = 'do_not_list' - await server.config.updateCustomConfig({ newCustomConfig: customConfig }) - - for (const body of await getVideosFunctions(userAccessToken)) { - expect(body.total).to.equal(2) - - const videos = body.data - expect(videos).to.have.lengthOf(2) - expect(videos[0].name).to.equal('normal') - expect(videos[1].name).to.equal('nsfw') - } - }) - - it('Should display NSFW videos with display user NSFW policy', async function () { - await server.users.updateMe({ nsfwPolicy: 'display' }) - - for (const body of await getVideosFunctions(server.accessToken)) { - expect(body.total).to.equal(2) - - const videos = body.data - expect(videos).to.have.lengthOf(2) - expect(videos[0].name).to.equal('normal') - expect(videos[1].name).to.equal('nsfw') - } - }) - - it('Should not display NSFW videos with do_not_list user NSFW policy', async function () { - await server.users.updateMe({ nsfwPolicy: 'do_not_list' }) - - for (const body of await getVideosFunctions(server.accessToken)) { - expect(body.total).to.equal(1) - - const videos = body.data - expect(videos).to.have.lengthOf(1) - expect(videos[0].name).to.equal('normal') - } - }) - - it('Should be able to see my NSFW videos even with do_not_list user NSFW policy', async function () { - const { total, data } = await server.videos.listMyVideos() - expect(total).to.equal(2) - - expect(data).to.have.lengthOf(2) - expect(data[0].name).to.equal('normal') - expect(data[1].name).to.equal('nsfw') - }) - - it('Should display NSFW videos when the nsfw param === true', async function () { - for (const body of await getVideosFunctions(server.accessToken, { nsfw: 'true' })) { - expect(body.total).to.equal(1) - - const videos = body.data - expect(videos).to.have.lengthOf(1) - expect(videos[0].name).to.equal('nsfw') - } - }) - - it('Should hide NSFW videos when the nsfw param === true', async function () { - for (const body of await getVideosFunctions(server.accessToken, { nsfw: 'false' })) { - expect(body.total).to.equal(1) - - const videos = body.data - expect(videos).to.have.lengthOf(1) - expect(videos[0].name).to.equal('normal') - } - }) - - it('Should display both videos when the nsfw param === both', async function () { - for (const body of await getVideosFunctions(server.accessToken, { nsfw: 'both' })) { - expect(body.total).to.equal(2) - - const videos = body.data - expect(videos).to.have.lengthOf(2) - expect(videos[0].name).to.equal('normal') - expect(videos[1].name).to.equal('nsfw') - } - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/videos/video-passwords.ts b/server/tests/api/videos/video-passwords.ts deleted file mode 100644 index e01a93a4d..000000000 --- a/server/tests/api/videos/video-passwords.ts +++ /dev/null @@ -1,97 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { - cleanupTests, - createSingleServer, - VideoPasswordsCommand, - PeerTubeServer, - setAccessTokensToServers, - setDefaultAccountAvatar, - setDefaultChannelAvatar -} from '@shared/server-commands' -import { VideoPrivacy } from '@shared/models' - -describe('Test video passwords', function () { - let server: PeerTubeServer - let videoUUID: string - - let userAccessTokenServer1: string - - let videoPasswords: string[] = [] - let command: VideoPasswordsCommand - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - - for (let i = 0; i < 10; i++) { - videoPasswords.push(`password ${i + 1}`) - } - const { uuid } = await server.videos.upload({ attributes: { privacy: VideoPrivacy.PASSWORD_PROTECTED, videoPasswords } }) - videoUUID = uuid - - await setDefaultChannelAvatar(server) - await setDefaultAccountAvatar(server) - - userAccessTokenServer1 = await server.users.generateUserAndToken('user1') - await setDefaultChannelAvatar(server, 'user1_channel') - await setDefaultAccountAvatar(server, userAccessTokenServer1) - - command = server.videoPasswords - }) - - it('Should list video passwords', async function () { - const body = await command.list({ videoId: videoUUID }) - - expect(body.total).to.equal(10) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(10) - }) - - it('Should filter passwords on this video', async function () { - const body = await command.list({ videoId: videoUUID, count: 2, start: 3, sort: 'createdAt' }) - - expect(body.total).to.equal(10) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(2) - expect(body.data[0].password).to.equal('password 4') - expect(body.data[1].password).to.equal('password 5') - }) - - it('Should update password for this video', async function () { - videoPasswords = [ 'my super new password 1', 'my super new password 2' ] - - await command.updateAll({ videoId: videoUUID, passwords: videoPasswords }) - const body = await command.list({ videoId: videoUUID }) - expect(body.total).to.equal(2) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(2) - expect(body.data[0].password).to.equal('my super new password 2') - expect(body.data[1].password).to.equal('my super new password 1') - }) - - it('Should delete one password', async function () { - { - const body = await command.list({ videoId: videoUUID }) - expect(body.total).to.equal(2) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(2) - await command.remove({ id: body.data[0].id, videoId: videoUUID }) - } - { - const body = await command.list({ videoId: videoUUID }) - - expect(body.total).to.equal(1) - expect(body.data).to.be.an('array') - expect(body.data).to.have.lengthOf(1) - } - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/videos/video-playlist-thumbnails.ts b/server/tests/api/videos/video-playlist-thumbnails.ts deleted file mode 100644 index c274c20bf..000000000 --- a/server/tests/api/videos/video-playlist-thumbnails.ts +++ /dev/null @@ -1,234 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { testImageGeneratedByFFmpeg } from '@server/tests/shared' -import { VideoPlaylistPrivacy } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - waitJobs -} from '@shared/server-commands' - -describe('Playlist thumbnail', function () { - let servers: PeerTubeServer[] = [] - - let playlistWithoutThumbnailId: number - let playlistWithThumbnailId: number - - let withThumbnailE1: number - let withThumbnailE2: number - let withoutThumbnailE1: number - let withoutThumbnailE2: number - - let video1: number - let video2: number - - async function getPlaylistWithoutThumbnail (server: PeerTubeServer) { - const body = await server.playlists.list({ start: 0, count: 10 }) - - return body.data.find(p => p.displayName === 'playlist without thumbnail') - } - - async function getPlaylistWithThumbnail (server: PeerTubeServer) { - const body = await server.playlists.list({ start: 0, count: 10 }) - - return body.data.find(p => p.displayName === 'playlist with thumbnail') - } - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(2) - - // Get the access tokens - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - - for (const server of servers) { - await server.config.disableTranscoding() - } - - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - - video1 = (await servers[0].videos.quickUpload({ name: 'video 1' })).id - video2 = (await servers[0].videos.quickUpload({ name: 'video 2' })).id - - await waitJobs(servers) - }) - - it('Should automatically update the thumbnail when adding an element', async function () { - this.timeout(30000) - - const created = await servers[1].playlists.create({ - attributes: { - displayName: 'playlist without thumbnail', - privacy: VideoPlaylistPrivacy.PUBLIC, - videoChannelId: servers[1].store.channel.id - } - }) - playlistWithoutThumbnailId = created.id - - const added = await servers[1].playlists.addElement({ - playlistId: playlistWithoutThumbnailId, - attributes: { videoId: video1 } - }) - withoutThumbnailE1 = added.id - - await waitJobs(servers) - - for (const server of servers) { - const p = await getPlaylistWithoutThumbnail(server) - await testImageGeneratedByFFmpeg(server.url, 'thumbnail-playlist', p.thumbnailPath) - } - }) - - it('Should not update the thumbnail if we explicitly uploaded a thumbnail', async function () { - this.timeout(30000) - - const created = await servers[1].playlists.create({ - attributes: { - displayName: 'playlist with thumbnail', - privacy: VideoPlaylistPrivacy.PUBLIC, - videoChannelId: servers[1].store.channel.id, - thumbnailfile: 'custom-thumbnail.jpg' - } - }) - playlistWithThumbnailId = created.id - - const added = await servers[1].playlists.addElement({ - playlistId: playlistWithThumbnailId, - attributes: { videoId: video1 } - }) - withThumbnailE1 = added.id - - await waitJobs(servers) - - for (const server of servers) { - const p = await getPlaylistWithThumbnail(server) - await testImageGeneratedByFFmpeg(server.url, 'custom-thumbnail', p.thumbnailPath) - } - }) - - it('Should automatically update the thumbnail when moving the first element', async function () { - this.timeout(30000) - - const added = await servers[1].playlists.addElement({ - playlistId: playlistWithoutThumbnailId, - attributes: { videoId: video2 } - }) - withoutThumbnailE2 = added.id - - await servers[1].playlists.reorderElements({ - playlistId: playlistWithoutThumbnailId, - attributes: { - startPosition: 1, - insertAfterPosition: 2 - } - }) - - await waitJobs(servers) - - for (const server of servers) { - const p = await getPlaylistWithoutThumbnail(server) - await testImageGeneratedByFFmpeg(server.url, 'thumbnail-playlist', p.thumbnailPath) - } - }) - - it('Should not update the thumbnail when moving the first element if we explicitly uploaded a thumbnail', async function () { - this.timeout(30000) - - const added = await servers[1].playlists.addElement({ - playlistId: playlistWithThumbnailId, - attributes: { videoId: video2 } - }) - withThumbnailE2 = added.id - - await servers[1].playlists.reorderElements({ - playlistId: playlistWithThumbnailId, - attributes: { - startPosition: 1, - insertAfterPosition: 2 - } - }) - - await waitJobs(servers) - - for (const server of servers) { - const p = await getPlaylistWithThumbnail(server) - await testImageGeneratedByFFmpeg(server.url, 'custom-thumbnail', p.thumbnailPath) - } - }) - - it('Should automatically update the thumbnail when deleting the first element', async function () { - this.timeout(30000) - - await servers[1].playlists.removeElement({ - playlistId: playlistWithoutThumbnailId, - elementId: withoutThumbnailE1 - }) - - await waitJobs(servers) - - for (const server of servers) { - const p = await getPlaylistWithoutThumbnail(server) - await testImageGeneratedByFFmpeg(server.url, 'thumbnail-playlist', p.thumbnailPath) - } - }) - - it('Should not update the thumbnail when deleting the first element if we explicitly uploaded a thumbnail', async function () { - this.timeout(30000) - - await servers[1].playlists.removeElement({ - playlistId: playlistWithThumbnailId, - elementId: withThumbnailE1 - }) - - await waitJobs(servers) - - for (const server of servers) { - const p = await getPlaylistWithThumbnail(server) - await testImageGeneratedByFFmpeg(server.url, 'custom-thumbnail', p.thumbnailPath) - } - }) - - it('Should the thumbnail when we delete the last element', async function () { - this.timeout(30000) - - await servers[1].playlists.removeElement({ - playlistId: playlistWithoutThumbnailId, - elementId: withoutThumbnailE2 - }) - - await waitJobs(servers) - - for (const server of servers) { - const p = await getPlaylistWithoutThumbnail(server) - expect(p.thumbnailPath).to.be.null - } - }) - - it('Should not update the thumbnail when we delete the last element if we explicitly uploaded a thumbnail', async function () { - this.timeout(30000) - - await servers[1].playlists.removeElement({ - playlistId: playlistWithThumbnailId, - elementId: withThumbnailE2 - }) - - await waitJobs(servers) - - for (const server of servers) { - const p = await getPlaylistWithThumbnail(server) - await testImageGeneratedByFFmpeg(server.url, 'custom-thumbnail', p.thumbnailPath) - } - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/videos/video-playlists.ts b/server/tests/api/videos/video-playlists.ts deleted file mode 100644 index 3bfa874cb..000000000 --- a/server/tests/api/videos/video-playlists.ts +++ /dev/null @@ -1,1208 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { checkPlaylistFilesWereRemoved, testImageGeneratedByFFmpeg } from '@server/tests/shared' -import { wait } from '@shared/core-utils' -import { uuidToShort } from '@shared/extra-utils' -import { - HttpStatusCode, - VideoPlaylist, - VideoPlaylistCreateResult, - VideoPlaylistElementType, - VideoPlaylistPrivacy, - VideoPlaylistType, - VideoPrivacy -} from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - PlaylistsCommand, - setAccessTokensToServers, - setDefaultAccountAvatar, - setDefaultVideoChannel, - waitJobs -} from '@shared/server-commands' - -async function checkPlaylistElementType ( - servers: PeerTubeServer[], - playlistId: string, - type: VideoPlaylistElementType, - position: number, - name: string, - total: number -) { - for (const server of servers) { - const body = await server.playlists.listVideos({ token: server.accessToken, playlistId, start: 0, count: 10 }) - expect(body.total).to.equal(total) - - const videoElement = body.data.find(e => e.position === position) - expect(videoElement.type).to.equal(type, 'On server ' + server.url) - - if (type === VideoPlaylistElementType.REGULAR) { - expect(videoElement.video).to.not.be.null - expect(videoElement.video.name).to.equal(name) - } else { - expect(videoElement.video).to.be.null - } - } -} - -describe('Test video playlists', function () { - let servers: PeerTubeServer[] = [] - - let playlistServer2Id1: number - let playlistServer2Id2: number - let playlistServer2UUID2: string - - let playlistServer1Id: number - let playlistServer1DisplayName: string - let playlistServer1UUID: string - let playlistServer1UUID2: string - - let playlistElementServer1Video4: number - let playlistElementServer1Video5: number - let playlistElementNSFW: number - - let nsfwVideoServer1: number - - let userTokenServer1: string - - let commands: PlaylistsCommand[] - - before(async function () { - this.timeout(240000) - - servers = await createMultipleServers(3) - - // Get the access tokens - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - await setDefaultAccountAvatar(servers) - - for (const server of servers) { - await server.config.disableTranscoding() - } - - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - // Server 1 and server 3 follow each other - await doubleFollow(servers[0], servers[2]) - - commands = servers.map(s => s.playlists) - - { - servers[0].store.videos = [] - servers[1].store.videos = [] - servers[2].store.videos = [] - - for (const server of servers) { - for (let i = 0; i < 7; i++) { - const name = `video ${i} server ${server.serverNumber}` - const video = await server.videos.upload({ attributes: { name, nsfw: false } }) - - server.store.videos.push(video) - } - } - } - - nsfwVideoServer1 = (await servers[0].videos.quickUpload({ name: 'NSFW video', nsfw: true })).id - - userTokenServer1 = await servers[0].users.generateUserAndToken('user1') - - await waitJobs(servers) - }) - - describe('Check playlists filters and privacies', function () { - - it('Should list video playlist privacies', async function () { - const privacies = await commands[0].getPrivacies() - - expect(Object.keys(privacies)).to.have.length.at.least(3) - expect(privacies[3]).to.equal('Private') - }) - - it('Should filter on playlist type', async function () { - this.timeout(30000) - - const token = servers[0].accessToken - - await commands[0].create({ - attributes: { - displayName: 'my super playlist', - privacy: VideoPlaylistPrivacy.PUBLIC, - description: 'my super description', - thumbnailfile: 'custom-thumbnail.jpg', - videoChannelId: servers[0].store.channel.id - } - }) - - { - const body = await commands[0].listByAccount({ token, handle: 'root', playlistType: VideoPlaylistType.WATCH_LATER }) - - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - - const playlist = body.data[0] - expect(playlist.displayName).to.equal('Watch later') - expect(playlist.type.id).to.equal(VideoPlaylistType.WATCH_LATER) - expect(playlist.type.label).to.equal('Watch later') - expect(playlist.privacy.id).to.equal(VideoPlaylistPrivacy.PRIVATE) - } - - { - const bodyList = await commands[0].list({ playlistType: VideoPlaylistType.WATCH_LATER }) - const bodyChannel = await commands[0].listByChannel({ handle: 'root_channel', playlistType: VideoPlaylistType.WATCH_LATER }) - - for (const body of [ bodyList, bodyChannel ]) { - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - } - } - - { - const bodyList = await commands[0].list({ playlistType: VideoPlaylistType.REGULAR }) - const bodyChannel = await commands[0].listByChannel({ handle: 'root_channel', playlistType: VideoPlaylistType.REGULAR }) - - let playlist: VideoPlaylist = null - for (const body of [ bodyList, bodyChannel ]) { - - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - - playlist = body.data[0] - expect(playlist.displayName).to.equal('my super playlist') - expect(playlist.privacy.id).to.equal(VideoPlaylistPrivacy.PUBLIC) - expect(playlist.type.id).to.equal(VideoPlaylistType.REGULAR) - } - - await commands[0].update({ - playlistId: playlist.id, - attributes: { - privacy: VideoPlaylistPrivacy.PRIVATE - } - }) - } - - { - const bodyList = await commands[0].list({ playlistType: VideoPlaylistType.REGULAR }) - const bodyChannel = await commands[0].listByChannel({ handle: 'root_channel', playlistType: VideoPlaylistType.REGULAR }) - - for (const body of [ bodyList, bodyChannel ]) { - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - } - } - - { - const body = await commands[0].listByAccount({ handle: 'root' }) - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - } - }) - - it('Should get private playlist for a classic user', async function () { - const token = await servers[0].users.generateUserAndToken('toto') - - const body = await commands[0].listByAccount({ token, handle: 'toto' }) - - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - - const playlistId = body.data[0].id - await commands[0].listVideos({ token, playlistId }) - }) - }) - - describe('Create and federate playlists', function () { - - it('Should create a playlist on server 1 and have the playlist on server 2 and 3', async function () { - this.timeout(30000) - - await commands[0].create({ - attributes: { - displayName: 'my super playlist', - privacy: VideoPlaylistPrivacy.PUBLIC, - description: 'my super description', - thumbnailfile: 'custom-thumbnail.jpg', - videoChannelId: servers[0].store.channel.id - } - }) - - await waitJobs(servers) - // Processing a playlist by the receiver could be long - await wait(3000) - - for (const server of servers) { - const body = await server.playlists.list({ start: 0, count: 5 }) - expect(body.total).to.equal(1) - expect(body.data).to.have.lengthOf(1) - - const playlistFromList = body.data[0] - - const playlistFromGet = await server.playlists.get({ playlistId: playlistFromList.uuid }) - - for (const playlist of [ playlistFromGet, playlistFromList ]) { - expect(playlist.id).to.be.a('number') - expect(playlist.uuid).to.be.a('string') - - expect(playlist.isLocal).to.equal(server.serverNumber === 1) - - expect(playlist.displayName).to.equal('my super playlist') - expect(playlist.description).to.equal('my super description') - expect(playlist.privacy.id).to.equal(VideoPlaylistPrivacy.PUBLIC) - expect(playlist.privacy.label).to.equal('Public') - expect(playlist.type.id).to.equal(VideoPlaylistType.REGULAR) - expect(playlist.type.label).to.equal('Regular') - expect(playlist.embedPath).to.equal('/video-playlists/embed/' + playlist.uuid) - - expect(playlist.videosLength).to.equal(0) - - expect(playlist.ownerAccount.name).to.equal('root') - expect(playlist.ownerAccount.displayName).to.equal('root') - expect(playlist.videoChannel.name).to.equal('root_channel') - expect(playlist.videoChannel.displayName).to.equal('Main root channel') - } - } - }) - - it('Should create a playlist on server 2 and have the playlist on server 1 but not on server 3', async function () { - this.timeout(30000) - - { - const playlist = await servers[1].playlists.create({ - attributes: { - displayName: 'playlist 2', - privacy: VideoPlaylistPrivacy.PUBLIC, - videoChannelId: servers[1].store.channel.id - } - }) - playlistServer2Id1 = playlist.id - } - - { - const playlist = await servers[1].playlists.create({ - attributes: { - displayName: 'playlist 3', - privacy: VideoPlaylistPrivacy.PUBLIC, - thumbnailfile: 'custom-thumbnail.jpg', - videoChannelId: servers[1].store.channel.id - } - }) - - playlistServer2Id2 = playlist.id - playlistServer2UUID2 = playlist.uuid - } - - for (const id of [ playlistServer2Id1, playlistServer2Id2 ]) { - await servers[1].playlists.addElement({ - playlistId: id, - attributes: { videoId: servers[1].store.videos[0].id, startTimestamp: 1, stopTimestamp: 2 } - }) - await servers[1].playlists.addElement({ - playlistId: id, - attributes: { videoId: servers[1].store.videos[1].id } - }) - } - - await waitJobs(servers) - await wait(3000) - - for (const server of [ servers[0], servers[1] ]) { - const body = await server.playlists.list({ start: 0, count: 5 }) - - const playlist2 = body.data.find(p => p.displayName === 'playlist 2') - expect(playlist2).to.not.be.undefined - await testImageGeneratedByFFmpeg(server.url, 'thumbnail-playlist', playlist2.thumbnailPath) - - const playlist3 = body.data.find(p => p.displayName === 'playlist 3') - expect(playlist3).to.not.be.undefined - await testImageGeneratedByFFmpeg(server.url, 'custom-thumbnail', playlist3.thumbnailPath) - } - - const body = await servers[2].playlists.list({ start: 0, count: 5 }) - expect(body.data.find(p => p.displayName === 'playlist 2')).to.be.undefined - expect(body.data.find(p => p.displayName === 'playlist 3')).to.be.undefined - }) - - it('Should have the playlist on server 3 after a new follow', async function () { - this.timeout(30000) - - // Server 2 and server 3 follow each other - await doubleFollow(servers[1], servers[2]) - - const body = await servers[2].playlists.list({ start: 0, count: 5 }) - - const playlist2 = body.data.find(p => p.displayName === 'playlist 2') - expect(playlist2).to.not.be.undefined - await testImageGeneratedByFFmpeg(servers[2].url, 'thumbnail-playlist', playlist2.thumbnailPath) - - expect(body.data.find(p => p.displayName === 'playlist 3')).to.not.be.undefined - }) - }) - - describe('List playlists', function () { - - it('Should correctly list the playlists', async function () { - this.timeout(30000) - - { - const body = await servers[2].playlists.list({ start: 1, count: 2, sort: 'createdAt' }) - expect(body.total).to.equal(3) - - const data = body.data - expect(data).to.have.lengthOf(2) - expect(data[0].displayName).to.equal('playlist 2') - expect(data[1].displayName).to.equal('playlist 3') - } - - { - const body = await servers[2].playlists.list({ start: 1, count: 2, sort: '-createdAt' }) - expect(body.total).to.equal(3) - - const data = body.data - expect(data).to.have.lengthOf(2) - expect(data[0].displayName).to.equal('playlist 2') - expect(data[1].displayName).to.equal('my super playlist') - } - }) - - it('Should list video channel playlists', async function () { - this.timeout(30000) - - { - const body = await commands[0].listByChannel({ handle: 'root_channel', start: 0, count: 2, sort: '-createdAt' }) - expect(body.total).to.equal(1) - - const data = body.data - expect(data).to.have.lengthOf(1) - expect(data[0].displayName).to.equal('my super playlist') - } - }) - - it('Should list account playlists', async function () { - this.timeout(30000) - - { - const body = await servers[1].playlists.listByAccount({ handle: 'root', start: 1, count: 2, sort: '-createdAt' }) - expect(body.total).to.equal(2) - - const data = body.data - expect(data).to.have.lengthOf(1) - expect(data[0].displayName).to.equal('playlist 2') - } - - { - const body = await servers[1].playlists.listByAccount({ handle: 'root', start: 1, count: 2, sort: 'createdAt' }) - expect(body.total).to.equal(2) - - const data = body.data - expect(data).to.have.lengthOf(1) - expect(data[0].displayName).to.equal('playlist 3') - } - - { - const body = await servers[1].playlists.listByAccount({ handle: 'root', sort: 'createdAt', search: '3' }) - expect(body.total).to.equal(1) - - const data = body.data - expect(data).to.have.lengthOf(1) - expect(data[0].displayName).to.equal('playlist 3') - } - - { - const body = await servers[1].playlists.listByAccount({ handle: 'root', sort: 'createdAt', search: '4' }) - expect(body.total).to.equal(0) - - const data = body.data - expect(data).to.have.lengthOf(0) - } - }) - }) - - describe('Playlist rights', function () { - let unlistedPlaylist: VideoPlaylistCreateResult - let privatePlaylist: VideoPlaylistCreateResult - - before(async function () { - this.timeout(30000) - - { - unlistedPlaylist = await servers[1].playlists.create({ - attributes: { - displayName: 'playlist unlisted', - privacy: VideoPlaylistPrivacy.UNLISTED, - videoChannelId: servers[1].store.channel.id - } - }) - } - - { - privatePlaylist = await servers[1].playlists.create({ - attributes: { - displayName: 'playlist private', - privacy: VideoPlaylistPrivacy.PRIVATE - } - }) - } - - await waitJobs(servers) - await wait(3000) - }) - - it('Should not list unlisted or private playlists', async function () { - for (const server of servers) { - const results = [ - await server.playlists.listByAccount({ handle: 'root@' + servers[1].host, sort: '-createdAt' }), - await server.playlists.list({ start: 0, count: 2, sort: '-createdAt' }) - ] - - expect(results[0].total).to.equal(2) - expect(results[1].total).to.equal(3) - - for (const body of results) { - const data = body.data - expect(data).to.have.lengthOf(2) - expect(data[0].displayName).to.equal('playlist 3') - expect(data[1].displayName).to.equal('playlist 2') - } - } - }) - - it('Should not get unlisted playlist using only the id', async function () { - await servers[1].playlists.get({ playlistId: unlistedPlaylist.id, expectedStatus: 404 }) - }) - - it('Should get unlisted playlist using uuid or shortUUID', async function () { - await servers[1].playlists.get({ playlistId: unlistedPlaylist.uuid }) - await servers[1].playlists.get({ playlistId: unlistedPlaylist.shortUUID }) - }) - - it('Should not get private playlist without token', async function () { - for (const id of [ privatePlaylist.id, privatePlaylist.uuid, privatePlaylist.shortUUID ]) { - await servers[1].playlists.get({ playlistId: id, expectedStatus: 401 }) - } - }) - - it('Should get private playlist with a token', async function () { - for (const id of [ privatePlaylist.id, privatePlaylist.uuid, privatePlaylist.shortUUID ]) { - await servers[1].playlists.get({ token: servers[1].accessToken, playlistId: id }) - } - }) - }) - - describe('Update playlists', function () { - - it('Should update a playlist', async function () { - this.timeout(30000) - - await servers[1].playlists.update({ - attributes: { - displayName: 'playlist 3 updated', - description: 'description updated', - privacy: VideoPlaylistPrivacy.UNLISTED, - thumbnailfile: 'custom-thumbnail.jpg', - videoChannelId: servers[1].store.channel.id - }, - playlistId: playlistServer2Id2 - }) - - await waitJobs(servers) - - for (const server of servers) { - const playlist = await server.playlists.get({ playlistId: playlistServer2UUID2 }) - - expect(playlist.displayName).to.equal('playlist 3 updated') - expect(playlist.description).to.equal('description updated') - - expect(playlist.privacy.id).to.equal(VideoPlaylistPrivacy.UNLISTED) - expect(playlist.privacy.label).to.equal('Unlisted') - - expect(playlist.type.id).to.equal(VideoPlaylistType.REGULAR) - expect(playlist.type.label).to.equal('Regular') - - expect(playlist.videosLength).to.equal(2) - - expect(playlist.ownerAccount.name).to.equal('root') - expect(playlist.ownerAccount.displayName).to.equal('root') - expect(playlist.videoChannel.name).to.equal('root_channel') - expect(playlist.videoChannel.displayName).to.equal('Main root channel') - } - }) - }) - - describe('Element timestamps', function () { - - it('Should create a playlist containing different startTimestamp/endTimestamp videos', async function () { - this.timeout(30000) - - const addVideo = (attributes: any) => { - return commands[0].addElement({ playlistId: playlistServer1Id, attributes }) - } - - const playlistDisplayName = 'playlist 4' - const playlist = await commands[0].create({ - attributes: { - displayName: playlistDisplayName, - privacy: VideoPlaylistPrivacy.PUBLIC, - videoChannelId: servers[0].store.channel.id - } - }) - - playlistServer1Id = playlist.id - playlistServer1DisplayName = playlistDisplayName - playlistServer1UUID = playlist.uuid - - await addVideo({ videoId: servers[0].store.videos[0].uuid, startTimestamp: 15, stopTimestamp: 28 }) - await addVideo({ videoId: servers[2].store.videos[1].uuid, startTimestamp: 35 }) - await addVideo({ videoId: servers[2].store.videos[2].uuid }) - { - const element = await addVideo({ videoId: servers[0].store.videos[3].uuid, stopTimestamp: 35 }) - playlistElementServer1Video4 = element.id - } - - { - const element = await addVideo({ videoId: servers[0].store.videos[4].uuid, startTimestamp: 45, stopTimestamp: 60 }) - playlistElementServer1Video5 = element.id - } - - { - const element = await addVideo({ videoId: nsfwVideoServer1, startTimestamp: 5 }) - playlistElementNSFW = element.id - - await addVideo({ videoId: nsfwVideoServer1, startTimestamp: 4 }) - await addVideo({ videoId: nsfwVideoServer1 }) - } - - await waitJobs(servers) - }) - - it('Should correctly list playlist videos', async function () { - this.timeout(30000) - - for (const server of servers) { - { - const body = await server.playlists.listVideos({ playlistId: playlistServer1UUID, start: 0, count: 10 }) - - expect(body.total).to.equal(8) - - const videoElements = body.data - expect(videoElements).to.have.lengthOf(8) - - expect(videoElements[0].video.name).to.equal('video 0 server 1') - expect(videoElements[0].position).to.equal(1) - expect(videoElements[0].startTimestamp).to.equal(15) - expect(videoElements[0].stopTimestamp).to.equal(28) - - expect(videoElements[1].video.name).to.equal('video 1 server 3') - expect(videoElements[1].position).to.equal(2) - expect(videoElements[1].startTimestamp).to.equal(35) - expect(videoElements[1].stopTimestamp).to.be.null - - expect(videoElements[2].video.name).to.equal('video 2 server 3') - expect(videoElements[2].position).to.equal(3) - expect(videoElements[2].startTimestamp).to.be.null - expect(videoElements[2].stopTimestamp).to.be.null - - expect(videoElements[3].video.name).to.equal('video 3 server 1') - expect(videoElements[3].position).to.equal(4) - expect(videoElements[3].startTimestamp).to.be.null - expect(videoElements[3].stopTimestamp).to.equal(35) - - expect(videoElements[4].video.name).to.equal('video 4 server 1') - expect(videoElements[4].position).to.equal(5) - expect(videoElements[4].startTimestamp).to.equal(45) - expect(videoElements[4].stopTimestamp).to.equal(60) - - expect(videoElements[5].video.name).to.equal('NSFW video') - expect(videoElements[5].position).to.equal(6) - expect(videoElements[5].startTimestamp).to.equal(5) - expect(videoElements[5].stopTimestamp).to.be.null - - expect(videoElements[6].video.name).to.equal('NSFW video') - expect(videoElements[6].position).to.equal(7) - expect(videoElements[6].startTimestamp).to.equal(4) - expect(videoElements[6].stopTimestamp).to.be.null - - expect(videoElements[7].video.name).to.equal('NSFW video') - expect(videoElements[7].position).to.equal(8) - expect(videoElements[7].startTimestamp).to.be.null - expect(videoElements[7].stopTimestamp).to.be.null - } - - { - const body = await server.playlists.listVideos({ playlistId: playlistServer1UUID, start: 0, count: 2 }) - expect(body.data).to.have.lengthOf(2) - } - } - }) - }) - - describe('Element type', function () { - let groupUser1: PeerTubeServer[] - let groupWithoutToken1: PeerTubeServer[] - let group1: PeerTubeServer[] - let group2: PeerTubeServer[] - - let video1: string - let video2: string - let video3: string - - before(async function () { - this.timeout(60000) - - groupUser1 = [ Object.assign({}, servers[0], { accessToken: userTokenServer1 }) ] - groupWithoutToken1 = [ Object.assign({}, servers[0], { accessToken: undefined }) ] - group1 = [ servers[0] ] - group2 = [ servers[1], servers[2] ] - - const playlist = await commands[0].create({ - token: userTokenServer1, - attributes: { - displayName: 'playlist 56', - privacy: VideoPlaylistPrivacy.PUBLIC, - videoChannelId: servers[0].store.channel.id - } - }) - - const playlistServer1Id2 = playlist.id - playlistServer1UUID2 = playlist.uuid - - const addVideo = (attributes: any) => { - return commands[0].addElement({ token: userTokenServer1, playlistId: playlistServer1Id2, attributes }) - } - - video1 = (await servers[0].videos.quickUpload({ name: 'video 89', token: userTokenServer1 })).uuid - video2 = (await servers[1].videos.quickUpload({ name: 'video 90' })).uuid - video3 = (await servers[0].videos.quickUpload({ name: 'video 91', nsfw: true })).uuid - - await waitJobs(servers) - - await addVideo({ videoId: video1, startTimestamp: 15, stopTimestamp: 28 }) - await addVideo({ videoId: video2, startTimestamp: 35 }) - await addVideo({ videoId: video3 }) - - await waitJobs(servers) - }) - - it('Should update the element type if the video is private/password protected', async function () { - this.timeout(20000) - - const name = 'video 89' - const position = 1 - - { - await servers[0].videos.update({ id: video1, attributes: { privacy: VideoPrivacy.PRIVATE } }) - await waitJobs(servers) - - await checkPlaylistElementType(groupUser1, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) - await checkPlaylistElementType(groupWithoutToken1, playlistServer1UUID2, VideoPlaylistElementType.PRIVATE, position, name, 3) - await checkPlaylistElementType(group1, playlistServer1UUID2, VideoPlaylistElementType.PRIVATE, position, name, 3) - await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.DELETED, position, name, 3) - } - - { - await servers[0].videos.update({ - id: video1, - attributes: { privacy: VideoPrivacy.PASSWORD_PROTECTED, videoPasswords: [ 'password' ] } - }) - await waitJobs(servers) - - await checkPlaylistElementType(groupUser1, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) - await checkPlaylistElementType(groupWithoutToken1, playlistServer1UUID2, VideoPlaylistElementType.PRIVATE, position, name, 3) - await checkPlaylistElementType(group1, playlistServer1UUID2, VideoPlaylistElementType.PRIVATE, position, name, 3) - await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.DELETED, position, name, 3) - } - - { - await servers[0].videos.update({ id: video1, attributes: { privacy: VideoPrivacy.PUBLIC } }) - await waitJobs(servers) - - await checkPlaylistElementType(groupUser1, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) - await checkPlaylistElementType(groupWithoutToken1, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) - await checkPlaylistElementType(group1, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) - // We deleted the video, so even if we recreated it, the old entry is still deleted - await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.DELETED, position, name, 3) - } - }) - - it('Should update the element type if the video is blacklisted', async function () { - this.timeout(20000) - - const name = 'video 89' - const position = 1 - - { - await servers[0].blacklist.add({ videoId: video1, reason: 'reason', unfederate: true }) - await waitJobs(servers) - - await checkPlaylistElementType(groupUser1, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) - await checkPlaylistElementType(groupWithoutToken1, playlistServer1UUID2, VideoPlaylistElementType.UNAVAILABLE, position, name, 3) - await checkPlaylistElementType(group1, playlistServer1UUID2, VideoPlaylistElementType.UNAVAILABLE, position, name, 3) - await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.DELETED, position, name, 3) - } - - { - await servers[0].blacklist.remove({ videoId: video1 }) - await waitJobs(servers) - - await checkPlaylistElementType(groupUser1, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) - await checkPlaylistElementType(groupWithoutToken1, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) - await checkPlaylistElementType(group1, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) - // We deleted the video (because unfederated), so even if we recreated it, the old entry is still deleted - await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.DELETED, position, name, 3) - } - }) - - it('Should update the element type if the account or server of the video is blocked', async function () { - this.timeout(90000) - - const command = servers[0].blocklist - - const name = 'video 90' - const position = 2 - - { - await command.addToMyBlocklist({ token: userTokenServer1, account: 'root@' + servers[1].host }) - await waitJobs(servers) - - await checkPlaylistElementType(groupUser1, playlistServer1UUID2, VideoPlaylistElementType.UNAVAILABLE, position, name, 3) - await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) - - await command.removeFromMyBlocklist({ token: userTokenServer1, account: 'root@' + servers[1].host }) - await waitJobs(servers) - - await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) - } - - { - await command.addToMyBlocklist({ token: userTokenServer1, server: servers[1].host }) - await waitJobs(servers) - - await checkPlaylistElementType(groupUser1, playlistServer1UUID2, VideoPlaylistElementType.UNAVAILABLE, position, name, 3) - await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) - - await command.removeFromMyBlocklist({ token: userTokenServer1, server: servers[1].host }) - await waitJobs(servers) - - await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) - } - - { - await command.addToServerBlocklist({ account: 'root@' + servers[1].host }) - await waitJobs(servers) - - await checkPlaylistElementType(groupUser1, playlistServer1UUID2, VideoPlaylistElementType.UNAVAILABLE, position, name, 3) - await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) - - await command.removeFromServerBlocklist({ account: 'root@' + servers[1].host }) - await waitJobs(servers) - - await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) - } - - { - await command.addToServerBlocklist({ server: servers[1].host }) - await waitJobs(servers) - - await checkPlaylistElementType(groupUser1, playlistServer1UUID2, VideoPlaylistElementType.UNAVAILABLE, position, name, 3) - await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) - - await command.removeFromServerBlocklist({ server: servers[1].host }) - await waitJobs(servers) - - await checkPlaylistElementType(group2, playlistServer1UUID2, VideoPlaylistElementType.REGULAR, position, name, 3) - } - }) - }) - - describe('Managing playlist elements', function () { - - it('Should reorder the playlist', async function () { - this.timeout(30000) - - { - await commands[0].reorderElements({ - playlistId: playlistServer1Id, - attributes: { - startPosition: 2, - insertAfterPosition: 3 - } - }) - - await waitJobs(servers) - - for (const server of servers) { - const body = await server.playlists.listVideos({ playlistId: playlistServer1UUID, start: 0, count: 10 }) - const names = body.data.map(v => v.video.name) - - expect(names).to.deep.equal([ - 'video 0 server 1', - 'video 2 server 3', - 'video 1 server 3', - 'video 3 server 1', - 'video 4 server 1', - 'NSFW video', - 'NSFW video', - 'NSFW video' - ]) - } - } - - { - await commands[0].reorderElements({ - playlistId: playlistServer1Id, - attributes: { - startPosition: 1, - reorderLength: 3, - insertAfterPosition: 4 - } - }) - - await waitJobs(servers) - - for (const server of servers) { - const body = await server.playlists.listVideos({ playlistId: playlistServer1UUID, start: 0, count: 10 }) - const names = body.data.map(v => v.video.name) - - expect(names).to.deep.equal([ - 'video 3 server 1', - 'video 0 server 1', - 'video 2 server 3', - 'video 1 server 3', - 'video 4 server 1', - 'NSFW video', - 'NSFW video', - 'NSFW video' - ]) - } - } - - { - await commands[0].reorderElements({ - playlistId: playlistServer1Id, - attributes: { - startPosition: 6, - insertAfterPosition: 3 - } - }) - - await waitJobs(servers) - - for (const server of servers) { - const { data: elements } = await server.playlists.listVideos({ playlistId: playlistServer1UUID, start: 0, count: 10 }) - const names = elements.map(v => v.video.name) - - expect(names).to.deep.equal([ - 'video 3 server 1', - 'video 0 server 1', - 'video 2 server 3', - 'NSFW video', - 'video 1 server 3', - 'video 4 server 1', - 'NSFW video', - 'NSFW video' - ]) - - for (let i = 1; i <= elements.length; i++) { - expect(elements[i - 1].position).to.equal(i) - } - } - } - }) - - it('Should update startTimestamp/endTimestamp of some elements', async function () { - this.timeout(30000) - - await commands[0].updateElement({ - playlistId: playlistServer1Id, - elementId: playlistElementServer1Video4, - attributes: { - startTimestamp: 1 - } - }) - - await commands[0].updateElement({ - playlistId: playlistServer1Id, - elementId: playlistElementServer1Video5, - attributes: { - stopTimestamp: null - } - }) - - await waitJobs(servers) - - for (const server of servers) { - const { data: elements } = await server.playlists.listVideos({ playlistId: playlistServer1UUID, start: 0, count: 10 }) - - expect(elements[0].video.name).to.equal('video 3 server 1') - expect(elements[0].position).to.equal(1) - expect(elements[0].startTimestamp).to.equal(1) - expect(elements[0].stopTimestamp).to.equal(35) - - expect(elements[5].video.name).to.equal('video 4 server 1') - expect(elements[5].position).to.equal(6) - expect(elements[5].startTimestamp).to.equal(45) - expect(elements[5].stopTimestamp).to.be.null - } - }) - - it('Should check videos existence in my playlist', async function () { - const videoIds = [ - servers[0].store.videos[0].id, - 42000, - servers[0].store.videos[3].id, - 43000, - servers[0].store.videos[4].id - ] - const obj = await commands[0].videosExist({ videoIds }) - - { - const elem = obj[servers[0].store.videos[0].id] - expect(elem).to.have.lengthOf(1) - expect(elem[0].playlistElementId).to.exist - expect(elem[0].playlistDisplayName).to.equal(playlistServer1DisplayName) - expect(elem[0].playlistShortUUID).to.equal(uuidToShort(playlistServer1UUID)) - expect(elem[0].playlistId).to.equal(playlistServer1Id) - expect(elem[0].startTimestamp).to.equal(15) - expect(elem[0].stopTimestamp).to.equal(28) - } - - { - const elem = obj[servers[0].store.videos[3].id] - expect(elem).to.have.lengthOf(1) - expect(elem[0].playlistElementId).to.equal(playlistElementServer1Video4) - expect(elem[0].playlistDisplayName).to.equal(playlistServer1DisplayName) - expect(elem[0].playlistShortUUID).to.equal(uuidToShort(playlistServer1UUID)) - expect(elem[0].playlistId).to.equal(playlistServer1Id) - expect(elem[0].startTimestamp).to.equal(1) - expect(elem[0].stopTimestamp).to.equal(35) - } - - { - const elem = obj[servers[0].store.videos[4].id] - expect(elem).to.have.lengthOf(1) - expect(elem[0].playlistId).to.equal(playlistServer1Id) - expect(elem[0].playlistDisplayName).to.equal(playlistServer1DisplayName) - expect(elem[0].playlistShortUUID).to.equal(uuidToShort(playlistServer1UUID)) - expect(elem[0].startTimestamp).to.equal(45) - expect(elem[0].stopTimestamp).to.equal(null) - } - - expect(obj[42000]).to.have.lengthOf(0) - expect(obj[43000]).to.have.lengthOf(0) - }) - - it('Should automatically update updatedAt field of playlists', async function () { - const server = servers[1] - const videoId = servers[1].store.videos[5].id - - async function getPlaylistNames () { - const { data } = await server.playlists.listByAccount({ token: server.accessToken, handle: 'root', sort: '-updatedAt' }) - - return data.map(p => p.displayName) - } - - const attributes = { videoId } - const element1 = await server.playlists.addElement({ playlistId: playlistServer2Id1, attributes }) - const element2 = await server.playlists.addElement({ playlistId: playlistServer2Id2, attributes }) - - const names1 = await getPlaylistNames() - expect(names1[0]).to.equal('playlist 3 updated') - expect(names1[1]).to.equal('playlist 2') - - await server.playlists.removeElement({ playlistId: playlistServer2Id1, elementId: element1.id }) - - const names2 = await getPlaylistNames() - expect(names2[0]).to.equal('playlist 2') - expect(names2[1]).to.equal('playlist 3 updated') - - await server.playlists.removeElement({ playlistId: playlistServer2Id2, elementId: element2.id }) - - const names3 = await getPlaylistNames() - expect(names3[0]).to.equal('playlist 3 updated') - expect(names3[1]).to.equal('playlist 2') - }) - - it('Should delete some elements', async function () { - this.timeout(30000) - - await commands[0].removeElement({ playlistId: playlistServer1Id, elementId: playlistElementServer1Video4 }) - await commands[0].removeElement({ playlistId: playlistServer1Id, elementId: playlistElementNSFW }) - - await waitJobs(servers) - - for (const server of servers) { - const body = await server.playlists.listVideos({ playlistId: playlistServer1UUID, start: 0, count: 10 }) - expect(body.total).to.equal(6) - - const elements = body.data - expect(elements).to.have.lengthOf(6) - - expect(elements[0].video.name).to.equal('video 0 server 1') - expect(elements[0].position).to.equal(1) - - expect(elements[1].video.name).to.equal('video 2 server 3') - expect(elements[1].position).to.equal(2) - - expect(elements[2].video.name).to.equal('video 1 server 3') - expect(elements[2].position).to.equal(3) - - expect(elements[3].video.name).to.equal('video 4 server 1') - expect(elements[3].position).to.equal(4) - - expect(elements[4].video.name).to.equal('NSFW video') - expect(elements[4].position).to.equal(5) - - expect(elements[5].video.name).to.equal('NSFW video') - expect(elements[5].position).to.equal(6) - } - }) - - it('Should be able to create a public playlist, and set it to private', async function () { - this.timeout(30000) - - const videoPlaylistIds = await commands[0].create({ - attributes: { - displayName: 'my super public playlist', - privacy: VideoPlaylistPrivacy.PUBLIC, - videoChannelId: servers[0].store.channel.id - } - }) - - await waitJobs(servers) - - for (const server of servers) { - await server.playlists.get({ playlistId: videoPlaylistIds.uuid, expectedStatus: HttpStatusCode.OK_200 }) - } - - const attributes = { privacy: VideoPlaylistPrivacy.PRIVATE } - await commands[0].update({ playlistId: videoPlaylistIds.id, attributes }) - - await waitJobs(servers) - - for (const server of [ servers[1], servers[2] ]) { - await server.playlists.get({ playlistId: videoPlaylistIds.uuid, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - } - - await commands[0].get({ playlistId: videoPlaylistIds.uuid, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - await commands[0].get({ token: servers[0].accessToken, playlistId: videoPlaylistIds.uuid, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - describe('Playlist deletion', function () { - - it('Should delete the playlist on server 1 and delete on server 2 and 3', async function () { - this.timeout(30000) - - await commands[0].delete({ playlistId: playlistServer1Id }) - - await waitJobs(servers) - - for (const server of servers) { - await server.playlists.get({ playlistId: playlistServer1UUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - } - }) - - it('Should have deleted the thumbnail on server 1, 2 and 3', async function () { - this.timeout(30000) - - for (const server of servers) { - await checkPlaylistFilesWereRemoved(playlistServer1UUID, server) - } - }) - - it('Should unfollow servers 1 and 2 and hide their playlists', async function () { - this.timeout(30000) - - const finder = (data: VideoPlaylist[]) => data.find(p => p.displayName === 'my super playlist') - - { - const body = await servers[2].playlists.list({ start: 0, count: 5 }) - expect(body.total).to.equal(3) - - expect(finder(body.data)).to.not.be.undefined - } - - await servers[2].follows.unfollow({ target: servers[0] }) - - { - const body = await servers[2].playlists.list({ start: 0, count: 5 }) - expect(body.total).to.equal(1) - - expect(finder(body.data)).to.be.undefined - } - }) - - it('Should delete a channel and put the associated playlist in private mode', async function () { - this.timeout(30000) - - const channel = await servers[0].channels.create({ attributes: { name: 'super_channel', displayName: 'super channel' } }) - - const playlistCreated = await commands[0].create({ - attributes: { - displayName: 'channel playlist', - privacy: VideoPlaylistPrivacy.PUBLIC, - videoChannelId: channel.id - } - }) - - await waitJobs(servers) - - await servers[0].channels.delete({ channelName: 'super_channel' }) - - await waitJobs(servers) - - const body = await commands[0].get({ token: servers[0].accessToken, playlistId: playlistCreated.uuid }) - expect(body.displayName).to.equal('channel playlist') - expect(body.privacy.id).to.equal(VideoPlaylistPrivacy.PRIVATE) - - await servers[1].playlists.get({ playlistId: playlistCreated.uuid, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - - it('Should delete an account and delete its playlists', async function () { - this.timeout(30000) - - const { userId, token } = await servers[0].users.generate('user_1') - - const { videoChannels } = await servers[0].users.getMyInfo({ token }) - const userChannel = videoChannels[0] - - await commands[0].create({ - attributes: { - displayName: 'playlist to be deleted', - privacy: VideoPlaylistPrivacy.PUBLIC, - videoChannelId: userChannel.id - } - }) - - await waitJobs(servers) - - const finder = (data: VideoPlaylist[]) => data.find(p => p.displayName === 'playlist to be deleted') - - { - for (const server of [ servers[0], servers[1] ]) { - const body = await server.playlists.list({ start: 0, count: 15 }) - - expect(finder(body.data)).to.not.be.undefined - } - } - - await servers[0].users.remove({ userId }) - await waitJobs(servers) - - { - for (const server of [ servers[0], servers[1] ]) { - const body = await server.playlists.list({ start: 0, count: 15 }) - - expect(finder(body.data)).to.be.undefined - } - } - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/videos/video-privacy.ts b/server/tests/api/videos/video-privacy.ts deleted file mode 100644 index de96bcfcc..000000000 --- a/server/tests/api/videos/video-privacy.ts +++ /dev/null @@ -1,287 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { wait } from '@shared/core-utils' -import { HttpStatusCode, VideoCreateResult, VideoPrivacy } from '@shared/models' -import { cleanupTests, createSingleServer, doubleFollow, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/server-commands' - -describe('Test video privacy', function () { - const servers: PeerTubeServer[] = [] - let anotherUserToken: string - - let privateVideoId: number - let privateVideoUUID: string - - let internalVideoId: number - let internalVideoUUID: string - - let unlistedVideo: VideoCreateResult - let nonFederatedUnlistedVideoUUID: string - - let now: number - - const dontFederateUnlistedConfig = { - federation: { - videos: { - federate_unlisted: false - } - } - } - - before(async function () { - this.timeout(50000) - - // Run servers - servers.push(await createSingleServer(1, dontFederateUnlistedConfig)) - servers.push(await createSingleServer(2)) - - // Get the access tokens - await setAccessTokensToServers(servers) - - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - }) - - describe('Private and internal videos', function () { - - it('Should upload a private and internal videos on server 1', async function () { - this.timeout(50000) - - for (const privacy of [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]) { - const attributes = { privacy } - await servers[0].videos.upload({ attributes }) - } - - await waitJobs(servers) - }) - - it('Should not have these private and internal videos on server 2', async function () { - const { total, data } = await servers[1].videos.list() - - expect(total).to.equal(0) - expect(data).to.have.lengthOf(0) - }) - - it('Should not list the private and internal videos for an unauthenticated user on server 1', async function () { - const { total, data } = await servers[0].videos.list() - - expect(total).to.equal(0) - expect(data).to.have.lengthOf(0) - }) - - it('Should not list the private video and list the internal video for an authenticated user on server 1', async function () { - const { total, data } = await servers[0].videos.listWithToken() - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - - expect(data[0].privacy.id).to.equal(VideoPrivacy.INTERNAL) - }) - - it('Should list my (private and internal) videos', async function () { - const { total, data } = await servers[0].videos.listMyVideos() - - expect(total).to.equal(2) - expect(data).to.have.lengthOf(2) - - const privateVideo = data.find(v => v.privacy.id === VideoPrivacy.PRIVATE) - privateVideoId = privateVideo.id - privateVideoUUID = privateVideo.uuid - - const internalVideo = data.find(v => v.privacy.id === VideoPrivacy.INTERNAL) - internalVideoId = internalVideo.id - internalVideoUUID = internalVideo.uuid - }) - - it('Should not be able to watch the private/internal video with non authenticated user', async function () { - await servers[0].videos.get({ id: privateVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - await servers[0].videos.get({ id: internalVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should not be able to watch the private video with another user', async function () { - const user = { - username: 'hello', - password: 'super password' - } - await servers[0].users.create({ username: user.username, password: user.password }) - - anotherUserToken = await servers[0].login.getAccessToken(user) - - await servers[0].videos.getWithToken({ - token: anotherUserToken, - id: privateVideoUUID, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should be able to watch the internal video with another user', async function () { - await servers[0].videos.getWithToken({ token: anotherUserToken, id: internalVideoUUID }) - }) - - it('Should be able to watch the private video with the correct user', async function () { - await servers[0].videos.getWithToken({ id: privateVideoUUID }) - }) - }) - - describe('Unlisted videos', function () { - - it('Should upload an unlisted video on server 2', async function () { - this.timeout(120000) - - const attributes = { - name: 'unlisted video', - privacy: VideoPrivacy.UNLISTED - } - await servers[1].videos.upload({ attributes }) - - // Server 2 has transcoding enabled - await waitJobs(servers) - }) - - it('Should not have this unlisted video listed on server 1 and 2', async function () { - for (const server of servers) { - const { total, data } = await server.videos.list() - - expect(total).to.equal(0) - expect(data).to.have.lengthOf(0) - } - }) - - it('Should list my (unlisted) videos', async function () { - const { total, data } = await servers[1].videos.listMyVideos() - - expect(total).to.equal(1) - expect(data).to.have.lengthOf(1) - - unlistedVideo = data[0] - }) - - it('Should not be able to get this unlisted video using its id', async function () { - await servers[1].videos.get({ id: unlistedVideo.id, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should be able to get this unlisted video using its uuid/shortUUID', async function () { - for (const server of servers) { - for (const id of [ unlistedVideo.uuid, unlistedVideo.shortUUID ]) { - const video = await server.videos.get({ id }) - - expect(video.name).to.equal('unlisted video') - } - } - }) - - it('Should upload a non-federating unlisted video to server 1', async function () { - this.timeout(30000) - - const attributes = { - name: 'unlisted video', - privacy: VideoPrivacy.UNLISTED - } - await servers[0].videos.upload({ attributes }) - - await waitJobs(servers) - }) - - it('Should list my new unlisted video', async function () { - const { total, data } = await servers[0].videos.listMyVideos() - - expect(total).to.equal(3) - expect(data).to.have.lengthOf(3) - - nonFederatedUnlistedVideoUUID = data[0].uuid - }) - - it('Should be able to get non-federated unlisted video from origin', async function () { - const video = await servers[0].videos.get({ id: nonFederatedUnlistedVideoUUID }) - - expect(video.name).to.equal('unlisted video') - }) - - it('Should not be able to get non-federated unlisted video from federated server', async function () { - await servers[1].videos.get({ id: nonFederatedUnlistedVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - }) - }) - - describe('Privacy update', function () { - - it('Should update the private and internal videos to public on server 1', async function () { - this.timeout(100000) - - now = Date.now() - - { - const attributes = { - name: 'private video becomes public', - privacy: VideoPrivacy.PUBLIC - } - - await servers[0].videos.update({ id: privateVideoId, attributes }) - } - - { - const attributes = { - name: 'internal video becomes public', - privacy: VideoPrivacy.PUBLIC - } - await servers[0].videos.update({ id: internalVideoId, attributes }) - } - - await wait(10000) - await waitJobs(servers) - }) - - it('Should have this new public video listed on server 1 and 2', async function () { - for (const server of servers) { - const { total, data } = await server.videos.list() - expect(total).to.equal(2) - expect(data).to.have.lengthOf(2) - - const privateVideo = data.find(v => v.name === 'private video becomes public') - const internalVideo = data.find(v => v.name === 'internal video becomes public') - - expect(privateVideo).to.not.be.undefined - expect(internalVideo).to.not.be.undefined - - expect(new Date(privateVideo.publishedAt).getTime()).to.be.at.least(now) - // We don't change the publish date of internal videos - expect(new Date(internalVideo.publishedAt).getTime()).to.be.below(now) - - expect(privateVideo.privacy.id).to.equal(VideoPrivacy.PUBLIC) - expect(internalVideo.privacy.id).to.equal(VideoPrivacy.PUBLIC) - } - }) - - it('Should set these videos as private and internal', async function () { - await servers[0].videos.update({ id: internalVideoId, attributes: { privacy: VideoPrivacy.PRIVATE } }) - await servers[0].videos.update({ id: privateVideoId, attributes: { privacy: VideoPrivacy.INTERNAL } }) - - await waitJobs(servers) - - for (const server of servers) { - const { total, data } = await server.videos.list() - - expect(total).to.equal(0) - expect(data).to.have.lengthOf(0) - } - - { - const { total, data } = await servers[0].videos.listMyVideos() - expect(total).to.equal(3) - expect(data).to.have.lengthOf(3) - - const privateVideo = data.find(v => v.name === 'private video becomes public') - const internalVideo = data.find(v => v.name === 'internal video becomes public') - - expect(privateVideo).to.not.be.undefined - expect(internalVideo).to.not.be.undefined - - expect(privateVideo.privacy.id).to.equal(VideoPrivacy.INTERNAL) - expect(internalVideo.privacy.id).to.equal(VideoPrivacy.PRIVATE) - } - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/videos/video-schedule-update.ts b/server/tests/api/videos/video-schedule-update.ts deleted file mode 100644 index bf341c648..000000000 --- a/server/tests/api/videos/video-schedule-update.ts +++ /dev/null @@ -1,155 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { wait } from '@shared/core-utils' -import { VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -function in10Seconds () { - const now = new Date() - now.setSeconds(now.getSeconds() + 10) - - return now -} - -describe('Test video update scheduler', function () { - let servers: PeerTubeServer[] = [] - let video2UUID: string - - before(async function () { - this.timeout(30000) - - // Run servers - servers = await createMultipleServers(2) - - await setAccessTokensToServers(servers) - - await doubleFollow(servers[0], servers[1]) - }) - - it('Should upload a video and schedule an update in 10 seconds', async function () { - const attributes = { - name: 'video 1', - privacy: VideoPrivacy.PRIVATE, - scheduleUpdate: { - updateAt: in10Seconds().toISOString(), - privacy: VideoPrivacy.PUBLIC as VideoPrivacy.PUBLIC - } - } - - await servers[0].videos.upload({ attributes }) - - await waitJobs(servers) - }) - - it('Should not list the video (in privacy mode)', async function () { - for (const server of servers) { - const { total } = await server.videos.list() - - expect(total).to.equal(0) - } - }) - - it('Should have my scheduled video in my account videos', async function () { - const { total, data } = await servers[0].videos.listMyVideos() - expect(total).to.equal(1) - - const videoFromList = data[0] - const videoFromGet = await servers[0].videos.getWithToken({ id: videoFromList.uuid }) - - for (const video of [ videoFromList, videoFromGet ]) { - expect(video.name).to.equal('video 1') - expect(video.privacy.id).to.equal(VideoPrivacy.PRIVATE) - expect(new Date(video.scheduledUpdate.updateAt)).to.be.above(new Date()) - expect(video.scheduledUpdate.privacy).to.equal(VideoPrivacy.PUBLIC) - } - }) - - it('Should wait some seconds and have the video in public privacy', async function () { - this.timeout(50000) - - await wait(15000) - await waitJobs(servers) - - for (const server of servers) { - const { total, data } = await server.videos.list() - - expect(total).to.equal(1) - expect(data[0].name).to.equal('video 1') - } - }) - - it('Should upload a video without scheduling an update', async function () { - const attributes = { - name: 'video 2', - privacy: VideoPrivacy.PRIVATE - } - - const { uuid } = await servers[0].videos.upload({ attributes }) - video2UUID = uuid - - await waitJobs(servers) - }) - - it('Should update a video by scheduling an update', async function () { - const attributes = { - name: 'video 2 updated', - scheduleUpdate: { - updateAt: in10Seconds().toISOString(), - privacy: VideoPrivacy.PUBLIC as VideoPrivacy.PUBLIC - } - } - - await servers[0].videos.update({ id: video2UUID, attributes }) - await waitJobs(servers) - }) - - it('Should not display the updated video', async function () { - for (const server of servers) { - const { total } = await server.videos.list() - - expect(total).to.equal(1) - } - }) - - it('Should have my scheduled updated video in my account videos', async function () { - const { total, data } = await servers[0].videos.listMyVideos() - expect(total).to.equal(2) - - const video = data.find(v => v.uuid === video2UUID) - expect(video).not.to.be.undefined - - expect(video.name).to.equal('video 2 updated') - expect(video.privacy.id).to.equal(VideoPrivacy.PRIVATE) - - expect(new Date(video.scheduledUpdate.updateAt)).to.be.above(new Date()) - expect(video.scheduledUpdate.privacy).to.equal(VideoPrivacy.PUBLIC) - }) - - it('Should wait some seconds and have the updated video in public privacy', async function () { - this.timeout(20000) - - await wait(15000) - await waitJobs(servers) - - for (const server of servers) { - const { total, data } = await server.videos.list() - expect(total).to.equal(2) - - const video = data.find(v => v.uuid === video2UUID) - expect(video).not.to.be.undefined - expect(video.name).to.equal('video 2 updated') - } - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/videos/video-source.ts b/server/tests/api/videos/video-source.ts deleted file mode 100644 index 1f394f904..000000000 --- a/server/tests/api/videos/video-source.ts +++ /dev/null @@ -1,447 +0,0 @@ -import { expect } from 'chai' -import { expectStartWith } from '@server/tests/shared' -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { areMockObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils' -import { HttpStatusCode } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - makeGetRequest, - makeRawRequest, - ObjectStorageCommand, - PeerTubeServer, - setAccessTokensToServers, - setDefaultAccountAvatar, - setDefaultVideoChannel, - waitJobs -} from '@shared/server-commands' - -describe('Test a video file replacement', function () { - let servers: PeerTubeServer[] = [] - - let replaceDate: Date - let userToken: string - let uuid: string - - before(async function () { - this.timeout(50000) - - servers = await createMultipleServers(2) - - // Get the access tokens - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - await setDefaultAccountAvatar(servers) - - await servers[0].config.enableFileUpdate() - - userToken = await servers[0].users.generateUserAndToken('user1') - - // Server 1 and server 2 follow each other - await doubleFollow(servers[0], servers[1]) - }) - - describe('Getting latest video source', () => { - const fixture = 'video_short.webm' - const uuids: string[] = [] - - it('Should get the source filename with legacy upload', async function () { - this.timeout(30000) - - const { uuid } = await servers[0].videos.upload({ attributes: { name: 'my video', fixture }, mode: 'legacy' }) - uuids.push(uuid) - - const source = await servers[0].videos.getSource({ id: uuid }) - expect(source.filename).to.equal(fixture) - }) - - it('Should get the source filename with resumable upload', async function () { - this.timeout(30000) - - const { uuid } = await servers[0].videos.upload({ attributes: { name: 'my video', fixture }, mode: 'resumable' }) - uuids.push(uuid) - - const source = await servers[0].videos.getSource({ id: uuid }) - expect(source.filename).to.equal(fixture) - }) - - after(async function () { - this.timeout(60000) - - for (const uuid of uuids) { - await servers[0].videos.remove({ id: uuid }) - } - - await waitJobs(servers) - }) - }) - - describe('Updating video source', function () { - - describe('Filesystem', function () { - - it('Should replace a video file with transcoding disabled', async function () { - this.timeout(120000) - - await servers[0].config.disableTranscoding() - - const { uuid } = await servers[0].videos.quickUpload({ name: 'fs without transcoding', fixture: 'video_short_720p.mp4' }) - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: uuid }) - - const files = getAllFiles(video) - expect(files).to.have.lengthOf(1) - expect(files[0].resolution.id).to.equal(720) - } - - await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_360p.mp4' }) - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: uuid }) - - const files = getAllFiles(video) - expect(files).to.have.lengthOf(1) - expect(files[0].resolution.id).to.equal(360) - } - }) - - it('Should replace a video file with transcoding enabled', async function () { - this.timeout(120000) - - const previousPaths: string[] = [] - - await servers[0].config.enableTranscoding({ hls: true, webVideo: true, with0p: true }) - - const { uuid: videoUUID } = await servers[0].videos.quickUpload({ name: 'fs with transcoding', fixture: 'video_short_720p.mp4' }) - uuid = videoUUID - - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: uuid }) - expect(video.inputFileUpdatedAt).to.be.null - - const files = getAllFiles(video) - expect(files).to.have.lengthOf(6 * 2) - - // Grab old paths to ensure we'll regenerate - - previousPaths.push(video.previewPath) - previousPaths.push(video.thumbnailPath) - - for (const file of files) { - previousPaths.push(file.fileUrl) - previousPaths.push(file.torrentUrl) - previousPaths.push(file.metadataUrl) - - const metadata = await server.videos.getFileMetadata({ url: file.metadataUrl }) - previousPaths.push(JSON.stringify(metadata)) - } - - const { storyboards } = await server.storyboard.list({ id: uuid }) - for (const s of storyboards) { - previousPaths.push(s.storyboardPath) - } - } - - replaceDate = new Date() - - await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_360p.mp4' }) - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: uuid }) - - expect(video.inputFileUpdatedAt).to.not.be.null - expect(new Date(video.inputFileUpdatedAt)).to.be.above(replaceDate) - - const files = getAllFiles(video) - expect(files).to.have.lengthOf(4 * 2) - - expect(previousPaths).to.not.include(video.previewPath) - expect(previousPaths).to.not.include(video.thumbnailPath) - - await makeGetRequest({ url: server.url, path: video.previewPath, expectedStatus: HttpStatusCode.OK_200 }) - await makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) - - for (const file of files) { - expect(previousPaths).to.not.include(file.fileUrl) - expect(previousPaths).to.not.include(file.torrentUrl) - expect(previousPaths).to.not.include(file.metadataUrl) - - await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 }) - - const metadata = await server.videos.getFileMetadata({ url: file.metadataUrl }) - expect(previousPaths).to.not.include(JSON.stringify(metadata)) - } - - const { storyboards } = await server.storyboard.list({ id: uuid }) - for (const s of storyboards) { - expect(previousPaths).to.not.include(s.storyboardPath) - - await makeGetRequest({ url: server.url, path: s.storyboardPath, expectedStatus: HttpStatusCode.OK_200 }) - } - } - - await servers[0].config.enableMinimumTranscoding() - }) - - it('Should have cleaned up old files', async function () { - { - const count = await servers[0].servers.countFiles('storyboards') - expect(count).to.equal(2) - } - - { - const count = await servers[0].servers.countFiles('web-videos') - expect(count).to.equal(5 + 1) // +1 for private directory - } - - { - const count = await servers[0].servers.countFiles('streaming-playlists/hls') - expect(count).to.equal(1 + 1) // +1 for private directory - } - - { - const count = await servers[0].servers.countFiles('torrents') - expect(count).to.equal(9) - } - }) - - it('Should have the correct source input', async function () { - const source = await servers[0].videos.getSource({ id: uuid }) - - expect(source.filename).to.equal('video_short_360p.mp4') - expect(new Date(source.createdAt)).to.be.above(replaceDate) - }) - - it('Should not have regenerated miniatures that were previously uploaded', async function () { - this.timeout(120000) - - const { uuid } = await servers[0].videos.upload({ - attributes: { - name: 'custom miniatures', - thumbnailfile: 'custom-thumbnail.jpg', - previewfile: 'custom-preview.jpg' - } - }) - - await waitJobs(servers) - - const previousPaths: string[] = [] - - for (const server of servers) { - const video = await server.videos.get({ id: uuid }) - - previousPaths.push(video.previewPath) - previousPaths.push(video.thumbnailPath) - - await makeGetRequest({ url: server.url, path: video.previewPath, expectedStatus: HttpStatusCode.OK_200 }) - await makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) - } - - await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_360p.mp4' }) - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: uuid }) - - expect(previousPaths).to.include(video.previewPath) - expect(previousPaths).to.include(video.thumbnailPath) - - await makeGetRequest({ url: server.url, path: video.previewPath, expectedStatus: HttpStatusCode.OK_200 }) - await makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) - } - }) - }) - - describe('Autoblacklist', function () { - - function updateAutoBlacklist (enabled: boolean) { - return servers[0].config.updateExistingSubConfig({ - newConfig: { - autoBlacklist: { - videos: { - ofUsers: { - enabled - } - } - } - } - }) - } - - async function expectBlacklist (uuid: string, value: boolean) { - const video = await servers[0].videos.getWithToken({ id: uuid }) - - expect(video.blacklisted).to.equal(value) - } - - before(async function () { - await updateAutoBlacklist(true) - }) - - it('Should auto blacklist an unblacklisted video after file replacement', async function () { - this.timeout(120000) - - const { uuid } = await servers[0].videos.quickUpload({ token: userToken, name: 'user video' }) - await waitJobs(servers) - await expectBlacklist(uuid, true) - - await servers[0].blacklist.remove({ videoId: uuid }) - await expectBlacklist(uuid, false) - - await servers[0].videos.replaceSourceFile({ videoId: uuid, token: userToken, fixture: 'video_short_360p.mp4' }) - await waitJobs(servers) - - await expectBlacklist(uuid, true) - }) - - it('Should auto blacklist an already blacklisted video after file replacement', async function () { - this.timeout(120000) - - const { uuid } = await servers[0].videos.quickUpload({ token: userToken, name: 'user video' }) - await waitJobs(servers) - await expectBlacklist(uuid, true) - - await servers[0].videos.replaceSourceFile({ videoId: uuid, token: userToken, fixture: 'video_short_360p.mp4' }) - await waitJobs(servers) - - await expectBlacklist(uuid, true) - }) - - it('Should not auto blacklist if auto blacklist has been disabled between the upload and the replacement', async function () { - this.timeout(120000) - - const { uuid } = await servers[0].videos.quickUpload({ token: userToken, name: 'user video' }) - await waitJobs(servers) - await expectBlacklist(uuid, true) - - await servers[0].blacklist.remove({ videoId: uuid }) - await expectBlacklist(uuid, false) - - await updateAutoBlacklist(false) - - await servers[0].videos.replaceSourceFile({ videoId: uuid, token: userToken, fixture: 'video_short1.webm' }) - await waitJobs(servers) - - await expectBlacklist(uuid, false) - }) - }) - - describe('With object storage enabled', function () { - if (areMockObjectStorageTestsDisabled()) return - - const objectStorage = new ObjectStorageCommand() - - before(async function () { - this.timeout(120000) - - const configOverride = objectStorage.getDefaultMockConfig() - await objectStorage.prepareDefaultMockBuckets() - - await servers[0].kill() - await servers[0].run(configOverride) - }) - - it('Should replace a video file with transcoding disabled', async function () { - this.timeout(120000) - - await servers[0].config.disableTranscoding() - - const { uuid } = await servers[0].videos.quickUpload({ - name: 'object storage without transcoding', - fixture: 'video_short_720p.mp4' - }) - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: uuid }) - - const files = getAllFiles(video) - expect(files).to.have.lengthOf(1) - expect(files[0].resolution.id).to.equal(720) - expectStartWith(files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl()) - } - - await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_360p.mp4' }) - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: uuid }) - - const files = getAllFiles(video) - expect(files).to.have.lengthOf(1) - expect(files[0].resolution.id).to.equal(360) - expectStartWith(files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl()) - } - }) - - it('Should replace a video file with transcoding enabled', async function () { - this.timeout(120000) - - const previousPaths: string[] = [] - - await servers[0].config.enableTranscoding({ hls: true, webVideo: true, with0p: true }) - - const { uuid: videoUUID } = await servers[0].videos.quickUpload({ - name: 'object storage with transcoding', - fixture: 'video_short_360p.mp4' - }) - uuid = videoUUID - - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: uuid }) - - const files = getAllFiles(video) - expect(files).to.have.lengthOf(4 * 2) - - for (const file of files) { - previousPaths.push(file.fileUrl) - } - - for (const file of video.files) { - expectStartWith(file.fileUrl, objectStorage.getMockWebVideosBaseUrl()) - } - - for (const file of video.streamingPlaylists[0].files) { - expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl()) - } - } - - await servers[0].videos.replaceSourceFile({ videoId: uuid, fixture: 'video_short_240p.mp4' }) - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: uuid }) - - const files = getAllFiles(video) - expect(files).to.have.lengthOf(3 * 2) - - for (const file of files) { - expect(previousPaths).to.not.include(file.fileUrl) - } - - for (const file of video.files) { - expectStartWith(file.fileUrl, objectStorage.getMockWebVideosBaseUrl()) - } - - for (const file of video.streamingPlaylists[0].files) { - expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl()) - } - } - }) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/videos/video-static-file-privacy.ts b/server/tests/api/videos/video-static-file-privacy.ts deleted file mode 100644 index 0a9864134..000000000 --- a/server/tests/api/videos/video-static-file-privacy.ts +++ /dev/null @@ -1,600 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { decode } from 'magnet-uri' -import { checkVideoFileTokenReinjection, expectStartWith, parseTorrentVideo } from '@server/tests/shared' -import { getAllFiles, wait } from '@shared/core-utils' -import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createSingleServer, - findExternalSavedVideo, - makeRawRequest, - PeerTubeServer, - sendRTMPStream, - setAccessTokensToServers, - setDefaultVideoChannel, - stopFfmpeg, - waitJobs -} from '@shared/server-commands' - -describe('Test video static file privacy', function () { - let server: PeerTubeServer - let userToken: string - - before(async function () { - this.timeout(50000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - userToken = await server.users.generateUserAndToken('user1') - }) - - describe('VOD static file path', function () { - - function runSuite () { - - async function checkPrivateFiles (uuid: string) { - const video = await server.videos.getWithToken({ id: uuid }) - - for (const file of video.files) { - expect(file.fileDownloadUrl).to.not.include('/private/') - expectStartWith(file.fileUrl, server.url + '/static/web-videos/private/') - - const torrent = await parseTorrentVideo(server, file) - expect(torrent.urlList).to.have.lengthOf(0) - - const magnet = decode(file.magnetUri) - expect(magnet.urlList).to.have.lengthOf(0) - - await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - } - - const hls = video.streamingPlaylists[0] - if (hls) { - expectStartWith(hls.playlistUrl, server.url + '/static/streaming-playlists/hls/private/') - expectStartWith(hls.segmentsSha256Url, server.url + '/static/streaming-playlists/hls/private/') - - await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - } - } - - async function checkPublicFiles (uuid: string) { - const video = await server.videos.get({ id: uuid }) - - for (const file of getAllFiles(video)) { - expect(file.fileDownloadUrl).to.not.include('/private/') - expect(file.fileUrl).to.not.include('/private/') - - const torrent = await parseTorrentVideo(server, file) - expect(torrent.urlList[0]).to.not.include('private') - - const magnet = decode(file.magnetUri) - expect(magnet.urlList[0]).to.not.include('private') - - await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ url: torrent.urlList[0], expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ url: magnet.urlList[0], expectedStatus: HttpStatusCode.OK_200 }) - } - - const hls = video.streamingPlaylists[0] - if (hls) { - expect(hls.playlistUrl).to.not.include('private') - expect(hls.segmentsSha256Url).to.not.include('private') - - await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) - } - } - - it('Should upload a private/internal/password protected video and have a private static path', async function () { - this.timeout(120000) - - for (const privacy of [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]) { - const { uuid } = await server.videos.quickUpload({ name: 'video', privacy }) - await waitJobs([ server ]) - - await checkPrivateFiles(uuid) - } - - const { uuid } = await server.videos.quickUpload({ - name: 'video', - privacy: VideoPrivacy.PASSWORD_PROTECTED, - videoPasswords: [ 'my super password' ] - }) - await waitJobs([ server ]) - - await checkPrivateFiles(uuid) - }) - - it('Should upload a public video and update it as private/internal to have a private static path', async function () { - this.timeout(120000) - - for (const privacy of [ VideoPrivacy.PRIVATE, VideoPrivacy.INTERNAL ]) { - const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PUBLIC }) - await waitJobs([ server ]) - - await server.videos.update({ id: uuid, attributes: { privacy } }) - await waitJobs([ server ]) - - await checkPrivateFiles(uuid) - } - }) - - it('Should upload a private video and update it to unlisted to have a public static path', async function () { - this.timeout(120000) - - const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) - await waitJobs([ server ]) - - await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.UNLISTED } }) - await waitJobs([ server ]) - - await checkPublicFiles(uuid) - }) - - it('Should upload an internal video and update it to public to have a public static path', async function () { - this.timeout(120000) - - const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.INTERNAL }) - await waitJobs([ server ]) - - await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } }) - await waitJobs([ server ]) - - await checkPublicFiles(uuid) - }) - - it('Should upload an internal video and schedule a public publish', async function () { - this.timeout(120000) - - const attributes = { - name: 'video', - privacy: VideoPrivacy.PRIVATE, - scheduleUpdate: { - updateAt: new Date(Date.now() + 1000).toISOString(), - privacy: VideoPrivacy.PUBLIC as VideoPrivacy.PUBLIC - } - } - - const { uuid } = await server.videos.upload({ attributes }) - - await waitJobs([ server ]) - await wait(1000) - await server.debug.sendCommand({ body: { command: 'process-update-videos-scheduler' } }) - - await waitJobs([ server ]) - - await checkPublicFiles(uuid) - }) - } - - describe('Without transcoding', function () { - runSuite() - }) - - describe('With transcoding', function () { - - before(async function () { - await server.config.enableMinimumTranscoding() - }) - - runSuite() - }) - }) - - describe('VOD static file right check', function () { - let unrelatedFileToken: string - - async function checkVideoFiles (options: { - id: string - expectedStatus: HttpStatusCode - token: string - videoFileToken: string - videoPassword?: string - }) { - const { id, expectedStatus, token, videoFileToken, videoPassword } = options - - const video = await server.videos.getWithToken({ id }) - - for (const file of getAllFiles(video)) { - await makeRawRequest({ url: file.fileUrl, token, expectedStatus }) - await makeRawRequest({ url: file.fileDownloadUrl, token, expectedStatus }) - - await makeRawRequest({ url: file.fileUrl, query: { videoFileToken }, expectedStatus }) - await makeRawRequest({ url: file.fileDownloadUrl, query: { videoFileToken }, expectedStatus }) - - if (videoPassword) { - const headers = { 'x-peertube-video-password': videoPassword } - await makeRawRequest({ url: file.fileUrl, headers, expectedStatus }) - await makeRawRequest({ url: file.fileDownloadUrl, headers, expectedStatus }) - } - } - - const hls = video.streamingPlaylists[0] - await makeRawRequest({ url: hls.playlistUrl, token, expectedStatus }) - await makeRawRequest({ url: hls.segmentsSha256Url, token, expectedStatus }) - - await makeRawRequest({ url: hls.playlistUrl, query: { videoFileToken }, expectedStatus }) - await makeRawRequest({ url: hls.segmentsSha256Url, query: { videoFileToken }, expectedStatus }) - - if (videoPassword) { - const headers = { 'x-peertube-video-password': videoPassword } - await makeRawRequest({ url: hls.playlistUrl, token: null, headers, expectedStatus }) - await makeRawRequest({ url: hls.segmentsSha256Url, token: null, headers, expectedStatus }) - } - } - - before(async function () { - await server.config.enableMinimumTranscoding() - - const { uuid } = await server.videos.quickUpload({ name: 'another video' }) - unrelatedFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid }) - }) - - it('Should not be able to access a private video files without OAuth token and file token', async function () { - this.timeout(120000) - - const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) - await waitJobs([ server ]) - - await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.FORBIDDEN_403, token: null, videoFileToken: null }) - }) - - it('Should not be able to access password protected video files without OAuth token, file token and password', async function () { - this.timeout(120000) - const videoPassword = 'my super password' - - const { uuid } = await server.videos.quickUpload({ - name: 'password protected video', - privacy: VideoPrivacy.PASSWORD_PROTECTED, - videoPasswords: [ videoPassword ] - }) - await waitJobs([ server ]) - - await checkVideoFiles({ - id: uuid, - expectedStatus: HttpStatusCode.FORBIDDEN_403, - token: null, - videoFileToken: null, - videoPassword: null - }) - }) - - it('Should not be able to access an password video files with incorrect OAuth token, file token and password', async function () { - this.timeout(120000) - const videoPassword = 'my super password' - - const { uuid } = await server.videos.quickUpload({ - name: 'password protected video', - privacy: VideoPrivacy.PASSWORD_PROTECTED, - videoPasswords: [ videoPassword ] - }) - await waitJobs([ server ]) - - await checkVideoFiles({ - id: uuid, - expectedStatus: HttpStatusCode.FORBIDDEN_403, - token: userToken, - videoFileToken: unrelatedFileToken, - videoPassword: 'incorrectPassword' - }) - }) - - it('Should not be able to access an private video files without appropriate OAuth token and file token', async function () { - this.timeout(120000) - - const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) - await waitJobs([ server ]) - - await checkVideoFiles({ - id: uuid, - expectedStatus: HttpStatusCode.FORBIDDEN_403, - token: userToken, - videoFileToken: unrelatedFileToken - }) - }) - - it('Should be able to access a private video files with appropriate OAuth token or file token', async function () { - this.timeout(120000) - - const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) - const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid }) - - await waitJobs([ server ]) - - await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken, videoFileToken }) - }) - - it('Should be able to access a password protected video files with appropriate OAuth token or file token', async function () { - this.timeout(120000) - const videoPassword = 'my super password' - - const { uuid } = await server.videos.quickUpload({ - name: 'video', - privacy: VideoPrivacy.PASSWORD_PROTECTED, - videoPasswords: [ videoPassword ] - }) - - const videoFileToken = await server.videoToken.getVideoFileToken({ token: null, videoId: uuid, videoPassword }) - - await waitJobs([ server ]) - - await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken, videoFileToken, videoPassword }) - }) - - it('Should reinject video file token', async function () { - this.timeout(120000) - - const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) - - const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid }) - await waitJobs([ server ]) - - { - const video = await server.videos.getWithToken({ id: uuid }) - const hls = video.streamingPlaylists[0] - const query = { videoFileToken } - const { text } = await makeRawRequest({ url: hls.playlistUrl, query, expectedStatus: HttpStatusCode.OK_200 }) - - expect(text).to.not.include(videoFileToken) - } - - { - await checkVideoFileTokenReinjection({ - server, - videoUUID: uuid, - videoFileToken, - resolutions: [ 240, 720 ], - isLive: false - }) - } - }) - - it('Should be able to access a private video of another user with an admin OAuth token or file token', async function () { - this.timeout(120000) - - const { uuid } = await server.videos.quickUpload({ name: 'video', token: userToken, privacy: VideoPrivacy.PRIVATE }) - const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid }) - - await waitJobs([ server ]) - - await checkVideoFiles({ id: uuid, expectedStatus: HttpStatusCode.OK_200, token: server.accessToken, videoFileToken }) - }) - }) - - describe('Live static file path and check', function () { - let normalLiveId: string - let normalLive: LiveVideo - - let permanentLiveId: string - let permanentLive: LiveVideo - - let passwordProtectedLiveId: string - let passwordProtectedLive: LiveVideo - - const correctPassword = 'my super password' - - let unrelatedFileToken: string - - async function checkLiveFiles (options: { live: LiveVideo, liveId: string, videoPassword?: string }) { - const { live, liveId, videoPassword } = options - const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) - await server.live.waitUntilPublished({ videoId: liveId }) - - const video = await server.videos.getWithToken({ id: liveId }) - - const fileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid }) - - const hls = video.streamingPlaylists[0] - - for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) { - expectStartWith(url, server.url + '/static/streaming-playlists/hls/private/') - - await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) - - await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - - if (videoPassword) { - await makeRawRequest({ url, headers: { 'x-peertube-video-password': videoPassword }, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ - url, - headers: { 'x-peertube-video-password': 'incorrectPassword' }, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - } - - } - - await stopFfmpeg(ffmpegCommand) - } - - async function checkReplay (replay: VideoDetails) { - const fileToken = await server.videoToken.getVideoFileToken({ videoId: replay.uuid }) - - const hls = replay.streamingPlaylists[0] - expect(hls.files).to.not.have.lengthOf(0) - - for (const file of hls.files) { - await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ url: file.fileUrl, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) - - await makeRawRequest({ url: file.fileUrl, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeRawRequest({ - url: file.fileUrl, - query: { videoFileToken: unrelatedFileToken }, - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - } - - for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) { - expectStartWith(url, server.url + '/static/streaming-playlists/hls/private/') - - await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) - - await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - } - } - - before(async function () { - await server.config.enableMinimumTranscoding() - - const { uuid } = await server.videos.quickUpload({ name: 'another video' }) - unrelatedFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid }) - - await server.config.enableLive({ - allowReplay: true, - transcoding: true, - resolutions: 'min' - }) - - { - const { video, live } = await server.live.quickCreate({ - saveReplay: true, - permanentLive: false, - privacy: VideoPrivacy.PRIVATE - }) - normalLiveId = video.uuid - normalLive = live - } - - { - const { video, live } = await server.live.quickCreate({ - saveReplay: true, - permanentLive: true, - privacy: VideoPrivacy.PRIVATE - }) - permanentLiveId = video.uuid - permanentLive = live - } - - { - const { video, live } = await server.live.quickCreate({ - saveReplay: false, - permanentLive: false, - privacy: VideoPrivacy.PASSWORD_PROTECTED, - videoPasswords: [ correctPassword ] - }) - passwordProtectedLiveId = video.uuid - passwordProtectedLive = live - } - }) - - it('Should create a private normal live and have a private static path', async function () { - this.timeout(240000) - - await checkLiveFiles({ live: normalLive, liveId: normalLiveId }) - }) - - it('Should create a private permanent live and have a private static path', async function () { - this.timeout(240000) - - await checkLiveFiles({ live: permanentLive, liveId: permanentLiveId }) - }) - - it('Should create a password protected live and have a private static path', async function () { - this.timeout(240000) - - await checkLiveFiles({ live: passwordProtectedLive, liveId: passwordProtectedLiveId, videoPassword: correctPassword }) - }) - - it('Should reinject video file token on permanent live', async function () { - this.timeout(240000) - - const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: permanentLive.rtmpUrl, streamKey: permanentLive.streamKey }) - await server.live.waitUntilPublished({ videoId: permanentLiveId }) - - const video = await server.videos.getWithToken({ id: permanentLiveId }) - const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid }) - const hls = video.streamingPlaylists[0] - - { - const query = { videoFileToken } - const { text } = await makeRawRequest({ url: hls.playlistUrl, query, expectedStatus: HttpStatusCode.OK_200 }) - - expect(text).to.not.include(videoFileToken) - } - - { - await checkVideoFileTokenReinjection({ - server, - videoUUID: permanentLiveId, - videoFileToken, - resolutions: [ 720 ], - isLive: true - }) - } - - await stopFfmpeg(ffmpegCommand) - }) - - it('Should have created a replay of the normal live with a private static path', async function () { - this.timeout(240000) - - await server.live.waitUntilReplacedByReplay({ videoId: normalLiveId }) - - const replay = await server.videos.getWithToken({ id: normalLiveId }) - await checkReplay(replay) - }) - - it('Should have created a replay of the permanent live with a private static path', async function () { - this.timeout(240000) - - await server.live.waitUntilWaiting({ videoId: permanentLiveId }) - await waitJobs([ server ]) - - const live = await server.videos.getWithToken({ id: permanentLiveId }) - const replayFromList = await findExternalSavedVideo(server, live) - const replay = await server.videos.getWithToken({ id: replayFromList.id }) - - await checkReplay(replay) - }) - }) - - describe('With static file right check disabled', function () { - let videoUUID: string - - before(async function () { - this.timeout(240000) - - await server.kill() - - await server.run({ - static_files: { - private_files_require_auth: false - } - }) - - const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.INTERNAL }) - videoUUID = uuid - - await waitJobs([ server ]) - }) - - it('Should not check auth for private static files', async function () { - const video = await server.videos.getWithToken({ id: videoUUID }) - - for (const file of getAllFiles(video)) { - await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) - } - - const hls = video.streamingPlaylists[0] - await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/videos/video-storyboard.ts b/server/tests/api/videos/video-storyboard.ts deleted file mode 100644 index 07f371cad..000000000 --- a/server/tests/api/videos/video-storyboard.ts +++ /dev/null @@ -1,213 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { readdir } from 'fs-extra' -import { basename } from 'path' -import { FIXTURE_URLS } from '@server/tests/shared' -import { areHttpImportTestsDisabled } from '@shared/core-utils' -import { HttpStatusCode, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - makeGetRequest, - PeerTubeServer, - sendRTMPStream, - setAccessTokensToServers, - setDefaultVideoChannel, - stopFfmpeg, - waitJobs -} from '@shared/server-commands' - -async function checkStoryboard (options: { - server: PeerTubeServer - uuid: string - tilesCount?: number - minSize?: number -}) { - const { server, uuid, tilesCount, minSize = 1000 } = options - - const { storyboards } = await server.storyboard.list({ id: uuid }) - - expect(storyboards).to.have.lengthOf(1) - - const storyboard = storyboards[0] - - expect(storyboard.spriteDuration).to.equal(1) - expect(storyboard.spriteHeight).to.equal(108) - expect(storyboard.spriteWidth).to.equal(192) - expect(storyboard.storyboardPath).to.exist - - if (tilesCount) { - expect(storyboard.totalWidth).to.equal(192 * Math.min(tilesCount, 10)) - expect(storyboard.totalHeight).to.equal(108 * Math.max((tilesCount / 10), 1)) - } - - const { body } = await makeGetRequest({ url: server.url, path: storyboard.storyboardPath, expectedStatus: HttpStatusCode.OK_200 }) - expect(body.length).to.be.above(minSize) -} - -describe('Test video storyboard', function () { - let servers: PeerTubeServer[] - - let baseUUID: string - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(2) - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - - await doubleFollow(servers[0], servers[1]) - }) - - it('Should generate a storyboard after upload without transcoding', async function () { - this.timeout(120000) - - // 5s video - const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_short.webm' }) - baseUUID = uuid - await waitJobs(servers) - - for (const server of servers) { - await checkStoryboard({ server, uuid, tilesCount: 5 }) - } - }) - - it('Should generate a storyboard after upload without transcoding with a long video', async function () { - this.timeout(120000) - - // 124s video - const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_very_long_10p.mp4' }) - await waitJobs(servers) - - for (const server of servers) { - await checkStoryboard({ server, uuid, tilesCount: 100 }) - } - }) - - it('Should generate a storyboard after upload with transcoding', async function () { - this.timeout(120000) - - await servers[0].config.enableMinimumTranscoding() - - // 5s video - const { uuid } = await servers[0].videos.quickUpload({ name: 'upload', fixture: 'video_short.webm' }) - await waitJobs(servers) - - for (const server of servers) { - await checkStoryboard({ server, uuid, tilesCount: 5 }) - } - }) - - it('Should generate a storyboard after an audio upload', async function () { - this.timeout(120000) - - // 6s audio - const attributes = { name: 'audio', fixture: 'sample.ogg' } - const { uuid } = await servers[0].videos.upload({ attributes, mode: 'legacy' }) - await waitJobs(servers) - - for (const server of servers) { - try { - await checkStoryboard({ server, uuid, tilesCount: 6, minSize: 250 }) - } catch { // FIXME: to remove after ffmpeg CI upgrade, ffmpeg CI version (4.3) generates a 7.6s length video - await checkStoryboard({ server, uuid, tilesCount: 8, minSize: 250 }) - } - } - }) - - it('Should generate a storyboard after HTTP import', async function () { - this.timeout(120000) - - if (areHttpImportTestsDisabled()) return - - // 3s video - const { video } = await servers[0].imports.importVideo({ - attributes: { - targetUrl: FIXTURE_URLS.goodVideo, - channelId: servers[0].store.channel.id, - privacy: VideoPrivacy.PUBLIC - } - }) - await waitJobs(servers) - - for (const server of servers) { - await checkStoryboard({ server, uuid: video.uuid, tilesCount: 3 }) - } - }) - - it('Should generate a storyboard after torrent import', async function () { - this.timeout(120000) - - if (areHttpImportTestsDisabled()) return - - // 10s video - const { video } = await servers[0].imports.importVideo({ - attributes: { - magnetUri: FIXTURE_URLS.magnet, - channelId: servers[0].store.channel.id, - privacy: VideoPrivacy.PUBLIC - } - }) - await waitJobs(servers) - - for (const server of servers) { - await checkStoryboard({ server, uuid: video.uuid, tilesCount: 10 }) - } - }) - - it('Should generate a storyboard after a live', async function () { - this.timeout(240000) - - await servers[0].config.enableLive({ allowReplay: true, transcoding: true, resolutions: 'min' }) - - const { live, video } = await servers[0].live.quickCreate({ - saveReplay: true, - permanentLive: false, - privacy: VideoPrivacy.PUBLIC - }) - - const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) - await servers[0].live.waitUntilPublished({ videoId: video.id }) - - await stopFfmpeg(ffmpegCommand) - - await servers[0].live.waitUntilReplacedByReplay({ videoId: video.id }) - await waitJobs(servers) - - for (const server of servers) { - await checkStoryboard({ server, uuid: video.uuid }) - } - }) - - it('Should cleanup storyboards on video deletion', async function () { - this.timeout(60000) - - const { storyboards } = await servers[0].storyboard.list({ id: baseUUID }) - const storyboardName = basename(storyboards[0].storyboardPath) - - const listFiles = () => { - const storyboardPath = servers[0].getDirectoryPath('storyboards') - return readdir(storyboardPath) - } - - { - const storyboads = await listFiles() - expect(storyboads).to.include(storyboardName) - } - - await servers[0].videos.remove({ id: baseUUID }) - await waitJobs(servers) - - { - const storyboads = await listFiles() - expect(storyboads).to.not.include(storyboardName) - } - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/videos/videos-common-filters.ts b/server/tests/api/videos/videos-common-filters.ts deleted file mode 100644 index 48de7c537..000000000 --- a/server/tests/api/videos/videos-common-filters.ts +++ /dev/null @@ -1,489 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { pick } from '@shared/core-utils' -import { HttpStatusCode, UserRole, Video, VideoDetails, VideoInclude, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - makeGetRequest, - PeerTubeServer, - setAccessTokensToServers, - setDefaultAccountAvatar, - setDefaultVideoChannel, - waitJobs -} from '@shared/server-commands' - -describe('Test videos filter', function () { - let servers: PeerTubeServer[] - let paths: string[] - let remotePaths: string[] - - const subscriptionVideosPath = '/api/v1/users/me/subscriptions/videos' - - // --------------------------------------------------------------- - - before(async function () { - this.timeout(240000) - - servers = await createMultipleServers(2) - - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - await setDefaultAccountAvatar(servers) - - await servers[1].config.enableMinimumTranscoding() - - for (const server of servers) { - const moderator = { username: 'moderator', password: 'my super password' } - await server.users.create({ username: moderator.username, password: moderator.password, role: UserRole.MODERATOR }) - server['moderatorAccessToken'] = await server.login.getAccessToken(moderator) - - await server.videos.upload({ attributes: { name: 'public ' + server.serverNumber } }) - - { - const attributes = { name: 'unlisted ' + server.serverNumber, privacy: VideoPrivacy.UNLISTED } - await server.videos.upload({ attributes }) - } - - { - const attributes = { name: 'private ' + server.serverNumber, privacy: VideoPrivacy.PRIVATE } - await server.videos.upload({ attributes }) - } - - // Subscribing to itself - await server.subscriptions.add({ targetUri: 'root_channel@' + server.host }) - } - - await doubleFollow(servers[0], servers[1]) - - paths = [ - `/api/v1/video-channels/root_channel/videos`, - `/api/v1/accounts/root/videos`, - '/api/v1/videos', - '/api/v1/search/videos', - subscriptionVideosPath - ] - - remotePaths = [ - `/api/v1/video-channels/root_channel@${servers[1].host}/videos`, - `/api/v1/accounts/root@${servers[1].host}/videos`, - '/api/v1/videos', - '/api/v1/search/videos' - ] - }) - - describe('Check videos filters', function () { - - async function listVideos (options: { - server: PeerTubeServer - path: string - isLocal?: boolean - hasWebVideoFiles?: boolean - hasHLSFiles?: boolean - include?: VideoInclude - privacyOneOf?: VideoPrivacy[] - category?: number - tagsAllOf?: string[] - token?: string - expectedStatus?: HttpStatusCode - excludeAlreadyWatched?: boolean - }) { - const res = await makeGetRequest({ - url: options.server.url, - path: options.path, - token: options.token ?? options.server.accessToken, - query: { - ...pick(options, [ - 'isLocal', - 'include', - 'category', - 'tagsAllOf', - 'hasWebVideoFiles', - 'hasHLSFiles', - 'privacyOneOf', - 'excludeAlreadyWatched' - ]), - - sort: 'createdAt' - }, - expectedStatus: options.expectedStatus ?? HttpStatusCode.OK_200 - }) - - return res.body.data as Video[] - } - - async function getVideosNames ( - options: { - server: PeerTubeServer - isLocal?: boolean - include?: VideoInclude - privacyOneOf?: VideoPrivacy[] - token?: string - expectedStatus?: HttpStatusCode - skipSubscription?: boolean - excludeAlreadyWatched?: boolean - } - ) { - const { skipSubscription = false } = options - const videosResults: string[][] = [] - - for (const path of paths) { - if (skipSubscription && path === subscriptionVideosPath) continue - - const videos = await listVideos({ ...options, path }) - - videosResults.push(videos.map(v => v.name)) - } - - return videosResults - } - - it('Should display local videos', async function () { - for (const server of servers) { - const namesResults = await getVideosNames({ server, isLocal: true }) - - for (const names of namesResults) { - expect(names).to.have.lengthOf(1) - expect(names[0]).to.equal('public ' + server.serverNumber) - } - } - }) - - it('Should display local videos with hidden privacy by the admin or the moderator', async function () { - for (const server of servers) { - for (const token of [ server.accessToken, server['moderatorAccessToken'] ]) { - - const namesResults = await getVideosNames( - { - server, - token, - isLocal: true, - privacyOneOf: [ VideoPrivacy.UNLISTED, VideoPrivacy.PUBLIC, VideoPrivacy.PRIVATE ], - skipSubscription: true - } - ) - - for (const names of namesResults) { - expect(names).to.have.lengthOf(3) - - expect(names[0]).to.equal('public ' + server.serverNumber) - expect(names[1]).to.equal('unlisted ' + server.serverNumber) - expect(names[2]).to.equal('private ' + server.serverNumber) - } - } - } - }) - - it('Should display all videos by the admin or the moderator', async function () { - for (const server of servers) { - for (const token of [ server.accessToken, server['moderatorAccessToken'] ]) { - - const [ channelVideos, accountVideos, videos, searchVideos ] = await getVideosNames({ - server, - token, - privacyOneOf: [ VideoPrivacy.UNLISTED, VideoPrivacy.PUBLIC, VideoPrivacy.PRIVATE ] - }) - - expect(channelVideos).to.have.lengthOf(3) - expect(accountVideos).to.have.lengthOf(3) - - expect(videos).to.have.lengthOf(5) - expect(searchVideos).to.have.lengthOf(5) - } - } - }) - - it('Should display only remote videos', async function () { - this.timeout(120000) - - await servers[1].videos.upload({ attributes: { name: 'remote video' } }) - - await waitJobs(servers) - - const finder = (videos: Video[]) => videos.find(v => v.name === 'remote video') - - for (const path of remotePaths) { - { - const videos = await listVideos({ server: servers[0], path }) - const video = finder(videos) - expect(video).to.exist - } - - { - const videos = await listVideos({ server: servers[0], path, isLocal: false }) - const video = finder(videos) - expect(video).to.exist - } - - { - const videos = await listVideos({ server: servers[0], path, isLocal: true }) - const video = finder(videos) - expect(video).to.not.exist - } - } - }) - - it('Should include not published videos', async function () { - await servers[0].config.enableLive({ allowReplay: false, transcoding: false }) - await servers[0].live.create({ fields: { name: 'live video', channelId: servers[0].store.channel.id, privacy: VideoPrivacy.PUBLIC } }) - - const finder = (videos: Video[]) => videos.find(v => v.name === 'live video') - - for (const path of paths) { - { - const videos = await listVideos({ server: servers[0], path }) - const video = finder(videos) - expect(video).to.not.exist - expect(videos[0].state).to.not.exist - expect(videos[0].waitTranscoding).to.not.exist - } - - { - const videos = await listVideos({ server: servers[0], path, include: VideoInclude.NOT_PUBLISHED_STATE }) - const video = finder(videos) - expect(video).to.exist - expect(video.state).to.exist - } - } - }) - - it('Should include blacklisted videos', async function () { - const { id } = await servers[0].videos.upload({ attributes: { name: 'blacklisted' } }) - - await servers[0].blacklist.add({ videoId: id }) - - const finder = (videos: Video[]) => videos.find(v => v.name === 'blacklisted') - - for (const path of paths) { - { - const videos = await listVideos({ server: servers[0], path }) - const video = finder(videos) - expect(video).to.not.exist - expect(videos[0].blacklisted).to.not.exist - } - - { - const videos = await listVideos({ server: servers[0], path, include: VideoInclude.BLACKLISTED }) - const video = finder(videos) - expect(video).to.exist - expect(video.blacklisted).to.be.true - } - } - }) - - it('Should include videos from muted account', async function () { - const finder = (videos: Video[]) => videos.find(v => v.name === 'remote video') - - await servers[0].blocklist.addToServerBlocklist({ account: 'root@' + servers[1].host }) - - for (const path of remotePaths) { - { - const videos = await listVideos({ server: servers[0], path }) - const video = finder(videos) - expect(video).to.not.exist - - // Some paths won't have videos - if (videos[0]) { - expect(videos[0].blockedOwner).to.not.exist - expect(videos[0].blockedServer).to.not.exist - } - } - - { - const videos = await listVideos({ server: servers[0], path, include: VideoInclude.BLOCKED_OWNER }) - - const video = finder(videos) - expect(video).to.exist - expect(video.blockedServer).to.be.false - expect(video.blockedOwner).to.be.true - } - } - - await servers[0].blocklist.removeFromServerBlocklist({ account: 'root@' + servers[1].host }) - }) - - it('Should include videos from muted server', async function () { - const finder = (videos: Video[]) => videos.find(v => v.name === 'remote video') - - await servers[0].blocklist.addToServerBlocklist({ server: servers[1].host }) - - for (const path of remotePaths) { - { - const videos = await listVideos({ server: servers[0], path }) - const video = finder(videos) - expect(video).to.not.exist - - // Some paths won't have videos - if (videos[0]) { - expect(videos[0].blockedOwner).to.not.exist - expect(videos[0].blockedServer).to.not.exist - } - } - - { - const videos = await listVideos({ server: servers[0], path, include: VideoInclude.BLOCKED_OWNER }) - const video = finder(videos) - expect(video).to.exist - expect(video.blockedServer).to.be.true - expect(video.blockedOwner).to.be.false - } - } - - await servers[0].blocklist.removeFromServerBlocklist({ server: servers[1].host }) - }) - - it('Should include video files', async function () { - for (const path of paths) { - { - const videos = await listVideos({ server: servers[0], path }) - - for (const video of videos) { - const videoWithFiles = video as VideoDetails - - expect(videoWithFiles.files).to.not.exist - expect(videoWithFiles.streamingPlaylists).to.not.exist - } - } - - { - const videos = await listVideos({ server: servers[0], path, include: VideoInclude.FILES }) - - for (const video of videos) { - const videoWithFiles = video as VideoDetails - - expect(videoWithFiles.files).to.exist - expect(videoWithFiles.files).to.have.length.at.least(1) - } - } - } - }) - - it('Should filter by tags and category', async function () { - await servers[0].videos.upload({ attributes: { name: 'tag filter', tags: [ 'tag1', 'tag2' ] } }) - await servers[0].videos.upload({ attributes: { name: 'tag filter with category', tags: [ 'tag3' ], category: 4 } }) - - for (const path of paths) { - { - const videos = await listVideos({ server: servers[0], path, tagsAllOf: [ 'tag1', 'tag2' ] }) - expect(videos).to.have.lengthOf(1) - expect(videos[0].name).to.equal('tag filter') - } - - { - const videos = await listVideos({ server: servers[0], path, tagsAllOf: [ 'tag1', 'tag3' ] }) - expect(videos).to.have.lengthOf(0) - } - - { - const { data, total } = await servers[0].videos.list({ tagsAllOf: [ 'tag3' ], categoryOneOf: [ 4 ] }) - expect(total).to.equal(1) - expect(data[0].name).to.equal('tag filter with category') - } - - { - const { total } = await servers[0].videos.list({ tagsAllOf: [ 'tag4' ], categoryOneOf: [ 4 ] }) - expect(total).to.equal(0) - } - } - }) - - it('Should filter by HLS or Web Video files', async function () { - this.timeout(360000) - - const finderFactory = (name: string) => (videos: Video[]) => videos.some(v => v.name === name) - - await servers[0].config.enableTranscoding({ hls: false, webVideo: true }) - await servers[0].videos.upload({ attributes: { name: 'web video' } }) - const hasWebVideo = finderFactory('web video') - - await waitJobs(servers) - - await servers[0].config.enableTranscoding({ hls: true, webVideo: false }) - await servers[0].videos.upload({ attributes: { name: 'hls video' } }) - const hasHLS = finderFactory('hls video') - - await waitJobs(servers) - - await servers[0].config.enableTranscoding({ hls: true, webVideo: true }) - await servers[0].videos.upload({ attributes: { name: 'hls and web video' } }) - const hasBoth = finderFactory('hls and web video') - - await waitJobs(servers) - - for (const path of paths) { - { - const videos = await listVideos({ server: servers[0], path, hasWebVideoFiles: true }) - - expect(hasWebVideo(videos)).to.be.true - expect(hasHLS(videos)).to.be.false - expect(hasBoth(videos)).to.be.true - } - - { - const videos = await listVideos({ server: servers[0], path, hasWebVideoFiles: false }) - - expect(hasWebVideo(videos)).to.be.false - expect(hasHLS(videos)).to.be.true - expect(hasBoth(videos)).to.be.false - } - - { - const videos = await listVideos({ server: servers[0], path, hasHLSFiles: true }) - - expect(hasWebVideo(videos)).to.be.false - expect(hasHLS(videos)).to.be.true - expect(hasBoth(videos)).to.be.true - } - - { - const videos = await listVideos({ server: servers[0], path, hasHLSFiles: false }) - - expect(hasWebVideo(videos)).to.be.true - expect(hasHLS(videos)).to.be.false - expect(hasBoth(videos)).to.be.false - } - - { - const videos = await listVideos({ server: servers[0], path, hasHLSFiles: false, hasWebVideoFiles: false }) - - expect(hasWebVideo(videos)).to.be.false - expect(hasHLS(videos)).to.be.false - expect(hasBoth(videos)).to.be.false - } - - { - const videos = await listVideos({ server: servers[0], path, hasHLSFiles: true, hasWebVideoFiles: true }) - - expect(hasWebVideo(videos)).to.be.false - expect(hasHLS(videos)).to.be.false - expect(hasBoth(videos)).to.be.true - } - } - }) - - it('Should filter already watched videos by the user', async function () { - const { id } = await servers[0].videos.upload({ attributes: { name: 'video for history' } }) - - for (const path of paths) { - const videos = await listVideos({ server: servers[0], path, isLocal: true, excludeAlreadyWatched: true }) - const foundVideo = videos.find(video => video.id === id) - - expect(foundVideo).to.not.be.undefined - } - await servers[0].views.view({ id, currentTime: 1, token: servers[0].accessToken }) - - for (const path of paths) { - const videos = await listVideos({ server: servers[0], path, excludeAlreadyWatched: true }) - const foundVideo = videos.find(video => video.id === id) - - expect(foundVideo).to.be.undefined - } - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/videos/videos-history.ts b/server/tests/api/videos/videos-history.ts deleted file mode 100644 index 6df26ab7d..000000000 --- a/server/tests/api/videos/videos-history.ts +++ /dev/null @@ -1,224 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { wait } from '@shared/core-utils' -import { Video } from '@shared/models' -import { cleanupTests, createSingleServer, killallServers, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' - -describe('Test videos history', function () { - let server: PeerTubeServer = null - let video1Id: number - let video1UUID: string - let video2UUID: string - let video3UUID: string - let video3WatchedDate: Date - let userAccessToken: string - - before(async function () { - this.timeout(120000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - - // 10 seconds long - const fixture = 'video_short1.webm' - - { - const { id, uuid } = await server.videos.upload({ attributes: { name: 'video 1', fixture } }) - video1UUID = uuid - video1Id = id - } - - { - const { uuid } = await server.videos.upload({ attributes: { name: 'video 2', fixture } }) - video2UUID = uuid - } - - { - const { uuid } = await server.videos.upload({ attributes: { name: 'video 3', fixture } }) - video3UUID = uuid - } - - userAccessToken = await server.users.generateUserAndToken('user_1') - }) - - it('Should get videos, without watching history', async function () { - const { data } = await server.videos.listWithToken() - - for (const video of data) { - const videoDetails = await server.videos.getWithToken({ id: video.id }) - - expect(video.userHistory).to.be.undefined - expect(videoDetails.userHistory).to.be.undefined - } - }) - - it('Should watch the first and second video', async function () { - await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 }) - await server.views.view({ id: video1UUID, token: server.accessToken, currentTime: 3 }) - }) - - it('Should return the correct history when listing, searching and getting videos', async function () { - const videosOfVideos: Video[][] = [] - - { - const { data } = await server.videos.listWithToken() - videosOfVideos.push(data) - } - - { - const body = await server.search.searchVideos({ token: server.accessToken, search: 'video' }) - videosOfVideos.push(body.data) - } - - for (const videos of videosOfVideos) { - const video1 = videos.find(v => v.uuid === video1UUID) - const video2 = videos.find(v => v.uuid === video2UUID) - const video3 = videos.find(v => v.uuid === video3UUID) - - expect(video1.userHistory).to.not.be.undefined - expect(video1.userHistory.currentTime).to.equal(3) - - expect(video2.userHistory).to.not.be.undefined - expect(video2.userHistory.currentTime).to.equal(8) - - expect(video3.userHistory).to.be.undefined - } - - { - const videoDetails = await server.videos.getWithToken({ id: video1UUID }) - - expect(videoDetails.userHistory).to.not.be.undefined - expect(videoDetails.userHistory.currentTime).to.equal(3) - } - - { - const videoDetails = await server.videos.getWithToken({ id: video2UUID }) - - expect(videoDetails.userHistory).to.not.be.undefined - expect(videoDetails.userHistory.currentTime).to.equal(8) - } - - { - const videoDetails = await server.videos.getWithToken({ id: video3UUID }) - - expect(videoDetails.userHistory).to.be.undefined - } - }) - - it('Should have these videos when listing my history', async function () { - video3WatchedDate = new Date() - await server.views.view({ id: video3UUID, token: server.accessToken, currentTime: 2 }) - - const body = await server.history.list() - - expect(body.total).to.equal(3) - - const videos = body.data - expect(videos[0].name).to.equal('video 3') - expect(videos[1].name).to.equal('video 1') - expect(videos[2].name).to.equal('video 2') - }) - - it('Should not have videos history on another user', async function () { - const body = await server.history.list({ token: userAccessToken }) - - expect(body.total).to.equal(0) - expect(body.data).to.have.lengthOf(0) - }) - - it('Should be able to search through videos in my history', async function () { - const body = await server.history.list({ search: '2' }) - expect(body.total).to.equal(1) - - const videos = body.data - expect(videos[0].name).to.equal('video 2') - }) - - it('Should clear my history', async function () { - await server.history.removeAll({ beforeDate: video3WatchedDate.toISOString() }) - }) - - it('Should have my history cleared', async function () { - const body = await server.history.list() - expect(body.total).to.equal(1) - - const videos = body.data - expect(videos[0].name).to.equal('video 3') - }) - - it('Should disable videos history', async function () { - await server.users.updateMe({ - videosHistoryEnabled: false - }) - - await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 }) - - const { data } = await server.history.list() - expect(data[0].name).to.not.equal('video 2') - }) - - it('Should re-enable videos history', async function () { - await server.users.updateMe({ - videosHistoryEnabled: true - }) - - await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 }) - - const { data } = await server.history.list() - expect(data[0].name).to.equal('video 2') - }) - - it('Should not clean old history', async function () { - this.timeout(50000) - - await killallServers([ server ]) - - await server.run({ history: { videos: { max_age: '10 days' } } }) - - await wait(6000) - - // Should still have history - - const body = await server.history.list() - expect(body.total).to.equal(2) - }) - - it('Should clean old history', async function () { - this.timeout(50000) - - await killallServers([ server ]) - - await server.run({ history: { videos: { max_age: '5 seconds' } } }) - - await wait(6000) - - const body = await server.history.list() - expect(body.total).to.equal(0) - }) - - it('Should delete a specific history element', async function () { - { - await server.views.view({ id: video1UUID, token: server.accessToken, currentTime: 4 }) - await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 }) - } - - { - const body = await server.history.list() - expect(body.total).to.equal(2) - } - - { - await server.history.removeElement({ videoId: video1Id }) - - const body = await server.history.list() - expect(body.total).to.equal(1) - expect(body.data[0].uuid).to.equal(video2UUID) - } - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/videos/videos-overview.ts b/server/tests/api/videos/videos-overview.ts deleted file mode 100644 index f2496e35e..000000000 --- a/server/tests/api/videos/videos-overview.ts +++ /dev/null @@ -1,129 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { wait } from '@shared/core-utils' -import { VideosOverview } from '@shared/models' -import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' - -describe('Test a videos overview', function () { - let server: PeerTubeServer = null - - function testOverviewCount (overview: VideosOverview, expected: number) { - expect(overview.tags).to.have.lengthOf(expected) - expect(overview.categories).to.have.lengthOf(expected) - expect(overview.channels).to.have.lengthOf(expected) - } - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - }) - - it('Should send empty overview', async function () { - const body = await server.overviews.getVideos({ page: 1 }) - - testOverviewCount(body, 0) - }) - - it('Should upload 5 videos in a specific category, tag and channel but not include them in overview', async function () { - this.timeout(60000) - - await wait(3000) - - await server.videos.upload({ - attributes: { - name: 'video 0', - category: 3, - tags: [ 'coucou1', 'coucou2' ] - } - }) - - const body = await server.overviews.getVideos({ page: 1 }) - - testOverviewCount(body, 0) - }) - - it('Should upload another video and include all videos in the overview', async function () { - this.timeout(120000) - - { - for (let i = 1; i < 6; i++) { - await server.videos.upload({ - attributes: { - name: 'video ' + i, - category: 3, - tags: [ 'coucou1', 'coucou2' ] - } - }) - } - - await wait(3000) - } - - { - const body = await server.overviews.getVideos({ page: 1 }) - - testOverviewCount(body, 1) - } - - { - const overview = await server.overviews.getVideos({ page: 2 }) - - expect(overview.tags).to.have.lengthOf(1) - expect(overview.categories).to.have.lengthOf(0) - expect(overview.channels).to.have.lengthOf(0) - } - }) - - it('Should have the correct overview', async function () { - const overview1 = await server.overviews.getVideos({ page: 1 }) - const overview2 = await server.overviews.getVideos({ page: 2 }) - - for (const arr of [ overview1.tags, overview1.categories, overview1.channels, overview2.tags ]) { - expect(arr).to.have.lengthOf(1) - - const obj = arr[0] - - expect(obj.videos).to.have.lengthOf(6) - expect(obj.videos[0].name).to.equal('video 5') - expect(obj.videos[1].name).to.equal('video 4') - expect(obj.videos[2].name).to.equal('video 3') - expect(obj.videos[3].name).to.equal('video 2') - expect(obj.videos[4].name).to.equal('video 1') - expect(obj.videos[5].name).to.equal('video 0') - } - - const tags = [ overview1.tags[0].tag, overview2.tags[0].tag ] - expect(tags.find(t => t === 'coucou1')).to.not.be.undefined - expect(tags.find(t => t === 'coucou2')).to.not.be.undefined - - expect(overview1.categories[0].category.id).to.equal(3) - - expect(overview1.channels[0].channel.name).to.equal('root_channel') - }) - - it('Should hide muted accounts', async function () { - const token = await server.users.generateUserAndToken('choco') - - await server.blocklist.addToMyBlocklist({ token, account: 'root@' + server.host }) - - { - const body = await server.overviews.getVideos({ page: 1 }) - - testOverviewCount(body, 1) - } - - { - const body = await server.overviews.getVideos({ page: 1, token }) - - testOverviewCount(body, 0) - } - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/api/views/index.ts b/server/tests/api/views/index.ts deleted file mode 100644 index 5e06b31fb..000000000 --- a/server/tests/api/views/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -export * from './video-views-counter' -export * from './video-views-overall-stats' -export * from './video-views-retention-stats' -export * from './video-views-timeserie-stats' -export * from './videos-views-cleaner' diff --git a/server/tests/api/views/video-views-counter.ts b/server/tests/api/views/video-views-counter.ts deleted file mode 100644 index 0c1b7859c..000000000 --- a/server/tests/api/views/video-views-counter.ts +++ /dev/null @@ -1,153 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { FfmpegCommand } from 'fluent-ffmpeg' -import { prepareViewsServers, prepareViewsVideos, processViewsBuffer } from '@server/tests/shared' -import { wait } from '@shared/core-utils' -import { cleanupTests, PeerTubeServer, stopFfmpeg, waitJobs } from '@shared/server-commands' - -describe('Test video views/viewers counters', function () { - let servers: PeerTubeServer[] - - async function checkCounter (field: 'views' | 'viewers', id: string, expected: number) { - for (const server of servers) { - const video = await server.videos.get({ id }) - - const messageSuffix = video.isLive - ? 'live video' - : 'vod video' - - expect(video[field]).to.equal(expected, `${field} not valid on server ${server.serverNumber} for ${messageSuffix} ${video.uuid}`) - } - } - - before(async function () { - this.timeout(120000) - - servers = await prepareViewsServers() - }) - - describe('Test views counter on VOD', function () { - let videoUUID: string - - before(async function () { - this.timeout(120000) - - const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) - videoUUID = uuid - - await waitJobs(servers) - }) - - it('Should not view a video if watch time is below the threshold', async function () { - await servers[0].views.simulateViewer({ id: videoUUID, currentTimes: [ 1, 2 ] }) - await processViewsBuffer(servers) - - await checkCounter('views', videoUUID, 0) - }) - - it('Should view a video if watch time is above the threshold', async function () { - await servers[0].views.simulateViewer({ id: videoUUID, currentTimes: [ 1, 4 ] }) - await processViewsBuffer(servers) - - await checkCounter('views', videoUUID, 1) - }) - - it('Should not view again this video with the same IP', async function () { - await servers[0].views.simulateViewer({ id: videoUUID, xForwardedFor: '0.0.0.1,127.0.0.1', currentTimes: [ 1, 4 ] }) - await servers[0].views.simulateViewer({ id: videoUUID, xForwardedFor: '0.0.0.1,127.0.0.1', currentTimes: [ 1, 4 ] }) - await processViewsBuffer(servers) - - await checkCounter('views', videoUUID, 2) - }) - - it('Should view the video from server 2 and send the event', async function () { - await servers[1].views.simulateViewer({ id: videoUUID, currentTimes: [ 1, 4 ] }) - await waitJobs(servers) - await processViewsBuffer(servers) - - await checkCounter('views', videoUUID, 3) - }) - }) - - describe('Test views and viewers counters on live and VOD', function () { - let liveVideoId: string - let vodVideoId: string - let command: FfmpegCommand - - before(async function () { - this.timeout(240000); - - ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) - }) - - it('Should display no views and viewers', async function () { - await checkCounter('views', liveVideoId, 0) - await checkCounter('viewers', liveVideoId, 0) - - await checkCounter('views', vodVideoId, 0) - await checkCounter('viewers', vodVideoId, 0) - }) - - it('Should view twice and display 1 view/viewer', async function () { - this.timeout(30000) - - await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) - await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) - await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) - await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) - - await waitJobs(servers) - await checkCounter('viewers', liveVideoId, 1) - await checkCounter('viewers', vodVideoId, 1) - - await processViewsBuffer(servers) - - await checkCounter('views', liveVideoId, 1) - await checkCounter('views', vodVideoId, 1) - }) - - it('Should wait and display 0 viewers but still have 1 view', async function () { - this.timeout(30000) - - await wait(12000) - await waitJobs(servers) - - await checkCounter('views', liveVideoId, 1) - await checkCounter('viewers', liveVideoId, 0) - - await checkCounter('views', vodVideoId, 1) - await checkCounter('viewers', vodVideoId, 0) - }) - - it('Should view on a remote and on local and display 2 viewers and 3 views', async function () { - this.timeout(30000) - - await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) - await servers[1].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) - await servers[1].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) - - await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) - await servers[1].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) - await servers[1].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35 ] }) - - await waitJobs(servers) - - await checkCounter('viewers', liveVideoId, 2) - await checkCounter('viewers', vodVideoId, 2) - - await processViewsBuffer(servers) - - await checkCounter('views', liveVideoId, 3) - await checkCounter('views', vodVideoId, 3) - }) - - after(async function () { - await stopFfmpeg(command) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/views/video-views-overall-stats.ts b/server/tests/api/views/video-views-overall-stats.ts deleted file mode 100644 index ac636961e..000000000 --- a/server/tests/api/views/video-views-overall-stats.ts +++ /dev/null @@ -1,368 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { FfmpegCommand } from 'fluent-ffmpeg' -import { prepareViewsServers, prepareViewsVideos, processViewersStats } from '@server/tests/shared' -import { cleanupTests, PeerTubeServer, stopFfmpeg, waitJobs } from '@shared/server-commands' -import { wait } from '@shared/core-utils' -import { VideoStatsOverall } from '@shared/models' - -/** - * - * Simulate 5 sections of viewers - * * user0 started and ended before start date - * * user1 started before start date and ended in the interval - * * user2 started started in the interval and ended after end date - * * user3 started and ended in the interval - * * user4 started and ended after end date - */ -async function simulateComplexViewers (servers: PeerTubeServer[], videoUUID: string) { - const user0 = '8.8.8.8,127.0.0.1' - const user1 = '8.8.8.8,127.0.0.1' - const user2 = '8.8.8.9,127.0.0.1' - const user3 = '8.8.8.10,127.0.0.1' - const user4 = '8.8.8.11,127.0.0.1' - - await servers[0].views.view({ id: videoUUID, currentTime: 0, xForwardedFor: user0 }) // User 0 starts - await wait(500) - - await servers[0].views.view({ id: videoUUID, currentTime: 0, xForwardedFor: user1 }) // User 1 starts - await servers[0].views.view({ id: videoUUID, currentTime: 2, xForwardedFor: user0 }) // User 0 ends - await wait(500) - - const startDate = new Date().toISOString() - await servers[0].views.view({ id: videoUUID, currentTime: 0, xForwardedFor: user2 }) // User 2 starts - await wait(500) - - await servers[0].views.view({ id: videoUUID, currentTime: 0, xForwardedFor: user3 }) // User 3 starts - await wait(500) - - await servers[0].views.view({ id: videoUUID, currentTime: 4, xForwardedFor: user1 }) // User 1 ends - await wait(500) - - await servers[0].views.view({ id: videoUUID, currentTime: 3, xForwardedFor: user3 }) // User 3 ends - await wait(500) - - const endDate = new Date().toISOString() - await servers[0].views.view({ id: videoUUID, currentTime: 0, xForwardedFor: user4 }) // User 4 starts - await servers[0].views.view({ id: videoUUID, currentTime: 5, xForwardedFor: user2 }) // User 2 ends - await wait(500) - - await servers[0].views.view({ id: videoUUID, currentTime: 1, xForwardedFor: user4 }) // User 4 ends - - await processViewersStats(servers) - - return { startDate, endDate } -} - -describe('Test views overall stats', function () { - let servers: PeerTubeServer[] - - before(async function () { - this.timeout(120000) - - servers = await prepareViewsServers() - }) - - describe('Test watch time stats of local videos on live and VOD', function () { - let vodVideoId: string - let liveVideoId: string - let command: FfmpegCommand - - before(async function () { - this.timeout(240000); - - ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) - }) - - it('Should display overall stats of a video with no viewers', async function () { - for (const videoId of [ liveVideoId, vodVideoId ]) { - const stats = await servers[0].videoStats.getOverallStats({ videoId }) - const video = await servers[0].videos.get({ id: videoId }) - - expect(video.views).to.equal(0) - expect(stats.averageWatchTime).to.equal(0) - expect(stats.totalWatchTime).to.equal(0) - expect(stats.totalViewers).to.equal(0) - } - }) - - it('Should display overall stats with 1 viewer below the watch time limit', async function () { - this.timeout(60000) - - for (const videoId of [ liveVideoId, vodVideoId ]) { - await servers[0].views.simulateViewer({ id: videoId, currentTimes: [ 0, 1 ] }) - } - - await processViewersStats(servers) - - for (const videoId of [ liveVideoId, vodVideoId ]) { - const stats = await servers[0].videoStats.getOverallStats({ videoId }) - const video = await servers[0].videos.get({ id: videoId }) - - expect(video.views).to.equal(0) - expect(stats.averageWatchTime).to.equal(1) - expect(stats.totalWatchTime).to.equal(1) - expect(stats.totalViewers).to.equal(1) - } - }) - - it('Should display overall stats with 2 viewers', async function () { - this.timeout(60000) - - { - await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 3 ] }) - await servers[0].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 35, 40 ] }) - - await processViewersStats(servers) - - { - const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId }) - const video = await servers[0].videos.get({ id: vodVideoId }) - - expect(video.views).to.equal(1) - expect(stats.averageWatchTime).to.equal(2) - expect(stats.totalWatchTime).to.equal(4) - expect(stats.totalViewers).to.equal(2) - } - - { - const stats = await servers[0].videoStats.getOverallStats({ videoId: liveVideoId }) - const video = await servers[0].videos.get({ id: liveVideoId }) - - expect(video.views).to.equal(1) - expect(stats.averageWatchTime).to.equal(21) - expect(stats.totalWatchTime).to.equal(41) - expect(stats.totalViewers).to.equal(2) - } - } - }) - - it('Should display overall stats with a remote viewer below the watch time limit', async function () { - this.timeout(60000) - - for (const videoId of [ liveVideoId, vodVideoId ]) { - await servers[1].views.simulateViewer({ id: videoId, currentTimes: [ 0, 2 ] }) - } - - await processViewersStats(servers) - - { - const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId }) - const video = await servers[0].videos.get({ id: vodVideoId }) - - expect(video.views).to.equal(1) - expect(stats.averageWatchTime).to.equal(2) - expect(stats.totalWatchTime).to.equal(6) - expect(stats.totalViewers).to.equal(3) - } - - { - const stats = await servers[0].videoStats.getOverallStats({ videoId: liveVideoId }) - const video = await servers[0].videos.get({ id: liveVideoId }) - - expect(video.views).to.equal(1) - expect(stats.averageWatchTime).to.equal(14) - expect(stats.totalWatchTime).to.equal(43) - expect(stats.totalViewers).to.equal(3) - } - }) - - it('Should display overall stats with a remote viewer above the watch time limit', async function () { - this.timeout(60000) - - await servers[1].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 5 ] }) - await servers[1].views.simulateViewer({ id: liveVideoId, currentTimes: [ 0, 45 ] }) - await processViewersStats(servers) - - { - const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId }) - const video = await servers[0].videos.get({ id: vodVideoId }) - - expect(video.views).to.equal(2) - expect(stats.averageWatchTime).to.equal(3) - expect(stats.totalWatchTime).to.equal(11) - expect(stats.totalViewers).to.equal(4) - } - - { - const stats = await servers[0].videoStats.getOverallStats({ videoId: liveVideoId }) - const video = await servers[0].videos.get({ id: liveVideoId }) - - expect(video.views).to.equal(2) - expect(stats.averageWatchTime).to.equal(22) - expect(stats.totalWatchTime).to.equal(88) - expect(stats.totalViewers).to.equal(4) - } - }) - - it('Should filter overall stats by date', async function () { - this.timeout(60000) - - const beforeView = new Date() - - await servers[0].views.simulateViewer({ id: vodVideoId, currentTimes: [ 0, 3 ] }) - await processViewersStats(servers) - - { - const stats = await servers[0].videoStats.getOverallStats({ videoId: vodVideoId, startDate: beforeView.toISOString() }) - expect(stats.averageWatchTime).to.equal(3) - expect(stats.totalWatchTime).to.equal(3) - expect(stats.totalViewers).to.equal(1) - } - - { - const stats = await servers[0].videoStats.getOverallStats({ videoId: liveVideoId, endDate: beforeView.toISOString() }) - expect(stats.averageWatchTime).to.equal(22) - expect(stats.totalWatchTime).to.equal(88) - expect(stats.totalViewers).to.equal(4) - } - }) - - after(async function () { - await stopFfmpeg(command) - }) - }) - - describe('Test watchers peak stats of local videos on VOD', function () { - let videoUUID: string - let before2Watchers: Date - - before(async function () { - this.timeout(240000); - - ({ vodVideoId: videoUUID } = await prepareViewsVideos({ servers, live: true, vod: true })) - }) - - it('Should not have watchers peak', async function () { - const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID }) - - expect(stats.viewersPeak).to.equal(0) - expect(stats.viewersPeakDate).to.be.null - }) - - it('Should have watcher peak with 1 watcher', async function () { - this.timeout(60000) - - const before = new Date() - await servers[0].views.simulateViewer({ id: videoUUID, currentTimes: [ 0, 2 ] }) - const after = new Date() - - await processViewersStats(servers) - - const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID }) - - expect(stats.viewersPeak).to.equal(1) - expect(new Date(stats.viewersPeakDate)).to.be.above(before).and.below(after) - }) - - it('Should have watcher peak with 2 watchers', async function () { - this.timeout(60000) - - before2Watchers = new Date() - await servers[0].views.view({ id: videoUUID, currentTime: 0 }) - await servers[1].views.view({ id: videoUUID, currentTime: 0 }) - await servers[0].views.view({ id: videoUUID, currentTime: 2 }) - await servers[1].views.view({ id: videoUUID, currentTime: 2 }) - const after = new Date() - - await processViewersStats(servers) - - const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID }) - - expect(stats.viewersPeak).to.equal(2) - expect(new Date(stats.viewersPeakDate)).to.be.above(before2Watchers).and.below(after) - }) - - it('Should filter peak viewers stats by date', async function () { - { - const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID, startDate: new Date().toISOString() }) - expect(stats.viewersPeak).to.equal(0) - expect(stats.viewersPeakDate).to.not.exist - } - - { - const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID, endDate: before2Watchers.toISOString() }) - expect(stats.viewersPeak).to.equal(1) - expect(new Date(stats.viewersPeakDate)).to.be.below(before2Watchers) - } - }) - - it('Should complex filter peak viewers by date', async function () { - this.timeout(60000) - - const { startDate, endDate } = await simulateComplexViewers(servers, videoUUID) - - const expectCorrect = (stats: VideoStatsOverall) => { - expect(stats.viewersPeak).to.equal(3) - expect(new Date(stats.viewersPeakDate)).to.be.above(new Date(startDate)).and.below(new Date(endDate)) - } - - expectCorrect(await servers[0].videoStats.getOverallStats({ videoId: videoUUID, startDate, endDate })) - expectCorrect(await servers[0].videoStats.getOverallStats({ videoId: videoUUID, startDate })) - expectCorrect(await servers[0].videoStats.getOverallStats({ videoId: videoUUID, endDate })) - expectCorrect(await servers[0].videoStats.getOverallStats({ videoId: videoUUID })) - }) - }) - - describe('Test countries', function () { - let videoUUID: string - - it('Should not report countries if geoip is disabled', async function () { - this.timeout(120000) - - const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) - await waitJobs(servers) - - await servers[1].views.view({ id: uuid, xForwardedFor: '8.8.8.8,127.0.0.1', currentTime: 1 }) - - await processViewersStats(servers) - - const stats = await servers[0].videoStats.getOverallStats({ videoId: uuid }) - expect(stats.countries).to.have.lengthOf(0) - }) - - it('Should report countries if geoip is enabled', async function () { - this.timeout(240000) - - const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) - videoUUID = uuid - await waitJobs(servers) - - await Promise.all([ - servers[0].kill(), - servers[1].kill() - ]) - - const config = { geo_ip: { enabled: true } } - await Promise.all([ - servers[0].run(config), - servers[1].run(config) - ]) - - await servers[0].views.view({ id: uuid, xForwardedFor: '8.8.8.8,127.0.0.1', currentTime: 1 }) - await servers[1].views.view({ id: uuid, xForwardedFor: '8.8.8.4,127.0.0.1', currentTime: 3 }) - await servers[1].views.view({ id: uuid, xForwardedFor: '80.67.169.12,127.0.0.1', currentTime: 2 }) - - await processViewersStats(servers) - - const stats = await servers[0].videoStats.getOverallStats({ videoId: uuid }) - expect(stats.countries).to.have.lengthOf(2) - - expect(stats.countries[0].isoCode).to.equal('US') - expect(stats.countries[0].viewers).to.equal(2) - - expect(stats.countries[1].isoCode).to.equal('FR') - expect(stats.countries[1].viewers).to.equal(1) - }) - - it('Should filter countries stats by date', async function () { - const stats = await servers[0].videoStats.getOverallStats({ videoId: videoUUID, startDate: new Date().toISOString() }) - expect(stats.countries).to.have.lengthOf(0) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/views/video-views-retention-stats.ts b/server/tests/api/views/video-views-retention-stats.ts deleted file mode 100644 index 5b9ce4c92..000000000 --- a/server/tests/api/views/video-views-retention-stats.ts +++ /dev/null @@ -1,53 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { prepareViewsServers, prepareViewsVideos, processViewersStats } from '@server/tests/shared' -import { cleanupTests, PeerTubeServer } from '@shared/server-commands' - -describe('Test views retention stats', function () { - let servers: PeerTubeServer[] - - before(async function () { - this.timeout(120000) - - servers = await prepareViewsServers() - }) - - describe('Test retention stats on VOD', function () { - let vodVideoId: string - - before(async function () { - this.timeout(240000); - - ({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true })) - }) - - it('Should display empty retention', async function () { - const { data } = await servers[0].videoStats.getRetentionStats({ videoId: vodVideoId }) - expect(data).to.have.lengthOf(6) - - for (let i = 0; i < 6; i++) { - expect(data[i].second).to.equal(i) - expect(data[i].retentionPercent).to.equal(0) - } - }) - - it('Should display appropriate retention metrics', async function () { - await servers[0].views.simulateViewer({ xForwardedFor: '127.0.0.2,127.0.0.1', id: vodVideoId, currentTimes: [ 0, 1 ] }) - await servers[0].views.simulateViewer({ xForwardedFor: '127.0.0.3,127.0.0.1', id: vodVideoId, currentTimes: [ 1, 3 ] }) - await servers[1].views.simulateViewer({ xForwardedFor: '127.0.0.2,127.0.0.1', id: vodVideoId, currentTimes: [ 4 ] }) - await servers[1].views.simulateViewer({ xForwardedFor: '127.0.0.3,127.0.0.1', id: vodVideoId, currentTimes: [ 0, 1 ] }) - - await processViewersStats(servers) - - const { data } = await servers[0].videoStats.getRetentionStats({ videoId: vodVideoId }) - expect(data).to.have.lengthOf(6) - - expect(data.map(d => d.retentionPercent)).to.deep.equal([ 50, 75, 25, 25, 25, 0 ]) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/views/video-views-timeserie-stats.ts b/server/tests/api/views/video-views-timeserie-stats.ts deleted file mode 100644 index 2d991d7ea..000000000 --- a/server/tests/api/views/video-views-timeserie-stats.ts +++ /dev/null @@ -1,253 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { FfmpegCommand } from 'fluent-ffmpeg' -import { prepareViewsServers, prepareViewsVideos, processViewersStats } from '@server/tests/shared' -import { VideoStatsTimeserie, VideoStatsTimeserieMetric } from '@shared/models' -import { cleanupTests, PeerTubeServer, stopFfmpeg } from '@shared/server-commands' - -function buildOneMonthAgo () { - const monthAgo = new Date() - monthAgo.setHours(0, 0, 0, 0) - - monthAgo.setDate(monthAgo.getDate() - 29) - - return monthAgo -} - -describe('Test views timeserie stats', function () { - const availableMetrics: VideoStatsTimeserieMetric[] = [ 'viewers' ] - - let servers: PeerTubeServer[] - - before(async function () { - this.timeout(120000) - - servers = await prepareViewsServers() - }) - - describe('Common metric tests', function () { - let vodVideoId: string - - before(async function () { - this.timeout(240000); - - ({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true })) - }) - - it('Should display empty metric stats', async function () { - for (const metric of availableMetrics) { - const { data } = await servers[0].videoStats.getTimeserieStats({ videoId: vodVideoId, metric }) - - expect(data).to.have.length.at.least(1) - - for (const d of data) { - expect(d.value).to.equal(0) - } - } - }) - }) - - describe('Test viewer and watch time metrics on live and VOD', function () { - let vodVideoId: string - let liveVideoId: string - let command: FfmpegCommand - - function expectTodayLastValue (result: VideoStatsTimeserie, lastValue?: number) { - const { data } = result - - const last = data[data.length - 1] - const today = new Date().getDate() - expect(new Date(last.date).getDate()).to.equal(today) - - if (lastValue) expect(last.value).to.equal(lastValue) - } - - function expectTimeserieData (result: VideoStatsTimeserie, lastValue: number) { - const { data } = result - expect(data).to.have.length.at.least(25) - - expectTodayLastValue(result, lastValue) - - for (let i = 0; i < data.length - 2; i++) { - expect(data[i].value).to.equal(0) - } - } - - function expectInterval (result: VideoStatsTimeserie, intervalMs: number) { - const first = result.data[0] - const second = result.data[1] - expect(new Date(second.date).getTime() - new Date(first.date).getTime()).to.equal(intervalMs) - } - - before(async function () { - this.timeout(240000); - - ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) - }) - - it('Should display appropriate viewers metrics', async function () { - for (const videoId of [ vodVideoId, liveVideoId ]) { - await servers[0].views.simulateViewer({ id: videoId, currentTimes: [ 0, 3 ] }) - await servers[1].views.simulateViewer({ id: videoId, currentTimes: [ 0, 5 ] }) - } - - await processViewersStats(servers) - - for (const videoId of [ vodVideoId, liveVideoId ]) { - const result = await servers[0].videoStats.getTimeserieStats({ - videoId, - startDate: buildOneMonthAgo(), - endDate: new Date(), - metric: 'viewers' - }) - expectTimeserieData(result, 2) - } - }) - - it('Should display appropriate watch time metrics', async function () { - for (const videoId of [ vodVideoId, liveVideoId ]) { - const result = await servers[0].videoStats.getTimeserieStats({ - videoId, - startDate: buildOneMonthAgo(), - endDate: new Date(), - metric: 'aggregateWatchTime' - }) - expectTimeserieData(result, 8) - - await servers[1].views.simulateViewer({ id: videoId, currentTimes: [ 0, 1 ] }) - } - - await processViewersStats(servers) - - for (const videoId of [ vodVideoId, liveVideoId ]) { - const result = await servers[0].videoStats.getTimeserieStats({ - videoId, - startDate: buildOneMonthAgo(), - endDate: new Date(), - metric: 'aggregateWatchTime' - }) - expectTimeserieData(result, 9) - } - }) - - it('Should use a custom start/end date', async function () { - const now = new Date() - const twentyDaysAgo = new Date() - twentyDaysAgo.setDate(twentyDaysAgo.getDate() - 19) - - const result = await servers[0].videoStats.getTimeserieStats({ - videoId: vodVideoId, - metric: 'aggregateWatchTime', - startDate: twentyDaysAgo, - endDate: now - }) - - expect(result.groupInterval).to.equal('1 day') - expect(result.data).to.have.lengthOf(20) - - const first = result.data[0] - expect(new Date(first.date).toLocaleDateString()).to.equal(twentyDaysAgo.toLocaleDateString()) - - expectInterval(result, 24 * 3600 * 1000) - expectTodayLastValue(result, 9) - }) - - it('Should automatically group by months', async function () { - const now = new Date() - const heightYearsAgo = new Date() - heightYearsAgo.setFullYear(heightYearsAgo.getFullYear() - 7) - - const result = await servers[0].videoStats.getTimeserieStats({ - videoId: vodVideoId, - metric: 'aggregateWatchTime', - startDate: heightYearsAgo, - endDate: now - }) - - expect(result.groupInterval).to.equal('6 months') - expect(result.data).to.have.length.above(10).and.below(200) - }) - - it('Should automatically group by days', async function () { - const now = new Date() - const threeMonthsAgo = new Date() - threeMonthsAgo.setMonth(threeMonthsAgo.getMonth() - 3) - - const result = await servers[0].videoStats.getTimeserieStats({ - videoId: vodVideoId, - metric: 'aggregateWatchTime', - startDate: threeMonthsAgo, - endDate: now - }) - - expect(result.groupInterval).to.equal('2 days') - expect(result.data).to.have.length.above(10).and.below(200) - }) - - it('Should automatically group by hours', async function () { - const now = new Date() - const twoDaysAgo = new Date() - twoDaysAgo.setDate(twoDaysAgo.getDate() - 1) - - const result = await servers[0].videoStats.getTimeserieStats({ - videoId: vodVideoId, - metric: 'aggregateWatchTime', - startDate: twoDaysAgo, - endDate: now - }) - - expect(result.groupInterval).to.equal('1 hour') - expect(result.data).to.have.length.above(24).and.below(50) - - expectInterval(result, 3600 * 1000) - expectTodayLastValue(result, 9) - }) - - it('Should automatically group by ten minutes', async function () { - const now = new Date() - const twoHoursAgo = new Date() - twoHoursAgo.setHours(twoHoursAgo.getHours() - 4) - - const result = await servers[0].videoStats.getTimeserieStats({ - videoId: vodVideoId, - metric: 'aggregateWatchTime', - startDate: twoHoursAgo, - endDate: now - }) - - expect(result.groupInterval).to.equal('10 minutes') - expect(result.data).to.have.length.above(20).and.below(30) - - expectInterval(result, 60 * 10 * 1000) - expectTodayLastValue(result) - }) - - it('Should automatically group by one minute', async function () { - const now = new Date() - const thirtyAgo = new Date() - thirtyAgo.setMinutes(thirtyAgo.getMinutes() - 30) - - const result = await servers[0].videoStats.getTimeserieStats({ - videoId: vodVideoId, - metric: 'aggregateWatchTime', - startDate: thirtyAgo, - endDate: now - }) - - expect(result.groupInterval).to.equal('1 minute') - expect(result.data).to.have.length.above(20).and.below(40) - - expectInterval(result, 60 * 1000) - expectTodayLastValue(result) - }) - - after(async function () { - await stopFfmpeg(command) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/api/views/videos-views-cleaner.ts b/server/tests/api/views/videos-views-cleaner.ts deleted file mode 100644 index a84cd43c7..000000000 --- a/server/tests/api/views/videos-views-cleaner.ts +++ /dev/null @@ -1,98 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { SQLCommand } from '@server/tests/shared' -import { wait } from '@shared/core-utils' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - killallServers, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Test video views cleaner', function () { - let servers: PeerTubeServer[] - let sqlCommands: SQLCommand[] = [] - - let videoIdServer1: string - let videoIdServer2: string - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(2) - await setAccessTokensToServers(servers) - - await doubleFollow(servers[0], servers[1]) - - videoIdServer1 = (await servers[0].videos.quickUpload({ name: 'video server 1' })).uuid - videoIdServer2 = (await servers[1].videos.quickUpload({ name: 'video server 2' })).uuid - - await waitJobs(servers) - - await servers[0].views.simulateView({ id: videoIdServer1 }) - await servers[1].views.simulateView({ id: videoIdServer1 }) - await servers[0].views.simulateView({ id: videoIdServer2 }) - await servers[1].views.simulateView({ id: videoIdServer2 }) - - await waitJobs(servers) - - sqlCommands = servers.map(s => new SQLCommand(s)) - }) - - it('Should not clean old video views', async function () { - this.timeout(50000) - - await killallServers([ servers[0] ]) - - await servers[0].run({ views: { videos: { remote: { max_age: '10 days' } } } }) - - await wait(6000) - - // Should still have views - - for (let i = 0; i < servers.length; i++) { - const total = await sqlCommands[i].countVideoViewsOf(videoIdServer1) - expect(total).to.equal(2, 'Server ' + servers[i].serverNumber + ' does not have the correct amount of views') - } - - for (let i = 0; i < servers.length; i++) { - const total = await sqlCommands[i].countVideoViewsOf(videoIdServer2) - expect(total).to.equal(2, 'Server ' + servers[i].serverNumber + ' does not have the correct amount of views') - } - }) - - it('Should clean old video views', async function () { - this.timeout(50000) - - await killallServers([ servers[0] ]) - - await servers[0].run({ views: { videos: { remote: { max_age: '5 seconds' } } } }) - - await wait(6000) - - // Should still have views - - for (let i = 0; i < servers.length; i++) { - const total = await sqlCommands[i].countVideoViewsOf(videoIdServer1) - expect(total).to.equal(2) - } - - const totalServer1 = await sqlCommands[0].countVideoViewsOf(videoIdServer2) - expect(totalServer1).to.equal(0) - - const totalServer2 = await sqlCommands[1].countVideoViewsOf(videoIdServer2) - expect(totalServer2).to.equal(2) - }) - - after(async function () { - for (const sqlCommand of sqlCommands) { - await sqlCommand.cleanup() - } - - await cleanupTests(servers) - }) -}) diff --git a/server/tests/cli/create-generate-storyboard-job.ts b/server/tests/cli/create-generate-storyboard-job.ts deleted file mode 100644 index 02a4be8ae..000000000 --- a/server/tests/cli/create-generate-storyboard-job.ts +++ /dev/null @@ -1,120 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { readdir, remove } from 'fs-extra' -import { join } from 'path' -import { HttpStatusCode } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - makeGetRequest, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' -import { SQLCommand } from '../shared' - -function listStoryboardFiles (server: PeerTubeServer) { - const storage = server.getDirectoryPath('storyboards') - - return readdir(storage) -} - -describe('Test create generate storyboard job', function () { - let servers: PeerTubeServer[] = [] - const uuids: string[] = [] - let sql: SQLCommand - let existingStoryboardName: string - - before(async function () { - this.timeout(120000) - - // Run server 2 to have transcoding enabled - servers = await createMultipleServers(2) - await setAccessTokensToServers(servers) - - await doubleFollow(servers[0], servers[1]) - - for (let i = 0; i < 3; i++) { - const { uuid } = await servers[0].videos.quickUpload({ name: 'video ' + i }) - uuids.push(uuid) - } - - await waitJobs(servers) - - const storage = servers[0].getDirectoryPath('storyboards') - for (const storyboard of await listStoryboardFiles(servers[0])) { - await remove(join(storage, storyboard)) - } - - sql = new SQLCommand(servers[0]) - await sql.deleteAll('storyboard') - - const { uuid } = await servers[0].videos.quickUpload({ name: 'video 4' }) - uuids.push(uuid) - - await waitJobs(servers) - - const storyboards = await listStoryboardFiles(servers[0]) - existingStoryboardName = storyboards[0] - }) - - it('Should create a storyboard of a video', async function () { - this.timeout(120000) - - for (const uuid of [ uuids[0], uuids[3] ]) { - const command = `npm run create-generate-storyboard-job -- -v ${uuid}` - await servers[0].cli.execWithEnv(command) - } - - await waitJobs(servers) - - { - const storyboards = await listStoryboardFiles(servers[0]) - expect(storyboards).to.have.lengthOf(2) - expect(storyboards).to.not.include(existingStoryboardName) - - existingStoryboardName = storyboards[0] - } - - for (const server of servers) { - for (const uuid of [ uuids[0], uuids[3] ]) { - const { storyboards } = await server.storyboard.list({ id: uuid }) - expect(storyboards).to.have.lengthOf(1) - - await makeGetRequest({ url: server.url, path: storyboards[0].storyboardPath, expectedStatus: HttpStatusCode.OK_200 }) - } - } - }) - - it('Should create missing storyboards', async function () { - this.timeout(120000) - - const command = `npm run create-generate-storyboard-job -- -a` - await servers[0].cli.execWithEnv(command) - - await waitJobs(servers) - - { - const storyboards = await listStoryboardFiles(servers[0]) - expect(storyboards).to.have.lengthOf(4) - expect(storyboards).to.include(existingStoryboardName) - } - - for (const server of servers) { - for (const uuid of uuids) { - const { storyboards } = await server.storyboard.list({ id: uuid }) - expect(storyboards).to.have.lengthOf(1) - - await makeGetRequest({ url: server.url, path: storyboards[0].storyboardPath, expectedStatus: HttpStatusCode.OK_200 }) - } - } - }) - - after(async function () { - await sql.cleanup() - - await cleanupTests(servers) - }) -}) diff --git a/server/tests/cli/create-import-video-file-job.ts b/server/tests/cli/create-import-video-file-job.ts deleted file mode 100644 index edd727967..000000000 --- a/server/tests/cli/create-import-video-file-job.ts +++ /dev/null @@ -1,168 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' -import { HttpStatusCode, VideoDetails, VideoFile, VideoInclude } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - makeRawRequest, - ObjectStorageCommand, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' -import { expectStartWith } from '../shared' - -function assertVideoProperties (video: VideoFile, resolution: number, extname: string, size?: number) { - expect(video).to.have.nested.property('resolution.id', resolution) - expect(video).to.have.property('torrentUrl').that.includes(`-${resolution}.torrent`) - expect(video).to.have.property('fileUrl').that.includes(`.${extname}`) - expect(video).to.have.property('magnetUri').that.includes(`.${extname}`) - expect(video).to.have.property('size').that.is.above(0) - - if (size) expect(video.size).to.equal(size) -} - -async function checkFiles (video: VideoDetails, objectStorage: ObjectStorageCommand) { - for (const file of video.files) { - if (objectStorage) expectStartWith(file.fileUrl, objectStorage.getMockWebVideosBaseUrl()) - - await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) - } -} - -function runTests (enableObjectStorage: boolean) { - let video1ShortId: string - let video2UUID: string - - let servers: PeerTubeServer[] = [] - - const objectStorage = new ObjectStorageCommand() - - before(async function () { - this.timeout(90000) - - const config = enableObjectStorage - ? objectStorage.getDefaultMockConfig() - : {} - - // Run server 2 to have transcoding enabled - servers = await createMultipleServers(2, config) - await setAccessTokensToServers(servers) - - await doubleFollow(servers[0], servers[1]) - - if (enableObjectStorage) await objectStorage.prepareDefaultMockBuckets() - - // Upload two videos for our needs - { - const { shortUUID } = await servers[0].videos.upload({ attributes: { name: 'video1' } }) - video1ShortId = shortUUID - } - - { - const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video2' } }) - video2UUID = uuid - } - - await waitJobs(servers) - - for (const server of servers) { - await server.config.enableTranscoding() - } - }) - - it('Should run a import job on video 1 with a lower resolution', async function () { - const command = `npm run create-import-video-file-job -- -v ${video1ShortId} -i server/tests/fixtures/video_short_480.webm` - await servers[0].cli.execWithEnv(command) - - await waitJobs(servers) - - for (const server of servers) { - const { data: videos } = await server.videos.list() - expect(videos).to.have.lengthOf(2) - - const video = videos.find(({ shortUUID }) => shortUUID === video1ShortId) - const videoDetails = await server.videos.get({ id: video.shortUUID }) - - expect(videoDetails.files).to.have.lengthOf(2) - const [ originalVideo, transcodedVideo ] = videoDetails.files - assertVideoProperties(originalVideo, 720, 'webm', 218910) - assertVideoProperties(transcodedVideo, 480, 'webm', 69217) - - await checkFiles(videoDetails, enableObjectStorage && objectStorage) - } - }) - - it('Should run a import job on video 2 with the same resolution and a different extension', async function () { - const command = `npm run create-import-video-file-job -- -v ${video2UUID} -i server/tests/fixtures/video_short.ogv` - await servers[1].cli.execWithEnv(command) - - await waitJobs(servers) - - for (const server of servers) { - const { data: videos } = await server.videos.listWithToken({ include: VideoInclude.NOT_PUBLISHED_STATE }) - expect(videos).to.have.lengthOf(2) - - const video = videos.find(({ uuid }) => uuid === video2UUID) - const videoDetails = await server.videos.get({ id: video.uuid }) - - expect(videoDetails.files).to.have.lengthOf(4) - const [ originalVideo, transcodedVideo420, transcodedVideo320, transcodedVideo240 ] = videoDetails.files - assertVideoProperties(originalVideo, 720, 'ogv', 140849) - assertVideoProperties(transcodedVideo420, 480, 'mp4') - assertVideoProperties(transcodedVideo320, 360, 'mp4') - assertVideoProperties(transcodedVideo240, 240, 'mp4') - - await checkFiles(videoDetails, enableObjectStorage && objectStorage) - } - }) - - it('Should run a import job on video 2 with the same resolution and the same extension', async function () { - const command = `npm run create-import-video-file-job -- -v ${video1ShortId} -i server/tests/fixtures/video_short2.webm` - await servers[0].cli.execWithEnv(command) - - await waitJobs(servers) - - for (const server of servers) { - const { data: videos } = await server.videos.listWithToken({ include: VideoInclude.NOT_PUBLISHED_STATE }) - expect(videos).to.have.lengthOf(2) - - const video = videos.find(({ shortUUID }) => shortUUID === video1ShortId) - const videoDetails = await server.videos.get({ id: video.uuid }) - - expect(videoDetails.files).to.have.lengthOf(2) - const [ video720, video480 ] = videoDetails.files - assertVideoProperties(video720, 720, 'webm', 942961) - assertVideoProperties(video480, 480, 'webm', 69217) - - await checkFiles(videoDetails, enableObjectStorage && objectStorage) - } - }) - - it('Should not have run transcoding after an import job', async function () { - const { data } = await servers[0].jobs.list({ jobType: 'video-transcoding' }) - expect(data).to.have.lengthOf(0) - }) - - after(async function () { - await objectStorage.cleanupMock() - - await cleanupTests(servers) - }) -} - -describe('Test create import video jobs', function () { - - describe('On filesystem', function () { - runTests(false) - }) - - describe('On object storage', function () { - if (areMockObjectStorageTestsDisabled()) return - - runTests(true) - }) -}) diff --git a/server/tests/cli/create-move-video-storage-job.ts b/server/tests/cli/create-move-video-storage-job.ts deleted file mode 100644 index fc6a8e648..000000000 --- a/server/tests/cli/create-move-video-storage-job.ts +++ /dev/null @@ -1,124 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { join } from 'path' -import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' -import { HttpStatusCode, VideoDetails } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - makeRawRequest, - ObjectStorageCommand, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' -import { checkDirectoryIsEmpty, expectStartWith } from '../shared' - -async function checkFiles (origin: PeerTubeServer, video: VideoDetails, objectStorage?: ObjectStorageCommand) { - for (const file of video.files) { - const start = objectStorage - ? objectStorage.getMockWebVideosBaseUrl() - : origin.url - - expectStartWith(file.fileUrl, start) - - await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) - } - - const start = objectStorage - ? objectStorage.getMockPlaylistBaseUrl() - : origin.url - - const hls = video.streamingPlaylists[0] - expectStartWith(hls.playlistUrl, start) - expectStartWith(hls.segmentsSha256Url, start) - - for (const file of hls.files) { - expectStartWith(file.fileUrl, start) - - await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) - } -} - -describe('Test create move video storage job', function () { - if (areMockObjectStorageTestsDisabled()) return - - let servers: PeerTubeServer[] = [] - const uuids: string[] = [] - const objectStorage = new ObjectStorageCommand() - - before(async function () { - this.timeout(360000) - - // Run server 2 to have transcoding enabled - servers = await createMultipleServers(2) - await setAccessTokensToServers(servers) - - await doubleFollow(servers[0], servers[1]) - - await objectStorage.prepareDefaultMockBuckets() - - await servers[0].config.enableTranscoding() - - for (let i = 0; i < 3; i++) { - const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' + i } }) - uuids.push(uuid) - } - - await waitJobs(servers) - - await servers[0].kill() - await servers[0].run(objectStorage.getDefaultMockConfig()) - }) - - it('Should move only one file', async function () { - this.timeout(120000) - - const command = `npm run create-move-video-storage-job -- --to-object-storage -v ${uuids[1]}` - await servers[0].cli.execWithEnv(command, objectStorage.getDefaultMockConfig()) - await waitJobs(servers) - - for (const server of servers) { - const video = await server.videos.get({ id: uuids[1] }) - - await checkFiles(servers[0], video, objectStorage) - - for (const id of [ uuids[0], uuids[2] ]) { - const video = await server.videos.get({ id }) - - await checkFiles(servers[0], video) - } - } - }) - - it('Should move all files', async function () { - this.timeout(120000) - - const command = `npm run create-move-video-storage-job -- --to-object-storage --all-videos` - await servers[0].cli.execWithEnv(command, objectStorage.getDefaultMockConfig()) - await waitJobs(servers) - - for (const server of servers) { - for (const id of [ uuids[0], uuids[2] ]) { - const video = await server.videos.get({ id }) - - await checkFiles(servers[0], video, objectStorage) - } - } - }) - - it('Should not have files on disk anymore', async function () { - await checkDirectoryIsEmpty(servers[0], 'web-videos', [ 'private' ]) - await checkDirectoryIsEmpty(servers[0], join('web-videos', 'private')) - - await checkDirectoryIsEmpty(servers[0], join('streaming-playlists', 'hls'), [ 'private' ]) - await checkDirectoryIsEmpty(servers[0], join('streaming-playlists', 'hls', 'private')) - }) - - after(async function () { - await objectStorage.cleanupMock() - - await cleanupTests(servers) - }) -}) diff --git a/server/tests/cli/peertube.ts b/server/tests/cli/peertube.ts deleted file mode 100644 index ad14fde91..000000000 --- a/server/tests/cli/peertube.ts +++ /dev/null @@ -1,331 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { areHttpImportTestsDisabled, buildAbsoluteFixturePath } from '@shared/core-utils' -import { - cleanupTests, - CLICommand, - createSingleServer, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' -import { FIXTURE_URLS, testHelloWorldRegisteredSettings } from '../shared' - -describe('Test CLI wrapper', function () { - let server: PeerTubeServer - let userAccessToken: string - - let cliCommand: CLICommand - - const cmd = 'node ./dist/server/tools/peertube.js' - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1, { - rates_limit: { - login: { - max: 30 - } - } - }) - await setAccessTokensToServers([ server ]) - - await server.users.create({ username: 'user_1', password: 'super_password' }) - - userAccessToken = await server.login.getAccessToken({ username: 'user_1', password: 'super_password' }) - - { - const attributes = { name: 'user_channel', displayName: 'User channel', support: 'super support text' } - await server.channels.create({ token: userAccessToken, attributes }) - } - - cliCommand = server.cli - }) - - describe('Authentication and instance selection', function () { - - it('Should get an access token', async function () { - const stdout = await cliCommand.execWithEnv(`${cmd} token --url ${server.url} --username user_1 --password super_password`) - const token = stdout.trim() - - const body = await server.users.getMyInfo({ token }) - expect(body.username).to.equal('user_1') - }) - - it('Should display no selected instance', async function () { - this.timeout(60000) - - const stdout = await cliCommand.execWithEnv(`${cmd} --help`) - expect(stdout).to.contain('no instance selected') - }) - - it('Should add a user', async function () { - this.timeout(60000) - - await cliCommand.execWithEnv(`${cmd} auth add -u ${server.url} -U user_1 -p super_password`) - }) - - it('Should not fail to add a user if there is a slash at the end of the instance URL', async function () { - this.timeout(60000) - - let fullServerURL = server.url + '/' - - await cliCommand.execWithEnv(`${cmd} auth add -u ${fullServerURL} -U user_1 -p super_password`) - - fullServerURL = server.url + '/asdfasdf' - await cliCommand.execWithEnv(`${cmd} auth add -u ${fullServerURL} -U user_1 -p super_password`) - }) - - it('Should default to this user', async function () { - this.timeout(60000) - - const stdout = await cliCommand.execWithEnv(`${cmd} --help`) - expect(stdout).to.contain(`instance ${server.url} selected`) - }) - - it('Should remember the user', async function () { - this.timeout(60000) - - const stdout = await cliCommand.execWithEnv(`${cmd} auth list`) - expect(stdout).to.contain(server.url) - }) - }) - - describe('Video upload/import', function () { - - it('Should upload a video', async function () { - this.timeout(60000) - - const fixture = buildAbsoluteFixturePath('60fps_720p_small.mp4') - const params = `-f ${fixture} --video-name 'test upload' --channel-name user_channel --support 'support_text'` - - await cliCommand.execWithEnv(`${cmd} upload ${params}`) - }) - - it('Should have the video uploaded', async function () { - const { total, data } = await server.videos.list() - expect(total).to.equal(1) - - const video = await server.videos.get({ id: data[0].uuid }) - expect(video.name).to.equal('test upload') - expect(video.support).to.equal('support_text') - expect(video.channel.name).to.equal('user_channel') - }) - - it('Should import a video', async function () { - if (areHttpImportTestsDisabled()) return - - this.timeout(60000) - - const params = `--target-url ${FIXTURE_URLS.youtube} --channel-name user_channel` - await cliCommand.execWithEnv(`${cmd} import ${params}`) - }) - - it('Should have imported the video', async function () { - if (areHttpImportTestsDisabled()) return - - this.timeout(60000) - - await waitJobs([ server ]) - - const { total, data } = await server.videos.list() - expect(total).to.equal(2) - - const video = data.find(v => v.name === 'small video - youtube') - expect(video).to.not.be.undefined - - const videoDetails = await server.videos.get({ id: video.id }) - expect(videoDetails.channel.name).to.equal('user_channel') - expect(videoDetails.support).to.equal('super support text') - expect(videoDetails.nsfw).to.be.false - }) - - it('Should not import again the same video', async function () { - if (areHttpImportTestsDisabled()) return - - this.timeout(60000) - - const params = `--target-url ${FIXTURE_URLS.youtube} --channel-name user_channel` - await cliCommand.execWithEnv(`${cmd} import ${params}`) - - await waitJobs([ server ]) - - const { total, data } = await server.videos.list() - expect(total).to.equal(2) - - const videos = data.filter(v => v.name === 'small video - youtube') - expect(videos).to.have.lengthOf(1) - - // So we can reimport it - await server.videos.remove({ token: userAccessToken, id: videos[0].id }) - }) - - it('Should import and override some imported attributes', async function () { - if (areHttpImportTestsDisabled()) return - - this.timeout(60000) - - const params = `--target-url ${FIXTURE_URLS.youtube} ` + - `--channel-name user_channel --video-name toto --nsfw --support support` - await cliCommand.execWithEnv(`${cmd} import ${params}`) - - await waitJobs([ server ]) - - { - const { total, data } = await server.videos.list() - expect(total).to.equal(2) - - const video = data.find(v => v.name === 'toto') - expect(video).to.not.be.undefined - - const videoDetails = await server.videos.get({ id: video.id }) - expect(videoDetails.channel.name).to.equal('user_channel') - expect(videoDetails.support).to.equal('support') - expect(videoDetails.nsfw).to.be.true - expect(videoDetails.commentsEnabled).to.be.true - } - }) - }) - - describe('Admin auth', function () { - - it('Should remove the auth user', async function () { - await cliCommand.execWithEnv(`${cmd} auth del ${server.url}`) - - const stdout = await cliCommand.execWithEnv(`${cmd} --help`) - expect(stdout).to.contain('no instance selected') - }) - - it('Should add the admin user', async function () { - await cliCommand.execWithEnv(`${cmd} auth add -u ${server.url} -U root -p test${server.internalServerNumber}`) - }) - }) - - describe('Manage plugins', function () { - - it('Should install a plugin', async function () { - this.timeout(60000) - - await cliCommand.execWithEnv(`${cmd} plugins install --npm-name peertube-plugin-hello-world`) - }) - - it('Should have registered settings', async function () { - await testHelloWorldRegisteredSettings(server) - }) - - it('Should list installed plugins', async function () { - const res = await cliCommand.execWithEnv(`${cmd} plugins list`) - - expect(res).to.contain('peertube-plugin-hello-world') - }) - - it('Should uninstall the plugin', async function () { - const res = await cliCommand.execWithEnv(`${cmd} plugins uninstall --npm-name peertube-plugin-hello-world`) - - expect(res).to.not.contain('peertube-plugin-hello-world') - }) - - it('Should install a plugin in requested version', async function () { - this.timeout(60000) - - await cliCommand.execWithEnv(`${cmd} plugins install --npm-name peertube-plugin-hello-world --plugin-version 0.0.17`) - }) - - it('Should list installed plugins, in correct version', async function () { - const res = await cliCommand.execWithEnv(`${cmd} plugins list`) - - expect(res).to.contain('peertube-plugin-hello-world') - expect(res).to.contain('0.0.17') - }) - - it('Should uninstall the plugin again', async function () { - const res = await cliCommand.execWithEnv(`${cmd} plugins uninstall --npm-name peertube-plugin-hello-world`) - - expect(res).to.not.contain('peertube-plugin-hello-world') - }) - - it('Should install a plugin in requested beta version', async function () { - this.timeout(60000) - - await cliCommand.execWithEnv(`${cmd} plugins install --npm-name peertube-plugin-hello-world --plugin-version 0.0.21-beta.1`) - - const res = await cliCommand.execWithEnv(`${cmd} plugins list`) - - expect(res).to.contain('peertube-plugin-hello-world') - expect(res).to.contain('0.0.21-beta.1') - - await cliCommand.execWithEnv(`${cmd} plugins uninstall --npm-name peertube-plugin-hello-world`) - }) - }) - - describe('Manage video redundancies', function () { - let anotherServer: PeerTubeServer - let video1Server2: number - let servers: PeerTubeServer[] - - before(async function () { - this.timeout(120000) - - anotherServer = await createSingleServer(2) - await setAccessTokensToServers([ anotherServer ]) - - await doubleFollow(server, anotherServer) - - servers = [ server, anotherServer ] - await waitJobs(servers) - - const { uuid } = await anotherServer.videos.quickUpload({ name: 'super video' }) - await waitJobs(servers) - - video1Server2 = await server.videos.getId({ uuid }) - }) - - it('Should add a redundancy', async function () { - this.timeout(60000) - - const params = `add --video ${video1Server2}` - await cliCommand.execWithEnv(`${cmd} redundancy ${params}`) - - await waitJobs(servers) - }) - - it('Should list redundancies', async function () { - this.timeout(60000) - - { - const params = 'list-my-redundancies' - const stdout = await cliCommand.execWithEnv(`${cmd} redundancy ${params}`) - - expect(stdout).to.contain('super video') - expect(stdout).to.contain(server.host) - } - }) - - it('Should remove a redundancy', async function () { - this.timeout(60000) - - const params = `remove --video ${video1Server2}` - await cliCommand.execWithEnv(`${cmd} redundancy ${params}`) - - await waitJobs(servers) - - { - const params = 'list-my-redundancies' - const stdout = await cliCommand.execWithEnv(`${cmd} redundancy ${params}`) - - expect(stdout).to.not.contain('super video') - } - }) - - after(async function () { - await cleanupTests([ anotherServer ]) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/cli/plugins.ts b/server/tests/cli/plugins.ts deleted file mode 100644 index c646e20d9..000000000 --- a/server/tests/cli/plugins.ts +++ /dev/null @@ -1,76 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { - cleanupTests, - createSingleServer, - killallServers, - PeerTubeServer, - PluginsCommand, - setAccessTokensToServers -} from '@shared/server-commands' - -describe('Test plugin scripts', function () { - let server: PeerTubeServer - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - }) - - it('Should install a plugin from stateless CLI', async function () { - this.timeout(60000) - - const packagePath = PluginsCommand.getPluginTestPath() - - await server.cli.execWithEnv(`npm run plugin:install -- --plugin-path ${packagePath}`) - }) - - it('Should install a theme from stateless CLI', async function () { - this.timeout(60000) - - await server.cli.execWithEnv(`npm run plugin:install -- --npm-name peertube-theme-background-red`) - }) - - it('Should have the theme and the plugin registered when we restart peertube', async function () { - this.timeout(30000) - - await killallServers([ server ]) - await server.run() - - const config = await server.config.getConfig() - - const plugin = config.plugin.registered - .find(p => p.name === 'test') - expect(plugin).to.not.be.undefined - - const theme = config.theme.registered - .find(t => t.name === 'background-red') - expect(theme).to.not.be.undefined - }) - - it('Should uninstall a plugin from stateless CLI', async function () { - this.timeout(60000) - - await server.cli.execWithEnv(`npm run plugin:uninstall -- --npm-name peertube-plugin-test`) - }) - - it('Should have removed the plugin on another peertube restart', async function () { - this.timeout(30000) - - await killallServers([ server ]) - await server.run() - - const config = await server.config.getConfig() - - const plugin = config.plugin.registered - .find(p => p.name === 'test') - expect(plugin).to.be.undefined - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/cli/prune-storage.ts b/server/tests/cli/prune-storage.ts deleted file mode 100644 index 72a4b1332..000000000 --- a/server/tests/cli/prune-storage.ts +++ /dev/null @@ -1,223 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { createFile, readdir } from 'fs-extra' -import { join } from 'path' -import { wait } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { HttpStatusCode, VideoPlaylistPrivacy, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - CLICommand, - createMultipleServers, - doubleFollow, - killallServers, - makeGetRequest, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - waitJobs -} from '@shared/server-commands' - -async function assertNotExists (server: PeerTubeServer, directory: string, substring: string) { - const files = await readdir(server.servers.buildDirectory(directory)) - - for (const f of files) { - expect(f).to.not.contain(substring) - } -} - -async function assertCountAreOkay (servers: PeerTubeServer[]) { - for (const server of servers) { - const videosCount = await server.servers.countFiles('web-videos') - expect(videosCount).to.equal(9) // 2 videos with 4 resolutions + private directory - - const privateVideosCount = await server.servers.countFiles('web-videos/private') - expect(privateVideosCount).to.equal(4) - - const torrentsCount = await server.servers.countFiles('torrents') - expect(torrentsCount).to.equal(24) - - const previewsCount = await server.servers.countFiles('previews') - expect(previewsCount).to.equal(3) - - const thumbnailsCount = await server.servers.countFiles('thumbnails') - expect(thumbnailsCount).to.equal(5) // 3 local videos, 1 local playlist, 2 remotes videos (lazy downloaded) and 1 remote playlist - - const avatarsCount = await server.servers.countFiles('avatars') - expect(avatarsCount).to.equal(4) - - const hlsRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls')) - expect(hlsRootCount).to.equal(3) // 2 videos + private directory - - const hlsPrivateRootCount = await server.servers.countFiles(join('streaming-playlists', 'hls', 'private')) - expect(hlsPrivateRootCount).to.equal(1) - } -} - -describe('Test prune storage scripts', function () { - let servers: PeerTubeServer[] - const badNames: { [directory: string]: string[] } = {} - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(2, { transcoding: { enabled: true } }) - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - - for (const server of servers) { - await server.videos.upload({ attributes: { name: 'video 1', privacy: VideoPrivacy.PUBLIC } }) - await server.videos.upload({ attributes: { name: 'video 2', privacy: VideoPrivacy.PUBLIC } }) - - await server.videos.upload({ attributes: { name: 'video 3', privacy: VideoPrivacy.PRIVATE } }) - - await server.users.updateMyAvatar({ fixture: 'avatar.png' }) - - await server.playlists.create({ - attributes: { - displayName: 'playlist', - privacy: VideoPlaylistPrivacy.PUBLIC, - videoChannelId: server.store.channel.id, - thumbnailfile: 'custom-thumbnail.jpg' - } - }) - } - - await doubleFollow(servers[0], servers[1]) - - // Lazy load the remote avatars - { - const account = await servers[0].accounts.get({ accountName: 'root@' + servers[1].host }) - - for (const avatar of account.avatars) { - await makeGetRequest({ - url: servers[0].url, - path: avatar.path, - expectedStatus: HttpStatusCode.OK_200 - }) - } - } - - { - const account = await servers[1].accounts.get({ accountName: 'root@' + servers[0].host }) - for (const avatar of account.avatars) { - await makeGetRequest({ - url: servers[1].url, - path: avatar.path, - expectedStatus: HttpStatusCode.OK_200 - }) - } - } - - await wait(1000) - - await waitJobs(servers) - await killallServers(servers) - - await wait(1000) - }) - - it('Should have the files on the disk', async function () { - await assertCountAreOkay(servers) - }) - - it('Should create some dirty files', async function () { - for (let i = 0; i < 2; i++) { - { - const basePublic = servers[0].servers.buildDirectory('web-videos') - const basePrivate = servers[0].servers.buildDirectory(join('web-videos', 'private')) - - const n1 = buildUUID() + '.mp4' - const n2 = buildUUID() + '.webm' - - await createFile(join(basePublic, n1)) - await createFile(join(basePublic, n2)) - await createFile(join(basePrivate, n1)) - await createFile(join(basePrivate, n2)) - - badNames['web-videos'] = [ n1, n2 ] - } - - { - const base = servers[0].servers.buildDirectory('torrents') - - const n1 = buildUUID() + '-240.torrent' - const n2 = buildUUID() + '-480.torrent' - - await createFile(join(base, n1)) - await createFile(join(base, n2)) - - badNames['torrents'] = [ n1, n2 ] - } - - { - const base = servers[0].servers.buildDirectory('thumbnails') - - const n1 = buildUUID() + '.jpg' - const n2 = buildUUID() + '.jpg' - - await createFile(join(base, n1)) - await createFile(join(base, n2)) - - badNames['thumbnails'] = [ n1, n2 ] - } - - { - const base = servers[0].servers.buildDirectory('previews') - - const n1 = buildUUID() + '.jpg' - const n2 = buildUUID() + '.jpg' - - await createFile(join(base, n1)) - await createFile(join(base, n2)) - - badNames['previews'] = [ n1, n2 ] - } - - { - const base = servers[0].servers.buildDirectory('avatars') - - const n1 = buildUUID() + '.png' - const n2 = buildUUID() + '.jpg' - - await createFile(join(base, n1)) - await createFile(join(base, n2)) - - badNames['avatars'] = [ n1, n2 ] - } - - { - const directory = join('streaming-playlists', 'hls') - const basePublic = servers[0].servers.buildDirectory(directory) - const basePrivate = servers[0].servers.buildDirectory(join(directory, 'private')) - - const n1 = buildUUID() - await createFile(join(basePublic, n1)) - await createFile(join(basePrivate, n1)) - badNames[directory] = [ n1 ] - } - } - }) - - it('Should run prune storage', async function () { - this.timeout(30000) - - const env = servers[0].cli.getEnv() - await CLICommand.exec(`echo y | ${env} npm run prune-storage`) - }) - - it('Should have removed files', async function () { - await assertCountAreOkay(servers) - - for (const directory of Object.keys(badNames)) { - for (const name of badNames[directory]) { - await assertNotExists(servers[0], directory, name) - } - } - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/cli/regenerate-thumbnails.ts b/server/tests/cli/regenerate-thumbnails.ts deleted file mode 100644 index 66de7f79c..000000000 --- a/server/tests/cli/regenerate-thumbnails.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { expect } from 'chai' -import { writeFile } from 'fs-extra' -import { basename, join } from 'path' -import { HttpStatusCode, Video } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - makeGetRequest, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '../../../shared/server-commands' - -async function testThumbnail (server: PeerTubeServer, videoId: number | string) { - const video = await server.videos.get({ id: videoId }) - - const requests = [ - makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }), - makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) - ] - - for (const req of requests) { - const res = await req - expect(res.body).to.not.have.lengthOf(0) - } -} - -describe('Test regenerate thumbnails script', function () { - let servers: PeerTubeServer[] - - let video1: Video - let video2: Video - let remoteVideo: Video - - let thumbnail1Path: string - let thumbnailRemotePath: string - - before(async function () { - this.timeout(60000) - - servers = await createMultipleServers(2) - await setAccessTokensToServers(servers) - - await doubleFollow(servers[0], servers[1]) - - { - const videoUUID1 = (await servers[0].videos.quickUpload({ name: 'video 1' })).uuid - video1 = await servers[0].videos.get({ id: videoUUID1 }) - - thumbnail1Path = join(servers[0].servers.buildDirectory('thumbnails'), basename(video1.thumbnailPath)) - - const videoUUID2 = (await servers[0].videos.quickUpload({ name: 'video 2' })).uuid - video2 = await servers[0].videos.get({ id: videoUUID2 }) - } - - { - const videoUUID = (await servers[1].videos.quickUpload({ name: 'video 3' })).uuid - await waitJobs(servers) - - remoteVideo = await servers[0].videos.get({ id: videoUUID }) - - // Load remote thumbnail on disk - await makeGetRequest({ url: servers[0].url, path: remoteVideo.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) - - thumbnailRemotePath = join(servers[0].servers.buildDirectory('thumbnails'), basename(remoteVideo.thumbnailPath)) - } - - await writeFile(thumbnail1Path, '') - await writeFile(thumbnailRemotePath, '') - }) - - it('Should have empty thumbnails', async function () { - { - const res = await makeGetRequest({ url: servers[0].url, path: video1.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) - expect(res.body).to.have.lengthOf(0) - } - - { - const res = await makeGetRequest({ url: servers[0].url, path: video2.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) - expect(res.body).to.not.have.lengthOf(0) - } - - { - const res = await makeGetRequest({ url: servers[0].url, path: remoteVideo.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) - expect(res.body).to.have.lengthOf(0) - } - }) - - it('Should regenerate local thumbnails from the CLI', async function () { - this.timeout(15000) - - await servers[0].cli.execWithEnv(`npm run regenerate-thumbnails`) - }) - - it('Should have generated new thumbnail files', async function () { - await testThumbnail(servers[0], video1.uuid) - await testThumbnail(servers[0], video2.uuid) - - const res = await makeGetRequest({ url: servers[0].url, path: remoteVideo.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) - expect(res.body).to.have.lengthOf(0) - }) - - it('Should have deleted old thumbnail files', async function () { - { - await makeGetRequest({ url: servers[0].url, path: video1.thumbnailPath, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - } - - { - await makeGetRequest({ url: servers[0].url, path: video2.thumbnailPath, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) - } - - { - const res = await makeGetRequest({ url: servers[0].url, path: remoteVideo.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 }) - expect(res.body).to.have.lengthOf(0) - } - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/cli/reset-password.ts b/server/tests/cli/reset-password.ts deleted file mode 100644 index 79892173b..000000000 --- a/server/tests/cli/reset-password.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { cleanupTests, CLICommand, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' - -describe('Test reset password scripts', function () { - let server: PeerTubeServer - - before(async function () { - this.timeout(30000) - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - - await server.users.create({ username: 'user_1', password: 'super password' }) - }) - - it('Should change the user password from CLI', async function () { - this.timeout(60000) - - const env = server.cli.getEnv() - await CLICommand.exec(`echo coucou | ${env} npm run reset-password -- -u user_1`) - - await server.login.login({ user: { username: 'user_1', password: 'coucou' } }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/cli/update-host.ts b/server/tests/cli/update-host.ts deleted file mode 100644 index 386c384e6..000000000 --- a/server/tests/cli/update-host.ts +++ /dev/null @@ -1,134 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { getAllFiles } from '@shared/core-utils' -import { - cleanupTests, - createSingleServer, - killallServers, - makeActivityPubGetRequest, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' -import { parseTorrentVideo } from '../shared' - -describe('Test update host scripts', function () { - let server: PeerTubeServer - - before(async function () { - this.timeout(60000) - - const overrideConfig = { - webserver: { - port: 9256 - } - } - // Run server 2 to have transcoding enabled - server = await createSingleServer(2, overrideConfig) - await setAccessTokensToServers([ server ]) - - // Upload two videos for our needs - const { uuid: video1UUID } = await server.videos.upload() - await server.videos.upload() - - // Create a user - await server.users.create({ username: 'toto', password: 'coucou' }) - - // Create channel - const videoChannel = { - name: 'second_channel', - displayName: 'second video channel', - description: 'super video channel description' - } - await server.channels.create({ attributes: videoChannel }) - - // Create comments - const text = 'my super first comment' - await server.comments.createThread({ videoId: video1UUID, text }) - - await waitJobs(server) - }) - - it('Should run update host', async function () { - this.timeout(30000) - - await killallServers([ server ]) - // Run server with standard configuration - await server.run() - - await server.cli.execWithEnv(`npm run update-host`) - }) - - it('Should have updated videos url', async function () { - const { total, data } = await server.videos.list() - expect(total).to.equal(2) - - for (const video of data) { - const { body } = await makeActivityPubGetRequest(server.url, '/videos/watch/' + video.uuid) - - expect(body.id).to.equal('http://127.0.0.1:9002/videos/watch/' + video.uuid) - - const videoDetails = await server.videos.get({ id: video.uuid }) - - expect(videoDetails.trackerUrls[0]).to.include(server.host) - expect(videoDetails.streamingPlaylists[0].playlistUrl).to.include(server.host) - expect(videoDetails.streamingPlaylists[0].segmentsSha256Url).to.include(server.host) - } - }) - - it('Should have updated video channels url', async function () { - const { data, total } = await server.channels.list({ sort: '-name' }) - expect(total).to.equal(3) - - for (const channel of data) { - const { body } = await makeActivityPubGetRequest(server.url, '/video-channels/' + channel.name) - - expect(body.id).to.equal('http://127.0.0.1:9002/video-channels/' + channel.name) - } - }) - - it('Should have updated accounts url', async function () { - const body = await server.accounts.list() - expect(body.total).to.equal(3) - - for (const account of body.data) { - const usernameWithDomain = account.name - const { body } = await makeActivityPubGetRequest(server.url, '/accounts/' + usernameWithDomain) - - expect(body.id).to.equal('http://127.0.0.1:9002/accounts/' + usernameWithDomain) - } - }) - - it('Should have updated torrent hosts', async function () { - this.timeout(30000) - - const { data } = await server.videos.list() - expect(data).to.have.lengthOf(2) - - for (const video of data) { - const videoDetails = await server.videos.get({ id: video.id }) - const files = getAllFiles(videoDetails) - - expect(files).to.have.lengthOf(8) - - for (const file of files) { - expect(file.magnetUri).to.contain('127.0.0.1%3A9002%2Ftracker%2Fsocket') - expect(file.magnetUri).to.contain('127.0.0.1%3A9002%2Fstatic%2F') - - const torrent = await parseTorrentVideo(server, file) - const announceWS = torrent.announce.find(a => a === 'ws://127.0.0.1:9002/tracker/socket') - expect(announceWS).to.not.be.undefined - - const announceHttp = torrent.announce.find(a => a === 'http://127.0.0.1:9002/tracker/announce') - expect(announceHttp).to.not.be.undefined - - expect(torrent.urlList[0]).to.contain('http://127.0.0.1:9002/static/') - } - } - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/client.ts b/server/tests/client.ts deleted file mode 100644 index 68f3a1d14..000000000 --- a/server/tests/client.ts +++ /dev/null @@ -1,556 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { omit } from '@shared/core-utils' -import { - Account, - HTMLServerConfig, - HttpStatusCode, - ServerConfig, - VideoPlaylistCreateResult, - VideoPlaylistPrivacy, - VideoPrivacy -} from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - makeGetRequest, - makeHTMLRequest, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - waitJobs -} from '../../shared/server-commands' - -function checkIndexTags (html: string, title: string, description: string, css: string, config: ServerConfig) { - expect(html).to.contain('' + title + '') - expect(html).to.contain('') - expect(html).to.contain('') - - const htmlConfig: HTMLServerConfig = omit(config, [ 'signup' ]) - const configObjectString = JSON.stringify(htmlConfig) - const configEscapedString = JSON.stringify(configObjectString) - - expect(html).to.contain(``) -} - -describe('Test a client controllers', function () { - let servers: PeerTubeServer[] = [] - let account: Account - - const videoName = 'my super name for server 1' - const videoDescription = 'my
super __description__ for *server* 1

' - const videoDescriptionPlainText = 'my super description for server 1' - - const playlistName = 'super playlist name' - const playlistDescription = 'super playlist description' - let playlist: VideoPlaylistCreateResult - - const channelDescription = 'my super channel description' - - const watchVideoBasePaths = [ '/videos/watch/', '/w/' ] - const watchPlaylistBasePaths = [ '/videos/watch/playlist/', '/w/p/' ] - - let videoIds: (string | number)[] = [] - let privateVideoId: string - let internalVideoId: string - let unlistedVideoId: string - let passwordProtectedVideoId: string - - let playlistIds: (string | number)[] = [] - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(2) - - await setAccessTokensToServers(servers) - - await doubleFollow(servers[0], servers[1]) - - await setDefaultVideoChannel(servers) - - await servers[0].channels.update({ - channelName: servers[0].store.channel.name, - attributes: { description: channelDescription } - }) - - // Public video - - { - const attributes = { name: videoName, description: videoDescription } - await servers[0].videos.upload({ attributes }) - - const { data } = await servers[0].videos.list() - expect(data.length).to.equal(1) - - const video = data[0] - servers[0].store.video = video - videoIds = [ video.id, video.uuid, video.shortUUID ] - } - - { - ({ uuid: privateVideoId } = await servers[0].videos.quickUpload({ name: 'private', privacy: VideoPrivacy.PRIVATE })); - ({ uuid: unlistedVideoId } = await servers[0].videos.quickUpload({ name: 'unlisted', privacy: VideoPrivacy.UNLISTED })); - ({ uuid: internalVideoId } = await servers[0].videos.quickUpload({ name: 'internal', privacy: VideoPrivacy.INTERNAL })); - ({ uuid: passwordProtectedVideoId } = await servers[0].videos.quickUpload({ - name: 'password protected', - privacy: VideoPrivacy.PASSWORD_PROTECTED, - videoPasswords: [ 'password' ] - })) - } - - // Playlist - - { - const attributes = { - displayName: playlistName, - description: playlistDescription, - privacy: VideoPlaylistPrivacy.PUBLIC, - videoChannelId: servers[0].store.channel.id - } - - playlist = await servers[0].playlists.create({ attributes }) - playlistIds = [ playlist.id, playlist.shortUUID, playlist.uuid ] - - await servers[0].playlists.addElement({ playlistId: playlist.shortUUID, attributes: { videoId: servers[0].store.video.id } }) - } - - // Account - - { - await servers[0].users.updateMe({ description: 'my account description' }) - - account = await servers[0].accounts.get({ accountName: `${servers[0].store.user.username}@${servers[0].host}` }) - } - - await waitJobs(servers) - }) - - describe('oEmbed', function () { - - it('Should have valid oEmbed discovery tags for videos', async function () { - for (const basePath of watchVideoBasePaths) { - for (const id of videoIds) { - const res = await makeGetRequest({ - url: servers[0].url, - path: basePath + id, - accept: 'text/html', - expectedStatus: HttpStatusCode.OK_200 - }) - - const expectedLink = `` - - expect(res.text).to.contain(expectedLink) - } - } - }) - - it('Should have valid oEmbed discovery tags for a playlist', async function () { - for (const basePath of watchPlaylistBasePaths) { - for (const id of playlistIds) { - const res = await makeGetRequest({ - url: servers[0].url, - path: basePath + id, - accept: 'text/html', - expectedStatus: HttpStatusCode.OK_200 - }) - - const expectedLink = `` - - expect(res.text).to.contain(expectedLink) - } - } - }) - }) - - describe('Open Graph', function () { - - async function accountPageTest (path: string) { - const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) - const text = res.text - - expect(text).to.contain(``) - expect(text).to.contain(``) - expect(text).to.contain('') - expect(text).to.contain(``) - } - - async function channelPageTest (path: string) { - const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) - const text = res.text - - expect(text).to.contain(``) - expect(text).to.contain(``) - expect(text).to.contain('') - expect(text).to.contain(``) - } - - async function watchVideoPageTest (path: string) { - const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) - const text = res.text - - expect(text).to.contain(``) - expect(text).to.contain(``) - expect(text).to.contain('') - expect(text).to.contain(``) - } - - async function watchPlaylistPageTest (path: string) { - const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) - const text = res.text - - expect(text).to.contain(``) - expect(text).to.contain(``) - expect(text).to.contain('') - expect(text).to.contain(``) - } - - it('Should have valid Open Graph tags on the account page', async function () { - await accountPageTest('/accounts/' + servers[0].store.user.username) - await accountPageTest('/a/' + servers[0].store.user.username) - await accountPageTest('/@' + servers[0].store.user.username) - }) - - it('Should have valid Open Graph tags on the channel page', async function () { - await channelPageTest('/video-channels/' + servers[0].store.channel.name) - await channelPageTest('/c/' + servers[0].store.channel.name) - await channelPageTest('/@' + servers[0].store.channel.name) - }) - - it('Should have valid Open Graph tags on the watch page', async function () { - for (const path of watchVideoBasePaths) { - for (const id of videoIds) { - await watchVideoPageTest(path + id) - } - } - }) - - it('Should have valid Open Graph tags on the watch page with thread id Angular param', async function () { - for (const path of watchVideoBasePaths) { - for (const id of videoIds) { - await watchVideoPageTest(path + id + ';threadId=1') - } - } - }) - - it('Should have valid Open Graph tags on the watch playlist page', async function () { - for (const path of watchPlaylistBasePaths) { - for (const id of playlistIds) { - await watchPlaylistPageTest(path + id) - } - } - }) - }) - - describe('Twitter card', async function () { - - describe('Not whitelisted', function () { - - async function accountPageTest (path: string) { - const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) - const text = res.text - - expect(text).to.contain('') - expect(text).to.contain('') - expect(text).to.contain(``) - expect(text).to.contain(``) - } - - async function channelPageTest (path: string) { - const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) - const text = res.text - - expect(text).to.contain('') - expect(text).to.contain('') - expect(text).to.contain(``) - expect(text).to.contain(``) - } - - async function watchVideoPageTest (path: string) { - const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) - const text = res.text - - expect(text).to.contain('') - expect(text).to.contain('') - expect(text).to.contain(``) - expect(text).to.contain(``) - } - - async function watchPlaylistPageTest (path: string) { - const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) - const text = res.text - - expect(text).to.contain('') - expect(text).to.contain('') - expect(text).to.contain(``) - expect(text).to.contain(``) - } - - it('Should have valid twitter card on the watch video page', async function () { - for (const path of watchVideoBasePaths) { - for (const id of videoIds) { - await watchVideoPageTest(path + id) - } - } - }) - - it('Should have valid twitter card on the watch playlist page', async function () { - for (const path of watchPlaylistBasePaths) { - for (const id of playlistIds) { - await watchPlaylistPageTest(path + id) - } - } - }) - - it('Should have valid twitter card on the account page', async function () { - await accountPageTest('/accounts/' + account.name) - await accountPageTest('/a/' + account.name) - await accountPageTest('/@' + account.name) - }) - - it('Should have valid twitter card on the channel page', async function () { - await channelPageTest('/video-channels/' + servers[0].store.channel.name) - await channelPageTest('/c/' + servers[0].store.channel.name) - await channelPageTest('/@' + servers[0].store.channel.name) - }) - }) - - describe('Whitelisted', function () { - - before(async function () { - const config = await servers[0].config.getCustomConfig() - config.services.twitter = { - username: '@Kuja', - whitelisted: true - } - - await servers[0].config.updateCustomConfig({ newCustomConfig: config }) - }) - - async function accountPageTest (path: string) { - const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) - const text = res.text - - expect(text).to.contain('') - expect(text).to.contain('') - } - - async function channelPageTest (path: string) { - const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) - const text = res.text - - expect(text).to.contain('') - expect(text).to.contain('') - } - - async function watchVideoPageTest (path: string) { - const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) - const text = res.text - - expect(text).to.contain('') - expect(text).to.contain('') - } - - async function watchPlaylistPageTest (path: string) { - const res = await makeGetRequest({ url: servers[0].url, path, accept: 'text/html', expectedStatus: HttpStatusCode.OK_200 }) - const text = res.text - - expect(text).to.contain('') - expect(text).to.contain('') - } - - it('Should have valid twitter card on the watch video page', async function () { - for (const path of watchVideoBasePaths) { - for (const id of videoIds) { - await watchVideoPageTest(path + id) - } - } - }) - - it('Should have valid twitter card on the watch playlist page', async function () { - for (const path of watchPlaylistBasePaths) { - for (const id of playlistIds) { - await watchPlaylistPageTest(path + id) - } - } - }) - - it('Should have valid twitter card on the account page', async function () { - await accountPageTest('/accounts/' + account.name) - await accountPageTest('/a/' + account.name) - await accountPageTest('/@' + account.name) - }) - - it('Should have valid twitter card on the channel page', async function () { - await channelPageTest('/video-channels/' + servers[0].store.channel.name) - await channelPageTest('/c/' + servers[0].store.channel.name) - await channelPageTest('/@' + servers[0].store.channel.name) - }) - }) - }) - - describe('Index HTML', function () { - - it('Should have valid index html tags (title, description...)', async function () { - const config = await servers[0].config.getConfig() - const res = await makeHTMLRequest(servers[0].url, '/videos/trending') - - const description = 'PeerTube, an ActivityPub-federated video streaming platform using P2P directly in your web browser.' - checkIndexTags(res.text, 'PeerTube', description, '', config) - }) - - it('Should update the customized configuration and have the correct index html tags', async function () { - await servers[0].config.updateCustomSubConfig({ - newConfig: { - instance: { - name: 'PeerTube updated', - shortDescription: 'my short description', - description: 'my super description', - terms: 'my super terms', - defaultNSFWPolicy: 'blur', - defaultClientRoute: '/videos/recently-added', - customizations: { - javascript: 'alert("coucou")', - css: 'body { background-color: red; }' - } - } - } - }) - - const config = await servers[0].config.getConfig() - const res = await makeHTMLRequest(servers[0].url, '/videos/trending') - - checkIndexTags(res.text, 'PeerTube updated', 'my short description', 'body { background-color: red; }', config) - }) - - it('Should have valid index html updated tags (title, description...)', async function () { - const config = await servers[0].config.getConfig() - const res = await makeHTMLRequest(servers[0].url, '/videos/trending') - - checkIndexTags(res.text, 'PeerTube updated', 'my short description', 'body { background-color: red; }', config) - }) - - it('Should use the original video URL for the canonical tag', async function () { - for (const basePath of watchVideoBasePaths) { - for (const id of videoIds) { - const res = await makeHTMLRequest(servers[1].url, basePath + id) - expect(res.text).to.contain(``) - } - } - }) - - it('Should use the original account URL for the canonical tag', async function () { - const accountURLtest = res => { - expect(res.text).to.contain(``) - } - - accountURLtest(await makeHTMLRequest(servers[1].url, '/accounts/root@' + servers[0].host)) - accountURLtest(await makeHTMLRequest(servers[1].url, '/a/root@' + servers[0].host)) - accountURLtest(await makeHTMLRequest(servers[1].url, '/@root@' + servers[0].host)) - }) - - it('Should use the original channel URL for the canonical tag', async function () { - const channelURLtests = res => { - expect(res.text).to.contain(``) - } - - channelURLtests(await makeHTMLRequest(servers[1].url, '/video-channels/root_channel@' + servers[0].host)) - channelURLtests(await makeHTMLRequest(servers[1].url, '/c/root_channel@' + servers[0].host)) - channelURLtests(await makeHTMLRequest(servers[1].url, '/@root_channel@' + servers[0].host)) - }) - - it('Should use the original playlist URL for the canonical tag', async function () { - for (const basePath of watchPlaylistBasePaths) { - for (const id of playlistIds) { - const res = await makeHTMLRequest(servers[1].url, basePath + id) - expect(res.text).to.contain(``) - } - } - }) - - it('Should add noindex meta tag for remote accounts', async function () { - const handle = 'root@' + servers[0].host - const paths = [ '/accounts/', '/a/', '/@' ] - - for (const path of paths) { - { - const { text } = await makeHTMLRequest(servers[1].url, path + handle) - expect(text).to.contain('') - } - - { - const { text } = await makeHTMLRequest(servers[0].url, path + handle) - expect(text).to.not.contain('') - } - } - }) - - it('Should add noindex meta tag for remote channels', async function () { - const handle = 'root_channel@' + servers[0].host - const paths = [ '/video-channels/', '/c/', '/@' ] - - for (const path of paths) { - { - const { text } = await makeHTMLRequest(servers[1].url, path + handle) - expect(text).to.contain('') - } - - { - const { text } = await makeHTMLRequest(servers[0].url, path + handle) - expect(text).to.not.contain('') - } - } - }) - - it('Should not display internal/private/password protected video', async function () { - for (const basePath of watchVideoBasePaths) { - for (const id of [ privateVideoId, internalVideoId, passwordProtectedVideoId ]) { - const res = await makeGetRequest({ - url: servers[0].url, - path: basePath + id, - accept: 'text/html', - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - - expect(res.text).to.not.contain('internal') - expect(res.text).to.not.contain('private') - expect(res.text).to.not.contain('password protected') - } - } - }) - - it('Should add noindex meta tag for unlisted video', async function () { - for (const basePath of watchVideoBasePaths) { - const res = await makeGetRequest({ - url: servers[0].url, - path: basePath + unlistedVideoId, - accept: 'text/html', - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(res.text).to.contain('unlisted') - expect(res.text).to.contain('') - } - }) - }) - - describe('Embed HTML', function () { - - it('Should have the correct embed html tags', async function () { - const config = await servers[0].config.getConfig() - const res = await makeHTMLRequest(servers[0].url, servers[0].store.video.embedPath) - - checkIndexTags(res.text, 'PeerTube updated', 'my short description', 'body { background-color: red; }', config) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/external-plugins/akismet.ts b/server/tests/external-plugins/akismet.ts deleted file mode 100644 index e964bf0c2..000000000 --- a/server/tests/external-plugins/akismet.ts +++ /dev/null @@ -1,160 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { HttpStatusCode } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' - -describe('Official plugin Akismet', function () { - let servers: PeerTubeServer[] - let videoUUID: string - - before(async function () { - this.timeout(30000) - - servers = await createMultipleServers(2) - await setAccessTokensToServers(servers) - - await servers[0].plugins.install({ - npmName: 'peertube-plugin-akismet' - }) - - if (!process.env.AKISMET_KEY) throw new Error('Missing AKISMET_KEY from env') - - await servers[0].plugins.updateSettings({ - npmName: 'peertube-plugin-akismet', - settings: { - 'akismet-api-key': process.env.AKISMET_KEY - } - }) - - await doubleFollow(servers[0], servers[1]) - }) - - describe('Local threads/replies', function () { - - before(async function () { - const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' }) - videoUUID = uuid - }) - - it('Should not detect a thread as spam', async function () { - await servers[0].comments.createThread({ videoId: videoUUID, text: 'comment' }) - }) - - it('Should not detect a reply as spam', async function () { - await servers[0].comments.addReplyToLastThread({ text: 'reply' }) - }) - - it('Should detect a thread as spam', async function () { - await servers[0].comments.createThread({ - videoId: videoUUID, - text: 'akismet-guaranteed-spam', - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should detect a thread as spam', async function () { - await servers[0].comments.createThread({ videoId: videoUUID, text: 'comment' }) - await servers[0].comments.addReplyToLastThread({ text: 'akismet-guaranteed-spam', expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - }) - - describe('Remote threads/replies', function () { - - before(async function () { - this.timeout(60000) - - const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' }) - videoUUID = uuid - - await waitJobs(servers) - }) - - it('Should not detect a thread as spam', async function () { - this.timeout(30000) - - await servers[1].comments.createThread({ videoId: videoUUID, text: 'remote comment 1' }) - await waitJobs(servers) - - const { data } = await servers[0].comments.listThreads({ videoId: videoUUID }) - expect(data).to.have.lengthOf(1) - }) - - it('Should not detect a reply as spam', async function () { - this.timeout(30000) - - await servers[1].comments.addReplyToLastThread({ text: 'I agree with you' }) - await waitJobs(servers) - - const { data } = await servers[0].comments.listThreads({ videoId: videoUUID }) - expect(data).to.have.lengthOf(1) - - const tree = await servers[0].comments.getThread({ videoId: videoUUID, threadId: data[0].id }) - expect(tree.children).to.have.lengthOf(1) - }) - - it('Should detect a thread as spam', async function () { - this.timeout(30000) - - await servers[1].comments.createThread({ videoId: videoUUID, text: 'akismet-guaranteed-spam' }) - await waitJobs(servers) - - const { data } = await servers[0].comments.listThreads({ videoId: videoUUID }) - expect(data).to.have.lengthOf(1) - }) - - it('Should detect a thread as spam', async function () { - this.timeout(30000) - - await servers[1].comments.addReplyToLastThread({ text: 'akismet-guaranteed-spam' }) - await waitJobs(servers) - - const { data } = await servers[0].comments.listThreads({ videoId: videoUUID }) - expect(data).to.have.lengthOf(1) - - const thread = data[0] - const tree = await servers[0].comments.getThread({ videoId: videoUUID, threadId: thread.id }) - expect(tree.children).to.have.lengthOf(1) - }) - }) - - describe('Signup', function () { - - before(async function () { - await servers[0].config.updateExistingSubConfig({ - newConfig: { - signup: { - enabled: true - } - } - }) - }) - - it('Should allow signup', async function () { - await servers[0].registrations.register({ - username: 'user1', - displayName: 'user 1' - }) - }) - - it('Should detect a signup as SPAM', async function () { - await servers[0].registrations.register({ - username: 'user2', - displayName: 'user 2', - email: 'akismet-guaranteed-spam@example.com', - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/external-plugins/auth-ldap.ts b/server/tests/external-plugins/auth-ldap.ts deleted file mode 100644 index d51d337be..000000000 --- a/server/tests/external-plugins/auth-ldap.ts +++ /dev/null @@ -1,117 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' -import { HttpStatusCode } from '@shared/models' - -describe('Official plugin auth-ldap', function () { - let server: PeerTubeServer - let accessToken: string - let userId: number - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - - await server.plugins.install({ npmName: 'peertube-plugin-auth-ldap' }) - }) - - it('Should not login with without LDAP settings', async function () { - await server.login.login({ user: { username: 'fry', password: 'fry' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should not login with bad LDAP settings', async function () { - await server.plugins.updateSettings({ - npmName: 'peertube-plugin-auth-ldap', - settings: { - 'bind-credentials': 'GoodNewsEveryone', - 'bind-dn': 'cn=admin,dc=planetexpress,dc=com', - 'insecure-tls': false, - 'mail-property': 'mail', - 'search-base': 'ou=people,dc=planetexpress,dc=com', - 'search-filter': '(|(mail={{username}})(uid={{username}}))', - 'url': 'ldap://127.0.0.1:390', - 'username-property': 'uid' - } - }) - - await server.login.login({ user: { username: 'fry', password: 'fry' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should not login with good LDAP settings but wrong username/password', async function () { - await server.plugins.updateSettings({ - npmName: 'peertube-plugin-auth-ldap', - settings: { - 'bind-credentials': 'GoodNewsEveryone', - 'bind-dn': 'cn=admin,dc=planetexpress,dc=com', - 'insecure-tls': false, - 'mail-property': 'mail', - 'search-base': 'ou=people,dc=planetexpress,dc=com', - 'search-filter': '(|(mail={{username}})(uid={{username}}))', - 'url': 'ldap://127.0.0.1:10389', - 'username-property': 'uid' - } - }) - - await server.login.login({ user: { username: 'fry', password: 'bad password' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await server.login.login({ user: { username: 'fryr', password: 'fry' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should login with the appropriate username/password', async function () { - accessToken = await server.login.getAccessToken({ username: 'fry', password: 'fry' }) - }) - - it('Should login with the appropriate email/password', async function () { - accessToken = await server.login.getAccessToken({ username: 'fry@planetexpress.com', password: 'fry' }) - }) - - it('Should login get my profile', async function () { - const body = await server.users.getMyInfo({ token: accessToken }) - expect(body.username).to.equal('fry') - expect(body.email).to.equal('fry@planetexpress.com') - - userId = body.id - }) - - it('Should upload a video', async function () { - await server.videos.upload({ token: accessToken, attributes: { name: 'my super video' } }) - }) - - it('Should not be able to login if the user is banned', async function () { - await server.users.banUser({ userId }) - - await server.login.login({ - user: { username: 'fry@planetexpress.com', password: 'fry' }, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should be able to login if the user is unbanned', async function () { - await server.users.unbanUser({ userId }) - - await server.login.login({ user: { username: 'fry@planetexpress.com', password: 'fry' } }) - }) - - it('Should not be able to ask password reset', async function () { - await server.users.askResetPassword({ email: 'fry@planetexpress.com', expectedStatus: HttpStatusCode.CONFLICT_409 }) - }) - - it('Should not be able to ask email verification', async function () { - await server.users.askSendVerifyEmail({ email: 'fry@planetexpress.com', expectedStatus: HttpStatusCode.CONFLICT_409 }) - }) - - it('Should not login if the plugin is uninstalled', async function () { - await server.plugins.uninstall({ npmName: 'peertube-plugin-auth-ldap' }) - - await server.login.login({ - user: { username: 'fry@planetexpress.com', password: 'fry' }, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/external-plugins/auto-block-videos.ts b/server/tests/external-plugins/auto-block-videos.ts deleted file mode 100644 index 95d7a4b58..000000000 --- a/server/tests/external-plugins/auto-block-videos.ts +++ /dev/null @@ -1,167 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { wait } from '@shared/core-utils' -import { Video } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - killallServers, - PeerTubeServer, - setAccessTokensToServers -} from '@shared/server-commands' -import { MockBlocklist } from '../shared' - -async function check (server: PeerTubeServer, videoUUID: string, exists = true) { - const { data } = await server.videos.list() - - const video = data.find(v => v.uuid === videoUUID) - - if (exists) expect(video).to.not.be.undefined - else expect(video).to.be.undefined -} - -describe('Official plugin auto-block videos', function () { - let servers: PeerTubeServer[] - let blocklistServer: MockBlocklist - let server1Videos: Video[] = [] - let server2Videos: Video[] = [] - let port: number - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(2) - await setAccessTokensToServers(servers) - - for (const server of servers) { - await server.plugins.install({ npmName: 'peertube-plugin-auto-block-videos' }) - } - - blocklistServer = new MockBlocklist() - port = await blocklistServer.initialize() - - await servers[0].videos.quickUpload({ name: 'video server 1' }) - await servers[1].videos.quickUpload({ name: 'video server 2' }) - await servers[1].videos.quickUpload({ name: 'video 2 server 2' }) - await servers[1].videos.quickUpload({ name: 'video 3 server 2' }) - - { - const { data } = await servers[0].videos.list() - server1Videos = data.map(v => Object.assign(v, { url: servers[0].url + '/videos/watch/' + v.uuid })) - } - - { - const { data } = await servers[1].videos.list() - server2Videos = data.map(v => Object.assign(v, { url: servers[1].url + '/videos/watch/' + v.uuid })) - } - - await doubleFollow(servers[0], servers[1]) - }) - - it('Should update plugin settings', async function () { - await servers[0].plugins.updateSettings({ - npmName: 'peertube-plugin-auto-block-videos', - settings: { - 'blocklist-urls': `http://127.0.0.1:${port}/blocklist`, - 'check-seconds-interval': 1 - } - }) - }) - - it('Should auto block a video', async function () { - await check(servers[0], server2Videos[0].uuid, true) - - blocklistServer.replace({ - data: [ - { - value: server2Videos[0].url - } - ] - }) - - await wait(2000) - - await check(servers[0], server2Videos[0].uuid, false) - }) - - it('Should have video in blacklists', async function () { - const body = await servers[0].blacklist.list() - - const videoBlacklists = body.data - expect(videoBlacklists).to.have.lengthOf(1) - expect(videoBlacklists[0].reason).to.contains('Automatically blocked from auto block plugin') - expect(videoBlacklists[0].video.name).to.equal(server2Videos[0].name) - }) - - it('Should not block a local video', async function () { - await check(servers[0], server1Videos[0].uuid, true) - - blocklistServer.replace({ - data: [ - { - value: server1Videos[0].url - } - ] - }) - - await wait(2000) - - await check(servers[0], server1Videos[0].uuid, true) - }) - - it('Should remove a video block', async function () { - await check(servers[0], server2Videos[0].uuid, false) - - blocklistServer.replace({ - data: [ - { - value: server2Videos[0].url, - action: 'remove' - } - ] - }) - - await wait(2000) - - await check(servers[0], server2Videos[0].uuid, true) - }) - - it('Should auto block a video, manually unblock it and do not reblock it automatically', async function () { - this.timeout(20000) - - const video = server2Videos[1] - - await check(servers[0], video.uuid, true) - - blocklistServer.replace({ - data: [ - { - value: video.url, - updatedAt: new Date().toISOString() - } - ] - }) - - await wait(2000) - - await check(servers[0], video.uuid, false) - - await servers[0].blacklist.remove({ videoId: video.uuid }) - - await check(servers[0], video.uuid, true) - - await killallServers([ servers[0] ]) - await servers[0].run() - await wait(2000) - - await check(servers[0], video.uuid, true) - }) - - after(async function () { - await blocklistServer.terminate() - - await cleanupTests(servers) - }) -}) diff --git a/server/tests/external-plugins/auto-mute.ts b/server/tests/external-plugins/auto-mute.ts deleted file mode 100644 index a9bf3c173..000000000 --- a/server/tests/external-plugins/auto-mute.ts +++ /dev/null @@ -1,216 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { wait } from '@shared/core-utils' -import { HttpStatusCode } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - killallServers, - makeGetRequest, - PeerTubeServer, - setAccessTokensToServers -} from '@shared/server-commands' -import { MockBlocklist } from '../shared' - -describe('Official plugin auto-mute', function () { - const autoMuteListPath = '/plugins/auto-mute/router/api/v1/mute-list' - let servers: PeerTubeServer[] - let blocklistServer: MockBlocklist - let port: number - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(2) - await setAccessTokensToServers(servers) - - for (const server of servers) { - await server.plugins.install({ npmName: 'peertube-plugin-auto-mute' }) - } - - blocklistServer = new MockBlocklist() - port = await blocklistServer.initialize() - - await servers[0].videos.quickUpload({ name: 'video server 1' }) - await servers[1].videos.quickUpload({ name: 'video server 2' }) - - await doubleFollow(servers[0], servers[1]) - }) - - it('Should update plugin settings', async function () { - await servers[0].plugins.updateSettings({ - npmName: 'peertube-plugin-auto-mute', - settings: { - 'blocklist-urls': `http://127.0.0.1:${port}/blocklist`, - 'check-seconds-interval': 1 - } - }) - }) - - it('Should add a server blocklist', async function () { - blocklistServer.replace({ - data: [ - { - value: servers[1].host - } - ] - }) - - await wait(2000) - - const { total } = await servers[0].videos.list() - expect(total).to.equal(1) - }) - - it('Should remove a server blocklist', async function () { - blocklistServer.replace({ - data: [ - { - value: servers[1].host, - action: 'remove' - } - ] - }) - - await wait(2000) - - const { total } = await servers[0].videos.list() - expect(total).to.equal(2) - }) - - it('Should add an account blocklist', async function () { - blocklistServer.replace({ - data: [ - { - value: 'root@' + servers[1].host - } - ] - }) - - await wait(2000) - - const { total } = await servers[0].videos.list() - expect(total).to.equal(1) - }) - - it('Should remove an account blocklist', async function () { - blocklistServer.replace({ - data: [ - { - value: 'root@' + servers[1].host, - action: 'remove' - } - ] - }) - - await wait(2000) - - const { total } = await servers[0].videos.list() - expect(total).to.equal(2) - }) - - it('Should auto mute an account, manually unmute it and do not remute it automatically', async function () { - this.timeout(20000) - - const account = 'root@' + servers[1].host - - blocklistServer.replace({ - data: [ - { - value: account, - updatedAt: new Date().toISOString() - } - ] - }) - - await wait(2000) - - { - const { total } = await servers[0].videos.list() - expect(total).to.equal(1) - } - - await servers[0].blocklist.removeFromServerBlocklist({ account }) - - { - const { total } = await servers[0].videos.list() - expect(total).to.equal(2) - } - - await killallServers([ servers[0] ]) - await servers[0].run() - await wait(2000) - - { - const { total } = await servers[0].videos.list() - expect(total).to.equal(2) - } - }) - - it('Should not expose the auto mute list', async function () { - await makeGetRequest({ - url: servers[0].url, - path: '/plugins/auto-mute/router/api/v1/mute-list', - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should enable auto mute list', async function () { - await servers[0].plugins.updateSettings({ - npmName: 'peertube-plugin-auto-mute', - settings: { - 'blocklist-urls': '', - 'check-seconds-interval': 1, - 'expose-mute-list': true - } - }) - - await makeGetRequest({ - url: servers[0].url, - path: '/plugins/auto-mute/router/api/v1/mute-list', - expectedStatus: HttpStatusCode.OK_200 - }) - }) - - it('Should mute an account on server 1, and server 2 auto mutes it', async function () { - this.timeout(20000) - - await servers[1].plugins.updateSettings({ - npmName: 'peertube-plugin-auto-mute', - settings: { - 'blocklist-urls': 'http://' + servers[0].host + autoMuteListPath, - 'check-seconds-interval': 1, - 'expose-mute-list': false - } - }) - - await servers[0].blocklist.addToServerBlocklist({ account: 'root@' + servers[1].host }) - await servers[0].blocklist.addToMyBlocklist({ server: servers[1].host }) - - const res = await makeGetRequest({ - url: servers[0].url, - path: '/plugins/auto-mute/router/api/v1/mute-list', - expectedStatus: HttpStatusCode.OK_200 - }) - - const data = res.body.data - expect(data).to.have.lengthOf(1) - expect(data[0].updatedAt).to.exist - expect(data[0].value).to.equal('root@' + servers[1].host) - - await wait(2000) - - for (const server of servers) { - const { total } = await server.videos.list() - expect(total).to.equal(1) - } - }) - - after(async function () { - await blocklistServer.terminate() - - await cleanupTests(servers) - }) -}) diff --git a/server/tests/feeds/feeds.ts b/server/tests/feeds/feeds.ts deleted file mode 100644 index 1754ac466..000000000 --- a/server/tests/feeds/feeds.ts +++ /dev/null @@ -1,695 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import * as chai from 'chai' -import { XMLParser, XMLValidator } from 'fast-xml-parser' -import { HttpStatusCode, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - createSingleServer, - doubleFollow, - makeGetRequest, - makeRawRequest, - PeerTubeServer, - PluginsCommand, - setAccessTokensToServers, - setDefaultChannelAvatar, - setDefaultVideoChannel, - stopFfmpeg, - waitJobs -} from '@shared/server-commands' - -chai.use(require('chai-xml')) -chai.use(require('chai-json-schema')) -chai.config.includeStack = true - -const expect = chai.expect - -describe('Test syndication feeds', () => { - let servers: PeerTubeServer[] = [] - let serverHLSOnly: PeerTubeServer - - let userAccessToken: string - let rootAccountId: number - let rootChannelId: number - - let userAccountId: number - let userChannelId: number - let userFeedToken: string - - let liveId: string - - before(async function () { - this.timeout(120000) - - // Run servers - servers = await createMultipleServers(2) - serverHLSOnly = await createSingleServer(3, { - transcoding: { - enabled: true, - web_videos: { enabled: false }, - hls: { enabled: true } - } - }) - - await setAccessTokensToServers([ ...servers, serverHLSOnly ]) - await setDefaultChannelAvatar(servers[0]) - await setDefaultVideoChannel(servers) - await doubleFollow(servers[0], servers[1]) - - await servers[0].config.enableLive({ allowReplay: false, transcoding: false }) - - { - const user = await servers[0].users.getMyInfo() - rootAccountId = user.account.id - rootChannelId = user.videoChannels[0].id - } - - { - userAccessToken = await servers[0].users.generateUserAndToken('john') - - const user = await servers[0].users.getMyInfo({ token: userAccessToken }) - userAccountId = user.account.id - userChannelId = user.videoChannels[0].id - - const token = await servers[0].users.getMyScopedTokens({ token: userAccessToken }) - userFeedToken = token.feedToken - } - - { - await servers[0].videos.upload({ token: userAccessToken, attributes: { name: 'user video' } }) - } - - { - const attributes = { - name: 'my super name for server 1', - description: 'my super description for server 1', - fixture: 'video_short.webm' - } - const { id } = await servers[0].videos.upload({ attributes }) - - await servers[0].comments.createThread({ videoId: id, text: 'super comment 1' }) - await servers[0].comments.createThread({ videoId: id, text: 'super comment 2' }) - } - - { - const attributes = { name: 'unlisted video', privacy: VideoPrivacy.UNLISTED } - const { id } = await servers[0].videos.upload({ attributes }) - - await servers[0].comments.createThread({ videoId: id, text: 'comment on unlisted video' }) - } - - { - const attributes = { name: 'password protected video', privacy: VideoPrivacy.PASSWORD_PROTECTED, videoPasswords: [ 'password' ] } - const { id } = await servers[0].videos.upload({ attributes }) - - await servers[0].comments.createThread({ videoId: id, text: 'comment on password protected video' }) - } - - await serverHLSOnly.videos.upload({ attributes: { name: 'hls only video' } }) - - await waitJobs([ ...servers, serverHLSOnly ]) - - await servers[0].plugins.install({ path: PluginsCommand.getPluginTestPath('-podcast-custom-tags') }) - }) - - describe('All feed', function () { - - it('Should be well formed XML (covers RSS 2.0 and ATOM 1.0 endpoints)', async function () { - for (const feed of [ 'video-comments' as 'video-comments', 'videos' as 'videos' ]) { - const rss = await servers[0].feed.getXML({ feed, ignoreCache: true }) - expect(rss).xml.to.be.valid() - - const atom = await servers[0].feed.getXML({ feed, format: 'atom', ignoreCache: true }) - expect(atom).xml.to.be.valid() - } - }) - - it('Should be well formed XML (covers Podcast endpoint)', async function () { - const podcast = await servers[0].feed.getPodcastXML({ ignoreCache: true, channelId: rootChannelId }) - expect(podcast).xml.to.be.valid() - }) - - it('Should be well formed JSON (covers JSON feed 1.0 endpoint)', async function () { - for (const feed of [ 'video-comments' as 'video-comments', 'videos' as 'videos' ]) { - const jsonText = await servers[0].feed.getJSON({ feed, ignoreCache: true }) - expect(JSON.parse(jsonText)).to.be.jsonSchema({ type: 'object' }) - } - }) - - it('Should serve the endpoint with a classic request', async function () { - await makeGetRequest({ - url: servers[0].url, - path: '/feeds/videos.xml', - accept: 'application/xml', - expectedStatus: HttpStatusCode.OK_200 - }) - }) - - it('Should refuse to serve the endpoint without accept header', async function () { - await makeGetRequest({ url: servers[0].url, path: '/feeds/videos.xml', expectedStatus: HttpStatusCode.NOT_ACCEPTABLE_406 }) - }) - }) - - describe('Videos feed', function () { - - describe('Podcast feed', function () { - - it('Should contain a valid podcast:alternateEnclosure', async function () { - // Since podcast feeds should only work on the server they originate on, - // only test the first server where the videos reside - const rss = await servers[0].feed.getPodcastXML({ ignoreCache: false, channelId: rootChannelId }) - expect(XMLValidator.validate(rss)).to.be.true - - const parser = new XMLParser({ parseAttributeValue: true, ignoreAttributes: false }) - const xmlDoc = parser.parse(rss) - - const itemGuid = xmlDoc.rss.channel.item.guid - expect(itemGuid).to.exist - expect(itemGuid['@_isPermaLink']).to.equal(true) - - const enclosure = xmlDoc.rss.channel.item.enclosure - expect(enclosure).to.exist - const alternateEnclosure = xmlDoc.rss.channel.item['podcast:alternateEnclosure'] - expect(alternateEnclosure).to.exist - - expect(alternateEnclosure['@_type']).to.equal('video/webm') - expect(alternateEnclosure['@_length']).to.equal(218910) - expect(alternateEnclosure['@_lang']).to.equal('zh') - expect(alternateEnclosure['@_title']).to.equal('720p') - expect(alternateEnclosure['@_default']).to.equal(true) - - expect(alternateEnclosure['podcast:source'][0]['@_uri']).to.contain('-720.webm') - expect(alternateEnclosure['podcast:source'][0]['@_uri']).to.equal(enclosure['@_url']) - expect(alternateEnclosure['podcast:source'][1]['@_uri']).to.contain('-720.torrent') - expect(alternateEnclosure['podcast:source'][1]['@_contentType']).to.equal('application/x-bittorrent') - expect(alternateEnclosure['podcast:source'][2]['@_uri']).to.contain('magnet:?') - }) - - it('Should contain a valid podcast:alternateEnclosure with HLS only', async function () { - const rss = await serverHLSOnly.feed.getPodcastXML({ ignoreCache: false, channelId: rootChannelId }) - expect(XMLValidator.validate(rss)).to.be.true - - const parser = new XMLParser({ parseAttributeValue: true, ignoreAttributes: false }) - const xmlDoc = parser.parse(rss) - - const itemGuid = xmlDoc.rss.channel.item.guid - expect(itemGuid).to.exist - expect(itemGuid['@_isPermaLink']).to.equal(true) - - const enclosure = xmlDoc.rss.channel.item.enclosure - const alternateEnclosure = xmlDoc.rss.channel.item['podcast:alternateEnclosure'] - expect(alternateEnclosure).to.exist - - expect(alternateEnclosure['@_type']).to.equal('application/x-mpegURL') - expect(alternateEnclosure['@_lang']).to.equal('zh') - expect(alternateEnclosure['@_title']).to.equal('HLS') - expect(alternateEnclosure['@_default']).to.equal(true) - - expect(alternateEnclosure['podcast:source']['@_uri']).to.contain('-master.m3u8') - expect(alternateEnclosure['podcast:source']['@_uri']).to.equal(enclosure['@_url']) - }) - - it('Should contain a valid podcast:socialInteract', async function () { - const rss = await servers[0].feed.getPodcastXML({ ignoreCache: false, channelId: rootChannelId }) - expect(XMLValidator.validate(rss)).to.be.true - - const parser = new XMLParser({ parseAttributeValue: true, ignoreAttributes: false }) - const xmlDoc = parser.parse(rss) - - const item = xmlDoc.rss.channel.item - const socialInteract = item['podcast:socialInteract'] - expect(socialInteract).to.exist - expect(socialInteract['@_protocol']).to.equal('activitypub') - expect(socialInteract['@_uri']).to.exist - expect(socialInteract['@_accountUrl']).to.exist - }) - - it('Should contain a valid support custom tags for plugins', async function () { - const rss = await servers[0].feed.getPodcastXML({ ignoreCache: false, channelId: userChannelId }) - expect(XMLValidator.validate(rss)).to.be.true - - const parser = new XMLParser({ parseAttributeValue: true, ignoreAttributes: false }) - const xmlDoc = parser.parse(rss) - - const fooTag = xmlDoc.rss.channel.fooTag - expect(fooTag).to.exist - expect(fooTag['@_bar']).to.equal('baz') - expect(fooTag['#text']).to.equal(42) - - const bizzBuzzItem = xmlDoc.rss.channel['biz:buzzItem'] - expect(bizzBuzzItem).to.exist - - let nestedTag = bizzBuzzItem.nestedTag - expect(nestedTag).to.exist - expect(nestedTag).to.equal('example nested tag') - - const item = xmlDoc.rss.channel.item - const fizzTag = item.fizzTag - expect(fizzTag).to.exist - expect(fizzTag['@_bar']).to.equal('baz') - expect(fizzTag['#text']).to.equal(21) - - const bizzBuzz = item['biz:buzz'] - expect(bizzBuzz).to.exist - - nestedTag = bizzBuzz.nestedTag - expect(nestedTag).to.exist - expect(nestedTag).to.equal('example nested tag') - }) - - it('Should contain a valid podcast:liveItem for live streams', async function () { - this.timeout(120000) - - const { uuid } = await servers[0].live.create({ - fields: { - name: 'live-0', - privacy: VideoPrivacy.PUBLIC, - channelId: rootChannelId, - permanentLive: false - } - }) - liveId = uuid - - const ffmpeg = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveId, copyCodecs: true, fixtureName: 'video_short.mp4' }) - await servers[0].live.waitUntilPublished({ videoId: liveId }) - - const rss = await servers[0].feed.getPodcastXML({ ignoreCache: false, channelId: rootChannelId }) - expect(XMLValidator.validate(rss)).to.be.true - - const parser = new XMLParser({ parseAttributeValue: true, ignoreAttributes: false }) - const xmlDoc = parser.parse(rss) - const liveItem = xmlDoc.rss.channel['podcast:liveItem'] - expect(liveItem.title).to.equal('live-0') - expect(liveItem.guid['@_isPermaLink']).to.equal(false) - expect(liveItem.guid['#text']).to.contain(`${uuid}_`) - expect(liveItem['@_status']).to.equal('live') - - const enclosure = liveItem.enclosure - const alternateEnclosure = liveItem['podcast:alternateEnclosure'] - expect(alternateEnclosure).to.exist - expect(alternateEnclosure['@_type']).to.equal('application/x-mpegURL') - expect(alternateEnclosure['@_title']).to.equal('HLS live stream') - expect(alternateEnclosure['@_default']).to.equal(true) - - expect(alternateEnclosure['podcast:source']['@_uri']).to.contain('/master.m3u8') - expect(alternateEnclosure['podcast:source']['@_uri']).to.equal(enclosure['@_url']) - - await stopFfmpeg(ffmpeg) - - await servers[0].live.waitUntilEnded({ videoId: liveId }) - - await waitJobs(servers) - }) - }) - - describe('JSON feed', function () { - - it('Should contain a valid \'attachments\' object', async function () { - for (const server of servers) { - const json = await server.feed.getJSON({ feed: 'videos', ignoreCache: true }) - const jsonObj = JSON.parse(json) - expect(jsonObj.items.length).to.be.equal(2) - expect(jsonObj.items[0].attachments).to.exist - expect(jsonObj.items[0].attachments.length).to.be.eq(1) - expect(jsonObj.items[0].attachments[0].mime_type).to.be.eq('application/x-bittorrent') - expect(jsonObj.items[0].attachments[0].size_in_bytes).to.be.eq(218910) - expect(jsonObj.items[0].attachments[0].url).to.contain('720.torrent') - } - }) - - it('Should filter by account', async function () { - { - const json = await servers[0].feed.getJSON({ feed: 'videos', query: { accountId: rootAccountId }, ignoreCache: true }) - const jsonObj = JSON.parse(json) - expect(jsonObj.items.length).to.be.equal(1) - expect(jsonObj.items[0].title).to.equal('my super name for server 1') - expect(jsonObj.items[0].author.name).to.equal('Main root channel') - } - - { - const json = await servers[0].feed.getJSON({ feed: 'videos', query: { accountId: userAccountId }, ignoreCache: true }) - const jsonObj = JSON.parse(json) - expect(jsonObj.items.length).to.be.equal(1) - expect(jsonObj.items[0].title).to.equal('user video') - expect(jsonObj.items[0].author.name).to.equal('Main john channel') - } - - for (const server of servers) { - { - const json = await server.feed.getJSON({ feed: 'videos', query: { accountName: 'root@' + servers[0].host }, ignoreCache: true }) - const jsonObj = JSON.parse(json) - expect(jsonObj.items.length).to.be.equal(1) - expect(jsonObj.items[0].title).to.equal('my super name for server 1') - } - - { - const json = await server.feed.getJSON({ feed: 'videos', query: { accountName: 'john@' + servers[0].host }, ignoreCache: true }) - const jsonObj = JSON.parse(json) - expect(jsonObj.items.length).to.be.equal(1) - expect(jsonObj.items[0].title).to.equal('user video') - } - } - }) - - it('Should filter by video channel', async function () { - { - const json = await servers[0].feed.getJSON({ feed: 'videos', query: { videoChannelId: rootChannelId }, ignoreCache: true }) - const jsonObj = JSON.parse(json) - expect(jsonObj.items.length).to.be.equal(1) - expect(jsonObj.items[0].title).to.equal('my super name for server 1') - expect(jsonObj.items[0].author.name).to.equal('Main root channel') - } - - { - const json = await servers[0].feed.getJSON({ feed: 'videos', query: { videoChannelId: userChannelId }, ignoreCache: true }) - const jsonObj = JSON.parse(json) - expect(jsonObj.items.length).to.be.equal(1) - expect(jsonObj.items[0].title).to.equal('user video') - expect(jsonObj.items[0].author.name).to.equal('Main john channel') - } - - for (const server of servers) { - { - const query = { videoChannelName: 'root_channel@' + servers[0].host } - const json = await server.feed.getJSON({ feed: 'videos', query, ignoreCache: true }) - const jsonObj = JSON.parse(json) - expect(jsonObj.items.length).to.be.equal(1) - expect(jsonObj.items[0].title).to.equal('my super name for server 1') - } - - { - const query = { videoChannelName: 'john_channel@' + servers[0].host } - const json = await server.feed.getJSON({ feed: 'videos', query, ignoreCache: true }) - const jsonObj = JSON.parse(json) - expect(jsonObj.items.length).to.be.equal(1) - expect(jsonObj.items[0].title).to.equal('user video') - } - } - }) - - it('Should correctly have videos feed with HLS only', async function () { - this.timeout(120000) - - const json = await serverHLSOnly.feed.getJSON({ feed: 'videos', ignoreCache: true }) - const jsonObj = JSON.parse(json) - expect(jsonObj.items.length).to.be.equal(1) - expect(jsonObj.items[0].attachments).to.exist - expect(jsonObj.items[0].attachments.length).to.be.eq(4) - - for (let i = 0; i < 4; i++) { - expect(jsonObj.items[0].attachments[i].mime_type).to.be.eq('application/x-bittorrent') - expect(jsonObj.items[0].attachments[i].size_in_bytes).to.be.greaterThan(0) - expect(jsonObj.items[0].attachments[i].url).to.exist - } - }) - - it('Should not display waiting live videos', async function () { - const { uuid } = await servers[0].live.create({ - fields: { - name: 'live', - privacy: VideoPrivacy.PUBLIC, - channelId: rootChannelId - } - }) - liveId = uuid - - const json = await servers[0].feed.getJSON({ feed: 'videos', ignoreCache: true }) - - const jsonObj = JSON.parse(json) - expect(jsonObj.items.length).to.be.equal(2) - expect(jsonObj.items[0].title).to.equal('my super name for server 1') - expect(jsonObj.items[1].title).to.equal('user video') - }) - - it('Should display published live videos', async function () { - this.timeout(120000) - - const ffmpeg = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveId, copyCodecs: true, fixtureName: 'video_short.mp4' }) - await servers[0].live.waitUntilPublished({ videoId: liveId }) - - const json = await servers[0].feed.getJSON({ feed: 'videos', ignoreCache: true }) - - const jsonObj = JSON.parse(json) - expect(jsonObj.items.length).to.be.equal(3) - expect(jsonObj.items[0].title).to.equal('live') - expect(jsonObj.items[1].title).to.equal('my super name for server 1') - expect(jsonObj.items[2].title).to.equal('user video') - - await stopFfmpeg(ffmpeg) - }) - - it('Should have the channel avatar as feed icon', async function () { - const json = await servers[0].feed.getJSON({ feed: 'videos', query: { videoChannelId: rootChannelId }, ignoreCache: true }) - - const jsonObj = JSON.parse(json) - const imageUrl = jsonObj.icon - expect(imageUrl).to.include('/lazy-static/avatars/') - await makeRawRequest({ url: imageUrl, expectedStatus: HttpStatusCode.OK_200 }) - }) - }) - }) - - describe('Video comments feed', function () { - - it('Should contain valid comments (covers JSON feed 1.0 endpoint) and not from unlisted/password protected videos', async function () { - for (const server of servers) { - const json = await server.feed.getJSON({ feed: 'video-comments', ignoreCache: true }) - - const jsonObj = JSON.parse(json) - expect(jsonObj.items.length).to.be.equal(2) - expect(jsonObj.items[0].content_html).to.contain('

super comment 2

') - expect(jsonObj.items[1].content_html).to.contain('

super comment 1

') - } - }) - - it('Should not list comments from muted accounts or instances', async function () { - this.timeout(30000) - - const remoteHandle = 'root@' + servers[0].host - - await servers[1].blocklist.addToServerBlocklist({ account: remoteHandle }) - - { - const json = await servers[1].feed.getJSON({ feed: 'video-comments', ignoreCache: true }) - const jsonObj = JSON.parse(json) - expect(jsonObj.items.length).to.be.equal(0) - } - - await servers[1].blocklist.removeFromServerBlocklist({ account: remoteHandle }) - - { - const videoUUID = (await servers[1].videos.quickUpload({ name: 'server 2' })).uuid - await waitJobs(servers) - await servers[0].comments.createThread({ videoId: videoUUID, text: 'super comment' }) - await waitJobs(servers) - - const json = await servers[1].feed.getJSON({ feed: 'video-comments', ignoreCache: true }) - const jsonObj = JSON.parse(json) - expect(jsonObj.items.length).to.be.equal(3) - } - - await servers[1].blocklist.addToMyBlocklist({ account: remoteHandle }) - - { - const json = await servers[1].feed.getJSON({ feed: 'video-comments', ignoreCache: true }) - const jsonObj = JSON.parse(json) - expect(jsonObj.items.length).to.be.equal(2) - } - }) - }) - - describe('Video feed from my subscriptions', function () { - let feeduserAccountId: number - let feeduserFeedToken: string - - it('Should list no videos for a user with no videos and no subscriptions', async function () { - const attr = { username: 'feeduser', password: 'password' } - await servers[0].users.create({ username: attr.username, password: attr.password }) - const feeduserAccessToken = await servers[0].login.getAccessToken(attr) - - { - const user = await servers[0].users.getMyInfo({ token: feeduserAccessToken }) - feeduserAccountId = user.account.id - } - - { - const token = await servers[0].users.getMyScopedTokens({ token: feeduserAccessToken }) - feeduserFeedToken = token.feedToken - } - - { - const body = await servers[0].videos.listMySubscriptionVideos({ token: feeduserAccessToken }) - expect(body.total).to.equal(0) - - const query = { accountId: feeduserAccountId, token: feeduserFeedToken } - const json = await servers[0].feed.getJSON({ feed: 'subscriptions', query, ignoreCache: true }) - const jsonObj = JSON.parse(json) - expect(jsonObj.items.length).to.be.equal(0) // no subscription, it should not list the instance's videos but list 0 videos - } - }) - - it('Should fail with an invalid token', async function () { - const query = { accountId: feeduserAccountId, token: 'toto' } - await servers[0].feed.getJSON({ feed: 'subscriptions', query, expectedStatus: HttpStatusCode.FORBIDDEN_403, ignoreCache: true }) - }) - - it('Should fail with a token of another user', async function () { - const query = { accountId: feeduserAccountId, token: userFeedToken } - await servers[0].feed.getJSON({ feed: 'subscriptions', query, expectedStatus: HttpStatusCode.FORBIDDEN_403, ignoreCache: true }) - }) - - it('Should list no videos for a user with videos but no subscriptions', async function () { - const body = await servers[0].videos.listMySubscriptionVideos({ token: userAccessToken }) - expect(body.total).to.equal(0) - - const query = { accountId: userAccountId, token: userFeedToken } - const json = await servers[0].feed.getJSON({ feed: 'subscriptions', query, ignoreCache: true }) - const jsonObj = JSON.parse(json) - expect(jsonObj.items.length).to.be.equal(0) // no subscription, it should not list the instance's videos but list 0 videos - }) - - it('Should list self videos for a user with a subscription to themselves', async function () { - this.timeout(30000) - - await servers[0].subscriptions.add({ token: userAccessToken, targetUri: 'john_channel@' + servers[0].host }) - await waitJobs(servers) - - { - const body = await servers[0].videos.listMySubscriptionVideos({ token: userAccessToken }) - expect(body.total).to.equal(1) - expect(body.data[0].name).to.equal('user video') - - const query = { accountId: userAccountId, token: userFeedToken } - const json = await servers[0].feed.getJSON({ feed: 'subscriptions', query, ignoreCache: true }) - const jsonObj = JSON.parse(json) - expect(jsonObj.items.length).to.be.equal(1) // subscribed to self, it should not list the instance's videos but list john's - } - }) - - it('Should list videos of a user\'s subscription', async function () { - this.timeout(30000) - - await servers[0].subscriptions.add({ token: userAccessToken, targetUri: 'root_channel@' + servers[0].host }) - await waitJobs(servers) - - { - const body = await servers[0].videos.listMySubscriptionVideos({ token: userAccessToken }) - expect(body.total).to.equal(2, 'there should be 2 videos part of the subscription') - - const query = { accountId: userAccountId, token: userFeedToken } - const json = await servers[0].feed.getJSON({ feed: 'subscriptions', query, ignoreCache: true }) - const jsonObj = JSON.parse(json) - expect(jsonObj.items.length).to.be.equal(2) // subscribed to root, it should not list the instance's videos but list root/john's - } - }) - - it('Should renew the token, and so have an invalid old token', async function () { - await servers[0].users.renewMyScopedTokens({ token: userAccessToken }) - - const query = { accountId: userAccountId, token: userFeedToken } - await servers[0].feed.getJSON({ feed: 'subscriptions', query, expectedStatus: HttpStatusCode.FORBIDDEN_403, ignoreCache: true }) - }) - - it('Should succeed with the new token', async function () { - const token = await servers[0].users.getMyScopedTokens({ token: userAccessToken }) - userFeedToken = token.feedToken - - const query = { accountId: userAccountId, token: userFeedToken } - await servers[0].feed.getJSON({ feed: 'subscriptions', query, ignoreCache: true }) - }) - - }) - - describe('Cache', function () { - const uuids: string[] = [] - - function doPodcastRequest () { - return makeGetRequest({ - url: servers[0].url, - path: '/feeds/podcast/videos.xml', - query: { videoChannelId: servers[0].store.channel.id }, - accept: 'application/xml', - expectedStatus: HttpStatusCode.OK_200 - }) - } - - function doVideosRequest (query: { [id: string]: string } = {}) { - return makeGetRequest({ - url: servers[0].url, - path: '/feeds/videos.xml', - query, - accept: 'application/xml', - expectedStatus: HttpStatusCode.OK_200 - }) - } - - before(async function () { - { - const { uuid } = await servers[0].videos.quickUpload({ name: 'cache 1' }) - uuids.push(uuid) - } - - { - const { uuid } = await servers[0].videos.quickUpload({ name: 'cache 2' }) - uuids.push(uuid) - } - }) - - it('Should serve the videos endpoint as a cached request', async function () { - await doVideosRequest() - - const res = await doVideosRequest() - - expect(res.headers['x-api-cache-cached']).to.equal('true') - }) - - it('Should not serve the videos endpoint as a cached request', async function () { - const res = await doVideosRequest({ v: '186' }) - - expect(res.headers['x-api-cache-cached']).to.not.exist - }) - - it('Should invalidate the podcast feed cache after video deletion', async function () { - await doPodcastRequest() - - { - const res = await doPodcastRequest() - expect(res.headers['x-api-cache-cached']).to.exist - } - - await servers[0].videos.remove({ id: uuids[0] }) - - { - const res = await doPodcastRequest() - expect(res.headers['x-api-cache-cached']).to.not.exist - } - }) - - it('Should invalidate the podcast feed cache after video deletion, even after server restart', async function () { - this.timeout(120000) - - await doPodcastRequest() - - { - const res = await doPodcastRequest() - expect(res.headers['x-api-cache-cached']).to.exist - } - - await servers[0].kill() - await servers[0].run() - - await servers[0].videos.remove({ id: uuids[1] }) - - const res = await doPodcastRequest() - expect(res.headers['x-api-cache-cached']).to.not.exist - }) - - }) - - after(async function () { - await servers[0].plugins.uninstall({ npmName: 'peertube-plugin-test-podcast-custom-tags' }) - - await cleanupTests([ ...servers, serverHLSOnly ]) - }) -}) diff --git a/server/tests/helpers/comment-model.ts b/server/tests/helpers/comment-model.ts deleted file mode 100644 index e39cae442..000000000 --- a/server/tests/helpers/comment-model.ts +++ /dev/null @@ -1,24 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { VideoCommentModel } from '../../models/video/video-comment' - -class CommentMock { - text: string - - extractMentions = VideoCommentModel.prototype.extractMentions - - isOwned = () => true -} - -describe('Comment model', function () { - it('Should correctly extract mentions', async function () { - const comment = new CommentMock() - - comment.text = '@florian @jean@localhost:9000 @flo @another@localhost:9000 @flo2@jean.com hello ' + - 'email@localhost:9000 coucou.com no? @chocobozzz @chocobozzz @end' - const result = comment.extractMentions().sort((a, b) => a.localeCompare(b)) - - expect(result).to.deep.equal([ 'another', 'chocobozzz', 'end', 'flo', 'florian', 'jean' ]) - }) -}) diff --git a/server/tests/helpers/core-utils.ts b/server/tests/helpers/core-utils.ts deleted file mode 100644 index cd2f07e4a..000000000 --- a/server/tests/helpers/core-utils.ts +++ /dev/null @@ -1,150 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { snakeCase } from 'lodash' -import validator from 'validator' -import { getAverageTheoreticalBitrate, getMaxTheoreticalBitrate } from '@shared/core-utils' -import { VideoResolution } from '@shared/models' -import { objectConverter, parseBytes, parseDurationToMs } from '../../helpers/core-utils' - -describe('Parse Bytes', function () { - - it('Should pass on valid value', async function () { - // just return it - expect(parseBytes(-1024)).to.equal(-1024) - expect(parseBytes(1024)).to.equal(1024) - expect(parseBytes(1048576)).to.equal(1048576) - expect(parseBytes('1024')).to.equal(1024) - expect(parseBytes('1048576')).to.equal(1048576) - - // sizes - expect(parseBytes('1B')).to.equal(1024) - expect(parseBytes('1MB')).to.equal(1048576) - expect(parseBytes('1GB')).to.equal(1073741824) - expect(parseBytes('1TB')).to.equal(1099511627776) - - expect(parseBytes('5GB')).to.equal(5368709120) - expect(parseBytes('5TB')).to.equal(5497558138880) - - expect(parseBytes('1024B')).to.equal(1048576) - expect(parseBytes('1024MB')).to.equal(1073741824) - expect(parseBytes('1024GB')).to.equal(1099511627776) - expect(parseBytes('1024TB')).to.equal(1125899906842624) - - // with whitespace - expect(parseBytes('1 GB')).to.equal(1073741824) - expect(parseBytes('1\tGB')).to.equal(1073741824) - - // sum value - expect(parseBytes('1TB 1024MB')).to.equal(1100585369600) - expect(parseBytes('4GB 1024MB')).to.equal(5368709120) - expect(parseBytes('4TB 1024GB')).to.equal(5497558138880) - expect(parseBytes('4TB 1024GB 0MB')).to.equal(5497558138880) - expect(parseBytes('1024TB 1024GB 1024MB')).to.equal(1127000492212224) - }) - - it('Should be invalid when given invalid value', async function () { - expect(parseBytes('6GB 1GB')).to.equal(6) - }) -}) - -describe('Parse duration', function () { - - it('Should pass when given valid value', async function () { - expect(parseDurationToMs(35)).to.equal(35) - expect(parseDurationToMs(-35)).to.equal(-35) - expect(parseDurationToMs('35 seconds')).to.equal(35 * 1000) - expect(parseDurationToMs('1 minute')).to.equal(60 * 1000) - expect(parseDurationToMs('1 hour')).to.equal(3600 * 1000) - expect(parseDurationToMs('35 hours')).to.equal(3600 * 35 * 1000) - }) - - it('Should be invalid when given invalid value', async function () { - expect(parseBytes('35m 5s')).to.equal(35) - }) -}) - -describe('Object', function () { - - it('Should convert an object', async function () { - function keyConverter (k: string) { - return snakeCase(k) - } - - function valueConverter (v: any) { - if (validator.isNumeric(v + '')) return parseInt('' + v, 10) - - return v - } - - const obj = { - mySuperKey: 'hello', - mySuper2Key: '45', - mySuper3Key: { - mySuperSubKey: '15', - mySuperSub2Key: 'hello', - mySuperSub3Key: [ '1', 'hello', 2 ], - mySuperSub4Key: 4 - }, - mySuper4Key: 45, - toto: { - super_key: '15', - superKey2: 'hello' - }, - super_key: { - superKey4: 15 - } - } - - const res = objectConverter(obj, keyConverter, valueConverter) - - expect(res.my_super_key).to.equal('hello') - expect(res.my_super_2_key).to.equal(45) - expect(res.my_super_3_key.my_super_sub_key).to.equal(15) - expect(res.my_super_3_key.my_super_sub_2_key).to.equal('hello') - expect(res.my_super_3_key.my_super_sub_3_key).to.deep.equal([ 1, 'hello', 2 ]) - expect(res.my_super_3_key.my_super_sub_4_key).to.equal(4) - expect(res.toto.super_key).to.equal(15) - expect(res.toto.super_key_2).to.equal('hello') - expect(res.super_key.super_key_4).to.equal(15) - - // Immutable - expect(res.mySuperKey).to.be.undefined - expect(obj['my_super_key']).to.be.undefined - }) -}) - -describe('Bitrate', function () { - - it('Should get appropriate max bitrate', function () { - const tests = [ - { resolution: VideoResolution.H_144P, ratio: 16 / 9, fps: 24, min: 200, max: 400 }, - { resolution: VideoResolution.H_240P, ratio: 16 / 9, fps: 24, min: 600, max: 800 }, - { resolution: VideoResolution.H_360P, ratio: 16 / 9, fps: 24, min: 1200, max: 1600 }, - { resolution: VideoResolution.H_480P, ratio: 16 / 9, fps: 24, min: 2000, max: 2300 }, - { resolution: VideoResolution.H_720P, ratio: 16 / 9, fps: 24, min: 4000, max: 4400 }, - { resolution: VideoResolution.H_1080P, ratio: 16 / 9, fps: 24, min: 8000, max: 10000 }, - { resolution: VideoResolution.H_4K, ratio: 16 / 9, fps: 24, min: 25000, max: 30000 } - ] - - for (const test of tests) { - expect(getMaxTheoreticalBitrate(test)).to.be.above(test.min * 1000).and.below(test.max * 1000) - } - }) - - it('Should get appropriate average bitrate', function () { - const tests = [ - { resolution: VideoResolution.H_144P, ratio: 16 / 9, fps: 24, min: 50, max: 300 }, - { resolution: VideoResolution.H_240P, ratio: 16 / 9, fps: 24, min: 350, max: 450 }, - { resolution: VideoResolution.H_360P, ratio: 16 / 9, fps: 24, min: 700, max: 900 }, - { resolution: VideoResolution.H_480P, ratio: 16 / 9, fps: 24, min: 1100, max: 1300 }, - { resolution: VideoResolution.H_720P, ratio: 16 / 9, fps: 24, min: 2300, max: 2500 }, - { resolution: VideoResolution.H_1080P, ratio: 16 / 9, fps: 24, min: 4700, max: 5000 }, - { resolution: VideoResolution.H_4K, ratio: 16 / 9, fps: 24, min: 15000, max: 17000 } - ] - - for (const test of tests) { - expect(getAverageTheoreticalBitrate(test)).to.be.above(test.min * 1000).and.below(test.max * 1000) - } - }) -}) diff --git a/server/tests/helpers/crypto.ts b/server/tests/helpers/crypto.ts deleted file mode 100644 index b508c715b..000000000 --- a/server/tests/helpers/crypto.ts +++ /dev/null @@ -1,33 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { decrypt, encrypt } from '@server/helpers/peertube-crypto' - -describe('Encrypt/Descrypt', function () { - - it('Should encrypt and decrypt the string', async function () { - const secret = 'my_secret' - const str = 'my super string' - - const encrypted = await encrypt(str, secret) - const decrypted = await decrypt(encrypted, secret) - - expect(str).to.equal(decrypted) - }) - - it('Should not decrypt without the same secret', async function () { - const str = 'my super string' - - const encrypted = await encrypt(str, 'my_secret') - - let error = false - - try { - await decrypt(encrypted, 'my_sicret') - } catch (err) { - error = true - } - - expect(error).to.be.true - }) -}) diff --git a/server/tests/helpers/dns.ts b/server/tests/helpers/dns.ts deleted file mode 100644 index 49b506e7b..000000000 --- a/server/tests/helpers/dns.ts +++ /dev/null @@ -1,16 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { isResolvingToUnicastOnly } from '@server/helpers/dns' - -describe('DNS helpers', function () { - - it('Should correctly check unicast IPs', async function () { - expect(await isResolvingToUnicastOnly('cpy.re')).to.be.true - expect(await isResolvingToUnicastOnly('framasoft.org')).to.be.true - expect(await isResolvingToUnicastOnly('8.8.8.8')).to.be.true - - expect(await isResolvingToUnicastOnly('127.0.0.1')).to.be.false - expect(await isResolvingToUnicastOnly('127.0.0.1.cpy.re')).to.be.false - }) -}) diff --git a/server/tests/helpers/image.ts b/server/tests/helpers/image.ts deleted file mode 100644 index 6021ffc48..000000000 --- a/server/tests/helpers/image.ts +++ /dev/null @@ -1,96 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { readFile, remove } from 'fs-extra' -import { join } from 'path' -import { execPromise } from '@server/helpers/core-utils' -import { buildAbsoluteFixturePath, root } from '@shared/core-utils' -import { processImage } from '../../../server/helpers/image-utils' - -async function checkBuffers (path1: string, path2: string, equals: boolean) { - const [ buf1, buf2 ] = await Promise.all([ - readFile(path1), - readFile(path2) - ]) - - if (equals) { - expect(buf1.equals(buf2)).to.be.true - } else { - expect(buf1.equals(buf2)).to.be.false - } -} - -async function hasTitleExif (path: string) { - const result = JSON.parse(await execPromise(`exiftool -json ${path}`)) - - return result[0]?.Title === 'should be removed' -} - -describe('Image helpers', function () { - const imageDestDir = join(root(), 'test-images') - - const imageDestJPG = join(imageDestDir, 'test.jpg') - const imageDestPNG = join(imageDestDir, 'test.png') - - const thumbnailSize = { width: 280, height: 157 } - - it('Should skip processing if the source image is okay', async function () { - const input = buildAbsoluteFixturePath('custom-thumbnail.jpg') - await processImage({ path: input, destination: imageDestJPG, newSize: thumbnailSize, keepOriginal: true }) - - await checkBuffers(input, imageDestJPG, true) - }) - - it('Should not skip processing if the source image does not have the appropriate extension', async function () { - const input = buildAbsoluteFixturePath('custom-thumbnail.png') - await processImage({ path: input, destination: imageDestJPG, newSize: thumbnailSize, keepOriginal: true }) - - await checkBuffers(input, imageDestJPG, false) - }) - - it('Should not skip processing if the source image does not have the appropriate size', async function () { - const input = buildAbsoluteFixturePath('custom-preview.jpg') - await processImage({ path: input, destination: imageDestJPG, newSize: thumbnailSize, keepOriginal: true }) - - await checkBuffers(input, imageDestJPG, false) - }) - - it('Should not skip processing if the source image does not have the appropriate size', async function () { - const input = buildAbsoluteFixturePath('custom-thumbnail-big.jpg') - await processImage({ path: input, destination: imageDestJPG, newSize: thumbnailSize, keepOriginal: true }) - - await checkBuffers(input, imageDestJPG, false) - }) - - it('Should strip exif for a jpg file that can not be copied', async function () { - const input = buildAbsoluteFixturePath('exif.jpg') - expect(await hasTitleExif(input)).to.be.true - - await processImage({ path: input, destination: imageDestJPG, newSize: { width: 100, height: 100 }, keepOriginal: true }) - await checkBuffers(input, imageDestJPG, false) - - expect(await hasTitleExif(imageDestJPG)).to.be.false - }) - - it('Should strip exif for a jpg file that could be copied', async function () { - const input = buildAbsoluteFixturePath('exif.jpg') - expect(await hasTitleExif(input)).to.be.true - - await processImage({ path: input, destination: imageDestJPG, newSize: thumbnailSize, keepOriginal: true }) - await checkBuffers(input, imageDestJPG, false) - - expect(await hasTitleExif(imageDestJPG)).to.be.false - }) - - it('Should strip exif for png', async function () { - const input = buildAbsoluteFixturePath('exif.png') - expect(await hasTitleExif(input)).to.be.true - - await processImage({ path: input, destination: imageDestPNG, newSize: thumbnailSize, keepOriginal: true }) - expect(await hasTitleExif(imageDestPNG)).to.be.false - }) - - after(async function () { - await remove(imageDestDir) - }) -}) diff --git a/server/tests/helpers/index.ts b/server/tests/helpers/index.ts deleted file mode 100644 index 073ae6455..000000000 --- a/server/tests/helpers/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -import './comment-model' -import './core-utils' -import './crypto' -import './dns' -import './image' -import './markdown' -import './request' -import './validator' -import './version' diff --git a/server/tests/helpers/markdown.ts b/server/tests/helpers/markdown.ts deleted file mode 100644 index 6fab31d6f..000000000 --- a/server/tests/helpers/markdown.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { mdToOneLinePlainText } from '@server/helpers/markdown' -import { expect } from 'chai' - -describe('Markdown helpers', function () { - - describe('Plain text', function () { - - it('Should convert a list to plain text', function () { - const result = mdToOneLinePlainText(`* list 1 -* list 2 -* list 3`) - - expect(result).to.equal('list 1, list 2, list 3') - }) - - it('Should convert a list with indentation to plain text', function () { - const result = mdToOneLinePlainText(`Hello: - * list 1 - * list 2 - * list 3`) - - expect(result).to.equal('Hello: list 1, list 2, list 3') - }) - - it('Should convert HTML to plain text', function () { - const result = mdToOneLinePlainText(`**Hello** coucou`) - - expect(result).to.equal('Hello coucou') - }) - - it('Should convert tags to plain text', function () { - const result = mdToOneLinePlainText(`#déconversion\n#newage\n#histoire`) - - expect(result).to.equal('#déconversion #newage #histoire') - }) - }) -}) diff --git a/server/tests/helpers/request.ts b/server/tests/helpers/request.ts deleted file mode 100644 index 363237df5..000000000 --- a/server/tests/helpers/request.ts +++ /dev/null @@ -1,62 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { pathExists, remove } from 'fs-extra' -import { join } from 'path' -import { root, wait } from '@shared/core-utils' -import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' -import { FIXTURE_URLS, Mock429 } from '../shared' - -describe('Request helpers', function () { - const destPath1 = join(root(), 'test-output-1.txt') - const destPath2 = join(root(), 'test-output-2.txt') - - it('Should throw an error when the bytes limit is exceeded for request', async function () { - try { - await doRequest(FIXTURE_URLS.file4K, { bodyKBLimit: 3 }) - } catch { - return - } - - throw new Error('No error thrown by do request') - }) - - it('Should throw an error when the bytes limit is exceeded for request and save file', async function () { - try { - await doRequestAndSaveToFile(FIXTURE_URLS.file4K, destPath1, { bodyKBLimit: 3 }) - } catch { - - await wait(500) - expect(await pathExists(destPath1)).to.be.false - return - } - - throw new Error('No error thrown by do request and save to file') - }) - - it('Should correctly retry on 429 error', async function () { - this.timeout(25000) - - const mock = new Mock429() - const port = await mock.initialize() - - const before = new Date().getTime() - await doRequest('http://127.0.0.1:' + port) - - expect(new Date().getTime() - before).to.be.greaterThan(2000) - - await mock.terminate() - }) - - it('Should succeed if the file is below the limit', async function () { - await doRequest(FIXTURE_URLS.file4K, { bodyKBLimit: 5 }) - await doRequestAndSaveToFile(FIXTURE_URLS.file4K, destPath2, { bodyKBLimit: 5 }) - - expect(await pathExists(destPath2)).to.be.true - }) - - after(async function () { - await remove(destPath1) - await remove(destPath2) - }) -}) diff --git a/server/tests/helpers/validator.ts b/server/tests/helpers/validator.ts deleted file mode 100644 index f40a3aaae..000000000 --- a/server/tests/helpers/validator.ts +++ /dev/null @@ -1,32 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { isPluginStableOrUnstableVersionValid, isPluginStableVersionValid } from '@server/helpers/custom-validators/plugins' - -describe('Validators', function () { - - it('Should correctly check stable plugin versions', async function () { - expect(isPluginStableVersionValid('3.4.0')).to.be.true - expect(isPluginStableVersionValid('0.4.0')).to.be.true - expect(isPluginStableVersionValid('0.1.0')).to.be.true - - expect(isPluginStableVersionValid('0.1.0-beta-1')).to.be.false - expect(isPluginStableVersionValid('hello')).to.be.false - expect(isPluginStableVersionValid('0.x.a')).to.be.false - }) - - it('Should correctly check unstable plugin versions', async function () { - expect(isPluginStableOrUnstableVersionValid('3.4.0')).to.be.true - expect(isPluginStableOrUnstableVersionValid('0.4.0')).to.be.true - expect(isPluginStableOrUnstableVersionValid('0.1.0')).to.be.true - - expect(isPluginStableOrUnstableVersionValid('0.1.0-beta.1')).to.be.true - expect(isPluginStableOrUnstableVersionValid('0.1.0-alpha.45')).to.be.true - expect(isPluginStableOrUnstableVersionValid('0.1.0-rc.45')).to.be.true - - expect(isPluginStableOrUnstableVersionValid('hello')).to.be.false - expect(isPluginStableOrUnstableVersionValid('0.x.a')).to.be.false - expect(isPluginStableOrUnstableVersionValid('0.1.0-rc-45')).to.be.false - expect(isPluginStableOrUnstableVersionValid('0.1.0-rc.45d')).to.be.false - }) -}) diff --git a/server/tests/helpers/version.ts b/server/tests/helpers/version.ts deleted file mode 100644 index 2a90efba3..000000000 --- a/server/tests/helpers/version.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { compareSemVer } from '@shared/core-utils' - -describe('Version', function () { - - it('Should correctly compare two stable versions', async function () { - expect(compareSemVer('3.4.0', '3.5.0')).to.be.below(0) - expect(compareSemVer('3.5.0', '3.4.0')).to.be.above(0) - - expect(compareSemVer('3.4.0', '4.1.0')).to.be.below(0) - expect(compareSemVer('4.1.0', '3.4.0')).to.be.above(0) - - expect(compareSemVer('3.4.0', '3.4.1')).to.be.below(0) - expect(compareSemVer('3.4.1', '3.4.0')).to.be.above(0) - }) - - it('Should correctly compare two unstable version', async function () { - expect(compareSemVer('3.4.0-alpha', '3.4.0-beta.1')).to.be.below(0) - expect(compareSemVer('3.4.0-alpha.1', '3.4.0-beta.1')).to.be.below(0) - expect(compareSemVer('3.4.0-beta.1', '3.4.0-beta.2')).to.be.below(0) - expect(compareSemVer('3.4.0-beta.1', '3.5.0-alpha.1')).to.be.below(0) - - expect(compareSemVer('3.4.0-alpha.1', '3.4.0-nightly.4')).to.be.below(0) - expect(compareSemVer('3.4.0-nightly.3', '3.4.0-nightly.4')).to.be.below(0) - expect(compareSemVer('3.3.0-nightly.5', '3.4.0-nightly.4')).to.be.below(0) - }) - - it('Should correctly compare a stable and unstable versions', async function () { - expect(compareSemVer('3.4.0', '3.4.1-beta.1')).to.be.below(0) - expect(compareSemVer('3.4.0-beta.1', '3.4.0-beta.2')).to.be.below(0) - expect(compareSemVer('3.4.0-beta.1', '3.4.0')).to.be.below(0) - expect(compareSemVer('3.4.0-nightly.4', '3.4.0')).to.be.below(0) - }) -}) diff --git a/server/tests/index.ts b/server/tests/index.ts deleted file mode 100644 index 4ec1ebe67..000000000 --- a/server/tests/index.ts +++ /dev/null @@ -1,10 +0,0 @@ -// Order of the tests we want to execute -import './client' -import './misc-endpoints' -import './feeds/' -import './cli/' -import './api/' -import './peertube-runner/' -import './plugins/' -import './helpers/' -import './lib/' diff --git a/server/tests/lib/index.ts b/server/tests/lib/index.ts deleted file mode 100644 index a40df35fd..000000000 --- a/server/tests/lib/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './video-constant-registry-factory' diff --git a/server/tests/lib/video-constant-registry-factory.ts b/server/tests/lib/video-constant-registry-factory.ts deleted file mode 100644 index c3480dc12..000000000 --- a/server/tests/lib/video-constant-registry-factory.ts +++ /dev/null @@ -1,154 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions */ -import { expect } from 'chai' -import { VideoConstantManagerFactory } from '@server/lib/plugins/video-constant-manager-factory' -import { - VIDEO_CATEGORIES, - VIDEO_LANGUAGES, - VIDEO_LICENCES, - VIDEO_PLAYLIST_PRIVACIES, - VIDEO_PRIVACIES -} from '@server/initializers/constants' -import { - VideoPlaylistPrivacy, - VideoPrivacy -} from '@shared/models' - -describe('VideoConstantManagerFactory', function () { - const factory = new VideoConstantManagerFactory('peertube-plugin-constants') - - afterEach(() => { - factory.resetVideoConstants('peertube-plugin-constants') - }) - - describe('VideoCategoryManager', () => { - const videoCategoryManager = factory.createVideoConstantManager('category') - - it('Should be able to list all video category constants', () => { - const constants = videoCategoryManager.getConstants() - expect(constants).to.deep.equal(VIDEO_CATEGORIES) - }) - - it('Should be able to delete a video category constant', () => { - const successfullyDeleted = videoCategoryManager.deleteConstant(1) - expect(successfullyDeleted).to.be.true - expect(videoCategoryManager.getConstantValue(1)).to.be.undefined - }) - - it('Should be able to add a video category constant', () => { - const successfullyAdded = videoCategoryManager.addConstant(42, 'The meaning of life') - expect(successfullyAdded).to.be.true - expect(videoCategoryManager.getConstantValue(42)).to.equal('The meaning of life') - }) - - it('Should be able to reset video category constants', () => { - videoCategoryManager.deleteConstant(1) - videoCategoryManager.resetConstants() - expect(videoCategoryManager.getConstantValue(1)).not.be.undefined - }) - }) - - describe('VideoLicenceManager', () => { - const videoLicenceManager = factory.createVideoConstantManager('licence') - it('Should be able to list all video licence constants', () => { - const constants = videoLicenceManager.getConstants() - expect(constants).to.deep.equal(VIDEO_LICENCES) - }) - - it('Should be able to delete a video licence constant', () => { - const successfullyDeleted = videoLicenceManager.deleteConstant(1) - expect(successfullyDeleted).to.be.true - expect(videoLicenceManager.getConstantValue(1)).to.be.undefined - }) - - it('Should be able to add a video licence constant', () => { - const successfullyAdded = videoLicenceManager.addConstant(42, 'European Union Public Licence') - expect(successfullyAdded).to.be.true - expect(videoLicenceManager.getConstantValue(42 as any)).to.equal('European Union Public Licence') - }) - - it('Should be able to reset video licence constants', () => { - videoLicenceManager.deleteConstant(1) - videoLicenceManager.resetConstants() - expect(videoLicenceManager.getConstantValue(1)).not.be.undefined - }) - }) - - describe('PlaylistPrivacyManager', () => { - const playlistPrivacyManager = factory.createVideoConstantManager('playlistPrivacy') - it('Should be able to list all video playlist privacy constants', () => { - const constants = playlistPrivacyManager.getConstants() - expect(constants).to.deep.equal(VIDEO_PLAYLIST_PRIVACIES) - }) - - it('Should be able to delete a video playlist privacy constant', () => { - const successfullyDeleted = playlistPrivacyManager.deleteConstant(1) - expect(successfullyDeleted).to.be.true - expect(playlistPrivacyManager.getConstantValue(1)).to.be.undefined - }) - - it('Should be able to add a video playlist privacy constant', () => { - const successfullyAdded = playlistPrivacyManager.addConstant(42 as any, 'Friends only') - expect(successfullyAdded).to.be.true - expect(playlistPrivacyManager.getConstantValue(42 as any)).to.equal('Friends only') - }) - - it('Should be able to reset video playlist privacy constants', () => { - playlistPrivacyManager.deleteConstant(1) - playlistPrivacyManager.resetConstants() - expect(playlistPrivacyManager.getConstantValue(1)).not.be.undefined - }) - }) - - describe('VideoPrivacyManager', () => { - const videoPrivacyManager = factory.createVideoConstantManager('privacy') - it('Should be able to list all video privacy constants', () => { - const constants = videoPrivacyManager.getConstants() - expect(constants).to.deep.equal(VIDEO_PRIVACIES) - }) - - it('Should be able to delete a video privacy constant', () => { - const successfullyDeleted = videoPrivacyManager.deleteConstant(1) - expect(successfullyDeleted).to.be.true - expect(videoPrivacyManager.getConstantValue(1)).to.be.undefined - }) - - it('Should be able to add a video privacy constant', () => { - const successfullyAdded = videoPrivacyManager.addConstant(42 as any, 'Friends only') - expect(successfullyAdded).to.be.true - expect(videoPrivacyManager.getConstantValue(42 as any)).to.equal('Friends only') - }) - - it('Should be able to reset video privacy constants', () => { - videoPrivacyManager.deleteConstant(1) - videoPrivacyManager.resetConstants() - expect(videoPrivacyManager.getConstantValue(1)).not.be.undefined - }) - }) - - describe('VideoLanguageManager', () => { - const videoLanguageManager = factory.createVideoConstantManager('language') - it('Should be able to list all video language constants', () => { - const constants = videoLanguageManager.getConstants() - expect(constants).to.deep.equal(VIDEO_LANGUAGES) - }) - - it('Should be able to add a video language constant', () => { - const successfullyAdded = videoLanguageManager.addConstant('fr', 'Fr occitan') - expect(successfullyAdded).to.be.true - expect(videoLanguageManager.getConstantValue('fr')).to.equal('Fr occitan') - }) - - it('Should be able to delete a video language constant', () => { - videoLanguageManager.addConstant('fr', 'Fr occitan') - const successfullyDeleted = videoLanguageManager.deleteConstant('fr') - expect(successfullyDeleted).to.be.true - expect(videoLanguageManager.getConstantValue('fr')).to.be.undefined - }) - - it('Should be able to reset video language constants', () => { - videoLanguageManager.addConstant('fr', 'Fr occitan') - videoLanguageManager.resetConstants() - expect(videoLanguageManager.getConstantValue('fr')).to.be.undefined - }) - }) -}) diff --git a/server/tests/misc-endpoints.ts b/server/tests/misc-endpoints.ts deleted file mode 100644 index f9cf2b717..000000000 --- a/server/tests/misc-endpoints.ts +++ /dev/null @@ -1,237 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { writeJson } from 'fs-extra' -import { join } from 'path' -import { HttpStatusCode, VideoPrivacy } from '@shared/models' -import { cleanupTests, createSingleServer, makeGetRequest, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands' -import { expectLogDoesNotContain } from './shared' - -describe('Test misc endpoints', function () { - let server: PeerTubeServer - let wellKnownPath: string - - before(async function () { - this.timeout(120000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - - wellKnownPath = server.getDirectoryPath('well-known') - }) - - describe('Test a well known endpoints', function () { - - it('Should get security.txt', async function () { - const res = await makeGetRequest({ - url: server.url, - path: '/.well-known/security.txt', - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(res.text).to.contain('security issue') - }) - - it('Should get nodeinfo', async function () { - const res = await makeGetRequest({ - url: server.url, - path: '/.well-known/nodeinfo', - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(res.body.links).to.be.an('array') - expect(res.body.links).to.have.lengthOf(1) - expect(res.body.links[0].rel).to.equal('http://nodeinfo.diaspora.software/ns/schema/2.0') - }) - - it('Should get dnt policy text', async function () { - const res = await makeGetRequest({ - url: server.url, - path: '/.well-known/dnt-policy.txt', - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(res.text).to.contain('http://www.w3.org/TR/tracking-dnt') - }) - - it('Should get dnt policy', async function () { - const res = await makeGetRequest({ - url: server.url, - path: '/.well-known/dnt', - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(res.body.tracking).to.equal('N') - }) - - it('Should get change-password location', async function () { - const res = await makeGetRequest({ - url: server.url, - path: '/.well-known/change-password', - expectedStatus: HttpStatusCode.FOUND_302 - }) - - expect(res.header.location).to.equal('/my-account/settings') - }) - - it('Should test webfinger', async function () { - const resource = 'acct:peertube@' + server.host - const accountUrl = server.url + '/accounts/peertube' - - const res = await makeGetRequest({ - url: server.url, - path: '/.well-known/webfinger?resource=' + resource, - expectedStatus: HttpStatusCode.OK_200 - }) - - const data = res.body - - expect(data.subject).to.equal(resource) - expect(data.aliases).to.contain(accountUrl) - - const self = data.links.find(l => l.rel === 'self') - expect(self).to.exist - expect(self.type).to.equal('application/activity+json') - expect(self.href).to.equal(accountUrl) - - const remoteInteract = data.links.find(l => l.rel === 'http://ostatus.org/schema/1.0/subscribe') - expect(remoteInteract).to.exist - expect(remoteInteract.template).to.equal(server.url + '/remote-interaction?uri={uri}') - }) - - it('Should return 404 for non-existing files in /.well-known', async function () { - await makeGetRequest({ - url: server.url, - path: '/.well-known/non-existing-file', - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should return custom file from /.well-known', async function () { - const filename = 'existing-file.json' - - await writeJson(join(wellKnownPath, filename), { iThink: 'therefore I am' }) - - const { body } = await makeGetRequest({ - url: server.url, - path: '/.well-known/' + filename, - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(body.iThink).to.equal('therefore I am') - }) - }) - - describe('Test classic static endpoints', function () { - - it('Should get robots.txt', async function () { - const res = await makeGetRequest({ - url: server.url, - path: '/robots.txt', - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(res.text).to.contain('User-agent') - }) - - it('Should get security.txt', async function () { - await makeGetRequest({ - url: server.url, - path: '/security.txt', - expectedStatus: HttpStatusCode.MOVED_PERMANENTLY_301 - }) - }) - - it('Should get nodeinfo', async function () { - const res = await makeGetRequest({ - url: server.url, - path: '/nodeinfo/2.0.json', - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(res.body.software.name).to.equal('peertube') - expect(res.body.usage.users.activeMonth).to.equal(1) - expect(res.body.usage.users.activeHalfyear).to.equal(1) - }) - }) - - describe('Test bots endpoints', function () { - - it('Should get the empty sitemap', async function () { - const res = await makeGetRequest({ - url: server.url, - path: '/sitemap.xml', - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"') - expect(res.text).to.contain('' + server.url + '/about/instance') - }) - - it('Should get the empty cached sitemap', async function () { - const res = await makeGetRequest({ - url: server.url, - path: '/sitemap.xml', - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"') - expect(res.text).to.contain('' + server.url + '/about/instance') - }) - - it('Should add videos, channel and accounts and get sitemap', async function () { - this.timeout(35000) - - await server.videos.upload({ attributes: { name: 'video 1', nsfw: false } }) - await server.videos.upload({ attributes: { name: 'video 2', nsfw: false } }) - await server.videos.upload({ attributes: { name: 'video 3', privacy: VideoPrivacy.PRIVATE } }) - - await server.channels.create({ attributes: { name: 'channel1', displayName: 'channel 1' } }) - await server.channels.create({ attributes: { name: 'channel2', displayName: 'channel 2' } }) - - await server.users.create({ username: 'user1', password: 'password' }) - await server.users.create({ username: 'user2', password: 'password' }) - - const res = await makeGetRequest({ - url: server.url, - path: '/sitemap.xml?t=1', // avoid using cache - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(res.text).to.contain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"') - expect(res.text).to.contain('' + server.url + '/about/instance') - - expect(res.text).to.contain('video 1') - expect(res.text).to.contain('video 2') - expect(res.text).to.not.contain('video 3') - - expect(res.text).to.contain('' + server.url + '/video-channels/channel1') - expect(res.text).to.contain('' + server.url + '/video-channels/channel2') - - expect(res.text).to.contain('' + server.url + '/accounts/user1') - expect(res.text).to.contain('' + server.url + '/accounts/user2') - }) - - it('Should not fail with big title/description videos', async function () { - const name = 'v'.repeat(115) - - await server.videos.upload({ attributes: { name, description: 'd'.repeat(2500), nsfw: false } }) - - const res = await makeGetRequest({ - url: server.url, - path: '/sitemap.xml?t=2', // avoid using cache - expectedStatus: HttpStatusCode.OK_200 - }) - - await expectLogDoesNotContain(server, 'Warning in sitemap generation') - await expectLogDoesNotContain(server, 'Error in sitemap generation') - - expect(res.text).to.contain(`${'v'.repeat(97)}...`) - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/peertube-runner/client-cli.ts b/server/tests/peertube-runner/client-cli.ts deleted file mode 100644 index 5cbdc4e77..000000000 --- a/server/tests/peertube-runner/client-cli.ts +++ /dev/null @@ -1,70 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { PeerTubeRunnerProcess } from '@server/tests/shared' -import { cleanupTests, createSingleServer, PeerTubeServer, setAccessTokensToServers, setDefaultVideoChannel } from '@shared/server-commands' - -describe('Test peertube-runner program client CLI', function () { - let server: PeerTubeServer - let peertubeRunner: PeerTubeRunnerProcess - - before(async function () { - this.timeout(120_000) - - server = await createSingleServer(1) - - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - await server.config.enableRemoteTranscoding() - - peertubeRunner = new PeerTubeRunnerProcess(server) - await peertubeRunner.runServer() - }) - - it('Should not have PeerTube instance listed', async function () { - const data = await peertubeRunner.listRegisteredPeerTubeInstances() - - expect(data).to.not.contain(server.url) - }) - - it('Should register a new PeerTube instance', async function () { - const registrationToken = await server.runnerRegistrationTokens.getFirstRegistrationToken() - - await peertubeRunner.registerPeerTubeInstance({ - registrationToken, - runnerName: 'my super runner', - runnerDescription: 'super description' - }) - }) - - it('Should list this new PeerTube instance', async function () { - const data = await peertubeRunner.listRegisteredPeerTubeInstances() - - expect(data).to.contain(server.url) - expect(data).to.contain('my super runner') - expect(data).to.contain('super description') - }) - - it('Should still have the configuration after a restart', async function () { - peertubeRunner.kill() - - await peertubeRunner.runServer() - }) - - it('Should unregister the PeerTube instance', async function () { - await peertubeRunner.unregisterPeerTubeInstance({ runnerName: 'my super runner' }) - }) - - it('Should not have PeerTube instance listed', async function () { - const data = await peertubeRunner.listRegisteredPeerTubeInstances() - - expect(data).to.not.contain(server.url) - }) - - after(async function () { - peertubeRunner.kill() - - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/peertube-runner/index.ts b/server/tests/peertube-runner/index.ts deleted file mode 100644 index 470316417..000000000 --- a/server/tests/peertube-runner/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './client-cli' -export * from './live-transcoding' -export * from './studio-transcoding' -export * from './vod-transcoding' diff --git a/server/tests/peertube-runner/live-transcoding.ts b/server/tests/peertube-runner/live-transcoding.ts deleted file mode 100644 index 41b01f8d5..000000000 --- a/server/tests/peertube-runner/live-transcoding.ts +++ /dev/null @@ -1,201 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { expect } from 'chai' -import { - checkPeerTubeRunnerCacheIsEmpty, - expectStartWith, - PeerTubeRunnerProcess, - SQLCommand, - testLiveVideoResolutions -} from '@server/tests/shared' -import { areMockObjectStorageTestsDisabled, wait } from '@shared/core-utils' -import { HttpStatusCode, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - findExternalSavedVideo, - makeRawRequest, - ObjectStorageCommand, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - stopFfmpeg, - waitJobs, - waitUntilLivePublishedOnAllServers, - waitUntilLiveWaitingOnAllServers -} from '@shared/server-commands' - -describe('Test Live transcoding in peertube-runner program', function () { - let servers: PeerTubeServer[] = [] - let peertubeRunner: PeerTubeRunnerProcess - let sqlCommandServer1: SQLCommand - - function runSuite (options: { - objectStorage?: ObjectStorageCommand - } = {}) { - const { objectStorage } = options - - it('Should enable transcoding without additional resolutions', async function () { - this.timeout(120000) - - const { video } = await servers[0].live.quickCreate({ permanentLive: true, saveReplay: false, privacy: VideoPrivacy.PUBLIC }) - - const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: video.uuid }) - await waitUntilLivePublishedOnAllServers(servers, video.uuid) - await waitJobs(servers) - - await testLiveVideoResolutions({ - originServer: servers[0], - sqlCommand: sqlCommandServer1, - servers, - liveVideoId: video.uuid, - resolutions: [ 720, 480, 360, 240, 144 ], - objectStorage, - transcoded: true - }) - - await stopFfmpeg(ffmpegCommand) - - await waitUntilLiveWaitingOnAllServers(servers, video.uuid) - await servers[0].videos.remove({ id: video.id }) - }) - - it('Should transcode audio only RTMP stream', async function () { - this.timeout(120000) - - const { video } = await servers[0].live.quickCreate({ permanentLive: true, saveReplay: false, privacy: VideoPrivacy.UNLISTED }) - - const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: video.uuid, fixtureName: 'video_short_no_audio.mp4' }) - await waitUntilLivePublishedOnAllServers(servers, video.uuid) - await waitJobs(servers) - - await stopFfmpeg(ffmpegCommand) - - await waitUntilLiveWaitingOnAllServers(servers, video.uuid) - await servers[0].videos.remove({ id: video.id }) - }) - - it('Should save a replay', async function () { - this.timeout(240000) - - const { video } = await servers[0].live.quickCreate({ permanentLive: true, saveReplay: true }) - - const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: video.uuid }) - await waitUntilLivePublishedOnAllServers(servers, video.uuid) - - await testLiveVideoResolutions({ - originServer: servers[0], - sqlCommand: sqlCommandServer1, - servers, - liveVideoId: video.uuid, - resolutions: [ 720, 480, 360, 240, 144 ], - objectStorage, - transcoded: true - }) - - await stopFfmpeg(ffmpegCommand) - - await waitUntilLiveWaitingOnAllServers(servers, video.uuid) - await waitJobs(servers) - - const session = await servers[0].live.findLatestSession({ videoId: video.uuid }) - expect(session.endingProcessed).to.be.true - expect(session.endDate).to.exist - expect(session.saveReplay).to.be.true - - const videoLiveDetails = await servers[0].videos.get({ id: video.uuid }) - const replay = await findExternalSavedVideo(servers[0], videoLiveDetails) - - for (const server of servers) { - const video = await server.videos.get({ id: replay.uuid }) - - expect(video.files).to.have.lengthOf(0) - expect(video.streamingPlaylists).to.have.lengthOf(1) - - const files = video.streamingPlaylists[0].files - expect(files).to.have.lengthOf(5) - - for (const file of files) { - if (objectStorage) { - expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl()) - } - - await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) - } - } - }) - } - - before(async function () { - this.timeout(120_000) - - servers = await createMultipleServers(2) - - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - - await doubleFollow(servers[0], servers[1]) - - sqlCommandServer1 = new SQLCommand(servers[0]) - - await servers[0].config.enableRemoteTranscoding() - await servers[0].config.enableTranscoding({ hls: true, webVideo: true, with0p: true }) - await servers[0].config.enableLive({ allowReplay: true, resolutions: 'max', transcoding: true }) - - const registrationToken = await servers[0].runnerRegistrationTokens.getFirstRegistrationToken() - - peertubeRunner = new PeerTubeRunnerProcess(servers[0]) - await peertubeRunner.runServer() - await peertubeRunner.registerPeerTubeInstance({ registrationToken, runnerName: 'runner' }) - }) - - describe('With lives on local filesystem storage', function () { - - before(async function () { - await servers[0].config.enableTranscoding({ webVideo: true, hls: false, with0p: true }) - }) - - runSuite() - }) - - describe('With lives on object storage', function () { - if (areMockObjectStorageTestsDisabled()) return - - const objectStorage = new ObjectStorageCommand() - - before(async function () { - await objectStorage.prepareDefaultMockBuckets() - - await servers[0].kill() - - await servers[0].run(objectStorage.getDefaultMockConfig()) - - // Wait for peertube runner socket reconnection - await wait(1500) - }) - - runSuite({ objectStorage }) - - after(async function () { - await objectStorage.cleanupMock() - }) - }) - - describe('Check cleanup', function () { - - it('Should have an empty cache directory', async function () { - await checkPeerTubeRunnerCacheIsEmpty(peertubeRunner) - }) - }) - - after(async function () { - if (peertubeRunner) { - await peertubeRunner.unregisterPeerTubeInstance({ runnerName: 'runner' }) - peertubeRunner.kill() - } - - if (sqlCommandServer1) await sqlCommandServer1.cleanup() - - await cleanupTests(servers) - }) -}) diff --git a/server/tests/peertube-runner/studio-transcoding.ts b/server/tests/peertube-runner/studio-transcoding.ts deleted file mode 100644 index 56bfef897..000000000 --- a/server/tests/peertube-runner/studio-transcoding.ts +++ /dev/null @@ -1,124 +0,0 @@ - -import { expect } from 'chai' -import { checkPeerTubeRunnerCacheIsEmpty, checkVideoDuration, expectStartWith, PeerTubeRunnerProcess } from '@server/tests/shared' -import { areMockObjectStorageTestsDisabled, getAllFiles, wait } from '@shared/core-utils' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - ObjectStorageCommand, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - VideoStudioCommand, - waitJobs -} from '@shared/server-commands' - -describe('Test studio transcoding in peertube-runner program', function () { - let servers: PeerTubeServer[] = [] - let peertubeRunner: PeerTubeRunnerProcess - - function runSuite (options: { - objectStorage?: ObjectStorageCommand - } = {}) { - const { objectStorage } = options - - it('Should run a complex studio transcoding', async function () { - this.timeout(120000) - - const { uuid } = await servers[0].videos.quickUpload({ name: 'mp4', fixture: 'video_short.mp4' }) - await waitJobs(servers) - - const video = await servers[0].videos.get({ id: uuid }) - const oldFileUrls = getAllFiles(video).map(f => f.fileUrl) - - await servers[0].videoStudio.createEditionTasks({ videoId: uuid, tasks: VideoStudioCommand.getComplexTask() }) - await waitJobs(servers, { runnerJobs: true }) - - for (const server of servers) { - const video = await server.videos.get({ id: uuid }) - const files = getAllFiles(video) - - for (const f of files) { - expect(oldFileUrls).to.not.include(f.fileUrl) - } - - if (objectStorage) { - for (const webVideoFile of video.files) { - expectStartWith(webVideoFile.fileUrl, objectStorage.getMockWebVideosBaseUrl()) - } - - for (const hlsFile of video.streamingPlaylists[0].files) { - expectStartWith(hlsFile.fileUrl, objectStorage.getMockPlaylistBaseUrl()) - } - } - - await checkVideoDuration(server, uuid, 9) - } - }) - } - - before(async function () { - this.timeout(120_000) - - servers = await createMultipleServers(2) - - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - - await doubleFollow(servers[0], servers[1]) - - await servers[0].config.enableTranscoding({ hls: true, webVideo: true }) - await servers[0].config.enableStudio() - await servers[0].config.enableRemoteStudio() - - const registrationToken = await servers[0].runnerRegistrationTokens.getFirstRegistrationToken() - - peertubeRunner = new PeerTubeRunnerProcess(servers[0]) - await peertubeRunner.runServer() - await peertubeRunner.registerPeerTubeInstance({ registrationToken, runnerName: 'runner' }) - }) - - describe('With videos on local filesystem storage', function () { - runSuite() - }) - - describe('With videos on object storage', function () { - if (areMockObjectStorageTestsDisabled()) return - - const objectStorage = new ObjectStorageCommand() - - before(async function () { - await objectStorage.prepareDefaultMockBuckets() - - await servers[0].kill() - - await servers[0].run(objectStorage.getDefaultMockConfig()) - - // Wait for peertube runner socket reconnection - await wait(1500) - }) - - runSuite({ objectStorage }) - - after(async function () { - await objectStorage.cleanupMock() - }) - }) - - describe('Check cleanup', function () { - - it('Should have an empty cache directory', async function () { - await checkPeerTubeRunnerCacheIsEmpty(peertubeRunner) - }) - }) - - after(async function () { - if (peertubeRunner) { - await peertubeRunner.unregisterPeerTubeInstance({ runnerName: 'runner' }) - peertubeRunner.kill() - } - - await cleanupTests(servers) - }) -}) diff --git a/server/tests/peertube-runner/vod-transcoding.ts b/server/tests/peertube-runner/vod-transcoding.ts deleted file mode 100644 index b3b62e5e0..000000000 --- a/server/tests/peertube-runner/vod-transcoding.ts +++ /dev/null @@ -1,350 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { expect } from 'chai' -import { - checkPeerTubeRunnerCacheIsEmpty, - completeCheckHlsPlaylist, - completeWebVideoFilesCheck, - PeerTubeRunnerProcess -} from '@server/tests/shared' -import { areMockObjectStorageTestsDisabled, getAllFiles, wait } from '@shared/core-utils' -import { VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - ObjectStorageCommand, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - waitJobs -} from '@shared/server-commands' - -describe('Test VOD transcoding in peertube-runner program', function () { - let servers: PeerTubeServer[] = [] - let peertubeRunner: PeerTubeRunnerProcess - - function runSuite (options: { - webVideoEnabled: boolean - hlsEnabled: boolean - objectStorage?: ObjectStorageCommand - }) { - const { webVideoEnabled, hlsEnabled, objectStorage } = options - - const objectStorageBaseUrlWebVideo = objectStorage - ? objectStorage.getMockWebVideosBaseUrl() - : undefined - - const objectStorageBaseUrlHLS = objectStorage - ? objectStorage.getMockPlaylistBaseUrl() - : undefined - - it('Should upload a classic video mp4 and transcode it', async function () { - this.timeout(120000) - - const { uuid } = await servers[0].videos.quickUpload({ name: 'mp4', fixture: 'video_short.mp4' }) - - await waitJobs(servers, { runnerJobs: true }) - - for (const server of servers) { - if (webVideoEnabled) { - await completeWebVideoFilesCheck({ - server, - originServer: servers[0], - fixture: 'video_short.mp4', - videoUUID: uuid, - objectStorageBaseUrl: objectStorageBaseUrlWebVideo, - files: [ - { resolution: 0 }, - { resolution: 144 }, - { resolution: 240 }, - { resolution: 360 }, - { resolution: 480 }, - { resolution: 720 } - ] - }) - } - - if (hlsEnabled) { - await completeCheckHlsPlaylist({ - hlsOnly: !webVideoEnabled, - servers, - videoUUID: uuid, - objectStorageBaseUrl: objectStorageBaseUrlHLS, - resolutions: [ 720, 480, 360, 240, 144, 0 ] - }) - } - } - }) - - it('Should upload a webm video and transcode it', async function () { - this.timeout(120000) - - const { uuid } = await servers[0].videos.quickUpload({ name: 'mp4', fixture: 'video_short.webm' }) - - await waitJobs(servers, { runnerJobs: true }) - - for (const server of servers) { - if (webVideoEnabled) { - await completeWebVideoFilesCheck({ - server, - originServer: servers[0], - fixture: 'video_short.webm', - videoUUID: uuid, - objectStorageBaseUrl: objectStorageBaseUrlWebVideo, - files: [ - { resolution: 0 }, - { resolution: 144 }, - { resolution: 240 }, - { resolution: 360 }, - { resolution: 480 }, - { resolution: 720 } - ] - }) - } - - if (hlsEnabled) { - await completeCheckHlsPlaylist({ - hlsOnly: !webVideoEnabled, - servers, - videoUUID: uuid, - objectStorageBaseUrl: objectStorageBaseUrlHLS, - resolutions: [ 720, 480, 360, 240, 144, 0 ] - }) - } - } - }) - - it('Should upload an audio only video and transcode it', async function () { - this.timeout(120000) - - const attributes = { name: 'audio_without_preview', fixture: 'sample.ogg' } - const { uuid } = await servers[0].videos.upload({ attributes, mode: 'resumable' }) - - await waitJobs(servers, { runnerJobs: true }) - - for (const server of servers) { - if (webVideoEnabled) { - await completeWebVideoFilesCheck({ - server, - originServer: servers[0], - fixture: 'sample.ogg', - videoUUID: uuid, - objectStorageBaseUrl: objectStorageBaseUrlWebVideo, - files: [ - { resolution: 0 }, - { resolution: 144 }, - { resolution: 240 }, - { resolution: 360 }, - { resolution: 480 } - ] - }) - } - - if (hlsEnabled) { - await completeCheckHlsPlaylist({ - hlsOnly: !webVideoEnabled, - servers, - videoUUID: uuid, - objectStorageBaseUrl: objectStorageBaseUrlHLS, - resolutions: [ 480, 360, 240, 144, 0 ] - }) - } - } - }) - - it('Should upload a private video and transcode it', async function () { - this.timeout(120000) - - const { uuid } = await servers[0].videos.quickUpload({ name: 'mp4', fixture: 'video_short.mp4', privacy: VideoPrivacy.PRIVATE }) - - await waitJobs(servers, { runnerJobs: true }) - - if (webVideoEnabled) { - await completeWebVideoFilesCheck({ - server: servers[0], - originServer: servers[0], - fixture: 'video_short.mp4', - videoUUID: uuid, - objectStorageBaseUrl: objectStorageBaseUrlWebVideo, - files: [ - { resolution: 0 }, - { resolution: 144 }, - { resolution: 240 }, - { resolution: 360 }, - { resolution: 480 }, - { resolution: 720 } - ] - }) - } - - if (hlsEnabled) { - await completeCheckHlsPlaylist({ - hlsOnly: !webVideoEnabled, - servers: [ servers[0] ], - videoUUID: uuid, - objectStorageBaseUrl: objectStorageBaseUrlHLS, - resolutions: [ 720, 480, 360, 240, 144, 0 ] - }) - } - }) - - it('Should transcode videos on manual run', async function () { - this.timeout(120000) - - await servers[0].config.disableTranscoding() - - const { uuid } = await servers[0].videos.quickUpload({ name: 'manual transcoding', fixture: 'video_short.mp4' }) - await waitJobs(servers, { runnerJobs: true }) - - { - const video = await servers[0].videos.get({ id: uuid }) - expect(getAllFiles(video)).to.have.lengthOf(1) - } - - await servers[0].config.enableTranscoding({ hls: true, webVideo: true, with0p: true }) - - await servers[0].videos.runTranscoding({ transcodingType: 'web-video', videoId: uuid }) - await waitJobs(servers, { runnerJobs: true }) - - await completeWebVideoFilesCheck({ - server: servers[0], - originServer: servers[0], - fixture: 'video_short.mp4', - videoUUID: uuid, - objectStorageBaseUrl: objectStorageBaseUrlWebVideo, - files: [ - { resolution: 0 }, - { resolution: 144 }, - { resolution: 240 }, - { resolution: 360 }, - { resolution: 480 }, - { resolution: 720 } - ] - }) - - await servers[0].videos.runTranscoding({ transcodingType: 'hls', videoId: uuid }) - await waitJobs(servers, { runnerJobs: true }) - - await completeCheckHlsPlaylist({ - hlsOnly: false, - servers: [ servers[0] ], - videoUUID: uuid, - objectStorageBaseUrl: objectStorageBaseUrlHLS, - resolutions: [ 720, 480, 360, 240, 144, 0 ] - }) - }) - } - - before(async function () { - this.timeout(120_000) - - servers = await createMultipleServers(2) - - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - - await doubleFollow(servers[0], servers[1]) - - await servers[0].config.enableRemoteTranscoding() - - const registrationToken = await servers[0].runnerRegistrationTokens.getFirstRegistrationToken() - - peertubeRunner = new PeerTubeRunnerProcess(servers[0]) - await peertubeRunner.runServer() - await peertubeRunner.registerPeerTubeInstance({ registrationToken, runnerName: 'runner' }) - }) - - describe('With videos on local filesystem storage', function () { - - describe('Web video only enabled', function () { - - before(async function () { - await servers[0].config.enableTranscoding({ webVideo: true, hls: false, with0p: true }) - }) - - runSuite({ webVideoEnabled: true, hlsEnabled: false }) - }) - - describe('HLS videos only enabled', function () { - - before(async function () { - await servers[0].config.enableTranscoding({ webVideo: false, hls: true, with0p: true }) - }) - - runSuite({ webVideoEnabled: false, hlsEnabled: true }) - }) - - describe('Web video & HLS enabled', function () { - - before(async function () { - await servers[0].config.enableTranscoding({ hls: true, webVideo: true, with0p: true }) - }) - - runSuite({ webVideoEnabled: true, hlsEnabled: true }) - }) - }) - - describe('With videos on object storage', function () { - if (areMockObjectStorageTestsDisabled()) return - - const objectStorage = new ObjectStorageCommand() - - before(async function () { - await objectStorage.prepareDefaultMockBuckets() - - await servers[0].kill() - - await servers[0].run(objectStorage.getDefaultMockConfig()) - - // Wait for peertube runner socket reconnection - await wait(1500) - }) - - describe('Web video only enabled', function () { - - before(async function () { - await servers[0].config.enableTranscoding({ webVideo: true, hls: false, with0p: true }) - }) - - runSuite({ webVideoEnabled: true, hlsEnabled: false, objectStorage }) - }) - - describe('HLS videos only enabled', function () { - - before(async function () { - await servers[0].config.enableTranscoding({ webVideo: false, hls: true, with0p: true }) - }) - - runSuite({ webVideoEnabled: false, hlsEnabled: true, objectStorage }) - }) - - describe('Web video & HLS enabled', function () { - - before(async function () { - await servers[0].config.enableTranscoding({ hls: true, webVideo: true, with0p: true }) - }) - - runSuite({ webVideoEnabled: true, hlsEnabled: true, objectStorage }) - }) - - after(async function () { - await objectStorage.cleanupMock() - }) - }) - - describe('Check cleanup', function () { - - it('Should have an empty cache directory', async function () { - await checkPeerTubeRunnerCacheIsEmpty(peertubeRunner) - }) - }) - - after(async function () { - if (peertubeRunner) { - await peertubeRunner.unregisterPeerTubeInstance({ runnerName: 'runner' }) - peertubeRunner.kill() - } - - await cleanupTests(servers) - }) -}) diff --git a/server/tests/plugins/action-hooks.ts b/server/tests/plugins/action-hooks.ts deleted file mode 100644 index 773be0d76..000000000 --- a/server/tests/plugins/action-hooks.ts +++ /dev/null @@ -1,298 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { ServerHookName, VideoPlaylistPrivacy, VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - killallServers, - PeerTubeServer, - PluginsCommand, - setAccessTokensToServers, - setDefaultVideoChannel, - stopFfmpeg, - waitJobs -} from '@shared/server-commands' - -describe('Test plugin action hooks', function () { - let servers: PeerTubeServer[] - let videoUUID: string - let threadId: number - - function checkHook (hook: ServerHookName, strictCount = true, count = 1) { - return servers[0].servers.waitUntilLog('Run hook ' + hook, count, strictCount) - } - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(2) - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - - await servers[0].plugins.install({ path: PluginsCommand.getPluginTestPath() }) - - await killallServers([ servers[0] ]) - - await servers[0].run({ - live: { - enabled: true - } - }) - - await servers[0].config.enableFileUpdate() - - await doubleFollow(servers[0], servers[1]) - }) - - describe('Application hooks', function () { - it('Should run action:application.listening', async function () { - await checkHook('action:application.listening') - }) - }) - - describe('Videos hooks', function () { - - it('Should run action:api.video.uploaded', async function () { - const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' } }) - videoUUID = uuid - - await checkHook('action:api.video.uploaded') - }) - - it('Should run action:api.video.updated', async function () { - await servers[0].videos.update({ id: videoUUID, attributes: { name: 'video updated' } }) - - await checkHook('action:api.video.updated') - }) - - it('Should run action:api.video.viewed', async function () { - await servers[0].views.simulateView({ id: videoUUID }) - - await checkHook('action:api.video.viewed') - }) - - it('Should run action:api.video.file-updated', async function () { - await servers[0].videos.replaceSourceFile({ videoId: videoUUID, fixture: 'video_short.mp4' }) - - await checkHook('action:api.video.file-updated') - }) - - it('Should run action:api.video.deleted', async function () { - await servers[0].videos.remove({ id: videoUUID }) - - await checkHook('action:api.video.deleted') - }) - - after(async function () { - const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) - videoUUID = uuid - }) - }) - - describe('Video channel hooks', function () { - const channelName = 'my_super_channel' - - it('Should run action:api.video-channel.created', async function () { - await servers[0].channels.create({ attributes: { name: channelName } }) - - await checkHook('action:api.video-channel.created') - }) - - it('Should run action:api.video-channel.updated', async function () { - await servers[0].channels.update({ channelName, attributes: { displayName: 'my display name' } }) - - await checkHook('action:api.video-channel.updated') - }) - - it('Should run action:api.video-channel.deleted', async function () { - await servers[0].channels.delete({ channelName }) - - await checkHook('action:api.video-channel.deleted') - }) - }) - - describe('Live hooks', function () { - - it('Should run action:api.live-video.created', async function () { - const attributes = { - name: 'live', - privacy: VideoPrivacy.PUBLIC, - channelId: servers[0].store.channel.id - } - - await servers[0].live.create({ fields: attributes }) - - await checkHook('action:api.live-video.created') - }) - - it('Should run action:live.video.state.updated', async function () { - this.timeout(60000) - - const attributes = { - name: 'live', - privacy: VideoPrivacy.PUBLIC, - channelId: servers[0].store.channel.id - } - - const { uuid: liveVideoId } = await servers[0].live.create({ fields: attributes }) - const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoId }) - await servers[0].live.waitUntilPublished({ videoId: liveVideoId }) - await waitJobs(servers) - - await checkHook('action:live.video.state.updated', true, 1) - - await stopFfmpeg(ffmpegCommand) - await servers[0].live.waitUntilEnded({ videoId: liveVideoId }) - await waitJobs(servers) - - await checkHook('action:live.video.state.updated', true, 2) - }) - }) - - describe('Comments hooks', function () { - it('Should run action:api.video-thread.created', async function () { - const created = await servers[0].comments.createThread({ videoId: videoUUID, text: 'thread' }) - threadId = created.id - - await checkHook('action:api.video-thread.created') - }) - - it('Should run action:api.video-comment-reply.created', async function () { - await servers[0].comments.addReply({ videoId: videoUUID, toCommentId: threadId, text: 'reply' }) - - await checkHook('action:api.video-comment-reply.created') - }) - - it('Should run action:api.video-comment.deleted', async function () { - await servers[0].comments.delete({ videoId: videoUUID, commentId: threadId }) - - await checkHook('action:api.video-comment.deleted') - }) - }) - - describe('Captions hooks', function () { - it('Should run action:api.video-caption.created', async function () { - await servers[0].captions.add({ videoId: videoUUID, language: 'en', fixture: 'subtitle-good.srt' }) - - await checkHook('action:api.video-caption.created') - }) - - it('Should run action:api.video-caption.deleted', async function () { - await servers[0].captions.delete({ videoId: videoUUID, language: 'en' }) - - await checkHook('action:api.video-caption.deleted') - }) - }) - - describe('Users hooks', function () { - let userId: number - - it('Should run action:api.user.registered', async function () { - await servers[0].registrations.register({ username: 'registered_user' }) - - await checkHook('action:api.user.registered') - }) - - it('Should run action:api.user.created', async function () { - const user = await servers[0].users.create({ username: 'created_user' }) - userId = user.id - - await checkHook('action:api.user.created') - }) - - it('Should run action:api.user.oauth2-got-token', async function () { - await servers[0].login.login({ user: { username: 'created_user' } }) - - await checkHook('action:api.user.oauth2-got-token') - }) - - it('Should run action:api.user.blocked', async function () { - await servers[0].users.banUser({ userId }) - - await checkHook('action:api.user.blocked') - }) - - it('Should run action:api.user.unblocked', async function () { - await servers[0].users.unbanUser({ userId }) - - await checkHook('action:api.user.unblocked') - }) - - it('Should run action:api.user.updated', async function () { - await servers[0].users.update({ userId, videoQuota: 50 }) - - await checkHook('action:api.user.updated') - }) - - it('Should run action:api.user.deleted', async function () { - await servers[0].users.remove({ userId }) - - await checkHook('action:api.user.deleted') - }) - }) - - describe('Playlist hooks', function () { - let playlistId: number - let videoId: number - - before(async function () { - { - const { id } = await servers[0].playlists.create({ - attributes: { - displayName: 'My playlist', - privacy: VideoPlaylistPrivacy.PRIVATE - } - }) - playlistId = id - } - - { - const { id } = await servers[0].videos.upload({ attributes: { name: 'my super name' } }) - videoId = id - } - }) - - it('Should run action:api.video-playlist-element.created', async function () { - await servers[0].playlists.addElement({ playlistId, attributes: { videoId } }) - - await checkHook('action:api.video-playlist-element.created') - }) - }) - - describe('Notification hook', function () { - - it('Should run action:notifier.notification.created', async function () { - await checkHook('action:notifier.notification.created', false) - }) - }) - - describe('Activity Pub hooks', function () { - let videoUUID: string - - it('Should run action:activity-pub.remote-video.created', async function () { - this.timeout(30000) - - const { uuid } = await servers[1].videos.quickUpload({ name: 'remote video' }) - videoUUID = uuid - - await servers[0].servers.waitUntilLog('action:activity-pub.remote-video.created - AP remote video - video remote video') - }) - - it('Should run action:activity-pub.remote-video.updated', async function () { - this.timeout(30000) - - await servers[1].videos.update({ id: videoUUID, attributes: { name: 'remote video updated' } }) - - await servers[0].servers.waitUntilLog( - 'action:activity-pub.remote-video.updated - AP remote video updated - video remote video updated', - 1, - false - ) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/plugins/external-auth.ts b/server/tests/plugins/external-auth.ts deleted file mode 100644 index e4015939a..000000000 --- a/server/tests/plugins/external-auth.ts +++ /dev/null @@ -1,436 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { wait } from '@shared/core-utils' -import { HttpStatusCode, UserAdminFlag, UserRole } from '@shared/models' -import { - cleanupTests, - createSingleServer, - decodeQueryString, - PeerTubeServer, - PluginsCommand, - setAccessTokensToServers -} from '@shared/server-commands' - -async function loginExternal (options: { - server: PeerTubeServer - npmName: string - authName: string - username: string - query?: any - expectedStatus?: HttpStatusCode - expectedStatusStep2?: HttpStatusCode -}) { - const res = await options.server.plugins.getExternalAuth({ - npmName: options.npmName, - npmVersion: '0.0.1', - authName: options.authName, - query: options.query, - expectedStatus: options.expectedStatus || HttpStatusCode.FOUND_302 - }) - - if (res.status !== HttpStatusCode.FOUND_302) return - - const location = res.header.location - const { externalAuthToken } = decodeQueryString(location) - - const resLogin = await options.server.login.loginUsingExternalToken({ - username: options.username, - externalAuthToken: externalAuthToken as string, - expectedStatus: options.expectedStatusStep2 - }) - - return resLogin.body -} - -describe('Test external auth plugins', function () { - let server: PeerTubeServer - - let cyanAccessToken: string - let cyanRefreshToken: string - - let kefkaAccessToken: string - let kefkaRefreshToken: string - let kefkaId: number - - let externalAuthToken: string - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1, { - rates_limit: { - login: { - max: 30 - } - } - }) - - await setAccessTokensToServers([ server ]) - - for (const suffix of [ 'one', 'two', 'three' ]) { - await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-external-auth-' + suffix) }) - } - }) - - it('Should display the correct configuration', async function () { - const config = await server.config.getConfig() - - const auths = config.plugin.registeredExternalAuths - expect(auths).to.have.lengthOf(9) - - const auth2 = auths.find((a) => a.authName === 'external-auth-2') - expect(auth2).to.exist - expect(auth2.authDisplayName).to.equal('External Auth 2') - expect(auth2.npmName).to.equal('peertube-plugin-test-external-auth-one') - }) - - it('Should redirect for a Cyan login', async function () { - const res = await server.plugins.getExternalAuth({ - npmName: 'test-external-auth-one', - npmVersion: '0.0.1', - authName: 'external-auth-1', - query: { - username: 'cyan' - }, - expectedStatus: HttpStatusCode.FOUND_302 - }) - - const location = res.header.location - expect(location.startsWith('/login?')).to.be.true - - const searchParams = decodeQueryString(location) - - expect(searchParams.externalAuthToken).to.exist - expect(searchParams.username).to.equal('cyan') - - externalAuthToken = searchParams.externalAuthToken as string - }) - - it('Should reject auto external login with a missing or invalid token', async function () { - const command = server.login - - await command.loginUsingExternalToken({ username: 'cyan', externalAuthToken: '', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await command.loginUsingExternalToken({ username: 'cyan', externalAuthToken: 'blabla', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should reject auto external login with a missing or invalid username', async function () { - const command = server.login - - await command.loginUsingExternalToken({ username: '', externalAuthToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await command.loginUsingExternalToken({ username: '', externalAuthToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should reject auto external login with an expired token', async function () { - this.timeout(15000) - - await wait(5000) - - await server.login.loginUsingExternalToken({ - username: 'cyan', - externalAuthToken, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - - await server.servers.waitUntilLog('expired external auth token', 4) - }) - - it('Should auto login Cyan, create the user and use the token', async function () { - { - const res = await loginExternal({ - server, - npmName: 'test-external-auth-one', - authName: 'external-auth-1', - query: { - username: 'cyan' - }, - username: 'cyan' - }) - - cyanAccessToken = res.access_token - cyanRefreshToken = res.refresh_token - } - - { - const body = await server.users.getMyInfo({ token: cyanAccessToken }) - expect(body.username).to.equal('cyan') - expect(body.account.displayName).to.equal('cyan') - expect(body.email).to.equal('cyan@example.com') - expect(body.role.id).to.equal(UserRole.USER) - expect(body.adminFlags).to.equal(UserAdminFlag.NONE) - expect(body.videoQuota).to.equal(5242880) - expect(body.videoQuotaDaily).to.equal(-1) - } - }) - - it('Should auto login Kefka, create the user and use the token', async function () { - { - const res = await loginExternal({ - server, - npmName: 'test-external-auth-one', - authName: 'external-auth-2', - username: 'kefka' - }) - - kefkaAccessToken = res.access_token - kefkaRefreshToken = res.refresh_token - } - - { - const body = await server.users.getMyInfo({ token: kefkaAccessToken }) - expect(body.username).to.equal('kefka') - expect(body.account.displayName).to.equal('Kefka Palazzo') - expect(body.email).to.equal('kefka@example.com') - expect(body.role.id).to.equal(UserRole.ADMINISTRATOR) - expect(body.adminFlags).to.equal(UserAdminFlag.BYPASS_VIDEO_AUTO_BLACKLIST) - expect(body.videoQuota).to.equal(42000) - expect(body.videoQuotaDaily).to.equal(42100) - - kefkaId = body.id - } - }) - - it('Should refresh Cyan token, but not Kefka token', async function () { - { - const resRefresh = await server.login.refreshToken({ refreshToken: cyanRefreshToken }) - cyanAccessToken = resRefresh.body.access_token - cyanRefreshToken = resRefresh.body.refresh_token - - const body = await server.users.getMyInfo({ token: cyanAccessToken }) - expect(body.username).to.equal('cyan') - } - - { - await server.login.refreshToken({ refreshToken: kefkaRefreshToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - } - }) - - it('Should update Cyan profile', async function () { - await server.users.updateMe({ - token: cyanAccessToken, - displayName: 'Cyan Garamonde', - description: 'Retainer to the king of Doma' - }) - - const body = await server.users.getMyInfo({ token: cyanAccessToken }) - expect(body.account.displayName).to.equal('Cyan Garamonde') - expect(body.account.description).to.equal('Retainer to the king of Doma') - }) - - it('Should logout Cyan', async function () { - await server.login.logout({ token: cyanAccessToken }) - }) - - it('Should have logged out Cyan', async function () { - await server.servers.waitUntilLog('On logout cyan') - - await server.users.getMyInfo({ token: cyanAccessToken, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should login Cyan and keep the old existing profile', async function () { - { - const res = await loginExternal({ - server, - npmName: 'test-external-auth-one', - authName: 'external-auth-1', - query: { - username: 'cyan' - }, - username: 'cyan' - }) - - cyanAccessToken = res.access_token - } - - const body = await server.users.getMyInfo({ token: cyanAccessToken }) - expect(body.username).to.equal('cyan') - expect(body.account.displayName).to.equal('Cyan Garamonde') - expect(body.account.description).to.equal('Retainer to the king of Doma') - expect(body.role.id).to.equal(UserRole.USER) - }) - - it('Should login Kefka and update the profile', async function () { - { - await server.users.update({ userId: kefkaId, videoQuota: 43000, videoQuotaDaily: 43100 }) - await server.users.updateMe({ token: kefkaAccessToken, displayName: 'kefka updated' }) - - const body = await server.users.getMyInfo({ token: kefkaAccessToken }) - expect(body.username).to.equal('kefka') - expect(body.account.displayName).to.equal('kefka updated') - expect(body.videoQuota).to.equal(43000) - expect(body.videoQuotaDaily).to.equal(43100) - } - - { - const res = await loginExternal({ - server, - npmName: 'test-external-auth-one', - authName: 'external-auth-2', - username: 'kefka' - }) - - kefkaAccessToken = res.access_token - kefkaRefreshToken = res.refresh_token - - const body = await server.users.getMyInfo({ token: kefkaAccessToken }) - expect(body.username).to.equal('kefka') - expect(body.account.displayName).to.equal('Kefka Palazzo') - expect(body.videoQuota).to.equal(42000) - expect(body.videoQuotaDaily).to.equal(43100) - } - }) - - it('Should not update an external auth email', async function () { - await server.users.updateMe({ - token: cyanAccessToken, - email: 'toto@example.com', - currentPassword: 'toto', - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should reject token of Kefka by the plugin hook', async function () { - await wait(5000) - - await server.users.getMyInfo({ token: kefkaAccessToken, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should unregister external-auth-2 and do not login existing Kefka', async function () { - await server.plugins.updateSettings({ - npmName: 'peertube-plugin-test-external-auth-one', - settings: { disableKefka: true } - }) - - await server.login.login({ user: { username: 'kefka', password: 'fake' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - - await loginExternal({ - server, - npmName: 'test-external-auth-one', - authName: 'external-auth-2', - query: { - username: 'kefka' - }, - username: 'kefka', - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should have disabled this auth', async function () { - const config = await server.config.getConfig() - - const auths = config.plugin.registeredExternalAuths - expect(auths).to.have.lengthOf(8) - - const auth1 = auths.find(a => a.authName === 'external-auth-2') - expect(auth1).to.not.exist - }) - - it('Should uninstall the plugin one and do not login Cyan', async function () { - await server.plugins.uninstall({ npmName: 'peertube-plugin-test-external-auth-one' }) - - await loginExternal({ - server, - npmName: 'test-external-auth-one', - authName: 'external-auth-1', - query: { - username: 'cyan' - }, - username: 'cyan', - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - - await server.login.login({ user: { username: 'cyan', password: null }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await server.login.login({ user: { username: 'cyan', password: '' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await server.login.login({ user: { username: 'cyan', password: 'fake' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should not login kefka with another plugin', async function () { - await loginExternal({ - server, - npmName: 'test-external-auth-two', - authName: 'external-auth-4', - username: 'kefka2', - expectedStatusStep2: HttpStatusCode.BAD_REQUEST_400 - }) - - await loginExternal({ - server, - npmName: 'test-external-auth-two', - authName: 'external-auth-4', - username: 'kefka', - expectedStatusStep2: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should not login an existing user email', async function () { - await server.users.create({ username: 'existing_user', password: 'super_password' }) - - await loginExternal({ - server, - npmName: 'test-external-auth-two', - authName: 'external-auth-6', - username: 'existing_user', - expectedStatusStep2: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should be able to login an existing user username and channel', async function () { - await server.users.create({ username: 'existing_user2' }) - await server.users.create({ username: 'existing_user2-1_channel' }) - - // Test twice to ensure we don't generate a username on every login - for (let i = 0; i < 2; i++) { - const res = await loginExternal({ - server, - npmName: 'test-external-auth-two', - authName: 'external-auth-7', - username: 'existing_user2' - }) - - const token = res.access_token - - const myInfo = await server.users.getMyInfo({ token }) - expect(myInfo.username).to.equal('existing_user2-1') - - expect(myInfo.videoChannels[0].name).to.equal('existing_user2-1_channel-1') - } - }) - - it('Should display the correct configuration', async function () { - const config = await server.config.getConfig() - - const auths = config.plugin.registeredExternalAuths - expect(auths).to.have.lengthOf(7) - - const auth2 = auths.find((a) => a.authName === 'external-auth-2') - expect(auth2).to.not.exist - }) - - after(async function () { - await cleanupTests([ server ]) - }) - - it('Should forward the redirectUrl if the plugin returns one', async function () { - const resLogin = await loginExternal({ - server, - npmName: 'test-external-auth-three', - authName: 'external-auth-7', - username: 'cid' - }) - - const { redirectUrl } = await server.login.logout({ token: resLogin.access_token }) - expect(redirectUrl).to.equal('https://example.com/redirectUrl') - }) - - it('Should call the plugin\'s onLogout method with the request', async function () { - const resLogin = await loginExternal({ - server, - npmName: 'test-external-auth-three', - authName: 'external-auth-8', - username: 'cid' - }) - - const { redirectUrl } = await server.login.logout({ token: resLogin.access_token }) - expect(redirectUrl).to.equal('https://example.com/redirectUrl?access_token=' + resLogin.access_token) - }) -}) diff --git a/server/tests/plugins/filter-hooks.ts b/server/tests/plugins/filter-hooks.ts deleted file mode 100644 index 8382b400f..000000000 --- a/server/tests/plugins/filter-hooks.ts +++ /dev/null @@ -1,909 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { - HttpStatusCode, - PeerTubeProblemDocument, - VideoDetails, - VideoImportState, - VideoPlaylist, - VideoPlaylistPrivacy, - VideoPrivacy -} from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - makeActivityPubGetRequest, - makeGetRequest, - makeRawRequest, - PeerTubeServer, - PluginsCommand, - setAccessTokensToServers, - setDefaultVideoChannel, - waitJobs -} from '@shared/server-commands' -import { FIXTURE_URLS } from '../shared' - -describe('Test plugin filter hooks', function () { - let servers: PeerTubeServer[] - let videoUUID: string - let threadId: number - let videoPlaylistUUID: string - - before(async function () { - this.timeout(120000) - - servers = await createMultipleServers(2) - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - await doubleFollow(servers[0], servers[1]) - - await servers[0].plugins.install({ path: PluginsCommand.getPluginTestPath() }) - await servers[0].plugins.install({ path: PluginsCommand.getPluginTestPath('-filter-translations') }) - { - ({ uuid: videoPlaylistUUID } = await servers[0].playlists.create({ - attributes: { - displayName: 'my super playlist', - privacy: VideoPlaylistPrivacy.PUBLIC, - description: 'my super description', - videoChannelId: servers[0].store.channel.id - } - })) - } - - for (let i = 0; i < 10; i++) { - const video = await servers[0].videos.upload({ attributes: { name: 'default video ' + i } }) - await servers[0].playlists.addElement({ playlistId: videoPlaylistUUID, attributes: { videoId: video.id } }) - } - - const { data } = await servers[0].videos.list() - videoUUID = data[0].uuid - - await servers[0].config.updateCustomSubConfig({ - newConfig: { - live: { enabled: true }, - signup: { enabled: true }, - videoFile: { - update: { - enabled: true - } - }, - import: { - videos: { - http: { enabled: true }, - torrent: { enabled: true } - } - } - } - }) - - // Root subscribes to itself - await servers[0].subscriptions.add({ targetUri: 'root_channel@' + servers[0].host }) - }) - - describe('Videos', function () { - - it('Should run filter:api.videos.list.params', async function () { - const { data } = await servers[0].videos.list({ start: 0, count: 2 }) - - // 2 plugins do +1 to the count parameter - expect(data).to.have.lengthOf(4) - }) - - it('Should run filter:api.videos.list.result', async function () { - const { total } = await servers[0].videos.list({ start: 0, count: 0 }) - - // Plugin do +1 to the total result - expect(total).to.equal(11) - }) - - it('Should run filter:api.video-playlist.videos.list.params', async function () { - const { data } = await servers[0].playlists.listVideos({ - count: 2, - playlistId: videoPlaylistUUID - }) - - // 1 plugin do +1 to the count parameter - expect(data).to.have.lengthOf(3) - }) - - it('Should run filter:api.video-playlist.videos.list.result', async function () { - const { total } = await servers[0].playlists.listVideos({ - count: 0, - playlistId: videoPlaylistUUID - }) - - // Plugin do +1 to the total result - expect(total).to.equal(11) - }) - - it('Should run filter:api.accounts.videos.list.params', async function () { - const { data } = await servers[0].videos.listByAccount({ handle: 'root', start: 0, count: 2 }) - - // 1 plugin do +1 to the count parameter - expect(data).to.have.lengthOf(3) - }) - - it('Should run filter:api.accounts.videos.list.result', async function () { - const { total } = await servers[0].videos.listByAccount({ handle: 'root', start: 0, count: 2 }) - - // Plugin do +2 to the total result - expect(total).to.equal(12) - }) - - it('Should run filter:api.video-channels.videos.list.params', async function () { - const { data } = await servers[0].videos.listByChannel({ handle: 'root_channel', start: 0, count: 2 }) - - // 1 plugin do +3 to the count parameter - expect(data).to.have.lengthOf(5) - }) - - it('Should run filter:api.video-channels.videos.list.result', async function () { - const { total } = await servers[0].videos.listByChannel({ handle: 'root_channel', start: 0, count: 2 }) - - // Plugin do +3 to the total result - expect(total).to.equal(13) - }) - - it('Should run filter:api.user.me.videos.list.params', async function () { - const { data } = await servers[0].videos.listMyVideos({ start: 0, count: 2 }) - - // 1 plugin do +4 to the count parameter - expect(data).to.have.lengthOf(6) - }) - - it('Should run filter:api.user.me.videos.list.result', async function () { - const { total } = await servers[0].videos.listMyVideos({ start: 0, count: 2 }) - - // Plugin do +4 to the total result - expect(total).to.equal(14) - }) - - it('Should run filter:api.user.me.subscription-videos.list.params', async function () { - const { data } = await servers[0].videos.listMySubscriptionVideos({ start: 0, count: 2 }) - - // 1 plugin do +1 to the count parameter - expect(data).to.have.lengthOf(3) - }) - - it('Should run filter:api.user.me.subscription-videos.list.result', async function () { - const { total } = await servers[0].videos.listMySubscriptionVideos({ start: 0, count: 2 }) - - // Plugin do +4 to the total result - expect(total).to.equal(14) - }) - - it('Should run filter:api.video.get.result', async function () { - const video = await servers[0].videos.get({ id: videoUUID }) - expect(video.name).to.contain('<3') - }) - }) - - describe('Video/live/import accept', function () { - - it('Should run filter:api.video.upload.accept.result', async function () { - const options = { attributes: { name: 'video with bad word' }, expectedStatus: HttpStatusCode.FORBIDDEN_403 } - await servers[0].videos.upload({ mode: 'legacy', ...options }) - await servers[0].videos.upload({ mode: 'resumable', ...options }) - }) - - it('Should run filter:api.video.update-file.accept.result', async function () { - const res = await servers[0].videos.replaceSourceFile({ - videoId: videoUUID, - fixture: 'video_short1.webm', - completedExpectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - - expect((res as any)?.error).to.equal('no webm') - }) - - it('Should run filter:api.live-video.create.accept.result', async function () { - const attributes = { - name: 'video with bad word', - privacy: VideoPrivacy.PUBLIC, - channelId: servers[0].store.channel.id - } - - await servers[0].live.create({ fields: attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should run filter:api.video.pre-import-url.accept.result', async function () { - const attributes = { - name: 'normal title', - privacy: VideoPrivacy.PUBLIC, - channelId: servers[0].store.channel.id, - targetUrl: FIXTURE_URLS.goodVideo + 'bad' - } - await servers[0].imports.importVideo({ attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should run filter:api.video.pre-import-torrent.accept.result', async function () { - const attributes = { - name: 'bad torrent', - privacy: VideoPrivacy.PUBLIC, - channelId: servers[0].store.channel.id, - torrentfile: 'video-720p.torrent' as any - } - await servers[0].imports.importVideo({ attributes, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - }) - - it('Should run filter:api.video.post-import-url.accept.result', async function () { - this.timeout(60000) - - let videoImportId: number - - { - const attributes = { - name: 'title with bad word', - privacy: VideoPrivacy.PUBLIC, - channelId: servers[0].store.channel.id, - targetUrl: FIXTURE_URLS.goodVideo - } - const body = await servers[0].imports.importVideo({ attributes }) - videoImportId = body.id - } - - await waitJobs(servers) - - { - const body = await servers[0].imports.getMyVideoImports() - const videoImports = body.data - - const videoImport = videoImports.find(i => i.id === videoImportId) - - expect(videoImport.state.id).to.equal(VideoImportState.REJECTED) - expect(videoImport.state.label).to.equal('Rejected') - } - }) - - it('Should run filter:api.video.post-import-torrent.accept.result', async function () { - this.timeout(60000) - - let videoImportId: number - - { - const attributes = { - name: 'title with bad word', - privacy: VideoPrivacy.PUBLIC, - channelId: servers[0].store.channel.id, - torrentfile: 'video-720p.torrent' as any - } - const body = await servers[0].imports.importVideo({ attributes }) - videoImportId = body.id - } - - await waitJobs(servers) - - { - const { data: videoImports } = await servers[0].imports.getMyVideoImports() - - const videoImport = videoImports.find(i => i.id === videoImportId) - - expect(videoImport.state.id).to.equal(VideoImportState.REJECTED) - expect(videoImport.state.label).to.equal('Rejected') - } - }) - }) - - describe('Video comments accept', function () { - - it('Should run filter:api.video-thread.create.accept.result', async function () { - await servers[0].comments.createThread({ - videoId: videoUUID, - text: 'comment with bad word', - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - }) - - it('Should run filter:api.video-comment-reply.create.accept.result', async function () { - const created = await servers[0].comments.createThread({ videoId: videoUUID, text: 'thread' }) - threadId = created.id - - await servers[0].comments.addReply({ - videoId: videoUUID, - toCommentId: threadId, - text: 'comment with bad word', - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - await servers[0].comments.addReply({ - videoId: videoUUID, - toCommentId: threadId, - text: 'comment with good word', - expectedStatus: HttpStatusCode.OK_200 - }) - }) - - it('Should run filter:activity-pub.remote-video-comment.create.accept.result on a thread creation', async function () { - this.timeout(30000) - - await servers[1].comments.createThread({ videoId: videoUUID, text: 'comment with bad word' }) - - await waitJobs(servers) - - { - const thread = await servers[0].comments.listThreads({ videoId: videoUUID }) - expect(thread.data).to.have.lengthOf(1) - expect(thread.data[0].text).to.not.include(' bad ') - } - - { - const thread = await servers[1].comments.listThreads({ videoId: videoUUID }) - expect(thread.data).to.have.lengthOf(2) - } - }) - - it('Should run filter:activity-pub.remote-video-comment.create.accept.result on a reply creation', async function () { - this.timeout(30000) - - const { data } = await servers[1].comments.listThreads({ videoId: videoUUID }) - const threadIdServer2 = data.find(t => t.text === 'thread').id - - await servers[1].comments.addReply({ - videoId: videoUUID, - toCommentId: threadIdServer2, - text: 'comment with bad word' - }) - - await waitJobs(servers) - - { - const tree = await servers[0].comments.getThread({ videoId: videoUUID, threadId }) - expect(tree.children).to.have.lengthOf(1) - expect(tree.children[0].comment.text).to.not.include(' bad ') - } - - { - const tree = await servers[1].comments.getThread({ videoId: videoUUID, threadId: threadIdServer2 }) - expect(tree.children).to.have.lengthOf(2) - } - }) - }) - - describe('Video comments', function () { - - it('Should run filter:api.video-threads.list.params', async function () { - const { data } = await servers[0].comments.listThreads({ videoId: videoUUID, start: 0, count: 0 }) - - // our plugin do +1 to the count parameter - expect(data).to.have.lengthOf(1) - }) - - it('Should run filter:api.video-threads.list.result', async function () { - const { total } = await servers[0].comments.listThreads({ videoId: videoUUID, start: 0, count: 0 }) - - // Plugin do +1 to the total result - expect(total).to.equal(2) - }) - - it('Should run filter:api.video-thread-comments.list.params') - - it('Should run filter:api.video-thread-comments.list.result', async function () { - const thread = await servers[0].comments.getThread({ videoId: videoUUID, threadId }) - - expect(thread.comment.text.endsWith(' <3')).to.be.true - }) - - it('Should run filter:api.overviews.videos.list.{params,result}', async function () { - await servers[0].overviews.getVideos({ page: 1 }) - - // 3 because we get 3 samples per page - await servers[0].servers.waitUntilLog('Run hook filter:api.overviews.videos.list.params', 3) - await servers[0].servers.waitUntilLog('Run hook filter:api.overviews.videos.list.result', 3) - }) - }) - - describe('filter:video.auto-blacklist.result', function () { - - async function checkIsBlacklisted (id: number | string, value: boolean) { - const video = await servers[0].videos.getWithToken({ id }) - expect(video.blacklisted).to.equal(value) - } - - it('Should blacklist on upload', async function () { - const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video please blacklist me' } }) - await checkIsBlacklisted(uuid, true) - }) - - it('Should blacklist on import', async function () { - this.timeout(15000) - - const attributes = { - name: 'video please blacklist me', - targetUrl: FIXTURE_URLS.goodVideo, - channelId: servers[0].store.channel.id - } - const body = await servers[0].imports.importVideo({ attributes }) - await checkIsBlacklisted(body.video.uuid, true) - }) - - it('Should blacklist on update', async function () { - const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video' } }) - await checkIsBlacklisted(uuid, false) - - await servers[0].videos.update({ id: uuid, attributes: { name: 'please blacklist me' } }) - await checkIsBlacklisted(uuid, true) - }) - - it('Should blacklist on remote upload', async function () { - this.timeout(120000) - - const { uuid } = await servers[1].videos.upload({ attributes: { name: 'remote please blacklist me' } }) - await waitJobs(servers) - - await checkIsBlacklisted(uuid, true) - }) - - it('Should blacklist on remote update', async function () { - this.timeout(120000) - - const { uuid } = await servers[1].videos.upload({ attributes: { name: 'video' } }) - await waitJobs(servers) - - await checkIsBlacklisted(uuid, false) - - await servers[1].videos.update({ id: uuid, attributes: { name: 'please blacklist me' } }) - await waitJobs(servers) - - await checkIsBlacklisted(uuid, true) - }) - }) - - describe('Should run filter:api.user.signup.allowed.result', function () { - - before(async function () { - await servers[0].config.updateExistingSubConfig({ newConfig: { signup: { requiresApproval: false } } }) - }) - - it('Should run on config endpoint', async function () { - const body = await servers[0].config.getConfig() - expect(body.signup.allowed).to.be.true - }) - - it('Should allow a signup', async function () { - await servers[0].registrations.register({ username: 'john1' }) - }) - - it('Should not allow a signup', async function () { - const res = await servers[0].registrations.register({ - username: 'jma 1', - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - - expect(res.body.error).to.equal('No jma 1') - }) - }) - - describe('Should run filter:api.user.request-signup.allowed.result', function () { - - before(async function () { - await servers[0].config.updateExistingSubConfig({ newConfig: { signup: { requiresApproval: true } } }) - }) - - it('Should run on config endpoint', async function () { - const body = await servers[0].config.getConfig() - expect(body.signup.allowed).to.be.true - }) - - it('Should allow a signup request', async function () { - await servers[0].registrations.requestRegistration({ username: 'john2', registrationReason: 'tt' }) - }) - - it('Should not allow a signup request', async function () { - const body = await servers[0].registrations.requestRegistration({ - username: 'jma 2', - registrationReason: 'tt', - expectedStatus: HttpStatusCode.FORBIDDEN_403 - }) - - expect((body as unknown as PeerTubeProblemDocument).error).to.equal('No jma 2') - }) - }) - - describe('Download hooks', function () { - const downloadVideos: VideoDetails[] = [] - let downloadVideo2Token: string - - before(async function () { - this.timeout(120000) - - await servers[0].config.updateCustomSubConfig({ - newConfig: { - transcoding: { - webVideos: { - enabled: true - }, - hls: { - enabled: true - } - } - } - }) - - const uuids: string[] = [] - - for (const name of [ 'bad torrent', 'bad file', 'bad playlist file' ]) { - const uuid = (await servers[0].videos.quickUpload({ name })).uuid - uuids.push(uuid) - } - - await waitJobs(servers) - - for (const uuid of uuids) { - downloadVideos.push(await servers[0].videos.get({ id: uuid })) - } - - downloadVideo2Token = await servers[0].videoToken.getVideoFileToken({ videoId: downloadVideos[2].uuid }) - }) - - it('Should run filter:api.download.torrent.allowed.result', async function () { - const res = await makeRawRequest({ url: downloadVideos[0].files[0].torrentDownloadUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - expect(res.body.error).to.equal('Liu Bei') - - await makeRawRequest({ url: downloadVideos[1].files[0].torrentDownloadUrl, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ url: downloadVideos[2].files[0].torrentDownloadUrl, expectedStatus: HttpStatusCode.OK_200 }) - }) - - it('Should run filter:api.download.video.allowed.result', async function () { - { - const refused = downloadVideos[1].files[0].fileDownloadUrl - const allowed = [ - downloadVideos[0].files[0].fileDownloadUrl, - downloadVideos[2].files[0].fileDownloadUrl - ] - - const res = await makeRawRequest({ url: refused, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - expect(res.body.error).to.equal('Cao Cao') - - for (const url of allowed) { - await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) - } - } - - { - const refused = downloadVideos[2].streamingPlaylists[0].files[0].fileDownloadUrl - - const allowed = [ - downloadVideos[2].files[0].fileDownloadUrl, - downloadVideos[0].streamingPlaylists[0].files[0].fileDownloadUrl, - downloadVideos[1].streamingPlaylists[0].files[0].fileDownloadUrl - ] - - // Only streaming playlist is refuse - const res = await makeRawRequest({ url: refused, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) - expect(res.body.error).to.equal('Sun Jian') - - // But not we there is a user in res - await makeRawRequest({ url: refused, token: servers[0].accessToken, expectedStatus: HttpStatusCode.OK_200 }) - await makeRawRequest({ url: refused, query: { videoFileToken: downloadVideo2Token }, expectedStatus: HttpStatusCode.OK_200 }) - - // Other files work - for (const url of allowed) { - await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) - } - } - }) - }) - - describe('Embed filters', function () { - const embedVideos: VideoDetails[] = [] - const embedPlaylists: VideoPlaylist[] = [] - - before(async function () { - this.timeout(60000) - - await servers[0].config.disableTranscoding() - - for (const name of [ 'bad embed', 'good embed' ]) { - { - const uuid = (await servers[0].videos.quickUpload({ name })).uuid - embedVideos.push(await servers[0].videos.get({ id: uuid })) - } - - { - const attributes = { displayName: name, videoChannelId: servers[0].store.channel.id, privacy: VideoPlaylistPrivacy.PUBLIC } - const { id } = await servers[0].playlists.create({ attributes }) - - const playlist = await servers[0].playlists.get({ playlistId: id }) - embedPlaylists.push(playlist) - } - } - }) - - it('Should run filter:html.embed.video.allowed.result', async function () { - const res = await makeGetRequest({ url: servers[0].url, path: embedVideos[0].embedPath, expectedStatus: HttpStatusCode.OK_200 }) - expect(res.text).to.equal('Lu Bu') - }) - - it('Should run filter:html.embed.video-playlist.allowed.result', async function () { - const res = await makeGetRequest({ url: servers[0].url, path: embedPlaylists[0].embedPath, expectedStatus: HttpStatusCode.OK_200 }) - expect(res.text).to.equal('Diao Chan') - }) - }) - - describe('Client HTML filters', function () { - let videoUUID: string - - before(async function () { - this.timeout(60000) - - const { uuid } = await servers[0].videos.quickUpload({ name: 'html video' }) - videoUUID = uuid - }) - - it('Should run filter:html.client.json-ld.result', async function () { - const res = await makeGetRequest({ url: servers[0].url, path: '/w/' + videoUUID, expectedStatus: HttpStatusCode.OK_200 }) - expect(res.text).to.contain('"recordedAt":"http://example.com/recordedAt"') - }) - - it('Should not run filter:html.client.json-ld.result with an account', async function () { - const res = await makeGetRequest({ url: servers[0].url, path: '/a/root', expectedStatus: HttpStatusCode.OK_200 }) - expect(res.text).not.to.contain('"recordedAt":"http://example.com/recordedAt"') - }) - }) - - describe('Search filters', function () { - - before(async function () { - await servers[0].config.updateCustomSubConfig({ - newConfig: { - search: { - searchIndex: { - enabled: true, - isDefaultSearch: false, - disableLocalSearch: false - } - } - } - }) - }) - - it('Should run filter:api.search.videos.local.list.{params,result}', async function () { - await servers[0].search.advancedVideoSearch({ - search: { - search: 'Sun Quan' - } - }) - - await servers[0].servers.waitUntilLog('Run hook filter:api.search.videos.local.list.params', 1) - await servers[0].servers.waitUntilLog('Run hook filter:api.search.videos.local.list.result', 1) - }) - - it('Should run filter:api.search.videos.index.list.{params,result}', async function () { - await servers[0].search.advancedVideoSearch({ - search: { - search: 'Sun Quan', - searchTarget: 'search-index' - } - }) - - await servers[0].servers.waitUntilLog('Run hook filter:api.search.videos.local.list.params', 1) - await servers[0].servers.waitUntilLog('Run hook filter:api.search.videos.local.list.result', 1) - await servers[0].servers.waitUntilLog('Run hook filter:api.search.videos.index.list.params', 1) - await servers[0].servers.waitUntilLog('Run hook filter:api.search.videos.index.list.result', 1) - }) - - it('Should run filter:api.search.video-channels.local.list.{params,result}', async function () { - await servers[0].search.advancedChannelSearch({ - search: { - search: 'Sun Ce' - } - }) - - await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-channels.local.list.params', 1) - await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-channels.local.list.result', 1) - }) - - it('Should run filter:api.search.video-channels.index.list.{params,result}', async function () { - await servers[0].search.advancedChannelSearch({ - search: { - search: 'Sun Ce', - searchTarget: 'search-index' - } - }) - - await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-channels.local.list.params', 1) - await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-channels.local.list.result', 1) - await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-channels.index.list.params', 1) - await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-channels.index.list.result', 1) - }) - - it('Should run filter:api.search.video-playlists.local.list.{params,result}', async function () { - await servers[0].search.advancedPlaylistSearch({ - search: { - search: 'Sun Jian' - } - }) - - await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-playlists.local.list.params', 1) - await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-playlists.local.list.result', 1) - }) - - it('Should run filter:api.search.video-playlists.index.list.{params,result}', async function () { - await servers[0].search.advancedPlaylistSearch({ - search: { - search: 'Sun Jian', - searchTarget: 'search-index' - } - }) - - await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-playlists.local.list.params', 1) - await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-playlists.local.list.result', 1) - await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-playlists.index.list.params', 1) - await servers[0].servers.waitUntilLog('Run hook filter:api.search.video-playlists.index.list.result', 1) - }) - }) - - describe('Upload/import/live attributes filters', function () { - - before(async function () { - await servers[0].config.enableLive({ transcoding: false, allowReplay: false }) - await servers[0].config.enableImports() - await servers[0].config.disableTranscoding() - }) - - it('Should run filter:api.video.upload.video-attribute.result', async function () { - for (const mode of [ 'legacy' as 'legacy', 'resumable' as 'resumable' ]) { - const { id } = await servers[0].videos.upload({ attributes: { name: 'video', description: 'upload' }, mode }) - - const video = await servers[0].videos.get({ id }) - expect(video.description).to.equal('upload - filter:api.video.upload.video-attribute.result') - } - }) - - it('Should run filter:api.video.import-url.video-attribute.result', async function () { - const attributes = { - name: 'video', - description: 'import url', - channelId: servers[0].store.channel.id, - targetUrl: FIXTURE_URLS.goodVideo, - privacy: VideoPrivacy.PUBLIC - } - const { video: { id } } = await servers[0].imports.importVideo({ attributes }) - - const video = await servers[0].videos.get({ id }) - expect(video.description).to.equal('import url - filter:api.video.import-url.video-attribute.result') - }) - - it('Should run filter:api.video.import-torrent.video-attribute.result', async function () { - const attributes = { - name: 'video', - description: 'import torrent', - channelId: servers[0].store.channel.id, - magnetUri: FIXTURE_URLS.magnet, - privacy: VideoPrivacy.PUBLIC - } - const { video: { id } } = await servers[0].imports.importVideo({ attributes }) - - const video = await servers[0].videos.get({ id }) - expect(video.description).to.equal('import torrent - filter:api.video.import-torrent.video-attribute.result') - }) - - it('Should run filter:api.video.live.video-attribute.result', async function () { - const fields = { - name: 'live', - description: 'live', - channelId: servers[0].store.channel.id, - privacy: VideoPrivacy.PUBLIC - } - const { id } = await servers[0].live.create({ fields }) - - const video = await servers[0].videos.get({ id }) - expect(video.description).to.equal('live - filter:api.video.live.video-attribute.result') - }) - }) - - describe('Stats filters', function () { - - it('Should run filter:api.server.stats.get.result', async function () { - const data = await servers[0].stats.get() - - expect((data as any).customStats).to.equal(14) - }) - - }) - - describe('Job queue filters', function () { - let videoUUID: string - - before(async function () { - this.timeout(120_000) - - await servers[0].config.enableMinimumTranscoding() - const { uuid } = await servers[0].videos.quickUpload({ name: 'studio' }) - - const video = await servers[0].videos.get({ id: uuid }) - expect(video.duration).at.least(2) - videoUUID = video.uuid - - await waitJobs(servers) - - await servers[0].config.enableStudio() - }) - - it('Should run filter:job-queue.process.params', async function () { - this.timeout(120_000) - - await servers[0].videoStudio.createEditionTasks({ - videoId: videoUUID, - tasks: [ - { - name: 'add-intro', - options: { - file: 'video_very_short_240p.mp4' - } - } - ] - }) - - await waitJobs(servers) - - await servers[0].servers.waitUntilLog('Run hook filter:job-queue.process.params', 1, false) - - const video = await servers[0].videos.get({ id: videoUUID }) - expect(video.duration).at.most(2) - }) - - it('Should run filter:job-queue.process.result', async function () { - await servers[0].servers.waitUntilLog('Run hook filter:job-queue.process.result', 1, false) - }) - }) - - describe('Transcoding filters', async function () { - - it('Should run filter:transcoding.auto.resolutions-to-transcode.result', async function () { - const { uuid } = await servers[0].videos.quickUpload({ name: 'transcode-filter' }) - - await waitJobs(servers) - - const video = await servers[0].videos.get({ id: uuid }) - expect(video.files).to.have.lengthOf(2) - expect(video.files.find(f => f.resolution.id === 100 as any)).to.exist - }) - }) - - describe('Video channel filters', async function () { - - it('Should run filter:api.video-channels.list.params', async function () { - const { data } = await servers[0].channels.list({ start: 0, count: 0 }) - - // plugin do +1 to the count parameter - expect(data).to.have.lengthOf(1) - }) - - it('Should run filter:api.video-channels.list.result', async function () { - const { total } = await servers[0].channels.list({ start: 0, count: 1 }) - - // plugin do +1 to the total parameter - expect(total).to.equal(4) - }) - - it('Should run filter:api.video-channel.get.result', async function () { - const channel = await servers[0].channels.get({ channelName: 'root_channel' }) - expect(channel.displayName).to.equal('Main root channel <3') - }) - }) - - describe('Activity Pub', function () { - - it('Should run filter:activity-pub.activity.context.build.result', async function () { - const { body } = await makeActivityPubGetRequest(servers[0].url, '/w/' + videoUUID) - expect(body.type).to.equal('Video') - - expect(body['@context'].some(c => { - return typeof c === 'object' && c.recordedAt === 'https://schema.org/recordedAt' - })).to.be.true - }) - - it('Should run filter:activity-pub.video.json-ld.build.result', async function () { - const { body } = await makeActivityPubGetRequest(servers[0].url, '/w/' + videoUUID) - expect(body.name).to.equal('default video 0') - expect(body.videoName).to.equal('default video 0') - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/plugins/html-injection.ts b/server/tests/plugins/html-injection.ts deleted file mode 100644 index fe16bf1e6..000000000 --- a/server/tests/plugins/html-injection.ts +++ /dev/null @@ -1,73 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { - cleanupTests, - createSingleServer, - makeHTMLRequest, - PeerTubeServer, - PluginsCommand, - setAccessTokensToServers -} from '@shared/server-commands' - -describe('Test plugins HTML injection', function () { - let server: PeerTubeServer = null - let command: PluginsCommand - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - - command = server.plugins - }) - - it('Should not inject global css file in HTML', async function () { - { - const text = await command.getCSS() - expect(text).to.be.empty - } - - for (const path of [ '/', '/videos/embed/1', '/video-playlists/embed/1' ]) { - const res = await makeHTMLRequest(server.url, path) - expect(res.text).to.not.include('link rel="stylesheet" href="/plugins/global.css') - } - }) - - it('Should install a plugin and a theme', async function () { - this.timeout(30000) - - await command.install({ npmName: 'peertube-plugin-hello-world' }) - }) - - it('Should have the correct global css', async function () { - { - const text = await command.getCSS() - expect(text).to.contain('background-color: red') - } - - for (const path of [ '/', '/videos/embed/1', '/video-playlists/embed/1' ]) { - const res = await makeHTMLRequest(server.url, path) - expect(res.text).to.include('link rel="stylesheet" href="/plugins/global.css') - } - }) - - it('Should have an empty global css on uninstall', async function () { - await command.uninstall({ npmName: 'peertube-plugin-hello-world' }) - - { - const text = await command.getCSS() - expect(text).to.be.empty - } - - for (const path of [ '/', '/videos/embed/1', '/video-playlists/embed/1' ]) { - const res = await makeHTMLRequest(server.url, path) - expect(res.text).to.not.include('link rel="stylesheet" href="/plugins/global.css') - } - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/plugins/id-and-pass-auth.ts b/server/tests/plugins/id-and-pass-auth.ts deleted file mode 100644 index 127c29cbc..000000000 --- a/server/tests/plugins/id-and-pass-auth.ts +++ /dev/null @@ -1,242 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { wait } from '@shared/core-utils' -import { HttpStatusCode, UserRole } from '@shared/models' -import { cleanupTests, createSingleServer, PeerTubeServer, PluginsCommand, setAccessTokensToServers } from '@shared/server-commands' - -describe('Test id and pass auth plugins', function () { - let server: PeerTubeServer - - let crashAccessToken: string - let crashRefreshToken: string - - let lagunaAccessToken: string - let lagunaRefreshToken: string - let lagunaId: number - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - - for (const suffix of [ 'one', 'two', 'three' ]) { - await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-id-pass-auth-' + suffix) }) - } - }) - - it('Should display the correct configuration', async function () { - const config = await server.config.getConfig() - - const auths = config.plugin.registeredIdAndPassAuths - expect(auths).to.have.lengthOf(8) - - const crashAuth = auths.find(a => a.authName === 'crash-auth') - expect(crashAuth).to.exist - expect(crashAuth.npmName).to.equal('peertube-plugin-test-id-pass-auth-one') - expect(crashAuth.weight).to.equal(50) - }) - - it('Should not login', async function () { - await server.login.login({ user: { username: 'toto', password: 'password' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should login Spyro, create the user and use the token', async function () { - const accessToken = await server.login.getAccessToken({ username: 'spyro', password: 'spyro password' }) - - const body = await server.users.getMyInfo({ token: accessToken }) - - expect(body.username).to.equal('spyro') - expect(body.account.displayName).to.equal('Spyro the Dragon') - expect(body.role.id).to.equal(UserRole.USER) - }) - - it('Should login Crash, create the user and use the token', async function () { - { - const body = await server.login.login({ user: { username: 'crash', password: 'crash password' } }) - crashAccessToken = body.access_token - crashRefreshToken = body.refresh_token - } - - { - const body = await server.users.getMyInfo({ token: crashAccessToken }) - - expect(body.username).to.equal('crash') - expect(body.account.displayName).to.equal('Crash Bandicoot') - expect(body.role.id).to.equal(UserRole.MODERATOR) - } - }) - - it('Should login the first Laguna, create the user and use the token', async function () { - { - const body = await server.login.login({ user: { username: 'laguna', password: 'laguna password' } }) - lagunaAccessToken = body.access_token - lagunaRefreshToken = body.refresh_token - } - - { - const body = await server.users.getMyInfo({ token: lagunaAccessToken }) - - expect(body.username).to.equal('laguna') - expect(body.account.displayName).to.equal('Laguna Loire') - expect(body.role.id).to.equal(UserRole.USER) - - lagunaId = body.id - } - }) - - it('Should refresh crash token, but not laguna token', async function () { - { - const resRefresh = await server.login.refreshToken({ refreshToken: crashRefreshToken }) - crashAccessToken = resRefresh.body.access_token - crashRefreshToken = resRefresh.body.refresh_token - - const body = await server.users.getMyInfo({ token: crashAccessToken }) - expect(body.username).to.equal('crash') - } - - { - await server.login.refreshToken({ refreshToken: lagunaRefreshToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - } - }) - - it('Should update Crash profile', async function () { - await server.users.updateMe({ - token: crashAccessToken, - displayName: 'Beautiful Crash', - description: 'Mutant eastern barred bandicoot' - }) - - const body = await server.users.getMyInfo({ token: crashAccessToken }) - - expect(body.account.displayName).to.equal('Beautiful Crash') - expect(body.account.description).to.equal('Mutant eastern barred bandicoot') - }) - - it('Should logout Crash', async function () { - await server.login.logout({ token: crashAccessToken }) - }) - - it('Should have logged out Crash', async function () { - await server.servers.waitUntilLog('On logout for auth 1 - 2') - - await server.users.getMyInfo({ token: crashAccessToken, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should login Crash and keep the old existing profile', async function () { - crashAccessToken = await server.login.getAccessToken({ username: 'crash', password: 'crash password' }) - - const body = await server.users.getMyInfo({ token: crashAccessToken }) - - expect(body.username).to.equal('crash') - expect(body.account.displayName).to.equal('Beautiful Crash') - expect(body.account.description).to.equal('Mutant eastern barred bandicoot') - expect(body.role.id).to.equal(UserRole.MODERATOR) - }) - - it('Should login Laguna and update the profile', async function () { - { - await server.users.update({ userId: lagunaId, videoQuota: 43000, videoQuotaDaily: 43100 }) - await server.users.updateMe({ token: lagunaAccessToken, displayName: 'laguna updated' }) - - const body = await server.users.getMyInfo({ token: lagunaAccessToken }) - expect(body.username).to.equal('laguna') - expect(body.account.displayName).to.equal('laguna updated') - expect(body.videoQuota).to.equal(43000) - expect(body.videoQuotaDaily).to.equal(43100) - } - - { - const body = await server.login.login({ user: { username: 'laguna', password: 'laguna password' } }) - lagunaAccessToken = body.access_token - lagunaRefreshToken = body.refresh_token - } - - { - const body = await server.users.getMyInfo({ token: lagunaAccessToken }) - expect(body.username).to.equal('laguna') - expect(body.account.displayName).to.equal('Laguna Loire') - expect(body.videoQuota).to.equal(42000) - expect(body.videoQuotaDaily).to.equal(43100) - } - }) - - it('Should reject token of laguna by the plugin hook', async function () { - await wait(5000) - - await server.users.getMyInfo({ token: lagunaAccessToken, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) - }) - - it('Should reject an invalid username, email, role or display name', async function () { - const command = server.login - - await command.login({ user: { username: 'ward', password: 'ward password' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await server.servers.waitUntilLog('valid username') - - await command.login({ user: { username: 'kiros', password: 'kiros password' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await server.servers.waitUntilLog('valid displayName') - - await command.login({ user: { username: 'raine', password: 'raine password' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await server.servers.waitUntilLog('valid role') - - await command.login({ user: { username: 'ellone', password: 'elonne password' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await server.servers.waitUntilLog('valid email') - }) - - it('Should unregister spyro-auth and do not login existing Spyro', async function () { - await server.plugins.updateSettings({ - npmName: 'peertube-plugin-test-id-pass-auth-one', - settings: { disableSpyro: true } - }) - - const command = server.login - await command.login({ user: { username: 'spyro', password: 'spyro password' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - await command.login({ user: { username: 'spyro', password: 'fake' }, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should have disabled this auth', async function () { - const config = await server.config.getConfig() - - const auths = config.plugin.registeredIdAndPassAuths - expect(auths).to.have.lengthOf(7) - - const spyroAuth = auths.find(a => a.authName === 'spyro-auth') - expect(spyroAuth).to.not.exist - }) - - it('Should uninstall the plugin one and do not login existing Crash', async function () { - await server.plugins.uninstall({ npmName: 'peertube-plugin-test-id-pass-auth-one' }) - - await server.login.login({ - user: { username: 'crash', password: 'crash password' }, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - }) - - it('Should display the correct configuration', async function () { - const config = await server.config.getConfig() - - const auths = config.plugin.registeredIdAndPassAuths - expect(auths).to.have.lengthOf(6) - - const crashAuth = auths.find(a => a.authName === 'crash-auth') - expect(crashAuth).to.not.exist - }) - - it('Should display plugin auth information in users list', async function () { - const { data } = await server.users.list() - - const root = data.find(u => u.username === 'root') - const crash = data.find(u => u.username === 'crash') - const laguna = data.find(u => u.username === 'laguna') - - expect(root.pluginAuth).to.be.null - expect(crash.pluginAuth).to.equal('peertube-plugin-test-id-pass-auth-one') - expect(laguna.pluginAuth).to.equal('peertube-plugin-test-id-pass-auth-two') - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/plugins/plugin-helpers.ts b/server/tests/plugins/plugin-helpers.ts deleted file mode 100644 index f5a0cbe85..000000000 --- a/server/tests/plugins/plugin-helpers.ts +++ /dev/null @@ -1,383 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { pathExists } from 'fs-extra' -import { HttpStatusCode, ThumbnailType } from '@shared/models' -import { - cleanupTests, - createMultipleServers, - doubleFollow, - makeGetRequest, - makePostBodyRequest, - makeRawRequest, - PeerTubeServer, - PluginsCommand, - setAccessTokensToServers, - waitJobs -} from '@shared/server-commands' -import { checkVideoFilesWereRemoved } from '../shared' - -function postCommand (server: PeerTubeServer, command: string, bodyArg?: object) { - const body = { command } - if (bodyArg) Object.assign(body, bodyArg) - - return makePostBodyRequest({ - url: server.url, - path: '/plugins/test-four/router/commander', - fields: body, - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) -} - -describe('Test plugin helpers', function () { - let servers: PeerTubeServer[] - - before(async function () { - this.timeout(60000) - - servers = await createMultipleServers(2) - await setAccessTokensToServers(servers) - - await doubleFollow(servers[0], servers[1]) - - await servers[0].plugins.install({ path: PluginsCommand.getPluginTestPath('-four') }) - }) - - describe('Logger', function () { - - it('Should have logged things', async function () { - await servers[0].servers.waitUntilLog(servers[0].host + ' peertube-plugin-test-four', 1, false) - await servers[0].servers.waitUntilLog('Hello world from plugin four', 1) - }) - }) - - describe('Database', function () { - - it('Should have made a query', async function () { - await servers[0].servers.waitUntilLog(`root email is admin${servers[0].internalServerNumber}@example.com`) - }) - }) - - describe('Config', function () { - - it('Should have the correct webserver url', async function () { - await servers[0].servers.waitUntilLog(`server url is ${servers[0].url}`) - }) - - it('Should have the correct listening config', async function () { - const res = await makeGetRequest({ - url: servers[0].url, - path: '/plugins/test-four/router/server-listening-config', - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(res.body.config).to.exist - expect(res.body.config.hostname).to.equal('::') - expect(res.body.config.port).to.equal(servers[0].port) - }) - - it('Should have the correct config', async function () { - const res = await makeGetRequest({ - url: servers[0].url, - path: '/plugins/test-four/router/server-config', - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(res.body.serverConfig).to.exist - expect(res.body.serverConfig.instance.name).to.equal('PeerTube') - }) - }) - - describe('Server', function () { - - it('Should get the server actor', async function () { - await servers[0].servers.waitUntilLog('server actor name is peertube') - }) - }) - - describe('Socket', function () { - - it('Should sendNotification without any exceptions', async () => { - const user = await servers[0].users.create({ username: 'notis_redding', password: 'secret1234?' }) - await makePostBodyRequest({ - url: servers[0].url, - path: '/plugins/test-four/router/send-notification', - fields: { - userId: user.id - }, - expectedStatus: HttpStatusCode.CREATED_201 - }) - }) - - it('Should sendVideoLiveNewState without any exceptions', async () => { - const res = await servers[0].videos.quickUpload({ name: 'video server 1' }) - - await makePostBodyRequest({ - url: servers[0].url, - path: '/plugins/test-four/router/send-video-live-new-state/' + res.uuid, - expectedStatus: HttpStatusCode.CREATED_201 - }) - - await servers[0].videos.remove({ id: res.uuid }) - }) - }) - - describe('Plugin', function () { - - it('Should get the base static route', async function () { - const res = await makeGetRequest({ - url: servers[0].url, - path: '/plugins/test-four/router/static-route', - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(res.body.staticRoute).to.equal('/plugins/test-four/0.0.1/static/') - }) - - it('Should get the base static route', async function () { - const baseRouter = '/plugins/test-four/0.0.1/router/' - - const res = await makeGetRequest({ - url: servers[0].url, - path: baseRouter + 'router-route', - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(res.body.routerRoute).to.equal(baseRouter) - }) - }) - - describe('User', function () { - let rootId: number - - it('Should not get a user if not authenticated', async function () { - await makeGetRequest({ - url: servers[0].url, - path: '/plugins/test-four/router/user', - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should get a user if authenticated', async function () { - const res = await makeGetRequest({ - url: servers[0].url, - token: servers[0].accessToken, - path: '/plugins/test-four/router/user', - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(res.body.username).to.equal('root') - expect(res.body.displayName).to.equal('root') - expect(res.body.isAdmin).to.be.true - expect(res.body.isModerator).to.be.false - expect(res.body.isUser).to.be.false - - rootId = res.body.id - }) - - it('Should load a user by id', async function () { - { - const res = await makeGetRequest({ - url: servers[0].url, - path: '/plugins/test-four/router/user/' + rootId, - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(res.body.username).to.equal('root') - } - - { - await makeGetRequest({ - url: servers[0].url, - path: '/plugins/test-four/router/user/42', - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - } - }) - }) - - describe('Moderation', function () { - let videoUUIDServer1: string - - before(async function () { - this.timeout(60000) - - { - const res = await servers[0].videos.quickUpload({ name: 'video server 1' }) - videoUUIDServer1 = res.uuid - } - - { - await servers[1].videos.quickUpload({ name: 'video server 2' }) - } - - await waitJobs(servers) - - const { data } = await servers[0].videos.list() - - expect(data).to.have.lengthOf(2) - }) - - it('Should mute server 2', async function () { - await postCommand(servers[0], 'blockServer', { hostToBlock: servers[1].host }) - - const { data } = await servers[0].videos.list() - - expect(data).to.have.lengthOf(1) - expect(data[0].name).to.equal('video server 1') - }) - - it('Should unmute server 2', async function () { - await postCommand(servers[0], 'unblockServer', { hostToUnblock: servers[1].host }) - - const { data } = await servers[0].videos.list() - - expect(data).to.have.lengthOf(2) - }) - - it('Should mute account of server 2', async function () { - await postCommand(servers[0], 'blockAccount', { handleToBlock: `root@${servers[1].host}` }) - - const { data } = await servers[0].videos.list() - - expect(data).to.have.lengthOf(1) - expect(data[0].name).to.equal('video server 1') - }) - - it('Should unmute account of server 2', async function () { - await postCommand(servers[0], 'unblockAccount', { handleToUnblock: `root@${servers[1].host}` }) - - const { data } = await servers[0].videos.list() - - expect(data).to.have.lengthOf(2) - }) - - it('Should blacklist video', async function () { - await postCommand(servers[0], 'blacklist', { videoUUID: videoUUIDServer1, unfederate: true }) - - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.videos.list() - - expect(data).to.have.lengthOf(1) - expect(data[0].name).to.equal('video server 2') - } - }) - - it('Should unblacklist video', async function () { - await postCommand(servers[0], 'unblacklist', { videoUUID: videoUUIDServer1 }) - - await waitJobs(servers) - - for (const server of servers) { - const { data } = await server.videos.list() - - expect(data).to.have.lengthOf(2) - } - }) - }) - - describe('Videos', function () { - let videoUUID: string - let videoPath: string - - before(async function () { - this.timeout(240000) - - await servers[0].config.enableTranscoding() - - const res = await servers[0].videos.quickUpload({ name: 'video1' }) - videoUUID = res.uuid - - await waitJobs(servers) - }) - - it('Should get video files', async function () { - const { body } = await makeGetRequest({ - url: servers[0].url, - path: '/plugins/test-four/router/video-files/' + videoUUID, - expectedStatus: HttpStatusCode.OK_200 - }) - - // Video files check - { - expect(body.webVideo.videoFiles).to.be.an('array') - expect(body.hls.videoFiles).to.be.an('array') - - for (const resolution of [ 144, 240, 360, 480, 720 ]) { - for (const files of [ body.webVideo.videoFiles, body.hls.videoFiles ]) { - const file = files.find(f => f.resolution === resolution) - expect(file).to.exist - - expect(file.size).to.be.a('number') - expect(file.fps).to.equal(25) - - expect(await pathExists(file.path)).to.be.true - await makeRawRequest({ url: file.url, expectedStatus: HttpStatusCode.OK_200 }) - } - } - - videoPath = body.webVideo.videoFiles[0].path - } - - // Thumbnails check - { - expect(body.thumbnails).to.be.an('array') - - const miniature = body.thumbnails.find(t => t.type === ThumbnailType.MINIATURE) - expect(miniature).to.exist - expect(await pathExists(miniature.path)).to.be.true - await makeRawRequest({ url: miniature.url, expectedStatus: HttpStatusCode.OK_200 }) - - const preview = body.thumbnails.find(t => t.type === ThumbnailType.PREVIEW) - expect(preview).to.exist - expect(await pathExists(preview.path)).to.be.true - await makeRawRequest({ url: preview.url, expectedStatus: HttpStatusCode.OK_200 }) - } - }) - - it('Should probe a file', async function () { - const { body } = await makeGetRequest({ - url: servers[0].url, - path: '/plugins/test-four/router/ffprobe', - query: { - path: videoPath - }, - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(body.streams).to.be.an('array') - expect(body.streams).to.have.lengthOf(2) - }) - - it('Should remove a video after a view', async function () { - this.timeout(40000) - - // Should not throw -> video exists - const video = await servers[0].videos.get({ id: videoUUID }) - // Should delete the video - await servers[0].views.simulateView({ id: videoUUID }) - - await servers[0].servers.waitUntilLog('Video deleted by plugin four.') - - try { - // Should throw because the video should have been deleted - await servers[0].videos.get({ id: videoUUID }) - throw new Error('Video exists') - } catch (err) { - if (err.message.includes('exists')) throw err - } - - await checkVideoFilesWereRemoved({ server: servers[0], video }) - }) - - it('Should have fetched the video by URL', async function () { - await servers[0].servers.waitUntilLog(`video from DB uuid is ${videoUUID}`) - }) - }) - - after(async function () { - await cleanupTests(servers) - }) -}) diff --git a/server/tests/plugins/plugin-router.ts b/server/tests/plugins/plugin-router.ts deleted file mode 100644 index 40b15eb79..000000000 --- a/server/tests/plugins/plugin-router.ts +++ /dev/null @@ -1,105 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { - cleanupTests, - createSingleServer, - makeGetRequest, - makePostBodyRequest, - PeerTubeServer, - PluginsCommand, - setAccessTokensToServers -} from '@shared/server-commands' -import { HttpStatusCode } from '@shared/models' - -describe('Test plugin helpers', function () { - let server: PeerTubeServer - const basePaths = [ - '/plugins/test-five/router/', - '/plugins/test-five/0.0.1/router/' - ] - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - - await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-five') }) - }) - - it('Should answer "pong"', async function () { - for (const path of basePaths) { - const res = await makeGetRequest({ - url: server.url, - path: path + 'ping', - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(res.body.message).to.equal('pong') - } - }) - - it('Should check if authenticated', async function () { - for (const path of basePaths) { - const res = await makeGetRequest({ - url: server.url, - path: path + 'is-authenticated', - token: server.accessToken, - expectedStatus: 200 - }) - - expect(res.body.isAuthenticated).to.equal(true) - - const secRes = await makeGetRequest({ - url: server.url, - path: path + 'is-authenticated', - expectedStatus: 200 - }) - - expect(secRes.body.isAuthenticated).to.equal(false) - } - }) - - it('Should mirror post body', async function () { - const body = { - hello: 'world', - riri: 'fifi', - loulou: 'picsou' - } - - for (const path of basePaths) { - const res = await makePostBodyRequest({ - url: server.url, - path: path + 'form/post/mirror', - fields: body, - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(res.body).to.deep.equal(body) - } - }) - - it('Should remove the plugin and remove the routes', async function () { - await server.plugins.uninstall({ npmName: 'peertube-plugin-test-five' }) - - for (const path of basePaths) { - await makeGetRequest({ - url: server.url, - path: path + 'ping', - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - - await makePostBodyRequest({ - url: server.url, - path: path + 'ping', - fields: {}, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - } - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/plugins/plugin-storage.ts b/server/tests/plugins/plugin-storage.ts deleted file mode 100644 index 112652a1f..000000000 --- a/server/tests/plugins/plugin-storage.ts +++ /dev/null @@ -1,94 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { pathExists, readdir, readFile } from 'fs-extra' -import { join } from 'path' -import { - cleanupTests, - createSingleServer, - makeGetRequest, - PeerTubeServer, - PluginsCommand, - setAccessTokensToServers -} from '@shared/server-commands' -import { HttpStatusCode } from '@shared/models' - -describe('Test plugin storage', function () { - let server: PeerTubeServer - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - - await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-six') }) - }) - - describe('DB storage', function () { - it('Should correctly store a subkey', async function () { - await server.servers.waitUntilLog('superkey stored value is toto') - }) - - it('Should correctly retrieve an array as array from the storage.', async function () { - await server.servers.waitUntilLog('storedArrayKey isArray is true') - await server.servers.waitUntilLog('storedArrayKey stored value is toto, toto2') - }) - }) - - describe('Disk storage', function () { - let dataPath: string - let pluginDataPath: string - - async function getFileContent () { - const files = await readdir(pluginDataPath) - expect(files).to.have.lengthOf(1) - - return readFile(join(pluginDataPath, files[0]), 'utf8') - } - - before(function () { - dataPath = server.servers.buildDirectory('plugins/data') - pluginDataPath = join(dataPath, 'peertube-plugin-test-six') - }) - - it('Should have created the directory on install', async function () { - const dataPath = server.servers.buildDirectory('plugins/data') - const pluginDataPath = join(dataPath, 'peertube-plugin-test-six') - - expect(await pathExists(dataPath)).to.be.true - expect(await pathExists(pluginDataPath)).to.be.true - expect(await readdir(pluginDataPath)).to.have.lengthOf(0) - }) - - it('Should have created a file', async function () { - await makeGetRequest({ - url: server.url, - token: server.accessToken, - path: '/plugins/test-six/router/create-file', - expectedStatus: HttpStatusCode.OK_200 - }) - - const content = await getFileContent() - expect(content).to.equal('Prince Ali') - }) - - it('Should still have the file after an uninstallation', async function () { - await server.plugins.uninstall({ npmName: 'peertube-plugin-test-six' }) - - const content = await getFileContent() - expect(content).to.equal('Prince Ali') - }) - - it('Should still have the file after the reinstallation', async function () { - await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-six') }) - - const content = await getFileContent() - expect(content).to.equal('Prince Ali') - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/plugins/plugin-transcoding.ts b/server/tests/plugins/plugin-transcoding.ts deleted file mode 100644 index 21f82fbac..000000000 --- a/server/tests/plugins/plugin-transcoding.ts +++ /dev/null @@ -1,279 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { getAudioStream, getVideoStream, getVideoStreamFPS } from '@shared/ffmpeg' -import { VideoPrivacy } from '@shared/models' -import { - cleanupTests, - createSingleServer, - PeerTubeServer, - PluginsCommand, - setAccessTokensToServers, - setDefaultVideoChannel, - testFfmpegStreamError, - waitJobs -} from '@shared/server-commands' - -async function createLiveWrapper (server: PeerTubeServer) { - const liveAttributes = { - name: 'live video', - channelId: server.store.channel.id, - privacy: VideoPrivacy.PUBLIC - } - - const { uuid } = await server.live.create({ fields: liveAttributes }) - - return uuid -} - -function updateConf (server: PeerTubeServer, vodProfile: string, liveProfile: string) { - return server.config.updateCustomSubConfig({ - newConfig: { - transcoding: { - enabled: true, - profile: vodProfile, - hls: { - enabled: true - }, - webVideos: { - enabled: true - }, - resolutions: { - '240p': true, - '360p': false, - '480p': false, - '720p': true - } - }, - live: { - transcoding: { - profile: liveProfile, - enabled: true, - resolutions: { - '240p': true, - '360p': false, - '480p': false, - '720p': true - } - } - } - } - }) -} - -describe('Test transcoding plugins', function () { - let server: PeerTubeServer - - before(async function () { - this.timeout(60000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - await setDefaultVideoChannel([ server ]) - - await updateConf(server, 'default', 'default') - }) - - describe('When using a plugin adding profiles to existing encoders', function () { - - async function checkVideoFPS (uuid: string, type: 'above' | 'below', fps: number) { - const video = await server.videos.get({ id: uuid }) - const files = video.files.concat(...video.streamingPlaylists.map(p => p.files)) - - for (const file of files) { - if (type === 'above') { - expect(file.fps).to.be.above(fps) - } else { - expect(file.fps).to.be.below(fps) - } - } - } - - async function checkLiveFPS (uuid: string, type: 'above' | 'below', fps: number) { - const playlistUrl = `${server.url}/static/streaming-playlists/hls/${uuid}/0.m3u8` - const videoFPS = await getVideoStreamFPS(playlistUrl) - - if (type === 'above') { - expect(videoFPS).to.be.above(fps) - } else { - expect(videoFPS).to.be.below(fps) - } - } - - before(async function () { - await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-transcoding-one') }) - }) - - it('Should have the appropriate available profiles', async function () { - const config = await server.config.getConfig() - - expect(config.transcoding.availableProfiles).to.have.members([ 'default', 'low-vod', 'input-options-vod', 'bad-scale-vod' ]) - expect(config.live.transcoding.availableProfiles).to.have.members([ 'default', 'high-live', 'input-options-live', 'bad-scale-live' ]) - }) - - describe('VOD', function () { - - it('Should not use the plugin profile if not chosen by the admin', async function () { - this.timeout(240000) - - const videoUUID = (await server.videos.quickUpload({ name: 'video' })).uuid - await waitJobs([ server ]) - - await checkVideoFPS(videoUUID, 'above', 20) - }) - - it('Should use the vod profile', async function () { - this.timeout(240000) - - await updateConf(server, 'low-vod', 'default') - - const videoUUID = (await server.videos.quickUpload({ name: 'video' })).uuid - await waitJobs([ server ]) - - await checkVideoFPS(videoUUID, 'below', 12) - }) - - it('Should apply input options in vod profile', async function () { - this.timeout(240000) - - await updateConf(server, 'input-options-vod', 'default') - - const videoUUID = (await server.videos.quickUpload({ name: 'video' })).uuid - await waitJobs([ server ]) - - await checkVideoFPS(videoUUID, 'below', 6) - }) - - it('Should apply the scale filter in vod profile', async function () { - this.timeout(240000) - - await updateConf(server, 'bad-scale-vod', 'default') - - const videoUUID = (await server.videos.quickUpload({ name: 'video' })).uuid - await waitJobs([ server ]) - - // Transcoding failed - const video = await server.videos.get({ id: videoUUID }) - expect(video.files).to.have.lengthOf(1) - expect(video.streamingPlaylists).to.have.lengthOf(0) - }) - }) - - describe('Live', function () { - - it('Should not use the plugin profile if not chosen by the admin', async function () { - this.timeout(240000) - - const liveVideoId = await createLiveWrapper(server) - - await server.live.sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_very_short_240p.mp4' }) - await server.live.waitUntilPublished({ videoId: liveVideoId }) - await waitJobs([ server ]) - - await checkLiveFPS(liveVideoId, 'above', 20) - }) - - it('Should use the live profile', async function () { - this.timeout(240000) - - await updateConf(server, 'low-vod', 'high-live') - - const liveVideoId = await createLiveWrapper(server) - - await server.live.sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_very_short_240p.mp4' }) - await server.live.waitUntilPublished({ videoId: liveVideoId }) - await waitJobs([ server ]) - - await checkLiveFPS(liveVideoId, 'above', 45) - }) - - it('Should apply the input options on live profile', async function () { - this.timeout(240000) - - await updateConf(server, 'low-vod', 'input-options-live') - - const liveVideoId = await createLiveWrapper(server) - - await server.live.sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_very_short_240p.mp4' }) - await server.live.waitUntilPublished({ videoId: liveVideoId }) - await waitJobs([ server ]) - - await checkLiveFPS(liveVideoId, 'above', 45) - }) - - it('Should apply the scale filter name on live profile', async function () { - this.timeout(240000) - - await updateConf(server, 'low-vod', 'bad-scale-live') - - const liveVideoId = await createLiveWrapper(server) - - const command = await server.live.sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_very_short_240p.mp4' }) - await testFfmpegStreamError(command, true) - }) - - it('Should default to the default profile if the specified profile does not exist', async function () { - this.timeout(240000) - - await server.plugins.uninstall({ npmName: 'peertube-plugin-test-transcoding-one' }) - - const config = await server.config.getConfig() - - expect(config.transcoding.availableProfiles).to.deep.equal([ 'default' ]) - expect(config.live.transcoding.availableProfiles).to.deep.equal([ 'default' ]) - - const videoUUID = (await server.videos.quickUpload({ name: 'video', fixture: 'video_very_short_240p.mp4' })).uuid - await waitJobs([ server ]) - - await checkVideoFPS(videoUUID, 'above', 20) - }) - }) - - }) - - describe('When using a plugin adding new encoders', function () { - - before(async function () { - await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-transcoding-two') }) - - await updateConf(server, 'test-vod-profile', 'test-live-profile') - }) - - it('Should use the new vod encoders', async function () { - this.timeout(240000) - - const videoUUID = (await server.videos.quickUpload({ name: 'video', fixture: 'video_very_short_240p.mp4' })).uuid - await waitJobs([ server ]) - - const video = await server.videos.get({ id: videoUUID }) - - const path = server.servers.buildWebVideoFilePath(video.files[0].fileUrl) - const audioProbe = await getAudioStream(path) - expect(audioProbe.audioStream.codec_name).to.equal('opus') - - const videoProbe = await getVideoStream(path) - expect(videoProbe.codec_name).to.equal('vp9') - }) - - it('Should use the new live encoders', async function () { - this.timeout(240000) - - const liveVideoId = await createLiveWrapper(server) - - await server.live.sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_short2.webm' }) - await server.live.waitUntilPublished({ videoId: liveVideoId }) - await waitJobs([ server ]) - - const playlistUrl = `${server.url}/static/streaming-playlists/hls/${liveVideoId}/0.m3u8` - const audioProbe = await getAudioStream(playlistUrl) - expect(audioProbe.audioStream.codec_name).to.equal('opus') - - const videoProbe = await getVideoStream(playlistUrl) - expect(videoProbe.codec_name).to.equal('h264') - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/plugins/plugin-unloading.ts b/server/tests/plugins/plugin-unloading.ts deleted file mode 100644 index 5aca1a0c0..000000000 --- a/server/tests/plugins/plugin-unloading.ts +++ /dev/null @@ -1,75 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { - cleanupTests, - createSingleServer, - makeGetRequest, - PeerTubeServer, - PluginsCommand, - setAccessTokensToServers -} from '@shared/server-commands' -import { HttpStatusCode } from '@shared/models' - -describe('Test plugins module unloading', function () { - let server: PeerTubeServer = null - const requestPath = '/plugins/test-unloading/router/get' - let value: string = null - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - - await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-unloading') }) - }) - - it('Should return a numeric value', async function () { - const res = await makeGetRequest({ - url: server.url, - path: requestPath, - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(res.body.message).to.match(/^\d+$/) - value = res.body.message - }) - - it('Should return the same value the second time', async function () { - const res = await makeGetRequest({ - url: server.url, - path: requestPath, - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(res.body.message).to.be.equal(value) - }) - - it('Should uninstall the plugin and free the route', async function () { - await server.plugins.uninstall({ npmName: 'peertube-plugin-test-unloading' }) - - await makeGetRequest({ - url: server.url, - path: requestPath, - expectedStatus: HttpStatusCode.NOT_FOUND_404 - }) - }) - - it('Should return a different numeric value', async function () { - await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-unloading') }) - - const res = await makeGetRequest({ - url: server.url, - path: requestPath, - expectedStatus: HttpStatusCode.OK_200 - }) - - expect(res.body.message).to.match(/^\d+$/) - expect(res.body.message).to.be.not.equal(value) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/plugins/plugin-websocket.ts b/server/tests/plugins/plugin-websocket.ts deleted file mode 100644 index adaa28b1d..000000000 --- a/server/tests/plugins/plugin-websocket.ts +++ /dev/null @@ -1,70 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import WebSocket from 'ws' -import { cleanupTests, createSingleServer, PeerTubeServer, PluginsCommand, setAccessTokensToServers } from '@shared/server-commands' - -function buildWebSocket (server: PeerTubeServer, path: string) { - return new WebSocket('ws://' + server.host + path) -} - -function expectErrorOrTimeout (server: PeerTubeServer, path: string, expectedTimeout: number) { - return new Promise((res, rej) => { - const ws = buildWebSocket(server, path) - ws.on('error', () => res()) - - const timeout = setTimeout(() => res(), expectedTimeout) - - ws.on('open', () => { - clearTimeout(timeout) - - return rej(new Error('Connect did not timeout')) - }) - }) -} - -describe('Test plugin websocket', function () { - let server: PeerTubeServer - const basePaths = [ - '/plugins/test-websocket/ws/', - '/plugins/test-websocket/0.0.1/ws/' - ] - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - - await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-websocket') }) - }) - - it('Should not connect to the websocket without the appropriate path', async function () { - const paths = [ - '/plugins/unknown/ws/', - '/plugins/unknown/0.0.1/ws/' - ] - - for (const path of paths) { - await expectErrorOrTimeout(server, path, 1000) - } - }) - - it('Should not connect to the websocket without the appropriate sub path', async function () { - for (const path of basePaths) { - await expectErrorOrTimeout(server, path + '/unknown', 1000) - } - }) - - it('Should connect to the websocket and receive pong', function (done) { - const ws = buildWebSocket(server, basePaths[0]) - - ws.on('open', () => ws.send('ping')) - ws.on('message', data => { - if (data.toString() === 'pong') return done() - }) - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/plugins/translations.ts b/server/tests/plugins/translations.ts deleted file mode 100644 index 67e4683f8..000000000 --- a/server/tests/plugins/translations.ts +++ /dev/null @@ -1,74 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { cleanupTests, createSingleServer, PeerTubeServer, PluginsCommand, setAccessTokensToServers } from '@shared/server-commands' - -describe('Test plugin translations', function () { - let server: PeerTubeServer - let command: PluginsCommand - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - - command = server.plugins - - await command.install({ path: PluginsCommand.getPluginTestPath() }) - await command.install({ path: PluginsCommand.getPluginTestPath('-filter-translations') }) - }) - - it('Should not have translations for locale pt', async function () { - const body = await command.getTranslations({ locale: 'pt' }) - - expect(body).to.deep.equal({}) - }) - - it('Should have translations for locale fr', async function () { - const body = await command.getTranslations({ locale: 'fr-FR' }) - - expect(body).to.deep.equal({ - 'peertube-plugin-test': { - Hi: 'Coucou' - }, - 'peertube-plugin-test-filter-translations': { - 'Hello world': 'Bonjour le monde' - } - }) - }) - - it('Should have translations of locale it', async function () { - const body = await command.getTranslations({ locale: 'it-IT' }) - - expect(body).to.deep.equal({ - 'peertube-plugin-test-filter-translations': { - 'Hello world': 'Ciao, mondo!' - } - }) - }) - - it('Should remove the plugin and remove the locales', async function () { - await command.uninstall({ npmName: 'peertube-plugin-test-filter-translations' }) - - { - const body = await command.getTranslations({ locale: 'fr-FR' }) - - expect(body).to.deep.equal({ - 'peertube-plugin-test': { - Hi: 'Coucou' - } - }) - } - - { - const body = await command.getTranslations({ locale: 'it-IT' }) - - expect(body).to.deep.equal({}) - } - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/plugins/video-constants.ts b/server/tests/plugins/video-constants.ts deleted file mode 100644 index c388f02d1..000000000 --- a/server/tests/plugins/video-constants.ts +++ /dev/null @@ -1,180 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { - cleanupTests, - createSingleServer, - makeGetRequest, - PeerTubeServer, - PluginsCommand, - setAccessTokensToServers -} from '@shared/server-commands' -import { HttpStatusCode, VideoPlaylistPrivacy } from '@shared/models' - -describe('Test plugin altering video constants', function () { - let server: PeerTubeServer - - before(async function () { - this.timeout(30000) - - server = await createSingleServer(1) - await setAccessTokensToServers([ server ]) - - await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-video-constants') }) - }) - - it('Should have updated languages', async function () { - const languages = await server.videos.getLanguages() - - expect(languages['en']).to.not.exist - expect(languages['fr']).to.not.exist - - expect(languages['al_bhed']).to.equal('Al Bhed') - expect(languages['al_bhed2']).to.equal('Al Bhed 2') - expect(languages['al_bhed3']).to.not.exist - }) - - it('Should have updated categories', async function () { - const categories = await server.videos.getCategories() - - expect(categories[1]).to.not.exist - expect(categories[2]).to.not.exist - - expect(categories[42]).to.equal('Best category') - expect(categories[43]).to.equal('High best category') - }) - - it('Should have updated licences', async function () { - const licences = await server.videos.getLicences() - - expect(licences[1]).to.not.exist - expect(licences[7]).to.not.exist - - expect(licences[42]).to.equal('Best licence') - expect(licences[43]).to.equal('High best licence') - }) - - it('Should have updated video privacies', async function () { - const privacies = await server.videos.getPrivacies() - - expect(privacies[1]).to.exist - expect(privacies[2]).to.not.exist - expect(privacies[3]).to.exist - expect(privacies[4]).to.exist - }) - - it('Should have updated playlist privacies', async function () { - const playlistPrivacies = await server.playlists.getPrivacies() - - expect(playlistPrivacies[1]).to.exist - expect(playlistPrivacies[2]).to.exist - expect(playlistPrivacies[3]).to.not.exist - }) - - it('Should not be able to create a video with this privacy', async function () { - const attributes = { name: 'video', privacy: 2 } - await server.videos.upload({ attributes, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should not be able to create a video with this privacy', async function () { - const attributes = { displayName: 'video playlist', privacy: VideoPlaylistPrivacy.PRIVATE } - await server.playlists.create({ attributes, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) - }) - - it('Should be able to upload a video with these values', async function () { - const attributes = { name: 'video', category: 42, licence: 42, language: 'al_bhed2' } - const { uuid } = await server.videos.upload({ attributes }) - - const video = await server.videos.get({ id: uuid }) - expect(video.language.label).to.equal('Al Bhed 2') - expect(video.licence.label).to.equal('Best licence') - expect(video.category.label).to.equal('Best category') - }) - - it('Should uninstall the plugin and reset languages, categories, licences and privacies', async function () { - await server.plugins.uninstall({ npmName: 'peertube-plugin-test-video-constants' }) - - { - const languages = await server.videos.getLanguages() - - expect(languages['en']).to.equal('English') - expect(languages['fr']).to.equal('French') - - expect(languages['al_bhed']).to.not.exist - expect(languages['al_bhed2']).to.not.exist - expect(languages['al_bhed3']).to.not.exist - } - - { - const categories = await server.videos.getCategories() - - expect(categories[1]).to.equal('Music') - expect(categories[2]).to.equal('Films') - - expect(categories[42]).to.not.exist - expect(categories[43]).to.not.exist - } - - { - const licences = await server.videos.getLicences() - - expect(licences[1]).to.equal('Attribution') - expect(licences[7]).to.equal('Public Domain Dedication') - - expect(licences[42]).to.not.exist - expect(licences[43]).to.not.exist - } - - { - const privacies = await server.videos.getPrivacies() - - expect(privacies[1]).to.exist - expect(privacies[2]).to.exist - expect(privacies[3]).to.exist - expect(privacies[4]).to.exist - } - - { - const playlistPrivacies = await server.playlists.getPrivacies() - - expect(playlistPrivacies[1]).to.exist - expect(playlistPrivacies[2]).to.exist - expect(playlistPrivacies[3]).to.exist - } - }) - - it('Should be able to reset categories', async function () { - await server.plugins.install({ path: PluginsCommand.getPluginTestPath('-video-constants') }) - - { - const categories = await server.videos.getCategories() - - expect(categories[1]).to.not.exist - expect(categories[2]).to.not.exist - - expect(categories[42]).to.exist - expect(categories[43]).to.exist - } - - await makeGetRequest({ - url: server.url, - token: server.accessToken, - path: '/plugins/test-video-constants/router/reset-categories', - expectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - - { - const categories = await server.videos.getCategories() - - expect(categories[1]).to.exist - expect(categories[2]).to.exist - - expect(categories[42]).to.not.exist - expect(categories[43]).to.not.exist - } - }) - - after(async function () { - await cleanupTests([ server ]) - }) -}) diff --git a/server/tests/shared/actors.ts b/server/tests/shared/actors.ts deleted file mode 100644 index 41fd72e89..000000000 --- a/server/tests/shared/actors.ts +++ /dev/null @@ -1,69 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { pathExists, readdir } from 'fs-extra' -import { Account, VideoChannel } from '@shared/models' -import { PeerTubeServer } from '@shared/server-commands' - -async function expectChannelsFollows (options: { - server: PeerTubeServer - handle: string - followers: number - following: number -}) { - const { server } = options - const { data } = await server.channels.list() - - return expectActorFollow({ ...options, data }) -} - -async function expectAccountFollows (options: { - server: PeerTubeServer - handle: string - followers: number - following: number -}) { - const { server } = options - const { data } = await server.accounts.list() - - return expectActorFollow({ ...options, data }) -} - -async function checkActorFilesWereRemoved (filename: string, server: PeerTubeServer) { - for (const directory of [ 'avatars' ]) { - const directoryPath = server.getDirectoryPath(directory) - - const directoryExists = await pathExists(directoryPath) - expect(directoryExists).to.be.true - - const files = await readdir(directoryPath) - for (const file of files) { - expect(file).to.not.contain(filename) - } - } -} - -export { - expectAccountFollows, - expectChannelsFollows, - checkActorFilesWereRemoved -} - -// --------------------------------------------------------------------------- - -function expectActorFollow (options: { - server: PeerTubeServer - data: (Account | VideoChannel)[] - handle: string - followers: number - following: number -}) { - const { server, data, handle, followers, following } = options - - const actor = data.find(a => a.name + '@' + a.host === handle) - const message = `${handle} on ${server.url}` - - expect(actor, message).to.exist - expect(actor.followersCount).to.equal(followers, message) - expect(actor.followingCount).to.equal(following, message) -} diff --git a/server/tests/shared/captions.ts b/server/tests/shared/captions.ts deleted file mode 100644 index 35e722408..000000000 --- a/server/tests/shared/captions.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { expect } from 'chai' -import request from 'supertest' -import { HttpStatusCode } from '@shared/models' - -async function testCaptionFile (url: string, captionPath: string, toTest: RegExp | string) { - const res = await request(url) - .get(captionPath) - .expect(HttpStatusCode.OK_200) - - if (toTest instanceof RegExp) { - expect(res.text).to.match(toTest) - } else { - expect(res.text).to.contain(toTest) - } -} - -// --------------------------------------------------------------------------- - -export { - testCaptionFile -} diff --git a/server/tests/shared/checks.ts b/server/tests/shared/checks.ts deleted file mode 100644 index 90179c6ac..000000000 --- a/server/tests/shared/checks.ts +++ /dev/null @@ -1,174 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */ - -import { expect } from 'chai' -import { pathExists, readFile } from 'fs-extra' -import JPEG from 'jpeg-js' -import { join } from 'path' -import pixelmatch from 'pixelmatch' -import { PNG } from 'pngjs' -import { root } from '@shared/core-utils' -import { HttpStatusCode } from '@shared/models' -import { makeGetRequest, PeerTubeServer } from '@shared/server-commands' - -// Default interval -> 5 minutes -function dateIsValid (dateString: string | Date, interval = 300000) { - const dateToCheck = new Date(dateString) - const now = new Date() - - return Math.abs(now.getTime() - dateToCheck.getTime()) <= interval -} - -function expectStartWith (str: string, start: string) { - expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.true -} - -function expectNotStartWith (str: string, start: string) { - expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.false -} - -function expectEndWith (str: string, end: string) { - expect(str.endsWith(end), `${str} does not end with ${end}`).to.be.true -} - -// --------------------------------------------------------------------------- - -async function expectLogDoesNotContain (server: PeerTubeServer, str: string) { - const content = await server.servers.getLogContent() - - expect(content.toString()).to.not.contain(str) -} - -async function expectLogContain (server: PeerTubeServer, str: string) { - const content = await server.servers.getLogContent() - - expect(content.toString()).to.contain(str) -} - -async function testImageSize (url: string, imageName: string, imageHTTPPath: string, extension = '.jpg') { - const res = await makeGetRequest({ - url, - path: imageHTTPPath, - expectedStatus: HttpStatusCode.OK_200 - }) - - const body = res.body - - const data = await readFile(join(root(), 'server', 'tests', 'fixtures', imageName + extension)) - const minLength = data.length - ((40 * data.length) / 100) - const maxLength = data.length + ((40 * data.length) / 100) - - expect(body.length).to.be.above(minLength, 'the generated image is way smaller than the recorded fixture') - expect(body.length).to.be.below(maxLength, 'the generated image is way larger than the recorded fixture') -} - -async function testImageGeneratedByFFmpeg (url: string, imageName: string, imageHTTPPath: string, extension = '.jpg') { - if (process.env.ENABLE_FFMPEG_THUMBNAIL_PIXEL_COMPARISON_TESTS !== 'true') { - console.log( - 'Pixel comparison of image generated by ffmpeg is disabled. ' + - 'You can enable it using `ENABLE_FFMPEG_THUMBNAIL_PIXEL_COMPARISON_TESTS=true env variable') - } - - return testImage(url, imageName, imageHTTPPath, extension) -} - -async function testImage (url: string, imageName: string, imageHTTPPath: string, extension = '.jpg') { - const res = await makeGetRequest({ - url, - path: imageHTTPPath, - expectedStatus: HttpStatusCode.OK_200 - }) - - const body = res.body - const data = await readFile(join(root(), 'server', 'tests', 'fixtures', imageName + extension)) - - const img1 = imageHTTPPath.endsWith('.png') - ? PNG.sync.read(body) - : JPEG.decode(body) - - const img2 = extension === '.png' - ? PNG.sync.read(data) - : JPEG.decode(data) - - const result = pixelmatch(img1.data, img2.data, null, img1.width, img1.height, { threshold: 0.1 }) - - expect(result).to.equal(0, `${imageHTTPPath} image is not the same as ${imageName}${extension}`) -} - -async function testFileExistsOrNot (server: PeerTubeServer, directory: string, filePath: string, exist: boolean) { - const base = server.servers.buildDirectory(directory) - - expect(await pathExists(join(base, filePath))).to.equal(exist) -} - -// --------------------------------------------------------------------------- - -function checkBadStartPagination (url: string, path: string, token?: string, query = {}) { - return makeGetRequest({ - url, - path, - token, - query: { ...query, start: 'hello' }, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) -} - -async function checkBadCountPagination (url: string, path: string, token?: string, query = {}) { - await makeGetRequest({ - url, - path, - token, - query: { ...query, count: 'hello' }, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) - - await makeGetRequest({ - url, - path, - token, - query: { ...query, count: 2000 }, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) -} - -function checkBadSortPagination (url: string, path: string, token?: string, query = {}) { - return makeGetRequest({ - url, - path, - token, - query: { ...query, sort: 'hello' }, - expectedStatus: HttpStatusCode.BAD_REQUEST_400 - }) -} - -// --------------------------------------------------------------------------- - -async function checkVideoDuration (server: PeerTubeServer, videoUUID: string, duration: number) { - const video = await server.videos.get({ id: videoUUID }) - - expect(video.duration).to.be.approximately(duration, 1) - - for (const file of video.files) { - const metadata = await server.videos.getFileMetadata({ url: file.metadataUrl }) - - for (const stream of metadata.streams) { - expect(Math.round(stream.duration)).to.be.approximately(duration, 1) - } - } -} - -export { - dateIsValid, - testImageGeneratedByFFmpeg, - testImageSize, - testImage, - expectLogDoesNotContain, - testFileExistsOrNot, - expectStartWith, - expectNotStartWith, - expectEndWith, - checkBadStartPagination, - checkBadCountPagination, - checkBadSortPagination, - checkVideoDuration, - expectLogContain -} diff --git a/server/tests/shared/directories.ts b/server/tests/shared/directories.ts deleted file mode 100644 index 5ad12d78a..000000000 --- a/server/tests/shared/directories.ts +++ /dev/null @@ -1,43 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { pathExists, readdir } from 'fs-extra' -import { homedir } from 'os' -import { join } from 'path' -import { PeerTubeServer } from '@shared/server-commands' -import { PeerTubeRunnerProcess } from './peertube-runner-process' - -export async function checkTmpIsEmpty (server: PeerTubeServer) { - await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls', 'resumable-uploads' ]) - - if (await pathExists(server.getDirectoryPath('tmp/hls'))) { - await checkDirectoryIsEmpty(server, 'tmp/hls') - } -} - -export async function checkPersistentTmpIsEmpty (server: PeerTubeServer) { - await checkDirectoryIsEmpty(server, 'tmp-persistent') -} - -export async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) { - const directoryPath = server.getDirectoryPath(directory) - - const directoryExists = await pathExists(directoryPath) - expect(directoryExists).to.be.true - - const files = await readdir(directoryPath) - const filtered = files.filter(f => exceptions.includes(f) === false) - - expect(filtered).to.have.lengthOf(0) -} - -export async function checkPeerTubeRunnerCacheIsEmpty (runner: PeerTubeRunnerProcess) { - const directoryPath = join(homedir(), '.cache', 'peertube-runner-nodejs', runner.getId(), 'transcoding') - - const directoryExists = await pathExists(directoryPath) - expect(directoryExists).to.be.true - - const files = await readdir(directoryPath) - - expect(files, 'Directory content: ' + files.join(', ')).to.have.lengthOf(0) -} diff --git a/server/tests/shared/generate.ts b/server/tests/shared/generate.ts deleted file mode 100644 index 3788b049f..000000000 --- a/server/tests/shared/generate.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { expect } from 'chai' -import ffmpeg from 'fluent-ffmpeg' -import { ensureDir, pathExists } from 'fs-extra' -import { dirname } from 'path' -import { buildAbsoluteFixturePath, getMaxTheoreticalBitrate } from '@shared/core-utils' -import { getVideoStreamBitrate, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@shared/ffmpeg' - -async function ensureHasTooBigBitrate (fixturePath: string) { - const bitrate = await getVideoStreamBitrate(fixturePath) - const dataResolution = await getVideoStreamDimensionsInfo(fixturePath) - const fps = await getVideoStreamFPS(fixturePath) - - const maxBitrate = getMaxTheoreticalBitrate({ ...dataResolution, fps }) - expect(bitrate).to.be.above(maxBitrate) -} - -async function generateHighBitrateVideo () { - const tempFixturePath = buildAbsoluteFixturePath('video_high_bitrate_1080p.mp4', true) - - await ensureDir(dirname(tempFixturePath)) - - const exists = await pathExists(tempFixturePath) - if (!exists) { - console.log('Generating high bitrate video.') - - // Generate a random, high bitrate video on the fly, so we don't have to include - // a large file in the repo. The video needs to have a certain minimum length so - // that FFmpeg properly applies bitrate limits. - // https://stackoverflow.com/a/15795112 - return new Promise((res, rej) => { - ffmpeg() - .outputOptions([ '-f rawvideo', '-video_size 1920x1080', '-i /dev/urandom' ]) - .outputOptions([ '-ac 2', '-f s16le', '-i /dev/urandom', '-t 10' ]) - .outputOptions([ '-maxrate 10M', '-bufsize 10M' ]) - .output(tempFixturePath) - .on('error', rej) - .on('end', () => res(tempFixturePath)) - .run() - }) - } - - await ensureHasTooBigBitrate(tempFixturePath) - - return tempFixturePath -} - -async function generateVideoWithFramerate (fps = 60) { - const tempFixturePath = buildAbsoluteFixturePath(`video_${fps}fps.mp4`, true) - - await ensureDir(dirname(tempFixturePath)) - - const exists = await pathExists(tempFixturePath) - if (!exists) { - console.log('Generating video with framerate %d.', fps) - - return new Promise((res, rej) => { - ffmpeg() - .outputOptions([ '-f rawvideo', '-video_size 1280x720', '-i /dev/urandom' ]) - .outputOptions([ '-ac 2', '-f s16le', '-i /dev/urandom', '-t 10' ]) - .outputOptions([ `-r ${fps}` ]) - .output(tempFixturePath) - .on('error', rej) - .on('end', () => res(tempFixturePath)) - .run() - }) - } - - return tempFixturePath -} - -export { - generateHighBitrateVideo, - generateVideoWithFramerate -} diff --git a/server/tests/shared/index.ts b/server/tests/shared/index.ts deleted file mode 100644 index eda24adb5..000000000 --- a/server/tests/shared/index.ts +++ /dev/null @@ -1,19 +0,0 @@ -export * from './mock-servers' -export * from './actors' -export * from './captions' -export * from './checks' -export * from './directories' -export * from './generate' -export * from './live' -export * from './notifications' -export * from './peertube-runner-process' -export * from './video-playlists' -export * from './plugins' -export * from './requests' -export * from './sql-command' -export * from './streaming-playlists' -export * from './tests' -export * from './tracker' -export * from './videos' -export * from './views' -export * from './webtorrent' diff --git a/server/tests/shared/live.ts b/server/tests/shared/live.ts deleted file mode 100644 index 9d8c1d941..000000000 --- a/server/tests/shared/live.ts +++ /dev/null @@ -1,185 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { pathExists, readdir } from 'fs-extra' -import { join } from 'path' -import { sha1 } from '@shared/extra-utils' -import { LiveVideo, VideoStreamingPlaylistType } from '@shared/models' -import { ObjectStorageCommand, PeerTubeServer } from '@shared/server-commands' -import { SQLCommand } from './sql-command' -import { checkLiveSegmentHash, checkResolutionsInMasterPlaylist } from './streaming-playlists' - -async function checkLiveCleanup (options: { - server: PeerTubeServer - videoUUID: string - permanent: boolean - savedResolutions?: number[] -}) { - const { server, videoUUID, permanent, savedResolutions = [] } = options - - const basePath = server.servers.buildDirectory('streaming-playlists') - const hlsPath = join(basePath, 'hls', videoUUID) - - if (permanent) { - if (!await pathExists(hlsPath)) return - - const files = await readdir(hlsPath) - expect(files).to.have.lengthOf(0) - return - } - - if (savedResolutions.length === 0) { - return checkUnsavedLiveCleanup(server, videoUUID, hlsPath) - } - - return checkSavedLiveCleanup(hlsPath, savedResolutions) -} - -// --------------------------------------------------------------------------- - -async function testLiveVideoResolutions (options: { - sqlCommand: SQLCommand - originServer: PeerTubeServer - - servers: PeerTubeServer[] - liveVideoId: string - resolutions: number[] - transcoded: boolean - - objectStorage?: ObjectStorageCommand - objectStorageBaseUrl?: string -}) { - const { - originServer, - sqlCommand, - servers, - liveVideoId, - resolutions, - transcoded, - objectStorage, - objectStorageBaseUrl = objectStorage?.getMockPlaylistBaseUrl() - } = options - - for (const server of servers) { - const { data } = await server.videos.list() - expect(data.find(v => v.uuid === liveVideoId)).to.exist - - const video = await server.videos.get({ id: liveVideoId }) - expect(video.streamingPlaylists).to.have.lengthOf(1) - - const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS) - expect(hlsPlaylist).to.exist - expect(hlsPlaylist.files).to.have.lengthOf(0) // Only fragmented mp4 files are displayed - - await checkResolutionsInMasterPlaylist({ - server, - playlistUrl: hlsPlaylist.playlistUrl, - resolutions, - transcoded, - withRetry: !!objectStorage - }) - - if (objectStorage) { - expect(hlsPlaylist.playlistUrl).to.contain(objectStorageBaseUrl) - } - - for (let i = 0; i < resolutions.length; i++) { - const segmentNum = 3 - const segmentName = `${i}-00000${segmentNum}.ts` - await originServer.live.waitUntilSegmentGeneration({ - server: originServer, - videoUUID: video.uuid, - playlistNumber: i, - segment: segmentNum, - objectStorage, - objectStorageBaseUrl - }) - - const baseUrl = objectStorage - ? join(objectStorageBaseUrl, 'hls') - : originServer.url + '/static/streaming-playlists/hls' - - if (objectStorage) { - expect(hlsPlaylist.segmentsSha256Url).to.contain(objectStorageBaseUrl) - } - - const subPlaylist = await originServer.streamingPlaylists.get({ - url: `${baseUrl}/${video.uuid}/${i}.m3u8`, - withRetry: !!objectStorage // With object storage, the request may fail because of inconsistent data in S3 - }) - - expect(subPlaylist).to.contain(segmentName) - - await checkLiveSegmentHash({ - server, - baseUrlSegment: baseUrl, - videoUUID: video.uuid, - segmentName, - hlsPlaylist, - withRetry: !!objectStorage // With object storage, the request may fail because of inconsistent data in S3 - }) - - if (originServer.internalServerNumber === server.internalServerNumber) { - const infohash = sha1(`${2 + hlsPlaylist.playlistUrl}+V${i}`) - const dbInfohashes = await sqlCommand.getPlaylistInfohash(hlsPlaylist.id) - - expect(dbInfohashes).to.include(infohash) - } - } - } -} - -// --------------------------------------------------------------------------- - -export { - checkLiveCleanup, - testLiveVideoResolutions -} - -// --------------------------------------------------------------------------- - -async function checkSavedLiveCleanup (hlsPath: string, savedResolutions: number[] = []) { - const files = await readdir(hlsPath) - - // fragmented file and playlist per resolution + master playlist + segments sha256 json file - expect(files, `Directory content: ${files.join(', ')}`).to.have.lengthOf(savedResolutions.length * 2 + 2) - - for (const resolution of savedResolutions) { - const fragmentedFile = files.find(f => f.endsWith(`-${resolution}-fragmented.mp4`)) - expect(fragmentedFile).to.exist - - const playlistFile = files.find(f => f.endsWith(`${resolution}.m3u8`)) - expect(playlistFile).to.exist - } - - const masterPlaylistFile = files.find(f => f.endsWith('-master.m3u8')) - expect(masterPlaylistFile).to.exist - - const shaFile = files.find(f => f.endsWith('-segments-sha256.json')) - expect(shaFile).to.exist -} - -async function checkUnsavedLiveCleanup (server: PeerTubeServer, videoUUID: string, hlsPath: string) { - let live: LiveVideo - - try { - live = await server.live.get({ videoId: videoUUID }) - } catch {} - - if (live?.permanentLive) { - expect(await pathExists(hlsPath)).to.be.true - - const hlsFiles = await readdir(hlsPath) - expect(hlsFiles).to.have.lengthOf(1) // Only replays directory - - const replayDir = join(hlsPath, 'replay') - expect(await pathExists(replayDir)).to.be.true - - const replayFiles = await readdir(join(hlsPath, 'replay')) - expect(replayFiles).to.have.lengthOf(0) - - return - } - - expect(await pathExists(hlsPath)).to.be.false -} diff --git a/server/tests/shared/mock-servers/index.ts b/server/tests/shared/mock-servers/index.ts deleted file mode 100644 index 1fa983116..000000000 --- a/server/tests/shared/mock-servers/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -export * from './mock-429' -export * from './mock-email' -export * from './mock-http' -export * from './mock-instances-index' -export * from './mock-joinpeertube-versions' -export * from './mock-object-storage' -export * from './mock-plugin-blocklist' -export * from './mock-proxy' diff --git a/server/tests/shared/mock-servers/mock-429.ts b/server/tests/shared/mock-servers/mock-429.ts deleted file mode 100644 index 1fc20b079..000000000 --- a/server/tests/shared/mock-servers/mock-429.ts +++ /dev/null @@ -1,33 +0,0 @@ -import express from 'express' -import { Server } from 'http' -import { getPort, randomListen, terminateServer } from './shared' - -export class Mock429 { - private server: Server - private responseSent = false - - async initialize () { - const app = express() - - app.get('/', (req: express.Request, res: express.Response, next: express.NextFunction) => { - - if (!this.responseSent) { - this.responseSent = true - - // Retry after 5 seconds - res.header('retry-after', '2') - return res.sendStatus(429) - } - - return res.sendStatus(200) - }) - - this.server = await randomListen(app) - - return getPort(this.server) - } - - terminate () { - return terminateServer(this.server) - } -} diff --git a/server/tests/shared/mock-servers/mock-email.ts b/server/tests/shared/mock-servers/mock-email.ts deleted file mode 100644 index 6eda2dfda..000000000 --- a/server/tests/shared/mock-servers/mock-email.ts +++ /dev/null @@ -1,61 +0,0 @@ -import MailDev from '@peertube/maildev' -import { parallelTests, randomInt } from '@shared/core-utils' - -class MockSmtpServer { - - private static instance: MockSmtpServer - private started = false - private maildev: any - private emails: object[] - - private constructor () { } - - collectEmails (emailsCollection: object[]) { - return new Promise((res, rej) => { - const port = parallelTests() ? randomInt(1025, 2000) : 1025 - this.emails = emailsCollection - - if (this.started) { - return res(undefined) - } - - this.maildev = new MailDev({ - ip: '127.0.0.1', - smtp: port, - disableWeb: true, - silent: true - }) - - this.maildev.on('new', email => { - this.emails.push(email) - }) - - this.maildev.listen(err => { - if (err) return rej(err) - - this.started = true - - return res(port) - }) - }) - } - - kill () { - if (!this.maildev) return - - this.maildev.close() - - this.maildev = null - MockSmtpServer.instance = null - } - - static get Instance () { - return this.instance || (this.instance = new this()) - } -} - -// --------------------------------------------------------------------------- - -export { - MockSmtpServer -} diff --git a/server/tests/shared/mock-servers/mock-http.ts b/server/tests/shared/mock-servers/mock-http.ts deleted file mode 100644 index b7a019e07..000000000 --- a/server/tests/shared/mock-servers/mock-http.ts +++ /dev/null @@ -1,23 +0,0 @@ -import express from 'express' -import { Server } from 'http' -import { getPort, randomListen, terminateServer } from './shared' - -export class MockHTTP { - private server: Server - - async initialize () { - const app = express() - - app.get('/*', (req: express.Request, res: express.Response, next: express.NextFunction) => { - return res.sendStatus(200) - }) - - this.server = await randomListen(app) - - return getPort(this.server) - } - - terminate () { - return terminateServer(this.server) - } -} diff --git a/server/tests/shared/mock-servers/mock-instances-index.ts b/server/tests/shared/mock-servers/mock-instances-index.ts deleted file mode 100644 index 598b007f1..000000000 --- a/server/tests/shared/mock-servers/mock-instances-index.ts +++ /dev/null @@ -1,46 +0,0 @@ -import express from 'express' -import { Server } from 'http' -import { getPort, randomListen, terminateServer } from './shared' - -export class MockInstancesIndex { - private server: Server - - private readonly indexInstances: { host: string, createdAt: string }[] = [] - - async initialize () { - const app = express() - - app.use('/', (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (process.env.DEBUG) console.log('Receiving request on mocked server %s.', req.url) - - return next() - }) - - app.get('/api/v1/instances/hosts', (req: express.Request, res: express.Response) => { - const since = req.query.since - - const filtered = this.indexInstances.filter(i => { - if (!since) return true - - return i.createdAt > since - }) - - return res.json({ - total: filtered.length, - data: filtered - }) - }) - - this.server = await randomListen(app) - - return getPort(this.server) - } - - addInstance (host: string) { - this.indexInstances.push({ host, createdAt: new Date().toISOString() }) - } - - terminate () { - return terminateServer(this.server) - } -} diff --git a/server/tests/shared/mock-servers/mock-joinpeertube-versions.ts b/server/tests/shared/mock-servers/mock-joinpeertube-versions.ts deleted file mode 100644 index 502f4e2f5..000000000 --- a/server/tests/shared/mock-servers/mock-joinpeertube-versions.ts +++ /dev/null @@ -1,34 +0,0 @@ -import express from 'express' -import { Server } from 'http' -import { getPort, randomListen } from './shared' - -export class MockJoinPeerTubeVersions { - private server: Server - private latestVersion: string - - async initialize () { - const app = express() - - app.use('/', (req: express.Request, res: express.Response, next: express.NextFunction) => { - if (process.env.DEBUG) console.log('Receiving request on mocked server %s.', req.url) - - return next() - }) - - app.get('/versions.json', (req: express.Request, res: express.Response) => { - return res.json({ - peertube: { - latestVersion: this.latestVersion - } - }) - }) - - this.server = await randomListen(app) - - return getPort(this.server) - } - - setLatestVersion (latestVersion: string) { - this.latestVersion = latestVersion - } -} diff --git a/server/tests/shared/mock-servers/mock-object-storage.ts b/server/tests/shared/mock-servers/mock-object-storage.ts deleted file mode 100644 index ae76c4f3f..000000000 --- a/server/tests/shared/mock-servers/mock-object-storage.ts +++ /dev/null @@ -1,41 +0,0 @@ -import express from 'express' -import got, { RequestError } from 'got' -import { Server } from 'http' -import { pipeline } from 'stream' -import { ObjectStorageCommand } from '@shared/server-commands' -import { getPort, randomListen, terminateServer } from './shared' - -export class MockObjectStorageProxy { - private server: Server - - async initialize () { - const app = express() - - app.get('/:bucketName/:path(*)', (req: express.Request, res: express.Response, next: express.NextFunction) => { - const url = `http://${req.params.bucketName}.${ObjectStorageCommand.getMockEndpointHost()}/${req.params.path}` - - if (process.env.DEBUG) { - console.log('Receiving request on mocked server %s.', req.url) - console.log('Proxifying request to %s', url) - } - - return pipeline( - got.stream(url, { throwHttpErrors: false }), - res, - (err: RequestError) => { - if (!err) return - - console.error('Pipeline failed.', err) - } - ) - }) - - this.server = await randomListen(app) - - return getPort(this.server) - } - - terminate () { - return terminateServer(this.server) - } -} diff --git a/server/tests/shared/mock-servers/mock-plugin-blocklist.ts b/server/tests/shared/mock-servers/mock-plugin-blocklist.ts deleted file mode 100644 index 5d6e01816..000000000 --- a/server/tests/shared/mock-servers/mock-plugin-blocklist.ts +++ /dev/null @@ -1,36 +0,0 @@ -import express, { Request, Response } from 'express' -import { Server } from 'http' -import { getPort, randomListen, terminateServer } from './shared' - -type BlocklistResponse = { - data: { - value: string - action?: 'add' | 'remove' - updatedAt?: string - }[] -} - -export class MockBlocklist { - private body: BlocklistResponse - private server: Server - - async initialize () { - const app = express() - - app.get('/blocklist', (req: Request, res: Response) => { - return res.json(this.body) - }) - - this.server = await randomListen(app) - - return getPort(this.server) - } - - replace (body: BlocklistResponse) { - this.body = body - } - - terminate () { - return terminateServer(this.server) - } -} diff --git a/server/tests/shared/mock-servers/mock-proxy.ts b/server/tests/shared/mock-servers/mock-proxy.ts deleted file mode 100644 index e741d6735..000000000 --- a/server/tests/shared/mock-servers/mock-proxy.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { createServer, Server } from 'http' -import { createProxy } from 'proxy' -import { getPort, terminateServer } from './shared' - -class MockProxy { - private server: Server - - initialize () { - return new Promise(res => { - this.server = createProxy(createServer()) - this.server.listen(0, () => res(getPort(this.server))) - }) - } - - terminate () { - return terminateServer(this.server) - } -} - -// --------------------------------------------------------------------------- - -export { - MockProxy -} diff --git a/server/tests/shared/notifications.ts b/server/tests/shared/notifications.ts deleted file mode 100644 index 6c0688d5a..000000000 --- a/server/tests/shared/notifications.ts +++ /dev/null @@ -1,889 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { inspect } from 'util' -import { - AbuseState, - PluginType, - UserNotification, - UserNotificationSetting, - UserNotificationSettingValue, - UserNotificationType -} from '@shared/models' -import { - ConfigCommand, - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - setDefaultAccountAvatar, - setDefaultChannelAvatar, - setDefaultVideoChannel -} from '@shared/server-commands' -import { MockSmtpServer } from './mock-servers' - -type CheckerBaseParams = { - server: PeerTubeServer - emails: any[] - socketNotifications: UserNotification[] - token: string - check?: { web: boolean, mail: boolean } -} - -type CheckerType = 'presence' | 'absence' - -function getAllNotificationsSettings (): UserNotificationSetting { - return { - newVideoFromSubscription: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - newCommentOnMyVideo: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - abuseAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - videoAutoBlacklistAsModerator: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - blacklistOnMyVideo: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - myVideoImportFinished: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - myVideoPublished: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - commentMention: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - newFollow: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - newUserRegistration: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - newInstanceFollower: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - abuseNewMessage: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - abuseStateChange: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - autoInstanceFollowing: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - newPeerTubeVersion: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - myVideoStudioEditionFinished: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL, - newPluginVersion: UserNotificationSettingValue.WEB | UserNotificationSettingValue.EMAIL - } -} - -async function checkNewVideoFromSubscription (options: CheckerBaseParams & { - videoName: string - shortUUID: string - checkType: CheckerType -}) { - const { videoName, shortUUID } = options - const notificationType = UserNotificationType.NEW_VIDEO_FROM_SUBSCRIPTION - - function notificationChecker (notification: UserNotification, checkType: CheckerType) { - if (checkType === 'presence') { - expect(notification).to.not.be.undefined - expect(notification.type).to.equal(notificationType) - - checkVideo(notification.video, videoName, shortUUID) - checkActor(notification.video.channel) - } else { - expect(notification).to.satisfy((n: UserNotification) => { - return n === undefined || n.type !== UserNotificationType.NEW_VIDEO_FROM_SUBSCRIPTION || n.video.name !== videoName - }) - } - } - - function emailNotificationFinder (email: object) { - const text = email['text'] - return text.indexOf(shortUUID) !== -1 && text.indexOf('Your subscription') !== -1 - } - - await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) -} - -async function checkVideoIsPublished (options: CheckerBaseParams & { - videoName: string - shortUUID: string - checkType: CheckerType -}) { - const { videoName, shortUUID } = options - const notificationType = UserNotificationType.MY_VIDEO_PUBLISHED - - function notificationChecker (notification: UserNotification, checkType: CheckerType) { - if (checkType === 'presence') { - expect(notification).to.not.be.undefined - expect(notification.type).to.equal(notificationType) - - checkVideo(notification.video, videoName, shortUUID) - checkActor(notification.video.channel) - } else { - expect(notification.video).to.satisfy(v => v === undefined || v.name !== videoName) - } - } - - function emailNotificationFinder (email: object) { - const text: string = email['text'] - return text.includes(shortUUID) && text.includes('Your video') - } - - await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) -} - -async function checkVideoStudioEditionIsFinished (options: CheckerBaseParams & { - videoName: string - shortUUID: string - checkType: CheckerType -}) { - const { videoName, shortUUID } = options - const notificationType = UserNotificationType.MY_VIDEO_STUDIO_EDITION_FINISHED - - function notificationChecker (notification: UserNotification, checkType: CheckerType) { - if (checkType === 'presence') { - expect(notification).to.not.be.undefined - expect(notification.type).to.equal(notificationType) - - checkVideo(notification.video, videoName, shortUUID) - checkActor(notification.video.channel) - } else { - expect(notification.video).to.satisfy(v => v === undefined || v.name !== videoName) - } - } - - function emailNotificationFinder (email: object) { - const text: string = email['text'] - return text.includes(shortUUID) && text.includes('Edition of your video') - } - - await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) -} - -async function checkMyVideoImportIsFinished (options: CheckerBaseParams & { - videoName: string - shortUUID: string - url: string - success: boolean - checkType: CheckerType -}) { - const { videoName, shortUUID, url, success } = options - - const notificationType = success ? UserNotificationType.MY_VIDEO_IMPORT_SUCCESS : UserNotificationType.MY_VIDEO_IMPORT_ERROR - - function notificationChecker (notification: UserNotification, checkType: CheckerType) { - if (checkType === 'presence') { - expect(notification).to.not.be.undefined - expect(notification.type).to.equal(notificationType) - - expect(notification.videoImport.targetUrl).to.equal(url) - - if (success) checkVideo(notification.videoImport.video, videoName, shortUUID) - } else { - expect(notification.videoImport).to.satisfy(i => i === undefined || i.targetUrl !== url) - } - } - - function emailNotificationFinder (email: object) { - const text: string = email['text'] - const toFind = success ? ' finished' : ' error' - - return text.includes(url) && text.includes(toFind) - } - - await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) -} - -// --------------------------------------------------------------------------- - -async function checkUserRegistered (options: CheckerBaseParams & { - username: string - checkType: CheckerType -}) { - const { username } = options - const notificationType = UserNotificationType.NEW_USER_REGISTRATION - - function notificationChecker (notification: UserNotification, checkType: CheckerType) { - if (checkType === 'presence') { - expect(notification).to.not.be.undefined - expect(notification.type).to.equal(notificationType) - - checkActor(notification.account, { withAvatar: false }) - expect(notification.account.name).to.equal(username) - } else { - expect(notification).to.satisfy(n => n.type !== notificationType || n.account.name !== username) - } - } - - function emailNotificationFinder (email: object) { - const text: string = email['text'] - - return text.includes(' registered.') && text.includes(username) - } - - await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) -} - -async function checkRegistrationRequest (options: CheckerBaseParams & { - username: string - registrationReason: string - checkType: CheckerType -}) { - const { username, registrationReason } = options - const notificationType = UserNotificationType.NEW_USER_REGISTRATION_REQUEST - - function notificationChecker (notification: UserNotification, checkType: CheckerType) { - if (checkType === 'presence') { - expect(notification).to.not.be.undefined - expect(notification.type).to.equal(notificationType) - - expect(notification.registration.username).to.equal(username) - } else { - expect(notification).to.satisfy(n => n.type !== notificationType || n.registration.username !== username) - } - } - - function emailNotificationFinder (email: object) { - const text: string = email['text'] - - return text.includes(' wants to register ') && text.includes(username) && text.includes(registrationReason) - } - - await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) -} - -// --------------------------------------------------------------------------- - -async function checkNewActorFollow (options: CheckerBaseParams & { - followType: 'channel' | 'account' - followerName: string - followerDisplayName: string - followingDisplayName: string - checkType: CheckerType -}) { - const { followType, followerName, followerDisplayName, followingDisplayName } = options - const notificationType = UserNotificationType.NEW_FOLLOW - - function notificationChecker (notification: UserNotification, checkType: CheckerType) { - if (checkType === 'presence') { - expect(notification).to.not.be.undefined - expect(notification.type).to.equal(notificationType) - - checkActor(notification.actorFollow.follower) - expect(notification.actorFollow.follower.displayName).to.equal(followerDisplayName) - expect(notification.actorFollow.follower.name).to.equal(followerName) - expect(notification.actorFollow.follower.host).to.not.be.undefined - - const following = notification.actorFollow.following - expect(following.displayName).to.equal(followingDisplayName) - expect(following.type).to.equal(followType) - } else { - expect(notification).to.satisfy(n => { - return n.type !== notificationType || - (n.actorFollow.follower.name !== followerName && n.actorFollow.following !== followingDisplayName) - }) - } - } - - function emailNotificationFinder (email: object) { - const text: string = email['text'] - - return text.includes(followType) && text.includes(followingDisplayName) && text.includes(followerDisplayName) - } - - await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) -} - -async function checkNewInstanceFollower (options: CheckerBaseParams & { - followerHost: string - checkType: CheckerType -}) { - const { followerHost } = options - const notificationType = UserNotificationType.NEW_INSTANCE_FOLLOWER - - function notificationChecker (notification: UserNotification, checkType: CheckerType) { - if (checkType === 'presence') { - expect(notification).to.not.be.undefined - expect(notification.type).to.equal(notificationType) - - checkActor(notification.actorFollow.follower, { withAvatar: false }) - expect(notification.actorFollow.follower.name).to.equal('peertube') - expect(notification.actorFollow.follower.host).to.equal(followerHost) - - expect(notification.actorFollow.following.name).to.equal('peertube') - } else { - expect(notification).to.satisfy(n => { - return n.type !== notificationType || n.actorFollow.follower.host !== followerHost - }) - } - } - - function emailNotificationFinder (email: object) { - const text: string = email['text'] - - return text.includes('instance has a new follower') && text.includes(followerHost) - } - - await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) -} - -async function checkAutoInstanceFollowing (options: CheckerBaseParams & { - followerHost: string - followingHost: string - checkType: CheckerType -}) { - const { followerHost, followingHost } = options - const notificationType = UserNotificationType.AUTO_INSTANCE_FOLLOWING - - function notificationChecker (notification: UserNotification, checkType: CheckerType) { - if (checkType === 'presence') { - expect(notification).to.not.be.undefined - expect(notification.type).to.equal(notificationType) - - const following = notification.actorFollow.following - - checkActor(following, { withAvatar: false }) - expect(following.name).to.equal('peertube') - expect(following.host).to.equal(followingHost) - - expect(notification.actorFollow.follower.name).to.equal('peertube') - expect(notification.actorFollow.follower.host).to.equal(followerHost) - } else { - expect(notification).to.satisfy(n => { - return n.type !== notificationType || n.actorFollow.following.host !== followingHost - }) - } - } - - function emailNotificationFinder (email: object) { - const text: string = email['text'] - - return text.includes(' automatically followed a new instance') && text.includes(followingHost) - } - - await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) -} - -async function checkCommentMention (options: CheckerBaseParams & { - shortUUID: string - commentId: number - threadId: number - byAccountDisplayName: string - checkType: CheckerType -}) { - const { shortUUID, commentId, threadId, byAccountDisplayName } = options - const notificationType = UserNotificationType.COMMENT_MENTION - - function notificationChecker (notification: UserNotification, checkType: CheckerType) { - if (checkType === 'presence') { - expect(notification).to.not.be.undefined - expect(notification.type).to.equal(notificationType) - - checkComment(notification.comment, commentId, threadId) - checkActor(notification.comment.account) - expect(notification.comment.account.displayName).to.equal(byAccountDisplayName) - - checkVideo(notification.comment.video, undefined, shortUUID) - } else { - expect(notification).to.satisfy(n => n.type !== notificationType || n.comment.id !== commentId) - } - } - - function emailNotificationFinder (email: object) { - const text: string = email['text'] - - return text.includes(' mentioned ') && text.includes(shortUUID) && text.includes(byAccountDisplayName) - } - - await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) -} - -let lastEmailCount = 0 - -async function checkNewCommentOnMyVideo (options: CheckerBaseParams & { - shortUUID: string - commentId: number - threadId: number - checkType: CheckerType -}) { - const { server, shortUUID, commentId, threadId, checkType, emails } = options - const notificationType = UserNotificationType.NEW_COMMENT_ON_MY_VIDEO - - function notificationChecker (notification: UserNotification, checkType: CheckerType) { - if (checkType === 'presence') { - expect(notification).to.not.be.undefined - expect(notification.type).to.equal(notificationType) - - checkComment(notification.comment, commentId, threadId) - checkActor(notification.comment.account) - checkVideo(notification.comment.video, undefined, shortUUID) - } else { - expect(notification).to.satisfy((n: UserNotification) => { - return n === undefined || n.comment === undefined || n.comment.id !== commentId - }) - } - } - - const commentUrl = `${server.url}/w/${shortUUID};threadId=${threadId}` - - function emailNotificationFinder (email: object) { - return email['text'].indexOf(commentUrl) !== -1 - } - - await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) - - if (checkType === 'presence') { - // We cannot detect email duplicates, so check we received another email - expect(emails).to.have.length.above(lastEmailCount) - lastEmailCount = emails.length - } -} - -async function checkNewVideoAbuseForModerators (options: CheckerBaseParams & { - shortUUID: string - videoName: string - checkType: CheckerType -}) { - const { shortUUID, videoName } = options - const notificationType = UserNotificationType.NEW_ABUSE_FOR_MODERATORS - - function notificationChecker (notification: UserNotification, checkType: CheckerType) { - if (checkType === 'presence') { - expect(notification).to.not.be.undefined - expect(notification.type).to.equal(notificationType) - - expect(notification.abuse.id).to.be.a('number') - checkVideo(notification.abuse.video, videoName, shortUUID) - } else { - expect(notification).to.satisfy((n: UserNotification) => { - return n === undefined || n.abuse === undefined || n.abuse.video.shortUUID !== shortUUID - }) - } - } - - function emailNotificationFinder (email: object) { - const text = email['text'] - return text.indexOf(shortUUID) !== -1 && text.indexOf('abuse') !== -1 - } - - await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) -} - -async function checkNewAbuseMessage (options: CheckerBaseParams & { - abuseId: number - message: string - toEmail: string - checkType: CheckerType -}) { - const { abuseId, message, toEmail } = options - const notificationType = UserNotificationType.ABUSE_NEW_MESSAGE - - function notificationChecker (notification: UserNotification, checkType: CheckerType) { - if (checkType === 'presence') { - expect(notification).to.not.be.undefined - expect(notification.type).to.equal(notificationType) - - expect(notification.abuse.id).to.equal(abuseId) - } else { - expect(notification).to.satisfy((n: UserNotification) => { - return n === undefined || n.type !== notificationType || n.abuse === undefined || n.abuse.id !== abuseId - }) - } - } - - function emailNotificationFinder (email: object) { - const text = email['text'] - const to = email['to'].filter(t => t.address === toEmail) - - return text.indexOf(message) !== -1 && to.length !== 0 - } - - await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) -} - -async function checkAbuseStateChange (options: CheckerBaseParams & { - abuseId: number - state: AbuseState - checkType: CheckerType -}) { - const { abuseId, state } = options - const notificationType = UserNotificationType.ABUSE_STATE_CHANGE - - function notificationChecker (notification: UserNotification, checkType: CheckerType) { - if (checkType === 'presence') { - expect(notification).to.not.be.undefined - expect(notification.type).to.equal(notificationType) - - expect(notification.abuse.id).to.equal(abuseId) - expect(notification.abuse.state).to.equal(state) - } else { - expect(notification).to.satisfy((n: UserNotification) => { - return n === undefined || n.abuse === undefined || n.abuse.id !== abuseId - }) - } - } - - function emailNotificationFinder (email: object) { - const text = email['text'] - - const contains = state === AbuseState.ACCEPTED - ? ' accepted' - : ' rejected' - - return text.indexOf(contains) !== -1 - } - - await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) -} - -async function checkNewCommentAbuseForModerators (options: CheckerBaseParams & { - shortUUID: string - videoName: string - checkType: CheckerType -}) { - const { shortUUID, videoName } = options - const notificationType = UserNotificationType.NEW_ABUSE_FOR_MODERATORS - - function notificationChecker (notification: UserNotification, checkType: CheckerType) { - if (checkType === 'presence') { - expect(notification).to.not.be.undefined - expect(notification.type).to.equal(notificationType) - - expect(notification.abuse.id).to.be.a('number') - checkVideo(notification.abuse.comment.video, videoName, shortUUID) - } else { - expect(notification).to.satisfy((n: UserNotification) => { - return n === undefined || n.abuse === undefined || n.abuse.comment.video.shortUUID !== shortUUID - }) - } - } - - function emailNotificationFinder (email: object) { - const text = email['text'] - return text.indexOf(shortUUID) !== -1 && text.indexOf('abuse') !== -1 - } - - await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) -} - -async function checkNewAccountAbuseForModerators (options: CheckerBaseParams & { - displayName: string - checkType: CheckerType -}) { - const { displayName } = options - const notificationType = UserNotificationType.NEW_ABUSE_FOR_MODERATORS - - function notificationChecker (notification: UserNotification, checkType: CheckerType) { - if (checkType === 'presence') { - expect(notification).to.not.be.undefined - expect(notification.type).to.equal(notificationType) - - expect(notification.abuse.id).to.be.a('number') - expect(notification.abuse.account.displayName).to.equal(displayName) - } else { - expect(notification).to.satisfy((n: UserNotification) => { - return n === undefined || n.abuse === undefined || n.abuse.account.displayName !== displayName - }) - } - } - - function emailNotificationFinder (email: object) { - const text = email['text'] - return text.indexOf(displayName) !== -1 && text.indexOf('abuse') !== -1 - } - - await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) -} - -async function checkVideoAutoBlacklistForModerators (options: CheckerBaseParams & { - shortUUID: string - videoName: string - checkType: CheckerType -}) { - const { shortUUID, videoName } = options - const notificationType = UserNotificationType.VIDEO_AUTO_BLACKLIST_FOR_MODERATORS - - function notificationChecker (notification: UserNotification, checkType: CheckerType) { - if (checkType === 'presence') { - expect(notification).to.not.be.undefined - expect(notification.type).to.equal(notificationType) - - expect(notification.videoBlacklist.video.id).to.be.a('number') - checkVideo(notification.videoBlacklist.video, videoName, shortUUID) - } else { - expect(notification).to.satisfy((n: UserNotification) => { - return n === undefined || n.video === undefined || n.video.shortUUID !== shortUUID - }) - } - } - - function emailNotificationFinder (email: object) { - const text = email['text'] - return text.indexOf(shortUUID) !== -1 && email['text'].indexOf('video-auto-blacklist/list') !== -1 - } - - await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) -} - -async function checkNewBlacklistOnMyVideo (options: CheckerBaseParams & { - shortUUID: string - videoName: string - blacklistType: 'blacklist' | 'unblacklist' -}) { - const { videoName, shortUUID, blacklistType } = options - const notificationType = blacklistType === 'blacklist' - ? UserNotificationType.BLACKLIST_ON_MY_VIDEO - : UserNotificationType.UNBLACKLIST_ON_MY_VIDEO - - function notificationChecker (notification: UserNotification) { - expect(notification).to.not.be.undefined - expect(notification.type).to.equal(notificationType) - - const video = blacklistType === 'blacklist' ? notification.videoBlacklist.video : notification.video - - checkVideo(video, videoName, shortUUID) - } - - function emailNotificationFinder (email: object) { - const text = email['text'] - const blacklistText = blacklistType === 'blacklist' - ? 'blacklisted' - : 'unblacklisted' - - return text.includes(shortUUID) && text.includes(blacklistText) - } - - await checkNotification({ ...options, notificationChecker, emailNotificationFinder, checkType: 'presence' }) -} - -async function checkNewPeerTubeVersion (options: CheckerBaseParams & { - latestVersion: string - checkType: CheckerType -}) { - const { latestVersion } = options - const notificationType = UserNotificationType.NEW_PEERTUBE_VERSION - - function notificationChecker (notification: UserNotification, checkType: CheckerType) { - if (checkType === 'presence') { - expect(notification).to.not.be.undefined - expect(notification.type).to.equal(notificationType) - - expect(notification.peertube).to.exist - expect(notification.peertube.latestVersion).to.equal(latestVersion) - } else { - expect(notification).to.satisfy((n: UserNotification) => { - return n === undefined || n.peertube === undefined || n.peertube.latestVersion !== latestVersion - }) - } - } - - function emailNotificationFinder (email: object) { - const text = email['text'] - - return text.includes(latestVersion) - } - - await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) -} - -async function checkNewPluginVersion (options: CheckerBaseParams & { - pluginType: PluginType - pluginName: string - checkType: CheckerType -}) { - const { pluginName, pluginType } = options - const notificationType = UserNotificationType.NEW_PLUGIN_VERSION - - function notificationChecker (notification: UserNotification, checkType: CheckerType) { - if (checkType === 'presence') { - expect(notification).to.not.be.undefined - expect(notification.type).to.equal(notificationType) - - expect(notification.plugin.name).to.equal(pluginName) - expect(notification.plugin.type).to.equal(pluginType) - } else { - expect(notification).to.satisfy((n: UserNotification) => { - return n === undefined || n.plugin === undefined || n.plugin.name !== pluginName - }) - } - } - - function emailNotificationFinder (email: object) { - const text = email['text'] - - return text.includes(pluginName) - } - - await checkNotification({ ...options, notificationChecker, emailNotificationFinder }) -} - -async function prepareNotificationsTest (serversCount = 3, overrideConfigArg: any = {}) { - const userNotifications: UserNotification[] = [] - const adminNotifications: UserNotification[] = [] - const adminNotificationsServer2: UserNotification[] = [] - const emails: object[] = [] - - const port = await MockSmtpServer.Instance.collectEmails(emails) - - const overrideConfig = { - ...ConfigCommand.getEmailOverrideConfig(port), - - signup: { - limit: 20 - } - } - const servers = await createMultipleServers(serversCount, Object.assign(overrideConfig, overrideConfigArg)) - - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - await setDefaultChannelAvatar(servers) - await setDefaultAccountAvatar(servers) - - if (servers[1]) { - await servers[1].config.enableStudio() - await servers[1].config.enableLive({ allowReplay: true, transcoding: false }) - } - - if (serversCount > 1) { - await doubleFollow(servers[0], servers[1]) - } - - const user = { username: 'user_1', password: 'super password' } - await servers[0].users.create({ ...user, videoQuota: 10 * 1000 * 1000 }) - const userAccessToken = await servers[0].login.getAccessToken(user) - - await servers[0].notifications.updateMySettings({ token: userAccessToken, settings: getAllNotificationsSettings() }) - await servers[0].users.updateMyAvatar({ token: userAccessToken, fixture: 'avatar.png' }) - await servers[0].channels.updateImage({ channelName: 'user_1_channel', token: userAccessToken, fixture: 'avatar.png', type: 'avatar' }) - - await servers[0].notifications.updateMySettings({ settings: getAllNotificationsSettings() }) - - if (serversCount > 1) { - await servers[1].notifications.updateMySettings({ settings: getAllNotificationsSettings() }) - } - - { - const socket = servers[0].socketIO.getUserNotificationSocket({ token: userAccessToken }) - socket.on('new-notification', n => userNotifications.push(n)) - } - { - const socket = servers[0].socketIO.getUserNotificationSocket() - socket.on('new-notification', n => adminNotifications.push(n)) - } - - if (serversCount > 1) { - const socket = servers[1].socketIO.getUserNotificationSocket() - socket.on('new-notification', n => adminNotificationsServer2.push(n)) - } - - const { videoChannels } = await servers[0].users.getMyInfo() - const channelId = videoChannels[0].id - - return { - userNotifications, - adminNotifications, - adminNotificationsServer2, - userAccessToken, - emails, - servers, - channelId, - baseOverrideConfig: overrideConfig - } -} - -// --------------------------------------------------------------------------- - -export { - getAllNotificationsSettings, - - CheckerBaseParams, - CheckerType, - checkMyVideoImportIsFinished, - checkUserRegistered, - checkAutoInstanceFollowing, - checkVideoIsPublished, - checkNewVideoFromSubscription, - checkNewActorFollow, - checkNewCommentOnMyVideo, - checkNewBlacklistOnMyVideo, - checkCommentMention, - checkNewVideoAbuseForModerators, - checkVideoAutoBlacklistForModerators, - checkNewAbuseMessage, - checkAbuseStateChange, - checkNewInstanceFollower, - prepareNotificationsTest, - checkNewCommentAbuseForModerators, - checkNewAccountAbuseForModerators, - checkNewPeerTubeVersion, - checkNewPluginVersion, - checkVideoStudioEditionIsFinished, - checkRegistrationRequest -} - -// --------------------------------------------------------------------------- - -async function checkNotification (options: CheckerBaseParams & { - notificationChecker: (notification: UserNotification, checkType: CheckerType) => void - emailNotificationFinder: (email: object) => boolean - checkType: CheckerType -}) { - const { server, token, checkType, notificationChecker, emailNotificationFinder, socketNotifications, emails } = options - - const check = options.check || { web: true, mail: true } - - if (check.web) { - const notification = await server.notifications.getLatest({ token }) - - if (notification || checkType !== 'absence') { - notificationChecker(notification, checkType) - } - - const socketNotification = socketNotifications.find(n => { - try { - notificationChecker(n, 'presence') - return true - } catch { - return false - } - }) - - if (checkType === 'presence') { - const obj = inspect(socketNotifications, { depth: 5 }) - expect(socketNotification, 'The socket notification is absent when it should be present. ' + obj).to.not.be.undefined - } else { - const obj = inspect(socketNotification, { depth: 5 }) - expect(socketNotification, 'The socket notification is present when it should not be present. ' + obj).to.be.undefined - } - } - - if (check.mail) { - // Last email - const email = emails - .slice() - .reverse() - .find(e => emailNotificationFinder(e)) - - if (checkType === 'presence') { - const texts = emails.map(e => e.text) - expect(email, 'The email is absent when is should be present. ' + inspect(texts)).to.not.be.undefined - } else { - expect(email, 'The email is present when is should not be present. ' + inspect(email)).to.be.undefined - } - } -} - -function checkVideo (video: any, videoName?: string, shortUUID?: string) { - if (videoName) { - expect(video.name).to.be.a('string') - expect(video.name).to.not.be.empty - expect(video.name).to.equal(videoName) - } - - if (shortUUID) { - expect(video.shortUUID).to.be.a('string') - expect(video.shortUUID).to.not.be.empty - expect(video.shortUUID).to.equal(shortUUID) - } - - expect(video.id).to.be.a('number') -} - -function checkActor (actor: any, options: { withAvatar?: boolean } = {}) { - const { withAvatar = true } = options - - expect(actor.displayName).to.be.a('string') - expect(actor.displayName).to.not.be.empty - expect(actor.host).to.not.be.undefined - - if (withAvatar) { - expect(actor.avatars).to.be.an('array') - expect(actor.avatars).to.have.lengthOf(2) - expect(actor.avatars[0].path).to.exist.and.not.empty - } -} - -function checkComment (comment: any, commentId: number, threadId: number) { - expect(comment.id).to.equal(commentId) - expect(comment.threadId).to.equal(threadId) -} diff --git a/server/tests/shared/peertube-runner-process.ts b/server/tests/shared/peertube-runner-process.ts deleted file mode 100644 index 9304ebcc8..000000000 --- a/server/tests/shared/peertube-runner-process.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { ChildProcess, fork } from 'child_process' -import execa from 'execa' -import { join } from 'path' -import { root } from '@shared/core-utils' -import { PeerTubeServer } from '@shared/server-commands' - -export class PeerTubeRunnerProcess { - private app?: ChildProcess - - constructor (private readonly server: PeerTubeServer) { - - } - - runServer (options: { - hideLogs?: boolean // default true - } = {}) { - const { hideLogs = true } = options - - return new Promise((res, rej) => { - const args = [ 'server', '--verbose', ...this.buildIdArg() ] - - const forkOptions = { - detached: false, - silent: true - } - this.app = fork(this.getRunnerPath(), args, forkOptions) - - this.app.stdout.on('data', data => { - const str = data.toString() as string - - if (!hideLogs) { - console.log(str) - } - }) - - res() - }) - } - - registerPeerTubeInstance (options: { - registrationToken: string - runnerName: string - runnerDescription?: string - }) { - const { registrationToken, runnerName, runnerDescription } = options - - const args = [ - 'register', - '--url', this.server.url, - '--registration-token', registrationToken, - '--runner-name', runnerName, - ...this.buildIdArg() - ] - - if (runnerDescription) { - args.push('--runner-description') - args.push(runnerDescription) - } - - return execa.node(this.getRunnerPath(), args) - } - - unregisterPeerTubeInstance (options: { - runnerName: string - }) { - const { runnerName } = options - - const args = [ 'unregister', '--url', this.server.url, '--runner-name', runnerName, ...this.buildIdArg() ] - return execa.node(this.getRunnerPath(), args) - } - - async listRegisteredPeerTubeInstances () { - const args = [ 'list-registered', ...this.buildIdArg() ] - const { stdout } = await execa.node(this.getRunnerPath(), args) - - return stdout - } - - kill () { - if (!this.app) return - - process.kill(this.app.pid) - - this.app = null - } - - getId () { - return 'test-' + this.server.internalServerNumber - } - - private getRunnerPath () { - return join(root(), 'packages', 'peertube-runner', 'dist', 'peertube-runner.js') - } - - private buildIdArg () { - return [ '--id', this.getId() ] - } -} diff --git a/server/tests/shared/plugins.ts b/server/tests/shared/plugins.ts deleted file mode 100644 index 036fce2ff..000000000 --- a/server/tests/shared/plugins.ts +++ /dev/null @@ -1,18 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { PeerTubeServer } from '@shared/server-commands' - -async function testHelloWorldRegisteredSettings (server: PeerTubeServer) { - const body = await server.plugins.getRegisteredSettings({ npmName: 'peertube-plugin-hello-world' }) - - const registeredSettings = body.registeredSettings - expect(registeredSettings).to.have.length.at.least(1) - - const adminNameSettings = registeredSettings.find(s => s.name === 'admin-name') - expect(adminNameSettings).to.not.be.undefined -} - -export { - testHelloWorldRegisteredSettings -} diff --git a/server/tests/shared/requests.ts b/server/tests/shared/requests.ts deleted file mode 100644 index 0cfeab7b2..000000000 --- a/server/tests/shared/requests.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { doRequest } from '@server/helpers/requests' - -export function makePOSTAPRequest (url: string, body: any, httpSignature: any, headers: any) { - const options = { - method: 'POST' as 'POST', - json: body, - httpSignature, - headers - } - - return doRequest(url, options) -} diff --git a/server/tests/shared/sql-command.ts b/server/tests/shared/sql-command.ts deleted file mode 100644 index 5c53a8ac6..000000000 --- a/server/tests/shared/sql-command.ts +++ /dev/null @@ -1,150 +0,0 @@ -import { QueryTypes, Sequelize } from 'sequelize' -import { forceNumber } from '@shared/core-utils' -import { PeerTubeServer } from '@shared/server-commands' - -export class SQLCommand { - private sequelize: Sequelize - - constructor (private readonly server: PeerTubeServer) { - - } - - deleteAll (table: string) { - const seq = this.getSequelize() - - const options = { type: QueryTypes.DELETE } - - return seq.query(`DELETE FROM "${table}"`, options) - } - - async getVideoShareCount () { - const [ { total } ] = await this.selectQuery<{ total: string }>(`SELECT COUNT(*) as total FROM "videoShare"`) - if (total === null) return 0 - - return parseInt(total, 10) - } - - async getInternalFileUrl (fileId: number) { - return this.selectQuery<{ fileUrl: string }>(`SELECT "fileUrl" FROM "videoFile" WHERE id = :fileId`, { fileId }) - .then(rows => rows[0].fileUrl) - } - - setActorField (to: string, field: string, value: string) { - return this.updateQuery(`UPDATE actor SET ${this.escapeColumnName(field)} = :value WHERE url = :to`, { value, to }) - } - - setVideoField (uuid: string, field: string, value: string) { - return this.updateQuery(`UPDATE video SET ${this.escapeColumnName(field)} = :value WHERE uuid = :uuid`, { value, uuid }) - } - - setPlaylistField (uuid: string, field: string, value: string) { - return this.updateQuery(`UPDATE "videoPlaylist" SET ${this.escapeColumnName(field)} = :value WHERE uuid = :uuid`, { value, uuid }) - } - - async countVideoViewsOf (uuid: string) { - const query = 'SELECT SUM("videoView"."views") AS "total" FROM "videoView" ' + - `INNER JOIN "video" ON "video"."id" = "videoView"."videoId" WHERE "video"."uuid" = :uuid` - - const [ { total } ] = await this.selectQuery<{ total: number }>(query, { uuid }) - if (!total) return 0 - - return forceNumber(total) - } - - getActorImage (filename: string) { - return this.selectQuery<{ width: number, height: number }>(`SELECT * FROM "actorImage" WHERE filename = :filename`, { filename }) - .then(rows => rows[0]) - } - - // --------------------------------------------------------------------------- - - setPluginVersion (pluginName: string, newVersion: string) { - return this.setPluginField(pluginName, 'version', newVersion) - } - - setPluginLatestVersion (pluginName: string, newVersion: string) { - return this.setPluginField(pluginName, 'latestVersion', newVersion) - } - - setPluginField (pluginName: string, field: string, value: string) { - return this.updateQuery( - `UPDATE "plugin" SET ${this.escapeColumnName(field)} = :value WHERE "name" = :pluginName`, - { pluginName, value } - ) - } - - // --------------------------------------------------------------------------- - - selectQuery (query: string, replacements: { [id: string]: string | number } = {}) { - const seq = this.getSequelize() - const options = { - type: QueryTypes.SELECT as QueryTypes.SELECT, - replacements - } - - return seq.query(query, options) - } - - updateQuery (query: string, replacements: { [id: string]: string | number } = {}) { - const seq = this.getSequelize() - const options = { type: QueryTypes.UPDATE as QueryTypes.UPDATE, replacements } - - return seq.query(query, options) - } - - // --------------------------------------------------------------------------- - - async getPlaylistInfohash (playlistId: number) { - const query = 'SELECT "p2pMediaLoaderInfohashes" FROM "videoStreamingPlaylist" WHERE id = :playlistId' - - const result = await this.selectQuery<{ p2pMediaLoaderInfohashes: string }>(query, { playlistId }) - if (!result || result.length === 0) return [] - - return result[0].p2pMediaLoaderInfohashes - } - - // --------------------------------------------------------------------------- - - setActorFollowScores (newScore: number) { - return this.updateQuery(`UPDATE "actorFollow" SET "score" = :newScore`, { newScore }) - } - - setTokenField (accessToken: string, field: string, value: string) { - return this.updateQuery( - `UPDATE "oAuthToken" SET ${this.escapeColumnName(field)} = :value WHERE "accessToken" = :accessToken`, - { value, accessToken } - ) - } - - async cleanup () { - if (!this.sequelize) return - - await this.sequelize.close() - this.sequelize = undefined - } - - private getSequelize () { - if (this.sequelize) return this.sequelize - - const dbname = 'peertube_test' + this.server.internalServerNumber - const username = 'peertube' - const password = 'peertube' - const host = '127.0.0.1' - const port = 5432 - - this.sequelize = new Sequelize(dbname, username, password, { - dialect: 'postgres', - host, - port, - logging: false - }) - - return this.sequelize - } - - private escapeColumnName (columnName: string) { - return this.getSequelize().escape(columnName) - .replace(/^'/, '"') - .replace(/'$/, '"') - } -} diff --git a/server/tests/shared/streaming-playlists.ts b/server/tests/shared/streaming-playlists.ts deleted file mode 100644 index e4f88bc25..000000000 --- a/server/tests/shared/streaming-playlists.ts +++ /dev/null @@ -1,296 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { expect } from 'chai' -import { basename, dirname, join } from 'path' -import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils' -import { sha256 } from '@shared/extra-utils' -import { HttpStatusCode, VideoPrivacy, VideoResolution, VideoStreamingPlaylist, VideoStreamingPlaylistType } from '@shared/models' -import { makeRawRequest, PeerTubeServer } from '@shared/server-commands' -import { expectStartWith } from './checks' -import { hlsInfohashExist } from './tracker' -import { checkWebTorrentWorks } from './webtorrent' - -async function checkSegmentHash (options: { - server: PeerTubeServer - baseUrlPlaylist: string - baseUrlSegment: string - resolution: number - hlsPlaylist: VideoStreamingPlaylist - token?: string -}) { - const { server, baseUrlPlaylist, baseUrlSegment, resolution, hlsPlaylist, token } = options - const command = server.streamingPlaylists - - const file = hlsPlaylist.files.find(f => f.resolution.id === resolution) - const videoName = basename(file.fileUrl) - - const playlist = await command.get({ url: `${baseUrlPlaylist}/${removeFragmentedMP4Ext(videoName)}.m3u8`, token }) - - const matches = /#EXT-X-BYTERANGE:(\d+)@(\d+)/.exec(playlist) - - const length = parseInt(matches[1], 10) - const offset = parseInt(matches[2], 10) - const range = `${offset}-${offset + length - 1}` - - const segmentBody = await command.getFragmentedSegment({ - url: `${baseUrlSegment}/${videoName}`, - expectedStatus: HttpStatusCode.PARTIAL_CONTENT_206, - range: `bytes=${range}`, - token - }) - - const shaBody = await command.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url, token }) - expect(sha256(segmentBody)).to.equal(shaBody[videoName][range], `Invalid sha256 result for ${videoName} range ${range}`) -} - -// --------------------------------------------------------------------------- - -async function checkLiveSegmentHash (options: { - server: PeerTubeServer - baseUrlSegment: string - videoUUID: string - segmentName: string - hlsPlaylist: VideoStreamingPlaylist - withRetry?: boolean -}) { - const { server, baseUrlSegment, videoUUID, segmentName, hlsPlaylist, withRetry = false } = options - const command = server.streamingPlaylists - - const segmentBody = await command.getFragmentedSegment({ url: `${baseUrlSegment}/${videoUUID}/${segmentName}`, withRetry }) - const shaBody = await command.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url, withRetry }) - - expect(sha256(segmentBody)).to.equal(shaBody[segmentName]) -} - -// --------------------------------------------------------------------------- - -async function checkResolutionsInMasterPlaylist (options: { - server: PeerTubeServer - playlistUrl: string - resolutions: number[] - token?: string - transcoded?: boolean // default true - withRetry?: boolean // default false -}) { - const { server, playlistUrl, resolutions, token, withRetry = false, transcoded = true } = options - - const masterPlaylist = await server.streamingPlaylists.get({ url: playlistUrl, token, withRetry }) - - for (const resolution of resolutions) { - const base = '#EXT-X-STREAM-INF:BANDWIDTH=\\d+,RESOLUTION=\\d+x' + resolution - - if (resolution === VideoResolution.H_NOVIDEO) { - expect(masterPlaylist).to.match(new RegExp(`${base},CODECS="mp4a.40.2"`)) - } else if (transcoded) { - expect(masterPlaylist).to.match(new RegExp(`${base},(FRAME-RATE=\\d+,)?CODECS="avc1.64001f,mp4a.40.2"`)) - } else { - expect(masterPlaylist).to.match(new RegExp(`${base}`)) - } - } - - const playlistsLength = masterPlaylist.split('\n').filter(line => line.startsWith('#EXT-X-STREAM-INF:BANDWIDTH=')) - expect(playlistsLength).to.have.lengthOf(resolutions.length) -} - -async function completeCheckHlsPlaylist (options: { - servers: PeerTubeServer[] - videoUUID: string - hlsOnly: boolean - - resolutions?: number[] - objectStorageBaseUrl?: string -}) { - const { videoUUID, hlsOnly, objectStorageBaseUrl } = options - - const resolutions = options.resolutions ?? [ 240, 360, 480, 720 ] - - for (const server of options.servers) { - const videoDetails = await server.videos.getWithToken({ id: videoUUID }) - const requiresAuth = videoDetails.privacy.id === VideoPrivacy.PRIVATE || videoDetails.privacy.id === VideoPrivacy.INTERNAL - - const privatePath = requiresAuth - ? 'private/' - : '' - const token = requiresAuth - ? server.accessToken - : undefined - - const baseUrl = `http://${videoDetails.account.host}` - - expect(videoDetails.streamingPlaylists).to.have.lengthOf(1) - - const hlsPlaylist = videoDetails.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) - expect(hlsPlaylist).to.not.be.undefined - - const hlsFiles = hlsPlaylist.files - expect(hlsFiles).to.have.lengthOf(resolutions.length) - - if (hlsOnly) expect(videoDetails.files).to.have.lengthOf(0) - else expect(videoDetails.files).to.have.lengthOf(resolutions.length) - - // Check JSON files - for (const resolution of resolutions) { - const file = hlsFiles.find(f => f.resolution.id === resolution) - expect(file).to.not.be.undefined - - if (file.resolution.id === VideoResolution.H_NOVIDEO) { - expect(file.resolution.label).to.equal('Audio') - } else { - expect(file.resolution.label).to.equal(resolution + 'p') - } - - expect(file.magnetUri).to.have.lengthOf.above(2) - await checkWebTorrentWorks(file.magnetUri) - - { - const nameReg = `${uuidRegex}-${file.resolution.id}` - - expect(file.torrentUrl).to.match(new RegExp(`${server.url}/lazy-static/torrents/${nameReg}-hls.torrent`)) - - if (objectStorageBaseUrl && requiresAuth) { - // eslint-disable-next-line max-len - expect(file.fileUrl).to.match(new RegExp(`${server.url}/object-storage-proxy/streaming-playlists/hls/${privatePath}${videoDetails.uuid}/${nameReg}-fragmented.mp4`)) - } else if (objectStorageBaseUrl) { - expectStartWith(file.fileUrl, objectStorageBaseUrl) - } else { - expect(file.fileUrl).to.match( - new RegExp(`${baseUrl}/static/streaming-playlists/hls/${privatePath}${videoDetails.uuid}/${nameReg}-fragmented.mp4`) - ) - } - } - - { - await Promise.all([ - makeRawRequest({ url: file.torrentUrl, token, expectedStatus: HttpStatusCode.OK_200 }), - makeRawRequest({ url: file.torrentDownloadUrl, token, expectedStatus: HttpStatusCode.OK_200 }), - makeRawRequest({ url: file.metadataUrl, token, expectedStatus: HttpStatusCode.OK_200 }), - makeRawRequest({ url: file.fileUrl, token, expectedStatus: HttpStatusCode.OK_200 }), - - makeRawRequest({ - url: file.fileDownloadUrl, - token, - expectedStatus: objectStorageBaseUrl - ? HttpStatusCode.FOUND_302 - : HttpStatusCode.OK_200 - }) - ]) - } - } - - // Check master playlist - { - await checkResolutionsInMasterPlaylist({ server, token, playlistUrl: hlsPlaylist.playlistUrl, resolutions }) - - const masterPlaylist = await server.streamingPlaylists.get({ url: hlsPlaylist.playlistUrl, token }) - - let i = 0 - for (const resolution of resolutions) { - expect(masterPlaylist).to.contain(`${resolution}.m3u8`) - expect(masterPlaylist).to.contain(`${resolution}.m3u8`) - - const url = 'http://' + videoDetails.account.host - await hlsInfohashExist(url, hlsPlaylist.playlistUrl, i) - - i++ - } - } - - // Check resolution playlists - { - for (const resolution of resolutions) { - const file = hlsFiles.find(f => f.resolution.id === resolution) - const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8' - - let url: string - if (objectStorageBaseUrl && requiresAuth) { - url = `${baseUrl}/object-storage-proxy/streaming-playlists/hls/${privatePath}${videoUUID}/${playlistName}` - } else if (objectStorageBaseUrl) { - url = `${objectStorageBaseUrl}hls/${videoUUID}/${playlistName}` - } else { - url = `${baseUrl}/static/streaming-playlists/hls/${privatePath}${videoUUID}/${playlistName}` - } - - const subPlaylist = await server.streamingPlaylists.get({ url, token }) - - expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`)) - expect(subPlaylist).to.contain(basename(file.fileUrl)) - } - } - - { - let baseUrlAndPath: string - if (objectStorageBaseUrl && requiresAuth) { - baseUrlAndPath = `${baseUrl}/object-storage-proxy/streaming-playlists/hls/${privatePath}${videoUUID}` - } else if (objectStorageBaseUrl) { - baseUrlAndPath = `${objectStorageBaseUrl}hls/${videoUUID}` - } else { - baseUrlAndPath = `${baseUrl}/static/streaming-playlists/hls/${privatePath}${videoUUID}` - } - - for (const resolution of resolutions) { - await checkSegmentHash({ - server, - token, - baseUrlPlaylist: baseUrlAndPath, - baseUrlSegment: baseUrlAndPath, - resolution, - hlsPlaylist - }) - } - } - } -} - -async function checkVideoFileTokenReinjection (options: { - server: PeerTubeServer - videoUUID: string - videoFileToken: string - resolutions: number[] - isLive: boolean -}) { - const { server, resolutions, videoFileToken, videoUUID, isLive } = options - - const video = await server.videos.getWithToken({ id: videoUUID }) - const hls = video.streamingPlaylists[0] - - const query = { videoFileToken, reinjectVideoFileToken: 'true' } - const { text } = await makeRawRequest({ url: hls.playlistUrl, query, expectedStatus: HttpStatusCode.OK_200 }) - - for (let i = 0; i < resolutions.length; i++) { - const resolution = resolutions[i] - - const suffix = isLive - ? i - : `-${resolution}` - - expect(text).to.contain(`${suffix}.m3u8?videoFileToken=${videoFileToken}&reinjectVideoFileToken=true`) - } - - const resolutionPlaylists = extractResolutionPlaylistUrls(hls.playlistUrl, text) - expect(resolutionPlaylists).to.have.lengthOf(resolutions.length) - - for (const url of resolutionPlaylists) { - const { text } = await makeRawRequest({ url, query, expectedStatus: HttpStatusCode.OK_200 }) - - const extension = isLive - ? '.ts' - : '.mp4' - - expect(text).to.contain(`${extension}?videoFileToken=${videoFileToken}`) - expect(text).not.to.contain(`reinjectVideoFileToken=true`) - } -} - -function extractResolutionPlaylistUrls (masterPath: string, masterContent: string) { - return masterContent.match(/^([^.]+\.m3u8.*)/mg) - .map(filename => join(dirname(masterPath), filename)) -} - -export { - checkSegmentHash, - checkLiveSegmentHash, - checkResolutionsInMasterPlaylist, - completeCheckHlsPlaylist, - extractResolutionPlaylistUrls, - checkVideoFileTokenReinjection -} diff --git a/server/tests/shared/tracker.ts b/server/tests/shared/tracker.ts deleted file mode 100644 index 9c1f0246a..000000000 --- a/server/tests/shared/tracker.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { expect } from 'chai' -import { sha1 } from '@shared/extra-utils' -import { makeGetRequest } from '@shared/server-commands' - -async function hlsInfohashExist (serverUrl: string, masterPlaylistUrl: string, fileNumber: number) { - const path = '/tracker/announce' - - const infohash = sha1(`2${masterPlaylistUrl}+V${fileNumber}`) - - // From bittorrent-tracker - const infohashBinary = escape(Buffer.from(infohash, 'hex').toString('binary')).replace(/[@*/+]/g, function (char) { - return '%' + char.charCodeAt(0).toString(16).toUpperCase() - }) - - const res = await makeGetRequest({ - url: serverUrl, - path, - rawQuery: `peer_id=-WW0105-NkvYO/egUAr4&info_hash=${infohashBinary}&port=42100`, - expectedStatus: 200 - }) - - expect(res.text).to.not.contain('failure') -} - -export { - hlsInfohashExist -} diff --git a/server/tests/shared/video-playlists.ts b/server/tests/shared/video-playlists.ts deleted file mode 100644 index 8db303fd8..000000000 --- a/server/tests/shared/video-playlists.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { expect } from 'chai' -import { readdir } from 'fs-extra' -import { PeerTubeServer } from '@shared/server-commands' - -async function checkPlaylistFilesWereRemoved ( - playlistUUID: string, - server: PeerTubeServer, - directories = [ 'thumbnails' ] -) { - for (const directory of directories) { - const directoryPath = server.getDirectoryPath(directory) - - const files = await readdir(directoryPath) - for (const file of files) { - expect(file).to.not.contain(playlistUUID) - } - } -} - -export { - checkPlaylistFilesWereRemoved -} diff --git a/server/tests/shared/videos.ts b/server/tests/shared/videos.ts deleted file mode 100644 index ac24bb173..000000000 --- a/server/tests/shared/videos.ts +++ /dev/null @@ -1,315 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */ - -import { expect } from 'chai' -import { pathExists, readdir } from 'fs-extra' -import { basename, join } from 'path' -import { loadLanguages, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '@server/initializers/constants' -import { getLowercaseExtension, pick, uuidRegex } from '@shared/core-utils' -import { HttpStatusCode, VideoCaption, VideoDetails, VideoPrivacy, VideoResolution } from '@shared/models' -import { makeRawRequest, PeerTubeServer, VideoEdit, waitJobs } from '@shared/server-commands' -import { dateIsValid, expectStartWith, testImageGeneratedByFFmpeg } from './checks' -import { checkWebTorrentWorks } from './webtorrent' - -loadLanguages() - -async function completeWebVideoFilesCheck (options: { - server: PeerTubeServer - originServer: PeerTubeServer - videoUUID: string - fixture: string - files: { - resolution: number - size?: number - }[] - objectStorageBaseUrl?: string -}) { - const { originServer, server, videoUUID, files, fixture, objectStorageBaseUrl } = options - const video = await server.videos.getWithToken({ id: videoUUID }) - const serverConfig = await originServer.config.getConfig() - const requiresAuth = video.privacy.id === VideoPrivacy.PRIVATE || video.privacy.id === VideoPrivacy.INTERNAL - - const transcodingEnabled = serverConfig.transcoding.web_videos.enabled - - for (const attributeFile of files) { - const file = video.files.find(f => f.resolution.id === attributeFile.resolution) - expect(file, `resolution ${attributeFile.resolution} does not exist`).not.to.be.undefined - - let extension = getLowercaseExtension(fixture) - // Transcoding enabled: extension will always be .mp4 - if (transcodingEnabled) extension = '.mp4' - - expect(file.id).to.exist - expect(file.magnetUri).to.have.lengthOf.above(2) - - { - const privatePath = requiresAuth - ? 'private/' - : '' - const nameReg = `${uuidRegex}-${file.resolution.id}` - - expect(file.torrentDownloadUrl).to.match(new RegExp(`${server.url}/download/torrents/${nameReg}.torrent`)) - expect(file.torrentUrl).to.match(new RegExp(`${server.url}/lazy-static/torrents/${nameReg}.torrent`)) - - if (objectStorageBaseUrl && requiresAuth) { - const regexp = new RegExp(`${originServer.url}/object-storage-proxy/web-videos/${privatePath}${nameReg}${extension}`) - expect(file.fileUrl).to.match(regexp) - } else if (objectStorageBaseUrl) { - expectStartWith(file.fileUrl, objectStorageBaseUrl) - } else { - expect(file.fileUrl).to.match(new RegExp(`${originServer.url}/static/web-videos/${privatePath}${nameReg}${extension}`)) - } - - expect(file.fileDownloadUrl).to.match(new RegExp(`${originServer.url}/download/videos/${nameReg}${extension}`)) - } - - { - const token = requiresAuth - ? server.accessToken - : undefined - - await Promise.all([ - makeRawRequest({ url: file.torrentUrl, token, expectedStatus: HttpStatusCode.OK_200 }), - makeRawRequest({ url: file.torrentDownloadUrl, token, expectedStatus: HttpStatusCode.OK_200 }), - makeRawRequest({ url: file.metadataUrl, token, expectedStatus: HttpStatusCode.OK_200 }), - makeRawRequest({ url: file.fileUrl, token, expectedStatus: HttpStatusCode.OK_200 }), - makeRawRequest({ - url: file.fileDownloadUrl, - token, - expectedStatus: objectStorageBaseUrl ? HttpStatusCode.FOUND_302 : HttpStatusCode.OK_200 - }) - ]) - } - - expect(file.resolution.id).to.equal(attributeFile.resolution) - - if (file.resolution.id === VideoResolution.H_NOVIDEO) { - expect(file.resolution.label).to.equal('Audio') - } else { - expect(file.resolution.label).to.equal(attributeFile.resolution + 'p') - } - - if (attributeFile.size) { - const minSize = attributeFile.size - ((10 * attributeFile.size) / 100) - const maxSize = attributeFile.size + ((10 * attributeFile.size) / 100) - expect( - file.size, - 'File size for resolution ' + file.resolution.label + ' outside confidence interval (' + minSize + '> size <' + maxSize + ')' - ).to.be.above(minSize).and.below(maxSize) - } - - await checkWebTorrentWorks(file.magnetUri) - } -} - -async function completeVideoCheck (options: { - server: PeerTubeServer - originServer: PeerTubeServer - videoUUID: string - attributes: { - name: string - category: number - licence: number - language: string - nsfw: boolean - commentsEnabled: boolean - downloadEnabled: boolean - description: string - publishedAt?: string - support: string - originallyPublishedAt?: string - account: { - name: string - host: string - } - isLocal: boolean - tags: string[] - privacy: number - likes?: number - dislikes?: number - duration: number - channel: { - displayName: string - name: string - description: string - isLocal: boolean - } - fixture: string - files: { - resolution: number - size: number - }[] - thumbnailfile?: string - previewfile?: string - } -}) { - const { attributes, originServer, server, videoUUID } = options - - const video = await server.videos.get({ id: videoUUID }) - - if (!attributes.likes) attributes.likes = 0 - if (!attributes.dislikes) attributes.dislikes = 0 - - expect(video.name).to.equal(attributes.name) - expect(video.category.id).to.equal(attributes.category) - expect(video.category.label).to.equal(attributes.category !== null ? VIDEO_CATEGORIES[attributes.category] : 'Unknown') - expect(video.licence.id).to.equal(attributes.licence) - expect(video.licence.label).to.equal(attributes.licence !== null ? VIDEO_LICENCES[attributes.licence] : 'Unknown') - expect(video.language.id).to.equal(attributes.language) - expect(video.language.label).to.equal(attributes.language !== null ? VIDEO_LANGUAGES[attributes.language] : 'Unknown') - expect(video.privacy.id).to.deep.equal(attributes.privacy) - expect(video.privacy.label).to.deep.equal(VIDEO_PRIVACIES[attributes.privacy]) - expect(video.nsfw).to.equal(attributes.nsfw) - expect(video.description).to.equal(attributes.description) - expect(video.account.id).to.be.a('number') - expect(video.account.host).to.equal(attributes.account.host) - expect(video.account.name).to.equal(attributes.account.name) - expect(video.channel.displayName).to.equal(attributes.channel.displayName) - expect(video.channel.name).to.equal(attributes.channel.name) - expect(video.likes).to.equal(attributes.likes) - expect(video.dislikes).to.equal(attributes.dislikes) - expect(video.isLocal).to.equal(attributes.isLocal) - expect(video.duration).to.equal(attributes.duration) - expect(video.url).to.contain(originServer.host) - expect(dateIsValid(video.createdAt)).to.be.true - expect(dateIsValid(video.publishedAt)).to.be.true - expect(dateIsValid(video.updatedAt)).to.be.true - - if (attributes.publishedAt) { - expect(video.publishedAt).to.equal(attributes.publishedAt) - } - - if (attributes.originallyPublishedAt) { - expect(video.originallyPublishedAt).to.equal(attributes.originallyPublishedAt) - } else { - expect(video.originallyPublishedAt).to.be.null - } - - expect(video.files).to.have.lengthOf(attributes.files.length) - expect(video.tags).to.deep.equal(attributes.tags) - expect(video.account.name).to.equal(attributes.account.name) - expect(video.account.host).to.equal(attributes.account.host) - expect(video.channel.displayName).to.equal(attributes.channel.displayName) - expect(video.channel.name).to.equal(attributes.channel.name) - expect(video.channel.host).to.equal(attributes.account.host) - expect(video.channel.isLocal).to.equal(attributes.channel.isLocal) - expect(dateIsValid(video.channel.createdAt.toString())).to.be.true - expect(dateIsValid(video.channel.updatedAt.toString())).to.be.true - expect(video.commentsEnabled).to.equal(attributes.commentsEnabled) - expect(video.downloadEnabled).to.equal(attributes.downloadEnabled) - - expect(video.thumbnailPath).to.exist - await testImageGeneratedByFFmpeg(server.url, attributes.thumbnailfile || attributes.fixture, video.thumbnailPath) - - if (attributes.previewfile) { - expect(video.previewPath).to.exist - await testImageGeneratedByFFmpeg(server.url, attributes.previewfile, video.previewPath) - } - - await completeWebVideoFilesCheck({ server, originServer, videoUUID: video.uuid, ...pick(attributes, [ 'fixture', 'files' ]) }) -} - -async function checkVideoFilesWereRemoved (options: { - server: PeerTubeServer - video: VideoDetails - captions?: VideoCaption[] - onlyVideoFiles?: boolean // default false -}) { - const { video, server, captions = [], onlyVideoFiles = false } = options - - const webVideoFiles = video.files || [] - const hlsFiles = video.streamingPlaylists[0]?.files || [] - - const thumbnailName = basename(video.thumbnailPath) - const previewName = basename(video.previewPath) - - const torrentNames = webVideoFiles.concat(hlsFiles).map(f => basename(f.torrentUrl)) - - const captionNames = captions.map(c => basename(c.captionPath)) - - const webVideoFilenames = webVideoFiles.map(f => basename(f.fileUrl)) - const hlsFilenames = hlsFiles.map(f => basename(f.fileUrl)) - - let directories: { [ directory: string ]: string[] } = { - videos: webVideoFilenames, - redundancy: webVideoFilenames, - [join('playlists', 'hls')]: hlsFilenames, - [join('redundancy', 'hls')]: hlsFilenames - } - - if (onlyVideoFiles !== true) { - directories = { - ...directories, - - thumbnails: [ thumbnailName ], - previews: [ previewName ], - torrents: torrentNames, - captions: captionNames - } - } - - for (const directory of Object.keys(directories)) { - const directoryPath = server.servers.buildDirectory(directory) - - const directoryExists = await pathExists(directoryPath) - if (directoryExists === false) continue - - const existingFiles = await readdir(directoryPath) - for (const existingFile of existingFiles) { - for (const shouldNotExist of directories[directory]) { - expect(existingFile, `File ${existingFile} should not exist in ${directoryPath}`).to.not.contain(shouldNotExist) - } - } - } -} - -async function saveVideoInServers (servers: PeerTubeServer[], uuid: string) { - for (const server of servers) { - server.store.videoDetails = await server.videos.get({ id: uuid }) - } -} - -function checkUploadVideoParam (options: { - server: PeerTubeServer - token: string - attributes: Partial - expectedStatus?: HttpStatusCode - completedExpectedStatus?: HttpStatusCode - mode?: 'legacy' | 'resumable' -}) { - const { server, token, attributes, completedExpectedStatus, expectedStatus, mode = 'legacy' } = options - - return mode === 'legacy' - ? server.videos.buildLegacyUpload({ token, attributes, expectedStatus: expectedStatus || completedExpectedStatus }) - : server.videos.buildResumeUpload({ - token, - attributes, - expectedStatus, - completedExpectedStatus, - path: '/api/v1/videos/upload-resumable' - }) -} - -// serverNumber starts from 1 -async function uploadRandomVideoOnServers ( - servers: PeerTubeServer[], - serverNumber: number, - additionalParams?: VideoEdit & { prefixName?: string } -) { - const server = servers.find(s => s.serverNumber === serverNumber) - const res = await server.videos.randomUpload({ wait: false, additionalParams }) - - await waitJobs(servers) - - return res -} - -// --------------------------------------------------------------------------- - -export { - completeVideoCheck, - completeWebVideoFilesCheck, - checkUploadVideoParam, - uploadRandomVideoOnServers, - checkVideoFilesWereRemoved, - saveVideoInServers -} diff --git a/server/tests/shared/views.ts b/server/tests/shared/views.ts deleted file mode 100644 index e6b289715..000000000 --- a/server/tests/shared/views.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { FfmpegCommand } from 'fluent-ffmpeg' -import { wait } from '@shared/core-utils' -import { VideoCreateResult, VideoPrivacy } from '@shared/models' -import { - createMultipleServers, - doubleFollow, - PeerTubeServer, - setAccessTokensToServers, - setDefaultVideoChannel, - waitJobs, - waitUntilLivePublishedOnAllServers -} from '@shared/server-commands' - -async function processViewersStats (servers: PeerTubeServer[]) { - await wait(6000) - - for (const server of servers) { - await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } }) - await server.debug.sendCommand({ body: { command: 'process-video-viewers' } }) - } - - await waitJobs(servers) -} - -async function processViewsBuffer (servers: PeerTubeServer[]) { - for (const server of servers) { - await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } }) - } - - await waitJobs(servers) -} - -async function prepareViewsServers () { - const servers = await createMultipleServers(2) - await setAccessTokensToServers(servers) - await setDefaultVideoChannel(servers) - - await servers[0].config.updateCustomSubConfig({ - newConfig: { - live: { - enabled: true, - allowReplay: true, - transcoding: { - enabled: false - } - } - } - }) - - await doubleFollow(servers[0], servers[1]) - - return servers -} - -async function prepareViewsVideos (options: { - servers: PeerTubeServer[] - live: boolean - vod: boolean -}) { - const { servers } = options - - const liveAttributes = { - name: 'live video', - channelId: servers[0].store.channel.id, - privacy: VideoPrivacy.PUBLIC - } - - let ffmpegCommand: FfmpegCommand - let live: VideoCreateResult - let vod: VideoCreateResult - - if (options.live) { - live = await servers[0].live.create({ fields: liveAttributes }) - - ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: live.uuid }) - await waitUntilLivePublishedOnAllServers(servers, live.uuid) - } - - if (options.vod) { - vod = await servers[0].videos.quickUpload({ name: 'video' }) - } - - await waitJobs(servers) - - return { liveVideoId: live?.uuid, vodVideoId: vod?.uuid, ffmpegCommand } -} - -export { - processViewersStats, - prepareViewsServers, - processViewsBuffer, - prepareViewsVideos -} diff --git a/server/tests/shared/webtorrent.ts b/server/tests/shared/webtorrent.ts deleted file mode 100644 index d5bd86500..000000000 --- a/server/tests/shared/webtorrent.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { expect } from 'chai' -import { readFile } from 'fs-extra' -import parseTorrent from 'parse-torrent' -import { basename, join } from 'path' -import * as WebTorrent from 'webtorrent' -import { VideoFile } from '@shared/models' -import { PeerTubeServer } from '@shared/server-commands' - -let webtorrent: WebTorrent.Instance - -export async function checkWebTorrentWorks (magnetUri: string, pathMatch?: RegExp) { - const torrent = await webtorrentAdd(magnetUri, true) - - expect(torrent.files).to.be.an('array') - expect(torrent.files.length).to.equal(1) - expect(torrent.files[0].path).to.exist.and.to.not.equal('') - - if (pathMatch) { - expect(torrent.files[0].path).match(pathMatch) - } -} - -export async function parseTorrentVideo (server: PeerTubeServer, file: VideoFile) { - const torrentName = basename(file.torrentUrl) - const torrentPath = server.servers.buildDirectory(join('torrents', torrentName)) - - const data = await readFile(torrentPath) - - return parseTorrent(data) -} - -// --------------------------------------------------------------------------- -// Private -// --------------------------------------------------------------------------- - -function webtorrentAdd (torrentId: string, refreshWebTorrent = false) { - const WebTorrent = require('webtorrent') - - if (webtorrent && refreshWebTorrent) webtorrent.destroy() - if (!webtorrent || refreshWebTorrent) webtorrent = new WebTorrent() - - webtorrent.on('error', err => console.error('Error in webtorrent', err)) - - return new Promise(res => { - const torrent = webtorrent.add(torrentId, res) - - torrent.on('error', err => console.error('Error in webtorrent torrent', err)) - torrent.on('warning', warn => { - const msg = typeof warn === 'string' - ? warn - : warn.message - - if (msg.includes('Unsupported')) return - - console.error('Warning in webtorrent torrent', warn) - }) - }) -} diff --git a/server/tools/README.md b/server/tools/README.md deleted file mode 100644 index d7ecd4004..000000000 --- a/server/tools/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# PeerTube CLI - -See https://docs.joinpeertube.org/maintain/tools#remote-tools diff --git a/server/tools/package.json b/server/tools/package.json deleted file mode 100644 index b20f38244..000000000 --- a/server/tools/package.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name": "@peertube/cli", - "version": "1.0.0", - "private": true, - "dependencies": { - "application-config": "^2.0.0", - "cli-table3": "^0.6.0", - "netrc-parser": "^3.1.6" - }, - "devDependencies": {} -} diff --git a/server/tools/peertube-auth.ts b/server/tools/peertube-auth.ts deleted file mode 100644 index c853469c2..000000000 --- a/server/tools/peertube-auth.ts +++ /dev/null @@ -1,171 +0,0 @@ -import CliTable3 from 'cli-table3' -import { OptionValues, program } from 'commander' -import { isUserUsernameValid } from '../helpers/custom-validators/users' -import { assignToken, buildServer, getNetrc, getSettings, writeSettings } from './shared' - -import prompt = require('prompt') - -async function delInstance (url: string) { - const [ settings, netrc ] = await Promise.all([ getSettings(), getNetrc() ]) - - const index = settings.remotes.indexOf(url) - settings.remotes.splice(index) - - if (settings.default === index) settings.default = -1 - - await writeSettings(settings) - - delete netrc.machines[url] - - await netrc.save() -} - -async function setInstance (url: string, username: string, password: string, isDefault: boolean) { - const [ settings, netrc ] = await Promise.all([ getSettings(), getNetrc() ]) - - if (settings.remotes.includes(url) === false) { - settings.remotes.push(url) - } - - if (isDefault || settings.remotes.length === 1) { - settings.default = settings.remotes.length - 1 - } - - await writeSettings(settings) - - netrc.machines[url] = { login: username, password } - await netrc.save() -} - -function isURLaPeerTubeInstance (url: string) { - return url.startsWith('http://') || url.startsWith('https://') -} - -function stripExtraneousFromPeerTubeUrl (url: string) { - // Get everything before the 3rd /. - const urlLength = url.includes('/', 8) - ? url.indexOf('/', 8) - : url.length - - return url.substring(0, urlLength) -} - -program - .name('auth') - .usage('[command] [options]') - -program - .command('add') - .description('remember your accounts on remote instances for easier use') - .option('-u, --url ', 'Server url') - .option('-U, --username ', 'Username') - .option('-p, --password ', 'Password') - .option('--default', 'add the entry as the new default') - .action((options: OptionValues) => { - /* eslint-disable no-import-assign */ - prompt.override = options - prompt.start() - prompt.get({ - properties: { - url: { - description: 'instance url', - conform: (value) => isURLaPeerTubeInstance(value), - message: 'It should be an URL (https://peertube.example.com)', - required: true - }, - username: { - conform: (value) => isUserUsernameValid(value), - message: 'Name must be only letters, spaces, or dashes', - required: true - }, - password: { - hidden: true, - replace: '*', - required: true - } - } - }, async (_, result) => { - - // Check credentials - try { - // Strip out everything after the domain:port. - // See https://github.com/Chocobozzz/PeerTube/issues/3520 - result.url = stripExtraneousFromPeerTubeUrl(result.url) - - const server = buildServer(result.url) - await assignToken(server, result.username, result.password) - } catch (err) { - console.error(err.message) - process.exit(-1) - } - - await setInstance(result.url, result.username, result.password, options.default) - - process.exit(0) - }) - }) - -program - .command('del ') - .description('unregisters a remote instance') - .action(async url => { - await delInstance(url) - - process.exit(0) - }) - -program - .command('list') - .description('lists registered remote instances') - .action(async () => { - const [ settings, netrc ] = await Promise.all([ getSettings(), getNetrc() ]) - - const table = new CliTable3({ - head: [ 'instance', 'login' ], - colWidths: [ 30, 30 ] - }) as any - - settings.remotes.forEach(element => { - if (!netrc.machines[element]) return - - table.push([ - element, - netrc.machines[element].login - ]) - }) - - console.log(table.toString()) - - process.exit(0) - }) - -program - .command('set-default ') - .description('set an existing entry as default') - .action(async url => { - const settings = await getSettings() - const instanceExists = settings.remotes.includes(url) - - if (instanceExists) { - settings.default = settings.remotes.indexOf(url) - await writeSettings(settings) - - process.exit(0) - } else { - console.log(' is not a registered instance.') - process.exit(-1) - } - }) - -program.addHelpText('after', '\n\n Examples:\n\n' + - ' $ peertube auth add -u https://peertube.cpy.re -U "PEERTUBE_USER" --password "PEERTUBE_PASSWORD"\n' + - ' $ peertube auth add -u https://peertube.cpy.re -U root\n' + - ' $ peertube auth list\n' + - ' $ peertube auth del https://peertube.cpy.re\n' -) - -if (!process.argv.slice(2).length) { - program.outputHelp() -} - -program.parse(process.argv) diff --git a/server/tools/peertube-get-access-token.ts b/server/tools/peertube-get-access-token.ts deleted file mode 100644 index 71a4826e8..000000000 --- a/server/tools/peertube-get-access-token.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { program } from 'commander' -import { assignToken, buildServer } from './shared' - -program - .option('-u, --url ', 'Server url') - .option('-n, --username ', 'Username') - .option('-p, --password ', 'Password') - .parse(process.argv) - -const options = program.opts() - -if ( - !options.url || - !options.username || - !options.password -) { - if (!options.url) console.error('--url field is required.') - if (!options.username) console.error('--username field is required.') - if (!options.password) console.error('--password field is required.') - - process.exit(-1) -} - -const server = buildServer(options.url) - -assignToken(server, options.username, options.password) - .then(() => { - console.log(server.accessToken) - process.exit(0) - }) - .catch(err => { - console.error(err) - process.exit(-1) - }) diff --git a/server/tools/peertube-import-videos.ts b/server/tools/peertube-import-videos.ts deleted file mode 100644 index bbdaa09c0..000000000 --- a/server/tools/peertube-import-videos.ts +++ /dev/null @@ -1,351 +0,0 @@ -import { program } from 'commander' -import { accessSync, constants } from 'fs' -import { remove } from 'fs-extra' -import { join } from 'path' -import { YoutubeDLCLI, YoutubeDLInfo, YoutubeDLInfoBuilder } from '@server/helpers/youtube-dl' -import { wait } from '@shared/core-utils' -import { sha256 } from '@shared/extra-utils' -import { doRequestAndSaveToFile } from '../helpers/requests' -import { - assignToken, - buildCommonVideoOptions, - buildServer, - buildVideoAttributesFromCommander, - getLogger, - getServerCredentials -} from './shared' - -import prompt = require('prompt') - -const processOptions = { - maxBuffer: Infinity -} - -let command = program - .name('import-videos') - -command = buildCommonVideoOptions(command) - -command - .option('-u, --url ', 'Server url') - .option('-U, --username ', 'Username') - .option('-p, --password ', 'Password') - .option('--target-url ', 'Video target URL') - .option('--since ', 'Publication date (inclusive) since which the videos can be imported (YYYY-MM-DD)', parseDate) - .option('--until ', 'Publication date (inclusive) until which the videos can be imported (YYYY-MM-DD)', parseDate) - .option('--first ', 'Process first n elements of returned playlist') - .option('--last ', 'Process last n elements of returned playlist') - .option('--wait-interval ', 'Duration between two video imports (in seconds)', convertIntoMs) - .option('-T, --tmpdir ', 'Working directory', __dirname) - .usage('[global options] [ -- youtube-dl options]') - .parse(process.argv) - -const options = command.opts() - -const log = getLogger(options.verbose) - -getServerCredentials(command) - .then(({ url, username, password }) => { - if (!options.targetUrl) { - exitError('--target-url field is required.') - } - - try { - accessSync(options.tmpdir, constants.R_OK | constants.W_OK) - } catch (e) { - exitError('--tmpdir %s: directory does not exist or is not accessible', options.tmpdir) - } - - url = normalizeTargetUrl(url) - options.targetUrl = normalizeTargetUrl(options.targetUrl) - - run(url, username, password) - .catch(err => exitError(err)) - }) - .catch(err => console.error(err)) - -async function run (url: string, username: string, password: string) { - if (!password) password = await promptPassword() - - const youtubeDLBinary = await YoutubeDLCLI.safeGet() - - let info = await getYoutubeDLInfo(youtubeDLBinary, options.targetUrl, command.args) - - if (!Array.isArray(info)) info = [ info ] - - // Try to fix youtube channels upload - const uploadsObject = info.find(i => !i.ie_key && !i.duration && i.title === 'Uploads') - - if (uploadsObject) { - console.log('Fixing URL to %s.', uploadsObject.url) - - info = await getYoutubeDLInfo(youtubeDLBinary, uploadsObject.url, command.args) - } - - let infoArray: any[] - - infoArray = [].concat(info) - if (options.first) { - infoArray = infoArray.slice(0, options.first) - } else if (options.last) { - infoArray = infoArray.slice(-options.last) - } - - log.info('Will download and upload %d videos.\n', infoArray.length) - - let skipInterval = true - for (const [ index, info ] of infoArray.entries()) { - try { - if (index > 0 && options.waitInterval && !skipInterval) { - log.info('Wait for %d seconds before continuing.', options.waitInterval / 1000) - await wait(options.waitInterval) - } - - skipInterval = await processVideo({ - cwd: options.tmpdir, - url, - username, - password, - youtubeInfo: info - }) - } catch (err) { - console.error('Cannot process video.', { info, url, err }) - } - } - - log.info('Video/s for user %s imported: %s', username, options.targetUrl) - process.exit(0) -} - -async function processVideo (parameters: { - cwd: string - url: string - username: string - password: string - youtubeInfo: any -}) { - const { youtubeInfo, cwd, url, username, password } = parameters - - log.debug('Fetching object.', youtubeInfo) - - const videoInfo = await fetchObject(youtubeInfo) - log.debug('Fetched object.', videoInfo) - - if ( - options.since && - videoInfo.originallyPublishedAtWithoutTime && - videoInfo.originallyPublishedAtWithoutTime.getTime() < options.since.getTime() - ) { - log.info('Video "%s" has been published before "%s", don\'t upload it.\n', videoInfo.name, formatDate(options.since)) - return true - } - - if ( - options.until && - videoInfo.originallyPublishedAtWithoutTime && - videoInfo.originallyPublishedAtWithoutTime.getTime() > options.until.getTime() - ) { - log.info('Video "%s" has been published after "%s", don\'t upload it.\n', videoInfo.name, formatDate(options.until)) - return true - } - - const server = buildServer(url) - const { data } = await server.search.advancedVideoSearch({ - search: { - search: videoInfo.name, - sort: '-match', - searchTarget: 'local' - } - }) - - log.info('############################################################\n') - - if (data.find(v => v.name === videoInfo.name)) { - log.info('Video "%s" already exists, don\'t reupload it.\n', videoInfo.name) - return true - } - - const path = join(cwd, sha256(videoInfo.url) + '.mp4') - - log.info('Downloading video "%s"...', videoInfo.name) - - try { - const youtubeDLBinary = await YoutubeDLCLI.safeGet() - const output = await youtubeDLBinary.download({ - url: videoInfo.url, - format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false), - output: path, - additionalYoutubeDLArgs: command.args, - processOptions - }) - - log.info(output.join('\n')) - await uploadVideoOnPeerTube({ - cwd, - url, - username, - password, - videoInfo, - videoPath: path - }) - } catch (err) { - log.error(err.message) - } - - return false -} - -async function uploadVideoOnPeerTube (parameters: { - videoInfo: YoutubeDLInfo - videoPath: string - cwd: string - url: string - username: string - password: string -}) { - const { videoInfo, videoPath, cwd, url, username, password } = parameters - - const server = buildServer(url) - await assignToken(server, username, password) - - let thumbnailfile: string - if (videoInfo.thumbnailUrl) { - thumbnailfile = join(cwd, sha256(videoInfo.thumbnailUrl) + '.jpg') - - await doRequestAndSaveToFile(videoInfo.thumbnailUrl, thumbnailfile) - } - - const baseAttributes = await buildVideoAttributesFromCommander(server, program, videoInfo) - - const attributes = { - ...baseAttributes, - - originallyPublishedAtWithoutTime: videoInfo.originallyPublishedAtWithoutTime - ? videoInfo.originallyPublishedAtWithoutTime.toISOString() - : null, - - thumbnailfile, - previewfile: thumbnailfile, - fixture: videoPath - } - - log.info('\nUploading on PeerTube video "%s".', attributes.name) - - try { - await server.videos.upload({ attributes }) - } catch (err) { - if (err.message.indexOf('401') !== -1) { - log.info('Got 401 Unauthorized, token may have expired, renewing token and retry.') - - server.accessToken = await server.login.getAccessToken(username, password) - - await server.videos.upload({ attributes }) - } else { - exitError(err.message) - } - } - - await remove(videoPath) - if (thumbnailfile) await remove(thumbnailfile) - - log.info('Uploaded video "%s"!\n', attributes.name) -} - -/* ---------------------------------------------------------- */ - -async function fetchObject (info: any) { - const url = buildUrl(info) - - const youtubeDLCLI = await YoutubeDLCLI.safeGet() - const result = await youtubeDLCLI.getInfo({ - url, - format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false), - processOptions - }) - - const builder = new YoutubeDLInfoBuilder(result) - - const videoInfo = builder.getInfo() - - return { ...videoInfo, url } -} - -function buildUrl (info: any) { - const webpageUrl = info.webpage_url as string - if (webpageUrl?.match(/^https?:\/\//)) return webpageUrl - - const url = info.url as string - if (url?.match(/^https?:\/\//)) return url - - // It seems youtube-dl does not return the video url - return 'https://www.youtube.com/watch?v=' + info.id -} - -function normalizeTargetUrl (url: string) { - let normalizedUrl = url.replace(/\/+$/, '') - - if (!normalizedUrl.startsWith('http://') && !normalizedUrl.startsWith('https://')) { - normalizedUrl = 'https://' + normalizedUrl - } - - return normalizedUrl -} - -async function promptPassword () { - return new Promise((res, rej) => { - prompt.start() - const schema = { - properties: { - password: { - hidden: true, - required: true - } - } - } - prompt.get(schema, function (err, result) { - if (err) { - return rej(err) - } - return res(result.password) - }) - }) -} - -function parseDate (dateAsStr: string): Date { - if (!/\d{4}-\d{2}-\d{2}/.test(dateAsStr)) { - exitError(`Invalid date passed: ${dateAsStr}. Expected format: YYYY-MM-DD. See help for usage.`) - } - const date = new Date(dateAsStr) - date.setHours(0, 0, 0) - if (isNaN(date.getTime())) { - exitError(`Invalid date passed: ${dateAsStr}. See help for usage.`) - } - return date -} - -function formatDate (date: Date): string { - return date.toISOString().split('T')[0] -} - -function convertIntoMs (secondsAsStr: string): number { - const seconds = parseInt(secondsAsStr, 10) - if (seconds <= 0) { - exitError(`Invalid duration passed: ${seconds}. Expected duration to be strictly positive and in seconds`) - } - return Math.round(seconds * 1000) -} - -function exitError (message: string, ...meta: any[]) { - // use console.error instead of log.error here - console.error(message, ...meta) - process.exit(-1) -} - -function getYoutubeDLInfo (youtubeDLCLI: YoutubeDLCLI, url: string, args: string[]) { - return youtubeDLCLI.getInfo({ - url, - format: YoutubeDLCLI.getYoutubeDLVideoFormat([], false), - additionalYoutubeDLArgs: [ '-j', '--flat-playlist', '--playlist-reverse', ...args ], - processOptions - }) -} diff --git a/server/tools/peertube-plugins.ts b/server/tools/peertube-plugins.ts deleted file mode 100644 index 0660c855f..000000000 --- a/server/tools/peertube-plugins.ts +++ /dev/null @@ -1,165 +0,0 @@ -import CliTable3 from 'cli-table3' -import { Command, OptionValues, program } from 'commander' -import { isAbsolute } from 'path' -import { PluginType } from '../../shared/models' -import { assignToken, buildServer, getServerCredentials } from './shared' - -program - .name('plugins') - .usage('[command] [options]') - -program - .command('list') - .description('List installed plugins') - .option('-u, --url ', 'Server url') - .option('-U, --username ', 'Username') - .option('-p, --password ', 'Password') - .option('-t, --only-themes', 'List themes only') - .option('-P, --only-plugins', 'List plugins only') - .action((options, command) => pluginsListCLI(command, options)) - -program - .command('install') - .description('Install a plugin or a theme') - .option('-u, --url ', 'Server url') - .option('-U, --username ', 'Username') - .option('-p, --password ', 'Password') - .option('-P --path ', 'Install from a path') - .option('-n, --npm-name ', 'Install from npm') - .option('--plugin-version ', 'Specify the plugin version to install (only available when installing from npm)') - .action((options, command) => installPluginCLI(command, options)) - -program - .command('update') - .description('Update a plugin or a theme') - .option('-u, --url ', 'Server url') - .option('-U, --username ', 'Username') - .option('-p, --password ', 'Password') - .option('-P --path ', 'Update from a path') - .option('-n, --npm-name ', 'Update from npm') - .action((options, command) => updatePluginCLI(command, options)) - -program - .command('uninstall') - .description('Uninstall a plugin or a theme') - .option('-u, --url ', 'Server url') - .option('-U, --username ', 'Username') - .option('-p, --password ', 'Password') - .option('-n, --npm-name ', 'NPM plugin/theme name') - .action((options, command) => uninstallPluginCLI(command, options)) - -if (!process.argv.slice(2).length) { - program.outputHelp() -} - -program.parse(process.argv) - -// ---------------------------------------------------------------------------- - -async function pluginsListCLI (command: Command, options: OptionValues) { - const { url, username, password } = await getServerCredentials(command) - const server = buildServer(url) - await assignToken(server, username, password) - - let pluginType: PluginType - if (options.onlyThemes) pluginType = PluginType.THEME - if (options.onlyPlugins) pluginType = PluginType.PLUGIN - - const { data } = await server.plugins.list({ start: 0, count: 100, sort: 'name', pluginType }) - - const table = new CliTable3({ - head: [ 'name', 'version', 'homepage' ], - colWidths: [ 50, 20, 50 ] - }) as any - - for (const plugin of data) { - const npmName = plugin.type === PluginType.PLUGIN - ? 'peertube-plugin-' + plugin.name - : 'peertube-theme-' + plugin.name - - table.push([ - npmName, - plugin.version, - plugin.homepage - ]) - } - - console.log(table.toString()) - process.exit(0) -} - -async function installPluginCLI (command: Command, options: OptionValues) { - if (!options.path && !options.npmName) { - console.error('You need to specify the npm name or the path of the plugin you want to install.\n') - program.outputHelp() - process.exit(-1) - } - - if (options.path && !isAbsolute(options.path)) { - console.error('Path should be absolute.') - process.exit(-1) - } - - const { url, username, password } = await getServerCredentials(command) - const server = buildServer(url) - await assignToken(server, username, password) - - try { - await server.plugins.install({ npmName: options.npmName, path: options.path, pluginVersion: options.pluginVersion }) - } catch (err) { - console.error('Cannot install plugin.', err) - process.exit(-1) - } - - console.log('Plugin installed.') - process.exit(0) -} - -async function updatePluginCLI (command: Command, options: OptionValues) { - if (!options.path && !options.npmName) { - console.error('You need to specify the npm name or the path of the plugin you want to update.\n') - program.outputHelp() - process.exit(-1) - } - - if (options.path && !isAbsolute(options.path)) { - console.error('Path should be absolute.') - process.exit(-1) - } - - const { url, username, password } = await getServerCredentials(command) - const server = buildServer(url) - await assignToken(server, username, password) - - try { - await server.plugins.update({ npmName: options.npmName, path: options.path }) - } catch (err) { - console.error('Cannot update plugin.', err) - process.exit(-1) - } - - console.log('Plugin updated.') - process.exit(0) -} - -async function uninstallPluginCLI (command: Command, options: OptionValues) { - if (!options.npmName) { - console.error('You need to specify the npm name of the plugin/theme you want to uninstall.\n') - program.outputHelp() - process.exit(-1) - } - - const { url, username, password } = await getServerCredentials(command) - const server = buildServer(url) - await assignToken(server, username, password) - - try { - await server.plugins.uninstall({ npmName: options.npmName }) - } catch (err) { - console.error('Cannot uninstall plugin.', err) - process.exit(-1) - } - - console.log('Plugin uninstalled.') - process.exit(0) -} diff --git a/server/tools/peertube-redundancy.ts b/server/tools/peertube-redundancy.ts deleted file mode 100644 index c24eb5233..000000000 --- a/server/tools/peertube-redundancy.ts +++ /dev/null @@ -1,172 +0,0 @@ -import CliTable3 from 'cli-table3' -import { Command, program } from 'commander' -import { URL } from 'url' -import validator from 'validator' -import { forceNumber, uniqify } from '@shared/core-utils' -import { HttpStatusCode, VideoRedundanciesTarget } from '@shared/models' -import { assignToken, buildServer, getServerCredentials } from './shared' - -import bytes = require('bytes') -program - .name('redundancy') - .usage('[command] [options]') - -program - .command('list-remote-redundancies') - .description('List remote redundancies on your videos') - .option('-u, --url ', 'Server url') - .option('-U, --username ', 'Username') - .option('-p, --password ', 'Password') - .action(() => listRedundanciesCLI('my-videos')) - -program - .command('list-my-redundancies') - .description('List your redundancies of remote videos') - .option('-u, --url ', 'Server url') - .option('-U, --username ', 'Username') - .option('-p, --password ', 'Password') - .action(() => listRedundanciesCLI('remote-videos')) - -program - .command('add') - .description('Duplicate a video in your redundancy system') - .option('-u, --url ', 'Server url') - .option('-U, --username ', 'Username') - .option('-p, --password ', 'Password') - .option('-v, --video ', 'Video id to duplicate') - .action((options, command) => addRedundancyCLI(options, command)) - -program - .command('remove') - .description('Remove a video from your redundancies') - .option('-u, --url ', 'Server url') - .option('-U, --username ', 'Username') - .option('-p, --password ', 'Password') - .option('-v, --video ', 'Video id to remove from redundancies') - .action((options, command) => removeRedundancyCLI(options, command)) - -if (!process.argv.slice(2).length) { - program.outputHelp() -} - -program.parse(process.argv) - -// ---------------------------------------------------------------------------- - -async function listRedundanciesCLI (target: VideoRedundanciesTarget) { - const { url, username, password } = await getServerCredentials(program) - const server = buildServer(url) - await assignToken(server, username, password) - - const { data } = await server.redundancy.listVideos({ start: 0, count: 100, sort: 'name', target }) - - const table = new CliTable3({ - head: [ 'video id', 'video name', 'video url', 'files', 'playlists', 'by instances', 'total size' ] - }) as any - - for (const redundancy of data) { - const webVideoFiles = redundancy.redundancies.files - const streamingPlaylists = redundancy.redundancies.streamingPlaylists - - let totalSize = '' - if (target === 'remote-videos') { - const tmp = webVideoFiles.concat(streamingPlaylists) - .reduce((a, b) => a + b.size, 0) - - totalSize = bytes(tmp) - } - - const instances = uniqify( - webVideoFiles.concat(streamingPlaylists) - .map(r => r.fileUrl) - .map(u => new URL(u).host) - ) - - table.push([ - redundancy.id.toString(), - redundancy.name, - redundancy.url, - webVideoFiles.length, - streamingPlaylists.length, - instances.join('\n'), - totalSize - ]) - } - - console.log(table.toString()) - process.exit(0) -} - -async function addRedundancyCLI (options: { video: number }, command: Command) { - const { url, username, password } = await getServerCredentials(command) - const server = buildServer(url) - await assignToken(server, username, password) - - if (!options.video || validator.isInt('' + options.video) === false) { - console.error('You need to specify the video id to duplicate and it should be a number.\n') - command.outputHelp() - process.exit(-1) - } - - try { - await server.redundancy.addVideo({ videoId: options.video }) - - console.log('Video will be duplicated by your instance!') - - process.exit(0) - } catch (err) { - if (err.message.includes(HttpStatusCode.CONFLICT_409)) { - console.error('This video is already duplicated by your instance.') - } else if (err.message.includes(HttpStatusCode.NOT_FOUND_404)) { - console.error('This video id does not exist.') - } else { - console.error(err) - } - - process.exit(-1) - } -} - -async function removeRedundancyCLI (options: { video: number }, command: Command) { - const { url, username, password } = await getServerCredentials(command) - const server = buildServer(url) - await assignToken(server, username, password) - - if (!options.video || validator.isInt('' + options.video) === false) { - console.error('You need to specify the video id to remove from your redundancies.\n') - command.outputHelp() - process.exit(-1) - } - - const videoId = forceNumber(options.video) - - const myVideoRedundancies = await server.redundancy.listVideos({ target: 'my-videos' }) - let videoRedundancy = myVideoRedundancies.data.find(r => videoId === r.id) - - if (!videoRedundancy) { - const remoteVideoRedundancies = await server.redundancy.listVideos({ target: 'remote-videos' }) - videoRedundancy = remoteVideoRedundancies.data.find(r => videoId === r.id) - } - - if (!videoRedundancy) { - console.error('Video redundancy not found.') - process.exit(-1) - } - - try { - const ids = videoRedundancy.redundancies.files - .concat(videoRedundancy.redundancies.streamingPlaylists) - .map(r => r.id) - - for (const id of ids) { - await server.redundancy.removeVideo({ redundancyId: id }) - } - - console.log('Video redundancy removed!') - - process.exit(0) - } catch (err) { - console.error(err) - process.exit(-1) - } -} diff --git a/server/tools/peertube-upload.ts b/server/tools/peertube-upload.ts deleted file mode 100644 index 87da55005..000000000 --- a/server/tools/peertube-upload.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { program } from 'commander' -import { access, constants } from 'fs-extra' -import { isAbsolute } from 'path' -import { assignToken, buildCommonVideoOptions, buildServer, buildVideoAttributesFromCommander, getServerCredentials } from './shared' - -let command = program - .name('upload') - -command = buildCommonVideoOptions(command) - -command - .option('-u, --url ', 'Server url') - .option('-U, --username ', 'Username') - .option('-p, --password ', 'Password') - .option('-b, --thumbnail ', 'Thumbnail path') - .option('-v, --preview ', 'Preview path') - .option('-f, --file ', 'Video absolute file path') - .parse(process.argv) - -const options = command.opts() - -getServerCredentials(command) - .then(({ url, username, password }) => { - if (!options.videoName || !options.file) { - if (!options.videoName) console.error('--video-name is required.') - if (!options.file) console.error('--file is required.') - - process.exit(-1) - } - - if (isAbsolute(options.file) === false) { - console.error('File path should be absolute.') - process.exit(-1) - } - - run(url, username, password).catch(err => { - console.error(err) - process.exit(-1) - }) - }) - .catch(err => console.error(err)) - -async function run (url: string, username: string, password: string) { - const server = buildServer(url) - await assignToken(server, username, password) - - await access(options.file, constants.F_OK) - - console.log('Uploading %s video...', options.videoName) - - const baseAttributes = await buildVideoAttributesFromCommander(server, program) - - const attributes = { - ...baseAttributes, - - fixture: options.file, - thumbnailfile: options.thumbnail, - previewfile: options.preview - } - - try { - await server.videos.upload({ attributes }) - console.log(`Video ${options.videoName} uploaded.`) - process.exit(0) - } catch (err) { - const message = err.message || '' - if (message.includes('413')) { - console.error('Aborted: user quota is exceeded or video file is too big for this PeerTube instance.') - } else { - console.error(require('util').inspect(err)) - } - - process.exit(-1) - } -} - -// ---------------------------------------------------------------------------- diff --git a/server/tools/peertube.ts b/server/tools/peertube.ts deleted file mode 100644 index b79917b4f..000000000 --- a/server/tools/peertube.ts +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env node - -import { CommandOptions, program } from 'commander' -import { getSettings, version } from './shared' - -program - .version(version, '-v, --version') - .usage('[command] [options]') - -/* Subcommands automatically loaded in the directory and beginning by peertube-* */ -program - .command('auth [action]', 'register your accounts on remote instances to use them with other commands') - .command('upload', 'upload a video').alias('up') - .command('import-videos', 'import a video from a streaming platform').alias('import') - .command('get-access-token', 'get a peertube access token', { noHelp: true }).alias('token') - .command('plugins [action]', 'manage instance plugins/themes').alias('p') - .command('redundancy [action]', 'manage instance redundancies').alias('r') - -/* Not Yet Implemented */ -program - .command( - 'diagnostic [action]', - 'like couple therapy, but for your instance', - { noHelp: true } as CommandOptions - ).alias('d') - .command('admin', - 'manage an instance where you have elevated rights', - { noHelp: true } as CommandOptions - ).alias('a') - -// help on no command -if (!process.argv.slice(2).length) { - const logo = '░P░e░e░r░T░u░b░e░' - console.log(` - ___/),.._ ` + logo + ` -/' ,. ."'._ -( "' '-.__"-._ ,- -\\'='='), "\\ -._-"-. -"/ - / ""/"\\,_\\,__"" _" /,- - / / -" _/"/ - / | ._\\\\ |\\ |_.".-" / - / | __\\)|)|),/|_." _,." - / \\_." " ") | ).-""---''-- - ( "/.""7__-""'' - | " ."._--._ - \\ \\ (_ __ "" ".,_ - \\.,. \\ "" -"".-" - ".,_, (",_-,,,-".- - "'-,\\_ __,-" - ",)" ") - /"\\-" - ,"\\/ - _,.__/"\\/_ (the CLI for red chocobos) - / \\) "./, ". - --/---"---" "-) )---- by Chocobozzz et al.\n`) -} - -getSettings() - .then(settings => { - const state = (settings.default === undefined || settings.default === -1) - ? 'no instance selected, commands will require explicit arguments' - : 'instance ' + settings.remotes[settings.default] + ' selected' - - program - .addHelpText('after', '\n\n State: ' + state + '\n\n' + - ' Examples:\n\n' + - ' $ peertube auth add -u "PEERTUBE_URL" -U "PEERTUBE_USER" --password "PEERTUBE_PASSWORD"\n' + - ' $ peertube up \n' - ) - .parse(process.argv) - }) - .catch(err => console.error(err)) diff --git a/server/tools/shared/cli.ts b/server/tools/shared/cli.ts deleted file mode 100644 index e010ab320..000000000 --- a/server/tools/shared/cli.ts +++ /dev/null @@ -1,262 +0,0 @@ -import { Command } from 'commander' -import { Netrc } from 'netrc-parser' -import { join } from 'path' -import { createLogger, format, transports } from 'winston' -import { getAppNumber, isTestInstance } from '@server/helpers/core-utils' -import { loadLanguages } from '@server/initializers/constants' -import { root } from '@shared/core-utils' -import { UserRole, VideoPrivacy } from '@shared/models' -import { PeerTubeServer } from '@shared/server-commands' - -let configName = 'PeerTube/CLI' -if (isTestInstance()) configName += `-${getAppNumber()}` - -const config = require('application-config')(configName) - -const version = require(join(root(), 'package.json')).version - -async function getAdminTokenOrDie (server: PeerTubeServer, username: string, password: string) { - const token = await server.login.getAccessToken(username, password) - const me = await server.users.getMyInfo({ token }) - - if (me.role.id !== UserRole.ADMINISTRATOR) { - console.error('You must be an administrator.') - process.exit(-1) - } - - return token -} - -interface Settings { - remotes: any[] - default: number -} - -async function getSettings (): Promise { - const defaultSettings = { - remotes: [], - default: -1 - } - - const data = await config.read() - - return Object.keys(data).length === 0 - ? defaultSettings - : data -} - -async function getNetrc () { - const Netrc = require('netrc-parser').Netrc - - const netrc = isTestInstance() - ? new Netrc(join(root(), 'test' + getAppNumber(), 'netrc')) - : new Netrc() - - await netrc.load() - - return netrc -} - -function writeSettings (settings: Settings) { - return config.write(settings) -} - -function deleteSettings () { - return config.trash() -} - -function getRemoteObjectOrDie ( - program: Command, - settings: Settings, - netrc: Netrc -): { url: string, username: string, password: string } { - const options = program.opts() - - function exitIfNoOptions (optionNames: string[], errorPrefix: string = '') { - let exit = false - - for (const key of optionNames) { - if (!options[key]) { - if (exit === false && errorPrefix) console.error(errorPrefix) - - console.error(`--${key} field is required`) - exit = true - } - } - - if (exit) process.exit(-1) - } - - // If username or password are specified, both are mandatory - if (options.username || options.password) { - exitIfNoOptions([ 'username', 'password' ]) - } - - // If no available machines, url, username and password args are mandatory - if (Object.keys(netrc.machines).length === 0) { - exitIfNoOptions([ 'url', 'username', 'password' ], 'No account found in netrc') - } - - if (settings.remotes.length === 0 || settings.default === -1) { - exitIfNoOptions([ 'url' ], 'No default instance found') - } - - let url: string = options.url - let username: string = options.username - let password: string = options.password - - if (!url && settings.default !== -1) url = settings.remotes[settings.default] - - const machine = netrc.machines[url] - if ((!username || !password) && !machine) { - console.error('Cannot find existing configuration for %s.', url) - process.exit(-1) - } - - if (!username && machine) username = machine.login - if (!password && machine) password = machine.password - - return { url, username, password } -} - -function buildCommonVideoOptions (command: Command) { - function list (val) { - return val.split(',') - } - - return command - .option('-n, --video-name ', 'Video name') - .option('-c, --category ', 'Category number') - .option('-l, --licence ', 'Licence number') - .option('-L, --language ', 'Language ISO 639 code (fr or en...)') - .option('-t, --tags ', 'Video tags', list) - .option('-N, --nsfw', 'Video is Not Safe For Work') - .option('-d, --video-description ', 'Video description') - .option('-P, --privacy ', 'Privacy') - .option('-C, --channel-name ', 'Channel name') - .option('--no-comments-enabled', 'Disable video comments') - .option('-s, --support ', 'Video support text') - .option('--no-wait-transcoding', 'Do not wait transcoding before publishing the video') - .option('--no-download-enabled', 'Disable video download') - .option('-v, --verbose ', 'Verbosity, from 0/\'error\' to 4/\'debug\'', 'info') -} - -async function buildVideoAttributesFromCommander (server: PeerTubeServer, command: Command, defaultAttributes: any = {}) { - const options = command.opts() - - const defaultBooleanAttributes = { - nsfw: false, - commentsEnabled: true, - downloadEnabled: true, - waitTranscoding: true - } - - const booleanAttributes: { [id in keyof typeof defaultBooleanAttributes]: boolean } | {} = {} - - for (const key of Object.keys(defaultBooleanAttributes)) { - if (options[key] !== undefined) { - booleanAttributes[key] = options[key] - } else if (defaultAttributes[key] !== undefined) { - booleanAttributes[key] = defaultAttributes[key] - } else { - booleanAttributes[key] = defaultBooleanAttributes[key] - } - } - - const videoAttributes = { - name: options.videoName || defaultAttributes.name, - category: options.category || defaultAttributes.category || undefined, - licence: options.licence || defaultAttributes.licence || undefined, - language: options.language || defaultAttributes.language || undefined, - privacy: options.privacy || defaultAttributes.privacy || VideoPrivacy.PUBLIC, - support: options.support || defaultAttributes.support || undefined, - description: options.videoDescription || defaultAttributes.description || undefined, - tags: options.tags || defaultAttributes.tags || undefined - } - - Object.assign(videoAttributes, booleanAttributes) - - if (options.channelName) { - const videoChannel = await server.channels.get({ channelName: options.channelName }) - - Object.assign(videoAttributes, { channelId: videoChannel.id }) - - if (!videoAttributes.support && videoChannel.support) { - Object.assign(videoAttributes, { support: videoChannel.support }) - } - } - - return videoAttributes -} - -function getServerCredentials (program: Command) { - return Promise.all([ getSettings(), getNetrc() ]) - .then(([ settings, netrc ]) => { - return getRemoteObjectOrDie(program, settings, netrc) - }) -} - -function buildServer (url: string) { - loadLanguages() - return new PeerTubeServer({ url }) -} - -async function assignToken (server: PeerTubeServer, username: string, password: string) { - const bodyClient = await server.login.getClient() - const client = { id: bodyClient.client_id, secret: bodyClient.client_secret } - - const body = await server.login.login({ client, user: { username, password } }) - - server.accessToken = body.access_token -} - -function getLogger (logLevel = 'info') { - const logLevels = { - 0: 0, - error: 0, - 1: 1, - warn: 1, - 2: 2, - info: 2, - 3: 3, - verbose: 3, - 4: 4, - debug: 4 - } - - const logger = createLogger({ - levels: logLevels, - format: format.combine( - format.splat(), - format.simple() - ), - transports: [ - new (transports.Console)({ - level: logLevel - }) - ] - }) - - return logger -} - -// --------------------------------------------------------------------------- - -export { - version, - getLogger, - getSettings, - getNetrc, - getRemoteObjectOrDie, - writeSettings, - deleteSettings, - - getServerCredentials, - - buildCommonVideoOptions, - buildVideoAttributesFromCommander, - - getAdminTokenOrDie, - buildServer, - assignToken -} diff --git a/server/tools/shared/index.ts b/server/tools/shared/index.ts deleted file mode 100644 index 8a3f31e2f..000000000 --- a/server/tools/shared/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './cli' diff --git a/server/tools/tsconfig.json b/server/tools/tsconfig.json deleted file mode 100644 index 39f8e74e4..000000000 --- a/server/tools/tsconfig.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - "outDir": "../../dist/server/tools" - }, - "include": [ ".", "../typings" ], - "references": [ - { "path": "../" } - ], - "files": [], - "exclude": [ ] // Overwrite exclude property -} diff --git a/server/tools/yarn.lock b/server/tools/yarn.lock deleted file mode 100644 index 025ef208d..000000000 --- a/server/tools/yarn.lock +++ /dev/null @@ -1,373 +0,0 @@ -# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. -# yarn lockfile v1 - - -"@babel/code-frame@^7.0.0": - version "7.16.7" - resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.16.7.tgz#44416b6bd7624b998f5b1af5d470856c40138789" - integrity sha512-iAXqUn8IIeBTNd72xsFlgaXHkMBMt6y4HJp1tIaK465CWLT/fG1aqB7ykr95gHHmlBdGbFeWWfyB4NJJ0nmeIg== - dependencies: - "@babel/highlight" "^7.16.7" - -"@babel/helper-validator-identifier@^7.16.7": - version "7.16.7" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.16.7.tgz#e8c602438c4a8195751243da9031d1607d247cad" - integrity sha512-hsEnFemeiW4D08A5gUAZxLBTXpZ39P+a+DGDsHw1yxqyQ/jzFEnxf5uTEGp+3bzAbNOxU1paTgYS4ECU/IgfDw== - -"@babel/highlight@^7.16.7": - version "7.16.10" - resolved "https://registry.yarnpkg.com/@babel/highlight/-/highlight-7.16.10.tgz#744f2eb81579d6eea753c227b0f570ad785aba88" - integrity sha512-5FnTQLSLswEj6IkgVw5KusNUUFY9ZGqe/TRFnP/BKYHYgfh7tc+C7mwiy95/yNP7Dh9x580Vv8r7u7ZfTBFxdw== - dependencies: - "@babel/helper-validator-identifier" "^7.16.7" - chalk "^2.0.0" - js-tokens "^4.0.0" - -ansi-regex@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" - integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== - -ansi-styles@^3.2.1: - version "3.2.1" - resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" - integrity sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA== - dependencies: - color-convert "^1.9.0" - -application-config-path@^0.1.0: - version "0.1.0" - resolved "https://registry.yarnpkg.com/application-config-path/-/application-config-path-0.1.0.tgz#193c5f0a86541a4c66fba1e2dc38583362ea5e8f" - integrity sha1-GTxfCoZUGkxm+6Hi3DhYM2LqXo8= - -application-config@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/application-config/-/application-config-2.0.0.tgz#15b4d54d61c0c082f9802227e3e85de876b47747" - integrity sha512-NC5/0guSZK3/UgUDfCk/riByXzqz0owL1L3r63JPSBzYk5QALrp3bLxbsR7qeSfvYfFmAhnp3dbqYsW3U9MpZQ== - dependencies: - application-config-path "^0.1.0" - load-json-file "^6.2.0" - write-json-file "^4.2.0" - -chalk@^2.0.0: - version "2.4.2" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" - integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== - dependencies: - ansi-styles "^3.2.1" - escape-string-regexp "^1.0.5" - supports-color "^5.3.0" - -cli-table3@^0.6.0: - version "0.6.1" - resolved "https://registry.yarnpkg.com/cli-table3/-/cli-table3-0.6.1.tgz#36ce9b7af4847f288d3cdd081fbd09bf7bd237b8" - integrity sha512-w0q/enDHhPLq44ovMGdQeeDLvwxwavsJX7oQGYt/LrBlYsyaxyDnp6z3QzFut/6kLLKnlcUVJLrpB7KBfgG/RA== - dependencies: - string-width "^4.2.0" - optionalDependencies: - colors "1.4.0" - -color-convert@^1.9.0: - version "1.9.3" - resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" - integrity sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg== - dependencies: - color-name "1.1.3" - -color-name@1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" - integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU= - -colors@1.4.0: - version "1.4.0" - resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78" - integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA== - -cross-spawn@^6.0.0: - version "6.0.5" - resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4" - integrity sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ== - dependencies: - nice-try "^1.0.4" - path-key "^2.0.1" - semver "^5.5.0" - shebang-command "^1.2.0" - which "^1.2.9" - -debug@^3.1.0: - version "3.2.7" - resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" - integrity sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ== - dependencies: - ms "^2.1.1" - -detect-indent@^6.0.0: - version "6.1.0" - resolved "https://registry.yarnpkg.com/detect-indent/-/detect-indent-6.1.0.tgz#592485ebbbf6b3b1ab2be175c8393d04ca0d57e6" - integrity sha512-reYkTUJAZb9gUuZ2RvVCNhVHdg62RHnJ7WJl8ftMi4diZ6NWlciOzQN88pUhSELEwflJht4oQDv0F0BMlwaYtA== - -emoji-regex@^8.0.0: - version "8.0.0" - resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" - integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== - -error-ex@^1.3.1: - version "1.3.2" - resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.2.tgz#b4ac40648107fdcdcfae242f428bea8a14d4f1bf" - integrity sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g== - dependencies: - is-arrayish "^0.2.1" - -escape-string-regexp@^1.0.5: - version "1.0.5" - resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz#1b61c0562190a8dff6ae3bb2cf0200ca130b86d4" - integrity sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ= - -execa@^0.10.0: - version "0.10.0" - resolved "https://registry.yarnpkg.com/execa/-/execa-0.10.0.tgz#ff456a8f53f90f8eccc71a96d11bdfc7f082cb50" - integrity sha512-7XOMnz8Ynx1gGo/3hyV9loYNPWM94jG3+3T3Y8tsfSstFmETmENCMU/A/zj8Lyaj1lkgEepKepvd6240tBRvlw== - dependencies: - cross-spawn "^6.0.0" - get-stream "^3.0.0" - is-stream "^1.1.0" - npm-run-path "^2.0.0" - p-finally "^1.0.0" - signal-exit "^3.0.0" - strip-eof "^1.0.0" - -get-stream@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-3.0.0.tgz#8e943d1358dc37555054ecbe2edb05aa174ede14" - integrity sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ= - -graceful-fs@^4.1.15: - version "4.2.9" - resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.9.tgz#041b05df45755e587a24942279b9d113146e1c96" - integrity sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ== - -has-flag@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" - integrity sha1-tdRU3CGZriJWmfNGfloH87lVuv0= - -imurmurhash@^0.1.4: - version "0.1.4" - resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" - integrity sha1-khi5srkoojixPcT7a21XbyMUU+o= - -is-arrayish@^0.2.1: - version "0.2.1" - resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" - integrity sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0= - -is-fullwidth-code-point@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d" - integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg== - -is-plain-obj@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/is-plain-obj/-/is-plain-obj-2.1.0.tgz#45e42e37fccf1f40da8e5f76ee21515840c09287" - integrity sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA== - -is-stream@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44" - integrity sha1-EtSj3U5o4Lec6428hBc66A2RykQ= - -is-typedarray@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" - integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo= - -isexe@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" - integrity sha1-6PvzdNxVb/iUehDcsFctYz8s+hA= - -js-tokens@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-4.0.0.tgz#19203fb59991df98e3a287050d4647cdeaf32499" - integrity sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ== - -json-parse-even-better-errors@^2.3.0: - version "2.3.1" - resolved "https://registry.yarnpkg.com/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz#7c47805a94319928e05777405dc12e1f7a4ee02d" - integrity sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w== - -lines-and-columns@^1.1.6: - version "1.2.4" - resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632" - integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg== - -load-json-file@^6.2.0: - version "6.2.0" - resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-6.2.0.tgz#5c7770b42cafa97074ca2848707c61662f4251a1" - integrity sha512-gUD/epcRms75Cw8RT1pUdHugZYM5ce64ucs2GEISABwkRsOQr0q2wm/MV2TKThycIe5e0ytRweW2RZxclogCdQ== - dependencies: - graceful-fs "^4.1.15" - parse-json "^5.0.0" - strip-bom "^4.0.0" - type-fest "^0.6.0" - -make-dir@^3.0.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/make-dir/-/make-dir-3.1.0.tgz#415e967046b3a7f1d185277d84aa58203726a13f" - integrity sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw== - dependencies: - semver "^6.0.0" - -ms@^2.1.1: - version "2.1.3" - resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" - integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== - -netrc-parser@^3.1.6: - version "3.1.6" - resolved "https://registry.yarnpkg.com/netrc-parser/-/netrc-parser-3.1.6.tgz#7243c9ec850b8e805b9bdc7eae7b1450d4a96e72" - integrity sha512-lY+fmkqSwntAAjfP63jB4z5p5WbuZwyMCD3pInT7dpHU/Gc6Vv90SAC6A0aNiqaRGHiuZFBtiwu+pu8W/Eyotw== - dependencies: - debug "^3.1.0" - execa "^0.10.0" - -nice-try@^1.0.4: - version "1.0.5" - resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" - integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== - -npm-run-path@^2.0.0: - version "2.0.2" - resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-2.0.2.tgz#35a9232dfa35d7067b4cb2ddf2357b1871536c5f" - integrity sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8= - dependencies: - path-key "^2.0.0" - -p-finally@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae" - integrity sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4= - -parse-json@^5.0.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-5.2.0.tgz#c76fc66dee54231c962b22bcc8a72cf2f99753cd" - integrity sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg== - dependencies: - "@babel/code-frame" "^7.0.0" - error-ex "^1.3.1" - json-parse-even-better-errors "^2.3.0" - lines-and-columns "^1.1.6" - -path-key@^2.0.0, path-key@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40" - integrity sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A= - -semver@^5.5.0: - version "5.7.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" - integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ== - -semver@^6.0.0: - version "6.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" - integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== - -shebang-command@^1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-1.2.0.tgz#44aac65b695b03398968c39f363fee5deafdf1ea" - integrity sha1-RKrGW2lbAzmJaMOfNj/uXer98eo= - dependencies: - shebang-regex "^1.0.0" - -shebang-regex@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-1.0.0.tgz#da42f49740c0b42db2ca9728571cb190c98efea3" - integrity sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM= - -signal-exit@^3.0.0, signal-exit@^3.0.2: - version "3.0.7" - resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9" - integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ== - -sort-keys@^4.0.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/sort-keys/-/sort-keys-4.2.0.tgz#6b7638cee42c506fff8c1cecde7376d21315be18" - integrity sha512-aUYIEU/UviqPgc8mHR6IW1EGxkAXpeRETYcrzg8cLAvUPZcpAlleSXHV2mY7G12GphSH6Gzv+4MMVSSkbdteHg== - dependencies: - is-plain-obj "^2.0.0" - -string-width@^4.2.0: - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - -strip-bom@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-4.0.0.tgz#9c3505c1db45bcedca3d9cf7a16f5c5aa3901878" - integrity sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w== - -strip-eof@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/strip-eof/-/strip-eof-1.0.0.tgz#bb43ff5598a6eb05d89b59fcd129c983313606bf" - integrity sha1-u0P/VZim6wXYm1n80SnJgzE2Br8= - -supports-color@^5.3.0: - version "5.5.0" - resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f" - integrity sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow== - dependencies: - has-flag "^3.0.0" - -type-fest@^0.6.0: - version "0.6.0" - resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.6.0.tgz#8d2a2370d3df886eb5c90ada1c5bf6188acf838b" - integrity sha512-q+MB8nYR1KDLrgr4G5yemftpMC7/QLqVndBmEEdqzmNj5dcFOO4Oo8qlwZE3ULT3+Zim1F8Kq4cBnikNhlCMlg== - -typedarray-to-buffer@^3.1.5: - version "3.1.5" - resolved "https://registry.yarnpkg.com/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz#a97ee7a9ff42691b9f783ff1bc5112fe3fca9080" - integrity sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q== - dependencies: - is-typedarray "^1.0.0" - -which@^1.2.9: - version "1.3.1" - resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a" - integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ== - dependencies: - isexe "^2.0.0" - -write-file-atomic@^3.0.0: - version "3.0.3" - resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-3.0.3.tgz#56bd5c5a5c70481cd19c571bd39ab965a5de56e8" - integrity sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q== - dependencies: - imurmurhash "^0.1.4" - is-typedarray "^1.0.0" - signal-exit "^3.0.2" - typedarray-to-buffer "^3.1.5" - -write-json-file@^4.2.0: - version "4.3.0" - resolved "https://registry.yarnpkg.com/write-json-file/-/write-json-file-4.3.0.tgz#908493d6fd23225344af324016e4ca8f702dd12d" - integrity sha512-PxiShnxf0IlnQuMYOPPhPkhExoCQuTUNPOa/2JWCYTmBquU9njyyDuwRKN26IZBlp4yn1nt+Agh2HOOBl+55HQ== - dependencies: - detect-indent "^6.0.0" - graceful-fs "^4.1.15" - is-plain-obj "^2.0.0" - make-dir "^3.0.0" - sort-keys "^4.0.0" - write-file-atomic "^3.0.0" diff --git a/server/tsconfig.json b/server/tsconfig.json index 240bd3bfe..2b799749f 100644 --- a/server/tsconfig.json +++ b/server/tsconfig.json @@ -1,13 +1,22 @@ { "extends": "../tsconfig.base.json", "compilerOptions": { - "outDir": "../dist/server" + "outDir": "../dist", + "baseUrl": "../", + "rootDir": ".", + "tsBuildInfoFile": "../dist/.tsbuildinfo", + "paths": { + "@server/*": [ "server/server/*" ] + } }, "references": [ - { "path": "../shared" } + { "path": "../packages/core-utils" }, + { "path": "../packages/ffmpeg" }, + { "path": "../packages/models" }, + { "path": "../packages/node-utils" }, + { "path": "../packages/typescript-utils" } ], - "exclude": [ - "tools/", - "tests/fixtures" + "include": [ + "./**/*.ts" ] } diff --git a/server/tsconfig.lib.json b/server/tsconfig.lib.json new file mode 100644 index 000000000..dfc83c0ec --- /dev/null +++ b/server/tsconfig.lib.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./dist", + "tsBuildInfoFile": "./dist/.tsbuildinfo", + "baseUrl": "../", + "rootDir": ".", + "paths": { + "@server/*": [ "server/server/*" ] + } + } +} diff --git a/server/tsconfig.types.json b/server/tsconfig.types.json index da6b572ea..696fe7059 100644 --- a/server/tsconfig.types.json +++ b/server/tsconfig.types.json @@ -1,16 +1,21 @@ { "extends": "./tsconfig.json", "compilerOptions": { - "outDir": "../packages/types/dist/server", + "outDir": "../packages/types-generator/dist/server", + "tsBuildInfoFile": "../packages/types-generator/dist/server/.tsbuildinfo", "stripInternal": true, "removeComments": false, "emitDeclarationOnly": true }, "references": [ - { "path": "../shared/tsconfig.types.json" } + { "path": "../packages/core-utils" }, + { "path": "../packages/ffmpeg" }, + { "path": "../packages/models" }, + { "path": "../packages/node-utils" }, + { "path": "../packages/server-commands" }, + { "path": "../packages/typescript-utils" } ], "exclude": [ - "tools/", "tests/" ] } diff --git a/server/types/activitypub-processor.model.ts b/server/types/activitypub-processor.model.ts deleted file mode 100644 index 7ed3a65b1..000000000 --- a/server/types/activitypub-processor.model.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { Activity } from '../../shared/models/activitypub' -import { MActorDefault, MActorSignature } from './models' - -export type APProcessorOptions = { - activity: T - byActor: MActorSignature - inboxActor?: MActorDefault - fromFetch?: boolean -} diff --git a/server/types/express.d.ts b/server/types/express.d.ts deleted file mode 100644 index 4729c4534..000000000 --- a/server/types/express.d.ts +++ /dev/null @@ -1,222 +0,0 @@ -import { OutgoingHttpHeaders } from 'http' -import { Writable } from 'stream' -import { RegisterServerAuthExternalOptions } from '@server/types' -import { - MAbuseMessage, - MAbuseReporter, - MAccountBlocklist, - MActorFollowActorsDefault, - MActorUrl, - MChannelBannerAccountDefault, - MChannelSyncChannel, - MRegistration, - MStreamingPlaylist, - MUserAccountUrl, - MVideoChangeOwnershipFull, - MVideoFile, - MVideoFormattableDetails, - MVideoId, - MVideoImmutable, - MVideoLiveFormattable, - MVideoPassword, - MVideoPlaylistFull, - MVideoPlaylistFullSummary -} from '@server/types/models' -import { MOAuthTokenUser } from '@server/types/models/oauth/oauth-token' -import { MPlugin, MServer, MServerBlocklist } from '@server/types/models/server' -import { MVideoImportDefault } from '@server/types/models/video/video-import' -import { MVideoPlaylistElement, MVideoPlaylistElementVideoUrlPlaylistPrivacy } from '@server/types/models/video/video-playlist-element' -import { MAccountVideoRateAccountVideo } from '@server/types/models/video/video-rate' -import { HttpMethod, PeerTubeProblemDocumentData, ServerErrorCode, VideoCreate } from '@shared/models' -import { File as UploadXFile, Metadata } from '@uploadx/core' -import { RegisteredPlugin } from '../../lib/plugins/plugin-manager' -import { - MAccountDefault, - MActorAccountChannelId, - MActorFollowActorsDefaultSubscription, - MActorFull, - MComment, - MCommentOwnerVideoReply, - MUserDefault, - MVideoBlacklist, - MVideoCaptionVideo, - MVideoFullLight, - MVideoRedundancyVideo, - MVideoShareActor, - MVideoThumbnail -} from './models' -import { MRunner, MRunnerJobRunner, MRunnerRegistrationToken } from './models/runners' -import { MVideoSource } from './models/video/video-source' - -declare module 'express' { - export interface Request { - query: any - method: HttpMethod - } - - // Upload using multer or uploadx middleware - export type MulterOrUploadXFile = UploadXFile | Express.Multer.File - - export type UploadFiles = { - [fieldname: string]: MulterOrUploadXFile[] - } | MulterOrUploadXFile[] - - // Partial object used by some functions to check the file mimetype/extension - export type UploadFileForCheck = { - originalname: string - mimetype: string - size: number - } - - export type UploadFilesForCheck = { - [fieldname: string]: UploadFileForCheck[] - } | UploadFileForCheck[] - - // Upload file with a duration added by our middleware - export type VideoUploadFile = Pick & { - duration: number - } - - // Extends Metadata property of UploadX object - export type UploadXFileMetadata = Metadata & VideoCreate & { - previewfile: Express.Multer.File[] - thumbnailfile: Express.Multer.File[] - } - - // Our custom UploadXFile object using our custom metadata - export type CustomUploadXFile = UploadXFile & { metadata: T } - - export type EnhancedUploadXFile = CustomUploadXFile & { - duration: number - path: string - filename: string - originalname: string - } - - export type UploadNewVideoUploadXFile = EnhancedUploadXFile & CustomUploadXFile - - // Extends Response with added functions and potential variables passed by middlewares - interface Response { - fail: (options: { - message: string - - title?: string - status?: number - type?: ServerErrorCode | string - instance?: string - - data?: PeerTubeProblemDocumentData - - tags?: string[] - }) => void - - locals: { - requestStart: number - - apicacheGroups: string[] - - apicache: { - content: string | Buffer - write: Writable['write'] - writeHead: Response['writeHead'] - end: Response['end'] - cacheable: boolean - headers: OutgoingHttpHeaders - } - - docUrl?: string - - videoAPI?: MVideoFormattableDetails - videoAll?: MVideoFullLight - onlyImmutableVideo?: MVideoImmutable - onlyVideo?: MVideoThumbnail - videoId?: MVideoId - - videoLive?: MVideoLiveFormattable - videoLiveSession?: MVideoLiveSession - - videoShare?: MVideoShareActor - - videoSource?: MVideoSource - - videoFile?: MVideoFile - - uploadVideoFileResumable?: UploadNewVideoUploadXFile - updateVideoFileResumable?: EnhancedUploadXFile - - videoImport?: MVideoImportDefault - - videoBlacklist?: MVideoBlacklist - - videoCaption?: MVideoCaptionVideo - - abuse?: MAbuseReporter - abuseMessage?: MAbuseMessage - - videoStreamingPlaylist?: MStreamingPlaylist - - videoChannel?: MChannelBannerAccountDefault - videoChannelSync?: MChannelSyncChannel - - videoPlaylistFull?: MVideoPlaylistFull - videoPlaylistSummary?: MVideoPlaylistFullSummary - - videoPlaylistElement?: MVideoPlaylistElement - videoPlaylistElementAP?: MVideoPlaylistElementVideoUrlPlaylistPrivacy - - accountVideoRate?: MAccountVideoRateAccountVideo - - videoCommentFull?: MCommentOwnerVideoReply - videoCommentThread?: MComment - - videoPassword?: MVideoPassword - - follow?: MActorFollowActorsDefault - subscription?: MActorFollowActorsDefaultSubscription - - nextOwner?: MAccountDefault - videoChangeOwnership?: MVideoChangeOwnershipFull - - account?: MAccountDefault - - actorUrl?: MActorUrl - actorFull?: MActorFull - - user?: MUserDefault - userRegistration?: MRegistration - - server?: MServer - - videoRedundancy?: MVideoRedundancyVideo - - accountBlock?: MAccountBlocklist - serverBlock?: MServerBlocklist - - oauth?: { - token: MOAuthTokenUser - } - - signature?: { - actor: MActorAccountChannelId - } - - videoFileToken?: { - user: MUserAccountUrl - } - - authenticated?: boolean - - registeredPlugin?: RegisteredPlugin - - externalAuth?: RegisterServerAuthExternalOptions - - plugin?: MPlugin - - localViewerFull?: MLocalVideoViewerWithWatchSections - - runner?: MRunner - runnerRegistrationToken?: MRunnerRegistrationToken - runnerJob?: MRunnerJobRunner - } - } -} diff --git a/server/types/index.ts b/server/types/index.ts deleted file mode 100644 index 18d3827a5..000000000 --- a/server/types/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './plugins' -export * from './activitypub-processor.model' -export * from './sequelize' diff --git a/server/types/models/abuse/abuse-message.ts b/server/types/models/abuse/abuse-message.ts deleted file mode 100644 index 2d7d09316..000000000 --- a/server/types/models/abuse/abuse-message.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { AbuseMessageModel } from '@server/models/abuse/abuse-message' -import { PickWith } from '@shared/typescript-utils' -import { AbuseModel } from '../../../models/abuse/abuse' -import { MAccountFormattable } from '../account' - -type Use = PickWith - -// ############################################################################ - -export type MAbuseMessage = Omit - -export type MAbuseMessageId = Pick - -// ############################################################################ - -// Format for API - -export type MAbuseMessageFormattable = - MAbuseMessage & - Use<'Account', MAccountFormattable> diff --git a/server/types/models/abuse/abuse.ts b/server/types/models/abuse/abuse.ts deleted file mode 100644 index 1b45b3879..000000000 --- a/server/types/models/abuse/abuse.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { VideoAbuseModel } from '@server/models/abuse/video-abuse' -import { VideoCommentAbuseModel } from '@server/models/abuse/video-comment-abuse' -import { VideoCommentModel } from '@server/models/video/video-comment' -import { PickWith } from '@shared/typescript-utils' -import { AbuseModel } from '../../../models/abuse/abuse' -import { MAccountDefault, MAccountFormattable, MAccountLight, MAccountUrl } from '../account' -import { MComment, MCommentOwner, MCommentUrl, MCommentVideo, MVideoUrl } from '../video' -import { MVideo, MVideoAccountLightBlacklistAllFiles } from '../video/video' - -type Use = PickWith -type UseVideoAbuse = PickWith -type UseCommentAbuse = PickWith - -// ############################################################################ - -export type MAbuse = Omit - -export type MVideoAbuse = Omit - -export type MCommentAbuse = Omit - -export type MAbuseReporter = - MAbuse & - Use<'ReporterAccount', MAccountDefault> - -// ############################################################################ - -export type MVideoAbuseVideo = - MVideoAbuse & - UseVideoAbuse<'Video', MVideo> - -export type MVideoAbuseVideoUrl = - MVideoAbuse & - UseVideoAbuse<'Video', MVideoUrl> - -export type MVideoAbuseVideoFull = - MVideoAbuse & - UseVideoAbuse<'Video', Omit> - -export type MVideoAbuseFormattable = - MVideoAbuse & - UseVideoAbuse<'Video', Pick> - -// ############################################################################ - -export type MCommentAbuseAccount = - MCommentAbuse & - UseCommentAbuse<'VideoComment', MCommentOwner> - -export type MCommentAbuseAccountVideo = - MCommentAbuse & - UseCommentAbuse<'VideoComment', MCommentOwner & PickWith> - -export type MCommentAbuseUrl = - MCommentAbuse & - UseCommentAbuse<'VideoComment', MCommentUrl> - -export type MCommentAbuseFormattable = - MCommentAbuse & - UseCommentAbuse<'VideoComment', MComment & PickWith>> - -// ############################################################################ - -export type MAbuseId = Pick - -export type MAbuseVideo = - MAbuse & - Pick & - Use<'VideoAbuse', MVideoAbuseVideo> - -export type MAbuseUrl = - MAbuse & - Use<'VideoAbuse', MVideoAbuseVideoUrl> & - Use<'VideoCommentAbuse', MCommentAbuseUrl> - -export type MAbuseAccountVideo = - MAbuse & - Pick & - Use<'VideoAbuse', MVideoAbuseVideoFull> & - Use<'ReporterAccount', MAccountDefault> - -export type MAbuseFull = - MAbuse & - Pick & - Use<'ReporterAccount', MAccountLight> & - Use<'FlaggedAccount', MAccountLight> & - Use<'VideoAbuse', MVideoAbuseVideoFull> & - Use<'VideoCommentAbuse', MCommentAbuseAccountVideo> - -// ############################################################################ - -// Format for API or AP object - -export type MAbuseAdminFormattable = - MAbuse & - Use<'ReporterAccount', MAccountFormattable> & - Use<'FlaggedAccount', MAccountFormattable> & - Use<'VideoAbuse', MVideoAbuseFormattable> & - Use<'VideoCommentAbuse', MCommentAbuseFormattable> - -export type MAbuseUserFormattable = - MAbuse & - Use<'FlaggedAccount', MAccountFormattable> & - Use<'VideoAbuse', MVideoAbuseFormattable> & - Use<'VideoCommentAbuse', MCommentAbuseFormattable> - -export type MAbuseAP = - MAbuse & - Pick & - Use<'ReporterAccount', MAccountUrl> & - Use<'FlaggedAccount', MAccountUrl> & - Use<'VideoAbuse', MVideoAbuseVideo> & - Use<'VideoCommentAbuse', MCommentAbuseAccount> diff --git a/server/types/models/abuse/index.ts b/server/types/models/abuse/index.ts deleted file mode 100644 index 1ed91b249..000000000 --- a/server/types/models/abuse/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './abuse' -export * from './abuse-message' diff --git a/server/types/models/account/account-blocklist.ts b/server/types/models/account/account-blocklist.ts deleted file mode 100644 index 9dae10915..000000000 --- a/server/types/models/account/account-blocklist.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { AccountBlocklistModel } from '../../../models/account/account-blocklist' -import { PickWith } from '@shared/typescript-utils' -import { MAccountDefault, MAccountFormattable } from './account' - -type Use = PickWith - -// ############################################################################ - -export type MAccountBlocklist = Omit - -// ############################################################################ - -export type MAccountBlocklistId = Pick - -export type MAccountBlocklistAccounts = - MAccountBlocklist & - Use<'ByAccount', MAccountDefault> & - Use<'BlockedAccount', MAccountDefault> - -// ############################################################################ - -// Format for API or AP object - -export type MAccountBlocklistFormattable = - Pick & - Use<'ByAccount', MAccountFormattable> & - Use<'BlockedAccount', MAccountFormattable> diff --git a/server/types/models/account/account.ts b/server/types/models/account/account.ts deleted file mode 100644 index d10b904ab..000000000 --- a/server/types/models/account/account.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { FunctionProperties, PickWith } from '@shared/typescript-utils' -import { AccountModel } from '../../../models/account/account' -import { - MActor, - MActorAPAccount, - MActorAPI, - MActorAudience, - MActorDefault, - MActorDefaultLight, - MActorFormattable, - MActorHost, - MActorId, - MActorSummary, - MActorSummaryFormattable, - MActorUrl -} from '../actor' -import { MChannelDefault } from '../video/video-channels' -import { MAccountBlocklistId } from './account-blocklist' - -type Use = PickWith - -// ############################################################################ - -export type MAccount = - Omit - -// ############################################################################ - -// Only some attributes -export type MAccountId = Pick -export type MAccountUserId = Pick - -// Only some Actor attributes -export type MAccountUrl = Use<'Actor', MActorUrl> -export type MAccountAudience = Use<'Actor', MActorAudience> - -export type MAccountIdActor = - MAccountId & - Use<'Actor', MActor> - -export type MAccountIdActorId = - MAccountId & - Use<'Actor', MActorId> - -// ############################################################################ - -// Default scope -export type MAccountDefault = - MAccount & - Use<'Actor', MActorDefault> - -// Default with default association scopes -export type MAccountDefaultChannelDefault = - MAccount & - Use<'Actor', MActorDefault> & - Use<'VideoChannels', MChannelDefault[]> - -// We don't need some actors attributes -export type MAccountLight = - MAccount & - Use<'Actor', MActorDefaultLight> - -// ############################################################################ - -// Full actor -export type MAccountActor = - MAccount & - Use<'Actor', MActor> - -export type MAccountHost = - MAccount & - Use<'Actor', MActorHost> - -// ############################################################################ - -// For API - -export type MAccountSummary = - FunctionProperties & - Pick & - Use<'Actor', MActorSummary> - -export type MAccountSummaryBlocks = - MAccountSummary & - Use<'BlockedBy', MAccountBlocklistId[]> - -export type MAccountAPI = - MAccount & - Use<'Actor', MActorAPI> - -// ############################################################################ - -// Format for API or AP object - -export type MAccountSummaryFormattable = - FunctionProperties & - Pick & - Use<'Actor', MActorSummaryFormattable> - -export type MAccountFormattable = - FunctionProperties & - Pick & - Use<'Actor', MActorFormattable> - -export type MAccountAP = - Pick & - Use<'Actor', MActorAPAccount> diff --git a/server/types/models/account/actor-custom-page.ts b/server/types/models/account/actor-custom-page.ts deleted file mode 100644 index fcd8069be..000000000 --- a/server/types/models/account/actor-custom-page.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { ActorCustomPageModel } from '../../../models/account/actor-custom-page' - -export type MActorCustomPage = Omit diff --git a/server/types/models/account/index.ts b/server/types/models/account/index.ts deleted file mode 100644 index 9679c01e4..000000000 --- a/server/types/models/account/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './account' -export * from './actor-custom-page' -export * from './account-blocklist' diff --git a/server/types/models/actor/actor-follow.ts b/server/types/models/actor/actor-follow.ts deleted file mode 100644 index 84042e228..000000000 --- a/server/types/models/actor/actor-follow.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { PickWith } from '@shared/typescript-utils' -import { ActorFollowModel } from '../../../models/actor/actor-follow' -import { - MActor, - MActorChannelAccountActor, - MActorDefault, - MActorDefaultAccountChannel, - MActorDefaultChannelId, - MActorFormattable, - MActorHostOnly, - MActorUsername -} from './actor' - -type Use = PickWith - -// ############################################################################ - -export type MActorFollow = Omit - -// ############################################################################ - -export type MActorFollowFollowingHost = - MActorFollow & - Use<'ActorFollowing', MActorUsername & MActorHostOnly> - -// ############################################################################ - -// With actors or actors default - -export type MActorFollowActors = - MActorFollow & - Use<'ActorFollower', MActor> & - Use<'ActorFollowing', MActor> - -export type MActorFollowActorsDefault = - MActorFollow & - Use<'ActorFollower', MActorDefault> & - Use<'ActorFollowing', MActorDefault> - -export type MActorFollowFull = - MActorFollow & - Use<'ActorFollower', MActorDefaultAccountChannel> & - Use<'ActorFollowing', MActorDefaultAccountChannel> - -// ############################################################################ - -// For subscriptions - -export type MActorFollowActorsDefaultSubscription = - MActorFollow & - Use<'ActorFollower', MActorDefault> & - Use<'ActorFollowing', MActorDefaultChannelId> - -export type MActorFollowSubscriptions = - MActorFollow & - Use<'ActorFollowing', MActorChannelAccountActor> - -// ############################################################################ - -// Format for API or AP object - -export type MActorFollowFormattable = - Pick & - Use<'ActorFollower', MActorFormattable> & - Use<'ActorFollowing', MActorFormattable> diff --git a/server/types/models/actor/actor-image.ts b/server/types/models/actor/actor-image.ts deleted file mode 100644 index e8f32b71e..000000000 --- a/server/types/models/actor/actor-image.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { FunctionProperties } from '@shared/typescript-utils' -import { ActorImageModel } from '../../../models/actor/actor-image' - -export type MActorImage = ActorImageModel - -// ############################################################################ - -// Format for API or AP object - -export type MActorImageFormattable = - FunctionProperties & - Pick diff --git a/server/types/models/actor/actor.ts b/server/types/models/actor/actor.ts deleted file mode 100644 index 47e7b7091..000000000 --- a/server/types/models/actor/actor.ts +++ /dev/null @@ -1,170 +0,0 @@ -import { FunctionProperties, PickWith, PickWithOpt } from '@shared/typescript-utils' -import { ActorModel } from '../../../models/actor/actor' -import { MAccount, MAccountDefault, MAccountId, MAccountIdActor } from '../account' -import { MServer, MServerHost, MServerHostBlocks, MServerRedundancyAllowed } from '../server' -import { MChannel, MChannelAccountActor, MChannelAccountDefault, MChannelId, MChannelIdActor } from '../video' -import { MActorImage, MActorImageFormattable } from './actor-image' - -type Use = PickWith -type UseOpt = PickWithOpt - -// ############################################################################ - -export type MActor = Omit - -// ############################################################################ - -export type MActorUrl = Pick -export type MActorId = Pick -export type MActorUsername = Pick - -export type MActorFollowersUrl = Pick -export type MActorAudience = MActorUrl & MActorFollowersUrl -export type MActorWithInboxes = Pick -export type MActorSignature = MActorAccountChannelId - -export type MActorLight = Omit - -// ############################################################################ - -// Some association attributes - -export type MActorHostOnly = Use<'Server', MServerHost> -export type MActorHost = - MActorLight & - Use<'Server', MServerHost> - -export type MActorRedundancyAllowedOpt = PickWithOpt - -export type MActorDefaultLight = - MActorLight & - Use<'Server', MServerHost> & - Use<'Avatars', MActorImage[]> - -export type MActorAccountId = - MActor & - Use<'Account', MAccountId> -export type MActorAccountIdActor = - MActor & - Use<'Account', MAccountIdActor> - -export type MActorChannelId = - MActor & - Use<'VideoChannel', MChannelId> -export type MActorChannelIdActor = - MActor & - Use<'VideoChannel', MChannelIdActor> - -export type MActorAccountChannelId = MActorAccountId & MActorChannelId -export type MActorAccountChannelIdActor = MActorAccountIdActor & MActorChannelIdActor - -// ############################################################################ - -// Include raw account/channel/server - -export type MActorAccount = - MActor & - Use<'Account', MAccount> - -export type MActorChannel = - MActor & - Use<'VideoChannel', MChannel> - -export type MActorDefaultAccountChannel = MActorDefault & MActorAccount & MActorChannel - -export type MActorServerLight = - MActorLight & - Use<'Server', MServer> - -// ############################################################################ - -// Complex actor associations - -export type MActorImages = - MActor & - Use<'Avatars', MActorImage[]> & - UseOpt<'Banners', MActorImage[]> - -export type MActorDefault = - MActor & - Use<'Server', MServer> & - Use<'Avatars', MActorImage[]> - -export type MActorDefaultChannelId = - MActorDefault & - Use<'VideoChannel', MChannelId> - -export type MActorDefaultBanner = - MActor & - Use<'Server', MServer> & - Use<'Avatars', MActorImage[]> & - Use<'Banners', MActorImage[]> - -// Actor with channel that is associated to an account and its actor -// Actor -> VideoChannel -> Account -> Actor -export type MActorChannelAccountActor = - MActor & - Use<'VideoChannel', MChannelAccountActor> - -export type MActorFull = - MActor & - Use<'Server', MServer> & - Use<'Avatars', MActorImage[]> & - Use<'Banners', MActorImage[]> & - Use<'Account', MAccount> & - Use<'VideoChannel', MChannelAccountActor> - -// Same than ActorFull, but the account and the channel have their actor -export type MActorFullActor = - MActor & - Use<'Server', MServer> & - Use<'Avatars', MActorImage[]> & - Use<'Banners', MActorImage[]> & - Use<'Account', MAccountDefault> & - Use<'VideoChannel', MChannelAccountDefault> - -// ############################################################################ - -// API - -export type MActorSummary = - FunctionProperties & - Pick & - Use<'Server', MServerHost> & - Use<'Avatars', MActorImage[]> - -export type MActorSummaryBlocks = - MActorSummary & - Use<'Server', MServerHostBlocks> - -export type MActorAPI = - Omit - -// ############################################################################ - -// Format for API or AP object - -export type MActorSummaryFormattable = - FunctionProperties & - Pick & - Use<'Server', MServerHost> & - Use<'Avatars', MActorImageFormattable[]> - -export type MActorFormattable = - MActorSummaryFormattable & - Pick & - Use<'Server', MServerHost & Partial>> & - UseOpt<'Banners', MActorImageFormattable[]> & - UseOpt<'Avatars', MActorImageFormattable[]> - -type MActorAPBase = - MActor & - Use<'Avatars', MActorImage[]> - -export type MActorAPAccount = - MActorAPBase - -export type MActorAPChannel = - MActorAPBase & - Use<'Banners', MActorImage[]> diff --git a/server/types/models/actor/index.ts b/server/types/models/actor/index.ts deleted file mode 100644 index b27815255..000000000 --- a/server/types/models/actor/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './actor-follow' -export * from './actor-image' -export * from './actor' diff --git a/server/types/models/application/application.ts b/server/types/models/application/application.ts deleted file mode 100644 index 9afb9ad70..000000000 --- a/server/types/models/application/application.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { ApplicationModel } from '@server/models/application/application' - -// ############################################################################ - -export type MApplication = Omit diff --git a/server/types/models/application/index.ts b/server/types/models/application/index.ts deleted file mode 100644 index 26e4b031f..000000000 --- a/server/types/models/application/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './application' diff --git a/server/types/models/index.ts b/server/types/models/index.ts deleted file mode 100644 index 704cb9844..000000000 --- a/server/types/models/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -export * from './abuse' -export * from './account' -export * from './actor' -export * from './application' -export * from './oauth' -export * from './server' -export * from './user' -export * from './video' diff --git a/server/types/models/oauth/index.ts b/server/types/models/oauth/index.ts deleted file mode 100644 index 36b7ea8ca..000000000 --- a/server/types/models/oauth/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './oauth-client' -export * from './oauth-token' diff --git a/server/types/models/oauth/oauth-client.ts b/server/types/models/oauth/oauth-client.ts deleted file mode 100644 index 904a07863..000000000 --- a/server/types/models/oauth/oauth-client.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { OAuthClientModel } from '@server/models/oauth/oauth-client' - -export type MOAuthClient = Omit diff --git a/server/types/models/oauth/oauth-token.ts b/server/types/models/oauth/oauth-token.ts deleted file mode 100644 index 6af087e3c..000000000 --- a/server/types/models/oauth/oauth-token.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { OAuthTokenModel } from '@server/models/oauth/oauth-token' -import { PickWith } from '@shared/typescript-utils' -import { MUserAccountUrl } from '../user/user' - -type Use = PickWith - -// ############################################################################ - -export type MOAuthToken = Omit - -export type MOAuthTokenUser = - MOAuthToken & - Use<'User', MUserAccountUrl> & - { user?: MUserAccountUrl } diff --git a/server/types/models/runners/index.ts b/server/types/models/runners/index.ts deleted file mode 100644 index e94d4794e..000000000 --- a/server/types/models/runners/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './runner' -export * from './runner-job' -export * from './runner-registration-token' diff --git a/server/types/models/runners/runner-job.ts b/server/types/models/runners/runner-job.ts deleted file mode 100644 index ec983ba32..000000000 --- a/server/types/models/runners/runner-job.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { RunnerJobModel } from '@server/models/runner/runner-job' -import { PickWith } from '@shared/typescript-utils' -import { MRunner } from './runner' - -type Use = PickWith - -// ############################################################################ - -export type MRunnerJob = Omit - -// ############################################################################ - -export type MRunnerJobRunner = - MRunnerJob & - Use<'Runner', MRunner> - -export type MRunnerJobRunnerParent = - MRunnerJob & - Use<'Runner', MRunner> & - Use<'DependsOnRunnerJob', MRunnerJob> diff --git a/server/types/models/runners/runner-registration-token.ts b/server/types/models/runners/runner-registration-token.ts deleted file mode 100644 index 83b8614ad..000000000 --- a/server/types/models/runners/runner-registration-token.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token' - -// ############################################################################ - -export type MRunnerRegistrationToken = Omit diff --git a/server/types/models/runners/runner.ts b/server/types/models/runners/runner.ts deleted file mode 100644 index d35356378..000000000 --- a/server/types/models/runners/runner.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { RunnerModel } from '@server/models/runner/runner' - -// ############################################################################ - -export type MRunner = Omit diff --git a/server/types/models/server/index.ts b/server/types/models/server/index.ts deleted file mode 100644 index c853795ad..000000000 --- a/server/types/models/server/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './plugin' -export * from './server' -export * from './server-blocklist' diff --git a/server/types/models/server/plugin.ts b/server/types/models/server/plugin.ts deleted file mode 100644 index 83eb83794..000000000 --- a/server/types/models/server/plugin.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { PluginModel } from '@server/models/server/plugin' - -export type MPlugin = PluginModel - -// ############################################################################ - -// Format for API or AP object - -export type MPluginFormattable = - Pick diff --git a/server/types/models/server/server-blocklist.ts b/server/types/models/server/server-blocklist.ts deleted file mode 100644 index 71a4ea963..000000000 --- a/server/types/models/server/server-blocklist.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { ServerBlocklistModel } from '@server/models/server/server-blocklist' -import { PickWith } from '@shared/typescript-utils' -import { MAccountDefault, MAccountFormattable } from '../account/account' -import { MServer, MServerFormattable } from './server' - -type Use = PickWith - -// ############################################################################ - -export type MServerBlocklist = Omit - -// ############################################################################ - -export type MServerBlocklistAccountServer = - MServerBlocklist & - Use<'ByAccount', MAccountDefault> & - Use<'BlockedServer', MServer> - -// ############################################################################ - -// Format for API or AP object - -export type MServerBlocklistFormattable = - Pick & - Use<'ByAccount', MAccountFormattable> & - Use<'BlockedServer', MServerFormattable> diff --git a/server/types/models/server/server.ts b/server/types/models/server/server.ts deleted file mode 100644 index 0b16186cd..000000000 --- a/server/types/models/server/server.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { FunctionProperties, PickWith } from '@shared/typescript-utils' -import { ServerModel } from '../../../models/server/server' -import { MAccountBlocklistId } from '../account' - -type Use = PickWith - -// ############################################################################ - -export type MServer = Omit - -// ############################################################################ - -export type MServerHost = Pick -export type MServerRedundancyAllowed = Pick - -export type MServerHostBlocks = - MServerHost & - Use<'BlockedBy', MAccountBlocklistId[]> - -// ############################################################################ - -// Format for API or AP object - -export type MServerFormattable = - FunctionProperties & - Pick diff --git a/server/types/models/server/tracker.ts b/server/types/models/server/tracker.ts deleted file mode 100644 index 5fe03f8c0..000000000 --- a/server/types/models/server/tracker.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { TrackerModel } from '../../../models/server/tracker' - -export type MTracker = Omit - -// ############################################################################ - -export type MTrackerUrl = Pick diff --git a/server/types/models/user/index.ts b/server/types/models/user/index.ts deleted file mode 100644 index 5738f4107..000000000 --- a/server/types/models/user/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -export * from './user' -export * from './user-notification' -export * from './user-notification-setting' -export * from './user-registration' -export * from './user-video-history' diff --git a/server/types/models/user/user-notification-setting.ts b/server/types/models/user/user-notification-setting.ts deleted file mode 100644 index d1db645e7..000000000 --- a/server/types/models/user/user-notification-setting.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { UserNotificationSettingModel } from '@server/models/user/user-notification-setting' - -export type MNotificationSetting = Omit - -// ############################################################################ - -// Format for API or AP object - -export type MNotificationSettingFormattable = MNotificationSetting diff --git a/server/types/models/user/user-notification.ts b/server/types/models/user/user-notification.ts deleted file mode 100644 index a732c8aa9..000000000 --- a/server/types/models/user/user-notification.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { VideoAbuseModel } from '@server/models/abuse/video-abuse' -import { VideoCommentAbuseModel } from '@server/models/abuse/video-comment-abuse' -import { ApplicationModel } from '@server/models/application/application' -import { PluginModel } from '@server/models/server/plugin' -import { UserNotificationModel } from '@server/models/user/user-notification' -import { UserRegistrationModel } from '@server/models/user/user-registration' -import { PickWith, PickWithOpt } from '@shared/typescript-utils' -import { AbuseModel } from '../../../models/abuse/abuse' -import { AccountModel } from '../../../models/account/account' -import { ActorModel } from '../../../models/actor/actor' -import { ActorFollowModel } from '../../../models/actor/actor-follow' -import { ActorImageModel } from '../../../models/actor/actor-image' -import { ServerModel } from '../../../models/server/server' -import { VideoModel } from '../../../models/video/video' -import { VideoBlacklistModel } from '../../../models/video/video-blacklist' -import { VideoChannelModel } from '../../../models/video/video-channel' -import { VideoCommentModel } from '../../../models/video/video-comment' -import { VideoImportModel } from '../../../models/video/video-import' - -type Use = PickWith - -// ############################################################################ - -export module UserNotificationIncludes { - export type ActorImageInclude = Pick - - export type VideoInclude = Pick - export type VideoIncludeChannel = - VideoInclude & - PickWith - - export type ActorInclude = - Pick & - PickWith & - PickWith> - - export type VideoChannelInclude = Pick - export type VideoChannelIncludeActor = - VideoChannelInclude & - PickWith - - export type AccountInclude = Pick - export type AccountIncludeActor = - AccountInclude & - PickWith - - export type VideoCommentInclude = - Pick & - PickWith & - PickWith - - export type VideoAbuseInclude = - Pick & - PickWith - - export type VideoCommentAbuseInclude = - Pick & - PickWith & - PickWith>> - - export type AbuseInclude = - Pick & - PickWith & - PickWith & - PickWith - - export type VideoBlacklistInclude = - Pick & - PickWith - - export type VideoImportInclude = - Pick & - PickWith - - export type ActorFollower = - Pick & - PickWith & - PickWith> & - PickWithOpt - - export type ActorFollowing = - Pick & - PickWith & - PickWith & - PickWith> - - export type ActorFollowInclude = - Pick & - PickWith & - PickWith - - export type PluginInclude = - Pick - - export type ApplicationInclude = - Pick - - export type UserRegistrationInclude = - Pick -} - -// ############################################################################ - -export type MUserNotification = - Omit - -// ############################################################################ - -export type UserNotificationModelForApi = - MUserNotification & - Use<'Video', UserNotificationIncludes.VideoIncludeChannel> & - Use<'VideoComment', UserNotificationIncludes.VideoCommentInclude> & - Use<'Abuse', UserNotificationIncludes.AbuseInclude> & - Use<'VideoBlacklist', UserNotificationIncludes.VideoBlacklistInclude> & - Use<'VideoImport', UserNotificationIncludes.VideoImportInclude> & - Use<'ActorFollow', UserNotificationIncludes.ActorFollowInclude> & - Use<'Plugin', UserNotificationIncludes.PluginInclude> & - Use<'Application', UserNotificationIncludes.ApplicationInclude> & - Use<'Account', UserNotificationIncludes.AccountIncludeActor> & - Use<'UserRegistration', UserNotificationIncludes.UserRegistrationInclude> diff --git a/server/types/models/user/user-registration.ts b/server/types/models/user/user-registration.ts deleted file mode 100644 index 216423cc9..000000000 --- a/server/types/models/user/user-registration.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { UserRegistrationModel } from '@server/models/user/user-registration' -import { PickWith } from '@shared/typescript-utils' -import { MUserId } from './user' - -type Use = PickWith - -// ############################################################################ - -export type MRegistration = Omit - -// ############################################################################ - -export type MRegistrationFormattable = - MRegistration & - Use<'User', MUserId> diff --git a/server/types/models/user/user-video-history.ts b/server/types/models/user/user-video-history.ts deleted file mode 100644 index 34e2930e7..000000000 --- a/server/types/models/user/user-video-history.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { UserVideoHistoryModel } from '../../../models/user/user-video-history' - -export type MUserVideoHistory = Omit - -export type MUserVideoHistoryTime = Pick diff --git a/server/types/models/user/user.ts b/server/types/models/user/user.ts deleted file mode 100644 index 89092c242..000000000 --- a/server/types/models/user/user.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { AccountModel } from '@server/models/account/account' -import { UserModel } from '@server/models/user/user' -import { MVideoPlaylist } from '@server/types/models' -import { PickWith, PickWithOpt } from '@shared/typescript-utils' -import { - MAccount, - MAccountDefault, - MAccountDefaultChannelDefault, - MAccountFormattable, - MAccountId, - MAccountIdActorId, - MAccountUrl -} from '../account' -import { MChannelFormattable } from '../video/video-channels' -import { MNotificationSetting, MNotificationSettingFormattable } from './user-notification-setting' - -type Use = PickWith - -// ############################################################################ - -export type MUser = Omit - -// ############################################################################ - -export type MUserQuotaUsed = MUser & { videoQuotaUsed?: number, videoQuotaUsedDaily?: number } -export type MUserId = Pick - -// ############################################################################ - -// With account - -export type MUserAccountId = - MUser & - Use<'Account', MAccountId> - -export type MUserAccountUrl = - MUser & - Use<'Account', MAccountUrl & MAccountIdActorId> - -export type MUserAccount = - MUser & - Use<'Account', MAccount> - -export type MUserAccountDefault = - MUser & - Use<'Account', MAccountDefault> - -// With channel - -export type MUserNotifSettingChannelDefault = - MUser & - Use<'NotificationSetting', MNotificationSetting> & - Use<'Account', MAccountDefaultChannelDefault> - -// With notification settings - -export type MUserWithNotificationSetting = - MUser & - Use<'NotificationSetting', MNotificationSetting> - -export type MUserNotifSettingAccount = - MUser & - Use<'NotificationSetting', MNotificationSetting> & - Use<'Account', MAccount> - -// Default scope - -export type MUserDefault = - MUser & - Use<'NotificationSetting', MNotificationSetting> & - Use<'Account', MAccountDefault> - -// ############################################################################ - -// Format for API or AP object - -type MAccountWithChannels = MAccountFormattable & PickWithOpt -type MAccountWithChannelsAndSpecialPlaylists = - MAccountWithChannels & - PickWithOpt - -export type MUserFormattable = - MUserQuotaUsed & - Use<'Account', MAccountWithChannels> & - PickWithOpt - -export type MMyUserFormattable = - MUserFormattable & - Use<'Account', MAccountWithChannelsAndSpecialPlaylists> diff --git a/server/types/models/video/index.ts b/server/types/models/video/index.ts deleted file mode 100644 index 7f05db666..000000000 --- a/server/types/models/video/index.ts +++ /dev/null @@ -1,26 +0,0 @@ -export * from './local-video-viewer-watch-section' -export * from './local-video-viewer-watch-section' -export * from './local-video-viewer' -export * from './storyboard' -export * from './schedule-video-update' -export * from './tag' -export * from './thumbnail' -export * from './video' -export * from './video-blacklist' -export * from './video-caption' -export * from './video-change-ownership' -export * from './video-channel-sync' -export * from './video-channels' -export * from './video-comment' -export * from './video-file' -export * from './video-import' -export * from './video-live-replay-setting' -export * from './video-live-session' -export * from './video-live' -export * from './video-password' -export * from './video-playlist' -export * from './video-playlist-element' -export * from './video-rate' -export * from './video-redundancy' -export * from './video-share' -export * from './video-streaming-playlist' diff --git a/server/types/models/video/local-video-viewer-watch-section.ts b/server/types/models/video/local-video-viewer-watch-section.ts deleted file mode 100644 index be7a3bba0..000000000 --- a/server/types/models/video/local-video-viewer-watch-section.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section' - -// ############################################################################ - -export type MLocalVideoViewerWatchSection = Omit diff --git a/server/types/models/video/local-video-viewer.ts b/server/types/models/video/local-video-viewer.ts deleted file mode 100644 index b78ef5507..000000000 --- a/server/types/models/video/local-video-viewer.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' -import { PickWith } from '@shared/typescript-utils' -import { MLocalVideoViewerWatchSection } from './local-video-viewer-watch-section' -import { MVideo } from './video' - -type Use = PickWith - -// ############################################################################ - -export type MLocalVideoViewer = Omit - -export type MLocalVideoViewerVideo = - MLocalVideoViewer & - Use<'Video', MVideo> - -export type MLocalVideoViewerWithWatchSections = - MLocalVideoViewer & - Use<'Video', MVideo> & - Use<'WatchSections', MLocalVideoViewerWatchSection[]> diff --git a/server/types/models/video/schedule-video-update.ts b/server/types/models/video/schedule-video-update.ts deleted file mode 100644 index 39fd73501..000000000 --- a/server/types/models/video/schedule-video-update.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update' - -// ############################################################################ - -export type MScheduleVideoUpdate = Omit - -// ############################################################################ - -// Format for API or AP object - -export type MScheduleVideoUpdateFormattable = Pick diff --git a/server/types/models/video/storyboard.ts b/server/types/models/video/storyboard.ts deleted file mode 100644 index a0403d4f0..000000000 --- a/server/types/models/video/storyboard.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { StoryboardModel } from '@server/models/video/storyboard' -import { PickWith } from '@shared/typescript-utils' -import { MVideo } from './video' - -type Use = PickWith - -// ############################################################################ - -export type MStoryboard = Omit - -// ############################################################################ - -export type MStoryboardVideo = - MStoryboard & - Use<'Video', MVideo> diff --git a/server/types/models/video/tag.ts b/server/types/models/video/tag.ts deleted file mode 100644 index 64a68873e..000000000 --- a/server/types/models/video/tag.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { TagModel } from '../../../models/video/tag' - -export type MTag = Omit diff --git a/server/types/models/video/thumbnail.ts b/server/types/models/video/thumbnail.ts deleted file mode 100644 index c3b27681f..000000000 --- a/server/types/models/video/thumbnail.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { PickWith } from '@shared/typescript-utils' -import { ThumbnailModel } from '../../../models/video/thumbnail' -import { MVideo } from './video' - -type Use = PickWith - -// ############################################################################ - -export type MThumbnail = Omit - -// ############################################################################ - -export type MThumbnailVideo = - MThumbnail & - Use<'Video', MVideo> diff --git a/server/types/models/video/video-blacklist.ts b/server/types/models/video/video-blacklist.ts deleted file mode 100644 index 048b63bd2..000000000 --- a/server/types/models/video/video-blacklist.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { PickWith } from '@shared/typescript-utils' -import { VideoBlacklistModel } from '../../../models/video/video-blacklist' -import { MVideo, MVideoFormattable } from './video' - -type Use = PickWith - -// ############################################################################ - -export type MVideoBlacklist = Omit - -export type MVideoBlacklistLight = Pick -export type MVideoBlacklistUnfederated = Pick - -// ############################################################################ - -export type MVideoBlacklistLightVideo = - MVideoBlacklistLight & - Use<'Video', MVideo> - -export type MVideoBlacklistVideo = - MVideoBlacklist & - Use<'Video', MVideo> - -// ############################################################################ - -// Format for API or AP object - -export type MVideoBlacklistFormattable = - MVideoBlacklist & - Use<'Video', MVideoFormattable> diff --git a/server/types/models/video/video-caption.ts b/server/types/models/video/video-caption.ts deleted file mode 100644 index d3adec362..000000000 --- a/server/types/models/video/video-caption.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { PickWith } from '@shared/typescript-utils' -import { VideoCaptionModel } from '../../../models/video/video-caption' -import { MVideo, MVideoUUID } from './video' - -type Use = PickWith - -// ############################################################################ - -export type MVideoCaption = Omit - -// ############################################################################ - -export type MVideoCaptionLanguage = Pick -export type MVideoCaptionLanguageUrl = Pick - -export type MVideoCaptionVideo = - MVideoCaption & - Use<'Video', Pick> - -// ############################################################################ - -// Format for API or AP object - -export type MVideoCaptionFormattable = - MVideoCaption & - Pick & - Use<'Video', MVideoUUID> diff --git a/server/types/models/video/video-change-ownership.ts b/server/types/models/video/video-change-ownership.ts deleted file mode 100644 index d99f25071..000000000 --- a/server/types/models/video/video-change-ownership.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { VideoChangeOwnershipModel } from '@server/models/video/video-change-ownership' -import { PickWith } from '@shared/typescript-utils' -import { MAccountDefault, MAccountFormattable } from '../account/account' -import { MVideoFormattable, MVideoWithAllFiles } from './video' - -type Use = PickWith - -// ############################################################################ - -export type MVideoChangeOwnership = Omit - -export type MVideoChangeOwnershipFull = - MVideoChangeOwnership & - Use<'Initiator', MAccountDefault> & - Use<'NextOwner', MAccountDefault> & - Use<'Video', MVideoWithAllFiles> - -// ############################################################################ - -// Format for API or AP object - -export type MVideoChangeOwnershipFormattable = - Pick & - Use<'Initiator', MAccountFormattable> & - Use<'NextOwner', MAccountFormattable> & - Use<'Video', MVideoFormattable> diff --git a/server/types/models/video/video-channel-sync.ts b/server/types/models/video/video-channel-sync.ts deleted file mode 100644 index 429ab70b0..000000000 --- a/server/types/models/video/video-channel-sync.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { VideoChannelSyncModel } from '@server/models/video/video-channel-sync' -import { FunctionProperties, PickWith } from '@shared/typescript-utils' -import { MChannelAccountDefault, MChannelFormattable } from './video-channels' - -type Use = PickWith - -export type MChannelSync = Omit - -export type MChannelSyncChannel = - MChannelSync & - Use<'VideoChannel', MChannelAccountDefault> & - FunctionProperties - -export type MChannelSyncFormattable = - FunctionProperties & - Use<'VideoChannel', MChannelFormattable> & - MChannelSync diff --git a/server/types/models/video/video-channels.ts b/server/types/models/video/video-channels.ts deleted file mode 100644 index 57e991494..000000000 --- a/server/types/models/video/video-channels.ts +++ /dev/null @@ -1,153 +0,0 @@ -import { FunctionProperties, PickWith, PickWithOpt } from '@shared/typescript-utils' -import { VideoChannelModel } from '../../../models/video/video-channel' -import { - MAccountActor, - MAccountAPI, - MAccountDefault, - MAccountFormattable, - MAccountLight, - MAccountSummaryBlocks, - MAccountSummaryFormattable, - MAccountUrl, - MAccountUserId -} from '../account' -import { - MActor, - MActorAccountChannelId, - MActorAPChannel, - MActorAPI, - MActorDefault, - MActorDefaultBanner, - MActorDefaultLight, - MActorFormattable, - MActorHost, - MActorHostOnly, - MActorLight, - MActorSummary, - MActorSummaryFormattable, - MActorUrl -} from '../actor' -import { MVideo } from './video' - -type Use = PickWith - -// ############################################################################ - -export type MChannel = Omit - -// ############################################################################ - -export type MChannelId = Pick - -// ############################################################################ - -export type MChannelIdActor = - MChannelId & - Use<'Actor', MActorAccountChannelId> - -export type MChannelUserId = - Pick & - Use<'Account', MAccountUserId> - -export type MChannelActor = - MChannel & - Use<'Actor', MActor> - -export type MChannelUrl = Use<'Actor', MActorUrl> - -// Default scope -export type MChannelDefault = - MChannel & - Use<'Actor', MActorDefault> - -export type MChannelBannerDefault = - MChannel & - Use<'Actor', MActorDefaultBanner> - -// ############################################################################ - -// Not all association attributes - -export type MChannelActorLight = - MChannel & - Use<'Actor', MActorLight> - -export type MChannelAccountLight = - MChannel & - Use<'Actor', MActorDefaultLight> & - Use<'Account', MAccountLight> - -export type MChannelHost = - MChannel & - Use<'Actor', MActorHost> - -export type MChannelHostOnly = - MChannelId & - Use<'Actor', MActorHostOnly> - -// ############################################################################ - -// Account associations - -export type MChannelAccountActor = - MChannel & - Use<'Account', MAccountActor> - -export type MChannelBannerAccountDefault = - MChannel & - Use<'Actor', MActorDefaultBanner> & - Use<'Account', MAccountDefault> - -export type MChannelAccountDefault = - MChannel & - Use<'Actor', MActorDefault> & - Use<'Account', MAccountDefault> - -// ############################################################################ - -// Videos associations -export type MChannelVideos = - MChannel & - Use<'Videos', MVideo[]> - -// ############################################################################ - -// For API - -export type MChannelSummary = - FunctionProperties & - Pick & - Use<'Actor', MActorSummary> - -export type MChannelSummaryAccount = - MChannelSummary & - Use<'Account', MAccountSummaryBlocks> - -export type MChannelAPI = - MChannel & - Use<'Actor', MActorAPI> & - Use<'Account', MAccountAPI> - -// ############################################################################ - -// Format for API or AP object - -export type MChannelSummaryFormattable = - FunctionProperties & - Pick & - Use<'Actor', MActorSummaryFormattable> - -export type MChannelAccountSummaryFormattable = - MChannelSummaryFormattable & - Use<'Account', MAccountSummaryFormattable> - -export type MChannelFormattable = - FunctionProperties & - Pick & - Use<'Actor', MActorFormattable> & - PickWithOpt - -export type MChannelAP = - Pick & - Use<'Actor', MActorAPChannel> & - Use<'Account', MAccountUrl> diff --git a/server/types/models/video/video-comment.ts b/server/types/models/video/video-comment.ts deleted file mode 100644 index b66de064f..000000000 --- a/server/types/models/video/video-comment.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { PickWith, PickWithOpt } from '@shared/typescript-utils' -import { VideoCommentModel } from '../../../models/video/video-comment' -import { MAccountDefault, MAccountFormattable, MAccountUrl } from '../account' -import { MVideo, MVideoAccountLight, MVideoFeed, MVideoIdUrl, MVideoUrl } from './video' - -type Use = PickWith - -// ############################################################################ - -export type MComment = Omit -export type MCommentTotalReplies = MComment & { totalReplies?: number } -export type MCommentId = Pick -export type MCommentUrl = Pick - -// ############################################################################ - -export type MCommentOwner = - MComment & - Use<'Account', MAccountDefault> - -export type MCommentVideo = - MComment & - Use<'Video', MVideoAccountLight> - -export type MCommentReply = - MComment & - Use<'InReplyToVideoComment', MComment> - -export type MCommentOwnerVideo = - MComment & - Use<'Account', MAccountDefault> & - Use<'Video', MVideoAccountLight> - -export type MCommentOwnerVideoReply = - MComment & - Use<'Account', MAccountDefault> & - Use<'Video', MVideoAccountLight> & - Use<'InReplyToVideoComment', MComment> - -export type MCommentOwnerReplyVideoLight = - MComment & - Use<'Account', MAccountDefault> & - Use<'InReplyToVideoComment', MComment> & - Use<'Video', MVideoIdUrl> - -export type MCommentOwnerVideoFeed = - MCommentOwner & - Use<'Video', MVideoFeed> - -// ############################################################################ - -export type MCommentAPI = MComment & { totalReplies: number } - -// ############################################################################ - -// Format for API or AP object - -export type MCommentFormattable = - MCommentTotalReplies & - Use<'Account', MAccountFormattable> - -export type MCommentAdminFormattable = - MComment & - Use<'Account', MAccountFormattable> & - Use<'Video', MVideo> - -export type MCommentAP = - MComment & - Use<'Account', MAccountUrl> & - PickWithOpt & - PickWithOpt diff --git a/server/types/models/video/video-file.ts b/server/types/models/video/video-file.ts deleted file mode 100644 index 68106788d..000000000 --- a/server/types/models/video/video-file.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { PickWith, PickWithOpt } from '@shared/typescript-utils' -import { VideoFileModel } from '../../../models/video/video-file' -import { MVideo, MVideoUUID } from './video' -import { MVideoRedundancy, MVideoRedundancyFileUrl } from './video-redundancy' -import { MStreamingPlaylist, MStreamingPlaylistVideo } from './video-streaming-playlist' - -type Use = PickWith - -// ############################################################################ - -export type MVideoFile = Omit - -export type MVideoFileVideo = - MVideoFile & - Use<'Video', MVideo> - -export type MVideoFileStreamingPlaylist = - MVideoFile & - Use<'VideoStreamingPlaylist', MStreamingPlaylist> - -export type MVideoFileStreamingPlaylistVideo = - MVideoFile & - Use<'VideoStreamingPlaylist', MStreamingPlaylistVideo> - -export type MVideoFileVideoUUID = - MVideoFile & - Use<'Video', MVideoUUID> - -export type MVideoFileRedundanciesAll = - MVideoFile & - PickWithOpt - -export type MVideoFileRedundanciesOpt = - MVideoFile & - PickWithOpt - -export function isStreamingPlaylistFile (file: any): file is MVideoFileStreamingPlaylist { - return !!file.videoStreamingPlaylistId -} - -export function isWebVideoFile (file: any): file is MVideoFileVideo { - return !!file.videoId -} diff --git a/server/types/models/video/video-import.ts b/server/types/models/video/video-import.ts deleted file mode 100644 index 650c293f7..000000000 --- a/server/types/models/video/video-import.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { VideoImportModel } from '@server/models/video/video-import' -import { PickWith, PickWithOpt } from '@shared/typescript-utils' -import { MUser } from '../user/user' -import { MVideo, MVideoAccountLight, MVideoFormattable, MVideoTag, MVideoThumbnail, MVideoWithFile } from './video' - -type Use = PickWith - -// ############################################################################ - -export type MVideoImport = Omit - -export type MVideoImportVideo = - MVideoImport & - Use<'Video', MVideo> - -// ############################################################################ - -type VideoAssociation = MVideoTag & MVideoAccountLight & MVideoThumbnail - -export type MVideoImportDefault = - MVideoImport & - Use<'User', MUser> & - Use<'Video', VideoAssociation> - -export type MVideoImportDefaultFiles = - MVideoImport & - Use<'User', MUser> & - Use<'Video', VideoAssociation & MVideoWithFile> - -// ############################################################################ - -// Format for API or AP object - -export type MVideoImportFormattable = - MVideoImport & - PickWithOpt diff --git a/server/types/models/video/video-live-replay-setting.ts b/server/types/models/video/video-live-replay-setting.ts deleted file mode 100644 index c5a5adf54..000000000 --- a/server/types/models/video/video-live-replay-setting.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { VideoLiveReplaySettingModel } from '@server/models/video/video-live-replay-setting' - -export type MLiveReplaySetting = Omit diff --git a/server/types/models/video/video-live-session.ts b/server/types/models/video/video-live-session.ts deleted file mode 100644 index 852e2c24b..000000000 --- a/server/types/models/video/video-live-session.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { VideoLiveSessionModel } from '@server/models/video/video-live-session' -import { PickWith } from '@shared/typescript-utils' -import { MVideo } from './video' -import { MLiveReplaySetting } from './video-live-replay-setting' - -type Use = PickWith - -// ############################################################################ - -export type MVideoLiveSession = Omit - -// ############################################################################ - -export type MVideoLiveSessionReplay = - MVideoLiveSession & - Use<'ReplayVideo', MVideo> & - Use<'ReplaySetting', MLiveReplaySetting> diff --git a/server/types/models/video/video-live.ts b/server/types/models/video/video-live.ts deleted file mode 100644 index a899edfa6..000000000 --- a/server/types/models/video/video-live.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { VideoLiveModel } from '@server/models/video/video-live' -import { PickWith } from '@shared/typescript-utils' -import { MVideo } from './video' -import { MLiveReplaySetting } from './video-live-replay-setting' - -type Use = PickWith - -// ############################################################################ - -export type MVideoLive = Omit - -// ############################################################################ - -export type MVideoLiveVideo = - MVideoLive & - Use<'Video', MVideo> - -// ############################################################################ - -export type MVideoLiveVideoWithSetting = - MVideoLiveVideo & - Use<'ReplaySetting', MLiveReplaySetting> diff --git a/server/types/models/video/video-password.ts b/server/types/models/video/video-password.ts deleted file mode 100644 index 313cc3e0c..000000000 --- a/server/types/models/video/video-password.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { VideoPasswordModel } from '@server/models/video/video-password' - -export type MVideoPassword = Omit diff --git a/server/types/models/video/video-playlist-element.ts b/server/types/models/video/video-playlist-element.ts deleted file mode 100644 index eae676096..000000000 --- a/server/types/models/video/video-playlist-element.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element' -import { PickWith } from '@shared/typescript-utils' -import { MVideoFormattable, MVideoThumbnail, MVideoUrl } from './video' -import { MVideoPlaylistPrivacy } from './video-playlist' - -type Use = PickWith - -// ############################################################################ - -export type MVideoPlaylistElement = Omit - -// ############################################################################ - -export type MVideoPlaylistElementId = Pick - -export type MVideoPlaylistElementLight = Pick - -// ############################################################################ - -export type MVideoPlaylistVideoThumbnail = - MVideoPlaylistElement & - Use<'Video', MVideoThumbnail> - -export type MVideoPlaylistElementVideoUrlPlaylistPrivacy = - MVideoPlaylistElement & - Use<'Video', MVideoUrl> & - Use<'VideoPlaylist', MVideoPlaylistPrivacy> - -// ############################################################################ - -// Format for API or AP object - -export type MVideoPlaylistElementFormattable = - MVideoPlaylistElement & - Use<'Video', MVideoFormattable> - -export type MVideoPlaylistElementAP = - MVideoPlaylistElement & - Use<'Video', MVideoUrl> diff --git a/server/types/models/video/video-playlist.ts b/server/types/models/video/video-playlist.ts deleted file mode 100644 index 40f0dfc14..000000000 --- a/server/types/models/video/video-playlist.ts +++ /dev/null @@ -1,104 +0,0 @@ -import { MVideoPlaylistElementLight } from '@server/types/models/video/video-playlist-element' -import { PickWith } from '@shared/typescript-utils' -import { VideoPlaylistModel } from '../../../models/video/video-playlist' -import { MAccount, MAccountDefault, MAccountSummary, MAccountSummaryFormattable } from '../account' -import { MThumbnail } from './thumbnail' -import { MChannelDefault, MChannelSummary, MChannelSummaryFormattable, MChannelUrl } from './video-channels' - -type Use = PickWith - -// ############################################################################ - -export type MVideoPlaylist = Omit - -// ############################################################################ - -export type MVideoPlaylistId = Pick -export type MVideoPlaylistSummary = - Pick & - Pick & - Pick -export type MVideoPlaylistPrivacy = Pick -export type MVideoPlaylistUUID = Pick -export type MVideoPlaylistVideosLength = MVideoPlaylist & { videosLength?: number } - -// ############################################################################ - -// With elements - -export type MVideoPlaylistSummaryWithElements = - MVideoPlaylistSummary & - Use<'VideoPlaylistElements', MVideoPlaylistElementLight[]> - -// ############################################################################ - -// With account - -export type MVideoPlaylistOwner = - MVideoPlaylist & - Use<'OwnerAccount', MAccount> - -export type MVideoPlaylistOwnerDefault = - MVideoPlaylist & - Use<'OwnerAccount', MAccountDefault> - -// ############################################################################ - -// With thumbnail - -export type MVideoPlaylistThumbnail = - MVideoPlaylist & - Use<'Thumbnail', MThumbnail> - -export type MVideoPlaylistAccountThumbnail = - MVideoPlaylist & - Use<'OwnerAccount', MAccountDefault> & - Use<'Thumbnail', MThumbnail> - -// ############################################################################ - -// With channel - -export type MVideoPlaylistAccountChannelDefault = - MVideoPlaylist & - Use<'OwnerAccount', MAccountDefault> & - Use<'VideoChannel', MChannelDefault> - -// ############################################################################ - -// With all associations - -export type MVideoPlaylistFull = - MVideoPlaylistVideosLength & - Use<'OwnerAccount', MAccountDefault> & - Use<'VideoChannel', MChannelDefault> & - Use<'Thumbnail', MThumbnail> - -// ############################################################################ - -// For API - -export type MVideoPlaylistAccountChannelSummary = - MVideoPlaylist & - Use<'OwnerAccount', MAccountSummary> & - Use<'VideoChannel', MChannelSummary> - -export type MVideoPlaylistFullSummary = - MVideoPlaylistVideosLength & - Use<'Thumbnail', MThumbnail> & - Use<'OwnerAccount', MAccountSummary> & - Use<'VideoChannel', MChannelSummary> - -// ############################################################################ - -// Format for API or AP object - -export type MVideoPlaylistFormattable = - MVideoPlaylistVideosLength & - Use<'OwnerAccount', MAccountSummaryFormattable> & - Use<'VideoChannel', MChannelSummaryFormattable> - -export type MVideoPlaylistAP = - MVideoPlaylist & - Use<'Thumbnail', MThumbnail> & - Use<'VideoChannel', MChannelUrl> diff --git a/server/types/models/video/video-rate.ts b/server/types/models/video/video-rate.ts deleted file mode 100644 index 0dbdf3c41..000000000 --- a/server/types/models/video/video-rate.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { AccountVideoRateModel } from '@server/models/account/account-video-rate' -import { PickWith } from '@shared/typescript-utils' -import { MAccountAudience, MAccountUrl } from '../account/account' -import { MVideo, MVideoFormattable } from './video' - -type Use = PickWith - -// ############################################################################ - -export type MAccountVideoRate = Omit - -export type MAccountVideoRateAccountUrl = - MAccountVideoRate & - Use<'Account', MAccountUrl> - -export type MAccountVideoRateAccountVideo = - MAccountVideoRate & - Use<'Account', MAccountAudience> & - Use<'Video', MVideo> - -// ############################################################################ - -// Format for API or AP object - -export type MAccountVideoRateFormattable = - Pick & - Use<'Video', MVideoFormattable> diff --git a/server/types/models/video/video-redundancy.ts b/server/types/models/video/video-redundancy.ts deleted file mode 100644 index e2a9beb93..000000000 --- a/server/types/models/video/video-redundancy.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { VideoFileModel } from '@server/models/video/video-file' -import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist' -import { PickWith, PickWithOpt } from '@shared/typescript-utils' -import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' -import { MVideoUrl } from './video' -import { MVideoFile, MVideoFileVideo } from './video-file' -import { MStreamingPlaylistVideo } from './video-streaming-playlist' - -type Use = PickWith - -// ############################################################################ - -export type MVideoRedundancy = Omit - -export type MVideoRedundancyFileUrl = Pick - -// ############################################################################ - -export type MVideoRedundancyFile = - MVideoRedundancy & - Use<'VideoFile', MVideoFile> - -export type MVideoRedundancyFileVideo = - MVideoRedundancy & - Use<'VideoFile', MVideoFileVideo> - -export type MVideoRedundancyStreamingPlaylistVideo = - MVideoRedundancy & - Use<'VideoStreamingPlaylist', MStreamingPlaylistVideo> - -export type MVideoRedundancyVideo = - MVideoRedundancy & - Use<'VideoFile', MVideoFileVideo> & - Use<'VideoStreamingPlaylist', MStreamingPlaylistVideo> - -// ############################################################################ - -// Format for API or AP object - -export type MVideoRedundancyAP = - MVideoRedundancy & - PickWithOpt> & - PickWithOpt> diff --git a/server/types/models/video/video-share.ts b/server/types/models/video/video-share.ts deleted file mode 100644 index ffc0edad6..000000000 --- a/server/types/models/video/video-share.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { PickWith } from '@shared/typescript-utils' -import { VideoShareModel } from '../../../models/video/video-share' -import { MActorDefault } from '../actor' -import { MVideo } from './video' - -type Use = PickWith - -// ############################################################################ - -export type MVideoShare = Omit - -export type MVideoShareActor = - MVideoShare & - Use<'Actor', MActorDefault> - -export type MVideoShareFull = - MVideoShare & - Use<'Actor', MActorDefault> & - Use<'Video', MVideo> diff --git a/server/types/models/video/video-source.ts b/server/types/models/video/video-source.ts deleted file mode 100644 index 0948f3b2e..000000000 --- a/server/types/models/video/video-source.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { VideoSourceModel } from '@server/models/video/video-source' - -export type MVideoSource = Omit diff --git a/server/types/models/video/video-streaming-playlist.ts b/server/types/models/video/video-streaming-playlist.ts deleted file mode 100644 index 1c2f83489..000000000 --- a/server/types/models/video/video-streaming-playlist.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { PickWith, PickWithOpt } from '@shared/typescript-utils' -import { VideoStreamingPlaylistModel } from '../../../models/video/video-streaming-playlist' -import { MVideo } from './video' -import { MVideoFile } from './video-file' -import { MVideoRedundancy, MVideoRedundancyFileUrl } from './video-redundancy' - -type Use = PickWith - -// ############################################################################ - -export type MStreamingPlaylist = Omit - -export type MStreamingPlaylistFiles = - MStreamingPlaylist & - Use<'VideoFiles', MVideoFile[]> - -export type MStreamingPlaylistVideo = - MStreamingPlaylist & - Use<'Video', MVideo> - -export type MStreamingPlaylistFilesVideo = - MStreamingPlaylist & - Use<'VideoFiles', MVideoFile[]> & - Use<'Video', MVideo> - -export type MStreamingPlaylistRedundanciesAll = - MStreamingPlaylist & - Use<'VideoFiles', MVideoFile[]> & - Use<'RedundancyVideos', MVideoRedundancy[]> - -export type MStreamingPlaylistRedundancies = - MStreamingPlaylist & - Use<'VideoFiles', MVideoFile[]> & - Use<'RedundancyVideos', MVideoRedundancyFileUrl[]> - -export type MStreamingPlaylistRedundanciesOpt = - MStreamingPlaylist & - Use<'VideoFiles', MVideoFile[]> & - PickWithOpt - -export function isStreamingPlaylist (value: MVideo | MStreamingPlaylistVideo): value is MStreamingPlaylistVideo { - return !!(value as MStreamingPlaylist).videoId -} diff --git a/server/types/models/video/video.ts b/server/types/models/video/video.ts deleted file mode 100644 index 53ee94269..000000000 --- a/server/types/models/video/video.ts +++ /dev/null @@ -1,225 +0,0 @@ -import { PickWith, PickWithOpt } from '@shared/typescript-utils' -import { VideoModel } from '../../../models/video/video' -import { MTrackerUrl } from '../server/tracker' -import { MUserVideoHistoryTime } from '../user/user-video-history' -import { MScheduleVideoUpdate } from './schedule-video-update' -import { MStoryboard } from './storyboard' -import { MTag } from './tag' -import { MThumbnail } from './thumbnail' -import { MVideoBlacklist, MVideoBlacklistLight, MVideoBlacklistUnfederated } from './video-blacklist' -import { MVideoCaptionLanguage, MVideoCaptionLanguageUrl } from './video-caption' -import { - MChannelAccountDefault, - MChannelAccountLight, - MChannelAccountSummaryFormattable, - MChannelActor, - MChannelFormattable, - MChannelHostOnly, - MChannelUserId -} from './video-channels' -import { MVideoFile, MVideoFileRedundanciesAll, MVideoFileRedundanciesOpt } from './video-file' -import { MVideoLive } from './video-live' -import { - MStreamingPlaylistFiles, - MStreamingPlaylistRedundancies, - MStreamingPlaylistRedundanciesAll, - MStreamingPlaylistRedundanciesOpt -} from './video-streaming-playlist' - -type Use = PickWith - -// ############################################################################ - -export type MVideo = - Omit - -// ############################################################################ - -export type MVideoId = Pick -export type MVideoUrl = Pick -export type MVideoUUID = Pick - -export type MVideoImmutable = Pick -export type MVideoIdUrl = MVideoId & MVideoUrl -export type MVideoFeed = Pick - -// ############################################################################ - -// Video raw associations: schedules, video files, tags, thumbnails, captions, streaming playlists, passwords - -// "With" to not confuse with the VideoFile model -export type MVideoWithFile = - MVideo & - Use<'VideoFiles', MVideoFile[]> & - Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]> - -export type MVideoThumbnail = - MVideo & - Use<'Thumbnails', MThumbnail[]> - -export type MVideoIdThumbnail = - MVideoId & - Use<'Thumbnails', MThumbnail[]> - -export type MVideoWithFileThumbnail = - MVideo & - Use<'VideoFiles', MVideoFile[]> & - Use<'Thumbnails', MThumbnail[]> - -export type MVideoThumbnailBlacklist = - MVideo & - Use<'Thumbnails', MThumbnail[]> & - Use<'VideoBlacklist', MVideoBlacklistLight> - -export type MVideoTag = - MVideo & - Use<'Tags', MTag[]> - -export type MVideoWithSchedule = - MVideo & - PickWithOpt - -export type MVideoWithCaptions = - MVideo & - Use<'VideoCaptions', MVideoCaptionLanguage[]> - -export type MVideoWithStreamingPlaylist = - MVideo & - Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]> - -// ############################################################################ - -// Associations with not all their attributes - -export type MVideoUserHistory = - MVideo & - Use<'UserVideoHistories', MUserVideoHistoryTime[]> - -export type MVideoWithBlacklistLight = - MVideo & - Use<'VideoBlacklist', MVideoBlacklistLight> - -export type MVideoAccountLight = - MVideo & - Use<'VideoChannel', MChannelAccountLight> - -export type MVideoWithRights = - MVideo & - Use<'VideoBlacklist', MVideoBlacklistLight> & - Use<'VideoChannel', MChannelUserId> - -// ############################################################################ - -// All files with some additional associations - -export type MVideoWithAllFiles = - MVideo & - Use<'VideoFiles', MVideoFile[]> & - Use<'Thumbnails', MThumbnail[]> & - Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]> - -export type MVideoAccountLightBlacklistAllFiles = - MVideo & - Use<'VideoFiles', MVideoFile[]> & - Use<'Thumbnails', MThumbnail[]> & - Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]> & - Use<'VideoChannel', MChannelAccountLight> & - Use<'VideoBlacklist', MVideoBlacklistLight> - -// ############################################################################ - -// With account - -export type MVideoAccountDefault = - MVideo & - Use<'VideoChannel', MChannelAccountDefault> - -export type MVideoThumbnailAccountDefault = - MVideo & - Use<'Thumbnails', MThumbnail[]> & - Use<'VideoChannel', MChannelAccountDefault> - -export type MVideoWithChannelActor = - MVideo & - Use<'VideoChannel', MChannelActor> - -export type MVideoWithHost = - MVideo & - Use<'VideoChannel', MChannelHostOnly> - -export type MVideoFullLight = - MVideo & - Use<'Thumbnails', MThumbnail[]> & - Use<'VideoBlacklist', MVideoBlacklistLight> & - Use<'Tags', MTag[]> & - Use<'VideoChannel', MChannelAccountLight> & - Use<'UserVideoHistories', MUserVideoHistoryTime[]> & - Use<'VideoFiles', MVideoFile[]> & - Use<'ScheduleVideoUpdate', MScheduleVideoUpdate> & - Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]> & - Use<'VideoLive', MVideoLive> - -// ############################################################################ - -// API - -export type MVideoAP = - MVideo & - Use<'Tags', MTag[]> & - Use<'VideoChannel', MChannelAccountLight> & - Use<'VideoStreamingPlaylists', MStreamingPlaylistFiles[]> & - Use<'VideoCaptions', MVideoCaptionLanguageUrl[]> & - Use<'VideoBlacklist', MVideoBlacklistUnfederated> & - Use<'VideoFiles', MVideoFileRedundanciesOpt[]> & - Use<'Thumbnails', MThumbnail[]> & - Use<'VideoLive', MVideoLive> & - Use<'Storyboard', MStoryboard> - -export type MVideoAPLight = Omit - -export type MVideoDetails = - MVideo & - Use<'VideoBlacklist', MVideoBlacklistLight> & - Use<'Tags', MTag[]> & - Use<'VideoChannel', MChannelAccountLight> & - Use<'ScheduleVideoUpdate', MScheduleVideoUpdate> & - Use<'Thumbnails', MThumbnail[]> & - Use<'UserVideoHistories', MUserVideoHistoryTime[]> & - Use<'VideoStreamingPlaylists', MStreamingPlaylistRedundancies[]> & - Use<'VideoFiles', MVideoFileRedundanciesOpt[]> & - Use<'Trackers', MTrackerUrl[]> - -export type MVideoForUser = - MVideo & - Use<'VideoChannel', MChannelAccountDefault> & - Use<'ScheduleVideoUpdate', MScheduleVideoUpdate> & - Use<'VideoBlacklist', MVideoBlacklistLight> & - Use<'Thumbnails', MThumbnail[]> - -export type MVideoForRedundancyAPI = - MVideo & - Use<'VideoFiles', MVideoFileRedundanciesAll[]> & - Use<'VideoStreamingPlaylists', MStreamingPlaylistRedundanciesAll[]> - -// ############################################################################ - -// Format for API or AP object - -export type MVideoFormattable = - MVideo & - PickWithOpt & - Use<'VideoChannel', MChannelAccountSummaryFormattable> & - PickWithOpt> & - PickWithOpt> & - PickWithOpt & - PickWithOpt - -export type MVideoFormattableDetails = - MVideoFormattable & - Use<'VideoChannel', MChannelFormattable> & - Use<'Tags', MTag[]> & - Use<'VideoStreamingPlaylists', MStreamingPlaylistRedundanciesOpt[]> & - Use<'VideoFiles', MVideoFileRedundanciesOpt[]> & - PickWithOpt diff --git a/server/types/plugins/index.ts b/server/types/plugins/index.ts deleted file mode 100644 index bf9c35d49..000000000 --- a/server/types/plugins/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './plugin-library.model' -export * from './register-server-auth.model' -export * from './register-server-option.model' -export * from './register-server-websocket-route.model' diff --git a/server/types/plugins/plugin-library.model.ts b/server/types/plugins/plugin-library.model.ts deleted file mode 100644 index 5b517ee9f..000000000 --- a/server/types/plugins/plugin-library.model.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { RegisterServerOptions } from './register-server-option.model' - -export interface PluginLibrary { - register: (options: RegisterServerOptions) => Promise - - unregister: () => Promise -} diff --git a/server/types/plugins/register-server-auth.model.ts b/server/types/plugins/register-server-auth.model.ts deleted file mode 100644 index e10968c20..000000000 --- a/server/types/plugins/register-server-auth.model.ts +++ /dev/null @@ -1,72 +0,0 @@ -import express from 'express' -import { UserAdminFlag, UserRole } from '@shared/models' -import { MOAuthToken, MUser } from '../models' - -export type RegisterServerAuthOptions = RegisterServerAuthPassOptions | RegisterServerAuthExternalOptions - -export type AuthenticatedResultUpdaterFieldName = 'displayName' | 'role' | 'adminFlags' | 'videoQuota' | 'videoQuotaDaily' - -export interface RegisterServerAuthenticatedResult { - // Update the user profile if it already exists - // Default behaviour is no update - // Introduced in PeerTube >= 5.1 - userUpdater?: (options: { - fieldName: AuthenticatedResultUpdaterFieldName - currentValue: T - newValue: T - }) => T - - username: string - email: string - role?: UserRole - displayName?: string - - // PeerTube >= 5.1 - adminFlags?: UserAdminFlag - - // PeerTube >= 5.1 - videoQuota?: number - // PeerTube >= 5.1 - videoQuotaDaily?: number -} - -export interface RegisterServerExternalAuthenticatedResult extends RegisterServerAuthenticatedResult { - req: express.Request - res: express.Response -} - -interface RegisterServerAuthBase { - // Authentication name (a plugin can register multiple auth strategies) - authName: string - - // Called by PeerTube when a user from your plugin logged out - // Returns a redirectUrl sent to the client or nothing - onLogout?(user: MUser, req: express.Request): Promise - - // Your plugin can hook PeerTube access/refresh token validity - // So you can control for your plugin the user session lifetime - hookTokenValidity?(options: { token: MOAuthToken, type: 'access' | 'refresh' }): Promise<{ valid: boolean }> -} - -export interface RegisterServerAuthPassOptions extends RegisterServerAuthBase { - // Weight of this authentication so PeerTube tries the auth methods in DESC weight order - getWeight(): number - - // Used by PeerTube to login a user - // Returns null if the login failed, or { username, email } on success - login(body: { - id: string - password: string - }): Promise -} - -export interface RegisterServerAuthExternalOptions extends RegisterServerAuthBase { - // Will be displayed in a block next to the login form - authDisplayName: () => string - - onAuthRequest: (req: express.Request, res: express.Response) => void -} - -export interface RegisterServerAuthExternalResult { - userAuthenticated (options: RegisterServerExternalAuthenticatedResult): void -} diff --git a/server/types/plugins/register-server-option.model.ts b/server/types/plugins/register-server-option.model.ts deleted file mode 100644 index 103ef234b..000000000 --- a/server/types/plugins/register-server-option.model.ts +++ /dev/null @@ -1,171 +0,0 @@ -import { Response, Router } from 'express' -import { Server } from 'http' -import { Logger } from 'winston' -import { ActorModel } from '@server/models/actor/actor' -import { - PluginPlaylistPrivacyManager, - PluginSettingsManager, - PluginStorageManager, - PluginTranscodingManager, - PluginVideoCategoryManager, - PluginVideoLanguageManager, - PluginVideoLicenceManager, - PluginVideoPrivacyManager, - RegisterServerHookOptions, - RegisterServerSettingOptions, - ServerConfig, - ThumbnailType, - VideoBlacklistCreate -} from '@shared/models' -import { MUserDefault, MVideo, MVideoThumbnail, UserNotificationModelForApi } from '../models' -import { - RegisterServerAuthExternalOptions, - RegisterServerAuthExternalResult, - RegisterServerAuthPassOptions -} from './register-server-auth.model' -import { RegisterServerWebSocketRouteOptions } from './register-server-websocket-route.model' - -export type PeerTubeHelpers = { - logger: Logger - - database: { - query: Function - } - - videos: { - loadByUrl: (url: string) => Promise - loadByIdOrUUID: (id: number | string) => Promise - - removeVideo: (videoId: number) => Promise - - ffprobe: (path: string) => Promise - - getFiles: (id: number | string) => Promise<{ - webtorrent: { // TODO: remove in v7 - videoFiles: { - path: string // Could be null if using remote storage - url: string - resolution: number - size: number - fps: number - }[] - } - - webVideo: { - videoFiles: { - path: string // Could be null if using remote storage - url: string - resolution: number - size: number - fps: number - }[] - } - - hls: { - videoFiles: { - path: string // Could be null if using remote storage - url: string - resolution: number - size: number - fps: number - }[] - } - - thumbnails: { - type: ThumbnailType - path: string - }[] - }> - } - - config: { - getWebserverUrl: () => string - - // PeerTube >= 5.1 - getServerListeningConfig: () => { hostname: string, port: number } - - getServerConfig: () => Promise - } - - moderation: { - blockServer: (options: { byAccountId: number, hostToBlock: string }) => Promise - unblockServer: (options: { byAccountId: number, hostToUnblock: string }) => Promise - blockAccount: (options: { byAccountId: number, handleToBlock: string }) => Promise - unblockAccount: (options: { byAccountId: number, handleToUnblock: string }) => Promise - - blacklistVideo: (options: { videoIdOrUUID: number | string, createOptions: VideoBlacklistCreate }) => Promise - unblacklistVideo: (options: { videoIdOrUUID: number | string }) => Promise - } - - server: { - // PeerTube >= 5.0 - getHTTPServer: () => Server - - getServerActor: () => Promise - } - - socket: { - sendNotification: (userId: number, notification: UserNotificationModelForApi) => void - sendVideoLiveNewState: (video: MVideo) => void - } - - plugin: { - // PeerTube >= 3.2 - getBaseStaticRoute: () => string - - // PeerTube >= 3.2 - getBaseRouterRoute: () => string - // PeerTube >= 5.0 - getBaseWebSocketRoute: () => string - - // PeerTube >= 3.2 - getDataDirectoryPath: () => string - } - - user: { - // PeerTube >= 3.2 - getAuthUser: (response: Response) => Promise - - // PeerTube >= 4.3 - loadById: (id: number) => Promise - } -} - -export type RegisterServerOptions = { - registerHook: (options: RegisterServerHookOptions) => void - - registerSetting: (options: RegisterServerSettingOptions) => void - - settingsManager: PluginSettingsManager - - storageManager: PluginStorageManager - - videoCategoryManager: PluginVideoCategoryManager - videoLanguageManager: PluginVideoLanguageManager - videoLicenceManager: PluginVideoLicenceManager - - videoPrivacyManager: PluginVideoPrivacyManager - playlistPrivacyManager: PluginPlaylistPrivacyManager - - transcodingManager: PluginTranscodingManager - - registerIdAndPassAuth: (options: RegisterServerAuthPassOptions) => void - registerExternalAuth: (options: RegisterServerAuthExternalOptions) => RegisterServerAuthExternalResult - unregisterIdAndPassAuth: (authName: string) => void - unregisterExternalAuth: (authName: string) => void - - // Get plugin router to create custom routes - // Base routes of this router are - // * /plugins/:pluginName/:pluginVersion/router/... - // * /plugins/:pluginName/router/... - getRouter(): Router - - // PeerTube >= 5.0 - // Register WebSocket route - // Base routes of the WebSocket router are - // * /plugins/:pluginName/:pluginVersion/ws/... - // * /plugins/:pluginName/ws/... - registerWebSocketRoute: (options: RegisterServerWebSocketRouteOptions) => void - - peertubeHelpers: PeerTubeHelpers -} diff --git a/server/types/sequelize.ts b/server/types/sequelize.ts deleted file mode 100644 index e399c3d5d..000000000 --- a/server/types/sequelize.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { AttributesOnly } from '@shared/typescript-utils' -import { Model } from 'sequelize' - -// Thanks to sequelize-typescript: https://github.com/RobinBuschmann/sequelize-typescript - -export type Diff = - ({ [P in T]: P } & { [P in U]: never } & { [ x: string ]: never })[T] - -export type Omit = { [P in Diff]: T[P] } - -export type RecursivePartial = { [P in keyof T]?: RecursivePartial } - -export type FilteredModelAttributes> = Partial> & { - id?: number | any - createdAt?: Date | any - updatedAt?: Date | any - deletedAt?: Date | any - version?: number | any -} diff --git a/shared/core-utils/abuse/abuse-predefined-reasons.ts b/shared/core-utils/abuse/abuse-predefined-reasons.ts deleted file mode 100644 index 9967e54dd..000000000 --- a/shared/core-utils/abuse/abuse-predefined-reasons.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { AbusePredefinedReasons, AbusePredefinedReasonsString } from '../../models/moderation/abuse/abuse-reason.model' - -export const abusePredefinedReasonsMap: { - [key in AbusePredefinedReasonsString]: AbusePredefinedReasons -} = { - violentOrRepulsive: AbusePredefinedReasons.VIOLENT_OR_REPULSIVE, - hatefulOrAbusive: AbusePredefinedReasons.HATEFUL_OR_ABUSIVE, - spamOrMisleading: AbusePredefinedReasons.SPAM_OR_MISLEADING, - privacy: AbusePredefinedReasons.PRIVACY, - rights: AbusePredefinedReasons.RIGHTS, - serverRules: AbusePredefinedReasons.SERVER_RULES, - thumbnails: AbusePredefinedReasons.THUMBNAILS, - captions: AbusePredefinedReasons.CAPTIONS -} diff --git a/shared/core-utils/abuse/index.ts b/shared/core-utils/abuse/index.ts deleted file mode 100644 index 244b83cff..000000000 --- a/shared/core-utils/abuse/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './abuse-predefined-reasons' diff --git a/shared/core-utils/common/env.ts b/shared/core-utils/common/env.ts deleted file mode 100644 index 973f895d4..000000000 --- a/shared/core-utils/common/env.ts +++ /dev/null @@ -1,46 +0,0 @@ -function parallelTests () { - return process.env.MOCHA_PARALLEL === 'true' -} - -function isGithubCI () { - return !!process.env.GITHUB_WORKSPACE -} - -function areHttpImportTestsDisabled () { - const disabled = process.env.DISABLE_HTTP_IMPORT_TESTS === 'true' - - if (disabled) console.log('DISABLE_HTTP_IMPORT_TESTS env set to "true" so import tests are disabled') - - return disabled -} - -function areMockObjectStorageTestsDisabled () { - const disabled = process.env.ENABLE_OBJECT_STORAGE_TESTS !== 'true' - - if (disabled) console.log('ENABLE_OBJECT_STORAGE_TESTS env is not set to "true" so object storage tests are disabled') - - return disabled -} - -function areScalewayObjectStorageTestsDisabled () { - if (areMockObjectStorageTestsDisabled()) return true - - const enabled = process.env.OBJECT_STORAGE_SCALEWAY_KEY_ID && process.env.OBJECT_STORAGE_SCALEWAY_ACCESS_KEY - if (!enabled) { - console.log( - 'OBJECT_STORAGE_SCALEWAY_KEY_ID and/or OBJECT_STORAGE_SCALEWAY_ACCESS_KEY are not set, so scaleway object storage tests are disabled' - ) - - return true - } - - return false -} - -export { - parallelTests, - isGithubCI, - areHttpImportTestsDisabled, - areMockObjectStorageTestsDisabled, - areScalewayObjectStorageTestsDisabled -} diff --git a/shared/core-utils/common/index.ts b/shared/core-utils/common/index.ts deleted file mode 100644 index 8d63ee1b2..000000000 --- a/shared/core-utils/common/index.ts +++ /dev/null @@ -1,12 +0,0 @@ -export * from './array' -export * from './random' -export * from './date' -export * from './env' -export * from './number' -export * from './object' -export * from './path' -export * from './regexp' -export * from './time' -export * from './promises' -export * from './url' -export * from './version' diff --git a/shared/core-utils/common/path.ts b/shared/core-utils/common/path.ts deleted file mode 100644 index 006505316..000000000 --- a/shared/core-utils/common/path.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { basename, extname, isAbsolute, join, resolve } from 'path' - -let rootPath: string - -function root () { - if (rootPath) return rootPath - - rootPath = __dirname - - if (basename(rootPath) === 'tools') rootPath = resolve(rootPath, '..') - if (basename(rootPath) === 'scripts') rootPath = resolve(rootPath, '..') - if (basename(rootPath) === 'common') rootPath = resolve(rootPath, '..') - if (basename(rootPath) === 'core-utils') rootPath = resolve(rootPath, '..') - if (basename(rootPath) === 'shared') rootPath = resolve(rootPath, '..') - if (basename(rootPath) === 'server') rootPath = resolve(rootPath, '..') - if (basename(rootPath) === 'dist') rootPath = resolve(rootPath, '..') - - return rootPath -} - -function buildPath (path: string) { - if (isAbsolute(path)) return path - - return join(root(), path) -} - -function getLowercaseExtension (filename: string) { - const ext = extname(filename) || '' - - return ext.toLowerCase() -} - -function buildAbsoluteFixturePath (path: string, customCIPath = false) { - if (isAbsolute(path)) return path - - if (customCIPath && process.env.GITHUB_WORKSPACE) { - return join(process.env.GITHUB_WORKSPACE, 'fixtures', path) - } - - return join(root(), 'server', 'tests', 'fixtures', path) -} - -export { - root, - buildPath, - buildAbsoluteFixturePath, - getLowercaseExtension -} diff --git a/shared/core-utils/common/url.ts b/shared/core-utils/common/url.ts deleted file mode 100644 index 33fc5ee3a..000000000 --- a/shared/core-utils/common/url.ts +++ /dev/null @@ -1,150 +0,0 @@ -import { Video, VideoPlaylist } from '../../models' -import { secondsToTime } from './date' - -function addQueryParams (url: string, params: { [ id: string ]: string }) { - const objUrl = new URL(url) - - for (const key of Object.keys(params)) { - objUrl.searchParams.append(key, params[key]) - } - - return objUrl.toString() -} - -function removeQueryParams (url: string) { - const objUrl = new URL(url) - - objUrl.searchParams.forEach((_v, k) => objUrl.searchParams.delete(k)) - - return objUrl.toString() -} - -function buildPlaylistLink (playlist: Pick, base?: string) { - return (base ?? window.location.origin) + buildPlaylistWatchPath(playlist) -} - -function buildPlaylistWatchPath (playlist: Pick) { - return '/w/p/' + playlist.shortUUID -} - -function buildVideoWatchPath (video: Pick) { - return '/w/' + video.shortUUID -} - -function buildVideoLink (video: Pick, base?: string) { - return (base ?? window.location.origin) + buildVideoWatchPath(video) -} - -function buildPlaylistEmbedPath (playlist: Pick) { - return '/video-playlists/embed/' + playlist.uuid -} - -function buildPlaylistEmbedLink (playlist: Pick, base?: string) { - return (base ?? window.location.origin) + buildPlaylistEmbedPath(playlist) -} - -function buildVideoEmbedPath (video: Pick) { - return '/videos/embed/' + video.uuid -} - -function buildVideoEmbedLink (video: Pick, base?: string) { - return (base ?? window.location.origin) + buildVideoEmbedPath(video) -} - -function decorateVideoLink (options: { - url: string - - startTime?: number - stopTime?: number - - subtitle?: string - - loop?: boolean - autoplay?: boolean - muted?: boolean - - // Embed options - title?: boolean - warningTitle?: boolean - - controls?: boolean - controlBar?: boolean - - peertubeLink?: boolean - p2p?: boolean -}) { - const { url } = options - - const params = new URLSearchParams() - - if (options.startTime !== undefined && options.startTime !== null) { - const startTimeInt = Math.floor(options.startTime) - params.set('start', secondsToTime(startTimeInt)) - } - - if (options.stopTime) { - const stopTimeInt = Math.floor(options.stopTime) - params.set('stop', secondsToTime(stopTimeInt)) - } - - if (options.subtitle) params.set('subtitle', options.subtitle) - - if (options.loop === true) params.set('loop', '1') - if (options.autoplay === true) params.set('autoplay', '1') - if (options.muted === true) params.set('muted', '1') - if (options.title === false) params.set('title', '0') - if (options.warningTitle === false) params.set('warningTitle', '0') - - if (options.controls === false) params.set('controls', '0') - if (options.controlBar === false) params.set('controlBar', '0') - - if (options.peertubeLink === false) params.set('peertubeLink', '0') - if (options.p2p !== undefined) params.set('p2p', options.p2p ? '1' : '0') - - return buildUrl(url, params) -} - -function decoratePlaylistLink (options: { - url: string - - playlistPosition?: number -}) { - const { url } = options - - const params = new URLSearchParams() - - if (options.playlistPosition) params.set('playlistPosition', '' + options.playlistPosition) - - return buildUrl(url, params) -} - -// --------------------------------------------------------------------------- - -export { - addQueryParams, - removeQueryParams, - - buildPlaylistLink, - buildVideoLink, - - buildVideoWatchPath, - buildPlaylistWatchPath, - - buildPlaylistEmbedPath, - buildVideoEmbedPath, - - buildPlaylistEmbedLink, - buildVideoEmbedLink, - - decorateVideoLink, - decoratePlaylistLink -} - -function buildUrl (url: string, params: URLSearchParams) { - let hasParams = false - params.forEach(() => { hasParams = true }) - - if (hasParams) return url + '?' + params.toString() - - return url -} diff --git a/shared/core-utils/i18n/index.ts b/shared/core-utils/i18n/index.ts deleted file mode 100644 index 8f7cbe2c7..000000000 --- a/shared/core-utils/i18n/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './i18n' diff --git a/shared/core-utils/index.ts b/shared/core-utils/index.ts deleted file mode 100644 index 8daaa2d04..000000000 --- a/shared/core-utils/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export * from './abuse' -export * from './common' -export * from './i18n' -export * from './plugins' -export * from './renderer' -export * from './users' -export * from './videos' diff --git a/shared/core-utils/plugins/hooks.ts b/shared/core-utils/plugins/hooks.ts deleted file mode 100644 index 96bcc945e..000000000 --- a/shared/core-utils/plugins/hooks.ts +++ /dev/null @@ -1,61 +0,0 @@ -import { RegisteredExternalAuthConfig } from '@shared/models' -import { HookType } from '../../models/plugins/hook-type.enum' -import { isCatchable, isPromise } from '../common/promises' - -function getHookType (hookName: string) { - if (hookName.startsWith('filter:')) return HookType.FILTER - if (hookName.startsWith('action:')) return HookType.ACTION - - return HookType.STATIC -} - -async function internalRunHook (options: { - handler: Function - hookType: HookType - result: T - params: any - onError: (err: Error) => void -}) { - const { handler, hookType, result, params, onError } = options - - try { - if (hookType === HookType.FILTER) { - const p = handler(result, params) - - const newResult = isPromise(p) - ? await p - : p - - return newResult - } - - // Action/static hooks do not have result value - const p = handler(params) - - if (hookType === HookType.STATIC) { - if (isPromise(p)) await p - - return undefined - } - - if (hookType === HookType.ACTION) { - if (isCatchable(p)) p.catch((err: any) => onError(err)) - - return undefined - } - } catch (err) { - onError(err) - } - - return result -} - -function getExternalAuthHref (apiUrl: string, auth: RegisteredExternalAuthConfig) { - return apiUrl + `/plugins/${auth.name}/${auth.version}/auth/${auth.authName}` -} - -export { - getHookType, - internalRunHook, - getExternalAuthHref -} diff --git a/shared/core-utils/plugins/index.ts b/shared/core-utils/plugins/index.ts deleted file mode 100644 index fc78d3512..000000000 --- a/shared/core-utils/plugins/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './hooks' diff --git a/shared/core-utils/renderer/index.ts b/shared/core-utils/renderer/index.ts deleted file mode 100644 index 0ad29d782..000000000 --- a/shared/core-utils/renderer/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './markdown' -export * from './html' diff --git a/shared/core-utils/users/index.ts b/shared/core-utils/users/index.ts deleted file mode 100644 index 1cbf0af1b..000000000 --- a/shared/core-utils/users/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './user-role' diff --git a/shared/core-utils/users/user-role.ts b/shared/core-utils/users/user-role.ts deleted file mode 100644 index 5f3b9a10f..000000000 --- a/shared/core-utils/users/user-role.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { UserRight, UserRole } from '../../models/users' - -export const USER_ROLE_LABELS: { [ id in UserRole ]: string } = { - [UserRole.USER]: 'User', - [UserRole.MODERATOR]: 'Moderator', - [UserRole.ADMINISTRATOR]: 'Administrator' -} - -const userRoleRights: { [ id in UserRole ]: UserRight[] } = { - [UserRole.ADMINISTRATOR]: [ - UserRight.ALL - ], - - [UserRole.MODERATOR]: [ - UserRight.MANAGE_VIDEO_BLACKLIST, - UserRight.MANAGE_ABUSES, - UserRight.MANAGE_ANY_VIDEO_CHANNEL, - UserRight.REMOVE_ANY_VIDEO, - UserRight.REMOVE_ANY_VIDEO_PLAYLIST, - UserRight.REMOVE_ANY_VIDEO_COMMENT, - UserRight.UPDATE_ANY_VIDEO, - UserRight.SEE_ALL_VIDEOS, - UserRight.MANAGE_ACCOUNTS_BLOCKLIST, - UserRight.MANAGE_SERVERS_BLOCKLIST, - UserRight.MANAGE_USERS, - UserRight.SEE_ALL_COMMENTS, - UserRight.MANAGE_REGISTRATIONS - ], - - [UserRole.USER]: [] -} - -export function hasUserRight (userRole: UserRole, userRight: UserRight) { - const userRights = userRoleRights[userRole] - - return userRights.includes(UserRight.ALL) || userRights.includes(userRight) -} diff --git a/shared/core-utils/videos/bitrate.ts b/shared/core-utils/videos/bitrate.ts deleted file mode 100644 index 6be027826..000000000 --- a/shared/core-utils/videos/bitrate.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { VideoResolution } from '@shared/models' - -type BitPerPixel = { [ id in VideoResolution ]: number } - -// https://bitmovin.com/video-bitrate-streaming-hls-dash/ - -const minLimitBitPerPixel: BitPerPixel = { - [VideoResolution.H_NOVIDEO]: 0, - [VideoResolution.H_144P]: 0.02, - [VideoResolution.H_240P]: 0.02, - [VideoResolution.H_360P]: 0.02, - [VideoResolution.H_480P]: 0.02, - [VideoResolution.H_720P]: 0.02, - [VideoResolution.H_1080P]: 0.02, - [VideoResolution.H_1440P]: 0.02, - [VideoResolution.H_4K]: 0.02 -} - -const averageBitPerPixel: BitPerPixel = { - [VideoResolution.H_NOVIDEO]: 0, - [VideoResolution.H_144P]: 0.19, - [VideoResolution.H_240P]: 0.17, - [VideoResolution.H_360P]: 0.15, - [VideoResolution.H_480P]: 0.12, - [VideoResolution.H_720P]: 0.11, - [VideoResolution.H_1080P]: 0.10, - [VideoResolution.H_1440P]: 0.09, - [VideoResolution.H_4K]: 0.08 -} - -const maxBitPerPixel: BitPerPixel = { - [VideoResolution.H_NOVIDEO]: 0, - [VideoResolution.H_144P]: 0.32, - [VideoResolution.H_240P]: 0.29, - [VideoResolution.H_360P]: 0.26, - [VideoResolution.H_480P]: 0.22, - [VideoResolution.H_720P]: 0.19, - [VideoResolution.H_1080P]: 0.17, - [VideoResolution.H_1440P]: 0.16, - [VideoResolution.H_4K]: 0.14 -} - -function getAverageTheoreticalBitrate (options: { - resolution: VideoResolution - ratio: number - fps: number -}) { - const targetBitrate = calculateBitrate({ ...options, bitPerPixel: averageBitPerPixel }) - if (!targetBitrate) return 192 * 1000 - - return targetBitrate -} - -function getMaxTheoreticalBitrate (options: { - resolution: VideoResolution - ratio: number - fps: number -}) { - const targetBitrate = calculateBitrate({ ...options, bitPerPixel: maxBitPerPixel }) - if (!targetBitrate) return 256 * 1000 - - return targetBitrate -} - -function getMinTheoreticalBitrate (options: { - resolution: VideoResolution - ratio: number - fps: number -}) { - const minLimitBitrate = calculateBitrate({ ...options, bitPerPixel: minLimitBitPerPixel }) - if (!minLimitBitrate) return 10 * 1000 - - return minLimitBitrate -} - -// --------------------------------------------------------------------------- - -export { - getAverageTheoreticalBitrate, - getMaxTheoreticalBitrate, - getMinTheoreticalBitrate -} - -// --------------------------------------------------------------------------- - -function calculateBitrate (options: { - bitPerPixel: BitPerPixel - resolution: VideoResolution - ratio: number - fps: number -}) { - const { bitPerPixel, resolution, ratio, fps } = options - - const resolutionsOrder = [ - VideoResolution.H_4K, - VideoResolution.H_1440P, - VideoResolution.H_1080P, - VideoResolution.H_720P, - VideoResolution.H_480P, - VideoResolution.H_360P, - VideoResolution.H_240P, - VideoResolution.H_144P, - VideoResolution.H_NOVIDEO - ] - - for (const toTestResolution of resolutionsOrder) { - if (toTestResolution <= resolution) { - return Math.floor(resolution * resolution * ratio * fps * bitPerPixel[toTestResolution]) - } - } - - throw new Error('Unknown resolution ' + resolution) -} diff --git a/shared/core-utils/videos/common.ts b/shared/core-utils/videos/common.ts deleted file mode 100644 index 0431edaaf..000000000 --- a/shared/core-utils/videos/common.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { VideoStreamingPlaylistType } from '@shared/models' -import { VideoPrivacy } from '../../models/videos/video-privacy.enum' -import { VideoDetails } from '../../models/videos/video.model' - -function getAllPrivacies () { - return [ VideoPrivacy.PUBLIC, VideoPrivacy.INTERNAL, VideoPrivacy.PRIVATE, VideoPrivacy.UNLISTED, VideoPrivacy.PASSWORD_PROTECTED ] -} - -function getAllFiles (video: Partial>) { - const files = video.files - - const hls = getHLS(video) - if (hls) return files.concat(hls.files) - - return files -} - -function getHLS (video: Partial>) { - return video.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) -} - -export { - getAllPrivacies, - getAllFiles, - getHLS -} diff --git a/shared/core-utils/videos/index.ts b/shared/core-utils/videos/index.ts deleted file mode 100644 index 2cf319395..000000000 --- a/shared/core-utils/videos/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './bitrate' -export * from './common' diff --git a/shared/extra-utils/file.ts b/shared/extra-utils/file.ts deleted file mode 100644 index 8060ab520..000000000 --- a/shared/extra-utils/file.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { stat } from 'fs-extra' - -async function getFileSize (path: string) { - const stats = await stat(path) - - return stats.size -} - -export { - getFileSize -} diff --git a/shared/extra-utils/index.ts b/shared/extra-utils/index.ts deleted file mode 100644 index d4cfcbec8..000000000 --- a/shared/extra-utils/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './crypto' -export * from './file' -export * from './uuid' diff --git a/shared/extra-utils/uuid.ts b/shared/extra-utils/uuid.ts deleted file mode 100644 index f3c80e046..000000000 --- a/shared/extra-utils/uuid.ts +++ /dev/null @@ -1,32 +0,0 @@ -import short, { uuid } from 'short-uuid' - -const translator = short() - -function buildUUID () { - return uuid() -} - -function uuidToShort (uuid: string) { - if (!uuid) return uuid - - return translator.fromUUID(uuid) -} - -function shortToUUID (shortUUID: string) { - if (!shortUUID) return shortUUID - - return translator.toUUID(shortUUID) -} - -function isShortUUID (value: string) { - if (!value) return false - - return value.length === translator.maxLength -} - -export { - buildUUID, - uuidToShort, - shortToUUID, - isShortUUID -} diff --git a/shared/ffmpeg/ffmpeg-command-wrapper.ts b/shared/ffmpeg/ffmpeg-command-wrapper.ts deleted file mode 100644 index efb75c198..000000000 --- a/shared/ffmpeg/ffmpeg-command-wrapper.ts +++ /dev/null @@ -1,246 +0,0 @@ -import ffmpeg, { FfmpegCommand, getAvailableEncoders } from 'fluent-ffmpeg' -import { pick, promisify0 } from '@shared/core-utils' -import { AvailableEncoders, EncoderOptionsBuilder, EncoderOptionsBuilderParams, EncoderProfile } from '@shared/models' - -type FFmpegLogger = { - info: (msg: string, obj?: any) => void - debug: (msg: string, obj?: any) => void - warn: (msg: string, obj?: any) => void - error: (msg: string, obj?: any) => void -} - -export interface FFmpegCommandWrapperOptions { - availableEncoders?: AvailableEncoders - profile?: string - - niceness: number - tmpDirectory: string - threads: number - - logger: FFmpegLogger - lTags?: { tags: string[] } - - updateJobProgress?: (progress?: number) => void - onEnd?: () => void - onError?: (err: Error) => void -} - -export class FFmpegCommandWrapper { - private static supportedEncoders: Map - - private readonly availableEncoders: AvailableEncoders - private readonly profile: string - - private readonly niceness: number - private readonly tmpDirectory: string - private readonly threads: number - - private readonly logger: FFmpegLogger - private readonly lTags: { tags: string[] } - - private readonly updateJobProgress: (progress?: number) => void - private readonly onEnd?: () => void - private readonly onError?: (err: Error) => void - - private command: FfmpegCommand - - constructor (options: FFmpegCommandWrapperOptions) { - this.availableEncoders = options.availableEncoders - this.profile = options.profile - this.niceness = options.niceness - this.tmpDirectory = options.tmpDirectory - this.threads = options.threads - this.logger = options.logger - this.lTags = options.lTags || { tags: [] } - - this.updateJobProgress = options.updateJobProgress - - this.onEnd = options.onEnd - this.onError = options.onError - } - - getAvailableEncoders () { - return this.availableEncoders - } - - getProfile () { - return this.profile - } - - getCommand () { - return this.command - } - - // --------------------------------------------------------------------------- - - debugLog (msg: string, meta: any) { - this.logger.debug(msg, { ...meta, ...this.lTags }) - } - - // --------------------------------------------------------------------------- - - buildCommand (input: string) { - if (this.command) throw new Error('Command is already built') - - // We set cwd explicitly because ffmpeg appears to create temporary files when trancoding which fails in read-only file systems - this.command = ffmpeg(input, { - niceness: this.niceness, - cwd: this.tmpDirectory - }) - - if (this.threads > 0) { - // If we don't set any threads ffmpeg will chose automatically - this.command.outputOption('-threads ' + this.threads) - } - - return this.command - } - - async runCommand (options: { - silent?: boolean // false by default - } = {}) { - const { silent = false } = options - - return new Promise((res, rej) => { - let shellCommand: string - - this.command.on('start', cmdline => { shellCommand = cmdline }) - - this.command.on('error', (err, stdout, stderr) => { - if (silent !== true) this.logger.error('Error in ffmpeg.', { stdout, stderr, shellCommand, ...this.lTags }) - - if (this.onError) this.onError(err) - - rej(err) - }) - - this.command.on('end', (stdout, stderr) => { - this.logger.debug('FFmpeg command ended.', { stdout, stderr, shellCommand, ...this.lTags }) - - if (this.onEnd) this.onEnd() - - res() - }) - - if (this.updateJobProgress) { - this.command.on('progress', progress => { - if (!progress.percent) return - - // Sometimes ffmpeg returns an invalid progress - let percent = Math.round(progress.percent) - if (percent < 0) percent = 0 - if (percent > 100) percent = 100 - - this.updateJobProgress(percent) - }) - } - - this.command.run() - }) - } - - // --------------------------------------------------------------------------- - - static resetSupportedEncoders () { - FFmpegCommandWrapper.supportedEncoders = undefined - } - - // Run encoder builder depending on available encoders - // Try encoders by priority: if the encoder is available, run the chosen profile or fallback to the default one - // If the default one does not exist, check the next encoder - async getEncoderBuilderResult (options: EncoderOptionsBuilderParams & { - streamType: 'video' | 'audio' - input: string - - videoType: 'vod' | 'live' - }) { - if (!this.availableEncoders) { - throw new Error('There is no available encoders') - } - - const { streamType, videoType } = options - - const encodersToTry = this.availableEncoders.encodersToTry[videoType][streamType] - const encoders = this.availableEncoders.available[videoType] - - for (const encoder of encodersToTry) { - if (!(await this.checkFFmpegEncoders(this.availableEncoders)).get(encoder)) { - this.logger.debug(`Encoder ${encoder} not available in ffmpeg, skipping.`, this.lTags) - continue - } - - if (!encoders[encoder]) { - this.logger.debug(`Encoder ${encoder} not available in peertube encoders, skipping.`, this.lTags) - continue - } - - // An object containing available profiles for this encoder - const builderProfiles: EncoderProfile = encoders[encoder] - let builder = builderProfiles[this.profile] - - if (!builder) { - this.logger.debug(`Profile ${this.profile} for encoder ${encoder} not available. Fallback to default.`, this.lTags) - builder = builderProfiles.default - - if (!builder) { - this.logger.debug(`Default profile for encoder ${encoder} not available. Try next available encoder.`, this.lTags) - continue - } - } - - const result = await builder( - pick(options, [ - 'input', - 'canCopyAudio', - 'canCopyVideo', - 'resolution', - 'inputBitrate', - 'fps', - 'inputRatio', - 'streamNum' - ]) - ) - - return { - result, - - // If we don't have output options, then copy the input stream - encoder: result.copy === true - ? 'copy' - : encoder - } - } - - return null - } - - // Detect supported encoders by ffmpeg - private async checkFFmpegEncoders (peertubeAvailableEncoders: AvailableEncoders): Promise> { - if (FFmpegCommandWrapper.supportedEncoders !== undefined) { - return FFmpegCommandWrapper.supportedEncoders - } - - const getAvailableEncodersPromise = promisify0(getAvailableEncoders) - const availableFFmpegEncoders = await getAvailableEncodersPromise() - - const searchEncoders = new Set() - for (const type of [ 'live', 'vod' ]) { - for (const streamType of [ 'audio', 'video' ]) { - for (const encoder of peertubeAvailableEncoders.encodersToTry[type][streamType]) { - searchEncoders.add(encoder) - } - } - } - - const supportedEncoders = new Map() - - for (const searchEncoder of searchEncoders) { - supportedEncoders.set(searchEncoder, availableFFmpegEncoders[searchEncoder] !== undefined) - } - - this.logger.info('Built supported ffmpeg encoders.', { supportedEncoders, searchEncoders, ...this.lTags }) - - FFmpegCommandWrapper.supportedEncoders = supportedEncoders - return supportedEncoders - } -} diff --git a/shared/ffmpeg/ffmpeg-default-transcoding-profile.ts b/shared/ffmpeg/ffmpeg-default-transcoding-profile.ts deleted file mode 100644 index 8a3f32011..000000000 --- a/shared/ffmpeg/ffmpeg-default-transcoding-profile.ts +++ /dev/null @@ -1,187 +0,0 @@ -import { FfprobeData } from 'fluent-ffmpeg' -import { getAverageTheoreticalBitrate, getMaxTheoreticalBitrate, getMinTheoreticalBitrate } from '@shared/core-utils' -import { - buildStreamSuffix, - ffprobePromise, - getAudioStream, - getMaxAudioBitrate, - getVideoStream, - getVideoStreamBitrate, - getVideoStreamDimensionsInfo, - getVideoStreamFPS -} from '@shared/ffmpeg' -import { EncoderOptionsBuilder, EncoderOptionsBuilderParams, VideoResolution } from '@shared/models' - -const defaultX264VODOptionsBuilder: EncoderOptionsBuilder = (options: EncoderOptionsBuilderParams) => { - const { fps, inputRatio, inputBitrate, resolution } = options - - const targetBitrate = getTargetBitrate({ inputBitrate, ratio: inputRatio, fps, resolution }) - - return { - outputOptions: [ - ...getCommonOutputOptions(targetBitrate), - - `-r ${fps}` - ] - } -} - -const defaultX264LiveOptionsBuilder: EncoderOptionsBuilder = (options: EncoderOptionsBuilderParams) => { - const { streamNum, fps, inputBitrate, inputRatio, resolution } = options - - const targetBitrate = getTargetBitrate({ inputBitrate, ratio: inputRatio, fps, resolution }) - - return { - outputOptions: [ - ...getCommonOutputOptions(targetBitrate, streamNum), - - `${buildStreamSuffix('-r:v', streamNum)} ${fps}`, - `${buildStreamSuffix('-b:v', streamNum)} ${targetBitrate}` - ] - } -} - -const defaultAACOptionsBuilder: EncoderOptionsBuilder = async ({ input, streamNum, canCopyAudio }) => { - const probe = await ffprobePromise(input) - - if (canCopyAudio && await canDoQuickAudioTranscode(input, probe)) { - return { copy: true, outputOptions: [ ] } - } - - const parsedAudio = await getAudioStream(input, probe) - - // We try to reduce the ceiling bitrate by making rough matches of bitrates - // Of course this is far from perfect, but it might save some space in the end - - const audioCodecName = parsedAudio.audioStream['codec_name'] - - const bitrate = getMaxAudioBitrate(audioCodecName, parsedAudio.bitrate) - - // Force stereo as it causes some issues with HLS playback in Chrome - const base = [ '-channel_layout', 'stereo' ] - - if (bitrate !== -1) { - return { outputOptions: base.concat([ buildStreamSuffix('-b:a', streamNum), bitrate + 'k' ]) } - } - - return { outputOptions: base } -} - -const defaultLibFDKAACVODOptionsBuilder: EncoderOptionsBuilder = ({ streamNum }) => { - return { outputOptions: [ buildStreamSuffix('-q:a', streamNum), '5' ] } -} - -export function getDefaultAvailableEncoders () { - return { - vod: { - libx264: { - default: defaultX264VODOptionsBuilder - }, - aac: { - default: defaultAACOptionsBuilder - }, - libfdk_aac: { - default: defaultLibFDKAACVODOptionsBuilder - } - }, - live: { - libx264: { - default: defaultX264LiveOptionsBuilder - }, - aac: { - default: defaultAACOptionsBuilder - } - } - } -} - -export function getDefaultEncodersToTry () { - return { - vod: { - video: [ 'libx264' ], - audio: [ 'libfdk_aac', 'aac' ] - }, - - live: { - video: [ 'libx264' ], - audio: [ 'libfdk_aac', 'aac' ] - } - } -} - -export async function canDoQuickAudioTranscode (path: string, probe?: FfprobeData): Promise { - const parsedAudio = await getAudioStream(path, probe) - - if (!parsedAudio.audioStream) return true - - if (parsedAudio.audioStream['codec_name'] !== 'aac') return false - - const audioBitrate = parsedAudio.bitrate - if (!audioBitrate) return false - - const maxAudioBitrate = getMaxAudioBitrate('aac', audioBitrate) - if (maxAudioBitrate !== -1 && audioBitrate > maxAudioBitrate) return false - - const channelLayout = parsedAudio.audioStream['channel_layout'] - // Causes playback issues with Chrome - if (!channelLayout || channelLayout === 'unknown' || channelLayout === 'quad') return false - - return true -} - -export async function canDoQuickVideoTranscode (path: string, probe?: FfprobeData): Promise { - const videoStream = await getVideoStream(path, probe) - const fps = await getVideoStreamFPS(path, probe) - const bitRate = await getVideoStreamBitrate(path, probe) - const resolutionData = await getVideoStreamDimensionsInfo(path, probe) - - // If ffprobe did not manage to guess the bitrate - if (!bitRate) return false - - // check video params - if (!videoStream) return false - if (videoStream['codec_name'] !== 'h264') return false - if (videoStream['pix_fmt'] !== 'yuv420p') return false - if (fps < 2 || fps > 65) return false - if (bitRate > getMaxTheoreticalBitrate({ ...resolutionData, fps })) return false - - return true -} - -// --------------------------------------------------------------------------- - -function getTargetBitrate (options: { - inputBitrate: number - resolution: VideoResolution - ratio: number - fps: number -}) { - const { inputBitrate, resolution, ratio, fps } = options - - const capped = capBitrate(inputBitrate, getAverageTheoreticalBitrate({ resolution, fps, ratio })) - const limit = getMinTheoreticalBitrate({ resolution, fps, ratio }) - - return Math.max(limit, capped) -} - -function capBitrate (inputBitrate: number, targetBitrate: number) { - if (!inputBitrate) return targetBitrate - - // Add 30% margin to input bitrate - const inputBitrateWithMargin = inputBitrate + (inputBitrate * 0.3) - - return Math.min(targetBitrate, inputBitrateWithMargin) -} - -function getCommonOutputOptions (targetBitrate: number, streamNum?: number) { - return [ - `-preset veryfast`, - `${buildStreamSuffix('-maxrate:v', streamNum)} ${targetBitrate}`, - `${buildStreamSuffix('-bufsize:v', streamNum)} ${targetBitrate * 2}`, - - // NOTE: b-strategy 1 - heuristic algorithm, 16 is optimal B-frames for it - `-b_strategy 1`, - // NOTE: Why 16: https://github.com/Chocobozzz/PeerTube/pull/774. b-strategy 2 -> B-frames<16 - `-bf 16` - ] -} diff --git a/shared/ffmpeg/ffmpeg-edition.ts b/shared/ffmpeg/ffmpeg-edition.ts deleted file mode 100644 index 724ca1ea9..000000000 --- a/shared/ffmpeg/ffmpeg-edition.ts +++ /dev/null @@ -1,239 +0,0 @@ -import { FilterSpecification } from 'fluent-ffmpeg' -import { FFmpegCommandWrapper, FFmpegCommandWrapperOptions } from './ffmpeg-command-wrapper' -import { presetVOD } from './shared/presets' -import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamDuration, getVideoStreamFPS, hasAudioStream } from './ffprobe' - -export class FFmpegEdition { - private readonly commandWrapper: FFmpegCommandWrapper - - constructor (options: FFmpegCommandWrapperOptions) { - this.commandWrapper = new FFmpegCommandWrapper(options) - } - - async cutVideo (options: { - inputPath: string - outputPath: string - start?: number - end?: number - }) { - const { inputPath, outputPath } = options - - const mainProbe = await ffprobePromise(inputPath) - const fps = await getVideoStreamFPS(inputPath, mainProbe) - const { resolution } = await getVideoStreamDimensionsInfo(inputPath, mainProbe) - - const command = this.commandWrapper.buildCommand(inputPath) - .output(outputPath) - - await presetVOD({ - commandWrapper: this.commandWrapper, - input: inputPath, - resolution, - fps, - canCopyAudio: false, - canCopyVideo: false - }) - - if (options.start) { - command.outputOption('-ss ' + options.start) - } - - if (options.end) { - command.outputOption('-to ' + options.end) - } - - await this.commandWrapper.runCommand() - } - - async addWatermark (options: { - inputPath: string - watermarkPath: string - outputPath: string - - videoFilters: { - watermarkSizeRatio: number - horitonzalMarginRatio: number - verticalMarginRatio: number - } - }) { - const { watermarkPath, inputPath, outputPath, videoFilters } = options - - const videoProbe = await ffprobePromise(inputPath) - const fps = await getVideoStreamFPS(inputPath, videoProbe) - const { resolution } = await getVideoStreamDimensionsInfo(inputPath, videoProbe) - - const command = this.commandWrapper.buildCommand(inputPath) - .output(outputPath) - - command.input(watermarkPath) - - await presetVOD({ - commandWrapper: this.commandWrapper, - input: inputPath, - resolution, - fps, - canCopyAudio: true, - canCopyVideo: false - }) - - const complexFilter: FilterSpecification[] = [ - // Scale watermark - { - inputs: [ '[1]', '[0]' ], - filter: 'scale2ref', - options: { - w: 'oh*mdar', - h: `ih*${videoFilters.watermarkSizeRatio}` - }, - outputs: [ '[watermark]', '[video]' ] - }, - - { - inputs: [ '[video]', '[watermark]' ], - filter: 'overlay', - options: { - x: `main_w - overlay_w - (main_h * ${videoFilters.horitonzalMarginRatio})`, - y: `main_h * ${videoFilters.verticalMarginRatio}` - } - } - ] - - command.complexFilter(complexFilter) - - await this.commandWrapper.runCommand() - } - - async addIntroOutro (options: { - inputPath: string - introOutroPath: string - outputPath: string - type: 'intro' | 'outro' - }) { - const { introOutroPath, inputPath, outputPath, type } = options - - const mainProbe = await ffprobePromise(inputPath) - const fps = await getVideoStreamFPS(inputPath, mainProbe) - const { resolution } = await getVideoStreamDimensionsInfo(inputPath, mainProbe) - const mainHasAudio = await hasAudioStream(inputPath, mainProbe) - - const introOutroProbe = await ffprobePromise(introOutroPath) - const introOutroHasAudio = await hasAudioStream(introOutroPath, introOutroProbe) - - const command = this.commandWrapper.buildCommand(inputPath) - .output(outputPath) - - command.input(introOutroPath) - - if (!introOutroHasAudio && mainHasAudio) { - const duration = await getVideoStreamDuration(introOutroPath, introOutroProbe) - - command.input('anullsrc') - command.withInputFormat('lavfi') - command.withInputOption('-t ' + duration) - } - - await presetVOD({ - commandWrapper: this.commandWrapper, - input: inputPath, - resolution, - fps, - canCopyAudio: false, - canCopyVideo: false - }) - - // Add black background to correctly scale intro/outro with padding - const complexFilter: FilterSpecification[] = [ - { - inputs: [ '1', '0' ], - filter: 'scale2ref', - options: { - w: 'iw', - h: `ih` - }, - outputs: [ 'intro-outro', 'main' ] - }, - { - inputs: [ 'intro-outro', 'main' ], - filter: 'scale2ref', - options: { - w: 'iw', - h: `ih` - }, - outputs: [ 'to-scale', 'main' ] - }, - { - inputs: 'to-scale', - filter: 'drawbox', - options: { - t: 'fill' - }, - outputs: [ 'to-scale-bg' ] - }, - { - inputs: [ '1', 'to-scale-bg' ], - filter: 'scale2ref', - options: { - w: 'iw', - h: 'ih', - force_original_aspect_ratio: 'decrease', - flags: 'spline' - }, - outputs: [ 'to-scale', 'to-scale-bg' ] - }, - { - inputs: [ 'to-scale-bg', 'to-scale' ], - filter: 'overlay', - options: { - x: '(main_w - overlay_w)/2', - y: '(main_h - overlay_h)/2' - }, - outputs: 'intro-outro-resized' - } - ] - - const concatFilter = { - inputs: [], - filter: 'concat', - options: { - n: 2, - v: 1, - unsafe: 1 - }, - outputs: [ 'v' ] - } - - const introOutroFilterInputs = [ 'intro-outro-resized' ] - const mainFilterInputs = [ 'main' ] - - if (mainHasAudio) { - mainFilterInputs.push('0:a') - - if (introOutroHasAudio) { - introOutroFilterInputs.push('1:a') - } else { - // Silent input - introOutroFilterInputs.push('2:a') - } - } - - if (type === 'intro') { - concatFilter.inputs = [ ...introOutroFilterInputs, ...mainFilterInputs ] - } else { - concatFilter.inputs = [ ...mainFilterInputs, ...introOutroFilterInputs ] - } - - if (mainHasAudio) { - concatFilter.options['a'] = 1 - concatFilter.outputs.push('a') - - command.outputOption('-map [a]') - } - - command.outputOption('-map [v]') - - complexFilter.push(concatFilter) - command.complexFilter(complexFilter) - - await this.commandWrapper.runCommand() - } -} diff --git a/shared/ffmpeg/ffmpeg-images.ts b/shared/ffmpeg/ffmpeg-images.ts deleted file mode 100644 index 618fac7d1..000000000 --- a/shared/ffmpeg/ffmpeg-images.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { FFmpegCommandWrapper, FFmpegCommandWrapperOptions } from './ffmpeg-command-wrapper' -import { getVideoStreamDuration } from './ffprobe' - -export class FFmpegImage { - private readonly commandWrapper: FFmpegCommandWrapper - - constructor (options: FFmpegCommandWrapperOptions) { - this.commandWrapper = new FFmpegCommandWrapper(options) - } - - convertWebPToJPG (options: { - path: string - destination: string - }): Promise { - const { path, destination } = options - - this.commandWrapper.buildCommand(path) - .output(destination) - - return this.commandWrapper.runCommand({ silent: true }) - } - - processGIF (options: { - path: string - destination: string - newSize: { width: number, height: number } - }): Promise { - const { path, destination, newSize } = options - - this.commandWrapper.buildCommand(path) - .fps(20) - .size(`${newSize.width}x${newSize.height}`) - .output(destination) - - return this.commandWrapper.runCommand() - } - - async generateThumbnailFromVideo (options: { - fromPath: string - output: string - }) { - const { fromPath, output } = options - - let duration = await getVideoStreamDuration(fromPath) - if (isNaN(duration)) duration = 0 - - this.commandWrapper.buildCommand(fromPath) - .seekInput(duration / 2) - .videoFilter('thumbnail=500') - .outputOption('-frames:v 1') - .output(output) - - return this.commandWrapper.runCommand() - } - - async generateStoryboardFromVideo (options: { - path: string - destination: string - - sprites: { - size: { - width: number - height: number - } - - count: { - width: number - height: number - } - - duration: number - } - }) { - const { path, destination, sprites } = options - - const command = this.commandWrapper.buildCommand(path) - - const filter = [ - `setpts=N/round(FRAME_RATE)/TB`, - `select='not(mod(t,${options.sprites.duration}))'`, - `scale=${sprites.size.width}:${sprites.size.height}`, - `tile=layout=${sprites.count.width}x${sprites.count.height}` - ].join(',') - - command.outputOption('-filter_complex', filter) - command.outputOption('-frames:v', '1') - command.outputOption('-q:v', '2') - command.output(destination) - - return this.commandWrapper.runCommand() - } -} diff --git a/shared/ffmpeg/ffmpeg-live.ts b/shared/ffmpeg/ffmpeg-live.ts deleted file mode 100644 index cca4c6474..000000000 --- a/shared/ffmpeg/ffmpeg-live.ts +++ /dev/null @@ -1,184 +0,0 @@ -import { FilterSpecification } from 'fluent-ffmpeg' -import { join } from 'path' -import { pick } from '@shared/core-utils' -import { FFmpegCommandWrapper, FFmpegCommandWrapperOptions } from './ffmpeg-command-wrapper' -import { buildStreamSuffix, getScaleFilter, StreamType } from './ffmpeg-utils' -import { addDefaultEncoderGlobalParams, addDefaultEncoderParams, applyEncoderOptions } from './shared' - -export class FFmpegLive { - private readonly commandWrapper: FFmpegCommandWrapper - - constructor (options: FFmpegCommandWrapperOptions) { - this.commandWrapper = new FFmpegCommandWrapper(options) - } - - async getLiveTranscodingCommand (options: { - inputUrl: string - - outPath: string - masterPlaylistName: string - - toTranscode: { - resolution: number - fps: number - }[] - - // Input information - bitrate: number - ratio: number - hasAudio: boolean - - segmentListSize: number - segmentDuration: number - }) { - const { - inputUrl, - outPath, - toTranscode, - bitrate, - masterPlaylistName, - ratio, - hasAudio - } = options - const command = this.commandWrapper.buildCommand(inputUrl) - - const varStreamMap: string[] = [] - - const complexFilter: FilterSpecification[] = [ - { - inputs: '[v:0]', - filter: 'split', - options: toTranscode.length, - outputs: toTranscode.map(t => `vtemp${t.resolution}`) - } - ] - - command.outputOption('-sc_threshold 0') - - addDefaultEncoderGlobalParams(command) - - for (let i = 0; i < toTranscode.length; i++) { - const streamMap: string[] = [] - const { resolution, fps } = toTranscode[i] - - const baseEncoderBuilderParams = { - input: inputUrl, - - canCopyAudio: true, - canCopyVideo: true, - - inputBitrate: bitrate, - inputRatio: ratio, - - resolution, - fps, - - streamNum: i, - videoType: 'live' as 'live' - } - - { - const streamType: StreamType = 'video' - const builderResult = await this.commandWrapper.getEncoderBuilderResult({ ...baseEncoderBuilderParams, streamType }) - if (!builderResult) { - throw new Error('No available live video encoder found') - } - - command.outputOption(`-map [vout${resolution}]`) - - addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps, streamNum: i }) - - this.commandWrapper.debugLog( - `Apply ffmpeg live video params from ${builderResult.encoder} using ${this.commandWrapper.getProfile()} profile.`, - { builderResult, fps, toTranscode } - ) - - command.outputOption(`${buildStreamSuffix('-c:v', i)} ${builderResult.encoder}`) - applyEncoderOptions(command, builderResult.result) - - complexFilter.push({ - inputs: `vtemp${resolution}`, - filter: getScaleFilter(builderResult.result), - options: `w=-2:h=${resolution}`, - outputs: `vout${resolution}` - }) - - streamMap.push(`v:${i}`) - } - - if (hasAudio) { - const streamType: StreamType = 'audio' - const builderResult = await this.commandWrapper.getEncoderBuilderResult({ ...baseEncoderBuilderParams, streamType }) - if (!builderResult) { - throw new Error('No available live audio encoder found') - } - - command.outputOption('-map a:0') - - addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps, streamNum: i }) - - this.commandWrapper.debugLog( - `Apply ffmpeg live audio params from ${builderResult.encoder} using ${this.commandWrapper.getProfile()} profile.`, - { builderResult, fps, resolution } - ) - - command.outputOption(`${buildStreamSuffix('-c:a', i)} ${builderResult.encoder}`) - applyEncoderOptions(command, builderResult.result) - - streamMap.push(`a:${i}`) - } - - varStreamMap.push(streamMap.join(',')) - } - - command.complexFilter(complexFilter) - - this.addDefaultLiveHLSParams({ ...pick(options, [ 'segmentDuration', 'segmentListSize' ]), outPath, masterPlaylistName }) - - command.outputOption('-var_stream_map', varStreamMap.join(' ')) - - return command - } - - getLiveMuxingCommand (options: { - inputUrl: string - outPath: string - masterPlaylistName: string - - segmentListSize: number - segmentDuration: number - }) { - const { inputUrl, outPath, masterPlaylistName } = options - - const command = this.commandWrapper.buildCommand(inputUrl) - - command.outputOption('-c:v copy') - command.outputOption('-c:a copy') - command.outputOption('-map 0:a?') - command.outputOption('-map 0:v?') - - this.addDefaultLiveHLSParams({ ...pick(options, [ 'segmentDuration', 'segmentListSize' ]), outPath, masterPlaylistName }) - - return command - } - - private addDefaultLiveHLSParams (options: { - outPath: string - masterPlaylistName: string - segmentListSize: number - segmentDuration: number - }) { - const { outPath, masterPlaylistName, segmentListSize, segmentDuration } = options - - const command = this.commandWrapper.getCommand() - - command.outputOption('-hls_time ' + segmentDuration) - command.outputOption('-hls_list_size ' + segmentListSize) - command.outputOption('-hls_flags delete_segments+independent_segments+program_date_time') - command.outputOption(`-hls_segment_filename ${join(outPath, '%v-%06d.ts')}`) - command.outputOption('-master_pl_name ' + masterPlaylistName) - command.outputOption(`-f hls`) - - command.output(join(outPath, '%v.m3u8')) - } -} diff --git a/shared/ffmpeg/ffmpeg-utils.ts b/shared/ffmpeg/ffmpeg-utils.ts deleted file mode 100644 index 7d09c32ca..000000000 --- a/shared/ffmpeg/ffmpeg-utils.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { EncoderOptions } from '@shared/models' - -export type StreamType = 'audio' | 'video' - -export function buildStreamSuffix (base: string, streamNum?: number) { - if (streamNum !== undefined) { - return `${base}:${streamNum}` - } - - return base -} - -export function getScaleFilter (options: EncoderOptions): string { - if (options.scaleFilter) return options.scaleFilter.name - - return 'scale' -} diff --git a/shared/ffmpeg/ffmpeg-vod.ts b/shared/ffmpeg/ffmpeg-vod.ts deleted file mode 100644 index e40ca0a1e..000000000 --- a/shared/ffmpeg/ffmpeg-vod.ts +++ /dev/null @@ -1,256 +0,0 @@ -import { MutexInterface } from 'async-mutex' -import { FfmpegCommand } from 'fluent-ffmpeg' -import { readFile, writeFile } from 'fs-extra' -import { dirname } from 'path' -import { pick } from '@shared/core-utils' -import { VideoResolution } from '@shared/models' -import { FFmpegCommandWrapper, FFmpegCommandWrapperOptions } from './ffmpeg-command-wrapper' -import { ffprobePromise, getVideoStreamDimensionsInfo } from './ffprobe' -import { presetCopy, presetOnlyAudio, presetVOD } from './shared/presets' - -export type TranscodeVODOptionsType = 'hls' | 'hls-from-ts' | 'quick-transcode' | 'video' | 'merge-audio' | 'only-audio' - -export interface BaseTranscodeVODOptions { - type: TranscodeVODOptionsType - - inputPath: string - outputPath: string - - // Will be released after the ffmpeg started - // To prevent a bug where the input file does not exist anymore when running ffmpeg - inputFileMutexReleaser: MutexInterface.Releaser - - resolution: number - fps: number -} - -export interface HLSTranscodeOptions extends BaseTranscodeVODOptions { - type: 'hls' - - copyCodecs: boolean - - hlsPlaylist: { - videoFilename: string - } -} - -export interface HLSFromTSTranscodeOptions extends BaseTranscodeVODOptions { - type: 'hls-from-ts' - - isAAC: boolean - - hlsPlaylist: { - videoFilename: string - } -} - -export interface QuickTranscodeOptions extends BaseTranscodeVODOptions { - type: 'quick-transcode' -} - -export interface VideoTranscodeOptions extends BaseTranscodeVODOptions { - type: 'video' -} - -export interface MergeAudioTranscodeOptions extends BaseTranscodeVODOptions { - type: 'merge-audio' - audioPath: string -} - -export interface OnlyAudioTranscodeOptions extends BaseTranscodeVODOptions { - type: 'only-audio' -} - -export type TranscodeVODOptions = - HLSTranscodeOptions - | HLSFromTSTranscodeOptions - | VideoTranscodeOptions - | MergeAudioTranscodeOptions - | OnlyAudioTranscodeOptions - | QuickTranscodeOptions - -// --------------------------------------------------------------------------- - -export class FFmpegVOD { - private readonly commandWrapper: FFmpegCommandWrapper - - private ended = false - - constructor (options: FFmpegCommandWrapperOptions) { - this.commandWrapper = new FFmpegCommandWrapper(options) - } - - async transcode (options: TranscodeVODOptions) { - const builders: { - [ type in TranscodeVODOptionsType ]: (options: TranscodeVODOptions) => Promise | void - } = { - 'quick-transcode': this.buildQuickTranscodeCommand.bind(this), - 'hls': this.buildHLSVODCommand.bind(this), - 'hls-from-ts': this.buildHLSVODFromTSCommand.bind(this), - 'merge-audio': this.buildAudioMergeCommand.bind(this), - // TODO: remove, we merge this in buildWebVideoCommand - 'only-audio': this.buildOnlyAudioCommand.bind(this), - 'video': this.buildWebVideoCommand.bind(this) - } - - this.commandWrapper.debugLog('Will run transcode.', { options }) - - const command = this.commandWrapper.buildCommand(options.inputPath) - .output(options.outputPath) - - await builders[options.type](options) - - command.on('start', () => { - setTimeout(() => { - options.inputFileMutexReleaser() - }, 1000) - }) - - await this.commandWrapper.runCommand() - - await this.fixHLSPlaylistIfNeeded(options) - - this.ended = true - } - - isEnded () { - return this.ended - } - - private async buildWebVideoCommand (options: TranscodeVODOptions) { - const { resolution, fps, inputPath } = options - - if (resolution === VideoResolution.H_NOVIDEO) { - presetOnlyAudio(this.commandWrapper) - return - } - - let scaleFilterValue: string - - if (resolution !== undefined) { - const probe = await ffprobePromise(inputPath) - const videoStreamInfo = await getVideoStreamDimensionsInfo(inputPath, probe) - - scaleFilterValue = videoStreamInfo?.isPortraitMode === true - ? `w=${resolution}:h=-2` - : `w=-2:h=${resolution}` - } - - await presetVOD({ - commandWrapper: this.commandWrapper, - - resolution, - input: inputPath, - canCopyAudio: true, - canCopyVideo: true, - fps, - scaleFilterValue - }) - } - - private buildQuickTranscodeCommand (_options: TranscodeVODOptions) { - const command = this.commandWrapper.getCommand() - - presetCopy(this.commandWrapper) - - command.outputOption('-map_metadata -1') // strip all metadata - .outputOption('-movflags faststart') - } - - // --------------------------------------------------------------------------- - // Audio transcoding - // --------------------------------------------------------------------------- - - private async buildAudioMergeCommand (options: MergeAudioTranscodeOptions) { - const command = this.commandWrapper.getCommand() - - command.loop(undefined) - - await presetVOD({ - ...pick(options, [ 'resolution' ]), - - commandWrapper: this.commandWrapper, - input: options.audioPath, - canCopyAudio: true, - canCopyVideo: true, - fps: options.fps, - scaleFilterValue: this.getMergeAudioScaleFilterValue() - }) - - command.outputOption('-preset:v veryfast') - - command.input(options.audioPath) - .outputOption('-tune stillimage') - .outputOption('-shortest') - } - - private buildOnlyAudioCommand (_options: OnlyAudioTranscodeOptions) { - presetOnlyAudio(this.commandWrapper) - } - - // Avoid "height not divisible by 2" error - private getMergeAudioScaleFilterValue () { - return 'trunc(iw/2)*2:trunc(ih/2)*2' - } - - // --------------------------------------------------------------------------- - // HLS transcoding - // --------------------------------------------------------------------------- - - private async buildHLSVODCommand (options: HLSTranscodeOptions) { - const command = this.commandWrapper.getCommand() - - const videoPath = this.getHLSVideoPath(options) - - if (options.copyCodecs) presetCopy(this.commandWrapper) - else if (options.resolution === VideoResolution.H_NOVIDEO) presetOnlyAudio(this.commandWrapper) - else await this.buildWebVideoCommand(options) - - this.addCommonHLSVODCommandOptions(command, videoPath) - } - - private buildHLSVODFromTSCommand (options: HLSFromTSTranscodeOptions) { - const command = this.commandWrapper.getCommand() - - const videoPath = this.getHLSVideoPath(options) - - command.outputOption('-c copy') - - if (options.isAAC) { - // Required for example when copying an AAC stream from an MPEG-TS - // Since it's a bitstream filter, we don't need to reencode the audio - command.outputOption('-bsf:a aac_adtstoasc') - } - - this.addCommonHLSVODCommandOptions(command, videoPath) - } - - private addCommonHLSVODCommandOptions (command: FfmpegCommand, outputPath: string) { - return command.outputOption('-hls_time 4') - .outputOption('-hls_list_size 0') - .outputOption('-hls_playlist_type vod') - .outputOption('-hls_segment_filename ' + outputPath) - .outputOption('-hls_segment_type fmp4') - .outputOption('-f hls') - .outputOption('-hls_flags single_file') - } - - private async fixHLSPlaylistIfNeeded (options: TranscodeVODOptions) { - if (options.type !== 'hls' && options.type !== 'hls-from-ts') return - - const fileContent = await readFile(options.outputPath) - - const videoFileName = options.hlsPlaylist.videoFilename - const videoFilePath = this.getHLSVideoPath(options) - - // Fix wrong mapping with some ffmpeg versions - const newContent = fileContent.toString() - .replace(`#EXT-X-MAP:URI="${videoFilePath}",`, `#EXT-X-MAP:URI="${videoFileName}",`) - - await writeFile(options.outputPath, newContent) - } - - private getHLSVideoPath (options: HLSTranscodeOptions | HLSFromTSTranscodeOptions) { - return `${dirname(options.outputPath)}/${options.hlsPlaylist.videoFilename}` - } -} diff --git a/shared/ffmpeg/ffprobe.ts b/shared/ffmpeg/ffprobe.ts deleted file mode 100644 index fda08c28e..000000000 --- a/shared/ffmpeg/ffprobe.ts +++ /dev/null @@ -1,184 +0,0 @@ -import { ffprobe, FfprobeData } from 'fluent-ffmpeg' -import { forceNumber } from '@shared/core-utils' -import { VideoResolution } from '@shared/models/videos' - -/** - * - * Helpers to run ffprobe and extract data from the JSON output - * - */ - -function ffprobePromise (path: string) { - return new Promise((res, rej) => { - ffprobe(path, (err, data) => { - if (err) return rej(err) - - return res(data) - }) - }) -} - -// --------------------------------------------------------------------------- -// Audio -// --------------------------------------------------------------------------- - -const imageCodecs = new Set([ - 'ansi', 'apng', 'bintext', 'bmp', 'brender_pix', 'dpx', 'exr', 'fits', 'gem', 'gif', 'jpeg2000', 'jpgls', 'mjpeg', 'mjpegb', 'msp2', - 'pam', 'pbm', 'pcx', 'pfm', 'pgm', 'pgmyuv', 'pgx', 'photocd', 'pictor', 'png', 'ppm', 'psd', 'sgi', 'sunrast', 'svg', 'targa', 'tiff', - 'txd', 'webp', 'xbin', 'xbm', 'xface', 'xpm', 'xwd' -]) - -async function isAudioFile (path: string, existingProbe?: FfprobeData) { - const videoStream = await getVideoStream(path, existingProbe) - if (!videoStream) return true - - if (imageCodecs.has(videoStream.codec_name)) return true - - return false -} - -async function hasAudioStream (path: string, existingProbe?: FfprobeData) { - const { audioStream } = await getAudioStream(path, existingProbe) - - return !!audioStream -} - -async function getAudioStream (videoPath: string, existingProbe?: FfprobeData) { - // without position, ffprobe considers the last input only - // we make it consider the first input only - // if you pass a file path to pos, then ffprobe acts on that file directly - const data = existingProbe || await ffprobePromise(videoPath) - - if (Array.isArray(data.streams)) { - const audioStream = data.streams.find(stream => stream['codec_type'] === 'audio') - - if (audioStream) { - return { - absolutePath: data.format.filename, - audioStream, - bitrate: forceNumber(audioStream['bit_rate']) - } - } - } - - return { absolutePath: data.format.filename } -} - -function getMaxAudioBitrate (type: 'aac' | 'mp3' | string, bitrate: number) { - const maxKBitrate = 384 - const kToBits = (kbits: number) => kbits * 1000 - - // If we did not manage to get the bitrate, use an average value - if (!bitrate) return 256 - - if (type === 'aac') { - switch (true) { - case bitrate > kToBits(maxKBitrate): - return maxKBitrate - - default: - return -1 // we interpret it as a signal to copy the audio stream as is - } - } - - /* - a 192kbit/sec mp3 doesn't hold as much information as a 192kbit/sec aac. - That's why, when using aac, we can go to lower kbit/sec. The equivalences - made here are not made to be accurate, especially with good mp3 encoders. - */ - switch (true) { - case bitrate <= kToBits(192): - return 128 - - case bitrate <= kToBits(384): - return 256 - - default: - return maxKBitrate - } -} - -// --------------------------------------------------------------------------- -// Video -// --------------------------------------------------------------------------- - -async function getVideoStreamDimensionsInfo (path: string, existingProbe?: FfprobeData) { - const videoStream = await getVideoStream(path, existingProbe) - if (!videoStream) { - return { - width: 0, - height: 0, - ratio: 0, - resolution: VideoResolution.H_NOVIDEO, - isPortraitMode: false - } - } - - return { - width: videoStream.width, - height: videoStream.height, - ratio: Math.max(videoStream.height, videoStream.width) / Math.min(videoStream.height, videoStream.width), - resolution: Math.min(videoStream.height, videoStream.width), - isPortraitMode: videoStream.height > videoStream.width - } -} - -async function getVideoStreamFPS (path: string, existingProbe?: FfprobeData) { - const videoStream = await getVideoStream(path, existingProbe) - if (!videoStream) return 0 - - for (const key of [ 'avg_frame_rate', 'r_frame_rate' ]) { - const valuesText: string = videoStream[key] - if (!valuesText) continue - - const [ frames, seconds ] = valuesText.split('/') - if (!frames || !seconds) continue - - const result = parseInt(frames, 10) / parseInt(seconds, 10) - if (result > 0) return Math.round(result) - } - - return 0 -} - -async function getVideoStreamBitrate (path: string, existingProbe?: FfprobeData): Promise { - const metadata = existingProbe || await ffprobePromise(path) - - let bitrate = metadata.format.bit_rate - if (bitrate && !isNaN(bitrate)) return bitrate - - const videoStream = await getVideoStream(path, existingProbe) - if (!videoStream) return undefined - - bitrate = forceNumber(videoStream?.bit_rate) - if (bitrate && !isNaN(bitrate)) return bitrate - - return undefined -} - -async function getVideoStreamDuration (path: string, existingProbe?: FfprobeData) { - const metadata = existingProbe || await ffprobePromise(path) - - return Math.round(metadata.format.duration) -} - -async function getVideoStream (path: string, existingProbe?: FfprobeData) { - const metadata = existingProbe || await ffprobePromise(path) - - return metadata.streams.find(s => s.codec_type === 'video') -} - -// --------------------------------------------------------------------------- - -export { - getVideoStreamDimensionsInfo, - getMaxAudioBitrate, - getVideoStream, - getVideoStreamDuration, - getAudioStream, - getVideoStreamFPS, - isAudioFile, - ffprobePromise, - getVideoStreamBitrate, - hasAudioStream -} diff --git a/shared/ffmpeg/index.ts b/shared/ffmpeg/index.ts deleted file mode 100644 index 1dab292da..000000000 --- a/shared/ffmpeg/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -export * from './ffmpeg-command-wrapper' -export * from './ffmpeg-default-transcoding-profile' -export * from './ffmpeg-edition' -export * from './ffmpeg-images' -export * from './ffmpeg-live' -export * from './ffmpeg-utils' -export * from './ffmpeg-version' -export * from './ffmpeg-vod' -export * from './ffprobe' diff --git a/shared/ffmpeg/shared/encoder-options.ts b/shared/ffmpeg/shared/encoder-options.ts deleted file mode 100644 index 9692a6b02..000000000 --- a/shared/ffmpeg/shared/encoder-options.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { FfmpegCommand } from 'fluent-ffmpeg' -import { EncoderOptions } from '@shared/models' -import { buildStreamSuffix } from '../ffmpeg-utils' - -export function addDefaultEncoderGlobalParams (command: FfmpegCommand) { - // avoid issues when transcoding some files: https://trac.ffmpeg.org/ticket/6375 - command.outputOption('-max_muxing_queue_size 1024') - // strip all metadata - .outputOption('-map_metadata -1') - // allows import of source material with incompatible pixel formats (e.g. MJPEG video) - .outputOption('-pix_fmt yuv420p') -} - -export function addDefaultEncoderParams (options: { - command: FfmpegCommand - encoder: 'libx264' | string - fps: number - - streamNum?: number -}) { - const { command, encoder, fps, streamNum } = options - - if (encoder === 'libx264') { - // 3.1 is the minimal resource allocation for our highest supported resolution - command.outputOption(buildStreamSuffix('-level:v', streamNum) + ' 3.1') - - if (fps) { - // Keyframe interval of 2 seconds for faster seeking and resolution switching. - // https://streaminglearningcenter.com/blogs/whats-the-right-keyframe-interval.html - // https://superuser.com/a/908325 - command.outputOption(buildStreamSuffix('-g:v', streamNum) + ' ' + (fps * 2)) - } - } -} - -export function applyEncoderOptions (command: FfmpegCommand, options: EncoderOptions) { - command.inputOptions(options.inputOptions ?? []) - .outputOptions(options.outputOptions ?? []) -} diff --git a/shared/ffmpeg/shared/index.ts b/shared/ffmpeg/shared/index.ts deleted file mode 100644 index 51de0316f..000000000 --- a/shared/ffmpeg/shared/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './encoder-options' -export * from './presets' diff --git a/shared/ffmpeg/shared/presets.ts b/shared/ffmpeg/shared/presets.ts deleted file mode 100644 index dcebdc1cf..000000000 --- a/shared/ffmpeg/shared/presets.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { pick } from '@shared/core-utils' -import { FFmpegCommandWrapper } from '../ffmpeg-command-wrapper' -import { ffprobePromise, getVideoStreamBitrate, getVideoStreamDimensionsInfo, hasAudioStream } from '../ffprobe' -import { addDefaultEncoderGlobalParams, addDefaultEncoderParams, applyEncoderOptions } from './encoder-options' -import { getScaleFilter, StreamType } from '../ffmpeg-utils' - -export async function presetVOD (options: { - commandWrapper: FFmpegCommandWrapper - - input: string - - canCopyAudio: boolean - canCopyVideo: boolean - - resolution: number - fps: number - - scaleFilterValue?: string -}) { - const { commandWrapper, input, resolution, fps, scaleFilterValue } = options - const command = commandWrapper.getCommand() - - command.format('mp4') - .outputOption('-movflags faststart') - - addDefaultEncoderGlobalParams(command) - - const probe = await ffprobePromise(input) - - // Audio encoder - const bitrate = await getVideoStreamBitrate(input, probe) - const videoStreamDimensions = await getVideoStreamDimensionsInfo(input, probe) - - let streamsToProcess: StreamType[] = [ 'audio', 'video' ] - - if (!await hasAudioStream(input, probe)) { - command.noAudio() - streamsToProcess = [ 'video' ] - } - - for (const streamType of streamsToProcess) { - const builderResult = await commandWrapper.getEncoderBuilderResult({ - ...pick(options, [ 'canCopyAudio', 'canCopyVideo' ]), - - input, - inputBitrate: bitrate, - inputRatio: videoStreamDimensions?.ratio || 0, - - resolution, - fps, - streamType, - - videoType: 'vod' as 'vod' - }) - - if (!builderResult) { - throw new Error('No available encoder found for stream ' + streamType) - } - - commandWrapper.debugLog( - `Apply ffmpeg params from ${builderResult.encoder} for ${streamType} ` + - `stream of input ${input} using ${commandWrapper.getProfile()} profile.`, - { builderResult, resolution, fps } - ) - - if (streamType === 'video') { - command.videoCodec(builderResult.encoder) - - if (scaleFilterValue) { - command.outputOption(`-vf ${getScaleFilter(builderResult.result)}=${scaleFilterValue}`) - } - } else if (streamType === 'audio') { - command.audioCodec(builderResult.encoder) - } - - applyEncoderOptions(command, builderResult.result) - addDefaultEncoderParams({ command, encoder: builderResult.encoder, fps }) - } -} - -export function presetCopy (commandWrapper: FFmpegCommandWrapper) { - commandWrapper.getCommand() - .format('mp4') - .videoCodec('copy') - .audioCodec('copy') -} - -export function presetOnlyAudio (commandWrapper: FFmpegCommandWrapper) { - commandWrapper.getCommand() - .format('mp4') - .audioCodec('copy') - .noVideo() -} diff --git a/shared/models/activitypub/activity.ts b/shared/models/activitypub/activity.ts deleted file mode 100644 index 10cf53ead..000000000 --- a/shared/models/activitypub/activity.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { ActivityPubActor } from './activitypub-actor' -import { ActivityPubSignature } from './activitypub-signature' -import { - ActivityFlagReasonObject, - ActivityObject, - APObjectId, - CacheFileObject, - PlaylistObject, - VideoCommentObject, - VideoObject, - WatchActionObject -} from './objects' - -export type ActivityUpdateObject = - Extract | ActivityPubActor - -// Cannot Extract from Activity because of circular reference -export type ActivityUndoObject = - ActivityFollow | ActivityLike | ActivityDislike | ActivityCreate | ActivityAnnounce - -export type ActivityCreateObject = - Extract - -export type Activity = - ActivityCreate | - ActivityUpdate | - ActivityDelete | - ActivityFollow | - ActivityAccept | - ActivityAnnounce | - ActivityUndo | - ActivityLike | - ActivityReject | - ActivityView | - ActivityDislike | - ActivityFlag - -export type ActivityType = - 'Create' | - 'Update' | - 'Delete' | - 'Follow' | - 'Accept' | - 'Announce' | - 'Undo' | - 'Like' | - 'Reject' | - 'View' | - 'Dislike' | - 'Flag' - -export interface ActivityAudience { - to: string[] - cc: string[] -} - -export interface BaseActivity { - '@context'?: any[] - id: string - to?: string[] - cc?: string[] - actor: string | ActivityPubActor - type: ActivityType - signature?: ActivityPubSignature -} - -export interface ActivityCreate extends BaseActivity { - type: 'Create' - object: T -} - -export interface ActivityUpdate extends BaseActivity { - type: 'Update' - object: T -} - -export interface ActivityDelete extends BaseActivity { - type: 'Delete' - object: APObjectId -} - -export interface ActivityFollow extends BaseActivity { - type: 'Follow' - object: string -} - -export interface ActivityAccept extends BaseActivity { - type: 'Accept' - object: ActivityFollow -} - -export interface ActivityReject extends BaseActivity { - type: 'Reject' - object: ActivityFollow -} - -export interface ActivityAnnounce extends BaseActivity { - type: 'Announce' - object: APObjectId -} - -export interface ActivityUndo extends BaseActivity { - type: 'Undo' - object: T -} - -export interface ActivityLike extends BaseActivity { - type: 'Like' - object: APObjectId -} - -export interface ActivityView extends BaseActivity { - type: 'View' - actor: string - object: APObjectId - - // If sending a "viewer" event - expires?: string -} - -export interface ActivityDislike extends BaseActivity { - id: string - type: 'Dislike' - actor: string - object: APObjectId -} - -export interface ActivityFlag extends BaseActivity { - type: 'Flag' - content: string - object: APObjectId | APObjectId[] - tag?: ActivityFlagReasonObject[] - startAt?: number - endAt?: number -} diff --git a/shared/models/activitypub/activitypub-actor.ts b/shared/models/activitypub/activitypub-actor.ts deleted file mode 100644 index b86bcb764..000000000 --- a/shared/models/activitypub/activitypub-actor.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { ActivityIconObject, ActivityPubAttributedTo } from './objects/common-objects' - -export type ActivityPubActorType = 'Person' | 'Application' | 'Group' | 'Service' | 'Organization' - -export interface ActivityPubActor { - '@context': any[] - type: ActivityPubActorType - id: string - following: string - followers: string - playlists?: string - inbox: string - outbox: string - preferredUsername: string - url: string - name: string - endpoints: { - sharedInbox: string - } - summary: string - attributedTo: ActivityPubAttributedTo[] - - support?: string - publicKey: { - id: string - owner: string - publicKeyPem: string - } - - image?: ActivityIconObject | ActivityIconObject[] - icon?: ActivityIconObject | ActivityIconObject[] - - published?: string -} diff --git a/shared/models/activitypub/activitypub-collection.ts b/shared/models/activitypub/activitypub-collection.ts deleted file mode 100644 index 60a6a6b04..000000000 --- a/shared/models/activitypub/activitypub-collection.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { Activity } from './activity' - -export interface ActivityPubCollection { - '@context': string[] - type: 'Collection' | 'CollectionPage' - totalItems: number - partOf?: string - items: Activity[] -} diff --git a/shared/models/activitypub/activitypub-root.ts b/shared/models/activitypub/activitypub-root.ts deleted file mode 100644 index 22dff83a2..000000000 --- a/shared/models/activitypub/activitypub-root.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { Activity } from './activity' -import { ActivityPubCollection } from './activitypub-collection' -import { ActivityPubOrderedCollection } from './activitypub-ordered-collection' - -export type RootActivity = Activity | ActivityPubCollection | ActivityPubOrderedCollection diff --git a/shared/models/activitypub/index.ts b/shared/models/activitypub/index.ts deleted file mode 100644 index fa07b6a64..000000000 --- a/shared/models/activitypub/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -export * from './objects' -export * from './activity' -export * from './activitypub-actor' -export * from './activitypub-collection' -export * from './activitypub-ordered-collection' -export * from './activitypub-root' -export * from './activitypub-signature' -export * from './context' -export * from './webfinger' diff --git a/shared/models/activitypub/objects/abuse-object.ts b/shared/models/activitypub/objects/abuse-object.ts deleted file mode 100644 index d938b8693..000000000 --- a/shared/models/activitypub/objects/abuse-object.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { ActivityFlagReasonObject } from './common-objects' - -export interface AbuseObject { - type: 'Flag' - - content: string - mediaType: 'text/markdown' - - object: string | string[] - - tag?: ActivityFlagReasonObject[] - - startAt?: number - endAt?: number -} diff --git a/shared/models/activitypub/objects/activitypub-object.ts b/shared/models/activitypub/objects/activitypub-object.ts deleted file mode 100644 index faeac2618..000000000 --- a/shared/models/activitypub/objects/activitypub-object.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { AbuseObject } from './abuse-object' -import { CacheFileObject } from './cache-file-object' -import { PlaylistObject } from './playlist-object' -import { VideoCommentObject } from './video-comment-object' -import { VideoObject } from './video-object' -import { WatchActionObject } from './watch-action-object' - -export type ActivityObject = - VideoObject | - AbuseObject | - VideoCommentObject | - CacheFileObject | - PlaylistObject | - WatchActionObject | - string - -export type APObjectId = string | { id: string } diff --git a/shared/models/activitypub/objects/cache-file-object.ts b/shared/models/activitypub/objects/cache-file-object.ts deleted file mode 100644 index 19a817582..000000000 --- a/shared/models/activitypub/objects/cache-file-object.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { ActivityVideoUrlObject, ActivityPlaylistUrlObject } from './common-objects' - -export interface CacheFileObject { - id: string - type: 'CacheFile' - object: string - expires: string - url: ActivityVideoUrlObject | ActivityPlaylistUrlObject -} diff --git a/shared/models/activitypub/objects/common-objects.ts b/shared/models/activitypub/objects/common-objects.ts deleted file mode 100644 index db9c73658..000000000 --- a/shared/models/activitypub/objects/common-objects.ts +++ /dev/null @@ -1,130 +0,0 @@ -import { AbusePredefinedReasonsString } from '../../moderation/abuse/abuse-reason.model' - -export interface ActivityIdentifierObject { - identifier: string - name: string - url?: string -} - -export interface ActivityIconObject { - type: 'Image' - url: string - mediaType: string - width?: number - height?: number -} - -export type ActivityVideoUrlObject = { - type: 'Link' - mediaType: 'video/mp4' | 'video/webm' | 'video/ogg' - href: string - height: number - size: number - fps: number -} - -export type ActivityPlaylistSegmentHashesObject = { - type: 'Link' - name: 'sha256' - mediaType: 'application/json' - href: string -} - -export type ActivityVideoFileMetadataUrlObject = { - type: 'Link' - rel: [ 'metadata', any ] - mediaType: 'application/json' - height: number - href: string - fps: number -} - -export type ActivityTrackerUrlObject = { - type: 'Link' - rel: [ 'tracker', 'websocket' | 'http' ] - name: string - href: string -} - -export type ActivityStreamingPlaylistInfohashesObject = { - type: 'Infohash' - name: string -} - -export type ActivityPlaylistUrlObject = { - type: 'Link' - mediaType: 'application/x-mpegURL' - href: string - tag?: ActivityTagObject[] -} - -export type ActivityBitTorrentUrlObject = { - type: 'Link' - mediaType: 'application/x-bittorrent' | 'application/x-bittorrent;x-scheme-handler/magnet' - href: string - height: number -} - -export type ActivityMagnetUrlObject = { - type: 'Link' - mediaType: 'application/x-bittorrent;x-scheme-handler/magnet' - href: string - height: number -} - -export type ActivityHtmlUrlObject = { - type: 'Link' - mediaType: 'text/html' - href: string -} - -export interface ActivityHashTagObject { - type: 'Hashtag' - href?: string - name: string -} - -export interface ActivityMentionObject { - type: 'Mention' - href?: string - name: string -} - -export interface ActivityFlagReasonObject { - type: 'Hashtag' - name: AbusePredefinedReasonsString -} - -export type ActivityTagObject = - ActivityPlaylistSegmentHashesObject - | ActivityStreamingPlaylistInfohashesObject - | ActivityVideoUrlObject - | ActivityHashTagObject - | ActivityMentionObject - | ActivityBitTorrentUrlObject - | ActivityMagnetUrlObject - | ActivityVideoFileMetadataUrlObject - -export type ActivityUrlObject = - ActivityVideoUrlObject - | ActivityPlaylistUrlObject - | ActivityBitTorrentUrlObject - | ActivityMagnetUrlObject - | ActivityHtmlUrlObject - | ActivityVideoFileMetadataUrlObject - | ActivityTrackerUrlObject - -export type ActivityPubAttributedTo = { type: 'Group' | 'Person', id: string } | string - -export interface ActivityTombstoneObject { - '@context'?: any - id: string - url?: string - type: 'Tombstone' - name?: string - formerType?: string - inReplyTo?: string - published: string - updated: string - deleted: string -} diff --git a/shared/models/activitypub/objects/index.ts b/shared/models/activitypub/objects/index.ts deleted file mode 100644 index 753e02003..000000000 --- a/shared/models/activitypub/objects/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -export * from './abuse-object' -export * from './activitypub-object' -export * from './cache-file-object' -export * from './common-objects' -export * from './playlist-element-object' -export * from './playlist-object' -export * from './video-comment-object' -export * from './video-object' -export * from './watch-action-object' diff --git a/shared/models/activitypub/objects/playlist-object.ts b/shared/models/activitypub/objects/playlist-object.ts deleted file mode 100644 index 0ccb71828..000000000 --- a/shared/models/activitypub/objects/playlist-object.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { ActivityIconObject, ActivityPubAttributedTo } from './common-objects' - -export interface PlaylistObject { - id: string - type: 'Playlist' - - name: string - - content: string - mediaType: 'text/markdown' - - uuid: string - - totalItems: number - attributedTo: ActivityPubAttributedTo[] - - icon?: ActivityIconObject - - published: string - updated: string - - orderedItems?: string[] - - partOf?: string - next?: string - first?: string - - to?: string[] -} diff --git a/shared/models/activitypub/objects/video-comment-object.ts b/shared/models/activitypub/objects/video-comment-object.ts deleted file mode 100644 index fb1e6f8db..000000000 --- a/shared/models/activitypub/objects/video-comment-object.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { ActivityPubAttributedTo, ActivityTagObject } from './common-objects' - -export interface VideoCommentObject { - type: 'Note' - id: string - - content: string - mediaType: 'text/markdown' - - inReplyTo: string - published: string - updated: string - url: string - attributedTo: ActivityPubAttributedTo - tag: ActivityTagObject[] -} diff --git a/shared/models/activitypub/objects/video-object.ts b/shared/models/activitypub/objects/video-object.ts deleted file mode 100644 index 409504f0f..000000000 --- a/shared/models/activitypub/objects/video-object.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { - ActivityIconObject, - ActivityIdentifierObject, - ActivityPubAttributedTo, - ActivityTagObject, - ActivityUrlObject -} from './common-objects' -import { LiveVideoLatencyMode, VideoState } from '../../videos' - -export interface VideoObject { - type: 'Video' - id: string - name: string - duration: string - uuid: string - tag: ActivityTagObject[] - category: ActivityIdentifierObject - licence: ActivityIdentifierObject - language: ActivityIdentifierObject - subtitleLanguage: ActivityIdentifierObject[] - views: number - - sensitive: boolean - - isLiveBroadcast: boolean - liveSaveReplay: boolean - permanentLive: boolean - latencyMode: LiveVideoLatencyMode - - commentsEnabled: boolean - downloadEnabled: boolean - waitTranscoding: boolean - state: VideoState - - published: string - originallyPublishedAt: string - updated: string - uploadDate: string - - mediaType: 'text/markdown' - content: string - - support: string - - icon: ActivityIconObject[] - - url: ActivityUrlObject[] - - likes: string - dislikes: string - shares: string - comments: string - - attributedTo: ActivityPubAttributedTo[] - - preview?: ActivityPubStoryboard[] - - to?: string[] - cc?: string[] -} - -export interface ActivityPubStoryboard { - type: 'Image' - rel: [ 'storyboard' ] - url: { - href: string - mediaType: string - width: number - height: number - tileWidth: number - tileHeight: number - tileDuration: string - }[] -} diff --git a/shared/models/actors/account.model.ts b/shared/models/actors/account.model.ts deleted file mode 100644 index 9fbec60ba..000000000 --- a/shared/models/actors/account.model.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { ActorImage } from './actor-image.model' -import { Actor } from './actor.model' - -export interface Account extends Actor { - displayName: string - description: string - avatars: ActorImage[] - - updatedAt: Date | string - - userId?: number -} - -export interface AccountSummary { - id: number - name: string - displayName: string - url: string - host: string - - avatars: ActorImage[] -} diff --git a/shared/models/actors/actor-image.type.ts b/shared/models/actors/actor-image.type.ts deleted file mode 100644 index ac8eb6bf2..000000000 --- a/shared/models/actors/actor-image.type.ts +++ /dev/null @@ -1,4 +0,0 @@ -export const enum ActorImageType { - AVATAR = 1, - BANNER = 2 -} diff --git a/shared/models/actors/actor.model.ts b/shared/models/actors/actor.model.ts deleted file mode 100644 index ab0760263..000000000 --- a/shared/models/actors/actor.model.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { ActorImage } from './actor-image.model' - -export interface Actor { - id: number - url: string - name: string - host: string - followingCount: number - followersCount: number - createdAt: Date | string - - avatars: ActorImage[] -} diff --git a/shared/models/actors/follow.model.ts b/shared/models/actors/follow.model.ts deleted file mode 100644 index 244d6d97e..000000000 --- a/shared/models/actors/follow.model.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { Actor } from './actor.model' - -export type FollowState = 'pending' | 'accepted' | 'rejected' - -export interface ActorFollow { - id: number - follower: Actor & { hostRedundancyAllowed: boolean } - following: Actor & { hostRedundancyAllowed: boolean } - score: number - state: FollowState - createdAt: Date - updatedAt: Date -} diff --git a/shared/models/actors/index.ts b/shared/models/actors/index.ts deleted file mode 100644 index e03f168cd..000000000 --- a/shared/models/actors/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from './account.model' -export * from './actor-image.model' -export * from './actor-image.type' -export * from './actor.model' -export * from './custom-page.model' -export * from './follow.model' diff --git a/shared/models/bulk/index.ts b/shared/models/bulk/index.ts deleted file mode 100644 index 168c8cd48..000000000 --- a/shared/models/bulk/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './bulk-remove-comments-of-body.model' diff --git a/shared/models/common/index.ts b/shared/models/common/index.ts deleted file mode 100644 index 4db85eff2..000000000 --- a/shared/models/common/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './result-list.model' diff --git a/shared/models/custom-markup/index.ts b/shared/models/custom-markup/index.ts deleted file mode 100644 index 2898dfa90..000000000 --- a/shared/models/custom-markup/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './custom-markup-data.model' diff --git a/shared/models/feeds/feed-format.enum.ts b/shared/models/feeds/feed-format.enum.ts deleted file mode 100644 index d3d574331..000000000 --- a/shared/models/feeds/feed-format.enum.ts +++ /dev/null @@ -1,5 +0,0 @@ -export const enum FeedFormat { - RSS = 'xml', - ATOM = 'atom', - JSON = 'json' -} diff --git a/shared/models/feeds/index.ts b/shared/models/feeds/index.ts deleted file mode 100644 index d56c8458c..000000000 --- a/shared/models/feeds/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './feed-format.enum' diff --git a/shared/models/http/http-error-codes.ts b/shared/models/http/http-error-codes.ts deleted file mode 100644 index 5ebff1cb5..000000000 --- a/shared/models/http/http-error-codes.ts +++ /dev/null @@ -1,364 +0,0 @@ -/** - * Hypertext Transfer Protocol (HTTP) response status codes. - * @see {@link https://en.wikipedia.org/wiki/List_of_HTTP_status_codes} - * - * WebDAV and other codes useless with regards to PeerTube are not listed. - */ -export const enum HttpStatusCode { - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.2.1 - * - * The server has received the request headers and the client should proceed to send the request body - * (in the case of a request for which a body needs to be sent; for example, a POST request). - * Sending a large request body to a server after a request has been rejected for inappropriate headers would be inefficient. - * To have a server check the request's headers, a client must send Expect: 100-continue as a header in its initial request - * and receive a 100 Continue status code in response before sending the body. The response 417 Expectation Failed indicates - * the request should not be continued. - */ - CONTINUE_100 = 100, - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.2.2 - * - * This code is sent in response to an Upgrade request header by the client, and indicates the protocol the server is switching too. - */ - SWITCHING_PROTOCOLS_101 = 101, - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.3.1 - * - * Standard response for successful HTTP requests. The actual response will depend on the request method used: - * GET: The resource has been fetched and is transmitted in the message body. - * HEAD: The entity headers are in the message body. - * POST: The resource describing the result of the action is transmitted in the message body. - * TRACE: The message body contains the request message as received by the server - */ - OK_200 = 200, - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.3.2 - * - * The request has been fulfilled, resulting in the creation of a new resource, typically after a PUT. - */ - CREATED_201 = 201, - - /** - * The request has been accepted for processing, but the processing has not been completed. - * The request might or might not be eventually acted upon, and may be disallowed when processing occurs. - */ - ACCEPTED_202 = 202, - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.3.5 - * - * There is no content to send for this request, but the headers may be useful. - * The user-agent may update its cached headers for this resource with the new ones. - */ - NO_CONTENT_204 = 204, - - /** - * The server successfully processed the request, but is not returning any content. - * Unlike a 204 response, this response requires that the requester reset the document view. - */ - RESET_CONTENT_205 = 205, - - /** - * The server is delivering only part of the resource (byte serving) due to a range header sent by the client. - * The range header is used by HTTP clients to enable resuming of interrupted downloads, - * or split a download into multiple simultaneous streams. - */ - PARTIAL_CONTENT_206 = 206, - - /** - * Indicates multiple options for the resource from which the client may choose (via agent-driven content negotiation). - * For example, this code could be used to present multiple video format options, - * to list files with different filename extensions, or to suggest word-sense disambiguation. - */ - MULTIPLE_CHOICES_300 = 300, - - /** - * This and all future requests should be directed to the given URI. - */ - MOVED_PERMANENTLY_301 = 301, - - /** - * This is an example of industry practice contradicting the standard. - * The HTTP/1.0 specification (RFC 1945) required the client to perform a temporary redirect - * (the original describing phrase was "Moved Temporarily"), but popular browsers implemented 302 - * with the functionality of a 303 See Other. Therefore, HTTP/1.1 added status codes 303 and 307 - * to distinguish between the two behaviours. However, some Web applications and frameworks - * use the 302 status code as if it were the 303. - */ - FOUND_302 = 302, - - /** - * SINCE HTTP/1.1 - * The response to the request can be found under another URI using a GET method. - * When received in response to a POST (or PUT/DELETE), the client should presume that - * the server has received the data and should issue a redirect with a separate GET message. - */ - SEE_OTHER_303 = 303, - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc7232#section-4.1 - * - * Indicates that the resource has not been modified since the version specified by the request headers - * `If-Modified-Since` or `If-None-Match`. - * In such case, there is no need to retransmit the resource since the client still has a previously-downloaded copy. - */ - NOT_MODIFIED_304 = 304, - - /** - * SINCE HTTP/1.1 - * In this case, the request should be repeated with another URI; however, future requests should still use the original URI. - * In contrast to how 302 was historically implemented, the request method is not allowed to be changed when reissuing the - * original request. - * For example, a POST request should be repeated using another POST request. - */ - TEMPORARY_REDIRECT_307 = 307, - - /** - * The request and all future requests should be repeated using another URI. - * 307 and 308 parallel the behaviors of 302 and 301, but do not allow the HTTP method to change. - * So, for example, submitting a form to a permanently redirected resource may continue smoothly. - */ - PERMANENT_REDIRECT_308 = 308, - - /** - * The server cannot or will not process the request due to an apparent client error - * (e.g., malformed request syntax, too large size, invalid request message framing, or deceptive request routing). - */ - BAD_REQUEST_400 = 400, - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc7235#section-3.1 - * - * Similar to 403 Forbidden, but specifically for use when authentication is required and has failed or has not yet - * been provided. The response must include a `WWW-Authenticate` header field containing a challenge applicable to the - * requested resource. See Basic access authentication and Digest access authentication. 401 semantically means - * "unauthenticated",i.e. the user does not have the necessary credentials. - */ - UNAUTHORIZED_401 = 401, - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.2 - * - * Reserved for future use. The original intention was that this code might be used as part of some form of digital - * cash or micro payment scheme, but that has not happened, and this code is not usually used. - * Google Developers API uses this status if a particular developer has exceeded the daily limit on requests. - */ - PAYMENT_REQUIRED_402 = 402, - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.3 - * - * The client does not have access rights to the content, i.e. they are unauthorized, so server is rejecting to - * give proper response. Unlike 401, the client's identity is known to the server. - */ - FORBIDDEN_403 = 403, - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.6.2 - * - * The requested resource could not be found but may be available in the future. - * Subsequent requests by the client are permissible. - */ - NOT_FOUND_404 = 404, - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.5 - * - * A request method is not supported for the requested resource; - * for example, a GET request on a form that requires data to be presented via POST, or a PUT request on a read-only resource. - */ - METHOD_NOT_ALLOWED_405 = 405, - - /** - * The requested resource is capable of generating only content not acceptable according to the Accept headers sent in the request. - */ - NOT_ACCEPTABLE_406 = 406, - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.7 - * - * This response is sent on an idle connection by some servers, even without any previous request by the client. - * It means that the server would like to shut down this unused connection. This response is used much more since - * some browsers, like Chrome, Firefox 27+, or IE9, use HTTP pre-connection mechanisms to speed up surfing. Also - * note that some servers merely shut down the connection without sending this message. - * - * @ - */ - REQUEST_TIMEOUT_408 = 408, - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.8 - * - * Indicates that the request could not be processed because of conflict in the request, - * such as an edit conflict between multiple simultaneous updates. - * - * @see HttpStatusCode.UNPROCESSABLE_ENTITY_422 to denote a disabled feature - */ - CONFLICT_409 = 409, - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.9 - * - * Indicates that the resource requested is no longer available and will not be available again. - * This should be used when a resource has been intentionally removed and the resource should be purged. - * Upon receiving a 410 status code, the client should not request the resource in the future. - * Clients such as search engines should remove the resource from their indices. - * Most use cases do not require clients and search engines to purge the resource, and a "404 Not Found" may be used instead. - */ - GONE_410 = 410, - - /** - * The request did not specify the length of its content, which is required by the requested resource. - */ - LENGTH_REQUIRED_411 = 411, - - /** - * The server does not meet one of the preconditions that the requester put on the request. - */ - PRECONDITION_FAILED_412 = 412, - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.11 - * - * The request is larger than the server is willing or able to process ; the server might close the connection - * or return an Retry-After header field. - * Previously called "Request Entity Too Large". - */ - PAYLOAD_TOO_LARGE_413 = 413, - - /** - * The URI provided was too long for the server to process. Often the result of too much data being encoded as a - * query-string of a GET request, in which case it should be converted to a POST request. - * Called "Request-URI Too Long" previously. - */ - URI_TOO_LONG_414 = 414, - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.5.13 - * - * The request entity has a media type which the server or resource does not support. - * For example, the client uploads an image as image/svg+xml, but the server requires that images use a different format. - */ - UNSUPPORTED_MEDIA_TYPE_415 = 415, - - /** - * The client has asked for a portion of the file (byte serving), but the server cannot supply that portion. - * For example, if the client asked for a part of the file that lies beyond the end of the file. - * Called "Requested Range Not Satisfiable" previously. - */ - RANGE_NOT_SATISFIABLE_416 = 416, - - /** - * The server cannot meet the requirements of the `Expect` request-header field. - */ - EXPECTATION_FAILED_417 = 417, - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc2324 - * - * This code was defined in 1998 as one of the traditional IETF April Fools' jokes, in RFC 2324, Hyper Text Coffee Pot Control Protocol, - * and is not expected to be implemented by actual HTTP servers. The RFC specifies this code should be returned by - * teapots requested to brew coffee. This HTTP status is used as an Easter egg in some websites, including PeerTube instances ;-). - */ - I_AM_A_TEAPOT_418 = 418, - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc2518#section-10.3 - * - * The request was well-formed but was unable to be followed due to semantic errors. - * The server understands the content type of the request entity (hence a 415 (Unsupported Media Type) status code is inappropriate), - * and the syntax of the request entity is correct (thus a 400 (Bad Request) status code is inappropriate) but was unable to process - * the contained instructions. For example, this error condition may occur if an JSON request body contains well-formed (i.e., - * syntactically correct), but semantically erroneous, JSON instructions. - * - * Can also be used to denote disabled features (akin to disabled syntax). - * - * @see HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415 if the `Content-Type` was not supported. - * @see HttpStatusCode.BAD_REQUEST_400 if the request was not parsable (broken JSON, XML) - */ - UNPROCESSABLE_ENTITY_422 = 422, - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc4918#section-11.3 - * - * The resource that is being accessed is locked. WebDAV-specific but used by some HTTP services. - * - * @deprecated use `If-Match` / `If-None-Match` instead - * @see {@link https://evertpot.com/http/423-locked} - */ - LOCKED_423 = 423, - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc6585#section-4 - * - * The user has sent too many requests in a given amount of time. Intended for use with rate-limiting schemes. - */ - TOO_MANY_REQUESTS_429 = 429, - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc6585#section-5 - * - * The server is unwilling to process the request because either an individual header field, - * or all the header fields collectively, are too large. - */ - REQUEST_HEADER_FIELDS_TOO_LARGE_431 = 431, - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc7725 - * - * A server operator has received a legal demand to deny access to a resource or to a set of resources - * that includes the requested resource. The code 451 was chosen as a reference to the novel Fahrenheit 451. - */ - UNAVAILABLE_FOR_LEGAL_REASONS_451 = 451, - - /** - * A generic error message, given when an unexpected condition was encountered and no more specific message is suitable. - */ - INTERNAL_SERVER_ERROR_500 = 500, - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc7231#section-6.6.2 - * - * The server either does not recognize the request method, or it lacks the ability to fulfill the request. - * Usually this implies future availability (e.g., a new feature of a web-service API). - */ - NOT_IMPLEMENTED_501 = 501, - - /** - * The server was acting as a gateway or proxy and received an invalid response from the upstream server. - */ - BAD_GATEWAY_502 = 502, - - /** - * The server is currently unavailable (because it is overloaded or down for maintenance). - * Generally, this is a temporary state. - */ - SERVICE_UNAVAILABLE_503 = 503, - - /** - * The server was acting as a gateway or proxy and did not receive a timely response from the upstream server. - */ - GATEWAY_TIMEOUT_504 = 504, - - /** - * The server does not support the HTTP protocol version used in the request - */ - HTTP_VERSION_NOT_SUPPORTED_505 = 505, - - /** - * Official Documentation @ https://tools.ietf.org/html/rfc2518#section-10.6 - * - * The 507 (Insufficient Storage) status code means the method could not be performed on the resource because the - * server is unable to store the representation needed to successfully complete the request. This condition is - * considered to be temporary. If the request which received this status code was the result of a user action, - * the request MUST NOT be repeated until it is requested by a separate user action. - * - * @see HttpStatusCode.PAYLOAD_TOO_LARGE_413 for quota errors - */ - INSUFFICIENT_STORAGE_507 = 507, -} diff --git a/shared/models/http/http-methods.ts b/shared/models/http/http-methods.ts deleted file mode 100644 index 3f4adafe2..000000000 --- a/shared/models/http/http-methods.ts +++ /dev/null @@ -1,21 +0,0 @@ -/** HTTP request method to indicate the desired action to be performed for a given resource. */ -export const enum HttpMethod { - /** The CONNECT method establishes a tunnel to the server identified by the target resource. */ - CONNECT = 'CONNECT', - /** The DELETE method deletes the specified resource. */ - DELETE = 'DELETE', - /** The GET method requests a representation of the specified resource. Requests using GET should only retrieve data. */ - GET = 'GET', - /** The HEAD method asks for a response identical to that of a GET request, but without the response body. */ - HEAD = 'HEAD', - /** The OPTIONS method is used to describe the communication options for the target resource. */ - OPTIONS = 'OPTIONS', - /** The PATCH method is used to apply partial modifications to a resource. */ - PATCH = 'PATCH', - /** The POST method is used to submit an entity to the specified resource */ - POST = 'POST', - /** The PUT method replaces all current representations of the target resource with the request payload. */ - PUT = 'PUT', - /** The TRACE method performs a message loop-back test along the path to the target resource. */ - TRACE = 'TRACE' -} diff --git a/shared/models/http/index.ts b/shared/models/http/index.ts deleted file mode 100644 index ec991afe0..000000000 --- a/shared/models/http/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './http-error-codes' -export * from './http-methods' diff --git a/shared/models/index.ts b/shared/models/index.ts deleted file mode 100644 index 78f6e73e3..000000000 --- a/shared/models/index.ts +++ /dev/null @@ -1,19 +0,0 @@ -export * from './activitypub' -export * from './actors' -export * from './bulk' -export * from './common' -export * from './custom-markup' -export * from './feeds' -export * from './http' -export * from './joinpeertube' -export * from './metrics' -export * from './moderation' -export * from './overviews' -export * from './plugins' -export * from './redundancy' -export * from './runners' -export * from './search' -export * from './server' -export * from './tokens' -export * from './users' -export * from './videos' diff --git a/shared/models/joinpeertube/index.ts b/shared/models/joinpeertube/index.ts deleted file mode 100644 index 9681c35ad..000000000 --- a/shared/models/joinpeertube/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './versions.model' diff --git a/shared/models/metrics/index.ts b/shared/models/metrics/index.ts deleted file mode 100644 index 24194cce3..000000000 --- a/shared/models/metrics/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './playback-metric-create.model' diff --git a/shared/models/metrics/playback-metric-create.model.ts b/shared/models/metrics/playback-metric-create.model.ts deleted file mode 100644 index 1d47421c3..000000000 --- a/shared/models/metrics/playback-metric-create.model.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { VideoResolution } from '../videos' - -export interface PlaybackMetricCreate { - playerMode: 'p2p-media-loader' | 'webtorrent' | 'web-video' // FIXME: remove webtorrent player mode not used anymore in PeerTube v6 - - resolution?: VideoResolution - fps?: number - - p2pEnabled: boolean - p2pPeers?: number - - resolutionChanges: number - - errors: number - - downloadedBytesP2P: number - downloadedBytesHTTP: number - - uploadedBytesP2P: number - - videoId: number | string -} diff --git a/shared/models/moderation/abuse/abuse-create.model.ts b/shared/models/moderation/abuse/abuse-create.model.ts deleted file mode 100644 index 7d35555c3..000000000 --- a/shared/models/moderation/abuse/abuse-create.model.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { AbusePredefinedReasonsString } from './abuse-reason.model' - -export interface AbuseCreate { - reason: string - - predefinedReasons?: AbusePredefinedReasonsString[] - - account?: { - id: number - } - - video?: { - id: number | string - startAt?: number - endAt?: number - } - - comment?: { - id: number - } -} diff --git a/shared/models/moderation/abuse/abuse-message.model.ts b/shared/models/moderation/abuse/abuse-message.model.ts deleted file mode 100644 index 9edd9daff..000000000 --- a/shared/models/moderation/abuse/abuse-message.model.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { AccountSummary } from '../../actors/account.model' - -export interface AbuseMessage { - id: number - message: string - byModerator: boolean - createdAt: Date | string - - account: AccountSummary -} diff --git a/shared/models/moderation/abuse/abuse-reason.model.ts b/shared/models/moderation/abuse/abuse-reason.model.ts deleted file mode 100644 index 57359aef6..000000000 --- a/shared/models/moderation/abuse/abuse-reason.model.ts +++ /dev/null @@ -1,20 +0,0 @@ -export const enum AbusePredefinedReasons { - VIOLENT_OR_REPULSIVE = 1, - HATEFUL_OR_ABUSIVE, - SPAM_OR_MISLEADING, - PRIVACY, - RIGHTS, - SERVER_RULES, - THUMBNAILS, - CAPTIONS -} - -export type AbusePredefinedReasonsString = - 'violentOrRepulsive' | - 'hatefulOrAbusive' | - 'spamOrMisleading' | - 'privacy' | - 'rights' | - 'serverRules' | - 'thumbnails' | - 'captions' diff --git a/shared/models/moderation/abuse/abuse-state.model.ts b/shared/models/moderation/abuse/abuse-state.model.ts deleted file mode 100644 index 8ef6fdada..000000000 --- a/shared/models/moderation/abuse/abuse-state.model.ts +++ /dev/null @@ -1,5 +0,0 @@ -export const enum AbuseState { - PENDING = 1, - REJECTED = 2, - ACCEPTED = 3 -} diff --git a/shared/models/moderation/abuse/abuse-update.model.ts b/shared/models/moderation/abuse/abuse-update.model.ts deleted file mode 100644 index 4360fe7ac..000000000 --- a/shared/models/moderation/abuse/abuse-update.model.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { AbuseState } from './abuse-state.model' - -export interface AbuseUpdate { - moderationComment?: string - - state?: AbuseState -} diff --git a/shared/models/moderation/abuse/abuse.model.ts b/shared/models/moderation/abuse/abuse.model.ts deleted file mode 100644 index 6048777ff..000000000 --- a/shared/models/moderation/abuse/abuse.model.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { Account } from '../../actors/account.model' -import { AbuseState } from './abuse-state.model' -import { AbusePredefinedReasonsString } from './abuse-reason.model' -import { VideoConstant } from '../../videos/video-constant.model' -import { VideoChannel } from '../../videos/channel/video-channel.model' - -export interface AdminVideoAbuse { - id: number - name: string - uuid: string - nsfw: boolean - - deleted: boolean - blacklisted: boolean - - startAt: number | null - endAt: number | null - - thumbnailPath?: string - channel?: VideoChannel - - countReports: number - nthReport: number -} - -export interface AdminVideoCommentAbuse { - id: number - threadId: number - - video: { - id: number - name: string - uuid: string - } - - text: string - - deleted: boolean -} - -export interface AdminAbuse { - id: number - - reason: string - predefinedReasons?: AbusePredefinedReasonsString[] - - reporterAccount: Account - flaggedAccount: Account - - state: VideoConstant - moderationComment?: string - - video?: AdminVideoAbuse - comment?: AdminVideoCommentAbuse - - createdAt: Date - updatedAt: Date - - countReportsForReporter?: number - countReportsForReportee?: number - - countMessages: number -} - -export type UserVideoAbuse = Omit - -export type UserVideoCommentAbuse = AdminVideoCommentAbuse - -export type UserAbuse = Omit diff --git a/shared/models/moderation/abuse/index.ts b/shared/models/moderation/abuse/index.ts deleted file mode 100644 index b518517a6..000000000 --- a/shared/models/moderation/abuse/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -export * from './abuse-create.model' -export * from './abuse-filter.type' -export * from './abuse-message.model' -export * from './abuse-reason.model' -export * from './abuse-state.model' -export * from './abuse-update.model' -export * from './abuse-video-is.type' -export * from './abuse.model' diff --git a/shared/models/moderation/account-block.model.ts b/shared/models/moderation/account-block.model.ts deleted file mode 100644 index a942ed614..000000000 --- a/shared/models/moderation/account-block.model.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { Account } from '../actors' - -export interface AccountBlock { - byAccount: Account - blockedAccount: Account - createdAt: Date | string -} diff --git a/shared/models/moderation/index.ts b/shared/models/moderation/index.ts deleted file mode 100644 index f8e6d351c..000000000 --- a/shared/models/moderation/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './abuse' -export * from './block-status.model' -export * from './account-block.model' -export * from './server-block.model' diff --git a/shared/models/moderation/server-block.model.ts b/shared/models/moderation/server-block.model.ts deleted file mode 100644 index a8b8af0b7..000000000 --- a/shared/models/moderation/server-block.model.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { Account } from '../actors' - -export interface ServerBlock { - byAccount: Account - blockedServer: { - host: string - } - createdAt: Date | string -} diff --git a/shared/models/nodeinfo/index.ts b/shared/models/nodeinfo/index.ts deleted file mode 100644 index faa64302a..000000000 --- a/shared/models/nodeinfo/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './nodeinfo.model' diff --git a/shared/models/overviews/index.ts b/shared/models/overviews/index.ts deleted file mode 100644 index 468507c6b..000000000 --- a/shared/models/overviews/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './videos-overview.model' diff --git a/shared/models/overviews/videos-overview.model.ts b/shared/models/overviews/videos-overview.model.ts deleted file mode 100644 index 0f3cb4a52..000000000 --- a/shared/models/overviews/videos-overview.model.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { Video, VideoChannelSummary, VideoConstant } from '../videos' - -export interface ChannelOverview { - channel: VideoChannelSummary - videos: Video[] -} - -export interface CategoryOverview { - category: VideoConstant - videos: Video[] -} - -export interface TagOverview { - tag: string - videos: Video[] -} - -export interface VideosOverview { - channels: ChannelOverview[] - - categories: CategoryOverview[] - - tags: TagOverview[] -} diff --git a/shared/models/plugins/client/index.ts b/shared/models/plugins/client/index.ts deleted file mode 100644 index f3e3fcbcf..000000000 --- a/shared/models/plugins/client/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -export * from './client-hook.model' -export * from './plugin-client-scope.type' -export * from './plugin-element-placeholder.type' -export * from './plugin-selector-id.type' -export * from './register-client-form-field.model' -export * from './register-client-hook.model' -export * from './register-client-route.model' -export * from './register-client-settings-script.model' diff --git a/shared/models/plugins/client/register-client-hook.model.ts b/shared/models/plugins/client/register-client-hook.model.ts deleted file mode 100644 index 81047b21d..000000000 --- a/shared/models/plugins/client/register-client-hook.model.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { ClientHookName } from './client-hook.model' - -export interface RegisterClientHookOptions { - target: ClientHookName - handler: Function - priority?: number -} diff --git a/shared/models/plugins/client/register-client-settings-script.model.ts b/shared/models/plugins/client/register-client-settings-script.model.ts deleted file mode 100644 index 117ca4739..000000000 --- a/shared/models/plugins/client/register-client-settings-script.model.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { RegisterServerSettingOptions } from '../server' - -export interface RegisterClientSettingsScriptOptions { - isSettingHidden (options: { - setting: RegisterServerSettingOptions - formValues: { [name: string]: any } - }): boolean -} diff --git a/shared/models/plugins/hook-type.enum.ts b/shared/models/plugins/hook-type.enum.ts deleted file mode 100644 index a96c943f1..000000000 --- a/shared/models/plugins/hook-type.enum.ts +++ /dev/null @@ -1,5 +0,0 @@ -export const enum HookType { - STATIC = 1, - ACTION = 2, - FILTER = 3 -} diff --git a/shared/models/plugins/index.ts b/shared/models/plugins/index.ts deleted file mode 100644 index cbbe4916e..000000000 --- a/shared/models/plugins/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from './client' -export * from './plugin-index' -export * from './server' -export * from './hook-type.enum' -export * from './plugin-package-json.model' -export * from './plugin.type' diff --git a/shared/models/plugins/plugin-index/index.ts b/shared/models/plugins/plugin-index/index.ts deleted file mode 100644 index 913846638..000000000 --- a/shared/models/plugins/plugin-index/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './peertube-plugin-index-list.model' -export * from './peertube-plugin-index.model' -export * from './peertube-plugin-latest-version.model' diff --git a/shared/models/plugins/plugin-index/peertube-plugin-index-list.model.ts b/shared/models/plugins/plugin-index/peertube-plugin-index-list.model.ts deleted file mode 100644 index ecb46482e..000000000 --- a/shared/models/plugins/plugin-index/peertube-plugin-index-list.model.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { PluginType } from '../plugin.type' - -export interface PeertubePluginIndexList { - start: number - count: number - sort: string - pluginType?: PluginType - currentPeerTubeEngine?: string - search?: string -} diff --git a/shared/models/plugins/plugin-package-json.model.ts b/shared/models/plugins/plugin-package-json.model.ts deleted file mode 100644 index 7ce968ff2..000000000 --- a/shared/models/plugins/plugin-package-json.model.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { PluginClientScope } from './client/plugin-client-scope.type' - -export type PluginTranslationPathsJSON = { - [ locale: string ]: string -} - -export type ClientScriptJSON = { - script: string - scopes: PluginClientScope[] -} - -export type PluginPackageJSON = { - name: string - version: string - description: string - engine: { peertube: string } - - homepage: string - author: string - bugs: string - library: string - - staticDirs: { [ name: string ]: string } - css: string[] - - clientScripts: ClientScriptJSON[] - - translations: PluginTranslationPathsJSON -} diff --git a/shared/models/plugins/plugin.type.ts b/shared/models/plugins/plugin.type.ts deleted file mode 100644 index 016219ceb..000000000 --- a/shared/models/plugins/plugin.type.ts +++ /dev/null @@ -1,4 +0,0 @@ -export const enum PluginType { - PLUGIN = 1, - THEME = 2 -} diff --git a/shared/models/plugins/server/api/index.ts b/shared/models/plugins/server/api/index.ts deleted file mode 100644 index eb59a03f0..000000000 --- a/shared/models/plugins/server/api/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './install-plugin.model' -export * from './manage-plugin.model' -export * from './peertube-plugin.model' diff --git a/shared/models/plugins/server/api/peertube-plugin.model.ts b/shared/models/plugins/server/api/peertube-plugin.model.ts deleted file mode 100644 index 54c383f57..000000000 --- a/shared/models/plugins/server/api/peertube-plugin.model.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { PluginType } from '../../plugin.type' - -export interface PeerTubePlugin { - name: string - type: PluginType - latestVersion: string - version: string - enabled: boolean - uninstalled: boolean - peertubeEngine: string - description: string - homepage: string - settings: { [ name: string ]: string } - createdAt: Date - updatedAt: Date -} diff --git a/shared/models/plugins/server/index.ts b/shared/models/plugins/server/index.ts deleted file mode 100644 index d3ff49d3b..000000000 --- a/shared/models/plugins/server/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from './api' -export * from './managers' -export * from './settings' -export * from './plugin-translation.model' -export * from './register-server-hook.model' -export * from './server-hook.model' diff --git a/shared/models/plugins/server/managers/index.ts b/shared/models/plugins/server/managers/index.ts deleted file mode 100644 index 49365a854..000000000 --- a/shared/models/plugins/server/managers/index.ts +++ /dev/null @@ -1,9 +0,0 @@ - -export * from './plugin-playlist-privacy-manager.model' -export * from './plugin-settings-manager.model' -export * from './plugin-storage-manager.model' -export * from './plugin-transcoding-manager.model' -export * from './plugin-video-category-manager.model' -export * from './plugin-video-language-manager.model' -export * from './plugin-video-licence-manager.model' -export * from './plugin-video-privacy-manager.model' diff --git a/shared/models/plugins/server/managers/plugin-playlist-privacy-manager.model.ts b/shared/models/plugins/server/managers/plugin-playlist-privacy-manager.model.ts deleted file mode 100644 index 35247c1e3..000000000 --- a/shared/models/plugins/server/managers/plugin-playlist-privacy-manager.model.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { VideoPlaylistPrivacy } from '../../../videos/playlist/video-playlist-privacy.model' -import { ConstantManager } from '../plugin-constant-manager.model' - -export interface PluginPlaylistPrivacyManager extends ConstantManager { - /** - * PUBLIC = 1, - * UNLISTED = 2, - * PRIVATE = 3 - * @deprecated use `deleteConstant` instead - */ - deletePlaylistPrivacy: (privacyKey: VideoPlaylistPrivacy) => boolean -} diff --git a/shared/models/plugins/server/managers/plugin-transcoding-manager.model.ts b/shared/models/plugins/server/managers/plugin-transcoding-manager.model.ts deleted file mode 100644 index b6fb46ba0..000000000 --- a/shared/models/plugins/server/managers/plugin-transcoding-manager.model.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { EncoderOptionsBuilder } from '../../../videos/transcoding' - -export interface PluginTranscodingManager { - addLiveProfile (encoder: string, profile: string, builder: EncoderOptionsBuilder): boolean - - addVODProfile (encoder: string, profile: string, builder: EncoderOptionsBuilder): boolean - - addLiveEncoderPriority (streamType: 'audio' | 'video', encoder: string, priority: number): void - - addVODEncoderPriority (streamType: 'audio' | 'video', encoder: string, priority: number): void - - removeAllProfilesAndEncoderPriorities(): void -} diff --git a/shared/models/plugins/server/managers/plugin-video-category-manager.model.ts b/shared/models/plugins/server/managers/plugin-video-category-manager.model.ts deleted file mode 100644 index cf3d828fe..000000000 --- a/shared/models/plugins/server/managers/plugin-video-category-manager.model.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { ConstantManager } from '../plugin-constant-manager.model' - -export interface PluginVideoCategoryManager extends ConstantManager { - /** - * @deprecated use `addConstant` instead - */ - addCategory: (categoryKey: number, categoryLabel: string) => boolean - - /** - * @deprecated use `deleteConstant` instead - */ - deleteCategory: (categoryKey: number) => boolean -} diff --git a/shared/models/plugins/server/managers/plugin-video-language-manager.model.ts b/shared/models/plugins/server/managers/plugin-video-language-manager.model.ts deleted file mode 100644 index 69fc8e503..000000000 --- a/shared/models/plugins/server/managers/plugin-video-language-manager.model.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { ConstantManager } from '../plugin-constant-manager.model' - -export interface PluginVideoLanguageManager extends ConstantManager { - /** - * @deprecated use `addConstant` instead - */ - addLanguage: (languageKey: string, languageLabel: string) => boolean - - /** - * @deprecated use `deleteConstant` instead - */ - deleteLanguage: (languageKey: string) => boolean -} diff --git a/shared/models/plugins/server/managers/plugin-video-licence-manager.model.ts b/shared/models/plugins/server/managers/plugin-video-licence-manager.model.ts deleted file mode 100644 index 21b422270..000000000 --- a/shared/models/plugins/server/managers/plugin-video-licence-manager.model.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { ConstantManager } from '../plugin-constant-manager.model' - -export interface PluginVideoLicenceManager extends ConstantManager { - /** - * @deprecated use `addConstant` instead - */ - addLicence: (licenceKey: number, licenceLabel: string) => boolean - - /** - * @deprecated use `deleteConstant` instead - */ - deleteLicence: (licenceKey: number) => boolean -} diff --git a/shared/models/plugins/server/managers/plugin-video-privacy-manager.model.ts b/shared/models/plugins/server/managers/plugin-video-privacy-manager.model.ts deleted file mode 100644 index a237037db..000000000 --- a/shared/models/plugins/server/managers/plugin-video-privacy-manager.model.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { VideoPrivacy } from '../../../videos/video-privacy.enum' -import { ConstantManager } from '../plugin-constant-manager.model' - -export interface PluginVideoPrivacyManager extends ConstantManager { - /** - * PUBLIC = 1, - * UNLISTED = 2, - * PRIVATE = 3 - * INTERNAL = 4 - * @deprecated use `deleteConstant` instead - */ - deletePrivacy: (privacyKey: VideoPrivacy) => boolean -} diff --git a/shared/models/plugins/server/register-server-hook.model.ts b/shared/models/plugins/server/register-server-hook.model.ts deleted file mode 100644 index 746fdc329..000000000 --- a/shared/models/plugins/server/register-server-hook.model.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { ServerHookName } from './server-hook.model' - -export interface RegisterServerHookOptions { - target: ServerHookName - handler: Function - priority?: number -} diff --git a/shared/models/plugins/server/settings/index.ts b/shared/models/plugins/server/settings/index.ts deleted file mode 100644 index b456de019..000000000 --- a/shared/models/plugins/server/settings/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './public-server.setting' -export * from './register-server-setting.model' diff --git a/shared/models/plugins/server/settings/public-server.setting.ts b/shared/models/plugins/server/settings/public-server.setting.ts deleted file mode 100644 index d38e5424a..000000000 --- a/shared/models/plugins/server/settings/public-server.setting.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { SettingEntries } from '../managers/plugin-settings-manager.model' - -export interface PublicServerSetting { - publicSettings: SettingEntries -} diff --git a/shared/models/plugins/server/settings/register-server-setting.model.ts b/shared/models/plugins/server/settings/register-server-setting.model.ts deleted file mode 100644 index d9a798cac..000000000 --- a/shared/models/plugins/server/settings/register-server-setting.model.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { RegisterClientFormFieldOptions } from '../../client' - -export type RegisterServerSettingOptions = RegisterClientFormFieldOptions & { - // If the setting is not private, anyone can view its value (client code included) - // If the setting is private, only server-side hooks can access it - // Mainly used by the PeerTube client to get admin config - private: boolean -} - -export interface RegisteredServerSettings { - registeredSettings: RegisterServerSettingOptions[] -} diff --git a/shared/models/redundancy/index.ts b/shared/models/redundancy/index.ts deleted file mode 100644 index 641a5d625..000000000 --- a/shared/models/redundancy/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './video-redundancies-filters.model' -export * from './video-redundancy-config-filter.type' -export * from './video-redundancy.model' -export * from './videos-redundancy-strategy.model' diff --git a/shared/models/runners/accept-runner-job-result.model.ts b/shared/models/runners/accept-runner-job-result.model.ts deleted file mode 100644 index f2094b945..000000000 --- a/shared/models/runners/accept-runner-job-result.model.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { RunnerJobPayload } from './runner-job-payload.model' -import { RunnerJob } from './runner-job.model' - -export interface AcceptRunnerJobResult { - job: RunnerJob & { jobToken: string } -} diff --git a/shared/models/runners/index.ts b/shared/models/runners/index.ts deleted file mode 100644 index a52b82d2e..000000000 --- a/shared/models/runners/index.ts +++ /dev/null @@ -1,21 +0,0 @@ -export * from './abort-runner-job-body.model' -export * from './accept-runner-job-body.model' -export * from './accept-runner-job-result.model' -export * from './error-runner-job-body.model' -export * from './list-runner-jobs-query.model' -export * from './list-runner-registration-tokens.model' -export * from './list-runners-query.model' -export * from './register-runner-body.model' -export * from './register-runner-result.model' -export * from './request-runner-job-body.model' -export * from './request-runner-job-result.model' -export * from './runner-job-payload.model' -export * from './runner-job-private-payload.model' -export * from './runner-job-state.model' -export * from './runner-job-success-body.model' -export * from './runner-job-type.type' -export * from './runner-job-update-body.model' -export * from './runner-job.model' -export * from './runner-registration-token' -export * from './runner.model' -export * from './unregister-runner-body.model' diff --git a/shared/models/runners/list-runner-jobs-query.model.ts b/shared/models/runners/list-runner-jobs-query.model.ts deleted file mode 100644 index ef19b31fa..000000000 --- a/shared/models/runners/list-runner-jobs-query.model.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { RunnerJobState } from './runner-job-state.model' - -export interface ListRunnerJobsQuery { - start?: number - count?: number - sort?: string - search?: string - stateOneOf?: RunnerJobState[] -} diff --git a/shared/models/runners/request-runner-job-result.model.ts b/shared/models/runners/request-runner-job-result.model.ts deleted file mode 100644 index 98601c42c..000000000 --- a/shared/models/runners/request-runner-job-result.model.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { RunnerJobPayload } from './runner-job-payload.model' -import { RunnerJobType } from './runner-job-type.type' - -export interface RequestRunnerJobResult

{ - availableJobs: { - uuid: string - type: RunnerJobType - payload: P - }[] -} diff --git a/shared/models/runners/runner-job-payload.model.ts b/shared/models/runners/runner-job-payload.model.ts deleted file mode 100644 index 3dda6c51f..000000000 --- a/shared/models/runners/runner-job-payload.model.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { VideoStudioTaskPayload } from '../server' - -export type RunnerJobVODPayload = - RunnerJobVODWebVideoTranscodingPayload | - RunnerJobVODHLSTranscodingPayload | - RunnerJobVODAudioMergeTranscodingPayload - -export type RunnerJobPayload = - RunnerJobVODPayload | - RunnerJobLiveRTMPHLSTranscodingPayload | - RunnerJobStudioTranscodingPayload - -// --------------------------------------------------------------------------- - -export interface RunnerJobVODWebVideoTranscodingPayload { - input: { - videoFileUrl: string - } - - output: { - resolution: number - fps: number - } -} - -export interface RunnerJobVODHLSTranscodingPayload { - input: { - videoFileUrl: string - } - - output: { - resolution: number - fps: number - } -} - -export interface RunnerJobVODAudioMergeTranscodingPayload { - input: { - audioFileUrl: string - previewFileUrl: string - } - - output: { - resolution: number - fps: number - } -} - -export interface RunnerJobStudioTranscodingPayload { - input: { - videoFileUrl: string - } - - tasks: VideoStudioTaskPayload[] -} - -// --------------------------------------------------------------------------- - -export function isAudioMergeTranscodingPayload (payload: RunnerJobPayload): payload is RunnerJobVODAudioMergeTranscodingPayload { - return !!(payload as RunnerJobVODAudioMergeTranscodingPayload).input.audioFileUrl -} - -// --------------------------------------------------------------------------- - -export interface RunnerJobLiveRTMPHLSTranscodingPayload { - input: { - rtmpUrl: string - } - - output: { - toTranscode: { - resolution: number - fps: number - }[] - - segmentDuration: number - segmentListSize: number - } -} diff --git a/shared/models/runners/runner-job-private-payload.model.ts b/shared/models/runners/runner-job-private-payload.model.ts deleted file mode 100644 index 529034db8..000000000 --- a/shared/models/runners/runner-job-private-payload.model.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { VideoStudioTaskPayload } from '../server' - -export type RunnerJobVODPrivatePayload = - RunnerJobVODWebVideoTranscodingPrivatePayload | - RunnerJobVODAudioMergeTranscodingPrivatePayload | - RunnerJobVODHLSTranscodingPrivatePayload - -export type RunnerJobPrivatePayload = - RunnerJobVODPrivatePayload | - RunnerJobLiveRTMPHLSTranscodingPrivatePayload | - RunnerJobVideoStudioTranscodingPrivatePayload - -// --------------------------------------------------------------------------- - -export interface RunnerJobVODWebVideoTranscodingPrivatePayload { - videoUUID: string - isNewVideo: boolean -} - -export interface RunnerJobVODAudioMergeTranscodingPrivatePayload { - videoUUID: string - isNewVideo: boolean -} - -export interface RunnerJobVODHLSTranscodingPrivatePayload { - videoUUID: string - isNewVideo: boolean - deleteWebVideoFiles: boolean -} - -// --------------------------------------------------------------------------- - -export interface RunnerJobLiveRTMPHLSTranscodingPrivatePayload { - videoUUID: string - masterPlaylistName: string - outputDirectory: string - sessionId: string -} - -// --------------------------------------------------------------------------- - -export interface RunnerJobVideoStudioTranscodingPrivatePayload { - videoUUID: string - originalTasks: VideoStudioTaskPayload[] -} diff --git a/shared/models/runners/runner-job-state.model.ts b/shared/models/runners/runner-job-state.model.ts deleted file mode 100644 index 7ed34b3bf..000000000 --- a/shared/models/runners/runner-job-state.model.ts +++ /dev/null @@ -1,11 +0,0 @@ -export enum RunnerJobState { - PENDING = 1, - PROCESSING = 2, - COMPLETED = 3, - ERRORED = 4, - WAITING_FOR_PARENT_JOB = 5, - CANCELLED = 6, - PARENT_ERRORED = 7, - PARENT_CANCELLED = 8, - COMPLETING = 9 -} diff --git a/shared/models/runners/runner-job.model.ts b/shared/models/runners/runner-job.model.ts deleted file mode 100644 index 080093563..000000000 --- a/shared/models/runners/runner-job.model.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { VideoConstant } from '../videos' -import { RunnerJobPayload } from './runner-job-payload.model' -import { RunnerJobPrivatePayload } from './runner-job-private-payload.model' -import { RunnerJobState } from './runner-job-state.model' -import { RunnerJobType } from './runner-job-type.type' - -export interface RunnerJob { - uuid: string - - type: RunnerJobType - - state: VideoConstant - - payload: T - - failures: number - error: string | null - - progress: number - priority: number - - startedAt: Date | string - createdAt: Date | string - updatedAt: Date | string - finishedAt: Date | string - - parent?: { - type: RunnerJobType - state: VideoConstant - uuid: string - } - - // If associated to a runner - runner?: { - id: number - name: string - - description: string - } -} - -// eslint-disable-next-line max-len -export interface RunnerJobAdmin extends RunnerJob { - privatePayload: U -} diff --git a/shared/models/search/index.ts b/shared/models/search/index.ts deleted file mode 100644 index 50aeeddc8..000000000 --- a/shared/models/search/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from './boolean-both-query.model' -export * from './search-target-query.model' -export * from './videos-common-query.model' -export * from './video-channels-search-query.model' -export * from './video-playlists-search-query.model' -export * from './videos-search-query.model' diff --git a/shared/models/search/video-channels-search-query.model.ts b/shared/models/search/video-channels-search-query.model.ts deleted file mode 100644 index b68a1e80b..000000000 --- a/shared/models/search/video-channels-search-query.model.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { SearchTargetQuery } from './search-target-query.model' - -export interface VideoChannelsSearchQuery extends SearchTargetQuery { - search?: string - - start?: number - count?: number - sort?: string - - host?: string - handles?: string[] -} - -export interface VideoChannelsSearchQueryAfterSanitize extends VideoChannelsSearchQuery { - start: number - count: number - sort: string -} diff --git a/shared/models/search/video-playlists-search-query.model.ts b/shared/models/search/video-playlists-search-query.model.ts deleted file mode 100644 index d9027eb5b..000000000 --- a/shared/models/search/video-playlists-search-query.model.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { SearchTargetQuery } from './search-target-query.model' - -export interface VideoPlaylistsSearchQuery extends SearchTargetQuery { - search?: string - - start?: number - count?: number - sort?: string - - host?: string - - // UUIDs or short UUIDs - uuids?: string[] -} - -export interface VideoPlaylistsSearchQueryAfterSanitize extends VideoPlaylistsSearchQuery { - start: number - count: number - sort: string -} diff --git a/shared/models/search/videos-common-query.model.ts b/shared/models/search/videos-common-query.model.ts deleted file mode 100644 index f783d7534..000000000 --- a/shared/models/search/videos-common-query.model.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { VideoPrivacy } from '@shared/models' -import { VideoInclude } from '../videos/video-include.enum' -import { BooleanBothQuery } from './boolean-both-query.model' - -// These query parameters can be used with any endpoint that list videos -export interface VideosCommonQuery { - start?: number - count?: number - sort?: string - - nsfw?: BooleanBothQuery - - isLive?: boolean - - isLocal?: boolean - include?: VideoInclude - - categoryOneOf?: number[] - - licenceOneOf?: number[] - - languageOneOf?: string[] - - privacyOneOf?: VideoPrivacy[] - - tagsOneOf?: string[] - tagsAllOf?: string[] - - hasHLSFiles?: boolean - - hasWebtorrentFiles?: boolean // TODO: remove in v7 - hasWebVideoFiles?: boolean - - skipCount?: boolean - - search?: string - - excludeAlreadyWatched?: boolean -} - -export interface VideosCommonQueryAfterSanitize extends VideosCommonQuery { - start: number - count: number - sort: string -} diff --git a/shared/models/search/videos-search-query.model.ts b/shared/models/search/videos-search-query.model.ts deleted file mode 100644 index a5436879d..000000000 --- a/shared/models/search/videos-search-query.model.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { SearchTargetQuery } from './search-target-query.model' -import { VideosCommonQuery } from './videos-common-query.model' - -export interface VideosSearchQuery extends SearchTargetQuery, VideosCommonQuery { - search?: string - - host?: string - - startDate?: string // ISO 8601 - endDate?: string // ISO 8601 - - originallyPublishedStartDate?: string // ISO 8601 - originallyPublishedEndDate?: string // ISO 8601 - - durationMin?: number // seconds - durationMax?: number // seconds - - // UUIDs or short UUIDs - uuids?: string[] -} - -export interface VideosSearchQueryAfterSanitize extends VideosSearchQuery { - start: number - count: number - sort: string -} diff --git a/shared/models/server/client-log-create.model.ts b/shared/models/server/client-log-create.model.ts deleted file mode 100644 index c9dc65568..000000000 --- a/shared/models/server/client-log-create.model.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { ClientLogLevel } from './client-log-level.type' - -export interface ClientLogCreate { - message: string - url: string - level: ClientLogLevel - - stackTrace?: string - userAgent?: string - meta?: string -} diff --git a/shared/models/server/custom-config.model.ts b/shared/models/server/custom-config.model.ts deleted file mode 100644 index 0dbb46fa8..000000000 --- a/shared/models/server/custom-config.model.ts +++ /dev/null @@ -1,259 +0,0 @@ -import { NSFWPolicyType } from '../videos/nsfw-policy.type' -import { BroadcastMessageLevel } from './broadcast-message-level.type' - -export type ConfigResolutions = { - '144p': boolean - '240p': boolean - '360p': boolean - '480p': boolean - '720p': boolean - '1080p': boolean - '1440p': boolean - '2160p': boolean -} - -export interface CustomConfig { - instance: { - name: string - shortDescription: string - description: string - terms: string - codeOfConduct: string - - creationReason: string - moderationInformation: string - administrator: string - maintenanceLifetime: string - businessModel: string - hardwareInformation: string - - languages: string[] - categories: number[] - - isNSFW: boolean - defaultNSFWPolicy: NSFWPolicyType - - defaultClientRoute: string - - customizations: { - javascript?: string - css?: string - } - } - - theme: { - default: string - } - - services: { - twitter: { - username: string - whitelisted: boolean - } - } - - client: { - videos: { - miniature: { - preferAuthorDisplayName: boolean - } - } - - menu: { - login: { - redirectOnSingleExternalAuth: boolean - } - } - } - - cache: { - previews: { - size: number - } - - captions: { - size: number - } - - torrents: { - size: number - } - - storyboards: { - size: number - } - } - - signup: { - enabled: boolean - limit: number - requiresApproval: boolean - requiresEmailVerification: boolean - minimumAge: number - } - - admin: { - email: string - } - - contactForm: { - enabled: boolean - } - - user: { - history: { - videos: { - enabled: boolean - } - } - videoQuota: number - videoQuotaDaily: number - } - - videoChannels: { - maxPerUser: number - } - - transcoding: { - enabled: boolean - - allowAdditionalExtensions: boolean - allowAudioFiles: boolean - - remoteRunners: { - enabled: boolean - } - - threads: number - concurrency: number - - profile: string - - resolutions: ConfigResolutions & { '0p': boolean } - - alwaysTranscodeOriginalResolution: boolean - - webVideos: { - enabled: boolean - } - - hls: { - enabled: boolean - } - } - - live: { - enabled: boolean - - allowReplay: boolean - - latencySetting: { - enabled: boolean - } - - maxDuration: number - maxInstanceLives: number - maxUserLives: number - - transcoding: { - enabled: boolean - remoteRunners: { - enabled: boolean - } - threads: number - profile: string - resolutions: ConfigResolutions - alwaysTranscodeOriginalResolution: boolean - } - } - - videoStudio: { - enabled: boolean - - remoteRunners: { - enabled: boolean - } - } - - videoFile: { - update: { - enabled: boolean - } - } - - import: { - videos: { - concurrency: number - - http: { - enabled: boolean - } - torrent: { - enabled: boolean - } - } - videoChannelSynchronization: { - enabled: boolean - maxPerUser: number - } - } - - trending: { - videos: { - algorithms: { - enabled: string[] - default: string - } - } - } - - autoBlacklist: { - videos: { - ofUsers: { - enabled: boolean - } - } - } - - followers: { - instance: { - enabled: boolean - manualApproval: boolean - } - } - - followings: { - instance: { - autoFollowBack: { - enabled: boolean - } - - autoFollowIndex: { - enabled: boolean - indexUrl: string - } - } - } - - broadcastMessage: { - enabled: boolean - message: string - level: BroadcastMessageLevel - dismissable: boolean - } - - search: { - remoteUri: { - users: boolean - anonymous: boolean - } - - searchIndex: { - enabled: boolean - url: string - disableLocalSearch: boolean - isDefaultSearch: boolean - } - } - -} diff --git a/shared/models/server/index.ts b/shared/models/server/index.ts deleted file mode 100644 index a9136f3d4..000000000 --- a/shared/models/server/index.ts +++ /dev/null @@ -1,16 +0,0 @@ -export * from './about.model' -export * from './broadcast-message-level.type' -export * from './client-log-create.model' -export * from './client-log-level.type' -export * from './contact-form.model' -export * from './custom-config.model' -export * from './debug.model' -export * from './emailer.model' -export * from './job.model' -export * from './peertube-problem-document.model' -export * from './server-config.model' -export * from './server-debug.model' -export * from './server-error-code.enum' -export * from './server-follow-create.model' -export * from './server-log-level.type' -export * from './server-stats.model' diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts deleted file mode 100644 index c14806dab..000000000 --- a/shared/models/server/job.model.ts +++ /dev/null @@ -1,304 +0,0 @@ -import { ContextType } from '../activitypub/context' -import { VideoState } from '../videos' -import { VideoResolution } from '../videos/file/video-resolution.enum' -import { VideoStudioTaskCut } from '../videos/studio' -import { SendEmailOptions } from './emailer.model' - -export type JobState = 'active' | 'completed' | 'failed' | 'waiting' | 'delayed' | 'paused' | 'waiting-children' - -export type JobType = - | 'activitypub-cleaner' - | 'activitypub-follow' - | 'activitypub-http-broadcast-parallel' - | 'activitypub-http-broadcast' - | 'activitypub-http-fetcher' - | 'activitypub-http-unicast' - | 'activitypub-refresher' - | 'actor-keys' - | 'after-video-channel-import' - | 'email' - | 'federate-video' - | 'transcoding-job-builder' - | 'manage-video-torrent' - | 'move-to-object-storage' - | 'notify' - | 'video-channel-import' - | 'video-file-import' - | 'video-import' - | 'video-live-ending' - | 'video-redundancy' - | 'video-studio-edition' - | 'video-transcoding' - | 'videos-views-stats' - | 'generate-video-storyboard' - -export interface Job { - id: number | string - state: JobState | 'unknown' - type: JobType - data: any - priority: number - progress: number - error: any - createdAt: Date | string - finishedOn: Date | string - processedOn: Date | string - - parent?: { - id: string - } -} - -export type ActivitypubHttpBroadcastPayload = { - uris: string[] - contextType: ContextType - body: any - signatureActorId?: number -} - -export type ActivitypubFollowPayload = { - followerActorId: number - name: string - host: string - isAutoFollow?: boolean - assertIsChannel?: boolean -} - -export type FetchType = 'activity' | 'video-shares' | 'video-comments' | 'account-playlists' -export type ActivitypubHttpFetcherPayload = { - uri: string - type: FetchType - videoId?: number -} - -export type ActivitypubHttpUnicastPayload = { - uri: string - contextType: ContextType - signatureActorId?: number - body: object -} - -export type RefreshPayload = { - type: 'video' | 'video-playlist' | 'actor' - url: string -} - -export type EmailPayload = SendEmailOptions - -export type VideoFileImportPayload = { - videoUUID: string - filePath: string -} - -// --------------------------------------------------------------------------- - -export type VideoImportTorrentPayloadType = 'magnet-uri' | 'torrent-file' -export type VideoImportYoutubeDLPayloadType = 'youtube-dl' - -export interface VideoImportYoutubeDLPayload { - type: VideoImportYoutubeDLPayloadType - videoImportId: number - - fileExt?: string -} - -export interface VideoImportTorrentPayload { - type: VideoImportTorrentPayloadType - videoImportId: number -} - -export type VideoImportPayload = (VideoImportYoutubeDLPayload | VideoImportTorrentPayload) & { - preventException: boolean -} - -export interface VideoImportPreventExceptionResult { - resultType: 'success' | 'error' -} - -// --------------------------------------------------------------------------- - -export type VideoRedundancyPayload = { - videoId: number -} - -export type ManageVideoTorrentPayload = - { - action: 'create' - videoId: number - videoFileId: number - } | { - action: 'update-metadata' - - videoId?: number - streamingPlaylistId?: number - - videoFileId: number - } - -// Video transcoding payloads - -interface BaseTranscodingPayload { - videoUUID: string - isNewVideo?: boolean -} - -export interface HLSTranscodingPayload extends BaseTranscodingPayload { - type: 'new-resolution-to-hls' - resolution: VideoResolution - fps: number - copyCodecs: boolean - - deleteWebVideoFiles: boolean -} - -export interface NewWebVideoResolutionTranscodingPayload extends BaseTranscodingPayload { - type: 'new-resolution-to-web-video' - resolution: VideoResolution - fps: number -} - -export interface MergeAudioTranscodingPayload extends BaseTranscodingPayload { - type: 'merge-audio-to-web-video' - - resolution: VideoResolution - fps: number - - hasChildren: boolean -} - -export interface OptimizeTranscodingPayload extends BaseTranscodingPayload { - type: 'optimize-to-web-video' - - quickTranscode: boolean - - hasChildren: boolean -} - -export type VideoTranscodingPayload = - HLSTranscodingPayload - | NewWebVideoResolutionTranscodingPayload - | OptimizeTranscodingPayload - | MergeAudioTranscodingPayload - -export interface VideoLiveEndingPayload { - videoId: number - publishedAt: string - liveSessionId: number - streamingPlaylistId: number - - replayDirectory?: string -} - -export interface ActorKeysPayload { - actorId: number -} - -export interface DeleteResumableUploadMetaFilePayload { - filepath: string -} - -export interface MoveObjectStoragePayload { - videoUUID: string - isNewVideo: boolean - previousVideoState: VideoState -} - -export type VideoStudioTaskCutPayload = VideoStudioTaskCut - -export type VideoStudioTaskIntroPayload = { - name: 'add-intro' - - options: { - file: string - } -} - -export type VideoStudioTaskOutroPayload = { - name: 'add-outro' - - options: { - file: string - } -} - -export type VideoStudioTaskWatermarkPayload = { - name: 'add-watermark' - - options: { - file: string - - watermarkSizeRatio: number - horitonzalMarginRatio: number - verticalMarginRatio: number - } -} - -export type VideoStudioTaskPayload = - VideoStudioTaskCutPayload | - VideoStudioTaskIntroPayload | - VideoStudioTaskOutroPayload | - VideoStudioTaskWatermarkPayload - -export interface VideoStudioEditionPayload { - videoUUID: string - tasks: VideoStudioTaskPayload[] -} - -// --------------------------------------------------------------------------- - -export interface VideoChannelImportPayload { - externalChannelUrl: string - videoChannelId: number - - partOfChannelSyncId?: number -} - -export interface AfterVideoChannelImportPayload { - channelSyncId: number -} - -// --------------------------------------------------------------------------- - -export type NotifyPayload = - { - action: 'new-video' - videoUUID: string - } - -// --------------------------------------------------------------------------- - -export interface FederateVideoPayload { - videoUUID: string - isNewVideo: boolean -} - -// --------------------------------------------------------------------------- - -export interface TranscodingJobBuilderPayload { - videoUUID: string - - optimizeJob?: { - isNewVideo: boolean - } - - // Array of jobs to create - jobs?: { - type: 'video-transcoding' - payload: VideoTranscodingPayload - priority?: number - }[] - - // Array of sequential jobs to create - sequentialJobs?: { - type: 'video-transcoding' - payload: VideoTranscodingPayload - priority?: number - }[][] -} - -// --------------------------------------------------------------------------- - -export interface GenerateStoryboardPayload { - videoUUID: string - federate: boolean -} diff --git a/shared/models/server/peertube-problem-document.model.ts b/shared/models/server/peertube-problem-document.model.ts deleted file mode 100644 index 83d9cea9b..000000000 --- a/shared/models/server/peertube-problem-document.model.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { HttpStatusCode } from '../../models' -import { OAuth2ErrorCode, ServerErrorCode } from './server-error-code.enum' - -export interface PeerTubeProblemDocumentData { - 'invalid-params'?: Record - - originUrl?: string - - keyId?: string - - targetUrl?: string - - actorUrl?: string - - // Feeds - format?: string - url?: string -} - -export interface PeerTubeProblemDocument extends PeerTubeProblemDocumentData { - type: string - title: string - - detail: string - // Compat PeerTube <= 3.2 - error: string - - status: HttpStatusCode - - docs?: string - code?: ServerErrorCode | OAuth2ErrorCode -} diff --git a/shared/models/server/server-config.model.ts b/shared/models/server/server-config.model.ts deleted file mode 100644 index 3f61e93b5..000000000 --- a/shared/models/server/server-config.model.ts +++ /dev/null @@ -1,305 +0,0 @@ -import { ClientScriptJSON } from '../plugins/plugin-package-json.model' -import { NSFWPolicyType } from '../videos/nsfw-policy.type' -import { VideoPrivacy } from '../videos/video-privacy.enum' -import { BroadcastMessageLevel } from './broadcast-message-level.type' - -export interface ServerConfigPlugin { - name: string - npmName: string - version: string - description: string - clientScripts: { [name: string]: ClientScriptJSON } -} - -export interface ServerConfigTheme extends ServerConfigPlugin { - css: string[] -} - -export interface RegisteredExternalAuthConfig { - npmName: string - name: string - version: string - authName: string - authDisplayName: string -} - -export interface RegisteredIdAndPassAuthConfig { - npmName: string - name: string - version: string - authName: string - weight: number -} - -export interface ServerConfig { - serverVersion: string - serverCommit?: string - - client: { - videos: { - miniature: { - displayAuthorAvatar: boolean - preferAuthorDisplayName: boolean - } - resumableUpload: { - maxChunkSize: number - } - } - - menu: { - login: { - redirectOnSingleExternalAuth: boolean - } - } - } - - defaults: { - publish: { - downloadEnabled: boolean - commentsEnabled: boolean - privacy: VideoPrivacy - licence: number - } - - p2p: { - webapp: { - enabled: boolean - } - - embed: { - enabled: boolean - } - } - } - - webadmin: { - configuration: { - edition: { - allowed: boolean - } - } - } - - instance: { - name: string - shortDescription: string - isNSFW: boolean - defaultNSFWPolicy: NSFWPolicyType - defaultClientRoute: string - customizations: { - javascript: string - css: string - } - } - - search: { - remoteUri: { - users: boolean - anonymous: boolean - } - - searchIndex: { - enabled: boolean - url: string - disableLocalSearch: boolean - isDefaultSearch: boolean - } - } - - plugin: { - registered: ServerConfigPlugin[] - - registeredExternalAuths: RegisteredExternalAuthConfig[] - - registeredIdAndPassAuths: RegisteredIdAndPassAuthConfig[] - } - - theme: { - registered: ServerConfigTheme[] - default: string - } - - email: { - enabled: boolean - } - - contactForm: { - enabled: boolean - } - - signup: { - allowed: boolean - allowedForCurrentIP: boolean - requiresEmailVerification: boolean - requiresApproval: boolean - minimumAge: number - } - - transcoding: { - hls: { - enabled: boolean - } - - web_videos: { - enabled: boolean - } - - enabledResolutions: number[] - - profile: string - availableProfiles: string[] - - remoteRunners: { - enabled: boolean - } - } - - live: { - enabled: boolean - - allowReplay: boolean - latencySetting: { - enabled: boolean - } - - maxDuration: number - maxInstanceLives: number - maxUserLives: number - - transcoding: { - enabled: boolean - - remoteRunners: { - enabled: boolean - } - - enabledResolutions: number[] - - profile: string - availableProfiles: string[] - } - - rtmp: { - port: number - } - } - - videoStudio: { - enabled: boolean - - remoteRunners: { - enabled: boolean - } - } - - videoFile: { - update: { - enabled: boolean - } - } - - import: { - videos: { - http: { - enabled: boolean - } - torrent: { - enabled: boolean - } - } - videoChannelSynchronization: { - enabled: boolean - } - } - - autoBlacklist: { - videos: { - ofUsers: { - enabled: boolean - } - } - } - - avatar: { - file: { - size: { - max: number - } - extensions: string[] - } - } - - banner: { - file: { - size: { - max: number - } - extensions: string[] - } - } - - video: { - image: { - size: { - max: number - } - extensions: string[] - } - file: { - extensions: string[] - } - } - - videoCaption: { - file: { - size: { - max: number - } - extensions: string[] - } - } - - user: { - videoQuota: number - videoQuotaDaily: number - } - - videoChannels: { - maxPerUser: number - } - - trending: { - videos: { - intervalDays: number - algorithms: { - enabled: string[] - default: string - } - } - } - - tracker: { - enabled: boolean - } - - followings: { - instance: { - autoFollowIndex: { - indexUrl: string - } - } - } - - broadcastMessage: { - enabled: boolean - message: string - level: BroadcastMessageLevel - dismissable: boolean - } - - homepage: { - enabled: boolean - } -} - -export type HTMLServerConfig = Omit diff --git a/shared/models/server/server-error-code.enum.ts b/shared/models/server/server-error-code.enum.ts deleted file mode 100644 index 583e8245f..000000000 --- a/shared/models/server/server-error-code.enum.ts +++ /dev/null @@ -1,89 +0,0 @@ -export const enum ServerErrorCode { - /** - * The simplest form of payload too large: when the file size is over the - * global file size limit - */ - MAX_FILE_SIZE_REACHED = 'max_file_size_reached', - - /** - * The payload is too large for the user quota set - */ - QUOTA_REACHED = 'quota_reached', - - /** - * Error yielded upon trying to access a video that is not federated, nor can - * be. This may be due to: remote videos on instances that are not followed by - * yours, and with your instance disallowing unknown instances being accessed. - */ - DOES_NOT_RESPECT_FOLLOW_CONSTRAINTS = 'does_not_respect_follow_constraints', - - LIVE_NOT_ENABLED = 'live_not_enabled', - LIVE_NOT_ALLOWING_REPLAY = 'live_not_allowing_replay', - LIVE_CONFLICTING_PERMANENT_AND_SAVE_REPLAY = 'live_conflicting_permanent_and_save_replay', - /** - * Pretty self-explanatory: the set maximum number of simultaneous lives was - * reached, and this error is typically there to inform the user trying to - * broadcast one. - */ - MAX_INSTANCE_LIVES_LIMIT_REACHED = 'max_instance_lives_limit_reached', - /** - * Pretty self-explanatory: the set maximum number of simultaneous lives FOR - * THIS USER was reached, and this error is typically there to inform the user - * trying to broadcast one. - */ - MAX_USER_LIVES_LIMIT_REACHED = 'max_user_lives_limit_reached', - - /** - * A torrent should have at most one correct video file. Any more and we will - * not be able to choose automatically. - */ - INCORRECT_FILES_IN_TORRENT = 'incorrect_files_in_torrent', - - COMMENT_NOT_ASSOCIATED_TO_VIDEO = 'comment_not_associated_to_video', - - MISSING_TWO_FACTOR = 'missing_two_factor', - INVALID_TWO_FACTOR = 'invalid_two_factor', - - ACCOUNT_WAITING_FOR_APPROVAL = 'account_waiting_for_approval', - ACCOUNT_APPROVAL_REJECTED = 'account_approval_rejected', - - RUNNER_JOB_NOT_IN_PROCESSING_STATE = 'runner_job_not_in_processing_state', - RUNNER_JOB_NOT_IN_PENDING_STATE = 'runner_job_not_in_pending_state', - UNKNOWN_RUNNER_TOKEN = 'unknown_runner_token', - - VIDEO_REQUIRES_PASSWORD = 'video_requires_password', - INCORRECT_VIDEO_PASSWORD = 'incorrect_video_password', - - VIDEO_ALREADY_BEING_TRANSCODED = 'video_already_being_transcoded' -} - -/** - * oauthjs/oauth2-server error codes - * @see https://datatracker.ietf.org/doc/html/rfc6749#section-5.2 - **/ -export const enum OAuth2ErrorCode { - /** - * The provided authorization grant (e.g., authorization code, resource owner - * credentials) or refresh token is invalid, expired, revoked, does not match - * the redirection URI used in the authorization request, or was issued to - * another client. - * - * @see https://github.com/oauthjs/node-oauth2-server/blob/master/lib/errors/invalid-grant-error.js - */ - INVALID_GRANT = 'invalid_grant', - - /** - * Client authentication failed (e.g., unknown client, no client authentication - * included, or unsupported authentication method). - * - * @see https://github.com/oauthjs/node-oauth2-server/blob/master/lib/errors/invalid-client-error.js - */ - INVALID_CLIENT = 'invalid_client', - - /** - * The access token provided is expired, revoked, malformed, or invalid for other reasons - * - * @see https://github.com/oauthjs/node-oauth2-server/blob/master/lib/errors/invalid-token-error.js - */ - INVALID_TOKEN = 'invalid_token' -} diff --git a/shared/models/server/server-stats.model.ts b/shared/models/server/server-stats.model.ts deleted file mode 100644 index 82f5a737f..000000000 --- a/shared/models/server/server-stats.model.ts +++ /dev/null @@ -1,47 +0,0 @@ -import { ActivityType } from '../activitypub' -import { VideoRedundancyStrategyWithManual } from '../redundancy' - -type ActivityPubMessagesSuccess = Record<`totalActivityPub${ActivityType}MessagesSuccesses`, number> -type ActivityPubMessagesErrors = Record<`totalActivityPub${ActivityType}MessagesErrors`, number> - -export interface ServerStats extends ActivityPubMessagesSuccess, ActivityPubMessagesErrors { - totalUsers: number - totalDailyActiveUsers: number - totalWeeklyActiveUsers: number - totalMonthlyActiveUsers: number - - totalLocalVideos: number - totalLocalVideoViews: number - totalLocalVideoComments: number - totalLocalVideoFilesSize: number - - totalVideos: number - totalVideoComments: number - - totalLocalVideoChannels: number - totalLocalDailyActiveVideoChannels: number - totalLocalWeeklyActiveVideoChannels: number - totalLocalMonthlyActiveVideoChannels: number - - totalLocalPlaylists: number - - totalInstanceFollowers: number - totalInstanceFollowing: number - - videosRedundancy: VideosRedundancyStats[] - - totalActivityPubMessagesProcessed: number - totalActivityPubMessagesSuccesses: number - totalActivityPubMessagesErrors: number - - activityPubMessagesProcessedPerSecond: number - totalActivityPubMessagesWaiting: number -} - -export interface VideosRedundancyStats { - strategy: VideoRedundancyStrategyWithManual - totalSize: number - totalUsed: number - totalVideoFiles: number - totalVideos: number -} diff --git a/shared/models/tokens/index.ts b/shared/models/tokens/index.ts deleted file mode 100644 index fe130f153..000000000 --- a/shared/models/tokens/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './oauth-client-local.model' diff --git a/shared/models/users/index.ts b/shared/models/users/index.ts deleted file mode 100644 index 4a050c870..000000000 --- a/shared/models/users/index.ts +++ /dev/null @@ -1,16 +0,0 @@ -export * from './registration' -export * from './two-factor-enable-result.model' -export * from './user-create-result.model' -export * from './user-create.model' -export * from './user-flag.model' -export * from './user-login.model' -export * from './user-notification-setting.model' -export * from './user-notification.model' -export * from './user-refresh-token.model' -export * from './user-right.enum' -export * from './user-role' -export * from './user-scoped-token' -export * from './user-update-me.model' -export * from './user-update.model' -export * from './user-video-quota.model' -export * from './user.model' diff --git a/shared/models/users/registration/index.ts b/shared/models/users/registration/index.ts deleted file mode 100644 index 593740c4f..000000000 --- a/shared/models/users/registration/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -export * from './user-register.model' -export * from './user-registration-request.model' -export * from './user-registration-state.model' -export * from './user-registration-update-state.model' -export * from './user-registration.model' diff --git a/shared/models/users/registration/user-registration-request.model.ts b/shared/models/users/registration/user-registration-request.model.ts deleted file mode 100644 index 6c38817e0..000000000 --- a/shared/models/users/registration/user-registration-request.model.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { UserRegister } from './user-register.model' - -export interface UserRegistrationRequest extends UserRegister { - registrationReason: string -} diff --git a/shared/models/users/registration/user-registration-state.model.ts b/shared/models/users/registration/user-registration-state.model.ts deleted file mode 100644 index e4c835f78..000000000 --- a/shared/models/users/registration/user-registration-state.model.ts +++ /dev/null @@ -1,5 +0,0 @@ -export const enum UserRegistrationState { - PENDING = 1, - REJECTED = 2, - ACCEPTED = 3 -} diff --git a/shared/models/users/registration/user-registration.model.ts b/shared/models/users/registration/user-registration.model.ts deleted file mode 100644 index 0d74dc28b..000000000 --- a/shared/models/users/registration/user-registration.model.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { UserRegistrationState } from './user-registration-state.model' - -export interface UserRegistration { - id: number - - state: { - id: UserRegistrationState - label: string - } - - registrationReason: string - moderationResponse: string - - username: string - email: string - emailVerified: boolean - - accountDisplayName: string - - channelHandle: string - channelDisplayName: string - - createdAt: Date - updatedAt: Date - - user?: { - id: number - } -} diff --git a/shared/models/users/user-create.model.ts b/shared/models/users/user-create.model.ts deleted file mode 100644 index ddc71dd59..000000000 --- a/shared/models/users/user-create.model.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { UserRole } from './user-role' -import { UserAdminFlag } from './user-flag.model' - -export interface UserCreate { - username: string - password: string - email: string - videoQuota: number - videoQuotaDaily: number - role: UserRole - adminFlags?: UserAdminFlag - channelName?: string -} diff --git a/shared/models/users/user-flag.model.ts b/shared/models/users/user-flag.model.ts deleted file mode 100644 index b791a1263..000000000 --- a/shared/models/users/user-flag.model.ts +++ /dev/null @@ -1,4 +0,0 @@ -export const enum UserAdminFlag { - NONE = 0, - BYPASS_VIDEO_AUTO_BLACKLIST = 1 << 0 -} diff --git a/shared/models/users/user-notification-setting.model.ts b/shared/models/users/user-notification-setting.model.ts deleted file mode 100644 index 278a05e7a..000000000 --- a/shared/models/users/user-notification-setting.model.ts +++ /dev/null @@ -1,32 +0,0 @@ -export const enum UserNotificationSettingValue { - NONE = 0, - WEB = 1 << 0, - EMAIL = 1 << 1 -} - -export interface UserNotificationSetting { - abuseAsModerator: UserNotificationSettingValue - videoAutoBlacklistAsModerator: UserNotificationSettingValue - newUserRegistration: UserNotificationSettingValue - - newVideoFromSubscription: UserNotificationSettingValue - - blacklistOnMyVideo: UserNotificationSettingValue - myVideoPublished: UserNotificationSettingValue - myVideoImportFinished: UserNotificationSettingValue - - commentMention: UserNotificationSettingValue - newCommentOnMyVideo: UserNotificationSettingValue - - newFollow: UserNotificationSettingValue - newInstanceFollower: UserNotificationSettingValue - autoInstanceFollowing: UserNotificationSettingValue - - abuseStateChange: UserNotificationSettingValue - abuseNewMessage: UserNotificationSettingValue - - newPeerTubeVersion: UserNotificationSettingValue - newPluginVersion: UserNotificationSettingValue - - myVideoStudioEditionFinished: UserNotificationSettingValue -} diff --git a/shared/models/users/user-notification.model.ts b/shared/models/users/user-notification.model.ts deleted file mode 100644 index 294c921bd..000000000 --- a/shared/models/users/user-notification.model.ts +++ /dev/null @@ -1,138 +0,0 @@ -import { FollowState } from '../actors' -import { AbuseState } from '../moderation' -import { PluginType } from '../plugins' - -export const enum UserNotificationType { - NEW_VIDEO_FROM_SUBSCRIPTION = 1, - NEW_COMMENT_ON_MY_VIDEO = 2, - NEW_ABUSE_FOR_MODERATORS = 3, - - BLACKLIST_ON_MY_VIDEO = 4, - UNBLACKLIST_ON_MY_VIDEO = 5, - - MY_VIDEO_PUBLISHED = 6, - - MY_VIDEO_IMPORT_SUCCESS = 7, - MY_VIDEO_IMPORT_ERROR = 8, - - NEW_USER_REGISTRATION = 9, - NEW_FOLLOW = 10, - COMMENT_MENTION = 11, - - VIDEO_AUTO_BLACKLIST_FOR_MODERATORS = 12, - - NEW_INSTANCE_FOLLOWER = 13, - - AUTO_INSTANCE_FOLLOWING = 14, - - ABUSE_STATE_CHANGE = 15, - - ABUSE_NEW_MESSAGE = 16, - - NEW_PLUGIN_VERSION = 17, - NEW_PEERTUBE_VERSION = 18, - - MY_VIDEO_STUDIO_EDITION_FINISHED = 19, - - NEW_USER_REGISTRATION_REQUEST = 20 -} - -export interface VideoInfo { - id: number - uuid: string - shortUUID: string - name: string -} - -export interface AvatarInfo { - width: number - path: string -} - -export interface ActorInfo { - id: number - displayName: string - name: string - host: string - - avatars: AvatarInfo[] - avatar: AvatarInfo -} - -export interface UserNotification { - id: number - type: UserNotificationType - read: boolean - - video?: VideoInfo & { - channel: ActorInfo - } - - videoImport?: { - id: number - video?: VideoInfo - torrentName?: string - magnetUri?: string - targetUrl?: string - } - - comment?: { - id: number - threadId: number - account: ActorInfo - video: VideoInfo - } - - abuse?: { - id: number - state: AbuseState - - video?: VideoInfo - - comment?: { - threadId: number - - video: VideoInfo - } - - account?: ActorInfo - } - - videoBlacklist?: { - id: number - video: VideoInfo - } - - account?: ActorInfo - - actorFollow?: { - id: number - follower: ActorInfo - state: FollowState - - following: { - type: 'account' | 'channel' | 'instance' - name: string - displayName: string - host: string - } - } - - plugin?: { - name: string - type: PluginType - latestVersion: string - } - - peertube?: { - latestVersion: string - } - - registration?: { - id: number - username: string - } - - createdAt: string - updatedAt: string -} diff --git a/shared/models/users/user-right.enum.ts b/shared/models/users/user-right.enum.ts deleted file mode 100644 index a5a770b75..000000000 --- a/shared/models/users/user-right.enum.ts +++ /dev/null @@ -1,51 +0,0 @@ -export const enum UserRight { - ALL = 0, - - MANAGE_USERS = 1, - - MANAGE_SERVER_FOLLOW = 2, - - MANAGE_LOGS = 3, - - MANAGE_DEBUG = 4, - - MANAGE_SERVER_REDUNDANCY = 5, - - MANAGE_ABUSES = 6, - - MANAGE_JOBS = 7, - - MANAGE_CONFIGURATION = 8, - MANAGE_INSTANCE_CUSTOM_PAGE = 9, - - MANAGE_ACCOUNTS_BLOCKLIST = 10, - MANAGE_SERVERS_BLOCKLIST = 11, - - MANAGE_VIDEO_BLACKLIST = 12, - MANAGE_ANY_VIDEO_CHANNEL = 13, - - REMOVE_ANY_VIDEO = 14, - REMOVE_ANY_VIDEO_PLAYLIST = 15, - REMOVE_ANY_VIDEO_COMMENT = 16, - - UPDATE_ANY_VIDEO = 17, - UPDATE_ANY_VIDEO_PLAYLIST = 18, - - GET_ANY_LIVE = 19, - SEE_ALL_VIDEOS = 20, - SEE_ALL_COMMENTS = 21, - CHANGE_VIDEO_OWNERSHIP = 22, - - MANAGE_PLUGINS = 23, - - MANAGE_VIDEOS_REDUNDANCIES = 24, - - MANAGE_VIDEO_FILES = 25, - RUN_VIDEO_TRANSCODING = 26, - - MANAGE_VIDEO_IMPORTS = 27, - - MANAGE_REGISTRATIONS = 28, - - MANAGE_RUNNERS = 29 -} diff --git a/shared/models/users/user-role.ts b/shared/models/users/user-role.ts deleted file mode 100644 index 687a2aa0d..000000000 --- a/shared/models/users/user-role.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Keep the order -export const enum UserRole { - ADMINISTRATOR = 0, - MODERATOR = 1, - USER = 2 -} diff --git a/shared/models/users/user-update-me.model.ts b/shared/models/users/user-update-me.model.ts deleted file mode 100644 index c1d5ffba4..000000000 --- a/shared/models/users/user-update-me.model.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { NSFWPolicyType } from '../videos/nsfw-policy.type' - -export interface UserUpdateMe { - displayName?: string - description?: string - nsfwPolicy?: NSFWPolicyType - - p2pEnabled?: boolean - - autoPlayVideo?: boolean - autoPlayNextVideo?: boolean - autoPlayNextVideoPlaylist?: boolean - videosHistoryEnabled?: boolean - videoLanguages?: string[] - - email?: string - emailPublic?: boolean - currentPassword?: string - password?: string - - theme?: string - - noInstanceConfigWarningModal?: boolean - noWelcomeModal?: boolean - noAccountSetupWarningModal?: boolean -} diff --git a/shared/models/users/user-update.model.ts b/shared/models/users/user-update.model.ts deleted file mode 100644 index 158738545..000000000 --- a/shared/models/users/user-update.model.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { UserRole } from './user-role' -import { UserAdminFlag } from './user-flag.model' - -export interface UserUpdate { - password?: string - email?: string - emailVerified?: boolean - videoQuota?: number - videoQuotaDaily?: number - role?: UserRole - adminFlags?: UserAdminFlag - pluginAuth?: string -} diff --git a/shared/models/users/user.model.ts b/shared/models/users/user.model.ts deleted file mode 100644 index 9de4118b4..000000000 --- a/shared/models/users/user.model.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { Account } from '../actors' -import { VideoChannel } from '../videos/channel/video-channel.model' -import { UserRole } from './user-role' -import { NSFWPolicyType } from '../videos/nsfw-policy.type' -import { UserNotificationSetting } from './user-notification-setting.model' -import { UserAdminFlag } from './user-flag.model' -import { VideoPlaylistType } from '../videos/playlist/video-playlist-type.model' - -export interface User { - id: number - username: string - email: string - pendingEmail: string | null - - emailVerified: boolean - emailPublic: boolean - nsfwPolicy: NSFWPolicyType - - adminFlags?: UserAdminFlag - - autoPlayVideo: boolean - autoPlayNextVideo: boolean - autoPlayNextVideoPlaylist: boolean - - p2pEnabled: boolean - - videosHistoryEnabled: boolean - videoLanguages: string[] - - role: { - id: UserRole - label: string - } - - videoQuota: number - videoQuotaDaily: number - videoQuotaUsed?: number - videoQuotaUsedDaily?: number - - videosCount?: number - - abusesCount?: number - abusesAcceptedCount?: number - abusesCreatedCount?: number - - videoCommentsCount?: number - - theme: string - - account: Account - notificationSettings?: UserNotificationSetting - videoChannels?: VideoChannel[] - - blocked: boolean - blockedReason?: string - - noInstanceConfigWarningModal: boolean - noWelcomeModal: boolean - noAccountSetupWarningModal: boolean - - createdAt: Date - - pluginAuth: string | null - - lastLoginDate: Date | null - - twoFactorEnabled: boolean -} - -export interface MyUserSpecialPlaylist { - id: number - name: string - type: VideoPlaylistType -} - -export interface MyUser extends User { - specialPlaylists: MyUserSpecialPlaylist[] -} diff --git a/shared/models/videos/blacklist/index.ts b/shared/models/videos/blacklist/index.ts deleted file mode 100644 index 66082be34..000000000 --- a/shared/models/videos/blacklist/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './video-blacklist.model' -export * from './video-blacklist-create.model' -export * from './video-blacklist-update.model' diff --git a/shared/models/videos/blacklist/video-blacklist.model.ts b/shared/models/videos/blacklist/video-blacklist.model.ts deleted file mode 100644 index 982a34592..000000000 --- a/shared/models/videos/blacklist/video-blacklist.model.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Video } from '../video.model' - -export const enum VideoBlacklistType { - MANUAL = 1, - AUTO_BEFORE_PUBLISHED = 2 -} - -export interface VideoBlacklist { - id: number - unfederated: boolean - reason?: string - type: VideoBlacklistType - - video: Video - - createdAt: Date - updatedAt: Date -} diff --git a/shared/models/videos/caption/index.ts b/shared/models/videos/caption/index.ts deleted file mode 100644 index 2a5ff512d..000000000 --- a/shared/models/videos/caption/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './video-caption.model' -export * from './video-caption-update.model' diff --git a/shared/models/videos/caption/video-caption.model.ts b/shared/models/videos/caption/video-caption.model.ts deleted file mode 100644 index 6d5665006..000000000 --- a/shared/models/videos/caption/video-caption.model.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { VideoConstant } from '../video-constant.model' - -export interface VideoCaption { - language: VideoConstant - captionPath: string - updatedAt: string -} diff --git a/shared/models/videos/change-ownership/index.ts b/shared/models/videos/change-ownership/index.ts deleted file mode 100644 index a942fb2cd..000000000 --- a/shared/models/videos/change-ownership/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './video-change-ownership-accept.model' -export * from './video-change-ownership-create.model' -export * from './video-change-ownership.model' diff --git a/shared/models/videos/change-ownership/video-change-ownership.model.ts b/shared/models/videos/change-ownership/video-change-ownership.model.ts deleted file mode 100644 index 3d31cad0a..000000000 --- a/shared/models/videos/change-ownership/video-change-ownership.model.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { Account } from '../../actors' -import { Video } from '../video.model' - -export interface VideoChangeOwnership { - id: number - status: VideoChangeOwnershipStatus - initiatorAccount: Account - nextOwnerAccount: Account - video: Video - createdAt: Date -} - -export const enum VideoChangeOwnershipStatus { - WAITING = 'WAITING', - ACCEPTED = 'ACCEPTED', - REFUSED = 'REFUSED' -} diff --git a/shared/models/videos/channel-sync/index.ts b/shared/models/videos/channel-sync/index.ts deleted file mode 100644 index 7d25aaac3..000000000 --- a/shared/models/videos/channel-sync/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './video-channel-sync-state.enum' -export * from './video-channel-sync.model' -export * from './video-channel-sync-create.model' diff --git a/shared/models/videos/channel-sync/video-channel-sync-state.enum.ts b/shared/models/videos/channel-sync/video-channel-sync-state.enum.ts deleted file mode 100644 index 3e9f5ddc2..000000000 --- a/shared/models/videos/channel-sync/video-channel-sync-state.enum.ts +++ /dev/null @@ -1,6 +0,0 @@ -export const enum VideoChannelSyncState { - WAITING_FIRST_RUN = 1, - PROCESSING = 2, - SYNCED = 3, - FAILED = 4 -} diff --git a/shared/models/videos/channel-sync/video-channel-sync.model.ts b/shared/models/videos/channel-sync/video-channel-sync.model.ts deleted file mode 100644 index 73ac0615b..000000000 --- a/shared/models/videos/channel-sync/video-channel-sync.model.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { VideoChannelSummary } from '../channel/video-channel.model' -import { VideoConstant } from '../video-constant.model' -import { VideoChannelSyncState } from './video-channel-sync-state.enum' - -export interface VideoChannelSync { - id: number - - externalChannelUrl: string - - createdAt: string - channel: VideoChannelSummary - state: VideoConstant - lastSyncAt: string -} diff --git a/shared/models/videos/channel/index.ts b/shared/models/videos/channel/index.ts deleted file mode 100644 index 6cdabffbd..000000000 --- a/shared/models/videos/channel/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './video-channel-create-result.model' -export * from './video-channel-create.model' -export * from './video-channel-update.model' -export * from './video-channel.model' diff --git a/shared/models/videos/channel/video-channel.model.ts b/shared/models/videos/channel/video-channel.model.ts deleted file mode 100644 index ce5fc0e8d..000000000 --- a/shared/models/videos/channel/video-channel.model.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { Account, ActorImage } from '../../actors' -import { Actor } from '../../actors/actor.model' - -export type ViewsPerDate = { - date: Date - views: number -} - -export interface VideoChannel extends Actor { - displayName: string - description: string - support: string - isLocal: boolean - - updatedAt: Date | string - - ownerAccount?: Account - - videosCount?: number - viewsPerDay?: ViewsPerDate[] // chronologically ordered - totalViews?: number - - banners: ActorImage[] -} - -export interface VideoChannelSummary { - id: number - name: string - displayName: string - url: string - host: string - - avatars: ActorImage[] -} diff --git a/shared/models/videos/comment/index.ts b/shared/models/videos/comment/index.ts deleted file mode 100644 index 80c6c0724..000000000 --- a/shared/models/videos/comment/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './video-comment-create.model' -export * from './video-comment.model' diff --git a/shared/models/videos/comment/video-comment.model.ts b/shared/models/videos/comment/video-comment.model.ts deleted file mode 100644 index 737cfe098..000000000 --- a/shared/models/videos/comment/video-comment.model.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { ResultList } from '../../common' -import { Account } from '../../actors' - -export interface VideoComment { - id: number - url: string - text: string - threadId: number - inReplyToCommentId: number - videoId: number - createdAt: Date | string - updatedAt: Date | string - deletedAt: Date | string - isDeleted: boolean - totalRepliesFromVideoAuthor: number - totalReplies: number - account: Account -} - -export interface VideoCommentAdmin { - id: number - url: string - text: string - - threadId: number - inReplyToCommentId: number - - createdAt: Date | string - updatedAt: Date | string - - account: Account - - video: { - id: number - uuid: string - name: string - } -} - -export type VideoCommentThreads = ResultList & { totalNotDeletedComments: number } - -export interface VideoCommentThreadTree { - comment: VideoComment - children: VideoCommentThreadTree[] -} diff --git a/shared/models/videos/file/index.ts b/shared/models/videos/file/index.ts deleted file mode 100644 index 78a784a3c..000000000 --- a/shared/models/videos/file/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './video-file-metadata.model' -export * from './video-file.model' -export * from './video-resolution.enum' diff --git a/shared/models/videos/file/video-file.model.ts b/shared/models/videos/file/video-file.model.ts deleted file mode 100644 index 2bbff48eb..000000000 --- a/shared/models/videos/file/video-file.model.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { VideoConstant } from '../video-constant.model' -import { VideoFileMetadata } from './video-file-metadata.model' -import { VideoResolution } from './video-resolution.enum' - -export interface VideoFile { - id: number - - resolution: VideoConstant - size: number // Bytes - - torrentUrl: string - torrentDownloadUrl: string - - fileUrl: string - fileDownloadUrl: string - - fps: number - - metadata?: VideoFileMetadata - metadataUrl?: string - - magnetUri: string | null -} diff --git a/shared/models/videos/file/video-resolution.enum.ts b/shared/models/videos/file/video-resolution.enum.ts deleted file mode 100644 index 5b48ad353..000000000 --- a/shared/models/videos/file/video-resolution.enum.ts +++ /dev/null @@ -1,11 +0,0 @@ -export const enum VideoResolution { - H_NOVIDEO = 0, - H_144P = 144, - H_240P = 240, - H_360P = 360, - H_480P = 480, - H_720P = 720, - H_1080P = 1080, - H_1440P = 1440, - H_4K = 2160 -} diff --git a/shared/models/videos/import/index.ts b/shared/models/videos/import/index.ts deleted file mode 100644 index b38a67b5f..000000000 --- a/shared/models/videos/import/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './video-import-create.model' -export * from './video-import-state.enum' -export * from './video-import.model' -export * from './videos-import-in-channel-create.model' diff --git a/shared/models/videos/import/video-import-create.model.ts b/shared/models/videos/import/video-import-create.model.ts deleted file mode 100644 index 425477389..000000000 --- a/shared/models/videos/import/video-import-create.model.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { VideoUpdate } from '../video-update.model' - -export interface VideoImportCreate extends VideoUpdate { - targetUrl?: string - magnetUri?: string - torrentfile?: Blob - - channelId: number // Required -} diff --git a/shared/models/videos/import/video-import-state.enum.ts b/shared/models/videos/import/video-import-state.enum.ts deleted file mode 100644 index ff5c6beff..000000000 --- a/shared/models/videos/import/video-import-state.enum.ts +++ /dev/null @@ -1,8 +0,0 @@ -export const enum VideoImportState { - PENDING = 1, - SUCCESS = 2, - FAILED = 3, - REJECTED = 4, - CANCELLED = 5, - PROCESSING = 6 -} diff --git a/shared/models/videos/import/video-import.model.ts b/shared/models/videos/import/video-import.model.ts deleted file mode 100644 index 6aed7a91a..000000000 --- a/shared/models/videos/import/video-import.model.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { Video } from '../video.model' -import { VideoConstant } from '../video-constant.model' -import { VideoImportState } from './video-import-state.enum' - -export interface VideoImport { - id: number - - targetUrl: string - magnetUri: string - torrentName: string - - createdAt: string - updatedAt: string - originallyPublishedAt?: string - state: VideoConstant - error?: string - - video?: Video & { tags: string[] } - - videoChannelSync?: { - id: number - externalChannelUrl: string - } -} diff --git a/shared/models/videos/index.ts b/shared/models/videos/index.ts deleted file mode 100644 index f8f1ce081..000000000 --- a/shared/models/videos/index.ts +++ /dev/null @@ -1,42 +0,0 @@ -export * from './blacklist' -export * from './caption' -export * from './change-ownership' -export * from './channel' -export * from './comment' -export * from './studio' -export * from './live' -export * from './file' -export * from './import' -export * from './playlist' -export * from './rate' -export * from './stats' -export * from './transcoding' -export * from './channel-sync' - -export * from './nsfw-policy.type' - -export * from './storyboard.model' -export * from './thumbnail.type' - -export * from './video-constant.model' -export * from './video-create.model' - -export * from './video-privacy.enum' -export * from './video-include.enum' -export * from './video-rate.type' - -export * from './video-schedule-update.model' -export * from './video-sort-field.type' -export * from './video-state.enum' -export * from './video-storage.enum' - -export * from './video-streaming-playlist.model' -export * from './video-streaming-playlist.type' - -export * from './video-token.model' - -export * from './video-update.model' -export * from './video-view.model' -export * from './video.model' -export * from './video-create-result.model' -export * from './video-password.model' diff --git a/shared/models/videos/live/index.ts b/shared/models/videos/live/index.ts deleted file mode 100644 index 07b59fe2c..000000000 --- a/shared/models/videos/live/index.ts +++ /dev/null @@ -1,8 +0,0 @@ -export * from './live-video-create.model' -export * from './live-video-error.enum' -export * from './live-video-event-payload.model' -export * from './live-video-event.type' -export * from './live-video-latency-mode.enum' -export * from './live-video-session.model' -export * from './live-video-update.model' -export * from './live-video.model' diff --git a/shared/models/videos/live/live-video-create.model.ts b/shared/models/videos/live/live-video-create.model.ts deleted file mode 100644 index f8ae9e5a9..000000000 --- a/shared/models/videos/live/live-video-create.model.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { VideoCreate } from '../video-create.model' -import { VideoPrivacy } from '../video-privacy.enum' -import { LiveVideoLatencyMode } from './live-video-latency-mode.enum' - -export interface LiveVideoCreate extends VideoCreate { - permanentLive?: boolean - latencyMode?: LiveVideoLatencyMode - - saveReplay?: boolean - replaySettings?: { privacy: VideoPrivacy } -} diff --git a/shared/models/videos/live/live-video-error.enum.ts b/shared/models/videos/live/live-video-error.enum.ts deleted file mode 100644 index a26453505..000000000 --- a/shared/models/videos/live/live-video-error.enum.ts +++ /dev/null @@ -1,9 +0,0 @@ -export const enum LiveVideoError { - BAD_SOCKET_HEALTH = 1, - DURATION_EXCEEDED = 2, - QUOTA_EXCEEDED = 3, - FFMPEG_ERROR = 4, - BLACKLISTED = 5, - RUNNER_JOB_ERROR = 6, - RUNNER_JOB_CANCEL = 7 -} diff --git a/shared/models/videos/live/live-video-event-payload.model.ts b/shared/models/videos/live/live-video-event-payload.model.ts deleted file mode 100644 index 646856ac3..000000000 --- a/shared/models/videos/live/live-video-event-payload.model.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { VideoState } from '../video-state.enum' - -export interface LiveVideoEventPayload { - state?: VideoState - - viewers?: number -} diff --git a/shared/models/videos/live/live-video-latency-mode.enum.ts b/shared/models/videos/live/live-video-latency-mode.enum.ts deleted file mode 100644 index 4285e1d41..000000000 --- a/shared/models/videos/live/live-video-latency-mode.enum.ts +++ /dev/null @@ -1,5 +0,0 @@ -export const enum LiveVideoLatencyMode { - DEFAULT = 1, - HIGH_LATENCY = 2, - SMALL_LATENCY = 3 -} diff --git a/shared/models/videos/live/live-video-session.model.ts b/shared/models/videos/live/live-video-session.model.ts deleted file mode 100644 index 888c20a8a..000000000 --- a/shared/models/videos/live/live-video-session.model.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { VideoPrivacy } from '../video-privacy.enum' -import { LiveVideoError } from './live-video-error.enum' - -export interface LiveVideoSession { - id: number - - startDate: string - endDate: string - - error: LiveVideoError - - saveReplay: boolean - endingProcessed: boolean - - replaySettings?: { privacy: VideoPrivacy } - - replayVideo: { - id: number - uuid: string - shortUUID: string - } -} diff --git a/shared/models/videos/live/live-video-update.model.ts b/shared/models/videos/live/live-video-update.model.ts deleted file mode 100644 index d6aa6fb37..000000000 --- a/shared/models/videos/live/live-video-update.model.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { VideoPrivacy } from '../video-privacy.enum' -import { LiveVideoLatencyMode } from './live-video-latency-mode.enum' - -export interface LiveVideoUpdate { - permanentLive?: boolean - saveReplay?: boolean - replaySettings?: { privacy: VideoPrivacy } - latencyMode?: LiveVideoLatencyMode -} diff --git a/shared/models/videos/live/live-video.model.ts b/shared/models/videos/live/live-video.model.ts deleted file mode 100644 index fd8454123..000000000 --- a/shared/models/videos/live/live-video.model.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { VideoPrivacy } from '../video-privacy.enum' -import { LiveVideoLatencyMode } from './live-video-latency-mode.enum' - -export interface LiveVideo { - // If owner - rtmpUrl?: string - rtmpsUrl?: string - streamKey?: string - - saveReplay: boolean - replaySettings?: { privacy: VideoPrivacy } - permanentLive: boolean - latencyMode: LiveVideoLatencyMode -} diff --git a/shared/models/videos/playlist/index.ts b/shared/models/videos/playlist/index.ts deleted file mode 100644 index a9e8ce496..000000000 --- a/shared/models/videos/playlist/index.ts +++ /dev/null @@ -1,12 +0,0 @@ -export * from './video-exist-in-playlist.model' -export * from './video-playlist-create-result.model' -export * from './video-playlist-create.model' -export * from './video-playlist-element-create-result.model' -export * from './video-playlist-element-create.model' -export * from './video-playlist-element-update.model' -export * from './video-playlist-element.model' -export * from './video-playlist-privacy.model' -export * from './video-playlist-reorder.model' -export * from './video-playlist-type.model' -export * from './video-playlist-update.model' -export * from './video-playlist.model' diff --git a/shared/models/videos/playlist/video-playlist-create.model.ts b/shared/models/videos/playlist/video-playlist-create.model.ts deleted file mode 100644 index 67a33fa35..000000000 --- a/shared/models/videos/playlist/video-playlist-create.model.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { VideoPlaylistPrivacy } from './video-playlist-privacy.model' - -export interface VideoPlaylistCreate { - displayName: string - privacy: VideoPlaylistPrivacy - - description?: string - videoChannelId?: number - - thumbnailfile?: any -} diff --git a/shared/models/videos/playlist/video-playlist-element.model.ts b/shared/models/videos/playlist/video-playlist-element.model.ts deleted file mode 100644 index df9e3b5cf..000000000 --- a/shared/models/videos/playlist/video-playlist-element.model.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Video } from '../video.model' - -export const enum VideoPlaylistElementType { - REGULAR = 0, - DELETED = 1, - PRIVATE = 2, - UNAVAILABLE = 3 // Blacklisted, blocked by the user/instance, NSFW... -} - -export interface VideoPlaylistElement { - id: number - position: number - startTimestamp: number - stopTimestamp: number - - type: VideoPlaylistElementType - - video?: Video -} diff --git a/shared/models/videos/playlist/video-playlist-privacy.model.ts b/shared/models/videos/playlist/video-playlist-privacy.model.ts deleted file mode 100644 index 480e1f104..000000000 --- a/shared/models/videos/playlist/video-playlist-privacy.model.ts +++ /dev/null @@ -1,5 +0,0 @@ -export const enum VideoPlaylistPrivacy { - PUBLIC = 1, - UNLISTED = 2, - PRIVATE = 3 -} diff --git a/shared/models/videos/playlist/video-playlist-type.model.ts b/shared/models/videos/playlist/video-playlist-type.model.ts deleted file mode 100644 index 7f51a6354..000000000 --- a/shared/models/videos/playlist/video-playlist-type.model.ts +++ /dev/null @@ -1,4 +0,0 @@ -export const enum VideoPlaylistType { - REGULAR = 1, - WATCH_LATER = 2 -} diff --git a/shared/models/videos/playlist/video-playlist-update.model.ts b/shared/models/videos/playlist/video-playlist-update.model.ts deleted file mode 100644 index a6a3f74d9..000000000 --- a/shared/models/videos/playlist/video-playlist-update.model.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { VideoPlaylistPrivacy } from './video-playlist-privacy.model' - -export interface VideoPlaylistUpdate { - displayName?: string - privacy?: VideoPlaylistPrivacy - - description?: string - videoChannelId?: number - thumbnailfile?: any -} diff --git a/shared/models/videos/playlist/video-playlist.model.ts b/shared/models/videos/playlist/video-playlist.model.ts deleted file mode 100644 index b8a9955d9..000000000 --- a/shared/models/videos/playlist/video-playlist.model.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { AccountSummary } from '../../actors/index' -import { VideoChannelSummary, VideoConstant } from '..' -import { VideoPlaylistPrivacy } from './video-playlist-privacy.model' -import { VideoPlaylistType } from './video-playlist-type.model' - -export interface VideoPlaylist { - id: number - uuid: string - shortUUID: string - - isLocal: boolean - - url: string - - displayName: string - description: string - privacy: VideoConstant - - thumbnailPath: string - thumbnailUrl?: string - - videosLength: number - - type: VideoConstant - - embedPath: string - embedUrl?: string - - createdAt: Date | string - updatedAt: Date | string - - ownerAccount: AccountSummary - videoChannel?: VideoChannelSummary -} diff --git a/shared/models/videos/rate/account-video-rate.model.ts b/shared/models/videos/rate/account-video-rate.model.ts deleted file mode 100644 index e789367dc..000000000 --- a/shared/models/videos/rate/account-video-rate.model.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { UserVideoRateType } from './user-video-rate.type' -import { Video } from '../video.model' - -export interface AccountVideoRate { - video: Video - rating: UserVideoRateType -} diff --git a/shared/models/videos/rate/index.ts b/shared/models/videos/rate/index.ts deleted file mode 100644 index 06aa691bd..000000000 --- a/shared/models/videos/rate/index.ts +++ /dev/null @@ -1,5 +0,0 @@ - -export * from './user-video-rate-update.model' -export * from './user-video-rate.model' -export * from './account-video-rate.model' -export * from './user-video-rate.type' diff --git a/shared/models/videos/rate/user-video-rate-update.model.ts b/shared/models/videos/rate/user-video-rate-update.model.ts deleted file mode 100644 index 85e89271a..000000000 --- a/shared/models/videos/rate/user-video-rate-update.model.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { UserVideoRateType } from './user-video-rate.type' - -export interface UserVideoRateUpdate { - rating: UserVideoRateType -} diff --git a/shared/models/videos/rate/user-video-rate.model.ts b/shared/models/videos/rate/user-video-rate.model.ts deleted file mode 100644 index d39a1c3d5..000000000 --- a/shared/models/videos/rate/user-video-rate.model.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { UserVideoRateType } from './user-video-rate.type' - -export interface UserVideoRate { - videoId: number - rating: UserVideoRateType -} diff --git a/shared/models/videos/stats/index.ts b/shared/models/videos/stats/index.ts deleted file mode 100644 index a9b203f58..000000000 --- a/shared/models/videos/stats/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from './video-stats-overall-query.model' -export * from './video-stats-overall.model' -export * from './video-stats-retention.model' -export * from './video-stats-timeserie-query.model' -export * from './video-stats-timeserie-metric.type' -export * from './video-stats-timeserie.model' diff --git a/shared/models/videos/studio/index.ts b/shared/models/videos/studio/index.ts deleted file mode 100644 index a1eb98a49..000000000 --- a/shared/models/videos/studio/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './video-studio-create-edit.model' diff --git a/shared/models/videos/thumbnail.type.ts b/shared/models/videos/thumbnail.type.ts deleted file mode 100644 index 6907b2802..000000000 --- a/shared/models/videos/thumbnail.type.ts +++ /dev/null @@ -1,4 +0,0 @@ -export const enum ThumbnailType { - MINIATURE = 1, - PREVIEW = 2 -} diff --git a/shared/models/videos/transcoding/index.ts b/shared/models/videos/transcoding/index.ts deleted file mode 100644 index 14472d900..000000000 --- a/shared/models/videos/transcoding/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './video-transcoding-create.model' -export * from './video-transcoding-fps.model' -export * from './video-transcoding.model' diff --git a/shared/models/videos/transcoding/video-transcoding.model.ts b/shared/models/videos/transcoding/video-transcoding.model.ts deleted file mode 100644 index 91eacf8dc..000000000 --- a/shared/models/videos/transcoding/video-transcoding.model.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { VideoResolution } from '../file/video-resolution.enum' - -// Types used by plugins and ffmpeg-utils - -export type EncoderOptionsBuilderParams = { - input: string - - resolution: VideoResolution - - // If PeerTube applies a filter, transcoding profile must not copy input stream - canCopyAudio: boolean - canCopyVideo: boolean - - fps: number - - // Could be undefined if we could not get input bitrate (some RTMP streams for example) - inputBitrate: number - inputRatio: number - - // For lives - streamNum?: number -} - -export type EncoderOptionsBuilder = (params: EncoderOptionsBuilderParams) => Promise | EncoderOptions - -export interface EncoderOptions { - copy?: boolean // Copy stream? Default to false - - scaleFilter?: { - name: string - } - - inputOptions?: string[] - outputOptions?: string[] -} - -// All our encoders - -export interface EncoderProfile { - [ profile: string ]: T - - default: T -} - -export type AvailableEncoders = { - available: { - live: { - [ encoder: string ]: EncoderProfile - } - - vod: { - [ encoder: string ]: EncoderProfile - } - } - - encodersToTry: { - vod: { - video: string[] - audio: string[] - } - - live: { - video: string[] - audio: string[] - } - } -} diff --git a/shared/models/videos/video-create.model.ts b/shared/models/videos/video-create.model.ts deleted file mode 100644 index 7a34b5afe..000000000 --- a/shared/models/videos/video-create.model.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { VideoPrivacy } from './video-privacy.enum' -import { VideoScheduleUpdate } from './video-schedule-update.model' - -export interface VideoCreate { - name: string - channelId: number - - category?: number - licence?: number - language?: string - description?: string - support?: string - nsfw?: boolean - waitTranscoding?: boolean - tags?: string[] - commentsEnabled?: boolean - downloadEnabled?: boolean - privacy: VideoPrivacy - scheduleUpdate?: VideoScheduleUpdate - originallyPublishedAt?: Date | string - videoPasswords?: string[] - - thumbnailfile?: Blob | string - previewfile?: Blob | string -} diff --git a/shared/models/videos/video-include.enum.ts b/shared/models/videos/video-include.enum.ts deleted file mode 100644 index 32ee12e86..000000000 --- a/shared/models/videos/video-include.enum.ts +++ /dev/null @@ -1,8 +0,0 @@ -export const enum VideoInclude { - NONE = 0, - NOT_PUBLISHED_STATE = 1 << 0, - BLACKLISTED = 1 << 1, - BLOCKED_OWNER = 1 << 2, - FILES = 1 << 3, - CAPTIONS = 1 << 4 -} diff --git a/shared/models/videos/video-privacy.enum.ts b/shared/models/videos/video-privacy.enum.ts deleted file mode 100644 index 12e1d196f..000000000 --- a/shared/models/videos/video-privacy.enum.ts +++ /dev/null @@ -1,7 +0,0 @@ -export const enum VideoPrivacy { - PUBLIC = 1, - UNLISTED = 2, - PRIVATE = 3, - INTERNAL = 4, - PASSWORD_PROTECTED = 5 -} diff --git a/shared/models/videos/video-schedule-update.model.ts b/shared/models/videos/video-schedule-update.model.ts deleted file mode 100644 index 87d74f654..000000000 --- a/shared/models/videos/video-schedule-update.model.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { VideoPrivacy } from './video-privacy.enum' - -export interface VideoScheduleUpdate { - updateAt: Date | string - privacy?: VideoPrivacy.PUBLIC | VideoPrivacy.UNLISTED | VideoPrivacy.INTERNAL // Cannot schedule an update to PRIVATE -} diff --git a/shared/models/videos/video-state.enum.ts b/shared/models/videos/video-state.enum.ts deleted file mode 100644 index e45e4adc2..000000000 --- a/shared/models/videos/video-state.enum.ts +++ /dev/null @@ -1,11 +0,0 @@ -export const enum VideoState { - PUBLISHED = 1, - TO_TRANSCODE = 2, - TO_IMPORT = 3, - WAITING_FOR_LIVE = 4, - LIVE_ENDED = 5, - TO_MOVE_TO_EXTERNAL_STORAGE = 6, - TRANSCODING_FAILED = 7, - TO_MOVE_TO_EXTERNAL_STORAGE_FAILED = 8, - TO_EDIT = 9 -} diff --git a/shared/models/videos/video-storage.enum.ts b/shared/models/videos/video-storage.enum.ts deleted file mode 100644 index 7c6690db2..000000000 --- a/shared/models/videos/video-storage.enum.ts +++ /dev/null @@ -1,4 +0,0 @@ -export const enum VideoStorage { - FILE_SYSTEM, - OBJECT_STORAGE, -} diff --git a/shared/models/videos/video-streaming-playlist.model.ts b/shared/models/videos/video-streaming-playlist.model.ts deleted file mode 100644 index 11919a4ee..000000000 --- a/shared/models/videos/video-streaming-playlist.model.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { VideoStreamingPlaylistType } from './video-streaming-playlist.type' -import { VideoFile } from './file' - -export interface VideoStreamingPlaylist { - id: number - type: VideoStreamingPlaylistType - playlistUrl: string - segmentsSha256Url: string - - redundancies: { - baseUrl: string - }[] - - files: VideoFile[] -} diff --git a/shared/models/videos/video-streaming-playlist.type.ts b/shared/models/videos/video-streaming-playlist.type.ts deleted file mode 100644 index e2e2b93ea..000000000 --- a/shared/models/videos/video-streaming-playlist.type.ts +++ /dev/null @@ -1,3 +0,0 @@ -export const enum VideoStreamingPlaylistType { - HLS = 1 -} diff --git a/shared/models/videos/video-update.model.ts b/shared/models/videos/video-update.model.ts deleted file mode 100644 index 43537b5af..000000000 --- a/shared/models/videos/video-update.model.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { VideoPrivacy } from './video-privacy.enum' -import { VideoScheduleUpdate } from './video-schedule-update.model' - -export interface VideoUpdate { - name?: string - category?: number - licence?: number - language?: string - description?: string - support?: string - privacy?: VideoPrivacy - tags?: string[] - commentsEnabled?: boolean - downloadEnabled?: boolean - nsfw?: boolean - waitTranscoding?: boolean - channelId?: number - thumbnailfile?: Blob - previewfile?: Blob - scheduleUpdate?: VideoScheduleUpdate - originallyPublishedAt?: Date | string - videoPasswords?: string[] - - pluginData?: any -} diff --git a/shared/models/videos/video.model.ts b/shared/models/videos/video.model.ts deleted file mode 100644 index 7e5930067..000000000 --- a/shared/models/videos/video.model.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { Account, AccountSummary } from '../actors' -import { VideoChannel, VideoChannelSummary } from './channel/video-channel.model' -import { VideoFile } from './file' -import { VideoConstant } from './video-constant.model' -import { VideoPrivacy } from './video-privacy.enum' -import { VideoScheduleUpdate } from './video-schedule-update.model' -import { VideoState } from './video-state.enum' -import { VideoStreamingPlaylist } from './video-streaming-playlist.model' - -export interface Video extends Partial { - id: number - uuid: string - shortUUID: string - - createdAt: Date | string - updatedAt: Date | string - publishedAt: Date | string - originallyPublishedAt: Date | string - category: VideoConstant - licence: VideoConstant - language: VideoConstant - privacy: VideoConstant - - // Deprecated in 5.0 in favour of truncatedDescription - description: string - truncatedDescription: string - - duration: number - isLocal: boolean - name: string - - isLive: boolean - - thumbnailPath: string - thumbnailUrl?: string - - previewPath: string - previewUrl?: string - - embedPath: string - embedUrl?: string - - url: string - - views: number - viewers: number - - likes: number - dislikes: number - nsfw: boolean - - account: AccountSummary - channel: VideoChannelSummary - - userHistory?: { - currentTime: number - } - - pluginData?: any -} - -// Not included by default, needs query params -export interface VideoAdditionalAttributes { - waitTranscoding: boolean - state: VideoConstant - scheduledUpdate: VideoScheduleUpdate - - blacklisted: boolean - blacklistedReason: string - - blockedOwner: boolean - blockedServer: boolean - - files: VideoFile[] - streamingPlaylists: VideoStreamingPlaylist[] -} - -export interface VideoDetails extends Video { - // Deprecated in 5.0 - descriptionPath: string - - support: string - channel: VideoChannel - account: Account - tags: string[] - commentsEnabled: boolean - downloadEnabled: boolean - - // Not optional in details (unlike in parent Video) - waitTranscoding: boolean - state: VideoConstant - - trackerUrls: string[] - - files: VideoFile[] - streamingPlaylists: VideoStreamingPlaylist[] - - inputFileUpdatedAt: string | Date -} diff --git a/shared/server-commands/bulk/bulk-command.ts b/shared/server-commands/bulk/bulk-command.ts deleted file mode 100644 index b5c5673ce..000000000 --- a/shared/server-commands/bulk/bulk-command.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { BulkRemoveCommentsOfBody, HttpStatusCode } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class BulkCommand extends AbstractCommand { - - removeCommentsOf (options: OverrideCommandOptions & { - attributes: BulkRemoveCommentsOfBody - }) { - const { attributes } = options - - return this.postBodyRequest({ - ...options, - - path: '/api/v1/bulk/remove-comments-of', - fields: attributes, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } -} diff --git a/shared/server-commands/bulk/index.ts b/shared/server-commands/bulk/index.ts deleted file mode 100644 index 391597243..000000000 --- a/shared/server-commands/bulk/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './bulk-command' diff --git a/shared/server-commands/cli/cli-command.ts b/shared/server-commands/cli/cli-command.ts deleted file mode 100644 index ab9738174..000000000 --- a/shared/server-commands/cli/cli-command.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { exec } from 'child_process' -import { AbstractCommand } from '../shared' - -export class CLICommand extends AbstractCommand { - - static exec (command: string) { - return new Promise((res, rej) => { - exec(command, (err, stdout, _stderr) => { - if (err) return rej(err) - - return res(stdout) - }) - }) - } - - getEnv () { - return `NODE_ENV=test NODE_APP_INSTANCE=${this.server.internalServerNumber}` - } - - async execWithEnv (command: string, configOverride?: any) { - const prefix = configOverride - ? `NODE_CONFIG='${JSON.stringify(configOverride)}'` - : '' - - return CLICommand.exec(`${prefix} ${this.getEnv()} ${command}`) - } -} diff --git a/shared/server-commands/cli/index.ts b/shared/server-commands/cli/index.ts deleted file mode 100644 index 91b5abfbe..000000000 --- a/shared/server-commands/cli/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './cli-command' diff --git a/shared/server-commands/custom-pages/custom-pages-command.ts b/shared/server-commands/custom-pages/custom-pages-command.ts deleted file mode 100644 index cd869a8de..000000000 --- a/shared/server-commands/custom-pages/custom-pages-command.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { CustomPage, HttpStatusCode } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class CustomPagesCommand extends AbstractCommand { - - getInstanceHomepage (options: OverrideCommandOptions = {}) { - const path = '/api/v1/custom-pages/homepage/instance' - - return this.getRequestBody({ - ...options, - - path, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - updateInstanceHomepage (options: OverrideCommandOptions & { - content: string - }) { - const { content } = options - const path = '/api/v1/custom-pages/homepage/instance' - - return this.putBodyRequest({ - ...options, - - path, - fields: { content }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } -} diff --git a/shared/server-commands/custom-pages/index.ts b/shared/server-commands/custom-pages/index.ts deleted file mode 100644 index 58aed04f2..000000000 --- a/shared/server-commands/custom-pages/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './custom-pages-command' diff --git a/shared/server-commands/feeds/feeds-command.ts b/shared/server-commands/feeds/feeds-command.ts deleted file mode 100644 index 26763b43e..000000000 --- a/shared/server-commands/feeds/feeds-command.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { buildUUID } from '@shared/extra-utils' -import { HttpStatusCode } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -type FeedType = 'videos' | 'video-comments' | 'subscriptions' - -export class FeedCommand extends AbstractCommand { - - getXML (options: OverrideCommandOptions & { - feed: FeedType - ignoreCache: boolean - format?: string - }) { - const { feed, format, ignoreCache } = options - const path = '/feeds/' + feed + '.xml' - - const query: { [id: string]: string } = {} - - if (ignoreCache) query.v = buildUUID() - if (format) query.format = format - - return this.getRequestText({ - ...options, - - path, - query, - accept: 'application/xml', - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getPodcastXML (options: OverrideCommandOptions & { - ignoreCache: boolean - channelId: number - }) { - const { ignoreCache, channelId } = options - const path = `/feeds/podcast/videos.xml` - - const query: { [id: string]: string } = {} - - if (ignoreCache) query.v = buildUUID() - if (channelId) query.videoChannelId = channelId + '' - - return this.getRequestText({ - ...options, - - path, - query, - accept: 'application/xml', - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getJSON (options: OverrideCommandOptions & { - feed: FeedType - ignoreCache: boolean - query?: { [ id: string ]: any } - }) { - const { feed, query = {}, ignoreCache } = options - const path = '/feeds/' + feed + '.json' - - const cacheQuery = ignoreCache - ? { v: buildUUID() } - : {} - - return this.getRequestText({ - ...options, - - path, - query: { ...query, ...cacheQuery }, - accept: 'application/json', - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } -} diff --git a/shared/server-commands/feeds/index.ts b/shared/server-commands/feeds/index.ts deleted file mode 100644 index 662a22b6f..000000000 --- a/shared/server-commands/feeds/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './feeds-command' diff --git a/shared/server-commands/index.ts b/shared/server-commands/index.ts deleted file mode 100644 index a4581dbc0..000000000 --- a/shared/server-commands/index.ts +++ /dev/null @@ -1,14 +0,0 @@ -export * from './bulk' -export * from './cli' -export * from './custom-pages' -export * from './feeds' -export * from './logs' -export * from './moderation' -export * from './overviews' -export * from './requests' -export * from './runners' -export * from './search' -export * from './server' -export * from './socket' -export * from './users' -export * from './videos' diff --git a/shared/server-commands/logs/index.ts b/shared/server-commands/logs/index.ts deleted file mode 100644 index 69452d7f0..000000000 --- a/shared/server-commands/logs/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './logs-command' diff --git a/shared/server-commands/logs/logs-command.ts b/shared/server-commands/logs/logs-command.ts deleted file mode 100644 index 1c5de7f59..000000000 --- a/shared/server-commands/logs/logs-command.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { ClientLogCreate, HttpStatusCode, ServerLogLevel } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class LogsCommand extends AbstractCommand { - - createLogClient (options: OverrideCommandOptions & { payload: ClientLogCreate }) { - const path = '/api/v1/server/logs/client' - - return this.postBodyRequest({ - ...options, - - path, - fields: options.payload, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - getLogs (options: OverrideCommandOptions & { - startDate: Date - endDate?: Date - level?: ServerLogLevel - tagsOneOf?: string[] - }) { - const { startDate, endDate, tagsOneOf, level } = options - const path = '/api/v1/server/logs' - - return this.getRequestBody({ - ...options, - - path, - query: { startDate, endDate, level, tagsOneOf }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getAuditLogs (options: OverrideCommandOptions & { - startDate: Date - endDate?: Date - }) { - const { startDate, endDate } = options - - const path = '/api/v1/server/audit-logs' - - return this.getRequestBody({ - ...options, - - path, - query: { startDate, endDate }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - -} diff --git a/shared/server-commands/moderation/abuses-command.ts b/shared/server-commands/moderation/abuses-command.ts deleted file mode 100644 index 0db32ba46..000000000 --- a/shared/server-commands/moderation/abuses-command.ts +++ /dev/null @@ -1,228 +0,0 @@ -import { pick } from '@shared/core-utils' -import { - AbuseFilter, - AbuseMessage, - AbusePredefinedReasonsString, - AbuseState, - AbuseUpdate, - AbuseVideoIs, - AdminAbuse, - HttpStatusCode, - ResultList, - UserAbuse -} from '@shared/models' -import { unwrapBody } from '../requests/requests' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class AbusesCommand extends AbstractCommand { - - report (options: OverrideCommandOptions & { - reason: string - - accountId?: number - videoId?: number - commentId?: number - - predefinedReasons?: AbusePredefinedReasonsString[] - - startAt?: number - endAt?: number - }) { - const path = '/api/v1/abuses' - - const video = options.videoId - ? { - id: options.videoId, - startAt: options.startAt, - endAt: options.endAt - } - : undefined - - const comment = options.commentId - ? { id: options.commentId } - : undefined - - const account = options.accountId - ? { id: options.accountId } - : undefined - - const body = { - account, - video, - comment, - - reason: options.reason, - predefinedReasons: options.predefinedReasons - } - - return unwrapBody<{ abuse: { id: number } }>(this.postBodyRequest({ - ...options, - - path, - fields: body, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - })) - } - - getAdminList (options: OverrideCommandOptions & { - start?: number - count?: number - sort?: string - - id?: number - predefinedReason?: AbusePredefinedReasonsString - search?: string - filter?: AbuseFilter - state?: AbuseState - videoIs?: AbuseVideoIs - searchReporter?: string - searchReportee?: string - searchVideo?: string - searchVideoChannel?: string - } = {}) { - const toPick: (keyof typeof options)[] = [ - 'count', - 'filter', - 'id', - 'predefinedReason', - 'search', - 'searchReportee', - 'searchReporter', - 'searchVideo', - 'searchVideoChannel', - 'sort', - 'start', - 'state', - 'videoIs' - ] - - const path = '/api/v1/abuses' - - const defaultQuery = { sort: 'createdAt' } - const query = { ...defaultQuery, ...pick(options, toPick) } - - return this.getRequestBody>({ - ...options, - - path, - query, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getUserList (options: OverrideCommandOptions & { - start?: number - count?: number - sort?: string - - id?: number - search?: string - state?: AbuseState - }) { - const toPick: (keyof typeof options)[] = [ - 'id', - 'search', - 'state', - 'start', - 'count', - 'sort' - ] - - const path = '/api/v1/users/me/abuses' - - const defaultQuery = { sort: 'createdAt' } - const query = { ...defaultQuery, ...pick(options, toPick) } - - return this.getRequestBody>({ - ...options, - - path, - query, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - update (options: OverrideCommandOptions & { - abuseId: number - body: AbuseUpdate - }) { - const { abuseId, body } = options - const path = '/api/v1/abuses/' + abuseId - - return this.putBodyRequest({ - ...options, - - path, - fields: body, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - delete (options: OverrideCommandOptions & { - abuseId: number - }) { - const { abuseId } = options - const path = '/api/v1/abuses/' + abuseId - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - listMessages (options: OverrideCommandOptions & { - abuseId: number - }) { - const { abuseId } = options - const path = '/api/v1/abuses/' + abuseId + '/messages' - - return this.getRequestBody>({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - deleteMessage (options: OverrideCommandOptions & { - abuseId: number - messageId: number - }) { - const { abuseId, messageId } = options - const path = '/api/v1/abuses/' + abuseId + '/messages/' + messageId - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - addMessage (options: OverrideCommandOptions & { - abuseId: number - message: string - }) { - const { abuseId, message } = options - const path = '/api/v1/abuses/' + abuseId + '/messages' - - return this.postBodyRequest({ - ...options, - - path, - fields: { message }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - -} diff --git a/shared/server-commands/moderation/index.ts b/shared/server-commands/moderation/index.ts deleted file mode 100644 index b37643956..000000000 --- a/shared/server-commands/moderation/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './abuses-command' diff --git a/shared/server-commands/overviews/index.ts b/shared/server-commands/overviews/index.ts deleted file mode 100644 index e19551907..000000000 --- a/shared/server-commands/overviews/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './overviews-command' diff --git a/shared/server-commands/overviews/overviews-command.ts b/shared/server-commands/overviews/overviews-command.ts deleted file mode 100644 index 06b4892d2..000000000 --- a/shared/server-commands/overviews/overviews-command.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { HttpStatusCode, VideosOverview } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class OverviewsCommand extends AbstractCommand { - - getVideos (options: OverrideCommandOptions & { - page: number - }) { - const { page } = options - const path = '/api/v1/overviews/videos' - - const query = { page } - - return this.getRequestBody({ - ...options, - - path, - query, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } -} diff --git a/shared/server-commands/requests/index.ts b/shared/server-commands/requests/index.ts deleted file mode 100644 index 802982301..000000000 --- a/shared/server-commands/requests/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './requests' diff --git a/shared/server-commands/requests/requests.ts b/shared/server-commands/requests/requests.ts deleted file mode 100644 index 8227017eb..000000000 --- a/shared/server-commands/requests/requests.ts +++ /dev/null @@ -1,259 +0,0 @@ -/* eslint-disable @typescript-eslint/no-floating-promises */ - -import { decode } from 'querystring' -import request from 'supertest' -import { URL } from 'url' -import { buildAbsoluteFixturePath, pick } from '@shared/core-utils' -import { HttpStatusCode } from '@shared/models' - -export type CommonRequestParams = { - url: string - path?: string - contentType?: string - responseType?: string - range?: string - redirects?: number - accept?: string - host?: string - token?: string - headers?: { [ name: string ]: string } - type?: string - xForwardedFor?: string - expectedStatus?: HttpStatusCode -} - -function makeRawRequest (options: { - url: string - token?: string - expectedStatus?: HttpStatusCode - range?: string - query?: { [ id: string ]: string } - method?: 'GET' | 'POST' - headers?: { [ name: string ]: string } -}) { - const { host, protocol, pathname } = new URL(options.url) - - const reqOptions = { - url: `${protocol}//${host}`, - path: pathname, - contentType: undefined, - - ...pick(options, [ 'expectedStatus', 'range', 'token', 'query', 'headers' ]) - } - - if (options.method === 'POST') { - return makePostBodyRequest(reqOptions) - } - - return makeGetRequest(reqOptions) -} - -function makeGetRequest (options: CommonRequestParams & { - query?: any - rawQuery?: string -}) { - const req = request(options.url).get(options.path) - - if (options.query) req.query(options.query) - if (options.rawQuery) req.query(options.rawQuery) - - return buildRequest(req, { contentType: 'application/json', expectedStatus: HttpStatusCode.BAD_REQUEST_400, ...options }) -} - -function makeHTMLRequest (url: string, path: string) { - return makeGetRequest({ - url, - path, - accept: 'text/html', - expectedStatus: HttpStatusCode.OK_200 - }) -} - -function makeActivityPubGetRequest (url: string, path: string, expectedStatus = HttpStatusCode.OK_200) { - return makeGetRequest({ - url, - path, - expectedStatus, - accept: 'application/activity+json,text/html;q=0.9,\\*/\\*;q=0.8' - }) -} - -function makeDeleteRequest (options: CommonRequestParams & { - query?: any - rawQuery?: string -}) { - const req = request(options.url).delete(options.path) - - if (options.query) req.query(options.query) - if (options.rawQuery) req.query(options.rawQuery) - - return buildRequest(req, { accept: 'application/json', expectedStatus: HttpStatusCode.BAD_REQUEST_400, ...options }) -} - -function makeUploadRequest (options: CommonRequestParams & { - method?: 'POST' | 'PUT' - - fields: { [ fieldName: string ]: any } - attaches?: { [ attachName: string ]: any | any[] } -}) { - let req = options.method === 'PUT' - ? request(options.url).put(options.path) - : request(options.url).post(options.path) - - req = buildRequest(req, { accept: 'application/json', expectedStatus: HttpStatusCode.BAD_REQUEST_400, ...options }) - - buildFields(req, options.fields) - - Object.keys(options.attaches || {}).forEach(attach => { - const value = options.attaches[attach] - if (!value) return - - if (Array.isArray(value)) { - req.attach(attach, buildAbsoluteFixturePath(value[0]), value[1]) - } else { - req.attach(attach, buildAbsoluteFixturePath(value)) - } - }) - - return req -} - -function makePostBodyRequest (options: CommonRequestParams & { - fields?: { [ fieldName: string ]: any } -}) { - const req = request(options.url).post(options.path) - .send(options.fields) - - return buildRequest(req, { accept: 'application/json', expectedStatus: HttpStatusCode.BAD_REQUEST_400, ...options }) -} - -function makePutBodyRequest (options: { - url: string - path: string - token?: string - fields: { [ fieldName: string ]: any } - expectedStatus?: HttpStatusCode - headers?: { [name: string]: string } -}) { - const req = request(options.url).put(options.path) - .send(options.fields) - - return buildRequest(req, { accept: 'application/json', expectedStatus: HttpStatusCode.BAD_REQUEST_400, ...options }) -} - -function decodeQueryString (path: string) { - return decode(path.split('?')[1]) -} - -// --------------------------------------------------------------------------- - -function unwrapBody (test: request.Test): Promise { - return test.then(res => res.body) -} - -function unwrapText (test: request.Test): Promise { - return test.then(res => res.text) -} - -function unwrapBodyOrDecodeToJSON (test: request.Test): Promise { - return test.then(res => { - if (res.body instanceof Buffer) { - try { - return JSON.parse(new TextDecoder().decode(res.body)) - } catch (err) { - console.error('Cannot decode JSON.', { res, body: res.body instanceof Buffer ? res.body.toString() : res.body }) - throw err - } - } - - if (res.text) { - try { - return JSON.parse(res.text) - } catch (err) { - console.error('Cannot decode json', { res, text: res.text }) - throw err - } - } - - return res.body - }) -} - -function unwrapTextOrDecode (test: request.Test): Promise { - return test.then(res => res.text || new TextDecoder().decode(res.body)) -} - -// --------------------------------------------------------------------------- - -export { - makeHTMLRequest, - makeGetRequest, - decodeQueryString, - makeUploadRequest, - makePostBodyRequest, - makePutBodyRequest, - makeDeleteRequest, - makeRawRequest, - makeActivityPubGetRequest, - unwrapBody, - unwrapTextOrDecode, - unwrapBodyOrDecodeToJSON, - unwrapText -} - -// --------------------------------------------------------------------------- - -function buildRequest (req: request.Test, options: CommonRequestParams) { - if (options.contentType) req.set('Accept', options.contentType) - if (options.responseType) req.responseType(options.responseType) - if (options.token) req.set('Authorization', 'Bearer ' + options.token) - if (options.range) req.set('Range', options.range) - if (options.accept) req.set('Accept', options.accept) - if (options.host) req.set('Host', options.host) - if (options.redirects) req.redirects(options.redirects) - if (options.xForwardedFor) req.set('X-Forwarded-For', options.xForwardedFor) - if (options.type) req.type(options.type) - - Object.keys(options.headers || {}).forEach(name => { - req.set(name, options.headers[name]) - }) - - return req.expect(res => { - if (options.expectedStatus && res.status !== options.expectedStatus) { - const err = new Error(`Expected status ${options.expectedStatus}, got ${res.status}. ` + - `\nThe server responded: "${res.body?.error ?? res.text}".\n` + - 'You may take a closer look at the logs. To see how to do so, check out this page: ' + - 'https://github.com/Chocobozzz/PeerTube/blob/develop/support/doc/development/tests.md#debug-server-logs'); - - (err as any).res = res - - throw err - } - - return res - }) -} - -function buildFields (req: request.Test, fields: { [ fieldName: string ]: any }, namespace?: string) { - if (!fields) return - - let formKey: string - - for (const key of Object.keys(fields)) { - if (namespace) formKey = `${namespace}[${key}]` - else formKey = key - - if (fields[key] === undefined) continue - - if (Array.isArray(fields[key]) && fields[key].length === 0) { - req.field(key, []) - continue - } - - if (fields[key] !== null && typeof fields[key] === 'object') { - buildFields(req, fields[key], formKey) - } else { - req.field(formKey, fields[key]) - } - } -} diff --git a/shared/server-commands/runners/index.ts b/shared/server-commands/runners/index.ts deleted file mode 100644 index 9e8e1baf2..000000000 --- a/shared/server-commands/runners/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './runner-jobs-command' -export * from './runner-registration-tokens-command' -export * from './runners-command' diff --git a/shared/server-commands/runners/runner-jobs-command.ts b/shared/server-commands/runners/runner-jobs-command.ts deleted file mode 100644 index 0a0ffb5d3..000000000 --- a/shared/server-commands/runners/runner-jobs-command.ts +++ /dev/null @@ -1,294 +0,0 @@ -import { omit, pick, wait } from '@shared/core-utils' -import { - AbortRunnerJobBody, - AcceptRunnerJobBody, - AcceptRunnerJobResult, - ErrorRunnerJobBody, - HttpStatusCode, - isHLSTranscodingPayloadSuccess, - isLiveRTMPHLSTranscodingUpdatePayload, - isWebVideoOrAudioMergeTranscodingPayloadSuccess, - ListRunnerJobsQuery, - RequestRunnerJobBody, - RequestRunnerJobResult, - ResultList, - RunnerJobAdmin, - RunnerJobLiveRTMPHLSTranscodingPayload, - RunnerJobPayload, - RunnerJobState, - RunnerJobSuccessBody, - RunnerJobSuccessPayload, - RunnerJobType, - RunnerJobUpdateBody, - RunnerJobVODPayload -} from '@shared/models' -import { unwrapBody } from '../requests' -import { waitJobs } from '../server' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class RunnerJobsCommand extends AbstractCommand { - - list (options: OverrideCommandOptions & ListRunnerJobsQuery = {}) { - const path = '/api/v1/runners/jobs' - - return this.getRequestBody>({ - ...options, - - path, - query: pick(options, [ 'start', 'count', 'sort', 'search', 'stateOneOf' ]), - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - cancelByAdmin (options: OverrideCommandOptions & { jobUUID: string }) { - const path = '/api/v1/runners/jobs/' + options.jobUUID + '/cancel' - - return this.postBodyRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - deleteByAdmin (options: OverrideCommandOptions & { jobUUID: string }) { - const path = '/api/v1/runners/jobs/' + options.jobUUID - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - // --------------------------------------------------------------------------- - - request (options: OverrideCommandOptions & RequestRunnerJobBody) { - const path = '/api/v1/runners/jobs/request' - - return unwrapBody(this.postBodyRequest({ - ...options, - - path, - fields: pick(options, [ 'runnerToken' ]), - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - })) - } - - async requestVOD (options: OverrideCommandOptions & RequestRunnerJobBody) { - const vodTypes = new Set([ 'vod-audio-merge-transcoding', 'vod-hls-transcoding', 'vod-web-video-transcoding' ]) - - const { availableJobs } = await this.request(options) - - return { - availableJobs: availableJobs.filter(j => vodTypes.has(j.type)) - } as RequestRunnerJobResult - } - - async requestLive (options: OverrideCommandOptions & RequestRunnerJobBody) { - const vodTypes = new Set([ 'live-rtmp-hls-transcoding' ]) - - const { availableJobs } = await this.request(options) - - return { - availableJobs: availableJobs.filter(j => vodTypes.has(j.type)) - } as RequestRunnerJobResult - } - - // --------------------------------------------------------------------------- - - accept (options: OverrideCommandOptions & AcceptRunnerJobBody & { jobUUID: string }) { - const path = '/api/v1/runners/jobs/' + options.jobUUID + '/accept' - - return unwrapBody>(this.postBodyRequest({ - ...options, - - path, - fields: pick(options, [ 'runnerToken' ]), - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - })) - } - - abort (options: OverrideCommandOptions & AbortRunnerJobBody & { jobUUID: string }) { - const path = '/api/v1/runners/jobs/' + options.jobUUID + '/abort' - - return this.postBodyRequest({ - ...options, - - path, - fields: pick(options, [ 'reason', 'jobToken', 'runnerToken' ]), - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - update (options: OverrideCommandOptions & RunnerJobUpdateBody & { jobUUID: string }) { - const path = '/api/v1/runners/jobs/' + options.jobUUID + '/update' - - const { payload } = options - const attaches: { [id: string]: any } = {} - let payloadWithoutFiles = payload - - if (isLiveRTMPHLSTranscodingUpdatePayload(payload)) { - if (payload.masterPlaylistFile) { - attaches[`payload[masterPlaylistFile]`] = payload.masterPlaylistFile - } - - attaches[`payload[resolutionPlaylistFile]`] = payload.resolutionPlaylistFile - attaches[`payload[videoChunkFile]`] = payload.videoChunkFile - - payloadWithoutFiles = omit(payloadWithoutFiles as any, [ 'masterPlaylistFile', 'resolutionPlaylistFile', 'videoChunkFile' ]) - } - - return this.postUploadRequest({ - ...options, - - path, - fields: { - ...pick(options, [ 'progress', 'jobToken', 'runnerToken' ]), - - payload: payloadWithoutFiles - }, - attaches, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - error (options: OverrideCommandOptions & ErrorRunnerJobBody & { jobUUID: string }) { - const path = '/api/v1/runners/jobs/' + options.jobUUID + '/error' - - return this.postBodyRequest({ - ...options, - - path, - fields: pick(options, [ 'message', 'jobToken', 'runnerToken' ]), - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - success (options: OverrideCommandOptions & RunnerJobSuccessBody & { jobUUID: string }) { - const { payload } = options - - const path = '/api/v1/runners/jobs/' + options.jobUUID + '/success' - const attaches: { [id: string]: any } = {} - let payloadWithoutFiles = payload - - if ((isWebVideoOrAudioMergeTranscodingPayloadSuccess(payload) || isHLSTranscodingPayloadSuccess(payload)) && payload.videoFile) { - attaches[`payload[videoFile]`] = payload.videoFile - - payloadWithoutFiles = omit(payloadWithoutFiles as any, [ 'videoFile' ]) - } - - if (isHLSTranscodingPayloadSuccess(payload) && payload.resolutionPlaylistFile) { - attaches[`payload[resolutionPlaylistFile]`] = payload.resolutionPlaylistFile - - payloadWithoutFiles = omit(payloadWithoutFiles as any, [ 'resolutionPlaylistFile' ]) - } - - return this.postUploadRequest({ - ...options, - - path, - attaches, - fields: { - ...pick(options, [ 'jobToken', 'runnerToken' ]), - - payload: payloadWithoutFiles - }, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - getJobFile (options: OverrideCommandOptions & { url: string, jobToken: string, runnerToken: string }) { - const { host, protocol, pathname } = new URL(options.url) - - return this.postBodyRequest({ - url: `${protocol}//${host}`, - path: pathname, - - fields: pick(options, [ 'jobToken', 'runnerToken' ]), - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - // --------------------------------------------------------------------------- - - async autoAccept (options: OverrideCommandOptions & RequestRunnerJobBody & { type?: RunnerJobType }) { - const { availableJobs } = await this.request(options) - - const job = options.type - ? availableJobs.find(j => j.type === options.type) - : availableJobs[0] - - return this.accept({ ...options, jobUUID: job.uuid }) - } - - async autoProcessWebVideoJob (runnerToken: string, jobUUIDToProcess?: string) { - let jobUUID = jobUUIDToProcess - - if (!jobUUID) { - const { availableJobs } = await this.request({ runnerToken }) - jobUUID = availableJobs[0].uuid - } - - const { job } = await this.accept({ runnerToken, jobUUID }) - const jobToken = job.jobToken - - const payload: RunnerJobSuccessPayload = { videoFile: 'video_short.mp4' } - await this.success({ runnerToken, jobUUID, jobToken, payload }) - - await waitJobs([ this.server ]) - - return job - } - - async cancelAllJobs (options: { state?: RunnerJobState } = {}) { - const { state } = options - - const { data } = await this.list({ count: 100 }) - - const allowedStates = new Set([ - RunnerJobState.PENDING, - RunnerJobState.PROCESSING, - RunnerJobState.WAITING_FOR_PARENT_JOB - ]) - - for (const job of data) { - if (state && job.state.id !== state) continue - else if (allowedStates.has(job.state.id) !== true) continue - - await this.cancelByAdmin({ jobUUID: job.uuid }) - } - } - - async getJob (options: OverrideCommandOptions & { uuid: string }) { - const { data } = await this.list({ ...options, count: 100, sort: '-updatedAt' }) - - return data.find(j => j.uuid === options.uuid) - } - - async requestLiveJob (runnerToken: string) { - let availableJobs: RequestRunnerJobResult['availableJobs'] = [] - - while (availableJobs.length === 0) { - const result = await this.requestLive({ runnerToken }) - availableJobs = result.availableJobs - - if (availableJobs.length === 1) break - - await wait(150) - } - - return availableJobs[0] - } -} diff --git a/shared/server-commands/runners/runner-registration-tokens-command.ts b/shared/server-commands/runners/runner-registration-tokens-command.ts deleted file mode 100644 index e4f2e3d95..000000000 --- a/shared/server-commands/runners/runner-registration-tokens-command.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { pick } from '@shared/core-utils' -import { HttpStatusCode, ResultList, RunnerRegistrationToken } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class RunnerRegistrationTokensCommand extends AbstractCommand { - - list (options: OverrideCommandOptions & { - start?: number - count?: number - sort?: string - } = {}) { - const path = '/api/v1/runners/registration-tokens' - - return this.getRequestBody>({ - ...options, - - path, - query: pick(options, [ 'start', 'count', 'sort' ]), - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - generate (options: OverrideCommandOptions = {}) { - const path = '/api/v1/runners/registration-tokens/generate' - - return this.postBodyRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - delete (options: OverrideCommandOptions & { - id: number - }) { - const path = '/api/v1/runners/registration-tokens/' + options.id - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - async getFirstRegistrationToken (options: OverrideCommandOptions = {}) { - const { data } = await this.list(options) - - return data[0].registrationToken - } -} diff --git a/shared/server-commands/runners/runners-command.ts b/shared/server-commands/runners/runners-command.ts deleted file mode 100644 index b0083e841..000000000 --- a/shared/server-commands/runners/runners-command.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { pick } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { HttpStatusCode, RegisterRunnerBody, RegisterRunnerResult, ResultList, Runner, UnregisterRunnerBody } from '@shared/models' -import { unwrapBody } from '../requests' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class RunnersCommand extends AbstractCommand { - - list (options: OverrideCommandOptions & { - start?: number - count?: number - sort?: string - } = {}) { - const path = '/api/v1/runners' - - return this.getRequestBody>({ - ...options, - - path, - query: pick(options, [ 'start', 'count', 'sort' ]), - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - register (options: OverrideCommandOptions & RegisterRunnerBody) { - const path = '/api/v1/runners/register' - - return unwrapBody(this.postBodyRequest({ - ...options, - - path, - fields: pick(options, [ 'name', 'registrationToken', 'description' ]), - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - })) - } - - unregister (options: OverrideCommandOptions & UnregisterRunnerBody) { - const path = '/api/v1/runners/unregister' - - return this.postBodyRequest({ - ...options, - - path, - fields: pick(options, [ 'runnerToken' ]), - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - delete (options: OverrideCommandOptions & { - id: number - }) { - const path = '/api/v1/runners/' + options.id - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - // --------------------------------------------------------------------------- - - async autoRegisterRunner () { - const { data } = await this.server.runnerRegistrationTokens.list({ sort: 'createdAt' }) - - const { runnerToken } = await this.register({ - name: 'runner ' + buildUUID(), - registrationToken: data[0].registrationToken - }) - - return runnerToken - } -} diff --git a/shared/server-commands/search/index.ts b/shared/server-commands/search/index.ts deleted file mode 100644 index 48dbe8ae9..000000000 --- a/shared/server-commands/search/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './search-command' diff --git a/shared/server-commands/search/search-command.ts b/shared/server-commands/search/search-command.ts deleted file mode 100644 index a5b498b66..000000000 --- a/shared/server-commands/search/search-command.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { - HttpStatusCode, - ResultList, - Video, - VideoChannel, - VideoChannelsSearchQuery, - VideoPlaylist, - VideoPlaylistsSearchQuery, - VideosSearchQuery -} from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class SearchCommand extends AbstractCommand { - - searchChannels (options: OverrideCommandOptions & { - search: string - }) { - return this.advancedChannelSearch({ - ...options, - - search: { search: options.search } - }) - } - - advancedChannelSearch (options: OverrideCommandOptions & { - search: VideoChannelsSearchQuery - }) { - const { search } = options - const path = '/api/v1/search/video-channels' - - return this.getRequestBody>({ - ...options, - - path, - query: search, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - searchPlaylists (options: OverrideCommandOptions & { - search: string - }) { - return this.advancedPlaylistSearch({ - ...options, - - search: { search: options.search } - }) - } - - advancedPlaylistSearch (options: OverrideCommandOptions & { - search: VideoPlaylistsSearchQuery - }) { - const { search } = options - const path = '/api/v1/search/video-playlists' - - return this.getRequestBody>({ - ...options, - - path, - query: search, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - searchVideos (options: OverrideCommandOptions & { - search: string - sort?: string - }) { - const { search, sort } = options - - return this.advancedVideoSearch({ - ...options, - - search: { - search, - sort: sort ?? '-publishedAt' - } - }) - } - - advancedVideoSearch (options: OverrideCommandOptions & { - search: VideosSearchQuery - }) { - const { search } = options - const path = '/api/v1/search/videos' - - return this.getRequestBody>({ - ...options, - - path, - query: search, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } -} diff --git a/shared/server-commands/server/config-command.ts b/shared/server-commands/server/config-command.ts deleted file mode 100644 index 5ee2fe021..000000000 --- a/shared/server-commands/server/config-command.ts +++ /dev/null @@ -1,576 +0,0 @@ -import { merge } from 'lodash' -import { About, CustomConfig, HttpStatusCode, ServerConfig } from '@shared/models' -import { DeepPartial } from '@shared/typescript-utils' -import { AbstractCommand, OverrideCommandOptions } from '../shared/abstract-command' - -export class ConfigCommand extends AbstractCommand { - - static getCustomConfigResolutions (enabled: boolean, with0p = false) { - return { - '0p': enabled && with0p, - '144p': enabled, - '240p': enabled, - '360p': enabled, - '480p': enabled, - '720p': enabled, - '1080p': enabled, - '1440p': enabled, - '2160p': enabled - } - } - - // --------------------------------------------------------------------------- - - static getEmailOverrideConfig (emailPort: number) { - return { - smtp: { - hostname: '127.0.0.1', - port: emailPort - } - } - } - - // --------------------------------------------------------------------------- - - enableSignup (requiresApproval: boolean, limit = -1) { - return this.updateExistingSubConfig({ - newConfig: { - signup: { - enabled: true, - requiresApproval, - limit - } - } - }) - } - - // --------------------------------------------------------------------------- - - disableImports () { - return this.setImportsEnabled(false) - } - - enableImports () { - return this.setImportsEnabled(true) - } - - private setImportsEnabled (enabled: boolean) { - return this.updateExistingSubConfig({ - newConfig: { - import: { - videos: { - http: { - enabled - }, - - torrent: { - enabled - } - } - } - } - }) - } - - // --------------------------------------------------------------------------- - - disableFileUpdate () { - return this.setFileUpdateEnabled(false) - } - - enableFileUpdate () { - return this.setFileUpdateEnabled(true) - } - - private setFileUpdateEnabled (enabled: boolean) { - return this.updateExistingSubConfig({ - newConfig: { - videoFile: { - update: { - enabled - } - } - } - }) - } - - // --------------------------------------------------------------------------- - - enableChannelSync () { - return this.setChannelSyncEnabled(true) - } - - disableChannelSync () { - return this.setChannelSyncEnabled(false) - } - - private setChannelSyncEnabled (enabled: boolean) { - return this.updateExistingSubConfig({ - newConfig: { - import: { - videoChannelSynchronization: { - enabled - } - } - } - }) - } - - // --------------------------------------------------------------------------- - - enableLive (options: { - allowReplay?: boolean - transcoding?: boolean - resolutions?: 'min' | 'max' // Default max - } = {}) { - const { allowReplay, transcoding, resolutions = 'max' } = options - - return this.updateExistingSubConfig({ - newConfig: { - live: { - enabled: true, - allowReplay: allowReplay ?? true, - transcoding: { - enabled: transcoding ?? true, - resolutions: ConfigCommand.getCustomConfigResolutions(resolutions === 'max') - } - } - } - }) - } - - disableTranscoding () { - return this.updateExistingSubConfig({ - newConfig: { - transcoding: { - enabled: false - }, - videoStudio: { - enabled: false - } - } - }) - } - - enableTranscoding (options: { - webVideo?: boolean // default true - hls?: boolean // default true - with0p?: boolean // default false - } = {}) { - const { webVideo = true, hls = true, with0p = false } = options - - return this.updateExistingSubConfig({ - newConfig: { - transcoding: { - enabled: true, - - allowAudioFiles: true, - allowAdditionalExtensions: true, - - resolutions: ConfigCommand.getCustomConfigResolutions(true, with0p), - - webVideos: { - enabled: webVideo - }, - hls: { - enabled: hls - } - } - } - }) - } - - enableMinimumTranscoding (options: { - webVideo?: boolean // default true - hls?: boolean // default true - } = {}) { - const { webVideo = true, hls = true } = options - - return this.updateExistingSubConfig({ - newConfig: { - transcoding: { - enabled: true, - - allowAudioFiles: true, - allowAdditionalExtensions: true, - - resolutions: { - ...ConfigCommand.getCustomConfigResolutions(false), - - '240p': true - }, - - webVideos: { - enabled: webVideo - }, - hls: { - enabled: hls - } - } - } - }) - } - - enableRemoteTranscoding () { - return this.updateExistingSubConfig({ - newConfig: { - transcoding: { - remoteRunners: { - enabled: true - } - }, - live: { - transcoding: { - remoteRunners: { - enabled: true - } - } - } - } - }) - } - - enableRemoteStudio () { - return this.updateExistingSubConfig({ - newConfig: { - videoStudio: { - remoteRunners: { - enabled: true - } - } - } - }) - } - - // --------------------------------------------------------------------------- - - enableStudio () { - return this.updateExistingSubConfig({ - newConfig: { - videoStudio: { - enabled: true - } - } - }) - } - - // --------------------------------------------------------------------------- - - getConfig (options: OverrideCommandOptions = {}) { - const path = '/api/v1/config' - - return this.getRequestBody({ - ...options, - - path, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - async getIndexHTMLConfig (options: OverrideCommandOptions = {}) { - const text = await this.getRequestText({ - ...options, - - path: '/', - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - - const match = text.match('') - - // We parse the string twice, first to extract the string and then to extract the JSON - return JSON.parse(JSON.parse(match[1])) as ServerConfig - } - - getAbout (options: OverrideCommandOptions = {}) { - const path = '/api/v1/config/about' - - return this.getRequestBody({ - ...options, - - path, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getCustomConfig (options: OverrideCommandOptions = {}) { - const path = '/api/v1/config/custom' - - return this.getRequestBody({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - updateCustomConfig (options: OverrideCommandOptions & { - newCustomConfig: CustomConfig - }) { - const path = '/api/v1/config/custom' - - return this.putBodyRequest({ - ...options, - - path, - fields: options.newCustomConfig, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - deleteCustomConfig (options: OverrideCommandOptions = {}) { - const path = '/api/v1/config/custom' - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - async updateExistingSubConfig (options: OverrideCommandOptions & { - newConfig: DeepPartial - }) { - const existing = await this.getCustomConfig({ ...options, expectedStatus: HttpStatusCode.OK_200 }) - - return this.updateCustomConfig({ ...options, newCustomConfig: merge({}, existing, options.newConfig) }) - } - - updateCustomSubConfig (options: OverrideCommandOptions & { - newConfig: DeepPartial - }) { - const newCustomConfig: CustomConfig = { - instance: { - name: 'PeerTube updated', - shortDescription: 'my short description', - description: 'my super description', - terms: 'my super terms', - codeOfConduct: 'my super coc', - - creationReason: 'my super creation reason', - moderationInformation: 'my super moderation information', - administrator: 'Kuja', - maintenanceLifetime: 'forever', - businessModel: 'my super business model', - hardwareInformation: '2vCore 3GB RAM', - - languages: [ 'en', 'es' ], - categories: [ 1, 2 ], - - isNSFW: true, - defaultNSFWPolicy: 'blur', - - defaultClientRoute: '/videos/recently-added', - - customizations: { - javascript: 'alert("coucou")', - css: 'body { background-color: red; }' - } - }, - theme: { - default: 'default' - }, - services: { - twitter: { - username: '@MySuperUsername', - whitelisted: true - } - }, - client: { - videos: { - miniature: { - preferAuthorDisplayName: false - } - }, - menu: { - login: { - redirectOnSingleExternalAuth: false - } - } - }, - cache: { - previews: { - size: 2 - }, - captions: { - size: 3 - }, - torrents: { - size: 4 - }, - storyboards: { - size: 5 - } - }, - signup: { - enabled: false, - limit: 5, - requiresApproval: true, - requiresEmailVerification: false, - minimumAge: 16 - }, - admin: { - email: 'superadmin1@example.com' - }, - contactForm: { - enabled: true - }, - user: { - history: { - videos: { - enabled: true - } - }, - videoQuota: 5242881, - videoQuotaDaily: 318742 - }, - videoChannels: { - maxPerUser: 20 - }, - transcoding: { - enabled: true, - remoteRunners: { - enabled: false - }, - allowAdditionalExtensions: true, - allowAudioFiles: true, - threads: 1, - concurrency: 3, - profile: 'default', - resolutions: { - '0p': false, - '144p': false, - '240p': false, - '360p': true, - '480p': true, - '720p': false, - '1080p': false, - '1440p': false, - '2160p': false - }, - alwaysTranscodeOriginalResolution: true, - webVideos: { - enabled: true - }, - hls: { - enabled: false - } - }, - live: { - enabled: true, - allowReplay: false, - latencySetting: { - enabled: false - }, - maxDuration: -1, - maxInstanceLives: -1, - maxUserLives: 50, - transcoding: { - enabled: true, - remoteRunners: { - enabled: false - }, - threads: 4, - profile: 'default', - resolutions: { - '144p': true, - '240p': true, - '360p': true, - '480p': true, - '720p': true, - '1080p': true, - '1440p': true, - '2160p': true - }, - alwaysTranscodeOriginalResolution: true - } - }, - videoStudio: { - enabled: false, - remoteRunners: { - enabled: false - } - }, - videoFile: { - update: { - enabled: false - } - }, - import: { - videos: { - concurrency: 3, - http: { - enabled: false - }, - torrent: { - enabled: false - } - }, - videoChannelSynchronization: { - enabled: false, - maxPerUser: 10 - } - }, - trending: { - videos: { - algorithms: { - enabled: [ 'hot', 'most-viewed', 'most-liked' ], - default: 'hot' - } - } - }, - autoBlacklist: { - videos: { - ofUsers: { - enabled: false - } - } - }, - followers: { - instance: { - enabled: true, - manualApproval: false - } - }, - followings: { - instance: { - autoFollowBack: { - enabled: false - }, - autoFollowIndex: { - indexUrl: 'https://instances.joinpeertube.org/api/v1/instances/hosts', - enabled: false - } - } - }, - broadcastMessage: { - enabled: true, - level: 'warning', - message: 'hello', - dismissable: true - }, - search: { - remoteUri: { - users: true, - anonymous: true - }, - searchIndex: { - enabled: true, - url: 'https://search.joinpeertube.org', - disableLocalSearch: true, - isDefaultSearch: true - } - } - } - - merge(newCustomConfig, options.newConfig) - - return this.updateCustomConfig({ ...options, newCustomConfig }) - } -} diff --git a/shared/server-commands/server/contact-form-command.ts b/shared/server-commands/server/contact-form-command.ts deleted file mode 100644 index 0e8fd6d84..000000000 --- a/shared/server-commands/server/contact-form-command.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { HttpStatusCode } from '@shared/models' -import { ContactForm } from '../../models/server' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class ContactFormCommand extends AbstractCommand { - - send (options: OverrideCommandOptions & { - fromEmail: string - fromName: string - subject: string - body: string - }) { - const path = '/api/v1/server/contact' - - const body: ContactForm = { - fromEmail: options.fromEmail, - fromName: options.fromName, - subject: options.subject, - body: options.body - } - - return this.postBodyRequest({ - ...options, - - path, - fields: body, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } -} diff --git a/shared/server-commands/server/debug-command.ts b/shared/server-commands/server/debug-command.ts deleted file mode 100644 index 3c5a785bb..000000000 --- a/shared/server-commands/server/debug-command.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Debug, HttpStatusCode, SendDebugCommand } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class DebugCommand extends AbstractCommand { - - getDebug (options: OverrideCommandOptions = {}) { - const path = '/api/v1/server/debug' - - return this.getRequestBody({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - sendCommand (options: OverrideCommandOptions & { - body: SendDebugCommand - }) { - const { body } = options - const path = '/api/v1/server/debug/run-command' - - return this.postBodyRequest({ - ...options, - - path, - fields: body, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } -} diff --git a/shared/server-commands/server/follows-command.ts b/shared/server-commands/server/follows-command.ts deleted file mode 100644 index 496e11df1..000000000 --- a/shared/server-commands/server/follows-command.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { pick } from '@shared/core-utils' -import { ActivityPubActorType, ActorFollow, FollowState, HttpStatusCode, ResultList, ServerFollowCreate } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' -import { PeerTubeServer } from './server' - -export class FollowsCommand extends AbstractCommand { - - getFollowers (options: OverrideCommandOptions & { - start?: number - count?: number - sort?: string - search?: string - actorType?: ActivityPubActorType - state?: FollowState - } = {}) { - const path = '/api/v1/server/followers' - - const query = pick(options, [ 'start', 'count', 'sort', 'search', 'state', 'actorType' ]) - - return this.getRequestBody>({ - ...options, - - path, - query, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getFollowings (options: OverrideCommandOptions & { - start?: number - count?: number - sort?: string - search?: string - actorType?: ActivityPubActorType - state?: FollowState - } = {}) { - const path = '/api/v1/server/following' - - const query = pick(options, [ 'start', 'count', 'sort', 'search', 'state', 'actorType' ]) - - return this.getRequestBody>({ - ...options, - - path, - query, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - follow (options: OverrideCommandOptions & { - hosts?: string[] - handles?: string[] - }) { - const path = '/api/v1/server/following' - - const fields: ServerFollowCreate = {} - - if (options.hosts) { - fields.hosts = options.hosts.map(f => f.replace(/^http:\/\//, '')) - } - - if (options.handles) { - fields.handles = options.handles - } - - return this.postBodyRequest({ - ...options, - - path, - fields, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - async unfollow (options: OverrideCommandOptions & { - target: PeerTubeServer | string - }) { - const { target } = options - - const handle = typeof target === 'string' - ? target - : target.host - - const path = '/api/v1/server/following/' + handle - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - acceptFollower (options: OverrideCommandOptions & { - follower: string - }) { - const path = '/api/v1/server/followers/' + options.follower + '/accept' - - return this.postBodyRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - rejectFollower (options: OverrideCommandOptions & { - follower: string - }) { - const path = '/api/v1/server/followers/' + options.follower + '/reject' - - return this.postBodyRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - removeFollower (options: OverrideCommandOptions & { - follower: PeerTubeServer - }) { - const path = '/api/v1/server/followers/peertube@' + options.follower.host - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } -} diff --git a/shared/server-commands/server/follows.ts b/shared/server-commands/server/follows.ts deleted file mode 100644 index 698238f29..000000000 --- a/shared/server-commands/server/follows.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { waitJobs } from './jobs' -import { PeerTubeServer } from './server' - -async function doubleFollow (server1: PeerTubeServer, server2: PeerTubeServer) { - await Promise.all([ - server1.follows.follow({ hosts: [ server2.url ] }), - server2.follows.follow({ hosts: [ server1.url ] }) - ]) - - // Wait request propagation - await waitJobs([ server1, server2 ]) - - return true -} - -// --------------------------------------------------------------------------- - -export { - doubleFollow -} diff --git a/shared/server-commands/server/index.ts b/shared/server-commands/server/index.ts deleted file mode 100644 index 9a2fbf8d3..000000000 --- a/shared/server-commands/server/index.ts +++ /dev/null @@ -1,15 +0,0 @@ -export * from './config-command' -export * from './contact-form-command' -export * from './debug-command' -export * from './follows-command' -export * from './follows' -export * from './jobs' -export * from './jobs-command' -export * from './metrics-command' -export * from './object-storage-command' -export * from './plugins-command' -export * from './redundancy-command' -export * from './server' -export * from './servers-command' -export * from './servers' -export * from './stats-command' diff --git a/shared/server-commands/server/jobs-command.ts b/shared/server-commands/server/jobs-command.ts deleted file mode 100644 index b8790ea00..000000000 --- a/shared/server-commands/server/jobs-command.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { pick } from '@shared/core-utils' -import { HttpStatusCode, Job, JobState, JobType, ResultList } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class JobsCommand extends AbstractCommand { - - async getLatest (options: OverrideCommandOptions & { - jobType: JobType - }) { - const { data } = await this.list({ ...options, start: 0, count: 1, sort: '-createdAt' }) - - if (data.length === 0) return undefined - - return data[0] - } - - pauseJobQueue (options: OverrideCommandOptions = {}) { - const path = '/api/v1/jobs/pause' - - return this.postBodyRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - resumeJobQueue (options: OverrideCommandOptions = {}) { - const path = '/api/v1/jobs/resume' - - return this.postBodyRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - list (options: OverrideCommandOptions & { - state?: JobState - jobType?: JobType - start?: number - count?: number - sort?: string - } = {}) { - const path = this.buildJobsUrl(options.state) - - const query = pick(options, [ 'start', 'count', 'sort', 'jobType' ]) - - return this.getRequestBody>({ - ...options, - - path, - query, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - listFailed (options: OverrideCommandOptions & { - jobType?: JobType - }) { - const path = this.buildJobsUrl('failed') - - return this.getRequestBody>({ - ...options, - - path, - query: { start: 0, count: 50 }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - private buildJobsUrl (state?: JobState) { - let path = '/api/v1/jobs' - - if (state) path += '/' + state - - return path - } -} diff --git a/shared/server-commands/server/jobs.ts b/shared/server-commands/server/jobs.ts deleted file mode 100644 index 8f131fba4..000000000 --- a/shared/server-commands/server/jobs.ts +++ /dev/null @@ -1,118 +0,0 @@ - -import { expect } from 'chai' -import { wait } from '@shared/core-utils' -import { JobState, JobType, RunnerJobState } from '../../models' -import { PeerTubeServer } from './server' - -async function waitJobs ( - serversArg: PeerTubeServer[] | PeerTubeServer, - options: { - skipDelayed?: boolean // default false - runnerJobs?: boolean // default false - } = {} -) { - const { skipDelayed = false, runnerJobs = false } = options - - const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT - ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10) - : 250 - - let servers: PeerTubeServer[] - - if (Array.isArray(serversArg) === false) servers = [ serversArg as PeerTubeServer ] - else servers = serversArg as PeerTubeServer[] - - const states: JobState[] = [ 'waiting', 'active' ] - if (!skipDelayed) states.push('delayed') - - const repeatableJobs: JobType[] = [ 'videos-views-stats', 'activitypub-cleaner' ] - let pendingRequests: boolean - - function tasksBuilder () { - const tasks: Promise[] = [] - - // Check if each server has pending request - for (const server of servers) { - if (process.env.DEBUG) console.log('Checking ' + server.url) - - for (const state of states) { - - const jobPromise = server.jobs.list({ - state, - start: 0, - count: 10, - sort: '-createdAt' - }).then(body => body.data) - .then(jobs => jobs.filter(j => !repeatableJobs.includes(j.type))) - .then(jobs => { - if (jobs.length !== 0) { - pendingRequests = true - - if (process.env.DEBUG) { - console.log(jobs) - } - } - }) - - tasks.push(jobPromise) - } - - const debugPromise = server.debug.getDebug() - .then(obj => { - if (obj.activityPubMessagesWaiting !== 0) { - pendingRequests = true - - if (process.env.DEBUG) { - console.log('AP messages waiting: ' + obj.activityPubMessagesWaiting) - } - } - }) - tasks.push(debugPromise) - - if (runnerJobs) { - const runnerJobsPromise = server.runnerJobs.list({ count: 100 }) - .then(({ data }) => { - for (const job of data) { - if (job.state.id !== RunnerJobState.COMPLETED) { - pendingRequests = true - - if (process.env.DEBUG) { - console.log(job) - } - } - } - }) - tasks.push(runnerJobsPromise) - } - } - - return tasks - } - - do { - pendingRequests = false - await Promise.all(tasksBuilder()) - - // Retry, in case of new jobs were created - if (pendingRequests === false) { - await wait(pendingJobWait) - await Promise.all(tasksBuilder()) - } - - if (pendingRequests) { - await wait(pendingJobWait) - } - } while (pendingRequests) -} - -async function expectNoFailedTranscodingJob (server: PeerTubeServer) { - const { data } = await server.jobs.listFailed({ jobType: 'video-transcoding' }) - expect(data).to.have.lengthOf(0) -} - -// --------------------------------------------------------------------------- - -export { - waitJobs, - expectNoFailedTranscodingJob -} diff --git a/shared/server-commands/server/metrics-command.ts b/shared/server-commands/server/metrics-command.ts deleted file mode 100644 index d22b4833d..000000000 --- a/shared/server-commands/server/metrics-command.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { HttpStatusCode, PlaybackMetricCreate } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class MetricsCommand extends AbstractCommand { - - addPlaybackMetric (options: OverrideCommandOptions & { metrics: PlaybackMetricCreate }) { - const path = '/api/v1/metrics/playback' - - return this.postBodyRequest({ - ...options, - - path, - fields: options.metrics, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } -} diff --git a/shared/server-commands/server/object-storage-command.ts b/shared/server-commands/server/object-storage-command.ts deleted file mode 100644 index 6bb232c36..000000000 --- a/shared/server-commands/server/object-storage-command.ts +++ /dev/null @@ -1,165 +0,0 @@ -import { randomInt } from 'crypto' -import { HttpStatusCode } from '@shared/models' -import { makePostBodyRequest } from '../requests' - -export class ObjectStorageCommand { - static readonly DEFAULT_SCALEWAY_BUCKET = 'peertube-ci-test' - - private readonly bucketsCreated: string[] = [] - private readonly seed: number - - // --------------------------------------------------------------------------- - - constructor () { - this.seed = randomInt(0, 10000) - } - - static getMockCredentialsConfig () { - return { - access_key_id: 'AKIAIOSFODNN7EXAMPLE', - secret_access_key: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' - } - } - - static getMockEndpointHost () { - return 'localhost:9444' - } - - static getMockRegion () { - return 'us-east-1' - } - - getDefaultMockConfig () { - return { - object_storage: { - enabled: true, - endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(), - region: ObjectStorageCommand.getMockRegion(), - - credentials: ObjectStorageCommand.getMockCredentialsConfig(), - - streaming_playlists: { - bucket_name: this.getMockStreamingPlaylistsBucketName() - }, - - web_videos: { - bucket_name: this.getMockWebVideosBucketName() - } - } - } - } - - getMockWebVideosBaseUrl () { - return `http://${this.getMockWebVideosBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/` - } - - getMockPlaylistBaseUrl () { - return `http://${this.getMockStreamingPlaylistsBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/` - } - - async prepareDefaultMockBuckets () { - await this.createMockBucket(this.getMockStreamingPlaylistsBucketName()) - await this.createMockBucket(this.getMockWebVideosBucketName()) - } - - async createMockBucket (name: string) { - this.bucketsCreated.push(name) - - await this.deleteMockBucket(name) - - await makePostBodyRequest({ - url: ObjectStorageCommand.getMockEndpointHost(), - path: '/ui/' + name + '?create', - expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 - }) - - await makePostBodyRequest({ - url: ObjectStorageCommand.getMockEndpointHost(), - path: '/ui/' + name + '?make-public', - expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 - }) - } - - async cleanupMock () { - for (const name of this.bucketsCreated) { - await this.deleteMockBucket(name) - } - } - - getMockStreamingPlaylistsBucketName (name = 'streaming-playlists') { - return this.getMockBucketName(name) - } - - getMockWebVideosBucketName (name = 'web-videos') { - return this.getMockBucketName(name) - } - - getMockBucketName (name: string) { - return `${this.seed}-${name}` - } - - private async deleteMockBucket (name: string) { - await makePostBodyRequest({ - url: ObjectStorageCommand.getMockEndpointHost(), - path: '/ui/' + name + '?delete', - expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 - }) - } - - // --------------------------------------------------------------------------- - - static getDefaultScalewayConfig (options: { - serverNumber: number - enablePrivateProxy?: boolean // default true - privateACL?: 'private' | 'public-read' // default 'private' - }) { - const { serverNumber, enablePrivateProxy = true, privateACL = 'private' } = options - - return { - object_storage: { - enabled: true, - endpoint: this.getScalewayEndpointHost(), - region: this.getScalewayRegion(), - - credentials: this.getScalewayCredentialsConfig(), - - upload_acl: { - private: privateACL - }, - - proxy: { - proxify_private_files: enablePrivateProxy - }, - - streaming_playlists: { - bucket_name: this.DEFAULT_SCALEWAY_BUCKET, - prefix: `test:server-${serverNumber}-streaming-playlists:` - }, - - web_videos: { - bucket_name: this.DEFAULT_SCALEWAY_BUCKET, - prefix: `test:server-${serverNumber}-web-videos:` - } - } - } - } - - static getScalewayCredentialsConfig () { - return { - access_key_id: process.env.OBJECT_STORAGE_SCALEWAY_KEY_ID, - secret_access_key: process.env.OBJECT_STORAGE_SCALEWAY_ACCESS_KEY - } - } - - static getScalewayEndpointHost () { - return 's3.fr-par.scw.cloud' - } - - static getScalewayRegion () { - return 'fr-par' - } - - static getScalewayBaseUrl () { - return `https://${this.DEFAULT_SCALEWAY_BUCKET}.${this.getScalewayEndpointHost()}/` - } -} diff --git a/shared/server-commands/server/plugins-command.ts b/shared/server-commands/server/plugins-command.ts deleted file mode 100644 index bb1277a7c..000000000 --- a/shared/server-commands/server/plugins-command.ts +++ /dev/null @@ -1,257 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { readJSON, writeJSON } from 'fs-extra' -import { join } from 'path' -import { root } from '@shared/core-utils' -import { - HttpStatusCode, - PeerTubePlugin, - PeerTubePluginIndex, - PeertubePluginIndexList, - PluginPackageJSON, - PluginTranslation, - PluginType, - PublicServerSetting, - RegisteredServerSettings, - ResultList -} from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class PluginsCommand extends AbstractCommand { - - static getPluginTestPath (suffix = '') { - return join(root(), 'server', 'tests', 'fixtures', 'peertube-plugin-test' + suffix) - } - - list (options: OverrideCommandOptions & { - start?: number - count?: number - sort?: string - pluginType?: PluginType - uninstalled?: boolean - }) { - const { start, count, sort, pluginType, uninstalled } = options - const path = '/api/v1/plugins' - - return this.getRequestBody>({ - ...options, - - path, - query: { - start, - count, - sort, - pluginType, - uninstalled - }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - listAvailable (options: OverrideCommandOptions & { - start?: number - count?: number - sort?: string - pluginType?: PluginType - currentPeerTubeEngine?: string - search?: string - expectedStatus?: HttpStatusCode - }) { - const { start, count, sort, pluginType, search, currentPeerTubeEngine } = options - const path = '/api/v1/plugins/available' - - const query: PeertubePluginIndexList = { - start, - count, - sort, - pluginType, - currentPeerTubeEngine, - search - } - - return this.getRequestBody>({ - ...options, - - path, - query, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - get (options: OverrideCommandOptions & { - npmName: string - }) { - const path = '/api/v1/plugins/' + options.npmName - - return this.getRequestBody({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - updateSettings (options: OverrideCommandOptions & { - npmName: string - settings: any - }) { - const { npmName, settings } = options - const path = '/api/v1/plugins/' + npmName + '/settings' - - return this.putBodyRequest({ - ...options, - - path, - fields: { settings }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - getRegisteredSettings (options: OverrideCommandOptions & { - npmName: string - }) { - const path = '/api/v1/plugins/' + options.npmName + '/registered-settings' - - return this.getRequestBody({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getPublicSettings (options: OverrideCommandOptions & { - npmName: string - }) { - const { npmName } = options - const path = '/api/v1/plugins/' + npmName + '/public-settings' - - return this.getRequestBody({ - ...options, - - path, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getTranslations (options: OverrideCommandOptions & { - locale: string - }) { - const { locale } = options - const path = '/plugins/translations/' + locale + '.json' - - return this.getRequestBody({ - ...options, - - path, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - install (options: OverrideCommandOptions & { - path?: string - npmName?: string - pluginVersion?: string - }) { - const { npmName, path, pluginVersion } = options - const apiPath = '/api/v1/plugins/install' - - return this.postBodyRequest({ - ...options, - - path: apiPath, - fields: { npmName, path, pluginVersion }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - update (options: OverrideCommandOptions & { - path?: string - npmName?: string - }) { - const { npmName, path } = options - const apiPath = '/api/v1/plugins/update' - - return this.postBodyRequest({ - ...options, - - path: apiPath, - fields: { npmName, path }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - uninstall (options: OverrideCommandOptions & { - npmName: string - }) { - const { npmName } = options - const apiPath = '/api/v1/plugins/uninstall' - - return this.postBodyRequest({ - ...options, - - path: apiPath, - fields: { npmName }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - getCSS (options: OverrideCommandOptions = {}) { - const path = '/plugins/global.css' - - return this.getRequestText({ - ...options, - - path, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getExternalAuth (options: OverrideCommandOptions & { - npmName: string - npmVersion: string - authName: string - query?: any - }) { - const { npmName, npmVersion, authName, query } = options - - const path = '/plugins/' + npmName + '/' + npmVersion + '/auth/' + authName - - return this.getRequest({ - ...options, - - path, - query, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200, - redirects: 0 - }) - } - - updatePackageJSON (npmName: string, json: any) { - const path = this.getPackageJSONPath(npmName) - - return writeJSON(path, json) - } - - getPackageJSON (npmName: string): Promise { - const path = this.getPackageJSONPath(npmName) - - return readJSON(path) - } - - private getPackageJSONPath (npmName: string) { - return this.server.servers.buildDirectory(join('plugins', 'node_modules', npmName, 'package.json')) - } -} diff --git a/shared/server-commands/server/redundancy-command.ts b/shared/server-commands/server/redundancy-command.ts deleted file mode 100644 index e7a8b3c29..000000000 --- a/shared/server-commands/server/redundancy-command.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { HttpStatusCode, ResultList, VideoRedundanciesTarget, VideoRedundancy } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class RedundancyCommand extends AbstractCommand { - - updateRedundancy (options: OverrideCommandOptions & { - host: string - redundancyAllowed: boolean - }) { - const { host, redundancyAllowed } = options - const path = '/api/v1/server/redundancy/' + host - - return this.putBodyRequest({ - ...options, - - path, - fields: { redundancyAllowed }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - listVideos (options: OverrideCommandOptions & { - target: VideoRedundanciesTarget - start?: number - count?: number - sort?: string - }) { - const path = '/api/v1/server/redundancy/videos' - - const { target, start, count, sort } = options - - return this.getRequestBody>({ - ...options, - - path, - - query: { - start: start ?? 0, - count: count ?? 5, - sort: sort ?? 'name', - target - }, - - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - addVideo (options: OverrideCommandOptions & { - videoId: number - }) { - const path = '/api/v1/server/redundancy/videos' - const { videoId } = options - - return this.postBodyRequest({ - ...options, - - path, - fields: { videoId }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - removeVideo (options: OverrideCommandOptions & { - redundancyId: number - }) { - const { redundancyId } = options - const path = '/api/v1/server/redundancy/videos/' + redundancyId - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } -} diff --git a/shared/server-commands/server/server.ts b/shared/server-commands/server/server.ts deleted file mode 100644 index 38568a890..000000000 --- a/shared/server-commands/server/server.ts +++ /dev/null @@ -1,450 +0,0 @@ -import { ChildProcess, fork } from 'child_process' -import { copy } from 'fs-extra' -import { join } from 'path' -import { parallelTests, randomInt, root } from '@shared/core-utils' -import { Video, VideoChannel, VideoChannelSync, VideoCreateResult, VideoDetails } from '@shared/models' -import { BulkCommand } from '../bulk' -import { CLICommand } from '../cli' -import { CustomPagesCommand } from '../custom-pages' -import { FeedCommand } from '../feeds' -import { LogsCommand } from '../logs' -import { AbusesCommand } from '../moderation' -import { OverviewsCommand } from '../overviews' -import { RunnerJobsCommand, RunnerRegistrationTokensCommand, RunnersCommand } from '../runners' -import { SearchCommand } from '../search' -import { SocketIOCommand } from '../socket' -import { - AccountsCommand, - BlocklistCommand, - LoginCommand, - NotificationsCommand, - RegistrationsCommand, - SubscriptionsCommand, - TwoFactorCommand, - UsersCommand -} from '../users' -import { - BlacklistCommand, - CaptionsCommand, - ChangeOwnershipCommand, - ChannelsCommand, - ChannelSyncsCommand, - HistoryCommand, - ImportsCommand, - LiveCommand, - VideoPasswordsCommand, - PlaylistsCommand, - ServicesCommand, - StoryboardCommand, - StreamingPlaylistsCommand, - VideosCommand, - VideoStudioCommand, - VideoTokenCommand, - ViewsCommand -} from '../videos' -import { CommentsCommand } from '../videos/comments-command' -import { VideoStatsCommand } from '../videos/video-stats-command' -import { ConfigCommand } from './config-command' -import { ContactFormCommand } from './contact-form-command' -import { DebugCommand } from './debug-command' -import { FollowsCommand } from './follows-command' -import { JobsCommand } from './jobs-command' -import { MetricsCommand } from './metrics-command' -import { PluginsCommand } from './plugins-command' -import { RedundancyCommand } from './redundancy-command' -import { ServersCommand } from './servers-command' -import { StatsCommand } from './stats-command' - -export type RunServerOptions = { - hideLogs?: boolean - nodeArgs?: string[] - peertubeArgs?: string[] - env?: { [ id: string ]: string } -} - -export class PeerTubeServer { - app?: ChildProcess - - url: string - host?: string - hostname?: string - port?: number - - rtmpPort?: number - rtmpsPort?: number - - parallel?: boolean - internalServerNumber: number - - serverNumber?: number - customConfigFile?: string - - store?: { - client?: { - id?: string - secret?: string - } - - user?: { - username: string - password: string - email?: string - } - - channel?: VideoChannel - videoChannelSync?: Partial - - video?: Video - videoCreated?: VideoCreateResult - videoDetails?: VideoDetails - - videos?: { id: number, uuid: string }[] - } - - accessToken?: string - refreshToken?: string - - bulk?: BulkCommand - cli?: CLICommand - customPage?: CustomPagesCommand - feed?: FeedCommand - logs?: LogsCommand - abuses?: AbusesCommand - overviews?: OverviewsCommand - search?: SearchCommand - contactForm?: ContactFormCommand - debug?: DebugCommand - follows?: FollowsCommand - jobs?: JobsCommand - metrics?: MetricsCommand - plugins?: PluginsCommand - redundancy?: RedundancyCommand - stats?: StatsCommand - config?: ConfigCommand - socketIO?: SocketIOCommand - accounts?: AccountsCommand - blocklist?: BlocklistCommand - subscriptions?: SubscriptionsCommand - live?: LiveCommand - services?: ServicesCommand - blacklist?: BlacklistCommand - captions?: CaptionsCommand - changeOwnership?: ChangeOwnershipCommand - playlists?: PlaylistsCommand - history?: HistoryCommand - imports?: ImportsCommand - channelSyncs?: ChannelSyncsCommand - streamingPlaylists?: StreamingPlaylistsCommand - channels?: ChannelsCommand - comments?: CommentsCommand - notifications?: NotificationsCommand - servers?: ServersCommand - login?: LoginCommand - users?: UsersCommand - videoStudio?: VideoStudioCommand - videos?: VideosCommand - videoStats?: VideoStatsCommand - views?: ViewsCommand - twoFactor?: TwoFactorCommand - videoToken?: VideoTokenCommand - registrations?: RegistrationsCommand - videoPasswords?: VideoPasswordsCommand - - storyboard?: StoryboardCommand - - runners?: RunnersCommand - runnerRegistrationTokens?: RunnerRegistrationTokensCommand - runnerJobs?: RunnerJobsCommand - - constructor (options: { serverNumber: number } | { url: string }) { - if ((options as any).url) { - this.setUrl((options as any).url) - } else { - this.setServerNumber((options as any).serverNumber) - } - - this.store = { - client: { - id: null, - secret: null - }, - user: { - username: null, - password: null - } - } - - this.assignCommands() - } - - setServerNumber (serverNumber: number) { - this.serverNumber = serverNumber - - this.parallel = parallelTests() - - this.internalServerNumber = this.parallel ? this.randomServer() : this.serverNumber - this.rtmpPort = this.parallel ? this.randomRTMP() : 1936 - this.rtmpsPort = this.parallel ? this.randomRTMP() : 1937 - this.port = 9000 + this.internalServerNumber - - this.url = `http://127.0.0.1:${this.port}` - this.host = `127.0.0.1:${this.port}` - this.hostname = '127.0.0.1' - } - - setUrl (url: string) { - const parsed = new URL(url) - - this.url = url - this.host = parsed.host - this.hostname = parsed.hostname - this.port = parseInt(parsed.port) - } - - getDirectoryPath (directoryName: string) { - const testDirectory = 'test' + this.internalServerNumber - - return join(root(), testDirectory, directoryName) - } - - async flushAndRun (configOverride?: object, options: RunServerOptions = {}) { - await ServersCommand.flushTests(this.internalServerNumber) - - return this.run(configOverride, options) - } - - async run (configOverrideArg?: any, options: RunServerOptions = {}) { - // These actions are async so we need to be sure that they have both been done - const serverRunString = { - 'HTTP server listening': false - } - const key = 'Database peertube_test' + this.internalServerNumber + ' is ready' - serverRunString[key] = false - - const regexps = { - client_id: 'Client id: (.+)', - client_secret: 'Client secret: (.+)', - user_username: 'Username: (.+)', - user_password: 'User password: (.+)' - } - - await this.assignCustomConfigFile() - - const configOverride = this.buildConfigOverride() - - if (configOverrideArg !== undefined) { - Object.assign(configOverride, configOverrideArg) - } - - // Share the environment - const env = { ...process.env } - env['NODE_ENV'] = 'test' - env['NODE_APP_INSTANCE'] = this.internalServerNumber.toString() - env['NODE_CONFIG'] = JSON.stringify(configOverride) - - if (options.env) { - Object.assign(env, options.env) - } - - const execArgv = options.nodeArgs || [] - // FIXME: too slow :/ - // execArgv.push('--enable-source-maps') - - const forkOptions = { - silent: true, - env, - detached: false, - execArgv - } - - const peertubeArgs = options.peertubeArgs || [] - - return new Promise((res, rej) => { - const self = this - let aggregatedLogs = '' - - this.app = fork(join(root(), 'dist', 'server.js'), peertubeArgs, forkOptions) - - const onPeerTubeExit = () => rej(new Error('Process exited:\n' + aggregatedLogs)) - const onParentExit = () => { - if (!this.app?.pid) return - - try { - process.kill(self.app.pid) - } catch { /* empty */ } - } - - this.app.on('exit', onPeerTubeExit) - process.on('exit', onParentExit) - - this.app.stdout.on('data', function onStdout (data) { - let dontContinue = false - - const log: string = data.toString() - aggregatedLogs += log - - // Capture things if we want to - for (const key of Object.keys(regexps)) { - const regexp = regexps[key] - const matches = log.match(regexp) - if (matches !== null) { - if (key === 'client_id') self.store.client.id = matches[1] - else if (key === 'client_secret') self.store.client.secret = matches[1] - else if (key === 'user_username') self.store.user.username = matches[1] - else if (key === 'user_password') self.store.user.password = matches[1] - } - } - - // Check if all required sentences are here - for (const key of Object.keys(serverRunString)) { - if (log.includes(key)) serverRunString[key] = true - if (serverRunString[key] === false) dontContinue = true - } - - // If no, there is maybe one thing not already initialized (client/user credentials generation...) - if (dontContinue === true) return - - if (options.hideLogs === false) { - console.log(log) - } else { - process.removeListener('exit', onParentExit) - self.app.stdout.removeListener('data', onStdout) - self.app.removeListener('exit', onPeerTubeExit) - } - - res() - }) - }) - } - - kill () { - if (!this.app) return Promise.resolve() - - process.kill(this.app.pid) - - this.app = null - - return Promise.resolve() - } - - private randomServer () { - const low = 2500 - const high = 10000 - - return randomInt(low, high) - } - - private randomRTMP () { - const low = 1900 - const high = 2100 - - return randomInt(low, high) - } - - private async assignCustomConfigFile () { - if (this.internalServerNumber === this.serverNumber) return - - const basePath = join(root(), 'config') - - const tmpConfigFile = join(basePath, `test-${this.internalServerNumber}.yaml`) - await copy(join(basePath, `test-${this.serverNumber}.yaml`), tmpConfigFile) - - this.customConfigFile = tmpConfigFile - } - - private buildConfigOverride () { - if (!this.parallel) return {} - - return { - listen: { - port: this.port - }, - webserver: { - port: this.port - }, - database: { - suffix: '_test' + this.internalServerNumber - }, - storage: { - tmp: this.getDirectoryPath('tmp') + '/', - tmp_persistent: this.getDirectoryPath('tmp-persistent') + '/', - bin: this.getDirectoryPath('bin') + '/', - avatars: this.getDirectoryPath('avatars') + '/', - web_videos: this.getDirectoryPath('web-videos') + '/', - streaming_playlists: this.getDirectoryPath('streaming-playlists') + '/', - redundancy: this.getDirectoryPath('redundancy') + '/', - logs: this.getDirectoryPath('logs') + '/', - previews: this.getDirectoryPath('previews') + '/', - thumbnails: this.getDirectoryPath('thumbnails') + '/', - storyboards: this.getDirectoryPath('storyboards') + '/', - torrents: this.getDirectoryPath('torrents') + '/', - captions: this.getDirectoryPath('captions') + '/', - cache: this.getDirectoryPath('cache') + '/', - plugins: this.getDirectoryPath('plugins') + '/', - well_known: this.getDirectoryPath('well-known') + '/' - }, - admin: { - email: `admin${this.internalServerNumber}@example.com` - }, - live: { - rtmp: { - port: this.rtmpPort - } - } - } - } - - private assignCommands () { - this.bulk = new BulkCommand(this) - this.cli = new CLICommand(this) - this.customPage = new CustomPagesCommand(this) - this.feed = new FeedCommand(this) - this.logs = new LogsCommand(this) - this.abuses = new AbusesCommand(this) - this.overviews = new OverviewsCommand(this) - this.search = new SearchCommand(this) - this.contactForm = new ContactFormCommand(this) - this.debug = new DebugCommand(this) - this.follows = new FollowsCommand(this) - this.jobs = new JobsCommand(this) - this.metrics = new MetricsCommand(this) - this.plugins = new PluginsCommand(this) - this.redundancy = new RedundancyCommand(this) - this.stats = new StatsCommand(this) - this.config = new ConfigCommand(this) - this.socketIO = new SocketIOCommand(this) - this.accounts = new AccountsCommand(this) - this.blocklist = new BlocklistCommand(this) - this.subscriptions = new SubscriptionsCommand(this) - this.live = new LiveCommand(this) - this.services = new ServicesCommand(this) - this.blacklist = new BlacklistCommand(this) - this.captions = new CaptionsCommand(this) - this.changeOwnership = new ChangeOwnershipCommand(this) - this.playlists = new PlaylistsCommand(this) - this.history = new HistoryCommand(this) - this.imports = new ImportsCommand(this) - this.channelSyncs = new ChannelSyncsCommand(this) - this.streamingPlaylists = new StreamingPlaylistsCommand(this) - this.channels = new ChannelsCommand(this) - this.comments = new CommentsCommand(this) - this.notifications = new NotificationsCommand(this) - this.servers = new ServersCommand(this) - this.login = new LoginCommand(this) - this.users = new UsersCommand(this) - this.videos = new VideosCommand(this) - this.videoStudio = new VideoStudioCommand(this) - this.videoStats = new VideoStatsCommand(this) - this.views = new ViewsCommand(this) - this.twoFactor = new TwoFactorCommand(this) - this.videoToken = new VideoTokenCommand(this) - this.registrations = new RegistrationsCommand(this) - - this.storyboard = new StoryboardCommand(this) - - this.runners = new RunnersCommand(this) - this.runnerRegistrationTokens = new RunnerRegistrationTokensCommand(this) - this.runnerJobs = new RunnerJobsCommand(this) - this.videoPasswords = new VideoPasswordsCommand(this) - } -} diff --git a/shared/server-commands/server/servers-command.ts b/shared/server-commands/server/servers-command.ts deleted file mode 100644 index 54e586a18..000000000 --- a/shared/server-commands/server/servers-command.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { exec } from 'child_process' -import { copy, ensureDir, readFile, readdir, remove } from 'fs-extra' -import { basename, join } from 'path' -import { isGithubCI, root, wait } from '@shared/core-utils' -import { getFileSize } from '@shared/extra-utils' -import { HttpStatusCode } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class ServersCommand extends AbstractCommand { - - static flushTests (internalServerNumber: number) { - return new Promise((res, rej) => { - const suffix = ` -- ${internalServerNumber}` - - return exec('npm run clean:server:test' + suffix, (err, _stdout, stderr) => { - if (err || stderr) return rej(err || new Error(stderr)) - - return res() - }) - }) - } - - ping (options: OverrideCommandOptions = {}) { - return this.getRequestBody({ - ...options, - - path: '/api/v1/ping', - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - cleanupTests () { - const promises: Promise[] = [] - - const saveGithubLogsIfNeeded = async () => { - if (!isGithubCI()) return - - await ensureDir('artifacts') - - const origin = this.buildDirectory('logs/peertube.log') - const destname = `peertube-${this.server.internalServerNumber}.log` - console.log('Saving logs %s.', destname) - - await copy(origin, join('artifacts', destname)) - } - - if (this.server.parallel) { - const promise = saveGithubLogsIfNeeded() - .then(() => ServersCommand.flushTests(this.server.internalServerNumber)) - - promises.push(promise) - } - - if (this.server.customConfigFile) { - promises.push(remove(this.server.customConfigFile)) - } - - return promises - } - - async waitUntilLog (str: string, count = 1, strictCount = true) { - const logfile = this.buildDirectory('logs/peertube.log') - - while (true) { - const buf = await readFile(logfile) - - const matches = buf.toString().match(new RegExp(str, 'g')) - if (matches && matches.length === count) return - if (matches && strictCount === false && matches.length >= count) return - - await wait(1000) - } - } - - buildDirectory (directory: string) { - return join(root(), 'test' + this.server.internalServerNumber, directory) - } - - async countFiles (directory: string) { - const files = await readdir(this.buildDirectory(directory)) - - return files.length - } - - buildWebVideoFilePath (fileUrl: string) { - return this.buildDirectory(join('web-videos', basename(fileUrl))) - } - - buildFragmentedFilePath (videoUUID: string, fileUrl: string) { - return this.buildDirectory(join('streaming-playlists', 'hls', videoUUID, basename(fileUrl))) - } - - getLogContent () { - return readFile(this.buildDirectory('logs/peertube.log')) - } - - async getServerFileSize (subPath: string) { - const path = this.server.servers.buildDirectory(subPath) - - return getFileSize(path) - } -} diff --git a/shared/server-commands/server/servers.ts b/shared/server-commands/server/servers.ts deleted file mode 100644 index fe9da9e63..000000000 --- a/shared/server-commands/server/servers.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { ensureDir } from 'fs-extra' -import { isGithubCI } from '@shared/core-utils' -import { PeerTubeServer, RunServerOptions } from './server' - -async function createSingleServer (serverNumber: number, configOverride?: object, options: RunServerOptions = {}) { - const server = new PeerTubeServer({ serverNumber }) - - await server.flushAndRun(configOverride, options) - - return server -} - -function createMultipleServers (totalServers: number, configOverride?: object, options: RunServerOptions = {}) { - const serverPromises: Promise[] = [] - - for (let i = 1; i <= totalServers; i++) { - serverPromises.push(createSingleServer(i, configOverride, options)) - } - - return Promise.all(serverPromises) -} - -function killallServers (servers: PeerTubeServer[]) { - return Promise.all(servers.map(s => s.kill())) -} - -async function cleanupTests (servers: PeerTubeServer[]) { - await killallServers(servers) - - if (isGithubCI()) { - await ensureDir('artifacts') - } - - let p: Promise[] = [] - for (const server of servers) { - p = p.concat(server.servers.cleanupTests()) - } - - return Promise.all(p) -} - -function getServerImportConfig (mode: 'youtube-dl' | 'yt-dlp') { - return { - import: { - videos: { - http: { - youtube_dl_release: { - url: mode === 'youtube-dl' - ? 'https://yt-dl.org/downloads/latest/youtube-dl' - : 'https://api.github.com/repos/yt-dlp/yt-dlp/releases', - - name: mode - } - } - } - } - } -} - -// --------------------------------------------------------------------------- - -export { - createSingleServer, - createMultipleServers, - cleanupTests, - killallServers, - getServerImportConfig -} diff --git a/shared/server-commands/server/stats-command.ts b/shared/server-commands/server/stats-command.ts deleted file mode 100644 index 64a452306..000000000 --- a/shared/server-commands/server/stats-command.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { HttpStatusCode, ServerStats } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class StatsCommand extends AbstractCommand { - - get (options: OverrideCommandOptions & { - useCache?: boolean // default false - } = {}) { - const { useCache = false } = options - const path = '/api/v1/server/stats' - - const query = { - t: useCache ? undefined : new Date().getTime() - } - - return this.getRequestBody({ - ...options, - - path, - query, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } -} diff --git a/shared/server-commands/shared/abstract-command.ts b/shared/server-commands/shared/abstract-command.ts deleted file mode 100644 index 463acc26b..000000000 --- a/shared/server-commands/shared/abstract-command.ts +++ /dev/null @@ -1,223 +0,0 @@ -import { isAbsolute, join } from 'path' -import { root } from '@shared/core-utils' -import { - makeDeleteRequest, - makeGetRequest, - makePostBodyRequest, - makePutBodyRequest, - makeUploadRequest, - unwrapBody, - unwrapText -} from '../requests/requests' -import { PeerTubeServer } from '../server/server' - -export interface OverrideCommandOptions { - token?: string - expectedStatus?: number -} - -interface InternalCommonCommandOptions extends OverrideCommandOptions { - // Default to server.url - url?: string - - path: string - // If we automatically send the server token if the token is not provided - implicitToken: boolean - defaultExpectedStatus: number - - // Common optional request parameters - contentType?: string - accept?: string - redirects?: number - range?: string - host?: string - headers?: { [ name: string ]: string } - requestType?: string - responseType?: string - xForwardedFor?: string -} - -interface InternalGetCommandOptions extends InternalCommonCommandOptions { - query?: { [ id: string ]: any } -} - -interface InternalDeleteCommandOptions extends InternalCommonCommandOptions { - query?: { [ id: string ]: any } - rawQuery?: string -} - -abstract class AbstractCommand { - - constructor ( - protected server: PeerTubeServer - ) { - - } - - protected getRequestBody (options: InternalGetCommandOptions) { - return unwrapBody(this.getRequest(options)) - } - - protected getRequestText (options: InternalGetCommandOptions) { - return unwrapText(this.getRequest(options)) - } - - protected getRawRequest (options: Omit) { - const { url, range } = options - const { host, protocol, pathname } = new URL(url) - - return this.getRequest({ - ...options, - - token: this.buildCommonRequestToken(options), - defaultExpectedStatus: this.buildExpectedStatus(options), - - url: `${protocol}//${host}`, - path: pathname, - range - }) - } - - protected getRequest (options: InternalGetCommandOptions) { - const { query } = options - - return makeGetRequest({ - ...this.buildCommonRequestOptions(options), - - query - }) - } - - protected deleteRequest (options: InternalDeleteCommandOptions) { - const { query, rawQuery } = options - - return makeDeleteRequest({ - ...this.buildCommonRequestOptions(options), - - query, - rawQuery - }) - } - - protected putBodyRequest (options: InternalCommonCommandOptions & { - fields?: { [ fieldName: string ]: any } - headers?: { [name: string]: string } - }) { - const { fields, headers } = options - - return makePutBodyRequest({ - ...this.buildCommonRequestOptions(options), - - fields, - headers - }) - } - - protected postBodyRequest (options: InternalCommonCommandOptions & { - fields?: { [ fieldName: string ]: any } - headers?: { [name: string]: string } - }) { - const { fields, headers } = options - - return makePostBodyRequest({ - ...this.buildCommonRequestOptions(options), - - fields, - headers - }) - } - - protected postUploadRequest (options: InternalCommonCommandOptions & { - fields?: { [ fieldName: string ]: any } - attaches?: { [ fieldName: string ]: any } - }) { - const { fields, attaches } = options - - return makeUploadRequest({ - ...this.buildCommonRequestOptions(options), - - method: 'POST', - fields, - attaches - }) - } - - protected putUploadRequest (options: InternalCommonCommandOptions & { - fields?: { [ fieldName: string ]: any } - attaches?: { [ fieldName: string ]: any } - }) { - const { fields, attaches } = options - - return makeUploadRequest({ - ...this.buildCommonRequestOptions(options), - - method: 'PUT', - fields, - attaches - }) - } - - protected updateImageRequest (options: InternalCommonCommandOptions & { - fixture: string - fieldname: string - }) { - const filePath = isAbsolute(options.fixture) - ? options.fixture - : join(root(), 'server', 'tests', 'fixtures', options.fixture) - - return this.postUploadRequest({ - ...options, - - fields: {}, - attaches: { [options.fieldname]: filePath } - }) - } - - protected buildCommonRequestOptions (options: InternalCommonCommandOptions) { - const { url, path, redirects, contentType, accept, range, host, headers, requestType, xForwardedFor, responseType } = options - - return { - url: url ?? this.server.url, - path, - - token: this.buildCommonRequestToken(options), - expectedStatus: this.buildExpectedStatus(options), - - redirects, - contentType, - range, - host, - accept, - headers, - type: requestType, - responseType, - xForwardedFor - } - } - - protected buildCommonRequestToken (options: Pick) { - const { token } = options - - const fallbackToken = options.implicitToken - ? this.server.accessToken - : undefined - - return token !== undefined ? token : fallbackToken - } - - protected buildExpectedStatus (options: Pick) { - const { expectedStatus, defaultExpectedStatus } = options - - return expectedStatus !== undefined ? expectedStatus : defaultExpectedStatus - } - - protected buildVideoPasswordHeader (videoPassword: string) { - return videoPassword !== undefined && videoPassword !== null - ? { 'x-peertube-video-password': videoPassword } - : undefined - } -} - -export { - AbstractCommand -} diff --git a/shared/server-commands/shared/index.ts b/shared/server-commands/shared/index.ts deleted file mode 100644 index e807ab4f7..000000000 --- a/shared/server-commands/shared/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './abstract-command' diff --git a/shared/server-commands/socket/index.ts b/shared/server-commands/socket/index.ts deleted file mode 100644 index 594329b2f..000000000 --- a/shared/server-commands/socket/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './socket-io-command' diff --git a/shared/server-commands/socket/socket-io-command.ts b/shared/server-commands/socket/socket-io-command.ts deleted file mode 100644 index c28a86366..000000000 --- a/shared/server-commands/socket/socket-io-command.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { io } from 'socket.io-client' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class SocketIOCommand extends AbstractCommand { - - getUserNotificationSocket (options: OverrideCommandOptions = {}) { - return io(this.server.url + '/user-notifications', { - query: { accessToken: options.token ?? this.server.accessToken } - }) - } - - getLiveNotificationSocket () { - return io(this.server.url + '/live-videos') - } - - getRunnersSocket (options: { - runnerToken: string - }) { - return io(this.server.url + '/runners', { - reconnection: false, - auth: { runnerToken: options.runnerToken } - }) - } -} diff --git a/shared/server-commands/users/accounts-command.ts b/shared/server-commands/users/accounts-command.ts deleted file mode 100644 index 5844b330b..000000000 --- a/shared/server-commands/users/accounts-command.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { Account, AccountVideoRate, ActorFollow, HttpStatusCode, ResultList, VideoRateType } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class AccountsCommand extends AbstractCommand { - - list (options: OverrideCommandOptions & { - sort?: string // default -createdAt - } = {}) { - const { sort = '-createdAt' } = options - const path = '/api/v1/accounts' - - return this.getRequestBody>({ - ...options, - - path, - query: { sort }, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - get (options: OverrideCommandOptions & { - accountName: string - }) { - const path = '/api/v1/accounts/' + options.accountName - - return this.getRequestBody({ - ...options, - - path, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - listRatings (options: OverrideCommandOptions & { - accountName: string - rating?: VideoRateType - }) { - const { rating, accountName } = options - const path = '/api/v1/accounts/' + accountName + '/ratings' - - const query = { rating } - - return this.getRequestBody>({ - ...options, - - path, - query, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - listFollowers (options: OverrideCommandOptions & { - accountName: string - start?: number - count?: number - sort?: string - search?: string - }) { - const { accountName, start, count, sort, search } = options - const path = '/api/v1/accounts/' + accountName + '/followers' - - const query = { start, count, sort, search } - - return this.getRequestBody>({ - ...options, - - path, - query, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } -} diff --git a/shared/server-commands/users/accounts.ts b/shared/server-commands/users/accounts.ts deleted file mode 100644 index 6387891f4..000000000 --- a/shared/server-commands/users/accounts.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { PeerTubeServer } from '../server/server' - -async function setDefaultAccountAvatar (serversArg: PeerTubeServer | PeerTubeServer[], token?: string) { - const servers = Array.isArray(serversArg) - ? serversArg - : [ serversArg ] - - for (const server of servers) { - await server.users.updateMyAvatar({ fixture: 'avatar.png', token }) - } -} - -export { - setDefaultAccountAvatar -} diff --git a/shared/server-commands/users/blocklist-command.ts b/shared/server-commands/users/blocklist-command.ts deleted file mode 100644 index 862d8945e..000000000 --- a/shared/server-commands/users/blocklist-command.ts +++ /dev/null @@ -1,165 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { AccountBlock, BlockStatus, HttpStatusCode, ResultList, ServerBlock } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -type ListBlocklistOptions = OverrideCommandOptions & { - start: number - count: number - - sort?: string // default -createdAt - - search?: string -} - -export class BlocklistCommand extends AbstractCommand { - - listMyAccountBlocklist (options: ListBlocklistOptions) { - const path = '/api/v1/users/me/blocklist/accounts' - - return this.listBlocklist(options, path) - } - - listMyServerBlocklist (options: ListBlocklistOptions) { - const path = '/api/v1/users/me/blocklist/servers' - - return this.listBlocklist(options, path) - } - - listServerAccountBlocklist (options: ListBlocklistOptions) { - const path = '/api/v1/server/blocklist/accounts' - - return this.listBlocklist(options, path) - } - - listServerServerBlocklist (options: ListBlocklistOptions) { - const path = '/api/v1/server/blocklist/servers' - - return this.listBlocklist(options, path) - } - - // --------------------------------------------------------------------------- - - getStatus (options: OverrideCommandOptions & { - accounts?: string[] - hosts?: string[] - }) { - const { accounts, hosts } = options - - const path = '/api/v1/blocklist/status' - - return this.getRequestBody({ - ...options, - - path, - query: { - accounts, - hosts - }, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - // --------------------------------------------------------------------------- - - addToMyBlocklist (options: OverrideCommandOptions & { - account?: string - server?: string - }) { - const { account, server } = options - - const path = account - ? '/api/v1/users/me/blocklist/accounts' - : '/api/v1/users/me/blocklist/servers' - - return this.postBodyRequest({ - ...options, - - path, - fields: { - accountName: account, - host: server - }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - addToServerBlocklist (options: OverrideCommandOptions & { - account?: string - server?: string - }) { - const { account, server } = options - - const path = account - ? '/api/v1/server/blocklist/accounts' - : '/api/v1/server/blocklist/servers' - - return this.postBodyRequest({ - ...options, - - path, - fields: { - accountName: account, - host: server - }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - // --------------------------------------------------------------------------- - - removeFromMyBlocklist (options: OverrideCommandOptions & { - account?: string - server?: string - }) { - const { account, server } = options - - const path = account - ? '/api/v1/users/me/blocklist/accounts/' + account - : '/api/v1/users/me/blocklist/servers/' + server - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - removeFromServerBlocklist (options: OverrideCommandOptions & { - account?: string - server?: string - }) { - const { account, server } = options - - const path = account - ? '/api/v1/server/blocklist/accounts/' + account - : '/api/v1/server/blocklist/servers/' + server - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - private listBlocklist (options: ListBlocklistOptions, path: string) { - const { start, count, search, sort = '-createdAt' } = options - - return this.getRequestBody>({ - ...options, - - path, - query: { start, count, sort, search }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - -} diff --git a/shared/server-commands/users/index.ts b/shared/server-commands/users/index.ts deleted file mode 100644 index 404756539..000000000 --- a/shared/server-commands/users/index.ts +++ /dev/null @@ -1,10 +0,0 @@ -export * from './accounts-command' -export * from './accounts' -export * from './blocklist-command' -export * from './login' -export * from './login-command' -export * from './notifications-command' -export * from './registrations-command' -export * from './subscriptions-command' -export * from './two-factor-command' -export * from './users-command' diff --git a/shared/server-commands/users/login-command.ts b/shared/server-commands/users/login-command.ts deleted file mode 100644 index f2fc6d1c5..000000000 --- a/shared/server-commands/users/login-command.ts +++ /dev/null @@ -1,159 +0,0 @@ -import { HttpStatusCode, PeerTubeProblemDocument } from '@shared/models' -import { unwrapBody } from '../requests' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -type LoginOptions = OverrideCommandOptions & { - client?: { id?: string, secret?: string } - user?: { username: string, password?: string } - otpToken?: string -} - -export class LoginCommand extends AbstractCommand { - - async login (options: LoginOptions = {}) { - const res = await this._login(options) - - return this.unwrapLoginBody(res.body) - } - - async loginAndGetResponse (options: LoginOptions = {}) { - const res = await this._login(options) - - return { - res, - body: this.unwrapLoginBody(res.body) - } - } - - getAccessToken (arg1?: { username: string, password?: string }): Promise - getAccessToken (arg1: string, password?: string): Promise - async getAccessToken (arg1?: { username: string, password?: string } | string, password?: string) { - let user: { username: string, password?: string } - - if (!arg1) user = this.server.store.user - else if (typeof arg1 === 'object') user = arg1 - else user = { username: arg1, password } - - try { - const body = await this.login({ user }) - - return body.access_token - } catch (err) { - throw new Error(`Cannot authenticate. Please check your username/password. (${err})`) - } - } - - loginUsingExternalToken (options: OverrideCommandOptions & { - username: string - externalAuthToken: string - }) { - const { username, externalAuthToken } = options - const path = '/api/v1/users/token' - - const body = { - client_id: this.server.store.client.id, - client_secret: this.server.store.client.secret, - username, - response_type: 'code', - grant_type: 'password', - scope: 'upload', - externalAuthToken - } - - return this.postBodyRequest({ - ...options, - - path, - requestType: 'form', - fields: body, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - logout (options: OverrideCommandOptions & { - token: string - }) { - const path = '/api/v1/users/revoke-token' - - return unwrapBody<{ redirectUrl: string }>(this.postBodyRequest({ - ...options, - - path, - requestType: 'form', - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - })) - } - - refreshToken (options: OverrideCommandOptions & { - refreshToken: string - }) { - const path = '/api/v1/users/token' - - const body = { - client_id: this.server.store.client.id, - client_secret: this.server.store.client.secret, - refresh_token: options.refreshToken, - response_type: 'code', - grant_type: 'refresh_token' - } - - return this.postBodyRequest({ - ...options, - - path, - requestType: 'form', - fields: body, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getClient (options: OverrideCommandOptions = {}) { - const path = '/api/v1/oauth-clients/local' - - return this.getRequestBody<{ client_id: string, client_secret: string }>({ - ...options, - - path, - host: this.server.host, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - private _login (options: LoginOptions) { - const { client = this.server.store.client, user = this.server.store.user, otpToken } = options - const path = '/api/v1/users/token' - - const body = { - client_id: client.id, - client_secret: client.secret, - username: user.username, - password: user.password ?? 'password', - response_type: 'code', - grant_type: 'password', - scope: 'upload' - } - - const headers = otpToken - ? { 'x-peertube-otp': otpToken } - : {} - - return this.postBodyRequest({ - ...options, - - path, - headers, - requestType: 'form', - fields: body, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - private unwrapLoginBody (body: any) { - return body as { access_token: string, refresh_token: string } & PeerTubeProblemDocument - } -} diff --git a/shared/server-commands/users/login.ts b/shared/server-commands/users/login.ts deleted file mode 100644 index f1df027d3..000000000 --- a/shared/server-commands/users/login.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { PeerTubeServer } from '../server/server' - -function setAccessTokensToServers (servers: PeerTubeServer[]) { - const tasks: Promise[] = [] - - for (const server of servers) { - const p = server.login.getAccessToken() - .then(t => { server.accessToken = t }) - tasks.push(p) - } - - return Promise.all(tasks) -} - -// --------------------------------------------------------------------------- - -export { - setAccessTokensToServers -} diff --git a/shared/server-commands/users/notifications-command.ts b/shared/server-commands/users/notifications-command.ts deleted file mode 100644 index 6bd815daa..000000000 --- a/shared/server-commands/users/notifications-command.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { HttpStatusCode, ResultList, UserNotification, UserNotificationSetting } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class NotificationsCommand extends AbstractCommand { - - updateMySettings (options: OverrideCommandOptions & { - settings: UserNotificationSetting - }) { - const path = '/api/v1/users/me/notification-settings' - - return this.putBodyRequest({ - ...options, - - path, - fields: options.settings, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - list (options: OverrideCommandOptions & { - start?: number - count?: number - unread?: boolean - sort?: string - }) { - const { start, count, unread, sort = '-createdAt' } = options - const path = '/api/v1/users/me/notifications' - - return this.getRequestBody>({ - ...options, - - path, - query: { - start, - count, - sort, - unread - }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - markAsRead (options: OverrideCommandOptions & { - ids: number[] - }) { - const { ids } = options - const path = '/api/v1/users/me/notifications/read' - - return this.postBodyRequest({ - ...options, - - path, - fields: { ids }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - markAsReadAll (options: OverrideCommandOptions) { - const path = '/api/v1/users/me/notifications/read-all' - - return this.postBodyRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - async getLatest (options: OverrideCommandOptions = {}) { - const { total, data } = await this.list({ - ...options, - start: 0, - count: 1, - sort: '-createdAt' - }) - - if (total === 0) return undefined - - return data[0] - } -} diff --git a/shared/server-commands/users/registrations-command.ts b/shared/server-commands/users/registrations-command.ts deleted file mode 100644 index f57f54b34..000000000 --- a/shared/server-commands/users/registrations-command.ts +++ /dev/null @@ -1,151 +0,0 @@ -import { pick } from '@shared/core-utils' -import { HttpStatusCode, ResultList, UserRegistration, UserRegistrationRequest, UserRegistrationUpdateState } from '@shared/models' -import { unwrapBody } from '../requests' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class RegistrationsCommand extends AbstractCommand { - - register (options: OverrideCommandOptions & Partial & Pick) { - const { password = 'password', email = options.username + '@example.com' } = options - const path = '/api/v1/users/register' - - return this.postBodyRequest({ - ...options, - - path, - fields: { - ...pick(options, [ 'username', 'displayName', 'channel' ]), - - password, - email - }, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - requestRegistration ( - options: OverrideCommandOptions & Partial & Pick - ) { - const { password = 'password', email = options.username + '@example.com' } = options - const path = '/api/v1/users/registrations/request' - - return unwrapBody(this.postBodyRequest({ - ...options, - - path, - fields: { - ...pick(options, [ 'username', 'displayName', 'channel', 'registrationReason' ]), - - password, - email - }, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - })) - } - - // --------------------------------------------------------------------------- - - accept (options: OverrideCommandOptions & { id: number } & UserRegistrationUpdateState) { - const { id } = options - const path = '/api/v1/users/registrations/' + id + '/accept' - - return this.postBodyRequest({ - ...options, - - path, - fields: pick(options, [ 'moderationResponse', 'preventEmailDelivery' ]), - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - reject (options: OverrideCommandOptions & { id: number } & UserRegistrationUpdateState) { - const { id } = options - const path = '/api/v1/users/registrations/' + id + '/reject' - - return this.postBodyRequest({ - ...options, - - path, - fields: pick(options, [ 'moderationResponse', 'preventEmailDelivery' ]), - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - // --------------------------------------------------------------------------- - - delete (options: OverrideCommandOptions & { - id: number - }) { - const { id } = options - const path = '/api/v1/users/registrations/' + id - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - // --------------------------------------------------------------------------- - - list (options: OverrideCommandOptions & { - start?: number - count?: number - sort?: string - search?: string - } = {}) { - const path = '/api/v1/users/registrations' - - return this.getRequestBody>({ - ...options, - - path, - query: pick(options, [ 'start', 'count', 'sort', 'search' ]), - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - // --------------------------------------------------------------------------- - - askSendVerifyEmail (options: OverrideCommandOptions & { - email: string - }) { - const { email } = options - const path = '/api/v1/users/registrations/ask-send-verify-email' - - return this.postBodyRequest({ - ...options, - - path, - fields: { email }, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - verifyEmail (options: OverrideCommandOptions & { - registrationId: number - verificationString: string - }) { - const { registrationId, verificationString } = options - const path = '/api/v1/users/registrations/' + registrationId + '/verify-email' - - return this.postBodyRequest({ - ...options, - - path, - fields: { - verificationString - }, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } -} diff --git a/shared/server-commands/users/subscriptions-command.ts b/shared/server-commands/users/subscriptions-command.ts deleted file mode 100644 index b92f037f8..000000000 --- a/shared/server-commands/users/subscriptions-command.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { HttpStatusCode, ResultList, VideoChannel } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class SubscriptionsCommand extends AbstractCommand { - - add (options: OverrideCommandOptions & { - targetUri: string - }) { - const path = '/api/v1/users/me/subscriptions' - - return this.postBodyRequest({ - ...options, - - path, - fields: { uri: options.targetUri }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - list (options: OverrideCommandOptions & { - sort?: string // default -createdAt - search?: string - } = {}) { - const { sort = '-createdAt', search } = options - const path = '/api/v1/users/me/subscriptions' - - return this.getRequestBody>({ - ...options, - - path, - query: { - sort, - search - }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - get (options: OverrideCommandOptions & { - uri: string - }) { - const path = '/api/v1/users/me/subscriptions/' + options.uri - - return this.getRequestBody({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - remove (options: OverrideCommandOptions & { - uri: string - }) { - const path = '/api/v1/users/me/subscriptions/' + options.uri - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - exist (options: OverrideCommandOptions & { - uris: string[] - }) { - const path = '/api/v1/users/me/subscriptions/exist' - - return this.getRequestBody<{ [id: string ]: boolean }>({ - ...options, - - path, - query: { 'uris[]': options.uris }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } -} diff --git a/shared/server-commands/users/two-factor-command.ts b/shared/server-commands/users/two-factor-command.ts deleted file mode 100644 index 5542acfda..000000000 --- a/shared/server-commands/users/two-factor-command.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { TOTP } from 'otpauth' -import { HttpStatusCode, TwoFactorEnableResult } from '@shared/models' -import { unwrapBody } from '../requests' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class TwoFactorCommand extends AbstractCommand { - - static buildOTP (options: { - secret: string - }) { - const { secret } = options - - return new TOTP({ - issuer: 'PeerTube', - algorithm: 'SHA1', - digits: 6, - period: 30, - secret - }) - } - - request (options: OverrideCommandOptions & { - userId: number - currentPassword?: string - }) { - const { currentPassword, userId } = options - - const path = '/api/v1/users/' + userId + '/two-factor/request' - - return unwrapBody(this.postBodyRequest({ - ...options, - - path, - fields: { currentPassword }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - })) - } - - confirmRequest (options: OverrideCommandOptions & { - userId: number - requestToken: string - otpToken: string - }) { - const { userId, requestToken, otpToken } = options - - const path = '/api/v1/users/' + userId + '/two-factor/confirm-request' - - return this.postBodyRequest({ - ...options, - - path, - fields: { requestToken, otpToken }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - disable (options: OverrideCommandOptions & { - userId: number - currentPassword?: string - }) { - const { userId, currentPassword } = options - const path = '/api/v1/users/' + userId + '/two-factor/disable' - - return this.postBodyRequest({ - ...options, - - path, - fields: { currentPassword }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - async requestAndConfirm (options: OverrideCommandOptions & { - userId: number - currentPassword?: string - }) { - const { userId, currentPassword } = options - - const { otpRequest } = await this.request({ userId, currentPassword }) - - await this.confirmRequest({ - userId, - requestToken: otpRequest.requestToken, - otpToken: TwoFactorCommand.buildOTP({ secret: otpRequest.secret }).generate() - }) - - return otpRequest - } -} diff --git a/shared/server-commands/users/users-command.ts b/shared/server-commands/users/users-command.ts deleted file mode 100644 index 5b39d3488..000000000 --- a/shared/server-commands/users/users-command.ts +++ /dev/null @@ -1,388 +0,0 @@ -import { omit, pick } from '@shared/core-utils' -import { - HttpStatusCode, - MyUser, - ResultList, - ScopedToken, - User, - UserAdminFlag, - UserCreateResult, - UserRole, - UserUpdate, - UserUpdateMe, - UserVideoQuota, - UserVideoRate -} from '@shared/models' -import { unwrapBody } from '../requests' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class UsersCommand extends AbstractCommand { - - askResetPassword (options: OverrideCommandOptions & { - email: string - }) { - const { email } = options - const path = '/api/v1/users/ask-reset-password' - - return this.postBodyRequest({ - ...options, - - path, - fields: { email }, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - resetPassword (options: OverrideCommandOptions & { - userId: number - verificationString: string - password: string - }) { - const { userId, verificationString, password } = options - const path = '/api/v1/users/' + userId + '/reset-password' - - return this.postBodyRequest({ - ...options, - - path, - fields: { password, verificationString }, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - // --------------------------------------------------------------------------- - - askSendVerifyEmail (options: OverrideCommandOptions & { - email: string - }) { - const { email } = options - const path = '/api/v1/users/ask-send-verify-email' - - return this.postBodyRequest({ - ...options, - - path, - fields: { email }, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - verifyEmail (options: OverrideCommandOptions & { - userId: number - verificationString: string - isPendingEmail?: boolean // default false - }) { - const { userId, verificationString, isPendingEmail = false } = options - const path = '/api/v1/users/' + userId + '/verify-email' - - return this.postBodyRequest({ - ...options, - - path, - fields: { - verificationString, - isPendingEmail - }, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - // --------------------------------------------------------------------------- - - banUser (options: OverrideCommandOptions & { - userId: number - reason?: string - }) { - const { userId, reason } = options - const path = '/api/v1/users' + '/' + userId + '/block' - - return this.postBodyRequest({ - ...options, - - path, - fields: { reason }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - unbanUser (options: OverrideCommandOptions & { - userId: number - }) { - const { userId } = options - const path = '/api/v1/users' + '/' + userId + '/unblock' - - return this.postBodyRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - // --------------------------------------------------------------------------- - - getMyScopedTokens (options: OverrideCommandOptions = {}) { - const path = '/api/v1/users/scoped-tokens' - - return this.getRequestBody({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - renewMyScopedTokens (options: OverrideCommandOptions = {}) { - const path = '/api/v1/users/scoped-tokens' - - return this.postBodyRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - // --------------------------------------------------------------------------- - - create (options: OverrideCommandOptions & { - username: string - password?: string - videoQuota?: number - videoQuotaDaily?: number - role?: UserRole - adminFlags?: UserAdminFlag - }) { - const { - username, - adminFlags, - password = 'password', - videoQuota, - videoQuotaDaily, - role = UserRole.USER - } = options - - const path = '/api/v1/users' - - return unwrapBody<{ user: UserCreateResult }>(this.postBodyRequest({ - ...options, - - path, - fields: { - username, - password, - role, - adminFlags, - email: username + '@example.com', - videoQuota, - videoQuotaDaily - }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - })).then(res => res.user) - } - - async generate (username: string, role?: UserRole) { - const password = 'password' - const user = await this.create({ username, password, role }) - - const token = await this.server.login.getAccessToken({ username, password }) - - const me = await this.getMyInfo({ token }) - - return { - token, - userId: user.id, - userChannelId: me.videoChannels[0].id, - userChannelName: me.videoChannels[0].name, - password - } - } - - async generateUserAndToken (username: string, role?: UserRole) { - const password = 'password' - await this.create({ username, password, role }) - - return this.server.login.getAccessToken({ username, password }) - } - - // --------------------------------------------------------------------------- - - getMyInfo (options: OverrideCommandOptions = {}) { - const path = '/api/v1/users/me' - - return this.getRequestBody({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getMyQuotaUsed (options: OverrideCommandOptions = {}) { - const path = '/api/v1/users/me/video-quota-used' - - return this.getRequestBody({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getMyRating (options: OverrideCommandOptions & { - videoId: number | string - }) { - const { videoId } = options - const path = '/api/v1/users/me/videos/' + videoId + '/rating' - - return this.getRequestBody({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - deleteMe (options: OverrideCommandOptions = {}) { - const path = '/api/v1/users/me' - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - updateMe (options: OverrideCommandOptions & UserUpdateMe) { - const path = '/api/v1/users/me' - - const toSend: UserUpdateMe = omit(options, [ 'expectedStatus', 'token' ]) - - return this.putBodyRequest({ - ...options, - - path, - fields: toSend, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - updateMyAvatar (options: OverrideCommandOptions & { - fixture: string - }) { - const { fixture } = options - const path = '/api/v1/users/me/avatar/pick' - - return this.updateImageRequest({ - ...options, - - path, - fixture, - fieldname: 'avatarfile', - - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - // --------------------------------------------------------------------------- - - get (options: OverrideCommandOptions & { - userId: number - withStats?: boolean // default false - }) { - const { userId, withStats } = options - const path = '/api/v1/users/' + userId - - return this.getRequestBody({ - ...options, - - path, - query: { withStats }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - list (options: OverrideCommandOptions & { - start?: number - count?: number - sort?: string - search?: string - blocked?: boolean - } = {}) { - const path = '/api/v1/users' - - return this.getRequestBody>({ - ...options, - - path, - query: pick(options, [ 'start', 'count', 'sort', 'search', 'blocked' ]), - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - remove (options: OverrideCommandOptions & { - userId: number - }) { - const { userId } = options - const path = '/api/v1/users/' + userId - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - update (options: OverrideCommandOptions & { - userId: number - email?: string - emailVerified?: boolean - videoQuota?: number - videoQuotaDaily?: number - password?: string - adminFlags?: UserAdminFlag - pluginAuth?: string - role?: UserRole - }) { - const path = '/api/v1/users/' + options.userId - - const toSend: UserUpdate = {} - if (options.password !== undefined && options.password !== null) toSend.password = options.password - if (options.email !== undefined && options.email !== null) toSend.email = options.email - if (options.emailVerified !== undefined && options.emailVerified !== null) toSend.emailVerified = options.emailVerified - if (options.videoQuota !== undefined && options.videoQuota !== null) toSend.videoQuota = options.videoQuota - if (options.videoQuotaDaily !== undefined && options.videoQuotaDaily !== null) toSend.videoQuotaDaily = options.videoQuotaDaily - if (options.role !== undefined && options.role !== null) toSend.role = options.role - if (options.adminFlags !== undefined && options.adminFlags !== null) toSend.adminFlags = options.adminFlags - if (options.pluginAuth !== undefined) toSend.pluginAuth = options.pluginAuth - - return this.putBodyRequest({ - ...options, - - path, - fields: toSend, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } -} diff --git a/shared/server-commands/videos/blacklist-command.ts b/shared/server-commands/videos/blacklist-command.ts deleted file mode 100644 index 47e23ebc8..000000000 --- a/shared/server-commands/videos/blacklist-command.ts +++ /dev/null @@ -1,75 +0,0 @@ - -import { HttpStatusCode, ResultList, VideoBlacklist, VideoBlacklistType } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class BlacklistCommand extends AbstractCommand { - - add (options: OverrideCommandOptions & { - videoId: number | string - reason?: string - unfederate?: boolean - }) { - const { videoId, reason, unfederate } = options - const path = '/api/v1/videos/' + videoId + '/blacklist' - - return this.postBodyRequest({ - ...options, - - path, - fields: { reason, unfederate }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - update (options: OverrideCommandOptions & { - videoId: number | string - reason?: string - }) { - const { videoId, reason } = options - const path = '/api/v1/videos/' + videoId + '/blacklist' - - return this.putBodyRequest({ - ...options, - - path, - fields: { reason }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - remove (options: OverrideCommandOptions & { - videoId: number | string - }) { - const { videoId } = options - const path = '/api/v1/videos/' + videoId + '/blacklist' - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - list (options: OverrideCommandOptions & { - sort?: string - type?: VideoBlacklistType - } = {}) { - const { sort, type } = options - const path = '/api/v1/videos/blacklist/' - - const query = { sort, type } - - return this.getRequestBody>({ - ...options, - - path, - query, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } -} diff --git a/shared/server-commands/videos/captions-command.ts b/shared/server-commands/videos/captions-command.ts deleted file mode 100644 index a26fcb57d..000000000 --- a/shared/server-commands/videos/captions-command.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { buildAbsoluteFixturePath } from '@shared/core-utils' -import { HttpStatusCode, ResultList, VideoCaption } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class CaptionsCommand extends AbstractCommand { - - add (options: OverrideCommandOptions & { - videoId: string | number - language: string - fixture: string - mimeType?: string - }) { - const { videoId, language, fixture, mimeType } = options - - const path = '/api/v1/videos/' + videoId + '/captions/' + language - - const captionfile = buildAbsoluteFixturePath(fixture) - const captionfileAttach = mimeType - ? [ captionfile, { contentType: mimeType } ] - : captionfile - - return this.putUploadRequest({ - ...options, - - path, - fields: {}, - attaches: { - captionfile: captionfileAttach - }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - list (options: OverrideCommandOptions & { - videoId: string | number - videoPassword?: string - }) { - const { videoId, videoPassword } = options - const path = '/api/v1/videos/' + videoId + '/captions' - - return this.getRequestBody>({ - ...options, - - path, - headers: this.buildVideoPasswordHeader(videoPassword), - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - delete (options: OverrideCommandOptions & { - videoId: string | number - language: string - }) { - const { videoId, language } = options - const path = '/api/v1/videos/' + videoId + '/captions/' + language - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } -} diff --git a/shared/server-commands/videos/change-ownership-command.ts b/shared/server-commands/videos/change-ownership-command.ts deleted file mode 100644 index ad4c726ef..000000000 --- a/shared/server-commands/videos/change-ownership-command.ts +++ /dev/null @@ -1,68 +0,0 @@ - -import { HttpStatusCode, ResultList, VideoChangeOwnership } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class ChangeOwnershipCommand extends AbstractCommand { - - create (options: OverrideCommandOptions & { - videoId: number | string - username: string - }) { - const { videoId, username } = options - const path = '/api/v1/videos/' + videoId + '/give-ownership' - - return this.postBodyRequest({ - ...options, - - path, - fields: { username }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - list (options: OverrideCommandOptions = {}) { - const path = '/api/v1/videos/ownership' - - return this.getRequestBody>({ - ...options, - - path, - query: { sort: '-createdAt' }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - accept (options: OverrideCommandOptions & { - ownershipId: number - channelId: number - }) { - const { ownershipId, channelId } = options - const path = '/api/v1/videos/ownership/' + ownershipId + '/accept' - - return this.postBodyRequest({ - ...options, - - path, - fields: { channelId }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - refuse (options: OverrideCommandOptions & { - ownershipId: number - }) { - const { ownershipId } = options - const path = '/api/v1/videos/ownership/' + ownershipId + '/refuse' - - return this.postBodyRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } -} diff --git a/shared/server-commands/videos/channel-syncs-command.ts b/shared/server-commands/videos/channel-syncs-command.ts deleted file mode 100644 index de4a160ec..000000000 --- a/shared/server-commands/videos/channel-syncs-command.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { HttpStatusCode, ResultList, VideoChannelSync, VideoChannelSyncCreate } from '@shared/models' -import { pick } from '@shared/core-utils' -import { unwrapBody } from '../requests' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class ChannelSyncsCommand extends AbstractCommand { - private static readonly API_PATH = '/api/v1/video-channel-syncs' - - listByAccount (options: OverrideCommandOptions & { - accountName: string - start?: number - count?: number - sort?: string - }) { - const { accountName, sort = 'createdAt' } = options - - const path = `/api/v1/accounts/${accountName}/video-channel-syncs` - - return this.getRequestBody>({ - ...options, - - path, - query: { sort, ...pick(options, [ 'start', 'count' ]) }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - async create (options: OverrideCommandOptions & { - attributes: VideoChannelSyncCreate - }) { - return unwrapBody<{ videoChannelSync: VideoChannelSync }>(this.postBodyRequest({ - ...options, - - path: ChannelSyncsCommand.API_PATH, - fields: options.attributes, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - })) - } - - delete (options: OverrideCommandOptions & { - channelSyncId: number - }) { - const path = `${ChannelSyncsCommand.API_PATH}/${options.channelSyncId}` - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } -} diff --git a/shared/server-commands/videos/channels-command.ts b/shared/server-commands/videos/channels-command.ts deleted file mode 100644 index 385d0fe73..000000000 --- a/shared/server-commands/videos/channels-command.ts +++ /dev/null @@ -1,202 +0,0 @@ -import { pick } from '@shared/core-utils' -import { - ActorFollow, - HttpStatusCode, - ResultList, - VideoChannel, - VideoChannelCreate, - VideoChannelCreateResult, - VideoChannelUpdate, - VideosImportInChannelCreate -} from '@shared/models' -import { unwrapBody } from '../requests' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class ChannelsCommand extends AbstractCommand { - - list (options: OverrideCommandOptions & { - start?: number - count?: number - sort?: string - withStats?: boolean - } = {}) { - const path = '/api/v1/video-channels' - - return this.getRequestBody>({ - ...options, - - path, - query: pick(options, [ 'start', 'count', 'sort', 'withStats' ]), - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - listByAccount (options: OverrideCommandOptions & { - accountName: string - start?: number - count?: number - sort?: string - withStats?: boolean - search?: string - }) { - const { accountName, sort = 'createdAt' } = options - const path = '/api/v1/accounts/' + accountName + '/video-channels' - - return this.getRequestBody>({ - ...options, - - path, - query: { sort, ...pick(options, [ 'start', 'count', 'withStats', 'search' ]) }, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - async create (options: OverrideCommandOptions & { - attributes: Partial - }) { - const path = '/api/v1/video-channels/' - - // Default attributes - const defaultAttributes = { - displayName: 'my super video channel', - description: 'my super channel description', - support: 'my super channel support' - } - const attributes = { ...defaultAttributes, ...options.attributes } - - const body = await unwrapBody<{ videoChannel: VideoChannelCreateResult }>(this.postBodyRequest({ - ...options, - - path, - fields: attributes, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - })) - - return body.videoChannel - } - - update (options: OverrideCommandOptions & { - channelName: string - attributes: VideoChannelUpdate - }) { - const { channelName, attributes } = options - const path = '/api/v1/video-channels/' + channelName - - return this.putBodyRequest({ - ...options, - - path, - fields: attributes, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - delete (options: OverrideCommandOptions & { - channelName: string - }) { - const path = '/api/v1/video-channels/' + options.channelName - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - get (options: OverrideCommandOptions & { - channelName: string - }) { - const path = '/api/v1/video-channels/' + options.channelName - - return this.getRequestBody({ - ...options, - - path, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - updateImage (options: OverrideCommandOptions & { - fixture: string - channelName: string | number - type: 'avatar' | 'banner' - }) { - const { channelName, fixture, type } = options - - const path = `/api/v1/video-channels/${channelName}/${type}/pick` - - return this.updateImageRequest({ - ...options, - - path, - fixture, - fieldname: type + 'file', - - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - deleteImage (options: OverrideCommandOptions & { - channelName: string | number - type: 'avatar' | 'banner' - }) { - const { channelName, type } = options - - const path = `/api/v1/video-channels/${channelName}/${type}` - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - listFollowers (options: OverrideCommandOptions & { - channelName: string - start?: number - count?: number - sort?: string - search?: string - }) { - const { channelName, start, count, sort, search } = options - const path = '/api/v1/video-channels/' + channelName + '/followers' - - const query = { start, count, sort, search } - - return this.getRequestBody>({ - ...options, - - path, - query, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - importVideos (options: OverrideCommandOptions & VideosImportInChannelCreate & { - channelName: string - }) { - const { channelName, externalChannelUrl, videoChannelSyncId } = options - - const path = `/api/v1/video-channels/${channelName}/import-videos` - - return this.postBodyRequest({ - ...options, - - path, - fields: { externalChannelUrl, videoChannelSyncId }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } -} diff --git a/shared/server-commands/videos/channels.ts b/shared/server-commands/videos/channels.ts deleted file mode 100644 index 3c0d4b723..000000000 --- a/shared/server-commands/videos/channels.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { PeerTubeServer } from '../server/server' - -function setDefaultVideoChannel (servers: PeerTubeServer[]) { - const tasks: Promise[] = [] - - for (const server of servers) { - const p = server.users.getMyInfo() - .then(user => { server.store.channel = user.videoChannels[0] }) - - tasks.push(p) - } - - return Promise.all(tasks) -} - -async function setDefaultChannelAvatar (serversArg: PeerTubeServer | PeerTubeServer[], channelName: string = 'root_channel') { - const servers = Array.isArray(serversArg) - ? serversArg - : [ serversArg ] - - for (const server of servers) { - await server.channels.updateImage({ channelName, fixture: 'avatar.png', type: 'avatar' }) - } -} - -export { - setDefaultVideoChannel, - setDefaultChannelAvatar -} diff --git a/shared/server-commands/videos/comments-command.ts b/shared/server-commands/videos/comments-command.ts deleted file mode 100644 index 0dab1b66a..000000000 --- a/shared/server-commands/videos/comments-command.ts +++ /dev/null @@ -1,159 +0,0 @@ -import { pick } from '@shared/core-utils' -import { HttpStatusCode, ResultList, VideoComment, VideoCommentThreads, VideoCommentThreadTree } from '@shared/models' -import { unwrapBody } from '../requests' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class CommentsCommand extends AbstractCommand { - - private lastVideoId: number | string - private lastThreadId: number - private lastReplyId: number - - listForAdmin (options: OverrideCommandOptions & { - start?: number - count?: number - sort?: string - isLocal?: boolean - onLocalVideo?: boolean - search?: string - searchAccount?: string - searchVideo?: string - } = {}) { - const { sort = '-createdAt' } = options - const path = '/api/v1/videos/comments' - - const query = { sort, ...pick(options, [ 'start', 'count', 'isLocal', 'onLocalVideo', 'search', 'searchAccount', 'searchVideo' ]) } - - return this.getRequestBody>({ - ...options, - - path, - query, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - listThreads (options: OverrideCommandOptions & { - videoId: number | string - videoPassword?: string - start?: number - count?: number - sort?: string - }) { - const { start, count, sort, videoId, videoPassword } = options - const path = '/api/v1/videos/' + videoId + '/comment-threads' - - return this.getRequestBody({ - ...options, - - path, - query: { start, count, sort }, - headers: this.buildVideoPasswordHeader(videoPassword), - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getThread (options: OverrideCommandOptions & { - videoId: number | string - threadId: number - }) { - const { videoId, threadId } = options - const path = '/api/v1/videos/' + videoId + '/comment-threads/' + threadId - - return this.getRequestBody({ - ...options, - - path, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - async createThread (options: OverrideCommandOptions & { - videoId: number | string - text: string - videoPassword?: string - }) { - const { videoId, text, videoPassword } = options - const path = '/api/v1/videos/' + videoId + '/comment-threads' - - const body = await unwrapBody<{ comment: VideoComment }>(this.postBodyRequest({ - ...options, - - path, - fields: { text }, - headers: this.buildVideoPasswordHeader(videoPassword), - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - })) - - this.lastThreadId = body.comment?.id - this.lastVideoId = videoId - - return body.comment - } - - async addReply (options: OverrideCommandOptions & { - videoId: number | string - toCommentId: number - text: string - videoPassword?: string - }) { - const { videoId, toCommentId, text, videoPassword } = options - const path = '/api/v1/videos/' + videoId + '/comments/' + toCommentId - - const body = await unwrapBody<{ comment: VideoComment }>(this.postBodyRequest({ - ...options, - - path, - fields: { text }, - headers: this.buildVideoPasswordHeader(videoPassword), - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - })) - - this.lastReplyId = body.comment?.id - - return body.comment - } - - async addReplyToLastReply (options: OverrideCommandOptions & { - text: string - }) { - return this.addReply({ ...options, videoId: this.lastVideoId, toCommentId: this.lastReplyId }) - } - - async addReplyToLastThread (options: OverrideCommandOptions & { - text: string - }) { - return this.addReply({ ...options, videoId: this.lastVideoId, toCommentId: this.lastThreadId }) - } - - async findCommentId (options: OverrideCommandOptions & { - videoId: number | string - text: string - }) { - const { videoId, text } = options - const { data } = await this.listThreads({ videoId, count: 25, sort: '-createdAt' }) - - return data.find(c => c.text === text).id - } - - delete (options: OverrideCommandOptions & { - videoId: number | string - commentId: number - }) { - const { videoId, commentId } = options - const path = '/api/v1/videos/' + videoId + '/comments/' + commentId - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } -} diff --git a/shared/server-commands/videos/history-command.ts b/shared/server-commands/videos/history-command.ts deleted file mode 100644 index d27afcff2..000000000 --- a/shared/server-commands/videos/history-command.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { HttpStatusCode, ResultList, Video } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class HistoryCommand extends AbstractCommand { - - list (options: OverrideCommandOptions & { - search?: string - } = {}) { - const { search } = options - const path = '/api/v1/users/me/history/videos' - - return this.getRequestBody>({ - ...options, - - path, - query: { - search - }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - removeElement (options: OverrideCommandOptions & { - videoId: number - }) { - const { videoId } = options - const path = '/api/v1/users/me/history/videos/' + videoId - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - removeAll (options: OverrideCommandOptions & { - beforeDate?: string - } = {}) { - const { beforeDate } = options - const path = '/api/v1/users/me/history/videos/remove' - - return this.postBodyRequest({ - ...options, - - path, - fields: { beforeDate }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } -} diff --git a/shared/server-commands/videos/imports-command.ts b/shared/server-commands/videos/imports-command.ts deleted file mode 100644 index e307a79be..000000000 --- a/shared/server-commands/videos/imports-command.ts +++ /dev/null @@ -1,77 +0,0 @@ - -import { HttpStatusCode, ResultList } from '@shared/models' -import { VideoImport, VideoImportCreate } from '../../models/videos' -import { unwrapBody } from '../requests' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class ImportsCommand extends AbstractCommand { - - importVideo (options: OverrideCommandOptions & { - attributes: (VideoImportCreate | { torrentfile?: string, previewfile?: string, thumbnailfile?: string }) - }) { - const { attributes } = options - const path = '/api/v1/videos/imports' - - let attaches: any = {} - if (attributes.torrentfile) attaches = { torrentfile: attributes.torrentfile } - if (attributes.thumbnailfile) attaches = { thumbnailfile: attributes.thumbnailfile } - if (attributes.previewfile) attaches = { previewfile: attributes.previewfile } - - return unwrapBody(this.postUploadRequest({ - ...options, - - path, - attaches, - fields: options.attributes, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - })) - } - - delete (options: OverrideCommandOptions & { - importId: number - }) { - const path = '/api/v1/videos/imports/' + options.importId - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - cancel (options: OverrideCommandOptions & { - importId: number - }) { - const path = '/api/v1/videos/imports/' + options.importId + '/cancel' - - return this.postBodyRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - getMyVideoImports (options: OverrideCommandOptions & { - sort?: string - targetUrl?: string - videoChannelSyncId?: number - search?: string - } = {}) { - const { sort, targetUrl, videoChannelSyncId, search } = options - const path = '/api/v1/users/me/videos/imports' - - return this.getRequestBody>({ - ...options, - - path, - query: { sort, targetUrl, videoChannelSyncId, search }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } -} diff --git a/shared/server-commands/videos/index.ts b/shared/server-commands/videos/index.ts deleted file mode 100644 index 106d80af0..000000000 --- a/shared/server-commands/videos/index.ts +++ /dev/null @@ -1,21 +0,0 @@ -export * from './blacklist-command' -export * from './captions-command' -export * from './change-ownership-command' -export * from './channels' -export * from './channels-command' -export * from './channel-syncs-command' -export * from './comments-command' -export * from './history-command' -export * from './imports-command' -export * from './live-command' -export * from './live' -export * from './playlists-command' -export * from './services-command' -export * from './storyboard-command' -export * from './streaming-playlists-command' -export * from './comments-command' -export * from './video-studio-command' -export * from './video-token-command' -export * from './views-command' -export * from './videos-command' -export * from './video-passwords-command' diff --git a/shared/server-commands/videos/live-command.ts b/shared/server-commands/videos/live-command.ts deleted file mode 100644 index 6006d9fe9..000000000 --- a/shared/server-commands/videos/live-command.ts +++ /dev/null @@ -1,337 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { readdir } from 'fs-extra' -import { join } from 'path' -import { omit, wait } from '@shared/core-utils' -import { - HttpStatusCode, - LiveVideo, - LiveVideoCreate, - LiveVideoSession, - LiveVideoUpdate, - ResultList, - VideoCreateResult, - VideoDetails, - VideoPrivacy, - VideoState -} from '@shared/models' -import { unwrapBody } from '../requests' -import { ObjectStorageCommand, PeerTubeServer } from '../server' -import { AbstractCommand, OverrideCommandOptions } from '../shared' -import { sendRTMPStream, testFfmpegStreamError } from './live' - -export class LiveCommand extends AbstractCommand { - - get (options: OverrideCommandOptions & { - videoId: number | string - }) { - const path = '/api/v1/videos/live' - - return this.getRequestBody({ - ...options, - - path: path + '/' + options.videoId, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - // --------------------------------------------------------------------------- - - listSessions (options: OverrideCommandOptions & { - videoId: number | string - }) { - const path = `/api/v1/videos/live/${options.videoId}/sessions` - - return this.getRequestBody>({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - async findLatestSession (options: OverrideCommandOptions & { - videoId: number | string - }) { - const { data: sessions } = await this.listSessions(options) - - return sessions[sessions.length - 1] - } - - getReplaySession (options: OverrideCommandOptions & { - videoId: number | string - }) { - const path = `/api/v1/videos/${options.videoId}/live-session` - - return this.getRequestBody({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - // --------------------------------------------------------------------------- - - update (options: OverrideCommandOptions & { - videoId: number | string - fields: LiveVideoUpdate - }) { - const { videoId, fields } = options - const path = '/api/v1/videos/live' - - return this.putBodyRequest({ - ...options, - - path: path + '/' + videoId, - fields, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - async create (options: OverrideCommandOptions & { - fields: LiveVideoCreate - }) { - const { fields } = options - const path = '/api/v1/videos/live' - - const attaches: any = {} - if (fields.thumbnailfile) attaches.thumbnailfile = fields.thumbnailfile - if (fields.previewfile) attaches.previewfile = fields.previewfile - - const body = await unwrapBody<{ video: VideoCreateResult }>(this.postUploadRequest({ - ...options, - - path, - attaches, - fields: omit(fields, [ 'thumbnailfile', 'previewfile' ]), - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - })) - - return body.video - } - - async quickCreate (options: OverrideCommandOptions & { - saveReplay: boolean - permanentLive: boolean - privacy?: VideoPrivacy - videoPasswords?: string[] - }) { - const { saveReplay, permanentLive, privacy = VideoPrivacy.PUBLIC, videoPasswords } = options - - const replaySettings = privacy === VideoPrivacy.PASSWORD_PROTECTED - ? { privacy: VideoPrivacy.PRIVATE } - : { privacy } - - const { uuid } = await this.create({ - ...options, - - fields: { - name: 'live', - permanentLive, - saveReplay, - replaySettings, - channelId: this.server.store.channel.id, - privacy, - videoPasswords - } - }) - - const video = await this.server.videos.getWithToken({ id: uuid }) - const live = await this.get({ videoId: uuid }) - - return { video, live } - } - - // --------------------------------------------------------------------------- - - async sendRTMPStreamInVideo (options: OverrideCommandOptions & { - videoId: number | string - fixtureName?: string - copyCodecs?: boolean - }) { - const { videoId, fixtureName, copyCodecs } = options - const videoLive = await this.get({ videoId }) - - return sendRTMPStream({ rtmpBaseUrl: videoLive.rtmpUrl, streamKey: videoLive.streamKey, fixtureName, copyCodecs }) - } - - async runAndTestStreamError (options: OverrideCommandOptions & { - videoId: number | string - shouldHaveError: boolean - }) { - const command = await this.sendRTMPStreamInVideo(options) - - return testFfmpegStreamError(command, options.shouldHaveError) - } - - // --------------------------------------------------------------------------- - - waitUntilPublished (options: OverrideCommandOptions & { - videoId: number | string - }) { - const { videoId } = options - return this.waitUntilState({ videoId, state: VideoState.PUBLISHED }) - } - - waitUntilWaiting (options: OverrideCommandOptions & { - videoId: number | string - }) { - const { videoId } = options - return this.waitUntilState({ videoId, state: VideoState.WAITING_FOR_LIVE }) - } - - waitUntilEnded (options: OverrideCommandOptions & { - videoId: number | string - }) { - const { videoId } = options - return this.waitUntilState({ videoId, state: VideoState.LIVE_ENDED }) - } - - async waitUntilSegmentGeneration (options: OverrideCommandOptions & { - server: PeerTubeServer - videoUUID: string - playlistNumber: number - segment: number - objectStorage?: ObjectStorageCommand - objectStorageBaseUrl?: string - }) { - const { - server, - objectStorage, - playlistNumber, - segment, - videoUUID, - objectStorageBaseUrl - } = options - - const segmentName = `${playlistNumber}-00000${segment}.ts` - const baseUrl = objectStorage - ? join(objectStorageBaseUrl || objectStorage.getMockPlaylistBaseUrl(), 'hls') - : server.url + '/static/streaming-playlists/hls' - - let error = true - - while (error) { - try { - // Check fragment exists - await this.getRawRequest({ - ...options, - - url: `${baseUrl}/${videoUUID}/${segmentName}`, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - - const video = await server.videos.get({ id: videoUUID }) - const hlsPlaylist = video.streamingPlaylists[0] - - // Check SHA generation - const shaBody = await server.streamingPlaylists.getSegmentSha256({ url: hlsPlaylist.segmentsSha256Url, withRetry: !!objectStorage }) - if (!shaBody[segmentName]) { - throw new Error('Segment SHA does not exist') - } - - // Check fragment is in m3u8 playlist - const subPlaylist = await server.streamingPlaylists.get({ url: `${baseUrl}/${video.uuid}/${playlistNumber}.m3u8` }) - if (!subPlaylist.includes(segmentName)) throw new Error('Fragment does not exist in playlist') - - error = false - } catch { - error = true - await wait(100) - } - } - } - - async waitUntilReplacedByReplay (options: OverrideCommandOptions & { - videoId: number | string - }) { - let video: VideoDetails - - do { - video = await this.server.videos.getWithToken({ token: options.token, id: options.videoId }) - - await wait(500) - } while (video.isLive === true || video.state.id !== VideoState.PUBLISHED) - } - - // --------------------------------------------------------------------------- - - getSegmentFile (options: OverrideCommandOptions & { - videoUUID: string - playlistNumber: number - segment: number - objectStorage?: ObjectStorageCommand - }) { - const { playlistNumber, segment, videoUUID, objectStorage } = options - - const segmentName = `${playlistNumber}-00000${segment}.ts` - const baseUrl = objectStorage - ? objectStorage.getMockPlaylistBaseUrl() - : `${this.server.url}/static/streaming-playlists/hls` - - const url = `${baseUrl}/${videoUUID}/${segmentName}` - - return this.getRawRequest({ - ...options, - - url, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getPlaylistFile (options: OverrideCommandOptions & { - videoUUID: string - playlistName: string - objectStorage?: ObjectStorageCommand - }) { - const { playlistName, videoUUID, objectStorage } = options - - const baseUrl = objectStorage - ? objectStorage.getMockPlaylistBaseUrl() - : `${this.server.url}/static/streaming-playlists/hls` - - const url = `${baseUrl}/${videoUUID}/${playlistName}` - - return this.getRawRequest({ - ...options, - - url, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - // --------------------------------------------------------------------------- - - async countPlaylists (options: OverrideCommandOptions & { - videoUUID: string - }) { - const basePath = this.server.servers.buildDirectory('streaming-playlists') - const hlsPath = join(basePath, 'hls', options.videoUUID) - - const files = await readdir(hlsPath) - - return files.filter(f => f.endsWith('.m3u8')).length - } - - private async waitUntilState (options: OverrideCommandOptions & { - videoId: number | string - state: VideoState - }) { - let video: VideoDetails - - do { - video = await this.server.videos.getWithToken({ token: options.token, id: options.videoId }) - - await wait(500) - } while (video.state.id !== options.state) - } -} diff --git a/shared/server-commands/videos/live.ts b/shared/server-commands/videos/live.ts deleted file mode 100644 index cebadb1db..000000000 --- a/shared/server-commands/videos/live.ts +++ /dev/null @@ -1,128 +0,0 @@ -import ffmpeg, { FfmpegCommand } from 'fluent-ffmpeg' -import { truncate } from 'lodash' -import { buildAbsoluteFixturePath, wait } from '@shared/core-utils' -import { VideoDetails, VideoInclude, VideoPrivacy } from '@shared/models' -import { PeerTubeServer } from '../server/server' - -function sendRTMPStream (options: { - rtmpBaseUrl: string - streamKey: string - fixtureName?: string // default video_short.mp4 - copyCodecs?: boolean // default false -}) { - const { rtmpBaseUrl, streamKey, fixtureName = 'video_short.mp4', copyCodecs = false } = options - - const fixture = buildAbsoluteFixturePath(fixtureName) - - const command = ffmpeg(fixture) - command.inputOption('-stream_loop -1') - command.inputOption('-re') - - if (copyCodecs) { - command.outputOption('-c copy') - } else { - command.outputOption('-c:v libx264') - command.outputOption('-g 120') - command.outputOption('-x264-params "no-scenecut=1"') - command.outputOption('-r 60') - } - - command.outputOption('-f flv') - - const rtmpUrl = rtmpBaseUrl + '/' + streamKey - command.output(rtmpUrl) - - command.on('error', err => { - if (err?.message?.includes('Exiting normally')) return - - if (process.env.DEBUG) console.error(err) - }) - - if (process.env.DEBUG) { - command.on('stderr', data => console.log(data)) - command.on('stdout', data => console.log(data)) - } - - command.run() - - return command -} - -function waitFfmpegUntilError (command: FfmpegCommand, successAfterMS = 10000) { - return new Promise((res, rej) => { - command.on('error', err => { - return rej(err) - }) - - setTimeout(() => { - res() - }, successAfterMS) - }) -} - -async function testFfmpegStreamError (command: FfmpegCommand, shouldHaveError: boolean) { - let error: Error - - try { - await waitFfmpegUntilError(command, 45000) - } catch (err) { - error = err - } - - await stopFfmpeg(command) - - if (shouldHaveError && !error) throw new Error('Ffmpeg did not have an error') - if (!shouldHaveError && error) throw error -} - -async function stopFfmpeg (command: FfmpegCommand) { - command.kill('SIGINT') - - await wait(500) -} - -async function waitUntilLivePublishedOnAllServers (servers: PeerTubeServer[], videoId: string) { - for (const server of servers) { - await server.live.waitUntilPublished({ videoId }) - } -} - -async function waitUntilLiveWaitingOnAllServers (servers: PeerTubeServer[], videoId: string) { - for (const server of servers) { - await server.live.waitUntilWaiting({ videoId }) - } -} - -async function waitUntilLiveReplacedByReplayOnAllServers (servers: PeerTubeServer[], videoId: string) { - for (const server of servers) { - await server.live.waitUntilReplacedByReplay({ videoId }) - } -} - -async function findExternalSavedVideo (server: PeerTubeServer, liveDetails: VideoDetails) { - const include = VideoInclude.BLACKLISTED - const privacyOneOf = [ VideoPrivacy.INTERNAL, VideoPrivacy.PRIVATE, VideoPrivacy.PUBLIC, VideoPrivacy.UNLISTED ] - - const { data } = await server.videos.list({ token: server.accessToken, sort: '-publishedAt', include, privacyOneOf }) - - const videoNameSuffix = ` - ${new Date(liveDetails.publishedAt).toLocaleString()}` - const truncatedVideoName = truncate(liveDetails.name, { - length: 120 - videoNameSuffix.length - }) - const toFind = truncatedVideoName + videoNameSuffix - - return data.find(v => v.name === toFind) -} - -export { - sendRTMPStream, - waitFfmpegUntilError, - testFfmpegStreamError, - stopFfmpeg, - - waitUntilLivePublishedOnAllServers, - waitUntilLiveReplacedByReplayOnAllServers, - waitUntilLiveWaitingOnAllServers, - - findExternalSavedVideo -} diff --git a/shared/server-commands/videos/playlists-command.ts b/shared/server-commands/videos/playlists-command.ts deleted file mode 100644 index da3bef7b0..000000000 --- a/shared/server-commands/videos/playlists-command.ts +++ /dev/null @@ -1,281 +0,0 @@ -import { omit, pick } from '@shared/core-utils' -import { - BooleanBothQuery, - HttpStatusCode, - ResultList, - VideoExistInPlaylist, - VideoPlaylist, - VideoPlaylistCreate, - VideoPlaylistCreateResult, - VideoPlaylistElement, - VideoPlaylistElementCreate, - VideoPlaylistElementCreateResult, - VideoPlaylistElementUpdate, - VideoPlaylistReorder, - VideoPlaylistType, - VideoPlaylistUpdate -} from '@shared/models' -import { unwrapBody } from '../requests' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class PlaylistsCommand extends AbstractCommand { - - list (options: OverrideCommandOptions & { - start?: number - count?: number - sort?: string - playlistType?: VideoPlaylistType - }) { - const path = '/api/v1/video-playlists' - const query = pick(options, [ 'start', 'count', 'sort', 'playlistType' ]) - - return this.getRequestBody>({ - ...options, - - path, - query, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - listByChannel (options: OverrideCommandOptions & { - handle: string - start?: number - count?: number - sort?: string - playlistType?: VideoPlaylistType - }) { - const path = '/api/v1/video-channels/' + options.handle + '/video-playlists' - const query = pick(options, [ 'start', 'count', 'sort', 'playlistType' ]) - - return this.getRequestBody>({ - ...options, - - path, - query, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - listByAccount (options: OverrideCommandOptions & { - handle: string - start?: number - count?: number - sort?: string - search?: string - playlistType?: VideoPlaylistType - }) { - const path = '/api/v1/accounts/' + options.handle + '/video-playlists' - const query = pick(options, [ 'start', 'count', 'sort', 'search', 'playlistType' ]) - - return this.getRequestBody>({ - ...options, - - path, - query, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - get (options: OverrideCommandOptions & { - playlistId: number | string - }) { - const { playlistId } = options - const path = '/api/v1/video-playlists/' + playlistId - - return this.getRequestBody({ - ...options, - - path, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - listVideos (options: OverrideCommandOptions & { - playlistId: number | string - start?: number - count?: number - query?: { nsfw?: BooleanBothQuery } - }) { - const path = '/api/v1/video-playlists/' + options.playlistId + '/videos' - const query = options.query ?? {} - - return this.getRequestBody>({ - ...options, - - path, - query: { - ...query, - start: options.start, - count: options.count - }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - delete (options: OverrideCommandOptions & { - playlistId: number | string - }) { - const path = '/api/v1/video-playlists/' + options.playlistId - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - async create (options: OverrideCommandOptions & { - attributes: VideoPlaylistCreate - }) { - const path = '/api/v1/video-playlists' - - const fields = omit(options.attributes, [ 'thumbnailfile' ]) - - const attaches = options.attributes.thumbnailfile - ? { thumbnailfile: options.attributes.thumbnailfile } - : {} - - const body = await unwrapBody<{ videoPlaylist: VideoPlaylistCreateResult }>(this.postUploadRequest({ - ...options, - - path, - fields, - attaches, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - })) - - return body.videoPlaylist - } - - update (options: OverrideCommandOptions & { - attributes: VideoPlaylistUpdate - playlistId: number | string - }) { - const path = '/api/v1/video-playlists/' + options.playlistId - - const fields = omit(options.attributes, [ 'thumbnailfile' ]) - - const attaches = options.attributes.thumbnailfile - ? { thumbnailfile: options.attributes.thumbnailfile } - : {} - - return this.putUploadRequest({ - ...options, - - path, - fields, - attaches, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - async addElement (options: OverrideCommandOptions & { - playlistId: number | string - attributes: VideoPlaylistElementCreate | { videoId: string } - }) { - const attributes = { - ...options.attributes, - - videoId: await this.server.videos.getId({ ...options, uuid: options.attributes.videoId }) - } - - const path = '/api/v1/video-playlists/' + options.playlistId + '/videos' - - const body = await unwrapBody<{ videoPlaylistElement: VideoPlaylistElementCreateResult }>(this.postBodyRequest({ - ...options, - - path, - fields: attributes, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - })) - - return body.videoPlaylistElement - } - - updateElement (options: OverrideCommandOptions & { - playlistId: number | string - elementId: number | string - attributes: VideoPlaylistElementUpdate - }) { - const path = '/api/v1/video-playlists/' + options.playlistId + '/videos/' + options.elementId - - return this.putBodyRequest({ - ...options, - - path, - fields: options.attributes, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - removeElement (options: OverrideCommandOptions & { - playlistId: number | string - elementId: number - }) { - const path = '/api/v1/video-playlists/' + options.playlistId + '/videos/' + options.elementId - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - reorderElements (options: OverrideCommandOptions & { - playlistId: number | string - attributes: VideoPlaylistReorder - }) { - const path = '/api/v1/video-playlists/' + options.playlistId + '/videos/reorder' - - return this.postBodyRequest({ - ...options, - - path, - fields: options.attributes, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - getPrivacies (options: OverrideCommandOptions = {}) { - const path = '/api/v1/video-playlists/privacies' - - return this.getRequestBody<{ [ id: number ]: string }>({ - ...options, - - path, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - videosExist (options: OverrideCommandOptions & { - videoIds: number[] - }) { - const { videoIds } = options - const path = '/api/v1/users/me/video-playlists/videos-exist' - - return this.getRequestBody({ - ...options, - - path, - query: { videoIds }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } -} diff --git a/shared/server-commands/videos/services-command.ts b/shared/server-commands/videos/services-command.ts deleted file mode 100644 index 06760df42..000000000 --- a/shared/server-commands/videos/services-command.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { HttpStatusCode } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class ServicesCommand extends AbstractCommand { - - getOEmbed (options: OverrideCommandOptions & { - oembedUrl: string - format?: string - maxHeight?: number - maxWidth?: number - }) { - const path = '/services/oembed' - const query = { - url: options.oembedUrl, - format: options.format, - maxheight: options.maxHeight, - maxwidth: options.maxWidth - } - - return this.getRequest({ - ...options, - - path, - query, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } -} diff --git a/shared/server-commands/videos/storyboard-command.ts b/shared/server-commands/videos/storyboard-command.ts deleted file mode 100644 index 06d90fc12..000000000 --- a/shared/server-commands/videos/storyboard-command.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { HttpStatusCode, Storyboard } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class StoryboardCommand extends AbstractCommand { - - list (options: OverrideCommandOptions & { - id: number | string - }) { - const path = '/api/v1/videos/' + options.id + '/storyboards' - - return this.getRequestBody<{ storyboards: Storyboard[] }>({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } -} diff --git a/shared/server-commands/videos/streaming-playlists-command.ts b/shared/server-commands/videos/streaming-playlists-command.ts deleted file mode 100644 index b988ac4b2..000000000 --- a/shared/server-commands/videos/streaming-playlists-command.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { wait } from '@shared/core-utils' -import { HttpStatusCode } from '@shared/models' -import { unwrapBody, unwrapBodyOrDecodeToJSON, unwrapTextOrDecode } from '../requests' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class StreamingPlaylistsCommand extends AbstractCommand { - - async get (options: OverrideCommandOptions & { - url: string - - videoFileToken?: string - reinjectVideoFileToken?: boolean - - withRetry?: boolean // default false - currentRetry?: number - }): Promise { - const { videoFileToken, reinjectVideoFileToken, expectedStatus, withRetry = false, currentRetry = 1 } = options - - try { - const result = await unwrapTextOrDecode(this.getRawRequest({ - ...options, - - url: options.url, - query: { - videoFileToken, - reinjectVideoFileToken - }, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - })) - - // master.m3u8 could be empty - if (!result && (!expectedStatus || expectedStatus === HttpStatusCode.OK_200)) { - throw new Error('Empty result') - } - - return result - } catch (err) { - if (!withRetry || currentRetry > 10) throw err - - await wait(250) - - return this.get({ - ...options, - - withRetry, - currentRetry: currentRetry + 1 - }) - } - } - - async getFragmentedSegment (options: OverrideCommandOptions & { - url: string - range?: string - - withRetry?: boolean // default false - currentRetry?: number - }) { - const { withRetry = false, currentRetry = 1 } = options - - try { - const result = await unwrapBody(this.getRawRequest({ - ...options, - - url: options.url, - range: options.range, - implicitToken: false, - responseType: 'application/octet-stream', - defaultExpectedStatus: HttpStatusCode.OK_200 - })) - - return result - } catch (err) { - if (!withRetry || currentRetry > 10) throw err - - await wait(250) - - return this.getFragmentedSegment({ - ...options, - - withRetry, - currentRetry: currentRetry + 1 - }) - } - } - - async getSegmentSha256 (options: OverrideCommandOptions & { - url: string - - withRetry?: boolean // default false - currentRetry?: number - }) { - const { withRetry = false, currentRetry = 1 } = options - - try { - const result = await unwrapBodyOrDecodeToJSON<{ [ id: string ]: string }>(this.getRawRequest({ - ...options, - - url: options.url, - contentType: 'application/json', - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - })) - - return result - } catch (err) { - if (!withRetry || currentRetry > 10) throw err - - await wait(250) - - return this.getSegmentSha256({ - ...options, - - withRetry, - currentRetry: currentRetry + 1 - }) - } - } -} diff --git a/shared/server-commands/videos/video-passwords-command.ts b/shared/server-commands/videos/video-passwords-command.ts deleted file mode 100644 index bf10335b4..000000000 --- a/shared/server-commands/videos/video-passwords-command.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { HttpStatusCode, ResultList, VideoPassword } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' -export class VideoPasswordsCommand extends AbstractCommand { - - list (options: OverrideCommandOptions & { - videoId: number | string - start?: number - count?: number - sort?: string - }) { - const { start, count, sort, videoId } = options - const path = '/api/v1/videos/' + videoId + '/passwords' - - return this.getRequestBody>({ - ...options, - - path, - query: { start, count, sort }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - updateAll (options: OverrideCommandOptions & { - videoId: number | string - passwords: string[] - }) { - const { videoId, passwords } = options - const path = `/api/v1/videos/${videoId}/passwords` - - return this.putBodyRequest({ - ...options, - path, - fields: { passwords }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - remove (options: OverrideCommandOptions & { - id: number - videoId: number | string - }) { - const { id, videoId } = options - const path = `/api/v1/videos/${videoId}/passwords/${id}` - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } -} diff --git a/shared/server-commands/videos/video-stats-command.ts b/shared/server-commands/videos/video-stats-command.ts deleted file mode 100644 index b9b99bfb5..000000000 --- a/shared/server-commands/videos/video-stats-command.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { pick } from '@shared/core-utils' -import { HttpStatusCode, VideoStatsOverall, VideoStatsRetention, VideoStatsTimeserie, VideoStatsTimeserieMetric } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class VideoStatsCommand extends AbstractCommand { - - getOverallStats (options: OverrideCommandOptions & { - videoId: number | string - startDate?: string - endDate?: string - }) { - const path = '/api/v1/videos/' + options.videoId + '/stats/overall' - - return this.getRequestBody({ - ...options, - path, - - query: pick(options, [ 'startDate', 'endDate' ]), - - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getTimeserieStats (options: OverrideCommandOptions & { - videoId: number | string - metric: VideoStatsTimeserieMetric - startDate?: Date - endDate?: Date - }) { - const path = '/api/v1/videos/' + options.videoId + '/stats/timeseries/' + options.metric - - return this.getRequestBody({ - ...options, - path, - - query: pick(options, [ 'startDate', 'endDate' ]), - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getRetentionStats (options: OverrideCommandOptions & { - videoId: number | string - }) { - const path = '/api/v1/videos/' + options.videoId + '/stats/retention' - - return this.getRequestBody({ - ...options, - path, - - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } -} diff --git a/shared/server-commands/videos/video-studio-command.ts b/shared/server-commands/videos/video-studio-command.ts deleted file mode 100644 index 675cd84b7..000000000 --- a/shared/server-commands/videos/video-studio-command.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { HttpStatusCode, VideoStudioTask } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class VideoStudioCommand extends AbstractCommand { - - static getComplexTask (): VideoStudioTask[] { - return [ - // Total duration: 2 - { - name: 'cut', - options: { - start: 1, - end: 3 - } - }, - - // Total duration: 7 - { - name: 'add-outro', - options: { - file: 'video_short.webm' - } - }, - - { - name: 'add-watermark', - options: { - file: 'custom-thumbnail.png' - } - }, - - // Total duration: 9 - { - name: 'add-intro', - options: { - file: 'video_very_short_240p.mp4' - } - } - ] - } - - createEditionTasks (options: OverrideCommandOptions & { - videoId: number | string - tasks: VideoStudioTask[] - }) { - const path = '/api/v1/videos/' + options.videoId + '/studio/edit' - const attaches: { [id: string]: any } = {} - - for (let i = 0; i < options.tasks.length; i++) { - const task = options.tasks[i] - - if (task.name === 'add-intro' || task.name === 'add-outro' || task.name === 'add-watermark') { - attaches[`tasks[${i}][options][file]`] = task.options.file - } - } - - return this.postUploadRequest({ - ...options, - - path, - attaches, - fields: { tasks: options.tasks }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } -} diff --git a/shared/server-commands/videos/video-token-command.ts b/shared/server-commands/videos/video-token-command.ts deleted file mode 100644 index c4ed29a8c..000000000 --- a/shared/server-commands/videos/video-token-command.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */ - -import { HttpStatusCode, VideoToken } from '@shared/models' -import { unwrapBody } from '../requests' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class VideoTokenCommand extends AbstractCommand { - - create (options: OverrideCommandOptions & { - videoId: number | string - videoPassword?: string - }) { - const { videoId, videoPassword } = options - const path = '/api/v1/videos/' + videoId + '/token' - - return unwrapBody(this.postBodyRequest({ - ...options, - headers: this.buildVideoPasswordHeader(videoPassword), - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - })) - } - - async getVideoFileToken (options: OverrideCommandOptions & { - videoId: number | string - videoPassword?: string - }) { - const { files } = await this.create(options) - - return files.token - } -} diff --git a/shared/server-commands/videos/videos-command.ts b/shared/server-commands/videos/videos-command.ts deleted file mode 100644 index 4c3513ed4..000000000 --- a/shared/server-commands/videos/videos-command.ts +++ /dev/null @@ -1,829 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */ - -import { expect } from 'chai' -import { createReadStream, stat } from 'fs-extra' -import got, { Response as GotResponse } from 'got' -import validator from 'validator' -import { buildAbsoluteFixturePath, getAllPrivacies, omit, pick, wait } from '@shared/core-utils' -import { buildUUID } from '@shared/extra-utils' -import { - HttpStatusCode, - ResultList, - UserVideoRateType, - Video, - VideoCreate, - VideoCreateResult, - VideoDetails, - VideoFileMetadata, - VideoInclude, - VideoPrivacy, - VideosCommonQuery, - VideoTranscodingCreate -} from '@shared/models' -import { VideoSource } from '@shared/models/videos/video-source' -import { unwrapBody } from '../requests' -import { waitJobs } from '../server' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export type VideoEdit = Partial> & { - fixture?: string - thumbnailfile?: string - previewfile?: string -} - -export class VideosCommand extends AbstractCommand { - - getCategories (options: OverrideCommandOptions = {}) { - const path = '/api/v1/videos/categories' - - return this.getRequestBody<{ [id: number]: string }>({ - ...options, - path, - - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getLicences (options: OverrideCommandOptions = {}) { - const path = '/api/v1/videos/licences' - - return this.getRequestBody<{ [id: number]: string }>({ - ...options, - path, - - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getLanguages (options: OverrideCommandOptions = {}) { - const path = '/api/v1/videos/languages' - - return this.getRequestBody<{ [id: string]: string }>({ - ...options, - path, - - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getPrivacies (options: OverrideCommandOptions = {}) { - const path = '/api/v1/videos/privacies' - - return this.getRequestBody<{ [id in VideoPrivacy]: string }>({ - ...options, - path, - - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - // --------------------------------------------------------------------------- - - getDescription (options: OverrideCommandOptions & { - descriptionPath: string - }) { - return this.getRequestBody<{ description: string }>({ - ...options, - path: options.descriptionPath, - - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getFileMetadata (options: OverrideCommandOptions & { - url: string - }) { - return unwrapBody(this.getRawRequest({ - ...options, - - url: options.url, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - })) - } - - // --------------------------------------------------------------------------- - - rate (options: OverrideCommandOptions & { - id: number | string - rating: UserVideoRateType - videoPassword?: string - }) { - const { id, rating, videoPassword } = options - const path = '/api/v1/videos/' + id + '/rate' - - return this.putBodyRequest({ - ...options, - - path, - fields: { rating }, - headers: this.buildVideoPasswordHeader(videoPassword), - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - // --------------------------------------------------------------------------- - - get (options: OverrideCommandOptions & { - id: number | string - }) { - const path = '/api/v1/videos/' + options.id - - return this.getRequestBody({ - ...options, - - path, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getWithToken (options: OverrideCommandOptions & { - id: number | string - }) { - return this.get({ - ...options, - - token: this.buildCommonRequestToken({ ...options, implicitToken: true }) - }) - } - - getWithPassword (options: OverrideCommandOptions & { - id: number | string - password?: string - }) { - const path = '/api/v1/videos/' + options.id - - return this.getRequestBody({ - ...options, - headers:{ - 'x-peertube-video-password': options.password - }, - path, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getSource (options: OverrideCommandOptions & { - id: number | string - }) { - const path = '/api/v1/videos/' + options.id + '/source' - - return this.getRequestBody({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - async getId (options: OverrideCommandOptions & { - uuid: number | string - }) { - const { uuid } = options - - if (validator.isUUID('' + uuid) === false) return uuid as number - - const { id } = await this.get({ ...options, id: uuid }) - - return id - } - - async listFiles (options: OverrideCommandOptions & { - id: number | string - }) { - const video = await this.get(options) - - const files = video.files || [] - const hlsFiles = video.streamingPlaylists[0]?.files || [] - - return files.concat(hlsFiles) - } - - // --------------------------------------------------------------------------- - - listMyVideos (options: OverrideCommandOptions & { - start?: number - count?: number - sort?: string - search?: string - isLive?: boolean - channelId?: number - } = {}) { - const path = '/api/v1/users/me/videos' - - return this.getRequestBody>({ - ...options, - - path, - query: pick(options, [ 'start', 'count', 'sort', 'search', 'isLive', 'channelId' ]), - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - listMySubscriptionVideos (options: OverrideCommandOptions & VideosCommonQuery = {}) { - const { sort = '-createdAt' } = options - const path = '/api/v1/users/me/subscriptions/videos' - - return this.getRequestBody>({ - ...options, - - path, - query: { sort, ...this.buildListQuery(options) }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - // --------------------------------------------------------------------------- - - list (options: OverrideCommandOptions & VideosCommonQuery = {}) { - const path = '/api/v1/videos' - - const query = this.buildListQuery(options) - - return this.getRequestBody>({ - ...options, - - path, - query: { sort: 'name', ...query }, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - listWithToken (options: OverrideCommandOptions & VideosCommonQuery = {}) { - return this.list({ - ...options, - - token: this.buildCommonRequestToken({ ...options, implicitToken: true }) - }) - } - - listAllForAdmin (options: OverrideCommandOptions & VideosCommonQuery = {}) { - const include = VideoInclude.NOT_PUBLISHED_STATE | VideoInclude.BLACKLISTED | VideoInclude.BLOCKED_OWNER - const nsfw = 'both' - const privacyOneOf = getAllPrivacies() - - return this.list({ - ...options, - - include, - nsfw, - privacyOneOf, - - token: this.buildCommonRequestToken({ ...options, implicitToken: true }) - }) - } - - listByAccount (options: OverrideCommandOptions & VideosCommonQuery & { - handle: string - }) { - const { handle, search } = options - const path = '/api/v1/accounts/' + handle + '/videos' - - return this.getRequestBody>({ - ...options, - - path, - query: { search, ...this.buildListQuery(options) }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - listByChannel (options: OverrideCommandOptions & VideosCommonQuery & { - handle: string - }) { - const { handle } = options - const path = '/api/v1/video-channels/' + handle + '/videos' - - return this.getRequestBody>({ - ...options, - - path, - query: this.buildListQuery(options), - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - // --------------------------------------------------------------------------- - - async find (options: OverrideCommandOptions & { - name: string - }) { - const { data } = await this.list(options) - - return data.find(v => v.name === options.name) - } - - // --------------------------------------------------------------------------- - - update (options: OverrideCommandOptions & { - id: number | string - attributes?: VideoEdit - }) { - const { id, attributes = {} } = options - const path = '/api/v1/videos/' + id - - // Upload request - if (attributes.thumbnailfile || attributes.previewfile) { - const attaches: any = {} - if (attributes.thumbnailfile) attaches.thumbnailfile = attributes.thumbnailfile - if (attributes.previewfile) attaches.previewfile = attributes.previewfile - - return this.putUploadRequest({ - ...options, - - path, - fields: options.attributes, - attaches: { - thumbnailfile: attributes.thumbnailfile, - previewfile: attributes.previewfile - }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - return this.putBodyRequest({ - ...options, - - path, - fields: options.attributes, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - remove (options: OverrideCommandOptions & { - id: number | string - }) { - const path = '/api/v1/videos/' + options.id - - return unwrapBody(this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - })) - } - - async removeAll () { - const { data } = await this.list() - - for (const v of data) { - await this.remove({ id: v.id }) - } - } - - // --------------------------------------------------------------------------- - - async upload (options: OverrideCommandOptions & { - attributes?: VideoEdit - mode?: 'legacy' | 'resumable' // default legacy - waitTorrentGeneration?: boolean // default true - completedExpectedStatus?: HttpStatusCode - } = {}) { - const { mode = 'legacy', waitTorrentGeneration = true } = options - let defaultChannelId = 1 - - try { - const { videoChannels } = await this.server.users.getMyInfo({ token: options.token }) - defaultChannelId = videoChannels[0].id - } catch (e) { /* empty */ } - - // Override default attributes - const attributes = { - name: 'my super video', - category: 5, - licence: 4, - language: 'zh', - channelId: defaultChannelId, - nsfw: true, - waitTranscoding: false, - description: 'my super description', - support: 'my super support text', - tags: [ 'tag' ], - privacy: VideoPrivacy.PUBLIC, - commentsEnabled: true, - downloadEnabled: true, - fixture: 'video_short.webm', - - ...options.attributes - } - - const created = mode === 'legacy' - ? await this.buildLegacyUpload({ ...options, attributes }) - : await this.buildResumeUpload({ ...options, path: '/api/v1/videos/upload-resumable', attributes }) - - // Wait torrent generation - const expectedStatus = this.buildExpectedStatus({ ...options, defaultExpectedStatus: HttpStatusCode.OK_200 }) - if (expectedStatus === HttpStatusCode.OK_200 && waitTorrentGeneration) { - let video: VideoDetails - - do { - video = await this.getWithToken({ ...options, id: created.uuid }) - - await wait(50) - } while (!video.files[0].torrentUrl) - } - - return created - } - - async buildLegacyUpload (options: OverrideCommandOptions & { - attributes: VideoEdit - }): Promise { - const path = '/api/v1/videos/upload' - - return unwrapBody<{ video: VideoCreateResult }>(this.postUploadRequest({ - ...options, - - path, - fields: this.buildUploadFields(options.attributes), - attaches: this.buildUploadAttaches(options.attributes), - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - })).then(body => body.video || body as any) - } - - async buildResumeUpload (options: OverrideCommandOptions & { - path: string - attributes: { fixture?: string } & { [id: string]: any } - completedExpectedStatus?: HttpStatusCode // When the upload is finished - }): Promise { - const { path, attributes, expectedStatus = HttpStatusCode.OK_200, completedExpectedStatus } = options - - let size = 0 - let videoFilePath: string - let mimetype = 'video/mp4' - - if (attributes.fixture) { - videoFilePath = buildAbsoluteFixturePath(attributes.fixture) - size = (await stat(videoFilePath)).size - - if (videoFilePath.endsWith('.mkv')) { - mimetype = 'video/x-matroska' - } else if (videoFilePath.endsWith('.webm')) { - mimetype = 'video/webm' - } - } - - // Do not check status automatically, we'll check it manually - const initializeSessionRes = await this.prepareResumableUpload({ - ...options, - - path, - expectedStatus: null, - attributes, - size, - mimetype - }) - const initStatus = initializeSessionRes.status - - if (videoFilePath && initStatus === HttpStatusCode.CREATED_201) { - const locationHeader = initializeSessionRes.header['location'] - expect(locationHeader).to.not.be.undefined - - const pathUploadId = locationHeader.split('?')[1] - - const result = await this.sendResumableChunks({ - ...options, - - path, - pathUploadId, - videoFilePath, - size, - expectedStatus: completedExpectedStatus - }) - - if (result.statusCode === HttpStatusCode.OK_200) { - await this.endResumableUpload({ - ...options, - - expectedStatus: HttpStatusCode.NO_CONTENT_204, - path, - pathUploadId - }) - } - - return result.body?.video || result.body as any - } - - const expectedInitStatus = expectedStatus === HttpStatusCode.OK_200 - ? HttpStatusCode.CREATED_201 - : expectedStatus - - expect(initStatus).to.equal(expectedInitStatus) - - return initializeSessionRes.body.video || initializeSessionRes.body - } - - async prepareResumableUpload (options: OverrideCommandOptions & { - path: string - attributes: { fixture?: string } & { [id: string]: any } - size: number - mimetype: string - - originalName?: string - lastModified?: number - }) { - const { path, attributes, originalName, lastModified, size, mimetype } = options - - const attaches = this.buildUploadAttaches(omit(options.attributes, [ 'fixture' ])) - - const uploadOptions = { - ...options, - - path, - headers: { - 'X-Upload-Content-Type': mimetype, - 'X-Upload-Content-Length': size.toString() - }, - fields: { - filename: attributes.fixture, - originalName, - lastModified, - - ...this.buildUploadFields(options.attributes) - }, - - // Fixture will be sent later - attaches: this.buildUploadAttaches(omit(options.attributes, [ 'fixture' ])), - implicitToken: true, - - defaultExpectedStatus: null - } - - if (Object.keys(attaches).length === 0) return this.postBodyRequest(uploadOptions) - - return this.postUploadRequest(uploadOptions) - } - - sendResumableChunks (options: OverrideCommandOptions & { - pathUploadId: string - path: string - videoFilePath: string - size: number - contentLength?: number - contentRangeBuilder?: (start: number, chunk: any) => string - digestBuilder?: (chunk: any) => string - }) { - const { - path, - pathUploadId, - videoFilePath, - size, - contentLength, - contentRangeBuilder, - digestBuilder, - expectedStatus = HttpStatusCode.OK_200 - } = options - - let start = 0 - - const token = this.buildCommonRequestToken({ ...options, implicitToken: true }) - const url = this.server.url - - const readable = createReadStream(videoFilePath, { highWaterMark: 8 * 1024 }) - return new Promise>((resolve, reject) => { - readable.on('data', async function onData (chunk) { - try { - readable.pause() - - const byterangeStart = start + chunk.length - 1 - - const headers = { - 'Authorization': 'Bearer ' + token, - 'Content-Type': 'application/octet-stream', - 'Content-Range': contentRangeBuilder - ? contentRangeBuilder(start, chunk) - : `bytes ${start}-${byterangeStart}/${size}`, - 'Content-Length': contentLength ? contentLength + '' : chunk.length + '' - } - - if (digestBuilder) { - Object.assign(headers, { digest: digestBuilder(chunk) }) - } - - const res = await got<{ video: VideoCreateResult }>({ - url, - method: 'put', - headers, - path: path + '?' + pathUploadId, - body: chunk, - responseType: 'json', - throwHttpErrors: false - }) - - start += chunk.length - - // Last request, check final status - if (byterangeStart + 1 === size) { - if (res.statusCode === expectedStatus) { - return resolve(res) - } - - if (res.statusCode !== HttpStatusCode.PERMANENT_REDIRECT_308) { - readable.off('data', onData) - - // eslint-disable-next-line max-len - const message = `Incorrect transient behaviour sending intermediary chunks. Status code is ${res.statusCode} instead of ${expectedStatus}` - return reject(new Error(message)) - } - } - - readable.resume() - } catch (err) { - reject(err) - } - }) - }) - } - - endResumableUpload (options: OverrideCommandOptions & { - path: string - pathUploadId: string - }) { - return this.deleteRequest({ - ...options, - - path: options.path, - rawQuery: options.pathUploadId, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - quickUpload (options: OverrideCommandOptions & { - name: string - nsfw?: boolean - privacy?: VideoPrivacy - fixture?: string - videoPasswords?: string[] - }) { - const attributes: VideoEdit = { name: options.name } - if (options.nsfw) attributes.nsfw = options.nsfw - if (options.privacy) attributes.privacy = options.privacy - if (options.fixture) attributes.fixture = options.fixture - if (options.videoPasswords) attributes.videoPasswords = options.videoPasswords - - return this.upload({ ...options, attributes }) - } - - async randomUpload (options: OverrideCommandOptions & { - wait?: boolean // default true - additionalParams?: VideoEdit & { prefixName?: string } - } = {}) { - const { wait = true, additionalParams } = options - const prefixName = additionalParams?.prefixName || '' - const name = prefixName + buildUUID() - - const attributes = { name, ...additionalParams } - - const result = await this.upload({ ...options, attributes }) - - if (wait) await waitJobs([ this.server ]) - - return { ...result, name } - } - - // --------------------------------------------------------------------------- - - replaceSourceFile (options: OverrideCommandOptions & { - videoId: number | string - fixture: string - completedExpectedStatus?: HttpStatusCode - }) { - return this.buildResumeUpload({ - ...options, - - path: '/api/v1/videos/' + options.videoId + '/source/replace-resumable', - attributes: { fixture: options.fixture } - }) - } - - // --------------------------------------------------------------------------- - - removeHLSPlaylist (options: OverrideCommandOptions & { - videoId: number | string - }) { - const path = '/api/v1/videos/' + options.videoId + '/hls' - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - removeHLSFile (options: OverrideCommandOptions & { - videoId: number | string - fileId: number - }) { - const path = '/api/v1/videos/' + options.videoId + '/hls/' + options.fileId - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - removeAllWebVideoFiles (options: OverrideCommandOptions & { - videoId: number | string - }) { - const path = '/api/v1/videos/' + options.videoId + '/web-videos' - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - removeWebVideoFile (options: OverrideCommandOptions & { - videoId: number | string - fileId: number - }) { - const path = '/api/v1/videos/' + options.videoId + '/web-videos/' + options.fileId - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - runTranscoding (options: OverrideCommandOptions & VideoTranscodingCreate & { - videoId: number | string - }) { - const path = '/api/v1/videos/' + options.videoId + '/transcoding' - - return this.postBodyRequest({ - ...options, - - path, - fields: pick(options, [ 'transcodingType', 'forceTranscoding' ]), - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - // --------------------------------------------------------------------------- - - private buildListQuery (options: VideosCommonQuery) { - return pick(options, [ - 'start', - 'count', - 'sort', - 'nsfw', - 'isLive', - 'categoryOneOf', - 'licenceOneOf', - 'languageOneOf', - 'privacyOneOf', - 'tagsOneOf', - 'tagsAllOf', - 'isLocal', - 'include', - 'skipCount' - ]) - } - - private buildUploadFields (attributes: VideoEdit) { - return omit(attributes, [ 'fixture', 'thumbnailfile', 'previewfile' ]) - } - - private buildUploadAttaches (attributes: VideoEdit) { - const attaches: { [ name: string ]: string } = {} - - for (const key of [ 'thumbnailfile', 'previewfile' ]) { - if (attributes[key]) attaches[key] = buildAbsoluteFixturePath(attributes[key]) - } - - if (attributes.fixture) attaches.videofile = buildAbsoluteFixturePath(attributes.fixture) - - return attaches - } -} diff --git a/shared/server-commands/videos/views-command.ts b/shared/server-commands/videos/views-command.ts deleted file mode 100644 index bdb8daaa4..000000000 --- a/shared/server-commands/videos/views-command.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */ -import { HttpStatusCode, VideoViewEvent } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class ViewsCommand extends AbstractCommand { - - view (options: OverrideCommandOptions & { - id: number | string - currentTime: number - viewEvent?: VideoViewEvent - xForwardedFor?: string - }) { - const { id, xForwardedFor, viewEvent, currentTime } = options - const path = '/api/v1/videos/' + id + '/views' - - return this.postBodyRequest({ - ...options, - - path, - xForwardedFor, - fields: { - currentTime, - viewEvent - }, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - async simulateView (options: OverrideCommandOptions & { - id: number | string - xForwardedFor?: string - }) { - await this.view({ ...options, currentTime: 0 }) - await this.view({ ...options, currentTime: 5 }) - } - - async simulateViewer (options: OverrideCommandOptions & { - id: number | string - currentTimes: number[] - xForwardedFor?: string - }) { - let viewEvent: VideoViewEvent = 'seek' - - for (const currentTime of options.currentTimes) { - await this.view({ ...options, currentTime, viewEvent }) - - viewEvent = undefined - } - } -} diff --git a/shared/tsconfig.json b/shared/tsconfig.json deleted file mode 100644 index 95892077b..000000000 --- a/shared/tsconfig.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "extends": "../tsconfig.base.json", - "compilerOptions": { - "outDir": "../dist/shared" - } -} diff --git a/shared/tsconfig.types.json b/shared/tsconfig.types.json deleted file mode 100644 index 6acfc05e1..000000000 --- a/shared/tsconfig.types.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "extends": "./tsconfig.json", - "compilerOptions": { - "outDir": "../packages/types/dist/shared", - "stripInternal": true, - "removeComments": false, - "emitDeclarationOnly": true - }, - "exclude": [ - "server-commands/" - ] -} diff --git a/shared/typescript-utils/index.ts b/shared/typescript-utils/index.ts deleted file mode 100644 index c9f6f047d..000000000 --- a/shared/typescript-utils/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './types' diff --git a/support/doc/api/embeds.md b/support/doc/api/embeds.md index fd5507e38..989c8e98e 100644 --- a/support/doc/api/embeds.md +++ b/support/doc/api/embeds.md @@ -20,7 +20,7 @@ yarn add @peertube/embed-api Now just use the `PeerTubePlayer` class exported by the module: ```typescript -import { PeerTubePlayer } from '@peertube/embed-api' +import { PeerTubePlayer } from '@peertube/embed-api.js' ... ``` diff --git a/support/doc/api/openapi.yaml b/support/doc/api/openapi.yaml index 0cbc58678..5d54a7a51 100644 --- a/support/doc/api/openapi.yaml +++ b/support/doc/api/openapi.yaml @@ -53,7 +53,7 @@ info: } ``` - We provide error `type` values for [a growing number of cases](https://github.com/Chocobozzz/PeerTube/blob/develop/shared/models/server/server-error-code.enum.ts), + We provide error `type` values for [a growing number of cases](https://github.com/Chocobozzz/PeerTube/blob/develop/packages/models/server/server-error-code.enum.ts), but it is still optional. Types are used to disambiguate errors that bear the same status code and are non-obvious: @@ -8752,7 +8752,7 @@ components: password: $ref: '#/components/schemas/password' UpdateMe: - # see shared/models/users/user-update-me.model.ts: + # see packages/models/users/user-update-me.model.ts: properties: password: $ref: '#/components/schemas/password' diff --git a/support/doc/development/lib.md b/support/doc/development/lib.md index 25fe3068e..1ea09f2bc 100644 --- a/support/doc/development/lib.md +++ b/support/doc/development/lib.md @@ -18,7 +18,7 @@ The complete types package is generated via: ``` npm run generate-types-package 4.x.x -cd packages/types/dist +cd packages/types-generator/dist npm publish --access=public ``` diff --git a/support/doc/development/localization.md b/support/doc/development/localization.md index a38ed6f55..4aca9b18b 100644 --- a/support/doc/development/localization.md +++ b/support/doc/development/localization.md @@ -26,7 +26,7 @@ Nothing to do here, Github will automatically send a webhook to Weblate that wil ## Support a new language - * Add it to [/shared/models/i18n/i18n.ts](/shared/models/i18n/i18n.ts) + * Add it to [/packages/models/i18n/i18n.ts](/packages/models/i18n/i18n.ts) * Add it to [/scripts/build/client.sh](/scripts/build/client.sh) * Add it to [/client/angular.json](/client/angular.json) * Add it to [/scripts/i18n/update.sh](/scripts/i18n/update.sh) diff --git a/support/doc/development/monitoring.md b/support/doc/development/monitoring.md index 93fd1403e..023be9e92 100644 --- a/support/doc/development/monitoring.md +++ b/support/doc/development/monitoring.md @@ -13,11 +13,11 @@ npm run build -- --analyze-bundle && npm run client-report To benchmark the REST API and save result in `benchmark.json`: ``` -node dist/scripts/benchmark.js -o benchmark.json +npm run benchmark-server -- -o benchmark.json ``` You can also grep on a specific test: ``` -node dist/scripts/benchmark.js --grep homepage +npm run benchmark-server -- --grep homepage ``` diff --git a/support/doc/development/server.md b/support/doc/development/server.md index 7a9fa571f..5c83af704 100644 --- a/support/doc/development/server.md +++ b/support/doc/development/server.md @@ -1,11 +1,11 @@ -# Server code + # Server code ## Database model typing Sequelize models contain optional fields corresponding to table joins. For example, `VideoModel` has a `VideoChannel?: VideoChannelModel` field. It can be filled if the SQL query joined with the `videoChannel` table or empty if not. It can be difficult in TypeScript to understand if a function argument expects associations to be filled or not. -To improve clarity and reduce bugs, PeerTube defines multiple versions of a database model depending on its associations in `server/types/models/`. +To improve clarity and reduce bugs, PeerTube defines multiple versions of a database model depending on its associations in `server/server/types/models/`. These models start with `M` and by default do not include any association. `MVideo` for example corresponds to `VideoModel` without any association, where `VideoChannel` attribute doesn't exist. On the other hand, `MVideoWithChannel` is a `MVideo` that has a `VideoChannel` field. This way, a function that accepts `video: MVideoWithChannel` argument expects a video with channel populated. Main PeerTube code should never use `...Model` (`VideoModel`) database type, but always `M...` instead (`MVideo`, `MVideoChannel` etc). ## Add a new feature walkthrough @@ -16,67 +16,67 @@ Some of these may be optional (for example your new endpoint may not need to sen * Configuration: - Add you new configuration key in `config/default.yaml` and `config/production.yaml` - If you configuration needs to be different in dev or tests environments, also update `config/dev.yaml` and `config/test.yaml` - - Load your configuration in `server/initializers/config.ts` - - Check new configuration keys are set in `server/initializers/checker-before-init.ts` - - You can also ensure configuration consistency in `server/initializers/checker-after-init.ts` + - Load your configuration in `server/server/initializers/config.ts` + - Check new configuration keys are set in `server/server/initializers/checker-before-init.ts` + - You can also ensure configuration consistency in `server/server/initializers/checker-after-init.ts` - If you want your configuration to be available in the client: - + Add your field in `shared/models/server/server-config.model.ts` - + Update `server/lib/server-config-manager.ts` to include your new configuration + + Add your field in `packages/models/server/server/server-config.model.ts` + + Update `server/server/lib/server-config-manager.ts` to include your new configuration - If you want your configuration to be updatable by the web admin in the client: - + Add your field in `shared/models/server/custom-config.model.ts` - + Add the configuration to the config object in the `server/controllers/api/config.ts` controller + + Add your field in `packages/models/server/server/custom-config.model.ts` + + Add the configuration to the config object in the `server/server/controllers/api/config.ts` controller * Controllers: - Create the controller file and fill it with your REST API routes - Import and use your controller in the parent controller * Middlewares: - - Create your validator middleware in `server/middlewares/validators` that will be used by your controllers - - Add your new middleware file `server/middlewares/validators/index.ts` so it's easier to import - - Create the entry in `server/types/express.d.ts` to attach the database model loaded by your middleware to the express response + - Create your validator middleware in `server/server/middlewares/validators` that will be used by your controllers + - Add your new middleware file `server/server/middlewares/validators/index.ts` so it's easier to import + - Create the entry in `server/server/types/express.d.ts` to attach the database model loaded by your middleware to the express response * Validators: - - Create your validators that will be used by your middlewares in `server/helpers/custom-validators` + - Create your validators that will be used by your middlewares in `server/server/helpers/custom-validators` * Typescript models: - - Create the API models (request parameters or response) in `shared/models` + - Create the API models (request parameters or response) in `packages/models` - Add your models in `index.ts` of current directory to facilitate the imports * Sequelize model (BDD): - If you need to create a new table: - + Create the Sequelize model in `server/models/`: + + Create the Sequelize model in `server/server/models/`: * Create the `@Column` * Add some indexes if you need * Create static methods to load a specific from the database `loadBy...` * Create static methods to load a list of models from the database `listBy...` * Create the instance method `toFormattedJSON` that creates the JSON to send to the REST API from the model - + Add your new Sequelize model to `server/initializers/database.ts` - + Create a new file in `server/types` to define multiple versions of your Sequelize model depending on database associations - + Add this new file to `server/types/*/index.ts` to facilitate the imports + + Add your new Sequelize model to `server/server/initializers/database.ts` + + Create a new file in `server/server/types` to define multiple versions of your Sequelize model depending on database associations + + Add this new file to `server/server/types/*/index.ts` to facilitate the imports + Create database migrations: - * Create the migration file in `server/initializers/migrations` using raw SQL (copy the same SQL query as at PeerTube startup) - * Update `LAST_MIGRATION_VERSION` in `server/initializers/constants.ts` + * Create the migration file in `server/server/initializers/migrations` using raw SQL (copy the same SQL query as at PeerTube startup) + * Update `LAST_MIGRATION_VERSION` in `server/server/initializers/constants.ts` - If updating database schema (adding/removing/renaming a column): - + Update the sequelize models in `server/models/` + + Update the sequelize models in `server/server/models/` + Add migrations: * Create the migration file in `initializers/migrations` using Sequelize Query Interface (`.addColumn`, `.dropTable`, `.changeColumn`) - * Update `LAST_MIGRATION_VERSION` in `server/initializers/constants.ts` + * Update `LAST_MIGRATION_VERSION` in `server/server/initializers/constants.ts` * Notifications: - - Create the new notification model in `shared/models/users/user-notification.model.ts` - - Create the notification logic in `server/lib/notifier/shared`: + - Create the new notification model in `packages/models/users/user-notification.model.ts` + - Create the notification logic in `server/server/lib/notifier/shared`: + Email subject has a common prefix (defined by the admin in PeerTube configuration) - - Add your notification to `server/lib/notifier/notifier.ts` - - Create the email template in `server/lib/emails`: + - Add your notification to `server/server/lib/notifier/notifier.ts` + - Create the email template in `server/server/lib/emails`: + A text version is automatically generated from the HTML + The template usually extends `../common/grettings` that already says "Hi" and "Cheers". You just have to write the title and the content blocks that will be inserted in the appropriate places in the HTML template - If you need to associate a new table with `userNotification`: + Associate the new table in `UserNotificationModel` (don't forget the index) - + Add the object property in the API model definition (`shared/models/users/user-notification.model.ts`) + + Add the object property in the API model definition (`packages/models/users/user-notification.model.ts`) + Add the object in `UserNotificationModel.toFormattedJSON` + Handle this new notification type in client (`UserNotificationsComponent`) + Handle the new object property in client model (`UserNotification`) * Tests: - - Create your command class in `shared/server-commands/` that will wrap HTTP requests to your new endpoint + - Create your command class in `packages/server-commands/` that will wrap HTTP requests to your new endpoint - Add your command file in `index.ts` of current directory - - Instantiate your command class in `shared/server-commands/server/server.ts` - - Create your test file in `server/tests/api/check-params` to test middleware validators/authentification/user rights (offensive tests) - - Add it to `server/tests/api/check-params/index.ts` - - Create your test file in `server/tests/api` to test your new endpoints + - Instantiate your command class in `packages/server-commands/server/server/server.ts` + - Create your test file in `server/server/tests/api/check-params` to test middleware validators/authentification/user rights (offensive tests) + - Add it to `server/server/tests/api/check-params/index.ts` + - Create your test file in `server/server/tests/api` to test your new endpoints - Add it to `index.ts` of current directory - - Add your notification test in `server/tests/api/notifications` + - Add your notification test in `server/server/tests/api/notifications` * Update REST API documentation in `support/doc/api/openapi.yaml` diff --git a/support/doc/development/tests.md b/support/doc/development/tests.md index 1c2589c8a..2e4c6ff6a 100644 --- a/support/doc/development/tests.md +++ b/support/doc/development/tests.md @@ -8,7 +8,7 @@ Prepare PostgreSQL user so PeerTube can delete/create the test databases: sudo -u postgres createuser you_username --createdb --superuser ``` -Prepare databases: +Prepare the databases: ```bash npm run clean:server:test @@ -45,22 +45,19 @@ sudo apt-get install parallel libimage-exiftool-perl ### Test -To run all test suites: +To run all test suites (can be long!): ```bash npm run test # See scripts/test.sh to run a particular suite ``` -Most of tests can be run using: +To run a specific test: ```bash -TS_NODE_TRANSPILE_ONLY=true npm run mocha -- --timeout 30000 --exit -r ts-node/register -r tsconfig-paths/register --bail server/tests/api/videos/video-transcoder.ts -``` +npm run mocha -- --exit --bail packages/tests/src/your-test.ts -`server/tests/api/activitypub` tests will need different options: - -``` -TS_NODE_FILES=true mocha -- --timeout 30000 --exit -r ts-node/register -r tsconfig-paths/register --bail server/tests/api/activitypub/security.ts +# For example +npm run mocha -- --exit --bail packages/tests/src/api/videos/single-server.ts ``` ### Configuration diff --git a/support/doc/plugins/guide.md b/support/doc/plugins/guide.md index ff08ce8c4..07f484934 100644 --- a/support/doc/plugins/guide.md +++ b/support/doc/plugins/guide.md @@ -387,7 +387,7 @@ function register (...) { displayName: 'User display name', // Custom admin flags (bypass video auto moderation etc.) - // https://github.com/Chocobozzz/PeerTube/blob/develop/shared/models/users/user-flag.model.ts + // https://github.com/Chocobozzz/PeerTube/blob/develop/packages/models/users/user-flag.model.ts // PeerTube >= 5.1 adminFlags: 0, // Quota in bytes @@ -977,7 +977,7 @@ npm install --save-dev @peertube/peertube-types This package exposes *server* definition files by default: ```ts -import { RegisterServerOptions } from '@peertube/peertube-types' +import { RegisterServerOptions } from '@peertube/peertube-types.js' export async function register ({ registerHook }: RegisterServerOptions) { registerHook({ @@ -989,8 +989,8 @@ export async function register ({ registerHook }: RegisterServerOptions) { But it also exposes client types and various models used in __PeerTube__: ```ts -import { Video } from '@peertube/peertube-types'; -import { RegisterClientOptions } from '@peertube/peertube-types/client'; +import { Video } from '@peertube/peertube-types.js'; +import { RegisterClientOptions } from '@peertube/peertube-types/client.js'; function register({ registerHook, peertubeHelpers }: RegisterClientOptions) { registerHook({ @@ -1032,7 +1032,7 @@ If you want to translate strings of your plugin (like labels of your registered } ``` -The key should be one of the locales defined in [i18n.ts](https://github.com/Chocobozzz/PeerTube/blob/develop/shared/models/i18n/i18n.ts). +The key should be one of the locales defined in [i18n.ts](https://github.com/Chocobozzz/PeerTube/blob/develop/packages/models/i18n/i18n.ts). Translation files are just objects, with the english sentence as the key and the translation as the value. `fr.json` could contain for example: @@ -1070,40 +1070,25 @@ You built files are in the `dist/` directory. Check `package.json` to correctly ### Test your plugin/theme -PeerTube dev server (ran with `npm run dev` on `localhost:3000`) can't inject plugin CSS. -It's the reason why we don't use the dev mode but build PeerTube instead. +You need to have a running PeerTube instance with an administrator account. +If you're using dev server on your local computer, test your plugin on `localhost:9000` using `npm run dev` because plugin CSS is not injected in Angular webserver (`localhost:3000`). -You'll need to have a local PeerTube instance: - * Follow the [dev prerequisites](https://github.com/Chocobozzz/PeerTube/blob/develop/.github/CONTRIBUTING.md#prerequisites) - (to clone the repository, install dependencies and prepare the database) - * Build PeerTube: +Install PeerTube CLI (can be installed on another computer/server than the PeerTube instance): -```sh -npm run build +```bash +npm install -g @peertube/peertube-cli ``` - * Build the CLI: +Register the PeerTube instance via the CLI: ```sh -npm run setup:cli +peertube-cli auth add -u 'https://peertube.example.com' -U 'root' --password 'test' ``` - * Run PeerTube (you can access to your instance on `localhost:9000`): +Then, you can install your local plugin/theme by running: ```sh -NODE_ENV=dev npm start -``` - - * Register the instance via the CLI: - -```sh -node ./dist/server/tools/peertube.js auth add -u 'http://localhost:9000' -U 'root' --password 'test' -``` - -Then, you can install or reinstall your local plugin/theme by running: - -```sh -node ./dist/server/tools/peertube.js plugins install --path /your/absolute/plugin-or-theme/path +peertube-cli plugins install --path /your/absolute/plugin-or-theme/path ``` ### Publish diff --git a/support/doc/tools.md b/support/doc/tools.md index 2b3ebf159..40d9ec66a 100644 --- a/support/doc/tools.md +++ b/support/doc/tools.md @@ -4,59 +4,48 @@ ## Remote PeerTube CLI -You need at least 512MB RAM to run the script. -Scripts can be launched directly from a PeerTube server, or from a separate server, even a desktop PC. -You need to follow all the following steps even if you are on a PeerTube server (including cloning the git repository in a different directory than your production installation because the scripts utilize non-production dependencies). - -### Dependencies - -Install the [PeerTube dependencies](/support/doc/dependencies.md) except PostgreSQL and Redis. +`peertube-cli` is a tool that communicates with a PeerTube instance using its [REST API](https://docs.joinpeertube.org/api-rest-reference.html). +It can be launched from a remote server/computer to easily upload videos, manage plugins, redundancies etc. ### Installation -Clone the PeerTube repo to get the latest version (even if you are on your PeerTube server): +Ensure you have `node` installed on your system: ```bash -git clone https://github.com/Chocobozzz/PeerTube.git -CLONE="$(pwd)/PeerTube" -cd ${CLONE} +node --version # Should be >= 16.x ``` -Install dependencies and build CLI tools: +Then install the CLI: ```bash -NOCLIENT=1 yarn install --pure-lockfile -npm run setup:cli +sudo npm install -g @peertube/peertube-cli ``` ### CLI wrapper -The wrapper provides a convenient interface to the following scripts. -You can access it as `peertube` via an alias in your `.bashrc` like `alias peertube="cd /your/peertube/directory/ && node ./dist/server/tools/peertube.js"` (you have to keep the `cd` command): +The wrapper provides a convenient interface to the following sub-commands. ``` - Usage: peertube [command] [options] +Usage: peertube-cli [command] [options] - Options: +Options: + -v, --version output the version number + -h, --help display help for command - -v, --version output the version number - -h, --help output usage information - - Commands: - - auth [action] register your accounts on remote instances to use them with other commands - upload|up upload a video - import-videos|import import a video from a streaming platform - plugins|p [action] manage instance plugins - redundancy|r [action] manage video redundancies - help [cmd] display help for [cmd] +Commands: + auth Register your accounts on remote instances to use them with other commands + upload|up [options] Upload a video on a PeerTube instance + redundancy|r Manage instance redundancies + plugins|p Manage instance plugins/themes + get-access-token|token [options] Get a peertube access token + help [command] display help for command ``` The wrapper can keep track of instances you have an account on. We limit to one account per instance for now. ```bash -peertube auth add -u 'PEERTUBE_URL' -U 'PEERTUBE_USER' --password 'PEERTUBE_PASSWORD' -peertube auth list +peertube-cli auth add -u 'PEERTUBE_URL' -U 'PEERTUBE_USER' --password 'PEERTUBE_PASSWORD' +peertube-cli auth list ┌──────────────────────────────┬──────────────────────────────┐ │ instance │ login │ ├──────────────────────────────┼──────────────────────────────┤ @@ -64,331 +53,80 @@ peertube auth list └──────────────────────────────┴──────────────────────────────┘ ``` -You can now use that account to upload videos without feeding the same parameters again. +You can now use that account to execute sub-commands without feeding the `--url`, `--username` and `--password` parameters: ```bash -peertube up +peertube-cli upload +peertube-cli plugins list +... ``` -To list, install, uninstall dynamically plugins/themes of an instance: +#### peertube-cli upload -```bash -peertube plugins list -peertube plugins install --path /local/plugin/path -peertube plugins install --npm-name peertube-plugin-myplugin -peertube plugins uninstall --npm-name peertube-plugin-myplugin -``` - -#### peertube-import-videos.js - -You can use this script to import videos from all [supported sites of youtube-dl](https://rg3.github.io/youtube-dl/supportedsites.html) into PeerTube. -Be sure you own the videos or have the author's authorization to do so. - -```sh -node dist/server/tools/peertube-import-videos.js \ - -u 'PEERTUBE_URL' \ - -U 'PEERTUBE_USER' \ - --password 'PEERTUBE_PASSWORD' \ - --target-url 'TARGET_URL' -``` - -* `PEERTUBE_URL` : the full URL of your PeerTube server where you want to import, eg: https://peertube.cpy.re -* `PEERTUBE_USER` : your PeerTube account where videos will be uploaded -* `PEERTUBE_PASSWORD` : password of your PeerTube account (if `--password PEERTUBE_PASSWORD` is omitted, you will be prompted for it) -* `TARGET_URL` : the target url you want to import. Examples: - * YouTube: - * Channel: https://www.youtube.com/channel/ChannelId - * User https://www.youtube.com/c/UserName or https://www.youtube.com/user/UserName - * Video https://www.youtube.com/watch?v=blabla - * Vimeo: https://vimeo.com/xxxxxx - * Dailymotion: https://www.dailymotion.com/xxxxx - -The script will get all public videos from Youtube, download them and upload to PeerTube. -Already downloaded videos will not be uploaded twice, so you can run and re-run the script in case of crash, disconnection... - -Videos will be publicly available after transcoding (you can see them before that in your account on the web interface). - -**NB**: If you want to synchronize a Youtube channel to your PeerTube instance (ensure you have the agreement from the author), -you can add a [crontab rule](https://help.ubuntu.com/community/CronHowto) (or an equivalent of your OS) and insert -these rules (ensure to customize them to your needs): - -``` -# Update youtube-dl every day at midnight -0 0 * * * /usr/bin/npm rebuild youtube-dl --prefix /PATH/TO/PEERTUBE/ - -# Synchronize the YT channel every sunday at 22:00 all the videos published since last monday included -0 22 * * 0 /usr/bin/node /PATH/TO/PEERTUBE/dist/server/tools/peertube-import-videos.js -u '__PEERTUBE_URL__' -U '__USER__' --password '__PASSWORD__' --target-url 'https://www.youtube.com/channel/___CHANNEL__' --since $(date --date="-6 days" +\%Y-\%m-\%d) -``` - -Also you may want to subscribe to the PeerTube channel in order to manually check the synchronization is successful. - -#### peertube-upload.js - -You can use this script to import videos directly from the CLI. +You can use this script to upload videos directly from the CLI. Videos will be publicly available after transcoding (you can see them before that in your account on the web interface). ```bash cd ${CLONE} -node dist/server/tools/peertube-upload.js --help +peertube-cli upload --help ``` -#### peertube-plugins.js +#### peertube-cli plugins Install/update/uninstall or list local or NPM PeerTube plugins: ```bash cd ${CLONE} -node dist/server/tools/peertube-plugins.js --help -node dist/server/tools/peertube-plugins.js list --help -node dist/server/tools/peertube-plugins.js install --help -node dist/server/tools/peertube-plugins.js update --help -node dist/server/tools/peertube-plugins.js uninstall --help +peertube-cli plugins --help +peertube-cli plugins list --help +peertube-cli plugins install --help +peertube-cli plugins update --help +peertube-cli plugins uninstall --help -node dist/server/tools/peertube-plugins.js install --path /my/plugin/path -node dist/server/tools/peertube-plugins.js install --npm-name peertube-theme-example +peertube-cli plugins install --path /my/plugin/path +peertube-cli plugins install --npm-name peertube-theme-example ``` -#### peertube-redundancy.js +#### peertube-cli redundancy Manage (list/add/remove) video redundancies: To list your videos that are duplicated by remote instances: ```bash -node dist/server/tools/peertube.js redundancy list-remote-redundancies +peertube-cli redundancy list-remote-redundancies ``` To list remote videos that your instance duplicated: ```bash -node dist/server/tools/peertube.js redundancy list-my-redundancies +peertube-cli redundancy list-my-redundancies ``` To duplicate a specific video in your redundancy system: ```bash -node dist/server/tools/peertube.js redundancy add --video 823 +peertube-cli redundancy add --video 823 ``` To remove a video redundancy: ```bash -node dist/server/tools/peertube.js redundancy remove --video 823 +peertube-cli redundancy remove --video 823 ``` -## Server tools - -These scripts should be run on the server, in `peertube-latest` directory. - -### parse-log - -To parse PeerTube last log file: - -```bash -# Basic installation -cd /var/www/peertube/peertube-latest -sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run parse-log -- --level info - -# Docker installation -cd /var/www/peertube-docker -docker-compose exec -u peertube peertube npm run parse-log -- --level info -``` - -`--level` is optional and could be `info`/`warn`/`error` - -You can also remove SQL or HTTP logs using `--not-tags` (PeerTube >= 3.2): - -```bash -# Basic installation -cd /var/www/peertube/peertube-latest -sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run parse-log -- --level debug --not-tags http sql - -# Docker installation -cd /var/www/peertube-docker -docker-compose exec -u peertube peertube npm run parse-log -- --level debug --not-tags http sql -``` - -### regenerate-thumbnails.js - -**PeerTube >= 3.2** - -Regenerating local video thumbnails could be useful because new PeerTube releases may increase thumbnail sizes: - -```bash -# Basic installation -cd /var/www/peertube/peertube-latest -sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run regenerate-thumbnails - -# Docker installation -cd /var/www/peertube-docker -docker-compose exec -u peertube peertube npm run regenerate-thumbnails -``` - -### create-import-video-file-job.js - -You can use this script to import a video file to replace an already uploaded file or to add a new web compatible resolution to a video. PeerTube needs to be running. -You can then create a transcoding job using the web interface if you need to optimize your file or create an HLS version of it. - -```bash -# Basic installation -cd /var/www/peertube/peertube-latest -sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run create-import-video-file-job -- -v [videoUUID] -i [videoFile] - -# Docker installation -cd /var/www/peertube-docker -docker-compose exec -u peertube peertube npm run create-import-video-file-job -- -v [videoUUID] -i [videoFile] -``` - -### create-move-video-storage-job.js - -**PeerTube >= 4.0** - -Use this script to move all video files or a specific video file to object storage. - -```bash -# Basic installation -cd /var/www/peertube/peertube-latest -sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run create-move-video-storage-job -- --to-object-storage -v [videoUUID] - -# Docker installation -cd /var/www/peertube-docker -docker-compose exec -u peertube peertube npm run create-move-video-storage-job -- --to-object-storage -v [videoUUID] -``` - -The script can also move all video files that are not already in object storage: - -```bash -# Basic installation -cd /var/www/peertube/peertube-latest -sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run create-move-video-storage-job -- --to-object-storage --all-videos - -# Docker installation -cd /var/www/peertube-docker -docker-compose exec -u peertube peertube npm run create-move-video-storage-job -- --to-object-storage --all-videos -``` - - - -### prune-storage.js - -Some transcoded videos or shutdown at a bad time can leave some unused files on your storage. -Stop PeerTube and delete these files (a confirmation will be demanded first): - -```bash -cd /var/www/peertube/peertube-latest -sudo systemctl stop peertube && sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run prune-storage -``` - - -### update-host.js - -**Changing the hostname is unsupported and may be a risky operation, especially if you have already federated.** -If you started PeerTube with a domain, and then changed it you will have -invalid torrent files and invalid URLs in your database. To fix this, you have -to run the command below (keep in mind your follower instances will NOT update their URLs). - -```bash -# Basic installation -cd /var/www/peertube/peertube-latest -sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run update-host - -# Docker installation -cd /var/www/peertube-docker -docker-compose exec -u peertube peertube npm run update-host -``` - -### reset-password.js - -To reset a user password from CLI, run: - -```bash -# Basic installation -cd /var/www/peertube/peertube-latest -sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run reset-password -- -u target_username - -# Docker installation -cd /var/www/peertube-docker -docker-compose exec -u peertube peertube npm run reset-password -- -u target_username -``` - - -### plugin install/uninstall - -The difference with `peertube plugins` CLI is that these scripts can be used even if PeerTube is not running. -If PeerTube is running, you need to restart it for the changes to take effect (whereas with `peertube plugins` CLI, plugins/themes are dynamically loaded on the server). - -To install/update a plugin or a theme from the disk: - -```bash -cd /var/www/peertube/peertube-latest -sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run plugin:install -- --plugin-path /local/plugin/path - -# Docker installation -cd /var/www/peertube-docker -docker-compose exec -u peertube peertube npm run plugin:install -- --plugin-path /local/plugin/path -``` - -From NPM: - -```bash -cd /var/www/peertube/peertube-latest -sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run plugin:install -- --npm-name peertube-plugin-myplugin - -# Docker installation -cd /var/www/peertube-docker -docker-compose exec -u peertube peertube npm run plugin:install -- --npm-name peertube-plugin-myplugin -``` - -To uninstall a plugin or a theme: - -```bash -cd /var/www/peertube/peertube-latest -sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run plugin:uninstall -- --npm-name peertube-plugin-myplugin - -# Docker installation -cd /var/www/peertube-docker -docker-compose exec -u peertube peertube npm run plugin:uninstall -- --npm-name peertube-plugin-myplugin -``` ## PeerTube runner PeerTube >= 5.2 supports VOD or Live transcoding by a remote PeerTube runner. - ### Installation -Ensure you have `ffmpeg` and `ffprobe` installed on your system: +Ensure you have `node`, `ffmpeg` and `ffprobe` installed on your system: ```bash +node --version # Should be >= 16.x ffprobe -version # Should be >= 4.3 ffmpeg -version # Should be >= 4.3 ``` @@ -444,3 +182,201 @@ peertube-runner unregister --url http://peertube.example.com --runner-name my-ru ```bash peertube-runner list-registered ``` + +## Server tools + +Server tools are scripts that interect directly with the code of your PeerTube instance. +They must be run on the server, in `peertube-latest` directory. + +### Parse logs + +To parse PeerTube last log file: + +```bash +# Basic installation +cd /var/www/peertube/peertube-latest +sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run parse-log -- --level info + +# Docker installation +cd /var/www/peertube-docker +docker-compose exec -u peertube peertube npm run parse-log -- --level info +``` + +`--level` is optional and could be `info`/`warn`/`error` + +You can also remove SQL or HTTP logs using `--not-tags` (PeerTube >= 3.2): + +```bash +# Basic installation +cd /var/www/peertube/peertube-latest +sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run parse-log -- --level debug --not-tags http sql + +# Docker installation +cd /var/www/peertube-docker +docker-compose exec -u peertube peertube npm run parse-log -- --level debug --not-tags http sql +``` + +### Regenerate video thumbnails + +Regenerating local video thumbnails could be useful because new PeerTube releases may increase thumbnail sizes: + +```bash +# Basic installation +cd /var/www/peertube/peertube-latest +sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run regenerate-thumbnails + +# Docker installation +cd /var/www/peertube-docker +docker-compose exec -u peertube peertube npm run regenerate-thumbnails +``` + +### Add or replace specific video file + +You can use this script to import a video file to replace an already uploaded file or to add a new web compatible resolution to a video. PeerTube needs to be running. +You can then create a transcoding job using the web interface if you need to optimize your file or create an HLS version of it. + +```bash +# Basic installation +cd /var/www/peertube/peertube-latest +sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run create-import-video-file-job -- -v [videoUUID] -i [videoFile] + +# Docker installation +cd /var/www/peertube-docker +docker-compose exec -u peertube peertube npm run create-import-video-file-job -- -v [videoUUID] -i [videoFile] +``` + +### Move video files to object storage + +Use this script to move all video files or a specific video file to object storage. + +```bash +# Basic installation +cd /var/www/peertube/peertube-latest +sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run create-move-video-storage-job -- --to-object-storage -v [videoUUID] + +# Docker installation +cd /var/www/peertube-docker +docker-compose exec -u peertube peertube npm run create-move-video-storage-job -- --to-object-storage -v [videoUUID] +``` + +The script can also move all video files that are not already in object storage: + +```bash +# Basic installation +cd /var/www/peertube/peertube-latest +sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run create-move-video-storage-job -- --to-object-storage --all-videos + +# Docker installation +cd /var/www/peertube-docker +docker-compose exec -u peertube peertube npm run create-move-video-storage-job -- --to-object-storage --all-videos +``` + + + +### Prune filesystem storage + +Some transcoded videos or shutdown at a bad time can leave some unused files on your storage. +Stop PeerTube and delete these files (a confirmation will be demanded first): + +```bash +cd /var/www/peertube/peertube-latest +sudo systemctl stop peertube && sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run prune-storage +``` + +### Update PeerTube instance domain name + +**Changing the hostname is unsupported and may be a risky operation, especially if you have already federated.** +If you started PeerTube with a domain, and then changed it you will have +invalid torrent files and invalid URLs in your database. To fix this, you have +to run the command below (keep in mind your follower instances will NOT update their URLs). + +```bash +# Basic installation +cd /var/www/peertube/peertube-latest +sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run update-host + +# Docker installation +cd /var/www/peertube-docker +docker-compose exec -u peertube peertube npm run update-host +``` + +### Reset user password + +To reset a user password from CLI, run: + +```bash +# Basic installation +cd /var/www/peertube/peertube-latest +sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run reset-password -- -u target_username + +# Docker installation +cd /var/www/peertube-docker +docker-compose exec -u peertube peertube npm run reset-password -- -u target_username +``` + + +### Install or uninstall plugins + +The difference with `peertube plugins` CLI is that these scripts can be used even if PeerTube is not running. +If PeerTube is running, you need to restart it for the changes to take effect (whereas with `peertube plugins` CLI, plugins/themes are dynamically loaded on the server). + +To install/update a plugin or a theme from the disk: + +```bash +cd /var/www/peertube/peertube-latest +sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run plugin:install -- --plugin-path /local/plugin/path + +# Docker installation +cd /var/www/peertube-docker +docker-compose exec -u peertube peertube npm run plugin:install -- --plugin-path /local/plugin/path +``` + +From NPM: + +```bash +cd /var/www/peertube/peertube-latest +sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run plugin:install -- --npm-name peertube-plugin-myplugin + +# Docker installation +cd /var/www/peertube-docker +docker-compose exec -u peertube peertube npm run plugin:install -- --npm-name peertube-plugin-myplugin +``` + +To uninstall a plugin or a theme: + +```bash +cd /var/www/peertube/peertube-latest +sudo -u peertube NODE_CONFIG_DIR=/var/www/peertube/config NODE_ENV=production npm run plugin:uninstall -- --npm-name peertube-plugin-myplugin + +# Docker installation +cd /var/www/peertube-docker +docker-compose exec -u peertube peertube npm run plugin:uninstall -- --npm-name peertube-plugin-myplugin +``` diff --git a/support/nginx/peertube b/support/nginx/peertube index 5ce59a112..7028566c7 100644 --- a/support/nginx/peertube +++ b/support/nginx/peertube @@ -183,7 +183,7 @@ server { #client_body_temp_path /var/www/peertube/storage/nginx/; # Bypass PeerTube for performance reasons. Optional. - # Should be consistent with client-overrides assets list in /server/controllers/client.ts + # Should be consistent with client-overrides assets list in client.ts server controller location ~ ^/client/(assets/images/(icons/icon-36x36\.png|icons/icon-48x48\.png|icons/icon-72x72\.png|icons/icon-96x96\.png|icons/icon-144x144\.png|icons/icon-192x192\.png|icons/icon-512x512\.png|logo\.svg|favicon\.png|default-playlist\.jpg|default-avatar-account\.png|default-avatar-account-48x48\.png|default-avatar-video-channel\.png|default-avatar-video-channel-48x48\.png))$ { add_header Cache-Control "public, max-age=31536000, immutable"; # Cache 1 year diff --git a/tsconfig.base.json b/tsconfig.base.json index 18ba8f06c..cce74235b 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -1,11 +1,12 @@ { "compilerOptions": { - "module": "commonjs", - "target": "es2015", + "module": "NodeNext", + "target": "ES2017", "noImplicitAny": false, "sourceMap": true, "experimentalDecorators": true, "emitDecoratorMetadata": true, + "isolatedModules": true, "importHelpers": true, "removeComments": true, "esModuleInterop": true, @@ -19,11 +20,6 @@ "es2019" ], "baseUrl": "./", - "paths": { - "@server/*": [ "server/*" ], - "@shared/*": [ "shared/*" ], - "@client/*": [ "client/src/*" ] - }, "resolveJsonModule": true, "strict": false, "strictBindCallApply": true, diff --git a/tsconfig.eslint.json b/tsconfig.eslint.json new file mode 100644 index 000000000..c2e868173 --- /dev/null +++ b/tsconfig.eslint.json @@ -0,0 +1,32 @@ +{ + "extends": "./tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist/", + "baseUrl": "./", + "paths": { + "@server/*": [ "server/*" ] + }, + "typeRoots": [ + "node_modules/@types" + ] + }, + "include": [ + "./server.ts", + "server/**/*.ts", + "packages/**/*.ts", + "apps/**/*.ts", + "scripts/**/*.ts" + ], + "references": [ + { "path": "./server" }, + { "path": "./scripts" }, + { "path": "./apps/peertube-runner" }, + { "path": "./apps/peertube-cli" }, + { "path": "./packages/core-utils" }, + { "path": "./packages/ffmpeg" }, + { "path": "./packages/models" }, + { "path": "./packages/node-utils" }, + { "path": "./packages/server-commands" }, + { "path": "./packages/typescript-utils" } + ] +} diff --git a/tsconfig.json b/tsconfig.json deleted file mode 100644 index 8bcd944e3..000000000 --- a/tsconfig.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "extends": "./tsconfig.base.json", - "compilerOptions": { - "outDir": "./dist/", - "baseUrl": "./", - "paths": { - "@server/*": [ "server/*" ], - "@shared/*": [ "shared/*" ] - }, - "typeRoots": [ - "node_modules/@types" - ] - }, - "references": [ - { "path": "./shared" }, - { "path": "./server" }, - { "path": "./scripts" } - ], - "files": [ "server.ts", "server/types/express.d.ts", "server/types/lib.d.ts" ] -} diff --git a/yarn.lock b/yarn.lock index 2686c4d4a..e9c5c22d1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1269,12 +1269,10 @@ resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9" integrity sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ== -"@cspotcode/source-map-support@^0.8.0": - version "0.8.1" - resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz#00629c35a688e05a88b1cda684fb9d5e73f000a1" - integrity sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw== - dependencies: - "@jridgewell/trace-mapping" "0.3.9" +"@commander-js/extra-typings@^11.0.0": + version "11.0.0" + resolved "https://registry.yarnpkg.com/@commander-js/extra-typings/-/extra-typings-11.0.0.tgz#eb922a59550454cad1f319d3d33e675e12e92fa0" + integrity sha512-06ol6Kn5gPjFY6v0vWOZ84nQwyqhZdaeZCHYH3vhwewjpOEjniF1KHZxh18887G3poWiJ8qyq5pb6ANuiddfPQ== "@dabh/diagnostics@^2.0.2": version "2.0.3" @@ -1294,6 +1292,250 @@ ky-universal "^0.11.0" undici "^5.21.2" +"@esbuild-kit/cjs-loader@^2.4.2": + version "2.4.2" + resolved "https://registry.yarnpkg.com/@esbuild-kit/cjs-loader/-/cjs-loader-2.4.2.tgz#cb4dde00fbf744a68c4f20162ea15a8242d0fa54" + integrity sha512-BDXFbYOJzT/NBEtp71cvsrGPwGAMGRB/349rwKuoxNSiKjPraNNnlK6MIIabViCjqZugu6j+xeMDlEkWdHHJSg== + dependencies: + "@esbuild-kit/core-utils" "^3.0.0" + get-tsconfig "^4.4.0" + +"@esbuild-kit/core-utils@^3.0.0": + version "3.1.0" + resolved "https://registry.yarnpkg.com/@esbuild-kit/core-utils/-/core-utils-3.1.0.tgz#49945d533dbd5e1b7620aa0fc522c15e6ec089c5" + integrity sha512-Uuk8RpCg/7fdHSceR1M6XbSZFSuMrxcePFuGgyvsBn+u339dk5OeL4jv2EojwTN2st/unJGsVm4qHWjWNmJ/tw== + dependencies: + esbuild "~0.17.6" + source-map-support "^0.5.21" + +"@esbuild-kit/esm-loader@^2.5.5": + version "2.5.5" + resolved "https://registry.yarnpkg.com/@esbuild-kit/esm-loader/-/esm-loader-2.5.5.tgz#b82da14fcee3fc1d219869756c06f43f67d1ca71" + integrity sha512-Qwfvj/qoPbClxCRNuac1Du01r9gvNOT+pMYtJDapfB1eoGN1YlJ1BixLyL9WVENRx5RXgNLdfYdx/CuswlGhMw== + dependencies: + "@esbuild-kit/core-utils" "^3.0.0" + get-tsconfig "^4.4.0" + +"@esbuild/android-arm64@0.17.19": + version "0.17.19" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.17.19.tgz#bafb75234a5d3d1b690e7c2956a599345e84a2fd" + integrity sha512-KBMWvEZooR7+kzY0BtbTQn0OAYY7CsiydT63pVEaPtVYF0hXbUaOyZog37DKxK7NF3XacBJOpYT4adIJh+avxA== + +"@esbuild/android-arm64@0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.19.0.tgz#c5ea635bdbe9b83d1f78a711120814e716439029" + integrity sha512-AzsozJnB+RNaDncBCs3Ys5g3kqhPFUueItfEaCpp89JH2naFNX2mYDIvUgPYMqqjm8hiFoo+jklb3QHZyR3ubw== + +"@esbuild/android-arm@0.17.19": + version "0.17.19" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.17.19.tgz#5898f7832c2298bc7d0ab53701c57beb74d78b4d" + integrity sha512-rIKddzqhmav7MSmoFCmDIb6e2W57geRsM94gV2l38fzhXMwq7hZoClug9USI2pFRGL06f4IOPHHpFNOkWieR8A== + +"@esbuild/android-arm@0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.19.0.tgz#6eb6e1fbc0dbfafa035aaef8b5ecde25b539fcf9" + integrity sha512-GAkjUyHgWTYuex3evPd5V7uV/XS4LMKr1PWHRPW1xNyy/Jx08x3uTrDFRefBYLKT/KpaWM8/YMQcwbp5a3yIDA== + +"@esbuild/android-x64@0.17.19": + version "0.17.19" + resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.17.19.tgz#658368ef92067866d95fb268719f98f363d13ae1" + integrity sha512-uUTTc4xGNDT7YSArp/zbtmbhO0uEEK9/ETW29Wk1thYUJBz3IVnvgEiEwEa9IeLyvnpKrWK64Utw2bgUmDveww== + +"@esbuild/android-x64@0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.19.0.tgz#99f154f71f5b92e778468bcf0f425d166c17bf20" + integrity sha512-SUG8/qiVhljBDpdkHQ9DvOWbp7hFFIP0OzxOTptbmVsgBgzY6JWowmMd6yJuOhapfxmj/DrvwKmjRLvVSIAKZg== + +"@esbuild/darwin-arm64@0.17.19": + version "0.17.19" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.17.19.tgz#584c34c5991b95d4d48d333300b1a4e2ff7be276" + integrity sha512-80wEoCfF/hFKM6WE1FyBHc9SfUblloAWx6FJkFWTWiCoht9Mc0ARGEM47e67W9rI09YoUxJL68WHfDRYEAvOhg== + +"@esbuild/darwin-arm64@0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.19.0.tgz#2fcc11abf95fbabbf9167db6a11d899385bd777b" + integrity sha512-HkxZ8k3Jvcw0FORPNTavA8BMgQjLOB6AajT+iXmil7BwY3gU1hWvJJAyWyEogCmA4LdbGvKF8vEykdmJ4xNJJQ== + +"@esbuild/darwin-x64@0.17.19": + version "0.17.19" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.17.19.tgz#7751d236dfe6ce136cce343dce69f52d76b7f6cb" + integrity sha512-IJM4JJsLhRYr9xdtLytPLSH9k/oxR3boaUIYiHkAawtwNOXKE8KoU8tMvryogdcT8AU+Bflmh81Xn6Q0vTZbQw== + +"@esbuild/darwin-x64@0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.19.0.tgz#b5bbde35468db093fdf994880b0eb4b62613b67c" + integrity sha512-9IRWJjqpWFHM9a5Qs3r3bK834NCFuDY5ZaLrmTjqE+10B6w65UMQzeZjh794JcxpHolsAHqwsN/33crUXNCM2Q== + +"@esbuild/freebsd-arm64@0.17.19": + version "0.17.19" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.17.19.tgz#cacd171665dd1d500f45c167d50c6b7e539d5fd2" + integrity sha512-pBwbc7DufluUeGdjSU5Si+P3SoMF5DQ/F/UmTSb8HXO80ZEAJmrykPyzo1IfNbAoaqw48YRpv8shwd1NoI0jcQ== + +"@esbuild/freebsd-arm64@0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.19.0.tgz#3f64c76dc590f79cc40acef6b22dd5eb89fc2125" + integrity sha512-s7i2WcXcK0V1PJHVBe7NsGddsL62a9Vhpz2U7zapPrwKoFuxPP9jybwX8SXnropR/AOj3ppt2ern4ItblU6UQQ== + +"@esbuild/freebsd-x64@0.17.19": + version "0.17.19" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.17.19.tgz#0769456eee2a08b8d925d7c00b79e861cb3162e4" + integrity sha512-4lu+n8Wk0XlajEhbEffdy2xy53dpR06SlzvhGByyg36qJw6Kpfk7cp45DR/62aPH9mtJRmIyrXAS5UWBrJT6TQ== + +"@esbuild/freebsd-x64@0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.19.0.tgz#14d497e9e858fba2bb9b16130602b7f5944bc09c" + integrity sha512-NMdBSSdgwHCqCsucU5k1xflIIRU0qi1QZnM6+vdGy5fvxm1c8rKh50VzsWsIVTFUG3l91AtRxVwoz3Lcvy3I5w== + +"@esbuild/linux-arm64@0.17.19": + version "0.17.19" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.17.19.tgz#38e162ecb723862c6be1c27d6389f48960b68edb" + integrity sha512-ct1Tg3WGwd3P+oZYqic+YZF4snNl2bsnMKRkb3ozHmnM0dGWuxcPTTntAF6bOP0Sp4x0PjSF+4uHQ1xvxfRKqg== + +"@esbuild/linux-arm64@0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.19.0.tgz#0f2f4d8889f7dc89681c306d7312aa76445a5f65" + integrity sha512-I4zvE2srSZxRPapFnNqj+NL3sDJ1wkvEZqt903OZUlBBgigrQMvzUowvP/TTTu2OGYe1oweg5MFilfyrElIFag== + +"@esbuild/linux-arm@0.17.19": + version "0.17.19" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.17.19.tgz#1a2cd399c50040184a805174a6d89097d9d1559a" + integrity sha512-cdmT3KxjlOQ/gZ2cjfrQOtmhG4HJs6hhvm3mWSRDPtZ/lP5oe8FWceS10JaSJC13GBd4eH/haHnqf7hhGNLerA== + +"@esbuild/linux-arm@0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.19.0.tgz#0b0f79dc72884f0ad02c0aabfc969a0bee7f6775" + integrity sha512-2F1+lH7ZBcCcgxiSs8EXQV0PPJJdTNiNcXxDb61vzxTRJJkXX1I/ye9mAhfHyScXzHaEibEXg1Jq9SW586zz7w== + +"@esbuild/linux-ia32@0.17.19": + version "0.17.19" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.17.19.tgz#e28c25266b036ce1cabca3c30155222841dc035a" + integrity sha512-w4IRhSy1VbsNxHRQpeGCHEmibqdTUx61Vc38APcsRbuVgK0OPEnQ0YD39Brymn96mOx48Y2laBQGqgZ0j9w6SQ== + +"@esbuild/linux-ia32@0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.19.0.tgz#dfcece1f5e74d0e7db090475e48b28d9aa270687" + integrity sha512-dz2Q7+P92r1Evc8kEN+cQnB3qqPjmCrOZ+EdBTn8lEc1yN8WDgaDORQQiX+mxaijbH8npXBT9GxUqE52Gt6Y+g== + +"@esbuild/linux-loong64@0.17.19": + version "0.17.19" + resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.17.19.tgz#0f887b8bb3f90658d1a0117283e55dbd4c9dcf72" + integrity sha512-2iAngUbBPMq439a+z//gE+9WBldoMp1s5GWsUSgqHLzLJ9WoZLZhpwWuym0u0u/4XmZ3gpHmzV84PonE+9IIdQ== + +"@esbuild/linux-loong64@0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.19.0.tgz#710f5bd55db3f5d9ebac8773ea49795261a35ca7" + integrity sha512-IcVJovJVflih4oFahhUw+N7YgNbuMSVFNr38awb0LNzfaiIfdqIh518nOfYaNQU3aVfiJnOIRVJDSAP4k35WxA== + +"@esbuild/linux-mips64el@0.17.19": + version "0.17.19" + resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.17.19.tgz#f5d2a0b8047ea9a5d9f592a178ea054053a70289" + integrity sha512-LKJltc4LVdMKHsrFe4MGNPp0hqDFA1Wpt3jE1gEyM3nKUvOiO//9PheZZHfYRfYl6AwdTH4aTcXSqBerX0ml4A== + +"@esbuild/linux-mips64el@0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.19.0.tgz#a918b310f9bf31fced3853ca52fee6e7acc09824" + integrity sha512-bZGRAGySMquWsKw0gIdsClwfvgbsSq/7oq5KVu1H1r9Il+WzOcfkV1hguntIuBjRVL8agI95i4AukjdAV2YpUw== + +"@esbuild/linux-ppc64@0.17.19": + version "0.17.19" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.17.19.tgz#876590e3acbd9fa7f57a2c7d86f83717dbbac8c7" + integrity sha512-/c/DGybs95WXNS8y3Ti/ytqETiW7EU44MEKuCAcpPto3YjQbyK3IQVKfF6nbghD7EcLUGl0NbiL5Rt5DMhn5tg== + +"@esbuild/linux-ppc64@0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.19.0.tgz#104771ef6ce2719ac17031f6b9ed8aa98f8e5faf" + integrity sha512-3LC6H5/gCDorxoRBUdpLV/m7UthYSdar0XcCu+ypycQxMS08MabZ06y1D1yZlDzL/BvOYliRNRWVG/YJJvQdbg== + +"@esbuild/linux-riscv64@0.17.19": + version "0.17.19" + resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.17.19.tgz#7f49373df463cd9f41dc34f9b2262d771688bf09" + integrity sha512-FC3nUAWhvFoutlhAkgHf8f5HwFWUL6bYdvLc/TTuxKlvLi3+pPzdZiFKSWz/PF30TB1K19SuCxDTI5KcqASJqA== + +"@esbuild/linux-riscv64@0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.19.0.tgz#83beafa472ad4224adcd4d7469e3a17ba1fbd976" + integrity sha512-jfvdKjWk+Cp2sgLtEEdSHXO7qckrw2B2eFBaoRdmfhThqZs29GMMg7q/LsQpybA7BxCLLEs4di5ucsWzZC5XPA== + +"@esbuild/linux-s390x@0.17.19": + version "0.17.19" + resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.17.19.tgz#e2afd1afcaf63afe2c7d9ceacd28ec57c77f8829" + integrity sha512-IbFsFbxMWLuKEbH+7sTkKzL6NJmG2vRyy6K7JJo55w+8xDk7RElYn6xvXtDW8HCfoKBFK69f3pgBJSUSQPr+4Q== + +"@esbuild/linux-s390x@0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.19.0.tgz#edc26cb41d8745716bda9c26bac1f0001eaad029" + integrity sha512-ofcucfNLkoXmcnJaw9ugdEOf40AWKGt09WBFCkpor+vFJVvmk/8OPjl/qRtks2Z7BuZbG3ztJuK1zS9z5Cgx9A== + +"@esbuild/linux-x64@0.17.19": + version "0.17.19" + resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.17.19.tgz#8a0e9738b1635f0c53389e515ae83826dec22aa4" + integrity sha512-68ngA9lg2H6zkZcyp22tsVt38mlhWde8l3eJLWkyLrp4HwMUr3c1s/M2t7+kHIhvMjglIBrFpncX1SzMckomGw== + +"@esbuild/linux-x64@0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.19.0.tgz#80a6b5e55ad454e0c0af5bdb267335287e331007" + integrity sha512-Fpf7zNDBti3xrQKQKLdXT0hTyOxgFdRJIMtNy8x1az9ATR9/GJ1brYbB/GLWoXhKiHsoWs+2DLkFVNNMTCLEwA== + +"@esbuild/netbsd-x64@0.17.19": + version "0.17.19" + resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.17.19.tgz#c29fb2453c6b7ddef9a35e2c18b37bda1ae5c462" + integrity sha512-CwFq42rXCR8TYIjIfpXCbRX0rp1jo6cPIUPSaWwzbVI4aOfX96OXY8M6KNmtPcg7QjYeDmN+DD0Wp3LaBOLf4Q== + +"@esbuild/netbsd-x64@0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.19.0.tgz#2e6e8d869b58aea34bab9c0c47f15ae1bda29a90" + integrity sha512-AMQAp/5oENgDOvVhvOlbhVe1pWii7oFAMRHlmTjSEMcpjTpIHtFXhv9uAFgUERHm3eYtNvS9Vf+gT55cwuI6Aw== + +"@esbuild/openbsd-x64@0.17.19": + version "0.17.19" + resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.17.19.tgz#95e75a391403cb10297280d524d66ce04c920691" + integrity sha512-cnq5brJYrSZ2CF6c35eCmviIN3k3RczmHz8eYaVlNasVqsNY+JKohZU5MKmaOI+KkllCdzOKKdPs762VCPC20g== + +"@esbuild/openbsd-x64@0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.19.0.tgz#ca0817d3ab332afb0d8d96a2eb42b4d8ebaa8715" + integrity sha512-fDztEve1QUs3h/Dw2AUmBlWGkNQbhDoD05ppm5jKvzQv+HVuV13so7m5RYeiSMIC2XQy7PAjZh+afkxAnCRZxA== + +"@esbuild/sunos-x64@0.17.19": + version "0.17.19" + resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.17.19.tgz#722eaf057b83c2575937d3ffe5aeb16540da7273" + integrity sha512-vCRT7yP3zX+bKWFeP/zdS6SqdWB8OIpaRq/mbXQxTGHnIxspRtigpkUcDMlSCOejlHowLqII7K2JKevwyRP2rg== + +"@esbuild/sunos-x64@0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.19.0.tgz#8de27de2563cb3eb6c1af066b6d7fcb1229fe3d4" + integrity sha512-bKZzJ2/rvUjDzA5Ddyva2tMk89WzNJEibZEaq+wY6SiqPlwgFbqyQLimouxLHiHh1itb5P3SNCIF1bc2bw5H9w== + +"@esbuild/win32-arm64@0.17.19": + version "0.17.19" + resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.17.19.tgz#9aa9dc074399288bdcdd283443e9aeb6b9552b6f" + integrity sha512-yYx+8jwowUstVdorcMdNlzklLYhPxjniHWFKgRqH7IFlUEa0Umu3KuYplf1HUZZ422e3NU9F4LGb+4O0Kdcaag== + +"@esbuild/win32-arm64@0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.19.0.tgz#67c2b410ff8862be2cd61145ad21e11be00fb914" + integrity sha512-NQJ+4jmnA79saI+sE+QzcEls19uZkoEmdxo7r//PDOjIpX8pmoWtTnWg6XcbnO7o4fieyAwb5U2LvgWynF4diA== + +"@esbuild/win32-ia32@0.17.19": + version "0.17.19" + resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.17.19.tgz#95ad43c62ad62485e210f6299c7b2571e48d2b03" + integrity sha512-eggDKanJszUtCdlVs0RB+h35wNlb5v4TWEkq4vZcmVt5u/HiDZrTXe2bWFQUez3RgNHwx/x4sk5++4NSSicKkw== + +"@esbuild/win32-ia32@0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.19.0.tgz#cac8992219c6d943bb22226e4afeb3774a29cca1" + integrity sha512-uyxiZAnsfu9diHm9/rIH2soecF/HWLXYUhJKW4q1+/LLmNQ+55lRjvSUDhUmsgJtSUscRJB/3S4RNiTb9o9mCg== + +"@esbuild/win32-x64@0.17.19": + version "0.17.19" + resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.17.19.tgz#8cfaf2ff603e9aabb910e9c0558c26cf32744061" + integrity sha512-lAhycmKnVOuRYNtRtatQR1LPQf2oYCkRGkSFnseDAKPl8lu5SOsK/e1sXe5a0Pc5kHIHe6P2I/ilntNv2xf3cA== + +"@esbuild/win32-x64@0.19.0": + version "0.19.0" + resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.19.0.tgz#fa5f12c96811cec3233a53bdbf61d1a05ba9018f" + integrity sha512-jl+NXUjK2StMgqnZnqgNjZuerFG8zQqWXMBZdMMv4W/aO1ZKQaYWZBxTrtWKphkCBVEMh0wMVfGgOd2BjOZqUQ== + "@eslint-community/eslint-utils@^4.1.2", "@eslint-community/eslint-utils@^4.2.0", "@eslint-community/eslint-utils@^4.4.0": version "4.4.0" resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz#a23514e8fb9af1269d5f7788aa556798d61c6b59" @@ -1634,11 +1876,6 @@ resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz#2203b118c157721addfe69d47b70465463066d78" integrity sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w== -"@jridgewell/resolve-uri@^3.0.3": - version "3.1.1" - resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz#c08679063f279615a3326583ba3a90d1d82cc721" - integrity sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA== - "@jridgewell/set-array@^1.0.1": version "1.1.2" resolved "https://registry.yarnpkg.com/@jridgewell/set-array/-/set-array-1.1.2.tgz#7c6cf998d6d20b914c0a55a91ae928ff25965e72" @@ -1654,14 +1891,6 @@ resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz#d7c6e6755c78567a951e04ab52ef0fd26de59f32" integrity sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg== -"@jridgewell/trace-mapping@0.3.9": - version "0.3.9" - resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz#6534fd5933a53ba7cbf3a17615e273a0d1273ff9" - integrity sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ== - dependencies: - "@jridgewell/resolve-uri" "^3.0.3" - "@jridgewell/sourcemap-codec" "^1.4.10" - "@jridgewell/trace-mapping@^0.3.17", "@jridgewell/trace-mapping@^0.3.9": version "0.3.18" resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.18.tgz#25783b2086daf6ff1dcb53c9249ae480e4dd4cd6" @@ -1828,176 +2057,196 @@ resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.4.1.tgz#ff22eb2e5d476fbc2450a196e40dd243cc20c28f" integrity sha512-O2yRJce1GOc6PAy3QxFM4NzFiWzvScDC1/5ihYBL6BUEVdq0XMWN01sppE+H6bBXbaFYipjwFLEWLg5PaSOThA== -"@opentelemetry/context-async-hooks@1.13.0": - version "1.13.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/context-async-hooks/-/context-async-hooks-1.13.0.tgz#b697317c1670eaa9b1c23201d09dd29250dcc8fa" - integrity sha512-pS5fU4lrRjOIPZQqA2V1SUM9QUFXbO+8flubAiy6ntLjnAjJJUdRFOUOxK6v86ZHI2p2S8A0vD0BTu95FZYvjA== +"@opentelemetry/context-async-hooks@1.15.1": + version "1.15.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/context-async-hooks/-/context-async-hooks-1.15.1.tgz#694798afeb83eab5fe31c6e15762815b27615c65" + integrity sha512-JHPs/o15OO902lI5jkWWPz0JyOpQav7hfOY20MZFH/elq6kSvjBTw5cCu1v7SJwN0Ac3n08fOjYK+jtNlYP0LA== -"@opentelemetry/core@1.13.0", "@opentelemetry/core@^1.8.0": +"@opentelemetry/core@1.15.1", "@opentelemetry/core@^1.1.0", "@opentelemetry/core@^1.13.0": + version "1.15.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/core/-/core-1.15.1.tgz#a580a547c1006cc411ae7aacd4991b52555b3f1d" + integrity sha512-V6GoRTY6aANMDDOQ9CiHOiLWEK2b2b3OGZK+zk05Li5merb9jadFeV5ooTSGtjxfxVNMpQUaQERO1cdbdbeEGg== + dependencies: + "@opentelemetry/semantic-conventions" "1.15.1" + +"@opentelemetry/core@^1.8.0": version "1.13.0" resolved "https://registry.yarnpkg.com/@opentelemetry/core/-/core-1.13.0.tgz#7cdcb4176d260d279b0aa31456c4ce2ba7f410aa" integrity sha512-2dBX3Sj99H96uwJKvc2w9NOiNgbvAO6mOFJFramNkKfS9O4Um+VWgpnlAazoYjT6kUJ1MP70KQ5ngD4ed+4NUw== dependencies: "@opentelemetry/semantic-conventions" "1.13.0" -"@opentelemetry/exporter-jaeger@^1.3.1": - version "1.13.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-jaeger/-/exporter-jaeger-1.13.0.tgz#e96436438d3f8cc7b262ab4e517d55f96f413161" - integrity sha512-ke/STs/erRDqKmNv6Dv+5SetXsVD+Zm1/Wo8cLdAGrZn6kG6Fyp5EXVO/BJuzx6q+jHCdODm8jV4veXl4m71nQ== +"@opentelemetry/exporter-jaeger@^1.15.1": + version "1.15.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-jaeger/-/exporter-jaeger-1.15.1.tgz#2081eae6ea7906ff39ad82b4aeed0f8913692fe9" + integrity sha512-Rh+0OQXeOzyTpDJ48+iJRIp7Sq1RKTycG9seoiBecwxyevnNLPghAZvj/DgOc7SGK8kmK2CQ0aqkhBE3kT6hKw== dependencies: - "@opentelemetry/core" "1.13.0" - "@opentelemetry/sdk-trace-base" "1.13.0" - "@opentelemetry/semantic-conventions" "1.13.0" + "@opentelemetry/core" "1.15.1" + "@opentelemetry/sdk-trace-base" "1.15.1" + "@opentelemetry/semantic-conventions" "1.15.1" jaeger-client "^3.15.0" -"@opentelemetry/exporter-prometheus@~0.39.1": - version "0.39.1" - resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-prometheus/-/exporter-prometheus-0.39.1.tgz#2ce574e54faae628a260579b4ffa7b73456fb331" - integrity sha512-FRedkVIUgLApKd9aF7cpflXzXRkHxKiV9pJwvY8dqk9amP5QuZKZgP3d+2L1IIKe10+JbcKIrijddZnT9jID2g== +"@opentelemetry/exporter-prometheus@~0.41.1": + version "0.41.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/exporter-prometheus/-/exporter-prometheus-0.41.1.tgz#1721c918908f71cdfc5174bb8d2fa0b59eb760e9" + integrity sha512-6XpRLbzZ0AWmECSa3mpHcN7E5ilfUf2dbBB0VhdnNvTycBnLNMbFFJ06lVVLFLg8BgnxH+S30KaPP8Jng979pA== dependencies: - "@opentelemetry/core" "1.13.0" - "@opentelemetry/resources" "1.13.0" - "@opentelemetry/sdk-metrics" "1.13.0" + "@opentelemetry/core" "1.15.1" + "@opentelemetry/resources" "1.15.1" + "@opentelemetry/sdk-metrics" "1.15.1" -"@opentelemetry/instrumentation-dns@^0.31.2": - version "0.31.4" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-dns/-/instrumentation-dns-0.31.4.tgz#515dc3afac32fdf192e3b91553baade738634145" - integrity sha512-TUNybmyCYxKQwvFo+6gzaTBYP5aO9i2wqo/gBCAgd/TnHZzzEpRl4PZIwU1qzNRTcHUzpHXYA05F7GyQGebEVw== +"@opentelemetry/instrumentation-dns@^0.32.0": + version "0.32.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-dns/-/instrumentation-dns-0.32.0.tgz#2262e6b68bb415508cde4958b914a17ea5356c0b" + integrity sha512-Q6RHePHnMQdKsAKzKvPSN0nPfKVlzFlbPa9/nb3r0FhThCP/Ucjob138X4LEDy0ZyZs3mq8Vqr9riyyRHIW6iA== dependencies: - "@opentelemetry/instrumentation" "^0.39.1" + "@opentelemetry/instrumentation" "^0.41.0" "@opentelemetry/semantic-conventions" "^1.0.0" semver "^7.3.2" + tslib "^2.3.1" -"@opentelemetry/instrumentation-express@^0.32.1": - version "0.32.3" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-express/-/instrumentation-express-0.32.3.tgz#cd381cbcf048146731f407ce7aaef19e272ae197" - integrity sha512-/A9eJAA7XXj6GkktlsM9YKORQiIpgFRZT3J79MEGNbMwNHTPh4sOuzjAnARcpUQ3JKuYs7T98fs35aRH+Ms43w== +"@opentelemetry/instrumentation-express@^0.33.0": + version "0.33.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-express/-/instrumentation-express-0.33.0.tgz#d972ef43420759761f7dbfe78a6ae159cd925a69" + integrity sha512-Cem3AssubzUoBK5ab89rBt2kY90i/FFyQwMC9wPjBQldkOaT4cR+5ufvWritXRfoPltqEeX2imLavujNH6EzCw== dependencies: "@opentelemetry/core" "^1.8.0" - "@opentelemetry/instrumentation" "^0.39.1" + "@opentelemetry/instrumentation" "^0.41.0" "@opentelemetry/semantic-conventions" "^1.0.0" "@types/express" "4.17.13" + tslib "^2.3.1" -"@opentelemetry/instrumentation-fs@^0.7.0": - version "0.7.3" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-fs/-/instrumentation-fs-0.7.3.tgz#3fd631f0606ea25fc08936b7e8a89f6eb3571cb3" - integrity sha512-GUJvcU6/lZI4gpA3Mu7FP7hVHYk9IS6C2gGJlEhzzBOrStIw+xWzupFbra+sA2+ds1IPDUdAOBvNp0fhBrou5A== +"@opentelemetry/instrumentation-fs@^0.8.0": + version "0.8.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-fs/-/instrumentation-fs-0.8.0.tgz#63d6936b7e5d3de6aeffb4a37b0c5f65accd132c" + integrity sha512-uZMLqy1LKkLlRKC84HkjU3DYmVixTzRlxdfINHFyStBEEw345fI4xPs0cXg1KcQDoxWscFyX2nhB/Q6cpHurbA== dependencies: "@opentelemetry/core" "^1.8.0" - "@opentelemetry/instrumentation" "^0.39.1" + "@opentelemetry/instrumentation" "^0.41.0" "@opentelemetry/semantic-conventions" "^1.0.0" + tslib "^2.3.1" -"@opentelemetry/instrumentation-http@^0.39.1": - version "0.39.1" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-http/-/instrumentation-http-0.39.1.tgz#1bc63d4773fe7996a933a1351877e9a6ea73859a" - integrity sha512-JX1HTvNOqqel2fuMSRiSzFREyk2iMQ2B4/1Y46AGa0u6i4XQRCbCuy64FZ1YYMrQ2e5P917iiGrEUFkB+33Tlw== +"@opentelemetry/instrumentation-http@^0.41.1": + version "0.41.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-http/-/instrumentation-http-0.41.1.tgz#94bc5df488db9f46d65ce22d83bf7dbe256205bd" + integrity sha512-fhLBlSxTg+jw5HZVzOvH4tIUQHJkP8L2dyYYXu60sppYZHFVltL/DyfoMErdq5cSn97WHWfRqnbYrG0wlPJedA== dependencies: - "@opentelemetry/core" "1.13.0" - "@opentelemetry/instrumentation" "0.39.1" - "@opentelemetry/semantic-conventions" "1.13.0" - semver "^7.3.5" + "@opentelemetry/core" "1.15.1" + "@opentelemetry/instrumentation" "0.41.1" + "@opentelemetry/semantic-conventions" "1.15.1" + semver "^7.5.1" -"@opentelemetry/instrumentation-ioredis@^0.34.2": - version "0.34.2" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-ioredis/-/instrumentation-ioredis-0.34.2.tgz#3810c34cd30bd39523fcaae6c6250048f92eb5ad" - integrity sha512-tlXYJzBUytjN3UbFFVxuCJkZc6y/OmeAuH4VKoCV1fwx8iveQar1I9+mzf6H2Ur8CnzoCv4cq7bEhZAJepLN8g== +"@opentelemetry/instrumentation-ioredis@^0.35.0": + version "0.35.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-ioredis/-/instrumentation-ioredis-0.35.0.tgz#aa6fb1ff9965980e407950bc44f6e5127eb877df" + integrity sha512-tdM1BkrYmx+fXH+t1DViTXqFr9LUJHl0Qdcr6G8PjscsK+bVssSHhi5a3zYPFFFHpjks1mXMZHMr/Vsj2hNQAw== dependencies: - "@opentelemetry/instrumentation" "^0.39.1" - "@opentelemetry/redis-common" "^0.35.1" + "@opentelemetry/instrumentation" "^0.41.0" + "@opentelemetry/redis-common" "^0.36.0" "@opentelemetry/semantic-conventions" "^1.0.0" "@types/ioredis4" "npm:@types/ioredis@^4.28.10" + tslib "^2.3.1" -"@opentelemetry/instrumentation-pg@^0.35.2": - version "0.35.2" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-pg/-/instrumentation-pg-0.35.2.tgz#2eb56e24318aa67f8b8a3ac4d8314a1622385c82" - integrity sha512-DsRHUgacDZKc2obohpgCeVSyew3lWH7QHqk6awfz/e2/i+Zl6KvhcOUH3H3pFbcXScWliJlLlNa8XE6omFiI/Q== +"@opentelemetry/instrumentation-pg@^0.36.0": + version "0.36.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation-pg/-/instrumentation-pg-0.36.0.tgz#43270d2849a0e074ec6eb705dc12216e5c7cc84c" + integrity sha512-S9RmzTILWTl7AVfdp/e8lWQTlpwrpoPbpxAfEJ1ENzTlPMmdw0jWPAQTgrTLQa6cpzhiypDHts3g2b6hc1zFYQ== dependencies: "@opentelemetry/core" "^1.8.0" - "@opentelemetry/instrumentation" "^0.39.1" + "@opentelemetry/instrumentation" "^0.41.0" "@opentelemetry/semantic-conventions" "^1.0.0" + "@opentelemetry/sql-common" "^0.40.0" "@types/pg" "8.6.1" "@types/pg-pool" "2.0.3" + tslib "^2.3.1" -"@opentelemetry/instrumentation@0.39.1", "@opentelemetry/instrumentation@^0.39.1": - version "0.39.1" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation/-/instrumentation-0.39.1.tgz#46d03b4c7ce9f8d08f575d756acc801fa1283615" - integrity sha512-s7/9tPmM0l5KCd07VQizC4AO2/5UJdkXq5gMSHPdCeiMKSeBEdyDyQX7A+Cq+RYZM452qzFmrJ4ut628J5bnSg== +"@opentelemetry/instrumentation@0.41.1", "@opentelemetry/instrumentation@^0.41.0", "@opentelemetry/instrumentation@^0.41.1": + version "0.41.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation/-/instrumentation-0.41.1.tgz#20b8b1b15812a18acfa3b744abf90223b1d26eb0" + integrity sha512-IsOidIIgI7Sg2NhWGYRZRifiv9kLyrxT89hBK1YVPDetuBEBUgFzD5VXdwqwfOKL3kgT4KiERMmLJ8gqig0o1A== dependencies: - require-in-the-middle "^7.1.0" - semver "^7.3.2" + "@types/shimmer" "^1.0.2" + import-in-the-middle "1.4.1" + require-in-the-middle "^7.1.1" + semver "^7.5.1" shimmer "^1.2.1" -"@opentelemetry/instrumentation@^0.35.1": - version "0.35.1" - resolved "https://registry.yarnpkg.com/@opentelemetry/instrumentation/-/instrumentation-0.35.1.tgz#065bdbc4771137347e648eb4c6c6de6e9e97e4d1" - integrity sha512-EZsvXqxenbRTSNsft6LDcrT4pjAiyZOx3rkDNeqKpwZZe6GmZtsXaZZKuDkJtz9fTjOGjDHjZj9/h80Ya9iIJw== +"@opentelemetry/propagator-b3@1.15.1": + version "1.15.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/propagator-b3/-/propagator-b3-1.15.1.tgz#d3c625d18945c9fd501ed7e2a628f56d0a80c378" + integrity sha512-Rgzp5CgxSLDLdtiUx/nv+1jkyyU/qbhTqTBxMUvk4fqPfddzQNZyllyJ9IMNp9Xh4pzYlPP5ZBlN5Sw5isjuaw== dependencies: - require-in-the-middle "^5.0.3" - semver "^7.3.2" - shimmer "^1.2.1" + "@opentelemetry/core" "1.15.1" -"@opentelemetry/propagator-b3@1.13.0": - version "1.13.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/propagator-b3/-/propagator-b3-1.13.0.tgz#30a19a24e61ae8dbc26c2d7d7d3423d804d48f07" - integrity sha512-HOo91EI4UbuG8xQVLFziTzrcIn0MJQhy8m9jorh8aonb94jFVFi3CFNIiAnIGOabmnshJLOABxpYXsiPB8Xnzg== +"@opentelemetry/propagator-jaeger@1.15.1": + version "1.15.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/propagator-jaeger/-/propagator-jaeger-1.15.1.tgz#1af170b9cee5cba556ccb2e21d547260cb5c33ad" + integrity sha512-27cljZFnbUv5e459e2BhcsHCn2yePYq+07dZNW51e6F05GDWHC86fpwdh+WKvrfKSRMddUMkufHyoBWxtUN/Vg== dependencies: - "@opentelemetry/core" "1.13.0" + "@opentelemetry/core" "1.15.1" -"@opentelemetry/propagator-jaeger@1.13.0": - version "1.13.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/propagator-jaeger/-/propagator-jaeger-1.13.0.tgz#94a79d5301409d49b149227ee5568fcf6b21f9fe" - integrity sha512-IV9TO+u1Jzm9mUDAD3gyXf89eyvgEJUY1t+GB5QmS4wjVeWrSMUtD0JjH3yG9SNqkrQOqOGJq7YUSSetW+Lf5Q== +"@opentelemetry/redis-common@^0.36.0": + version "0.36.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/redis-common/-/redis-common-0.36.0.tgz#5b992d76838a385820f452235b7714e0f270e4ac" + integrity sha512-rTKuBszerwzKm0uBmffJ8j47+5YBGu6HGUWnez5gev2B4by8TKkY37E/QMq7/3KRL9NkZ08VJCtl3piCCFW30g== dependencies: - "@opentelemetry/core" "1.13.0" + tslib "^2.3.1" -"@opentelemetry/redis-common@^0.35.1": - version "0.35.1" - resolved "https://registry.yarnpkg.com/@opentelemetry/redis-common/-/redis-common-0.35.1.tgz#01356f6845d4f9f9fdfd2c4c562a74316d2d24d3" - integrity sha512-qLXe7h9VzFLx3LaizFiUlpuohCRyvHlDW5b9synE6omHKTZr/n0EHEdmhp3GezBeAqMGI+q499Mht4SmStaSqQ== - -"@opentelemetry/resources@1.13.0", "@opentelemetry/resources@^1.3.1": - version "1.13.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/resources/-/resources-1.13.0.tgz#436b33ea950004e66fce6575f2776a05faca7f8e" - integrity sha512-euqjOkiN6xhjE//0vQYGvbStxoD/WWQRhDiO0OTLlnLBO9Yw2Gd/VoSx2H+svsebjzYk5OxLuREBmcdw6rbUNg== +"@opentelemetry/resources@1.15.1", "@opentelemetry/resources@^1.15.1": + version "1.15.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/resources/-/resources-1.15.1.tgz#6a0da2eb5d394d302701d428a1cbbb2cd924ac50" + integrity sha512-15JcpyKZHhFYQ1uiC08vR02sRY/2seSnqSJ0tIUhcdYDzOhd0FrqPYpLj3WkLhVdQP6vgJ+pelAmSaOrCxCpKA== dependencies: - "@opentelemetry/core" "1.13.0" - "@opentelemetry/semantic-conventions" "1.13.0" + "@opentelemetry/core" "1.15.1" + "@opentelemetry/semantic-conventions" "1.15.1" -"@opentelemetry/sdk-metrics@1.13.0", "@opentelemetry/sdk-metrics@^1.8.0": - version "1.13.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-metrics/-/sdk-metrics-1.13.0.tgz#4e859107a7a4389bcda7b37d3952bc7dd34211d7" - integrity sha512-MOjZX6AnSOqLliCcZUrb+DQKjAWXBiGeICGbHAGe5w0BB18PJIeIo995lO5JSaFfHpmUMgJButTPfJJD27W3Vg== +"@opentelemetry/sdk-metrics@1.15.1", "@opentelemetry/sdk-metrics@^1.15.1": + version "1.15.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-metrics/-/sdk-metrics-1.15.1.tgz#e0d2844191ecd7fce3fccf18ae50ed35389f0885" + integrity sha512-ojcrzexOQfto83NvKfIvsJap4SHH3ZvLjsDGhQ04AfvWWGR7mPcqLSlLedoSkEdIe0k1H6uBEsHBtIprkMpTHA== dependencies: - "@opentelemetry/core" "1.13.0" - "@opentelemetry/resources" "1.13.0" - lodash.merge "4.6.2" + "@opentelemetry/core" "1.15.1" + "@opentelemetry/resources" "1.15.1" + lodash.merge "^4.6.2" -"@opentelemetry/sdk-trace-base@1.13.0", "@opentelemetry/sdk-trace-base@^1.3.1": - version "1.13.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.13.0.tgz#096cc2759430d880c5d886e009df2605097403dc" - integrity sha512-moTiQtc0uPR1hQLt6gLDJH9IIkeBhgRb71OKjNHZPE1VF45fHtD6nBDi5J/DkTHTwYP5X3kBJLa3xN7ub6J4eg== +"@opentelemetry/sdk-trace-base@1.15.1", "@opentelemetry/sdk-trace-base@^1.15.1": + version "1.15.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-trace-base/-/sdk-trace-base-1.15.1.tgz#8eabc0827769d91ac86cde8a86ebf0bf2a7d22ad" + integrity sha512-5hccBe2yXzzXyExJNkTsIzDe1AM7HK0al+y/D2yEpslJqS1HUzsUSuCMY7Z4+Sfz5Gf0kTa6KYEt1QUQppnoBA== dependencies: - "@opentelemetry/core" "1.13.0" - "@opentelemetry/resources" "1.13.0" - "@opentelemetry/semantic-conventions" "1.13.0" + "@opentelemetry/core" "1.15.1" + "@opentelemetry/resources" "1.15.1" + "@opentelemetry/semantic-conventions" "1.15.1" -"@opentelemetry/sdk-trace-node@^1.3.1": - version "1.13.0" - resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-trace-node/-/sdk-trace-node-1.13.0.tgz#baadf62218ca69e37486debfbcf15b2563f75979" - integrity sha512-FXA85lXKTsnbOflA/TBuBf2pmhD3c8uDjNjG0YqK+ap8UayfALmfJhf+aG1yBOUHevCY0JXJ4/xtbXExxpsMog== +"@opentelemetry/sdk-trace-node@^1.15.1": + version "1.15.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/sdk-trace-node/-/sdk-trace-node-1.15.1.tgz#d1589ead5fe8aa2dc6789ec31e16965b4dcaf259" + integrity sha512-aZDcuYHwh+qyOD/FLFAEAh32V2DlAp8Ubyaohh51oSssC3cxmN9JmpkyPbp2PQX3Mn48gBubwTXr9g++3+NB5w== dependencies: - "@opentelemetry/context-async-hooks" "1.13.0" - "@opentelemetry/core" "1.13.0" - "@opentelemetry/propagator-b3" "1.13.0" - "@opentelemetry/propagator-jaeger" "1.13.0" - "@opentelemetry/sdk-trace-base" "1.13.0" - semver "^7.3.5" + "@opentelemetry/context-async-hooks" "1.15.1" + "@opentelemetry/core" "1.15.1" + "@opentelemetry/propagator-b3" "1.15.1" + "@opentelemetry/propagator-jaeger" "1.15.1" + "@opentelemetry/sdk-trace-base" "1.15.1" + semver "^7.5.1" -"@opentelemetry/semantic-conventions@1.13.0", "@opentelemetry/semantic-conventions@^1.0.0", "@opentelemetry/semantic-conventions@^1.3.1", "@opentelemetry/semantic-conventions@^1.8.0": +"@opentelemetry/semantic-conventions@1.13.0", "@opentelemetry/semantic-conventions@^1.0.0": version "1.13.0" resolved "https://registry.yarnpkg.com/@opentelemetry/semantic-conventions/-/semantic-conventions-1.13.0.tgz#0290398b3eaebc6029c348988a78c3b688fe9219" integrity sha512-LMGqfSZkaMQXqewO0o1wvWr/2fQdCh4a3Sqlxka/UsJCe0cfLulh6x2aqnKLnsrSGiCq5rSCwvINd152i0nCqw== +"@opentelemetry/semantic-conventions@1.15.1", "@opentelemetry/semantic-conventions@^1.14.0", "@opentelemetry/semantic-conventions@^1.15.1": + version "1.15.1" + resolved "https://registry.yarnpkg.com/@opentelemetry/semantic-conventions/-/semantic-conventions-1.15.1.tgz#3d745996b2bd11095b515515fd3d68d46092a02d" + integrity sha512-n8Kur1/CZlYG32YCEj30CoUqA8R7UyDVZzoEU6SDP+13+kXDT2kFVu6MpcnEUTyGP3i058ID6Qjp5h6IJxdPPQ== + +"@opentelemetry/sql-common@^0.40.0": + version "0.40.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/sql-common/-/sql-common-0.40.0.tgz#8cbed0722354d62997c3b9e1adf0e16257be6b15" + integrity sha512-vSqRJYUPJVjMFQpYkQS3ruexCPSZJ8esne3LazLwtCPaPRvzZ7WG3tX44RouAn7w4wMp8orKguBqtt+ng2UTnw== + dependencies: + "@opentelemetry/core" "^1.1.0" + "@peertube/feed@^5.1.0": version "5.1.0" resolved "https://registry.yarnpkg.com/@peertube/feed/-/feed-5.1.0.tgz#e2fec950459ebaa32ea35791c45177f8b6fa85e9" @@ -2026,6 +2275,15 @@ smtp-server "^3.9.0" wildstring "1.0.9" +"@peertube/resolve-tspaths@^0.8.14": + version "0.8.14" + resolved "https://registry.yarnpkg.com/@peertube/resolve-tspaths/-/resolve-tspaths-0.8.14.tgz#a4acee0b6ff713fe38c92e6e7f4f4c0c3eabce3c" + integrity sha512-EnYh2GSY5qDPJ0TVt/UexuLQrB1vshOFXOM6OnqaV4e1WOYpxmtzu5amPFBRBLcGH9aJ4HM5eyDR3ka7Corjjw== + dependencies: + ansi-colors "4.1.3" + commander "11.0.0" + fast-glob "3.3.1" + "@selderee/plugin-htmlparser2@^0.10.0": version "0.10.0" resolved "https://registry.yarnpkg.com/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.10.0.tgz#8a304d18df907e086f3cfc71ea0ced52d6524430" @@ -2042,10 +2300,10 @@ domhandler "^5.0.3" selderee "^0.11.0" -"@sindresorhus/is@^4.0.0": - version "4.6.0" - resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-4.6.0.tgz#3c7c9c46e678feefe7a2e5bb609d3dbd665ffb3f" - integrity sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw== +"@sindresorhus/is@^5.2.0": + version "5.6.0" + resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-5.6.0.tgz#41dd6093d34652cddb5d5bdeee04eafc33826668" + integrity sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g== "@smithy/protocol-http@^1.0.1": version "1.0.1" @@ -2067,38 +2325,18 @@ resolved "https://registry.yarnpkg.com/@socket.io/component-emitter/-/component-emitter-3.1.0.tgz#96116f2a912e0c02817345b3c10751069920d553" integrity sha512-+9jVqKhRSpsc591z5vX+X5Yyw+he/HCB4iQ/RYxw35CEPaY1gnsNE43nf9n9AaYjAQrTiI/mOwKUKdUs9vf7Xg== -"@szmarczak/http-timer@^4.0.5": - version "4.0.6" - resolved "https://registry.yarnpkg.com/@szmarczak/http-timer/-/http-timer-4.0.6.tgz#b4a914bb62e7c272d4e5989fe4440f812ab1d807" - integrity sha512-4BAffykYOgO+5nzBWYwE3W90sBgLJoUPRWWcL8wlyiM8IB8ipJz3UMJ9KXQd1RKQXpKp8Tutn80HZtWsu2u76w== +"@szmarczak/http-timer@^5.0.1": + version "5.0.1" + resolved "https://registry.yarnpkg.com/@szmarczak/http-timer/-/http-timer-5.0.1.tgz#c7c1bf1141cdd4751b0399c8fc7b8b664cd5be3a" + integrity sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw== dependencies: - defer-to-connect "^2.0.0" + defer-to-connect "^2.0.1" "@tokenizer/token@^0.3.0": version "0.3.0" resolved "https://registry.yarnpkg.com/@tokenizer/token/-/token-0.3.0.tgz#fe98a93fe789247e998c75e74e9c7c63217aa276" integrity sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A== -"@tsconfig/node10@^1.0.7": - version "1.0.9" - resolved "https://registry.yarnpkg.com/@tsconfig/node10/-/node10-1.0.9.tgz#df4907fc07a886922637b15e02d4cebc4c0021b2" - integrity sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA== - -"@tsconfig/node12@^1.0.7": - version "1.0.11" - resolved "https://registry.yarnpkg.com/@tsconfig/node12/-/node12-1.0.11.tgz#ee3def1f27d9ed66dac6e46a295cffb0152e058d" - integrity sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag== - -"@tsconfig/node14@^1.0.0": - version "1.0.3" - resolved "https://registry.yarnpkg.com/@tsconfig/node14/-/node14-1.0.3.tgz#e4386316284f00b98435bf40f72f75a09dabf6c1" - integrity sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow== - -"@tsconfig/node16@^1.0.2": - version "1.0.4" - resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.4.tgz#0b92dcc0cc1c81f6f306a381f28e31b1a56536e9" - integrity sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA== - "@types/bcrypt@^5.0.0": version "5.0.0" resolved "https://registry.yarnpkg.com/@types/bcrypt/-/bcrypt-5.0.0.tgz#a835afa2882d165aff5690893db314eaa98b9f20" @@ -2138,16 +2376,6 @@ resolved "https://registry.yarnpkg.com/@types/bytes/-/bytes-3.1.1.tgz#67a876422e660dc4c10a27f3e5bcfbd5455f01d0" integrity sha512-lOGyCnw+2JVPKU3wIV0srU0NyALwTBJlVSx5DfMQOFuuohA8y9S8orImpuIQikZ0uIQ8gehrRjxgQC1rLRi11w== -"@types/cacheable-request@^6.0.1": - version "6.0.3" - resolved "https://registry.yarnpkg.com/@types/cacheable-request/-/cacheable-request-6.0.3.tgz#a430b3260466ca7b5ca5bfd735693b36e7a9d183" - integrity sha512-IQ3EbTzGxIigb1I3qPZc1rWJnH0BmSKv5QYTalEwweFvyBDLSAe24zP0le/hyi7ecGfZVlIVAg4BZqb8WBwKqw== - dependencies: - "@types/http-cache-semantics" "*" - "@types/keyv" "^3.1.4" - "@types/node" "*" - "@types/responselike" "^1.0.0" - "@types/caseless@*": version "0.12.2" resolved "https://registry.yarnpkg.com/@types/caseless/-/caseless-0.12.2.tgz#f65d3d6389e01eeb458bd54dc8f52b95a9463bc8" @@ -2270,7 +2498,7 @@ "@types/jsonfile" "*" "@types/node" "*" -"@types/http-cache-semantics@*": +"@types/http-cache-semantics@^4.0.1": version "4.0.1" resolved "https://registry.yarnpkg.com/@types/http-cache-semantics/-/http-cache-semantics-4.0.1.tgz#0ea7b61496902b95890dc4c3a116b60cb8dae812" integrity sha512-SZs7ekbP8CN0txVG2xVRH6EgKmEm31BOxA07vkFaETzZz1xh+cbt8BcI0slpymvwhx5dlFnQG2rTlPVQn+iRPQ== @@ -2299,17 +2527,22 @@ dependencies: "@types/node" "*" -"@types/keyv@^3.1.4": - version "3.1.4" - resolved "https://registry.yarnpkg.com/@types/keyv/-/keyv-3.1.4.tgz#3ccdb1c6751b0c7e52300bcdacd5bcbf8faa75b6" - integrity sha512-BQ5aZNSCpj7D6K2ksrRCTmKRLEpnPvWDiLPfoGyhZ++8YtiK9d/3DBKPJgry359X/P1PfruyYwvnvwFjuEiEIg== - dependencies: - "@types/node" "*" +"@types/jsonld@^1.5.9": + version "1.5.9" + resolved "https://registry.yarnpkg.com/@types/jsonld/-/jsonld-1.5.9.tgz#aca18d90b91488d15f8b4e941f660da4093be783" + integrity sha512-K76ImkErPYL2wGPZpNFSKp6wE+h/APecZLJrU7UfDaGqt/f+D9Rrg1aR7VdRrQ6k5DUNRZ2vn9yACwmpOr9QcA== -"@types/lodash@^4.14.64": - version "4.14.194" - resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.194.tgz#b71eb6f7a0ff11bff59fc987134a093029258a76" - integrity sha512-r22s9tAS7imvBt2lyHC9B8AGwWnXaYb1tY09oyLkXDs4vArpYJzw09nj8MLx5VfciBPGIb+ZwG0ssYnEPJxn/g== +"@types/lodash-es@^4.17.8": + version "4.17.8" + resolved "https://registry.yarnpkg.com/@types/lodash-es/-/lodash-es-4.17.8.tgz#cfffd0969507830c22da18dbb20d2ca126fdaa8b" + integrity sha512-euY3XQcZmIzSy7YH5+Unb3b2X12Wtk54YWINBvvGQ5SmMvwb11JQskGsfkH/5HXK77Kr8GF0wkVDIxzAisWtog== + dependencies: + "@types/lodash" "*" + +"@types/lodash@*": + version "4.14.196" + resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.196.tgz#a7c3d6fc52d8d71328b764e28e080b4169ec7a95" + integrity sha512-22y3o88f4a94mKljsZcanlNWPzO0uBsBdzLAngf2tp533LzZcQzb6+eZPJ+vCTt+bqF2XnvT9gejTLsAcJAJyQ== "@types/magnet-uri@*", "@types/magnet-uri@^5.1.1": version "5.1.3" @@ -2385,9 +2618,9 @@ integrity sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw== "@types/node@^18.13.0": - version "18.16.14" - resolved "https://registry.yarnpkg.com/@types/node/-/node-18.16.14.tgz#ab67bb907f1146afc6fedb9ce60ae8a99c989631" - integrity sha512-+ImzUB3mw2c5ISJUq0punjDilUQ5GnUim0ZRvchHIWJmOC0G+p0kzhXBqj6cDjK0QdPFwzrHWgrJp3RPvCG5qg== + version "18.17.1" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.17.1.tgz#84c32903bf3a09f7878c391d31ff08f6fe7d8335" + integrity sha512-xlR1jahfizdplZYRU59JlUx9uzF1ARa8jbhM11ccpCJya8kvos5jwdm2ZAgxSCwOl0fq21svP18EVwPBXMQudw== "@types/nodemailer@^6.2.0": version "6.4.8" @@ -2469,13 +2702,6 @@ "@types/tough-cookie" "*" form-data "^2.5.0" -"@types/responselike@^1.0.0": - version "1.0.0" - resolved "https://registry.yarnpkg.com/@types/responselike/-/responselike-1.0.0.tgz#251f4fe7d154d2bad125abe1b429b23afd262e29" - integrity sha512-85Y2BjiufFzaMIlvJDvTTB8Fxl2xfLo4HgmHzVBz08w4wDePCTjYw66PdrolO0kzli3yam/YCgRufyo1DdQVTA== - dependencies: - "@types/node" "*" - "@types/sax@^1.2.1": version "1.2.4" resolved "https://registry.yarnpkg.com/@types/sax/-/sax-1.2.4.tgz#8221affa7f4f3cb21abd22f244cfabfa63e6a69e" @@ -2504,6 +2730,11 @@ "@types/mime" "*" "@types/node" "*" +"@types/shimmer@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@types/shimmer/-/shimmer-1.0.2.tgz#93eb2c243c351f3f17d5c580c7467ae5d686b65f" + integrity sha512-dKkr1bTxbEsFlh2ARpKzcaAmsYixqt9UyCdoEZk8rHyE4iQYcDCyvSjDSf7JUWJHlJiTtbIoQjxKh6ViywqDAg== + "@types/simple-peer@*": version "9.11.5" resolved "https://registry.yarnpkg.com/@types/simple-peer/-/simple-peer-9.11.5.tgz#6baa00edbbd0f632f8561e8fb03b4d21d62f076e" @@ -2541,11 +2772,16 @@ resolved "https://registry.yarnpkg.com/@types/tv4/-/tv4-1.2.31.tgz#b33f3e6f082782a90f1fc5f414ad8722db8c9baa" integrity sha512-P97XU07fcpauSw3/fE2Q7eF6bHl4oHhwkikjnM7zlQLENrdC2rZuHSdNlMBhnW82NyBEsVJHII1Jk3d/MtQsQQ== -"@types/validator@^13.0.0", "@types/validator@^13.7.1": +"@types/validator@^13.7.1": version "13.7.17" resolved "https://registry.yarnpkg.com/@types/validator/-/validator-13.7.17.tgz#0a6d1510395065171e3378a4afc587a3aefa7cc1" integrity sha512-aqayTNmeWrZcvnG2MG9eGYI6b7S5fl+yKgPs6bAjOTwPS316R5SxBGKvtSExfyoJU7pIeHJfsHI0Ji41RVMkvQ== +"@types/validator@^13.9.0": + version "13.9.0" + resolved "https://registry.yarnpkg.com/@types/validator/-/validator-13.9.0.tgz#e7a96da3ea6a936222c6e76bb54abdd3dc4c9e4a" + integrity sha512-NclP0IbzHj/4tJZKFqKh8E7kZdgss+MCUYV9G+TLltFfDA4lFgE4PKPpDIyS2FlcdANIfSx273emkupvChigbw== + "@types/webtorrent@^0.109.0": version "0.109.3" resolved "https://registry.yarnpkg.com/@types/webtorrent/-/webtorrent-0.109.3.tgz#95df708d98bcea235b37f49a9a348b11f3511670" @@ -2742,26 +2978,31 @@ accepts@~1.3.4, accepts@~1.3.8: mime-types "~2.1.34" negotiator "0.6.3" +acorn-import-assertions@^1.9.0: + version "1.9.0" + resolved "https://registry.yarnpkg.com/acorn-import-assertions/-/acorn-import-assertions-1.9.0.tgz#507276249d684797c84e0734ef84860334cfb1ac" + integrity sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA== + acorn-jsx@^5.3.2: version "5.3.2" resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937" integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ== -acorn-walk@^8.1.1: - version "8.2.0" - resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.2.0.tgz#741210f2e2426454508853a2f44d0ab83b7f69c1" - integrity sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA== - acorn@^7.1.1: version "7.4.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== -acorn@^8.4.1, acorn@^8.8.0: +acorn@^8.8.0: version "8.8.2" resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.2.tgz#1b2f25db02af965399b9776b0c2c391276d37c4a" integrity sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw== +acorn@^8.8.2: + version "8.10.0" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.10.0.tgz#8be5b3907a67221a81ab23c7889c4c5526b62ec5" + integrity sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw== + addr-to-ip-port@^1.0.1, addr-to-ip-port@^1.5.4: version "1.5.4" resolved "https://registry.yarnpkg.com/addr-to-ip-port/-/addr-to-ip-port-1.5.4.tgz#9542b1c6219fdb8c9ce6cc72c14ee880ab7ddd88" @@ -2877,11 +3118,6 @@ are-we-there-yet@^2.0.0: delegates "^1.0.0" readable-stream "^3.6.0" -arg@^4.1.0: - version "4.1.3" - resolved "https://registry.yarnpkg.com/arg/-/arg-4.1.3.tgz#269fc7ad5b8e42cb63c896d5666017261c144089" - integrity sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA== - arg@^5.0.0: version "5.0.2" resolved "https://registry.yarnpkg.com/arg/-/arg-5.0.2.tgz#c81433cc427c92c4dcf4865142dbca6f15acd59c" @@ -3087,6 +3323,11 @@ base32.js@0.1.0: resolved "https://registry.yarnpkg.com/base32.js/-/base32.js-0.1.0.tgz#b582dec693c2f11e893cf064ee6ac5b6131a2202" integrity sha512-n3TkB02ixgBOhTvANakDb4xaMXnYUVkNoRFJjQflcqMQhyEKxEHdj3E6N8t8sUQ0mjH/3/JxzlXuz3ul/J90pQ== +base64-arraybuffer@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/base64-arraybuffer/-/base64-arraybuffer-1.0.2.tgz#1c37589a7c4b0746e34bd1feb951da2df01c1bdc" + integrity sha512-I3yl4r9QB5ZRY3XuJVEPfc2XhZO6YweFPI+UovAzn+8/hb3oJ6lnysaFcjVpkCPfVWFUDvoZ8kmVDP7WyRtYtQ== + base64-js@^1.0.2, base64-js@^1.2.0, base64-js@^1.3.1: version "1.5.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" @@ -3134,6 +3375,13 @@ bencode@^2.0.0, bencode@^2.0.1, bencode@^2.0.2, bencode@^2.0.3: resolved "https://registry.yarnpkg.com/bencode/-/bencode-2.0.3.tgz#89b9c80ea1b8573554915a7d0c15f62b0aa7fc52" integrity sha512-D/vrAD4dLVX23NalHwb8dSvsUsxeRPO8Y7ToKA015JQYq69MLDOMkC0uGZYA/MPpltLO8rt8eqFC2j8DxjTZ/w== +bencode@^3.1.1: + version "3.1.1" + resolved "https://registry.yarnpkg.com/bencode/-/bencode-3.1.1.tgz#359901f9a93724ee0c709f9f8265456b368c60e6" + integrity sha512-btsxX9201yoWh45TdqYg6+OZ5O1xTYKTYSGvJndICDFtznE/9zXgow8yjMvvhOqKKuzuL7h+iiCMpfkG8+QuBA== + dependencies: + uint8-util "^2.1.6" + bep53-range@^1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/bep53-range/-/bep53-range-1.1.1.tgz#20fd125b00a413254a77d42f63a43750ca7e64ac" @@ -3446,23 +3694,23 @@ cache-chunk-store@^3.2.2: lru "^3.1.0" queue-microtask "^1.2.3" -cacheable-lookup@^5.0.3: - version "5.0.4" - resolved "https://registry.yarnpkg.com/cacheable-lookup/-/cacheable-lookup-5.0.4.tgz#5a6b865b2c44357be3d5ebc2a467b032719a7005" - integrity sha512-2/kNscPhpcxrOigMZzbiWF7dz8ilhb/nIHU3EyZiXWXpeq/au8qJ8VhdftMkty3n7Gj6HIGalQG8oiBNB3AJgA== +cacheable-lookup@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/cacheable-lookup/-/cacheable-lookup-7.0.0.tgz#3476a8215d046e5a3202a9209dd13fec1f933a27" + integrity sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w== -cacheable-request@^7.0.2: - version "7.0.2" - resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-7.0.2.tgz#ea0d0b889364a25854757301ca12b2da77f91d27" - integrity sha512-pouW8/FmiPQbuGpkXQ9BAPv/Mo5xDGANgSNXzTzJ8DrKGuXOssM4wIQRjfanNRh3Yu5cfYPvcorqbhg2KIJtew== +cacheable-request@^10.2.8: + version "10.2.13" + resolved "https://registry.yarnpkg.com/cacheable-request/-/cacheable-request-10.2.13.tgz#b7012bb4a2acdb18cb54d2dff751d766b3500842" + integrity sha512-3SD4rrMu1msNGEtNSt8Od6enwdo//U9s4ykmXfA2TD58kcLkCobtCDiby7kNyj7a/Q7lz/mAesAFI54rTdnvBA== dependencies: - clone-response "^1.0.2" - get-stream "^5.1.0" - http-cache-semantics "^4.0.0" - keyv "^4.0.0" - lowercase-keys "^2.0.0" - normalize-url "^6.0.1" - responselike "^2.0.0" + "@types/http-cache-semantics" "^4.0.1" + get-stream "^6.0.1" + http-cache-semantics "^4.1.1" + keyv "^4.5.3" + mimic-response "^4.0.0" + normalize-url "^8.0.0" + responselike "^3.0.0" call-bind@^1.0.0, call-bind@^1.0.2: version "1.0.2" @@ -3664,6 +3912,11 @@ cidr-regex@^3.1.1: dependencies: ip-regex "^4.1.0" +cjs-module-lexer@^1.2.2: + version "1.2.3" + resolved "https://registry.yarnpkg.com/cjs-module-lexer/-/cjs-module-lexer-1.2.3.tgz#6c370ab19f8a3394e318fe682686ec0ac684d107" + integrity sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ== + cli-table3@^0.6.0: version "0.6.3" resolved "https://registry.yarnpkg.com/cli-table3/-/cli-table3-0.6.3.tgz#61ab765aac156b52f222954ffc607a6f01dbeeb2" @@ -3700,13 +3953,6 @@ cliui@^8.0.1: strip-ansi "^6.0.1" wrap-ansi "^7.0.0" -clone-response@^1.0.2: - version "1.0.3" - resolved "https://registry.yarnpkg.com/clone-response/-/clone-response-1.0.3.tgz#af2032aa47816399cf5f0a1d0db902f517abb8c3" - integrity sha512-ROoL94jJH2dUVML2Y/5PEDNaSHgeOdSDicUyS7izcF63G6sTc/FTjLub4b8Il9S8S0beOfYt0TaA5qvFK+w0wA== - dependencies: - mimic-response "^1.0.0" - clone@^2.0.0: version "2.1.2" resolved "https://registry.yarnpkg.com/clone/-/clone-2.1.2.tgz#1b7f4b9f591f1e8f83670401600345a02887435f" @@ -3795,10 +4041,10 @@ combined-stream@^1.0.6, combined-stream@^1.0.8: dependencies: delayed-stream "~1.0.0" -commander@10.0.0: - version "10.0.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-10.0.0.tgz#71797971162cd3cf65f0b9d24eb28f8d303acdf1" - integrity sha512-zS5PnTI22FIRM6ylNW8G4Ap0IEOyk62fhLSD0+uHRT9McRCLGpkVNvao4bjimpK/GShynyQkFFxHhwMcETmduA== +commander@11.0.0: + version "11.0.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-11.0.0.tgz#43e19c25dbedc8256203538e8d7e9346877a6f67" + integrity sha512-9HMlXtt/BNoYr8ooyjjNRdIilOTkVJXB+GhxMTtOKwk0R4j4lS4NpjuqmRxroBfnfTSHQIHQB7wryHhXarNjmQ== commander@^10.0.0: version "10.0.1" @@ -3968,11 +4214,6 @@ cpus@^1.0.3: resolved "https://registry.yarnpkg.com/cpus/-/cpus-1.0.3.tgz#4ef6deea461968d6329d07dd01205685df2934a2" integrity sha512-PXHBvGLuL69u55IkLa5e5838fLhIMHxmkV4ge42a8alGyn7BtawYgI0hQ849EedvtHIOLNNH3i6eQU1BiE9SUA== -create-require@^1.1.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" - integrity sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ== - create-torrent@^5, create-torrent@^5.0.9: version "5.0.9" resolved "https://registry.yarnpkg.com/create-torrent/-/create-torrent-5.0.9.tgz#850f198f7568e3d0e1e73b6858d43d44659a69d0" @@ -4140,7 +4381,7 @@ deepmerge@^4.2.2, deepmerge@^4.3.0, deepmerge@^4.3.1: resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.3.1.tgz#44b5f2147cd3b00d4b56137685966f26fd25dd4a" integrity sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A== -defer-to-connect@^2.0.0: +defer-to-connect@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/defer-to-connect/-/defer-to-connect-2.0.1.tgz#8016bdb4143e4632b77a3449c6236277de520587" integrity sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg== @@ -4245,11 +4486,6 @@ diff@5.0.0: resolved "https://registry.yarnpkg.com/diff/-/diff-5.0.0.tgz#7ed6ad76d859d030787ec35855f5b1daf31d852b" integrity sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w== -diff@^4.0.1: - version "4.0.2" - resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d" - integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A== - dir-glob@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f" @@ -4605,6 +4841,62 @@ es6-weak-map@^2.0.3: es6-iterator "^2.0.3" es6-symbol "^3.1.1" +esbuild@^0.19.0: + version "0.19.0" + resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.19.0.tgz#f187e4ce3bcc7396d13f408a991655efeba65282" + integrity sha512-i7i8TP4vuG55bKeLyqqk5sTPu1ZjPH3wkcLvAj/0X/222iWFo3AJUYRKjbOoY6BWFMH3teizxHEdV9Su5ESl0w== + optionalDependencies: + "@esbuild/android-arm" "0.19.0" + "@esbuild/android-arm64" "0.19.0" + "@esbuild/android-x64" "0.19.0" + "@esbuild/darwin-arm64" "0.19.0" + "@esbuild/darwin-x64" "0.19.0" + "@esbuild/freebsd-arm64" "0.19.0" + "@esbuild/freebsd-x64" "0.19.0" + "@esbuild/linux-arm" "0.19.0" + "@esbuild/linux-arm64" "0.19.0" + "@esbuild/linux-ia32" "0.19.0" + "@esbuild/linux-loong64" "0.19.0" + "@esbuild/linux-mips64el" "0.19.0" + "@esbuild/linux-ppc64" "0.19.0" + "@esbuild/linux-riscv64" "0.19.0" + "@esbuild/linux-s390x" "0.19.0" + "@esbuild/linux-x64" "0.19.0" + "@esbuild/netbsd-x64" "0.19.0" + "@esbuild/openbsd-x64" "0.19.0" + "@esbuild/sunos-x64" "0.19.0" + "@esbuild/win32-arm64" "0.19.0" + "@esbuild/win32-ia32" "0.19.0" + "@esbuild/win32-x64" "0.19.0" + +esbuild@~0.17.6: + version "0.17.19" + resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.17.19.tgz#087a727e98299f0462a3d0bcdd9cd7ff100bd955" + integrity sha512-XQ0jAPFkK/u3LcVRcvVHQcTIqD6E2H1fvZMA5dQPSOWb3suUbWbfbRf94pjc0bNzRYLfIrDRQXr7X+LHIm5oHw== + optionalDependencies: + "@esbuild/android-arm" "0.17.19" + "@esbuild/android-arm64" "0.17.19" + "@esbuild/android-x64" "0.17.19" + "@esbuild/darwin-arm64" "0.17.19" + "@esbuild/darwin-x64" "0.17.19" + "@esbuild/freebsd-arm64" "0.17.19" + "@esbuild/freebsd-x64" "0.17.19" + "@esbuild/linux-arm" "0.17.19" + "@esbuild/linux-arm64" "0.17.19" + "@esbuild/linux-ia32" "0.17.19" + "@esbuild/linux-loong64" "0.17.19" + "@esbuild/linux-mips64el" "0.17.19" + "@esbuild/linux-ppc64" "0.17.19" + "@esbuild/linux-riscv64" "0.17.19" + "@esbuild/linux-s390x" "0.17.19" + "@esbuild/linux-x64" "0.17.19" + "@esbuild/netbsd-x64" "0.17.19" + "@esbuild/openbsd-x64" "0.17.19" + "@esbuild/sunos-x64" "0.17.19" + "@esbuild/win32-arm64" "0.17.19" + "@esbuild/win32-ia32" "0.17.19" + "@esbuild/win32-x64" "0.17.19" + escalade@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40" @@ -4904,7 +5196,7 @@ eventemitter-asyncresource@^1.0.0: resolved "https://registry.yarnpkg.com/eventemitter-asyncresource/-/eventemitter-asyncresource-1.0.0.tgz#734ff2e44bf448e627f7748f905d6bdd57bdb65b" integrity sha512-39F7TBIV0G7gTelxwbEqnwhp90eqCPON1k0NwNfwhgKn4Co4ybUbj2pECcXT0B3ztRKZ7Pw1JujUUgmQJHcVAQ== -eventemitter3@^4.0.4: +eventemitter3@^4.0.7: version "4.0.7" resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f" integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw== @@ -5047,7 +5339,18 @@ fast-fifo@^1.1.0: resolved "https://registry.yarnpkg.com/fast-fifo/-/fast-fifo-1.2.0.tgz#2ee038da2468e8623066dee96958b0c1763aa55a" integrity sha512-NcvQXt7Cky1cNau15FWy64IjuO8X0JijhTBBrJj1YlxlDfRkJXNaK9RFUjwpfDPzMdv7wB38jr53l9tkNLxnWg== -fast-glob@3.2.12, fast-glob@^3.2.9: +fast-glob@3.3.1: + version "3.3.1" + resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.3.1.tgz#784b4e897340f3dbbef17413b3f11acf03c874c4" + integrity sha512-kNFPyjhh5cKjrUltxs+wFx+ZkbRaxxmZ+X0ZU31SOsxCEtP9VPgtq2teZw1DebupL5GmDaNQ6yKMMVcM41iqDg== + dependencies: + "@nodelib/fs.stat" "^2.0.2" + "@nodelib/fs.walk" "^1.2.3" + glob-parent "^5.1.2" + merge2 "^1.3.0" + micromatch "^4.0.4" + +fast-glob@^3.2.9: version "3.2.12" resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.2.12.tgz#7f39ec99c2e6ab030337142da9e0c18f37afae80" integrity sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w== @@ -5221,6 +5524,11 @@ for-each@^0.3.3: dependencies: is-callable "^1.1.3" +form-data-encoder@^2.1.2: + version "2.1.4" + resolved "https://registry.yarnpkg.com/form-data-encoder/-/form-data-encoder-2.1.4.tgz#261ea35d2a70d48d30ec7a9603130fa5515e9cd5" + integrity sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw== + form-data@^2.5.0: version "2.5.1" resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.5.1.tgz#f2cbec57b5e59e23716e128fe44d4e5dd23895f4" @@ -5395,14 +5703,7 @@ get-stream@^3.0.0: resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-3.0.0.tgz#8e943d1358dc37555054ecbe2edb05aa174ede14" integrity sha512-GlhdIUuVakc8SJ6kK0zAFbiGzRFzNnY4jUuEbV9UROo4Y+0Ny4fjvcZFVTeDA4odpFyOQzaw6hXukJSq/f28sQ== -get-stream@^5.1.0: - version "5.2.0" - resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-5.2.0.tgz#4966a1795ee5ace65e706c4b7beb71257d6e22d3" - integrity sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA== - dependencies: - pump "^3.0.0" - -get-stream@^6.0.0: +get-stream@^6.0.0, get-stream@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-6.0.1.tgz#a262d8eef67aced57c2852ad6167526a43cbf7b7" integrity sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg== @@ -5415,6 +5716,13 @@ get-symbol-description@^1.0.0: call-bind "^1.0.2" get-intrinsic "^1.1.1" +get-tsconfig@^4.4.0: + version "4.6.2" + resolved "https://registry.yarnpkg.com/get-tsconfig/-/get-tsconfig-4.6.2.tgz#831879a5e6c2aa24fe79b60340e2233a1e0f472e" + integrity sha512-E5XrT4CbbXcXWy+1jChlZmrmCwd5KGx502kDCXJJ7y898TtWW9FwoG5HfOLVRKmlmDGkWN2HM9Ho+/Y8F0sJDg== + dependencies: + resolve-pkg-maps "^1.0.0" + getpass@^0.1.1: version "0.1.7" resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" @@ -5525,22 +5833,22 @@ gopd@^1.0.1: dependencies: get-intrinsic "^1.1.3" -got@^11.8.2: - version "11.8.6" - resolved "https://registry.yarnpkg.com/got/-/got-11.8.6.tgz#276e827ead8772eddbcfc97170590b841823233a" - integrity sha512-6tfZ91bOr7bOXnK7PRDCGBLa1H4U080YHNaAQ2KsMGlLEzRbk44nsZF2E1IeRc3vtJHPVbKCYgdFbaGO2ljd8g== +got@^13.0.0: + version "13.0.0" + resolved "https://registry.yarnpkg.com/got/-/got-13.0.0.tgz#a2402862cef27a5d0d1b07c0fb25d12b58175422" + integrity sha512-XfBk1CxOOScDcMr9O1yKkNaQyy865NbYs+F7dr4H0LZMVgCj2Le59k6PqbNHoL5ToeaEQUYh6c6yMfVcc6SJxA== dependencies: - "@sindresorhus/is" "^4.0.0" - "@szmarczak/http-timer" "^4.0.5" - "@types/cacheable-request" "^6.0.1" - "@types/responselike" "^1.0.0" - cacheable-lookup "^5.0.3" - cacheable-request "^7.0.2" + "@sindresorhus/is" "^5.2.0" + "@szmarczak/http-timer" "^5.0.1" + cacheable-lookup "^7.0.0" + cacheable-request "^10.2.8" decompress-response "^6.0.0" - http2-wrapper "^1.0.0-beta.5.2" - lowercase-keys "^2.0.0" - p-cancelable "^2.0.0" - responselike "^2.0.0" + form-data-encoder "^2.1.2" + get-stream "^6.0.1" + http2-wrapper "^2.1.10" + lowercase-keys "^3.0.0" + p-cancelable "^3.0.0" + responselike "^3.0.0" graceful-fs@^4.1.6, graceful-fs@^4.2.0: version "4.2.11" @@ -5708,7 +6016,7 @@ htmlparser2@^8.0.0, htmlparser2@^8.0.1, htmlparser2@^8.0.2: domutils "^3.0.1" entities "^4.4.0" -http-cache-semantics@^4.0.0: +http-cache-semantics@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz#abe02fcb2985460bf0323be664436ec3476a6d5a" integrity sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ== @@ -5750,13 +6058,13 @@ http-problem-details@^0.1.5: resolved "https://registry.yarnpkg.com/http-problem-details/-/http-problem-details-0.1.5.tgz#f8f94f4ab9d4050749e9f8566fb85bb8caa2be56" integrity sha512-GHxfQZ0POP4FWbAM0guOyZyJNWwbLUXp+4XOJdmitS2tp3gHVSatrSX59Yyq/dCkhk4KiGtTWIlXZC83yCkBkA== -http2-wrapper@^1.0.0-beta.5.2: - version "1.0.3" - resolved "https://registry.yarnpkg.com/http2-wrapper/-/http2-wrapper-1.0.3.tgz#b8f55e0c1f25d4ebd08b3b0c2c079f9590800b3d" - integrity sha512-V+23sDMr12Wnz7iTcDeJr3O6AIxlnvT/bmaAAAP/Xda35C90p9599p0F1eHR/N1KILWSoWVAiOMFjBBXaXSMxg== +http2-wrapper@^2.1.10: + version "2.2.0" + resolved "https://registry.yarnpkg.com/http2-wrapper/-/http2-wrapper-2.2.0.tgz#b80ad199d216b7d3680195077bd7b9060fa9d7f3" + integrity sha512-kZB0wxMo0sh1PehyjJUWRFEd99KC5TLjZ2cULC4f9iqJBAmKQQXEICjxl5iPJRwP40dpeHFqqhm7tYCvODpqpQ== dependencies: quick-lru "^5.1.1" - resolve-alpn "^1.0.0" + resolve-alpn "^1.2.0" https-proxy-agent@^5.0.0: version "5.0.1" @@ -5849,6 +6157,16 @@ import-fresh@^3.0.0, import-fresh@^3.2.1: parent-module "^1.0.0" resolve-from "^4.0.0" +import-in-the-middle@1.4.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/import-in-the-middle/-/import-in-the-middle-1.4.1.tgz#31b25123bc35d556986a172bb398a3e6c32af9be" + integrity sha512-hGG0PcCsykVo8MBVH8l0uEWLWW6DXMgJA9jvC0yps6M3uIJ8L/tagTCbyF8Ud5TtqJ8/jmZL1YkyySyeVkVQrA== + dependencies: + acorn "^8.8.2" + acorn-import-assertions "^1.9.0" + cjs-module-lexer "^1.2.2" + module-details-from-path "^1.0.3" + imurmurhash@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" @@ -6193,10 +6511,10 @@ isexe@^2.0.0: resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" integrity sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw== -iso-639-3@2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/iso-639-3/-/iso-639-3-2.2.0.tgz#eb01d7734d61396efec934979e8b0806550837f1" - integrity sha512-v9w/U4XDSfXCrXxf4E6ertGC/lTRX8MLLv7XC1j6N5oL3ympe38jp77zgeyMsn3MbufuAAoGeVzDJbOXnPTMhQ== +iso-639-3@3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/iso-639-3/-/iso-639-3-3.0.1.tgz#4be56987c46fbda79da63a3d90d6552d7429dcea" + integrity sha512-SdljCYXOexv/JmbQ0tvigHN43yECoscVpe2y2hlEqy/CStXQlroPhZLj7zKLRiGqLJfw8k7B973UAMDoQczVgQ== isomorphic-fetch@^3.0.0: version "3.0.0" @@ -6314,7 +6632,7 @@ json5@^1.0.2: dependencies: minimist "^1.2.0" -json5@^2.1.3, json5@^2.2.2, json5@^2.2.3: +json5@^2.1.3, json5@^2.2.3: version "2.2.3" resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283" integrity sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg== @@ -6408,10 +6726,10 @@ k-rpc@^5.1.0: k-rpc-socket "^1.7.2" randombytes "^2.0.5" -keyv@^4.0.0: - version "4.5.2" - resolved "https://registry.yarnpkg.com/keyv/-/keyv-4.5.2.tgz#0e310ce73bf7851ec702f2eaf46ec4e3805cce56" - integrity sha512-5MHbFaKn8cNSmVW7BYnijeAVlE4cYA/SVkifVgrh7yotnfhKmjuXpDKjrABLnT0SfHWV21P8ow07OGfRrNDg8g== +keyv@^4.5.3: + version "4.5.3" + resolved "https://registry.yarnpkg.com/keyv/-/keyv-4.5.3.tgz#00873d2b046df737963157bd04f294ca818c9c25" + integrity sha512-QCiSav9WaX1PgETJ+SpNnx2PRRapJ/oRSXM4VO5OGYGSjrxbKPVFVhB3l2OCbLCk329N8qyAtsJjSjvVBWzEug== dependencies: json-buffer "3.0.1" @@ -6542,6 +6860,11 @@ locate-path@^6.0.0: dependencies: p-locate "^5.0.0" +lodash-es@^4.17.21: + version "4.17.21" + resolved "https://registry.yarnpkg.com/lodash-es/-/lodash-es-4.17.21.tgz#43e626c46e6591b7750beb2b50117390c609e3ee" + integrity sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw== + lodash.chunk@^4.2.0: version "4.2.0" resolved "https://registry.yarnpkg.com/lodash.chunk/-/lodash.chunk-4.2.0.tgz#66e5ce1f76ed27b4303d8c6512e8d1216e8106bc" @@ -6567,7 +6890,7 @@ lodash.isarguments@^3.1.0: resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a" integrity sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg== -lodash.merge@4.6.2, lodash.merge@^4.6.2: +lodash.merge@^4.6.2: version "4.6.2" resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" integrity sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ== @@ -6616,10 +6939,10 @@ loupe@^2.3.1: dependencies: get-func-name "^2.0.0" -lowercase-keys@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-2.0.0.tgz#2603e78b7b4b0006cbca2fbcc8a3202558ac9479" - integrity sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA== +lowercase-keys@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/lowercase-keys/-/lowercase-keys-3.0.0.tgz#c5e7d442e37ead247ae9db117a9d0a467c89d4f2" + integrity sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ== lru-cache@4.1.x: version "4.1.5" @@ -6724,11 +7047,6 @@ make-dir@^3.1.0: dependencies: semver "^6.0.0" -make-error@^1.1.1: - version "1.3.6" - resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2" - integrity sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw== - make-plural@^7.0.0: version "7.3.0" resolved "https://registry.yarnpkg.com/make-plural/-/make-plural-7.3.0.tgz#2889dbafca2fb097037c47967d3e3afa7e48a52c" @@ -6878,16 +7196,16 @@ mimic-fn@^2.1.0: resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== -mimic-response@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b" - integrity sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ== - mimic-response@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9" integrity sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ== +mimic-response@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-4.0.0.tgz#35468b19e7c75d10f5165ea25e75a5ceea7cf70f" + integrity sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg== + min-document@^2.19.0: version "2.19.0" resolved "https://registry.yarnpkg.com/min-document/-/min-document-2.19.0.tgz#7bd282e3f5842ed295bb748cdd9f1ffa2c824685" @@ -7289,10 +7607,10 @@ normalize-path@^3.0.0, normalize-path@~3.0.0: resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65" integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA== -normalize-url@^6.0.1: - version "6.1.0" - resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-6.1.0.tgz#40d0885b535deffe3f3147bec877d05fe4c5668a" - integrity sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A== +normalize-url@^8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/normalize-url/-/normalize-url-8.0.0.tgz#593dbd284f743e8dcf6a5ddf8fadff149c82701a" + integrity sha512-uVFpKhj5MheNBJRTiMZ9pE/7hD1QTeEvugSJW/OmLzAp78PB5O6adfMNTvmfKhXBkvCzC+rqifWcVYpGFwTjnw== npm-run-path@^2.0.0: version "2.0.2" @@ -7417,14 +7735,14 @@ open@7: is-docker "^2.0.0" is-wsl "^2.1.1" -opentelemetry-instrumentation-sequelize@^0.35.0: - version "0.35.0" - resolved "https://registry.yarnpkg.com/opentelemetry-instrumentation-sequelize/-/opentelemetry-instrumentation-sequelize-0.35.0.tgz#ac583d0a2283e251f71c35b983ffeb06305f8b2b" - integrity sha512-bYcw98dFTbAGcGzIwNa+09mnW7tRgew0qR9L09oo3X3+L6NtA3C6GxGVGBa9ExxHJeokSl5F5ytSMTYVUG4iBA== +opentelemetry-instrumentation-sequelize@^0.39.1: + version "0.39.1" + resolved "https://registry.yarnpkg.com/opentelemetry-instrumentation-sequelize/-/opentelemetry-instrumentation-sequelize-0.39.1.tgz#1fb3e1fa517ef776e749b0c5c5b043c4132c1659" + integrity sha512-LMZ9UxMbD23JhAqrHdzwd6w0khEYNkBK7ic+aP2vTLI06RutfiOmk0OZkKvzCW5ZozAsO2vI4zihJCqJGN98gA== dependencies: - "@opentelemetry/core" "^1.8.0" - "@opentelemetry/instrumentation" "^0.35.1" - "@opentelemetry/semantic-conventions" "^1.8.0" + "@opentelemetry/core" "^1.13.0" + "@opentelemetry/instrumentation" "^0.41.0" + "@opentelemetry/semantic-conventions" "^1.14.0" opentracing@^0.14.4: version "0.14.7" @@ -7455,10 +7773,10 @@ otpauth@^9.0.2: dependencies: jssha "~3.3.0" -p-cancelable@^2.0.0: - version "2.1.1" - resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-2.1.1.tgz#aab7fbd416582fa32a3db49859c122487c5ed2cf" - integrity sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg== +p-cancelable@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/p-cancelable/-/p-cancelable-3.0.0.tgz#63826694b54d61ca1c20ebcb6d3ecf5e14cd8050" + integrity sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw== p-event@4.2.0: version "4.2.0" @@ -7500,21 +7818,26 @@ p-locate@^5.0.0: dependencies: p-limit "^3.0.2" -p-queue@^6: - version "6.6.2" - resolved "https://registry.yarnpkg.com/p-queue/-/p-queue-6.6.2.tgz#2068a9dcf8e67dd0ec3e7a2bcb76810faa85e426" - integrity sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ== +p-queue@^7.3.4: + version "7.3.4" + resolved "https://registry.yarnpkg.com/p-queue/-/p-queue-7.3.4.tgz#7ef7d89b6c1a0563596d98adbc9dc404e9ed4a84" + integrity sha512-esox8CWt0j9EZECFvkFl2WNPat8LN4t7WWeXq73D9ha0V96qPRufApZi4ZhPwXAln1uVVal429HVVKPa2X0yQg== dependencies: - eventemitter3 "^4.0.4" - p-timeout "^3.2.0" + eventemitter3 "^4.0.7" + p-timeout "^5.0.2" -p-timeout@^3.0.0, p-timeout@^3.1.0, p-timeout@^3.2.0: +p-timeout@^3.0.0, p-timeout@^3.1.0: version "3.2.0" resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-3.2.0.tgz#c7e17abc971d2a7962ef83626b35d635acf23dfe" integrity sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg== dependencies: p-finally "^1.0.0" +p-timeout@^5.0.2: + version "5.1.0" + resolved "https://registry.yarnpkg.com/p-timeout/-/p-timeout-5.1.0.tgz#b3c691cf4415138ce2d9cfe071dba11f0fee085b" + integrity sha512-auFDyzzzGZZZdHz3BtET9VEz0SE/uMEAx7uWfGPucfzEwwe/xH0iVeZibQmANYE/hp9T2+UUZT5m+BKyrDp3Ew== + p-try@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6" @@ -8383,19 +8706,10 @@ require-from-string@^2.0.2: resolved "https://registry.yarnpkg.com/require-from-string/-/require-from-string-2.0.2.tgz#89a7fdd938261267318eafe14f9c32e598c36909" integrity sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw== -require-in-the-middle@^5.0.3: - version "5.2.0" - resolved "https://registry.yarnpkg.com/require-in-the-middle/-/require-in-the-middle-5.2.0.tgz#4b71e3cc7f59977100af9beb76bf2d056a5a6de2" - integrity sha512-efCx3b+0Z69/LGJmm9Yvi4cqEdxnoGnxYxGxBghkkTTFeXRtTCmmhO0AnAfHz59k957uTSuy8WaHqOs8wbYUWg== - dependencies: - debug "^4.1.1" - module-details-from-path "^1.0.3" - resolve "^1.22.1" - -require-in-the-middle@^7.1.0: - version "7.1.0" - resolved "https://registry.yarnpkg.com/require-in-the-middle/-/require-in-the-middle-7.1.0.tgz#8ab4089383e7b7879ed134d8e9d1887bd48195ec" - integrity sha512-6f86Mh0vWCxqKKatRPwgY6VzYmcVay3WUTIpJ1ILBCNh+dTWabMR1swKGKz3XcEZ5mgjndzRu7fQ+44G2H9Gew== +require-in-the-middle@^7.1.1: + version "7.2.0" + resolved "https://registry.yarnpkg.com/require-in-the-middle/-/require-in-the-middle-7.2.0.tgz#b539de8f00955444dc8aed95e17c69b0a4f10fcf" + integrity sha512-3TLx5TGyAY6AOqLBoXmHkNql0HIf2RGbuMgCDT2WO/uGVAPJs6h7Kl+bN6TIZGd9bWhWPwnDnTHGtW8Iu77sdw== dependencies: debug "^4.1.1" module-details-from-path "^1.0.3" @@ -8411,7 +8725,7 @@ require-package-name@^2.0.1: resolved "https://registry.yarnpkg.com/require-package-name/-/require-package-name-2.0.1.tgz#c11e97276b65b8e2923f75dabf5fb2ef0c3841b9" integrity sha512-uuoJ1hU/k6M0779t3VMVIYpb2VMJk05cehCaABFhXaibcbvfgR8wKiozLjVFSzJPmQMRqIcO0HMyTFqfV09V6Q== -resolve-alpn@^1.0.0: +resolve-alpn@^1.2.0: version "1.2.1" resolved "https://registry.yarnpkg.com/resolve-alpn/-/resolve-alpn-1.2.1.tgz#b7adbdac3546aaaec20b45e7d8265927072726f9" integrity sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g== @@ -8421,14 +8735,10 @@ resolve-from@^4.0.0: resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6" integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g== -resolve-tspaths@^0.8.8: - version "0.8.13" - resolved "https://registry.yarnpkg.com/resolve-tspaths/-/resolve-tspaths-0.8.13.tgz#83e37d7fb40b0ed241c40e91dbcd15c0fb82224c" - integrity sha512-eHlHinC2qt3jQLFiZyUE4HXZOTlT1abHO2fb+OI9Ybsn8wdhKiAtIFVy1+QVTaIQNphCLvm42EkqJt/+ZAA8Sw== - dependencies: - ansi-colors "4.1.3" - commander "10.0.0" - fast-glob "3.2.12" +resolve-pkg-maps@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz#616b3dc2c57056b5588c31cdf4b3d64db133720f" + integrity sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw== resolve@^1.10.1, resolve@^1.15.1, resolve@^1.18.1, resolve@^1.22.1, resolve@^1.22.2: version "1.22.2" @@ -8439,12 +8749,12 @@ resolve@^1.10.1, resolve@^1.15.1, resolve@^1.18.1, resolve@^1.22.1, resolve@^1.2 path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" -responselike@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/responselike/-/responselike-2.0.1.tgz#9a0bc8fdc252f3fb1cca68b016591059ba1422bc" - integrity sha512-4gl03wn3hj1HP3yzgdI7d3lCkF95F21Pz4BPGvKHinyQzALR5CapwC8yIi0Rh58DEMQ/SguC03wFj2k0M/mHhw== +responselike@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/responselike/-/responselike-3.0.0.tgz#20decb6c298aff0dbee1c355ca95461d42823626" + integrity sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg== dependencies: - lowercase-keys "^2.0.0" + lowercase-keys "^3.0.0" ret@~0.1.10: version "0.1.15" @@ -8620,6 +8930,13 @@ semver@^7.0.0, semver@^7.3.2, semver@^7.3.5, semver@^7.3.7, semver@^7.3.8, semve dependencies: lru-cache "^6.0.0" +semver@^7.5.1: + version "7.5.4" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.5.4.tgz#483986ec4ed38e1c6c48c34894a9182dbff68a6e" + integrity sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA== + dependencies: + lru-cache "^6.0.0" + send@0.18.0: version "0.18.0" resolved "https://registry.yarnpkg.com/send/-/send-0.18.0.tgz#670167cc654b05f5aa4a767f9113bb371bc706be" @@ -8898,6 +9215,19 @@ socks@^2.0.0: resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.2.tgz#adbc361d9c62df380125e7f161f71c826f1e490c" integrity sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw== +source-map-support@^0.5.21: + version "0.5.21" + resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.21.tgz#04fe7c7f9e1ed2d662233c28cb2b35b9f63f6e4f" + integrity sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w== + dependencies: + buffer-from "^1.0.0" + source-map "^0.6.0" + +source-map@^0.6.0: + version "0.6.1" + resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" + integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== + spawn-command@0.0.2-1: version "0.0.2-1" resolved "https://registry.yarnpkg.com/spawn-command/-/spawn-command-0.0.2-1.tgz#62f5e9466981c1b796dc5929937e11c9c6921bd0" @@ -9460,25 +9790,6 @@ triple-beam@^1.3.0: resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.3.0.tgz#a595214c7298db8339eeeee083e4d10bd8cb8dd9" integrity sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw== -ts-node@^10.8.1: - version "10.9.1" - resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.9.1.tgz#e73de9102958af9e1f0b168a6ff320e25adcff4b" - integrity sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw== - dependencies: - "@cspotcode/source-map-support" "^0.8.0" - "@tsconfig/node10" "^1.0.7" - "@tsconfig/node12" "^1.0.7" - "@tsconfig/node14" "^1.0.0" - "@tsconfig/node16" "^1.0.2" - acorn "^8.4.1" - acorn-walk "^8.1.1" - arg "^4.1.0" - create-require "^1.1.0" - diff "^4.0.1" - make-error "^1.1.1" - v8-compile-cache-lib "^3.0.1" - yn "3.1.1" - tsc-watch@^6.0.0: version "6.0.4" resolved "https://registry.yarnpkg.com/tsc-watch/-/tsc-watch-6.0.4.tgz#af15229f03cd53086771a97b10653db063bc6c59" @@ -9499,15 +9810,6 @@ tsconfig-paths@^3.14.1: minimist "^1.2.6" strip-bom "^3.0.0" -tsconfig-paths@^4.0.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-4.2.0.tgz#ef78e19039133446d244beac0fd6a1632e2d107c" - integrity sha512-NoZ4roiN7LnbKn9QqE1amc9DJfzvZXxF4xDavcOWt1BPkdx+m+0gJuPM+S0vCe7zTJMYUP0R8pO2XMr+Y8oLIg== - dependencies: - json5 "^2.2.2" - minimist "^1.2.6" - strip-bom "^3.0.0" - tslib@^1.11.1, tslib@^1.8.1: version "1.14.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" @@ -9525,6 +9827,17 @@ tsutils@^3.21.0: dependencies: tslib "^1.8.1" +tsx@^3.12.7: + version "3.12.7" + resolved "https://registry.yarnpkg.com/tsx/-/tsx-3.12.7.tgz#b3b8b0fc79afc8260d1e14f9e995616c859a91e9" + integrity sha512-C2Ip+jPmqKd1GWVQDvz/Eyc6QJbGfE7NrR3fx5BpEHMZsEHoIxHL1j+lKdGobr8ovEyqeNkPLSKp6SCSOt7gmw== + dependencies: + "@esbuild-kit/cjs-loader" "^2.4.2" + "@esbuild-kit/core-utils" "^3.0.0" + "@esbuild-kit/esm-loader" "^2.5.5" + optionalDependencies: + fsevents "~2.3.2" + tv4@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/tv4/-/tv4-1.3.0.tgz#d020c846fadd50c855abb25ebaecc68fc10f7963" @@ -9608,6 +9921,13 @@ uint64be@^2.0.2: dependencies: buffer-alloc "^1.1.0" +uint8-util@^2.1.6: + version "2.2.2" + resolved "https://registry.yarnpkg.com/uint8-util/-/uint8-util-2.2.2.tgz#d1830e02957c7b5e1913c519174ff2c63fb7e2a3" + integrity sha512-zqDacLmV6UPJguIUKezcW8V9NzWJQxF6KX0hHiJWq2YbgHcTS8RnsfcganIRI51Pla59OIq6MjjjEMDEBnEW0A== + dependencies: + base64-arraybuffer "^1.0.2" + unbox-primitive@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.0.2.tgz#29032021057d5e6cdbd08c5129c226dff8ed6f9e" @@ -9747,11 +10067,6 @@ uuid@^9.0.0: resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.0.tgz#592f550650024a38ceb0c562f2f6aa435761efb5" integrity sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg== -v8-compile-cache-lib@^3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf" - integrity sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg== - valid-data-url@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/valid-data-url/-/valid-data-url-3.0.1.tgz#826c1744e71b5632e847dd15dbd45b9fb38aa34f" @@ -10208,11 +10523,6 @@ yargs@^17.7.1: y18n "^5.0.5" yargs-parser "^21.1.1" -yn@3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50" - integrity sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q== - yocto-queue@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b"